Node.js For Loop async request

Question

I have for loop with a request (and another request in callback).

I have problem with memory usage (when i am go through a lot of request at same time)

here is the sample code :

var request = require('request');

for(var j=1;j<=10;j++){

            request({
              method: 'GET',
              url: 'https://api.domain.com/items/?page='+j+'&limit=1000',
              headers: {
                'Content-Type': 'application/json'
              }}, function (error, response, body) {

              var data = JSON.parse(body)


              for(var i=0;i<data.length;i++){

                    request({
                      method: 'GET',
                      url: 'https://api.domain.com/itemDetail/'+data[i].id,
                      headers: {
                        'Content-Type': 'application/json',
                      }}, function (error, response, body) {


                        var itemDetails = JSON.parse(body);

                        // save items to mongodb          

                        }

              }
    });


}

The solution suggested to me it was using async module.

After reading documentation i find out eachlimit suit my needs.

The problem i have its i can use eachlimit for second loop but for first loop i dont know how use it (because first loop in not an array).

here code so far i get , and its not working:

var request = require('request');
var mongodb = require('mongodb');
var async = require('async');

var MongoClient = mongodb.MongoClient;

var url = 'mongodb://localhost:27017/subtitle';

for(var j=1;j<=10;j++){

        request({
              method: 'GET',
              url: 'https://api-v2.domain.com/news/popular?page='+j+'&limit=1000',
              headers: {
                'Content-Type': 'application/json',
                'api-version': '2',
                'api-key': 'my-api-key'
              }}, function (error, response, body) {

                    try {
                        var data = JSON.parse(body);
                        } catch (e) {
                          // An error has occured, handle it, by e.g. logging it
                              console.log(e);
                        }

                            async.each(data, function(item, callback) {

                                    request({
                                      method: 'GET',
                                      url: 'https://api-v2.domain.com/news/'+item.id,
                                      headers: {
                                            'Content-Type': 'application/json',
                                            'api-version': '2',
                                            'api-key': 'my-api-key'
                                          }}, function (error, response, body) {
                                            // sava item in mongodb
                                      })



                                callback();

                            }, function(err){

                                if( err ) {
                                  console.log('A item failed to process');
                                } else {
                                  console.log('All items have been saved successfully');
                                }
                            });


              })


}

cviejo · Accepted Answer · 2015-08-31 12:31:05Z

You can achieve this in a number of ways, one of them would be using whilst. With your code it'd look something like this (removed error handling and request options for readability here):

var requestCount = 0;

async.whilst(
    function () { 
        return requestCount < 10; 
    },
    function (firstCallback) {

        request({url: "https://api-v2.domain.com/news/popular?page=" + requestCount + "&limit=1000"}, function (err1, res1, body1) {

            var data = JSON.parse(body1);

            async.each(
                data, 
                function(item, secondCallback) {

                    request({url: "https://api-v2.domain.com/news/" + item.id}, function (err2, res2, body2) {
                        // sava item in mongodb
                        secondCallback();
                    }); 
                },
                function(err){
                    requestCount++;
                    firstCallback();
                }
            );
        });
    },
    function (err) {
        //all requests done
    }
);

Once you start to nest so many callbacks, you should probably refactor this and put the second loop in it's own function. For example:

var requestCount = 0;

async.whilst(
    function () { 
        return requestCount < 10; 
    },
    function (callback) {

        request({url: "https://api-v2.domain.com/news/popular?page=" + requestCount + "&limit=1000"}, function (err, res, body) {

            var items = JSON.parse(data);

            getNews(items, function(newsErr){
                requestCount++;
                callback(newsErr);
            });
        });
    },
    function (err) {
        //all requests done
    }
);


//----------------------------------------------------
function getNews(items, callback){

    async.each(items, function(item, itemCallback) {

        request({url: "https://api-v2.domain.com/news/" + item.id}, function (err2, res2, body2) {
            // sava item in mongodb
            itemCallback();
        }); 
    }, callback);
}

Also make sure to call the async callbacks within the callbacks of the request (you weren't doing it so in your code).

we dont need add requestCount++ ? and another question whilst run in parallel ?

Collectives™ on Stack Overflow

Node.js For Loop async request

1 Answer 1

1 Comment

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

1 Comment

Related