How to handle multiple arrays in asynchronous methods in node.js

Question

I have a very large array (10K) and i want to split it (i did according to this: https://stackoverflow.com/a/8495740/2183053 and it worked) but i need to pass the tempArray to request and wait for response which will be passed to savetodb.

Can someone help me out please. To be clear, I want to split the large Array and pass the separated arrays to request function then pass to save to db and continue this process until all the arrays are cleared.

the following is the code i did:

//i used waterfall because it worked in waiting to finish the first task before starting another 
async.waterfall([
  async.apply(handleFile, './jsonFile.json')
  runRequest1,
  savetoDb
], function(err) {
  console.log('waterfall1 complete')
})

function handleFile(data, callback) {
  var name
  var authorNames = []
  require('fs').readFile(data, 'utf8', function(err, data) {
    if (err) throw err
    var content = _.get(JSON.parse(data), [2, 'data'])
    for (var x = 0; x < content.length; x++) {
      authorNames.push(JSON.stringify(content[x]['author']))
    }
    //the large array is authorNames and im splitting it below:
    for (i = 0, j = authorNames.length; i < j; i += chunk) {
      temparray = authorNames.slice(i, i + chunk);
      setTimeout(function() {
        callback(null, temparray)
      }, 2000);
    }
  })
}

Answer 1

You need promise to handle async things in nodejs

let findAllReceipts = function (accountArray) {
    const a = [];

    for (let i = 0; i < accountArray.length; i++) {
        a.push(new Promise((resolve, reject) => {
            receiptTable.find({accountNo: accountArray[i].accountNo}, function (err, data) {
                if (!err) {
                    resolve(data);
                } else {
                    reject(new Error('findPrice ERROR : ' + err));
                }
            });
        }));
    }

    return Promise.all(a);
};

Answer 2

I added some promise.

const data = await handleFile('./jsonFile.json');
// save to db 


async function handleFile(filePath) {
    let arrayWillReturn = []; 

    var name
    var authorNames = []
    let data = await getFileData(filePath)
    var content = _.get(JSON.parse(data), [2, 'data'])
    for (var x = 0; x < content.length; x++) {
        authorNames.push(JSON.stringify(content[x]['author']))
    }
    //the large array is authorNames and im splitting it below:
    for (i = 0, j = authorNames.length; i < j; i += chunk) {
        arrayWillReturn.push(authorNames.slice(i, i + chunk));
    }
    return arrayWillReturn;
}

async function getFileData(fileName) {
    return new Promise(function (resolve, reject) {
        fs.readFile(fileName, type, (err, data) => {
            err ? reject(err) : resolve(data);
        });
    });
}

Answer 3

I'm answering my own question for future references. the only thing that I added to make my code work is Promise. It sounds easy but it took me a while to grasp the function and implementing it but it worked and it was worth it. Thank you so much for the responses.