async function inside a for loop

Question

Hi i´m trying to convert sqlite database to NeDb, with this code:

const sqliteJSON = require('sqlite-json');
const Datastore = require('nedb')

const exporter = sqliteJSON('etecsa.db');

db = new Datastore('etecsa.nedb');
db.loadDatabase();

tables = ['fix','movil'];

tables.forEach(function(table) {
    sql = 'select count(1) from ' + table;

    exporter.json(sql, function (err, json) {
        toNeDB(table, JSON.parse(json)[0]['count(1)'])
    });

}, this);

var toNeDB = function(table, count) { 
    var inc = 10000;
    console.log(table + ' => ' + count)

    for (var i = 0; i < count + inc; i += inc) {
        var sql = 'SELECT * FROM ' + table + ' ORDER BY province ASC, number DESC LIMIT '+ i + ' , ' + inc;
        console.log(i)
        exporter.json(sql, function(err, json) {
            var data = JSON.parse(json);
            db.insert(data, function (err, newDoc) {});
        });
    }
}

the problem is that the for loop its not working as I desire. I need to use it to change the sql pagination because the sqlite database is very huge and I can´t pass all the data on a single query.

UPDATE using async.map

const sqliteJSON = require('sqlite-json');
const Datastore = require('nedb')
var range = require("range");
var async = require("async");

const exporter = sqliteJSON('etecsa.db');

db = new Datastore('etecsa.nedb');
db.loadDatabase();

tables = ['fix','movil'];

tables.forEach(function(table) {
    sql = 'select count(1) from ' + table;

    exporter.json(sql, function (err, json) {
        toNeDB(table, JSON.parse(json)[0]['count(1)'])
    });

}, this);


var toNeDB = function(table, count, cb) { 
    var inc = 10000;
    var pagination = range.range(1,count+inc,inc)

    async.map(pagination, function (page, cb){
        var sql = 'SELECT * FROM ' + table + ' ORDER BY province ASC, number DESC LIMIT '+ page + ' , ' + inc;
        console.log(page, table, inc);
        exporter.json(sql, function(err, json) {
            var data = JSON.parse(json);
            console.log(data[0])
            db.insert(data, function (err, newDoc) {});
        });
    }.bind({ table: table, inc: inc }), function(err,results){

    })

}

and the output:

1 'fix' 10000
10001 'fix' 10000
....
1150001 'fix' 10000

1 'movil' 10000
10001 'movil' 10000
...
3730001 'movil' 10000


{ number: '8775031',
  name: 'UNION ELECTRICA',
  address: 'S ALLENDE #666 OQUENDO SOLEDAD',
  province: 7 }
{ number: '8734454',
  name: 'EMP ESTB ESP Y SERVICIOS',
  address: 'ESAPDA #256 CONCORDIA S LAZARO',
  province: 7 }

@will, but to use map, i need to have a list of pagination to iterate, so I dont know it its a good solution — efirvida
– efirvida, Commented Aug 29, 2017 at 17:40
@FelixMosheev. for example the actual code print first all the console.log(i) iterations and then the script stay quiet, so y use the console.log, to monitor the process but its not working as i need it, — efirvida
– efirvida, Commented Aug 29, 2017 at 18:16
Put the console log inside the callback, but because of the fact it is a closure, you should wrap it with iffi. — felixmosh
– felixmosh, Commented Aug 29, 2017 at 18:22

felixmosh · Accepted Answer · 2017-08-29 18:36:37Z

If you need to know when each action occurred, you should put the console.log inside the callback.

Something like that:

var toNeDB = function(table, count) { 
var inc = 10000;
    console.log(table + ' => ' + count)

    for (var i = 0; i < count + inc; i += inc) {
        var sql = 'SELECT * FROM ' + table + ' ORDER BY province ASC, number DESC LIMIT '+ i + ' , ' + inc;

        exporter.json(sql, (function(i) {
           return function(err, json) {
              console.log(i)
              var data = JSON.parse(json);
              db.insert(data, function (err, newDoc) {});
           }
        })(i));
    }
}

madllamas · Accepted Answer · 2017-08-31 13:34:24Z

You could use recursion instead of a loop, that way you would be sure the next iteration won't execute until the first is done.

var proc = function (i, count, table) {
  var sql = 'SELECT * FROM ' + table + ' ORDER BY province ASC, number DESC 
             LIMIT ' + i + ' , ' + inc'
  console.log(i)
  exporter.json(sql, function (err, json) {
    var data = JSON.parse(json)
    db.insert(data, function (err, newDoc) {
      if (i < count) {
        i += inc
        proc(i, count, table)
      }
    })
  })
}
var toNeDB = function (table, count) {
  var inc = 10000
  console.log(table + ' => ' + count)
  proc(0, count, table)
}

let me know if that works

Collectives™ on Stack Overflow

async function inside a for loop

2 Answers 2

Comments

Comments

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

Comments

Comments

Related