running async code in parallel

Question

Using the aync library, is it possible to create truly asynchronous tasks without using pre-built asynchronous i/o functions?

For example, if you run this code, it will always run the functions sequentially, since node.js is "single-threaded".

var async = require('async');

async.parallel([

    function(){
        console.log('1')
    },

    function(){
        console.log('2')
    },

    function(){
        console.log('3')
    },

    function(){
        console.log('4')
    }

]);

We aren't supposed to block Node - yes - but node.js runs on a single-core. If we wanted to utilize other cores, we could spawn processes and use the async library to good effect in that way.

Is there a best practice way to spawn async processes on other cores, besides the cluster module?

Answer 1

You can use something like - https://github.com/rvagg/node-worker-farm to run your tasks inside child processes.

Answer 2

Ok, I figured out a good solution. For the simplest possible situation, we use the core module 'child_process'. You don't even have to run npm install to use this module, it is a core module.

To solve this one, I used information from here:

http://blog.carbonfive.com/2014/02/28/taking-advantage-of-multi-processor-environments-in-node-js/

https://nodejs.org/api/child_process.html

//main.js

var cp = require('child_process');
var async = require('async');

async.parallel([

    function one(callback){
        console.log('1');
        var k = cp.fork('./doOther.js',['01']);
        k.once('message',function(msg){
            console.log(msg);
            callback();
        });
    },
    function two(callback){
        console.log('2');
        var k = cp.fork('./doOther.js',['02']);
        k.once('message',function(msg){
            console.log(msg);
            callback();
        });
    },
    function three(callback){
        console.log('3');
        var k = cp.fork('./doOther.js',['03']);
        k.once('message',function(msg){
            console.log(msg);
            callback();
        });

    },
    function four(callback){
        console.log('4');
        var k = cp.fork('./doOther.js',['04']);
        k.once('message',function(msg){
            console.log(msg);
            callback();
        });

    }

],function(){

    console.log('done.');
    process.exit(0);

});

a separate file in the same directory:

//doOther.js

process.on('message', function(msg) {
    console.log('child process got message:', msg);
});

setTimeout(function(){
    process.send({ foo: process.argv[2] });
},1000);

put the two files in the same directory, and away you go. Feel free to ask any questions. I will respond.

The above could be simplified to something like this:

  async.parallel([1,2,3,4].map(function(val){

       return function(cb){
            console.log(val);
            var k = cp.fork('./other-script.js',[val]);
            k.once('message',function(msg){
                console.log(msg);
                cb(null, msg);
            });
        },

     }),
     function(err, results){

        console.log('done.');
        process.exit(0);

    });