setTimeout inside iteration of iteration

Question

I have the code below and i would like to put a setTimeout between each iteration of Myurl. There are a number of classes and each of them contains a number of elements.

//Some calculations before...
var i = 0;
async.whilst(
function () {
    return i <= thefooz.length - 1;
},
function (innerCallback) {

    //Some calculations where I get classes array.

    async.forEachOfSeries(classes, function (Myurl, m, eachDone) {
        // Here I want a delay
        async.waterfall([
            function (next) {
                connection.query(
                    'SELECT * FROM mydata WHERE UrlLink=? LIMIT 1', [Myurl],
                    next
                );
            },
            function (results, fields, next) {
                if (results.length !== 0) {
                    console.log("Already Present");
                    return next();
                }
                console.log("New Thing!");
                request(options2, function (err, resp, body) {
                    if (!err && resp.statusCode == 200) {
                        var $ = cheerio.load(body);
                        //Some calculations, where I get AllLinks.
                        var post = {
                            ThisUrl: AllLinks[0],
                            Time: AllLinks[1],
                        };
                        var query = connection.query('Insert INTO mydata Set ?', post, next);
                    };
                });
            }
        ], eachDone);

    }, function (err) {
        if (err) throw err;
    });
    setTimeout(function () {
        i++;
        innerCallback();
        console.log("Done");
    }, 20000);

    //Some calculations after...

So how could I set a delay between each Myurl in async.waterfall? Say I want a 5 seconds delay. I managed to set setTimeout between each async.whilstiteration but not between each async.forEachOfSeries iteration. It simply does not wait, instead it continues to loop until each async.forEachOfSeries is done and then calls the async.whilst setTimeout.

EDIT: Queue solution does not work. That solution seems to just go to next page, and next page and so on, without outputting to my database. Of course I could apply it in the wrong way, but I really tried to do exactly as the example said.

Answer 1

I think you don't fully understand how setTimeout works:

(function () {
    var seconds=0;
[1,2,3].forEach(function(value) {
    setTimeout(function() {
        console.log('Showing value '+value+ 'at '+Date());
    },1000*seconds++);
})
})()

This code, for each element, creates a callback function to be executed after a second. Please note that JS is single threaded, so what the code really does is adding "executions" to a queue. So if the current execution does not stop, the callbacks are not called. So the time (in millis) that you pass to the setTimeout function as second parameter is just the minimum time to have that code executed.

Then, the execution of those callbacks is made in FIFO order.

UPDATE: Here is an example of what I'm explaining:

function myFunction() { 
    var test=0;
    setTimeout(function(){
        console.log("This is the current value of test: "+test); 
    }, 0);
    console.log("This is run first");
    for (var i=0;i<50;i++) {
        test++;
    }
    console.log("Main thread ending, not the callbacks will be executed");
} 

The setTimeout will wait 0 (zero) before executing, but as the main thread has not finished, it cannot be executed. Then, when the loop ends, the callback is executed, finding that test is 50, not 0.

Answer 2

First we must implement a simple queue

Queue

function Queue() {
  var obj = {};  
  var queue = [];
  var _delay;
  
  function next() {
    // If queue is empty stops execution
    if(queue.length == 0) return;
    
    // Prepare next call to next
    setTimeout(next, _delay);
    // Take out an element from the queue and execute it.
    (queue.shift())();          
  }
  
  // Add a new function to the queue
  obj.add = function (myFunc) {
    queue.push(myFunc);
  };
  
  // Start the queue execution passing the delay between each call
  obj.run = function(delay) {
    _delay = delay;
    
    // call next function
    next();
  }
  
  return obj;
  
}

Then we use it inside the code

// create the queue
var myQueue = Queue();

async.forEachOfSeries(classes, function (Myurl, m, eachDone) {
  // Add the function to the queue
  myQueue.add(executeWaterfall.bind(this));
}, function (err) {
  if (err) throw err;
});

// Start the queue with 5 second delay
myQueue.run(5000);

function executeWaterfall() {
  async.waterfall([
    function (next) {
      connection.query(
        'SELECT * FROM mydata WHERE UrlLink=? LIMIT 1', [Myurl],
        next
      );
    },
    function (results, fields, next) {
      if (results.length !== 0) {
        console.log("Already Present");
        return next();
      }
      console.log("New Thing!");
      request(options2, function (err, resp, body) {
        if (!err && resp.statusCode == 200) {
          var $ = cheerio.load(body);
          //Some calculations, where I get AllLinks.
          var post = {
            ThisUrl: AllLinks[0],
            Time: AllLinks[1],
          };
          var query = connection.query('Insert INTO mydata Set ?', post, next);
        };
      });
    }
  ], eachDone);
}

This is far from optimal because anyway you are falling in what is called the Pyramid of doom

Bonus

Pyramid of doom

When handling asynchronous operations in succession with normal callbacks, you'll end up nesting calls within each other; with this nesting comes more indentation, creating a pyramid (pointing to the right), hence the name "Pyramid of Doom".

Solution

In this case is better to use some promise pattern to save your code from the pyramid of doom and facilitate the solution of this kind of issues.

• More Info
• PromiseJS

Answer 3

setTimeout will only delay the result to be shown in the output...it will not delay the execution of methods inside or outside the setTimeout method...In the background the thread will keep running the code after the setTimeout function...Since you are using async you nend to use COMPLETE method of ajax that will get invoked when you have completed received all the data from the server

Answer 4

I'm not intimately familiar with the async library, but to me it looks like async.waterfall will call eachdone after each run, so that async.forEachOfSeries knows that it should execute the next iteration. Assuming that eachdone is called without parameters, I would expect the following to work:

function executeWaterfall() {
  async.waterfall([
      ......
  ], function () { window.setTimeout(eachDone, 5000));
}

If eachdone does get parameters, you'd have to send them over as well.

As an alternative I expect you could add another step to your waterfall, one that waits 5 seconds. This will work regardless of parameters for eachdone (but may fail if the third waterfall function expects more paraneters):

function executeWaterfall() {
  async.waterfall([
    function (next) {
      connection.query(
        'SELECT * FROM mydata WHERE UrlLink=? LIMIT 1', [Myurl],
        next
      );
    },
    function (results, fields, next) {
      if (results.length !== 0) {
        console.log("Already Present");
        return next();
      }
      console.log("New Thing!");
      request(options2, function (err, resp, body) {
        if (!err && resp.statusCode == 200) {
          var $ = cheerio.load(body);
          //Some calculations, where I get AllLinks.
          var post = {
            ThisUrl: AllLinks[0],
            Time: AllLinks[1],
          };
          var query = connection.query('Insert INTO mydata Set ?', post, next);
        };
      });
    },
    function (next) {
      window.setTimeout(next, 5000);
    }
  ], eachDone);
}

Now, again I'd like to stress that I'm not familiar with async and all examples are untested. Perhaps everything is majorly wrong, but this is my gut feeling.