Having an issue with order of operations in node.js function

Question

I have a function that's I would like to use to poll historical data beginning from the start of Ethereum trading on the gdax exchange to current. I want the code to poll 100 results (max allowable at a time) insert results into mongodb, add 100, then repeat, until I've got all historical data. However, what I've wrote is executing in a different order. First it sums all numbers in the for next, then executes all the http authedclient requests (although it properly uses the variable adding 100 to each request), then drops all the data into mongodb. The problem I'm having is by the time the data is inserted into the mongodb client, the connection is closed. How can I get it to execute each operation in the order I want it to? Pastebin posted below. Note the code I use has changed significantly to accommodate other errors, changed the insert loop to insertMany but I can integrate suggestions properly if anyone can help. Thanks in advance.
https://pastebin.com/1BHi9RQV

var i = 100;

for (; i < 18750000; ) {

    publicClient.getProductTrades({ after: i }, function (err, response, data) {

        if (err) {

            console.log(err);

            return;

        }

        MongoClient.connect(url, function (err, db) {

            if (err) {

                console.log(err);

                return;

            }

            var myobj = { data };

            for (item in data) {

                db.collection("EthTest").insertOne(data[item], function (err, res) {

                    if (err) {

                        console.log(err);

                        return;

                    }

                });

                console.log(data[item].trade_id);

            }

            db.close();

        });

    });

    i = i + 100;

};

Also downloaded stack app to properly post link and code. Mobile site didn't allow me to do that. Sorry!

Answer 1

Don't put db.close(); in your for loop. Only call it once all your inserts are completed.

If this order is really important (get 100 results, store them, and so on), I suggest you split this code in 2 functions that call each other. Something like that:

var i = 100;
var mongo;

function getData() {
    publicClient.getProductTrades({ after: i }, function (err, response, data) {
        if (err) {
            console.log(err);
            return;
        }

        i = i + 100;
        saveData(data);
    });
}

function saveData(data) {

    var keys = Object.keys(data);

    // loop through the data
    for (j = 0, end = keys.length; j < end; j++) {
        db.collection("EthTest").insertOne(data[item], function (err, res) {
            if (err) {
                console.log(err);
                return;
            }

            console.log(data[item].trade_id);

            if (keys[j] === keys[keys.length-1]) {
                // last item in data was inserted
                if (i < 18750000) {
                    // get the next 100 results if there's any
                    getData();
                } else {
                    mongo.close();
                }
            }
        });
    };
}


MongoClient.connect(url, function (err, db) {
    if (err) {
        console.log(err);
        return;
    }

    mongo = db;
    getData();
});

Or course I cannot test it for you, but I hope you get the idea.

The other option would be to separate your 2 async calls. Have the first loop get all the data (100 at a time), store them in an array, and once that is done, loop through that array and insert the data in Mongo.