NodeJS script and high memory usage while processing mongodb documents with jexl

147 views Asked by At

i've created a nodejs script for processing a mongodb collection of products. Each product needs to be processed with jexl, an expression language. The final result should be a text file with all product data. The script works but i need some help to optimize it because it consumes a lot of memory. Within the script i'm using a mongodb cursor with a batchsize of 1-5 to limit the received documents. The query receives around ~9200 documents with an average size of 8.7KB.

With a batchsize of 1 the script takes hours and consumes around 200-600 MB of memory. When i set the batchsize to ~10 i get an out of memory exception. (CALL_AND_RETRY_LAST Allocation failed - process out of memory).

I've tried to comment out the jexl.eval and the script runs in a few seconds.

Does someone have an idea how to optimize this script? The lineTemplate is a joined string of all jexl expressions. I cut this expression string to make the whole script more readable. Normally the last expression "{{attributes[.id == 1 && .language == ""] in attributes ? attributes[.id == 1 && .language == ""].value : ""}}|" repeats 106 times with different ids.

var async = require("async");

var lineTemplate = '{{no}}|Parent_ID|{{no}}|{{translations[.language == "DE-DE"] in translations ? translations[.language == "DE-DE"].title : ""}}|{{prices[.channel == "DE" && .specialPrice == false] in prices ? prices[.channel == "DE" && .specialPrice == false].price : ""}}|{{prices[.channel == "DE" && .specialPrice == true] in prices ? prices[.channel == "DE" && .specialPrice == true].price : ""}}|{{itemCategory}}|{{productGroup}}|{{groupOfGoods}}|http://www.google.de/test.html|{{crossReferenceNo}}|{{brand}}|Replenishment_in_days|Quantity_in_Stock|Availability|EUR|{{attributes[.id == 1 && .language == ""] in attributes ? attributes[.id == 1 && .language == ""].value : ""}}| ... \r\n'

var MongoClient = require('mongodb').MongoClient
  , assert = require('assert');

var fs = require('fs');
var filename = '/tmp/products.txt'; 
fs.writeFileSync(filename, 'no;name;price\r\n');

function ProcessProduct(product, expression, cb) {
  var jexl = require('Jexl');
  var regex = /{{([^{}]*)}}/g;
  var line = lineTemplate;
  var matches = lineTemplate.match(regex);

  async.each(matches,
    function(match, callback){
        var query = match.substring(2, match.length - 2);
        jexl.eval(query, product, function(err, res) {
            if(err) console.log(err);
            //console.log(res);
            var strToReplace = "{{" + query + "}}";
            line = line.replace(strToReplace, res);
            callback();
        });
    },
    function(err){
        delete(jexl);
        if(err) return cb(err, null);
        cb(null, line);
    }
  );
}

var url = 'mongodb://localhost/test';
MongoClient.connect(url, function(err, db) {
  assert.equal(null, err);
  console.log("Connected correctly to server");

  var collection = db.collection('products');
  var cursor = collection.find({ "saleschannel": { $elemMatch: { "channel": "DE" } }}, { "batchSize": 1, fields: {} }).each(function(err, product) {
    ProcessProduct(product, lineTemplate, function(err, data) {
      fs.appendFile(filename, data, function(err) {
        if (err) throw err;
      });
    });
  });
});
0

There are 0 answers