i've created a nodejs script for processing a mongodb collection of products. Each product needs to be processed with jexl, an expression language. The final result should be a text file with all product data. The script works but i need some help to optimize it because it consumes a lot of memory. Within the script i'm using a mongodb cursor with a batchsize of 1-5 to limit the received documents. The query receives around ~9200 documents with an average size of 8.7KB.
With a batchsize of 1 the script takes hours and consumes around 200-600 MB of memory. When i set the batchsize to ~10 i get an out of memory exception. (CALL_AND_RETRY_LAST Allocation failed - process out of memory).
I've tried to comment out the jexl.eval and the script runs in a few seconds.
Does someone have an idea how to optimize this script? The lineTemplate is a joined string of all jexl expressions. I cut this expression string to make the whole script more readable. Normally the last expression "{{attributes[.id == 1 && .language == ""] in attributes ? attributes[.id == 1 && .language == ""].value : ""}}|" repeats 106 times with different ids.
var async = require("async");
var lineTemplate = '{{no}}|Parent_ID|{{no}}|{{translations[.language == "DE-DE"] in translations ? translations[.language == "DE-DE"].title : ""}}|{{prices[.channel == "DE" && .specialPrice == false] in prices ? prices[.channel == "DE" && .specialPrice == false].price : ""}}|{{prices[.channel == "DE" && .specialPrice == true] in prices ? prices[.channel == "DE" && .specialPrice == true].price : ""}}|{{itemCategory}}|{{productGroup}}|{{groupOfGoods}}|http://www.google.de/test.html|{{crossReferenceNo}}|{{brand}}|Replenishment_in_days|Quantity_in_Stock|Availability|EUR|{{attributes[.id == 1 && .language == ""] in attributes ? attributes[.id == 1 && .language == ""].value : ""}}| ... \r\n'
var MongoClient = require('mongodb').MongoClient
, assert = require('assert');
var fs = require('fs');
var filename = '/tmp/products.txt';
fs.writeFileSync(filename, 'no;name;price\r\n');
function ProcessProduct(product, expression, cb) {
var jexl = require('Jexl');
var regex = /{{([^{}]*)}}/g;
var line = lineTemplate;
var matches = lineTemplate.match(regex);
async.each(matches,
function(match, callback){
var query = match.substring(2, match.length - 2);
jexl.eval(query, product, function(err, res) {
if(err) console.log(err);
//console.log(res);
var strToReplace = "{{" + query + "}}";
line = line.replace(strToReplace, res);
callback();
});
},
function(err){
delete(jexl);
if(err) return cb(err, null);
cb(null, line);
}
);
}
var url = 'mongodb://localhost/test';
MongoClient.connect(url, function(err, db) {
assert.equal(null, err);
console.log("Connected correctly to server");
var collection = db.collection('products');
var cursor = collection.find({ "saleschannel": { $elemMatch: { "channel": "DE" } }}, { "batchSize": 1, fields: {} }).each(function(err, product) {
ProcessProduct(product, lineTemplate, function(err, data) {
fs.appendFile(filename, data, function(err) {
if (err) throw err;
});
});
});
});