I am developing a little program with nodejs and Puppeteer which goal is to generate a certain amount of PDF files. The program is working with the bluebird module in order to achieve concurrency. The problem is that the use of Physical and virtual memory does not stop to increase. The size of all the generated documents is approximately 25GB, but the used memory in the docker container is much bigger:
pdf-generator 64.2GB (virtual 68.4GB)
We generate the PDF with Puppeteer in this way:
async function generatePDF(browser, num) {
const page = await browser.newPage();
try {
const pdfUrl = pdfPathURL(num);
await page.goto(pdfUrl, {
waitUntil: ["domcontentloaded", "networkidle0", "load"],
timeout: 300000,
});
// await page.waitForLoadState({ waitUntil: "domcontentloaded" });
const buffer = await page.pdf({
format: "A4",
printBackground: true,
landscape: true,
preferCSSPageSize: true,
});
await page.close()
return buffer.toString("base64");
} catch (error) {
let messageError = error;
console.log(messageError);
return "";
}
} finally {
await page.close();
}
}
[EDIT] This is the code that opens the Chromium instance. One per request:
async function generatePDFFile(id) {
let pdfFile;
let errorGeneration = "";
const browser = await launchPuppeteer();
try {
if (!browser) {
errorGeneration = `Error getting Chromium instance`;
}
if (!browser.isConnected()) {
errorGeneration = `Error connecting to Chromium`;
}
pdfFile = await generatePDFPage(browser, id);
console.log(`PDF file generated of id:`, id);
} catch (error) {
errorGeneration = error;
console.log("errorGeneration: ", error);
}
finally {
await browser.close();
}
return { id, pdf: pdfFile, error: errorGeneration };
}
const puppeteerParams = {
headless: true,
args: [
"--disable-gpu",
"--disable-dev-shm-usage",
"--disable-setuid-sandbox",
"--no-sandbox",
"--font-render-hinting=none",
'--single-process',
'--no-zygote'
],
};
The top command in the container
Tasks: 19 total, 1 running, 18 sleeping, 0 stopped, 0 zombie
%Cpu(s): 45.4 us, 11.0 sy, 0.0 ni, 42.5 id, 0.3 wa, 0.0 hi, 0.9 si, 0.0 st
MiB Mem : 15862.5 total, 1418.4 free, 9686.2 used, 4757.9 buff/cache
MiB Swap: 2048.0 total, 1291.0 free, 757.0 used. 4953.4 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
34855 myuser 20 0 1124.8g 222708 149664 S 81.0 1.4 0:02.74 chrome
34810 myuser 20 0 1124.8g 212544 146712 S 77.3 1.3 0:02.61 chrome
34918 myuser 20 0 1124.8g 184764 141052 S 36.7 1.1 0:01.10 chrome
31 myuser 20 0 706628 142100 33080 S 35.7 0.9 2:19.22 node
34968 myuser 20 0 1124.7g 136748 112832 S 9.3 0.8 0:00.28 chrome
35062 myuser 20 0 1124.7g 138452 114036 S 9.0 0.9 0:00.27 chrome
35013 myuser 20 0 1124.8g 137448 113456 S 8.3 0.8 0:00.25 chrome
60 myuser 20 0 965160 103512 33040 S 7.7 0.6 0:22.25 node
35106 myuser 20 0 1124.6g 105352 89208 S 5.0 0.6 0:00.15 chrome
8 myuser 20 0 630596 51892 32908 S 0.7 0.3 0:04.14 node
1 myuser 20 0 2420 524 452 S 0.0 0.0 0:00.05 sh
19 myuser 20 0 707412 57724 35064 S 0.0 0.4 0:02.26 npm start
30 myuser 20 0 2420 580 512 S 0.0 0.0 0:00.00 sh
48 myuser 20 0 705296 53336 34512 S 0.0 0.3 0:01.71 npm run example
59 myuser 20 0 2420 524 456 S 0.0 0.0 0:00.00 sh
24495 myuser 20 0 1124.7g 140500 116068 S 0.0 0.9 0:00.31 chrome
31812 myuser 20 0 4100 3376 2948 S 0.0 0.0 0:00.05 bash
31920 myuser 20 0 7012 3420 2848 R 0.0 0.0 0:00.03 top
34415 myuser 20 0 1124.8g 138368 114276 S 0.0 0.9 0:00.28 chrome
There are 8 chrome processes because I am concurrently doing 8 requests. The memory continues increasing