Configure Nginx for file upload using connect-busboy

Question

I have an API in NodeJS where I can upload a file, and I receive it on the server side using the connect-busboy package.

So, for example, here is a part of the code that handles the request:

var app = require('express')();
var busboy = require('connect-busboy');

app.use(busboy({
        highWaterMark: 2 * 1024 * 1024,
        limits: {
            fileSize: 1024 * 1024 * 1024 // 1 GB
        },
        immediate: true
    }));

var busboyHandler = function (req, res) {
    if (req.busboy) {
        req.busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
            console.log('received file ', req.path, fieldname, file, filename, encoding, mimetype);
        });

        req.busboy.on('field', function(key, value, keyTruncated, valueTruncated) {
            console.log('field..', key, value, keyTruncated, valueTruncated);
        });

        req.busboy.on('finish', function() {
            console.log('busboy finished');
        });
    }
};

app.post('api/documents/files', busboyHandler); 

This works well when I start the API with npm start and upload the file to this API directly, however, when I configure an Nginx Docker, it works for really small files, but for most files, they don't get uploaded successfully.

Here is an excerpt from my nginx.conf file:

user nobody nogroup;
worker_processes auto;          # auto-detect number of logical CPU cores

events {
  worker_connections 512;       # set the max number of simultaneous connections (per worker process)
}

http {
  include mime.types;

  client_max_body_size 100M;
  client_body_buffer_size 256k;

  upstream api_doc {
    server 192.168.2.16:4956;
  }

  server {
    listen *:4000;                # Listen for incoming connections from any interface on port 80
    server_name localhost;             # Don't worry if "Host" HTTP Header is empty or not set
    root /usr/share/nginx/html; # serve static files from here

    client_max_body_size 100M;
    client_body_buffer_size 256k;

    location /api/documents {
        proxy_pass http://api_doc;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection 'upgrade';
        proxy_set_header Host $host;
        proxy_cache_bypass $http_upgrade;
     }
  }

}

I see the received file log, but when it's under Nginx, there is never a busboy finished log, unlike when calling the API directly without Nginx.

I tried changing these Nginx configs but it did not work: client_max_body_size, client_body_buffer_size. It looks to me like the API receives only a chunk of the file for larger files, instead of the whole file, or all chunks, as it should.

Any help would be appreciated.

Thanks, Simon

Answer 1

It turned out that the problem was elsewhere, I was starting to read the incoming stream before the file was fully uploaded, so it led to breaking the incoming stream for some reasons.