nodejs, use oboe to send large stream or large JSON files have some wrong

Question

I send a file from one server(ex:127.0.0.1:3000) to another server(ex:127.0.0.1:3001),I found that when the file is too large, the transfer will be a problem.

for example:

//127.0.0.1:3000
router.post('/getStream',function(req, res, next){

function getResultStream(){
    return new Promise(function(resolve, reject){
            oboe('http://127.0.0.1:3001/getEnvelopeResult_2')
                .done(function(things) {
                resolve(things);
            }).node(function(e){
                console.log(e);
            })
            .fail(function(e) {
                console.log(e);
                reject(e);
            });

        });
    }

   getResultStream().then(function(body){
       res.json({data:body});
   }).catch(function(err){
       res.json({message:err});
   });


});

for example another server: I will be to get a json file.json and converted into Stream.

//127.0.0.1:3001
app.get('/getStream',function(req, res, next){

  let path="json/file.json";

  oboe(fs.createReadStream(path))
  .done(function(things) {

    send(things,res);

   })
   .fail(function(e) {
      res.json({message:e});
   });


  function send(things,res){

    things.errorArray = JSON.parse(things.errorArray);
    let json = JSON.stringify(things);
    highland([
      json
    ])
    .invoke('split', [''])
    .sequence()
    .pipe(res)
  }

});

the file.json approximately 3000 KB in size. the content like this:

{"array":"[{\"No\":1,\"tyep\":\"none\",\"checkflag\":true,\"active\":true,\"time\":\"2017-11-08T07:04:49.024Z\"}...30000 pieces ]
"file":"axzf",
"last":30000,
"start":1
}

This array for the vast majority of capacity of about 30,000. I tried 10,000 and 20,000 and 30,000..., is failed. the array only about less than 1100 will be successfully transmitted.

the part of the wrong information:

{ statusCode: undefined,
  body: undefined,
  jsonBody: undefined,
  thrown:
   Error: Max buffer length exceeded: textNode
   Ln: 1
   Col: 131072
   Chr: undefined
       at Error (native)
       at emitError

---

at emitOne (events.js:96:13)
       at IncomingMessage.emit (events.js:188:7)
       at readableAddChunk (_stream_readable.js:176:18)
       at IncomingMessage.Readable.push (_stream_readable.js:134:10)
       at HTTPParser.parserOnBody (_http_common.js:123:22)
       at Socket.socketOnData (_http_client.js:363:20)
       at emitOne (events.js:96:13)
       at Socket.emit (events.js:188:7)
       at readableAddChunk (_stream_readable.js:176:18)
       at Socket.Readable.push (_stream_readable.js:134:10)
       at TCP.onread (net.js:547:20) }

---

 maxActual = Math.max(maxActual, textNode.length);
                                              ^

TypeError: Cannot read property 'length' of undefined
    at checkBufferLength

---

How can I improve, or do I have to do something in other ways if I can not reduce the information?

the highland: http://highlandjs.org/ the oboe: http://oboejs.com/

Answer 1

Hey I see this post is a bit old, but I was having similar issues, and I was able to solve this only using node.

My directory is set up accordingly for this test:

.
├── s1
│   └── lorem.txt
├── s2
├── server.js
└── transmitter.js

Given that, I was able to solve this using the following code:

transmitter.js

const http = require('http');
const fs = require('fs');

const requestOpts = {
  hostname: 'localhost',
  port: 1337,
  method: 'POST'
};

const readStream = fs.createReadStream('./s1/lorem.txt');
const request = http.request(requestOpts);

readStream.pipe(request);

server.js

const http = require('http');
const fs = require('fs');

http.createServer((request, response) => {
  const writeStream = fs.createWriteStream('./s2/lorem.txt');

  request.on('data', (chunk) => {
    writeStream.write(chunk);
  });

  request.on('end', () => {
    writeStream.close();
    response.end();
  });
}).listen(1337);

You can test this by running node server.js in one terminal window, and, in another, running transmitter.js.

The result should be lorem.txt appearing in the s2 directory exactly as it appears in s1.

Obviously there's a lot more to be done to make this production worthy, but hopefully that helps the next person to wander through here as I did.