Node.js transform file stream and write to same file results in empty file

Question

I am trying to modify some files using node file streams and custom transform function. This is the transform function:

const TransformStream = function() {
  Transform.call(this, {objectMode: true});
};
util.inherits(TransformStream, Transform);

TransformStream.prototype._transform = function(chunk, encoding, callback) {
  let line = chunk.toString()
  if (!this.findLinesMode && lineStartRe.test(line)) {
    this.findLinesMode = true
    this.lines = []
  }
  if (this.findLinesMode) {
    this.lines.push(line)
  }
  if (this.findLinesMode && lineEndRe.test(line)) {
    this.findLinesMode = false
    line = this.lines.join('').replace(re, (str, match) => match.trim())
  }
  if (!this.findLinesMode) {
    this.push(line + '\n')
  }

  callback()
};

And I tried to use it in the following code:

byline(fs.createReadStream(filePath, {encoding: 'utf8'}))
  .pipe(new TransformStream())
  .pipe(fs.createWriteStream(filePath))

However, the file ends up empty.

I am confident that the transformer code works as expected, because I tried pipe it to process.stdout and the output is exactly how I want it.

My question is: What I am doing wrong and what can I try to fix it?

Answer 1

This is not a problem with your transformer code but a problem that you open a file for writing that you overwrite probably before you even read anything from it.

It would be the same in shell. If you run:

cat < file.txt > file.txt

or:

tr a-z A-Z < x.txt > x.txt

it will result in making the file empty.

You have to pipe to a temporary file and then substitute the old file with the new one. Or alternatively rename the old one to some other temporary name, open the new file under the correct name and pipe the renamed file to the old file, making your transforms on the way.

Make sure to use a secure way to make a name of the temporary file. You can use modules like:

• https://www.npmjs.com/package/tmp
• https://www.npmjs.com/package/temp
• https://www.npmjs.com/package/tempfile

Answer 2

Using @rsp's suggestion, we can use the tmp package from NPM to create a temporary file and write onto it after transformation. The next step is to copy this file to the original file.

const fs = require("fs");
const {Transform} = require("stream");
const tmp = require("tmp")
const readStream = fs.createReadStream(this.databasePath);
const tmpFile = tmp.fileSync({'postfix' : '.json'});
const writeStream = fs.createWriteStream(tmpFile.name);
const filePath = this.databasePath;
try{
      readStream.pipe(
        new Transform({
          transform(chunk, encoding, callback){
            let fileData = JSON.parse(chunk);
            fileData.data = fileData.data.map((_doc, _index) => {
              let updatedDoc = _doc;
              if(_doc._id===_id){
                updatedDoc = {..._doc,[ _field] : _value};
              }
              return updatedDoc;
            });

        callback(null,JSON.stringify(fileData));
      }
    })
  ).pipe(writeStream).on('finish', () =>{
    return fs.copyFile(tmpFile.name, filePath, (err) =>{
      if(err) {
        throw err;
      }
    })
  });
  this.data = this.updatedData;
}
catch(error){
  // undo the changes in the local copy
  this.updatedData = this.data;
  return this._handleData(error, null);
}

This is an example to show how to use the tmp package to update the original file using a temporary copy.