Removing the last character from file stream in node.js (fs module)

Question

Using node.js, I am trying to build an array of objects and write them to a file. To do this, I'm using the built in fs library.

After calling var file = fs.createWriteStream('arrayOfObjects.json'); and file.write('[') I run several asynchronous functions to eventually append objects like this:

file.write(JSON.stringify(objectToAppend) + ',\n')

I can determine when all of the objects have stopped appending, and this is where I run file.write(']') and file.end(). My problem is that adding the last comma to the end of the last object causes the JSON to be invalid.

It is very difficult to determine where and when the last object is being created due to the asynchronous nature of the script, so I was wondering if there is a way to strip or remove characters from a file-stream. If so, I could do this before adding the last ']' character.

I could do this manually, but I was hoping to pipe this to another application. The only solution I've thought about is using the fs.truncate() function, however this doesn't seem to work for file streams, and neither file.length or file.length() will give me the length of the contents because it is not a string so it's difficult to determine how or where to truncate the file.

For now I have just been adding '{}]' to the end of the array to make it valid JSON, but this empty object may cause some problems later.

Also note: the array of objects I am writing in this stream is VERY large, so I would rather not end the stream and re-open the file.

Answer 1

I'd recommend to prepend the separator instead, so that you dynamically can adjust it after the first call:

file.write('[\n')
var sep = "";
forEach(function(objectToAppen) {
    file.write(sep + JSON.stringify(objectToAppend))
    if (!sep)
        sep = ",\n";
});

Answer 2

Example using JSONStream:

var JSONStream = require('JSONStream');
var fs         = require('fs');

var jsonwriter = JSONStream.stringify();
var file       = fs.createWriteStream('arrayOfObjects.json');

// Pipe the JSON data to the file.
jsonwriter.pipe(file);

// Write your objects to the JSON stream.
jsonwriter.write({ foo : 'bar#1' });
jsonwriter.write({ foo : 'bar#2' });
jsonwriter.write({ foo : 'bar#3' });
jsonwriter.write({ foo : 'bar#4' });

// When you're done, end it.
jsonwriter.end();

Answer 3

The accepted answer is interesting (prepending the separator) but in my case I have found it easier to append the separator and remove the last character of the file, just as suggested in the question.

This is how you remove the last character of a file with Node.js :

import fs from 'fs'

async function removeLastCharacter(filename) {
  const stat = await fs.promises.stat(filename)
  const fileSize = stat.size

  await fs.promises.truncate(filename, fileSize - 1)
}

explanation :

• fs.promises.stat gives us some information about the file, we will use its size.
• fs.promises.truncate remove from the file what is after a certain position
• We use the position fileSize - 1 which is the last character.

Note : Yes I know that we need to wait until the stream is closed, but this is ok because truncate and stat functions are very fast and don't depend on the file size, they don't have to read its content.

Answer 4

Here's a snippet incorporating robertklep's answer. This converts from a pipe-separated file to json:

var fs = require('fs');
var readline = require('readline');
var JSONStream = require('JSONStream');


// Make sure we got a filename on the command line.
if (process.argv.length < 3) {
  console.log('Usage: node ' + process.argv[1] + ' FILENAME');
  process.exit(1);
}

var filename = process.argv[2];
var outputFilename = filename + '.json';
console.log("Converting psv to json. Please wait.");
var jsonwriter = JSONStream.stringify();
var outputFile = fs.createWriteStream(outputFilename);
jsonwriter.pipe(outputFile);

var rl = readline.createInterface({
   input: fs.createReadStream(filename),
   terminal: false
}).on('line', function(line) {
  console.log('Line: ' + line);
   if(!/ADDRESS_DETAIL_PID/.test(line))
   {     
     var split = line.split('|');
     var line_as_json = { "address_detail_pid":  split[0], "flat_type": split[1], "flat_number": split[2], "level_type": split[3], "level_number": split[4], "number_first": split[5], "street_name": split[6], "street_type_code": split[7], "locality_name": split[8], "state_abbreviation": split[9], "postcode": split[10], "longitude": split[11], "latitude": split[12] };
     jsonwriter.write(line_as_json);
   }    
}).on('close', () => {
  jsonwriter.end();
});;

console.log('psv2json complete.');