Pipe data once it's ready in Node.js

552 views Asked by At

I have a large JSON file like this:

[
 {
  "id": 2000,
  "city": "New York",
  "lat": "",
  "lon": "",
 },
...
]

and I'm searching for the lat and lon in each object. I'm using the JSONStream module to pipe the data:

var writeStream = fs.createWriteStream('data/stream.json');
var readStream = fs.createReadStream('data/dummy/institucion_1_2000.json', {encoding: 'utf8'})

// Search for lat and lon
var search = es.mapSync(function(data){
  if(data.lat == ""){
    // Search for the lat and lon
    geocoder.geocode({address: data.city, country: "US"}) 
      .then(function(res) {
        console.log("Searched for " + res[0].formattedAddress);
        data.lat = res[0].latitude;
        data.lon = res[0].longitude;
      })
      .catch(function(err) {
        console.log("There was an error with element with id = " + data.id);
        console.log("Here is the error: " + err);
        if(err == 'Error: Status is OVER_QUERY_LIMIT. You have exceeded your rate-limit for this API.') {
          process.exit();
        }
      });
    return data;
  }
})

// Pipe
readStream
  .pipe(JSONStream.parse('*'))
  .pipe(search) 
  .pipe(JSONStream.stringify()) // This doesent wait until the search is finish
  .pipe(writeStream)

The geocoding parts works.

My problem is that the JSONStream.stringify reads and then pipes the data before the search function ends. So I'm getting the same JSON File without the modifications I need. If I just try this:

if(data.lat == ""){
    lat = 1;
}

instead of the geocoding, which takes more time, it works. I'm guessing my problem is on the time it takes to modify the data streamed. So, is there a way to pipe the data once it has been modified?

EDIT I had a mess between sync and async. Thanks to djones

var search = es.map(function (data, callback) {
  if(data.lat == ""){
    geocoder.geocode({address: data.city, country: "USA"}) // Choose between Comuna_Empresa and Comuna_Institucion
    .then(function(res) {
      console.log("Searched for " + res[0].formattedAddress);
      data.lat = res[0].latitude;
      data.lon = res[0].longitude;
      callback(null,data);
    })
    .catch(function(err) {
      console.log("There was an error with element at index = " + index);
      console.log("Here is the error: " + err);
    });
  }
})
0

There are 0 answers