Retrieving SFTP listings asynchorously in Node.js

Question

I'm trying to retrieve SFTP listings using Node, from multiple servers. I'm using the ssh2-sftp-client library, and I'm trying to handle the asynchronous connections using a futzed Promise.all().

The SFTP servers are stored in a config file (servers.config) like this:

{
  "myhost1": {
    "host": "sftp.myhost1.com",
    "port": "22 ",
    "username":  "userid1",
    "password": "password1"
  },
  "myhost2": {
    "host": "sftp.myhost2.com",
    "port": "22 ",
    "username":  "userid2",
    "password": "password2"
  },
  ...
}

My code looks like this...

#!/node
let fs = require('fs');
let Client = require('ssh2-sftp-client');

// which servers should we process?
const serverList = fs.readFileSync('./servers.config', {encoding:'utf8', flag:'r'});
const servers = JSON.parse(serverList);

const servers_to_process = Object.keys(servers);

function getDirectoryListing(config) {
  let sftp = new Client();
  sftp.connect(config)
    .then(() => {
      return sftp.list('/');
    })
    .then(data => {
      console.log('Data retrieved for: ',config.host);
      //console.log(data);  // Line B
      sftp.end();
      return data;
    })
    .catch(err => {
      console.log('Error for: ',config.host);
      return [];
    });
}


const processes_to_run = [];

// generate array of promises to run
servers_to_process.forEach( key => {
    log('==========================');
    log("Provider: "+key+"; "+timestamp);
    processes_to_run.push(getDirectoryListing(servers[key]));
  });


// wait for all the promises to resolve...
Promise.allSettled(processes_to_run).
  then((results) => results.forEach((result) => console.log(result)));

What I'm not getting is any console logged data from line A... but if I uncomment Line B, I get each listing, asynchronously.

The output looks something like this:

JSON file read correctly
==========================
Provider: myhost1; 01/06/2021, 14:57:25
==========================
Provider: myhost2; 01/06/2021, 14:57:25
{ status: 'fulfilled', value: undefined }
{ status: 'fulfilled', value: undefined }
Data retrieved for:  sftp.myhost1.com
Data retrieved for:  sftp.myhost2.com

So, clearly I'm dropping the ball on returning the data from the promises...

Is this the correct approach to getting all the listings into array, prior to processing? Is there a cleaner approach, given the asynchronous nature of the SFTP list fetching?

Answer 1

You need to actually return the promise from your function - getDirectoryListing() isn't returning anything. Thus, you are passing an array full of undefined to Promise.allSettled()

Try this:

function getDirectoryListing(config) {
  let sftp = new Client();
  return sftp.connect(config)
    .then(() => {
    // ...stuff
}

Answer 2

Your getDirectoryListing isn't actually returning a promise. Something like this should work for you:

#!/node
let fs = require('fs');
let Client = require('ssh2-sftp-client');

// which servers should we process?
const serverList = fs.readFileSync('./servers.config', {encoding:'utf8', flag:'r'});
const servers = JSON.parse(serverList);

const servers_to_process = Object.keys(servers);

//Ensure this is returning a promise by making it async
//and controlling the flow with await rather than callbacks
async function getDirectoryListing(config) {
    let sftp = new Client();
    await sftp.connect(config)
    let list = await sftp.list('/');
    console.log('Data retrieved for: ',config.host);
    console.log(list);  // Line B
    sftp.end();
    return list;
}


const processes_to_run = [];

// generate array of promises to run
servers_to_process.forEach( key => {
    console.log('==========================');
    console.log("Provider: "+key+"; "+Date.now());
    processes_to_run.push(getDirectoryListing(servers[key]));
  });


// wait for all the promises to resolve...
Promise.allSettled(processes_to_run).
  then((results) => results.forEach((result) => console.log(result)));

Answer 3

To make your code work you just need to return the promises from the function getDirectoryListing() to ensure the proper ordering of execution of statements.

Your fix:

function getDirectoryListing(config) {
  let sftp = new Client();
  return sftp.connect(config) // just add a return here
    // ...rest code will be same
}

But you must also understand the reason why such unexpected results are coming. (Read this section if you want to understand what's happening under the hood)

When you call the method getDirectoryListing(), you add the promises to the event loop and return undefined. Since the processes_to_run array is full of undefined there is no promise to execute in the processes_to_run array. That's why the execution first goes to the console.log(result).

// wait for all the promises to resolve...
Promise.allSettled(processes_to_run).
  then((results) => results.forEach((result) => console.log(result)));

Once the event loop completes the promises, it adds them to the callback queue and they are then processed. Thus, line A is printed afterwards.

  sftp.connect(config)
    .then(() => {
      return sftp.list('/');
    })
    .then(data => {
      console.log('Data retrieved for: ',config.host); // Line A
      //console.log(data);  // Line B
      sftp.end();
      return data;
    })
    .catch(err => {
      console.log('Error for: ',config.host);
      return [];
    });

If you want to understand more about the event loop, you can watch this amazing video.