Parallel or Sequential copying of files in WinRT

Question

I have this recursive copyFolder method that copies (source modified) files/sub folders in parallel.

copyFileIfNewerAsync = function(sourceFile, destFolder) {
  return destFolder.tryGetItemAsync(sourceFile.name).then(function(destFile) {
    if (destFile) {
      return WinJS.Promise.join({
        sourceProperties: sourceFile.getBasicPropertiesAsync(),
        destProperties: destFile.getBasicPropertiesAsync()
      }).then(function(_arg) {
        var destProperties, sourceProperties;
        sourceProperties = _arg.sourceProperties, destProperties = _arg.destProperties;
        if (sourceProperties.size !== destProperties.size && sourceProperties.dateModified > destProperties.dateModified) {
          logger.debug("Updating " + destFolder.path + "\\" + destFile.name);
          return sourceFile.copyAsync(destFolder, sourceFile.name, NameCollisionOption.replaceExisting);
        } else {
          return logger.trace("" + destFolder.path + "\\" + destFile.name + " up-to-date (" + destProperties.dateModified + ")");
        }
      });
    } else {
      logger.debug("Copying " + sourceFile.name + " to " + destFolder.path);
      return sourceFile.copyAsync(destFolder, sourceFile.name, NameCollisionOption.replaceExisting);
    }
  });
};
copyFolderAsync = function(destFolder, sourceFolder) {
  return destFolder.createFolderAsync(sourceFolder.name, CreationCollisionOption.openIfExists).then(function(destSubFolder) {
    return sourceFolder.getItemsAsync().then(function(items) {
      return WinJS.Promise.join(items.map(function(item) {
        if (item instanceof Windows.Storage.StorageFile) {
          return copyFileIfNewerAsync(item, destSubFolder);
        } else {
          return copyFolderAsync(destSubFolder, item);
        }
      }));
    });
  });
};

It seems this stresses the system pretty much. Maybe a sequential copying approach would be less stressful for the system and in the end even faster?

If so, how would I have to refactor the code to make it run sequentially?

update

about the WinJS.Scheduler

preloadAsync = (serialNumbers, expert) ->
    assert(expert)
    assert(serialNumbers)
    serialNumbers = [serialNumbers] unless Array.isArray(serialNumbers)
    preloadPromise?.cancel()
    preloadPromise = serialNumbers.reduce((p, serialNumber) ->
      p.then(WinJS.Utilities.Scheduler.schedulePromiseBelowNormal)
      .then () ->
        preloadOneAsync(serialNumber, expert)
      .then null, (error) ->
        if error.name isnt "Canceled"
          logger.error("Could not create preloaded inspection #{serialNumber}", error)
    , WinJS.Promise.as())

This should then schedule all downloads at low prio?

Answer 1

Yes, I imagine that you're overloading the file system with too many parallel operations, as each async call will spin off its own thread. I think a reasonable approach to try would be to let each folder run in parallel and have the files within each folder run sequentially.

The trick is to run each async file copy sequentially. For this there's a useful promise pattern to do sequential chaining from an array of input arguments, which I describe on page 1208 (Appendix A) of my free ebook Programming Windows Store Apps with HTML, CSS, and JavaScript, 2nd Edition.

In your case, I think you'll want to do a separate call to sourceFolder.getFoldersAsync and iterate the results calling copyFolderAsync. Then call sourceFolder.getFilesAsync to get the array of files. This you can use with the array.reduce method as shown in the pattern, which builds an accumulated array of promises one at a time. Using the code from the book, your "op" would be a call to copyFileIfNewerAsync.

Here's a modified code snippet from my example just to show it:

//This op function attached other arguments to each call
var op = function (file) { return copyFileIfNewerAsync(file); };  

//The arguments we want to process are in "files" from getFilesAsync

//This code creates a parameterized promise chain from the items array and the async call
//in op. By using WinJS.Promise.as for the initializer we can just call p.then inside
//the callback.
var endPromise = files.reduce(function (p, file) {
    return p.then(function (r) {
        //The first result from WinJS.Promise.as will be undefined, so skip logging
        if (r !== undefined) { App.log("operation completed with results = " + r) }; 

        return op(file);
    });
}, WinJS.Promise.as());  

//endPromise is fulfilled with the last operation's results when the whole chain is complete.
endPromise.done(function (r) {
    App.log("Final promise complete with results = " + r);
});