How to perform bulk upload in mongoose (node.js)

Question

I am trying to perform bulk upload in node js and mongodb is my db,can anyone suggest me some best articles regarding this.Thanks in advance.

Answer 1

You can insert multiple records with batch/bulk insert in mongoose.

var arr = [{ name: 'Star Wars' }, { name: 'The Empire Strikes Back' }];
Movies.insertMany(arr, function(error, docs) {});

Let's say i have an excel file employees.xlsx with following data and i want perform bulk write.

There are several libraries out there for converting excel data to json in node, i use xlsx but it's personal taste you can use whatever is convenient for you.

Here is a helper i use for reading "/public/employees.xlsx" file.I found the content from here here.

 //** helper/excel-reader.js **//

var excelReader = {};
excelReader.readExcel =  function(filePath){     

  var XLSX = require('xlsx');
  var workbook = XLSX.readFile(filePath);
  var sheet_name_list = workbook.SheetNames;
  var data = [];

  sheet_name_list.forEach(function(y) {
    var worksheet = workbook.Sheets[y];
    var headers = {};

    for(z in worksheet) {
        if(z[0] === '!') continue;
        //parse out the column, row, and value
        var tt = 0;
        for (var i = 0; i < z.length; i++) {
            if (!isNaN(z[i])) {
                tt = i;
                break;
            }
        };
        var col = z.substring(0,tt);
        var row = parseInt(z.substring(tt));
        var value = worksheet[z].v;

        //store header names
        if(row == 1 && value) {
            headers[col] = value;
            continue;
        }

        if(!data[row]) data[row]={};
        data[row][headers[col]] = value;
    }
    //drop those first two rows which are empty
    data.shift();
    data.shift();       
  });
 return data;
}

module.exports = excelReader;

Now the employee model somehow looks like this.

var mongoose =  require('mongoose');
var Schema =  mongoose.Schema;

var employee = new Schema({
    name: String,
    adderess: String,
    phonenumber: String
});

module.exports = mongoose.model('Employee', employee);

Now let's use the above code so here is my users.js route whenever i type "localhost:3000/users" it write the csv content to database.

var express = require('express');
var router = express.Router();
var excelReader = require('../helpers/excel-reader');
var mongoose = require('mongoose');
var Employee = require('../models/employee');

 /* GET users listing. */
router.get('/', function(req, res, next) {
  var employeesJsonArray = excelReader.readExcel('./public/employees.xlsx') 
  Employee.insertMany(employeesJsonArray,function(error, docs) {
    if(error){
      next(error)
    }
    else{
      res.json(docs);//just rendering the document i got
    } 
   });
});
module.exports = router;

Hope this helps!!

Answer 2

You can use Model.collection.insert or Model.insertMany as below where collections is array of items in bulk.

Model.collection.insert(collections, function (err, models) {
   next(err, models);
});

OR,

Model.insertMany(collections, function (err, models) {
   next(err, models);
});

Mongoose reference: http://mongoosejs.com/docs/api.html#model_Model.insertMany

Mongo reference: https://docs.mongodb.com/v3.2/reference/method/db.collection.insert/