linkedin bot by adding people automatically

Question

I try to learn the web scraping by using nodejs and nightmarejs.

I wanted to create a bot that can connect to my linkedin account, then look for a technology, as javascript for example, then add the link of each profile in the sheet into a table, and with this table, I wanted to use a loop with "for" to click on the each link. At the end I send an invitation for each profile on the sheet.

Here my code :

var Nightmare = require('nightmare');
var nightmare = Nightmare({ show: true })

 var langage = process.argv[2];

 var urls = [];
 nightmare
    .viewport(1000, 720)
    .goto('https://www.linkedin.com/')
    .type('#login-email', '**********@gmail.com')
    .type('#login-password', '*********')
    .click('#login-submit')
    .wait(3000)
    .type('#main-search-box', langage)
    .wait(3000)
    .click('.search-button')
    .wait(2000)
    .click('a[data-li-vertical-type="people"]')
    .wait()
    .evaluate(function () {
 var link = [];
    $('.primary-action-button.label').each(function(i, item) {
    link.push($(item).attr('href'));
    urls = link;
})

    }
})

.run(function (err, nightmare) {
if (err) return console.log(err);
console.log('Done!');


for (var i = 0; i< urls.length; i++ ) {
nightmare
    .goto(urls[i])
    .click('#IF-reason-iweReconnect')
    .wait()
    .click('.btn-primary')
}    


  });

I type node script.js 'jquery' into the console and it told me that 'urls is not defined', and I don't undestand where is my mistake. If you can help me, it would be really helpful :) Thank you,

Kev

Answer 1

You're hitting two common pitfalls with Nightmare. First, variable lifting - like you're attempting to do with the urls array - does not work like you might think. Variables from the enclosing scope are not available to a scope inside an .evaluate() because of how the .evaluate()d function is passed to Electron. Fixing this is reasonably easy: return the array from the evaluated function and have an intermediate call to .then(). Something like (edited for brevity):

.evaluate(function () {
    var link = [];
    $('.primary-action-button.label').each(function(i, item) {
      link.push($(item).attr('href'));
    })
    return link;
 })
 .then(function(urls){
    nightmare.goto(urls[0]) ...
 })

For a more in-depth explanation, you may want to read Variable Lifting and .evaluate().

The second issue you're almost immediately going to hit is looping over something and using a Nightmare instance to execute some command. By design, Nightmare is asynchronous, and as such looping as you have it will not work. You will also probably want to give Async Operations and Loops a read. Lifted relevant vanilla JS example from there:

var urls = ['http://example1.com', 'http://example2.com', 'http://example3.com'];
urls.reduce(function(accumulator, url) {
  return accumulator.then(function(results) {
    return nightmare.goto(url)
      .wait('body')
      .title()
      .then(function(result){
        results.push(result);
        return results;
      });
  });
}, Promise.resolve([])).then(function(results){
    console.dir(results);
});