NIghtmare error in web scraping course details

167 views Asked by At

Hi i am using nightmare for scrape data from website and also course details. I occur a issue :-

err: { message: 'navigation error', code: 0, details: 'OK', url: 'https://www.myskills.gov.au/courses/details?Code=CHC14015' }

on each url traversal. Please suggest me to resolve this:

   var Nightmare = require('nightmare')

var vo = require('vo')

var fs = require('fs')

var filesystem = require('file-system')
// var nightmare = Nightmare({show:true});

var sleep = require('sleep');

vo(run)(function(err, result) {

 if (err) throw err

  console.log("Hi");

})

function *run() {
 
 var nightmare = Nightmare({show:true});

  console.log("1st step *run");

  yield nightmare

    .goto('https://www.myskills.gov.au/courses/search/')

    .wait(12000)

    //.click('#select3 value="100")

    .evaluate(function () {

          var hrefs = [];

        $('.search-result h4 a').each(function()

        {
            var course = new Object();

            course.title = $(this).html();

            course.link = $(this).attr('href');

            course.code = $('.search-result .col-md-2.text-small 
span').html();
           hrefs.push(course);

        });

       return hrefs;

    })

  .then(function(str){

    console.log("2nd step evaluate then on page load");

       console.log(str);

      for(var i=0; i< 8; i++)

      {
           console.log(i);

           sleep.sleep(1);
           var link = "https://www.myskills.gov.au";

           var coursetitle =  str[i].title

           var courselink = link +str[i].link+"\n";

           var coursesinglelink = link +str[i].link;

           var courseData = coursetitle+"\n"+str[i].code+"\n"+courselink;

        fs.appendFile('getcourselink.txt', courseData, function (err) {
              if(err) console.log(err);
            });

        vo(run1)(function(err, result) {
          if (err) console.log ("err: ",err);

          console.log("Hi in run1");
        //console.log(result);

      });
          function *run1() {
            console.log("I  2nd time:-"+i);

            console.log(coursesinglelink);

            sleep.sleep(2);
          var nightmare1 = Nightmare({show:true});

          yield nightmare1

            .goto(coursesinglelink)

            .wait(9000)

            .evaluate(function () {
                var str="Hi";

          var  CourseDetails =  $('#details #courseStructureDiv 
#packagingrules').text();
                str = str+"\n"+CourseDetails;
            return str;

            })

            .then(function(str){
              console.log("Run inner then",str);

            fs.appendFile('getcourselink.txt', str, function (err) {
                  if(err) console.log(err);
                });

            //  nightmare1.end();
            });

        }
      //nightmare.end();
    //  nightmare.proc.disconnect();
    // nightmare.proc.kill();
    // nightmare.ended = true;
    // nightmare = null;

      }

    });


}
0

There are 0 answers