Hi i am using nightmare for scrape data from website and also course details. I occur a issue :-
err: { message: 'navigation error', code: 0, details: 'OK', url: 'https://www.myskills.gov.au/courses/details?Code=CHC14015' }
on each url traversal. Please suggest me to resolve this:
var Nightmare = require('nightmare')
var vo = require('vo')
var fs = require('fs')
var filesystem = require('file-system')
// var nightmare = Nightmare({show:true});
var sleep = require('sleep');
vo(run)(function(err, result) {
if (err) throw err
console.log("Hi");
})
function *run() {
var nightmare = Nightmare({show:true});
console.log("1st step *run");
yield nightmare
.goto('https://www.myskills.gov.au/courses/search/')
.wait(12000)
//.click('#select3 value="100")
.evaluate(function () {
var hrefs = [];
$('.search-result h4 a').each(function()
{
var course = new Object();
course.title = $(this).html();
course.link = $(this).attr('href');
course.code = $('.search-result .col-md-2.text-small
span').html();
hrefs.push(course);
});
return hrefs;
})
.then(function(str){
console.log("2nd step evaluate then on page load");
console.log(str);
for(var i=0; i< 8; i++)
{
console.log(i);
sleep.sleep(1);
var link = "https://www.myskills.gov.au";
var coursetitle = str[i].title
var courselink = link +str[i].link+"\n";
var coursesinglelink = link +str[i].link;
var courseData = coursetitle+"\n"+str[i].code+"\n"+courselink;
fs.appendFile('getcourselink.txt', courseData, function (err) {
if(err) console.log(err);
});
vo(run1)(function(err, result) {
if (err) console.log ("err: ",err);
console.log("Hi in run1");
//console.log(result);
});
function *run1() {
console.log("I 2nd time:-"+i);
console.log(coursesinglelink);
sleep.sleep(2);
var nightmare1 = Nightmare({show:true});
yield nightmare1
.goto(coursesinglelink)
.wait(9000)
.evaluate(function () {
var str="Hi";
var CourseDetails = $('#details #courseStructureDiv
#packagingrules').text();
str = str+"\n"+CourseDetails;
return str;
})
.then(function(str){
console.log("Run inner then",str);
fs.appendFile('getcourselink.txt', str, function (err) {
if(err) console.log(err);
});
// nightmare1.end();
});
}
//nightmare.end();
// nightmare.proc.disconnect();
// nightmare.proc.kill();
// nightmare.ended = true;
// nightmare = null;
}
});
}