I'm working on a nightmare JS script that logs in to a website, builds a list of links based on results in a table then opens each of those links. On each link opened, some information is scraped and added to the end results.
I'm having lots of trouble with the loop of opening each link. This process must be synchronous due to restrictions at the website.
Below is a very simplified example of what I have so far, and what I'm getting returned.
I'm new at this library, and while I've been looking at the documentation I'm finding it pretty confusing.
Any advice on getting a .goto()
loop working?
const Nightmare = require('nightmare')
const moment = require('moment')
const opts = {
show: true,
openDevTools: { mode: 'detach' },
pollInterval: 250,
waitTimeout: 10000,
webPreferences: {
webSecurity: false
}
}
const nightmare = Nightmare(opts)
nightmare
.goto('https://www.google.co.nz/')
.evaluate(links => {
var allHrefs = document.querySelectorAll('#fbar #fsl a')
var allLinks = []
allHrefs.forEach(function(a) {
allLinks.push(a.href)
})
console.log('allLinks:', allLinks)
return allLinks
}, '.what')
.end()
.then(result => {
console.log('result:', result)
let titles = []
result.forEach(link => {
return nightmare
.goto(link)
.wait('#navheader')
.evaluate(getTitle => {
var thisTitle = document.title
console.log('this title:', thisTitle)
titles.push(thisTitle)
})
})
console.log('titles:', titles)
return titles
})
and here, the console results after running the script:
$ node scripts/nm_test.js
result: [ 'https://www.google.co.nz/intl/en/ads/?fg=1',
'https://www.google.co.nz/services/?fg=1',
'https://www.google.co.nz/intl/en/about.html?fg=1' ]
titles: []