I'm following a web scraping course that uses Cheerio. I practice on a different website then they use in the course and now I run into the problem that all my scraped text end up in one big object. But every title should end up in it's own object. Can someone see what I did wrong? I already bumbed my head 2 hours on this problem.

const request = require('request-promise');
const cheerio = require('cheerio');

const url = "https://huurgoed.nl/gehele-aanbod";

const scrapeResults = [];

async function scrapeHuurgoed() {
    try {
        const htmlResult = await request.get(url);
        const $ = await cheerio.load(htmlResult);

        $("div.aanbod").each((index, element) => {
            const result = $(element).children(".item");
            const title = result.find("h2").text().trim();
            const characteristics = result.find("h4").text();
            const scrapeResult =  {title, characteristics};
            scrapeResults.push(scrapeResult);
        });

        console.log(scrapeResults);
    } catch(err) {
        console.error(err);
    }
}

scrapeHuurgoed();

This is the link to the repo: https://github.com/danielkroon/huurgoed-scraper/blob/master/index.js Thanks!

1 Answers

1
SIM On Best Solutions

That is because of the way you used selectors. I've modified your script to fetch the content as you expected. Currently the script is collecting titles and characteristics. Feel free to add the rest within your script.

This is how you can get the required output:

const request = require('request-promise');
const cheerio = require('cheerio');

const url = "https://huurgoed.nl/gehele-aanbod";

const scrapeResults = [];

async function scrapeHuurgoed() {
    try {
        const htmlResult = await request.get(url);
        const $ = await cheerio.load(htmlResult);

        $("div.item").each((index, element) => {
            const title = $(element).find(".kenmerken > h2").text().trim();
            const characteristics = $(element).find("h4").text().trim();
            scrapeResults.push({title,characteristics});
        });
        console.log(scrapeResults);
    } catch(err) {
        console.error(err);
    }
}

scrapeHuurgoed();