How to store every network request into an array given the loop behaviour of request.continue()?

488 views Asked by At

I'm trying to get all the network requests when a page is accessed and store them into an array.

My code looks like this:

  await page.setRequestInterceptionEnabled(true);
  page.on('request', request => {
      if(request.url) {
          var networkRequests = request.url;
          var networkArray = [];
          for (var i = 0; i < networkRequests; i++) {
              networkArray.push(networkRequests[i]);
          }
          console.log(networkArray);
          console.log(typeof networkArray);
          request.continue();
      } else {
          request.abort();
      }

  });
  await page.goto('http://www.example.com', {waitUntil: 'networkidle'});

I find that the problem is with the request.continue(). It creates several iterations for each fetched request, and for each iteration it shows that request and returns it as string. That means that I end up with several strings.

The problem is that I couldn't managed to insert all those strings into one array, so I can make use of them lately. I tried several for loops but didn't succeed.

1

There are 1 answers

0
Carol-Theodor Pelu On BEST ANSWER

A quick fix has been found in the meantime:

const puppeteer = require('puppeteer');

function extractRequests(url) {
    return new Promise((resolve, reject) => {
        (async() => {
            const browser = await puppeteer.launch();
            const page = await browser.newPage();
            await page.setRequestInterceptionEnabled(true);
            let result = [];
            page.on('request', request => {
                if (request.url) {
                    var networkRequests = request.url;
                    result.push(networkRequests);
                    request.continue();
                } else {
                    request.abort();
                }

            });
            page.goto(url, {
                     waitUntil: 'networkidle'
                })
                .then( _=> setTimeout( _=> resolve(result), 1000));
        })();
    });
}

extractRequests('http://example.com').then(requests => {
    console.log(requests.filter(x => x.includes('event-name') && x.includes('other-event-name')));
    process.exit(0);
});