I am trying to scrape some startups data of a site (bestsecret.at) with Puppeteer and when I try to navigate to the next page the Cloudflare waiting screen comes in and disrupts the scraper. Is there a way to bypass it with Puppeteer?
const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
const randomUseragent = require('random-useragent');
const userAgent = randomUseragent.getRandom();
puppeteer.use(StealthPlugin());
(async () => {
var argarr = ['--no-sandbox', '--disable-setuid-sandbox'];
const browser = await puppeteer.launch({args: argarr});
const page = await browser.newPage();
await page.setUserAgent(userAgent);
await page.goto('https://www.bestsecret.at', {waitUntil: 'networkidle2'});
const bodyWidth = await page.evaluate(() => document.body.scrollWidth);
const bodyHeight = await page.evaluate(() => document.body.scrollHeight);
await page.setViewport({ width: bodyWidth, height: bodyHeight });
await page.type('#login-username', 'USARENAME');
await page.type('#j_password', 'PASSWORD');
await page.waitForTimeout(1000);
await Promise.all([
page.click('#login-button'),
page.waitForNavigation({waitUntil: 'networkidle2'}),
]);
let bodyHTML = await page.content();
console.log(bodyHTML);
await browser.close();
})();