deepkoluguri
/
agentic-os
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Fix puppeteer bot detection and timeout issues

This commit is contained in:
Deep Koluguri 2026-06-28 20:50:41 -04:00
parent 25ecda9cb7
commit a77fe1216d
1 changed files with 14 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
utility-agent/scraper.js
View File

@ -13,7 +13,7 @@ async function runScrapers(credentials) {
const browser = await puppeteer.launch({ const browser = await puppeteer.launch({
headless: 'new', headless: 'new',
executablePath: '/home/pptruser/.cache/puppeteer/chrome/linux-150.0.7871.24/chrome-linux64/chrome', executablePath: '/home/pptruser/.cache/puppeteer/chrome/linux-150.0.7871.24/chrome-linux64/chrome',
args: ['--no-sandbox', '--disable-setuid-sandbox'] args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-blink-features=AutomationControlled']
}); });
if (credentials.firstEnergy?.username && credentials.firstEnergy?.password) { if (credentials.firstEnergy?.username && credentials.firstEnergy?.password) {
@ -22,24 +22,25 @@ async function runScrapers(credentials) {
try { try {
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36'); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36');
await page.goto('https://www.firstenergycorp.com/content/customer/log_in.html', { waitUntil: 'networkidle2' }); await page.goto('https://www.firstenergycorp.com/content/customer/log_in.html', { waitUntil: 'domcontentloaded', timeout: 60000 });
// Wait for the login button to appear and click it, which triggers the B2C redirect // Try to extract the direct URL instead of clicking, which is much more reliable
await page.waitForSelector('a[href*="b2clogin.firstenergycorp.com"]', { timeout: 10000 }).catch(()=>null); await page.waitForSelector('a[href*="b2clogin.firstenergycorp.com"]', { timeout: 15000 }).catch(()=>null);
await page.evaluate(() => { const b2cUrl = await page.evaluate(() => {
const loginBtn = document.querySelector('a[href*="b2clogin.firstenergycorp.com"]'); const loginBtn = document.querySelector('a[href*="b2clogin.firstenergycorp.com"]');
if (loginBtn) loginBtn.click(); return loginBtn ? loginBtn.href : null;
}); });
// Wait for navigation to the B2C login page if (b2cUrl) {
await page.waitForNavigation({ waitUntil: 'networkidle0', timeout: 15000 }).catch(()=>null); await page.goto(b2cUrl, { waitUntil: 'domcontentloaded', timeout: 60000 });
}
await page.waitForSelector('#signInName, input[name="signInName"]', { timeout: 15000 }); await page.waitForSelector('#signInName, input[name="signInName"]', { timeout: 30000 });
await page.type('#signInName, input[name="signInName"]', credentials.firstEnergy.username); await page.type('#signInName, input[name="signInName"]', credentials.firstEnergy.username);
await page.type('#password, input[name="password"]', credentials.firstEnergy.password); await page.type('#password, input[name="password"]', credentials.firstEnergy.password);
await Promise.all([ await Promise.all([
page.waitForNavigation({ waitUntil: 'networkidle2' }), page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 60000 }),
page.click('#next, button[type="submit"]') page.click('#next, button[type="submit"]')
]); ]);
@ -62,14 +63,14 @@ async function runScrapers(credentials) {
try { try {
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36'); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36');
await page.goto('https://peopleseaccount.com/Portal/default.aspx', { waitUntil: 'networkidle2' }); await page.goto('https://peopleseaccount.com/Portal/default.aspx', { waitUntil: 'domcontentloaded', timeout: 60000 });
await page.waitForSelector('#txtLogin, input[name="txtLogin"]', { timeout: 15000 }); await page.waitForSelector('#txtLogin, input[name="txtLogin"]', { timeout: 30000 });
await page.type('#txtLogin, input[name="txtLogin"]', credentials.peoplesGas.username); await page.type('#txtLogin, input[name="txtLogin"]', credentials.peoplesGas.username);
await page.type('#txtpwd, input[name="txtpwd"]', credentials.peoplesGas.password); await page.type('#txtpwd, input[name="txtpwd"]', credentials.peoplesGas.password);
await Promise.all([ await Promise.all([
page.waitForNavigation({ waitUntil: 'networkidle2' }), page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 60000 }),
page.click('button[type="submit"], .login-btn') page.click('button[type="submit"], .login-btn')
]); ]);