diff --git a/utility-agent/package.json b/utility-agent/package.json index 791c7a5..087f907 100644 --- a/utility-agent/package.json +++ b/utility-agent/package.json @@ -11,6 +11,8 @@ "dotenv": "^16.4.5", "express": "^4.19.2", "node-cron": "^3.0.3", - "puppeteer": "^22.12.1" + "puppeteer": "^22.12.1", + "puppeteer-extra": "^3.3.6", + "puppeteer-extra-plugin-stealth": "^2.11.2" } } diff --git a/utility-agent/scraper.js b/utility-agent/scraper.js index f1d1afd..e140cf8 100644 --- a/utility-agent/scraper.js +++ b/utility-agent/scraper.js @@ -1,5 +1,6 @@ -const puppeteer = require('puppeteer'); - +const puppeteer = require('puppeteer-extra'); +const StealthPlugin = require('puppeteer-extra-plugin-stealth'); +puppeteer.use(StealthPlugin()); const WHATSAPP_GATEWAY_URL = process.env.WHATSAPP_GATEWAY_URL || 'http://whatsapp-gateway.ai-agents.svc.cluster.local:5001/api/send-message'; async function runScrapers(credentials) { @@ -22,25 +23,26 @@ async function runScrapers(credentials) { try { await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36'); - await page.goto('https://www.firstenergycorp.com/content/customer/log_in.html', { waitUntil: 'domcontentloaded', timeout: 60000 }); + // Use networkidle2 so we wait for any Javascript redirects or B2C login loads + await page.goto('https://www.firstenergycorp.com/content/customer/log_in.html', { waitUntil: 'networkidle2', timeout: 60000 }); // Try to extract the direct URL instead of clicking, which is much more reliable - await page.waitForSelector('a[href*="b2clogin.firstenergycorp.com"]', { timeout: 15000 }).catch(()=>null); + await page.waitForSelector('a[href*="b2clogin.firstenergycorp.com"]', { timeout: 10000 }).catch(()=>null); const b2cUrl = await page.evaluate(() => { - const loginBtn = document.querySelector('a[href*="b2clogin.firstenergycorp.com"]'); + const loginBtn = document.querySelector('a[href*="b2clogin"]'); return loginBtn ? loginBtn.href : null; }); if (b2cUrl) { - await page.goto(b2cUrl, { waitUntil: 'domcontentloaded', timeout: 60000 }); + await page.goto(b2cUrl, { waitUntil: 'networkidle2', timeout: 60000 }); } - await page.waitForSelector('#signInName, input[name="signInName"]', { timeout: 30000 }); - await page.type('#signInName, input[name="signInName"]', credentials.firstEnergy.username); + await page.waitForSelector('#signInName, input[name="signInName"], #username, input[name="username"]', { timeout: 30000 }); + await page.type('#signInName, input[name="signInName"], #username, input[name="username"]', credentials.firstEnergy.username); await page.type('#password, input[name="password"]', credentials.firstEnergy.password); await Promise.all([ - page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 60000 }), + page.waitForNavigation({ waitUntil: 'networkidle2', timeout: 60000 }).catch(()=>null), page.click('#next, button[type="submit"]') ]); @@ -71,7 +73,7 @@ async function runScrapers(credentials) { await Promise.all([ page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 60000 }), - page.click('button[type="submit"], .login-btn') + page.click('#btnlogin') ]); console.log('[Scraper] Logged into Peoples Gas. Extracting usage...');