diff --git a/utility-agent/scraper.js b/utility-agent/scraper.js index e140cf8..4bc7411 100644 --- a/utility-agent/scraper.js +++ b/utility-agent/scraper.js @@ -26,23 +26,37 @@ async function runScrapers(credentials) { // Use networkidle2 so we wait for any Javascript redirects or B2C login loads await page.goto('https://www.firstenergycorp.com/content/customer/log_in.html', { waitUntil: 'networkidle2', timeout: 60000 }); - // Try to extract the direct URL instead of clicking, which is much more reliable - await page.waitForSelector('a[href*="b2clogin.firstenergycorp.com"]', { timeout: 10000 }).catch(()=>null); - const b2cUrl = await page.evaluate(() => { - const loginBtn = document.querySelector('a[href*="b2clogin"]'); - return loginBtn ? loginBtn.href : null; + // Dismiss cookie modal if present + try { + for (const frame of page.frames()) { + if (frame.url().includes('usercentrics')) { + await frame.evaluate(() => { + const btns = Array.from(document.querySelectorAll('button')); + const accept = btns.find(b => b.innerText && b.innerText.toUpperCase().includes('ACCEPT ALL')); + if (accept) accept.click(); + }); + } + } + } catch(e) {} + await new Promise(r => setTimeout(r, 2000)); + + // Force click the 'Log In' button by text + await page.evaluate(() => { + const els = Array.from(document.querySelectorAll('a, button, div.btn')); + // search backwards to find the most specific inner element + const loginBtn = els.reverse().find(e => e.innerText && e.innerText.trim() === 'Log In'); + if (loginBtn) loginBtn.click(); }); - if (b2cUrl) { - await page.goto(b2cUrl, { waitUntil: 'networkidle2', timeout: 60000 }); - } + // Wait for B2C navigation + await new Promise(r => setTimeout(r, 5000)); await page.waitForSelector('#signInName, input[name="signInName"], #username, input[name="username"]', { timeout: 30000 }); await page.type('#signInName, input[name="signInName"], #username, input[name="username"]', credentials.firstEnergy.username); await page.type('#password, input[name="password"]', credentials.firstEnergy.password); await Promise.all([ - page.waitForNavigation({ waitUntil: 'networkidle2', timeout: 60000 }).catch(()=>null), + page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 30000 }).catch(()=>null), page.click('#next, button[type="submit"]') ]); @@ -71,10 +85,7 @@ async function runScrapers(credentials) { await page.type('#txtLogin, input[name="txtLogin"]', credentials.peoplesGas.username); await page.type('#txtpwd, input[name="txtpwd"]', credentials.peoplesGas.password); - await Promise.all([ - page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 60000 }), - page.click('#btnlogin') - ]); + await page.click('#btnlogin'); console.log('[Scraper] Logged into Peoples Gas. Extracting usage...'); await page.waitForSelector('.ccf-usage', { timeout: 10000 });