From 858e3b4ffd08beb4a3a9496d933263e82cf6b2d8 Mon Sep 17 00:00:00 2001 From: Deep Koluguri Date: Sun, 28 Jun 2026 14:29:59 -0400 Subject: [PATCH] Isolate scraping logic to prevent crashes --- utility-agent/scraper.js | 83 +++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 44 deletions(-) diff --git a/utility-agent/scraper.js b/utility-agent/scraper.js index 5ea59e1..3cbac62 100644 --- a/utility-agent/scraper.js +++ b/utility-agent/scraper.js @@ -16,44 +16,40 @@ async function runScrapers(credentials) { args: ['--no-sandbox', '--disable-setuid-sandbox'] }); - try { - if (credentials.firstEnergy?.username && credentials.firstEnergy?.password) { - console.log('[Scraper] Scraping FirstEnergy...'); - const page = await browser.newPage(); - // Stealth settings to avoid basic bot detection + if (credentials.firstEnergy?.username && credentials.firstEnergy?.password) { + console.log('[Scraper] Scraping FirstEnergy...'); + const page = await browser.newPage(); + try { await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36'); await page.goto('https://www.firstenergycorp.com/log_in.html', { waitUntil: 'networkidle2' }); - // Wait for and fill login form - await page.waitForSelector('input[name="USER"]', { timeout: 15000 }); - await page.type('input[name="USER"]', credentials.firstEnergy.username); - await page.type('input[name="PASSWORD"]', credentials.firstEnergy.password); + await page.waitForSelector('input[name="USER"], input[name="username"], #username', { timeout: 15000 }); + await page.type('input[name="USER"], input[name="username"], #username', credentials.firstEnergy.username); + await page.type('input[name="PASSWORD"], input[name="password"], #password', credentials.firstEnergy.password); await Promise.all([ page.waitForNavigation({ waitUntil: 'networkidle2' }), page.click('button[type="submit"], input[type="submit"]') ]); - // Navigate to usage page or extract from dashboard console.log('[Scraper] Logged into FirstEnergy. Extracting usage...'); - // Note: These selectors are standard guesses and will likely need adjustment - // based on the actual DOM structure of the authenticated dashboard. - try { - await page.waitForSelector('.usage-summary, .kwh-value', { timeout: 10000 }); - const usage = await page.$eval('.kwh-value', el => el.innerText); - const cost = await page.$eval('.cost-value', el => el.innerText).catch(() => 'N/A'); - report += `💡 *FirstEnergy*: ${usage.trim()} kWh used today (~${cost.trim()})\n`; - } catch (err) { - console.warn('[Scraper] Could not find FirstEnergy usage selectors. Falling back to mock data for demonstration.'); - report += `💡 *FirstEnergy (Simulated)*: 24.5 kWh used today (~$3.15)\n`; - } - await page.close(); + await page.waitForSelector('.usage-summary, .kwh-value', { timeout: 10000 }); + const usage = await page.$eval('.kwh-value', el => el.innerText); + const cost = await page.$eval('.cost-value', el => el.innerText).catch(() => 'N/A'); + report += `💡 *FirstEnergy*: ${usage.trim()} kWh used today (~${cost.trim()})\n`; + } catch (err) { + console.warn('[Scraper] FirstEnergy scraping failed:', err.message); + report += `💡 *FirstEnergy (Simulated Fallback)*: 24.5 kWh used today (~$3.15) [Actual scrape failed]\n`; + } finally { + await page.close().catch(()=>null); } + } - if (credentials.peoplesGas?.username && credentials.peoplesGas?.password) { - console.log('[Scraper] Scraping Peoples Gas...'); - const page = await browser.newPage(); + if (credentials.peoplesGas?.username && credentials.peoplesGas?.password) { + console.log('[Scraper] Scraping Peoples Gas...'); + const page = await browser.newPage(); + try { await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36'); await page.goto('https://www.peoples-gas.com/login', { waitUntil: 'networkidle2' }); @@ -68,33 +64,32 @@ async function runScrapers(credentials) { ]); console.log('[Scraper] Logged into Peoples Gas. Extracting usage...'); - try { - await page.waitForSelector('.ccf-usage', { timeout: 10000 }); - const usage = await page.$eval('.ccf-usage', el => el.innerText); - const cost = await page.$eval('.ccf-cost', el => el.innerText).catch(() => 'N/A'); - report += `🔥 *Peoples Gas*: ${usage.trim()} CCF used today (~${cost.trim()})\n`; - } catch (err) { - console.warn('[Scraper] Could not find Peoples Gas usage selectors. Falling back to mock data for demonstration.'); - report += `🔥 *Peoples Gas (Simulated)*: 3.2 CCF used today (~$2.40)\n`; - } - await page.close(); + await page.waitForSelector('.ccf-usage', { timeout: 10000 }); + const usage = await page.$eval('.ccf-usage', el => el.innerText); + const cost = await page.$eval('.ccf-cost', el => el.innerText).catch(() => 'N/A'); + report += `🔥 *Peoples Gas*: ${usage.trim()} CCF used today (~${cost.trim()})\n`; + } catch (err) { + console.warn('[Scraper] Peoples Gas scraping failed:', err.message); + report += `🔥 *Peoples Gas (Simulated Fallback)*: 3.2 CCF used today (~$2.40) [Actual scrape failed]\n`; + } finally { + await page.close().catch(()=>null); } - - report += `\n_Stats pulled automatically by Utility Agent_`; - - console.log('[Scraper] Report generated, sending to WhatsApp...'); + } + + report += `\n_Stats pulled automatically by Utility Agent_`; + + console.log('[Scraper] Report generated, sending to WhatsApp...'); + try { await fetch(WHATSAPP_GATEWAY_URL, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ to: 'default', message: report }) }); console.log('[Scraper] Report sent successfully.'); - - } catch (e) { - console.error('[Scraper] Error during scraping:', e); - } finally { - await browser.close(); + } catch (err) { + console.error('[Scraper] Failed to send WhatsApp message:', err); } + await browser.close(); } module.exports = { runScrapers };