deepkoluguri
/
agentic-os
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Isolate scraping logic to prevent crashes

This commit is contained in:
Deep Koluguri 2026-06-28 14:29:59 -04:00
parent 23ef48647f
commit 858e3b4ffd
1 changed files with 39 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
utility-agent/scraper.js
View File

@ -16,44 +16,40 @@ async function runScrapers(credentials) {
args: ['--no-sandbox', '--disable-setuid-sandbox'] args: ['--no-sandbox', '--disable-setuid-sandbox']
}); });
try { if (credentials.firstEnergy?.username && credentials.firstEnergy?.password) {
if (credentials.firstEnergy?.username && credentials.firstEnergy?.password) { console.log('[Scraper] Scraping FirstEnergy...');
console.log('[Scraper] Scraping FirstEnergy...'); const page = await browser.newPage();
const page = await browser.newPage(); try {
// Stealth settings to avoid basic bot detection
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36'); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36');
await page.goto('https://www.firstenergycorp.com/log_in.html', { waitUntil: 'networkidle2' }); await page.goto('https://www.firstenergycorp.com/log_in.html', { waitUntil: 'networkidle2' });
// Wait for and fill login form await page.waitForSelector('input[name="USER"], input[name="username"], #username', { timeout: 15000 });
await page.waitForSelector('input[name="USER"]', { timeout: 15000 }); await page.type('input[name="USER"], input[name="username"], #username', credentials.firstEnergy.username);
await page.type('input[name="USER"]', credentials.firstEnergy.username); await page.type('input[name="PASSWORD"], input[name="password"], #password', credentials.firstEnergy.password);
await page.type('input[name="PASSWORD"]', credentials.firstEnergy.password);
await Promise.all([ await Promise.all([
page.waitForNavigation({ waitUntil: 'networkidle2' }), page.waitForNavigation({ waitUntil: 'networkidle2' }),
page.click('button[type="submit"], input[type="submit"]') page.click('button[type="submit"], input[type="submit"]')
]); ]);
// Navigate to usage page or extract from dashboard
console.log('[Scraper] Logged into FirstEnergy. Extracting usage...'); console.log('[Scraper] Logged into FirstEnergy. Extracting usage...');
// Note: These selectors are standard guesses and will likely need adjustment await page.waitForSelector('.usage-summary, .kwh-value', { timeout: 10000 });
// based on the actual DOM structure of the authenticated dashboard. const usage = await page.$eval('.kwh-value', el => el.innerText);
try { const cost = await page.$eval('.cost-value', el => el.innerText).catch(() => 'N/A');
await page.waitForSelector('.usage-summary, .kwh-value', { timeout: 10000 }); report += `💡 *FirstEnergy*: ${usage.trim()} kWh used today (~${cost.trim()})\n`;
const usage = await page.$eval('.kwh-value', el => el.innerText); } catch (err) {
const cost = await page.$eval('.cost-value', el => el.innerText).catch(() => 'N/A'); console.warn('[Scraper] FirstEnergy scraping failed:', err.message);
report += `💡 *FirstEnergy*: ${usage.trim()} kWh used today (~${cost.trim()})\n`; report += `💡 *FirstEnergy (Simulated Fallback)*: 24.5 kWh used today (~$3.15) [Actual scrape failed]\n`;
} catch (err) { } finally {
console.warn('[Scraper] Could not find FirstEnergy usage selectors. Falling back to mock data for demonstration.'); await page.close().catch(()=>null);
report += `💡 *FirstEnergy (Simulated)*: 24.5 kWh used today (~$3.15)\n`;
}
await page.close();
} }
}
if (credentials.peoplesGas?.username && credentials.peoplesGas?.password) { if (credentials.peoplesGas?.username && credentials.peoplesGas?.password) {
console.log('[Scraper] Scraping Peoples Gas...'); console.log('[Scraper] Scraping Peoples Gas...');
const page = await browser.newPage(); const page = await browser.newPage();
try {
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36'); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36');
await page.goto('https://www.peoples-gas.com/login', { waitUntil: 'networkidle2' }); await page.goto('https://www.peoples-gas.com/login', { waitUntil: 'networkidle2' });
@ -68,33 +64,32 @@ async function runScrapers(credentials) {
]); ]);
console.log('[Scraper] Logged into Peoples Gas. Extracting usage...'); console.log('[Scraper] Logged into Peoples Gas. Extracting usage...');
try { await page.waitForSelector('.ccf-usage', { timeout: 10000 });
await page.waitForSelector('.ccf-usage', { timeout: 10000 }); const usage = await page.$eval('.ccf-usage', el => el.innerText);
const usage = await page.$eval('.ccf-usage', el => el.innerText); const cost = await page.$eval('.ccf-cost', el => el.innerText).catch(() => 'N/A');
const cost = await page.$eval('.ccf-cost', el => el.innerText).catch(() => 'N/A'); report += `🔥 *Peoples Gas*: ${usage.trim()} CCF used today (~${cost.trim()})\n`;
report += `🔥 *Peoples Gas*: ${usage.trim()} CCF used today (~${cost.trim()})\n`; } catch (err) {
} catch (err) { console.warn('[Scraper] Peoples Gas scraping failed:', err.message);
console.warn('[Scraper] Could not find Peoples Gas usage selectors. Falling back to mock data for demonstration.'); report += `🔥 *Peoples Gas (Simulated Fallback)*: 3.2 CCF used today (~$2.40) [Actual scrape failed]\n`;
report += `🔥 *Peoples Gas (Simulated)*: 3.2 CCF used today (~$2.40)\n`; } finally {
} await page.close().catch(()=>null);
await page.close();
} }
}
report += `\n_Stats pulled automatically by Utility Agent_`;
report += `\n_Stats pulled automatically by Utility Agent_`;
console.log('[Scraper] Report generated, sending to WhatsApp...');
console.log('[Scraper] Report generated, sending to WhatsApp...');
try {
await fetch(WHATSAPP_GATEWAY_URL, { await fetch(WHATSAPP_GATEWAY_URL, {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ to: 'default', message: report }) body: JSON.stringify({ to: 'default', message: report })
}); });
console.log('[Scraper] Report sent successfully.'); console.log('[Scraper] Report sent successfully.');
} catch (err) {
} catch (e) { console.error('[Scraper] Failed to send WhatsApp message:', err);
console.error('[Scraper] Error during scraping:', e);
} finally {
await browser.close();
} }
await browser.close();
} }
module.exports = { runScrapers }; module.exports = { runScrapers };