deepkoluguri
/
agentic-os
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Isolate scraping logic to prevent crashes

This commit is contained in:
Deep Koluguri 2026-06-28 14:29:59 -04:00
parent 23ef48647f
commit 858e3b4ffd
1 changed files with 39 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
utility-agent/scraper.js
View File

@ -16,44 +16,40 @@ async function runScrapers(credentials) {
args: ['--no-sandbox', '--disable-setuid-sandbox']
});
try {
if (credentials.firstEnergy?.username && credentials.firstEnergy?.password) {
console.log('[Scraper] Scraping FirstEnergy...');
const page = await browser.newPage();
// Stealth settings to avoid basic bot detection
if (credentials.firstEnergy?.username && credentials.firstEnergy?.password) {
console.log('[Scraper] Scraping FirstEnergy...');
const page = await browser.newPage();
try {
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36');
await page.goto('https://www.firstenergycorp.com/log_in.html', { waitUntil: 'networkidle2' });
// Wait for and fill login form
await page.waitForSelector('input[name="USER"]', { timeout: 15000 });
await page.type('input[name="USER"]', credentials.firstEnergy.username);
await page.type('input[name="PASSWORD"]', credentials.firstEnergy.password);
await page.waitForSelector('input[name="USER"], input[name="username"], #username', { timeout: 15000 });
await page.type('input[name="USER"], input[name="username"], #username', credentials.firstEnergy.username);
await page.type('input[name="PASSWORD"], input[name="password"], #password', credentials.firstEnergy.password);
await Promise.all([
page.waitForNavigation({ waitUntil: 'networkidle2' }),
page.click('button[type="submit"], input[type="submit"]')
]);
// Navigate to usage page or extract from dashboard
console.log('[Scraper] Logged into FirstEnergy. Extracting usage...');
// Note: These selectors are standard guesses and will likely need adjustment
// based on the actual DOM structure of the authenticated dashboard.
try {
await page.waitForSelector('.usage-summary, .kwh-value', { timeout: 10000 });
const usage = await page.$eval('.kwh-value', el => el.innerText);
const cost = await page.$eval('.cost-value', el => el.innerText).catch(() => 'N/A');
report += `💡 *FirstEnergy*: ${usage.trim()} kWh used today (~${cost.trim()})\n`;
} catch (err) {
console.warn('[Scraper] Could not find FirstEnergy usage selectors. Falling back to mock data for demonstration.');
report += `💡 *FirstEnergy (Simulated)*: 24.5 kWh used today (~$3.15)\n`;
}
await page.close();
await page.waitForSelector('.usage-summary, .kwh-value', { timeout: 10000 });
const usage = await page.$eval('.kwh-value', el => el.innerText);
const cost = await page.$eval('.cost-value', el => el.innerText).catch(() => 'N/A');
report += `💡 *FirstEnergy*: ${usage.trim()} kWh used today (~${cost.trim()})\n`;
} catch (err) {
console.warn('[Scraper] FirstEnergy scraping failed:', err.message);
report += `💡 *FirstEnergy (Simulated Fallback)*: 24.5 kWh used today (~$3.15) [Actual scrape failed]\n`;
} finally {
await page.close().catch(()=>null);
}
}
if (credentials.peoplesGas?.username && credentials.peoplesGas?.password) {
console.log('[Scraper] Scraping Peoples Gas...');
const page = await browser.newPage();
if (credentials.peoplesGas?.username && credentials.peoplesGas?.password) {
console.log('[Scraper] Scraping Peoples Gas...');
const page = await browser.newPage();
try {
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36');
await page.goto('https://www.peoples-gas.com/login', { waitUntil: 'networkidle2' });
@ -68,33 +64,32 @@ async function runScrapers(credentials) {
]);
console.log('[Scraper] Logged into Peoples Gas. Extracting usage...');
try {
await page.waitForSelector('.ccf-usage', { timeout: 10000 });
const usage = await page.$eval('.ccf-usage', el => el.innerText);
const cost = await page.$eval('.ccf-cost', el => el.innerText).catch(() => 'N/A');
report += `🔥 *Peoples Gas*: ${usage.trim()} CCF used today (~${cost.trim()})\n`;
} catch (err) {
console.warn('[Scraper] Could not find Peoples Gas usage selectors. Falling back to mock data for demonstration.');
report += `🔥 *Peoples Gas (Simulated)*: 3.2 CCF used today (~$2.40)\n`;
}
await page.close();
await page.waitForSelector('.ccf-usage', { timeout: 10000 });
const usage = await page.$eval('.ccf-usage', el => el.innerText);
const cost = await page.$eval('.ccf-cost', el => el.innerText).catch(() => 'N/A');
report += `🔥 *Peoples Gas*: ${usage.trim()} CCF used today (~${cost.trim()})\n`;
} catch (err) {
console.warn('[Scraper] Peoples Gas scraping failed:', err.message);
report += `🔥 *Peoples Gas (Simulated Fallback)*: 3.2 CCF used today (~$2.40) [Actual scrape failed]\n`;
} finally {
await page.close().catch(()=>null);
}
}
report += `\n_Stats pulled automatically by Utility Agent_`;
report += `\n_Stats pulled automatically by Utility Agent_`;
console.log('[Scraper] Report generated, sending to WhatsApp...');
console.log('[Scraper] Report generated, sending to WhatsApp...');
try {
await fetch(WHATSAPP_GATEWAY_URL, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ to: 'default', message: report })
});
console.log('[Scraper] Report sent successfully.');
} catch (e) {
console.error('[Scraper] Error during scraping:', e);
} finally {
await browser.close();
} catch (err) {
console.error('[Scraper] Failed to send WhatsApp message:', err);
}
await browser.close();
}
module.exports = { runScrapers };