deepkoluguri
/
agentic-os
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(utility-agent): Use APIs and robust text parsing for utility scrapers
Build Newspaper Extractor Backend / build-newspaper-backend (push) Failing after 29m27s Details
Build Newspaper Extractor Frontend / build-newspaper-frontend (push) Failing after 17s Details

This commit is contained in:
Deep Koluguri 2026-07-08 10:18:16 -04:00
parent 10656615d1
commit e4a7d07fdf
1 changed files with 50 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
utility-agent/scraper.js
View File

@ -61,14 +61,21 @@ async function runScrapers(credentials) {
]);
console.log('[Scraper] Logged into FirstEnergy. Extracting usage...');
await page.waitForSelector('.kwh, .usage-summary, .kwh-value', { timeout: 10000 });
const usage = await page.evaluate(() => {
const el = document.querySelector('.kwh, .usage-summary, .kwh-value');
if (!el) return 'N/A';
return (el.getAttribute('data-chart-value') || el.innerText).trim();
// Navigate to Usage History page
await page.goto('https://www.firstenergycorp.com/my_account/view_usage_history.html', { waitUntil: 'networkidle2', timeout: 60000 });
await new Promise(r => setTimeout(r, 5000));
// Extract the Current Month - $X text
const feData = await page.evaluate(() => {
const textDump = document.body.innerText || '';
const matchCurrent = textDump.match(/Current Month\s*-\s*(\$[0-9.]+)/i);
return {
currentMonthCost: matchCurrent ? matchCurrent[1] : 'N/A'
};
});
const cost = await page.$eval('.cost-value', el => el.innerText).catch(() => 'N/A');
report += `💡 *FirstEnergy*: ${usage} kWh used today (~${cost.trim()})\n`;
report += `💡 *FirstEnergy*: Bill: ${feData.currentMonthCost} (Current Month)\n`;
} catch (err) {
console.warn('[Scraper] FirstEnergy scraping failed:', err.message);
report += `💡 *FirstEnergy*: ❌ Error scraping data (${err.message})\n`;
@ -98,14 +105,43 @@ async function runScrapers(credentials) {
await page.click('#btnlogin');
console.log('[Scraper] Logged into Peoples Gas. Extracting usage...');
await page.waitForSelector('.ccf-usage', { timeout: 15000 }).catch(() => null);
const usage = await page.evaluate(() => {
const el = document.querySelector('.ccf-usage, #aGasUsageCCF');
return el ? el.innerText.replace('View in', '').trim() : 'N/A';
console.log('[Scraper] Logged into Peoples Gas. Extracting usage via API...');
// We are currently on dashboard. Extract Amount Due first.
await new Promise(r => setTimeout(r, 3000));
const pgCost = await page.evaluate(() => {
const text = document.body.innerText || '';
const match = text.match(/Please Pay\s*(\$[0-9.]+)/i);
return match ? match[1] : 'N/A';
});
const cost = await page.$eval('.cost-value', el => el.innerText).catch(() => 'N/A');
report += `🔥 *Peoples Gas*: ${usage} CCF used today (~${cost.trim()})\n`;
// Intercept LoadGasUsage API
const usagePromise = page.waitForResponse(response =>
response.url().includes('LoadGasUsage') && response.request().method() === 'POST',
{ timeout: 30000 }
).catch(() => null);
await page.goto('https://peopleseaccount.com/Portal/Usages.aspx?type=GU', { waitUntil: 'domcontentloaded', timeout: 60000 });
let pgUsage = 'N/A';
const apiResponse = await usagePromise;
if (apiResponse) {
try {
const json = await apiResponse.json();
if (json && json.d) {
const data = JSON.parse(json.d);
const usageArray = data.objUsageGenerationResultSetTwo;
if (usageArray && usageArray.length > 0) {
const lastEntry = usageArray[usageArray.length - 1];
pgUsage = `${lastEntry.UsageValue} MCF (ending ${lastEntry.ToDate})`;
}
}
} catch(e) {
console.error('[Scraper] PG API parsing error:', e);
}
}
report += `🔥 *Peoples Gas*: ${pgUsage} | Bill: ${pgCost}\n`;
} catch (err) {
console.warn('[Scraper] Peoples Gas scraping failed:', err.message);
report += `🔥 *Peoples Gas*: ❌ Error scraping data (${err.message})\n`;