agentic-os/utility-agent/scraper.js

const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
puppeteer.use(StealthPlugin());
const WHATSAPP_GATEWAY_URL = process.env.WHATSAPP_GATEWAY_URL || 'http://whatsapp-gateway.ai-agents.svc.cluster.local:5001/api/send-message';

async function runScrapers(credentials) {
    if (!credentials.firstEnergy && !credentials.peoplesGas) {
        console.log('[Scraper] No credentials configured. Skipping run.');
        return;
    }

    let report = "⚡ *Daily Utility Usage Report*\n\n";

    const browser = await puppeteer.launch({
        headless: 'new',
        executablePath: '/home/pptruser/.cache/puppeteer/chrome/linux-150.0.7871.24/chrome-linux64/chrome',
        args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-blink-features=AutomationControlled']
    });

    if (credentials.firstEnergy?.username && credentials.firstEnergy?.password) {
        console.log('[Scraper] Scraping FirstEnergy...');
        const page = await browser.newPage();
        try {
            await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36');
            
            // Use networkidle2 so we wait for any Javascript redirects or B2C login loads
            await page.goto('https://www.firstenergycorp.com/content/customer/log_in.html', { waitUntil: 'networkidle2', timeout: 60000 });
            
            // Dismiss cookie modal if present
            try {
                for (const frame of page.frames()) {
                    if (frame.url().includes('usercentrics')) {
                        await frame.evaluate(() => {
                            const btns = Array.from(document.querySelectorAll('button'));
                            const accept = btns.find(b => b.innerText && b.innerText.toUpperCase().includes('ACCEPT ALL'));
                            if (accept) accept.click();
                        });
                    }
                }
            } catch(e) {}
            await new Promise(r => setTimeout(r, 2000));
            
            // Force click the 'Log In' button by text
            await page.evaluate(() => {
                const els = Array.from(document.querySelectorAll('a, button, div.btn'));
                // search backwards to find the most specific inner element
                const loginBtn = els.reverse().find(e => e.innerText && e.innerText.trim() === 'Log In');
                if (loginBtn) loginBtn.click();
            });
            
            // Wait for B2C navigation
            await new Promise(r => setTimeout(r, 5000));
            
            await page.waitForSelector('#signInName, input[name="signInName"], #username, input[name="username"]', { timeout: 30000 });
            await page.type('#signInName, input[name="signInName"], #username, input[name="username"]', credentials.firstEnergy.username);
            await page.type('#password, input[name="password"]', credentials.firstEnergy.password);
            
            await Promise.all([
                page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 30000 }).catch(()=>null),
                page.click('#next, button[type="submit"]')
            ]);
            
            console.log('[Scraper] Logged into FirstEnergy. Extracting usage...');
            
            // Navigate to Usage History page
            await page.goto('https://www.firstenergycorp.com/my_account/view_usage_history.html', { waitUntil: 'networkidle2', timeout: 60000 });
            await new Promise(r => setTimeout(r, 5000));
            
            // Extract the Current Month - $X text
            const feData = await page.evaluate(() => {
                const textDump = document.body.innerText || '';
                const matchCurrent = textDump.match(/Current Month\s*-\s*(\$[0-9.]+)/i);
                return {
                    currentMonthCost: matchCurrent ? matchCurrent[1] : 'N/A'
                };
            });
            
            report += `💡 *FirstEnergy*: Bill: ${feData.currentMonthCost} (Current Month)\n`;
        } catch (err) {
            console.warn('[Scraper] FirstEnergy scraping failed:', err.message);
            report += `💡 *FirstEnergy*: ❌ Error scraping data (${err.message})\n`;
        } finally {
            await page.close().catch(()=>null);
        }
    }

    if (credentials.peoplesGas?.username && credentials.peoplesGas?.password) {
        console.log('[Scraper] Scraping Peoples Gas...');
        const page = await browser.newPage();
        try {
            await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36');
            
            await page.goto('https://peopleseaccount.com/Portal/default.aspx', { waitUntil: 'domcontentloaded', timeout: 60000 });
            
            await page.waitForSelector('#txtLogin', { timeout: 30000 });
            
            // Clear fields first and type cleanly
            await page.evaluate(() => {
                document.querySelector('#txtLogin').value = '';
                document.querySelector('#txtpwd').value = '';
            });
            
            await page.type('#txtLogin', credentials.peoplesGas.username);
            await page.type('#txtpwd', credentials.peoplesGas.password);
            
            await Promise.all([
                page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 30000 }).catch(()=>null),
                page.click('#btnlogin')
            ]);
            
            console.log('[Scraper] Logged into Peoples Gas. Extracting usage via API...');
            
            // We are currently on dashboard. Extract Amount Due first.
            await new Promise(r => setTimeout(r, 3000));
            const pgCost = await page.evaluate(() => {
                const text = document.body.innerText || '';
                const match = text.match(/Please Pay\s*(\$[0-9.]+)/i);
                return match ? match[1] : 'N/A';
            });
            
            // Intercept LoadGasUsage API
            const usagePromise = page.waitForResponse(response => 
                response.url().includes('LoadGasUsage') && response.request().method() === 'POST',
                { timeout: 30000 }
            ).catch(() => null);
            
            await page.goto('https://peopleseaccount.com/Portal/Usages.aspx?type=GU', { waitUntil: 'domcontentloaded', timeout: 60000 }).catch(e => console.log('[Scraper] PG goto error ignored:', e.message));
            
            let pgUsage = 'N/A';
            const apiResponse = await usagePromise;
            if (apiResponse) {
                try {
                    const json = await apiResponse.json();
                    if (json && json.d) {
                        const data = JSON.parse(json.d);
                        const usageArray = data.objUsageGenerationResultSetTwo;
                        if (usageArray && usageArray.length > 0) {
                            const lastEntry = usageArray[usageArray.length - 1];
                            pgUsage = `${lastEntry.UsageValue} MCF (ending ${lastEntry.ToDate})`;
                        }
                    }
                } catch(e) {
                    console.error('[Scraper] PG API parsing error:', e);
                }
            }
            
            report += `🔥 *Peoples Gas*: ${pgUsage} | Bill: ${pgCost}\n`;
        } catch (err) {
            console.warn('[Scraper] Peoples Gas scraping failed:', err.message);
            report += `🔥 *Peoples Gas*: ❌ Error scraping data (${err.message})\n`;
        } finally {
            await page.close().catch(()=>null);
        }
    }
    
    report += `\n_Stats pulled automatically by Utility Agent_`;
    
    console.log('[Scraper] Report generated:');
    console.log(report);
    
    console.log('[Scraper] Sending to WhatsApp...');
    try {
        await fetch(WHATSAPP_GATEWAY_URL, {
            method: 'POST',
            headers: { 'Content-Type': 'application/json' },
            body: JSON.stringify({ to: 'default', message: report })
        });
        console.log('[Scraper] Report sent successfully.');
    } catch (err) {
        console.error('[Scraper] Failed to send WhatsApp message:', err);
    }
    await browser.close();
}

module.exports = { runScrapers };