deepkoluguri
/
agentic-os
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add StealthPlugin and fix PeoplesGas selector

This commit is contained in:
Deep Koluguri 2026-06-28 21:12:16 -04:00
parent a77fe1216d
commit f136001d1b
2 changed files with 15 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
utility-agent/package.json
View File

@ -11,6 +11,8 @@
"dotenv": "^16.4.5", "dotenv": "^16.4.5",
"express": "^4.19.2", "express": "^4.19.2",
"node-cron": "^3.0.3", "node-cron": "^3.0.3",
"puppeteer": "^22.12.1" "puppeteer": "^22.12.1",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2"
} }
} }

22
utility-agent/scraper.js
View File

@ -1,5 +1,6 @@
const puppeteer = require('puppeteer'); const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
puppeteer.use(StealthPlugin());
const WHATSAPP_GATEWAY_URL = process.env.WHATSAPP_GATEWAY_URL || 'http://whatsapp-gateway.ai-agents.svc.cluster.local:5001/api/send-message'; const WHATSAPP_GATEWAY_URL = process.env.WHATSAPP_GATEWAY_URL || 'http://whatsapp-gateway.ai-agents.svc.cluster.local:5001/api/send-message';
async function runScrapers(credentials) { async function runScrapers(credentials) {
@ -22,25 +23,26 @@ async function runScrapers(credentials) {
try { try {
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36'); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36');
await page.goto('https://www.firstenergycorp.com/content/customer/log_in.html', { waitUntil: 'domcontentloaded', timeout: 60000 }); // Use networkidle2 so we wait for any Javascript redirects or B2C login loads
await page.goto('https://www.firstenergycorp.com/content/customer/log_in.html', { waitUntil: 'networkidle2', timeout: 60000 });
// Try to extract the direct URL instead of clicking, which is much more reliable // Try to extract the direct URL instead of clicking, which is much more reliable
await page.waitForSelector('a[href*="b2clogin.firstenergycorp.com"]', { timeout: 15000 }).catch(()=>null); await page.waitForSelector('a[href*="b2clogin.firstenergycorp.com"]', { timeout: 10000 }).catch(()=>null);
const b2cUrl = await page.evaluate(() => { const b2cUrl = await page.evaluate(() => {
const loginBtn = document.querySelector('a[href*="b2clogin.firstenergycorp.com"]'); const loginBtn = document.querySelector('a[href*="b2clogin"]');
return loginBtn ? loginBtn.href : null; return loginBtn ? loginBtn.href : null;
}); });
if (b2cUrl) { if (b2cUrl) {
await page.goto(b2cUrl, { waitUntil: 'domcontentloaded', timeout: 60000 }); await page.goto(b2cUrl, { waitUntil: 'networkidle2', timeout: 60000 });
} }
await page.waitForSelector('#signInName, input[name="signInName"]', { timeout: 30000 }); await page.waitForSelector('#signInName, input[name="signInName"], #username, input[name="username"]', { timeout: 30000 });
await page.type('#signInName, input[name="signInName"]', credentials.firstEnergy.username); await page.type('#signInName, input[name="signInName"], #username, input[name="username"]', credentials.firstEnergy.username);
await page.type('#password, input[name="password"]', credentials.firstEnergy.password); await page.type('#password, input[name="password"]', credentials.firstEnergy.password);
await Promise.all([ await Promise.all([
page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 60000 }), page.waitForNavigation({ waitUntil: 'networkidle2', timeout: 60000 }).catch(()=>null),
page.click('#next, button[type="submit"]') page.click('#next, button[type="submit"]')
]); ]);
@ -71,7 +73,7 @@ async function runScrapers(credentials) {
await Promise.all([ await Promise.all([
page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 60000 }), page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 60000 }),
page.click('button[type="submit"], .login-btn') page.click('#btnlogin')
]); ]);
console.log('[Scraper] Logged into Peoples Gas. Extracting usage...'); console.log('[Scraper] Logged into Peoples Gas. Extracting usage...');