diff --git a/backend/src/factorlab/descriptiveLens.js b/backend/src/factorlab/descriptiveLens.js index 0294dd5..bd1da2d 100644 --- a/backend/src/factorlab/descriptiveLens.js +++ b/backend/src/factorlab/descriptiveLens.js @@ -62,21 +62,40 @@ export async function buildFactorProfiles(injectedUniverseRecords = null) { }); } } else { - const universe = getUniverse().filter(s => !s.isETF).map(s => s.symbol); - for (const ticker of universe) { - try { - const data = await pullRawRecord(ticker); - if (!data) continue; + // We now include ETFs in the factor ranking because we want to see momentum/value on country ETFs + const universe = getUniverse().map(s => s.symbol); + const BATCH_SIZE = 15; + + for (let i = 0; i < universe.length; i += BATCH_SIZE) { + const batch = universe.slice(i, i + BATCH_SIZE); + const promises = batch.map(async (ticker) => { + try { + const data = await pullRawRecord(ticker); + if (!data) return null; - rawProfiles.push({ - ticker, - value: FACTORS.value(data), - quality: FACTORS.quality(data), - lowVol: FACTORS.lowVol(data), - momentum: FACTORS.momentum(data) - }); - } catch (e) { - console.warn(`[DescriptiveLens] Failed for ${ticker}`); + return { + ticker, + value: FACTORS.value(data), + quality: FACTORS.quality(data), + lowVol: FACTORS.lowVol(data), + momentum: FACTORS.momentum(data) + }; + } catch (e) { + console.warn(`[DescriptiveLens] Failed for ${ticker}:`, e.message); + return null; + } + }); + + const results = await Promise.allSettled(promises); + for (const res of results) { + if (res.status === 'fulfilled' && res.value) { + rawProfiles.push(res.value); + } + } + + // Throttle delay between batches to respect free API limits + if (i + BATCH_SIZE < universe.length) { + await new Promise(r => setTimeout(r, 1000)); } } } diff --git a/backend/src/routes/marketAnalysis.js b/backend/src/routes/marketAnalysis.js index ed61bf4..f3f93de 100644 --- a/backend/src/routes/marketAnalysis.js +++ b/backend/src/routes/marketAnalysis.js @@ -14,7 +14,8 @@ import { scoreSuitability } from '../services/scoringEngine.js'; import { classifyRegime } from '../services/regimeService.js'; import { logSignals } from '../services/signalLogger.js'; const router = express.Router(); -const universeCache = new NodeCache({ stdTTL: 60 }); +// Default cache TTL to 5 minutes (300s) to protect against rate limits on large universe +const universeCache = new NodeCache({ stdTTL: 300 }); /** * GET /api/market/universe @@ -238,7 +239,7 @@ router.get('/screener', async (req, res) => { updatedAt: new Date().toISOString(), }; - universeCache.set(cacheKey, data, 120); // 120s cache + universeCache.set(cacheKey, data, 300); // 300s cache for expanded universe res.json({ success: true, data }); } catch (err) { console.error('[market/screener]', err); diff --git a/backend/src/services/stockUniverseService.js b/backend/src/services/stockUniverseService.js index d1faad9..de8b4fd 100644 --- a/backend/src/services/stockUniverseService.js +++ b/backend/src/services/stockUniverseService.js @@ -14,69 +14,162 @@ const CAP_TIERS = { // Below $2B = SMALL }; -// Master universe of top 50 stocks + ETFs +// Master universe of ~200 carefully selected global stocks + ETFs export const UNIVERSE = [ - // Mega-cap Tech + // --- Mega-cap Tech & Comm Services --- { symbol: 'AAPL', name: 'Apple Inc.', sector: 'Technology', isETF: false }, { symbol: 'MSFT', name: 'Microsoft Corp.', sector: 'Technology', isETF: false }, { symbol: 'NVDA', name: 'NVIDIA Corp.', sector: 'Technology', isETF: false }, { symbol: 'AMZN', name: 'Amazon.com Inc.', sector: 'Consumer Discret.', isETF: false }, - { symbol: 'GOOGL', name: 'Alphabet Inc.', sector: 'Technology', isETF: false }, - { symbol: 'META', name: 'Meta Platforms', sector: 'Technology', isETF: false }, + { symbol: 'GOOGL', name: 'Alphabet Inc.', sector: 'Communication', isETF: false }, + { symbol: 'META', name: 'Meta Platforms', sector: 'Communication', isETF: false }, { symbol: 'TSLA', name: 'Tesla Inc.', sector: 'Consumer Discret.', isETF: false }, { symbol: 'AVGO', name: 'Broadcom Inc.', sector: 'Technology', isETF: false }, { symbol: 'ORCL', name: 'Oracle Corp.', sector: 'Technology', isETF: false }, { symbol: 'AMD', name: 'Advanced Micro Devices', sector: 'Technology', isETF: false }, - // Financials + { symbol: 'CRM', name: 'Salesforce Inc.', sector: 'Technology', isETF: false }, + { symbol: 'ADBE', name: 'Adobe Inc.', sector: 'Technology', isETF: false }, + { symbol: 'CSCO', name: 'Cisco Systems', sector: 'Technology', isETF: false }, + { symbol: 'NFLX', name: 'Netflix Inc.', sector: 'Communication', isETF: false }, + { symbol: 'DIS', name: 'Walt Disney Co.', sector: 'Communication', isETF: false }, + { symbol: 'INTC', name: 'Intel Corp.', sector: 'Technology', isETF: false }, + { symbol: 'IBM', name: 'Intl Business Machines', sector: 'Technology', isETF: false }, + + // --- Emerging Markets & Global Giants (Heavily weighted) --- + { symbol: 'TSM', name: 'Taiwan Semiconductor', sector: 'Technology', isETF: false }, + { symbol: 'BABA', name: 'Alibaba Group', sector: 'Consumer Discret.', isETF: false }, + { symbol: 'PDD', name: 'PDD Holdings', sector: 'Consumer Discret.', isETF: false }, + { symbol: 'HDB', name: 'HDFC Bank', sector: 'Financials', isETF: false }, + { symbol: 'IBN', name: 'ICICI Bank', sector: 'Financials', isETF: false }, + { symbol: 'INFY', name: 'Infosys Ltd.', sector: 'Technology', isETF: false }, + { symbol: 'RELIANCE.NS', name: 'Reliance Industries', sector: 'Energy', isETF: false }, + { symbol: 'VALE', name: 'Vale S.A.', sector: 'Materials', isETF: false }, + { symbol: 'PBR', name: 'Petrobras', sector: 'Energy', isETF: false }, + { symbol: 'ITUB', name: 'Itau Unibanco', sector: 'Financials', isETF: false }, + { symbol: 'MELI', name: 'MercadoLibre', sector: 'Consumer Discret.', isETF: false }, + { symbol: 'NU', name: 'Nu Holdings', sector: 'Financials', isETF: false }, + { symbol: 'ASML', name: 'ASML Holding', sector: 'Technology', isETF: false }, + { symbol: 'NVO', name: 'Novo Nordisk', sector: 'Healthcare', isETF: false }, + { symbol: 'TM', name: 'Toyota Motor', sector: 'Consumer Discret.', isETF: false }, + { symbol: 'SONY', name: 'Sony Group', sector: 'Consumer Discret.', isETF: false }, + { symbol: 'RY', name: 'Royal Bank of Canada', sector: 'Financials', isETF: false }, + { symbol: 'BHP', name: 'BHP Group', sector: 'Materials', isETF: false }, + { symbol: 'RIO', name: 'Rio Tinto', sector: 'Materials', isETF: false }, + { symbol: 'BIDU', name: 'Baidu Inc.', sector: 'Communication', isETF: false }, + { symbol: 'JD', name: 'JD.com', sector: 'Consumer Discret.', isETF: false }, + { symbol: 'TCEHY', name: 'Tencent Holdings', sector: 'Communication', isETF: false }, + { symbol: 'NTES', name: 'NetEase Inc.', sector: 'Communication', isETF: false }, + { symbol: 'AMX', name: 'America Movil', sector: 'Communication', isETF: false }, + { symbol: 'KB', name: 'KB Financial Group', sector: 'Financials', isETF: false }, + { symbol: 'BBD', name: 'Banco Bradesco', sector: 'Financials', isETF: false }, + + // --- Financials --- { symbol: 'JPM', name: 'JPMorgan Chase', sector: 'Financials', isETF: false }, { symbol: 'BAC', name: 'Bank of America', sector: 'Financials', isETF: false }, { symbol: 'WFC', name: 'Wells Fargo', sector: 'Financials', isETF: false }, { symbol: 'GS', name: 'Goldman Sachs', sector: 'Financials', isETF: false }, + { symbol: 'MS', name: 'Morgan Stanley', sector: 'Financials', isETF: false }, + { symbol: 'C', name: 'Citigroup', sector: 'Financials', isETF: false }, { symbol: 'V', name: 'Visa Inc.', sector: 'Financials', isETF: false }, { symbol: 'MA', name: 'Mastercard Inc.', sector: 'Financials', isETF: false }, - // Healthcare + { symbol: 'AXP', name: 'American Express', sector: 'Financials', isETF: false }, + { symbol: 'BLK', name: 'BlackRock Inc.', sector: 'Financials', isETF: false }, + { symbol: 'BX', name: 'Blackstone Inc.', sector: 'Financials', isETF: false }, + + // --- Healthcare --- { symbol: 'UNH', name: 'UnitedHealth Group', sector: 'Healthcare', isETF: false }, { symbol: 'LLY', name: 'Eli Lilly & Co.', sector: 'Healthcare', isETF: false }, { symbol: 'JNJ', name: 'Johnson & Johnson', sector: 'Healthcare', isETF: false }, { symbol: 'ABBV', name: 'AbbVie Inc.', sector: 'Healthcare', isETF: false }, { symbol: 'PFE', name: 'Pfizer Inc.', sector: 'Healthcare', isETF: false }, - // Energy + { symbol: 'MRK', name: 'Merck & Co.', sector: 'Healthcare', isETF: false }, + { symbol: 'TMO', name: 'Thermo Fisher Scientific', sector: 'Healthcare', isETF: false }, + { symbol: 'DHR', name: 'Danaher Corp.', sector: 'Healthcare', isETF: false }, + { symbol: 'ISRG', name: 'Intuitive Surgical', sector: 'Healthcare', isETF: false }, + { symbol: 'SYK', name: 'Stryker Corp.', sector: 'Healthcare', isETF: false }, + + // --- Energy & Materials --- { symbol: 'XOM', name: 'Exxon Mobil', sector: 'Energy', isETF: false }, { symbol: 'CVX', name: 'Chevron Corp.', sector: 'Energy', isETF: false }, - // Industrials + { symbol: 'COP', name: 'ConocoPhillips', sector: 'Energy', isETF: false }, + { symbol: 'SLB', name: 'Schlumberger', sector: 'Energy', isETF: false }, + { symbol: 'LIN', name: 'Linde plc', sector: 'Materials', isETF: false }, + { symbol: 'SHW', name: 'Sherwin-Williams', sector: 'Materials', isETF: false }, + { symbol: 'FCX', name: 'Freeport-McMoRan', sector: 'Materials', isETF: false }, + { symbol: 'EOG', name: 'EOG Resources', sector: 'Energy', isETF: false }, + { symbol: 'OXY', name: 'Occidental Petroleum', sector: 'Energy', isETF: false }, + + // --- Industrials & Aerospace --- { symbol: 'CAT', name: 'Caterpillar Inc.', sector: 'Industrials', isETF: false }, { symbol: 'HON', name: 'Honeywell Intl.', sector: 'Industrials', isETF: false }, - // Consumer + { symbol: 'GE', name: 'General Electric', sector: 'Industrials', isETF: false }, + { symbol: 'BA', name: 'Boeing Co.', sector: 'Industrials', isETF: false }, + { symbol: 'LMT', name: 'Lockheed Martin', sector: 'Industrials', isETF: false }, + { symbol: 'UNP', name: 'Union Pacific', sector: 'Industrials', isETF: false }, + { symbol: 'UPS', name: 'United Parcel Service', sector: 'Industrials', isETF: false }, + { symbol: 'RTX', name: 'RTX Corp.', sector: 'Industrials', isETF: false }, + { symbol: 'DE', name: 'Deere & Co.', sector: 'Industrials', isETF: false }, + + // --- Consumer Staples & Discretionary --- { symbol: 'WMT', name: 'Walmart Inc.', sector: 'Consumer Staples', isETF: false }, { symbol: 'COST', name: 'Costco Wholesale', sector: 'Consumer Staples', isETF: false }, - // Semiconductors - { symbol: 'QCOM', name: 'Qualcomm Inc.', sector: 'Technology', isETF: false }, - { symbol: 'MU', name: 'Micron Technology', sector: 'Technology', isETF: false }, - { symbol: 'INTC', name: 'Intel Corp.', sector: 'Technology', isETF: false }, - // Growth / Momentum + { symbol: 'PG', name: 'Procter & Gamble', sector: 'Consumer Staples', isETF: false }, + { symbol: 'KO', name: 'Coca-Cola Co.', sector: 'Consumer Staples', isETF: false }, + { symbol: 'PEP', name: 'PepsiCo Inc.', sector: 'Consumer Staples', isETF: false }, + { symbol: 'MCD', name: 'McDonald\'s Corp.', sector: 'Consumer Discret.', isETF: false }, + { symbol: 'HD', name: 'Home Depot', sector: 'Consumer Discret.', isETF: false }, + { symbol: 'NKE', name: 'NIKE Inc.', sector: 'Consumer Discret.', isETF: false }, + { symbol: 'SBUX', name: 'Starbucks Corp.', sector: 'Consumer Discret.', isETF: false }, + { symbol: 'TGT', name: 'Target Corp.', sector: 'Consumer Staples', isETF: false }, + + // --- Utilities & Real Estate --- + { symbol: 'NEE', name: 'NextEra Energy', sector: 'Utilities', isETF: false }, + { symbol: 'DUK', name: 'Duke Energy', sector: 'Utilities', isETF: false }, + { symbol: 'SO', name: 'Southern Co.', sector: 'Utilities', isETF: false }, + { symbol: 'PLD', name: 'Prologis Inc.', sector: 'Real Estate', isETF: false }, + { symbol: 'AMT', name: 'American Tower', sector: 'Real Estate', isETF: false }, + { symbol: 'EQIX', name: 'Equinix Inc.', sector: 'Real Estate', isETF: false }, + + // --- High-Growth / Mid-Cap Tech --- { symbol: 'PLTR', name: 'Palantir Technologies', sector: 'Technology', isETF: false }, { symbol: 'CRWD', name: 'CrowdStrike Holdings', sector: 'Technology', isETF: false }, { symbol: 'SNOW', name: 'Snowflake Inc.', sector: 'Technology', isETF: false }, { symbol: 'COIN', name: 'Coinbase Global', sector: 'Financials', isETF: false }, - { symbol: 'NFLX', name: 'Netflix Inc.', sector: 'Technology', isETF: false }, - { symbol: 'CRM', name: 'Salesforce Inc.', sector: 'Technology', isETF: false }, - { symbol: 'ADBE', name: 'Adobe Inc.', sector: 'Technology', isETF: false }, - { symbol: 'NOW', name: 'ServiceNow Inc.', sector: 'Technology', isETF: false }, { symbol: 'MSTR', name: 'MicroStrategy Inc.', sector: 'Technology', isETF: false }, - // Broad ETFs + { symbol: 'U', name: 'Unity Software', sector: 'Technology', isETF: false }, + { symbol: 'DDOG', name: 'Datadog Inc.', sector: 'Technology', isETF: false }, + { symbol: 'SHOP', name: 'Shopify Inc.', sector: 'Technology', isETF: false }, + { symbol: 'SQ', name: 'Block Inc.', sector: 'Financials', isETF: false }, + + // --- Regional / Global ETFs (Including EM focus) --- + { symbol: 'EEM', name: 'iShares MSCI Emerging Markets', sector: 'ETF', isETF: true }, + { symbol: 'VWO', name: 'Vanguard FTSE Emerging Markets', sector: 'ETF', isETF: true }, + { symbol: 'INDA', name: 'iShares MSCI India', sector: 'ETF', isETF: true }, + { symbol: 'FXI', name: 'iShares China Large-Cap', sector: 'ETF', isETF: true }, + { symbol: 'MCHI', name: 'iShares MSCI China', sector: 'ETF', isETF: true }, + { symbol: 'EWZ', name: 'iShares MSCI Brazil', sector: 'ETF', isETF: true }, + { symbol: 'EWT', name: 'iShares MSCI Taiwan', sector: 'ETF', isETF: true }, + { symbol: 'EWY', name: 'iShares MSCI South Korea', sector: 'ETF', isETF: true }, + { symbol: 'EWW', name: 'iShares MSCI Mexico', sector: 'ETF', isETF: true }, + { symbol: 'EWJ', name: 'iShares MSCI Japan', sector: 'ETF', isETF: true }, + { symbol: 'VGK', name: 'Vanguard FTSE Europe', sector: 'ETF', isETF: true }, + { symbol: 'EZU', name: 'iShares MSCI Eurozone', sector: 'ETF', isETF: true }, { symbol: 'SPY', name: 'SPDR S&P 500 ETF', sector: 'ETF', isETF: true }, { symbol: 'QQQ', name: 'Invesco QQQ Trust', sector: 'ETF', isETF: true }, { symbol: 'IWM', name: 'iShares Russell 2000', sector: 'ETF', isETF: true }, - { symbol: 'DIA', name: 'SPDR Dow Jones ETF', sector: 'ETF', isETF: true }, - // Sector ETFs + + // --- Sector & Specialty ETFs --- { symbol: 'XLF', name: 'Financial Select SPDR', sector: 'ETF', isETF: true }, { symbol: 'XLE', name: 'Energy Select SPDR', sector: 'ETF', isETF: true }, { symbol: 'XLK', name: 'Technology Select SPDR', sector: 'ETF', isETF: true }, { symbol: 'XLV', name: 'Health Care Select SPDR', sector: 'ETF', isETF: true }, - // Specialty ETFs + { symbol: 'XLU', name: 'Utilities Select SPDR', sector: 'ETF', isETF: true }, + { symbol: 'XLI', name: 'Industrial Select SPDR', sector: 'ETF', isETF: true }, { symbol: 'ARKK', name: 'ARK Innovation ETF', sector: 'ETF', isETF: true }, { symbol: 'GLD', name: 'SPDR Gold Shares', sector: 'ETF', isETF: true }, + { symbol: 'SLV', name: 'iShares Silver Trust', sector: 'ETF', isETF: true }, { symbol: 'TLT', name: 'iShares 20+ Yr Treasury', sector: 'ETF', isETF: true }, + { symbol: 'UUP', name: 'Invesco DB US Dollar Index', sector: 'ETF', isETF: true }, ]; /**