feat: Expand universe to ~125 symbols covering EM and sectors, optimize fetching with concurrency and rate limit delays
Build Institutional Trader / build-and-deploy (push) Successful in 23s
Details
Build Institutional Trader / build-and-deploy (push) Successful in 23s
Details
This commit is contained in:
parent
1278b5d4eb
commit
85e11dca88
|
|
@ -62,21 +62,40 @@ export async function buildFactorProfiles(injectedUniverseRecords = null) {
|
|||
});
|
||||
}
|
||||
} else {
|
||||
const universe = getUniverse().filter(s => !s.isETF).map(s => s.symbol);
|
||||
for (const ticker of universe) {
|
||||
try {
|
||||
const data = await pullRawRecord(ticker);
|
||||
if (!data) continue;
|
||||
// We now include ETFs in the factor ranking because we want to see momentum/value on country ETFs
|
||||
const universe = getUniverse().map(s => s.symbol);
|
||||
const BATCH_SIZE = 15;
|
||||
|
||||
for (let i = 0; i < universe.length; i += BATCH_SIZE) {
|
||||
const batch = universe.slice(i, i + BATCH_SIZE);
|
||||
const promises = batch.map(async (ticker) => {
|
||||
try {
|
||||
const data = await pullRawRecord(ticker);
|
||||
if (!data) return null;
|
||||
|
||||
rawProfiles.push({
|
||||
ticker,
|
||||
value: FACTORS.value(data),
|
||||
quality: FACTORS.quality(data),
|
||||
lowVol: FACTORS.lowVol(data),
|
||||
momentum: FACTORS.momentum(data)
|
||||
});
|
||||
} catch (e) {
|
||||
console.warn(`[DescriptiveLens] Failed for ${ticker}`);
|
||||
return {
|
||||
ticker,
|
||||
value: FACTORS.value(data),
|
||||
quality: FACTORS.quality(data),
|
||||
lowVol: FACTORS.lowVol(data),
|
||||
momentum: FACTORS.momentum(data)
|
||||
};
|
||||
} catch (e) {
|
||||
console.warn(`[DescriptiveLens] Failed for ${ticker}:`, e.message);
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
const results = await Promise.allSettled(promises);
|
||||
for (const res of results) {
|
||||
if (res.status === 'fulfilled' && res.value) {
|
||||
rawProfiles.push(res.value);
|
||||
}
|
||||
}
|
||||
|
||||
// Throttle delay between batches to respect free API limits
|
||||
if (i + BATCH_SIZE < universe.length) {
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,8 @@ import { scoreSuitability } from '../services/scoringEngine.js';
|
|||
import { classifyRegime } from '../services/regimeService.js';
|
||||
import { logSignals } from '../services/signalLogger.js';
|
||||
const router = express.Router();
|
||||
const universeCache = new NodeCache({ stdTTL: 60 });
|
||||
// Default cache TTL to 5 minutes (300s) to protect against rate limits on large universe
|
||||
const universeCache = new NodeCache({ stdTTL: 300 });
|
||||
|
||||
/**
|
||||
* GET /api/market/universe
|
||||
|
|
@ -238,7 +239,7 @@ router.get('/screener', async (req, res) => {
|
|||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
universeCache.set(cacheKey, data, 120); // 120s cache
|
||||
universeCache.set(cacheKey, data, 300); // 300s cache for expanded universe
|
||||
res.json({ success: true, data });
|
||||
} catch (err) {
|
||||
console.error('[market/screener]', err);
|
||||
|
|
|
|||
|
|
@ -14,69 +14,162 @@ const CAP_TIERS = {
|
|||
// Below $2B = SMALL
|
||||
};
|
||||
|
||||
// Master universe of top 50 stocks + ETFs
|
||||
// Master universe of ~200 carefully selected global stocks + ETFs
|
||||
export const UNIVERSE = [
|
||||
// Mega-cap Tech
|
||||
// --- Mega-cap Tech & Comm Services ---
|
||||
{ symbol: 'AAPL', name: 'Apple Inc.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'MSFT', name: 'Microsoft Corp.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'NVDA', name: 'NVIDIA Corp.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'AMZN', name: 'Amazon.com Inc.', sector: 'Consumer Discret.', isETF: false },
|
||||
{ symbol: 'GOOGL', name: 'Alphabet Inc.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'META', name: 'Meta Platforms', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'GOOGL', name: 'Alphabet Inc.', sector: 'Communication', isETF: false },
|
||||
{ symbol: 'META', name: 'Meta Platforms', sector: 'Communication', isETF: false },
|
||||
{ symbol: 'TSLA', name: 'Tesla Inc.', sector: 'Consumer Discret.', isETF: false },
|
||||
{ symbol: 'AVGO', name: 'Broadcom Inc.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'ORCL', name: 'Oracle Corp.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'AMD', name: 'Advanced Micro Devices', sector: 'Technology', isETF: false },
|
||||
// Financials
|
||||
{ symbol: 'CRM', name: 'Salesforce Inc.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'ADBE', name: 'Adobe Inc.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'CSCO', name: 'Cisco Systems', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'NFLX', name: 'Netflix Inc.', sector: 'Communication', isETF: false },
|
||||
{ symbol: 'DIS', name: 'Walt Disney Co.', sector: 'Communication', isETF: false },
|
||||
{ symbol: 'INTC', name: 'Intel Corp.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'IBM', name: 'Intl Business Machines', sector: 'Technology', isETF: false },
|
||||
|
||||
// --- Emerging Markets & Global Giants (Heavily weighted) ---
|
||||
{ symbol: 'TSM', name: 'Taiwan Semiconductor', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'BABA', name: 'Alibaba Group', sector: 'Consumer Discret.', isETF: false },
|
||||
{ symbol: 'PDD', name: 'PDD Holdings', sector: 'Consumer Discret.', isETF: false },
|
||||
{ symbol: 'HDB', name: 'HDFC Bank', sector: 'Financials', isETF: false },
|
||||
{ symbol: 'IBN', name: 'ICICI Bank', sector: 'Financials', isETF: false },
|
||||
{ symbol: 'INFY', name: 'Infosys Ltd.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'RELIANCE.NS', name: 'Reliance Industries', sector: 'Energy', isETF: false },
|
||||
{ symbol: 'VALE', name: 'Vale S.A.', sector: 'Materials', isETF: false },
|
||||
{ symbol: 'PBR', name: 'Petrobras', sector: 'Energy', isETF: false },
|
||||
{ symbol: 'ITUB', name: 'Itau Unibanco', sector: 'Financials', isETF: false },
|
||||
{ symbol: 'MELI', name: 'MercadoLibre', sector: 'Consumer Discret.', isETF: false },
|
||||
{ symbol: 'NU', name: 'Nu Holdings', sector: 'Financials', isETF: false },
|
||||
{ symbol: 'ASML', name: 'ASML Holding', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'NVO', name: 'Novo Nordisk', sector: 'Healthcare', isETF: false },
|
||||
{ symbol: 'TM', name: 'Toyota Motor', sector: 'Consumer Discret.', isETF: false },
|
||||
{ symbol: 'SONY', name: 'Sony Group', sector: 'Consumer Discret.', isETF: false },
|
||||
{ symbol: 'RY', name: 'Royal Bank of Canada', sector: 'Financials', isETF: false },
|
||||
{ symbol: 'BHP', name: 'BHP Group', sector: 'Materials', isETF: false },
|
||||
{ symbol: 'RIO', name: 'Rio Tinto', sector: 'Materials', isETF: false },
|
||||
{ symbol: 'BIDU', name: 'Baidu Inc.', sector: 'Communication', isETF: false },
|
||||
{ symbol: 'JD', name: 'JD.com', sector: 'Consumer Discret.', isETF: false },
|
||||
{ symbol: 'TCEHY', name: 'Tencent Holdings', sector: 'Communication', isETF: false },
|
||||
{ symbol: 'NTES', name: 'NetEase Inc.', sector: 'Communication', isETF: false },
|
||||
{ symbol: 'AMX', name: 'America Movil', sector: 'Communication', isETF: false },
|
||||
{ symbol: 'KB', name: 'KB Financial Group', sector: 'Financials', isETF: false },
|
||||
{ symbol: 'BBD', name: 'Banco Bradesco', sector: 'Financials', isETF: false },
|
||||
|
||||
// --- Financials ---
|
||||
{ symbol: 'JPM', name: 'JPMorgan Chase', sector: 'Financials', isETF: false },
|
||||
{ symbol: 'BAC', name: 'Bank of America', sector: 'Financials', isETF: false },
|
||||
{ symbol: 'WFC', name: 'Wells Fargo', sector: 'Financials', isETF: false },
|
||||
{ symbol: 'GS', name: 'Goldman Sachs', sector: 'Financials', isETF: false },
|
||||
{ symbol: 'MS', name: 'Morgan Stanley', sector: 'Financials', isETF: false },
|
||||
{ symbol: 'C', name: 'Citigroup', sector: 'Financials', isETF: false },
|
||||
{ symbol: 'V', name: 'Visa Inc.', sector: 'Financials', isETF: false },
|
||||
{ symbol: 'MA', name: 'Mastercard Inc.', sector: 'Financials', isETF: false },
|
||||
// Healthcare
|
||||
{ symbol: 'AXP', name: 'American Express', sector: 'Financials', isETF: false },
|
||||
{ symbol: 'BLK', name: 'BlackRock Inc.', sector: 'Financials', isETF: false },
|
||||
{ symbol: 'BX', name: 'Blackstone Inc.', sector: 'Financials', isETF: false },
|
||||
|
||||
// --- Healthcare ---
|
||||
{ symbol: 'UNH', name: 'UnitedHealth Group', sector: 'Healthcare', isETF: false },
|
||||
{ symbol: 'LLY', name: 'Eli Lilly & Co.', sector: 'Healthcare', isETF: false },
|
||||
{ symbol: 'JNJ', name: 'Johnson & Johnson', sector: 'Healthcare', isETF: false },
|
||||
{ symbol: 'ABBV', name: 'AbbVie Inc.', sector: 'Healthcare', isETF: false },
|
||||
{ symbol: 'PFE', name: 'Pfizer Inc.', sector: 'Healthcare', isETF: false },
|
||||
// Energy
|
||||
{ symbol: 'MRK', name: 'Merck & Co.', sector: 'Healthcare', isETF: false },
|
||||
{ symbol: 'TMO', name: 'Thermo Fisher Scientific', sector: 'Healthcare', isETF: false },
|
||||
{ symbol: 'DHR', name: 'Danaher Corp.', sector: 'Healthcare', isETF: false },
|
||||
{ symbol: 'ISRG', name: 'Intuitive Surgical', sector: 'Healthcare', isETF: false },
|
||||
{ symbol: 'SYK', name: 'Stryker Corp.', sector: 'Healthcare', isETF: false },
|
||||
|
||||
// --- Energy & Materials ---
|
||||
{ symbol: 'XOM', name: 'Exxon Mobil', sector: 'Energy', isETF: false },
|
||||
{ symbol: 'CVX', name: 'Chevron Corp.', sector: 'Energy', isETF: false },
|
||||
// Industrials
|
||||
{ symbol: 'COP', name: 'ConocoPhillips', sector: 'Energy', isETF: false },
|
||||
{ symbol: 'SLB', name: 'Schlumberger', sector: 'Energy', isETF: false },
|
||||
{ symbol: 'LIN', name: 'Linde plc', sector: 'Materials', isETF: false },
|
||||
{ symbol: 'SHW', name: 'Sherwin-Williams', sector: 'Materials', isETF: false },
|
||||
{ symbol: 'FCX', name: 'Freeport-McMoRan', sector: 'Materials', isETF: false },
|
||||
{ symbol: 'EOG', name: 'EOG Resources', sector: 'Energy', isETF: false },
|
||||
{ symbol: 'OXY', name: 'Occidental Petroleum', sector: 'Energy', isETF: false },
|
||||
|
||||
// --- Industrials & Aerospace ---
|
||||
{ symbol: 'CAT', name: 'Caterpillar Inc.', sector: 'Industrials', isETF: false },
|
||||
{ symbol: 'HON', name: 'Honeywell Intl.', sector: 'Industrials', isETF: false },
|
||||
// Consumer
|
||||
{ symbol: 'GE', name: 'General Electric', sector: 'Industrials', isETF: false },
|
||||
{ symbol: 'BA', name: 'Boeing Co.', sector: 'Industrials', isETF: false },
|
||||
{ symbol: 'LMT', name: 'Lockheed Martin', sector: 'Industrials', isETF: false },
|
||||
{ symbol: 'UNP', name: 'Union Pacific', sector: 'Industrials', isETF: false },
|
||||
{ symbol: 'UPS', name: 'United Parcel Service', sector: 'Industrials', isETF: false },
|
||||
{ symbol: 'RTX', name: 'RTX Corp.', sector: 'Industrials', isETF: false },
|
||||
{ symbol: 'DE', name: 'Deere & Co.', sector: 'Industrials', isETF: false },
|
||||
|
||||
// --- Consumer Staples & Discretionary ---
|
||||
{ symbol: 'WMT', name: 'Walmart Inc.', sector: 'Consumer Staples', isETF: false },
|
||||
{ symbol: 'COST', name: 'Costco Wholesale', sector: 'Consumer Staples', isETF: false },
|
||||
// Semiconductors
|
||||
{ symbol: 'QCOM', name: 'Qualcomm Inc.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'MU', name: 'Micron Technology', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'INTC', name: 'Intel Corp.', sector: 'Technology', isETF: false },
|
||||
// Growth / Momentum
|
||||
{ symbol: 'PG', name: 'Procter & Gamble', sector: 'Consumer Staples', isETF: false },
|
||||
{ symbol: 'KO', name: 'Coca-Cola Co.', sector: 'Consumer Staples', isETF: false },
|
||||
{ symbol: 'PEP', name: 'PepsiCo Inc.', sector: 'Consumer Staples', isETF: false },
|
||||
{ symbol: 'MCD', name: 'McDonald\'s Corp.', sector: 'Consumer Discret.', isETF: false },
|
||||
{ symbol: 'HD', name: 'Home Depot', sector: 'Consumer Discret.', isETF: false },
|
||||
{ symbol: 'NKE', name: 'NIKE Inc.', sector: 'Consumer Discret.', isETF: false },
|
||||
{ symbol: 'SBUX', name: 'Starbucks Corp.', sector: 'Consumer Discret.', isETF: false },
|
||||
{ symbol: 'TGT', name: 'Target Corp.', sector: 'Consumer Staples', isETF: false },
|
||||
|
||||
// --- Utilities & Real Estate ---
|
||||
{ symbol: 'NEE', name: 'NextEra Energy', sector: 'Utilities', isETF: false },
|
||||
{ symbol: 'DUK', name: 'Duke Energy', sector: 'Utilities', isETF: false },
|
||||
{ symbol: 'SO', name: 'Southern Co.', sector: 'Utilities', isETF: false },
|
||||
{ symbol: 'PLD', name: 'Prologis Inc.', sector: 'Real Estate', isETF: false },
|
||||
{ symbol: 'AMT', name: 'American Tower', sector: 'Real Estate', isETF: false },
|
||||
{ symbol: 'EQIX', name: 'Equinix Inc.', sector: 'Real Estate', isETF: false },
|
||||
|
||||
// --- High-Growth / Mid-Cap Tech ---
|
||||
{ symbol: 'PLTR', name: 'Palantir Technologies', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'CRWD', name: 'CrowdStrike Holdings', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'SNOW', name: 'Snowflake Inc.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'COIN', name: 'Coinbase Global', sector: 'Financials', isETF: false },
|
||||
{ symbol: 'NFLX', name: 'Netflix Inc.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'CRM', name: 'Salesforce Inc.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'ADBE', name: 'Adobe Inc.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'NOW', name: 'ServiceNow Inc.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'MSTR', name: 'MicroStrategy Inc.', sector: 'Technology', isETF: false },
|
||||
// Broad ETFs
|
||||
{ symbol: 'U', name: 'Unity Software', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'DDOG', name: 'Datadog Inc.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'SHOP', name: 'Shopify Inc.', sector: 'Technology', isETF: false },
|
||||
{ symbol: 'SQ', name: 'Block Inc.', sector: 'Financials', isETF: false },
|
||||
|
||||
// --- Regional / Global ETFs (Including EM focus) ---
|
||||
{ symbol: 'EEM', name: 'iShares MSCI Emerging Markets', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'VWO', name: 'Vanguard FTSE Emerging Markets', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'INDA', name: 'iShares MSCI India', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'FXI', name: 'iShares China Large-Cap', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'MCHI', name: 'iShares MSCI China', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'EWZ', name: 'iShares MSCI Brazil', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'EWT', name: 'iShares MSCI Taiwan', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'EWY', name: 'iShares MSCI South Korea', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'EWW', name: 'iShares MSCI Mexico', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'EWJ', name: 'iShares MSCI Japan', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'VGK', name: 'Vanguard FTSE Europe', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'EZU', name: 'iShares MSCI Eurozone', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'SPY', name: 'SPDR S&P 500 ETF', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'QQQ', name: 'Invesco QQQ Trust', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'IWM', name: 'iShares Russell 2000', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'DIA', name: 'SPDR Dow Jones ETF', sector: 'ETF', isETF: true },
|
||||
// Sector ETFs
|
||||
|
||||
// --- Sector & Specialty ETFs ---
|
||||
{ symbol: 'XLF', name: 'Financial Select SPDR', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'XLE', name: 'Energy Select SPDR', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'XLK', name: 'Technology Select SPDR', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'XLV', name: 'Health Care Select SPDR', sector: 'ETF', isETF: true },
|
||||
// Specialty ETFs
|
||||
{ symbol: 'XLU', name: 'Utilities Select SPDR', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'XLI', name: 'Industrial Select SPDR', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'ARKK', name: 'ARK Innovation ETF', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'GLD', name: 'SPDR Gold Shares', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'SLV', name: 'iShares Silver Trust', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'TLT', name: 'iShares 20+ Yr Treasury', sector: 'ETF', isETF: true },
|
||||
{ symbol: 'UUP', name: 'Invesco DB US Dollar Index', sector: 'ETF', isETF: true },
|
||||
];
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Reference in New Issue