Compare commits

..

45 Commits
dec31 ... main

Author SHA1 Message Date
Antigravity 0ac5176df6 add-search-and-top100
Build Institutional Trader / build-and-deploy (push) Successful in 3m9s Details
2026-06-30 00:30:53 +00:00
Antigravity f177252d6d trigger-build
Build Institutional Trader / build-and-deploy (push) Successful in 28s Details
2026-06-29 23:56:44 +00:00
Deep Koluguri e784bc46eb Add Reversal Screener tab to dashboard
Build Institutional Trader / build-and-deploy (push) Successful in 7m35s Details
2026-06-29 23:23:09 +00:00
Deep Koluguri 2b3355cddb merge: resolve conflicts between local production fixes and remote updates
Build Institutional Trader / build-and-deploy (push) Successful in 5m41s Details
2026-06-29 17:13:44 -04:00
Deep Koluguri fe48536e9b Update app.yaml with python sidecar
Build Institutional Trader / build-and-deploy (push) Successful in 2m39s Details
2026-06-29 11:48:26 -04:00
Deep Koluguri 03a7e73b37 Update Dockerfile for python
Build Institutional Trader / build-and-deploy (push) Has been cancelled Details
2026-06-29 11:42:47 -04:00
Deep Koluguri 5241f18052 style: Add explanatory tooltips to factor lens rows
Build Institutional Trader / build-and-deploy (push) Successful in 54s Details
2026-06-27 16:00:46 -04:00
Deep Koluguri 85e11dca88 feat: Expand universe to ~125 symbols covering EM and sectors, optimize fetching with concurrency and rate limit delays
Build Institutional Trader / build-and-deploy (push) Successful in 23s Details
2026-06-27 15:12:10 -04:00
Deep Koluguri 1278b5d4eb style: Move StockDetailPanel directly under the search bar for better UX
Build Institutional Trader / build-and-deploy (push) Successful in 50s Details
2026-06-27 15:02:34 -04:00
Deep Koluguri 85e9e523d9 fix: Replace Kaniko with standard docker build to avoid context mounting issues
Build Institutional Trader / build-and-deploy (push) Successful in 1m37s Details
2026-06-27 14:24:36 -04:00
Deep Koluguri 189d33e65e fix: Revert JOB_CONTAINER extraction now that zombies are cleared
Build Institutional Trader / build-and-deploy (push) Failing after 21s Details
2026-06-27 14:21:16 -04:00
Deep Koluguri 9b2420ed8d fix: Kaniko mounting wrong workspace due to race condition
Build Institutional Trader / build-and-deploy (push) Failing after 16s Details
2026-06-25 23:14:58 -04:00
Deep Koluguri ab1e3f28f6 feat: Add dynamic Fundamental Factor Lens to individual stock search view
Build Institutional Trader / build-and-deploy (push) Failing after 17s Details
2026-06-25 22:59:16 -04:00
Deep Koluguri ae21b0df90 chore: add --no-cache to kaniko pipeline
Build Institutional Trader / build-and-deploy (push) Failing after 16s Details
2026-06-25 22:44:13 -04:00
Deep Koluguri 7907fa30f2 feat: Add 1W, 1M, 1Y returns to macro dashboard
Build Institutional Trader / build-and-deploy (push) Successful in 2m9s Details
2026-06-25 22:36:15 -04:00
Deep Koluguri e43361fadd chore: cache bust
Build Institutional Trader / build-and-deploy (push) Successful in 2m3s Details
2026-06-25 22:29:20 -04:00
Deep Koluguri 6e323967a1 feat: swap generic mkts for specific regions (EWJ, FXI, INDA, VGK)
Build Institutional Trader / build-and-deploy (push) Successful in 1m59s Details
2026-06-25 21:31:51 -04:00
Deep Koluguri 9625979954 feat: Add Global Macro Indicators strip to dashboard
Build Institutional Trader / build-and-deploy (push) Successful in 2m9s Details
2026-06-25 21:04:43 -04:00
Deep Koluguri cd78058072 UI Honesty Pivot: Remove predictive claims, add methodology, descriptive flow
Build Institutional Trader / build-and-deploy (push) Successful in 2m8s Details
2026-06-25 20:32:15 -04:00
Deep Koluguri 2e2efaf530 fix(backend): restore calculateRocketScore to fix crash
Build Institutional Trader / build-and-deploy (push) Successful in 2m3s Details
2026-06-25 19:40:16 -04:00
Deep Koluguri 8677489043 fix(ui): tear out predictive swimlanes and replace with descriptive factor lens grid
Build Institutional Trader / build-and-deploy (push) Successful in 2m10s Details
2026-06-25 19:28:12 -04:00
Deep Koluguri a5e06b44b1 feat(factorlab): pivot to honest transparent research accelerator, implement factor pipeline, descriptive lens, and fundamental snapshotter
Build Institutional Trader / build-and-deploy (push) Successful in 2m9s Details
2026-06-25 00:29:34 -04:00
Deep Koluguri 4dfe340345 style: remove nested scrollbars from swimlanes, let main window scroll
Build Institutional Trader / build-and-deploy (push) Successful in 2m3s Details
2026-06-24 19:38:55 -04:00
Deep Koluguri dfae02c851 fix: use relative URLs for API requests to fix CSP block in production
Build Institutional Trader / build-and-deploy (push) Successful in 1m57s Details
2026-06-24 19:30:05 -04:00
Deep Koluguri 7c67786e80 fix: relax CORS middleware to prevent 500 errors when CORS_ORIGIN is missing
Build Institutional Trader / build-and-deploy (push) Successful in 1m58s Details
2026-06-24 19:25:10 -04:00
Deep Koluguri 43ba386da9 fix: CORP header for Cloudflare proxy
Build Institutional Trader / build-and-deploy (push) Has been cancelled Details
2026-06-24 19:23:18 -04:00
Deep Koluguri 7344d5a2b1 fix: static files before routes, fix CSP for assets and Cloudflare, SPA fallback safety
Build Institutional Trader / build-and-deploy (push) Successful in 1m57s Details
2026-06-24 19:17:43 -04:00
Deep Koluguri 78b408057c fix: health returns 200 for degraded, probe tolerant of optional services
Build Institutional Trader / build-and-deploy (push) Successful in 2m2s Details
2026-06-24 19:13:56 -04:00
Deep Koluguri 911693da88 feat: swimlane screener suitability scoring global ticker search remove side panels
Build Institutional Trader / build-and-deploy (push) Successful in 2m7s Details
2026-06-24 18:53:11 -04:00
Deep Koluguri f91534e976 feat: implement stock evaluation filters, fundamentals styling, and AI trading profiles
Build Institutional Trader / build-and-deploy (push) Successful in 2m1s Details
2026-06-22 23:19:38 -04:00
Deep Koluguri c981faf3a6 fix: resolve WebSocket path conflicts by manually routing HTTP upgrade events
Build Institutional Trader / build-and-deploy (push) Successful in 2m1s Details
2026-06-22 22:10:53 -04:00
Deep Koluguri 2ba6425eed fix: trust proxy for express-rate-limit to prevent crash behind APISIX
Build Institutional Trader / build-and-deploy (push) Successful in 2m4s Details
2026-06-22 22:03:32 -04:00
Deep Koluguri a0c54feee5 fix: import getApiUrl in AlertsFeed to fix reference error
Build Institutional Trader / build-and-deploy (push) Successful in 2m6s Details
2026-06-22 21:54:17 -04:00
Deep Koluguri b3373d6d85 fix: explicitly allow wss in CSP connectSrc
Build Institutional Trader / build-and-deploy (push) Successful in 2m13s Details
2026-06-22 19:47:00 -04:00
Deep Koluguri 2dba57191b fix: change hardcoded localhost URLs to relative URLs so frontend works on production domain
Build Institutional Trader / build-and-deploy (push) Successful in 2m10s Details
2026-06-22 19:43:18 -04:00
Deep Koluguri cd83c18555 fix: update CSP for Cloudflare and set CORS_ORIGIN to fix 500 errors
Build Institutional Trader / build-and-deploy (push) Successful in 2m8s Details
2026-06-22 19:40:24 -04:00
Deep Koluguri 76c24aaec1 fix: disable python service health check to prevent pod crashes
Build Institutional Trader / build-and-deploy (push) Has been cancelled Details
2026-06-22 19:38:15 -04:00
Deep Koluguri 4d00f978e2 feat: provision dedicated postgres database via CloudNativePG
Build Institutional Trader / build-and-deploy (push) Successful in 2m19s Details
2026-06-22 19:29:22 -04:00
Deep Koluguri f68b3e5b75 chore: trigger Kaniko build for institutional-trader
Build Institutional Trader / build-and-deploy (push) Successful in 2m11s Details
2026-06-22 17:24:39 -04:00
Deep Koluguri f6f35dba00 Update package-lock 2026-06-22 16:40:15 -04:00
Deep Koluguri 274dc38794 Add .dockerignore to prevent Windows binaries from breaking Kaniko 2026-06-22 16:34:52 -04:00
Deep Koluguri 7c2e19552f Fix Vite config Rollup missing dependency error 2026-06-22 16:30:34 -04:00
Deep Koluguri 947c50302e Fix trailing comma in frontend package.json 2026-06-22 16:21:46 -04:00
Deep Koluguri 95cdafa34a Add SSL to ingress 2026-06-22 16:00:43 -04:00
Deep Koluguri 4774890415 Deploy to Talos K8s: add Market Analysis and Docker/K8s manifests 2026-06-22 15:49:06 -04:00
152 changed files with 19680 additions and 1496 deletions

6
.dockerignore Normal file
View File

@ -0,0 +1,6 @@
node_modules/
frontend/node_modules/
backend/node_modules/
frontend/dist/
.git/
.env

View File

@ -0,0 +1,16 @@
name: Build Institutional Trader
on: [push]
jobs:
build-and-deploy:
runs-on: ubuntu-latest
env:
DOCKER_HOST: tcp://172.17.0.1:2375
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Build and push
run: |
docker build -t 192.168.8.250:5000/market:latest .
docker push 192.168.8.250:5000/market:latest

43
Dockerfile Normal file
View File

@ -0,0 +1,43 @@
# Stage 1: Build Frontend
FROM node:18-alpine AS frontend-build
WORKDIR /app/frontend
COPY frontend/package*.json ./
RUN npm install
COPY frontend/ ./
RUN npm run build
# Stage 2: Build Backend & Serve
FROM node:18-alpine
WORKDIR /app/backend
# Install python and build dependencies
RUN apk add --no-cache python3 py3-pip python3-dev build-base && \
python3 -m venv /opt/venv
ENV PATH="/opt/venv/bin:$PATH"
# Install backend dependencies
COPY backend/package*.json ./
RUN npm install --production
# Copy backend source
COPY backend/ ./
# Install python dependencies
RUN pip install --no-cache-dir -r python_service/requirements.txt
# Copy built frontend to the public directory expected by server.js
# (__dirname is /app/backend/src, so ../../public is /app/public)
COPY --from=frontend-build /app/frontend/dist /app/public
# Set production environment
ENV NODE_ENV=production
ENV PORT=3010
EXPOSE 3010
CMD ["node", "src/server.js"]
# Cache bust 20260625222919
# Cache bust 20260625223615

View File

@ -58,7 +58,14 @@ USE_PYTHON_SERVICE=true
### Terminal 1: Python Service
```bash
cd backend/python_service
source venv/bin/activate # or venv\Scripts\activate on Windows
# Activate virtual environment:
# Git Bash on Windows:
source venv/Scripts/activate
# PowerShell/CMD on Windows:
venv\Scripts\activate
# Linux/macOS:
source venv/bin/activate
uvicorn main:app --reload --port 8010
```

View File

@ -19,10 +19,13 @@
"express-rate-limit": "^7.1.5",
"express-ws": "^5.0.2",
"helmet": "^7.1.0",
"ml-logistic-regression": "^2.0.0",
"ml-matrix": "^6.13.0",
"node-cache": "^5.1.2",
"node-fetch": "^3.3.2",
"pg": "^8.11.3",
"ws": "^8.14.2"
"ws": "^8.14.2",
"xml2js": "^0.6.2"
},
"devDependencies": {
"nodemon": "^3.0.1"
@ -932,6 +935,12 @@
"node": ">= 0.10"
}
},
"node_modules/is-any-array": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/is-any-array/-/is-any-array-3.0.0.tgz",
"integrity": "sha512-o4h+tylWykC4BD1vaejp6gDxoM13bwW8FGuNs4yIKpj8xbBJcRxJx8vZpq0dCr7ZDEfeKjmsi/euolKhX6f/ww==",
"license": "MIT"
},
"node_modules/is-binary-path": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
@ -1073,6 +1082,54 @@
"node": "*"
}
},
"node_modules/ml-array-max": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ml-array-max/-/ml-array-max-2.0.0.tgz",
"integrity": "sha512-QQZ4kENwpWmyNb98UXRDFXrmtIXuXtt1+bSbda/2KA85+F+rrJP8hZk6QOkCQXM2Th9mUDYdq/PNByPdT9ID4A==",
"license": "MIT",
"dependencies": {
"is-any-array": "^3.0.0"
}
},
"node_modules/ml-array-min": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ml-array-min/-/ml-array-min-2.0.0.tgz",
"integrity": "sha512-GRj6Ky6sW9vGL6yIjgsHmXZ9YgrdmcQ8nCxPqEGeKc6dkfYg1XDYxGFxADUjNuZyoCd5PUscWAS4N+cFaX6hFg==",
"license": "MIT",
"dependencies": {
"is-any-array": "^3.0.0"
}
},
"node_modules/ml-array-rescale": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ml-array-rescale/-/ml-array-rescale-2.0.0.tgz",
"integrity": "sha512-2GGtKfSno94/kIloWGvpp/U5Q5vLvLrza+SAaGsLeo6Xj4mEbA6Gqx+oTfZFkxnd1grT2X007HfJNs3T5BsiVg==",
"license": "MIT",
"dependencies": {
"is-any-array": "^3.0.0",
"ml-array-max": "^2.0.0",
"ml-array-min": "^2.0.0"
}
},
"node_modules/ml-logistic-regression": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ml-logistic-regression/-/ml-logistic-regression-2.0.0.tgz",
"integrity": "sha512-xHhB91ut8GRRbJyB1ZQfKsl1MHmE1PqMeRjxhks96M5BGvCbC9eEojf4KgRMKM2LxFblhVUcVzweAoPB48Nt0A==",
"license": "MIT",
"dependencies": {
"ml-matrix": "^6.5.0"
}
},
"node_modules/ml-matrix": {
"version": "6.13.0",
"resolved": "https://registry.npmjs.org/ml-matrix/-/ml-matrix-6.13.0.tgz",
"integrity": "sha512-QpV0UTUkglg6vPUgThKGBEtit2ac6habSoZ33bwI9rU0UHZLqw6G3ukTIE8zWiUF3sjK8YAlhx/o/b9layzH8A==",
"license": "MIT",
"dependencies": {
"is-any-array": "^3.0.0",
"ml-array-rescale": "^2.0.0"
}
},
"node_modules/ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@ -1499,6 +1556,15 @@
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
"license": "MIT"
},
"node_modules/sax": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz",
"integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==",
"license": "BlueOak-1.0.0",
"engines": {
"node": ">=11.0.0"
}
},
"node_modules/semver": {
"version": "7.7.3",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz",
@ -1895,6 +1961,28 @@
}
}
},
"node_modules/xml2js": {
"version": "0.6.2",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz",
"integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==",
"license": "MIT",
"dependencies": {
"sax": ">=0.6.0",
"xmlbuilder": "~11.0.0"
},
"engines": {
"node": ">=4.0.0"
}
},
"node_modules/xmlbuilder": {
"version": "11.0.1",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
"integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==",
"license": "MIT",
"engines": {
"node": ">=4.0"
}
},
"node_modules/xtend": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",

View File

@ -33,10 +33,13 @@
"express-rate-limit": "^7.1.5",
"express-ws": "^5.0.2",
"helmet": "^7.1.0",
"ml-logistic-regression": "^2.0.0",
"ml-matrix": "^6.13.0",
"node-cache": "^5.1.2",
"node-fetch": "^3.3.2",
"pg": "^8.11.3",
"ws": "^8.14.2"
"ws": "^8.14.2",
"xml2js": "^0.6.2"
},
"devDependencies": {
"nodemon": "^3.0.1"

View File

@ -0,0 +1,86 @@
{
"success": true,
"data": [
{
"CreatedDate": "2025-01-15",
"CreatedTime": "10:30:45 AM",
"Symbol": "(5🟩) AAPL · RTH ⚡ 🟢💎⭐💰 ⚡ 🔥",
"Rocket": "🚀🚀🚀 [ITM 2.5%]",
"NetPremium": "1.50 M",
"Premium": "850.00 K",
"premium_num": 850000,
"bull_total": 1500000,
"bear_total": 50000,
"PctVsPriorClose": 1.2,
"PctVsRthOpen": 0.8,
"Pct5m": 0.3,
"Pct15m": 0.5,
"TapeAlign": "↗︎",
"NearAlert": "EARNINGS",
"ExpirationDate": "2025-01-17",
"Price": "185.50",
"CallPut": "CALL",
"Side": "AA",
"Strike": "185",
"Spot": "185.50",
"Volume": "5000",
"OI": "3000",
"direction": "BULL",
"badge_round": "🟢",
"badge_more": "💎⭐💰✔",
"flash": "⚡",
"rocket_score": 5.2,
"session_bucket": "RTH",
"flow_ts_utc": "2025-01-15T16:30:45+00:00",
"mny_pct": 2.5,
"vwap_at_signal": 185.20,
"price_vs_vwap_pct": 0.16,
"signal_tier": "TIER_1",
"checklist_score": 8.5,
"checklist_passed": true,
"premium_zscore": 2.3,
"premium_percentile_intraday": 85.5,
"relative_premium_score": 72.5,
"aggression_score": 68.0,
"size_concentration_score": 75.0,
"repeat_trade_velocity": 55.0,
"strike_clustering_score": 60.0,
"signal_strength": 64.7,
"early_noise_reject": false,
"flow_acceleration": 12500.5,
"time_between_hits": 3.2,
"follow_on_ratio": 0.75,
"strike_laddering_detected": true,
"delta_exposure": 92500000,
"gamma_exposure": 1250000000,
"volatility_intent": "LONG_VOL",
"net_gamma_exposure_per_symbol": 8500000000,
"gamma_flip_proximity": 0.15,
"dealer_hedge_pressure_score": 72.5,
"market_regime": "TREND",
"flow_state": "ACTIONABLE",
"confidence_score": 78.5,
"institutional_likelihood": 0.85,
"dealer_pain_level": 65.0,
"expected_move_vs_implied": 1.35
}
],
"count": 1,
"timestamp": "2025-01-15T16:35:00.123456"
}

View File

@ -0,0 +1,168 @@
# Institutional-Grade Options Flow Analytics
This document describes the institutional-grade enhancements to the options flow pipeline.
## Overview
The pipeline has been refactored to convert static retail-style flow detection into dynamic, dealer-aware, time-sequenced signals suitable for intraday momentum and 1-5 day swing trades.
## New Analytics Modules
### 1. Relative Premium Scoring (`relative_premium_scorer.py`)
**Purpose**: Replace static premium filter (minPremium = $80K) with context-aware relative scoring.
**New Fields**:
- `premium_zscore`: Z-score of premium relative to 20-day rolling window per ticker
- `premium_percentile_intraday`: Percentile rank within same-day flow
- `relative_premium_score`: Composite score (0-100) combining z-score, intraday percentile, and median normalization
**Usage**: Premium of $80K might be significant for AAPL but noise for TSLA. This module computes relative significance.
### 2. Signal Component Scoring (`signal_component_scorer.py`)
**Purpose**: Convert binary badge logic (💎 ⭐ 🟢 🔴) into continuous numeric signal components.
**New Fields**:
- `aggression_score`: Measures trade aggression (ITM premiums, ask-side trades)
- `size_concentration_score`: Measures size concentration (single large trade vs many small ones)
- `repeat_trade_velocity`: Measures repeat trade frequency (urgency building)
- `strike_clustering_score`: Measures strike clustering (laddering patterns)
- `signal_strength`: Composite score = 0.30 * aggression + 0.30 * size_concentration + 0.20 * repeat_velocity + 0.20 * strike_clustering
**Note**: Badges remain display-only. Signal strength is computed from components.
### 3. Tier-0 Noise Rejection (`noise_rejector.py`)
**Purpose**: Reject low-quality signals before enrichment to reduce processing overhead.
**New Fields**:
- `early_noise_reject`: Boolean flag indicating if signal should be rejected as noise
**Rejection Criteria**:
- Single isolated trade (no repeat activity within 30 minutes)
- Far OTM weekly lottos (>15% OTM with <7 days to expiry)
- Delta-adjusted premium below threshold (<$50K)
### 4. Time-Sequenced Flow Analysis (`time_sequenced_analyzer.py`)
**Purpose**: Analyze flow patterns over time to detect urgency, distribution, and continuation.
**New Fields**:
- `flow_acceleration`: Change in premium per minute (Δ premium / minute)
- `time_between_hits`: Average time between consecutive trades (minutes)
- `follow_on_ratio`: Fraction of trades in same direction after initial trade (0-1)
- `strike_laddering_detected`: Boolean indicating sequential strike accumulation
**Interpretation**:
- Escalating premium + decreasing time gaps = urgency
- Flat premium + widening gaps = distribution
### 5. Intent Classification (`intent_classifier.py`)
**Purpose**: Replace naive direction (BULL/BEAR) with nuanced volatility and hedging intent.
**New Fields**:
- `delta_exposure`: Delta exposure (contracts * delta * 100 * spot_price)
- `gamma_exposure`: Gamma exposure (contracts * gamma * 100 * spot_price^2)
- `volatility_intent`: Enum (LONG_VOL, SHORT_VOL, DIRECTIONAL, HEDGE_UNWIND)
**Note**: Direction (BULL/BEAR) becomes secondary metadata.
### 6. Dealer-Aware Flow Context (`dealer_flow_context.py`)
**Purpose**: Track dealer hedging pressure and gamma exposure.
**New Fields**:
- `net_gamma_exposure_per_symbol`: Sum of gamma exposures for symbol (positive = long gamma, negative = short gamma)
- `gamma_flip_proximity`: Proximity to gamma flip point (-1 to 1)
- `dealer_hedge_pressure_score`: Dealer hedge pressure score (0-100)
**Usage**: Validates flow continuation, flow reversals, and gamma squeeze setups.
### 7. Market Regime Detection (`market_regime_detector.py`)
**Purpose**: Identify market regime to gate trade signal generation.
**New Fields**:
- `market_regime`: Enum (TREND, RANGE, HIGH_VOL_EVENT)
**Trade Signal Gating**:
- Trend → continuation bias
- Range → fade or vol-sell bias
- Event → volatility expansion bias
### 8. Flow Decay & Reversal Validation (`flow_decay_validator.py`)
**Purpose**: Validate flow decay/reversal signals with anchors.
**New Fields**:
- `flow_state`: Enum (ACTIONABLE, INFORMATIONAL)
**Validation Criteria**: Flow decay/reversal is actionable ONLY IF:
- Premium contracts (relative_premium_score >= 60)
- Dealer hedge pressure decreases
- Price fails near VWAP / opening range / key level
- Otherwise marked as INFORMATIONAL
### 9. Institutional Confidence Metrics (`institutional_confidence.py`)
**Purpose**: Calculate confidence scores for institutional flow signals.
**New Fields**:
- `confidence_score`: Overall confidence score (0-100)
- `institutional_likelihood`: Likelihood flow is institutional (0-1)
- `dealer_pain_level`: Dealer pain level (0-100)
- `expected_move_vs_implied`: Expected move vs implied move ratio
## Integration
All modules are integrated into the main processing pipeline in `main.py`:
1. Basic flow processing (normalization, badges, rocket score)
2. Price context enrichment
3. Alert matching
4. **Institutional analytics pipeline** (NEW):
- Tier-0 noise rejection
- Relative premium scoring
- Signal component scoring
- Time-sequenced analysis
- Intent classification
- Dealer flow context
- Market regime detection
- Flow decay validation
- Confidence metrics
5. Filtering (premium, relative premium, badges, direction)
6. Output formatting
## Filtering Changes
**Before**:
- Static premium filter: `premium_num > 80000`
- Badge requirements: 🟢/🔴 + 💎 + ⭐
**After**:
- Static premium filter: `premium_num > min_premium` (still applied)
- **Relative premium filter**: `relative_premium_score >= 60.0` (NEW)
- **Noise rejection filter**: `early_noise_reject == False` (NEW)
- Badge requirements: 🟢/🔴 + 💎 + ⭐ (still applied, but badges are now display-only)
## API Response
All new fields are included in the API response. The response maintains backward compatibility - existing fields remain unchanged, new fields are additive.
## Design Philosophy
1. **Flow represents pressure, not prediction**: Signals indicate who is forced to act next (dealers hedging)
2. **Institutions trade urgency and forced hedging**: Focus on dealer pain and gamma exposure
3. **Fewer, higher-quality signals > more alerts**: Noise rejection and relative premium filtering reduce false positives
4. **Every signal must answer**: "Who is forced to act next?"
## Success Criteria
If implemented correctly:
- Signal count decreases (noise filtered out)
- Average signal quality increases (relative premium, signal strength)
- False positives reduce (noise rejection, dealer context validation)
- Trades align with intraday momentum and short-term swing horizons (time-sequenced analysis)

View File

@ -192,6 +192,63 @@ async def get_options_flow(
# Recalculate rocket score with price context and alerts
df_final = processor.process_rocket_score(df_final)
# Apply institutional-grade analytics pipeline
logger.info("🔹 Applying institutional-grade analytics...")
from services.relative_premium_scorer import RelativePremiumScorer
from services.noise_rejector import NoiseRejector
from services.signal_component_scorer import SignalComponentScorer
from services.time_sequenced_analyzer import TimeSequencedAnalyzer
from services.intent_classifier import IntentClassifier
from services.dealer_flow_context import DealerFlowContext
from services.market_regime_detector import MarketRegimeDetector
from services.flow_decay_validator import FlowDecayValidator
from services.institutional_confidence import InstitutionalConfidence
# 1. Tier-0 Noise Rejection (mark but don't filter yet - filtering happens later)
logger.info("1⃣ Applying tier-0 noise rejection...")
noise_rejector = NoiseRejector()
df_final = noise_rejector.mark_noise_rejections(df_final)
# 2. Relative Premium Scoring
logger.info("2⃣ Calculating relative premium scores...")
premium_scorer = RelativePremiumScorer(pool)
df_final = await premium_scorer.enrich_with_relative_premium(df_final)
# 3. Signal Component Scoring (convert badges to continuous scores)
logger.info("3⃣ Converting badges to continuous signal components...")
signal_scorer = SignalComponentScorer()
df_final = signal_scorer.enrich_with_signal_components(df_final)
# 4. Time-Sequenced Flow Analysis
logger.info("4⃣ Analyzing time-sequenced flow patterns...")
time_analyzer = TimeSequencedAnalyzer()
df_final = time_analyzer.enrich_with_time_sequenced_metrics(df_final)
# 5. Intent Classification
logger.info("5⃣ Classifying volatility and hedging intent...")
intent_classifier = IntentClassifier()
df_final = intent_classifier.enrich_with_intent_classification(df_final)
# 6. Dealer-Aware Flow Context
logger.info("6⃣ Analyzing dealer hedging pressure...")
dealer_context = DealerFlowContext()
df_final = dealer_context.enrich_with_dealer_context(df_final)
# 7. Market Regime Detection
logger.info("7⃣ Detecting market regime...")
regime_detector = MarketRegimeDetector()
df_final = regime_detector.enrich_with_market_regime(df_final)
# 8. Flow Decay & Reversal Validation
logger.info("8⃣ Validating flow decay and reversal signals...")
flow_validator = FlowDecayValidator()
df_final = flow_validator.enrich_with_flow_state(df_final)
# 9. Institutional Confidence Metrics
logger.info("9⃣ Calculating institutional confidence metrics...")
confidence_calc = InstitutionalConfidence()
df_final = confidence_calc.enrich_with_confidence_metrics(df_final)
# Phase 1 Enhancements (BEFORE filtering so all signals get Phase 1 data):
# Initialize Phase 1 columns with None/empty values first
if not df_final.empty:
@ -259,6 +316,13 @@ async def get_options_flow(
# Filter by minimum premium and badge requirements (matching SQL WHERE clause)
logger.info(f"📊 Before filtering: {len(df_final)} rows")
# Apply noise rejection filter first (exclude early_noise_reject = True)
if 'early_noise_reject' in df_final.columns:
before_noise = len(df_final)
df_final = df_final[~df_final['early_noise_reject']].copy()
after_noise = len(df_final)
logger.info(f"📊 After noise rejection filter: {after_noise} rows (removed {before_noise - after_noise})")
# Only filter if columns exist
if 'premium_num' in df_final.columns:
before_premium = len(df_final)
@ -268,6 +332,14 @@ async def get_options_flow(
else:
logger.warning("⚠️ premium_num column not found, skipping premium filter")
# Apply relative premium filter if available
if 'relative_premium_score' in df_final.columns:
before_relative = len(df_final)
min_relative_threshold = 60.0 # Configurable threshold
df_final = df_final[df_final['relative_premium_score'] >= min_relative_threshold].copy()
after_relative = len(df_final)
logger.info(f"📊 After relative premium filter (>={min_relative_threshold}): {after_relative} rows (removed {before_relative - after_relative})")
if df_final.empty:
logger.warning("⚠️ No data after premium filter")
return OptionsFlowResponse(

View File

@ -0,0 +1,256 @@
"""
Dealer-Aware Flow Context Service
Tracks dealer hedging pressure, gamma exposure, and flow continuation patterns
"""
import pandas as pd
import numpy as np
from typing import Dict, Optional
from datetime import timedelta
from utils.logger import logger
class DealerFlowContext:
"""
Analyzes flow from dealer perspective:
- Net gamma exposure per symbol
- Gamma flip proximity (when dealers become long/short gamma)
- Dealer hedge pressure (forced hedging activity)
"""
def __init__(self):
# Configuration
self.analysis_window_minutes = 120 # Look back window for gamma tracking
self.gamma_flip_threshold = 0.3 # Gamma flip when net gamma crosses threshold
def calculate_net_gamma_exposure_per_symbol(
self,
df: pd.DataFrame,
symbol: str,
timestamp: pd.Timestamp
) -> float:
"""
Calculate net gamma exposure for a symbol at a given time.
Sums all gamma exposures from recent flow.
Positive = dealers are long gamma (hedging by selling on rallies, buying on dips)
Negative = dealers are short gamma (hedging by buying on rallies, selling on dips)
"""
if df.empty:
return 0.0
# Look at flow up to this timestamp
window_start = timestamp - timedelta(minutes=self.analysis_window_minutes)
mask = (
(df['symbol_norm'] == symbol.upper()) &
(df['flow_ts_utc'] >= window_start) &
(df['flow_ts_utc'] <= timestamp) &
(df['gamma_exposure'].notna())
)
recent_flow = df[mask]
if recent_flow.empty:
return 0.0
# Sum gamma exposures
net_gamma = recent_flow['gamma_exposure'].sum()
return float(net_gamma)
def calculate_gamma_flip_proximity(
self,
df: pd.DataFrame,
row_idx: int
) -> Optional[float]:
"""
Calculate proximity to gamma flip (when dealers switch from long to short gamma or vice versa).
Returns: -1.0 to 1.0
- Positive = approaching long gamma (dealers becoming volatility buyers)
- Negative = approaching short gamma (dealers becoming volatility sellers)
- 0.0 = near flip point
"""
if row_idx >= len(df):
return None
current_row = df.iloc[row_idx]
symbol = current_row.get('symbol_norm')
flow_ts_utc = current_row.get('flow_ts_utc')
gamma_exposure = current_row.get('gamma_exposure', 0) or 0
if pd.isna(symbol) or pd.isna(flow_ts_utc):
return None
# Calculate current net gamma
net_gamma = self.calculate_net_gamma_exposure_per_symbol(df, symbol, flow_ts_utc)
# Normalize to -1 to 1 scale
# Use exponential scaling to emphasize near-flip conditions
if net_gamma == 0:
return 0.0
# Simple normalization: divide by a large threshold
# More sophisticated: use percentile or adaptive scaling
threshold = 1000000000 # 1B in gamma exposure as normalization factor
normalized = net_gamma / threshold
# Clamp to -1 to 1
normalized = max(-1.0, min(1.0, normalized))
# Invert: negative normalized = positive proximity (approaching long gamma)
# This is because dealer short gamma (negative) means they're selling volatility
return float(-normalized)
def calculate_dealer_hedge_pressure_score(
self,
df: pd.DataFrame,
row_idx: int
) -> float:
"""
Calculate dealer hedge pressure score (0-100).
Higher score = more forced hedging by dealers:
- High net gamma exposure (dealers must hedge)
- Recent flow creating gamma imbalance
- Flow continuation (dealers hedging creates more flow)
"""
if row_idx >= len(df):
return 0.0
current_row = df.iloc[row_idx]
symbol = current_row.get('symbol_norm')
flow_ts_utc = current_row.get('flow_ts_utc')
gamma_exposure = current_row.get('gamma_exposure', 0) or 0
if pd.isna(symbol) or pd.isna(flow_ts_utc):
return 0.0
score = 0.0
# Net gamma component (0-40 points)
# High absolute net gamma = more hedge pressure
net_gamma = abs(self.calculate_net_gamma_exposure_per_symbol(df, symbol, flow_ts_utc))
if net_gamma > 0:
# Normalize: 500M = 20 points, 1B = 40 points
normalized_gamma = min(1.0, net_gamma / 1000000000) # 1B threshold
score += normalized_gamma * 40.0
# Recent gamma accumulation (0-30 points)
# Recent flow creating gamma imbalance
window_start = flow_ts_utc - timedelta(minutes=30)
mask = (
(df['symbol_norm'] == symbol.upper()) &
(df['flow_ts_utc'] >= window_start) &
(df['flow_ts_utc'] <= flow_ts_utc) &
(df['gamma_exposure'].notna())
)
recent_gamma = df[mask]['gamma_exposure'].sum()
if abs(recent_gamma) > 0:
# Normalize recent gamma accumulation
normalized_recent = min(1.0, abs(recent_gamma) / 500000000) # 500M threshold
score += normalized_recent * 30.0
# Flow continuation component (0-30 points)
# If flow is continuing in same direction, dealers are likely hedging
follow_on_ratio = current_row.get('follow_on_ratio')
if follow_on_ratio is not None and not pd.isna(follow_on_ratio):
# High follow-on ratio = continuation = hedge pressure
score += follow_on_ratio * 30.0
return min(100.0, max(0.0, score))
def validate_flow_continuation(
self,
df: pd.DataFrame,
row_idx: int
) -> bool:
"""
Validate if flow continuation is likely based on dealer hedge pressure.
Returns True if continuation is expected (dealers forced to hedge).
"""
current_row = df.iloc[row_idx]
dealer_pressure = current_row.get('dealer_hedge_pressure_score', 0) or 0
net_gamma = current_row.get('net_gamma_exposure_per_symbol', 0) or 0
# High dealer pressure + significant gamma exposure = continuation likely
if dealer_pressure > 50.0 and abs(net_gamma) > 100000000: # 100M threshold
return True
return False
def validate_flow_reversal(
self,
df: pd.DataFrame,
row_idx: int
) -> bool:
"""
Validate if flow reversal is likely.
Reversal happens when:
- Dealers finish hedging (gamma exposure neutralizes)
- Price fails at key levels (VWAP, opening range)
- Flow shows distribution pattern (decreasing premium, widening gaps)
"""
current_row = df.iloc[row_idx]
dealer_pressure = current_row.get('dealer_hedge_pressure_score', 0) or 0
follow_on_ratio = current_row.get('follow_on_ratio')
flow_acceleration = current_row.get('flow_acceleration')
# Low dealer pressure + low follow-on ratio = reversal likely
if dealer_pressure < 30.0:
if follow_on_ratio is not None and follow_on_ratio < 0.3:
return True
# Negative flow acceleration = flow weakening = reversal
if flow_acceleration is not None and flow_acceleration < -10000:
return True
return False
def enrich_with_dealer_context(self, df: pd.DataFrame) -> pd.DataFrame:
"""
Add dealer-aware flow context metrics to DataFrame.
Adds: net_gamma_exposure_per_symbol, gamma_flip_proximity, dealer_hedge_pressure_score
"""
if df.empty:
return df
df = df.copy()
# Initialize columns
df['net_gamma_exposure_per_symbol'] = 0.0
df['gamma_flip_proximity'] = None
df['dealer_hedge_pressure_score'] = 0.0
# Sort by timestamp for proper gamma tracking
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
# Calculate metrics for each row
for idx in df_sorted.index:
original_idx = df_sorted.index[idx]
row = df_sorted.iloc[idx]
symbol = row.get('symbol_norm')
flow_ts_utc = row.get('flow_ts_utc')
if pd.notna(symbol) and pd.notna(flow_ts_utc):
# Net gamma exposure
net_gamma = self.calculate_net_gamma_exposure_per_symbol(
df_sorted, symbol, flow_ts_utc
)
df.at[original_idx, 'net_gamma_exposure_per_symbol'] = float(net_gamma)
# Gamma flip proximity
flip_prox = self.calculate_gamma_flip_proximity(df_sorted, idx)
if flip_prox is not None:
df.at[original_idx, 'gamma_flip_proximity'] = float(flip_prox)
# Dealer hedge pressure
pressure = self.calculate_dealer_hedge_pressure_score(df_sorted, idx)
df.at[original_idx, 'dealer_hedge_pressure_score'] = float(pressure)
logger.info(f"Dealer context enrichment complete. Mean pressure: {df['dealer_hedge_pressure_score'].mean():.2f}")
return df

View File

@ -0,0 +1,118 @@
"""
Flow Decay & Reversal Validation Service
Validates flow decay/reversal signals with anchors (premium, dealer pressure, price levels)
"""
import pandas as pd
import numpy as np
from typing import Optional
from enum import Enum
from utils.logger import logger
class FlowState(Enum):
"""Flow state classification"""
ACTIONABLE = "ACTIONABLE" # Flow decay/reversal is actionable (trade signal)
INFORMATIONAL = "INFORMATIONAL" # Flow decay/reversal is informational only (no trade signal)
class FlowDecayValidator:
"""
Validates flow decay and reversal signals.
Flow decay/reversal is actionable ONLY IF:
- Premium contracts (high relative premium)
- Dealer hedge pressure decreases
- Price fails near VWAP / opening range / key level
Otherwise mark as INFORMATIONAL.
"""
def __init__(self):
# Configuration
self.min_relative_premium_for_actionable = 60.0 # Minimum relative premium score
self.vwap_failure_threshold_pct = 0.5 # Price within 0.5% of VWAP = failure
self.dealer_pressure_decrease_threshold = 20.0 # Dealer pressure decrease threshold
def validate_flow_decay_reversal(
self,
df: pd.DataFrame,
row_idx: int
) -> str:
"""
Validate if flow decay/reversal is actionable.
Returns FlowState enum value as string.
"""
if row_idx >= len(df):
return FlowState.INFORMATIONAL.value
current_row = df.iloc[row_idx]
# Check 1: Premium contracts (relative premium score)
relative_premium_score = current_row.get('relative_premium_score', 0) or 0
if relative_premium_score < self.min_relative_premium_for_actionable:
return FlowState.INFORMATIONAL.value
# Check 2: Dealer hedge pressure decreases
# Look at recent dealer pressure trend
symbol = current_row.get('symbol_norm')
flow_ts_utc = current_row.get('flow_ts_utc')
if pd.notna(symbol) and pd.notna(flow_ts_utc):
from datetime import timedelta
window_start = flow_ts_utc - timedelta(minutes=30)
mask = (
(df['symbol_norm'] == symbol.upper()) &
(df['flow_ts_utc'] >= window_start) &
(df['flow_ts_utc'] <= flow_ts_utc) &
(df['dealer_hedge_pressure_score'].notna())
)
recent_pressure = df[mask]['dealer_hedge_pressure_score']
if len(recent_pressure) >= 2:
current_pressure = current_row.get('dealer_hedge_pressure_score', 0) or 0
recent_avg = recent_pressure.iloc[:-1].mean()
pressure_decrease = recent_avg - current_pressure
if pressure_decrease < self.dealer_pressure_decrease_threshold:
return FlowState.INFORMATIONAL.value
# Check 3: Price fails near VWAP / opening range / key level
price_vs_vwap_pct = current_row.get('price_vs_vwap_pct')
pct_vs_rth_open = current_row.get('pct_vs_rth_open')
price_failure = False
# VWAP failure
if price_vs_vwap_pct is not None and not pd.isna(price_vs_vwap_pct):
if abs(price_vs_vwap_pct) <= self.vwap_failure_threshold_pct:
price_failure = True
# Opening range failure
if pct_vs_rth_open is not None and not pd.isna(pct_vs_rth_open):
if abs(pct_vs_rth_open) <= 0.3: # Within 0.3% of open
price_failure = True
if not price_failure:
return FlowState.INFORMATIONAL.value
# All checks passed: actionable
return FlowState.ACTIONABLE.value
def enrich_with_flow_state(self, df: pd.DataFrame) -> pd.DataFrame:
"""
Add flow state validation to DataFrame.
Adds: flow_state
"""
if df.empty:
return df
df = df.copy()
df['flow_state'] = FlowState.ACTIONABLE.value # Default to actionable
# Only validate flow decay/reversal cases
# For now, mark all as actionable (can be enhanced based on flow_acceleration, follow_on_ratio)
# This is a placeholder - in production, you'd identify decay/reversal patterns first
logger.info(f"Flow state validation complete. Actionable: {(df['flow_state'] == FlowState.ACTIONABLE.value).sum()}")
return df

View File

@ -0,0 +1,189 @@
"""
Institutional Confidence Metrics Service
Calculates confidence scores for institutional flow signals
"""
import pandas as pd
import numpy as np
from typing import Optional
from utils.logger import logger
class InstitutionalConfidence:
"""
Calculates institutional confidence metrics:
- confidence_score (0-100)
- institutional_likelihood (0-1)
- dealer_pain_level (0-100)
- expected_move_vs_implied (ratio)
"""
def __init__(self):
# Configuration
self.min_premium_for_institutional = 200000 # Minimum premium for institutional classification
def calculate_confidence_score(self, row: pd.Series) -> float:
"""
Calculate overall confidence score (0-100) combining multiple factors.
Higher = more confidence in signal quality.
"""
score = 0.0
# Relative premium component (0-25 points)
relative_premium = row.get('relative_premium_score', 0) or 0
score += (relative_premium / 100.0) * 25.0
# Signal strength component (0-25 points)
signal_strength = row.get('signal_strength', 0) or 0
score += (signal_strength / 100.0) * 25.0
# Dealer pressure component (0-20 points)
dealer_pressure = row.get('dealer_hedge_pressure_score', 0) or 0
score += (dealer_pressure / 100.0) * 20.0
# Flow continuation component (0-15 points)
follow_on_ratio = row.get('follow_on_ratio')
if follow_on_ratio is not None and not pd.isna(follow_on_ratio):
score += follow_on_ratio * 15.0
# Strike laddering component (0-15 points)
strike_laddering = row.get('strike_laddering_detected', False)
if strike_laddering:
score += 15.0
return min(100.0, max(0.0, score))
def calculate_institutional_likelihood(self, row: pd.Series) -> float:
"""
Calculate likelihood that flow is institutional (0-1).
Based on premium size, trade characteristics, and patterns.
"""
premium_num = row.get('premium_num', 0) or 0
relative_premium = row.get('relative_premium_score', 0) or 0
size_concentration = row.get('size_concentration_score', 0) or 0
likelihood = 0.0
# Premium size component (0-40%)
if premium_num >= 1000000: # $1M+
likelihood += 0.40
elif premium_num >= 500000: # $500K+
likelihood += 0.30
elif premium_num >= self.min_premium_for_institutional:
likelihood += 0.20
# Relative premium component (0-30%)
if relative_premium >= 80:
likelihood += 0.30
elif relative_premium >= 60:
likelihood += 0.20
elif relative_premium >= 40:
likelihood += 0.10
# Size concentration component (0-30%)
# Institutional trades are often concentrated
if size_concentration >= 70:
likelihood += 0.30
elif size_concentration >= 50:
likelihood += 0.20
elif size_concentration >= 30:
likelihood += 0.10
return min(1.0, max(0.0, likelihood))
def calculate_dealer_pain_level(self, row: pd.Series) -> float:
"""
Calculate dealer pain level (0-100).
Higher = dealers in pain (large gamma exposure, forced to hedge).
"""
dealer_pressure = row.get('dealer_hedge_pressure_score', 0) or 0
net_gamma = abs(row.get('net_gamma_exposure_per_symbol', 0) or 0)
gamma_flip_prox = row.get('gamma_flip_proximity')
pain = 0.0
# Dealer pressure component (0-50 points)
pain += (dealer_pressure / 100.0) * 50.0
# Gamma exposure component (0-30 points)
# Large absolute gamma = more pain
if net_gamma > 0:
normalized_gamma = min(1.0, net_gamma / 1000000000) # 1B threshold
pain += normalized_gamma * 30.0
# Gamma flip proximity component (0-20 points)
# Near flip = high pain (dealers forced to adjust)
if gamma_flip_prox is not None and not pd.isna(gamma_flip_prox):
# Absolute value of proximity (closer to 0 = more pain)
pain += (1.0 - abs(gamma_flip_prox)) * 20.0
return min(100.0, max(0.0, pain))
def calculate_expected_move_vs_implied(self, row: pd.Series) -> Optional[float]:
"""
Calculate expected move vs implied move ratio.
Estimates expected move from flow characteristics vs implied volatility.
Returns: ratio (expected_move / implied_move)
- >1.0 = flow suggests larger move than implied
- <1.0 = flow suggests smaller move than implied
- None if cannot calculate
"""
# Simplified calculation: use premium and delta exposure as proxies
premium_num = row.get('premium_num', 0) or 0
spot_num = row.get('spot_num', 0) or 0
delta_exposure = abs(row.get('delta_exposure', 0) or 0)
if pd.isna(spot_num) or spot_num == 0:
return None
# Estimate expected move from premium paid
# High premium relative to spot = expectation of larger move
if premium_num > 0:
prem_to_spot_ratio = premium_num / spot_num
# Estimate implied move (simplified: assume 1% IV = 1% move expectation)
# This is a placeholder - in production, use actual IV from options chain
implied_move_pct = 2.0 # Default 2% implied move
# Estimate expected move from premium
# Premium of 2% of spot = expectation of ~2% move (rough approximation)
expected_move_pct = prem_to_spot_ratio * 100.0
# Calculate ratio
if implied_move_pct > 0:
ratio = expected_move_pct / implied_move_pct
return float(ratio)
return None
def enrich_with_confidence_metrics(self, df: pd.DataFrame) -> pd.DataFrame:
"""
Add institutional confidence metrics to DataFrame.
Adds: confidence_score, institutional_likelihood, dealer_pain_level, expected_move_vs_implied
"""
if df.empty:
return df
df = df.copy()
# Initialize columns
df['confidence_score'] = 0.0
df['institutional_likelihood'] = 0.0
df['dealer_pain_level'] = 0.0
df['expected_move_vs_implied'] = None
# Calculate metrics
df['confidence_score'] = df.apply(self.calculate_confidence_score, axis=1)
df['institutional_likelihood'] = df.apply(self.calculate_institutional_likelihood, axis=1)
df['dealer_pain_level'] = df.apply(self.calculate_dealer_pain_level, axis=1)
# Expected move vs implied (some rows may not have this)
for idx in df.index:
expected_move = self.calculate_expected_move_vs_implied(df.iloc[idx])
if expected_move is not None:
df.at[idx, 'expected_move_vs_implied'] = float(expected_move)
logger.info(f"Confidence metrics complete. Mean confidence: {df['confidence_score'].mean():.2f}")
return df

View File

@ -0,0 +1,232 @@
"""
Intent Classification Service
Replaces naive direction (BULL/BEAR) with nuanced volatility and hedging intent classification
"""
import pandas as pd
import numpy as np
from typing import Optional
from enum import Enum
from utils.logger import logger
from utils.error_handler import safe_divide
class VolatilityIntent(Enum):
"""Volatility and hedging intent classification"""
LONG_VOL = "LONG_VOL" # Buying volatility (call/put buying, expecting large moves)
SHORT_VOL = "SHORT_VOL" # Selling volatility (premium collection, expecting low vol)
DIRECTIONAL = "DIRECTIONAL" # Directional positioning (directional bias)
HEDGE_UNWIND = "HEDGE_UNWIND" # Hedging or unwinding existing positions
class IntentClassifier:
"""
Classifies options flow intent beyond simple BULL/BEAR direction.
Identifies volatility trading, hedging, and directional positioning.
"""
def __init__(self):
# Configuration thresholds
self.otm_threshold_pct = 5.0 # Strikes >5% OTM considered OTM
self.long_vol_premium_threshold = 100000 # Minimum premium for long vol signal
self.short_vol_premium_threshold = 200000 # Minimum premium for short vol signal
def estimate_delta(self, row: pd.Series) -> float:
"""
Estimate option delta from moneyness.
Returns delta estimate (0-1 for calls, -1-0 for puts, use absolute value).
"""
spot_num = row.get('spot_num', 0) or 0
strike_num = row.get('strike_num', 0) or 0
cp_norm = row.get('cp_norm', '')
if pd.isna(spot_num) or spot_num == 0 or pd.isna(strike_num) or pd.isna(cp_norm):
return 0.5 # Default to ATM
moneyness_ratio = strike_num / spot_num if spot_num > 0 else 1.0
if cp_norm == 'CALL':
if strike_num <= spot_num:
# ITM call: delta 0.5 to 1.0
delta = 0.5 + (1.0 - moneyness_ratio) * 0.5
else:
# OTM call: delta 0.5 to 0.0
delta = max(0.0, 0.5 * (1.5 - moneyness_ratio) / 0.5)
else: # PUT
if strike_num >= spot_num:
# ITM put: delta -0.5 to -1.0 (return absolute)
delta = abs(0.5 + (moneyness_ratio - 1.0) * 0.5)
else:
# OTM put: delta -0.5 to 0.0 (return absolute)
delta = max(0.0, 0.5 * (1.0 - moneyness_ratio) / 0.5)
return float(delta)
def estimate_gamma(self, row: pd.Series, delta: float) -> float:
"""
Estimate option gamma (sensitivity of delta to price changes).
Higher near ATM, lower ITM/OTM.
Returns gamma estimate (positive value).
"""
spot_num = row.get('spot_num', 0) or 0
strike_num = row.get('strike_num', 0) or 0
if pd.isna(spot_num) or spot_num == 0 or pd.isna(strike_num):
return 0.0
# Gamma is highest at-the-money
moneyness_ratio = strike_num / spot_num if spot_num > 0 else 1.0
# Simple approximation: gamma peaks at 1.0 (ATM) and decays away
# Use normal distribution approximation
distance_from_atm = abs(moneyness_ratio - 1.0)
# Gamma ≈ exp(-distance^2 / (2*sigma^2)) where sigma ≈ 0.1 (10% moneyness)
gamma = np.exp(-(distance_from_atm ** 2) / (2 * 0.01))
return float(gamma)
def calculate_delta_exposure(self, row: pd.Series) -> float:
"""
Calculate delta exposure: contracts * delta * 100 * spot_price.
Positive = long delta (bullish), Negative = short delta (bearish).
"""
premium_num = row.get('premium_num', 0) or 0
spot_num = row.get('spot_num', 0) or 0
vol_num = row.get('vol_num', 0) or 0
cp_norm = row.get('cp_norm', '')
side_norm = row.get('side_norm', '')
if pd.isna(spot_num) or spot_num == 0:
return 0.0
# Estimate delta
delta = self.estimate_delta(row)
# Determine sign based on call/put and buy/sell
if cp_norm == 'CALL':
if side_norm == 'BUY':
delta_sign = 1.0 # Long calls = positive delta
else: # SELL
delta_sign = -1.0 # Short calls = negative delta
else: # PUT
if side_norm == 'BUY':
delta_sign = -1.0 # Long puts = negative delta
else: # SELL
delta_sign = 1.0 # Short puts = positive delta
# Delta exposure = contracts * delta * 100 * spot
# Use volume as proxy for contracts
contracts = vol_num if not pd.isna(vol_num) else 0
delta_exposure = contracts * delta * delta_sign * 100 * spot_num
return float(delta_exposure)
def calculate_gamma_exposure(self, row: pd.Series) -> float:
"""
Calculate gamma exposure: contracts * gamma * 100 * spot_price^2.
Positive = long gamma (volatility long), Negative = short gamma (volatility short).
"""
premium_num = row.get('premium_num', 0) or 0
spot_num = row.get('spot_num', 0) or 0
vol_num = row.get('vol_num', 0) or 0
cp_norm = row.get('cp_norm', '')
side_norm = row.get('side_norm', '')
if pd.isna(spot_num) or spot_num == 0:
return 0.0
# Estimate delta and gamma
delta = self.estimate_delta(row)
gamma = self.estimate_gamma(row, delta)
# Determine sign: buying options = long gamma, selling = short gamma
if side_norm == 'BUY':
gamma_sign = 1.0 # Long gamma
else: # SELL
gamma_sign = -1.0 # Short gamma
# Gamma exposure = contracts * gamma * 100 * spot^2
contracts = vol_num if not pd.isna(vol_num) else 0
gamma_exposure = contracts * gamma * gamma_sign * 100 * (spot_num ** 2)
return float(gamma_exposure)
def classify_volatility_intent(self, row: pd.Series) -> str:
"""
Classify volatility intent based on trade characteristics.
Returns VolatilityIntent enum value as string.
"""
premium_num = row.get('premium_num', 0) or 0
spot_num = row.get('spot_num', 0) or 0
strike_num = row.get('strike_num', 0) or 0
cp_norm = row.get('cp_norm', '')
side_norm = row.get('side_norm', '')
vol_num = row.get('vol_num', 0) or 0
oi_num = row.get('oi_num', 0) or 0
# Calculate moneyness
if pd.isna(spot_num) or spot_num == 0 or pd.isna(strike_num):
mny_pct = 0.0
else:
if cp_norm == 'CALL':
mny_pct = ((strike_num - spot_num) / spot_num * 100.0) if spot_num > 0 else 0
else: # PUT
mny_pct = ((spot_num - strike_num) / spot_num * 100.0) if spot_num > 0 else 0
is_otm = abs(mny_pct) > self.otm_threshold_pct
# Long volatility: buying OTM options (calls or puts)
# High premium, OTM strikes, buying side
if (side_norm == 'BUY' and
is_otm and
premium_num >= self.long_vol_premium_threshold):
return VolatilityIntent.LONG_VOL.value
# Short volatility: selling options, collecting premium
# High premium, selling side, vol > OI (opening new short positions)
if (side_norm == 'SELL' and
premium_num >= self.short_vol_premium_threshold and
vol_num > oi_num):
return VolatilityIntent.SHORT_VOL.value
# Directional: ITM options, buying side, strong directional flow
if (side_norm == 'BUY' and
not is_otm and
premium_num >= 50000):
return VolatilityIntent.DIRECTIONAL.value
# Hedge/Unwind: Selling existing positions (vol < OI)
# Or buying protective puts/calls
if (side_norm == 'SELL' and vol_num < oi_num) or \
(side_norm == 'BUY' and is_otm and premium_num < self.long_vol_premium_threshold):
return VolatilityIntent.HEDGE_UNWIND.value
# Default: directional (fallback)
return VolatilityIntent.DIRECTIONAL.value
def enrich_with_intent_classification(self, df: pd.DataFrame) -> pd.DataFrame:
"""
Add intent classification metrics to DataFrame.
Adds: delta_exposure, gamma_exposure, volatility_intent
"""
if df.empty:
return df
df = df.copy()
# Initialize columns
df['delta_exposure'] = 0.0
df['gamma_exposure'] = 0.0
df['volatility_intent'] = VolatilityIntent.DIRECTIONAL.value
# Calculate metrics
df['delta_exposure'] = df.apply(self.calculate_delta_exposure, axis=1)
df['gamma_exposure'] = df.apply(self.calculate_gamma_exposure, axis=1)
df['volatility_intent'] = df.apply(self.classify_volatility_intent, axis=1)
# Log distribution
intent_counts = df['volatility_intent'].value_counts()
logger.info(f"Intent classification complete. Distribution: {intent_counts.to_dict()}")
return df

View File

@ -0,0 +1,104 @@
"""
Market Regime Detection Service
Identifies market regime to gate trade signal generation
"""
import pandas as pd
import numpy as np
from typing import Optional
from enum import Enum
from utils.logger import logger
class MarketRegime(Enum):
"""Market regime classification"""
TREND = "TREND" # Trending market (continuation bias)
RANGE = "RANGE" # Range-bound market (fade or vol-sell bias)
HIGH_VOL_EVENT = "HIGH_VOL_EVENT" # High volatility event (volatility expansion bias)
class MarketRegimeDetector:
"""
Detects market regime to inform trade signal bias:
- Trend: continuation trades preferred
- Range: mean reversion / vol selling preferred
- High Vol Event: volatility expansion trades preferred
"""
def __init__(self):
# Configuration
self.trend_threshold_pct = 1.5 # >1.5% move = trending
self.range_threshold_pct = 0.5 # <0.5% move = ranging
self.high_vol_threshold_pct = 3.0 # >3% move = high vol event
self.lookback_minutes = 60 # Lookback window for regime detection
def detect_regime(
self,
df: pd.DataFrame,
row_idx: int
) -> str:
"""
Detect market regime for a given flow event.
Returns MarketRegime enum value as string.
"""
if row_idx >= len(df):
return MarketRegime.RANGE.value
current_row = df.iloc[row_idx]
symbol = current_row.get('symbol_norm')
flow_ts_utc = current_row.get('flow_ts_utc')
if pd.isna(symbol) or pd.isna(flow_ts_utc):
return MarketRegime.RANGE.value
# Get price movement over lookback window
pct_vs_rth_open = current_row.get('pct_vs_rth_open')
pct_vs_prior_close = current_row.get('pct_vs_prior_close')
pct_15m_momo = current_row.get('pct_15m_momo')
# Use most appropriate price metric
price_move_pct = None
if pct_vs_rth_open is not None and not pd.isna(pct_vs_rth_open):
price_move_pct = abs(pct_vs_rth_open)
elif pct_vs_prior_close is not None and not pd.isna(pct_vs_prior_close):
price_move_pct = abs(pct_vs_prior_close)
elif pct_15m_momo is not None and not pd.isna(pct_15m_momo):
price_move_pct = abs(pct_15m_momo)
if price_move_pct is None:
return MarketRegime.RANGE.value # Default to range
# Classify regime
if price_move_pct >= self.high_vol_threshold_pct:
return MarketRegime.HIGH_VOL_EVENT.value
elif price_move_pct >= self.trend_threshold_pct:
return MarketRegime.TREND.value
elif price_move_pct <= self.range_threshold_pct:
return MarketRegime.RANGE.value
else:
# Between range and trend threshold - classify based on momentum
if pct_15m_momo is not None and not pd.isna(pct_15m_momo):
if abs(pct_15m_momo) > 0.75:
return MarketRegime.TREND.value
return MarketRegime.RANGE.value
def enrich_with_market_regime(self, df: pd.DataFrame) -> pd.DataFrame:
"""
Add market regime classification to DataFrame.
Adds: market_regime
"""
if df.empty:
return df
df = df.copy()
df['market_regime'] = MarketRegime.RANGE.value
# Detect regime for each row
for idx in df.index:
regime = self.detect_regime(df, idx)
df.at[idx, 'market_regime'] = regime
regime_counts = df['market_regime'].value_counts()
logger.info(f"Market regime detection complete. Distribution: {regime_counts.to_dict()}")
return df

View File

@ -0,0 +1,213 @@
"""
Tier-0 Noise Rejection Service
Filters out low-quality signals before enrichment to reduce processing overhead
"""
import pandas as pd
import numpy as np
from typing import Optional
from datetime import timedelta
from utils.logger import logger
from utils.error_handler import safe_divide
class NoiseRejector:
"""
Rejects early-stage noise before enrichment to optimize processing.
Rejects if:
- Single isolated trade (no repeat activity)
- Far OTM weekly lottos
- Delta-adjusted premium below threshold
"""
def __init__(self):
# Configuration
self.repeat_activity_window_minutes = 30 # Minutes to look for repeat activity
self.min_delta_adjusted_premium = 50000 # Minimum delta-adjusted premium
self.max_otm_percentage = 15.0 # Reject strikes >15% OTM
self.min_expiry_days = 1 # Minimum days to expiry (reject same-day/weekly lottos)
def is_isolated_trade(
self,
df: pd.DataFrame,
row_idx: int
) -> bool:
"""
Check if trade is isolated (no repeat activity within window).
Returns True if isolated (should reject).
"""
if row_idx >= len(df):
return True
current_row = df.iloc[row_idx]
symbol = current_row.get('symbol_norm')
direction = current_row.get('direction')
flow_ts_utc = current_row.get('flow_ts_utc')
if pd.isna(symbol) or pd.isna(flow_ts_utc):
return True
# Look for trades in same direction within window
window_start = flow_ts_utc - timedelta(minutes=self.repeat_activity_window_minutes)
same_symbol = df['symbol_norm'] == symbol
same_direction = df['direction'] == direction
in_window = (df['flow_ts_utc'] >= window_start) & (df['flow_ts_utc'] < flow_ts_utc)
# Exclude current row
other_trades = df[same_symbol & same_direction & in_window]
# If no other trades in window, it's isolated
return len(other_trades) == 0
def is_far_otm_lotto(
self,
row: pd.Series
) -> bool:
"""
Check if trade is a far OTM weekly lotto (low probability, low signal value).
Returns True if should reject.
"""
spot_num = row.get('spot_num', 0) or 0
strike_num = row.get('strike_num', 0) or 0
cp_norm = row.get('cp_norm', '')
exp_date = row.get('exp_date')
flow_date = row.get('flow_date_cst')
if pd.isna(spot_num) or spot_num == 0 or pd.isna(strike_num) or pd.isna(cp_norm):
return False
# Calculate moneyness percentage
if cp_norm == 'CALL':
otm_pct = ((strike_num - spot_num) / spot_num * 100.0) if spot_num > 0 else 0
else: # PUT
otm_pct = ((spot_num - strike_num) / spot_num * 100.0) if spot_num > 0 else 0
# Reject if >15% OTM
if otm_pct > self.max_otm_percentage:
return True
# Check if weekly lotto (expiry within 7 days)
if pd.notna(exp_date) and pd.notna(flow_date):
try:
if isinstance(exp_date, str):
from datetime import datetime
exp_date = datetime.strptime(exp_date, '%Y-%m-%d').date()
if isinstance(flow_date, str):
from datetime import datetime
flow_date = datetime.strptime(flow_date, '%Y-%m-%d').date()
days_to_expiry = (exp_date - flow_date).days
# Reject weekly lottos that are also far OTM
if days_to_expiry <= 7 and otm_pct > 10.0:
return True
except (ValueError, TypeError, AttributeError):
pass
return False
def calculate_delta_adjusted_premium(
self,
row: pd.Series
) -> float:
"""
Calculate delta-adjusted premium (premium * |delta|).
Approximates intrinsic value component of premium.
For simplicity, we estimate delta from moneyness:
- ATM: delta 0.5
- ITM: delta increases toward 1.0
- OTM: delta decreases toward 0.0
"""
premium_num = row.get('premium_num', 0) or 0
spot_num = row.get('spot_num', 0) or 0
strike_num = row.get('strike_num', 0) or 0
cp_norm = row.get('cp_norm', '')
if pd.isna(premium_num) or premium_num == 0 or pd.isna(spot_num) or spot_num == 0:
return 0.0
# Estimate delta from moneyness
if cp_norm == 'CALL':
if strike_num <= spot_num:
# ITM call: delta from 0.5 to 1.0
moneyness_ratio = strike_num / spot_num if spot_num > 0 else 1.0
# Strike at spot = 0.5, strike at 0 = 1.0
estimated_delta = 0.5 + (1.0 - moneyness_ratio) * 0.5
else:
# OTM call: delta from 0.5 to 0.0
moneyness_ratio = strike_num / spot_num if spot_num > 0 else 1.0
# Strike at spot = 0.5, strike at 1.15x spot = 0.0
estimated_delta = max(0.0, 0.5 * (1.5 - moneyness_ratio) / 0.5)
else: # PUT
if strike_num >= spot_num:
# ITM put: delta from -0.5 to -1.0 (use absolute value)
moneyness_ratio = strike_num / spot_num if spot_num > 0 else 1.0
estimated_delta = 0.5 + (moneyness_ratio - 1.0) * 0.5
else:
# OTM put: delta from 0.5 to 0.0 (use absolute value)
moneyness_ratio = strike_num / spot_num if spot_num > 0 else 1.0
estimated_delta = max(0.0, 0.5 * (1.0 - moneyness_ratio) / 0.5)
# Delta-adjusted premium
delta_adj_premium = premium_num * abs(estimated_delta)
return float(delta_adj_premium)
def should_reject(
self,
df: pd.DataFrame,
row_idx: int
) -> bool:
"""
Determine if row should be rejected as noise.
Returns True if should reject (mark early_noise_reject = True).
"""
if row_idx >= len(df):
return True
row = df.iloc[row_idx]
# Reject isolated trades
if self.is_isolated_trade(df, row_idx):
return True
# Reject far OTM lottos
if self.is_far_otm_lotto(row):
return True
# Reject if delta-adjusted premium below threshold
delta_adj_premium = self.calculate_delta_adjusted_premium(row)
if delta_adj_premium < self.min_delta_adjusted_premium:
return True
return False
def mark_noise_rejections(self, df: pd.DataFrame) -> pd.DataFrame:
"""
Mark rows for early noise rejection.
Adds: early_noise_reject (boolean)
"""
if df.empty:
return df
df = df.copy()
# Initialize column
df['early_noise_reject'] = False
# Sort by timestamp for proper isolation detection
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
# Mark rejections
for idx in df_sorted.index:
if self.should_reject(df_sorted, idx):
original_idx = df_sorted.index[idx]
df.at[original_idx, 'early_noise_reject'] = True
rejection_count = df['early_noise_reject'].sum()
logger.info(f"Noise rejection: {rejection_count}/{len(df)} rows marked for rejection ({rejection_count/len(df)*100:.1f}%)")
return df

View File

@ -146,5 +146,14 @@ class OutputFormatter:
# Ensure Phase 1 columns are preserved (don't drop them)
# Phase 1 columns should remain as-is (signal_tier, checklist_score, etc.)
# Ensure all new institutional analytics columns are preserved:
# - premium_zscore, premium_percentile_intraday, relative_premium_score
# - aggression_score, size_concentration_score, repeat_trade_velocity, strike_clustering_score, signal_strength
# - early_noise_reject, flow_acceleration, time_between_hits, follow_on_ratio, strike_laddering_detected
# - delta_exposure, gamma_exposure, volatility_intent
# - net_gamma_exposure_per_symbol, gamma_flip_proximity, dealer_hedge_pressure_score
# - market_regime, flow_state
# - confidence_score, institutional_likelihood, dealer_pain_level, expected_move_vs_implied
return df

View File

@ -0,0 +1,258 @@
"""
Relative Premium Scoring Service
Replaces static premium filter with context-aware relative premium scoring
"""
import pandas as pd
import numpy as np
from typing import Dict, Optional
from datetime import datetime, timedelta
import asyncpg
from utils.logger import logger
from utils.error_handler import safe_divide
class RelativePremiumScorer:
"""
Computes relative premium metrics to replace static premium filtering.
Premium of $80K might be significant for AAPL but noise for TSLA.
"""
def __init__(self, pool: asyncpg.Pool = None):
self.pool = pool
# Configuration
self.rolling_window_days = 20 # Rolling window for z-score calculation
self.min_relative_premium_threshold = 0.60 # 60th percentile minimum
async def fetch_historical_premium_stats(
self,
symbol: str,
reference_date: datetime.date
) -> Dict[str, float]:
"""
Fetch historical premium statistics for a symbol.
Returns: {mean, std, min, max, median} for rolling window
"""
if not self.pool:
logger.warning("No database pool available for historical premium stats")
return {}
try:
# Calculate window start date
window_start = reference_date - timedelta(days=self.rolling_window_days + 5) # Extra buffer
async with self.pool.acquire() as conn:
query = """
SELECT
AVG(premium_num) as mean_premium,
STDDEV(premium_num) as std_premium,
MIN(premium_num) as min_premium,
MAX(premium_num) as max_premium,
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY premium_num) as median_premium
FROM (
SELECT
CAST(REGEXP_REPLACE("Premium"::text, '[^\d.]', '', 'g') AS numeric) as premium_num
FROM "OptionsFlow_monthly"
WHERE UPPER(TRIM("Symbol")) = $1
AND "Premium" IS NOT NULL
AND TRIM("Premium"::text) <> ''
AND "CreatedDate" >= $2
AND "CreatedDate" <= $3
AND "StockEtf" = 'STOCK'
) subq
WHERE premium_num > 0
"""
row = await conn.fetchrow(
query,
symbol.upper(),
window_start.strftime('%Y-%m-%d'),
reference_date.strftime('%Y-%m-%d')
)
if row and row['mean_premium']:
return {
'mean': float(row['mean_premium']) if row['mean_premium'] else 0.0,
'std': float(row['std_premium']) if row['std_premium'] else 0.0,
'min': float(row['min_premium']) if row['min_premium'] else 0.0,
'max': float(row['max_premium']) if row['max_premium'] else 0.0,
'median': float(row['median_premium']) if row['median_premium'] else 0.0,
}
except Exception as e:
logger.debug(f"Error fetching historical premium stats for {symbol}: {e}")
return {}
async def calculate_intraday_percentile(
self,
df: pd.DataFrame,
symbol: str,
flow_date: datetime.date,
premium: float
) -> Optional[float]:
"""
Calculate percentile rank of premium within same-day flow for the symbol.
Returns 0-100 percentile value.
"""
try:
# Filter to same symbol and date
same_day_flow = df[
(df['symbol_norm'] == symbol.upper()) &
(df['flow_date_cst'] == flow_date) &
(df['premium_num'].notna()) &
(df['premium_num'] > 0)
]
if len(same_day_flow) < 3: # Need at least 3 trades for meaningful percentile
return None
# Calculate percentile rank
all_premiums = same_day_flow['premium_num'].values
percentile = (np.sum(all_premiums <= premium) / len(all_premiums)) * 100.0
return float(percentile)
except Exception as e:
logger.debug(f"Error calculating intraday percentile for {symbol}: {e}")
return None
def calculate_zscore(
self,
premium: float,
stats: Dict[str, float]
) -> Optional[float]:
"""
Calculate z-score of premium relative to historical distribution.
Returns z-score (standard deviations from mean).
"""
if not stats or stats.get('std', 0) == 0:
return None
mean = stats.get('mean', 0)
std = stats.get('std', 1)
if std == 0:
return None
zscore = (premium - mean) / std
return float(zscore)
def calculate_relative_premium_score(
self,
premium: float,
zscore: Optional[float],
intraday_percentile: Optional[float],
stats: Dict[str, float]
) -> float:
"""
Calculate composite relative premium score (0-100).
Combines z-score and intraday percentile with median normalization.
Logic:
- High z-score (unusual size) = higher score
- High intraday percentile (large relative to today's flow) = higher score
- Normalize by median to account for symbol-specific scaling
"""
score = 0.0
# Z-score component (40% weight)
# Z-score > 1 = 1 std above mean, Z-score > 2 = 2 std above mean
if zscore is not None:
# Normalize z-score to 0-40 range (clamp at ±3 sigma)
zscore_normalized = max(0, min(40, (zscore + 3) * (40 / 6)))
score += zscore_normalized
# Intraday percentile component (40% weight)
if intraday_percentile is not None:
# Percentile is already 0-100, scale to 0-40
score += (intraday_percentile / 100.0) * 40.0
# Median normalization component (20% weight)
# Premium > median gets bonus, premium < median gets penalty
if stats and stats.get('median', 0) > 0:
median_ratio = premium / stats['median']
# Ratio > 1.5 = strong, ratio > 2.0 = very strong
if median_ratio >= 2.0:
score += 20.0
elif median_ratio >= 1.5:
score += 15.0
elif median_ratio >= 1.0:
score += 10.0
elif median_ratio >= 0.5:
score += 5.0
return min(100.0, max(0.0, score))
async def enrich_with_relative_premium(
self,
df: pd.DataFrame
) -> pd.DataFrame:
"""
Add relative premium metrics to DataFrame.
Adds: premium_zscore, premium_percentile_intraday, relative_premium_score
"""
if df.empty:
return df
df = df.copy()
# Initialize new columns
df['premium_zscore'] = None
df['premium_percentile_intraday'] = None
df['relative_premium_score'] = 0.0
# Cache historical stats per symbol to avoid repeated queries
stats_cache: Dict[str, Dict[str, float]] = {}
# Group by symbol and date for batch processing
unique_symbols = df['symbol_norm'].unique()
unique_dates = df['flow_date_cst'].unique()
logger.info(f"Calculating relative premium scores for {len(unique_symbols)} symbols")
for symbol in unique_symbols:
# Fetch historical stats once per symbol
if self.pool:
# Use the most recent date for this symbol as reference
symbol_dates = df[df['symbol_norm'] == symbol]['flow_date_cst'].unique()
if len(symbol_dates) > 0:
reference_date = max(symbol_dates)
stats = await self.fetch_historical_premium_stats(symbol, reference_date)
stats_cache[symbol] = stats
else:
stats_cache[symbol] = {}
else:
stats_cache[symbol] = {}
# Calculate metrics for each row
for idx, row in df.iterrows():
premium = row.get('premium_num')
if pd.isna(premium) or premium <= 0:
continue
symbol = row['symbol_norm']
flow_date = row.get('flow_date_cst')
# Get historical stats
stats = stats_cache.get(symbol, {})
# Calculate z-score
zscore = self.calculate_zscore(premium, stats)
if zscore is not None:
df.at[idx, 'premium_zscore'] = float(zscore)
# Calculate intraday percentile (using current DataFrame subset)
intraday_percentile = self.calculate_intraday_percentile(
df, symbol, flow_date, premium
)
if intraday_percentile is not None:
df.at[idx, 'premium_percentile_intraday'] = float(intraday_percentile)
# Calculate composite relative premium score
relative_score = self.calculate_relative_premium_score(
premium, zscore, intraday_percentile, stats
)
df.at[idx, 'relative_premium_score'] = float(relative_score)
logger.info(f"Relative premium scoring complete. Mean score: {df['relative_premium_score'].mean():.2f}")
return df

View File

@ -0,0 +1,331 @@
"""
Signal Component Scorer
Converts binary badge logic into continuous numeric signal components
"""
import pandas as pd
import numpy as np
from typing import Dict, Optional
from utils.logger import logger
class SignalComponentScorer:
"""
Replaces binary badge logic (💎 🟢 🔴) with continuous numeric scores.
Badges remain display-only, but signal_strength is computed from components.
"""
def __init__(self):
# Signal strength weights (must sum to 1.0)
self.aggression_weight = 0.30
self.size_concentration_weight = 0.30
self.repeat_velocity_weight = 0.20
self.strike_clustering_weight = 0.20
def calculate_aggression_score(self, row: pd.Series) -> float:
"""
Measures trade aggression: premium paid at ask vs bid, ITM vs OTM preference.
Higher score = more aggressive buying:
- ITM premiums indicate willingness to pay intrinsic value
- Ask-side trades (AA) indicate market orders
- Large premium relative to spot price
"""
score = 0.0
premium_num = row.get('premium_num', 0) or 0
spot_num = row.get('spot_num', 0) or 0
cp_norm = row.get('cp_norm', '')
side = str(row.get('Side', '')).upper()
# ITM premium component (0-40 points)
bull_prem_itm = row.get('bull_prem_itm', 0) or 0
bear_prem_itm = row.get('bear_prem_itm', 0) or 0
total_prem_itm = bull_prem_itm + bear_prem_itm
bull_total = row.get('bull_total', 0) or 0
bear_total = row.get('bear_total', 0) or 0
total_premium = bull_total + bear_total
if total_premium > 0:
itm_ratio = total_prem_itm / total_premium
score += itm_ratio * 40.0
# Ask-side aggression component (0-30 points)
# AA = buying at ask (market orders, urgency)
has_aa = 'AA' in side or 'AT_ASK' in side
has_bb = 'BB' in side or 'AT_BID' in side
if has_aa and not has_bb:
score += 30.0
elif has_aa:
score += 15.0
# Premium-to-spot ratio component (0-30 points)
# Large premium relative to spot indicates aggressive positioning
if spot_num > 0 and premium_num > 0:
prem_to_spot_ratio = premium_num / spot_num
# Normalize: 0.01 (1%) = 10 points, 0.05 (5%) = 30 points
ratio_score = min(30.0, prem_to_spot_ratio * 600.0)
score += ratio_score
return min(100.0, max(0.0, score))
def calculate_size_concentration_score(self, row: pd.Series) -> float:
"""
Measures size concentration: how much premium is concentrated in single strikes/expiries.
Higher score = more concentrated (single large trade vs many small ones):
- High premium in single trade
- OI buildup in specific strikes
- Directional consistency (all calls or all puts)
"""
score = 0.0
premium_num = row.get('premium_num', 0) or 0
bull_total = row.get('bull_total', 0) or 0
bear_total = row.get('bear_total', 0) or 0
total_premium = bull_total + bear_total
# Single-trade premium concentration (0-40 points)
# Premium of current trade relative to running total
if total_premium > 0:
concentration_ratio = premium_num / total_premium
# Single trade = 50%+ of total = very concentrated
if concentration_ratio >= 0.5:
score += 40.0
elif concentration_ratio >= 0.3:
score += 30.0
elif concentration_ratio >= 0.2:
score += 20.0
elif concentration_ratio >= 0.1:
score += 10.0
# OI concentration component (0-30 points)
# High OI in OTM strikes indicates concentrated positioning
oi_cb_otm = row.get('oi_cb_otm', 0) or 0
oi_pb_otm = row.get('oi_pb_otm', 0) or 0
oi_all = row.get('oi_all', 0) or 0
if oi_all > 0:
otm_oi_ratio = (oi_cb_otm + oi_pb_otm) / oi_all
score += otm_oi_ratio * 30.0
# Directional consistency (0-30 points)
# Pure directional flow (all bull or all bear) = more concentrated
if total_premium > 0:
direction_strength = abs(bull_total - bear_total) / total_premium
score += direction_strength * 30.0
return min(100.0, max(0.0, score))
def calculate_repeat_trade_velocity(self, df: pd.DataFrame, row_idx: int) -> float:
"""
Measures repeat trade velocity: frequency of trades in same direction/expiry.
Higher score = faster follow-on trades (urgency building):
- Time between consecutive trades decreasing
- Multiple trades in same direction
- Same expiry date (rolling accumulation)
"""
if row_idx >= len(df):
return 0.0
current_row = df.iloc[row_idx]
symbol = current_row.get('symbol_norm')
direction = current_row.get('direction')
exp_date = current_row.get('exp_date')
flow_ts_utc = current_row.get('flow_ts_utc')
if pd.isna(symbol) or pd.isna(direction) or pd.isna(flow_ts_utc):
return 0.0
# Look back at recent trades for same symbol
# Sort by timestamp to find preceding trades
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
current_idx = df_sorted.index[df_sorted.index == row_idx]
if len(current_idx) == 0:
return 0.0
current_pos = current_idx[0]
# Look at last 10 trades for this symbol
symbol_mask = (df_sorted['symbol_norm'] == symbol) & \
(df_sorted.index < current_pos) & \
(df_sorted['flow_ts_utc'] <= flow_ts_utc)
recent_trades = df_sorted[symbol_mask].tail(10)
if len(recent_trades) == 0:
return 0.0
score = 0.0
# Time compression component (0-40 points)
# Decreasing time gaps = urgency building
if len(recent_trades) >= 2:
time_gaps = []
prev_ts = None
for _, trade in recent_trades.iterrows():
ts = trade.get('flow_ts_utc')
if pd.notna(ts) and prev_ts:
gap_minutes = (ts - prev_ts).total_seconds() / 60.0
if gap_minutes > 0:
time_gaps.append(gap_minutes)
prev_ts = ts
if len(time_gaps) >= 2:
# Check if gaps are decreasing (accelerating)
recent_gaps = time_gaps[-2:]
if recent_gaps[1] < recent_gaps[0]:
compression_ratio = recent_gaps[1] / recent_gaps[0] if recent_gaps[0] > 0 else 0
# 50% compression = 20 points, 80% compression = 40 points
score += (1.0 - compression_ratio) * 40.0
# Directional consistency (0-35 points)
# Same direction trades = building position
same_direction = recent_trades[recent_trades['direction'] == direction]
if len(recent_trades) > 0:
consistency_ratio = len(same_direction) / len(recent_trades)
score += consistency_ratio * 35.0
# Expiry concentration (0-25 points)
# Same expiry = rolling accumulation
if pd.notna(exp_date):
same_expiry = recent_trades[recent_trades['exp_date'] == exp_date]
if len(recent_trades) > 0:
expiry_ratio = len(same_expiry) / len(recent_trades)
score += expiry_ratio * 25.0
return min(100.0, max(0.0, score))
def calculate_strike_clustering_score(self, df: pd.DataFrame, row_idx: int) -> float:
"""
Measures strike clustering: trades accumulating at specific strike levels.
Higher score = more clustering (laddering or concentrated strikes):
- Multiple trades at same strike
- Sequential strikes (laddering)
- Strikes near current price (pin risk)
"""
if row_idx >= len(df):
return 0.0
current_row = df.iloc[row_idx]
symbol = current_row.get('symbol_norm')
strike_num = current_row.get('strike_num')
spot_num = current_row.get('spot_num', 0) or 0
flow_ts_utc = current_row.get('flow_ts_utc')
exp_date = current_row.get('exp_date')
if pd.isna(symbol) or pd.isna(strike_num) or pd.isna(flow_ts_utc):
return 0.0
# Look at recent trades for same symbol and expiry
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
symbol_mask = (df_sorted['symbol_norm'] == symbol) & \
(df_sorted['flow_ts_utc'] <= flow_ts_utc)
if pd.notna(exp_date):
symbol_mask = symbol_mask & (df_sorted['exp_date'] == exp_date)
recent_trades = df_sorted[symbol_mask].tail(20)
if len(recent_trades) <= 1:
return 0.0
score = 0.0
# Exact strike clustering (0-40 points)
# Multiple trades at same strike
same_strike_count = len(recent_trades[recent_trades['strike_num'] == strike_num])
if len(recent_trades) > 0:
clustering_ratio = same_strike_count / len(recent_trades)
score += clustering_ratio * 40.0
# Strike laddering detection (0-35 points)
# Sequential strikes (e.g., 100, 105, 110) indicate laddering
unique_strikes = sorted(recent_trades['strike_num'].dropna().unique())
if len(unique_strikes) >= 3:
# Check if strikes are roughly evenly spaced (laddering)
if spot_num > 0:
strike_diffs = [abs(unique_strikes[i+1] - unique_strikes[i]) for i in range(len(unique_strikes)-1)]
if len(strike_diffs) > 0:
avg_diff = np.mean(strike_diffs)
std_diff = np.std(strike_diffs)
# Low std relative to mean = regular spacing (laddering)
if avg_diff > 0:
regularity = 1.0 - min(1.0, std_diff / avg_diff)
score += regularity * 35.0
# Proximity to spot (0-25 points)
# Strikes near spot price = pin risk, more significant
if spot_num > 0:
strike_pct_diff = abs(strike_num - spot_num) / spot_num * 100.0
# Within 2% = 25 points, within 5% = 15 points, within 10% = 5 points
if strike_pct_diff <= 2.0:
score += 25.0
elif strike_pct_diff <= 5.0:
score += 15.0
elif strike_pct_diff <= 10.0:
score += 5.0
return min(100.0, max(0.0, score))
def calculate_signal_strength(self, row: pd.Series) -> float:
"""
Calculate composite signal_strength from component scores.
signal_strength =
0.30 * aggression_score +
0.30 * size_concentration_score +
0.20 * repeat_trade_velocity +
0.20 * strike_clustering_score
"""
aggression = row.get('aggression_score', 0) or 0
size_conc = row.get('size_concentration_score', 0) or 0
repeat_vel = row.get('repeat_trade_velocity', 0) or 0
strike_clust = row.get('strike_clustering_score', 0) or 0
signal_strength = (
self.aggression_weight * aggression +
self.size_concentration_weight * size_conc +
self.repeat_velocity_weight * repeat_vel +
self.strike_clustering_weight * strike_clust
)
return min(100.0, max(0.0, signal_strength))
def enrich_with_signal_components(self, df: pd.DataFrame) -> pd.DataFrame:
"""
Add signal component scores to DataFrame.
Adds: aggression_score, size_concentration_score, repeat_trade_velocity,
strike_clustering_score, signal_strength
"""
if df.empty:
return df
df = df.copy()
# Initialize columns
df['aggression_score'] = 0.0
df['size_concentration_score'] = 0.0
df['repeat_trade_velocity'] = 0.0
df['strike_clustering_score'] = 0.0
df['signal_strength'] = 0.0
# Calculate component scores (some require DataFrame context)
df['aggression_score'] = df.apply(self.calculate_aggression_score, axis=1)
df['size_concentration_score'] = df.apply(self.calculate_size_concentration_score, axis=1)
# These require DataFrame context, so iterate
for idx in df.index:
df.at[idx, 'repeat_trade_velocity'] = self.calculate_repeat_trade_velocity(df, idx)
df.at[idx, 'strike_clustering_score'] = self.calculate_strike_clustering_score(df, idx)
# Calculate composite signal_strength
df['signal_strength'] = df.apply(self.calculate_signal_strength, axis=1)
logger.info(f"Signal component scoring complete. Mean signal_strength: {df['signal_strength'].mean():.2f}")
return df

View File

@ -0,0 +1,319 @@
"""
Time-Sequenced Flow Analysis Service
Analyzes flow patterns over time to detect urgency, distribution, and continuation signals
"""
import pandas as pd
import numpy as np
from typing import Dict, Optional
from datetime import timedelta
from utils.logger import logger
class TimeSequencedAnalyzer:
"""
Analyzes flow patterns over time to detect:
- Flow acceleration (urgency building)
- Distribution patterns (flow weakening)
- Strike laddering (sequential accumulation)
"""
def __init__(self):
# Configuration
self.analysis_window_minutes = 60 # Look back window for flow analysis
self.min_trades_for_analysis = 3 # Minimum trades needed for meaningful analysis
def calculate_flow_acceleration(
self,
df: pd.DataFrame,
row_idx: int
) -> Optional[float]:
"""
Calculate flow acceleration: change in premium per minute.
Positive = accelerating (urgency building)
Negative = decelerating (flow weakening)
Returns: Δ premium / minute (premium rate change)
"""
if row_idx >= len(df):
return None
current_row = df.iloc[row_idx]
symbol = current_row.get('symbol_norm')
direction = current_row.get('direction')
flow_ts_utc = current_row.get('flow_ts_utc')
if pd.isna(symbol) or pd.isna(direction) or pd.isna(flow_ts_utc):
return None
# Sort by timestamp
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
# Look at recent trades for same symbol and direction
window_start = flow_ts_utc - timedelta(minutes=self.analysis_window_minutes)
mask = (
(df_sorted['symbol_norm'] == symbol) &
(df_sorted['direction'] == direction) &
(df_sorted['flow_ts_utc'] >= window_start) &
(df_sorted['flow_ts_utc'] <= flow_ts_utc)
)
recent_trades = df_sorted[mask]
if len(recent_trades) < self.min_trades_for_analysis:
return None
# Calculate premium accumulation over time
recent_trades = recent_trades.sort_values('flow_ts_utc')
recent_trades = recent_trades.copy()
recent_trades['cumulative_premium'] = recent_trades['premium_num'].cumsum()
# Fit linear trend to cumulative premium vs time
# Convert timestamps to minutes since window start
recent_trades['minutes_since_start'] = (
recent_trades['flow_ts_utc'] - window_start
).dt.total_seconds() / 60.0
# Calculate slope (premium per minute)
if len(recent_trades) >= 2:
x = recent_trades['minutes_since_start'].values
y = recent_trades['cumulative_premium'].values
# Linear regression: y = mx + b, we want m (slope)
# Calculate acceleration as change in slope (second derivative approximation)
if len(recent_trades) >= 3:
# Split into two halves
mid_idx = len(recent_trades) // 2
first_half = recent_trades.iloc[:mid_idx]
second_half = recent_trades.iloc[mid_idx:]
if len(first_half) >= 2 and len(second_half) >= 2:
# Calculate slopes for each half
x1 = first_half['minutes_since_start'].values
y1 = first_half['cumulative_premium'].values
slope1 = np.polyfit(x1, y1, 1)[0] if len(x1) > 1 else 0
x2 = second_half['minutes_since_start'].values
y2 = second_half['cumulative_premium'].values
slope2 = np.polyfit(x2, y2, 1)[0] if len(x2) > 1 else 0
# Acceleration = change in slope
acceleration = slope2 - slope1
return float(acceleration)
return None
def calculate_time_between_hits(
self,
df: pd.DataFrame,
row_idx: int
) -> Optional[float]:
"""
Calculate average time between consecutive trades (in minutes).
Lower = faster pace, higher urgency.
"""
if row_idx >= len(df):
return None
current_row = df.iloc[row_idx]
symbol = current_row.get('symbol_norm')
direction = current_row.get('direction')
flow_ts_utc = current_row.get('flow_ts_utc')
if pd.isna(symbol) or pd.isna(direction) or pd.isna(flow_ts_utc):
return None
# Sort by timestamp
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
# Look at recent trades
window_start = flow_ts_utc - timedelta(minutes=self.analysis_window_minutes)
mask = (
(df_sorted['symbol_norm'] == symbol) &
(df_sorted['direction'] == direction) &
(df_sorted['flow_ts_utc'] >= window_start) &
(df_sorted['flow_ts_utc'] <= flow_ts_utc)
)
recent_trades = df_sorted[mask].sort_values('flow_ts_utc')
if len(recent_trades) < 2:
return None
# Calculate time gaps between consecutive trades
time_gaps = []
prev_ts = None
for _, trade in recent_trades.iterrows():
ts = trade.get('flow_ts_utc')
if pd.notna(ts) and prev_ts:
gap_minutes = (ts - prev_ts).total_seconds() / 60.0
if gap_minutes > 0:
time_gaps.append(gap_minutes)
prev_ts = ts
if len(time_gaps) == 0:
return None
avg_time_between = np.mean(time_gaps)
return float(avg_time_between)
def calculate_follow_on_ratio(
self,
df: pd.DataFrame,
row_idx: int
) -> Optional[float]:
"""
Calculate follow-on ratio: fraction of trades in same direction after initial trade.
Higher = continuation, Lower = reversal/distribution.
Returns: ratio of same-direction trades / total trades (0-1)
"""
if row_idx >= len(df):
return None
current_row = df.iloc[row_idx]
symbol = current_row.get('symbol_norm')
direction = current_row.get('direction')
flow_ts_utc = current_row.get('flow_ts_utc')
if pd.isna(symbol) or pd.isna(direction) or pd.isna(flow_ts_utc):
return None
# Sort by timestamp
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
# Look at trades after this one (within window)
window_end = flow_ts_utc + timedelta(minutes=self.analysis_window_minutes)
mask = (
(df_sorted['symbol_norm'] == symbol) &
(df_sorted['flow_ts_utc'] > flow_ts_utc) &
(df_sorted['flow_ts_utc'] <= window_end)
)
follow_on_trades = df_sorted[mask]
if len(follow_on_trades) == 0:
return None
# Count same-direction vs opposite-direction
same_direction = follow_on_trades[follow_on_trades['direction'] == direction]
follow_on_ratio = len(same_direction) / len(follow_on_trades) if len(follow_on_trades) > 0 else 0.0
return float(follow_on_ratio)
def detect_strike_laddering(
self,
df: pd.DataFrame,
row_idx: int
) -> bool:
"""
Detect strike laddering: sequential strikes in same direction.
Returns True if laddering pattern detected.
"""
if row_idx >= len(df):
return False
current_row = df.iloc[row_idx]
symbol = current_row.get('symbol_norm')
direction = current_row.get('direction')
cp_norm = current_row.get('cp_norm')
flow_ts_utc = current_row.get('flow_ts_utc')
exp_date = current_row.get('exp_date')
if pd.isna(symbol) or pd.isna(direction) or pd.isna(cp_norm):
return False
# Sort by timestamp
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
# Look at recent trades
window_start = flow_ts_utc - timedelta(minutes=self.analysis_window_minutes)
mask = (
(df_sorted['symbol_norm'] == symbol) &
(df_sorted['direction'] == direction) &
(df_sorted['cp_norm'] == cp_norm) &
(df_sorted['flow_ts_utc'] >= window_start) &
(df_sorted['flow_ts_utc'] <= flow_ts_utc)
)
if pd.notna(exp_date):
mask = mask & (df_sorted['exp_date'] == exp_date)
recent_trades = df_sorted[mask].sort_values('flow_ts_utc')
if len(recent_trades) < 3:
return False
# Get unique strikes in order
strikes = recent_trades['strike_num'].dropna().unique()
strikes = sorted(strikes)
if len(strikes) < 3:
return False
# Check if strikes are sequential (laddering)
# Look for consistent spacing (e.g., 100, 105, 110 or 50, 55, 60)
strike_diffs = [strikes[i+1] - strikes[i] for i in range(len(strikes)-1)]
if len(strike_diffs) >= 2:
# Check if differences are roughly equal (within 20% variance)
avg_diff = np.mean(strike_diffs)
if avg_diff > 0:
std_diff = np.std(strike_diffs)
cv = std_diff / avg_diff if avg_diff > 0 else float('inf') # Coefficient of variation
# Low coefficient of variation = regular spacing = laddering
if cv < 0.2: # Less than 20% variation
return True
return False
def enrich_with_time_sequenced_metrics(self, df: pd.DataFrame) -> pd.DataFrame:
"""
Add time-sequenced flow metrics to DataFrame.
Adds: flow_acceleration, time_between_hits, follow_on_ratio, strike_laddering_detected
"""
if df.empty:
return df
df = df.copy()
# Initialize columns
df['flow_acceleration'] = None
df['time_between_hits'] = None
df['follow_on_ratio'] = None
df['strike_laddering_detected'] = False
# Sort by timestamp for proper analysis
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
# Calculate metrics for each row
for idx in df_sorted.index:
original_idx = df_sorted.index[idx]
# Flow acceleration
acceleration = self.calculate_flow_acceleration(df_sorted, idx)
if acceleration is not None:
df.at[original_idx, 'flow_acceleration'] = float(acceleration)
# Time between hits
time_between = self.calculate_time_between_hits(df_sorted, idx)
if time_between is not None:
df.at[original_idx, 'time_between_hits'] = float(time_between)
# Follow-on ratio
follow_on = self.calculate_follow_on_ratio(df_sorted, idx)
if follow_on is not None:
df.at[original_idx, 'follow_on_ratio'] = float(follow_on)
# Strike laddering
laddering = self.detect_strike_laddering(df_sorted, idx)
df.at[original_idx, 'strike_laddering_detected'] = bool(laddering)
logger.info(f"Time-sequenced analysis complete. Strike laddering detected: {df['strike_laddering_detected'].sum()}")
return df

View File

@ -3,7 +3,13 @@
# Activate virtual environment if it exists
if [ -d "venv" ]; then
# Try Windows Scripts path first (Git Bash on Windows)
if [ -f "venv/Scripts/activate" ]; then
source venv/Scripts/activate
# Fallback to Unix bin path (Linux/macOS)
elif [ -f "venv/bin/activate" ]; then
source venv/bin/activate
fi
fi
# Start the service

View File

View File

@ -0,0 +1,35 @@
import { rawQuery } from '../src/db.js';
async function run() {
try {
await rawQuery(`
CREATE TABLE IF NOT EXISTS signals (
id SERIAL PRIMARY KEY,
ticker TEXT NOT NULL,
ts TIMESTAMPTZ NOT NULL,
lane TEXT NOT NULL,
score NUMERIC NOT NULL,
grade TEXT NOT NULL,
entry_price NUMERIC NOT NULL,
features JSONB NOT NULL,
regime JSONB NOT NULL,
data_age_sec INT,
resolved BOOLEAN DEFAULT FALSE,
intraday_move NUMERIC,
hit_2R_first BOOLEAN,
ret_5d NUMERIC,
ret_10d NUMERIC,
excess_3m NUMERIC,
excess_6m NUMERIC,
excess_12m NUMERIC
);
CREATE INDEX IF NOT EXISTS idx_signals_lane_grade ON signals(lane, grade, ts);
`);
console.log('Signals table initialized.');
process.exit(0);
} catch (err) {
console.error(err);
process.exit(1);
}
}
run();

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

1
backend/src/backtest/cache/GS_2y.json vendored Normal file

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

1
backend/src/backtest/cache/MA_2y.json vendored Normal file

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

1
backend/src/backtest/cache/MU_2y.json vendored Normal file

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

1
backend/src/backtest/cache/V_2y.json vendored Normal file

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,16 @@
export const LANES = {
swing: {
maxHoldDays: 10,
targetAtr: 2.0,
stopAtr: 1.0,
entry: 'next_open',
burnIn: 200,
},
overnight: {
maxHoldDays: 1, // exit at close of day 1 (the entry day)
targetAtr: 1.0, // +1 ATR (reachable intraday)
stopAtr: 1.0, // -1 ATR
entry: 'next_open', // signal at close[t], enter open[t+1]
burnIn: 200,
}
};

View File

@ -0,0 +1,26 @@
/**
* Apply slippage and spread costs
* @param {number} price - The raw price (e.g. open/close)
* @param {string} side - 'buy' or 'sell'
* @param {Array} candles - Full candles array
* @param {number} idx - Index of the candle to compute ADV from
* @param {number} orderDollar - Estimated order size (default $5000)
*/
export function applyCosts(price, side, candles, idx, orderDollar = 5000) {
// Compute ADV from the 10 days prior to idx, or default to 1M if not enough data
let advDollar = 1000000;
if (idx >= 10) {
let volSum = 0;
for (let i = idx - 10; i < idx; i++) {
volSum += (candles[i].volume || 0) * (candles[i].close || 0);
}
if (volSum > 0) advDollar = volSum / 10;
}
// tightest spread ~ 0.02%, scaling up for less liquid
const spreadPct = Math.max(0.0002, 0.01 / Math.sqrt(advDollar));
// size-aware slippage
const slippagePct = 0.0005 * Math.sqrt(orderDollar / advDollar);
const drag = (spreadPct / 2) + slippagePct;
return side === 'buy' ? price * (1 + drag) : price * (1 - drag);
}

View File

@ -0,0 +1,96 @@
export function sma(candles, period) {
if (candles.length < period) return null;
const slice = candles.slice(-period);
return slice.reduce((a, c) => a + c.close, 0) / period;
}
export function atr14(candles, period = 14) {
if (!candles || candles.length <= period) return null;
const trs = [];
for (let i = 1; i < candles.length; i++) {
const { high, low } = candles[i];
const prevClose = candles[i - 1].close;
trs.push(Math.max(high - low, Math.abs(high - prevClose), Math.abs(low - prevClose)));
}
const recent = trs.slice(-period);
return recent.reduce((a, b) => a + b, 0) / recent.length;
}
export function rsi14(candles, period = 14) {
if (candles.length < period + 1) return null;
const recent = candles.slice(-period - 1);
let gains = 0, losses = 0;
for (let i = 1; i <= period; i++) {
const diff = recent[i].close - recent[i - 1].close;
if (diff >= 0) gains += diff;
else losses += Math.abs(diff);
}
if (losses === 0) return 100;
const rs = gains / losses;
return 100 - (100 / (1 + rs));
}
export function macd(candles, fast = 12, slow = 26, sig = 9) {
if (candles.length < slow + sig) return { macdLine: null, signalLine: null, hist: null };
const emaFast = sma(candles, fast); // simplified to SMA for backtest proxy
const emaSlow = sma(candles, slow);
const macdLine = emaFast - emaSlow;
// fake signal line for proxy (just using a small offset since we can't easily compute EMA of MACD over full history here without more state)
// Actually, let's just do a rough proxy
const signalLine = macdLine * 0.9;
return { macdLine, signalLine, hist: macdLine - signalLine };
}
export function slope(candles, period = 50, slopeDays = 5) {
if (candles.length < period + slopeDays) return null;
const currentSma = sma(candles, period);
const oldSma = sma(candles.slice(0, candles.length - slopeDays), period);
return (currentSma - oldSma) / oldSma; // % change in SMA
}
export function volume(candles, period) {
if (candles.length < period) return null;
return candles.slice(-period).reduce((a, c) => a + c.volume, 0) / period;
}
export function avgVolume(candles, period) {
return volume(candles, period);
}
export function regimeScore(spyCandles, vixClose) {
if (!spyCandles || spyCandles.length < 50) return 0;
const spySma20 = sma(spyCandles, 20);
const spySma50 = sma(spyCandles, 50);
const spyClose = spyCandles[spyCandles.length - 1].close;
if (spyClose > spySma20 && spyClose > spySma50 && vixClose < 20) return 1; // Risk-On
if (spyClose < spySma50 || vixClose > 25) return -1; // Risk-Off
return 0; // Neutral
}
/**
* Compute point-in-time features for the Swing lane.
* GUARANTEE: `candles` and `spyCandles` passed to this function must ALREADY
* be sliced to contain ONLY data up to and including the current day `t`.
*/
export function swingFeatures(candles, spyCandles, vix) {
const px = candles.at(-1).close;
const sma50 = sma(candles, 50);
const sma200 = sma(candles, 200);
const atr = atr14(candles);
const { macdLine, signalLine, hist } = macd(candles);
if (!sma50 || !sma200 || !atr || macdLine === null) return null;
return {
dist50Atr: (px - sma50) / atr,
dist200Atr: (px - sma200) / atr,
sma50Slope: slope(candles, 50, 5),
macdHistNorm: hist / atr,
macdCrossUp: macdLine > signalLine ? 1 : 0,
rsi: rsi14(candles),
atrPct: (atr / px) * 100,
rvol: volume(candles, 1) / avgVolume(candles, 50),
regime: regimeScore(spyCandles, vix)
};
}

View File

@ -0,0 +1,9 @@
export function fitGradeThresholds(trainProbs) {
const sorted = [...trainProbs].sort((a, b) => a - b);
const q = p => sorted[Math.floor(p * sorted.length)];
return { aMin: q(0.80), bMin: q(0.60), cMin: q(0.40) }; // A=top20%, B=top40%, C=top60%
}
export function toGrade(prob, t) {
return prob >= t.aMin ? 'A' : prob >= t.bMin ? 'B' : prob >= t.cMin ? 'C' : 'D';
}

View File

@ -0,0 +1,188 @@
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import { UNIVERSE } from '../services/stockUniverseService.js';
import { LANES } from './config.js';
import { simulateSwing } from './outcomeSim.js';
import { swingFeatures } from './features.js';
import { StandardScaler } from './model/scaler.js';
import { LogisticModel } from './model/logistic.js';
import { fitGradeThresholds, toGrade } from './grades.js';
import { generateResultsTable } from './report/resultsTable.js';
import { calibration } from './report/calibration.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const CACHE_DIR = path.join(__dirname, 'cache');
const BACKTEST_RANGE = '2y';
const YF_BASE = 'https://query1.finance.yahoo.com';
const BURN_IN_DAYS = 200; // Need 200 days for 200-SMA
// Fetch historical data with caching
async function fetchHistoricalData(symbol) {
const cachePath = path.join(CACHE_DIR, `${symbol}_${BACKTEST_RANGE}.json`);
if (fs.existsSync(cachePath)) {
return JSON.parse(fs.readFileSync(cachePath, 'utf8'));
}
console.log(`[Fetch] Downloading ${symbol} data...`);
const url = `${YF_BASE}/v8/finance/chart/${symbol}?interval=1d&range=${BACKTEST_RANGE}`;
const res = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0' } });
if (!res.ok) return null;
const data = await res.json();
const result = data?.chart?.result?.[0];
if (result) fs.writeFileSync(cachePath, JSON.stringify(result));
return result;
}
function extractCandles(yfResult) {
const quotes = yfResult.indicators?.quote?.[0] || {};
const timestamps = yfResult.timestamp || [];
const candles = [];
for (let i = 0; i < timestamps.length; i++) {
if (quotes.high?.[i] != null && quotes.low?.[i] != null && quotes.close?.[i] != null && quotes.open?.[i] != null) {
candles.push({
ts: timestamps[i] * 1000,
date: new Date(timestamps[i] * 1000).toISOString().split('T')[0],
open: quotes.open[i],
high: quotes.high[i],
low: quotes.low[i],
close: quotes.close[i],
volume: quotes.volume?.[i] || 0
});
}
}
return candles;
}
async function runValidationGate() {
console.log(`\n======================================================`);
console.log(` PATH C (SWING) VALIDATION GATE`);
console.log(`======================================================\n`);
const cfg = LANES.swing;
const dataMap = new Map();
const symbols = ['SPY', '^VIX', ...UNIVERSE.map(s => s.symbol)];
// 0. Load Data
process.stdout.write("Loading historical data... ");
for (const sym of symbols) {
const raw = await fetchHistoricalData(sym);
if (raw) dataMap.set(sym, extractCandles(raw));
}
console.log("Done.");
const spyCandles = dataMap.get('SPY');
const vixCandles = dataMap.get('^VIX');
const allSignals = [];
// 1. Generate ALL signals over full window
process.stdout.write("Generating signals (simulating multi-day paths)... ");
for (let i = BURN_IN_DAYS; i < spyCandles.length - 1; i++) {
const currentSpyDate = spyCandles[i].date;
const currentVixDate = spyCandles[i].date; // approx
const spySlice = spyCandles.slice(0, i + 1);
// get VIX close
const vixSlice = vixCandles?.filter(c => c.date <= currentSpyDate);
const vixClose = vixSlice?.length ? vixSlice[vixSlice.length - 1].close : 15;
for (const u of UNIVERSE) {
const candles = dataMap.get(u.symbol);
if (!candles) continue;
const stockIdx = candles.findIndex(c => c.date === currentSpyDate);
if (stockIdx === -1) continue;
const stockSlice = candles.slice(0, stockIdx + 1);
// Compute point-in-time features (no lookahead!)
const features = swingFeatures(stockSlice, spySlice, vixClose);
if (!features) continue;
// Simulate outcome (multi-day) using the real entry index which is stockIdx + 1 (tomorrow's open)
const entryIdx = stockIdx + 1;
const result = simulateSwing(candles, entryIdx, cfg);
if (result) {
allSignals.push({
date: currentSpyDate, // The day the signal fired (market close)
symbol: u.symbol,
features,
outcome: result.outcome,
rMultiple: result.r
});
}
}
}
console.log(`Done. (${allSignals.length} signals generated)`);
if (allSignals.length === 0) {
console.error("No signals generated!");
return;
}
// 2. SPLIT: train = first 12 months, test = last 12 months
// We'll split roughly down the middle of the available dates
const uniqueDates = [...new Set(allSignals.map(s => s.date))].sort();
const splitDate = uniqueDates[Math.floor(uniqueDates.length / 2)];
const trainSet = allSignals.filter(s => s.date < splitDate);
const testSet = allSignals.filter(s => s.date >= splitDate);
console.log(`Split Date: ${splitDate}`);
console.log(`TRAIN Set: ${trainSet.length} signals`);
console.log(`TEST Set: ${testSet.length} signals`);
// 3. scaler.fit(train.features)
const scaler = new StandardScaler();
scaler.fit(trainSet.map(s => s.features));
// 4. model = logistic.fit(scaler.transform(train.features), train.win)
const model = new LogisticModel();
const trainFeaturesScaled = scaler.transformArray(trainSet.map(s => s.features));
const trainLabels = trainSet.map(s => s.outcome === 'WIN' ? 1 : 0);
const coeffs = model.fit(trainFeaturesScaled, trainLabels);
console.log(`\n=== MODEL FIT (Standardized Coefficients) ===`);
console.log(`Intercept: ${coeffs.intercept.toFixed(4)}`);
for (const k of Object.keys(coeffs)) {
if (k !== 'intercept') console.log(`${k.padEnd(15)} ${coeffs[k].toFixed(4)}`);
}
// 5. thresholds = fitGradeThresholds(model.predict(train))
const trainProbs = model.predict(trainFeaturesScaled);
const thresholds = fitGradeThresholds(trainProbs);
console.log(`\n=== GRADE THRESHOLDS (FROZEN ON TRAIN) ===`);
console.log(`Grade A min P(win): ${(thresholds.aMin*100).toFixed(1)}%`);
console.log(`Grade B min P(win): ${(thresholds.bMin*100).toFixed(1)}%`);
console.log(`Grade C min P(win): ${(thresholds.cMin*100).toFixed(1)}%`);
// ===================== NO TRAIN DATA PAST THIS POINT =====================
// 6. test.prob = model.predict(scaler.transform(test.features))
const testFeaturesScaled = scaler.transformArray(testSet.map(s => s.features));
const testProbs = model.predict(testFeaturesScaled);
for (let i = 0; i < testSet.length; i++) {
testSet[i].prob = testProbs[i];
testSet[i].grade = toGrade(testProbs[i], thresholds);
}
// 7. resultsTable(test)
generateResultsTable(testSet);
// 8. calibration(test)
calibration(testSet);
console.log(`\n======================================================`);
console.log(` VALIDATION GATE COMPLETE`);
console.log(`======================================================\n`);
}
runValidationGate().catch(console.error);

View File

@ -0,0 +1,188 @@
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import { UNIVERSE } from '../services/stockUniverseService.js';
import { LANES } from './config.js';
import { simulateSwing } from './outcomeSim.js';
import { swingFeatures } from './features.js';
import { StandardScaler } from './model/scaler.js';
import { LogisticModel } from './model/logistic.js';
import { fitGradeThresholds, toGrade } from './grades.js';
import { generateResultsTable } from './report/resultsTable.js';
import { calibration } from './report/calibration.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const CACHE_DIR = path.join(__dirname, 'cache');
const BACKTEST_RANGE = '2y';
const YF_BASE = 'https://query1.finance.yahoo.com';
const BURN_IN_DAYS = 200; // Need 200 days for 200-SMA
// Fetch historical data with caching
async function fetchHistoricalData(symbol) {
const cachePath = path.join(CACHE_DIR, `${symbol}_${BACKTEST_RANGE}.json`);
if (fs.existsSync(cachePath)) {
return JSON.parse(fs.readFileSync(cachePath, 'utf8'));
}
console.log(`[Fetch] Downloading ${symbol} data...`);
const url = `${YF_BASE}/v8/finance/chart/${symbol}?interval=1d&range=${BACKTEST_RANGE}`;
const res = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0' } });
if (!res.ok) return null;
const data = await res.json();
const result = data?.chart?.result?.[0];
if (result) fs.writeFileSync(cachePath, JSON.stringify(result));
return result;
}
function extractCandles(yfResult) {
const quotes = yfResult.indicators?.quote?.[0] || {};
const timestamps = yfResult.timestamp || [];
const candles = [];
for (let i = 0; i < timestamps.length; i++) {
if (quotes.high?.[i] != null && quotes.low?.[i] != null && quotes.close?.[i] != null && quotes.open?.[i] != null) {
candles.push({
ts: timestamps[i] * 1000,
date: new Date(timestamps[i] * 1000).toISOString().split('T')[0],
open: quotes.open[i],
high: quotes.high[i],
low: quotes.low[i],
close: quotes.close[i],
volume: quotes.volume?.[i] || 0
});
}
}
return candles;
}
async function runValidationGate() {
console.log(`\n======================================================`);
console.log(` PATH B (OVERNIGHT) VALIDATION GATE`);
console.log(`======================================================\n`);
const cfg = LANES.overnight;
const dataMap = new Map();
const symbols = ['SPY', '^VIX', ...UNIVERSE.map(s => s.symbol)];
// 0. Load Data
process.stdout.write("Loading historical data... ");
for (const sym of symbols) {
const raw = await fetchHistoricalData(sym);
if (raw) dataMap.set(sym, extractCandles(raw));
}
console.log("Done.");
const spyCandles = dataMap.get('SPY');
const vixCandles = dataMap.get('^VIX');
const allSignals = [];
// 1. Generate ALL signals over full window
process.stdout.write("Generating signals (simulating multi-day paths)... ");
for (let i = BURN_IN_DAYS; i < spyCandles.length - 1; i++) {
const currentSpyDate = spyCandles[i].date;
const currentVixDate = spyCandles[i].date; // approx
const spySlice = spyCandles.slice(0, i + 1);
// get VIX close
const vixSlice = vixCandles?.filter(c => c.date <= currentSpyDate);
const vixClose = vixSlice?.length ? vixSlice[vixSlice.length - 1].close : 15;
for (const u of UNIVERSE) {
const candles = dataMap.get(u.symbol);
if (!candles) continue;
const stockIdx = candles.findIndex(c => c.date === currentSpyDate);
if (stockIdx === -1) continue;
const stockSlice = candles.slice(0, stockIdx + 1);
// Compute point-in-time features (no lookahead!)
const features = swingFeatures(stockSlice, spySlice, vixClose);
if (!features) continue;
// Simulate outcome (multi-day) using the real entry index which is stockIdx + 1 (tomorrow's open)
const entryIdx = stockIdx + 1;
const result = simulateSwing(candles, entryIdx, cfg);
if (result) {
allSignals.push({
date: currentSpyDate, // The day the signal fired (market close)
symbol: u.symbol,
features,
outcome: result.outcome,
rMultiple: result.r
});
}
}
}
console.log(`Done. (${allSignals.length} signals generated)`);
if (allSignals.length === 0) {
console.error("No signals generated!");
return;
}
// 2. SPLIT: train = first 12 months, test = last 12 months
// We'll split roughly down the middle of the available dates
const uniqueDates = [...new Set(allSignals.map(s => s.date))].sort();
const splitDate = uniqueDates[Math.floor(uniqueDates.length / 2)];
const trainSet = allSignals.filter(s => s.date < splitDate);
const testSet = allSignals.filter(s => s.date >= splitDate);
console.log(`Split Date: ${splitDate}`);
console.log(`TRAIN Set: ${trainSet.length} signals`);
console.log(`TEST Set: ${testSet.length} signals`);
// 3. scaler.fit(train.features)
const scaler = new StandardScaler();
scaler.fit(trainSet.map(s => s.features));
// 4. model = logistic.fit(scaler.transform(train.features), train.win)
const model = new LogisticModel();
const trainFeaturesScaled = scaler.transformArray(trainSet.map(s => s.features));
const trainLabels = trainSet.map(s => s.outcome === 'WIN' ? 1 : 0);
const coeffs = model.fit(trainFeaturesScaled, trainLabels);
console.log(`\n=== MODEL FIT (Standardized Coefficients) ===`);
console.log(`Intercept: ${coeffs.intercept.toFixed(4)}`);
for (const k of Object.keys(coeffs)) {
if (k !== 'intercept') console.log(`${k.padEnd(15)} ${coeffs[k].toFixed(4)}`);
}
// 5. thresholds = fitGradeThresholds(model.predict(train))
const trainProbs = model.predict(trainFeaturesScaled);
const thresholds = fitGradeThresholds(trainProbs);
console.log(`\n=== GRADE THRESHOLDS (FROZEN ON TRAIN) ===`);
console.log(`Grade A min P(win): ${(thresholds.aMin*100).toFixed(1)}%`);
console.log(`Grade B min P(win): ${(thresholds.bMin*100).toFixed(1)}%`);
console.log(`Grade C min P(win): ${(thresholds.cMin*100).toFixed(1)}%`);
// ===================== NO TRAIN DATA PAST THIS POINT =====================
// 6. test.prob = model.predict(scaler.transform(test.features))
const testFeaturesScaled = scaler.transformArray(testSet.map(s => s.features));
const testProbs = model.predict(testFeaturesScaled);
for (let i = 0; i < testSet.length; i++) {
testSet[i].prob = testProbs[i];
testSet[i].grade = toGrade(testProbs[i], thresholds);
}
// 7. resultsTable(test)
generateResultsTable(testSet);
// 8. calibration(test)
calibration(testSet);
console.log(`\n======================================================`);
console.log(` VALIDATION GATE COMPLETE`);
console.log(`======================================================\n`);
}
runValidationGate().catch(console.error);

View File

@ -0,0 +1,153 @@
import fs from 'fs';
function sigmoid(z) {
// Cap z to avoid overflow
if (z > 20) return 1.0;
if (z < -20) return 0.0;
return 1 / (1 + Math.exp(-z));
}
function computeLogLoss(features, labels, weights) {
let loss = 0;
for (let i = 0; i < features.length; i++) {
const f = features[i];
let z = weights[0];
for (let j = 0; j < f.length; j++) z += weights[j + 1] * f[j];
const p = sigmoid(z);
// clip p to avoid log(0)
const pSafe = Math.max(1e-15, Math.min(1 - 1e-15, p));
loss += -labels[i] * Math.log(pSafe) - (1 - labels[i]) * Math.log(1 - pSafe);
}
return loss / features.length;
}
export class LogisticModel {
constructor() {
this.weights = null;
this.featureKeys = [];
}
// Internal full-batch gradient descent
_train(X, Y, lambda, learningRate, maxSteps, tol) {
const numFeatures = X[0].length;
let w = new Array(numFeatures + 1).fill(0.0);
const N = X.length;
let prevLoss = Infinity;
for (let step = 0; step < maxSteps; step++) {
const grad = new Array(numFeatures + 1).fill(0.0);
for (let i = 0; i < N; i++) {
let z = w[0];
for (let j = 0; j < numFeatures; j++) z += w[j + 1] * X[i][j];
const p = sigmoid(z);
const err = p - Y[i];
grad[0] += err;
for (let j = 0; j < numFeatures; j++) {
grad[j + 1] += err * X[i][j];
}
}
// Add L2 penalty (skip intercept grad[0])
for (let j = 0; j < numFeatures; j++) {
grad[j + 1] += lambda * w[j + 1];
}
// Update weights
let maxDelta = 0;
for (let j = 0; j < w.length; j++) {
const update = (learningRate * grad[j]) / N;
w[j] -= update;
if (Math.abs(update) > maxDelta) maxDelta = Math.abs(update);
}
if (maxDelta < tol) {
// Converged
break;
}
}
return w;
}
fit(featuresArray, labels) {
if (!featuresArray.length) return;
this.featureKeys = Object.keys(featuresArray[0]);
// Convert to 2D array
const X = featuresArray.map(f => this.featureKeys.map(k => f[k]));
const Y = labels;
const N = X.length;
// 1. Carve a 80/20 validation slice from TRAIN to tune lambda
const splitIdx = Math.floor(N * 0.8);
const X_train = X.slice(0, splitIdx);
const Y_train = Y.slice(0, splitIdx);
const X_val = X.slice(splitIdx);
const Y_val = Y.slice(splitIdx);
// 2. Tune lambda
const lambdas = [0.01, 0.1, 1.0, 10.0, 100.0, 500.0, 1000.0];
let bestLambda = 0;
let bestLoss = Infinity;
console.log(`\nTuning L2 Regularization (Lambda) on Validation Slice...`);
for (const l of lambdas) {
const w = this._train(X_train, Y_train, l, 0.5, 2000, 1e-5);
const loss = computeLogLoss(X_val, Y_val, w);
console.log(` Lambda ${l.toString().padEnd(6)} -> LogLoss: ${loss.toFixed(4)}`);
if (loss < bestLoss) {
bestLoss = loss;
bestLambda = l;
}
}
console.log(`Selected Best Lambda: ${bestLambda}`);
// 3. Refit on FULL TRAIN set using best lambda
this.weights = this._train(X, Y, bestLambda, 0.5, 5000, 1e-5);
return this.getCoefficients();
}
predict(featuresArray) {
if (!this.weights) throw new Error("Model not trained");
const probs = [];
for (let i = 0; i < featuresArray.length; i++) {
const f = featuresArray[i];
let z = this.weights[0]; // intercept
for (let j = 0; j < this.featureKeys.length; j++) {
z += this.weights[j + 1] * f[this.featureKeys[j]];
}
probs.push(sigmoid(z));
}
return probs;
}
getCoefficients() {
if (!this.weights) return {};
const coeffs = { intercept: this.weights[0] };
for (let i = 0; i < this.featureKeys.length; i++) {
coeffs[this.featureKeys[i]] = this.weights[i + 1];
}
return coeffs;
}
save(filepath) {
fs.writeFileSync(filepath, JSON.stringify({
featureKeys: this.featureKeys,
weights: this.weights
}, null, 2));
}
load(filepath) {
if (!fs.existsSync(filepath)) return;
const data = JSON.parse(fs.readFileSync(filepath, 'utf8'));
this.featureKeys = data.featureKeys;
this.weights = data.weights;
}
}

View File

@ -0,0 +1,51 @@
import fs from 'fs';
import path from 'path';
export class StandardScaler {
constructor() {
this.means = {};
this.stds = {};
}
fit(featuresArray) {
if (!featuresArray.length) return;
// Get all feature keys
const keys = Object.keys(featuresArray[0]);
for (const key of keys) {
const values = featuresArray.map(f => f[key]);
const mean = values.reduce((sum, v) => sum + v, 0) / values.length;
const variance = values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / values.length;
const std = Math.sqrt(variance);
this.means[key] = mean;
this.stds[key] = std || 1; // Avoid divide by 0
}
}
transform(features) {
const scaled = {};
for (const key of Object.keys(this.means)) {
if (features[key] === undefined) continue;
scaled[key] = (features[key] - this.means[key]) / this.stds[key];
}
return scaled;
}
transformArray(featuresArray) {
return featuresArray.map(f => this.transform(f));
}
save(filepath) {
fs.writeFileSync(filepath, JSON.stringify({ means: this.means, stds: this.stds }, null, 2));
}
load(filepath) {
if (!fs.existsSync(filepath)) return;
const data = JSON.parse(fs.readFileSync(filepath, 'utf8'));
this.means = data.means;
this.stds = data.stds;
}
}

View File

@ -0,0 +1,36 @@
{
"name": "LogisticRegression",
"numSteps": 1000,
"learningRate": 0.05,
"numberClasses": 2,
"classifiers": [
{
"numSteps": 1000,
"learningRate": 0.05,
"weights": [
[
-94.4420175451543,
-5.766427757134977,
-74.50180063587948,
-66.23257473781456,
-1817.0926888106162,
-166.35311896181236
]
]
},
{
"numSteps": 1000,
"learningRate": 0.05,
"weights": [
[
-84.54027063090014,
-50.36483530223822,
13.904576636453635,
37.27497755449738,
-395.0000542460034,
41.80818011695695
]
]
}
]
}

View File

@ -0,0 +1,46 @@
import { swingFeatures } from './features.js';
function runTest() {
console.log("Running no-lookahead guardrail test...");
// Create mock candles where index = day
const candles = [];
for (let i = 0; i < 250; i++) {
candles.push({
date: `2024-01-${i}`,
open: 100 + i,
high: 105 + i,
low: 95 + i,
close: 102 + i,
volume: 1000000
});
}
const spyCandles = [...candles]; // clone
const vix = 15;
// We are at day index 210.
const currentDayIdx = 210;
// Guardrail: Pass ONLY the slice up to current day
const slicedCandles = candles.slice(0, currentDayIdx + 1);
const slicedSpy = spyCandles.slice(0, currentDayIdx + 1);
// Compute features
const feats = swingFeatures(slicedCandles, slicedSpy, vix);
if (!feats) {
throw new Error("Features returned null unexpectedly.");
}
// The slice literally does not contain any future data in memory.
// We can verify the length of the array inside the feature to be absolutely sure.
if (slicedCandles.length > currentDayIdx + 1) {
throw new Error("Lookahead leak: slicedCandles contains future data!");
}
console.log("PASS: Features successfully computed without lookahead bias.");
console.log("Computed Features:", feats);
}
runTest();

View File

@ -0,0 +1,44 @@
import { applyCosts } from './costs.js';
function atr14(candles, period = 14) {
if (!candles || candles.length <= period) return null;
const trs = [];
for (let i = 1; i < candles.length; i++) {
const { high, low } = candles[i];
const prevClose = candles[i - 1].close;
trs.push(Math.max(high - low, Math.abs(high - prevClose), Math.abs(low - prevClose)));
}
const recent = trs.slice(-period);
return recent.reduce((a, b) => a + b, 0) / recent.length;
}
export function simulateSwing(candles, entryIdx, cfg) {
// If entry is impossible (out of bounds)
if (entryIdx >= candles.length) return null;
const rawEntry = candles[entryIdx].open;
const entry = applyCosts(rawEntry, 'buy', candles, entryIdx);
// ATR known at signal (close of previous day). candles slice goes UP TO the entryIdx (so it includes the signal bar, which is entryIdx - 1)
// Wait, if entryIdx is the open of tomorrow, we pass candles.slice(0, entryIdx) so the last candle is today (the signal bar).
const preEntryCandles = candles.slice(0, entryIdx);
const atr = atr14(preEntryCandles);
if (!atr) return null;
const stop = entry - cfg.stopAtr * atr;
const target = entry + cfg.targetAtr * atr;
const R = entry - stop; // 1 ATR worth of price
const end = Math.min(entryIdx + cfg.maxHoldDays - 1, candles.length - 1);
for (let d = entryIdx; d <= end; d++) {
// pessimistic tie-break: if both hit same day, assume STOP first
if (candles[d].low <= stop) return { outcome: 'LOSS', r: -1.0, exitDay: d, stopPrice: stop, targetPrice: target, atr };
if (candles[d].high >= target) return { outcome: 'WIN', r: cfg.targetAtr, exitDay: d, stopPrice: stop, targetPrice: target, atr };
}
// unresolved → exit at close of last day, net of exit costs
const rawExit = candles[end].close;
const exit = applyCosts(rawExit, 'sell', candles, end);
return { outcome: 'SCRATCH', r: (exit - entry) / R, exitDay: end, stopPrice: stop, targetPrice: target, atr };
}

View File

@ -0,0 +1,31 @@
export function calibration(testResults) {
// We bucket predicted probabilities into deciles (0-10%, 10-20%, etc)
const buckets = Array(10).fill(null).map(() => ({ total: 0, wins: 0, sumProb: 0 }));
for (const r of testResults) {
if (r.prob === undefined || r.prob === null) continue;
let bIdx = Math.floor(r.prob * 10);
if (bIdx > 9) bIdx = 9; // Handle p=1.0 edge case
if (bIdx < 0) bIdx = 0;
buckets[bIdx].total++;
buckets[bIdx].sumProb += r.prob;
if (r.outcome === 'WIN') buckets[bIdx].wins++;
}
console.log(`\n=== CALIBRATION TABLE ===`);
console.log(`Bucket\t\tN\tPred P(Win)\tActual Win%\tDiff`);
console.log(`------------------------------------------------------------------`);
for (let i = 0; i < 10; i++) {
const b = buckets[i];
if (b.total === 0) continue;
const label = `${(i * 10).toString().padStart(2, '0')}-${((i + 1) * 10).toString().padStart(2, '0')}%`;
const predP = (b.sumProb / b.total) * 100;
const actP = (b.wins / b.total) * 100;
const diff = actP - predP;
console.log(`[${label}]\t${b.total}\t${predP.toFixed(1)}%\t\t${actP.toFixed(1)}%\t\t${diff > 0 ? '+' : ''}${diff.toFixed(1)}pp`);
}
}

View File

@ -0,0 +1,75 @@
export function ciExpectancy(rValues) {
const n = rValues.length;
if (n === 0) return { mean: 0, lo: 0, hi: 0, n: 0 };
const mean = rValues.reduce((a, b) => a + b, 0) / n;
if (n === 1) return { mean, lo: mean, hi: mean, n };
const variance = rValues.reduce((a, b) => a + Math.pow(b - mean, 2), 0) / (n - 1);
const sd = Math.sqrt(variance);
const se = sd / Math.sqrt(n);
return { mean, lo: mean - 1.96 * se, hi: mean + 1.96 * se, n };
}
export function generateResultsTable(testResults) {
const grades = ['A', 'B', 'C', 'D'];
const grouped = {
A: { wins: 0, losses: 0, scratches: 0, rVals: [] },
B: { wins: 0, losses: 0, scratches: 0, rVals: [] },
C: { wins: 0, losses: 0, scratches: 0, rVals: [] },
D: { wins: 0, losses: 0, scratches: 0, rVals: [] }
};
for (const r of testResults) {
if (grouped[r.grade]) {
grouped[r.grade].rVals.push(r.rMultiple);
if (r.outcome === 'WIN') grouped[r.grade].wins++;
else if (r.outcome === 'LOSS') grouped[r.grade].losses++;
else grouped[r.grade].scratches++;
}
}
console.log(`\n=== OUT-OF-SAMPLE TEST RESULTS (EXPECTANCY) ===`);
console.log(`Grade\tExp(R)\t\t95% CI (Exp)\t\tWin%\tLoss%\tScratch%\tn`);
console.log(`-----------------------------------------------------------------------------------------`);
for (const g of grades) {
const st = grouped[g];
const n = st.rVals.length;
if (n === 0) continue;
const pWin = (st.wins / n) * 100;
const pLoss = (st.losses / n) * 100;
const pScratch = (st.scratches / n) * 100;
const ci = ciExpectancy(st.rVals);
const meanFmt = `${ci.mean > 0 ? '+' : ''}${ci.mean.toFixed(3)}R`;
const ciFmt = `[${ci.lo.toFixed(3)}, ${ci.hi.toFixed(3)}]`;
console.log(`${g}\t${meanFmt}\t\t${ciFmt}\t\t${pWin.toFixed(1)}%\t${pLoss.toFixed(1)}%\t${pScratch.toFixed(1)}%\t\t${n}`);
}
// Random Control = The entire test set
const allR = testResults.map(r => r.rMultiple);
const totalN = allR.length;
if (totalN > 0) {
const totalWins = testResults.filter(r => r.outcome === 'WIN').length;
const totalLosses = testResults.filter(r => r.outcome === 'LOSS').length;
const totalScratches = testResults.filter(r => r.outcome === 'SCRATCH').length;
const ciRand = ciExpectancy(allR);
const meanFmt = `${ciRand.mean > 0 ? '+' : ''}${ciRand.mean.toFixed(3)}R`;
const ciFmt = `[${ciRand.lo.toFixed(3)}, ${ciRand.hi.toFixed(3)}]`;
const pWin = (totalWins / totalN) * 100;
const pLoss = (totalLosses / totalN) * 100;
const pScratch = (totalScratches / totalN) * 100;
console.log(`Rand\t${meanFmt}\t\t${ciFmt}\t\t${pWin.toFixed(1)}%\t${pLoss.toFixed(1)}%\t${pScratch.toFixed(1)}%\t\t${totalN}`);
}
console.log(`-----------------------------------------------------------------------------------------`);
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,68 @@
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import { parse } from 'csv-parse/sync';
import LogisticRegression from 'ml-logistic-regression';
import { Matrix } from 'ml-matrix';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
async function train() {
console.log('Loading results.csv...');
const csvPath = path.join(__dirname, 'results.csv');
const fileContent = fs.readFileSync(csvPath, 'utf8');
const records = parse(fileContent, {
columns: true,
skip_empty_lines: true
});
const X = [];
const Y = [];
for (const row of records) {
if (row.type !== 'signal') continue; // Skip controls if they exist
// Parse features
const rsi = parseFloat(row.rsi);
const atrPct = parseFloat(row.atrPct);
const rvol = parseFloat(row.rvol);
const macd = parseFloat(row.macd);
const regime = parseFloat(row.regime_risk_on);
// Skip rows with missing data
if (isNaN(rsi) || isNaN(atrPct) || isNaN(rvol) || isNaN(macd) || isNaN(regime)) continue;
X.push([
1.0, // Intercept
rsi / 100,
atrPct / 10,
rvol / 5,
macd / 2,
regime
]);
// Target: 1 if win or win_partial, 0 otherwise
const isWin = (row.outcome === 'win' || row.outcome === 'win_partial') ? 1 : 0;
Y.push(isWin);
}
console.log(`Loaded ${X.length} valid training samples.`);
const xMat = new Matrix(X);
const yMat = Matrix.columnVector(Y);
console.log('Training Logistic Regression model...');
const logreg = new LogisticRegression({ numSteps: 1000, learningRate: 0.05 });
logreg.train(xMat, yMat);
const modelJson = logreg.toJSON();
const outputPath = path.join(__dirname, 'model_weights.json');
fs.writeFileSync(outputPath, JSON.stringify(modelJson, null, 2));
console.log(`Model trained and saved to ${outputPath}`);
}
train().catch(console.error);

View File

@ -0,0 +1,60 @@
import { loadFundamentals, getFundamentalsAsOf } from '../fundamentals.js';
function runTest() {
console.log("Running fundamentals lookahead test...");
const testData = [
{
symbol: 'AAPL',
datekey: '2020-02-15', // Actually filed on Feb 15
fiscalEnd: '2019-12-31',
metrics: { roe: 0.10 }
},
{
symbol: 'AAPL',
datekey: '2020-05-15', // Q1 filed on May 15
fiscalEnd: '2020-03-31',
metrics: { roe: 0.12 }
},
{
symbol: 'MSFT',
datekey: null, // Missing datekey, should fallback to fiscalEnd + 120 days
fiscalEnd: '2019-12-31', // + 120 days = 2020-04-29
metrics: { roe: 0.15 }
}
];
loadFundamentals(testData);
// Test 1: Jan 31, 2020.
// The Dec 31 fiscal data isn't filed until Feb 15. We MUST return null.
const a1 = getFundamentalsAsOf('AAPL', '2020-01-31');
if (a1 !== null) throw new Error("Lookahead leak! Returned Q4 data before filing date.");
// Test 2: Feb 16, 2020.
// The Dec 31 fiscal data was filed Feb 15. We should get it.
const a2 = getFundamentalsAsOf('AAPL', '2020-02-16');
if (!a2 || a2.roe !== 0.10) throw new Error("Failed to return available data.");
// Test 3: May 14, 2020.
// Q1 isn't filed until May 15. We should still get Q4 (roe: 0.10).
const a3 = getFundamentalsAsOf('AAPL', '2020-05-14');
if (!a3 || a3.roe !== 0.10) throw new Error("Lookahead leak! Returned Q1 data early.");
// Test 4: May 16, 2020.
// Now we should get Q1 (roe: 0.12).
const a4 = getFundamentalsAsOf('AAPL', '2020-05-16');
if (!a4 || a4.roe !== 0.12) throw new Error("Failed to roll forward to new data.");
// Test 5: MSFT fallback logic. Target = April 28 (119 days after fiscalEnd). Should be null.
const m1 = getFundamentalsAsOf('MSFT', '2020-04-28');
if (m1 !== null) throw new Error("Lookahead leak! MSFT fallback data returned early.");
// Test 6: MSFT fallback logic. Target = April 30 (121 days after fiscalEnd). Should be available.
const m2 = getFundamentalsAsOf('MSFT', '2020-04-30');
if (!m2 || m2.roe !== 0.15) throw new Error("Failed to return fallback data after 120 days.");
console.log("PASS: Fundamentals guardrail strictly prevents lookahead bias.");
}
runTest();

View File

@ -0,0 +1,54 @@
import { buildFactorProfiles } from '../descriptiveLens.js';
test('strictly-better stock ranks higher on every factor', async () => {
const good = {
ticker: 'GOOD',
trailingPE: 8,
priceToBook: 1,
priceToSales: 0.5,
freeCashflow: 1e10,
marketCap: 1e11,
returnOnEquity: 0.30,
grossMargins: 0.60,
debtToEquity: 10,
mom_12_1: 0.25,
realizedVol252: 0.15
};
const bad = {
ticker: 'BAD',
trailingPE: 50,
priceToBook: 10,
priceToSales: 12,
freeCashflow: 1e8,
marketCap: 1e11,
returnOnEquity: 0.02,
grossMargins: 0.10,
debtToEquity: 300,
mom_12_1: -0.10,
realizedVol252: 0.60
};
// buildFactorProfiles supports injecting a universe payload directly for testing
const out = await buildFactorProfiles([good, bad]);
// 'out' returns a map { 'GOOD': { ... }, 'BAD': { ... } } during test injection
for (const f of ['value', 'quality', 'lowVol', 'momentum', 'composite']) {
// A strictly better stock MUST have a higher percentile rank
if (out['GOOD'][f] <= out['BAD'][f]) {
throw new Error(`Sign catastrophe: GOOD ranked ${out['GOOD'][f]} on ${f}, but BAD ranked ${out['BAD'][f]}. GOOD must strictly outrank BAD.`);
}
}
console.log("PASS: Orientation logic correctly handles signs. 'Higher percentile' always means 'more favorable'.");
});
// Jest polyfill for simple node execution
function test(name, fn) {
console.log(`Running test: ${name}`);
fn().catch(e => {
console.error(`FAIL: ${e.message}`);
process.exit(1);
});
}

View File

@ -0,0 +1,142 @@
import { getUniverseAsOf } from './universe.js';
import { getFundamentalsAsOf } from './fundamentals.js';
import { standardizeCrossSection, FACTORS } from './factors.js';
import { spearman, calculateTurnover } from './metrics.js';
// Global reference for forward returns in our testing environment
// In production, this would query a price DB
let globalReturnsMap = new Map();
export function setForwardReturnsMap(map) {
globalReturnsMap = map;
}
function getForwardReturn(ticker, dateStr, horizonMonths, allDates) {
// Find the index of the current date
const idx = allDates.indexOf(dateStr);
if (idx === -1 || idx + horizonMonths >= allDates.length) return null; // Not enough forward data
let cumulativeRet = 0;
// Simple sum for log returns, or compound. We'll use simple compound: (1+r1)*(1+r2) - 1
let mult = 1.0;
for (let m = 0; m < horizonMonths; m++) {
const nextDate = allDates[idx + m];
const r = globalReturnsMap.get(`${ticker}_${nextDate}`);
if (r === undefined || r === null) return null; // missing data
mult *= (1 + r);
}
return mult - 1.0;
}
function bucketIntoDeciles(rankedScores) {
const deciles = Array(10).fill(null).map(() => []);
const n = rankedScores.length;
if (n === 0) return deciles;
for (let i = 0; i < n; i++) {
const d = Math.min(9, Math.floor((i / n) * 10));
deciles[d].push(rankedScores[i].ticker);
}
return deciles; // deciles[0] = highest scores (Top Decile), deciles[9] = lowest scores
}
export function runFactor(factorName, allDates, horizonMonths = 1, costPerSideBps = 5) {
const results = [];
let prevD10Weights = null;
let prevD1Weights = null;
const costPct = costPerSideBps / 10000.0;
for (const date of allDates) {
const members = getUniverseAsOf(date);
if (!members || members.length === 0) continue;
const rawScores = {};
for (const t of members) {
const fund = getFundamentalsAsOf(t, date);
if (!fund) continue;
const score = FACTORS[factorName](fund);
if (score !== null && !isNaN(score)) {
rawScores[t] = score;
}
}
// Standardize cross-sectionally
const zScores = standardizeCrossSection(rawScores, false);
// Sort descending (High Score = Best)
const ranked = Object.keys(zScores)
.map(t => ({ ticker: t, score: zScores[t] }))
.sort((a, b) => b.score - a.score);
if (ranked.length < 10) continue; // need enough names for deciles
const deciles = bucketIntoDeciles(ranked);
const d10Tickers = deciles[0]; // Top decile
const d1Tickers = deciles[9]; // Bottom decile
// Calculate equal-weight portfolio weights for this month
const curD10Weights = {};
d10Tickers.forEach(t => curD10Weights[t] = 1.0 / d10Tickers.length);
const curD1Weights = {};
d1Tickers.forEach(t => curD1Weights[t] = 1.0 / d1Tickers.length);
// Calculate turnover
const d10Turnover = calculateTurnover(prevD10Weights, curD10Weights);
const d1Turnover = calculateTurnover(prevD1Weights, curD1Weights);
// Forward returns
const decileReturns = [];
let allFwdReturns = [];
let allZScores = [];
for (let d = 0; d < 10; d++) {
const decTickers = deciles[d];
let sumRet = 0;
let count = 0;
for (const t of decTickers) {
const ret = getForwardReturn(t, date, horizonMonths, allDates);
if (ret !== null) {
sumRet += ret;
count++;
allFwdReturns.push(ret);
allZScores.push(zScores[t]);
}
}
decileReturns.push(count > 0 ? sumRet / count : 0);
}
if (allFwdReturns.length > 0) {
const ic = spearman(allZScores, allFwdReturns);
const universeMeanRet = allFwdReturns.reduce((a, b) => a + b, 0) / allFwdReturns.length;
// Gross Returns
const d10Gross = decileReturns[0];
const d1Gross = decileReturns[9];
// Net Returns
const d10Net = d10Gross - (d10Turnover * costPct);
const d1Net = d1Gross - (d1Turnover * costPct);
results.push({
date,
ic,
decileReturns, // Gross decile returns for the staircase plot
d10Gross,
d1Gross,
d10Net,
d1Net,
universeMeanRet,
d10Turnover,
d1Turnover
});
}
prevD10Weights = curD10Weights;
prevD1Weights = curD1Weights;
}
return results;
}

View File

@ -0,0 +1,228 @@
import { FACTORS, standardizeCrossSection } from './factors.js';
import { pullRawRecord } from './ingest.js';
import { getUniverse } from '../services/stockUniverseService.js';
/**
* Assigns percentiles (0-100) to a raw factor array using the standardizer.
* @param {Array} raw - Array of objects: { ticker, value: 0.5, quality: 1.2 }
* @param {String} factorKey - The factor to assign percentiles for (e.g. 'value')
*/
function assignPercentiles(raw, factorKey) {
const rawByTicker = {};
for (const r of raw) {
if (r[factorKey] !== null && r[factorKey] !== undefined) {
rawByTicker[r.ticker] = r[factorKey];
}
}
// Uses winsorization and z-scoring from factors.js
const zScores = standardizeCrossSection(rawByTicker, false);
// Convert z-scores to percentiles relative to the universe
const validTickers = Object.keys(zScores);
const sortedZScores = validTickers.map(t => zScores[t]).sort((a, b) => a - b);
const n = sortedZScores.length;
for (const r of raw) {
const z = zScores[r.ticker];
if (z === undefined) {
r[`${factorKey}Percentile`] = null;
continue;
}
// Find percentile rank
let index = sortedZScores.findIndex(val => val >= z);
if (index === -1) index = n - 1;
r[`${factorKey}Percentile`] = Math.round((index / n) * 100);
}
}
/**
* Generates the descriptive Factor Profile for the entire universe based on CURRENT data.
* Does NOT generate forward predictions.
*/
let _cachedRawProfiles = null;
let _cachedZScoreMaps = null;
let _cachedRawTime = 0;
export async function buildFactorProfiles(injectedUniverseRecords = null) {
const rawProfiles = [];
// Allow test injection
if (injectedUniverseRecords) {
for (const data of injectedUniverseRecords) {
rawProfiles.push({
ticker: data.ticker || 'TEST',
value: FACTORS.value(data),
quality: FACTORS.quality(data),
lowVol: FACTORS.lowVol(data),
momentum: FACTORS.momentum(data)
});
}
} else {
// We now include ETFs in the factor ranking because we want to see momentum/value on country ETFs
const universe = getUniverse().map(s => s.symbol);
const BATCH_SIZE = 15;
for (let i = 0; i < universe.length; i += BATCH_SIZE) {
const batch = universe.slice(i, i + BATCH_SIZE);
const promises = batch.map(async (ticker) => {
try {
const data = await pullRawRecord(ticker);
if (!data) return null;
return {
ticker,
value: FACTORS.value(data),
quality: FACTORS.quality(data),
lowVol: FACTORS.lowVol(data),
momentum: FACTORS.momentum(data)
};
} catch (e) {
console.warn(`[DescriptiveLens] Failed for ${ticker}:`, e.message);
return null;
}
});
const results = await Promise.allSettled(promises);
for (const res of results) {
if (res.status === 'fulfilled' && res.value) {
rawProfiles.push(res.value);
}
}
// Throttle delay between batches to respect free API limits
if (i + BATCH_SIZE < universe.length) {
await new Promise(r => setTimeout(r, 1000));
}
}
}
// Composite is the equal-weight average of the non-null standardized z-scores
// First, we must standardize the individual factors to compute composite z-score
const factors = ['value', 'quality', 'lowVol', 'momentum'];
const zScoreMaps = {};
for (const f of factors) {
const rawByTicker = {};
for (const r of rawProfiles) {
if (r[f] !== null && r[f] !== undefined) rawByTicker[r.ticker] = r[f];
}
zScoreMaps[f] = standardizeCrossSection(rawByTicker, false);
}
// Compute composite raw score
for (const r of rawProfiles) {
let sum = 0;
let count = 0;
for (const f of factors) {
const z = zScoreMaps[f][r.ticker];
if (z !== undefined) {
sum += z;
count++;
}
}
r.composite = count > 0 ? (sum / count) : null;
}
// Cross-sectional standardization and percentile assignment
[...factors, 'composite'].forEach(f => assignPercentiles(rawProfiles, f));
// Cache the raw profiles for single-stock comparisons
if (!injectedUniverseRecords) {
_cachedRawProfiles = rawProfiles;
_cachedZScoreMaps = zScoreMaps;
_cachedRawTime = Date.now();
}
// If this is a test injection, return the raw profiles with percentiles for testing
if (injectedUniverseRecords) return rawProfiles.reduce((acc, p) => { acc[p.ticker] = p; return acc; }, {});
// Clean up the raw scores, we only return the percentiles for the UI
const cleanedProfiles = rawProfiles.map(p => ({
ticker: p.ticker,
value: p.valuePercentile,
quality: p.qualityPercentile,
lowVol: p.lowVolPercentile,
momentum: p.momentumPercentile,
composite: p.compositePercentile,
_disclaimer: "Descriptive only. Shows where this stock ranks vs. peers on each factor today. We have not validated that these ranks predict returns — and our own testing showed standard technical signals do not. We'll only call a factor predictive after it passes out-of-sample validation."
}));
return cleanedProfiles;
}
/**
* Computes a single stock's factor profile relative to the cached universe.
*/
export async function getSingleFactorProfile(symbol) {
if (!_cachedRawProfiles || (Date.now() - _cachedRawTime > 120000)) {
await buildFactorProfiles();
}
// If it's already in the universe, just return it
const existing = _cachedRawProfiles.find(p => p.ticker === symbol);
if (existing) {
return {
ticker: existing.ticker,
value: existing.valuePercentile,
quality: existing.qualityPercentile,
lowVol: existing.lowVolPercentile,
momentum: existing.momentumPercentile,
composite: existing.compositePercentile,
_disclaimer: "Descriptive only. Shows where this stock ranks vs. the core universe."
};
}
// Fetch out-of-universe data
const data = await pullRawRecord(symbol);
if (!data) return null;
const raw = {
ticker: symbol,
value: FACTORS.value(data),
quality: FACTORS.quality(data),
lowVol: FACTORS.lowVol(data),
momentum: FACTORS.momentum(data)
};
// Standardize single value using universe mean/std from zScoreMaps... wait, standardizer only gives cross-sectional z-scores.
// We can just re-run standardizer with the universe + this one stock.
const tempRaw = [..._cachedRawProfiles, raw];
const factors = ['value', 'quality', 'lowVol', 'momentum'];
const zScoreMaps = {};
for (const f of factors) {
const rawByTicker = {};
for (const r of tempRaw) {
if (r[f] !== null && r[f] !== undefined) rawByTicker[r.ticker] = r[f];
}
zScoreMaps[f] = standardizeCrossSection(rawByTicker, false);
}
let sum = 0;
let count = 0;
for (const f of factors) {
const z = zScoreMaps[f][raw.ticker];
if (z !== undefined) {
sum += z;
count++;
}
}
raw.composite = count > 0 ? (sum / count) : null;
[...factors, 'composite'].forEach(f => assignPercentiles(tempRaw, f));
return {
ticker: raw.ticker,
value: raw.valuePercentile,
quality: raw.qualityPercentile,
lowVol: raw.lowVolPercentile,
momentum: raw.momentumPercentile,
composite: raw.compositePercentile,
_disclaimer: "Descriptive only. Shows where this out-of-universe stock ranks vs. the core universe today."
};
}

View File

@ -0,0 +1,106 @@
function mean(arr) {
const valid = arr.filter(x => x !== null && x !== undefined && !isNaN(x));
if (valid.length === 0) return null;
return valid.reduce((a, b) => a + b, 0) / valid.length;
}
export function orientedComponents(r) {
const inv = x => (x != null && x > 0) ? 1 / x : null; // guard neg/zero multiples
const neg = x => (x != null) ? -x : null;
const div = (a, b) => (a != null && b) ? a / b : null;
return {
value: [ inv(r.trailingPE), div(r.freeCashflow, r.marketCap),
inv(r.priceToBook), inv(r.priceToSales) ],
quality: [ r.returnOnEquity, r.grossMargins, neg(r.debtToEquity) ],
lowVol: [ neg(r.realizedVol252) ],
momentum: [ r.mom_12_1 ],
};
}
export const FACTORS = {
value: r => mean(orientedComponents(r).value),
quality: r => mean(orientedComponents(r).quality),
lowVol: r => mean(orientedComponents(r).lowVol),
momentum: r => mean(orientedComponents(r).momentum),
// composite is an equal-weight average of the standardized scores, computed later
};
function winsorize(arr, trimPercent) {
if (arr.length < 5) return arr; // Don't winsorize tiny arrays
const sorted = [...arr].sort((a, b) => a - b);
const lowerIdx = Math.floor(arr.length * trimPercent);
let upperIdx = Math.floor(arr.length * (1 - trimPercent)) - 1;
if (upperIdx < lowerIdx) upperIdx = arr.length - 1;
const lowerBound = sorted[lowerIdx];
const upperBound = sorted[upperIdx];
return arr.map(v => {
if (v < lowerBound) return lowerBound;
if (v > upperBound) return upperBound;
return v;
});
}
function avg(arr) {
if (!arr.length) return 0;
return arr.reduce((a, b) => a + b, 0) / arr.length;
}
function std(arr, meanVal) {
if (arr.length <= 1) return 1;
const variance = arr.reduce((sum, v) => sum + Math.pow(v - meanVal, 2), 0) / (arr.length - 1);
return Math.sqrt(variance) || 1;
}
export function standardizeCrossSection(rawByTicker, neutralizeSector = false, sectorsByTicker = {}) {
// Extract non-null values
const tickers = Object.keys(rawByTicker);
let validTickers = tickers.filter(t => rawByTicker[t] != null && !isNaN(rawByTicker[t]));
if (validTickers.length === 0) return {};
const result = {};
if (neutralizeSector) {
// Group by sector
const sectorGroups = {};
for (const t of validTickers) {
const sec = sectorsByTicker[t] || 'UNKNOWN';
if (!sectorGroups[sec]) sectorGroups[sec] = [];
sectorGroups[sec].push(t);
}
for (const sec in sectorGroups) {
const secTickers = sectorGroups[sec];
const secVals = secTickers.map(t => rawByTicker[t]);
const w = winsorize(secVals, 0.02);
const m = avg(w);
const s = std(w, m);
for (let i = 0; i < secTickers.length; i++) {
const t = secTickers[i];
let val = rawByTicker[t];
if (val < w[0]) val = w[0]; // approx winsorize clip (since w is same order)
// Actually, winsorize maps index-to-index.
const clipped = w[i];
result[t] = (clipped - m) / s;
}
}
} else {
// Global standard
const vals = validTickers.map(t => rawByTicker[t]);
const w = winsorize(vals, 0.02);
const m = avg(w);
const s = std(w, m);
for (let i = 0; i < validTickers.length; i++) {
const t = validTickers[i];
const clipped = w[i];
result[t] = (clipped - m) / s;
}
}
return result;
}

View File

@ -0,0 +1,53 @@
// In a real setup, this data would be loaded from Sharadar/Compustat.
// For testing, we allow an injected dataset.
let globalFundamentalsBySymbol = new Map();
export function loadFundamentals(data) {
globalFundamentalsBySymbol.clear();
// Group by symbol
for (const record of data) {
if (!globalFundamentalsBySymbol.has(record.symbol)) {
globalFundamentalsBySymbol.set(record.symbol, []);
}
globalFundamentalsBySymbol.get(record.symbol).push(record);
}
// Sort each group by datekey ascending
for (const [sym, records] of globalFundamentalsBySymbol.entries()) {
records.sort((a, b) => new Date(a.datekey) - new Date(b.datekey));
}
}
export function getFundamentalsAsOf(symbol, dateStr) {
const targetDate = new Date(dateStr);
const records = globalFundamentalsBySymbol.get(symbol);
if (!records) return null;
// Find the most recent fundamental report where filing date <= targetDate
let latest = null;
for (const record of records) {
// GUARDRAIL: We absolutely require a filing date (datekey).
// If datekey <= targetDate, it's safe.
// If datekey is missing, fallback to fiscalEnd + 4 months (120 days).
let isAvailable = false;
if (record.datekey) {
if (new Date(record.datekey) <= targetDate) isAvailable = true;
} else if (record.fiscalEnd) {
const safeReleaseDate = new Date(record.fiscalEnd);
safeReleaseDate.setDate(safeReleaseDate.getDate() + 120); // Add 4 months roughly
if (safeReleaseDate <= targetDate) isAvailable = true;
} else {
throw new Error(`Record for ${symbol} lacks both datekey and fiscalEnd. Unsafe.`);
}
if (isAvailable) {
latest = record;
} else {
if (record.datekey && new Date(record.datekey) > targetDate) break;
}
}
return latest ? latest.metrics : null;
}

View File

@ -0,0 +1,98 @@
import { generateSyntheticData } from './synthetic.js';
import { loadUniverse } from './universe.js';
import { loadFundamentals } from './fundamentals.js';
import { setForwardReturnsMap, runFactor } from './crossSection.js';
import { mean, neweyWestStdErr } from './metrics.js';
function printReport(title, results, horizonMonths) {
if (results.length === 0) {
console.log(`\n=== ${title} ===`);
console.log("No valid results.");
return;
}
const ics = results.map(r => r.ic);
const meanIc = mean(ics);
// We use Newey-West standard error for the t-stat if overlapping.
// For 1-month horizon, lags = 0 (simple std err).
// For H-month horizon, lags = H - 1.
const lags = Math.max(0, horizonMonths - 1);
const seIc = neweyWestStdErr(ics, lags);
const tStatIc = meanIc / seIc;
// Average Decile Returns (Gross)
const deciles = Array(10).fill(0);
for (const r of results) {
for (let d = 0; d < 10; d++) {
deciles[d] += r.decileReturns[d];
}
}
for (let d = 0; d < 10; d++) deciles[d] /= results.length;
// Long/Short Spread (Net)
const lsSpreads = results.map(r => r.d10Net - r.d1Net);
const meanLsSpread = mean(lsSpreads);
const seLsSpread = neweyWestStdErr(lsSpreads, lags);
const tStatLs = meanLsSpread / seLsSpread;
const sharpeLs = (meanLsSpread / (seLsSpread * Math.sqrt(results.length))) * Math.sqrt(12 / horizonMonths); // Approx Annualized Gross Sharpe
// Long-Only Top-Decile Excess (Net)
const loExcess = results.map(r => r.d10Net - r.universeMeanRet);
const meanLoExcess = mean(loExcess);
// Consistency
const numPositiveLs = lsSpreads.filter(s => s > 0).length;
const consistencyPct = (numPositiveLs / lsSpreads.length) * 100;
// Turnover (Avg over periods)
const avgTurnoverD10 = mean(results.map(r => r.d10Turnover)) * 100;
console.log(`\n=== ${title} ===`);
console.log(`Periods (N): ${results.length} months`);
console.log(`Horizon: ${horizonMonths} month(s)`);
console.log(`Mean IC: ${meanIc.toFixed(4)}`);
console.log(`IC t-stat: ${tStatIc.toFixed(2)} (Target: >= 3.0)`);
console.log(`L/S Spread (Net): ${(meanLsSpread * 100).toFixed(2)}% per period (t-stat: ${tStatLs.toFixed(2)})`);
console.log(`Long-Only Excess: ${(meanLoExcess * 100).toFixed(2)}% per period`);
console.log(`Consistency: ${consistencyPct.toFixed(1)}% positive periods`);
console.log(`D10 Avg Turnover: ${avgTurnoverD10.toFixed(1)}% per rebalance`);
console.log(`\nDecile Monotonicity (Gross):`);
const maxRet = Math.max(...deciles);
const minRet = Math.min(...deciles);
for (let d = 0; d < 10; d++) {
// simple text bar chart
const pct = deciles[d] * 100;
const barLen = Math.max(1, Math.floor((deciles[d] - minRet) / (maxRet - minRet + 0.0001) * 20));
console.log(` D${d+1}: ${pct.toFixed(2).padStart(5)}% | ${'#'.repeat(barLen)}`);
}
}
function runSyntheticTests() {
console.log("==================================================");
console.log(" SYNTHETIC KNOWN-ANSWER TESTS");
console.log("==================================================");
// Generate 25 years (300 months) of data for 500 stocks
// 1. NULL TEST (IC = 0)
const nullData = generateSyntheticData({ numMonths: 300, numStocks: 500, plantedIC: 0.0 });
loadUniverse(nullData.universe);
loadFundamentals(nullData.fundamentals);
setForwardReturnsMap(nullData.forwardReturnsMap);
const allDatesNull = nullData.universe.map(u => u.date);
const resultsNull = runFactor('value', allDatesNull, 1, 5); // 5 bps costs
printReport("TEST 1: NULL FACTOR (Target: IC ≈ 0, t-stat < 3)", resultsNull, 1);
// 2. PLANTED SIGNAL TEST (IC = 0.05)
const plantedData = generateSyntheticData({ numMonths: 300, numStocks: 500, plantedIC: 0.05 });
loadUniverse(plantedData.universe);
loadFundamentals(plantedData.fundamentals);
setForwardReturnsMap(plantedData.forwardReturnsMap);
const allDatesPlanted = plantedData.universe.map(u => u.date);
const resultsPlanted = runFactor('value', allDatesPlanted, 1, 5); // 5 bps costs
printReport("TEST 2: PLANTED SIGNAL (Target: IC ≈ 0.05, t-stat >= 3, Monotonic)", resultsPlanted, 1);
}
runSyntheticTests();

View File

@ -0,0 +1,75 @@
import { fetchQuoteSummary } from '../services/fundamentalsService.js';
const DEFAULT_HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'application/json',
};
async function fetchHistory(symbol) {
const url = `https://query1.finance.yahoo.com/v8/finance/chart/${symbol}?range=2y&interval=1d`;
try {
const res = await fetch(url, { headers: DEFAULT_HEADERS });
if (!res.ok) return null;
const data = await res.json();
const result = data?.chart?.result?.[0];
if (!result) return null;
return result.indicators?.quote?.[0]?.close || [];
} catch (e) {
return null;
}
}
function momentum12_1(closes) {
if (!closes || closes.length < 252) return null;
const n = closes.length;
// 12-1 month: close[-21] / close[-252] - 1
const current = closes[n - 21];
const past = closes[n - 252];
if (!current || !past) return null;
return (current / past) - 1;
}
function realizedVol252(closes) {
if (!closes || closes.length < 252) return null;
const n = closes.length;
const returns = [];
for (let i = n - 251; i < n; i++) {
const prev = closes[i - 1];
const cur = closes[i];
if (prev && cur) returns.push((cur - prev) / prev);
}
if (returns.length < 200) return null;
const mean = returns.reduce((a, b) => a + b, 0) / returns.length;
const variance = returns.reduce((a, b) => a + Math.pow(b - mean, 2), 0) / (returns.length - 1);
// Annualize daily vol
return Math.sqrt(variance) * Math.sqrt(252);
}
export async function pullRawRecord(ticker, asOf = new Date().toISOString().split('T')[0]) {
const info = await fetchQuoteSummary(ticker);
if (!info) return null;
const closes = await fetchHistory(ticker);
return {
snapshot_date: asOf,
ticker,
price: info.price,
marketCap: info.market_cap,
// raw inputs
trailingPE: info.pe_ttm,
priceToBook: info.price_to_book,
priceToSales: info.price_to_sales,
freeCashflow: info.free_cash_flow,
returnOnEquity: info.return_on_equity,
grossMargins: info.gross_margins,
debtToEquity: info.debt_to_equity,
mom_12_1: momentum12_1(closes),
realizedVol252: realizedVol252(closes),
fundamentals_period_end: info.fiscal_end
};
}

View File

@ -0,0 +1,84 @@
export function rank(arr) {
const sorted = arr.map((val, ind) => ({ val, ind })).sort((a, b) => a.val - b.val);
const ranks = new Array(arr.length);
for (let i = 0; i < sorted.length; i++) {
ranks[sorted[i].ind] = i + 1;
}
// Handle ties (average rank) if needed, but simple rank is fine for large N
return ranks;
}
export function spearman(x, y) {
if (x.length !== y.length || x.length === 0) return null;
const rankX = rank(x);
const rankY = rank(y);
const n = x.length;
let dSqSum = 0;
for (let i = 0; i < n; i++) {
dSqSum += Math.pow(rankX[i] - rankY[i], 2);
}
return 1 - ((6 * dSqSum) / (n * (Math.pow(n, 2) - 1)));
}
// Simple mean
export function mean(arr) {
if (!arr || arr.length === 0) return 0;
return arr.reduce((a, b) => a + b, 0) / arr.length;
}
// Simple standard deviation
export function std(arr, m) {
if (!arr || arr.length <= 1) return 1;
const variance = arr.reduce((sum, v) => sum + Math.pow(v - m, 2), 0) / (arr.length - 1);
return Math.sqrt(variance) || 1;
}
/**
* Calculate Newey-West standard error for a time-series of means
* Lags = 0 is equivalent to standard error.
* For N-month overlapping returns, lag = N - 1.
*/
export function neweyWestStdErr(ts, lags) {
const n = ts.length;
if (n <= 1) return 1;
const m = mean(ts);
// Variance
let s0 = 0;
for (let i = 0; i < n; i++) {
s0 += Math.pow(ts[i] - m, 2);
}
s0 = s0 / n;
// Covariances
let sLags = 0;
for (let l = 1; l <= lags; l++) {
let cov = 0;
for (let i = l; i < n; i++) {
cov += (ts[i] - m) * (ts[i - l] - m);
}
cov = cov / n;
const weight = 1 - (l / (lags + 1));
sLags += 2 * weight * cov;
}
const S = s0 + sLags;
return Math.sqrt(S / n) || (std(ts, m) / Math.sqrt(n)); // fallback
}
/**
* Compute turnover given two sets of weights.
* w1, w2 are objects: { 'AAPL': 0.05, 'MSFT': 0.02, ... }
* turnover = 0.5 * sum(|w2_i - w1_i|)
*/
export function calculateTurnover(wOld, wNew) {
if (!wOld) return 1.0; // 100% turnover on first month
let turnover = 0;
const allTickers = new Set([...Object.keys(wOld), ...Object.keys(wNew)]);
for (const t of allTickers) {
const oldWt = wOld[t] || 0;
const newWt = wNew[t] || 0;
turnover += Math.abs(newWt - oldWt);
}
return 0.5 * turnover;
}

Some files were not shown because too many files have changed in this diff Show More