Compare commits
No commits in common. "main" and "dec31" have entirely different histories.
|
|
@ -1,6 +0,0 @@
|
|||
node_modules/
|
||||
frontend/node_modules/
|
||||
backend/node_modules/
|
||||
frontend/dist/
|
||||
.git/
|
||||
.env
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
name: Build Institutional Trader
|
||||
on: [push]
|
||||
|
||||
jobs:
|
||||
build-and-deploy:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
DOCKER_HOST: tcp://172.17.0.1:2375
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Build and push
|
||||
run: |
|
||||
docker build -t 192.168.8.250:5000/market:latest .
|
||||
docker push 192.168.8.250:5000/market:latest
|
||||
43
Dockerfile
43
Dockerfile
|
|
@ -1,43 +0,0 @@
|
|||
# Stage 1: Build Frontend
|
||||
FROM node:18-alpine AS frontend-build
|
||||
WORKDIR /app/frontend
|
||||
COPY frontend/package*.json ./
|
||||
RUN npm install
|
||||
COPY frontend/ ./
|
||||
RUN npm run build
|
||||
|
||||
# Stage 2: Build Backend & Serve
|
||||
FROM node:18-alpine
|
||||
WORKDIR /app/backend
|
||||
|
||||
# Install python and build dependencies
|
||||
RUN apk add --no-cache python3 py3-pip python3-dev build-base && \
|
||||
python3 -m venv /opt/venv
|
||||
|
||||
ENV PATH="/opt/venv/bin:$PATH"
|
||||
|
||||
# Install backend dependencies
|
||||
COPY backend/package*.json ./
|
||||
RUN npm install --production
|
||||
|
||||
# Copy backend source
|
||||
COPY backend/ ./
|
||||
|
||||
# Install python dependencies
|
||||
RUN pip install --no-cache-dir -r python_service/requirements.txt
|
||||
|
||||
# Copy built frontend to the public directory expected by server.js
|
||||
# (__dirname is /app/backend/src, so ../../public is /app/public)
|
||||
COPY --from=frontend-build /app/frontend/dist /app/public
|
||||
|
||||
# Set production environment
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=3010
|
||||
|
||||
EXPOSE 3010
|
||||
|
||||
CMD ["node", "src/server.js"]
|
||||
|
||||
# Cache bust 20260625222919
|
||||
|
||||
# Cache bust 20260625223615
|
||||
|
|
@ -58,14 +58,7 @@ USE_PYTHON_SERVICE=true
|
|||
### Terminal 1: Python Service
|
||||
```bash
|
||||
cd backend/python_service
|
||||
# Activate virtual environment:
|
||||
# Git Bash on Windows:
|
||||
source venv/Scripts/activate
|
||||
# PowerShell/CMD on Windows:
|
||||
venv\Scripts\activate
|
||||
# Linux/macOS:
|
||||
source venv/bin/activate
|
||||
|
||||
source venv/bin/activate # or venv\Scripts\activate on Windows
|
||||
uvicorn main:app --reload --port 8010
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -19,13 +19,10 @@
|
|||
"express-rate-limit": "^7.1.5",
|
||||
"express-ws": "^5.0.2",
|
||||
"helmet": "^7.1.0",
|
||||
"ml-logistic-regression": "^2.0.0",
|
||||
"ml-matrix": "^6.13.0",
|
||||
"node-cache": "^5.1.2",
|
||||
"node-fetch": "^3.3.2",
|
||||
"pg": "^8.11.3",
|
||||
"ws": "^8.14.2",
|
||||
"xml2js": "^0.6.2"
|
||||
"ws": "^8.14.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "^3.0.1"
|
||||
|
|
@ -935,12 +932,6 @@
|
|||
"node": ">= 0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/is-any-array": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-any-array/-/is-any-array-3.0.0.tgz",
|
||||
"integrity": "sha512-o4h+tylWykC4BD1vaejp6gDxoM13bwW8FGuNs4yIKpj8xbBJcRxJx8vZpq0dCr7ZDEfeKjmsi/euolKhX6f/ww==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/is-binary-path": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
|
||||
|
|
@ -1082,54 +1073,6 @@
|
|||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/ml-array-max": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/ml-array-max/-/ml-array-max-2.0.0.tgz",
|
||||
"integrity": "sha512-QQZ4kENwpWmyNb98UXRDFXrmtIXuXtt1+bSbda/2KA85+F+rrJP8hZk6QOkCQXM2Th9mUDYdq/PNByPdT9ID4A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"is-any-array": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ml-array-min": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/ml-array-min/-/ml-array-min-2.0.0.tgz",
|
||||
"integrity": "sha512-GRj6Ky6sW9vGL6yIjgsHmXZ9YgrdmcQ8nCxPqEGeKc6dkfYg1XDYxGFxADUjNuZyoCd5PUscWAS4N+cFaX6hFg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"is-any-array": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ml-array-rescale": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/ml-array-rescale/-/ml-array-rescale-2.0.0.tgz",
|
||||
"integrity": "sha512-2GGtKfSno94/kIloWGvpp/U5Q5vLvLrza+SAaGsLeo6Xj4mEbA6Gqx+oTfZFkxnd1grT2X007HfJNs3T5BsiVg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"is-any-array": "^3.0.0",
|
||||
"ml-array-max": "^2.0.0",
|
||||
"ml-array-min": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ml-logistic-regression": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/ml-logistic-regression/-/ml-logistic-regression-2.0.0.tgz",
|
||||
"integrity": "sha512-xHhB91ut8GRRbJyB1ZQfKsl1MHmE1PqMeRjxhks96M5BGvCbC9eEojf4KgRMKM2LxFblhVUcVzweAoPB48Nt0A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ml-matrix": "^6.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ml-matrix": {
|
||||
"version": "6.13.0",
|
||||
"resolved": "https://registry.npmjs.org/ml-matrix/-/ml-matrix-6.13.0.tgz",
|
||||
"integrity": "sha512-QpV0UTUkglg6vPUgThKGBEtit2ac6habSoZ33bwI9rU0UHZLqw6G3ukTIE8zWiUF3sjK8YAlhx/o/b9layzH8A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"is-any-array": "^3.0.0",
|
||||
"ml-array-rescale": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ms": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
|
||||
|
|
@ -1556,15 +1499,6 @@
|
|||
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/sax": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz",
|
||||
"integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==",
|
||||
"license": "BlueOak-1.0.0",
|
||||
"engines": {
|
||||
"node": ">=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/semver": {
|
||||
"version": "7.7.3",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz",
|
||||
|
|
@ -1961,28 +1895,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/xml2js": {
|
||||
"version": "0.6.2",
|
||||
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz",
|
||||
"integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"sax": ">=0.6.0",
|
||||
"xmlbuilder": "~11.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/xmlbuilder": {
|
||||
"version": "11.0.1",
|
||||
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
|
||||
"integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/xtend": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
|
||||
|
|
|
|||
|
|
@ -33,13 +33,10 @@
|
|||
"express-rate-limit": "^7.1.5",
|
||||
"express-ws": "^5.0.2",
|
||||
"helmet": "^7.1.0",
|
||||
"ml-logistic-regression": "^2.0.0",
|
||||
"ml-matrix": "^6.13.0",
|
||||
"node-cache": "^5.1.2",
|
||||
"node-fetch": "^3.3.2",
|
||||
"pg": "^8.11.3",
|
||||
"ws": "^8.14.2",
|
||||
"xml2js": "^0.6.2"
|
||||
"ws": "^8.14.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "^3.0.1"
|
||||
|
|
|
|||
|
|
@ -1,86 +0,0 @@
|
|||
{
|
||||
"success": true,
|
||||
"data": [
|
||||
{
|
||||
"CreatedDate": "2025-01-15",
|
||||
"CreatedTime": "10:30:45 AM",
|
||||
"Symbol": "(5🟩) AAPL · RTH ⚡ 🟢💎⭐💰 ⚡ 🔥",
|
||||
"Rocket": "🚀🚀🚀 [ITM 2.5%]",
|
||||
"NetPremium": "1.50 M",
|
||||
"Premium": "850.00 K",
|
||||
|
||||
"premium_num": 850000,
|
||||
"bull_total": 1500000,
|
||||
"bear_total": 50000,
|
||||
|
||||
"PctVsPriorClose": 1.2,
|
||||
"PctVsRthOpen": 0.8,
|
||||
"Pct5m": 0.3,
|
||||
"Pct15m": 0.5,
|
||||
"TapeAlign": "↗︎",
|
||||
"NearAlert": "EARNINGS",
|
||||
|
||||
"ExpirationDate": "2025-01-17",
|
||||
"Price": "185.50",
|
||||
"CallPut": "CALL",
|
||||
"Side": "AA",
|
||||
"Strike": "185",
|
||||
"Spot": "185.50",
|
||||
"Volume": "5000",
|
||||
"OI": "3000",
|
||||
|
||||
"direction": "BULL",
|
||||
"badge_round": "🟢",
|
||||
"badge_more": "💎⭐💰✔",
|
||||
"flash": "⚡",
|
||||
"rocket_score": 5.2,
|
||||
|
||||
"session_bucket": "RTH",
|
||||
"flow_ts_utc": "2025-01-15T16:30:45+00:00",
|
||||
"mny_pct": 2.5,
|
||||
|
||||
"vwap_at_signal": 185.20,
|
||||
"price_vs_vwap_pct": 0.16,
|
||||
|
||||
"signal_tier": "TIER_1",
|
||||
"checklist_score": 8.5,
|
||||
"checklist_passed": true,
|
||||
|
||||
"premium_zscore": 2.3,
|
||||
"premium_percentile_intraday": 85.5,
|
||||
"relative_premium_score": 72.5,
|
||||
|
||||
"aggression_score": 68.0,
|
||||
"size_concentration_score": 75.0,
|
||||
"repeat_trade_velocity": 55.0,
|
||||
"strike_clustering_score": 60.0,
|
||||
"signal_strength": 64.7,
|
||||
|
||||
"early_noise_reject": false,
|
||||
|
||||
"flow_acceleration": 12500.5,
|
||||
"time_between_hits": 3.2,
|
||||
"follow_on_ratio": 0.75,
|
||||
"strike_laddering_detected": true,
|
||||
|
||||
"delta_exposure": 92500000,
|
||||
"gamma_exposure": 1250000000,
|
||||
"volatility_intent": "LONG_VOL",
|
||||
|
||||
"net_gamma_exposure_per_symbol": 8500000000,
|
||||
"gamma_flip_proximity": 0.15,
|
||||
"dealer_hedge_pressure_score": 72.5,
|
||||
|
||||
"market_regime": "TREND",
|
||||
"flow_state": "ACTIONABLE",
|
||||
|
||||
"confidence_score": 78.5,
|
||||
"institutional_likelihood": 0.85,
|
||||
"dealer_pain_level": 65.0,
|
||||
"expected_move_vs_implied": 1.35
|
||||
}
|
||||
],
|
||||
"count": 1,
|
||||
"timestamp": "2025-01-15T16:35:00.123456"
|
||||
}
|
||||
|
||||
|
|
@ -1,168 +0,0 @@
|
|||
# Institutional-Grade Options Flow Analytics
|
||||
|
||||
This document describes the institutional-grade enhancements to the options flow pipeline.
|
||||
|
||||
## Overview
|
||||
|
||||
The pipeline has been refactored to convert static retail-style flow detection into dynamic, dealer-aware, time-sequenced signals suitable for intraday momentum and 1-5 day swing trades.
|
||||
|
||||
## New Analytics Modules
|
||||
|
||||
### 1. Relative Premium Scoring (`relative_premium_scorer.py`)
|
||||
|
||||
**Purpose**: Replace static premium filter (minPremium = $80K) with context-aware relative scoring.
|
||||
|
||||
**New Fields**:
|
||||
- `premium_zscore`: Z-score of premium relative to 20-day rolling window per ticker
|
||||
- `premium_percentile_intraday`: Percentile rank within same-day flow
|
||||
- `relative_premium_score`: Composite score (0-100) combining z-score, intraday percentile, and median normalization
|
||||
|
||||
**Usage**: Premium of $80K might be significant for AAPL but noise for TSLA. This module computes relative significance.
|
||||
|
||||
### 2. Signal Component Scoring (`signal_component_scorer.py`)
|
||||
|
||||
**Purpose**: Convert binary badge logic (💎 ⭐ 🟢 🔴) into continuous numeric signal components.
|
||||
|
||||
**New Fields**:
|
||||
- `aggression_score`: Measures trade aggression (ITM premiums, ask-side trades)
|
||||
- `size_concentration_score`: Measures size concentration (single large trade vs many small ones)
|
||||
- `repeat_trade_velocity`: Measures repeat trade frequency (urgency building)
|
||||
- `strike_clustering_score`: Measures strike clustering (laddering patterns)
|
||||
- `signal_strength`: Composite score = 0.30 * aggression + 0.30 * size_concentration + 0.20 * repeat_velocity + 0.20 * strike_clustering
|
||||
|
||||
**Note**: Badges remain display-only. Signal strength is computed from components.
|
||||
|
||||
### 3. Tier-0 Noise Rejection (`noise_rejector.py`)
|
||||
|
||||
**Purpose**: Reject low-quality signals before enrichment to reduce processing overhead.
|
||||
|
||||
**New Fields**:
|
||||
- `early_noise_reject`: Boolean flag indicating if signal should be rejected as noise
|
||||
|
||||
**Rejection Criteria**:
|
||||
- Single isolated trade (no repeat activity within 30 minutes)
|
||||
- Far OTM weekly lottos (>15% OTM with <7 days to expiry)
|
||||
- Delta-adjusted premium below threshold (<$50K)
|
||||
|
||||
### 4. Time-Sequenced Flow Analysis (`time_sequenced_analyzer.py`)
|
||||
|
||||
**Purpose**: Analyze flow patterns over time to detect urgency, distribution, and continuation.
|
||||
|
||||
**New Fields**:
|
||||
- `flow_acceleration`: Change in premium per minute (Δ premium / minute)
|
||||
- `time_between_hits`: Average time between consecutive trades (minutes)
|
||||
- `follow_on_ratio`: Fraction of trades in same direction after initial trade (0-1)
|
||||
- `strike_laddering_detected`: Boolean indicating sequential strike accumulation
|
||||
|
||||
**Interpretation**:
|
||||
- Escalating premium + decreasing time gaps = urgency
|
||||
- Flat premium + widening gaps = distribution
|
||||
|
||||
### 5. Intent Classification (`intent_classifier.py`)
|
||||
|
||||
**Purpose**: Replace naive direction (BULL/BEAR) with nuanced volatility and hedging intent.
|
||||
|
||||
**New Fields**:
|
||||
- `delta_exposure`: Delta exposure (contracts * delta * 100 * spot_price)
|
||||
- `gamma_exposure`: Gamma exposure (contracts * gamma * 100 * spot_price^2)
|
||||
- `volatility_intent`: Enum (LONG_VOL, SHORT_VOL, DIRECTIONAL, HEDGE_UNWIND)
|
||||
|
||||
**Note**: Direction (BULL/BEAR) becomes secondary metadata.
|
||||
|
||||
### 6. Dealer-Aware Flow Context (`dealer_flow_context.py`)
|
||||
|
||||
**Purpose**: Track dealer hedging pressure and gamma exposure.
|
||||
|
||||
**New Fields**:
|
||||
- `net_gamma_exposure_per_symbol`: Sum of gamma exposures for symbol (positive = long gamma, negative = short gamma)
|
||||
- `gamma_flip_proximity`: Proximity to gamma flip point (-1 to 1)
|
||||
- `dealer_hedge_pressure_score`: Dealer hedge pressure score (0-100)
|
||||
|
||||
**Usage**: Validates flow continuation, flow reversals, and gamma squeeze setups.
|
||||
|
||||
### 7. Market Regime Detection (`market_regime_detector.py`)
|
||||
|
||||
**Purpose**: Identify market regime to gate trade signal generation.
|
||||
|
||||
**New Fields**:
|
||||
- `market_regime`: Enum (TREND, RANGE, HIGH_VOL_EVENT)
|
||||
|
||||
**Trade Signal Gating**:
|
||||
- Trend → continuation bias
|
||||
- Range → fade or vol-sell bias
|
||||
- Event → volatility expansion bias
|
||||
|
||||
### 8. Flow Decay & Reversal Validation (`flow_decay_validator.py`)
|
||||
|
||||
**Purpose**: Validate flow decay/reversal signals with anchors.
|
||||
|
||||
**New Fields**:
|
||||
- `flow_state`: Enum (ACTIONABLE, INFORMATIONAL)
|
||||
|
||||
**Validation Criteria**: Flow decay/reversal is actionable ONLY IF:
|
||||
- Premium contracts (relative_premium_score >= 60)
|
||||
- Dealer hedge pressure decreases
|
||||
- Price fails near VWAP / opening range / key level
|
||||
- Otherwise marked as INFORMATIONAL
|
||||
|
||||
### 9. Institutional Confidence Metrics (`institutional_confidence.py`)
|
||||
|
||||
**Purpose**: Calculate confidence scores for institutional flow signals.
|
||||
|
||||
**New Fields**:
|
||||
- `confidence_score`: Overall confidence score (0-100)
|
||||
- `institutional_likelihood`: Likelihood flow is institutional (0-1)
|
||||
- `dealer_pain_level`: Dealer pain level (0-100)
|
||||
- `expected_move_vs_implied`: Expected move vs implied move ratio
|
||||
|
||||
## Integration
|
||||
|
||||
All modules are integrated into the main processing pipeline in `main.py`:
|
||||
|
||||
1. Basic flow processing (normalization, badges, rocket score)
|
||||
2. Price context enrichment
|
||||
3. Alert matching
|
||||
4. **Institutional analytics pipeline** (NEW):
|
||||
- Tier-0 noise rejection
|
||||
- Relative premium scoring
|
||||
- Signal component scoring
|
||||
- Time-sequenced analysis
|
||||
- Intent classification
|
||||
- Dealer flow context
|
||||
- Market regime detection
|
||||
- Flow decay validation
|
||||
- Confidence metrics
|
||||
5. Filtering (premium, relative premium, badges, direction)
|
||||
6. Output formatting
|
||||
|
||||
## Filtering Changes
|
||||
|
||||
**Before**:
|
||||
- Static premium filter: `premium_num > 80000`
|
||||
- Badge requirements: 🟢/🔴 + 💎 + ⭐
|
||||
|
||||
**After**:
|
||||
- Static premium filter: `premium_num > min_premium` (still applied)
|
||||
- **Relative premium filter**: `relative_premium_score >= 60.0` (NEW)
|
||||
- **Noise rejection filter**: `early_noise_reject == False` (NEW)
|
||||
- Badge requirements: 🟢/🔴 + 💎 + ⭐ (still applied, but badges are now display-only)
|
||||
|
||||
## API Response
|
||||
|
||||
All new fields are included in the API response. The response maintains backward compatibility - existing fields remain unchanged, new fields are additive.
|
||||
|
||||
## Design Philosophy
|
||||
|
||||
1. **Flow represents pressure, not prediction**: Signals indicate who is forced to act next (dealers hedging)
|
||||
2. **Institutions trade urgency and forced hedging**: Focus on dealer pain and gamma exposure
|
||||
3. **Fewer, higher-quality signals > more alerts**: Noise rejection and relative premium filtering reduce false positives
|
||||
4. **Every signal must answer**: "Who is forced to act next?"
|
||||
|
||||
## Success Criteria
|
||||
|
||||
If implemented correctly:
|
||||
- Signal count decreases (noise filtered out)
|
||||
- Average signal quality increases (relative premium, signal strength)
|
||||
- False positives reduce (noise rejection, dealer context validation)
|
||||
- Trades align with intraday momentum and short-term swing horizons (time-sequenced analysis)
|
||||
|
||||
Binary file not shown.
|
|
@ -192,63 +192,6 @@ async def get_options_flow(
|
|||
# Recalculate rocket score with price context and alerts
|
||||
df_final = processor.process_rocket_score(df_final)
|
||||
|
||||
# Apply institutional-grade analytics pipeline
|
||||
logger.info("🔹 Applying institutional-grade analytics...")
|
||||
from services.relative_premium_scorer import RelativePremiumScorer
|
||||
from services.noise_rejector import NoiseRejector
|
||||
from services.signal_component_scorer import SignalComponentScorer
|
||||
from services.time_sequenced_analyzer import TimeSequencedAnalyzer
|
||||
from services.intent_classifier import IntentClassifier
|
||||
from services.dealer_flow_context import DealerFlowContext
|
||||
from services.market_regime_detector import MarketRegimeDetector
|
||||
from services.flow_decay_validator import FlowDecayValidator
|
||||
from services.institutional_confidence import InstitutionalConfidence
|
||||
|
||||
# 1. Tier-0 Noise Rejection (mark but don't filter yet - filtering happens later)
|
||||
logger.info("1️⃣ Applying tier-0 noise rejection...")
|
||||
noise_rejector = NoiseRejector()
|
||||
df_final = noise_rejector.mark_noise_rejections(df_final)
|
||||
|
||||
# 2. Relative Premium Scoring
|
||||
logger.info("2️⃣ Calculating relative premium scores...")
|
||||
premium_scorer = RelativePremiumScorer(pool)
|
||||
df_final = await premium_scorer.enrich_with_relative_premium(df_final)
|
||||
|
||||
# 3. Signal Component Scoring (convert badges to continuous scores)
|
||||
logger.info("3️⃣ Converting badges to continuous signal components...")
|
||||
signal_scorer = SignalComponentScorer()
|
||||
df_final = signal_scorer.enrich_with_signal_components(df_final)
|
||||
|
||||
# 4. Time-Sequenced Flow Analysis
|
||||
logger.info("4️⃣ Analyzing time-sequenced flow patterns...")
|
||||
time_analyzer = TimeSequencedAnalyzer()
|
||||
df_final = time_analyzer.enrich_with_time_sequenced_metrics(df_final)
|
||||
|
||||
# 5. Intent Classification
|
||||
logger.info("5️⃣ Classifying volatility and hedging intent...")
|
||||
intent_classifier = IntentClassifier()
|
||||
df_final = intent_classifier.enrich_with_intent_classification(df_final)
|
||||
|
||||
# 6. Dealer-Aware Flow Context
|
||||
logger.info("6️⃣ Analyzing dealer hedging pressure...")
|
||||
dealer_context = DealerFlowContext()
|
||||
df_final = dealer_context.enrich_with_dealer_context(df_final)
|
||||
|
||||
# 7. Market Regime Detection
|
||||
logger.info("7️⃣ Detecting market regime...")
|
||||
regime_detector = MarketRegimeDetector()
|
||||
df_final = regime_detector.enrich_with_market_regime(df_final)
|
||||
|
||||
# 8. Flow Decay & Reversal Validation
|
||||
logger.info("8️⃣ Validating flow decay and reversal signals...")
|
||||
flow_validator = FlowDecayValidator()
|
||||
df_final = flow_validator.enrich_with_flow_state(df_final)
|
||||
|
||||
# 9. Institutional Confidence Metrics
|
||||
logger.info("9️⃣ Calculating institutional confidence metrics...")
|
||||
confidence_calc = InstitutionalConfidence()
|
||||
df_final = confidence_calc.enrich_with_confidence_metrics(df_final)
|
||||
|
||||
# Phase 1 Enhancements (BEFORE filtering so all signals get Phase 1 data):
|
||||
# Initialize Phase 1 columns with None/empty values first
|
||||
if not df_final.empty:
|
||||
|
|
@ -316,13 +259,6 @@ async def get_options_flow(
|
|||
# Filter by minimum premium and badge requirements (matching SQL WHERE clause)
|
||||
logger.info(f"📊 Before filtering: {len(df_final)} rows")
|
||||
|
||||
# Apply noise rejection filter first (exclude early_noise_reject = True)
|
||||
if 'early_noise_reject' in df_final.columns:
|
||||
before_noise = len(df_final)
|
||||
df_final = df_final[~df_final['early_noise_reject']].copy()
|
||||
after_noise = len(df_final)
|
||||
logger.info(f"📊 After noise rejection filter: {after_noise} rows (removed {before_noise - after_noise})")
|
||||
|
||||
# Only filter if columns exist
|
||||
if 'premium_num' in df_final.columns:
|
||||
before_premium = len(df_final)
|
||||
|
|
@ -332,14 +268,6 @@ async def get_options_flow(
|
|||
else:
|
||||
logger.warning("⚠️ premium_num column not found, skipping premium filter")
|
||||
|
||||
# Apply relative premium filter if available
|
||||
if 'relative_premium_score' in df_final.columns:
|
||||
before_relative = len(df_final)
|
||||
min_relative_threshold = 60.0 # Configurable threshold
|
||||
df_final = df_final[df_final['relative_premium_score'] >= min_relative_threshold].copy()
|
||||
after_relative = len(df_final)
|
||||
logger.info(f"📊 After relative premium filter (>={min_relative_threshold}): {after_relative} rows (removed {before_relative - after_relative})")
|
||||
|
||||
if df_final.empty:
|
||||
logger.warning("⚠️ No data after premium filter")
|
||||
return OptionsFlowResponse(
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
|
|
@ -1,256 +0,0 @@
|
|||
"""
|
||||
Dealer-Aware Flow Context Service
|
||||
Tracks dealer hedging pressure, gamma exposure, and flow continuation patterns
|
||||
"""
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from typing import Dict, Optional
|
||||
from datetime import timedelta
|
||||
from utils.logger import logger
|
||||
|
||||
|
||||
class DealerFlowContext:
|
||||
"""
|
||||
Analyzes flow from dealer perspective:
|
||||
- Net gamma exposure per symbol
|
||||
- Gamma flip proximity (when dealers become long/short gamma)
|
||||
- Dealer hedge pressure (forced hedging activity)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Configuration
|
||||
self.analysis_window_minutes = 120 # Look back window for gamma tracking
|
||||
self.gamma_flip_threshold = 0.3 # Gamma flip when net gamma crosses threshold
|
||||
|
||||
def calculate_net_gamma_exposure_per_symbol(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
symbol: str,
|
||||
timestamp: pd.Timestamp
|
||||
) -> float:
|
||||
"""
|
||||
Calculate net gamma exposure for a symbol at a given time.
|
||||
Sums all gamma exposures from recent flow.
|
||||
|
||||
Positive = dealers are long gamma (hedging by selling on rallies, buying on dips)
|
||||
Negative = dealers are short gamma (hedging by buying on rallies, selling on dips)
|
||||
"""
|
||||
if df.empty:
|
||||
return 0.0
|
||||
|
||||
# Look at flow up to this timestamp
|
||||
window_start = timestamp - timedelta(minutes=self.analysis_window_minutes)
|
||||
|
||||
mask = (
|
||||
(df['symbol_norm'] == symbol.upper()) &
|
||||
(df['flow_ts_utc'] >= window_start) &
|
||||
(df['flow_ts_utc'] <= timestamp) &
|
||||
(df['gamma_exposure'].notna())
|
||||
)
|
||||
|
||||
recent_flow = df[mask]
|
||||
|
||||
if recent_flow.empty:
|
||||
return 0.0
|
||||
|
||||
# Sum gamma exposures
|
||||
net_gamma = recent_flow['gamma_exposure'].sum()
|
||||
|
||||
return float(net_gamma)
|
||||
|
||||
def calculate_gamma_flip_proximity(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
row_idx: int
|
||||
) -> Optional[float]:
|
||||
"""
|
||||
Calculate proximity to gamma flip (when dealers switch from long to short gamma or vice versa).
|
||||
|
||||
Returns: -1.0 to 1.0
|
||||
- Positive = approaching long gamma (dealers becoming volatility buyers)
|
||||
- Negative = approaching short gamma (dealers becoming volatility sellers)
|
||||
- 0.0 = near flip point
|
||||
"""
|
||||
if row_idx >= len(df):
|
||||
return None
|
||||
|
||||
current_row = df.iloc[row_idx]
|
||||
symbol = current_row.get('symbol_norm')
|
||||
flow_ts_utc = current_row.get('flow_ts_utc')
|
||||
gamma_exposure = current_row.get('gamma_exposure', 0) or 0
|
||||
|
||||
if pd.isna(symbol) or pd.isna(flow_ts_utc):
|
||||
return None
|
||||
|
||||
# Calculate current net gamma
|
||||
net_gamma = self.calculate_net_gamma_exposure_per_symbol(df, symbol, flow_ts_utc)
|
||||
|
||||
# Normalize to -1 to 1 scale
|
||||
# Use exponential scaling to emphasize near-flip conditions
|
||||
if net_gamma == 0:
|
||||
return 0.0
|
||||
|
||||
# Simple normalization: divide by a large threshold
|
||||
# More sophisticated: use percentile or adaptive scaling
|
||||
threshold = 1000000000 # 1B in gamma exposure as normalization factor
|
||||
normalized = net_gamma / threshold
|
||||
|
||||
# Clamp to -1 to 1
|
||||
normalized = max(-1.0, min(1.0, normalized))
|
||||
|
||||
# Invert: negative normalized = positive proximity (approaching long gamma)
|
||||
# This is because dealer short gamma (negative) means they're selling volatility
|
||||
return float(-normalized)
|
||||
|
||||
def calculate_dealer_hedge_pressure_score(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
row_idx: int
|
||||
) -> float:
|
||||
"""
|
||||
Calculate dealer hedge pressure score (0-100).
|
||||
|
||||
Higher score = more forced hedging by dealers:
|
||||
- High net gamma exposure (dealers must hedge)
|
||||
- Recent flow creating gamma imbalance
|
||||
- Flow continuation (dealers hedging creates more flow)
|
||||
"""
|
||||
if row_idx >= len(df):
|
||||
return 0.0
|
||||
|
||||
current_row = df.iloc[row_idx]
|
||||
symbol = current_row.get('symbol_norm')
|
||||
flow_ts_utc = current_row.get('flow_ts_utc')
|
||||
gamma_exposure = current_row.get('gamma_exposure', 0) or 0
|
||||
|
||||
if pd.isna(symbol) or pd.isna(flow_ts_utc):
|
||||
return 0.0
|
||||
|
||||
score = 0.0
|
||||
|
||||
# Net gamma component (0-40 points)
|
||||
# High absolute net gamma = more hedge pressure
|
||||
net_gamma = abs(self.calculate_net_gamma_exposure_per_symbol(df, symbol, flow_ts_utc))
|
||||
if net_gamma > 0:
|
||||
# Normalize: 500M = 20 points, 1B = 40 points
|
||||
normalized_gamma = min(1.0, net_gamma / 1000000000) # 1B threshold
|
||||
score += normalized_gamma * 40.0
|
||||
|
||||
# Recent gamma accumulation (0-30 points)
|
||||
# Recent flow creating gamma imbalance
|
||||
window_start = flow_ts_utc - timedelta(minutes=30)
|
||||
mask = (
|
||||
(df['symbol_norm'] == symbol.upper()) &
|
||||
(df['flow_ts_utc'] >= window_start) &
|
||||
(df['flow_ts_utc'] <= flow_ts_utc) &
|
||||
(df['gamma_exposure'].notna())
|
||||
)
|
||||
recent_gamma = df[mask]['gamma_exposure'].sum()
|
||||
|
||||
if abs(recent_gamma) > 0:
|
||||
# Normalize recent gamma accumulation
|
||||
normalized_recent = min(1.0, abs(recent_gamma) / 500000000) # 500M threshold
|
||||
score += normalized_recent * 30.0
|
||||
|
||||
# Flow continuation component (0-30 points)
|
||||
# If flow is continuing in same direction, dealers are likely hedging
|
||||
follow_on_ratio = current_row.get('follow_on_ratio')
|
||||
if follow_on_ratio is not None and not pd.isna(follow_on_ratio):
|
||||
# High follow-on ratio = continuation = hedge pressure
|
||||
score += follow_on_ratio * 30.0
|
||||
|
||||
return min(100.0, max(0.0, score))
|
||||
|
||||
def validate_flow_continuation(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
row_idx: int
|
||||
) -> bool:
|
||||
"""
|
||||
Validate if flow continuation is likely based on dealer hedge pressure.
|
||||
Returns True if continuation is expected (dealers forced to hedge).
|
||||
"""
|
||||
current_row = df.iloc[row_idx]
|
||||
dealer_pressure = current_row.get('dealer_hedge_pressure_score', 0) or 0
|
||||
net_gamma = current_row.get('net_gamma_exposure_per_symbol', 0) or 0
|
||||
|
||||
# High dealer pressure + significant gamma exposure = continuation likely
|
||||
if dealer_pressure > 50.0 and abs(net_gamma) > 100000000: # 100M threshold
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def validate_flow_reversal(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
row_idx: int
|
||||
) -> bool:
|
||||
"""
|
||||
Validate if flow reversal is likely.
|
||||
Reversal happens when:
|
||||
- Dealers finish hedging (gamma exposure neutralizes)
|
||||
- Price fails at key levels (VWAP, opening range)
|
||||
- Flow shows distribution pattern (decreasing premium, widening gaps)
|
||||
"""
|
||||
current_row = df.iloc[row_idx]
|
||||
dealer_pressure = current_row.get('dealer_hedge_pressure_score', 0) or 0
|
||||
follow_on_ratio = current_row.get('follow_on_ratio')
|
||||
flow_acceleration = current_row.get('flow_acceleration')
|
||||
|
||||
# Low dealer pressure + low follow-on ratio = reversal likely
|
||||
if dealer_pressure < 30.0:
|
||||
if follow_on_ratio is not None and follow_on_ratio < 0.3:
|
||||
return True
|
||||
|
||||
# Negative flow acceleration = flow weakening = reversal
|
||||
if flow_acceleration is not None and flow_acceleration < -10000:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def enrich_with_dealer_context(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Add dealer-aware flow context metrics to DataFrame.
|
||||
Adds: net_gamma_exposure_per_symbol, gamma_flip_proximity, dealer_hedge_pressure_score
|
||||
"""
|
||||
if df.empty:
|
||||
return df
|
||||
|
||||
df = df.copy()
|
||||
|
||||
# Initialize columns
|
||||
df['net_gamma_exposure_per_symbol'] = 0.0
|
||||
df['gamma_flip_proximity'] = None
|
||||
df['dealer_hedge_pressure_score'] = 0.0
|
||||
|
||||
# Sort by timestamp for proper gamma tracking
|
||||
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
|
||||
|
||||
# Calculate metrics for each row
|
||||
for idx in df_sorted.index:
|
||||
original_idx = df_sorted.index[idx]
|
||||
row = df_sorted.iloc[idx]
|
||||
|
||||
symbol = row.get('symbol_norm')
|
||||
flow_ts_utc = row.get('flow_ts_utc')
|
||||
|
||||
if pd.notna(symbol) and pd.notna(flow_ts_utc):
|
||||
# Net gamma exposure
|
||||
net_gamma = self.calculate_net_gamma_exposure_per_symbol(
|
||||
df_sorted, symbol, flow_ts_utc
|
||||
)
|
||||
df.at[original_idx, 'net_gamma_exposure_per_symbol'] = float(net_gamma)
|
||||
|
||||
# Gamma flip proximity
|
||||
flip_prox = self.calculate_gamma_flip_proximity(df_sorted, idx)
|
||||
if flip_prox is not None:
|
||||
df.at[original_idx, 'gamma_flip_proximity'] = float(flip_prox)
|
||||
|
||||
# Dealer hedge pressure
|
||||
pressure = self.calculate_dealer_hedge_pressure_score(df_sorted, idx)
|
||||
df.at[original_idx, 'dealer_hedge_pressure_score'] = float(pressure)
|
||||
|
||||
logger.info(f"Dealer context enrichment complete. Mean pressure: {df['dealer_hedge_pressure_score'].mean():.2f}")
|
||||
|
||||
return df
|
||||
|
||||
|
|
@ -1,118 +0,0 @@
|
|||
"""
|
||||
Flow Decay & Reversal Validation Service
|
||||
Validates flow decay/reversal signals with anchors (premium, dealer pressure, price levels)
|
||||
"""
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from typing import Optional
|
||||
from enum import Enum
|
||||
from utils.logger import logger
|
||||
|
||||
|
||||
class FlowState(Enum):
|
||||
"""Flow state classification"""
|
||||
ACTIONABLE = "ACTIONABLE" # Flow decay/reversal is actionable (trade signal)
|
||||
INFORMATIONAL = "INFORMATIONAL" # Flow decay/reversal is informational only (no trade signal)
|
||||
|
||||
|
||||
class FlowDecayValidator:
|
||||
"""
|
||||
Validates flow decay and reversal signals.
|
||||
Flow decay/reversal is actionable ONLY IF:
|
||||
- Premium contracts (high relative premium)
|
||||
- Dealer hedge pressure decreases
|
||||
- Price fails near VWAP / opening range / key level
|
||||
Otherwise mark as INFORMATIONAL.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Configuration
|
||||
self.min_relative_premium_for_actionable = 60.0 # Minimum relative premium score
|
||||
self.vwap_failure_threshold_pct = 0.5 # Price within 0.5% of VWAP = failure
|
||||
self.dealer_pressure_decrease_threshold = 20.0 # Dealer pressure decrease threshold
|
||||
|
||||
def validate_flow_decay_reversal(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
row_idx: int
|
||||
) -> str:
|
||||
"""
|
||||
Validate if flow decay/reversal is actionable.
|
||||
Returns FlowState enum value as string.
|
||||
"""
|
||||
if row_idx >= len(df):
|
||||
return FlowState.INFORMATIONAL.value
|
||||
|
||||
current_row = df.iloc[row_idx]
|
||||
|
||||
# Check 1: Premium contracts (relative premium score)
|
||||
relative_premium_score = current_row.get('relative_premium_score', 0) or 0
|
||||
if relative_premium_score < self.min_relative_premium_for_actionable:
|
||||
return FlowState.INFORMATIONAL.value
|
||||
|
||||
# Check 2: Dealer hedge pressure decreases
|
||||
# Look at recent dealer pressure trend
|
||||
symbol = current_row.get('symbol_norm')
|
||||
flow_ts_utc = current_row.get('flow_ts_utc')
|
||||
|
||||
if pd.notna(symbol) and pd.notna(flow_ts_utc):
|
||||
from datetime import timedelta
|
||||
window_start = flow_ts_utc - timedelta(minutes=30)
|
||||
|
||||
mask = (
|
||||
(df['symbol_norm'] == symbol.upper()) &
|
||||
(df['flow_ts_utc'] >= window_start) &
|
||||
(df['flow_ts_utc'] <= flow_ts_utc) &
|
||||
(df['dealer_hedge_pressure_score'].notna())
|
||||
)
|
||||
|
||||
recent_pressure = df[mask]['dealer_hedge_pressure_score']
|
||||
if len(recent_pressure) >= 2:
|
||||
current_pressure = current_row.get('dealer_hedge_pressure_score', 0) or 0
|
||||
recent_avg = recent_pressure.iloc[:-1].mean()
|
||||
|
||||
pressure_decrease = recent_avg - current_pressure
|
||||
if pressure_decrease < self.dealer_pressure_decrease_threshold:
|
||||
return FlowState.INFORMATIONAL.value
|
||||
|
||||
# Check 3: Price fails near VWAP / opening range / key level
|
||||
price_vs_vwap_pct = current_row.get('price_vs_vwap_pct')
|
||||
pct_vs_rth_open = current_row.get('pct_vs_rth_open')
|
||||
|
||||
price_failure = False
|
||||
|
||||
# VWAP failure
|
||||
if price_vs_vwap_pct is not None and not pd.isna(price_vs_vwap_pct):
|
||||
if abs(price_vs_vwap_pct) <= self.vwap_failure_threshold_pct:
|
||||
price_failure = True
|
||||
|
||||
# Opening range failure
|
||||
if pct_vs_rth_open is not None and not pd.isna(pct_vs_rth_open):
|
||||
if abs(pct_vs_rth_open) <= 0.3: # Within 0.3% of open
|
||||
price_failure = True
|
||||
|
||||
if not price_failure:
|
||||
return FlowState.INFORMATIONAL.value
|
||||
|
||||
# All checks passed: actionable
|
||||
return FlowState.ACTIONABLE.value
|
||||
|
||||
def enrich_with_flow_state(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Add flow state validation to DataFrame.
|
||||
Adds: flow_state
|
||||
"""
|
||||
if df.empty:
|
||||
return df
|
||||
|
||||
df = df.copy()
|
||||
df['flow_state'] = FlowState.ACTIONABLE.value # Default to actionable
|
||||
|
||||
# Only validate flow decay/reversal cases
|
||||
# For now, mark all as actionable (can be enhanced based on flow_acceleration, follow_on_ratio)
|
||||
# This is a placeholder - in production, you'd identify decay/reversal patterns first
|
||||
|
||||
logger.info(f"Flow state validation complete. Actionable: {(df['flow_state'] == FlowState.ACTIONABLE.value).sum()}")
|
||||
|
||||
return df
|
||||
|
||||
|
|
@ -1,189 +0,0 @@
|
|||
"""
|
||||
Institutional Confidence Metrics Service
|
||||
Calculates confidence scores for institutional flow signals
|
||||
"""
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from typing import Optional
|
||||
from utils.logger import logger
|
||||
|
||||
|
||||
class InstitutionalConfidence:
|
||||
"""
|
||||
Calculates institutional confidence metrics:
|
||||
- confidence_score (0-100)
|
||||
- institutional_likelihood (0-1)
|
||||
- dealer_pain_level (0-100)
|
||||
- expected_move_vs_implied (ratio)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Configuration
|
||||
self.min_premium_for_institutional = 200000 # Minimum premium for institutional classification
|
||||
|
||||
def calculate_confidence_score(self, row: pd.Series) -> float:
|
||||
"""
|
||||
Calculate overall confidence score (0-100) combining multiple factors.
|
||||
Higher = more confidence in signal quality.
|
||||
"""
|
||||
score = 0.0
|
||||
|
||||
# Relative premium component (0-25 points)
|
||||
relative_premium = row.get('relative_premium_score', 0) or 0
|
||||
score += (relative_premium / 100.0) * 25.0
|
||||
|
||||
# Signal strength component (0-25 points)
|
||||
signal_strength = row.get('signal_strength', 0) or 0
|
||||
score += (signal_strength / 100.0) * 25.0
|
||||
|
||||
# Dealer pressure component (0-20 points)
|
||||
dealer_pressure = row.get('dealer_hedge_pressure_score', 0) or 0
|
||||
score += (dealer_pressure / 100.0) * 20.0
|
||||
|
||||
# Flow continuation component (0-15 points)
|
||||
follow_on_ratio = row.get('follow_on_ratio')
|
||||
if follow_on_ratio is not None and not pd.isna(follow_on_ratio):
|
||||
score += follow_on_ratio * 15.0
|
||||
|
||||
# Strike laddering component (0-15 points)
|
||||
strike_laddering = row.get('strike_laddering_detected', False)
|
||||
if strike_laddering:
|
||||
score += 15.0
|
||||
|
||||
return min(100.0, max(0.0, score))
|
||||
|
||||
def calculate_institutional_likelihood(self, row: pd.Series) -> float:
|
||||
"""
|
||||
Calculate likelihood that flow is institutional (0-1).
|
||||
Based on premium size, trade characteristics, and patterns.
|
||||
"""
|
||||
premium_num = row.get('premium_num', 0) or 0
|
||||
relative_premium = row.get('relative_premium_score', 0) or 0
|
||||
size_concentration = row.get('size_concentration_score', 0) or 0
|
||||
|
||||
likelihood = 0.0
|
||||
|
||||
# Premium size component (0-40%)
|
||||
if premium_num >= 1000000: # $1M+
|
||||
likelihood += 0.40
|
||||
elif premium_num >= 500000: # $500K+
|
||||
likelihood += 0.30
|
||||
elif premium_num >= self.min_premium_for_institutional:
|
||||
likelihood += 0.20
|
||||
|
||||
# Relative premium component (0-30%)
|
||||
if relative_premium >= 80:
|
||||
likelihood += 0.30
|
||||
elif relative_premium >= 60:
|
||||
likelihood += 0.20
|
||||
elif relative_premium >= 40:
|
||||
likelihood += 0.10
|
||||
|
||||
# Size concentration component (0-30%)
|
||||
# Institutional trades are often concentrated
|
||||
if size_concentration >= 70:
|
||||
likelihood += 0.30
|
||||
elif size_concentration >= 50:
|
||||
likelihood += 0.20
|
||||
elif size_concentration >= 30:
|
||||
likelihood += 0.10
|
||||
|
||||
return min(1.0, max(0.0, likelihood))
|
||||
|
||||
def calculate_dealer_pain_level(self, row: pd.Series) -> float:
|
||||
"""
|
||||
Calculate dealer pain level (0-100).
|
||||
Higher = dealers in pain (large gamma exposure, forced to hedge).
|
||||
"""
|
||||
dealer_pressure = row.get('dealer_hedge_pressure_score', 0) or 0
|
||||
net_gamma = abs(row.get('net_gamma_exposure_per_symbol', 0) or 0)
|
||||
gamma_flip_prox = row.get('gamma_flip_proximity')
|
||||
|
||||
pain = 0.0
|
||||
|
||||
# Dealer pressure component (0-50 points)
|
||||
pain += (dealer_pressure / 100.0) * 50.0
|
||||
|
||||
# Gamma exposure component (0-30 points)
|
||||
# Large absolute gamma = more pain
|
||||
if net_gamma > 0:
|
||||
normalized_gamma = min(1.0, net_gamma / 1000000000) # 1B threshold
|
||||
pain += normalized_gamma * 30.0
|
||||
|
||||
# Gamma flip proximity component (0-20 points)
|
||||
# Near flip = high pain (dealers forced to adjust)
|
||||
if gamma_flip_prox is not None and not pd.isna(gamma_flip_prox):
|
||||
# Absolute value of proximity (closer to 0 = more pain)
|
||||
pain += (1.0 - abs(gamma_flip_prox)) * 20.0
|
||||
|
||||
return min(100.0, max(0.0, pain))
|
||||
|
||||
def calculate_expected_move_vs_implied(self, row: pd.Series) -> Optional[float]:
|
||||
"""
|
||||
Calculate expected move vs implied move ratio.
|
||||
Estimates expected move from flow characteristics vs implied volatility.
|
||||
|
||||
Returns: ratio (expected_move / implied_move)
|
||||
- >1.0 = flow suggests larger move than implied
|
||||
- <1.0 = flow suggests smaller move than implied
|
||||
- None if cannot calculate
|
||||
"""
|
||||
# Simplified calculation: use premium and delta exposure as proxies
|
||||
premium_num = row.get('premium_num', 0) or 0
|
||||
spot_num = row.get('spot_num', 0) or 0
|
||||
delta_exposure = abs(row.get('delta_exposure', 0) or 0)
|
||||
|
||||
if pd.isna(spot_num) or spot_num == 0:
|
||||
return None
|
||||
|
||||
# Estimate expected move from premium paid
|
||||
# High premium relative to spot = expectation of larger move
|
||||
if premium_num > 0:
|
||||
prem_to_spot_ratio = premium_num / spot_num
|
||||
|
||||
# Estimate implied move (simplified: assume 1% IV = 1% move expectation)
|
||||
# This is a placeholder - in production, use actual IV from options chain
|
||||
implied_move_pct = 2.0 # Default 2% implied move
|
||||
|
||||
# Estimate expected move from premium
|
||||
# Premium of 2% of spot = expectation of ~2% move (rough approximation)
|
||||
expected_move_pct = prem_to_spot_ratio * 100.0
|
||||
|
||||
# Calculate ratio
|
||||
if implied_move_pct > 0:
|
||||
ratio = expected_move_pct / implied_move_pct
|
||||
return float(ratio)
|
||||
|
||||
return None
|
||||
|
||||
def enrich_with_confidence_metrics(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Add institutional confidence metrics to DataFrame.
|
||||
Adds: confidence_score, institutional_likelihood, dealer_pain_level, expected_move_vs_implied
|
||||
"""
|
||||
if df.empty:
|
||||
return df
|
||||
|
||||
df = df.copy()
|
||||
|
||||
# Initialize columns
|
||||
df['confidence_score'] = 0.0
|
||||
df['institutional_likelihood'] = 0.0
|
||||
df['dealer_pain_level'] = 0.0
|
||||
df['expected_move_vs_implied'] = None
|
||||
|
||||
# Calculate metrics
|
||||
df['confidence_score'] = df.apply(self.calculate_confidence_score, axis=1)
|
||||
df['institutional_likelihood'] = df.apply(self.calculate_institutional_likelihood, axis=1)
|
||||
df['dealer_pain_level'] = df.apply(self.calculate_dealer_pain_level, axis=1)
|
||||
|
||||
# Expected move vs implied (some rows may not have this)
|
||||
for idx in df.index:
|
||||
expected_move = self.calculate_expected_move_vs_implied(df.iloc[idx])
|
||||
if expected_move is not None:
|
||||
df.at[idx, 'expected_move_vs_implied'] = float(expected_move)
|
||||
|
||||
logger.info(f"Confidence metrics complete. Mean confidence: {df['confidence_score'].mean():.2f}")
|
||||
|
||||
return df
|
||||
|
||||
|
|
@ -1,232 +0,0 @@
|
|||
"""
|
||||
Intent Classification Service
|
||||
Replaces naive direction (BULL/BEAR) with nuanced volatility and hedging intent classification
|
||||
"""
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from typing import Optional
|
||||
from enum import Enum
|
||||
from utils.logger import logger
|
||||
from utils.error_handler import safe_divide
|
||||
|
||||
|
||||
class VolatilityIntent(Enum):
|
||||
"""Volatility and hedging intent classification"""
|
||||
LONG_VOL = "LONG_VOL" # Buying volatility (call/put buying, expecting large moves)
|
||||
SHORT_VOL = "SHORT_VOL" # Selling volatility (premium collection, expecting low vol)
|
||||
DIRECTIONAL = "DIRECTIONAL" # Directional positioning (directional bias)
|
||||
HEDGE_UNWIND = "HEDGE_UNWIND" # Hedging or unwinding existing positions
|
||||
|
||||
|
||||
class IntentClassifier:
|
||||
"""
|
||||
Classifies options flow intent beyond simple BULL/BEAR direction.
|
||||
Identifies volatility trading, hedging, and directional positioning.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Configuration thresholds
|
||||
self.otm_threshold_pct = 5.0 # Strikes >5% OTM considered OTM
|
||||
self.long_vol_premium_threshold = 100000 # Minimum premium for long vol signal
|
||||
self.short_vol_premium_threshold = 200000 # Minimum premium for short vol signal
|
||||
|
||||
def estimate_delta(self, row: pd.Series) -> float:
|
||||
"""
|
||||
Estimate option delta from moneyness.
|
||||
Returns delta estimate (0-1 for calls, -1-0 for puts, use absolute value).
|
||||
"""
|
||||
spot_num = row.get('spot_num', 0) or 0
|
||||
strike_num = row.get('strike_num', 0) or 0
|
||||
cp_norm = row.get('cp_norm', '')
|
||||
|
||||
if pd.isna(spot_num) or spot_num == 0 or pd.isna(strike_num) or pd.isna(cp_norm):
|
||||
return 0.5 # Default to ATM
|
||||
|
||||
moneyness_ratio = strike_num / spot_num if spot_num > 0 else 1.0
|
||||
|
||||
if cp_norm == 'CALL':
|
||||
if strike_num <= spot_num:
|
||||
# ITM call: delta 0.5 to 1.0
|
||||
delta = 0.5 + (1.0 - moneyness_ratio) * 0.5
|
||||
else:
|
||||
# OTM call: delta 0.5 to 0.0
|
||||
delta = max(0.0, 0.5 * (1.5 - moneyness_ratio) / 0.5)
|
||||
else: # PUT
|
||||
if strike_num >= spot_num:
|
||||
# ITM put: delta -0.5 to -1.0 (return absolute)
|
||||
delta = abs(0.5 + (moneyness_ratio - 1.0) * 0.5)
|
||||
else:
|
||||
# OTM put: delta -0.5 to 0.0 (return absolute)
|
||||
delta = max(0.0, 0.5 * (1.0 - moneyness_ratio) / 0.5)
|
||||
|
||||
return float(delta)
|
||||
|
||||
def estimate_gamma(self, row: pd.Series, delta: float) -> float:
|
||||
"""
|
||||
Estimate option gamma (sensitivity of delta to price changes).
|
||||
Higher near ATM, lower ITM/OTM.
|
||||
Returns gamma estimate (positive value).
|
||||
"""
|
||||
spot_num = row.get('spot_num', 0) or 0
|
||||
strike_num = row.get('strike_num', 0) or 0
|
||||
|
||||
if pd.isna(spot_num) or spot_num == 0 or pd.isna(strike_num):
|
||||
return 0.0
|
||||
|
||||
# Gamma is highest at-the-money
|
||||
moneyness_ratio = strike_num / spot_num if spot_num > 0 else 1.0
|
||||
|
||||
# Simple approximation: gamma peaks at 1.0 (ATM) and decays away
|
||||
# Use normal distribution approximation
|
||||
distance_from_atm = abs(moneyness_ratio - 1.0)
|
||||
|
||||
# Gamma ≈ exp(-distance^2 / (2*sigma^2)) where sigma ≈ 0.1 (10% moneyness)
|
||||
gamma = np.exp(-(distance_from_atm ** 2) / (2 * 0.01))
|
||||
|
||||
return float(gamma)
|
||||
|
||||
def calculate_delta_exposure(self, row: pd.Series) -> float:
|
||||
"""
|
||||
Calculate delta exposure: contracts * delta * 100 * spot_price.
|
||||
Positive = long delta (bullish), Negative = short delta (bearish).
|
||||
"""
|
||||
premium_num = row.get('premium_num', 0) or 0
|
||||
spot_num = row.get('spot_num', 0) or 0
|
||||
vol_num = row.get('vol_num', 0) or 0
|
||||
cp_norm = row.get('cp_norm', '')
|
||||
side_norm = row.get('side_norm', '')
|
||||
|
||||
if pd.isna(spot_num) or spot_num == 0:
|
||||
return 0.0
|
||||
|
||||
# Estimate delta
|
||||
delta = self.estimate_delta(row)
|
||||
|
||||
# Determine sign based on call/put and buy/sell
|
||||
if cp_norm == 'CALL':
|
||||
if side_norm == 'BUY':
|
||||
delta_sign = 1.0 # Long calls = positive delta
|
||||
else: # SELL
|
||||
delta_sign = -1.0 # Short calls = negative delta
|
||||
else: # PUT
|
||||
if side_norm == 'BUY':
|
||||
delta_sign = -1.0 # Long puts = negative delta
|
||||
else: # SELL
|
||||
delta_sign = 1.0 # Short puts = positive delta
|
||||
|
||||
# Delta exposure = contracts * delta * 100 * spot
|
||||
# Use volume as proxy for contracts
|
||||
contracts = vol_num if not pd.isna(vol_num) else 0
|
||||
delta_exposure = contracts * delta * delta_sign * 100 * spot_num
|
||||
|
||||
return float(delta_exposure)
|
||||
|
||||
def calculate_gamma_exposure(self, row: pd.Series) -> float:
|
||||
"""
|
||||
Calculate gamma exposure: contracts * gamma * 100 * spot_price^2.
|
||||
Positive = long gamma (volatility long), Negative = short gamma (volatility short).
|
||||
"""
|
||||
premium_num = row.get('premium_num', 0) or 0
|
||||
spot_num = row.get('spot_num', 0) or 0
|
||||
vol_num = row.get('vol_num', 0) or 0
|
||||
cp_norm = row.get('cp_norm', '')
|
||||
side_norm = row.get('side_norm', '')
|
||||
|
||||
if pd.isna(spot_num) or spot_num == 0:
|
||||
return 0.0
|
||||
|
||||
# Estimate delta and gamma
|
||||
delta = self.estimate_delta(row)
|
||||
gamma = self.estimate_gamma(row, delta)
|
||||
|
||||
# Determine sign: buying options = long gamma, selling = short gamma
|
||||
if side_norm == 'BUY':
|
||||
gamma_sign = 1.0 # Long gamma
|
||||
else: # SELL
|
||||
gamma_sign = -1.0 # Short gamma
|
||||
|
||||
# Gamma exposure = contracts * gamma * 100 * spot^2
|
||||
contracts = vol_num if not pd.isna(vol_num) else 0
|
||||
gamma_exposure = contracts * gamma * gamma_sign * 100 * (spot_num ** 2)
|
||||
|
||||
return float(gamma_exposure)
|
||||
|
||||
def classify_volatility_intent(self, row: pd.Series) -> str:
|
||||
"""
|
||||
Classify volatility intent based on trade characteristics.
|
||||
Returns VolatilityIntent enum value as string.
|
||||
"""
|
||||
premium_num = row.get('premium_num', 0) or 0
|
||||
spot_num = row.get('spot_num', 0) or 0
|
||||
strike_num = row.get('strike_num', 0) or 0
|
||||
cp_norm = row.get('cp_norm', '')
|
||||
side_norm = row.get('side_norm', '')
|
||||
vol_num = row.get('vol_num', 0) or 0
|
||||
oi_num = row.get('oi_num', 0) or 0
|
||||
|
||||
# Calculate moneyness
|
||||
if pd.isna(spot_num) or spot_num == 0 or pd.isna(strike_num):
|
||||
mny_pct = 0.0
|
||||
else:
|
||||
if cp_norm == 'CALL':
|
||||
mny_pct = ((strike_num - spot_num) / spot_num * 100.0) if spot_num > 0 else 0
|
||||
else: # PUT
|
||||
mny_pct = ((spot_num - strike_num) / spot_num * 100.0) if spot_num > 0 else 0
|
||||
|
||||
is_otm = abs(mny_pct) > self.otm_threshold_pct
|
||||
|
||||
# Long volatility: buying OTM options (calls or puts)
|
||||
# High premium, OTM strikes, buying side
|
||||
if (side_norm == 'BUY' and
|
||||
is_otm and
|
||||
premium_num >= self.long_vol_premium_threshold):
|
||||
return VolatilityIntent.LONG_VOL.value
|
||||
|
||||
# Short volatility: selling options, collecting premium
|
||||
# High premium, selling side, vol > OI (opening new short positions)
|
||||
if (side_norm == 'SELL' and
|
||||
premium_num >= self.short_vol_premium_threshold and
|
||||
vol_num > oi_num):
|
||||
return VolatilityIntent.SHORT_VOL.value
|
||||
|
||||
# Directional: ITM options, buying side, strong directional flow
|
||||
if (side_norm == 'BUY' and
|
||||
not is_otm and
|
||||
premium_num >= 50000):
|
||||
return VolatilityIntent.DIRECTIONAL.value
|
||||
|
||||
# Hedge/Unwind: Selling existing positions (vol < OI)
|
||||
# Or buying protective puts/calls
|
||||
if (side_norm == 'SELL' and vol_num < oi_num) or \
|
||||
(side_norm == 'BUY' and is_otm and premium_num < self.long_vol_premium_threshold):
|
||||
return VolatilityIntent.HEDGE_UNWIND.value
|
||||
|
||||
# Default: directional (fallback)
|
||||
return VolatilityIntent.DIRECTIONAL.value
|
||||
|
||||
def enrich_with_intent_classification(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Add intent classification metrics to DataFrame.
|
||||
Adds: delta_exposure, gamma_exposure, volatility_intent
|
||||
"""
|
||||
if df.empty:
|
||||
return df
|
||||
|
||||
df = df.copy()
|
||||
|
||||
# Initialize columns
|
||||
df['delta_exposure'] = 0.0
|
||||
df['gamma_exposure'] = 0.0
|
||||
df['volatility_intent'] = VolatilityIntent.DIRECTIONAL.value
|
||||
|
||||
# Calculate metrics
|
||||
df['delta_exposure'] = df.apply(self.calculate_delta_exposure, axis=1)
|
||||
df['gamma_exposure'] = df.apply(self.calculate_gamma_exposure, axis=1)
|
||||
df['volatility_intent'] = df.apply(self.classify_volatility_intent, axis=1)
|
||||
|
||||
# Log distribution
|
||||
intent_counts = df['volatility_intent'].value_counts()
|
||||
logger.info(f"Intent classification complete. Distribution: {intent_counts.to_dict()}")
|
||||
|
||||
return df
|
||||
|
||||
|
|
@ -1,104 +0,0 @@
|
|||
"""
|
||||
Market Regime Detection Service
|
||||
Identifies market regime to gate trade signal generation
|
||||
"""
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from typing import Optional
|
||||
from enum import Enum
|
||||
from utils.logger import logger
|
||||
|
||||
|
||||
class MarketRegime(Enum):
|
||||
"""Market regime classification"""
|
||||
TREND = "TREND" # Trending market (continuation bias)
|
||||
RANGE = "RANGE" # Range-bound market (fade or vol-sell bias)
|
||||
HIGH_VOL_EVENT = "HIGH_VOL_EVENT" # High volatility event (volatility expansion bias)
|
||||
|
||||
|
||||
class MarketRegimeDetector:
|
||||
"""
|
||||
Detects market regime to inform trade signal bias:
|
||||
- Trend: continuation trades preferred
|
||||
- Range: mean reversion / vol selling preferred
|
||||
- High Vol Event: volatility expansion trades preferred
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Configuration
|
||||
self.trend_threshold_pct = 1.5 # >1.5% move = trending
|
||||
self.range_threshold_pct = 0.5 # <0.5% move = ranging
|
||||
self.high_vol_threshold_pct = 3.0 # >3% move = high vol event
|
||||
self.lookback_minutes = 60 # Lookback window for regime detection
|
||||
|
||||
def detect_regime(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
row_idx: int
|
||||
) -> str:
|
||||
"""
|
||||
Detect market regime for a given flow event.
|
||||
Returns MarketRegime enum value as string.
|
||||
"""
|
||||
if row_idx >= len(df):
|
||||
return MarketRegime.RANGE.value
|
||||
|
||||
current_row = df.iloc[row_idx]
|
||||
symbol = current_row.get('symbol_norm')
|
||||
flow_ts_utc = current_row.get('flow_ts_utc')
|
||||
|
||||
if pd.isna(symbol) or pd.isna(flow_ts_utc):
|
||||
return MarketRegime.RANGE.value
|
||||
|
||||
# Get price movement over lookback window
|
||||
pct_vs_rth_open = current_row.get('pct_vs_rth_open')
|
||||
pct_vs_prior_close = current_row.get('pct_vs_prior_close')
|
||||
pct_15m_momo = current_row.get('pct_15m_momo')
|
||||
|
||||
# Use most appropriate price metric
|
||||
price_move_pct = None
|
||||
if pct_vs_rth_open is not None and not pd.isna(pct_vs_rth_open):
|
||||
price_move_pct = abs(pct_vs_rth_open)
|
||||
elif pct_vs_prior_close is not None and not pd.isna(pct_vs_prior_close):
|
||||
price_move_pct = abs(pct_vs_prior_close)
|
||||
elif pct_15m_momo is not None and not pd.isna(pct_15m_momo):
|
||||
price_move_pct = abs(pct_15m_momo)
|
||||
|
||||
if price_move_pct is None:
|
||||
return MarketRegime.RANGE.value # Default to range
|
||||
|
||||
# Classify regime
|
||||
if price_move_pct >= self.high_vol_threshold_pct:
|
||||
return MarketRegime.HIGH_VOL_EVENT.value
|
||||
elif price_move_pct >= self.trend_threshold_pct:
|
||||
return MarketRegime.TREND.value
|
||||
elif price_move_pct <= self.range_threshold_pct:
|
||||
return MarketRegime.RANGE.value
|
||||
else:
|
||||
# Between range and trend threshold - classify based on momentum
|
||||
if pct_15m_momo is not None and not pd.isna(pct_15m_momo):
|
||||
if abs(pct_15m_momo) > 0.75:
|
||||
return MarketRegime.TREND.value
|
||||
return MarketRegime.RANGE.value
|
||||
|
||||
def enrich_with_market_regime(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Add market regime classification to DataFrame.
|
||||
Adds: market_regime
|
||||
"""
|
||||
if df.empty:
|
||||
return df
|
||||
|
||||
df = df.copy()
|
||||
df['market_regime'] = MarketRegime.RANGE.value
|
||||
|
||||
# Detect regime for each row
|
||||
for idx in df.index:
|
||||
regime = self.detect_regime(df, idx)
|
||||
df.at[idx, 'market_regime'] = regime
|
||||
|
||||
regime_counts = df['market_regime'].value_counts()
|
||||
logger.info(f"Market regime detection complete. Distribution: {regime_counts.to_dict()}")
|
||||
|
||||
return df
|
||||
|
||||
|
|
@ -1,213 +0,0 @@
|
|||
"""
|
||||
Tier-0 Noise Rejection Service
|
||||
Filters out low-quality signals before enrichment to reduce processing overhead
|
||||
"""
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from typing import Optional
|
||||
from datetime import timedelta
|
||||
from utils.logger import logger
|
||||
from utils.error_handler import safe_divide
|
||||
|
||||
|
||||
class NoiseRejector:
|
||||
"""
|
||||
Rejects early-stage noise before enrichment to optimize processing.
|
||||
|
||||
Rejects if:
|
||||
- Single isolated trade (no repeat activity)
|
||||
- Far OTM weekly lottos
|
||||
- Delta-adjusted premium below threshold
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Configuration
|
||||
self.repeat_activity_window_minutes = 30 # Minutes to look for repeat activity
|
||||
self.min_delta_adjusted_premium = 50000 # Minimum delta-adjusted premium
|
||||
self.max_otm_percentage = 15.0 # Reject strikes >15% OTM
|
||||
self.min_expiry_days = 1 # Minimum days to expiry (reject same-day/weekly lottos)
|
||||
|
||||
def is_isolated_trade(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
row_idx: int
|
||||
) -> bool:
|
||||
"""
|
||||
Check if trade is isolated (no repeat activity within window).
|
||||
Returns True if isolated (should reject).
|
||||
"""
|
||||
if row_idx >= len(df):
|
||||
return True
|
||||
|
||||
current_row = df.iloc[row_idx]
|
||||
symbol = current_row.get('symbol_norm')
|
||||
direction = current_row.get('direction')
|
||||
flow_ts_utc = current_row.get('flow_ts_utc')
|
||||
|
||||
if pd.isna(symbol) or pd.isna(flow_ts_utc):
|
||||
return True
|
||||
|
||||
# Look for trades in same direction within window
|
||||
window_start = flow_ts_utc - timedelta(minutes=self.repeat_activity_window_minutes)
|
||||
|
||||
same_symbol = df['symbol_norm'] == symbol
|
||||
same_direction = df['direction'] == direction
|
||||
in_window = (df['flow_ts_utc'] >= window_start) & (df['flow_ts_utc'] < flow_ts_utc)
|
||||
|
||||
# Exclude current row
|
||||
other_trades = df[same_symbol & same_direction & in_window]
|
||||
|
||||
# If no other trades in window, it's isolated
|
||||
return len(other_trades) == 0
|
||||
|
||||
def is_far_otm_lotto(
|
||||
self,
|
||||
row: pd.Series
|
||||
) -> bool:
|
||||
"""
|
||||
Check if trade is a far OTM weekly lotto (low probability, low signal value).
|
||||
Returns True if should reject.
|
||||
"""
|
||||
spot_num = row.get('spot_num', 0) or 0
|
||||
strike_num = row.get('strike_num', 0) or 0
|
||||
cp_norm = row.get('cp_norm', '')
|
||||
exp_date = row.get('exp_date')
|
||||
flow_date = row.get('flow_date_cst')
|
||||
|
||||
if pd.isna(spot_num) or spot_num == 0 or pd.isna(strike_num) or pd.isna(cp_norm):
|
||||
return False
|
||||
|
||||
# Calculate moneyness percentage
|
||||
if cp_norm == 'CALL':
|
||||
otm_pct = ((strike_num - spot_num) / spot_num * 100.0) if spot_num > 0 else 0
|
||||
else: # PUT
|
||||
otm_pct = ((spot_num - strike_num) / spot_num * 100.0) if spot_num > 0 else 0
|
||||
|
||||
# Reject if >15% OTM
|
||||
if otm_pct > self.max_otm_percentage:
|
||||
return True
|
||||
|
||||
# Check if weekly lotto (expiry within 7 days)
|
||||
if pd.notna(exp_date) and pd.notna(flow_date):
|
||||
try:
|
||||
if isinstance(exp_date, str):
|
||||
from datetime import datetime
|
||||
exp_date = datetime.strptime(exp_date, '%Y-%m-%d').date()
|
||||
if isinstance(flow_date, str):
|
||||
from datetime import datetime
|
||||
flow_date = datetime.strptime(flow_date, '%Y-%m-%d').date()
|
||||
|
||||
days_to_expiry = (exp_date - flow_date).days
|
||||
|
||||
# Reject weekly lottos that are also far OTM
|
||||
if days_to_expiry <= 7 and otm_pct > 10.0:
|
||||
return True
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
def calculate_delta_adjusted_premium(
|
||||
self,
|
||||
row: pd.Series
|
||||
) -> float:
|
||||
"""
|
||||
Calculate delta-adjusted premium (premium * |delta|).
|
||||
Approximates intrinsic value component of premium.
|
||||
|
||||
For simplicity, we estimate delta from moneyness:
|
||||
- ATM: delta ≈ 0.5
|
||||
- ITM: delta increases toward 1.0
|
||||
- OTM: delta decreases toward 0.0
|
||||
"""
|
||||
premium_num = row.get('premium_num', 0) or 0
|
||||
spot_num = row.get('spot_num', 0) or 0
|
||||
strike_num = row.get('strike_num', 0) or 0
|
||||
cp_norm = row.get('cp_norm', '')
|
||||
|
||||
if pd.isna(premium_num) or premium_num == 0 or pd.isna(spot_num) or spot_num == 0:
|
||||
return 0.0
|
||||
|
||||
# Estimate delta from moneyness
|
||||
if cp_norm == 'CALL':
|
||||
if strike_num <= spot_num:
|
||||
# ITM call: delta from 0.5 to 1.0
|
||||
moneyness_ratio = strike_num / spot_num if spot_num > 0 else 1.0
|
||||
# Strike at spot = 0.5, strike at 0 = 1.0
|
||||
estimated_delta = 0.5 + (1.0 - moneyness_ratio) * 0.5
|
||||
else:
|
||||
# OTM call: delta from 0.5 to 0.0
|
||||
moneyness_ratio = strike_num / spot_num if spot_num > 0 else 1.0
|
||||
# Strike at spot = 0.5, strike at 1.15x spot = 0.0
|
||||
estimated_delta = max(0.0, 0.5 * (1.5 - moneyness_ratio) / 0.5)
|
||||
else: # PUT
|
||||
if strike_num >= spot_num:
|
||||
# ITM put: delta from -0.5 to -1.0 (use absolute value)
|
||||
moneyness_ratio = strike_num / spot_num if spot_num > 0 else 1.0
|
||||
estimated_delta = 0.5 + (moneyness_ratio - 1.0) * 0.5
|
||||
else:
|
||||
# OTM put: delta from 0.5 to 0.0 (use absolute value)
|
||||
moneyness_ratio = strike_num / spot_num if spot_num > 0 else 1.0
|
||||
estimated_delta = max(0.0, 0.5 * (1.0 - moneyness_ratio) / 0.5)
|
||||
|
||||
# Delta-adjusted premium
|
||||
delta_adj_premium = premium_num * abs(estimated_delta)
|
||||
|
||||
return float(delta_adj_premium)
|
||||
|
||||
def should_reject(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
row_idx: int
|
||||
) -> bool:
|
||||
"""
|
||||
Determine if row should be rejected as noise.
|
||||
Returns True if should reject (mark early_noise_reject = True).
|
||||
"""
|
||||
if row_idx >= len(df):
|
||||
return True
|
||||
|
||||
row = df.iloc[row_idx]
|
||||
|
||||
# Reject isolated trades
|
||||
if self.is_isolated_trade(df, row_idx):
|
||||
return True
|
||||
|
||||
# Reject far OTM lottos
|
||||
if self.is_far_otm_lotto(row):
|
||||
return True
|
||||
|
||||
# Reject if delta-adjusted premium below threshold
|
||||
delta_adj_premium = self.calculate_delta_adjusted_premium(row)
|
||||
if delta_adj_premium < self.min_delta_adjusted_premium:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def mark_noise_rejections(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Mark rows for early noise rejection.
|
||||
Adds: early_noise_reject (boolean)
|
||||
"""
|
||||
if df.empty:
|
||||
return df
|
||||
|
||||
df = df.copy()
|
||||
|
||||
# Initialize column
|
||||
df['early_noise_reject'] = False
|
||||
|
||||
# Sort by timestamp for proper isolation detection
|
||||
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
|
||||
|
||||
# Mark rejections
|
||||
for idx in df_sorted.index:
|
||||
if self.should_reject(df_sorted, idx):
|
||||
original_idx = df_sorted.index[idx]
|
||||
df.at[original_idx, 'early_noise_reject'] = True
|
||||
|
||||
rejection_count = df['early_noise_reject'].sum()
|
||||
logger.info(f"Noise rejection: {rejection_count}/{len(df)} rows marked for rejection ({rejection_count/len(df)*100:.1f}%)")
|
||||
|
||||
return df
|
||||
|
||||
|
|
@ -146,14 +146,5 @@ class OutputFormatter:
|
|||
# Ensure Phase 1 columns are preserved (don't drop them)
|
||||
# Phase 1 columns should remain as-is (signal_tier, checklist_score, etc.)
|
||||
|
||||
# Ensure all new institutional analytics columns are preserved:
|
||||
# - premium_zscore, premium_percentile_intraday, relative_premium_score
|
||||
# - aggression_score, size_concentration_score, repeat_trade_velocity, strike_clustering_score, signal_strength
|
||||
# - early_noise_reject, flow_acceleration, time_between_hits, follow_on_ratio, strike_laddering_detected
|
||||
# - delta_exposure, gamma_exposure, volatility_intent
|
||||
# - net_gamma_exposure_per_symbol, gamma_flip_proximity, dealer_hedge_pressure_score
|
||||
# - market_regime, flow_state
|
||||
# - confidence_score, institutional_likelihood, dealer_pain_level, expected_move_vs_implied
|
||||
|
||||
return df
|
||||
|
||||
|
|
|
|||
|
|
@ -1,258 +0,0 @@
|
|||
"""
|
||||
Relative Premium Scoring Service
|
||||
Replaces static premium filter with context-aware relative premium scoring
|
||||
"""
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from typing import Dict, Optional
|
||||
from datetime import datetime, timedelta
|
||||
import asyncpg
|
||||
from utils.logger import logger
|
||||
from utils.error_handler import safe_divide
|
||||
|
||||
|
||||
class RelativePremiumScorer:
|
||||
"""
|
||||
Computes relative premium metrics to replace static premium filtering.
|
||||
Premium of $80K might be significant for AAPL but noise for TSLA.
|
||||
"""
|
||||
|
||||
def __init__(self, pool: asyncpg.Pool = None):
|
||||
self.pool = pool
|
||||
# Configuration
|
||||
self.rolling_window_days = 20 # Rolling window for z-score calculation
|
||||
self.min_relative_premium_threshold = 0.60 # 60th percentile minimum
|
||||
|
||||
async def fetch_historical_premium_stats(
|
||||
self,
|
||||
symbol: str,
|
||||
reference_date: datetime.date
|
||||
) -> Dict[str, float]:
|
||||
"""
|
||||
Fetch historical premium statistics for a symbol.
|
||||
Returns: {mean, std, min, max, median} for rolling window
|
||||
"""
|
||||
if not self.pool:
|
||||
logger.warning("No database pool available for historical premium stats")
|
||||
return {}
|
||||
|
||||
try:
|
||||
# Calculate window start date
|
||||
window_start = reference_date - timedelta(days=self.rolling_window_days + 5) # Extra buffer
|
||||
|
||||
async with self.pool.acquire() as conn:
|
||||
query = """
|
||||
SELECT
|
||||
AVG(premium_num) as mean_premium,
|
||||
STDDEV(premium_num) as std_premium,
|
||||
MIN(premium_num) as min_premium,
|
||||
MAX(premium_num) as max_premium,
|
||||
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY premium_num) as median_premium
|
||||
FROM (
|
||||
SELECT
|
||||
CAST(REGEXP_REPLACE("Premium"::text, '[^\d.]', '', 'g') AS numeric) as premium_num
|
||||
FROM "OptionsFlow_monthly"
|
||||
WHERE UPPER(TRIM("Symbol")) = $1
|
||||
AND "Premium" IS NOT NULL
|
||||
AND TRIM("Premium"::text) <> ''
|
||||
AND "CreatedDate" >= $2
|
||||
AND "CreatedDate" <= $3
|
||||
AND "StockEtf" = 'STOCK'
|
||||
) subq
|
||||
WHERE premium_num > 0
|
||||
"""
|
||||
|
||||
row = await conn.fetchrow(
|
||||
query,
|
||||
symbol.upper(),
|
||||
window_start.strftime('%Y-%m-%d'),
|
||||
reference_date.strftime('%Y-%m-%d')
|
||||
)
|
||||
|
||||
if row and row['mean_premium']:
|
||||
return {
|
||||
'mean': float(row['mean_premium']) if row['mean_premium'] else 0.0,
|
||||
'std': float(row['std_premium']) if row['std_premium'] else 0.0,
|
||||
'min': float(row['min_premium']) if row['min_premium'] else 0.0,
|
||||
'max': float(row['max_premium']) if row['max_premium'] else 0.0,
|
||||
'median': float(row['median_premium']) if row['median_premium'] else 0.0,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.debug(f"Error fetching historical premium stats for {symbol}: {e}")
|
||||
|
||||
return {}
|
||||
|
||||
async def calculate_intraday_percentile(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
symbol: str,
|
||||
flow_date: datetime.date,
|
||||
premium: float
|
||||
) -> Optional[float]:
|
||||
"""
|
||||
Calculate percentile rank of premium within same-day flow for the symbol.
|
||||
Returns 0-100 percentile value.
|
||||
"""
|
||||
try:
|
||||
# Filter to same symbol and date
|
||||
same_day_flow = df[
|
||||
(df['symbol_norm'] == symbol.upper()) &
|
||||
(df['flow_date_cst'] == flow_date) &
|
||||
(df['premium_num'].notna()) &
|
||||
(df['premium_num'] > 0)
|
||||
]
|
||||
|
||||
if len(same_day_flow) < 3: # Need at least 3 trades for meaningful percentile
|
||||
return None
|
||||
|
||||
# Calculate percentile rank
|
||||
all_premiums = same_day_flow['premium_num'].values
|
||||
percentile = (np.sum(all_premiums <= premium) / len(all_premiums)) * 100.0
|
||||
|
||||
return float(percentile)
|
||||
except Exception as e:
|
||||
logger.debug(f"Error calculating intraday percentile for {symbol}: {e}")
|
||||
return None
|
||||
|
||||
def calculate_zscore(
|
||||
self,
|
||||
premium: float,
|
||||
stats: Dict[str, float]
|
||||
) -> Optional[float]:
|
||||
"""
|
||||
Calculate z-score of premium relative to historical distribution.
|
||||
Returns z-score (standard deviations from mean).
|
||||
"""
|
||||
if not stats or stats.get('std', 0) == 0:
|
||||
return None
|
||||
|
||||
mean = stats.get('mean', 0)
|
||||
std = stats.get('std', 1)
|
||||
|
||||
if std == 0:
|
||||
return None
|
||||
|
||||
zscore = (premium - mean) / std
|
||||
return float(zscore)
|
||||
|
||||
def calculate_relative_premium_score(
|
||||
self,
|
||||
premium: float,
|
||||
zscore: Optional[float],
|
||||
intraday_percentile: Optional[float],
|
||||
stats: Dict[str, float]
|
||||
) -> float:
|
||||
"""
|
||||
Calculate composite relative premium score (0-100).
|
||||
Combines z-score and intraday percentile with median normalization.
|
||||
|
||||
Logic:
|
||||
- High z-score (unusual size) = higher score
|
||||
- High intraday percentile (large relative to today's flow) = higher score
|
||||
- Normalize by median to account for symbol-specific scaling
|
||||
"""
|
||||
score = 0.0
|
||||
|
||||
# Z-score component (40% weight)
|
||||
# Z-score > 1 = 1 std above mean, Z-score > 2 = 2 std above mean
|
||||
if zscore is not None:
|
||||
# Normalize z-score to 0-40 range (clamp at ±3 sigma)
|
||||
zscore_normalized = max(0, min(40, (zscore + 3) * (40 / 6)))
|
||||
score += zscore_normalized
|
||||
|
||||
# Intraday percentile component (40% weight)
|
||||
if intraday_percentile is not None:
|
||||
# Percentile is already 0-100, scale to 0-40
|
||||
score += (intraday_percentile / 100.0) * 40.0
|
||||
|
||||
# Median normalization component (20% weight)
|
||||
# Premium > median gets bonus, premium < median gets penalty
|
||||
if stats and stats.get('median', 0) > 0:
|
||||
median_ratio = premium / stats['median']
|
||||
# Ratio > 1.5 = strong, ratio > 2.0 = very strong
|
||||
if median_ratio >= 2.0:
|
||||
score += 20.0
|
||||
elif median_ratio >= 1.5:
|
||||
score += 15.0
|
||||
elif median_ratio >= 1.0:
|
||||
score += 10.0
|
||||
elif median_ratio >= 0.5:
|
||||
score += 5.0
|
||||
|
||||
return min(100.0, max(0.0, score))
|
||||
|
||||
async def enrich_with_relative_premium(
|
||||
self,
|
||||
df: pd.DataFrame
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Add relative premium metrics to DataFrame.
|
||||
Adds: premium_zscore, premium_percentile_intraday, relative_premium_score
|
||||
"""
|
||||
if df.empty:
|
||||
return df
|
||||
|
||||
df = df.copy()
|
||||
|
||||
# Initialize new columns
|
||||
df['premium_zscore'] = None
|
||||
df['premium_percentile_intraday'] = None
|
||||
df['relative_premium_score'] = 0.0
|
||||
|
||||
# Cache historical stats per symbol to avoid repeated queries
|
||||
stats_cache: Dict[str, Dict[str, float]] = {}
|
||||
|
||||
# Group by symbol and date for batch processing
|
||||
unique_symbols = df['symbol_norm'].unique()
|
||||
unique_dates = df['flow_date_cst'].unique()
|
||||
|
||||
logger.info(f"Calculating relative premium scores for {len(unique_symbols)} symbols")
|
||||
|
||||
for symbol in unique_symbols:
|
||||
# Fetch historical stats once per symbol
|
||||
if self.pool:
|
||||
# Use the most recent date for this symbol as reference
|
||||
symbol_dates = df[df['symbol_norm'] == symbol]['flow_date_cst'].unique()
|
||||
if len(symbol_dates) > 0:
|
||||
reference_date = max(symbol_dates)
|
||||
stats = await self.fetch_historical_premium_stats(symbol, reference_date)
|
||||
stats_cache[symbol] = stats
|
||||
else:
|
||||
stats_cache[symbol] = {}
|
||||
else:
|
||||
stats_cache[symbol] = {}
|
||||
|
||||
# Calculate metrics for each row
|
||||
for idx, row in df.iterrows():
|
||||
premium = row.get('premium_num')
|
||||
if pd.isna(premium) or premium <= 0:
|
||||
continue
|
||||
|
||||
symbol = row['symbol_norm']
|
||||
flow_date = row.get('flow_date_cst')
|
||||
|
||||
# Get historical stats
|
||||
stats = stats_cache.get(symbol, {})
|
||||
|
||||
# Calculate z-score
|
||||
zscore = self.calculate_zscore(premium, stats)
|
||||
if zscore is not None:
|
||||
df.at[idx, 'premium_zscore'] = float(zscore)
|
||||
|
||||
# Calculate intraday percentile (using current DataFrame subset)
|
||||
intraday_percentile = self.calculate_intraday_percentile(
|
||||
df, symbol, flow_date, premium
|
||||
)
|
||||
if intraday_percentile is not None:
|
||||
df.at[idx, 'premium_percentile_intraday'] = float(intraday_percentile)
|
||||
|
||||
# Calculate composite relative premium score
|
||||
relative_score = self.calculate_relative_premium_score(
|
||||
premium, zscore, intraday_percentile, stats
|
||||
)
|
||||
df.at[idx, 'relative_premium_score'] = float(relative_score)
|
||||
|
||||
logger.info(f"Relative premium scoring complete. Mean score: {df['relative_premium_score'].mean():.2f}")
|
||||
|
||||
return df
|
||||
|
||||
|
|
@ -1,331 +0,0 @@
|
|||
"""
|
||||
Signal Component Scorer
|
||||
Converts binary badge logic into continuous numeric signal components
|
||||
"""
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from typing import Dict, Optional
|
||||
from utils.logger import logger
|
||||
|
||||
|
||||
class SignalComponentScorer:
|
||||
"""
|
||||
Replaces binary badge logic (💎 ⭐ 🟢 🔴) with continuous numeric scores.
|
||||
Badges remain display-only, but signal_strength is computed from components.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Signal strength weights (must sum to 1.0)
|
||||
self.aggression_weight = 0.30
|
||||
self.size_concentration_weight = 0.30
|
||||
self.repeat_velocity_weight = 0.20
|
||||
self.strike_clustering_weight = 0.20
|
||||
|
||||
def calculate_aggression_score(self, row: pd.Series) -> float:
|
||||
"""
|
||||
Measures trade aggression: premium paid at ask vs bid, ITM vs OTM preference.
|
||||
|
||||
Higher score = more aggressive buying:
|
||||
- ITM premiums indicate willingness to pay intrinsic value
|
||||
- Ask-side trades (AA) indicate market orders
|
||||
- Large premium relative to spot price
|
||||
"""
|
||||
score = 0.0
|
||||
|
||||
premium_num = row.get('premium_num', 0) or 0
|
||||
spot_num = row.get('spot_num', 0) or 0
|
||||
cp_norm = row.get('cp_norm', '')
|
||||
side = str(row.get('Side', '')).upper()
|
||||
|
||||
# ITM premium component (0-40 points)
|
||||
bull_prem_itm = row.get('bull_prem_itm', 0) or 0
|
||||
bear_prem_itm = row.get('bear_prem_itm', 0) or 0
|
||||
total_prem_itm = bull_prem_itm + bear_prem_itm
|
||||
bull_total = row.get('bull_total', 0) or 0
|
||||
bear_total = row.get('bear_total', 0) or 0
|
||||
total_premium = bull_total + bear_total
|
||||
|
||||
if total_premium > 0:
|
||||
itm_ratio = total_prem_itm / total_premium
|
||||
score += itm_ratio * 40.0
|
||||
|
||||
# Ask-side aggression component (0-30 points)
|
||||
# AA = buying at ask (market orders, urgency)
|
||||
has_aa = 'AA' in side or 'AT_ASK' in side
|
||||
has_bb = 'BB' in side or 'AT_BID' in side
|
||||
|
||||
if has_aa and not has_bb:
|
||||
score += 30.0
|
||||
elif has_aa:
|
||||
score += 15.0
|
||||
|
||||
# Premium-to-spot ratio component (0-30 points)
|
||||
# Large premium relative to spot indicates aggressive positioning
|
||||
if spot_num > 0 and premium_num > 0:
|
||||
prem_to_spot_ratio = premium_num / spot_num
|
||||
# Normalize: 0.01 (1%) = 10 points, 0.05 (5%) = 30 points
|
||||
ratio_score = min(30.0, prem_to_spot_ratio * 600.0)
|
||||
score += ratio_score
|
||||
|
||||
return min(100.0, max(0.0, score))
|
||||
|
||||
def calculate_size_concentration_score(self, row: pd.Series) -> float:
|
||||
"""
|
||||
Measures size concentration: how much premium is concentrated in single strikes/expiries.
|
||||
|
||||
Higher score = more concentrated (single large trade vs many small ones):
|
||||
- High premium in single trade
|
||||
- OI buildup in specific strikes
|
||||
- Directional consistency (all calls or all puts)
|
||||
"""
|
||||
score = 0.0
|
||||
|
||||
premium_num = row.get('premium_num', 0) or 0
|
||||
bull_total = row.get('bull_total', 0) or 0
|
||||
bear_total = row.get('bear_total', 0) or 0
|
||||
total_premium = bull_total + bear_total
|
||||
|
||||
# Single-trade premium concentration (0-40 points)
|
||||
# Premium of current trade relative to running total
|
||||
if total_premium > 0:
|
||||
concentration_ratio = premium_num / total_premium
|
||||
# Single trade = 50%+ of total = very concentrated
|
||||
if concentration_ratio >= 0.5:
|
||||
score += 40.0
|
||||
elif concentration_ratio >= 0.3:
|
||||
score += 30.0
|
||||
elif concentration_ratio >= 0.2:
|
||||
score += 20.0
|
||||
elif concentration_ratio >= 0.1:
|
||||
score += 10.0
|
||||
|
||||
# OI concentration component (0-30 points)
|
||||
# High OI in OTM strikes indicates concentrated positioning
|
||||
oi_cb_otm = row.get('oi_cb_otm', 0) or 0
|
||||
oi_pb_otm = row.get('oi_pb_otm', 0) or 0
|
||||
oi_all = row.get('oi_all', 0) or 0
|
||||
|
||||
if oi_all > 0:
|
||||
otm_oi_ratio = (oi_cb_otm + oi_pb_otm) / oi_all
|
||||
score += otm_oi_ratio * 30.0
|
||||
|
||||
# Directional consistency (0-30 points)
|
||||
# Pure directional flow (all bull or all bear) = more concentrated
|
||||
if total_premium > 0:
|
||||
direction_strength = abs(bull_total - bear_total) / total_premium
|
||||
score += direction_strength * 30.0
|
||||
|
||||
return min(100.0, max(0.0, score))
|
||||
|
||||
def calculate_repeat_trade_velocity(self, df: pd.DataFrame, row_idx: int) -> float:
|
||||
"""
|
||||
Measures repeat trade velocity: frequency of trades in same direction/expiry.
|
||||
|
||||
Higher score = faster follow-on trades (urgency building):
|
||||
- Time between consecutive trades decreasing
|
||||
- Multiple trades in same direction
|
||||
- Same expiry date (rolling accumulation)
|
||||
"""
|
||||
if row_idx >= len(df):
|
||||
return 0.0
|
||||
|
||||
current_row = df.iloc[row_idx]
|
||||
symbol = current_row.get('symbol_norm')
|
||||
direction = current_row.get('direction')
|
||||
exp_date = current_row.get('exp_date')
|
||||
flow_ts_utc = current_row.get('flow_ts_utc')
|
||||
|
||||
if pd.isna(symbol) or pd.isna(direction) or pd.isna(flow_ts_utc):
|
||||
return 0.0
|
||||
|
||||
# Look back at recent trades for same symbol
|
||||
# Sort by timestamp to find preceding trades
|
||||
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
|
||||
current_idx = df_sorted.index[df_sorted.index == row_idx]
|
||||
|
||||
if len(current_idx) == 0:
|
||||
return 0.0
|
||||
|
||||
current_pos = current_idx[0]
|
||||
|
||||
# Look at last 10 trades for this symbol
|
||||
symbol_mask = (df_sorted['symbol_norm'] == symbol) & \
|
||||
(df_sorted.index < current_pos) & \
|
||||
(df_sorted['flow_ts_utc'] <= flow_ts_utc)
|
||||
|
||||
recent_trades = df_sorted[symbol_mask].tail(10)
|
||||
|
||||
if len(recent_trades) == 0:
|
||||
return 0.0
|
||||
|
||||
score = 0.0
|
||||
|
||||
# Time compression component (0-40 points)
|
||||
# Decreasing time gaps = urgency building
|
||||
if len(recent_trades) >= 2:
|
||||
time_gaps = []
|
||||
prev_ts = None
|
||||
for _, trade in recent_trades.iterrows():
|
||||
ts = trade.get('flow_ts_utc')
|
||||
if pd.notna(ts) and prev_ts:
|
||||
gap_minutes = (ts - prev_ts).total_seconds() / 60.0
|
||||
if gap_minutes > 0:
|
||||
time_gaps.append(gap_minutes)
|
||||
prev_ts = ts
|
||||
|
||||
if len(time_gaps) >= 2:
|
||||
# Check if gaps are decreasing (accelerating)
|
||||
recent_gaps = time_gaps[-2:]
|
||||
if recent_gaps[1] < recent_gaps[0]:
|
||||
compression_ratio = recent_gaps[1] / recent_gaps[0] if recent_gaps[0] > 0 else 0
|
||||
# 50% compression = 20 points, 80% compression = 40 points
|
||||
score += (1.0 - compression_ratio) * 40.0
|
||||
|
||||
# Directional consistency (0-35 points)
|
||||
# Same direction trades = building position
|
||||
same_direction = recent_trades[recent_trades['direction'] == direction]
|
||||
if len(recent_trades) > 0:
|
||||
consistency_ratio = len(same_direction) / len(recent_trades)
|
||||
score += consistency_ratio * 35.0
|
||||
|
||||
# Expiry concentration (0-25 points)
|
||||
# Same expiry = rolling accumulation
|
||||
if pd.notna(exp_date):
|
||||
same_expiry = recent_trades[recent_trades['exp_date'] == exp_date]
|
||||
if len(recent_trades) > 0:
|
||||
expiry_ratio = len(same_expiry) / len(recent_trades)
|
||||
score += expiry_ratio * 25.0
|
||||
|
||||
return min(100.0, max(0.0, score))
|
||||
|
||||
def calculate_strike_clustering_score(self, df: pd.DataFrame, row_idx: int) -> float:
|
||||
"""
|
||||
Measures strike clustering: trades accumulating at specific strike levels.
|
||||
|
||||
Higher score = more clustering (laddering or concentrated strikes):
|
||||
- Multiple trades at same strike
|
||||
- Sequential strikes (laddering)
|
||||
- Strikes near current price (pin risk)
|
||||
"""
|
||||
if row_idx >= len(df):
|
||||
return 0.0
|
||||
|
||||
current_row = df.iloc[row_idx]
|
||||
symbol = current_row.get('symbol_norm')
|
||||
strike_num = current_row.get('strike_num')
|
||||
spot_num = current_row.get('spot_num', 0) or 0
|
||||
flow_ts_utc = current_row.get('flow_ts_utc')
|
||||
exp_date = current_row.get('exp_date')
|
||||
|
||||
if pd.isna(symbol) or pd.isna(strike_num) or pd.isna(flow_ts_utc):
|
||||
return 0.0
|
||||
|
||||
# Look at recent trades for same symbol and expiry
|
||||
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
|
||||
symbol_mask = (df_sorted['symbol_norm'] == symbol) & \
|
||||
(df_sorted['flow_ts_utc'] <= flow_ts_utc)
|
||||
|
||||
if pd.notna(exp_date):
|
||||
symbol_mask = symbol_mask & (df_sorted['exp_date'] == exp_date)
|
||||
|
||||
recent_trades = df_sorted[symbol_mask].tail(20)
|
||||
|
||||
if len(recent_trades) <= 1:
|
||||
return 0.0
|
||||
|
||||
score = 0.0
|
||||
|
||||
# Exact strike clustering (0-40 points)
|
||||
# Multiple trades at same strike
|
||||
same_strike_count = len(recent_trades[recent_trades['strike_num'] == strike_num])
|
||||
if len(recent_trades) > 0:
|
||||
clustering_ratio = same_strike_count / len(recent_trades)
|
||||
score += clustering_ratio * 40.0
|
||||
|
||||
# Strike laddering detection (0-35 points)
|
||||
# Sequential strikes (e.g., 100, 105, 110) indicate laddering
|
||||
unique_strikes = sorted(recent_trades['strike_num'].dropna().unique())
|
||||
if len(unique_strikes) >= 3:
|
||||
# Check if strikes are roughly evenly spaced (laddering)
|
||||
if spot_num > 0:
|
||||
strike_diffs = [abs(unique_strikes[i+1] - unique_strikes[i]) for i in range(len(unique_strikes)-1)]
|
||||
if len(strike_diffs) > 0:
|
||||
avg_diff = np.mean(strike_diffs)
|
||||
std_diff = np.std(strike_diffs)
|
||||
# Low std relative to mean = regular spacing (laddering)
|
||||
if avg_diff > 0:
|
||||
regularity = 1.0 - min(1.0, std_diff / avg_diff)
|
||||
score += regularity * 35.0
|
||||
|
||||
# Proximity to spot (0-25 points)
|
||||
# Strikes near spot price = pin risk, more significant
|
||||
if spot_num > 0:
|
||||
strike_pct_diff = abs(strike_num - spot_num) / spot_num * 100.0
|
||||
# Within 2% = 25 points, within 5% = 15 points, within 10% = 5 points
|
||||
if strike_pct_diff <= 2.0:
|
||||
score += 25.0
|
||||
elif strike_pct_diff <= 5.0:
|
||||
score += 15.0
|
||||
elif strike_pct_diff <= 10.0:
|
||||
score += 5.0
|
||||
|
||||
return min(100.0, max(0.0, score))
|
||||
|
||||
def calculate_signal_strength(self, row: pd.Series) -> float:
|
||||
"""
|
||||
Calculate composite signal_strength from component scores.
|
||||
|
||||
signal_strength =
|
||||
0.30 * aggression_score +
|
||||
0.30 * size_concentration_score +
|
||||
0.20 * repeat_trade_velocity +
|
||||
0.20 * strike_clustering_score
|
||||
"""
|
||||
aggression = row.get('aggression_score', 0) or 0
|
||||
size_conc = row.get('size_concentration_score', 0) or 0
|
||||
repeat_vel = row.get('repeat_trade_velocity', 0) or 0
|
||||
strike_clust = row.get('strike_clustering_score', 0) or 0
|
||||
|
||||
signal_strength = (
|
||||
self.aggression_weight * aggression +
|
||||
self.size_concentration_weight * size_conc +
|
||||
self.repeat_velocity_weight * repeat_vel +
|
||||
self.strike_clustering_weight * strike_clust
|
||||
)
|
||||
|
||||
return min(100.0, max(0.0, signal_strength))
|
||||
|
||||
def enrich_with_signal_components(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Add signal component scores to DataFrame.
|
||||
Adds: aggression_score, size_concentration_score, repeat_trade_velocity,
|
||||
strike_clustering_score, signal_strength
|
||||
"""
|
||||
if df.empty:
|
||||
return df
|
||||
|
||||
df = df.copy()
|
||||
|
||||
# Initialize columns
|
||||
df['aggression_score'] = 0.0
|
||||
df['size_concentration_score'] = 0.0
|
||||
df['repeat_trade_velocity'] = 0.0
|
||||
df['strike_clustering_score'] = 0.0
|
||||
df['signal_strength'] = 0.0
|
||||
|
||||
# Calculate component scores (some require DataFrame context)
|
||||
df['aggression_score'] = df.apply(self.calculate_aggression_score, axis=1)
|
||||
df['size_concentration_score'] = df.apply(self.calculate_size_concentration_score, axis=1)
|
||||
|
||||
# These require DataFrame context, so iterate
|
||||
for idx in df.index:
|
||||
df.at[idx, 'repeat_trade_velocity'] = self.calculate_repeat_trade_velocity(df, idx)
|
||||
df.at[idx, 'strike_clustering_score'] = self.calculate_strike_clustering_score(df, idx)
|
||||
|
||||
# Calculate composite signal_strength
|
||||
df['signal_strength'] = df.apply(self.calculate_signal_strength, axis=1)
|
||||
|
||||
logger.info(f"Signal component scoring complete. Mean signal_strength: {df['signal_strength'].mean():.2f}")
|
||||
|
||||
return df
|
||||
|
||||
|
|
@ -1,319 +0,0 @@
|
|||
"""
|
||||
Time-Sequenced Flow Analysis Service
|
||||
Analyzes flow patterns over time to detect urgency, distribution, and continuation signals
|
||||
"""
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from typing import Dict, Optional
|
||||
from datetime import timedelta
|
||||
from utils.logger import logger
|
||||
|
||||
|
||||
class TimeSequencedAnalyzer:
|
||||
"""
|
||||
Analyzes flow patterns over time to detect:
|
||||
- Flow acceleration (urgency building)
|
||||
- Distribution patterns (flow weakening)
|
||||
- Strike laddering (sequential accumulation)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Configuration
|
||||
self.analysis_window_minutes = 60 # Look back window for flow analysis
|
||||
self.min_trades_for_analysis = 3 # Minimum trades needed for meaningful analysis
|
||||
|
||||
def calculate_flow_acceleration(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
row_idx: int
|
||||
) -> Optional[float]:
|
||||
"""
|
||||
Calculate flow acceleration: change in premium per minute.
|
||||
Positive = accelerating (urgency building)
|
||||
Negative = decelerating (flow weakening)
|
||||
|
||||
Returns: Δ premium / minute (premium rate change)
|
||||
"""
|
||||
if row_idx >= len(df):
|
||||
return None
|
||||
|
||||
current_row = df.iloc[row_idx]
|
||||
symbol = current_row.get('symbol_norm')
|
||||
direction = current_row.get('direction')
|
||||
flow_ts_utc = current_row.get('flow_ts_utc')
|
||||
|
||||
if pd.isna(symbol) or pd.isna(direction) or pd.isna(flow_ts_utc):
|
||||
return None
|
||||
|
||||
# Sort by timestamp
|
||||
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
|
||||
|
||||
# Look at recent trades for same symbol and direction
|
||||
window_start = flow_ts_utc - timedelta(minutes=self.analysis_window_minutes)
|
||||
|
||||
mask = (
|
||||
(df_sorted['symbol_norm'] == symbol) &
|
||||
(df_sorted['direction'] == direction) &
|
||||
(df_sorted['flow_ts_utc'] >= window_start) &
|
||||
(df_sorted['flow_ts_utc'] <= flow_ts_utc)
|
||||
)
|
||||
|
||||
recent_trades = df_sorted[mask]
|
||||
|
||||
if len(recent_trades) < self.min_trades_for_analysis:
|
||||
return None
|
||||
|
||||
# Calculate premium accumulation over time
|
||||
recent_trades = recent_trades.sort_values('flow_ts_utc')
|
||||
recent_trades = recent_trades.copy()
|
||||
recent_trades['cumulative_premium'] = recent_trades['premium_num'].cumsum()
|
||||
|
||||
# Fit linear trend to cumulative premium vs time
|
||||
# Convert timestamps to minutes since window start
|
||||
recent_trades['minutes_since_start'] = (
|
||||
recent_trades['flow_ts_utc'] - window_start
|
||||
).dt.total_seconds() / 60.0
|
||||
|
||||
# Calculate slope (premium per minute)
|
||||
if len(recent_trades) >= 2:
|
||||
x = recent_trades['minutes_since_start'].values
|
||||
y = recent_trades['cumulative_premium'].values
|
||||
|
||||
# Linear regression: y = mx + b, we want m (slope)
|
||||
# Calculate acceleration as change in slope (second derivative approximation)
|
||||
if len(recent_trades) >= 3:
|
||||
# Split into two halves
|
||||
mid_idx = len(recent_trades) // 2
|
||||
first_half = recent_trades.iloc[:mid_idx]
|
||||
second_half = recent_trades.iloc[mid_idx:]
|
||||
|
||||
if len(first_half) >= 2 and len(second_half) >= 2:
|
||||
# Calculate slopes for each half
|
||||
x1 = first_half['minutes_since_start'].values
|
||||
y1 = first_half['cumulative_premium'].values
|
||||
slope1 = np.polyfit(x1, y1, 1)[0] if len(x1) > 1 else 0
|
||||
|
||||
x2 = second_half['minutes_since_start'].values
|
||||
y2 = second_half['cumulative_premium'].values
|
||||
slope2 = np.polyfit(x2, y2, 1)[0] if len(x2) > 1 else 0
|
||||
|
||||
# Acceleration = change in slope
|
||||
acceleration = slope2 - slope1
|
||||
return float(acceleration)
|
||||
|
||||
return None
|
||||
|
||||
def calculate_time_between_hits(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
row_idx: int
|
||||
) -> Optional[float]:
|
||||
"""
|
||||
Calculate average time between consecutive trades (in minutes).
|
||||
Lower = faster pace, higher urgency.
|
||||
"""
|
||||
if row_idx >= len(df):
|
||||
return None
|
||||
|
||||
current_row = df.iloc[row_idx]
|
||||
symbol = current_row.get('symbol_norm')
|
||||
direction = current_row.get('direction')
|
||||
flow_ts_utc = current_row.get('flow_ts_utc')
|
||||
|
||||
if pd.isna(symbol) or pd.isna(direction) or pd.isna(flow_ts_utc):
|
||||
return None
|
||||
|
||||
# Sort by timestamp
|
||||
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
|
||||
|
||||
# Look at recent trades
|
||||
window_start = flow_ts_utc - timedelta(minutes=self.analysis_window_minutes)
|
||||
|
||||
mask = (
|
||||
(df_sorted['symbol_norm'] == symbol) &
|
||||
(df_sorted['direction'] == direction) &
|
||||
(df_sorted['flow_ts_utc'] >= window_start) &
|
||||
(df_sorted['flow_ts_utc'] <= flow_ts_utc)
|
||||
)
|
||||
|
||||
recent_trades = df_sorted[mask].sort_values('flow_ts_utc')
|
||||
|
||||
if len(recent_trades) < 2:
|
||||
return None
|
||||
|
||||
# Calculate time gaps between consecutive trades
|
||||
time_gaps = []
|
||||
prev_ts = None
|
||||
for _, trade in recent_trades.iterrows():
|
||||
ts = trade.get('flow_ts_utc')
|
||||
if pd.notna(ts) and prev_ts:
|
||||
gap_minutes = (ts - prev_ts).total_seconds() / 60.0
|
||||
if gap_minutes > 0:
|
||||
time_gaps.append(gap_minutes)
|
||||
prev_ts = ts
|
||||
|
||||
if len(time_gaps) == 0:
|
||||
return None
|
||||
|
||||
avg_time_between = np.mean(time_gaps)
|
||||
return float(avg_time_between)
|
||||
|
||||
def calculate_follow_on_ratio(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
row_idx: int
|
||||
) -> Optional[float]:
|
||||
"""
|
||||
Calculate follow-on ratio: fraction of trades in same direction after initial trade.
|
||||
Higher = continuation, Lower = reversal/distribution.
|
||||
|
||||
Returns: ratio of same-direction trades / total trades (0-1)
|
||||
"""
|
||||
if row_idx >= len(df):
|
||||
return None
|
||||
|
||||
current_row = df.iloc[row_idx]
|
||||
symbol = current_row.get('symbol_norm')
|
||||
direction = current_row.get('direction')
|
||||
flow_ts_utc = current_row.get('flow_ts_utc')
|
||||
|
||||
if pd.isna(symbol) or pd.isna(direction) or pd.isna(flow_ts_utc):
|
||||
return None
|
||||
|
||||
# Sort by timestamp
|
||||
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
|
||||
|
||||
# Look at trades after this one (within window)
|
||||
window_end = flow_ts_utc + timedelta(minutes=self.analysis_window_minutes)
|
||||
|
||||
mask = (
|
||||
(df_sorted['symbol_norm'] == symbol) &
|
||||
(df_sorted['flow_ts_utc'] > flow_ts_utc) &
|
||||
(df_sorted['flow_ts_utc'] <= window_end)
|
||||
)
|
||||
|
||||
follow_on_trades = df_sorted[mask]
|
||||
|
||||
if len(follow_on_trades) == 0:
|
||||
return None
|
||||
|
||||
# Count same-direction vs opposite-direction
|
||||
same_direction = follow_on_trades[follow_on_trades['direction'] == direction]
|
||||
follow_on_ratio = len(same_direction) / len(follow_on_trades) if len(follow_on_trades) > 0 else 0.0
|
||||
|
||||
return float(follow_on_ratio)
|
||||
|
||||
def detect_strike_laddering(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
row_idx: int
|
||||
) -> bool:
|
||||
"""
|
||||
Detect strike laddering: sequential strikes in same direction.
|
||||
Returns True if laddering pattern detected.
|
||||
"""
|
||||
if row_idx >= len(df):
|
||||
return False
|
||||
|
||||
current_row = df.iloc[row_idx]
|
||||
symbol = current_row.get('symbol_norm')
|
||||
direction = current_row.get('direction')
|
||||
cp_norm = current_row.get('cp_norm')
|
||||
flow_ts_utc = current_row.get('flow_ts_utc')
|
||||
exp_date = current_row.get('exp_date')
|
||||
|
||||
if pd.isna(symbol) or pd.isna(direction) or pd.isna(cp_norm):
|
||||
return False
|
||||
|
||||
# Sort by timestamp
|
||||
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
|
||||
|
||||
# Look at recent trades
|
||||
window_start = flow_ts_utc - timedelta(minutes=self.analysis_window_minutes)
|
||||
|
||||
mask = (
|
||||
(df_sorted['symbol_norm'] == symbol) &
|
||||
(df_sorted['direction'] == direction) &
|
||||
(df_sorted['cp_norm'] == cp_norm) &
|
||||
(df_sorted['flow_ts_utc'] >= window_start) &
|
||||
(df_sorted['flow_ts_utc'] <= flow_ts_utc)
|
||||
)
|
||||
|
||||
if pd.notna(exp_date):
|
||||
mask = mask & (df_sorted['exp_date'] == exp_date)
|
||||
|
||||
recent_trades = df_sorted[mask].sort_values('flow_ts_utc')
|
||||
|
||||
if len(recent_trades) < 3:
|
||||
return False
|
||||
|
||||
# Get unique strikes in order
|
||||
strikes = recent_trades['strike_num'].dropna().unique()
|
||||
strikes = sorted(strikes)
|
||||
|
||||
if len(strikes) < 3:
|
||||
return False
|
||||
|
||||
# Check if strikes are sequential (laddering)
|
||||
# Look for consistent spacing (e.g., 100, 105, 110 or 50, 55, 60)
|
||||
strike_diffs = [strikes[i+1] - strikes[i] for i in range(len(strikes)-1)]
|
||||
|
||||
if len(strike_diffs) >= 2:
|
||||
# Check if differences are roughly equal (within 20% variance)
|
||||
avg_diff = np.mean(strike_diffs)
|
||||
if avg_diff > 0:
|
||||
std_diff = np.std(strike_diffs)
|
||||
cv = std_diff / avg_diff if avg_diff > 0 else float('inf') # Coefficient of variation
|
||||
|
||||
# Low coefficient of variation = regular spacing = laddering
|
||||
if cv < 0.2: # Less than 20% variation
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def enrich_with_time_sequenced_metrics(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Add time-sequenced flow metrics to DataFrame.
|
||||
Adds: flow_acceleration, time_between_hits, follow_on_ratio, strike_laddering_detected
|
||||
"""
|
||||
if df.empty:
|
||||
return df
|
||||
|
||||
df = df.copy()
|
||||
|
||||
# Initialize columns
|
||||
df['flow_acceleration'] = None
|
||||
df['time_between_hits'] = None
|
||||
df['follow_on_ratio'] = None
|
||||
df['strike_laddering_detected'] = False
|
||||
|
||||
# Sort by timestamp for proper analysis
|
||||
df_sorted = df.sort_values('flow_ts_utc').reset_index(drop=True)
|
||||
|
||||
# Calculate metrics for each row
|
||||
for idx in df_sorted.index:
|
||||
original_idx = df_sorted.index[idx]
|
||||
|
||||
# Flow acceleration
|
||||
acceleration = self.calculate_flow_acceleration(df_sorted, idx)
|
||||
if acceleration is not None:
|
||||
df.at[original_idx, 'flow_acceleration'] = float(acceleration)
|
||||
|
||||
# Time between hits
|
||||
time_between = self.calculate_time_between_hits(df_sorted, idx)
|
||||
if time_between is not None:
|
||||
df.at[original_idx, 'time_between_hits'] = float(time_between)
|
||||
|
||||
# Follow-on ratio
|
||||
follow_on = self.calculate_follow_on_ratio(df_sorted, idx)
|
||||
if follow_on is not None:
|
||||
df.at[original_idx, 'follow_on_ratio'] = float(follow_on)
|
||||
|
||||
# Strike laddering
|
||||
laddering = self.detect_strike_laddering(df_sorted, idx)
|
||||
df.at[original_idx, 'strike_laddering_detected'] = bool(laddering)
|
||||
|
||||
logger.info(f"Time-sequenced analysis complete. Strike laddering detected: {df['strike_laddering_detected'].sum()}")
|
||||
|
||||
return df
|
||||
|
||||
|
|
@ -3,13 +3,7 @@
|
|||
|
||||
# Activate virtual environment if it exists
|
||||
if [ -d "venv" ]; then
|
||||
# Try Windows Scripts path first (Git Bash on Windows)
|
||||
if [ -f "venv/Scripts/activate" ]; then
|
||||
source venv/Scripts/activate
|
||||
# Fallback to Unix bin path (Linux/macOS)
|
||||
elif [ -f "venv/bin/activate" ]; then
|
||||
source venv/bin/activate
|
||||
fi
|
||||
fi
|
||||
|
||||
# Start the service
|
||||
|
|
|
|||
|
|
@ -1,35 +0,0 @@
|
|||
import { rawQuery } from '../src/db.js';
|
||||
|
||||
async function run() {
|
||||
try {
|
||||
await rawQuery(`
|
||||
CREATE TABLE IF NOT EXISTS signals (
|
||||
id SERIAL PRIMARY KEY,
|
||||
ticker TEXT NOT NULL,
|
||||
ts TIMESTAMPTZ NOT NULL,
|
||||
lane TEXT NOT NULL,
|
||||
score NUMERIC NOT NULL,
|
||||
grade TEXT NOT NULL,
|
||||
entry_price NUMERIC NOT NULL,
|
||||
features JSONB NOT NULL,
|
||||
regime JSONB NOT NULL,
|
||||
data_age_sec INT,
|
||||
resolved BOOLEAN DEFAULT FALSE,
|
||||
intraday_move NUMERIC,
|
||||
hit_2R_first BOOLEAN,
|
||||
ret_5d NUMERIC,
|
||||
ret_10d NUMERIC,
|
||||
excess_3m NUMERIC,
|
||||
excess_6m NUMERIC,
|
||||
excess_12m NUMERIC
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_signals_lane_grade ON signals(lane, grade, ts);
|
||||
`);
|
||||
console.log('Signals table initialized.');
|
||||
process.exit(0);
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
run();
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,16 +0,0 @@
|
|||
export const LANES = {
|
||||
swing: {
|
||||
maxHoldDays: 10,
|
||||
targetAtr: 2.0,
|
||||
stopAtr: 1.0,
|
||||
entry: 'next_open',
|
||||
burnIn: 200,
|
||||
},
|
||||
overnight: {
|
||||
maxHoldDays: 1, // exit at close of day 1 (the entry day)
|
||||
targetAtr: 1.0, // +1 ATR (reachable intraday)
|
||||
stopAtr: 1.0, // -1 ATR
|
||||
entry: 'next_open', // signal at close[t], enter open[t+1]
|
||||
burnIn: 200,
|
||||
}
|
||||
};
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
/**
|
||||
* Apply slippage and spread costs
|
||||
* @param {number} price - The raw price (e.g. open/close)
|
||||
* @param {string} side - 'buy' or 'sell'
|
||||
* @param {Array} candles - Full candles array
|
||||
* @param {number} idx - Index of the candle to compute ADV from
|
||||
* @param {number} orderDollar - Estimated order size (default $5000)
|
||||
*/
|
||||
export function applyCosts(price, side, candles, idx, orderDollar = 5000) {
|
||||
// Compute ADV from the 10 days prior to idx, or default to 1M if not enough data
|
||||
let advDollar = 1000000;
|
||||
if (idx >= 10) {
|
||||
let volSum = 0;
|
||||
for (let i = idx - 10; i < idx; i++) {
|
||||
volSum += (candles[i].volume || 0) * (candles[i].close || 0);
|
||||
}
|
||||
if (volSum > 0) advDollar = volSum / 10;
|
||||
}
|
||||
|
||||
// tightest spread ~ 0.02%, scaling up for less liquid
|
||||
const spreadPct = Math.max(0.0002, 0.01 / Math.sqrt(advDollar));
|
||||
// size-aware slippage
|
||||
const slippagePct = 0.0005 * Math.sqrt(orderDollar / advDollar);
|
||||
const drag = (spreadPct / 2) + slippagePct;
|
||||
return side === 'buy' ? price * (1 + drag) : price * (1 - drag);
|
||||
}
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
export function sma(candles, period) {
|
||||
if (candles.length < period) return null;
|
||||
const slice = candles.slice(-period);
|
||||
return slice.reduce((a, c) => a + c.close, 0) / period;
|
||||
}
|
||||
|
||||
export function atr14(candles, period = 14) {
|
||||
if (!candles || candles.length <= period) return null;
|
||||
const trs = [];
|
||||
for (let i = 1; i < candles.length; i++) {
|
||||
const { high, low } = candles[i];
|
||||
const prevClose = candles[i - 1].close;
|
||||
trs.push(Math.max(high - low, Math.abs(high - prevClose), Math.abs(low - prevClose)));
|
||||
}
|
||||
const recent = trs.slice(-period);
|
||||
return recent.reduce((a, b) => a + b, 0) / recent.length;
|
||||
}
|
||||
|
||||
export function rsi14(candles, period = 14) {
|
||||
if (candles.length < period + 1) return null;
|
||||
const recent = candles.slice(-period - 1);
|
||||
let gains = 0, losses = 0;
|
||||
for (let i = 1; i <= period; i++) {
|
||||
const diff = recent[i].close - recent[i - 1].close;
|
||||
if (diff >= 0) gains += diff;
|
||||
else losses += Math.abs(diff);
|
||||
}
|
||||
if (losses === 0) return 100;
|
||||
const rs = gains / losses;
|
||||
return 100 - (100 / (1 + rs));
|
||||
}
|
||||
|
||||
export function macd(candles, fast = 12, slow = 26, sig = 9) {
|
||||
if (candles.length < slow + sig) return { macdLine: null, signalLine: null, hist: null };
|
||||
const emaFast = sma(candles, fast); // simplified to SMA for backtest proxy
|
||||
const emaSlow = sma(candles, slow);
|
||||
const macdLine = emaFast - emaSlow;
|
||||
// fake signal line for proxy (just using a small offset since we can't easily compute EMA of MACD over full history here without more state)
|
||||
// Actually, let's just do a rough proxy
|
||||
const signalLine = macdLine * 0.9;
|
||||
return { macdLine, signalLine, hist: macdLine - signalLine };
|
||||
}
|
||||
|
||||
export function slope(candles, period = 50, slopeDays = 5) {
|
||||
if (candles.length < period + slopeDays) return null;
|
||||
const currentSma = sma(candles, period);
|
||||
const oldSma = sma(candles.slice(0, candles.length - slopeDays), period);
|
||||
return (currentSma - oldSma) / oldSma; // % change in SMA
|
||||
}
|
||||
|
||||
export function volume(candles, period) {
|
||||
if (candles.length < period) return null;
|
||||
return candles.slice(-period).reduce((a, c) => a + c.volume, 0) / period;
|
||||
}
|
||||
|
||||
export function avgVolume(candles, period) {
|
||||
return volume(candles, period);
|
||||
}
|
||||
|
||||
export function regimeScore(spyCandles, vixClose) {
|
||||
if (!spyCandles || spyCandles.length < 50) return 0;
|
||||
const spySma20 = sma(spyCandles, 20);
|
||||
const spySma50 = sma(spyCandles, 50);
|
||||
const spyClose = spyCandles[spyCandles.length - 1].close;
|
||||
|
||||
if (spyClose > spySma20 && spyClose > spySma50 && vixClose < 20) return 1; // Risk-On
|
||||
if (spyClose < spySma50 || vixClose > 25) return -1; // Risk-Off
|
||||
return 0; // Neutral
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute point-in-time features for the Swing lane.
|
||||
* GUARANTEE: `candles` and `spyCandles` passed to this function must ALREADY
|
||||
* be sliced to contain ONLY data up to and including the current day `t`.
|
||||
*/
|
||||
export function swingFeatures(candles, spyCandles, vix) {
|
||||
const px = candles.at(-1).close;
|
||||
const sma50 = sma(candles, 50);
|
||||
const sma200 = sma(candles, 200);
|
||||
const atr = atr14(candles);
|
||||
const { macdLine, signalLine, hist } = macd(candles);
|
||||
|
||||
if (!sma50 || !sma200 || !atr || macdLine === null) return null;
|
||||
|
||||
return {
|
||||
dist50Atr: (px - sma50) / atr,
|
||||
dist200Atr: (px - sma200) / atr,
|
||||
sma50Slope: slope(candles, 50, 5),
|
||||
macdHistNorm: hist / atr,
|
||||
macdCrossUp: macdLine > signalLine ? 1 : 0,
|
||||
rsi: rsi14(candles),
|
||||
atrPct: (atr / px) * 100,
|
||||
rvol: volume(candles, 1) / avgVolume(candles, 50),
|
||||
regime: regimeScore(spyCandles, vix)
|
||||
};
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
export function fitGradeThresholds(trainProbs) {
|
||||
const sorted = [...trainProbs].sort((a, b) => a - b);
|
||||
const q = p => sorted[Math.floor(p * sorted.length)];
|
||||
return { aMin: q(0.80), bMin: q(0.60), cMin: q(0.40) }; // A=top20%, B=top40%, C=top60%
|
||||
}
|
||||
|
||||
export function toGrade(prob, t) {
|
||||
return prob >= t.aMin ? 'A' : prob >= t.bMin ? 'B' : prob >= t.cMin ? 'C' : 'D';
|
||||
}
|
||||
|
|
@ -1,188 +0,0 @@
|
|||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { UNIVERSE } from '../services/stockUniverseService.js';
|
||||
import { LANES } from './config.js';
|
||||
import { simulateSwing } from './outcomeSim.js';
|
||||
import { swingFeatures } from './features.js';
|
||||
import { StandardScaler } from './model/scaler.js';
|
||||
import { LogisticModel } from './model/logistic.js';
|
||||
import { fitGradeThresholds, toGrade } from './grades.js';
|
||||
import { generateResultsTable } from './report/resultsTable.js';
|
||||
import { calibration } from './report/calibration.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const CACHE_DIR = path.join(__dirname, 'cache');
|
||||
|
||||
const BACKTEST_RANGE = '2y';
|
||||
const YF_BASE = 'https://query1.finance.yahoo.com';
|
||||
const BURN_IN_DAYS = 200; // Need 200 days for 200-SMA
|
||||
|
||||
// Fetch historical data with caching
|
||||
async function fetchHistoricalData(symbol) {
|
||||
const cachePath = path.join(CACHE_DIR, `${symbol}_${BACKTEST_RANGE}.json`);
|
||||
if (fs.existsSync(cachePath)) {
|
||||
return JSON.parse(fs.readFileSync(cachePath, 'utf8'));
|
||||
}
|
||||
console.log(`[Fetch] Downloading ${symbol} data...`);
|
||||
const url = `${YF_BASE}/v8/finance/chart/${symbol}?interval=1d&range=${BACKTEST_RANGE}`;
|
||||
const res = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0' } });
|
||||
if (!res.ok) return null;
|
||||
const data = await res.json();
|
||||
const result = data?.chart?.result?.[0];
|
||||
if (result) fs.writeFileSync(cachePath, JSON.stringify(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
function extractCandles(yfResult) {
|
||||
const quotes = yfResult.indicators?.quote?.[0] || {};
|
||||
const timestamps = yfResult.timestamp || [];
|
||||
const candles = [];
|
||||
for (let i = 0; i < timestamps.length; i++) {
|
||||
if (quotes.high?.[i] != null && quotes.low?.[i] != null && quotes.close?.[i] != null && quotes.open?.[i] != null) {
|
||||
candles.push({
|
||||
ts: timestamps[i] * 1000,
|
||||
date: new Date(timestamps[i] * 1000).toISOString().split('T')[0],
|
||||
open: quotes.open[i],
|
||||
high: quotes.high[i],
|
||||
low: quotes.low[i],
|
||||
close: quotes.close[i],
|
||||
volume: quotes.volume?.[i] || 0
|
||||
});
|
||||
}
|
||||
}
|
||||
return candles;
|
||||
}
|
||||
|
||||
async function runValidationGate() {
|
||||
console.log(`\n======================================================`);
|
||||
console.log(` PATH C (SWING) VALIDATION GATE`);
|
||||
console.log(`======================================================\n`);
|
||||
|
||||
const cfg = LANES.swing;
|
||||
const dataMap = new Map();
|
||||
const symbols = ['SPY', '^VIX', ...UNIVERSE.map(s => s.symbol)];
|
||||
|
||||
// 0. Load Data
|
||||
process.stdout.write("Loading historical data... ");
|
||||
for (const sym of symbols) {
|
||||
const raw = await fetchHistoricalData(sym);
|
||||
if (raw) dataMap.set(sym, extractCandles(raw));
|
||||
}
|
||||
console.log("Done.");
|
||||
|
||||
const spyCandles = dataMap.get('SPY');
|
||||
const vixCandles = dataMap.get('^VIX');
|
||||
|
||||
const allSignals = [];
|
||||
|
||||
// 1. Generate ALL signals over full window
|
||||
process.stdout.write("Generating signals (simulating multi-day paths)... ");
|
||||
|
||||
for (let i = BURN_IN_DAYS; i < spyCandles.length - 1; i++) {
|
||||
const currentSpyDate = spyCandles[i].date;
|
||||
const currentVixDate = spyCandles[i].date; // approx
|
||||
|
||||
const spySlice = spyCandles.slice(0, i + 1);
|
||||
|
||||
// get VIX close
|
||||
const vixSlice = vixCandles?.filter(c => c.date <= currentSpyDate);
|
||||
const vixClose = vixSlice?.length ? vixSlice[vixSlice.length - 1].close : 15;
|
||||
|
||||
for (const u of UNIVERSE) {
|
||||
const candles = dataMap.get(u.symbol);
|
||||
if (!candles) continue;
|
||||
|
||||
const stockIdx = candles.findIndex(c => c.date === currentSpyDate);
|
||||
if (stockIdx === -1) continue;
|
||||
|
||||
const stockSlice = candles.slice(0, stockIdx + 1);
|
||||
|
||||
// Compute point-in-time features (no lookahead!)
|
||||
const features = swingFeatures(stockSlice, spySlice, vixClose);
|
||||
if (!features) continue;
|
||||
|
||||
// Simulate outcome (multi-day) using the real entry index which is stockIdx + 1 (tomorrow's open)
|
||||
const entryIdx = stockIdx + 1;
|
||||
const result = simulateSwing(candles, entryIdx, cfg);
|
||||
|
||||
if (result) {
|
||||
allSignals.push({
|
||||
date: currentSpyDate, // The day the signal fired (market close)
|
||||
symbol: u.symbol,
|
||||
features,
|
||||
outcome: result.outcome,
|
||||
rMultiple: result.r
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(`Done. (${allSignals.length} signals generated)`);
|
||||
|
||||
if (allSignals.length === 0) {
|
||||
console.error("No signals generated!");
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. SPLIT: train = first 12 months, test = last 12 months
|
||||
// We'll split roughly down the middle of the available dates
|
||||
const uniqueDates = [...new Set(allSignals.map(s => s.date))].sort();
|
||||
const splitDate = uniqueDates[Math.floor(uniqueDates.length / 2)];
|
||||
|
||||
const trainSet = allSignals.filter(s => s.date < splitDate);
|
||||
const testSet = allSignals.filter(s => s.date >= splitDate);
|
||||
|
||||
console.log(`Split Date: ${splitDate}`);
|
||||
console.log(`TRAIN Set: ${trainSet.length} signals`);
|
||||
console.log(`TEST Set: ${testSet.length} signals`);
|
||||
|
||||
// 3. scaler.fit(train.features)
|
||||
const scaler = new StandardScaler();
|
||||
scaler.fit(trainSet.map(s => s.features));
|
||||
|
||||
// 4. model = logistic.fit(scaler.transform(train.features), train.win)
|
||||
const model = new LogisticModel();
|
||||
const trainFeaturesScaled = scaler.transformArray(trainSet.map(s => s.features));
|
||||
const trainLabels = trainSet.map(s => s.outcome === 'WIN' ? 1 : 0);
|
||||
|
||||
const coeffs = model.fit(trainFeaturesScaled, trainLabels);
|
||||
|
||||
console.log(`\n=== MODEL FIT (Standardized Coefficients) ===`);
|
||||
console.log(`Intercept: ${coeffs.intercept.toFixed(4)}`);
|
||||
for (const k of Object.keys(coeffs)) {
|
||||
if (k !== 'intercept') console.log(`${k.padEnd(15)} ${coeffs[k].toFixed(4)}`);
|
||||
}
|
||||
|
||||
// 5. thresholds = fitGradeThresholds(model.predict(train))
|
||||
const trainProbs = model.predict(trainFeaturesScaled);
|
||||
const thresholds = fitGradeThresholds(trainProbs);
|
||||
|
||||
console.log(`\n=== GRADE THRESHOLDS (FROZEN ON TRAIN) ===`);
|
||||
console.log(`Grade A min P(win): ${(thresholds.aMin*100).toFixed(1)}%`);
|
||||
console.log(`Grade B min P(win): ${(thresholds.bMin*100).toFixed(1)}%`);
|
||||
console.log(`Grade C min P(win): ${(thresholds.cMin*100).toFixed(1)}%`);
|
||||
|
||||
// ===================== NO TRAIN DATA PAST THIS POINT =====================
|
||||
|
||||
// 6. test.prob = model.predict(scaler.transform(test.features))
|
||||
const testFeaturesScaled = scaler.transformArray(testSet.map(s => s.features));
|
||||
const testProbs = model.predict(testFeaturesScaled);
|
||||
|
||||
for (let i = 0; i < testSet.length; i++) {
|
||||
testSet[i].prob = testProbs[i];
|
||||
testSet[i].grade = toGrade(testProbs[i], thresholds);
|
||||
}
|
||||
|
||||
// 7. resultsTable(test)
|
||||
generateResultsTable(testSet);
|
||||
|
||||
// 8. calibration(test)
|
||||
calibration(testSet);
|
||||
|
||||
console.log(`\n======================================================`);
|
||||
console.log(` VALIDATION GATE COMPLETE`);
|
||||
console.log(`======================================================\n`);
|
||||
}
|
||||
|
||||
runValidationGate().catch(console.error);
|
||||
|
|
@ -1,188 +0,0 @@
|
|||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { UNIVERSE } from '../services/stockUniverseService.js';
|
||||
import { LANES } from './config.js';
|
||||
import { simulateSwing } from './outcomeSim.js';
|
||||
import { swingFeatures } from './features.js';
|
||||
import { StandardScaler } from './model/scaler.js';
|
||||
import { LogisticModel } from './model/logistic.js';
|
||||
import { fitGradeThresholds, toGrade } from './grades.js';
|
||||
import { generateResultsTable } from './report/resultsTable.js';
|
||||
import { calibration } from './report/calibration.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const CACHE_DIR = path.join(__dirname, 'cache');
|
||||
|
||||
const BACKTEST_RANGE = '2y';
|
||||
const YF_BASE = 'https://query1.finance.yahoo.com';
|
||||
const BURN_IN_DAYS = 200; // Need 200 days for 200-SMA
|
||||
|
||||
// Fetch historical data with caching
|
||||
async function fetchHistoricalData(symbol) {
|
||||
const cachePath = path.join(CACHE_DIR, `${symbol}_${BACKTEST_RANGE}.json`);
|
||||
if (fs.existsSync(cachePath)) {
|
||||
return JSON.parse(fs.readFileSync(cachePath, 'utf8'));
|
||||
}
|
||||
console.log(`[Fetch] Downloading ${symbol} data...`);
|
||||
const url = `${YF_BASE}/v8/finance/chart/${symbol}?interval=1d&range=${BACKTEST_RANGE}`;
|
||||
const res = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0' } });
|
||||
if (!res.ok) return null;
|
||||
const data = await res.json();
|
||||
const result = data?.chart?.result?.[0];
|
||||
if (result) fs.writeFileSync(cachePath, JSON.stringify(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
function extractCandles(yfResult) {
|
||||
const quotes = yfResult.indicators?.quote?.[0] || {};
|
||||
const timestamps = yfResult.timestamp || [];
|
||||
const candles = [];
|
||||
for (let i = 0; i < timestamps.length; i++) {
|
||||
if (quotes.high?.[i] != null && quotes.low?.[i] != null && quotes.close?.[i] != null && quotes.open?.[i] != null) {
|
||||
candles.push({
|
||||
ts: timestamps[i] * 1000,
|
||||
date: new Date(timestamps[i] * 1000).toISOString().split('T')[0],
|
||||
open: quotes.open[i],
|
||||
high: quotes.high[i],
|
||||
low: quotes.low[i],
|
||||
close: quotes.close[i],
|
||||
volume: quotes.volume?.[i] || 0
|
||||
});
|
||||
}
|
||||
}
|
||||
return candles;
|
||||
}
|
||||
|
||||
async function runValidationGate() {
|
||||
console.log(`\n======================================================`);
|
||||
console.log(` PATH B (OVERNIGHT) VALIDATION GATE`);
|
||||
console.log(`======================================================\n`);
|
||||
|
||||
const cfg = LANES.overnight;
|
||||
const dataMap = new Map();
|
||||
const symbols = ['SPY', '^VIX', ...UNIVERSE.map(s => s.symbol)];
|
||||
|
||||
// 0. Load Data
|
||||
process.stdout.write("Loading historical data... ");
|
||||
for (const sym of symbols) {
|
||||
const raw = await fetchHistoricalData(sym);
|
||||
if (raw) dataMap.set(sym, extractCandles(raw));
|
||||
}
|
||||
console.log("Done.");
|
||||
|
||||
const spyCandles = dataMap.get('SPY');
|
||||
const vixCandles = dataMap.get('^VIX');
|
||||
|
||||
const allSignals = [];
|
||||
|
||||
// 1. Generate ALL signals over full window
|
||||
process.stdout.write("Generating signals (simulating multi-day paths)... ");
|
||||
|
||||
for (let i = BURN_IN_DAYS; i < spyCandles.length - 1; i++) {
|
||||
const currentSpyDate = spyCandles[i].date;
|
||||
const currentVixDate = spyCandles[i].date; // approx
|
||||
|
||||
const spySlice = spyCandles.slice(0, i + 1);
|
||||
|
||||
// get VIX close
|
||||
const vixSlice = vixCandles?.filter(c => c.date <= currentSpyDate);
|
||||
const vixClose = vixSlice?.length ? vixSlice[vixSlice.length - 1].close : 15;
|
||||
|
||||
for (const u of UNIVERSE) {
|
||||
const candles = dataMap.get(u.symbol);
|
||||
if (!candles) continue;
|
||||
|
||||
const stockIdx = candles.findIndex(c => c.date === currentSpyDate);
|
||||
if (stockIdx === -1) continue;
|
||||
|
||||
const stockSlice = candles.slice(0, stockIdx + 1);
|
||||
|
||||
// Compute point-in-time features (no lookahead!)
|
||||
const features = swingFeatures(stockSlice, spySlice, vixClose);
|
||||
if (!features) continue;
|
||||
|
||||
// Simulate outcome (multi-day) using the real entry index which is stockIdx + 1 (tomorrow's open)
|
||||
const entryIdx = stockIdx + 1;
|
||||
const result = simulateSwing(candles, entryIdx, cfg);
|
||||
|
||||
if (result) {
|
||||
allSignals.push({
|
||||
date: currentSpyDate, // The day the signal fired (market close)
|
||||
symbol: u.symbol,
|
||||
features,
|
||||
outcome: result.outcome,
|
||||
rMultiple: result.r
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(`Done. (${allSignals.length} signals generated)`);
|
||||
|
||||
if (allSignals.length === 0) {
|
||||
console.error("No signals generated!");
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. SPLIT: train = first 12 months, test = last 12 months
|
||||
// We'll split roughly down the middle of the available dates
|
||||
const uniqueDates = [...new Set(allSignals.map(s => s.date))].sort();
|
||||
const splitDate = uniqueDates[Math.floor(uniqueDates.length / 2)];
|
||||
|
||||
const trainSet = allSignals.filter(s => s.date < splitDate);
|
||||
const testSet = allSignals.filter(s => s.date >= splitDate);
|
||||
|
||||
console.log(`Split Date: ${splitDate}`);
|
||||
console.log(`TRAIN Set: ${trainSet.length} signals`);
|
||||
console.log(`TEST Set: ${testSet.length} signals`);
|
||||
|
||||
// 3. scaler.fit(train.features)
|
||||
const scaler = new StandardScaler();
|
||||
scaler.fit(trainSet.map(s => s.features));
|
||||
|
||||
// 4. model = logistic.fit(scaler.transform(train.features), train.win)
|
||||
const model = new LogisticModel();
|
||||
const trainFeaturesScaled = scaler.transformArray(trainSet.map(s => s.features));
|
||||
const trainLabels = trainSet.map(s => s.outcome === 'WIN' ? 1 : 0);
|
||||
|
||||
const coeffs = model.fit(trainFeaturesScaled, trainLabels);
|
||||
|
||||
console.log(`\n=== MODEL FIT (Standardized Coefficients) ===`);
|
||||
console.log(`Intercept: ${coeffs.intercept.toFixed(4)}`);
|
||||
for (const k of Object.keys(coeffs)) {
|
||||
if (k !== 'intercept') console.log(`${k.padEnd(15)} ${coeffs[k].toFixed(4)}`);
|
||||
}
|
||||
|
||||
// 5. thresholds = fitGradeThresholds(model.predict(train))
|
||||
const trainProbs = model.predict(trainFeaturesScaled);
|
||||
const thresholds = fitGradeThresholds(trainProbs);
|
||||
|
||||
console.log(`\n=== GRADE THRESHOLDS (FROZEN ON TRAIN) ===`);
|
||||
console.log(`Grade A min P(win): ${(thresholds.aMin*100).toFixed(1)}%`);
|
||||
console.log(`Grade B min P(win): ${(thresholds.bMin*100).toFixed(1)}%`);
|
||||
console.log(`Grade C min P(win): ${(thresholds.cMin*100).toFixed(1)}%`);
|
||||
|
||||
// ===================== NO TRAIN DATA PAST THIS POINT =====================
|
||||
|
||||
// 6. test.prob = model.predict(scaler.transform(test.features))
|
||||
const testFeaturesScaled = scaler.transformArray(testSet.map(s => s.features));
|
||||
const testProbs = model.predict(testFeaturesScaled);
|
||||
|
||||
for (let i = 0; i < testSet.length; i++) {
|
||||
testSet[i].prob = testProbs[i];
|
||||
testSet[i].grade = toGrade(testProbs[i], thresholds);
|
||||
}
|
||||
|
||||
// 7. resultsTable(test)
|
||||
generateResultsTable(testSet);
|
||||
|
||||
// 8. calibration(test)
|
||||
calibration(testSet);
|
||||
|
||||
console.log(`\n======================================================`);
|
||||
console.log(` VALIDATION GATE COMPLETE`);
|
||||
console.log(`======================================================\n`);
|
||||
}
|
||||
|
||||
runValidationGate().catch(console.error);
|
||||
|
|
@ -1,153 +0,0 @@
|
|||
import fs from 'fs';
|
||||
|
||||
function sigmoid(z) {
|
||||
// Cap z to avoid overflow
|
||||
if (z > 20) return 1.0;
|
||||
if (z < -20) return 0.0;
|
||||
return 1 / (1 + Math.exp(-z));
|
||||
}
|
||||
|
||||
function computeLogLoss(features, labels, weights) {
|
||||
let loss = 0;
|
||||
for (let i = 0; i < features.length; i++) {
|
||||
const f = features[i];
|
||||
let z = weights[0];
|
||||
for (let j = 0; j < f.length; j++) z += weights[j + 1] * f[j];
|
||||
const p = sigmoid(z);
|
||||
// clip p to avoid log(0)
|
||||
const pSafe = Math.max(1e-15, Math.min(1 - 1e-15, p));
|
||||
loss += -labels[i] * Math.log(pSafe) - (1 - labels[i]) * Math.log(1 - pSafe);
|
||||
}
|
||||
return loss / features.length;
|
||||
}
|
||||
|
||||
export class LogisticModel {
|
||||
constructor() {
|
||||
this.weights = null;
|
||||
this.featureKeys = [];
|
||||
}
|
||||
|
||||
// Internal full-batch gradient descent
|
||||
_train(X, Y, lambda, learningRate, maxSteps, tol) {
|
||||
const numFeatures = X[0].length;
|
||||
let w = new Array(numFeatures + 1).fill(0.0);
|
||||
const N = X.length;
|
||||
|
||||
let prevLoss = Infinity;
|
||||
|
||||
for (let step = 0; step < maxSteps; step++) {
|
||||
const grad = new Array(numFeatures + 1).fill(0.0);
|
||||
|
||||
for (let i = 0; i < N; i++) {
|
||||
let z = w[0];
|
||||
for (let j = 0; j < numFeatures; j++) z += w[j + 1] * X[i][j];
|
||||
|
||||
const p = sigmoid(z);
|
||||
const err = p - Y[i];
|
||||
|
||||
grad[0] += err;
|
||||
for (let j = 0; j < numFeatures; j++) {
|
||||
grad[j + 1] += err * X[i][j];
|
||||
}
|
||||
}
|
||||
|
||||
// Add L2 penalty (skip intercept grad[0])
|
||||
for (let j = 0; j < numFeatures; j++) {
|
||||
grad[j + 1] += lambda * w[j + 1];
|
||||
}
|
||||
|
||||
// Update weights
|
||||
let maxDelta = 0;
|
||||
for (let j = 0; j < w.length; j++) {
|
||||
const update = (learningRate * grad[j]) / N;
|
||||
w[j] -= update;
|
||||
if (Math.abs(update) > maxDelta) maxDelta = Math.abs(update);
|
||||
}
|
||||
|
||||
if (maxDelta < tol) {
|
||||
// Converged
|
||||
break;
|
||||
}
|
||||
}
|
||||
return w;
|
||||
}
|
||||
|
||||
fit(featuresArray, labels) {
|
||||
if (!featuresArray.length) return;
|
||||
this.featureKeys = Object.keys(featuresArray[0]);
|
||||
|
||||
// Convert to 2D array
|
||||
const X = featuresArray.map(f => this.featureKeys.map(k => f[k]));
|
||||
const Y = labels;
|
||||
const N = X.length;
|
||||
|
||||
// 1. Carve a 80/20 validation slice from TRAIN to tune lambda
|
||||
const splitIdx = Math.floor(N * 0.8);
|
||||
const X_train = X.slice(0, splitIdx);
|
||||
const Y_train = Y.slice(0, splitIdx);
|
||||
const X_val = X.slice(splitIdx);
|
||||
const Y_val = Y.slice(splitIdx);
|
||||
|
||||
// 2. Tune lambda
|
||||
const lambdas = [0.01, 0.1, 1.0, 10.0, 100.0, 500.0, 1000.0];
|
||||
let bestLambda = 0;
|
||||
let bestLoss = Infinity;
|
||||
|
||||
console.log(`\nTuning L2 Regularization (Lambda) on Validation Slice...`);
|
||||
for (const l of lambdas) {
|
||||
const w = this._train(X_train, Y_train, l, 0.5, 2000, 1e-5);
|
||||
const loss = computeLogLoss(X_val, Y_val, w);
|
||||
console.log(` Lambda ${l.toString().padEnd(6)} -> LogLoss: ${loss.toFixed(4)}`);
|
||||
if (loss < bestLoss) {
|
||||
bestLoss = loss;
|
||||
bestLambda = l;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Selected Best Lambda: ${bestLambda}`);
|
||||
|
||||
// 3. Refit on FULL TRAIN set using best lambda
|
||||
this.weights = this._train(X, Y, bestLambda, 0.5, 5000, 1e-5);
|
||||
|
||||
return this.getCoefficients();
|
||||
}
|
||||
|
||||
predict(featuresArray) {
|
||||
if (!this.weights) throw new Error("Model not trained");
|
||||
|
||||
const probs = [];
|
||||
for (let i = 0; i < featuresArray.length; i++) {
|
||||
const f = featuresArray[i];
|
||||
let z = this.weights[0]; // intercept
|
||||
for (let j = 0; j < this.featureKeys.length; j++) {
|
||||
z += this.weights[j + 1] * f[this.featureKeys[j]];
|
||||
}
|
||||
probs.push(sigmoid(z));
|
||||
}
|
||||
|
||||
return probs;
|
||||
}
|
||||
|
||||
getCoefficients() {
|
||||
if (!this.weights) return {};
|
||||
const coeffs = { intercept: this.weights[0] };
|
||||
for (let i = 0; i < this.featureKeys.length; i++) {
|
||||
coeffs[this.featureKeys[i]] = this.weights[i + 1];
|
||||
}
|
||||
return coeffs;
|
||||
}
|
||||
|
||||
save(filepath) {
|
||||
fs.writeFileSync(filepath, JSON.stringify({
|
||||
featureKeys: this.featureKeys,
|
||||
weights: this.weights
|
||||
}, null, 2));
|
||||
}
|
||||
|
||||
load(filepath) {
|
||||
if (!fs.existsSync(filepath)) return;
|
||||
const data = JSON.parse(fs.readFileSync(filepath, 'utf8'));
|
||||
this.featureKeys = data.featureKeys;
|
||||
this.weights = data.weights;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
export class StandardScaler {
|
||||
constructor() {
|
||||
this.means = {};
|
||||
this.stds = {};
|
||||
}
|
||||
|
||||
fit(featuresArray) {
|
||||
if (!featuresArray.length) return;
|
||||
|
||||
// Get all feature keys
|
||||
const keys = Object.keys(featuresArray[0]);
|
||||
|
||||
for (const key of keys) {
|
||||
const values = featuresArray.map(f => f[key]);
|
||||
const mean = values.reduce((sum, v) => sum + v, 0) / values.length;
|
||||
|
||||
const variance = values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / values.length;
|
||||
const std = Math.sqrt(variance);
|
||||
|
||||
this.means[key] = mean;
|
||||
this.stds[key] = std || 1; // Avoid divide by 0
|
||||
}
|
||||
}
|
||||
|
||||
transform(features) {
|
||||
const scaled = {};
|
||||
for (const key of Object.keys(this.means)) {
|
||||
if (features[key] === undefined) continue;
|
||||
scaled[key] = (features[key] - this.means[key]) / this.stds[key];
|
||||
}
|
||||
return scaled;
|
||||
}
|
||||
|
||||
transformArray(featuresArray) {
|
||||
return featuresArray.map(f => this.transform(f));
|
||||
}
|
||||
|
||||
save(filepath) {
|
||||
fs.writeFileSync(filepath, JSON.stringify({ means: this.means, stds: this.stds }, null, 2));
|
||||
}
|
||||
|
||||
load(filepath) {
|
||||
if (!fs.existsSync(filepath)) return;
|
||||
const data = JSON.parse(fs.readFileSync(filepath, 'utf8'));
|
||||
this.means = data.means;
|
||||
this.stds = data.stds;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
{
|
||||
"name": "LogisticRegression",
|
||||
"numSteps": 1000,
|
||||
"learningRate": 0.05,
|
||||
"numberClasses": 2,
|
||||
"classifiers": [
|
||||
{
|
||||
"numSteps": 1000,
|
||||
"learningRate": 0.05,
|
||||
"weights": [
|
||||
[
|
||||
-94.4420175451543,
|
||||
-5.766427757134977,
|
||||
-74.50180063587948,
|
||||
-66.23257473781456,
|
||||
-1817.0926888106162,
|
||||
-166.35311896181236
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"numSteps": 1000,
|
||||
"learningRate": 0.05,
|
||||
"weights": [
|
||||
[
|
||||
-84.54027063090014,
|
||||
-50.36483530223822,
|
||||
13.904576636453635,
|
||||
37.27497755449738,
|
||||
-395.0000542460034,
|
||||
41.80818011695695
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
import { swingFeatures } from './features.js';
|
||||
|
||||
function runTest() {
|
||||
console.log("Running no-lookahead guardrail test...");
|
||||
|
||||
// Create mock candles where index = day
|
||||
const candles = [];
|
||||
for (let i = 0; i < 250; i++) {
|
||||
candles.push({
|
||||
date: `2024-01-${i}`,
|
||||
open: 100 + i,
|
||||
high: 105 + i,
|
||||
low: 95 + i,
|
||||
close: 102 + i,
|
||||
volume: 1000000
|
||||
});
|
||||
}
|
||||
|
||||
const spyCandles = [...candles]; // clone
|
||||
const vix = 15;
|
||||
|
||||
// We are at day index 210.
|
||||
const currentDayIdx = 210;
|
||||
|
||||
// Guardrail: Pass ONLY the slice up to current day
|
||||
const slicedCandles = candles.slice(0, currentDayIdx + 1);
|
||||
const slicedSpy = spyCandles.slice(0, currentDayIdx + 1);
|
||||
|
||||
// Compute features
|
||||
const feats = swingFeatures(slicedCandles, slicedSpy, vix);
|
||||
|
||||
if (!feats) {
|
||||
throw new Error("Features returned null unexpectedly.");
|
||||
}
|
||||
|
||||
// The slice literally does not contain any future data in memory.
|
||||
// We can verify the length of the array inside the feature to be absolutely sure.
|
||||
if (slicedCandles.length > currentDayIdx + 1) {
|
||||
throw new Error("Lookahead leak: slicedCandles contains future data!");
|
||||
}
|
||||
|
||||
console.log("PASS: Features successfully computed without lookahead bias.");
|
||||
console.log("Computed Features:", feats);
|
||||
}
|
||||
|
||||
runTest();
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
import { applyCosts } from './costs.js';
|
||||
|
||||
function atr14(candles, period = 14) {
|
||||
if (!candles || candles.length <= period) return null;
|
||||
const trs = [];
|
||||
for (let i = 1; i < candles.length; i++) {
|
||||
const { high, low } = candles[i];
|
||||
const prevClose = candles[i - 1].close;
|
||||
trs.push(Math.max(high - low, Math.abs(high - prevClose), Math.abs(low - prevClose)));
|
||||
}
|
||||
const recent = trs.slice(-period);
|
||||
return recent.reduce((a, b) => a + b, 0) / recent.length;
|
||||
}
|
||||
|
||||
export function simulateSwing(candles, entryIdx, cfg) {
|
||||
// If entry is impossible (out of bounds)
|
||||
if (entryIdx >= candles.length) return null;
|
||||
|
||||
const rawEntry = candles[entryIdx].open;
|
||||
const entry = applyCosts(rawEntry, 'buy', candles, entryIdx);
|
||||
|
||||
// ATR known at signal (close of previous day). candles slice goes UP TO the entryIdx (so it includes the signal bar, which is entryIdx - 1)
|
||||
// Wait, if entryIdx is the open of tomorrow, we pass candles.slice(0, entryIdx) so the last candle is today (the signal bar).
|
||||
const preEntryCandles = candles.slice(0, entryIdx);
|
||||
const atr = atr14(preEntryCandles);
|
||||
if (!atr) return null;
|
||||
|
||||
const stop = entry - cfg.stopAtr * atr;
|
||||
const target = entry + cfg.targetAtr * atr;
|
||||
const R = entry - stop; // 1 ATR worth of price
|
||||
|
||||
const end = Math.min(entryIdx + cfg.maxHoldDays - 1, candles.length - 1);
|
||||
|
||||
for (let d = entryIdx; d <= end; d++) {
|
||||
// pessimistic tie-break: if both hit same day, assume STOP first
|
||||
if (candles[d].low <= stop) return { outcome: 'LOSS', r: -1.0, exitDay: d, stopPrice: stop, targetPrice: target, atr };
|
||||
if (candles[d].high >= target) return { outcome: 'WIN', r: cfg.targetAtr, exitDay: d, stopPrice: stop, targetPrice: target, atr };
|
||||
}
|
||||
|
||||
// unresolved → exit at close of last day, net of exit costs
|
||||
const rawExit = candles[end].close;
|
||||
const exit = applyCosts(rawExit, 'sell', candles, end);
|
||||
return { outcome: 'SCRATCH', r: (exit - entry) / R, exitDay: end, stopPrice: stop, targetPrice: target, atr };
|
||||
}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
export function calibration(testResults) {
|
||||
// We bucket predicted probabilities into deciles (0-10%, 10-20%, etc)
|
||||
const buckets = Array(10).fill(null).map(() => ({ total: 0, wins: 0, sumProb: 0 }));
|
||||
|
||||
for (const r of testResults) {
|
||||
if (r.prob === undefined || r.prob === null) continue;
|
||||
let bIdx = Math.floor(r.prob * 10);
|
||||
if (bIdx > 9) bIdx = 9; // Handle p=1.0 edge case
|
||||
if (bIdx < 0) bIdx = 0;
|
||||
|
||||
buckets[bIdx].total++;
|
||||
buckets[bIdx].sumProb += r.prob;
|
||||
if (r.outcome === 'WIN') buckets[bIdx].wins++;
|
||||
}
|
||||
|
||||
console.log(`\n=== CALIBRATION TABLE ===`);
|
||||
console.log(`Bucket\t\tN\tPred P(Win)\tActual Win%\tDiff`);
|
||||
console.log(`------------------------------------------------------------------`);
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const b = buckets[i];
|
||||
if (b.total === 0) continue;
|
||||
|
||||
const label = `${(i * 10).toString().padStart(2, '0')}-${((i + 1) * 10).toString().padStart(2, '0')}%`;
|
||||
const predP = (b.sumProb / b.total) * 100;
|
||||
const actP = (b.wins / b.total) * 100;
|
||||
const diff = actP - predP;
|
||||
|
||||
console.log(`[${label}]\t${b.total}\t${predP.toFixed(1)}%\t\t${actP.toFixed(1)}%\t\t${diff > 0 ? '+' : ''}${diff.toFixed(1)}pp`);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
export function ciExpectancy(rValues) {
|
||||
const n = rValues.length;
|
||||
if (n === 0) return { mean: 0, lo: 0, hi: 0, n: 0 };
|
||||
|
||||
const mean = rValues.reduce((a, b) => a + b, 0) / n;
|
||||
|
||||
if (n === 1) return { mean, lo: mean, hi: mean, n };
|
||||
|
||||
const variance = rValues.reduce((a, b) => a + Math.pow(b - mean, 2), 0) / (n - 1);
|
||||
const sd = Math.sqrt(variance);
|
||||
const se = sd / Math.sqrt(n);
|
||||
|
||||
return { mean, lo: mean - 1.96 * se, hi: mean + 1.96 * se, n };
|
||||
}
|
||||
|
||||
export function generateResultsTable(testResults) {
|
||||
const grades = ['A', 'B', 'C', 'D'];
|
||||
const grouped = {
|
||||
A: { wins: 0, losses: 0, scratches: 0, rVals: [] },
|
||||
B: { wins: 0, losses: 0, scratches: 0, rVals: [] },
|
||||
C: { wins: 0, losses: 0, scratches: 0, rVals: [] },
|
||||
D: { wins: 0, losses: 0, scratches: 0, rVals: [] }
|
||||
};
|
||||
|
||||
for (const r of testResults) {
|
||||
if (grouped[r.grade]) {
|
||||
grouped[r.grade].rVals.push(r.rMultiple);
|
||||
if (r.outcome === 'WIN') grouped[r.grade].wins++;
|
||||
else if (r.outcome === 'LOSS') grouped[r.grade].losses++;
|
||||
else grouped[r.grade].scratches++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n=== OUT-OF-SAMPLE TEST RESULTS (EXPECTANCY) ===`);
|
||||
console.log(`Grade\tExp(R)\t\t95% CI (Exp)\t\tWin%\tLoss%\tScratch%\tn`);
|
||||
console.log(`-----------------------------------------------------------------------------------------`);
|
||||
|
||||
for (const g of grades) {
|
||||
const st = grouped[g];
|
||||
const n = st.rVals.length;
|
||||
if (n === 0) continue;
|
||||
|
||||
const pWin = (st.wins / n) * 100;
|
||||
const pLoss = (st.losses / n) * 100;
|
||||
const pScratch = (st.scratches / n) * 100;
|
||||
|
||||
const ci = ciExpectancy(st.rVals);
|
||||
|
||||
const meanFmt = `${ci.mean > 0 ? '+' : ''}${ci.mean.toFixed(3)}R`;
|
||||
const ciFmt = `[${ci.lo.toFixed(3)}, ${ci.hi.toFixed(3)}]`;
|
||||
|
||||
console.log(`${g}\t${meanFmt}\t\t${ciFmt}\t\t${pWin.toFixed(1)}%\t${pLoss.toFixed(1)}%\t${pScratch.toFixed(1)}%\t\t${n}`);
|
||||
}
|
||||
|
||||
// Random Control = The entire test set
|
||||
const allR = testResults.map(r => r.rMultiple);
|
||||
const totalN = allR.length;
|
||||
if (totalN > 0) {
|
||||
const totalWins = testResults.filter(r => r.outcome === 'WIN').length;
|
||||
const totalLosses = testResults.filter(r => r.outcome === 'LOSS').length;
|
||||
const totalScratches = testResults.filter(r => r.outcome === 'SCRATCH').length;
|
||||
|
||||
const ciRand = ciExpectancy(allR);
|
||||
const meanFmt = `${ciRand.mean > 0 ? '+' : ''}${ciRand.mean.toFixed(3)}R`;
|
||||
const ciFmt = `[${ciRand.lo.toFixed(3)}, ${ciRand.hi.toFixed(3)}]`;
|
||||
|
||||
const pWin = (totalWins / totalN) * 100;
|
||||
const pLoss = (totalLosses / totalN) * 100;
|
||||
const pScratch = (totalScratches / totalN) * 100;
|
||||
|
||||
console.log(`Rand\t${meanFmt}\t\t${ciFmt}\t\t${pWin.toFixed(1)}%\t${pLoss.toFixed(1)}%\t${pScratch.toFixed(1)}%\t\t${totalN}`);
|
||||
}
|
||||
|
||||
console.log(`-----------------------------------------------------------------------------------------`);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,68 +0,0 @@
|
|||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { parse } from 'csv-parse/sync';
|
||||
import LogisticRegression from 'ml-logistic-regression';
|
||||
import { Matrix } from 'ml-matrix';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
async function train() {
|
||||
console.log('Loading results.csv...');
|
||||
const csvPath = path.join(__dirname, 'results.csv');
|
||||
const fileContent = fs.readFileSync(csvPath, 'utf8');
|
||||
|
||||
const records = parse(fileContent, {
|
||||
columns: true,
|
||||
skip_empty_lines: true
|
||||
});
|
||||
|
||||
const X = [];
|
||||
const Y = [];
|
||||
|
||||
for (const row of records) {
|
||||
if (row.type !== 'signal') continue; // Skip controls if they exist
|
||||
|
||||
// Parse features
|
||||
const rsi = parseFloat(row.rsi);
|
||||
const atrPct = parseFloat(row.atrPct);
|
||||
const rvol = parseFloat(row.rvol);
|
||||
const macd = parseFloat(row.macd);
|
||||
const regime = parseFloat(row.regime_risk_on);
|
||||
|
||||
// Skip rows with missing data
|
||||
if (isNaN(rsi) || isNaN(atrPct) || isNaN(rvol) || isNaN(macd) || isNaN(regime)) continue;
|
||||
|
||||
X.push([
|
||||
1.0, // Intercept
|
||||
rsi / 100,
|
||||
atrPct / 10,
|
||||
rvol / 5,
|
||||
macd / 2,
|
||||
regime
|
||||
]);
|
||||
|
||||
// Target: 1 if win or win_partial, 0 otherwise
|
||||
const isWin = (row.outcome === 'win' || row.outcome === 'win_partial') ? 1 : 0;
|
||||
Y.push(isWin);
|
||||
}
|
||||
|
||||
console.log(`Loaded ${X.length} valid training samples.`);
|
||||
|
||||
const xMat = new Matrix(X);
|
||||
const yMat = Matrix.columnVector(Y);
|
||||
|
||||
console.log('Training Logistic Regression model...');
|
||||
const logreg = new LogisticRegression({ numSteps: 1000, learningRate: 0.05 });
|
||||
logreg.train(xMat, yMat);
|
||||
|
||||
const modelJson = logreg.toJSON();
|
||||
|
||||
const outputPath = path.join(__dirname, 'model_weights.json');
|
||||
fs.writeFileSync(outputPath, JSON.stringify(modelJson, null, 2));
|
||||
|
||||
console.log(`Model trained and saved to ${outputPath}`);
|
||||
}
|
||||
|
||||
train().catch(console.error);
|
||||
|
|
@ -1,60 +0,0 @@
|
|||
import { loadFundamentals, getFundamentalsAsOf } from '../fundamentals.js';
|
||||
|
||||
function runTest() {
|
||||
console.log("Running fundamentals lookahead test...");
|
||||
|
||||
const testData = [
|
||||
{
|
||||
symbol: 'AAPL',
|
||||
datekey: '2020-02-15', // Actually filed on Feb 15
|
||||
fiscalEnd: '2019-12-31',
|
||||
metrics: { roe: 0.10 }
|
||||
},
|
||||
{
|
||||
symbol: 'AAPL',
|
||||
datekey: '2020-05-15', // Q1 filed on May 15
|
||||
fiscalEnd: '2020-03-31',
|
||||
metrics: { roe: 0.12 }
|
||||
},
|
||||
{
|
||||
symbol: 'MSFT',
|
||||
datekey: null, // Missing datekey, should fallback to fiscalEnd + 120 days
|
||||
fiscalEnd: '2019-12-31', // + 120 days = 2020-04-29
|
||||
metrics: { roe: 0.15 }
|
||||
}
|
||||
];
|
||||
|
||||
loadFundamentals(testData);
|
||||
|
||||
// Test 1: Jan 31, 2020.
|
||||
// The Dec 31 fiscal data isn't filed until Feb 15. We MUST return null.
|
||||
const a1 = getFundamentalsAsOf('AAPL', '2020-01-31');
|
||||
if (a1 !== null) throw new Error("Lookahead leak! Returned Q4 data before filing date.");
|
||||
|
||||
// Test 2: Feb 16, 2020.
|
||||
// The Dec 31 fiscal data was filed Feb 15. We should get it.
|
||||
const a2 = getFundamentalsAsOf('AAPL', '2020-02-16');
|
||||
if (!a2 || a2.roe !== 0.10) throw new Error("Failed to return available data.");
|
||||
|
||||
// Test 3: May 14, 2020.
|
||||
// Q1 isn't filed until May 15. We should still get Q4 (roe: 0.10).
|
||||
const a3 = getFundamentalsAsOf('AAPL', '2020-05-14');
|
||||
if (!a3 || a3.roe !== 0.10) throw new Error("Lookahead leak! Returned Q1 data early.");
|
||||
|
||||
// Test 4: May 16, 2020.
|
||||
// Now we should get Q1 (roe: 0.12).
|
||||
const a4 = getFundamentalsAsOf('AAPL', '2020-05-16');
|
||||
if (!a4 || a4.roe !== 0.12) throw new Error("Failed to roll forward to new data.");
|
||||
|
||||
// Test 5: MSFT fallback logic. Target = April 28 (119 days after fiscalEnd). Should be null.
|
||||
const m1 = getFundamentalsAsOf('MSFT', '2020-04-28');
|
||||
if (m1 !== null) throw new Error("Lookahead leak! MSFT fallback data returned early.");
|
||||
|
||||
// Test 6: MSFT fallback logic. Target = April 30 (121 days after fiscalEnd). Should be available.
|
||||
const m2 = getFundamentalsAsOf('MSFT', '2020-04-30');
|
||||
if (!m2 || m2.roe !== 0.15) throw new Error("Failed to return fallback data after 120 days.");
|
||||
|
||||
console.log("PASS: Fundamentals guardrail strictly prevents lookahead bias.");
|
||||
}
|
||||
|
||||
runTest();
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
import { buildFactorProfiles } from '../descriptiveLens.js';
|
||||
|
||||
test('strictly-better stock ranks higher on every factor', async () => {
|
||||
const good = {
|
||||
ticker: 'GOOD',
|
||||
trailingPE: 8,
|
||||
priceToBook: 1,
|
||||
priceToSales: 0.5,
|
||||
freeCashflow: 1e10,
|
||||
marketCap: 1e11,
|
||||
returnOnEquity: 0.30,
|
||||
grossMargins: 0.60,
|
||||
debtToEquity: 10,
|
||||
mom_12_1: 0.25,
|
||||
realizedVol252: 0.15
|
||||
};
|
||||
|
||||
const bad = {
|
||||
ticker: 'BAD',
|
||||
trailingPE: 50,
|
||||
priceToBook: 10,
|
||||
priceToSales: 12,
|
||||
freeCashflow: 1e8,
|
||||
marketCap: 1e11,
|
||||
returnOnEquity: 0.02,
|
||||
grossMargins: 0.10,
|
||||
debtToEquity: 300,
|
||||
mom_12_1: -0.10,
|
||||
realizedVol252: 0.60
|
||||
};
|
||||
|
||||
// buildFactorProfiles supports injecting a universe payload directly for testing
|
||||
const out = await buildFactorProfiles([good, bad]);
|
||||
|
||||
// 'out' returns a map { 'GOOD': { ... }, 'BAD': { ... } } during test injection
|
||||
|
||||
for (const f of ['value', 'quality', 'lowVol', 'momentum', 'composite']) {
|
||||
// A strictly better stock MUST have a higher percentile rank
|
||||
if (out['GOOD'][f] <= out['BAD'][f]) {
|
||||
throw new Error(`Sign catastrophe: GOOD ranked ${out['GOOD'][f]} on ${f}, but BAD ranked ${out['BAD'][f]}. GOOD must strictly outrank BAD.`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("PASS: Orientation logic correctly handles signs. 'Higher percentile' always means 'more favorable'.");
|
||||
});
|
||||
|
||||
// Jest polyfill for simple node execution
|
||||
function test(name, fn) {
|
||||
console.log(`Running test: ${name}`);
|
||||
fn().catch(e => {
|
||||
console.error(`FAIL: ${e.message}`);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
|
@ -1,142 +0,0 @@
|
|||
import { getUniverseAsOf } from './universe.js';
|
||||
import { getFundamentalsAsOf } from './fundamentals.js';
|
||||
import { standardizeCrossSection, FACTORS } from './factors.js';
|
||||
import { spearman, calculateTurnover } from './metrics.js';
|
||||
|
||||
// Global reference for forward returns in our testing environment
|
||||
// In production, this would query a price DB
|
||||
let globalReturnsMap = new Map();
|
||||
|
||||
export function setForwardReturnsMap(map) {
|
||||
globalReturnsMap = map;
|
||||
}
|
||||
|
||||
function getForwardReturn(ticker, dateStr, horizonMonths, allDates) {
|
||||
// Find the index of the current date
|
||||
const idx = allDates.indexOf(dateStr);
|
||||
if (idx === -1 || idx + horizonMonths >= allDates.length) return null; // Not enough forward data
|
||||
|
||||
let cumulativeRet = 0;
|
||||
// Simple sum for log returns, or compound. We'll use simple compound: (1+r1)*(1+r2) - 1
|
||||
let mult = 1.0;
|
||||
for (let m = 0; m < horizonMonths; m++) {
|
||||
const nextDate = allDates[idx + m];
|
||||
const r = globalReturnsMap.get(`${ticker}_${nextDate}`);
|
||||
if (r === undefined || r === null) return null; // missing data
|
||||
mult *= (1 + r);
|
||||
}
|
||||
return mult - 1.0;
|
||||
}
|
||||
|
||||
function bucketIntoDeciles(rankedScores) {
|
||||
const deciles = Array(10).fill(null).map(() => []);
|
||||
const n = rankedScores.length;
|
||||
if (n === 0) return deciles;
|
||||
|
||||
for (let i = 0; i < n; i++) {
|
||||
const d = Math.min(9, Math.floor((i / n) * 10));
|
||||
deciles[d].push(rankedScores[i].ticker);
|
||||
}
|
||||
return deciles; // deciles[0] = highest scores (Top Decile), deciles[9] = lowest scores
|
||||
}
|
||||
|
||||
export function runFactor(factorName, allDates, horizonMonths = 1, costPerSideBps = 5) {
|
||||
const results = [];
|
||||
let prevD10Weights = null;
|
||||
let prevD1Weights = null;
|
||||
const costPct = costPerSideBps / 10000.0;
|
||||
|
||||
for (const date of allDates) {
|
||||
const members = getUniverseAsOf(date);
|
||||
if (!members || members.length === 0) continue;
|
||||
|
||||
const rawScores = {};
|
||||
for (const t of members) {
|
||||
const fund = getFundamentalsAsOf(t, date);
|
||||
if (!fund) continue;
|
||||
|
||||
const score = FACTORS[factorName](fund);
|
||||
if (score !== null && !isNaN(score)) {
|
||||
rawScores[t] = score;
|
||||
}
|
||||
}
|
||||
|
||||
// Standardize cross-sectionally
|
||||
const zScores = standardizeCrossSection(rawScores, false);
|
||||
|
||||
// Sort descending (High Score = Best)
|
||||
const ranked = Object.keys(zScores)
|
||||
.map(t => ({ ticker: t, score: zScores[t] }))
|
||||
.sort((a, b) => b.score - a.score);
|
||||
|
||||
if (ranked.length < 10) continue; // need enough names for deciles
|
||||
|
||||
const deciles = bucketIntoDeciles(ranked);
|
||||
const d10Tickers = deciles[0]; // Top decile
|
||||
const d1Tickers = deciles[9]; // Bottom decile
|
||||
|
||||
// Calculate equal-weight portfolio weights for this month
|
||||
const curD10Weights = {};
|
||||
d10Tickers.forEach(t => curD10Weights[t] = 1.0 / d10Tickers.length);
|
||||
|
||||
const curD1Weights = {};
|
||||
d1Tickers.forEach(t => curD1Weights[t] = 1.0 / d1Tickers.length);
|
||||
|
||||
// Calculate turnover
|
||||
const d10Turnover = calculateTurnover(prevD10Weights, curD10Weights);
|
||||
const d1Turnover = calculateTurnover(prevD1Weights, curD1Weights);
|
||||
|
||||
// Forward returns
|
||||
const decileReturns = [];
|
||||
let allFwdReturns = [];
|
||||
let allZScores = [];
|
||||
|
||||
for (let d = 0; d < 10; d++) {
|
||||
const decTickers = deciles[d];
|
||||
let sumRet = 0;
|
||||
let count = 0;
|
||||
for (const t of decTickers) {
|
||||
const ret = getForwardReturn(t, date, horizonMonths, allDates);
|
||||
if (ret !== null) {
|
||||
sumRet += ret;
|
||||
count++;
|
||||
|
||||
allFwdReturns.push(ret);
|
||||
allZScores.push(zScores[t]);
|
||||
}
|
||||
}
|
||||
decileReturns.push(count > 0 ? sumRet / count : 0);
|
||||
}
|
||||
|
||||
if (allFwdReturns.length > 0) {
|
||||
const ic = spearman(allZScores, allFwdReturns);
|
||||
const universeMeanRet = allFwdReturns.reduce((a, b) => a + b, 0) / allFwdReturns.length;
|
||||
|
||||
// Gross Returns
|
||||
const d10Gross = decileReturns[0];
|
||||
const d1Gross = decileReturns[9];
|
||||
|
||||
// Net Returns
|
||||
const d10Net = d10Gross - (d10Turnover * costPct);
|
||||
const d1Net = d1Gross - (d1Turnover * costPct);
|
||||
|
||||
results.push({
|
||||
date,
|
||||
ic,
|
||||
decileReturns, // Gross decile returns for the staircase plot
|
||||
d10Gross,
|
||||
d1Gross,
|
||||
d10Net,
|
||||
d1Net,
|
||||
universeMeanRet,
|
||||
d10Turnover,
|
||||
d1Turnover
|
||||
});
|
||||
}
|
||||
|
||||
prevD10Weights = curD10Weights;
|
||||
prevD1Weights = curD1Weights;
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
|
@ -1,228 +0,0 @@
|
|||
import { FACTORS, standardizeCrossSection } from './factors.js';
|
||||
import { pullRawRecord } from './ingest.js';
|
||||
import { getUniverse } from '../services/stockUniverseService.js';
|
||||
|
||||
/**
|
||||
* Assigns percentiles (0-100) to a raw factor array using the standardizer.
|
||||
* @param {Array} raw - Array of objects: { ticker, value: 0.5, quality: 1.2 }
|
||||
* @param {String} factorKey - The factor to assign percentiles for (e.g. 'value')
|
||||
*/
|
||||
function assignPercentiles(raw, factorKey) {
|
||||
const rawByTicker = {};
|
||||
for (const r of raw) {
|
||||
if (r[factorKey] !== null && r[factorKey] !== undefined) {
|
||||
rawByTicker[r.ticker] = r[factorKey];
|
||||
}
|
||||
}
|
||||
|
||||
// Uses winsorization and z-scoring from factors.js
|
||||
const zScores = standardizeCrossSection(rawByTicker, false);
|
||||
|
||||
// Convert z-scores to percentiles relative to the universe
|
||||
const validTickers = Object.keys(zScores);
|
||||
const sortedZScores = validTickers.map(t => zScores[t]).sort((a, b) => a - b);
|
||||
|
||||
const n = sortedZScores.length;
|
||||
|
||||
for (const r of raw) {
|
||||
const z = zScores[r.ticker];
|
||||
if (z === undefined) {
|
||||
r[`${factorKey}Percentile`] = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Find percentile rank
|
||||
let index = sortedZScores.findIndex(val => val >= z);
|
||||
if (index === -1) index = n - 1;
|
||||
|
||||
r[`${factorKey}Percentile`] = Math.round((index / n) * 100);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates the descriptive Factor Profile for the entire universe based on CURRENT data.
|
||||
* Does NOT generate forward predictions.
|
||||
*/
|
||||
let _cachedRawProfiles = null;
|
||||
let _cachedZScoreMaps = null;
|
||||
let _cachedRawTime = 0;
|
||||
|
||||
export async function buildFactorProfiles(injectedUniverseRecords = null) {
|
||||
const rawProfiles = [];
|
||||
|
||||
// Allow test injection
|
||||
if (injectedUniverseRecords) {
|
||||
for (const data of injectedUniverseRecords) {
|
||||
rawProfiles.push({
|
||||
ticker: data.ticker || 'TEST',
|
||||
value: FACTORS.value(data),
|
||||
quality: FACTORS.quality(data),
|
||||
lowVol: FACTORS.lowVol(data),
|
||||
momentum: FACTORS.momentum(data)
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// We now include ETFs in the factor ranking because we want to see momentum/value on country ETFs
|
||||
const universe = getUniverse().map(s => s.symbol);
|
||||
const BATCH_SIZE = 15;
|
||||
|
||||
for (let i = 0; i < universe.length; i += BATCH_SIZE) {
|
||||
const batch = universe.slice(i, i + BATCH_SIZE);
|
||||
const promises = batch.map(async (ticker) => {
|
||||
try {
|
||||
const data = await pullRawRecord(ticker);
|
||||
if (!data) return null;
|
||||
|
||||
return {
|
||||
ticker,
|
||||
value: FACTORS.value(data),
|
||||
quality: FACTORS.quality(data),
|
||||
lowVol: FACTORS.lowVol(data),
|
||||
momentum: FACTORS.momentum(data)
|
||||
};
|
||||
} catch (e) {
|
||||
console.warn(`[DescriptiveLens] Failed for ${ticker}:`, e.message);
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
const results = await Promise.allSettled(promises);
|
||||
for (const res of results) {
|
||||
if (res.status === 'fulfilled' && res.value) {
|
||||
rawProfiles.push(res.value);
|
||||
}
|
||||
}
|
||||
|
||||
// Throttle delay between batches to respect free API limits
|
||||
if (i + BATCH_SIZE < universe.length) {
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Composite is the equal-weight average of the non-null standardized z-scores
|
||||
// First, we must standardize the individual factors to compute composite z-score
|
||||
const factors = ['value', 'quality', 'lowVol', 'momentum'];
|
||||
const zScoreMaps = {};
|
||||
|
||||
for (const f of factors) {
|
||||
const rawByTicker = {};
|
||||
for (const r of rawProfiles) {
|
||||
if (r[f] !== null && r[f] !== undefined) rawByTicker[r.ticker] = r[f];
|
||||
}
|
||||
zScoreMaps[f] = standardizeCrossSection(rawByTicker, false);
|
||||
}
|
||||
|
||||
// Compute composite raw score
|
||||
for (const r of rawProfiles) {
|
||||
let sum = 0;
|
||||
let count = 0;
|
||||
for (const f of factors) {
|
||||
const z = zScoreMaps[f][r.ticker];
|
||||
if (z !== undefined) {
|
||||
sum += z;
|
||||
count++;
|
||||
}
|
||||
}
|
||||
r.composite = count > 0 ? (sum / count) : null;
|
||||
}
|
||||
|
||||
// Cross-sectional standardization and percentile assignment
|
||||
[...factors, 'composite'].forEach(f => assignPercentiles(rawProfiles, f));
|
||||
|
||||
// Cache the raw profiles for single-stock comparisons
|
||||
if (!injectedUniverseRecords) {
|
||||
_cachedRawProfiles = rawProfiles;
|
||||
_cachedZScoreMaps = zScoreMaps;
|
||||
_cachedRawTime = Date.now();
|
||||
}
|
||||
|
||||
// If this is a test injection, return the raw profiles with percentiles for testing
|
||||
if (injectedUniverseRecords) return rawProfiles.reduce((acc, p) => { acc[p.ticker] = p; return acc; }, {});
|
||||
|
||||
// Clean up the raw scores, we only return the percentiles for the UI
|
||||
const cleanedProfiles = rawProfiles.map(p => ({
|
||||
ticker: p.ticker,
|
||||
value: p.valuePercentile,
|
||||
quality: p.qualityPercentile,
|
||||
lowVol: p.lowVolPercentile,
|
||||
momentum: p.momentumPercentile,
|
||||
composite: p.compositePercentile,
|
||||
_disclaimer: "Descriptive only. Shows where this stock ranks vs. peers on each factor today. We have not validated that these ranks predict returns — and our own testing showed standard technical signals do not. We'll only call a factor predictive after it passes out-of-sample validation."
|
||||
}));
|
||||
|
||||
return cleanedProfiles;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes a single stock's factor profile relative to the cached universe.
|
||||
*/
|
||||
export async function getSingleFactorProfile(symbol) {
|
||||
if (!_cachedRawProfiles || (Date.now() - _cachedRawTime > 120000)) {
|
||||
await buildFactorProfiles();
|
||||
}
|
||||
|
||||
// If it's already in the universe, just return it
|
||||
const existing = _cachedRawProfiles.find(p => p.ticker === symbol);
|
||||
if (existing) {
|
||||
return {
|
||||
ticker: existing.ticker,
|
||||
value: existing.valuePercentile,
|
||||
quality: existing.qualityPercentile,
|
||||
lowVol: existing.lowVolPercentile,
|
||||
momentum: existing.momentumPercentile,
|
||||
composite: existing.compositePercentile,
|
||||
_disclaimer: "Descriptive only. Shows where this stock ranks vs. the core universe."
|
||||
};
|
||||
}
|
||||
|
||||
// Fetch out-of-universe data
|
||||
const data = await pullRawRecord(symbol);
|
||||
if (!data) return null;
|
||||
|
||||
const raw = {
|
||||
ticker: symbol,
|
||||
value: FACTORS.value(data),
|
||||
quality: FACTORS.quality(data),
|
||||
lowVol: FACTORS.lowVol(data),
|
||||
momentum: FACTORS.momentum(data)
|
||||
};
|
||||
|
||||
// Standardize single value using universe mean/std from zScoreMaps... wait, standardizer only gives cross-sectional z-scores.
|
||||
// We can just re-run standardizer with the universe + this one stock.
|
||||
const tempRaw = [..._cachedRawProfiles, raw];
|
||||
|
||||
const factors = ['value', 'quality', 'lowVol', 'momentum'];
|
||||
const zScoreMaps = {};
|
||||
|
||||
for (const f of factors) {
|
||||
const rawByTicker = {};
|
||||
for (const r of tempRaw) {
|
||||
if (r[f] !== null && r[f] !== undefined) rawByTicker[r.ticker] = r[f];
|
||||
}
|
||||
zScoreMaps[f] = standardizeCrossSection(rawByTicker, false);
|
||||
}
|
||||
|
||||
let sum = 0;
|
||||
let count = 0;
|
||||
for (const f of factors) {
|
||||
const z = zScoreMaps[f][raw.ticker];
|
||||
if (z !== undefined) {
|
||||
sum += z;
|
||||
count++;
|
||||
}
|
||||
}
|
||||
raw.composite = count > 0 ? (sum / count) : null;
|
||||
|
||||
[...factors, 'composite'].forEach(f => assignPercentiles(tempRaw, f));
|
||||
|
||||
return {
|
||||
ticker: raw.ticker,
|
||||
value: raw.valuePercentile,
|
||||
quality: raw.qualityPercentile,
|
||||
lowVol: raw.lowVolPercentile,
|
||||
momentum: raw.momentumPercentile,
|
||||
composite: raw.compositePercentile,
|
||||
_disclaimer: "Descriptive only. Shows where this out-of-universe stock ranks vs. the core universe today."
|
||||
};
|
||||
}
|
||||
|
|
@ -1,106 +0,0 @@
|
|||
function mean(arr) {
|
||||
const valid = arr.filter(x => x !== null && x !== undefined && !isNaN(x));
|
||||
if (valid.length === 0) return null;
|
||||
return valid.reduce((a, b) => a + b, 0) / valid.length;
|
||||
}
|
||||
|
||||
export function orientedComponents(r) {
|
||||
const inv = x => (x != null && x > 0) ? 1 / x : null; // guard neg/zero multiples
|
||||
const neg = x => (x != null) ? -x : null;
|
||||
const div = (a, b) => (a != null && b) ? a / b : null;
|
||||
return {
|
||||
value: [ inv(r.trailingPE), div(r.freeCashflow, r.marketCap),
|
||||
inv(r.priceToBook), inv(r.priceToSales) ],
|
||||
quality: [ r.returnOnEquity, r.grossMargins, neg(r.debtToEquity) ],
|
||||
lowVol: [ neg(r.realizedVol252) ],
|
||||
momentum: [ r.mom_12_1 ],
|
||||
};
|
||||
}
|
||||
|
||||
export const FACTORS = {
|
||||
value: r => mean(orientedComponents(r).value),
|
||||
quality: r => mean(orientedComponents(r).quality),
|
||||
lowVol: r => mean(orientedComponents(r).lowVol),
|
||||
momentum: r => mean(orientedComponents(r).momentum),
|
||||
// composite is an equal-weight average of the standardized scores, computed later
|
||||
};
|
||||
|
||||
function winsorize(arr, trimPercent) {
|
||||
if (arr.length < 5) return arr; // Don't winsorize tiny arrays
|
||||
const sorted = [...arr].sort((a, b) => a - b);
|
||||
const lowerIdx = Math.floor(arr.length * trimPercent);
|
||||
let upperIdx = Math.floor(arr.length * (1 - trimPercent)) - 1;
|
||||
if (upperIdx < lowerIdx) upperIdx = arr.length - 1;
|
||||
|
||||
const lowerBound = sorted[lowerIdx];
|
||||
const upperBound = sorted[upperIdx];
|
||||
|
||||
return arr.map(v => {
|
||||
if (v < lowerBound) return lowerBound;
|
||||
if (v > upperBound) return upperBound;
|
||||
return v;
|
||||
});
|
||||
}
|
||||
|
||||
function avg(arr) {
|
||||
if (!arr.length) return 0;
|
||||
return arr.reduce((a, b) => a + b, 0) / arr.length;
|
||||
}
|
||||
|
||||
function std(arr, meanVal) {
|
||||
if (arr.length <= 1) return 1;
|
||||
const variance = arr.reduce((sum, v) => sum + Math.pow(v - meanVal, 2), 0) / (arr.length - 1);
|
||||
return Math.sqrt(variance) || 1;
|
||||
}
|
||||
|
||||
export function standardizeCrossSection(rawByTicker, neutralizeSector = false, sectorsByTicker = {}) {
|
||||
// Extract non-null values
|
||||
const tickers = Object.keys(rawByTicker);
|
||||
let validTickers = tickers.filter(t => rawByTicker[t] != null && !isNaN(rawByTicker[t]));
|
||||
|
||||
if (validTickers.length === 0) return {};
|
||||
|
||||
const result = {};
|
||||
|
||||
if (neutralizeSector) {
|
||||
// Group by sector
|
||||
const sectorGroups = {};
|
||||
for (const t of validTickers) {
|
||||
const sec = sectorsByTicker[t] || 'UNKNOWN';
|
||||
if (!sectorGroups[sec]) sectorGroups[sec] = [];
|
||||
sectorGroups[sec].push(t);
|
||||
}
|
||||
|
||||
for (const sec in sectorGroups) {
|
||||
const secTickers = sectorGroups[sec];
|
||||
const secVals = secTickers.map(t => rawByTicker[t]);
|
||||
|
||||
const w = winsorize(secVals, 0.02);
|
||||
const m = avg(w);
|
||||
const s = std(w, m);
|
||||
|
||||
for (let i = 0; i < secTickers.length; i++) {
|
||||
const t = secTickers[i];
|
||||
let val = rawByTicker[t];
|
||||
if (val < w[0]) val = w[0]; // approx winsorize clip (since w is same order)
|
||||
// Actually, winsorize maps index-to-index.
|
||||
const clipped = w[i];
|
||||
result[t] = (clipped - m) / s;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Global standard
|
||||
const vals = validTickers.map(t => rawByTicker[t]);
|
||||
const w = winsorize(vals, 0.02);
|
||||
const m = avg(w);
|
||||
const s = std(w, m);
|
||||
|
||||
for (let i = 0; i < validTickers.length; i++) {
|
||||
const t = validTickers[i];
|
||||
const clipped = w[i];
|
||||
result[t] = (clipped - m) / s;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
// In a real setup, this data would be loaded from Sharadar/Compustat.
|
||||
// For testing, we allow an injected dataset.
|
||||
|
||||
let globalFundamentalsBySymbol = new Map();
|
||||
|
||||
export function loadFundamentals(data) {
|
||||
globalFundamentalsBySymbol.clear();
|
||||
// Group by symbol
|
||||
for (const record of data) {
|
||||
if (!globalFundamentalsBySymbol.has(record.symbol)) {
|
||||
globalFundamentalsBySymbol.set(record.symbol, []);
|
||||
}
|
||||
globalFundamentalsBySymbol.get(record.symbol).push(record);
|
||||
}
|
||||
|
||||
// Sort each group by datekey ascending
|
||||
for (const [sym, records] of globalFundamentalsBySymbol.entries()) {
|
||||
records.sort((a, b) => new Date(a.datekey) - new Date(b.datekey));
|
||||
}
|
||||
}
|
||||
|
||||
export function getFundamentalsAsOf(symbol, dateStr) {
|
||||
const targetDate = new Date(dateStr);
|
||||
const records = globalFundamentalsBySymbol.get(symbol);
|
||||
if (!records) return null;
|
||||
|
||||
// Find the most recent fundamental report where filing date <= targetDate
|
||||
let latest = null;
|
||||
for (const record of records) {
|
||||
|
||||
// GUARDRAIL: We absolutely require a filing date (datekey).
|
||||
// If datekey <= targetDate, it's safe.
|
||||
// If datekey is missing, fallback to fiscalEnd + 4 months (120 days).
|
||||
let isAvailable = false;
|
||||
if (record.datekey) {
|
||||
if (new Date(record.datekey) <= targetDate) isAvailable = true;
|
||||
} else if (record.fiscalEnd) {
|
||||
const safeReleaseDate = new Date(record.fiscalEnd);
|
||||
safeReleaseDate.setDate(safeReleaseDate.getDate() + 120); // Add 4 months roughly
|
||||
if (safeReleaseDate <= targetDate) isAvailable = true;
|
||||
} else {
|
||||
throw new Error(`Record for ${symbol} lacks both datekey and fiscalEnd. Unsafe.`);
|
||||
}
|
||||
|
||||
if (isAvailable) {
|
||||
latest = record;
|
||||
} else {
|
||||
if (record.datekey && new Date(record.datekey) > targetDate) break;
|
||||
}
|
||||
}
|
||||
|
||||
return latest ? latest.metrics : null;
|
||||
}
|
||||
|
|
@ -1,98 +0,0 @@
|
|||
import { generateSyntheticData } from './synthetic.js';
|
||||
import { loadUniverse } from './universe.js';
|
||||
import { loadFundamentals } from './fundamentals.js';
|
||||
import { setForwardReturnsMap, runFactor } from './crossSection.js';
|
||||
import { mean, neweyWestStdErr } from './metrics.js';
|
||||
|
||||
function printReport(title, results, horizonMonths) {
|
||||
if (results.length === 0) {
|
||||
console.log(`\n=== ${title} ===`);
|
||||
console.log("No valid results.");
|
||||
return;
|
||||
}
|
||||
|
||||
const ics = results.map(r => r.ic);
|
||||
const meanIc = mean(ics);
|
||||
// We use Newey-West standard error for the t-stat if overlapping.
|
||||
// For 1-month horizon, lags = 0 (simple std err).
|
||||
// For H-month horizon, lags = H - 1.
|
||||
const lags = Math.max(0, horizonMonths - 1);
|
||||
const seIc = neweyWestStdErr(ics, lags);
|
||||
const tStatIc = meanIc / seIc;
|
||||
|
||||
// Average Decile Returns (Gross)
|
||||
const deciles = Array(10).fill(0);
|
||||
for (const r of results) {
|
||||
for (let d = 0; d < 10; d++) {
|
||||
deciles[d] += r.decileReturns[d];
|
||||
}
|
||||
}
|
||||
for (let d = 0; d < 10; d++) deciles[d] /= results.length;
|
||||
|
||||
// Long/Short Spread (Net)
|
||||
const lsSpreads = results.map(r => r.d10Net - r.d1Net);
|
||||
const meanLsSpread = mean(lsSpreads);
|
||||
const seLsSpread = neweyWestStdErr(lsSpreads, lags);
|
||||
const tStatLs = meanLsSpread / seLsSpread;
|
||||
const sharpeLs = (meanLsSpread / (seLsSpread * Math.sqrt(results.length))) * Math.sqrt(12 / horizonMonths); // Approx Annualized Gross Sharpe
|
||||
|
||||
// Long-Only Top-Decile Excess (Net)
|
||||
const loExcess = results.map(r => r.d10Net - r.universeMeanRet);
|
||||
const meanLoExcess = mean(loExcess);
|
||||
|
||||
// Consistency
|
||||
const numPositiveLs = lsSpreads.filter(s => s > 0).length;
|
||||
const consistencyPct = (numPositiveLs / lsSpreads.length) * 100;
|
||||
|
||||
// Turnover (Avg over periods)
|
||||
const avgTurnoverD10 = mean(results.map(r => r.d10Turnover)) * 100;
|
||||
|
||||
console.log(`\n=== ${title} ===`);
|
||||
console.log(`Periods (N): ${results.length} months`);
|
||||
console.log(`Horizon: ${horizonMonths} month(s)`);
|
||||
console.log(`Mean IC: ${meanIc.toFixed(4)}`);
|
||||
console.log(`IC t-stat: ${tStatIc.toFixed(2)} (Target: >= 3.0)`);
|
||||
console.log(`L/S Spread (Net): ${(meanLsSpread * 100).toFixed(2)}% per period (t-stat: ${tStatLs.toFixed(2)})`);
|
||||
console.log(`Long-Only Excess: ${(meanLoExcess * 100).toFixed(2)}% per period`);
|
||||
console.log(`Consistency: ${consistencyPct.toFixed(1)}% positive periods`);
|
||||
console.log(`D10 Avg Turnover: ${avgTurnoverD10.toFixed(1)}% per rebalance`);
|
||||
|
||||
console.log(`\nDecile Monotonicity (Gross):`);
|
||||
const maxRet = Math.max(...deciles);
|
||||
const minRet = Math.min(...deciles);
|
||||
for (let d = 0; d < 10; d++) {
|
||||
// simple text bar chart
|
||||
const pct = deciles[d] * 100;
|
||||
const barLen = Math.max(1, Math.floor((deciles[d] - minRet) / (maxRet - minRet + 0.0001) * 20));
|
||||
console.log(` D${d+1}: ${pct.toFixed(2).padStart(5)}% | ${'#'.repeat(barLen)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function runSyntheticTests() {
|
||||
console.log("==================================================");
|
||||
console.log(" SYNTHETIC KNOWN-ANSWER TESTS");
|
||||
console.log("==================================================");
|
||||
|
||||
// Generate 25 years (300 months) of data for 500 stocks
|
||||
// 1. NULL TEST (IC = 0)
|
||||
const nullData = generateSyntheticData({ numMonths: 300, numStocks: 500, plantedIC: 0.0 });
|
||||
loadUniverse(nullData.universe);
|
||||
loadFundamentals(nullData.fundamentals);
|
||||
setForwardReturnsMap(nullData.forwardReturnsMap);
|
||||
|
||||
const allDatesNull = nullData.universe.map(u => u.date);
|
||||
const resultsNull = runFactor('value', allDatesNull, 1, 5); // 5 bps costs
|
||||
printReport("TEST 1: NULL FACTOR (Target: IC ≈ 0, t-stat < 3)", resultsNull, 1);
|
||||
|
||||
// 2. PLANTED SIGNAL TEST (IC = 0.05)
|
||||
const plantedData = generateSyntheticData({ numMonths: 300, numStocks: 500, plantedIC: 0.05 });
|
||||
loadUniverse(plantedData.universe);
|
||||
loadFundamentals(plantedData.fundamentals);
|
||||
setForwardReturnsMap(plantedData.forwardReturnsMap);
|
||||
|
||||
const allDatesPlanted = plantedData.universe.map(u => u.date);
|
||||
const resultsPlanted = runFactor('value', allDatesPlanted, 1, 5); // 5 bps costs
|
||||
printReport("TEST 2: PLANTED SIGNAL (Target: IC ≈ 0.05, t-stat >= 3, Monotonic)", resultsPlanted, 1);
|
||||
}
|
||||
|
||||
runSyntheticTests();
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
import { fetchQuoteSummary } from '../services/fundamentalsService.js';
|
||||
|
||||
const DEFAULT_HEADERS = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Accept': 'application/json',
|
||||
};
|
||||
|
||||
async function fetchHistory(symbol) {
|
||||
const url = `https://query1.finance.yahoo.com/v8/finance/chart/${symbol}?range=2y&interval=1d`;
|
||||
try {
|
||||
const res = await fetch(url, { headers: DEFAULT_HEADERS });
|
||||
if (!res.ok) return null;
|
||||
const data = await res.json();
|
||||
const result = data?.chart?.result?.[0];
|
||||
if (!result) return null;
|
||||
return result.indicators?.quote?.[0]?.close || [];
|
||||
} catch (e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function momentum12_1(closes) {
|
||||
if (!closes || closes.length < 252) return null;
|
||||
const n = closes.length;
|
||||
// 12-1 month: close[-21] / close[-252] - 1
|
||||
const current = closes[n - 21];
|
||||
const past = closes[n - 252];
|
||||
if (!current || !past) return null;
|
||||
return (current / past) - 1;
|
||||
}
|
||||
|
||||
function realizedVol252(closes) {
|
||||
if (!closes || closes.length < 252) return null;
|
||||
const n = closes.length;
|
||||
const returns = [];
|
||||
for (let i = n - 251; i < n; i++) {
|
||||
const prev = closes[i - 1];
|
||||
const cur = closes[i];
|
||||
if (prev && cur) returns.push((cur - prev) / prev);
|
||||
}
|
||||
if (returns.length < 200) return null;
|
||||
|
||||
const mean = returns.reduce((a, b) => a + b, 0) / returns.length;
|
||||
const variance = returns.reduce((a, b) => a + Math.pow(b - mean, 2), 0) / (returns.length - 1);
|
||||
// Annualize daily vol
|
||||
return Math.sqrt(variance) * Math.sqrt(252);
|
||||
}
|
||||
|
||||
export async function pullRawRecord(ticker, asOf = new Date().toISOString().split('T')[0]) {
|
||||
const info = await fetchQuoteSummary(ticker);
|
||||
if (!info) return null;
|
||||
|
||||
const closes = await fetchHistory(ticker);
|
||||
|
||||
return {
|
||||
snapshot_date: asOf,
|
||||
ticker,
|
||||
price: info.price,
|
||||
marketCap: info.market_cap,
|
||||
|
||||
// raw inputs
|
||||
trailingPE: info.pe_ttm,
|
||||
priceToBook: info.price_to_book,
|
||||
priceToSales: info.price_to_sales,
|
||||
freeCashflow: info.free_cash_flow,
|
||||
returnOnEquity: info.return_on_equity,
|
||||
grossMargins: info.gross_margins,
|
||||
debtToEquity: info.debt_to_equity,
|
||||
|
||||
mom_12_1: momentum12_1(closes),
|
||||
realizedVol252: realizedVol252(closes),
|
||||
|
||||
fundamentals_period_end: info.fiscal_end
|
||||
};
|
||||
}
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
export function rank(arr) {
|
||||
const sorted = arr.map((val, ind) => ({ val, ind })).sort((a, b) => a.val - b.val);
|
||||
const ranks = new Array(arr.length);
|
||||
for (let i = 0; i < sorted.length; i++) {
|
||||
ranks[sorted[i].ind] = i + 1;
|
||||
}
|
||||
// Handle ties (average rank) if needed, but simple rank is fine for large N
|
||||
return ranks;
|
||||
}
|
||||
|
||||
export function spearman(x, y) {
|
||||
if (x.length !== y.length || x.length === 0) return null;
|
||||
const rankX = rank(x);
|
||||
const rankY = rank(y);
|
||||
const n = x.length;
|
||||
let dSqSum = 0;
|
||||
for (let i = 0; i < n; i++) {
|
||||
dSqSum += Math.pow(rankX[i] - rankY[i], 2);
|
||||
}
|
||||
return 1 - ((6 * dSqSum) / (n * (Math.pow(n, 2) - 1)));
|
||||
}
|
||||
|
||||
// Simple mean
|
||||
export function mean(arr) {
|
||||
if (!arr || arr.length === 0) return 0;
|
||||
return arr.reduce((a, b) => a + b, 0) / arr.length;
|
||||
}
|
||||
|
||||
// Simple standard deviation
|
||||
export function std(arr, m) {
|
||||
if (!arr || arr.length <= 1) return 1;
|
||||
const variance = arr.reduce((sum, v) => sum + Math.pow(v - m, 2), 0) / (arr.length - 1);
|
||||
return Math.sqrt(variance) || 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate Newey-West standard error for a time-series of means
|
||||
* Lags = 0 is equivalent to standard error.
|
||||
* For N-month overlapping returns, lag = N - 1.
|
||||
*/
|
||||
export function neweyWestStdErr(ts, lags) {
|
||||
const n = ts.length;
|
||||
if (n <= 1) return 1;
|
||||
const m = mean(ts);
|
||||
|
||||
// Variance
|
||||
let s0 = 0;
|
||||
for (let i = 0; i < n; i++) {
|
||||
s0 += Math.pow(ts[i] - m, 2);
|
||||
}
|
||||
s0 = s0 / n;
|
||||
|
||||
// Covariances
|
||||
let sLags = 0;
|
||||
for (let l = 1; l <= lags; l++) {
|
||||
let cov = 0;
|
||||
for (let i = l; i < n; i++) {
|
||||
cov += (ts[i] - m) * (ts[i - l] - m);
|
||||
}
|
||||
cov = cov / n;
|
||||
const weight = 1 - (l / (lags + 1));
|
||||
sLags += 2 * weight * cov;
|
||||
}
|
||||
|
||||
const S = s0 + sLags;
|
||||
return Math.sqrt(S / n) || (std(ts, m) / Math.sqrt(n)); // fallback
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute turnover given two sets of weights.
|
||||
* w1, w2 are objects: { 'AAPL': 0.05, 'MSFT': 0.02, ... }
|
||||
* turnover = 0.5 * sum(|w2_i - w1_i|)
|
||||
*/
|
||||
export function calculateTurnover(wOld, wNew) {
|
||||
if (!wOld) return 1.0; // 100% turnover on first month
|
||||
let turnover = 0;
|
||||
const allTickers = new Set([...Object.keys(wOld), ...Object.keys(wNew)]);
|
||||
for (const t of allTickers) {
|
||||
const oldWt = wOld[t] || 0;
|
||||
const newWt = wNew[t] || 0;
|
||||
turnover += Math.abs(newWt - oldWt);
|
||||
}
|
||||
return 0.5 * turnover;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue