831 lines
38 KiB
Python
831 lines
38 KiB
Python
"""
|
||
FastAPI service for options flow processing
|
||
Replaces complex SQL with Python/pandas logic
|
||
"""
|
||
from fastapi import FastAPI, HTTPException, Query, Body
|
||
from fastapi.middleware.cors import CORSMiddleware
|
||
from pydantic import BaseModel
|
||
from typing import Optional, List, Dict, Any
|
||
from datetime import datetime, timedelta
|
||
import pandas as pd
|
||
import asyncpg
|
||
import pytz
|
||
|
||
from db import get_pool, close_pool
|
||
from services.options_flow_processor import OptionsFlowProcessor
|
||
from services.price_context import PriceContextService
|
||
from services.alert_service import AlertService
|
||
from services.output_formatter import OutputFormatter
|
||
from services.price_reaction_tracker import PriceReactionTracker
|
||
from services.signal_tier_classifier import SignalTierClassifier
|
||
from services.trade_checklist import TradeChecklist
|
||
from utils.logger import logger
|
||
from utils.error_handler import handle_processing_error
|
||
|
||
app = FastAPI(title="Options Flow Processing Service", version="1.0.0")
|
||
|
||
# CORS middleware
|
||
app.add_middleware(
|
||
CORSMiddleware,
|
||
allow_origins=["*"], # Configure appropriately for production
|
||
allow_credentials=True,
|
||
allow_methods=["*"],
|
||
allow_headers=["*"],
|
||
)
|
||
|
||
|
||
class OptionsFlowRequest(BaseModel):
|
||
start_date: Optional[str] = None
|
||
end_date: Optional[str] = None
|
||
min_premium: Optional[float] = 80000
|
||
tol_pct: Optional[float] = 0.20
|
||
|
||
|
||
class OptionsFlowResponse(BaseModel):
|
||
success: bool
|
||
data: List[dict]
|
||
count: int
|
||
timestamp: str
|
||
|
||
|
||
@app.on_event("startup")
|
||
async def startup():
|
||
"""Initialize database pool on startup"""
|
||
try:
|
||
pool = await get_pool()
|
||
logger.info("✅ Database pool initialized")
|
||
except Exception as e:
|
||
logger.error(f"❌ Failed to initialize database pool: {e}")
|
||
|
||
|
||
@app.on_event("shutdown")
|
||
async def shutdown():
|
||
"""Close database pool on shutdown"""
|
||
await close_pool()
|
||
|
||
|
||
@app.get("/health")
|
||
async def health():
|
||
"""Health check endpoint"""
|
||
try:
|
||
pool = await get_pool()
|
||
async with pool.acquire() as conn:
|
||
await conn.fetchval("SELECT 1")
|
||
return {"status": "healthy"}
|
||
except Exception as e:
|
||
return {"status": "unhealthy", "error": str(e)}
|
||
|
||
|
||
@app.get("/api/options-flow", response_model=OptionsFlowResponse)
|
||
async def get_options_flow(
|
||
start_date: Optional[str] = Query(None, description="Start date (YYYY-MM-DD)"),
|
||
end_date: Optional[str] = Query(None, description="End date (YYYY-MM-DD)"),
|
||
min_premium: Optional[float] = Query(80000, description="Minimum premium filter"),
|
||
tol_pct: Optional[float] = Query(0.20, description="Tape alignment tolerance")
|
||
):
|
||
"""
|
||
Get processed options flow data
|
||
Replaces the complex SQL query with Python processing
|
||
"""
|
||
try:
|
||
logger.info(f"Options flow request: start={start_date}, end={end_date}, min_premium={min_premium}")
|
||
pool = await get_pool()
|
||
|
||
# Default dates (only if not provided)
|
||
if not start_date:
|
||
start_date = (datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d')
|
||
logger.info(f"No start_date provided, using default: {start_date}")
|
||
if not end_date:
|
||
end_date = datetime.now().strftime('%Y-%m-%d')
|
||
logger.info(f"No end_date provided, using default: {end_date}")
|
||
|
||
logger.info(f"Processing with date range: {start_date} to {end_date}")
|
||
start_dt = datetime.strptime(start_date, '%Y-%m-%d')
|
||
end_dt = datetime.strptime(end_date, '%Y-%m-%d')
|
||
|
||
# Load raw options flow data (with timeout handling)
|
||
try:
|
||
async with pool.acquire() as conn:
|
||
# Build query with date filtering
|
||
# Note: CreatedDate is TEXT, so we need to handle date comparisons carefully
|
||
query = """
|
||
SELECT *
|
||
FROM "OptionsFlow_monthly"
|
||
WHERE "Premium" IS NOT NULL
|
||
AND TRIM("Premium"::text) <> ''
|
||
AND "StockEtf" = 'STOCK'
|
||
AND "Symbol" NOT IN ('TSLA', 'NVDA')
|
||
"""
|
||
|
||
# Add date filtering using the parsed dates
|
||
params = []
|
||
if start_date:
|
||
# CreatedDate is TEXT, so compare as strings (assuming YYYY-MM-DD format)
|
||
query += ' AND "CreatedDate" >= $1'
|
||
params.append(start_date)
|
||
if end_date:
|
||
param_idx = len(params) + 1
|
||
query += ' AND "CreatedDate" <= $' + str(param_idx)
|
||
params.append(end_date)
|
||
|
||
# Add LIMIT for safety (prevent loading millions of rows)
|
||
# Limit to 500k rows max
|
||
query += ' LIMIT 500000'
|
||
|
||
logger.info(f"Executing query with date range: {start_date} to {end_date}")
|
||
|
||
# Execute with timeout
|
||
try:
|
||
# Set statement timeout for this query (60 seconds)
|
||
await conn.execute('SET statement_timeout = 60000')
|
||
rows = await conn.fetch(query, *params)
|
||
await conn.execute('RESET statement_timeout')
|
||
logger.info(f"✅ Fetched {len(rows)} rows from database")
|
||
except Exception as query_error:
|
||
await conn.execute('RESET statement_timeout')
|
||
raise query_error
|
||
except Exception as e:
|
||
error_type = type(e).__name__
|
||
error_msg = str(e)
|
||
logger.error(f"Database query error: {error_type} - {error_msg}")
|
||
|
||
# Provide more helpful error messages
|
||
if 'TimeoutError' in error_type or 'timeout' in error_msg.lower():
|
||
raise HTTPException(
|
||
status_code=504,
|
||
detail=f"Database query timed out. The query may be too large. Try narrowing the date range or ensure database indexes are optimized."
|
||
)
|
||
elif 'Connection' in error_type or 'connection' in error_msg.lower():
|
||
raise HTTPException(
|
||
status_code=503,
|
||
detail=f"Database connection error: {error_msg}. Check database connectivity and configuration."
|
||
)
|
||
else:
|
||
raise HTTPException(
|
||
status_code=500,
|
||
detail=f"Database query failed: {error_msg}"
|
||
)
|
||
|
||
if not rows:
|
||
return OptionsFlowResponse(
|
||
success=True,
|
||
data=[],
|
||
count=0,
|
||
timestamp=datetime.now().isoformat()
|
||
)
|
||
|
||
# Convert to DataFrame
|
||
df = pd.DataFrame([dict(row) for row in rows])
|
||
|
||
# Process with Python service
|
||
processor = OptionsFlowProcessor(tol_pct=tol_pct)
|
||
df_processed = processor.process(df, start_dt, end_dt)
|
||
|
||
# Enrich with price context (optimized batch queries) - includes VWAP
|
||
price_service = PriceContextService(pool)
|
||
df_with_prices = await price_service.enrich_flow_with_prices(df_processed, pool)
|
||
|
||
# Match alerts (batch processing)
|
||
alert_service = AlertService(pool)
|
||
df_final = await alert_service.match_alerts_to_flows(df_with_prices)
|
||
|
||
# Recalculate rocket score with price context and alerts
|
||
df_final = processor.process_rocket_score(df_final)
|
||
|
||
# Apply institutional-grade analytics pipeline
|
||
logger.info("🔹 Applying institutional-grade analytics...")
|
||
from services.relative_premium_scorer import RelativePremiumScorer
|
||
from services.noise_rejector import NoiseRejector
|
||
from services.signal_component_scorer import SignalComponentScorer
|
||
from services.time_sequenced_analyzer import TimeSequencedAnalyzer
|
||
from services.intent_classifier import IntentClassifier
|
||
from services.dealer_flow_context import DealerFlowContext
|
||
from services.market_regime_detector import MarketRegimeDetector
|
||
from services.flow_decay_validator import FlowDecayValidator
|
||
from services.institutional_confidence import InstitutionalConfidence
|
||
|
||
# 1. Tier-0 Noise Rejection (mark but don't filter yet - filtering happens later)
|
||
logger.info("1️⃣ Applying tier-0 noise rejection...")
|
||
noise_rejector = NoiseRejector()
|
||
df_final = noise_rejector.mark_noise_rejections(df_final)
|
||
|
||
# 2. Relative Premium Scoring
|
||
logger.info("2️⃣ Calculating relative premium scores...")
|
||
premium_scorer = RelativePremiumScorer(pool)
|
||
df_final = await premium_scorer.enrich_with_relative_premium(df_final)
|
||
|
||
# 3. Signal Component Scoring (convert badges to continuous scores)
|
||
logger.info("3️⃣ Converting badges to continuous signal components...")
|
||
signal_scorer = SignalComponentScorer()
|
||
df_final = signal_scorer.enrich_with_signal_components(df_final)
|
||
|
||
# 4. Time-Sequenced Flow Analysis
|
||
logger.info("4️⃣ Analyzing time-sequenced flow patterns...")
|
||
time_analyzer = TimeSequencedAnalyzer()
|
||
df_final = time_analyzer.enrich_with_time_sequenced_metrics(df_final)
|
||
|
||
# 5. Intent Classification
|
||
logger.info("5️⃣ Classifying volatility and hedging intent...")
|
||
intent_classifier = IntentClassifier()
|
||
df_final = intent_classifier.enrich_with_intent_classification(df_final)
|
||
|
||
# 6. Dealer-Aware Flow Context
|
||
logger.info("6️⃣ Analyzing dealer hedging pressure...")
|
||
dealer_context = DealerFlowContext()
|
||
df_final = dealer_context.enrich_with_dealer_context(df_final)
|
||
|
||
# 7. Market Regime Detection
|
||
logger.info("7️⃣ Detecting market regime...")
|
||
regime_detector = MarketRegimeDetector()
|
||
df_final = regime_detector.enrich_with_market_regime(df_final)
|
||
|
||
# 8. Flow Decay & Reversal Validation
|
||
logger.info("8️⃣ Validating flow decay and reversal signals...")
|
||
flow_validator = FlowDecayValidator()
|
||
df_final = flow_validator.enrich_with_flow_state(df_final)
|
||
|
||
# 9. Institutional Confidence Metrics
|
||
logger.info("9️⃣ Calculating institutional confidence metrics...")
|
||
confidence_calc = InstitutionalConfidence()
|
||
df_final = confidence_calc.enrich_with_confidence_metrics(df_final)
|
||
|
||
# Phase 1 Enhancements (BEFORE filtering so all signals get Phase 1 data):
|
||
# Initialize Phase 1 columns with None/empty values first
|
||
if not df_final.empty:
|
||
# Initialize all Phase 1 columns to ensure they exist
|
||
df_final['signal_tier'] = None
|
||
df_final['is_tradeable'] = False
|
||
df_final['checklist_score'] = None
|
||
df_final['checklist_passed'] = False
|
||
df_final['checklist_details'] = None
|
||
df_final['price_reaction_5m_pct'] = None
|
||
df_final['price_reaction_15m_pct'] = None
|
||
df_final['price_reaction_30m_pct'] = None
|
||
df_final['high_break_5m'] = None
|
||
df_final['low_break_5m'] = None
|
||
df_final['flow_led_to_move'] = None
|
||
# VWAP fields should already be set by PriceContextService, but ensure they exist
|
||
if 'vwap_at_signal' not in df_final.columns:
|
||
df_final['vwap_at_signal'] = None
|
||
if 'price_vs_vwap_pct' not in df_final.columns:
|
||
df_final['price_vs_vwap_pct'] = None
|
||
|
||
# 1. Signal Tier Classification
|
||
logger.info("🔍 Classifying signal tiers...")
|
||
if not df_final.empty:
|
||
try:
|
||
tier_classifier = SignalTierClassifier()
|
||
df_final = tier_classifier.classify_tiers(df_final)
|
||
# Debug: Check if tiers were calculated
|
||
if 'signal_tier' in df_final.columns:
|
||
tier_counts = df_final['signal_tier'].value_counts()
|
||
logger.info(f"✅ Signal tiers calculated: {tier_counts.to_dict()}")
|
||
else:
|
||
logger.warning("⚠️ signal_tier column not found after classification")
|
||
except Exception as e:
|
||
logger.error(f"❌ Error in signal tier classification: {str(e)}", exc_info=True)
|
||
|
||
# 2. Price Reaction Tracking (disabled - will be calculated on-demand)
|
||
# Price reaction requires Yahoo Finance and is calculated when modal opens
|
||
|
||
# 3. Trade Checklist Evaluation
|
||
logger.info("✅ Evaluating trade checklist...")
|
||
if not df_final.empty:
|
||
try:
|
||
checklist = TradeChecklist()
|
||
df_final = checklist.evaluate_all(df_final)
|
||
# Debug: Check if checklist was calculated
|
||
if 'checklist_score' in df_final.columns:
|
||
checklist_count = df_final['checklist_score'].notna().sum()
|
||
logger.info(f"✅ Checklist scores calculated: {checklist_count}/{len(df_final)} signals")
|
||
else:
|
||
logger.warning("⚠️ checklist_score column not found after evaluation")
|
||
except Exception as e:
|
||
logger.error(f"❌ Error in trade checklist evaluation: {str(e)}", exc_info=True)
|
||
|
||
# Check if DataFrame is empty before filtering
|
||
if df_final.empty:
|
||
logger.warning("⚠️ No data after processing, returning empty result")
|
||
return OptionsFlowResponse(
|
||
success=True,
|
||
data=[],
|
||
count=0,
|
||
timestamp=datetime.now().isoformat()
|
||
)
|
||
|
||
# Filter by minimum premium and badge requirements (matching SQL WHERE clause)
|
||
logger.info(f"📊 Before filtering: {len(df_final)} rows")
|
||
|
||
# Apply noise rejection filter first (exclude early_noise_reject = True)
|
||
if 'early_noise_reject' in df_final.columns:
|
||
before_noise = len(df_final)
|
||
df_final = df_final[~df_final['early_noise_reject']].copy()
|
||
after_noise = len(df_final)
|
||
logger.info(f"📊 After noise rejection filter: {after_noise} rows (removed {before_noise - after_noise})")
|
||
|
||
# Only filter if columns exist
|
||
if 'premium_num' in df_final.columns:
|
||
before_premium = len(df_final)
|
||
df_final = df_final[df_final['premium_num'] > min_premium].copy()
|
||
after_premium = len(df_final)
|
||
logger.info(f"📊 After premium filter (>${min_premium:,.0f}): {after_premium} rows (removed {before_premium - after_premium})")
|
||
else:
|
||
logger.warning("⚠️ premium_num column not found, skipping premium filter")
|
||
|
||
# Apply relative premium filter if available
|
||
if 'relative_premium_score' in df_final.columns:
|
||
before_relative = len(df_final)
|
||
min_relative_threshold = 60.0 # Configurable threshold
|
||
df_final = df_final[df_final['relative_premium_score'] >= min_relative_threshold].copy()
|
||
after_relative = len(df_final)
|
||
logger.info(f"📊 After relative premium filter (>={min_relative_threshold}): {after_relative} rows (removed {before_relative - after_relative})")
|
||
|
||
if df_final.empty:
|
||
logger.warning("⚠️ No data after premium filter")
|
||
return OptionsFlowResponse(
|
||
success=True,
|
||
data=[],
|
||
count=0,
|
||
timestamp=datetime.now().isoformat()
|
||
)
|
||
|
||
# Filter by badge requirements (only if columns exist)
|
||
if 'badge_round' in df_final.columns and 'badge_more' in df_final.columns:
|
||
before_badges = len(df_final)
|
||
df_final = df_final[
|
||
(df_final['badge_round'].isin(['🟢', '🔴'])) &
|
||
(df_final['badge_more'].str.contains('💎', na=False)) &
|
||
(df_final['badge_more'].str.contains('⭐', na=False))
|
||
].copy()
|
||
after_badges = len(df_final)
|
||
logger.info(f"📊 After badge filter (🟢/🔴 + 💎 + ⭐): {after_badges} rows (removed {before_badges - after_badges})")
|
||
else:
|
||
logger.warning("⚠️ badge_round or badge_more columns not found, skipping badge filter")
|
||
|
||
if df_final.empty:
|
||
logger.warning("⚠️ No data after badge filter")
|
||
return OptionsFlowResponse(
|
||
success=True,
|
||
data=[],
|
||
count=0,
|
||
timestamp=datetime.now().isoformat()
|
||
)
|
||
|
||
# Additional direction filter (only if columns exist)
|
||
if 'direction' in df_final.columns and 'badge_round' in df_final.columns and 'bull_total' in df_final.columns and 'bear_total' in df_final.columns:
|
||
before_direction = len(df_final)
|
||
df_final = df_final[
|
||
((df_final['direction'] == 'BULL') &
|
||
(df_final['badge_round'] == '🟢') &
|
||
((df_final['bull_total'] - df_final['bear_total']) > 0)) |
|
||
((df_final['direction'] == 'BEAR') &
|
||
(df_final['badge_round'] == '🔴') &
|
||
((df_final['bull_total'] - df_final['bear_total']) < 0))
|
||
].copy()
|
||
after_direction = len(df_final)
|
||
logger.info(f"📊 After direction/net premium filter: {after_direction} rows (removed {before_direction - after_direction})")
|
||
else:
|
||
logger.warning("⚠️ Required columns for direction filter not found, skipping")
|
||
|
||
# Sort by timestamp descending
|
||
df_final = df_final.sort_values(['flow_ts_utc', 'rid'], ascending=[False, False])
|
||
|
||
# Format output to match SQL format
|
||
df_final = OutputFormatter.format_final_output(df_final)
|
||
|
||
# Debug: Log Phase 1 columns before converting to dict
|
||
phase1_columns = ['signal_tier', 'checklist_score', 'checklist_passed', 'price_reaction_5m_pct', 'flow_led_to_move', 'vwap_at_signal', 'price_vs_vwap_pct']
|
||
existing_phase1_cols = [col for col in phase1_columns if col in df_final.columns]
|
||
if existing_phase1_cols:
|
||
logger.info(f"✅ Phase 1 columns in final output: {existing_phase1_cols}")
|
||
# Sample a row to check values
|
||
if len(df_final) > 0:
|
||
sample_row = df_final.iloc[0]
|
||
sample_phase1 = {col: sample_row.get(col) for col in existing_phase1_cols}
|
||
logger.info(f"📊 Sample Phase 1 data: {sample_phase1}")
|
||
else:
|
||
logger.warning("⚠️ No Phase 1 columns found in final output!")
|
||
|
||
# Convert DataFrame to list of dicts
|
||
result_data = df_final.to_dict('records')
|
||
|
||
# Format dates and handle NaN values
|
||
for record in result_data:
|
||
# Convert datetime objects to strings
|
||
for key, value in record.items():
|
||
if isinstance(value, datetime):
|
||
record[key] = value.isoformat()
|
||
elif pd.isna(value):
|
||
record[key] = None
|
||
elif isinstance(value, pd.Timestamp):
|
||
# Check if it's NaT (Not a Time)
|
||
if pd.isna(value):
|
||
record[key] = None
|
||
else:
|
||
record[key] = value.isoformat()
|
||
elif isinstance(value, dict):
|
||
# Keep dicts as-is (e.g., checklist_details)
|
||
pass
|
||
elif isinstance(value, (list, tuple)):
|
||
# Keep lists as-is
|
||
pass
|
||
|
||
return OptionsFlowResponse(
|
||
success=True,
|
||
data=result_data,
|
||
count=len(result_data),
|
||
timestamp=datetime.now().isoformat()
|
||
)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error processing options flow: {e}", exc_info=True)
|
||
raise HTTPException(
|
||
status_code=500,
|
||
detail=f"Processing failed: {str(e)}"
|
||
)
|
||
|
||
|
||
@app.get("/api/options-flow/stats")
|
||
async def get_flow_stats(
|
||
symbol: Optional[str] = Query(None, description="Symbol to get stats for")
|
||
):
|
||
"""Get flow statistics"""
|
||
try:
|
||
pool = await get_pool()
|
||
|
||
query = """
|
||
SELECT
|
||
symbol,
|
||
COUNT(*) as total_trades,
|
||
SUM(premium_num) as total_premium,
|
||
SUM(CASE WHEN cp_norm = 'CALL' THEN vol_num ELSE 0 END) as call_volume,
|
||
SUM(CASE WHEN cp_norm = 'PUT' THEN vol_num ELSE 0 END) as put_volume
|
||
FROM processed_options_flow
|
||
"""
|
||
|
||
params = []
|
||
if symbol:
|
||
query += " WHERE symbol_norm = $1"
|
||
params.append(symbol.upper())
|
||
|
||
query += " GROUP BY symbol"
|
||
|
||
async with pool.acquire() as conn:
|
||
rows = await conn.fetch(query, *params)
|
||
|
||
return {
|
||
"success": True,
|
||
"data": [dict(row) for row in rows]
|
||
}
|
||
|
||
except Exception as e:
|
||
raise HTTPException(status_code=500, detail=str(e))
|
||
|
||
|
||
@app.post("/api/phase1/calculate")
|
||
async def calculate_phase1_for_signal(request: Dict[str, Any] = Body(...)):
|
||
"""
|
||
Calculate Phase 1 metrics for a specific signal on-demand
|
||
This is called when the Phase 1 modal opens - fetches Yahoo Finance data and calculates Phase 1
|
||
"""
|
||
try:
|
||
from services.yahoo_finance_service import YahooFinanceService
|
||
|
||
symbol = request.get('symbol')
|
||
flow_ts_utc = request.get('flow_ts_utc')
|
||
flow_date_cst = request.get('flow_date_cst')
|
||
row_data = request.get('row_data', {})
|
||
|
||
if not symbol or not flow_ts_utc:
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail="symbol and flow_ts_utc are required"
|
||
)
|
||
|
||
# Log the symbol we received
|
||
logger.info(f"📥 Phase 1 calculation request for symbol: '{symbol}' (type: {type(symbol).__name__}, length: {len(symbol) if symbol else 0})")
|
||
|
||
pool = await get_pool()
|
||
|
||
# Parse timestamps - handle various formats
|
||
logger.info(f"📥 Received flow_ts_utc: {flow_ts_utc} (type: {type(flow_ts_utc).__name__})")
|
||
|
||
if isinstance(flow_ts_utc, str):
|
||
try:
|
||
original_ts = flow_ts_utc
|
||
# Remove any trailing Z and handle ISO format
|
||
ts_str = flow_ts_utc.strip()
|
||
if ts_str.endswith('Z'):
|
||
# Replace Z with +00:00 for fromisoformat
|
||
ts_str = ts_str[:-1] + '+00:00'
|
||
logger.info(f"📅 Parsing UTC timestamp (Z suffix): {original_ts} -> {ts_str}")
|
||
elif 'T' in ts_str and '+' not in ts_str and '-' not in ts_str[-6:]:
|
||
# ISO format without timezone - check if it might be local time
|
||
# For now, assume UTC as the field name suggests, but log it
|
||
ts_str = ts_str + '+00:00'
|
||
logger.info(f"📅 Parsing timestamp without timezone, assuming UTC: {original_ts} -> {ts_str}")
|
||
|
||
# Try parsing with fromisoformat
|
||
try:
|
||
flow_ts_utc = datetime.fromisoformat(ts_str)
|
||
logger.info(f"✅ Parsed flow_ts_utc: {flow_ts_utc} (timezone: {flow_ts_utc.tzinfo})")
|
||
except ValueError:
|
||
# Fallback: try parsing just the date part
|
||
date_part = ts_str.split('T')[0].split(' ')[0]
|
||
flow_ts_utc = datetime.strptime(date_part, '%Y-%m-%d')
|
||
logger.warning(f"⚠️ Could only parse date part from {original_ts}, using {flow_ts_utc}")
|
||
except (ValueError, AttributeError) as e:
|
||
logger.error(f"❌ Error parsing flow_ts_utc '{flow_ts_utc}': {e}")
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail=f"Invalid flow_ts_utc format: {flow_ts_utc}. Error: {str(e)}"
|
||
)
|
||
elif isinstance(flow_ts_utc, (int, float)):
|
||
# Unix timestamp - assume UTC
|
||
flow_ts_utc = datetime.fromtimestamp(flow_ts_utc, tz=pytz.UTC)
|
||
logger.info(f"📅 Parsed Unix timestamp: {flow_ts_utc} (UTC)")
|
||
|
||
# Ensure flow_ts_utc is timezone-aware (assume UTC if naive)
|
||
if flow_ts_utc.tzinfo is None:
|
||
flow_ts_utc = pytz.UTC.localize(flow_ts_utc)
|
||
logger.info(f"📅 flow_ts_utc was naive, localized to UTC: {flow_ts_utc}")
|
||
|
||
if isinstance(flow_date_cst, str):
|
||
try:
|
||
# Extract just the date part if it's a datetime string
|
||
# Handle formats like "2025-12-10T05:00:00.000Z" or "2025-12-10"
|
||
original_date = flow_date_cst.strip()
|
||
|
||
# Split on 'T' first, then on space, take first part
|
||
if 'T' in original_date:
|
||
date_str = original_date.split('T')[0]
|
||
elif ' ' in original_date:
|
||
date_str = original_date.split(' ')[0]
|
||
else:
|
||
date_str = original_date
|
||
|
||
# Validate it's in YYYY-MM-DD format (exactly 10 characters, 2 dashes)
|
||
if len(date_str) == 10 and date_str.count('-') == 2:
|
||
flow_date_cst = datetime.strptime(date_str, '%Y-%m-%d').date()
|
||
logger.debug(f"Parsed flow_date_cst: {flow_date_cst} from '{original_date}'")
|
||
else:
|
||
raise ValueError(f"Date string '{date_str}' is not in YYYY-MM-DD format (length={len(date_str)}, dashes={date_str.count('-')})")
|
||
except (ValueError, AttributeError) as e:
|
||
logger.warning(f"Error parsing flow_date_cst '{flow_date_cst}': {e}")
|
||
# Try to extract date from flow_ts_utc if available
|
||
if flow_ts_utc and isinstance(flow_ts_utc, datetime):
|
||
flow_date_cst = flow_ts_utc.date()
|
||
logger.info(f"Using date from flow_ts_utc: {flow_date_cst}")
|
||
else:
|
||
# Default to today if we can't parse
|
||
flow_date_cst = datetime.now().date()
|
||
logger.warning(f"Using current date as fallback for flow_date_cst")
|
||
elif flow_date_cst is None:
|
||
# If not provided, try to get from flow_ts_utc
|
||
if flow_ts_utc and isinstance(flow_ts_utc, datetime):
|
||
flow_date_cst = flow_ts_utc.date()
|
||
else:
|
||
flow_date_cst = datetime.now().date()
|
||
logger.warning(f"flow_date_cst not provided, using current date as fallback")
|
||
elif flow_date_cst is None:
|
||
# If not provided, try to get from flow_ts_utc
|
||
if flow_ts_utc and isinstance(flow_ts_utc, datetime):
|
||
flow_date_cst = flow_ts_utc.date()
|
||
else:
|
||
flow_date_cst = datetime.now().date()
|
||
logger.warning(f"flow_date_cst not provided, using current date as fallback")
|
||
|
||
# Initialize services with Yahoo Finance enabled
|
||
price_service = PriceContextService(pool)
|
||
price_service.use_yahoo_finance = True
|
||
price_service.yahoo_service = YahooFinanceService()
|
||
|
||
reaction_tracker = PriceReactionTracker()
|
||
reaction_tracker.use_yahoo_finance = True
|
||
reaction_tracker.yahoo_service = YahooFinanceService()
|
||
|
||
tier_classifier = SignalTierClassifier()
|
||
checklist = TradeChecklist()
|
||
|
||
# Create a minimal DataFrame row for processing
|
||
df_row = pd.DataFrame([{
|
||
'symbol_norm': symbol,
|
||
'flow_ts_utc': flow_ts_utc,
|
||
'flow_date_cst': flow_date_cst,
|
||
**row_data # Include all other row data (badges, premium, etc.)
|
||
}])
|
||
|
||
# Calculate Phase 1 metrics
|
||
result = {}
|
||
|
||
# 1. Signal Tier (doesn't need price data)
|
||
try:
|
||
df_with_tier = tier_classifier.classify_tiers(df_row)
|
||
if not df_with_tier.empty:
|
||
result['signal_tier'] = df_with_tier.iloc[0].get('signal_tier')
|
||
# Convert numpy bool to Python bool
|
||
is_tradeable_val = df_with_tier.iloc[0].get('is_tradeable', False)
|
||
result['is_tradeable'] = bool(is_tradeable_val) if is_tradeable_val is not None else False
|
||
except Exception as e:
|
||
logger.error(f"Error calculating signal tier: {e}")
|
||
result['signal_tier'] = None
|
||
result['is_tradeable'] = False
|
||
|
||
# 2. Price Reaction (needs Yahoo Finance)
|
||
try:
|
||
# Check if signal is recent enough for intraday data
|
||
now = datetime.now(pytz.timezone('America/Chicago'))
|
||
time_diff_hours = (now - flow_ts_utc.replace(tzinfo=now.tzinfo)).total_seconds() / 3600
|
||
|
||
if time_diff_hours > 168: # 7 days
|
||
logger.warning(f"⚠️ Price reaction unavailable: Signal is {time_diff_hours/24:.1f} days old")
|
||
logger.warning(f" Yahoo Finance only provides intraday data for the last 7 days")
|
||
logger.warning(f" For historical signals, price reaction data requires your own intraday price database")
|
||
else:
|
||
logger.info(f"📊 Calculating price reaction for {symbol} (signal is {time_diff_hours:.1f} hours old)")
|
||
|
||
df_with_reactions = await reaction_tracker.enrich_with_reactions(df_row, pool)
|
||
if not df_with_reactions.empty:
|
||
result['price_reaction_5m_pct'] = df_with_reactions.iloc[0].get('price_reaction_5m_pct')
|
||
result['price_reaction_15m_pct'] = df_with_reactions.iloc[0].get('price_reaction_15m_pct')
|
||
result['price_reaction_30m_pct'] = df_with_reactions.iloc[0].get('price_reaction_30m_pct')
|
||
# Convert numpy bools to Python bools
|
||
flow_led = df_with_reactions.iloc[0].get('flow_led_to_move')
|
||
result['flow_led_to_move'] = bool(flow_led) if flow_led is not None else None
|
||
high_break = df_with_reactions.iloc[0].get('high_break_5m')
|
||
result['high_break_5m'] = bool(high_break) if high_break is not None else None
|
||
low_break = df_with_reactions.iloc[0].get('low_break_5m')
|
||
result['low_break_5m'] = bool(low_break) if low_break is not None else None
|
||
except Exception as e:
|
||
logger.error(f"Error calculating price reaction: {e}")
|
||
result['price_reaction_5m_pct'] = None
|
||
result['price_reaction_15m_pct'] = None
|
||
result['price_reaction_30m_pct'] = None
|
||
result['flow_led_to_move'] = None
|
||
result['high_break_5m'] = None
|
||
result['low_break_5m'] = None
|
||
|
||
# 3. VWAP (needs Yahoo Finance)
|
||
try:
|
||
# Ensure flow_ts_utc is timezone-aware
|
||
# If it's naive (no timezone), assume it's UTC (as the name suggests)
|
||
# If it has timezone info, use it
|
||
if flow_ts_utc.tzinfo is None:
|
||
# Naive datetime - assume UTC (as name suggests)
|
||
flow_ts_utc = pytz.UTC.localize(flow_ts_utc)
|
||
logger.debug(f"flow_ts_utc was naive, assumed UTC: {flow_ts_utc}")
|
||
|
||
# Convert to Eastern Time for market hours check (US market opens at 9:30 AM ET)
|
||
et_tz = pytz.timezone('America/New_York')
|
||
signal_time_et = flow_ts_utc.astimezone(et_tz)
|
||
cst_tz = pytz.timezone('America/Chicago')
|
||
signal_time_cst = flow_ts_utc.astimezone(cst_tz)
|
||
|
||
# Check if signal is recent enough and during market hours
|
||
now_et = datetime.now(et_tz)
|
||
now_cst = datetime.now(cst_tz)
|
||
time_diff_hours = (now_et - signal_time_et).total_seconds() / 3600
|
||
signal_hour_et = signal_time_et.hour
|
||
signal_minute_et = signal_time_et.minute
|
||
|
||
logger.info(f"📅 Signal time: {signal_time_et.strftime('%Y-%m-%d %H:%M:%S %Z')} (ET) / {signal_time_cst.strftime('%H:%M:%S %Z')} (CST)")
|
||
logger.info(f"📅 Current time: {now_et.strftime('%Y-%m-%d %H:%M:%S %Z')} (ET) / {now_cst.strftime('%H:%M:%S %Z')} (CST)")
|
||
|
||
if time_diff_hours > 168: # 7 days
|
||
logger.warning(f"⚠️ VWAP unavailable: Signal is {time_diff_hours/24:.1f} days old")
|
||
logger.warning(f" Yahoo Finance only provides intraday data for the last 7 days")
|
||
elif signal_hour_et < 9 or (signal_hour_et == 9 and signal_minute_et < 30):
|
||
logger.warning(f"⚠️ VWAP unavailable: Signal time is {signal_time_et.strftime('%H:%M')} ET (before RTH open at 9:30 AM ET)")
|
||
logger.warning(f" VWAP can only be calculated after market open (9:30 AM Eastern Time)")
|
||
else:
|
||
logger.info(f"📊 Calculating VWAP for {symbol} at {signal_time_et.strftime('%Y-%m-%d %H:%M')} ET")
|
||
|
||
vwap_data = await price_service.calculate_vwap_at_time(symbol, flow_ts_utc)
|
||
if vwap_data:
|
||
result['vwap_at_signal'] = vwap_data.get('vwap')
|
||
|
||
# Get price at signal time
|
||
price_at_time = await price_service.get_price_at_time(symbol, flow_ts_utc)
|
||
if price_at_time and price_at_time.get('close') and result.get('vwap_at_signal'):
|
||
price_vs_vwap = ((price_at_time['close'] - result['vwap_at_signal']) / result['vwap_at_signal']) * 100
|
||
result['price_vs_vwap_pct'] = round(price_vs_vwap, 2)
|
||
else:
|
||
result['price_vs_vwap_pct'] = None
|
||
else:
|
||
result['vwap_at_signal'] = None
|
||
result['price_vs_vwap_pct'] = None
|
||
except Exception as e:
|
||
logger.error(f"Error calculating VWAP: {e}")
|
||
result['vwap_at_signal'] = None
|
||
result['price_vs_vwap_pct'] = None
|
||
|
||
# 4. Trade Checklist (needs VWAP, but we can calculate with what we have)
|
||
try:
|
||
# Add VWAP data to row for checklist
|
||
df_row['vwap_at_signal'] = result.get('vwap_at_signal')
|
||
df_row['price_vs_vwap_pct'] = result.get('price_vs_vwap_pct')
|
||
|
||
df_with_checklist = checklist.evaluate_all(df_row)
|
||
if not df_with_checklist.empty:
|
||
result['checklist_score'] = df_with_checklist.iloc[0].get('checklist_score')
|
||
# Convert numpy bool to Python bool
|
||
checklist_passed_val = df_with_checklist.iloc[0].get('checklist_passed')
|
||
result['checklist_passed'] = bool(checklist_passed_val) if checklist_passed_val is not None else False
|
||
|
||
# Extract and convert checklist_details immediately
|
||
checklist_details_raw = df_with_checklist.iloc[0].get('checklist_details')
|
||
if checklist_details_raw is not None:
|
||
# Convert nested numpy types in checklist_details
|
||
import numpy as np
|
||
if isinstance(checklist_details_raw, dict):
|
||
result['checklist_details'] = {
|
||
k: bool(v) if isinstance(v, np.bool_) else
|
||
(int(v) if isinstance(v, np.integer) else
|
||
(float(v) if isinstance(v, np.floating) else v))
|
||
for k, v in checklist_details_raw.items()
|
||
}
|
||
else:
|
||
result['checklist_details'] = {}
|
||
else:
|
||
result['checklist_details'] = {}
|
||
except Exception as e:
|
||
logger.error(f"Error calculating checklist: {e}")
|
||
result['checklist_score'] = None
|
||
result['checklist_passed'] = False
|
||
result['checklist_details'] = {}
|
||
|
||
# Convert numpy types to native Python types for JSON serialization
|
||
def convert_to_native(obj):
|
||
"""Recursively convert numpy types to native Python types"""
|
||
import numpy as np
|
||
|
||
# Handle None and NaN first
|
||
if obj is None:
|
||
return None
|
||
try:
|
||
if hasattr(pd, 'isna') and pd.isna(obj):
|
||
return None
|
||
except (TypeError, ValueError):
|
||
pass
|
||
try:
|
||
if isinstance(obj, float) and np.isnan(obj):
|
||
return None
|
||
except (TypeError, ValueError):
|
||
pass
|
||
|
||
# Handle numpy types
|
||
try:
|
||
if isinstance(obj, np.bool_):
|
||
return bool(obj)
|
||
elif isinstance(obj, np.integer):
|
||
return int(obj)
|
||
elif isinstance(obj, np.floating):
|
||
return float(obj)
|
||
elif isinstance(obj, np.ndarray):
|
||
return obj.tolist()
|
||
except (TypeError, AttributeError):
|
||
pass
|
||
|
||
# Handle collections
|
||
try:
|
||
if isinstance(obj, dict):
|
||
return {str(k): convert_to_native(v) for k, v in obj.items()}
|
||
except (TypeError, AttributeError):
|
||
return {}
|
||
|
||
try:
|
||
if isinstance(obj, (list, tuple)):
|
||
return [convert_to_native(item) for item in obj]
|
||
except (TypeError, AttributeError):
|
||
return []
|
||
|
||
# Handle pandas Series/DataFrame (shouldn't happen, but just in case)
|
||
if hasattr(obj, '__dict__') and not isinstance(obj, dict):
|
||
try:
|
||
# Try to convert to string representation
|
||
return str(obj)
|
||
except:
|
||
return None
|
||
|
||
return obj
|
||
|
||
# Convert result dictionary
|
||
result_converted = convert_to_native(result)
|
||
|
||
return {
|
||
'success': True,
|
||
'data': result_converted
|
||
}
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error in Phase 1 calculation: {e}", exc_info=True)
|
||
raise HTTPException(
|
||
status_code=500,
|
||
detail=f"Phase 1 calculation failed: {str(e)}"
|
||
)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
import uvicorn
|
||
uvicorn.run(app, host="0.0.0.0", port=8010)
|