added 5 day volume
This commit is contained in:
parent
dbcfc65752
commit
b1cee2aa43
|
|
@ -0,0 +1,317 @@
|
||||||
|
# Implementation Roadmap - Quick Reference
|
||||||
|
|
||||||
|
## File-by-File Implementation Guide
|
||||||
|
|
||||||
|
### Phase 1: Critical Features (Start Here)
|
||||||
|
|
||||||
|
#### 1. Price Reaction Tracking
|
||||||
|
**New File:** `backend/python_service/services/price_reaction_tracker.py`
|
||||||
|
- Class: `PriceReactionTracker`
|
||||||
|
- Method: `track_reaction(flow_row, pool)` → returns dict with 5m/15m/30m reactions
|
||||||
|
- Integration: Call in `main.py` after `enrich_flow_with_prices()`
|
||||||
|
|
||||||
|
**Modify:** `backend/python_service/main.py`
|
||||||
|
```python
|
||||||
|
# After line 145 (after price enrichment):
|
||||||
|
from services.price_reaction_tracker import PriceReactionTracker
|
||||||
|
|
||||||
|
reaction_tracker = PriceReactionTracker()
|
||||||
|
df_final = await reaction_tracker.enrich_with_reactions(df_final, pool)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### 2. VWAP Integration
|
||||||
|
**Modify:** `backend/python_service/services/price_context.py`
|
||||||
|
- Add method: `async def calculate_vwap_at_time(symbol, timestamp, pool)`
|
||||||
|
- Add method: `async def get_vwap_for_date(symbol, date, pool)`
|
||||||
|
- Integration: Call in `enrich_flow_with_prices()` method
|
||||||
|
|
||||||
|
**Add to enrichment:**
|
||||||
|
```python
|
||||||
|
# In enrich_flow_with_prices(), add:
|
||||||
|
vwap_data = await self.get_vwap_at_time(symbol, flow_ts_utc, pool)
|
||||||
|
df['vwap_at_signal'] = vwap_data['vwap']
|
||||||
|
df['price_vs_vwap_pct'] = ((df['u_close'] - df['vwap_at_signal']) / df['vwap_at_signal']) * 100
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### 3. Signal Tier Classification
|
||||||
|
**New File:** `backend/python_service/services/signal_tier_classifier.py`
|
||||||
|
- Class: `SignalTierClassifier`
|
||||||
|
- Method: `classify_tier(row)` → returns 'TIER_1', 'TIER_2', or 'IGNORE'
|
||||||
|
|
||||||
|
**Modify:** `backend/python_service/services/options_flow_processor.py`
|
||||||
|
- Add method: `process_tier_classification(df)` → adds `signal_tier` column
|
||||||
|
- Call in `process()` method after `process_badges()`
|
||||||
|
|
||||||
|
**Integration:**
|
||||||
|
```python
|
||||||
|
# In process() method, after process_badges():
|
||||||
|
df = self.process_tier_classification(df)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### 4. Trade Checklist
|
||||||
|
**New File:** `backend/python_service/services/trade_checklist.py`
|
||||||
|
- Class: `TradeChecklist`
|
||||||
|
- Method: `evaluate(flow_row)` → returns checklist score and details
|
||||||
|
|
||||||
|
**Modify:** `backend/python_service/main.py`
|
||||||
|
- After all enrichments, add checklist evaluation:
|
||||||
|
```python
|
||||||
|
from services.trade_checklist import TradeChecklist
|
||||||
|
|
||||||
|
checklist = TradeChecklist()
|
||||||
|
df_final['checklist_result'] = df_final.apply(
|
||||||
|
lambda row: checklist.evaluate(row), axis=1
|
||||||
|
)
|
||||||
|
df_final['checklist_score'] = df_final['checklist_result'].apply(lambda x: x['checklist_score'])
|
||||||
|
df_final['checklist_passed'] = df_final['checklist_result'].apply(lambda x: x['checklist_passed'])
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 2: High Value Features
|
||||||
|
|
||||||
|
#### 5. Strike Clustering
|
||||||
|
**New File:** `backend/python_service/services/strike_cluster_detector.py`
|
||||||
|
- Class: `StrikeClusterDetector`
|
||||||
|
- Method: `detect_clusters(df, window_minutes=30)` → adds cluster flags
|
||||||
|
|
||||||
|
**Integration:** Call in `main.py` after aggregations
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### 6. Delta Weighting
|
||||||
|
**Modify:** `backend/python_service/services/options_flow_processor.py`
|
||||||
|
- Add method: `calculate_delta_weighted_value(row)`
|
||||||
|
- Add to `process_aggregations()` or create new method `process_delta_weighting()`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### 7. Index Correlation
|
||||||
|
**New File:** `backend/python_service/services/index_correlation.py`
|
||||||
|
- Class: `IndexCorrelationService`
|
||||||
|
- Method: `check_index_alignment(flow_row, pool)` → returns alignment data
|
||||||
|
|
||||||
|
**Integration:** Call in `main.py` after price enrichment
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 3: Advanced Features
|
||||||
|
|
||||||
|
#### 8. Gamma Exposure
|
||||||
|
**New File:** `backend/python_service/services/gamma_calculator.py`
|
||||||
|
- Class: `GammaCalculator`
|
||||||
|
- Method: `calculate_gex(df)` → adds GEX columns
|
||||||
|
|
||||||
|
**Note:** Requires options pricing library (e.g., `py_vollib` or simplified approximation)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### 9. Sweep vs Block Detection
|
||||||
|
**New File:** `backend/python_service/services/trade_type_detector.py`
|
||||||
|
- Class: `TradeTypeDetector`
|
||||||
|
- Method: `detect_trade_type(df)` → adds trade_type column
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### 10. DTE Buckets
|
||||||
|
**Modify:** `backend/python_service/services/options_flow_processor.py`
|
||||||
|
- Add method: `calculate_dte_bucket(row)`
|
||||||
|
- Add to `process_moneyness()` or create new method
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 4: Analytics
|
||||||
|
|
||||||
|
#### 11. Historical Win Rate
|
||||||
|
**New File:** `backend/python_service/services/pattern_analyzer.py`
|
||||||
|
- Class: `PatternAnalyzer`
|
||||||
|
- Methods: `track_pattern()`, `get_pattern_stats()`
|
||||||
|
|
||||||
|
**Database:** Create table `signal_patterns_history`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### 12. Enhanced Entry/Exit Logic
|
||||||
|
**Modify:** `backend/src/services/tradePlanGenerator.js`
|
||||||
|
- Enhance `generateEntryStrategy()` with VWAP logic
|
||||||
|
- Enhance `generateExitStrategy()` with flow-based exits
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Database Migrations
|
||||||
|
|
||||||
|
### Migration 1: Add Enrichment Columns
|
||||||
|
```sql
|
||||||
|
ALTER TABLE processed_options_flow ADD COLUMN IF NOT EXISTS
|
||||||
|
signal_tier VARCHAR(10),
|
||||||
|
is_tradeable BOOLEAN,
|
||||||
|
vwap_at_signal NUMERIC,
|
||||||
|
price_vs_vwap_pct NUMERIC,
|
||||||
|
price_reaction_5m_pct NUMERIC,
|
||||||
|
price_reaction_15m_pct NUMERIC,
|
||||||
|
flow_led_to_move BOOLEAN,
|
||||||
|
checklist_score INTEGER,
|
||||||
|
checklist_passed BOOLEAN;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Migration 2: Add Pattern Tracking Table
|
||||||
|
```sql
|
||||||
|
CREATE TABLE IF NOT EXISTS signal_patterns_history (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
pattern_hash VARCHAR(100),
|
||||||
|
signal_time TIMESTAMPTZ,
|
||||||
|
symbol VARCHAR(10),
|
||||||
|
price_at_signal NUMERIC,
|
||||||
|
price_5m_after NUMERIC,
|
||||||
|
price_15m_after NUMERIC,
|
||||||
|
outcome VARCHAR(20),
|
||||||
|
return_pct NUMERIC,
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_pattern_hash ON signal_patterns_history(pattern_hash);
|
||||||
|
CREATE INDEX idx_signal_time ON signal_patterns_history(signal_time);
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## API Endpoint Additions
|
||||||
|
|
||||||
|
### Modify: `backend/python_service/main.py`
|
||||||
|
|
||||||
|
Add new endpoints:
|
||||||
|
|
||||||
|
```python
|
||||||
|
@app.get("/api/options-flow/enhanced")
|
||||||
|
async def get_enhanced_flow(...):
|
||||||
|
# Same as existing endpoint but with all enrichments enabled
|
||||||
|
pass
|
||||||
|
|
||||||
|
@app.get("/api/options-flow/tier-1")
|
||||||
|
async def get_tier1_signals(...):
|
||||||
|
# Filter to only Tier-1 signals
|
||||||
|
df_final = df_final[df_final['signal_tier'] == 'TIER_1']
|
||||||
|
pass
|
||||||
|
|
||||||
|
@app.get("/api/options-flow/checklist-passed")
|
||||||
|
async def get_checklist_passed(...):
|
||||||
|
# Filter to only checklist-passed signals
|
||||||
|
df_final = df_final[df_final['checklist_passed'] == True]
|
||||||
|
pass
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Testing Checklist
|
||||||
|
|
||||||
|
### Unit Tests to Add
|
||||||
|
|
||||||
|
1. **Price Reaction Tests**
|
||||||
|
- `test_price_reaction_5m_positive()`
|
||||||
|
- `test_price_reaction_no_move()`
|
||||||
|
- `test_flow_led_to_move_detection()`
|
||||||
|
|
||||||
|
2. **Tier Classification Tests**
|
||||||
|
- `test_tier1_classification()`
|
||||||
|
- `test_tier2_classification()`
|
||||||
|
- `test_ignore_classification()`
|
||||||
|
|
||||||
|
3. **Checklist Tests**
|
||||||
|
- `test_checklist_5_5_passes()`
|
||||||
|
- `test_checklist_4_5_passes()`
|
||||||
|
- `test_checklist_3_5_fails()`
|
||||||
|
|
||||||
|
4. **VWAP Tests**
|
||||||
|
- `test_vwap_calculation()`
|
||||||
|
- `test_vwap_pullback_detection()`
|
||||||
|
- `test_vwap_reclaim_detection()`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Performance Considerations
|
||||||
|
|
||||||
|
### Optimization Tips
|
||||||
|
|
||||||
|
1. **Price Reaction Tracking**
|
||||||
|
- Batch fetch prices for all signals at once
|
||||||
|
- Use async queries with connection pooling
|
||||||
|
- Cache VWAP calculations per symbol/date
|
||||||
|
|
||||||
|
2. **Strike Clustering**
|
||||||
|
- Use pandas groupby operations (already efficient)
|
||||||
|
- Consider windowing for large datasets
|
||||||
|
|
||||||
|
3. **Index Correlation**
|
||||||
|
- Cache SPY/QQQ flow data (update every minute)
|
||||||
|
- Use materialized views for index flow aggregations
|
||||||
|
|
||||||
|
4. **Gamma Calculation**
|
||||||
|
- Use simplified approximation (no full Black-Scholes)
|
||||||
|
- Pre-calculate common strikes
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Rollout Strategy
|
||||||
|
|
||||||
|
### Week 1: Phase 1 (Critical)
|
||||||
|
- Day 1-2: Price Reaction Tracking
|
||||||
|
- Day 3-4: VWAP Integration
|
||||||
|
- Day 5: Signal Tier Classification
|
||||||
|
- Day 6-7: Trade Checklist
|
||||||
|
|
||||||
|
### Week 2: Phase 2 (High Value)
|
||||||
|
- Day 1-2: Strike Clustering
|
||||||
|
- Day 3: Delta Weighting
|
||||||
|
- Day 4-5: Index Correlation
|
||||||
|
|
||||||
|
### Week 3: Phase 3 (Advanced)
|
||||||
|
- Day 1-2: Gamma Exposure
|
||||||
|
- Day 3: Sweep vs Block
|
||||||
|
- Day 4: DTE Buckets
|
||||||
|
|
||||||
|
### Week 4: Phase 4 (Analytics)
|
||||||
|
- Day 1-3: Historical Win Rate Tracking
|
||||||
|
- Day 4-5: Enhanced Entry/Exit Logic
|
||||||
|
- Day 6-7: Testing & Refinement
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Monitoring & Metrics
|
||||||
|
|
||||||
|
### Key Metrics to Track
|
||||||
|
|
||||||
|
1. **Signal Quality**
|
||||||
|
- Tier-1 signal percentage
|
||||||
|
- Checklist pass rate
|
||||||
|
- Price reaction success rate
|
||||||
|
|
||||||
|
2. **Trade Performance**
|
||||||
|
- Win rate by tier
|
||||||
|
- Win rate by checklist score
|
||||||
|
- Average return by pattern
|
||||||
|
|
||||||
|
3. **System Performance**
|
||||||
|
- Enrichment processing time
|
||||||
|
- Database query performance
|
||||||
|
- API response times
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
1. ✅ Review this roadmap
|
||||||
|
2. ✅ Prioritize features based on your needs
|
||||||
|
3. ✅ Start with Phase 1 (Price Reaction + VWAP + Tier + Checklist)
|
||||||
|
4. ✅ Test each feature before moving to next
|
||||||
|
5. ✅ Monitor metrics and refine
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Remember:** Don't change existing code - extend it with new services and enrichments!
|
||||||
|
|
||||||
|
|
@ -1042,6 +1042,111 @@ sudo -u postgres psql institutional_trader < backup_20240101.sql
|
||||||
|
|
||||||
## 12. Troubleshooting
|
## 12. Troubleshooting
|
||||||
|
|
||||||
|
### Git Pull Fails - DNS Resolution Error
|
||||||
|
|
||||||
|
**Error:** `fatal: unable to access 'https://github.com/...': Could not resolve host: github.com`
|
||||||
|
|
||||||
|
**This means your container/VM cannot resolve DNS names.**
|
||||||
|
|
||||||
|
**Quick Fix:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Check current DNS configuration
|
||||||
|
cat /etc/resolv.conf
|
||||||
|
|
||||||
|
# 2. If empty or missing, add DNS servers
|
||||||
|
sudo nano /etc/resolv.conf
|
||||||
|
```
|
||||||
|
|
||||||
|
Add these lines:
|
||||||
|
```
|
||||||
|
nameserver 8.8.8.8
|
||||||
|
nameserver 8.8.4.4
|
||||||
|
nameserver 1.1.1.1
|
||||||
|
```
|
||||||
|
|
||||||
|
**For LXC Containers (Persistent Fix):**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Edit container DNS configuration
|
||||||
|
# On Proxmox host, edit container config:
|
||||||
|
nano /etc/pve/lxc/<container-id>.conf
|
||||||
|
|
||||||
|
# Add DNS servers:
|
||||||
|
nameserver: 8.8.8.8
|
||||||
|
nameserver: 8.8.4.4
|
||||||
|
```
|
||||||
|
|
||||||
|
Or via Proxmox Web UI:
|
||||||
|
- Go to your container → Options → DNS
|
||||||
|
- Set DNS servers: `8.8.8.8, 8.8.4.4`
|
||||||
|
|
||||||
|
**For Ubuntu VMs/Containers (Systemd-resolved):**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Edit systemd-resolved config
|
||||||
|
sudo nano /etc/systemd/resolved.conf
|
||||||
|
```
|
||||||
|
|
||||||
|
Uncomment and set:
|
||||||
|
```
|
||||||
|
[Resolve]
|
||||||
|
DNS=8.8.8.8 8.8.4.4 1.1.1.1
|
||||||
|
FallbackDNS=1.1.1.1 8.8.8.8
|
||||||
|
```
|
||||||
|
|
||||||
|
**If using Tailscale (DNS server 100.100.100.100):**
|
||||||
|
|
||||||
|
If your `/etc/resolv.conf` shows Tailscale DNS (`100.100.100.100`) but it's timing out, add fallback DNS servers:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Edit systemd-resolved config
|
||||||
|
sudo nano /etc/systemd/resolved.conf
|
||||||
|
```
|
||||||
|
|
||||||
|
Set:
|
||||||
|
```
|
||||||
|
[Resolve]
|
||||||
|
DNS=100.100.100.100 8.8.8.8 8.8.4.4 1.1.1.1
|
||||||
|
FallbackDNS=8.8.8.8 8.8.4.4 1.1.1.1
|
||||||
|
```
|
||||||
|
|
||||||
|
This keeps Tailscale DNS as primary (for Tailscale network access) but adds public DNS as fallback.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Restart systemd-resolved
|
||||||
|
sudo systemctl restart systemd-resolved
|
||||||
|
|
||||||
|
# Test DNS
|
||||||
|
nslookup github.com
|
||||||
|
ping -c 2 github.com
|
||||||
|
```
|
||||||
|
|
||||||
|
**Alternative: Use IP Address (Temporary Workaround)**
|
||||||
|
|
||||||
|
If DNS still doesn't work, you can manually update git remote:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Get GitHub IP
|
||||||
|
nslookup github.com 8.8.8.8
|
||||||
|
|
||||||
|
# Or use known GitHub IPs (may change)
|
||||||
|
# Update git remote to use IP (not recommended for long-term)
|
||||||
|
git remote set-url origin https://140.82.121.3/deepkoluguri/INSTITUTIONAL-FLOW-TRADING-PLATFORM.git
|
||||||
|
```
|
||||||
|
|
||||||
|
**Verify DNS is Working:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Test DNS resolution
|
||||||
|
nslookup github.com
|
||||||
|
dig github.com
|
||||||
|
|
||||||
|
# Test connectivity
|
||||||
|
ping -c 2 github.com
|
||||||
|
curl -I https://github.com
|
||||||
|
```
|
||||||
|
|
||||||
### Backend Won't Start
|
### Backend Won't Start
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|
@ -0,0 +1,863 @@
|
||||||
|
# Trading Playbook Implementation Suggestions
|
||||||
|
|
||||||
|
## Current State Analysis
|
||||||
|
|
||||||
|
### ✅ What You Already Have
|
||||||
|
- Badge system: 🟢/🔴, 💎, ⭐, 💰, ⚡, 🚀
|
||||||
|
- Rocket scoring algorithm
|
||||||
|
- Price context: RTH open, prior close, 5m/15m momentum
|
||||||
|
- Tape alignment detection
|
||||||
|
- Trade signal generation
|
||||||
|
- Session bucketing (PRE/RTH/POST)
|
||||||
|
- Premium filtering and aggregations
|
||||||
|
|
||||||
|
### ❌ What's Missing (High Impact)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## PART A — TRADING LOGIC ENHANCEMENTS
|
||||||
|
|
||||||
|
### 1️⃣ Signal Tier Classification System
|
||||||
|
|
||||||
|
**Current Gap:** All signals are treated equally. You need to classify them into Tier-1, Tier-2, and Ignore categories.
|
||||||
|
|
||||||
|
**Suggestion:**
|
||||||
|
- **Add a new service:** `backend/python_service/services/signal_tier_classifier.py`
|
||||||
|
- Classify signals based on badge combinations
|
||||||
|
- Tier-1: 🟢/🔴 + 💎 + ⭐ + premium > 500K + direction aligned
|
||||||
|
- Tier-2: 🟢 + 💎 (no ⭐) OR ⭐ without 💎
|
||||||
|
- Ignore: OTM-only, mixed signals, low volume/OI ratio
|
||||||
|
|
||||||
|
**Implementation Approach:**
|
||||||
|
```python
|
||||||
|
# In options_flow_processor.py, add after process_badges():
|
||||||
|
def classify_signal_tier(row):
|
||||||
|
badge_round = row.get('badge_round', '')
|
||||||
|
badge_more = row.get('badge_more', '')
|
||||||
|
premium = row.get('premium_num', 0) or 0
|
||||||
|
direction = row.get('direction', '')
|
||||||
|
bull_total = row.get('bull_total', 0) or 0
|
||||||
|
bear_total = row.get('bear_total', 0) or 0
|
||||||
|
|
||||||
|
has_diamond = '💎' in badge_more
|
||||||
|
has_star = '⭐' in badge_more
|
||||||
|
|
||||||
|
# Tier-1 conditions
|
||||||
|
if (badge_round in ['🟢', '🔴'] and
|
||||||
|
has_diamond and has_star and
|
||||||
|
premium > 500000):
|
||||||
|
# Check direction alignment
|
||||||
|
if (badge_round == '🟢' and direction == 'BULL' and (bull_total - bear_total) > 0):
|
||||||
|
return 'TIER_1'
|
||||||
|
elif (badge_round == '🔴' and direction == 'BEAR' and (bear_total - bull_total) > 0):
|
||||||
|
return 'TIER_1'
|
||||||
|
|
||||||
|
# Tier-2 conditions
|
||||||
|
if (badge_round == '🟢' and has_diamond and not has_star):
|
||||||
|
return 'TIER_2'
|
||||||
|
if (has_star and not has_diamond):
|
||||||
|
return 'TIER_2'
|
||||||
|
|
||||||
|
# Ignore conditions
|
||||||
|
# (Add logic for OTM-only, mixed signals, etc.)
|
||||||
|
|
||||||
|
return 'IGNORE'
|
||||||
|
```
|
||||||
|
|
||||||
|
**Database Addition:**
|
||||||
|
- Add `signal_tier` column to processed flow output
|
||||||
|
- Add `is_tradeable` boolean flag
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2️⃣ VWAP Integration
|
||||||
|
|
||||||
|
**Current Gap:** You have price context but no VWAP calculation or VWAP-based entry/exit logic.
|
||||||
|
|
||||||
|
**Suggestion:**
|
||||||
|
- **Extend:** `backend/python_service/services/price_context.py`
|
||||||
|
- Add `calculate_vwap()` method
|
||||||
|
- Calculate VWAP for each symbol on each trading day
|
||||||
|
- Store VWAP at signal time
|
||||||
|
- Calculate distance from VWAP (percentage)
|
||||||
|
|
||||||
|
**Implementation Approach:**
|
||||||
|
```python
|
||||||
|
# Add to PriceContextService:
|
||||||
|
async def get_vwap_at_time(self, symbol: str, timestamp: datetime, pool: asyncpg.Pool):
|
||||||
|
"""Calculate VWAP up to the given timestamp for the trading day"""
|
||||||
|
# Query all 1m bars from RTH open to timestamp
|
||||||
|
# Calculate: SUM(price * volume) / SUM(volume)
|
||||||
|
# Return VWAP value and distance from current price
|
||||||
|
```
|
||||||
|
|
||||||
|
**New Fields to Add:**
|
||||||
|
- `vwap_at_signal` - VWAP value at signal time
|
||||||
|
- `price_vs_vwap_pct` - Percentage distance from VWAP
|
||||||
|
- `vwap_reclaimed` - Boolean: did price reclaim VWAP after signal?
|
||||||
|
|
||||||
|
**Entry Strategy Integration:**
|
||||||
|
- Best entry: VWAP pullback or VWAP reclaim
|
||||||
|
- Good entry: Break & hold above prior high
|
||||||
|
- Avoid: Chasing vertical candles
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 3️⃣ Price Reaction Tracking (MOST IMPORTANT)
|
||||||
|
|
||||||
|
**Current Gap:** No tracking of how price moves AFTER the signal appears.
|
||||||
|
|
||||||
|
**Suggestion:**
|
||||||
|
- **New service:** `backend/python_service/services/price_reaction_tracker.py`
|
||||||
|
- Track price 5 minutes, 15 minutes, 30 minutes after signal
|
||||||
|
- Calculate price change percentage
|
||||||
|
- Identify if flow led to price movement or was just hedging
|
||||||
|
|
||||||
|
**Implementation Approach:**
|
||||||
|
```python
|
||||||
|
class PriceReactionTracker:
|
||||||
|
async def track_reaction(self, flow_row, pool):
|
||||||
|
signal_time = flow_row['flow_ts_utc']
|
||||||
|
symbol = flow_row['symbol_norm']
|
||||||
|
price_at_signal = flow_row['u_close']
|
||||||
|
|
||||||
|
# Get price 5m, 15m, 30m after signal
|
||||||
|
price_5m = await get_price_at_time(symbol, signal_time + timedelta(minutes=5))
|
||||||
|
price_15m = await get_price_at_time(symbol, signal_time + timedelta(minutes=15))
|
||||||
|
price_30m = await get_price_at_time(symbol, signal_time + timedelta(minutes=30))
|
||||||
|
|
||||||
|
# Calculate reactions
|
||||||
|
reaction_5m = ((price_5m - price_at_signal) / price_at_signal) * 100 if price_5m else None
|
||||||
|
reaction_15m = ((price_15m - price_at_signal) / price_at_signal) * 100 if price_15m else None
|
||||||
|
reaction_30m = ((price_30m - price_at_signal) / price_at_signal) * 100 if price_30m else None
|
||||||
|
|
||||||
|
# High/Low break confirmation
|
||||||
|
high_break = price_5m > flow_row.get('u_high', 0)
|
||||||
|
low_break = price_5m < flow_row.get('u_low', 0)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'price_reaction_5m_pct': reaction_5m,
|
||||||
|
'price_reaction_15m_pct': reaction_15m,
|
||||||
|
'price_reaction_30m_pct': reaction_30m,
|
||||||
|
'high_break_5m': high_break,
|
||||||
|
'low_break_5m': low_break,
|
||||||
|
'flow_led_to_move': reaction_5m and abs(reaction_5m) > 0.5 # 0.5% threshold
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Database Addition:**
|
||||||
|
- Add columns: `price_reaction_5m_pct`, `price_reaction_15m_pct`, `high_break_5m`, `low_break_5m`
|
||||||
|
- Add flag: `flow_led_to_move` (boolean)
|
||||||
|
|
||||||
|
**Why This Matters:**
|
||||||
|
- Flow without price reaction = hedge or roll (ignore)
|
||||||
|
- Flow with price reaction = real positioning (trade it)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 4️⃣ Strike Clustering Detection
|
||||||
|
|
||||||
|
**Current Gap:** No detection of multiple large trades at the same strike (institutional layering).
|
||||||
|
|
||||||
|
**Suggestion:**
|
||||||
|
- **New service:** `backend/python_service/services/strike_cluster_detector.py`
|
||||||
|
- Group trades by strike and expiration
|
||||||
|
- Identify clusters: 3+ trades at same strike within 30 minutes
|
||||||
|
- Calculate cluster premium total
|
||||||
|
- Flag as "institutional positioning" vs "single trade"
|
||||||
|
|
||||||
|
**Implementation Approach:**
|
||||||
|
```python
|
||||||
|
class StrikeClusterDetector:
|
||||||
|
def detect_clusters(self, df: pd.DataFrame, window_minutes: int = 30):
|
||||||
|
"""Detect strike clusters within time window"""
|
||||||
|
df = df.copy()
|
||||||
|
|
||||||
|
# Group by symbol, exp_date, strike
|
||||||
|
clusters = df.groupby(['symbol_norm', 'exp_date', 'strike_num']).apply(
|
||||||
|
lambda g: self._find_clusters_in_group(g, window_minutes)
|
||||||
|
)
|
||||||
|
|
||||||
|
return clusters
|
||||||
|
|
||||||
|
def _find_clusters_in_group(self, group, window_minutes):
|
||||||
|
"""Find time-based clusters within a strike group"""
|
||||||
|
# Sort by time
|
||||||
|
group = group.sort_values('flow_ts_utc')
|
||||||
|
|
||||||
|
# Rolling window: if 3+ trades within window_minutes, it's a cluster
|
||||||
|
# Return cluster flags and cluster IDs
|
||||||
|
```
|
||||||
|
|
||||||
|
**New Fields:**
|
||||||
|
- `is_cluster_trade` - Boolean
|
||||||
|
- `cluster_id` - Unique ID for the cluster
|
||||||
|
- `cluster_size` - Number of trades in cluster
|
||||||
|
- `cluster_total_premium` - Sum of all premiums in cluster
|
||||||
|
|
||||||
|
**Why This Matters:**
|
||||||
|
- Institutions rarely place one order — they layer
|
||||||
|
- Clusters = stronger signal than single prints
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 5️⃣ Gamma Exposure (GEX) Calculation
|
||||||
|
|
||||||
|
**Current Gap:** No gamma exposure tracking. This explains why some rockets fail.
|
||||||
|
|
||||||
|
**Suggestion:**
|
||||||
|
- **New service:** `backend/python_service/services/gamma_calculator.py`
|
||||||
|
- Calculate call GEX and put GEX per strike
|
||||||
|
- Net dealer gamma = Call GEX - Put GEX
|
||||||
|
- Positive GEX = price pinned (resistance)
|
||||||
|
- Negative GEX = explosive moves possible
|
||||||
|
|
||||||
|
**Implementation Approach:**
|
||||||
|
```python
|
||||||
|
class GammaCalculator:
|
||||||
|
def calculate_gex(self, df: pd.DataFrame):
|
||||||
|
"""
|
||||||
|
Calculate Gamma Exposure (GEX)
|
||||||
|
GEX = OI * Spot^2 * Gamma * 0.01 * Multiplier
|
||||||
|
Simplified: GEX ≈ OI * Spot^2 * 0.01 (for rough estimate)
|
||||||
|
"""
|
||||||
|
# For each strike, calculate:
|
||||||
|
# - Call GEX (positive for calls)
|
||||||
|
# - Put GEX (negative for puts)
|
||||||
|
# - Net GEX = Call GEX + Put GEX
|
||||||
|
|
||||||
|
# Add to flow row:
|
||||||
|
# - strike_gex (GEX at this strike)
|
||||||
|
# - net_dealer_gex (aggregate GEX for symbol)
|
||||||
|
# - gex_pin_level (strike with highest GEX)
|
||||||
|
```
|
||||||
|
|
||||||
|
**New Fields:**
|
||||||
|
- `strike_gex` - GEX at this strike
|
||||||
|
- `net_dealer_gex` - Net GEX for the symbol
|
||||||
|
- `gex_pin_level` - Strike where GEX is highest (pin level)
|
||||||
|
- `is_gex_positive` - Boolean: positive GEX = pinning, negative = explosive
|
||||||
|
|
||||||
|
**Why This Matters:**
|
||||||
|
- +GEX = Price pinned (rockets may fail at pin level)
|
||||||
|
- -GEX = Explosive moves (rockets more likely to work)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 6️⃣ Delta Weighting (Smart Money Filter)
|
||||||
|
|
||||||
|
**Current Gap:** No delta weighting. ITM delta > OTM lottery tickets.
|
||||||
|
|
||||||
|
**Suggestion:**
|
||||||
|
- **Extend:** `backend/python_service/services/options_flow_processor.py`
|
||||||
|
- Add delta calculation (approximate: use Black-Scholes or simplified formula)
|
||||||
|
- Calculate: `delta_weighted_premium = delta * volume * premium`
|
||||||
|
- Filter out low delta-weighted trades (YOLO prints)
|
||||||
|
|
||||||
|
**Implementation Approach:**
|
||||||
|
```python
|
||||||
|
def calculate_delta_weighted_value(row):
|
||||||
|
"""Calculate delta-weighted premium value"""
|
||||||
|
# Simplified delta approximation:
|
||||||
|
# For CALL: delta ≈ N(d1) where d1 = (ln(S/K) + (r+σ²/2)*T) / (σ*√T)
|
||||||
|
# For rough estimate: delta ≈ 0.5 for ATM, 0.8+ for ITM, 0.2- for OTM
|
||||||
|
|
||||||
|
spot = row.get('spot_num', 0)
|
||||||
|
strike = row.get('strike_num', 0)
|
||||||
|
cp = row.get('cp_norm', '')
|
||||||
|
moneyness = row.get('moneyness', '')
|
||||||
|
|
||||||
|
# Simplified delta based on moneyness
|
||||||
|
if moneyness == 'ITM':
|
||||||
|
delta = 0.7 if cp == 'CALL' else 0.7
|
||||||
|
elif moneyness == 'OTM':
|
||||||
|
delta = 0.3 if cp == 'CALL' else 0.3
|
||||||
|
else: # ATM
|
||||||
|
delta = 0.5
|
||||||
|
|
||||||
|
volume = row.get('vol_num', 0) or 0
|
||||||
|
premium = row.get('premium_num', 0) or 0
|
||||||
|
|
||||||
|
return delta * volume * premium
|
||||||
|
```
|
||||||
|
|
||||||
|
**New Fields:**
|
||||||
|
- `delta_approx` - Approximate delta value
|
||||||
|
- `delta_weighted_premium` - Delta * Volume * Premium
|
||||||
|
- `is_smart_money` - Boolean: delta_weighted_premium > threshold
|
||||||
|
|
||||||
|
**Why This Matters:**
|
||||||
|
- Filters out YOLO OTM lottery prints
|
||||||
|
- ITM delta > OTM = real positioning
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 7️⃣ Time-to-Expiration Buckets
|
||||||
|
|
||||||
|
**Current Gap:** No DTE-based classification.
|
||||||
|
|
||||||
|
**Suggestion:**
|
||||||
|
- **Extend:** `backend/python_service/services/options_flow_processor.py`
|
||||||
|
- Calculate DTE (days to expiration)
|
||||||
|
- Bucket into: 0DTE, 1-3 DTE, 7-14 DTE, Monthly
|
||||||
|
- Different logic per bucket
|
||||||
|
|
||||||
|
**Implementation Approach:**
|
||||||
|
```python
|
||||||
|
def calculate_dte_bucket(row):
|
||||||
|
"""Calculate days to expiration and bucket"""
|
||||||
|
exp_date = row.get('exp_date')
|
||||||
|
flow_date = row.get('flow_date_cst')
|
||||||
|
|
||||||
|
if not exp_date or not flow_date:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if isinstance(flow_date, datetime):
|
||||||
|
flow_date = flow_date.date()
|
||||||
|
if isinstance(exp_date, datetime):
|
||||||
|
exp_date = exp_date.date()
|
||||||
|
|
||||||
|
dte = (exp_date - flow_date).days
|
||||||
|
|
||||||
|
if dte == 0:
|
||||||
|
return '0DTE'
|
||||||
|
elif 1 <= dte <= 3:
|
||||||
|
return '1-3DTE'
|
||||||
|
elif 4 <= dte <= 6:
|
||||||
|
return '4-6DTE'
|
||||||
|
elif 7 <= dte <= 14:
|
||||||
|
return '7-14DTE'
|
||||||
|
elif 15 <= dte <= 30:
|
||||||
|
return 'MONTHLY'
|
||||||
|
else:
|
||||||
|
return 'LONG_TERM'
|
||||||
|
```
|
||||||
|
|
||||||
|
**New Fields:**
|
||||||
|
- `dte` - Days to expiration
|
||||||
|
- `dte_bucket` - Bucket classification
|
||||||
|
- `is_0dte` - Boolean flag
|
||||||
|
|
||||||
|
**Why This Matters:**
|
||||||
|
- 0DTE → intraday pressure (gamma risk)
|
||||||
|
- Longer DTE → directional thesis (less gamma risk)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 8️⃣ Sweep vs Block Detection
|
||||||
|
|
||||||
|
**Current Gap:** No distinction between sweeps (urgency) and blocks (positioning).
|
||||||
|
|
||||||
|
**Suggestion:**
|
||||||
|
- **New service:** `backend/python_service/services/trade_type_detector.py`
|
||||||
|
- Detect multiple trades at same strike/expiration within 2 seconds = SWEEP
|
||||||
|
- Single large trade = BLOCK
|
||||||
|
- Different trading implications
|
||||||
|
|
||||||
|
**Implementation Approach:**
|
||||||
|
```python
|
||||||
|
class TradeTypeDetector:
|
||||||
|
def detect_trade_type(self, df: pd.DataFrame):
|
||||||
|
"""Detect if trade is sweep or block"""
|
||||||
|
df = df.copy()
|
||||||
|
df = df.sort_values(['symbol_norm', 'exp_date', 'strike_num', 'flow_ts_utc'])
|
||||||
|
|
||||||
|
# Group by symbol, exp, strike
|
||||||
|
groups = df.groupby(['symbol_norm', 'exp_date', 'strike_num'])
|
||||||
|
|
||||||
|
def classify_group(group):
|
||||||
|
# If multiple trades within 2 seconds = sweep
|
||||||
|
# If single large trade = block
|
||||||
|
# Otherwise = regular trade
|
||||||
|
|
||||||
|
if len(group) == 1:
|
||||||
|
return 'BLOCK' if group.iloc[0]['premium_num'] > 500000 else 'REGULAR'
|
||||||
|
|
||||||
|
# Check time differences
|
||||||
|
time_diffs = group['flow_ts_utc'].diff().dt.total_seconds()
|
||||||
|
has_sweep = (time_diffs <= 2).any()
|
||||||
|
|
||||||
|
if has_sweep:
|
||||||
|
return 'SWEEP'
|
||||||
|
else:
|
||||||
|
return 'CLUSTER'
|
||||||
|
|
||||||
|
df['trade_type'] = groups.apply(classify_group).values
|
||||||
|
|
||||||
|
return df
|
||||||
|
```
|
||||||
|
|
||||||
|
**New Fields:**
|
||||||
|
- `trade_type` - 'SWEEP', 'BLOCK', 'CLUSTER', 'REGULAR'
|
||||||
|
- `is_sweep` - Boolean
|
||||||
|
- `is_block` - Boolean
|
||||||
|
|
||||||
|
**Why This Matters:**
|
||||||
|
- Sweeps = urgency (institutions hitting multiple exchanges)
|
||||||
|
- Blocks = positioning (single large order)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 9️⃣ Historical Win Rate Tracking
|
||||||
|
|
||||||
|
**Current Gap:** No tracking of which patterns actually work.
|
||||||
|
|
||||||
|
**Suggestion:**
|
||||||
|
- **New service:** `backend/python_service/services/pattern_analyzer.py`
|
||||||
|
- Track pattern → outcome mapping
|
||||||
|
- Calculate win rate per pattern
|
||||||
|
- Average return per pattern
|
||||||
|
- Max drawdown per pattern
|
||||||
|
|
||||||
|
**Database Addition:**
|
||||||
|
- **New table:** `signal_patterns_history`
|
||||||
|
- Columns: pattern_hash, signal_time, price_at_signal, price_5m_after, price_15m_after, outcome, return_pct
|
||||||
|
|
||||||
|
**Implementation Approach:**
|
||||||
|
```python
|
||||||
|
class PatternAnalyzer:
|
||||||
|
def track_pattern(self, flow_row, price_reaction):
|
||||||
|
"""Track pattern and outcome"""
|
||||||
|
pattern_hash = self._hash_pattern(flow_row)
|
||||||
|
|
||||||
|
# Store in database:
|
||||||
|
# - Pattern signature (badge combo + premium tier + DTE)
|
||||||
|
# - Outcome (price reaction)
|
||||||
|
# - Return percentage
|
||||||
|
|
||||||
|
def get_pattern_stats(self, pattern_hash):
|
||||||
|
"""Get historical stats for a pattern"""
|
||||||
|
# Query database for all instances of this pattern
|
||||||
|
# Calculate: win_rate, avg_return, max_drawdown
|
||||||
|
```
|
||||||
|
|
||||||
|
**New Fields:**
|
||||||
|
- `pattern_hash` - Unique identifier for pattern
|
||||||
|
- `historical_win_rate` - Win rate for this pattern
|
||||||
|
- `historical_avg_return` - Average return for this pattern
|
||||||
|
- `pattern_confidence` - Confidence based on historical performance
|
||||||
|
|
||||||
|
**Why This Matters:**
|
||||||
|
- Discover which patterns actually work
|
||||||
|
- 🚀🚀 without 💎 fails more often
|
||||||
|
- 🟢💎⭐ + VWAP reclaim wins most
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 🔟 Index & Correlation Filter
|
||||||
|
|
||||||
|
**Current Gap:** No SPY/QQQ/VIX alignment check.
|
||||||
|
|
||||||
|
**Suggestion:**
|
||||||
|
- **New service:** `backend/python_service/services/index_correlation.py`
|
||||||
|
- Fetch SPY/QQQ flow at signal time
|
||||||
|
- Check VIX direction
|
||||||
|
- Rule: Single stock flow works best when index agrees
|
||||||
|
|
||||||
|
**Implementation Approach:**
|
||||||
|
```python
|
||||||
|
class IndexCorrelationService:
|
||||||
|
async def check_index_alignment(self, flow_row, pool):
|
||||||
|
"""Check if index flow aligns with stock flow"""
|
||||||
|
symbol = flow_row['symbol_norm']
|
||||||
|
signal_time = flow_row['flow_ts_utc']
|
||||||
|
direction = flow_row['direction']
|
||||||
|
|
||||||
|
# Get SPY/QQQ flow in same time window
|
||||||
|
spy_flow = await self.get_index_flow('SPY', signal_time, pool)
|
||||||
|
qqq_flow = await self.get_index_flow('QQQ', signal_time, pool)
|
||||||
|
|
||||||
|
# Get VIX direction
|
||||||
|
vix_direction = await self.get_vix_direction(signal_time, pool)
|
||||||
|
|
||||||
|
# Check alignment
|
||||||
|
index_bullish = (spy_flow.get('net_premium', 0) > 0) or (qqq_flow.get('net_premium', 0) > 0)
|
||||||
|
index_bearish = (spy_flow.get('net_premium', 0) < 0) or (qqq_flow.get('net_premium', 0) < 0)
|
||||||
|
|
||||||
|
aligned = (
|
||||||
|
(direction == 'BULL' and index_bullish) or
|
||||||
|
(direction == 'BEAR' and index_bearish)
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'index_aligned': aligned,
|
||||||
|
'spy_flow_direction': 'BULL' if spy_flow.get('net_premium', 0) > 0 else 'BEAR',
|
||||||
|
'qqq_flow_direction': 'BULL' if qqq_flow.get('net_premium', 0) > 0 else 'BEAR',
|
||||||
|
'vix_direction': vix_direction
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**New Fields:**
|
||||||
|
- `index_aligned` - Boolean: does index flow agree?
|
||||||
|
- `spy_flow_direction` - SPY flow direction
|
||||||
|
- `qqq_flow_direction` - QQQ flow direction
|
||||||
|
- `vix_direction` - VIX direction (up/down)
|
||||||
|
|
||||||
|
**Why This Matters:**
|
||||||
|
- Single stock flow works best when index agrees
|
||||||
|
- Contrarian flow (stock vs index) = lower probability
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## PART B — TRADE CHECKLIST IMPLEMENTATION
|
||||||
|
|
||||||
|
### Trade Entry Checklist
|
||||||
|
|
||||||
|
**Suggestion:**
|
||||||
|
- **New service:** `backend/python_service/services/trade_checklist.py`
|
||||||
|
- Implement 5-point checklist
|
||||||
|
- Return checklist score (0-5)
|
||||||
|
- Only allow trades with 4/5 or 5/5
|
||||||
|
|
||||||
|
**Implementation Approach:**
|
||||||
|
```python
|
||||||
|
class TradeChecklist:
|
||||||
|
def evaluate(self, flow_row):
|
||||||
|
"""Evaluate trade checklist"""
|
||||||
|
checks = {
|
||||||
|
'has_direction': flow_row.get('badge_round') in ['🟢', '🔴'],
|
||||||
|
'has_diamond': '💎' in flow_row.get('badge_more', ''),
|
||||||
|
'has_star': '⭐' in flow_row.get('badge_more', ''),
|
||||||
|
'price_respects_vwap': self._check_vwap_respect(flow_row),
|
||||||
|
'index_confirms': flow_row.get('index_aligned', False)
|
||||||
|
}
|
||||||
|
|
||||||
|
score = sum(checks.values())
|
||||||
|
passed = score >= 4
|
||||||
|
|
||||||
|
return {
|
||||||
|
'checklist_score': score,
|
||||||
|
'checklist_passed': passed,
|
||||||
|
'checks': checks
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**New Fields:**
|
||||||
|
- `checklist_score` - 0-5 score
|
||||||
|
- `checklist_passed` - Boolean: 4/5 or 5/5
|
||||||
|
- `checklist_details` - JSON with individual check results
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## PART C — ENHANCED ENTRY/EXIT LOGIC
|
||||||
|
|
||||||
|
### Entry Strategy Enhancement
|
||||||
|
|
||||||
|
**Current Gap:** Entry logic exists but doesn't use VWAP pullback/reclaim.
|
||||||
|
|
||||||
|
**Suggestion:**
|
||||||
|
- **Extend:** `backend/src/services/tradePlanGenerator.js`
|
||||||
|
- Add VWAP pullback entry
|
||||||
|
- Add VWAP reclaim entry
|
||||||
|
- Add prior high break entry
|
||||||
|
- Avoid chasing vertical candles
|
||||||
|
|
||||||
|
**Implementation:**
|
||||||
|
```javascript
|
||||||
|
function generateEntryStrategy(signal, currentPrice, priceContext) {
|
||||||
|
const vwap = priceContext.vwap;
|
||||||
|
const priorHigh = priceContext.priorHigh;
|
||||||
|
const vwapDistance = ((currentPrice - vwap) / vwap) * 100;
|
||||||
|
|
||||||
|
if (signal === 'BUY') {
|
||||||
|
// Best: VWAP pullback or VWAP reclaim
|
||||||
|
if (currentPrice < vwap && vwapDistance > -1) {
|
||||||
|
return {
|
||||||
|
type: 'VWAP_PULLBACK',
|
||||||
|
entry: vwap * 0.998, // Slightly below VWAP
|
||||||
|
reason: 'VWAP pullback entry'
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Good: Break & hold above prior high
|
||||||
|
if (currentPrice > priorHigh) {
|
||||||
|
return {
|
||||||
|
type: 'BREAKOUT',
|
||||||
|
entry: priorHigh * 1.001, // Slightly above prior high
|
||||||
|
reason: 'Prior high breakout'
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Avoid: Chasing vertical candles
|
||||||
|
if (vwapDistance > 2) {
|
||||||
|
return {
|
||||||
|
type: 'WAIT',
|
||||||
|
reason: 'Price too extended from VWAP - wait for pullback'
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Similar for SELL signals...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Exit Strategy Enhancement
|
||||||
|
|
||||||
|
**Current Gap:** Exit logic is basic. Need flow-based exits.
|
||||||
|
|
||||||
|
**Suggestion:**
|
||||||
|
- **Extend:** `backend/src/services/tradePlanGenerator.js`
|
||||||
|
- Exit when flow stalls
|
||||||
|
- Exit when opposite 💎 appears
|
||||||
|
- Exit when net premium flips
|
||||||
|
- Exit when price rejects VWAP
|
||||||
|
- Scale out at +30-50% option gain
|
||||||
|
|
||||||
|
**Implementation:**
|
||||||
|
```javascript
|
||||||
|
function generateExitStrategy(signal, entryPrice, currentPrice, flowData) {
|
||||||
|
const exits = [];
|
||||||
|
|
||||||
|
// Flow stalls
|
||||||
|
if (flowData.recentFlowVolume < flowData.avgFlowVolume * 0.3) {
|
||||||
|
exits.push({
|
||||||
|
type: 'FLOW_STALL',
|
||||||
|
reason: 'Flow volume dropped significantly'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Opposite diamond appears
|
||||||
|
if (signal === 'BUY' && flowData.hasBearDiamond) {
|
||||||
|
exits.push({
|
||||||
|
type: 'OPPOSITE_SIGNAL',
|
||||||
|
reason: 'Bear diamond (💎) appeared - exit long'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Net premium flips
|
||||||
|
if (signal === 'BUY' && flowData.netPremium < 0) {
|
||||||
|
exits.push({
|
||||||
|
type: 'PREMIUM_FLIP',
|
||||||
|
reason: 'Net premium flipped negative'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Price rejects VWAP
|
||||||
|
if (currentPrice < priceContext.vwap && signal === 'BUY') {
|
||||||
|
exits.push({
|
||||||
|
type: 'VWAP_REJECTION',
|
||||||
|
reason: 'Price rejected VWAP - exit'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scale out at gains
|
||||||
|
const gainPct = ((currentPrice - entryPrice) / entryPrice) * 100;
|
||||||
|
if (gainPct >= 30) {
|
||||||
|
exits.push({
|
||||||
|
type: 'SCALE_OUT',
|
||||||
|
reason: `+${gainPct.toFixed(1)}% gain - scale out 50%`
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return exits;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## PART D — DATABASE SCHEMA ADDITIONS
|
||||||
|
|
||||||
|
### New Columns for `processed_options_flow` (or new enrichment table)
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Signal classification
|
||||||
|
signal_tier VARCHAR(10), -- 'TIER_1', 'TIER_2', 'IGNORE'
|
||||||
|
is_tradeable BOOLEAN,
|
||||||
|
|
||||||
|
-- VWAP
|
||||||
|
vwap_at_signal NUMERIC,
|
||||||
|
price_vs_vwap_pct NUMERIC,
|
||||||
|
vwap_reclaimed BOOLEAN,
|
||||||
|
|
||||||
|
-- Price reaction
|
||||||
|
price_reaction_5m_pct NUMERIC,
|
||||||
|
price_reaction_15m_pct NUMERIC,
|
||||||
|
price_reaction_30m_pct NUMERIC,
|
||||||
|
high_break_5m BOOLEAN,
|
||||||
|
low_break_5m BOOLEAN,
|
||||||
|
flow_led_to_move BOOLEAN,
|
||||||
|
|
||||||
|
-- Strike clustering
|
||||||
|
is_cluster_trade BOOLEAN,
|
||||||
|
cluster_id VARCHAR(50),
|
||||||
|
cluster_size INTEGER,
|
||||||
|
cluster_total_premium NUMERIC,
|
||||||
|
|
||||||
|
-- Gamma exposure
|
||||||
|
strike_gex NUMERIC,
|
||||||
|
net_dealer_gex NUMERIC,
|
||||||
|
gex_pin_level NUMERIC,
|
||||||
|
is_gex_positive BOOLEAN,
|
||||||
|
|
||||||
|
-- Delta weighting
|
||||||
|
delta_approx NUMERIC,
|
||||||
|
delta_weighted_premium NUMERIC,
|
||||||
|
is_smart_money BOOLEAN,
|
||||||
|
|
||||||
|
-- DTE
|
||||||
|
dte INTEGER,
|
||||||
|
dte_bucket VARCHAR(20),
|
||||||
|
is_0dte BOOLEAN,
|
||||||
|
|
||||||
|
-- Trade type
|
||||||
|
trade_type VARCHAR(20), -- 'SWEEP', 'BLOCK', 'CLUSTER', 'REGULAR'
|
||||||
|
is_sweep BOOLEAN,
|
||||||
|
is_block BOOLEAN,
|
||||||
|
|
||||||
|
-- Index correlation
|
||||||
|
index_aligned BOOLEAN,
|
||||||
|
spy_flow_direction VARCHAR(10),
|
||||||
|
qqq_flow_direction VARCHAR(10),
|
||||||
|
vix_direction VARCHAR(10),
|
||||||
|
|
||||||
|
-- Checklist
|
||||||
|
checklist_score INTEGER,
|
||||||
|
checklist_passed BOOLEAN,
|
||||||
|
checklist_details JSONB,
|
||||||
|
|
||||||
|
-- Pattern tracking
|
||||||
|
pattern_hash VARCHAR(100),
|
||||||
|
historical_win_rate NUMERIC,
|
||||||
|
historical_avg_return NUMERIC,
|
||||||
|
pattern_confidence NUMERIC
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## PART E — IMPLEMENTATION PRIORITY
|
||||||
|
|
||||||
|
### Phase 1 (Highest Impact - Do First)
|
||||||
|
1. ✅ **Price Reaction Tracking** - Most important filter
|
||||||
|
2. ✅ **VWAP Integration** - Critical for entry/exit
|
||||||
|
3. ✅ **Signal Tier Classification** - Filter noise
|
||||||
|
4. ✅ **Trade Checklist** - Prevent bad trades
|
||||||
|
|
||||||
|
### Phase 2 (High Value)
|
||||||
|
5. ✅ **Strike Clustering** - Identify institutional layering
|
||||||
|
6. ✅ **Delta Weighting** - Filter YOLO prints
|
||||||
|
7. ✅ **Index Correlation** - Context filter
|
||||||
|
|
||||||
|
### Phase 3 (Nice to Have)
|
||||||
|
8. ✅ **Gamma Exposure** - Explains pinning behavior
|
||||||
|
9. ✅ **Sweep vs Block** - Trade type classification
|
||||||
|
10. ✅ **DTE Buckets** - Time-based filtering
|
||||||
|
|
||||||
|
### Phase 4 (Analytics)
|
||||||
|
11. ✅ **Historical Win Rate** - Pattern analysis
|
||||||
|
12. ✅ **Enhanced Entry/Exit** - Refine trading logic
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## PART F — API ENDPOINT SUGGESTIONS
|
||||||
|
|
||||||
|
### New Endpoints to Add
|
||||||
|
|
||||||
|
1. **`GET /api/options-flow/enhanced`**
|
||||||
|
- Returns flow with all new enrichments
|
||||||
|
- Parameters: `include_price_reaction`, `include_gex`, etc.
|
||||||
|
|
||||||
|
2. **`GET /api/options-flow/checklist`**
|
||||||
|
- Returns only signals that pass checklist (4/5 or 5/5)
|
||||||
|
|
||||||
|
3. **`GET /api/options-flow/tier-1`**
|
||||||
|
- Returns only Tier-1 tradeable signals
|
||||||
|
|
||||||
|
4. **`GET /api/patterns/stats`**
|
||||||
|
- Returns historical win rates per pattern
|
||||||
|
|
||||||
|
5. **`GET /api/options-flow/vwap-analysis`**
|
||||||
|
- Returns VWAP-based entry opportunities
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## PART G — FRONTEND DISPLAY SUGGESTIONS
|
||||||
|
|
||||||
|
### New UI Elements to Add
|
||||||
|
|
||||||
|
1. **Signal Tier Badge**
|
||||||
|
- Display "TIER-1", "TIER-2", or "IGNORE" badge
|
||||||
|
- Color code: Green (Tier-1), Yellow (Tier-2), Gray (Ignore)
|
||||||
|
|
||||||
|
2. **Price Reaction Indicator**
|
||||||
|
- Show 5m/15m price reaction percentage
|
||||||
|
- Green if positive reaction, Red if negative
|
||||||
|
- "Flow Led to Move" indicator
|
||||||
|
|
||||||
|
3. **VWAP Distance Display**
|
||||||
|
- Show current price vs VWAP
|
||||||
|
- Visual indicator: Above/Below VWAP
|
||||||
|
- Entry opportunity: "VWAP Pullback" or "VWAP Reclaim"
|
||||||
|
|
||||||
|
4. **Checklist Score Display**
|
||||||
|
- Show checklist score (X/5)
|
||||||
|
- Green if passed (4/5+), Red if failed
|
||||||
|
- Expandable details showing each check
|
||||||
|
|
||||||
|
5. **Index Alignment Indicator**
|
||||||
|
- Show SPY/QQQ flow direction
|
||||||
|
- Show if aligned (green) or not (red)
|
||||||
|
|
||||||
|
6. **Gamma Pin Level**
|
||||||
|
- Display GEX pin level on chart
|
||||||
|
- Show if price is near pin (resistance)
|
||||||
|
|
||||||
|
7. **Strike Cluster Visualization**
|
||||||
|
- Show cluster size and total premium
|
||||||
|
- Highlight clustered strikes
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## PART H — TESTING SUGGESTIONS
|
||||||
|
|
||||||
|
### Test Cases to Add
|
||||||
|
|
||||||
|
1. **Price Reaction Tests**
|
||||||
|
- Test: Flow with no price reaction = should be filtered
|
||||||
|
- Test: Flow with 5m price reaction = should be tradeable
|
||||||
|
|
||||||
|
2. **Tier Classification Tests**
|
||||||
|
- Test: 🟢 + 💎 + ⭐ + premium > 500K = Tier-1
|
||||||
|
- Test: 🟢 + 💎 (no ⭐) = Tier-2
|
||||||
|
- Test: OTM-only = Ignore
|
||||||
|
|
||||||
|
3. **Checklist Tests**
|
||||||
|
- Test: 4/5 checks = passed
|
||||||
|
- Test: 3/5 checks = failed
|
||||||
|
|
||||||
|
4. **VWAP Tests**
|
||||||
|
- Test: VWAP pullback entry detection
|
||||||
|
- Test: VWAP reclaim entry detection
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## SUMMARY
|
||||||
|
|
||||||
|
### Key Takeaways
|
||||||
|
|
||||||
|
1. **Price Reaction is #1 Priority** - This filters out hedges/rolls
|
||||||
|
2. **VWAP Integration is Critical** - Needed for proper entry/exit
|
||||||
|
3. **Tier Classification Reduces Noise** - Focus on Tier-1 signals
|
||||||
|
4. **Checklist Prevents Bad Trades** - Enforce 4/5 minimum
|
||||||
|
5. **Strike Clustering Identifies Institutions** - Multiple trades = stronger signal
|
||||||
|
6. **Index Correlation Adds Context** - Single stock works best with index alignment
|
||||||
|
|
||||||
|
### Implementation Strategy
|
||||||
|
|
||||||
|
- Start with Phase 1 (Price Reaction + VWAP + Tier Classification + Checklist)
|
||||||
|
- These 4 features will have the biggest impact on trade quality
|
||||||
|
- Then move to Phase 2 for additional filtering
|
||||||
|
- Phase 3 and 4 can be added over time as analytics improve
|
||||||
|
|
||||||
|
### Expected Outcomes
|
||||||
|
|
||||||
|
- **Higher Win Rate**: Filtering out hedges/rolls and low-quality signals
|
||||||
|
- **Better Entries**: VWAP-based entry logic
|
||||||
|
- **Better Exits**: Flow-based exit signals
|
||||||
|
- **Reduced Noise**: Tier classification and checklist
|
||||||
|
- **Institutional Detection**: Strike clustering and delta weighting
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Note:** All suggestions are additive - they don't change existing code, just extend it with new services and enrichments.
|
||||||
|
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -553,13 +553,14 @@ SELECT
|
||||||
TO_CHAR(fe.flow_ts_local, 'HH12:MI:SS AM') AS "CreatedTime",
|
TO_CHAR(fe.flow_ts_local, 'HH12:MI:SS AM') AS "CreatedTime",
|
||||||
|
|
||||||
/* Symbol line with session + catalyst + your badges */
|
/* Symbol line with session + catalyst + your badges */
|
||||||
|
/* Use symbol_norm (clean symbol) instead of fe."Symbol" to avoid duplication */
|
||||||
(
|
(
|
||||||
CASE
|
CASE
|
||||||
WHEN fe.direction = 'BULL' THEN '(' || fe.dir_count || '🟩) '
|
WHEN fe.direction = 'BULL' THEN '(' || fe.dir_count || '🟩) '
|
||||||
WHEN fe.direction = 'BEAR' THEN '(' || fe.dir_count || '🟥) '
|
WHEN fe.direction = 'BEAR' THEN '(' || fe.dir_count || '🟥) '
|
||||||
ELSE ''
|
ELSE ''
|
||||||
END
|
END
|
||||||
|| fe."Symbol"
|
|| fe.symbol_norm
|
||||||
|| ' · ' || COALESCE(fe.session_bucket,'?')
|
|| ' · ' || COALESCE(fe.session_bucket,'?')
|
||||||
|| CASE WHEN fe.near_alert_type IS NOT NULL THEN ' ⚡' ELSE '' END
|
|| CASE WHEN fe.near_alert_type IS NOT NULL THEN ' ⚡' ELSE '' END
|
||||||
|| CASE WHEN (fe.badge_round || fe.badge_more) <> '' THEN ' ' || fe.badge_round || fe.badge_more ELSE '' END
|
|| CASE WHEN (fe.badge_round || fe.badge_more) <> '' THEN ' ' || fe.badge_round || fe.badge_more ELSE '' END
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,7 @@ import { QueryProfiler } from '../utils/queryProfiler.js';
|
||||||
import { calculateFlowMomentum, getMomentumLabel } from '../services/momentumScore.js';
|
import { calculateFlowMomentum, getMomentumLabel } from '../services/momentumScore.js';
|
||||||
import { getMoneynessContext, isGammaSqueezeSetup, getMoneynessLabel } from '../utils/moneynessHelper.js';
|
import { getMoneynessContext, isGammaSqueezeSetup, getMoneynessLabel } from '../utils/moneynessHelper.js';
|
||||||
import { batchFetchYahooFinanceData } from '../services/yahooFinanceService.js';
|
import { batchFetchYahooFinanceData } from '../services/yahooFinanceService.js';
|
||||||
|
import { fetchVolumeHistoryBulk } from '../services/volumeHistoryService.js';
|
||||||
|
|
||||||
const router = express.Router();
|
const router = express.Router();
|
||||||
|
|
||||||
|
|
@ -29,10 +30,16 @@ const USE_PYTHON_SERVICE = process.env.USE_PYTHON_SERVICE !== 'false';
|
||||||
|
|
||||||
router.get('/flow', async (req, res) => {
|
router.get('/flow', async (req, res) => {
|
||||||
try {
|
try {
|
||||||
const { startDate, endDate, minPremium: minPremiumParam = 80000, tolPct = 0.20 } = req.query;
|
const { startDate, endDate, minPremium: minPremiumParam = 80000, tolPct = 0.20, skipPrices = 'false' } = req.query;
|
||||||
// Parse minPremium to number (query params come as strings)
|
// Parse minPremium to number (query params come as strings)
|
||||||
const minPremium = parseFloat(minPremiumParam) || 80000;
|
const minPremium = parseFloat(minPremiumParam) || 80000;
|
||||||
|
|
||||||
|
// Ensure dates include full 24-hour day (00:00:00 to 23:59:59)
|
||||||
|
// SQL query uses date comparison, so we just need to ensure dates are properly formatted
|
||||||
|
// The SQL query's date BETWEEN clause will include the full day automatically
|
||||||
|
const normalizedStartDate = startDate ? startDate.split('T')[0] : null; // Extract just YYYY-MM-DD
|
||||||
|
const normalizedEndDate = endDate ? endDate.split('T')[0] : null; // Extract just YYYY-MM-DD
|
||||||
|
|
||||||
let rawData = [];
|
let rawData = [];
|
||||||
|
|
||||||
// Try Python service first (if enabled)
|
// Try Python service first (if enabled)
|
||||||
|
|
@ -73,18 +80,58 @@ router.get('/flow', async (req, res) => {
|
||||||
// Fallback to SQL if Python service not used or failed
|
// Fallback to SQL if Python service not used or failed
|
||||||
if (rawData.length === 0) {
|
if (rawData.length === 0) {
|
||||||
console.log('📊 Using SQL query (fallback or Python disabled)');
|
console.log('📊 Using SQL query (fallback or Python disabled)');
|
||||||
console.log(`📅 Date range: ${startDate} to ${endDate}`);
|
const queryStartDate = normalizedStartDate || startDate;
|
||||||
|
const queryEndDate = normalizedEndDate || endDate;
|
||||||
|
console.log(`📅 Date range: ${queryStartDate} to ${queryEndDate} (full 24-hour days: 00:00:00 to 23:59:59)`);
|
||||||
|
|
||||||
|
// First, check if there's any data in the database for these dates
|
||||||
try {
|
try {
|
||||||
// Note: SQL query has hardcoded minPremium=80000 in WHERE clause
|
const checkQuery = `
|
||||||
// We'll filter by minPremium in JavaScript instead
|
SELECT COUNT(*) as total_count,
|
||||||
|
COUNT(CASE WHEN "Premium" IS NOT NULL AND TRIM("Premium"::text) <> '' THEN 1 END) as with_premium,
|
||||||
|
COUNT(CASE WHEN "StockEtf" = 'STOCK' THEN 1 END) as stocks,
|
||||||
|
MIN("CreatedDate") as min_date,
|
||||||
|
MAX("CreatedDate") as max_date
|
||||||
|
FROM public."OptionsFlow_monthly"
|
||||||
|
WHERE "CreatedDate"::date >= $1::date
|
||||||
|
AND "CreatedDate"::date <= $2::date
|
||||||
|
`;
|
||||||
|
const checkResult = await rawQuery(checkQuery, [normalizedStartDate || startDate, normalizedEndDate || endDate]);
|
||||||
|
if (checkResult && checkResult.length > 0) {
|
||||||
|
const stats = checkResult[0];
|
||||||
|
console.log(`📊 Database stats for date range:`);
|
||||||
|
console.log(` Total rows: ${stats.total_count}`);
|
||||||
|
console.log(` With premium: ${stats.with_premium}`);
|
||||||
|
console.log(` Stocks: ${stats.stocks}`);
|
||||||
|
console.log(` Date range in DB: ${stats.min_date} to ${stats.max_date}`);
|
||||||
|
}
|
||||||
|
} catch (checkError) {
|
||||||
|
console.warn('⚠️ Could not check database stats:', checkError.message);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Use SQL query as source of truth - it has restrictive filters
|
||||||
// Profile the query execution
|
// Profile the query execution
|
||||||
|
// Pass normalized dates to ensure full 24-hour day coverage (00:00:00 to 23:59:59)
|
||||||
|
const queryStartDate = normalizedStartDate || startDate;
|
||||||
|
const queryEndDate = normalizedEndDate || endDate;
|
||||||
const { result, metrics } = await QueryProfiler.profile(
|
const { result, metrics } = await QueryProfiler.profile(
|
||||||
() => rawQuery(optionsFlowQuery, [startDate, endDate]),
|
() => rawQuery(optionsFlowQuery, [queryStartDate, queryEndDate]),
|
||||||
'optionsFlowQuery',
|
'optionsFlowQuery',
|
||||||
{ logSlowThreshold: 500 } // Log queries > 500ms
|
{ logSlowThreshold: 500 } // Log queries > 500ms
|
||||||
);
|
);
|
||||||
rawData = result;
|
rawData = result;
|
||||||
console.log(`✅ SQL query returned ${rawData.length} rows (${metrics.duration.toFixed(2)}ms)`);
|
console.log(`✅ SQL query (source of truth) returned ${rawData.length} rows (${metrics.duration.toFixed(2)}ms)`);
|
||||||
|
|
||||||
|
if (rawData.length === 0) {
|
||||||
|
console.warn('⚠️ SQL query returned 0 rows. The SQL query (source of truth) has restrictive filters:');
|
||||||
|
console.warn(' - premium_num > 80000');
|
||||||
|
console.warn(' - badge_round IN (🟢,🔴)');
|
||||||
|
console.warn(' - Must have 💎 (diamond) badge');
|
||||||
|
console.warn(' - Must have ⭐ (star) badge');
|
||||||
|
console.warn(' - Direction alignment (BULL/🟢/positive net OR BEAR/🔴/negative net)');
|
||||||
|
console.warn(' These filters are defined in the SQL query and are not modified.');
|
||||||
|
}
|
||||||
|
|
||||||
// Add performance metrics to response metadata
|
// Add performance metrics to response metadata
|
||||||
res.locals.queryMetrics = metrics;
|
res.locals.queryMetrics = metrics;
|
||||||
|
|
@ -153,9 +200,9 @@ router.get('/flow', async (req, res) => {
|
||||||
});
|
});
|
||||||
|
|
||||||
// Filter and sort
|
// Filter and sort
|
||||||
console.log(`📊 Raw data from SQL: ${rawData.length} rows`);
|
console.log(`📊 Raw data from SQL (source of truth): ${rawData.length} rows`);
|
||||||
console.log(`📊 After enrichment: ${enrichedData.length} rows`);
|
console.log(`📊 After enrichment: ${enrichedData.length} rows`);
|
||||||
console.log(`📊 Filtering with minPremium: ${minPremium}`);
|
console.log(`📊 SQL query already applied filters: premium > 80000, badges (🟢/🔴 + 💎 + ⭐), direction alignment`);
|
||||||
|
|
||||||
// Debug: show sample of enriched data
|
// Debug: show sample of enriched data
|
||||||
if (enrichedData.length > 0) {
|
if (enrichedData.length > 0) {
|
||||||
|
|
@ -169,33 +216,25 @@ router.get('/flow', async (req, res) => {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Filter by premium (compare premium_num against minPremium)
|
// Note: SQL query is the source of truth and already applies all restrictive filters:
|
||||||
const afterPremium = enrichedData.filter(row => {
|
// - premium_num > 80000
|
||||||
const premium = parseFloat(row.premium_num) || 0;
|
// - badge_round IN ('🟢','🔴')
|
||||||
return premium > minPremium;
|
// - Must have 💎 (diamond) badge
|
||||||
});
|
// - Must have ⭐ (star) badge
|
||||||
console.log(`📊 After premium filter (>${minPremium}): ${afterPremium.length} rows`);
|
// - Direction alignment (BULL/🟢/positive net OR BEAR/🔴/negative net)
|
||||||
|
// So we don't need to re-apply these filters here - SQL query is authoritative
|
||||||
|
|
||||||
// Filter by badges
|
// Only apply additional JavaScript-side filters if minPremium differs from SQL's hardcoded 80000
|
||||||
const afterBadges = afterPremium.filter(row => {
|
let filtered = enrichedData;
|
||||||
// Must have core badges - use badgesRaw object, not badges array
|
if (minPremium !== 80000) {
|
||||||
const badges = row.badgesRaw || {};
|
filtered = filtered.filter(row => {
|
||||||
const hasRound = badges.round === '🟢' || badges.round === '🔴';
|
const premium = parseFloat(row.premium_num) || 0;
|
||||||
const hasDiamond = badges.more && badges.more.includes('💎');
|
return premium > minPremium;
|
||||||
const hasStar = badges.more && badges.more.includes('⭐');
|
});
|
||||||
return hasRound && hasDiamond && hasStar;
|
console.log(`📊 Applied additional premium filter (>${minPremium}): ${filtered.length} rows`);
|
||||||
});
|
} else {
|
||||||
console.log(`📊 After badge filter (🟢/🔴 + 💎 + ⭐): ${afterBadges.length} rows`);
|
console.log(`📊 Using SQL query filters as-is (no additional filtering needed)`);
|
||||||
|
}
|
||||||
// Filter by direction/net premium alignment
|
|
||||||
const filtered = afterBadges.filter(row => {
|
|
||||||
// Direction must match net premium
|
|
||||||
const badges = row.badgesRaw || {};
|
|
||||||
const netPrem = (row.bull_total || 0) - (row.bear_total || 0);
|
|
||||||
return (row.direction === 'BULL' && badges.round === '🟢' && netPrem > 0) ||
|
|
||||||
(row.direction === 'BEAR' && badges.round === '🔴' && netPrem < 0);
|
|
||||||
});
|
|
||||||
console.log(`📊 After direction/net premium filter: ${filtered.length} rows`);
|
|
||||||
|
|
||||||
// Sort by timestamp descending
|
// Sort by timestamp descending
|
||||||
const sorted = filtered.sort((a, b) => {
|
const sorted = filtered.sort((a, b) => {
|
||||||
|
|
@ -219,17 +258,52 @@ router.get('/flow', async (req, res) => {
|
||||||
const flowReversals = batchDetectFlowReversals(tickerFlows, 30);
|
const flowReversals = batchDetectFlowReversals(tickerFlows, 30);
|
||||||
const flowTrends = batchDetectFlowTrends(tickerFlows);
|
const flowTrends = batchDetectFlowTrends(tickerFlows);
|
||||||
|
|
||||||
// Fetch stock price data for all unique symbols
|
// Fetch stock price data for all unique symbols (includes volume history)
|
||||||
|
// Skip if skipPrices=true for faster initial response
|
||||||
const uniqueSymbols = [...new Set(sorted.map(row => (row.symbol_norm || row.Symbol || '').toUpperCase()).filter(Boolean))];
|
const uniqueSymbols = [...new Set(sorted.map(row => (row.symbol_norm || row.Symbol || '').toUpperCase()).filter(Boolean))];
|
||||||
console.log(`📈 Fetching stock prices for ${uniqueSymbols.length} symbols from Yahoo Finance...`);
|
const shouldSkipPrices = skipPrices === 'true' || skipPrices === true;
|
||||||
|
|
||||||
let stockPrices = {};
|
let stockPrices = {};
|
||||||
|
let volumeHistory = {};
|
||||||
|
|
||||||
|
if (!shouldSkipPrices) {
|
||||||
|
console.log(`📈 Fetching stock prices and volume history for ${uniqueSymbols.length} symbols from Yahoo Finance...`);
|
||||||
try {
|
try {
|
||||||
stockPrices = await batchFetchYahooFinanceData(uniqueSymbols, 5);
|
stockPrices = await batchFetchYahooFinanceData(uniqueSymbols, 5);
|
||||||
console.log(`✅ Successfully fetched stock prices for ${Object.keys(stockPrices).length} symbols`);
|
console.log(`✅ Successfully fetched stock prices for ${Object.keys(stockPrices).length} symbols`);
|
||||||
|
|
||||||
|
// Extract volume history from stock price data
|
||||||
|
Object.keys(stockPrices).forEach(symbol => {
|
||||||
|
const priceData = stockPrices[symbol];
|
||||||
|
if (priceData && priceData.volumeHistory && priceData.volumeHistory.length > 0) {
|
||||||
|
volumeHistory[symbol] = priceData.volumeHistory;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
console.log(`✅ Extracted volume history for ${Object.keys(volumeHistory).length} symbols`);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn('⚠️ Error fetching stock prices from Yahoo Finance:', error.message);
|
console.warn('⚠️ Error fetching stock prices from Yahoo Finance:', error.message);
|
||||||
// Continue without stock prices if fetch fails
|
// Continue without stock prices if fetch fails
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: Fetch volume history from database for symbols that didn't get data from Yahoo Finance
|
||||||
|
const missingSymbols = uniqueSymbols.filter(s => !volumeHistory[s]);
|
||||||
|
if (missingSymbols.length > 0) {
|
||||||
|
console.log(`📊 Fetching volume history from database for ${missingSymbols.length} symbols...`);
|
||||||
|
try {
|
||||||
|
const dbVolumeHistory = await fetchVolumeHistoryBulk(missingSymbols);
|
||||||
|
// Merge database results
|
||||||
|
Object.keys(dbVolumeHistory).forEach(symbol => {
|
||||||
|
if (!volumeHistory[symbol] && dbVolumeHistory[symbol].length > 0) {
|
||||||
|
volumeHistory[symbol] = dbVolumeHistory[symbol];
|
||||||
|
}
|
||||||
|
});
|
||||||
|
console.log(`✅ Fetched volume history from database for ${Object.keys(dbVolumeHistory).length} additional symbols`);
|
||||||
|
} catch (error) {
|
||||||
|
console.warn('⚠️ Error fetching volume history from database:', error.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
console.log(`⏩ Skipping stock price/volume fetch for faster response (skipPrices=true)`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add flow decay, reversal, and trend info to rows, then regenerate trade signals with trend data
|
// Add flow decay, reversal, and trend info to rows, then regenerate trade signals with trend data
|
||||||
|
|
@ -251,6 +325,9 @@ router.get('/flow', async (req, res) => {
|
||||||
// Get stock price data for this symbol
|
// Get stock price data for this symbol
|
||||||
const stockPriceData = stockPrices[symbolUpper] || null;
|
const stockPriceData = stockPrices[symbolUpper] || null;
|
||||||
|
|
||||||
|
// Get volume history for this symbol
|
||||||
|
const volumeHistoryData = volumeHistory[symbolUpper] || [];
|
||||||
|
|
||||||
const rowWithFlowInfo = {
|
const rowWithFlowInfo = {
|
||||||
...row,
|
...row,
|
||||||
flowDecayRaw: decay,
|
flowDecayRaw: decay,
|
||||||
|
|
@ -266,7 +343,9 @@ router.get('/flow', async (req, res) => {
|
||||||
momentumColor: momentumLabel.color,
|
momentumColor: momentumLabel.color,
|
||||||
momentumIcon: momentumLabel.icon,
|
momentumIcon: momentumLabel.icon,
|
||||||
// Add stock price data
|
// Add stock price data
|
||||||
stockPrice: stockPriceData
|
stockPrice: stockPriceData,
|
||||||
|
// Add volume history (last 5 days)
|
||||||
|
volumeHistory: volumeHistoryData
|
||||||
};
|
};
|
||||||
|
|
||||||
// Regenerate trade signal with flow trend data and moneyness context
|
// Regenerate trade signal with flow trend data and moneyness context
|
||||||
|
|
@ -315,16 +394,21 @@ router.get('/flow', async (req, res) => {
|
||||||
|
|
||||||
// Helper functions
|
// Helper functions
|
||||||
function formatSymbolDisplay(row, badges) {
|
function formatSymbolDisplay(row, badges) {
|
||||||
|
// Use symbol_norm (raw symbol) instead of Symbol (which may already be formatted by SQL)
|
||||||
|
const rawSymbol = row.symbol_norm || (row.Symbol && !row.Symbol.includes('·') ? row.Symbol : null) || 'UNKNOWN';
|
||||||
|
|
||||||
const dirCount = row.direction === 'BULL'
|
const dirCount = row.direction === 'BULL'
|
||||||
? `(${row.dir_count}🟩)`
|
? `(${row.dir_count}🟩)`
|
||||||
: `(${row.dir_count}🟥)`;
|
: row.direction === 'BEAR'
|
||||||
|
? `(${row.dir_count}🟥)`
|
||||||
|
: '';
|
||||||
|
|
||||||
const catalyst = row.near_alert_type ? ' ⚡' : '';
|
const catalyst = row.near_alert_type ? ' ⚡' : '';
|
||||||
const badgeStr = (badges.round + badges.more + badges.flash).trim();
|
const badgeStr = (badges.round + badges.more + badges.flash).trim();
|
||||||
const fire = row.premium_num > 1000000 ? ' 🔥' :
|
const fire = row.premium_num > 1000000 ? ' 🔥' :
|
||||||
row.premium_num > 500000 ? ' 💵' : '';
|
row.premium_num > 500000 ? ' 💵' : '';
|
||||||
|
|
||||||
return `${dirCount} ${row.Symbol || row.symbol_norm} · ${row.session_bucket || '?'}${catalyst}${badgeStr ? ' ' + badgeStr : ''}${fire}`;
|
return `${dirCount}${dirCount ? ' ' : ''}${rawSymbol} · ${row.session_bucket || '?'}${catalyst}${badgeStr ? ' ' + badgeStr : ''}${fire}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function formatRocketDisplay(row, score, badges) {
|
function formatRocketDisplay(row, score, badges) {
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,94 @@
|
||||||
|
import express from 'express';
|
||||||
|
import { batchFetchYahooFinanceData } from '../services/yahooFinanceService.js';
|
||||||
|
import { fetchVolumeHistoryBulk } from '../services/volumeHistoryService.js';
|
||||||
|
|
||||||
|
const router = express.Router();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/stock-prices
|
||||||
|
* Fetch stock prices and volume history for multiple symbols
|
||||||
|
* Query params: symbols (comma-separated list)
|
||||||
|
*/
|
||||||
|
router.get('/', async (req, res) => {
|
||||||
|
try {
|
||||||
|
const { symbols } = req.query;
|
||||||
|
|
||||||
|
if (!symbols) {
|
||||||
|
return res.status(400).json({
|
||||||
|
success: false,
|
||||||
|
error: 'symbols parameter is required (comma-separated list)'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const symbolList = symbols.split(',').map(s => s.trim().toUpperCase()).filter(Boolean);
|
||||||
|
|
||||||
|
if (symbolList.length === 0) {
|
||||||
|
return res.json({
|
||||||
|
success: true,
|
||||||
|
data: {}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`📈 Fetching stock prices and volume for ${symbolList.length} symbols...`);
|
||||||
|
|
||||||
|
let stockPrices = {};
|
||||||
|
let volumeHistory = {};
|
||||||
|
|
||||||
|
// Fetch from Yahoo Finance
|
||||||
|
try {
|
||||||
|
stockPrices = await batchFetchYahooFinanceData(symbolList, 5);
|
||||||
|
console.log(`✅ Successfully fetched stock prices for ${Object.keys(stockPrices).length} symbols`);
|
||||||
|
|
||||||
|
// Extract volume history from stock price data
|
||||||
|
Object.keys(stockPrices).forEach(symbol => {
|
||||||
|
const priceData = stockPrices[symbol];
|
||||||
|
if (priceData && priceData.volumeHistory && priceData.volumeHistory.length > 0) {
|
||||||
|
volumeHistory[symbol] = priceData.volumeHistory;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
console.log(`✅ Extracted volume history for ${Object.keys(volumeHistory).length} symbols`);
|
||||||
|
} catch (error) {
|
||||||
|
console.warn('⚠️ Error fetching stock prices from Yahoo Finance:', error.message);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: Fetch volume history from database for symbols that didn't get data from Yahoo Finance
|
||||||
|
const missingSymbols = symbolList.filter(s => !volumeHistory[s]);
|
||||||
|
if (missingSymbols.length > 0) {
|
||||||
|
console.log(`📊 Fetching volume history from database for ${missingSymbols.length} symbols...`);
|
||||||
|
try {
|
||||||
|
const dbVolumeHistory = await fetchVolumeHistoryBulk(missingSymbols);
|
||||||
|
// Merge database results
|
||||||
|
Object.keys(dbVolumeHistory).forEach(symbol => {
|
||||||
|
if (!volumeHistory[symbol] && dbVolumeHistory[symbol].length > 0) {
|
||||||
|
volumeHistory[symbol] = dbVolumeHistory[symbol];
|
||||||
|
}
|
||||||
|
});
|
||||||
|
console.log(`✅ Fetched volume history from database for ${Object.keys(dbVolumeHistory).length} additional symbols`);
|
||||||
|
} catch (error) {
|
||||||
|
console.warn('⚠️ Error fetching volume history from database:', error.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Combine stock prices and volume history
|
||||||
|
const result = {};
|
||||||
|
symbolList.forEach(symbol => {
|
||||||
|
result[symbol] = {
|
||||||
|
stockPrice: stockPrices[symbol] || null,
|
||||||
|
volumeHistory: volumeHistory[symbol] || []
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
res.json({
|
||||||
|
success: true,
|
||||||
|
data: result,
|
||||||
|
timestamp: new Date().toISOString()
|
||||||
|
});
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Stock prices error:', error);
|
||||||
|
res.status(500).json({ success: false, error: error.message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
export default router;
|
||||||
|
|
||||||
|
|
@ -4,6 +4,7 @@ import dotenv from 'dotenv';
|
||||||
import optionsFlowRouter from './routes/optionsFlow.js';
|
import optionsFlowRouter from './routes/optionsFlow.js';
|
||||||
import dailyAnalysisRouter from './routes/dailyAnalysis.js';
|
import dailyAnalysisRouter from './routes/dailyAnalysis.js';
|
||||||
import pricesRouter from './routes/prices.js';
|
import pricesRouter from './routes/prices.js';
|
||||||
|
import stockPricesRouter from './routes/stockPrices.js';
|
||||||
import alertsRouter, { setupAlertsWebSocket } from './routes/alerts.js';
|
import alertsRouter, { setupAlertsWebSocket } from './routes/alerts.js';
|
||||||
import scannerRouter from './routes/scanner.js';
|
import scannerRouter from './routes/scanner.js';
|
||||||
import tradePlansRouter from './routes/tradePlans.js';
|
import tradePlansRouter from './routes/tradePlans.js';
|
||||||
|
|
@ -42,6 +43,7 @@ app.use(express.json());
|
||||||
app.use('/api/options', optionsFlowRouter);
|
app.use('/api/options', optionsFlowRouter);
|
||||||
app.use('/api/analysis', dailyAnalysisRouter);
|
app.use('/api/analysis', dailyAnalysisRouter);
|
||||||
app.use('/api/prices', pricesRouter);
|
app.use('/api/prices', pricesRouter);
|
||||||
|
app.use('/api/stock-prices', stockPricesRouter);
|
||||||
app.use('/api/alerts', alertsRouter);
|
app.use('/api/alerts', alertsRouter);
|
||||||
app.use('/api/scanner', scannerRouter);
|
app.use('/api/scanner', scannerRouter);
|
||||||
app.use('/api/trade-plans', tradePlansRouter);
|
app.use('/api/trade-plans', tradePlansRouter);
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,144 @@
|
||||||
|
/**
|
||||||
|
* Volume History Service
|
||||||
|
* Fetches last 5 days of volume data from database (used as fallback when Yahoo Finance data unavailable)
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { rawQuery } from '../db.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch last 5 days of volume data for a single symbol
|
||||||
|
* @param {string} symbol - Stock symbol (e.g., 'AAPL')
|
||||||
|
* @returns {Promise<Array>} Array of volume data objects with date and volume
|
||||||
|
*/
|
||||||
|
export async function fetchVolumeHistory(symbol) {
|
||||||
|
try {
|
||||||
|
const query = `
|
||||||
|
SELECT
|
||||||
|
"Date" as date,
|
||||||
|
volume,
|
||||||
|
close
|
||||||
|
FROM public.prices_daily
|
||||||
|
WHERE UPPER(symbol) = UPPER($1)
|
||||||
|
AND "Date" <= CURRENT_DATE
|
||||||
|
AND "Date" >= CURRENT_DATE - INTERVAL '5 days'
|
||||||
|
ORDER BY "Date" DESC
|
||||||
|
LIMIT 5
|
||||||
|
`;
|
||||||
|
|
||||||
|
const data = await rawQuery(query, [symbol]);
|
||||||
|
|
||||||
|
// Format the data
|
||||||
|
return data.map(row => ({
|
||||||
|
date: row.date,
|
||||||
|
volume: parseFloat(row.volume) || 0,
|
||||||
|
close: parseFloat(row.close) || 0
|
||||||
|
}));
|
||||||
|
} catch (error) {
|
||||||
|
console.warn(`Failed to fetch volume history for ${symbol}:`, error.message);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Batch fetch volume history for multiple symbols
|
||||||
|
* @param {string[]} symbols - Array of stock symbols
|
||||||
|
* @param {number} concurrency - Number of concurrent requests (default: 10)
|
||||||
|
* @returns {Promise<Object>} Map of symbol to volume history array
|
||||||
|
*/
|
||||||
|
export async function batchFetchVolumeHistory(symbols, concurrency = 10) {
|
||||||
|
if (!symbols || symbols.length === 0) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove duplicates and normalize symbols
|
||||||
|
const uniqueSymbols = [...new Set(symbols.map(s => s.toUpperCase().trim()))].filter(Boolean);
|
||||||
|
|
||||||
|
if (uniqueSymbols.length === 0) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
const volumeHistoryMap = {};
|
||||||
|
|
||||||
|
// Process in batches to avoid overwhelming the database
|
||||||
|
for (let i = 0; i < uniqueSymbols.length; i += concurrency) {
|
||||||
|
const batch = uniqueSymbols.slice(i, i + concurrency);
|
||||||
|
const batchPromises = batch.map(async (symbol) => {
|
||||||
|
const data = await fetchVolumeHistory(symbol);
|
||||||
|
return { symbol, data };
|
||||||
|
});
|
||||||
|
|
||||||
|
const batchResults = await Promise.allSettled(batchPromises);
|
||||||
|
|
||||||
|
batchResults.forEach((result) => {
|
||||||
|
if (result.status === 'fulfilled') {
|
||||||
|
const { symbol, data } = result.value;
|
||||||
|
if (data && data.length > 0) {
|
||||||
|
volumeHistoryMap[symbol] = data;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
console.warn(`Failed to fetch volume history for symbol in batch:`, result.reason);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return volumeHistoryMap;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch volume history for multiple symbols from database (fallback when Yahoo Finance unavailable)
|
||||||
|
* @param {string[]} symbols - Array of stock symbols
|
||||||
|
* @returns {Promise<Object>} Map of symbol to volume history array
|
||||||
|
*/
|
||||||
|
export async function fetchVolumeHistoryBulk(symbols) {
|
||||||
|
if (!symbols || symbols.length === 0) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove duplicates and normalize symbols
|
||||||
|
const uniqueSymbols = [...new Set(symbols.map(s => s.toUpperCase().trim()))].filter(Boolean);
|
||||||
|
|
||||||
|
if (uniqueSymbols.length === 0) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
const volumeHistoryMap = {};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const query = `
|
||||||
|
SELECT
|
||||||
|
UPPER(symbol) as symbol,
|
||||||
|
"Date" as date,
|
||||||
|
volume,
|
||||||
|
close
|
||||||
|
FROM public.prices_daily
|
||||||
|
WHERE UPPER(symbol) = ANY($1::text[])
|
||||||
|
AND "Date" <= CURRENT_DATE
|
||||||
|
AND "Date" >= CURRENT_DATE - INTERVAL '5 days'
|
||||||
|
ORDER BY symbol, "Date" DESC
|
||||||
|
`;
|
||||||
|
|
||||||
|
const data = await rawQuery(query, [uniqueSymbols]);
|
||||||
|
|
||||||
|
// Group by symbol
|
||||||
|
data.forEach(row => {
|
||||||
|
const symbol = row.symbol;
|
||||||
|
if (!volumeHistoryMap[symbol]) {
|
||||||
|
volumeHistoryMap[symbol] = [];
|
||||||
|
}
|
||||||
|
// Only keep the last 5 days per symbol
|
||||||
|
if (volumeHistoryMap[symbol].length < 5) {
|
||||||
|
volumeHistoryMap[symbol].push({
|
||||||
|
date: row.date,
|
||||||
|
volume: parseFloat(row.volume) || 0,
|
||||||
|
close: parseFloat(row.close) || 0
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return volumeHistoryMap;
|
||||||
|
} catch (error) {
|
||||||
|
console.warn('⚠️ Failed to fetch bulk volume history from database:', error.message);
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@ -24,6 +24,37 @@ export async function fetchYahooFinanceData(symbol) {
|
||||||
const result = data.chart.result[0];
|
const result = data.chart.result[0];
|
||||||
const meta = result.meta;
|
const meta = result.meta;
|
||||||
const quotes = result.indicators?.quote?.[0];
|
const quotes = result.indicators?.quote?.[0];
|
||||||
|
const timestamps = result.timestamp || [];
|
||||||
|
|
||||||
|
// Extract historical volume data (last 5 days)
|
||||||
|
const volumeHistory = [];
|
||||||
|
if (quotes && quotes.volume && timestamps.length > 0) {
|
||||||
|
const volumes = quotes.volume || [];
|
||||||
|
const closes = quotes.close || [];
|
||||||
|
const opens = quotes.open || [];
|
||||||
|
const highs = quotes.high || [];
|
||||||
|
const lows = quotes.low || [];
|
||||||
|
|
||||||
|
// Get the last 5 days (or all available if less than 5)
|
||||||
|
const startIdx = Math.max(0, timestamps.length - 5);
|
||||||
|
for (let i = startIdx; i < timestamps.length; i++) {
|
||||||
|
if (volumes[i] !== null && volumes[i] !== undefined) {
|
||||||
|
const timestamp = timestamps[i];
|
||||||
|
const date = new Date(timestamp * 1000);
|
||||||
|
volumeHistory.push({
|
||||||
|
date: date.toISOString().split('T')[0], // Convert to YYYY-MM-DD
|
||||||
|
volume: Math.round(volumes[i]) || 0,
|
||||||
|
close: closes[i] || 0,
|
||||||
|
open: opens[i] || 0,
|
||||||
|
high: highs[i] || 0,
|
||||||
|
low: lows[i] || 0
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by date descending (most recent first)
|
||||||
|
volumeHistory.sort((a, b) => new Date(b.date) - new Date(a.date));
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
symbol: symbol.toUpperCase(),
|
symbol: symbol.toUpperCase(),
|
||||||
|
|
@ -42,6 +73,8 @@ export async function fetchYahooFinanceData(symbol) {
|
||||||
: 0,
|
: 0,
|
||||||
// Get recent price history
|
// Get recent price history
|
||||||
recentPrices: quotes?.close?.slice(-5) || [],
|
recentPrices: quotes?.close?.slice(-5) || [],
|
||||||
|
// Get volume history (last 5 days)
|
||||||
|
volumeHistory: volumeHistory,
|
||||||
timestamp: new Date().toISOString()
|
timestamp: new Date().toISOString()
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ import PhaseClassifierPanel from '@/components/dashboard/PhaseClassifierPanel';
|
||||||
import AlertsFeed from '@/components/dashboard/AlertsFeed';
|
import AlertsFeed from '@/components/dashboard/AlertsFeed';
|
||||||
import Watchlist from '@/components/dashboard/Watchlist';
|
import Watchlist from '@/components/dashboard/Watchlist';
|
||||||
import PerformanceTrackingPanel from '@/components/dashboard/PerformanceTrackingPanel';
|
import PerformanceTrackingPanel from '@/components/dashboard/PerformanceTrackingPanel';
|
||||||
|
import FlowInfoPanel from '@/components/dashboard/FlowInfoPanel';
|
||||||
|
|
||||||
export default function App() {
|
export default function App() {
|
||||||
return (
|
return (
|
||||||
|
|
@ -74,13 +75,18 @@ export default function App() {
|
||||||
</Tabs>
|
</Tabs>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Right: Alerts Feed & Watchlist */}
|
{/* Right: Flow Info, Watchlist & Alerts Feed */}
|
||||||
<div className="col-span-3 space-y-6">
|
<div className="col-span-3 space-y-6">
|
||||||
<PerformanceTrackingPanel />
|
<FlowInfoPanel />
|
||||||
<Watchlist />
|
<Watchlist />
|
||||||
<AlertsFeed />
|
<AlertsFeed />
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Bottom: Today's Signals */}
|
||||||
|
<div className="mt-6">
|
||||||
|
<PerformanceTrackingPanel />
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,186 @@
|
||||||
|
import { useState } from 'react';
|
||||||
|
import { Info, ChevronDown, ChevronUp } from 'lucide-react';
|
||||||
|
|
||||||
|
export default function FlowInfoPanel() {
|
||||||
|
const [isExpanded, setIsExpanded] = useState(false);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="bg-slate-900/50 backdrop-blur-sm rounded-lg border border-slate-800/50 p-4">
|
||||||
|
<button
|
||||||
|
onClick={() => setIsExpanded(!isExpanded)}
|
||||||
|
className="w-full flex items-center justify-between text-left"
|
||||||
|
>
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<Info className="w-5 h-5 text-blue-400" />
|
||||||
|
<h3 className="font-semibold text-slate-100">📚 Flow Data Legend & Info</h3>
|
||||||
|
</div>
|
||||||
|
{isExpanded ? (
|
||||||
|
<ChevronUp className="w-5 h-5 text-slate-400" />
|
||||||
|
) : (
|
||||||
|
<ChevronDown className="w-5 h-5 text-slate-400" />
|
||||||
|
)}
|
||||||
|
</button>
|
||||||
|
|
||||||
|
{isExpanded && (
|
||||||
|
<div className="mt-4 space-y-4 text-sm">
|
||||||
|
{/* Badges Section */}
|
||||||
|
<div>
|
||||||
|
<h4 className="text-slate-200 font-semibold mb-2">🎯 Badges & Indicators</h4>
|
||||||
|
<div className="grid grid-cols-1 md:grid-cols-2 gap-2 text-xs">
|
||||||
|
<div className="flex items-start gap-2">
|
||||||
|
<span className="text-lg">⚡</span>
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Flash:</span>
|
||||||
|
<span className="text-slate-400 ml-1">Aggressive sweep (AA/BB) with premium > $10K</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-start gap-2">
|
||||||
|
<span className="text-lg">🟢</span>
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Green Circle:</span>
|
||||||
|
<span className="text-slate-400 ml-1">Bullish ITM premium dominance</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-start gap-2">
|
||||||
|
<span className="text-lg">🔴</span>
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Red Circle:</span>
|
||||||
|
<span className="text-slate-400 ml-1">Bearish ITM premium dominance</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-start gap-2">
|
||||||
|
<span className="text-lg">💎</span>
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Diamond:</span>
|
||||||
|
<span className="text-slate-400 ml-1">ITM premium dominance in direction</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-start gap-2">
|
||||||
|
<span className="text-lg">⭐</span>
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Star:</span>
|
||||||
|
<span className="text-slate-400 ml-1">OTM flow spread > $10K</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-start gap-2">
|
||||||
|
<span className="text-lg">💰</span>
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Money:</span>
|
||||||
|
<span className="text-slate-400 ml-1">Open Interest accumulation > $100K</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-start gap-2">
|
||||||
|
<span className="text-lg">✔</span>
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Check:</span>
|
||||||
|
<span className="text-slate-400 ml-1">Volume > OI (new positioning)</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-start gap-2">
|
||||||
|
<span className="text-lg">🔥</span>
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Fire:</span>
|
||||||
|
<span className="text-slate-400 ml-1">Premium > $1M</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-start gap-2">
|
||||||
|
<span className="text-lg">💵</span>
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Cash:</span>
|
||||||
|
<span className="text-slate-400 ml-1">Premium > $500K</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-start gap-2">
|
||||||
|
<span className="text-lg">🚀</span>
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Rocket:</span>
|
||||||
|
<span className="text-slate-400 ml-1">Single rocket - Vol > OI + (Flash OR Premium > $500K)</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-start gap-2">
|
||||||
|
<span className="text-lg">🚀🚀</span>
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Double Rocket:</span>
|
||||||
|
<span className="text-slate-400 ml-1">Vol > OI + Flash + Money badge</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-start gap-2">
|
||||||
|
<span className="text-lg">🚀🚀🚀</span>
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Triple Rocket:</span>
|
||||||
|
<span className="text-slate-400 ml-1">Vol > OI + Flash + Money + Premium > $500K</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Trading Terms */}
|
||||||
|
<div>
|
||||||
|
<h4 className="text-slate-200 font-semibold mb-2">📖 Trading Terms</h4>
|
||||||
|
<div className="grid grid-cols-1 md:grid-cols-2 gap-2 text-xs">
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Side:</span>
|
||||||
|
<ul className="text-slate-400 ml-4 mt-1 space-y-0.5">
|
||||||
|
<li>• <strong>A/AA:</strong> Ask / Above Ask (aggressive buy)</li>
|
||||||
|
<li>• <strong>B/BB:</strong> Bid / Below Bid (aggressive sell)</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Moneyness:</span>
|
||||||
|
<ul className="text-slate-400 ml-4 mt-1 space-y-0.5">
|
||||||
|
<li>• <strong>ITM:</strong> In The Money</li>
|
||||||
|
<li>• <strong>OTM:</strong> Out The Money</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Option Type:</span>
|
||||||
|
<ul className="text-slate-400 ml-4 mt-1 space-y-0.5">
|
||||||
|
<li>• <strong>CALL:</strong> Right to buy at strike</li>
|
||||||
|
<li>• <strong>PUT:</strong> Right to sell at strike</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Direction:</span>
|
||||||
|
<ul className="text-slate-400 ml-4 mt-1 space-y-0.5">
|
||||||
|
<li>• <strong>BULL:</strong> Call Buy or Put Sell</li>
|
||||||
|
<li>• <strong>BEAR:</strong> Put Buy or Call Sell</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Sessions:</span>
|
||||||
|
<ul className="text-slate-400 ml-4 mt-1 space-y-0.5">
|
||||||
|
<li>• <strong>PRE:</strong> Pre-market (4:00 AM - 9:30 AM)</li>
|
||||||
|
<li>• <strong>RTH:</strong> Regular Trading Hours (9:30 AM - 4:00 PM)</li>
|
||||||
|
<li>• <strong>POST:</strong> After-hours (4:00 PM - 8:00 PM)</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<span className="text-slate-300 font-medium">Key Metrics:</span>
|
||||||
|
<ul className="text-slate-400 ml-4 mt-1 space-y-0.5">
|
||||||
|
<li>• <strong>Premium:</strong> Total dollar value of trade</li>
|
||||||
|
<li>• <strong>Volume:</strong> Contracts traded</li>
|
||||||
|
<li>• <strong>OI:</strong> Open Interest (outstanding contracts)</li>
|
||||||
|
<li>• <strong>Net Premium:</strong> Bull total - Bear total</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Tape Alignment */}
|
||||||
|
<div>
|
||||||
|
<h4 className="text-slate-200 font-semibold mb-2">📊 Tape Alignment</h4>
|
||||||
|
<div className="text-xs text-slate-400">
|
||||||
|
<p className="mb-1">
|
||||||
|
<strong className="text-slate-300">↗︎ (Up Arrow):</strong> Bullish flow with price moving up ≥ 0.20%
|
||||||
|
</p>
|
||||||
|
<p>
|
||||||
|
<strong className="text-slate-300">↘︎ (Down Arrow):</strong> Bearish flow with price moving down ≥ 0.20%
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@ -35,6 +35,7 @@ export function OptionsFlowCardList({ data, onCardClick, selectedRow }) {
|
||||||
const momentum = row.momentumScore || 0;
|
const momentum = row.momentumScore || 0;
|
||||||
const signal = row.tradeSignal;
|
const signal = row.tradeSignal;
|
||||||
const isSelected = selectedRow && (selectedRow.Symbol === row.Symbol || selectedRow.symbol_norm === row.symbol_norm);
|
const isSelected = selectedRow && (selectedRow.Symbol === row.Symbol || selectedRow.symbol_norm === row.symbol_norm);
|
||||||
|
const volumeHistory = row.volumeHistory || [];
|
||||||
|
|
||||||
// Date fields
|
// Date fields
|
||||||
const createdDate = row.CreatedDate;
|
const createdDate = row.CreatedDate;
|
||||||
|
|
@ -390,6 +391,74 @@ export function OptionsFlowCardList({ data, onCardClick, selectedRow }) {
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Last 5 Days Volume History - Table Format (New Row) */}
|
||||||
|
{volumeHistory && volumeHistory.length > 0 && (
|
||||||
|
<div className="mt-4 pt-4 border-t border-slate-700/50 w-full">
|
||||||
|
<div className="text-xs font-semibold text-slate-400 uppercase tracking-wide mb-3">
|
||||||
|
Last 5 Days Volume
|
||||||
|
</div>
|
||||||
|
<table className="w-full text-xs">
|
||||||
|
<thead>
|
||||||
|
<tr className="border-b border-slate-700/50">
|
||||||
|
<th className="text-left py-2 px-3 text-slate-500 font-semibold">Date</th>
|
||||||
|
<th className="text-right py-2 px-3 text-slate-500 font-semibold">Volume</th>
|
||||||
|
<th className="text-right py-2 px-3 text-slate-500 font-semibold">Open</th>
|
||||||
|
<th className="text-right py-2 px-3 text-slate-500 font-semibold">High</th>
|
||||||
|
<th className="text-right py-2 px-3 text-slate-500 font-semibold">Low</th>
|
||||||
|
<th className="text-right py-2 px-3 text-slate-500 font-semibold">Close</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{volumeHistory.map((day, idx) => {
|
||||||
|
const date = new Date(day.date);
|
||||||
|
const today = new Date();
|
||||||
|
today.setHours(0, 0, 0, 0);
|
||||||
|
const yesterday = new Date(today);
|
||||||
|
yesterday.setDate(yesterday.getDate() - 1);
|
||||||
|
const dayDate = new Date(date);
|
||||||
|
dayDate.setHours(0, 0, 0, 0);
|
||||||
|
|
||||||
|
let dayLabel;
|
||||||
|
if (dayDate.getTime() === today.getTime()) {
|
||||||
|
dayLabel = 'Today';
|
||||||
|
} else if (dayDate.getTime() === yesterday.getTime()) {
|
||||||
|
dayLabel = 'Yesterday';
|
||||||
|
} else {
|
||||||
|
dayLabel = date.toLocaleDateString('en-US', { month: 'short', day: 'numeric' });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Format volume with K/M/B
|
||||||
|
const formatVolume = (vol) => {
|
||||||
|
if (!vol || vol === 0) return '—';
|
||||||
|
const absVal = Math.abs(vol);
|
||||||
|
if (absVal >= 1e9) return `${(absVal / 1e9).toFixed(2)}B`;
|
||||||
|
if (absVal >= 1e6) return `${(absVal / 1e6).toFixed(2)}M`;
|
||||||
|
if (absVal >= 1e3) return `${(absVal / 1e3).toFixed(2)}K`;
|
||||||
|
return absVal.toLocaleString();
|
||||||
|
};
|
||||||
|
|
||||||
|
// Format price
|
||||||
|
const formatPrice = (price) => {
|
||||||
|
if (!price || price === 0) return '—';
|
||||||
|
return `$${parseFloat(price).toFixed(2)}`;
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<tr key={idx} className="border-b border-slate-700/30 hover:bg-slate-800/30">
|
||||||
|
<td className="py-2 px-3 text-slate-300 font-medium">{dayLabel}</td>
|
||||||
|
<td className="py-2 px-3 text-right text-slate-200 font-mono">{formatVolume(day.volume)}</td>
|
||||||
|
<td className="py-2 px-3 text-right text-slate-300 font-mono">{formatPrice(day.open)}</td>
|
||||||
|
<td className="py-2 px-3 text-right text-green-400 font-mono">{formatPrice(day.high)}</td>
|
||||||
|
<td className="py-2 px-3 text-right text-red-400 font-mono">{formatPrice(day.low)}</td>
|
||||||
|
<td className="py-2 px-3 text-right text-slate-200 font-mono font-semibold">{formatPrice(day.close)}</td>
|
||||||
|
</tr>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
})}
|
})}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
import { useEffect, useState, useMemo, useRef } from 'react';
|
import { useEffect, useState, useMemo, useRef } from 'react';
|
||||||
import { useOptionsFlow } from '@/hooks/useOptionsFlow';
|
import { useOptionsFlow } from '@/hooks/useOptionsFlow';
|
||||||
|
import { useStockPrices } from '@/hooks/useStockPrices';
|
||||||
import { DataTable } from '@/components/ui/data-table';
|
import { DataTable } from '@/components/ui/data-table';
|
||||||
import { Badge } from '@/components/ui/Badge';
|
import { Badge } from '@/components/ui/Badge';
|
||||||
import { Input } from '@/components/ui/input';
|
import { Input } from '@/components/ui/input';
|
||||||
|
|
@ -73,22 +74,144 @@ export default function OptionsFlowPanel() {
|
||||||
return score;
|
return score;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Rank data by best trade score
|
// Extract unique symbols for stock price fetching
|
||||||
|
const uniqueSymbols = useMemo(() => {
|
||||||
|
if (!data) return [];
|
||||||
|
return [...new Set(data.map(row => (row.symbol_norm || row.Symbol || '').toUpperCase()).filter(Boolean))];
|
||||||
|
}, [data]);
|
||||||
|
|
||||||
|
// Fetch stock prices and volume asynchronously
|
||||||
|
const { data: stockPricesData } = useStockPrices(uniqueSymbols);
|
||||||
|
|
||||||
|
// Rank data by best trade score and merge stock price/volume data
|
||||||
const filteredData = useMemo(() => {
|
const filteredData = useMemo(() => {
|
||||||
if (!data) return [];
|
if (!data) return [];
|
||||||
|
|
||||||
// Add best trade score and sort by it
|
// Add best trade score and sort by it
|
||||||
const filtered = data.map(row => ({
|
const filtered = data.map(row => {
|
||||||
|
const symbolUpper = (row.symbol_norm || row.Symbol || '').toUpperCase();
|
||||||
|
const stockPriceInfo = stockPricesData[symbolUpper];
|
||||||
|
|
||||||
|
// Merge stock price and volume history data
|
||||||
|
const mergedRow = {
|
||||||
...row,
|
...row,
|
||||||
bestTradeScore: calculateBestTradeScore(row)
|
bestTradeScore: calculateBestTradeScore(row)
|
||||||
})).sort((a, b) => b.bestTradeScore - a.bestTradeScore);
|
};
|
||||||
|
|
||||||
|
// Add stock price data if available
|
||||||
|
if (stockPriceInfo) {
|
||||||
|
if (stockPriceInfo.stockPrice) {
|
||||||
|
mergedRow.stockPrice = stockPriceInfo.stockPrice;
|
||||||
|
}
|
||||||
|
if (stockPriceInfo.volumeHistory && stockPriceInfo.volumeHistory.length > 0) {
|
||||||
|
mergedRow.volumeHistory = stockPriceInfo.volumeHistory;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return mergedRow;
|
||||||
|
}).sort((a, b) => b.bestTradeScore - a.bestTradeScore);
|
||||||
|
|
||||||
return filtered;
|
return filtered;
|
||||||
}, [data]);
|
}, [data, stockPricesData]);
|
||||||
|
|
||||||
// Get top 5 trades for summary
|
// Get top 5 trades for summary
|
||||||
|
// Filter: items repeated more than once, more than 2 rockets, highest premium
|
||||||
const topTrades = useMemo(() => {
|
const topTrades = useMemo(() => {
|
||||||
return filteredData.slice(0, 5);
|
if (!filteredData || filteredData.length === 0) return [];
|
||||||
|
|
||||||
|
// Helper function to extract clean symbol
|
||||||
|
const getCleanSymbol = (row) => {
|
||||||
|
// Prioritize symbol_norm (clean, normalized)
|
||||||
|
if (row.symbol_norm) {
|
||||||
|
return row.symbol_norm.toUpperCase().trim();
|
||||||
|
}
|
||||||
|
// If Symbol is formatted like "(35) (35) HOOD · RTH ⚡ 🟢💎⭐💰⚡ 🔥", extract just the symbol
|
||||||
|
if (row.Symbol) {
|
||||||
|
const symbolStr = String(row.Symbol);
|
||||||
|
// Try to extract symbol from formatted string (look for pattern like "HOOD ·" or just "HOOD")
|
||||||
|
// The symbol is usually before the "·" separator
|
||||||
|
const match = symbolStr.match(/([A-Z]{1,5})\s*·/);
|
||||||
|
if (match && match[1]) {
|
||||||
|
return match[1].toUpperCase().trim();
|
||||||
|
}
|
||||||
|
// If no separator, try to find a stock symbol (1-5 uppercase letters)
|
||||||
|
const symbolMatch = symbolStr.match(/\b([A-Z]{1,5})\b/);
|
||||||
|
if (symbolMatch && symbolMatch[1]) {
|
||||||
|
return symbolMatch[1].toUpperCase().trim();
|
||||||
|
}
|
||||||
|
// Fallback: use the whole string if it's short and looks like a symbol
|
||||||
|
const cleaned = symbolStr.replace(/[^A-Z]/g, '').toUpperCase();
|
||||||
|
if (cleaned.length >= 1 && cleaned.length <= 5) {
|
||||||
|
return cleaned;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return '';
|
||||||
|
};
|
||||||
|
|
||||||
|
// Count symbol occurrences using clean symbols
|
||||||
|
const symbolCounts = {};
|
||||||
|
filteredData.forEach(row => {
|
||||||
|
const symbol = getCleanSymbol(row);
|
||||||
|
if (symbol) {
|
||||||
|
symbolCounts[symbol] = (symbolCounts[symbol] || 0) + 1;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Helper function to count rockets
|
||||||
|
const countRockets = (rocketStr) => {
|
||||||
|
if (!rocketStr) return 0;
|
||||||
|
// Count occurrences of 🚀 emoji
|
||||||
|
return (rocketStr.match(/🚀/g) || []).length;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Helper function to get premium as number
|
||||||
|
const getPremiumNum = (row) => {
|
||||||
|
// Try premium_num first (raw numeric value)
|
||||||
|
if (row.premium_num !== undefined && row.premium_num !== null) {
|
||||||
|
return parseFloat(row.premium_num) || 0;
|
||||||
|
}
|
||||||
|
// Try Premium field (might be formatted string like "500 K" or "1.2 M")
|
||||||
|
if (row.Premium) {
|
||||||
|
const premiumStr = String(row.Premium).trim();
|
||||||
|
// Try to parse if it's a number string
|
||||||
|
const num = parseFloat(premiumStr.replace(/[^0-9.-]/g, ''));
|
||||||
|
if (!isNaN(num)) {
|
||||||
|
// Check for K or M suffix
|
||||||
|
if (premiumStr.toUpperCase().includes('M')) {
|
||||||
|
return num * 1000000;
|
||||||
|
} else if (premiumStr.toUpperCase().includes('K')) {
|
||||||
|
return num * 1000;
|
||||||
|
}
|
||||||
|
return num;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Filter: items that appear more than once AND have 2 or more rockets (>= 2, so 🚀🚀 or 🚀🚀🚀)
|
||||||
|
const filtered = filteredData.filter(row => {
|
||||||
|
const symbol = getCleanSymbol(row);
|
||||||
|
// Check multiple rocket field variations
|
||||||
|
const rocket = row.Rocket || row.rocketDisplay || row.rocket || row.rocket_with_mny || '';
|
||||||
|
const rocketCount = countRockets(rocket);
|
||||||
|
|
||||||
|
// Must appear more than once (duplicate)
|
||||||
|
const isRepeated = symbolCounts[symbol] > 1;
|
||||||
|
// Must have 2 or more rockets (count >= 2, so 🚀🚀 or 🚀🚀🚀)
|
||||||
|
const has2OrMoreRockets = rocketCount >= 2;
|
||||||
|
|
||||||
|
return isRepeated && has2OrMoreRockets;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Sort by premium (highest first)
|
||||||
|
filtered.sort((a, b) => {
|
||||||
|
const premiumA = getPremiumNum(a);
|
||||||
|
const premiumB = getPremiumNum(b);
|
||||||
|
return premiumB - premiumA;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Return top 5
|
||||||
|
return filtered.slice(0, 5);
|
||||||
}, [filteredData]);
|
}, [filteredData]);
|
||||||
|
|
||||||
const handleRowClick = (row) => {
|
const handleRowClick = (row) => {
|
||||||
|
|
@ -776,16 +899,22 @@ export default function OptionsFlowPanel() {
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Top Trades Summary */}
|
{/* Top Trades Summary */}
|
||||||
{!loading && !error && topTrades.length > 0 && (
|
{!loading && !error && (
|
||||||
<div className="bg-gradient-to-r from-blue-500/10 to-purple-500/10 border border-blue-500/30 rounded-lg p-4">
|
<div className="bg-gradient-to-r from-blue-500/10 to-purple-500/10 border border-blue-500/30 rounded-lg p-4">
|
||||||
<div className="flex items-center justify-between mb-3">
|
<div className="flex items-center justify-between mb-3">
|
||||||
<h3 className="text-sm font-semibold text-slate-200">🏆 TOP 5 TRADES FOR TODAY</h3>
|
<h3 className="text-sm font-semibold text-slate-200">🏆 TOP 5 TRADES FOR TODAY</h3>
|
||||||
<span className="text-xs text-slate-400">Ranked by best trade potential</span>
|
<span className="text-xs text-slate-400">
|
||||||
|
{topTrades.length > 0
|
||||||
|
? `Filtered: Repeated symbols with 2+ rockets, sorted by highest premium`
|
||||||
|
: `No trades found matching criteria (repeated symbols with 2+ rockets)`}
|
||||||
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<div className="grid grid-cols-1 md:grid-cols-5 gap-3">
|
{topTrades.length > 0 ? (
|
||||||
{topTrades.map((row, idx) => {
|
<div className="grid grid-cols-1 md:grid-cols-5 gap-3">
|
||||||
|
{topTrades.map((row, idx) => {
|
||||||
const signal = row.tradeSignal;
|
const signal = row.tradeSignal;
|
||||||
const symbol = row.Symbol || row.symbol_norm;
|
// Use clean symbol extraction (same logic as filtering)
|
||||||
|
const symbol = row.symbol_norm || (row.Symbol && row.Symbol.match(/([A-Z]{1,5})\s*·/)?.[1]) || row.Symbol || 'UNKNOWN';
|
||||||
const score = row.bestTradeScore || 0;
|
const score = row.bestTradeScore || 0;
|
||||||
const hasSignal = signal && signal.signal !== 'NEUTRAL' && signal.signal !== 'WAIT';
|
const hasSignal = signal && signal.signal !== 'NEUTRAL' && signal.signal !== 'WAIT';
|
||||||
|
|
||||||
|
|
@ -828,7 +957,15 @@ export default function OptionsFlowPanel() {
|
||||||
</button>
|
</button>
|
||||||
);
|
);
|
||||||
})}
|
})}
|
||||||
</div>
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="text-center py-8 text-slate-400 text-sm">
|
||||||
|
<p>No trades found that meet the criteria:</p>
|
||||||
|
<p className="mt-2 text-xs">• Symbol appears more than once in the data</p>
|
||||||
|
<p className="text-xs">• Has 2 or more rockets (🚀🚀 or 🚀🚀🚀)</p>
|
||||||
|
<p className="text-xs">• Sorted by highest premium</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -39,33 +39,40 @@ export default function PerformanceTrackingPanel() {
|
||||||
|
|
||||||
const result = await response.json();
|
const result = await response.json();
|
||||||
|
|
||||||
|
// Check if response has the expected trading stats format
|
||||||
if (result.success && result.data) {
|
if (result.success && result.data) {
|
||||||
setStats({
|
setStats({
|
||||||
totalSignals: result.data.totalSignals || 0,
|
totalSignals: result.data.totalSignals || 0,
|
||||||
highConviction: result.data.highConviction || 0,
|
highConviction: result.data.highConviction || 0,
|
||||||
currentlyTracking: result.data.currentlyTracking || 0,
|
currentlyTracking: result.data.currentlyTracking || 0,
|
||||||
winRate: {
|
winRate: {
|
||||||
all: result.data.winRate?.all || 0,
|
all: result.data.winRate?.all || stats.winRate.all,
|
||||||
highScore: result.data.winRate?.highScore || 0,
|
highScore: result.data.winRate?.highScore || stats.winRate.highScore,
|
||||||
tapeAligned: result.data.winRate?.tapeAligned || 0,
|
tapeAligned: result.data.winRate?.tapeAligned || stats.winRate.tapeAligned,
|
||||||
patternMatched: result.data.winRate?.patternMatched || 0
|
patternMatched: result.data.winRate?.patternMatched || stats.winRate.patternMatched
|
||||||
},
|
},
|
||||||
avgPerformance: {
|
avgPerformance: {
|
||||||
avgWinner: result.data.avgPerformance?.avgWinner || 0,
|
avgWinner: result.data.avgPerformance?.avgWinner || stats.avgPerformance.avgWinner,
|
||||||
avgLoser: result.data.avgPerformance?.avgLoser || 0,
|
avgLoser: result.data.avgPerformance?.avgLoser || stats.avgPerformance.avgLoser,
|
||||||
rrRatio: result.data.avgPerformance?.rrRatio || 0,
|
rrRatio: result.data.avgPerformance?.rrRatio || stats.avgPerformance.rrRatio,
|
||||||
expectancy: result.data.avgPerformance?.expectancy || 0
|
expectancy: result.data.avgPerformance?.expectancy || stats.avgPerformance.expectancy
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
} else if (result.success && result.stats) {
|
||||||
|
// Endpoint exists but returns query performance stats, not trading stats
|
||||||
|
// Keep default values - this endpoint is for query metrics, not trading stats
|
||||||
|
console.warn('Performance endpoint returned query stats, not trading stats. Using default values.');
|
||||||
} else {
|
} else {
|
||||||
throw new Error(result.error || 'Failed to fetch stats');
|
// If endpoint doesn't return expected format, silently use defaults
|
||||||
|
console.warn('Performance stats endpoint returned unexpected format. Using default values.');
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
// Only log if it's not a 404 (endpoint doesn't exist)
|
// Silently handle errors - endpoint may not exist or may be unavailable
|
||||||
if (!error.message.includes('404')) {
|
// Keep existing default stats on error (don't reset to 0)
|
||||||
console.error('Failed to fetch performance stats:', error);
|
// Only log non-network errors for debugging
|
||||||
|
if (error.name !== 'TypeError' && !error.message.includes('fetch')) {
|
||||||
|
console.warn('Performance stats endpoint error:', error.message);
|
||||||
}
|
}
|
||||||
// Keep existing stats on error (don't reset to 0)
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,9 @@ export function useOptionsFlow({ autoRefresh = false, interval = 30000, ...filte
|
||||||
)
|
)
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Add skipPrices=true for faster initial response
|
||||||
|
params.append('skipPrices', 'true');
|
||||||
|
|
||||||
const response = await fetch(`${getApiUrl('/api/options/flow')}?${params}`);
|
const response = await fetch(`${getApiUrl('/api/options/flow')}?${params}`);
|
||||||
const result = await response.json();
|
const result = await response.json();
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,61 @@
|
||||||
|
import { useState, useEffect, useCallback } from 'react';
|
||||||
|
import { getApiUrl } from '@/config/api';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Hook to fetch stock prices and volume history for symbols
|
||||||
|
* @param {string[]} symbols - Array of stock symbols to fetch
|
||||||
|
* @returns {Object} { data, loading, error, refetch }
|
||||||
|
*/
|
||||||
|
export function useStockPrices(symbols) {
|
||||||
|
const [data, setData] = useState({});
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
const [error, setError] = useState(null);
|
||||||
|
|
||||||
|
const fetchStockPrices = useCallback(async (symbolList) => {
|
||||||
|
if (!symbolList || symbolList.length === 0) {
|
||||||
|
setData({});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
setLoading(true);
|
||||||
|
setError(null);
|
||||||
|
|
||||||
|
// Remove duplicates and normalize
|
||||||
|
const uniqueSymbols = [...new Set(symbolList.map(s => s.toUpperCase().trim()))].filter(Boolean);
|
||||||
|
|
||||||
|
if (uniqueSymbols.length === 0) {
|
||||||
|
setData({});
|
||||||
|
setLoading(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const params = new URLSearchParams({
|
||||||
|
symbols: uniqueSymbols.join(',')
|
||||||
|
});
|
||||||
|
|
||||||
|
const response = await fetch(`${getApiUrl('/api/stock-prices')}?${params}`);
|
||||||
|
const result = await response.json();
|
||||||
|
|
||||||
|
if (result.success) {
|
||||||
|
setData(result.data || {});
|
||||||
|
} else {
|
||||||
|
throw new Error(result.error || 'Failed to fetch stock prices');
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Error fetching stock prices:', err);
|
||||||
|
setError(err.message);
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (symbols && symbols.length > 0) {
|
||||||
|
fetchStockPrices(symbols);
|
||||||
|
}
|
||||||
|
}, [symbols, fetchStockPrices]);
|
||||||
|
|
||||||
|
return { data, loading, error, refetch: () => fetchStockPrices(symbols) };
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@ -0,0 +1,733 @@
|
||||||
|
# sync_blackbox_flow.py — BlackBox API sync to PostgreSQL
|
||||||
|
# Fetches options flow data from BlackBox API and syncs to PostgreSQL database
|
||||||
|
|
||||||
|
import os, time, hashlib, re, argparse
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict, Optional, List
|
||||||
|
from datetime import datetime, date
|
||||||
|
import pandas as pd
|
||||||
|
import numpy as np
|
||||||
|
import requests
|
||||||
|
import json
|
||||||
|
import psycopg2
|
||||||
|
from psycopg2.extras import execute_values
|
||||||
|
from psycopg2 import sql
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
|
load_dotenv()
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────
|
||||||
|
# Config
|
||||||
|
# ─────────────────────────────────────────────
|
||||||
|
WRITE_POSTGRES = True
|
||||||
|
|
||||||
|
# BlackBox API Configuration
|
||||||
|
BLACKBOX_API_URL = "https://api.blackboxstocks.com/api/v2/options/getFlowMobile"
|
||||||
|
BLACKBOX_API_TOKEN = os.getenv("BLACKBOX_API_TOKEN", "eyJhbGciOiJodHRwOi8vd3d3LnczLm9yZy8yMDAxLzA0L3htbGRzaWctbW9yZSNobWFjLXNoYTI1NiIsInR5cCI6IkpXVCJ9.eyJodHRwOi8vc2NoZW1hcy54bWxzb2FwLm9yZy93cy8yMDA1LzA1L2lkZW50aXR5L2NsYWltcy9uYW1lIjoic3JlZGR5MDgxOUBnbWFpbC5jb20iLCJodHRwOi8vc2NoZW1hcy54bWxzb2FwLm9yZy93cy8yMDA1LzA1L2lkZW50aXR5L2NsYWltcy9uYW1laWRlbnRpZmllciI6Ijk3ZDA5NThjLTBhOTktNDA5OC05OWQwLWNkYTg3Njg5N2Q3ZiIsImh0dHA6Ly9zY2hlbWFzLnhtbHNvYXAub3JnL3dzLzIwMDUvMDUvaWRlbnRpdHkvY2xhaW1zL3NpZCI6InBpS1plMktZN2d2NnErbDZyM3g3YkpHYU9ESlUrZVdOZmMwc1NicWZZNU09IiwiaHR0cDovL3NjaGVtYXMubWljcm9zb2Z0LmNvbS93cy8yMDA4LzA2L2lkZW50aXR5L2NsYWltcy9yb2xlIjpbIlN1YnNjcmliZXIiLCJPcHRpb25zIiwiRXF1aXRpZXMiXSwiZXhwIjoxNzY1NzQ4NjI1fQ.Evq__DugD7s1kAytUqtsknQFIeOPjKM3iwp6cDI0hJI")
|
||||||
|
|
||||||
|
# PostgreSQL connection parameters
|
||||||
|
POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost")
|
||||||
|
POSTGRES_PORT = int(os.getenv("POSTGRES_PORT", "5432"))
|
||||||
|
POSTGRES_DB = os.getenv("POSTGRES_DB", "institutional_trader")
|
||||||
|
POSTGRES_USER = os.getenv("POSTGRES_USER", "postgres")
|
||||||
|
POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD", "postgres")
|
||||||
|
POSTGRES_SCHEMA = os.getenv("POSTGRES_SCHEMA", "public")
|
||||||
|
|
||||||
|
# ⚠️ Per-table schema style: 'snake' or 'camel'
|
||||||
|
TABLE_STYLE = {
|
||||||
|
"AlertStream": "snake",
|
||||||
|
"AlertStream_monthly": "snake",
|
||||||
|
"OptionsFlow": "camel", # ← your Supabase columns are CamelCase here
|
||||||
|
"OptionsFlow_monthly": "camel", # ← same
|
||||||
|
"OptionsVolume": "camel", # adjust if needed
|
||||||
|
"Short_Long": "camel", # adjust if needed
|
||||||
|
}
|
||||||
|
|
||||||
|
_TARGET_STEMS = set(TABLE_STYLE.keys())
|
||||||
|
|
||||||
|
# Upsert behavior / logging
|
||||||
|
INCREMENTAL = True
|
||||||
|
SINGLE_CHUNK = False # chunked to avoid 57014
|
||||||
|
UPSERT_CHUNK = 3000 # drop to 1000 if still timeouts
|
||||||
|
MAX_RETRIES = 3
|
||||||
|
PRINT_PROGRESS = False
|
||||||
|
PRINT_PRUNE_LOGS = False
|
||||||
|
USE_RETURNING_MINIMAL= True
|
||||||
|
|
||||||
|
print_flush = lambda *a, **k: print(*a, **k, flush=True)
|
||||||
|
def _ts(): return time.strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────
|
||||||
|
# PostgreSQL connection
|
||||||
|
# ─────────────────────────────────────────────
|
||||||
|
def _pg_conn():
|
||||||
|
"""Create a PostgreSQL connection."""
|
||||||
|
return psycopg2.connect(
|
||||||
|
host=POSTGRES_HOST,
|
||||||
|
port=POSTGRES_PORT,
|
||||||
|
database=POSTGRES_DB,
|
||||||
|
user=POSTGRES_USER,
|
||||||
|
password=POSTGRES_PASSWORD,
|
||||||
|
options=f"-c search_path={POSTGRES_SCHEMA}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────
|
||||||
|
# Expected headers (both styles) + mappings
|
||||||
|
# ─────────────────────────────────────────────
|
||||||
|
# OptionsFlow (CamelCase)
|
||||||
|
EXPECTED_OPTIONSFLOW_CAMEL = [
|
||||||
|
"CreatedDate","CreatedTime","Symbol","Type","Volume","Price","Side",
|
||||||
|
"CallPut","Strike","Spot","Premium","ExpirationDate","Color",
|
||||||
|
"ImpliedVolatility","Dte","ER","StockEtf","Sector","Uoa",
|
||||||
|
"Weekly","MktCap","OI"
|
||||||
|
]
|
||||||
|
# OptionsFlow (snake_case)
|
||||||
|
EXPECTED_OPTIONSFLOW_SNAKE = [
|
||||||
|
"created_date","created_time","symbol","type","volume","price","side",
|
||||||
|
"callput","strike","spot","premium","expiration_date","color",
|
||||||
|
"implied_volatility","dte","er","stock_etf","sector","uoa",
|
||||||
|
"weekly","mktcap","oi"
|
||||||
|
]
|
||||||
|
# AlertStream (CamelCase in files → snake in DB)
|
||||||
|
EXPECTED_ALERTSTREAM_SNAKE = [
|
||||||
|
"date","timestamp","ticker","volume","price","pct_of_avg30",
|
||||||
|
"notional","message","type","securitytype","industry","sector",
|
||||||
|
"avg30day","float","earningsdate"
|
||||||
|
]
|
||||||
|
|
||||||
|
# Camel→snake renames (if file arrives CamelCase but DB expects snake)
|
||||||
|
MAP_OPTIONSFLOW_SNAKE = {
|
||||||
|
"CreatedDate":"created_date","CreatedTime":"created_time","Symbol":"symbol","Type":"type",
|
||||||
|
"Volume":"volume","Price":"price","Side":"side","CallPut":"callput","Strike":"strike",
|
||||||
|
"Spot":"spot","Premium":"premium","ExpirationDate":"expiration_date","Color":"color",
|
||||||
|
"ImpliedVolatility":"implied_volatility","Dte":"dte","ER":"er","StockEtf":"stock_etf",
|
||||||
|
"Sector":"sector","Uoa":"uoa","Weekly":"weekly","MktCap":"mktcap","OI":"oi"
|
||||||
|
}
|
||||||
|
MAP_ALERTSTREAM_SNAKE = {
|
||||||
|
"Date":"date","Timestamp":"timestamp","Ticker":"ticker","Volume":"volume","Price":"price",
|
||||||
|
"Pct_of_Avg30Day":"pct_of_avg30","Notional":"notional","Message":"message","Type":"type",
|
||||||
|
"SecurityType":"securitytype","Industry":"industry","Sector":"sector",
|
||||||
|
"Avg30Day":"avg30day","Float":"float","EarningsDate":"earningsdate"
|
||||||
|
}
|
||||||
|
|
||||||
|
# snake→Camel renames (if file is snake but DB expects Camel)
|
||||||
|
MAP_OPTIONSFLOW_CAMEL = {v:k for k,v in MAP_OPTIONSFLOW_SNAKE.items()}
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────
|
||||||
|
# BlackBox API Integration
|
||||||
|
# ─────────────────────────────────────────────
|
||||||
|
def build_filter_bitmask(_filter_options: Optional[Dict] = None) -> int:
|
||||||
|
"""Build the filter bitmask based on filter options."""
|
||||||
|
# Default filter value that enables common filters
|
||||||
|
return 2198487171967
|
||||||
|
|
||||||
|
def build_filters(start_date: datetime, end_date: datetime, custom_filters: Optional[Dict] = None) -> Dict:
|
||||||
|
"""Build default filters object."""
|
||||||
|
date_str = start_date.isoformat()
|
||||||
|
|
||||||
|
filters = {
|
||||||
|
"optionsDate": {
|
||||||
|
"start": date_str,
|
||||||
|
"end": end_date.isoformat()
|
||||||
|
},
|
||||||
|
"expireOptionsDate": {
|
||||||
|
"start": date_str,
|
||||||
|
"end": end_date.isoformat()
|
||||||
|
},
|
||||||
|
"optionsFlowPuts": True,
|
||||||
|
"optionsFlowCalls": True,
|
||||||
|
"optionsFlowYellow": True,
|
||||||
|
"optionsFlowWhite": True,
|
||||||
|
"optionsFlowMagenta": True,
|
||||||
|
"optionsFlowAboveAskOnly": True,
|
||||||
|
"optionsFlowBelowBidOnly": True,
|
||||||
|
"optionsFlowAtOrAboveAsk": True,
|
||||||
|
"optionsFlowAtOrBelowBid": True,
|
||||||
|
"optionsFlowMultileg": False,
|
||||||
|
"optionsFlowOnlyMultiLeg": False,
|
||||||
|
"optionsFlowBelowPoint5": False,
|
||||||
|
"optionsFlowBelow5": False,
|
||||||
|
"optionsFlow100Contracts": False,
|
||||||
|
"optionsFlow500Contracts": False,
|
||||||
|
"optionsFlow5000Contracts": False,
|
||||||
|
"optionsFlowStock": True,
|
||||||
|
"optionsFlowEtf": True,
|
||||||
|
"optionsFlowAbove50k": False,
|
||||||
|
"optionsFlowAbove100k": False,
|
||||||
|
"optionsFlowAbove200k": False,
|
||||||
|
"optionsFlowAbove500k": False,
|
||||||
|
"optionsFlowAbove1m": False,
|
||||||
|
"marketCapAbove750B": False,
|
||||||
|
"optionsFlowInTheMoney": False,
|
||||||
|
"optionsFlowOutOfTheMoney": False,
|
||||||
|
"optionsFlowSweepOnly": False,
|
||||||
|
"optionsFlowWeeklyOnly": False,
|
||||||
|
"optionsFlowEarningsReportOnly": False,
|
||||||
|
"optionsFlowUnusualOnly": False,
|
||||||
|
"optionsFlowExDiv": False,
|
||||||
|
"optionsFlowConsumerDiscretionary": True,
|
||||||
|
"optionsFlowIndustrials": True,
|
||||||
|
"optionsFlowInformationTechnology": True,
|
||||||
|
"optionsFlowRealEstate": True,
|
||||||
|
"optionsFlowHealthCare": True,
|
||||||
|
"optionsFlowEnergy": True,
|
||||||
|
"optionsFlowFinancials": True,
|
||||||
|
"optionsFlowMaterials": True,
|
||||||
|
"optionsFlowConsumerStaples": True,
|
||||||
|
"optionsFlowCommunicationServices": True,
|
||||||
|
"optionsFlowUtilities": True,
|
||||||
|
"optionsExpirationRange": False,
|
||||||
|
"optionsFlowSectorNone": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
if custom_filters:
|
||||||
|
filters.update(custom_filters)
|
||||||
|
|
||||||
|
return filters
|
||||||
|
|
||||||
|
# Constants
|
||||||
|
TIMEZONE_SUFFIX = "+00:00"
|
||||||
|
|
||||||
|
def fetch_blackbox_flow(options: Optional[Dict] = None) -> List[Dict]:
|
||||||
|
"""Fetch options flow data from BlackBox Stocks API."""
|
||||||
|
if options is None:
|
||||||
|
options = {}
|
||||||
|
|
||||||
|
if not BLACKBOX_API_TOKEN:
|
||||||
|
raise ValueError(
|
||||||
|
"BLACKBOX_API_TOKEN not found in environment variables.\n"
|
||||||
|
"Please add BLACKBOX_API_TOKEN to your .env file or set it as an environment variable."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Parse dates - default to today if not provided
|
||||||
|
if options.get("startDate"):
|
||||||
|
if isinstance(options["startDate"], str):
|
||||||
|
start_date = datetime.fromisoformat(options["startDate"].replace("Z", TIMEZONE_SUFFIX))
|
||||||
|
elif isinstance(options["startDate"], date):
|
||||||
|
start_date = datetime.combine(options["startDate"], datetime.min.time())
|
||||||
|
else:
|
||||||
|
start_date = options["startDate"]
|
||||||
|
else:
|
||||||
|
start_date = datetime.now()
|
||||||
|
|
||||||
|
if options.get("endDate"):
|
||||||
|
if isinstance(options["endDate"], str):
|
||||||
|
end_date = datetime.fromisoformat(options["endDate"].replace("Z", TIMEZONE_SUFFIX))
|
||||||
|
elif isinstance(options["endDate"], date):
|
||||||
|
end_date = datetime.combine(options["endDate"], datetime.max.time())
|
||||||
|
else:
|
||||||
|
end_date = options["endDate"]
|
||||||
|
else:
|
||||||
|
end_date = start_date
|
||||||
|
|
||||||
|
# Build request body
|
||||||
|
body = {
|
||||||
|
"historical": options.get("historical", False),
|
||||||
|
"symbol": options.get("symbol", ""),
|
||||||
|
"strike": options.get("strike", 0),
|
||||||
|
"count": options.get("count") or options.get("limit", 300),
|
||||||
|
"filter": build_filter_bitmask(options.get("filters")),
|
||||||
|
"filters": build_filters(start_date, end_date, options.get("filters")),
|
||||||
|
"fromDate": start_date.isoformat(),
|
||||||
|
"toDate": end_date.isoformat()
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = requests.post(
|
||||||
|
BLACKBOX_API_URL,
|
||||||
|
headers={
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Accept": "application/json",
|
||||||
|
"Authorization": f"Bearer {BLACKBOX_API_TOKEN}"
|
||||||
|
},
|
||||||
|
json=body,
|
||||||
|
timeout=30
|
||||||
|
)
|
||||||
|
|
||||||
|
if not response.ok:
|
||||||
|
error_text = response.text
|
||||||
|
raise RuntimeError(
|
||||||
|
f"BlackBox API error: {response.status_code} {response.reason}\n"
|
||||||
|
f"Response: {error_text}"
|
||||||
|
)
|
||||||
|
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
# Handle different response structures
|
||||||
|
if isinstance(data, list):
|
||||||
|
return data
|
||||||
|
elif isinstance(data, dict):
|
||||||
|
if "data" in data and isinstance(data["data"], list):
|
||||||
|
return data["data"]
|
||||||
|
elif "flows" in data and isinstance(data["flows"], list):
|
||||||
|
return data["flows"]
|
||||||
|
elif "results" in data and isinstance(data["results"], list):
|
||||||
|
return data["results"]
|
||||||
|
else:
|
||||||
|
print_flush(f"{_ts()} | ⚠️ Unexpected API response structure: {list(data.keys())}")
|
||||||
|
return []
|
||||||
|
else:
|
||||||
|
return []
|
||||||
|
except Exception as e:
|
||||||
|
print_flush(f"{_ts()} | ❌ Error fetching BlackBox flow data: {e}")
|
||||||
|
raise
|
||||||
|
|
||||||
|
def map_blackbox_to_database(api_record: Dict) -> Dict:
|
||||||
|
"""Map BlackBox API response to database schema."""
|
||||||
|
def get_value(obj: Dict, *keys: str) -> Optional[str]:
|
||||||
|
"""Safely extract values from object."""
|
||||||
|
for key in keys:
|
||||||
|
if key in obj and obj[key] is not None:
|
||||||
|
return str(obj[key])
|
||||||
|
return None
|
||||||
|
|
||||||
|
def format_date(date_str: Optional[str]) -> Optional[str]:
|
||||||
|
"""Format date as YYYY-MM-DD."""
|
||||||
|
if not date_str:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
dt = datetime.fromisoformat(str(date_str).replace("Z", TIMEZONE_SUFFIX))
|
||||||
|
return dt.strftime("%Y-%m-%d")
|
||||||
|
except (ValueError, AttributeError):
|
||||||
|
try:
|
||||||
|
dt = datetime.strptime(str(date_str), "%Y-%m-%d")
|
||||||
|
return dt.strftime("%Y-%m-%d")
|
||||||
|
except (ValueError, AttributeError):
|
||||||
|
return str(date_str)
|
||||||
|
|
||||||
|
def format_time(time_str: Optional[str]) -> Optional[str]:
|
||||||
|
"""Format time string."""
|
||||||
|
if not time_str:
|
||||||
|
return None
|
||||||
|
return str(time_str)
|
||||||
|
|
||||||
|
# Map fields - try multiple possible field names from API
|
||||||
|
mapped = {
|
||||||
|
"CreatedDate": format_date(
|
||||||
|
get_value(api_record, "createdDate", "CreatedDate", "date", "Date", "timestamp", "Timestamp")
|
||||||
|
),
|
||||||
|
"CreatedTime": format_time(
|
||||||
|
get_value(api_record, "createdTime", "CreatedTime", "time", "Time", "timestamp", "Timestamp")
|
||||||
|
),
|
||||||
|
"Symbol": get_value(api_record, "symbol", "Symbol", "ticker", "Ticker", "underlying", "Underlying"),
|
||||||
|
"Type": get_value(api_record, "type", "Type", "tradeType", "TradeType"),
|
||||||
|
"Volume": get_value(api_record, "volume", "Volume", "vol", "Vol", "contracts", "Contracts"),
|
||||||
|
"Price": get_value(api_record, "price", "Price", "lastPrice", "LastPrice", "tradePrice", "TradePrice"),
|
||||||
|
"Side": get_value(api_record, "side", "Side", "tradeSide", "TradeSide", "direction", "Direction"),
|
||||||
|
"CallPut": get_value(api_record, "callPut", "CallPut", "optionType", "OptionType", "putCall", "PutCall", "type", "Type"),
|
||||||
|
"Strike": get_value(api_record, "strike", "Strike", "strikePrice", "StrikePrice"),
|
||||||
|
"Spot": get_value(api_record, "spot", "Spot", "underlyingPrice", "UnderlyingPrice", "stockPrice", "StockPrice"),
|
||||||
|
"Premium": get_value(api_record, "premium", "Premium", "totalPremium", "TotalPremium", "notional", "Notional"),
|
||||||
|
"ExpirationDate": format_date(
|
||||||
|
get_value(api_record, "expirationDate", "ExpirationDate", "expiry", "Expiry", "expiration", "Expiration")
|
||||||
|
),
|
||||||
|
"Color": get_value(api_record, "color", "Color", "tradeColor", "TradeColor"),
|
||||||
|
"ImpliedVolatility": get_value(api_record, "impliedVolatility", "ImpliedVolatility", "iv", "IV", "volatility", "Volatility"),
|
||||||
|
"Dte": get_value(api_record, "dte", "Dte", "DTE", "daysToExpiration", "DaysToExpiration", "daysToExpiry", "DaysToExpiry"),
|
||||||
|
"ER": get_value(api_record, "er", "ER", "earnings", "Earnings", "earningsReport", "EarningsReport"),
|
||||||
|
"StockEtf": get_value(api_record, "stockEtf", "StockEtf", "assetType", "AssetType", "securityType", "SecurityType"),
|
||||||
|
"Sector": get_value(api_record, "sector", "Sector", "industry", "Industry"),
|
||||||
|
"Uoa": get_value(api_record, "uoa", "Uoa", "UOA", "underlyingOfAsset", "UnderlyingOfAsset"),
|
||||||
|
"Weekly": get_value(api_record, "weekly", "Weekly", "isWeekly", "IsWeekly", "weeklies", "Weeklies"),
|
||||||
|
"MktCap": get_value(api_record, "mktCap", "MktCap", "marketCap", "MarketCap", "marketCapitalization", "MarketCapitalization"),
|
||||||
|
"OI": get_value(api_record, "oi", "OI", "openInterest", "OpenInterest", "openInt", "OpenInt")
|
||||||
|
}
|
||||||
|
|
||||||
|
return mapped
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────
|
||||||
|
# Normalization + JSON safety
|
||||||
|
# ─────────────────────────────────────────────
|
||||||
|
WEIRD_STR = {"inf","+inf","-inf","infinity","+infinity","-infinity","∞","+∞","-∞",
|
||||||
|
"nan","-nan","NaN","N/A","NA","NULL","null",""}
|
||||||
|
|
||||||
|
def _coerce_weird_numbers(df: pd.DataFrame) -> pd.DataFrame:
|
||||||
|
df = df.copy()
|
||||||
|
for c in df.columns:
|
||||||
|
if df[c].dtype == object:
|
||||||
|
df[c] = df[c].replace(list(WEIRD_STR), np.nan)
|
||||||
|
return df
|
||||||
|
|
||||||
|
def _normalize_for_table(df: pd.DataFrame, table: str) -> pd.DataFrame:
|
||||||
|
"""Rename/select columns to match the DB style of this table."""
|
||||||
|
style = TABLE_STYLE.get(table, "snake").lower()
|
||||||
|
df = df.copy()
|
||||||
|
df.columns = [c.strip() for c in df.columns]
|
||||||
|
|
||||||
|
if table in ("OptionsFlow","OptionsFlow_monthly"):
|
||||||
|
if style == "camel":
|
||||||
|
# Ensure CamelCase headers, no renaming needed if file already CamelCase
|
||||||
|
# If file is snake, map to Camel
|
||||||
|
lower_cols = {c for c in df.columns if c.islower()}
|
||||||
|
if lower_cols:
|
||||||
|
df = df.rename(columns=MAP_OPTIONSFLOW_CAMEL)
|
||||||
|
for c in EXPECTED_OPTIONSFLOW_CAMEL:
|
||||||
|
if c not in df.columns: df[c] = None
|
||||||
|
df = df[EXPECTED_OPTIONSFLOW_CAMEL]
|
||||||
|
else:
|
||||||
|
# snake_case target
|
||||||
|
# If file CamelCase, map to snake
|
||||||
|
has_upper = any(any(ch.isupper() for ch in c) for c in df.columns)
|
||||||
|
if has_upper:
|
||||||
|
df = df.rename(columns=MAP_OPTIONSFLOW_SNAKE)
|
||||||
|
for c in EXPECTED_OPTIONSFLOW_SNAKE:
|
||||||
|
if c not in df.columns: df[c] = None
|
||||||
|
df = df[EXPECTED_OPTIONSFLOW_SNAKE]
|
||||||
|
|
||||||
|
elif table in ("AlertStream","AlertStream_monthly"):
|
||||||
|
# DB is snake_case per your screenshot
|
||||||
|
# If file CamelCase, map to snake
|
||||||
|
has_upper = any(any(ch.isupper() for ch in c) for c in df.columns)
|
||||||
|
if has_upper:
|
||||||
|
df = df.rename(columns=MAP_ALERTSTREAM_SNAKE)
|
||||||
|
for c in EXPECTED_ALERTSTREAM_SNAKE:
|
||||||
|
if c not in df.columns: df[c] = None
|
||||||
|
df = df[EXPECTED_ALERTSTREAM_SNAKE]
|
||||||
|
|
||||||
|
# Standardize any *date columns → YYYY-MM-DD* strings
|
||||||
|
for col in [c for c in df.columns if c.lower().endswith("date")]:
|
||||||
|
df[col] = pd.to_datetime(df[col], errors="coerce").dt.strftime("%Y-%m-%d")
|
||||||
|
|
||||||
|
return df
|
||||||
|
|
||||||
|
def _json_safe(df: pd.DataFrame) -> pd.DataFrame:
|
||||||
|
df = df.copy()
|
||||||
|
# numeric: drop non-finite
|
||||||
|
for c in df.columns:
|
||||||
|
if pd.api.types.is_numeric_dtype(df[c]):
|
||||||
|
s = pd.to_numeric(df[c], errors="coerce")
|
||||||
|
s[~np.isfinite(s)] = np.nan
|
||||||
|
df[c] = s
|
||||||
|
# datetimes -> strings
|
||||||
|
for c in df.columns:
|
||||||
|
if pd.api.types.is_datetime64_any_dtype(df[c]):
|
||||||
|
df[c] = df[c].dt.strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
# NA -> None
|
||||||
|
return df.astype(object).where(pd.notnull(df), None)
|
||||||
|
|
||||||
|
def _row_hash_from_series(s: pd.Series) -> str:
|
||||||
|
vals=[]
|
||||||
|
for _,v in s.items():
|
||||||
|
if v is None or (isinstance(v,float) and pd.isna(v)): vals.append("NULL")
|
||||||
|
elif isinstance(v,str): vals.append(v.strip())
|
||||||
|
else: vals.append(str(v))
|
||||||
|
return hashlib.sha1("\x1f".join(vals).encode("utf-8","ignore")).hexdigest()
|
||||||
|
|
||||||
|
def _df_prepare_for_postgres(df: pd.DataFrame) -> pd.DataFrame:
|
||||||
|
if df.empty: return df
|
||||||
|
df = _json_safe(df)
|
||||||
|
df["row_hash"] = df.apply(_row_hash_from_series, axis=1)
|
||||||
|
return df.drop_duplicates(subset=["row_hash"], keep="first").reset_index(drop=True)
|
||||||
|
|
||||||
|
def _extract_missing_col_from_error(msg: str) -> Optional[str]:
|
||||||
|
"""Extract missing column name from PostgreSQL error messages."""
|
||||||
|
patterns = [
|
||||||
|
r"column \"([^\"]+)\" does not exist",
|
||||||
|
r"Could not find the '([^']+)' column",
|
||||||
|
]
|
||||||
|
for pattern in patterns:
|
||||||
|
m = re.search(pattern, msg, re.IGNORECASE)
|
||||||
|
if m:
|
||||||
|
return m.group(1)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────
|
||||||
|
# PostgreSQL upload (quiet, chunked)
|
||||||
|
# ─────────────────────────────────────────────
|
||||||
|
def _ensure_table_exists(conn, table_name: str, columns: list):
|
||||||
|
"""Ensure table exists with row_hash column and unique constraint."""
|
||||||
|
cur = conn.cursor()
|
||||||
|
try:
|
||||||
|
# Check if table exists
|
||||||
|
cur.execute("""
|
||||||
|
SELECT EXISTS (
|
||||||
|
SELECT FROM information_schema.tables
|
||||||
|
WHERE table_schema = current_schema()
|
||||||
|
AND table_name = %s
|
||||||
|
)
|
||||||
|
""", (table_name,))
|
||||||
|
exists = cur.fetchone()[0]
|
||||||
|
|
||||||
|
if not exists:
|
||||||
|
# Create table with all columns as text initially (we'll let PostgreSQL infer types)
|
||||||
|
# For now, we'll create a basic structure - actual schema should match your data
|
||||||
|
col_defs = ", ".join([f'"{col}" TEXT' for col in columns if col != "row_hash"])
|
||||||
|
col_defs += ', "row_hash" TEXT UNIQUE'
|
||||||
|
|
||||||
|
cur.execute(f'CREATE TABLE IF NOT EXISTS "{table_name}" ({col_defs})')
|
||||||
|
conn.commit()
|
||||||
|
else:
|
||||||
|
# Ensure row_hash column and unique constraint exist
|
||||||
|
cur.execute("""
|
||||||
|
SELECT column_name FROM information_schema.columns
|
||||||
|
WHERE table_schema = current_schema()
|
||||||
|
AND table_name = %s AND column_name = 'row_hash'
|
||||||
|
""", (table_name,))
|
||||||
|
if not cur.fetchone():
|
||||||
|
cur.execute(f'ALTER TABLE "{table_name}" ADD COLUMN IF NOT EXISTS "row_hash" TEXT')
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
# Check for unique constraint on row_hash
|
||||||
|
cur.execute("""
|
||||||
|
SELECT constraint_name FROM information_schema.table_constraints
|
||||||
|
WHERE table_schema = current_schema()
|
||||||
|
AND table_name = %s
|
||||||
|
AND constraint_type = 'UNIQUE'
|
||||||
|
AND constraint_name LIKE %s
|
||||||
|
""", (table_name, f'%{table_name}%row_hash%'))
|
||||||
|
if not cur.fetchone():
|
||||||
|
try:
|
||||||
|
cur.execute(f'CREATE UNIQUE INDEX IF NOT EXISTS "{table_name}_row_hash_idx" ON "{table_name}" ("row_hash")')
|
||||||
|
conn.commit()
|
||||||
|
except Exception:
|
||||||
|
conn.rollback()
|
||||||
|
finally:
|
||||||
|
cur.close()
|
||||||
|
|
||||||
|
def _upsert_slice(conn, tname: str, rows: list, columns: list):
|
||||||
|
"""Upsert a slice of rows using PostgreSQL INSERT ... ON CONFLICT."""
|
||||||
|
if not rows:
|
||||||
|
return
|
||||||
|
|
||||||
|
cur = conn.cursor()
|
||||||
|
try:
|
||||||
|
# Ensure table exists
|
||||||
|
_ensure_table_exists(conn, tname, columns)
|
||||||
|
|
||||||
|
# Build INSERT ... ON CONFLICT statement
|
||||||
|
cols_quoted = ", ".join([f'"{col}"' for col in columns])
|
||||||
|
updates = ", ".join([f'"{col}" = EXCLUDED."{col}"' for col in columns if col != "row_hash"])
|
||||||
|
|
||||||
|
# Build the base query template for execute_values
|
||||||
|
base_query = f'INSERT INTO "{tname}" ({cols_quoted}) VALUES %s'
|
||||||
|
if updates:
|
||||||
|
conflict_query = f'{base_query} ON CONFLICT ("row_hash") DO UPDATE SET {updates}'
|
||||||
|
else:
|
||||||
|
conflict_query = f'{base_query} ON CONFLICT ("row_hash") DO NOTHING'
|
||||||
|
|
||||||
|
# Prepare values as list of tuples
|
||||||
|
values = [tuple(row.get(col) for col in columns) for row in rows]
|
||||||
|
|
||||||
|
execute_values(cur, conflict_query, values, template=None, page_size=len(rows))
|
||||||
|
conn.commit()
|
||||||
|
except Exception:
|
||||||
|
conn.rollback()
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
cur.close()
|
||||||
|
|
||||||
|
def _postgres_upsert(table: str, df: pd.DataFrame):
|
||||||
|
if df.empty:
|
||||||
|
return
|
||||||
|
|
||||||
|
tname = table
|
||||||
|
total = len(df)
|
||||||
|
columns = [col for col in df.columns if col != "row_hash"] + ["row_hash"]
|
||||||
|
|
||||||
|
ranges = [(0, min(UPSERT_CHUNK,total))] if SINGLE_CHUNK else \
|
||||||
|
[(i, min(i+UPSERT_CHUNK,total)) for i in range(0,total,UPSERT_CHUNK)]
|
||||||
|
|
||||||
|
if not PRINT_PROGRESS:
|
||||||
|
lbl = "(single-chunk)" if SINGLE_CHUNK else f"chunk_size={UPSERT_CHUNK}"
|
||||||
|
print_flush(f"{_ts()} | ☁️ Starting upsert → [{tname}] rows={total:,} {lbl}")
|
||||||
|
|
||||||
|
sent = 0
|
||||||
|
conn = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
for start, end in ranges:
|
||||||
|
part = df.iloc[start:end].copy()
|
||||||
|
if "row_hash" in part.columns:
|
||||||
|
part = part.drop_duplicates(subset=["row_hash"], keep="first")
|
||||||
|
|
||||||
|
# Ensure all columns are present
|
||||||
|
for col in columns:
|
||||||
|
if col not in part.columns:
|
||||||
|
part[col] = None
|
||||||
|
|
||||||
|
rows = part[columns].to_dict(orient="records")
|
||||||
|
|
||||||
|
tried_prune = False
|
||||||
|
for attempt in range(1, MAX_RETRIES + 1):
|
||||||
|
try:
|
||||||
|
if conn is None or conn.closed:
|
||||||
|
conn = _pg_conn()
|
||||||
|
_upsert_slice(conn, tname, rows, columns)
|
||||||
|
break
|
||||||
|
except Exception as e:
|
||||||
|
missing = _extract_missing_col_from_error(str(e))
|
||||||
|
if (missing is not None) and (missing in part.columns) and (not tried_prune):
|
||||||
|
if PRINT_PRUNE_LOGS:
|
||||||
|
print_flush(f"{_ts()} | ⚠️ [{tname}] pruning missing column '{missing}'")
|
||||||
|
part = part.drop(columns=[missing])
|
||||||
|
columns = [c for c in columns if c != missing]
|
||||||
|
rows = part[columns].to_dict(orient="records")
|
||||||
|
tried_prune = True
|
||||||
|
continue
|
||||||
|
if attempt == MAX_RETRIES:
|
||||||
|
raise
|
||||||
|
time.sleep(1.0 * attempt)
|
||||||
|
if conn and not conn.closed:
|
||||||
|
conn.close()
|
||||||
|
conn = None
|
||||||
|
|
||||||
|
sent += len(part)
|
||||||
|
if PRINT_PROGRESS:
|
||||||
|
print_flush(f"{_ts()} | ☁️ [{tname}] {sent:,}/{total:,}")
|
||||||
|
|
||||||
|
if not PRINT_PROGRESS:
|
||||||
|
print_flush(f"{_ts()} | ☁️ [{tname}] done {sent:,}/{total:,}")
|
||||||
|
finally:
|
||||||
|
if conn and not conn.closed:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def _postgres_replace(table: str, df: pd.DataFrame):
|
||||||
|
"""Replace all data in table (delete then insert)."""
|
||||||
|
conn = _pg_conn()
|
||||||
|
try:
|
||||||
|
cur = conn.cursor()
|
||||||
|
cur.execute(f'DELETE FROM "{table}"')
|
||||||
|
conn.commit()
|
||||||
|
cur.close()
|
||||||
|
except Exception:
|
||||||
|
conn.rollback()
|
||||||
|
# Table might not exist, that's okay
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
_postgres_upsert(table, df)
|
||||||
|
|
||||||
|
def _load_to_postgres(df: pd.DataFrame, table_name: str, _source_path: str = ""):
|
||||||
|
ndf = _normalize_for_table(df, table_name)
|
||||||
|
if ndf.empty:
|
||||||
|
print_flush(f"{_ts()} | ☁️ Empty after normalize. Skipping PostgreSQL [{table_name}]")
|
||||||
|
return
|
||||||
|
|
||||||
|
ndf = _df_prepare_for_postgres(ndf)
|
||||||
|
|
||||||
|
if INCREMENTAL:
|
||||||
|
_postgres_upsert(table_name, ndf)
|
||||||
|
print_flush(f"{_ts()} | ☁️✅ Upserted {len(ndf):,} rows → PostgreSQL [{table_name}]")
|
||||||
|
else:
|
||||||
|
_postgres_replace(table_name, ndf)
|
||||||
|
print_flush(f"{_ts()} | ☁️✅ Replaced table with {len(ndf):,} rows → PostgreSQL [{table_name}]")
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────
|
||||||
|
# Orchestrator
|
||||||
|
# ─────────────────────────────────────────────
|
||||||
|
def _load_records_to_databases(records: List[Dict], table_name: str):
|
||||||
|
"""Load records from API into both SQLite and PostgreSQL."""
|
||||||
|
if not records:
|
||||||
|
print_flush(f"{_ts()} | ⚠️ No records to process")
|
||||||
|
return
|
||||||
|
|
||||||
|
# Convert records to DataFrame
|
||||||
|
df = pd.DataFrame(records)
|
||||||
|
df = _coerce_weird_numbers(df)
|
||||||
|
|
||||||
|
if WRITE_POSTGRES:
|
||||||
|
try:
|
||||||
|
_load_to_postgres(df, table_name)
|
||||||
|
except Exception as e:
|
||||||
|
print_flush(f"{_ts()} | ❌ (PostgreSQL) Error: {e}")
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────
|
||||||
|
# Main
|
||||||
|
# ─────────────────────────────────────────────
|
||||||
|
def sync_blackbox_flow():
|
||||||
|
"""Main sync function."""
|
||||||
|
print_flush(f"{_ts()} | 🚀 Starting BlackBox Stocks flow data sync...\n")
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Parse command line arguments
|
||||||
|
parser = argparse.ArgumentParser(description="Sync BlackBox Stocks options flow data to databases")
|
||||||
|
parser.add_argument("--start-date", type=str, help="Start date (YYYY-MM-DD)")
|
||||||
|
parser.add_argument("--end-date", type=str, help="End date (YYYY-MM-DD)")
|
||||||
|
parser.add_argument("--limit", type=int, help="Maximum number of records to fetch")
|
||||||
|
parser.add_argument("--count", type=int, help="Maximum number of records to fetch (alias for --limit)")
|
||||||
|
parser.add_argument("--symbol", type=str, help="Filter by specific symbol")
|
||||||
|
parser.add_argument("--table", type=str, default="OptionsFlow_monthly", help="Target table name (default: OptionsFlow_monthly)")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
options = {}
|
||||||
|
if args.start_date:
|
||||||
|
options["startDate"] = args.start_date
|
||||||
|
if args.end_date:
|
||||||
|
options["endDate"] = args.end_date
|
||||||
|
if args.limit:
|
||||||
|
options["count"] = args.limit
|
||||||
|
elif args.count:
|
||||||
|
options["count"] = args.count
|
||||||
|
if args.symbol:
|
||||||
|
options["symbol"] = args.symbol
|
||||||
|
|
||||||
|
table_name = args.table
|
||||||
|
|
||||||
|
# Default to today if no dates provided
|
||||||
|
if not options.get("startDate") and not options.get("endDate"):
|
||||||
|
today = date.today().isoformat()
|
||||||
|
options["startDate"] = today
|
||||||
|
options["endDate"] = today
|
||||||
|
print_flush(f"{_ts()} | 📅 No date range specified, using today: {today}")
|
||||||
|
|
||||||
|
print_flush(f"{_ts()} | 📥 Fetching flow data from BlackBox API...")
|
||||||
|
print_flush(f"{_ts()} | Options: {options}")
|
||||||
|
|
||||||
|
# Fetch data from API
|
||||||
|
api_records = fetch_blackbox_flow(options)
|
||||||
|
print_flush(f"{_ts()} | ✅ Fetched {len(api_records)} records from API")
|
||||||
|
|
||||||
|
if len(api_records) == 0:
|
||||||
|
print_flush(f"{_ts()} | ⚠️ No records returned from API")
|
||||||
|
return
|
||||||
|
|
||||||
|
# Log sample record
|
||||||
|
if len(api_records) > 0:
|
||||||
|
print_flush(f"\n{_ts()} | 📋 Sample API record structure:")
|
||||||
|
print_flush(json.dumps(api_records[0], indent=2, default=str))
|
||||||
|
|
||||||
|
# Map API records to database schema
|
||||||
|
print_flush(f"\n{_ts()} | 🔄 Mapping records to database schema...")
|
||||||
|
mapped_records = [map_blackbox_to_database(record) for record in api_records]
|
||||||
|
print_flush(f"{_ts()} | ✅ Mapped {len(mapped_records)} records")
|
||||||
|
|
||||||
|
# Log sample mapped record
|
||||||
|
if len(mapped_records) > 0:
|
||||||
|
print_flush(f"\n{_ts()} | 📋 Sample mapped record:")
|
||||||
|
print_flush(json.dumps(mapped_records[0], indent=2, default=str))
|
||||||
|
|
||||||
|
# Insert into databases
|
||||||
|
print_flush(f"\n{_ts()} | 💾 Inserting records into databases...")
|
||||||
|
_load_records_to_databases(mapped_records, table_name)
|
||||||
|
|
||||||
|
print_flush(f"\n{_ts()} | ✅ Successfully synced {len(mapped_records)} records")
|
||||||
|
|
||||||
|
# Summary
|
||||||
|
print_flush(f"\n{_ts()} | 📊 Sync Summary:")
|
||||||
|
print_flush(f"{_ts()} | Fetched from API: {len(api_records)} records")
|
||||||
|
print_flush(f"{_ts()} | Inserted into DB: {len(mapped_records)} records")
|
||||||
|
|
||||||
|
# Get total count in PostgreSQL database
|
||||||
|
if WRITE_POSTGRES:
|
||||||
|
try:
|
||||||
|
conn = _pg_conn()
|
||||||
|
cur = conn.cursor()
|
||||||
|
cur.execute(f'SELECT COUNT(*) FROM "{table_name}"')
|
||||||
|
total_count = cur.fetchone()[0]
|
||||||
|
print_flush(f"{_ts()} | Total records in PostgreSQL DB: {total_count}")
|
||||||
|
conn.close()
|
||||||
|
except Exception as e:
|
||||||
|
print_flush(f"{_ts()} | ⚠️ Could not get PostgreSQL count: {e}")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print_flush(f"\n{_ts()} | ❌ Sync failed: {e}")
|
||||||
|
import traceback
|
||||||
|
traceback.print_exc()
|
||||||
|
raise
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
if WRITE_POSTGRES:
|
||||||
|
print_flush(f"{_ts()} | ☁️ PostgreSQL: {POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB} "
|
||||||
|
f"(schema={POSTGRES_SCHEMA}) | INCREMENTAL={INCREMENTAL} | chunk={UPSERT_CHUNK}")
|
||||||
|
sync_blackbox_flow()
|
||||||
Loading…
Reference in New Issue