updated schema sql

backend/database/schema.sql

@@ -1,5 +1,14 @@
--- Institutional Flow Trading Platform Database Schema
+-- ============================================
--- Run this SQL in your Supabase SQL Editor
+-- INSTITUTIONAL FLOW TRADING PLATFORM
+-- COMPLETE DATABASE SCHEMA
+-- ============================================
+-- This script creates all tables and indexes needed for the platform
+-- Run this in your Supabase SQL Editor to set up the complete database
+-- ============================================
+
+-- ============================================
+-- 1. CORE TABLES (from original schema.sql)
+-- ============================================
 
 -- Options Flow Table
 CREATE TABLE IF NOT EXISTS options_flow (
@@ -17,12 +26,6 @@ CREATE TABLE IF NOT EXISTS options_flow (
   created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
 );
 
--- Indexes for options_flow
-CREATE INDEX IF NOT EXISTS idx_options_flow_symbol ON options_flow(symbol);
-CREATE INDEX IF NOT EXISTS idx_options_flow_created_at ON options_flow(created_at DESC);
-CREATE INDEX IF NOT EXISTS idx_options_flow_type ON options_flow(type);
-CREATE INDEX IF NOT EXISTS idx_options_flow_rocket_score ON options_flow(rocket_score DESC);
-
 -- Daily Analysis Table
 CREATE TABLE IF NOT EXISTS daily_analysis (
   id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
@@ -37,31 +40,492 @@ CREATE TABLE IF NOT EXISTS daily_analysis (
   UNIQUE(symbol, analysis_date)
 );
 
+-- ============================================
+-- 2. DATA SOURCE TABLES (from missing_tables.sql)
+-- ============================================
+
+-- OptionsFlow_monthly (CamelCase) - Main options flow data
+CREATE TABLE IF NOT EXISTS public."OptionsFlow_monthly" (
+  "CreatedDate" TEXT,
+  "CreatedTime" TEXT,
+  "Symbol" TEXT,
+  "Type" TEXT,
+  "Volume" TEXT,
+  "Price" TEXT,
+  "Side" TEXT,
+  "CallPut" TEXT,
+  "Strike" TEXT,
+  "Spot" TEXT,
+  "Premium" TEXT,
+  "ExpirationDate" TEXT,
+  "Color" TEXT,
+  "ImpliedVolatility" TEXT,
+  "Dte" TEXT,
+  "ER" TEXT,
+  "StockEtf" TEXT,
+  "Sector" TEXT,
+  "Uoa" TEXT,
+  "Weekly" TEXT,
+  "MktCap" TEXT,
+  "OI" TEXT,
+  "timestamp" TIMESTAMP WITH TIME ZONE,
+  "aggressor" TEXT
+);
+
+-- AlertStream_monthly (CamelCase) - Alert stream data
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema='public' AND table_name='AlertStream_monthly') THEN
+    CREATE TABLE public."AlertStream_monthly" (
+      "Date" TEXT,
+      "AlertDate" TEXT,
+      "date" TEXT,
+      "alert_date" TEXT,
+      "Ticker" TEXT,
+      "ticker" TEXT,
+      "Symbol" TEXT,
+      "symbol" TEXT,
+      "Type" TEXT,
+      "type" TEXT,
+      "AlertType" TEXT,
+      "alert_type" TEXT,
+      "Message" TEXT,
+      "message" TEXT,
+      "Timestamp" TEXT,
+      "timestamp" TEXT,
+      "alert_time_txt" TEXT,
+      "Notional" TEXT,
+      "notional" TEXT,
+      "Pct_of_Avg30Day" TEXT,
+      "pct_of_avg30day" TEXT,
+      "Pct_of_Avg30" TEXT,
+      "pct_of_avg30" TEXT,
+      "Price" NUMERIC(12, 4),
+      "price" NUMERIC(12, 4),
+      "Volume" NUMERIC(15, 2),
+      "volume" NUMERIC(15, 2)
+    );
+  END IF;
+END $$;
+
+-- AlertStream_monthly (lowercase) - Alternative alert stream table
+CREATE TABLE IF NOT EXISTS public.alertstream_monthly (
+  symbol VARCHAR(10),
+  alert_type VARCHAR(50),
+  message TEXT,
+  notional NUMERIC(15, 2),
+  pct_of_avg30 NUMERIC(10, 2),
+  alert_date DATE,
+  alert_time_txt TEXT
+);
+
+-- prices_intraday_1m - Intraday 1-minute price bars
+CREATE TABLE IF NOT EXISTS public.prices_intraday_1m (
+  symbol VARCHAR(10) NOT NULL,
+  ts TIMESTAMP WITH TIME ZONE NOT NULL,
+  open NUMERIC(12, 4),
+  high NUMERIC(12, 4),
+  low NUMERIC(12, 4),
+  close NUMERIC(12, 4),
+  volume NUMERIC(15, 2),
+  short_volume_ratio NUMERIC(5, 4),
+  "timestamp" TIMESTAMP WITH TIME ZONE,
+  PRIMARY KEY (symbol, ts)
+);
+
+-- prices_daily - Daily price bars
+CREATE TABLE IF NOT EXISTS public.prices_daily (
+  symbol VARCHAR(10) NOT NULL,
+  "Date" DATE NOT NULL,
+  open NUMERIC(12, 4),
+  high NUMERIC(12, 4),
+  low NUMERIC(12, 4),
+  close NUMERIC(12, 4),
+  volume NUMERIC(15, 2),
+  PRIMARY KEY (symbol, "Date")
+);
+
+-- price_daily - Alternative daily price table (used by some queries)
+CREATE TABLE IF NOT EXISTS public.price_daily (
+  "Ticker" VARCHAR(10) NOT NULL,
+  dt DATE NOT NULL,
+  open NUMERIC(12, 4),
+  high NUMERIC(12, 4),
+  low NUMERIC(12, 4),
+  close NUMERIC(12, 4),
+  volume NUMERIC(15, 2),
+  PRIMARY KEY ("Ticker", dt)
+);
+
+-- price_1m - Alternative 1-minute price table (used by some queries)
+CREATE TABLE IF NOT EXISTS public.price_1m (
+  "Ticker" VARCHAR(10) NOT NULL,
+  ts TIMESTAMP WITH TIME ZONE NOT NULL,
+  open NUMERIC(12, 4),
+  high NUMERIC(12, 4),
+  low NUMERIC(12, 4),
+  close NUMERIC(12, 4),
+  volume NUMERIC(15, 2),
+  PRIMARY KEY ("Ticker", ts)
+);
+
+-- minute_bars_et - Minute bars in Eastern Time
+CREATE TABLE IF NOT EXISTS public.minute_bars_et (
+  symbol VARCHAR(10) NOT NULL,
+  d_et DATE NOT NULL,
+  ts TIMESTAMP WITH TIME ZONE NOT NULL,
+  session VARCHAR(10) NOT NULL CHECK (session IN ('pre', 'rth', 'post')),
+  hh INTEGER,
+  mm INTEGER,
+  open NUMERIC(12, 4),
+  high NUMERIC(12, 4),
+  low NUMERIC(12, 4),
+  close NUMERIC(12, 4),
+  volume NUMERIC(15, 2),
+  PRIMARY KEY (symbol, ts)
+);
+
+-- daily_bars - Daily bars table
+CREATE TABLE IF NOT EXISTS public.daily_bars (
+  symbol VARCHAR(10) NOT NULL,
+  date DATE NOT NULL,
+  open NUMERIC(12, 4),
+  high NUMERIC(12, 4),
+  low NUMERIC(12, 4),
+  close NUMERIC(12, 4),
+  volume NUMERIC(15, 2),
+  PRIMARY KEY (symbol, date)
+);
+
+-- bucket_members - Symbol bucket groupings
+CREATE TABLE IF NOT EXISTS public.bucket_members (
+  symbol VARCHAR(10) NOT NULL,
+  bucket VARCHAR(50) NOT NULL,
+  PRIMARY KEY (symbol, bucket)
+);
+
+-- ============================================
+-- 3. INDEXES FOR CORE TABLES
+-- ============================================
+
+-- Indexes for options_flow
+CREATE INDEX IF NOT EXISTS idx_options_flow_symbol ON options_flow(symbol);
+CREATE INDEX IF NOT EXISTS idx_options_flow_created_at ON options_flow(created_at DESC);
+CREATE INDEX IF NOT EXISTS idx_options_flow_type ON options_flow(type);
+CREATE INDEX IF NOT EXISTS idx_options_flow_rocket_score ON options_flow(rocket_score DESC);
+
 -- Indexes for daily_analysis
 CREATE INDEX IF NOT EXISTS idx_daily_analysis_date ON daily_analysis(analysis_date DESC);
 CREATE INDEX IF NOT EXISTS idx_daily_analysis_symbol ON daily_analysis(symbol);
 CREATE INDEX IF NOT EXISTS idx_daily_analysis_score ON daily_analysis(final_score DESC);
 
--- Enable Row Level Security (RLS) - Adjust policies as needed
+-- ============================================
+-- 4. PERFORMANCE INDEXES FOR DATA SOURCE TABLES
+-- ============================================
+
+-- OptionsFlow_monthly Table Indexes
+CREATE INDEX IF NOT EXISTS idx_ofm_date_symbol_premium 
+ON "OptionsFlow_monthly"("CreatedDate", "Symbol", "Premium" DESC)
+WHERE "Premium" IS NOT NULL 
+  AND btrim("Premium"::text) <> ''
+  AND "StockEtf" = 'STOCK';
+
+CREATE INDEX IF NOT EXISTS idx_ofm_exp_symbol_strike 
+ON "OptionsFlow_monthly"("ExpirationDate", "Symbol", "Strike");
+
+CREATE INDEX IF NOT EXISTS idx_ofm_high_premium 
+ON "OptionsFlow_monthly"("CreatedDate" DESC, "Symbol", "Premium" DESC)
+WHERE "Premium"::numeric > 80000
+  AND "StockEtf" = 'STOCK';
+
+CREATE INDEX IF NOT EXISTS idx_ofm_cp_side 
+ON "OptionsFlow_monthly"(
+  UPPER(TRIM("CallPut")), 
+  UPPER(TRIM("Side")),
+  "CreatedDate" DESC
+)
+WHERE "Premium" IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS idx_ofm_symbol_filter 
+ON "OptionsFlow_monthly"("Symbol", "CreatedDate" DESC)
+WHERE "Symbol" NOT IN ('TSLA','NVDA')
+  AND "StockEtf" = 'STOCK';
+
+CREATE INDEX IF NOT EXISTS idx_optionsflow_monthly_symbol ON public."OptionsFlow_monthly"("Symbol");
+CREATE INDEX IF NOT EXISTS idx_optionsflow_monthly_timestamp ON public."OptionsFlow_monthly"("timestamp" DESC);
+CREATE INDEX IF NOT EXISTS idx_optionsflow_monthly_expiration ON public."OptionsFlow_monthly"("ExpirationDate");
+
+-- prices_intraday_1m Table Indexes
+CREATE INDEX IF NOT EXISTS idx_prices_symbol_ts_desc 
+ON prices_intraday_1m(UPPER(symbol), ts DESC)
+WHERE symbol IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS idx_prices_recent 
+ON prices_intraday_1m(UPPER(symbol), ts DESC)
+WHERE symbol IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS idx_prices_session 
+ON prices_intraday_1m(
+  UPPER(symbol), 
+  ts,
+  EXTRACT(HOUR FROM ts AT TIME ZONE 'America/Chicago')
+)
+WHERE EXTRACT(HOUR FROM ts AT TIME ZONE 'America/Chicago') BETWEEN 4 AND 20;
+
+CREATE INDEX IF NOT EXISTS idx_prices_rth_open 
+ON prices_intraday_1m(
+  UPPER(symbol),
+  ts
+)
+WHERE symbol IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS idx_prices_intraday_1m_symbol_ts ON public.prices_intraday_1m(symbol, ts DESC);
+CREATE INDEX IF NOT EXISTS idx_prices_intraday_1m_ts ON public.prices_intraday_1m(ts DESC);
+CREATE INDEX IF NOT EXISTS idx_prices_intraday_1m_timestamp ON public.prices_intraday_1m("timestamp" DESC);
+
+-- prices_daily Table Indexes
+CREATE INDEX IF NOT EXISTS idx_prices_daily_symbol_date 
+ON prices_daily(UPPER(symbol), "Date" DESC)
+WHERE symbol IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS idx_prices_daily_symbol_date_alt ON public.prices_daily(symbol, "Date" DESC);
+CREATE INDEX IF NOT EXISTS idx_prices_daily_date ON public.prices_daily("Date" DESC);
+
+-- AlertStream_monthly Table Indexes
+CREATE INDEX IF NOT EXISTS idx_alert_ticker_time 
+ON "AlertStream_monthly"(
+  UPPER("ticker"), 
+  "date", 
+  "timestamp"
+)
+WHERE "ticker" IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS idx_alert_recent 
+ON "AlertStream_monthly"(
+  UPPER("ticker"), 
+  "date", 
+  "timestamp"
+)
+WHERE "ticker" IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS idx_alert_type 
+ON "AlertStream_monthly"("type", "date", UPPER("ticker"))
+WHERE "ticker" IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS idx_alert_dark_block 
+ON "AlertStream_monthly"(
+  UPPER("ticker"),
+  "date",
+  "timestamp"
+)
+WHERE "type" IN ('DarkPool','Block')
+  AND "ticker" IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_ticker ON public."AlertStream_monthly"("Ticker");
+CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_symbol ON public."AlertStream_monthly"("symbol");
+CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_type ON public."AlertStream_monthly"("Type");
+
+-- alertstream_monthly (lowercase) Indexes
+CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_lower_symbol ON public.alertstream_monthly(symbol);
+CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_lower_date ON public.alertstream_monthly(alert_date);
+CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_lower_type ON public.alertstream_monthly(alert_type);
+
+-- price_daily Table Indexes
+CREATE INDEX IF NOT EXISTS idx_price_daily_ticker_dt ON public.price_daily("Ticker", dt DESC);
+CREATE INDEX IF NOT EXISTS idx_price_daily_dt ON public.price_daily(dt DESC);
+
+-- price_1m Table Indexes
+CREATE INDEX IF NOT EXISTS idx_price_1m_ticker_ts ON public.price_1m("Ticker", ts DESC);
+CREATE INDEX IF NOT EXISTS idx_price_1m_ts ON public.price_1m(ts DESC);
+
+-- minute_bars_et Table Indexes
+CREATE INDEX IF NOT EXISTS idx_minute_bars_et_symbol_date ON public.minute_bars_et(symbol, d_et);
+CREATE INDEX IF NOT EXISTS idx_minute_bars_et_session ON public.minute_bars_et(session, d_et);
+CREATE INDEX IF NOT EXISTS idx_minute_bars_et_ts ON public.minute_bars_et(ts DESC);
+
+-- daily_bars Table Indexes
+CREATE INDEX IF NOT EXISTS idx_daily_bars_symbol_date ON public.daily_bars(symbol, date DESC);
+CREATE INDEX IF NOT EXISTS idx_daily_bars_date ON public.daily_bars(date DESC);
+
+-- bucket_members Table Indexes
+CREATE INDEX IF NOT EXISTS idx_bucket_members_bucket ON public.bucket_members(bucket);
+CREATE INDEX IF NOT EXISTS idx_bucket_members_symbol ON public.bucket_members(symbol);
+
+-- ============================================
+-- 5. ROW LEVEL SECURITY (RLS)
+-- ============================================
+
+-- Enable RLS on all tables
 ALTER TABLE options_flow ENABLE ROW LEVEL SECURITY;
 ALTER TABLE daily_analysis ENABLE ROW LEVEL SECURITY;
+ALTER TABLE public.minute_bars_et ENABLE ROW LEVEL SECURITY;
+ALTER TABLE public.bucket_members ENABLE ROW LEVEL SECURITY;
+ALTER TABLE public.daily_bars ENABLE ROW LEVEL SECURITY;
+ALTER TABLE public."AlertStream_monthly" ENABLE ROW LEVEL SECURITY;
+ALTER TABLE public.alertstream_monthly ENABLE ROW LEVEL SECURITY;
+ALTER TABLE public."OptionsFlow_monthly" ENABLE ROW LEVEL SECURITY;
+ALTER TABLE public.prices_intraday_1m ENABLE ROW LEVEL SECURITY;
+ALTER TABLE public.prices_daily ENABLE ROW LEVEL SECURITY;
+ALTER TABLE public.price_daily ENABLE ROW LEVEL SECURITY;
+ALTER TABLE public.price_1m ENABLE ROW LEVEL SECURITY;
 
--- Create a policy that allows all operations (adjust for your security needs)
+-- Create permissive policies for development (adjust for production)
--- For development, you might want to allow all. For production, create specific policies.
+-- Note: If policies already exist, you may need to drop them first manually
+-- or modify this section to handle existing policies
+
+DO $$
+BEGIN
+  -- options_flow
+  BEGIN
+    DROP POLICY "Allow all operations on options_flow" ON options_flow;
+  EXCEPTION WHEN undefined_object THEN NULL;
+  END;
   CREATE POLICY "Allow all operations on options_flow" ON options_flow
-  FOR ALL
+    FOR ALL USING (true) WITH CHECK (true);
-  USING (true)
+END $$;
-  WITH CHECK (true);
 
+DO $$
+BEGIN
+  -- daily_analysis
+  BEGIN
+    DROP POLICY "Allow all operations on daily_analysis" ON daily_analysis;
+  EXCEPTION WHEN undefined_object THEN NULL;
+  END;
   CREATE POLICY "Allow all operations on daily_analysis" ON daily_analysis
-  FOR ALL
+    FOR ALL USING (true) WITH CHECK (true);
-  USING (true)
+END $$;
-  WITH CHECK (true);
 
--- Sample data (optional - for testing)
+DO $$
--- INSERT INTO options_flow (symbol, type, strike, expiration, total_premium, volume, sentiment, volume_ratio)
+BEGIN
--- VALUES 
+  -- minute_bars_et
---   ('AAPL', 'CALL', 180.00, NOW() + INTERVAL '30 days', 50000, 100, 0.75, 5.2),
+  BEGIN
---   ('TSLA', 'PUT', 250.00, NOW() + INTERVAL '45 days', 75000, 200, 0.25, 8.5),
+    DROP POLICY "Allow all operations on minute_bars_et" ON public.minute_bars_et;
---   ('NVDA', 'CALL', 500.00, NOW() + INTERVAL '60 days', 100000, 150, 0.85, 12.3);
+  EXCEPTION WHEN undefined_object THEN NULL;
+  END;
+  CREATE POLICY "Allow all operations on minute_bars_et" ON public.minute_bars_et
+    FOR ALL USING (true) WITH CHECK (true);
+END $$;
 
+DO $$
+BEGIN
+  -- bucket_members
+  BEGIN
+    DROP POLICY "Allow all operations on bucket_members" ON public.bucket_members;
+  EXCEPTION WHEN undefined_object THEN NULL;
+  END;
+  CREATE POLICY "Allow all operations on bucket_members" ON public.bucket_members
+    FOR ALL USING (true) WITH CHECK (true);
+END $$;
+
+DO $$
+BEGIN
+  -- daily_bars
+  BEGIN
+    DROP POLICY "Allow all operations on daily_bars" ON public.daily_bars;
+  EXCEPTION WHEN undefined_object THEN NULL;
+  END;
+  CREATE POLICY "Allow all operations on daily_bars" ON public.daily_bars
+    FOR ALL USING (true) WITH CHECK (true);
+END $$;
+
+DO $$
+BEGIN
+  -- AlertStream_monthly
+  BEGIN
+    DROP POLICY "Allow all operations on AlertStream_monthly" ON public."AlertStream_monthly";
+  EXCEPTION WHEN undefined_object THEN NULL;
+  END;
+  CREATE POLICY "Allow all operations on AlertStream_monthly" ON public."AlertStream_monthly"
+    FOR ALL USING (true) WITH CHECK (true);
+END $$;
+
+DO $$
+BEGIN
+  -- alertstream_monthly
+  BEGIN
+    DROP POLICY "Allow all operations on alertstream_monthly" ON public.alertstream_monthly;
+  EXCEPTION WHEN undefined_object THEN NULL;
+  END;
+  CREATE POLICY "Allow all operations on alertstream_monthly" ON public.alertstream_monthly
+    FOR ALL USING (true) WITH CHECK (true);
+END $$;
+
+DO $$
+BEGIN
+  -- OptionsFlow_monthly
+  BEGIN
+    DROP POLICY "Allow all operations on OptionsFlow_monthly" ON public."OptionsFlow_monthly";
+  EXCEPTION WHEN undefined_object THEN NULL;
+  END;
+  CREATE POLICY "Allow all operations on OptionsFlow_monthly" ON public."OptionsFlow_monthly"
+    FOR ALL USING (true) WITH CHECK (true);
+END $$;
+
+DO $$
+BEGIN
+  -- prices_intraday_1m
+  BEGIN
+    DROP POLICY "Allow all operations on prices_intraday_1m" ON public.prices_intraday_1m;
+  EXCEPTION WHEN undefined_object THEN NULL;
+  END;
+  CREATE POLICY "Allow all operations on prices_intraday_1m" ON public.prices_intraday_1m
+    FOR ALL USING (true) WITH CHECK (true);
+END $$;
+
+DO $$
+BEGIN
+  -- prices_daily
+  BEGIN
+    DROP POLICY "Allow all operations on prices_daily" ON public.prices_daily;
+  EXCEPTION WHEN undefined_object THEN NULL;
+  END;
+  CREATE POLICY "Allow all operations on prices_daily" ON public.prices_daily
+    FOR ALL USING (true) WITH CHECK (true);
+END $$;
+
+DO $$
+BEGIN
+  -- price_daily
+  BEGIN
+    DROP POLICY "Allow all operations on price_daily" ON public.price_daily;
+  EXCEPTION WHEN undefined_object THEN NULL;
+  END;
+  CREATE POLICY "Allow all operations on price_daily" ON public.price_daily
+    FOR ALL USING (true) WITH CHECK (true);
+END $$;
+
+DO $$
+BEGIN
+  -- price_1m
+  BEGIN
+    DROP POLICY "Allow all operations on price_1m" ON public.price_1m;
+  EXCEPTION WHEN undefined_object THEN NULL;
+  END;
+  CREATE POLICY "Allow all operations on price_1m" ON public.price_1m
+    FOR ALL USING (true) WITH CHECK (true);
+END $$;
+
+-- ============================================
+-- 6. ANALYZE TABLES FOR QUERY PLANNER
+-- ============================================
+
+-- Update statistics for query planner
+ANALYZE options_flow;
+ANALYZE daily_analysis;
+ANALYZE "OptionsFlow_monthly";
+ANALYZE prices_intraday_1m;
+ANALYZE prices_daily;
+ANALYZE price_daily;
+ANALYZE price_1m;
+ANALYZE "AlertStream_monthly";
+ANALYZE alertstream_monthly;
+ANALYZE minute_bars_et;
+ANALYZE daily_bars;
+ANALYZE bucket_members;
+
+-- ============================================
+-- SCHEMA COMPLETE
+-- ============================================
+-- All tables and indexes have been created
+-- The database is now ready for use
+-- ============================================