diff --git a/backend/database/schema.sql b/backend/database/schema.sql index 38ee393..c54cea9 100644 --- a/backend/database/schema.sql +++ b/backend/database/schema.sql @@ -1,5 +1,14 @@ --- Institutional Flow Trading Platform Database Schema --- Run this SQL in your Supabase SQL Editor +-- ============================================ +-- INSTITUTIONAL FLOW TRADING PLATFORM +-- COMPLETE DATABASE SCHEMA +-- ============================================ +-- This script creates all tables and indexes needed for the platform +-- Run this in your Supabase SQL Editor to set up the complete database +-- ============================================ + +-- ============================================ +-- 1. CORE TABLES (from original schema.sql) +-- ============================================ -- Options Flow Table CREATE TABLE IF NOT EXISTS options_flow ( @@ -17,12 +26,6 @@ CREATE TABLE IF NOT EXISTS options_flow ( created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() ); --- Indexes for options_flow -CREATE INDEX IF NOT EXISTS idx_options_flow_symbol ON options_flow(symbol); -CREATE INDEX IF NOT EXISTS idx_options_flow_created_at ON options_flow(created_at DESC); -CREATE INDEX IF NOT EXISTS idx_options_flow_type ON options_flow(type); -CREATE INDEX IF NOT EXISTS idx_options_flow_rocket_score ON options_flow(rocket_score DESC); - -- Daily Analysis Table CREATE TABLE IF NOT EXISTS daily_analysis ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), @@ -37,31 +40,492 @@ CREATE TABLE IF NOT EXISTS daily_analysis ( UNIQUE(symbol, analysis_date) ); +-- ============================================ +-- 2. DATA SOURCE TABLES (from missing_tables.sql) +-- ============================================ + +-- OptionsFlow_monthly (CamelCase) - Main options flow data +CREATE TABLE IF NOT EXISTS public."OptionsFlow_monthly" ( + "CreatedDate" TEXT, + "CreatedTime" TEXT, + "Symbol" TEXT, + "Type" TEXT, + "Volume" TEXT, + "Price" TEXT, + "Side" TEXT, + "CallPut" TEXT, + "Strike" TEXT, + "Spot" TEXT, + "Premium" TEXT, + "ExpirationDate" TEXT, + "Color" TEXT, + "ImpliedVolatility" TEXT, + "Dte" TEXT, + "ER" TEXT, + "StockEtf" TEXT, + "Sector" TEXT, + "Uoa" TEXT, + "Weekly" TEXT, + "MktCap" TEXT, + "OI" TEXT, + "timestamp" TIMESTAMP WITH TIME ZONE, + "aggressor" TEXT +); + +-- AlertStream_monthly (CamelCase) - Alert stream data +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema='public' AND table_name='AlertStream_monthly') THEN + CREATE TABLE public."AlertStream_monthly" ( + "Date" TEXT, + "AlertDate" TEXT, + "date" TEXT, + "alert_date" TEXT, + "Ticker" TEXT, + "ticker" TEXT, + "Symbol" TEXT, + "symbol" TEXT, + "Type" TEXT, + "type" TEXT, + "AlertType" TEXT, + "alert_type" TEXT, + "Message" TEXT, + "message" TEXT, + "Timestamp" TEXT, + "timestamp" TEXT, + "alert_time_txt" TEXT, + "Notional" TEXT, + "notional" TEXT, + "Pct_of_Avg30Day" TEXT, + "pct_of_avg30day" TEXT, + "Pct_of_Avg30" TEXT, + "pct_of_avg30" TEXT, + "Price" NUMERIC(12, 4), + "price" NUMERIC(12, 4), + "Volume" NUMERIC(15, 2), + "volume" NUMERIC(15, 2) + ); + END IF; +END $$; + +-- AlertStream_monthly (lowercase) - Alternative alert stream table +CREATE TABLE IF NOT EXISTS public.alertstream_monthly ( + symbol VARCHAR(10), + alert_type VARCHAR(50), + message TEXT, + notional NUMERIC(15, 2), + pct_of_avg30 NUMERIC(10, 2), + alert_date DATE, + alert_time_txt TEXT +); + +-- prices_intraday_1m - Intraday 1-minute price bars +CREATE TABLE IF NOT EXISTS public.prices_intraday_1m ( + symbol VARCHAR(10) NOT NULL, + ts TIMESTAMP WITH TIME ZONE NOT NULL, + open NUMERIC(12, 4), + high NUMERIC(12, 4), + low NUMERIC(12, 4), + close NUMERIC(12, 4), + volume NUMERIC(15, 2), + short_volume_ratio NUMERIC(5, 4), + "timestamp" TIMESTAMP WITH TIME ZONE, + PRIMARY KEY (symbol, ts) +); + +-- prices_daily - Daily price bars +CREATE TABLE IF NOT EXISTS public.prices_daily ( + symbol VARCHAR(10) NOT NULL, + "Date" DATE NOT NULL, + open NUMERIC(12, 4), + high NUMERIC(12, 4), + low NUMERIC(12, 4), + close NUMERIC(12, 4), + volume NUMERIC(15, 2), + PRIMARY KEY (symbol, "Date") +); + +-- price_daily - Alternative daily price table (used by some queries) +CREATE TABLE IF NOT EXISTS public.price_daily ( + "Ticker" VARCHAR(10) NOT NULL, + dt DATE NOT NULL, + open NUMERIC(12, 4), + high NUMERIC(12, 4), + low NUMERIC(12, 4), + close NUMERIC(12, 4), + volume NUMERIC(15, 2), + PRIMARY KEY ("Ticker", dt) +); + +-- price_1m - Alternative 1-minute price table (used by some queries) +CREATE TABLE IF NOT EXISTS public.price_1m ( + "Ticker" VARCHAR(10) NOT NULL, + ts TIMESTAMP WITH TIME ZONE NOT NULL, + open NUMERIC(12, 4), + high NUMERIC(12, 4), + low NUMERIC(12, 4), + close NUMERIC(12, 4), + volume NUMERIC(15, 2), + PRIMARY KEY ("Ticker", ts) +); + +-- minute_bars_et - Minute bars in Eastern Time +CREATE TABLE IF NOT EXISTS public.minute_bars_et ( + symbol VARCHAR(10) NOT NULL, + d_et DATE NOT NULL, + ts TIMESTAMP WITH TIME ZONE NOT NULL, + session VARCHAR(10) NOT NULL CHECK (session IN ('pre', 'rth', 'post')), + hh INTEGER, + mm INTEGER, + open NUMERIC(12, 4), + high NUMERIC(12, 4), + low NUMERIC(12, 4), + close NUMERIC(12, 4), + volume NUMERIC(15, 2), + PRIMARY KEY (symbol, ts) +); + +-- daily_bars - Daily bars table +CREATE TABLE IF NOT EXISTS public.daily_bars ( + symbol VARCHAR(10) NOT NULL, + date DATE NOT NULL, + open NUMERIC(12, 4), + high NUMERIC(12, 4), + low NUMERIC(12, 4), + close NUMERIC(12, 4), + volume NUMERIC(15, 2), + PRIMARY KEY (symbol, date) +); + +-- bucket_members - Symbol bucket groupings +CREATE TABLE IF NOT EXISTS public.bucket_members ( + symbol VARCHAR(10) NOT NULL, + bucket VARCHAR(50) NOT NULL, + PRIMARY KEY (symbol, bucket) +); + +-- ============================================ +-- 3. INDEXES FOR CORE TABLES +-- ============================================ + +-- Indexes for options_flow +CREATE INDEX IF NOT EXISTS idx_options_flow_symbol ON options_flow(symbol); +CREATE INDEX IF NOT EXISTS idx_options_flow_created_at ON options_flow(created_at DESC); +CREATE INDEX IF NOT EXISTS idx_options_flow_type ON options_flow(type); +CREATE INDEX IF NOT EXISTS idx_options_flow_rocket_score ON options_flow(rocket_score DESC); + -- Indexes for daily_analysis CREATE INDEX IF NOT EXISTS idx_daily_analysis_date ON daily_analysis(analysis_date DESC); CREATE INDEX IF NOT EXISTS idx_daily_analysis_symbol ON daily_analysis(symbol); CREATE INDEX IF NOT EXISTS idx_daily_analysis_score ON daily_analysis(final_score DESC); --- Enable Row Level Security (RLS) - Adjust policies as needed +-- ============================================ +-- 4. PERFORMANCE INDEXES FOR DATA SOURCE TABLES +-- ============================================ + +-- OptionsFlow_monthly Table Indexes +CREATE INDEX IF NOT EXISTS idx_ofm_date_symbol_premium +ON "OptionsFlow_monthly"("CreatedDate", "Symbol", "Premium" DESC) +WHERE "Premium" IS NOT NULL + AND btrim("Premium"::text) <> '' + AND "StockEtf" = 'STOCK'; + +CREATE INDEX IF NOT EXISTS idx_ofm_exp_symbol_strike +ON "OptionsFlow_monthly"("ExpirationDate", "Symbol", "Strike"); + +CREATE INDEX IF NOT EXISTS idx_ofm_high_premium +ON "OptionsFlow_monthly"("CreatedDate" DESC, "Symbol", "Premium" DESC) +WHERE "Premium"::numeric > 80000 + AND "StockEtf" = 'STOCK'; + +CREATE INDEX IF NOT EXISTS idx_ofm_cp_side +ON "OptionsFlow_monthly"( + UPPER(TRIM("CallPut")), + UPPER(TRIM("Side")), + "CreatedDate" DESC +) +WHERE "Premium" IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_ofm_symbol_filter +ON "OptionsFlow_monthly"("Symbol", "CreatedDate" DESC) +WHERE "Symbol" NOT IN ('TSLA','NVDA') + AND "StockEtf" = 'STOCK'; + +CREATE INDEX IF NOT EXISTS idx_optionsflow_monthly_symbol ON public."OptionsFlow_monthly"("Symbol"); +CREATE INDEX IF NOT EXISTS idx_optionsflow_monthly_timestamp ON public."OptionsFlow_monthly"("timestamp" DESC); +CREATE INDEX IF NOT EXISTS idx_optionsflow_monthly_expiration ON public."OptionsFlow_monthly"("ExpirationDate"); + +-- prices_intraday_1m Table Indexes +CREATE INDEX IF NOT EXISTS idx_prices_symbol_ts_desc +ON prices_intraday_1m(UPPER(symbol), ts DESC) +WHERE symbol IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_prices_recent +ON prices_intraday_1m(UPPER(symbol), ts DESC) +WHERE symbol IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_prices_session +ON prices_intraday_1m( + UPPER(symbol), + ts, + EXTRACT(HOUR FROM ts AT TIME ZONE 'America/Chicago') +) +WHERE EXTRACT(HOUR FROM ts AT TIME ZONE 'America/Chicago') BETWEEN 4 AND 20; + +CREATE INDEX IF NOT EXISTS idx_prices_rth_open +ON prices_intraday_1m( + UPPER(symbol), + ts +) +WHERE symbol IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_prices_intraday_1m_symbol_ts ON public.prices_intraday_1m(symbol, ts DESC); +CREATE INDEX IF NOT EXISTS idx_prices_intraday_1m_ts ON public.prices_intraday_1m(ts DESC); +CREATE INDEX IF NOT EXISTS idx_prices_intraday_1m_timestamp ON public.prices_intraday_1m("timestamp" DESC); + +-- prices_daily Table Indexes +CREATE INDEX IF NOT EXISTS idx_prices_daily_symbol_date +ON prices_daily(UPPER(symbol), "Date" DESC) +WHERE symbol IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_prices_daily_symbol_date_alt ON public.prices_daily(symbol, "Date" DESC); +CREATE INDEX IF NOT EXISTS idx_prices_daily_date ON public.prices_daily("Date" DESC); + +-- AlertStream_monthly Table Indexes +CREATE INDEX IF NOT EXISTS idx_alert_ticker_time +ON "AlertStream_monthly"( + UPPER("ticker"), + "date", + "timestamp" +) +WHERE "ticker" IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_alert_recent +ON "AlertStream_monthly"( + UPPER("ticker"), + "date", + "timestamp" +) +WHERE "ticker" IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_alert_type +ON "AlertStream_monthly"("type", "date", UPPER("ticker")) +WHERE "ticker" IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_alert_dark_block +ON "AlertStream_monthly"( + UPPER("ticker"), + "date", + "timestamp" +) +WHERE "type" IN ('DarkPool','Block') + AND "ticker" IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_ticker ON public."AlertStream_monthly"("Ticker"); +CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_symbol ON public."AlertStream_monthly"("symbol"); +CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_type ON public."AlertStream_monthly"("Type"); + +-- alertstream_monthly (lowercase) Indexes +CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_lower_symbol ON public.alertstream_monthly(symbol); +CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_lower_date ON public.alertstream_monthly(alert_date); +CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_lower_type ON public.alertstream_monthly(alert_type); + +-- price_daily Table Indexes +CREATE INDEX IF NOT EXISTS idx_price_daily_ticker_dt ON public.price_daily("Ticker", dt DESC); +CREATE INDEX IF NOT EXISTS idx_price_daily_dt ON public.price_daily(dt DESC); + +-- price_1m Table Indexes +CREATE INDEX IF NOT EXISTS idx_price_1m_ticker_ts ON public.price_1m("Ticker", ts DESC); +CREATE INDEX IF NOT EXISTS idx_price_1m_ts ON public.price_1m(ts DESC); + +-- minute_bars_et Table Indexes +CREATE INDEX IF NOT EXISTS idx_minute_bars_et_symbol_date ON public.minute_bars_et(symbol, d_et); +CREATE INDEX IF NOT EXISTS idx_minute_bars_et_session ON public.minute_bars_et(session, d_et); +CREATE INDEX IF NOT EXISTS idx_minute_bars_et_ts ON public.minute_bars_et(ts DESC); + +-- daily_bars Table Indexes +CREATE INDEX IF NOT EXISTS idx_daily_bars_symbol_date ON public.daily_bars(symbol, date DESC); +CREATE INDEX IF NOT EXISTS idx_daily_bars_date ON public.daily_bars(date DESC); + +-- bucket_members Table Indexes +CREATE INDEX IF NOT EXISTS idx_bucket_members_bucket ON public.bucket_members(bucket); +CREATE INDEX IF NOT EXISTS idx_bucket_members_symbol ON public.bucket_members(symbol); + +-- ============================================ +-- 5. ROW LEVEL SECURITY (RLS) +-- ============================================ + +-- Enable RLS on all tables ALTER TABLE options_flow ENABLE ROW LEVEL SECURITY; ALTER TABLE daily_analysis ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.minute_bars_et ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.bucket_members ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.daily_bars ENABLE ROW LEVEL SECURITY; +ALTER TABLE public."AlertStream_monthly" ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.alertstream_monthly ENABLE ROW LEVEL SECURITY; +ALTER TABLE public."OptionsFlow_monthly" ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.prices_intraday_1m ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.prices_daily ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.price_daily ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.price_1m ENABLE ROW LEVEL SECURITY; --- Create a policy that allows all operations (adjust for your security needs) --- For development, you might want to allow all. For production, create specific policies. -CREATE POLICY "Allow all operations on options_flow" ON options_flow - FOR ALL - USING (true) - WITH CHECK (true); +-- Create permissive policies for development (adjust for production) +-- Note: If policies already exist, you may need to drop them first manually +-- or modify this section to handle existing policies -CREATE POLICY "Allow all operations on daily_analysis" ON daily_analysis - FOR ALL - USING (true) - WITH CHECK (true); +DO $$ +BEGIN + -- options_flow + BEGIN + DROP POLICY "Allow all operations on options_flow" ON options_flow; + EXCEPTION WHEN undefined_object THEN NULL; + END; + CREATE POLICY "Allow all operations on options_flow" ON options_flow + FOR ALL USING (true) WITH CHECK (true); +END $$; --- Sample data (optional - for testing) --- INSERT INTO options_flow (symbol, type, strike, expiration, total_premium, volume, sentiment, volume_ratio) --- VALUES --- ('AAPL', 'CALL', 180.00, NOW() + INTERVAL '30 days', 50000, 100, 0.75, 5.2), --- ('TSLA', 'PUT', 250.00, NOW() + INTERVAL '45 days', 75000, 200, 0.25, 8.5), --- ('NVDA', 'CALL', 500.00, NOW() + INTERVAL '60 days', 100000, 150, 0.85, 12.3); +DO $$ +BEGIN + -- daily_analysis + BEGIN + DROP POLICY "Allow all operations on daily_analysis" ON daily_analysis; + EXCEPTION WHEN undefined_object THEN NULL; + END; + CREATE POLICY "Allow all operations on daily_analysis" ON daily_analysis + FOR ALL USING (true) WITH CHECK (true); +END $$; +DO $$ +BEGIN + -- minute_bars_et + BEGIN + DROP POLICY "Allow all operations on minute_bars_et" ON public.minute_bars_et; + EXCEPTION WHEN undefined_object THEN NULL; + END; + CREATE POLICY "Allow all operations on minute_bars_et" ON public.minute_bars_et + FOR ALL USING (true) WITH CHECK (true); +END $$; + +DO $$ +BEGIN + -- bucket_members + BEGIN + DROP POLICY "Allow all operations on bucket_members" ON public.bucket_members; + EXCEPTION WHEN undefined_object THEN NULL; + END; + CREATE POLICY "Allow all operations on bucket_members" ON public.bucket_members + FOR ALL USING (true) WITH CHECK (true); +END $$; + +DO $$ +BEGIN + -- daily_bars + BEGIN + DROP POLICY "Allow all operations on daily_bars" ON public.daily_bars; + EXCEPTION WHEN undefined_object THEN NULL; + END; + CREATE POLICY "Allow all operations on daily_bars" ON public.daily_bars + FOR ALL USING (true) WITH CHECK (true); +END $$; + +DO $$ +BEGIN + -- AlertStream_monthly + BEGIN + DROP POLICY "Allow all operations on AlertStream_monthly" ON public."AlertStream_monthly"; + EXCEPTION WHEN undefined_object THEN NULL; + END; + CREATE POLICY "Allow all operations on AlertStream_monthly" ON public."AlertStream_monthly" + FOR ALL USING (true) WITH CHECK (true); +END $$; + +DO $$ +BEGIN + -- alertstream_monthly + BEGIN + DROP POLICY "Allow all operations on alertstream_monthly" ON public.alertstream_monthly; + EXCEPTION WHEN undefined_object THEN NULL; + END; + CREATE POLICY "Allow all operations on alertstream_monthly" ON public.alertstream_monthly + FOR ALL USING (true) WITH CHECK (true); +END $$; + +DO $$ +BEGIN + -- OptionsFlow_monthly + BEGIN + DROP POLICY "Allow all operations on OptionsFlow_monthly" ON public."OptionsFlow_monthly"; + EXCEPTION WHEN undefined_object THEN NULL; + END; + CREATE POLICY "Allow all operations on OptionsFlow_monthly" ON public."OptionsFlow_monthly" + FOR ALL USING (true) WITH CHECK (true); +END $$; + +DO $$ +BEGIN + -- prices_intraday_1m + BEGIN + DROP POLICY "Allow all operations on prices_intraday_1m" ON public.prices_intraday_1m; + EXCEPTION WHEN undefined_object THEN NULL; + END; + CREATE POLICY "Allow all operations on prices_intraday_1m" ON public.prices_intraday_1m + FOR ALL USING (true) WITH CHECK (true); +END $$; + +DO $$ +BEGIN + -- prices_daily + BEGIN + DROP POLICY "Allow all operations on prices_daily" ON public.prices_daily; + EXCEPTION WHEN undefined_object THEN NULL; + END; + CREATE POLICY "Allow all operations on prices_daily" ON public.prices_daily + FOR ALL USING (true) WITH CHECK (true); +END $$; + +DO $$ +BEGIN + -- price_daily + BEGIN + DROP POLICY "Allow all operations on price_daily" ON public.price_daily; + EXCEPTION WHEN undefined_object THEN NULL; + END; + CREATE POLICY "Allow all operations on price_daily" ON public.price_daily + FOR ALL USING (true) WITH CHECK (true); +END $$; + +DO $$ +BEGIN + -- price_1m + BEGIN + DROP POLICY "Allow all operations on price_1m" ON public.price_1m; + EXCEPTION WHEN undefined_object THEN NULL; + END; + CREATE POLICY "Allow all operations on price_1m" ON public.price_1m + FOR ALL USING (true) WITH CHECK (true); +END $$; + +-- ============================================ +-- 6. ANALYZE TABLES FOR QUERY PLANNER +-- ============================================ + +-- Update statistics for query planner +ANALYZE options_flow; +ANALYZE daily_analysis; +ANALYZE "OptionsFlow_monthly"; +ANALYZE prices_intraday_1m; +ANALYZE prices_daily; +ANALYZE price_daily; +ANALYZE price_1m; +ANALYZE "AlertStream_monthly"; +ANALYZE alertstream_monthly; +ANALYZE minute_bars_et; +ANALYZE daily_bars; +ANALYZE bucket_members; + +-- ============================================ +-- SCHEMA COMPLETE +-- ============================================ +-- All tables and indexes have been created +-- The database is now ready for use +-- ============================================ \ No newline at end of file