institutional-trader/backend/database/schema.sql

531 lines
17 KiB
SQL

-- ============================================
-- INSTITUTIONAL FLOW TRADING PLATFORM
-- COMPLETE DATABASE SCHEMA
-- ============================================
-- This script creates all tables and indexes needed for the platform
-- Run this in your Supabase SQL Editor to set up the complete database
-- ============================================
-- ============================================
-- 1. CORE TABLES (from original schema.sql)
-- ============================================
-- Options Flow Table
CREATE TABLE IF NOT EXISTS options_flow (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
symbol VARCHAR(10) NOT NULL,
type VARCHAR(4) NOT NULL CHECK (type IN ('CALL', 'PUT')),
strike DECIMAL(10, 2) NOT NULL,
expiration TIMESTAMP WITH TIME ZONE,
total_premium DECIMAL(15, 2) DEFAULT 0,
volume INTEGER DEFAULT 0,
sentiment DECIMAL(3, 2) DEFAULT 0.5 CHECK (sentiment >= 0 AND sentiment <= 1),
volume_ratio DECIMAL(10, 2) DEFAULT 1,
execution_time INTEGER DEFAULT 0,
rocket_score INTEGER CHECK (rocket_score >= 0 AND rocket_score <= 100),
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
);
-- Daily Analysis Table
CREATE TABLE IF NOT EXISTS daily_analysis (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
symbol VARCHAR(10) NOT NULL,
analysis_date DATE NOT NULL,
final_score INTEGER CHECK (final_score >= 0 AND final_score <= 100),
price_change_percent DECIMAL(5, 2) DEFAULT 0,
flow_count INTEGER DEFAULT 0,
total_premium DECIMAL(15, 2) DEFAULT 0,
call_put_ratio DECIMAL(5, 2) DEFAULT 1,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
UNIQUE(symbol, analysis_date)
);
-- ============================================
-- 2. DATA SOURCE TABLES (from missing_tables.sql)
-- ============================================
-- OptionsFlow_monthly (CamelCase) - Main options flow data
CREATE TABLE IF NOT EXISTS public."OptionsFlow_monthly" (
"CreatedDate" TEXT,
"CreatedTime" TEXT,
"Symbol" TEXT,
"Type" TEXT,
"Volume" TEXT,
"Price" TEXT,
"Side" TEXT,
"CallPut" TEXT,
"Strike" TEXT,
"Spot" TEXT,
"Premium" TEXT,
"ExpirationDate" TEXT,
"Color" TEXT,
"ImpliedVolatility" TEXT,
"Dte" TEXT,
"ER" TEXT,
"StockEtf" TEXT,
"Sector" TEXT,
"Uoa" TEXT,
"Weekly" TEXT,
"MktCap" TEXT,
"OI" TEXT,
"timestamp" TIMESTAMP WITH TIME ZONE,
"aggressor" TEXT
);
-- AlertStream_monthly (CamelCase) - Alert stream data
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema='public' AND table_name='AlertStream_monthly') THEN
CREATE TABLE public."AlertStream_monthly" (
"Date" TEXT,
"AlertDate" TEXT,
"date" TEXT,
"alert_date" TEXT,
"Ticker" TEXT,
"ticker" TEXT,
"Symbol" TEXT,
"symbol" TEXT,
"Type" TEXT,
"type" TEXT,
"AlertType" TEXT,
"alert_type" TEXT,
"Message" TEXT,
"message" TEXT,
"Timestamp" TEXT,
"timestamp" TEXT,
"alert_time_txt" TEXT,
"Notional" TEXT,
"notional" TEXT,
"Pct_of_Avg30Day" TEXT,
"pct_of_avg30day" TEXT,
"Pct_of_Avg30" TEXT,
"pct_of_avg30" TEXT,
"Price" NUMERIC(12, 4),
"price" NUMERIC(12, 4),
"Volume" NUMERIC(15, 2),
"volume" NUMERIC(15, 2)
);
END IF;
END $$;
-- AlertStream_monthly (lowercase) - Alternative alert stream table
CREATE TABLE IF NOT EXISTS public.alertstream_monthly (
symbol VARCHAR(10),
alert_type VARCHAR(50),
message TEXT,
notional NUMERIC(15, 2),
pct_of_avg30 NUMERIC(10, 2),
alert_date DATE,
alert_time_txt TEXT
);
-- prices_intraday_1m - Intraday 1-minute price bars
CREATE TABLE IF NOT EXISTS public.prices_intraday_1m (
symbol VARCHAR(10) NOT NULL,
ts TIMESTAMP WITH TIME ZONE NOT NULL,
open NUMERIC(12, 4),
high NUMERIC(12, 4),
low NUMERIC(12, 4),
close NUMERIC(12, 4),
volume NUMERIC(15, 2),
short_volume_ratio NUMERIC(5, 4),
"timestamp" TIMESTAMP WITH TIME ZONE,
PRIMARY KEY (symbol, ts)
);
-- prices_daily - Daily price bars
CREATE TABLE IF NOT EXISTS public.prices_daily (
symbol VARCHAR(10) NOT NULL,
"Date" DATE NOT NULL,
open NUMERIC(12, 4),
high NUMERIC(12, 4),
low NUMERIC(12, 4),
close NUMERIC(12, 4),
volume NUMERIC(15, 2),
PRIMARY KEY (symbol, "Date")
);
-- price_daily - Alternative daily price table (used by some queries)
CREATE TABLE IF NOT EXISTS public.price_daily (
"Ticker" VARCHAR(10) NOT NULL,
dt DATE NOT NULL,
open NUMERIC(12, 4),
high NUMERIC(12, 4),
low NUMERIC(12, 4),
close NUMERIC(12, 4),
volume NUMERIC(15, 2),
PRIMARY KEY ("Ticker", dt)
);
-- price_1m - Alternative 1-minute price table (used by some queries)
CREATE TABLE IF NOT EXISTS public.price_1m (
"Ticker" VARCHAR(10) NOT NULL,
ts TIMESTAMP WITH TIME ZONE NOT NULL,
open NUMERIC(12, 4),
high NUMERIC(12, 4),
low NUMERIC(12, 4),
close NUMERIC(12, 4),
volume NUMERIC(15, 2),
PRIMARY KEY ("Ticker", ts)
);
-- minute_bars_et - Minute bars in Eastern Time
CREATE TABLE IF NOT EXISTS public.minute_bars_et (
symbol VARCHAR(10) NOT NULL,
d_et DATE NOT NULL,
ts TIMESTAMP WITH TIME ZONE NOT NULL,
session VARCHAR(10) NOT NULL CHECK (session IN ('pre', 'rth', 'post')),
hh INTEGER,
mm INTEGER,
open NUMERIC(12, 4),
high NUMERIC(12, 4),
low NUMERIC(12, 4),
close NUMERIC(12, 4),
volume NUMERIC(15, 2),
PRIMARY KEY (symbol, ts)
);
-- daily_bars - Daily bars table
CREATE TABLE IF NOT EXISTS public.daily_bars (
symbol VARCHAR(10) NOT NULL,
date DATE NOT NULL,
open NUMERIC(12, 4),
high NUMERIC(12, 4),
low NUMERIC(12, 4),
close NUMERIC(12, 4),
volume NUMERIC(15, 2),
PRIMARY KEY (symbol, date)
);
-- bucket_members - Symbol bucket groupings
CREATE TABLE IF NOT EXISTS public.bucket_members (
symbol VARCHAR(10) NOT NULL,
bucket VARCHAR(50) NOT NULL,
PRIMARY KEY (symbol, bucket)
);
-- ============================================
-- 3. INDEXES FOR CORE TABLES
-- ============================================
-- Indexes for options_flow
CREATE INDEX IF NOT EXISTS idx_options_flow_symbol ON options_flow(symbol);
CREATE INDEX IF NOT EXISTS idx_options_flow_created_at ON options_flow(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_options_flow_type ON options_flow(type);
CREATE INDEX IF NOT EXISTS idx_options_flow_rocket_score ON options_flow(rocket_score DESC);
-- Indexes for daily_analysis
CREATE INDEX IF NOT EXISTS idx_daily_analysis_date ON daily_analysis(analysis_date DESC);
CREATE INDEX IF NOT EXISTS idx_daily_analysis_symbol ON daily_analysis(symbol);
CREATE INDEX IF NOT EXISTS idx_daily_analysis_score ON daily_analysis(final_score DESC);
-- ============================================
-- 4. PERFORMANCE INDEXES FOR DATA SOURCE TABLES
-- ============================================
-- OptionsFlow_monthly Table Indexes
CREATE INDEX IF NOT EXISTS idx_ofm_date_symbol_premium
ON "OptionsFlow_monthly"("CreatedDate", "Symbol", "Premium" DESC)
WHERE "Premium" IS NOT NULL
AND btrim("Premium"::text) <> ''
AND "StockEtf" = 'STOCK';
CREATE INDEX IF NOT EXISTS idx_ofm_exp_symbol_strike
ON "OptionsFlow_monthly"("ExpirationDate", "Symbol", "Strike");
CREATE INDEX IF NOT EXISTS idx_ofm_high_premium
ON "OptionsFlow_monthly"("CreatedDate" DESC, "Symbol", "Premium" DESC)
WHERE "Premium"::numeric > 80000
AND "StockEtf" = 'STOCK';
CREATE INDEX IF NOT EXISTS idx_ofm_cp_side
ON "OptionsFlow_monthly"(
UPPER(TRIM("CallPut")),
UPPER(TRIM("Side")),
"CreatedDate" DESC
)
WHERE "Premium" IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_ofm_symbol_filter
ON "OptionsFlow_monthly"("Symbol", "CreatedDate" DESC)
WHERE "Symbol" NOT IN ('TSLA','NVDA')
AND "StockEtf" = 'STOCK';
CREATE INDEX IF NOT EXISTS idx_optionsflow_monthly_symbol ON public."OptionsFlow_monthly"("Symbol");
CREATE INDEX IF NOT EXISTS idx_optionsflow_monthly_timestamp ON public."OptionsFlow_monthly"("timestamp" DESC);
CREATE INDEX IF NOT EXISTS idx_optionsflow_monthly_expiration ON public."OptionsFlow_monthly"("ExpirationDate");
-- prices_intraday_1m Table Indexes
CREATE INDEX IF NOT EXISTS idx_prices_symbol_ts_desc
ON prices_intraday_1m(UPPER(symbol), ts DESC)
WHERE symbol IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_prices_recent
ON prices_intraday_1m(UPPER(symbol), ts DESC)
WHERE symbol IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_prices_session
ON prices_intraday_1m(
UPPER(symbol),
ts,
EXTRACT(HOUR FROM ts AT TIME ZONE 'America/Chicago')
)
WHERE EXTRACT(HOUR FROM ts AT TIME ZONE 'America/Chicago') BETWEEN 4 AND 20;
CREATE INDEX IF NOT EXISTS idx_prices_rth_open
ON prices_intraday_1m(
UPPER(symbol),
ts
)
WHERE symbol IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_prices_intraday_1m_symbol_ts ON public.prices_intraday_1m(symbol, ts DESC);
CREATE INDEX IF NOT EXISTS idx_prices_intraday_1m_ts ON public.prices_intraday_1m(ts DESC);
CREATE INDEX IF NOT EXISTS idx_prices_intraday_1m_timestamp ON public.prices_intraday_1m("timestamp" DESC);
-- prices_daily Table Indexes
CREATE INDEX IF NOT EXISTS idx_prices_daily_symbol_date
ON prices_daily(UPPER(symbol), "Date" DESC)
WHERE symbol IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_prices_daily_symbol_date_alt ON public.prices_daily(symbol, "Date" DESC);
CREATE INDEX IF NOT EXISTS idx_prices_daily_date ON public.prices_daily("Date" DESC);
-- AlertStream_monthly Table Indexes
CREATE INDEX IF NOT EXISTS idx_alert_ticker_time
ON "AlertStream_monthly"(
UPPER("ticker"),
"date",
"timestamp"
)
WHERE "ticker" IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_alert_recent
ON "AlertStream_monthly"(
UPPER("ticker"),
"date",
"timestamp"
)
WHERE "ticker" IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_alert_type
ON "AlertStream_monthly"("type", "date", UPPER("ticker"))
WHERE "ticker" IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_alert_dark_block
ON "AlertStream_monthly"(
UPPER("ticker"),
"date",
"timestamp"
)
WHERE "type" IN ('DarkPool','Block')
AND "ticker" IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_ticker ON public."AlertStream_monthly"("Ticker");
CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_symbol ON public."AlertStream_monthly"("symbol");
CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_type ON public."AlertStream_monthly"("Type");
-- alertstream_monthly (lowercase) Indexes
CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_lower_symbol ON public.alertstream_monthly(symbol);
CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_lower_date ON public.alertstream_monthly(alert_date);
CREATE INDEX IF NOT EXISTS idx_alertstream_monthly_lower_type ON public.alertstream_monthly(alert_type);
-- price_daily Table Indexes
CREATE INDEX IF NOT EXISTS idx_price_daily_ticker_dt ON public.price_daily("Ticker", dt DESC);
CREATE INDEX IF NOT EXISTS idx_price_daily_dt ON public.price_daily(dt DESC);
-- price_1m Table Indexes
CREATE INDEX IF NOT EXISTS idx_price_1m_ticker_ts ON public.price_1m("Ticker", ts DESC);
CREATE INDEX IF NOT EXISTS idx_price_1m_ts ON public.price_1m(ts DESC);
-- minute_bars_et Table Indexes
CREATE INDEX IF NOT EXISTS idx_minute_bars_et_symbol_date ON public.minute_bars_et(symbol, d_et);
CREATE INDEX IF NOT EXISTS idx_minute_bars_et_session ON public.minute_bars_et(session, d_et);
CREATE INDEX IF NOT EXISTS idx_minute_bars_et_ts ON public.minute_bars_et(ts DESC);
-- daily_bars Table Indexes
CREATE INDEX IF NOT EXISTS idx_daily_bars_symbol_date ON public.daily_bars(symbol, date DESC);
CREATE INDEX IF NOT EXISTS idx_daily_bars_date ON public.daily_bars(date DESC);
-- bucket_members Table Indexes
CREATE INDEX IF NOT EXISTS idx_bucket_members_bucket ON public.bucket_members(bucket);
CREATE INDEX IF NOT EXISTS idx_bucket_members_symbol ON public.bucket_members(symbol);
-- ============================================
-- 5. ROW LEVEL SECURITY (RLS)
-- ============================================
-- Enable RLS on all tables
ALTER TABLE options_flow ENABLE ROW LEVEL SECURITY;
ALTER TABLE daily_analysis ENABLE ROW LEVEL SECURITY;
ALTER TABLE public.minute_bars_et ENABLE ROW LEVEL SECURITY;
ALTER TABLE public.bucket_members ENABLE ROW LEVEL SECURITY;
ALTER TABLE public.daily_bars ENABLE ROW LEVEL SECURITY;
ALTER TABLE public."AlertStream_monthly" ENABLE ROW LEVEL SECURITY;
ALTER TABLE public.alertstream_monthly ENABLE ROW LEVEL SECURITY;
ALTER TABLE public."OptionsFlow_monthly" ENABLE ROW LEVEL SECURITY;
ALTER TABLE public.prices_intraday_1m ENABLE ROW LEVEL SECURITY;
ALTER TABLE public.prices_daily ENABLE ROW LEVEL SECURITY;
ALTER TABLE public.price_daily ENABLE ROW LEVEL SECURITY;
ALTER TABLE public.price_1m ENABLE ROW LEVEL SECURITY;
-- Create permissive policies for development (adjust for production)
-- Note: If policies already exist, you may need to drop them first manually
-- or modify this section to handle existing policies
DO $$
BEGIN
-- options_flow
BEGIN
DROP POLICY "Allow all operations on options_flow" ON options_flow;
EXCEPTION WHEN undefined_object THEN NULL;
END;
CREATE POLICY "Allow all operations on options_flow" ON options_flow
FOR ALL USING (true) WITH CHECK (true);
END $$;
DO $$
BEGIN
-- daily_analysis
BEGIN
DROP POLICY "Allow all operations on daily_analysis" ON daily_analysis;
EXCEPTION WHEN undefined_object THEN NULL;
END;
CREATE POLICY "Allow all operations on daily_analysis" ON daily_analysis
FOR ALL USING (true) WITH CHECK (true);
END $$;
DO $$
BEGIN
-- minute_bars_et
BEGIN
DROP POLICY "Allow all operations on minute_bars_et" ON public.minute_bars_et;
EXCEPTION WHEN undefined_object THEN NULL;
END;
CREATE POLICY "Allow all operations on minute_bars_et" ON public.minute_bars_et
FOR ALL USING (true) WITH CHECK (true);
END $$;
DO $$
BEGIN
-- bucket_members
BEGIN
DROP POLICY "Allow all operations on bucket_members" ON public.bucket_members;
EXCEPTION WHEN undefined_object THEN NULL;
END;
CREATE POLICY "Allow all operations on bucket_members" ON public.bucket_members
FOR ALL USING (true) WITH CHECK (true);
END $$;
DO $$
BEGIN
-- daily_bars
BEGIN
DROP POLICY "Allow all operations on daily_bars" ON public.daily_bars;
EXCEPTION WHEN undefined_object THEN NULL;
END;
CREATE POLICY "Allow all operations on daily_bars" ON public.daily_bars
FOR ALL USING (true) WITH CHECK (true);
END $$;
DO $$
BEGIN
-- AlertStream_monthly
BEGIN
DROP POLICY "Allow all operations on AlertStream_monthly" ON public."AlertStream_monthly";
EXCEPTION WHEN undefined_object THEN NULL;
END;
CREATE POLICY "Allow all operations on AlertStream_monthly" ON public."AlertStream_monthly"
FOR ALL USING (true) WITH CHECK (true);
END $$;
DO $$
BEGIN
-- alertstream_monthly
BEGIN
DROP POLICY "Allow all operations on alertstream_monthly" ON public.alertstream_monthly;
EXCEPTION WHEN undefined_object THEN NULL;
END;
CREATE POLICY "Allow all operations on alertstream_monthly" ON public.alertstream_monthly
FOR ALL USING (true) WITH CHECK (true);
END $$;
DO $$
BEGIN
-- OptionsFlow_monthly
BEGIN
DROP POLICY "Allow all operations on OptionsFlow_monthly" ON public."OptionsFlow_monthly";
EXCEPTION WHEN undefined_object THEN NULL;
END;
CREATE POLICY "Allow all operations on OptionsFlow_monthly" ON public."OptionsFlow_monthly"
FOR ALL USING (true) WITH CHECK (true);
END $$;
DO $$
BEGIN
-- prices_intraday_1m
BEGIN
DROP POLICY "Allow all operations on prices_intraday_1m" ON public.prices_intraday_1m;
EXCEPTION WHEN undefined_object THEN NULL;
END;
CREATE POLICY "Allow all operations on prices_intraday_1m" ON public.prices_intraday_1m
FOR ALL USING (true) WITH CHECK (true);
END $$;
DO $$
BEGIN
-- prices_daily
BEGIN
DROP POLICY "Allow all operations on prices_daily" ON public.prices_daily;
EXCEPTION WHEN undefined_object THEN NULL;
END;
CREATE POLICY "Allow all operations on prices_daily" ON public.prices_daily
FOR ALL USING (true) WITH CHECK (true);
END $$;
DO $$
BEGIN
-- price_daily
BEGIN
DROP POLICY "Allow all operations on price_daily" ON public.price_daily;
EXCEPTION WHEN undefined_object THEN NULL;
END;
CREATE POLICY "Allow all operations on price_daily" ON public.price_daily
FOR ALL USING (true) WITH CHECK (true);
END $$;
DO $$
BEGIN
-- price_1m
BEGIN
DROP POLICY "Allow all operations on price_1m" ON public.price_1m;
EXCEPTION WHEN undefined_object THEN NULL;
END;
CREATE POLICY "Allow all operations on price_1m" ON public.price_1m
FOR ALL USING (true) WITH CHECK (true);
END $$;
-- ============================================
-- 6. ANALYZE TABLES FOR QUERY PLANNER
-- ============================================
-- Update statistics for query planner
ANALYZE options_flow;
ANALYZE daily_analysis;
ANALYZE "OptionsFlow_monthly";
ANALYZE prices_intraday_1m;
ANALYZE prices_daily;
ANALYZE price_daily;
ANALYZE price_1m;
ANALYZE "AlertStream_monthly";
ANALYZE alertstream_monthly;
ANALYZE minute_bars_et;
ANALYZE daily_bars;
ANALYZE bucket_members;
-- ============================================
-- SCHEMA COMPLETE
-- ============================================
-- All tables and indexes have been created
-- The database is now ready for use
-- ============================================