institutional-trader/backend/scripts/validateSchema.js

/**
 * Database Schema Validation Script
 * Validates that all required tables, indexes, and constraints exist
 */

import { rawQuery } from '../src/db.js';

const REQUIRED_TABLES = [
  'options_flow',
  'daily_analysis',
  'OptionsFlow_monthly',
  'AlertStream_monthly',
  'alertstream_monthly',
  'prices_intraday_1m',
  'prices_daily',
  'price_daily',
  'price_1m',
  'minute_bars_et',
  'daily_bars',
  'bucket_members'
];

async function validateSchema() {
  console.log('🔍 Starting database schema validation...\n');

  const issues = [];
  const warnings = [];

  try {
    // 1. Check all required tables exist
    console.log('📋 Checking required tables...');
    const tablesQuery = `
      SELECT table_name 
      FROM information_schema.tables 
      WHERE table_schema = 'public' 
        AND table_name = ANY($1::text[])
      ORDER BY table_name;
    `;
    const existingTables = await rawQuery(tablesQuery, [REQUIRED_TABLES]);
    const existingTableNames = existingTables.map(t => t.table_name);

    for (const table of REQUIRED_TABLES) {
      if (existingTableNames.includes(table)) {
        console.log(`  ✅ ${table}`);
      } else {
        console.log(`  ❌ ${table} - MISSING`);
        issues.push(`Missing table: ${table}`);
      }
    }

    // 2. Check indexes on critical tables
    console.log('\n📊 Checking indexes on critical tables...');
    const indexesQuery = `
      SELECT tablename, indexname, indexdef
      FROM pg_indexes
      WHERE schemaname = 'public'
        AND tablename IN ('prices_intraday_1m', 'prices_daily', 'OptionsFlow_monthly')
      ORDER BY tablename, indexname;
    `;
    const indexes = await rawQuery(indexesQuery);
    
    const criticalIndexes = {
      'prices_intraday_1m': ['idx_prices_intraday_1m_symbol_ts', 'idx_prices_intraday_1m_ts'],
      'prices_daily': ['idx_prices_daily_symbol_date', 'idx_prices_daily_date'],
      'OptionsFlow_monthly': ['idx_optionsflow_monthly_symbol', 'idx_optionsflow_monthly_timestamp']
    };

    for (const [table, requiredIndexes] of Object.entries(criticalIndexes)) {
      const tableIndexes = indexes.filter(idx => idx.tablename === table);
      const existingIndexNames = tableIndexes.map(idx => idx.indexname);
      
      for (const indexName of requiredIndexes) {
        if (existingIndexNames.some(name => name.includes(indexName.split('_').slice(-2).join('_')))) {
          console.log(`  ✅ ${table}.${indexName}`);
        } else {
          console.log(`  ⚠️  ${table}.${indexName} - Missing or different name`);
          warnings.push(`Index ${indexName} on ${table} may be missing`);
        }
      }
    }

    // 3. Verify PRIMARY KEY constraints
    console.log('\n🔑 Checking PRIMARY KEY constraints...');
    const pkQuery = `
      SELECT tc.table_name, kc.column_name, tc.constraint_name
      FROM information_schema.table_constraints tc
      JOIN information_schema.key_column_usage kc
        ON tc.constraint_name = kc.constraint_name
      WHERE tc.constraint_type = 'PRIMARY KEY'
        AND tc.table_schema = 'public'
        AND tc.table_name IN ('prices_intraday_1m', 'prices_daily')
      ORDER BY tc.table_name;
    `;
    const primaryKeys = await rawQuery(pkQuery);

    const expectedPKs = {
      'prices_intraday_1m': ['symbol', 'ts'],
      'prices_daily': ['symbol', 'Date']
    };

    for (const [table, expectedCols] of Object.entries(expectedPKs)) {
      const tablePKs = primaryKeys.filter(pk => pk.table_name === table);
      const pkColumns = tablePKs.map(pk => pk.column_name);
      
      const hasAllColumns = expectedCols.every(col => 
        pkColumns.some(pkCol => pkCol.toLowerCase() === col.toLowerCase())
      );
      
      if (hasAllColumns && tablePKs.length > 0) {
        console.log(`  ✅ ${table} - PRIMARY KEY on (${expectedCols.join(', ')})`);
      } else {
        console.log(`  ❌ ${table} - Missing PRIMARY KEY on (${expectedCols.join(', ')})`);
        issues.push(`Missing PRIMARY KEY on ${table}`);
      }
    }

    // 4. Check for data type inconsistencies
    console.log('\n🔤 Checking data type consistency...');
    const dataTypeQuery = `
      SELECT 
        table_name,
        column_name,
        data_type,
        character_maximum_length
      FROM information_schema.columns
      WHERE table_schema = 'public'
        AND table_name IN ('prices_intraday_1m', 'prices_daily')
        AND column_name IN ('symbol', 'ts', 'Date', 'timestamp')
      ORDER BY table_name, column_name;
    `;
    const columns = await rawQuery(dataTypeQuery);

    // Check for redundant timestamp columns
    const intradayColumns = columns.filter(c => c.table_name === 'prices_intraday_1m');
    const hasTs = intradayColumns.some(c => c.column_name === 'ts');
    const hasTimestamp = intradayColumns.some(c => c.column_name === 'timestamp');
    
    if (hasTs && hasTimestamp) {
      console.log(`  ⚠️  prices_intraday_1m has both 'ts' and 'timestamp' columns (may be redundant)`);
      warnings.push('prices_intraday_1m has redundant timestamp columns');
    } else if (hasTs) {
      console.log(`  ✅ prices_intraday_1m uses 'ts' column`);
    } else if (hasTimestamp) {
      console.log(`  ⚠️  prices_intraday_1m only has 'timestamp' column (expected 'ts')`);
      warnings.push('prices_intraday_1m missing expected ts column');
    }

    // Check Date column capitalization
    const dailyColumns = columns.filter(c => c.table_name === 'prices_daily');
    const hasDate = dailyColumns.some(c => c.column_name === 'Date');
    if (hasDate) {
      console.log(`  ✅ prices_daily uses 'Date' column (capital D)`);
    } else {
      console.log(`  ⚠️  prices_daily may not have 'Date' column with capital D`);
      warnings.push('prices_daily Date column capitalization check');
    }

    // 5. Summary
    console.log('\n' + '='.repeat(60));
    console.log('📊 VALIDATION SUMMARY');
    console.log('='.repeat(60));
    
    if (issues.length === 0 && warnings.length === 0) {
      console.log('✅ Schema validation passed! All checks successful.');
      return true;
    }

    if (warnings.length > 0) {
      console.log(`\n⚠️  Warnings (${warnings.length}):`);
      warnings.forEach(w => console.log(`   - ${w}`));
    }

    if (issues.length > 0) {
      console.log(`\n❌ Issues found (${issues.length}):`);
      issues.forEach(i => console.log(`   - ${i}`));
      console.log('\n💡 Run the schema.sql and missing_tables.sql scripts to fix issues.');
      return false;
    }

    console.log('\n✅ Schema validation passed with warnings.');
    return true;

  } catch (error) {
    console.error('\n❌ Schema validation failed with error:');
    console.error(error);
    return false;
  }
}

// Run validation
validateSchema()
  .then(success => {
    process.exit(success ? 0 : 1);
  })
  .catch(error => {
    console.error('Fatal error:', error);
    process.exit(1);
  });