/**
 * CSV Import Script
 * Imports OptionsFlow, OptionsVolume, and OptionsOpenInterest CSV files into database
 * 
 * Usage: node scripts/importCSV.js
 */

import fs from 'fs';
import { parse } from 'csv-parse/sync';
import { rawQuery } from '../src/db.js';
import path from 'path';
import { fileURLToPath } from 'url';
import dotenv from 'dotenv';

// Load environment variables
dotenv.config();

// Ensure we're using local DB for imports
if (process.env.USE_LOCAL_DB !== 'true' && !process.env.DATABASE_URL) {
  console.warn('⚠️  Warning: Not using local database. CSV import works best with local PostgreSQL.');
  console.warn('   Set USE_LOCAL_DB=true in .env for local development.');
}

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

/**
 * Parse CSV file
 */
function parseCSV(filePath) {
  try {
    const fileContent = fs.readFileSync(filePath, 'utf-8');
    const records = parse(fileContent, {
      columns: true,
      skip_empty_lines: true,
      trim: true,
      cast: true
    });
    return records;
  } catch (error) {
    console.error(`Error reading ${filePath}:`, error.message);
    throw error;
  }
}

/**
 * Normalize date string to PostgreSQL date format
 */
function normalizeDate(dateStr) {
  if (!dateStr) return null;
  
  // Handle MM/DD/YYYY format
  if (dateStr.match(/^\d{1,2}\/\d{1,2}\/\d{4}$/)) {
    const [month, day, year] = dateStr.split('/');
    return `${year}-${month.padStart(2, '0')}-${day.padStart(2, '0')}`;
  }
  
  // Handle YYYY-MM-DD format
  if (dateStr.match(/^\d{4}-\d{2}-\d{2}$/)) {
    return dateStr;
  }
  
  return null;
}

/**
 * Normalize time string
 * Fixes common issues like "34 AM" (missing hour) or ":34 AM" (missing hour with colon)
 */
function normalizeTime(timeStr) {
  if (!timeStr) return null;
  
  let time = timeStr.trim();
  
  // Fix pattern: ":34 AM" or ":34 PM" (missing hour, has colon)
  // Convert to "9:34 AM" or "1:34 PM"
  if (time.match(/^:\d{1,2}\s*(AM|PM)$/i)) {
    const match = time.match(/^:(\d{1,2})\s*(AM|PM)$/i);
    if (match) {
      const minute = match[1].padStart(2, '0');
      const ampm = match[2].toUpperCase();
      const hour = ampm === 'AM' ? '9' : '1';  // Default hour
      time = `${hour}:${minute}:00 ${ampm}`;
    }
  }
  
  // Fix pattern: "34 AM" or "34 PM" (missing hour, no colon)
  // If number > 12, it's definitely minutes, add default hour
  // If number <= 12, it could be hour or minute - default to minute with hour 9/1
  if (time.match(/^\d{1,2}\s*(AM|PM)$/i) && !time.includes(':')) {
    const match = time.match(/^(\d{1,2})\s*(AM|PM)$/i);
    if (match) {
      const num = parseInt(match[1]);
      const ampm = match[2].toUpperCase();
      
      if (num > 12) {
        // Definitely minutes (can't be hour > 12)
        const minute = match[1].padStart(2, '0');
        const hour = ampm === 'AM' ? '9' : '1';
        time = `${hour}:${minute}:00 ${ampm}`;
      } else {
        // Could be hour or minute - assume it's minute with default hour
        const minute = match[1].padStart(2, '0');
        const hour = ampm === 'AM' ? '9' : '1';
        time = `${hour}:${minute}:00 ${ampm}`;
      }
    }
  }
  
  return time;
}

/**
 * Clean numeric value
 */
function cleanNumeric(value) {
  if (!value || value === '' || value === null || value === undefined) return null;
  if (typeof value === 'number') return value;
  
  // Remove commas, dollar signs, spaces
  const cleaned = String(value).replace(/[\s$,]/g, '');
  const num = parseFloat(cleaned);
  return isNaN(num) ? null : num;
}

/**
 * Import OptionsFlow CSV
 */
async function importOptionsFlow(csvPath) {
  console.log('📥 Importing OptionsFlow data...');
  
  const records = parseCSV(csvPath);
  console.log(`   Found ${records.length} records`);
  
  // Ensure table exists
  const createTableSQL = `
    CREATE TABLE IF NOT EXISTS "OptionsFlow_monthly" (
      "CreatedDate" TEXT,
      "CreatedTime" TEXT,
      "Symbol" TEXT,
      "Type" TEXT,
      "Volume" TEXT,
      "Price" TEXT,
      "Side" TEXT,
      "CallPut" TEXT,
      "Strike" TEXT,
      "Spot" TEXT,
      "Premium" TEXT,
      "ExpirationDate" TEXT,
      "Color" TEXT,
      "ImpliedVolatility" TEXT,
      "Dte" TEXT,
      "ER" TEXT,
      "StockEtf" TEXT,
      "Sector" TEXT,
      "Uoa" TEXT,
      "Weekly" TEXT,
      "MktCap" TEXT,
      "OI" TEXT
    );
  `;
  
  await rawQuery(createTableSQL);
  console.log('   ✅ Table created/verified');
  
  // Batch insert (100 records at a time to avoid parameter limits)
  const batchSize = 100;
  let imported = 0;
  
  for (let i = 0; i < records.length; i += batchSize) {
    const batch = records.slice(i, i + batchSize);
    
    // Build parameterized query
    const placeholders = batch.map((_, idx) => {
      const start = idx * 22 + 1;
      return `($${start}, $${start + 1}, $${start + 2}, $${start + 3}, $${start + 4}, $${start + 5}, $${start + 6}, $${start + 7}, $${start + 8}, $${start + 9}, $${start + 10}, $${start + 11}, $${start + 12}, $${start + 13}, $${start + 14}, $${start + 15}, $${start + 16}, $${start + 17}, $${start + 18}, $${start + 19}, $${start + 20}, $${start + 21})`;
    }).join(', ');
    
    const params = batch.flatMap(row => [
      row.CreatedDate || null,
      row.CreatedTime || null,
      row.Symbol || null,
      row.Type || null,
      row.Volume || null,
      row.Price || null,
      row.Side || null,
      row.CallPut || null,
      row.Strike || null,
      row.Spot || null,
      row.Premium || null,
      row.ExpirationDate || null,
      row.Color || null,
      row.ImpliedVolatility || null,
      row.Dte || null,
      row.ER || null,
      row.StockEtf || null,
      row.Sector || null,
      row.Uoa || null,
      row.Weekly || null,
      row.MktCap || null,
      row.OI || null
    ]);
    
    const insertSQL = `
      INSERT INTO "OptionsFlow_monthly" (
        "CreatedDate", "CreatedTime", "Symbol", "Type", "Volume", "Price",
        "Side", "CallPut", "Strike", "Spot", "Premium", "ExpirationDate",
        "Color", "ImpliedVolatility", "Dte", "ER", "StockEtf", "Sector",
        "Uoa", "Weekly", "MktCap", "OI"
      ) VALUES ${placeholders}
    `;
    
    try {
      await rawQuery(insertSQL, params);
      imported += batch.length;
      console.log(`   ✅ Imported ${imported}/${records.length} records...`);
    } catch (error) {
      console.error(`   ❌ Error importing batch ${Math.floor(i / batchSize) + 1}:`, error.message);
      throw error;
    }
  }
  
  console.log(`   ✅ Successfully imported ${imported} OptionsFlow records`);
  return imported;
}

/**
 * Import OptionsVolume CSV
 */
async function importOptionsVolume(csvPath) {
  console.log('📥 Importing OptionsVolume data...');
  
  const records = parseCSV(csvPath);
  console.log(`   Found ${records.length} records`);
  
  // Create table if it doesn't exist
  const createTableSQL = `
    CREATE TABLE IF NOT EXISTS "OptionsVolume" (
      "SYMBOL" TEXT,
      "VOL" NUMERIC,
      "C VOL" NUMERIC,
      "P VOL" NUMERIC,
      "P/C" NUMERIC,
      "AVG VOL" NUMERIC,
      "RATIO" NUMERIC,
      "COI" NUMERIC,
      "POI" NUMERIC,
      "DCOI" NUMERIC,
      "DPOI" NUMERIC
    );
  `;
  
  await rawQuery(createTableSQL);
  console.log('   ✅ Table created/verified');
  
  // Batch insert
  const batchSize = 100;
  let imported = 0;
  
  for (let i = 0; i < records.length; i += batchSize) {
    const batch = records.slice(i, i + batchSize);
    
    const placeholders = batch.map((_, idx) => {
      const start = idx * 11 + 1;
      return `($${start}, $${start + 1}, $${start + 2}, $${start + 3}, $${start + 4}, $${start + 5}, $${start + 6}, $${start + 7}, $${start + 8}, $${start + 9}, $${start + 10})`;
    }).join(', ');
    
    const params = batch.flatMap(row => [
      row.SYMBOL || null,
      cleanNumeric(row.VOL),
      cleanNumeric(row['C VOL']),
      cleanNumeric(row['P VOL']),
      cleanNumeric(row['P/C']),
      cleanNumeric(row['AVG VOL']),
      cleanNumeric(row.RATIO),
      cleanNumeric(row.COI),
      cleanNumeric(row.POI),
      cleanNumeric(row.DCOI),
      cleanNumeric(row.DPOI)
    ]);
    
    const insertSQL = `
      INSERT INTO "OptionsVolume" (
        "SYMBOL", "VOL", "C VOL", "P VOL", "P/C", "AVG VOL",
        "RATIO", "COI", "POI", "DCOI", "DPOI"
      ) VALUES ${placeholders}
    `;
    
    try {
      await rawQuery(insertSQL, params);
      imported += batch.length;
      console.log(`   ✅ Imported ${imported}/${records.length} records...`);
    } catch (error) {
      console.error(`   ❌ Error importing batch:`, error.message);
      throw error;
    }
  }
  
  console.log(`   ✅ Successfully imported ${imported} OptionsVolume records`);
  return imported;
}

/**
 * Import OptionsOpenInterest CSV
 */
async function importOptionsOpenInterest(csvPath) {
  console.log('📥 Importing OptionsOpenInterest data...');
  
  const records = parseCSV(csvPath);
  console.log(`   Found ${records.length} records`);
  
  // Create table if it doesn't exist
  const createTableSQL = `
    CREATE TABLE IF NOT EXISTS "OptionsOpenInterest" (
      "SYMBOL" TEXT,
      "EXP" TEXT,
      "C/P" TEXT,
      "STRIKE" NUMERIC,
      "OI" NUMERIC,
      "DOI" NUMERIC
    );
  `;
  
  await rawQuery(createTableSQL);
  console.log('   ✅ Table created/verified');
  
  // Batch insert
  const batchSize = 100;
  let imported = 0;
  
  for (let i = 0; i < records.length; i += batchSize) {
    const batch = records.slice(i, i + batchSize);
    
    const placeholders = batch.map((_, idx) => {
      const start = idx * 6 + 1;
      return `($${start}, $${start + 1}, $${start + 2}, $${start + 3}, $${start + 4}, $${start + 5})`;
    }).join(', ');
    
    const params = batch.flatMap(row => [
      row.SYMBOL || null,
      row.EXP || null,
      row['C/P'] || null,
      cleanNumeric(row.STRIKE),
      cleanNumeric(row.OI),
      cleanNumeric(row.DOI)
    ]);
    
    const insertSQL = `
      INSERT INTO "OptionsOpenInterest" (
        "SYMBOL", "EXP", "C/P", "STRIKE", "OI", "DOI"
      ) VALUES ${placeholders}
    `;
    
    try {
      await rawQuery(insertSQL, params);
      imported += batch.length;
      console.log(`   ✅ Imported ${imported}/${records.length} records...`);
    } catch (error) {
      console.error(`   ❌ Error importing batch:`, error.message);
      throw error;
    }
  }
  
  console.log(`   ✅ Successfully imported ${imported} OptionsOpenInterest records`);
  return imported;
}

/**
 * Import AlertStream CSV
 */
async function importAlertStream(csvPath) {
  console.log('📥 Importing AlertStream data...');
  
  const records = parseCSV(csvPath);
  console.log(`   Found ${records.length} records`);
  
  // Ensure table exists (columns match what queries expect)
  const createTableSQL = `
    DO $$
    BEGIN
      IF NOT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema='public' AND table_name='AlertStream_monthly') THEN
        CREATE TABLE public."AlertStream_monthly" (
          "Date" TEXT,
          "AlertDate" TEXT,
          "date" TEXT,
          "alert_date" TEXT,
          "Ticker" TEXT,
          "ticker" TEXT,
          "Symbol" TEXT,
          "symbol" TEXT,
          "Type" TEXT,
          "type" TEXT,
          "AlertType" TEXT,
          "alert_type" TEXT,
          "Message" TEXT,
          "message" TEXT,
          "Timestamp" TEXT,
          "timestamp" TEXT,
          "alert_time_txt" TEXT,
          "Notional" TEXT,
          "notional" TEXT,
          "Pct_of_Avg30Day" TEXT,
          "pct_of_avg30day" TEXT,
          "Pct_of_Avg30" TEXT,
          "pct_of_avg30" TEXT,
          "Price" NUMERIC(12, 4),
          "price" NUMERIC(12, 4),
          "Volume" NUMERIC(15, 2),
          "volume" NUMERIC(15, 2)
        );
      END IF;
    END $$;
  `;
  
  await rawQuery(createTableSQL);
  
  // Add missing columns if they don't exist
  const addColumnSQL = `
    DO $$
    BEGIN
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Ticker') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Ticker" TEXT;
      END IF;
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='ticker') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "ticker" TEXT;
      END IF;
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Timestamp') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Timestamp" TEXT;
      END IF;
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='timestamp') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "timestamp" TEXT;
      END IF;
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Type') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Type" TEXT;
      END IF;
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='type') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "type" TEXT;
      END IF;
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Message') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Message" TEXT;
      END IF;
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='message') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "message" TEXT;
      END IF;
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Date') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Date" TEXT;
      END IF;
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='date') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "date" TEXT;
      END IF;
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Price') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Price" NUMERIC(12, 4);
      END IF;
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='price') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "price" NUMERIC(12, 4);
      END IF;
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Volume') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Volume" NUMERIC(15, 2);
      END IF;
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='volume') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "volume" NUMERIC(15, 2);
      END IF;
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Notional') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Notional" TEXT;
      END IF;
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='notional') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "notional" TEXT;
      END IF;
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Pct_of_Avg30Day') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Pct_of_Avg30Day" TEXT;
      END IF;
      IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='pct_of_avg30day') THEN
        ALTER TABLE public."AlertStream_monthly" ADD COLUMN "pct_of_avg30day" TEXT;
      END IF;
    END $$;
  `;
  
  await rawQuery(addColumnSQL);
  console.log('   ✅ Table created/verified');
  
  // Batch insert
  const batchSize = 100;
  let imported = 0;
  
  for (let i = 0; i < records.length; i += batchSize) {
    const batch = records.slice(i, i + batchSize);
    
    // Map CSV columns to both CamelCase and lowercase versions for compatibility
    const placeholders = batch.map((_, idx) => {
      const start = idx * 18 + 1;
      return `($${start}, $${start + 1}, $${start + 2}, $${start + 3}, $${start + 4}, $${start + 5}, $${start + 6}, $${start + 7}, $${start + 8}, $${start + 9}, $${start + 10}, $${start + 11}, $${start + 12}, $${start + 13}, $${start + 14}, $${start + 15}, $${start + 16}, $${start + 17})`;
    }).join(', ');
    
    const params = batch.flatMap(row => [
      row.Date || null, row.Date || null,  // Date, date
      row.Timestamp || null, row.Timestamp || null,  // Timestamp, timestamp
      row.Ticker || null, row.Ticker || null,  // Ticker, ticker
      row.Type || null, row.Type || null,  // Type, type
      row.Message || null, row.Message || null,  // Message, message
      cleanNumeric(row.Price), cleanNumeric(row.Price),  // Price, price
      cleanNumeric(row.Volume), cleanNumeric(row.Volume),  // Volume, volume
      cleanNumeric(row.Notional)?.toString() || null, cleanNumeric(row.Notional)?.toString() || null,  // Notional, notional
      row.Pct_of_Avg30Day || null, row.Pct_of_Avg30Day || null  // Pct_of_Avg30Day, pct_of_avg30day
    ]);
    
    const insertSQL = `
      INSERT INTO public."AlertStream_monthly" (
        "Date", "date", "Timestamp", "timestamp", "Ticker", "ticker",
        "Type", "type", "Message", "message", "Price", "price",
        "Volume", "volume", "Notional", "notional", "Pct_of_Avg30Day", "pct_of_avg30day"
      ) VALUES ${placeholders}
    `;
    
    try {
      await rawQuery(insertSQL, params);
      imported += batch.length;
      console.log(`   ✅ Imported ${imported}/${records.length} records...`);
    } catch (error) {
      console.error(`   ❌ Error importing batch:`, error.message);
      throw error;
    }
  }
  
  console.log(`   ✅ Successfully imported ${imported} AlertStream records`);
  return imported;
}

/**
 * Main import function
 */
async function main() {
  console.log('🚀 Starting CSV import process...\n');
  
  try {
    // Find CSV files in data folder
    const projectRoot = path.resolve(__dirname, '../..');
    const dataFolder = path.join(projectRoot, 'data');
    
    // Check if data folder exists
    if (!fs.existsSync(dataFolder)) {
      throw new Error(`Data folder not found at ${dataFolder}`);
    }
    
    // Find all CSV files
    const optionsFlowFiles = [
      path.join(dataFolder, 'OptionsFlow.csv'),
      path.join(dataFolder, 'OptionsFlowMonthly (1).csv'),
      path.join(dataFolder, 'HistoricalOptionsFlow.csv')
    ].filter(f => fs.existsSync(f));
    
    const optionsVolumePath = path.join(dataFolder, 'OptionsVolume.csv');
    const optionsOpenInterestPath = path.join(dataFolder, 'OptionsOpenInterest.csv');
    const alertStreamPath = path.join(dataFolder, 'AlertStream.csv');
    
    console.log('📁 Found CSV files:');
    optionsFlowFiles.forEach(f => console.log(`   - ${f}`));
    if (fs.existsSync(optionsVolumePath)) console.log(`   - ${optionsVolumePath}`);
    if (fs.existsSync(optionsOpenInterestPath)) console.log(`   - ${optionsOpenInterestPath}`);
    if (fs.existsSync(alertStreamPath)) console.log(`   - ${alertStreamPath}`);
    console.log('');
    
    // Import each file
    const results = {
      optionsFlow: 0,
      optionsVolume: 0,
      optionsOpenInterest: 0,
      alertStream: 0
    };
    
    // Import all OptionsFlow files
    for (const filePath of optionsFlowFiles) {
      console.log(`📥 Importing ${path.basename(filePath)}...`);
      const count = await importOptionsFlow(filePath);
      results.optionsFlow += count;
      console.log('');
    }
    
    if (fs.existsSync(optionsVolumePath)) {
      results.optionsVolume = await importOptionsVolume(optionsVolumePath);
      console.log('');
    }
    
    if (fs.existsSync(optionsOpenInterestPath)) {
      results.optionsOpenInterest = await importOptionsOpenInterest(optionsOpenInterestPath);
      console.log('');
    }
    
    if (fs.existsSync(alertStreamPath)) {
      results.alertStream = await importAlertStream(alertStreamPath);
      console.log('');
    }
    
    // Summary
    console.log('✅ Import Summary:');
    console.log(`   OptionsFlow: ${results.optionsFlow} records (from ${optionsFlowFiles.length} file(s))`);
    if (results.optionsVolume > 0) console.log(`   OptionsVolume: ${results.optionsVolume} records`);
    if (results.optionsOpenInterest > 0) console.log(`   OptionsOpenInterest: ${results.optionsOpenInterest} records`);
    if (results.alertStream > 0) console.log(`   AlertStream: ${results.alertStream} records`);
    const total = results.optionsFlow + results.optionsVolume + results.optionsOpenInterest + results.alertStream;
    console.log(`   Total: ${total} records`);
    
  } catch (error) {
    console.error('\n❌ Import failed:', error.message);
    console.error(error);
    process.exit(1);
  }
}

// Run if called directly
const isMainModule = import.meta.url === `file://${process.argv[1]}` || 
                     import.meta.url.endsWith('importCSV.js') ||
                     process.argv[1]?.endsWith('importCSV.js');

if (isMainModule || !process.env.NODE_ENV) {
  main().catch(error => {
    console.error('Fatal error:', error);
    process.exit(1);
  });
}

export { importOptionsFlow, importOptionsVolume, importOptionsOpenInterest, importAlertStream };