623 lines
22 KiB
JavaScript
623 lines
22 KiB
JavaScript
/**
|
|
* CSV Import Script
|
|
* Imports OptionsFlow, OptionsVolume, and OptionsOpenInterest CSV files into database
|
|
*
|
|
* Usage: node scripts/importCSV.js
|
|
*/
|
|
|
|
import fs from 'fs';
|
|
import { parse } from 'csv-parse/sync';
|
|
import { rawQuery } from '../src/db.js';
|
|
import path from 'path';
|
|
import { fileURLToPath } from 'url';
|
|
import dotenv from 'dotenv';
|
|
|
|
// Load environment variables
|
|
dotenv.config();
|
|
|
|
// Ensure we're using local DB for imports
|
|
if (process.env.USE_LOCAL_DB !== 'true' && !process.env.DATABASE_URL) {
|
|
console.warn('⚠️ Warning: Not using local database. CSV import works best with local PostgreSQL.');
|
|
console.warn(' Set USE_LOCAL_DB=true in .env for local development.');
|
|
}
|
|
|
|
const __filename = fileURLToPath(import.meta.url);
|
|
const __dirname = path.dirname(__filename);
|
|
|
|
/**
|
|
* Parse CSV file
|
|
*/
|
|
function parseCSV(filePath) {
|
|
try {
|
|
const fileContent = fs.readFileSync(filePath, 'utf-8');
|
|
const records = parse(fileContent, {
|
|
columns: true,
|
|
skip_empty_lines: true,
|
|
trim: true,
|
|
cast: true
|
|
});
|
|
return records;
|
|
} catch (error) {
|
|
console.error(`Error reading ${filePath}:`, error.message);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Normalize date string to PostgreSQL date format
|
|
*/
|
|
function normalizeDate(dateStr) {
|
|
if (!dateStr) return null;
|
|
|
|
// Handle MM/DD/YYYY format
|
|
if (dateStr.match(/^\d{1,2}\/\d{1,2}\/\d{4}$/)) {
|
|
const [month, day, year] = dateStr.split('/');
|
|
return `${year}-${month.padStart(2, '0')}-${day.padStart(2, '0')}`;
|
|
}
|
|
|
|
// Handle YYYY-MM-DD format
|
|
if (dateStr.match(/^\d{4}-\d{2}-\d{2}$/)) {
|
|
return dateStr;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Normalize time string
|
|
* Fixes common issues like "34 AM" (missing hour) or ":34 AM" (missing hour with colon)
|
|
*/
|
|
function normalizeTime(timeStr) {
|
|
if (!timeStr) return null;
|
|
|
|
let time = timeStr.trim();
|
|
|
|
// Fix pattern: ":34 AM" or ":34 PM" (missing hour, has colon)
|
|
// Convert to "9:34 AM" or "1:34 PM"
|
|
if (time.match(/^:\d{1,2}\s*(AM|PM)$/i)) {
|
|
const match = time.match(/^:(\d{1,2})\s*(AM|PM)$/i);
|
|
if (match) {
|
|
const minute = match[1].padStart(2, '0');
|
|
const ampm = match[2].toUpperCase();
|
|
const hour = ampm === 'AM' ? '9' : '1'; // Default hour
|
|
time = `${hour}:${minute}:00 ${ampm}`;
|
|
}
|
|
}
|
|
|
|
// Fix pattern: "34 AM" or "34 PM" (missing hour, no colon)
|
|
// If number > 12, it's definitely minutes, add default hour
|
|
// If number <= 12, it could be hour or minute - default to minute with hour 9/1
|
|
if (time.match(/^\d{1,2}\s*(AM|PM)$/i) && !time.includes(':')) {
|
|
const match = time.match(/^(\d{1,2})\s*(AM|PM)$/i);
|
|
if (match) {
|
|
const num = parseInt(match[1]);
|
|
const ampm = match[2].toUpperCase();
|
|
|
|
if (num > 12) {
|
|
// Definitely minutes (can't be hour > 12)
|
|
const minute = match[1].padStart(2, '0');
|
|
const hour = ampm === 'AM' ? '9' : '1';
|
|
time = `${hour}:${minute}:00 ${ampm}`;
|
|
} else {
|
|
// Could be hour or minute - assume it's minute with default hour
|
|
const minute = match[1].padStart(2, '0');
|
|
const hour = ampm === 'AM' ? '9' : '1';
|
|
time = `${hour}:${minute}:00 ${ampm}`;
|
|
}
|
|
}
|
|
}
|
|
|
|
return time;
|
|
}
|
|
|
|
/**
|
|
* Clean numeric value
|
|
*/
|
|
function cleanNumeric(value) {
|
|
if (!value || value === '' || value === null || value === undefined) return null;
|
|
if (typeof value === 'number') return value;
|
|
|
|
// Remove commas, dollar signs, spaces
|
|
const cleaned = String(value).replace(/[\s$,]/g, '');
|
|
const num = parseFloat(cleaned);
|
|
return isNaN(num) ? null : num;
|
|
}
|
|
|
|
/**
|
|
* Import OptionsFlow CSV
|
|
*/
|
|
async function importOptionsFlow(csvPath) {
|
|
console.log('📥 Importing OptionsFlow data...');
|
|
|
|
const records = parseCSV(csvPath);
|
|
console.log(` Found ${records.length} records`);
|
|
|
|
// Ensure table exists
|
|
const createTableSQL = `
|
|
CREATE TABLE IF NOT EXISTS "OptionsFlow_monthly" (
|
|
"CreatedDate" TEXT,
|
|
"CreatedTime" TEXT,
|
|
"Symbol" TEXT,
|
|
"Type" TEXT,
|
|
"Volume" TEXT,
|
|
"Price" TEXT,
|
|
"Side" TEXT,
|
|
"CallPut" TEXT,
|
|
"Strike" TEXT,
|
|
"Spot" TEXT,
|
|
"Premium" TEXT,
|
|
"ExpirationDate" TEXT,
|
|
"Color" TEXT,
|
|
"ImpliedVolatility" TEXT,
|
|
"Dte" TEXT,
|
|
"ER" TEXT,
|
|
"StockEtf" TEXT,
|
|
"Sector" TEXT,
|
|
"Uoa" TEXT,
|
|
"Weekly" TEXT,
|
|
"MktCap" TEXT,
|
|
"OI" TEXT
|
|
);
|
|
`;
|
|
|
|
await rawQuery(createTableSQL);
|
|
console.log(' ✅ Table created/verified');
|
|
|
|
// Batch insert (100 records at a time to avoid parameter limits)
|
|
const batchSize = 100;
|
|
let imported = 0;
|
|
|
|
for (let i = 0; i < records.length; i += batchSize) {
|
|
const batch = records.slice(i, i + batchSize);
|
|
|
|
// Build parameterized query
|
|
const placeholders = batch.map((_, idx) => {
|
|
const start = idx * 22 + 1;
|
|
return `($${start}, $${start + 1}, $${start + 2}, $${start + 3}, $${start + 4}, $${start + 5}, $${start + 6}, $${start + 7}, $${start + 8}, $${start + 9}, $${start + 10}, $${start + 11}, $${start + 12}, $${start + 13}, $${start + 14}, $${start + 15}, $${start + 16}, $${start + 17}, $${start + 18}, $${start + 19}, $${start + 20}, $${start + 21})`;
|
|
}).join(', ');
|
|
|
|
const params = batch.flatMap(row => [
|
|
row.CreatedDate || null,
|
|
row.CreatedTime || null,
|
|
row.Symbol || null,
|
|
row.Type || null,
|
|
row.Volume || null,
|
|
row.Price || null,
|
|
row.Side || null,
|
|
row.CallPut || null,
|
|
row.Strike || null,
|
|
row.Spot || null,
|
|
row.Premium || null,
|
|
row.ExpirationDate || null,
|
|
row.Color || null,
|
|
row.ImpliedVolatility || null,
|
|
row.Dte || null,
|
|
row.ER || null,
|
|
row.StockEtf || null,
|
|
row.Sector || null,
|
|
row.Uoa || null,
|
|
row.Weekly || null,
|
|
row.MktCap || null,
|
|
row.OI || null
|
|
]);
|
|
|
|
const insertSQL = `
|
|
INSERT INTO "OptionsFlow_monthly" (
|
|
"CreatedDate", "CreatedTime", "Symbol", "Type", "Volume", "Price",
|
|
"Side", "CallPut", "Strike", "Spot", "Premium", "ExpirationDate",
|
|
"Color", "ImpliedVolatility", "Dte", "ER", "StockEtf", "Sector",
|
|
"Uoa", "Weekly", "MktCap", "OI"
|
|
) VALUES ${placeholders}
|
|
`;
|
|
|
|
try {
|
|
await rawQuery(insertSQL, params);
|
|
imported += batch.length;
|
|
console.log(` ✅ Imported ${imported}/${records.length} records...`);
|
|
} catch (error) {
|
|
console.error(` ❌ Error importing batch ${Math.floor(i / batchSize) + 1}:`, error.message);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
console.log(` ✅ Successfully imported ${imported} OptionsFlow records`);
|
|
return imported;
|
|
}
|
|
|
|
/**
|
|
* Import OptionsVolume CSV
|
|
*/
|
|
async function importOptionsVolume(csvPath) {
|
|
console.log('📥 Importing OptionsVolume data...');
|
|
|
|
const records = parseCSV(csvPath);
|
|
console.log(` Found ${records.length} records`);
|
|
|
|
// Create table if it doesn't exist
|
|
const createTableSQL = `
|
|
CREATE TABLE IF NOT EXISTS "OptionsVolume" (
|
|
"SYMBOL" TEXT,
|
|
"VOL" NUMERIC,
|
|
"C VOL" NUMERIC,
|
|
"P VOL" NUMERIC,
|
|
"P/C" NUMERIC,
|
|
"AVG VOL" NUMERIC,
|
|
"RATIO" NUMERIC,
|
|
"COI" NUMERIC,
|
|
"POI" NUMERIC,
|
|
"DCOI" NUMERIC,
|
|
"DPOI" NUMERIC
|
|
);
|
|
`;
|
|
|
|
await rawQuery(createTableSQL);
|
|
console.log(' ✅ Table created/verified');
|
|
|
|
// Batch insert
|
|
const batchSize = 100;
|
|
let imported = 0;
|
|
|
|
for (let i = 0; i < records.length; i += batchSize) {
|
|
const batch = records.slice(i, i + batchSize);
|
|
|
|
const placeholders = batch.map((_, idx) => {
|
|
const start = idx * 11 + 1;
|
|
return `($${start}, $${start + 1}, $${start + 2}, $${start + 3}, $${start + 4}, $${start + 5}, $${start + 6}, $${start + 7}, $${start + 8}, $${start + 9}, $${start + 10})`;
|
|
}).join(', ');
|
|
|
|
const params = batch.flatMap(row => [
|
|
row.SYMBOL || null,
|
|
cleanNumeric(row.VOL),
|
|
cleanNumeric(row['C VOL']),
|
|
cleanNumeric(row['P VOL']),
|
|
cleanNumeric(row['P/C']),
|
|
cleanNumeric(row['AVG VOL']),
|
|
cleanNumeric(row.RATIO),
|
|
cleanNumeric(row.COI),
|
|
cleanNumeric(row.POI),
|
|
cleanNumeric(row.DCOI),
|
|
cleanNumeric(row.DPOI)
|
|
]);
|
|
|
|
const insertSQL = `
|
|
INSERT INTO "OptionsVolume" (
|
|
"SYMBOL", "VOL", "C VOL", "P VOL", "P/C", "AVG VOL",
|
|
"RATIO", "COI", "POI", "DCOI", "DPOI"
|
|
) VALUES ${placeholders}
|
|
`;
|
|
|
|
try {
|
|
await rawQuery(insertSQL, params);
|
|
imported += batch.length;
|
|
console.log(` ✅ Imported ${imported}/${records.length} records...`);
|
|
} catch (error) {
|
|
console.error(` ❌ Error importing batch:`, error.message);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
console.log(` ✅ Successfully imported ${imported} OptionsVolume records`);
|
|
return imported;
|
|
}
|
|
|
|
/**
|
|
* Import OptionsOpenInterest CSV
|
|
*/
|
|
async function importOptionsOpenInterest(csvPath) {
|
|
console.log('📥 Importing OptionsOpenInterest data...');
|
|
|
|
const records = parseCSV(csvPath);
|
|
console.log(` Found ${records.length} records`);
|
|
|
|
// Create table if it doesn't exist
|
|
const createTableSQL = `
|
|
CREATE TABLE IF NOT EXISTS "OptionsOpenInterest" (
|
|
"SYMBOL" TEXT,
|
|
"EXP" TEXT,
|
|
"C/P" TEXT,
|
|
"STRIKE" NUMERIC,
|
|
"OI" NUMERIC,
|
|
"DOI" NUMERIC
|
|
);
|
|
`;
|
|
|
|
await rawQuery(createTableSQL);
|
|
console.log(' ✅ Table created/verified');
|
|
|
|
// Batch insert
|
|
const batchSize = 100;
|
|
let imported = 0;
|
|
|
|
for (let i = 0; i < records.length; i += batchSize) {
|
|
const batch = records.slice(i, i + batchSize);
|
|
|
|
const placeholders = batch.map((_, idx) => {
|
|
const start = idx * 6 + 1;
|
|
return `($${start}, $${start + 1}, $${start + 2}, $${start + 3}, $${start + 4}, $${start + 5})`;
|
|
}).join(', ');
|
|
|
|
const params = batch.flatMap(row => [
|
|
row.SYMBOL || null,
|
|
row.EXP || null,
|
|
row['C/P'] || null,
|
|
cleanNumeric(row.STRIKE),
|
|
cleanNumeric(row.OI),
|
|
cleanNumeric(row.DOI)
|
|
]);
|
|
|
|
const insertSQL = `
|
|
INSERT INTO "OptionsOpenInterest" (
|
|
"SYMBOL", "EXP", "C/P", "STRIKE", "OI", "DOI"
|
|
) VALUES ${placeholders}
|
|
`;
|
|
|
|
try {
|
|
await rawQuery(insertSQL, params);
|
|
imported += batch.length;
|
|
console.log(` ✅ Imported ${imported}/${records.length} records...`);
|
|
} catch (error) {
|
|
console.error(` ❌ Error importing batch:`, error.message);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
console.log(` ✅ Successfully imported ${imported} OptionsOpenInterest records`);
|
|
return imported;
|
|
}
|
|
|
|
/**
|
|
* Import AlertStream CSV
|
|
*/
|
|
async function importAlertStream(csvPath) {
|
|
console.log('📥 Importing AlertStream data...');
|
|
|
|
const records = parseCSV(csvPath);
|
|
console.log(` Found ${records.length} records`);
|
|
|
|
// Ensure table exists (columns match what queries expect)
|
|
const createTableSQL = `
|
|
DO $$
|
|
BEGIN
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema='public' AND table_name='AlertStream_monthly') THEN
|
|
CREATE TABLE public."AlertStream_monthly" (
|
|
"Date" TEXT,
|
|
"AlertDate" TEXT,
|
|
"date" TEXT,
|
|
"alert_date" TEXT,
|
|
"Ticker" TEXT,
|
|
"ticker" TEXT,
|
|
"Symbol" TEXT,
|
|
"symbol" TEXT,
|
|
"Type" TEXT,
|
|
"type" TEXT,
|
|
"AlertType" TEXT,
|
|
"alert_type" TEXT,
|
|
"Message" TEXT,
|
|
"message" TEXT,
|
|
"Timestamp" TEXT,
|
|
"timestamp" TEXT,
|
|
"alert_time_txt" TEXT,
|
|
"Notional" TEXT,
|
|
"notional" TEXT,
|
|
"Pct_of_Avg30Day" TEXT,
|
|
"pct_of_avg30day" TEXT,
|
|
"Pct_of_Avg30" TEXT,
|
|
"pct_of_avg30" TEXT,
|
|
"Price" NUMERIC(12, 4),
|
|
"price" NUMERIC(12, 4),
|
|
"Volume" NUMERIC(15, 2),
|
|
"volume" NUMERIC(15, 2)
|
|
);
|
|
END IF;
|
|
END $$;
|
|
`;
|
|
|
|
await rawQuery(createTableSQL);
|
|
|
|
// Add missing columns if they don't exist
|
|
const addColumnSQL = `
|
|
DO $$
|
|
BEGIN
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Ticker') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Ticker" TEXT;
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='ticker') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "ticker" TEXT;
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Timestamp') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Timestamp" TEXT;
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='timestamp') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "timestamp" TEXT;
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Type') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Type" TEXT;
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='type') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "type" TEXT;
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Message') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Message" TEXT;
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='message') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "message" TEXT;
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Date') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Date" TEXT;
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='date') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "date" TEXT;
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Price') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Price" NUMERIC(12, 4);
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='price') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "price" NUMERIC(12, 4);
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Volume') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Volume" NUMERIC(15, 2);
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='volume') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "volume" NUMERIC(15, 2);
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Notional') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Notional" TEXT;
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='notional') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "notional" TEXT;
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='Pct_of_Avg30Day') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "Pct_of_Avg30Day" TEXT;
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='AlertStream_monthly' AND column_name='pct_of_avg30day') THEN
|
|
ALTER TABLE public."AlertStream_monthly" ADD COLUMN "pct_of_avg30day" TEXT;
|
|
END IF;
|
|
END $$;
|
|
`;
|
|
|
|
await rawQuery(addColumnSQL);
|
|
console.log(' ✅ Table created/verified');
|
|
|
|
// Batch insert
|
|
const batchSize = 100;
|
|
let imported = 0;
|
|
|
|
for (let i = 0; i < records.length; i += batchSize) {
|
|
const batch = records.slice(i, i + batchSize);
|
|
|
|
// Map CSV columns to both CamelCase and lowercase versions for compatibility
|
|
const placeholders = batch.map((_, idx) => {
|
|
const start = idx * 18 + 1;
|
|
return `($${start}, $${start + 1}, $${start + 2}, $${start + 3}, $${start + 4}, $${start + 5}, $${start + 6}, $${start + 7}, $${start + 8}, $${start + 9}, $${start + 10}, $${start + 11}, $${start + 12}, $${start + 13}, $${start + 14}, $${start + 15}, $${start + 16}, $${start + 17})`;
|
|
}).join(', ');
|
|
|
|
const params = batch.flatMap(row => [
|
|
row.Date || null, row.Date || null, // Date, date
|
|
row.Timestamp || null, row.Timestamp || null, // Timestamp, timestamp
|
|
row.Ticker || null, row.Ticker || null, // Ticker, ticker
|
|
row.Type || null, row.Type || null, // Type, type
|
|
row.Message || null, row.Message || null, // Message, message
|
|
cleanNumeric(row.Price), cleanNumeric(row.Price), // Price, price
|
|
cleanNumeric(row.Volume), cleanNumeric(row.Volume), // Volume, volume
|
|
cleanNumeric(row.Notional)?.toString() || null, cleanNumeric(row.Notional)?.toString() || null, // Notional, notional
|
|
row.Pct_of_Avg30Day || null, row.Pct_of_Avg30Day || null // Pct_of_Avg30Day, pct_of_avg30day
|
|
]);
|
|
|
|
const insertSQL = `
|
|
INSERT INTO public."AlertStream_monthly" (
|
|
"Date", "date", "Timestamp", "timestamp", "Ticker", "ticker",
|
|
"Type", "type", "Message", "message", "Price", "price",
|
|
"Volume", "volume", "Notional", "notional", "Pct_of_Avg30Day", "pct_of_avg30day"
|
|
) VALUES ${placeholders}
|
|
`;
|
|
|
|
try {
|
|
await rawQuery(insertSQL, params);
|
|
imported += batch.length;
|
|
console.log(` ✅ Imported ${imported}/${records.length} records...`);
|
|
} catch (error) {
|
|
console.error(` ❌ Error importing batch:`, error.message);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
console.log(` ✅ Successfully imported ${imported} AlertStream records`);
|
|
return imported;
|
|
}
|
|
|
|
/**
|
|
* Main import function
|
|
*/
|
|
async function main() {
|
|
console.log('🚀 Starting CSV import process...\n');
|
|
|
|
try {
|
|
// Find CSV files in data folder
|
|
const projectRoot = path.resolve(__dirname, '../..');
|
|
const dataFolder = path.join(projectRoot, 'data');
|
|
|
|
// Check if data folder exists
|
|
if (!fs.existsSync(dataFolder)) {
|
|
throw new Error(`Data folder not found at ${dataFolder}`);
|
|
}
|
|
|
|
// Find all CSV files
|
|
const optionsFlowFiles = [
|
|
path.join(dataFolder, 'OptionsFlow.csv'),
|
|
path.join(dataFolder, 'OptionsFlowMonthly (1).csv'),
|
|
path.join(dataFolder, 'HistoricalOptionsFlow.csv')
|
|
].filter(f => fs.existsSync(f));
|
|
|
|
const optionsVolumePath = path.join(dataFolder, 'OptionsVolume.csv');
|
|
const optionsOpenInterestPath = path.join(dataFolder, 'OptionsOpenInterest.csv');
|
|
const alertStreamPath = path.join(dataFolder, 'AlertStream.csv');
|
|
|
|
console.log('📁 Found CSV files:');
|
|
optionsFlowFiles.forEach(f => console.log(` - ${f}`));
|
|
if (fs.existsSync(optionsVolumePath)) console.log(` - ${optionsVolumePath}`);
|
|
if (fs.existsSync(optionsOpenInterestPath)) console.log(` - ${optionsOpenInterestPath}`);
|
|
if (fs.existsSync(alertStreamPath)) console.log(` - ${alertStreamPath}`);
|
|
console.log('');
|
|
|
|
// Import each file
|
|
const results = {
|
|
optionsFlow: 0,
|
|
optionsVolume: 0,
|
|
optionsOpenInterest: 0,
|
|
alertStream: 0
|
|
};
|
|
|
|
// Import all OptionsFlow files
|
|
for (const filePath of optionsFlowFiles) {
|
|
console.log(`📥 Importing ${path.basename(filePath)}...`);
|
|
const count = await importOptionsFlow(filePath);
|
|
results.optionsFlow += count;
|
|
console.log('');
|
|
}
|
|
|
|
if (fs.existsSync(optionsVolumePath)) {
|
|
results.optionsVolume = await importOptionsVolume(optionsVolumePath);
|
|
console.log('');
|
|
}
|
|
|
|
if (fs.existsSync(optionsOpenInterestPath)) {
|
|
results.optionsOpenInterest = await importOptionsOpenInterest(optionsOpenInterestPath);
|
|
console.log('');
|
|
}
|
|
|
|
if (fs.existsSync(alertStreamPath)) {
|
|
results.alertStream = await importAlertStream(alertStreamPath);
|
|
console.log('');
|
|
}
|
|
|
|
// Summary
|
|
console.log('✅ Import Summary:');
|
|
console.log(` OptionsFlow: ${results.optionsFlow} records (from ${optionsFlowFiles.length} file(s))`);
|
|
if (results.optionsVolume > 0) console.log(` OptionsVolume: ${results.optionsVolume} records`);
|
|
if (results.optionsOpenInterest > 0) console.log(` OptionsOpenInterest: ${results.optionsOpenInterest} records`);
|
|
if (results.alertStream > 0) console.log(` AlertStream: ${results.alertStream} records`);
|
|
const total = results.optionsFlow + results.optionsVolume + results.optionsOpenInterest + results.alertStream;
|
|
console.log(` Total: ${total} records`);
|
|
|
|
} catch (error) {
|
|
console.error('\n❌ Import failed:', error.message);
|
|
console.error(error);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
// Run if called directly
|
|
const isMainModule = import.meta.url === `file://${process.argv[1]}` ||
|
|
import.meta.url.endsWith('importCSV.js') ||
|
|
process.argv[1]?.endsWith('importCSV.js');
|
|
|
|
if (isMainModule || !process.env.NODE_ENV) {
|
|
main().catch(error => {
|
|
console.error('Fatal error:', error);
|
|
process.exit(1);
|
|
});
|
|
}
|
|
|
|
export { importOptionsFlow, importOptionsVolume, importOptionsOpenInterest, importAlertStream };
|
|
|