254 lines
8.3 KiB
JavaScript
254 lines
8.3 KiB
JavaScript
/**
|
|
* Sync BlackBox Stocks Options Flow Data
|
|
* Fetches latest flow data from BlackBox API and updates the database
|
|
*
|
|
* Usage:
|
|
* node scripts/syncBlackBoxFlow.js
|
|
* node scripts/syncBlackBoxFlow.js --start-date 2024-01-15 --end-date 2024-01-16
|
|
* node scripts/syncBlackBoxFlow.js --limit 100
|
|
*/
|
|
|
|
import { fetchBlackBoxFlow, mapBlackBoxToDatabase } from '../src/services/blackboxApiService.js';
|
|
import { rawQuery } from '../src/db.js';
|
|
import dotenv from 'dotenv';
|
|
|
|
dotenv.config();
|
|
|
|
/**
|
|
* Ensure OptionsFlow_monthly table exists
|
|
*/
|
|
async function ensureTableExists() {
|
|
const createTableSQL = `
|
|
CREATE TABLE IF NOT EXISTS "OptionsFlow_monthly" (
|
|
"CreatedDate" TEXT,
|
|
"CreatedTime" TEXT,
|
|
"Symbol" TEXT,
|
|
"Type" TEXT,
|
|
"Volume" TEXT,
|
|
"Price" TEXT,
|
|
"Side" TEXT,
|
|
"CallPut" TEXT,
|
|
"Strike" TEXT,
|
|
"Spot" TEXT,
|
|
"Premium" TEXT,
|
|
"ExpirationDate" TEXT,
|
|
"Color" TEXT,
|
|
"ImpliedVolatility" TEXT,
|
|
"Dte" TEXT,
|
|
"ER" TEXT,
|
|
"StockEtf" TEXT,
|
|
"Sector" TEXT,
|
|
"Uoa" TEXT,
|
|
"Weekly" TEXT,
|
|
"MktCap" TEXT,
|
|
"OI" TEXT
|
|
);
|
|
`;
|
|
|
|
await rawQuery(createTableSQL);
|
|
console.log('✅ Table OptionsFlow_monthly verified');
|
|
}
|
|
|
|
/**
|
|
* Insert flow records into database
|
|
* @param {Array} records - Array of mapped records
|
|
* @param {boolean} skipDuplicates - Skip duplicate records based on key fields
|
|
*/
|
|
async function insertFlowRecords(records, skipDuplicates = true) {
|
|
if (records.length === 0) {
|
|
console.log(' No records to insert');
|
|
return 0;
|
|
}
|
|
|
|
const batchSize = 100;
|
|
let inserted = 0;
|
|
let skipped = 0;
|
|
|
|
for (let i = 0; i < records.length; i += batchSize) {
|
|
const batch = records.slice(i, i + batchSize);
|
|
|
|
// Build parameterized query
|
|
const placeholders = batch.map((_, idx) => {
|
|
const start = idx * 22 + 1;
|
|
return `($${start}, $${start + 1}, $${start + 2}, $${start + 3}, $${start + 4}, $${start + 5}, $${start + 6}, $${start + 7}, $${start + 8}, $${start + 9}, $${start + 10}, $${start + 11}, $${start + 12}, $${start + 13}, $${start + 14}, $${start + 15}, $${start + 16}, $${start + 17}, $${start + 18}, $${start + 19}, $${start + 20}, $${start + 21})`;
|
|
}).join(', ');
|
|
|
|
const params = batch.flatMap(row => [
|
|
row.CreatedDate || null,
|
|
row.CreatedTime || null,
|
|
row.Symbol || null,
|
|
row.Type || null,
|
|
row.Volume || null,
|
|
row.Price || null,
|
|
row.Side || null,
|
|
row.CallPut || null,
|
|
row.Strike || null,
|
|
row.Spot || null,
|
|
row.Premium || null,
|
|
row.ExpirationDate || null,
|
|
row.Color || null,
|
|
row.ImpliedVolatility || null,
|
|
row.Dte || null,
|
|
row.ER || null,
|
|
row.StockEtf || null,
|
|
row.Sector || null,
|
|
row.Uoa || null,
|
|
row.Weekly || null,
|
|
row.MktCap || null,
|
|
row.OI || null
|
|
]);
|
|
|
|
const insertSQL = skipDuplicates ? `
|
|
INSERT INTO "OptionsFlow_monthly" (
|
|
"CreatedDate", "CreatedTime", "Symbol", "Type", "Volume", "Price",
|
|
"Side", "CallPut", "Strike", "Spot", "Premium", "ExpirationDate",
|
|
"Color", "ImpliedVolatility", "Dte", "ER", "StockEtf", "Sector",
|
|
"Uoa", "Weekly", "MktCap", "OI"
|
|
) VALUES ${placeholders}
|
|
ON CONFLICT DO NOTHING
|
|
` : `
|
|
INSERT INTO "OptionsFlow_monthly" (
|
|
"CreatedDate", "CreatedTime", "Symbol", "Type", "Volume", "Price",
|
|
"Side", "CallPut", "Strike", "Spot", "Premium", "ExpirationDate",
|
|
"Color", "ImpliedVolatility", "Dte", "ER", "StockEtf", "Sector",
|
|
"Uoa", "Weekly", "MktCap", "OI"
|
|
) VALUES ${placeholders}
|
|
`;
|
|
|
|
try {
|
|
await rawQuery(insertSQL, params);
|
|
inserted += batch.length;
|
|
console.log(` ✅ Inserted batch ${Math.floor(i / batchSize) + 1}: ${batch.length} records (Total: ${inserted})`);
|
|
} catch (error) {
|
|
// If ON CONFLICT doesn't work (SQLite), try without it
|
|
if (error.message.includes('ON CONFLICT') || error.message.includes('syntax error')) {
|
|
console.log(' ⚠️ ON CONFLICT not supported, checking for duplicates manually...');
|
|
// For databases without ON CONFLICT, we'll insert and let duplicates exist
|
|
// or implement a check before insert
|
|
const simpleInsertSQL = `
|
|
INSERT INTO "OptionsFlow_monthly" (
|
|
"CreatedDate", "CreatedTime", "Symbol", "Type", "Volume", "Price",
|
|
"Side", "CallPut", "Strike", "Spot", "Premium", "ExpirationDate",
|
|
"Color", "ImpliedVolatility", "Dte", "ER", "StockEtf", "Sector",
|
|
"Uoa", "Weekly", "MktCap", "OI"
|
|
) VALUES ${placeholders}
|
|
`;
|
|
await rawQuery(simpleInsertSQL, params);
|
|
inserted += batch.length;
|
|
console.log(` ✅ Inserted batch ${Math.floor(i / batchSize) + 1}: ${batch.length} records (Total: ${inserted})`);
|
|
} else {
|
|
console.error(` ❌ Error inserting batch ${Math.floor(i / batchSize) + 1}:`, error.message);
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
|
|
return inserted;
|
|
}
|
|
|
|
/**
|
|
* Main sync function
|
|
*/
|
|
async function syncBlackBoxFlow() {
|
|
console.log('🚀 Starting BlackBox Stocks flow data sync...\n');
|
|
|
|
try {
|
|
// Parse command line arguments
|
|
const args = process.argv.slice(2);
|
|
const options = {};
|
|
|
|
for (let i = 0; i < args.length; i++) {
|
|
if (args[i] === '--start-date' && args[i + 1]) {
|
|
options.startDate = args[i + 1];
|
|
i++;
|
|
} else if (args[i] === '--end-date' && args[i + 1]) {
|
|
options.endDate = args[i + 1];
|
|
i++;
|
|
} else if (args[i] === '--limit' && args[i + 1]) {
|
|
options.count = parseInt(args[i + 1]);
|
|
i++;
|
|
} else if (args[i] === '--count' && args[i + 1]) {
|
|
options.count = parseInt(args[i + 1]);
|
|
i++;
|
|
} else if (args[i] === '--symbol' && args[i + 1]) {
|
|
options.symbol = args[i + 1];
|
|
i++;
|
|
}
|
|
}
|
|
|
|
// Default to today if no dates provided
|
|
if (!options.startDate && !options.endDate) {
|
|
const today = new Date().toISOString().split('T')[0];
|
|
options.startDate = today;
|
|
options.endDate = today;
|
|
console.log(`📅 No date range specified, using today: ${today}`);
|
|
}
|
|
|
|
console.log(`📥 Fetching flow data from BlackBox API...`);
|
|
console.log(` Options:`, options);
|
|
|
|
// Ensure table exists
|
|
await ensureTableExists();
|
|
|
|
// Fetch data from API
|
|
const apiRecords = await fetchBlackBoxFlow(options);
|
|
console.log(` ✅ Fetched ${apiRecords.length} records from API`);
|
|
|
|
if (apiRecords.length === 0) {
|
|
console.log('⚠️ No records returned from API');
|
|
return;
|
|
}
|
|
|
|
// Log sample record to understand structure
|
|
if (apiRecords.length > 0) {
|
|
console.log('\n📋 Sample API record structure:');
|
|
console.log(JSON.stringify(apiRecords[0], null, 2));
|
|
}
|
|
|
|
// Map API records to database schema
|
|
console.log('\n🔄 Mapping records to database schema...');
|
|
const mappedRecords = apiRecords.map(mapBlackBoxToDatabase);
|
|
console.log(` ✅ Mapped ${mappedRecords.length} records`);
|
|
|
|
// Log sample mapped record
|
|
if (mappedRecords.length > 0) {
|
|
console.log('\n📋 Sample mapped record:');
|
|
console.log(JSON.stringify(mappedRecords[0], null, 2));
|
|
}
|
|
|
|
// Insert into database
|
|
console.log('\n💾 Inserting records into database...');
|
|
const inserted = await insertFlowRecords(mappedRecords, true);
|
|
console.log(`\n✅ Successfully synced ${inserted} records`);
|
|
|
|
// Summary
|
|
console.log('\n📊 Sync Summary:');
|
|
console.log(` Fetched from API: ${apiRecords.length} records`);
|
|
console.log(` Inserted into DB: ${inserted} records`);
|
|
|
|
// Get total count in database
|
|
const totalCount = await rawQuery('SELECT COUNT(*) as count FROM "OptionsFlow_monthly"');
|
|
console.log(` Total records in DB: ${totalCount[0]?.count || 0}`);
|
|
|
|
} catch (error) {
|
|
console.error('\n❌ Sync failed:', error.message);
|
|
console.error(error);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
// Run if called directly
|
|
const isMainModule = import.meta.url === `file://${process.argv[1]}` ||
|
|
import.meta.url.endsWith('syncBlackBoxFlow.js') ||
|
|
process.argv[1]?.endsWith('syncBlackBoxFlow.js');
|
|
|
|
if (isMainModule || !process.env.NODE_ENV) {
|
|
syncBlackBoxFlow().catch(error => {
|
|
console.error('Fatal error:', error);
|
|
process.exit(1);
|
|
});
|
|
}
|
|
|
|
export { syncBlackBoxFlow };
|
|
|