393 lines
14 KiB
JavaScript
393 lines
14 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* Copy all LuckyChit data from PostgreSQL → SQLite (local file).
|
||
*
|
||
* Prerequisites:
|
||
* 1. PostgreSQL reachable with your old data (set SOURCE_DATABASE_URL or DB_* below).
|
||
* 2. SQLite schema already exists: run `npm start` once with DB_DIALECT=sqlite so tables are created,
|
||
* or the script will exit with instructions.
|
||
*
|
||
* Usage (from backend/):
|
||
* node scripts/migrate-pg-to-sqlite.js
|
||
*
|
||
* Env files (backend/): loads `.env`, then merges `.env.prod` and `.env.local` without
|
||
* overriding keys already set (so sqlite-only `.env` can still pick up Postgres from `.env.prod`).
|
||
*
|
||
* SOURCE_DATABASE_URL=postgresql://... (recommended if DATABASE_URL in .env is sqlite)
|
||
* Or POSTGRES_URL / DATABASE_URL if postgres:// or postgresql://
|
||
* SQLITE_STORAGE=./data/luckychit.sqlite (destination)
|
||
* Or DB_* / PGHOST, PGDATABASE, PGUSER, PGPASSWORD
|
||
*/
|
||
|
||
const path = require('path');
|
||
const backendRoot = path.join(__dirname, '..');
|
||
require('dotenv').config({ path: path.join(backendRoot, '.env') });
|
||
require('dotenv').config({ path: path.join(backendRoot, '.env.prod'), override: false });
|
||
require('dotenv').config({ path: path.join(backendRoot, '.env.local'), override: false });
|
||
|
||
const { Sequelize, QueryTypes } = require('sequelize');
|
||
|
||
const TABLES_ORDER = [
|
||
'users',
|
||
'chit_groups',
|
||
'group_members',
|
||
'payments',
|
||
'payment_audit_events',
|
||
'monthly_draws',
|
||
'notifications',
|
||
];
|
||
|
||
const TABLES_DELETE_REVERSE = [...TABLES_ORDER].reverse();
|
||
|
||
function isPostgresConnectionString(s) {
|
||
const t = String(s).trim();
|
||
return /^postgres(ql)?:\/\//i.test(t);
|
||
}
|
||
|
||
function getPostgresUrl() {
|
||
const candidates = [
|
||
process.env.SOURCE_DATABASE_URL,
|
||
process.env.POSTGRES_URL,
|
||
process.env.DATABASE_URL,
|
||
].filter(Boolean);
|
||
|
||
for (const u of candidates) {
|
||
if (isPostgresConnectionString(u)) return String(u).trim();
|
||
}
|
||
|
||
const host = process.env.DB_HOST || process.env.PGHOST;
|
||
const port = process.env.DB_PORT || process.env.PGPORT || 5432;
|
||
const name = process.env.DB_NAME || process.env.PGDATABASE;
|
||
const user = process.env.DB_USER || process.env.PGUSER;
|
||
const pass = process.env.DB_PASSWORD ?? process.env.PGPASSWORD ?? '';
|
||
|
||
if (host && name && user != null && String(user).length > 0) {
|
||
const userEnc = encodeURIComponent(String(user));
|
||
const passEnc = encodeURIComponent(String(pass));
|
||
return `postgresql://${userEnc}:${passEnc}@${host}:${port}/${name}`;
|
||
}
|
||
|
||
throw new Error(
|
||
'Could not build PostgreSQL URL. Your .env likely uses SQLite only.\n' +
|
||
' Add one line to backend/.env (use your real Postgres host/user/db), then run again:\n' +
|
||
' SOURCE_DATABASE_URL=postgresql://USER:PASSWORD@HOST:5432/DBNAME\n' +
|
||
' Or set POSTGRES_URL / DATABASE_URL to a postgresql:// or postgres:// URL.\n' +
|
||
' Or set DB_HOST, DB_NAME, DB_USER, DB_PASSWORD (and optional DB_PORT),\n' +
|
||
' or libpq vars: PGHOST, PGDATABASE, PGUSER, PGPASSWORD (optional PGPORT).'
|
||
);
|
||
}
|
||
|
||
function getSqlitePath() {
|
||
const raw = process.env.SQLITE_STORAGE || path.join('data', 'luckychit.sqlite');
|
||
return path.isAbsolute(raw) ? raw : path.join(__dirname, '..', raw);
|
||
}
|
||
|
||
function serializeForSqlite(value) {
|
||
if (value === null || value === undefined) return null;
|
||
if (value instanceof Date) return value.toISOString();
|
||
if (typeof value === 'boolean') return value ? 1 : 0;
|
||
if (typeof value === 'bigint') return Number(value);
|
||
if (Buffer.isBuffer(value)) return value.toString('hex');
|
||
if (typeof value === 'object') return JSON.stringify(value);
|
||
return value;
|
||
}
|
||
|
||
async function tableExists(sqlite, name) {
|
||
const rows = await sqlite.query(
|
||
`SELECT name FROM sqlite_master WHERE type='table' AND name=:name`,
|
||
{ replacements: { name }, type: QueryTypes.SELECT }
|
||
);
|
||
return Array.isArray(rows) && rows.length > 0;
|
||
}
|
||
|
||
function colSetKey(cols) {
|
||
return [...cols].sort().join(',');
|
||
}
|
||
|
||
/** Detect old/broken SQLite schemas (e.g. UNIQUE(group_id) only → one row per group). */
|
||
async function groupMembersSqliteNeedsRepair(sqlite) {
|
||
const list = await sqlite.query(`PRAGMA index_list('group_members')`, { type: QueryTypes.SELECT });
|
||
let hasGroupUser = false;
|
||
let hasGroupMemberNum = false;
|
||
let badGroupIdOnly = false;
|
||
|
||
for (const row of list) {
|
||
const unique = row.unique === 1 || row.unique === true;
|
||
if (!unique) continue;
|
||
const info = await sqlite.query(`PRAGMA index_info('${String(row.name).replace(/'/g, "''")}')`, {
|
||
type: QueryTypes.SELECT,
|
||
});
|
||
const cols = info
|
||
.slice()
|
||
.sort((a, b) => (a.seqno ?? 0) - (b.seqno ?? 0))
|
||
.map((i) => i.name)
|
||
.filter(Boolean);
|
||
if (cols.length === 1 && cols[0] === 'group_id') {
|
||
badGroupIdOnly = true;
|
||
}
|
||
if (colSetKey(cols) === 'group_id,user_id') hasGroupUser = true;
|
||
if (colSetKey(cols) === 'group_id,member_number') hasGroupMemberNum = true;
|
||
}
|
||
|
||
return badGroupIdOnly || !hasGroupUser || !hasGroupMemberNum;
|
||
}
|
||
|
||
/** Must match `src/models/GroupMember.js` + Sequelize SQLite DDL (table is empty). */
|
||
async function recreateGroupMembersTable(sqlite) {
|
||
await sqlite.query('DROP TABLE IF EXISTS `group_members`');
|
||
await sqlite.query(`
|
||
CREATE TABLE \`group_members\` (
|
||
\`id\` UUID PRIMARY KEY,
|
||
\`group_id\` UUID NOT NULL REFERENCES \`chit_groups\` (\`id\`) ON DELETE CASCADE ON UPDATE CASCADE,
|
||
\`user_id\` UUID NOT NULL REFERENCES \`users\` (\`id\`) ON DELETE CASCADE ON UPDATE CASCADE,
|
||
\`member_number\` INTEGER NOT NULL,
|
||
\`joined_date\` DATETIME NOT NULL,
|
||
\`status\` TEXT DEFAULT 'active',
|
||
\`total_paid\` DECIMAL(15,2) DEFAULT 0,
|
||
\`total_won\` DECIMAL(15,2) DEFAULT 0,
|
||
\`created_at\` DATETIME NOT NULL,
|
||
\`updated_at\` DATETIME NOT NULL
|
||
)
|
||
`);
|
||
await sqlite.query(
|
||
'CREATE UNIQUE INDEX `group_members_group_id_user_id` ON `group_members` (`group_id`, `user_id`)'
|
||
);
|
||
await sqlite.query(
|
||
'CREATE UNIQUE INDEX `group_members_group_id_member_number` ON `group_members` (`group_id`, `member_number`)'
|
||
);
|
||
}
|
||
|
||
/** Sorted column-set keys for each non-PK unique index (matches `PRAGMA index_list` / `index_info`). */
|
||
async function sqliteUniqueIndexColKeys(sqlite, tableName) {
|
||
if (!/^[a-z_]+$/.test(tableName)) {
|
||
throw new Error(`sqliteUniqueIndexColKeys: invalid table name "${tableName}"`);
|
||
}
|
||
const list = await sqlite.query(`PRAGMA index_list('${tableName}')`, { type: QueryTypes.SELECT });
|
||
const keys = [];
|
||
for (const row of list) {
|
||
const unique = row.unique === 1 || row.unique === true;
|
||
if (!unique) continue;
|
||
const info = await sqlite.query(`PRAGMA index_info('${String(row.name).replace(/'/g, "''")}')`, {
|
||
type: QueryTypes.SELECT,
|
||
});
|
||
const cols = info
|
||
.slice()
|
||
.sort((a, b) => (a.seqno ?? 0) - (b.seqno ?? 0))
|
||
.map((i) => i.name)
|
||
.filter(Boolean);
|
||
if (row.origin === 'pk') continue;
|
||
if (cols.length === 1 && cols[0] === 'id') continue;
|
||
keys.push(colSetKey(cols));
|
||
}
|
||
return keys;
|
||
}
|
||
|
||
async function monthlyDrawsSqliteNeedsRepair(sqlite) {
|
||
const keys = await sqliteUniqueIndexColKeys(sqlite, 'monthly_draws');
|
||
const hasComposite = keys.includes('group_id,month,year');
|
||
const badSingleton = keys.some((k) => k === 'year' || k === 'month' || k === 'group_id');
|
||
return !hasComposite || badSingleton;
|
||
}
|
||
|
||
/** Must match `src/models/MonthlyDraw.js` + Sequelize SQLite DDL (table is empty). */
|
||
async function recreateMonthlyDrawsTable(sqlite) {
|
||
await sqlite.query('DROP TABLE IF EXISTS `monthly_draws`');
|
||
await sqlite.query(`
|
||
CREATE TABLE \`monthly_draws\` (
|
||
\`id\` UUID PRIMARY KEY,
|
||
\`group_id\` UUID NOT NULL REFERENCES \`chit_groups\` (\`id\`) ON DELETE CASCADE ON UPDATE CASCADE,
|
||
\`month\` INTEGER NOT NULL,
|
||
\`year\` INTEGER NOT NULL,
|
||
\`draw_date\` DATETIME NOT NULL,
|
||
\`eligible_members\` JSON NOT NULL DEFAULT '[]',
|
||
\`winner_id\` UUID REFERENCES \`users\` (\`id\`) ON DELETE SET NULL ON UPDATE CASCADE,
|
||
\`prize_amount\` DECIMAL(15,2),
|
||
\`server_seed\` VARCHAR(255) NOT NULL,
|
||
\`server_seed_hash\` VARCHAR(255) NOT NULL,
|
||
\`client_seed\` VARCHAR(255) NOT NULL,
|
||
\`nonce\` BIGINT NOT NULL,
|
||
\`result_hash\` VARCHAR(255) NOT NULL,
|
||
\`status\` TEXT DEFAULT 'pending',
|
||
\`notes\` TEXT,
|
||
\`created_at\` DATETIME NOT NULL,
|
||
\`updated_at\` DATETIME NOT NULL
|
||
)
|
||
`);
|
||
await sqlite.query(
|
||
'CREATE UNIQUE INDEX `monthly_draws_group_id_month_year` ON `monthly_draws` (`group_id`, `month`, `year`)'
|
||
);
|
||
}
|
||
|
||
async function paymentsSqliteNeedsRepair(sqlite) {
|
||
const keys = await sqliteUniqueIndexColKeys(sqlite, 'payments');
|
||
const hasComposite = keys.includes('group_id,month,year,user_id');
|
||
const badSingleton = keys.some((k) => ['year', 'month', 'group_id', 'user_id'].includes(k));
|
||
return !hasComposite || badSingleton;
|
||
}
|
||
|
||
/** Must match `src/models/Payment.js` + Sequelize SQLite DDL (table is empty). */
|
||
async function recreatePaymentsTable(sqlite) {
|
||
await sqlite.query('DROP TABLE IF EXISTS `payments`');
|
||
await sqlite.query(`
|
||
CREATE TABLE \`payments\` (
|
||
\`id\` UUID PRIMARY KEY,
|
||
\`group_id\` UUID NOT NULL REFERENCES \`chit_groups\` (\`id\`) ON DELETE CASCADE ON UPDATE CASCADE,
|
||
\`user_id\` UUID NOT NULL REFERENCES \`users\` (\`id\`) ON DELETE CASCADE ON UPDATE CASCADE,
|
||
\`month\` INTEGER NOT NULL,
|
||
\`year\` INTEGER NOT NULL,
|
||
\`amount\` DECIMAL(15,2) NOT NULL,
|
||
\`payment_method\` TEXT DEFAULT 'upi',
|
||
\`transaction_id\` VARCHAR(255),
|
||
\`source\` TEXT NOT NULL DEFAULT 'manual_manager_entry',
|
||
\`entered_by\` UUID REFERENCES \`users\` (\`id\`) ON DELETE SET NULL ON UPDATE CASCADE,
|
||
\`idempotency_key\` VARCHAR(128),
|
||
\`status\` TEXT DEFAULT 'pending',
|
||
\`paid_at\` DATETIME,
|
||
\`notes\` TEXT,
|
||
\`created_at\` DATETIME NOT NULL,
|
||
\`updated_at\` DATETIME NOT NULL
|
||
)
|
||
`);
|
||
await sqlite.query(
|
||
'CREATE UNIQUE INDEX `payments_group_id_user_id_month_year` ON `payments` (`group_id`, `user_id`, `month`, `year`)'
|
||
);
|
||
await sqlite.query('CREATE UNIQUE INDEX `payments_idempotency_key` ON `payments` (`idempotency_key`)');
|
||
}
|
||
|
||
async function fetchPostgresTable(pg, table) {
|
||
try {
|
||
return await pg.query(`SELECT * FROM "${table}"`, { type: QueryTypes.SELECT });
|
||
} catch (e) {
|
||
if (e.parent && e.parent.code === '42P01') {
|
||
console.warn(` ⚠️ Postgres: table "${table}" does not exist — skipping.`);
|
||
return [];
|
||
}
|
||
throw e;
|
||
}
|
||
}
|
||
|
||
async function insertRows(sqlite, table, rows) {
|
||
if (!rows.length) {
|
||
console.log(` ${table}: 0 rows`);
|
||
return;
|
||
}
|
||
|
||
const cols = Object.keys(rows[0]);
|
||
const colList = cols.map((c) => `"${c}"`).join(', ');
|
||
const placeholders = cols.map(() => '?').join(', ');
|
||
const sql = `INSERT INTO "${table}" (${colList}) VALUES (${placeholders})`;
|
||
|
||
const t = await sqlite.transaction();
|
||
try {
|
||
for (const row of rows) {
|
||
const values = cols.map((c) => serializeForSqlite(row[c]));
|
||
await sqlite.query(sql, { replacements: values, transaction: t });
|
||
}
|
||
await t.commit();
|
||
console.log(` ${table}: ${rows.length} rows copied`);
|
||
} catch (e) {
|
||
await t.rollback();
|
||
throw e;
|
||
}
|
||
}
|
||
|
||
async function main() {
|
||
console.log('🔄 PostgreSQL → SQLite migration\n');
|
||
|
||
const pgUrl = getPostgresUrl();
|
||
const sqlitePath = getSqlitePath();
|
||
|
||
console.log(` Source (Postgres): ${pgUrl.replace(/:[^:@]+@/, ':****@')}`);
|
||
console.log(` Dest (SQLite): ${sqlitePath}\n`);
|
||
|
||
const pg = new Sequelize(pgUrl, {
|
||
dialect: 'postgres',
|
||
logging: false,
|
||
pool: { max: 5, min: 0, idle: 10000 },
|
||
});
|
||
|
||
// Sequelize sqlite connection-manager runs PRAGMA FOREIGN_KEYS=ON on every new connection
|
||
// unless foreignKeys === false, which overrides manual PRAGMA OFF during pooled/transaction use.
|
||
const sqlite = new Sequelize({
|
||
dialect: 'sqlite',
|
||
storage: sqlitePath,
|
||
logging: false,
|
||
foreignKeys: false,
|
||
define: { underscored: true, freezeTableName: true },
|
||
});
|
||
|
||
await pg.authenticate();
|
||
console.log('✅ Connected to PostgreSQL');
|
||
|
||
await sqlite.authenticate();
|
||
console.log('✅ Connected to SQLite');
|
||
|
||
for (const t of TABLES_ORDER) {
|
||
if (!(await tableExists(sqlite, t))) {
|
||
console.error(
|
||
`\n❌ SQLite table "${t}" is missing. Create schema first:\n` +
|
||
` Set DB_DIALECT=sqlite in backend/.env, then run: npm start\n` +
|
||
` (or ensure Sequelize sync has created tables.)\n`
|
||
);
|
||
await pg.close();
|
||
await sqlite.close();
|
||
process.exit(1);
|
||
}
|
||
}
|
||
|
||
console.log('\n🗑️ Clearing SQLite tables…');
|
||
for (const table of TABLES_DELETE_REVERSE) {
|
||
await sqlite.query(`DELETE FROM "${table}"`, { type: QueryTypes.DELETE });
|
||
}
|
||
console.log(' Done.\n');
|
||
|
||
if (await groupMembersSqliteNeedsRepair(sqlite)) {
|
||
console.log(
|
||
'🔧 Fixing `group_members` SQLite schema (wrong UNIQUE on `group_id` or missing composite indexes).\n' +
|
||
' Recreating empty table to match current Sequelize model…\n'
|
||
);
|
||
await recreateGroupMembersTable(sqlite);
|
||
console.log(' Done.\n');
|
||
}
|
||
|
||
if (await monthlyDrawsSqliteNeedsRepair(sqlite)) {
|
||
console.log(
|
||
'🔧 Fixing `monthly_draws` SQLite schema (wrong UNIQUE on `year` / missing composite on group+month+year).\n' +
|
||
' Recreating empty table to match current Sequelize model…\n'
|
||
);
|
||
await recreateMonthlyDrawsTable(sqlite);
|
||
console.log(' Done.\n');
|
||
}
|
||
|
||
if (await paymentsSqliteNeedsRepair(sqlite)) {
|
||
console.log(
|
||
'🔧 Fixing `payments` SQLite schema (legacy UNIQUE columns vs composite + idempotency_key).\n' +
|
||
' Recreating empty table to match current Sequelize model…\n'
|
||
);
|
||
await recreatePaymentsTable(sqlite);
|
||
console.log(' Done.\n');
|
||
}
|
||
|
||
console.log('📥 Copying from PostgreSQL → SQLite…\n');
|
||
for (const table of TABLES_ORDER) {
|
||
const rows = await fetchPostgresTable(pg, table);
|
||
try {
|
||
await insertRows(sqlite, table, rows);
|
||
} catch (e) {
|
||
e.message = `[${table}] ${e.message}`;
|
||
throw e;
|
||
}
|
||
}
|
||
|
||
await pg.close();
|
||
await sqlite.close();
|
||
|
||
console.log('\n✅ Migration finished. Point DB_DIALECT=sqlite at this file and restart the API.');
|
||
}
|
||
|
||
main().catch((err) => {
|
||
console.error('\n❌ Migration failed:', err.message);
|
||
if (err.parent) console.error(err.parent.message);
|
||
process.exit(1);
|
||
});
|