From c8deeaf9e0821e6f781ff54b873b937f78b014f1 Mon Sep 17 00:00:00 2001 From: Deep Koluguri Date: Fri, 12 Dec 2025 20:55:24 -0500 Subject: [PATCH] added tailscale --- DEPLOYMENT_QUICK_START.md | 24 +- PROXMOX_DEPLOYMENT.md | 231 ++++++++++++++---- backend/database/setup_friend_access.sql | 9 +- backend/env.production.example | 14 +- .../__pycache__/db.cpython-312.pyc | Bin 3964 -> 3982 bytes backend/python_service/db.py | 2 +- .../options_flow_processor.cpython-312.pyc | Bin 37936 -> 38185 bytes .../services/options_flow_processor.py | 12 +- backend/scripts/checkDbConnection.js | 64 +++++ backend/scripts/setupLocalDB.js | 2 +- backend/scripts/syncBlackBoxFlow.js | 16 +- backend/scripts/testAndSetup.js | 2 +- backend/src/db.js | 100 +++++++- 13 files changed, 403 insertions(+), 73 deletions(-) create mode 100644 backend/scripts/checkDbConnection.js diff --git a/DEPLOYMENT_QUICK_START.md b/DEPLOYMENT_QUICK_START.md index d740128..b08f58d 100644 --- a/DEPLOYMENT_QUICK_START.md +++ b/DEPLOYMENT_QUICK_START.md @@ -36,10 +36,19 @@ sudo bash deploy.sh ```env NODE_ENV=production PORT=3000 -SUPABASE_URL=your_supabase_url -SUPABASE_ANON_KEY=your_anon_key -SUPABASE_SERVICE_KEY=your_service_key -DATABASE_URL=your_database_url + +# Remote PostgreSQL Database +USE_LOCAL_DB=true +LOCAL_DB_HOST=100.121.163.23 +LOCAL_DB_PORT=5432 +LOCAL_DB_USER=postgres +LOCAL_DB_PASSWORD=your_postgres_password +LOCAL_DB_NAME=institutional_trader + +# OR use DATABASE_URL format: +# DATABASE_URL=postgresql://postgres:your_password@100.121.163.23:5432/institutional_trader + +# CORS Configuration CORS_ORIGIN=https://yourdomain.com JWT_SECRET=your_secret_key_32_chars_min ``` @@ -101,9 +110,9 @@ sudo certbot --nginx -d yourdomain.com -d www.yourdomain.com -d api.yourdomain.c **Option B: PostgreSQL User** ```bash -# Run the SQL script +# Connect to remote database and run the SQL script cd /opt/institutional_trader/backend/database -sudo -u postgres psql institutional_trader < setup_friend_access.sql +psql -h 100.121.163.23 -p 5432 -U postgres -d institutional_trader < setup_friend_access.sql # Edit the script first to set username/password! ``` @@ -114,6 +123,9 @@ sudo -u postgres psql institutional_trader < setup_friend_access.sql sudo systemctl status institutional-trader-backend sudo systemctl status nginx +# Test database connection +psql -h 100.121.163.23 -p 5432 -U postgres -d institutional_trader + # Test health endpoint curl https://api.yourdomain.com/health diff --git a/PROXMOX_DEPLOYMENT.md b/PROXMOX_DEPLOYMENT.md index de308c8..839d4b0 100644 --- a/PROXMOX_DEPLOYMENT.md +++ b/PROXMOX_DEPLOYMENT.md @@ -38,10 +38,18 @@ Complete guide for deploying the Institutional Trader platform on a Proxmox serv - Configure: - **Hostname**: `institutional-trader` - **Password**: Set root password - - **CPU**: 2-4 cores - - **Memory**: 4-8 GB RAM - - **Disk**: 20-50 GB + - **CPU**: 2-4 cores ✅ (You have 4 CPUs - perfect!) + - **Memory**: 4-8 GB RAM ✅ (You have 31.75 GiB - excellent!) + - **Disk**: 20-50 GB ✅ (You have 294.23 GiB - plenty of space!) - **Network**: Bridge with static IP (recommended) or DHCP + - **Unprivileged**: Yes/No (see note below) + +**Note on Unprivileged Containers:** +- ✅ **Your configuration will work** - Unprivileged containers are fine for this deployment +- Services run on ports > 1024 (Node.js: 3000, Python: 8000) - no issues +- Nginx can run as reverse proxy (may need to bind to ports > 1024 or use host network) +- PostgreSQL can be installed and run normally +- SSL certificates work fine with Certbot 2. **Start Container:** ```bash @@ -80,6 +88,43 @@ Complete guide for deploying the Institutional Trader platform on a Proxmox serv ## 3. System Dependencies +### Important: Unprivileged Container Considerations + +If your container is **unprivileged** (which yours is), there are a few adjustments: + +1. **Nginx Port Binding**: + - Unprivileged containers cannot bind to ports < 1024 + - **Solution**: Use Nginx on host or configure port forwarding + - Alternative: Run Nginx on port 8080/8443 and forward from host + +2. **Systemd**: + - May need to enable systemd in container + - Check: `systemctl status` should work + +3. **File Permissions**: + - Some operations may need different permissions + - Usually not an issue for this application + +**Your Configuration Analysis:** +- ✅ **4 CPUs** - Perfect (recommended 2-4) +- ✅ **31.75 GiB RAM** - Excellent (recommended 4-8 GB) +- ✅ **294.23 GiB Disk** - Plenty of space (recommended 20-50 GB) +- ⚠️ **Unprivileged: Yes** - Will work, but see Nginx note below + +### Enabling Systemd (if needed) + +If systemd doesn't work in your container, you may need to enable it: + +```bash +# Check if systemd is working +systemctl status + +# If it fails, you may need to enable it in Proxmox +# Edit container config on Proxmox host: +pct set -features nesting=1 +pct set -features keyctl=1 +``` + Run these commands on your Ubuntu container/VM: ```bash @@ -93,7 +138,7 @@ apt install -y nodejs # Install Python 3.11+ and pip apt install -y python3 python3-pip python3-venv -# Install PostgreSQL client (if using local DB) +# Install PostgreSQL client (for remote database connection) apt install -y postgresql-client # Install Nginx @@ -160,7 +205,7 @@ SUPABASE_ANON_KEY=your_anon_key SUPABASE_SERVICE_KEY=your_service_role_key # OR Direct PostgreSQL Connection (if using local/remote PostgreSQL) -DATABASE_URL=postgresql://postgres:password@localhost:5432/institutional_trader +DATABASE_URL=postgresql://postgres:password@100.121.163.23:5432/institutional_trader # CORS Configuration CORS_ORIGIN=https://yourdomain.com @@ -236,42 +281,40 @@ This creates a `dist` folder with production-ready files. - Use `DATABASE_URL` or `SUPABASE_URL` + keys in backend `.env` - See [Database Access for Friend](#9-database-access-for-friend) section for sharing access -### Option B: Local PostgreSQL (Self-Hosted) +### Option B: Remote PostgreSQL Database -```bash -# Install PostgreSQL -apt install -y postgresql postgresql-contrib +Your database is hosted at `100.121.163.23:5432`. Configure connection: -# Start PostgreSQL -systemctl start postgresql -systemctl enable postgresql - -# Create database and user -sudo -u postgres psql - -# In PostgreSQL prompt: -CREATE DATABASE institutional_trader; -CREATE USER trader_user WITH PASSWORD 'secure_password_here'; -GRANT ALL PRIVILEGES ON DATABASE institutional_trader TO trader_user; -\q - -# Import schema -cd /opt/institutional_trader/backend/database -sudo -u postgres psql institutional_trader < schema.sql -sudo -u postgres psql institutional_trader < missing_tables.sql -# Import other SQL files as needed -``` - -Update backend `.env`: +**Update backend `.env`:** ```env USE_LOCAL_DB=true -LOCAL_DB_HOST=localhost +LOCAL_DB_HOST=100.121.163.23 LOCAL_DB_PORT=5432 -LOCAL_DB_USER=trader_user -LOCAL_DB_PASSWORD=secure_password_here +LOCAL_DB_USER=postgres +LOCAL_DB_PASSWORD=your_postgres_password LOCAL_DB_NAME=institutional_trader ``` +**Or use DATABASE_URL format:** +```env +DATABASE_URL=postgresql://postgres:your_password@100.121.163.23:5432/institutional_trader +``` + +**Test connection:** +```bash +# Test from container +psql -h 100.121.163.23 -p 5432 -U postgres -d institutional_trader + +# Or test from Proxmox host +apt install -y postgresql-client +psql -h 100.121.163.23 -p 5432 -U postgres -d institutional_trader +``` + +**Note:** Make sure the remote PostgreSQL server allows connections from your Proxmox container IP. You may need to: +1. Configure `pg_hba.conf` on the database server to allow your container's IP +2. Configure firewall rules to allow port 5432 from your container +3. Ensure PostgreSQL is listening on the correct interface (not just localhost) + --- ## 6. Systemd Services @@ -288,7 +331,7 @@ Add: ```ini [Unit] Description=Institutional Trader Backend API -After=network.target postgresql.service +After=network.target [Service] Type=simple @@ -326,7 +369,7 @@ Add: ```ini [Unit] Description=Institutional Trader Python Service -After=network.target postgresql.service +After=network.target [Service] Type=simple @@ -368,8 +411,30 @@ sudo journalctl -u institutional-trader-* -f ## 7. Nginx Configuration +### ⚠️ Important: Unprivileged Container Nginx Setup + +If your container is **unprivileged**, Nginx cannot bind to ports 80/443 directly. You have two options: + +**Option A: Run Nginx on Proxmox Host (Recommended)** +- Install Nginx on the Proxmox host +- Configure it to proxy to your container's services +- Container services run on ports 3000 (backend) and 8000 (Python) +- See "Nginx on Host" section below + +**Option B: Use Port Forwarding** +- Run Nginx in container on ports 8080/8443 +- Forward ports from Proxmox host to container +- Configure firewall rules + +**Option C: Enable Privileged Mode (Less Secure)** +- Convert container to privileged mode +- Allows binding to ports 80/443 +- Less secure but simpler + ### 7.1 Create Nginx Configuration +**For Container (if using Option B):** + ```bash sudo nano /etc/nginx/sites-available/institutional-trader ``` @@ -492,6 +557,77 @@ sudo nginx -t sudo systemctl reload nginx ``` +### 7.3 Nginx on Proxmox Host (For Unprivileged Containers) + +If your container is unprivileged, install Nginx on the Proxmox host: + +**On Proxmox Host:** +```bash +# Install Nginx +apt install -y nginx + +# Create configuration +nano /etc/nginx/sites-available/institutional-trader +``` + +**Nginx Config (on Proxmox host, pointing to container):** +```nginx +# Backend API Server +server { + listen 443 ssl http2; + server_name api.yourdomain.com; + + ssl_certificate /etc/letsencrypt/live/yourdomain.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/yourdomain.com/privkey.pem; + + location / { + proxy_pass http://:3000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /ws { + proxy_pass http://:3000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_read_timeout 86400; + } +} + +# Frontend Server +server { + listen 443 ssl http2; + server_name yourdomain.com www.yourdomain.com; + + ssl_certificate /etc/letsencrypt/live/yourdomain.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/yourdomain.com/privkey.pem; + + root /var/www/institutional-trader/dist; + index index.html; + + location / { + try_files $uri $uri/ /index.html; + } +} +``` + +**Copy frontend files to host:** +```bash +# On container, create archive +cd /opt/institutional_trader/frontend +tar czf /tmp/frontend-dist.tar.gz dist/ + +# On Proxmox host, copy from container +pct pull /tmp/frontend-dist.tar.gz /tmp/ +tar xzf /tmp/frontend-dist.tar.gz -C /var/www/institutional-trader/ +``` + --- ## 8. SSL Certificates @@ -601,13 +737,14 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, USAGE ON SEQUENCES TO fr **Connection String for Friend:** ``` -postgresql://friend_user:secure_password_here@your-server-ip:5432/institutional_trader +postgresql://friend_user:secure_password_here@100.121.163.23:5432/institutional_trader ``` -**Important:** If your friend needs to connect from outside your network: -1. Configure PostgreSQL to accept remote connections -2. Set up firewall rules -3. Consider using SSH tunnel for security +**Important:** Make sure the PostgreSQL server at `100.121.163.23`: +1. Allows remote connections from your friend's IP +2. Has `pg_hba.conf` configured to allow connections +3. Has firewall rules allowing port 5432 +4. Consider using SSH tunnel for additional security ### Option C: SSH Tunnel (Most Secure) @@ -615,12 +752,14 @@ Create an SSH tunnel for your friend: ```bash # On your friend's machine -ssh -L 5432:localhost:5432 user@your-server-ip +ssh -L 5432:100.121.163.23:5432 user@your-proxmox-server-ip # Then they can connect using: # postgresql://friend_user:password@localhost:5432/institutional_trader ``` +This creates a secure tunnel through your Proxmox server to the database. + ### Option D: Read-Only Access (If Friend Only Needs to Read Data) ```sql @@ -696,13 +835,13 @@ sudo systemctl reload nginx - Use Supabase Dashboard > Database > Backups - Or use `pg_dump` with connection string -**For Local PostgreSQL:** +**For Remote PostgreSQL:** ```bash # Create backup -sudo -u postgres pg_dump institutional_trader > backup_$(date +%Y%m%d).sql +pg_dump -h 100.121.163.23 -p 5432 -U postgres -d institutional_trader > backup_$(date +%Y%m%d).sql # Restore backup -sudo -u postgres psql institutional_trader < backup_20240101.sql +psql -h 100.121.163.23 -p 5432 -U postgres -d institutional_trader < backup_20240101.sql ``` --- @@ -751,8 +890,8 @@ sudo systemctl reload nginx ### Database Connection Issues ```bash -# Test PostgreSQL connection (local) -psql -h localhost -U trader_user -d institutional_trader +# Test PostgreSQL connection (remote) +psql -h 100.121.163.23 -p 5432 -U postgres -d institutional_trader # Test Supabase connection psql "postgresql://postgres:password@db.project.supabase.co:5432/postgres" diff --git a/backend/database/setup_friend_access.sql b/backend/database/setup_friend_access.sql index 1b1e8cc..fb01131 100644 --- a/backend/database/setup_friend_access.sql +++ b/backend/database/setup_friend_access.sql @@ -3,6 +3,7 @@ -- -- Usage: -- For Supabase: Run in Supabase SQL Editor +-- For Remote PostgreSQL: psql -h 100.121.163.23 -p 5432 -U postgres -d institutional_trader < setup_friend_access.sql -- For Local PostgreSQL: sudo -u postgres psql institutional_trader < setup_friend_access.sql -- -- IMPORTANT: Replace 'friend_user' and 'secure_password_here' with actual values! @@ -126,13 +127,13 @@ ORDER BY grantee, table_name; -- Connection Strings for Friend -- ============================================ +-- For Remote PostgreSQL (100.121.163.23): +-- postgresql://friend_user:secure_password_here@100.121.163.23:5432/institutional_trader +-- -- For Supabase: -- postgresql://friend_user:secure_password_here@db.[project-ref].supabase.co:5432/postgres -- --- For Local PostgreSQL: --- postgresql://friend_user:secure_password_here@your-server-ip:5432/institutional_trader --- -- For SSH Tunnel (most secure): --- 1. Friend runs: ssh -L 5432:localhost:5432 user@your-server-ip +-- 1. Friend runs: ssh -L 5432:100.121.163.23:5432 user@your-proxmox-server-ip -- 2. Friend connects to: postgresql://friend_user:password@localhost:5432/institutional_trader diff --git a/backend/env.production.example b/backend/env.production.example index 588ea3d..71ffd18 100644 --- a/backend/env.production.example +++ b/backend/env.production.example @@ -10,8 +10,20 @@ SUPABASE_ANON_KEY=your_production_anon_key SUPABASE_SERVICE_KEY=your_service_role_key # Direct PostgreSQL Connection (optional, for heavy queries) +# Option 1: Remote PostgreSQL Database +DATABASE_URL=postgresql://postgres:[password]@100.121.163.23:5432/institutional_trader + +# Option 2: Supabase Database # Get this from: Supabase Dashboard > Settings > Database > Connection String (URI) -DATABASE_URL=postgresql://postgres:[password]@db.[project-ref].supabase.co:5432/postgres +# DATABASE_URL=postgresql://postgres:[password]@db.[project-ref].supabase.co:5432/postgres + +# Option 3: Use LOCAL_DB variables (for remote database) +USE_LOCAL_DB=true +LOCAL_DB_HOST=100.121.163.23 +LOCAL_DB_PORT=5432 +LOCAL_DB_USER=postgres +LOCAL_DB_PASSWORD=your_postgres_password +LOCAL_DB_NAME=institutional_trader # CORS Configuration CORS_ORIGIN=https://your-frontend-domain.com diff --git a/backend/python_service/__pycache__/db.cpython-312.pyc b/backend/python_service/__pycache__/db.cpython-312.pyc index 544b702c987b8076eef3358e5144807b1db56910..694cf6cc029db4d91fe5941cdd438e31a4b77e81 100644 GIT binary patch delta 424 zcmew(*C)?=nwOW00SFxD+GMWW$h((Cpo-7Xz(CK?$WYJF%vjIJc=J`3IgA1#KsC2G zN{dsA@)C1Xi^L{3vhCn90SXELak1HCXLd`r3&zfuC0sT)vTtXUQ2+{m24aRHV<2&h zD>)~>I5oZ?KR>6)Vsi**Gou7Z^DXwo(Bwb7^=yhD<8&uC@>$t|?Y98(Z!s5_6y0LU r$plgkAbDpH0W!Tv0YsRC2$1E!IBatBQ%ZAE?TUOS|KzJ@0m%RWZA@}^ delta 399 zcmeB^|0BnHnwOW00SIPGTV*cU$h((?`xa+TesW??Mt*V0=36Xt7zIRu(ziHDi&Km8 z5_40F#3#40?cg#63JL&mvFT(Fc1zX`)|VyhH@CBIXOvL{ihc%Sh9VOnaf>TCC%-r~ zz92t8r^s@16lXIdqv_;(Tw;t?lfQDwi-Fa!7M7+KRTen{)w=+3aoA)nZgVLH0lEIV z&bklG3_LPFZt%+jvGxxjPe5*SBliWyNRShXY(RE!q@|W5XB6d^-(pG0OfJ!6EwTd1 z*?|aq5a9qMZZQKD6te asyncpg.Pool: if use_local_db: config = { - 'host': os.getenv('LOCAL_DB_HOST', 'localhost'), + 'host': os.getenv('LOCAL_DB_HOST', '100.121.163.23'), 'port': int(os.getenv('LOCAL_DB_PORT', '5432')), 'user': os.getenv('LOCAL_DB_USER', 'postgres'), 'password': os.getenv('LOCAL_DB_PASSWORD', 'postgres'), diff --git a/backend/python_service/services/__pycache__/options_flow_processor.cpython-312.pyc b/backend/python_service/services/__pycache__/options_flow_processor.cpython-312.pyc index 60d95f8d3b49f5c3797dfc64063c68288aa48a32..9534e491a449276eb18f3cf9d96e15a40bcd3ad7 100644 GIT binary patch delta 2201 zcmah~eN0nV6o2=XmX=ydfm%OG`EtWDD>_6hV@6HKH=W9dqA*0Lp-4ZtEo!0IB7#mt z!Lvy+v+2| z-22WwZ|D>|aGH(0rcgvMx+*X?f7ZNu- zS0&1%)Kl%)+yJ`hc*u<|XGI8#BHzl({4le3*-g3jm~pSgv?QLVhsIg$oSTv?79*HWx6Qg+mBhmfQm(j{N zQ!+c8tTbhahnFBz2Iq)+Nx5xUv$kkGlOZp&vlIxld9vS@bvD zSkQ$!p~8?X@21}U6nbEfVZq`Ox;#mNo*3>T5D?YaJ%O-#OHCdAh#L11&_`l=`nU=n z7)H=}?#qSYVtGBaG*Ym`mGng9f!-=LJWtP;h)8$2Jm6cZgRM*IB&{^Y%l*7$CX(=^ z26&jU7e#U{OQo#bJ}FZVJZVSac`#@db!O1^jBPM9;bGta;&?&D6Vv z!f6Vm^_WsUBCi`eoY+B4v_)|f1s4T@0R1cNNkSdec$or?{+;~zXMoX%VGpFN+J@eR z&sIItc!(L_A>g8d34W}hh87roe=T(8#>{TMYPp$Oif2=+uvxMiVDUaMn37NnsB%_- zI4>4Xn*JpJ2uoc^P6*$l@h^~Iyl9RPjS6AH91B}=eiY*2S%C@;=SBvPaw-s73S(h* zUP!i#%Kq_c*)q72KOU5Hm}5XuIOG444zn0OHYwpip&+|1YU0(3)sS5rR77>)H0!48 zU0P#;$L3#xySCEup1xYlN_MS*!PS+)Q?Nn)n(e{09oV(q@%(GkQH2HyZ&OI3V5HCl z4@=*G(Y5)g3H> zngU#;039q|XGUG5fDp3>`qqE?ntTFtYlRv0lAO!oX+>W!Z$DhyfM1%I@7!!|b^U)EwzHvCn{2azA556?-bfy!Xghy2&)@gY9K|<2h-MUH>(5?>1>a4iB*3oQt z>u2Cj$f`~=(z_XuY^!N(wzvW^H?Fa}wm7hjpCovQI;FMEHl7m@H9BeuF*rz$S06_P z?!8TEGD!=GYsrAba5b!6DupxdD9Fy2qI)o8%@4jm?jP$dR?gpK{!Y7Kym3Am6*Ajg ziEzIm)wUVk=Xz`xS>%91XPhuuqSTW0`G_|o&FG%yT zaL%;?oo+RJEZR~M}FA1|8P1TjKZAc75R1-WG9bhGP^J%cXA~?Y9tAM zDeMO}5b+-lL6yxhpVH{j4W`Y&k{XP96PlB>-y1*+{X-APZS|8@6cVOER&8PiujGpyISe5D{b`f-B<3MDP=*f`AOGSZ$S}eU;mS728qdV=z>Z zvuWburp`=Iw`ty%><^uD=;k!ra5H{%B-;{_ITsV+mTYQFcJI3sHEQ;f{&IWn?|06< z=bdw2&vE`#Hy?RLs~y3S=l#4{PG{}u$jT`G0s=vu%6no!muiN)*=qPkV}aAr*@g&z zL_W91!Esie;{vjPT+KnB+7#(We%T3PqEA3UO$tBnm!IIFKOrGP=~Y8t(m39)fT?4Q z&=;kGle*CwJ>zqn*9hP1y0bKR60zVU3X>_MP>7&VKp-UZI$e&Ca)aBo-YGajGP}*| zfKT-k;1|6~MgLT|*H^(qeOcytLJx8S8f}NA&C;2DO*8H&e@uB;*{!-3mvPfDdcZKM z(+9I}7$#jcO!~<%xmDEarh0oT@SZYdQhX-HO0cC0l55f3u=PGv;@stbhn+tE~rmS%_yeJqZFik zG?P^*!X|7F$toQ-JARw)v=K;-#G~li3jJw)=o9gybfsLgiCQ*OaKX8(3CIV(Wv%7! zbC5f|MNvoXJH+p%k3tHG=!8FBX+<&-TU5NpH7wl(%`?)8^s^amq=&?uL;5DVCk2&< zgWkxoqQAwgnG=;3FWn`T!r|i->;#IF5qWFay%h`8MC%1_rGP0g0d&k0;@LTwF{V7T+0iSU?@# zexbtxIJYG1FtA8xtohP$spMM9%IAiMM~jTmvNY_vgt{Jlq3aUJS{8PFufPCx%bz(b zDu>(48D~}Wzc2Ju0lNx4ECCxJw`dz%+TlS_yIFcACMtDOm_{LAn z7cN>aF!6r4vZjNH*=;CW%F6S^2iGn|2jI@yA@Ly?tcZ($+##2!nATuOZFk~|4elDj zn>-RX!L)Vhv9!ZNidx&28iyyO_F|jh+33bD+zfT=%w+H!Sa$>^h_h|!YEp`>svPtq zbXP47_w_$j*LjW9WKz3fxapcqS_TO(DOl@4ys#eK5}Spy{KUs?Ig{4htMph?=_-SQ zR05T-u{txxM|ULl6Wj1{wVQv4L^BTX(NBbwxNy?DIa-pju*{fL^Dc@Jduw!Pv@}f; zXq=}<%Ad8R=#)5b+jSKZAU3!ld~8Rs%nX>7jB`b0u741$5E6<#(#(^Hh2I4 diff --git a/backend/python_service/services/options_flow_processor.py b/backend/python_service/services/options_flow_processor.py index 196e3e6..0c9773a 100644 --- a/backend/python_service/services/options_flow_processor.py +++ b/backend/python_service/services/options_flow_processor.py @@ -84,10 +84,20 @@ class OptionsFlowProcessor: def parse_timestamp(self, date_str, time_str) -> Optional[datetime]: """Parse timestamp from date and time strings""" - if pd.isna(date_str) or pd.isna(time_str): + if pd.isna(date_str): return None date_str = str(date_str).strip() + + # If time_str is None/NaN, parse just the date and use midnight as default + if pd.isna(time_str) or time_str is None: + # Try to parse just the date + date_obj = self.parse_date(date_str) + if date_obj: + # Use midnight (00:00:00) as default time + return datetime.combine(date_obj.date(), datetime.min.time()) + return None + time_str = str(time_str).strip() # Try various formats diff --git a/backend/scripts/checkDbConnection.js b/backend/scripts/checkDbConnection.js new file mode 100644 index 0000000..e535e42 --- /dev/null +++ b/backend/scripts/checkDbConnection.js @@ -0,0 +1,64 @@ +/** + * Check Database Connection Configuration + * Shows which database the application will connect to + * + * Usage: node scripts/checkDbConnection.js + */ + +import dotenv from 'dotenv'; +import { testConnection } from '../src/db.js'; + +dotenv.config(); + +console.log('🔍 Database Connection Configuration Check\n'); +console.log('='.repeat(50)); + +console.log('\n📋 Environment Variables:'); +console.log(' USE_LOCAL_DB:', process.env.USE_LOCAL_DB || '❌ not set'); +console.log(' LOCAL_DB_HOST:', process.env.LOCAL_DB_HOST || '❌ not set (will default to: 100.121.163.23)'); +console.log(' LOCAL_DB_PORT:', process.env.LOCAL_DB_PORT || '❌ not set (will default to: 5432)'); +console.log(' LOCAL_DB_USER:', process.env.LOCAL_DB_USER || '❌ not set (will default to: postgres)'); +console.log(' LOCAL_DB_NAME:', process.env.LOCAL_DB_NAME || '❌ not set (will default to: institutional_trader)'); +console.log(' LOCAL_DB_PASSWORD:', process.env.LOCAL_DB_PASSWORD ? '✅ set' : '❌ not set'); +console.log(' DATABASE_URL:', process.env.DATABASE_URL ? '✅ set (masked)' : '❌ not set'); + +console.log('\n🔌 Actual Connection Details:'); +const actualHost = process.env.LOCAL_DB_HOST || '100.121.163.23'; +const actualPort = process.env.LOCAL_DB_PORT || '5432'; +const actualUser = process.env.LOCAL_DB_USER || 'postgres'; +const actualDb = process.env.LOCAL_DB_NAME || 'institutional_trader'; + +console.log(` Host: ${actualHost}`); +console.log(` Port: ${actualPort}`); +console.log(` User: ${actualUser}`); +console.log(` Database: ${actualDb}`); + +if (actualHost === 'localhost' || actualHost === '127.0.0.1') { + console.log('\n⚠️ WARNING: You are connecting to localhost!'); + console.log(' If you want to use the remote database at 100.121.163.23,'); + console.log(' either:'); + console.log(' 1. Remove LOCAL_DB_HOST from your .env file, OR'); + console.log(' 2. Set LOCAL_DB_HOST=100.121.163.23 in your .env file'); +} else if (actualHost === '100.121.163.23') { + console.log('\n✅ Correct: Connecting to remote database at 100.121.163.23'); +} else { + console.log(`\n📌 Connecting to: ${actualHost}`); +} + +console.log('\n🧪 Testing connection...'); +testConnection() + .then(connected => { + if (connected) { + console.log('\n✅ Connection test successful!'); + console.log(` Data will be saved to: ${actualHost}:${actualPort}/${actualDb}`); + } else { + console.log('\n❌ Connection test failed!'); + console.log(' Please check your database credentials and network connectivity.'); + } + process.exit(connected ? 0 : 1); + }) + .catch(error => { + console.error('\n❌ Error testing connection:', error.message); + process.exit(1); + }); + diff --git a/backend/scripts/setupLocalDB.js b/backend/scripts/setupLocalDB.js index 0bed675..d4169c6 100644 --- a/backend/scripts/setupLocalDB.js +++ b/backend/scripts/setupLocalDB.js @@ -18,7 +18,7 @@ const __dirname = path.dirname(__filename); // Default local PostgreSQL connection (can be overridden via env) const localDbConfig = { - host: process.env.LOCAL_DB_HOST || 'localhost', + host: process.env.LOCAL_DB_HOST || '100.121.163.23', port: process.env.LOCAL_DB_PORT || 5432, user: process.env.LOCAL_DB_USER || 'postgres', password: process.env.LOCAL_DB_PASSWORD || 'postgres', diff --git a/backend/scripts/syncBlackBoxFlow.js b/backend/scripts/syncBlackBoxFlow.js index a3d3c5e..2751e28 100644 --- a/backend/scripts/syncBlackBoxFlow.js +++ b/backend/scripts/syncBlackBoxFlow.js @@ -9,7 +9,7 @@ */ import { fetchBlackBoxFlow, mapBlackBoxToDatabase } from '../src/services/blackboxApiService.js'; -import { rawQuery } from '../src/db.js'; +import { rawQuery, testConnection } from '../src/db.js'; import dotenv from 'dotenv'; dotenv.config(); @@ -153,6 +153,15 @@ async function syncBlackBoxFlow() { console.log('🚀 Starting BlackBox Stocks flow data sync...\n'); try { + // Test database connection first + console.log('🔌 Testing database connection...'); + const dbConnected = await testConnection(); + if (!dbConnected) { + console.error('❌ Database connection failed. Please check your .env configuration.'); + process.exit(1); + } + console.log(''); + // Parse command line arguments const args = process.argv.slice(2); const options = {}; @@ -218,8 +227,11 @@ async function syncBlackBoxFlow() { // Insert into database console.log('\n💾 Inserting records into database...'); + const dbHost = process.env.LOCAL_DB_HOST || '100.121.163.23'; + const dbName = process.env.LOCAL_DB_NAME || 'institutional_trader'; + console.log(` Target: ${dbHost}/${dbName}`); const inserted = await insertFlowRecords(mappedRecords, true); - console.log(`\n✅ Successfully synced ${inserted} records`); + console.log(`\n✅ Successfully synced ${inserted} records to ${dbHost}/${dbName}`); // Summary console.log('\n📊 Sync Summary:'); diff --git a/backend/scripts/testAndSetup.js b/backend/scripts/testAndSetup.js index e67d1d4..29d63a7 100644 --- a/backend/scripts/testAndSetup.js +++ b/backend/scripts/testAndSetup.js @@ -7,7 +7,7 @@ import dotenv from 'dotenv'; dotenv.config(); const localDbConfig = { - host: process.env.LOCAL_DB_HOST || 'localhost', + host: process.env.LOCAL_DB_HOST || '100.121.163.23', port: parseInt(process.env.LOCAL_DB_PORT || '5432'), user: process.env.LOCAL_DB_USER || 'postgres', password: process.env.LOCAL_DB_PASSWORD || 'postgres', diff --git a/backend/src/db.js b/backend/src/db.js index ef3d01f..616db55 100644 --- a/backend/src/db.js +++ b/backend/src/db.js @@ -13,7 +13,7 @@ let pgPool = null; if (useLocalDb) { // Use local PostgreSQL database const localDbConfig = { - host: process.env.LOCAL_DB_HOST || 'localhost', + host: process.env.LOCAL_DB_HOST || '100.121.163.23', port: parseInt(process.env.LOCAL_DB_PORT || '5432'), user: process.env.LOCAL_DB_USER || 'postgres', password: process.env.LOCAL_DB_PASSWORD || 'postgres', @@ -30,6 +30,20 @@ if (useLocalDb) { ); } + // Warn if connecting to localhost (might be wrong) + if (localDbConfig.host === 'localhost' || localDbConfig.host === '127.0.0.1') { + console.warn('⚠️ WARNING: Connecting to localhost!'); + console.warn(' If you meant to use remote database at 100.121.163.23,'); + console.warn(' set LOCAL_DB_HOST=100.121.163.23 in your .env file'); + } + + console.log('🔌 Initializing PostgreSQL connection...'); + console.log(' Host:', localDbConfig.host); + console.log(' Port:', localDbConfig.port); + console.log(' User:', localDbConfig.user); + console.log(' Database:', localDbConfig.database); + console.log(' Max connections:', 20); + pgPool = new Pool({ host: localDbConfig.host, port: localDbConfig.port, @@ -42,13 +56,32 @@ if (useLocalDb) { }); pgPool.on('error', (err) => { - console.error('Unexpected error on idle client', err); + console.error('❌ Unexpected error on idle PostgreSQL client:', err.message); }); - console.log('📦 Using local PostgreSQL database'); + pgPool.on('connect', () => { + console.log('✅ PostgreSQL client connected'); + }); + + console.log('📦 PostgreSQL connection pool created'); } else if (process.env.DATABASE_URL) { // Use remote PostgreSQL via connection string try { + // Parse connection string for logging (mask password) + const dbUrl = process.env.DATABASE_URL; + const urlMatch = dbUrl.match(/postgresql:\/\/([^:]+):([^@]+)@([^:]+):(\d+)\/(.+)/); + + console.log('🔌 Initializing PostgreSQL connection from DATABASE_URL...'); + if (urlMatch) { + console.log(' Host:', urlMatch[3]); + console.log(' Port:', urlMatch[4]); + console.log(' User:', urlMatch[1]); + console.log(' Database:', urlMatch[5]); + } else { + console.log(' Using connection string (masked)'); + } + console.log(' Max connections:', 20); + pgPool = new Pool({ connectionString: process.env.DATABASE_URL, max: 20, @@ -57,11 +90,17 @@ if (useLocalDb) { }); pgPool.on('error', (err) => { - console.error('Unexpected error on idle client', err); + console.error('❌ Unexpected error on idle PostgreSQL client:', err.message); }); + + pgPool.on('connect', () => { + console.log('✅ PostgreSQL client connected'); + }); + + console.log('📦 PostgreSQL connection pool created from DATABASE_URL'); } catch (err) { - console.warn('Failed to create PostgreSQL pool:', err.message); - console.warn('Will fall back to Supabase client'); + console.warn('⚠️ Failed to create PostgreSQL pool:', err.message); + console.warn(' Will fall back to Supabase client'); } } @@ -75,6 +114,11 @@ if (!useLocalDb) { const supabaseAnonKey = process.env.SUPABASE_ANON_KEY || process.env.SUPABASE_KEY; if (supabaseUrl && supabaseKey) { + console.log('🔌 Initializing Supabase connection...'); + console.log(' URL:', supabaseUrl); + console.log(' Using service key:', supabaseKey ? 'Yes' : 'No'); + console.log(' Using anon key:', supabaseAnonKey ? 'Yes' : 'No'); + // Create Supabase client with service key for admin operations supabase = createClient(supabaseUrl, supabaseKey, { auth: { @@ -90,9 +134,40 @@ if (!useLocalDb) { // Create Supabase client with anon key for regular operations supabaseAnon = createClient(supabaseUrl, supabaseAnonKey); + + console.log('📦 Supabase clients created'); } } +// Log connection summary with actual connection details +console.log('\n📊 Database Connection Summary:'); +console.log(' USE_LOCAL_DB:', process.env.USE_LOCAL_DB || 'not set'); +console.log(' LOCAL_DB_HOST:', process.env.LOCAL_DB_HOST || 'not set (default: 100.121.163.23)'); +console.log(' DATABASE_URL:', process.env.DATABASE_URL ? 'set (masked)' : 'not set'); + +if (pgPool) { + console.log(' Type: PostgreSQL (Direct Connection)'); + if (useLocalDb) { + const actualHost = process.env.LOCAL_DB_HOST || '100.121.163.23'; + const actualPort = process.env.LOCAL_DB_PORT || '5432'; + const actualDb = process.env.LOCAL_DB_NAME || 'institutional_trader'; + console.log(` ✅ ACTUAL CONNECTION: ${actualHost}:${actualPort}/${actualDb}`); + } else if (process.env.DATABASE_URL) { + const urlMatch = process.env.DATABASE_URL.match(/postgresql:\/\/([^:]+):([^@]+)@([^:]+):(\d+)\/(.+)/); + if (urlMatch) { + console.log(` ✅ ACTUAL CONNECTION: ${urlMatch[3]}:${urlMatch[4]}/${urlMatch[5]}`); + } else { + console.log(' Using DATABASE_URL connection string'); + } + } +} else if (supabase) { + console.log(' Type: Supabase'); + console.log(` URL: ${process.env.SUPABASE_URL}`); +} else { + console.log(' ⚠️ No database connection configured'); +} +console.log(''); + export { pgPool, supabase, supabaseAnon }; /** @@ -158,13 +233,15 @@ export async function testConnection() { // Prefer direct PostgreSQL connection if available if (pgPool) { try { + const startTime = Date.now(); await Promise.race([ pgPool.query('SELECT 1'), new Promise((_, reject) => setTimeout(() => reject(new Error('Connection timeout')), 10000) ) ]); - console.log('✅ Database connection successful (PostgreSQL direct)'); + const latency = Date.now() - startTime; + console.log(`✅ Database connection successful (PostgreSQL direct) - Latency: ${latency}ms`); return true; } catch (pgErr) { if (pgErr.code === 'ENOTFOUND' || pgErr.message?.includes('ENOTFOUND')) { @@ -206,12 +283,15 @@ export async function testConnection() { return false; } - const { data, error } = await supabase.from('options_flow').select('count').limit(1); + const startTime = Date.now(); + const { error } = await supabase.from('options_flow').select('count').limit(1); + const latency = Date.now() - startTime; + if (error) { // PGRST116 = table doesn't exist (ok for setup) // PGRST301 = invalid API key if (error.code === 'PGRST116') { - console.log('✅ Database connection successful (table may not exist yet)'); + console.log(`✅ Database connection successful (Supabase) - Table may not exist yet - Latency: ${latency}ms`); return true; } if (error.code === 'PGRST301' || error.message?.includes('Invalid API key')) { @@ -238,7 +318,7 @@ export async function testConnection() { console.error('Database connection error:', error); return false; } - console.log('✅ Database connection successful (Supabase)'); + console.log(`✅ Database connection successful (Supabase) - Latency: ${latency}ms`); return true; } catch (err) { console.error('Database connection failed:', err);