462 lines
22 KiB
Python
462 lines
22 KiB
Python
"""
|
||
Yahoo Finance Service
|
||
Fetches real-time stock price data from Yahoo Finance
|
||
"""
|
||
import yfinance as yf
|
||
import pandas as pd
|
||
from datetime import datetime, timedelta
|
||
from typing import Dict, Optional, List
|
||
import pytz
|
||
import time
|
||
from utils.logger import logger
|
||
|
||
|
||
class YahooFinanceService:
|
||
"""Service for fetching real-time stock price data from Yahoo Finance"""
|
||
|
||
def __init__(self):
|
||
self.ct_tz = pytz.timezone('America/Chicago')
|
||
self._last_request_time = {}
|
||
self._min_request_interval = 0.1 # 100ms between requests
|
||
self._failed_symbols = set()
|
||
self._cache = {}
|
||
|
||
def _normalize_symbol(self, symbol: str) -> str:
|
||
"""
|
||
Normalize symbol for Yahoo Finance API
|
||
Converts dots to hyphens (e.g., BRK.B -> BRK-B)
|
||
"""
|
||
if not symbol:
|
||
return symbol
|
||
# Convert dots to hyphens for Yahoo Finance API
|
||
return symbol.upper().replace('.', '-')
|
||
|
||
def get_current_price(self, symbol: str) -> Optional[float]:
|
||
"""Get current price for a symbol"""
|
||
try:
|
||
# Normalize symbol for Yahoo Finance
|
||
normalized_symbol = self._normalize_symbol(symbol)
|
||
ticker = yf.Ticker(normalized_symbol)
|
||
data = ticker.history(period="1d", interval="1m")
|
||
if not data.empty:
|
||
return float(data['Close'].iloc[-1])
|
||
# Fallback to info
|
||
info = ticker.info
|
||
return info.get('regularMarketPrice') or info.get('previousClose')
|
||
except Exception as e:
|
||
logger.debug(f"Error fetching current price for {symbol}: {e}")
|
||
return None
|
||
|
||
def _rate_limit(self, symbol: str):
|
||
"""Rate limit requests to avoid overwhelming Yahoo Finance"""
|
||
now = time.time()
|
||
last_time = self._last_request_time.get(symbol, 0)
|
||
elapsed = now - last_time
|
||
|
||
if elapsed < self._min_request_interval:
|
||
time.sleep(self._min_request_interval - elapsed)
|
||
|
||
self._last_request_time[symbol] = time.time()
|
||
|
||
def _get_ticker_safe(self, symbol: str):
|
||
"""Safely get ticker with error handling"""
|
||
# Normalize symbol for Yahoo Finance
|
||
normalized_symbol = self._normalize_symbol(symbol)
|
||
|
||
if normalized_symbol in self._failed_symbols:
|
||
return None
|
||
|
||
try:
|
||
self._rate_limit(normalized_symbol)
|
||
ticker = yf.Ticker(normalized_symbol)
|
||
|
||
# Try to get info to validate symbol, but don't fail if it errors
|
||
# Sometimes ticker.info fails due to API issues but history() still works
|
||
try:
|
||
info = ticker.info
|
||
if not info or len(info) < 2: # Valid ticker should have multiple info fields
|
||
logger.warning(f"⚠️ Symbol {symbol} (normalized: {normalized_symbol}) appears invalid or delisted - info has {len(info) if info else 0} fields")
|
||
# Don't add to failed symbols yet - try history() first
|
||
else:
|
||
logger.debug(f"✅ Validated ticker {symbol} (normalized: {normalized_symbol}) - {len(info)} info fields")
|
||
except Exception as info_error:
|
||
# ticker.info can fail due to API issues (JSON parsing errors, rate limits, etc.)
|
||
# But history() might still work, so we'll try it anyway
|
||
error_msg = str(info_error)
|
||
if 'Expecting value' in error_msg or 'JSON' in error_msg:
|
||
logger.warning(f"⚠️ Yahoo Finance API returned invalid JSON for {symbol} (normalized: {normalized_symbol}) - may be rate limited or API issue")
|
||
logger.warning(f" Will attempt to fetch history data anyway (history() sometimes works when info() fails)")
|
||
else:
|
||
logger.debug(f"⚠️ Could not get ticker.info for {symbol} (normalized: {normalized_symbol}): {info_error}")
|
||
|
||
# Return ticker even if info() failed - history() might still work
|
||
return ticker
|
||
except Exception as e:
|
||
error_msg = str(e)
|
||
if 'Expecting value' in error_msg or 'JSON' in error_msg:
|
||
logger.warning(f"⚠️ Yahoo Finance API error (likely rate limiting or API issue) for {symbol} (normalized: {normalized_symbol})")
|
||
logger.warning(f" Error: {type(e).__name__} - {error_msg}")
|
||
# Don't add to failed symbols for API errors - might work on retry
|
||
else:
|
||
self._failed_symbols.add(normalized_symbol)
|
||
logger.warning(f"⚠️ Error creating ticker for {symbol} (normalized: {normalized_symbol}): {type(e).__name__} - {error_msg}")
|
||
return None
|
||
|
||
def get_price_at_time(
|
||
self,
|
||
symbol: str,
|
||
timestamp: datetime
|
||
) -> Optional[float]:
|
||
"""
|
||
Get price at or before a specific timestamp
|
||
For historical data, fetches intraday data and finds closest match
|
||
"""
|
||
# Check cache first
|
||
cache_key = f"{symbol}_{timestamp.isoformat()}"
|
||
if cache_key in self._cache:
|
||
return self._cache[cache_key]
|
||
|
||
try:
|
||
# Convert timestamp to aware datetime if needed
|
||
if timestamp.tzinfo is None:
|
||
timestamp = self.ct_tz.localize(timestamp)
|
||
|
||
# Skip future dates - no data available
|
||
now = datetime.now(self.ct_tz)
|
||
if timestamp > now:
|
||
logger.warning(f"⚠️ Future date for {symbol}: {timestamp} (now: {now}) - Yahoo Finance cannot provide future data")
|
||
return None
|
||
|
||
ticker = self._get_ticker_safe(symbol)
|
||
if not ticker:
|
||
logger.warning(f"⚠️ Could not get ticker for {symbol} - symbol may be invalid or delisted")
|
||
return None
|
||
|
||
# For current/recent timestamps, fetch intraday data
|
||
time_diff = (now - timestamp).total_seconds() / 3600 # hours
|
||
|
||
# Yahoo Finance only provides intraday data for the last 7 days
|
||
# For older data, we can only get daily close prices
|
||
if time_diff <= 168: # 7 days
|
||
try:
|
||
# Fetch 1-minute data for the day
|
||
data = ticker.history(
|
||
start=timestamp.date(),
|
||
end=(timestamp + timedelta(days=1)).date(),
|
||
interval="1m",
|
||
timeout=5 # 5 second timeout
|
||
)
|
||
|
||
if not data.empty:
|
||
# Find closest timestamp before or equal to target
|
||
data = data[data.index <= timestamp]
|
||
if not data.empty:
|
||
price = float(data['Close'].iloc[-1])
|
||
self._cache[cache_key] = price
|
||
return price
|
||
except Exception as e:
|
||
logger.warning(f"⚠️ Intraday fetch failed for {symbol} at {timestamp}: {e}")
|
||
logger.warning(f" Note: Yahoo Finance only provides intraday data for the last 7 days")
|
||
logger.warning(f" Signal is {time_diff:.1f} hours old - trying daily data instead")
|
||
|
||
# For older data or if intraday fails, use daily data
|
||
# Note: Daily data doesn't give us exact price at a specific time, only the day's close
|
||
try:
|
||
data = ticker.history(
|
||
start=timestamp.date() - timedelta(days=1),
|
||
end=timestamp.date() + timedelta(days=1),
|
||
interval="1d",
|
||
timeout=5
|
||
)
|
||
|
||
if not data.empty:
|
||
# Get closest date
|
||
data = data[data.index.date <= timestamp.date()]
|
||
if not data.empty:
|
||
price = float(data['Close'].iloc[-1])
|
||
logger.info(f"✅ Got daily close price for {symbol} on {timestamp.date()}: ${price:.2f}")
|
||
self._cache[cache_key] = price
|
||
return price
|
||
else:
|
||
logger.warning(f"⚠️ No daily data found for {symbol} on or before {timestamp.date()}")
|
||
else:
|
||
logger.warning(f"⚠️ No daily data returned for {symbol} around {timestamp.date()}")
|
||
except Exception as e:
|
||
logger.warning(f"⚠️ Daily fetch failed for {symbol}: {e}")
|
||
|
||
logger.warning(f"❌ Could not fetch price for {symbol} at {timestamp}")
|
||
return None
|
||
except Exception as e:
|
||
logger.debug(f"Error fetching price at time for {symbol} at {timestamp}: {e}")
|
||
return None
|
||
|
||
def get_intraday_data(
|
||
self,
|
||
symbol: str,
|
||
start_time: datetime,
|
||
end_time: datetime
|
||
) -> pd.DataFrame:
|
||
"""
|
||
Get intraday 1-minute data for a symbol between start_time and end_time
|
||
Returns DataFrame with columns: Open, High, Low, Close, Volume
|
||
"""
|
||
try:
|
||
normalized_symbol = self._normalize_symbol(symbol)
|
||
logger.debug(f"📊 Fetching intraday data for {symbol} (normalized: {normalized_symbol})")
|
||
|
||
ticker = self._get_ticker_safe(symbol)
|
||
if not ticker:
|
||
logger.warning(f"⚠️ Could not get ticker for {symbol} (normalized: {normalized_symbol}) - cannot fetch intraday data")
|
||
logger.warning(f" This symbol may be invalid, delisted, or Yahoo Finance may be rate-limiting")
|
||
return pd.DataFrame()
|
||
|
||
logger.debug(f"📊 Fetching intraday data for {symbol} from {start_time.strftime('%Y-%m-%d %H:%M:%S %Z')} to {end_time.strftime('%Y-%m-%d %H:%M:%S %Z')}")
|
||
|
||
# Fetch 1-minute data with retry logic
|
||
data = pd.DataFrame()
|
||
max_retries = 3
|
||
for attempt in range(max_retries):
|
||
try:
|
||
logger.debug(f"📊 Attempt {attempt + 1}/{max_retries}: Fetching intraday data for {symbol} (normalized: {normalized_symbol})")
|
||
data = ticker.history(
|
||
start=start_time.date(),
|
||
end=(end_time + timedelta(days=1)).date(),
|
||
interval="1m",
|
||
timeout=15 # 15 second timeout for intraday data
|
||
)
|
||
if not data.empty:
|
||
logger.debug(f"✅ Successfully fetched {len(data)} bars for {symbol}")
|
||
break # Success, exit retry loop
|
||
else:
|
||
logger.warning(f"⚠️ Empty data returned (attempt {attempt + 1}/{max_retries}) for {symbol}")
|
||
if attempt < max_retries - 1:
|
||
time.sleep(2) # Wait 2 seconds before retry
|
||
except Exception as fetch_error:
|
||
error_msg = str(fetch_error)
|
||
error_type = type(fetch_error).__name__
|
||
|
||
# Check for JSON parsing errors (API issues)
|
||
if 'Expecting value' in error_msg or 'JSON' in error_msg or 'ValueError' in error_type:
|
||
logger.warning(f"⚠️ Yahoo Finance API error (attempt {attempt + 1}/{max_retries}) for {symbol}: {error_msg}")
|
||
if attempt < max_retries - 1:
|
||
# Wait before retry (exponential backoff: 2s, 4s, 6s)
|
||
wait_time = (attempt + 1) * 2
|
||
logger.info(f"⏳ Waiting {wait_time}s before retry (Yahoo Finance may be rate-limiting)...")
|
||
time.sleep(wait_time)
|
||
else:
|
||
logger.error(f"❌ Failed to fetch intraday data after {max_retries} attempts for {symbol}")
|
||
logger.error(f" Yahoo Finance API appears to be having issues or rate-limiting requests")
|
||
else:
|
||
logger.warning(f"⚠️ Error fetching intraday data for {symbol} (attempt {attempt + 1}/{max_retries}): {error_type} - {error_msg}")
|
||
if attempt < max_retries - 1:
|
||
time.sleep(1) # Short wait for other errors
|
||
else:
|
||
break # Don't retry further for non-API errors
|
||
|
||
if data.empty:
|
||
logger.warning(f"⚠️ No intraday data returned from Yahoo Finance for {symbol} on {start_time.date()}")
|
||
logger.warning(f" This could be due to:")
|
||
logger.warning(f" - Yahoo Finance API rate limiting or temporary issues")
|
||
logger.warning(f" - Symbol {symbol} may need a different format (e.g., crypto symbols)")
|
||
logger.warning(f" - Data is older than 7 days (Yahoo Finance limitation)")
|
||
return pd.DataFrame()
|
||
|
||
logger.debug(f"📊 Fetched {len(data)} bars from Yahoo Finance for {symbol}")
|
||
|
||
# Filter to time range
|
||
if data.index.tz is None:
|
||
# Assume data is in market timezone (Eastern Time for US markets)
|
||
# Yahoo Finance typically returns data in ET
|
||
et_tz = pytz.timezone('America/New_York')
|
||
data.index = data.index.tz_localize(et_tz)
|
||
logger.debug(f"📊 Localized data index to ET (was naive)")
|
||
|
||
# Convert start_time and end_time to same timezone as data
|
||
if data.index.tz:
|
||
start_time_tz = start_time.astimezone(data.index.tz) if start_time.tzinfo else data.index.tz.localize(start_time)
|
||
end_time_tz = end_time.astimezone(data.index.tz) if end_time.tzinfo else data.index.tz.localize(end_time)
|
||
else:
|
||
start_time_tz = start_time
|
||
end_time_tz = end_time
|
||
|
||
logger.debug(f"📊 Filtering data: {start_time_tz.strftime('%H:%M:%S')} to {end_time_tz.strftime('%H:%M:%S')}")
|
||
|
||
# Store original data range for logging
|
||
original_data = data.copy()
|
||
data = data[(data.index >= start_time_tz) & (data.index <= end_time_tz)]
|
||
|
||
logger.debug(f"📊 After filtering: {len(data)} bars remain (from {len(original_data)} total bars)")
|
||
|
||
if data.empty:
|
||
logger.warning(f"⚠️ No data in time range {start_time_tz.strftime('%H:%M:%S')} to {end_time_tz.strftime('%H:%M:%S')} for {symbol}")
|
||
if not original_data.empty:
|
||
logger.warning(f" Available data range: {original_data.index.min().strftime('%H:%M:%S')} to {original_data.index.max().strftime('%H:%M:%S')}")
|
||
else:
|
||
logger.warning(f" No data available at all for {symbol} on {start_time.date()}")
|
||
|
||
return data[['Open', 'High', 'Low', 'Close', 'Volume']]
|
||
except Exception as e:
|
||
logger.error(f"❌ Error fetching intraday data for {symbol}: {type(e).__name__} - {str(e)}")
|
||
import traceback
|
||
logger.debug(f"Traceback: {traceback.format_exc()}")
|
||
return pd.DataFrame()
|
||
|
||
def get_rth_open(self, symbol: str, date: datetime.date) -> Optional[float]:
|
||
"""Get RTH open price (9:30 AM CST) for a given date"""
|
||
try:
|
||
# Skip future dates
|
||
today = datetime.now(self.ct_tz).date()
|
||
if date > today:
|
||
logger.debug(f"Skipping future date for RTH open: {symbol} on {date}")
|
||
return None
|
||
|
||
ticker = self._get_ticker_safe(symbol)
|
||
if not ticker:
|
||
return None
|
||
|
||
# Fetch 1-minute data for the day
|
||
data = ticker.history(
|
||
start=date,
|
||
end=date + timedelta(days=1),
|
||
interval="1m",
|
||
timeout=5
|
||
)
|
||
|
||
if data.empty:
|
||
return None
|
||
|
||
# Find first bar at or after 9:30 AM CST
|
||
rth_start = self.ct_tz.localize(
|
||
datetime.combine(date, datetime.min.time().replace(hour=9, minute=30))
|
||
)
|
||
|
||
if data.index.tz is None:
|
||
data.index = data.index.tz_localize(self.ct_tz)
|
||
|
||
# Filter to RTH hours
|
||
rth_data = data[data.index >= rth_start]
|
||
|
||
if not rth_data.empty:
|
||
return float(rth_data['Open'].iloc[0])
|
||
|
||
return None
|
||
except Exception as e:
|
||
logger.debug(f"Error fetching RTH open for {symbol} on {date}: {e}")
|
||
return None
|
||
|
||
def get_prior_close(self, symbol: str, date: datetime.date) -> Optional[float]:
|
||
"""Get prior day's close price"""
|
||
try:
|
||
ticker = self._get_ticker_safe(symbol)
|
||
if not ticker:
|
||
return None
|
||
|
||
# Fetch daily data
|
||
prior_date = date - timedelta(days=1)
|
||
data = ticker.history(
|
||
start=prior_date - timedelta(days=5), # Get a few days to account for weekends
|
||
end=date,
|
||
interval="1d",
|
||
timeout=5
|
||
)
|
||
|
||
if data.empty:
|
||
return None
|
||
|
||
# Get most recent close before the date
|
||
data = data[data.index.date < date]
|
||
if not data.empty:
|
||
return float(data['Close'].iloc[-1])
|
||
|
||
return None
|
||
except Exception as e:
|
||
logger.debug(f"Error fetching prior close for {symbol} before {date}: {e}")
|
||
return None
|
||
|
||
async def calculate_vwap(
|
||
self,
|
||
symbol: str,
|
||
end_time: datetime
|
||
) -> Optional[float]:
|
||
"""
|
||
Calculate VWAP from RTH open (9:30 AM ET) to end_time
|
||
VWAP = Σ(Price × Volume) / Σ(Volume)
|
||
US market opens at 9:30 AM Eastern Time
|
||
|
||
Uses database (Bookmap) data first, falls back to Yahoo Finance
|
||
"""
|
||
try:
|
||
# Use Eastern Time for market hours (US market is in ET)
|
||
et_tz = pytz.timezone('America/New_York')
|
||
|
||
# Ensure end_time is timezone-aware and convert to ET
|
||
if end_time.tzinfo is None:
|
||
# Assume UTC if naive
|
||
end_time = pytz.UTC.localize(end_time)
|
||
end_time = end_time.astimezone(et_tz)
|
||
|
||
# Skip future dates
|
||
now = datetime.now(et_tz)
|
||
if end_time > now:
|
||
logger.debug(f"Skipping future date for VWAP: {symbol} at {end_time}")
|
||
return None
|
||
|
||
# Get RTH open time for the day (9:30 AM ET)
|
||
rth_start = et_tz.localize(
|
||
datetime.combine(end_time.date(), datetime.min.time().replace(hour=9, minute=30))
|
||
)
|
||
|
||
logger.debug(f"📊 Calculating VWAP for {symbol} at {end_time.strftime('%Y-%m-%d %H:%M ET')}")
|
||
logger.debug(f" RTH start = {rth_start.strftime('%Y-%m-%d %H:%M:%S %Z')}, End = {end_time.strftime('%Y-%m-%d %H:%M:%S %Z')}")
|
||
|
||
# Fetch intraday data from RTH open to end_time
|
||
# Convert to CST for get_intraday_data (Yahoo Finance data is typically in market timezone)
|
||
cst_tz = pytz.timezone('America/Chicago')
|
||
rth_start_cst = rth_start.astimezone(cst_tz)
|
||
end_time_cst = end_time.astimezone(cst_tz)
|
||
|
||
logger.debug(f"📊 Fetching intraday data: {rth_start_cst.strftime('%H:%M')} CST to {end_time_cst.strftime('%H:%M')} CST")
|
||
data = await self.get_intraday_data(symbol, rth_start_cst, end_time_cst)
|
||
|
||
if data.empty:
|
||
logger.warning(f"⚠️ No intraday data available for VWAP calculation for {symbol} at {end_time.strftime('%Y-%m-%d %H:%M:%S %Z')}")
|
||
logger.warning(f" RTH start: {rth_start.strftime('%H:%M:%S %Z')} (9:30 AM ET)")
|
||
logger.warning(f" Checked: Database (prices_intraday_1m) and Yahoo Finance")
|
||
logger.warning(f" Possible reasons:")
|
||
logger.warning(f" - No data in database (Bookmap may not be sending data for this symbol)")
|
||
logger.warning(f" - Data is older than 7 days (Yahoo Finance limitation)")
|
||
logger.warning(f" - Symbol is invalid or delisted")
|
||
logger.warning(f" - Market was closed at this time")
|
||
logger.warning(f" - Yahoo Finance API rate limiting or timeout")
|
||
return None
|
||
|
||
# Calculate VWAP using standard formula (same as TradingView)
|
||
# VWAP = Σ(Price × Volume) / Σ(Volume)
|
||
# Where Price = Typical Price = (High + Low + Close) / 3
|
||
# This matches the industry-standard VWAP calculation
|
||
|
||
# Calculate typical price for each bar: Pt = (Ht + Lt + Ct) / 3
|
||
data['TypicalPrice'] = (data['High'] + data['Low'] + data['Close']) / 3
|
||
|
||
# Calculate price × volume for each bar: PVt = Pt × Vt
|
||
data['PriceVolume'] = data['TypicalPrice'] * data['Volume']
|
||
|
||
# Calculate cumulative sums from RTH open to end_time
|
||
# Cumulative PV = Σ(Pt × Vt), Cumulative V = Σ(Vt)
|
||
total_pv = data['PriceVolume'].sum()
|
||
total_volume = data['Volume'].sum()
|
||
|
||
# VWAP = Cumulative PV / Cumulative V
|
||
if total_volume > 0:
|
||
vwap = total_pv / total_volume
|
||
logger.info(f"✅ VWAP calculated for {symbol}: ${vwap:.2f} (from {len(data)} bars, {total_volume:,.0f} total volume)")
|
||
return float(vwap)
|
||
else:
|
||
logger.warning(f"⚠️ Zero volume for {symbol} - cannot calculate VWAP")
|
||
|
||
return None
|
||
except Exception as e:
|
||
logger.debug(f"Error calculating VWAP for {symbol}: {e}")
|
||
return None
|
||
|