institutional-trader/backend/python_service/utils/validation.py

"""
Validation utilities for comparing Python vs SQL output
"""
import pandas as pd
from typing import Dict, List, Optional, Tuple
from utils.logger import logger


def compare_outputs(
    python_output: List[Dict],
    sql_output: List[Dict],
    key_columns: Optional[List[str]] = None
) -> Dict:
    """
    Compare Python output with SQL output
    
    Args:
        python_output: List of dicts from Python service
        sql_output: List of dicts from SQL query
        key_columns: Columns to use for matching rows
    
    Returns:
        Comparison report dict
    """
    python_df = pd.DataFrame(python_output)
    sql_df = pd.DataFrame(sql_output)
    
    report = {
        'python_count': len(python_output),
        'sql_count': len(sql_output),
        'count_match': len(python_output) == len(sql_output),
        'differences': [],
        'missing_in_python': [],
        'missing_in_sql': [],
        'column_differences': {}
    }
    
    if python_df.empty and sql_df.empty:
        report['status'] = 'both_empty'
        return report
    
    if python_df.empty:
        report['status'] = 'python_empty'
        return report
    
    if sql_df.empty:
        report['status'] = 'sql_empty'
        return report
    
    # Compare columns
    python_cols = set(python_df.columns)
    sql_cols = set(sql_df.columns)
    
    missing_in_python = sql_cols - python_cols
    missing_in_sql = python_cols - sql_cols
    
    if missing_in_python:
        report['missing_in_python'] = list(missing_in_python)
    if missing_in_sql:
        report['missing_in_sql'] = list(missing_in_sql)
    
    # Compare common columns
    common_cols = python_cols & sql_cols
    
    if key_columns:
        # Match rows by key columns
        for key_col in key_columns:
            if key_col not in common_cols:
                logger.warning(f"Key column {key_col} not found in both outputs")
                key_columns = None
                break
    
    if key_columns:
        # Merge on key columns
        python_key = python_df[key_columns].copy()
        sql_key = sql_df[key_columns].copy()
        
        merged = python_key.merge(
            sql_key,
            on=key_columns,
            how='outer',
            indicator=True
        )
        
        report['only_in_python'] = len(merged[merged['_merge'] == 'left_only'])
        report['only_in_sql'] = len(merged[merged['_merge'] == 'right_only'])
        report['in_both'] = len(merged[merged['_merge'] == 'both'])
    else:
        # Compare by index (assume same order)
        min_len = min(len(python_df), len(sql_df))
        report['compared_rows'] = min_len
    
    # Compare values in common columns
    numeric_cols = []
    text_cols = []
    
    for col in common_cols:
        if python_df[col].dtype in ['int64', 'float64'] or sql_df[col].dtype in ['int64', 'float64']:
            numeric_cols.append(col)
        else:
            text_cols.append(col)
    
    differences = []
    
    # Compare numeric columns
    for col in numeric_cols:
        if col in python_df.columns and col in sql_df.columns:
            python_vals = python_df[col].fillna(0)
            sql_vals = sql_df[col].fillna(0)
            
            # Handle different lengths
            min_len = min(len(python_vals), len(sql_vals))
            python_vals = python_vals[:min_len]
            sql_vals = sql_vals[:min_len]
            
            diff = (python_vals - sql_vals).abs()
            max_diff = diff.max()
            mean_diff = diff.mean()
            
            if max_diff > 0.01:  # Tolerance for floating point
                differences.append({
                    'column': col,
                    'type': 'numeric',
                    'max_difference': float(max_diff),
                    'mean_difference': float(mean_diff),
                    'different_count': int((diff > 0.01).sum())
                })
    
    # Compare text columns
    for col in text_cols:
        if col in python_df.columns and col in sql_df.columns:
            python_vals = python_df[col].fillna('').astype(str)
            sql_vals = sql_df[col].fillna('').astype(str)
            
            min_len = min(len(python_vals), len(sql_vals))
            python_vals = python_vals[:min_len]
            sql_vals = sql_vals[:min_len]
            
            different = (python_vals != sql_vals).sum()
            
            if different > 0:
                differences.append({
                    'column': col,
                    'type': 'text',
                    'different_count': int(different),
                    'total_compared': int(min_len)
                })
    
    report['differences'] = differences
    report['status'] = 'compared'
    
    return report


def print_comparison_report(report: Dict):
    """Print a formatted comparison report"""
    print("\n" + "="*60)
    print("PYTHON vs SQL OUTPUT COMPARISON")
    print("="*60)
    
    print(f"\nRow Counts:")
    print(f"  Python: {report['python_count']}")
    print(f"  SQL:    {report['sql_count']}")
    print(f"  Match:  {'✅' if report['count_match'] else '❌'}")
    
    if report.get('only_in_python'):
        print(f"\n  Only in Python: {report['only_in_python']}")
    if report.get('only_in_sql'):
        print(f"  Only in SQL: {report['only_in_sql']}")
    if report.get('in_both'):
        print(f"  In both: {report['in_both']}")
    
    if report.get('missing_in_python'):
        print(f"\n⚠️  Columns missing in Python: {report['missing_in_python']}")
    
    if report.get('missing_in_sql'):
        print(f"⚠️  Columns missing in SQL: {report['missing_in_sql']}")
    
    if report.get('differences'):
        print(f"\n📊 Column Differences:")
        for diff in report['differences']:
            print(f"\n  Column: {diff['column']} ({diff['type']})")
            if diff['type'] == 'numeric':
                print(f"    Max difference: {diff['max_difference']:.6f}")
                print(f"    Mean difference: {diff['mean_difference']:.6f}")
                print(f"    Different rows: {diff['different_count']}")
            else:
                print(f"    Different rows: {diff['different_count']} / {diff['total_compared']}")
    else:
        print("\n✅ No significant differences found!")
    
    print("\n" + "="*60 + "\n")