institutional-trader/backend/src/factorlab/descriptiveLens.js

import { FACTORS, standardizeCrossSection } from './factors.js';
import { pullRawRecord } from './ingest.js';
import { getUniverse } from '../services/stockUniverseService.js';

/**
 * Assigns percentiles (0-100) to a raw factor array using the standardizer.
 * @param {Array} raw - Array of objects: { ticker, value: 0.5, quality: 1.2 }
 * @param {String} factorKey - The factor to assign percentiles for (e.g. 'value')
 */
function assignPercentiles(raw, factorKey) {
  const rawByTicker = {};
  for (const r of raw) {
    if (r[factorKey] !== null && r[factorKey] !== undefined) {
      rawByTicker[r.ticker] = r[factorKey];
    }
  }

  // Uses winsorization and z-scoring from factors.js
  const zScores = standardizeCrossSection(rawByTicker, false);

  // Convert z-scores to percentiles relative to the universe
  const validTickers = Object.keys(zScores);
  const sortedZScores = validTickers.map(t => zScores[t]).sort((a, b) => a - b);
  
  const n = sortedZScores.length;

  for (const r of raw) {
    const z = zScores[r.ticker];
    if (z === undefined) {
      r[`${factorKey}Percentile`] = null;
      continue;
    }
    
    // Find percentile rank
    let index = sortedZScores.findIndex(val => val >= z);
    if (index === -1) index = n - 1;
    
    r[`${factorKey}Percentile`] = Math.round((index / n) * 100);
  }
}

/**
 * Generates the descriptive Factor Profile for the entire universe based on CURRENT data.
 * Does NOT generate forward predictions.
 */
let _cachedRawProfiles = null;
let _cachedZScoreMaps = null;
let _cachedRawTime = 0;

export async function buildFactorProfiles(injectedUniverseRecords = null) {
  const rawProfiles = [];
  
  // Allow test injection
  if (injectedUniverseRecords) {
    for (const data of injectedUniverseRecords) {
      rawProfiles.push({
        ticker: data.ticker || 'TEST',
        value: FACTORS.value(data),
        quality: FACTORS.quality(data),
        lowVol: FACTORS.lowVol(data),
        momentum: FACTORS.momentum(data)
      });
    }
  } else {
    const universe = getUniverse().filter(s => !s.isETF).map(s => s.symbol);
    for (const ticker of universe) {
      try {
        const data = await pullRawRecord(ticker);
        if (!data) continue;

        rawProfiles.push({
          ticker,
          value: FACTORS.value(data),
          quality: FACTORS.quality(data),
          lowVol: FACTORS.lowVol(data),
          momentum: FACTORS.momentum(data)
        });
      } catch (e) {
        console.warn(`[DescriptiveLens] Failed for ${ticker}`);
      }
    }
  }

  // Composite is the equal-weight average of the non-null standardized z-scores
  // First, we must standardize the individual factors to compute composite z-score
  const factors = ['value', 'quality', 'lowVol', 'momentum'];
  const zScoreMaps = {};
  
  for (const f of factors) {
    const rawByTicker = {};
    for (const r of rawProfiles) {
      if (r[f] !== null && r[f] !== undefined) rawByTicker[r.ticker] = r[f];
    }
    zScoreMaps[f] = standardizeCrossSection(rawByTicker, false);
  }

  // Compute composite raw score
  for (const r of rawProfiles) {
    let sum = 0;
    let count = 0;
    for (const f of factors) {
      const z = zScoreMaps[f][r.ticker];
      if (z !== undefined) {
        sum += z;
        count++;
      }
    }
    r.composite = count > 0 ? (sum / count) : null;
  }

  // Cross-sectional standardization and percentile assignment
  [...factors, 'composite'].forEach(f => assignPercentiles(rawProfiles, f));

  // Cache the raw profiles for single-stock comparisons
  if (!injectedUniverseRecords) {
    _cachedRawProfiles = rawProfiles;
    _cachedZScoreMaps = zScoreMaps;
    _cachedRawTime = Date.now();
  }

  // If this is a test injection, return the raw profiles with percentiles for testing
  if (injectedUniverseRecords) return rawProfiles.reduce((acc, p) => { acc[p.ticker] = p; return acc; }, {});

  // Clean up the raw scores, we only return the percentiles for the UI
  const cleanedProfiles = rawProfiles.map(p => ({
    ticker: p.ticker,
    value: p.valuePercentile,
    quality: p.qualityPercentile,
    lowVol: p.lowVolPercentile,
    momentum: p.momentumPercentile,
    composite: p.compositePercentile,
    _disclaimer: "Descriptive only. Shows where this stock ranks vs. peers on each factor today. We have not validated that these ranks predict returns — and our own testing showed standard technical signals do not. We'll only call a factor predictive after it passes out-of-sample validation."
  }));

  return cleanedProfiles;
}

/**
 * Computes a single stock's factor profile relative to the cached universe.
 */
export async function getSingleFactorProfile(symbol) {
  if (!_cachedRawProfiles || (Date.now() - _cachedRawTime > 120000)) {
    await buildFactorProfiles();
  }

  // If it's already in the universe, just return it
  const existing = _cachedRawProfiles.find(p => p.ticker === symbol);
  if (existing) {
    return {
      ticker: existing.ticker,
      value: existing.valuePercentile,
      quality: existing.qualityPercentile,
      lowVol: existing.lowVolPercentile,
      momentum: existing.momentumPercentile,
      composite: existing.compositePercentile,
      _disclaimer: "Descriptive only. Shows where this stock ranks vs. the core universe."
    };
  }

  // Fetch out-of-universe data
  const data = await pullRawRecord(symbol);
  if (!data) return null;

  const raw = {
    ticker: symbol,
    value: FACTORS.value(data),
    quality: FACTORS.quality(data),
    lowVol: FACTORS.lowVol(data),
    momentum: FACTORS.momentum(data)
  };

  // Standardize single value using universe mean/std from zScoreMaps... wait, standardizer only gives cross-sectional z-scores.
  // We can just re-run standardizer with the universe + this one stock.
  const tempRaw = [..._cachedRawProfiles, raw];
  
  const factors = ['value', 'quality', 'lowVol', 'momentum'];
  const zScoreMaps = {};
  
  for (const f of factors) {
    const rawByTicker = {};
    for (const r of tempRaw) {
      if (r[f] !== null && r[f] !== undefined) rawByTicker[r.ticker] = r[f];
    }
    zScoreMaps[f] = standardizeCrossSection(rawByTicker, false);
  }

  let sum = 0;
  let count = 0;
  for (const f of factors) {
    const z = zScoreMaps[f][raw.ticker];
    if (z !== undefined) {
      sum += z;
      count++;
    }
  }
  raw.composite = count > 0 ? (sum / count) : null;

  [...factors, 'composite'].forEach(f => assignPercentiles(tempRaw, f));

  return {
    ticker: raw.ticker,
    value: raw.valuePercentile,
    quality: raw.qualityPercentile,
    lowVol: raw.lowVolPercentile,
    momentum: raw.momentumPercentile,
    composite: raw.compositePercentile,
    _disclaimer: "Descriptive only. Shows where this out-of-universe stock ranks vs. the core universe today."
  };
}