32 lines
1.2 KiB
JavaScript
32 lines
1.2 KiB
JavaScript
export function calibration(testResults) {
|
|
// We bucket predicted probabilities into deciles (0-10%, 10-20%, etc)
|
|
const buckets = Array(10).fill(null).map(() => ({ total: 0, wins: 0, sumProb: 0 }));
|
|
|
|
for (const r of testResults) {
|
|
if (r.prob === undefined || r.prob === null) continue;
|
|
let bIdx = Math.floor(r.prob * 10);
|
|
if (bIdx > 9) bIdx = 9; // Handle p=1.0 edge case
|
|
if (bIdx < 0) bIdx = 0;
|
|
|
|
buckets[bIdx].total++;
|
|
buckets[bIdx].sumProb += r.prob;
|
|
if (r.outcome === 'WIN') buckets[bIdx].wins++;
|
|
}
|
|
|
|
console.log(`\n=== CALIBRATION TABLE ===`);
|
|
console.log(`Bucket\t\tN\tPred P(Win)\tActual Win%\tDiff`);
|
|
console.log(`------------------------------------------------------------------`);
|
|
|
|
for (let i = 0; i < 10; i++) {
|
|
const b = buckets[i];
|
|
if (b.total === 0) continue;
|
|
|
|
const label = `${(i * 10).toString().padStart(2, '0')}-${((i + 1) * 10).toString().padStart(2, '0')}%`;
|
|
const predP = (b.sumProb / b.total) * 100;
|
|
const actP = (b.wins / b.total) * 100;
|
|
const diff = actP - predP;
|
|
|
|
console.log(`[${label}]\t${b.total}\t${predP.toFixed(1)}%\t\t${actP.toFixed(1)}%\t\t${diff > 0 ? '+' : ''}${diff.toFixed(1)}pp`);
|
|
}
|
|
}
|