diff --git a/packages/loopover-engine/src/calibration/backtest-compare.ts b/packages/loopover-engine/src/calibration/backtest-compare.ts new file mode 100644 index 0000000000..b8152b9c0b --- /dev/null +++ b/packages/loopover-engine/src/calibration/backtest-compare.ts @@ -0,0 +1,52 @@ +// Pareto-floor comparator between two BacktestScoreReports (#8086) -- the dual-axis no-regression method: +// a candidate rule change may not regress on ANY measured axis even while improving another; "trading one +// axis for the other" is a regression, not a net win. This is deliberately NOT a weighted/averaged score -- +// a single regressed axis decides the verdict, which is the entire point of the floor. +// +// Same purity contract as the rest of this module family: no IO, no randomness, no wall-clock reads. + +import type { BacktestScoreReport } from "./backtest-score.js"; + +/** The two comparable axes of a {@link BacktestScoreReport}. */ +type ComparisonAxis = "precision" | "recall"; + +export type BacktestComparison = { + ruleId: string; + baseline: BacktestScoreReport; + candidate: BacktestScoreReport; + regressedAxes: Array<"precision" | "recall">; + improvedAxes: Array<"precision" | "recall">; + verdict: "improved" | "regressed" | "unchanged"; +}; + +/** + * Compare a candidate rule change's backtest score against its baseline under the Pareto-floor rule: an + * axis regresses when the candidate's value is strictly below the baseline's, improves when strictly above, + * and is excluded from BOTH lists when either side is null (insufficient decided data is never treated as 0 + * or as "no change" -- the same "unknown stays unknown" discipline the reports themselves use). The verdict + * is "regressed" whenever ANY axis regressed -- even if the other axis improved -- else "improved" when any + * axis improved, else "unchanged". Throws when the two reports describe different rules: that is a caller + * bug, not a valid comparison. + */ +export function compareBacktestScores(baseline: BacktestScoreReport, candidate: BacktestScoreReport): BacktestComparison { + if (baseline.ruleId !== candidate.ruleId) { + throw new Error(`cannot compare backtest scores for different rules: ${baseline.ruleId} vs ${candidate.ruleId}`); + } + const regressedAxes: ComparisonAxis[] = []; + const improvedAxes: ComparisonAxis[] = []; + for (const axis of ["precision", "recall"] as const) { + const baselineValue = baseline[axis]; + const candidateValue = candidate[axis]; + if (baselineValue === null || candidateValue === null) continue; + if (candidateValue < baselineValue) regressedAxes.push(axis); + else if (candidateValue > baselineValue) improvedAxes.push(axis); + } + return { + ruleId: baseline.ruleId, + baseline, + candidate, + regressedAxes, + improvedAxes, + verdict: regressedAxes.length > 0 ? "regressed" : improvedAxes.length > 0 ? "improved" : "unchanged", + }; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 6761786a77..51ab2d1714 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -165,6 +165,7 @@ export * from "./governor/chokepoint.js"; export * from "./calibration/signal-tracking.js"; export * from "./calibration/backtest-corpus.js"; export * from "./calibration/backtest-score.js"; +export * from "./calibration/backtest-compare.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/backtest-compare.test.ts b/packages/loopover-engine/test/backtest-compare.test.ts new file mode 100644 index 0000000000..adfbd7b3a5 --- /dev/null +++ b/packages/loopover-engine/test/backtest-compare.test.ts @@ -0,0 +1,63 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { compareBacktestScores, type BacktestScoreReport } from "../dist/index.js"; + +function report(overrides: Partial = {}): BacktestScoreReport { + return { + ruleId: "missing_linked_issue", + caseCount: 10, + truePositive: 4, + falsePositive: 2, + trueNegative: 3, + falseNegative: 1, + precision: 0.5, + recall: 0.5, + ...overrides, + }; +} + +test("barrel: the public entrypoint re-exports the Pareto-floor comparator (#8086)", () => { + assert.equal(typeof compareBacktestScores, "function"); +}); + +test("compareBacktestScores: both axes improving is an improved verdict with empty regressedAxes", () => { + const comparison = compareBacktestScores(report(), report({ precision: 0.7, recall: 0.6 })); + assert.deepEqual(comparison.improvedAxes, ["precision", "recall"]); + assert.deepEqual(comparison.regressedAxes, []); + assert.equal(comparison.verdict, "improved"); +}); + +test("compareBacktestScores: PARETO FLOOR -- one axis improving while the other regresses is a regressed verdict", () => { + const comparison = compareBacktestScores(report(), report({ precision: 0.9, recall: 0.3 })); + assert.deepEqual(comparison.improvedAxes, ["precision"]); + assert.deepEqual(comparison.regressedAxes, ["recall"]); + assert.equal(comparison.verdict, "regressed"); +}); + +test("compareBacktestScores: an axis with a null on either side is excluded from both lists", () => { + const nullBaseline = compareBacktestScores(report({ precision: null }), report({ precision: 0.9, recall: 0.6 })); + assert.deepEqual(nullBaseline.improvedAxes, ["recall"]); + assert.deepEqual(nullBaseline.regressedAxes, []); + assert.equal(nullBaseline.verdict, "improved"); + + const nullCandidate = compareBacktestScores(report(), report({ recall: null })); + assert.deepEqual(nullCandidate.improvedAxes, []); + assert.deepEqual(nullCandidate.regressedAxes, []); + assert.equal(nullCandidate.verdict, "unchanged"); +}); + +test("compareBacktestScores: equal non-null axes land in neither list and yield an unchanged verdict", () => { + const comparison = compareBacktestScores(report(), report()); + assert.deepEqual(comparison.improvedAxes, []); + assert.deepEqual(comparison.regressedAxes, []); + assert.equal(comparison.verdict, "unchanged"); + assert.equal(comparison.ruleId, "missing_linked_issue"); +}); + +test("compareBacktestScores: mismatched ruleIds throw, naming both rules", () => { + assert.throws( + () => compareBacktestScores(report(), report({ ruleId: "other_rule" })), + /cannot compare backtest scores for different rules: missing_linked_issue vs other_rule/, + ); +}); diff --git a/test/unit/backtest-compare-engine.test.ts b/test/unit/backtest-compare-engine.test.ts new file mode 100644 index 0000000000..9fe9e5caed --- /dev/null +++ b/test/unit/backtest-compare-engine.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; + +// Import the engine SOURCE directly (not the built dist) -- coverage.include lists +// packages/loopover-engine/src/**, so only a source-path import exercises the .ts these branches live in +// (the dist-importing twin in packages/loopover-engine/test/ covers the built barrel for the workspace +// suite). Same pattern as backtest-corpus-engine.test.ts / miner-deny-hook-synthesis.test.ts. +import { compareBacktestScores } from "../../packages/loopover-engine/src/calibration/backtest-compare"; +import type { BacktestScoreReport } from "../../packages/loopover-engine/src/calibration/backtest-score"; + +function report(overrides: Partial = {}): BacktestScoreReport { + return { + ruleId: "missing_linked_issue", + caseCount: 10, + truePositive: 4, + falsePositive: 2, + trueNegative: 3, + falseNegative: 1, + precision: 0.5, + recall: 0.5, + ...overrides, + }; +} + +describe("compareBacktestScores (#8086)", () => { + it("marks both-axes improvement as improved with empty regressedAxes", () => { + const comparison = compareBacktestScores(report(), report({ precision: 0.7, recall: 0.6 })); + expect(comparison.improvedAxes).toEqual(["precision", "recall"]); + expect(comparison.regressedAxes).toEqual([]); + expect(comparison.verdict).toBe("improved"); + expect(comparison.baseline.precision).toBe(0.5); + expect(comparison.candidate.precision).toBe(0.7); + }); + + it("PARETO FLOOR: one axis improving while the other regresses is a regressed verdict", () => { + const comparison = compareBacktestScores(report(), report({ precision: 0.9, recall: 0.3 })); + expect(comparison.improvedAxes).toEqual(["precision"]); + expect(comparison.regressedAxes).toEqual(["recall"]); + expect(comparison.verdict).toBe("regressed"); + }); + + it("marks a regression on both axes as regressed with empty improvedAxes", () => { + const comparison = compareBacktestScores(report(), report({ precision: 0.1, recall: 0.2 })); + expect(comparison.regressedAxes).toEqual(["precision", "recall"]); + expect(comparison.improvedAxes).toEqual([]); + expect(comparison.verdict).toBe("regressed"); + }); + + it("excludes an axis from both lists when either side is null -- null is never 0 and never 'no change'", () => { + const nullBaseline = compareBacktestScores(report({ precision: null }), report({ precision: 0.9, recall: 0.6 })); + expect(nullBaseline.improvedAxes).toEqual(["recall"]); + expect(nullBaseline.regressedAxes).toEqual([]); + expect(nullBaseline.verdict).toBe("improved"); + + const nullCandidate = compareBacktestScores(report(), report({ recall: null })); + expect(nullCandidate.improvedAxes).toEqual([]); + expect(nullCandidate.regressedAxes).toEqual([]); + expect(nullCandidate.verdict).toBe("unchanged"); + }); + + it("yields unchanged when every comparable axis is equal", () => { + const comparison = compareBacktestScores(report(), report()); + expect(comparison.improvedAxes).toEqual([]); + expect(comparison.regressedAxes).toEqual([]); + expect(comparison.verdict).toBe("unchanged"); + expect(comparison.ruleId).toBe("missing_linked_issue"); + }); + + it("throws on mismatched ruleIds, naming both rules in the message", () => { + expect(() => compareBacktestScores(report(), report({ ruleId: "other_rule" }))).toThrow( + "cannot compare backtest scores for different rules: missing_linked_issue vs other_rule", + ); + }); +});