diff --git a/packages/loopover-engine/src/calibration/backtest-report.ts b/packages/loopover-engine/src/calibration/backtest-report.ts new file mode 100644 index 0000000000..0baddb406b --- /dev/null +++ b/packages/loopover-engine/src/calibration/backtest-report.ts @@ -0,0 +1,68 @@ +// Markdown renderers for backtest results (#8088) -- the human-readable "receipt" a maintainer (and, per +// the parent epic, eventually an advisory CI comment) reads for a BacktestScoreReport (#8085) or a +// BacktestComparison (#8086). Deterministic pure functions producing stable Markdown, not ad-hoc console +// logging: byte-identical input always renders byte-identical output. +// +// Same purity contract as the rest of this module family: no IO, no randomness, no wall-clock reads. + +import type { BacktestComparison } from "./backtest-compare.js"; +import type { BacktestScoreReport } from "./backtest-score.js"; + +/** Render a nullable ratio for display: `null` is `N/A` -- never `0`, the word `null`, or an empty cell + * (the same null-is-not-zero discipline BacktestScoreReport itself establishes). */ +function renderRatio(value: number | null): string { + return value === null ? "N/A" : String(value); +} + +/** + * Render one {@link BacktestScoreReport} as a Markdown table: the rule ID as a heading, then every count + * and both (nullable) ratios. Pure string-in/string-out; the exact layout is pinned by a snapshot test. + */ +export function renderBacktestScoreReport(report: BacktestScoreReport): string { + return [ + `### Backtest score — \`${report.ruleId}\``, + "", + "| Metric | Value |", + "| --- | --- |", + `| Cases scored | ${report.caseCount} |`, + `| True positives | ${report.truePositive} |`, + `| False positives | ${report.falsePositive} |`, + `| True negatives | ${report.trueNegative} |`, + `| False negatives | ${report.falseNegative} |`, + `| Precision | ${renderRatio(report.precision)} |`, + `| Recall | ${renderRatio(report.recall)} |`, + ].join("\n"); +} + +/** + * Render one {@link BacktestComparison} as Markdown: the rule ID as a heading, a "Regressed" section for + * every regressed axis, a visually separate "Improved" section for every improved axis (an axis can only + * ever appear under its own section -- the two lists are disjoint by construction upstream), and a closing + * verdict line. The `"regressed"` closing line contains the literal word `REGRESSED` and states the change + * should not be merged -- pinned wording, so a future automated consumer can detect the regressed case by + * string match without re-implementing the comparison logic. Sections with no axes render as "(none)" + * rather than listing anything, so an empty regression list can never read as if something regressed. + */ +export function renderBacktestComparison(comparison: BacktestComparison): string { + const axisLines = (axes: ReadonlyArray<"precision" | "recall">): string[] => + axes.length === 0 ? ["- (none)"] : axes.map((axis) => `- ${axis}`); + const verdictLine = + comparison.verdict === "regressed" + ? "Verdict: REGRESSED — do not merge" + : comparison.verdict === "improved" + ? "Verdict: improved" + : "Verdict: unchanged"; + return [ + `### Backtest comparison — \`${comparison.ruleId}\``, + "", + "**Regressed**", + "", + ...axisLines(comparison.regressedAxes), + "", + "**Improved**", + "", + ...axisLines(comparison.improvedAxes), + "", + verdictLine, + ].join("\n"); +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 51ab2d1714..a36a7e8410 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -166,6 +166,7 @@ export * from "./calibration/signal-tracking.js"; export * from "./calibration/backtest-corpus.js"; export * from "./calibration/backtest-score.js"; export * from "./calibration/backtest-compare.js"; +export * from "./calibration/backtest-report.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/backtest-report.test.ts b/packages/loopover-engine/test/backtest-report.test.ts new file mode 100644 index 0000000000..8a53a10f70 --- /dev/null +++ b/packages/loopover-engine/test/backtest-report.test.ts @@ -0,0 +1,94 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { + renderBacktestComparison, + renderBacktestScoreReport, + type BacktestComparison, + type BacktestScoreReport, +} from "../dist/index.js"; + +function report(overrides: Partial = {}): BacktestScoreReport { + return { + ruleId: "missing_linked_issue", + caseCount: 4, + truePositive: 1, + falsePositive: 1, + trueNegative: 1, + falseNegative: 1, + precision: 0.5, + recall: 0.5, + ...overrides, + }; +} + +function comparison(overrides: Partial = {}): BacktestComparison { + return { + ruleId: "missing_linked_issue", + baseline: report(), + candidate: report({ precision: 0.75 }), + regressedAxes: [], + improvedAxes: ["precision"], + verdict: "improved", + ...overrides, + }; +} + +test("renderBacktestScoreReport: snapshot -- a non-null report renders every count and both ratios", () => { + assert.equal( + renderBacktestScoreReport(report()), + [ + "### Backtest score — `missing_linked_issue`", + "", + "| Metric | Value |", + "| --- | --- |", + "| Cases scored | 4 |", + "| True positives | 1 |", + "| False positives | 1 |", + "| True negatives | 1 |", + "| False negatives | 1 |", + "| Precision | 0.5 |", + "| Recall | 0.5 |", + ].join("\n"), + ); +}); + +test("renderBacktestScoreReport: null precision/recall render as N/A, never 0, null, or an empty cell", () => { + const rendered = renderBacktestScoreReport(report({ precision: null, recall: null })); + assert.match(rendered, /\| Precision \| N\/A \|/); + assert.match(rendered, /\| Recall \| N\/A \|/); + assert.doesNotMatch(rendered, /\| Precision \| (0|null)? \|/); + assert.doesNotMatch(rendered, /\| Recall \| (0|null)? \|/); +}); + +test("renderBacktestComparison: a regressed comparison names the regressed axis under Regressed and closes with the literal REGRESSED wording", () => { + const rendered = renderBacktestComparison( + comparison({ regressedAxes: ["recall"], improvedAxes: ["precision"], verdict: "regressed" }), + ); + assert.match(rendered, /\*\*Regressed\*\*\n\n- recall/); + assert.match(rendered, /\*\*Improved\*\*\n\n- precision/); + assert.match(rendered, /Verdict: REGRESSED — do not merge/); +}); + +test("renderBacktestComparison: an improved comparison claims no regressed axis", () => { + const rendered = renderBacktestComparison(comparison()); + assert.match(rendered, /\*\*Regressed\*\*\n\n- \(none\)/); + assert.match(rendered, /\*\*Improved\*\*\n\n- precision/); + assert.match(rendered, /Verdict: improved/); + assert.doesNotMatch(rendered, /REGRESSED/); +}); + +test("renderBacktestComparison: an unchanged comparison lists no axis on either side", () => { + const rendered = renderBacktestComparison( + comparison({ improvedAxes: [], verdict: "unchanged", candidate: report() }), + ); + assert.match(rendered, /\*\*Regressed\*\*\n\n- \(none\)/); + assert.match(rendered, /\*\*Improved\*\*\n\n- \(none\)/); + assert.match(rendered, /Verdict: unchanged/); +}); + +test("both renderers are deterministic: identical input renders byte-identical output", () => { + assert.equal(renderBacktestScoreReport(report()), renderBacktestScoreReport(report())); + const regressed = comparison({ regressedAxes: ["recall"], verdict: "regressed" }); + assert.equal(renderBacktestComparison(regressed), renderBacktestComparison(regressed)); +}); diff --git a/test/unit/backtest-report-engine.test.ts b/test/unit/backtest-report-engine.test.ts new file mode 100644 index 0000000000..05a4a10074 --- /dev/null +++ b/test/unit/backtest-report-engine.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from "vitest"; + +// Import the engine SOURCE directly (not the built dist) -- coverage.include lists +// packages/loopover-engine/src/**, so only a source-path import exercises the .ts these branches live in +// (the dist-importing twin in packages/loopover-engine/test/ covers the built barrel for the workspace +// suite). Same pattern as backtest-corpus-engine.test.ts / miner-deny-hook-synthesis.test.ts. +import { + renderBacktestComparison, + renderBacktestScoreReport, +} from "../../packages/loopover-engine/src/calibration/backtest-report"; +import type { BacktestComparison } from "../../packages/loopover-engine/src/calibration/backtest-compare"; +import type { BacktestScoreReport } from "../../packages/loopover-engine/src/calibration/backtest-score"; + +function report(overrides: Partial = {}): BacktestScoreReport { + return { + ruleId: "missing_linked_issue", + caseCount: 4, + truePositive: 1, + falsePositive: 1, + trueNegative: 1, + falseNegative: 1, + precision: 0.5, + recall: 0.5, + ...overrides, + }; +} + +function comparison(overrides: Partial = {}): BacktestComparison { + return { + ruleId: "missing_linked_issue", + baseline: report(), + candidate: report({ precision: 0.75 }), + regressedAxes: [], + improvedAxes: ["precision"], + verdict: "improved", + ...overrides, + }; +} + +describe("renderBacktestScoreReport (#8088)", () => { + it("renders the exact snapshot for a non-null report", () => { + expect(renderBacktestScoreReport(report())).toBe( + [ + "### Backtest score — `missing_linked_issue`", + "", + "| Metric | Value |", + "| --- | --- |", + "| Cases scored | 4 |", + "| True positives | 1 |", + "| False positives | 1 |", + "| True negatives | 1 |", + "| False negatives | 1 |", + "| Precision | 0.5 |", + "| Recall | 0.5 |", + ].join("\n"), + ); + }); + + it("renders null precision/recall as N/A -- never 0, null, or an empty cell", () => { + const rendered = renderBacktestScoreReport(report({ precision: null, recall: null })); + expect(rendered).toContain("| Precision | N/A |"); + expect(rendered).toContain("| Recall | N/A |"); + expect(rendered).not.toContain("| Precision | 0 |"); + expect(rendered).not.toContain("null"); + }); + + it("is deterministic for identical input", () => { + expect(renderBacktestScoreReport(report())).toBe(renderBacktestScoreReport(report())); + }); +}); + +describe("renderBacktestComparison (#8088)", () => { + it("puts each axis under its own section and pins the literal REGRESSED do-not-merge wording", () => { + const rendered = renderBacktestComparison( + comparison({ regressedAxes: ["recall"], improvedAxes: ["precision"], verdict: "regressed" }), + ); + expect(rendered).toContain("**Regressed**\n\n- recall"); + expect(rendered).toContain("**Improved**\n\n- precision"); + expect(rendered).toContain("Verdict: REGRESSED — do not merge"); + }); + + it("claims no regressed axis for an improved comparison", () => { + const rendered = renderBacktestComparison(comparison()); + expect(rendered).toContain("**Regressed**\n\n- (none)"); + expect(rendered).toContain("**Improved**\n\n- precision"); + expect(rendered).toContain("Verdict: improved"); + expect(rendered).not.toContain("REGRESSED"); + }); + + it("lists no axis on either side for an unchanged comparison", () => { + const rendered = renderBacktestComparison( + comparison({ improvedAxes: [], verdict: "unchanged", candidate: report() }), + ); + expect(rendered).toContain("**Regressed**\n\n- (none)"); + expect(rendered).toContain("**Improved**\n\n- (none)"); + expect(rendered).toContain("Verdict: unchanged"); + }); + + it("is deterministic for identical input", () => { + const regressed = comparison({ regressedAxes: ["recall"], verdict: "regressed" }); + expect(renderBacktestComparison(regressed)).toBe(renderBacktestComparison(regressed)); + }); +});