diff --git a/packages/loopover-engine/src/calibration/backtest-track-record.ts b/packages/loopover-engine/src/calibration/backtest-track-record.ts new file mode 100644 index 0000000000..08decbdc45 --- /dev/null +++ b/packages/loopover-engine/src/calibration/backtest-track-record.ts @@ -0,0 +1,49 @@ +// REGRESSED-verdict track-record aggregation (#8140, parent epic #8082) -- turns the individual +// BacktestComparison results the advisory CI check persists (#8138) into the summary #8105's Phase-2 +// merge-gating decision actually needs: how often REGRESSED fired, per rule, against the totals. +// +// Pure, like everything in this module: no IO, no wall-clock reads. The thin CLI wrapper does the reading. + +import type { BacktestComparison } from "./backtest-compare.js"; + +export type RegressedVerdictRuleBreakdown = { + total: number; + regressed: number; + improved: number; + unchanged: number; +}; + +export type RegressedVerdictTrackRecord = { + totalRuns: number; + regressedRuns: number; + /** regressedRuns / totalRuns, or null when totalRuns is 0 -- the same "unknown stays unknown, never + * coerced to 0" discipline as BacktestScoreReport's own null rates. */ + regressedRate: number | null; + perRule: Map; +}; + +/** Aggregate historical comparisons into the #8105 decision summary. Verdict counting is exhaustive per + * comparison; per-rule buckets are keyed by each comparison's own ruleId. */ +export function computeRegressedVerdictTrackRecord(comparisons: readonly BacktestComparison[]): RegressedVerdictTrackRecord { + const perRule = new Map(); + let regressedRuns = 0; + for (const comparison of comparisons) { + const bucket = perRule.get(comparison.ruleId) ?? { total: 0, regressed: 0, improved: 0, unchanged: 0 }; + bucket.total += 1; + if (comparison.verdict === "regressed") { + bucket.regressed += 1; + regressedRuns += 1; + } else if (comparison.verdict === "improved") { + bucket.improved += 1; + } else { + bucket.unchanged += 1; + } + perRule.set(comparison.ruleId, bucket); + } + return { + totalRuns: comparisons.length, + regressedRuns, + regressedRate: comparisons.length > 0 ? regressedRuns / comparisons.length : null, + perRule, + }; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 1c3bd3e07b..40566f3b27 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -167,6 +167,7 @@ export * from "./calibration/backtest-corpus.js"; export * from "./calibration/backtest-score.js"; export * from "./calibration/backtest-compare.js"; export * from "./calibration/backtest-report.js"; +export * from "./calibration/backtest-track-record.js"; // #8087 shipped this file but never added its barrel export -- every existing consumer happened to import // it via the direct relative source path instead, so this was latent rather than broken. Fixing it here // since #8138 is the first consumer that actually needs the package-name import (@loopover/engine), the diff --git a/packages/loopover-engine/test/backtest-track-record.test.ts b/packages/loopover-engine/test/backtest-track-record.test.ts new file mode 100644 index 0000000000..d13868c090 --- /dev/null +++ b/packages/loopover-engine/test/backtest-track-record.test.ts @@ -0,0 +1,53 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { computeRegressedVerdictTrackRecord, compareBacktestScores, type BacktestComparison, type BacktestScoreReport } from "../dist/index.js"; + +// #8140: the REGRESSED-verdict track-record aggregation feeding #8105's Phase-2 merge-gating decision. + +function score(ruleId: string, overrides: Partial = {}): BacktestScoreReport { + return { ruleId, caseCount: 10, truePositive: 4, falsePositive: 1, trueNegative: 4, falseNegative: 1, precision: 0.8, recall: 0.8, ...overrides }; +} + +function comparison(ruleId: string, candidate: Partial): BacktestComparison { + return compareBacktestScores(score(ruleId), score(ruleId, candidate)); +} + +test("zero runs -> zero totals and a null rate (unknown stays unknown, never coerced to 0)", () => { + const record = computeRegressedVerdictTrackRecord([]); + assert.equal(record.totalRuns, 0); + assert.equal(record.regressedRuns, 0); + assert.equal(record.regressedRate, null); + assert.equal(record.perRule.size, 0); +}); + +test("all-clean runs -> regressedRuns 0 with a real 0 rate", () => { + const record = computeRegressedVerdictTrackRecord([comparison("a", { precision: 0.9 }), comparison("a", {})]); + assert.equal(record.totalRuns, 2); + assert.equal(record.regressedRuns, 0); + assert.equal(record.regressedRate, 0); + assert.deepEqual(record.perRule.get("a"), { total: 2, regressed: 0, improved: 1, unchanged: 1 }); +}); + +test("some-regressed runs -> counted in the totals and the rate", () => { + const record = computeRegressedVerdictTrackRecord([ + comparison("a", { precision: 0.6 }), + comparison("a", { precision: 0.9 }), + comparison("a", { recall: 0.5 }), + comparison("a", {}), + ]); + assert.equal(record.totalRuns, 4); + assert.equal(record.regressedRuns, 2); + assert.equal(record.regressedRate, 0.5); +}); + +test("per-ruleId breakdown separates more than one ruleId", () => { + const record = computeRegressedVerdictTrackRecord([ + comparison("rule_a", { precision: 0.6 }), + comparison("rule_a", { precision: 0.95 }), + comparison("rule_b", {}), + ]); + assert.deepEqual(record.perRule.get("rule_a"), { total: 2, regressed: 1, improved: 1, unchanged: 0 }); + assert.deepEqual(record.perRule.get("rule_b"), { total: 1, regressed: 0, improved: 0, unchanged: 1 }); + assert.equal(record.perRule.size, 2); +}); diff --git a/scripts/backtest-track-record.ts b/scripts/backtest-track-record.ts new file mode 100644 index 0000000000..742812ce0e --- /dev/null +++ b/scripts/backtest-track-record.ts @@ -0,0 +1,75 @@ +#!/usr/bin/env node +// Read-only D1 → REGRESSED-verdict track-record summary (#8140, epic #8082). Reads the BacktestComparison +// results the advisory backtest CI check persists (#8138) out of audit_events via `wrangler d1 execute +// --json`, aggregates them with the pure computeRegressedVerdictTrackRecord (@loopover/engine), and prints +// the summary #8105's Phase-2 merge-gating decision needs. The aggregation lives in the engine (pure, +// unit-tested); this file is the thin IO wrapper — mirrors backtest-corpus-export.ts's identical split. +// +// tsx scripts/backtest-track-record.ts --db loopover [--remote] +// +// --remote reads the deployed D1 (default is the local miniflare DB). NEVER pass a write command. +import { spawnSync } from "node:child_process"; +import { computeRegressedVerdictTrackRecord, type BacktestComparison } from "@loopover/engine"; + +// Mirrors THRESHOLD_BACKTEST_EVENT_TYPE in src/services/threshold-backtest-run.ts (#8138's writer) and must +// be kept in sync with it by hand — that module is Worker-bound (D1 repositories import graph) and +// deliberately not imported into this standalone script, the same posture backtest-corpus-export.ts takes +// toward signal-tracking-wire's private helpers. +const THRESHOLD_BACKTEST_EVENT_TYPE = "calibration.threshold_backtest_run"; + +type Args = { db: string | undefined; remote: boolean }; + +function parseArgs(argv: string[]): Args { + const args: Args = { db: undefined, remote: false }; + for (let i = 0; i < argv.length; i += 1) { + const flag = argv[i]; + if (flag === "--remote") args.remote = true; + else if (flag === "--db") args.db = argv[++i]; + } + return args; +} + +// Mirrors export-d1-data.ts's d1Query: read-only, fail-loud so a partial read never passes as a full record. +function d1Query(db: string, remote: boolean, sql: string): Array> { + const result = spawnSync("npx", ["wrangler", "d1", "execute", db, remote ? "--remote" : "--local", "--json", "--command", sql], { + encoding: "utf8", + maxBuffer: 256 * 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error(`wrangler d1 execute failed (${result.status}): ${(result.stderr || result.stdout || "").slice(0, 500)}`); + } + const parsed = JSON.parse(result.stdout); + const first = Array.isArray(parsed) ? parsed[0] : parsed; + return first?.results ?? []; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + if (!args.db) { + console.error("Usage: tsx scripts/backtest-track-record.ts --db [--remote]"); + process.exit(2); + } + const rows = d1Query( + args.db, + args.remote, + `SELECT metadata_json FROM audit_events WHERE event_type = '${THRESHOLD_BACKTEST_EVENT_TYPE}' ORDER BY created_at ASC`, + ); + const comparisons: BacktestComparison[] = []; + for (const row of rows) { + try { + const metadata: unknown = JSON.parse(typeof row.metadata_json === "string" ? row.metadata_json : "{}"); + const comparison = (metadata as { comparison?: BacktestComparison }).comparison; + if (comparison && typeof comparison === "object" && typeof comparison.ruleId === "string") comparisons.push(comparison); + } catch { + /* corrupt row -- skip, matching listAuditEventsByType's own fail-open metadata parse */ + } + } + const record = computeRegressedVerdictTrackRecord(comparisons); + console.log(`Backtest CI track record: ${record.totalRuns} run(s), ${record.regressedRuns} REGRESSED`); + console.log(`REGRESSED rate: ${record.regressedRate === null ? "N/A (no runs yet)" : record.regressedRate.toFixed(3)}`); + for (const [ruleId, bucket] of record.perRule) { + console.log(` ${ruleId}: total=${bucket.total} regressed=${bucket.regressed} improved=${bucket.improved} unchanged=${bucket.unchanged}`); + } +} + +main(); diff --git a/test/unit/backtest-track-record.test.ts b/test/unit/backtest-track-record.test.ts new file mode 100644 index 0000000000..5da54b5e2a --- /dev/null +++ b/test/unit/backtest-track-record.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +// Direct src-path import — the coverage-twin pattern this calibration module's merged tests established +// (the engine's node:test suite runs against dist/, outside root vitest's coverage instrumentation). +import { compareBacktestScores } from "../../packages/loopover-engine/src/calibration/backtest-compare.js"; +import type { BacktestScoreReport } from "../../packages/loopover-engine/src/calibration/backtest-score.js"; +import { computeRegressedVerdictTrackRecord } from "../../packages/loopover-engine/src/calibration/backtest-track-record.js"; + +function score(ruleId: string, overrides: Partial = {}): BacktestScoreReport { + return { ruleId, caseCount: 10, truePositive: 4, falsePositive: 1, trueNegative: 4, falseNegative: 1, precision: 0.8, recall: 0.8, ...overrides }; +} + +function comparison(ruleId: string, candidate: Partial) { + return compareBacktestScores(score(ruleId), score(ruleId, candidate)); +} + +describe("computeRegressedVerdictTrackRecord (#8140)", () => { + it("returns zero totals and a null rate for zero runs", () => { + expect(computeRegressedVerdictTrackRecord([])).toEqual({ totalRuns: 0, regressedRuns: 0, regressedRate: null, perRule: new Map() }); + }); + + it("counts all-clean runs with a real 0 rate and full per-rule buckets", () => { + const record = computeRegressedVerdictTrackRecord([comparison("a", { precision: 0.9 }), comparison("a", {})]); + expect(record).toMatchObject({ totalRuns: 2, regressedRuns: 0, regressedRate: 0 }); + expect(record.perRule.get("a")).toEqual({ total: 2, regressed: 0, improved: 1, unchanged: 1 }); + }); + + it("counts regressed runs into totals and the rate", () => { + const record = computeRegressedVerdictTrackRecord([ + comparison("a", { precision: 0.6 }), + comparison("a", { precision: 0.9 }), + comparison("a", { recall: 0.5 }), + comparison("a", {}), + ]); + expect(record).toMatchObject({ totalRuns: 4, regressedRuns: 2, regressedRate: 0.5 }); + }); + + it("separates the per-ruleId breakdown across multiple rules", () => { + const record = computeRegressedVerdictTrackRecord([ + comparison("rule_a", { precision: 0.6 }), + comparison("rule_a", { precision: 0.95 }), + comparison("rule_b", {}), + ]); + expect(record.perRule.get("rule_a")).toEqual({ total: 2, regressed: 1, improved: 1, unchanged: 0 }); + expect(record.perRule.get("rule_b")).toEqual({ total: 1, regressed: 0, improved: 0, unchanged: 1 }); + }); +});