feat(engine): aggregate REGRESSED-verdict track-record statistics from persisted backtest CI results

packages/loopover-engine/src/calibration/backtest-track-record.ts

@@ -0,0 +1,49 @@
+// REGRESSED-verdict track-record aggregation (#8140, parent epic #8082) -- turns the individual
+// BacktestComparison results the advisory CI check persists (#8138) into the summary #8105's Phase-2
+// merge-gating decision actually needs: how often REGRESSED fired, per rule, against the totals.
+//
+// Pure, like everything in this module: no IO, no wall-clock reads. The thin CLI wrapper does the reading.
+
+import type { BacktestComparison } from "./backtest-compare.js";
+
+export type RegressedVerdictRuleBreakdown = {
+  total: number;
+  regressed: number;
+  improved: number;
+  unchanged: number;
+};
+
+export type RegressedVerdictTrackRecord = {
+  totalRuns: number;
+  regressedRuns: number;
+  /** regressedRuns / totalRuns, or null when totalRuns is 0 -- the same "unknown stays unknown, never
+   *  coerced to 0" discipline as BacktestScoreReport's own null rates. */
+  regressedRate: number | null;
+  perRule: Map<string, RegressedVerdictRuleBreakdown>;
+};
+
+/** Aggregate historical comparisons into the #8105 decision summary. Verdict counting is exhaustive per
+ *  comparison; per-rule buckets are keyed by each comparison's own ruleId. */
+export function computeRegressedVerdictTrackRecord(comparisons: readonly BacktestComparison[]): RegressedVerdictTrackRecord {
+  const perRule = new Map<string, RegressedVerdictRuleBreakdown>();
+  let regressedRuns = 0;
+  for (const comparison of comparisons) {
+    const bucket = perRule.get(comparison.ruleId) ?? { total: 0, regressed: 0, improved: 0, unchanged: 0 };
+    bucket.total += 1;
+    if (comparison.verdict === "regressed") {
+      bucket.regressed += 1;
+      regressedRuns += 1;
+    } else if (comparison.verdict === "improved") {
+      bucket.improved += 1;
+    } else {
+      bucket.unchanged += 1;
+    }
+    perRule.set(comparison.ruleId, bucket);
+  }
+  return {
+    totalRuns: comparisons.length,
+    regressedRuns,
+    regressedRate: comparisons.length > 0 ? regressedRuns / comparisons.length : null,
+    perRule,
+  };
+}

packages/loopover-engine/src/index.ts

@@ -167,6 +167,7 @@ export * from "./calibration/backtest-corpus.js";
 export * from "./calibration/backtest-score.js";
 export * from "./calibration/backtest-compare.js";
 export * from "./calibration/backtest-report.js";
+export * from "./calibration/backtest-track-record.js";
 // #8087 shipped this file but never added its barrel export -- every existing consumer happened to import
 // it via the direct relative source path instead, so this was latent rather than broken. Fixing it here
 // since #8138 is the first consumer that actually needs the package-name import (@loopover/engine), the

packages/loopover-engine/test/backtest-track-record.test.ts

@@ -0,0 +1,53 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { computeRegressedVerdictTrackRecord, compareBacktestScores, type BacktestComparison, type BacktestScoreReport } from "../dist/index.js";
+
+// #8140: the REGRESSED-verdict track-record aggregation feeding #8105's Phase-2 merge-gating decision.
+
+function score(ruleId: string, overrides: Partial<BacktestScoreReport> = {}): BacktestScoreReport {
+  return { ruleId, caseCount: 10, truePositive: 4, falsePositive: 1, trueNegative: 4, falseNegative: 1, precision: 0.8, recall: 0.8, ...overrides };
+}
+
+function comparison(ruleId: string, candidate: Partial<BacktestScoreReport>): BacktestComparison {
+  return compareBacktestScores(score(ruleId), score(ruleId, candidate));
+}
+
+test("zero runs -> zero totals and a null rate (unknown stays unknown, never coerced to 0)", () => {
+  const record = computeRegressedVerdictTrackRecord([]);
+  assert.equal(record.totalRuns, 0);
+  assert.equal(record.regressedRuns, 0);
+  assert.equal(record.regressedRate, null);
+  assert.equal(record.perRule.size, 0);
+});
+
+test("all-clean runs -> regressedRuns 0 with a real 0 rate", () => {
+  const record = computeRegressedVerdictTrackRecord([comparison("a", { precision: 0.9 }), comparison("a", {})]);
+  assert.equal(record.totalRuns, 2);
+  assert.equal(record.regressedRuns, 0);
+  assert.equal(record.regressedRate, 0);
+  assert.deepEqual(record.perRule.get("a"), { total: 2, regressed: 0, improved: 1, unchanged: 1 });
+});
+
+test("some-regressed runs -> counted in the totals and the rate", () => {
+  const record = computeRegressedVerdictTrackRecord([
+    comparison("a", { precision: 0.6 }),
+    comparison("a", { precision: 0.9 }),
+    comparison("a", { recall: 0.5 }),
+    comparison("a", {}),
+  ]);
+  assert.equal(record.totalRuns, 4);
+  assert.equal(record.regressedRuns, 2);
+  assert.equal(record.regressedRate, 0.5);
+});
+
+test("per-ruleId breakdown separates more than one ruleId", () => {
+  const record = computeRegressedVerdictTrackRecord([
+    comparison("rule_a", { precision: 0.6 }),
+    comparison("rule_a", { precision: 0.95 }),
+    comparison("rule_b", {}),
+  ]);
+  assert.deepEqual(record.perRule.get("rule_a"), { total: 2, regressed: 1, improved: 1, unchanged: 0 });
+  assert.deepEqual(record.perRule.get("rule_b"), { total: 1, regressed: 0, improved: 0, unchanged: 1 });
+  assert.equal(record.perRule.size, 2);
+});

scripts/backtest-track-record.ts

@@ -0,0 +1,75 @@
+#!/usr/bin/env node
+// Read-only D1 → REGRESSED-verdict track-record summary (#8140, epic #8082). Reads the BacktestComparison
+// results the advisory backtest CI check persists (#8138) out of audit_events via `wrangler d1 execute
+// --json`, aggregates them with the pure computeRegressedVerdictTrackRecord (@loopover/engine), and prints
+// the summary #8105's Phase-2 merge-gating decision needs. The aggregation lives in the engine (pure,
+// unit-tested); this file is the thin IO wrapper — mirrors backtest-corpus-export.ts's identical split.
+//
+//   tsx scripts/backtest-track-record.ts --db loopover [--remote]
+//
+// --remote reads the deployed D1 (default is the local miniflare DB). NEVER pass a write command.
+import { spawnSync } from "node:child_process";
+import { computeRegressedVerdictTrackRecord, type BacktestComparison } from "@loopover/engine";
+
+// Mirrors THRESHOLD_BACKTEST_EVENT_TYPE in src/services/threshold-backtest-run.ts (#8138's writer) and must
+// be kept in sync with it by hand — that module is Worker-bound (D1 repositories import graph) and
+// deliberately not imported into this standalone script, the same posture backtest-corpus-export.ts takes
+// toward signal-tracking-wire's private helpers.
+const THRESHOLD_BACKTEST_EVENT_TYPE = "calibration.threshold_backtest_run";
+
+type Args = { db: string | undefined; remote: boolean };
+
+function parseArgs(argv: string[]): Args {
+  const args: Args = { db: undefined, remote: false };
+  for (let i = 0; i < argv.length; i += 1) {
+    const flag = argv[i];
+    if (flag === "--remote") args.remote = true;
+    else if (flag === "--db") args.db = argv[++i];
+  }
+  return args;
+}
+
+// Mirrors export-d1-data.ts's d1Query: read-only, fail-loud so a partial read never passes as a full record.
+function d1Query(db: string, remote: boolean, sql: string): Array<Record<string, unknown>> {
+  const result = spawnSync("npx", ["wrangler", "d1", "execute", db, remote ? "--remote" : "--local", "--json", "--command", sql], {
+    encoding: "utf8",
+    maxBuffer: 256 * 1024 * 1024,
+  });
+  if (result.status !== 0) {
+    throw new Error(`wrangler d1 execute failed (${result.status}): ${(result.stderr || result.stdout || "").slice(0, 500)}`);
+  }
+  const parsed = JSON.parse(result.stdout);
+  const first = Array.isArray(parsed) ? parsed[0] : parsed;
+  return first?.results ?? [];
+}
+
+function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (!args.db) {
+    console.error("Usage: tsx scripts/backtest-track-record.ts --db <database> [--remote]");
+    process.exit(2);
+  }
+  const rows = d1Query(
+    args.db,
+    args.remote,
+    `SELECT metadata_json FROM audit_events WHERE event_type = '${THRESHOLD_BACKTEST_EVENT_TYPE}' ORDER BY created_at ASC`,
+  );
+  const comparisons: BacktestComparison[] = [];
+  for (const row of rows) {
+    try {
+      const metadata: unknown = JSON.parse(typeof row.metadata_json === "string" ? row.metadata_json : "{}");
+      const comparison = (metadata as { comparison?: BacktestComparison }).comparison;
+      if (comparison && typeof comparison === "object" && typeof comparison.ruleId === "string") comparisons.push(comparison);
+    } catch {
+      /* corrupt row -- skip, matching listAuditEventsByType's own fail-open metadata parse */
+    }
+  }
+  const record = computeRegressedVerdictTrackRecord(comparisons);
+  console.log(`Backtest CI track record: ${record.totalRuns} run(s), ${record.regressedRuns} REGRESSED`);
+  console.log(`REGRESSED rate: ${record.regressedRate === null ? "N/A (no runs yet)" : record.regressedRate.toFixed(3)}`);
+  for (const [ruleId, bucket] of record.perRule) {
+    console.log(`  ${ruleId}: total=${bucket.total} regressed=${bucket.regressed} improved=${bucket.improved} unchanged=${bucket.unchanged}`);
+  }
+}
+
+main();

test/unit/backtest-track-record.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, it } from "vitest";
+// Direct src-path import — the coverage-twin pattern this calibration module's merged tests established
+// (the engine's node:test suite runs against dist/, outside root vitest's coverage instrumentation).
+import { compareBacktestScores } from "../../packages/loopover-engine/src/calibration/backtest-compare.js";
+import type { BacktestScoreReport } from "../../packages/loopover-engine/src/calibration/backtest-score.js";
+import { computeRegressedVerdictTrackRecord } from "../../packages/loopover-engine/src/calibration/backtest-track-record.js";
+
+function score(ruleId: string, overrides: Partial<BacktestScoreReport> = {}): BacktestScoreReport {
+  return { ruleId, caseCount: 10, truePositive: 4, falsePositive: 1, trueNegative: 4, falseNegative: 1, precision: 0.8, recall: 0.8, ...overrides };
+}
+
+function comparison(ruleId: string, candidate: Partial<BacktestScoreReport>) {
+  return compareBacktestScores(score(ruleId), score(ruleId, candidate));
+}
+
+describe("computeRegressedVerdictTrackRecord (#8140)", () => {
+  it("returns zero totals and a null rate for zero runs", () => {
+    expect(computeRegressedVerdictTrackRecord([])).toEqual({ totalRuns: 0, regressedRuns: 0, regressedRate: null, perRule: new Map() });
+  });
+
+  it("counts all-clean runs with a real 0 rate and full per-rule buckets", () => {
+    const record = computeRegressedVerdictTrackRecord([comparison("a", { precision: 0.9 }), comparison("a", {})]);
+    expect(record).toMatchObject({ totalRuns: 2, regressedRuns: 0, regressedRate: 0 });
+    expect(record.perRule.get("a")).toEqual({ total: 2, regressed: 0, improved: 1, unchanged: 1 });
+  });
+
+  it("counts regressed runs into totals and the rate", () => {
+    const record = computeRegressedVerdictTrackRecord([
+      comparison("a", { precision: 0.6 }),
+      comparison("a", { precision: 0.9 }),
+      comparison("a", { recall: 0.5 }),
+      comparison("a", {}),
+    ]);
+    expect(record).toMatchObject({ totalRuns: 4, regressedRuns: 2, regressedRate: 0.5 });
+  });
+
+  it("separates the per-ruleId breakdown across multiple rules", () => {
+    const record = computeRegressedVerdictTrackRecord([
+      comparison("rule_a", { precision: 0.6 }),
+      comparison("rule_a", { precision: 0.95 }),
+      comparison("rule_b", {}),
+    ]);
+    expect(record.perRule.get("rule_a")).toEqual({ total: 2, regressed: 1, improved: 1, unchanged: 0 });
+    expect(record.perRule.get("rule_b")).toEqual({ total: 1, regressed: 0, improved: 0, unchanged: 1 });
+  });
+});