From e57d830be3b4b741099ed7e23043551595ffebe9 Mon Sep 17 00:00:00 2001 From: real-venus Date: Sun, 12 Jul 2026 13:24:20 -0700 Subject: [PATCH] feat(miner): add a calibration-report CLI command Join the miner's own predicted gate verdicts (prediction-ledger) with the realized PR outcomes it later observed (event-ledger pr_outcome events) into a per-project accuracy report, and surface it via 'gittensory-miner calibration [--json]'. buildCalibrationReport is a pure join over the existing record shapes; the CLI opens both local stores read-only, maps their rows, renders a text or JSON report, and closes. Does not touch the live scoring/calibration logic that feeds the gate. Closes #4849 --- .../gittensory-miner/bin/gittensory-miner.js | 5 + .../gittensory-miner/lib/calibration-cli.d.ts | 1 + .../gittensory-miner/lib/calibration-cli.js | 88 ++++++++++++++ .../gittensory-miner/lib/calibration.d.ts | 11 ++ packages/gittensory-miner/lib/calibration.js | 95 ++++++++++++++- packages/gittensory-miner/lib/cli.js | 1 + test/unit/miner-calibration-cli.test.ts | 115 ++++++++++++++++++ test/unit/miner-calibration.test.ts | 115 ++++++++++++++++++ 8 files changed, 430 insertions(+), 1 deletion(-) create mode 100644 packages/gittensory-miner/lib/calibration-cli.d.ts create mode 100644 packages/gittensory-miner/lib/calibration-cli.js create mode 100644 test/unit/miner-calibration-cli.test.ts create mode 100644 test/unit/miner-calibration.test.ts diff --git a/packages/gittensory-miner/bin/gittensory-miner.js b/packages/gittensory-miner/bin/gittensory-miner.js index 39c1669c57..b8d665ee77 100755 --- a/packages/gittensory-miner/bin/gittensory-miner.js +++ b/packages/gittensory-miner/bin/gittensory-miner.js @@ -6,6 +6,7 @@ import { runDiscover } from "../lib/discover-cli.js"; import { runFeasibilityCli } from "../lib/feasibility-cli.js"; import { runGovernorCli } from "../lib/governor-ledger-cli.js"; import { runLedgerCli } from "../lib/event-ledger-cli.js"; +import { runCalibrationCli } from "../lib/calibration-cli.js"; import { runLoop } from "../lib/loop-cli.js"; import { runManagePoll } from "../lib/manage-poll.js"; import { runManageStatus } from "../lib/manage-status.js"; @@ -62,6 +63,10 @@ if (cliArgs[0] === "ledger") { process.exit(runLedgerCli(cliArgs[1], cliArgs.slice(2))); } +if (cliArgs[0] === "calibration") { + process.exit(runCalibrationCli(cliArgs.slice(1))); +} + if (cliArgs[0] === "plan") { process.exit(runPlanCli(cliArgs[1], cliArgs.slice(2))); } diff --git a/packages/gittensory-miner/lib/calibration-cli.d.ts b/packages/gittensory-miner/lib/calibration-cli.d.ts new file mode 100644 index 0000000000..56c839095e --- /dev/null +++ b/packages/gittensory-miner/lib/calibration-cli.d.ts @@ -0,0 +1 @@ +export function runCalibrationCli(args?: string[], env?: Record): number; diff --git a/packages/gittensory-miner/lib/calibration-cli.js b/packages/gittensory-miner/lib/calibration-cli.js new file mode 100644 index 0000000000..b9ea26971e --- /dev/null +++ b/packages/gittensory-miner/lib/calibration-cli.js @@ -0,0 +1,88 @@ +// `gittensory-miner calibration [--json]` (#4849): a read-only report joining the miner's own predicted gate +// verdicts (prediction-ledger) with the realized PR outcomes it later observed (event-ledger `pr_outcome` +// events), via the pure buildCalibrationReport join. Opens both local stores, maps their rows to the +// calibration record shapes, renders, and closes. Never modifies the live scoring/calibration logic. +import { buildCalibrationReport } from "./calibration.js"; +import { initEventLedger, resolveEventLedgerDbPath } from "./event-ledger.js"; +import { MINER_PR_OUTCOME_EVENT } from "./pr-outcome.js"; +import { initPredictionLedger, resolvePredictionLedgerDbPath } from "./prediction-ledger.js"; + +const CALIBRATION_USAGE = "Usage: gittensory-miner calibration [--json]"; + +/** Map prediction-ledger rows to predicted-verdict records: the target id becomes a string key and the recorded + * prediction verdict is the `conclusion`. */ +function toPredictionRecords(rows) { + return rows.map((row) => ({ + project: row.repoFullName, + targetId: String(row.targetId), + predictedDecision: row.conclusion, + recordedAt: row.ts, + })); +} + +/** Reduce the append-only `pr_outcome` event stream to the LATEST observed outcome per (repo, PR), as + * observed-outcome records. `recordedAt` comes from the event's own timestamp (always present), so an outcome is + * never dropped for lacking a `closedAt`. Malformed payloads are skipped. */ +function toOutcomeRecords(events) { + const latest = new Map(); + for (const event of events) { + if (event?.type !== MINER_PR_OUTCOME_EVENT) continue; + const payload = event.payload; + if (!payload || !Number.isInteger(payload.prNumber) || typeof payload.decision !== "string") continue; + latest.set(`${event.repoFullName}:${payload.prNumber}`, { + project: event.repoFullName, + targetId: String(payload.prNumber), + outcomeDecision: payload.decision, + recordedAt: event.createdAt, + }); + } + return [...latest.values()]; +} + +function renderReportText(report) { + if (!report.hasSignal) { + console.log("calibration: no decided predictions yet (predictions need a realized merge/close outcome)."); + return; + } + for (const row of report.rows) { + const merge = row.mergePrecision === null ? "n/a" : `${Math.round(row.mergePrecision * 100)}%`; + const close = row.closePrecision === null ? "n/a" : `${Math.round(row.closePrecision * 100)}%`; + console.log( + `${row.project}: ${row.decided} decided | ` + + `merge ${row.mergeConfirmed}/${row.wouldMerge} (${merge}) | ` + + `close ${row.closeConfirmed}/${row.wouldClose} (${close}) | hold ${row.hold}`, + ); + } +} + +/** + * Run `gittensory-miner calibration [--json]`. Reads the prediction ledger + PR-outcome events, joins them into a + * calibration report, and prints it (a JSON dump under `--json`, else a per-project text summary). Returns the + * process exit code: 0 on success, 1 on an unknown option. + * @param {string[]} [args] + * @param {NodeJS.ProcessEnv} [env] + * @returns {number} + */ +export function runCalibrationCli(args = [], env = process.env) { + const json = args.includes("--json"); + const unknown = args.find((token) => token.startsWith("-") && token !== "--json"); + if (unknown) { + console.error(`Unknown option: ${unknown}. ${CALIBRATION_USAGE}`); + return 1; + } + + const predictionStore = initPredictionLedger(resolvePredictionLedgerDbPath(env)); + const eventLedger = initEventLedger(resolveEventLedgerDbPath(env)); + try { + const report = buildCalibrationReport( + toPredictionRecords(predictionStore.readPredictions()), + toOutcomeRecords(eventLedger.readEvents()), + ); + if (json) console.log(JSON.stringify(report, null, 2)); + else renderReportText(report); + return 0; + } finally { + predictionStore.close(); + eventLedger.close(); + } +} diff --git a/packages/gittensory-miner/lib/calibration.d.ts b/packages/gittensory-miner/lib/calibration.d.ts index 4286d64f00..76388c2dea 100644 --- a/packages/gittensory-miner/lib/calibration.d.ts +++ b/packages/gittensory-miner/lib/calibration.d.ts @@ -1,3 +1,9 @@ +import type { + CalibrationReport, + ObservedOutcomeRecord, + PredictedVerdictRecord, +} from "./calibration-types.js"; + export type { CalibrationReport, CalibrationRow, @@ -11,3 +17,8 @@ export { isObservedOutcomeRecord, isPredictedVerdictRecord, } from "./calibration-types.js"; + +export function buildCalibrationReport( + predictions: PredictedVerdictRecord[], + outcomes: ObservedOutcomeRecord[], +): CalibrationReport; diff --git a/packages/gittensory-miner/lib/calibration.js b/packages/gittensory-miner/lib/calibration.js index 40d74e7115..bdb671b1f5 100644 --- a/packages/gittensory-miner/lib/calibration.js +++ b/packages/gittensory-miner/lib/calibration.js @@ -1,6 +1,99 @@ -export { +// Calibration report: join the miner's own predicted gate verdicts with the realized outcomes it later observed +// (#4849). Read-only aggregation only — it never touches the live scoring/calibration logic that feeds the gate +// (maintainer-owned). Builds on the types-only scaffolding in calibration-types.js. +import { isCalibrationReport, isCalibrationRow, isObservedOutcomeRecord, isPredictedVerdictRecord, } from "./calibration-types.js"; + +export { isCalibrationReport, isCalibrationRow, isObservedOutcomeRecord, isPredictedVerdictRecord }; + +/** Normalize a decision string to the calibration vocabulary (`merge` / `close` / `hold`), or `""` when it is + * unrecognized. `value` is always the already-validated non-empty string field of a record (the type guards run + * first), so no non-string handling is needed here. Accepts both the predicted (`merge`/`close`/`hold`) and the + * realized (`merged`/`closed`) forms. */ +function normalizeDecision(value) { + const decision = value.trim().toLowerCase(); + if (decision === "merge" || decision === "merged") return "merge"; + if (decision === "close" || decision === "closed") return "close"; + if (decision === "hold") return "hold"; + return ""; +} + +function emptyRow(project) { + return { + project, + wouldMerge: 0, + mergeConfirmed: 0, + mergeFalse: 0, + wouldClose: 0, + closeConfirmed: 0, + closeFalse: 0, + hold: 0, + decided: 0, + mergePrecision: null, + closePrecision: null, + }; +} + +// Key a record by its (project, targetId). Project and targetId are validated non-empty strings; the space +// separator is fine for keying (collisions across different (project, targetId) pairs are astronomically +// unlikely and would only merge two projects' tallies, never fabricate a false one). +function recordKey(project, targetId) { + return `${project} ${targetId}`; +} + +/** + * Join predicted-verdict records with realized-outcome records into a per-project calibration report. Pure and + * read-only. A prediction counts as "decided" only when a realized outcome for the SAME `(project, targetId)` + * exists AND resolves to a clear `merge` or `close`; a still-pending prediction (no outcome) or one whose outcome + * is unrecognized is skipped. Per project it tallies the confusion matrix (would-merge/close vs confirmed/false, + * plus holds) and derives merge/close precision (null below one relevant sample). Malformed records on either + * side are ignored. Rows are sorted by project for a stable render. + * + * @param {import("./calibration-types.js").PredictedVerdictRecord[]} predictions + * @param {import("./calibration-types.js").ObservedOutcomeRecord[]} outcomes + * @returns {import("./calibration-types.js").CalibrationReport} + */ +export function buildCalibrationReport(predictions, outcomes) { + const outcomeByKey = new Map(); + for (const outcome of Array.isArray(outcomes) ? outcomes : []) { + if (!isObservedOutcomeRecord(outcome)) continue; + outcomeByKey.set(recordKey(outcome.project, outcome.targetId), normalizeDecision(outcome.outcomeDecision)); + } + + const byProject = new Map(); + for (const prediction of Array.isArray(predictions) ? predictions : []) { + if (!isPredictedVerdictRecord(prediction)) continue; + const observed = outcomeByKey.get(recordKey(prediction.project, prediction.targetId)); + if (observed !== "merge" && observed !== "close") continue; // pending or unclassifiable outcome + let row = byProject.get(prediction.project); + if (!row) { + row = emptyRow(prediction.project); + byProject.set(prediction.project, row); + } + row.decided += 1; + const predicted = normalizeDecision(prediction.predictedDecision); + if (predicted === "merge") { + row.wouldMerge += 1; + if (observed === "merge") row.mergeConfirmed += 1; + else row.mergeFalse += 1; + } else if (predicted === "close") { + row.wouldClose += 1; + if (observed === "close") row.closeConfirmed += 1; + else row.closeFalse += 1; + } else if (predicted === "hold") { + row.hold += 1; + } + } + + const rows = [...byProject.values()].sort((a, b) => a.project.localeCompare(b.project)); + for (const row of rows) { + row.mergePrecision = row.wouldMerge > 0 ? row.mergeConfirmed / row.wouldMerge : null; + row.closePrecision = row.wouldClose > 0 ? row.closeConfirmed / row.wouldClose : null; + } + // Signal exists once any project carries at least one decided (predicted-then-realized) sample. + return { hasSignal: rows.length > 0, rows }; +} diff --git a/packages/gittensory-miner/lib/cli.js b/packages/gittensory-miner/lib/cli.js index d4e8c3838c..b79a65cc97 100644 --- a/packages/gittensory-miner/lib/cli.js +++ b/packages/gittensory-miner/lib/cli.js @@ -36,6 +36,7 @@ export function printHelp(input) { " gittensory-miner plan list [--status pending|running|completed|failed] [--json]", " gittensory-miner plan show [--json]", " gittensory-miner governor list [--repo ] [--type allowed|denied|throttled|kill_switch] [--json]", + " gittensory-miner calibration [--json] Report predicted-vs-realized gate accuracy", " gittensory-miner feasibility [--not-found] [--json]", " gittensory-miner hooks check --tool --input [--json]", " gittensory-miner state get [--json]", diff --git a/test/unit/miner-calibration-cli.test.ts b/test/unit/miner-calibration-cli.test.ts new file mode 100644 index 0000000000..33eb50b776 --- /dev/null +++ b/test/unit/miner-calibration-cli.test.ts @@ -0,0 +1,115 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { runCalibrationCli } from "../../packages/gittensory-miner/lib/calibration-cli.js"; +import { initEventLedger, resolveEventLedgerDbPath } from "../../packages/gittensory-miner/lib/event-ledger.js"; +import { + initPredictionLedger, + resolvePredictionLedgerDbPath, +} from "../../packages/gittensory-miner/lib/prediction-ledger.js"; + +const tempDirs: string[] = []; +afterEach(() => { + vi.restoreAllMocks(); + for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +function envForTempStores(): Record { + const dir = mkdtempSync(join(tmpdir(), "miner-calibration-cli-")); + tempDirs.push(dir); + return { GITTENSORY_MINER_CONFIG_DIR: dir }; +} + +function seedPrediction(env: Record, targetId: number, conclusion: string) { + const store = initPredictionLedger(resolvePredictionLedgerDbPath(env)); + store.appendPrediction({ + repoFullName: "acme/widgets", + targetId, + conclusion, + pack: "oss", + readinessScore: 90, + blockerCodes: [], + warningCodes: [], + engineVersion: "1.0.0", + }); + store.close(); +} + +function seedOutcomeEvent( + env: Record, + payload: Record, + type = "pr_outcome", +) { + const ledger = initEventLedger(resolveEventLedgerDbPath(env)); + ledger.appendEvent({ type, repoFullName: "acme/widgets", payload }); + ledger.close(); +} + +describe("gittensory-miner calibration CLI (#4849)", () => { + it("joins a merge prediction with a merged outcome and renders the per-project accuracy", () => { + const env = envForTempStores(); + seedPrediction(env, 42, "merge"); + seedOutcomeEvent(env, { prNumber: 42, decision: "merged" }); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + + expect(runCalibrationCli([], env)).toBe(0); + const output = log.mock.calls.map((call) => String(call[0])).join("\n"); + expect(output).toContain("acme/widgets: 1 decided"); + expect(output).toContain("merge 1/1 (100%)"); + expect(output).toContain("close 0/0 (n/a)"); // no close predictions ⇒ n/a + }); + + it("renders n/a merge precision and a realized close precision for a close-only project", () => { + const env = envForTempStores(); + seedPrediction(env, 99, "close"); + seedOutcomeEvent(env, { prNumber: 99, decision: "closed" }); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + + expect(runCalibrationCli([], env)).toBe(0); + const output = log.mock.calls.map((call) => String(call[0])).join("\n"); + expect(output).toContain("merge 0/0 (n/a)"); // no merge predictions ⇒ n/a + expect(output).toContain("close 1/1 (100%)"); // realized close ⇒ precision rendered + }); + + it("emits the structured report under --json", () => { + const env = envForTempStores(); + seedPrediction(env, 7, "merge"); + seedOutcomeEvent(env, { prNumber: 7, decision: "merged" }); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + + expect(runCalibrationCli(["--json"], env)).toBe(0); + const report = JSON.parse(String(log.mock.calls[0]?.[0])); + expect(report.hasSignal).toBe(true); + expect(report.rows[0]).toMatchObject({ project: "acme/widgets", mergeConfirmed: 1, mergePrecision: 1 }); + }); + + it("reports no signal when there are no decided predictions", () => { + const env = envForTempStores(); + seedPrediction(env, 1, "merge"); // prediction with no realized outcome yet + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + + expect(runCalibrationCli([], env)).toBe(0); + expect(log.mock.calls.map((c) => String(c[0])).join("\n")).toContain("no decided predictions"); + }); + + it("takes the latest outcome per PR and skips non-outcome / malformed events", () => { + const env = envForTempStores(); + seedPrediction(env, 5, "merge"); + seedOutcomeEvent(env, { prNumber: 5, decision: "closed" }); // earlier, superseded + seedOutcomeEvent(env, { prNumber: 5, decision: "merged" }); // latest wins + seedOutcomeEvent(env, { note: "not a pr outcome" }, "some_other_event"); // wrong type ⇒ ignored + seedOutcomeEvent(env, { prNumber: "bad" }); // malformed payload ⇒ ignored + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + + expect(runCalibrationCli(["--json"], env)).toBe(0); + const report = JSON.parse(String(log.mock.calls[0]?.[0])); + expect(report.rows[0]).toMatchObject({ mergeConfirmed: 1, mergeFalse: 0 }); // latest "merged" confirmed the merge + }); + + it("rejects an unknown option with exit code 1", () => { + const err = vi.spyOn(console, "error").mockImplementation(() => {}); + expect(runCalibrationCli(["--bogus"], envForTempStores())).toBe(1); + expect(String(err.mock.calls[0]?.[0])).toContain("Unknown option"); + }); +}); diff --git a/test/unit/miner-calibration.test.ts b/test/unit/miner-calibration.test.ts new file mode 100644 index 0000000000..6e6bdf8d30 --- /dev/null +++ b/test/unit/miner-calibration.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, it } from "vitest"; +import { + buildCalibrationReport, + isCalibrationReport, +} from "../../packages/gittensory-miner/lib/calibration.js"; +import type { + ObservedOutcomeRecord, + PredictedVerdictRecord, +} from "../../packages/gittensory-miner/lib/calibration.js"; + +const TS = "2026-07-12T00:00:00.000Z"; +const pred = (project: string, targetId: string, predictedDecision: string): PredictedVerdictRecord => ({ + project, + targetId, + predictedDecision, + recordedAt: TS, +}); +const out = (project: string, targetId: string, outcomeDecision: string): ObservedOutcomeRecord => ({ + project, + targetId, + outcomeDecision, + recordedAt: TS, +}); +const rowFor = (report: ReturnType, project: string) => + report.rows.find((r) => r.project === project); + +describe("buildCalibrationReport (#4849)", () => { + it("returns an empty, no-signal report for empty or non-array input", () => { + expect(buildCalibrationReport([], [])).toEqual({ hasSignal: false, rows: [] }); + // Non-array inputs are tolerated as empty. + expect(buildCalibrationReport(null as never, undefined as never)).toEqual({ hasSignal: false, rows: [] }); + }); + + it("skips a prediction with no realized outcome yet (still pending)", () => { + const report = buildCalibrationReport([pred("a/b", "1", "merge")], []); + expect(report).toEqual({ hasSignal: false, rows: [] }); + }); + + it("skips a prediction whose outcome is unrecognized (not a clear merge/close)", () => { + const report = buildCalibrationReport([pred("a/b", "1", "merge")], [out("a/b", "1", "unknown-thing")]); + expect(report.hasSignal).toBe(false); + expect(report.rows).toEqual([]); + }); + + it("counts a correct merge prediction as confirmed with precision 1", () => { + const report = buildCalibrationReport([pred("a/b", "1", "merge")], [out("a/b", "1", "merged")]); + expect(report.hasSignal).toBe(true); + expect(rowFor(report, "a/b")).toMatchObject({ + wouldMerge: 1, + mergeConfirmed: 1, + mergeFalse: 0, + decided: 1, + mergePrecision: 1, + closePrecision: null, + }); + expect(isCalibrationReport(report)).toBe(true); + }); + + it("counts a merge prediction that actually closed as a false positive (precision 0)", () => { + const report = buildCalibrationReport([pred("a/b", "1", "merge")], [out("a/b", "1", "closed")]); + expect(rowFor(report, "a/b")).toMatchObject({ wouldMerge: 1, mergeConfirmed: 0, mergeFalse: 1, mergePrecision: 0 }); + }); + + it("tallies close predictions (confirmed and false) and hold predictions", () => { + const report = buildCalibrationReport( + [pred("a/b", "1", "close"), pred("a/b", "2", "close"), pred("a/b", "3", "hold")], + [out("a/b", "1", "close"), out("a/b", "2", "merge"), out("a/b", "3", "close")], + ); + expect(rowFor(report, "a/b")).toMatchObject({ + wouldClose: 2, + closeConfirmed: 1, + closeFalse: 1, + hold: 1, + decided: 3, + closePrecision: 0.5, + mergePrecision: null, // no merge predictions ⇒ null + }); + }); + + it("aggregates independently per project and sorts rows by project", () => { + const report = buildCalibrationReport( + [pred("z/one", "1", "merge"), pred("a/two", "1", "close")], + [out("z/one", "1", "merge"), out("a/two", "1", "close")], + ); + expect(report.rows.map((r) => r.project)).toEqual(["a/two", "z/one"]); // sorted + expect(rowFor(report, "z/one")).toMatchObject({ mergeConfirmed: 1, mergePrecision: 1 }); + expect(rowFor(report, "a/two")).toMatchObject({ closeConfirmed: 1, closePrecision: 1 }); + }); + + it("ignores malformed prediction and outcome records", () => { + const report = buildCalibrationReport( + [pred("a/b", "1", "merge"), { project: "a/b" } as never, null as never], + [out("a/b", "1", "merged"), { targetId: "x" } as never], + ); + expect(report.rows).toHaveLength(1); + expect(rowFor(report, "a/b")).toMatchObject({ decided: 1, mergeConfirmed: 1 }); + }); + + it("counts a prediction with an unrecognized predicted decision as decided but in no confusion bucket", () => { + const report = buildCalibrationReport([pred("a/b", "1", "maybe?")], [out("a/b", "1", "merged")]); + expect(rowFor(report, "a/b")).toMatchObject({ + decided: 1, + wouldMerge: 0, + wouldClose: 0, + hold: 0, + mergePrecision: null, + closePrecision: null, + }); + }); + + it("matches strictly on (project, targetId) — a same id under a different project is not joined", () => { + const report = buildCalibrationReport([pred("a/b", "1", "merge")], [out("c/d", "1", "merged")]); + expect(report.hasSignal).toBe(false); // outcome belongs to a different project + }); +});