feat(miner): add a calibration-report CLI command

packages/gittensory-miner/bin/gittensory-miner.js

@@ -6,6 +6,7 @@ import { runDiscover } from "../lib/discover-cli.js";
 import { runFeasibilityCli } from "../lib/feasibility-cli.js";
 import { runGovernorCli } from "../lib/governor-ledger-cli.js";
 import { runLedgerCli } from "../lib/event-ledger-cli.js";
+import { runCalibrationCli } from "../lib/calibration-cli.js";
 import { runLoop } from "../lib/loop-cli.js";
 import { runManagePoll } from "../lib/manage-poll.js";
 import { runManageStatus } from "../lib/manage-status.js";
@@ -62,6 +63,10 @@ if (cliArgs[0] === "ledger") {
   process.exit(runLedgerCli(cliArgs[1], cliArgs.slice(2)));
 }
 
+if (cliArgs[0] === "calibration") {
+  process.exit(runCalibrationCli(cliArgs.slice(1)));
+}
+
 if (cliArgs[0] === "plan") {
   process.exit(runPlanCli(cliArgs[1], cliArgs.slice(2)));
 }

packages/gittensory-miner/lib/calibration-cli.d.ts

@@ -0,0 +1 @@
+export function runCalibrationCli(args?: string[], env?: Record<string, string | undefined>): number;

packages/gittensory-miner/lib/calibration-cli.js

@@ -0,0 +1,88 @@
+// `gittensory-miner calibration [--json]` (#4849): a read-only report joining the miner's own predicted gate
+// verdicts (prediction-ledger) with the realized PR outcomes it later observed (event-ledger `pr_outcome`
+// events), via the pure buildCalibrationReport join. Opens both local stores, maps their rows to the
+// calibration record shapes, renders, and closes. Never modifies the live scoring/calibration logic.
+import { buildCalibrationReport } from "./calibration.js";
+import { initEventLedger, resolveEventLedgerDbPath } from "./event-ledger.js";
+import { MINER_PR_OUTCOME_EVENT } from "./pr-outcome.js";
+import { initPredictionLedger, resolvePredictionLedgerDbPath } from "./prediction-ledger.js";
+
+const CALIBRATION_USAGE = "Usage: gittensory-miner calibration [--json]";
+
+/** Map prediction-ledger rows to predicted-verdict records: the target id becomes a string key and the recorded
+ *  prediction verdict is the `conclusion`. */
+function toPredictionRecords(rows) {
+  return rows.map((row) => ({
+    project: row.repoFullName,
+    targetId: String(row.targetId),
+    predictedDecision: row.conclusion,
+    recordedAt: row.ts,
+  }));
+}
+
+/** Reduce the append-only `pr_outcome` event stream to the LATEST observed outcome per (repo, PR), as
+ *  observed-outcome records. `recordedAt` comes from the event's own timestamp (always present), so an outcome is
+ *  never dropped for lacking a `closedAt`. Malformed payloads are skipped. */
+function toOutcomeRecords(events) {
+  const latest = new Map();
+  for (const event of events) {
+    if (event?.type !== MINER_PR_OUTCOME_EVENT) continue;
+    const payload = event.payload;
+    if (!payload || !Number.isInteger(payload.prNumber) || typeof payload.decision !== "string") continue;
+    latest.set(`${event.repoFullName}:${payload.prNumber}`, {
+      project: event.repoFullName,
+      targetId: String(payload.prNumber),
+      outcomeDecision: payload.decision,
+      recordedAt: event.createdAt,
+    });
+  }
+  return [...latest.values()];
+}
+
+function renderReportText(report) {
+  if (!report.hasSignal) {
+    console.log("calibration: no decided predictions yet (predictions need a realized merge/close outcome).");
+    return;
+  }
+  for (const row of report.rows) {
+    const merge = row.mergePrecision === null ? "n/a" : `${Math.round(row.mergePrecision * 100)}%`;
+    const close = row.closePrecision === null ? "n/a" : `${Math.round(row.closePrecision * 100)}%`;
+    console.log(
+      `${row.project}: ${row.decided} decided | ` +
+        `merge ${row.mergeConfirmed}/${row.wouldMerge} (${merge}) | ` +
+        `close ${row.closeConfirmed}/${row.wouldClose} (${close}) | hold ${row.hold}`,
+    );
+  }
+}
+
+/**
+ * Run `gittensory-miner calibration [--json]`. Reads the prediction ledger + PR-outcome events, joins them into a
+ * calibration report, and prints it (a JSON dump under `--json`, else a per-project text summary). Returns the
+ * process exit code: 0 on success, 1 on an unknown option.
+ * @param {string[]} [args]
+ * @param {NodeJS.ProcessEnv} [env]
+ * @returns {number}
+ */
+export function runCalibrationCli(args = [], env = process.env) {
+  const json = args.includes("--json");
+  const unknown = args.find((token) => token.startsWith("-") && token !== "--json");
+  if (unknown) {
+    console.error(`Unknown option: ${unknown}. ${CALIBRATION_USAGE}`);
+    return 1;
+  }
+
+  const predictionStore = initPredictionLedger(resolvePredictionLedgerDbPath(env));
+  const eventLedger = initEventLedger(resolveEventLedgerDbPath(env));
+  try {
+    const report = buildCalibrationReport(
+      toPredictionRecords(predictionStore.readPredictions()),
+      toOutcomeRecords(eventLedger.readEvents()),
+    );
+    if (json) console.log(JSON.stringify(report, null, 2));
+    else renderReportText(report);
+    return 0;
+  } finally {
+    predictionStore.close();
+    eventLedger.close();
+  }
+}

packages/gittensory-miner/lib/calibration.d.ts

@@ -1,3 +1,9 @@
+import type {
+  CalibrationReport,
+  ObservedOutcomeRecord,
+  PredictedVerdictRecord,
+} from "./calibration-types.js";
+
 export type {
   CalibrationReport,
   CalibrationRow,
@@ -11,3 +17,8 @@ export {
   isObservedOutcomeRecord,
   isPredictedVerdictRecord,
 } from "./calibration-types.js";
+
+export function buildCalibrationReport(
+  predictions: PredictedVerdictRecord[],
+  outcomes: ObservedOutcomeRecord[],
+): CalibrationReport;

packages/gittensory-miner/lib/calibration.js

@@ -1,6 +1,99 @@
-export {
+// Calibration report: join the miner's own predicted gate verdicts with the realized outcomes it later observed
+// (#4849). Read-only aggregation only — it never touches the live scoring/calibration logic that feeds the gate
+// (maintainer-owned). Builds on the types-only scaffolding in calibration-types.js.
+import {
   isCalibrationReport,
   isCalibrationRow,
   isObservedOutcomeRecord,
   isPredictedVerdictRecord,
 } from "./calibration-types.js";
+
+export { isCalibrationReport, isCalibrationRow, isObservedOutcomeRecord, isPredictedVerdictRecord };
+
+/** Normalize a decision string to the calibration vocabulary (`merge` / `close` / `hold`), or `""` when it is
+ *  unrecognized. `value` is always the already-validated non-empty string field of a record (the type guards run
+ *  first), so no non-string handling is needed here. Accepts both the predicted (`merge`/`close`/`hold`) and the
+ *  realized (`merged`/`closed`) forms. */
+function normalizeDecision(value) {
+  const decision = value.trim().toLowerCase();
+  if (decision === "merge" || decision === "merged") return "merge";
+  if (decision === "close" || decision === "closed") return "close";
+  if (decision === "hold") return "hold";
+  return "";
+}
+
+function emptyRow(project) {
+  return {
+    project,
+    wouldMerge: 0,
+    mergeConfirmed: 0,
+    mergeFalse: 0,
+    wouldClose: 0,
+    closeConfirmed: 0,
+    closeFalse: 0,
+    hold: 0,
+    decided: 0,
+    mergePrecision: null,
+    closePrecision: null,
+  };
+}
+
+// Key a record by its (project, targetId). Project and targetId are validated non-empty strings; the space
+// separator is fine for keying (collisions across different (project, targetId) pairs are astronomically
+// unlikely and would only merge two projects' tallies, never fabricate a false one).
+function recordKey(project, targetId) {
+  return `${project} ${targetId}`;
+}
+
+/**
+ * Join predicted-verdict records with realized-outcome records into a per-project calibration report. Pure and
+ * read-only. A prediction counts as "decided" only when a realized outcome for the SAME `(project, targetId)`
+ * exists AND resolves to a clear `merge` or `close`; a still-pending prediction (no outcome) or one whose outcome
+ * is unrecognized is skipped. Per project it tallies the confusion matrix (would-merge/close vs confirmed/false,
+ * plus holds) and derives merge/close precision (null below one relevant sample). Malformed records on either
+ * side are ignored. Rows are sorted by project for a stable render.
+ *
+ * @param {import("./calibration-types.js").PredictedVerdictRecord[]} predictions
+ * @param {import("./calibration-types.js").ObservedOutcomeRecord[]} outcomes
+ * @returns {import("./calibration-types.js").CalibrationReport}
+ */
+export function buildCalibrationReport(predictions, outcomes) {
+  const outcomeByKey = new Map();
+  for (const outcome of Array.isArray(outcomes) ? outcomes : []) {
+    if (!isObservedOutcomeRecord(outcome)) continue;
+    outcomeByKey.set(recordKey(outcome.project, outcome.targetId), normalizeDecision(outcome.outcomeDecision));
+  }
+
+  const byProject = new Map();
+  for (const prediction of Array.isArray(predictions) ? predictions : []) {
+    if (!isPredictedVerdictRecord(prediction)) continue;
+    const observed = outcomeByKey.get(recordKey(prediction.project, prediction.targetId));
+    if (observed !== "merge" && observed !== "close") continue; // pending or unclassifiable outcome
+    let row = byProject.get(prediction.project);
+    if (!row) {
+      row = emptyRow(prediction.project);
+      byProject.set(prediction.project, row);
+    }
+    row.decided += 1;
+    const predicted = normalizeDecision(prediction.predictedDecision);
+    if (predicted === "merge") {
+      row.wouldMerge += 1;
+      if (observed === "merge") row.mergeConfirmed += 1;
+      else row.mergeFalse += 1;
+    } else if (predicted === "close") {
+      row.wouldClose += 1;
+      if (observed === "close") row.closeConfirmed += 1;
+      else row.closeFalse += 1;
+    } else if (predicted === "hold") {
+      row.hold += 1;
+    }
+  }
+
+  const rows = [...byProject.values()].sort((a, b) => a.project.localeCompare(b.project));
+  for (const row of rows) {
+    row.mergePrecision = row.wouldMerge > 0 ? row.mergeConfirmed / row.wouldMerge : null;
+    row.closePrecision = row.wouldClose > 0 ? row.closeConfirmed / row.wouldClose : null;
+  }
+  // Signal exists once any project carries at least one decided (predicted-then-realized) sample.
+  return { hasSignal: rows.length > 0, rows };
+}

packages/gittensory-miner/lib/cli.js

@@ -36,6 +36,7 @@ export function printHelp(input) {
       "  gittensory-miner plan list [--status pending|running|completed|failed] [--json]",
       "  gittensory-miner plan show <planId> [--json]",
       "  gittensory-miner governor list [--repo <owner/repo>] [--type allowed|denied|throttled|kill_switch] [--json]",
+      "  gittensory-miner calibration [--json]                         Report predicted-vs-realized gate accuracy",
       "  gittensory-miner feasibility <claimStatus> <duplicateClusterRisk> <issueStatus> [--not-found] [--json]",
       "  gittensory-miner hooks check --tool <name> --input <json> [--json]",
       "  gittensory-miner state get <owner/repo> [--json]",

test/unit/miner-calibration-cli.test.ts

@@ -0,0 +1,115 @@
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { runCalibrationCli } from "../../packages/gittensory-miner/lib/calibration-cli.js";
+import { initEventLedger, resolveEventLedgerDbPath } from "../../packages/gittensory-miner/lib/event-ledger.js";
+import {
+  initPredictionLedger,
+  resolvePredictionLedgerDbPath,
+} from "../../packages/gittensory-miner/lib/prediction-ledger.js";
+
+const tempDirs: string[] = [];
+afterEach(() => {
+  vi.restoreAllMocks();
+  for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+function envForTempStores(): Record<string, string | undefined> {
+  const dir = mkdtempSync(join(tmpdir(), "miner-calibration-cli-"));
+  tempDirs.push(dir);
+  return { GITTENSORY_MINER_CONFIG_DIR: dir };
+}
+
+function seedPrediction(env: Record<string, string | undefined>, targetId: number, conclusion: string) {
+  const store = initPredictionLedger(resolvePredictionLedgerDbPath(env));
+  store.appendPrediction({
+    repoFullName: "acme/widgets",
+    targetId,
+    conclusion,
+    pack: "oss",
+    readinessScore: 90,
+    blockerCodes: [],
+    warningCodes: [],
+    engineVersion: "1.0.0",
+  });
+  store.close();
+}
+
+function seedOutcomeEvent(
+  env: Record<string, string | undefined>,
+  payload: Record<string, unknown>,
+  type = "pr_outcome",
+) {
+  const ledger = initEventLedger(resolveEventLedgerDbPath(env));
+  ledger.appendEvent({ type, repoFullName: "acme/widgets", payload });
+  ledger.close();
+}
+
+describe("gittensory-miner calibration CLI (#4849)", () => {
+  it("joins a merge prediction with a merged outcome and renders the per-project accuracy", () => {
+    const env = envForTempStores();
+    seedPrediction(env, 42, "merge");
+    seedOutcomeEvent(env, { prNumber: 42, decision: "merged" });
+    const log = vi.spyOn(console, "log").mockImplementation(() => {});
+
+    expect(runCalibrationCli([], env)).toBe(0);
+    const output = log.mock.calls.map((call) => String(call[0])).join("\n");
+    expect(output).toContain("acme/widgets: 1 decided");
+    expect(output).toContain("merge 1/1 (100%)");
+    expect(output).toContain("close 0/0 (n/a)"); // no close predictions ⇒ n/a
+  });
+
+  it("renders n/a merge precision and a realized close precision for a close-only project", () => {
+    const env = envForTempStores();
+    seedPrediction(env, 99, "close");
+    seedOutcomeEvent(env, { prNumber: 99, decision: "closed" });
+    const log = vi.spyOn(console, "log").mockImplementation(() => {});
+
+    expect(runCalibrationCli([], env)).toBe(0);
+    const output = log.mock.calls.map((call) => String(call[0])).join("\n");
+    expect(output).toContain("merge 0/0 (n/a)"); // no merge predictions ⇒ n/a
+    expect(output).toContain("close 1/1 (100%)"); // realized close ⇒ precision rendered
+  });
+
+  it("emits the structured report under --json", () => {
+    const env = envForTempStores();
+    seedPrediction(env, 7, "merge");
+    seedOutcomeEvent(env, { prNumber: 7, decision: "merged" });
+    const log = vi.spyOn(console, "log").mockImplementation(() => {});
+
+    expect(runCalibrationCli(["--json"], env)).toBe(0);
+    const report = JSON.parse(String(log.mock.calls[0]?.[0]));
+    expect(report.hasSignal).toBe(true);
+    expect(report.rows[0]).toMatchObject({ project: "acme/widgets", mergeConfirmed: 1, mergePrecision: 1 });
+  });
+
+  it("reports no signal when there are no decided predictions", () => {
+    const env = envForTempStores();
+    seedPrediction(env, 1, "merge"); // prediction with no realized outcome yet
+    const log = vi.spyOn(console, "log").mockImplementation(() => {});
+
+    expect(runCalibrationCli([], env)).toBe(0);
+    expect(log.mock.calls.map((c) => String(c[0])).join("\n")).toContain("no decided predictions");
+  });
+
+  it("takes the latest outcome per PR and skips non-outcome / malformed events", () => {
+    const env = envForTempStores();
+    seedPrediction(env, 5, "merge");
+    seedOutcomeEvent(env, { prNumber: 5, decision: "closed" }); // earlier, superseded
+    seedOutcomeEvent(env, { prNumber: 5, decision: "merged" }); // latest wins
+    seedOutcomeEvent(env, { note: "not a pr outcome" }, "some_other_event"); // wrong type ⇒ ignored
+    seedOutcomeEvent(env, { prNumber: "bad" }); // malformed payload ⇒ ignored
+    const log = vi.spyOn(console, "log").mockImplementation(() => {});
+
+    expect(runCalibrationCli(["--json"], env)).toBe(0);
+    const report = JSON.parse(String(log.mock.calls[0]?.[0]));
+    expect(report.rows[0]).toMatchObject({ mergeConfirmed: 1, mergeFalse: 0 }); // latest "merged" confirmed the merge
+  });
+
+  it("rejects an unknown option with exit code 1", () => {
+    const err = vi.spyOn(console, "error").mockImplementation(() => {});
+    expect(runCalibrationCli(["--bogus"], envForTempStores())).toBe(1);
+    expect(String(err.mock.calls[0]?.[0])).toContain("Unknown option");
+  });
+});