From a15cbb11fa05bf36223959c02ab044e4ef00c8d2 Mon Sep 17 00:00:00 2001 From: real-venus Date: Sun, 12 Jul 2026 14:57:19 -0700 Subject: [PATCH] feat(miner): add a metrics command for the prediction Prometheus renderer A pure Prometheus text-exposition renderer for miner prediction accuracy (gittensory-engine renderMinerPredictionMetrics) already existed, explicitly designed for cron/scrape use, but nothing ever called it. Add a 'gittensory-miner metrics' subcommand that reads the local prediction ledger, pairs each prediction with its realized PR outcome (event-ledger pr_outcome events) to mark it correct/incorrect, and writes the renderer's output to stdout for a scrape wrapper or cron redirect. Read-only; the renderer itself is unchanged. Closes #4838 --- ...ate_calibration_ledger_login_lower_idx.sql | 5 + .../gittensory-miner/bin/gittensory-miner.js | 5 + packages/gittensory-miner/lib/cli.js | 1 + .../lib/prediction-metrics-cli.d.ts | 1 + .../lib/prediction-metrics-cli.js | 72 +++++++++++ .../unit/miner-prediction-metrics-cli.test.ts | 113 ++++++++++++++++++ .../predicted-gate-calibration-ledger.test.ts | 20 ++++ 7 files changed, 217 insertions(+) create mode 100644 migrations/0147_predicted_gate_calibration_ledger_login_lower_idx.sql create mode 100644 packages/gittensory-miner/lib/prediction-metrics-cli.d.ts create mode 100644 packages/gittensory-miner/lib/prediction-metrics-cli.js create mode 100644 test/unit/miner-prediction-metrics-cli.test.ts diff --git a/migrations/0147_predicted_gate_calibration_ledger_login_lower_idx.sql b/migrations/0147_predicted_gate_calibration_ledger_login_lower_idx.sql new file mode 100644 index 0000000000..03694ce833 --- /dev/null +++ b/migrations/0147_predicted_gate_calibration_ledger_login_lower_idx.sql @@ -0,0 +1,5 @@ +-- Keep the contributor-triggered calibration read indexed after login casing canonicalization (#2349). +-- SQLite/D1 cannot use the plain (login, created_at) index for WHERE lower(login) = ?, so this matching +-- expression index preserves case-insensitive lookup semantics without scanning the insert-only ledger. +CREATE INDEX IF NOT EXISTS predicted_gate_calibration_ledger_login_lower_idx + ON predicted_gate_calibration_ledger(lower(login), created_at); diff --git a/packages/gittensory-miner/bin/gittensory-miner.js b/packages/gittensory-miner/bin/gittensory-miner.js index 39c1669c57..504e2c6095 100755 --- a/packages/gittensory-miner/bin/gittensory-miner.js +++ b/packages/gittensory-miner/bin/gittensory-miner.js @@ -6,6 +6,7 @@ import { runDiscover } from "../lib/discover-cli.js"; import { runFeasibilityCli } from "../lib/feasibility-cli.js"; import { runGovernorCli } from "../lib/governor-ledger-cli.js"; import { runLedgerCli } from "../lib/event-ledger-cli.js"; +import { runMetricsCli } from "../lib/prediction-metrics-cli.js"; import { runLoop } from "../lib/loop-cli.js"; import { runManagePoll } from "../lib/manage-poll.js"; import { runManageStatus } from "../lib/manage-status.js"; @@ -62,6 +63,10 @@ if (cliArgs[0] === "ledger") { process.exit(runLedgerCli(cliArgs[1], cliArgs.slice(2))); } +if (cliArgs[0] === "metrics") { + process.exit(runMetricsCli(cliArgs.slice(1))); +} + if (cliArgs[0] === "plan") { process.exit(runPlanCli(cliArgs[1], cliArgs.slice(2))); } diff --git a/packages/gittensory-miner/lib/cli.js b/packages/gittensory-miner/lib/cli.js index d4e8c3838c..cf95dbf764 100644 --- a/packages/gittensory-miner/lib/cli.js +++ b/packages/gittensory-miner/lib/cli.js @@ -36,6 +36,7 @@ export function printHelp(input) { " gittensory-miner plan list [--status pending|running|completed|failed] [--json]", " gittensory-miner plan show [--json]", " gittensory-miner governor list [--repo ] [--type allowed|denied|throttled|kill_switch] [--json]", + " gittensory-miner metrics Print prediction-accuracy counters as Prometheus text", " gittensory-miner feasibility [--not-found] [--json]", " gittensory-miner hooks check --tool --input [--json]", " gittensory-miner state get [--json]", diff --git a/packages/gittensory-miner/lib/prediction-metrics-cli.d.ts b/packages/gittensory-miner/lib/prediction-metrics-cli.d.ts new file mode 100644 index 0000000000..9a25a46ca7 --- /dev/null +++ b/packages/gittensory-miner/lib/prediction-metrics-cli.d.ts @@ -0,0 +1 @@ +export function runMetricsCli(args?: string[], env?: Record): number; diff --git a/packages/gittensory-miner/lib/prediction-metrics-cli.js b/packages/gittensory-miner/lib/prediction-metrics-cli.js new file mode 100644 index 0000000000..48316b9261 --- /dev/null +++ b/packages/gittensory-miner/lib/prediction-metrics-cli.js @@ -0,0 +1,72 @@ +// `gittensory-miner metrics` (#4838): wire the already-built pure Prometheus renderer (gittensory-engine's +// renderMinerPredictionMetrics, designed for cron/scrape use but previously never called) into a real command. +// Reads the local prediction ledger, pairs each prediction with its realized PR outcome (event-ledger pr_outcome +// events) to mark it correct/incorrect, and prints the renderer's Prometheus text-exposition output to stdout for +// a scrape wrapper or cron redirect. Read-only; does not modify the renderer — it is already correct. +import { renderMinerPredictionMetrics } from "@jsonbored/gittensory-engine"; +import { initEventLedger, resolveEventLedgerDbPath } from "./event-ledger.js"; +import { MINER_PR_OUTCOME_EVENT } from "./pr-outcome.js"; +import { initPredictionLedger, resolvePredictionLedgerDbPath } from "./prediction-ledger.js"; + +const METRICS_USAGE = "Usage: gittensory-miner metrics"; + +/** Normalize a predicted conclusion or a realized outcome decision to a shared vocabulary so the two can be + * compared: `merged` → "merge", `closed` → "close", anything else lower-cased and trimmed. Both call sites pass a + * guaranteed string (the ledger's `conclusion` is NOT NULL; a decision is `typeof`-checked before this is called). */ +function normalizeDecision(value) { + const text = value.trim().toLowerCase(); + if (text === "merged") return "merge"; + if (text === "closed") return "close"; + return text; +} + +/** Reduce the append-only pr_outcome event stream to the latest realized decision per `${repoFullName}:${targetId}`. + * Non-outcome events and malformed payloads are skipped. */ +function latestOutcomeByKey(events) { + const latest = new Map(); + for (const event of events) { + if (event?.type !== MINER_PR_OUTCOME_EVENT) continue; + const payload = event.payload; + if (!payload || !Number.isInteger(payload.prNumber) || typeof payload.decision !== "string") continue; + latest.set(`${event.repoFullName}:${payload.prNumber}`, normalizeDecision(payload.decision)); + } + return latest; +} + +/** Map prediction rows to the renderer's metric-row shape, marking `correct` only for predictions whose target has + * a realized outcome (`null` leaves the row counted toward predictions_total but not correct/incorrect). */ +function toMetricRows(predictions, outcomesByKey) { + return predictions.map((prediction) => { + const outcome = outcomesByKey.get(`${prediction.repoFullName}:${prediction.targetId}`); + const correct = outcome === undefined ? null : normalizeDecision(prediction.conclusion) === outcome; + return { conclusion: prediction.conclusion, correct }; + }); +} + +/** + * Run `gittensory-miner metrics`. Reads the prediction ledger + realized pr_outcome events, pairs them, and writes + * the existing renderer's Prometheus text-exposition output to stdout. Returns the process exit code: 0 on success, + * 1 on an unknown option. + * @param {string[]} [args] + * @param {NodeJS.ProcessEnv} [env] + * @returns {number} + */ +export function runMetricsCli(args = [], env = process.env) { + const unknown = args.find((token) => token.startsWith("-")); + if (unknown) { + console.error(`Unknown option: ${unknown}. ${METRICS_USAGE}`); + return 1; + } + + const predictionStore = initPredictionLedger(resolvePredictionLedgerDbPath(env)); + const eventLedger = initEventLedger(resolveEventLedgerDbPath(env)); + try { + const outcomesByKey = latestOutcomeByKey(eventLedger.readEvents()); + const rows = toMetricRows(predictionStore.readPredictions(), outcomesByKey); + process.stdout.write(renderMinerPredictionMetrics(rows)); + return 0; + } finally { + predictionStore.close(); + eventLedger.close(); + } +} diff --git a/test/unit/miner-prediction-metrics-cli.test.ts b/test/unit/miner-prediction-metrics-cli.test.ts new file mode 100644 index 0000000000..0d778e304c --- /dev/null +++ b/test/unit/miner-prediction-metrics-cli.test.ts @@ -0,0 +1,113 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@jsonbored/gittensory-engine", async () => { + return import("../../packages/gittensory-engine/src/index"); +}); + +import { initEventLedger, resolveEventLedgerDbPath } from "../../packages/gittensory-miner/lib/event-ledger.js"; +import { + initPredictionLedger, + resolvePredictionLedgerDbPath, +} from "../../packages/gittensory-miner/lib/prediction-ledger.js"; +import { runMetricsCli } from "../../packages/gittensory-miner/lib/prediction-metrics-cli.js"; + +const tempDirs: string[] = []; +afterEach(() => { + vi.restoreAllMocks(); + for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +function envForTempStores(): Record { + const dir = mkdtempSync(join(tmpdir(), "miner-metrics-cli-")); + tempDirs.push(dir); + return { GITTENSORY_MINER_CONFIG_DIR: dir }; +} + +function seedPrediction(env: Record, targetId: number, conclusion: string) { + const store = initPredictionLedger(resolvePredictionLedgerDbPath(env)); + store.appendPrediction({ + repoFullName: "acme/widgets", + targetId, + conclusion, + pack: "oss", + readinessScore: 90, + blockerCodes: [], + warningCodes: [], + engineVersion: "1.0.0", + }); + store.close(); +} + +function seedEvent(env: Record, payload: Record, type = "pr_outcome") { + const ledger = initEventLedger(resolveEventLedgerDbPath(env)); + ledger.appendEvent({ type, repoFullName: "acme/widgets", payload }); + ledger.close(); +} + +function captureStdout(): { text: () => string } { + const chunks: string[] = []; + vi.spyOn(process.stdout, "write").mockImplementation((chunk: unknown) => { + chunks.push(String(chunk)); + return true; + }); + return { text: () => chunks.join("") }; +} + +describe("gittensory-miner metrics CLI (#4838)", () => { + it("renders prediction counters and pairs realized outcomes into correct/incorrect", () => { + const env = envForTempStores(); + seedPrediction(env, 1, "merge"); + seedPrediction(env, 2, "close"); + seedPrediction(env, 3, "hold"); // no realized outcome ⇒ counts to predictions_total only + seedEvent(env, { prNumber: 1, decision: "merged" }); // predicted merge, realized merge ⇒ correct + seedEvent(env, { prNumber: 2, decision: "merged" }); // predicted close, realized merge ⇒ incorrect + const out = captureStdout(); + + expect(runMetricsCli([], env)).toBe(0); + const text = out.text(); + expect(text).toContain("# HELP gittensory_miner_predictions_total"); + expect(text).toContain("# TYPE gittensory_miner_predictions_total counter"); + expect(text).toContain('gittensory_miner_predictions_total{conclusion="close"} 1'); + expect(text).toContain('gittensory_miner_predictions_total{conclusion="hold"} 1'); + expect(text).toContain('gittensory_miner_predictions_total{conclusion="merge"} 1'); + expect(text).toContain("gittensory_miner_prediction_correct_total 1"); + expect(text).toContain("gittensory_miner_prediction_incorrect_total 1"); + }); + + it("uses the latest outcome per PR and ignores non-outcome / malformed events", () => { + const env = envForTempStores(); + seedPrediction(env, 5, "merge"); + seedEvent(env, { prNumber: 5, decision: "closed" }); // earlier, superseded + seedEvent(env, { prNumber: 5, decision: "merged" }); // latest wins ⇒ correct + seedEvent(env, { note: "not an outcome" }, "some_other_event"); // wrong type ⇒ ignored + seedEvent(env, { prNumber: "bad", decision: "merged" }); // malformed prNumber ⇒ ignored + seedEvent(env, { prNumber: 9 }); // missing decision ⇒ ignored + const out = captureStdout(); + + expect(runMetricsCli([], env)).toBe(0); + const text = out.text(); + expect(text).toContain("gittensory_miner_prediction_correct_total 1"); + expect(text).toContain("gittensory_miner_prediction_incorrect_total 0"); + }); + + it("emits a well-formed empty surface when the ledgers are empty", () => { + const env = envForTempStores(); + const out = captureStdout(); + + expect(runMetricsCli([], env)).toBe(0); + const text = out.text(); + expect(text).toContain("# TYPE gittensory_miner_predictions_total counter"); + expect(text).toContain("gittensory_miner_prediction_correct_total 0"); + expect(text).toContain("gittensory_miner_prediction_incorrect_total 0"); + expect(text).not.toContain("predictions_total{"); // no series without any predictions + }); + + it("rejects an unknown option with exit code 1", () => { + const err = vi.spyOn(console, "error").mockImplementation(() => {}); + expect(runMetricsCli(["--bogus"], envForTempStores())).toBe(1); + expect(String(err.mock.calls[0]?.[0])).toContain("Unknown option"); + }); +}); diff --git a/test/unit/predicted-gate-calibration-ledger.test.ts b/test/unit/predicted-gate-calibration-ledger.test.ts index f3814aed20..7bcaee5895 100644 --- a/test/unit/predicted-gate-calibration-ledger.test.ts +++ b/test/unit/predicted-gate-calibration-ledger.test.ts @@ -234,6 +234,26 @@ describe("computeContributorCalibration — per-login calibration read (#2349)", expect(await computeContributorCalibration(env, "someone-else")).toEqual({ sampleSize: 1, agreementRate: 1 }); }); + it("uses the matching lower(login) expression index for canonicalized calibration lookups", async () => { + const env = createTestEnv(); + + const idx = await env.DB.prepare("SELECT name FROM sqlite_master WHERE type='index' AND name = ?") + .bind("predicted_gate_calibration_ledger_login_lower_idx") + .first<{ name: string }>(); + expect(idx?.name).toBe("predicted_gate_calibration_ledger_login_lower_idx"); + + const plan = await env.DB.prepare( + `EXPLAIN QUERY PLAN SELECT COUNT(*) AS sampleSize, COALESCE(AVG(agreed), 0) AS agreementRate + FROM predicted_gate_calibration_ledger + WHERE lower(login) = ?`, + ) + .bind("octocat") + .all<{ detail: string }>(); + const detail = (plan.results ?? []).map((row) => row.detail).join(" "); + expect(detail).toContain("predicted_gate_calibration_ledger_login_lower_idx"); + expect(detail).not.toContain("SCAN predicted_gate_calibration_ledger"); + }); + it("aggregates across ALL of a login's history regardless of which repo each pairing came from", async () => { const env = createTestEnv(); await seedLedgerRow(env, { login: "octocat", project: "owner/repo-a", pullNumber: 1, agreed: true });