diff --git a/packages/loopover-miner/README.md b/packages/loopover-miner/README.md index 269808eea2..97e033dc2d 100644 --- a/packages/loopover-miner/README.md +++ b/packages/loopover-miner/README.md @@ -255,7 +255,9 @@ It exposes these read-only tools: - `loopover_miner_status` (#5154) — read-only status + doctor diagnostics, returning `{ status, doctor }`: `status` = package/engine versions (and skew), node version, state-dir + config-file paths, and the resolved coding-agent driver (provider name, the model **env-var NAME** never its value, CLI-present boolean); `doctor` = the checks `loopover-miner doctor` runs (Docker/CLI presence, config validity, …) as `{ name, ok, detail }`. Reuses `collectStatus` / `runDoctorChecks` so it can't drift from the CLI, and returns only names / booleans / paths — never any env-var value, token, or credential. -This completes the read-only AMS MCP tool surface (status, portfolio, claims, event-ledger, governor-ledger, run-state, plan-store). +- `loopover_miner_get_calibration_report` (#5821) — read-only miner-local prediction-accuracy report: per-project merge/close precision, joining this miner's own recorded gate predictions (prediction ledger) with the realized PR outcomes it later observed (`pr_outcome` events). Wraps `calibration-cli.js`'s existing `toPredictionRecords` / `toOutcomeRecords` mappers and `calibration.js`'s `buildCalibrationReport` composer — no new join/scoring logic. Strictly local and offline; distinct from ORB's hosted, maintainer-authenticated `loopover_get_outcome_calibration` tool, which reads a different (D1) data source. + +This completes the read-only AMS MCP tool surface (status, portfolio, claims, event-ledger, governor-ledger, run-state, plan-store, calibration). ### Client config @@ -276,7 +278,7 @@ This completes the read-only AMS MCP tool surface (status, portfolio, claims, ev } ``` -`loopover` exposes ORB's hosted contributor-workflow tools (issue ranking, PR packet prep, decision packs). `loopover-miner` exposes AMS's own local state-visibility tools listed above (portfolio dashboard, claims, audit feed, run state, plans) — a fully separate, 100% local tool surface with no shared code or network calls between the two. Both follow the same `loopover_*` tool-naming convention (`loopover_...` vs. `loopover_miner_...`), but back onto different stores: ORB's tools read the hosted loopover backend, AMS's tools read this machine's own local SQLite files (see [Local storage](#local-storage)) — a handful of AMS tools even name the ORB tool they mirror (e.g. `loopover_miner_get_run_state` is the read-only analog of `loopover_get_automation_state`) so the relationship is explicit at the point of use, not just here. +`loopover` exposes ORB's hosted contributor-workflow tools (issue ranking, PR packet prep, decision packs). `loopover-miner` exposes AMS's own local state-visibility tools listed above (portfolio dashboard, claims, audit feed, run state, plans, calibration) — a fully separate, 100% local tool surface with no shared code or network calls between the two. Both follow the same `loopover_*` tool-naming convention (`loopover_...` vs. `loopover_miner_...`), but back onto different stores: ORB's tools read the hosted loopover backend, AMS's tools read this machine's own local SQLite files (see [Local storage](#local-storage)) — a handful of AMS tools even name the ORB tool they mirror (e.g. `loopover_miner_get_run_state` is the read-only analog of `loopover_get_automation_state`) so the relationship is explicit at the point of use, not just here. ## Version check diff --git a/packages/loopover-miner/bin/loopover-miner-mcp.d.ts b/packages/loopover-miner/bin/loopover-miner-mcp.d.ts index 653e1474d2..9168980017 100644 --- a/packages/loopover-miner/bin/loopover-miner-mcp.d.ts +++ b/packages/loopover-miner/bin/loopover-miner-mcp.d.ts @@ -1,5 +1,6 @@ import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import type { EventLedger } from "../lib/event-ledger.js"; +import type { PredictionLedgerEntry } from "../lib/prediction-ledger.js"; /** The static, non-secret payload the loopover_miner_ping tool always returns, independent of input. */ export const MINER_PING_STATUS: { status: "ok"; tool: "loopover_miner_ping" }; @@ -52,13 +53,21 @@ export interface MinerMcpServerOptions { collectStatus?: () => unknown; /** Override the doctor-checks reader (defaults to status.js's runDoctorChecks); injection seam for tests. */ runDoctorChecks?: () => unknown[]; + /** + * Override the prediction-ledger opener (defaults to the real on-disk ledger); injection seam for tests. Typed + * to the minimal read surface the calibration-report tool uses (never appendPrediction). + */ + initPredictionLedger?: () => { + readPredictions(filter?: { repoFullName?: string | null }): PredictionLedgerEntry[]; + close(): void; + }; } /** * Build the miner MCP server with its tools registered (loopover_miner_ping, * loopover_miner_get_portfolio_dashboard, loopover_miner_list_claims, loopover_miner_get_audit_feed, * loopover_miner_get_run_state, loopover_miner_list_plans, loopover_miner_get_plan, - * loopover_miner_get_governor_decisions, loopover_miner_status). `options` supplies test injection seams; - * production callers pass nothing. + * loopover_miner_get_governor_decisions, loopover_miner_status, loopover_miner_get_calibration_report). `options` + * supplies test injection seams; production callers pass nothing. */ export function createMinerMcpServer(options?: MinerMcpServerOptions): McpServer; diff --git a/packages/loopover-miner/bin/loopover-miner-mcp.js b/packages/loopover-miner/bin/loopover-miner-mcp.js index fe2d95c7a9..4609ccada0 100755 --- a/packages/loopover-miner/bin/loopover-miner-mcp.js +++ b/packages/loopover-miner/bin/loopover-miner-mcp.js @@ -16,6 +16,9 @@ import { initRunStateStore } from "../lib/run-state.js"; import { PLAN_STATUSES, openPlanStore } from "../lib/plan-store.js"; import { initGovernorLedger } from "../lib/governor-ledger.js"; import { collectStatus, runDoctorChecks } from "../lib/status.js"; +import { buildCalibrationReport } from "../lib/calibration.js"; +import { toOutcomeRecords, toPredictionRecords } from "../lib/calibration-cli.js"; +import { initPredictionLedger } from "../lib/prediction-ledger.js"; // MCP stdio server for @loopover/miner (scaffold #5153). Mirrors the packages/loopover-mcp // harness (MCP SDK server + stdio transport). Tools: @@ -34,6 +37,10 @@ import { collectStatus, runDoctorChecks } from "../lib/status.js"; // governor-ledger.js's readGovernorDecisions -- an explicit named-column read that excludes payload_json. // - loopover_miner_status (#5154): read-only status + doctor diagnostics via status.js's collectStatus/ // runDoctorChecks (names/booleans/paths only -- never any env-var value, token, key, or credential). +// - loopover_miner_get_calibration_report (#5821): read-only miner-local prediction-accuracy report, joining +// the prediction ledger with observed pr_outcome events via calibration-cli.js's existing toPredictionRecords/ +// toOutcomeRecords mappers and calibration.js's buildCalibrationReport composer (no new join logic). Distinct +// from ORB's hosted, maintainer-authenticated loopover_get_outcome_calibration tool. // Read the version from this package's own package.json (always shipped) rather than a hand-synced // literal, so a release bump never has a second place to forget -- same approach as the mcp harness. @@ -263,6 +270,37 @@ export function createMinerMcpServer(options = {}) { return { content: [{ type: "text", text: JSON.stringify({ status, doctor }) }] }; }, ); + server.registerTool( + "loopover_miner_get_calibration_report", + { + description: + "Read-only miner-local prediction-accuracy report: per-project merge/close precision, joining this " + + "miner's own recorded gate predictions (prediction ledger) with the realized PR outcomes it later " + + "observed (pr_outcome events). Wraps calibration-cli.js's existing toPredictionRecords/toOutcomeRecords " + + "mappers and calibration.js's buildCalibrationReport composer -- no new join/scoring logic, no mutation. " + + "Strictly local and offline; distinct from ORB's hosted, maintainer-authenticated " + + "loopover_get_outcome_calibration tool, which reads a different (D1) data source. Takes no arguments.", + inputSchema: {}, + }, + async () => { + const ownsPredictionLedger = options.initPredictionLedger === undefined; + const ownsEventLedger = options.initEventLedger === undefined; + let predictionLedger; + let eventLedger; + try { + predictionLedger = (options.initPredictionLedger ?? initPredictionLedger)(); + eventLedger = (options.initEventLedger ?? initEventLedger)(); + const report = buildCalibrationReport( + toPredictionRecords(predictionLedger.readPredictions()), + toOutcomeRecords(eventLedger.readEvents()), + ); + return { content: [{ type: "text", text: JSON.stringify(report) }] }; + } finally { + if (ownsPredictionLedger) predictionLedger?.close(); + if (ownsEventLedger) eventLedger?.close(); + } + }, + ); return server; } diff --git a/packages/loopover-miner/lib/calibration-cli.d.ts b/packages/loopover-miner/lib/calibration-cli.d.ts index 56c839095e..e8d749add8 100644 --- a/packages/loopover-miner/lib/calibration-cli.d.ts +++ b/packages/loopover-miner/lib/calibration-cli.d.ts @@ -1 +1,9 @@ +import type { PredictedVerdictRecord, ObservedOutcomeRecord } from "./calibration-types.js"; +import type { PredictionLedgerEntry } from "./prediction-ledger.js"; +import type { LedgerEntry } from "./event-ledger.js"; + export function runCalibrationCli(args?: string[], env?: Record): number; + +export function toPredictionRecords(rows: PredictionLedgerEntry[]): PredictedVerdictRecord[]; + +export function toOutcomeRecords(events: LedgerEntry[]): ObservedOutcomeRecord[]; diff --git a/packages/loopover-miner/lib/calibration-cli.js b/packages/loopover-miner/lib/calibration-cli.js index 0e103ff7f5..4c2fb472f7 100644 --- a/packages/loopover-miner/lib/calibration-cli.js +++ b/packages/loopover-miner/lib/calibration-cli.js @@ -11,8 +11,9 @@ import { reportCliFailure, describeCliError } from "./cli-error.js"; const CALIBRATION_USAGE = "Usage: loopover-miner calibration [--json]"; /** Map prediction-ledger rows to predicted-verdict records: the target id becomes a string key and the recorded - * prediction verdict is the `conclusion`. */ -function toPredictionRecords(rows) { + * prediction verdict is the `conclusion`. Exported so callers other than this CLI (the MCP calibration-report + * tool, #5821) can build the identical join without re-implementing the mapping. */ +export function toPredictionRecords(rows) { return rows.map((row) => ({ project: row.repoFullName, targetId: String(row.targetId), @@ -23,8 +24,9 @@ function toPredictionRecords(rows) { /** Reduce the append-only `pr_outcome` event stream to the LATEST observed outcome per (repo, PR), as * observed-outcome records. `recordedAt` comes from the event's own timestamp (always present), so an outcome is - * never dropped for lacking a `closedAt`. Malformed payloads are skipped. */ -function toOutcomeRecords(events) { + * never dropped for lacking a `closedAt`. Malformed payloads are skipped. Exported for the same reason as + * {@link toPredictionRecords} above. */ +export function toOutcomeRecords(events) { const latest = new Map(); for (const event of events) { if (event?.type !== MINER_PR_OUTCOME_EVENT) continue; diff --git a/test/unit/miner-mcp-calibration-report.test.ts b/test/unit/miner-mcp-calibration-report.test.ts new file mode 100644 index 0000000000..24c41e4e6e --- /dev/null +++ b/test/unit/miner-mcp-calibration-report.test.ts @@ -0,0 +1,129 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { afterEach, describe, expect, it } from "vitest"; +import { createMinerMcpServer } from "../../packages/loopover-miner/bin/loopover-miner-mcp.js"; +import { initEventLedger } from "../../packages/loopover-miner/lib/event-ledger.js"; +import { initPredictionLedger } from "../../packages/loopover-miner/lib/prediction-ledger.js"; + +// loopover_miner_get_calibration_report (#5821): read-only wrapper joining the prediction ledger with pr_outcome +// events via calibration-cli.js's existing toPredictionRecords/toOutcomeRecords mappers and calibration.js's +// buildCalibrationReport composer. Driven against REAL temp stores (not fakes) so the has-signal/no-signal +// assertions exercise the actual join, mirroring miner-mcp-governor-decisions.test.ts's approach. + +type Content = { content: Array<{ type: string; text?: string }> }; +type PredictionLedgerHandle = ReturnType; +type EventLedgerHandle = ReturnType; + +const roots: string[] = []; +function tempPredictionLedger(): PredictionLedgerHandle { + const root = mkdtempSync(join(tmpdir(), "gittensory-miner-mcp-calibration-pred-")); + roots.push(root); + return initPredictionLedger(join(root, "prediction-ledger.sqlite3")); +} +function tempEventLedger(): EventLedgerHandle { + const root = mkdtempSync(join(tmpdir(), "gittensory-miner-mcp-calibration-event-")); + roots.push(root); + return initEventLedger(join(root, "event-ledger.sqlite3")); +} +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function toolText(result: Content): string { + const first = result.content[0]; + if (!first || first.type !== "text" || typeof first.text !== "string") { + throw new Error("expected a single text content block"); + } + return first.text; +} + +async function callCalibrationReport( + predictionLedger: PredictionLedgerHandle, + eventLedger: EventLedgerHandle, +): Promise { + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + const client = new Client({ name: "miner-mcp-calibration-test", version: "0.0.0" }); + await Promise.all([ + createMinerMcpServer({ + initPredictionLedger: () => predictionLedger, + initEventLedger: () => eventLedger, + }).connect(serverTransport), + client.connect(clientTransport), + ]); + const result = (await client.callTool({ + name: "loopover_miner_get_calibration_report", + arguments: {}, + })) as Content; + return JSON.parse(toolText(result)); +} + +describe("loopover_miner_get_calibration_report (#5821)", () => { + it("returns hasSignal: false with no rows when neither store has data (calibration-cli.js's own no-signal branch)", async () => { + const predictionLedger = tempPredictionLedger(); + const eventLedger = tempEventLedger(); + expect(await callCalibrationReport(predictionLedger, eventLedger)).toEqual({ hasSignal: false, rows: [] }); + }); + + it("joins a decided prediction with its realized outcome into a per-project row (has-signal branch)", async () => { + const predictionLedger = tempPredictionLedger(); + const eventLedger = tempEventLedger(); + predictionLedger.appendPrediction({ + repoFullName: "acme/widgets", + targetId: 42, + conclusion: "merge", + pack: "default", + engineVersion: "1.0.0", + }); + eventLedger.appendEvent({ + type: "pr_outcome", + repoFullName: "acme/widgets", + payload: { prNumber: 42, decision: "merged" }, + }); + + const report = (await callCalibrationReport(predictionLedger, eventLedger)) as { + hasSignal: boolean; + rows: Array>; + }; + expect(report.hasSignal).toBe(true); + expect(report.rows).toEqual([ + { + project: "acme/widgets", + wouldMerge: 1, + mergeConfirmed: 1, + mergeFalse: 0, + wouldClose: 0, + closeConfirmed: 0, + closeFalse: 0, + hold: 0, + decided: 1, + mergePrecision: 1, + closePrecision: null, + }, + ]); + }); + + it("does NOT count a prediction with no matching realized outcome yet (still pending)", async () => { + const predictionLedger = tempPredictionLedger(); + const eventLedger = tempEventLedger(); + predictionLedger.appendPrediction({ + repoFullName: "acme/widgets", + targetId: 7, + conclusion: "merge", + pack: "default", + engineVersion: "1.0.0", + }); + expect(await callCalibrationReport(predictionLedger, eventLedger)).toEqual({ hasSignal: false, rows: [] }); + }); + + it("does not close an injected store — the caller retains ownership (mirrors the sibling tools' seam contract)", async () => { + const predictionLedger = tempPredictionLedger(); + const eventLedger = tempEventLedger(); + await callCalibrationReport(predictionLedger, eventLedger); + // If the tool had closed either injected store, this read would throw against a closed native handle. + expect(() => predictionLedger.readPredictions()).not.toThrow(); + expect(() => eventLedger.readEvents()).not.toThrow(); + }); +}); diff --git a/test/unit/miner-mcp-contract.test.ts b/test/unit/miner-mcp-contract.test.ts index d7e2537d1e..c9bbb0939d 100644 --- a/test/unit/miner-mcp-contract.test.ts +++ b/test/unit/miner-mcp-contract.test.ts @@ -210,6 +210,23 @@ const READ_ONLY_TOOLS: ToolContract[] = [ corrupt: { initGovernorLedger: () => ({ readGovernorDecisions: readThrows, close() {} }) }, excluded: ["payload", "payload_json", "reputation", "self_plagiarism", "budget"], }, + { + tool: "loopover_miner_get_calibration_report", + args: {}, + valid: { + initPredictionLedger: () => ({ readPredictions: () => [], close() {} }), + initEventLedger: () => ({ dbPath: "", appendEvent: readThrows, readEvents: () => [], purgeByRepo: readThrows, close() {} }), + }, + missing: { + initPredictionLedger: openerThrows, + initEventLedger: () => ({ dbPath: "", appendEvent: readThrows, readEvents: () => [], purgeByRepo: readThrows, close() {} }), + }, + corrupt: { + initPredictionLedger: () => ({ readPredictions: readThrows, close() {} }), + initEventLedger: () => ({ dbPath: "", appendEvent: readThrows, readEvents: () => [], purgeByRepo: readThrows, close() {} }), + }, + excluded: [], + }, ]; describe("read-only AMS MCP tool contract (#5199)", () => { diff --git a/test/unit/miner-mcp-scaffold.test.ts b/test/unit/miner-mcp-scaffold.test.ts index f6fe29f359..b3584dc218 100644 --- a/test/unit/miner-mcp-scaffold.test.ts +++ b/test/unit/miner-mcp-scaffold.test.ts @@ -92,6 +92,7 @@ describe("loopover-miner MCP server (#5153 scaffold)", () => { const { tools } = await client.listTools(); expect(tools.map((tool) => tool.name).sort()).toEqual([ "loopover_miner_get_audit_feed", + "loopover_miner_get_calibration_report", "loopover_miner_get_governor_decisions", "loopover_miner_get_plan", "loopover_miner_get_portfolio_dashboard",