From 0d0f26a6ce117beee5c0bc6415de7987cfc96d36 Mon Sep 17 00:00:00 2001 From: jaso0n0818 Date: Wed, 24 Jun 2026 05:09:22 +0000 Subject: [PATCH 1/3] feat(mcp): expose outcome calibration via gittensory_get_outcome_calibration Wire the slop-band and recommendation outcome calibration service into MCP so maintainers can inspect predictive accuracy without a separate HTTP call. Includes structured output schema validation tests. Co-authored-by: Cursor --- src/mcp/server.ts | 49 ++++++++++++++++++++++++++++ test/integration/api.test.ts | 1 + test/unit/mcp-output-schemas.test.ts | 28 +++++++++++++++- 3 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index b57b117e83..f6514abcf8 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -68,6 +68,7 @@ import { loadOrComputeIssueQualityResponse } from "../services/issue-quality"; import { loadOrComputeBurdenForecastResponse } from "../services/burden-forecast"; import { buildMcpClientTelemetry } from "../services/client-telemetry"; import { loadOrComputeRepoOutcomePatternsResponse } from "../services/repo-outcome-patterns"; +import { buildRepoOutcomeCalibration } from "../services/outcome-calibration"; import { buildUnavailableQueueTrendReport } from "../services/queue-trends"; import { applyMcpPlanningChoices, @@ -137,6 +138,12 @@ const ownerRepoShape = { repo: z.string().min(1), }; +const ownerRepoWindowShape = { + owner: z.string().min(1), + repo: z.string().min(1), + windowDays: z.number().int().positive().optional(), +}; + const loginShape = { login: z.string().min(1), }; @@ -587,6 +594,16 @@ const freshnessResponseOutputSchema = { report: z.unknown().optional(), }; +const maintainerMeasurementReportOutputSchema = { + repoFullName: z.string().optional(), + generatedAt: z.string().optional(), + windowDays: z.number().nullable().optional(), + slop: z.unknown().optional(), + recommendations: z.unknown().optional(), + signals: z.array(z.string()).optional(), + status: z.string().optional(), +}; + const contributorProfileOutputSchema = { login: z.string().optional(), github: z.unknown().optional(), @@ -1026,6 +1043,17 @@ export class GittensoryMcp { async (input) => this.toolResult(await this.getRepoOutcomePatterns(input)), ); + server.registerTool( + "gittensory_get_outcome_calibration", + { + description: + "Return slop-band and recommendation outcome calibration for a repo: whether higher-slop bands merge less often and how agent recommendations are panning out. Maintainer-authenticated; measurement only.", + inputSchema: ownerRepoWindowShape, + outputSchema: maintainerMeasurementReportOutputSchema, + }, + async (input) => this.toolResult(await this.getOutcomeCalibration(input)), + ); + server.registerTool( "gittensory_get_contributor_profile", { @@ -1884,6 +1912,27 @@ export class GittensoryMcp { }; } + private async getOutcomeCalibration(input: { owner: string; repo: string; windowDays?: number | undefined }): Promise { + const fullName = `${input.owner}/${input.repo}`; + await this.requireRepoAccess(fullName); + const report = await buildRepoOutcomeCalibration( + this.env, + fullName, + input.windowDays !== undefined ? input.windowDays : undefined, + ); + const slop = report.slop; + const summary = + slop.discriminates === true + ? `Outcome calibration for ${fullName}: slop bands are predictive across ${slop.totalResolved} resolved PRs.` + : slop.discriminates === false + ? `Outcome calibration for ${fullName}: slop bands are NOT discriminating on current data (${slop.totalResolved} resolved PRs).` + : `Outcome calibration for ${fullName}: not enough resolved PR data to judge slop calibration yet.`; + return { + summary, + data: report as unknown as Record, + }; + } + private async loadOpenQueueCounts(fullName: string): Promise<{ openIssues: number; openPullRequests: number }> { const [totals, openIssues, openPullRequests] = await Promise.all([ getLatestRepoGithubTotalsSnapshot(this.env, fullName), diff --git a/test/integration/api.test.ts b/test/integration/api.test.ts index 7f2bf062a6..f91cad1263 100644 --- a/test/integration/api.test.ts +++ b/test/integration/api.test.ts @@ -4872,6 +4872,7 @@ describe("api routes", () => { expect(toolNames).toContain("gittensory_preflight_local_diff"); expect(toolNames).toContain("gittensory_preview_local_pr_score"); expect(toolNames).toContain("gittensory_explain_score_breakdown"); + expect(toolNames).toContain("gittensory_get_outcome_calibration"); expect(toolNames).toContain("gittensory_get_registry_changes"); expect(toolNames).toContain("gittensory_get_upstream_drift"); expect(toolNames).toContain("gittensory_explain_review_risk"); diff --git a/test/unit/mcp-output-schemas.test.ts b/test/unit/mcp-output-schemas.test.ts index 9c505bbd84..02eb3c9626 100644 --- a/test/unit/mcp-output-schemas.test.ts +++ b/test/unit/mcp-output-schemas.test.ts @@ -1,7 +1,7 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; import { describe, expect, it } from "vitest"; -import { persistSignalSnapshot, upsertBounty, upsertIssueFromGitHub, upsertRepositoryFromGitHub } from "../../src/db/repositories"; +import { persistSignalSnapshot, upsertBounty, upsertIssueFromGitHub, upsertPullRequestFromGitHub, upsertRepositoryFromGitHub, updatePullRequestSlopAssessment } from "../../src/db/repositories"; import { GittensoryMcp } from "../../src/mcp/server"; import { normalizeRegistryPayload } from "../../src/registry/normalize"; import { persistRegistrySnapshot } from "../../src/registry/sync"; @@ -13,6 +13,7 @@ const TOOLS_WITH_OUTPUT_SCHEMA = [ "gittensory_get_repo_context", "gittensory_get_burden_forecast", "gittensory_get_repo_outcome_patterns", + "gittensory_get_outcome_calibration", "gittensory_get_contributor_profile", "gittensory_get_decision_pack", "gittensory_monitor_open_prs", @@ -380,6 +381,31 @@ describe("MCP tool calls return schema-valid structured content", () => { expect(cached.isError).toBeFalsy(); expect(cached.structuredContent).toMatchObject({ status: "ready", source: "snapshot", freshness: "fresh", repoFullName: "owner/cached" }); }); + + it("gittensory_get_outcome_calibration returns structured slop calibration for a repo", async () => { + const env = createTestEnv(); + await upsertRepositoryFromGitHub(env, { name: "demo", full_name: "octo/demo", private: false, owner: { login: "octo" }, default_branch: "main" }); + await upsertPullRequestFromGitHub(env, "octo/demo", { + number: 1, + title: "merged clean", + state: "closed", + user: { login: "alice" }, + merged_at: "2026-06-01T00:00:00.000Z", + }); + await updatePullRequestSlopAssessment(env, "octo/demo", 1, { slopRisk: 0, slopBand: "clean" }); + const { client } = await connectTestClient(env); + const result = await client.callTool({ + name: "gittensory_get_outcome_calibration", + arguments: { owner: "octo", repo: "demo", windowDays: 30 }, + }); + expect(result.isError).toBeFalsy(); + const data = result.structuredContent as Record; + expect(data.repoFullName).toBe("octo/demo"); + expect(data.windowDays).toBe(30); + expect(data.slop).toBeTruthy(); + expect(data.recommendations).toBeTruthy(); + expect(Array.isArray(data.signals)).toBe(true); + }); }); // ── Public/private safety ───────────────────────────────────────────────────── From 9be17a82830bb6bc54733a42dc694e204fc6dbe9 Mon Sep 17 00:00:00 2001 From: jaso0n0818 Date: Wed, 24 Jun 2026 05:19:59 +0000 Subject: [PATCH 2/3] fix(mcp): guard outcome-calibration summary when slop data is absent Use optional chaining on the slop calibration slice so the MCP tool returns a safe fallback summary instead of throwing on sparse reports. Co-authored-by: Cursor --- src/mcp/server.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index f6514abcf8..805d600ed9 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -1921,11 +1921,12 @@ export class GittensoryMcp { input.windowDays !== undefined ? input.windowDays : undefined, ); const slop = report.slop; + const totalResolved = slop?.totalResolved ?? 0; const summary = - slop.discriminates === true - ? `Outcome calibration for ${fullName}: slop bands are predictive across ${slop.totalResolved} resolved PRs.` - : slop.discriminates === false - ? `Outcome calibration for ${fullName}: slop bands are NOT discriminating on current data (${slop.totalResolved} resolved PRs).` + slop?.discriminates === true + ? `Outcome calibration for ${fullName}: slop bands are predictive across ${totalResolved} resolved PRs.` + : slop?.discriminates === false + ? `Outcome calibration for ${fullName}: slop bands are NOT discriminating on current data (${totalResolved} resolved PRs).` : `Outcome calibration for ${fullName}: not enough resolved PR data to judge slop calibration yet.`; return { summary, From 7cdec26827b337e8aa12e8e11d52a596faafd866 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 23 Jun 2026 23:27:41 -0700 Subject: [PATCH 3/3] refactor(mcp): derive the outcome-calibration summary from a tested pure helper --- src/mcp/server.ts | 18 +++-------------- src/services/outcome-calibration.ts | 7 +++++++ test/unit/outcome-calibration.test.ts | 29 +++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 805d600ed9..bddf7263de 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -68,7 +68,7 @@ import { loadOrComputeIssueQualityResponse } from "../services/issue-quality"; import { loadOrComputeBurdenForecastResponse } from "../services/burden-forecast"; import { buildMcpClientTelemetry } from "../services/client-telemetry"; import { loadOrComputeRepoOutcomePatternsResponse } from "../services/repo-outcome-patterns"; -import { buildRepoOutcomeCalibration } from "../services/outcome-calibration"; +import { buildRepoOutcomeCalibration, outcomeCalibrationSummary } from "../services/outcome-calibration"; import { buildUnavailableQueueTrendReport } from "../services/queue-trends"; import { applyMcpPlanningChoices, @@ -1915,21 +1915,9 @@ export class GittensoryMcp { private async getOutcomeCalibration(input: { owner: string; repo: string; windowDays?: number | undefined }): Promise { const fullName = `${input.owner}/${input.repo}`; await this.requireRepoAccess(fullName); - const report = await buildRepoOutcomeCalibration( - this.env, - fullName, - input.windowDays !== undefined ? input.windowDays : undefined, - ); - const slop = report.slop; - const totalResolved = slop?.totalResolved ?? 0; - const summary = - slop?.discriminates === true - ? `Outcome calibration for ${fullName}: slop bands are predictive across ${totalResolved} resolved PRs.` - : slop?.discriminates === false - ? `Outcome calibration for ${fullName}: slop bands are NOT discriminating on current data (${totalResolved} resolved PRs).` - : `Outcome calibration for ${fullName}: not enough resolved PR data to judge slop calibration yet.`; + const report = await buildRepoOutcomeCalibration(this.env, fullName, input.windowDays); return { - summary, + summary: outcomeCalibrationSummary(fullName, report.slop), data: report as unknown as Record, }; } diff --git a/src/services/outcome-calibration.ts b/src/services/outcome-calibration.ts index 7681c0f803..1f13ea0a42 100644 --- a/src/services/outcome-calibration.ts +++ b/src/services/outcome-calibration.ts @@ -130,6 +130,13 @@ function sameRepo(a: string | null | undefined, b: string): boolean { return (a ?? "").toLowerCase() === b.toLowerCase(); } +/** One-line human summary of a repo's slop-band calibration verdict (mirrors the discriminates signal). Pure. */ +export function outcomeCalibrationSummary(fullName: string, slop: SlopOutcomeCalibration): string { + if (slop.discriminates === true) return `Outcome calibration for ${fullName}: slop bands are predictive across ${slop.totalResolved} resolved PRs.`; + if (slop.discriminates === false) return `Outcome calibration for ${fullName}: slop bands are NOT discriminating on current data (${slop.totalResolved} resolved PRs).`; + return `Outcome calibration for ${fullName}: not enough resolved PR data to judge slop calibration yet.`; +} + /** Load a repo's PRs + recommendation outcomes and assemble the calibration report. */ export async function buildRepoOutcomeCalibration(env: Env, repoFullName: string, windowDays?: number): Promise { const [pullRequests, outcomes] = await Promise.all([ diff --git a/test/unit/outcome-calibration.test.ts b/test/unit/outcome-calibration.test.ts index c901439a43..bebcb41d72 100644 --- a/test/unit/outcome-calibration.test.ts +++ b/test/unit/outcome-calibration.test.ts @@ -4,6 +4,8 @@ import { buildRecommendationOutcomeCalibration, buildRepoOutcomeCalibration, buildSlopOutcomeCalibration, + outcomeCalibrationSummary, + type SlopOutcomeCalibration, } from "../../src/services/outcome-calibration"; import { createAgentRun, replaceAgentActions, updatePullRequestSlopAssessment, upsertAgentRecommendationOutcome, upsertPullRequestFromGitHub } from "../../src/db/repositories"; import type { SlopBand } from "../../src/signals/slop"; @@ -216,3 +218,30 @@ function actionRecord(id: string, runId: string): AgentActionRecord { createdAt: "2026-06-01T00:00:00.000Z", }; } + +describe("outcomeCalibrationSummary", () => { + const slop = (discriminates: boolean | null, totalResolved: number): SlopOutcomeCalibration => ({ + totalResolved, + bands: [], + overallMergeRate: null, + discriminates, + }); + + it("reports a predictive verdict when bands discriminate", () => { + expect(outcomeCalibrationSummary("octo/demo", slop(true, 12))).toBe( + "Outcome calibration for octo/demo: slop bands are predictive across 12 resolved PRs.", + ); + }); + + it("reports a non-discriminating verdict when bands invert", () => { + expect(outcomeCalibrationSummary("octo/demo", slop(false, 11))).toBe( + "Outcome calibration for octo/demo: slop bands are NOT discriminating on current data (11 resolved PRs).", + ); + }); + + it("reports an insufficient-data verdict when discrimination cannot be judged", () => { + expect(outcomeCalibrationSummary("octo/demo", slop(null, 2))).toBe( + "Outcome calibration for octo/demo: not enough resolved PR data to judge slop calibration yet.", + ); + }); +});