From c2be4386aa40e1bc2df93ee1d5593a335c423e14 Mon Sep 17 00:00:00 2001 From: Jeff <158072326+jeffrey701@users.noreply.github.com> Date: Wed, 8 Jul 2026 19:24:17 +0100 Subject: [PATCH] feat(mcp): add gittensory_check_test_evidence tool (deterministic coverage-gap self-check) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surface the engine's changed-path->test-evidence classifier (classifyTestCoverage, src/signals/test-evidence.ts) as a metadata-only MCP tool so an agent can ask whether its changed files carry enough test evidence before opening a PR — paths in, coverage band (strong/adequate/weak/absent) + guidance out, no source uploaded. Mirrors the existing gittensory_check_slop_risk source-free self-checks. Closes #2235 --- src/mcp/server.ts | 50 ++++++++++++++++- test/unit/mcp-check-test-evidence.test.ts | 66 +++++++++++++++++++++++ 2 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 test/unit/mcp-check-test-evidence.test.ts diff --git a/src/mcp/server.ts b/src/mcp/server.ts index efbca2a444..e261c18a82 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -135,7 +135,7 @@ import { buildTestGenSpec, type LocalWriteActionSpec, } from "./local-write-tools"; -import { TEST_FRAMEWORKS } from "../signals/test-evidence"; +import { classifyTestCoverage, isCodeFile, isTestPath, TEST_FRAMEWORKS } from "../signals/test-evidence"; import { applyStepResult, buildPlanDag, nextReadySteps, planProgress, validatePlanDag, type PlanDag } from "../services/plan-dag"; import { buildFocusManifestValidation } from "../services/focus-manifest-validation"; import { isGlobalAgentPause, resolveAgentActionMode, resolveAgentPermissionReadiness } from "../settings/agent-execution"; @@ -852,6 +852,21 @@ const checkSlopRiskOutputSchema = { rubric: z.string().optional(), }; +// Coverage-gap self-check (#2235): pure local-metadata, like checkSlopRisk — the agent supplies its changed +// paths (plus any test paths) and asks whether the change carries enough test evidence, no source uploaded. +const checkTestEvidenceShape = { + changedPaths: z.array(z.string().min(1).max(400)).max(2000), + testFiles: z.array(z.string().min(1).max(400)).max(2000).optional(), +}; + +const checkTestEvidenceOutputSchema = { + classification: z.enum(["strong", "adequate", "weak", "absent"]).optional(), + changedFileCount: z.number().optional(), + codeFileCount: z.number().optional(), + testFileCount: z.number().optional(), + guidance: z.array(z.string()).optional(), +}; + // Issue-side slop triage (#533): pure local-metadata, like checkSlopRisk — the agent supplies the issue // title + body, nothing to scope. Advisory-only; issues never block. const checkIssueSlopShape = { @@ -1433,6 +1448,17 @@ export class GittensoryMcp { async (input) => this.toolResult(await this.checkSlopRisk(input)), ); + server.registerTool( + "gittensory_check_test_evidence", + { + description: + "Classify whether a planned change's changed files carry enough test evidence, from path metadata alone (no source uploaded) — an agent-native coverage-gap self-check before opening a PR. Returns a coverage band (strong/adequate/weak/absent) plus actionable guidance.", + inputSchema: checkTestEvidenceShape, + outputSchema: checkTestEvidenceOutputSchema, + }, + async (input) => this.toolResult(await this.checkTestEvidence(input)), + ); + server.registerTool( "gittensory_check_issue_slop", { @@ -2619,6 +2645,28 @@ export class GittensoryMcp { }; } + private async checkTestEvidence(input: z.infer>): Promise { + await this.enforceToolRateLimit("gittensory_check_test_evidence"); + const allPaths = [...input.changedPaths, ...(input.testFiles ?? [])]; + const classification = classifyTestCoverage(allPaths); + const codeFileCount = input.changedPaths.filter(isCodeFile).length; + const testFileCount = allPaths.filter(isTestPath).length; + const guidance: string[] = []; + if (codeFileCount === 0) { + guidance.push("No hand-authored code files changed, so the missing-test-evidence signal does not apply (e.g. a docs- or config-only change)."); + } else if (classification === "absent") { + guidance.push("Changed code files carry no test evidence — add or update a test that exercises the change before opening the PR."); + } else if (classification === "strong") { + guidance.push("Test coverage looks strong for this change."); + } else { + guidance.push(`Test coverage is ${classification} for this change — adding another focused test would strengthen the evidence.`); + } + return { + summary: `Test evidence: ${classification}.`, + data: { classification, changedFileCount: allPaths.length, codeFileCount, testFileCount, guidance } as unknown as Record, + }; + } + private async checkIssueSlop(input: z.infer>): Promise { await this.enforceToolRateLimit("gittensory_check_issue_slop"); const assessment = buildIssueSlopAssessment(input); diff --git a/test/unit/mcp-check-test-evidence.test.ts b/test/unit/mcp-check-test-evidence.test.ts new file mode 100644 index 0000000000..91b2105f8e --- /dev/null +++ b/test/unit/mcp-check-test-evidence.test.ts @@ -0,0 +1,66 @@ +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { describe, expect, it } from "vitest"; +import { GittensoryMcp } from "../../src/mcp/server"; +import { createTestEnv } from "../helpers/d1"; + +async function connect() { + const server = new GittensoryMcp(createTestEnv()).createServer(); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + const client = new Client({ name: "gittensory-test-evidence-test", version: "0.1.0" }, { capabilities: {} }); + await client.connect(clientTransport); + return client; +} + +type Result = { classification: string; changedFileCount: number; codeFileCount: number; testFileCount: number; guidance: string[] }; + +describe("MCP gittensory_check_test_evidence (#2235)", () => { + it("flags code changes with no tests as absent (no source/auth needed)", async () => { + const client = await connect(); + const result = await client.callTool({ name: "gittensory_check_test_evidence", arguments: { changedPaths: ["src/a.ts", "src/b.ts"] } }); + expect(result.isError).toBeFalsy(); + const data = result.structuredContent as Result; + expect(data.classification).toBe("absent"); + expect(data.codeFileCount).toBe(2); + expect(data.testFileCount).toBe(0); + expect(data.guidance.join(" ")).toMatch(/no test evidence/i); + expect(JSON.stringify(data)).not.toMatch(/wallet|hotkey|reward|payout|trust score/i); + }); + + it("classifies a well-tested change as strong", async () => { + const client = await connect(); + // 1 test / 2 total = 0.5 → strong + const result = await client.callTool({ name: "gittensory_check_test_evidence", arguments: { changedPaths: ["src/a.ts"], testFiles: ["test/a.test.ts"] } }); + const data = result.structuredContent as Result; + expect(data.classification).toBe("strong"); + expect(data.testFileCount).toBe(1); + expect(data.guidance.join(" ")).toMatch(/strong/i); + }); + + it("classifies a lightly-tested change as adequate", async () => { + const client = await connect(); + // 1 test / 5 total = 0.2 → adequate + const result = await client.callTool({ name: "gittensory_check_test_evidence", arguments: { changedPaths: ["src/a.ts", "src/b.ts", "src/c.ts", "src/d.ts", "test/a.test.ts"] } }); + const data = result.structuredContent as Result; + expect(data.classification).toBe("adequate"); + expect(data.guidance.join(" ")).toMatch(/adequate/i); + }); + + it("classifies a barely-tested change as weak", async () => { + const client = await connect(); + // 1 test / 6 total ≈ 0.17 → weak + const result = await client.callTool({ name: "gittensory_check_test_evidence", arguments: { changedPaths: ["src/a.ts", "src/b.ts", "src/c.ts", "src/d.ts", "src/e.ts"], testFiles: ["test/a.test.ts"] } }); + const data = result.structuredContent as Result; + expect(data.classification).toBe("weak"); + expect(data.guidance.join(" ")).toMatch(/weak/i); + }); + + it("treats a docs-only change as not applicable (no code files)", async () => { + const client = await connect(); + const result = await client.callTool({ name: "gittensory_check_test_evidence", arguments: { changedPaths: ["README.md", "docs/guide.md"] } }); + const data = result.structuredContent as Result; + expect(data.codeFileCount).toBe(0); + expect(data.guidance.join(" ")).toMatch(/does not apply/i); + }); +});