Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 49 additions & 1 deletion src/mcp/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ import {
buildTestGenSpec,
type LocalWriteActionSpec,
} from "./local-write-tools";
import { TEST_FRAMEWORKS } from "../signals/test-evidence";
import { classifyTestCoverage, isCodeFile, isTestPath, TEST_FRAMEWORKS } from "../signals/test-evidence";
import { applyStepResult, buildPlanDag, nextReadySteps, planProgress, validatePlanDag, type PlanDag } from "../services/plan-dag";
import { buildFocusManifestValidation } from "../services/focus-manifest-validation";
import { isGlobalAgentPause, resolveAgentActionMode, resolveAgentPermissionReadiness } from "../settings/agent-execution";
Expand Down Expand Up @@ -852,6 +852,21 @@ const checkSlopRiskOutputSchema = {
rubric: z.string().optional(),
};

// Coverage-gap self-check (#2235): pure local-metadata, like checkSlopRisk — the agent supplies its changed
// paths (plus any test paths) and asks whether the change carries enough test evidence, no source uploaded.
const checkTestEvidenceShape = {
changedPaths: z.array(z.string().min(1).max(400)).max(2000),
testFiles: z.array(z.string().min(1).max(400)).max(2000).optional(),
};

const checkTestEvidenceOutputSchema = {
classification: z.enum(["strong", "adequate", "weak", "absent"]).optional(),
changedFileCount: z.number().optional(),
codeFileCount: z.number().optional(),
testFileCount: z.number().optional(),
guidance: z.array(z.string()).optional(),
};

// Issue-side slop triage (#533): pure local-metadata, like checkSlopRisk — the agent supplies the issue
// title + body, nothing to scope. Advisory-only; issues never block.
const checkIssueSlopShape = {
Expand Down Expand Up @@ -1433,6 +1448,17 @@ export class GittensoryMcp {
async (input) => this.toolResult(await this.checkSlopRisk(input)),
);

server.registerTool(
"gittensory_check_test_evidence",
{
description:
"Classify whether a planned change's changed files carry enough test evidence, from path metadata alone (no source uploaded) — an agent-native coverage-gap self-check before opening a PR. Returns a coverage band (strong/adequate/weak/absent) plus actionable guidance.",
inputSchema: checkTestEvidenceShape,
outputSchema: checkTestEvidenceOutputSchema,
},
async (input) => this.toolResult(await this.checkTestEvidence(input)),
);

server.registerTool(
"gittensory_check_issue_slop",
{
Expand Down Expand Up @@ -2619,6 +2645,28 @@ export class GittensoryMcp {
};
}

private async checkTestEvidence(input: z.infer<z.ZodObject<typeof checkTestEvidenceShape>>): Promise<ToolPayload> {
await this.enforceToolRateLimit("gittensory_check_test_evidence");
const allPaths = [...input.changedPaths, ...(input.testFiles ?? [])];
const classification = classifyTestCoverage(allPaths);
const codeFileCount = input.changedPaths.filter(isCodeFile).length;
const testFileCount = allPaths.filter(isTestPath).length;
const guidance: string[] = [];
if (codeFileCount === 0) {
guidance.push("No hand-authored code files changed, so the missing-test-evidence signal does not apply (e.g. a docs- or config-only change).");
} else if (classification === "absent") {
guidance.push("Changed code files carry no test evidence — add or update a test that exercises the change before opening the PR.");
} else if (classification === "strong") {
guidance.push("Test coverage looks strong for this change.");
} else {
guidance.push(`Test coverage is ${classification} for this change — adding another focused test would strengthen the evidence.`);
}
return {
summary: `Test evidence: ${classification}.`,
data: { classification, changedFileCount: allPaths.length, codeFileCount, testFileCount, guidance } as unknown as Record<string, unknown>,
};
}

private async checkIssueSlop(input: z.infer<z.ZodObject<typeof checkIssueSlopShape>>): Promise<ToolPayload> {
await this.enforceToolRateLimit("gittensory_check_issue_slop");
const assessment = buildIssueSlopAssessment(input);
Expand Down
66 changes: 66 additions & 0 deletions test/unit/mcp-check-test-evidence.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
import { describe, expect, it } from "vitest";
import { GittensoryMcp } from "../../src/mcp/server";
import { createTestEnv } from "../helpers/d1";

async function connect() {
const server = new GittensoryMcp(createTestEnv()).createServer();
const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
await server.connect(serverTransport);
const client = new Client({ name: "gittensory-test-evidence-test", version: "0.1.0" }, { capabilities: {} });
await client.connect(clientTransport);
return client;
}

type Result = { classification: string; changedFileCount: number; codeFileCount: number; testFileCount: number; guidance: string[] };

describe("MCP gittensory_check_test_evidence (#2235)", () => {
it("flags code changes with no tests as absent (no source/auth needed)", async () => {
const client = await connect();
const result = await client.callTool({ name: "gittensory_check_test_evidence", arguments: { changedPaths: ["src/a.ts", "src/b.ts"] } });
expect(result.isError).toBeFalsy();
const data = result.structuredContent as Result;
expect(data.classification).toBe("absent");
expect(data.codeFileCount).toBe(2);
expect(data.testFileCount).toBe(0);
expect(data.guidance.join(" ")).toMatch(/no test evidence/i);
expect(JSON.stringify(data)).not.toMatch(/wallet|hotkey|reward|payout|trust score/i);
});

it("classifies a well-tested change as strong", async () => {
const client = await connect();
// 1 test / 2 total = 0.5 → strong
const result = await client.callTool({ name: "gittensory_check_test_evidence", arguments: { changedPaths: ["src/a.ts"], testFiles: ["test/a.test.ts"] } });
const data = result.structuredContent as Result;
expect(data.classification).toBe("strong");
expect(data.testFileCount).toBe(1);
expect(data.guidance.join(" ")).toMatch(/strong/i);
});

it("classifies a lightly-tested change as adequate", async () => {
const client = await connect();
// 1 test / 5 total = 0.2 → adequate
const result = await client.callTool({ name: "gittensory_check_test_evidence", arguments: { changedPaths: ["src/a.ts", "src/b.ts", "src/c.ts", "src/d.ts", "test/a.test.ts"] } });
const data = result.structuredContent as Result;
expect(data.classification).toBe("adequate");
expect(data.guidance.join(" ")).toMatch(/adequate/i);
});

it("classifies a barely-tested change as weak", async () => {
const client = await connect();
// 1 test / 6 total ≈ 0.17 → weak
const result = await client.callTool({ name: "gittensory_check_test_evidence", arguments: { changedPaths: ["src/a.ts", "src/b.ts", "src/c.ts", "src/d.ts", "src/e.ts"], testFiles: ["test/a.test.ts"] } });
const data = result.structuredContent as Result;
expect(data.classification).toBe("weak");
expect(data.guidance.join(" ")).toMatch(/weak/i);
});

it("treats a docs-only change as not applicable (no code files)", async () => {
const client = await connect();
const result = await client.callTool({ name: "gittensory_check_test_evidence", arguments: { changedPaths: ["README.md", "docs/guide.md"] } });
const data = result.structuredContent as Result;
expect(data.codeFileCount).toBe(0);
expect(data.guidance.join(" ")).toMatch(/does not apply/i);
});
});