diff --git a/examples/demo-mcp/README.md b/examples/demo-mcp/README.md new file mode 100644 index 0000000..682248f --- /dev/null +++ b/examples/demo-mcp/README.md @@ -0,0 +1,48 @@ +# Demo MCP variants (#102) + +Tiny **stdio** MCP server with four interface variants for local tutorials and +CI smoke tests. No network dependencies. + +## Variants + +| Variant | Intent | Expected compare vs `v1` | +| --- | --- | --- | +| `v1` | Baseline | — | +| `v2-safe` | Additive (optional param + new tool) | Compatible / info–warning only | +| `v2-breaking` | Remove `search_notes` + required `folder` | Breaking | +| `v2-confusing` | Near-duplicate `find_notes` | Info adds; selection collisions | + +## Walkthrough + +```bash +# Capture each variant +for v in v1 v2-safe v2-breaking v2-confusing; do + tool-semantics capture-mcp -o ".tool-semantics/demo-$v.json" -- \ + python examples/demo-mcp/server.py "$v" +done + +# Safe candidate — should stay policy-compatible under default breaking gate +tool-semantics compare \ + .tool-semantics/demo-v1.json .tool-semantics/demo-v2-safe.json \ + --markdown-output .tool-semantics/demo-safe.md + +# Breaking candidate — expect tool.removed / parameter.added_required +tool-semantics compare \ + .tool-semantics/demo-v1.json .tool-semantics/demo-v2-breaking.json \ + --markdown-output .tool-semantics/demo-breaking.md + +# Confusing candidate — new overlapping tool for selection risk demos +tool-semantics compare \ + .tool-semantics/demo-v1.json .tool-semantics/demo-v2-confusing.json \ + --markdown-output .tool-semantics/demo-confusing.md +``` + +Scorecard / eval PRs can plug these snapshots into behavioral reports; structural +diffs alone already show the intended contrast. + +## Run the server alone + +```bash +DEMO_MCP_VARIANT=v2-breaking python examples/demo-mcp/server.py +# or: python examples/demo-mcp/server.py v2-confusing +``` diff --git a/examples/demo-mcp/server.py b/examples/demo-mcp/server.py new file mode 100644 index 0000000..7f12da6 --- /dev/null +++ b/examples/demo-mcp/server.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""Tiny demo MCP stdio server with interface variants for tutorials (#102). + +Variants (pass as argv[1] or DEMO_MCP_VARIANT): + v1 — baseline tools + v2-safe — additive / compatible changes only + v2-breaking — removed tool + required param + v2-confusing — near-duplicate tools that collide on selection +""" + +from __future__ import annotations + +import json +import os +import sys + +VARIANTS = ("v1", "v2-safe", "v2-breaking", "v2-confusing") + + +def read_message() -> dict: + header = b"" + while b"\r\n\r\n" not in header: + chunk = sys.stdin.buffer.read(1) + if not chunk: + raise EOFError + header += chunk + content_length = None + for line in header.decode().split("\r\n"): + if line.lower().startswith("content-length:"): + content_length = int(line.split(":", 1)[1].strip()) + if content_length is None: + raise RuntimeError("missing content-length") + body = sys.stdin.buffer.read(content_length) + return json.loads(body.decode()) + + +def write_message(payload: dict) -> None: + raw = json.dumps(payload).encode() + sys.stdout.buffer.write(f"Content-Length: {len(raw)}\r\n\r\n".encode() + raw) + sys.stdout.buffer.flush() + + +def _tools_for(variant: str) -> list[dict]: + search = { + "name": "search_notes", + "description": "Search personal notes by keyword.", + "inputSchema": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query"}, + }, + "required": ["query"], + }, + } + create = { + "name": "create_note", + "description": "Create a personal note.", + "inputSchema": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "body": {"type": "string"}, + }, + "required": ["title", "body"], + }, + } + if variant == "v1": + return [search, create] + if variant == "v2-safe": + # Additive: optional tag on create; new read-only list tool. + create_safe = { + **create, + "inputSchema": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "body": {"type": "string"}, + "tag": {"type": "string"}, + }, + "required": ["title", "body"], + }, + } + list_notes = { + "name": "list_notes", + "description": "List recent personal notes.", + "inputSchema": {"type": "object", "properties": {}}, + } + return [search, create_safe, list_notes] + if variant == "v2-breaking": + # Remove search_notes; require folder on create_note. + create_breaking = { + **create, + "inputSchema": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "body": {"type": "string"}, + "folder": {"type": "string"}, + }, + "required": ["title", "body", "folder"], + }, + } + return [create_breaking] + if variant == "v2-confusing": + # Keep search_notes but add a near-duplicate find_notes. + find = { + "name": "find_notes", + "description": "Search personal notes by keyword.", + "inputSchema": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query"}, + }, + "required": ["query"], + }, + } + return [search, find, create] + raise ValueError(f"Unknown variant {variant!r}; expected one of {VARIANTS}") + + +def main() -> None: + variant = ( + (sys.argv[1] if len(sys.argv) > 1 else None) or os.environ.get("DEMO_MCP_VARIANT") or "v1" + ) + if variant not in VARIANTS: + raise SystemExit(f"Unknown variant {variant!r}; expected one of {VARIANTS}") + tools = _tools_for(variant) + + while True: + try: + message = read_message() + except EOFError: + return + method = message.get("method") + request_id = message.get("id") + if method == "initialize": + write_message( + { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "demo-mcp", "version": variant}, + }, + } + ) + elif method == "notifications/initialized": + continue + elif method == "tools/list": + write_message( + { + "jsonrpc": "2.0", + "id": request_id, + "result": {"tools": tools}, + } + ) + elif method in {"prompts/list", "resources/list"}: + key = "prompts" if method.startswith("prompts") else "resources" + write_message({"jsonrpc": "2.0", "id": request_id, "result": {key: []}}) + elif request_id is not None: + write_message( + { + "jsonrpc": "2.0", + "id": request_id, + "error": {"code": -32601, "message": f"Unknown method {method}"}, + } + ) + + +if __name__ == "__main__": + main() diff --git a/tests/test_demo_mcp.py b/tests/test_demo_mcp.py new file mode 100644 index 0000000..1e925ad --- /dev/null +++ b/tests/test_demo_mcp.py @@ -0,0 +1,48 @@ +"""CI smoke: capture demo-mcp stdio variants and compare (#102).""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from tool_semantics.diff import Severity, compare_snapshots +from tool_semantics.mcp_capture import capture_mcp_stdio + +SERVER = Path(__file__).resolve().parents[1] / "examples" / "demo-mcp" / "server.py" + + +def _capture(variant: str): + return capture_mcp_stdio([sys.executable, str(SERVER), variant]) + + +def test_demo_mcp_variants_capture_and_compare() -> None: + v1 = _capture("v1") + safe = _capture("v2-safe") + breaking = _capture("v2-breaking") + confusing = _capture("v2-confusing") + + assert v1.server_name == "demo-mcp" + assert {tool.name for tool in v1.tools} == {"search_notes", "create_note"} + + safe_report = compare_snapshots(v1, safe) + assert safe_report.is_compatible + assert any(change.code == "tool.added" for change in safe_report.changes) + + breaking_report = compare_snapshots(v1, breaking) + assert not breaking_report.is_compatible + assert any( + change.code == "tool.removed" and change.subject == "search_notes" + for change in breaking_report.changes + ) + assert any( + change.severity == Severity.BREAKING and change.code == "parameter.added_required" + for change in breaking_report.changes + ) + + confusing_report = compare_snapshots(v1, confusing) + assert any( + change.code == "tool.added" and change.subject == "find_notes" + for change in confusing_report.changes + ) + # Additive only → still structurally compatible under default gate + assert confusing_report.is_compatible