feat(bench): correction rounds and multi-reference paired lift for the agent benchmark

benchmarks/agent/agent_bench.py

@@ -99,6 +99,12 @@ def baseline_path(path: str) -> str:
     return os.pathsep.join(kept)
 
 
+def references_of(args: argparse.Namespace) -> list[str]:
+    """The paired-comparison reference labels: `action="append"` collects them; a programmatic caller may pass a bare string."""
+    value = getattr(args, "reference", None) or bench_report.BASELINE
+    return value if isinstance(value, list) else [value]
+
+
 def normalize_cell(raw: dict) -> dict:
     return {"tool": raw["tool"], "level": raw.get("level") or "bin", "skills": sorted(raw.get("skills") or []), "knowledge": raw["knowledge"], "network": raw["network"]}
 
@@ -455,6 +461,7 @@ def run_trial(scenario: dict, cell: dict, args: argparse.Namespace, out: Path) -
     entry = workspace / scenario["entry"]
     final_sha = hashlib.sha256(entry.read_bytes()).hexdigest() if entry.is_file() else None
     result["friction"] = bench_trace.friction(events)
+    result["correctionRounds"] = bench_trace.correction_rounds(events, snaps)
     result["expectations"] = bench_trace.detect_expectations(events, snaps, scenario, final_sha)
     result["snapshots"] = snapshot_validity(scenario, snaps, args.wright, out)
     first_valid = next((s["t"] for s in result["snapshots"]["series"] if s["valid"]), None)
@@ -747,7 +754,7 @@ def cmd_evaluate(args: argparse.Namespace) -> int:
     status = cmd_matrix(args)
     if not list(args.out.glob("*/*/*/result.json")):
         return status or 1
-    bench_report.main([args.out], args.wright, False, load_scenario, bench_report.BASELINE)
+    bench_report.main([args.out], args.wright, False, load_scenario, references_of(args))
     languages = ["workshop", "opy"]
     expected = {lang: [s for s in all_scenario_ids() if load_scenario(s)["language"] == lang and load_scenario(s).get("split") == "test"] for lang in languages}
     bench_score.main([args.out], languages, expected, None)
@@ -863,10 +870,11 @@ def main() -> int:
     ev.add_argument("--trials", type=int, default=3)
     ev.add_argument("--parallel", type=int, default=1, help="trials at a time; sequential by default so provider limits are not hit, and a run can continue across sessions")
     ev.add_argument("--seed", type=int, default=1)
+    ev.add_argument("--reference", action="append", help="condition label the report's paired comparison is made against; repeatable (default: none/none/off)")
     ev.add_argument("--dry-run", action="store_true", help="check the setup and print what would run, without running it")
     ev.add_argument("--no-file-sandbox", action="store_true", help="run without the macOS file sandbox: the agent can then read the scenario answer keys")
     sub.add_parser("setup-oracle", help="install the pinned upstream OverPy oracle")
-    skill = sub.add_parser("wiki-skill", help="build the progressive-disclosure workshop-wiki skill from a wiki snapshot")
+    skill = sub.add_parser("wiki-skill", help="build the progressive-disclosure workshop-skill from a wiki snapshot")
     skill.add_argument("--snapshot", type=Path, required=True)
     skill.add_argument("--out-dir", type=Path, required=True, help="new skill directory (not overwritten)")
     skill.add_argument("--catalog", type=Path, required=True, help="workshop-rs catalog.json, for Workshop names and ids")
@@ -879,7 +887,7 @@ def main() -> int:
     report.add_argument("dirs", nargs="+", type=Path)
     report.add_argument("--regrade", action="store_true", help="re-grade stored workspaces twice and flag unstable graders")
     report.add_argument("--wright", default=str(ROOT / "target/debug/wright"))
-    report.add_argument("--reference", default=bench_report.BASELINE, help="condition label the paired comparison is made against")
+    report.add_argument("--reference", action="append", help="condition label a paired comparison is made against; repeatable for lift against several references (default: none/none/off)")
     compare = sub.add_parser("compare", help="one table from the score.json of several evaluation runs, warning when they are not comparable")
     compare.add_argument("dirs", nargs="+", type=Path)
     score = sub.add_parser("score", help="compute the Wright Agent Score card of each language track from canonical test runs")
@@ -915,7 +923,7 @@ def main() -> int:
     if args.command == "wiki-skill":
         return cmd_wiki_skill(args)
     if args.command == "report":
-        return bench_report.main(args.dirs, args.wright, args.regrade, lambda s: load_scenario(s), args.reference)
+        return bench_report.main(args.dirs, args.wright, args.regrade, lambda s: load_scenario(s), references_of(args))
     if args.command == "leaderboard":
         return bench_leaderboard.main(args.dirs, args.page_out or args.dirs[0].parent / "leaderboard")
     if args.command == "compare":

benchmarks/agent/bench_report.py

@@ -85,6 +85,7 @@ def group_rows(runs: list[dict]) -> dict:
         "peakContext": mean(peaks) if peaks else None,
         "seconds": mean(r["agent"]["seconds"] for r in runs),
         "usedTool": sum(1 for r in runs if any(u["invocations"] for u in r.get("toolUse", {}).values())),
+        "correctionRounds": mean_of([r.get("correctionRounds") for r in runs]),
     }
 
 
@@ -238,13 +239,13 @@ def setup_rows(runs: list[dict]) -> list[str]:
     return out
 
 
-def render(results: list[dict], regrade: list[str] | None = None, reference: str = BASELINE) -> tuple[str, dict]:
+def render(results: list[dict], regrade: list[str] | None = None, references: list[str] | None = None) -> tuple[str, dict]:
     invalid = [r for r in results if r["status"] == "invalid"]
     infrastructure = [r for r in results if r["status"] == "provider-interrupted"]
     runs = [r for r in results if r["status"] not in ("invalid", "provider-interrupted")]
     out = ["# Agent benchmark report", "", f"{len(runs)} valid run(s), {len(invalid)} invalid, {len(infrastructure)} infrastructure failures excluded.", ""]
     summary: dict = {"cells": {}, "infrastructureFailures": len(infrastructure)}
-    out += ["## Outcome by agent and condition", "", "| agent | condition | runs | usable | passed | used a tool | tokens/run | tokens per usable | peak context | s/run |", "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"]
+    out += ["## Outcome by agent and condition", "", "| agent | condition | runs | usable | passed | used a tool | tokens/run | tokens per usable | peak context | s/run | corr |", "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"]
     for agent in sorted({r["agent"]["id"] for r in runs}):
         for cell in sorted({label(r) for r in runs}):
             group = [r for r in runs if r["agent"]["id"] == agent and label(r) == cell]
@@ -253,7 +254,7 @@ def render(results: list[dict], regrade: list[str] | None = None, reference: str
             row = group_rows(group)
             summary["cells"][f"{agent}|{cell}"] = row
             out.append(f"| {agent} | {cell} | {row['n']} | {rate_runs(group)} | {row['passed']}/{row['n']} | {row['usedTool']}/{row['n']} | "
-                       f"{fmt(row['tokens'])} | {fmt(row['tokensPerUsable'])} | {fmt(row['peakContext'])} | {fmt(row['seconds'], 1)} |")
+                       f"{fmt(row['tokens'])} | {fmt(row['tokensPerUsable'])} | {fmt(row['peakContext'])} | {fmt(row['seconds'], 1)} | {fmt(row['correctionRounds'], 1)} |")
     out += setup_rows(runs)
     out += ["", "## By scenario", "", "| scenario | agent | condition | usable |", "| --- | --- | --- | --- |"]
     groups: dict[tuple, list[dict]] = defaultdict(list)
@@ -283,9 +284,10 @@ def render(results: list[dict], regrade: list[str] | None = None, reference: str
                 "search/read shell command (a `bash` call invoking the wright CLI counts once, as a wright call).", "",
                 "| agent | cell | level | runs | usable | passed | search/read | wright calls | bash calls | tool calls | turns | tokens/run |",
                 "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |", *level_lines]
-    pairs = paired(runs, reference)
-    if pairs:
-        out += ["", f"## Paired against `{reference}` (same scenario, agent, trial)", "", "| agent | comparison | pairs | usable gained/lost | tokens where both usable |", "| --- | --- | --- | --- | --- |", *pairs]
+    for reference in references or [BASELINE]:
+        pairs = paired(runs, reference)
+        if pairs:
+            out += ["", f"## Paired against `{reference}` (same scenario, agent, trial)", "", "| agent | comparison | pairs | usable gained/lost | tokens where both usable |", "| --- | --- | --- | --- | --- |", *pairs]
     exp = expectations(runs)
     if exp:
         out += ["", "## Expectation rates (pass/(pass+fail); n/a and unavailable excluded)", "", "| condition | expectations |", "| --- | --- |", *exp]
@@ -324,13 +326,13 @@ def render(results: list[dict], regrade: list[str] | None = None, reference: str
     return "\n".join(out) + "\n", summary
 
 
-def main(dirs: list[Path], wright: str, regrade: bool, load_scenario, reference: str = BASELINE) -> int:
+def main(dirs: list[Path], wright: str, regrade: bool, load_scenario, references: list[str] | None = None) -> int:
     results = load(dirs)
     if not results:
         print("no results found")
         return 1
     notes = regrade_notes([r for r in results if r["status"] != "invalid"], wright, load_scenario) if regrade else None
-    text, summary = render(results, notes, reference)
+    text, summary = render(results, notes, references)
     (dirs[0] / "report.md").write_text(text)
     write_json(dirs[0] / "summary.json", summary)
     print(text)

benchmarks/agent/bench_trace.py

@@ -15,7 +15,7 @@
 TOKEN_BYTES = 4  # estimation only: bytes per token for Wright output attribution
 DECISION_COMMANDS = ("check", "lint", "analyze", "inspect")
 JSONRPC_METHODS = ("compile", "check", "analyze", "inspect")  # serve.rs's direct methods — `lint` exists only as a CLI command
-VALIDATING = ("check", "lint", "analyze", "compile")
+VALIDATING = {"wright": ("check", "lint", "analyze", "compile"), "overpy": ("compile",)}  # ops whose exit 1 means "ran fine, found problems"
 OUTPUT_FORMAT_FLAGS = ("--format", "-f")
 
 
@@ -222,6 +222,58 @@ def serve_ops(events: list[dict]) -> list[str]:
     return [request["op"] for _, request, _ in serve_pairs(events) if request["op"]]
 
 
+def serve_result_payload(response: dict | None) -> dict | None:
+    """The op's result payload from a serve response, unwrapped across transports.
+
+    stdio and jsonrpc put the payload in `result`; an MCP tool result carries the same payload as JSON text
+    inside `result.content[]`. Service-level errors carry `error` instead and yield None."""
+    if response is None:
+        return None
+    try:
+        message = json.loads(response["line"])
+    except (json.JSONDecodeError, TypeError, KeyError):
+        return None
+    result = message.get("result") if isinstance(message, dict) else None
+    if isinstance(result, dict) and isinstance(result.get("content"), list):  # an MCP tool result
+        text = "".join(
+            block.get("text") or "" for block in result["content"] if isinstance(block, dict) and block.get("type") == "text"
+        )
+        try:
+            result = json.loads(text) if text else None
+        except json.JSONDecodeError:
+            return None
+    return result if isinstance(result, dict) else None
+
+
+def correction_rounds(events: list[dict], snapshots: list[dict]) -> int:
+    """`failed validation -> edit` rounds (#466): a validating call or serve op that reported problems
+    (`exit` 1) followed by a workspace edit. Only clean exits mark a verdict: usage errors, refusals,
+    crashes, and protocol errors are all neutral — they neither count nor clear a pending failure.
+    The condition's tool sets the ops that count (`overpy compile` under the `opy` cell).
+
+    Consecutive failures before one edit count as one round; an edit after a passing validation does not.
+    Edit markers use the poller's detection time, so ordering inside one poll interval may merge rounds."""
+    markers = [(s["t"], "edit") for s in snapshots]
+    for call in (e for e in events if e["type"] == "call"):
+        if command_of(call["argv"]) in VALIDATING.get(call.get("tool"), ()) and call["exit"] in (0, 1):
+            markers.append((call["t"] + call.get("seconds", 0), call["exit"] == 1))
+    for _event, request, response in serve_pairs(tool_events(events, "wright")):
+        if request["op"] in VALIDATING["wright"]:
+            payload = serve_result_payload(response)
+            exit_code = payload.get("exit") if payload else None  # an MCP refusal payload has `code`, no `exit`
+            if exit_code in (0, 1):
+                markers.append((response["t"], exit_code == 1))
+    rounds, pending = 0, False
+    for _t, kind in sorted(markers, key=lambda m: m[0]):
+        if kind == "edit":
+            if pending:
+                rounds += 1
+                pending = False
+        else:
+            pending = kind
+    return rounds
+
+
 def transcript_events(path: Path):
     """Parsed transcript events (dicts only); empty when the transcript does not exist."""
     if not path.is_file():
@@ -412,8 +464,8 @@ def detect_expectations(events: list[dict], snapshots: list[dict], scenario: dic
     if last_edit is None:
         result["E04"] = expectation("na", "no edits observed")
     else:
-        after = [c for c in calls if command_of(c["argv"]) in VALIDATING and c["t"] + c["seconds"] >= last_edit]
-        after += [e for e, request in ((e, p) for e, p, _ in pairs) if request["op"] in VALIDATING and e["t"] >= last_edit]
+        after = [c for c in calls if command_of(c["argv"]) in VALIDATING["wright"] and c["t"] + c["seconds"] >= last_edit]
+        after += [e for e, request in ((e, p) for e, p, _ in pairs) if request["op"] in VALIDATING["wright"] and e["t"] >= last_edit]
         matched = [c for c in after if c.get("envelope") and c["envelope"].get("inputIdentity") == final_sha256]
         result["E04"] = expectation("pass" if after else "fail", f"{len(after)} validation(s) after last edit; {len(matched)} match the final content")
     withheld = [c for c in calls if ((c.get("envelope") or {}).get("selection") or {}).get("withheld")]

benchmarks/agent/matrix.example.json

@@ -72,7 +72,7 @@
     ],
     "skill_dirs": {
       "wright-skill": "/abs/path/skills/skills/wright",
-      "workshop-skill": "/abs/path/local/workshop-wiki"
+      "workshop-skill": "/abs/path/local/workshop-skill"
     }
   }
 }

benchmarks/agent/matrix.pilot.example.json

@@ -56,7 +56,7 @@
     "timeout": 3600,
     "skill_dirs": {
       "wright-skill": "/abs/path/skills/skills/wright",
-      "workshop-skill": "/abs/path/local/workshop-wiki"
+      "workshop-skill": "/abs/path/local/workshop-skill"
     }
   }
 }