diff --git a/evals/completion/lib.test.ts b/evals/completion/lib.test.ts index df4766737..89ad669a2 100644 --- a/evals/completion/lib.test.ts +++ b/evals/completion/lib.test.ts @@ -4,10 +4,12 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { CompletionReport, + REPORT_VERSION, assertTaskSetContained, computeTotals, formatSummary, isCompletedRun, + migrateLegacyCompletionReport, parseResponderScript, parseTaskSetFile, resolveTaskRelativePath, @@ -25,9 +27,9 @@ const result = (overrides: Partial = {}): TaskResultType => completed: true, runStatus: "completed", turnsUsed: 3, + turnsEstimated: false, toolCallCount: 2, failedToolCalls: 0, - retryCount: 0, compactionEvents: 0, doomLoopInterventions: 0, thrashInterventions: 0, @@ -52,7 +54,7 @@ describe("completion predicate", () => { describe("completion totals", () => { test("mixed outcomes yield a fractional rate and completion-only turn mean", () => { const totals = computeTotals([ - result({ turnsUsed: 3, agentDurationMs: 1000 }), + result({ turnsUsed: 3, agentDurationMs: 1000, verifyDurationMs: 100 }), result({ taskId: "stall-read", profile: "stall", @@ -60,10 +62,11 @@ describe("completion totals", () => { runStatus: "failed", turnsUsed: 4, toolCallCount: 4, - retryCount: 1, + failedToolCalls: 1, doomLoopInterventions: 1, verifyExitCode: 1, agentDurationMs: 3000, + verifyDurationMs: 300, }), ]); expect(totals.tasksTotal).toBe(2); @@ -72,7 +75,8 @@ describe("completion totals", () => { expect(totals.completionRate).toBe(0.5); expect(totals.meanTurnsToCompletion).toBe(3); expect(totals.meanAgentDurationMs).toBe(2000); - expect(totals.totalRetries).toBe(1); + expect(totals.meanVerifyDurationMs).toBe(200); + expect(totals.totalFailedToolCalls).toBe(1); expect(totals.totalDoomLoopInterventions).toBe(1); }); @@ -81,6 +85,8 @@ describe("completion totals", () => { expect(totals.completionRate).toBe(0); expect(totals.meanTurnsToCompletion).toBe(0); expect(totals.meanAgentDurationMs).toBe(0); + expect(totals.meanVerifyDurationMs).toBe(0); + expect(totals.totalFailedToolCalls).toBe(0); }); }); @@ -171,11 +177,11 @@ describe("human summary", () => { test("names the harness, provenance, rate, and every run", () => { const results = [ result(), - result({ taskId: "stall-read", completed: false }), + result({ taskId: "stall-read", completed: false, turnsEstimated: true }), ]; const report: CompletionReportType = CompletionReport.assert({ harness: "completion-baseline", - version: 1, + version: REPORT_VERSION, startedAt: "2026-09-14T00:00:00.000Z", finishedAt: "2026-09-14T00:01:00.000Z", commitSha: "deadbeef", @@ -191,16 +197,105 @@ describe("human summary", () => { expect(summary).toContain("completion rate 50.0% (1/2 runs)"); expect(summary).toContain("deadbeef"); expect(summary).toContain("stub-scripted"); + expect(summary).toContain("mean verify time 300ms"); + expect(summary).toContain("failed tool calls 0"); + expect(summary).not.toContain("retries"); expect(summary).toContain("version-endpoint r0: complete"); expect(summary).toContain("stall-read r0: incomplete"); + expect(summary).toContain("turns 3 (estimated)"); + expect(summary).toContain("verify 300ms"); + }); +}); + +describe("legacy v1 migration", () => { + const legacyResult = (overrides: Record = {}) => ({ + taskId: "version-endpoint", + title: "Add GET /version", + profile: "solve", + repeat: 0, + completed: true, + runStatus: "completed", + turnsUsed: 3, + toolCallCount: 2, + failedToolCalls: 0, + retryCount: 0, + compactionEvents: 0, + doomLoopInterventions: 0, + thrashInterventions: 0, + gateSuspensions: 0, + agentDurationMs: 1200, + verifyDurationMs: 300, + verifyExitCode: 0, + overBudget: false, + ...overrides, + }); + + const legacyReport = (results: Record[]) => ({ + harness: "completion-baseline", + version: 1, + startedAt: "2026-09-14T00:00:00.000Z", + finishedAt: "2026-09-14T00:01:00.000Z", + commitSha: "deadbeef", + provider: "stub-scripted", + model: "completion-baseline-v1", + repeats: 1, + taskSetVersion: 1, + taskIds: ["version-endpoint"], + results, + totals: { + tasksTotal: 1, + runsTotal: results.length, + completedRuns: 1, + completionRate: 1, + meanTurnsToCompletion: 3, + meanAgentDurationMs: 1200, + totalRetries: 0, + totalCompactionEvents: 0, + totalDoomLoopInterventions: 0, + totalThrashInterventions: 0, + totalGateSuspensions: 0, + }, + }); + + test("drops the mislabeled retry count and aggregates verify durations", () => { + const migrated = migrateLegacyCompletionReport( + legacyReport([legacyResult(), legacyResult({ repeat: 1 })]), + ); + expect(migrated.version).toBe(REPORT_VERSION); + expect(migrated.totals.totalFailedToolCalls).toBe(0); + expect(migrated.totals.meanVerifyDurationMs).toBe(300); + for (const migratedResult of migrated.results) { + expect("retryCount" in migratedResult).toBe(false); + expect("turnsEstimated" in migratedResult).toBe(false); + } + }); + + test("rejects a legacy result whose retry count is not a failure count", () => { + expect(() => + migrateLegacyCompletionReport( + legacyReport([legacyResult({ retryCount: 2, failedToolCalls: 1 })]), + ), + ).toThrow(/mislabels retries/); + }); + + test("rejects a non-v1 report", () => { + const payload = legacyReport([legacyResult()]); + payload.version = REPORT_VERSION; + expect(() => migrateLegacyCompletionReport(payload)).toThrow( + /Only v1 reports can be migrated/, + ); }); }); describe("checked-in baseline", () => { - test("the frozen baseline validates against the current report schema", async () => { + test("the frozen v1 baseline migrates to the current report schema", async () => { const dir = dirname(fileURLToPath(import.meta.url)); const raw = await readFile(join(dir, "baseline-2026-09-14.json"), "utf8"); - const report = CompletionReport.assert(JSON.parse(raw)); + const report = migrateLegacyCompletionReport(JSON.parse(raw)); expect(report.totals.runsTotal).toBe(report.results.length); + expect(report.totals.runsTotal).toBe(8); + expect(report.totals.completionRate).toBe(0.5); + expect(report.totals.totalFailedToolCalls).toBe(0); + expect(report.totals.meanVerifyDurationMs).toBe(120.75); }); }); diff --git a/evals/completion/lib.ts b/evals/completion/lib.ts index d41bf2177..17e0a9924 100644 --- a/evals/completion/lib.ts +++ b/evals/completion/lib.ts @@ -69,9 +69,15 @@ export const TaskResult = type({ completed: "boolean", runStatus: RunStatus, turnsUsed: "number.integer >= 0", + // Absent on v1 reports recorded before estimation tracking: there the + // turnsUsed provenance is unknown, so migration leaves it unset rather + // than guessing. + "turnsEstimated?": "boolean", toolCallCount: "number.integer >= 0", + // Failed tool calls, not retries: v1 called this retryCount, which + // mislabeled the count. The loop never re-issues a failed call, so the + // honest name is the failure count itself. failedToolCalls: "number.integer >= 0", - retryCount: "number.integer >= 0", compactionEvents: "number.integer >= 0", doomLoopInterventions: "number.integer >= 0", thrashInterventions: "number.integer >= 0", @@ -91,7 +97,8 @@ export const CompletionTotals = type({ completionRate: "0<=number<=1", meanTurnsToCompletion: "number >= 0", meanAgentDurationMs: "number >= 0", - totalRetries: "number.integer >= 0", + meanVerifyDurationMs: "number >= 0", + totalFailedToolCalls: "number.integer >= 0", totalCompactionEvents: "number.integer >= 0", totalDoomLoopInterventions: "number.integer >= 0", totalThrashInterventions: "number.integer >= 0", @@ -115,6 +122,10 @@ export const CompletionReport = type({ }); export type CompletionReport = typeof CompletionReport.infer; +// Report schema revision: bump when field names or totals change so a v1 +// baseline file can never be mistaken for a current-schema report. +export const REPORT_VERSION = 2; + /** Parse and validate an unknown task-set payload (e.g. tasks.json). */ export function parseTaskSetFile(payload: unknown): CompletionTaskSet { return CompletionTaskSet.assert(payload); @@ -189,7 +200,12 @@ export function computeTotals( results.length === 0 ? 0 : sum(results.map((result) => result.agentDurationMs)) / results.length, - totalRetries: sum(results.map((result) => result.retryCount)), + meanVerifyDurationMs: + results.length === 0 + ? 0 + : sum(results.map((result) => result.verifyDurationMs)) / + results.length, + totalFailedToolCalls: sum(results.map((result) => result.failedToolCalls)), totalCompactionEvents: sum( results.map((result) => result.compactionEvents), ), @@ -213,16 +229,120 @@ export function formatSummary(report: CompletionReport): string { `window ${report.startedAt} .. ${report.finishedAt} repeats ${report.repeats}`, `task set v${report.taskSetVersion}: ${report.taskIds.join(", ")}`, `completion rate ${formatRate(report.totals.completionRate)} (${report.totals.completedRuns}/${report.totals.runsTotal} runs)`, - `mean turns to completion ${report.totals.meanTurnsToCompletion.toFixed(1)} mean agent time ${Math.round(report.totals.meanAgentDurationMs)}ms`, - `retries ${report.totals.totalRetries} compaction events ${report.totals.totalCompactionEvents} doom-loop interventions ${report.totals.totalDoomLoopInterventions} thrash interventions ${report.totals.totalThrashInterventions} gate suspensions ${report.totals.totalGateSuspensions}`, + `mean turns to completion ${report.totals.meanTurnsToCompletion.toFixed(1)} mean agent time ${Math.round(report.totals.meanAgentDurationMs)}ms mean verify time ${Math.round(report.totals.meanVerifyDurationMs)}ms`, + `failed tool calls ${report.totals.totalFailedToolCalls} compaction events ${report.totals.totalCompactionEvents} doom-loop interventions ${report.totals.totalDoomLoopInterventions} thrash interventions ${report.totals.totalThrashInterventions} gate suspensions ${report.totals.totalGateSuspensions}`, "", ...report.results.map( (result) => `- ${result.taskId} r${result.repeat}: ${result.completed ? "complete" : "incomplete"} ` + - `(status ${result.runStatus}, turns ${result.turnsUsed}, tools ${result.toolCallCount}, ` + - `retries ${result.retryCount}, doom-loop ${result.doomLoopInterventions}, ` + - `verify exit ${result.verifyExitCode}, agent ${result.agentDurationMs}ms)`, + `(status ${result.runStatus}, turns ${result.turnsUsed}${result.turnsEstimated === true ? " (estimated)" : ""}, tools ${result.toolCallCount}, ` + + `failed tool calls ${result.failedToolCalls}, doom-loop ${result.doomLoopInterventions}, ` + + `verify exit ${result.verifyExitCode}, agent ${result.agentDurationMs}ms verify ${result.verifyDurationMs}ms)`, ), ]; return `${lines.join("\n")}\n`; } + +// Frozen v1 report shapes (baseline-2026-09-14.json): v1 stored failed +// tool calls twice — as failedToolCalls and, mislabeled, as retryCount — +// and never aggregated verify durations into totals. These readers exist +// only so the frozen baseline stays byte-identical while remaining +// machine-checkable; new reports must use the v2 shapes above. +const LegacyTaskResult = type({ + taskId: "string", + title: "string", + profile: ResponderProfile, + repeat: "number.integer >= 0", + completed: "boolean", + runStatus: RunStatus, + turnsUsed: "number.integer >= 0", + toolCallCount: "number.integer >= 0", + failedToolCalls: "number.integer >= 0", + retryCount: "number.integer >= 0", + compactionEvents: "number.integer >= 0", + doomLoopInterventions: "number.integer >= 0", + thrashInterventions: "number.integer >= 0", + gateSuspensions: "number.integer >= 0", + agentDurationMs: "number.integer >= 0", + verifyDurationMs: "number.integer >= 0", + verifyExitCode: "number.integer", + overBudget: "boolean", + "error?": "string", +}); + +const LegacyCompletionReport = type({ + harness: "string", + version: "number.integer >= 1", + startedAt: "string", + finishedAt: "string", + commitSha: "string", + provider: "string", + model: "string", + repeats: "number.integer >= 1", + taskSetVersion: "number.integer >= 1", + taskIds: "string[]", + results: LegacyTaskResult.array(), +}); + +/** + * Migrate a frozen v1 report to the current schema without guessing: the + * mislabeled retryCount is dropped only after proving it equals + * failedToolCalls (a mismatch means hand-edited data and is rejected), + * verify means are recomputed from the recorded per-run durations, and + * turnsEstimated stays unset because v1 never tracked turns provenance. + */ +export function migrateLegacyCompletionReport( + payload: unknown, +): CompletionReport { + const legacy = LegacyCompletionReport.assert(payload); + if (legacy.version !== 1) { + throw new Error( + `Only v1 reports can be migrated, got version ${legacy.version}`, + ); + } + const results = legacy.results.map((legacyResult) => { + if (legacyResult.retryCount !== legacyResult.failedToolCalls) { + throw new Error( + `Legacy result for ${legacyResult.taskId} r${legacyResult.repeat} ` + + `mislabels retries: retryCount ${legacyResult.retryCount} !== ` + + `failedToolCalls ${legacyResult.failedToolCalls}`, + ); + } + return { + taskId: legacyResult.taskId, + title: legacyResult.title, + profile: legacyResult.profile, + repeat: legacyResult.repeat, + completed: legacyResult.completed, + runStatus: legacyResult.runStatus, + turnsUsed: legacyResult.turnsUsed, + toolCallCount: legacyResult.toolCallCount, + failedToolCalls: legacyResult.failedToolCalls, + compactionEvents: legacyResult.compactionEvents, + doomLoopInterventions: legacyResult.doomLoopInterventions, + thrashInterventions: legacyResult.thrashInterventions, + gateSuspensions: legacyResult.gateSuspensions, + agentDurationMs: legacyResult.agentDurationMs, + verifyDurationMs: legacyResult.verifyDurationMs, + verifyExitCode: legacyResult.verifyExitCode, + overBudget: legacyResult.overBudget, + ...(legacyResult.error !== undefined + ? { error: legacyResult.error } + : {}), + }; + }); + return CompletionReport.assert({ + harness: legacy.harness, + version: REPORT_VERSION, + startedAt: legacy.startedAt, + finishedAt: legacy.finishedAt, + commitSha: legacy.commitSha, + provider: legacy.provider, + model: legacy.model, + repeats: legacy.repeats, + taskSetVersion: legacy.taskSetVersion, + taskIds: legacy.taskIds, + results, + totals: computeTotals(TaskResult.array().assert(results)), + }); +} diff --git a/scripts/eval-completion.ts b/scripts/eval-completion.ts index ff763aec5..fc31eb85a 100644 --- a/scripts/eval-completion.ts +++ b/scripts/eval-completion.ts @@ -7,7 +7,7 @@ * provider credentials): each task's version-controlled responder script * plays the model, the real reactor/director/toolset executes, and the * task's verify.sh grades the outcome. Reports completion rate plus the - * control-layer secondary signals (turns, retries, compaction events, + * control-layer secondary signals (turns, failed tool calls, compaction events, * doom-loop/thrash interventions, wall clock) as JSON and a human summary. * * Scripted responders isolate the control layer from model variance on @@ -15,6 +15,17 @@ * guard-trip paths deterministically so the 0.4.x re-measure sees the * control layer move, not provider noise. * + * Re-measuring (canonical): + * bun scripts/eval-completion.ts --repeats 2 --out evals/completion/baseline-.json + * Conventions: the task set is frozen — re-measures reuse tasks.json as-is + * so runs stay comparable. Never edit tasks.json, per-task + * script.json/verify.sh/fixture, or a recorded baseline to hit a target + * number; a task-set change needs a version bump plus a new baseline file. + * Field honesty: turnsUsed is the persisted assistant-turn count and + * turnsEstimated marks the tool-call fallback estimate; failed tool calls + * are reported as failed tool calls, never "retries"; durations are wall + * clock, aggregated as means in totals and the summary. + * * TRUST BOUNDARY: the version-controlled files under evals/completion/tasks/ * (tasks.json, per-task script.json, verify.sh, fixture/) are the trusted * grading boundary — edits there are grading changes requiring owner review. @@ -37,6 +48,7 @@ import { } from "../tests/integration/harness.js"; import { CompletionReport, + REPORT_VERSION, assertTaskSetContained, computeTotals, formatSummary, @@ -202,9 +214,9 @@ function deriveSignals(events: TurnResult["events"]) { ); return { toolCallCount: toolStarts.length, + // Failed tool calls only: the loop never re-issues a failed call, so + // this count must not be reported as retries. failedToolCalls, - // Each failed tool call the loop continues past is a retried attempt. - retryCount: failedToolCalls, doomLoopInterventions, compactionEvents, thrashInterventions, @@ -312,6 +324,10 @@ async function runTask( const verifyExitCode = verify.status ?? 1; const persistedTurns = await countAssistantTurns(session.workdir); + // The persisted assistant-turn count is the measurement; the + // tool-call heuristic below is an estimate used only when turns.jsonl + // is missing, and turnsEstimated says which one a row holds. + const turnsEstimated = persistedTurns === null; const turnsUsed = persistedTurns ?? (signals.toolCallCount > 0 ? signals.toolCallCount + 1 : 1); @@ -325,9 +341,9 @@ async function runTask( completed, runStatus, turnsUsed, + turnsEstimated, toolCallCount: signals.toolCallCount, failedToolCalls: signals.failedToolCalls, - retryCount: signals.retryCount, compactionEvents: signals.compactionEvents, doomLoopInterventions: signals.doomLoopInterventions, thrashInterventions: signals.thrashInterventions, @@ -376,7 +392,7 @@ async function main(): Promise { const report = CompletionReport.assert({ harness: "completion-baseline", - version: 1, + version: REPORT_VERSION, startedAt, finishedAt: new Date().toISOString(), commitSha: commitSha(),