From 19625f9eb19bb642b78b63e6f984341d688e9f4e Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 30 Aug 2026 18:29:34 +0000 Subject: [PATCH] Harden benchmark workflow execution --- .github/workflows/benchmark-clawbench.yml | 5 +- benchmarks/harbor/README.md | 4 +- benchmarks/harbor/clawbench/prepare-task.py | 1 - benchmarks/harbor/clawbench/run.sh | 16 ++++-- benchmarks/harbor/publish-braintrust.ts | 10 ++++ benchmarks/harbor/report.ts | 12 +++- benchmarks/harbor/results.test.ts | 63 +++++++++++++++++++++ benchmarks/harbor/verify-project-scope.ts | 29 ++++++++++ 8 files changed, 129 insertions(+), 11 deletions(-) create mode 100644 benchmarks/harbor/verify-project-scope.ts diff --git a/.github/workflows/benchmark-clawbench.yml b/.github/workflows/benchmark-clawbench.yml index 4b8f6ca5..57531614 100644 --- a/.github/workflows/benchmark-clawbench.yml +++ b/.github/workflows/benchmark-clawbench.yml @@ -200,6 +200,7 @@ jobs: permissions: contents: read issues: write + pull-requests: write concurrency: group: benchmark-clawbench-${{ needs.resolve.outputs.pr_number || needs.resolve.outputs.head_sha }} cancel-in-progress: false @@ -207,7 +208,6 @@ jobs: HYPEMAN_API_KEY: ${{ secrets.HYPEMAN_API_KEY }} HYPEMAN_BASE_URL: ${{ vars.HYPEMAN_BASE_URL }} KERNEL_MCP_BENCHMARK_API_KEY: ${{ secrets.KERNEL_MCP_BENCHMARK_API_KEY }} - KERNEL_PROJECT: ${{ vars.KERNEL_PROJECT }} PURELY_MAIL_API_KEY: ${{ secrets.PURELY_MAIL_API_KEY }} PURELY_MAIL_DOMAIN: ${{ vars.PURELY_MAIL_DOMAIN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -231,6 +231,7 @@ jobs: steps: - name: Mark the PR benchmark as running if: needs.resolve.outputs.pr_number != '' + continue-on-error: true uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 env: PR_NUMBER: ${{ needs.resolve.outputs.pr_number }} @@ -434,6 +435,7 @@ jobs: - name: Update PR benchmark comment if: always() && needs.resolve.outputs.pr_number != '' + continue-on-error: true uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 env: PR_NUMBER: ${{ needs.resolve.outputs.pr_number }} @@ -477,3 +479,4 @@ jobs: [[ "$BASELINE_STATUS" == "0" ]] [[ "$PUBLISH_OUTCOME" == "success" ]] [[ "$REPORT_OUTCOME" == "success" ]] + jq -e 'all(.arms[]; .scored > 0)' "$RUNNER_TEMP/benchmark-summary.json" >/dev/null diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index b2d8c1f6..8ff24460 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -19,12 +19,14 @@ The image records the current Git SHA, and the generated task records the ClawBe - `uv`, Harbor 0.21.0, and `harbor-hypeman` 0.1.1 - Hypeman CLI credentials - a ClawBench checkout containing pinned commit `45a71c4` -- `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project, plus its `KERNEL_PROJECT` name +- `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project; its credential scope is the project source of truth - `PURELY_MAIL_API_KEY` and `PURELY_MAIL_DOMAIN` for ClawBench account tasks - `OPENAI_API_KEY` for Codex, or Anthropic credentials for Claude Code - the ClawBench judge variables when using a hosted judge: `CLAWBENCH_JUDGE_BASE_URL`, `CLAWBENCH_JUDGE_API_KEY`, `CLAWBENCH_JUDGE_MODEL`, and `CLAWBENCH_JUDGE_API_TYPE` - `BRAINTRUST_API_KEY` and `BRAINTRUST_PROJECT` when publishing results +The runner checks `/auth/context` before generating trials and stops unless the benchmark credential and effective connection resolve to the same non-empty project scope. + ## Build the trial image From the `kernel-mcp-server` checkout: diff --git a/benchmarks/harbor/clawbench/prepare-task.py b/benchmarks/harbor/clawbench/prepare-task.py index 5dc0bc84..3539a2f7 100755 --- a/benchmarks/harbor/clawbench/prepare-task.py +++ b/benchmarks/harbor/clawbench/prepare-task.py @@ -52,7 +52,6 @@ def _add_environment( f"CLAWBENCH_SOURCE_SHA = {json.dumps(clawbench_sha)}", f"KERNEL_MCP_ENABLED_TOOLSETS = {json.dumps(ENABLED_TOOLSETS)}", 'API_BASE_URL = "${KERNEL_API_BASE_URL:-}"', - 'KERNEL_PROJECT = "${KERNEL_PROJECT:-}"', 'REDIS_URL = "redis://127.0.0.1:6379"', ] ) diff --git a/benchmarks/harbor/clawbench/run.sh b/benchmarks/harbor/clawbench/run.sh index 4f59be5f..772e9cf6 100755 --- a/benchmarks/harbor/clawbench/run.sh +++ b/benchmarks/harbor/clawbench/run.sh @@ -71,6 +71,17 @@ esac harbor_version=${HARBOR_VERSION:-0.21.0} harbor_hypeman_version=${HARBOR_HYPEMAN_VERSION:-0.1.1} +export KERNEL_API_KEY=$KERNEL_MCP_BENCHMARK_API_KEY +export KERNEL_BASE_URL=${KERNEL_BASE_URL:-https://api.onkernel.com} +export KERNEL_API_BASE_URL=${KERNEL_API_BASE_URL:-$KERNEL_BASE_URL} + +curl --silent --show-error --fail-with-body \ + --retry 3 \ + --retry-all-errors \ + --header "Authorization: Bearer $KERNEL_API_KEY" \ + "${KERNEL_API_BASE_URL%/}/auth/context" | + bun "$benchmark_dir/verify-project-scope.ts" + runtime_root=$(mktemp -d) runtime_env=$(mktemp) trap 'rm -rf "$runtime_root"; rm -f "$runtime_env"' EXIT @@ -100,16 +111,11 @@ for task_dir in "${task_dirs[@]}"; do --clawbench-sha "$clawbench_ref" done -export KERNEL_API_KEY=$KERNEL_MCP_BENCHMARK_API_KEY -export KERNEL_BASE_URL=${KERNEL_BASE_URL:-https://api.onkernel.com} -export KERNEL_API_BASE_URL=${KERNEL_API_BASE_URL:-$KERNEL_BASE_URL} - { printf 'KERNEL_API_KEY=%s\n' "$KERNEL_API_KEY" printf 'KERNEL_BASE_URL=%s\n' "$KERNEL_BASE_URL" printf 'KERNEL_API_BASE_URL=%s\n' "$KERNEL_API_BASE_URL" printf 'API_BASE_URL=%s\n' "$KERNEL_API_BASE_URL" - printf 'KERNEL_PROJECT=%s\n' "${KERNEL_PROJECT:-}" printf 'PURELY_MAIL_API_KEY=%s\n' "$PURELY_MAIL_API_KEY" printf 'PURELY_MAIL_DOMAIN=%s\n' "$PURELY_MAIL_DOMAIN" printf 'CLAWBENCH_JUDGE_BASE_URL=%s\n' "${CLAWBENCH_JUDGE_BASE_URL:-}" diff --git a/benchmarks/harbor/publish-braintrust.ts b/benchmarks/harbor/publish-braintrust.ts index 51b032c1..fc342d75 100644 --- a/benchmarks/harbor/publish-braintrust.ts +++ b/benchmarks/harbor/publish-braintrust.ts @@ -354,6 +354,16 @@ export async function publishBenchmark( experimentName: string, apiKey: string, ): Promise> { + const ungraded = arms + .map(summarizeArm) + .filter((summary) => summary.scored === 0) + .map((summary) => summary.arm); + if (ungraded.length > 0) { + throw new Error( + `Cannot publish benchmark without graded trials for: ${ungraded.join(", ")}`, + ); + } + const api = new BraintrustApi(apiKey); const project = await api.request("/v1/project", "POST", { name: projectName, diff --git a/benchmarks/harbor/report.ts b/benchmarks/harbor/report.ts index 55eb7523..cb91a6ce 100644 --- a/benchmarks/harbor/report.ts +++ b/benchmarks/harbor/report.ts @@ -89,10 +89,16 @@ export function renderMarkdown( ): string { const lines = ["", `## ${title}`]; const failed = Object.entries(statuses).filter(([, status]) => status !== 0); - if (failed.length > 0) { + const ungraded = summaries.filter((summary) => summary.scored === 0); + const incomplete = failed.length > 0 || ungraded.length > 0; + if (incomplete) { + const reasons = [ + ...failed.map(([arm, status]) => `${arm} exited ${status}`), + ...ungraded.map((summary) => `${summary.arm} produced no graded trials`), + ]; lines.push( "", - `> [!WARNING]\n> Incomplete benchmark: ${failed.map(([arm, status]) => `${arm} exited ${status}`).join(", ")}. Scores below include only completed Harbor results and are not a complete comparison.`, + `> [!WARNING]\n> Incomplete benchmark: ${reasons.join(", ")}. Scores below include only completed Harbor results; comparison deltas are suppressed.`, ); } lines.push( @@ -108,7 +114,7 @@ export function renderMarkdown( const candidate = summaries.find((summary) => summary.arm === "candidate"); const baseline = summaries.find((summary) => summary.arm === "baseline"); - if (candidate && baseline && failed.length === 0) { + if (candidate && baseline && !incomplete) { const signed = (value: number) => { const rounded = Number(value.toFixed(3)); return `${rounded >= 0 ? "+" : ""}${rounded}`; diff --git a/benchmarks/harbor/results.test.ts b/benchmarks/harbor/results.test.ts index dbe80fe9..941c841d 100644 --- a/benchmarks/harbor/results.test.ts +++ b/benchmarks/harbor/results.test.ts @@ -12,6 +12,7 @@ import { buildExperimentEvents, publishBenchmark } from "./publish-braintrust"; import { renderMarkdown } from "./report"; import { readBenchmarkArm, selectPrimaryReward, summarizeArm } from "./results"; import { redactString, redactValue } from "./redact"; +import { assertProjectScopedCredential } from "./verify-project-scope"; const temporaryDirectories: string[] = []; @@ -257,6 +258,16 @@ describe("Harbor result ingestion", () => { } }); + test("does not publish arms without graded trials", async () => { + const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); + for (const trial of arm.trials) trial.rewards = {}; + await expect( + publishBenchmark([arm], "project", "experiment", "api-key"), + ).rejects.toThrow( + "Cannot publish benchmark without graded trials for: candidate", + ); + }); + test("uses the lenient reward per trial and reports incomplete arms", () => { expect(selectPrimaryReward({ reward: 0, reward_lenient: 1 })).toEqual({ key: "reward_lenient", @@ -288,6 +299,12 @@ describe("Harbor result ingestion", () => { { ...summary, arm: "baseline", lenient: 0.2 }, ]), ).toContain("+0.1 lenient"); + const ungraded = renderMarkdown("test", [ + { ...summary, arm: "candidate", scored: 0, ungraded: summary.trials }, + { ...summary, arm: "baseline", scored: 0, ungraded: summary.trials }, + ]); + expect(ungraded).toContain("candidate produced no graded trials"); + expect(ungraded).not.toContain("Candidate minus baseline"); }); test("keeps full errors until redaction and clamps derived scores", () => { @@ -369,11 +386,49 @@ describe("benchmark workflow hardening", () => { expect(workflow).not.toContain("baseSha = pull.base.sha"); expect(workflow).toContain('HARBOR_VERSION: "0.21.0"'); expect(workflow).toContain('CODEX_BENCHMARK_VERSION: "0.120.0"'); + expect(workflow).toContain("issues: write\n pull-requests: write"); + expect(workflow).not.toContain( + "KERNEL_PROJECT: ${{ vars.KERNEL_PROJECT }}", + ); + expect(workflow).toContain("all(.arms[]; .scored > 0)"); + expect(workflow).toMatch( + /- name: Mark the PR benchmark as running\n\s+if:.*\n\s+continue-on-error: true/, + ); + expect(workflow).toMatch( + /- name: Update PR benchmark comment\n\s+if:.*\n\s+continue-on-error: true/, + ); expect(workflow).toContain( 'statuses=(--status "candidate=${CANDIDATE_STATUS:-1}")', ); }); + test("requires the benchmark credential to resolve to one project", () => { + expect(() => + assertProjectScopedCredential({ + authorization: { + credential_scope: { project_id: "project" }, + effective_scope: { project_id: "project" }, + }, + }), + ).not.toThrow(); + expect(() => + assertProjectScopedCredential({ + authorization: { + credential_scope: { project_id: null }, + effective_scope: { project_id: null }, + }, + }), + ).toThrow(); + expect(() => + assertProjectScopedCredential({ + authorization: { + credential_scope: { project_id: "credential-project" }, + effective_scope: { project_id: "other-project" }, + }, + }), + ).toThrow(); + }); + test("excludes private keys and forwards only the selected provider", () => { const dockerignore = readFileSync( join(process.cwd(), ".dockerignore"), @@ -387,7 +442,15 @@ describe("benchmark workflow hardening", () => { join(process.cwd(), "benchmarks/harbor/clawbench/verify-task.py"), "utf8", ); + const taskPreparer = readFileSync( + join(process.cwd(), "benchmarks/harbor/clawbench/prepare-task.py"), + "utf8", + ); expect(dockerignore.split("\n")).toContain("*.pem"); + expect(runner).not.toContain("KERNEL_PROJECT"); + expect(runner).toContain('"${KERNEL_API_BASE_URL%/}/auth/context"'); + expect(runner).toContain('bun "$benchmark_dir/verify-project-scope.ts"'); + expect(taskPreparer).not.toContain("KERNEL_PROJECT"); expect(verifier).toContain('"mcp__kernel__execute_playwright_code"'); expect(verifier).toContain('"kernel__execute_playwright_code"'); expect(verifier).toContain('"execute_playwright_code"'); diff --git a/benchmarks/harbor/verify-project-scope.ts b/benchmarks/harbor/verify-project-scope.ts new file mode 100644 index 00000000..4863a0b8 --- /dev/null +++ b/benchmarks/harbor/verify-project-scope.ts @@ -0,0 +1,29 @@ +import { z } from "zod"; + +const authContextSchema = z.object({ + authorization: z.object({ + credential_scope: z.object({ project_id: z.string().min(1) }), + effective_scope: z.object({ project_id: z.string().min(1) }), + }), +}); + +export function assertProjectScopedCredential(value: unknown): void { + const context = authContextSchema.parse(value); + if ( + context.authorization.effective_scope.project_id !== + context.authorization.credential_scope.project_id + ) { + throw new Error("Credential and effective project scopes differ"); + } +} + +if (import.meta.main) { + try { + assertProjectScopedCredential(JSON.parse(await Bun.stdin.text())); + } catch { + console.error( + "The benchmark credential must resolve to one matching project scope", + ); + process.exit(1); + } +}