Harden benchmark workflow execution

.github/workflows/benchmark-clawbench.yml

@@ -200,14 +200,14 @@ jobs:
     permissions:
       contents: read
       issues: write
+      pull-requests: write
     concurrency:
       group: benchmark-clawbench-${{ needs.resolve.outputs.pr_number || needs.resolve.outputs.head_sha }}
       cancel-in-progress: false
     env:
       HYPEMAN_API_KEY: ${{ secrets.HYPEMAN_API_KEY }}
       HYPEMAN_BASE_URL: ${{ vars.HYPEMAN_BASE_URL }}
       KERNEL_MCP_BENCHMARK_API_KEY: ${{ secrets.KERNEL_MCP_BENCHMARK_API_KEY }}
-      KERNEL_PROJECT: ${{ vars.KERNEL_PROJECT }}
       PURELY_MAIL_API_KEY: ${{ secrets.PURELY_MAIL_API_KEY }}
       PURELY_MAIL_DOMAIN: ${{ vars.PURELY_MAIL_DOMAIN }}
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
@@ -231,6 +231,7 @@ jobs:
     steps:
       - name: Mark the PR benchmark as running
         if: needs.resolve.outputs.pr_number != ''
+        continue-on-error: true
         uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
         env:
           PR_NUMBER: ${{ needs.resolve.outputs.pr_number }}
@@ -434,6 +435,7 @@ jobs:
 
       - name: Update PR benchmark comment
         if: always() && needs.resolve.outputs.pr_number != ''
+        continue-on-error: true
         uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
         env:
           PR_NUMBER: ${{ needs.resolve.outputs.pr_number }}
@@ -477,3 +479,4 @@ jobs:
           [[ "$BASELINE_STATUS" == "0" ]]
           [[ "$PUBLISH_OUTCOME" == "success" ]]
           [[ "$REPORT_OUTCOME" == "success" ]]
+          jq -e 'all(.arms[]; .scored > 0)' "$RUNNER_TEMP/benchmark-summary.json" >/dev/null

benchmarks/harbor/README.md

@@ -19,12 +19,14 @@ The image records the current Git SHA, and the generated task records the ClawBe
 - `uv`, Harbor 0.21.0, and `harbor-hypeman` 0.1.1
 - Hypeman CLI credentials
 - a ClawBench checkout containing pinned commit `45a71c4`
-- `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project, plus its `KERNEL_PROJECT` name
+- `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project; its credential scope is the project source of truth
 - `PURELY_MAIL_API_KEY` and `PURELY_MAIL_DOMAIN` for ClawBench account tasks
 - `OPENAI_API_KEY` for Codex, or Anthropic credentials for Claude Code
 - the ClawBench judge variables when using a hosted judge: `CLAWBENCH_JUDGE_BASE_URL`, `CLAWBENCH_JUDGE_API_KEY`, `CLAWBENCH_JUDGE_MODEL`, and `CLAWBENCH_JUDGE_API_TYPE`
 - `BRAINTRUST_API_KEY` and `BRAINTRUST_PROJECT` when publishing results
 
+The runner checks `/auth/context` before generating trials and stops unless the benchmark credential and effective connection resolve to the same non-empty project scope.
+
 ## Build the trial image
 
 From the `kernel-mcp-server` checkout:

benchmarks/harbor/clawbench/prepare-task.py

@@ -52,7 +52,6 @@ def _add_environment(
                     f"CLAWBENCH_SOURCE_SHA = {json.dumps(clawbench_sha)}",
                     f"KERNEL_MCP_ENABLED_TOOLSETS = {json.dumps(ENABLED_TOOLSETS)}",
                     'API_BASE_URL = "${KERNEL_API_BASE_URL:-}"',
-                    'KERNEL_PROJECT = "${KERNEL_PROJECT:-}"',
                     'REDIS_URL = "redis://127.0.0.1:6379"',
                 ]
             )

benchmarks/harbor/clawbench/run.sh

@@ -71,6 +71,17 @@ esac
 harbor_version=${HARBOR_VERSION:-0.21.0}
 harbor_hypeman_version=${HARBOR_HYPEMAN_VERSION:-0.1.1}
 
+export KERNEL_API_KEY=$KERNEL_MCP_BENCHMARK_API_KEY
+export KERNEL_BASE_URL=${KERNEL_BASE_URL:-https://api.onkernel.com}
+export KERNEL_API_BASE_URL=${KERNEL_API_BASE_URL:-$KERNEL_BASE_URL}
+
+curl --silent --show-error --fail-with-body \
+  --retry 3 \
+  --retry-all-errors \
+  --header "Authorization: Bearer $KERNEL_API_KEY" \
+  "${KERNEL_API_BASE_URL%/}/auth/context" |
+  bun "$benchmark_dir/verify-project-scope.ts"
+
 runtime_root=$(mktemp -d)
 runtime_env=$(mktemp)
 trap 'rm -rf "$runtime_root"; rm -f "$runtime_env"' EXIT
@@ -100,16 +111,11 @@ for task_dir in "${task_dirs[@]}"; do
     --clawbench-sha "$clawbench_ref"
 done
 
-export KERNEL_API_KEY=$KERNEL_MCP_BENCHMARK_API_KEY
-export KERNEL_BASE_URL=${KERNEL_BASE_URL:-https://api.onkernel.com}
-export KERNEL_API_BASE_URL=${KERNEL_API_BASE_URL:-$KERNEL_BASE_URL}
-
 {
   printf 'KERNEL_API_KEY=%s\n' "$KERNEL_API_KEY"
   printf 'KERNEL_BASE_URL=%s\n' "$KERNEL_BASE_URL"
   printf 'KERNEL_API_BASE_URL=%s\n' "$KERNEL_API_BASE_URL"
   printf 'API_BASE_URL=%s\n' "$KERNEL_API_BASE_URL"
-  printf 'KERNEL_PROJECT=%s\n' "${KERNEL_PROJECT:-}"
   printf 'PURELY_MAIL_API_KEY=%s\n' "$PURELY_MAIL_API_KEY"
   printf 'PURELY_MAIL_DOMAIN=%s\n' "$PURELY_MAIL_DOMAIN"
   printf 'CLAWBENCH_JUDGE_BASE_URL=%s\n' "${CLAWBENCH_JUDGE_BASE_URL:-}"

benchmarks/harbor/publish-braintrust.ts

@@ -354,6 +354,16 @@ export async function publishBenchmark(
   experimentName: string,
   apiKey: string,
 ): Promise<Record<string, unknown>> {
+  const ungraded = arms
+    .map(summarizeArm)
+    .filter((summary) => summary.scored === 0)
+    .map((summary) => summary.arm);
+  if (ungraded.length > 0) {
+    throw new Error(
+      `Cannot publish benchmark without graded trials for: ${ungraded.join(", ")}`,
+    );
+  }
+
   const api = new BraintrustApi(apiKey);
   const project = await api.request<BraintrustProject>("/v1/project", "POST", {
     name: projectName,

benchmarks/harbor/report.ts

@@ -89,10 +89,16 @@ export function renderMarkdown(
 ): string {
   const lines = ["<!-- kernel-mcp-clawbench -->", `## ${title}`];
   const failed = Object.entries(statuses).filter(([, status]) => status !== 0);
-  if (failed.length > 0) {
+  const ungraded = summaries.filter((summary) => summary.scored === 0);
+  const incomplete = failed.length > 0 || ungraded.length > 0;
+  if (incomplete) {
+    const reasons = [
+      ...failed.map(([arm, status]) => `${arm} exited ${status}`),
+      ...ungraded.map((summary) => `${summary.arm} produced no graded trials`),
+    ];
     lines.push(
       "",
-      `> [!WARNING]\n> Incomplete benchmark: ${failed.map(([arm, status]) => `${arm} exited ${status}`).join(", ")}. Scores below include only completed Harbor results and are not a complete comparison.`,
+      `> [!WARNING]\n> Incomplete benchmark: ${reasons.join(", ")}. Scores below include only completed Harbor results; comparison deltas are suppressed.`,
     );
   }
   lines.push(
@@ -108,7 +114,7 @@ export function renderMarkdown(
 
   const candidate = summaries.find((summary) => summary.arm === "candidate");
   const baseline = summaries.find((summary) => summary.arm === "baseline");
-  if (candidate && baseline && failed.length === 0) {
+  if (candidate && baseline && !incomplete) {
     const signed = (value: number) => {
       const rounded = Number(value.toFixed(3));
       return `${rounded >= 0 ? "+" : ""}${rounded}`;

benchmarks/harbor/results.test.ts

@@ -12,6 +12,7 @@ import { buildExperimentEvents, publishBenchmark } from "./publish-braintrust";
 import { renderMarkdown } from "./report";
 import { readBenchmarkArm, selectPrimaryReward, summarizeArm } from "./results";
 import { redactString, redactValue } from "./redact";
+import { assertProjectScopedCredential } from "./verify-project-scope";
 
 const temporaryDirectories: string[] = [];
 
@@ -257,6 +258,16 @@ describe("Harbor result ingestion", () => {
     }
   });
 
+  test("does not publish arms without graded trials", async () => {
+    const arm = readBenchmarkArm({ name: "candidate", path: fixture() });
+    for (const trial of arm.trials) trial.rewards = {};
+    await expect(
+      publishBenchmark([arm], "project", "experiment", "api-key"),
+    ).rejects.toThrow(
+      "Cannot publish benchmark without graded trials for: candidate",
+    );
+  });
+
   test("uses the lenient reward per trial and reports incomplete arms", () => {
     expect(selectPrimaryReward({ reward: 0, reward_lenient: 1 })).toEqual({
       key: "reward_lenient",
@@ -288,6 +299,12 @@ describe("Harbor result ingestion", () => {
         { ...summary, arm: "baseline", lenient: 0.2 },
       ]),
     ).toContain("+0.1 lenient");
+    const ungraded = renderMarkdown("test", [
+      { ...summary, arm: "candidate", scored: 0, ungraded: summary.trials },
+      { ...summary, arm: "baseline", scored: 0, ungraded: summary.trials },
+    ]);
+    expect(ungraded).toContain("candidate produced no graded trials");
+    expect(ungraded).not.toContain("Candidate minus baseline");
   });
 
   test("keeps full errors until redaction and clamps derived scores", () => {
@@ -369,11 +386,49 @@ describe("benchmark workflow hardening", () => {
     expect(workflow).not.toContain("baseSha = pull.base.sha");
     expect(workflow).toContain('HARBOR_VERSION: "0.21.0"');
     expect(workflow).toContain('CODEX_BENCHMARK_VERSION: "0.120.0"');
+    expect(workflow).toContain("issues: write\n      pull-requests: write");
+    expect(workflow).not.toContain(
+      "KERNEL_PROJECT: ${{ vars.KERNEL_PROJECT }}",
+    );
+    expect(workflow).toContain("all(.arms[]; .scored > 0)");
+    expect(workflow).toMatch(
+      /- name: Mark the PR benchmark as running\n\s+if:.*\n\s+continue-on-error: true/,
+    );
+    expect(workflow).toMatch(
+      /- name: Update PR benchmark comment\n\s+if:.*\n\s+continue-on-error: true/,
+    );
     expect(workflow).toContain(
       'statuses=(--status "candidate=${CANDIDATE_STATUS:-1}")',
     );
   });
 
+  test("requires the benchmark credential to resolve to one project", () => {
+    expect(() =>
+      assertProjectScopedCredential({
+        authorization: {
+          credential_scope: { project_id: "project" },
+          effective_scope: { project_id: "project" },
+        },
+      }),
+    ).not.toThrow();
+    expect(() =>
+      assertProjectScopedCredential({
+        authorization: {
+          credential_scope: { project_id: null },
+          effective_scope: { project_id: null },
+        },
+      }),
+    ).toThrow();
+    expect(() =>
+      assertProjectScopedCredential({
+        authorization: {
+          credential_scope: { project_id: "credential-project" },
+          effective_scope: { project_id: "other-project" },
+        },
+      }),
+    ).toThrow();
+  });
+
   test("excludes private keys and forwards only the selected provider", () => {
     const dockerignore = readFileSync(
       join(process.cwd(), ".dockerignore"),
@@ -387,7 +442,15 @@ describe("benchmark workflow hardening", () => {
       join(process.cwd(), "benchmarks/harbor/clawbench/verify-task.py"),
       "utf8",
     );
+    const taskPreparer = readFileSync(
+      join(process.cwd(), "benchmarks/harbor/clawbench/prepare-task.py"),
+      "utf8",
+    );
     expect(dockerignore.split("\n")).toContain("*.pem");
+    expect(runner).not.toContain("KERNEL_PROJECT");
+    expect(runner).toContain('"${KERNEL_API_BASE_URL%/}/auth/context"');
+    expect(runner).toContain('bun "$benchmark_dir/verify-project-scope.ts"');
+    expect(taskPreparer).not.toContain("KERNEL_PROJECT");
     expect(verifier).toContain('"mcp__kernel__execute_playwright_code"');
     expect(verifier).toContain('"kernel__execute_playwright_code"');
     expect(verifier).toContain('"execute_playwright_code"');

benchmarks/harbor/verify-project-scope.ts

@@ -0,0 +1,29 @@
+import { z } from "zod";
+
+const authContextSchema = z.object({
+  authorization: z.object({
+    credential_scope: z.object({ project_id: z.string().min(1) }),
+    effective_scope: z.object({ project_id: z.string().min(1) }),
+  }),
+});
+
+export function assertProjectScopedCredential(value: unknown): void {
+  const context = authContextSchema.parse(value);
+  if (
+    context.authorization.effective_scope.project_id !==
+    context.authorization.credential_scope.project_id
+  ) {
+    throw new Error("Credential and effective project scopes differ");
+  }
+}
+
+if (import.meta.main) {
+  try {
+    assertProjectScopedCredential(JSON.parse(await Bun.stdin.text()));
+  } catch {
+    console.error(
+      "The benchmark credential must resolve to one matching project scope",
+    );
+    process.exit(1);
+  }
+}