diff --git a/.github/workflows/benchmark-clawbench.yml b/.github/workflows/benchmark-clawbench.yml index 57531614..3d87ee99 100644 --- a/.github/workflows/benchmark-clawbench.yml +++ b/.github/workflows/benchmark-clawbench.yml @@ -284,7 +284,7 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: repository: kernel/ClawBench - ref: 45a71c4b0c78186851c94cfc77bfe619c9e01387 + ref: c7feaa2435ca8115c0762c44e13885fe5adf3e98 path: clawbench persist-credentials: false @@ -326,7 +326,7 @@ jobs: shell: bash env: CLAWBENCH_REPO: ${{ github.workspace }}/clawbench - CLAWBENCH_REF: 45a71c4b0c78186851c94cfc77bfe619c9e01387 + CLAWBENCH_REF: c7feaa2435ca8115c0762c44e13885fe5adf3e98 HARBOR_BENCHMARK_TIMEOUT: 4h BENCHMARK_AGENT: ${{ needs.resolve.outputs.agent }} BENCHMARK_TASK: ${{ needs.resolve.outputs.task }} diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 8ff24460..771e7b6e 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -18,7 +18,7 @@ The image records the current Git SHA, and the generated task records the ClawBe - `uv`, Harbor 0.21.0, and `harbor-hypeman` 0.1.1 - Hypeman CLI credentials -- a ClawBench checkout containing pinned commit `45a71c4` +- a ClawBench checkout containing pinned commit `c7feaa2` - `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project; its credential scope is the project source of truth - `PURELY_MAIL_API_KEY` and `PURELY_MAIL_DOMAIN` for ClawBench account tasks - `OPENAI_API_KEY` for Codex, or Anthropic credentials for Claude Code diff --git a/benchmarks/harbor/clawbench/run.sh b/benchmarks/harbor/clawbench/run.sh index 772e9cf6..93a752a9 100755 --- a/benchmarks/harbor/clawbench/run.sh +++ b/benchmarks/harbor/clawbench/run.sh @@ -19,7 +19,7 @@ repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) benchmark_dir="$repo_root/benchmarks/harbor" image_env="$benchmark_dir/.image.env" clawbench_repo=${CLAWBENCH_REPO:-$repo_root/../ClawBench} -clawbench_ref=${CLAWBENCH_REF:-45a71c4b0c78186851c94cfc77bfe619c9e01387} +clawbench_ref=${CLAWBENCH_REF:-c7feaa2435ca8115c0762c44e13885fe5adf3e98} [[ -f "$image_env" ]] || { echo "Missing $image_env; run benchmarks/harbor/build-image.sh first" >&2 diff --git a/benchmarks/harbor/results.test.ts b/benchmarks/harbor/results.test.ts index 941c841d..7634ad6d 100644 --- a/benchmarks/harbor/results.test.ts +++ b/benchmarks/harbor/results.test.ts @@ -386,6 +386,9 @@ describe("benchmark workflow hardening", () => { expect(workflow).not.toContain("baseSha = pull.base.sha"); expect(workflow).toContain('HARBOR_VERSION: "0.21.0"'); expect(workflow).toContain('CODEX_BENCHMARK_VERSION: "0.120.0"'); + expect( + workflow.match(/c7feaa2435ca8115c0762c44e13885fe5adf3e98/g), + ).toHaveLength(2); expect(workflow).toContain("issues: write\n pull-requests: write"); expect(workflow).not.toContain( "KERNEL_PROJECT: ${{ vars.KERNEL_PROJECT }}", @@ -448,6 +451,7 @@ describe("benchmark workflow hardening", () => { ); expect(dockerignore.split("\n")).toContain("*.pem"); expect(runner).not.toContain("KERNEL_PROJECT"); + expect(runner).toContain("c7feaa2435ca8115c0762c44e13885fe5adf3e98"); expect(runner).toContain('"${KERNEL_API_BASE_URL%/}/auth/context"'); expect(runner).toContain('bun "$benchmark_dir/verify-project-scope.ts"'); expect(taskPreparer).not.toContain("KERNEL_PROJECT");