From e7248be5b7b7479f27cf80a633e8337ed519c8b5 Mon Sep 17 00:00:00 2001 From: Sayan- <1415138+Sayan-@users.noreply.github.com> Date: Fri, 28 Aug 2026 17:59:19 +0000 Subject: [PATCH 1/2] Build chromium images on the shared buildx builder Both image jobs ran on 2-core hosted runners with type=gha layer caching. Chromium layers are large, so mode=max churned against the repo's 10GB Actions cache quota and builds swung 79s to 700s depending on eviction luck. Move the jobs to the self-hosted pool and attach to the persistent deft-shared builder, whose local layer store replaces the gha cache entirely. Docker Hub push behavior and tags are unchanged; the e2e job pulls the same images as before. The pristine-context reset and isolated Docker config mirror kernel/kernel's auth-flow-simulator workflow, which pioneered this setup: persistent runner workdirs carry file modes that bust COPY cache keys, and the shared BUILDX_CONFIG path is the one root-run CI from other repos cannot clobber. Co-Authored-By: Claude Fable 5 --- .github/buildkitd.toml | 34 +++++++++++++++++++ .github/workflows/chromium-headful-image.yaml | 31 +++++++++++++++-- .../workflows/chromium-headless-image.yaml | 31 +++++++++++++++-- 3 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 .github/buildkitd.toml diff --git a/.github/buildkitd.toml b/.github/buildkitd.toml new file mode 100644 index 00000000..d6ed545e --- /dev/null +++ b/.github/buildkitd.toml @@ -0,0 +1,34 @@ +# GC config for the shared persistent builder (deft-shared) on the +# self-hosted runners. Mirrors BuildKit's default policy structure with two +# changes: cache-mount/local-context records survive 7 days of disuse instead +# of 48h (api-ecs is path-filtered and can idle past 48h), and total builder +# storage is capped at 150GB with a 100GB free-space floor. +# +# The authoritative copy is installed on the host by kernel/infra +# (roles/dev_shared_buildx_builder); this copy is the fallback if a job ever +# recreates the builder. Keep the two in sync. BuildKit reads this only at +# builder creation; applying a change requires `docker buildx rm deft-shared`. +[worker.oci] + gc = true + +[[worker.oci.gcpolicy]] + filters = ["type==source.local", "type==exec.cachemount", "type==source.git.checkout"] + keepDuration = "168h" + maxUsedSpace = "40GB" + +[[worker.oci.gcpolicy]] + keepDuration = "1440h" + reservedSpace = "10GB" + minFreeSpace = "100GB" + maxUsedSpace = "150GB" + +[[worker.oci.gcpolicy]] + reservedSpace = "10GB" + minFreeSpace = "100GB" + maxUsedSpace = "150GB" + +[[worker.oci.gcpolicy]] + all = true + reservedSpace = "10GB" + minFreeSpace = "100GB" + maxUsedSpace = "150GB" diff --git a/.github/workflows/chromium-headful-image.yaml b/.github/workflows/chromium-headful-image.yaml index ea56f29d..f47a35de 100644 --- a/.github/workflows/chromium-headful-image.yaml +++ b/.github/workflows/chromium-headful-image.yaml @@ -5,7 +5,7 @@ on: jobs: docker: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux] permissions: contents: read steps: @@ -17,6 +17,28 @@ jobs: shell: bash run: echo "short_sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + # The docker build context must match the git tree exactly, including + # file modes: BuildKit's COPY cache key covers permission bits, and the + # runners' persistent _work checkouts carry modes frozen from whatever + # umask was active when each file was first written, which differs per + # runner and busts the layer cache on the shared builder. + - name: Reset workspace to a pristine build context + run: | + git checkout -- . + git clean -xdf + find . -path ./.git -prune -o -type f -perm -u+x -print0 | xargs -0 -r chmod 755 + find . -path ./.git -prune -o -type f ! -perm -u+x -print0 | xargs -0 -r chmod 644 + + - name: Create isolated Docker config + run: | + DOCKER_CONFIG="$(mktemp -d "${RUNNER_TEMP}/docker-config.XXXXXX")" + echo "DOCKER_CONFIG=${DOCKER_CONFIG}" >> "$GITHUB_ENV" + # BUILDX_CONFIG defaults to $DOCKER_CONFIG/buildx; without this the + # isolated DOCKER_CONFIG above would hide the shared builder. The + # buildx-ci path (not the default ~/.docker/buildx) avoids the tree + # that root-run CI from other repos clobbers to root-owned. + echo "BUILDX_CONFIG=${HOME}/.docker/buildx-ci" >> "$GITHUB_ENV" + - name: Login to Docker Hub uses: docker/login-action@v3 with: @@ -25,13 +47,16 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + name: deft-shared + cleanup: false + buildkitd-config: .github/buildkitd.toml - name: Build and push uses: docker/build-push-action@v6 with: + builder: deft-shared context: . file: images/chromium-headful/Dockerfile push: true tags: onkernel/chromium-headful:${{ steps.vars.outputs.short_sha }} - cache-from: type=gha - cache-to: type=gha,mode=max diff --git a/.github/workflows/chromium-headless-image.yaml b/.github/workflows/chromium-headless-image.yaml index be00d67b..5cff25e9 100644 --- a/.github/workflows/chromium-headless-image.yaml +++ b/.github/workflows/chromium-headless-image.yaml @@ -5,7 +5,7 @@ on: jobs: docker: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux] permissions: contents: read steps: @@ -17,6 +17,28 @@ jobs: shell: bash run: echo "short_sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + # The docker build context must match the git tree exactly, including + # file modes: BuildKit's COPY cache key covers permission bits, and the + # runners' persistent _work checkouts carry modes frozen from whatever + # umask was active when each file was first written, which differs per + # runner and busts the layer cache on the shared builder. + - name: Reset workspace to a pristine build context + run: | + git checkout -- . + git clean -xdf + find . -path ./.git -prune -o -type f -perm -u+x -print0 | xargs -0 -r chmod 755 + find . -path ./.git -prune -o -type f ! -perm -u+x -print0 | xargs -0 -r chmod 644 + + - name: Create isolated Docker config + run: | + DOCKER_CONFIG="$(mktemp -d "${RUNNER_TEMP}/docker-config.XXXXXX")" + echo "DOCKER_CONFIG=${DOCKER_CONFIG}" >> "$GITHUB_ENV" + # BUILDX_CONFIG defaults to $DOCKER_CONFIG/buildx; without this the + # isolated DOCKER_CONFIG above would hide the shared builder. The + # buildx-ci path (not the default ~/.docker/buildx) avoids the tree + # that root-run CI from other repos clobbers to root-owned. + echo "BUILDX_CONFIG=${HOME}/.docker/buildx-ci" >> "$GITHUB_ENV" + - name: Login to Docker Hub uses: docker/login-action@v3 with: @@ -25,13 +47,16 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + name: deft-shared + cleanup: false + buildkitd-config: .github/buildkitd.toml - name: Build and push uses: docker/build-push-action@v6 with: + builder: deft-shared context: . file: images/chromium-headless/image/Dockerfile push: true tags: onkernel/chromium-headless:${{ steps.vars.outputs.short_sha }} - cache-from: type=gha - cache-to: type=gha,mode=max From 5d3eb63ea6fe448bb892c95809f21871abd54f25 Mon Sep 17 00:00:00 2001 From: Sayan- <1415138+Sayan-@users.noreply.github.com> Date: Fri, 28 Aug 2026 19:28:51 +0000 Subject: [PATCH 2/2] Skip image builds for fork PRs The docker jobs now run on the persistent self-hosted pool, and this is a public repo: a fork PR must not execute on those runners. Fork PRs previously failed at Docker Hub login (forks get no secrets); with the guard they and the dependent e2e job skip instead. Co-Authored-By: Claude Fable 5 --- .github/workflows/chromium-headful-image.yaml | 4 ++++ .github/workflows/chromium-headless-image.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/chromium-headful-image.yaml b/.github/workflows/chromium-headful-image.yaml index f47a35de..417126e6 100644 --- a/.github/workflows/chromium-headful-image.yaml +++ b/.github/workflows/chromium-headful-image.yaml @@ -5,6 +5,10 @@ on: jobs: docker: + # Fork PRs get no secrets and now must not reach the self-hosted pool + # either: skipping here (rather than failing at registry login, as + # before) also skips the dependent e2e job via its needs chain. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: [self-hosted, linux] permissions: contents: read diff --git a/.github/workflows/chromium-headless-image.yaml b/.github/workflows/chromium-headless-image.yaml index 5cff25e9..26c6b64a 100644 --- a/.github/workflows/chromium-headless-image.yaml +++ b/.github/workflows/chromium-headless-image.yaml @@ -5,6 +5,10 @@ on: jobs: docker: + # Fork PRs get no secrets and now must not reach the self-hosted pool + # either: skipping here (rather than failing at registry login, as + # before) also skips the dependent e2e job via its needs chain. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: [self-hosted, linux] permissions: contents: read