From c8a1adf01a6c11a810a7de42f427fc5366a24c0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Van=C4=9Bk?= Date: Tue, 23 Jun 2026 09:01:00 +0200 Subject: [PATCH 1/4] feat(orchestrator): make build-reserved-disk-space-mb default 256MB This value has been set to 256 in production for a couple of weeks without issues, so promote it to the default for the feature flag. --- packages/shared/pkg/featureflags/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/shared/pkg/featureflags/flags.go b/packages/shared/pkg/featureflags/flags.go index 11a3f9c8ff..7cc83972cb 100644 --- a/packages/shared/pkg/featureflags/flags.go +++ b/packages/shared/pkg/featureflags/flags.go @@ -299,7 +299,7 @@ var ( // BuildReservedDiskSpaceMB is the amount of disk space in MB reserved for root on the guest filesystem. // Reserved blocks are only usable by root (uid 0), protecting the guest OS from disk-full conditions. - BuildReservedDiskSpaceMB = NewIntFlag("build-reserved-disk-space-mb", 0) + BuildReservedDiskSpaceMB = NewIntFlag("build-reserved-disk-space-mb", 256) // MaxStartingInstancesPerNode limits concurrent sandbox start/resume operations on a single orchestrator node. // Must be > 0. From fc83febb14197e7e69b8beca0a812957e3c29749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Van=C4=9Bk?= Date: Tue, 23 Jun 2026 14:19:26 +0200 Subject: [PATCH 2/4] test(integration): bump seeded base build disk to 1024MB Raise the integration test base template build's free_disk_size_mb from 512 to 1024 (and total accordingly) to give disk-heavy template steps headroom above the default 256MB build-reserved-disk-space-mb reservation. --- tests/integration/seed.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/seed.go b/tests/integration/seed.go index ba611f8252..33511ef8b5 100644 --- a/tests/integration/seed.go +++ b/tests/integration/seed.go @@ -223,7 +223,7 @@ INSERT INTO env_builds ( cluster_node_id, version, created_at, updated_at ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, CURRENT_TIMESTAMP) `, build.id, "FROM e2bdev/base:latest", dbtypes.BuildStatusUploaded, - 2, 512, 512, 1982, "vmlinux-6.1.158-c1a568c", "v1.14.1_431f1fc", pkg.Version, + 2, 512, 1024, 2494, "vmlinux-6.1.158-c1a568c", "v1.14.1_431f1fc", pkg.Version, "integration-test-node", templates.TemplateV1Version, build.createdAt) } else { err = db.TestsRawSQL(ctx, ` @@ -233,7 +233,7 @@ INSERT INTO env_builds ( cluster_node_id, version, updated_at ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, CURRENT_TIMESTAMP) `, build.id, "FROM e2bdev/base:latest", dbtypes.BuildStatusUploaded, - 2, 512, 512, 1982, "vmlinux-6.1.158-c1a568c", "v1.14.1_431f1fc", pkg.Version, + 2, 512, 1024, 2494, "vmlinux-6.1.158-c1a568c", "v1.14.1_431f1fc", pkg.Version, "integration-test-node", templates.TemplateV1Version) } if err != nil { From cb71c35fa5ea25805ffe9cedf357cd5933a00742 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Van=C4=9Bk?= Date: Tue, 23 Jun 2026 14:42:59 +0200 Subject: [PATCH 3/4] test(integration): bump seeded base build disk to 2048MB 1024MB was insufficient: after the default 256MB build-reserved-disk-space-mb reservation, disk-heavy template steps like create-next-app still ran out of space in CI. Raise free_disk_size_mb to 2048 (total adjusted accordingly). --- tests/integration/seed.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/seed.go b/tests/integration/seed.go index 33511ef8b5..c9e53d2c37 100644 --- a/tests/integration/seed.go +++ b/tests/integration/seed.go @@ -223,7 +223,7 @@ INSERT INTO env_builds ( cluster_node_id, version, created_at, updated_at ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, CURRENT_TIMESTAMP) `, build.id, "FROM e2bdev/base:latest", dbtypes.BuildStatusUploaded, - 2, 512, 1024, 2494, "vmlinux-6.1.158-c1a568c", "v1.14.1_431f1fc", pkg.Version, + 2, 512, 2048, 3727, "vmlinux-6.1.158-c1a568c", "v1.14.1_431f1fc", pkg.Version, "integration-test-node", templates.TemplateV1Version, build.createdAt) } else { err = db.TestsRawSQL(ctx, ` @@ -233,7 +233,7 @@ INSERT INTO env_builds ( cluster_node_id, version, updated_at ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, CURRENT_TIMESTAMP) `, build.id, "FROM e2bdev/base:latest", dbtypes.BuildStatusUploaded, - 2, 512, 1024, 2494, "vmlinux-6.1.158-c1a568c", "v1.14.1_431f1fc", pkg.Version, + 2, 512, 2048, 3727, "vmlinux-6.1.158-c1a568c", "v1.14.1_431f1fc", pkg.Version, "integration-test-node", templates.TemplateV1Version) } if err != nil { From 73f489ca75d2a085c95ab28e9e3366260020e307 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Van=C4=9Bk?= Date: Tue, 23 Jun 2026 15:15:43 +0200 Subject: [PATCH 4/4] test(integration): build CI base template with larger disk The seeded env_builds disk columns are metadata only and do not size the actual base template rootfs, so bumping them did not fix CI. The base template CI builds from is created by create-build with the default -disk 1024MB; after the 256MB build-reserved-disk-space-mb reservation, non-root steps like create-next-app in TestCommandKillNextApp run out of space (exit 228). Revert the ineffective seed changes, make build-template's disk size configurable (default unchanged at 1024MB), and have the integration-test composite action build the base template with 2048MB so disk-heavy steps fit above the reservation. Verified locally: 1024MB base fails with exit 228, 2048MB base passes. --- .github/actions/build-sandbox-template/action.yml | 8 +++++++- packages/orchestrator/Makefile | 7 ++++++- tests/integration/seed.go | 4 ++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/actions/build-sandbox-template/action.yml b/.github/actions/build-sandbox-template/action.yml index 4e185f4941..08109a29ca 100644 --- a/.github/actions/build-sandbox-template/action.yml +++ b/.github/actions/build-sandbox-template/action.yml @@ -18,6 +18,10 @@ inputs: description: "Number of frame encode workers" required: false default: "" + disk_size_mb: + description: "Base template rootfs disk size in MB. Larger than the create-build default to leave headroom for disk-heavy build steps after the build-reserved-disk-space-mb reservation." + required: false + default: "2048" runs: using: "composite" @@ -27,6 +31,7 @@ runs: TEMPLATE_ID: "2j6ly824owf4awgai1xo" KERNEL_VERSION: "vmlinux-6.1.158-c1a568c" FIRECRACKER_VERSION: "v1.14.1_431f1fc" + DISK_SIZE_MB: ${{ inputs.disk_size_mb }} COMPRESS_ENABLED: ${{ inputs.compress_enabled }} COMPRESS_TYPE: ${{ inputs.compress_type }} COMPRESS_LEVEL: ${{ inputs.compress_level }} @@ -50,5 +55,6 @@ runs: TEMPLATE_ID=${TEMPLATE_ID} \ BUILD_ID=${BUILD_ID} \ KERNEL_VERSION=${KERNEL_VERSION} \ - FC_VERSION=${FC_VERSION} + FC_VERSION=${FC_VERSION} \ + DISK_SIZE_MB=${DISK_SIZE_MB} shell: bash diff --git a/packages/orchestrator/Makefile b/packages/orchestrator/Makefile index 06f9fe67d1..473a72aec5 100644 --- a/packages/orchestrator/Makefile +++ b/packages/orchestrator/Makefile @@ -147,6 +147,10 @@ test-docker: @echo "Done" .PHONY: build-template +# Disk size (MB) of the base template rootfs. Matches create-build's own +# default; callers (e.g. integration-test CI) can override to leave headroom +# for disk-heavy build steps after the build-reserved-disk-space-mb reservation. +DISK_SIZE_MB ?= 1024 build-template: fetch-busybox TEMPLATE_BUCKET_NAME=$(TEMPLATE_BUCKET_NAME) \ GOOGLE_SERVICE_ACCOUNT_BASE64=$(GOOGLE_SERVICE_ACCOUNT_BASE64) \ @@ -163,7 +167,8 @@ build-template: fetch-busybox -template $(TEMPLATE_ID) \ -to-build $(BUILD_ID) \ -kernel $(KERNEL_VERSION) \ - -firecracker $(FIRECRACKER_VERSION) + -firecracker $(FIRECRACKER_VERSION) \ + -disk $(DISK_SIZE_MB) .PHONY: migrate migrate: diff --git a/tests/integration/seed.go b/tests/integration/seed.go index c9e53d2c37..ba611f8252 100644 --- a/tests/integration/seed.go +++ b/tests/integration/seed.go @@ -223,7 +223,7 @@ INSERT INTO env_builds ( cluster_node_id, version, created_at, updated_at ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, CURRENT_TIMESTAMP) `, build.id, "FROM e2bdev/base:latest", dbtypes.BuildStatusUploaded, - 2, 512, 2048, 3727, "vmlinux-6.1.158-c1a568c", "v1.14.1_431f1fc", pkg.Version, + 2, 512, 512, 1982, "vmlinux-6.1.158-c1a568c", "v1.14.1_431f1fc", pkg.Version, "integration-test-node", templates.TemplateV1Version, build.createdAt) } else { err = db.TestsRawSQL(ctx, ` @@ -233,7 +233,7 @@ INSERT INTO env_builds ( cluster_node_id, version, updated_at ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, CURRENT_TIMESTAMP) `, build.id, "FROM e2bdev/base:latest", dbtypes.BuildStatusUploaded, - 2, 512, 2048, 3727, "vmlinux-6.1.158-c1a568c", "v1.14.1_431f1fc", pkg.Version, + 2, 512, 512, 1982, "vmlinux-6.1.158-c1a568c", "v1.14.1_431f1fc", pkg.Version, "integration-test-node", templates.TemplateV1Version) } if err != nil {