refactor(test): make all Docker tests hermetic

.bazelrc

@@ -4,3 +4,21 @@ common --repository_cache=~/.cache/bazel-repo-cache
 # Resolve protoc from a registered toolchain (toolchains_protoc provides a
 # prebuilt binary) instead of compiling protoc from source. Introduced in Bazel 7.
 common --incompatible_enable_proto_toolchain_resolution
+
+# Hermetic execution: actions and tests run with a static environment instead
+# of inheriting the client's, so results are reproducible and cache correctly
+# across shells and machines.
+common --incompatible_strict_action_env
+
+# Stricter sandbox: sandboxed actions get no network access by default (each
+# runs in its own network namespace with only a private loopback). Tests that
+# talk to the Docker daemon and connect to its published host ports opt back
+# in with tags = ["requires-network"].
+common --sandbox_default_allow_network=false
+
+# Docker-based tests are hermetic (all inputs are declared Bazel data deps),
+# but a few environment knobs are deliberately passed through when set:
+# DOCKER_HOST selects a non-default Docker daemon; SKIP_CLEANUP=true keeps
+# containers running after a failure for inspection.
+test --test_env=DOCKER_HOST
+test --test_env=SKIP_CLEANUP

AGENTS.md

@@ -308,7 +308,7 @@ deps = [
 
 **Integration tests** use Docker Compose via `testutil.ComposeStack`:
 - Package naming: folder name as package (NOT `*_test` suffix)
-- Bazel: add `tags = ["integration"]` and `data = [...]` for compose/schema files
+- Bazel: add `tags = ["integration", "requires-network"]` and `data = [...]` for every input the test reads (compose file, schema dirs, Dockerfiles, Bazel-built `*_linux` service binaries). Tests are hermetic: never resolve the repo root — resolve inputs from runfiles via `testutil.Runfile` and stage docker build contexts with `testutil.WithBuildContext`.
 - Use `testutil.NewComposeStack()` with meaningful context (e.g., `"ext-storage-mysql"`)
 
 See [doc/howto/TESTING.md](doc/howto/TESTING.md) for full testing guide.

Makefile

@@ -60,7 +60,9 @@ build: ## Build all services and examples
 	@$(BAZEL) build //...
 	@echo "Build complete!"
 
-# Build Linux binaries required for Docker containers
+# Build Linux binaries required for the local docker-compose flows (local-*).
+# Bazel-run integration/e2e tests do NOT need these: they build the service
+# binaries hermetically as data dependencies.
 build-all-linux: build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-linux build-runway-linux ## Build all Linux binaries for Docker
 	@echo "All Linux binaries ready for Docker"
 
@@ -132,7 +134,7 @@ clean-proto: ## Clean generated proto files
 deps: tidy-go ## Download and tidy Go dependencies
 	@echo "Dependencies installed!"
 
-e2e-test: build-all-linux ## Run end-to-end tests (hermetic, auto-builds binaries; runs in parallel)
+e2e-test: ## Run end-to-end tests (hermetic; Bazel builds all inputs; runs in parallel)
 	@echo "Running end-to-end tests (parallel)..."
 	@$(BAZEL) test //test/e2e/... --test_output=errors
 
@@ -147,7 +149,7 @@ gazelle: ## Update BUILD.bazel files
 	@echo "Running Gazelle to update BUILD files..."
 	@$(BAZEL) run //:gazelle
 
-integration-test: build-all-linux ## Run all integration tests (auto-builds binaries; runs in parallel)
+integration-test: ## Run all integration tests (hermetic; Bazel builds all inputs; runs in parallel)
 	@echo "Running all integration tests (parallel)..."
 	@$(BAZEL) test //test/integration/... --test_output=errors
 
@@ -159,11 +161,11 @@ integration-test-extensions: ## Run extension integration tests (runs in paralle
 	@echo "Running extension integration tests (parallel)..."
 	@$(BAZEL) test //test/integration/submitqueue/extension/... //test/integration/extension/... --test_output=errors
 
-integration-test-submitqueue-gateway: build-submitqueue-gateway-linux ## Run Gateway integration tests (auto-builds binary)
+integration-test-submitqueue-gateway: ## Run Gateway integration tests
 	@echo "Running Gateway integration tests..."
 	@$(BAZEL) test //test/integration/submitqueue/gateway:go_default_test --test_output=streamed
 
-integration-test-submitqueue-orchestrator: build-submitqueue-orchestrator-linux ## Run Orchestrator integration tests (auto-builds binary)
+integration-test-submitqueue-orchestrator: ## Run Orchestrator integration tests
 	@echo "Running Orchestrator integration tests..."
 	@$(BAZEL) test //test/integration/submitqueue/orchestrator:go_default_test --test_output=streamed
 

doc/howto/TESTING.md

@@ -62,7 +62,7 @@ make e2e-test
 
 # Build
 make build                          # Build all targets
-make build-all-linux                # Build Linux binaries for Docker
+make build-all-linux                # Build Linux binaries for the local docker-compose flows (tests build their own)
 ```
 
 ### Testing Levels

service/runway/server/BUILD.bazel

@@ -1,4 +1,12 @@
-load("@rules_go//go:def.bzl", "go_binary", "go_library")
+load("@rules_go//go:def.bzl", "go_binary", "go_cross_binary", "go_library")
+
+exports_files(
+    [
+        "Dockerfile",
+        "docker-compose.yml",
+    ],
+    visibility = ["//visibility:public"],
+)
 
 go_library(
     name = "go_default_library",
@@ -32,3 +40,13 @@ go_binary(
     embed = [":go_default_library"],
     visibility = ["//visibility:public"],
 )
+
+# Linux binary consumed by the Docker image build. Docker-based tests depend on
+# this target as data so Bazel builds the exact binary that ends up in the
+# container image — no pre-built .docker-bin artifacts required.
+go_cross_binary(
+    name = "runway_linux",
+    platform = "@rules_go//go/toolchain:linux_amd64",
+    target = ":runway",
+    visibility = ["//visibility:public"],
+)

service/stovepipe/server/BUILD.bazel

@@ -1,4 +1,9 @@
-load("@rules_go//go:def.bzl", "go_binary", "go_library")
+load("@rules_go//go:def.bzl", "go_binary", "go_cross_binary", "go_library")
+
+exports_files(
+    ["Dockerfile"],
+    visibility = ["//visibility:public"],
+)
 
 go_library(
     name = "go_default_library",
@@ -40,3 +45,13 @@ go_binary(
     embed = [":go_default_library"],
     visibility = ["//visibility:public"],
 )
+
+# Linux binary consumed by the Docker image build. Docker-based tests depend on
+# this target as data so Bazel builds the exact binary that ends up in the
+# container image — no pre-built .docker-bin artifacts required.
+go_cross_binary(
+    name = "stovepipe_linux",
+    platform = "@rules_go//go/toolchain:linux_amd64",
+    target = ":stovepipe",
+    visibility = ["//visibility:public"],
+)

service/submitqueue/gateway/server/BUILD.bazel

@@ -1,7 +1,11 @@
-load("@rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
+load("@rules_go//go:def.bzl", "go_binary", "go_cross_binary", "go_library", "go_test")
 
 exports_files(
-    ["docker-compose.yml"],
+    [
+        "Dockerfile",
+        "docker-compose.yml",
+        "queues.yaml",
+    ],
     visibility = ["//visibility:public"],
 )
 
@@ -41,6 +45,16 @@ go_binary(
     visibility = ["//visibility:public"],
 )
 
+# Linux binary consumed by the Docker image build. Docker-based tests depend on
+# this target as data so Bazel builds the exact binary that ends up in the
+# container image — no pre-built .docker-bin artifacts required.
+go_cross_binary(
+    name = "gateway_linux",
+    platform = "@rules_go//go/toolchain:linux_amd64",
+    target = ":gateway",
+    visibility = ["//visibility:public"],
+)
+
 go_test(
     name = "go_default_test",
     srcs = ["main_test.go"],

service/submitqueue/orchestrator/server/BUILD.bazel

@@ -1,7 +1,10 @@
-load("@rules_go//go:def.bzl", "go_binary", "go_library")
+load("@rules_go//go:def.bzl", "go_binary", "go_cross_binary", "go_library")
 
 exports_files(
-    ["docker-compose.yml"],
+    [
+        "Dockerfile",
+        "docker-compose.yml",
+    ],
     visibility = ["//visibility:public"],
 )
 
@@ -71,3 +74,13 @@ go_binary(
     embed = [":orchestrator_lib"],
     visibility = ["//visibility:public"],
 )
+
+# Linux binary consumed by the Docker image build. Docker-based tests depend on
+# this target as data so Bazel builds the exact binary that ends up in the
+# container image — no pre-built .docker-bin artifacts required.
+go_cross_binary(
+    name = "orchestrator_linux",
+    platform = "@rules_go//go/toolchain:linux_amd64",
+    target = ":orchestrator",
+    visibility = ["//visibility:public"],
+)

test/e2e/stovepipe/BUILD.bazel

@@ -7,16 +7,16 @@ go_test(
         "suite_test.go",
     ],
     data = [
-        "//:MODULE.bazel",
-        "//:go.mod",
         "//platform/extension/messagequeue/mysql/schema",
         "//service/stovepipe:docker-compose.yml",
+        "//service/stovepipe/server:Dockerfile",
+        "//service/stovepipe/server:stovepipe_linux",
         "//stovepipe/extension/storage/mysql/schema",
     ],
     tags = [
         "e2e",
-        "external",
         "integration",
+        "requires-network",
     ],
     deps = [
         "//api/stovepipe/protopb:go_default_library",

test/e2e/stovepipe/suite_test.go

@@ -16,13 +16,16 @@ package e2e_test
 
 // Stovepipe end-to-end tests.
 //
-// These tests use docker-compose from service/stovepipe/docker-compose.yml,
-// which requires a pre-built Linux binary. Run with the make target (builds
-// binaries + runs the test):
+// These tests use docker-compose from service/stovepipe/docker-compose.yml.
+// They are hermetic: the stovepipe image is built from a staged context whose
+// inputs (Bazel-built Linux binary, Dockerfile) are all declared data
+// dependencies of the test target.
+//
+// Run with:
 //
 //   make e2e-test
 //
-// or only this package (after building the binary):
+// or only this package:
 //
 //   bazel test //test/e2e/stovepipe:stovepipe_test
 //
@@ -35,7 +38,6 @@ package e2e_test
 import (
 	"context"
 	"database/sql"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -80,12 +82,15 @@ func (s *StovepipeE2ESuite) SetupSuite() {
 
 	s.log.Logf("Starting Stovepipe e2e test suite using docker-compose")
 
-	// Set REPO_ROOT for the docker-compose build context.
-	repoRoot := testutil.FindRepoRoot(t)
-	t.Setenv("REPO_ROOT", repoRoot)
-
-	composeFile := filepath.Join(repoRoot, "service/stovepipe/docker-compose.yml")
-	s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "e2e-stovepipe")
+	// Compose file and image build inputs come from the test runfiles; the
+	// stovepipe image is built from a staged build context assembled entirely
+	// from declared data dependencies.
+	composeFile := testutil.Runfile("service/stovepipe/docker-compose.yml")
+	s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "e2e-stovepipe",
+		testutil.WithBuildContext(map[string]string{
+			".docker-bin/stovepipe":               "service/stovepipe/server/stovepipe_linux",
+			"service/stovepipe/server/Dockerfile": "service/stovepipe/server/Dockerfile",
+		}))
 
 	err := s.stack.Up()
 	require.NoError(t, err, "failed to start compose stack")

test/e2e/submitqueue/BUILD.bazel

@@ -7,17 +7,22 @@ go_test(
         "suite_test.go",
     ],
     data = [
-        "//:MODULE.bazel",
-        "//:go.mod",
         "//platform/extension/counter/mysql/schema",
         "//platform/extension/messagequeue/mysql/schema",
+        "//service/runway/server:Dockerfile",
+        "//service/runway/server:runway_linux",
         "//service/submitqueue:docker-compose.yml",
+        "//service/submitqueue/gateway/server:Dockerfile",
+        "//service/submitqueue/gateway/server:gateway_linux",
+        "//service/submitqueue/gateway/server:queues.yaml",
+        "//service/submitqueue/orchestrator/server:Dockerfile",
+        "//service/submitqueue/orchestrator/server:orchestrator_linux",
         "//submitqueue/extension/storage/mysql/schema",
     ],
     tags = [
         "e2e",
-        "external",
         "integration",
+        "requires-network",
     ],
     deps = [
         "//api/base/change/protopb:go_default_library",

test/e2e/submitqueue/suite_test.go

@@ -16,16 +16,17 @@ package e2e_test
 
 // E2E Integration Tests
 //
-// These tests use docker-compose from service/submitqueue/docker-compose.yml
-// which requires pre-built Linux binaries.
+// These tests use docker-compose from service/submitqueue/docker-compose.yml.
+// They are hermetic: the service images are built from a staged context whose
+// inputs (Bazel-built Linux binaries, Dockerfiles, queues.yaml) are all
+// declared data dependencies of the test target.
 //
-// Run with make target (builds binaries + runs test):
+// Run with:
 //   make e2e-test
 
 import (
 	"context"
 	"database/sql"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -76,14 +77,20 @@ func (s *E2EIntegrationSuite) SetupSuite() {
 
 	s.log.Logf("Starting E2E integration test suite using docker-compose")
 
-	// Set REPO_ROOT for docker-compose volume mounts and build context
-	repoRoot := testutil.FindRepoRoot(t)
-	t.Setenv("REPO_ROOT", repoRoot)
-
-	// Use docker-compose from service/submitqueue (full stack)
-	// NOTE: Assumes Linux binaries are pre-built via make target
-	composeFile := filepath.Join(repoRoot, "service/submitqueue/docker-compose.yml")
-	s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "e2e-submitqueue")
+	// Use docker-compose from service/submitqueue (full stack), resolved from
+	// the test runfiles. All three service images are built from a staged
+	// build context assembled entirely from declared data dependencies.
+	composeFile := testutil.Runfile("service/submitqueue/docker-compose.yml")
+	s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "e2e-submitqueue",
+		testutil.WithBuildContext(map[string]string{
+			".docker-bin/gateway":                                "service/submitqueue/gateway/server/gateway_linux",
+			".docker-bin/orchestrator":                           "service/submitqueue/orchestrator/server/orchestrator_linux",
+			".docker-bin/runway":                                 "service/runway/server/runway_linux",
+			"service/submitqueue/gateway/server/Dockerfile":      "service/submitqueue/gateway/server/Dockerfile",
+			"service/submitqueue/gateway/server/queues.yaml":     "service/submitqueue/gateway/server/queues.yaml",
+			"service/submitqueue/orchestrator/server/Dockerfile": "service/submitqueue/orchestrator/server/Dockerfile",
+			"service/runway/server/Dockerfile":                   "service/runway/server/Dockerfile",
+		}))
 
 	// Start the compose stack (Gateway + Orchestrator + 2 MySQL DBs)
 	err := s.stack.Up()

test/integration/extension/counter/mysql/BUILD.bazel

@@ -5,13 +5,11 @@ go_test(
     srcs = ["counter_test.go"],
     data = [
         "docker-compose.yml",
-        "//:MODULE.bazel",
-        "//:go.mod",
         "//platform/extension/counter/mysql/schema",
     ],
     tags = [
-        "external",
         "integration",
+        "requires-network",
     ],
     deps = [
         "//platform/extension/counter/mysql:go_default_library",

test/integration/extension/messagequeue/mysql/BUILD.bazel

@@ -5,13 +5,11 @@ go_test(
     srcs = ["queue_test.go"],
     data = [
         "docker-compose.yml",
-        "//:MODULE.bazel",
-        "//:go.mod",
         "//platform/extension/messagequeue/mysql/schema",
     ],
     tags = [
-        "external",
         "integration",
+        "requires-network",
     ],
     deps = [
         "//platform/base/messagequeue:go_default_library",

test/integration/stovepipe/BUILD.bazel

@@ -4,15 +4,15 @@ go_test(
     name = "go_default_test",
     srcs = ["suite_test.go"],
     data = [
-        "//:MODULE.bazel",
-        "//:go.mod",
         "//platform/extension/messagequeue/mysql/schema",
         "//service/stovepipe:docker-compose.yml",
+        "//service/stovepipe/server:Dockerfile",
+        "//service/stovepipe/server:stovepipe_linux",
         "//stovepipe/extension/storage/mysql/schema",
     ],
     tags = [
-        "external",
         "integration",
+        "requires-network",
     ],
     deps = [
         "//api/stovepipe/protopb:go_default_library",