diff --git a/.bazelrc b/.bazelrc index 55b803eb..0bf6c32e 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,3 +4,21 @@ common --repository_cache=~/.cache/bazel-repo-cache # Resolve protoc from a registered toolchain (toolchains_protoc provides a # prebuilt binary) instead of compiling protoc from source. Introduced in Bazel 7. common --incompatible_enable_proto_toolchain_resolution + +# Hermetic execution: actions and tests run with a static environment instead +# of inheriting the client's, so results are reproducible and cache correctly +# across shells and machines. +common --incompatible_strict_action_env + +# Stricter sandbox: sandboxed actions get no network access by default (each +# runs in its own network namespace with only a private loopback). Tests that +# talk to the Docker daemon and connect to its published host ports opt back +# in with tags = ["requires-network"]. +common --sandbox_default_allow_network=false + +# Docker-based tests are hermetic (all inputs are declared Bazel data deps), +# but a few environment knobs are deliberately passed through when set: +# DOCKER_HOST selects a non-default Docker daemon; SKIP_CLEANUP=true keeps +# containers running after a failure for inspection. +test --test_env=DOCKER_HOST +test --test_env=SKIP_CLEANUP diff --git a/AGENTS.md b/AGENTS.md index f8b2579e..1a63ddfd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -308,7 +308,7 @@ deps = [ **Integration tests** use Docker Compose via `testutil.ComposeStack`: - Package naming: folder name as package (NOT `*_test` suffix) -- Bazel: add `tags = ["integration"]` and `data = [...]` for compose/schema files +- Bazel: add `tags = ["integration", "requires-network"]` and `data = [...]` for every input the test reads (compose file, schema dirs, Dockerfiles, Bazel-built `*_linux` service binaries). Tests are hermetic: never resolve the repo root — resolve inputs from runfiles via `testutil.Runfile` and stage docker build contexts with `testutil.WithBuildContext`. - Use `testutil.NewComposeStack()` with meaningful context (e.g., `"ext-storage-mysql"`) See [doc/howto/TESTING.md](doc/howto/TESTING.md) for full testing guide. diff --git a/Makefile b/Makefile index a0178520..98be4828 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,9 @@ build: ## Build all services and examples @$(BAZEL) build //... @echo "Build complete!" -# Build Linux binaries required for Docker containers +# Build Linux binaries required for the local docker-compose flows (local-*). +# Bazel-run integration/e2e tests do NOT need these: they build the service +# binaries hermetically as data dependencies. build-all-linux: build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-linux build-runway-linux ## Build all Linux binaries for Docker @echo "All Linux binaries ready for Docker" @@ -132,7 +134,7 @@ clean-proto: ## Clean generated proto files deps: tidy-go ## Download and tidy Go dependencies @echo "Dependencies installed!" -e2e-test: build-all-linux ## Run end-to-end tests (hermetic, auto-builds binaries; runs in parallel) +e2e-test: ## Run end-to-end tests (hermetic; Bazel builds all inputs; runs in parallel) @echo "Running end-to-end tests (parallel)..." @$(BAZEL) test //test/e2e/... --test_output=errors @@ -147,7 +149,7 @@ gazelle: ## Update BUILD.bazel files @echo "Running Gazelle to update BUILD files..." @$(BAZEL) run //:gazelle -integration-test: build-all-linux ## Run all integration tests (auto-builds binaries; runs in parallel) +integration-test: ## Run all integration tests (hermetic; Bazel builds all inputs; runs in parallel) @echo "Running all integration tests (parallel)..." @$(BAZEL) test //test/integration/... --test_output=errors @@ -159,11 +161,11 @@ integration-test-extensions: ## Run extension integration tests (runs in paralle @echo "Running extension integration tests (parallel)..." @$(BAZEL) test //test/integration/submitqueue/extension/... //test/integration/extension/... --test_output=errors -integration-test-submitqueue-gateway: build-submitqueue-gateway-linux ## Run Gateway integration tests (auto-builds binary) +integration-test-submitqueue-gateway: ## Run Gateway integration tests @echo "Running Gateway integration tests..." @$(BAZEL) test //test/integration/submitqueue/gateway:go_default_test --test_output=streamed -integration-test-submitqueue-orchestrator: build-submitqueue-orchestrator-linux ## Run Orchestrator integration tests (auto-builds binary) +integration-test-submitqueue-orchestrator: ## Run Orchestrator integration tests @echo "Running Orchestrator integration tests..." @$(BAZEL) test //test/integration/submitqueue/orchestrator:go_default_test --test_output=streamed diff --git a/doc/howto/TESTING.md b/doc/howto/TESTING.md index d2839740..08893b8f 100644 --- a/doc/howto/TESTING.md +++ b/doc/howto/TESTING.md @@ -62,7 +62,7 @@ make e2e-test # Build make build # Build all targets -make build-all-linux # Build Linux binaries for Docker +make build-all-linux # Build Linux binaries for the local docker-compose flows (tests build their own) ``` ### Testing Levels diff --git a/service/runway/server/BUILD.bazel b/service/runway/server/BUILD.bazel index 49067b0f..fec51e28 100644 --- a/service/runway/server/BUILD.bazel +++ b/service/runway/server/BUILD.bazel @@ -1,4 +1,12 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_go//go:def.bzl", "go_binary", "go_cross_binary", "go_library") + +exports_files( + [ + "Dockerfile", + "docker-compose.yml", + ], + visibility = ["//visibility:public"], +) go_library( name = "go_default_library", @@ -32,3 +40,13 @@ go_binary( embed = [":go_default_library"], visibility = ["//visibility:public"], ) + +# Linux binary consumed by the Docker image build. Docker-based tests depend on +# this target as data so Bazel builds the exact binary that ends up in the +# container image — no pre-built .docker-bin artifacts required. +go_cross_binary( + name = "runway_linux", + platform = "@rules_go//go/toolchain:linux_amd64", + target = ":runway", + visibility = ["//visibility:public"], +) diff --git a/service/stovepipe/server/BUILD.bazel b/service/stovepipe/server/BUILD.bazel index faf7b91e..ddc13418 100644 --- a/service/stovepipe/server/BUILD.bazel +++ b/service/stovepipe/server/BUILD.bazel @@ -1,4 +1,9 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_go//go:def.bzl", "go_binary", "go_cross_binary", "go_library") + +exports_files( + ["Dockerfile"], + visibility = ["//visibility:public"], +) go_library( name = "go_default_library", @@ -40,3 +45,13 @@ go_binary( embed = [":go_default_library"], visibility = ["//visibility:public"], ) + +# Linux binary consumed by the Docker image build. Docker-based tests depend on +# this target as data so Bazel builds the exact binary that ends up in the +# container image — no pre-built .docker-bin artifacts required. +go_cross_binary( + name = "stovepipe_linux", + platform = "@rules_go//go/toolchain:linux_amd64", + target = ":stovepipe", + visibility = ["//visibility:public"], +) diff --git a/service/submitqueue/gateway/server/BUILD.bazel b/service/submitqueue/gateway/server/BUILD.bazel index 39241b0a..711d1279 100644 --- a/service/submitqueue/gateway/server/BUILD.bazel +++ b/service/submitqueue/gateway/server/BUILD.bazel @@ -1,7 +1,11 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library", "go_test") +load("@rules_go//go:def.bzl", "go_binary", "go_cross_binary", "go_library", "go_test") exports_files( - ["docker-compose.yml"], + [ + "Dockerfile", + "docker-compose.yml", + "queues.yaml", + ], visibility = ["//visibility:public"], ) @@ -41,6 +45,16 @@ go_binary( visibility = ["//visibility:public"], ) +# Linux binary consumed by the Docker image build. Docker-based tests depend on +# this target as data so Bazel builds the exact binary that ends up in the +# container image — no pre-built .docker-bin artifacts required. +go_cross_binary( + name = "gateway_linux", + platform = "@rules_go//go/toolchain:linux_amd64", + target = ":gateway", + visibility = ["//visibility:public"], +) + go_test( name = "go_default_test", srcs = ["main_test.go"], diff --git a/service/submitqueue/orchestrator/server/BUILD.bazel b/service/submitqueue/orchestrator/server/BUILD.bazel index 72005d10..0b64c2e7 100644 --- a/service/submitqueue/orchestrator/server/BUILD.bazel +++ b/service/submitqueue/orchestrator/server/BUILD.bazel @@ -1,7 +1,10 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_go//go:def.bzl", "go_binary", "go_cross_binary", "go_library") exports_files( - ["docker-compose.yml"], + [ + "Dockerfile", + "docker-compose.yml", + ], visibility = ["//visibility:public"], ) @@ -71,3 +74,13 @@ go_binary( embed = [":orchestrator_lib"], visibility = ["//visibility:public"], ) + +# Linux binary consumed by the Docker image build. Docker-based tests depend on +# this target as data so Bazel builds the exact binary that ends up in the +# container image — no pre-built .docker-bin artifacts required. +go_cross_binary( + name = "orchestrator_linux", + platform = "@rules_go//go/toolchain:linux_amd64", + target = ":orchestrator", + visibility = ["//visibility:public"], +) diff --git a/test/e2e/stovepipe/BUILD.bazel b/test/e2e/stovepipe/BUILD.bazel index b68903aa..c599fdfd 100644 --- a/test/e2e/stovepipe/BUILD.bazel +++ b/test/e2e/stovepipe/BUILD.bazel @@ -7,16 +7,16 @@ go_test( "suite_test.go", ], data = [ - "//:MODULE.bazel", - "//:go.mod", "//platform/extension/messagequeue/mysql/schema", "//service/stovepipe:docker-compose.yml", + "//service/stovepipe/server:Dockerfile", + "//service/stovepipe/server:stovepipe_linux", "//stovepipe/extension/storage/mysql/schema", ], tags = [ "e2e", - "external", "integration", + "requires-network", ], deps = [ "//api/stovepipe/protopb:go_default_library", diff --git a/test/e2e/stovepipe/suite_test.go b/test/e2e/stovepipe/suite_test.go index 04c75d38..383a1432 100644 --- a/test/e2e/stovepipe/suite_test.go +++ b/test/e2e/stovepipe/suite_test.go @@ -16,13 +16,16 @@ package e2e_test // Stovepipe end-to-end tests. // -// These tests use docker-compose from service/stovepipe/docker-compose.yml, -// which requires a pre-built Linux binary. Run with the make target (builds -// binaries + runs the test): +// These tests use docker-compose from service/stovepipe/docker-compose.yml. +// They are hermetic: the stovepipe image is built from a staged context whose +// inputs (Bazel-built Linux binary, Dockerfile) are all declared data +// dependencies of the test target. +// +// Run with: // // make e2e-test // -// or only this package (after building the binary): +// or only this package: // // bazel test //test/e2e/stovepipe:stovepipe_test // @@ -35,7 +38,6 @@ package e2e_test import ( "context" "database/sql" - "path/filepath" "testing" "time" @@ -80,12 +82,15 @@ func (s *StovepipeE2ESuite) SetupSuite() { s.log.Logf("Starting Stovepipe e2e test suite using docker-compose") - // Set REPO_ROOT for the docker-compose build context. - repoRoot := testutil.FindRepoRoot(t) - t.Setenv("REPO_ROOT", repoRoot) - - composeFile := filepath.Join(repoRoot, "service/stovepipe/docker-compose.yml") - s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "e2e-stovepipe") + // Compose file and image build inputs come from the test runfiles; the + // stovepipe image is built from a staged build context assembled entirely + // from declared data dependencies. + composeFile := testutil.Runfile("service/stovepipe/docker-compose.yml") + s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "e2e-stovepipe", + testutil.WithBuildContext(map[string]string{ + ".docker-bin/stovepipe": "service/stovepipe/server/stovepipe_linux", + "service/stovepipe/server/Dockerfile": "service/stovepipe/server/Dockerfile", + })) err := s.stack.Up() require.NoError(t, err, "failed to start compose stack") diff --git a/test/e2e/submitqueue/BUILD.bazel b/test/e2e/submitqueue/BUILD.bazel index eda42174..da0f7b0f 100644 --- a/test/e2e/submitqueue/BUILD.bazel +++ b/test/e2e/submitqueue/BUILD.bazel @@ -7,17 +7,22 @@ go_test( "suite_test.go", ], data = [ - "//:MODULE.bazel", - "//:go.mod", "//platform/extension/counter/mysql/schema", "//platform/extension/messagequeue/mysql/schema", + "//service/runway/server:Dockerfile", + "//service/runway/server:runway_linux", "//service/submitqueue:docker-compose.yml", + "//service/submitqueue/gateway/server:Dockerfile", + "//service/submitqueue/gateway/server:gateway_linux", + "//service/submitqueue/gateway/server:queues.yaml", + "//service/submitqueue/orchestrator/server:Dockerfile", + "//service/submitqueue/orchestrator/server:orchestrator_linux", "//submitqueue/extension/storage/mysql/schema", ], tags = [ "e2e", - "external", "integration", + "requires-network", ], deps = [ "//api/base/change/protopb:go_default_library", diff --git a/test/e2e/submitqueue/suite_test.go b/test/e2e/submitqueue/suite_test.go index 84052b0e..d4085da3 100644 --- a/test/e2e/submitqueue/suite_test.go +++ b/test/e2e/submitqueue/suite_test.go @@ -16,16 +16,17 @@ package e2e_test // E2E Integration Tests // -// These tests use docker-compose from service/submitqueue/docker-compose.yml -// which requires pre-built Linux binaries. +// These tests use docker-compose from service/submitqueue/docker-compose.yml. +// They are hermetic: the service images are built from a staged context whose +// inputs (Bazel-built Linux binaries, Dockerfiles, queues.yaml) are all +// declared data dependencies of the test target. // -// Run with make target (builds binaries + runs test): +// Run with: // make e2e-test import ( "context" "database/sql" - "path/filepath" "testing" "time" @@ -76,14 +77,20 @@ func (s *E2EIntegrationSuite) SetupSuite() { s.log.Logf("Starting E2E integration test suite using docker-compose") - // Set REPO_ROOT for docker-compose volume mounts and build context - repoRoot := testutil.FindRepoRoot(t) - t.Setenv("REPO_ROOT", repoRoot) - - // Use docker-compose from service/submitqueue (full stack) - // NOTE: Assumes Linux binaries are pre-built via make target - composeFile := filepath.Join(repoRoot, "service/submitqueue/docker-compose.yml") - s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "e2e-submitqueue") + // Use docker-compose from service/submitqueue (full stack), resolved from + // the test runfiles. All three service images are built from a staged + // build context assembled entirely from declared data dependencies. + composeFile := testutil.Runfile("service/submitqueue/docker-compose.yml") + s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "e2e-submitqueue", + testutil.WithBuildContext(map[string]string{ + ".docker-bin/gateway": "service/submitqueue/gateway/server/gateway_linux", + ".docker-bin/orchestrator": "service/submitqueue/orchestrator/server/orchestrator_linux", + ".docker-bin/runway": "service/runway/server/runway_linux", + "service/submitqueue/gateway/server/Dockerfile": "service/submitqueue/gateway/server/Dockerfile", + "service/submitqueue/gateway/server/queues.yaml": "service/submitqueue/gateway/server/queues.yaml", + "service/submitqueue/orchestrator/server/Dockerfile": "service/submitqueue/orchestrator/server/Dockerfile", + "service/runway/server/Dockerfile": "service/runway/server/Dockerfile", + })) // Start the compose stack (Gateway + Orchestrator + 2 MySQL DBs) err := s.stack.Up() diff --git a/test/integration/extension/counter/mysql/BUILD.bazel b/test/integration/extension/counter/mysql/BUILD.bazel index d5686dfd..def70a25 100644 --- a/test/integration/extension/counter/mysql/BUILD.bazel +++ b/test/integration/extension/counter/mysql/BUILD.bazel @@ -5,13 +5,11 @@ go_test( srcs = ["counter_test.go"], data = [ "docker-compose.yml", - "//:MODULE.bazel", - "//:go.mod", "//platform/extension/counter/mysql/schema", ], tags = [ - "external", "integration", + "requires-network", ], deps = [ "//platform/extension/counter/mysql:go_default_library", diff --git a/test/integration/extension/messagequeue/mysql/BUILD.bazel b/test/integration/extension/messagequeue/mysql/BUILD.bazel index a74d8997..22747841 100644 --- a/test/integration/extension/messagequeue/mysql/BUILD.bazel +++ b/test/integration/extension/messagequeue/mysql/BUILD.bazel @@ -5,13 +5,11 @@ go_test( srcs = ["queue_test.go"], data = [ "docker-compose.yml", - "//:MODULE.bazel", - "//:go.mod", "//platform/extension/messagequeue/mysql/schema", ], tags = [ - "external", "integration", + "requires-network", ], deps = [ "//platform/base/messagequeue:go_default_library", diff --git a/test/integration/stovepipe/BUILD.bazel b/test/integration/stovepipe/BUILD.bazel index 3ca8251d..457b8113 100644 --- a/test/integration/stovepipe/BUILD.bazel +++ b/test/integration/stovepipe/BUILD.bazel @@ -4,15 +4,15 @@ go_test( name = "go_default_test", srcs = ["suite_test.go"], data = [ - "//:MODULE.bazel", - "//:go.mod", "//platform/extension/messagequeue/mysql/schema", "//service/stovepipe:docker-compose.yml", + "//service/stovepipe/server:Dockerfile", + "//service/stovepipe/server:stovepipe_linux", "//stovepipe/extension/storage/mysql/schema", ], tags = [ - "external", "integration", + "requires-network", ], deps = [ "//api/stovepipe/protopb:go_default_library", diff --git a/test/integration/stovepipe/extension/storage/mysql/BUILD.bazel b/test/integration/stovepipe/extension/storage/mysql/BUILD.bazel index 5beb0fee..5eabac00 100644 --- a/test/integration/stovepipe/extension/storage/mysql/BUILD.bazel +++ b/test/integration/stovepipe/extension/storage/mysql/BUILD.bazel @@ -5,13 +5,11 @@ go_test( srcs = ["storage_test.go"], data = [ "docker-compose.yml", - "//:MODULE.bazel", - "//:go.mod", "//stovepipe/extension/storage/mysql/schema", ], tags = [ - "external", "integration", + "requires-network", ], deps = [ "//stovepipe/entity:go_default_library", diff --git a/test/integration/stovepipe/suite_test.go b/test/integration/stovepipe/suite_test.go index 3f7495eb..5788bfc4 100644 --- a/test/integration/stovepipe/suite_test.go +++ b/test/integration/stovepipe/suite_test.go @@ -16,11 +16,11 @@ package stovepipe // Stovepipe integration tests // -// These tests use compose from service/stovepipe/docker-compose.yml and require -// a pre-built Linux binary (make integration-test runs //test/integration/... -// and builds all Linux binaries via build-all-linux). The stack runs the -// Stovepipe gRPC service plus a storage MySQL (request, request_uri) and a queue -// MySQL (process stage). +// These tests use compose from service/stovepipe/docker-compose.yml. They are +// hermetic: the stovepipe image is built from a staged context whose inputs +// (Bazel-built Linux binary, Dockerfile) are all declared data dependencies of +// the test target. The stack runs the Stovepipe gRPC service plus a storage +// MySQL (request, request_uri) and a queue MySQL (process stage). // // Run with: // make integration-test @@ -30,7 +30,6 @@ package stovepipe import ( "context" "database/sql" - "path/filepath" "testing" _ "github.com/go-sql-driver/mysql" @@ -63,11 +62,15 @@ func (s *StovepipeIntegrationSuite) SetupSuite() { s.log.Logf("Starting Stovepipe integration test suite using compose") - repoRoot := testutil.FindRepoRoot(t) - t.Setenv("REPO_ROOT", repoRoot) - - composeFile := filepath.Join(repoRoot, "service/stovepipe/docker-compose.yml") - s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "svc-stovepipe") + // Compose file and image build inputs come from the test runfiles; the + // stovepipe image is built from a staged build context assembled entirely + // from declared data dependencies. + composeFile := testutil.Runfile("service/stovepipe/docker-compose.yml") + s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "svc-stovepipe", + testutil.WithBuildContext(map[string]string{ + ".docker-bin/stovepipe": "service/stovepipe/server/stovepipe_linux", + "service/stovepipe/server/Dockerfile": "service/stovepipe/server/Dockerfile", + })) err := s.stack.Up() require.NoError(t, err, "failed to start compose stack") diff --git a/test/integration/submitqueue/core/consumer/BUILD.bazel b/test/integration/submitqueue/core/consumer/BUILD.bazel index ce6b2203..3d9c3a48 100644 --- a/test/integration/submitqueue/core/consumer/BUILD.bazel +++ b/test/integration/submitqueue/core/consumer/BUILD.bazel @@ -5,13 +5,11 @@ go_test( srcs = ["consumer_test.go"], data = [ "docker-compose.yml", - "//:MODULE.bazel", - "//:go.mod", "//platform/extension/messagequeue/mysql/schema", ], tags = [ - "external", "integration", + "requires-network", ], deps = [ "//platform/base/messagequeue:go_default_library", diff --git a/test/integration/submitqueue/extension/storage/mysql/BUILD.bazel b/test/integration/submitqueue/extension/storage/mysql/BUILD.bazel index d96ad07f..e14e602c 100644 --- a/test/integration/submitqueue/extension/storage/mysql/BUILD.bazel +++ b/test/integration/submitqueue/extension/storage/mysql/BUILD.bazel @@ -5,13 +5,11 @@ go_test( srcs = ["storage_test.go"], data = [ "docker-compose.yml", - "//:MODULE.bazel", - "//:go.mod", "//submitqueue/extension/storage/mysql/schema", ], tags = [ - "external", "integration", + "requires-network", ], deps = [ "//submitqueue/extension/storage/mysql:go_default_library", diff --git a/test/integration/submitqueue/gateway/BUILD.bazel b/test/integration/submitqueue/gateway/BUILD.bazel index 34fa0ad0..5fa2fb34 100644 --- a/test/integration/submitqueue/gateway/BUILD.bazel +++ b/test/integration/submitqueue/gateway/BUILD.bazel @@ -4,16 +4,17 @@ go_test( name = "go_default_test", srcs = ["suite_test.go"], data = [ - "//:MODULE.bazel", - "//:go.mod", "//platform/extension/counter/mysql/schema", "//platform/extension/messagequeue/mysql/schema", + "//service/submitqueue/gateway/server:Dockerfile", "//service/submitqueue/gateway/server:docker-compose.yml", + "//service/submitqueue/gateway/server:gateway_linux", + "//service/submitqueue/gateway/server:queues.yaml", "//submitqueue/extension/storage/mysql/schema", ], tags = [ - "external", "integration", + "requires-network", ], deps = [ "//api/base/change/protopb:go_default_library", diff --git a/test/integration/submitqueue/gateway/suite_test.go b/test/integration/submitqueue/gateway/suite_test.go index c3882c5d..ba3059fc 100644 --- a/test/integration/submitqueue/gateway/suite_test.go +++ b/test/integration/submitqueue/gateway/suite_test.go @@ -16,20 +16,18 @@ package gateway // Gateway Integration Tests // -// These tests use docker-compose from service/submitqueue/gateway/server/docker-compose.yml -// which requires pre-built Linux binaries. +// These tests use docker-compose from service/submitqueue/gateway/server/docker-compose.yml. +// They are hermetic: the gateway image is built from a staged context whose +// inputs (Bazel-built Linux binary, Dockerfile, queues.yaml) are all declared +// data dependencies of the test target. // -// Run with make target (builds binary + runs test): -// make integration-test-gateway -// -// For manual testing with docker-compose: -// make docker-gateway +// Run with: +// make integration-test-submitqueue-gateway import ( "context" "database/sql" "fmt" - "path/filepath" "testing" "time" @@ -85,14 +83,16 @@ func (s *GatewayIntegrationSuite) SetupSuite() { s.log.Logf("Starting Gateway integration test suite using docker-compose") - // Set REPO_ROOT for docker-compose volume mounts and build context - repoRoot := testutil.FindRepoRoot(t) - t.Setenv("REPO_ROOT", repoRoot) - - // Use docker-compose from service/submitqueue/gateway/server - // NOTE: Assumes Linux binary is pre-built via make target - composeFile := filepath.Join(repoRoot, "service/submitqueue/gateway/server/docker-compose.yml") - s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "svc-submitqueue-gateway") + // Use docker-compose from service/submitqueue/gateway/server, resolved + // from the test runfiles. The gateway image is built from a staged build + // context assembled entirely from declared data dependencies. + composeFile := testutil.Runfile("service/submitqueue/gateway/server/docker-compose.yml") + s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "svc-submitqueue-gateway", + testutil.WithBuildContext(map[string]string{ + ".docker-bin/gateway": "service/submitqueue/gateway/server/gateway_linux", + "service/submitqueue/gateway/server/Dockerfile": "service/submitqueue/gateway/server/Dockerfile", + "service/submitqueue/gateway/server/queues.yaml": "service/submitqueue/gateway/server/queues.yaml", + })) // Start the compose stack (Gateway + 2 MySQL DBs) err := s.stack.Up() diff --git a/test/integration/submitqueue/orchestrator/BUILD.bazel b/test/integration/submitqueue/orchestrator/BUILD.bazel index fa2d2743..1df538b8 100644 --- a/test/integration/submitqueue/orchestrator/BUILD.bazel +++ b/test/integration/submitqueue/orchestrator/BUILD.bazel @@ -4,16 +4,16 @@ go_test( name = "go_default_test", srcs = ["suite_test.go"], data = [ - "//:MODULE.bazel", - "//:go.mod", "//platform/extension/counter/mysql/schema", "//platform/extension/messagequeue/mysql/schema", + "//service/submitqueue/orchestrator/server:Dockerfile", "//service/submitqueue/orchestrator/server:docker-compose.yml", + "//service/submitqueue/orchestrator/server:orchestrator_linux", "//submitqueue/extension/storage/mysql/schema", ], tags = [ - "external", "integration", + "requires-network", ], deps = [ "//api/submitqueue/orchestrator/protopb:go_default_library", diff --git a/test/integration/submitqueue/orchestrator/suite_test.go b/test/integration/submitqueue/orchestrator/suite_test.go index bcdba170..24fe9148 100644 --- a/test/integration/submitqueue/orchestrator/suite_test.go +++ b/test/integration/submitqueue/orchestrator/suite_test.go @@ -16,19 +16,17 @@ package orchestrator // Orchestrator Integration Tests // -// These tests use docker-compose from service/submitqueue/orchestrator/server/docker-compose.yml -// which requires pre-built Linux binaries. +// These tests use docker-compose from service/submitqueue/orchestrator/server/docker-compose.yml. +// They are hermetic: the orchestrator image is built from a staged context +// whose inputs (Bazel-built Linux binary, Dockerfile) are all declared data +// dependencies of the test target. // -// Run with make target (builds binary + runs test): -// make integration-test-orchestrator -// -// For manual testing with docker-compose: -// make docker-orchestrator +// Run with: +// make integration-test-submitqueue-orchestrator import ( "context" "database/sql" - "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -60,14 +58,15 @@ func (s *OrchestratorIntegrationSuite) SetupSuite() { s.log.Logf("Starting Orchestrator integration test suite using docker-compose") - // Set REPO_ROOT for docker-compose volume mounts and build context - repoRoot := testutil.FindRepoRoot(t) - t.Setenv("REPO_ROOT", repoRoot) - - // Use docker-compose from service/submitqueue/orchestrator/server - // NOTE: Assumes Linux binary is pre-built via make target - composeFile := filepath.Join(repoRoot, "service/submitqueue/orchestrator/server/docker-compose.yml") - s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "svc-submitqueue-orchestrator") + // Use docker-compose from service/submitqueue/orchestrator/server, + // resolved from the test runfiles. The orchestrator image is built from a + // staged build context assembled entirely from declared data dependencies. + composeFile := testutil.Runfile("service/submitqueue/orchestrator/server/docker-compose.yml") + s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "svc-submitqueue-orchestrator", + testutil.WithBuildContext(map[string]string{ + ".docker-bin/orchestrator": "service/submitqueue/orchestrator/server/orchestrator_linux", + "service/submitqueue/orchestrator/server/Dockerfile": "service/submitqueue/orchestrator/server/Dockerfile", + })) // Start the compose stack (Orchestrator + 2 MySQL DBs) err := s.stack.Up() diff --git a/test/testutil/compose.go b/test/testutil/compose.go index ab1ab5b1..52754a8a 100644 --- a/test/testutil/compose.go +++ b/test/testutil/compose.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "os/exec" + "os/user" "path/filepath" "strings" "testing" @@ -44,24 +45,53 @@ type ComposeStack struct { logCmd *exec.Cmd // background "docker compose logs -f" process } -// getDockerComposeCommand returns the docker-compose command to use. -// Tries "docker-compose" first (V1), falls back to "docker compose" (V2). +// getDockerComposeCommand returns the docker compose command to use. +// Prefers the Compose V2 plugin ("docker compose") and falls back to a +// standalone docker-compose binary only when the plugin is unavailable: a +// standalone binary is often the legacy Python V1, which lacks flags the +// stack relies on (e.g. `up --wait`). func getDockerComposeCommand() []string { - // Try docker-compose (V1) - if _, err := exec.LookPath("docker-compose"); err == nil { - return []string{"docker-compose"} + if exec.Command("docker", "compose", "version").Run() == nil { + return []string{"docker", "compose"} } - // Fall back to docker compose (V2) - return []string{"docker", "compose"} + return []string{"docker-compose"} +} + +// ComposeOption customizes how a ComposeStack is created. +type ComposeOption func(*composeOptions) + +type composeOptions struct { + // buildContextFiles maps a path inside the synthesized docker build + // context to the workspace-relative runfile it is staged from. + buildContextFiles map[string]string +} + +// WithBuildContext stages the given files into a synthesized docker build +// context directory and points the compose build context variable (REPO_ROOT) +// at it. Keys are context-relative destination paths that must match the +// Dockerfile COPY sources and the compose `dockerfile:` path (e.g. +// ".docker-bin/gateway", "service/submitqueue/gateway/server/Dockerfile"); +// values are workspace-relative runfile paths, each of which must be declared +// as a `data` dependency of the test target. This keeps image builds hermetic: +// docker only ever sees files Bazel knows about, never the source checkout. +func WithBuildContext(files map[string]string) ComposeOption { + return func(o *composeOptions) { + o.buildContextFiles = files + } } // NewComposeStack creates a new compose stack from the given docker-compose file. // Automatically registers cleanup to tear down the stack. // testContext should describe what's being tested (e.g., "gateway", "storage", "e2e-submitqueue"). -func NewComposeStack(t *testing.T, log *TestLogger, ctx context.Context, composeFile, testContext string) *ComposeStack { +func NewComposeStack(t *testing.T, log *TestLogger, ctx context.Context, composeFile, testContext string, opts ...ComposeOption) *ComposeStack { t.Helper() + var options composeOptions + for _, opt := range opts { + opt(&options) + } + // Setup Docker environment setupDockerEnv(t) @@ -69,6 +99,11 @@ func NewComposeStack(t *testing.T, log *TestLogger, ctx context.Context, compose absPath, err := filepath.Abs(composeFile) require.NoError(t, err, "failed to get absolute path to compose file") + buildContextDir := "" + if len(options.buildContextFiles) > 0 { + buildContextDir = stageBuildContext(t, options.buildContextFiles) + } + // Generate meaningful project name: sq-test-{context}-{short-timestamp} // Results in container names like: sq-test-gateway-a1b2c3d-mysql-app-1 timestamp := fmt.Sprintf("%x", time.Now().UnixNano()&0xFFFFFF) // Last 6 hex digits @@ -81,7 +116,7 @@ func NewComposeStack(t *testing.T, log *TestLogger, ctx context.Context, compose log: log, ctx: ctx, composeCmd: getDockerComposeCommand(), - composeEnv: composeEnvironment(t, absPath), + composeEnv: composeEnvironment(buildContextDir), } // Register cleanup @@ -351,7 +386,7 @@ func (s *ComposeStack) ServiceExitCode(serviceName string) (int, error) { return exitCode, nil } -// setupDockerEnv configures Docker environment for docker-compose. +// setupDockerEnv configures the Docker CLI environment for compose commands. func setupDockerEnv(t *testing.T) { t.Helper() @@ -359,90 +394,64 @@ func setupDockerEnv(t *testing.T) { if os.Getenv("HOME") == "" { t.Setenv("HOME", t.TempDir()) } -} -// composeEnvironment returns the environment used by every compose command. -// The image prefix is stable within a worktree so --build can reuse images -// between test runs, but differs across worktrees to avoid stale cross-checkout -// images when multiple changes are tested on the same Docker daemon. -func composeEnvironment(t *testing.T, composeFile string) []string { - t.Helper() - - repoRoot := composeRepoRoot(t, composeFile) - sum := sha256.Sum256([]byte(repoRoot)) - imagePrefix := fmt.Sprintf("sq-test-%x", sum[:6]) - - env := os.Environ() - env = append(env, "SQ_DOCKER_IMAGE_PREFIX="+imagePrefix) - env = append(env, "SQ_MYSQL_DATA_MOUNT_TYPE=tmpfs") - env = append(env, "SQ_MYSQL_INITDB_SKIP_TZINFO=1") - return env -} - -// composeRepoRoot resolves the repository containing composeFile. Bazel exposes -// repository markers as runfile symlinks, so resolving a marker identifies the -// source checkout without invoking a host-provided Git executable. -func composeRepoRoot(t *testing.T, composeFile string) string { - t.Helper() - - if repoRoot := os.Getenv("REPO_ROOT"); repoRoot != "" { - return repoRoot - } - - dir := filepath.Dir(composeFile) - for { - for _, marker := range []string{"MODULE.bazel", "go.mod"} { - markerPath := filepath.Join(dir, marker) - if realMarker, err := filepath.EvalSymlinks(markerPath); err == nil { - return filepath.Dir(realMarker) + // Bazel gives tests a scratch HOME, which hides user-level Docker CLI + // plugin installs (e.g. compose v2 in ~/.docker/cli-plugins). Point + // DOCKER_CONFIG at the invoking user's docker config dir when not set + // explicitly — mirroring how the docker CLI resolves it outside the + // sandbox. Machine-level plugin installs are unaffected either way. + if os.Getenv("DOCKER_CONFIG") == "" { + if u, err := user.Current(); err == nil && u.HomeDir != "" { + dockerConfig := filepath.Join(u.HomeDir, ".docker") + if _, statErr := os.Stat(dockerConfig); statErr == nil { + t.Setenv("DOCKER_CONFIG", dockerConfig) } } - - parent := filepath.Dir(dir) - if parent == dir { - t.Fatalf("repository root not found") - } - dir = parent } } -// FindRepoRoot finds the repository root. -// Checks REPO_ROOT env var, then git, then walks up to find marker files. -func FindRepoRoot(t *testing.T) string { +// stageBuildContext copies the given runfiles into a temporary directory that +// serves as the docker build context. Runfiles are symlinks into Bazel's +// output tree, and docker's context upload does not follow symlinks, so the +// content is copied. Files are staged with the execute bit set because the +// context contains service binaries; docker preserves the mode on COPY. +func stageBuildContext(t *testing.T, files map[string]string) string { t.Helper() - // Check if REPO_ROOT is set (from .envrc or test environment) - if repoRoot := os.Getenv("REPO_ROOT"); repoRoot != "" { - return repoRoot - } + dir := t.TempDir() + for dest, src := range files { + content, err := os.ReadFile(Runfile(src)) + require.NoError(t, err, "failed to read build context input %s (is it declared as a data dependency?)", src) - // Try git (works outside Bazel sandbox) - cmd := exec.Command("git", "rev-parse", "--show-toplevel") - if output, err := cmd.Output(); err == nil { - if repoRoot := strings.TrimSpace(string(output)); repoRoot != "" { - return repoRoot - } + destPath := filepath.Join(dir, dest) + require.NoError(t, os.MkdirAll(filepath.Dir(destPath), 0o755), "failed to create build context dir for %s", dest) + require.NoError(t, os.WriteFile(destPath, content, 0o755), "failed to stage build context file %s", dest) } + return dir +} - // Walk up from current directory to find marker files - // In Bazel sandbox, marker files are symlinks - resolve them to get source location - dir, err := os.Getwd() - require.NoError(t, err, "failed to get working directory") - - for { - // Try to find and resolve marker file symlinks - for _, marker := range []string{"MODULE.bazel", "go.mod"} { - markerPath := filepath.Join(dir, marker) - if realMarker, err := filepath.EvalSymlinks(markerPath); err == nil { - return filepath.Dir(realMarker) - } - } +// composeEnvironment returns the environment used by every compose command. +// The image prefix is stable per test target so --build can reuse the docker +// build cache between runs of the same test. Concurrent runs of the same +// target against one daemon race on the tag, but every run rebuilds via +// `up --build` from its own staged context, so a stale image is never used +// for a completed run. +func composeEnvironment(buildContextDir string) []string { + seed := os.Getenv("TEST_TARGET") + if seed == "" { + seed, _ = os.Getwd() + } + sum := sha256.Sum256([]byte(seed)) + imagePrefix := fmt.Sprintf("sq-test-%x", sum[:6]) - // Move up one directory - parent := filepath.Dir(dir) - if parent == dir { - t.Fatalf("repository root not found") - } - dir = parent + env := os.Environ() + env = append(env, "SQ_DOCKER_IMAGE_PREFIX="+imagePrefix) + env = append(env, "SQ_MYSQL_DATA_MOUNT_TYPE=tmpfs") + env = append(env, "SQ_MYSQL_INITDB_SKIP_TZINFO=1") + if buildContextDir != "" { + // The compose files resolve their build context from REPO_ROOT; in + // tests it points at the staged minimal context, not the checkout. + env = append(env, "REPO_ROOT="+buildContextDir) } + return env } diff --git a/test/testutil/schema.go b/test/testutil/schema.go index e4271802..c736c88a 100644 --- a/test/testutil/schema.go +++ b/test/testutil/schema.go @@ -26,18 +26,26 @@ import ( "github.com/stretchr/testify/require" ) -// SchemaDir returns the path to a schema directory. -// It checks for both Bazel runfiles and direct go test paths. -// relativePath should be like "submitqueue/extension/storage/mysql/schema" or "platform/extension/messagequeue/mysql/schema" -func SchemaDir(relativePath string) string { - // Bazel runfiles path +// Runfile resolves a workspace-relative path (e.g. +// "service/submitqueue/docker-compose.yml") to an absolute path inside the +// Bazel runfiles tree. Every resolved path must be declared as a `data` +// dependency of the test target — that is what keeps Docker-based tests +// hermetic. Outside Bazel (plain `go test` from the repo root) the path is +// returned unchanged. +func Runfile(relativePath string) string { if dir := os.Getenv("TEST_SRCDIR"); dir != "" { return filepath.Join(dir, os.Getenv("TEST_WORKSPACE"), relativePath) } - // Direct go test path (run from repo root) return relativePath } +// SchemaDir returns the path to a schema directory. +// It checks for both Bazel runfiles and direct go test paths. +// relativePath should be like "submitqueue/extension/storage/mysql/schema" or "platform/extension/messagequeue/mysql/schema" +func SchemaDir(relativePath string) string { + return Runfile(relativePath) +} + // ApplySchema reads all .sql files from the schema directory and executes them on the database. func ApplySchema(t *testing.T, log *TestLogger, db *sql.DB, schemaDirectory string) { t.Helper()