From bdfe150dbbbc07e0883c16a7b92b64ab122dc835 Mon Sep 17 00:00:00 2001 From: Chris Burns <29541485+ChrisJBurns@users.noreply.github.com> Date: Mon, 27 Jul 2026 23:40:05 +0100 Subject: [PATCH] Speed up e2e(core) suite with Ginkgo procs and concurrent waits Run the core test suite across 4 Ginkgo processes in CI instead of serially. The 8-core runner has capacity to spare, and the ~30 core specs are independent enough to distribute safely. Two issues blocked parallelism: - Four test files named servers using only GinkgoRandomSeed(), which is identical across all procs; concurrent processes would attempt to create the same Docker container name. Switch to GenerateUniqueServerName which includes the OS PID (unique per proc). - ExpectMCPServersRunning polled workloads sequentially; specs that start 4 containers (group_rm "delete with workloads") paid the full sum of readiness waits. Replace the serial loop with concurrent goroutines so the wait is bounded by the slowest workload, not the sum. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .github/workflows/e2e-tests.yml | 2 ++ test/e2e/export_test.go | 3 +-- test/e2e/helpers.go | 22 +++++++++++++++------- test/e2e/restart_test.go | 3 +-- test/e2e/run_tests.sh | 6 +++++- test/e2e/status_test.go | 3 +-- test/e2e/thvignore_test.go | 2 +- 7 files changed, 26 insertions(+), 15 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index b993a3f9ae..7d99bc17f2 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -50,6 +50,7 @@ jobs: - title: core label_filter: core artifact: e2e-test-results-core + procs: 4 - title: mcp-run label_filter: mcp-run artifact: e2e-test-results-mcp-run @@ -178,6 +179,7 @@ jobs: TOOLHIVE_EGRESS_IMAGE: ghcr.io/stacklok/toolhive/egress-proxy:latest TEST_TIMEOUT: ${{ matrix.test_timeout || '15m' }} LABEL_FILTER: ${{ matrix.label_filter }} + PROCS: ${{ matrix.procs || 1 }} run: ./test/e2e/run_tests.sh - name: Upload test results (${{ matrix.title }}) diff --git a/test/e2e/export_test.go b/test/e2e/export_test.go index ab92ed3d0b..55c9794e46 100644 --- a/test/e2e/export_test.go +++ b/test/e2e/export_test.go @@ -5,7 +5,6 @@ package e2e_test import ( "encoding/json" - "fmt" "os" "path/filepath" "time" @@ -212,5 +211,5 @@ var _ = Describe("Export Command", Label("core", "export", "e2e"), func() { // generateExportTestServerName creates a unique server name for export tests func generateExportTestServerName(prefix string) string { - return fmt.Sprintf("%s-%d", prefix, GinkgoRandomSeed()) + return e2e.GenerateUniqueServerName(prefix) } diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index fa2cd036d8..90ebb52913 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -12,6 +12,7 @@ import ( "os/exec" "path/filepath" "strings" + "sync" "syscall" "time" @@ -216,14 +217,21 @@ func WaitForMCPServer(config *TestConfig, serverName string, timeout time.Durati // ExpectMCPServersRunning waits for each named workload to reach the running // state within ServerReadyTimeout, and fails naming the workload that did not. // -// Specs that start several workloads used to wait in a loop over a bare -// Expect(err).ToNot(HaveOccurred()), so a timeout said only that some workload -// in the set was not ready. The failure is reported at the caller's line so a -// CI annotation still points at the spec rather than at this helper. +// All workloads are polled concurrently so that the total wait is bounded by +// the slowest workload rather than the sum of all waits. func ExpectMCPServersRunning(config *TestConfig, serverNames ...string) { - for _, serverName := range serverNames { - err := WaitForMCPServer(config, serverName, ServerReadyTimeout()) - ExpectWithOffset(1, err).ToNot(HaveOccurred(), + errs := make([]error, len(serverNames)) + var wg sync.WaitGroup + for i, name := range serverNames { + wg.Add(1) + go func(i int, name string) { + defer wg.Done() + errs[i] = WaitForMCPServer(config, name, ServerReadyTimeout()) + }(i, name) + } + wg.Wait() + for i, serverName := range serverNames { + ExpectWithOffset(1, errs[i]).ToNot(HaveOccurred(), "workload %s (of %v) should reach the running state", serverName, serverNames) } } diff --git a/test/e2e/restart_test.go b/test/e2e/restart_test.go index d2463b1c7c..db11126f74 100644 --- a/test/e2e/restart_test.go +++ b/test/e2e/restart_test.go @@ -4,7 +4,6 @@ package e2e_test import ( - "fmt" "strings" "time" @@ -203,5 +202,5 @@ var _ = Describe("Server Restart", Label("core", "restart", "e2e"), func() { // generateTestServerName creates a unique server name for restart tests func generateTestServerName(prefix string) string { - return fmt.Sprintf("%s-%d", prefix, GinkgoRandomSeed()) + return e2e.GenerateUniqueServerName(prefix) } diff --git a/test/e2e/run_tests.sh b/test/e2e/run_tests.sh index a6c55f5162..e948aae9da 100755 --- a/test/e2e/run_tests.sh +++ b/test/e2e/run_tests.sh @@ -45,6 +45,10 @@ fi TEST_TIMEOUT="${TEST_TIMEOUT:-20m}" echo -e "${GREEN}✓${NC} Test timeout: $TEST_TIMEOUT" +# Set number of parallel Ginkgo processes (default 1 = sequential) +PROCS="${PROCS:-1}" +echo -e "${GREEN}✓${NC} Ginkgo procs: $PROCS" + # Export environment variables for tests export THV_BINARY export TEST_TIMEOUT @@ -57,7 +61,7 @@ echo "" cd "$(dirname "$0")" # Build ginkgo command with conditional GitHub output flag -GINKGO_CMD="ginkgo run --timeout=\"$TEST_TIMEOUT\"" +GINKGO_CMD="ginkgo run --timeout=\"$TEST_TIMEOUT\" --procs=$PROCS" GINKGO_CMD="$GINKGO_CMD --junit-report=junit-report.xml --output-dir=." GINKGO_CMD="$GINKGO_CMD --silence-skips" if [ -n "$GITHUB_ACTIONS" ]; then diff --git a/test/e2e/status_test.go b/test/e2e/status_test.go index 01f58012e3..93efba2cc1 100644 --- a/test/e2e/status_test.go +++ b/test/e2e/status_test.go @@ -5,7 +5,6 @@ package e2e_test import ( "encoding/json" - "fmt" "strings" "time" @@ -24,7 +23,7 @@ var _ = Describe("Status Command", Label("core", "status", "e2e"), func() { BeforeEach(func() { config = e2e.NewTestConfig() - serverName = fmt.Sprintf("status-test-%d", GinkgoRandomSeed()) + serverName = e2e.GenerateUniqueServerName("status-test") // Check if thv binary is available err := e2e.CheckTHVBinaryAvailable(config) diff --git a/test/e2e/thvignore_test.go b/test/e2e/thvignore_test.go index 66a9cd894f..03616baca4 100644 --- a/test/e2e/thvignore_test.go +++ b/test/e2e/thvignore_test.go @@ -25,7 +25,7 @@ var _ = Describe("THVIgnore E2E Tests", Label("core", "thvignore", "e2e"), func( BeforeEach(func() { config = e2e.NewTestConfig() - serverName = fmt.Sprintf("thvignore-test-%d", GinkgoRandomSeed()) + serverName = e2e.GenerateUniqueServerName("thvignore-test") // Check if thv binary is available err := e2e.CheckTHVBinaryAvailable(config)