From 4a44dda3a02bb77d81689dff2c4995440a1781da Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Mon, 18 May 2026 13:08:01 -0700 Subject: [PATCH] test(integration): fix flaky tests with real root causes - TestTeamMetrics: Eventually returned on any metrics, but assertions required start-rate and concurrent-sandboxes signals that lag behind the first non-empty response (ClickHouse aggregation). Move the assertions inside the loop and bump the window to 60s. - TestUpdateNetworkConfig pause/resume subtests: firewall rule reapply on resume is async; a one-shot curl races it. New helper verifyConnectivityEventually polls connectivity checks for 30s. - TestBindLocalhost: fixed 5s sleep was insufficient for python's http.server to start under CI load. Replaced with WaitForStatus poll (now extended to 60s total so any caller starting a service has time to come up). - TestListDir: parent sandbox used the 30s default timeout, which expired while the four parallel subtests waited to be scheduled. Bumped to 120s. - TestSandboxMemoryIntegrity subtests: dd + pause/resume iterations routinely exceed the 30s default timeout (tmpfs_hash alone takes ~44s). Bumped all three subtests to 300s. --- .../tests/api/metrics/team_metrics_test.go | 34 ++++++++----------- .../sandboxes/sandbox_network_update_test.go | 34 ++++++++++++++++--- .../internal/tests/envd/filesystem_test.go | 2 +- .../tests/envd/localhost_bind_test.go | 8 ++--- .../sandbox_memory_integrity_test.go | 8 ++--- tests/integration/internal/utils/request.go | 4 ++- 6 files changed, 53 insertions(+), 37 deletions(-) diff --git a/tests/integration/internal/tests/api/metrics/team_metrics_test.go b/tests/integration/internal/tests/api/metrics/team_metrics_test.go index e2fc105453..67990700cc 100644 --- a/tests/integration/internal/tests/api/metrics/team_metrics_test.go +++ b/tests/integration/internal/tests/api/metrics/team_metrics_test.go @@ -22,44 +22,38 @@ func TestTeamMetrics(t *testing.T) { utils.SetupSandboxWithCleanup(t, c) var metrics []api.TeamMetric - maxDuration := 15 * time.Second + maxDuration := 60 * time.Second tick := 500 * time.Millisecond require.Eventually(t, func() bool { response, err := c.GetTeamsTeamIDMetricsWithResponse(t.Context(), setup.TeamID, nil, setup.WithAPIKey()) - require.NoError(t, err) - require.Equal(t, http.StatusOK, response.StatusCode()) - - require.NotNil(t, response.JSON200) - if len(*response.JSON200) == 0 { + if err != nil || response == nil || response.StatusCode() != http.StatusOK || response.JSON200 == nil { return false } metrics = *response.JSON200 + startRateGreaterThanZero := false + concurrentSandboxesGreaterThanZero := false + for _, metric := range metrics { + if metric.SandboxStartRate > 0 { + startRateGreaterThanZero = true + } + if metric.ConcurrentSandboxes > 0 { + concurrentSandboxesGreaterThanZero = true + } + } - return true - }, maxDuration, tick, "team metrics not available in time") + return startRateGreaterThanZero && concurrentSandboxesGreaterThanZero + }, maxDuration, tick, "team metrics did not reach expected state in time") // Test getting team metrics require.NotEmpty(t, metrics, "Expected at least one team metric in the response") // Verify the structure of team metrics - startRateGreaterThanZero := false - concurrentSandboxesGreaterThanZero := false - for _, metric := range metrics { require.NotEmpty(t, metric.Timestamp, "Timestamp should not be empty") require.NotEmpty(t, metric.TimestampUnix, "Timestamp should not be empty") - if metric.SandboxStartRate > 0 { - startRateGreaterThanZero = true - } - if metric.ConcurrentSandboxes > 0 { - concurrentSandboxesGreaterThanZero = true - } } - - require.True(t, concurrentSandboxesGreaterThanZero, "MaxConcurrentSandboxes should be >= 0") - require.True(t, startRateGreaterThanZero, "StartedSandboxes should be >= 0") } func TestTeamMetricsWithTimeRange(t *testing.T) { diff --git a/tests/integration/internal/tests/api/sandboxes/sandbox_network_update_test.go b/tests/integration/internal/tests/api/sandboxes/sandbox_network_update_test.go index 558f1afbe6..7f319c2dea 100644 --- a/tests/integration/internal/tests/api/sandboxes/sandbox_network_update_test.go +++ b/tests/integration/internal/tests/api/sandboxes/sandbox_network_update_test.go @@ -3,7 +3,9 @@ package sandboxes import ( "context" "net/http" + "strings" "testing" + "time" "github.com/stretchr/testify/require" @@ -13,6 +15,32 @@ import ( "github.com/e2b-dev/infra/tests/integration/internal/utils" ) +func verifyConnectivityEventually( + t *testing.T, + ctx context.Context, + sbx *api.Sandbox, + envdClient *setup.EnvdClient, + checks []connectivityCheck, +) { + t.Helper() + require.Eventually(t, func() bool { + for _, c := range checks { + err := utils.ExecCommand(t, ctx, sbx, envdClient, "curl", "--connect-timeout", "3", "--max-time", "5", "-Iks", c.url) + if c.allowed { + if err != nil { + return false + } + } else { + if err == nil || !strings.Contains(err.Error(), "failed with exit code") { + return false + } + } + } + + return true + }, 30*time.Second, time.Second, "connectivity did not match expected state in time") +} + // ============================================================================= // PUT /sandboxes/{sandboxID}/network — Dynamic network config update tests // ============================================================================= @@ -356,8 +384,7 @@ func TestUpdateNetworkConfig(t *testing.T) { //nolint:tparallel // subtests are require.NoError(t, err) require.Equal(t, http.StatusCreated, resumeResp.StatusCode()) - // Verify rules survived - verifyConnectivity(t, ctx, sbx, envdClient, []connectivityCheck{ + verifyConnectivityEventually(t, ctx, sbx, envdClient, []connectivityCheck{ {"https://8.8.8.8", true}, {"https://1.1.1.1", false}, }) @@ -387,8 +414,7 @@ func TestUpdateNetworkConfig(t *testing.T) { //nolint:tparallel // subtests are require.NoError(t, err) require.Equal(t, http.StatusCreated, resumeResp.StatusCode()) - // Verify block survived pause/resume - verifyConnectivity(t, ctx, sbx, envdClient, []connectivityCheck{ + verifyConnectivityEventually(t, ctx, sbx, envdClient, []connectivityCheck{ {"https://8.8.8.8", false}, {"https://1.1.1.1", false}, }) diff --git a/tests/integration/internal/tests/envd/filesystem_test.go b/tests/integration/internal/tests/envd/filesystem_test.go index 5c83c40bd5..42bbeda424 100644 --- a/tests/integration/internal/tests/envd/filesystem_test.go +++ b/tests/integration/internal/tests/envd/filesystem_test.go @@ -33,7 +33,7 @@ func TestListDir(t *testing.T) { t.Cleanup(cancel) c := setup.GetAPIClient() - sbx := utils.SetupSandboxWithCleanup(t, c) + sbx := utils.SetupSandboxWithCleanup(t, c, utils.WithTimeout(120)) envdClient := setup.GetEnvdClient(t, ctx) utils.CreateDir(t, sbx, testFolder) diff --git a/tests/integration/internal/tests/envd/localhost_bind_test.go b/tests/integration/internal/tests/envd/localhost_bind_test.go index b0bbdfc9f7..a3ad08ff65 100644 --- a/tests/integration/internal/tests/envd/localhost_bind_test.go +++ b/tests/integration/internal/tests/envd/localhost_bind_test.go @@ -72,9 +72,6 @@ func TestBindLocalhost(t *testing.T) { } }() - // Give the server time to start - time.Sleep(5 * time.Second) - baseURL, err := url.Parse(setup.EnvdProxy) require.NoError(t, err) @@ -82,9 +79,8 @@ func TestBindLocalhost(t *testing.T) { Timeout: 10 * time.Second, } - req := utils.NewRequest(sbx, baseURL, port, nil) - resp, err := httpClient.Do(req) - require.NoErrorf(t, err, "Failed to connect to server bound to %s", tc.bindAddress) + resp := utils.WaitForStatus(t, httpClient, sbx, baseURL, port, nil, tc.expectStatus) + require.NotNilf(t, resp, "Server bound to %s did not become reachable in time", tc.bindAddress) defer resp.Body.Close() assert.Equalf(t, tc.expectStatus, resp.StatusCode, "Unexpected status code %d for bind address %s", resp.StatusCode, tc.bindAddress) diff --git a/tests/integration/internal/tests/orchestrator/sandbox_memory_integrity_test.go b/tests/integration/internal/tests/orchestrator/sandbox_memory_integrity_test.go index 2e90e1dc79..36005abbd8 100644 --- a/tests/integration/internal/tests/orchestrator/sandbox_memory_integrity_test.go +++ b/tests/integration/internal/tests/orchestrator/sandbox_memory_integrity_test.go @@ -38,8 +38,7 @@ func TestSandboxMemoryIntegrity(t *testing.T) { t.Run("tmpfs hash", func(t *testing.T) { t.Parallel() - // Create a sandbox with auto-pause disabled - sbx := utils.SetupSandboxWithCleanup(t, c, utils.WithAutoPause(false), utils.WithTemplateID(tmpl.TemplateID)) + sbx := utils.SetupSandboxWithCleanup(t, c, utils.WithAutoPause(false), utils.WithTimeout(300), utils.WithTemplateID(tmpl.TemplateID)) sbxId := sbx.SandboxID envdClient := setup.GetEnvdClient(t, t.Context()) @@ -107,7 +106,7 @@ echo "Used memory after tmpfs mount and file fill: ${USED_MEM_MB_AFTER} MB" t.Run("write after read survives pause", func(t *testing.T) { t.Parallel() - sbx := utils.SetupSandboxWithCleanup(t, c, utils.WithAutoPause(false)) + sbx := utils.SetupSandboxWithCleanup(t, c, utils.WithAutoPause(false), utils.WithTimeout(300)) sbxId := sbx.SandboxID envdClient := setup.GetEnvdClient(t, t.Context()) @@ -143,8 +142,7 @@ echo "Used memory after tmpfs mount and file fill: ${USED_MEM_MB_AFTER} MB" t.Run("stress-ng verify", func(t *testing.T) { t.Parallel() - // Create a sandbox with auto-pause disabled - sbx := utils.SetupSandboxWithCleanup(t, c, utils.WithAutoPause(false), utils.WithTemplateID(tmpl.TemplateID)) + sbx := utils.SetupSandboxWithCleanup(t, c, utils.WithAutoPause(false), utils.WithTimeout(300), utils.WithTemplateID(tmpl.TemplateID)) envdClient := setup.GetEnvdClient(t, t.Context()) diff --git a/tests/integration/internal/utils/request.go b/tests/integration/internal/utils/request.go index f08681b8aa..cdea744ffd 100644 --- a/tests/integration/internal/utils/request.go +++ b/tests/integration/internal/utils/request.go @@ -61,11 +61,13 @@ func NewRequest(sbx *api.Sandbox, url *url.URL, port int, extraHeaders *http.Hea func WaitForStatus(tb testing.TB, client *http.Client, sbx *api.Sandbox, url *url.URL, port int, headers *http.Header, expectedStatus int) *http.Response { tb.Helper() - for range 10 { + deadline := time.Now().Add(60 * time.Second) + for time.Now().Before(deadline) { req := NewRequest(sbx, url, port, headers) resp, err := client.Do(req) if err != nil { tb.Logf("Error: %v", err) + time.Sleep(500 * time.Millisecond) continue }