test(integration): fix flaky tests with root-cause changes

tests/integration/internal/tests/api/metrics/team_metrics_test.go

@@ -22,44 +22,38 @@ func TestTeamMetrics(t *testing.T) {
 	utils.SetupSandboxWithCleanup(t, c)
 	var metrics []api.TeamMetric
 
-	maxDuration := 15 * time.Second
+	maxDuration := 60 * time.Second
 	tick := 500 * time.Millisecond
 
 	require.Eventually(t, func() bool {
 		response, err := c.GetTeamsTeamIDMetricsWithResponse(t.Context(), setup.TeamID, nil, setup.WithAPIKey())
-		require.NoError(t, err)
-		require.Equal(t, http.StatusOK, response.StatusCode())
-
-		require.NotNil(t, response.JSON200)
-		if len(*response.JSON200) == 0 {
+		if err != nil || response == nil || response.StatusCode() != http.StatusOK || response.JSON200 == nil {
 			return false
 		}
 
 		metrics = *response.JSON200
+		startRateGreaterThanZero := false
+		concurrentSandboxesGreaterThanZero := false
+		for _, metric := range metrics {
+			if metric.SandboxStartRate > 0 {
+				startRateGreaterThanZero = true
+			}
+			if metric.ConcurrentSandboxes > 0 {
+				concurrentSandboxesGreaterThanZero = true
+			}
+		}
 
-		return true
-	}, maxDuration, tick, "team metrics not available in time")
+		return startRateGreaterThanZero && concurrentSandboxesGreaterThanZero
+	}, maxDuration, tick, "team metrics did not reach expected state in time")
 
 	// Test getting team metrics
 	require.NotEmpty(t, metrics, "Expected at least one team metric in the response")
 
 	// Verify the structure of team metrics
-	startRateGreaterThanZero := false
-	concurrentSandboxesGreaterThanZero := false
-
 	for _, metric := range metrics {
 		require.NotEmpty(t, metric.Timestamp, "Timestamp should not be empty")
 		require.NotEmpty(t, metric.TimestampUnix, "Timestamp should not be empty")
-		if metric.SandboxStartRate > 0 {
-			startRateGreaterThanZero = true
-		}
-		if metric.ConcurrentSandboxes > 0 {
-			concurrentSandboxesGreaterThanZero = true
-		}
 	}
-
-	require.True(t, concurrentSandboxesGreaterThanZero, "MaxConcurrentSandboxes should be >= 0")
-	require.True(t, startRateGreaterThanZero, "StartedSandboxes should be >= 0")
 }
 
 func TestTeamMetricsWithTimeRange(t *testing.T) {

tests/integration/internal/tests/api/sandboxes/sandbox_network_update_test.go

@@ -3,7 +3,9 @@ package sandboxes
 import (
 	"context"
 	"net/http"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 
@@ -13,6 +15,32 @@ import (
 	"github.com/e2b-dev/infra/tests/integration/internal/utils"
 )
 
+func verifyConnectivityEventually(
+	t *testing.T,
+	ctx context.Context,
+	sbx *api.Sandbox,
+	envdClient *setup.EnvdClient,
+	checks []connectivityCheck,
+) {
+	t.Helper()
+	require.Eventually(t, func() bool {
+		for _, c := range checks {
+			err := utils.ExecCommand(t, ctx, sbx, envdClient, "curl", "--connect-timeout", "3", "--max-time", "5", "-Iks", c.url)
+			if c.allowed {
+				if err != nil {
+					return false
+				}
+			} else {
+				if err == nil || !strings.Contains(err.Error(), "failed with exit code") {
+					return false
+				}
+			}
+		}
+
+		return true
+	}, 30*time.Second, time.Second, "connectivity did not match expected state in time")
+}
+
 // =============================================================================
 // PUT /sandboxes/{sandboxID}/network — Dynamic network config update tests
 // =============================================================================
@@ -356,8 +384,7 @@ func TestUpdateNetworkConfig(t *testing.T) { //nolint:tparallel // subtests are
 		require.NoError(t, err)
 		require.Equal(t, http.StatusCreated, resumeResp.StatusCode())
 
-		// Verify rules survived
-		verifyConnectivity(t, ctx, sbx, envdClient, []connectivityCheck{
+		verifyConnectivityEventually(t, ctx, sbx, envdClient, []connectivityCheck{
 			{"https://8.8.8.8", true},
 			{"https://1.1.1.1", false},
 		})
@@ -387,8 +414,7 @@ func TestUpdateNetworkConfig(t *testing.T) { //nolint:tparallel // subtests are
 		require.NoError(t, err)
 		require.Equal(t, http.StatusCreated, resumeResp.StatusCode())
 
-		// Verify block survived pause/resume
-		verifyConnectivity(t, ctx, sbx, envdClient, []connectivityCheck{
+		verifyConnectivityEventually(t, ctx, sbx, envdClient, []connectivityCheck{
 			{"https://8.8.8.8", false},
 			{"https://1.1.1.1", false},
 		})

tests/integration/internal/tests/envd/filesystem_test.go

@@ -33,7 +33,7 @@ func TestListDir(t *testing.T) {
 	t.Cleanup(cancel)
 
 	c := setup.GetAPIClient()
-	sbx := utils.SetupSandboxWithCleanup(t, c)
+	sbx := utils.SetupSandboxWithCleanup(t, c, utils.WithTimeout(120))
 	envdClient := setup.GetEnvdClient(t, ctx)
 
 	utils.CreateDir(t, sbx, testFolder)

tests/integration/internal/tests/envd/localhost_bind_test.go

@@ -72,19 +72,15 @@ func TestBindLocalhost(t *testing.T) {
 				}
 			}()
 
-			// Give the server time to start
-			time.Sleep(5 * time.Second)
-
 			baseURL, err := url.Parse(setup.EnvdProxy)
 			require.NoError(t, err)
 
 			httpClient := &http.Client{
 				Timeout: 10 * time.Second,
 			}
 
-			req := utils.NewRequest(sbx, baseURL, port, nil)
-			resp, err := httpClient.Do(req)
-			require.NoErrorf(t, err, "Failed to connect to server bound to %s", tc.bindAddress)
+			resp := utils.WaitForStatus(t, httpClient, sbx, baseURL, port, nil, tc.expectStatus)
+			require.NotNilf(t, resp, "Server bound to %s did not become reachable in time", tc.bindAddress)
 			defer resp.Body.Close()
 
 			assert.Equalf(t, tc.expectStatus, resp.StatusCode, "Unexpected status code %d for bind address %s", resp.StatusCode, tc.bindAddress)

tests/integration/internal/tests/orchestrator/sandbox_memory_integrity_test.go

@@ -38,8 +38,7 @@ func TestSandboxMemoryIntegrity(t *testing.T) {
 	t.Run("tmpfs hash", func(t *testing.T) {
 		t.Parallel()
 
-		// Create a sandbox with auto-pause disabled
-		sbx := utils.SetupSandboxWithCleanup(t, c, utils.WithAutoPause(false), utils.WithTemplateID(tmpl.TemplateID))
+		sbx := utils.SetupSandboxWithCleanup(t, c, utils.WithAutoPause(false), utils.WithTimeout(300), utils.WithTemplateID(tmpl.TemplateID))
 		sbxId := sbx.SandboxID
 
 		envdClient := setup.GetEnvdClient(t, t.Context())
@@ -107,7 +106,7 @@ echo "Used memory after tmpfs mount and file fill: ${USED_MEM_MB_AFTER} MB"
 	t.Run("write after read survives pause", func(t *testing.T) {
 		t.Parallel()
 
-		sbx := utils.SetupSandboxWithCleanup(t, c, utils.WithAutoPause(false))
+		sbx := utils.SetupSandboxWithCleanup(t, c, utils.WithAutoPause(false), utils.WithTimeout(300))
 		sbxId := sbx.SandboxID
 		envdClient := setup.GetEnvdClient(t, t.Context())
 
@@ -143,8 +142,7 @@ echo "Used memory after tmpfs mount and file fill: ${USED_MEM_MB_AFTER} MB"
 	t.Run("stress-ng verify", func(t *testing.T) {
 		t.Parallel()
 
-		// Create a sandbox with auto-pause disabled
-		sbx := utils.SetupSandboxWithCleanup(t, c, utils.WithAutoPause(false), utils.WithTemplateID(tmpl.TemplateID))
+		sbx := utils.SetupSandboxWithCleanup(t, c, utils.WithAutoPause(false), utils.WithTimeout(300), utils.WithTemplateID(tmpl.TemplateID))
 
 		envdClient := setup.GetEnvdClient(t, t.Context())
 

tests/integration/internal/utils/request.go

@@ -61,11 +61,13 @@ func NewRequest(sbx *api.Sandbox, url *url.URL, port int, extraHeaders *http.Hea
 func WaitForStatus(tb testing.TB, client *http.Client, sbx *api.Sandbox, url *url.URL, port int, headers *http.Header, expectedStatus int) *http.Response {
 	tb.Helper()
 
-	for range 10 {
+	deadline := time.Now().Add(60 * time.Second)
+	for time.Now().Before(deadline) {
 		req := NewRequest(sbx, url, port, headers)
 		resp, err := client.Do(req)
 		if err != nil {
 			tb.Logf("Error: %v", err)
+			time.Sleep(500 * time.Millisecond)
 
 			continue
 		}