From 845f8ec6d898dcca323cca87e55507f0b2e300f8 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Mon, 18 May 2026 12:38:48 +0000 Subject: [PATCH 1/8] fix(api): keep API alive until in-flight requests finish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the API received SIGTERM, in-flight HTTP requests (sandbox pause, snapshot, delete) could be cut off mid-flight, leaving sandboxes in inconsistent state on the orchestrator node. Root cause: each serve goroutine had `defer cancel()` on the process root context. http.Server.Shutdown stops the listener — which makes ListenAndServe return — and the serve goroutine's deferred cancel fired during the drain. Shutdown's context was a child of the root, so it self-aborted; in-flight r.Context() (BaseContext = root) died with it. Fixes: - Serve goroutines now `defer serveErrCancel()` on a dedicated serveErrCtx (child of ctx) so a serve exit wakes the shutdown watcher without cancelling the root. - Shutdown watcher selects on signalCtx OR serveErrCtx and drains HTTP via a bounded child of the still-live root ctx. - Bound grpc.Server.GracefulStop via GracefulStopWithTimeout with a Stop() fallback; run both gRPC servers in parallel via sync.WaitGroup.Go. - Bump Nomad kill_timeout 30s -> 120s to fit the drain budget (15s LB drain + 75s HTTP + 10s gRPC + cleanup slack). --- iac/modules/job-api/jobs/api.hcl | 4 +- packages/api/main.go | 103 ++++++++++++++++------ packages/shared/pkg/grpc/shutdown.go | 36 ++++++++ packages/shared/pkg/grpc/shutdown_test.go | 54 ++++++++++++ 4 files changed, 167 insertions(+), 30 deletions(-) create mode 100644 packages/shared/pkg/grpc/shutdown.go create mode 100644 packages/shared/pkg/grpc/shutdown_test.go diff --git a/iac/modules/job-api/jobs/api.hcl b/iac/modules/job-api/jobs/api.hcl index ed518a7a32..5d33409ea2 100644 --- a/iac/modules/job-api/jobs/api.hcl +++ b/iac/modules/job-api/jobs/api.hcl @@ -139,9 +139,9 @@ job "api" { task "start" { driver = "docker" - # If we need more than 30s we will need to update the max_kill_timeout in nomad + # Budget = shutdownDrainWait (15s) + httpShutdownTimeout (requestTimeout 70s) + grpcShutdownTimeout (10s) + cleanup (30s) + slack. # https://developer.hashicorp.com/nomad/docs/configuration/client#max_kill_timeout - kill_timeout = "30s" + kill_timeout = "150s" kill_signal = "SIGTERM" resources { diff --git a/packages/api/main.go b/packages/api/main.go index 81380cb56e..097ec67d04 100644 --- a/packages/api/main.go +++ b/packages/api/main.go @@ -72,6 +72,26 @@ const ( idleTimeout = 620 * time.Second defaultPort = 80 + + // shutdownDrainWait is how long /health returns 503 before we begin + // stopping the HTTP listener, giving the load balancer time to drain + // us off active backends. + shutdownDrainWait = 15 * time.Second + + // httpShutdownTimeout caps how long s.Shutdown waits for in-flight + // requests to complete. Must be >= requestTimeout so a request, that + // arrived before load balancer stopped sending new traffic, + // has its full deadline to finish + // (e.g. a slow sandbox pause / snapshot RPC). + httpShutdownTimeout = requestTimeout + 5*time.Second + + // grpcShutdownTimeout caps grpc.Server.GracefulStop. After this we + // fall back to Stop() so a stuck stream cannot block the process + // past Nomad's kill_timeout. + grpcShutdownTimeout = 10 * time.Second + + // pprofShutdownTimeout is a best-effort bound for pprof drain. + pprofShutdownTimeout = 5 * time.Second ) var ( @@ -228,6 +248,12 @@ func NewGinServer(ctx context.Context, config cfg.Config, tel *telemetry.Client, // Configure timeouts to be greater than the proxy timeouts. IdleTimeout: idleTimeout, + // BaseContext is the parent of every incoming request's r.Context(). + // It MUST NOT be derived from a context that the serve goroutines + // (HTTP/gRPC) cancel on exit: s.Shutdown stops the listener, the + // serve goroutine returns, and its `defer serveErrCancel()` fires + // while in-flight requests (sandbox pause/snapshot/delete) are + // still running. Per-request deadlines come from middleware. BaseContext: func(net.Listener) context.Context { return ctx }, } httpserver.ConfigureH2C(s) @@ -432,22 +458,25 @@ func run() int { } proxygrpc.RegisterSandboxServiceServer(edgeGrpcServer, handlers.NewSandboxService(apiStore, true, clientProxyOAuthVerifier)) - // pass the signal context so that handlers know when shutdown is happening. + // Pass ctx so in-flight requests survive the serve goroutines' exit during graceful shutdown. s := NewGinServer(ctx, config, tel, l, apiStore, redisClient, featureFlags, swagger, port) // //////////////////////// // // Start the HTTP service - // set up the signal handlers so that we can trigger a - // shutdown of the HTTP service when the process catches the - // specified signal. The parent context isn't canceled until - // after the HTTP service returns, to avoid terminating - // connections to databases and other upstream services before - // the HTTP server has shut down. + // signalCtx is cancelled when the process receives SIGTERM/SIGINT. + // It is the trigger for the shutdown watcher below. signalCtx, sigCancel := signal.NotifyContext(ctx, syscall.SIGTERM, syscall.SIGINT) defer sigCancel() + // serveErrCtx is cancelled by any serve goroutine when it exits — both + // on fatal Serve() error and on the normal "listener closed by Shutdown" + // exit. The shutdown watcher selects on this so that a startup-time + // listener failure (e.g. port in use) also triggers the drain sequence. + serveErrCtx, serveErrCancel := context.WithCancel(ctx) + defer serveErrCancel() + wg := &sync.WaitGroup{} // in the event of an unhandled panic *still* wait for the @@ -455,11 +484,10 @@ func run() int { defer wg.Wait() wg.Go(func() { - // make sure to cancel the parent context before this - // goroutine returns, so that in the case of a panic - // or error here, the other thread won't block until - // signaled. - defer cancel() + // Signal sibling goroutines via serveErrCtx (NOT the root ctx) so + // that a startup error or a normal Shutdown-triggered exit wakes + // the shutdown watcher without aborting the drain. + defer serveErrCancel() l.Info(ctx, "Http service starting", zap.Int("port", port)) @@ -479,7 +507,7 @@ func run() int { }) wg.Go(func() { - defer cancel() + defer serveErrCancel() l.Info(ctx, "internal gRPC service starting", zap.Uint16("port", config.APIInternalGrpcPort)) err := grpcServer.Serve(grpcListener) @@ -490,7 +518,7 @@ func run() int { }) wg.Go(func() { - defer cancel() + defer serveErrCancel() l.Info(ctx, "edge gRPC service starting", zap.Uint16("port", config.APIEdgeGrpcPort)) err := edgeGrpcServer.Serve(edgeGrpcListener) @@ -511,7 +539,14 @@ func run() int { }) wg.Go(func() { - <-signalCtx.Done() + // Wake on signal OR on a serve goroutine exiting unexpectedly at + // startup. Either way, run the full drain sequence. + select { + case <-signalCtx.Done(): + l.Info(ctx, "shutdown signal received, beginning graceful shutdown") + case <-serveErrCtx.Done(): + l.Info(ctx, "serve goroutine exited, beginning graceful shutdown") + } // Start returning 503s for health checks // to signal that the service is shutting down. @@ -521,27 +556,39 @@ func run() int { // Skip the delay in local environment for instant shutdown if !env.IsLocal() { - time.Sleep(15 * time.Second) + time.Sleep(shutdownDrainWait) } - // if the parent context `ctx` is canceled the - // shutdown will return early. This should only happen - // if there's an error in starting the http service - // (and would be a noop), or if there's an unhandled - // panic and defers start running, _probably_ won't - // even have a chance to return before the program - // returns. - if err := s.Shutdown(ctx); err != nil { + // Drain in-flight HTTP requests + httpShutdownCtx, httpShutdownCancel := context.WithTimeout(ctx, httpShutdownTimeout) + defer httpShutdownCancel() + if err := s.Shutdown(httpShutdownCtx); err != nil { exitCode.Add(1) l.Error(ctx, "Http service shutdown error", zap.Int("port", port), zap.Error(err)) } - if err := pprofServer.Shutdown(ctx); err != nil { + // Bounded gRPC stop: GracefulStop has no built-in deadline, so a + // stuck stream would block past Nomad's kill_timeout. Fall back to + // Stop() after the budget elapses. Run both stops in parallel — + // they are independent and serial draining would double the budget. + grpcWG := &sync.WaitGroup{} + grpcWG.Go(func() { + if !e2bgrpc.GracefulStopWithTimeout(grpcServer, grpcShutdownTimeout) { + l.Warn(ctx, "internal gRPC forced stop after graceful timeout", zap.Duration("budget", grpcShutdownTimeout)) + } + }) + grpcWG.Go(func() { + if !e2bgrpc.GracefulStopWithTimeout(edgeGrpcServer, grpcShutdownTimeout) { + l.Warn(ctx, "edge gRPC forced stop after graceful timeout", zap.Duration("budget", grpcShutdownTimeout)) + } + }) + grpcWG.Wait() + + pprofShutdownCtx, pprofCancel := context.WithTimeout(ctx, pprofShutdownTimeout) + if err := pprofServer.Shutdown(pprofShutdownCtx); err != nil { l.Error(ctx, "pprof server shutdown error", zap.Error(err)) } - - grpcServer.GracefulStop() - edgeGrpcServer.GracefulStop() + pprofCancel() }) // wait for the HTTP service to complete shutting down first diff --git a/packages/shared/pkg/grpc/shutdown.go b/packages/shared/pkg/grpc/shutdown.go new file mode 100644 index 0000000000..a71209d6d2 --- /dev/null +++ b/packages/shared/pkg/grpc/shutdown.go @@ -0,0 +1,36 @@ +package grpc + +import ( + "time" + + "google.golang.org/grpc" +) + +// GracefulStopWithTimeout invokes srv.GracefulStop and falls back to Stop() if +// it does not return within d. Returns true if graceful stop completed before +// the deadline. +// +// grpc.Server.GracefulStop blocks until all pending RPCs finish, with no +// built-in deadline. A stuck stream would otherwise block process shutdown +// past Nomad's kill_timeout and result in SIGKILL. +// +// After Stop() is called, we still wait for the inner goroutine to return so +// the caller can rely on all server resources being released. +func GracefulStopWithTimeout(srv *grpc.Server, d time.Duration) bool { + done := make(chan struct{}) + + go func() { + srv.GracefulStop() + close(done) + }() + + select { + case <-done: + return true + case <-time.After(d): + srv.Stop() + <-done + + return false + } +} diff --git a/packages/shared/pkg/grpc/shutdown_test.go b/packages/shared/pkg/grpc/shutdown_test.go new file mode 100644 index 0000000000..1c727b076e --- /dev/null +++ b/packages/shared/pkg/grpc/shutdown_test.go @@ -0,0 +1,54 @@ +package grpc + +import ( + "net" + "testing" + "time" + + "google.golang.org/grpc" +) + +// TestGracefulStopWithTimeout_Clean ensures the helper returns true when the +// server has no in-flight RPCs and GracefulStop completes immediately. +func TestGracefulStopWithTimeout_Clean(t *testing.T) { + t.Parallel() + + srv := grpc.NewServer() + listenCfg := &net.ListenConfig{} + ln, err := listenCfg.Listen(t.Context(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = srv.Serve(ln) }() + + start := time.Now() + ok := GracefulStopWithTimeout(srv, time.Second) + if !ok { + t.Fatal("expected clean stop") + } + if elapsed := time.Since(start); elapsed > 500*time.Millisecond { + t.Fatalf("stop took too long: %s", elapsed) + } +} + +// TestGracefulStopWithTimeout_AlreadyStopped is a sanity check that calling +// the helper on an already-stopped server doesn't hang. +func TestGracefulStopWithTimeout_AlreadyStopped(t *testing.T) { + t.Parallel() + + srv := grpc.NewServer() + listenCfg := &net.ListenConfig{} + ln, err := listenCfg.Listen(t.Context(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = srv.Serve(ln) }() + + if !GracefulStopWithTimeout(srv, time.Second) { + t.Fatal("first stop should be clean") + } + // Second invocation: GracefulStop on a stopped server is a no-op. + if !GracefulStopWithTimeout(srv, time.Second) { + t.Fatal("second stop should also report clean") + } +} From d92c6f000fa1dfc78ecf29dc2546a8ddad5c5557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Nov=C3=A1k?= Date: Mon, 18 May 2026 06:15:05 -0700 Subject: [PATCH 2/8] Apply suggestion from @jakubno --- iac/modules/job-api/jobs/api.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iac/modules/job-api/jobs/api.hcl b/iac/modules/job-api/jobs/api.hcl index 5d33409ea2..057f19e052 100644 --- a/iac/modules/job-api/jobs/api.hcl +++ b/iac/modules/job-api/jobs/api.hcl @@ -139,7 +139,7 @@ job "api" { task "start" { driver = "docker" - # Budget = shutdownDrainWait (15s) + httpShutdownTimeout (requestTimeout 70s) + grpcShutdownTimeout (10s) + cleanup (30s) + slack. + # Budget = shutdownDrainWait (15s) + httpShutdownTimeout (requestTimeout 70s + 5s) + grpcShutdownTimeout (10s) + cleanup (30s) + slack. # https://developer.hashicorp.com/nomad/docs/configuration/client#max_kill_timeout kill_timeout = "150s" kill_signal = "SIGTERM" From 9d5e7cfcb6c569289e0012498b037b1736449551 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Tue, 19 May 2026 10:00:12 +0000 Subject: [PATCH 3/8] fix(api): drain HTTP, gRPC, and pprof in parallel on shutdown Previously the shutdown sequence ran s.Shutdown, the two gRPC GracefulStopWithTimeout calls, and pprofServer.Shutdown serially, making the worst-case wallclock the sum of their timeouts and pushing the process uncomfortably close to Nomad's kill_timeout. The listeners are independent (separate ports / sockets) and share no mutable state during shutdown, so run them concurrently. Wallclock is now bounded by max(httpShutdownTimeout, grpcShutdownTimeout, pprofShutdownTimeout) instead of the sum. --- packages/api/main.go | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/packages/api/main.go b/packages/api/main.go index 097ec67d04..f1e24c1aa5 100644 --- a/packages/api/main.go +++ b/packages/api/main.go @@ -559,36 +559,39 @@ func run() int { time.Sleep(shutdownDrainWait) } - // Drain in-flight HTTP requests - httpShutdownCtx, httpShutdownCancel := context.WithTimeout(ctx, httpShutdownTimeout) - defer httpShutdownCancel() - if err := s.Shutdown(httpShutdownCtx); err != nil { - exitCode.Add(1) - l.Error(ctx, "Http service shutdown error", zap.Int("port", port), zap.Error(err)) - } + // Drain HTTP, gRPC in parallel. + drainWG := &sync.WaitGroup{} + + drainWG.Go(func() { + httpShutdownCtx, httpShutdownCancel := context.WithTimeout(ctx, httpShutdownTimeout) + defer httpShutdownCancel() + if err := s.Shutdown(httpShutdownCtx); err != nil { + exitCode.Add(1) + l.Error(ctx, "Http service shutdown error", zap.Int("port", port), zap.Error(err)) + } + }) // Bounded gRPC stop: GracefulStop has no built-in deadline, so a // stuck stream would block past Nomad's kill_timeout. Fall back to - // Stop() after the budget elapses. Run both stops in parallel — - // they are independent and serial draining would double the budget. - grpcWG := &sync.WaitGroup{} - grpcWG.Go(func() { + // Stop() after the budget elapses. + drainWG.Go(func() { if !e2bgrpc.GracefulStopWithTimeout(grpcServer, grpcShutdownTimeout) { l.Warn(ctx, "internal gRPC forced stop after graceful timeout", zap.Duration("budget", grpcShutdownTimeout)) } }) - grpcWG.Go(func() { + drainWG.Go(func() { if !e2bgrpc.GracefulStopWithTimeout(edgeGrpcServer, grpcShutdownTimeout) { l.Warn(ctx, "edge gRPC forced stop after graceful timeout", zap.Duration("budget", grpcShutdownTimeout)) } }) - grpcWG.Wait() + drainWG.Wait() + // Drain pprof after, so that it is still available during the shutdown process for debugging if needed. pprofShutdownCtx, pprofCancel := context.WithTimeout(ctx, pprofShutdownTimeout) + defer pprofCancel() if err := pprofServer.Shutdown(pprofShutdownCtx); err != nil { l.Error(ctx, "pprof server shutdown error", zap.Error(err)) } - pprofCancel() }) // wait for the HTTP service to complete shutting down first From cf00e240fa5ca77bfb59bf68f7d097268a3abd25 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Tue, 19 May 2026 11:09:52 +0000 Subject: [PATCH 4/8] fix(api): don't wait on GracefulStop goroutine after forced Stop Server.Stop() force-closes transports but still waits internally for handler goroutines to return. A handler that ignores its canceled context can keep GracefulStop blocked even after Stop(), so the previous <-done re-introduced the unbounded wait this helper was meant to prevent and could push shutdown past Nomad's kill_timeout. Return as soon as the timeout fires; the leaked GracefulStop goroutine dies with the process. --- packages/shared/pkg/grpc/shutdown.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/shared/pkg/grpc/shutdown.go b/packages/shared/pkg/grpc/shutdown.go index a71209d6d2..edc04ba94d 100644 --- a/packages/shared/pkg/grpc/shutdown.go +++ b/packages/shared/pkg/grpc/shutdown.go @@ -14,8 +14,7 @@ import ( // built-in deadline. A stuck stream would otherwise block process shutdown // past Nomad's kill_timeout and result in SIGKILL. // -// After Stop() is called, we still wait for the inner goroutine to return so -// the caller can rely on all server resources being released. +// Stop() force-closes transports func GracefulStopWithTimeout(srv *grpc.Server, d time.Duration) bool { done := make(chan struct{}) @@ -29,7 +28,6 @@ func GracefulStopWithTimeout(srv *grpc.Server, d time.Duration) bool { return true case <-time.After(d): srv.Stop() - <-done return false } From f5d8d8db67e662a43892cb878c0a25a7c1a2f3ae Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Tue, 19 May 2026 11:16:23 +0000 Subject: [PATCH 5/8] fix(api): match gRPC shutdown budget to HTTP ResumeSandbox can intentionally wait up to autoResumeTransitionWaitBudget (1 minute) for a transitioning sandbox before starting it. The previous 10s gRPC drain budget caused Stop() to cancel these in-flight RPCs during rolling deploys, aborting valid auto-resume requests. Use httpShutdownTimeout (requestTimeout + 5s = 75s) for both servers so gRPC handlers get the same drain budget as HTTP requests. Both still run in parallel and complete well within Nomad's 150s kill_timeout. --- packages/api/main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/api/main.go b/packages/api/main.go index f1e24c1aa5..f9db15394a 100644 --- a/packages/api/main.go +++ b/packages/api/main.go @@ -87,8 +87,11 @@ const ( // grpcShutdownTimeout caps grpc.Server.GracefulStop. After this we // fall back to Stop() so a stuck stream cannot block the process - // past Nomad's kill_timeout. - grpcShutdownTimeout = 10 * time.Second + // past Nomad's kill_timeout. Matches httpShutdownTimeout so legitimate + // long-running handlers (e.g. ResumeSandbox waiting up to + // autoResumeTransitionWaitBudget for a transitioning sandbox) get the + // same drain budget as HTTP requests during rolling deploys. + grpcShutdownTimeout = httpShutdownTimeout // pprofShutdownTimeout is a best-effort bound for pprof drain. pprofShutdownTimeout = 5 * time.Second From af42f715112fd30d8040a2740b565fba105717d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Nov=C3=A1k?= Date: Tue, 19 May 2026 04:17:16 -0700 Subject: [PATCH 6/8] Apply suggestion from @jakubno --- iac/modules/job-api/jobs/api.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iac/modules/job-api/jobs/api.hcl b/iac/modules/job-api/jobs/api.hcl index 057f19e052..768bd71561 100644 --- a/iac/modules/job-api/jobs/api.hcl +++ b/iac/modules/job-api/jobs/api.hcl @@ -139,7 +139,7 @@ job "api" { task "start" { driver = "docker" - # Budget = shutdownDrainWait (15s) + httpShutdownTimeout (requestTimeout 70s + 5s) + grpcShutdownTimeout (10s) + cleanup (30s) + slack. + # Budget = shutdownDrainWait (15s) + shutdownTimeout (requestTimeout 70s + 5s) + cleanup (30s) + slack. # https://developer.hashicorp.com/nomad/docs/configuration/client#max_kill_timeout kill_timeout = "150s" kill_signal = "SIGTERM" From 1b127c9b5d52e85ca1f45692524f5e76841a6562 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Tue, 19 May 2026 11:18:32 +0000 Subject: [PATCH 7/8] chore: merge timeouts --- packages/api/main.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/packages/api/main.go b/packages/api/main.go index f9db15394a..6bd1bc30be 100644 --- a/packages/api/main.go +++ b/packages/api/main.go @@ -78,20 +78,16 @@ const ( // us off active backends. shutdownDrainWait = 15 * time.Second - // httpShutdownTimeout caps how long s.Shutdown waits for in-flight + // shutdownTimeout caps how long s.Shutdown waits for in-flight // requests to complete. Must be >= requestTimeout so a request, that // arrived before load balancer stopped sending new traffic, // has its full deadline to finish // (e.g. a slow sandbox pause / snapshot RPC). - httpShutdownTimeout = requestTimeout + 5*time.Second - - // grpcShutdownTimeout caps grpc.Server.GracefulStop. After this we + // + // Also caps grpc.Server.GracefulStop. After this we // fall back to Stop() so a stuck stream cannot block the process - // past Nomad's kill_timeout. Matches httpShutdownTimeout so legitimate - // long-running handlers (e.g. ResumeSandbox waiting up to - // autoResumeTransitionWaitBudget for a transitioning sandbox) get the - // same drain budget as HTTP requests during rolling deploys. - grpcShutdownTimeout = httpShutdownTimeout + // past Nomad's kill_timeout. + shutdownTimeout = requestTimeout + 5*time.Second // pprofShutdownTimeout is a best-effort bound for pprof drain. pprofShutdownTimeout = 5 * time.Second @@ -566,7 +562,7 @@ func run() int { drainWG := &sync.WaitGroup{} drainWG.Go(func() { - httpShutdownCtx, httpShutdownCancel := context.WithTimeout(ctx, httpShutdownTimeout) + httpShutdownCtx, httpShutdownCancel := context.WithTimeout(ctx, shutdownTimeout) defer httpShutdownCancel() if err := s.Shutdown(httpShutdownCtx); err != nil { exitCode.Add(1) @@ -578,8 +574,8 @@ func run() int { // stuck stream would block past Nomad's kill_timeout. Fall back to // Stop() after the budget elapses. drainWG.Go(func() { - if !e2bgrpc.GracefulStopWithTimeout(grpcServer, grpcShutdownTimeout) { - l.Warn(ctx, "internal gRPC forced stop after graceful timeout", zap.Duration("budget", grpcShutdownTimeout)) + if !e2bgrpc.GracefulStopWithTimeout(grpcServer, shutdownTimeout) { + l.Warn(ctx, "internal gRPC forced stop after graceful timeout", zap.Duration("budget", shutdownTimeout)) } }) drainWG.Go(func() { From a1d6b4421b1e28897234e06769d2624abbf732c9 Mon Sep 17 00:00:00 2001 From: Jakub Novak Date: Tue, 19 May 2026 11:43:36 +0000 Subject: [PATCH 8/8] chore: merge timeouts --- packages/api/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/main.go b/packages/api/main.go index 6bd1bc30be..b025c21a4b 100644 --- a/packages/api/main.go +++ b/packages/api/main.go @@ -579,8 +579,8 @@ func run() int { } }) drainWG.Go(func() { - if !e2bgrpc.GracefulStopWithTimeout(edgeGrpcServer, grpcShutdownTimeout) { - l.Warn(ctx, "edge gRPC forced stop after graceful timeout", zap.Duration("budget", grpcShutdownTimeout)) + if !e2bgrpc.GracefulStopWithTimeout(edgeGrpcServer, shutdownTimeout) { + l.Warn(ctx, "edge gRPC forced stop after graceful timeout", zap.Duration("budget", shutdownTimeout)) } }) drainWG.Wait()