From 8832fd50313fc9a96d0fd836486b1d9a07c35084 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Sun, 17 May 2026 01:46:38 -0700 Subject: [PATCH 1/3] fix(orch): keep PostInit Timestamp stable across retries to break envd livelock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit doRequestWithInfiniteRetries previously set jsonBody.Timestamp = time.Now() on every iteration of the retry loop. envd's PostInit guards SetData with lastSetTime.SetToGreater(initRequest.Timestamp.UnixNano()), which is meant to skip work for an out-of-order or replayed request — but with a fresh timestamp per retry the guard always observed a newer value and ran SetData end-to-end every time. When the orchestrator's 50 ms EnvdInitRequestTimeout fires while envd is still serving the cold first request, envd queues the next retry on initLock and the next one and the next one. Each one then runs SetData against a connection the orchestrator already gave up on, writes 204 to a dead socket, and the orchestrator never observes a success — the livelock described in the bitfrost / Mode-B investigation. Pin Timestamp once before the loop. On the cold path envd's idempotency guard now actually fires for replays, the queue drains by skipping the duplicates, and the next live retry's response reaches the orchestrator. Also marshal the body once instead of on every retry (it doesn't change). --- packages/orchestrator/pkg/sandbox/envd.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/packages/orchestrator/pkg/sandbox/envd.go b/packages/orchestrator/pkg/sandbox/envd.go index 49ffe1ccd0..07fa29f25e 100644 --- a/packages/orchestrator/pkg/sandbox/envd.go +++ b/packages/orchestrator/pkg/sandbox/envd.go @@ -37,6 +37,14 @@ func (s *Sandbox) doRequestWithInfiniteRetries( ) (*http.Response, int64, error) { requestCount := int64(0) + // Pin Timestamp for the whole retry loop so envd's lastSetTime.SetToGreater + // idempotency guard can actually fire on a re-delivered request. With a + // fresh time.Now() per retry every queued PostInit looked newer than the + // previous one, and envd ran SetData end-to-end on every queued retry — + // writing the eventual 204 to a dead socket each time and never letting + // the orchestrator observe a success (the livelock described in the + // bitfrost / Mode-B investigation). + now := time.Now() jsonBody := &envd.PostInitJSONBody{ LifecycleID: s.LifecycleID, EnvVars: s.Config.Envd.Vars, @@ -46,16 +54,15 @@ func (s *Sandbox) doRequestWithInfiniteRetries( DefaultWorkdir: utils.DerefOrDefault(s.Config.Envd.DefaultWorkdir, ""), VolumeMounts: s.convertMounts(s.Config.VolumeMounts), CaBundle: s.CABundle, + Timestamp: now, } - for { - jsonBody.Timestamp = time.Now() - - body, err := json.Marshal(jsonBody) - if err != nil { - return nil, requestCount, err - } + body, err := json.Marshal(jsonBody) + if err != nil { + return nil, requestCount, err + } + for { requestCount++ reqCtx, cancel := context.WithTimeout(ctx, s.internalConfig.EnvdInitRequestTimeout) request, err := http.NewRequestWithContext(reqCtx, method, address, bytes.NewReader(body)) From 360f37c0be6dbc672d8a1c1b81438cfea3ea6f04 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Sun, 17 May 2026 02:36:18 -0700 Subject: [PATCH 2/3] chore: trim verbose comments --- packages/orchestrator/pkg/sandbox/envd.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/orchestrator/pkg/sandbox/envd.go b/packages/orchestrator/pkg/sandbox/envd.go index 07fa29f25e..c5af3dbfa6 100644 --- a/packages/orchestrator/pkg/sandbox/envd.go +++ b/packages/orchestrator/pkg/sandbox/envd.go @@ -37,13 +37,8 @@ func (s *Sandbox) doRequestWithInfiniteRetries( ) (*http.Response, int64, error) { requestCount := int64(0) - // Pin Timestamp for the whole retry loop so envd's lastSetTime.SetToGreater - // idempotency guard can actually fire on a re-delivered request. With a - // fresh time.Now() per retry every queued PostInit looked newer than the - // previous one, and envd ran SetData end-to-end on every queued retry — - // writing the eventual 204 to a dead socket each time and never letting - // the orchestrator observe a success (the livelock described in the - // bitfrost / Mode-B investigation). + // Pin Timestamp once so envd's lastSetTime.SetToGreater guard can skip + // re-delivered retries instead of re-running SetData against dead sockets. now := time.Now() jsonBody := &envd.PostInitJSONBody{ LifecycleID: s.LifecycleID, From df5007c7bd7c38638a6bc1f0c5efe63e77f4f055 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Sun, 17 May 2026 02:53:03 -0700 Subject: [PATCH 3/3] fix(envd): don't update lastSetTime until SetData succeeds With the orchestrator now pinning Timestamp once across retries, a SetData failure would otherwise leave lastSetTime updated and cause subsequent retries with the same timestamp to skip the (still-needed) SetData and return 204 success. Move the SetToGreater after the success check. --- packages/envd/internal/api/init.go | 11 +++++++++-- packages/envd/internal/utils/atomic.go | 8 ++++++++ packages/envd/pkg/version.go | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/envd/internal/api/init.go b/packages/envd/internal/api/init.go index 4d0081b0a2..557dc4b94a 100644 --- a/packages/envd/internal/api/init.go +++ b/packages/envd/internal/api/init.go @@ -131,8 +131,12 @@ func (a *API) PostInit(w http.ResponseWriter, r *http.Request) { a.initLock.Lock() defer a.initLock.Unlock() - // Update data only if the request is newer or if there's no timestamp at all - if initRequest.Timestamp == nil || a.lastSetTime.SetToGreater(initRequest.Timestamp.UnixNano()) { + // Update data only if the request is newer or if there's no timestamp at all. + // We check without mutating so that a failed SetData doesn't permanently + // raise lastSetTime — retries with the same pinned Timestamp must be able + // to re-run SetData until it succeeds. + shouldRun := initRequest.Timestamp == nil || initRequest.Timestamp.UnixNano() > a.lastSetTime.Load() + if shouldRun { err = a.SetData(ctx, logger, initRequest) if err != nil { switch { @@ -146,6 +150,9 @@ func (a *API) PostInit(w http.ResponseWriter, r *http.Request) { return } + if initRequest.Timestamp != nil { + a.lastSetTime.SetToGreater(initRequest.Timestamp.UnixNano()) + } } } diff --git a/packages/envd/internal/utils/atomic.go b/packages/envd/internal/utils/atomic.go index d34d6afc70..438cf9b41c 100644 --- a/packages/envd/internal/utils/atomic.go +++ b/packages/envd/internal/utils/atomic.go @@ -25,3 +25,11 @@ func (a *AtomicMax) SetToGreater(newValue int64) bool { return true } + +// Load returns the current value. +func (a *AtomicMax) Load() int64 { + a.mu.Lock() + defer a.mu.Unlock() + + return a.val +} diff --git a/packages/envd/pkg/version.go b/packages/envd/pkg/version.go index 88bc32c769..4cfbb08b29 100644 --- a/packages/envd/pkg/version.go +++ b/packages/envd/pkg/version.go @@ -1,3 +1,3 @@ package pkg -const Version = "0.5.23" +const Version = "0.5.24"