diff --git a/packages/envd/internal/api/init.go b/packages/envd/internal/api/init.go index 4d0081b0a2..557dc4b94a 100644 --- a/packages/envd/internal/api/init.go +++ b/packages/envd/internal/api/init.go @@ -131,8 +131,12 @@ func (a *API) PostInit(w http.ResponseWriter, r *http.Request) { a.initLock.Lock() defer a.initLock.Unlock() - // Update data only if the request is newer or if there's no timestamp at all - if initRequest.Timestamp == nil || a.lastSetTime.SetToGreater(initRequest.Timestamp.UnixNano()) { + // Update data only if the request is newer or if there's no timestamp at all. + // We check without mutating so that a failed SetData doesn't permanently + // raise lastSetTime — retries with the same pinned Timestamp must be able + // to re-run SetData until it succeeds. + shouldRun := initRequest.Timestamp == nil || initRequest.Timestamp.UnixNano() > a.lastSetTime.Load() + if shouldRun { err = a.SetData(ctx, logger, initRequest) if err != nil { switch { @@ -146,6 +150,9 @@ func (a *API) PostInit(w http.ResponseWriter, r *http.Request) { return } + if initRequest.Timestamp != nil { + a.lastSetTime.SetToGreater(initRequest.Timestamp.UnixNano()) + } } } diff --git a/packages/envd/internal/utils/atomic.go b/packages/envd/internal/utils/atomic.go index d34d6afc70..438cf9b41c 100644 --- a/packages/envd/internal/utils/atomic.go +++ b/packages/envd/internal/utils/atomic.go @@ -25,3 +25,11 @@ func (a *AtomicMax) SetToGreater(newValue int64) bool { return true } + +// Load returns the current value. +func (a *AtomicMax) Load() int64 { + a.mu.Lock() + defer a.mu.Unlock() + + return a.val +} diff --git a/packages/envd/pkg/version.go b/packages/envd/pkg/version.go index 88bc32c769..4cfbb08b29 100644 --- a/packages/envd/pkg/version.go +++ b/packages/envd/pkg/version.go @@ -1,3 +1,3 @@ package pkg -const Version = "0.5.23" +const Version = "0.5.24" diff --git a/packages/orchestrator/pkg/sandbox/envd.go b/packages/orchestrator/pkg/sandbox/envd.go index 49ffe1ccd0..c5af3dbfa6 100644 --- a/packages/orchestrator/pkg/sandbox/envd.go +++ b/packages/orchestrator/pkg/sandbox/envd.go @@ -37,6 +37,9 @@ func (s *Sandbox) doRequestWithInfiniteRetries( ) (*http.Response, int64, error) { requestCount := int64(0) + // Pin Timestamp once so envd's lastSetTime.SetToGreater guard can skip + // re-delivered retries instead of re-running SetData against dead sockets. + now := time.Now() jsonBody := &envd.PostInitJSONBody{ LifecycleID: s.LifecycleID, EnvVars: s.Config.Envd.Vars, @@ -46,16 +49,15 @@ func (s *Sandbox) doRequestWithInfiniteRetries( DefaultWorkdir: utils.DerefOrDefault(s.Config.Envd.DefaultWorkdir, ""), VolumeMounts: s.convertMounts(s.Config.VolumeMounts), CaBundle: s.CABundle, + Timestamp: now, } - for { - jsonBody.Timestamp = time.Now() - - body, err := json.Marshal(jsonBody) - if err != nil { - return nil, requestCount, err - } + body, err := json.Marshal(jsonBody) + if err != nil { + return nil, requestCount, err + } + for { requestCount++ reqCtx, cancel := context.WithTimeout(ctx, s.internalConfig.EnvdInitRequestTimeout) request, err := http.NewRequestWithContext(reqCtx, method, address, bytes.NewReader(body))