From 1b56a8a1df2eb104a42a79a6a21e2eb33383c778 Mon Sep 17 00:00:00 2001
From: Lev Brouk <lev@e2b.dev>
Date: Thu, 18 Jun 2026 11:38:13 -0700
Subject: [PATCH] fix(orchestrator): order envd.service after local-fs.target
 to stop /tmp race
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

updateEnvd stages the new envd binary at /tmp/envd_updated right after
WaitForEnvd. #3020 set DefaultDependencies=no and dropped envd's implicit
ordering, so it answers POST /init — and runs that /tmp staging — before the
early-boot filesystem window settles; if tmp.mount lands mid-upload, the path
lookup resolves through the fresh mount and returns ENOENT (xattr.list
/tmp/envd_updated: no such file or directory). Order envd after
local-fs.target so it no longer starts inside that window.

The .tpl change alone is inert on warm caches: the base-layer hash keyed on
(indexVersion, provisionVersion, diskSizeMB, baseSource) ignores baked file
content, so a changed unit doesn't invalidate cached base layers. Fold a hash
of the embedded rootfs file templates (rootfs.FilesHash) into the base-layer
key so unit-file changes force a rebuild.
---
 .../build/core/rootfs/files/envd.service.tpl  |  5 ++++-
 .../pkg/template/build/core/rootfs/rootfs.go  | 22 +++++++++++++++++++
 .../pkg/template/build/phases/base/hash.go    |  5 +++++
 3 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl
index 9754a534ab..7f59a239ad 100644
--- a/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl
+++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/envd.service.tpl
@@ -8,7 +8,10 @@ Description=Env Daemon Service
 # userspace. Default dependencies would gate it on sysinit/basic.target
 # (~0.5s), and the previous After=multi-user.target on chrony-wait (~8s).
 DefaultDependencies=no
-After=systemd-journald.socket systemd-remount-fs.service
+# local-fs.target keeps envd from answering /init before /tmp (a systemd-managed
+# tmpfs) is mounted: updateEnvd stages an update binary in /tmp during early
+# boot, and without this ordering the upload can race tmp.mount and fail ENOENT.
+After=systemd-journald.socket systemd-remount-fs.service local-fs.target
 Wants=systemd-journald.socket
 Conflicts=shutdown.target
 Before=shutdown.target
diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/rootfs.go b/packages/orchestrator/pkg/template/build/core/rootfs/rootfs.go
index 1499a61c13..3c0781142d 100644
--- a/packages/orchestrator/pkg/template/build/core/rootfs/rootfs.go
+++ b/packages/orchestrator/pkg/template/build/core/rootfs/rootfs.go
@@ -4,10 +4,13 @@ package rootfs
 
 import (
 	"context"
+	"crypto/sha256"
 	"embed"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -38,6 +41,25 @@ var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/orchestrator/pkg/tem
 var files embed.FS
 var fileTemplates = template.Must(template.ParseFS(files, "files/*"))
 
+// filesHash is a stable hash of the embedded rootfs file templates. Folded into
+// the base-layer cache key so a change to any baked unit (e.g. envd.service.tpl)
+// invalidates cached base layers — without it the base hash ignores file content
+// and stale layers keep the old units. Reads on an embed.FS can't fail at runtime
+// (content is baked at compile time), so the errors are safely ignored.
+var filesHash = func() string {
+	entries, _ := fs.ReadDir(files, "files") // sorted by name
+	h := sha256.New()
+	for _, e := range entries {
+		data, _ := files.ReadFile("files/" + e.Name())
+		fmt.Fprintf(h, "%s\x00%x\x00", e.Name(), data)
+	}
+
+	return hex.EncodeToString(h.Sum(nil))
+}()
+
+// FilesHash returns a stable hash over the embedded rootfs file templates.
+func FilesHash() string { return filesHash }
+
 const (
 	BusyBoxPath     = "usr/bin/busybox"
 	BusyBoxInitPath = "usr/bin/init"
diff --git a/packages/orchestrator/pkg/template/build/phases/base/hash.go b/packages/orchestrator/pkg/template/build/phases/base/hash.go
index 62d5284022..45f2704c51 100644
--- a/packages/orchestrator/pkg/template/build/phases/base/hash.go
+++ b/packages/orchestrator/pkg/template/build/phases/base/hash.go
@@ -9,6 +9,7 @@ import (
 
 	"go.opentelemetry.io/otel/attribute"
 
+	"github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs"
 	"github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/phases"
 	"github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/storage/cache"
 	"github.com/e2b-dev/infra/packages/shared/pkg/featureflags"
@@ -56,5 +57,9 @@ func (bb *BaseBuilder) Hash(ctx context.Context, _ phases.LayerResult) (string,
 		provisionVersion,
 		strconv.FormatInt(bb.Config.DiskSizeMB, 10),
 		baseSource,
+		// Invalidate when a baked rootfs file (e.g. envd.service) changes; the
+		// keys above don't otherwise reflect file content, so stale base layers
+		// would keep old units.
+		rootfs.FilesHash(),
 	), nil
 }