From da3757bd6e8395f29f1c0d6816566e305d558eef Mon Sep 17 00:00:00 2001 From: e2b Date: Sun, 29 Mar 2026 17:12:25 +0200 Subject: [PATCH 1/3] feat: ARM64 runtime guards for SMT, CPU info, seccomp, and UFFD - Disable SMT on ARM64 (Firecracker rejects SMT=true on ARM processors) - Add --no-seccomp flag on ARM64 (upstream seccomp filter lacks uffd syscall) - Provide fallback CPU Family/Model on ARM64 (gopsutil doesn't populate these) - Gracefully skip hugepage tests when ENOMEM (insufficient hugepages on CI) - Use runtime.GOARCH instead of hardcoded amd64 in smoketest envd build Co-Authored-By: Claude Opus 4.6 (1M context) --- .../orchestrator/cmd/smoketest/smoke_test.go | 3 ++- .../orchestrator/pkg/sandbox/fc/client.go | 11 +++++++++- .../pkg/sandbox/fc/script_builder.go | 16 +++++++++++++-- .../pkg/sandbox/uffd/testutils/page_mmap.go | 12 ++++++++++- .../pkg/service/machineinfo/main.go | 20 ++++++++++++++++--- 5 files changed, 54 insertions(+), 8 deletions(-) diff --git a/packages/orchestrator/cmd/smoketest/smoke_test.go b/packages/orchestrator/cmd/smoketest/smoke_test.go index 3718dadb8e..dbec4a721a 100644 --- a/packages/orchestrator/cmd/smoketest/smoke_test.go +++ b/packages/orchestrator/cmd/smoketest/smoke_test.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "testing" "time" @@ -285,7 +286,7 @@ func findOrBuildEnvd(t *testing.T) string { cmd := exec.CommandContext(t.Context(), "go", "build", "-o", binPath, ".") //nolint:gosec // trusted input cmd.Dir = envdDir - cmd.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=linux", "GOARCH=amd64") + cmd.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=linux", "GOARCH="+runtime.GOARCH) out, err := cmd.CombinedOutput() if err != nil { t.Skipf("failed to build envd: %v\n%s", err, out) diff --git a/packages/orchestrator/pkg/sandbox/fc/client.go b/packages/orchestrator/pkg/sandbox/fc/client.go index 1af2db0ec8..2b67174205 100644 --- a/packages/orchestrator/pkg/sandbox/fc/client.go +++ b/packages/orchestrator/pkg/sandbox/fc/client.go @@ -3,6 +3,7 @@ package fc import ( "context" "fmt" + "runtime" "github.com/bits-and-blooms/bitset" "github.com/firecracker-microvm/firecracker-go-sdk" @@ -326,7 +327,15 @@ func (c *apiClient) setMachineConfig( memoryMB int64, hugePages bool, ) error { - smt := true + // SMT (Simultaneous Multi-Threading / Hyper-Threading) must be disabled on + // ARM64 because ARM processors use a different core topology (big.LITTLE, + // efficiency/performance cores) rather than hardware threads per core. + // Firecracker validates this against the host CPU and rejects SMT=true on ARM. + // See: https://github.com/firecracker-microvm/firecracker/blob/main/docs/cpu_templates/cpu-features.md + // We use runtime.GOARCH (not TARGET_ARCH) because the orchestrator binary + // always runs on the same architecture as Firecracker. + const archARM64 = "arm64" + smt := runtime.GOARCH != archARM64 trackDirtyPages := false machineConfig := &models.MachineConfiguration{ VcpuCount: &vCPUCount, diff --git a/packages/orchestrator/pkg/sandbox/fc/script_builder.go b/packages/orchestrator/pkg/sandbox/fc/script_builder.go index 71bbf497f6..c1773d7fd3 100644 --- a/packages/orchestrator/pkg/sandbox/fc/script_builder.go +++ b/packages/orchestrator/pkg/sandbox/fc/script_builder.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "path/filepath" + "runtime" txtTemplate "text/template" "github.com/e2b-dev/infra/packages/orchestrator/pkg/cfg" @@ -25,6 +26,7 @@ type startScriptArgs struct { NamespaceID string FirecrackerPath string FirecrackerSocket string + ExtraArgs string } // StartScriptResult contains the generated script and computed paths @@ -47,7 +49,7 @@ ln -s {{ .HostRootfsPath }} {{ .DeprecatedSandboxRootfsDir }}/{{ .SandboxRootfsF mount -t tmpfs tmpfs {{ .SandboxDir }}/{{ .SandboxKernelDir }} -o X-mount.mkdir && ln -s {{ .HostKernelPath }} {{ .SandboxDir }}/{{ .SandboxKernelDir }}/{{ .SandboxKernelFile }} && -ip netns exec {{ .NamespaceID }} {{ .FirecrackerPath }} --api-sock {{ .FirecrackerSocket }}` +ip netns exec {{ .NamespaceID }} {{ .FirecrackerPath }} --api-sock {{ .FirecrackerSocket }}{{ .ExtraArgs }}` const startScriptV2 = `mount --make-rprivate / && mount -t tmpfs tmpfs {{ .SandboxDir }} -o X-mount.mkdir && @@ -57,7 +59,7 @@ ln -s {{ .HostRootfsPath }} {{ .SandboxDir }}/{{ .SandboxRootfsFile }} && mkdir -p {{ .SandboxDir }}/{{ .SandboxKernelDir }} && ln -s {{ .HostKernelPath }} {{ .SandboxDir }}/{{ .SandboxKernelDir }}/{{ .SandboxKernelFile }} && -ip netns exec {{ .NamespaceID }} {{ .FirecrackerPath }} --api-sock {{ .FirecrackerSocket }}` +ip netns exec {{ .NamespaceID }} {{ .FirecrackerPath }} --api-sock {{ .FirecrackerSocket }}{{ .ExtraArgs }}` // StartScriptBuilder handles the creation and execution of firecracker start scripts type StartScriptBuilder struct { @@ -85,6 +87,15 @@ func (sb *StartScriptBuilder) buildArgs( rootfsPaths RootfsPaths, namespaceID string, ) startScriptArgs { + // On ARM64, disable seccomp to allow userfaultfd syscall for snapshot restore. + // The upstream Firecracker seccomp filter for aarch64 does not include the + // userfaultfd syscall (nr 282), causing snapshot loading to fail with + // "Failed to UFFD object: System error". + var extraArgs string + if runtime.GOARCH == "arm64" { + extraArgs = " --no-seccomp" + } + return startScriptArgs{ // General SandboxDir: sb.builderConfig.SandboxDir, @@ -103,6 +114,7 @@ func (sb *StartScriptBuilder) buildArgs( NamespaceID: namespaceID, FirecrackerPath: versions.FirecrackerPath(sb.builderConfig), FirecrackerSocket: files.SandboxFirecrackerSocketPath(), + ExtraArgs: extraArgs, } } diff --git a/packages/orchestrator/pkg/sandbox/uffd/testutils/page_mmap.go b/packages/orchestrator/pkg/sandbox/uffd/testutils/page_mmap.go index 929a396702..ac17b0788f 100644 --- a/packages/orchestrator/pkg/sandbox/uffd/testutils/page_mmap.go +++ b/packages/orchestrator/pkg/sandbox/uffd/testutils/page_mmap.go @@ -1,6 +1,7 @@ package testutils import ( + "errors" "fmt" "math" "syscall" @@ -20,7 +21,16 @@ func NewPageMmap(t *testing.T, size, pagesize uint64) ([]byte, uintptr, error) { } if pagesize == header.HugepageSize { - return newMmap(t, size, header.HugepageSize, unix.MAP_HUGETLB|unix.MAP_HUGE_2MB) + b, addr, err := newMmap(t, size, header.HugepageSize, unix.MAP_HUGETLB|unix.MAP_HUGE_2MB) + // Hugepage allocation can fail with ENOMEM on CI runners that don't + // have enough (or any) hugepages pre-allocated in /proc/sys/vm/nr_hugepages. + // Skip gracefully rather than failing the test. + if err != nil && errors.Is(err, syscall.ENOMEM) { + pages := int(math.Ceil(float64(size) / float64(header.HugepageSize))) + t.Skipf("skipping: hugepage mmap failed (need %d hugepages): %v", pages, err) + } + + return b, addr, err } return nil, 0, fmt.Errorf("unsupported page size: %d", pagesize) diff --git a/packages/orchestrator/pkg/service/machineinfo/main.go b/packages/orchestrator/pkg/service/machineinfo/main.go index 27d280da8b..f221902f87 100644 --- a/packages/orchestrator/pkg/service/machineinfo/main.go +++ b/packages/orchestrator/pkg/service/machineinfo/main.go @@ -22,13 +22,27 @@ func Detect() (MachineInfo, error) { } if len(info) > 0 { - if info[0].Family == "" || info[0].Model == "" { + family := info[0].Family + model := info[0].Model + + // On ARM64, gopsutil doesn't populate Family/Model from /proc/cpuinfo. + // Provide fallback values so callers don't get an error. + if runtime.GOARCH == "arm64" { + if family == "" { + family = "arm64" + } + if model == "" { + model = "0" + } + } + + if family == "" || model == "" { return MachineInfo{}, fmt.Errorf("unable to detect CPU platform from CPU info: %+v", info[0]) } return MachineInfo{ - Family: info[0].Family, - Model: info[0].Model, + Family: family, + Model: model, ModelName: info[0].ModelName, Flags: info[0].Flags, Arch: runtime.GOARCH, From 064b1a6e853a637684ba10d66ef29b19a0038caf Mon Sep 17 00:00:00 2001 From: e2b Date: Mon, 30 Mar 2026 15:44:42 +0200 Subject: [PATCH 2/3] fix: move archARM64 const to package level, document CPU compat limits - Move const archARM64 out of setMachineConfig per review feedback - Add comment about ARM64 CPU compatibility limitations for cross-host snapshot restore Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/orchestrator/pkg/sandbox/fc/client.go | 3 ++- packages/orchestrator/pkg/service/machineinfo/main.go | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/orchestrator/pkg/sandbox/fc/client.go b/packages/orchestrator/pkg/sandbox/fc/client.go index 2b67174205..46b82f5722 100644 --- a/packages/orchestrator/pkg/sandbox/fc/client.go +++ b/packages/orchestrator/pkg/sandbox/fc/client.go @@ -19,6 +19,8 @@ import ( "github.com/e2b-dev/infra/packages/shared/pkg/utils" ) +const archARM64 = "arm64" + type apiClient struct { client *client.Firecracker } @@ -334,7 +336,6 @@ func (c *apiClient) setMachineConfig( // See: https://github.com/firecracker-microvm/firecracker/blob/main/docs/cpu_templates/cpu-features.md // We use runtime.GOARCH (not TARGET_ARCH) because the orchestrator binary // always runs on the same architecture as Firecracker. - const archARM64 = "arm64" smt := runtime.GOARCH != archARM64 trackDirtyPages := false machineConfig := &models.MachineConfiguration{ diff --git a/packages/orchestrator/pkg/service/machineinfo/main.go b/packages/orchestrator/pkg/service/machineinfo/main.go index f221902f87..0934880df7 100644 --- a/packages/orchestrator/pkg/service/machineinfo/main.go +++ b/packages/orchestrator/pkg/service/machineinfo/main.go @@ -27,6 +27,10 @@ func Detect() (MachineInfo, error) { // On ARM64, gopsutil doesn't populate Family/Model from /proc/cpuinfo. // Provide fallback values so callers don't get an error. + // NOTE: Using a generic "arm64" family treats all ARM64 CPUs as compatible. + // This works for same-host snapshot restore but cross-host restore between + // different ARM CPU implementations (e.g. Graviton2 vs Graviton3) may fail. + // For finer granularity, consider using MIDR_EL1 register values. if runtime.GOARCH == "arm64" { if family == "" { family = "arm64" From 32ed0312aa40abe34550efc98e3f07b7f17760c7 Mon Sep 17 00:00:00 2001 From: e2b Date: Mon, 30 Mar 2026 09:10:06 +0200 Subject: [PATCH 3/3] fix: remove --no-seccomp on ARM64, verified unnecessary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tested full sandbox lifecycle (UFFD snapshot restore + VM resume) on ARM64 with seccomp ENABLED using Firecracker v1.12 on kernel 6.17: - Lima VM (Apple Silicon), full E2B local-infra stack - Sandbox created, Firecracker launched without --no-seccomp - UFFD page fault handling worked correctly - VM resumed and envd initialized successfully The userfaultfd fd is created via /dev/userfaultfd (kernel 6.1+) before seccomp is installed, so the userfaultfd syscall is not needed in the seccomp filter. The original "Failed to UFFD object" error was likely caused by host configuration (missing /dev/userfaultfd device, permissions, or vm.unprivileged_userfaultfd=0). Reverts script_builder.go to match main — no ARM64-specific Firecracker args needed. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../pkg/sandbox/fc/script_builder.go | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/packages/orchestrator/pkg/sandbox/fc/script_builder.go b/packages/orchestrator/pkg/sandbox/fc/script_builder.go index c1773d7fd3..71bbf497f6 100644 --- a/packages/orchestrator/pkg/sandbox/fc/script_builder.go +++ b/packages/orchestrator/pkg/sandbox/fc/script_builder.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "path/filepath" - "runtime" txtTemplate "text/template" "github.com/e2b-dev/infra/packages/orchestrator/pkg/cfg" @@ -26,7 +25,6 @@ type startScriptArgs struct { NamespaceID string FirecrackerPath string FirecrackerSocket string - ExtraArgs string } // StartScriptResult contains the generated script and computed paths @@ -49,7 +47,7 @@ ln -s {{ .HostRootfsPath }} {{ .DeprecatedSandboxRootfsDir }}/{{ .SandboxRootfsF mount -t tmpfs tmpfs {{ .SandboxDir }}/{{ .SandboxKernelDir }} -o X-mount.mkdir && ln -s {{ .HostKernelPath }} {{ .SandboxDir }}/{{ .SandboxKernelDir }}/{{ .SandboxKernelFile }} && -ip netns exec {{ .NamespaceID }} {{ .FirecrackerPath }} --api-sock {{ .FirecrackerSocket }}{{ .ExtraArgs }}` +ip netns exec {{ .NamespaceID }} {{ .FirecrackerPath }} --api-sock {{ .FirecrackerSocket }}` const startScriptV2 = `mount --make-rprivate / && mount -t tmpfs tmpfs {{ .SandboxDir }} -o X-mount.mkdir && @@ -59,7 +57,7 @@ ln -s {{ .HostRootfsPath }} {{ .SandboxDir }}/{{ .SandboxRootfsFile }} && mkdir -p {{ .SandboxDir }}/{{ .SandboxKernelDir }} && ln -s {{ .HostKernelPath }} {{ .SandboxDir }}/{{ .SandboxKernelDir }}/{{ .SandboxKernelFile }} && -ip netns exec {{ .NamespaceID }} {{ .FirecrackerPath }} --api-sock {{ .FirecrackerSocket }}{{ .ExtraArgs }}` +ip netns exec {{ .NamespaceID }} {{ .FirecrackerPath }} --api-sock {{ .FirecrackerSocket }}` // StartScriptBuilder handles the creation and execution of firecracker start scripts type StartScriptBuilder struct { @@ -87,15 +85,6 @@ func (sb *StartScriptBuilder) buildArgs( rootfsPaths RootfsPaths, namespaceID string, ) startScriptArgs { - // On ARM64, disable seccomp to allow userfaultfd syscall for snapshot restore. - // The upstream Firecracker seccomp filter for aarch64 does not include the - // userfaultfd syscall (nr 282), causing snapshot loading to fail with - // "Failed to UFFD object: System error". - var extraArgs string - if runtime.GOARCH == "arm64" { - extraArgs = " --no-seccomp" - } - return startScriptArgs{ // General SandboxDir: sb.builderConfig.SandboxDir, @@ -114,7 +103,6 @@ func (sb *StartScriptBuilder) buildArgs( NamespaceID: namespaceID, FirecrackerPath: versions.FirecrackerPath(sb.builderConfig), FirecrackerSocket: files.SandboxFirecrackerSocketPath(), - ExtraArgs: extraArgs, } }