From 0576ecd5499bda4bf5ae3571ae2c50a1e495e3a9 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Thu, 16 Apr 2026 18:44:28 -0700 Subject: [PATCH] perf(sandbox): reduce memory dirtying from journald and envd logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add journald.conf drop-in with Storage=none to prevent journal writes from dirtying guest memory pages during pause-resume cycles - Remove stdout writer from envd in FC mode — debug logs ship via HTTP exporter only, never hitting journald/disk inside the VM - Add measure-memory-dirtying.sh script to benchmark diff sizes across pause-resume cycles with the orchestrator CLI tools - Bump envd version to 0.5.15 --- packages/envd/internal/logs/logger.go | 7 +- packages/envd/pkg/version.go | 2 +- .../resume-build/measure-memory-dirtying.sh | 82 +++++++++++++++++++ .../build/core/rootfs/files/journald.conf.tpl | 9 ++ .../template/build/core/rootfs/rootfs_test.go | 6 +- 5 files changed, 102 insertions(+), 4 deletions(-) create mode 100755 packages/orchestrator/cmd/resume-build/measure-memory-dirtying.sh create mode 100644 packages/orchestrator/pkg/template/build/core/rootfs/files/journald.conf.tpl diff --git a/packages/envd/internal/logs/logger.go b/packages/envd/internal/logs/logger.go index 4d8fbaabb2..8a15c24889 100644 --- a/packages/envd/internal/logs/logger.go +++ b/packages/envd/internal/logs/logger.go @@ -18,18 +18,21 @@ func NewLogger(ctx context.Context, isNotFC bool, mmdsChan <-chan *host.MMDSOpts exporters := []io.Writer{} + var level zerolog.Level if isNotFC { exporters = append(exporters, os.Stdout) } else { - exporters = append(exporters, exporter.NewHTTPLogsExporter(ctx, isNotFC, mmdsChan), os.Stdout) + // HTTP exporter only — stdout goes to journald which dirties guest memory pages. + exporters = append(exporters, exporter.NewHTTPLogsExporter(ctx, isNotFC, mmdsChan)) } + level = zerolog.DebugLevel l := zerolog. New(io.MultiWriter(exporters...)). With(). Timestamp(). Logger(). - Level(zerolog.DebugLevel) + Level(level) return &l } diff --git a/packages/envd/pkg/version.go b/packages/envd/pkg/version.go index 3f77a570c7..9bbb0c85e9 100644 --- a/packages/envd/pkg/version.go +++ b/packages/envd/pkg/version.go @@ -1,3 +1,3 @@ package pkg -const Version = "0.5.14" +const Version = "0.5.15" diff --git a/packages/orchestrator/cmd/resume-build/measure-memory-dirtying.sh b/packages/orchestrator/cmd/resume-build/measure-memory-dirtying.sh new file mode 100755 index 0000000000..0b179b05d9 --- /dev/null +++ b/packages/orchestrator/cmd/resume-build/measure-memory-dirtying.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Measure memory/rootfs diff sizes across pause-resume cycles. +# +# This script builds a base template, then runs a series of pause-resume +# cycles to measure how many pages get dirtied during each cycle — +# both idle and under normal envd operations (file writes, process starts). +# +# Usage: +# sudo ./measure-memory-dirtying.sh [storage-path] +# +# Requires: root, KVM, Docker, NBD, hugepages +set -euo pipefail + +STORAGE="${1:-.local-build}" +CREATE_BUILD="go run ./packages/orchestrator/cmd/create-build" +RESUME_BUILD="go run ./packages/orchestrator/cmd/resume-build" + +BASE_ID="measure-base-$(date +%s)" +echo "=== Step 1: Build base template ===" +$CREATE_BUILD \ + -to-build "$BASE_ID" \ + -storage "$STORAGE" \ + -hugepages \ + -v + +echo "" +echo "=== Step 2: Immediate pause (baseline — no activity) ===" +LAYER_IDLE="$BASE_ID-idle" +$RESUME_BUILD \ + -from-build "$BASE_ID" \ + -to-build "$LAYER_IDLE" \ + -storage "$STORAGE" \ + -pause + +echo "" +echo "=== Step 3: Resume + sleep 2s + pause (idle drift) ===" +LAYER_SLEEP="$LAYER_IDLE-sleep2" +$RESUME_BUILD \ + -from-build "$LAYER_IDLE" \ + -to-build "$LAYER_SLEEP" \ + -storage "$STORAGE" \ + -cmd-pause "sleep 2" + +echo "" +echo "=== Step 4: Resume + sleep 5s + pause (longer idle drift) ===" +LAYER_SLEEP5="$LAYER_SLEEP-sleep5" +$RESUME_BUILD \ + -from-build "$LAYER_SLEEP" \ + -to-build "$LAYER_SLEEP5" \ + -storage "$STORAGE" \ + -cmd-pause "sleep 5" + +echo "" +echo "=== Step 5: Resume + write files via envd + pause ===" +LAYER_WRITE="$LAYER_SLEEP5-write" +$RESUME_BUILD \ + -from-build "$LAYER_SLEEP5" \ + -to-build "$LAYER_WRITE" \ + -storage "$STORAGE" \ + -cmd-pause "dd if=/dev/urandom of=/tmp/testfile bs=1K count=64 2>/dev/null && echo written" + +echo "" +echo "=== Step 6: Resume + start process via envd + pause ===" +LAYER_PROC="$LAYER_WRITE-proc" +$RESUME_BUILD \ + -from-build "$LAYER_WRITE" \ + -to-build "$LAYER_PROC" \ + -storage "$STORAGE" \ + -cmd-pause "python3 -c 'print(sum(range(10000)))' || echo 'python not available, using echo'; echo done" + +echo "" +echo "=== Step 7: Multi-iteration pause benchmark (10x immediate pause) ===" +$RESUME_BUILD \ + -from-build "$LAYER_IDLE" \ + -storage "$STORAGE" \ + -pause \ + -iterations 10 + +echo "" +echo "=== Done ===" +echo "Compare the '📦 Artifacts' memfile/rootfs diff sizes above." +echo "Smaller diffs = fewer dirty pages = faster snapshot restore." diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/files/journald.conf.tpl b/packages/orchestrator/pkg/template/build/core/rootfs/files/journald.conf.tpl new file mode 100644 index 0000000000..68e3235ebc --- /dev/null +++ b/packages/orchestrator/pkg/template/build/core/rootfs/files/journald.conf.tpl @@ -0,0 +1,9 @@ +{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}} +{{ .WriteFile "etc/systemd/journald.conf.d/e2b.conf" 0o644 }} + +[Journal] +Storage=none +MaxLevelConsole=warning +MaxLevelKMsg=warning +MaxLevelWall=emerg +ForwardToSyslog=no diff --git a/packages/orchestrator/pkg/template/build/core/rootfs/rootfs_test.go b/packages/orchestrator/pkg/template/build/core/rootfs/rootfs_test.go index e07370c1f2..b5dca80e85 100644 --- a/packages/orchestrator/pkg/template/build/core/rootfs/rootfs_test.go +++ b/packages/orchestrator/pkg/template/build/core/rootfs/rootfs_test.go @@ -88,7 +88,7 @@ func TestAdditionalOCILayers(t *testing.T) { keysIter := maps.Keys(actualFiles) keys := slices.Collect(keysIter) - assert.Len(t, keys, 13) + assert.Len(t, keys, 14) assert.Equal(t, "e2b.local", actualFiles["etc/hostname"]) assert.Equal(t, "nameserver 8.8.8.8", actualFiles["etc/resolv.conf"]) @@ -101,5 +101,9 @@ func TestAdditionalOCILayers(t *testing.T) { WatchdogSec=0`) assert.Equal(t, disabledContent, actualFiles["etc/systemd/system/systemd-journald.service.d/override.conf"]) assert.Equal(t, disabledContent, actualFiles["etc/systemd/system/systemd-networkd.service.d/override.conf"]) + + journaldConf := strings.TrimSpace(actualFiles["etc/systemd/journald.conf.d/e2b.conf"]) + assert.Contains(t, journaldConf, "Storage=none") + assert.Contains(t, journaldConf, "MaxLevelConsole=warning") }) }