From 87ea017fd4a286f6ec8d8bff1e0ce1c45b460de8 Mon Sep 17 00:00:00 2001 From: Giulio Micheloni Date: Mon, 13 Jul 2026 15:34:09 +0200 Subject: [PATCH] fix(orchestrator): measure ext4 free space from block groups The ensure-free-disk phase previously trusted debugfs's top-level "Free blocks" value, which comes from the ext4 superblock. After an unclean shutdown, Linux can leave this aggregate stale, and e2fsck -E journal_only returns before rebuilding it from the block-group descriptors. This could cause builds to make an incorrect resize decision and leave less free space than requested. Flush the NBD device after replaying the journal, then calculate free space by summing debugfs's per-group counters and subtracting reserved blocks. Require one counter for every group so incomplete or unexpected output fails instead of silently producing an incorrect result. Pin LC_ALL=C because the parser relies on debugfs's English field names. Add Linux unit coverage for stale global counters, multi-group sums, singular block output, incomplete group output, and global-only output. --- .../template/build/core/filesystem/ext4.go | 44 ++++++++++++++----- .../build/core/filesystem/ext4_test.go | 31 +++++++++---- .../build/phases/ensurefreedisk/grow.go | 5 +++ 3 files changed, 60 insertions(+), 20 deletions(-) diff --git a/packages/orchestrator/pkg/template/build/core/filesystem/ext4.go b/packages/orchestrator/pkg/template/build/core/filesystem/ext4.go index 9ffe2aa626..0252763406 100644 --- a/packages/orchestrator/pkg/template/build/core/filesystem/ext4.go +++ b/packages/orchestrator/pkg/template/build/core/filesystem/ext4.go @@ -26,6 +26,12 @@ import ( var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/filesystem") +var ( + groupHeaderPattern = regexp.MustCompile(`(?m)^[ \t]*Group[ \t]+\d+:`) + groupFreeBlocksPattern = regexp.MustCompile(`(?m)^[ \t]+(\d+)[ \t]+free blocks?,`) + reservedBlocksPattern = regexp.MustCompile(`Reserved block count:\s+(\d+)`) +) + const ( // creates an inode for every bytes-per-inode byte of space on the disk inodesRatio = int64(4096) @@ -196,6 +202,8 @@ func GetFreeSpace(ctx context.Context, rootfsPath string, blockSize int64) (int6 defer statSpan.End() cmd := exec.CommandContext(ctx, "debugfs", "-R", "stats", rootfsPath) + // The parser below relies on debugfs's English field names. + cmd.Env = append(os.Environ(), "LC_ALL=C") var out bytes.Buffer cmd.Stdout = &out err := cmd.Run() @@ -206,7 +214,6 @@ func GetFreeSpace(ctx context.Context, rootfsPath string, blockSize int64) (int6 return 0, fmt.Errorf("error statting ext4: %w", err) } - // Extract block size and free blocks freeBlocks, err := parseFreeBlocks(output) if err != nil { return 0, fmt.Errorf("could not parse free blocks: %w", err) @@ -376,25 +383,38 @@ func LogMetadata(ctx context.Context, rootfsPath string, extraFields ...zap.Fiel logger.L().With(extraFields...).Debug(ctx, "tune2fs -l output", zap.String("path", rootfsPath), zap.String("output", string(output)), zap.Error(err)) } -// parseFreeBlocks extracts the "Free blocks:" value from debugfs output +// parseFreeBlocks returns the sum of the free-block counters stored in the +// block-group descriptors. e2fsck documents that the global count can be stale +// after an unclean unmount and normally rebuilds it from the group counts, but +// its journal_only path skips that repair: +// https://github.com/tytso/e2fsprogs/blob/v1.47.0/e2fsck/unix.c#L370-L441 +// debugfs prints each group from ext2fs_bg_free_blocks_count(), so those are the +// counters we must sum after replaying only the journal: +// https://github.com/tytso/e2fsprogs/blob/v1.47.0/debugfs/debugfs.c#L486-L501 func parseFreeBlocks(debugfsOutput string) (int64, error) { - re := regexp.MustCompile(`Free blocks:\s+(\d+)`) - matches := re.FindStringSubmatch(debugfsOutput) - if len(matches) < 2 { - return 0, errors.New("could not find free blocks in debugfs output") + groups := groupHeaderPattern.FindAllStringIndex(debugfsOutput, -1) + matches := groupFreeBlocksPattern.FindAllStringSubmatch(debugfsOutput, -1) + // Require one counter per group so truncated or unexpected debugfs output + // cannot silently undercount free space and trigger an incorrect resize. + if len(groups) == 0 || len(matches) != len(groups) { + return 0, fmt.Errorf("could not parse free blocks for every block group: found %d groups and %d counters", len(groups), len(matches)) } - freeBlocks, err := strconv.ParseInt(matches[1], 10, 64) - if err != nil { - return 0, fmt.Errorf("could not parse free blocks: %w", err) + + var freeBlocks int64 + for _, match := range matches { + groupFreeBlocks, err := strconv.ParseInt(match[1], 10, 64) + if err != nil { + return 0, fmt.Errorf("could not parse block-group free blocks: %w", err) + } + freeBlocks += groupFreeBlocks } return freeBlocks, nil } -// parseReservedBlocks extracts the "Reserved block count:" value from debugfs output +// parseReservedBlocks extracts the "Reserved block count:" value from debugfs output. func parseReservedBlocks(debugfsOutput string) (int64, error) { - re := regexp.MustCompile(`Reserved block count:\s+(\d+)`) - matches := re.FindStringSubmatch(debugfsOutput) + matches := reservedBlocksPattern.FindStringSubmatch(debugfsOutput) if len(matches) < 2 { return 0, errors.New("could not find reserved blocks in debugfs output") } diff --git a/packages/orchestrator/pkg/template/build/core/filesystem/ext4_test.go b/packages/orchestrator/pkg/template/build/core/filesystem/ext4_test.go index 63eeab19a7..86d7a29103 100644 --- a/packages/orchestrator/pkg/template/build/core/filesystem/ext4_test.go +++ b/packages/orchestrator/pkg/template/build/core/filesystem/ext4_test.go @@ -19,18 +19,33 @@ func TestParseFreeBlocks(t *testing.T) { wantErr bool }{ { - name: "standard debugfs output", - input: "Block count: 131072\nFree blocks: 120000\nFirst block: 0\n", - expected: 120000, + name: "sums group counters instead of stale superblock", + input: `Free blocks: 50000 + Group 0: block bitmap at 1, inode bitmap at 2, inode table at 3 + 28629 free blocks, 100 free inodes, 1 used directory + Group 1: block bitmap at 4, inode bitmap at 5, inode table at 6 + 28639 free blocks, 100 free inodes, 1 used directory +`, + expected: 57268, + }, + { + name: "supports singular block counter", + input: ` Group 0: block bitmap at 1, inode bitmap at 2, inode table at 3 + 1 free block, 1 free inode, 1 used directory +`, + expected: 1, }, { - name: "large block count", - input: "Free blocks: 999999999\n", - expected: 999999999, + name: "rejects incomplete group output", + input: ` Group 0: block bitmap at 1, inode bitmap at 2, inode table at 3 + 10 free blocks, 100 free inodes, 1 used directory + Group 1: block bitmap at 4, inode bitmap at 5, inode table at 6 +`, + wantErr: true, }, { - name: "missing free blocks", - input: "Block count: 131072\n", + name: "rejects global counter without block groups", + input: "Free blocks: 50000\n", wantErr: true, }, } diff --git a/packages/orchestrator/pkg/template/build/phases/ensurefreedisk/grow.go b/packages/orchestrator/pkg/template/build/phases/ensurefreedisk/grow.go index 4816537b5d..3dab1ed536 100644 --- a/packages/orchestrator/pkg/template/build/phases/ensurefreedisk/grow.go +++ b/packages/orchestrator/pkg/template/build/phases/ensurefreedisk/grow.go @@ -126,6 +126,11 @@ func (b *EnsureFreeDiskBuilder) measureFree( if _, err := filesystem.ReplayJournal(ctx, device.path); err != nil { return 0, fmt.Errorf("replay source journal: %w", err) } + // Make the recovered block-group metadata visible from the backend before + // debugfs reopens the device and reads its free-space counters. + if err := device.mnt.Flush(ctx); err != nil { + return 0, fmt.Errorf("flush recovered source journal: %w", err) + } free, err = filesystem.GetFreeSpace(ctx, device.path, blockSize) if err != nil { return 0, fmt.Errorf("measure source free space: %w", err)