diff --git a/packages/orchestrator/pkg/template/build/core/filesystem/ext4.go b/packages/orchestrator/pkg/template/build/core/filesystem/ext4.go index 9ffe2aa626..0252763406 100644 --- a/packages/orchestrator/pkg/template/build/core/filesystem/ext4.go +++ b/packages/orchestrator/pkg/template/build/core/filesystem/ext4.go @@ -26,6 +26,12 @@ import ( var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/filesystem") +var ( + groupHeaderPattern = regexp.MustCompile(`(?m)^[ \t]*Group[ \t]+\d+:`) + groupFreeBlocksPattern = regexp.MustCompile(`(?m)^[ \t]+(\d+)[ \t]+free blocks?,`) + reservedBlocksPattern = regexp.MustCompile(`Reserved block count:\s+(\d+)`) +) + const ( // creates an inode for every bytes-per-inode byte of space on the disk inodesRatio = int64(4096) @@ -196,6 +202,8 @@ func GetFreeSpace(ctx context.Context, rootfsPath string, blockSize int64) (int6 defer statSpan.End() cmd := exec.CommandContext(ctx, "debugfs", "-R", "stats", rootfsPath) + // The parser below relies on debugfs's English field names. + cmd.Env = append(os.Environ(), "LC_ALL=C") var out bytes.Buffer cmd.Stdout = &out err := cmd.Run() @@ -206,7 +214,6 @@ func GetFreeSpace(ctx context.Context, rootfsPath string, blockSize int64) (int6 return 0, fmt.Errorf("error statting ext4: %w", err) } - // Extract block size and free blocks freeBlocks, err := parseFreeBlocks(output) if err != nil { return 0, fmt.Errorf("could not parse free blocks: %w", err) @@ -376,25 +383,38 @@ func LogMetadata(ctx context.Context, rootfsPath string, extraFields ...zap.Fiel logger.L().With(extraFields...).Debug(ctx, "tune2fs -l output", zap.String("path", rootfsPath), zap.String("output", string(output)), zap.Error(err)) } -// parseFreeBlocks extracts the "Free blocks:" value from debugfs output +// parseFreeBlocks returns the sum of the free-block counters stored in the +// block-group descriptors. e2fsck documents that the global count can be stale +// after an unclean unmount and normally rebuilds it from the group counts, but +// its journal_only path skips that repair: +// https://github.com/tytso/e2fsprogs/blob/v1.47.0/e2fsck/unix.c#L370-L441 +// debugfs prints each group from ext2fs_bg_free_blocks_count(), so those are the +// counters we must sum after replaying only the journal: +// https://github.com/tytso/e2fsprogs/blob/v1.47.0/debugfs/debugfs.c#L486-L501 func parseFreeBlocks(debugfsOutput string) (int64, error) { - re := regexp.MustCompile(`Free blocks:\s+(\d+)`) - matches := re.FindStringSubmatch(debugfsOutput) - if len(matches) < 2 { - return 0, errors.New("could not find free blocks in debugfs output") + groups := groupHeaderPattern.FindAllStringIndex(debugfsOutput, -1) + matches := groupFreeBlocksPattern.FindAllStringSubmatch(debugfsOutput, -1) + // Require one counter per group so truncated or unexpected debugfs output + // cannot silently undercount free space and trigger an incorrect resize. + if len(groups) == 0 || len(matches) != len(groups) { + return 0, fmt.Errorf("could not parse free blocks for every block group: found %d groups and %d counters", len(groups), len(matches)) } - freeBlocks, err := strconv.ParseInt(matches[1], 10, 64) - if err != nil { - return 0, fmt.Errorf("could not parse free blocks: %w", err) + + var freeBlocks int64 + for _, match := range matches { + groupFreeBlocks, err := strconv.ParseInt(match[1], 10, 64) + if err != nil { + return 0, fmt.Errorf("could not parse block-group free blocks: %w", err) + } + freeBlocks += groupFreeBlocks } return freeBlocks, nil } -// parseReservedBlocks extracts the "Reserved block count:" value from debugfs output +// parseReservedBlocks extracts the "Reserved block count:" value from debugfs output. func parseReservedBlocks(debugfsOutput string) (int64, error) { - re := regexp.MustCompile(`Reserved block count:\s+(\d+)`) - matches := re.FindStringSubmatch(debugfsOutput) + matches := reservedBlocksPattern.FindStringSubmatch(debugfsOutput) if len(matches) < 2 { return 0, errors.New("could not find reserved blocks in debugfs output") } diff --git a/packages/orchestrator/pkg/template/build/core/filesystem/ext4_test.go b/packages/orchestrator/pkg/template/build/core/filesystem/ext4_test.go index 63eeab19a7..86d7a29103 100644 --- a/packages/orchestrator/pkg/template/build/core/filesystem/ext4_test.go +++ b/packages/orchestrator/pkg/template/build/core/filesystem/ext4_test.go @@ -19,18 +19,33 @@ func TestParseFreeBlocks(t *testing.T) { wantErr bool }{ { - name: "standard debugfs output", - input: "Block count: 131072\nFree blocks: 120000\nFirst block: 0\n", - expected: 120000, + name: "sums group counters instead of stale superblock", + input: `Free blocks: 50000 + Group 0: block bitmap at 1, inode bitmap at 2, inode table at 3 + 28629 free blocks, 100 free inodes, 1 used directory + Group 1: block bitmap at 4, inode bitmap at 5, inode table at 6 + 28639 free blocks, 100 free inodes, 1 used directory +`, + expected: 57268, + }, + { + name: "supports singular block counter", + input: ` Group 0: block bitmap at 1, inode bitmap at 2, inode table at 3 + 1 free block, 1 free inode, 1 used directory +`, + expected: 1, }, { - name: "large block count", - input: "Free blocks: 999999999\n", - expected: 999999999, + name: "rejects incomplete group output", + input: ` Group 0: block bitmap at 1, inode bitmap at 2, inode table at 3 + 10 free blocks, 100 free inodes, 1 used directory + Group 1: block bitmap at 4, inode bitmap at 5, inode table at 6 +`, + wantErr: true, }, { - name: "missing free blocks", - input: "Block count: 131072\n", + name: "rejects global counter without block groups", + input: "Free blocks: 50000\n", wantErr: true, }, } diff --git a/packages/orchestrator/pkg/template/build/phases/ensurefreedisk/grow.go b/packages/orchestrator/pkg/template/build/phases/ensurefreedisk/grow.go index 4816537b5d..3dab1ed536 100644 --- a/packages/orchestrator/pkg/template/build/phases/ensurefreedisk/grow.go +++ b/packages/orchestrator/pkg/template/build/phases/ensurefreedisk/grow.go @@ -126,6 +126,11 @@ func (b *EnsureFreeDiskBuilder) measureFree( if _, err := filesystem.ReplayJournal(ctx, device.path); err != nil { return 0, fmt.Errorf("replay source journal: %w", err) } + // Make the recovered block-group metadata visible from the backend before + // debugfs reopens the device and reads its free-space counters. + if err := device.mnt.Flush(ctx); err != nil { + return 0, fmt.Errorf("flush recovered source journal: %w", err) + } free, err = filesystem.GetFreeSpace(ctx, device.path, blockSize) if err != nil { return 0, fmt.Errorf("measure source free space: %w", err)