From b0f58ff6b6d44c36f3b935d244fd7175a2d6daff Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Thu, 11 Jun 2026 20:25:35 +0800 Subject: [PATCH 1/3] Wire disk-pressure self-heal into the Tentacle host lifecycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The self-heal scaffolding (SelfHealController, DiskPressureHealAction, DefaultWorkspaceCleanupPolicy, and their tests) was built but never scheduled — TentacleFlavor returned BackgroundTasks = [], so a disk-full agent failed every deployment via LocalScriptService's admission gate with no auto-reclaim of completed-script workspaces. Wire it: - WorkspaceProbe enumerates {tempRoot}/squid-tentacle-{ticketId} dirs and classifies each from its persisted ScriptState — Complete+0 => Succeeded, Complete+nonzero => Failed, started-not-complete => Active, no state => Unknown — plus recursive size + last-write, so the cleanup policy can pick oldest-evictable workspaces while protecting a retention window. - SelfHealBackgroundTask adapts SelfHealController to the host's ITentacleBackgroundTask lifecycle (Start on launch, drain on shutdown); ForLocalWorkspaces builds the disk-pressure action over the temp root with the running script backend as the live-workspace veto. - TentacleFlavor now schedules it (BackgroundTasks = [selfHeal]). The running-script reporter veto means an in-flight deployment's workspace is never swept; only completed/orphaned workspaces are reclaimed, and only under disk pressure. Tests: - WorkspaceProbeTests: real temp dirs + real ScriptState files — status classification matrix, prefix filtering, size, missing-root - SelfHealBackgroundTaskTests: RunAsync starts the heal loop + drains on cancel - TentacleFlavorSelfHealWiringTests: the flavor schedules a SelfHeal task (the dead-code-is-now-live regression guard) --- .../Flavors/Tentacle/TentacleFlavor.cs | 8 +- .../SelfHeal/SelfHealBackgroundTask.cs | 66 +++++++++ src/Squid.Tentacle/SelfHeal/WorkspaceProbe.cs | 104 +++++++++++++ .../TentacleFlavorSelfHealWiringTests.cs | 39 +++++ .../SelfHeal/SelfHealBackgroundTaskTests.cs | 62 ++++++++ .../SelfHeal/WorkspaceProbeTests.cs | 140 ++++++++++++++++++ 6 files changed, 418 insertions(+), 1 deletion(-) create mode 100644 src/Squid.Tentacle/SelfHeal/SelfHealBackgroundTask.cs create mode 100644 src/Squid.Tentacle/SelfHeal/WorkspaceProbe.cs create mode 100644 tests/Squid.Tentacle.Tests/Flavors/Tentacle/TentacleFlavorSelfHealWiringTests.cs create mode 100644 tests/Squid.Tentacle.Tests/SelfHeal/SelfHealBackgroundTaskTests.cs create mode 100644 tests/Squid.Tentacle.Tests/SelfHeal/WorkspaceProbeTests.cs diff --git a/src/Squid.Tentacle/Flavors/Tentacle/TentacleFlavor.cs b/src/Squid.Tentacle/Flavors/Tentacle/TentacleFlavor.cs index 592172e8..da448c4d 100644 --- a/src/Squid.Tentacle/Flavors/Tentacle/TentacleFlavor.cs +++ b/src/Squid.Tentacle/Flavors/Tentacle/TentacleFlavor.cs @@ -3,6 +3,7 @@ using Squid.Tentacle.Configuration; using Squid.Tentacle.Flavors.Tentacle.Configuration; using Squid.Tentacle.ScriptExecution; +using Squid.Tentacle.SelfHeal; using Serilog; namespace Squid.Tentacle.Flavors.Tentacle; @@ -41,7 +42,12 @@ public TentacleFlavorRuntime CreateRuntime(TentacleFlavorContext context) ScriptBackend = backend, CommunicationMode = communicationMode, ListeningPort = settings.ListeningPort, - BackgroundTasks = [], + // Disk-pressure self-heal: reclaim completed-script workspaces when the + // temp disk runs low, so a flood of deployments can't fill the disk and + // turn every subsequent deploy into a disk-full failure. The backend is + // the running-script reporter, so an in-flight deployment's workspace is + // never swept. + BackgroundTasks = [SelfHealBackgroundTask.ForLocalWorkspaces(backend)], StartupHooks = [], ReadinessCheck = null, Metadata = new Dictionary diff --git a/src/Squid.Tentacle/SelfHeal/SelfHealBackgroundTask.cs b/src/Squid.Tentacle/SelfHeal/SelfHealBackgroundTask.cs new file mode 100644 index 00000000..c3d98291 --- /dev/null +++ b/src/Squid.Tentacle/SelfHeal/SelfHealBackgroundTask.cs @@ -0,0 +1,66 @@ +using Squid.Tentacle.Abstractions; +using Squid.Tentacle.ScriptExecution; +using Squid.Tentacle.ScriptExecution.State; + +namespace Squid.Tentacle.SelfHeal; + +/// +/// Adapts to the host's +/// lifecycle: starts the heal loops when the +/// host launches background tasks and drains them when the host's cancellation +/// token trips on shutdown. This is the wiring that takes the self-heal +/// scaffolding from dead code to live — before it, no flavor scheduled any heal +/// action, so a disk-full agent failed deployments with no auto-reclaim. +/// +public sealed class SelfHealBackgroundTask : ITentacleBackgroundTask, IAsyncDisposable +{ + private readonly SelfHealController _controller; + + public SelfHealBackgroundTask(SelfHealController controller) + { + _controller = controller ?? throw new ArgumentNullException(nameof(controller)); + } + + public string Name => "SelfHeal"; + + public async Task RunAsync(CancellationToken ct) + { + _controller.Start(); + + try + { + await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + // Expected on shutdown. + } + finally + { + await _controller.DisposeAsync().ConfigureAwait(false); + } + } + + public ValueTask DisposeAsync() => _controller.DisposeAsync(); + + /// + /// Builds the disk-pressure self-heal for the local script workspaces that + /// creates under the temp root + /// ({tempRoot}/squid-tentacle-{ticketId}). The + /// (the live script backend) vetoes deletion of any workspace still running a + /// script, so the sweep can never remove an in-flight deployment's directory. + /// + public static SelfHealBackgroundTask ForLocalWorkspaces(IRunningScriptReporter reporter) + { + var stateStoreFactory = new ScriptStateStoreFactory(); + + var action = new DiskPressureHealAction( + workspaceRootProvider: Path.GetTempPath, + candidateProbe: root => WorkspaceProbe.Probe(root, stateStoreFactory), + policy: new DefaultWorkspaceCleanupPolicy(), + removeWorkspace: path => Directory.Delete(path, recursive: true), + runningScriptReporters: reporter == null ? null : new[] { reporter }); + + return new SelfHealBackgroundTask(new SelfHealController(new ISelfHealAction[] { action })); + } +} diff --git a/src/Squid.Tentacle/SelfHeal/WorkspaceProbe.cs b/src/Squid.Tentacle/SelfHeal/WorkspaceProbe.cs new file mode 100644 index 00000000..5273a8c2 --- /dev/null +++ b/src/Squid.Tentacle/SelfHeal/WorkspaceProbe.cs @@ -0,0 +1,104 @@ +using Serilog; +using Squid.Tentacle.ScriptExecution.State; + +namespace Squid.Tentacle.SelfHeal; + +/// +/// Enumerates completed-script workspace directories under a root and classifies +/// each one for the disk-pressure heal sweep: maps the persisted +/// to a , measures the +/// directory's size + last-write time, and ignores the root itself and any +/// directory that does not follow the squid-tentacle-{ticketId} +/// convention. It is pure of pressure/eviction logic — the +/// decides what to evict; this only reports +/// what exists, so the heal action and the policy stay independently testable. +/// +public static class WorkspaceProbe +{ + // Matches LocalScriptService.ResolveWorkDir: {tempRoot}/squid-tentacle-{ticketId}. + internal const string WorkspacePrefix = "squid-tentacle-"; + + public static IReadOnlyList Probe(string root, IScriptStateStoreFactory stateStoreFactory) + { + if (string.IsNullOrWhiteSpace(root) || !Directory.Exists(root)) + return Array.Empty(); + + var candidates = new List(); + + foreach (var dir in SafeEnumerateDirectories(root)) + { + var name = Path.GetFileName(dir); + if (string.IsNullOrEmpty(name) || !name.StartsWith(WorkspacePrefix, StringComparison.Ordinal)) + continue; + + try + { + var status = ClassifyStatus(dir, stateStoreFactory); + var lastModified = new DateTimeOffset(Directory.GetLastWriteTimeUtc(dir), TimeSpan.Zero); + var size = ComputeSizeBytes(dir); + + candidates.Add(new WorkspaceCandidate(dir, lastModified, size, status)); + } + catch (Exception ex) + { + // A workspace that vanished mid-probe or is momentarily locked must + // not abort the whole sweep — skip it; the next tick re-evaluates. + Log.Debug(ex, "[SelfHeal] Skipping unreadable workspace {Path}", dir); + } + } + + return candidates; + } + + /// + /// Maps the persisted state to a heal status. A workspace with no readable + /// state file is (treated as evictable + /// under pressure but kept out of the per-status retention windows); a + /// started-but-not-complete script is so + /// the policy never evicts it (the running-script-reporter veto in + /// is the authoritative second guard). + /// + private static WorkspaceStatus ClassifyStatus(string workDir, IScriptStateStoreFactory stateStoreFactory) + { + var store = stateStoreFactory.Create(workDir); + + if (!store.Exists()) + return WorkspaceStatus.Unknown; + + var state = store.Load(); + + if (!state.IsComplete()) + return state.HasStarted() ? WorkspaceStatus.Active : WorkspaceStatus.Unknown; + + return state.ExitCode == 0 ? WorkspaceStatus.Succeeded : WorkspaceStatus.Failed; + } + + private static long ComputeSizeBytes(string dir) + { + long total = 0; + + foreach (var file in SafeEnumerateFiles(dir)) + { + try { total += new FileInfo(file).Length; } + catch { /* file removed mid-walk — ignore */ } + } + + return total; + } + + private static IEnumerable SafeEnumerateDirectories(string root) + { + try { return Directory.EnumerateDirectories(root); } + catch (Exception ex) + { + Log.Debug(ex, "[SelfHeal] Could not enumerate workspace root {Root}", root); + return Array.Empty(); + } + } + + private static IEnumerable SafeEnumerateFiles(string dir) + { + try { return Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories); } + catch { return Array.Empty(); } + } +} diff --git a/tests/Squid.Tentacle.Tests/Flavors/Tentacle/TentacleFlavorSelfHealWiringTests.cs b/tests/Squid.Tentacle.Tests/Flavors/Tentacle/TentacleFlavorSelfHealWiringTests.cs new file mode 100644 index 00000000..b4cd1d53 --- /dev/null +++ b/tests/Squid.Tentacle.Tests/Flavors/Tentacle/TentacleFlavorSelfHealWiringTests.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.Configuration; +using Shouldly; +using Squid.Tentacle.Abstractions; +using Squid.Tentacle.Configuration; +using Squid.Tentacle.Flavors.Tentacle; +using Squid.Tentacle.SelfHeal; +using Squid.Tentacle.Tests.Support; +using Xunit; + +namespace Squid.Tentacle.Tests.Flavors.Tentacle; + +/// +/// Pins the wiring that took the disk self-heal from dead code to live: the +/// regular Tentacle flavor must schedule a as +/// a host background task. Before this, TentacleFlavor returned +/// BackgroundTasks = [], so a disk-full agent failed deployments with no +/// auto-reclaim even though the whole heal mechanism existed. +/// +[Trait("Category", TentacleTestCategories.Core)] +public sealed class TentacleFlavorSelfHealWiringTests +{ + [Fact] + public void CreateRuntime_SchedulesSelfHealBackgroundTask() + { + var context = new TentacleFlavorContext + { + TentacleSettings = new TentacleSettings(), + Configuration = new ConfigurationBuilder().Build() + }; + + var runtime = new TentacleFlavor().CreateRuntime(context); + + var selfHeal = runtime.BackgroundTasks.OfType().SingleOrDefault(); + + selfHeal.ShouldNotBeNull( + customMessage: "TentacleFlavor must schedule the disk self-heal background task — otherwise the heal mechanism stays dead code and a disk-full agent fails deployments with no auto-reclaim."); + selfHeal.Name.ShouldBe("SelfHeal"); + } +} diff --git a/tests/Squid.Tentacle.Tests/SelfHeal/SelfHealBackgroundTaskTests.cs b/tests/Squid.Tentacle.Tests/SelfHeal/SelfHealBackgroundTaskTests.cs new file mode 100644 index 00000000..8bfa9794 --- /dev/null +++ b/tests/Squid.Tentacle.Tests/SelfHeal/SelfHealBackgroundTaskTests.cs @@ -0,0 +1,62 @@ +using Shouldly; +using Squid.Tentacle.SelfHeal; +using Squid.Tentacle.Tests.Support; +using Xunit; + +namespace Squid.Tentacle.Tests.SelfHeal; + +/// +/// Pins — the adapter that schedules the +/// self-heal loops on the host's ITentacleBackgroundTask lifecycle. This is +/// the wiring that takes the heal scaffolding from dead code to live; the test +/// proves the controller's actions actually fire under RunAsync and that +/// cancellation drains cleanly (so host shutdown doesn't hang). +/// +[Trait("Category", TentacleTestCategories.Core)] +public sealed class SelfHealBackgroundTaskTests +{ + [Fact] + public void ForLocalWorkspaces_ProducesNamedSelfHealTask() + { + var task = SelfHealBackgroundTask.ForLocalWorkspaces(reporter: null); + + task.Name.ShouldBe("SelfHeal"); + } + + [Fact] + public async Task RunAsync_StartsHealActions_AndDrainsOnCancellation() + { + var fired = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var task = new SelfHealBackgroundTask(new SelfHealController(new ISelfHealAction[] { new ProbeAction(fired) })); + + using var cts = new CancellationTokenSource(); + var run = task.RunAsync(cts.Token); + + // The action loop must actually run — proves Start() was invoked. + await fired.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + cts.Cancel(); + + // RunAsync must return promptly on cancellation (drains the controller), + // never hang the host shutdown. + await run.WaitAsync(TimeSpan.FromSeconds(5)); + run.IsCompletedSuccessfully.ShouldBeTrue(); + } + + private sealed class ProbeAction : ISelfHealAction + { + private readonly TaskCompletionSource _fired; + + public ProbeAction(TaskCompletionSource fired) => _fired = fired; + + public string Name => "probe"; + + public TimeSpan CheckInterval => TimeSpan.FromMilliseconds(10); + + public Task RunAsync(CancellationToken ct) + { + _fired.TrySetResult(); + return Task.FromResult(SelfHealOutcome.Healthy(Name)); + } + } +} diff --git a/tests/Squid.Tentacle.Tests/SelfHeal/WorkspaceProbeTests.cs b/tests/Squid.Tentacle.Tests/SelfHeal/WorkspaceProbeTests.cs new file mode 100644 index 00000000..b4b36985 --- /dev/null +++ b/tests/Squid.Tentacle.Tests/SelfHeal/WorkspaceProbeTests.cs @@ -0,0 +1,140 @@ +using Shouldly; +using Squid.Tentacle.ScriptExecution.State; +using Squid.Tentacle.SelfHeal; +using Squid.Tentacle.Tests.Support; +using Xunit; + +namespace Squid.Tentacle.Tests.SelfHeal; + +/// +/// Pins — the enumeration + status classification +/// that turns on-disk script workspaces into s for +/// the disk-pressure heal sweep. This is the wiring glue between the persisted +/// (Progress + ExitCode) and the cleanup policy; the +/// status mapping decides what the per-status retention windows protect. +/// +[Trait("Category", TentacleTestCategories.Core)] +public sealed class WorkspaceProbeTests : IDisposable +{ + private readonly string _root = Path.Combine(Path.GetTempPath(), $"squid-probe-test-{Guid.NewGuid():N}"); + private readonly ScriptStateStoreFactory _factory = new(); + + public WorkspaceProbeTests() => Directory.CreateDirectory(_root); + + public void Dispose() + { + try { if (Directory.Exists(_root)) Directory.Delete(_root, recursive: true); } + catch { /* best-effort */ } + } + + private string MakeWorkspace(string ticketId, ScriptState? state = null, string? content = null) + { + var dir = Path.Combine(_root, $"squid-tentacle-{ticketId}"); + Directory.CreateDirectory(dir); + + if (content != null) + File.WriteAllText(Path.Combine(dir, "output.log"), content); + + if (state != null) + _factory.Create(dir).Save(state); + + return dir; + } + + private static ScriptState Complete(string ticketId, int exitCode) => new() + { + TicketId = ticketId, + Progress = ScriptProgress.Complete, + ExitCode = exitCode, + CreatedAt = DateTimeOffset.UtcNow, + CompletedAt = DateTimeOffset.UtcNow + }; + + private static ScriptState Running(string ticketId) => new() + { + TicketId = ticketId, + Progress = ScriptProgress.Running, + CreatedAt = DateTimeOffset.UtcNow + }; + + [Fact] + public void Probe_CompleteExitZero_IsSucceeded() + { + MakeWorkspace("ok", Complete("ok", exitCode: 0)); + + var c = WorkspaceProbe.Probe(_root, _factory).ShouldHaveSingleItem(); + + c.Status.ShouldBe(WorkspaceStatus.Succeeded); + } + + [Fact] + public void Probe_CompleteNonZero_IsFailed() + { + MakeWorkspace("bad", Complete("bad", exitCode: 1)); + + WorkspaceProbe.Probe(_root, _factory).ShouldHaveSingleItem() + .Status.ShouldBe(WorkspaceStatus.Failed); + } + + [Fact] + public void Probe_RunningScript_IsActive_SoThePolicyNeverEvictsIt() + { + MakeWorkspace("live", Running("live")); + + WorkspaceProbe.Probe(_root, _factory).ShouldHaveSingleItem() + .Status.ShouldBe(WorkspaceStatus.Active); + } + + [Fact] + public void Probe_NoStateFile_IsUnknown() + { + MakeWorkspace("orphan"); // dir exists, no .squid-state.json + + WorkspaceProbe.Probe(_root, _factory).ShouldHaveSingleItem() + .Status.ShouldBe(WorkspaceStatus.Unknown); + } + + [Fact] + public void Probe_IgnoresDirectoriesNotMatchingTheWorkspacePrefix() + { + Directory.CreateDirectory(Path.Combine(_root, "some-other-dir")); + Directory.CreateDirectory(Path.Combine(_root, "calamari-cache")); + MakeWorkspace("real", Complete("real", 0)); + + var candidates = WorkspaceProbe.Probe(_root, _factory); + + candidates.ShouldHaveSingleItem() + .Path.ShouldEndWith("squid-tentacle-real"); + } + + [Fact] + public void Probe_MeasuresWorkspaceSize() + { + var payload = new string('x', 4096); + MakeWorkspace("sized", Complete("sized", 0), content: payload); + + var c = WorkspaceProbe.Probe(_root, _factory).ShouldHaveSingleItem(); + + c.SizeBytes.ShouldBeGreaterThanOrEqualTo(4096, + customMessage: "Size must include the workspace's files so the policy can reclaim enough space."); + } + + [Fact] + public void Probe_MissingRoot_ReturnsEmpty() + => WorkspaceProbe.Probe(Path.Combine(_root, "does-not-exist"), _factory).ShouldBeEmpty(); + + [Fact] + public void Probe_MultipleWorkspaces_ClassifiesEachIndependently() + { + MakeWorkspace("s1", Complete("s1", 0)); + MakeWorkspace("f1", Complete("f1", 2)); + MakeWorkspace("r1", Running("r1")); + + var byStatus = WorkspaceProbe.Probe(_root, _factory) + .ToLookup(c => c.Status); + + byStatus[WorkspaceStatus.Succeeded].Count().ShouldBe(1); + byStatus[WorkspaceStatus.Failed].Count().ShouldBe(1); + byStatus[WorkspaceStatus.Active].Count().ShouldBe(1); + } +} From a6d3fa4545758eb0acd2e21a34ca606bd5ad15d3 Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Thu, 11 Jun 2026 21:57:01 +0800 Subject: [PATCH 2/3] Harden disk self-heal: retention TTL floor, symlink-safe sizing, corrupt-state reclaim, backoff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wiring that takes the disk self-heal live (the 30s DiskPressureHealAction over {tempRoot}/squid-tentacle-*) is a second sweeper over the same directories the 24h orphan sweep owns, and the probe/policy had several robustness gaps. Address them before the sweep runs in production: - Honour the operator's orphan-workspace TTL as a retention floor: under mild (non-critical) pressure the new sweep no longer evicts inside the post-mortem window the existing age-based orphan sweep promises; under critical pressure the floor is bypassed so reclaiming disk still wins. - Protect a just-created workspace via a 60s fresh-grace floor (even under critical pressure), closing the StartScript TOCTOU gap between CreateDirectory and the first state Save / running-script registration. - Classify a present-but-corrupt state file as Unknown instead of skipping the workspace forever — otherwise a dead-but-corrupt workspace is permanently un-reclaimable, exactly when disk pressure most needs the space. - Skip reparse points in the recursive size walk so a directory symlink (extracted package or runtime ln -s) cannot loop the enumeration or mis-attribute a symlink target's size to the workspace. - Warn once + back off exponentially when a sweep reclaims everything it is allowed to but the disk stays under pressure (non-workspace usage), instead of churning every 30s. - Expose the retention counts + low-disk trigger behind env vars with pinned const names (SelfHealOptions), mirroring the orphan-TTL env-var pattern. All floors default off in DefaultWorkspaceCleanupPolicy/DiskPressure so the parameterless constructors behave exactly as before; only the live ForLocalWorkspaces wiring injects them — non-breaking. Tests: SelfHealOptions env-var/default pins; corrupt-state, recursive nested-size and symlink-safety probe cases; non-critical-TTL + critical-bypass + fresh-grace policy cases; backoff growth/cap/reset; and a high-fidelity wired-chain E2E (real probe + policy + ScriptStateStore + Directory.Delete) proving real reclaim while a Running workspace and the in-TTL window survive. --- .../SelfHeal/DiskPressureHealAction.cs | 84 +++++++++-- .../SelfHeal/IWorkspaceCleanupPolicy.cs | 64 +++++++- .../SelfHeal/SelfHealBackgroundTask.cs | 25 +++- .../SelfHeal/SelfHealOptions.cs | 107 ++++++++++++++ src/Squid.Tentacle/SelfHeal/WorkspaceProbe.cs | 34 ++++- .../SelfHeal/DiskPressureHealActionTests.cs | 59 ++++++++ .../SelfHeal/DiskSelfHealWiredE2ETests.cs | 137 ++++++++++++++++++ .../SelfHeal/SelfHealOptionsTests.cs | 75 ++++++++++ .../SelfHeal/WorkspaceCleanupPolicyTests.cs | 56 +++++++ .../SelfHeal/WorkspaceProbeTests.cs | 72 ++++++++- 10 files changed, 688 insertions(+), 25 deletions(-) create mode 100644 src/Squid.Tentacle/SelfHeal/SelfHealOptions.cs create mode 100644 tests/Squid.Tentacle.Tests/SelfHeal/DiskSelfHealWiredE2ETests.cs create mode 100644 tests/Squid.Tentacle.Tests/SelfHeal/SelfHealOptionsTests.cs diff --git a/src/Squid.Tentacle/SelfHeal/DiskPressureHealAction.cs b/src/Squid.Tentacle/SelfHeal/DiskPressureHealAction.cs index 70afedf6..7784321c 100644 --- a/src/Squid.Tentacle/SelfHeal/DiskPressureHealAction.cs +++ b/src/Squid.Tentacle/SelfHeal/DiskPressureHealAction.cs @@ -30,6 +30,16 @@ public sealed class DiskPressureHealAction : ISelfHealAction private readonly TicketExtractor _ticketExtractor; private readonly RetentionQuota _quota; + // Backoff state: when a sweep reclaims everything it is allowed to but the + // disk is STILL under pressure (non-workspace usage, or everything protected + // by the retention window), re-running every CheckInterval just churns. We + // back off exponentially up to _maxInterval and warn once per episode, then + // reset the moment pressure clears. + private readonly TimeSpan _baseInterval; + private readonly TimeSpan _maxInterval; + private TimeSpan _currentInterval; + private bool _underPressureWarned; + public DiskPressureHealAction( Func workspaceRootProvider, Func> candidateProbe, @@ -49,7 +59,10 @@ public DiskPressureHealAction( _runningScriptReporters = runningScriptReporters?.ToList() ?? new List(); _ticketExtractor = ticketExtractor ?? DefaultTicketExtractor; _quota = quota ?? RetentionQuota.Default; - CheckInterval = checkInterval ?? TimeSpan.FromSeconds(30); + + _baseInterval = checkInterval ?? TimeSpan.FromSeconds(30); + _maxInterval = TimeSpan.FromTicks(_baseInterval.Ticks * 16); + _currentInterval = _baseInterval; } private static string? DefaultTicketExtractor(string workspacePath) @@ -83,12 +96,13 @@ private bool IsLiveScript(string workspacePath) public string Name => "disk-pressure-cleanup"; - public TimeSpan CheckInterval { get; } + public TimeSpan CheckInterval => _currentInterval; private static DiskPressure DefaultDiskProbe(string path) { var (available, total) = DiskSpaceChecker.GetDiskSpace(path); - return new DiskPressure(available, total); + return new DiskPressure(available, total, + SelfHealOptions.Default.LowFreePercentage, SelfHealOptions.Default.CriticalFreePercentage); } public Task RunAsync(CancellationToken ct) @@ -102,20 +116,46 @@ public Task RunAsync(CancellationToken ct) return Task.FromResult(SelfHealOutcome.Healthy(Name)); if (!pressure.IsLow) + { + ResetBackoff(); return Task.FromResult(SelfHealOutcome.Healthy(Name)); - - var candidates = _candidateProbe(workspaceRoot); + } // Veto candidates that a script backend still reports as live. This is the // race-safe cleanup guarantee: even if the workspace's Output.log looks // stale, we never delete a ticket that is still being tracked in memory. - candidates = candidates.Where(c => !IsLiveScript(c.Path)).ToList(); + var candidates = _candidateProbe(workspaceRoot) + .Where(c => !IsLiveScript(c.Path)) + .ToList(); var toRemove = _policy.SelectForRemoval(candidates, pressure, _quota); - if (toRemove.Count == 0) - return Task.FromResult(SelfHealOutcome.Healthy(Name)); + var (freed, removed) = RemoveWorkspaces(toRemove, ct); + + // Re-measure only if we actually freed something; otherwise the pressure is unchanged. + var post = removed > 0 ? _diskProbe(workspaceRoot) : pressure; + + if (!post.IsLow) + { + ResetBackoff(); + return Task.FromResult(SelfHealOutcome.RepairPerformed(Name, Summary(pressure, removed, freed))); + } + + // Reclaimed everything we were allowed to, but the disk is STILL under + // pressure — remaining usage is protected by the retention window or is not + // workspace-driven. Warn once per episode + back off so we don't churn the + // disk every tick (and re-delete a freshly-completed workspace the instant + // it ages past the keep-set). + WarnUnderPressureOnce(post, removed, freed); + BackOff(); + + return Task.FromResult(removed > 0 + ? SelfHealOutcome.RepairPerformed(Name, $"{Summary(pressure, removed, freed)}; still {post.FreePercentage:P1} free — backing off to {_currentInterval}") + : SelfHealOutcome.Healthy(Name)); + } + private (long Freed, int Removed) RemoveWorkspaces(IReadOnlyList toRemove, CancellationToken ct) + { var freed = 0L; var removed = 0; @@ -134,7 +174,31 @@ public Task RunAsync(CancellationToken ct) } } - return Task.FromResult(SelfHealOutcome.RepairPerformed(Name, - $"disk pressure {pressure.FreePercentage:P1} free — removed {removed} workspace(s), reclaimed {DiskSpaceChecker.FormatBytes(freed)}")); + return (freed, removed); + } + + private static string Summary(DiskPressure pressure, int removed, long freed) + => $"disk pressure {pressure.FreePercentage:P1} free — removed {removed} workspace(s), reclaimed {DiskSpaceChecker.FormatBytes(freed)}"; + + private void WarnUnderPressureOnce(DiskPressure post, int removed, long freed) + { + if (_underPressureWarned) return; + + _underPressureWarned = true; + Log.Warning("[SelfHeal] Disk still under pressure ({Free:P1} free) after reclaiming {Removed} workspace(s) ({Freed}) — " + + "remaining usage is protected by the retention window or is not workspace-driven. Backing off the heal sweep.", + post.FreePercentage, removed, DiskSpaceChecker.FormatBytes(freed)); + } + + private void BackOff() + { + var doubled = TimeSpan.FromTicks(Math.Min(_currentInterval.Ticks * 2, _maxInterval.Ticks)); + _currentInterval = doubled < _baseInterval ? _baseInterval : doubled; + } + + private void ResetBackoff() + { + _currentInterval = _baseInterval; + _underPressureWarned = false; } } diff --git a/src/Squid.Tentacle/SelfHeal/IWorkspaceCleanupPolicy.cs b/src/Squid.Tentacle/SelfHeal/IWorkspaceCleanupPolicy.cs index 7dd03f33..86a9e6e6 100644 --- a/src/Squid.Tentacle/SelfHeal/IWorkspaceCleanupPolicy.cs +++ b/src/Squid.Tentacle/SelfHeal/IWorkspaceCleanupPolicy.cs @@ -29,16 +29,20 @@ public enum WorkspaceStatus Unknown } -public sealed record DiskPressure(long FreeBytes, long TotalBytes) +public sealed record DiskPressure( + long FreeBytes, + long TotalBytes, + double LowFreePercentage = SelfHealOptions.DefaultLowFreePercentage, + double CriticalFreePercentage = SelfHealOptions.DefaultCriticalFreePercentage) { public double FreePercentage => TotalBytes > 0 ? (double)FreeBytes / TotalBytes : 0.0; - public bool IsLow => FreePercentage < 0.20; - public bool IsCritical => FreePercentage < 0.10; + public bool IsLow => FreePercentage < LowFreePercentage; + public bool IsCritical => FreePercentage < CriticalFreePercentage; } public sealed record RetentionQuota(int KeepLatestSucceeded, int KeepLatestFailed) { - public static RetentionQuota Default => new(KeepLatestSucceeded: 10, KeepLatestFailed: 20); + public static RetentionQuota Default => new(SelfHealOptions.DefaultKeepLatestSucceeded, SelfHealOptions.DefaultKeepLatestFailed); } /// @@ -47,10 +51,47 @@ public sealed record RetentionQuota(int KeepLatestSucceeded, int KeepLatestFaile /// - Not under disk pressure (IsLow == false): nothing to do. /// - Under pressure: keep the most recent K succeeded + M failed; evict the /// rest in oldest-first order until free space climbs back above the -/// low-pressure threshold (20% free, or 30% under critical pressure). +/// low-pressure target (the normal target equals the low threshold, raised +/// to criticalTargetFreePercentage under critical pressure). +/// +/// Two age floors keep the sweep safe and operator-friendly: +/// +/// Fresh-grace () protects a +/// workspace whose directory was written within the window — even under +/// critical pressure — so a deployment initialising a brand-new +/// workspace is never deleted out from under it (the TOCTOU gap before +/// the running-script reporter knows about the ticket). +/// Retention TTL (, the +/// operator's orphan-workspace TTL) protects recent completed +/// workspaces so the post-mortem window an operator pinned is honoured — +/// except under critical pressure, where reclaiming disk wins. +/// +/// Both default to zero so the parameterless policy keeps its original +/// no-floor behaviour; the live wiring (SelfHealBackgroundTask.ForLocalWorkspaces) +/// injects the real floors. /// public sealed class DefaultWorkspaceCleanupPolicy : IWorkspaceCleanupPolicy { + private readonly double _targetFreePercentage; + private readonly double _criticalTargetFreePercentage; + private readonly TimeSpan _minRetentionAge; + private readonly TimeSpan _freshGraceWindow; + private readonly Func _clock; + + public DefaultWorkspaceCleanupPolicy( + double targetFreePercentage = SelfHealOptions.DefaultLowFreePercentage, + double criticalTargetFreePercentage = SelfHealOptions.DefaultCriticalTargetFreePercentage, + TimeSpan minRetentionAge = default, + TimeSpan freshGraceWindow = default, + Func clock = null) + { + _targetFreePercentage = targetFreePercentage; + _criticalTargetFreePercentage = criticalTargetFreePercentage; + _minRetentionAge = minRetentionAge; + _freshGraceWindow = freshGraceWindow; + _clock = clock ?? (() => DateTimeOffset.UtcNow); + } + public IReadOnlyList SelectForRemoval( IReadOnlyList candidates, DiskPressure pressure, @@ -63,14 +104,18 @@ public IReadOnlyList SelectForRemoval( keep.UnionWith(LatestByStatus(candidates, WorkspaceStatus.Succeeded, quota.KeepLatestSucceeded)); keep.UnionWith(LatestByStatus(candidates, WorkspaceStatus.Failed, quota.KeepLatestFailed)); + var floor = EffectiveAgeFloor(pressure); + var now = _clock(); + var removable = candidates .Where(c => c.Status != WorkspaceStatus.Active && !keep.Contains(c.Path)) + .Where(c => now - c.LastModifiedUtc >= floor) // honour fresh-grace + retention TTL .OrderBy(c => c.LastModifiedUtc) // oldest first .ToList(); if (removable.Count == 0) return Array.Empty(); - var targetFreePct = pressure.IsCritical ? 0.30 : 0.20; + var targetFreePct = pressure.IsCritical ? _criticalTargetFreePercentage : _targetFreePercentage; var requiredBytes = (long)(pressure.TotalBytes * targetFreePct) - pressure.FreeBytes; if (requiredBytes <= 0) return Array.Empty(); @@ -88,6 +133,13 @@ public IReadOnlyList SelectForRemoval( return selected; } + // Under critical pressure only the short fresh-grace floor applies (reclaim + // disk wins over post-mortem retention); otherwise the longer of the two. + private TimeSpan EffectiveAgeFloor(DiskPressure pressure) + => pressure.IsCritical + ? _freshGraceWindow + : (_freshGraceWindow > _minRetentionAge ? _freshGraceWindow : _minRetentionAge); + private static IEnumerable LatestByStatus(IReadOnlyList candidates, WorkspaceStatus status, int count) => candidates .Where(c => c.Status == status) diff --git a/src/Squid.Tentacle/SelfHeal/SelfHealBackgroundTask.cs b/src/Squid.Tentacle/SelfHeal/SelfHealBackgroundTask.cs index c3d98291..a9085669 100644 --- a/src/Squid.Tentacle/SelfHeal/SelfHealBackgroundTask.cs +++ b/src/Squid.Tentacle/SelfHeal/SelfHealBackgroundTask.cs @@ -48,17 +48,38 @@ public async Task RunAsync(CancellationToken ct) /// creates under the temp root /// ({tempRoot}/squid-tentacle-{ticketId}). The /// (the live script backend) vetoes deletion of any workspace still running a - /// script, so the sweep can never remove an in-flight deployment's directory. + /// script, and the policy layers two age floors on top so the sweep can never + /// remove an in-flight deployment's directory: + /// + /// the protects a + /// just-created workspace before the reporter knows its ticket (the + /// StartScript TOCTOU gap), even under critical pressure; + /// (the operator's + /// orphan-workspace TTL) is honoured as a retention floor so the new + /// disk sweep does not silently override the post-mortem window the + /// existing age-based orphan sweep promises — except under critical + /// pressure, where reclaiming disk wins. + /// + /// Tunables (retention counts, low-disk trigger) come from + /// (env-var overridable). /// public static SelfHealBackgroundTask ForLocalWorkspaces(IRunningScriptReporter reporter) { var stateStoreFactory = new ScriptStateStoreFactory(); + var options = SelfHealOptions.Default; + + var policy = new DefaultWorkspaceCleanupPolicy( + targetFreePercentage: options.LowFreePercentage, + criticalTargetFreePercentage: options.CriticalTargetFreePercentage, + minRetentionAge: LocalScriptService.OrphanMaxAge, + freshGraceWindow: SelfHealOptions.FreshWorkspaceGraceWindow); var action = new DiskPressureHealAction( workspaceRootProvider: Path.GetTempPath, candidateProbe: root => WorkspaceProbe.Probe(root, stateStoreFactory), - policy: new DefaultWorkspaceCleanupPolicy(), + policy: policy, removeWorkspace: path => Directory.Delete(path, recursive: true), + quota: options.Quota, runningScriptReporters: reporter == null ? null : new[] { reporter }); return new SelfHealBackgroundTask(new SelfHealController(new ISelfHealAction[] { action })); diff --git a/src/Squid.Tentacle/SelfHeal/SelfHealOptions.cs b/src/Squid.Tentacle/SelfHeal/SelfHealOptions.cs new file mode 100644 index 00000000..1acc4df3 --- /dev/null +++ b/src/Squid.Tentacle/SelfHeal/SelfHealOptions.cs @@ -0,0 +1,107 @@ +using Serilog; + +namespace Squid.Tentacle.SelfHeal; + +/// +/// Operator-tunable knobs for the disk-pressure self-heal sweep, resolved once +/// from environment variables at process start (mirrors +/// LocalScriptService.OrphanMaxAge). Rule 8: anything an air-gapped / +/// fork operator might need to override lives behind an env var with a pinned +/// public const string name, and the literal defaults are pinned by a +/// unit test so a "harmless" change is a visible decision. +/// +/// Tunable via env var: the per-status retention counts (how many recent +/// succeeded / failed workspaces to keep for post-mortem) and the +/// low-disk trigger (also the normal reclaim target — reclaim back to the same +/// boundary that triggered the sweep). The critical thresholds are deliberately +/// immutable internal hysteresis; they are pinned by test, not exposed. +/// +public sealed record SelfHealOptions( + int KeepLatestSucceeded, + int KeepLatestFailed, + double LowFreePercentage, + double CriticalFreePercentage, + double CriticalTargetFreePercentage) +{ + // ── Env-var override surface (Rule 8 — pinned by SelfHealOptionsTests) ── + public const string KeepSucceededEnvVar = "SQUID_TENTACLE_SELFHEAL_KEEP_SUCCEEDED"; + public const string KeepFailedEnvVar = "SQUID_TENTACLE_SELFHEAL_KEEP_FAILED"; + public const string LowFreePercentageEnvVar = "SQUID_TENTACLE_SELFHEAL_LOW_FREE_PCT"; + + // ── Literal defaults (pinned by SelfHealOptionsTests) ── + public const int DefaultKeepLatestSucceeded = 10; + public const int DefaultKeepLatestFailed = 20; + public const double DefaultLowFreePercentage = 0.20; + public const double DefaultCriticalFreePercentage = 0.10; + public const double DefaultCriticalTargetFreePercentage = 0.30; + + // Bounds: keep-counts 0..10_000 (0 = keep nothing under pressure); free + // percentages strictly inside (0, 1). Out-of-range / unparseable input + // falls back to the default with a Serilog warning. + private const int MinKeepCount = 0; + private const int MaxKeepCount = 10_000; + + /// + /// Short safety floor: a workspace whose directory was last written within + /// this window is never evicted — even under critical pressure. It guards + /// the TOCTOU gap in LocalScriptService.StartScript between + /// Directory.CreateDirectory and the first state Save (and + /// before the ticket is registered with the running-script reporter), so a + /// sweep can never delete a workspace a deployment is initialising into. A + /// freshly-created dir reclaims ~nothing anyway, so protecting it costs + /// nothing. Not env-tunable — it is a correctness floor, not a policy knob. + /// + public const int FreshWorkspaceGraceSeconds = 60; + + public static TimeSpan FreshWorkspaceGraceWindow => TimeSpan.FromSeconds(FreshWorkspaceGraceSeconds); + + /// Read once, cached for process lifetime. + public static SelfHealOptions Default { get; } = Resolve(); + + public RetentionQuota Quota => new(KeepLatestSucceeded, KeepLatestFailed); + + private static SelfHealOptions Resolve() => new( + ResolveKeepCount(KeepSucceededEnvVar, DefaultKeepLatestSucceeded), + ResolveKeepCount(KeepFailedEnvVar, DefaultKeepLatestFailed), + ResolveFreePercentage(LowFreePercentageEnvVar, DefaultLowFreePercentage), + DefaultCriticalFreePercentage, + DefaultCriticalTargetFreePercentage); + + private static int ResolveKeepCount(string envVar, int defaultValue) + { + var raw = Environment.GetEnvironmentVariable(envVar); + + if (string.IsNullOrWhiteSpace(raw)) + return defaultValue; + + if (!int.TryParse(raw, out var value) || value < MinKeepCount || value > MaxKeepCount) + { + Log.Warning("{EnvVar}='{RawValue}' is not a valid integer in [{Min}..{Max}]; falling back to default {Default}", + envVar, raw, MinKeepCount, MaxKeepCount, defaultValue); + return defaultValue; + } + + Log.Information("Self-heal retention {EnvVar} configured to {Value}", envVar, value); + + return value; + } + + private static double ResolveFreePercentage(string envVar, double defaultValue) + { + var raw = Environment.GetEnvironmentVariable(envVar); + + if (string.IsNullOrWhiteSpace(raw)) + return defaultValue; + + if (!double.TryParse(raw, out var value) || value <= 0.0 || value >= 1.0) + { + Log.Warning("{EnvVar}='{RawValue}' is not a valid fraction in (0, 1); falling back to default {Default}", + envVar, raw, defaultValue); + return defaultValue; + } + + Log.Information("Self-heal {EnvVar} configured to {Value}", envVar, value); + + return value; + } +} diff --git a/src/Squid.Tentacle/SelfHeal/WorkspaceProbe.cs b/src/Squid.Tentacle/SelfHeal/WorkspaceProbe.cs index 5273a8c2..a753d20b 100644 --- a/src/Squid.Tentacle/SelfHeal/WorkspaceProbe.cs +++ b/src/Squid.Tentacle/SelfHeal/WorkspaceProbe.cs @@ -65,7 +65,23 @@ private static WorkspaceStatus ClassifyStatus(string workDir, IScriptStateStoreF if (!store.Exists()) return WorkspaceStatus.Unknown; - var state = store.Load(); + ScriptState state; + try + { + state = store.Load(); + } + catch (Exception ex) + { + // State file is present but unreadable (corrupt primary with no usable + // backup — e.g. a partial write when the disk filled mid-Save). Classify + // Unknown rather than letting the throw bubble to Probe's catch-and-skip, + // which would drop the workspace from the candidate list forever and make + // a dead-but-corrupt workspace permanently un-reclaimable — exactly when + // disk pressure most needs the space. A still-live script is separately + // protected by the running-script-reporter veto + the fresh-grace floor. + Log.Debug(ex, "[SelfHeal] Unreadable script state at {Path}; classifying Unknown", workDir); + return WorkspaceStatus.Unknown; + } if (!state.IsComplete()) return state.HasStarted() ? WorkspaceStatus.Active : WorkspaceStatus.Unknown; @@ -96,9 +112,23 @@ private static IEnumerable SafeEnumerateDirectories(string root) } } + // Recurse subdirectories but SKIP reparse points (directory symlinks / + // junctions): the default SearchOption.AllDirectories walk follows them, so a + // deployment package that extracted a symlink-to-parent (or a runtime `ln -s`) + // makes the walk loop until it exhausts the path-length limit — burning CPU/IO + // on the exact low-disk tick the heal exists to relieve, and mis-attributing a + // symlink target's size to the workspace. AttributesToSkip overrides the default + // (Hidden|System) so hidden/system files still count toward disk usage. + private static readonly EnumerationOptions SizeWalkOptions = new() + { + RecurseSubdirectories = true, + AttributesToSkip = FileAttributes.ReparsePoint, + IgnoreInaccessible = true + }; + private static IEnumerable SafeEnumerateFiles(string dir) { - try { return Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories); } + try { return Directory.EnumerateFiles(dir, "*", SizeWalkOptions); } catch { return Array.Empty(); } } } diff --git a/tests/Squid.Tentacle.Tests/SelfHeal/DiskPressureHealActionTests.cs b/tests/Squid.Tentacle.Tests/SelfHeal/DiskPressureHealActionTests.cs index a1a29c09..26998a3a 100644 --- a/tests/Squid.Tentacle.Tests/SelfHeal/DiskPressureHealActionTests.cs +++ b/tests/Squid.Tentacle.Tests/SelfHeal/DiskPressureHealActionTests.cs @@ -119,6 +119,65 @@ public async Task Run_WorkspaceRootEmpty_ReportsHealthy() removed.ShouldBeEmpty(); } + [Fact] + public async Task Run_StillUnderPressureWithNothingEvictable_BacksOff_ThenResetsWhenRelieved() + { + // Disk filled by non-workspace usage (or everything protected by retention): + // the sweep can't help. It must back off (exponentially) instead of churning + // every CheckInterval, and reset the moment pressure clears. + var free = 50L; // start low (5%) + var action = new DiskPressureHealAction( + workspaceRootProvider: () => _workspace, + candidateProbe: _ => Array.Empty(), // nothing evictable + policy: new DefaultWorkspaceCleanupPolicy(), + removeWorkspace: _ => { }, + checkInterval: TimeSpan.FromMilliseconds(100), + diskProbe: _ => new DiskPressure(free, 1000)); + + action.CheckInterval.ShouldBe(TimeSpan.FromMilliseconds(100), "starts at the base interval"); + + await action.RunAsync(CancellationToken.None); + var afterFirst = action.CheckInterval; + afterFirst.ShouldBeGreaterThan(TimeSpan.FromMilliseconds(100), "a futile-under-pressure tick backs off"); + + await action.RunAsync(CancellationToken.None); + action.CheckInterval.ShouldBeGreaterThan(afterFirst, "consecutive futile ticks keep backing off"); + + // Backoff is bounded — never exceeds the cap (base x 16) no matter how many ticks. + for (var i = 0; i < 10; i++) await action.RunAsync(CancellationToken.None); + action.CheckInterval.ShouldBeLessThanOrEqualTo(TimeSpan.FromMilliseconds(100 * 16)); + + free = 900; // pressure relieved (90% free) + await action.RunAsync(CancellationToken.None); + action.CheckInterval.ShouldBe(TimeSpan.FromMilliseconds(100), "the interval resets to base once pressure clears"); + } + + [Fact] + public async Task Run_RemovedSomeButStillUnderPressure_ReportsBackingOff() + { + var ws = Path.Combine(_workspace, "old"); + Directory.CreateDirectory(ws); + var candidates = new List + { + new(ws, DateTimeOffset.UtcNow.AddHours(-50), 100, WorkspaceStatus.Succeeded) + }; + + var action = new DiskPressureHealAction( + workspaceRootProvider: () => _workspace, + candidateProbe: _ => candidates, + policy: new AlwaysRemovePolicy(candidates), + removeWorkspace: p => { }, + checkInterval: TimeSpan.FromMilliseconds(100), + diskProbe: HighPressure); // re-probe after removal still shows pressure + + var outcome = await action.RunAsync(CancellationToken.None); + + outcome.Healed.ShouldBeTrue(); + outcome.Message.ShouldContain("removed 1", customMessage: "the reclaim summary is preserved"); + outcome.Message.ShouldContain("backing off", customMessage: "and it signals it could not relieve the pressure"); + action.CheckInterval.ShouldBeGreaterThan(TimeSpan.FromMilliseconds(100)); + } + // Injected so CI runners (which have plenty of real free disk) don't cause the // action to exit early before exercising the cleanup code paths. private static DiskPressure HighPressure(string _) => new(FreeBytes: 50, TotalBytes: 1000); diff --git a/tests/Squid.Tentacle.Tests/SelfHeal/DiskSelfHealWiredE2ETests.cs b/tests/Squid.Tentacle.Tests/SelfHeal/DiskSelfHealWiredE2ETests.cs new file mode 100644 index 00000000..099e390c --- /dev/null +++ b/tests/Squid.Tentacle.Tests/SelfHeal/DiskSelfHealWiredE2ETests.cs @@ -0,0 +1,137 @@ +using Shouldly; +using Squid.Tentacle.ScriptExecution.State; +using Squid.Tentacle.SelfHeal; +using Squid.Tentacle.Tests.Support; +using Xunit; + +namespace Squid.Tentacle.Tests.SelfHeal; + +/// +/// 🟢 HIGH-fidelity end-to-end coverage of the WIRED disk self-heal chain: real +/// + real +/// (with the live retention + fresh-grace floors) + real +/// state files + real recursive +/// against real OS directories. Only +/// the disk-pressure MEASUREMENT is injected (forced low/critical) — the same +/// substitution the unit tests use, because CI runners have plenty of free disk and +/// would otherwise no-op the sweep. +/// +/// The unit tests cover each component in isolation; this proves the +/// composed production path (the one SelfHealBackgroundTask.ForLocalWorkspaces +/// builds) actually reclaims a real completed workspace while a real Running-state +/// workspace survives, honours the operator's retention TTL under mild pressure, +/// and reclaims a corrupt-state workspace under critical pressure (the finding that +/// a corrupt state file would otherwise be un-reclaimable forever). +/// +[Trait("Category", TentacleTestCategories.Integration)] +public sealed class DiskSelfHealWiredE2ETests : IDisposable +{ + private readonly string _root = Path.Combine(Path.GetTempPath(), $"squid-selfheal-e2e-{Guid.NewGuid():N}"); + private readonly ScriptStateStoreFactory _factory = new(); + + public DiskSelfHealWiredE2ETests() => Directory.CreateDirectory(_root); + + public void Dispose() + { + try { if (Directory.Exists(_root)) Directory.Delete(_root, recursive: true); } + catch { /* best-effort */ } + } + + [Fact] + public async Task WiredChain_CriticalPressure_ReclaimsOldSucceeded_KeepsRunningAndRecent() + { + var oldSucceeded = StageWorkspace("old-ok", Complete(0), age: TimeSpan.FromHours(50)); + var recentSucceeded = StageWorkspace("recent-ok", Complete(0), age: TimeSpan.FromHours(1)); + var running = StageWorkspace("live", Running(), age: TimeSpan.FromHours(2)); + + var action = BuildWiredAction(Critical, keepSucceeded: 1, keepFailed: 1); + + var outcome = await action.RunAsync(CancellationToken.None); + + outcome.Healed.ShouldBeTrue(); + Directory.Exists(oldSucceeded).ShouldBeFalse("an old succeeded workspace beyond the keep-set is reclaimed under critical pressure"); + Directory.Exists(recentSucceeded).ShouldBeTrue("the newest succeeded workspace is kept by the retention quota"); + Directory.Exists(running).ShouldBeTrue("a Running-state workspace is Active and must NEVER be deleted out from under a live script"); + } + + [Fact] + public async Task WiredChain_NonCriticalPressure_HonoursRetentionTtl() + { + var withinTtl = StageWorkspace("within-ttl", Complete(0), age: TimeSpan.FromHours(5)); + var beyondTtl = StageWorkspace("beyond-ttl", Complete(0), age: TimeSpan.FromHours(50)); + + // keep 0 so neither sits in the keep-set — only the retention TTL decides. + var action = BuildWiredAction(NonCriticalLow, keepSucceeded: 0, keepFailed: 0); + + await action.RunAsync(CancellationToken.None); + + Directory.Exists(withinTtl).ShouldBeTrue("under mild pressure a workspace inside the operator's orphan TTL is preserved for post-mortem"); + Directory.Exists(beyondTtl).ShouldBeFalse("a workspace older than the TTL is reclaimed"); + } + + [Fact] + public async Task WiredChain_CriticalPressure_ReclaimsCorruptStateWorkspace() + { + // A dead workspace whose state file is corrupt (partial write when the disk + // filled mid-Save) must still be reclaimable — classified Unknown, not skipped + // forever. This is the exact scenario disk pressure produces. + var corrupt = StageWorkspace("corrupt", state: null, age: TimeSpan.FromHours(50)); + File.WriteAllText(Path.Combine(corrupt, "scriptstate.json"), "{ partial-write garbage"); + Directory.SetLastWriteTimeUtc(corrupt, DateTime.UtcNow - TimeSpan.FromHours(50)); + + var action = BuildWiredAction(Critical, keepSucceeded: 0, keepFailed: 0); + + await action.RunAsync(CancellationToken.None); + + Directory.Exists(corrupt).ShouldBeFalse("a corrupt-state dead workspace must be reclaimable under critical pressure, not un-reclaimable forever"); + } + + // ── Helpers ── + + private static DiskPressure Critical(string _) => new(FreeBytes: 50, TotalBytes: 1000); // 5% — critical + private static DiskPressure NonCriticalLow(string _) => new(FreeBytes: 150, TotalBytes: 1000); // 15% — low, not critical + + private DiskPressureHealAction BuildWiredAction(Func diskProbe, int keepSucceeded, int keepFailed) + => new( + workspaceRootProvider: () => _root, + candidateProbe: root => WorkspaceProbe.Probe(root, _factory), + policy: new DefaultWorkspaceCleanupPolicy( + minRetentionAge: TimeSpan.FromHours(24), + freshGraceWindow: SelfHealOptions.FreshWorkspaceGraceWindow), + removeWorkspace: path => Directory.Delete(path, recursive: true), + quota: new RetentionQuota(keepSucceeded, keepFailed), + diskProbe: diskProbe); + + private string StageWorkspace(string ticketId, ScriptState state, TimeSpan age) + { + var dir = Path.Combine(_root, $"squid-tentacle-{ticketId}-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + + File.WriteAllBytes(Path.Combine(dir, "output.log"), new byte[4096]); + + if (state != null) + _factory.Create(dir).Save(state); + + // Set the dir mtime LAST — Save() touches the directory, so the age must be + // stamped after every write to reflect the intended workspace age. + Directory.SetLastWriteTimeUtc(dir, DateTime.UtcNow - age); + + return dir; + } + + private static ScriptState Complete(int exitCode) => new() + { + TicketId = "t", + Progress = ScriptProgress.Complete, + ExitCode = exitCode, + CreatedAt = DateTimeOffset.UtcNow, + CompletedAt = DateTimeOffset.UtcNow + }; + + private static ScriptState Running() => new() + { + TicketId = "t", + Progress = ScriptProgress.Running, + CreatedAt = DateTimeOffset.UtcNow + }; +} diff --git a/tests/Squid.Tentacle.Tests/SelfHeal/SelfHealOptionsTests.cs b/tests/Squid.Tentacle.Tests/SelfHeal/SelfHealOptionsTests.cs new file mode 100644 index 00000000..f002c981 --- /dev/null +++ b/tests/Squid.Tentacle.Tests/SelfHeal/SelfHealOptionsTests.cs @@ -0,0 +1,75 @@ +using Shouldly; +using Squid.Tentacle.SelfHeal; +using Squid.Tentacle.Tests.Support; +using Xunit; + +namespace Squid.Tentacle.Tests.SelfHeal; + +/// +/// Rule 8 pin for the disk-pressure self-heal tunables. The retention quota + +/// low-disk trigger are operator-tunable behaviour (a tiny-disk agent wants fewer +/// kept workspaces; a debug-heavy operator wants more), so they live behind env +/// vars with pinned public const string names + pinned literal defaults — +/// a "harmless" rename / default change becomes a visible, test-gated decision +/// instead of a silent prod surprise for an operator who pinned the old name. +/// +/// Like OrphanWorkspaceCleanupConfigTests, the env vars are read once +/// at process start (cached static), so these tests pin the contract (names + +/// defaults), not a runtime re-read. +/// +[Trait("Category", TentacleTestCategories.Core)] +public sealed class SelfHealOptionsTests +{ + [Fact] + public void EnvVarNames_Pinned() + { + // Operators reference these in runbooks / Helm charts / air-gapped configs. + SelfHealOptions.KeepSucceededEnvVar.ShouldBe("SQUID_TENTACLE_SELFHEAL_KEEP_SUCCEEDED"); + SelfHealOptions.KeepFailedEnvVar.ShouldBe("SQUID_TENTACLE_SELFHEAL_KEEP_FAILED"); + SelfHealOptions.LowFreePercentageEnvVar.ShouldBe("SQUID_TENTACLE_SELFHEAL_LOW_FREE_PCT"); + } + + [Fact] + public void LiteralDefaults_Pinned() + { + // Changing any of these alters how aggressively a disk-full agent reclaims + // workspaces — must be a documented decision, not an invisible refactor. + SelfHealOptions.DefaultKeepLatestSucceeded.ShouldBe(10); + SelfHealOptions.DefaultKeepLatestFailed.ShouldBe(20); + SelfHealOptions.DefaultLowFreePercentage.ShouldBe(0.20); + SelfHealOptions.DefaultCriticalFreePercentage.ShouldBe(0.10); + SelfHealOptions.DefaultCriticalTargetFreePercentage.ShouldBe(0.30); + } + + [Fact] + public void FreshGraceWindow_Pinned() + { + // The TOCTOU safety floor — never delete a workspace a deployment is + // initialising into. A change here widens/narrows the race window. + SelfHealOptions.FreshWorkspaceGraceSeconds.ShouldBe(60); + SelfHealOptions.FreshWorkspaceGraceWindow.ShouldBe(TimeSpan.FromSeconds(60)); + } + + [Fact] + public void Default_WithNoEnvOverrides_UsesLiteralDefaults() + { + // The CI / dev environment sets none of these env vars, so Default must + // resolve to the pinned literals — proving Resolve()'s no-override path. + var options = SelfHealOptions.Default; + + options.KeepLatestSucceeded.ShouldBe(SelfHealOptions.DefaultKeepLatestSucceeded); + options.KeepLatestFailed.ShouldBe(SelfHealOptions.DefaultKeepLatestFailed); + options.LowFreePercentage.ShouldBe(SelfHealOptions.DefaultLowFreePercentage); + options.CriticalFreePercentage.ShouldBe(SelfHealOptions.DefaultCriticalFreePercentage); + options.CriticalTargetFreePercentage.ShouldBe(SelfHealOptions.DefaultCriticalTargetFreePercentage); + } + + [Fact] + public void Quota_MirrorsRetentionCounts() + { + var options = SelfHealOptions.Default; + + options.Quota.KeepLatestSucceeded.ShouldBe(options.KeepLatestSucceeded); + options.Quota.KeepLatestFailed.ShouldBe(options.KeepLatestFailed); + } +} diff --git a/tests/Squid.Tentacle.Tests/SelfHeal/WorkspaceCleanupPolicyTests.cs b/tests/Squid.Tentacle.Tests/SelfHeal/WorkspaceCleanupPolicyTests.cs index 51a66155..534a9673 100644 --- a/tests/Squid.Tentacle.Tests/SelfHeal/WorkspaceCleanupPolicyTests.cs +++ b/tests/Squid.Tentacle.Tests/SelfHeal/WorkspaceCleanupPolicyTests.cs @@ -103,6 +103,62 @@ public void NullCandidates_ReturnsEmpty() _policy.SelectForRemoval(null, pressure, RetentionQuota.Default).ShouldBeEmpty(); } + [Fact] + public void RetentionFloor_NonCriticalPressure_KeepsWorkspacesWithinTtl_EvictsOlder() + { + // The new disk sweep must NOT silently override the operator's orphan-workspace + // TTL (post-mortem window). Under mild (non-critical) pressure, a completed + // workspace younger than minRetentionAge is protected even though it is beyond + // the keep-set; only those older than the TTL are reclaimed. + var now = DateTimeOffset.UtcNow; + var policy = new DefaultWorkspaceCleanupPolicy(minRetentionAge: TimeSpan.FromHours(24), clock: () => now); + + var pressure = new DiskPressure(FreeBytes: 150, TotalBytes: 1000); // 15% — low but NOT critical + pressure.IsLow.ShouldBeTrue(); + pressure.IsCritical.ShouldBeFalse(); + + var candidates = new[] + { + new WorkspaceCandidate("within-ttl", now.AddHours(-5), 500, WorkspaceStatus.Succeeded), + new WorkspaceCandidate("beyond-ttl", now.AddHours(-48), 500, WorkspaceStatus.Succeeded) + }; + + var selected = _policy.SelectForRemoval(candidates, pressure, new RetentionQuota(0, 0)); + + var paths = selected.Select(s => s.Path).ToList(); + paths.ShouldContain("beyond-ttl", "a workspace older than the operator's TTL is reclaimable under pressure"); + paths.ShouldNotContain("within-ttl", "a workspace inside the operator's post-mortem TTL must NOT be evicted under mild pressure"); + } + + [Fact] + public void RetentionFloor_CriticalPressure_BypassesTtl_ButFreshGraceStillProtects() + { + // Under CRITICAL pressure the retention TTL is bypassed (reclaiming disk wins), + // BUT the short fresh-grace floor still protects a just-created workspace — that + // is the TOCTOU guard for a deployment initialising a brand-new workspace, and + // it must hold even in an emergency. + var now = DateTimeOffset.UtcNow; + var policy = new DefaultWorkspaceCleanupPolicy( + minRetentionAge: TimeSpan.FromHours(24), + freshGraceWindow: TimeSpan.FromSeconds(60), + clock: () => now); + + var pressure = new DiskPressure(FreeBytes: 50, TotalBytes: 1000); // 5% — critical + pressure.IsCritical.ShouldBeTrue(); + + var candidates = new[] + { + new WorkspaceCandidate("recent-within-ttl", now.AddHours(-1), 500, WorkspaceStatus.Succeeded), + new WorkspaceCandidate("just-created", now.AddSeconds(-10), 500, WorkspaceStatus.Unknown) + }; + + var selected = _policy.SelectForRemoval(candidates, pressure, new RetentionQuota(0, 0)); + + var paths = selected.Select(s => s.Path).ToList(); + paths.ShouldContain("recent-within-ttl", "critical pressure bypasses the retention TTL to reclaim disk"); + paths.ShouldNotContain("just-created", "the fresh-grace floor protects a just-created workspace even under critical pressure (TOCTOU guard)"); + } + [Fact] public void CriticalPressure_RaisesTargetTo30Percent() { diff --git a/tests/Squid.Tentacle.Tests/SelfHeal/WorkspaceProbeTests.cs b/tests/Squid.Tentacle.Tests/SelfHeal/WorkspaceProbeTests.cs index b4b36985..3eca2c82 100644 --- a/tests/Squid.Tentacle.Tests/SelfHeal/WorkspaceProbeTests.cs +++ b/tests/Squid.Tentacle.Tests/SelfHeal/WorkspaceProbeTests.cs @@ -94,6 +94,28 @@ public void Probe_NoStateFile_IsUnknown() .Status.ShouldBe(WorkspaceStatus.Unknown); } + [Fact] + public void Probe_StateFileExistsButCorrupt_ClassifiedUnknown_SweepContinues() + { + // A present-but-unreadable state file (corrupt primary, no usable backup — + // e.g. a partial write when the disk filled mid-Save) makes ScriptStateStore + // Exists()==true but Load() throw. The probe must classify it Unknown + // (reclaimable under pressure) — NOT let the throw drop it from the candidate + // list forever (which would make a dead-but-corrupt workspace permanently + // un-reclaimable, defeating the heal precisely when disk is full) and NOT + // abort the whole sweep so the valid workspace beside it is still classified. + var corrupt = MakeWorkspace("corrupt"); + File.WriteAllText(Path.Combine(corrupt, "scriptstate.json"), "{ this is no longer valid json"); + MakeWorkspace("good", Complete("good", 0)); + + var byStatus = WorkspaceProbe.Probe(_root, _factory).ToLookup(c => c.Status); + + byStatus[WorkspaceStatus.Unknown].ShouldHaveSingleItem() + .Path.ShouldEndWith("squid-tentacle-corrupt"); + byStatus[WorkspaceStatus.Succeeded].ShouldHaveSingleItem() + .Path.ShouldEndWith("squid-tentacle-good"); + } + [Fact] public void Probe_IgnoresDirectoriesNotMatchingTheWorkspacePrefix() { @@ -108,15 +130,55 @@ public void Probe_IgnoresDirectoriesNotMatchingTheWorkspacePrefix() } [Fact] - public void Probe_MeasuresWorkspaceSize() + public void Probe_MeasuresWorkspaceSize_RecursivelyAcrossNestedDirs() + { + // Real script workspaces nest artefacts (extracted packages, Calamari trees), + // so the recursive walk is the load-bearing part: the policy reclaims by + // SizeBytes, and a TopDirectoryOnly regression would silently halve the + // measured size for deep workspaces and starve the heal. Put payload at TWO + // depths and assert the summed size, pinning recursion AND per-file accrual. + var dir = MakeWorkspace("sized", Complete("sized", 0)); + File.WriteAllBytes(Path.Combine(dir, "top.bin"), new byte[4096]); + var deep = Path.Combine(dir, "sub", "deeper"); + Directory.CreateDirectory(deep); + File.WriteAllBytes(Path.Combine(deep, "nested.bin"), new byte[8192]); + + var c = WorkspaceProbe.Probe(_root, _factory).ShouldHaveSingleItem(); + + c.SizeBytes.ShouldBeGreaterThanOrEqualTo(4096 + 8192, + customMessage: "Size must recurse into nested dirs (nested.bin under sub/deeper) AND sum every file " + + "(top.bin) — a TopDirectoryOnly walk would miss the 8 KB nested file."); + } + + [Fact] + public void Probe_SizeWalk_DoesNotFollowDirectorySymlinks() { - var payload = new string('x', 4096); - MakeWorkspace("sized", Complete("sized", 0), content: payload); + // A deployment package (or a runtime `ln -s`) can plant a directory symlink + // inside the workspace. The default AllDirectories walk follows it — a + // symlink-to-parent loops until the path-length limit (CPU/IO burn on the + // exact low-disk tick the heal targets) and a symlink to /var or C:\ mis- + // attributes unrelated disk to the workspace. The walk must skip reparse points. + var dir = MakeWorkspace("withsymlink", Complete("withsymlink", 0)); + File.WriteAllBytes(Path.Combine(dir, "real.bin"), new byte[4096]); + + try + { + // Self-referential loop: {dir}/loop -> {dir}. A followed walk would recurse forever. + Directory.CreateSymbolicLink(Path.Combine(dir, "loop"), dir); + } + catch (Exception ex) when (ex is UnauthorizedAccessException or IOException or PlatformNotSupportedException) + { + return; // host doesn't permit symlink creation (e.g. Windows without dev mode) — skip + } var c = WorkspaceProbe.Probe(_root, _factory).ShouldHaveSingleItem(); - c.SizeBytes.ShouldBeGreaterThanOrEqualTo(4096, - customMessage: "Size must include the workspace's files so the policy can reclaim enough space."); + // Completes (no hang) AND counts only the real file + state json — the symlink + // loop contributes nothing. If reparse points were followed this would either + // hang the test or balloon the size. + c.SizeBytes.ShouldBeGreaterThanOrEqualTo(4096); + c.SizeBytes.ShouldBeLessThan(4096 * 4, + customMessage: "Size must exclude the symlink target — a followed loop would re-count files unboundedly."); } [Fact] From 2a65bf749e8e7b1f1f862a9e4156cfd06ea21744 Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Thu, 11 Jun 2026 22:16:54 +0800 Subject: [PATCH 3/3] Pin SelfHealOptions bounds + parse branches, invariant-culture percentage parse Make MinKeepCount/MaxKeepCount public (mirroring LocalScriptService's orphan-TTL bounds) and extract the env-value parsing into internal testable helpers so the reject/fallback branches are directly pinned (Rule 9 branch coverage). Parse the free-percentage with invariant culture so an operator's '0.20' reads the same on a comma-decimal locale server. --- .../SelfHeal/SelfHealOptions.cs | 27 ++++++---- .../SelfHeal/SelfHealOptionsTests.cs | 52 +++++++++++++++++++ 2 files changed, 69 insertions(+), 10 deletions(-) diff --git a/src/Squid.Tentacle/SelfHeal/SelfHealOptions.cs b/src/Squid.Tentacle/SelfHeal/SelfHealOptions.cs index 1acc4df3..3cfa0f2a 100644 --- a/src/Squid.Tentacle/SelfHeal/SelfHealOptions.cs +++ b/src/Squid.Tentacle/SelfHeal/SelfHealOptions.cs @@ -1,3 +1,4 @@ +using System.Globalization; using Serilog; namespace Squid.Tentacle.SelfHeal; @@ -37,9 +38,10 @@ public sealed record SelfHealOptions( // Bounds: keep-counts 0..10_000 (0 = keep nothing under pressure); free // percentages strictly inside (0, 1). Out-of-range / unparseable input - // falls back to the default with a Serilog warning. - private const int MinKeepCount = 0; - private const int MaxKeepCount = 10_000; + // falls back to the default with a Serilog warning. Public + pinned by test + // (mirrors LocalScriptService.Min/MaxOrphanMaxAgeHours). + public const int MinKeepCount = 0; + public const int MaxKeepCount = 10_000; /// /// Short safety floor: a workspace whose directory was last written within @@ -68,13 +70,18 @@ public sealed record SelfHealOptions( DefaultCriticalTargetFreePercentage); private static int ResolveKeepCount(string envVar, int defaultValue) - { - var raw = Environment.GetEnvironmentVariable(envVar); + => ParseKeepCount(Environment.GetEnvironmentVariable(envVar), envVar, defaultValue); + + private static double ResolveFreePercentage(string envVar, double defaultValue) + => ParseFreePercentage(Environment.GetEnvironmentVariable(envVar), envVar, defaultValue); + // Pure parse + validate (no env read) so every branch is directly unit-testable. + internal static int ParseKeepCount(string raw, string envVar, int defaultValue) + { if (string.IsNullOrWhiteSpace(raw)) return defaultValue; - if (!int.TryParse(raw, out var value) || value < MinKeepCount || value > MaxKeepCount) + if (!int.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var value) || value < MinKeepCount || value > MaxKeepCount) { Log.Warning("{EnvVar}='{RawValue}' is not a valid integer in [{Min}..{Max}]; falling back to default {Default}", envVar, raw, MinKeepCount, MaxKeepCount, defaultValue); @@ -86,14 +93,14 @@ private static int ResolveKeepCount(string envVar, int defaultValue) return value; } - private static double ResolveFreePercentage(string envVar, double defaultValue) + // Invariant culture so an operator's "0.20" parses identically regardless of the + // server's locale (a comma-decimal locale would otherwise mis-read it). + internal static double ParseFreePercentage(string raw, string envVar, double defaultValue) { - var raw = Environment.GetEnvironmentVariable(envVar); - if (string.IsNullOrWhiteSpace(raw)) return defaultValue; - if (!double.TryParse(raw, out var value) || value <= 0.0 || value >= 1.0) + if (!double.TryParse(raw, NumberStyles.Float, CultureInfo.InvariantCulture, out var value) || value <= 0.0 || value >= 1.0) { Log.Warning("{EnvVar}='{RawValue}' is not a valid fraction in (0, 1); falling back to default {Default}", envVar, raw, defaultValue); diff --git a/tests/Squid.Tentacle.Tests/SelfHeal/SelfHealOptionsTests.cs b/tests/Squid.Tentacle.Tests/SelfHeal/SelfHealOptionsTests.cs index f002c981..2bbe8d06 100644 --- a/tests/Squid.Tentacle.Tests/SelfHeal/SelfHealOptionsTests.cs +++ b/tests/Squid.Tentacle.Tests/SelfHeal/SelfHealOptionsTests.cs @@ -72,4 +72,56 @@ public void Quota_MirrorsRetentionCounts() options.Quota.KeepLatestSucceeded.ShouldBe(options.KeepLatestSucceeded); options.Quota.KeepLatestFailed.ShouldBe(options.KeepLatestFailed); } + + [Fact] + public void KeepCountBounds_Pinned() + { + // Mirrors LocalScriptService.Min/MaxOrphanMaxAgeHours: a widening of the cap + // (or dropping the bounds check) must be a visible, test-gated decision. + SelfHealOptions.MinKeepCount.ShouldBe(0); + SelfHealOptions.MaxKeepCount.ShouldBe(10_000); + } + + [Theory] + [InlineData(null, 10)] // unset → default + [InlineData("", 10)] // blank → default + [InlineData(" ", 10)] // whitespace → default + [InlineData("abc", 10)] // unparseable → default + [InlineData("-1", 10)] // below MinKeepCount → default + [InlineData("10001", 10)] // above MaxKeepCount → default + [InlineData("0", 0)] // valid lower bound (keep nothing under pressure) + [InlineData("5", 5)] // valid + [InlineData("10000", 10000)] // valid upper bound + public void ParseKeepCount_AcceptsValid_RejectsOutOfRangeOrGarbage(string raw, int expected) + => SelfHealOptions.ParseKeepCount(raw, "SQUID_TENTACLE_SELFHEAL_KEEP_SUCCEEDED", defaultValue: 10).ShouldBe(expected); + + [Theory] + [InlineData(null, 0.20)] // unset → default + [InlineData("", 0.20)] // blank → default + [InlineData("abc", 0.20)] // unparseable → default + [InlineData("0", 0.20)] // <= 0 → default (must be a strict fraction) + [InlineData("1", 0.20)] // >= 1 → default + [InlineData("-0.1", 0.20)] // negative → default + [InlineData("1.5", 0.20)] // above 1 → default + [InlineData("0.35", 0.35)] // valid fraction + [InlineData("0.99", 0.99)] // valid near-upper + public void ParseFreePercentage_AcceptsValidFraction_RejectsOutOfRangeOrGarbage(string raw, double expected) + => SelfHealOptions.ParseFreePercentage(raw, "SQUID_TENTACLE_SELFHEAL_LOW_FREE_PCT", defaultValue: 0.20).ShouldBe(expected); + + [Fact] + public void ParseFreePercentage_IsCultureInvariant() + { + // An operator's "0.20" must parse the same on a comma-decimal locale server — + // invariant culture means the dot is always the decimal separator. + var previous = System.Threading.Thread.CurrentThread.CurrentCulture; + try + { + System.Threading.Thread.CurrentThread.CurrentCulture = new System.Globalization.CultureInfo("de-DE"); + SelfHealOptions.ParseFreePercentage("0.30", "SQUID_TENTACLE_SELFHEAL_LOW_FREE_PCT", defaultValue: 0.20).ShouldBe(0.30); + } + finally + { + System.Threading.Thread.CurrentThread.CurrentCulture = previous; + } + } }