Wire disk-pressure self-heal into the Tentacle host lifecycle

src/Squid.Tentacle/Flavors/Tentacle/TentacleFlavor.cs

@@ -3,6 +3,7 @@
 using Squid.Tentacle.Configuration;
 using Squid.Tentacle.Flavors.Tentacle.Configuration;
 using Squid.Tentacle.ScriptExecution;
+using Squid.Tentacle.SelfHeal;
 using Serilog;
 
 namespace Squid.Tentacle.Flavors.Tentacle;
@@ -41,7 +42,12 @@ public TentacleFlavorRuntime CreateRuntime(TentacleFlavorContext context)
             ScriptBackend = backend,
             CommunicationMode = communicationMode,
             ListeningPort = settings.ListeningPort,
-            BackgroundTasks = [],
+            // Disk-pressure self-heal: reclaim completed-script workspaces when the
+            // temp disk runs low, so a flood of deployments can't fill the disk and
+            // turn every subsequent deploy into a disk-full failure. The backend is
+            // the running-script reporter, so an in-flight deployment's workspace is
+            // never swept.
+            BackgroundTasks = [SelfHealBackgroundTask.ForLocalWorkspaces(backend)],
             StartupHooks = [],
             ReadinessCheck = null,
             Metadata = new Dictionary<string, string>

src/Squid.Tentacle/SelfHeal/DiskPressureHealAction.cs

@@ -30,6 +30,16 @@ public sealed class DiskPressureHealAction : ISelfHealAction
     private readonly TicketExtractor _ticketExtractor;
     private readonly RetentionQuota _quota;
 
+    // Backoff state: when a sweep reclaims everything it is allowed to but the
+    // disk is STILL under pressure (non-workspace usage, or everything protected
+    // by the retention window), re-running every CheckInterval just churns. We
+    // back off exponentially up to _maxInterval and warn once per episode, then
+    // reset the moment pressure clears.
+    private readonly TimeSpan _baseInterval;
+    private readonly TimeSpan _maxInterval;
+    private TimeSpan _currentInterval;
+    private bool _underPressureWarned;
+
     public DiskPressureHealAction(
         Func<string> workspaceRootProvider,
         Func<string, IReadOnlyList<WorkspaceCandidate>> candidateProbe,
@@ -49,7 +59,10 @@ public DiskPressureHealAction(
         _runningScriptReporters = runningScriptReporters?.ToList() ?? new List<IRunningScriptReporter>();
         _ticketExtractor = ticketExtractor ?? DefaultTicketExtractor;
         _quota = quota ?? RetentionQuota.Default;
-        CheckInterval = checkInterval ?? TimeSpan.FromSeconds(30);
+
+        _baseInterval = checkInterval ?? TimeSpan.FromSeconds(30);
+        _maxInterval = TimeSpan.FromTicks(_baseInterval.Ticks * 16);
+        _currentInterval = _baseInterval;
     }
 
     private static string? DefaultTicketExtractor(string workspacePath)
@@ -83,12 +96,13 @@ private bool IsLiveScript(string workspacePath)
 
     public string Name => "disk-pressure-cleanup";
 
-    public TimeSpan CheckInterval { get; }
+    public TimeSpan CheckInterval => _currentInterval;
 
     private static DiskPressure DefaultDiskProbe(string path)
     {
         var (available, total) = DiskSpaceChecker.GetDiskSpace(path);
-        return new DiskPressure(available, total);
+        return new DiskPressure(available, total,
+            SelfHealOptions.Default.LowFreePercentage, SelfHealOptions.Default.CriticalFreePercentage);
     }
 
     public Task<SelfHealOutcome> RunAsync(CancellationToken ct)
@@ -102,20 +116,46 @@ public Task<SelfHealOutcome> RunAsync(CancellationToken ct)
             return Task.FromResult(SelfHealOutcome.Healthy(Name));
 
         if (!pressure.IsLow)
+        {
+            ResetBackoff();
             return Task.FromResult(SelfHealOutcome.Healthy(Name));
-
-        var candidates = _candidateProbe(workspaceRoot);
+        }
 
         // Veto candidates that a script backend still reports as live. This is the
         // race-safe cleanup guarantee: even if the workspace's Output.log looks
         // stale, we never delete a ticket that is still being tracked in memory.
-        candidates = candidates.Where(c => !IsLiveScript(c.Path)).ToList();
+        var candidates = _candidateProbe(workspaceRoot)
+            .Where(c => !IsLiveScript(c.Path))
+            .ToList();
 
         var toRemove = _policy.SelectForRemoval(candidates, pressure, _quota);
 
-        if (toRemove.Count == 0)
-            return Task.FromResult(SelfHealOutcome.Healthy(Name));
+        var (freed, removed) = RemoveWorkspaces(toRemove, ct);
+
+        // Re-measure only if we actually freed something; otherwise the pressure is unchanged.
+        var post = removed > 0 ? _diskProbe(workspaceRoot) : pressure;
+
+        if (!post.IsLow)
+        {
+            ResetBackoff();
+            return Task.FromResult(SelfHealOutcome.RepairPerformed(Name, Summary(pressure, removed, freed)));
+        }
+
+        // Reclaimed everything we were allowed to, but the disk is STILL under
+        // pressure — remaining usage is protected by the retention window or is not
+        // workspace-driven. Warn once per episode + back off so we don't churn the
+        // disk every tick (and re-delete a freshly-completed workspace the instant
+        // it ages past the keep-set).
+        WarnUnderPressureOnce(post, removed, freed);
+        BackOff();
+
+        return Task.FromResult(removed > 0
+            ? SelfHealOutcome.RepairPerformed(Name, $"{Summary(pressure, removed, freed)}; still {post.FreePercentage:P1} free — backing off to {_currentInterval}")
+            : SelfHealOutcome.Healthy(Name));
+    }
 
+    private (long Freed, int Removed) RemoveWorkspaces(IReadOnlyList<WorkspaceCandidate> toRemove, CancellationToken ct)
+    {
         var freed = 0L;
         var removed = 0;
 
@@ -134,7 +174,31 @@ public Task<SelfHealOutcome> RunAsync(CancellationToken ct)
             }
         }
 
-        return Task.FromResult(SelfHealOutcome.RepairPerformed(Name,
-            $"disk pressure {pressure.FreePercentage:P1} free — removed {removed} workspace(s), reclaimed {DiskSpaceChecker.FormatBytes(freed)}"));
+        return (freed, removed);
+    }
+
+    private static string Summary(DiskPressure pressure, int removed, long freed)
+        => $"disk pressure {pressure.FreePercentage:P1} free — removed {removed} workspace(s), reclaimed {DiskSpaceChecker.FormatBytes(freed)}";
+
+    private void WarnUnderPressureOnce(DiskPressure post, int removed, long freed)
+    {
+        if (_underPressureWarned) return;
+
+        _underPressureWarned = true;
+        Log.Warning("[SelfHeal] Disk still under pressure ({Free:P1} free) after reclaiming {Removed} workspace(s) ({Freed}) — " +
+            "remaining usage is protected by the retention window or is not workspace-driven. Backing off the heal sweep.",
+            post.FreePercentage, removed, DiskSpaceChecker.FormatBytes(freed));
+    }
+
+    private void BackOff()
+    {
+        var doubled = TimeSpan.FromTicks(Math.Min(_currentInterval.Ticks * 2, _maxInterval.Ticks));
+        _currentInterval = doubled < _baseInterval ? _baseInterval : doubled;
+    }
+
+    private void ResetBackoff()
+    {
+        _currentInterval = _baseInterval;
+        _underPressureWarned = false;
     }
 }

src/Squid.Tentacle/SelfHeal/IWorkspaceCleanupPolicy.cs

@@ -29,16 +29,20 @@ public enum WorkspaceStatus
     Unknown
 }
 
-public sealed record DiskPressure(long FreeBytes, long TotalBytes)
+public sealed record DiskPressure(
+    long FreeBytes,
+    long TotalBytes,
+    double LowFreePercentage = SelfHealOptions.DefaultLowFreePercentage,
+    double CriticalFreePercentage = SelfHealOptions.DefaultCriticalFreePercentage)
 {
     public double FreePercentage => TotalBytes > 0 ? (double)FreeBytes / TotalBytes : 0.0;
-    public bool IsLow => FreePercentage < 0.20;
-    public bool IsCritical => FreePercentage < 0.10;
+    public bool IsLow => FreePercentage < LowFreePercentage;
+    public bool IsCritical => FreePercentage < CriticalFreePercentage;
 }
 
 public sealed record RetentionQuota(int KeepLatestSucceeded, int KeepLatestFailed)
 {
-    public static RetentionQuota Default => new(KeepLatestSucceeded: 10, KeepLatestFailed: 20);
+    public static RetentionQuota Default => new(SelfHealOptions.DefaultKeepLatestSucceeded, SelfHealOptions.DefaultKeepLatestFailed);
 }
 
 /// <summary>
@@ -47,10 +51,47 @@ public sealed record RetentionQuota(int KeepLatestSucceeded, int KeepLatestFaile
 ///   - Not under disk pressure (IsLow == false): nothing to do.
 ///   - Under pressure: keep the most recent K succeeded + M failed; evict the
 ///     rest in oldest-first order until free space climbs back above the
-///     low-pressure threshold (20% free, or 30% under critical pressure).
+///     low-pressure target (the normal target equals the low threshold, raised
+///     to <c>criticalTargetFreePercentage</c> under critical pressure).
+///
+/// <para>Two age floors keep the sweep safe and operator-friendly:</para>
+/// <list type="bullet">
+///   <item><b>Fresh-grace</b> (<paramref name="freshGraceWindow"/>) protects a
+///         workspace whose directory was written within the window — even under
+///         critical pressure — so a deployment initialising a brand-new
+///         workspace is never deleted out from under it (the TOCTOU gap before
+///         the running-script reporter knows about the ticket).</item>
+///   <item><b>Retention TTL</b> (<paramref name="minRetentionAge"/>, the
+///         operator's orphan-workspace TTL) protects recent completed
+///         workspaces so the post-mortem window an operator pinned is honoured —
+///         <i>except</i> under critical pressure, where reclaiming disk wins.</item>
+/// </list>
+/// Both default to zero so the parameterless policy keeps its original
+/// no-floor behaviour; the live wiring (<c>SelfHealBackgroundTask.ForLocalWorkspaces</c>)
+/// injects the real floors.
 /// </summary>
 public sealed class DefaultWorkspaceCleanupPolicy : IWorkspaceCleanupPolicy
 {
+    private readonly double _targetFreePercentage;
+    private readonly double _criticalTargetFreePercentage;
+    private readonly TimeSpan _minRetentionAge;
+    private readonly TimeSpan _freshGraceWindow;
+    private readonly Func<DateTimeOffset> _clock;
+
+    public DefaultWorkspaceCleanupPolicy(
+        double targetFreePercentage = SelfHealOptions.DefaultLowFreePercentage,
+        double criticalTargetFreePercentage = SelfHealOptions.DefaultCriticalTargetFreePercentage,
+        TimeSpan minRetentionAge = default,
+        TimeSpan freshGraceWindow = default,
+        Func<DateTimeOffset> clock = null)
+    {
+        _targetFreePercentage = targetFreePercentage;
+        _criticalTargetFreePercentage = criticalTargetFreePercentage;
+        _minRetentionAge = minRetentionAge;
+        _freshGraceWindow = freshGraceWindow;
+        _clock = clock ?? (() => DateTimeOffset.UtcNow);
+    }
+
     public IReadOnlyList<WorkspaceCandidate> SelectForRemoval(
         IReadOnlyList<WorkspaceCandidate> candidates,
         DiskPressure pressure,
@@ -63,14 +104,18 @@ public IReadOnlyList<WorkspaceCandidate> SelectForRemoval(
         keep.UnionWith(LatestByStatus(candidates, WorkspaceStatus.Succeeded, quota.KeepLatestSucceeded));
         keep.UnionWith(LatestByStatus(candidates, WorkspaceStatus.Failed, quota.KeepLatestFailed));
 
+        var floor = EffectiveAgeFloor(pressure);
+        var now = _clock();
+
         var removable = candidates
             .Where(c => c.Status != WorkspaceStatus.Active && !keep.Contains(c.Path))
+            .Where(c => now - c.LastModifiedUtc >= floor)       // honour fresh-grace + retention TTL
             .OrderBy(c => c.LastModifiedUtc)                    // oldest first
             .ToList();
 
         if (removable.Count == 0) return Array.Empty<WorkspaceCandidate>();
 
-        var targetFreePct = pressure.IsCritical ? 0.30 : 0.20;
+        var targetFreePct = pressure.IsCritical ? _criticalTargetFreePercentage : _targetFreePercentage;
         var requiredBytes = (long)(pressure.TotalBytes * targetFreePct) - pressure.FreeBytes;
 
         if (requiredBytes <= 0) return Array.Empty<WorkspaceCandidate>();
@@ -88,6 +133,13 @@ public IReadOnlyList<WorkspaceCandidate> SelectForRemoval(
         return selected;
     }
 
+    // Under critical pressure only the short fresh-grace floor applies (reclaim
+    // disk wins over post-mortem retention); otherwise the longer of the two.
+    private TimeSpan EffectiveAgeFloor(DiskPressure pressure)
+        => pressure.IsCritical
+            ? _freshGraceWindow
+            : (_freshGraceWindow > _minRetentionAge ? _freshGraceWindow : _minRetentionAge);
+
     private static IEnumerable<string> LatestByStatus(IReadOnlyList<WorkspaceCandidate> candidates, WorkspaceStatus status, int count)
         => candidates
             .Where(c => c.Status == status)

src/Squid.Tentacle/SelfHeal/SelfHealBackgroundTask.cs

@@ -0,0 +1,87 @@
+using Squid.Tentacle.Abstractions;
+using Squid.Tentacle.ScriptExecution;
+using Squid.Tentacle.ScriptExecution.State;
+
+namespace Squid.Tentacle.SelfHeal;
+
+/// <summary>
+/// Adapts <see cref="SelfHealController"/> to the host's
+/// <see cref="ITentacleBackgroundTask"/> lifecycle: starts the heal loops when the
+/// host launches background tasks and drains them when the host's cancellation
+/// token trips on shutdown. This is the wiring that takes the self-heal
+/// scaffolding from dead code to live — before it, no flavor scheduled any heal
+/// action, so a disk-full agent failed deployments with no auto-reclaim.
+/// </summary>
+public sealed class SelfHealBackgroundTask : ITentacleBackgroundTask, IAsyncDisposable
+{
+    private readonly SelfHealController _controller;
+
+    public SelfHealBackgroundTask(SelfHealController controller)
+    {
+        _controller = controller ?? throw new ArgumentNullException(nameof(controller));
+    }
+
+    public string Name => "SelfHeal";
+
+    public async Task RunAsync(CancellationToken ct)
+    {
+        _controller.Start();
+
+        try
+        {
+            await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false);
+        }
+        catch (OperationCanceledException) when (ct.IsCancellationRequested)
+        {
+            // Expected on shutdown.
+        }
+        finally
+        {
+            await _controller.DisposeAsync().ConfigureAwait(false);
+        }
+    }
+
+    public ValueTask DisposeAsync() => _controller.DisposeAsync();
+
+    /// <summary>
+    /// Builds the disk-pressure self-heal for the local script workspaces that
+    /// <see cref="LocalScriptService"/> creates under the temp root
+    /// (<c>{tempRoot}/squid-tentacle-{ticketId}</c>). The <paramref name="reporter"/>
+    /// (the live script backend) vetoes deletion of any workspace still running a
+    /// script, and the policy layers two age floors on top so the sweep can never
+    /// remove an in-flight deployment's directory:
+    /// <list type="bullet">
+    ///   <item>the <see cref="SelfHealOptions.FreshWorkspaceGraceWindow"/> protects a
+    ///         just-created workspace before the reporter knows its ticket (the
+    ///         StartScript TOCTOU gap), even under critical pressure;</item>
+    ///   <item><see cref="LocalScriptService.OrphanMaxAge"/> (the operator's
+    ///         orphan-workspace TTL) is honoured as a retention floor so the new
+    ///         disk sweep does not silently override the post-mortem window the
+    ///         existing age-based orphan sweep promises — except under critical
+    ///         pressure, where reclaiming disk wins.</item>
+    /// </list>
+    /// Tunables (retention counts, low-disk trigger) come from
+    /// <see cref="SelfHealOptions.Default"/> (env-var overridable).
+    /// </summary>
+    public static SelfHealBackgroundTask ForLocalWorkspaces(IRunningScriptReporter reporter)
+    {
+        var stateStoreFactory = new ScriptStateStoreFactory();
+        var options = SelfHealOptions.Default;
+
+        var policy = new DefaultWorkspaceCleanupPolicy(
+            targetFreePercentage: options.LowFreePercentage,
+            criticalTargetFreePercentage: options.CriticalTargetFreePercentage,
+            minRetentionAge: LocalScriptService.OrphanMaxAge,
+            freshGraceWindow: SelfHealOptions.FreshWorkspaceGraceWindow);
+
+        var action = new DiskPressureHealAction(
+            workspaceRootProvider: Path.GetTempPath,
+            candidateProbe: root => WorkspaceProbe.Probe(root, stateStoreFactory),
+            policy: policy,
+            removeWorkspace: path => Directory.Delete(path, recursive: true),
+            quota: options.Quota,
+            runningScriptReporters: reporter == null ? null : new[] { reporter });
+
+        return new SelfHealBackgroundTask(new SelfHealController(new ISelfHealAction[] { action }));
+    }
+}