Make deployment kill-timeout configurable via env var

src/Squid.Core/Services/DeploymentExecution/Pipeline/DeploymentPipelineRunner.cs

@@ -8,11 +8,24 @@ namespace Squid.Core.Services.DeploymentExecution.Pipeline;
 public sealed class DeploymentPipelineRunner(IEnumerable<IDeploymentPipelinePhase> phases, IDeploymentLifecycle lifecycle, IDeploymentCompletionHandler completion, ITaskCancellationRegistry registry, IServerTaskDataProvider serverTaskDataProvider) : IDeploymentTaskExecutor
 {
     private const int CompletionTimeoutSeconds = 30;
-    private static readonly TimeSpan DefaultDeploymentTimeout = TimeSpan.FromMinutes(60);
     private static readonly TimeSpan DefaultConcurrencyMaxWait = TimeSpan.FromSeconds(300);
     private static readonly TimeSpan DefaultConcurrencyPollInterval = TimeSpan.FromMilliseconds(3000);
 
-    internal TimeSpan DeploymentTimeout { get; init; } = DefaultDeploymentTimeout;
+    /// <summary>
+    /// Operator escape hatch (Rule 8): maximum wall-clock minutes a single
+    /// deployment may run before the pipeline is force-cancelled and the task
+    /// fails with <see cref="DeploymentTimeoutException"/>. Unset / blank /
+    /// non-positive-integer → <see cref="DefaultDeploymentTimeoutMinutes"/>.
+    /// Raise it for long-running deployments (large DB migrations, multi-stage
+    /// rollouts) that legitimately exceed the default; leaving it unset
+    /// preserves the historical 60-minute behaviour exactly.
+    /// </summary>
+    public const string DeploymentTimeoutMinutesEnvVar = "SQUID_DEPLOYMENT_TIMEOUT_MINUTES";
+
+    internal const int DefaultDeploymentTimeoutMinutes = 60;
+    private static readonly TimeSpan DefaultDeploymentTimeout = TimeSpan.FromMinutes(DefaultDeploymentTimeoutMinutes);
+
+    internal TimeSpan DeploymentTimeout { get; init; } = ResolveDeploymentTimeout();
     internal TimeSpan ConcurrencyMaxWait { get; init; } = DefaultConcurrencyMaxWait;
     internal TimeSpan ConcurrencyPollInterval { get; init; } = DefaultConcurrencyPollInterval;
 
@@ -141,4 +154,30 @@ private async Task WaitForConcurrencySlotAsync(int serverTaskId, CancellationTok
 
         Log.Warning("[Deploy] Task {TaskId} exceeded concurrency wait timeout ({Timeout}), proceeding anyway", serverTaskId, ConcurrencyMaxWait);
     }
+
+    /// <summary>
+    /// Parses the operator-supplied deployment timeout (in minutes). Blank,
+    /// non-integer, or non-positive input falls back to the historical
+    /// <see cref="DefaultDeploymentTimeoutMinutes"/> default so a typo'd env var
+    /// can never disable the safety timer or crash construction. Pure + static
+    /// (internal, surfaced to the unit suite via InternalsVisibleTo) so the full
+    /// input matrix is testable without the pipeline.
+    /// </summary>
+    internal static TimeSpan ParseDeploymentTimeout(string raw)
+    {
+        if (string.IsNullOrWhiteSpace(raw)) return DefaultDeploymentTimeout;
+
+        if (!int.TryParse(raw.Trim(), out var minutes) || minutes <= 0)
+        {
+            Log.Warning(
+                "{EnvVar}='{RawValue}' is not a valid positive integer (minutes); falling back to default {Default} min.",
+                DeploymentTimeoutMinutesEnvVar, raw, DefaultDeploymentTimeoutMinutes);
+            return DefaultDeploymentTimeout;
+        }
+
+        return TimeSpan.FromMinutes(minutes);
+    }
+
+    private static TimeSpan ResolveDeploymentTimeout()
+        => ParseDeploymentTimeout(Environment.GetEnvironmentVariable(DeploymentTimeoutMinutesEnvVar));
 }

tests/Squid.UnitTests/Services/Deployments/Pipeline/DeploymentPipelineRunnerTimeoutTests.cs

@@ -78,6 +78,92 @@ public async Task UserCancel_DuringTimeout_TreatedAsCancellation()
         _completion.Verify(c => c.OnFailureAsync(It.IsAny<DeploymentTaskContext>(), It.IsAny<DeploymentTimeoutException>(), It.IsAny<CancellationToken>()), Times.Never);
     }
 
+    // ── Configurable deployment timeout (Rule 8 env-var escape hatch) ─────────
+    // Long-running deployments (large DB migrations, multi-stage rollouts) can
+    // legitimately exceed the 60-min default. Before this knob the timer was a
+    // hardcoded internal-init constant with no operator override, so any such
+    // deployment was killed at 60 min. The env var lets operators raise it
+    // without a code change; default behaviour (60 min when unset) is unchanged.
+
+    [Fact]
+    public void DeploymentTimeoutMinutesEnvVar_ConstantNamePinned()
+    {
+        // Operators set this in Helm overrides / container env. Renaming it
+        // silently reverts every tuned tenant back to the 60-min default.
+        DeploymentPipelineRunner.DeploymentTimeoutMinutesEnvVar
+            .ShouldBe("SQUID_DEPLOYMENT_TIMEOUT_MINUTES");
+    }
+
+    [Fact]
+    public void DefaultDeploymentTimeoutMinutes_Is60()
+    {
+        DeploymentPipelineRunner.DefaultDeploymentTimeoutMinutes.ShouldBe(60);
+    }
+
+    [Theory]
+    [InlineData(null,        60)]   // unset → default
+    [InlineData("",          60)]   // empty → default
+    [InlineData("   ",       60)]   // whitespace → default
+    [InlineData("90",        90)]   // explicit override
+    [InlineData(" 120 ",    120)]   // surrounding whitespace trimmed
+    [InlineData("+90",       90)]   // leading sign accepted by int.TryParse
+    [InlineData("0060",      60)]   // leading zeros accepted
+    [InlineData("240",      240)]   // larger override (4h migration)
+    [InlineData("60",        60)]   // explicit value equal to default still parses
+    [InlineData("not-int",   60)]   // garbage → default + warn
+    [InlineData("12.5",      60)]   // non-integer → default
+    [InlineData("0",         60)]   // zero would disable the safety timer → default
+    [InlineData("-5",        60)]   // negative → default
+    public void ParseDeploymentTimeout_HandlesAllInputs(string raw, int expectedMinutes)
+    {
+        DeploymentPipelineRunner.ParseDeploymentTimeout(raw)
+            .ShouldBe(TimeSpan.FromMinutes(expectedMinutes));
+    }
+
+    [Fact]
+    public void EnvVar_Unset_EffectiveTimeoutIs60Min()
+    {
+        var original = Environment.GetEnvironmentVariable(DeploymentPipelineRunner.DeploymentTimeoutMinutesEnvVar);
+        Environment.SetEnvironmentVariable(DeploymentPipelineRunner.DeploymentTimeoutMinutesEnvVar, null);
+
+        try
+        {
+            var runner = CreateRunnerWithoutTimeoutOverride();
+
+            runner.DeploymentTimeout.ShouldBe(TimeSpan.FromMinutes(60));
+        }
+        finally
+        {
+            Environment.SetEnvironmentVariable(DeploymentPipelineRunner.DeploymentTimeoutMinutesEnvVar, original);
+        }
+    }
+
+    [Fact]
+    public void EnvVar_Set_DrivesEffectiveDeploymentTimeout()
+    {
+        // 123 min: distinct from the 60-min default (proves env→property wiring)
+        // AND far longer than any unit test's runtime, so a parallel no-override
+        // runner construction that briefly reads this value can never time out.
+        var original = Environment.GetEnvironmentVariable(DeploymentPipelineRunner.DeploymentTimeoutMinutesEnvVar);
+        Environment.SetEnvironmentVariable(DeploymentPipelineRunner.DeploymentTimeoutMinutesEnvVar, "123");
+
+        try
+        {
+            var runner = CreateRunnerWithoutTimeoutOverride();
+
+            runner.DeploymentTimeout.ShouldBe(TimeSpan.FromMinutes(123));
+        }
+        finally
+        {
+            Environment.SetEnvironmentVariable(DeploymentPipelineRunner.DeploymentTimeoutMinutesEnvVar, original);
+        }
+    }
+
+    private DeploymentPipelineRunner CreateRunnerWithoutTimeoutOverride()
+    {
+        return new DeploymentPipelineRunner(Array.Empty<IDeploymentPipelinePhase>(), _lifecycle.Object, _completion.Object, _registry, _taskDataProvider.Object);
+    }
+
     private IDeploymentPipelinePhase CreateHangingPhase()
     {
         var phase = new Mock<IDeploymentPipelinePhase>();