diff --git a/src/Squid.Core/Services/DeploymentExecution/Pipeline/DeploymentPipelineRunner.cs b/src/Squid.Core/Services/DeploymentExecution/Pipeline/DeploymentPipelineRunner.cs index 18c4eea5..4685d66b 100644 --- a/src/Squid.Core/Services/DeploymentExecution/Pipeline/DeploymentPipelineRunner.cs +++ b/src/Squid.Core/Services/DeploymentExecution/Pipeline/DeploymentPipelineRunner.cs @@ -8,11 +8,24 @@ namespace Squid.Core.Services.DeploymentExecution.Pipeline; public sealed class DeploymentPipelineRunner(IEnumerable phases, IDeploymentLifecycle lifecycle, IDeploymentCompletionHandler completion, ITaskCancellationRegistry registry, IServerTaskDataProvider serverTaskDataProvider) : IDeploymentTaskExecutor { private const int CompletionTimeoutSeconds = 30; - private static readonly TimeSpan DefaultDeploymentTimeout = TimeSpan.FromMinutes(60); private static readonly TimeSpan DefaultConcurrencyMaxWait = TimeSpan.FromSeconds(300); private static readonly TimeSpan DefaultConcurrencyPollInterval = TimeSpan.FromMilliseconds(3000); - internal TimeSpan DeploymentTimeout { get; init; } = DefaultDeploymentTimeout; + /// + /// Operator escape hatch (Rule 8): maximum wall-clock minutes a single + /// deployment may run before the pipeline is force-cancelled and the task + /// fails with . Unset / blank / + /// non-positive-integer → . + /// Raise it for long-running deployments (large DB migrations, multi-stage + /// rollouts) that legitimately exceed the default; leaving it unset + /// preserves the historical 60-minute behaviour exactly. + /// + public const string DeploymentTimeoutMinutesEnvVar = "SQUID_DEPLOYMENT_TIMEOUT_MINUTES"; + + internal const int DefaultDeploymentTimeoutMinutes = 60; + private static readonly TimeSpan DefaultDeploymentTimeout = TimeSpan.FromMinutes(DefaultDeploymentTimeoutMinutes); + + internal TimeSpan DeploymentTimeout { get; init; } = ResolveDeploymentTimeout(); internal TimeSpan ConcurrencyMaxWait { get; init; } = DefaultConcurrencyMaxWait; internal TimeSpan ConcurrencyPollInterval { get; init; } = DefaultConcurrencyPollInterval; @@ -141,4 +154,30 @@ private async Task WaitForConcurrencySlotAsync(int serverTaskId, CancellationTok Log.Warning("[Deploy] Task {TaskId} exceeded concurrency wait timeout ({Timeout}), proceeding anyway", serverTaskId, ConcurrencyMaxWait); } + + /// + /// Parses the operator-supplied deployment timeout (in minutes). Blank, + /// non-integer, or non-positive input falls back to the historical + /// default so a typo'd env var + /// can never disable the safety timer or crash construction. Pure + static + /// (internal, surfaced to the unit suite via InternalsVisibleTo) so the full + /// input matrix is testable without the pipeline. + /// + internal static TimeSpan ParseDeploymentTimeout(string raw) + { + if (string.IsNullOrWhiteSpace(raw)) return DefaultDeploymentTimeout; + + if (!int.TryParse(raw.Trim(), out var minutes) || minutes <= 0) + { + Log.Warning( + "{EnvVar}='{RawValue}' is not a valid positive integer (minutes); falling back to default {Default} min.", + DeploymentTimeoutMinutesEnvVar, raw, DefaultDeploymentTimeoutMinutes); + return DefaultDeploymentTimeout; + } + + return TimeSpan.FromMinutes(minutes); + } + + private static TimeSpan ResolveDeploymentTimeout() + => ParseDeploymentTimeout(Environment.GetEnvironmentVariable(DeploymentTimeoutMinutesEnvVar)); } diff --git a/tests/Squid.UnitTests/Services/Deployments/Pipeline/DeploymentPipelineRunnerTimeoutTests.cs b/tests/Squid.UnitTests/Services/Deployments/Pipeline/DeploymentPipelineRunnerTimeoutTests.cs index 7f41b7a1..c1559875 100644 --- a/tests/Squid.UnitTests/Services/Deployments/Pipeline/DeploymentPipelineRunnerTimeoutTests.cs +++ b/tests/Squid.UnitTests/Services/Deployments/Pipeline/DeploymentPipelineRunnerTimeoutTests.cs @@ -78,6 +78,92 @@ public async Task UserCancel_DuringTimeout_TreatedAsCancellation() _completion.Verify(c => c.OnFailureAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); } + // ── Configurable deployment timeout (Rule 8 env-var escape hatch) ───────── + // Long-running deployments (large DB migrations, multi-stage rollouts) can + // legitimately exceed the 60-min default. Before this knob the timer was a + // hardcoded internal-init constant with no operator override, so any such + // deployment was killed at 60 min. The env var lets operators raise it + // without a code change; default behaviour (60 min when unset) is unchanged. + + [Fact] + public void DeploymentTimeoutMinutesEnvVar_ConstantNamePinned() + { + // Operators set this in Helm overrides / container env. Renaming it + // silently reverts every tuned tenant back to the 60-min default. + DeploymentPipelineRunner.DeploymentTimeoutMinutesEnvVar + .ShouldBe("SQUID_DEPLOYMENT_TIMEOUT_MINUTES"); + } + + [Fact] + public void DefaultDeploymentTimeoutMinutes_Is60() + { + DeploymentPipelineRunner.DefaultDeploymentTimeoutMinutes.ShouldBe(60); + } + + [Theory] + [InlineData(null, 60)] // unset → default + [InlineData("", 60)] // empty → default + [InlineData(" ", 60)] // whitespace → default + [InlineData("90", 90)] // explicit override + [InlineData(" 120 ", 120)] // surrounding whitespace trimmed + [InlineData("+90", 90)] // leading sign accepted by int.TryParse + [InlineData("0060", 60)] // leading zeros accepted + [InlineData("240", 240)] // larger override (4h migration) + [InlineData("60", 60)] // explicit value equal to default still parses + [InlineData("not-int", 60)] // garbage → default + warn + [InlineData("12.5", 60)] // non-integer → default + [InlineData("0", 60)] // zero would disable the safety timer → default + [InlineData("-5", 60)] // negative → default + public void ParseDeploymentTimeout_HandlesAllInputs(string raw, int expectedMinutes) + { + DeploymentPipelineRunner.ParseDeploymentTimeout(raw) + .ShouldBe(TimeSpan.FromMinutes(expectedMinutes)); + } + + [Fact] + public void EnvVar_Unset_EffectiveTimeoutIs60Min() + { + var original = Environment.GetEnvironmentVariable(DeploymentPipelineRunner.DeploymentTimeoutMinutesEnvVar); + Environment.SetEnvironmentVariable(DeploymentPipelineRunner.DeploymentTimeoutMinutesEnvVar, null); + + try + { + var runner = CreateRunnerWithoutTimeoutOverride(); + + runner.DeploymentTimeout.ShouldBe(TimeSpan.FromMinutes(60)); + } + finally + { + Environment.SetEnvironmentVariable(DeploymentPipelineRunner.DeploymentTimeoutMinutesEnvVar, original); + } + } + + [Fact] + public void EnvVar_Set_DrivesEffectiveDeploymentTimeout() + { + // 123 min: distinct from the 60-min default (proves env→property wiring) + // AND far longer than any unit test's runtime, so a parallel no-override + // runner construction that briefly reads this value can never time out. + var original = Environment.GetEnvironmentVariable(DeploymentPipelineRunner.DeploymentTimeoutMinutesEnvVar); + Environment.SetEnvironmentVariable(DeploymentPipelineRunner.DeploymentTimeoutMinutesEnvVar, "123"); + + try + { + var runner = CreateRunnerWithoutTimeoutOverride(); + + runner.DeploymentTimeout.ShouldBe(TimeSpan.FromMinutes(123)); + } + finally + { + Environment.SetEnvironmentVariable(DeploymentPipelineRunner.DeploymentTimeoutMinutesEnvVar, original); + } + } + + private DeploymentPipelineRunner CreateRunnerWithoutTimeoutOverride() + { + return new DeploymentPipelineRunner(Array.Empty(), _lifecycle.Object, _completion.Object, _registry, _taskDataProvider.Object); + } + private IDeploymentPipelinePhase CreateHangingPhase() { var phase = new Mock();