diff --git a/Cargo.lock b/Cargo.lock index 880989d1f..3868841ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,7 +85,7 @@ dependencies = [ [[package]] name = "alien-agent" -version = "1.10.0" +version = "1.10.1" dependencies = [ "aegis", "alien-client-config", @@ -127,7 +127,7 @@ dependencies = [ [[package]] name = "alien-aws-clients" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-aws-clients", "alien-client-core", @@ -169,7 +169,7 @@ dependencies = [ [[package]] name = "alien-azure-clients" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-client-core", "alien-core", @@ -212,7 +212,7 @@ dependencies = [ [[package]] name = "alien-bindings" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-aws-clients", "alien-azure-clients", @@ -262,7 +262,7 @@ dependencies = [ [[package]] name = "alien-build" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-build", "alien-core", @@ -296,7 +296,7 @@ dependencies = [ [[package]] name = "alien-cli" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-bindings", "alien-build", @@ -365,7 +365,7 @@ dependencies = [ [[package]] name = "alien-cli-common" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-core", "alien-deployment", @@ -379,7 +379,7 @@ dependencies = [ [[package]] name = "alien-client-config" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-aws-clients", "alien-azure-clients", @@ -397,7 +397,7 @@ dependencies = [ [[package]] name = "alien-client-core" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-error", "anyhow", @@ -416,7 +416,7 @@ dependencies = [ [[package]] name = "alien-cloudformation" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-core", "alien-error", @@ -431,7 +431,7 @@ dependencies = [ [[package]] name = "alien-commands" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-aws-clients", "alien-azure-clients", @@ -465,7 +465,7 @@ dependencies = [ [[package]] name = "alien-commands-client" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-core", "base64 0.22.1", @@ -480,7 +480,7 @@ dependencies = [ [[package]] name = "alien-core" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-error", "alien-macros", @@ -511,7 +511,7 @@ dependencies = [ [[package]] name = "alien-deploy-cli" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-agent", "alien-cli-common", @@ -549,7 +549,7 @@ dependencies = [ [[package]] name = "alien-deployment" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-aws-clients", "alien-azure-clients", @@ -579,7 +579,7 @@ dependencies = [ [[package]] name = "alien-error" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-error-derive", "anyhow", @@ -592,7 +592,7 @@ dependencies = [ [[package]] name = "alien-error-derive" -version = "1.10.0" +version = "1.10.1" dependencies = [ "proc-macro2", "quote", @@ -602,7 +602,7 @@ dependencies = [ [[package]] name = "alien-gcp-clients" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-client-core", "alien-core", @@ -635,7 +635,7 @@ dependencies = [ [[package]] name = "alien-helm" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-core", "alien-error", @@ -649,7 +649,7 @@ dependencies = [ [[package]] name = "alien-infra" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-aws-clients", "alien-azure-clients", @@ -709,7 +709,7 @@ dependencies = [ [[package]] name = "alien-k8s-clients" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-client-core", "alien-core", @@ -738,7 +738,7 @@ dependencies = [ [[package]] name = "alien-local" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-bindings", "alien-build", @@ -775,7 +775,7 @@ dependencies = [ [[package]] name = "alien-macros" -version = "1.10.0" +version = "1.10.1" dependencies = [ "proc-macro2", "quote", @@ -784,7 +784,7 @@ dependencies = [ [[package]] name = "alien-manager" -version = "1.10.0" +version = "1.10.1" dependencies = [ "aegis", "alien-bindings", @@ -843,7 +843,7 @@ dependencies = [ [[package]] name = "alien-manager-api" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-error", "chrono", @@ -860,7 +860,7 @@ dependencies = [ [[package]] name = "alien-permissions" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-core", "alien-error", @@ -879,7 +879,7 @@ dependencies = [ [[package]] name = "alien-platform-api" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-error", "chrono", @@ -896,7 +896,7 @@ dependencies = [ [[package]] name = "alien-preflights" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-core", "alien-error", @@ -913,7 +913,7 @@ dependencies = [ [[package]] name = "alien-runtime" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-bindings", "alien-commands", @@ -971,14 +971,14 @@ dependencies = [ [[package]] name = "alien-sdk" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-bindings", ] [[package]] name = "alien-terraform" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-core", "alien-error", @@ -993,7 +993,7 @@ dependencies = [ [[package]] name = "alien-test" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-aws-clients", "alien-azure-clients", @@ -1039,7 +1039,7 @@ dependencies = [ [[package]] name = "alien-test-app" -version = "1.10.0" +version = "1.10.1" dependencies = [ "alien-bindings", "alien-error", diff --git a/crates/alien-deploy-cli/src/commands/up.rs b/crates/alien-deploy-cli/src/commands/up.rs index 438483aef..fd3f145c6 100644 --- a/crates/alien-deploy-cli/src/commands/up.rs +++ b/crates/alien-deploy-cli/src/commands/up.rs @@ -21,7 +21,7 @@ use alien_deployment::{ loop_contract::{LoopOperation, LoopOutcome, LoopResult, LoopStopReason}, manager_api_transport::{ acquire_setup_delete_deployment, acquire_setup_run_deployment, final_reconcile, - release_deployment, ManagerApiTransport, + release_deployment, ManagerApiTransport, SetupDeleteAcquireOutcome, }, runner::{run_step_loop as shared_run_step_loop, RunnerPolicy, RunnerResult}, }; @@ -3054,12 +3054,17 @@ pub async fn push_deletion( // Acquire sync lock with retry let session = format!("push-deletion-{}", uuid::Uuid::new_v4()); - acquire_setup_delete_deployment(client, deployment_id, &session) + let acquire_outcome = acquire_setup_delete_deployment(client, deployment_id, &session) .await .context(ErrorData::DeploymentFailed { operation: "acquire sync lock for deletion".to_string(), })?; + if matches!(acquire_outcome, SetupDeleteAcquireOutcome::AlreadyDeleted) { + output::success("Deployment deleted successfully."); + return Ok(()); + } + // Re-fetch deployment under lock let deployment = client .get_deployment() diff --git a/crates/alien-deployment/src/manager_api_transport.rs b/crates/alien-deployment/src/manager_api_transport.rs index 6c0a03731..dd4fb24ee 100644 --- a/crates/alien-deployment/src/manager_api_transport.rs +++ b/crates/alien-deployment/src/manager_api_transport.rs @@ -153,9 +153,19 @@ fn to_manager_api_heartbeats( /// Maximum number of acquire attempts (60 × 2s = 2 minutes). const MAX_ACQUIRE_ATTEMPTS: usize = 60; +/// Maximum number of setup delete handoff attempts (1,350 × 2s = 45 minutes). +const MAX_SETUP_DELETE_ACQUIRE_ATTEMPTS: usize = 1_350; /// Delay between acquire attempts in seconds. const ACQUIRE_RETRY_DELAY_SECS: u64 = 2; +/// Result of waiting for setup-owned deletion work. +pub enum SetupDeleteAcquireOutcome { + /// The setup teardown lock was acquired and must be released. + Acquired, + /// Runtime cleanup already deleted the deployment record. + AlreadyDeleted, +} + /// Acquire a deployment lock from the manager, retrying until the lock is granted /// or the timeout is reached. /// @@ -216,22 +226,80 @@ pub async fn acquire_setup_delete_deployment( client: &ManagerClient, deployment_id: &str, session: &str, -) -> Result<(), AlienError> { - acquire_deployment_with_statuses( - client, - deployment_id, - session, - Some("setup-teardown".to_string()), - Some("cli".to_string()), - Some(vec![ - "delete-pending".to_string(), - "deleting".to_string(), - "teardown-required".to_string(), - "teardown-failed".to_string(), - ]), - ) - .await - .map(|_| ()) +) -> Result { + let statuses = vec![ + "teardown-required".to_string(), + "teardown-failed".to_string(), + ]; + + for attempt in 1..=MAX_SETUP_DELETE_ACQUIRE_ATTEMPTS { + let resp = client + .acquire() + .body(alien_manager_api::types::AcquireRequest { + acquire_mode: Some("setup-teardown".to_string()), + session: session.to_string(), + deployment_ids: Some(vec![deployment_id.to_string()]), + setup_method: Some("cli".to_string()), + statuses: Some(statuses.clone()), + platforms: None, + limit: None, + }) + .send() + .await + .into_sdk_error() + .context(alien_error::GenericError { + message: "Failed to acquire setup teardown sync lock".to_string(), + })?; + + if resp.into_inner().deployments.into_iter().next().is_some() { + return Ok(SetupDeleteAcquireOutcome::Acquired); + } + + let status = match client.get_deployment().id(deployment_id).send().await { + Ok(resp) => resp.into_inner().status, + Err(err) => { + let message = err.to_string(); + if message.contains("404") || message.contains("not found") { + return Ok(SetupDeleteAcquireOutcome::AlreadyDeleted); + } + return Err(AlienError::new(alien_error::GenericError { + message: format!( + "Failed to read deployment while waiting for setup teardown: {message}" + ), + })); + } + }; + + match status.as_str() { + "teardown-required" | "teardown-failed" => {} + "deleted" => return Ok(SetupDeleteAcquireOutcome::AlreadyDeleted), + "delete-failed" => { + return Err(AlienError::new(alien_error::GenericError { + message: + "Runtime deletion failed before setup teardown became available. Retry destroy after resolving the runtime cleanup failure." + .to_string(), + })); + } + _ => {} + } + + if attempt == MAX_SETUP_DELETE_ACQUIRE_ATTEMPTS { + return Err(AlienError::new(alien_error::GenericError { + message: "Timed out waiting for runtime cleanup to reach setup teardown handoff" + .to_string(), + })); + } + + info!( + attempt = attempt, + max = MAX_SETUP_DELETE_ACQUIRE_ATTEMPTS, + status = %status, + "Waiting for runtime cleanup handoff before setup teardown" + ); + tokio::time::sleep(std::time::Duration::from_secs(ACQUIRE_RETRY_DELAY_SECS)).await; + } + + unreachable!() } async fn acquire_deployment_with_statuses(