Expose a VM's SSH host keys in the customer API

API_CHANGELOG.md

@@ -24,6 +24,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 - **Contact form hardens email header values** — `POST /api/v1/contact` strips control characters from the sender name and subject (and caps the subject) before placing them in outgoing mail headers.
 
+### Added
+
+- **`host_ssh_keys` on VM status** (issue #154) — `GET /api/v1/vm/{id}` and `GET /api/v1/vm` now return the VM's own SSH host keys as `[{ key_type, public_key, fingerprint_sha256 }]`, so a client can verify the host on first connect instead of accepting whatever key answers. The list is empty until the keys are captured (scanned from the host once the VM is running, public keys only) and is re-captured after a reinstall, which regenerates them. Additive: no existing field changed. Not to be confused with `ssh_key`, which is the customer's authorized key.
+
 ### Fixed
 
 - **`GET /api/v1/payment/{id}` now works for every subscription type** — it previously resolved the payer through the VM back-reference, so a payment for a managed app, IP range or other non-VPS subscription errored instead of returning. Ownership is now asserted on the subscription and the embedded `vm_id` is `0` for non-VPS payments (the endpoint remains deprecated in favour of `GET /api/v1/subscriptions/{id}/payments/{payment_id}`).

API_DOCUMENTATION.md

@@ -256,6 +256,13 @@ interface VmStatus {
   host_sunset_date?: string; // ISO 8601 datetime — set when the VM's host is being decommissioned; migrate before this date. Renewals are blocked once expires reaches it. Omitted when the host is not being sunset
   max_prepay_days: number; // Max days this VM may be prepaid/renewed in advance. A renewal is rejected once it would push `expires` beyond now + max_prepay_days; cap the renewal interval selector accordingly
   cpu_arch?: string; // CPU architecture of the host this VM runs on ("x86_64" | "arm64"), from the host record. Unlike template.cpu_arch (an optional constraint) this is present whenever the host arch is known; use it to always pass ?arch= when listing OS images for a reinstall. Omitted when unknown
+  host_ssh_keys: VmHostKey[]; // The VM's own SSH host keys, for verifying the host on first connect. Empty until captured after first boot; re-captured after a reinstall. Not the customer's authorized key (that is ssh_key)
+}
+
+interface VmHostKey {
+  key_type: string;           // "ssh-ed25519" | "ssh-rsa" | "ecdsa-sha2-nistp256" | "ecdsa-sha2-nistp384" | "ecdsa-sha2-nistp521"
+  public_key: string;         // base64 key blob, as in the third field of a known_hosts line
+  fingerprint_sha256: string; // "SHA256:…", matching `ssh-keygen -lf` and the banner ssh prints for an unknown host
 }
 
 interface VmRunningState {

lnvps_api/src/api/routes.rs

@@ -2991,6 +2991,7 @@ mod tests {
                     ssh_key_id: Some(1),
                     disk_id: 1,
                     mac_address: "ff:ff:ff:ff:ff:ff".to_string(),
+                    ssh_host_keys: None,
                     deleted: false,
                     ref_code: None,
                     disabled: false,

lnvps_api/src/host/mod.rs

@@ -384,6 +384,7 @@ mod tests {
                 ssh_key_id: Some(1),
                 disk_id: 1,
                 mac_address: "ff:ff:ff:ff:ff:fe".to_string(),
+                ssh_host_keys: None,
                 deleted: false,
                 ref_code: None,
                 disabled: false,

lnvps_api/src/provisioner/integration_retry_tests.rs

@@ -198,6 +198,7 @@ mod tests {
             ssh_key_id: Some(ssh_key.id),
             disk_id: 1,
             mac_address: "bc:24:11:00:00:01".to_string(),
+            ssh_host_keys: None,
             deleted: false,
             ref_code: None,
             disabled: false,
@@ -245,6 +246,7 @@ mod tests {
             ssh_key_id: Some(ssh_key.id),
             disk_id: 1,
             mac_address: "bc:24:11:00:00:01".to_string(),
+            ssh_host_keys: None,
             deleted: false,
             ref_code: None,
             disabled: false,
@@ -287,6 +289,7 @@ mod tests {
             ssh_key_id: Some(ssh_key.id),
             disk_id: 1,
             mac_address: "bc:24:11:00:00:01".to_string(),
+            ssh_host_keys: None,
             deleted: false,
             ref_code: None,
             disabled: false,

lnvps_api/src/provisioner/retry_tests.rs

@@ -217,6 +217,7 @@ mod tests {
                 ssh_key_id: Some(ssh_key.id),
                 disk_id: 1,
                 mac_address: "bc:24:11:00:00:01".to_string(),
+                ssh_host_keys: None,
                 deleted: false,
                 ref_code: None,
                 disabled: false,

lnvps_api/src/provisioner/rollback_tests.rs

@@ -684,6 +684,7 @@ mod tests {
             subscription_line_item_id: 0,
             disk_id: 1,
             mac_address: "02:00:00:00:00:01".to_string(), // A valid MAC
+            ssh_host_keys: None,
             ref_code: None,
             deleted: false,
             disabled: false,

lnvps_api/src/provisioner/vm.rs

@@ -191,6 +191,7 @@ impl VmProvisioner {
             ssh_key_id: Some(ssh_key.id),
             disk_id: pick_disk.disk.id,
             mac_address: "ff:ff:ff:ff:ff:ff".to_string(),
+            ssh_host_keys: None,
             deleted: false,
             ref_code,
             disabled: false,
@@ -327,6 +328,7 @@ impl VmProvisioner {
             ssh_key_id: Some(ssh_key.id),
             disk_id: pick_disk.disk.id,
             mac_address: "ff:ff:ff:ff:ff:ff".to_string(),
+            ssh_host_keys: None,
             deleted: false,
             ref_code,
             disabled: false,
@@ -504,6 +506,7 @@ impl VmProvisioner {
                 .mac_address
                 .clone()
                 .unwrap_or_else(|| "ff:ff:ff:ff:ff:ff".to_string()),
+            ssh_host_keys: None,
             deleted: false,
             ref_code: None,
             disabled: false,

lnvps_api/src/worker.rs

@@ -10,8 +10,9 @@ use hickory_resolver::TokioResolver;
 use lnvps_api_common::{
     BlackholeWorkFeedback, ChannelWorkCommander, InMemoryKeyValueStore, JobFeedback, KeyValueStore,
     NetworkProvisioner, RedisConfig, RedisKeyValueStore, RedisWorkCommander, RedisWorkFeedback,
-    UpgradeConfig, VmHistoryLogger, VmRunningState, VmStateCache, WorkCommander, WorkFeedback,
-    WorkJob, WorkJobMessage, op_fatal,
+    SCANNED_KEY_FAMILIES, UpgradeConfig, VmHistoryLogger, VmRunningState, VmRunningStates,
+    VmStateCache, WorkCommander, WorkFeedback, WorkJob, WorkJobMessage, capture_is_complete,
+    merge_ssh_host_keys, op_fatal, parse_ssh_host_keys,
     retry::{OpError, Pipeline, RetryPolicy},
 };
 use lnvps_db::{
@@ -69,6 +70,15 @@ fn payment_blocks_unpaid_vm_deletion(p: &SubscriptionPayment, now: DateTime<Utc>
             || (p.payment_method == PaymentMethod::OnChain && p.external_id.is_some()))
 }
 
+/// How long to wait before scanning a VM's host keys again while the capture is
+/// still missing keys.
+const HOST_KEY_SCAN_RETRY_SECS: u64 = 3600;
+
+/// Key holding when a VM's host keys were last scanned.
+fn host_key_attempt_key(vm_id: u64) -> String {
+    format!("worker-host-keys-attempt-{vm_id}")
+}
+
 /// Extract hostname/IP from a URL or return the input if it's already a plain host
 /// e.g. "https://192.168.1.1:8006/" -> "192.168.1.1"
 ///      "192.168.1.1" -> "192.168.1.1"
@@ -894,9 +904,146 @@ impl Worker {
         )
         .await?;
         self.reconcile_vm_dns(vm).await;
+        self.capture_vm_ssh_host_keys(vm).await;
         Ok(())
     }
 
+    /// Best-effort capture of a VM's SSH host keys, so a customer can verify
+    /// the host on first connect instead of trusting whatever key answers.
+    ///
+    /// Scanned from the Proxmox node rather than from here: the VM's address is
+    /// often not routable from the API, and the node is already trusted with
+    /// the VM. Only public keys are read — nothing runs inside the guest.
+    ///
+    /// Runs on the periodic VM check rather than at spawn: the keys do not
+    /// exist until cloud-init has generated them and sshd is up, and a VM whose
+    /// keys were never captured (or were cleared by a reinstall) self-heals on
+    /// the next pass. A VM whose capture already covers every algorithm is
+    /// skipped, so this costs nothing for a healthy VM.
+    async fn capture_vm_ssh_host_keys(&self, vm: &Vm) {
+        if vm.deleted {
+            return;
+        }
+        let captured = vm
+            .ssh_host_keys
+            .as_deref()
+            .map(parse_ssh_host_keys)
+            .unwrap_or_default();
+        if capture_is_complete(&captured) {
+            return;
+        }
+        if !matches!(
+            self.vm_state_cache.get_state(vm.id).await.map(|s| s.state),
+            Some(VmRunningStates::Running)
+        ) {
+            return;
+        }
+        let Some(ip) = self.vm_scan_address(vm).await else {
+            return;
+        };
+        // A guest that blocks port 22, never runs sshd, or offers fewer
+        // algorithms than a scan asks for would otherwise be scanned on every
+        // check for the life of the VM. One attempt per hour still captures
+        // the keys within an hour of the guest becoming reachable.
+        let attempt_key = host_key_attempt_key(vm.id);
+        if let Ok(Some(v)) = self.kv.get(&attempt_key).await
+            && v.len() == 8
+        {
+            let last = u64::from_le_bytes(v.as_slice().try_into().unwrap_or_default());
+            // A clock stepped backwards leaves a stamp in the future; wait it
+            // out rather than panic.
+            let waited = (Utc::now().timestamp() as u64).saturating_sub(last);
+            if waited < HOST_KEY_SCAN_RETRY_SECS {
+                return;
+            }
+        }
+        let now = Utc::now().timestamp() as u64;
+        if let Err(e) = self.kv.store(&attempt_key, &now.to_le_bytes()).await {
+            warn!("[host-keys] vm {}: failed to record attempt: {}", vm.id, e);
+        }
+        let host = match self.db.get_host(vm.host_id).await {
+            Ok(h) => h,
+            Err(e) => {
+                warn!("[host-keys] vm {}: host lookup failed: {}", vm.id, e);
+                return;
+            }
+        };
+        let Some(ssh_key) = host.ssh_key.as_ref() else {
+            return;
+        };
+        let ssh_user = host.ssh_user.as_deref().unwrap_or("root");
+
+        let mut ssh = match SshClient::new() {
+            Ok(c) => c,
+            Err(e) => {
+                warn!("[host-keys] vm {}: ssh client failed: {}", vm.id, e);
+                return;
+            }
+        };
+        let ssh_host = extract_host_from_url(&host.ip);
+        if let Err(e) = ssh
+            .connect_with_key((ssh_host.as_str(), 22), ssh_user, ssh_key.as_str())
+            .await
+        {
+            warn!(
+                "[host-keys] vm {}: connect to {} failed: {}",
+                vm.id, host.name, e
+            );
+            return;
+        }
+        // Bounded so an unreachable or half-open guest cannot hold the check.
+        let scan = match ssh
+            .execute(&format!(
+                "ssh-keyscan -T 5 -t {} {ip}",
+                SCANNED_KEY_FAMILIES.join(",")
+            ))
+            .await
+        {
+            Ok((_, out)) => out,
+            Err(e) => {
+                warn!("[host-keys] vm {}: keyscan failed: {}", vm.id, e);
+                return;
+            }
+        };
+        // A non-zero exit still prints the keys it did get, so the output is
+        // what decides. A scan opens one connection per algorithm and can time
+        // out on some of them, so what came back is merged into what is stored
+        // rather than replacing it, and a short capture is scanned again later.
+        if parse_ssh_host_keys(&scan).is_empty() {
+            debug!("[host-keys] vm {}: no keys in scan of {}", vm.id, ip);
+            return;
+        }
+        let merged = merge_ssh_host_keys(&ip, vm.ssh_host_keys.as_deref(), &scan);
+        if Some(merged.as_str()) == vm.ssh_host_keys.as_deref() {
+            return;
+        }
+        if let Err(e) = self.db.set_vm_ssh_host_keys(vm.id, Some(&merged)).await {
+            warn!("[host-keys] vm {}: failed to store keys: {}", vm.id, e);
+        }
+    }
+
+    /// The address to scan a VM's host keys on: its first assigned IP, v4
+    /// preferred because a node without IPv6 egress cannot reach the other.
+    async fn vm_scan_address(&self, vm: &Vm) -> Option<String> {
+        // Assignments are stored as CIDR; ssh-keyscan wants the bare address.
+        // Parsed rather than trimmed: the result is interpolated into a command
+        // on the host, so only something that is definitely an address goes in.
+        let addrs: Vec<std::net::IpAddr> = self
+            .db
+            .list_vm_ip_assignments(vm.id)
+            .await
+            .ok()?
+            .into_iter()
+            .filter(|i| !i.deleted)
+            .filter_map(|i| i.ip.split('/').next()?.parse().ok())
+            .collect();
+        addrs
+            .iter()
+            .find(|a| a.is_ipv4())
+            .or_else(|| addrs.first())
+            .map(|a| a.to_string())
+    }
+
     /// Best-effort reconciliation of missing DNS records for a VM's IPs.
     ///
     /// DNS is best-effort during spawn (a failed forward/reverse record must not
@@ -2364,6 +2511,28 @@ impl Worker {
 
                 let feedback = match &result {
                     Ok(()) => {
+                        // The guest generates fresh host keys on reinstall, so
+                        // the stored ones now belong to an image that is gone;
+                        // clearing them makes the next check re-capture. The
+                        // scan-attempt stamp goes with them, or the customer
+                        // would be shown no keys until the retry window passed
+                        // — exactly when they are looking for the fingerprint.
+                        if let Err(e) = self.db.set_vm_ssh_host_keys(*vm_id, None).await {
+                            warn!(
+                                "Failed to clear ssh host keys after reinstall of VM {}: {}",
+                                vm_id, e
+                            );
+                        }
+                        if let Err(e) = self
+                            .kv
+                            .store(&host_key_attempt_key(*vm_id), &0u64.to_le_bytes())
+                            .await
+                        {
+                            warn!(
+                                "Failed to reset ssh host key scan stamp for VM {}: {}",
+                                vm_id, e
+                            );
+                        }
                         // Record history + refresh cached state only on success.
                         self.vm_history_logger
                             .log_vm_reinstalled(

lnvps_api_admin/src/bin/generate_demo_data.rs

@@ -1206,6 +1206,7 @@ async fn create_vms(
             ssh_key_id: Some(ssh_key.id),
             disk_id: disk.id,
             mac_address: mac_address.clone(),
+            ssh_host_keys: None,
             deleted: false,
             ref_code: ref_code.clone(),
             disabled: false,
@@ -1253,6 +1254,7 @@ async fn create_vms(
             ssh_key_id: Some(ssh_key.id),
             disk_id: disk.id,
             mac_address: mac_address.clone(),
+            ssh_host_keys: None,
             deleted: false,
             ref_code: None,
             disabled: false,

lnvps_api_common/src/lib.rs

@@ -19,6 +19,7 @@ pub mod retry;
 mod routes;
 mod session;
 pub mod shasum;
+mod ssh_host_key;
 mod status;
 mod vat;
 mod vm_history;
@@ -43,6 +44,7 @@ pub use registry::*;
 pub use routes::*;
 use serde::{Deserialize, Deserializer};
 pub use session::*;
+pub use ssh_host_key::*;
 pub use status::*;
 pub use vat::*;
 pub use vm_history::*;