diff --git a/Dockerfile b/Dockerfile index 8e142fcd1..34e11effe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -171,7 +171,22 @@ COPY --from=builder /build/q2/target/release/q2-cockpit ./q2-cockpit COPY --from=builder /build/q2/cockpit/public/aiwar_graph.json ./data/aiwar_graph.json COPY --from=builder /build/q2/cockpit/public/aiwar_weapons.json ./data/aiwar_weapons.json -HEALTHCHECK --interval=30s --timeout=3s \ +# `--start-period` is load-bearing, not decoration. On a COLD boot (empty +# volume) the OSM slab hydrates from S3 before the listener binds — measured +# ~60-90s for the 1.35 GB Berlin bake. Without a start period, Docker begins +# probing immediately and the default 3 retries at 30s intervals mark the +# container unhealthy inside ~90s, which overlaps the hydrate window exactly: +# the deploy would fail while doing precisely what it is supposed to do. +# +# Failures during the start period do not count, so this only widens the +# window for a legitimately slow FIRST start. It is not a mask for a hang: +# `ensure_slab_local` returns `None` fast on any error (no bucket, bad creds, +# missing checksum), so a long boot means a real transfer is in progress. A +# warm boot re-verifies the cached copy in ~0.8s and is unaffected. +# +# The cold path runs once per volume, not once per deploy — persisting across +# rebuilds is the volume's whole purpose. +HEALTHCHECK --interval=30s --timeout=3s --start-period=600s \ CMD curl -f http://localhost:8080/health || exit 1 ENV PORT=8080 diff --git a/claude-notes/plans/2026-08-08-osm-soa-cockpit-wiring.md b/claude-notes/plans/2026-08-08-osm-soa-cockpit-wiring.md index 0537ab919..87af6a528 100644 --- a/claude-notes/plans/2026-08-08-osm-soa-cockpit-wiring.md +++ b/claude-notes/plans/2026-08-08-osm-soa-cockpit-wiring.md @@ -1005,3 +1005,63 @@ it exactly at `q2/bakes/berlin-v1/` and touches nothing under `MedCare-rs/`. - [x] `OSM_SLAB_PATH` gains an S3 sibling: hydrate → volume, then mmap. - [x] Checksum-pin the object; no unverified-fetch path. + +### Deploy readiness — the Dockerfile already existed, and my hydrate broke it + +The root `Dockerfile` **already builds and deploys `q2-cockpit`** (Vite stage → +Rust stage → slim runtime, `PORT=8080`, siblings cloned at HEAD). No new deploy +surface was needed. Two things checked rather than assumed: + +- **Toolchain.** The Dockerfile pins `1.94.0`, but `rust-toolchain.toml` pins + `nightly-2026-04-28` and rustup honours the toml — so the build resolves to + the nightly, which satisfies `ogar-vocab`'s ≥1.95 floor. The apparent + mismatch is not one. +- **TLS.** `object_store` resolves to **rustls** (`hyper-rustls`), not + native-tls, so the new S3 leg adds no OpenSSL requirement to the runtime + image. +- **Features.** The deploy builds `--features embed-cockpit,planner`. `planner` + is already `default`, and `embed-cockpit` only gates `include_dir` for static + assets — neither touches the hydrate path. + +**⚠ The defect I introduced.** `ensure_slab_local().await` (main.rs:194) blocks +`TcpListener::bind` (main.rs:346). On a **cold** boot that is ~60-90s of +transfer before the port opens — and `HEALTHCHECK` had `--interval=30s +--timeout=3s` with **no `--start-period`**, so Docker probes immediately and +the default 3 retries mark the container unhealthy inside ~90s. The unhealthy +window overlapped the hydrate window exactly: the deploy would have failed +while doing precisely what it was supposed to do. + +Fixed with `--start-period=600s`. Failures during the start period do not +count, so this only widens the window for a legitimately slow FIRST start. It +is not masking a hang — verified, not asserted: + +``` +11:25:12 INFO osm slab: resolving from S3 (cold boot transfers ~1.35 GB …) + bucket=… prefix=q2/bakes/does-not-exist dir=…/probevol/osm +11:25:13 ERROR osm slab: SHA256SUMS not readable — 404 … in 765.929422ms + (server still starts and serves) +``` + +A misconfigured prefix costs **0.8s** and names the exact URL. Only a real +transfer is slow, and the cold path runs **once per volume**, not once per +deploy — persisting across rebuilds is the volume's whole purpose. + +Also added a boot line *before* the transfer. Without it the log is silent for +60-90s on a cold start, which is indistinguishable from a hang for whoever is +watching a deploy — the same observability gap as the `q2_cockpit` filter bug, +one layer up. + +**What Railway needs** (all already set per the operator): + +| variable | purpose | +|---|---| +| `AWS_ENDPOINT_URL` `AWS_ACCESS_KEY_ID` `AWS_SECRET_ACCESS_KEY` `AWS_DEFAULT_REGION` | read by `AmazonS3Builder::from_env()` | +| `AWS_S3_BUCKET_NAME` | set explicitly via `with_bucket_name` | +| `RAILWAY_VOL` | volume root; `/osm/` holds the cache | + +The volume needs **≥ 2 GB** (1.35 GB of artifacts plus headroom for a `.part` +during re-fetch, which transiently doubles one file). `OSM_SLAB_S3_PREFIX` +overrides the default `q2/bakes/berlin-v1`; `OSM_SLAB_CACHE_DIR` overrides the +volume root and is what made all of this testable off-Railway. + +- [x] Deploy image for the POC — already existed; made cold-boot-safe. diff --git a/crates/cockpit-server/src/osm_slab_hydrate.rs b/crates/cockpit-server/src/osm_slab_hydrate.rs index 4d92c219b..ff92e3d94 100644 --- a/crates/cockpit-server/src/osm_slab_hydrate.rs +++ b/crates/cockpit-server/src/osm_slab_hydrate.rs @@ -98,6 +98,18 @@ pub async fn ensure_slab_local() -> Option { return None; } + // Announce BEFORE the transfer, not after. This call blocks the listener + // bind, and a cold boot moves ~1.35 GB — so without a line here the boot + // log is silent for 60-90s, which is indistinguishable from a hang for + // whoever is watching a deploy. Naming the bucket and destination also + // makes a misconfigured prefix obvious from the first line rather than + // from a later "not readable" error. + tracing::info!( + %bucket, %prefix, dir = %dir.display(), + "osm slab: resolving from S3 (cold boot transfers ~1.35 GB and delays the listener; \ + a warm volume re-verifies in ~1s)" + ); + // `from_env()` reads AWS_ENDPOINT_URL / AWS_ACCESS_KEY_ID / // AWS_SECRET_ACCESS_KEY / AWS_DEFAULT_REGION with no glue — `aws_endpoint_url` // is an accepted alias for the endpoint key, so a non-AWS S3 endpoint needs