diff --git a/apps/loopover-ui/src/lib/selfhost-env-reference.ts b/apps/loopover-ui/src/lib/selfhost-env-reference.ts index f0451f4148..dd6b6992a9 100644 --- a/apps/loopover-ui/src/lib/selfhost-env-reference.ts +++ b/apps/loopover-ui/src/lib/selfhost-env-reference.ts @@ -129,6 +129,10 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ name: "CODEX_HOME", firstReference: "src/selfhost/ai.ts", }, + { + name: "CONFIG_DIR_EMPTY_ACKNOWLEDGED", + firstReference: "src/server.ts", + }, { name: "CRON_INTERVAL_MS", firstReference: "src/server.ts", @@ -513,6 +517,7 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `CODEX_AI_MODEL` | `src/selfhost/ai.ts` |", "| `CODEX_AI_TIMEOUT_MS` | `src/selfhost/ai.ts` |", "| `CODEX_HOME` | `src/selfhost/ai.ts` |", + "| `CONFIG_DIR_EMPTY_ACKNOWLEDGED` | `src/server.ts` |", "| `CRON_INTERVAL_MS` | `src/server.ts` |", "| `DATABASE_PATH` | `src/server.ts` |", "| `DATABASE_URL` | `src/selfhost/preflight.ts` |", diff --git a/apps/loopover-ui/src/routes/docs.self-hosting-operations.tsx b/apps/loopover-ui/src/routes/docs.self-hosting-operations.tsx index 49ddc7965c..bcd9442e4f 100644 --- a/apps/loopover-ui/src/routes/docs.self-hosting-operations.tsx +++ b/apps/loopover-ui/src/routes/docs.self-hosting-operations.tsx @@ -1142,7 +1142,8 @@ git merge --ff-only origin/main curl -sf http://localhost:8787/ready docker compose ps loopover grep -E '^(LOOPOVER_IMAGE|LOOPOVER_VERSION|SENTRY_RELEASE)=' .env -docker inspect --format '{{.Config.Image}}' "$(docker compose ps -q loopover)"`} +docker inspect --format '{{.Config.Image}}' "$(docker compose ps -q loopover)" +docker exec "$(docker compose ps -q loopover)" sh -c 'ls -A "\${LOOPOVER_REPO_CONFIG_DIR:-/config}" | wc -l'`} />

If any check fails, see Troubleshooting diff --git a/scripts/selfhost-post-update-check.sh b/scripts/selfhost-post-update-check.sh index 0adc4012e8..f697d909bd 100755 --- a/scripts/selfhost-post-update-check.sh +++ b/scripts/selfhost-post-update-check.sh @@ -4,7 +4,8 @@ # Run after deploy-selfhost-image.sh, deploy-selfhost-prebuilt.sh, or any manual # `docker compose up -d --no-deps loopover` that ships a new app image. # -# Checks /ready, compose health, .env release metadata, and the running container image. +# Checks /ready, compose health, .env release metadata, the running container image, and whether the +# container-private config mount (LOOPOVER_REPO_CONFIG_DIR) is unexpectedly empty. # Does not modify .env, volumes, loopover-config/, or any profile service. set -euo pipefail @@ -93,4 +94,18 @@ fi running_image="$(docker inspect --format '{{.Config.Image}}' "$container_id")" echo "selfhost post-update check: running image=$running_image" + +# Config-drift guardrail (a live incident during the gittensory->loopover rename): docker-compose.yml's +# LOOPOVER_REPO_CONFIG_DIR bind mount silently degrades to an empty directory -- not an error -- when its host +# source directory doesn't exist (e.g. renamed/moved without updating the mount, or simply never created). Every +# per-repo and global setting then falls back to built-in defaults with zero visible symptoms until someone +# notices the behavior change. This is a READ-ONLY check (matches the file header above: never modifies +# loopover-config/) run every time this script runs, i.e. after every deploy -- exactly when a mount-path change +# would land. Non-fatal: an empty mount is also the correct, expected state for a fresh install with no private +# config written yet, so this warns rather than exits non-zero. +config_dir_entries="$(docker exec "$container_id" sh -c 'dir="${LOOPOVER_REPO_CONFIG_DIR:-/config}"; [ -d "$dir" ] && ls -A "$dir" | wc -l || echo 0' 2>/dev/null || true)" +if [[ "$config_dir_entries" =~ ^[0-9]+$ ]] && [ "$config_dir_entries" -eq 0 ]; then + echo "selfhost post-update check: warning — the container's private config directory (LOOPOVER_REPO_CONFIG_DIR, default /config) is empty; every per-repo and global setting is silently using built-in defaults. If you expected private config to apply, verify the host directory wasn't renamed or moved without updating docker-compose.yml's bind mount." >&2 +fi + echo "selfhost post-update check: ok" diff --git a/src/selfhost/health.ts b/src/selfhost/health.ts index 9f495f6a48..2036e5a728 100644 --- a/src/selfhost/health.ts +++ b/src/selfhost/health.ts @@ -231,3 +231,26 @@ export function publicOriginAcknowledgedGaugeValue(opts: { }): 0 | 1 { return publicOriginReachabilityAdvisory(opts) === null ? 1 : 0; } + +/** Boot-time advisory (a live incident during the gittensory->loopover rename): `LOOPOVER_REPO_CONFIG_DIR` + * points the focus-manifest loader at a container-private per-repo config mount (`private-config.ts`) that + * silently and validly degrades to "no local config" when the mounted directory is empty — every setting + * (labels, gate, autonomy, ...) then falls back to built-in defaults with NO error, because an empty mount is + * also the correct, expected state for a brand-new install that hasn't written any `.loopover.yml` yet. The + * incident: a docker-compose.yml change renamed the bind-mount source directory convention + * (`./gittensory-config` -> `./loopover-config`) as a documented breaking change requiring operators to `mv` + * their existing directory to match — that manual step was missed on deploy, Docker silently created an empty + * directory at the new path, and every repo's config-driven settings (including `autoLabelEnabled`) reverted + * to defaults for about a day before anyone noticed. Mirrors {@link sqliteBackupAdvisory}'s shape: warns + * rather than blocks (an empty dir is legitimate for a fresh install), and the operator can silence it with + * `CONFIG_DIR_EMPTY_ACKNOWLEDGED=true` once they've confirmed it's intentional. */ +export function emptyConfigDirAdvisory(opts: { configured: boolean; entryCount: number; acknowledged: boolean }): string | null { + if (!opts.configured || opts.acknowledged || opts.entryCount > 0) return null; + return `LOOPOVER_REPO_CONFIG_DIR is set but the mounted directory is empty — every per-repo and global setting (labels, gate, autonomy, ...) is silently using built-in defaults instead of your .loopover.yml config. This usually means the host directory was renamed or moved without updating the bind mount (see docker-compose.yml's "volumes:" comment), or the volume didn't mount as expected. If this is intentional — a fresh install with no config written yet — set CONFIG_DIR_EMPTY_ACKNOWLEDGED=true to silence this warning.`; +} + +/** Prometheus gauge value mirroring {@link emptyConfigDirAdvisory}: 1 when the mount isn't configured, has + * entries, or the operator acknowledged it, 0 when the advisory would fire. */ +export function emptyConfigDirAcknowledgedGaugeValue(opts: { configured: boolean; entryCount: number; acknowledged: boolean }): 0 | 1 { + return emptyConfigDirAdvisory(opts) === null ? 1 : 0; +} diff --git a/src/selfhost/metrics.ts b/src/selfhost/metrics.ts index 9561838c1a..0f7f52f94a 100644 --- a/src/selfhost/metrics.ts +++ b/src/selfhost/metrics.ts @@ -54,6 +54,7 @@ export const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [ ["loopover_clock_skew_seconds", { help: "Clock skew in seconds between this process and GitHub's server time (positive = ahead), sampled from GitHub App JWT-mint response Date headers.", type: "gauge" }], ["loopover_uptime_seconds", { help: "Self-host process uptime in seconds.", type: "gauge" }], ["loopover_backup_acknowledged", { help: "1 when SQLite backup is acknowledged or Postgres is in use; 0 when the boot backup advisory would fire.", type: "gauge" }], + ["loopover_config_dir_empty_acknowledged", { help: "1 when LOOPOVER_REPO_CONFIG_DIR is unset, has entries, or is acknowledged; 0 when it's configured but the mounted directory is empty.", type: "gauge" }], ["loopover_http_requests_total", { help: "HTTP app requests by response status class.", type: "counter" }], ["loopover_http_request_duration_seconds", { help: "HTTP app request duration in seconds.", type: "histogram" }], ["loopover_webhook_dedup_total", { help: "Webhook deliveries deduplicated before enqueue.", type: "counter" }], diff --git a/src/server.ts b/src/server.ts index 9a625f90ef..1584a10dda 100644 --- a/src/server.ts +++ b/src/server.ts @@ -6,7 +6,7 @@ // Serves the Hono app via @hono/node-server, drives the queue with the same processJob, ticks the same // scheduled handler on a timer, exposes /health /ready /metrics, and shuts down gracefully. The Cloudflare // Worker (src/index.ts) is untouched — this is a parallel entry the self-host esbuild build bundles. -import { existsSync, writeFileSync } from "node:fs"; +import { existsSync, readdirSync, writeFileSync } from "node:fs"; import { delimiter, join } from "node:path"; import { randomUUID } from "node:crypto"; import { DatabaseSync } from "node:sqlite"; @@ -48,6 +48,8 @@ import { backupAcknowledgedGaugeValue, buildHealthBody, codexAuthReadinessProbe, + emptyConfigDirAcknowledgedGaugeValue, + emptyConfigDirAdvisory, githubAppReadinessProbe, publicOriginAcknowledgedGaugeValue, publicOriginReachabilityAdvisory, @@ -113,6 +115,16 @@ function nonBlank(value: string | undefined): string | undefined { return trimmed ? trimmed : undefined; } +/** Top-level entry count of `dir` (files + subdirectories, dotfiles included), or 0 on any read error -- + * a missing/unreadable directory is reported the same as an empty one rather than crashing boot. */ +function safeReaddirCount(dir: string): number { + try { + return readdirSync(dir).length; + } catch { + return 0; + } +} + interface Backend { db: D1Database; @@ -298,15 +310,36 @@ async function main(): Promise { // Boot-time visibility (config-drift guardrail): state which config dir is actually in effect, unconditionally // -- neither reader above logs anything, so an operator previously had no way to confirm from the logs alone // which directory (if any) was live, which is exactly the ambiguity that let a stale, no-longer-mounted config - // path get mistaken for the real one during a past incident. Never touches or validates any file; this is - // purely a log line, same "state what's in effect" shape as the sentry/otel boot logs below. + // path get mistaken for the real one during a past incident. `entryCount` is a cheap, one-time top-level + // listing (never recursive, never touches file contents) so a SECOND incident of the same shape -- the mount + // resolving but landing on an empty directory -- is visible in the log line itself, not just "some path is + // configured" (see emptyConfigDirAdvisory below for the loud version of this same signal). + const configDirOpts = { + configured: Boolean(repoConfigDir), + // A missing (as opposed to merely empty) directory is treated the same as zero entries -- both mean "no + // local config was actually read" -- rather than letting a bad path crash the whole server at boot. + entryCount: repoConfigDir ? safeReaddirCount(repoConfigDir) : 0, + acknowledged: process.env.CONFIG_DIR_EMPTY_ACKNOWLEDGED === "true", + }; console.log( JSON.stringify({ event: "selfhost_config_dir", - configured: Boolean(repoConfigDir), + configured: configDirOpts.configured, dir: repoConfigDir ?? null, + entryCount: repoConfigDir ? configDirOpts.entryCount : null, }), ); + // Config-drift advisory: warn LOUDLY (not just the log line above) when the mount resolves but is empty -- + // see emptyConfigDirAdvisory's own doc comment for the incident this guards against. + const configDirAdvisory = emptyConfigDirAdvisory(configDirOpts); + if (configDirAdvisory) + console.warn( + JSON.stringify({ + level: "warn", + event: "selfhost_config_dir_empty_advisory", + message: configDirAdvisory, + }), + ); // Error tracking (#1468): opt-in via SENTRY_DSN — a complete no-op when unset. When on, capture uncaught crashes // + unhandled rejections (flush before exit for the fatal case); per-subsystem captures (queue dead-letter, // review failures) are wired at their sites. @@ -781,6 +814,7 @@ async function main(): Promise { ); gauge("loopover_backup_acknowledged", () => backupAcknowledgedGaugeValue(sqliteBackupOpts)); gauge("loopover_public_origin_acknowledged", () => publicOriginAcknowledgedGaugeValue(publicOriginOpts)); + gauge("loopover_config_dir_empty_acknowledged", () => emptyConfigDirAcknowledgedGaugeValue(configDirOpts)); // Pre-initialize job counters to 0 so they appear in the first Prometheus scrape (lazy counters // created on first use would otherwise cause "No data" in Grafana until the first job event). for (const c of [ diff --git a/test/unit/selfhost-health.test.ts b/test/unit/selfhost-health.test.ts index f61fcba10c..999d40d9df 100644 --- a/test/unit/selfhost-health.test.ts +++ b/test/unit/selfhost-health.test.ts @@ -8,6 +8,8 @@ import { backupAcknowledgedGaugeValue, buildHealthBody, codexAuthReadinessProbe, + emptyConfigDirAcknowledgedGaugeValue, + emptyConfigDirAdvisory, githubAppReadinessProbe, publicOriginAcknowledgedGaugeValue, publicOriginReachabilityAdvisory, @@ -311,6 +313,35 @@ describe("publicOriginAcknowledgedGaugeValue (#4180)", () => { }); }); +describe("emptyConfigDirAdvisory (gittensory->loopover rename incident)", () => { + it("is silent when LOOPOVER_REPO_CONFIG_DIR is unset entirely — a normal, unconfigured install", () => { + expect(emptyConfigDirAdvisory({ configured: false, entryCount: 0, acknowledged: false })).toBeNull(); + }); + + it("is silent when the mounted directory has at least one entry", () => { + expect(emptyConfigDirAdvisory({ configured: true, entryCount: 1, acknowledged: false })).toBeNull(); + }); + + it("is silent when acknowledged, even if configured and empty", () => { + expect(emptyConfigDirAdvisory({ configured: true, entryCount: 0, acknowledged: true })).toBeNull(); + }); + + it("regression: warns when configured but the mounted directory is empty — the exact incident shape", () => { + const message = emptyConfigDirAdvisory({ configured: true, entryCount: 0, acknowledged: false }); + expect(message).toMatch(/mounted directory is empty/); + expect(message).toMatch(/CONFIG_DIR_EMPTY_ACKNOWLEDGED/); + }); +}); + +describe("emptyConfigDirAcknowledgedGaugeValue (gittensory->loopover rename incident)", () => { + it("mirrors the advisory: 0 only when configured and empty and unacknowledged", () => { + expect(emptyConfigDirAcknowledgedGaugeValue({ configured: true, entryCount: 0, acknowledged: false })).toBe(0); + expect(emptyConfigDirAcknowledgedGaugeValue({ configured: true, entryCount: 0, acknowledged: true })).toBe(1); + expect(emptyConfigDirAcknowledgedGaugeValue({ configured: true, entryCount: 1, acknowledged: false })).toBe(1); + expect(emptyConfigDirAcknowledgedGaugeValue({ configured: false, entryCount: 0, acknowledged: false })).toBe(1); + }); +}); + describe("readiness (#982)", () => { afterEach(() => { vi.restoreAllMocks(); diff --git a/test/unit/selfhost-post-update-check-script.test.ts b/test/unit/selfhost-post-update-check-script.test.ts index 8b2230b8fc..656d594829 100644 --- a/test/unit/selfhost-post-update-check-script.test.ts +++ b/test/unit/selfhost-post-update-check-script.test.ts @@ -44,6 +44,10 @@ if [ "$1" = "inspect" ] && [ "$2" = "--format" ]; then fi exit 0 fi +if [ "$1" = "exec" ]; then + printf '%s\\n' "\${CONFIG_DIR_ENTRIES:-3}" + exit 0 +fi exit 0 `, ); @@ -100,4 +104,19 @@ describe("selfhost-post-update-check.sh", () => { expect(result.status).toBe(1); expect(result.stderr).toContain("after 2 attempts (6s)"); }); + + it("regression: warns (without failing) when the container's private config mount is empty", () => { + const result = run({ CONFIG_DIR_ENTRIES: "0" }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toContain("private config directory (LOOPOVER_REPO_CONFIG_DIR, default /config) is empty"); + expect(result.stdout).toContain("selfhost post-update check: ok"); + }); + + it("is silent when the container's private config mount has entries", () => { + const result = run({ CONFIG_DIR_ENTRIES: "4" }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).not.toContain("private config directory"); + }); });