From 7ce3966d48d257feba638dd92502c5a97e0837d7 Mon Sep 17 00:00:00 2001 From: luciferlive112116 <291889058+luciferlive112116@users.noreply.github.com> Date: Wed, 15 Jul 2026 18:03:28 +0800 Subject: [PATCH] docs(selfhost): document every docker-compose.yml var in .env.example docker-compose.yml's header calls .env.example "the exhaustive reference" an operator copies to .env before running the stack. Cross-referencing every ${VAR} interpolation against every name the sample file documents found 18 real gaps -- knobs the stack reads that an operator could only discover by reading docker-compose.yml itself: - --profile backup, entirely undocumented: BACKUP_RETAIN, BACKUP_INTERVAL_SECONDS, and the opt-in restore-drill pair VERIFY_RESTORE_SCRATCH + LOOPOVER_VERIFY_SCRATCH_DATABASE_URL. - The four AMS exporter source/reporting DB path overrides, and the core exporter's LOOPOVER_REPORTING_DB output path. - LOOPOVER_REPO_CONFIG_DIR, read by the always-on loopover service. - The OTEL metrics-side siblings of the documented traces vars: OTEL_METRICS_EXPORTER, OTEL_EXPORTER_OTLP_PROTOCOL, OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE. - PROMETHEUS_RETENTION_TIME, REES_MEM_LIMIT (the one omission from the otherwise-exhaustive *_MEM_LIMIT block), INSTALL_VISUAL_REVIEW. - COMPOSE_PROJECT_NAME and TZ, which the drift guard below also requires and which the issue's own "exhaustive" outcome covers. Every documented default is the one docker-compose.yml already uses; no default or behavior is invented, and each var is grouped under the section it belongs to in the file's existing comment style. The guard parses compose's interpolations and both reference files, then asserts none is undocumented. Its negative lookbehind is load-bearing: compose escapes a literal $ as $$, so $${GF_SECURITY_ADMIN_PASSWORD:-} is resolved by the container's shell and is not a compose variable -- demanding .env.example document it would be wrong. Covers the pass case, a stripped-fixture fail case, the $$-escape case, and the secrets/ README.md path, so this gap class cannot silently reopen. Closes #5814 --- .env.example | 34 ++++++++ .../docker-compose-env-example-parity.test.ts | 80 +++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 test/unit/docker-compose-env-example-parity.test.ts diff --git a/.env.example b/.env.example index aa61778589..424be8605c 100644 --- a/.env.example +++ b/.env.example @@ -240,6 +240,10 @@ LOOPOVER_REVIEW_DRAFT=false # # scopes dashboard sign-in, not the shared MCP token's per-repo reach. # PORT=8787 # DATABASE_PATH=/data/loopover.sqlite # SQLite file on the mounted data volume; all migrations auto-apply +# LOOPOVER_REPO_CONFIG_DIR=/config # in-container dir the app reads per-repo .loopover.yml files from +# COMPOSE_PROJECT_NAME=loopover # Docker Compose's own project name; also labels the log stream +# # Promtail ships to Loki. Change it to run two stacks on one host. +# TZ=UTC # timezone passed to n8n (--profile workflows) for cron schedules # POSTGRES_PASSWORD=change-this-long-random-value # used by the --profile postgres / --profile pgbouncer services # DATABASE_URL= # set to postgres://user:pw@host:5432/db to use Postgres instead of # # SQLite (shared DB → multi-instance). Overrides DATABASE_PATH. @@ -348,6 +352,7 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # REDIS_MEM_LIMIT=512m # core app (always runs) # POSTGRES_MEM_LIMIT=2g # --profile postgres / --profile pgbouncer # QDRANT_MEM_LIMIT=2g # --profile qdrant +# REES_MEM_LIMIT=512m # --profile rees (the review-enrichment service) # OLLAMA_MEM_LIMIT=20g # --profile ollama; raise this before pulling a large local model # # (default sized for an embed model + one vision model resident # # together, #4335 -- raise further for additional/larger models) @@ -377,6 +382,15 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # acknowledged backup once you've wired Litestream (or an # equivalent) above. Default false (warning shown). +# --- Snapshot backups (--profile backup; the backup + backup-exporter sidecars) --- +# Separate from Litestream above: this runs scripts/backup.sh on a loop and exports the result as metrics. +# BACKUP_RETAIN=7 # how many snapshots to keep before pruning the oldest +# BACKUP_INTERVAL_SECONDS=86400 # seconds between runs (default 86400 = daily) +# Opt-in restore drill (scripts/verify-backup.sh): both are empty by default, which skips the drill. Set the +# pair together to restore each snapshot into a scratch database and prove it is actually restorable. +# VERIFY_RESTORE_SCRATCH= # non-empty enables the drill +# LOOPOVER_VERIFY_SCRATCH_DATABASE_URL= # throwaway scratch DB the drill restores into (never your live DB) + # --- n8n workflow automation (--profile workflows) --- # N8N_PASSWORD=changeme # REQUIRED at runtime when using --profile workflows # N8N_USER=admin @@ -519,6 +533,10 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # prometheus/rules/, routing in alertmanager/alertmanager.yml — silent until you fill in a receiver) + # Loki + Promtail (ship every container's logs to Loki) + Grafana (dashboards for metrics AND logs). # GRAFANA_ADMIN_PASSWORD= # REQUIRED at runtime when using --profile observability; generate a strong value +# PROMETHEUS_RETENTION_TIME=90d # metrics history window (a full quarter for trend/capacity review). +# # Deliberately longer than Loki's 14d / Tempo's 7d: the TSDB +# # compresses samples to ~1-2 bytes each, so metrics cost far less +# # disk per retained day than raw logs or span trees (#1828). # GRAFANA_LOCAL_SMOKE_PASSWORD= # optional local-only fallback for smoke tests; never expose Grafana with this # # Maintainer dashboards (in addition to the infra dashboard): @@ -527,6 +545,7 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # • "Resource hub" — links to every integrated service. # GRAFANA_REPORTING_EXPORT_INTERVAL_SECONDS=30 # refresh cadence for the redacted reporting SQLite export # LOOPOVER_REPORTING_SOURCE_DB=/appdb/loopover.sqlite # if DATABASE_PATH=/data/custom.sqlite, set /appdb/custom.sqlite +# LOOPOVER_REPORTING_DB=/reporting/loopover-reporting.sqlite # where the redacted export is written (Grafana reads this) # # AMS (loopover-miner) ledger dashboards — only useful when a miner ALSO runs on this same host (see # packages/loopover-miner/docs/observability.md). Requires --profile ams-observability, separate from the @@ -535,6 +554,13 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # snapshot (free-form attempt_log_events.reason/.payload_json dropped) into the same reporting volume. # LOOPOVER_MINER_CONFIG_DIR=~/.config/loopover-miner # host dir the exporter mounts read-only at /ams-ledgers # LOOPOVER_AMS_REPORTING_EXPORT_INTERVAL_SECONDS=30 # refresh cadence for the redacted AMS ledger exports +# Path overrides for the same exporter — in-container paths, not host paths. The source pair lives under the +# read-only /ams-ledgers mount above; the reporting pair is written into the shared reporting volume. Override +# only if your miner names its ledgers differently. +# LOOPOVER_AMS_ATTEMPT_LOG_SOURCE_DB=/ams-ledgers/attempt-log.sqlite3 +# LOOPOVER_AMS_PREDICTION_LEDGER_SOURCE_DB=/ams-ledgers/prediction-ledger.sqlite3 +# LOOPOVER_AMS_ATTEMPT_LOG_REPORTING_DB=/reporting/ams-attempt-log.sqlite +# LOOPOVER_AMS_PREDICTION_LEDGER_REPORTING_DB=/reporting/ams-prediction-ledger.sqlite # # Claude usage telemetry → OTEL collector → Prometheus → the Claude usage dashboard. OFF by default. # CLAUDE_CODE_ENABLE_TELEMETRY=1 # enable; needs --profile observability (starts the otel-collector) @@ -557,7 +583,11 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # # images bake LOOPOVER_VERSION=gittensory-selfhost@, so do # # not override SENTRY_RELEASE for those images. # OTEL_METRIC_EXPORT_INTERVAL=10000 # ms between metric exports (default 10s here; CLI default is 60s) +# OTEL_METRICS_EXPORTER=otlp # metrics-side counterpart to OTEL_TRACES_EXPORTER; empty = no app metrics # OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 # override only for an external collector +# OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf # wire protocol for the endpoint above (http/protobuf | grpc) +# OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE=cumulative # cumulative is what Prometheus expects; change +# # only for a backend that requires delta temporality # OTEL_SERVICE_NAME=gittensory-selfhost # # Live upstream PR/issue census (the GitHub data source). Install is automatic (GF_INSTALL_PLUGINS); add the @@ -603,6 +633,10 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # runtime-only: set AI_PROVIDER plus the provider-specific auth below. Set INSTALL_AI_CLIS=false only for a # custom minimal local build that will never use the subscription CLI providers. # INSTALL_AI_CLIS=true +# INSTALL_VISUAL_REVIEW=false # sibling build arg: installs puppeteer-core for a LOCAL build. +# # Official release images already bundle it, so this only matters +# # when you build the image yourself and want LOOPOVER_REVIEW_SCREENSHOTS +# # (the --profile visual-review browserless service is the alternative). # # Deprecated shared AI_* knobs are intentionally rejected at startup: # AI_BASE_URL, AI_API_KEY, AI_MODEL, AI_EFFORT, AI_TIMEOUT_MS. Use the explicit diff --git a/test/unit/docker-compose-env-example-parity.test.ts b/test/unit/docker-compose-env-example-parity.test.ts new file mode 100644 index 0000000000..bd5ab0db0c --- /dev/null +++ b/test/unit/docker-compose-env-example-parity.test.ts @@ -0,0 +1,80 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +// Drift guard for #5814: docker-compose.yml's own header calls .env.example "the exhaustive reference", but +// ~18 variables compose actually interpolates were documented nowhere — the whole --profile backup block, the +// AMS exporter path overrides, the OTEL metrics-side siblings, PROMETHEUS_RETENTION_TIME, REES_MEM_LIMIT. This +// is a distinct check from `npm run selfhost:env-reference`, which scans process.env reads under +// src/selfhost/** and never looks at docker-compose.yml or .env.example at all. +// +// Every compose variable must be documented in .env.example (live `NAME=` or commented `# NAME=`) or, for the +// Docker-secrets `*_FILE` vars deliberately kept out of the sample file, in secrets/README.md's table. + +const ROOT = process.cwd(); +const composeText = readFileSync(join(ROOT, "docker-compose.yml"), "utf8"); +const envExampleText = readFileSync(join(ROOT, ".env.example"), "utf8"); +const secretsReadmeText = readFileSync(join(ROOT, "secrets/README.md"), "utf8"); + +/** + * `${VAR}` / `${VAR:-default}` / `${VAR:?err}` that compose really interpolates. + * + * The negative lookbehind is load-bearing: compose escapes a literal `$` as `$$`, so `$${GF_SECURITY_ADMIN_PASSWORD:-}` + * is passed through to the container's shell and resolved there — it is NOT a compose variable, and demanding + * `.env.example` document it would be wrong. + */ +function composeInterpolatedVars(compose: string): string[] { + return [...new Set([...compose.matchAll(/(? m[1]!))].sort(); +} + +/** Live `NAME=` and commented `# NAME=` reference lines alike — both count as documented. */ +function documentedInEnvExample(envExample: string): Set { + return new Set([...envExample.matchAll(/^\s*#?\s*([A-Z_][A-Z0-9_]*)=/gm)].map((m) => m[1]!)); +} + +/** The `*_FILE` vars in secrets/README.md's table, intentionally documented there instead of .env.example. */ +function documentedInSecretsReadme(secretsReadme: string): Set { + return new Set([...secretsReadme.matchAll(/\b([A-Z_][A-Z0-9_]*_FILE)\b/g)].map((m) => m[1]!)); +} + +function undocumentedComposeVars(compose: string, envExample: string, secretsReadme: string): string[] { + const documented = documentedInEnvExample(envExample); + const secrets = documentedInSecretsReadme(secretsReadme); + return composeInterpolatedVars(compose).filter((name) => !documented.has(name) && !secrets.has(name)); +} + +describe("docker-compose.yml ↔ .env.example parity (#5814)", () => { + it("finds the compose variables at all (guards the parser itself, so the check can't pass vacuously)", () => { + const vars = composeInterpolatedVars(composeText); + expect(vars.length).toBeGreaterThan(50); + expect(vars).toContain("PROMETHEUS_RETENTION_TIME"); + expect(documentedInEnvExample(envExampleText).size).toBeGreaterThan(50); + expect(documentedInSecretsReadme(secretsReadmeText)).toContain("GITHUB_APP_PRIVATE_KEY_FILE"); + }); + + it("ignores $$-escaped shell variables, which compose never interpolates", () => { + // `$${FOO}` reaches the container's shell as `${FOO}`; only the un-escaped `${BAR}` is a compose var. + expect(composeInterpolatedVars('a: "$${FOO:-x}"\nb: "${BAR:-y}"')).toEqual(["BAR"]); + }); + + it("INVARIANT: every compose variable is documented in .env.example or secrets/README.md", () => { + expect(undocumentedComposeVars(composeText, envExampleText, secretsReadmeText)).toEqual([]); + }); + + it("catches a real gap: a variable dropped from .env.example is reported (proves the guard works)", () => { + // Strip PROMETHEUS_RETENTION_TIME's documentation line — the check must notice. + const stripped = envExampleText + .split("\n") + .filter((line) => !/^\s*#?\s*PROMETHEUS_RETENTION_TIME=/.test(line)) + .join("\n"); + expect(undocumentedComposeVars(composeText, stripped, secretsReadmeText)).toEqual(["PROMETHEUS_RETENTION_TIME"]); + }); + + it("counts a *_FILE secret as documented via secrets/README.md, not .env.example", () => { + const compose = 'x: "${GITHUB_APP_PRIVATE_KEY_FILE}"'; + expect(undocumentedComposeVars(compose, "", secretsReadmeText)).toEqual([]); + // …and reports it when it is documented in neither. + expect(undocumentedComposeVars(compose, "", "")).toEqual(["GITHUB_APP_PRIVATE_KEY_FILE"]); + }); +});