diff --git a/.env.example b/.env.example index 34c478edfa..80ce97e535 100644 --- a/.env.example +++ b/.env.example @@ -130,6 +130,9 @@ GITTENSORY_REVIEW_DRAFT=false # DATABASE_PATH=/data/gittensory.sqlite # SQLite file on the mounted data volume; all migrations auto-apply # DATABASE_URL= # set to postgres://user:pw@host:5432/db to use Postgres instead of # # SQLite (shared DB → multi-instance). Overrides DATABASE_PATH. +# BACKUP_ACKNOWLEDGED= # set to "true" to silence the boot warning that a single SQLite file +# # has no backup — only after you've wired Litestream (docs §6) or +# # equivalent. Unset on SQLite ⇒ a loud data-loss warning at boot. # REDIS_URL= # set to redis://host:6379 for distributed rate limiting + webhook dedup # # cache (prevents double-processing of GitHub retries). Off when unset. # QDRANT_URL= # set to http://qdrant:6333 to use Qdrant as the RAG vector store diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 3334081fd4..5e67d983d7 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -238,11 +238,13 @@ accepted. Unset ⇒ the public file is fetched exactly as before. in-flight job, checkpoints the WAL, and closes the DB before exiting. - **Logs** are structured JSON (`selfhost_listening`, `selfhost_migrations_applied`, `selfhost_ai_provider`, `selfhost_queue_recovered`, `selfhost_job_dead`, `selfhost_cron_error`, `selfhost_shutdown`, …). -- **Data + backup.** Everything is the single SQLite file on the `gittensory-data` volume (WAL mode). Back up - by snapshotting the volume or copying the `.sqlite` file. Migrations are idempotent and re-checked at boot. - For **continuous, point-in-time backup**, enable the optional [Litestream](https://litestream.io) sidecar in +- **Data + backup (do NOT skip).** Everything is the single SQLite file on the `gittensory-data` volume (WAL + mode), so **without a backup, losing the volume loses all review state** — and `/ready` still answers `200`, + so the gap is silent. The container logs a loud `selfhost_backup_advisory` warning at boot until you set up a + backup. For **continuous, point-in-time backup**, enable the [Litestream](https://litestream.io) sidecar in `docker-compose.yml` (copy `litestream.yml.example` → `litestream.yml`, set your bucket + credentials); it - streams every change to S3/B2/MinIO/R2. + streams every change to S3/B2/MinIO/R2. Then set **`BACKUP_ACKNOWLEDGED=true`** to silence the warning. (For + multi-instance, use `DATABASE_URL=postgres://…` and back up Postgres instead.) - **App-level metrics.** Enable `GITTENSORY_REVIEW_OPS=true` for the read-only gate-block anomaly scan and the bearer-gated `GET /v1/internal/ops/stats` aggregate. diff --git a/src/selfhost/health.ts b/src/selfhost/health.ts index 4accc343a3..6f350391af 100644 --- a/src/selfhost/health.ts +++ b/src/selfhost/health.ts @@ -42,3 +42,12 @@ export async function readiness(db: D1Database, probes: ReadinessProbe[] = []): } return { ok: Object.values(checks).every(Boolean), checks }; } + +/** Boot-time DATA-SAFETY advisory. A single SQLite file with no acknowledged backup is a data-loss SPOF — yet + * `/ready` would still answer 200, so an operator can run with zero durability believing they're healthy. Returns + * the warning to log at boot (or null on Postgres, or once the operator sets `BACKUP_ACKNOWLEDGED=true` after + * wiring Litestream or another backup). */ +export function sqliteBackupAdvisory(opts: { usingSqlite: boolean; backupAcknowledged: boolean }): string | null { + if (!opts.usingSqlite || opts.backupAcknowledged) return null; + return "Running on a single SQLite file with no acknowledged backup — if the volume is lost, ALL review state is lost. Enable the Litestream sidecar (docs/self-hosting.md §6) to stream the WAL to S3/B2/MinIO, then set BACKUP_ACKNOWLEDGED=true to silence this warning. (Multi-instance: use DATABASE_URL=postgres://… instead.)"; +} diff --git a/src/server.ts b/src/server.ts index e0b1b687c2..82c41022c3 100644 --- a/src/server.ts +++ b/src/server.ts @@ -28,7 +28,7 @@ import { import { isOrbBrokerMode, registerOrbRelayTarget } from "./orb/broker-client"; import { exportOrbBatch } from "./selfhost/orb-collector"; import { createD1Adapter, nodeSqliteDriver } from "./selfhost/d1-adapter"; -import { readiness, type ReadinessProbe } from "./selfhost/health"; +import { readiness, sqliteBackupAdvisory, type ReadinessProbe } from "./selfhost/health"; import { gauge, incr, observe, renderMetrics } from "./selfhost/metrics"; import { runSelfHostMigrations } from "./selfhost/migrate"; import { createPgAdapter } from "./selfhost/pg-adapter"; @@ -177,6 +177,10 @@ async function main(): Promise { const usePostgres = !!databaseUrl && /^postgres(ql)?:\/\//i.test(databaseUrl); const backend = usePostgres ? await buildPostgresBackend(databaseUrl as string, consume) : buildSqliteBackend(consume); console.log(JSON.stringify({ event: "selfhost_backend", backend: usePostgres ? "postgres" : "sqlite" })); + // Data-safety advisory (#8): warn LOUDLY at boot if running on a single SQLite file with no acknowledged backup, + // so an operator doesn't run with zero durability while /ready answers 200. + const backupAdvisory = sqliteBackupAdvisory({ usingSqlite: !usePostgres, backupAcknowledged: process.env.BACKUP_ACKNOWLEDGED === "true" }); + if (backupAdvisory) console.warn(JSON.stringify({ level: "warn", event: "selfhost_backup_advisory", message: backupAdvisory })); const applied = await runSelfHostMigrations(backend.db, process.env.MIGRATIONS_DIR ?? "migrations"); console.log(JSON.stringify({ event: "selfhost_migrations_applied", count: applied })); diff --git a/test/unit/selfhost-health.test.ts b/test/unit/selfhost-health.test.ts index d9b3506b68..0c5c53132d 100644 --- a/test/unit/selfhost-health.test.ts +++ b/test/unit/selfhost-health.test.ts @@ -1,7 +1,15 @@ import { DatabaseSync } from "node:sqlite"; import { describe, expect, it } from "vitest"; import { createD1Adapter, nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; -import { readiness } from "../../src/selfhost/health"; +import { readiness, sqliteBackupAdvisory } from "../../src/selfhost/health"; + +describe("sqliteBackupAdvisory (#8 data-safety)", () => { + it("warns on SQLite without an acknowledged backup, and is silent otherwise", () => { + expect(sqliteBackupAdvisory({ usingSqlite: true, backupAcknowledged: false })).toMatch(/single SQLite file with no acknowledged backup/); + expect(sqliteBackupAdvisory({ usingSqlite: true, backupAcknowledged: true })).toBeNull(); // operator acknowledged + expect(sqliteBackupAdvisory({ usingSqlite: false, backupAcknowledged: false })).toBeNull(); // Postgres + }); +}); describe("readiness (#982)", () => { it("is not ready until the migrations table has applied rows", async () => {