diff --git a/apps/gittensory-ui/src/lib/selfhost-env-reference.ts b/apps/gittensory-ui/src/lib/selfhost-env-reference.ts index b8944dcb4f..35099dbd13 100644 --- a/apps/gittensory-ui/src/lib/selfhost-env-reference.ts +++ b/apps/gittensory-ui/src/lib/selfhost-env-reference.ts @@ -15,11 +15,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "AI_EMBED_API_KEY", - firstReference: "src/server.ts:441", + firstReference: "src/server.ts:443", }, { name: "AI_EMBED_BASE_URL", - firstReference: "src/server.ts:438", + firstReference: "src/server.ts:440", }, { name: "AI_EMBED_MODEL", @@ -47,7 +47,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "BACKUP_ACKNOWLEDGED", - firstReference: "src/server.ts:380", + firstReference: "src/server.ts:381", }, { name: "BROWSER_WS_ENDPOINT", @@ -87,11 +87,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "CRON_INTERVAL_MS", - firstReference: "src/server.ts:919", + firstReference: "src/server.ts:922", }, { name: "DATABASE_PATH", - firstReference: "src/server.ts:250", + firstReference: "src/server.ts:251", }, { name: "DATABASE_URL", @@ -131,7 +131,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "GITHUB_CACHE_TTL_SECONDS", - firstReference: "src/server.ts:510", + firstReference: "src/server.ts:512", }, { name: "GITHUB_INSTALLATION_CONCURRENCY_DEFER_MS", @@ -147,7 +147,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "GITTENSORY_REPO_CONFIG_DIR", - firstReference: "src/server.ts:289", + firstReference: "src/server.ts:290", }, { name: "GITTENSORY_VERSION", @@ -191,7 +191,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "MIGRATIONS_DIR", - firstReference: "src/server.ts:393", + firstReference: "src/server.ts:395", }, { name: "OBSERVABILITY_SMOKE_POLL_MS", @@ -251,7 +251,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "ORB_BROKER_URL", - firstReference: "src/server.ts:974", + firstReference: "src/server.ts:977", }, { name: "ORB_COLLECTOR_TOKEN", @@ -267,7 +267,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "ORB_RELAY_MODE", - firstReference: "src/server.ts:962", + firstReference: "src/server.ts:965", }, { name: "OTEL_EXPORTER_OTLP_ENDPOINT", @@ -303,11 +303,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "PGVECTOR_ENABLED", - firstReference: "src/server.ts:230", + firstReference: "src/server.ts:231", }, { name: "PORT", - firstReference: "src/server.ts:718", + firstReference: "src/server.ts:721", }, { name: "PUBLIC_API_ORIGIN", @@ -323,7 +323,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "QDRANT_URL", - firstReference: "src/server.ts:529", + firstReference: "src/server.ts:531", }, { name: "QUEUE_BACKGROUND_CONCURRENCY", @@ -347,7 +347,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "REVIEW_AUDIT_DIR", - firstReference: "src/server.ts:574", + firstReference: "src/server.ts:576", }, { name: "SELFHOST_BUNDLE_ALL", @@ -383,7 +383,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ }, { name: "SETUP_OUTPUT_PATH", - firstReference: "src/server.ts:835", + firstReference: "src/server.ts:838", }, { name: "SLACK_WEBHOOK_URL", @@ -396,15 +396,15 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| --- | --- |", "| `AI_COMBINE` | `src/selfhost/ai.ts:1160` |", "| `AI_DUAL_REVIEW` | `src/selfhost/ai.ts:1135` |", - "| `AI_EMBED_API_KEY` | `src/server.ts:441` |", - "| `AI_EMBED_BASE_URL` | `src/server.ts:438` |", + "| `AI_EMBED_API_KEY` | `src/server.ts:443` |", + "| `AI_EMBED_BASE_URL` | `src/server.ts:440` |", "| `AI_EMBED_MODEL` | `src/selfhost/ai.ts:1032` |", "| `AI_ON_MERGE` | `src/selfhost/ai.ts:1162` |", "| `AI_PROVIDER` | `src/selfhost/ai-config.ts:43` |", "| `ANTHROPIC_AI_BASE_URL` | `src/selfhost/ai.ts:1036` |", "| `ANTHROPIC_AI_MODEL` | `src/selfhost/ai.ts:96` |", "| `ANTHROPIC_API_KEY` | `src/selfhost/ai.ts:1035` |", - "| `BACKUP_ACKNOWLEDGED` | `src/server.ts:380` |", + "| `BACKUP_ACKNOWLEDGED` | `src/server.ts:381` |", "| `BROWSER_WS_ENDPOINT` | `src/selfhost/stubs/puppeteer.ts:11` |", "| `CLAUDE_AI_EFFORT` | `src/selfhost/ai.ts:147` |", "| `CLAUDE_AI_MODEL` | `src/selfhost/ai.ts:88` |", @@ -414,8 +414,8 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `CODEX_AI_MODEL` | `src/selfhost/ai.ts:92` |", "| `CODEX_AI_TIMEOUT_MS` | `src/selfhost/ai.ts:151` |", "| `CODEX_HOME` | `src/selfhost/ai.ts:340` |", - "| `CRON_INTERVAL_MS` | `src/server.ts:919` |", - "| `DATABASE_PATH` | `src/server.ts:250` |", + "| `CRON_INTERVAL_MS` | `src/server.ts:922` |", + "| `DATABASE_PATH` | `src/server.ts:251` |", "| `DATABASE_URL` | `src/selfhost/preflight.ts:201` |", "| `DISCORD_REPO_WEBHOOKS` | `src/services/notify-discord.ts:41` |", "| `DISCORD_WEBHOOK_URL` | `src/services/notify-discord.ts:78` |", @@ -425,11 +425,11 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `FOREGROUND_LIVENESS_MAX_RELEASE_PER_SWEEP` | `src/selfhost/foreground-liveness.ts:53` |", "| `GITHUB_APP_ID` | `src/selfhost/orb-collector.ts:59` |", "| `GITHUB_APP_PRIVATE_KEY` | `src/selfhost/orb-collector.ts:166` |", - "| `GITHUB_CACHE_TTL_SECONDS` | `src/server.ts:510` |", + "| `GITHUB_CACHE_TTL_SECONDS` | `src/server.ts:512` |", "| `GITHUB_INSTALLATION_CONCURRENCY_DEFER_MS` | `src/selfhost/installation-concurrency-admission.ts:47` |", "| `GITHUB_INSTALLATION_CONCURRENCY_ENABLED` | `src/selfhost/installation-concurrency-admission.ts:34` |", "| `GITHUB_INSTALLATION_CONCURRENCY_LIMIT` | `src/selfhost/installation-concurrency-admission.ts:43` |", - "| `GITTENSORY_REPO_CONFIG_DIR` | `src/server.ts:289` |", + "| `GITTENSORY_REPO_CONFIG_DIR` | `src/server.ts:290` |", "| `GITTENSORY_VERSION` | `src/selfhost/otel.ts:62` |", "| `HOME` | `src/selfhost/ai.ts:340` |", "| `MAINTENANCE_ADMISSION_DEFER_MS` | `src/selfhost/maintenance-admission.ts:171` |", @@ -440,7 +440,7 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `MAINTENANCE_ADMISSION_MAX_LIVE_AGE_MS` | `src/selfhost/maintenance-admission.ts:155` |", "| `MAINTENANCE_ADMISSION_MAX_LIVE_PENDING` | `src/selfhost/maintenance-admission.ts:151` |", "| `MAINTENANCE_ADMISSION_MAX_PENDING` | `src/selfhost/maintenance-admission.ts:159` |", - "| `MIGRATIONS_DIR` | `src/server.ts:393` |", + "| `MIGRATIONS_DIR` | `src/server.ts:395` |", "| `OBSERVABILITY_SMOKE_POLL_MS` | `scripts/smoke-observability-traces.mjs:8` |", "| `OBSERVABILITY_SMOKE_TIMEOUT_MS` | `scripts/smoke-observability-traces.mjs:6` |", "| `OLLAMA_AI_API_KEY` | `src/selfhost/ai.ts:1029` |", @@ -455,11 +455,11 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `ORB_AIR_GAP` | `src/selfhost/orb-collector.ts:161` |", "| `ORB_ANONYMIZE` | `src/selfhost/orb-collector.ts:174` |", "| `ORB_APP_ID` | `src/selfhost/orb-collector.ts:59` |", - "| `ORB_BROKER_URL` | `src/server.ts:974` |", + "| `ORB_BROKER_URL` | `src/server.ts:977` |", "| `ORB_COLLECTOR_TOKEN` | `src/selfhost/orb-collector.ts:205` |", "| `ORB_COLLECTOR_URL` | `src/selfhost/orb-collector.ts:172` |", "| `ORB_ENROLLMENT_SECRET` | `src/selfhost/orb-collector.ts:165` |", - "| `ORB_RELAY_MODE` | `src/server.ts:962` |", + "| `ORB_RELAY_MODE` | `src/server.ts:965` |", "| `OTEL_EXPORTER_OTLP_ENDPOINT` | `src/selfhost/otel.ts:47` |", "| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | `src/selfhost/otel.ts:45` |", "| `OTEL_SERVICE_ENVIRONMENT` | `src/selfhost/otel.ts:60` |", @@ -468,18 +468,18 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `OTEL_TRACES_SAMPLER` | `src/selfhost/otel.ts:74` |", "| `OTEL_TRACES_SAMPLER_ARG` | `src/selfhost/otel.ts:76` |", "| `PGPOOL_MAX` | `src/selfhost/queue-common.ts:713` |", - "| `PGVECTOR_ENABLED` | `src/server.ts:230` |", - "| `PORT` | `src/server.ts:718` |", + "| `PGVECTOR_ENABLED` | `src/server.ts:231` |", + "| `PORT` | `src/server.ts:721` |", "| `PUBLIC_API_ORIGIN` | `src/selfhost/preflight.ts:192` |", "| `QDRANT_API_KEY` | `src/selfhost/qdrant-vectorize.ts:50` |", "| `QDRANT_DIM` | `src/selfhost/qdrant-vectorize.ts:71` |", - "| `QDRANT_URL` | `src/server.ts:529` |", + "| `QDRANT_URL` | `src/server.ts:531` |", "| `QUEUE_BACKGROUND_CONCURRENCY` | `src/selfhost/queue-common.ts:130` |", "| `QUEUE_CONCURRENCY` | `src/selfhost/pg-queue.ts:286` |", "| `QUEUE_DEAD_LETTER_AUTO_RETRY_MAX_EXTRA_ATTEMPTS` | `src/selfhost/queue-common.ts:721` |", "| `QUEUE_STARTUP_JITTER_MIN_JOBS` | `src/selfhost/queue-common.ts:702` |", "| `REDIS_URL` | `src/selfhost/preflight.ts:144` |", - "| `REVIEW_AUDIT_DIR` | `src/server.ts:574` |", + "| `REVIEW_AUDIT_DIR` | `src/server.ts:576` |", "| `SELFHOST_BUNDLE_ALL` | `scripts/build-selfhost.mjs:13` |", "| `SELFHOST_SERVICE` | `scripts/smoke-observability-traces.mjs:5` |", "| `SELFHOST_SETUP_TOKEN` | `src/selfhost/preflight.ts:186` |", @@ -488,6 +488,6 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `SENTRY_RELEASE` | `src/selfhost/otel.ts:62` |", "| `SENTRY_SERVER_NAME` | `src/selfhost/sentry.ts:407` |", "| `SENTRY_TRACES_SAMPLE_RATE` | `src/selfhost/sentry.ts:195` |", - "| `SETUP_OUTPUT_PATH` | `src/server.ts:835` |", + "| `SETUP_OUTPUT_PATH` | `src/server.ts:838` |", "| `SLACK_WEBHOOK_URL` | `src/services/notify-discord.ts:173` |", ].join("\n"); diff --git a/src/selfhost/health.ts b/src/selfhost/health.ts index 265ae7de98..b02b6b7b8e 100644 --- a/src/selfhost/health.ts +++ b/src/selfhost/health.ts @@ -150,3 +150,8 @@ export function sqliteBackupAdvisory(opts: { usingSqlite: boolean; backupAcknowl if (!opts.usingSqlite || opts.backupAcknowledged) return null; return "Running on a single SQLite file with no acknowledged backup — if the volume is lost, ALL review state is lost. Enable the Litestream sidecar (see the maintainer self-hosting docs) to stream the WAL to S3/B2/MinIO, then set BACKUP_ACKNOWLEDGED=true to silence this warning. (Multi-instance: use DATABASE_URL=postgres://… instead.)"; } + +/** Prometheus gauge value mirroring {@link sqliteBackupAdvisory}: 1 when Postgres or backup is acknowledged, 0 when the advisory would fire. */ +export function backupAcknowledgedGaugeValue(opts: { usingSqlite: boolean; backupAcknowledged: boolean }): 0 | 1 { + return sqliteBackupAdvisory(opts) === null ? 1 : 0; +} diff --git a/src/selfhost/metrics.ts b/src/selfhost/metrics.ts index c6f21b78a0..5f84f09bee 100644 --- a/src/selfhost/metrics.ts +++ b/src/selfhost/metrics.ts @@ -52,6 +52,7 @@ const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [ ["gittensory_github_rest_rate_limit_remaining", { help: "Newest observed GitHub REST rate-limit remaining count, by key scope.", type: "gauge" }], ["gittensory_host_load_avg1_per_core", { help: "One-minute host load average normalized by CPU core count.", type: "gauge" }], ["gittensory_uptime_seconds", { help: "Self-host process uptime in seconds.", type: "gauge" }], + ["gittensory_backup_acknowledged", { help: "1 when SQLite backup is acknowledged or Postgres is in use; 0 when the boot backup advisory would fire.", type: "gauge" }], ["gittensory_http_requests_total", { help: "HTTP app requests by response status class.", type: "counter" }], ["gittensory_http_request_duration_seconds", { help: "HTTP app request duration in seconds.", type: "histogram" }], ["gittensory_webhook_dedup_total", { help: "Webhook deliveries deduplicated before enqueue.", type: "counter" }], diff --git a/src/server.ts b/src/server.ts index 789d4a104d..9f5a3018bb 100644 --- a/src/server.ts +++ b/src/server.ts @@ -43,6 +43,7 @@ import { createOrbRelayRegistrationState, isOrbBrokerMode, registerOrbRelayTarge import { exportOrbBatch } from "./selfhost/orb-collector"; import { createD1Adapter, nodeSqliteDriver } from "./selfhost/d1-adapter"; import { + backupAcknowledgedGaugeValue, buildHealthBody, codexAuthReadinessProbe, githubAppReadinessProbe, @@ -375,10 +376,11 @@ async function main(): Promise { ); // Data-safety advisory (#8): warn LOUDLY at boot if running on a single SQLite file with no acknowledged backup, // so an operator doesn't run with zero durability while /ready answers 200. - const backupAdvisory = sqliteBackupAdvisory({ + const sqliteBackupOpts = { usingSqlite: !usePostgres, backupAcknowledged: process.env.BACKUP_ACKNOWLEDGED === "true", - }); + }; + const backupAdvisory = sqliteBackupAdvisory(sqliteBackupOpts); if (backupAdvisory) console.warn( JSON.stringify({ @@ -688,6 +690,7 @@ async function main(): Promise { gauge("gittensory_uptime_seconds", () => Math.floor((Date.now() - startedAt) / 1000), ); + gauge("gittensory_backup_acknowledged", () => backupAcknowledgedGaugeValue(sqliteBackupOpts)); // Pre-initialize job counters to 0 so they appear in the first Prometheus scrape (lazy counters // created on first use would otherwise cause "No data" in Grafana until the first job event). for (const c of [ diff --git a/test/unit/selfhost-health.test.ts b/test/unit/selfhost-health.test.ts index 65d93a0744..65a386a483 100644 --- a/test/unit/selfhost-health.test.ts +++ b/test/unit/selfhost-health.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; import { createD1Adapter, nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; import { + backupAcknowledgedGaugeValue, buildHealthBody, codexAuthReadinessProbe, githubAppReadinessProbe, @@ -210,6 +211,14 @@ describe("sqliteBackupAdvisory (#8 data-safety)", () => { }); }); +describe("backupAcknowledgedGaugeValue (#2089)", () => { + it("mirrors the advisory: 0 only when SQLite has no acknowledged backup", () => { + expect(backupAcknowledgedGaugeValue({ usingSqlite: true, backupAcknowledged: false })).toBe(0); + expect(backupAcknowledgedGaugeValue({ usingSqlite: true, backupAcknowledged: true })).toBe(1); + expect(backupAcknowledgedGaugeValue({ usingSqlite: false, backupAcknowledged: false })).toBe(1); + }); +}); + describe("readiness (#982)", () => { afterEach(() => { vi.restoreAllMocks(); diff --git a/test/unit/selfhost-metrics.test.ts b/test/unit/selfhost-metrics.test.ts index 49398bed44..738e29c287 100644 --- a/test/unit/selfhost-metrics.test.ts +++ b/test/unit/selfhost-metrics.test.ts @@ -1,4 +1,5 @@ import { afterEach, describe, expect, it } from "vitest"; +import { backupAcknowledgedGaugeValue } from "../../src/selfhost/health"; import { gauge, gaugeVector, incr, observe, registerMetricMeta, renderMetrics, resetMetrics, setSelfHostedMetricsMode } from "../../src/selfhost/metrics"; afterEach(() => { @@ -66,6 +67,16 @@ describe("metrics registry (#982)", () => { ); }); + it("renders gittensory_backup_acknowledged with seeded metadata (#2089)", async () => { + gauge("gittensory_backup_acknowledged", () => + backupAcknowledgedGaugeValue({ usingSqlite: true, backupAcknowledged: false }), + ); + + expect(await renderMetrics()).toBe( + "# HELP gittensory_backup_acknowledged 1 when SQLite backup is acknowledged or Postgres is in use; 0 when the boot backup advisory would fire.\n# TYPE gittensory_backup_acknowledged gauge\ngittensory_backup_acknowledged 0\n", + ); + }); + it("counters accumulate and render", async () => { incr("c_total"); incr("c_total", undefined, 2);