Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/selfhost/health.ts
Original file line number Diff line number Diff line change
Expand Up @@ -150,3 +150,8 @@ export function sqliteBackupAdvisory(opts: { usingSqlite: boolean; backupAcknowl
if (!opts.usingSqlite || opts.backupAcknowledged) return null;
return "Running on a single SQLite file with no acknowledged backup — if the volume is lost, ALL review state is lost. Enable the Litestream sidecar (see the maintainer self-hosting docs) to stream the WAL to S3/B2/MinIO, then set BACKUP_ACKNOWLEDGED=true to silence this warning. (Multi-instance: use DATABASE_URL=postgres://… instead.)";
}

/** Prometheus gauge value mirroring {@link sqliteBackupAdvisory}: 1 when Postgres or backup is acknowledged, 0 when the advisory would fire. */
export function backupAcknowledgedGaugeValue(opts: { usingSqlite: boolean; backupAcknowledged: boolean }): 0 | 1 {
return sqliteBackupAdvisory(opts) === null ? 1 : 0;
}
1 change: 1 addition & 0 deletions src/selfhost/metrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [
["gittensory_github_rest_rate_limit_remaining", { help: "Newest observed GitHub REST rate-limit remaining count, by key scope.", type: "gauge" }],
["gittensory_host_load_avg1_per_core", { help: "One-minute host load average normalized by CPU core count.", type: "gauge" }],
["gittensory_uptime_seconds", { help: "Self-host process uptime in seconds.", type: "gauge" }],
["gittensory_backup_acknowledged", { help: "1 when SQLite backup is acknowledged or Postgres is in use; 0 when the boot backup advisory would fire.", type: "gauge" }],
["gittensory_http_requests_total", { help: "HTTP app requests by response status class.", type: "counter" }],
["gittensory_http_request_duration_seconds", { help: "HTTP app request duration in seconds.", type: "histogram" }],
["gittensory_webhook_dedup_total", { help: "Webhook deliveries deduplicated before enqueue.", type: "counter" }],
Expand Down
7 changes: 5 additions & 2 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import { createOrbRelayRegistrationState, isOrbBrokerMode, registerOrbRelayTarge
import { exportOrbBatch } from "./selfhost/orb-collector";
import { createD1Adapter, nodeSqliteDriver } from "./selfhost/d1-adapter";
import {
backupAcknowledgedGaugeValue,
buildHealthBody,
codexAuthReadinessProbe,
githubAppReadinessProbe,
Expand Down Expand Up @@ -375,10 +376,11 @@ async function main(): Promise<void> {
);
// Data-safety advisory (#8): warn LOUDLY at boot if running on a single SQLite file with no acknowledged backup,
// so an operator doesn't run with zero durability while /ready answers 200.
const backupAdvisory = sqliteBackupAdvisory({
const sqliteBackupOpts = {
usingSqlite: !usePostgres,
backupAcknowledged: process.env.BACKUP_ACKNOWLEDGED === "true",
});
};
const backupAdvisory = sqliteBackupAdvisory(sqliteBackupOpts);
if (backupAdvisory)
console.warn(
JSON.stringify({
Expand Down Expand Up @@ -688,6 +690,7 @@ async function main(): Promise<void> {
gauge("gittensory_uptime_seconds", () =>
Math.floor((Date.now() - startedAt) / 1000),
);
gauge("gittensory_backup_acknowledged", () => backupAcknowledgedGaugeValue(sqliteBackupOpts));
// Pre-initialize job counters to 0 so they appear in the first Prometheus scrape (lazy counters
// created on first use would otherwise cause "No data" in Grafana until the first job event).
for (const c of [
Expand Down
9 changes: 9 additions & 0 deletions test/unit/selfhost-health.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { join } from "node:path";
import { describe, expect, it, vi } from "vitest";
import { createD1Adapter, nodeSqliteDriver } from "../../src/selfhost/d1-adapter";
import {
backupAcknowledgedGaugeValue,
buildHealthBody,
codexAuthReadinessProbe,
githubAppReadinessProbe,
Expand Down Expand Up @@ -210,6 +211,14 @@ describe("sqliteBackupAdvisory (#8 data-safety)", () => {
});
});

describe("backupAcknowledgedGaugeValue (#2089)", () => {
it("mirrors the advisory: 0 only when SQLite has no acknowledged backup", () => {
expect(backupAcknowledgedGaugeValue({ usingSqlite: true, backupAcknowledged: false })).toBe(0);
expect(backupAcknowledgedGaugeValue({ usingSqlite: true, backupAcknowledged: true })).toBe(1);
expect(backupAcknowledgedGaugeValue({ usingSqlite: false, backupAcknowledged: false })).toBe(1);
});
});

describe("readiness (#982)", () => {
afterEach(() => {
vi.restoreAllMocks();
Expand Down
Loading