From 687a72305892ac3e31bb5c6f2929e3ddd8d42f50 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Fri, 26 Jun 2026 04:12:31 -0700 Subject: [PATCH] feat(selfhost): /ready gates on configured optional backends (Redis, Qdrant) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit readiness() only checked the DB + migrations, so an instance could report ready while a backend it depends on was down — and a multi-instance load balancer would keep routing to it. readiness() now also runs a probe per CONFIGURED optional backend (Redis ping, Qdrant reachability), each owning a short timeout so a hung backend can't hang /ready, and reports every result in `checks`. A configured backend that fails to answer makes the instance not-ready. server.ts (codecov- ignored bootstrap) wires the probes only when REDIS_URL / QDRANT_URL are set. --- src/selfhost/health.ts | 22 +++++++++++++++++++--- src/server.ts | 17 +++++++++++++---- test/unit/selfhost-health.test.ts | 13 +++++++++++++ 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/src/selfhost/health.ts b/src/selfhost/health.ts index 26d0359d9a..4accc343a3 100644 --- a/src/selfhost/health.ts +++ b/src/selfhost/health.ts @@ -7,8 +7,16 @@ export interface Readiness { checks: Record; } -/** Readiness: the DB answers a trivial query and the migrations table shows applied rows. */ -export async function readiness(db: D1Database): Promise { +/** An extra readiness check for a CONFIGURED optional backend (Redis, Qdrant …). `check` resolves true when the + * backend is reachable; it OWNS its own timeout (the caller wires it that way) so a hung backend can't hang /ready. + * A configured backend that fails to answer means the instance is degraded — a multi-instance load balancer should + * stop routing to it — so every probe gates readiness. */ +export type ReadinessProbe = { name: string; check: () => Promise }; + +/** Readiness: the DB answers a trivial query, the migrations table shows applied rows, and every configured + * optional-backend probe (Redis/Qdrant, when wired) answers. An instance can no longer report ready while a + * backend it actually depends on is down. */ +export async function readiness(db: D1Database, probes: ReadinessProbe[] = []): Promise { let dbOk = false; let migrations = false; try { @@ -24,5 +32,13 @@ export async function readiness(db: D1Database): Promise { } catch { /* migrations table missing */ } - return { ok: dbOk && migrations, checks: { db: dbOk, migrations } }; + const checks: Record = { db: dbOk, migrations }; + for (const probe of probes) { + try { + checks[probe.name] = await probe.check(); + } catch { + checks[probe.name] = false; // an unreachable / erroring backend is not ready + } + } + return { ok: Object.values(checks).every(Boolean), checks }; } diff --git a/src/server.ts b/src/server.ts index 4b6bf2b19f..e0b1b687c2 100644 --- a/src/server.ts +++ b/src/server.ts @@ -28,7 +28,7 @@ import { import { isOrbBrokerMode, registerOrbRelayTarget } from "./orb/broker-client"; import { exportOrbBatch } from "./selfhost/orb-collector"; import { createD1Adapter, nodeSqliteDriver } from "./selfhost/d1-adapter"; -import { readiness } from "./selfhost/health"; +import { readiness, type ReadinessProbe } from "./selfhost/health"; import { gauge, incr, observe, renderMetrics } from "./selfhost/metrics"; import { runSelfHostMigrations } from "./selfhost/migrate"; import { createPgAdapter } from "./selfhost/pg-adapter"; @@ -188,6 +188,12 @@ async function main(): Promise { const aiReviewPlan = resolveAiReviewerPlan(process.env); if (aiReviewPlan) console.log(JSON.stringify({ event: "selfhost_ai_review_plan", reviewers: aiReviewPlan.reviewers.map((r) => r.model), combine: aiReviewPlan.combine })); + // /ready gates on every CONFIGURED optional backend (below) so a load balancer never routes to an instance whose + // Redis/Qdrant is down. Each probe owns a short timeout so a hung backend can't hang the readiness check. + const readinessProbes: ReadinessProbe[] = []; + const withTimeout = (p: Promise, ms = 1500): Promise => + Promise.race([p, new Promise((resolve) => setTimeout(() => resolve(false), ms))]); + // Redis fixed-window rate limiter + webhook dedup cache (else absent when REDIS_URL is unset). let rateLimiter: DurableObjectNamespace | undefined; let webhookCache: import("./selfhost/redis-cache").RedisCache | undefined; @@ -198,16 +204,19 @@ async function main(): Promise { const { createRedisCache } = await import("./selfhost/redis-cache"); rateLimiter = createRedisRateLimiter(redisClient); webhookCache = createRedisCache(redisClient); + readinessProbes.push({ name: "redis", check: () => withTimeout(redisClient.ping().then(() => true)) }); console.log(JSON.stringify({ event: "selfhost_rate_limiter", backend: "redis" })); } // Qdrant vector store — overrides the backend's built-in sqlite-vec / pgvector when QDRANT_URL is set. let vectorizeOverride: Vectorize | undefined; if (process.env.QDRANT_URL) { + const qdrantUrl = process.env.QDRANT_URL; const { createQdrantVectorize, initQdrantCollection } = await import("./selfhost/qdrant-vectorize"); // Retry until Qdrant accepts the collection PUT — the container may still be booting when we start. - await retryUntilReady("qdrant", () => initQdrantCollection(process.env.QDRANT_URL as string)); - vectorizeOverride = createQdrantVectorize(process.env.QDRANT_URL); + await retryUntilReady("qdrant", () => initQdrantCollection(qdrantUrl)); + vectorizeOverride = createQdrantVectorize(qdrantUrl); + readinessProbes.push({ name: "qdrant", check: () => withTimeout(fetch(qdrantUrl, { signal: AbortSignal.timeout(1500) }).then((r) => r.ok).catch(() => false)) }); console.log(JSON.stringify({ event: "selfhost_vectorize", backend: "qdrant" })); } @@ -255,7 +264,7 @@ async function main(): Promise { const path = new URL(request.url).pathname; if (path === "/health") return new Response(JSON.stringify({ status: "ok" }), { headers: { "content-type": "application/json" } }); if (path === "/ready") { - const r = await readiness(backend.db); + const r = await readiness(backend.db, readinessProbes); return new Response(JSON.stringify(r), { status: r.ok ? 200 : 503, headers: { "content-type": "application/json" } }); } if (path === "/metrics") return new Response(await renderMetrics(), { headers: { "content-type": "text/plain; version=0.0.4" } }); diff --git a/test/unit/selfhost-health.test.ts b/test/unit/selfhost-health.test.ts index add268dad0..d9b3506b68 100644 --- a/test/unit/selfhost-health.test.ts +++ b/test/unit/selfhost-health.test.ts @@ -32,4 +32,17 @@ describe("readiness (#982)", () => { } as unknown as D1Database; expect(await readiness(throwingDb)).toEqual({ ok: false, checks: { db: false, migrations: false } }); }); + + it("gates readiness on configured backend probes (#4) and reports each in checks", async () => { + const driver = nodeSqliteDriver(new DatabaseSync(":memory:") as never); + const db = createD1Adapter(driver); + driver.exec("CREATE TABLE _selfhost_migrations (name TEXT, applied_at INTEGER)"); + driver.query("INSERT INTO _selfhost_migrations (name, applied_at) VALUES (?, ?)", ["0001", 0]); + // A healthy probe → still ready, reported in checks. + expect(await readiness(db, [{ name: "redis", check: async () => true }])).toEqual({ ok: true, checks: { db: true, migrations: true, redis: true } }); + // A failing probe → NOT ready (a configured backend that's down means the instance is degraded). + expect(await readiness(db, [{ name: "redis", check: async () => false }])).toEqual({ ok: false, checks: { db: true, migrations: true, redis: false } }); + // A throwing probe → caught → false → not ready. + expect(await readiness(db, [{ name: "qdrant", check: async () => { throw new Error("unreachable"); } }])).toEqual({ ok: false, checks: { db: true, migrations: true, qdrant: false } }); + }); });