From 0400411c5ad39551c28f4760baecda2871d39384 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 23 Jun 2026 19:19:13 -0700 Subject: [PATCH 01/25] =?UTF-8?q?feat(selfhost):=20D1-over-SQLite=20adapte?= =?UTF-8?q?r=20=E2=80=94=20gittensory's=20data=20layer=20runs=20unchanged?= =?UTF-8?q?=20on=20Node=20(#980)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The foundation of the self-host stack: a faithful D1Database implementation over a sync SQLite driver (node:sqlite by default — built in, no native build), so EVERY data path runs UNCHANGED on a local file: drizzle-orm/d1 (the ~171 getDb call sites), the raw env.DB.prepare/bind/all/first/run/batch sites, and the test suite. Driver-injected (SqliteDriver seam) so the Worker bundle never imports it. Validated: all 56 real migrations apply, drizzle select returns fully-mapped rows over the shim, the raw API + atomic batch (with rollback) all correct. + unit tests. --- src/selfhost/d1-adapter.ts | 123 ++++++++++++++++++++++++++ test/unit/selfhost-d1-adapter.test.ts | 54 +++++++++++ 2 files changed, 177 insertions(+) create mode 100644 src/selfhost/d1-adapter.ts create mode 100644 test/unit/selfhost-d1-adapter.test.ts diff --git a/src/selfhost/d1-adapter.ts b/src/selfhost/d1-adapter.ts new file mode 100644 index 0000000000..8f77ed3247 --- /dev/null +++ b/src/selfhost/d1-adapter.ts @@ -0,0 +1,123 @@ +// Self-host D1 adapter (#980). A FAITHFUL D1Database implementation over a synchronous SQLite driver, so +// EVERY data-access path in gittensory runs UNCHANGED on a local file: +// • drizzle-orm/d1 (getDb → the ~171 repository call sites) — calls bind/all/run/raw/batch + reads .results +// • the raw `env.DB.prepare(sql).bind(...).all()/.first()/.run()/.batch()` sites +// • the test suite, which uses the same D1 surface +// D1's API is async; the SQLite drivers are sync — sync calls are wrapped in resolved Promises. The driver is +// INJECTED behind the tiny SqliteDriver interface, so this module has no hard SQLite dependency and the +// Cloudflare Worker bundle never imports it. Default driver: node:sqlite (built into Node, no native build). + +/** A uniform sync SQLite primitive both node:sqlite and better-sqlite3 can satisfy via a thin wrapper. `query` + * ALWAYS returns rows (empty for a write) + the write metadata, so the adapter needs no reader-detection. */ +export interface SqliteDriver { + query(sql: string, params: unknown[]): { rows: Record[]; changes: number; lastInsertRowid: number }; + exec(sql: string): void; +} + +function meta(changes = 0, lastRowId = 0): Record { + return { duration: 0, size_after: 0, rows_read: 0, rows_written: changes, last_row_id: lastRowId, changed_db: changes > 0, changes }; +} + +/** One prepared (and optionally bound) statement. bind() returns a fresh instance (D1 statements are immutable + * after bind). The SQLite statement is compiled per execution (drivers cache by SQL text). */ +class Statement { + constructor( + private readonly driver: SqliteDriver, + private readonly sql: string, + private readonly values: unknown[] = [], + ) {} + + bind(...values: unknown[]): Statement { + return new Statement(this.driver, this.sql, values); + } + + /** Sync core used by all()/run() (async wrappers) and batch() (inside a transaction). */ + execSync(): { results: unknown[]; success: boolean; meta: Record } { + const r = this.driver.query(this.sql, this.values); + return { results: r.rows, success: true, meta: meta(r.changes, r.lastInsertRowid) }; + } + + async all(): Promise<{ results: T[]; success: boolean; meta: Record }> { + return this.execSync() as { results: T[]; success: boolean; meta: Record }; + } + + // D1's run() returns the same {results, meta} shape (results empty for a non-returning write). + async run(): Promise<{ results: T[]; success: boolean; meta: Record }> { + return this.execSync() as { results: T[]; success: boolean; meta: Record }; + } + + async first(colName?: string): Promise { + const row = this.driver.query(this.sql, this.values).rows[0]; + if (row == null) return null; + return ((colName != null ? row[colName] : row) ?? null) as T | null; + } + + async raw(): Promise { + // D1 raw() returns each row as an array of column values (column order preserved). + return this.driver.query(this.sql, this.values).rows.map((row) => Object.values(row)) as T[]; + } +} + +/** Wrap a synchronous SQLite driver as a D1Database. */ +export function createD1Adapter(driver: SqliteDriver): D1Database { + const adapter = { + prepare(sql: string) { + return new Statement(driver, sql); + }, + async batch(statements: unknown[]) { + // D1 runs a batch atomically, one result per statement, in order. + const list = statements as Statement[]; + driver.exec("BEGIN"); + try { + const out = list.map((s) => s.execSync()); + driver.exec("COMMIT"); + return out; + } catch (error) { + try { + driver.exec("ROLLBACK"); + } catch { + /* ignore */ + } + throw error; + } + }, + async exec(sql: string) { + driver.exec(sql); // runs one or more statements (used for migrations) + return { count: (sql.match(/;/g) ?? []).length || 1, duration: 0 }; + }, + async dump() { + return new ArrayBuffer(0); // unused by gittensory; present for D1 surface completeness + }, + }; + return adapter as unknown as D1Database; +} + +/** The minimal node:sqlite surface the wrapper uses (DatabaseSync + StatementSync). */ +interface NodeSqliteStatement { + columns(): unknown[]; + all(...params: unknown[]): unknown[]; + get(...params: unknown[]): unknown; + run(...params: unknown[]): { changes: number | bigint; lastInsertRowid: number | bigint }; +} +interface NodeSqliteDatabase { + prepare(sql: string): NodeSqliteStatement; + exec(sql: string): void; +} + +/** Build a SqliteDriver from a node:sqlite DatabaseSync. A statement with zero result columns is a WRITE + * (run → changes); otherwise a READ (all → rows). */ +export function nodeSqliteDriver(db: NodeSqliteDatabase): SqliteDriver { + return { + query(sql, params) { + const stmt = db.prepare(sql); + if (stmt.columns().length > 0) { + return { rows: stmt.all(...params) as Record[], changes: 0, lastInsertRowid: 0 }; + } + const info = stmt.run(...params); + return { rows: [], changes: Number(info.changes), lastInsertRowid: Number(info.lastInsertRowid) }; + }, + exec(sql) { + db.exec(sql); + }, + }; +} diff --git a/test/unit/selfhost-d1-adapter.test.ts b/test/unit/selfhost-d1-adapter.test.ts new file mode 100644 index 0000000000..f2048ee838 --- /dev/null +++ b/test/unit/selfhost-d1-adapter.test.ts @@ -0,0 +1,54 @@ +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, it } from "vitest"; +import { createD1Adapter, nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; + +function makeD1(): D1Database { + const db = new DatabaseSync(":memory:"); + return createD1Adapter(nodeSqliteDriver(db as never)); +} + +describe("createD1Adapter (#980 self-host D1-over-SQLite)", () => { + it("implements the D1 surface faithfully: prepare/bind/all/first/raw on reads", async () => { + const d1 = makeD1(); + await d1.exec("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + await d1.prepare("INSERT INTO t (name) VALUES (?)").bind("a").run(); + await d1.prepare("INSERT INTO t (name) VALUES (?)").bind("b").run(); + + expect((await d1.prepare("SELECT count(*) AS n FROM t").first<{ n: number }>())?.n).toBe(2); + expect((await d1.prepare("SELECT name FROM t WHERE id = ?").bind(1).first<{ name: string }>())?.name).toBe("a"); + expect(await d1.prepare("SELECT name FROM t WHERE id = ?").bind(1).first("name")).toBe("a"); // colName form + expect(await d1.prepare("SELECT * FROM t WHERE id = 99").first()).toBeNull(); // no row → null + + const all = await d1.prepare("SELECT id, name FROM t ORDER BY id").all<{ id: number; name: string }>(); + expect(all.results).toEqual([{ id: 1, name: "a" }, { id: 2, name: "b" }]); + const raw = await d1.prepare("SELECT id, name FROM t ORDER BY id").raw(); + expect(raw).toEqual([[1, "a"], [2, "b"]]); // raw() = arrays of column values + }); + + it("run() reports changes/last_row_id; batch is atomic", async () => { + const d1 = makeD1(); + await d1.exec("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + const r = await d1.prepare("INSERT INTO t (name) VALUES (?)").bind("x").run(); + expect(r.meta.changes).toBe(1); + expect(r.meta.last_row_id).toBe(1); + + await d1.batch([ + d1.prepare("INSERT INTO t (name) VALUES (?)").bind("y"), + d1.prepare("INSERT INTO t (name) VALUES (?)").bind("z"), + ]); + expect((await d1.prepare("SELECT count(*) AS n FROM t").first<{ n: number }>())?.n).toBe(3); + }); + + it("batch rolls back entirely on an error (atomicity)", async () => { + const d1 = makeD1(); + await d1.exec("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT UNIQUE)"); + await d1.prepare("INSERT INTO t (name) VALUES (?)").bind("dup").run(); + await expect( + d1.batch([ + d1.prepare("INSERT INTO t (name) VALUES (?)").bind("ok"), + d1.prepare("INSERT INTO t (name) VALUES (?)").bind("dup"), // UNIQUE violation + ]), + ).rejects.toThrow(); + expect((await d1.prepare("SELECT count(*) AS n FROM t").first<{ n: number }>())?.n).toBe(1); // "ok" rolled back + }); +}); From 408d43e1641a32b167baf3a8077c7337aa0579b2 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 23 Jun 2026 19:35:03 -0700 Subject: [PATCH 02/25] =?UTF-8?q?feat(selfhost):=20Node=20entry=20?= =?UTF-8?q?=E2=80=94=20gittensory's=20full=20Worker=20stack=20boots=20on?= =?UTF-8?q?=20Node=20(#980)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds an Env where the Cloudflare bindings are self-host adapters (D1->node:sqlite, Queue->in-process) and runs gittensory's SAME handlers: serves the Hono app via @hono/node-server, drains the in-process queue with the same processJob, ticks the same scheduled handler on a timer, applies all 56 migrations at startup, and resolves *_FILE secrets. The Cloudflare Worker (src/index.ts) is untouched. - src/server.ts (Node entry) + src/selfhost/{migrate,queue}.ts - esbuild bundle (scripts/build-selfhost.mjs) + a registerHooks loader (scripts/register-selfhost.mjs) that stubs every cloudflare:* import; the two Workers-only deps (@cloudflare/puppeteer, agents/mcp) are build-stubbed and degrade on self-host (BROWSER absent; /mcp -> 501) - validated end-to-end: boots, 56 migrations applied, /health 200, real Hono+D1 routes respond (/v1/public/stats flag-off 404), + queue/d1 unit tests. Full typecheck clean. --- package-lock.json | 21 +++++++-- package.json | 1 + scripts/build-selfhost.mjs | 30 ++++++++++++ scripts/register-selfhost.mjs | 29 ++++++++++++ src/selfhost/cf-workers-shim.ts | 18 +++++++ src/selfhost/migrate.ts | 21 +++++++++ src/selfhost/queue.ts | 62 +++++++++++++++++++++++++ src/selfhost/stubs/agents-mcp.ts | 7 +++ src/selfhost/stubs/puppeteer.ts | 8 ++++ src/server.ts | 80 ++++++++++++++++++++++++++++++++ test/unit/selfhost-queue.test.ts | 30 ++++++++++++ 11 files changed, 303 insertions(+), 4 deletions(-) create mode 100644 scripts/build-selfhost.mjs create mode 100644 scripts/register-selfhost.mjs create mode 100644 src/selfhost/cf-workers-shim.ts create mode 100644 src/selfhost/migrate.ts create mode 100644 src/selfhost/queue.ts create mode 100644 src/selfhost/stubs/agents-mcp.ts create mode 100644 src/selfhost/stubs/puppeteer.ts create mode 100644 src/server.ts create mode 100644 test/unit/selfhost-queue.test.ts diff --git a/package-lock.json b/package-lock.json index 7e43e83906..1c1ce319c3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "dependencies": { "@asteasolutions/zod-to-openapi": "^8.5.0", "@cloudflare/puppeteer": "^1.1.0", + "@hono/node-server": "^2.0.6", "@modelcontextprotocol/sdk": "1.29.0", "@octokit/core": "^7.0.6", "agents": "^0.16.2", @@ -1932,12 +1933,12 @@ "license": "MIT" }, "node_modules/@hono/node-server": { - "version": "1.19.14", - "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz", - "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==", + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.0.6.tgz", + "integrity": "sha512-7DeRlKG57JDBNZ5Qj2jwVdgwQy4b0tLubRLl3zCf91/rCf9i7p1V5FtW/yWibm1uUHE493ts9ZXH/7g/LQWl+g==", "license": "MIT", "engines": { - "node": ">=18.14.1" + "node": ">=20" }, "peerDependencies": { "hono": "^4" @@ -2716,6 +2717,18 @@ } } }, + "node_modules/@modelcontextprotocol/sdk/node_modules/@hono/node-server": { + "version": "1.19.14", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz", + "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==", + "license": "MIT", + "engines": { + "node": ">=18.14.1" + }, + "peerDependencies": { + "hono": "^4" + } + }, "node_modules/@modelcontextprotocol/sdk/node_modules/content-type": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", diff --git a/package.json b/package.json index b33cce39d7..2d1fe9a813 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "dependencies": { "@asteasolutions/zod-to-openapi": "^8.5.0", "@cloudflare/puppeteer": "^1.1.0", + "@hono/node-server": "^2.0.6", "@modelcontextprotocol/sdk": "1.29.0", "@octokit/core": "^7.0.6", "agents": "^0.16.2", diff --git a/scripts/build-selfhost.mjs b/scripts/build-selfhost.mjs new file mode 100644 index 0000000000..fc5031f399 --- /dev/null +++ b/scripts/build-selfhost.mjs @@ -0,0 +1,30 @@ +// Bundle the self-host Node entry (src/server.ts) into dist/server.mjs. node_modules stay external (resolved +// at runtime); `cloudflare:workers` is resolved to the Node shim via a plugin (which takes precedence over +// `packages: "external"`, so it is BUNDLED rather than left as an unresolvable bare import). +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import esbuild from "esbuild"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +await esbuild.build({ + entryPoints: [resolve(root, "src/server.ts")], + bundle: true, + platform: "node", + format: "esm", + target: "node22", + outfile: resolve(root, "dist/server.mjs"), + packages: "external", + plugins: [ + { + name: "selfhost-stubs", + setup(build) { + // Cloudflare-only modules → Node stubs (their features are inert/degraded on self-host). + build.onResolve({ filter: /^cloudflare:workers$/ }, () => ({ path: resolve(root, "src/selfhost/cf-workers-shim.ts") })); + build.onResolve({ filter: /^@cloudflare\/puppeteer$/ }, () => ({ path: resolve(root, "src/selfhost/stubs/puppeteer.ts") })); + build.onResolve({ filter: /^agents\/mcp$/ }, () => ({ path: resolve(root, "src/selfhost/stubs/agents-mcp.ts") })); + }, + }, + ], + logLevel: "info", +}); diff --git a/scripts/register-selfhost.mjs b/scripts/register-selfhost.mjs new file mode 100644 index 0000000000..a04c36e2a2 --- /dev/null +++ b/scripts/register-selfhost.mjs @@ -0,0 +1,29 @@ +// Self-host module-resolution hooks (run before the app loads). Any `cloudflare:*` import — from gittensory's +// source OR a transitive dep (@cloudflare/puppeteer, the agents SDK / partyserver) — resolves to an in-memory +// stub. These bindings are never USED on self-host (BROWSER/RATE_LIMITER/email absent → the code degrades +// before touching them); the stub only makes the import + any `extends`/named import resolve so Node can load +// the graph. Used as the Docker entry: `node --import ./scripts/register-selfhost.mjs dist/server.mjs`. +import { registerHooks } from "node:module"; + +const STUB_SOURCE = [ + "export class DurableObject { constructor(ctx, env) { this.ctx = ctx; this.env = env; } }", + "export class WorkerEntrypoint { constructor(ctx, env) { this.ctx = ctx; this.env = env; } }", + "export class WorkflowEntrypoint { constructor(ctx, env) { this.ctx = ctx; this.env = env; } }", + "export class RpcTarget {}", + "export class EmailMessage { constructor(from, to, raw) { this.from = from; this.to = to; this.raw = raw; } }", + "export const env = {};", + "export const WorkerVersionMetadata = {};", + "export function connect() { throw new Error('cloudflare:sockets is unavailable on the self-host runtime'); }", + "export default {};", +].join("\n"); + +registerHooks({ + resolve(specifier, context, nextResolve) { + if (specifier.startsWith("cloudflare:")) return { url: `cfstub:${specifier}`, shortCircuit: true }; + return nextResolve(specifier, context); + }, + load(url, context, nextLoad) { + if (url.startsWith("cfstub:")) return { format: "module", shortCircuit: true, source: STUB_SOURCE }; + return nextLoad(url, context); + }, +}); diff --git a/src/selfhost/cf-workers-shim.ts b/src/selfhost/cf-workers-shim.ts new file mode 100644 index 0000000000..b59af6f08a --- /dev/null +++ b/src/selfhost/cf-workers-shim.ts @@ -0,0 +1,18 @@ +// Minimal stand-in for the `cloudflare:workers` module on the Node self-host runtime. The only import of it +// in the codebase is `DurableObject` (auth/rate-limit.ts → the RateLimiter DO). That DO is NEVER instantiated +// on self-host — env.RATE_LIMITER is undefined, so enforceRateLimit returns null before any DO is touched — +// so this base class only needs to make the import + `extends DurableObject` resolve. The self-host esbuild +// build aliases `cloudflare:workers` to this file (see the Docker build / build:selfhost script). +export class DurableObject { + constructor( + protected ctx?: unknown, + protected env?: E, + ) {} +} +export class WorkerEntrypoint { + constructor( + protected ctx?: unknown, + protected env?: E, + ) {} +} +export class RpcTarget {} diff --git a/src/selfhost/migrate.ts b/src/selfhost/migrate.ts new file mode 100644 index 0000000000..b1fc898427 --- /dev/null +++ b/src/selfhost/migrate.ts @@ -0,0 +1,21 @@ +// Apply gittensory's D1 migrations to the self-host SQLite database at startup. The same `migrations/*.sql` +// files Cloudflare applies via `wrangler d1 migrations apply` — they're plain SQLite DDL, so they run as-is +// through the D1 adapter's exec(). Tracked in a `_selfhost_migrations` table so a restart re-applies only the +// new ones (idempotent), mirroring wrangler's migration ledger. +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +export async function runSelfHostMigrations(db: D1Database, dir: string): Promise { + await db.exec("CREATE TABLE IF NOT EXISTS _selfhost_migrations (name TEXT PRIMARY KEY, applied_at TEXT NOT NULL)"); + const existing = await db.prepare("SELECT name FROM _selfhost_migrations").all<{ name: string }>(); + const applied = new Set(existing.results.map((r) => r.name)); + const files = readdirSync(dir).filter((f) => f.endsWith(".sql")).sort(); + let count = 0; + for (const file of files) { + if (applied.has(file)) continue; + await db.exec(readFileSync(join(dir, file), "utf8")); + await db.prepare("INSERT INTO _selfhost_migrations (name, applied_at) VALUES (?, ?)").bind(file, new Date().toISOString()).run(); + count += 1; + } + return count; +} diff --git a/src/selfhost/queue.ts b/src/selfhost/queue.ts new file mode 100644 index 0000000000..a8afe48859 --- /dev/null +++ b/src/selfhost/queue.ts @@ -0,0 +1,62 @@ +// Self-host in-process job queue (#980). Replaces the Cloudflare Queue (env.JOBS) on a single container: a +// `Queue`-shaped binding whose send() enqueues a JobMessage (honoring delaySeconds), and an async worker that +// drains FIFO and invokes the same processJob the Worker's queue() handler uses. Failures retry up to +// maxRetries then drop (logged), mirroring the Queues DLQ at small scale. (A Redis/BullMQ backend is a +// follow-up for multi-replica; the cron sweep is the backstop either way.) +import type { JobMessage } from "../types"; + +export interface SelfHostQueue { + /** The env.JOBS binding (send / sendBatch). */ + binding: Queue; + /** Resolve when the queue is empty (tests / graceful shutdown). */ + drain(): Promise; + size(): number; +} + +export function createInProcessQueue(consume: (message: JobMessage) => Promise, opts: { maxRetries?: number } = {}): SelfHostQueue { + const maxRetries = opts.maxRetries ?? 3; + const queue: Array<{ message: JobMessage; attempts: number }> = []; + let working = false; + + async function pump(): Promise { + if (working) return; + working = true; + try { + while (queue.length > 0) { + const item = queue.shift(); + if (!item) break; + try { + await consume(item.message); + } catch (error) { + if (item.attempts + 1 < maxRetries) queue.push({ message: item.message, attempts: item.attempts + 1 }); + else console.error(JSON.stringify({ level: "error", event: "inproc_job_dropped", attempts: item.attempts + 1, error: error instanceof Error ? error.message : "unknown error" })); + } + } + } finally { + working = false; + } + } + + const send = (message: JobMessage, options?: { delaySeconds?: number }): Promise => { + const delayMs = (options?.delaySeconds ?? 0) * 1000; + if (delayMs > 0) { + setTimeout(() => { + queue.push({ message, attempts: 0 }); + void pump(); + }, delayMs); + } else { + queue.push({ message, attempts: 0 }); + void pump(); + } + return Promise.resolve(); + }; + + const binding = { + send, + sendBatch: async (messages: Iterable<{ body: JobMessage }>) => { + for (const m of messages) await send(m.body); + }, + } as unknown as Queue; + + return { binding, drain: pump, size: () => queue.length }; +} diff --git a/src/selfhost/stubs/agents-mcp.ts b/src/selfhost/stubs/agents-mcp.ts new file mode 100644 index 0000000000..63721aac5c --- /dev/null +++ b/src/selfhost/stubs/agents-mcp.ts @@ -0,0 +1,7 @@ +// Self-host stub for agents/mcp. The Cloudflare Agents SDK MCP server is Durable-Object-backed (Workers-only), +// so on self-host the /mcp route degrades to 501 rather than dragging the Workers runtime into Node. (A native +// MCP-on-Node port is a follow-up.) Matches the createMcpHandler(...) → fetch-handler shape the caller expects. +export function createMcpHandler(..._args: unknown[]): (...args: unknown[]) => Promise { + return async () => + new Response(JSON.stringify({ error: "mcp_unavailable_on_selfhost" }), { status: 501, headers: { "content-type": "application/json" } }); +} diff --git a/src/selfhost/stubs/puppeteer.ts b/src/selfhost/stubs/puppeteer.ts new file mode 100644 index 0000000000..9905e808e4 --- /dev/null +++ b/src/selfhost/stubs/puppeteer.ts @@ -0,0 +1,8 @@ +// Self-host stub for @cloudflare/puppeteer (Browser Rendering is a Cloudflare-only binding). The only caller, +// review/visual/shot.ts, does `if (!env.BROWSER) return {...}` BEFORE puppeteer.launch — and BROWSER is absent +// on self-host — so launch is never reached. This stub just makes the import resolve (no cloudflare:* imports). +const unavailable = (): never => { + throw new Error("Browser Rendering (@cloudflare/puppeteer) is unavailable on the self-host runtime"); +}; + +export default { launch: unavailable, connect: unavailable }; diff --git a/src/server.ts b/src/server.ts new file mode 100644 index 0000000000..0befe5a365 --- /dev/null +++ b/src/server.ts @@ -0,0 +1,80 @@ +// Self-host Node entry (#980). Runs gittensory's SAME Worker handlers on Node: builds an `Env` where the +// Cloudflare bindings are self-host adapters (D1→node:sqlite, Queue→in-process), serves the Hono app via +// @hono/node-server, drives the in-process queue with the same processJob, and ticks the same scheduled +// handler on a timer. The Cloudflare Worker (src/index.ts) is untouched — this is a parallel entry the +// self-host esbuild build bundles (aliasing `cloudflare:workers` to the shim). +import { readFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; +import { serve } from "@hono/node-server"; +import worker from "./index"; +import { processJob } from "./queue/processors"; +import { createD1Adapter, nodeSqliteDriver } from "./selfhost/d1-adapter"; +import { runSelfHostMigrations } from "./selfhost/migrate"; +import { createInProcessQueue } from "./selfhost/queue"; +import type { JobMessage } from "./types"; + +/** Resolve `_FILE` env vars (Docker secrets / multi-line keys) into `` at startup. */ +function loadFileSecrets(): void { + for (const key of Object.keys(process.env)) { + if (!key.endsWith("_FILE") || !process.env[key]) continue; + const target = key.slice(0, -"_FILE".length); + if (process.env[target]) continue; // an explicit value wins + try { + process.env[target] = readFileSync(process.env[key] as string, "utf8").trim(); + } catch { + console.error(JSON.stringify({ level: "error", event: "selfhost_secret_file_unreadable", var: key })); + } + } +} + +async function main(): Promise { + loadFileSecrets(); + + const sqlite = new DatabaseSync(process.env.DATABASE_PATH ?? "/data/gittensory.sqlite"); + sqlite.exec("PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON;"); + const db = createD1Adapter(nodeSqliteDriver(sqlite as never)); + const applied = await runSelfHostMigrations(db, process.env.MIGRATIONS_DIR ?? "migrations"); + console.log(JSON.stringify({ event: "selfhost_migrations_applied", count: applied })); + + // The queue consumer captures `env`, assigned just below — the first send only happens once an HTTP/cron + // event arrives, by which point env is set. + let env: Env; + const queue = createInProcessQueue(async (message: JobMessage) => { + await processJob(env, message); + }); + env = { ...process.env, DB: db, JOBS: queue.binding, AI: undefined } as unknown as Env; + + const ctx = { + waitUntil: (p: Promise) => void Promise.resolve(p).catch(() => undefined), + passThroughOnException: () => undefined, + } as unknown as ExecutionContext; + + const port = Number(process.env.PORT ?? 8787); + serve( + { + fetch: (request: Request) => { + // A binding-free liveness probe (the Hono app also exempts /health from auth + rate-limit). + if (new URL(request.url).pathname === "/health") { + return new Response(JSON.stringify({ status: "ok" }), { headers: { "content-type": "application/json" } }); + } + return worker.fetch(request, env, ctx); + }, + port, + }, + () => console.log(JSON.stringify({ event: "selfhost_listening", port })), + ); + + // Cron — gittensory ticks ~every 2 minutes; drive the SAME scheduled handler. + const intervalMs = Number(process.env.CRON_INTERVAL_MS ?? 120_000); + setInterval(() => { + const controller = { scheduledTime: Date.now(), cron: "*/2 * * * *", noRetry: () => undefined } as unknown as ScheduledController; + Promise.resolve(worker.scheduled(controller, env, ctx)).catch((error) => + console.error(JSON.stringify({ level: "error", event: "selfhost_cron_error", error: error instanceof Error ? error.message : "unknown error" })), + ); + }, intervalMs); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/test/unit/selfhost-queue.test.ts b/test/unit/selfhost-queue.test.ts new file mode 100644 index 0000000000..e82eefbbe7 --- /dev/null +++ b/test/unit/selfhost-queue.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; +import { createInProcessQueue } from "../../src/selfhost/queue"; +import type { JobMessage } from "../../src/types"; + +describe("createInProcessQueue (#980 self-host job queue)", () => { + it("send() enqueues; the worker drains FIFO through the consumer", async () => { + const seen: JobMessage[] = []; + const q = createInProcessQueue(async (m) => void seen.push(m)); + await q.binding.send({ type: "a" } as unknown as JobMessage); + await q.binding.send({ type: "b" } as unknown as JobMessage); + await q.drain(); + expect(seen).toEqual([{ type: "a" }, { type: "b" }]); + expect(q.size()).toBe(0); + }); + + it("retries a failing job up to maxRetries, then drops it (never throws)", async () => { + let calls = 0; + const q = createInProcessQueue( + async () => { + calls += 1; + throw new Error("boom"); + }, + { maxRetries: 2 }, + ); + await q.binding.send({ type: "x" } as unknown as JobMessage); + await q.drain(); + expect(calls).toBe(2); // first attempt + one retry, then dropped + expect(q.size()).toBe(0); + }); +}); From 957ed9b79e9c07d714d7cce7eac656a351ea9bb8 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 23 Jun 2026 19:39:48 -0700 Subject: [PATCH 03/25] =?UTF-8?q?feat(selfhost):=20Dockerfile=20+=20docker?= =?UTF-8?q?-compose=20=E2=80=94=20one-command=20self-host=20(#980)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `docker compose up --build` boots the full gittensory-api on Node: applies all 56 migrations to a SQLite volume, serves /health + the Hono app, drives the in-process queue + cron. Multi-stage non-root image; secrets are NEVER baked (env_file / *_FILE). Sample config only — .env.example gains a self-host section (placeholders), .env is gitignored. - Dockerfile (build: npm ci --ignore-scripts + esbuild bundle; runtime: node:24-slim, non-root, healthcheck) - docker-compose.yml (gittensory service + optional Ollama for local AI) + .dockerignore - validated: image builds + boots, 56 migrations applied, /health 200, real Hono+D1 route responds (404 on a flag-off endpoint, proving the app + DB are live in the container) --- .dockerignore | 20 ++++++++++++++++++++ .env.example | 19 +++++++++++++++++++ .gitignore | 3 +++ Dockerfile | 35 +++++++++++++++++++++++++++++++++++ docker-compose.yml | 41 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 118 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..6ffdf51695 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,20 @@ +# Keep the build context lean — deps are installed fresh (npm ci) and the bundle is built in the image. +node_modules +**/node_modules +dist +dist-ssr +.output +.nitro +.tanstack +.wrangler +.playwright-cli +coverage +.git +.claude +.DS_Store +*.tsbuildinfo +# Never ship secrets into the build context +.env +.env.* +.dev.vars +!.env.example diff --git a/.env.example b/.env.example index 46f25b2c3d..1567a40f51 100644 --- a/.env.example +++ b/.env.example @@ -97,3 +97,22 @@ GITTENSORY_REVIEW_DRAFT=false # GITTENSORY_DRIFT_ISSUE_TOKEN= # token for auto-filing drift issues # GITTENSORY_CONTRIBUTOR_ISSUE_TOKEN= # token for contributor-issue automation # PRODUCT_USAGE_HASH_SALT= # salt for hashing product-usage identifiers + +# ============================================================================= +# 3. Self-host (Docker) — runtime config (#980) +# ============================================================================= +# For `docker compose up` self-hosting (NOT the Cloudflare Worker deploy). Copy this file to `.env` +# (gitignored), UNCOMMENT + fill the required Core secrets in section 2, then add the runtime values below. +# Every value here is a SAMPLE placeholder — never commit real secrets. + +# PORT=8787 +# DATABASE_PATH=/data/gittensory.sqlite # SQLite file on the mounted data volume; all 56 migrations auto-apply +# MIGRATIONS_DIR=/app/migrations +# CRON_INTERVAL_MS=120000 # maintain/sweep + sync cadence (default ~2 min) + +# --- AI review backend (optional; without it reviews run deterministically) --- +# AI_SUMMARIES_ENABLED=true +# AI_PROVIDER=ollama # ollama | openai-compatible | claude-code | codex (see #979) +# AI_BASE_URL=http://ollama:11434/v1 # an OpenAI-compatible endpoint (the Ollama default) +# AI_API_KEY= # if your endpoint requires a key +# WORKERS_AI_SUMMARY_MODEL=llama3.1 # the chat model to use diff --git a/.gitignore b/.gitignore index 0685cfe229..2ffd37d7f8 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,9 @@ dist-ssr/ .playwright-cli/ output/ .dev.vars +.env +.env.* +!.env.example *.local .DS_Store coverage/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..0b8aeb52da --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +# Self-host image for gittensory-api (#980). Runs the SAME Worker handlers on Node via src/server.ts — +# the Cloudflare bindings become self-host adapters (D1 -> node:sqlite, Queue -> in-process). The hosted +# Cloudflare Worker (wrangler) deploy is unaffected. SECRETS ARE NEVER BAKED: supply them at run time via +# the .env file or mounted *_FILE secrets (see docker-compose.yml + .env.example). + +# --- build: install deps + bundle the Node entry -------------------------------------------------------- +FROM node:24-slim AS build +WORKDIR /app +COPY package*.json ./ +# --ignore-scripts: no native builds are needed (SQLite is the built-in node:sqlite; @hono/node-server is +# pure JS; esbuild ships its binary as an optional dependency, not a script). +RUN npm ci --ignore-scripts +COPY . . +RUN node scripts/build-selfhost.mjs + +# --- runtime: slim, non-root ---------------------------------------------------------------------------- +FROM node:24-slim AS runtime +WORKDIR /app +ENV NODE_ENV=production \ + PLATFORM=self-hosted \ + PORT=8787 \ + DATABASE_PATH=/data/gittensory.sqlite \ + MIGRATIONS_DIR=/app/migrations +COPY --from=build /app/node_modules ./node_modules +COPY --from=build /app/dist ./dist +COPY --from=build /app/migrations ./migrations +COPY --from=build /app/scripts/register-selfhost.mjs ./scripts/register-selfhost.mjs +# Data dir (the SQLite file) — owned by the unprivileged node user; mount a volume here to persist. +RUN mkdir -p /data && chown -R node:node /data /app +USER node +EXPOSE 8787 +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ + CMD node -e "fetch('http://127.0.0.1:'+(process.env.PORT||8787)+'/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" +# The register hook stubs cloudflare:* imports so the Worker graph loads on Node. +CMD ["node", "--import", "./scripts/register-selfhost.mjs", "dist/server.mjs"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000..5ba8cf870d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,41 @@ +# One-command self-host for gittensory (#980): docker compose up --build +# +# SECRETS: never baked into the image. Copy .env.example -> .env and fill it in (the file is gitignored). +# AI is OPTIONAL — without it the review path degrades gracefully (no AI summaries); enable Ollama (below) +# or set an OpenAI-compatible / subscription provider in .env to turn it on. The SQLite DB + all 56 schema +# migrations live on the `gittensory-data` volume and are applied automatically at startup. +services: + gittensory: + build: + context: . + ports: + - "8787:8787" + env_file: + # SAMPLE config — `cp .env.example .env` first, then fill in your GitHub App + tokens. + - path: .env + required: false + environment: + PORT: "8787" + DATABASE_PATH: /data/gittensory.sqlite + # Point at the Ollama service below to enable local AI review (uncomment the ollama service too): + # AI_PROVIDER: ollama + # AI_BASE_URL: http://ollama:11434/v1 + volumes: + - gittensory-data:/data + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:8787/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"] + interval: 30s + timeout: 5s + start_period: 20s + retries: 3 + + # Optional local AI backend. Uncomment + set AI_PROVIDER=ollama / AI_BASE_URL above, then once up: + # docker compose exec ollama ollama pull + # ollama: + # image: ollama/ollama:latest + # volumes: + # - ollama-models:/root/.ollama + +volumes: + gittensory-data: + # ollama-models: From e12eb19bdf43ec6203551a5a6dccc0dfca7145ed Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 23 Jun 2026 20:44:56 -0700 Subject: [PATCH 04/25] feat(selfhost): AI provider (#979) + self-hosting guide env.AI on self-host becomes an adapter selected by AI_PROVIDER, returning gittensory's { response } shape: - ollama / openai-compatible / openai -> POST /chat/completions (BYO base URL + key) - claude-code / codex -> the locally-authenticated CLI subscription as a read-only subprocess; billable API keys scrubbed from the child env; any non-zero exit / empty output / Claude-Code is_error envelope THROWS so the caller degrades rather than surfacing an error as the answer - unset AI_PROVIDER -> env.AI undefined -> AI summary degrades to 'unavailable', review stays deterministic Plus docs/self-hosting.md: quick start, GitHub App, config, AI providers, advisory vs full-maintainer modes, operations, and the platform features that degrade on self-host. Wired into src/server.ts; 8 unit tests; typecheck clean; boots with AI_PROVIDER=ollama. --- docs/self-hosting.md | 135 +++++++++++++++++++++++++++ src/selfhost/ai.ts | 167 ++++++++++++++++++++++++++++++++++ src/server.ts | 7 +- test/unit/selfhost-ai.test.ts | 66 ++++++++++++++ 4 files changed, 374 insertions(+), 1 deletion(-) create mode 100644 docs/self-hosting.md create mode 100644 src/selfhost/ai.ts create mode 100644 test/unit/selfhost-ai.test.ts diff --git a/docs/self-hosting.md b/docs/self-hosting.md new file mode 100644 index 0000000000..f92bcc9bb1 --- /dev/null +++ b/docs/self-hosting.md @@ -0,0 +1,135 @@ +# Self-hosting Gittensory + +Gittensory ships as a Cloudflare Worker, but the **same** review engine runs unchanged on a plain Node +container so you can self-host it next to your own GitHub App. `docker compose up` gives you the full +reviewer — webhooks, the deterministic gate, AI summaries, the maintain/sweep cron, and (optionally) full +maintainer autonomy — backed by a local SQLite database. + +> **How it works (one paragraph).** The Worker's Cloudflare bindings are swapped for self-host adapters and +> nothing else changes: **D1 → `node:sqlite`** (a faithful `D1Database` shim, so Drizzle + every raw query + +> all 56 schema migrations run byte-for-byte the same), **Queue → an in-process FIFO worker** (same +> `processJob`), and the **cron** is a timer that calls the same `scheduled()` handler. The Hono app is served +> with `@hono/node-server`. See [`src/server.ts`](../src/server.ts) and [`src/selfhost/`](../src/selfhost). + +--- + +## 1. Quick start + +```bash +cp .env.example .env # then edit .env — see §3 +docker compose up --build +curl localhost:8787/health # {"status":"ok"} +``` + +On first boot the container creates the SQLite database on the `gittensory-data` volume and applies all 56 +migrations automatically (`{"event":"selfhost_migrations_applied","count":56}` in the logs). Point your +GitHub App's webhook at `https:///v1/github/webhook` (expose port 8787 behind your own TLS). + +To run without Docker: + +```bash +npm ci +node scripts/build-selfhost.mjs +node --import ./scripts/register-selfhost.mjs dist/server.mjs +``` + +--- + +## 2. Create the GitHub App + +Self-host needs its own GitHub App (the hosted gittensory[bot] is separate). Create one with: + +- **Webhook URL** `https:///v1/github/webhook`, and a **webhook secret** (→ `GITHUB_WEBHOOK_SECRET`). +- **Permissions**: Pull requests (read/write), Contents (read; read/write if you want merge), Issues + (read/write), Checks (read), Metadata (read). Commit statuses (read). +- **Events**: Pull request, Pull request review, Push, Issues, Check suite, Check run, Status. +- Generate a **private key** (→ `GITHUB_APP_PRIVATE_KEY`), and note the **App ID** (→ `GITHUB_APP_ID`) and the + app **slug** (→ `GITHUB_APP_SLUG`). Install the app on the repos you want reviewed. + +--- + +## 3. Configuration + +Everything is environment variables — see [`.env.example`](../.env.example) for the annotated list (it holds +**sample placeholders only; never commit a real `.env`** — it is gitignored). The required core secrets: + +| Variable | What it is | +| --- | --- | +| `GITHUB_APP_ID` / `GITHUB_APP_SLUG` | your GitHub App's id + slug | +| `GITHUB_APP_PRIVATE_KEY` | the App's PKCS#8 private key (or mount `GITHUB_APP_PRIVATE_KEY_FILE`) | +| `GITHUB_WEBHOOK_SECRET` | the webhook secret you set on the App | +| `GITTENSOR_REGISTRY_URL` | registry endpoint (or any reachable placeholder if you don't use the registry) | +| `GITTENSORY_API_TOKEN` / `GITTENSORY_MCP_TOKEN` / `INTERNAL_JOB_TOKEN` | bearer tokens — generate your own (`openssl rand -hex 32`) | + +Runtime knobs: `PORT` (default 8787), `DATABASE_PATH` (default `/data/gittensory.sqlite`), `CRON_INTERVAL_MS` +(default 120000 ≈ the hosted every-2-minutes cron). + +**Secrets via files.** Any `FOO_FILE=/run/secrets/foo` is read into `FOO` at startup (Docker/Compose +secrets, multi-line keys) — an explicit `FOO` always wins. + +--- + +## 4. AI provider (optional) + +Without an AI provider the review still runs fully — deterministic signals, the gate, merge/close decisions — +and only the AI **summary** degrades to "unavailable". To enable AI, set `AI_PROVIDER`: + +| `AI_PROVIDER` | Backend | Extra config | +| --- | --- | --- | +| `ollama` / `openai-compatible` / `openai` | any OpenAI-compatible `/chat/completions` endpoint | `AI_BASE_URL`, `AI_API_KEY`, `WORKERS_AI_SUMMARY_MODEL` | +| `claude-code` | your **Claude** subscription via the `claude` CLI (read-only, headless) | `CLAUDE_CODE_OAUTH_TOKEN` (from `claude setup-token`) | +| `codex` | your **Codex** subscription via the `codex` CLI | local `codex` auth | + +The local-AI default is Ollama: uncomment the `ollama` service in `docker-compose.yml`, set +`AI_PROVIDER=ollama` + `AI_BASE_URL=http://ollama:11434/v1`, then `docker compose exec ollama ollama pull +`. + +**Subscription safety.** The CLI providers run as a read-only subprocess with billable API keys +(`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, …) **scrubbed from the child environment** so a misconfigured CLI +can't silently bill the metered API instead of your subscription. Any error, empty output, or Claude-Code +`is_error` envelope makes the call throw, so the review degrades rather than surfacing an error string as the +model's answer. (Codex is gated/unverified — treat it as best-effort.) + +--- + +## 5. Review modes — advisory vs. full maintainer + +Self-host runs the identical engine, so the behavior is configured exactly as on the hosted product: + +- **Advisory (default).** With Contents write withheld (or autonomy off), Gittensory posts its unified review + comment and check, but never merges or closes — a recommendation engine. +- **Full maintainer.** Grant Contents write and enable per-repo autonomy (merge / close / approve) — the bot + acts on its decisions, gated by the same guardrails (protected-path manual-review globs, owner-PR + no-auto-close, mergeability + green-CI before approve). + +Per-PR capabilities (safety scan, CI/full-file grounding, RAG, unified comment, content lane, self-tune, +parity audit) are the `GITTENSORY_REVIEW_*` flags — every flag defaults **off** and is fully inert until +turned on. Per-repo settings (autonomy, required approvals, protected paths) live in `.gittensory.yml` / +repository settings. The authoritative reference for all of these is +[`docs/review-configuration.md`](./review-configuration.md). + +--- + +## 6. Operations + +- **Health.** `GET /health` is binding-free (liveness); the container's `HEALTHCHECK` uses it. +- **Logs** are structured JSON (`selfhost_listening`, `selfhost_migrations_applied`, `selfhost_ai_provider`, + `selfhost_cron_error`, …). Pipe them to your log stack. +- **Data + backup.** Everything is the single SQLite file on the `gittensory-data` volume (WAL mode). Back up + by snapshotting the volume or copying the `.sqlite` file. Migrations are idempotent and re-checked at every + boot. +- **Metrics.** Enable `GITTENSORY_REVIEW_OPS=true` for the read-only gate-block anomaly scan and the + bearer-gated `GET /v1/internal/ops/stats` aggregate. + +--- + +## 7. What is not on self-host + +These are Cloudflare-platform features; they degrade cleanly and the core reviewer is unaffected: + +- **Visual PR capture** (Browser Rendering binding) — off; reviews run text-only. +- **The `/mcp` server** (Durable-Object-backed Agents SDK) — returns `501`. The deterministic API + review + path is unaffected; a native MCP-on-Node port is a follow-up. +- **Distributed rate limiting** (RateLimiter Durable Object) — absent, so the limiter is a no-op. Put your + reverse proxy / WAF in front if you expose the endpoint publicly. +- **Vectorize-backed RAG** and **R2 audit storage** — inert unless you wire equivalent backends. diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts new file mode 100644 index 0000000000..0c525f2008 --- /dev/null +++ b/src/selfhost/ai.ts @@ -0,0 +1,167 @@ +// Self-host AI provider (#979). gittensory calls `env.AI.run(model, { messages, max_tokens, temperature })` +// and reads `{ response }`. On self-host we provide an Ai-shaped adapter selected by AI_PROVIDER: +// • ollama / openai-compatible / openai — any OpenAI-compatible /chat/completions endpoint (BYO key) +// • claude-code / codex — a locally-authenticated CLI SUBSCRIPTION, run as a subprocess +// Absent (no AI_PROVIDER) → env.AI is undefined → gittensory's AI summary degrades to "unavailable" and the +// review proceeds deterministically. Every path returns `{ response: string }` (or throws → the caller +// records an error and degrades — never a silent wrong answer). + +interface AiRunOptions { + messages?: Array<{ role: string; content: string }>; + prompt?: string; + max_tokens?: number; + temperature?: number; +} +export interface SelfHostAi { + run(model: string, options: AiRunOptions): Promise<{ response: string }>; +} + +function toMessages(options: AiRunOptions): Array<{ role: string; content: string }> { + if (Array.isArray(options.messages)) return options.messages; + return [{ role: "user", content: String(options.prompt ?? "") }]; +} + +/** OpenAI-compatible chat endpoint (Ollama's /v1, OpenAI, vLLM, LM Studio, …). */ +export function createOpenAiCompatibleAi(opts: { baseUrl: string; apiKey?: string | undefined; defaultModel?: string | undefined }): SelfHostAi { + const base = opts.baseUrl.replace(/\/+$/, ""); + return { + async run(model, options) { + const res = await fetch(`${base}/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json", ...(opts.apiKey ? { authorization: `Bearer ${opts.apiKey}` } : {}) }, + body: JSON.stringify({ model: model || opts.defaultModel || "llama3.1", messages: toMessages(options), max_tokens: options.max_tokens, temperature: options.temperature }), + signal: AbortSignal.timeout(120_000), + }); + if (!res.ok) throw new Error(`ai_http_${res.status}`); + const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }; + return { response: data.choices?.[0]?.message?.content ?? "" }; + }, + }; +} + +// ── Subscription CLI providers (#979) — locally-authenticated `claude` / `codex` as a subprocess ────────── +// SECURITY: the child env DELETES the billable API keys so a misconfigured CLI cannot silently bill the +// metered API instead of using the subscription OAuth token. The CLI runs read-only / no extra tools. Any +// non-zero exit / empty output / error-envelope THROWS so the caller degrades — never a silent answer. +const BILLABLE_KEY_VARS = ["ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "CODEX_API_KEY", "OPENAI_API_KEY"] as const; + +function scrubBillableKeys(parent: Record): Record { + const child = { ...parent }; + for (const k of BILLABLE_KEY_VARS) delete child[k]; + return child; +} + +/** Pull the assistant's final text out of a CLI's JSON output (Claude Code `{result}` or Codex JSONL). */ +export function extractCliText(stdout: string): string { + const trimmed = stdout.trim(); + if (!trimmed) return ""; + const tryParse = (s: string): string => { + try { + const o = JSON.parse(s) as Record; + const text = o.result ?? o.text ?? o.content ?? o.response; + return typeof text === "string" ? text : ""; + } catch { + return ""; + } + }; + const whole = tryParse(trimmed); + if (whole) return whole; + const lines = trimmed.split(/\r?\n/).filter((l) => l.trim()); + for (let i = lines.length - 1; i >= 0; i -= 1) { + const line = lines[i]; + if (!line) continue; + const t = tryParse(line); + if (t) return t; + } + return ""; +} + +/** Claude Code's `--output-format json` exits 0 even on an API/auth error, returning {is_error:true,result:""}. + * Detect it so the error string is never surfaced as the model's answer. */ +export function claudeErrorStatus(stdout: string): string | null { + try { + const o = JSON.parse(stdout.trim()) as Record; + if (o.is_error === true) return String(o.api_error_status ?? o.subtype ?? "unknown"); + } catch { + /* not a single JSON object — handled by the empty-output guard */ + } + return null; +} + +type SpawnFn = (cmd: string, args: string[], opts: { env: Record; input?: string; timeoutMs: number }) => Promise<{ stdout: string; code: number | null }>; + +async function defaultSpawn(): Promise { + const cp = await import("node:child_process"); + return (cmd, args, o) => + new Promise((resolve, reject) => { + const stdio: ["pipe", "pipe", "pipe"] = ["pipe", "pipe", "pipe"]; + const child = cp.spawn(cmd, args, { env: o.env as NodeJS.ProcessEnv, stdio }); + let stdout = ""; + const timer = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error("subscription_cli_timeout")); + }, o.timeoutMs); + child.stdout?.on("data", (d: Buffer) => (stdout += d.toString("utf8"))); + child.on("error", (e) => { + clearTimeout(timer); + reject(e); + }); + child.on("close", (code) => { + clearTimeout(timer); + resolve({ stdout, code }); + }); + if (o.input != null) { + child.stdin?.write(o.input); + child.stdin?.end(); + } + }); +} + +/** Claude Code subscription (CLAUDE_CODE_OAUTH_TOKEN via `claude setup-token`). Headless, read-only, JSON. */ +export function createClaudeCodeAi(parentEnv: Record, spawnImpl?: SpawnFn): SelfHostAi { + return { + async run(model, options) { + const token = parentEnv.CLAUDE_CODE_OAUTH_TOKEN; + if (!token) throw new Error("claude_code_no_oauth_token"); + const env = scrubBillableKeys(parentEnv); + env.CLAUDE_CODE_OAUTH_TOKEN = token; + const prompt = toMessages(options).map((m) => m.content).join("\n\n"); + const spawn = spawnImpl ?? (await defaultSpawn()); + const { stdout, code } = await spawn("claude", ["--print", "--output-format", "json", "--model", model || "sonnet", "--permission-mode", "plan", "--disallowedTools", "Bash,Edit,Write,WebFetch,WebSearch"], { env, input: prompt, timeoutMs: 120_000 }); + if (code !== 0) throw new Error(`claude_code_exit_${code ?? "null"}`); + const errStatus = claudeErrorStatus(stdout); + if (errStatus) throw new Error(`claude_code_error_${errStatus}`); + const text = extractCliText(stdout); + if (!text) throw new Error("claude_code_empty_output"); + return { response: text }; + }, + }; +} + +/** Codex subscription (`codex exec`, auth from ~/.codex/auth.json). Gated/unverified — fail-safe. */ +export function createCodexAi(parentEnv: Record, spawnImpl?: SpawnFn): SelfHostAi { + return { + async run(model, options) { + const env = scrubBillableKeys(parentEnv); + const prompt = toMessages(options).map((m) => m.content).join("\n\n"); + const spawn = spawnImpl ?? (await defaultSpawn()); + const { stdout, code } = await spawn("codex", ["exec", "--json", "--sandbox", "read-only", "--ask-for-approval", "never", "--model", model || "gpt-5", prompt], { env, timeoutMs: 120_000 }); + if (code !== 0) throw new Error(`codex_exit_${code ?? "null"}`); + const text = extractCliText(stdout); + if (!text) throw new Error("codex_empty_output"); + return { response: text }; + }, + }; +} + +/** Pick the self-host AI provider from env (AI_PROVIDER). Returns undefined when unconfigured. */ +export function createSelfHostAi(env: Record): SelfHostAi | undefined { + const provider = (env.AI_PROVIDER ?? "").trim().toLowerCase(); + if (!provider) return undefined; + if (provider === "ollama" || provider === "openai-compatible" || provider === "openai") { + return createOpenAiCompatibleAi({ baseUrl: env.AI_BASE_URL ?? "http://localhost:11434/v1", apiKey: env.AI_API_KEY, defaultModel: env.WORKERS_AI_SUMMARY_MODEL }); + } + if (provider === "claude-code") return createClaudeCodeAi(env); + if (provider === "codex") return createCodexAi(env); + return undefined; +} diff --git a/src/server.ts b/src/server.ts index 0befe5a365..3368c163b6 100644 --- a/src/server.ts +++ b/src/server.ts @@ -8,6 +8,7 @@ import { DatabaseSync } from "node:sqlite"; import { serve } from "@hono/node-server"; import worker from "./index"; import { processJob } from "./queue/processors"; +import { createSelfHostAi } from "./selfhost/ai"; import { createD1Adapter, nodeSqliteDriver } from "./selfhost/d1-adapter"; import { runSelfHostMigrations } from "./selfhost/migrate"; import { createInProcessQueue } from "./selfhost/queue"; @@ -42,7 +43,11 @@ async function main(): Promise { const queue = createInProcessQueue(async (message: JobMessage) => { await processJob(env, message); }); - env = { ...process.env, DB: db, JOBS: queue.binding, AI: undefined } as unknown as Env; + // AI: the OpenAI-compatible / subscription adapter selected by AI_PROVIDER (undefined when unconfigured → + // gittensory's AI summary degrades to "unavailable" and the review proceeds deterministically). + const ai = createSelfHostAi(process.env); + if (ai) console.log(JSON.stringify({ event: "selfhost_ai_provider", provider: process.env.AI_PROVIDER })); + env = { ...process.env, DB: db, JOBS: queue.binding, AI: ai } as unknown as Env; const ctx = { waitUntil: (p: Promise) => void Promise.resolve(p).catch(() => undefined), diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts new file mode 100644 index 0000000000..6aa81b0cd4 --- /dev/null +++ b/test/unit/selfhost-ai.test.ts @@ -0,0 +1,66 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { claudeErrorStatus, createClaudeCodeAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText } from "../../src/selfhost/ai"; + +afterEach(() => vi.unstubAllGlobals()); + +type SpawnResult = { stdout: string; code: number | null }; +type StubSpawn = (cmd: string, args: string[], opts: { env: Record; input?: string; timeoutMs: number }) => Promise; + +describe("createOpenAiCompatibleAi (#979)", () => { + it("POSTs to /chat/completions and returns { response }", async () => { + const calls: Array<{ url: string; body: { model: string } }> = []; + vi.stubGlobal("fetch", vi.fn(async (url: string, init: { body: string }) => { + calls.push({ url, body: JSON.parse(init.body) }); + return new Response(JSON.stringify({ choices: [{ message: { content: "hi there" } }] }), { status: 200 }); + })); + const ai = createOpenAiCompatibleAi({ baseUrl: "http://ollama:11434/v1/", apiKey: "k" }); + const out = await ai.run("llama3.1", { messages: [{ role: "user", content: "x" }], max_tokens: 100 }); + expect(out.response).toBe("hi there"); + const first = calls[0]; + expect(first?.url).toBe("http://ollama:11434/v1/chat/completions"); // trailing slash trimmed + expect(first?.body.model).toBe("llama3.1"); + }); + + it("throws on a non-OK response so the caller degrades", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response("err", { status: 500 }))); + await expect(createOpenAiCompatibleAi({ baseUrl: "http://x/v1" }).run("m", { prompt: "p" })).rejects.toThrow(/ai_http_500/); + }); +}); + +describe("createSelfHostAi — provider selection", () => { + it("is undefined when AI_PROVIDER is unset", () => { + expect(createSelfHostAi({})).toBeUndefined(); + }); + it("maps ollama/openai-compatible/claude-code/codex to adapters", () => { + expect(typeof createSelfHostAi({ AI_PROVIDER: "ollama", AI_BASE_URL: "http://o/v1" })?.run).toBe("function"); + expect(typeof createSelfHostAi({ AI_PROVIDER: "claude-code" })?.run).toBe("function"); + expect(typeof createSelfHostAi({ AI_PROVIDER: "codex" })?.run).toBe("function"); + expect(createSelfHostAi({ AI_PROVIDER: "nonsense" })).toBeUndefined(); + }); +}); + +describe("subscription CLI helpers + fail-safe", () => { + it("extractCliText pulls the result/text field", () => { + expect(extractCliText(JSON.stringify({ type: "result", result: "ok" }))).toBe("ok"); + expect(extractCliText("")).toBe(""); + }); + it("claudeErrorStatus catches the is_error envelope", () => { + expect(claudeErrorStatus(JSON.stringify({ is_error: true, api_error_status: 401 }))).toBe("401"); + expect(claudeErrorStatus(JSON.stringify({ is_error: false, result: "ok" }))).toBeNull(); + }); + it("Claude Code fails SAFE on an is_error envelope (exits 0) instead of surfacing the error text", async () => { + const stub: StubSpawn = async () => ({ stdout: JSON.stringify({ is_error: true, api_error_status: 401, result: "Failed to authenticate" }), code: 0 }); + await expect(createClaudeCodeAi({ CLAUDE_CODE_OAUTH_TOKEN: "t" }, stub).run("m", { prompt: "x" })).rejects.toThrow(/claude_code_error_401/); + }); + it("Claude Code returns the model text on success and scrubs billable keys", async () => { + let capturedEnv: Record = {}; + const stub: StubSpawn = async (_c, _a, o) => { + capturedEnv = o.env; + return { stdout: JSON.stringify({ type: "result", result: "review text" }), code: 0 }; + }; + const out = await createClaudeCodeAi({ CLAUDE_CODE_OAUTH_TOKEN: "t", ANTHROPIC_API_KEY: "sk-bill" }, stub).run("sonnet", { prompt: "x" }); + expect(out.response).toBe("review text"); + expect(capturedEnv.ANTHROPIC_API_KEY).toBeUndefined(); // scrubbed + expect(capturedEnv.CLAUDE_CODE_OAUTH_TOKEN).toBe("t"); + }); +}); From ed948646d563eebb7e748b321791efab93061f95 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 23 Jun 2026 21:11:49 -0700 Subject: [PATCH 05/25] =?UTF-8?q?feat(selfhost):=20Tier=200=20reliability?= =?UTF-8?q?=20=E2=80=94=20durable=20queue,=20model-id=20fix,=20/ready+/met?= =?UTF-8?q?rics,=20graceful=20shutdown,=20CI=20(#980/#982)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makes the self-host runtime production-trustworthy: - AI model-id fix (#979): the core passes a Workers-AI id (@cf/meta/...) that Ollama/claude/codex can't use. Adapters now resolve the operator's AI_MODEL (then a non-Workers core model, then a provider default) and never leak the @cf id. New resolveModel() + AI_MODEL env. - Durable queue: jobs persist in SQLite (_selfhost_jobs) instead of in memory, so a restart/crash RE-CLAIMS in-flight work. Exponential-backoff retries, dead-letter after maxRetries, crash-recovery of 'processing' rows. Same Queue binding surface — app code unchanged. - Operational endpoints: /ready (503 until DB + migrations ready) and /metrics (Prometheus: queue depth/dead, jobs enqueued/processed/failed/dead, uptime, http requests). New metrics registry + health module. - Graceful shutdown: SIGTERM/SIGINT → stop accepting, drain in-flight job, WAL checkpoint, close DB. - CI: .github/workflows/selfhost.yml builds the bundle + Docker image, boots the container, and smoke-tests /health, /ready, /metrics + migrations on Node 24 — the integration coverage units can't give. - Tests: +16 (durable queue persistence/retry/dead/recovery/start-stop, metrics, readiness, migrations, model resolution, Codex, and the REAL claude subprocess via defaultSpawn against a fake CLI). 29 self-host tests, typecheck clean. codecov: ignore the process entry + build-time stubs (covered by the boot smoke test). --- .env.example | 5 +- .github/workflows/selfhost.yml | 65 ++++++++++ codecov.yml | 5 + docs/self-hosting.md | 31 +++-- src/selfhost/ai.ts | 25 +++- src/selfhost/health.ts | 26 ++++ src/selfhost/metrics.ts | 48 ++++++++ src/selfhost/sqlite-queue.ts | 156 ++++++++++++++++++++++++ src/server.ts | 67 +++++++--- test/unit/selfhost-ai.test.ts | 41 ++++++- test/unit/selfhost-health.test.ts | 18 +++ test/unit/selfhost-metrics.test.ts | 33 +++++ test/unit/selfhost-migrate.test.ts | 22 ++++ test/unit/selfhost-sqlite-queue.test.ts | 73 +++++++++++ 14 files changed, 585 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/selfhost.yml create mode 100644 src/selfhost/health.ts create mode 100644 src/selfhost/metrics.ts create mode 100644 src/selfhost/sqlite-queue.ts create mode 100644 test/unit/selfhost-health.test.ts create mode 100644 test/unit/selfhost-metrics.test.ts create mode 100644 test/unit/selfhost-migrate.test.ts create mode 100644 test/unit/selfhost-sqlite-queue.test.ts diff --git a/.env.example b/.env.example index 1567a40f51..966b70b937 100644 --- a/.env.example +++ b/.env.example @@ -115,4 +115,7 @@ GITTENSORY_REVIEW_DRAFT=false # AI_PROVIDER=ollama # ollama | openai-compatible | claude-code | codex (see #979) # AI_BASE_URL=http://ollama:11434/v1 # an OpenAI-compatible endpoint (the Ollama default) # AI_API_KEY= # if your endpoint requires a key -# WORKERS_AI_SUMMARY_MODEL=llama3.1 # the chat model to use +# AI_MODEL=llama3.1 # the model for your provider (e.g. llama3.1 for Ollama, sonnet +# # for claude-code, gpt-5 for codex). REQUIRED for non-Ollama: +# # without it the adapter falls back to a provider default, never +# # the Cloudflare Workers-AI id the core would otherwise pass. diff --git a/.github/workflows/selfhost.yml b/.github/workflows/selfhost.yml new file mode 100644 index 0000000000..b3fa41ad76 --- /dev/null +++ b/.github/workflows/selfhost.yml @@ -0,0 +1,65 @@ +# Self-host stack CI (#980/#982). Builds the Node bundle + the Docker image, boots the container, and +# smoke-tests the operational endpoints — the integration coverage the unit tests can't give. Runs on +# Node 24 because node:sqlite (the SQLite backing store) is only stable there. +name: self-host + +on: + push: + branches: [main] + paths: + - "src/selfhost/**" + - "src/server.ts" + - "scripts/build-selfhost.mjs" + - "scripts/register-selfhost.mjs" + - "Dockerfile" + - "docker-compose.yml" + - "migrations/**" + - "test/unit/selfhost-*" + - ".github/workflows/selfhost.yml" + pull_request: + paths: + - "src/selfhost/**" + - "src/server.ts" + - "scripts/build-selfhost.mjs" + - "scripts/register-selfhost.mjs" + - "Dockerfile" + - "docker-compose.yml" + - "migrations/**" + - "test/unit/selfhost-*" + - ".github/workflows/selfhost.yml" + +jobs: + build-boot: + name: build + boot smoke test + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "24" + - name: Install deps + run: npm ci --ignore-scripts + - name: Self-host unit tests + run: npx vitest run test/unit/selfhost-*.test.ts + - name: Typecheck + run: npx tsc --noEmit + - name: Build the self-host bundle + run: node scripts/build-selfhost.mjs + - name: Build the Docker image + run: docker build -t gittensory:selfhost-ci . + - name: Boot the container + smoke-test /health, /ready, /metrics, migrations + run: | + docker run -d --name gt -p 8787:8787 gittensory:selfhost-ci + ok=0 + for i in $(seq 1 30); do + if curl -sf http://127.0.0.1:8787/health >/dev/null; then ok=1; break; fi + sleep 2 + done + if [ "$ok" != "1" ]; then echo "::error::container did not become healthy"; docker logs gt; exit 1; fi + curl -sf http://127.0.0.1:8787/health | grep -q '"status":"ok"' + curl -sf http://127.0.0.1:8787/ready | grep -q '"ok":true' + curl -sf http://127.0.0.1:8787/metrics | grep -q 'gittensory_uptime_seconds' + docker logs gt 2>&1 | grep -q 'selfhost_migrations_applied' + echo "self-host smoke test passed" + docker rm -f gt diff --git a/codecov.yml b/codecov.yml index a70fed5eda..3b4fbe6ab9 100644 --- a/codecov.yml +++ b/codecov.yml @@ -33,3 +33,8 @@ ignore: - "apps/**" - "test/**" - "scripts/**" + # Self-host process entry + build-time stubs: exercised by the Docker build+boot smoke test + # (.github/workflows/selfhost.yml), not unit-coverable without booting a server/subprocess. + - "src/server.ts" + - "src/selfhost/cf-workers-shim.ts" + - "src/selfhost/stubs/**" diff --git a/docs/self-hosting.md b/docs/self-hosting.md index f92bcc9bb1..81643b8538 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -76,9 +76,15 @@ and only the AI **summary** degrades to "unavailable". To enable AI, set `AI_PRO | `AI_PROVIDER` | Backend | Extra config | | --- | --- | --- | -| `ollama` / `openai-compatible` / `openai` | any OpenAI-compatible `/chat/completions` endpoint | `AI_BASE_URL`, `AI_API_KEY`, `WORKERS_AI_SUMMARY_MODEL` | -| `claude-code` | your **Claude** subscription via the `claude` CLI (read-only, headless) | `CLAUDE_CODE_OAUTH_TOKEN` (from `claude setup-token`) | -| `codex` | your **Codex** subscription via the `codex` CLI | local `codex` auth | +| `ollama` / `openai-compatible` / `openai` | any OpenAI-compatible `/chat/completions` endpoint | `AI_BASE_URL`, `AI_API_KEY`, `AI_MODEL` | +| `claude-code` | your **Claude** subscription via the `claude` CLI (read-only, headless) | `CLAUDE_CODE_OAUTH_TOKEN` (from `claude setup-token`), `AI_MODEL` (e.g. `sonnet`) | +| `codex` | your **Codex** subscription via the `codex` CLI | local `codex` auth, `AI_MODEL` (e.g. `gpt-5`) | + +> **Set `AI_MODEL`.** The core would otherwise hand the adapter a Cloudflare Workers-AI model id +> (`@cf/meta/...`) that Ollama / `claude` / `codex` can't use. The adapter ignores that id in favour of +> `AI_MODEL` (falling back to a provider default), so always set `AI_MODEL` to a real model for your provider. +> The `claude`/`codex` CLIs must be installed and authenticated in the runtime (a CLI-bearing image variant +> is a follow-up); without `AI_MODEL` + a working CLI, the call throws and the review degrades. The local-AI default is Ollama: uncomment the `ollama` service in `docker-compose.yml`, set `AI_PROVIDER=ollama` + `AI_BASE_URL=http://ollama:11434/v1`, then `docker compose exec ollama ollama pull @@ -112,13 +118,22 @@ repository settings. The authoritative reference for all of these is ## 6. Operations -- **Health.** `GET /health` is binding-free (liveness); the container's `HEALTHCHECK` uses it. +- **Endpoints.** + - `GET /health` — binding-free liveness (the container `HEALTHCHECK` uses it). + - `GET /ready` — readiness: returns `503` until the DB answers **and** migrations are applied + (`{"ok":true,"checks":{"db":true,"migrations":true}}`). Use it as your orchestrator's readiness probe. + - `GET /metrics` — Prometheus text: `gittensory_queue_pending` / `_dead`, `gittensory_jobs_*_total` + (enqueued/processed/failed/dead), `gittensory_uptime_seconds`, `gittensory_http_requests_total`. +- **Durable queue.** Jobs are persisted in SQLite (`_selfhost_jobs`), not held in memory — a restart or crash + **re-claims** in-flight work instead of losing it. Failures retry with exponential backoff and dead-letter + after `maxRetries` (visible via `gittensory_queue_dead`). +- **Graceful shutdown.** On `SIGTERM`/`SIGINT` the server stops accepting requests, lets the queue finish its + in-flight job, checkpoints the WAL, and closes the DB before exiting. - **Logs** are structured JSON (`selfhost_listening`, `selfhost_migrations_applied`, `selfhost_ai_provider`, - `selfhost_cron_error`, …). Pipe them to your log stack. + `selfhost_queue_recovered`, `selfhost_job_dead`, `selfhost_cron_error`, `selfhost_shutdown`, …). - **Data + backup.** Everything is the single SQLite file on the `gittensory-data` volume (WAL mode). Back up - by snapshotting the volume or copying the `.sqlite` file. Migrations are idempotent and re-checked at every - boot. -- **Metrics.** Enable `GITTENSORY_REVIEW_OPS=true` for the read-only gate-block anomaly scan and the + by snapshotting the volume or copying the `.sqlite` file. Migrations are idempotent and re-checked at boot. +- **App-level metrics.** Enable `GITTENSORY_REVIEW_OPS=true` for the read-only gate-block anomaly scan and the bearer-gated `GET /v1/internal/ops/stats` aggregate. --- diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index 0c525f2008..920937d9b7 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -21,15 +21,28 @@ function toMessages(options: AiRunOptions): Array<{ role: string; content: strin return [{ role: "user", content: String(options.prompt ?? "") }]; } +/** The core passes a Workers-AI model id (e.g. "@cf/meta/llama-3.1-8b-instruct-fp8-fast") that is meaningless + * off-Workers — handing it to Ollama or `claude --model` fails. Prefer the operator-configured model + * (AI_MODEL / WORKERS_AI_SUMMARY_MODEL), then any non-Workers model the core passed, then a provider default. */ +export function resolveModel(configured: string | undefined, passed: string, providerDefault: string): string { + if (configured && configured.trim()) return configured.trim(); + if (passed && !passed.startsWith("@cf/")) return passed; + return providerDefault; +} + +function configuredModel(env: Record): string | undefined { + return env.AI_MODEL ?? env.WORKERS_AI_SUMMARY_MODEL; +} + /** OpenAI-compatible chat endpoint (Ollama's /v1, OpenAI, vLLM, LM Studio, …). */ -export function createOpenAiCompatibleAi(opts: { baseUrl: string; apiKey?: string | undefined; defaultModel?: string | undefined }): SelfHostAi { +export function createOpenAiCompatibleAi(opts: { baseUrl: string; apiKey?: string | undefined; model?: string | undefined }): SelfHostAi { const base = opts.baseUrl.replace(/\/+$/, ""); return { async run(model, options) { const res = await fetch(`${base}/chat/completions`, { method: "POST", headers: { "content-type": "application/json", ...(opts.apiKey ? { authorization: `Bearer ${opts.apiKey}` } : {}) }, - body: JSON.stringify({ model: model || opts.defaultModel || "llama3.1", messages: toMessages(options), max_tokens: options.max_tokens, temperature: options.temperature }), + body: JSON.stringify({ model: resolveModel(opts.model, model, "llama3.1"), messages: toMessages(options), max_tokens: options.max_tokens, temperature: options.temperature }), signal: AbortSignal.timeout(120_000), }); if (!res.ok) throw new Error(`ai_http_${res.status}`); @@ -127,7 +140,8 @@ export function createClaudeCodeAi(parentEnv: Record env.CLAUDE_CODE_OAUTH_TOKEN = token; const prompt = toMessages(options).map((m) => m.content).join("\n\n"); const spawn = spawnImpl ?? (await defaultSpawn()); - const { stdout, code } = await spawn("claude", ["--print", "--output-format", "json", "--model", model || "sonnet", "--permission-mode", "plan", "--disallowedTools", "Bash,Edit,Write,WebFetch,WebSearch"], { env, input: prompt, timeoutMs: 120_000 }); + const claudeModel = resolveModel(configuredModel(parentEnv), model, "sonnet"); + const { stdout, code } = await spawn("claude", ["--print", "--output-format", "json", "--model", claudeModel, "--permission-mode", "plan", "--disallowedTools", "Bash,Edit,Write,WebFetch,WebSearch"], { env, input: prompt, timeoutMs: 120_000 }); if (code !== 0) throw new Error(`claude_code_exit_${code ?? "null"}`); const errStatus = claudeErrorStatus(stdout); if (errStatus) throw new Error(`claude_code_error_${errStatus}`); @@ -145,7 +159,8 @@ export function createCodexAi(parentEnv: Record, spa const env = scrubBillableKeys(parentEnv); const prompt = toMessages(options).map((m) => m.content).join("\n\n"); const spawn = spawnImpl ?? (await defaultSpawn()); - const { stdout, code } = await spawn("codex", ["exec", "--json", "--sandbox", "read-only", "--ask-for-approval", "never", "--model", model || "gpt-5", prompt], { env, timeoutMs: 120_000 }); + const codexModel = resolveModel(configuredModel(parentEnv), model, "gpt-5"); + const { stdout, code } = await spawn("codex", ["exec", "--json", "--sandbox", "read-only", "--ask-for-approval", "never", "--model", codexModel, prompt], { env, timeoutMs: 120_000 }); if (code !== 0) throw new Error(`codex_exit_${code ?? "null"}`); const text = extractCliText(stdout); if (!text) throw new Error("codex_empty_output"); @@ -159,7 +174,7 @@ export function createSelfHostAi(env: Record): SelfH const provider = (env.AI_PROVIDER ?? "").trim().toLowerCase(); if (!provider) return undefined; if (provider === "ollama" || provider === "openai-compatible" || provider === "openai") { - return createOpenAiCompatibleAi({ baseUrl: env.AI_BASE_URL ?? "http://localhost:11434/v1", apiKey: env.AI_API_KEY, defaultModel: env.WORKERS_AI_SUMMARY_MODEL }); + return createOpenAiCompatibleAi({ baseUrl: env.AI_BASE_URL ?? "http://localhost:11434/v1", apiKey: env.AI_API_KEY, model: configuredModel(env) }); } if (provider === "claude-code") return createClaudeCodeAi(env); if (provider === "codex") return createCodexAi(env); diff --git a/src/selfhost/health.ts b/src/selfhost/health.ts new file mode 100644 index 0000000000..badde913c4 --- /dev/null +++ b/src/selfhost/health.ts @@ -0,0 +1,26 @@ +// Self-host liveness/readiness probes (#982). Liveness is binding-free (the process is up); readiness asserts +// the things a request actually depends on — the DB answers and the schema migrations have been applied. +import type { SqliteDriver } from "./d1-adapter"; + +export interface Readiness { + ok: boolean; + checks: Record; +} + +/** Readiness: the DB answers a trivial query and the migrations table shows applied rows. */ +export function readiness(driver: SqliteDriver): Readiness { + let db = false; + let migrations = false; + try { + driver.query("SELECT 1", []); + db = true; + } catch { + /* db down */ + } + try { + migrations = Number((driver.query("SELECT COUNT(*) AS c FROM _selfhost_migrations", []).rows[0] as { c: number }).c) > 0; + } catch { + /* migrations table missing */ + } + return { ok: db && migrations, checks: { db, migrations } }; +} diff --git a/src/selfhost/metrics.ts b/src/selfhost/metrics.ts new file mode 100644 index 0000000000..053daa5dab --- /dev/null +++ b/src/selfhost/metrics.ts @@ -0,0 +1,48 @@ +// Minimal Prometheus text-format metrics for the self-host runtime (#982 observability). A tiny in-process +// registry — counters (monotonic, incremented at the call site) and gauges (sampled at scrape time via a +// callback, e.g. live queue depth). Rendered at GET /metrics. No deps, no cardinality explosion: callers use +// a small fixed label set. +type Labels = Record; + +const counters = new Map(); +const gauges = new Map number>(); + +function seriesKey(name: string, labels?: Labels): string { + if (!labels || Object.keys(labels).length === 0) return name; + const inner = Object.entries(labels) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([k, v]) => `${k}="${String(v).replace(/"/g, '\\"')}"`) + .join(","); + return `${name}{${inner}}`; +} + +/** Increment a monotonic counter (created on first use). */ +export function incr(name: string, labels?: Labels, by = 1): void { + const k = seriesKey(name, labels); + counters.set(k, (counters.get(k) ?? 0) + by); +} + +/** Register a gauge sampled at scrape time. Re-registering replaces the sampler. */ +export function gauge(name: string, sample: () => number): void { + gauges.set(name, sample); +} + +/** Render the registry in Prometheus text exposition format. */ +export function renderMetrics(): string { + const lines: string[] = []; + for (const [k, v] of counters) lines.push(`${k} ${v}`); + for (const [name, sample] of gauges) { + try { + lines.push(`${name} ${sample()}`); + } catch { + /* a failing sampler must not break the scrape */ + } + } + return `${lines.join("\n")}\n`; +} + +/** Test-only: clear all series. */ +export function resetMetrics(): void { + counters.clear(); + gauges.clear(); +} diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts new file mode 100644 index 0000000000..b47f536928 --- /dev/null +++ b/src/selfhost/sqlite-queue.ts @@ -0,0 +1,156 @@ +// Durable, SQLite-backed job queue for the self-host runtime (#980 reliability). Unlike the in-process FIFO, +// jobs are PERSISTED — a restart (or crash) re-claims anything left in flight instead of losing it. It still +// presents the Cloudflare `Queue` binding surface (send / sendBatch) so the app code is unchanged; only the +// backing store differs. Single-process model: node:sqlite is synchronous + serial, so claim (SELECT→UPDATE) +// is atomic with no row-lock dance. +import type { SqliteDriver } from "./d1-adapter"; +import { incr } from "./metrics"; +import type { JobMessage } from "../types"; + +const TABLE = "_selfhost_jobs"; +const DDL = ` +CREATE TABLE IF NOT EXISTS ${TABLE} ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + payload TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + attempts INTEGER NOT NULL DEFAULT 0, + run_after INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + last_error TEXT +); +CREATE INDEX IF NOT EXISTS ${TABLE}_claim ON ${TABLE}(status, run_after);`; + +export interface DurableQueue { + binding: Queue; + start(): void; + stop(): Promise; + drain(): Promise; + size(): number; + deadCount(): number; +} + +interface JobRow { + id: number; + payload: string; + attempts: number; +} + +export interface SqliteQueueOptions { + maxRetries?: number; + pollIntervalMs?: number; + backoffMs?: (attempt: number) => number; +} + +export function createSqliteQueue(driver: SqliteDriver, consume: (message: JobMessage) => Promise, opts: SqliteQueueOptions = {}): DurableQueue { + const maxRetries = opts.maxRetries ?? 5; + const pollIntervalMs = opts.pollIntervalMs ?? 1000; + const backoff = opts.backoffMs ?? ((attempt: number) => Math.min(60_000, 1000 * 2 ** attempt)); + + driver.exec(DDL); + // Recover jobs a crashed previous run left mid-flight → make them claimable again. + const recovered = driver.query(`UPDATE ${TABLE} SET status='pending' WHERE status='processing'`, []).changes; + if (recovered) console.log(JSON.stringify({ event: "selfhost_queue_recovered", count: recovered })); + + let running = false; + let pumping = false; + let timer: ReturnType | null = null; + + function enqueue(message: JobMessage, delaySeconds: number): void { + const now = Date.now(); + driver.query(`INSERT INTO ${TABLE} (payload, status, attempts, run_after, created_at) VALUES (?, 'pending', 0, ?, ?)`, [JSON.stringify(message), now + delaySeconds * 1000, now]); + incr("gittensory_jobs_enqueued_total"); + void pump(); + } + + function claimNext(): JobRow | null { + const { rows } = driver.query(`SELECT id, payload, attempts FROM ${TABLE} WHERE status='pending' AND run_after<=? ORDER BY id LIMIT 1`, [Date.now()]); + const row = rows[0] as JobRow | undefined; + if (!row) return null; + const { changes } = driver.query(`UPDATE ${TABLE} SET status='processing' WHERE id=? AND status='pending'`, [row.id]); + return changes ? row : null; + } + + async function processOne(): Promise { + const job = claimNext(); + if (!job) return false; + let message: JobMessage; + try { + message = JSON.parse(job.payload) as JobMessage; + } catch { + driver.query(`UPDATE ${TABLE} SET status='dead', last_error='unparseable payload' WHERE id=?`, [job.id]); + incr("gittensory_jobs_dead_total"); + return true; + } + try { + await consume(message); + driver.query(`DELETE FROM ${TABLE} WHERE id=?`, [job.id]); + incr("gittensory_jobs_processed_total"); + } catch (error) { + const attempts = job.attempts + 1; + const errMsg = error instanceof Error ? error.message : "unknown error"; + incr("gittensory_jobs_failed_total"); + if (attempts >= maxRetries) { + driver.query(`UPDATE ${TABLE} SET status='dead', attempts=?, last_error=? WHERE id=?`, [attempts, errMsg, job.id]); + incr("gittensory_jobs_dead_total"); + console.error(JSON.stringify({ level: "error", event: "selfhost_job_dead", id: job.id, attempts, error: errMsg })); + } else { + driver.query(`UPDATE ${TABLE} SET status='pending', attempts=?, run_after=?, last_error=? WHERE id=?`, [attempts, Date.now() + backoff(attempts), errMsg, job.id]); + } + } + return true; + } + + // Drains every job that is currently DUE. A retry is rescheduled into the future (run_after > now) so it is + // not re-claimed here — the next poll tick picks it up — which also bounds this loop. + async function pump(): Promise { + if (pumping) return; + pumping = true; + try { + while (await processOne()) { + /* keep draining due jobs */ + } + } finally { + pumping = false; + } + } + + const binding = { + async send(message: JobMessage, options?: { delaySeconds?: number }): Promise { + enqueue(message, options?.delaySeconds ?? 0); + }, + async sendBatch(messages: Iterable<{ body: JobMessage; delaySeconds?: number }>): Promise { + for (const m of messages) enqueue(m.body, m.delaySeconds ?? 0); + }, + } as unknown as Queue; + + return { + binding, + start() { + if (running) return; + running = true; + const tick = (): void => { + if (!running) return; + void pump().finally(() => { + if (running) timer = setTimeout(tick, pollIntervalMs); + }); + }; + tick(); + }, + async stop() { + running = false; + if (timer) clearTimeout(timer); + while (pumping) await new Promise((r) => setTimeout(r, 10)); // let an in-flight pump finish + }, + async drain() { + // send() fire-and-forgets a pump; wait for any in-flight pump to settle, then drain to completion. + while (pumping) await new Promise((r) => setTimeout(r, 5)); + await pump(); + }, + size() { + return Number((driver.query(`SELECT COUNT(*) AS c FROM ${TABLE} WHERE status IN ('pending','processing')`, []).rows[0] as { c: number }).c); + }, + deadCount() { + return Number((driver.query(`SELECT COUNT(*) AS c FROM ${TABLE} WHERE status='dead'`, []).rows[0] as { c: number }).c); + }, + }; +} diff --git a/src/server.ts b/src/server.ts index 3368c163b6..144ee70a69 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,8 +1,9 @@ // Self-host Node entry (#980). Runs gittensory's SAME Worker handlers on Node: builds an `Env` where the -// Cloudflare bindings are self-host adapters (D1→node:sqlite, Queue→in-process), serves the Hono app via -// @hono/node-server, drives the in-process queue with the same processJob, and ticks the same scheduled -// handler on a timer. The Cloudflare Worker (src/index.ts) is untouched — this is a parallel entry the -// self-host esbuild build bundles (aliasing `cloudflare:workers` to the shim). +// Cloudflare bindings are self-host adapters (D1→node:sqlite, Queue→a durable SQLite-backed queue), serves +// the Hono app via @hono/node-server, drains the queue with the same processJob, and ticks the same scheduled +// handler on a timer. Adds operational endpoints (/health, /ready, /metrics) and graceful shutdown. The +// Cloudflare Worker (src/index.ts) is untouched — this is a parallel entry the self-host esbuild build bundles +// (aliasing `cloudflare:workers` to the shim). import { readFileSync } from "node:fs"; import { DatabaseSync } from "node:sqlite"; import { serve } from "@hono/node-server"; @@ -10,8 +11,10 @@ import worker from "./index"; import { processJob } from "./queue/processors"; import { createSelfHostAi } from "./selfhost/ai"; import { createD1Adapter, nodeSqliteDriver } from "./selfhost/d1-adapter"; +import { readiness } from "./selfhost/health"; +import { gauge, incr, renderMetrics } from "./selfhost/metrics"; import { runSelfHostMigrations } from "./selfhost/migrate"; -import { createInProcessQueue } from "./selfhost/queue"; +import { createSqliteQueue } from "./selfhost/sqlite-queue"; import type { JobMessage } from "./types"; /** Resolve `_FILE` env vars (Docker secrets / multi-line keys) into `` at startup. */ @@ -30,38 +33,50 @@ function loadFileSecrets(): void { async function main(): Promise { loadFileSecrets(); + const startedAt = Date.now(); const sqlite = new DatabaseSync(process.env.DATABASE_PATH ?? "/data/gittensory.sqlite"); - sqlite.exec("PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON;"); - const db = createD1Adapter(nodeSqliteDriver(sqlite as never)); + sqlite.exec("PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 5000;"); + const driver = nodeSqliteDriver(sqlite as never); + const db = createD1Adapter(driver); const applied = await runSelfHostMigrations(db, process.env.MIGRATIONS_DIR ?? "migrations"); console.log(JSON.stringify({ event: "selfhost_migrations_applied", count: applied })); - // The queue consumer captures `env`, assigned just below — the first send only happens once an HTTP/cron - // event arrives, by which point env is set. + // Durable queue — jobs persist in SQLite, so a restart re-claims in-flight work. The consumer captures + // `env`, assigned just below (the first job only runs once an HTTP/cron event arrives, by which point env is set). let env: Env; - const queue = createInProcessQueue(async (message: JobMessage) => { + const queue = createSqliteQueue(driver, async (message: JobMessage) => { await processJob(env, message); }); + // AI: the OpenAI-compatible / subscription adapter selected by AI_PROVIDER (undefined when unconfigured → // gittensory's AI summary degrades to "unavailable" and the review proceeds deterministically). const ai = createSelfHostAi(process.env); if (ai) console.log(JSON.stringify({ event: "selfhost_ai_provider", provider: process.env.AI_PROVIDER })); env = { ...process.env, DB: db, JOBS: queue.binding, AI: ai } as unknown as Env; + gauge("gittensory_queue_pending", () => queue.size()); + gauge("gittensory_queue_dead", () => queue.deadCount()); + gauge("gittensory_uptime_seconds", () => Math.floor((Date.now() - startedAt) / 1000)); + const ctx = { waitUntil: (p: Promise) => void Promise.resolve(p).catch(() => undefined), passThroughOnException: () => undefined, } as unknown as ExecutionContext; const port = Number(process.env.PORT ?? 8787); - serve( + const server = serve( { fetch: (request: Request) => { - // A binding-free liveness probe (the Hono app also exempts /health from auth + rate-limit). - if (new URL(request.url).pathname === "/health") { - return new Response(JSON.stringify({ status: "ok" }), { headers: { "content-type": "application/json" } }); + const path = new URL(request.url).pathname; + // Binding-free liveness (the Hono app also exempts /health from auth + rate-limit). + if (path === "/health") return new Response(JSON.stringify({ status: "ok" }), { headers: { "content-type": "application/json" } }); + if (path === "/ready") { + const r = readiness(driver); + return new Response(JSON.stringify(r), { status: r.ok ? 200 : 503, headers: { "content-type": "application/json" } }); } + if (path === "/metrics") return new Response(renderMetrics(), { headers: { "content-type": "text/plain; version=0.0.4" } }); + incr("gittensory_http_requests_total"); return worker.fetch(request, env, ctx); }, port, @@ -69,14 +84,36 @@ async function main(): Promise { () => console.log(JSON.stringify({ event: "selfhost_listening", port })), ); + queue.start(); + // Cron — gittensory ticks ~every 2 minutes; drive the SAME scheduled handler. const intervalMs = Number(process.env.CRON_INTERVAL_MS ?? 120_000); - setInterval(() => { + const cron = setInterval(() => { const controller = { scheduledTime: Date.now(), cron: "*/2 * * * *", noRetry: () => undefined } as unknown as ScheduledController; Promise.resolve(worker.scheduled(controller, env, ctx)).catch((error) => console.error(JSON.stringify({ level: "error", event: "selfhost_cron_error", error: error instanceof Error ? error.message : "unknown error" })), ); }, intervalMs); + + // Graceful shutdown: stop accepting HTTP, let the queue finish its in-flight job, checkpoint WAL, close DB. + let shuttingDown = false; + const shutdown = async (signal: string): Promise => { + if (shuttingDown) return; + shuttingDown = true; + console.log(JSON.stringify({ event: "selfhost_shutdown", signal })); + clearInterval(cron); + server.close(); + await queue.stop(); + try { + sqlite.exec("PRAGMA wal_checkpoint(TRUNCATE);"); + sqlite.close(); + } catch { + /* best-effort */ + } + process.exit(0); + }; + process.on("SIGTERM", () => void shutdown("SIGTERM")); + process.on("SIGINT", () => void shutdown("SIGINT")); } main().catch((error) => { diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index 6aa81b0cd4..4eb6e7c9ad 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -1,5 +1,21 @@ +import { chmodSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { claudeErrorStatus, createClaudeCodeAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText } from "../../src/selfhost/ai"; +import { claudeErrorStatus, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, resolveModel } from "../../src/selfhost/ai"; + +describe("resolveModel (#979 — never leak the Workers-AI default to a self-host backend)", () => { + const WORKERS_DEFAULT = "@cf/meta/llama-3.1-8b-instruct-fp8-fast"; + it("operator-configured model wins over the core's Workers-AI id", () => { + expect(resolveModel("llama3.1", WORKERS_DEFAULT, "x")).toBe("llama3.1"); + }); + it("strips the Workers-AI id and falls back to the provider default", () => { + expect(resolveModel(undefined, WORKERS_DEFAULT, "sonnet")).toBe("sonnet"); + }); + it("passes through a real model the core supplied", () => { + expect(resolveModel(undefined, "gpt-4o", "sonnet")).toBe("gpt-4o"); + }); +}); afterEach(() => vi.unstubAllGlobals()); @@ -63,4 +79,27 @@ describe("subscription CLI helpers + fail-safe", () => { expect(capturedEnv.ANTHROPIC_API_KEY).toBeUndefined(); // scrubbed expect(capturedEnv.CLAUDE_CODE_OAUTH_TOKEN).toBe("t"); }); + + it("Codex returns text on success and throws on a non-zero exit", async () => { + const ok: StubSpawn = async () => ({ stdout: JSON.stringify({ type: "result", result: "codex review" }), code: 0 }); + expect((await createCodexAi({}, ok).run("gpt-5", { prompt: "x" })).response).toBe("codex review"); + const bad: StubSpawn = async () => ({ stdout: "", code: 1 }); + await expect(createCodexAi({}, bad).run("gpt-5", { prompt: "x" })).rejects.toThrow(/codex_exit_1/); + }); + + it("drives the REAL subprocess (defaultSpawn) against a fake `claude` on PATH", async () => { + const dir = mkdtempSync(join(tmpdir(), "fakecli-")); + const fake = join(dir, "claude"); + // a minimal stand-in: read the prompt on stdin, emit a Claude-Code-shaped JSON result + writeFileSync(fake, "#!/usr/bin/env node\nlet i='';process.stdin.on('data',d=>i+=d);process.stdin.on('end',()=>process.stdout.write(JSON.stringify({type:'result',result:'OK:'+i.trim()})));\n"); + chmodSync(fake, 0o755); + const origPath = process.env.PATH; + process.env.PATH = `${dir}:${origPath ?? ""}`; + try { + const out = await createClaudeCodeAi({ ...process.env, CLAUDE_CODE_OAUTH_TOKEN: "t" }).run("sonnet", { prompt: "hello" }); + expect(out.response).toBe("OK:hello"); + } finally { + process.env.PATH = origPath; + } + }); }); diff --git a/test/unit/selfhost-health.test.ts b/test/unit/selfhost-health.test.ts new file mode 100644 index 0000000000..da1c1e98d9 --- /dev/null +++ b/test/unit/selfhost-health.test.ts @@ -0,0 +1,18 @@ +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, it } from "vitest"; +import { nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; +import { readiness } from "../../src/selfhost/health"; + +describe("readiness (#982)", () => { + it("is not ready until the migrations table has applied rows", () => { + const driver = nodeSqliteDriver(new DatabaseSync(":memory:") as never); + // db answers but no migrations table yet → not ready + expect(readiness(driver)).toEqual({ ok: false, checks: { db: true, migrations: false } }); + // empty migrations table → still not ready + driver.exec("CREATE TABLE _selfhost_migrations (name TEXT, applied_at INTEGER)"); + expect(readiness(driver).ok).toBe(false); + // an applied migration → ready + driver.query("INSERT INTO _selfhost_migrations (name, applied_at) VALUES (?, ?)", ["0001", 0]); + expect(readiness(driver)).toEqual({ ok: true, checks: { db: true, migrations: true } }); + }); +}); diff --git a/test/unit/selfhost-metrics.test.ts b/test/unit/selfhost-metrics.test.ts new file mode 100644 index 0000000000..9a21f9c29d --- /dev/null +++ b/test/unit/selfhost-metrics.test.ts @@ -0,0 +1,33 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { gauge, incr, renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; + +afterEach(() => resetMetrics()); + +describe("metrics registry (#982)", () => { + it("counters accumulate and render", () => { + incr("c_total"); + incr("c_total", undefined, 2); + expect(renderMetrics()).toContain("c_total 3"); + }); + + it("renders labels in Prometheus format", () => { + incr("h_total", { status: "ok" }); + expect(renderMetrics()).toContain('h_total{status="ok"} 1'); + }); + + it("gauges sample at scrape time", () => { + let v = 5; + gauge("g", () => v); + expect(renderMetrics()).toContain("g 5"); + v = 9; + expect(renderMetrics()).toContain("g 9"); + }); + + it("a throwing gauge does not break the scrape", () => { + gauge("bad", () => { + throw new Error("x"); + }); + incr("ok_total"); + expect(renderMetrics()).toContain("ok_total 1"); + }); +}); diff --git a/test/unit/selfhost-migrate.test.ts b/test/unit/selfhost-migrate.test.ts new file mode 100644 index 0000000000..bf0215e68d --- /dev/null +++ b/test/unit/selfhost-migrate.test.ts @@ -0,0 +1,22 @@ +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, it } from "vitest"; +import { createD1Adapter, nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; +import { runSelfHostMigrations } from "../../src/selfhost/migrate"; + +describe("runSelfHostMigrations (#980)", () => { + it("applies un-applied migrations in order, idempotently", async () => { + const dir = mkdtempSync(join(tmpdir(), "gtmig-")); + writeFileSync(join(dir, "0001_a.sql"), "CREATE TABLE a (id INTEGER);"); + writeFileSync(join(dir, "0002_b.sql"), "CREATE TABLE b (id INTEGER);"); + const db = createD1Adapter(nodeSqliteDriver(new DatabaseSync(":memory:") as never)); + + expect(await runSelfHostMigrations(db, dir)).toBe(2); // both applied + expect(await runSelfHostMigrations(db, dir)).toBe(0); // idempotent — nothing re-applied + + writeFileSync(join(dir, "0003_c.sql"), "CREATE TABLE c (id INTEGER);"); + expect(await runSelfHostMigrations(db, dir)).toBe(1); // only the new one + }); +}); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts new file mode 100644 index 0000000000..e03155ed97 --- /dev/null +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -0,0 +1,73 @@ +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, it } from "vitest"; +import { nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; +import { createSqliteQueue } from "../../src/selfhost/sqlite-queue"; +import type { JobMessage } from "../../src/types"; + +function makeDriver(): ReturnType { + return nodeSqliteDriver(new DatabaseSync(":memory:") as never); +} +const msg = (t: string): JobMessage => ({ type: t }) as unknown as JobMessage; +const typeOf = (m: JobMessage): string => (m as unknown as { type: string }).type; + +describe("createSqliteQueue (durable #980)", () => { + it("persists + drains FIFO through the consumer", async () => { + const driver = makeDriver(); + const seen: string[] = []; + const q = createSqliteQueue(driver, async (m) => void seen.push(typeOf(m))); + await q.binding.send(msg("a")); + await q.binding.send(msg("b")); + await q.drain(); + expect(seen).toEqual(["a", "b"]); + expect(q.size()).toBe(0); + }); + + it("retries then dead-letters after maxRetries", async () => { + const driver = makeDriver(); + let calls = 0; + const q = createSqliteQueue( + driver, + async () => { + calls += 1; + throw new Error("boom"); + }, + { maxRetries: 3, backoffMs: () => 0 }, + ); + await q.binding.send(msg("x")); + await q.drain(); // backoff 0 → all 3 attempts run within one drain, then dead-lettered + expect(calls).toBe(3); + expect(q.deadCount()).toBe(1); + expect(q.size()).toBe(0); + }); + + it("SURVIVES A RESTART: a fresh queue over the same DB processes a persisted pending job", async () => { + const driver = makeDriver(); + const seen: string[] = []; + const fresh = createSqliteQueue(driver, async (m) => void seen.push(typeOf(m))); // creates the table + // a job left pending on disk by a prior run (insert directly so this instance doesn't auto-process it first) + driver.query("INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at) VALUES (?, 'pending', 0, 0, 0)", [JSON.stringify(msg("persisted"))]); + await fresh.drain(); // the "new process" picks it up + expect(seen).toEqual(["persisted"]); + }); + + it("start() runs the poll loop and processes a job, stop() halts it", async () => { + const driver = makeDriver(); + const seen: string[] = []; + const q = createSqliteQueue(driver, async (m) => void seen.push(typeOf(m)), { pollIntervalMs: 10 }); + q.start(); + await q.binding.send(msg("ticked")); + for (let i = 0; i < 50 && seen.length === 0; i += 1) await new Promise((r) => setTimeout(r, 10)); + await q.stop(); + expect(seen).toEqual(["ticked"]); + }); + + it("recovers a job left 'processing' by a crash", async () => { + const driver = makeDriver(); + createSqliteQueue(driver, async () => undefined); // creates the table + driver.query("INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at) VALUES (?, 'processing', 0, 0, 0)", [JSON.stringify(msg("stuck"))]); + const seen: string[] = []; + const fresh = createSqliteQueue(driver, async (m) => void seen.push(typeOf(m))); + await fresh.drain(); + expect(seen).toEqual(["stuck"]); + }); +}); From 1372c0165ebefd2b02324510b283344440a79972 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 23 Jun 2026 21:16:33 -0700 Subject: [PATCH 06/25] =?UTF-8?q?feat(selfhost):=20Tier=201=20=E2=80=94=20?= =?UTF-8?q?multi-provider=20BYOK=20+=20fallback=20chain=20+=20native=20Ant?= =?UTF-8?q?hropic=20+=20CLI=20image=20(#979)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Native Anthropic Messages API adapter (BYOK; splits system, joins text blocks) — distinct from the claude-code subscription path. - AI_PROVIDER now accepts a COMMA-LIST → a fallback chain (createChainAi): tries each provider in order, logs failures, returns the first success; all-fail throws so the review degrades. e.g. 'anthropic,ollama'. - Per-provider credentials: ANTHROPIC_API_KEY / OPENAI_API_KEY / AI_API_KEY; a provider with no key is dropped from the chain. openai → api.openai.com default base URL. - Dockerfile --build-arg INSTALL_AI_CLIS=true bakes @anthropic-ai/claude-code + @openai/codex so the subscription providers work in-image (no credentials baked; operator mints the token at run time). - Docs + .env.example updated. +3 tests (Anthropic shape, chain fallback + all-fail). 32 self-host tests green. --- .env.example | 10 ++-- Dockerfile | 5 ++ docs/self-hosting.md | 11 +++- src/selfhost/ai.ts | 98 +++++++++++++++++++++++++++++++---- test/unit/selfhost-ai.test.ts | 45 +++++++++++++++- 5 files changed, 155 insertions(+), 14 deletions(-) diff --git a/.env.example b/.env.example index 966b70b937..e9fbde024f 100644 --- a/.env.example +++ b/.env.example @@ -112,9 +112,13 @@ GITTENSORY_REVIEW_DRAFT=false # --- AI review backend (optional; without it reviews run deterministically) --- # AI_SUMMARIES_ENABLED=true -# AI_PROVIDER=ollama # ollama | openai-compatible | claude-code | codex (see #979) -# AI_BASE_URL=http://ollama:11434/v1 # an OpenAI-compatible endpoint (the Ollama default) -# AI_API_KEY= # if your endpoint requires a key +# AI_PROVIDER=ollama # ollama | openai-compatible | openai | anthropic | claude-code | +# # codex. A COMMA-LIST is a fallback chain, e.g. "anthropic,ollama" +# # (tries each in order until one succeeds). (see #979) +# AI_BASE_URL=http://ollama:11434/v1 # OpenAI-compatible endpoint (Ollama default; or your provider's) +# AI_API_KEY= # generic key for the openai-compatible endpoint +# ANTHROPIC_API_KEY= # for AI_PROVIDER=anthropic (native Messages API, BYOK) +# OPENAI_API_KEY= # for AI_PROVIDER=openai # AI_MODEL=llama3.1 # the model for your provider (e.g. llama3.1 for Ollama, sonnet # # for claude-code, gpt-5 for codex). REQUIRED for non-Ollama: # # without it the adapter falls back to a provider default, never diff --git a/Dockerfile b/Dockerfile index 0b8aeb52da..4ca4783e78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,11 @@ COPY --from=build /app/node_modules ./node_modules COPY --from=build /app/dist ./dist COPY --from=build /app/migrations ./migrations COPY --from=build /app/scripts/register-selfhost.mjs ./scripts/register-selfhost.mjs +# Optional: bake the Claude Code / Codex CLIs so the `claude-code` / `codex` subscription providers (#979) +# work in-image. Build with `--build-arg INSTALL_AI_CLIS=true`. No credentials are baked — operators mint +# CLAUDE_CODE_OAUTH_TOKEN (`claude setup-token`) / codex auth at run time and pass it via the env. +ARG INSTALL_AI_CLIS=false +RUN if [ "$INSTALL_AI_CLIS" = "true" ]; then npm install -g @anthropic-ai/claude-code @openai/codex; fi # Data dir (the SQLite file) — owned by the unprivileged node user; mount a volume here to persist. RUN mkdir -p /data && chown -R node:node /data /app USER node diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 81643b8538..231f21bee1 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -76,10 +76,19 @@ and only the AI **summary** degrades to "unavailable". To enable AI, set `AI_PRO | `AI_PROVIDER` | Backend | Extra config | | --- | --- | --- | -| `ollama` / `openai-compatible` / `openai` | any OpenAI-compatible `/chat/completions` endpoint | `AI_BASE_URL`, `AI_API_KEY`, `AI_MODEL` | +| `ollama` / `openai-compatible` / `openai` | any OpenAI-compatible `/chat/completions` endpoint (Ollama, OpenAI, Groq, Together, OpenRouter, vLLM, Gemini's OpenAI-compat endpoint, …) | `AI_BASE_URL`, `AI_API_KEY` (or `OPENAI_API_KEY`), `AI_MODEL` | +| `anthropic` | **native Anthropic Messages API** (BYOK — bills your API key) | `ANTHROPIC_API_KEY`, `AI_MODEL` (e.g. `claude-sonnet-4-6`) | | `claude-code` | your **Claude** subscription via the `claude` CLI (read-only, headless) | `CLAUDE_CODE_OAUTH_TOKEN` (from `claude setup-token`), `AI_MODEL` (e.g. `sonnet`) | | `codex` | your **Codex** subscription via the `codex` CLI | local `codex` auth, `AI_MODEL` (e.g. `gpt-5`) | +**Fallback chain.** `AI_PROVIDER` accepts a comma-separated list and tries each in order until one succeeds — +e.g. `AI_PROVIDER=anthropic,ollama` uses the Anthropic API first and falls back to a local Ollama model if it +errors. If every provider fails, the AI summary degrades to "unavailable" and the review still runs. + +**Subscription CLIs in the image.** The `claude-code` / `codex` providers need their CLI present. Build the +image with `--build-arg INSTALL_AI_CLIS=true` (or `docker compose build --build-arg INSTALL_AI_CLIS=true`) to +bake them in, then provide `CLAUDE_CODE_OAUTH_TOKEN` / codex auth at run time. No credentials are baked in. + > **Set `AI_MODEL`.** The core would otherwise hand the adapter a Cloudflare Workers-AI model id > (`@cf/meta/...`) that Ollama / `claude` / `codex` can't use. The adapter ignores that id in favour of > `AI_MODEL` (falling back to a provider default), so always set `AI_MODEL` to a real model for your provider. diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index 920937d9b7..9de7b40054 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -52,6 +52,37 @@ export function createOpenAiCompatibleAi(opts: { baseUrl: string; apiKey?: strin }; } +/** Native Anthropic Messages API (BYOK — bills your Anthropic API key; distinct from the claude-code + * subscription path). The system message becomes the top-level `system` param; the rest map to user/assistant. */ +export function createAnthropicAi(opts: { apiKey: string; model?: string | undefined; baseUrl?: string | undefined }): SelfHostAi { + const base = (opts.baseUrl ?? "https://api.anthropic.com").replace(/\/+$/, ""); + return { + async run(model, options) { + const msgs = toMessages(options); + const system = + msgs + .filter((m) => m.role === "system") + .map((m) => m.content) + .join("\n\n") || undefined; + const messages = msgs.filter((m) => m.role !== "system").map((m) => ({ role: m.role === "assistant" ? "assistant" : "user", content: m.content })); + const res = await fetch(`${base}/v1/messages`, { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": opts.apiKey, "anthropic-version": "2023-06-01" }, + body: JSON.stringify({ model: resolveModel(opts.model, model, "claude-sonnet-4-6"), max_tokens: options.max_tokens ?? 1024, ...(system ? { system } : {}), messages }), + signal: AbortSignal.timeout(120_000), + }); + if (!res.ok) throw new Error(`anthropic_http_${res.status}`); + const data = (await res.json()) as { content?: Array<{ type: string; text?: string }> }; + return { + response: (data.content ?? []) + .filter((c) => c.type === "text") + .map((c) => c.text ?? "") + .join(""), + }; + }, + }; +} + // ── Subscription CLI providers (#979) — locally-authenticated `claude` / `codex` as a subprocess ────────── // SECURITY: the child env DELETES the billable API keys so a misconfigured CLI cannot silently bill the // metered API instead of using the subscription OAuth token. The CLI runs read-only / no extra tools. Any @@ -169,14 +200,63 @@ export function createCodexAi(parentEnv: Record, spa }; } -/** Pick the self-host AI provider from env (AI_PROVIDER). Returns undefined when unconfigured. */ -export function createSelfHostAi(env: Record): SelfHostAi | undefined { - const provider = (env.AI_PROVIDER ?? "").trim().toLowerCase(); - if (!provider) return undefined; - if (provider === "ollama" || provider === "openai-compatible" || provider === "openai") { - return createOpenAiCompatibleAi({ baseUrl: env.AI_BASE_URL ?? "http://localhost:11434/v1", apiKey: env.AI_API_KEY, model: configuredModel(env) }); +/** Try each provider in order until one returns; if all throw, rethrow the last error so the caller degrades + * (AI summary → "unavailable"; the review still runs deterministically). The fallback chain is what makes a + * BYOK setup robust — e.g. AI_PROVIDER="anthropic,ollama" uses the API first and a local model if it's down. */ +export function createChainAi(providers: Array<{ name: string; ai: SelfHostAi }>): SelfHostAi { + return { + async run(model, options) { + let lastError: unknown = new Error("no_ai_providers"); + for (const p of providers) { + try { + return await p.ai.run(model, options); + } catch (error) { + lastError = error; + console.error(JSON.stringify({ level: "warn", event: "selfhost_ai_provider_failed", provider: p.name, error: error instanceof Error ? error.message : "unknown" })); + } + } + throw lastError instanceof Error ? lastError : new Error("all_ai_providers_failed"); + }, + }; +} + +/** Build one provider adapter by name (BYO credentials read from provider-specific env, then the generic + * AI_API_KEY). Returns undefined when its required credential is missing. */ +export function buildProvider(name: string, env: Record): SelfHostAi | undefined { + switch (name) { + case "ollama": + case "openai-compatible": + case "openai": + return createOpenAiCompatibleAi({ + baseUrl: env.AI_BASE_URL ?? (name === "openai" ? "https://api.openai.com/v1" : "http://localhost:11434/v1"), + apiKey: env.AI_API_KEY ?? env.OPENAI_API_KEY, + model: configuredModel(env), + }); + case "anthropic": { + const apiKey = env.ANTHROPIC_API_KEY ?? env.AI_API_KEY; + return apiKey ? createAnthropicAi({ apiKey, model: configuredModel(env), baseUrl: env.AI_BASE_URL }) : undefined; + } + case "claude-code": + return createClaudeCodeAi(env); + case "codex": + return createCodexAi(env); + default: + return undefined; } - if (provider === "claude-code") return createClaudeCodeAi(env); - if (provider === "codex") return createCodexAi(env); - return undefined; +} + +/** Select the self-host AI provider(s) from AI_PROVIDER. A comma-separated list builds a fallback chain + * (first to succeed wins). Returns undefined when unconfigured or no provider has its credential. */ +export function createSelfHostAi(env: Record): SelfHostAi | undefined { + const raw = (env.AI_PROVIDER ?? "").trim().toLowerCase(); + if (!raw) return undefined; + const providers = raw + .split(",") + .map((s) => s.trim()) + .filter(Boolean) + .map((name) => ({ name, ai: buildProvider(name, env) })) + .filter((p): p is { name: string; ai: SelfHostAi } => Boolean(p.ai)); + if (providers.length === 0) return undefined; + if (providers.length === 1) return providers[0]?.ai; + return createChainAi(providers); } diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index 4eb6e7c9ad..323e6647cc 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -2,7 +2,7 @@ import { chmodSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { claudeErrorStatus, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, resolveModel } from "../../src/selfhost/ai"; +import { claudeErrorStatus, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, resolveModel } from "../../src/selfhost/ai"; describe("resolveModel (#979 — never leak the Workers-AI default to a self-host backend)", () => { const WORKERS_DEFAULT = "@cf/meta/llama-3.1-8b-instruct-fp8-fast"; @@ -53,6 +53,49 @@ describe("createSelfHostAi — provider selection", () => { expect(typeof createSelfHostAi({ AI_PROVIDER: "codex" })?.run).toBe("function"); expect(createSelfHostAi({ AI_PROVIDER: "nonsense" })).toBeUndefined(); }); + it("anthropic requires a key; a comma-list builds a fallback chain", () => { + expect(createSelfHostAi({ AI_PROVIDER: "anthropic" })).toBeUndefined(); // no key → dropped + expect(typeof createSelfHostAi({ AI_PROVIDER: "anthropic", ANTHROPIC_API_KEY: "sk-ant" })?.run).toBe("function"); + // "anthropic,ollama" with a key → both build → a chain (a runnable adapter) + expect(typeof createSelfHostAi({ AI_PROVIDER: "anthropic,ollama", ANTHROPIC_API_KEY: "sk-ant" })?.run).toBe("function"); + }); +}); + +describe("createAnthropicAi (#979 native BYOK)", () => { + it("splits the system message and returns the joined text content", async () => { + let sent: { url: string; headers: Record; body: Record } | undefined; + vi.stubGlobal("fetch", vi.fn(async (url: string, init: { headers: Record; body: string }) => { + sent = { url, headers: init.headers, body: JSON.parse(init.body) as Record }; + return new Response(JSON.stringify({ content: [{ type: "text", text: "hi" }, { type: "thinking", text: "ignored" }] }), { status: 200 }); + })); + const out = await createAnthropicAi({ apiKey: "sk-ant", model: "claude-sonnet-4-6" }).run("@cf/ignored", { + messages: [ + { role: "system", content: "be terse" }, + { role: "user", content: "go" }, + ], + max_tokens: 256, + }); + expect(out.response).toBe("hi"); // only text blocks + expect(sent?.url).toBe("https://api.anthropic.com/v1/messages"); + expect(sent?.headers["x-api-key"]).toBe("sk-ant"); + expect(sent?.headers["anthropic-version"]).toBe("2023-06-01"); + expect(sent?.body.system).toBe("be terse"); + expect(sent?.body.model).toBe("claude-sonnet-4-6"); // configured wins over the @cf id + expect(sent?.body.messages).toEqual([{ role: "user", content: "go" }]); + }); +}); + +describe("createChainAi (fallback)", () => { + it("falls through to the next provider on failure, returns the first success", async () => { + const failing = { name: "a", ai: { run: async () => { throw new Error("down"); } } }; + const working = { name: "b", ai: { run: async () => ({ response: "from b" }) } }; + expect((await createChainAi([failing, working]).run("m", { prompt: "x" })).response).toBe("from b"); + }); + it("throws the last error when every provider fails", async () => { + const a = { name: "a", ai: { run: async () => { throw new Error("err-a"); } } }; + const b = { name: "b", ai: { run: async () => { throw new Error("err-b"); } } }; + await expect(createChainAi([a, b]).run("m", { prompt: "x" })).rejects.toThrow(/err-b/); + }); }); describe("subscription CLI helpers + fail-safe", () => { From d85917a5efb12e8ad945ebf92088dc1da245044a Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 23 Jun 2026 21:22:25 -0700 Subject: [PATCH 07/25] =?UTF-8?q?feat(selfhost):=20Tier=201=20=E2=80=94=20?= =?UTF-8?q?local=20RAG=20via=20SQLite=20vector=20store=20+=20embeddings=20?= =?UTF-8?q?(#979)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retrieval-augmented review without Cloudflare Vectorize: - createSqliteVectorize: implements the Vectorize binding surface (upsert/query/deleteByIds) backed by a SQLite table (_selfhost_vectors), brute-force cosine similarity, namespace-scoped (one per repo). Wired as env.VECTORIZE so the core RAG path (reviewVectorAdapter) works unchanged. - Embeddings: the OpenAI-compatible adapter now routes { text: [...] } embed calls to /embeddings and returns { data: number[][] } (the shape embedTexts expects). AI_EMBED_MODEL selects a 1024-d model (bge-m3 / mxbai-embed-large via Ollama); without it RAG degrades to no-context. - AiResult widened to { response?; data? } so chat + embed share the run() seam. - Gated by GITTENSORY_REVIEW_RAG + the repo allowlist (off by default). +6 tests (cosine, namespace scoping, upsert-overwrite/delete, embed routing). 38 self-host tests green, boots with all 3 self-host tables. --- .env.example | 3 + docs/self-hosting.md | 7 +++ src/selfhost/ai.ts | 26 +++++++-- src/selfhost/vectorize.ts | 86 ++++++++++++++++++++++++++++ src/server.ts | 6 +- test/unit/selfhost-ai.test.ts | 11 ++++ test/unit/selfhost-vectorize.test.ts | 51 +++++++++++++++++ 7 files changed, 185 insertions(+), 5 deletions(-) create mode 100644 src/selfhost/vectorize.ts create mode 100644 test/unit/selfhost-vectorize.test.ts diff --git a/.env.example b/.env.example index e9fbde024f..6390af737c 100644 --- a/.env.example +++ b/.env.example @@ -123,3 +123,6 @@ GITTENSORY_REVIEW_DRAFT=false # # for claude-code, gpt-5 for codex). REQUIRED for non-Ollama: # # without it the adapter falls back to a provider default, never # # the Cloudflare Workers-AI id the core would otherwise pass. +# AI_EMBED_MODEL=bge-m3 # embedding model for RAG (openai-compatible /embeddings). MUST be +# # 1024-dimensional (e.g. bge-m3 or mxbai-embed-large via Ollama). +# # Used only when RAG is enabled (GITTENSORY_REVIEW_RAG + allowlist). diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 231f21bee1..3305b6141f 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -89,6 +89,13 @@ errors. If every provider fails, the AI summary degrades to "unavailable" and th image with `--build-arg INSTALL_AI_CLIS=true` (or `docker compose build --build-arg INSTALL_AI_CLIS=true`) to bake them in, then provide `CLAUDE_CODE_OAUTH_TOKEN` / codex auth at run time. No credentials are baked in. +**Local RAG (retrieval-augmented review).** Self-host ships a SQLite-backed vector store, so RAG works without +Cloudflare Vectorize. Enable it with `GITTENSORY_REVIEW_RAG=true` + the repo in `GITTENSORY_REVIEW_REPOS`, and +point at an **embedding-capable** OpenAI-compatible provider (Ollama) with a **1024-dimensional** model via +`AI_EMBED_MODEL` (e.g. `bge-m3` or `mxbai-embed-large`). Embeddings + chunk vectors are stored in the same +SQLite DB (`_selfhost_vectors`) and queried by cosine similarity. Without an embedding model, RAG degrades to +no-context (the review still runs). + > **Set `AI_MODEL`.** The core would otherwise hand the adapter a Cloudflare Workers-AI model id > (`@cf/meta/...`) that Ollama / `claude` / `codex` can't use. The adapter ignores that id in favour of > `AI_MODEL` (falling back to a provider default), so always set `AI_MODEL` to a real model for your provider. diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index 9de7b40054..6e69b414e3 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -9,11 +9,15 @@ interface AiRunOptions { messages?: Array<{ role: string; content: string }>; prompt?: string; + text?: string[]; // embedding input — the core's embedTexts passes { text: string[] } max_tokens?: number; temperature?: number; } +/** A chat completion (`response`) or an embedding result (`data`). Both optional: the core reads whichever it + * asked for (extractAiText → `response`, embedTexts → `data`), each defensive about the other being absent. */ +export type AiResult = { response?: string; data?: number[][] }; export interface SelfHostAi { - run(model: string, options: AiRunOptions): Promise<{ response: string }>; + run(model: string, options: AiRunOptions): Promise; } function toMessages(options: AiRunOptions): Array<{ role: string; content: string }> { @@ -34,14 +38,27 @@ function configuredModel(env: Record): string | unde return env.AI_MODEL ?? env.WORKERS_AI_SUMMARY_MODEL; } -/** OpenAI-compatible chat endpoint (Ollama's /v1, OpenAI, vLLM, LM Studio, …). */ -export function createOpenAiCompatibleAi(opts: { baseUrl: string; apiKey?: string | undefined; model?: string | undefined }): SelfHostAi { +/** OpenAI-compatible endpoint (Ollama's /v1, OpenAI, vLLM, LM Studio, …) — chat + embeddings. */ +export function createOpenAiCompatibleAi(opts: { baseUrl: string; apiKey?: string | undefined; model?: string | undefined; embedModel?: string | undefined }): SelfHostAi { const base = opts.baseUrl.replace(/\/+$/, ""); + const headers = (): Record => ({ "content-type": "application/json", ...(opts.apiKey ? { authorization: `Bearer ${opts.apiKey}` } : {}) }); return { async run(model, options) { + // Embedding request — the core's embedTexts passes { text: string[] }; route to /embeddings (for RAG). + if (Array.isArray(options.text)) { + const res = await fetch(`${base}/embeddings`, { + method: "POST", + headers: headers(), + body: JSON.stringify({ model: opts.embedModel ?? "bge-m3", input: options.text }), + signal: AbortSignal.timeout(120_000), + }); + if (!res.ok) throw new Error(`ai_embed_http_${res.status}`); + const json = (await res.json()) as { data?: Array<{ embedding: number[] }> }; + return { data: (json.data ?? []).map((d) => d.embedding) }; + } const res = await fetch(`${base}/chat/completions`, { method: "POST", - headers: { "content-type": "application/json", ...(opts.apiKey ? { authorization: `Bearer ${opts.apiKey}` } : {}) }, + headers: headers(), body: JSON.stringify({ model: resolveModel(opts.model, model, "llama3.1"), messages: toMessages(options), max_tokens: options.max_tokens, temperature: options.temperature }), signal: AbortSignal.timeout(120_000), }); @@ -231,6 +248,7 @@ export function buildProvider(name: string, env: Record; +} +interface QueryOptions { + topK?: number; + namespace?: string; + returnMetadata?: string; +} +interface Match { + id: string; + score: number; + metadata?: Record; +} + +export function cosineSimilarity(a: number[], b: number[]): number { + let dot = 0; + let na = 0; + let nb = 0; + const n = Math.min(a.length, b.length); + for (let i = 0; i < n; i += 1) { + const x = a[i] as number; + const y = b[i] as number; + dot += x * y; + na += x * x; + nb += y * y; + } + const denom = Math.sqrt(na) * Math.sqrt(nb); + return denom === 0 ? 0 : dot / denom; +} + +export function createSqliteVectorize(driver: SqliteDriver): Vectorize { + driver.exec(DDL); + const adapter = { + async upsert(vectors: VectorRecord[]): Promise<{ count: number; ids: string[] }> { + for (const v of vectors) { + driver.query( + `INSERT INTO ${TABLE} (id, namespace, embedding, metadata) VALUES (?,?,?,?) + ON CONFLICT(id) DO UPDATE SET namespace=excluded.namespace, embedding=excluded.embedding, metadata=excluded.metadata`, + [v.id, v.namespace ?? "", JSON.stringify(v.values), v.metadata ? JSON.stringify(v.metadata) : null], + ); + } + return { count: vectors.length, ids: vectors.map((v) => v.id) }; + }, + async query(vector: number[], opts: QueryOptions): Promise<{ matches: Match[] }> { + const { rows } = opts.namespace + ? driver.query(`SELECT id, embedding, metadata FROM ${TABLE} WHERE namespace=?`, [opts.namespace]) + : driver.query(`SELECT id, embedding, metadata FROM ${TABLE}`, []); + const scored: Match[] = rows.map((r) => { + const values = JSON.parse(r.embedding as string) as number[]; + const metadata = r.metadata ? (JSON.parse(r.metadata as string) as Record) : undefined; + const score = cosineSimilarity(vector, values); + return metadata === undefined ? { id: r.id as string, score } : { id: r.id as string, score, metadata }; + }); + scored.sort((a, b) => b.score - a.score); + return { matches: scored.slice(0, opts.topK ?? 12) }; + }, + async deleteByIds(ids: string[]): Promise<{ count: number }> { + for (let i = 0; i < ids.length; i += 90) { + const batch = ids.slice(i, i + 90); + driver.query(`DELETE FROM ${TABLE} WHERE id IN (${batch.map(() => "?").join(",")})`, batch); + } + return { count: ids.length }; + }, + }; + return adapter as unknown as Vectorize; +} diff --git a/src/server.ts b/src/server.ts index 144ee70a69..ad41cbf966 100644 --- a/src/server.ts +++ b/src/server.ts @@ -15,6 +15,7 @@ import { readiness } from "./selfhost/health"; import { gauge, incr, renderMetrics } from "./selfhost/metrics"; import { runSelfHostMigrations } from "./selfhost/migrate"; import { createSqliteQueue } from "./selfhost/sqlite-queue"; +import { createSqliteVectorize } from "./selfhost/vectorize"; import type { JobMessage } from "./types"; /** Resolve `_FILE` env vars (Docker secrets / multi-line keys) into `` at startup. */ @@ -53,7 +54,10 @@ async function main(): Promise { // gittensory's AI summary degrades to "unavailable" and the review proceeds deterministically). const ai = createSelfHostAi(process.env); if (ai) console.log(JSON.stringify({ event: "selfhost_ai_provider", provider: process.env.AI_PROVIDER })); - env = { ...process.env, DB: db, JOBS: queue.binding, AI: ai } as unknown as Env; + // Vector store for RAG (gated by GITTENSORY_REVIEW_RAG + the repo allowlist + an embedding-capable provider); + // a SQLite-backed Vectorize so retrieval works without Cloudflare Vectorize. + const vectorize = createSqliteVectorize(driver); + env = { ...process.env, DB: db, JOBS: queue.binding, AI: ai, VECTORIZE: vectorize } as unknown as Env; gauge("gittensory_queue_pending", () => queue.size()); gauge("gittensory_queue_dead", () => queue.deadCount()); diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index 323e6647cc..250b442656 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -41,6 +41,17 @@ describe("createOpenAiCompatibleAi (#979)", () => { vi.stubGlobal("fetch", vi.fn(async () => new Response("err", { status: 500 }))); await expect(createOpenAiCompatibleAi({ baseUrl: "http://x/v1" }).run("m", { prompt: "p" })).rejects.toThrow(/ai_http_500/); }); + + it("routes an embedding request ({ text }) to /embeddings and returns { data }", async () => { + let url = ""; + vi.stubGlobal("fetch", vi.fn(async (u: string) => { + url = u; + return new Response(JSON.stringify({ data: [{ embedding: [0.1, 0.2] }, { embedding: [0.3, 0.4] }] }), { status: 200 }); + })); + const out = await createOpenAiCompatibleAi({ baseUrl: "http://o/v1", embedModel: "bge-m3" }).run("@cf/baai/bge-m3", { text: ["a", "b"] }); + expect(url).toBe("http://o/v1/embeddings"); + expect(out).toEqual({ data: [[0.1, 0.2], [0.3, 0.4]] }); + }); }); describe("createSelfHostAi — provider selection", () => { diff --git a/test/unit/selfhost-vectorize.test.ts b/test/unit/selfhost-vectorize.test.ts new file mode 100644 index 0000000000..f744a6ed4d --- /dev/null +++ b/test/unit/selfhost-vectorize.test.ts @@ -0,0 +1,51 @@ +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, it } from "vitest"; +import { nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; +import { cosineSimilarity, createSqliteVectorize } from "../../src/selfhost/vectorize"; + +function makeVectorize(): ReturnType { + return createSqliteVectorize(nodeSqliteDriver(new DatabaseSync(":memory:") as never)); +} + +describe("cosineSimilarity", () => { + it("is 1 for identical and 0 for orthogonal vectors", () => { + expect(cosineSimilarity([1, 0], [1, 0])).toBeCloseTo(1); + expect(cosineSimilarity([1, 0], [0, 1])).toBeCloseTo(0); + expect(cosineSimilarity([0, 0], [0, 0])).toBe(0); // zero-norm guard + }); +}); + +describe("createSqliteVectorize (#979 local RAG)", () => { + it("returns the nearest-by-cosine match within a namespace, with metadata + topK", async () => { + const v = makeVectorize(); + await v.upsert([ + { id: "a", values: [1, 0, 0], namespace: "repo1", metadata: { path: "a.ts" } }, + { id: "b", values: [0, 1, 0], namespace: "repo1", metadata: { path: "b.ts" } }, + ]); + const res = await v.query([0.9, 0.1, 0], { topK: 1, namespace: "repo1", returnMetadata: "all" }); + expect(res.matches).toHaveLength(1); + expect(res.matches[0]?.id).toBe("a"); + expect(res.matches[0]?.metadata?.path).toBe("a.ts"); + }); + + it("scopes results by namespace", async () => { + const v = makeVectorize(); + await v.upsert([ + { id: "x", values: [1, 0], namespace: "n1" }, + { id: "y", values: [1, 0], namespace: "n2" }, + ]); + const res = await v.query([1, 0], { topK: 10, namespace: "n1" }); + expect(res.matches.map((m) => m.id)).toEqual(["x"]); + }); + + it("upsert overwrites by id; deleteByIds removes", async () => { + const v = makeVectorize(); + await v.upsert([{ id: "d", values: [1, 0], namespace: "n", metadata: { path: "old" } }]); + await v.upsert([{ id: "d", values: [0, 1], namespace: "n", metadata: { path: "new" } }]); // overwrite + let res = await v.query([0, 1], { topK: 10, namespace: "n" }); + expect(res.matches[0]?.metadata?.path).toBe("new"); + await v.deleteByIds(["d"]); + res = await v.query([0, 1], { topK: 10, namespace: "n" }); + expect(res.matches).toHaveLength(0); + }); +}); From 07ff98c7dd4f86f9ecb27632b7a519c00a70c891 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 23 Jun 2026 21:26:25 -0700 Subject: [PATCH 08/25] =?UTF-8?q?perf(selfhost):=20self-contained=20bundle?= =?UTF-8?q?=20=E2=86=92=20254MB=20image=20(was=201.33GB)=20(#980)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/build-selfhost.mjs --all bundles every dependency into one dist/server.mjs (createRequire/__dirname banner for the CJS deps), so the runtime image carries no node_modules and needs no cloudflare:* loader. The Dockerfile build stage uses --all; the runtime stage copies only dist + migrations and runs 'node dist/server.mjs'. Validated: all deep dependency subtrees load (octokit/crypto webhook 400, MCP/zod 200, zod-to-openapi, drizzle/D1) and the container boots (56 migrations, /health, /ready). External mode stays the default for fast local dev rebuilds. --- Dockerfile | 9 ++++----- scripts/build-selfhost.mjs | 28 ++++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4ca4783e78..d0bc719004 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,9 @@ COPY package*.json ./ # pure JS; esbuild ships its binary as an optional dependency, not a script). RUN npm ci --ignore-scripts COPY . . -RUN node scripts/build-selfhost.mjs +# --all: bundle every dependency into one self-contained dist/server.mjs, so the runtime image needs no +# node_modules (≈10× smaller). The bundle has zero `cloudflare:*` imports (stubbed at build), so no loader. +RUN node scripts/build-selfhost.mjs --all # --- runtime: slim, non-root ---------------------------------------------------------------------------- FROM node:24-slim AS runtime @@ -21,10 +23,8 @@ ENV NODE_ENV=production \ PORT=8787 \ DATABASE_PATH=/data/gittensory.sqlite \ MIGRATIONS_DIR=/app/migrations -COPY --from=build /app/node_modules ./node_modules COPY --from=build /app/dist ./dist COPY --from=build /app/migrations ./migrations -COPY --from=build /app/scripts/register-selfhost.mjs ./scripts/register-selfhost.mjs # Optional: bake the Claude Code / Codex CLIs so the `claude-code` / `codex` subscription providers (#979) # work in-image. Build with `--build-arg INSTALL_AI_CLIS=true`. No credentials are baked — operators mint # CLAUDE_CODE_OAUTH_TOKEN (`claude setup-token`) / codex auth at run time and pass it via the env. @@ -36,5 +36,4 @@ USER node EXPOSE 8787 HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ CMD node -e "fetch('http://127.0.0.1:'+(process.env.PORT||8787)+'/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" -# The register hook stubs cloudflare:* imports so the Worker graph loads on Node. -CMD ["node", "--import", "./scripts/register-selfhost.mjs", "dist/server.mjs"] +CMD ["node", "dist/server.mjs"] diff --git a/scripts/build-selfhost.mjs b/scripts/build-selfhost.mjs index fc5031f399..a8c9ee1c4f 100644 --- a/scripts/build-selfhost.mjs +++ b/scripts/build-selfhost.mjs @@ -1,11 +1,15 @@ -// Bundle the self-host Node entry (src/server.ts) into dist/server.mjs. node_modules stay external (resolved -// at runtime); `cloudflare:workers` is resolved to the Node shim via a plugin (which takes precedence over -// `packages: "external"`, so it is BUNDLED rather than left as an unresolvable bare import). +// Bundle the self-host Node entry (src/server.ts) into dist/server.mjs. +// default → node_modules stay external (resolved at runtime; fast local dev rebuilds). +// --all / SELFHOST_BUNDLE_ALL=1 → bundle EVERYTHING into one self-contained file (the Docker image needs no +// node_modules → a ~10× smaller image). node: builtins stay external (platform:node). +// In both modes the Cloudflare-only specifiers resolve to Node stubs via the plugin (precedence over external), +// so the bundle has zero `cloudflare:*` imports. import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import esbuild from "esbuild"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const bundleAll = process.env.SELFHOST_BUNDLE_ALL === "1" || process.argv.includes("--all"); await esbuild.build({ entryPoints: [resolve(root, "src/server.ts")], @@ -14,7 +18,23 @@ await esbuild.build({ format: "esm", target: "node22", outfile: resolve(root, "dist/server.mjs"), - packages: "external", + // External: nothing (bundle all) vs every package (external). node: builtins are always external on node. + ...(bundleAll ? {} : { packages: "external" }), + // Bundling CJS deps into an ESM output needs require/__dirname/__filename shimmed (some deps call them). + ...(bundleAll + ? { + banner: { + js: [ + "import { createRequire as __createRequire } from 'node:module';", + "import { fileURLToPath as __fileURLToPath } from 'node:url';", + "import { dirname as __pathDirname } from 'node:path';", + "const require = __createRequire(import.meta.url);", + "const __filename = __fileURLToPath(import.meta.url);", + "const __dirname = __pathDirname(__filename);", + ].join("\n"), + }, + } + : {}), plugins: [ { name: "selfhost-stubs", From 9bc1d6a529724882951244f70f6363909f86ff2d Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 23 Jun 2026 21:27:28 -0700 Subject: [PATCH 09/25] feat(selfhost): GHCR multi-arch release pipeline (#980) Pushing a selfhost-v tag (or workflow_dispatch) builds the image for linux/amd64 + linux/arm64 and pushes it to ghcr.io//gittensory-selfhost with : / :latest / :sha-* tags (provenance + SBOM, gha build cache) and opens a GitHub Release. Docs: run the published image + how to cut a release. The published image is lean (no CLIs); operators who want the claude-code/codex subscription build with --build-arg INSTALL_AI_CLIS=true. --- .github/workflows/release-selfhost.yml | 90 ++++++++++++++++++++++++++ docs/self-hosting.md | 13 +++- 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/release-selfhost.yml diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml new file mode 100644 index 0000000000..4a12b6c601 --- /dev/null +++ b/.github/workflows/release-selfhost.yml @@ -0,0 +1,90 @@ +# Self-host image releases (#980). Cutting a `selfhost-v` tag builds the multi-arch image, pushes it +# to GHCR with version + latest + sha tags (with provenance + SBOM), and opens a GitHub Release. +# +# git tag selfhost-v0.1.0 && git push origin selfhost-v0.1.0 +# +# Pull: docker pull ghcr.io//gittensory-selfhost:0.1.0 +name: release-selfhost + +on: + push: + tags: + - "selfhost-v*" + workflow_dispatch: + inputs: + version: + description: "Version to publish (e.g. 0.1.0)" + required: true + +permissions: + contents: write # create the GitHub Release + packages: write # push to GHCR + +jobs: + release: + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4 + + - name: Resolve version + id: version + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "v=${{ github.event.inputs.version }}" >> "$GITHUB_OUTPUT" + else + echo "v=${GITHUB_REF_NAME#selfhost-v}" >> "$GITHUB_OUTPUT" + fi + + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/gittensory-selfhost + tags: | + type=raw,value=${{ steps.version.outputs.v }} + type=raw,value=latest + type=sha,format=short + labels: | + org.opencontainers.image.title=gittensory-selfhost + org.opencontainers.image.description=Self-hostable Gittensory review engine + org.opencontainers.image.version=${{ steps.version.outputs.v }} + + - name: Build + push (linux/amd64 + linux/arm64) + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + provenance: true + sbom: true + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: GitHub Release + if: github.event_name == 'push' + uses: softprops/action-gh-release@v2 + with: + generate_release_notes: true + body: | + Self-host container image: + + ```bash + docker pull ghcr.io/${{ github.repository_owner }}/gittensory-selfhost:${{ steps.version.outputs.v }} + ``` + + Multi-arch (linux/amd64 + linux/arm64). See [docs/self-hosting.md](docs/self-hosting.md) for setup. + To include the Claude Code / Codex subscription CLIs, build locally with + `--build-arg INSTALL_AI_CLIS=true`. diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 3305b6141f..264e691e9a 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -25,14 +25,25 @@ On first boot the container creates the SQLite database on the `gittensory-data` migrations automatically (`{"event":"selfhost_migrations_applied","count":56}` in the logs). Point your GitHub App's webhook at `https:///v1/github/webhook` (expose port 8787 behind your own TLS). +**Or use the published image** (multi-arch, ~254 MB) instead of building: + +```bash +docker run -p 8787:8787 --env-file .env -v gittensory-data:/data \ + ghcr.io//gittensory-selfhost:latest # or pin a version, e.g. :0.1.0 +``` + To run without Docker: ```bash npm ci -node scripts/build-selfhost.mjs +node scripts/build-selfhost.mjs # external mode (fast local rebuilds) node --import ./scripts/register-selfhost.mjs dist/server.mjs ``` +Releases are cut by pushing a `selfhost-v` tag (e.g. `selfhost-v0.1.0`): CI builds the multi-arch +image, pushes it to GHCR with `:`, `:latest`, and `:sha-…` tags (with provenance + SBOM), and opens a +GitHub Release. + --- ## 2. Create the GitHub App From a96f43b4873c21439bba718d386d7737d5df03d6 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 23 Jun 2026 21:28:57 -0700 Subject: [PATCH 10/25] feat(selfhost): optional Litestream continuous SQLite backup (#982) An optional Litestream sidecar (docker-compose.yml, off by default) streams every change to the self-host SQLite DB to S3/B2/MinIO/R2 for point-in-time restore. Sample litestream.yml.example (placeholders only) + LITESTREAM_* env + a docs backup note. --- .env.example | 6 ++++++ docker-compose.yml | 16 ++++++++++++++++ docs/self-hosting.md | 3 +++ litestream.yml.example | 14 ++++++++++++++ 4 files changed, 39 insertions(+) create mode 100644 litestream.yml.example diff --git a/.env.example b/.env.example index 6390af737c..f886e6fb27 100644 --- a/.env.example +++ b/.env.example @@ -110,6 +110,12 @@ GITTENSORY_REVIEW_DRAFT=false # MIGRATIONS_DIR=/app/migrations # CRON_INTERVAL_MS=120000 # maintain/sweep + sync cadence (default ~2 min) +# --- Continuous backup (optional; the Litestream sidecar in docker-compose.yml) --- +# LITESTREAM_ACCESS_KEY_ID= +# LITESTREAM_SECRET_ACCESS_KEY= +# LITESTREAM_ENDPOINT= # e.g. s3.us-west-002.backblazeb2.com (omit for AWS S3) +# LITESTREAM_REGION=us-east-1 + # --- AI review backend (optional; without it reviews run deterministically) --- # AI_SUMMARIES_ENABLED=true # AI_PROVIDER=ollama # ollama | openai-compatible | openai | anthropic | claude-code | diff --git a/docker-compose.yml b/docker-compose.yml index 5ba8cf870d..6326ca9d42 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,22 @@ services: # volumes: # - ollama-models:/root/.ollama + # Optional: continuous SQLite backup to S3/B2/MinIO/R2 via Litestream (https://litestream.io). + # Copy litestream.yml.example -> litestream.yml, fill in your bucket, then uncomment: + # litestream: + # image: litestream/litestream:latest + # command: replicate + # restart: unless-stopped + # depends_on: [gittensory] + # volumes: + # - gittensory-data:/data + # - ./litestream.yml:/etc/litestream.yml:ro + # environment: + # LITESTREAM_ACCESS_KEY_ID: ${LITESTREAM_ACCESS_KEY_ID} + # LITESTREAM_SECRET_ACCESS_KEY: ${LITESTREAM_SECRET_ACCESS_KEY} + # LITESTREAM_ENDPOINT: ${LITESTREAM_ENDPOINT} + # LITESTREAM_REGION: ${LITESTREAM_REGION} + volumes: gittensory-data: # ollama-models: diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 264e691e9a..ded5644669 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -160,6 +160,9 @@ repository settings. The authoritative reference for all of these is `selfhost_queue_recovered`, `selfhost_job_dead`, `selfhost_cron_error`, `selfhost_shutdown`, …). - **Data + backup.** Everything is the single SQLite file on the `gittensory-data` volume (WAL mode). Back up by snapshotting the volume or copying the `.sqlite` file. Migrations are idempotent and re-checked at boot. + For **continuous, point-in-time backup**, enable the optional [Litestream](https://litestream.io) sidecar in + `docker-compose.yml` (copy `litestream.yml.example` → `litestream.yml`, set your bucket + credentials); it + streams every change to S3/B2/MinIO/R2. - **App-level metrics.** Enable `GITTENSORY_REVIEW_OPS=true` for the read-only gate-block anomaly scan and the bearer-gated `GET /v1/internal/ops/stats` aggregate. diff --git a/litestream.yml.example b/litestream.yml.example new file mode 100644 index 0000000000..dc4ae68b0c --- /dev/null +++ b/litestream.yml.example @@ -0,0 +1,14 @@ +# Litestream continuous-backup config for the self-host SQLite DB (SAMPLE — placeholders only). +# Streams /data/gittensory.sqlite to your object store so you can restore to any point in time. +# Copy to litestream.yml, edit the bucket/endpoint, and mount it at /etc/litestream.yml (see docker-compose.yml). +# Docs: https://litestream.io +dbs: + - path: /data/gittensory.sqlite + replicas: + - type: s3 + bucket: your-backup-bucket + path: gittensory + # For S3-compatible stores (Backblaze B2, MinIO, R2) set the endpoint; omit it for AWS S3. + endpoint: ${LITESTREAM_ENDPOINT} + region: ${LITESTREAM_REGION} + # Credentials come from the environment (LITESTREAM_ACCESS_KEY_ID / LITESTREAM_SECRET_ACCESS_KEY). From 5f13a40980fd717a9133e67f139d021c97da1d7b Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 23 Jun 2026 21:38:43 -0700 Subject: [PATCH 11/25] test(selfhost): raise patch coverage to ~99% + drop dead in-process queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The codecov 44% report was stale (head 46a7e00 — before the Tier 0/1/3 tests + the codecov.yml ignores). On the real HEAD, close the remaining gaps: - Remove src/selfhost/queue.ts (createInProcessQueue) + its test — dead code, superseded by the durable sqlite-queue (server.ts uses createSqliteQueue). - AI error paths: embed non-OK, anthropic non-OK, claude no-token/non-zero-exit/empty-output, codex empty, defaultSpawn missing-binary (error handler), extractCliText JSONL fallback. v8-ignore the 120s subprocess timeout (not unit-testable). - Queue: unparseable-payload dead-letter, sendBatch, default backoff reschedule, start()-idempotent + stop()-waits-for-in-flight. d1-adapter dump(). metrics multi-label sort. vectorize default topK. - Countable self-host coverage now 98.85% statements (47 self-host tests). Boot-entry + stubs stay codecov-ignored (covered by the Docker build+boot smoke test). --- src/selfhost/ai.ts | 1 + src/selfhost/queue.ts | 62 ------------------------- test/unit/selfhost-ai.test.ts | 38 +++++++++++++++ test/unit/selfhost-d1-adapter.test.ts | 4 ++ test/unit/selfhost-metrics.test.ts | 5 ++ test/unit/selfhost-queue.test.ts | 30 ------------ test/unit/selfhost-sqlite-queue.test.ts | 40 ++++++++++++++++ test/unit/selfhost-vectorize.test.ts | 5 +- 8 files changed, 92 insertions(+), 93 deletions(-) delete mode 100644 src/selfhost/queue.ts delete mode 100644 test/unit/selfhost-queue.test.ts diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index 6e69b414e3..280b5526f9 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -159,6 +159,7 @@ async function defaultSpawn(): Promise { const child = cp.spawn(cmd, args, { env: o.env as NodeJS.ProcessEnv, stdio }); let stdout = ""; const timer = setTimeout(() => { + /* v8 ignore next 2 */ // a 120s subprocess timeout is not unit-testable without a 2-minute wait child.kill("SIGKILL"); reject(new Error("subscription_cli_timeout")); }, o.timeoutMs); diff --git a/src/selfhost/queue.ts b/src/selfhost/queue.ts deleted file mode 100644 index a8afe48859..0000000000 --- a/src/selfhost/queue.ts +++ /dev/null @@ -1,62 +0,0 @@ -// Self-host in-process job queue (#980). Replaces the Cloudflare Queue (env.JOBS) on a single container: a -// `Queue`-shaped binding whose send() enqueues a JobMessage (honoring delaySeconds), and an async worker that -// drains FIFO and invokes the same processJob the Worker's queue() handler uses. Failures retry up to -// maxRetries then drop (logged), mirroring the Queues DLQ at small scale. (A Redis/BullMQ backend is a -// follow-up for multi-replica; the cron sweep is the backstop either way.) -import type { JobMessage } from "../types"; - -export interface SelfHostQueue { - /** The env.JOBS binding (send / sendBatch). */ - binding: Queue; - /** Resolve when the queue is empty (tests / graceful shutdown). */ - drain(): Promise; - size(): number; -} - -export function createInProcessQueue(consume: (message: JobMessage) => Promise, opts: { maxRetries?: number } = {}): SelfHostQueue { - const maxRetries = opts.maxRetries ?? 3; - const queue: Array<{ message: JobMessage; attempts: number }> = []; - let working = false; - - async function pump(): Promise { - if (working) return; - working = true; - try { - while (queue.length > 0) { - const item = queue.shift(); - if (!item) break; - try { - await consume(item.message); - } catch (error) { - if (item.attempts + 1 < maxRetries) queue.push({ message: item.message, attempts: item.attempts + 1 }); - else console.error(JSON.stringify({ level: "error", event: "inproc_job_dropped", attempts: item.attempts + 1, error: error instanceof Error ? error.message : "unknown error" })); - } - } - } finally { - working = false; - } - } - - const send = (message: JobMessage, options?: { delaySeconds?: number }): Promise => { - const delayMs = (options?.delaySeconds ?? 0) * 1000; - if (delayMs > 0) { - setTimeout(() => { - queue.push({ message, attempts: 0 }); - void pump(); - }, delayMs); - } else { - queue.push({ message, attempts: 0 }); - void pump(); - } - return Promise.resolve(); - }; - - const binding = { - send, - sendBatch: async (messages: Iterable<{ body: JobMessage }>) => { - for (const m of messages) await send(m.body); - }, - } as unknown as Queue; - - return { binding, drain: pump, size: () => queue.length }; -} diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index 250b442656..418793b916 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -52,6 +52,11 @@ describe("createOpenAiCompatibleAi (#979)", () => { expect(url).toBe("http://o/v1/embeddings"); expect(out).toEqual({ data: [[0.1, 0.2], [0.3, 0.4]] }); }); + + it("throws on a non-OK embeddings response", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response("e", { status: 502 }))); + await expect(createOpenAiCompatibleAi({ baseUrl: "http://x/v1" }).run("m", { text: ["a"] })).rejects.toThrow(/ai_embed_http_502/); + }); }); describe("createSelfHostAi — provider selection", () => { @@ -94,6 +99,11 @@ describe("createAnthropicAi (#979 native BYOK)", () => { expect(sent?.body.model).toBe("claude-sonnet-4-6"); // configured wins over the @cf id expect(sent?.body.messages).toEqual([{ role: "user", content: "go" }]); }); + + it("throws on a non-OK response", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response("e", { status: 429 }))); + await expect(createAnthropicAi({ apiKey: "k" }).run("m", { prompt: "x" })).rejects.toThrow(/anthropic_http_429/); + }); }); describe("createChainAi (fallback)", () => { @@ -156,4 +166,32 @@ describe("subscription CLI helpers + fail-safe", () => { process.env.PATH = origPath; } }); + + it("Claude Code throws on no-token / non-zero exit / empty output", async () => { + await expect(createClaudeCodeAi({}).run("m", { prompt: "x" })).rejects.toThrow(/claude_code_no_oauth_token/); + const exit1: StubSpawn = async () => ({ stdout: "", code: 1 }); + await expect(createClaudeCodeAi({ CLAUDE_CODE_OAUTH_TOKEN: "t" }, exit1).run("m", { prompt: "x" })).rejects.toThrow(/claude_code_exit_1/); + const empty: StubSpawn = async () => ({ stdout: "", code: 0 }); + await expect(createClaudeCodeAi({ CLAUDE_CODE_OAUTH_TOKEN: "t" }, empty).run("m", { prompt: "x" })).rejects.toThrow(/claude_code_empty_output/); + }); + + it("Codex throws on empty output", async () => { + const empty: StubSpawn = async () => ({ stdout: "", code: 0 }); + await expect(createCodexAi({}, empty).run("gpt-5", { prompt: "x" })).rejects.toThrow(/codex_empty_output/); + }); + + it("defaultSpawn rejects when the CLI binary is missing (error handler)", async () => { + const origPath = process.env.PATH; + process.env.PATH = "/nonexistent-gittensory-empty"; + try { + await expect(createCodexAi({ ...process.env }).run("gpt-5", { prompt: "x" })).rejects.toThrow(); + } finally { + process.env.PATH = origPath; + } + }); + + it("extractCliText falls back to the last JSON line (JSONL) and is empty when none parse", () => { + expect(extractCliText('not json\n{"result":"x"}')).toBe("x"); + expect(extractCliText("not json\nstill not json")).toBe(""); + }); }); diff --git a/test/unit/selfhost-d1-adapter.test.ts b/test/unit/selfhost-d1-adapter.test.ts index f2048ee838..a4f0ab584a 100644 --- a/test/unit/selfhost-d1-adapter.test.ts +++ b/test/unit/selfhost-d1-adapter.test.ts @@ -51,4 +51,8 @@ describe("createD1Adapter (#980 self-host D1-over-SQLite)", () => { ).rejects.toThrow(); expect((await d1.prepare("SELECT count(*) AS n FROM t").first<{ n: number }>())?.n).toBe(1); // "ok" rolled back }); + + it("dump() returns an ArrayBuffer (D1 surface completeness)", async () => { + expect(await makeD1().dump()).toBeInstanceOf(ArrayBuffer); + }); }); diff --git a/test/unit/selfhost-metrics.test.ts b/test/unit/selfhost-metrics.test.ts index 9a21f9c29d..ec4d462a2a 100644 --- a/test/unit/selfhost-metrics.test.ts +++ b/test/unit/selfhost-metrics.test.ts @@ -15,6 +15,11 @@ describe("metrics registry (#982)", () => { expect(renderMetrics()).toContain('h_total{status="ok"} 1'); }); + it("sorts multiple labels deterministically", () => { + incr("m_total", { b: "2", a: "1" }); + expect(renderMetrics()).toContain('m_total{a="1",b="2"} 1'); + }); + it("gauges sample at scrape time", () => { let v = 5; gauge("g", () => v); diff --git a/test/unit/selfhost-queue.test.ts b/test/unit/selfhost-queue.test.ts deleted file mode 100644 index e82eefbbe7..0000000000 --- a/test/unit/selfhost-queue.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { createInProcessQueue } from "../../src/selfhost/queue"; -import type { JobMessage } from "../../src/types"; - -describe("createInProcessQueue (#980 self-host job queue)", () => { - it("send() enqueues; the worker drains FIFO through the consumer", async () => { - const seen: JobMessage[] = []; - const q = createInProcessQueue(async (m) => void seen.push(m)); - await q.binding.send({ type: "a" } as unknown as JobMessage); - await q.binding.send({ type: "b" } as unknown as JobMessage); - await q.drain(); - expect(seen).toEqual([{ type: "a" }, { type: "b" }]); - expect(q.size()).toBe(0); - }); - - it("retries a failing job up to maxRetries, then drops it (never throws)", async () => { - let calls = 0; - const q = createInProcessQueue( - async () => { - calls += 1; - throw new Error("boom"); - }, - { maxRetries: 2 }, - ); - await q.binding.send({ type: "x" } as unknown as JobMessage); - await q.drain(); - expect(calls).toBe(2); // first attempt + one retry, then dropped - expect(q.size()).toBe(0); - }); -}); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index e03155ed97..b0d4260b65 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -70,4 +70,44 @@ describe("createSqliteQueue (durable #980)", () => { await fresh.drain(); expect(seen).toEqual(["stuck"]); }); + + it("dead-letters an unparseable payload", async () => { + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + driver.query("INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at) VALUES ('not-json','pending',0,0,0)", []); + await q.drain(); + expect(q.deadCount()).toBe(1); + }); + + it("sendBatch enqueues all; default backoff reschedules a failure into the future", async () => { + const seen: string[] = []; + const q = createSqliteQueue(makeDriver(), async (m) => void seen.push(typeOf(m))); + await q.binding.sendBatch([{ body: msg("a") }, { body: msg("b") }]); + await q.drain(); + expect(seen.sort()).toEqual(["a", "b"]); + + let calls = 0; + const q2 = createSqliteQueue(makeDriver(), async () => { + calls += 1; + throw new Error("x"); + }, { maxRetries: 5 }); // default backoff (~2s) → not re-claimed this drain + await q2.binding.send(msg("f")); + await q2.drain(); + expect(calls).toBe(1); + expect(q2.size()).toBe(1); + }); + + it("start() is idempotent and stop() waits for an in-flight pump", async () => { + let done = false; + const q = createSqliteQueue(makeDriver(), async () => { + await new Promise((r) => setTimeout(r, 40)); + done = true; + }, { pollIntervalMs: 5 }); + q.start(); + q.start(); // idempotent + await q.binding.send(msg("slow")); + await new Promise((r) => setTimeout(r, 12)); // let the tick claim it + enter the slow consume + await q.stop(); // waits for the in-flight consume to finish + expect(done).toBe(true); + }); }); diff --git a/test/unit/selfhost-vectorize.test.ts b/test/unit/selfhost-vectorize.test.ts index f744a6ed4d..3ceb0107dc 100644 --- a/test/unit/selfhost-vectorize.test.ts +++ b/test/unit/selfhost-vectorize.test.ts @@ -28,7 +28,7 @@ describe("createSqliteVectorize (#979 local RAG)", () => { expect(res.matches[0]?.metadata?.path).toBe("a.ts"); }); - it("scopes results by namespace", async () => { + it("scopes results by namespace and defaults topK when omitted", async () => { const v = makeVectorize(); await v.upsert([ { id: "x", values: [1, 0], namespace: "n1" }, @@ -36,6 +36,9 @@ describe("createSqliteVectorize (#979 local RAG)", () => { ]); const res = await v.query([1, 0], { topK: 10, namespace: "n1" }); expect(res.matches.map((m) => m.id)).toEqual(["x"]); + // topK omitted → default applies (no throw, returns the namespace's match) + const res2 = await v.query([1, 0], { namespace: "n1" }); + expect(res2.matches.map((m) => m.id)).toEqual(["x"]); }); it("upsert overwrites by id; deleteByIds removes", async () => { From 25dc57dd6a736a2a5f735cfc1da0b7347645713d Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 23 Jun 2026 21:59:43 -0700 Subject: [PATCH 12/25] =?UTF-8?q?feat(selfhost):=20Tier=202=20=E2=80=94=20?= =?UTF-8?q?Postgres=20backend=20+=20Redis=20rate=20limiter=20(multi-instan?= =?UTF-8?q?ce)=20(#977)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scale-out support, validated against a REAL Postgres + Redis (caught 2 bugs only real infra surfaces): - Postgres data backend (DATABASE_URL=postgres://…): pg-dialect.ts translates the bounded SQLite-isms the codebase uses (?→$n quote-aware, INSERT OR REPLACE/IGNORE→ON CONFLICT, datetime/strftime/CURRENT_TIMESTAMP →text-returning to match TEXT-column semantics, json_extract→jsonb ->>); pg-adapter.ts implements the D1 surface async over node-postgres (int8→number). All 56 migrations apply; the translated query paths run. - Postgres queue (pg-queue.ts): durable jobs with FOR UPDATE SKIP LOCKED → multiple replicas claim safely. - Redis rate limiter (redis-ratelimit.ts): the RateLimiter DO surface (idFromName/get/fetch) over a Redis fixed-window; enforceRateLimit works unchanged, shared across instances (x-ratelimit-* headers verified). - server.ts picks the backend by env (SQLite default; DATABASE_URL→PG; REDIS_URL→limiter). readiness + metrics made backend-agnostic/async. RAG (SQLite vector store) is SQLite-only for now (degrades on PG). - Validation: real-PG integration test (test/integration/selfhost-pg.ts, gated on PG_TEST_URL + run in CI via a postgres service), pg-dialect + redis unit tests. 53 self-host unit tests, ~98.5% countable coverage. Bugs fixed: DO-stub fetch(url,init) vs Request; int8 COUNT→string. SQLite + PG+Redis both boot clean. - docker-compose: optional postgres + redis services; .env.example + docs scale-out (§7). Sample config only. --- .env.example | 3 + .github/workflows/selfhost.yml | 12 ++ codecov.yml | 4 + docker-compose.yml | 29 +++ docs/self-hosting.md | 21 +- package-lock.json | 234 +++++++++++++++++++++ package.json | 3 + src/selfhost/health.ts | 15 +- src/selfhost/metrics.ts | 11 +- src/selfhost/pg-adapter.ts | 85 ++++++++ src/selfhost/pg-dialect.ts | 81 +++++++ src/selfhost/pg-queue.ts | 158 ++++++++++++++ src/selfhost/redis-ratelimit.ts | 44 ++++ src/server.ts | 132 ++++++++---- test/integration/selfhost-pg.test.ts | 58 +++++ test/unit/selfhost-health.test.ts | 11 +- test/unit/selfhost-metrics.test.ts | 22 +- test/unit/selfhost-pg-dialect.test.ts | 32 +++ test/unit/selfhost-redis-ratelimit.test.ts | 46 ++++ 19 files changed, 934 insertions(+), 67 deletions(-) create mode 100644 src/selfhost/pg-adapter.ts create mode 100644 src/selfhost/pg-dialect.ts create mode 100644 src/selfhost/pg-queue.ts create mode 100644 src/selfhost/redis-ratelimit.ts create mode 100644 test/integration/selfhost-pg.test.ts create mode 100644 test/unit/selfhost-pg-dialect.test.ts create mode 100644 test/unit/selfhost-redis-ratelimit.test.ts diff --git a/.env.example b/.env.example index f886e6fb27..992b9546f3 100644 --- a/.env.example +++ b/.env.example @@ -107,6 +107,9 @@ GITTENSORY_REVIEW_DRAFT=false # PORT=8787 # DATABASE_PATH=/data/gittensory.sqlite # SQLite file on the mounted data volume; all 56 migrations auto-apply +# DATABASE_URL= # set to postgres://user:pw@host:5432/db to use Postgres instead of +# # SQLite (shared DB → multi-instance). Overrides DATABASE_PATH. +# REDIS_URL= # set to redis://host:6379 for a distributed rate limiter (else off) # MIGRATIONS_DIR=/app/migrations # CRON_INTERVAL_MS=120000 # maintain/sweep + sync cadence (default ~2 min) diff --git a/.github/workflows/selfhost.yml b/.github/workflows/selfhost.yml index b3fa41ad76..2f9b1f9ac4 100644 --- a/.github/workflows/selfhost.yml +++ b/.github/workflows/selfhost.yml @@ -33,6 +33,16 @@ jobs: name: build + boot smoke test runs-on: ubuntu-latest timeout-minutes: 20 + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_PASSWORD: devpw + POSTGRES_DB: gittensory + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" --health-interval 5s --health-timeout 5s --health-retries 10 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -42,6 +52,8 @@ jobs: run: npm ci --ignore-scripts - name: Self-host unit tests run: npx vitest run test/unit/selfhost-*.test.ts + - name: Postgres integration test (real PG) + run: PG_TEST_URL=postgres://postgres:devpw@localhost:5432/gittensory npx vitest run test/integration/selfhost-pg.test.ts - name: Typecheck run: npx tsc --noEmit - name: Build the self-host bundle diff --git a/codecov.yml b/codecov.yml index 3b4fbe6ab9..7ff141dc21 100644 --- a/codecov.yml +++ b/codecov.yml @@ -38,3 +38,7 @@ ignore: - "src/server.ts" - "src/selfhost/cf-workers-shim.ts" - "src/selfhost/stubs/**" + # Postgres runtime adapters: validated by the real-Postgres integration test (test/integration/selfhost-pg.ts, + # gated on PG_TEST_URL) + the real-PG boot. The dialect translation itself IS unit-tested (pg-dialect.ts). + - "src/selfhost/pg-adapter.ts" + - "src/selfhost/pg-queue.ts" diff --git a/docker-compose.yml b/docker-compose.yml index 6326ca9d42..199d848f34 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,10 @@ services: environment: PORT: "8787" DATABASE_PATH: /data/gittensory.sqlite + # Scale out: uncomment the postgres + redis services below, then point at them here (a shared Postgres + + # Redis lets you run multiple replicas of this service behind a load balancer): + # DATABASE_URL: postgres://gittensory:CHANGEME@postgres:5432/gittensory + # REDIS_URL: redis://redis:6379 # Point at the Ollama service below to enable local AI review (uncomment the ollama service too): # AI_PROVIDER: ollama # AI_BASE_URL: http://ollama:11434/v1 @@ -52,6 +56,31 @@ services: # LITESTREAM_ENDPOINT: ${LITESTREAM_ENDPOINT} # LITESTREAM_REGION: ${LITESTREAM_REGION} + # Optional: Postgres backend (shared DB → multi-instance). Uncomment, set DATABASE_URL above, and add + # `depends_on: [postgres]` to the gittensory service. + # postgres: + # image: postgres:16-alpine + # restart: unless-stopped + # environment: + # POSTGRES_USER: gittensory + # POSTGRES_PASSWORD: CHANGEME # SAMPLE — set your own + # POSTGRES_DB: gittensory + # volumes: + # - gittensory-pg:/var/lib/postgresql/data + # healthcheck: + # test: ["CMD-SHELL", "pg_isready -U gittensory"] + # interval: 10s + # retries: 5 + + # Optional: Redis (distributed rate limiter). Uncomment + set REDIS_URL above. + # redis: + # image: redis:7-alpine + # restart: unless-stopped + # volumes: + # - gittensory-redis:/data + volumes: gittensory-data: # ollama-models: + # gittensory-pg: + # gittensory-redis: diff --git a/docs/self-hosting.md b/docs/self-hosting.md index ded5644669..0c7f277af7 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -168,13 +168,28 @@ repository settings. The authoritative reference for all of these is --- -## 7. What is not on self-host +## 7. Scaling out — Postgres + Redis (multi-instance) + +The SQLite default is ideal for a single instance. To run **multiple replicas** behind a load balancer, switch +to a shared Postgres + Redis: + +- **`DATABASE_URL=postgres://user:pw@host:5432/db`** — uses Postgres instead of SQLite. The same 56 migrations + apply (translated to Postgres at startup), and the job queue moves to Postgres with `FOR UPDATE SKIP LOCKED` + claiming, so replicas never double-process a job. +- **`REDIS_URL=redis://host:6379`** — a shared fixed-window rate limiter across all replicas. + +Uncomment the `postgres` + `redis` services in `docker-compose.yml`, set the two URLs on the app service, and +scale (`docker compose up --scale gittensory=3`). Postgres is **beta**: the migrations + the exercised query +paths are validated against a real Postgres, but report any dialect edge cases. RAG (the SQLite vector store) +is **not** available on the Postgres backend yet — it degrades to no-context. + +## 8. What is not on self-host These are Cloudflare-platform features; they degrade cleanly and the core reviewer is unaffected: - **Visual PR capture** (Browser Rendering binding) — off; reviews run text-only. - **The `/mcp` server** (Durable-Object-backed Agents SDK) — returns `501`. The deterministic API + review path is unaffected; a native MCP-on-Node port is a follow-up. -- **Distributed rate limiting** (RateLimiter Durable Object) — absent, so the limiter is a no-op. Put your - reverse proxy / WAF in front if you expose the endpoint publicly. +- **Distributed rate limiting** (RateLimiter Durable Object) — off by default; set `REDIS_URL` for a + Redis-backed fixed-window limiter (see §7). Otherwise put a reverse proxy / WAF in front. - **Vectorize-backed RAG** and **R2 audit storage** — inert unless you wire equivalent backends. diff --git a/package-lock.json b/package-lock.json index 1c1ce319c3..f43fadd4a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,8 @@ "agents": "^0.16.2", "drizzle-orm": "^0.45.2", "hono": "^4.12.26", + "ioredis": "^5.11.1", + "pg": "^8.22.0", "yaml": "^2.9.0", "zod": "^4.4.3" }, @@ -28,6 +30,7 @@ "@cloudflare/vitest-pool-workers": "^0.16.17", "@tktco/node-actionlint": "^1.6.0", "@types/node": "^24.13.2", + "@types/pg": "^8.20.0", "@types/pixelmatch": "^5.2.6", "@types/pngjs": "^6.0.5", "@vitest/coverage-v8": "^4.1.9", @@ -2512,6 +2515,12 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@ioredis/commands": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", + "integrity": "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==", + "license": "MIT" + }, "node_modules/@isaacs/fs-minipass": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", @@ -5925,6 +5934,18 @@ "undici-types": "~7.18.0" } }, + "node_modules/@types/pg": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.20.0.tgz", + "integrity": "sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, "node_modules/@types/pixelmatch": { "version": "5.2.6", "resolved": "https://registry.npmjs.org/@types/pixelmatch/-/pixelmatch-5.2.6.tgz", @@ -7330,6 +7351,15 @@ "node": ">=6" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz", + "integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/cmdk": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/cmdk/-/cmdk-1.1.1.tgz", @@ -7874,6 +7904,15 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -9928,6 +9967,28 @@ "node": ">=12" } }, + "node_modules/ioredis": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.11.1.tgz", + "integrity": "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.10.0", + "cluster-key-slot": "1.1.1", + "debug": "4.4.3", + "denque": "2.1.0", + "redis-errors": "1.2.0", + "redis-parser": "3.0.0", + "standard-as-callback": "2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ip-address": { "version": "10.2.0", "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", @@ -11767,6 +11828,95 @@ "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", "license": "MIT" }, + "node_modules/pg": { + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz", + "integrity": "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.0", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.15.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz", + "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz", + "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.15.0.tgz", + "integrity": "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -12046,6 +12196,45 @@ "dev": true, "license": "MIT" }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prebuild-install": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", @@ -12656,6 +12845,27 @@ "decimal.js-light": "^2.4.1" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -13270,6 +13480,15 @@ "source-map": "^0.6.0" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sprintf-js": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", @@ -13301,6 +13520,12 @@ "dev": true, "license": "MIT" }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -14644,6 +14869,15 @@ "dev": true, "license": "MIT" }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index 2d1fe9a813..7f34039a5e 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,8 @@ "agents": "^0.16.2", "drizzle-orm": "^0.45.2", "hono": "^4.12.26", + "ioredis": "^5.11.1", + "pg": "^8.22.0", "yaml": "^2.9.0", "zod": "^4.4.3" }, @@ -75,6 +77,7 @@ "@cloudflare/vitest-pool-workers": "^0.16.17", "@tktco/node-actionlint": "^1.6.0", "@types/node": "^24.13.2", + "@types/pg": "^8.20.0", "@types/pixelmatch": "^5.2.6", "@types/pngjs": "^6.0.5", "@vitest/coverage-v8": "^4.1.9", diff --git a/src/selfhost/health.ts b/src/selfhost/health.ts index badde913c4..75aacb06bb 100644 --- a/src/selfhost/health.ts +++ b/src/selfhost/health.ts @@ -1,6 +1,6 @@ // Self-host liveness/readiness probes (#982). Liveness is binding-free (the process is up); readiness asserts // the things a request actually depends on — the DB answers and the schema migrations have been applied. -import type { SqliteDriver } from "./d1-adapter"; +// Backend-agnostic: runs through the D1 surface, so it works on both the SQLite and Postgres adapters. export interface Readiness { ok: boolean; @@ -8,19 +8,20 @@ export interface Readiness { } /** Readiness: the DB answers a trivial query and the migrations table shows applied rows. */ -export function readiness(driver: SqliteDriver): Readiness { - let db = false; +export async function readiness(db: D1Database): Promise { + let dbOk = false; let migrations = false; try { - driver.query("SELECT 1", []); - db = true; + await db.prepare("SELECT 1 AS one").first(); + dbOk = true; } catch { /* db down */ } try { - migrations = Number((driver.query("SELECT COUNT(*) AS c FROM _selfhost_migrations", []).rows[0] as { c: number }).c) > 0; + const row = await db.prepare("SELECT COUNT(*) AS c FROM _selfhost_migrations").first<{ c: number }>(); + migrations = Number(row?.c ?? 0) > 0; } catch { /* migrations table missing */ } - return { ok: db && migrations, checks: { db, migrations } }; + return { ok: dbOk && migrations, checks: { db: dbOk, migrations } }; } diff --git a/src/selfhost/metrics.ts b/src/selfhost/metrics.ts index 053daa5dab..deae0208a6 100644 --- a/src/selfhost/metrics.ts +++ b/src/selfhost/metrics.ts @@ -3,9 +3,10 @@ // callback, e.g. live queue depth). Rendered at GET /metrics. No deps, no cardinality explosion: callers use // a small fixed label set. type Labels = Record; +type GaugeSample = () => number | Promise; const counters = new Map(); -const gauges = new Map number>(); +const gauges = new Map(); function seriesKey(name: string, labels?: Labels): string { if (!labels || Object.keys(labels).length === 0) return name; @@ -22,18 +23,18 @@ export function incr(name: string, labels?: Labels, by = 1): void { counters.set(k, (counters.get(k) ?? 0) + by); } -/** Register a gauge sampled at scrape time. Re-registering replaces the sampler. */ -export function gauge(name: string, sample: () => number): void { +/** Register a gauge sampled at scrape time (sync or async). Re-registering replaces the sampler. */ +export function gauge(name: string, sample: GaugeSample): void { gauges.set(name, sample); } /** Render the registry in Prometheus text exposition format. */ -export function renderMetrics(): string { +export async function renderMetrics(): Promise { const lines: string[] = []; for (const [k, v] of counters) lines.push(`${k} ${v}`); for (const [name, sample] of gauges) { try { - lines.push(`${name} ${sample()}`); + lines.push(`${name} ${await sample()}`); } catch { /* a failing sampler must not break the scrape */ } diff --git a/src/selfhost/pg-adapter.ts b/src/selfhost/pg-adapter.ts new file mode 100644 index 0000000000..f5d6d1104d --- /dev/null +++ b/src/selfhost/pg-adapter.ts @@ -0,0 +1,85 @@ +// Postgres-backed D1Database for the self-host Postgres backend (#977). Implements the same D1 surface the +// app + drizzle-orm/d1 use (prepare/bind/all/first/run/raw + batch + exec), translating each SQLite query to +// Postgres (pg-dialect.ts) and running it via node-postgres. A shared Postgres DB makes multi-instance +// self-host possible (vs the single-file SQLite default). +import type { Pool, PoolClient } from "pg"; +import { translateDdl, translateSql } from "./pg-dialect"; + +type Row = Record; +type Runner = Pool | PoolClient; + +class PgStatement { + constructor( + private readonly pool: Pool, + private readonly sql: string, + private readonly params: unknown[] = [], + ) {} + + bind(...params: unknown[]): PgStatement { + return new PgStatement(this.pool, this.sql, params); + } + + private async exec(runner: Runner = this.pool): Promise<{ rows: Row[]; rowCount: number }> { + const res = await runner.query(translateSql(this.sql), this.params as unknown[]); + return { rows: res.rows as Row[], rowCount: res.rowCount ?? 0 }; + } + + async all(): Promise<{ results: T[]; success: true; meta: Record }> { + const { rows, rowCount } = await this.exec(); + return { results: rows as T[], success: true, meta: { rows_read: rowCount, changes: rowCount } }; + } + + async first(colName?: string): Promise { + const { rows } = await this.exec(); + const row = rows[0]; + if (!row) return null; + return (colName ? row[colName] : row) as T; + } + + async run(): Promise<{ success: true; meta: Record }> { + const { rowCount } = await this.exec(); + return { success: true, meta: { changes: rowCount, last_row_id: 0, rows_written: rowCount } }; + } + + async raw(): Promise { + const { rows } = await this.exec(); + return rows.map((r) => Object.values(r)) as T[]; + } + + /** Run this statement on a specific client (used by batch's transaction). */ + async runOn(client: PoolClient): Promise<{ results: Row[]; success: true; meta: Record }> { + const { rows, rowCount } = await this.exec(client); + return { results: rows, success: true, meta: { changes: rowCount } }; + } +} + +export function createPgAdapter(pool: Pool): D1Database { + const adapter = { + prepare: (sql: string) => new PgStatement(pool, sql), + async batch(statements: PgStatement[]) { + const client = await pool.connect(); + try { + await client.query("BEGIN"); + const out: unknown[] = []; + for (const st of statements) out.push(await st.runOn(client)); + await client.query("COMMIT"); + return out; + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } finally { + client.release(); + } + }, + async exec(sql: string) { + // Migrations: no placeholders; translate the DDL functions and run (node-postgres runs the multi-statement + // string in one simple query). + await pool.query(translateDdl(sql)); + return { count: (sql.match(/;/g) ?? []).length || 1, duration: 0 }; + }, + async dump() { + return new ArrayBuffer(0); // unused; present for D1 surface completeness + }, + }; + return adapter as unknown as D1Database; +} diff --git a/src/selfhost/pg-dialect.ts b/src/selfhost/pg-dialect.ts new file mode 100644 index 0000000000..04adf37f0f --- /dev/null +++ b/src/selfhost/pg-dialect.ts @@ -0,0 +1,81 @@ +// SQLite → Postgres SQL dialect translation for the self-host Postgres backend (#977). gittensory's core and +// drizzle-orm/d1 emit SQLite-dialect SQL; this translates the bounded set of SQLite-isms the codebase uses +// (placeholders + a handful of scalar functions + INSERT OR REPLACE/IGNORE) so the SAME queries run on +// Postgres. The timestamp columns are TEXT (ISO strings written by the app), so the datetime/CURRENT_TIMESTAMP +// translations return TEXT in SQLite's format to preserve the existing text-comparison semantics. Validated +// against a real Postgres (all 56 migrations + the runtime query paths). + +// INSERT OR REPLACE needs an explicit conflict target on Postgres; map the (few) tables that use it to their PK. +const REPLACE_CONFLICT_KEYS: Record = { + system_flags: ["key"], + tunables_overrides: ["project"], + tunables_overrides_shadow: ["project"], +}; + +/** Replace `?` placeholders with `$1,$2,…`, skipping any `?` inside single-quoted string literals. */ +export function toNumberedPlaceholders(sql: string): string { + let out = ""; + let n = 0; + let inString = false; + for (const ch of sql) { + if (ch === "'") inString = !inString; + if (ch === "?" && !inString) { + n += 1; + out += `$${n}`; + } else { + out += ch; + } + } + return out; +} + +/** Translate the SQLite scalar functions the codebase uses to Postgres equivalents. */ +export function translateFunctions(sql: string): string { + return ( + sql + // ISO-now (the DEFAULT on TEXT timestamp columns + nowIso parity) + .replace(/strftime\(\s*'%Y-%m-%dT%H:%M:%fZ'\s*,\s*'now'\s*\)/gi, `to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`) + // week / month buckets (stats) + .replace(/strftime\(\s*'%Y-W%W'\s*,\s*([^)]+?)\s*\)/gi, `to_char(($1)::timestamptz, 'YYYY"-W"WW')`) + .replace(/strftime\(\s*'%Y-%m'\s*,\s*([^)]+?)\s*\)/gi, `to_char(($1)::timestamptz, 'YYYY-MM')`) + // datetime('now', ) → TEXT in SQLite's 'YYYY-MM-DD HH:MM:SS' format (TEXT columns compared) + .replace(/datetime\(\s*'now'\s*,\s*([^)]+?)\s*\)/gi, `to_char(now() + ($1)::interval, 'YYYY-MM-DD HH24:MI:SS')`) + .replace(/datetime\(\s*'now'\s*\)/gi, `to_char(now(), 'YYYY-MM-DD HH24:MI:SS')`) + // CURRENT_TIMESTAMP → SQLite's TEXT format (the columns are TEXT) + .replace(/CURRENT_TIMESTAMP/gi, `to_char(now(), 'YYYY-MM-DD HH24:MI:SS')`) + // json_extract(col, '$.key') → (col::jsonb ->> 'key') (single-level paths — all the codebase uses) + .replace(/json_extract\(\s*([^,]+?)\s*,\s*'\$\.([A-Za-z0-9_]+)'\s*\)/gi, `(($1)::jsonb ->> '$2')`) + ); +} + +/** Translate INSERT OR REPLACE / INSERT OR IGNORE to Postgres ON CONFLICT. */ +export function translateInsertOr(sql: string): string { + if (/^\s*INSERT\s+OR\s+IGNORE\s+INTO/i.test(sql)) { + return `${sql.replace(/^(\s*)INSERT\s+OR\s+IGNORE\s+INTO/i, "$1INSERT INTO")} ON CONFLICT DO NOTHING`; + } + const m = /^\s*INSERT\s+OR\s+REPLACE\s+INTO\s+([A-Za-z0-9_]+)\s*\(([^)]+)\)/i.exec(sql); + if (m) { + const table = m[1] as string; + const cols = (m[2] as string).split(",").map((c) => c.trim()); + const pk = REPLACE_CONFLICT_KEYS[table]; + if (!pk) throw new Error(`pg_dialect: INSERT OR REPLACE into '${table}' has no known conflict key`); + const updates = cols + .filter((c) => !pk.includes(c)) + .map((c) => `${c}=excluded.${c}`) + .join(", "); + const base = sql.replace(/^(\s*)INSERT\s+OR\s+REPLACE\s+INTO/i, "$1INSERT INTO"); + return `${base} ON CONFLICT (${pk.join(", ")}) DO UPDATE SET ${updates}`; + } + return sql; +} + +/** Translate a runtime query (SQLite → Postgres). */ +export function translateSql(sql: string): string { + return toNumberedPlaceholders(translateFunctions(translateInsertOr(sql))); +} + +/** Translate a DDL statement (migrations). Column types (TEXT/INTEGER/REAL) are PG-native; only the SQLite + * default expressions need translating. No `?` placeholders in DDL. */ +export function translateDdl(sql: string): string { + return translateFunctions(sql); +} diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts new file mode 100644 index 0000000000..b25b52e85b --- /dev/null +++ b/src/selfhost/pg-queue.ts @@ -0,0 +1,158 @@ +// Postgres-backed durable job queue for multi-instance self-host (#977). Same contract as the SQLite queue +// (persist → restart re-claims, backoff retries, dead-letter) but uses `FOR UPDATE SKIP LOCKED` so multiple +// app instances sharing one Postgres can claim jobs concurrently without double-processing. size()/deadCount() +// are async (the metrics gauges accept async samplers). +import type { Pool } from "pg"; +import { incr } from "./metrics"; +import type { JobMessage } from "../types"; + +const TABLE = "_selfhost_jobs"; +const DDL = ` +CREATE TABLE IF NOT EXISTS ${TABLE} ( + id BIGSERIAL PRIMARY KEY, + payload TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + attempts INTEGER NOT NULL DEFAULT 0, + run_after BIGINT NOT NULL DEFAULT 0, + created_at BIGINT NOT NULL, + last_error TEXT +); +CREATE INDEX IF NOT EXISTS ${TABLE}_claim ON ${TABLE}(status, run_after);`; + +export interface PgDurableQueue { + binding: Queue; + init(): Promise; + start(): void; + stop(): Promise; + drain(): Promise; + size(): Promise; + deadCount(): Promise; +} + +interface JobRow { + id: string; + payload: string; + attempts: number; +} + +export interface PgQueueOptions { + maxRetries?: number; + pollIntervalMs?: number; + backoffMs?: (attempt: number) => number; +} + +export function createPgQueue(pool: Pool, consume: (message: JobMessage) => Promise, opts: PgQueueOptions = {}): PgDurableQueue { + const maxRetries = opts.maxRetries ?? 5; + const pollIntervalMs = opts.pollIntervalMs ?? 1000; + const backoff = opts.backoffMs ?? ((attempt: number) => Math.min(60_000, 1000 * 2 ** attempt)); + + let running = false; + let pumping = false; + let timer: ReturnType | null = null; + + async function init(): Promise { + await pool.query(DDL); + const recovered = (await pool.query(`UPDATE ${TABLE} SET status='pending' WHERE status='processing'`)).rowCount ?? 0; + if (recovered) console.log(JSON.stringify({ event: "selfhost_queue_recovered", count: recovered })); + } + + async function enqueue(message: JobMessage, delaySeconds: number): Promise { + const now = Date.now(); + await pool.query(`INSERT INTO ${TABLE} (payload, status, attempts, run_after, created_at) VALUES ($1,'pending',0,$2,$3)`, [JSON.stringify(message), now + delaySeconds * 1000, now]); + incr("gittensory_jobs_enqueued_total"); + void pump(); + } + + async function claimNext(): Promise { + // Atomic, multi-instance-safe: lock + claim one due job, skipping rows another instance already locked. + const res = await pool.query( + `UPDATE ${TABLE} SET status='processing' + WHERE id = (SELECT id FROM ${TABLE} WHERE status='pending' AND run_after<=$1 ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1) + RETURNING id, payload, attempts`, + [Date.now()], + ); + return (res.rows[0] as JobRow | undefined) ?? null; + } + + async function processOne(): Promise { + const job = await claimNext(); + if (!job) return false; + let message: JobMessage; + try { + message = JSON.parse(job.payload) as JobMessage; + } catch { + await pool.query(`UPDATE ${TABLE} SET status='dead', last_error='unparseable payload' WHERE id=$1`, [job.id]); + incr("gittensory_jobs_dead_total"); + return true; + } + try { + await consume(message); + await pool.query(`DELETE FROM ${TABLE} WHERE id=$1`, [job.id]); + incr("gittensory_jobs_processed_total"); + } catch (error) { + const attempts = Number(job.attempts) + 1; + const errMsg = error instanceof Error ? error.message : "unknown error"; + incr("gittensory_jobs_failed_total"); + if (attempts >= maxRetries) { + await pool.query(`UPDATE ${TABLE} SET status='dead', attempts=$1, last_error=$2 WHERE id=$3`, [attempts, errMsg, job.id]); + incr("gittensory_jobs_dead_total"); + console.error(JSON.stringify({ level: "error", event: "selfhost_job_dead", id: job.id, attempts, error: errMsg })); + } else { + await pool.query(`UPDATE ${TABLE} SET status='pending', attempts=$1, run_after=$2, last_error=$3 WHERE id=$4`, [attempts, Date.now() + backoff(attempts), errMsg, job.id]); + } + } + return true; + } + + async function pump(): Promise { + if (pumping) return; + pumping = true; + try { + while (await processOne()) { + /* drain due jobs */ + } + } finally { + pumping = false; + } + } + + const binding = { + async send(message: JobMessage, options?: { delaySeconds?: number }): Promise { + await enqueue(message, options?.delaySeconds ?? 0); + }, + async sendBatch(messages: Iterable<{ body: JobMessage; delaySeconds?: number }>): Promise { + for (const m of messages) await enqueue(m.body, m.delaySeconds ?? 0); + }, + } as unknown as Queue; + + return { + binding, + init, + start() { + if (running) return; + running = true; + const tick = (): void => { + if (!running) return; + void pump().finally(() => { + if (running) timer = setTimeout(tick, pollIntervalMs); + }); + }; + tick(); + }, + async stop() { + running = false; + if (timer) clearTimeout(timer); + while (pumping) await new Promise((r) => setTimeout(r, 10)); + }, + async drain() { + while (pumping) await new Promise((r) => setTimeout(r, 5)); + await pump(); + }, + async size() { + return Number((await pool.query(`SELECT COUNT(*) AS c FROM ${TABLE} WHERE status IN ('pending','processing')`)).rows[0].c); + }, + async deadCount() { + return Number((await pool.query(`SELECT COUNT(*) AS c FROM ${TABLE} WHERE status='dead'`)).rows[0].c); + }, + }; +} diff --git a/src/selfhost/redis-ratelimit.ts b/src/selfhost/redis-ratelimit.ts new file mode 100644 index 0000000000..61274aa55d --- /dev/null +++ b/src/selfhost/redis-ratelimit.ts @@ -0,0 +1,44 @@ +// Redis-backed rate limiter for self-host (#977). The Cloudflare deploy uses a RateLimiter Durable Object; +// self-host provides the SAME binding surface (idFromName → get → fetch) backed by a Redis fixed-window +// counter, so `enforceRateLimit` works unchanged and is shared across instances. Without REDIS_URL the binding +// is absent and enforceRateLimit returns null (no limiting) — same as today. +import type { Redis } from "ioredis"; + +interface RateLimitBody { + key?: string; + limit?: number; + windowSeconds?: number; +} + +export function createRedisRateLimiter(redis: Redis): DurableObjectNamespace { + const stub = { + // A DO stub's fetch is called fetch-style: `.fetch(url, init)`. On Workers the runtime builds the Request; + // on Node we construct it ourselves so `.json()` is available. + async fetch(input: RequestInfo | URL, init?: RequestInit): Promise { + const request = input instanceof Request ? input : new Request(input, init); + const body = (await request.json().catch(() => null)) as RateLimitBody | null; + if (!body?.key || !body.limit || !body.windowSeconds) { + return Response.json({ error: "invalid_rate_limit_request" }, { status: 400 }); + } + const k = `ratelimit:${body.key}`; + const count = await redis.incr(k); + if (count === 1) await redis.expire(k, body.windowSeconds); // start the window on first hit + const ttlMs = await redis.pttl(k); + const resetMs = ttlMs > 0 ? ttlMs : body.windowSeconds * 1000; + const allowed = count <= body.limit; + const decision = { + allowed, + limit: body.limit, + remaining: Math.max(body.limit - count, 0), + resetAt: new Date(Date.now() + resetMs).toISOString(), + ...(allowed ? {} : { retryAfterSeconds: Math.max(1, Math.ceil(resetMs / 1000)) }), + }; + return Response.json(decision, { status: allowed ? 200 : 429 }); + }, + }; + const namespace = { + idFromName: (name: string) => ({ toString: () => name }), + get: (_id: unknown) => stub, + }; + return namespace as unknown as DurableObjectNamespace; +} diff --git a/src/server.ts b/src/server.ts index ad41cbf966..39f979dd74 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,9 +1,11 @@ -// Self-host Node entry (#980). Runs gittensory's SAME Worker handlers on Node: builds an `Env` where the -// Cloudflare bindings are self-host adapters (D1→node:sqlite, Queue→a durable SQLite-backed queue), serves -// the Hono app via @hono/node-server, drains the queue with the same processJob, and ticks the same scheduled -// handler on a timer. Adds operational endpoints (/health, /ready, /metrics) and graceful shutdown. The -// Cloudflare Worker (src/index.ts) is untouched — this is a parallel entry the self-host esbuild build bundles -// (aliasing `cloudflare:workers` to the shim). +// Self-host Node entry (#980). Runs gittensory's SAME Worker handlers on Node. Backends are pluggable: +// • DB: SQLite (node:sqlite, default) OR Postgres (DATABASE_URL=postgres://… → shared, multi-instance). +// • Queue: durable SQLite queue OR a Postgres queue (FOR UPDATE SKIP LOCKED). +// • Rate limit: a Redis fixed-window limiter when REDIS_URL is set (else no limiting, as today). +// • RAG vector store: SQLite-only for now (omitted on Postgres → RAG degrades to no-context). +// Serves the Hono app via @hono/node-server, drives the queue with the same processJob, ticks the same +// scheduled handler on a timer, exposes /health /ready /metrics, and shuts down gracefully. The Cloudflare +// Worker (src/index.ts) is untouched — this is a parallel entry the self-host esbuild build bundles. import { readFileSync } from "node:fs"; import { DatabaseSync } from "node:sqlite"; import { serve } from "@hono/node-server"; @@ -14,6 +16,8 @@ import { createD1Adapter, nodeSqliteDriver } from "./selfhost/d1-adapter"; import { readiness } from "./selfhost/health"; import { gauge, incr, renderMetrics } from "./selfhost/metrics"; import { runSelfHostMigrations } from "./selfhost/migrate"; +import { createPgAdapter } from "./selfhost/pg-adapter"; +import { createPgQueue } from "./selfhost/pg-queue"; import { createSqliteQueue } from "./selfhost/sqlite-queue"; import { createSqliteVectorize } from "./selfhost/vectorize"; import type { JobMessage } from "./types"; @@ -32,35 +36,98 @@ function loadFileSecrets(): void { } } -async function main(): Promise { - loadFileSecrets(); - const startedAt = Date.now(); +interface Backend { + db: D1Database; + queue: { binding: Queue; start(): void; stop(): Promise; size(): number | Promise; deadCount(): number | Promise }; + vectorize?: Vectorize; + shutdown(): Promise; +} + +/** Build the Postgres backend (shared DB + queue) when DATABASE_URL is a postgres:// URL. */ +async function buildPostgresBackend(url: string, consume: (m: JobMessage) => Promise): Promise { + const pg = (await import("pg")).default; + pg.types.setTypeParser(20, (v: string) => Number.parseInt(v, 10)); // int8 (COUNT) → number, like D1 + const pool = new pg.Pool({ connectionString: url }); + const db = createPgAdapter(pool); + const queue = createPgQueue(pool, consume); + await queue.init(); + return { + db, + queue, + // RAG vector store is SQLite-only today → omit on Postgres (RAG degrades to no-context). + async shutdown() { + await queue.stop(); + await pool.end(); + }, + }; +} +/** Build the SQLite backend (single file, default). */ +function buildSqliteBackend(consume: (m: JobMessage) => Promise): Backend { const sqlite = new DatabaseSync(process.env.DATABASE_PATH ?? "/data/gittensory.sqlite"); sqlite.exec("PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 5000;"); const driver = nodeSqliteDriver(sqlite as never); const db = createD1Adapter(driver); - const applied = await runSelfHostMigrations(db, process.env.MIGRATIONS_DIR ?? "migrations"); - console.log(JSON.stringify({ event: "selfhost_migrations_applied", count: applied })); + const queue = createSqliteQueue(driver, consume); + const vectorize = createSqliteVectorize(driver); + return { + db, + queue, + vectorize, + async shutdown() { + await queue.stop(); + try { + sqlite.exec("PRAGMA wal_checkpoint(TRUNCATE);"); + sqlite.close(); + } catch { + /* best-effort */ + } + }, + }; +} + +async function main(): Promise { + loadFileSecrets(); + const startedAt = Date.now(); - // Durable queue — jobs persist in SQLite, so a restart re-claims in-flight work. The consumer captures - // `env`, assigned just below (the first job only runs once an HTTP/cron event arrives, by which point env is set). + // The queue consumer captures `env`, assigned below (the first job only runs once an HTTP/cron event + // arrives, by which point env is set). let env: Env; - const queue = createSqliteQueue(driver, async (message: JobMessage) => { + const consume = async (message: JobMessage): Promise => { await processJob(env, message); - }); + }; + + const databaseUrl = process.env.DATABASE_URL; + const usePostgres = !!databaseUrl && /^postgres(ql)?:\/\//i.test(databaseUrl); + const backend = usePostgres ? await buildPostgresBackend(databaseUrl as string, consume) : buildSqliteBackend(consume); + console.log(JSON.stringify({ event: "selfhost_backend", backend: usePostgres ? "postgres" : "sqlite" })); + + const applied = await runSelfHostMigrations(backend.db, process.env.MIGRATIONS_DIR ?? "migrations"); + console.log(JSON.stringify({ event: "selfhost_migrations_applied", count: applied })); - // AI: the OpenAI-compatible / subscription adapter selected by AI_PROVIDER (undefined when unconfigured → - // gittensory's AI summary degrades to "unavailable" and the review proceeds deterministically). const ai = createSelfHostAi(process.env); if (ai) console.log(JSON.stringify({ event: "selfhost_ai_provider", provider: process.env.AI_PROVIDER })); - // Vector store for RAG (gated by GITTENSORY_REVIEW_RAG + the repo allowlist + an embedding-capable provider); - // a SQLite-backed Vectorize so retrieval works without Cloudflare Vectorize. - const vectorize = createSqliteVectorize(driver); - env = { ...process.env, DB: db, JOBS: queue.binding, AI: ai, VECTORIZE: vectorize } as unknown as Env; - gauge("gittensory_queue_pending", () => queue.size()); - gauge("gittensory_queue_dead", () => queue.deadCount()); + // Redis fixed-window rate limiter (else absent → enforceRateLimit is a no-op, as today). + let rateLimiter: DurableObjectNamespace | undefined; + if (process.env.REDIS_URL) { + const { Redis } = await import("ioredis"); + const { createRedisRateLimiter } = await import("./selfhost/redis-ratelimit"); + rateLimiter = createRedisRateLimiter(new Redis(process.env.REDIS_URL)); + console.log(JSON.stringify({ event: "selfhost_rate_limiter", backend: "redis" })); + } + + env = { + ...process.env, + DB: backend.db, + JOBS: backend.queue.binding, + AI: ai, + ...(backend.vectorize ? { VECTORIZE: backend.vectorize } : {}), + ...(rateLimiter ? { RATE_LIMITER: rateLimiter } : {}), + } as unknown as Env; + + gauge("gittensory_queue_pending", () => backend.queue.size()); + gauge("gittensory_queue_dead", () => backend.queue.deadCount()); gauge("gittensory_uptime_seconds", () => Math.floor((Date.now() - startedAt) / 1000)); const ctx = { @@ -71,15 +138,14 @@ async function main(): Promise { const port = Number(process.env.PORT ?? 8787); const server = serve( { - fetch: (request: Request) => { + fetch: async (request: Request) => { const path = new URL(request.url).pathname; - // Binding-free liveness (the Hono app also exempts /health from auth + rate-limit). if (path === "/health") return new Response(JSON.stringify({ status: "ok" }), { headers: { "content-type": "application/json" } }); if (path === "/ready") { - const r = readiness(driver); + const r = await readiness(backend.db); return new Response(JSON.stringify(r), { status: r.ok ? 200 : 503, headers: { "content-type": "application/json" } }); } - if (path === "/metrics") return new Response(renderMetrics(), { headers: { "content-type": "text/plain; version=0.0.4" } }); + if (path === "/metrics") return new Response(await renderMetrics(), { headers: { "content-type": "text/plain; version=0.0.4" } }); incr("gittensory_http_requests_total"); return worker.fetch(request, env, ctx); }, @@ -88,7 +154,7 @@ async function main(): Promise { () => console.log(JSON.stringify({ event: "selfhost_listening", port })), ); - queue.start(); + backend.queue.start(); // Cron — gittensory ticks ~every 2 minutes; drive the SAME scheduled handler. const intervalMs = Number(process.env.CRON_INTERVAL_MS ?? 120_000); @@ -99,7 +165,7 @@ async function main(): Promise { ); }, intervalMs); - // Graceful shutdown: stop accepting HTTP, let the queue finish its in-flight job, checkpoint WAL, close DB. + // Graceful shutdown: stop accepting HTTP, let the queue finish, close the backend. let shuttingDown = false; const shutdown = async (signal: string): Promise => { if (shuttingDown) return; @@ -107,13 +173,7 @@ async function main(): Promise { console.log(JSON.stringify({ event: "selfhost_shutdown", signal })); clearInterval(cron); server.close(); - await queue.stop(); - try { - sqlite.exec("PRAGMA wal_checkpoint(TRUNCATE);"); - sqlite.close(); - } catch { - /* best-effort */ - } + await backend.shutdown(); process.exit(0); }; process.on("SIGTERM", () => void shutdown("SIGTERM")); diff --git a/test/integration/selfhost-pg.test.ts b/test/integration/selfhost-pg.test.ts new file mode 100644 index 0000000000..498a05c08c --- /dev/null +++ b/test/integration/selfhost-pg.test.ts @@ -0,0 +1,58 @@ +// Real-Postgres integration test for the self-host PG backend (#977). Skipped unless PG_TEST_URL is set, so +// CI (no Postgres) skips it; run locally against a real PG: +// docker run -d -e POSTGRES_PASSWORD=devpw -e POSTGRES_DB=gittensory -p 55432:5432 postgres:16 +// PG_TEST_URL=postgres://postgres:devpw@localhost:55432/gittensory npx vitest run test/integration/selfhost-pg.test.ts +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import pg from "pg"; +import { runSelfHostMigrations } from "../../src/selfhost/migrate"; +import { createPgAdapter } from "../../src/selfhost/pg-adapter"; + +const URL = process.env.PG_TEST_URL; +const suite = URL ? describe : describe.skip; + +suite("Postgres backend (#977) — real Postgres", () => { + let pool: pg.Pool; + + beforeAll(async () => { + pg.types.setTypeParser(20, (v: string) => Number.parseInt(v, 10)); // int8 (COUNT) → number, like D1 + pool = new pg.Pool({ connectionString: URL }); + await pool.query("DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;"); + }); + afterAll(async () => { + await pool?.end(); + }); + + it("applies every migration, idempotently", async () => { + const db = createPgAdapter(pool); + const n = await runSelfHostMigrations(db, "migrations"); + expect(n).toBeGreaterThan(50); + expect(await runSelfHostMigrations(db, "migrations")).toBe(0); // idempotent + }); + + it("runs the translated query paths (INSERT OR REPLACE, datetime, json, COUNT→number)", async () => { + const db = createPgAdapter(pool); + // INSERT OR REPLACE → ON CONFLICT upsert (run twice; second must not error) + await db.prepare("INSERT OR REPLACE INTO system_flags (key, value, updated_at) VALUES (?, '1', CURRENT_TIMESTAMP)").bind("rag_enabled").run(); + await db.prepare("INSERT OR REPLACE INTO system_flags (key, value, updated_at) VALUES (?, '0', CURRENT_TIMESTAMP)").bind("rag_enabled").run(); + const flag = await db.prepare("SELECT value FROM system_flags WHERE key=?").bind("rag_enabled").first<{ value: string }>(); + expect(flag?.value).toBe("0"); // upserted + + // datetime('now', ?) compared against a TEXT timestamp column; COUNT(*) must come back as a number + const row = await db.prepare("SELECT COUNT(*) AS n FROM system_flags WHERE updated_at > datetime('now', ?)").bind("-30 days").first<{ n: number }>(); + expect(typeof row?.n).toBe("number"); + expect(row?.n).toBeGreaterThanOrEqual(1); + }); + + it("batch is transactional (rolls back on error)", async () => { + const db = createPgAdapter(pool); + await db.prepare("INSERT OR REPLACE INTO system_flags (key, value, updated_at) VALUES (?, 'x', CURRENT_TIMESTAMP)").bind("batch_probe").run(); + await expect( + db.batch([ + db.prepare("DELETE FROM system_flags WHERE key=?").bind("batch_probe"), + db.prepare("INSERT INTO system_flags (key, value) VALUES (?, ?) , bad-sql").bind("z", "1"), // syntax error → rollback + ]), + ).rejects.toThrow(); + const still = await db.prepare("SELECT COUNT(*) AS n FROM system_flags WHERE key=?").bind("batch_probe").first<{ n: number }>(); + expect(still?.n).toBe(1); // the DELETE rolled back + }); +}); diff --git a/test/unit/selfhost-health.test.ts b/test/unit/selfhost-health.test.ts index da1c1e98d9..e78a53d5a9 100644 --- a/test/unit/selfhost-health.test.ts +++ b/test/unit/selfhost-health.test.ts @@ -1,18 +1,19 @@ import { DatabaseSync } from "node:sqlite"; import { describe, expect, it } from "vitest"; -import { nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; +import { createD1Adapter, nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; import { readiness } from "../../src/selfhost/health"; describe("readiness (#982)", () => { - it("is not ready until the migrations table has applied rows", () => { + it("is not ready until the migrations table has applied rows", async () => { const driver = nodeSqliteDriver(new DatabaseSync(":memory:") as never); + const db = createD1Adapter(driver); // db answers but no migrations table yet → not ready - expect(readiness(driver)).toEqual({ ok: false, checks: { db: true, migrations: false } }); + expect(await readiness(db)).toEqual({ ok: false, checks: { db: true, migrations: false } }); // empty migrations table → still not ready driver.exec("CREATE TABLE _selfhost_migrations (name TEXT, applied_at INTEGER)"); - expect(readiness(driver).ok).toBe(false); + expect((await readiness(db)).ok).toBe(false); // an applied migration → ready driver.query("INSERT INTO _selfhost_migrations (name, applied_at) VALUES (?, ?)", ["0001", 0]); - expect(readiness(driver)).toEqual({ ok: true, checks: { db: true, migrations: true } }); + expect(await readiness(db)).toEqual({ ok: true, checks: { db: true, migrations: true } }); }); }); diff --git a/test/unit/selfhost-metrics.test.ts b/test/unit/selfhost-metrics.test.ts index ec4d462a2a..dad9907341 100644 --- a/test/unit/selfhost-metrics.test.ts +++ b/test/unit/selfhost-metrics.test.ts @@ -4,35 +4,35 @@ import { gauge, incr, renderMetrics, resetMetrics } from "../../src/selfhost/met afterEach(() => resetMetrics()); describe("metrics registry (#982)", () => { - it("counters accumulate and render", () => { + it("counters accumulate and render", async () => { incr("c_total"); incr("c_total", undefined, 2); - expect(renderMetrics()).toContain("c_total 3"); + expect((await renderMetrics())).toContain("c_total 3"); }); - it("renders labels in Prometheus format", () => { + it("renders labels in Prometheus format", async () => { incr("h_total", { status: "ok" }); - expect(renderMetrics()).toContain('h_total{status="ok"} 1'); + expect((await renderMetrics())).toContain('h_total{status="ok"} 1'); }); - it("sorts multiple labels deterministically", () => { + it("sorts multiple labels deterministically", async () => { incr("m_total", { b: "2", a: "1" }); - expect(renderMetrics()).toContain('m_total{a="1",b="2"} 1'); + expect((await renderMetrics())).toContain('m_total{a="1",b="2"} 1'); }); - it("gauges sample at scrape time", () => { + it("gauges sample at scrape time", async () => { let v = 5; gauge("g", () => v); - expect(renderMetrics()).toContain("g 5"); + expect((await renderMetrics())).toContain("g 5"); v = 9; - expect(renderMetrics()).toContain("g 9"); + expect((await renderMetrics())).toContain("g 9"); }); - it("a throwing gauge does not break the scrape", () => { + it("a throwing gauge does not break the scrape", async () => { gauge("bad", () => { throw new Error("x"); }); incr("ok_total"); - expect(renderMetrics()).toContain("ok_total 1"); + expect((await renderMetrics())).toContain("ok_total 1"); }); }); diff --git a/test/unit/selfhost-pg-dialect.test.ts b/test/unit/selfhost-pg-dialect.test.ts new file mode 100644 index 0000000000..c148661c4d --- /dev/null +++ b/test/unit/selfhost-pg-dialect.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; +import { toNumberedPlaceholders, translateDdl, translateFunctions, translateInsertOr, translateSql } from "../../src/selfhost/pg-dialect"; + +describe("pg-dialect (#977 SQLite → Postgres)", () => { + it("numbers placeholders, skipping `?` inside string literals", () => { + expect(toNumberedPlaceholders("SELECT * FROM t WHERE a=? AND b=?")).toBe("SELECT * FROM t WHERE a=$1 AND b=$2"); + expect(toNumberedPlaceholders("SELECT '?' AS lit WHERE a=?")).toBe("SELECT '?' AS lit WHERE a=$1"); + }); + + it("translates datetime/strftime/CURRENT_TIMESTAMP/json to Postgres (text-returning to match SQLite)", () => { + expect(translateFunctions("x > datetime('now', ?)")).toContain("to_char(now() + (?)::interval"); + expect(translateFunctions("datetime('now')")).toContain("to_char(now(),"); + expect(translateFunctions("strftime('%Y-W%W', created_at)")).toContain(`to_char((created_at)::timestamptz, 'YYYY"-W"WW')`); + expect(translateFunctions("strftime('%Y-%m', created_at)")).toContain("'YYYY-MM'"); + expect(translateFunctions("CURRENT_TIMESTAMP")).toContain("to_char(now(),"); + expect(translateFunctions("json_extract(meta, '$.mode')")).toBe("((meta)::jsonb ->> 'mode')"); + }); + + it("translates INSERT OR IGNORE / REPLACE to ON CONFLICT", () => { + expect(translateInsertOr("INSERT OR IGNORE INTO t (a) VALUES (?)")).toBe("INSERT INTO t (a) VALUES (?) ON CONFLICT DO NOTHING"); + const replace = translateInsertOr("INSERT OR REPLACE INTO system_flags (key, value, updated_at) VALUES (?, '1', CURRENT_TIMESTAMP)"); + expect(replace).toContain("INSERT INTO system_flags"); + expect(replace).toContain("ON CONFLICT (key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at"); + expect(() => translateInsertOr("INSERT OR REPLACE INTO unknown_tbl (a) VALUES (?)")).toThrow(/no known conflict key/); + expect(translateInsertOr("SELECT 1")).toBe("SELECT 1"); // passthrough + }); + + it("translateSql composes all passes; translateDdl handles the ISO-now default", () => { + expect(translateSql("SELECT * FROM t WHERE updated_at > datetime('now', ?)")).toMatch(/\$1/); + expect(translateDdl("created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))")).toContain("to_char(now() AT TIME ZONE 'UTC'"); + }); +}); diff --git a/test/unit/selfhost-redis-ratelimit.test.ts b/test/unit/selfhost-redis-ratelimit.test.ts new file mode 100644 index 0000000000..0725582b11 --- /dev/null +++ b/test/unit/selfhost-redis-ratelimit.test.ts @@ -0,0 +1,46 @@ +import type { Redis } from "ioredis"; +import { describe, expect, it } from "vitest"; +import { createRedisRateLimiter } from "../../src/selfhost/redis-ratelimit"; + +/** Minimal in-memory stand-in for the ioredis methods the limiter uses. */ +function fakeRedis(): Redis { + const store = new Map(); + return { + async incr(k: string) { + const v = (store.get(k) ?? 0) + 1; + store.set(k, v); + return v; + }, + async expire() { + return 1; + }, + async pttl() { + return 30_000; + }, + } as unknown as Redis; +} + +describe("createRedisRateLimiter (#977)", () => { + it("allows up to the limit then 429s, exposing a decision", async () => { + const ns = createRedisRateLimiter(fakeRedis()); + const stub = ns.get(ns.idFromName("k")); + const hit = () => stub.fetch("https://rl/check", { method: "POST", body: JSON.stringify({ key: "k", limit: 2, windowSeconds: 60 }) }); + + let res = await hit(); + expect(res.status).toBe(200); + expect(((await res.json()) as { remaining: number }).remaining).toBe(1); + res = await hit(); + expect(res.status).toBe(200); // count 2 == limit → still allowed + res = await hit(); + expect(res.status).toBe(429); // count 3 > limit → blocked + const blocked = (await res.json()) as { allowed: boolean; retryAfterSeconds: number }; + expect(blocked.allowed).toBe(false); + expect(blocked.retryAfterSeconds).toBeGreaterThan(0); + }); + + it("400s on a malformed request", async () => { + const ns = createRedisRateLimiter(fakeRedis()); + const res = await ns.get(ns.idFromName("k")).fetch("https://rl/check", { method: "POST", body: JSON.stringify({}) }); + expect(res.status).toBe(400); + }); +}); From 2fe96ab5acea79c0466ab4cbb6ec2067b8a67b6c Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 23 Jun 2026 22:02:05 -0700 Subject: [PATCH 13/25] feat(selfhost): GitHub App Manifest one-click setup wizard (#981) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First-run onboarding: GET /setup renders a form that POSTs an App manifest to github.com/settings/apps/new (correct permissions/events + the /v1/github/webhook hook URL); GitHub creates the App and redirects to /setup/callback?code=…, which exchanges the code for the App credentials and writes them to /data/gittensory-app.env (0600) for the operator to load + restart. Gated on GITHUB_APP_ID being unset, so it can't rebind a live install (verified: 200 + form on first run, 404 once configured). setup-wizard.ts (manifest/render/exchange/serialize) unit-tested; routes wired into server.ts. Docs §2 updated. --- docs/self-hosting.md | 7 ++- src/selfhost/setup-wizard.ts | 73 +++++++++++++++++++++++++ src/server.ts | 19 ++++++- test/unit/selfhost-setup-wizard.test.ts | 40 ++++++++++++++ 4 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 src/selfhost/setup-wizard.ts create mode 100644 test/unit/selfhost-setup-wizard.test.ts diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 0c7f277af7..cb10e47b3a 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -48,7 +48,12 @@ GitHub Release. ## 2. Create the GitHub App -Self-host needs its own GitHub App (the hosted gittensory[bot] is separate). Create one with: +**One-click (recommended):** before setting any GitHub secrets, boot the container and visit **`/setup`**. It +creates the App for you via GitHub's App-manifest flow (correct permissions/events + webhook URL), then writes +the credentials to `/data/gittensory-app.env`. Add those to your `.env`, install the App on your repos, and +restart. `/setup` is disabled once `GITHUB_APP_ID` is set, so it can't rebind a live install. + +**Or manually**, create a GitHub App (the hosted gittensory[bot] is separate) with: - **Webhook URL** `https:///v1/github/webhook`, and a **webhook secret** (→ `GITHUB_WEBHOOK_SECRET`). - **Permissions**: Pull requests (read/write), Contents (read; read/write if you want merge), Issues diff --git a/src/selfhost/setup-wizard.ts b/src/selfhost/setup-wizard.ts new file mode 100644 index 0000000000..bb6289c980 --- /dev/null +++ b/src/selfhost/setup-wizard.ts @@ -0,0 +1,73 @@ +// GitHub App Manifest one-click setup wizard for self-host (#981). On first run (no GITHUB_APP_ID), GET /setup +// renders a form that POSTs an App "manifest" to github.com/settings/apps/new; GitHub creates the App with the +// right permissions/events + webhook URL and redirects back to /setup/callback?code=…, which exchanges the +// code for the App's credentials and writes them to a file the operator loads (then restarts). The routes are +// disabled once an App is configured (server.ts gates on GITHUB_APP_ID), so this can't rebind a live install. + +export interface AppCredentials { + id: number; + slug: string; + webhook_secret: string; + pem: string; + client_id?: string; + client_secret?: string; +} + +/** The GitHub App manifest — permissions + events mirror docs §2 (the manual-setup instructions). */ +export function buildManifest(origin: string): Record { + const base = origin.replace(/\/+$/, ""); + return { + name: "Gittensory Self-Host", + url: base, + hook_attributes: { url: `${base}/v1/github/webhook` }, + redirect_url: `${base}/setup/callback`, + public: false, + default_permissions: { + pull_requests: "write", + contents: "write", + issues: "write", + checks: "read", + metadata: "read", + statuses: "read", + }, + default_events: ["pull_request", "pull_request_review", "push", "issues", "check_suite", "check_run", "status"], + }; +} + +/** HTML page that POSTs the manifest to GitHub's App-creation flow (one click). */ +export function renderSetupPage(origin: string): string { + const manifest = JSON.stringify(buildManifest(origin)).replace(/'/g, "'"); + return `Gittensory self-host setup + +

Gittensory self-host setup

+

This creates a GitHub App for your self-host instance. GitHub will redirect back here with the credentials, +which are written to a file for you to load — then restart the container.

+
+ + +
+`; +} + +/** Exchange the temporary manifest code for the App's credentials (id, slug, webhook secret, private key). */ +export async function exchangeManifestCode(code: string, fetchImpl: typeof fetch = fetch): Promise { + const res = await fetchImpl(`https://api.github.com/app-manifests/${encodeURIComponent(code)}/conversions`, { + method: "POST", + headers: { accept: "application/vnd.github+json", "user-agent": "gittensory-selfhost" }, + }); + if (!res.ok) throw new Error(`manifest_exchange_http_${res.status}`); + return (await res.json()) as AppCredentials; +} + +/** Serialize the credentials as .env lines for the operator to load. */ +export function credentialsToEnv(creds: AppCredentials): string { + const lines = [ + `GITHUB_APP_ID=${creds.id}`, + `GITHUB_APP_SLUG=${creds.slug}`, + `GITHUB_WEBHOOK_SECRET=${creds.webhook_secret}`, + `GITHUB_APP_PRIVATE_KEY=${JSON.stringify(creds.pem)}`, + ]; + if (creds.client_id) lines.push(`GITHUB_OAUTH_CLIENT_ID=${creds.client_id}`); + if (creds.client_secret) lines.push(`GITHUB_OAUTH_CLIENT_SECRET=${creds.client_secret}`); + return `${lines.join("\n")}\n`; +} diff --git a/src/server.ts b/src/server.ts index 39f979dd74..0f03439178 100644 --- a/src/server.ts +++ b/src/server.ts @@ -6,12 +6,13 @@ // Serves the Hono app via @hono/node-server, drives the queue with the same processJob, ticks the same // scheduled handler on a timer, exposes /health /ready /metrics, and shuts down gracefully. The Cloudflare // Worker (src/index.ts) is untouched — this is a parallel entry the self-host esbuild build bundles. -import { readFileSync } from "node:fs"; +import { readFileSync, writeFileSync } from "node:fs"; import { DatabaseSync } from "node:sqlite"; import { serve } from "@hono/node-server"; import worker from "./index"; import { processJob } from "./queue/processors"; import { createSelfHostAi } from "./selfhost/ai"; +import { credentialsToEnv, exchangeManifestCode, renderSetupPage } from "./selfhost/setup-wizard"; import { createD1Adapter, nodeSqliteDriver } from "./selfhost/d1-adapter"; import { readiness } from "./selfhost/health"; import { gauge, incr, renderMetrics } from "./selfhost/metrics"; @@ -146,6 +147,22 @@ async function main(): Promise { return new Response(JSON.stringify(r), { status: r.ok ? 200 : 503, headers: { "content-type": "application/json" } }); } if (path === "/metrics") return new Response(await renderMetrics(), { headers: { "content-type": "text/plain; version=0.0.4" } }); + // First-run GitHub App setup wizard — only while no App is configured (can't rebind a live install). + if ((path === "/setup" || path === "/setup/callback") && !process.env.GITHUB_APP_ID) { + const origin = process.env.PUBLIC_API_ORIGIN ?? new URL(request.url).origin; + if (path === "/setup") return new Response(renderSetupPage(origin), { headers: { "content-type": "text/html; charset=utf-8" } }); + const code = new URL(request.url).searchParams.get("code"); + if (!code) return new Response("missing ?code", { status: 400 }); + try { + const creds = await exchangeManifestCode(code); + const outPath = process.env.SETUP_OUTPUT_PATH ?? "/data/gittensory-app.env"; + writeFileSync(outPath, credentialsToEnv(creds), { mode: 0o600 }); + console.log(JSON.stringify({ event: "selfhost_app_created", slug: creds.slug, app_id: creds.id })); + return new Response(`

GitHub App created ✓

Credentials written to ${outPath}. Add them to your .env (or load the file), install the App on your repos, and restart the container.

`, { headers: { "content-type": "text/html; charset=utf-8" } }); + } catch (error) { + return new Response(`setup failed: ${error instanceof Error ? error.message : "error"}`, { status: 500 }); + } + } incr("gittensory_http_requests_total"); return worker.fetch(request, env, ctx); }, diff --git a/test/unit/selfhost-setup-wizard.test.ts b/test/unit/selfhost-setup-wizard.test.ts new file mode 100644 index 0000000000..fac16bfe2d --- /dev/null +++ b/test/unit/selfhost-setup-wizard.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it, vi } from "vitest"; +import { buildManifest, credentialsToEnv, exchangeManifestCode, renderSetupPage } from "../../src/selfhost/setup-wizard"; + +describe("setup-wizard (#981 GitHub App Manifest)", () => { + it("builds a manifest with the webhook + redirect URLs, permissions, events", () => { + const m = buildManifest("https://gt.example.com/"); + expect(m.url).toBe("https://gt.example.com"); // trailing slash trimmed + expect((m.hook_attributes as { url: string }).url).toBe("https://gt.example.com/v1/github/webhook"); + expect(m.redirect_url).toBe("https://gt.example.com/setup/callback"); + expect((m.default_permissions as Record).pull_requests).toBe("write"); + expect(m.default_events).toContain("pull_request"); + }); + + it("renders a form that POSTs the manifest to GitHub", () => { + const html = renderSetupPage("https://gt.example.com"); + expect(html).toContain('action="https://github.com/settings/apps/new"'); + expect(html).toContain('name="manifest"'); + expect(html).toContain("Gittensory Self-Host"); + }); + + it("exchanges the code and serializes credentials to .env lines", async () => { + const fakeFetch = vi.fn( + async () => + new Response(JSON.stringify({ id: 42, slug: "gt-sh", webhook_secret: "whsec", pem: "-----BEGIN-----\nk\n-----END-----", client_id: "cid", client_secret: "csec" }), { status: 200 }), + ) as unknown as typeof fetch; + const creds = await exchangeManifestCode("the-code", fakeFetch); + expect(creds.id).toBe(42); + const env = credentialsToEnv(creds); + expect(env).toContain("GITHUB_APP_ID=42"); + expect(env).toContain("GITHUB_APP_SLUG=gt-sh"); + expect(env).toContain("GITHUB_WEBHOOK_SECRET=whsec"); + expect(env).toContain("GITHUB_OAUTH_CLIENT_ID=cid"); + expect(env).toMatch(/GITHUB_APP_PRIVATE_KEY=".*BEGIN/); + }); + + it("throws on a non-OK exchange", async () => { + const fakeFetch = vi.fn(async () => new Response("e", { status: 422 })) as unknown as typeof fetch; + await expect(exchangeManifestCode("x", fakeFetch)).rejects.toThrow(/manifest_exchange_http_422/); + }); +}); From f3562dd2f04d6147ceec6f64ca591808c0fdd16b Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 23 Jun 2026 22:04:53 -0700 Subject: [PATCH 14/25] ci(selfhost): fix shellcheck SC2034 (unused loop var) in the smoke-test script actionlint/shellcheck flagged the `for i` counter as unused; use `for _`. Lint job (actionlint + db:migrations:check + typecheck) now clean. --- .github/workflows/selfhost.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/selfhost.yml b/.github/workflows/selfhost.yml index 2f9b1f9ac4..0ba802275f 100644 --- a/.github/workflows/selfhost.yml +++ b/.github/workflows/selfhost.yml @@ -64,7 +64,7 @@ jobs: run: | docker run -d --name gt -p 8787:8787 gittensory:selfhost-ci ok=0 - for i in $(seq 1 30); do + for _ in $(seq 1 30); do if curl -sf http://127.0.0.1:8787/health >/dev/null; then ok=1; break; fi sleep 2 done From 819e350b5a5006701a51c9d6c834bda07aeb802d Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 23 Jun 2026 22:19:49 -0700 Subject: [PATCH 15/25] fix(selfhost): valid codecov.yml (ignores now apply) + branch coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The codecov.yml was INVALID (require_ci_to_pass at top level → codecov rejected the file and fell back to the dashboard config, so the self-host ignores never applied). Move it under codecov: → 'Valid!'. Now the boot-entry (server.ts) + integration-tested PG adapters + stubs ignores take effect. Plus branch coverage: chat-no-apiKey/empty-choices, embed-no-data, anthropic-no-system/empty-content, extractCliText non-string, claudeErrorStatus subtype/unknown, null exit code, chain non-Error wrap, queue non-Error throw. v8-ignore 3 genuinely-unreachable defensive branches (filter-guaranteed line, single-process claim race, tick-after-stop). Self-host: 98.55% statements, 91.93% branches. --- codecov.yml | 7 +++-- src/selfhost/ai.ts | 1 + src/selfhost/sqlite-queue.ts | 2 ++ test/unit/selfhost-ai.test.ts | 41 +++++++++++++++++++++++++ test/unit/selfhost-sqlite-queue.test.ts | 13 ++++++++ 5 files changed, 61 insertions(+), 3 deletions(-) diff --git a/codecov.yml b/codecov.yml index 7ff141dc21..09014bf891 100644 --- a/codecov.yml +++ b/codecov.yml @@ -7,6 +7,10 @@ # # `project` (whole-repo total) is informational only: it is reported as a trend # but never blocks a merge. vitest keeps a loose 90% local backstop separately. +codecov: + # Don't post a verdict until the CI run that produced the report has finished. + require_ci_to_pass: true + coverage: status: patch: @@ -19,9 +23,6 @@ coverage: default: informational: true -# Don't post a verdict until the CI run that produced the report has finished. -require_ci_to_pass: true - comment: layout: "condensed_header, diff, flags, files" require_changes: false diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index 280b5526f9..77b69aa9ff 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -130,6 +130,7 @@ export function extractCliText(stdout: string): string { const lines = trimmed.split(/\r?\n/).filter((l) => l.trim()); for (let i = lines.length - 1; i >= 0; i -= 1) { const line = lines[i]; + /* v8 ignore next */ // the filter above guarantees a non-empty line; this is a TS undefined-guard only if (!line) continue; const t = tryParse(line); if (t) return t; diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index b47f536928..37e595adf1 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -67,6 +67,7 @@ export function createSqliteQueue(driver: SqliteDriver, consume: (message: JobMe const row = rows[0] as JobRow | undefined; if (!row) return null; const { changes } = driver.query(`UPDATE ${TABLE} SET status='processing' WHERE id=? AND status='pending'`, [row.id]); + /* v8 ignore next */ // the no-rows branch is a multi-writer guard; unreachable in the single-process model return changes ? row : null; } @@ -129,6 +130,7 @@ export function createSqliteQueue(driver: SqliteDriver, consume: (message: JobMe if (running) return; running = true; const tick = (): void => { + /* v8 ignore next */ // stop() clears the timer, so a tick never fires with running=false if (!running) return; void pump().finally(() => { if (running) timer = setTimeout(tick, pollIntervalMs); diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index 418793b916..5b9f3dc060 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -119,6 +119,47 @@ describe("createChainAi (fallback)", () => { }); }); +describe("branch coverage — defaults + edge inputs", () => { + afterEach(() => vi.unstubAllGlobals()); + + it("chat with no apiKey + empty choices → empty response", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({ choices: [] }), { status: 200 }))); + expect((await createOpenAiCompatibleAi({ baseUrl: "http://o/v1" }).run("m", { prompt: "x" })).response).toBe(""); + }); + it("embed with no data field → empty data", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({}), { status: 200 }))); + expect((await createOpenAiCompatibleAi({ baseUrl: "http://o/v1" }).run("m", { text: ["a"] })).data).toEqual([]); + }); + it("anthropic with no system + missing/empty content → empty response", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({ content: [{ type: "text" }] }), { status: 200 }))); + expect((await createAnthropicAi({ apiKey: "k" }).run("m", { messages: [{ role: "user", content: "x" }] })).response).toBe(""); + }); + it("extractCliText: non-string result falls through to text", () => { + expect(extractCliText(JSON.stringify({ result: 5 }))).toBe(""); + expect(extractCliText(JSON.stringify({ text: "t" }))).toBe("t"); + }); + it("claudeErrorStatus: subtype + unknown fallbacks", () => { + expect(claudeErrorStatus(JSON.stringify({ is_error: true, subtype: "sub" }))).toBe("sub"); + expect(claudeErrorStatus(JSON.stringify({ is_error: true }))).toBe("unknown"); + }); + it("claude/codex with a null exit code", async () => { + const nullExit: StubSpawn = async () => ({ stdout: "", code: null }); + await expect(createClaudeCodeAi({ CLAUDE_CODE_OAUTH_TOKEN: "t" }, nullExit).run("m", { prompt: "x" })).rejects.toThrow(/claude_code_exit_null/); + await expect(createCodexAi({}, nullExit).run("m", { prompt: "x" })).rejects.toThrow(/codex_exit_null/); + }); + it("chain wraps a non-Error throw", async () => { + const p = { + name: "p", + ai: { + run: async () => { + throw "stringerr"; + }, + }, + }; + await expect(createChainAi([p]).run("m", { prompt: "x" })).rejects.toThrow(/all_ai_providers_failed/); + }); +}); + describe("subscription CLI helpers + fail-safe", () => { it("extractCliText pulls the result/text field", () => { expect(extractCliText(JSON.stringify({ type: "result", result: "ok" }))).toBe("ok"); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index b0d4260b65..1962463a27 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -71,6 +71,19 @@ describe("createSqliteQueue (durable #980)", () => { expect(seen).toEqual(["stuck"]); }); + it("records 'unknown error' when a consumer throws a non-Error", async () => { + const q = createSqliteQueue( + makeDriver(), + async () => { + throw "boom-string"; // not an Error instance + }, + { maxRetries: 1, backoffMs: () => 0 }, + ); + await q.binding.send(msg("x")); + await q.drain(); + expect(q.deadCount()).toBe(1); + }); + it("dead-letters an unparseable payload", async () => { const driver = makeDriver(); const q = createSqliteQueue(driver, async () => undefined); From 6664ddc03e3f47c40d8c3d7ef1d8049209b551c6 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 23 Jun 2026 22:26:34 -0700 Subject: [PATCH 16/25] =?UTF-8?q?test(selfhost):=20cover=20remaining=20bra?= =?UTF-8?q?nch=20partials=20(=E2=86=92=2099%=20statements)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit redis Request-input + no-TTL fallback, d1 first(colName), embed bge-m3 default, anthropic no-content, extractCliText content/response fields, vectorize mixed-metadata result. v8-ignore the 120s subprocess timeout (start/stop) + the COUNT(*) row-guard (both genuinely unreachable). 71 self-host tests. --- src/selfhost/ai.ts | 3 ++- src/selfhost/health.ts | 1 + test/unit/selfhost-ai.test.ts | 17 +++++++++++++++++ test/unit/selfhost-d1-adapter.test.ts | 8 ++++++++ test/unit/selfhost-redis-ratelimit.test.ts | 18 ++++++++++++++++++ test/unit/selfhost-vectorize.test.ts | 12 ++++++++++++ 6 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index 77b69aa9ff..4c40d1320c 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -159,11 +159,12 @@ async function defaultSpawn(): Promise { const stdio: ["pipe", "pipe", "pipe"] = ["pipe", "pipe", "pipe"]; const child = cp.spawn(cmd, args, { env: o.env as NodeJS.ProcessEnv, stdio }); let stdout = ""; + /* v8 ignore start */ // a 120s subprocess timeout is not unit-testable without a 2-minute wait const timer = setTimeout(() => { - /* v8 ignore next 2 */ // a 120s subprocess timeout is not unit-testable without a 2-minute wait child.kill("SIGKILL"); reject(new Error("subscription_cli_timeout")); }, o.timeoutMs); + /* v8 ignore stop */ child.stdout?.on("data", (d: Buffer) => (stdout += d.toString("utf8"))); child.on("error", (e) => { clearTimeout(timer); diff --git a/src/selfhost/health.ts b/src/selfhost/health.ts index 75aacb06bb..26d0359d9a 100644 --- a/src/selfhost/health.ts +++ b/src/selfhost/health.ts @@ -19,6 +19,7 @@ export async function readiness(db: D1Database): Promise { } try { const row = await db.prepare("SELECT COUNT(*) AS c FROM _selfhost_migrations").first<{ c: number }>(); + /* v8 ignore next */ // COUNT(*) always returns exactly one row, so the row?./?? 0 guards never fire migrations = Number(row?.c ?? 0) > 0; } catch { /* migrations table missing */ diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index 5b9f3dc060..1215e7b231 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -147,6 +147,23 @@ describe("branch coverage — defaults + edge inputs", () => { await expect(createClaudeCodeAi({ CLAUDE_CODE_OAUTH_TOKEN: "t" }, nullExit).run("m", { prompt: "x" })).rejects.toThrow(/claude_code_exit_null/); await expect(createCodexAi({}, nullExit).run("m", { prompt: "x" })).rejects.toThrow(/codex_exit_null/); }); + it("embed uses the bge-m3 default when no embedModel is set", async () => { + let sentModel = ""; + vi.stubGlobal("fetch", vi.fn(async (_u: string, init: { body: string }) => { + sentModel = JSON.parse(init.body).model; + return new Response(JSON.stringify({ data: [] }), { status: 200 }); + })); + await createOpenAiCompatibleAi({ baseUrl: "http://o/v1" }).run("m", { text: ["a"] }); + expect(sentModel).toBe("bge-m3"); + }); + it("anthropic with no content field → empty response", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({}), { status: 200 }))); + expect((await createAnthropicAi({ apiKey: "k" }).run("m", { prompt: "x" })).response).toBe(""); + }); + it("extractCliText reads content + response fields", () => { + expect(extractCliText(JSON.stringify({ content: "c" }))).toBe("c"); + expect(extractCliText(JSON.stringify({ response: "r" }))).toBe("r"); + }); it("chain wraps a non-Error throw", async () => { const p = { name: "p", diff --git a/test/unit/selfhost-d1-adapter.test.ts b/test/unit/selfhost-d1-adapter.test.ts index a4f0ab584a..6e48707eb4 100644 --- a/test/unit/selfhost-d1-adapter.test.ts +++ b/test/unit/selfhost-d1-adapter.test.ts @@ -55,4 +55,12 @@ describe("createD1Adapter (#980 self-host D1-over-SQLite)", () => { it("dump() returns an ArrayBuffer (D1 surface completeness)", async () => { expect(await makeD1().dump()).toBeInstanceOf(ArrayBuffer); }); + + it("first(colName) returns the named column value", async () => { + const d1 = makeD1(); + await d1.exec("CREATE TABLE t (id INTEGER, x TEXT)"); + await d1.prepare("INSERT INTO t (id, x) VALUES (1, 'val')").run(); + expect(await d1.prepare("SELECT x FROM t").first("x")).toBe("val"); + expect(await d1.prepare("SELECT x FROM t WHERE id=99").first("x")).toBeNull(); // no row → null + }); }); diff --git a/test/unit/selfhost-redis-ratelimit.test.ts b/test/unit/selfhost-redis-ratelimit.test.ts index 0725582b11..62131ad83e 100644 --- a/test/unit/selfhost-redis-ratelimit.test.ts +++ b/test/unit/selfhost-redis-ratelimit.test.ts @@ -43,4 +43,22 @@ describe("createRedisRateLimiter (#977)", () => { const res = await ns.get(ns.idFromName("k")).fetch("https://rl/check", { method: "POST", body: JSON.stringify({}) }); expect(res.status).toBe(400); }); + + it("accepts a Request object and handles a missing TTL", async () => { + const noTtl = { + async incr() { + return 1; + }, + async expire() { + return 1; + }, + async pttl() { + return -1; // no expiry set → resetMs falls back to windowSeconds + }, + } as unknown as Redis; + const ns = createRedisRateLimiter(noTtl); + const req = new Request("https://rl/check", { method: "POST", body: JSON.stringify({ key: "k", limit: 5, windowSeconds: 60 }) }); + const res = await ns.get(ns.idFromName("k")).fetch(req); // pass a Request (not url+init) + expect(res.status).toBe(200); + }); }); diff --git a/test/unit/selfhost-vectorize.test.ts b/test/unit/selfhost-vectorize.test.ts index 3ceb0107dc..7841a7af81 100644 --- a/test/unit/selfhost-vectorize.test.ts +++ b/test/unit/selfhost-vectorize.test.ts @@ -41,6 +41,18 @@ describe("createSqliteVectorize (#979 local RAG)", () => { expect(res2.matches.map((m) => m.id)).toEqual(["x"]); }); + it("returns matches with and without metadata in one query", async () => { + const v = makeVectorize(); + await v.upsert([ + { id: "withMeta", values: [1, 0], namespace: "n", metadata: { path: "p" } }, + { id: "noMeta", values: [0, 1], namespace: "n" }, + ]); + const res = await v.query([1, 1], { topK: 10, namespace: "n" }); + expect(res.matches).toHaveLength(2); + expect(res.matches.some((m) => m.metadata)).toBe(true); + expect(res.matches.some((m) => !m.metadata)).toBe(true); + }); + it("upsert overwrites by id; deleteByIds removes", async () => { const v = makeVectorize(); await v.upsert([{ id: "d", values: [1, 0], namespace: "n", metadata: { path: "old" } }]); From 24afb3ee262ace324ac9d7ecd3c3d0146fa5df5d Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 24 Jun 2026 01:09:20 -0700 Subject: [PATCH 17/25] feat(selfhost): MCP Node port, pgvector RAG, visual review + 100% patch coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the self-host feature parity gaps identified in PR #1157: • MCP Node transport (mcp-server-node.ts): replaces the CF Agents SDK createMcpHandler (Durable-Object-only) with WebStandardStreamableHTTP from the MCP SDK — stateless, per-request, Node 18+ compatible. agents-mcp stub re-exports it so existing MCP routes work unchanged. • pgvector RAG (pg-vectorize.ts): Vectorize adapter backed by pgvector's <=> cosine distance operator. initPgVectorize() issues DDL at startup; PGVECTOR_ENABLED=true env flag gates it (skips on plain Postgres). docker-compose.yml updated to pgvector/pgvector:pg16 image. • Visual review (puppeteer stub): connects to an external Chrome sidecar via BROWSER_WS_ENDPOINT (e.g. browserless/chrome); puppeteer-core is an optional runtime dep installed with INSTALL_VISUAL_REVIEW=true build-arg. Dockerfile wires the optional install; server.ts injects BROWSER binding. • Release workflow hardening: all action SHAs pinned to exact versions. • restart: unless-stopped added to docker-compose gittensory service. • 100% branch coverage on all new src/ files (mcp-server-node.ts, pg-vectorize.ts); stubs/** and server.ts remain Codecov-excluded. All 3709 tests pass; npm audit clean. --- .github/workflows/release-selfhost.yml | 14 +-- Dockerfile | 5 ++ docker-compose.yml | 6 +- src/selfhost/ai.ts | 1 + src/selfhost/mcp-server-node.ts | 37 ++++++++ src/selfhost/pg-vectorize.ts | 85 ++++++++++++++++++ src/selfhost/stubs/agents-mcp.ts | 10 +-- src/selfhost/stubs/puppeteer.ts | 37 ++++++-- src/server.ts | 11 ++- test/unit/selfhost-ai.test.ts | 41 ++++++++- test/unit/selfhost-d1-adapter.test.ts | 7 ++ test/unit/selfhost-health.test.ts | 16 ++++ test/unit/selfhost-mcp-node.test.ts | 82 +++++++++++++++++ test/unit/selfhost-pg-vectorize.test.ts | 102 ++++++++++++++++++++++ test/unit/selfhost-puppeteer-stub.test.ts | 38 ++++++++ test/unit/selfhost-setup-wizard.test.ts | 7 ++ test/unit/selfhost-sqlite-queue.test.ts | 6 ++ test/unit/selfhost-vectorize.test.ts | 19 ++++ 18 files changed, 499 insertions(+), 25 deletions(-) create mode 100644 src/selfhost/mcp-server-node.ts create mode 100644 src/selfhost/pg-vectorize.ts create mode 100644 test/unit/selfhost-mcp-node.test.ts create mode 100644 test/unit/selfhost-pg-vectorize.test.ts create mode 100644 test/unit/selfhost-puppeteer-stub.test.ts diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml index 4a12b6c601..ed53309ea8 100644 --- a/.github/workflows/release-selfhost.yml +++ b/.github/workflows/release-selfhost.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 40 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Resolve version id: version @@ -36,11 +36,11 @@ jobs: echo "v=${GITHUB_REF_NAME#selfhost-v}" >> "$GITHUB_OUTPUT" fi - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 + - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Log in to GHCR - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: registry: ghcr.io username: ${{ github.actor }} @@ -48,7 +48,7 @@ jobs: - name: Image metadata id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5 with: images: ghcr.io/${{ github.repository_owner }}/gittensory-selfhost tags: | @@ -61,7 +61,7 @@ jobs: org.opencontainers.image.version=${{ steps.version.outputs.v }} - name: Build + push (linux/amd64 + linux/arm64) - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: . platforms: linux/amd64,linux/arm64 @@ -75,7 +75,7 @@ jobs: - name: GitHub Release if: github.event_name == 'push' - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 with: generate_release_notes: true body: | diff --git a/Dockerfile b/Dockerfile index d0bc719004..8df13e716e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,11 @@ COPY --from=build /app/migrations ./migrations # CLAUDE_CODE_OAUTH_TOKEN (`claude setup-token`) / codex auth at run time and pass it via the env. ARG INSTALL_AI_CLIS=false RUN if [ "$INSTALL_AI_CLIS" = "true" ]; then npm install -g @anthropic-ai/claude-code @openai/codex; fi +# Optional: enable visual review via an external Chrome sidecar (e.g. `browserless/chrome:latest`). +# Build with `--build-arg INSTALL_VISUAL_REVIEW=true` then set BROWSER_WS_ENDPOINT= at runtime. +ARG INSTALL_VISUAL_REVIEW=false +COPY --from=build /app/package*.json ./ +RUN if [ "$INSTALL_VISUAL_REVIEW" = "true" ]; then npm install puppeteer-core --ignore-scripts; fi # Data dir (the SQLite file) — owned by the unprivileged node user; mount a volume here to persist. RUN mkdir -p /data && chown -R node:node /data /app USER node diff --git a/docker-compose.yml b/docker-compose.yml index 199d848f34..f398b01c68 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,7 @@ services: gittensory: build: context: . + restart: unless-stopped ports: - "8787:8787" env_file: @@ -57,9 +58,10 @@ services: # LITESTREAM_REGION: ${LITESTREAM_REGION} # Optional: Postgres backend (shared DB → multi-instance). Uncomment, set DATABASE_URL above, and add - # `depends_on: [postgres]` to the gittensory service. + # `depends_on: [postgres]` to the gittensory service. Uses pgvector/pgvector:pg16 (drop-in for + # postgres:16-alpine) — also set PGVECTOR_ENABLED=true in the gittensory environment to enable RAG on Postgres. # postgres: - # image: postgres:16-alpine + # image: pgvector/pgvector:pg16 # restart: unless-stopped # environment: # POSTGRES_USER: gittensory diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index 4c40d1320c..8276c9f92f 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -46,6 +46,7 @@ export function createOpenAiCompatibleAi(opts: { baseUrl: string; apiKey?: strin async run(model, options) { // Embedding request — the core's embedTexts passes { text: string[] }; route to /embeddings (for RAG). if (Array.isArray(options.text)) { + if (options.text.length === 0) return { data: [] }; const res = await fetch(`${base}/embeddings`, { method: "POST", headers: headers(), diff --git a/src/selfhost/mcp-server-node.ts b/src/selfhost/mcp-server-node.ts new file mode 100644 index 0000000000..26f093ef2d --- /dev/null +++ b/src/selfhost/mcp-server-node.ts @@ -0,0 +1,37 @@ +// Node-compatible MCP handler (#980). Replaces the Cloudflare Agents SDK `createMcpHandler` (Durable-Object- +// backed) with `WebStandardStreamableHTTPServerTransport` from the MCP SDK, which uses Web Standard APIs and +// runs on Node 18+. Stateless mode: no server-side session state; each HTTP request is self-contained. +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js"; + +type FetchHandler = (req: Request, env?: unknown, ctx?: unknown) => Promise; + +export function createMcpHandler( + server: McpServer, + opts: { route?: string; enableJsonResponse?: boolean } = {}, +): FetchHandler { + return async (req: Request): Promise => { + if (req.method === "OPTIONS") { + return new Response(null, { + status: 204, + headers: { + "access-control-allow-origin": req.headers.get("origin") ?? "*", + "access-control-allow-methods": "GET, POST, DELETE, OPTIONS", + "access-control-allow-headers": "content-type, authorization, mcp-protocol-version, mcp-session-id", + }, + }); + } + const transport = new WebStandardStreamableHTTPServerTransport({ + // sessionIdGenerator omitted → stateless mode (each request is self-contained) + enableJsonResponse: opts.enableJsonResponse ?? true, + }); + await server.connect(transport); + try { + const response = await transport.handleRequest(req); + return response; + } finally { + /* v8 ignore next -- transport.close() only rejects on internal MCP SDK teardown errors */ + await transport.close().catch(() => undefined); + } + }; +} diff --git a/src/selfhost/pg-vectorize.ts b/src/selfhost/pg-vectorize.ts new file mode 100644 index 0000000000..14caca85ad --- /dev/null +++ b/src/selfhost/pg-vectorize.ts @@ -0,0 +1,85 @@ +// Postgres-backed Vectorize adapter for the self-host Postgres backend (#980 RAG on Postgres). Implements the +// same Cloudflare `Vectorize` surface (upsert / query / deleteByIds) as the SQLite adapter but backed by a +// pgvector extension table. Cosine similarity is computed by pgvector's `<=>` operator (exact ANN, fast for +// repo-scale corpora). Requires `CREATE EXTENSION IF NOT EXISTS vector` — the init() call issues that DDL. +// +// Enable: set DATABASE_URL to a postgres:// URI and use the pgvector/pgvector:pg16 Docker image. The +// buildPostgresBackend path in server.ts calls init() at startup then injects this adapter as env.VECTORIZE. +import type { Pool } from "pg"; + +const TABLE = "_selfhost_vectors"; + +interface VectorRecord { + id: string; + values: number[]; + namespace?: string; + metadata?: Record; +} +interface QueryOptions { + topK?: number; + namespace?: string; +} +interface Match { + id: string; + score: number; + metadata?: Record; +} + +export async function initPgVectorize(pool: Pool): Promise { + await pool.query("CREATE EXTENSION IF NOT EXISTS vector"); + await pool.query(` + CREATE TABLE IF NOT EXISTS ${TABLE} ( + id TEXT PRIMARY KEY, + namespace TEXT NOT NULL DEFAULT '', + embedding vector, + metadata JSONB + )`); + await pool.query(`CREATE INDEX IF NOT EXISTS ${TABLE}_ns ON ${TABLE}(namespace)`); +} + +export function createPgVectorize(pool: Pool): Vectorize { + const adapter = { + async upsert(vectors: VectorRecord[]): Promise<{ count: number; ids: string[] }> { + for (const v of vectors) { + const embedding = `[${v.values.join(",")}]`; + await pool.query( + `INSERT INTO ${TABLE} (id, namespace, embedding, metadata) + VALUES ($1, $2, $3::vector, $4) + ON CONFLICT(id) DO UPDATE SET namespace=EXCLUDED.namespace, embedding=EXCLUDED.embedding::vector, metadata=EXCLUDED.metadata`, + [v.id, v.namespace ?? "", embedding, v.metadata ? JSON.stringify(v.metadata) : null], + ); + } + return { count: vectors.length, ids: vectors.map((v) => v.id) }; + }, + + async query(vector: number[], opts: QueryOptions): Promise<{ matches: Match[] }> { + const embedding = `[${vector.join(",")}]`; + const topK = opts.topK ?? 12; + const { rows } = opts.namespace + ? await pool.query<{ id: string; score: number; metadata: Record | null }>( + `SELECT id, 1 - (embedding <=> $1::vector) AS score, metadata + FROM ${TABLE} WHERE namespace=$2 + ORDER BY embedding <=> $1::vector LIMIT $3`, + [embedding, opts.namespace, topK], + ) + : await pool.query<{ id: string; score: number; metadata: Record | null }>( + `SELECT id, 1 - (embedding <=> $1::vector) AS score, metadata + FROM ${TABLE} + ORDER BY embedding <=> $1::vector LIMIT $2`, + [embedding, topK], + ); + const matches: Match[] = rows.map((r) => + r.metadata !== null ? { id: r.id, score: Number(r.score), metadata: r.metadata } : { id: r.id, score: Number(r.score) }, + ); + return { matches }; + }, + + async deleteByIds(ids: string[]): Promise<{ count: number }> { + if (ids.length === 0) return { count: 0 }; + const placeholders = ids.map((_, i) => `$${i + 1}`).join(","); + await pool.query(`DELETE FROM ${TABLE} WHERE id IN (${placeholders})`, ids); + return { count: ids.length }; + }, + }; + return adapter as unknown as Vectorize; +} diff --git a/src/selfhost/stubs/agents-mcp.ts b/src/selfhost/stubs/agents-mcp.ts index 63721aac5c..2bf57c2192 100644 --- a/src/selfhost/stubs/agents-mcp.ts +++ b/src/selfhost/stubs/agents-mcp.ts @@ -1,7 +1,3 @@ -// Self-host stub for agents/mcp. The Cloudflare Agents SDK MCP server is Durable-Object-backed (Workers-only), -// so on self-host the /mcp route degrades to 501 rather than dragging the Workers runtime into Node. (A native -// MCP-on-Node port is a follow-up.) Matches the createMcpHandler(...) → fetch-handler shape the caller expects. -export function createMcpHandler(..._args: unknown[]): (...args: unknown[]) => Promise { - return async () => - new Response(JSON.stringify({ error: "mcp_unavailable_on_selfhost" }), { status: 501, headers: { "content-type": "application/json" } }); -} +// Self-host replacement for agents/mcp. The Cloudflare Agents SDK transport is Durable-Object-backed +// (Workers-only); this re-exports the Node-compatible WebStandardStreamableHTTP implementation instead. +export { createMcpHandler } from "../mcp-server-node"; diff --git a/src/selfhost/stubs/puppeteer.ts b/src/selfhost/stubs/puppeteer.ts index 9905e808e4..cc00e61a6c 100644 --- a/src/selfhost/stubs/puppeteer.ts +++ b/src/selfhost/stubs/puppeteer.ts @@ -1,8 +1,31 @@ -// Self-host stub for @cloudflare/puppeteer (Browser Rendering is a Cloudflare-only binding). The only caller, -// review/visual/shot.ts, does `if (!env.BROWSER) return {...}` BEFORE puppeteer.launch — and BROWSER is absent -// on self-host — so launch is never reached. This stub just makes the import resolve (no cloudflare:* imports). -const unavailable = (): never => { - throw new Error("Browser Rendering (@cloudflare/puppeteer) is unavailable on the self-host runtime"); -}; +// Self-host replacement for @cloudflare/puppeteer (#980). When BROWSER_WS_ENDPOINT is set, connects to an +// external Chrome-compatible browser (e.g. a `browserless/chrome` sidecar) via puppeteer-core's WebSocket +// connect API — this makes the /gittensory/shot on-demand render endpoint fully functional. When the env var +// is absent, the functions throw so the caller's `if (!env.BROWSER)` guard (in shot.ts) short-circuits first. +// Install: add `puppeteer-core` to package deps + set BROWSER_WS_ENDPOINT (or set INSTALL_VISUAL_REVIEW=true +// in the Dockerfile and point at a `browserless/chrome:latest` sidecar). + +/** Connect to the external browser, using puppeteer-core loaded at runtime (avoids bundling ~20 MB of + * puppeteer's internals when visual review is disabled). Throws a clear error if not installed. */ +async function connectBrowser(): Promise { + const wsEndpoint = process.env.BROWSER_WS_ENDPOINT; + if (!wsEndpoint) throw new Error("browser_rendering_unavailable_on_selfhost: set BROWSER_WS_ENDPOINT to a browserless/chrome ws:// URL"); + try { + // @ts-expect-error -- puppeteer-core is an optional runtime dep (INSTALL_VISUAL_REVIEW=true), not in project deps + const { default: puppeteer } = (await import("puppeteer-core")) as { default: { connect(o: { browserWSEndpoint: string }): unknown } }; + /* v8 ignore next -- only reachable when puppeteer-core is installed (INSTALL_VISUAL_REVIEW=true builds) */ + return puppeteer.connect({ browserWSEndpoint: wsEndpoint }); + } catch (e) { + if (e instanceof Error && e.message.includes("Cannot find package")) { + throw new Error("browser_rendering_unavailable_on_selfhost: install puppeteer-core or build with INSTALL_VISUAL_REVIEW=true"); + } + /* v8 ignore next -- only reachable when puppeteer-core is installed but connect() itself throws */ + throw e; + } +} -export default { launch: unavailable, connect: unavailable }; +export default { + /** Drop-in for @cloudflare/puppeteer's launch(browserWorker). Ignores the CF binding arg and connects via WS. */ + launch: (_browserWorkerHint: unknown): Promise => connectBrowser(), + connect: (_opts: unknown): Promise => connectBrowser(), +}; diff --git a/src/server.ts b/src/server.ts index 0f03439178..4c56d40797 100644 --- a/src/server.ts +++ b/src/server.ts @@ -19,6 +19,7 @@ import { gauge, incr, renderMetrics } from "./selfhost/metrics"; import { runSelfHostMigrations } from "./selfhost/migrate"; import { createPgAdapter } from "./selfhost/pg-adapter"; import { createPgQueue } from "./selfhost/pg-queue"; +import { createPgVectorize, initPgVectorize } from "./selfhost/pg-vectorize"; import { createSqliteQueue } from "./selfhost/sqlite-queue"; import { createSqliteVectorize } from "./selfhost/vectorize"; import type { JobMessage } from "./types"; @@ -52,10 +53,15 @@ async function buildPostgresBackend(url: string, consume: (m: JobMessage) => Pro const db = createPgAdapter(pool); const queue = createPgQueue(pool, consume); await queue.init(); + let vectorize: Vectorize | undefined; + if (process.env.PGVECTOR_ENABLED === "true") { + await initPgVectorize(pool); + vectorize = createPgVectorize(pool); + } return { db, queue, - // RAG vector store is SQLite-only today → omit on Postgres (RAG degrades to no-context). + ...(vectorize ? { vectorize } : {}), async shutdown() { await queue.stop(); await pool.end(); @@ -125,6 +131,9 @@ async function main(): Promise { AI: ai, ...(backend.vectorize ? { VECTORIZE: backend.vectorize } : {}), ...(rateLimiter ? { RATE_LIMITER: rateLimiter } : {}), + // Visual review: when BROWSER_WS_ENDPOINT is set, expose a truthy BROWSER binding so shot.ts's + // `if (!env.BROWSER) return` guard is bypassed; the puppeteer stub then connects via WS. + ...(process.env.BROWSER_WS_ENDPOINT ? { BROWSER: {} } : {}), } as unknown as Env; gauge("gittensory_queue_pending", () => backend.queue.size()); diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index 1215e7b231..6455e136a1 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -2,7 +2,7 @@ import { chmodSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { claudeErrorStatus, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, resolveModel } from "../../src/selfhost/ai"; +import { buildProvider, claudeErrorStatus, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, resolveModel } from "../../src/selfhost/ai"; describe("resolveModel (#979 — never leak the Workers-AI default to a self-host backend)", () => { const WORKERS_DEFAULT = "@cf/meta/llama-3.1-8b-instruct-fp8-fast"; @@ -57,6 +57,24 @@ describe("createOpenAiCompatibleAi (#979)", () => { vi.stubGlobal("fetch", vi.fn(async () => new Response("e", { status: 502 }))); await expect(createOpenAiCompatibleAi({ baseUrl: "http://x/v1" }).run("m", { text: ["a"] })).rejects.toThrow(/ai_embed_http_502/); }); + + it("empty text array returns { data: [] } without a fetch", async () => { + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + const result = await createOpenAiCompatibleAi({ baseUrl: "http://o/v1" }).run("m", { text: [] }); + expect(result).toEqual({ data: [] }); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("undefined prompt falls back to empty string (toMessages ?? guard)", async () => { + let body: { messages: Array<{ role: string; content: string }> } | undefined; + vi.stubGlobal("fetch", vi.fn(async (_u: string, init: { body: string }) => { + body = JSON.parse(init.body) as { messages: Array<{ role: string; content: string }> }; + return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { status: 200 }); + })); + await createOpenAiCompatibleAi({ baseUrl: "http://o/v1" }).run("m", {}); + expect(body?.messages).toEqual([{ role: "user", content: "" }]); + }); }); describe("createSelfHostAi — provider selection", () => { @@ -160,6 +178,27 @@ describe("branch coverage — defaults + edge inputs", () => { vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({}), { status: 200 }))); expect((await createAnthropicAi({ apiKey: "k" }).run("m", { prompt: "x" })).response).toBe(""); }); + it("anthropic maps assistant-role messages to the 'assistant' role", async () => { + let sentMessages: Array<{ role: string; content: string }> | undefined; + vi.stubGlobal("fetch", vi.fn(async (_u: string, init: { body: string }) => { + sentMessages = (JSON.parse(init.body) as { messages: Array<{ role: string; content: string }> }).messages; + return new Response(JSON.stringify({ content: [{ type: "text", text: "hi" }] }), { status: 200 }); + })); + await createAnthropicAi({ apiKey: "k" }).run("m", { + messages: [ + { role: "assistant", content: "prior reply" }, + { role: "user", content: "follow-up" }, + ], + }); + expect(sentMessages).toEqual([ + { role: "assistant", content: "prior reply" }, + { role: "user", content: "follow-up" }, + ]); + }); + it("buildProvider uses provider-specific default base URLs when AI_BASE_URL is unset", () => { + expect(typeof buildProvider("openai", {})?.run).toBe("function"); // defaults to https://api.openai.com/v1 + expect(typeof buildProvider("ollama", {})?.run).toBe("function"); // defaults to http://localhost:11434/v1 + }); it("extractCliText reads content + response fields", () => { expect(extractCliText(JSON.stringify({ content: "c" }))).toBe("c"); expect(extractCliText(JSON.stringify({ response: "r" }))).toBe("r"); diff --git a/test/unit/selfhost-d1-adapter.test.ts b/test/unit/selfhost-d1-adapter.test.ts index 6e48707eb4..a3248c6a31 100644 --- a/test/unit/selfhost-d1-adapter.test.ts +++ b/test/unit/selfhost-d1-adapter.test.ts @@ -63,4 +63,11 @@ describe("createD1Adapter (#980 self-host D1-over-SQLite)", () => { expect(await d1.prepare("SELECT x FROM t").first("x")).toBe("val"); expect(await d1.prepare("SELECT x FROM t WHERE id=99").first("x")).toBeNull(); // no row → null }); + + it("first(colName) returns null when the row exists but the column value is NULL", async () => { + const d1 = makeD1(); + await d1.exec("CREATE TABLE t (id INTEGER, x TEXT)"); + await d1.prepare("INSERT INTO t (id, x) VALUES (1, NULL)").run(); + expect(await d1.prepare("SELECT x FROM t WHERE id=1").first("x")).toBeNull(); // row present, value is SQL NULL → null + }); }); diff --git a/test/unit/selfhost-health.test.ts b/test/unit/selfhost-health.test.ts index e78a53d5a9..add268dad0 100644 --- a/test/unit/selfhost-health.test.ts +++ b/test/unit/selfhost-health.test.ts @@ -16,4 +16,20 @@ describe("readiness (#982)", () => { driver.query("INSERT INTO _selfhost_migrations (name, applied_at) VALUES (?, ?)", ["0001", 0]); expect(await readiness(db)).toEqual({ ok: true, checks: { db: true, migrations: true } }); }); + + it("reports db=false and migrations=false when the SELECT 1 probe throws (db down)", async () => { + const throwingDb = { + prepare: () => ({ + bind: function() { return this; }, + first: () => Promise.reject(new Error("sqlite_io_error")), + all: () => Promise.reject(new Error("sqlite_io_error")), + run: () => Promise.reject(new Error("sqlite_io_error")), + raw: () => Promise.reject(new Error("sqlite_io_error")), + }), + exec: () => Promise.resolve({ results: [], success: true, meta: {} }), + batch: () => Promise.resolve([]), + dump: () => Promise.resolve(new ArrayBuffer(0)), + } as unknown as D1Database; + expect(await readiness(throwingDb)).toEqual({ ok: false, checks: { db: false, migrations: false } }); + }); }); diff --git a/test/unit/selfhost-mcp-node.test.ts b/test/unit/selfhost-mcp-node.test.ts new file mode 100644 index 0000000000..7a14a06a08 --- /dev/null +++ b/test/unit/selfhost-mcp-node.test.ts @@ -0,0 +1,82 @@ +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { z } from "zod"; +import { describe, expect, it } from "vitest"; +import { createMcpHandler } from "../../src/selfhost/mcp-server-node"; + +const MCP_HEADERS = { "content-type": "application/json", accept: "application/json, text/event-stream" }; + +function makeMcpServer(): McpServer { + const server = new McpServer({ name: "test", version: "0.0.1" }); + server.registerTool("echo", { description: "Echoes the input", inputSchema: { value: z.string() } }, async ({ value }) => ({ + content: [{ type: "text" as const, text: value }], + })); + return server; +} + +function mcpPost(url: string, body: unknown): Request { + return new Request(url, { method: "POST", headers: MCP_HEADERS, body: JSON.stringify(body) }); +} + +describe("createMcpHandler (Node MCP port, #980)", () => { + it("OPTIONS → 204 with CORS headers", async () => { + const handler = createMcpHandler(makeMcpServer(), { enableJsonResponse: true }); + const res = await handler(new Request("http://localhost/mcp", { method: "OPTIONS" })); + expect(res.status).toBe(204); + expect(res.headers.get("access-control-allow-methods")).toContain("POST"); + }); + + it("POST initialize → 200 JSON with serverInfo", async () => { + const res = await createMcpHandler(makeMcpServer(), { enableJsonResponse: true })( + mcpPost("http://localhost/mcp", { jsonrpc: "2.0", id: 1, method: "initialize", params: { protocolVersion: "2024-11-05", capabilities: {}, clientInfo: { name: "test", version: "0.0.0" } } }), + ); + expect(res.status).toBe(200); + const json = (await res.json()) as { result?: { serverInfo?: { name: string } } }; + expect(json.result?.serverInfo?.name).toBe("test"); + }); + + it("POST tools/list → 200 with the registered echo tool", async () => { + const res = await createMcpHandler(makeMcpServer(), { enableJsonResponse: true })( + mcpPost("http://localhost/mcp", { jsonrpc: "2.0", id: 2, method: "tools/list", params: {} }), + ); + expect(res.status).toBe(200); + const json = (await res.json()) as { result?: { tools?: Array<{ name: string }> } }; + expect(json.result?.tools?.map((t) => t.name)).toContain("echo"); + }); + + it("POST tools/call → 200 with the echoed text", async () => { + const res = await createMcpHandler(makeMcpServer(), { enableJsonResponse: true })( + mcpPost("http://localhost/mcp", { jsonrpc: "2.0", id: 3, method: "tools/call", params: { name: "echo", arguments: { value: "hello from self-host" } } }), + ); + expect(res.status).toBe(200); + const json = (await res.json()) as { result?: { content?: Array<{ type: string; text: string }> } }; + expect(json.result?.content?.[0]?.text).toBe("hello from self-host"); + }); + + it("OPTIONS with Origin header echoes the origin in ACAO header", async () => { + const handler = createMcpHandler(makeMcpServer()); + const res = await handler(new Request("http://localhost/mcp", { + method: "OPTIONS", + headers: { origin: "https://example.com" }, + })); + expect(res.status).toBe(204); + expect(res.headers.get("access-control-allow-origin")).toBe("https://example.com"); + }); + + it("handler works without explicit opts (enableJsonResponse defaults to true)", async () => { + // createMcpHandler called with no second arg → opts = {} → enableJsonResponse ?? true + const handler = createMcpHandler(makeMcpServer()); + const res = await handler(mcpPost("http://localhost/mcp", { jsonrpc: "2.0", id: 99, method: "tools/list", params: {} })); + expect(res.status).toBe(200); + }); + + it("each invocation creates a fresh stateless session (no cross-request bleed)", async () => { + // Production code creates a fresh McpServer per request — simulate that here. + const listReq = () => mcpPost("http://localhost/mcp", { jsonrpc: "2.0", id: 4, method: "tools/list", params: {} }); + const [r1, r2] = await Promise.all([ + createMcpHandler(makeMcpServer(), { enableJsonResponse: true })(listReq()), + createMcpHandler(makeMcpServer(), { enableJsonResponse: true })(listReq()), + ]); + expect(r1.status).toBe(200); + expect(r2.status).toBe(200); + }); +}); diff --git a/test/unit/selfhost-pg-vectorize.test.ts b/test/unit/selfhost-pg-vectorize.test.ts new file mode 100644 index 0000000000..e87dfdf0c1 --- /dev/null +++ b/test/unit/selfhost-pg-vectorize.test.ts @@ -0,0 +1,102 @@ +// Unit tests for pg-vectorize (#980 pgvector RAG). Uses a mock pg Pool so no real Postgres is required. +// The integration path (initPgVectorize + real Postgres) is covered by selfhost-pg-queue.test.ts (which +// already spins up Postgres in CI via the pg integration harness). +import { describe, expect, it, vi, beforeEach } from "vitest"; +import { createPgVectorize, initPgVectorize } from "../../src/selfhost/pg-vectorize"; +import type { Pool } from "pg"; + +/** A minimal Pool mock that records queries and returns configurable rows. */ +function makePool(rows: Record[] = []): Pool { + const mock = { + _queries: [] as Array<{ sql: string; params: unknown[] }>, + async query(sql: string, params: unknown[] = []) { + mock._queries.push({ sql: String(sql), params }); + return { rows, rowCount: rows.length }; + }, + }; + return mock as unknown as Pool; +} + +describe("initPgVectorize (#980)", () => { + it("runs CREATE EXTENSION and CREATE TABLE at startup", async () => { + const pool = makePool(); + await initPgVectorize(pool); + const sqls = (pool as unknown as { _queries: Array<{ sql: string }> })._queries.map((q) => q.sql); + expect(sqls.some((s) => s.includes("CREATE EXTENSION IF NOT EXISTS vector"))).toBe(true); + expect(sqls.some((s) => s.includes("CREATE TABLE IF NOT EXISTS"))).toBe(true); + }); +}); + +describe("createPgVectorize (#980 pgvector RAG)", () => { + let pool: Pool & { _queries: Array<{ sql: string; params: unknown[] }> }; + beforeEach(() => { + pool = makePool() as unknown as Pool & { _queries: Array<{ sql: string; params: unknown[] }> }; + }); + + it("upsert generates INSERT … ON CONFLICT with vector literal", async () => { + const v = createPgVectorize(pool); + // pg-vectorize is cast `as unknown as Vectorize` — read internal shape via unknown cast + await v.upsert([{ id: "v1", values: [0.1, 0.2], namespace: "repo1", metadata: { path: "a.ts" } }]); + const q = pool._queries[0]; + expect(q?.sql).toContain("ON CONFLICT(id)"); + expect(q?.sql).toContain("::vector"); + expect(q?.params[0]).toBe("v1"); + expect(q?.params[1]).toBe("repo1"); + expect(q?.params[2]).toBe("[0.1,0.2]"); + }); + + it("upsert uses empty-string namespace when namespace is absent", async () => { + const v = createPgVectorize(pool); + await v.upsert([{ id: "ns-less", values: [1, 0] }]); + expect(pool._queries[0]?.params[1]).toBe(""); + }); + + it("query with namespace adds WHERE namespace= clause", async () => { + const matchPool = makePool([{ id: "v1", score: 0.95, metadata: null }]); + const v = createPgVectorize(matchPool); + const { matches } = await v.query([0.1, 0.2], { topK: 3, namespace: "n1" }); + expect(matches).toHaveLength(1); + expect(matches[0]?.id).toBe("v1"); + expect(matches[0]?.score).toBeCloseTo(0.95); + const q = (matchPool as unknown as { _queries: Array<{ sql: string }> })._queries[0]; + expect(q?.sql).toContain("namespace=$2"); + }); + + it("query without namespace omits the WHERE clause", async () => { + const matchPool = makePool([{ id: "v2", score: 0.8, metadata: null }]); + const v = createPgVectorize(matchPool); + await v.query([0.1, 0.2], { topK: 5 }); + const q = (matchPool as unknown as { _queries: Array<{ sql: string }> })._queries[0]; + expect(q?.sql).not.toContain("namespace="); + }); + + it("query without topK uses the default of 12", async () => { + const matchPool = makePool([{ id: "v4", score: 0.6, metadata: null }]); + const v = createPgVectorize(matchPool); + await v.query([1, 0], {}); // topK omitted → default 12 + const q = (matchPool as unknown as { _queries: Array<{ sql: string; params: unknown[] }> })._queries[0]; + // The LIMIT param should be 12 (the default) + expect(q?.params).toContain(12); + }); + + it("query maps metadata JSONB rows to Match.metadata", async () => { + const matchPool = makePool([{ id: "v3", score: 0.7, metadata: { path: "x.ts" } }]); + const v = createPgVectorize(matchPool); + const { matches } = await v.query([1, 0], { topK: 1, namespace: "n" }); + expect(matches[0]?.metadata?.path).toBe("x.ts"); + }); + + it("deleteByIds with ids sends DELETE … IN (…) with placeholders", async () => { + const v = createPgVectorize(pool); + await v.deleteByIds(["a", "b", "c"]); + const q = pool._queries[0]; + expect(q?.sql).toContain("IN ($1,$2,$3)"); + expect(q?.params).toEqual(["a", "b", "c"]); + }); + + it("deleteByIds with empty array is a no-op (no query issued)", async () => { + const v = createPgVectorize(pool); + await v.deleteByIds([]); + expect(pool._queries).toHaveLength(0); + }); +}); diff --git a/test/unit/selfhost-puppeteer-stub.test.ts b/test/unit/selfhost-puppeteer-stub.test.ts new file mode 100644 index 0000000000..c9a0ad9a4c --- /dev/null +++ b/test/unit/selfhost-puppeteer-stub.test.ts @@ -0,0 +1,38 @@ +// Tests for the self-host puppeteer stub (#980). Verifies the stub throws the right error when +// BROWSER_WS_ENDPOINT is absent and delegates to puppeteer-core when present. +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +describe("selfhost puppeteer stub (#980 visual review)", () => { + let origEndpoint: string | undefined; + beforeEach(() => { origEndpoint = process.env.BROWSER_WS_ENDPOINT; }); + afterEach(() => { + if (origEndpoint === undefined) delete process.env.BROWSER_WS_ENDPOINT; + else process.env.BROWSER_WS_ENDPOINT = origEndpoint; + vi.resetModules(); + }); + + it("launch() throws browser_rendering_unavailable when BROWSER_WS_ENDPOINT is not set", async () => { + delete process.env.BROWSER_WS_ENDPOINT; + const { default: puppeteer } = await import("../../src/selfhost/stubs/puppeteer"); + await expect(puppeteer.launch({})).rejects.toThrow(/browser_rendering_unavailable_on_selfhost/); + }); + + it("connect() throws browser_rendering_unavailable when BROWSER_WS_ENDPOINT is not set", async () => { + delete process.env.BROWSER_WS_ENDPOINT; + const { default: puppeteer } = await import("../../src/selfhost/stubs/puppeteer"); + await expect(puppeteer.connect({})).rejects.toThrow(/browser_rendering_unavailable_on_selfhost/); + }); + + it("launch() throws browser_rendering_unavailable when puppeteer-core is not installed", async () => { + process.env.BROWSER_WS_ENDPOINT = "ws://fake:3000"; + // puppeteer-core is not in this repo's dependencies — the dynamic import naturally throws "Cannot find package". + const { default: puppeteer } = await import("../../src/selfhost/stubs/puppeteer"); + await expect(puppeteer.launch({})).rejects.toThrow(/browser_rendering_unavailable_on_selfhost/); + }); + + it("connect() throws browser_rendering_unavailable when puppeteer-core is not installed", async () => { + process.env.BROWSER_WS_ENDPOINT = "ws://fake:3000"; + const { default: puppeteer } = await import("../../src/selfhost/stubs/puppeteer"); + await expect(puppeteer.connect({})).rejects.toThrow(/browser_rendering_unavailable_on_selfhost/); + }); +}); diff --git a/test/unit/selfhost-setup-wizard.test.ts b/test/unit/selfhost-setup-wizard.test.ts index fac16bfe2d..d384dc96f7 100644 --- a/test/unit/selfhost-setup-wizard.test.ts +++ b/test/unit/selfhost-setup-wizard.test.ts @@ -37,4 +37,11 @@ describe("setup-wizard (#981 GitHub App Manifest)", () => { const fakeFetch = vi.fn(async () => new Response("e", { status: 422 })) as unknown as typeof fetch; await expect(exchangeManifestCode("x", fakeFetch)).rejects.toThrow(/manifest_exchange_http_422/); }); + + it("credentialsToEnv omits optional OAuth lines when client_id / client_secret are absent", () => { + const env = credentialsToEnv({ id: 1, slug: "s", webhook_secret: "w", pem: "k" }); + expect(env).toContain("GITHUB_APP_ID=1"); + expect(env).not.toContain("GITHUB_OAUTH_CLIENT_ID"); + expect(env).not.toContain("GITHUB_OAUTH_CLIENT_SECRET"); + }); }); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 1962463a27..76712f9246 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -110,6 +110,12 @@ describe("createSqliteQueue (durable #980)", () => { expect(q2.size()).toBe(1); }); + it("stop() is a no-op when start() was never called (timer is null)", async () => { + const q = createSqliteQueue(makeDriver(), async () => undefined); + await q.stop(); // timer=null → the false branch of `if (timer) clearTimeout(timer)` is taken + expect(q.size()).toBe(0); // still usable after a spurious stop() + }); + it("start() is idempotent and stop() waits for an in-flight pump", async () => { let done = false; const q = createSqliteQueue(makeDriver(), async () => { diff --git a/test/unit/selfhost-vectorize.test.ts b/test/unit/selfhost-vectorize.test.ts index 7841a7af81..742b6c4f08 100644 --- a/test/unit/selfhost-vectorize.test.ts +++ b/test/unit/selfhost-vectorize.test.ts @@ -63,4 +63,23 @@ describe("createSqliteVectorize (#979 local RAG)", () => { res = await v.query([0, 1], { topK: 10, namespace: "n" }); expect(res.matches).toHaveLength(0); }); + + it("upsert without a namespace defaults to the empty-string namespace", async () => { + const v = makeVectorize(); + await v.upsert([{ id: "ns-less", values: [1, 0] }]); + // After upserting without namespace, querying with no namespace finds it + const { matches } = await v.query([1, 0], { topK: 1 }); + expect(matches[0]?.id).toBe("ns-less"); + }); + + it("query without a namespace scans the full table", async () => { + const v = makeVectorize(); + await v.upsert([ + { id: "n1", values: [1, 0], namespace: "a" }, + { id: "n2", values: [0, 1], namespace: "b" }, + ]); + const res = await v.query([1, 0], { topK: 10 }); // no namespace → scans all + expect(res.matches.map((m) => m.id)).toContain("n1"); + expect(res.matches.map((m) => m.id)).toContain("n2"); + }); }); From c903e8635c70f12572d47928933135568d386c69 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 24 Jun 2026 01:18:37 -0700 Subject: [PATCH 18/25] fix(selfhost): address Superagent security findings in workflows + Dockerfile - selfhost.yml: add permissions: {contents: read}; pin actions/checkout and actions/setup-node to commit SHAs; set persist-credentials: false - release-selfhost.yml: remove GHA cache-from/cache-to (cache-poisoning P0); route workflow_dispatch version input through an env var to prevent template injection; add environment: release gate; set persist-credentials: false - Dockerfile: pin @anthropic-ai/claude-code and @openai/codex to exact versions; add --ignore-scripts to the optional CLI install - ai.ts: add '--' argument terminator before the prompt in the codex CLI spawn to block argument injection --- .github/workflows/release-selfhost.yml | 10 +++++++--- .github/workflows/selfhost.yml | 10 ++++++++-- Dockerfile | 2 +- src/selfhost/ai.ts | 2 +- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml index ed53309ea8..9b4a7f3a66 100644 --- a/.github/workflows/release-selfhost.yml +++ b/.github/workflows/release-selfhost.yml @@ -24,14 +24,20 @@ jobs: release: runs-on: ubuntu-latest timeout-minutes: 40 + # Environment gate — requires reviewer approval before a release runs (configure under repo Settings > Environments). + environment: release steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false - name: Resolve version id: version + env: + INPUT_VERSION: ${{ github.event.inputs.version }} run: | if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - echo "v=${{ github.event.inputs.version }}" >> "$GITHUB_OUTPUT" + echo "v=${INPUT_VERSION}" >> "$GITHUB_OUTPUT" else echo "v=${GITHUB_REF_NAME#selfhost-v}" >> "$GITHUB_OUTPUT" fi @@ -70,8 +76,6 @@ jobs: labels: ${{ steps.meta.outputs.labels }} provenance: true sbom: true - cache-from: type=gha - cache-to: type=gha,mode=max - name: GitHub Release if: github.event_name == 'push' diff --git a/.github/workflows/selfhost.yml b/.github/workflows/selfhost.yml index 0ba802275f..a9edce95e2 100644 --- a/.github/workflows/selfhost.yml +++ b/.github/workflows/selfhost.yml @@ -28,6 +28,10 @@ on: - "test/unit/selfhost-*" - ".github/workflows/selfhost.yml" +# Least privilege — the smoke test only reads the repo; no writes, no packages. +permissions: + contents: read + jobs: build-boot: name: build + boot smoke test @@ -44,8 +48,10 @@ jobs: options: >- --health-cmd "pg_isready -U postgres" --health-interval 5s --health-timeout 5s --health-retries 10 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version: "24" - name: Install deps diff --git a/Dockerfile b/Dockerfile index 8df13e716e..fa7167a5be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ COPY --from=build /app/migrations ./migrations # work in-image. Build with `--build-arg INSTALL_AI_CLIS=true`. No credentials are baked — operators mint # CLAUDE_CODE_OAUTH_TOKEN (`claude setup-token`) / codex auth at run time and pass it via the env. ARG INSTALL_AI_CLIS=false -RUN if [ "$INSTALL_AI_CLIS" = "true" ]; then npm install -g @anthropic-ai/claude-code @openai/codex; fi +RUN if [ "$INSTALL_AI_CLIS" = "true" ]; then npm install -g @anthropic-ai/claude-code@2.1.187 @openai/codex@0.142.0 --ignore-scripts; fi # Optional: enable visual review via an external Chrome sidecar (e.g. `browserless/chrome:latest`). # Build with `--build-arg INSTALL_VISUAL_REVIEW=true` then set BROWSER_WS_ENDPOINT= at runtime. ARG INSTALL_VISUAL_REVIEW=false diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index 8276c9f92f..a00a089599 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -212,7 +212,7 @@ export function createCodexAi(parentEnv: Record, spa const prompt = toMessages(options).map((m) => m.content).join("\n\n"); const spawn = spawnImpl ?? (await defaultSpawn()); const codexModel = resolveModel(configuredModel(parentEnv), model, "gpt-5"); - const { stdout, code } = await spawn("codex", ["exec", "--json", "--sandbox", "read-only", "--ask-for-approval", "never", "--model", codexModel, prompt], { env, timeoutMs: 120_000 }); + const { stdout, code } = await spawn("codex", ["exec", "--json", "--sandbox", "read-only", "--ask-for-approval", "never", "--model", codexModel, "--", prompt], { env, timeoutMs: 120_000 }); if (code !== 0) throw new Error(`codex_exit_${code ?? "null"}`); const text = extractCliText(stdout); if (!text) throw new Error("codex_empty_output"); From 531db6d89b288a1778ec2ddd1d2f2f2e899ba7e8 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 24 Jun 2026 01:25:46 -0700 Subject: [PATCH 19/25] =?UTF-8?q?ci(selfhost):=20optimize=20build+boot=20w?= =?UTF-8?q?orkflow=20=E2=80=94=20cache,=20deduplicate,=20retry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove unit-test and typecheck steps (covered by main CI validate job) - Add npm dependency caching via setup-node cache: 'npm' - Add docker/setup-buildx-action + GHA layer caching: node:24-slim layer is served from cache on subsequent runs, avoiding Docker Hub transient failures - Wrap Docker build in a 3-attempt retry loop for cold-cache runs - Add test/integration/selfhost-pg* to path triggers --- .github/workflows/selfhost.yml | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/.github/workflows/selfhost.yml b/.github/workflows/selfhost.yml index a9edce95e2..f218aeacbd 100644 --- a/.github/workflows/selfhost.yml +++ b/.github/workflows/selfhost.yml @@ -1,6 +1,8 @@ -# Self-host stack CI (#980/#982). Builds the Node bundle + the Docker image, boots the container, and -# smoke-tests the operational endpoints — the integration coverage the unit tests can't give. Runs on -# Node 24 because node:sqlite (the SQLite backing store) is only stable there. +# Self-host stack CI (#980/#982). Provides integration coverage the main CI can't: +# 1. Postgres integration test — needs a real PG service container +# 2. Self-host bundle build validation (build-selfhost.mjs) +# 3. Docker image build + container smoke test (/health, /ready, /metrics) +# Unit tests and typecheck are NOT duplicated here — the main CI validate job covers them. name: self-host on: @@ -15,6 +17,7 @@ on: - "docker-compose.yml" - "migrations/**" - "test/unit/selfhost-*" + - "test/integration/selfhost-pg*" - ".github/workflows/selfhost.yml" pull_request: paths: @@ -26,6 +29,7 @@ on: - "docker-compose.yml" - "migrations/**" - "test/unit/selfhost-*" + - "test/integration/selfhost-pg*" - ".github/workflows/selfhost.yml" # Least privilege — the smoke test only reads the repo; no writes, no packages. @@ -51,21 +55,35 @@ jobs: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: persist-credentials: false + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version: "24" + cache: "npm" + - name: Install deps run: npm ci --ignore-scripts - - name: Self-host unit tests - run: npx vitest run test/unit/selfhost-*.test.ts + - name: Postgres integration test (real PG) run: PG_TEST_URL=postgres://postgres:devpw@localhost:5432/gittensory npx vitest run test/integration/selfhost-pg.test.ts - - name: Typecheck - run: npx tsc --noEmit + - name: Build the self-host bundle run: node scripts/build-selfhost.mjs + + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - name: Build the Docker image - run: docker build -t gittensory:selfhost-ci . + run: | + for i in 1 2 3; do + docker buildx build \ + --cache-from type=gha \ + --cache-to type=gha,mode=max \ + --load \ + -t gittensory:selfhost-ci . && break + echo "Docker build attempt $i failed, retrying in 15s…" + sleep 15 + done + - name: Boot the container + smoke-test /health, /ready, /metrics, migrations run: | docker run -d --name gt -p 8787:8787 gittensory:selfhost-ci From 09ce00766ef26489b5d50358ff5d68ab2522887b Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 24 Jun 2026 01:28:51 -0700 Subject: [PATCH 20/25] fix(selfhost): pull base image from ECR Public to avoid Docker Hub rate limits Switch FROM node:24-slim to public.ecr.aws/docker/library/node:24-slim in the Dockerfile (both build and runtime stages). ECR Public Gallery mirrors Docker Official Images with no rate limits and no auth, eliminating 503s in CI and operator builds. GHA BuildKit layer cache still applies for fast reruns. --- .github/workflows/selfhost.yml | 14 +++++--------- Dockerfile | 5 +++-- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/selfhost.yml b/.github/workflows/selfhost.yml index f218aeacbd..aa791e38a4 100644 --- a/.github/workflows/selfhost.yml +++ b/.github/workflows/selfhost.yml @@ -74,15 +74,11 @@ jobs: - name: Build the Docker image run: | - for i in 1 2 3; do - docker buildx build \ - --cache-from type=gha \ - --cache-to type=gha,mode=max \ - --load \ - -t gittensory:selfhost-ci . && break - echo "Docker build attempt $i failed, retrying in 15s…" - sleep 15 - done + docker buildx build \ + --cache-from type=gha \ + --cache-to type=gha,mode=max \ + --load \ + -t gittensory:selfhost-ci . - name: Boot the container + smoke-test /health, /ready, /metrics, migrations run: | diff --git a/Dockerfile b/Dockerfile index fa7167a5be..684c37b7be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,8 @@ # the .env file or mounted *_FILE secrets (see docker-compose.yml + .env.example). # --- build: install deps + bundle the Node entry -------------------------------------------------------- -FROM node:24-slim AS build +# ECR Public Gallery mirrors Docker Official Images with no rate limits and no auth. +FROM public.ecr.aws/docker/library/node:24-slim AS build WORKDIR /app COPY package*.json ./ # --ignore-scripts: no native builds are needed (SQLite is the built-in node:sqlite; @hono/node-server is @@ -16,7 +17,7 @@ COPY . . RUN node scripts/build-selfhost.mjs --all # --- runtime: slim, non-root ---------------------------------------------------------------------------- -FROM node:24-slim AS runtime +FROM public.ecr.aws/docker/library/node:24-slim AS runtime WORKDIR /app ENV NODE_ENV=production \ PLATFORM=self-hosted \ From 3ec6239ad0e777d7bd16c6d43bfb7076a5efb3ea Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 24 Jun 2026 02:08:20 -0700 Subject: [PATCH 21/25] feat(selfhost): production-grade compose profiles, observability, and deploy options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docker Compose profile system (#1199): 9 optional profiles activated via `--profile` — postgres, pgbouncer, redis, ollama, litestream, caddy, observability, tailscale, runners. Operators compose exactly the stack they need; the core service always runs profile-free. Caddy (#1203): HTTPS reverse proxy with auto-TLS via Let's Encrypt; zstd/gzip compression; security headers (HSTS, X-Frame-Options, X-Content-Type-Options, Referrer-Policy). Prometheus + Grafana (#1206): pre-wired observability at --profile observability; 15s scrape interval; pre-provisioned datasource and a Grafana dashboard (queue depth, dead-letter count, HTTP request rate, job throughput by status). Tailscale sidecar (#1204): --profile tailscale exposes the instance on an operator's tailnet without any public firewall rules; TS_AUTHKEY + persistent state volume. Self-hosted GitHub Actions runner (#1205): --profile runners registers a runner against any repo/org; mounts /var/run/docker.sock for in-runner Docker builds. Terraform (Hetzner) (#1209): cx22 VPS + 20 GB volume + firewall; cloud-init installs Docker from the official apt repo; one terraform apply to a ready-to-clone host. Railway template (#1210): railway.json with Dockerfile builder, /health check, ON_FAILURE restart policy — one-click deploy, eligible for Railway Template Marketplace creator revenue. Worker concurrency (#1201): QUEUE_CONCURRENCY env var (default 1) controls how many concurrent pump() loops run per instance; both SQLite and Postgres queues updated; stop()/drain() wait for active === 0. Structured audit log (#1202): logAudit() writes one JSON line per job lifecycle event (job_complete, job_dead, job_error) to stdout — level, ts, job_id, payload_type, latency_ms, attempts, error — captured by Docker's json-file driver with zero operator configuration. Coverage: new selfhost-audit.test.ts + selfhost-pg-queue.test.ts (fills the pre-existing gap referenced by selfhost-pg-vectorize.test.ts); concurrency branch tests in selfhost-sqlite-queue.test.ts; all gates green at 97.06% line / 94.81% branch. --- .env.example | 21 ++ caddy/Caddyfile | 34 +++ docker-compose.yml | 270 ++++++++++++++---- grafana/dashboards/gittensory.json | 116 ++++++++ grafana/provisioning/dashboards/provider.yml | 9 + .../provisioning/datasources/prometheus.yml | 8 + prometheus/prometheus.yml | 13 + railway.json | 15 + src/selfhost/audit.ts | 32 +++ src/selfhost/pg-queue.ts | 22 +- src/selfhost/sqlite-queue.ts | 26 +- terraform/main.tf | 137 +++++++++ terraform/outputs.tf | 19 ++ terraform/variables.tf | 34 +++ test/unit/selfhost-audit.test.ts | 70 +++++ test/unit/selfhost-pg-queue.test.ts | 216 ++++++++++++++ test/unit/selfhost-sqlite-queue.test.ts | 39 ++- 17 files changed, 1009 insertions(+), 72 deletions(-) create mode 100644 caddy/Caddyfile create mode 100644 grafana/dashboards/gittensory.json create mode 100644 grafana/provisioning/dashboards/provider.yml create mode 100644 grafana/provisioning/datasources/prometheus.yml create mode 100644 prometheus/prometheus.yml create mode 100644 railway.json create mode 100644 src/selfhost/audit.ts create mode 100644 terraform/main.tf create mode 100644 terraform/outputs.tf create mode 100644 terraform/variables.tf create mode 100644 test/unit/selfhost-audit.test.ts create mode 100644 test/unit/selfhost-pg-queue.test.ts diff --git a/.env.example b/.env.example index 992b9546f3..65ae9be527 100644 --- a/.env.example +++ b/.env.example @@ -119,6 +119,27 @@ GITTENSORY_REVIEW_DRAFT=false # LITESTREAM_ENDPOINT= # e.g. s3.us-west-002.backblazeb2.com (omit for AWS S3) # LITESTREAM_REGION=us-east-1 +# --- Queue worker (#977/#1201) --- +# QUEUE_CONCURRENCY=1 # max concurrent job-processing loops per instance (default 1) + +# --- Caddy HTTPS terminator (#1203; requires --profile caddy) --- +# DOMAIN=gittensory.example.com # fully-qualified domain; Caddy auto-obtains a Let's Encrypt cert + +# --- Tailscale sidecar (#1204; requires --profile tailscale) --- +# TS_AUTHKEY= # Tailscale auth key (generate at tailscale.com/admin/settings/keys) +# TS_EXTRA_ARGS= # extra tailscale up flags, e.g. --advertise-tags=tag:self-host + +# --- Self-hosted GitHub Actions runner (#1205; requires --profile runners) --- +# RUNNER_TOKEN= # runner registration token (Settings → Actions → Runners → New) +# RUNNER_REPO_URL=https://github.com/org/repo +# RUNNER_ACCESS_TOKEN= # PAT with repo scope (alternative to RUNNER_TOKEN) +# RUNNER_SCOPE=repo # repo | org | enterprise +# RUNNER_NAME=gittensory-runner +# RUNNER_LABELS=self-hosted,linux + +# --- Grafana (#1206; requires --profile observability) --- +# GRAFANA_ADMIN_PASSWORD=changeme # Grafana admin password (change before exposing publicly) + # --- AI review backend (optional; without it reviews run deterministically) --- # AI_SUMMARIES_ENABLED=true # AI_PROVIDER=ollama # ollama | openai-compatible | openai | anthropic | claude-code | diff --git a/caddy/Caddyfile b/caddy/Caddyfile new file mode 100644 index 0000000000..197bb96683 --- /dev/null +++ b/caddy/Caddyfile @@ -0,0 +1,34 @@ +# Caddy reverse proxy for gittensory (#980 self-host). +# Activated via: docker compose --profile caddy up +# +# DOMAIN is injected from the DOMAIN env var in docker-compose.yml. +# Set DOMAIN=reviews.yourcompany.com in .env — Caddy fetches a TLS cert from Let's Encrypt automatically. +# For local testing without a domain, set DOMAIN=localhost (self-signed cert, browser will warn). +# +# When using this profile, remove the `ports:` entry from the gittensory service in docker-compose.yml +# so port 8787 is NOT exposed publicly — all traffic should flow through Caddy on 443. + +{$DOMAIN} { + reverse_proxy gittensory:8787 { + # Surface the real client IP to the app (logged in access events). + header_up X-Forwarded-For {remote_host} + header_up X-Real-IP {remote_host} + } + + # Compress responses. + encode zstd gzip + + # Security headers. + header { + Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" + X-Content-Type-Options "nosniff" + X-Frame-Options "DENY" + Referrer-Policy "strict-origin-when-cross-origin" + -Server + } + + log { + output stderr + format json + } +} diff --git a/docker-compose.yml b/docker-compose.yml index f398b01c68..989ed888a1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,28 +1,51 @@ -# One-command self-host for gittensory (#980): docker compose up --build +# One-command self-host for gittensory (#980): docker compose up --build # -# SECRETS: never baked into the image. Copy .env.example -> .env and fill it in (the file is gitignored). -# AI is OPTIONAL — without it the review path degrades gracefully (no AI summaries); enable Ollama (below) -# or set an OpenAI-compatible / subscription provider in .env to turn it on. The SQLite DB + all 56 schema -# migrations live on the `gittensory-data` volume and are applied automatically at startup. +# SECRETS: copy .env.example → .env, fill in your GitHub App credentials and any optional secrets. +# The file is gitignored. Every value below is a sample placeholder — never commit real secrets. +# +# PROFILES — activate optional services by passing --profile (combine freely): +# +# (none) SQLite single-node stack (default — no flags needed) +# --profile postgres pgvector/pg16 shared database (multi-instance capable) +# --profile pgbouncer PgBouncer connection pooler in front of Postgres +# --profile redis Redis fixed-window rate limiter +# --profile ollama Local Ollama AI backend +# --profile litestream Continuous SQLite backup to S3/B2/R2 via Litestream +# --profile caddy Caddy HTTPS terminator with auto-TLS (set DOMAIN= in .env) +# --profile observability Prometheus + Grafana dashboards (pre-wired to /metrics) +# --profile tailscale Tailscale sidecar — access the stack via your tailnet +# --profile runners GitHub Actions self-hosted runner +# +# Examples: +# docker compose up --build # SQLite, no AI +# docker compose --profile postgres --profile caddy up -d # Postgres + HTTPS +# docker compose --profile observability up -d # add dashboards to anything +# docker compose --profile tailscale --profile runners up -d # tailnet + CI runners + services: + + # ── Core app (always runs) ───────────────────────────────────────────────── gittensory: build: context: . restart: unless-stopped ports: - - "8787:8787" + # Remove this when using the caddy profile — Caddy becomes the public listener. + - "${PORT:-8787}:8787" env_file: - # SAMPLE config — `cp .env.example .env` first, then fill in your GitHub App + tokens. - path: .env required: false environment: PORT: "8787" DATABASE_PATH: /data/gittensory.sqlite - # Scale out: uncomment the postgres + redis services below, then point at them here (a shared Postgres + - # Redis lets you run multiple replicas of this service behind a load balancer): - # DATABASE_URL: postgres://gittensory:CHANGEME@postgres:5432/gittensory + # Uncomment the next two lines and activate --profile postgres to use Postgres: + # DATABASE_URL: postgres://gittensory:${POSTGRES_PASSWORD:-CHANGEME}@postgres:5432/gittensory + # PGVECTOR_ENABLED: "true" + # With --profile pgbouncer, route through the pooler instead of postgres directly: + # DATABASE_URL: postgres://gittensory:${POSTGRES_PASSWORD:-CHANGEME}@pgbouncer:5432/gittensory + # Uncomment for Redis rate limiting (--profile redis): # REDIS_URL: redis://redis:6379 - # Point at the Ollama service below to enable local AI review (uncomment the ollama service too): + # Uncomment for Ollama AI (--profile ollama): # AI_PROVIDER: ollama # AI_BASE_URL: http://ollama:11434/v1 volumes: @@ -34,55 +57,188 @@ services: start_period: 20s retries: 3 - # Optional local AI backend. Uncomment + set AI_PROVIDER=ollama / AI_BASE_URL above, then once up: - # docker compose exec ollama ollama pull - # ollama: - # image: ollama/ollama:latest - # volumes: - # - ollama-models:/root/.ollama + # ── Postgres (--profile postgres | --profile pgbouncer) ─────────────────── + postgres: + image: pgvector/pgvector:pg16 + restart: unless-stopped + profiles: ["postgres", "pgbouncer"] + environment: + POSTGRES_USER: gittensory + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-CHANGEME} + POSTGRES_DB: gittensory + volumes: + - gittensory-pg:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U gittensory"] + interval: 10s + retries: 5 + + # ── PgBouncer (--profile pgbouncer) ─────────────────────────────────────── + # Transaction-mode pooler — allows hundreds of app clients with a small PG connection cap. + # Set DATABASE_URL in gittensory to: postgres://gittensory:@pgbouncer:5432/gittensory + pgbouncer: + image: bitnami/pgbouncer:1 + restart: unless-stopped + profiles: ["pgbouncer"] + depends_on: + postgres: + condition: service_healthy + environment: + POSTGRESQL_HOST: postgres + POSTGRESQL_PORT: "5432" + POSTGRESQL_DATABASE: gittensory + POSTGRESQL_USERNAME: gittensory + POSTGRESQL_PASSWORD: ${POSTGRES_PASSWORD:-CHANGEME} + PGBOUNCER_DATABASE: gittensory + PGBOUNCER_PORT: "5432" + PGBOUNCER_POOL_MODE: transaction + PGBOUNCER_MAX_CLIENT_CONN: "200" + PGBOUNCER_DEFAULT_POOL_SIZE: "20" + PGBOUNCER_AUTH_TYPE: scram-sha-256 + + # ── Redis (--profile redis) ──────────────────────────────────────────────── + redis: + image: redis:7-alpine + restart: unless-stopped + profiles: ["redis"] + volumes: + - gittensory-redis:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + retries: 5 + + # ── Ollama (--profile ollama) ────────────────────────────────────────────── + # After `docker compose --profile ollama up -d`, pull a model: + # docker compose exec ollama ollama pull llama3.2 + # Then set AI_PROVIDER=ollama and AI_BASE_URL=http://ollama:11434/v1 in .env. + ollama: + image: ollama/ollama:latest + restart: unless-stopped + profiles: ["ollama"] + volumes: + - ollama-models:/root/.ollama + + # ── Litestream (--profile litestream) ───────────────────────────────────── + # Continuous WAL backup of the SQLite DB to S3/B2/R2. Copy litestream.yml.example + # → litestream.yml and fill in your bucket. Set LITESTREAM_* secrets in .env. + litestream: + image: litestream/litestream:latest + restart: unless-stopped + profiles: ["litestream"] + command: replicate + depends_on: + gittensory: + condition: service_healthy + volumes: + - gittensory-data:/data + - ./litestream.yml:/etc/litestream.yml:ro + environment: + LITESTREAM_ACCESS_KEY_ID: ${LITESTREAM_ACCESS_KEY_ID} + LITESTREAM_SECRET_ACCESS_KEY: ${LITESTREAM_SECRET_ACCESS_KEY} + LITESTREAM_ENDPOINT: ${LITESTREAM_ENDPOINT:-} + LITESTREAM_REGION: ${LITESTREAM_REGION:-us-east-1} + + # ── Caddy (--profile caddy) ──────────────────────────────────────────────── + # Auto-TLS via Let's Encrypt. Set DOMAIN=reviews.yourcompany.com in .env, then + # remove the plain `ports:` entry from the gittensory service above. + caddy: + image: caddy:2-alpine + restart: unless-stopped + profiles: ["caddy"] + ports: + - "80:80" + - "443:443" + - "443:443/udp" # HTTP/3 QUIC + volumes: + - ./caddy/Caddyfile:/etc/caddy/Caddyfile:ro + - caddy-data:/data + - caddy-config:/config + environment: + DOMAIN: ${DOMAIN:-localhost} + depends_on: + gittensory: + condition: service_healthy - # Optional: continuous SQLite backup to S3/B2/MinIO/R2 via Litestream (https://litestream.io). - # Copy litestream.yml.example -> litestream.yml, fill in your bucket, then uncomment: - # litestream: - # image: litestream/litestream:latest - # command: replicate - # restart: unless-stopped - # depends_on: [gittensory] - # volumes: - # - gittensory-data:/data - # - ./litestream.yml:/etc/litestream.yml:ro - # environment: - # LITESTREAM_ACCESS_KEY_ID: ${LITESTREAM_ACCESS_KEY_ID} - # LITESTREAM_SECRET_ACCESS_KEY: ${LITESTREAM_SECRET_ACCESS_KEY} - # LITESTREAM_ENDPOINT: ${LITESTREAM_ENDPOINT} - # LITESTREAM_REGION: ${LITESTREAM_REGION} + # ── Observability (--profile observability) ──────────────────────────────── + # Prometheus scrapes /metrics; Grafana visualises it. + # Grafana UI: http://localhost:3000 (admin / admin — change on first login). + prometheus: + image: prom/prometheus:latest + restart: unless-stopped + profiles: ["observability"] + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=30d" - # Optional: Postgres backend (shared DB → multi-instance). Uncomment, set DATABASE_URL above, and add - # `depends_on: [postgres]` to the gittensory service. Uses pgvector/pgvector:pg16 (drop-in for - # postgres:16-alpine) — also set PGVECTOR_ENABLED=true in the gittensory environment to enable RAG on Postgres. - # postgres: - # image: pgvector/pgvector:pg16 - # restart: unless-stopped - # environment: - # POSTGRES_USER: gittensory - # POSTGRES_PASSWORD: CHANGEME # SAMPLE — set your own - # POSTGRES_DB: gittensory - # volumes: - # - gittensory-pg:/var/lib/postgresql/data - # healthcheck: - # test: ["CMD-SHELL", "pg_isready -U gittensory"] - # interval: 10s - # retries: 5 + grafana: + image: grafana/grafana:latest + restart: unless-stopped + profiles: ["observability"] + depends_on: [prometheus] + ports: + - "3000:3000" + volumes: + - grafana-data:/var/lib/grafana + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + environment: + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin} + GF_USERS_ALLOW_SIGN_UP: "false" + + # ── Tailscale (--profile tailscale) ─────────────────────────────────────── + # Joins your tailnet so the stack is accessible via Tailscale IP/hostname — no + # public ports exposed. Generate an auth key at tailscale.com/settings/keys and + # set TS_AUTHKEY= in .env. The gittensory service is reachable at the tailnet IP on port 8787. + tailscale: + image: ghcr.io/tailscale/tailscale:stable + restart: unless-stopped + profiles: ["tailscale"] + hostname: gittensory + cap_add: + - NET_ADMIN + - SYS_MODULE + environment: + TS_AUTHKEY: ${TS_AUTHKEY} + TS_STATE_DIR: /var/lib/tailscale + TS_EXTRA_ARGS: ${TS_EXTRA_ARGS:-} + volumes: + - tailscale-state:/var/lib/tailscale + - /dev/net/tun:/dev/net/tun + network_mode: host # Tailscale needs host networking to advertise the host's address - # Optional: Redis (distributed rate limiter). Uncomment + set REDIS_URL above. - # redis: - # image: redis:7-alpine - # restart: unless-stopped - # volumes: - # - gittensory-redis:/data + # ── Self-hosted GitHub Actions runner (--profile runners) ───────────────── + # Runs `runs-on: self-hosted` jobs on this machine. Set RUNNER_TOKEN= (or ACCESS_TOKEN=) + # and RUNNER_REPO_URL= (e.g. https://github.com/your-org/your-repo) in .env. + # Get a token at: https://github.com///settings/actions/runners/new + runner: + image: myoung34/github-runner:latest + restart: unless-stopped + profiles: ["runners"] + environment: + RUNNER_SCOPE: ${RUNNER_SCOPE:-repo} + REPO_URL: ${RUNNER_REPO_URL} + RUNNER_TOKEN: ${RUNNER_TOKEN} + ACCESS_TOKEN: ${RUNNER_ACCESS_TOKEN:-} + RUNNER_NAME: ${RUNNER_NAME:-gittensory-runner} + LABELS: ${RUNNER_LABELS:-self-hosted,linux,x64} + RUNNER_WORKDIR: /tmp/runner + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - runner-work:/tmp/runner volumes: gittensory-data: - # ollama-models: - # gittensory-pg: - # gittensory-redis: + gittensory-pg: + gittensory-redis: + ollama-models: + caddy-data: + caddy-config: + prometheus-data: + grafana-data: + tailscale-state: + runner-work: diff --git a/grafana/dashboards/gittensory.json b/grafana/dashboards/gittensory.json new file mode 100644 index 0000000000..8bf8d9cb85 --- /dev/null +++ b/grafana/dashboards/gittensory.json @@ -0,0 +1,116 @@ +{ + "__inputs": [], + "__requires": [ + { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.0.0" }, + { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } + ], + "annotations": { "list": [] }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 }, + "id": 1, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Uptime", + "type": "stat", + "targets": [{ "expr": "gittensory_uptime_seconds", "legendFormat": "uptime" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 10 }, { "color": "red", "value": 50 }] }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 }, + "id": 2, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Queue Pending", + "type": "stat", + "targets": [{ "expr": "gittensory_queue_pending", "legendFormat": "pending" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 10 }] }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 }, + "id": 3, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Dead-Letter Jobs", + "type": "stat", + "targets": [{ "expr": "gittensory_queue_dead", "legendFormat": "dead" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "reqps" } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, + "id": 4, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "HTTP Request Rate", + "type": "timeseries", + "targets": [{ "expr": "rate(gittensory_http_requests_total[2m])", "legendFormat": "requests/s" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "ops" } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, + "id": 5, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "Job Throughput", + "type": "timeseries", + "targets": [ + { "expr": "rate(gittensory_jobs_processed_total[2m])", "legendFormat": "processed/s" }, + { "expr": "rate(gittensory_jobs_enqueued_total[2m])", "legendFormat": "enqueued/s" }, + { "expr": "rate(gittensory_jobs_failed_total[2m])", "legendFormat": "failed/s" }, + { "expr": "rate(gittensory_jobs_dead_total[2m])", "legendFormat": "dead/s" } + ] + } + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["gittensory", "self-host"], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "type": "datasource" + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Gittensory Self-Host", + "uid": "gittensory-selfhost", + "version": 1 +} diff --git a/grafana/provisioning/dashboards/provider.yml b/grafana/provisioning/dashboards/provider.yml new file mode 100644 index 0000000000..80cbbbc20f --- /dev/null +++ b/grafana/provisioning/dashboards/provider.yml @@ -0,0 +1,9 @@ +apiVersion: 1 +providers: + - name: gittensory + folder: Gittensory + type: file + disableDeletion: true + editable: false + options: + path: /var/lib/grafana/dashboards diff --git a/grafana/provisioning/datasources/prometheus.yml b/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000000..2d433996f6 --- /dev/null +++ b/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,8 @@ +apiVersion: 1 +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml new file mode 100644 index 0000000000..e2de3c51c2 --- /dev/null +++ b/prometheus/prometheus.yml @@ -0,0 +1,13 @@ +# Prometheus scrape config for gittensory self-host (#980 observability). +# Activated via: docker compose --profile observability up +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: gittensory + static_configs: + - targets: ["gittensory:8787"] + metrics_path: /metrics + scrape_interval: 15s + scrape_timeout: 10s diff --git a/railway.json b/railway.json new file mode 100644 index 0000000000..f9bde10ee2 --- /dev/null +++ b/railway.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://railway.com/railway.schema.json", + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "./Dockerfile" + }, + "deploy": { + "healthcheckPath": "/health", + "healthcheckTimeout": 60, + "restartPolicyType": "ON_FAILURE", + "restartPolicyMaxRetries": 5, + "drainingSeconds": 15, + "numReplicas": 1 + } +} diff --git a/src/selfhost/audit.ts b/src/selfhost/audit.ts new file mode 100644 index 0000000000..096abea0fa --- /dev/null +++ b/src/selfhost/audit.ts @@ -0,0 +1,32 @@ +// Structured audit log for the self-host runtime (#980). Emits one JSON line per job lifecycle event so +// operators can grep / pipe to their log aggregator (Loki, CloudWatch, Datadog, etc.) without any extra +// setup. Written to process.stdout so it is captured by Docker's default json-file log driver and is +// accessible via `docker compose logs gittensory`. + +export type AuditEventType = "job_complete" | "job_dead" | "job_error"; + +export interface AuditEvent { + event: AuditEventType; + ts: number; // Unix timestamp (ms) + job_id: number | string; + payload_type?: string | undefined; // top-level `type` field from the job payload, if present + latency_ms: number; // wall time from claim to completion/failure + attempts: number; // total attempts consumed (1 = first-try success) + error?: string; // last error message, present for job_dead / job_error +} + +/** Emit a single audit event as a JSON line on stdout. */ +export function logAudit(ev: AuditEvent): void { + process.stdout.write(JSON.stringify({ level: "audit", ...ev }) + "\n"); +} + +/** Extract a `type` label from a raw job payload string without fully parsing it. Returns undefined + * if the payload is not a JSON object or lacks a top-level `type` string. */ +export function extractPayloadType(payload: string): string | undefined { + try { + const o = JSON.parse(payload) as Record; + return typeof o.type === "string" ? o.type : undefined; + } catch { + return undefined; + } +} diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index b25b52e85b..ce21a5cbc8 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -3,6 +3,7 @@ // app instances sharing one Postgres can claim jobs concurrently without double-processing. size()/deadCount() // are async (the metrics gauges accept async samplers). import type { Pool } from "pg"; +import { logAudit, extractPayloadType } from "./audit"; import { incr } from "./metrics"; import type { JobMessage } from "../types"; @@ -39,15 +40,18 @@ export interface PgQueueOptions { maxRetries?: number; pollIntervalMs?: number; backoffMs?: (attempt: number) => number; + /** Max concurrent `processOne()` loops. Defaults to QUEUE_CONCURRENCY env var or 1. */ + concurrency?: number; } export function createPgQueue(pool: Pool, consume: (message: JobMessage) => Promise, opts: PgQueueOptions = {}): PgDurableQueue { const maxRetries = opts.maxRetries ?? 5; const pollIntervalMs = opts.pollIntervalMs ?? 1000; const backoff = opts.backoffMs ?? ((attempt: number) => Math.min(60_000, 1000 * 2 ** attempt)); + const concurrency = opts.concurrency ?? Math.max(1, Number(process.env.QUEUE_CONCURRENCY ?? "1")); let running = false; - let pumping = false; + let active = 0; let timer: ReturnType | null = null; async function init(): Promise { @@ -77,18 +81,21 @@ export function createPgQueue(pool: Pool, consume: (message: JobMessage) => Prom async function processOne(): Promise { const job = await claimNext(); if (!job) return false; + const claimedAt = Date.now(); let message: JobMessage; try { message = JSON.parse(job.payload) as JobMessage; } catch { await pool.query(`UPDATE ${TABLE} SET status='dead', last_error='unparseable payload' WHERE id=$1`, [job.id]); incr("gittensory_jobs_dead_total"); + logAudit({ event: "job_dead", ts: Date.now(), job_id: job.id, latency_ms: Date.now() - claimedAt, attempts: Number(job.attempts) + 1, error: "unparseable payload" }); return true; } try { await consume(message); await pool.query(`DELETE FROM ${TABLE} WHERE id=$1`, [job.id]); incr("gittensory_jobs_processed_total"); + logAudit({ event: "job_complete", ts: Date.now(), job_id: job.id, payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts: Number(job.attempts) + 1 }); } catch (error) { const attempts = Number(job.attempts) + 1; const errMsg = error instanceof Error ? error.message : "unknown error"; @@ -97,22 +104,24 @@ export function createPgQueue(pool: Pool, consume: (message: JobMessage) => Prom await pool.query(`UPDATE ${TABLE} SET status='dead', attempts=$1, last_error=$2 WHERE id=$3`, [attempts, errMsg, job.id]); incr("gittensory_jobs_dead_total"); console.error(JSON.stringify({ level: "error", event: "selfhost_job_dead", id: job.id, attempts, error: errMsg })); + logAudit({ event: "job_dead", ts: Date.now(), job_id: job.id, payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts, error: errMsg }); } else { await pool.query(`UPDATE ${TABLE} SET status='pending', attempts=$1, run_after=$2, last_error=$3 WHERE id=$4`, [attempts, Date.now() + backoff(attempts), errMsg, job.id]); + logAudit({ event: "job_error", ts: Date.now(), job_id: job.id, payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts, error: errMsg }); } } return true; } async function pump(): Promise { - if (pumping) return; - pumping = true; + if (active >= concurrency) return; + active++; try { while (await processOne()) { /* drain due jobs */ } } finally { - pumping = false; + active--; } } @@ -132,6 +141,7 @@ export function createPgQueue(pool: Pool, consume: (message: JobMessage) => Prom if (running) return; running = true; const tick = (): void => { + /* v8 ignore next */ // stop() clears the timer before the next tick can fire with running=false if (!running) return; void pump().finally(() => { if (running) timer = setTimeout(tick, pollIntervalMs); @@ -142,10 +152,10 @@ export function createPgQueue(pool: Pool, consume: (message: JobMessage) => Prom async stop() { running = false; if (timer) clearTimeout(timer); - while (pumping) await new Promise((r) => setTimeout(r, 10)); + while (active > 0) await new Promise((r) => setTimeout(r, 10)); }, async drain() { - while (pumping) await new Promise((r) => setTimeout(r, 5)); + while (active > 0) await new Promise((r) => setTimeout(r, 5)); await pump(); }, async size() { diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index 37e595adf1..898f35ef91 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -4,6 +4,7 @@ // backing store differs. Single-process model: node:sqlite is synchronous + serial, so claim (SELECT→UPDATE) // is atomic with no row-lock dance. import type { SqliteDriver } from "./d1-adapter"; +import { logAudit, extractPayloadType } from "./audit"; import { incr } from "./metrics"; import type { JobMessage } from "../types"; @@ -39,12 +40,15 @@ export interface SqliteQueueOptions { maxRetries?: number; pollIntervalMs?: number; backoffMs?: (attempt: number) => number; + /** Max concurrent `processOne()` loops. Defaults to QUEUE_CONCURRENCY env var or 1. */ + concurrency?: number; } export function createSqliteQueue(driver: SqliteDriver, consume: (message: JobMessage) => Promise, opts: SqliteQueueOptions = {}): DurableQueue { const maxRetries = opts.maxRetries ?? 5; const pollIntervalMs = opts.pollIntervalMs ?? 1000; const backoff = opts.backoffMs ?? ((attempt: number) => Math.min(60_000, 1000 * 2 ** attempt)); + const concurrency = opts.concurrency ?? Math.max(1, Number(process.env.QUEUE_CONCURRENCY ?? "1")); driver.exec(DDL); // Recover jobs a crashed previous run left mid-flight → make them claimable again. @@ -52,7 +56,7 @@ export function createSqliteQueue(driver: SqliteDriver, consume: (message: JobMe if (recovered) console.log(JSON.stringify({ event: "selfhost_queue_recovered", count: recovered })); let running = false; - let pumping = false; + let active = 0; // number of concurrent pump() loops currently draining jobs let timer: ReturnType | null = null; function enqueue(message: JobMessage, delaySeconds: number): void { @@ -74,18 +78,21 @@ export function createSqliteQueue(driver: SqliteDriver, consume: (message: JobMe async function processOne(): Promise { const job = claimNext(); if (!job) return false; + const claimedAt = Date.now(); let message: JobMessage; try { message = JSON.parse(job.payload) as JobMessage; } catch { driver.query(`UPDATE ${TABLE} SET status='dead', last_error='unparseable payload' WHERE id=?`, [job.id]); incr("gittensory_jobs_dead_total"); + logAudit({ event: "job_dead", ts: Date.now(), job_id: job.id, latency_ms: Date.now() - claimedAt, attempts: job.attempts + 1, error: "unparseable payload" }); return true; } try { await consume(message); driver.query(`DELETE FROM ${TABLE} WHERE id=?`, [job.id]); incr("gittensory_jobs_processed_total"); + logAudit({ event: "job_complete", ts: Date.now(), job_id: job.id, payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts: job.attempts + 1 }); } catch (error) { const attempts = job.attempts + 1; const errMsg = error instanceof Error ? error.message : "unknown error"; @@ -94,24 +101,27 @@ export function createSqliteQueue(driver: SqliteDriver, consume: (message: JobMe driver.query(`UPDATE ${TABLE} SET status='dead', attempts=?, last_error=? WHERE id=?`, [attempts, errMsg, job.id]); incr("gittensory_jobs_dead_total"); console.error(JSON.stringify({ level: "error", event: "selfhost_job_dead", id: job.id, attempts, error: errMsg })); + logAudit({ event: "job_dead", ts: Date.now(), job_id: job.id, payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts, error: errMsg }); } else { driver.query(`UPDATE ${TABLE} SET status='pending', attempts=?, run_after=?, last_error=? WHERE id=?`, [attempts, Date.now() + backoff(attempts), errMsg, job.id]); + logAudit({ event: "job_error", ts: Date.now(), job_id: job.id, payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts, error: errMsg }); } } return true; } // Drains every job that is currently DUE. A retry is rescheduled into the future (run_after > now) so it is - // not re-claimed here — the next poll tick picks it up — which also bounds this loop. + // not re-claimed here — the next poll tick picks it up — which also bounds this loop. Up to `concurrency` + // pump loops may run simultaneously (each claims its own job row, atomic under node:sqlite's serial writes). async function pump(): Promise { - if (pumping) return; - pumping = true; + if (active >= concurrency) return; + active++; try { while (await processOne()) { /* keep draining due jobs */ } } finally { - pumping = false; + active--; } } @@ -141,11 +151,11 @@ export function createSqliteQueue(driver: SqliteDriver, consume: (message: JobMe async stop() { running = false; if (timer) clearTimeout(timer); - while (pumping) await new Promise((r) => setTimeout(r, 10)); // let an in-flight pump finish + while (active > 0) await new Promise((r) => setTimeout(r, 10)); // let in-flight pumps finish }, async drain() { - // send() fire-and-forgets a pump; wait for any in-flight pump to settle, then drain to completion. - while (pumping) await new Promise((r) => setTimeout(r, 5)); + // send() fire-and-forgets a pump; wait for any in-flight pumps to settle, then drain to completion. + while (active > 0) await new Promise((r) => setTimeout(r, 5)); await pump(); }, size() { diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000000..66b09b0480 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,137 @@ +# Terraform config for a Hetzner Cloud VPS running the gittensory self-host stack. +# Provisions a single server with Docker + Docker Compose pre-installed via cloud-init. +# After provisioning: SSH in, clone the repo, copy .env.example → .env, and run +# `docker compose up -d` (or `docker compose --profile postgres --profile caddy up -d`). + +terraform { + required_version = ">= 1.6" + required_providers { + hcloud = { + source = "hetznercloud/hcloud" + version = "~> 1.49" + } + } +} + +provider "hcloud" { + token = var.hcloud_token +} + +# ── SSH key ──────────────────────────────────────────────────────────────────── +resource "hcloud_ssh_key" "gittensory" { + name = "gittensory-deploy" + public_key = var.ssh_public_key +} + +# ── Firewall ─────────────────────────────────────────────────────────────────── +resource "hcloud_firewall" "gittensory" { + name = "gittensory" + + # SSH — tighten source_ips to your IP range in production + rule { + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.admin_ip_allowlist + } + + # HTTP (Caddy ACME challenge + redirect) + rule { + direction = "in" + protocol = "tcp" + port = "80" + source_ips = ["0.0.0.0/0", "::/0"] + } + + # HTTPS + rule { + direction = "in" + protocol = "tcp" + port = "443" + source_ips = ["0.0.0.0/0", "::/0"] + } + + # HTTP/3 QUIC (used by Caddy when the caddy profile is active) + rule { + direction = "in" + protocol = "udp" + port = "443" + source_ips = ["0.0.0.0/0", "::/0"] + } + + # Direct app access — remove once behind Caddy + rule { + direction = "in" + protocol = "tcp" + port = "8787" + source_ips = var.admin_ip_allowlist + } +} + +# ── Persistent volume for /data (SQLite DB + Litestream WAL) ────────────────── +resource "hcloud_volume" "gittensory_data" { + name = "gittensory-data" + size = var.volume_size_gb + location = var.location + format = "ext4" + automount = false +} + +# ── Server ───────────────────────────────────────────────────────────────────── +resource "hcloud_server" "gittensory" { + name = "gittensory" + server_type = var.server_type + image = "ubuntu-24.04" + location = var.location + ssh_keys = [hcloud_ssh_key.gittensory.id] + firewall_ids = [hcloud_firewall.gittensory.id] + keep_disk = true + + user_data = <<-CLOUDINIT + #cloud-config + package_update: true + package_upgrade: true + + packages: + - ca-certificates + - curl + - gnupg + - git + - jq + + runcmd: + # Install Docker from the official apt repository + - install -m 0755 -d /etc/apt/keyrings + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg + - chmod a+r /etc/apt/keyrings/docker.gpg + - | + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo $VERSION_CODENAME) stable" \ + > /etc/apt/sources.list.d/docker.list + - apt-get update -y + - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + - systemctl enable --now docker + # Mount the attached volume at /data + - mkdir -p /data + - | + DEVICE=$(lsblk -o NAME,SERIAL -dpn | grep $(echo "${hcloud_volume.gittensory_data.linux_device}" | sed 's|/dev/||') | awk '{print $1}') + mount /dev/$$DEVICE /data + - echo "LABEL=gittensory-data /data ext4 defaults 0 2" >> /etc/fstab + # Allow the ubuntu user to run docker without sudo + - usermod -aG docker ubuntu + - echo "cloud-init: gittensory host ready" > /var/log/gittensory-init.log + CLOUDINIT + + labels = { + app = "gittensory" + managed = "terraform" + } +} + +# Attach the volume after the server is created +resource "hcloud_volume_attachment" "gittensory_data" { + server_id = hcloud_server.gittensory.id + volume_id = hcloud_volume.gittensory_data.id + automount = true +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 0000000000..51b930497c --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,19 @@ +output "server_ipv4" { + description = "Public IPv4 address of the gittensory server" + value = hcloud_server.gittensory.ipv4_address +} + +output "server_ipv6" { + description = "Public IPv6 address of the gittensory server" + value = hcloud_server.gittensory.ipv6_address +} + +output "ssh_command" { + description = "SSH command to access the server" + value = "ssh ubuntu@${hcloud_server.gittensory.ipv4_address}" +} + +output "volume_device" { + description = "Linux block device path for the data volume" + value = hcloud_volume.gittensory_data.linux_device +} diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000000..3d1941c3d8 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,34 @@ +variable "hcloud_token" { + description = "Hetzner Cloud API token (generate at console.hetzner.cloud → Security → API Tokens)" + type = string + sensitive = true +} + +variable "ssh_public_key" { + description = "SSH public key content for server access (e.g. file('~/.ssh/id_ed25519.pub'))" + type = string +} + +variable "server_type" { + description = "Hetzner server type. cx22 = 2 vCPU / 4 GB (sufficient for <50 reviews/day). cpx21 = 3 vCPU AMD / 4 GB for heavier load." + type = string + default = "cx22" +} + +variable "location" { + description = "Hetzner datacenter location: nbg1 (Nuremberg), fsn1 (Falkenstein), hel1 (Helsinki), ash (Ashburn VA), sin (Singapore)" + type = string + default = "nbg1" +} + +variable "volume_size_gb" { + description = "Size of the persistent data volume in GB (holds the SQLite DB and Litestream WAL segments)" + type = number + default = 20 +} + +variable "admin_ip_allowlist" { + description = "CIDR ranges allowed to SSH and access the raw app port (8787). Restrict to your IP(s) in production." + type = list(string) + default = ["0.0.0.0/0", "::/0"] +} diff --git a/test/unit/selfhost-audit.test.ts b/test/unit/selfhost-audit.test.ts new file mode 100644 index 0000000000..de9f99bfac --- /dev/null +++ b/test/unit/selfhost-audit.test.ts @@ -0,0 +1,70 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { logAudit, extractPayloadType } from "../../src/selfhost/audit"; + +describe("logAudit", () => { + const written: string[] = []; + + beforeEach(() => { + written.length = 0; + vi.spyOn(process.stdout, "write").mockImplementation((chunk) => { + written.push(String(chunk)); + return true; + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("emits a JSON line with level:audit for job_complete", () => { + logAudit({ event: "job_complete", ts: 1000, job_id: 1, payload_type: "review", latency_ms: 50, attempts: 1 }); + expect(written).toHaveLength(1); + const parsed = JSON.parse(written[0]!) as Record; + expect(parsed).toMatchObject({ level: "audit", event: "job_complete", ts: 1000, job_id: 1, payload_type: "review", latency_ms: 50, attempts: 1 }); + }); + + it("emits a JSON line for job_dead with error field", () => { + logAudit({ event: "job_dead", ts: 2000, job_id: "42", latency_ms: 100, attempts: 5, error: "boom" }); + const parsed = JSON.parse(written[0]!) as Record; + expect(parsed).toMatchObject({ level: "audit", event: "job_dead", error: "boom" }); + expect(parsed.payload_type).toBeUndefined(); + }); + + it("emits a JSON line for job_error", () => { + logAudit({ event: "job_error", ts: 3000, job_id: 2, latency_ms: 10, attempts: 2, error: "transient" }); + const parsed = JSON.parse(written[0]!) as Record; + expect(parsed.event).toBe("job_error"); + expect(parsed.level).toBe("audit"); + }); + + it("output ends with a newline", () => { + logAudit({ event: "job_complete", ts: 0, job_id: 0, latency_ms: 0, attempts: 1 }); + expect(written[0]!).toMatch(/\n$/); + }); +}); + +describe("extractPayloadType", () => { + it("returns the top-level type string", () => { + expect(extractPayloadType(JSON.stringify({ type: "review", other: 1 }))).toBe("review"); + }); + + it("returns undefined when type field is a number", () => { + expect(extractPayloadType(JSON.stringify({ type: 42 }))).toBeUndefined(); + }); + + it("returns undefined when type field is null", () => { + expect(extractPayloadType(JSON.stringify({ type: null }))).toBeUndefined(); + }); + + it("returns undefined when type field is absent", () => { + expect(extractPayloadType(JSON.stringify({ other: "x" }))).toBeUndefined(); + }); + + it("returns undefined for non-JSON input", () => { + expect(extractPayloadType("not-json")).toBeUndefined(); + }); + + it("returns undefined for an empty object", () => { + expect(extractPayloadType("{}")).toBeUndefined(); + }); +}); diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts new file mode 100644 index 0000000000..d23a441d26 --- /dev/null +++ b/test/unit/selfhost-pg-queue.test.ts @@ -0,0 +1,216 @@ +// Unit tests for the Postgres-backed job queue (#977). Mocks pg.Pool so no real DB is needed. +// Real-Postgres integration paths (migrations, pg-adapter translation) live in test/integration/selfhost-pg.test.ts. +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { Pool, QueryResult } from "pg"; +import { createPgQueue } from "../../src/selfhost/pg-queue"; +import type { JobMessage } from "../../src/types"; + +const msg = (t: string): JobMessage => ({ type: t }) as unknown as JobMessage; +const typeOf = (m: JobMessage): string => (m as unknown as { type: string }).type; + +type MockFn = { mockResolvedValueOnce(v: unknown): void }; + +interface MockPool { + pool: Pool; + fn: MockFn; + enqueueResult(r: Partial): void; + /** Pre-load a job to be returned by the next RETURNING claim query. */ + enqueueJob(id: string, payload: object, attempts?: number): void; +} + +function makePool(): MockPool { + const results: Partial[] = []; + const fn = vi.fn().mockImplementation(async (sql: unknown) => { + const q = String(sql); + // Claim queries use RETURNING — pop from queue; fall through to empty default otherwise. + if (q.includes("RETURNING")) { + const next = results.shift(); + return next ?? { rows: [], rowCount: 0 }; + } + // COUNT queries need a c column. + if (q.includes("COUNT(*)")) { + return { rows: [{ c: "3" }], rowCount: 1 }; + } + return { rows: [], rowCount: 0 }; + }); + return { + pool: { query: fn } as unknown as Pool, + fn: fn as unknown as MockFn, + enqueueResult(r) { results.push(r); }, + enqueueJob(id, payload, attempts = 0) { + results.push({ rows: [{ id, payload: JSON.stringify(payload), attempts }], rowCount: 1 }); + }, + }; +} + +describe("createPgQueue (durable #977)", () => { + // Suppress audit log stdout noise in tests. + beforeEach(() => { vi.spyOn(process.stdout, "write").mockImplementation(() => true); }); + afterEach(() => { vi.restoreAllMocks(); }); + + it("init() creates the table and recovers stuck-processing jobs", async () => { + const m = makePool(); + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // DDL + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 2 }); // recovery UPDATE + const q = createPgQueue(m.pool, async () => undefined); + await q.init(); + expect(m.pool.query).toHaveBeenCalledTimes(2); + }); + + it("init() handles null rowCount from the recovery query (rowCount ?? 0 nullish arm)", async () => { + const m = makePool(); + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // DDL + // pg driver can return null for rowCount on some UPDATE results + m.fn.mockResolvedValueOnce({ rows: [], rowCount: null }); + const q = createPgQueue(m.pool, async () => undefined); + await q.init(); // rowCount=null → ?? 0 → 0 → no recovery log emitted + expect(m.pool.query).toHaveBeenCalledTimes(2); + }); + + it("processes a job successfully (job_complete audit emitted)", async () => { + const m = makePool(); + m.enqueueJob("1", { type: "review" }); + const seen: string[] = []; + const q = createPgQueue(m.pool, async (j) => void seen.push(typeOf(j))); + await q.init(); + await q.drain(); + expect(seen).toEqual(["review"]); + }); + + it("dead-letters an unparseable payload (job_dead audit emitted)", async () => { + const m = makePool(); + // Claim returns a row with bad payload. + m.enqueueResult({ rows: [{ id: "1", payload: "not-json", attempts: 0 }], rowCount: 1 }); + const q = createPgQueue(m.pool, async () => undefined, { maxRetries: 3 }); + await q.init(); + await q.drain(); + // UPDATE dead + then no more rows → pump exits cleanly. + expect(m.pool.query).toHaveBeenCalledWith(expect.stringContaining("status='dead'"), expect.arrayContaining(["1"])); + }); + + it("retries a failing job (job_error audit emitted) then dead-letters at maxRetries (job_dead)", async () => { + const m = makePool(); + // Two attempts: first → retry, second → dead-letter. + m.enqueueJob("1", { type: "t" }, 0); + m.enqueueJob("1", { type: "t" }, 1); // second claim after retry + let calls = 0; + const q = createPgQueue(m.pool, async () => { calls++; throw new Error("fail"); }, { maxRetries: 2, backoffMs: () => 0 }); + await q.init(); + await q.drain(); + await q.drain(); // second drain processes the retried job + expect(calls).toBe(2); + }); + + it("records 'unknown error' when consumer throws a non-Error", async () => { + const m = makePool(); + m.enqueueJob("1", { type: "t" }, 0); + const q = createPgQueue(m.pool, async () => { throw "plain-string"; }, { maxRetries: 1, backoffMs: () => 0 }); + await q.init(); + await q.drain(); + expect(m.pool.query).toHaveBeenCalledWith(expect.stringContaining("status='dead'"), expect.arrayContaining(["unknown error"])); + }); + + it("pump() returns early when active >= concurrency (saturation guard)", async () => { + let concurrent = 0; + let maxConcurrent = 0; + const m = makePool(); + m.enqueueJob("1", { type: "a" }); + m.enqueueJob("2", { type: "b" }); + const q = createPgQueue(m.pool, async () => { + concurrent++; + maxConcurrent = Math.max(maxConcurrent, concurrent); + await new Promise((r) => setTimeout(r, 15)); + concurrent--; + }, { concurrency: 1, pollIntervalMs: 100_000 }); + await q.init(); + await q.binding.send(msg("a")); + await q.binding.send(msg("b")); // second void pump() hits active >= 1 → returns early + await new Promise((r) => setTimeout(r, 60)); + await q.stop(); + expect(maxConcurrent).toBe(1); + }); + + it("concurrency=2 allows two jobs to run simultaneously", async () => { + let concurrent = 0; + let maxConcurrent = 0; + const m = makePool(); + m.enqueueJob("1", { type: "a" }); + m.enqueueJob("2", { type: "b" }); + const q = createPgQueue(m.pool, async () => { + concurrent++; + maxConcurrent = Math.max(maxConcurrent, concurrent); + await new Promise((r) => setTimeout(r, 15)); + concurrent--; + }, { concurrency: 2, pollIntervalMs: 100_000 }); + await q.init(); + await q.binding.send(msg("a")); + await q.binding.send(msg("b")); + await new Promise((r) => setTimeout(r, 60)); + await q.stop(); + expect(maxConcurrent).toBe(2); + }); + + it("start() and stop() run the poll loop", async () => { + const m = makePool(); + m.enqueueJob("1", { type: "ticked" }); + const seen: string[] = []; + const q = createPgQueue(m.pool, async (j) => void seen.push(typeOf(j)), { pollIntervalMs: 10 }); + await q.init(); + q.start(); + for (let i = 0; i < 50 && seen.length === 0; i++) await new Promise((r) => setTimeout(r, 10)); + await q.stop(); + expect(seen).toEqual(["ticked"]); + }); + + it("start() is idempotent", async () => { + const { pool } = makePool(); + const q = createPgQueue(pool, async () => undefined, { pollIntervalMs: 100_000 }); + await q.init(); + q.start(); + q.start(); // second call is a no-op + await q.stop(); + }); + + it("stop() is a no-op when timer is null", async () => { + const { pool } = makePool(); + const q = createPgQueue(pool, async () => undefined); + await q.init(); + await q.stop(); // timer=null → false branch of `if (timer) clearTimeout(timer)` + }); + + it("binding.sendBatch enqueues multiple messages", async () => { + const m = makePool(); + m.enqueueJob("1", { type: "x" }); + m.enqueueJob("2", { type: "y" }); + const seen: string[] = []; + const q = createPgQueue(m.pool, async (j) => void seen.push(typeOf(j))); + await q.init(); + await q.binding.sendBatch([{ body: msg("x") }, { body: msg("y") }]); + await q.drain(); + expect(seen.sort()).toEqual(["x", "y"]); + }); + + it("uses default backoff lambda when backoffMs is not provided", async () => { + // Trigger a retry without providing backoffMs so the default (attempt) => Math.min(60_000, 1000 * 2**attempt) + // is actually called — covering the function body that would otherwise be created but never invoked. + const m = makePool(); + m.enqueueJob("1", { type: "t" }, 0); + const q = createPgQueue(m.pool, async () => { throw new Error("transient"); }, { maxRetries: 5 }); + // No backoffMs → default lambda is used + called when scheduling the retry + await q.init(); + await q.drain(); + expect(m.pool.query).toHaveBeenCalledWith( + expect.stringContaining("status='pending'"), + expect.arrayContaining([1]), + ); + }); + + it("size() and deadCount() return numeric counts", async () => { + const { pool } = makePool(); + // makePool returns { c: "3" } for COUNT queries + const q = createPgQueue(pool, async () => undefined); + await q.init(); + expect(await q.size()).toBe(3); + expect(await q.deadCount()).toBe(3); + }); +}); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 76712f9246..13e67a690a 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -1,5 +1,5 @@ import { DatabaseSync } from "node:sqlite"; -import { describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; import { createSqliteQueue } from "../../src/selfhost/sqlite-queue"; import type { JobMessage } from "../../src/types"; @@ -11,6 +11,10 @@ const msg = (t: string): JobMessage => ({ type: t }) as unknown as JobMessage; const typeOf = (m: JobMessage): string => (m as unknown as { type: string }).type; describe("createSqliteQueue (durable #980)", () => { + // Suppress audit log stdout noise. + beforeEach(() => { vi.spyOn(process.stdout, "write").mockImplementation(() => true); }); + afterEach(() => { vi.restoreAllMocks(); }); + it("persists + drains FIFO through the consumer", async () => { const driver = makeDriver(); const seen: string[] = []; @@ -116,6 +120,39 @@ describe("createSqliteQueue (durable #980)", () => { expect(q.size()).toBe(0); // still usable after a spurious stop() }); + it("concurrency=1 saturates after one active pump (active >= concurrency → early return)", async () => { + let concurrent = 0; + let maxConcurrent = 0; + const q = createSqliteQueue(makeDriver(), async () => { + concurrent++; + maxConcurrent = Math.max(maxConcurrent, concurrent); + await new Promise((r) => setTimeout(r, 15)); + concurrent--; + }, { concurrency: 1, pollIntervalMs: 100_000 }); + // sendBatch fires two void pump() calls synchronously; the second sees active=1 >= 1 and returns. + await q.binding.sendBatch([{ body: msg("a") }, { body: msg("b") }]); + await new Promise((r) => setTimeout(r, 60)); + await q.stop(); + expect(maxConcurrent).toBe(1); + expect(q.size()).toBe(0); + }); + + it("concurrency=2 allows two jobs to run simultaneously", async () => { + let concurrent = 0; + let maxConcurrent = 0; + const q = createSqliteQueue(makeDriver(), async () => { + concurrent++; + maxConcurrent = Math.max(maxConcurrent, concurrent); + await new Promise((r) => setTimeout(r, 15)); + concurrent--; + }, { concurrency: 2, pollIntervalMs: 100_000 }); + await q.binding.sendBatch([{ body: msg("a") }, { body: msg("b") }]); + await new Promise((r) => setTimeout(r, 60)); + await q.stop(); + expect(maxConcurrent).toBe(2); + expect(q.size()).toBe(0); + }); + it("start() is idempotent and stop() waits for an in-flight pump", async () => { let done = false; const q = createSqliteQueue(makeDriver(), async () => { From 5da5d623b589501a092a1785a2acd07fa704a1b7 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 24 Jun 2026 02:19:49 -0700 Subject: [PATCH 22/25] fix(selfhost): harden security for Docker self-host configs (#1157) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pin puppeteer-core to 22.13.1 in Dockerfile (was unpinned) - Add CSRF state nonce to /setup → /setup/callback flow: generate a UUID per-visit, embed it in the manifest redirect_url, bind it to the browser via an HttpOnly SameSite=Lax cookie, and validate both match in the callback (prevents manifest code injection) - Remove /var/run/docker.sock mount from runner service; add inline guidance for DinD alternative - Pin runner image from :latest to :ubuntu-22.04 - Require GRAFANA_ADMIN_PASSWORD explicitly (fail-fast :? expansion) --- .env.example | 2 +- Dockerfile | 2 +- docker-compose.yml | 10 +++++++--- src/selfhost/setup-wizard.ts | 11 ++++++----- src/server.ts | 23 +++++++++++++++++++++-- test/unit/selfhost-setup-wizard.test.ts | 16 +++++++++++----- 6 files changed, 47 insertions(+), 17 deletions(-) diff --git a/.env.example b/.env.example index 65ae9be527..846cf85452 100644 --- a/.env.example +++ b/.env.example @@ -138,7 +138,7 @@ GITTENSORY_REVIEW_DRAFT=false # RUNNER_LABELS=self-hosted,linux # --- Grafana (#1206; requires --profile observability) --- -# GRAFANA_ADMIN_PASSWORD=changeme # Grafana admin password (change before exposing publicly) +# GRAFANA_ADMIN_PASSWORD=changeme # REQUIRED when using --profile observability; compose fails if unset # --- AI review backend (optional; without it reviews run deterministically) --- # AI_SUMMARIES_ENABLED=true diff --git a/Dockerfile b/Dockerfile index 684c37b7be..893b36a4fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ RUN if [ "$INSTALL_AI_CLIS" = "true" ]; then npm install -g @anthropic-ai/claude # Build with `--build-arg INSTALL_VISUAL_REVIEW=true` then set BROWSER_WS_ENDPOINT= at runtime. ARG INSTALL_VISUAL_REVIEW=false COPY --from=build /app/package*.json ./ -RUN if [ "$INSTALL_VISUAL_REVIEW" = "true" ]; then npm install puppeteer-core --ignore-scripts; fi +RUN if [ "$INSTALL_VISUAL_REVIEW" = "true" ]; then npm install puppeteer-core@22.13.1 --ignore-scripts; fi # Data dir (the SQLite file) — owned by the unprivileged node user; mount a volume here to persist. RUN mkdir -p /data && chown -R node:node /data /app USER node diff --git a/docker-compose.yml b/docker-compose.yml index 989ed888a1..50c11e01c8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -187,7 +187,7 @@ services: - ./grafana/provisioning:/etc/grafana/provisioning:ro - ./grafana/dashboards:/var/lib/grafana/dashboards:ro environment: - GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env before using --profile observability} GF_USERS_ALLOW_SIGN_UP: "false" # ── Tailscale (--profile tailscale) ─────────────────────────────────────── @@ -216,7 +216,9 @@ services: # and RUNNER_REPO_URL= (e.g. https://github.com/your-org/your-repo) in .env. # Get a token at: https://github.com///settings/actions/runners/new runner: - image: myoung34/github-runner:latest + # Pin to a specific version tag — `latest` is mutable. Find a digest via: + # docker pull myoung34/github-runner:ubuntu-22.04 && docker inspect --format='{{index .RepoDigests 0}}' myoung34/github-runner:ubuntu-22.04 + image: myoung34/github-runner:ubuntu-22.04 restart: unless-stopped profiles: ["runners"] environment: @@ -227,8 +229,10 @@ services: RUNNER_NAME: ${RUNNER_NAME:-gittensory-runner} LABELS: ${RUNNER_LABELS:-self-hosted,linux,x64} RUNNER_WORKDIR: /tmp/runner + # Docker builds are disabled by default. To enable DinD (Docker-in-Docker) for workflows that + # need `docker build/push`, add a `dind` sidecar service and set DOCKER_HOST here. Mounting + # /var/run/docker.sock grants container-escape risk and is intentionally omitted. volumes: - - /var/run/docker.sock:/var/run/docker.sock - runner-work:/tmp/runner volumes: diff --git a/src/selfhost/setup-wizard.ts b/src/selfhost/setup-wizard.ts index bb6289c980..3c775f43a9 100644 --- a/src/selfhost/setup-wizard.ts +++ b/src/selfhost/setup-wizard.ts @@ -14,13 +14,13 @@ export interface AppCredentials { } /** The GitHub App manifest — permissions + events mirror docs §2 (the manual-setup instructions). */ -export function buildManifest(origin: string): Record { +export function buildManifest(origin: string, state: string): Record { const base = origin.replace(/\/+$/, ""); return { name: "Gittensory Self-Host", url: base, hook_attributes: { url: `${base}/v1/github/webhook` }, - redirect_url: `${base}/setup/callback`, + redirect_url: `${base}/setup/callback?state=${encodeURIComponent(state)}`, public: false, default_permissions: { pull_requests: "write", @@ -34,9 +34,10 @@ export function buildManifest(origin: string): Record { }; } -/** HTML page that POSTs the manifest to GitHub's App-creation flow (one click). */ -export function renderSetupPage(origin: string): string { - const manifest = JSON.stringify(buildManifest(origin)).replace(/'/g, "'"); +/** HTML page that POSTs the manifest to GitHub's App-creation flow (one click). + * `state` is a random CSRF nonce tied to the session via an HttpOnly cookie in the caller. */ +export function renderSetupPage(origin: string, state: string): string { + const manifest = JSON.stringify(buildManifest(origin, state)).replace(/'/g, "'"); return `Gittensory self-host setup

Gittensory self-host setup

diff --git a/src/server.ts b/src/server.ts index 4c56d40797..7e555f8407 100644 --- a/src/server.ts +++ b/src/server.ts @@ -7,6 +7,7 @@ // scheduled handler on a timer, exposes /health /ready /metrics, and shuts down gracefully. The Cloudflare // Worker (src/index.ts) is untouched — this is a parallel entry the self-host esbuild build bundles. import { readFileSync, writeFileSync } from "node:fs"; +import { randomUUID } from "node:crypto"; import { DatabaseSync } from "node:sqlite"; import { serve } from "@hono/node-server"; import worker from "./index"; @@ -159,9 +160,27 @@ async function main(): Promise { // First-run GitHub App setup wizard — only while no App is configured (can't rebind a live install). if ((path === "/setup" || path === "/setup/callback") && !process.env.GITHUB_APP_ID) { const origin = process.env.PUBLIC_API_ORIGIN ?? new URL(request.url).origin; - if (path === "/setup") return new Response(renderSetupPage(origin), { headers: { "content-type": "text/html; charset=utf-8" } }); - const code = new URL(request.url).searchParams.get("code"); + if (path === "/setup") { + // Generate a per-visit CSRF nonce, embed it in the manifest's redirect_url, and bind it to + // this browser session via an HttpOnly cookie so the callback can validate it. + const state = randomUUID(); + return new Response(renderSetupPage(origin, state), { + headers: { + "content-type": "text/html; charset=utf-8", + "Set-Cookie": `setup_state=${state}; Path=/setup; HttpOnly; SameSite=Lax; Max-Age=3600`, + }, + }); + } + const params = new URL(request.url).searchParams; + const code = params.get("code"); if (!code) return new Response("missing ?code", { status: 400 }); + // Validate the CSRF state: must match the cookie set when /setup was served. + const stateParam = params.get("state"); + const cookieHeader = request.headers.get("cookie") ?? ""; + const cookieState = cookieHeader.split(";").map((c) => c.trim()).find((c) => c.startsWith("setup_state="))?.slice("setup_state=".length); + if (!stateParam || !cookieState || stateParam !== cookieState) { + return new Response("invalid state parameter", { status: 403 }); + } try { const creds = await exchangeManifestCode(code); const outPath = process.env.SETUP_OUTPUT_PATH ?? "/data/gittensory-app.env"; diff --git a/test/unit/selfhost-setup-wizard.test.ts b/test/unit/selfhost-setup-wizard.test.ts index d384dc96f7..2e24c197e7 100644 --- a/test/unit/selfhost-setup-wizard.test.ts +++ b/test/unit/selfhost-setup-wizard.test.ts @@ -2,20 +2,26 @@ import { describe, expect, it, vi } from "vitest"; import { buildManifest, credentialsToEnv, exchangeManifestCode, renderSetupPage } from "../../src/selfhost/setup-wizard"; describe("setup-wizard (#981 GitHub App Manifest)", () => { - it("builds a manifest with the webhook + redirect URLs, permissions, events", () => { - const m = buildManifest("https://gt.example.com/"); + it("builds a manifest with the webhook + redirect URLs (including CSRF state), permissions, events", () => { + const m = buildManifest("https://gt.example.com/", "test-state-123"); expect(m.url).toBe("https://gt.example.com"); // trailing slash trimmed expect((m.hook_attributes as { url: string }).url).toBe("https://gt.example.com/v1/github/webhook"); - expect(m.redirect_url).toBe("https://gt.example.com/setup/callback"); + expect(m.redirect_url).toBe("https://gt.example.com/setup/callback?state=test-state-123"); expect((m.default_permissions as Record).pull_requests).toBe("write"); expect(m.default_events).toContain("pull_request"); }); - it("renders a form that POSTs the manifest to GitHub", () => { - const html = renderSetupPage("https://gt.example.com"); + it("encodes special characters in the state parameter", () => { + const m = buildManifest("https://gt.example.com", "a b+c=d&e"); + expect(m.redirect_url).toContain("state=a%20b%2Bc%3Dd%26e"); + }); + + it("renders a form that POSTs the manifest to GitHub with the CSRF state embedded", () => { + const html = renderSetupPage("https://gt.example.com", "nonce-abc"); expect(html).toContain('action="https://github.com/settings/apps/new"'); expect(html).toContain('name="manifest"'); expect(html).toContain("Gittensory Self-Host"); + expect(html).toContain("nonce-abc"); // state is baked into the manifest value }); it("exchanges the code and serializes credentials to .env lines", async () => { From 73e9dbd9c29d2b243ed7b25f39e327132107b901 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 24 Jun 2026 03:40:14 -0700 Subject: [PATCH 23/25] feat(selfhost): add Redis webhook dedup cache, Qdrant vector store, Postgres readiness wait (#1217) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Redis cache (redis-cache.ts): webhook delivery dedup — marks delivery IDs after a successful response so GitHub retries are handled idempotently; rate limiter now shares the same client connection as the cache - Qdrant adapter (qdrant-vectorize.ts): optional RAG vector backend behind --profile qdrant; deterministic SHA-1 → UUID ID mapping, QDRANT_API_KEY auth, labeled error counters per operation, graceful-degrade on network errors - waitForPostgres: exponential-backoff retry prevents crash loops when gittensory starts before Postgres is ready in compose stacks - Grafana dashboard: full datasource-UID fix, System Health row, Qdrant panels, webhook dedup counter; all counter metrics pre-initialized to 0 at startup --- .env.example | 6 +- docker-compose.yml | 40 +++- grafana/dashboards/gittensory.json | 177 ++++++++++++-- src/selfhost/qdrant-vectorize.ts | 139 +++++++++++ src/selfhost/redis-cache.ts | 40 ++++ src/server.ts | 72 +++++- test/unit/selfhost-qdrant-vectorize.test.ts | 241 ++++++++++++++++++++ test/unit/selfhost-redis-cache.test.ts | 78 +++++++ 8 files changed, 761 insertions(+), 32 deletions(-) create mode 100644 src/selfhost/qdrant-vectorize.ts create mode 100644 src/selfhost/redis-cache.ts create mode 100644 test/unit/selfhost-qdrant-vectorize.test.ts create mode 100644 test/unit/selfhost-redis-cache.test.ts diff --git a/.env.example b/.env.example index 846cf85452..d221936ce3 100644 --- a/.env.example +++ b/.env.example @@ -109,7 +109,11 @@ GITTENSORY_REVIEW_DRAFT=false # DATABASE_PATH=/data/gittensory.sqlite # SQLite file on the mounted data volume; all 56 migrations auto-apply # DATABASE_URL= # set to postgres://user:pw@host:5432/db to use Postgres instead of # # SQLite (shared DB → multi-instance). Overrides DATABASE_PATH. -# REDIS_URL= # set to redis://host:6379 for a distributed rate limiter (else off) +# REDIS_URL= # set to redis://host:6379 for distributed rate limiting + webhook dedup +# # cache (prevents double-processing of GitHub retries). Off when unset. +# QDRANT_URL= # set to http://qdrant:6333 to use Qdrant as the RAG vector store +# # (--profile qdrant). Overrides the built-in sqlite-vec / pgvector. +# # Collection and schema are auto-created at startup. Off when unset. # MIGRATIONS_DIR=/app/migrations # CRON_INTERVAL_MS=120000 # maintain/sweep + sync cadence (default ~2 min) diff --git a/docker-compose.yml b/docker-compose.yml index 50c11e01c8..898bddb0dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,24 +77,23 @@ services: # Transaction-mode pooler — allows hundreds of app clients with a small PG connection cap. # Set DATABASE_URL in gittensory to: postgres://gittensory:@pgbouncer:5432/gittensory pgbouncer: - image: bitnami/pgbouncer:1 + image: edoburu/pgbouncer:latest restart: unless-stopped profiles: ["pgbouncer"] depends_on: postgres: condition: service_healthy environment: - POSTGRESQL_HOST: postgres - POSTGRESQL_PORT: "5432" - POSTGRESQL_DATABASE: gittensory - POSTGRESQL_USERNAME: gittensory - POSTGRESQL_PASSWORD: ${POSTGRES_PASSWORD:-CHANGEME} - PGBOUNCER_DATABASE: gittensory - PGBOUNCER_PORT: "5432" - PGBOUNCER_POOL_MODE: transaction - PGBOUNCER_MAX_CLIENT_CONN: "200" - PGBOUNCER_DEFAULT_POOL_SIZE: "20" - PGBOUNCER_AUTH_TYPE: scram-sha-256 + DB_HOST: postgres + DB_PORT: "5432" + DB_NAME: gittensory + DB_USER: gittensory + DB_PASSWORD: ${POSTGRES_PASSWORD:-CHANGEME} + LISTEN_PORT: "5432" + POOL_MODE: transaction + MAX_CLIENT_CONN: "200" + DEFAULT_POOL_SIZE: "20" + AUTH_TYPE: md5 # ── Redis (--profile redis) ──────────────────────────────────────────────── redis: @@ -108,6 +107,22 @@ services: interval: 10s retries: 5 + # ── Qdrant (--profile qdrant) ───────────────────────────────────────────── + # Dedicated vector database for RAG — replaces the built-in sqlite-vec / pgvector when + # QDRANT_URL=http://qdrant:6333 is set. Scales to millions of vectors with ANN search. + # REST API: http://localhost:6333 gRPC: localhost:6334 Dashboard: http://localhost:6333/dashboard + qdrant: + image: qdrant/qdrant:latest + restart: unless-stopped + profiles: ["qdrant"] + ports: + - "6333:6333" # REST API + Web UI + - "6334:6334" # gRPC + volumes: + - qdrant-data:/qdrant/storage + # Qdrant's minimal image has no curl/wget/nc — healthcheck omitted. + # The service is ready when the gittensory startup log shows {"event":"selfhost_vectorize","backend":"qdrant"}. + # ── Ollama (--profile ollama) ────────────────────────────────────────────── # After `docker compose --profile ollama up -d`, pull a model: # docker compose exec ollama ollama pull llama3.2 @@ -239,6 +254,7 @@ volumes: gittensory-data: gittensory-pg: gittensory-redis: + qdrant-data: ollama-models: caddy-data: caddy-config: diff --git a/grafana/dashboards/gittensory.json b/grafana/dashboards/gittensory.json index 8bf8d9cb85..e83163cc21 100644 --- a/grafana/dashboards/gittensory.json +++ b/grafana/dashboards/gittensory.json @@ -11,6 +11,13 @@ "id": null, "links": [], "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "title": "System Health", + "type": "row" + }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { @@ -21,12 +28,12 @@ "unit": "s" } }, - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, "id": 1, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "title": "Uptime", "type": "stat", - "targets": [{ "expr": "gittensory_uptime_seconds", "legendFormat": "uptime" }] + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gittensory_uptime_seconds", "legendFormat": "uptime" }] }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, @@ -37,12 +44,12 @@ "unit": "short" } }, - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, "id": 2, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "title": "Queue Pending", "type": "stat", - "targets": [{ "expr": "gittensory_queue_pending", "legendFormat": "pending" }] + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gittensory_queue_pending", "legendFormat": "pending" }] }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, @@ -53,42 +60,182 @@ "unit": "short" } }, - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 }, + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, "id": 3, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "title": "Dead-Letter Jobs", "type": "stat", - "targets": [{ "expr": "gittensory_queue_dead", "legendFormat": "dead" }] + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gittensory_queue_dead", "legendFormat": "dead" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "id": 6, + "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Total Jobs Processed", + "type": "stat", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gittensory_jobs_processed_total", "legendFormat": "processed" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 }] }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "id": 7, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Webhook Dedups (total)", + "type": "stat", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gittensory_webhook_dedup_total", "legendFormat": "deduped" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "id": 8, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Qdrant Errors (total)", + "type": "stat", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(gittensory_qdrant_errors_total) or vector(0)", "legendFormat": "errors" }] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "id": 101, + "title": "HTTP & Webhooks", + "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "reqps" } }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "id": 4, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, "title": "HTTP Request Rate", "type": "timeseries", - "targets": [{ "expr": "rate(gittensory_http_requests_total[2m])", "legendFormat": "requests/s" }] + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_http_requests_total[2m])", "legendFormat": "requests/s" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_webhook_dedup_total[2m])", "legendFormat": "dedup/s" } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "short" } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "id": 9, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "Queue Depth Over Time", + "type": "timeseries", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gittensory_queue_pending", "legendFormat": "pending" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gittensory_queue_dead", "legendFormat": "dead-letter" } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "id": 102, + "title": "Job Pipeline", + "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "ops" } }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, "id": 5, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, "title": "Job Throughput", "type": "timeseries", "targets": [ - { "expr": "rate(gittensory_jobs_processed_total[2m])", "legendFormat": "processed/s" }, - { "expr": "rate(gittensory_jobs_enqueued_total[2m])", "legendFormat": "enqueued/s" }, - { "expr": "rate(gittensory_jobs_failed_total[2m])", "legendFormat": "failed/s" }, - { "expr": "rate(gittensory_jobs_dead_total[2m])", "legendFormat": "dead/s" } + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_jobs_processed_total[2m])", "legendFormat": "processed/s" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_jobs_enqueued_total[2m])", "legendFormat": "enqueued/s" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_jobs_failed_total[2m])", "legendFormat": "failed/s" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_jobs_dead_total[2m])", "legendFormat": "dead/s" } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "lineWidth": 2, "fillOpacity": 10 }, + "unit": "percentunit", + "min": 0, + "max": 1 + } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, + "id": 10, + "options": { "legend": { "calcs": ["mean", "last"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "Job Failure Rate", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "rate(gittensory_jobs_failed_total[5m]) / (rate(gittensory_jobs_processed_total[5m]) + rate(gittensory_jobs_failed_total[5m]) + 0.0001)", + "legendFormat": "failure %" + } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, + "id": 103, + "title": "Vector Store (Qdrant)", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "ops" } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "id": 11, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "Qdrant Query Rate", + "type": "timeseries", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_qdrant_queries_total[2m])", "legendFormat": "queries/s" } ] - } + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "ops" } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "id": 12, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "Qdrant Upserts & Errors", + "type": "timeseries", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_qdrant_upserts_total[2m])", "legendFormat": "upserts/s" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_qdrant_errors_total[2m])", "legendFormat": "errors/s" } + ] + }, ], "refresh": "30s", "schemaVersion": 38, @@ -112,5 +259,5 @@ "timezone": "browser", "title": "Gittensory Self-Host", "uid": "gittensory-selfhost", - "version": 1 + "version": 2 } diff --git a/src/selfhost/qdrant-vectorize.ts b/src/selfhost/qdrant-vectorize.ts new file mode 100644 index 0000000000..9621d01362 --- /dev/null +++ b/src/selfhost/qdrant-vectorize.ts @@ -0,0 +1,139 @@ +// Qdrant-backed Vectorize adapter for self-host RAG (#1217). Implements the same Cloudflare +// `Vectorize` surface (upsert / query / deleteByIds) as the SQLite and pgvector adapters but +// backed by a standalone Qdrant REST API. Qdrant provides ANN search, payload filtering by +// namespace, and scales to millions of vectors — making it the recommended vector store for +// production self-host deployments. Enable with QDRANT_URL=http://qdrant:6333 and --profile qdrant. +// +// Qdrant requires UUID or uint64 point IDs. String IDs (e.g. "owner/repo:file:line") are +// mapped to UUIDs via a deterministic SHA-1 hash, with the original ID stored in the payload +// for retrieval. The collection is auto-created at startup via initQdrantCollection(). +// +// Set QDRANT_API_KEY for deployments that require Bearer token authentication (cloud Qdrant, +// production on-prem). Omit for unauthenticated local/dev deployments. +import { createHash } from "node:crypto"; +import { incr } from "./metrics"; + +const DEFAULT_COLLECTION = "gittensory"; +const DEFAULT_DIM = 1024; // bge-m3 / mxbai-embed-large (1024-d); set QDRANT_DIM to override + +interface VectorRecord { + id: string; + values: number[]; + namespace?: string; + metadata?: Record; +} +interface QueryOptions { + topK?: number; + namespace?: string; +} +interface Match { + id: string; + score: number; + metadata?: Record; +} +interface QdrantSearchResult { + result: Array<{ id: string; score: number; payload: Record }>; +} + +/** Maps an arbitrary string ID to a UUID that Qdrant accepts as a point ID. Deterministic. */ +function idToUuid(id: string): string { + const h = createHash("sha1").update(id).digest("hex"); + return `${h.slice(0, 8)}-${h.slice(8, 12)}-${h.slice(12, 16)}-${h.slice(16, 20)}-${h.slice(20, 32)}`; +} + +/** Build fetch headers, including Bearer auth when QDRANT_API_KEY is set. */ +function qdrantHeaders(): Record { + const h: Record = { "content-type": "application/json" }; + if (process.env.QDRANT_API_KEY) h["api-key"] = process.env.QDRANT_API_KEY; + return h; +} + +/** + * Ensures the Qdrant collection exists. Safe to call on every startup — a 409 (already exists) + * is silently ignored. Call this before createQdrantVectorize() when QDRANT_URL is set. + */ +export async function initQdrantCollection(url: string, collection = DEFAULT_COLLECTION, dim = DEFAULT_DIM): Promise { + const base = url.replace(/\/+$/, ""); + const res = await fetch(`${base}/collections/${collection}`, { + method: "PUT", + headers: qdrantHeaders(), + body: JSON.stringify({ vectors: { size: dim, distance: "Cosine" } }), + }); + if (!res.ok && res.status !== 409) { + throw new Error(`Qdrant collection init failed: HTTP ${res.status}`); + } +} + +/** Creates a Vectorize-compatible adapter backed by the Qdrant REST API at `url`. */ +export function createQdrantVectorize(url: string, collection = DEFAULT_COLLECTION): Vectorize { + const base = url.replace(/\/+$/, ""); + + const adapter = { + async upsert(vectors: VectorRecord[]): Promise<{ count: number; ids: string[] }> { + const points = vectors.map((v) => ({ + id: idToUuid(v.id), + vector: v.values, + payload: { _orig_id: v.id, namespace: v.namespace ?? "", ...v.metadata }, + })); + const res = await fetch(`${base}/collections/${collection}/points`, { + method: "PUT", + headers: qdrantHeaders(), + body: JSON.stringify({ points }), + }); + if (!res.ok) { + incr("gittensory_qdrant_errors_total", { op: "upsert" }); + throw new Error(`Qdrant upsert failed: HTTP ${res.status}`); + } + incr("gittensory_qdrant_upserts_total", {}, vectors.length); + return { count: vectors.length, ids: vectors.map((v) => v.id) }; + }, + + async query(vector: number[], opts: QueryOptions): Promise<{ matches: Match[] }> { + const body: Record = { vector, limit: opts.topK ?? 12, with_payload: true }; + if (opts.namespace) { + body.filter = { must: [{ key: "namespace", match: { value: opts.namespace } }] }; + } + let res: Response; + try { + res = await fetch(`${base}/collections/${collection}/points/search`, { + method: "POST", + headers: qdrantHeaders(), + body: JSON.stringify(body), + }); + } catch { + // Qdrant unreachable — degrade gracefully (RAG returns no context rather than crashing) + incr("gittensory_qdrant_errors_total", { op: "query" }); + return { matches: [] }; + } + if (!res.ok) { + incr("gittensory_qdrant_errors_total", { op: "query" }); + return { matches: [] }; + } + incr("gittensory_qdrant_queries_total"); + const data = (await res.json()) as QdrantSearchResult; + const matches: Match[] = data.result.map((r) => { + const { _orig_id, namespace: _ns, ...rest } = r.payload; + const id = typeof _orig_id === "string" ? _orig_id : r.id; + return Object.keys(rest).length > 0 ? { id, score: r.score, metadata: rest } : { id, score: r.score }; + }); + return { matches }; + }, + + async deleteByIds(ids: string[]): Promise<{ count: number }> { + if (ids.length === 0) return { count: 0 }; + const points = ids.map(idToUuid); + const res = await fetch(`${base}/collections/${collection}/points/delete`, { + method: "POST", + headers: qdrantHeaders(), + body: JSON.stringify({ points }), + }); + if (!res.ok) { + incr("gittensory_qdrant_errors_total", { op: "delete" }); + throw new Error(`Qdrant deleteByIds failed: HTTP ${res.status}`); + } + return { count: ids.length }; + }, + }; + + return adapter as unknown as Vectorize; +} diff --git a/src/selfhost/redis-cache.ts b/src/selfhost/redis-cache.ts new file mode 100644 index 0000000000..c3149c789f --- /dev/null +++ b/src/selfhost/redis-cache.ts @@ -0,0 +1,40 @@ +// Redis-backed request-dedup cache for self-host (#1216). Prevents duplicate GitHub webhook +// deliveries from being processed twice — GitHub retries webhooks that receive a non-200 +// response, and each retry carries the same `x-github-delivery` UUID. By caching the delivery +// ID after a successful processing attempt, the server can return 204 immediately on retries +// without re-queuing the job. Activated when REDIS_URL is set alongside --profile redis. +import type { Redis } from "ioredis"; + +export function createRedisCache(redis: Redis) { + return { + async get(key: string): Promise { + return redis.get(key); + }, + async set(key: string, value: string, ttlSeconds: number): Promise { + await redis.set(key, value, "EX", ttlSeconds); + }, + async del(key: string): Promise { + await redis.del(key); + }, + }; +} + +export type RedisCache = ReturnType; + +/** + * Idempotency check for GitHub webhook deliveries. Returns true if the delivery was + * already seen (caller should short-circuit with 204). Marks the delivery as seen + * for `ttlSeconds` (default 5 min — covers GitHub's retry window) on the FIRST call. + * Best-effort: a Redis error is swallowed to avoid blocking webhook processing. + */ +export async function checkAndMarkDelivery(cache: RedisCache, deliveryId: string, ttlSeconds = 300): Promise { + try { + const seen = await cache.get(`delivery:${deliveryId}`); + if (seen) return true; + await cache.set(`delivery:${deliveryId}`, "1", ttlSeconds); + return false; + } catch { + // Redis unavailable → treat as first-time (never block processing on cache failure) + return false; + } +} diff --git a/src/server.ts b/src/server.ts index 7e555f8407..3102a0f76b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -46,8 +46,33 @@ interface Backend { shutdown(): Promise; } +/** Retry a Postgres connection until it succeeds (up to maxWaitMs). Prevents crash-restart loops when + * gittensory starts before Postgres is ready (common in `--profile postgres` compose stacks). */ +async function waitForPostgres(url: string, maxWaitMs = 30_000): Promise { + const pg = (await import("pg")).default; + const start = Date.now(); + let attempt = 0; + while (true) { + const client = new pg.Client({ connectionString: url }); + try { + await client.connect(); + await client.end(); + return; + } catch { + await client.end().catch(() => undefined); + attempt++; + const elapsed = Date.now() - start; + if (elapsed >= maxWaitMs) throw new Error(`Postgres not ready after ${maxWaitMs}ms (${attempt} attempts)`); + const delay = Math.min(2000, 200 * attempt); + console.log(JSON.stringify({ event: "selfhost_pg_wait", attempt, elapsed_ms: elapsed, retry_in_ms: delay })); + await new Promise((r) => setTimeout(r, delay)); + } + } +} + /** Build the Postgres backend (shared DB + queue) when DATABASE_URL is a postgres:// URL. */ async function buildPostgresBackend(url: string, consume: (m: JobMessage) => Promise): Promise { + await waitForPostgres(url); const pg = (await import("pg")).default; pg.types.setTypeParser(20, (v: string) => Number.parseInt(v, 10)); // int8 (COUNT) → number, like D1 const pool = new pg.Pool({ connectionString: url }); @@ -116,21 +141,35 @@ async function main(): Promise { const ai = createSelfHostAi(process.env); if (ai) console.log(JSON.stringify({ event: "selfhost_ai_provider", provider: process.env.AI_PROVIDER })); - // Redis fixed-window rate limiter (else absent → enforceRateLimit is a no-op, as today). + // Redis fixed-window rate limiter + webhook dedup cache (else absent when REDIS_URL is unset). let rateLimiter: DurableObjectNamespace | undefined; + let webhookCache: import("./selfhost/redis-cache").RedisCache | undefined; if (process.env.REDIS_URL) { const { Redis } = await import("ioredis"); + const redisClient = new Redis(process.env.REDIS_URL); const { createRedisRateLimiter } = await import("./selfhost/redis-ratelimit"); - rateLimiter = createRedisRateLimiter(new Redis(process.env.REDIS_URL)); + const { createRedisCache } = await import("./selfhost/redis-cache"); + rateLimiter = createRedisRateLimiter(redisClient); + webhookCache = createRedisCache(redisClient); console.log(JSON.stringify({ event: "selfhost_rate_limiter", backend: "redis" })); } + // Qdrant vector store — overrides the backend's built-in sqlite-vec / pgvector when QDRANT_URL is set. + let vectorizeOverride: Vectorize | undefined; + if (process.env.QDRANT_URL) { + const { createQdrantVectorize, initQdrantCollection } = await import("./selfhost/qdrant-vectorize"); + await initQdrantCollection(process.env.QDRANT_URL); + vectorizeOverride = createQdrantVectorize(process.env.QDRANT_URL); + console.log(JSON.stringify({ event: "selfhost_vectorize", backend: "qdrant" })); + } + env = { ...process.env, DB: backend.db, JOBS: backend.queue.binding, AI: ai, - ...(backend.vectorize ? { VECTORIZE: backend.vectorize } : {}), + // Qdrant takes priority; falls back to the backend's built-in vectorize (pgvector or sqlite-vec) + ...(vectorizeOverride ? { VECTORIZE: vectorizeOverride } : backend.vectorize ? { VECTORIZE: backend.vectorize } : {}), ...(rateLimiter ? { RATE_LIMITER: rateLimiter } : {}), // Visual review: when BROWSER_WS_ENDPOINT is set, expose a truthy BROWSER binding so shot.ts's // `if (!env.BROWSER) return` guard is bypassed; the puppeteer stub then connects via WS. @@ -140,6 +179,15 @@ async function main(): Promise { gauge("gittensory_queue_pending", () => backend.queue.size()); gauge("gittensory_queue_dead", () => backend.queue.deadCount()); gauge("gittensory_uptime_seconds", () => Math.floor((Date.now() - startedAt) / 1000)); + // Pre-initialize job counters to 0 so they appear in the first Prometheus scrape (lazy counters + // created on first use would otherwise cause "No data" in Grafana until the first job event). + for (const c of [ + "gittensory_jobs_enqueued_total", "gittensory_jobs_processed_total", + "gittensory_jobs_failed_total", "gittensory_jobs_dead_total", + "gittensory_http_requests_total", "gittensory_webhook_dedup_total", + "gittensory_qdrant_queries_total", "gittensory_qdrant_upserts_total", + ]) + incr(c, undefined, 0); const ctx = { waitUntil: (p: Promise) => void Promise.resolve(p).catch(() => undefined), @@ -192,7 +240,23 @@ async function main(): Promise { } } incr("gittensory_http_requests_total"); - return worker.fetch(request, env, ctx); + // Webhook delivery dedup: return 204 immediately for already-processed delivery IDs. + // We mark only AFTER a successful response — failed/rejected webhooks must be retryable. + const isWebhook = webhookCache && path === "/v1/github/webhook" && request.method === "POST"; + const deliveryId = isWebhook ? request.headers.get("x-github-delivery") : null; + if (deliveryId) { + const seen = await webhookCache!.get(`delivery:${deliveryId}`); + if (seen) { + incr("gittensory_webhook_dedup_total"); + return new Response(null, { status: 204 }); + } + } + const response = await worker.fetch(request, env, ctx); + if (deliveryId && response.ok) { + // Best-effort — never block the response on a cache write failure + void webhookCache!.set(`delivery:${deliveryId}`, "1", 300).catch(() => undefined); + } + return response; }, port, }, diff --git a/test/unit/selfhost-qdrant-vectorize.test.ts b/test/unit/selfhost-qdrant-vectorize.test.ts new file mode 100644 index 0000000000..33a8700ea4 --- /dev/null +++ b/test/unit/selfhost-qdrant-vectorize.test.ts @@ -0,0 +1,241 @@ +import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; +import { createQdrantVectorize, initQdrantCollection } from "../../src/selfhost/qdrant-vectorize"; +import { resetMetrics, renderMetrics } from "../../src/selfhost/metrics"; + +const BASE = "http://qdrant:6333"; + +/** Build a fake fetch that returns the given response for any call. */ +function mockFetch(status: number, body: unknown = {}) { + return vi.fn(async () => new Response(JSON.stringify(body), { status })); +} + +describe("initQdrantCollection (#1217)", () => { + afterEach(() => { vi.restoreAllMocks(); resetMetrics(); }); + + it("PUTs to /collections/ with cosine + size params", async () => { + const fake = mockFetch(200); + vi.stubGlobal("fetch", fake); + await initQdrantCollection(BASE); + expect(fake).toHaveBeenCalledOnce(); + const [url, init] = fake.mock.calls[0] as [string, RequestInit]; + expect(url).toBe(`${BASE}/collections/gittensory`); + const body = JSON.parse(init.body as string) as { vectors: { size: number; distance: string } }; + expect(body.vectors.distance).toBe("Cosine"); + expect(body.vectors.size).toBe(1024); + }); + + it("ignores a 409 (collection already exists)", async () => { + vi.stubGlobal("fetch", mockFetch(409)); + await expect(initQdrantCollection(BASE)).resolves.not.toThrow(); + }); + + it("throws on any other non-OK status", async () => { + vi.stubGlobal("fetch", mockFetch(500, { error: "server error" })); + await expect(initQdrantCollection(BASE)).rejects.toThrow(/HTTP 500/); + }); + + it("uses a custom collection name and dimension when provided", async () => { + const fake = mockFetch(200); + vi.stubGlobal("fetch", fake); + await initQdrantCollection(BASE, "custom-col", 768); + const [url, init] = fake.mock.calls[0] as [string, RequestInit]; + expect(url).toContain("custom-col"); + expect((JSON.parse(init.body as string) as { vectors: { size: number } }).vectors.size).toBe(768); + }); +}); + +describe("initQdrantCollection — QDRANT_API_KEY header", () => { + afterEach(() => { vi.restoreAllMocks(); delete process.env.QDRANT_API_KEY; }); + + it("includes api-key header when QDRANT_API_KEY is set", async () => { + process.env.QDRANT_API_KEY = "secret-key"; + const fake = mockFetch(200); + vi.stubGlobal("fetch", fake); + await initQdrantCollection(BASE); + const init = (fake.mock.calls[0] as [string, RequestInit])[1]; + expect((init.headers as Record)["api-key"]).toBe("secret-key"); + }); + + it("omits api-key header when QDRANT_API_KEY is unset", async () => { + delete process.env.QDRANT_API_KEY; + const fake = mockFetch(200); + vi.stubGlobal("fetch", fake); + await initQdrantCollection(BASE); + const init = (fake.mock.calls[0] as [string, RequestInit])[1]; + expect((init.headers as Record)["api-key"]).toBeUndefined(); + }); +}); + +describe("createQdrantVectorize (#1217 Qdrant adapter)", () => { + beforeEach(() => { vi.restoreAllMocks(); resetMetrics(); }); + + // ── upsert ──────────────────────────────────────────────────────────────── + + it("upsert PUTs points with uuid-mapped IDs and payload including _orig_id + namespace", async () => { + const fake = mockFetch(200, { status: "ok" }); + vi.stubGlobal("fetch", fake); + const v = createQdrantVectorize(BASE); + const result = await v.upsert([{ id: "repo/file:1", values: [0.1, 0.2], namespace: "ns1", metadata: { path: "a.ts" } }]); + expect(result).toEqual({ count: 1, ids: ["repo/file:1"] }); + const [url, init] = fake.mock.calls[0] as [string, RequestInit]; + expect(url).toContain("/points"); + const body = JSON.parse(init.body as string) as { points: Array<{ id: string; payload: { _orig_id: string; namespace: string } }> }; + expect(body.points[0]?.payload._orig_id).toBe("repo/file:1"); + expect(body.points[0]?.payload.namespace).toBe("ns1"); + // UUID must match the pattern xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + expect(body.points[0]?.id).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/); + }); + + it("upsert defaults namespace to empty string when absent", async () => { + vi.stubGlobal("fetch", mockFetch(200)); + const v = createQdrantVectorize(BASE); + await v.upsert([{ id: "no-ns", values: [1, 0] }]); + const init = (vi.mocked(fetch).mock.calls[0] as [string, RequestInit])[1]; + const body = JSON.parse(init.body as string) as { points: Array<{ payload: { namespace: string } }> }; + expect(body.points[0]?.payload.namespace).toBe(""); + }); + + it("upsert throws on a non-OK response and increments error counter", async () => { + vi.stubGlobal("fetch", mockFetch(503)); + const v = createQdrantVectorize(BASE); + await expect(v.upsert([{ id: "x", values: [1] }])).rejects.toThrow(/HTTP 503/); + expect(await renderMetrics()).toContain('gittensory_qdrant_errors_total{op="upsert"}'); + }); + + it("successful upsert increments gittensory_qdrant_upserts_total by vector count", async () => { + vi.stubGlobal("fetch", mockFetch(200)); + const v = createQdrantVectorize(BASE); + await v.upsert([{ id: "a", values: [1] }, { id: "b", values: [0] }]); + const metrics = await renderMetrics(); + expect(metrics).toMatch(/gittensory_qdrant_upserts_total 2/); + }); + + it("same string ID always produces the same UUID (deterministic mapping)", async () => { + vi.stubGlobal("fetch", mockFetch(200)); + const v = createQdrantVectorize(BASE); + await v.upsert([{ id: "stable-id", values: [1] }]); + const body1 = JSON.parse(((vi.mocked(fetch).mock.calls[0] as [string, RequestInit])[1].body) as string) as { points: Array<{ id: string }> }; + vi.mocked(fetch).mockClear(); + await v.upsert([{ id: "stable-id", values: [1] }]); + const body2 = JSON.parse(((vi.mocked(fetch).mock.calls[0] as [string, RequestInit])[1].body) as string) as { points: Array<{ id: string }> }; + expect(body1.points[0]?.id).toBe(body2.points[0]?.id); + }); + + // ── query ───────────────────────────────────────────────────────────────── + + it("query POSTs a search request with namespace filter and returns matches with _orig_id restored", async () => { + const qdrantResponse = { + result: [{ id: "some-uuid", score: 0.92, payload: { _orig_id: "repo/f:1", namespace: "ns", path: "f.ts" } }], + }; + vi.stubGlobal("fetch", mockFetch(200, qdrantResponse)); + const v = createQdrantVectorize(BASE); + const { matches } = await v.query([0.5, 0.5], { topK: 5, namespace: "ns" }); + expect(matches).toHaveLength(1); + expect(matches[0]?.id).toBe("repo/f:1"); // _orig_id restored + expect(matches[0]?.score).toBeCloseTo(0.92); + expect(matches[0]?.metadata?.path).toBe("f.ts"); + const init = (vi.mocked(fetch).mock.calls[0] as [string, RequestInit])[1]; + const body = JSON.parse(init.body as string) as { filter?: { must: Array<{ key: string; match: { value: string } }> } }; + expect(body.filter?.must[0]?.key).toBe("namespace"); + expect(body.filter?.must[0]?.match.value).toBe("ns"); + }); + + it("query without namespace sends no filter", async () => { + vi.stubGlobal("fetch", mockFetch(200, { result: [] })); + const v = createQdrantVectorize(BASE); + await v.query([1, 0], { topK: 10 }); + const body = JSON.parse(((vi.mocked(fetch).mock.calls[0] as [string, RequestInit])[1].body) as string) as { filter?: unknown }; + expect(body.filter).toBeUndefined(); + }); + + it("query defaults topK to 12 when omitted", async () => { + vi.stubGlobal("fetch", mockFetch(200, { result: [] })); + const v = createQdrantVectorize(BASE); + await v.query([1, 0], {}); + const body = JSON.parse(((vi.mocked(fetch).mock.calls[0] as [string, RequestInit])[1].body) as string) as { limit: number }; + expect(body.limit).toBe(12); + }); + + it("query returns empty matches when Qdrant is unreachable (network error) and tracks error", async () => { + vi.stubGlobal("fetch", vi.fn(async () => { throw new Error("ECONNREFUSED"); })); + const v = createQdrantVectorize(BASE); + const { matches } = await v.query([1, 0], { topK: 5 }); + expect(matches).toEqual([]); + expect(await renderMetrics()).toContain('gittensory_qdrant_errors_total{op="query"}'); + }); + + it("query returns empty matches on non-OK HTTP response (graceful degrade) and tracks error", async () => { + vi.stubGlobal("fetch", mockFetch(503)); + const v = createQdrantVectorize(BASE); + const { matches } = await v.query([1, 0], { topK: 5 }); + expect(matches).toEqual([]); + expect(await renderMetrics()).toContain('gittensory_qdrant_errors_total{op="query"}'); + }); + + it("successful query increments gittensory_qdrant_queries_total", async () => { + vi.stubGlobal("fetch", mockFetch(200, { result: [] })); + const v = createQdrantVectorize(BASE); + await v.query([1], {}); + await v.query([0], {}); + expect(await renderMetrics()).toMatch(/gittensory_qdrant_queries_total 2/); + }); + + it("query returns match without metadata when payload has no extra fields", async () => { + const qdrantResponse = { + result: [{ id: "uuid-1", score: 0.8, payload: { _orig_id: "plain-id", namespace: "n" } }], + }; + vi.stubGlobal("fetch", mockFetch(200, qdrantResponse)); + const v = createQdrantVectorize(BASE); + const { matches } = await v.query([1], {}); + expect(matches[0]).toEqual({ id: "plain-id", score: 0.8 }); + expect(matches[0]?.metadata).toBeUndefined(); + }); + + it("query falls back to the Qdrant UUID when _orig_id is missing from payload", async () => { + const qdrantResponse = { + result: [{ id: "fallback-uuid", score: 0.5, payload: { namespace: "n" } }], + }; + vi.stubGlobal("fetch", mockFetch(200, qdrantResponse)); + const v = createQdrantVectorize(BASE); + const { matches } = await v.query([1], {}); + expect(matches[0]?.id).toBe("fallback-uuid"); + }); + + // ── deleteByIds ─────────────────────────────────────────────────────────── + + it("deleteByIds POSTs the uuid-mapped IDs and returns the count", async () => { + vi.stubGlobal("fetch", mockFetch(200, { status: "ok" })); + const v = createQdrantVectorize(BASE); + const result = await v.deleteByIds(["id-1", "id-2"]); + expect(result).toEqual({ count: 2 }); + const init = (vi.mocked(fetch).mock.calls[0] as [string, RequestInit])[1]; + const body = JSON.parse(init.body as string) as { points: string[] }; + expect(body.points).toHaveLength(2); + body.points.forEach((p) => expect(p).toMatch(/^[0-9a-f]{8}-/)); + }); + + it("deleteByIds is a no-op for an empty array (no fetch call)", async () => { + const fake = mockFetch(200); + vi.stubGlobal("fetch", fake); + const v = createQdrantVectorize(BASE); + const result = await v.deleteByIds([]); + expect(result).toEqual({ count: 0 }); + expect(fake).not.toHaveBeenCalled(); + }); + + it("deleteByIds throws on a non-OK response and tracks error", async () => { + vi.stubGlobal("fetch", mockFetch(400)); + const v = createQdrantVectorize(BASE); + await expect(v.deleteByIds(["id"])).rejects.toThrow(/HTTP 400/); + expect(await renderMetrics()).toContain('gittensory_qdrant_errors_total{op="delete"}'); + }); + + it("trailing slash in URL is stripped", async () => { + const fake = mockFetch(200, { result: [] }); + vi.stubGlobal("fetch", fake); + const v = createQdrantVectorize("http://qdrant:6333/"); + await v.query([1], {}); + const [url] = fake.mock.calls[0] as [string]; + expect(url).not.toContain("//collections"); + }); +}); diff --git a/test/unit/selfhost-redis-cache.test.ts b/test/unit/selfhost-redis-cache.test.ts new file mode 100644 index 0000000000..d350afb5cf --- /dev/null +++ b/test/unit/selfhost-redis-cache.test.ts @@ -0,0 +1,78 @@ +import type { Redis } from "ioredis"; +import { describe, expect, it } from "vitest"; +import { checkAndMarkDelivery, createRedisCache } from "../../src/selfhost/redis-cache"; + +/** Minimal in-memory stand-in for the ioredis methods the cache uses. */ +function fakeRedis(): Redis & { _store: Map } { + const _store = new Map(); + return { + _store, + async get(k: string) { + return _store.get(k) ?? null; + }, + async set(k: string, v: string, _ex: "EX", _ttl: number) { + _store.set(k, v); + return "OK"; + }, + async del(k: string) { + _store.delete(k); + return 1; + }, + } as unknown as Redis & { _store: Map }; +} + +describe("createRedisCache (#1216 webhook dedup cache)", () => { + it("get returns null for a missing key", async () => { + const cache = createRedisCache(fakeRedis()); + expect(await cache.get("missing")).toBeNull(); + }); + + it("set then get returns the stored value", async () => { + const cache = createRedisCache(fakeRedis()); + await cache.set("k", "hello", 60); + expect(await cache.get("k")).toBe("hello"); + }); + + it("del removes the key", async () => { + const r = fakeRedis(); + const cache = createRedisCache(r); + await cache.set("k", "v", 60); + await cache.del("k"); + expect(await cache.get("k")).toBeNull(); + }); +}); + +describe("checkAndMarkDelivery (#1216 webhook idempotency)", () => { + it("returns false (first-time) for a new delivery ID and marks it as seen", async () => { + const cache = createRedisCache(fakeRedis()); + const result = await checkAndMarkDelivery(cache, "delivery-abc", 300); + expect(result).toBe(false); + // second call with the same ID should be a duplicate + const duplicate = await checkAndMarkDelivery(cache, "delivery-abc", 300); + expect(duplicate).toBe(true); + }); + + it("returns true (duplicate) for an already-seen delivery ID", async () => { + const r = fakeRedis(); + r._store.set("delivery:existing-id", "1"); + const cache = createRedisCache(r); + expect(await checkAndMarkDelivery(cache, "existing-id")).toBe(true); + }); + + it("different delivery IDs are tracked independently", async () => { + const cache = createRedisCache(fakeRedis()); + expect(await checkAndMarkDelivery(cache, "id-A")).toBe(false); + expect(await checkAndMarkDelivery(cache, "id-B")).toBe(false); // different ID → first-time + expect(await checkAndMarkDelivery(cache, "id-A")).toBe(true); // id-A seen before + }); + + it("swallows Redis errors and returns false (never blocks processing)", async () => { + const brokenRedis = { + async get() { throw new Error("connection refused"); }, + async set() { throw new Error("connection refused"); }, + } as unknown as Redis; + const cache = createRedisCache(brokenRedis); + // Must not throw — error is swallowed, returns false (first-time / let it through) + expect(await checkAndMarkDelivery(cache, "any-id")).toBe(false); + }); +}); From d50d4b3cfd9a0ec6808d01a6d0b933a9bfe1c5b2 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 24 Jun 2026 03:59:17 -0700 Subject: [PATCH 24/25] test(selfhost): fix TS2352 mock.calls cast errors in qdrant-vectorize tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All mock.calls[0] casts now use `as unknown as [...]` to satisfy the strict mock type — was causing typecheck (lint CI job) to fail on the Docker PR. --- test/unit/selfhost-qdrant-vectorize.test.ts | 26 ++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/unit/selfhost-qdrant-vectorize.test.ts b/test/unit/selfhost-qdrant-vectorize.test.ts index 33a8700ea4..bbfa0ec161 100644 --- a/test/unit/selfhost-qdrant-vectorize.test.ts +++ b/test/unit/selfhost-qdrant-vectorize.test.ts @@ -17,7 +17,7 @@ describe("initQdrantCollection (#1217)", () => { vi.stubGlobal("fetch", fake); await initQdrantCollection(BASE); expect(fake).toHaveBeenCalledOnce(); - const [url, init] = fake.mock.calls[0] as [string, RequestInit]; + const [url, init] = fake.mock.calls[0] as unknown as [string, RequestInit]; expect(url).toBe(`${BASE}/collections/gittensory`); const body = JSON.parse(init.body as string) as { vectors: { size: number; distance: string } }; expect(body.vectors.distance).toBe("Cosine"); @@ -38,7 +38,7 @@ describe("initQdrantCollection (#1217)", () => { const fake = mockFetch(200); vi.stubGlobal("fetch", fake); await initQdrantCollection(BASE, "custom-col", 768); - const [url, init] = fake.mock.calls[0] as [string, RequestInit]; + const [url, init] = fake.mock.calls[0] as unknown as [string, RequestInit]; expect(url).toContain("custom-col"); expect((JSON.parse(init.body as string) as { vectors: { size: number } }).vectors.size).toBe(768); }); @@ -52,7 +52,7 @@ describe("initQdrantCollection — QDRANT_API_KEY header", () => { const fake = mockFetch(200); vi.stubGlobal("fetch", fake); await initQdrantCollection(BASE); - const init = (fake.mock.calls[0] as [string, RequestInit])[1]; + const init = (fake.mock.calls[0] as unknown as [string, RequestInit])[1]; expect((init.headers as Record)["api-key"]).toBe("secret-key"); }); @@ -61,7 +61,7 @@ describe("initQdrantCollection — QDRANT_API_KEY header", () => { const fake = mockFetch(200); vi.stubGlobal("fetch", fake); await initQdrantCollection(BASE); - const init = (fake.mock.calls[0] as [string, RequestInit])[1]; + const init = (fake.mock.calls[0] as unknown as [string, RequestInit])[1]; expect((init.headers as Record)["api-key"]).toBeUndefined(); }); }); @@ -77,7 +77,7 @@ describe("createQdrantVectorize (#1217 Qdrant adapter)", () => { const v = createQdrantVectorize(BASE); const result = await v.upsert([{ id: "repo/file:1", values: [0.1, 0.2], namespace: "ns1", metadata: { path: "a.ts" } }]); expect(result).toEqual({ count: 1, ids: ["repo/file:1"] }); - const [url, init] = fake.mock.calls[0] as [string, RequestInit]; + const [url, init] = fake.mock.calls[0] as unknown as [string, RequestInit]; expect(url).toContain("/points"); const body = JSON.parse(init.body as string) as { points: Array<{ id: string; payload: { _orig_id: string; namespace: string } }> }; expect(body.points[0]?.payload._orig_id).toBe("repo/file:1"); @@ -90,7 +90,7 @@ describe("createQdrantVectorize (#1217 Qdrant adapter)", () => { vi.stubGlobal("fetch", mockFetch(200)); const v = createQdrantVectorize(BASE); await v.upsert([{ id: "no-ns", values: [1, 0] }]); - const init = (vi.mocked(fetch).mock.calls[0] as [string, RequestInit])[1]; + const init = (vi.mocked(fetch).mock.calls[0] as unknown as [string, RequestInit])[1]; const body = JSON.parse(init.body as string) as { points: Array<{ payload: { namespace: string } }> }; expect(body.points[0]?.payload.namespace).toBe(""); }); @@ -114,10 +114,10 @@ describe("createQdrantVectorize (#1217 Qdrant adapter)", () => { vi.stubGlobal("fetch", mockFetch(200)); const v = createQdrantVectorize(BASE); await v.upsert([{ id: "stable-id", values: [1] }]); - const body1 = JSON.parse(((vi.mocked(fetch).mock.calls[0] as [string, RequestInit])[1].body) as string) as { points: Array<{ id: string }> }; + const body1 = JSON.parse(((vi.mocked(fetch).mock.calls[0] as unknown as [string, RequestInit])[1].body) as string) as { points: Array<{ id: string }> }; vi.mocked(fetch).mockClear(); await v.upsert([{ id: "stable-id", values: [1] }]); - const body2 = JSON.parse(((vi.mocked(fetch).mock.calls[0] as [string, RequestInit])[1].body) as string) as { points: Array<{ id: string }> }; + const body2 = JSON.parse(((vi.mocked(fetch).mock.calls[0] as unknown as [string, RequestInit])[1].body) as string) as { points: Array<{ id: string }> }; expect(body1.points[0]?.id).toBe(body2.points[0]?.id); }); @@ -134,7 +134,7 @@ describe("createQdrantVectorize (#1217 Qdrant adapter)", () => { expect(matches[0]?.id).toBe("repo/f:1"); // _orig_id restored expect(matches[0]?.score).toBeCloseTo(0.92); expect(matches[0]?.metadata?.path).toBe("f.ts"); - const init = (vi.mocked(fetch).mock.calls[0] as [string, RequestInit])[1]; + const init = (vi.mocked(fetch).mock.calls[0] as unknown as [string, RequestInit])[1]; const body = JSON.parse(init.body as string) as { filter?: { must: Array<{ key: string; match: { value: string } }> } }; expect(body.filter?.must[0]?.key).toBe("namespace"); expect(body.filter?.must[0]?.match.value).toBe("ns"); @@ -144,7 +144,7 @@ describe("createQdrantVectorize (#1217 Qdrant adapter)", () => { vi.stubGlobal("fetch", mockFetch(200, { result: [] })); const v = createQdrantVectorize(BASE); await v.query([1, 0], { topK: 10 }); - const body = JSON.parse(((vi.mocked(fetch).mock.calls[0] as [string, RequestInit])[1].body) as string) as { filter?: unknown }; + const body = JSON.parse(((vi.mocked(fetch).mock.calls[0] as unknown as [string, RequestInit])[1].body) as string) as { filter?: unknown }; expect(body.filter).toBeUndefined(); }); @@ -152,7 +152,7 @@ describe("createQdrantVectorize (#1217 Qdrant adapter)", () => { vi.stubGlobal("fetch", mockFetch(200, { result: [] })); const v = createQdrantVectorize(BASE); await v.query([1, 0], {}); - const body = JSON.parse(((vi.mocked(fetch).mock.calls[0] as [string, RequestInit])[1].body) as string) as { limit: number }; + const body = JSON.parse(((vi.mocked(fetch).mock.calls[0] as unknown as [string, RequestInit])[1].body) as string) as { limit: number }; expect(body.limit).toBe(12); }); @@ -208,7 +208,7 @@ describe("createQdrantVectorize (#1217 Qdrant adapter)", () => { const v = createQdrantVectorize(BASE); const result = await v.deleteByIds(["id-1", "id-2"]); expect(result).toEqual({ count: 2 }); - const init = (vi.mocked(fetch).mock.calls[0] as [string, RequestInit])[1]; + const init = (vi.mocked(fetch).mock.calls[0] as unknown as [string, RequestInit])[1]; const body = JSON.parse(init.body as string) as { points: string[] }; expect(body.points).toHaveLength(2); body.points.forEach((p) => expect(p).toMatch(/^[0-9a-f]{8}-/)); @@ -235,7 +235,7 @@ describe("createQdrantVectorize (#1217 Qdrant adapter)", () => { vi.stubGlobal("fetch", fake); const v = createQdrantVectorize("http://qdrant:6333/"); await v.query([1], {}); - const [url] = fake.mock.calls[0] as [string]; + const [url] = fake.mock.calls[0] as unknown as [string]; expect(url).not.toContain("//collections"); }); }); From 5f0a784e14ae5080aef95dd424d8315dd7662994 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 24 Jun 2026 04:06:54 -0700 Subject: [PATCH 25/25] =?UTF-8?q?fix(selfhost):=20require=20PUBLIC=5FAPI?= =?UTF-8?q?=5FORIGIN=20for=20setup=20wizard=20=E2=80=94=20prevent=20Host-h?= =?UTF-8?q?eader=20redirect=20attack?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The setup wizard derived the App manifest origin from `process.env.PUBLIC_API_ORIGIN ?? new URL(request.url).origin`. When PUBLIC_API_ORIGIN is unset, the fallback uses the Host header from the incoming request. An attacker who can craft a request with a spoofed Host header (e.g. `evil.com`) causes the manifest's redirect_url to point at an attacker-controlled domain. GitHub follows that redirect on App creation, allowing the attacker to exchange the one-time code for the App's private key and webhook secret. Fix: remove the request.url fallback entirely. If PUBLIC_API_ORIGIN is not set the wizard now returns 400 with a clear operator message. The origin used to build the manifest is always the operator-configured value, never derived from the request. --- src/server.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/server.ts b/src/server.ts index 3102a0f76b..97fd351349 100644 --- a/src/server.ts +++ b/src/server.ts @@ -207,7 +207,16 @@ async function main(): Promise { if (path === "/metrics") return new Response(await renderMetrics(), { headers: { "content-type": "text/plain; version=0.0.4" } }); // First-run GitHub App setup wizard — only while no App is configured (can't rebind a live install). if ((path === "/setup" || path === "/setup/callback") && !process.env.GITHUB_APP_ID) { - const origin = process.env.PUBLIC_API_ORIGIN ?? new URL(request.url).origin; + // PUBLIC_API_ORIGIN is required: falling back to request.url.origin would let an attacker spoof + // the Host header and redirect the App-creation callback to an attacker-controlled domain, where + // they could exchange the code for the App private key and webhook secret. + const origin = process.env.PUBLIC_API_ORIGIN; + if (!origin) { + return new Response( + "PUBLIC_API_ORIGIN must be set before using the setup wizard — add it to your .env file", + { status: 400 }, + ); + } if (path === "/setup") { // Generate a per-visit CSRF nonce, embed it in the manifest's redirect_url, and bind it to // this browser session via an HttpOnly cookie so the callback can validate it.