diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..6ffdf51695 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,20 @@ +# Keep the build context lean — deps are installed fresh (npm ci) and the bundle is built in the image. +node_modules +**/node_modules +dist +dist-ssr +.output +.nitro +.tanstack +.wrangler +.playwright-cli +coverage +.git +.claude +.DS_Store +*.tsbuildinfo +# Never ship secrets into the build context +.env +.env.* +.dev.vars +!.env.example diff --git a/.env.example b/.env.example index 46f25b2c3d..d221936ce3 100644 --- a/.env.example +++ b/.env.example @@ -97,3 +97,66 @@ GITTENSORY_REVIEW_DRAFT=false # GITTENSORY_DRIFT_ISSUE_TOKEN= # token for auto-filing drift issues # GITTENSORY_CONTRIBUTOR_ISSUE_TOKEN= # token for contributor-issue automation # PRODUCT_USAGE_HASH_SALT= # salt for hashing product-usage identifiers + +# ============================================================================= +# 3. Self-host (Docker) — runtime config (#980) +# ============================================================================= +# For `docker compose up` self-hosting (NOT the Cloudflare Worker deploy). Copy this file to `.env` +# (gitignored), UNCOMMENT + fill the required Core secrets in section 2, then add the runtime values below. +# Every value here is a SAMPLE placeholder — never commit real secrets. + +# PORT=8787 +# DATABASE_PATH=/data/gittensory.sqlite # SQLite file on the mounted data volume; all 56 migrations auto-apply +# DATABASE_URL= # set to postgres://user:pw@host:5432/db to use Postgres instead of +# # SQLite (shared DB → multi-instance). Overrides DATABASE_PATH. +# REDIS_URL= # set to redis://host:6379 for distributed rate limiting + webhook dedup +# # cache (prevents double-processing of GitHub retries). Off when unset. +# QDRANT_URL= # set to http://qdrant:6333 to use Qdrant as the RAG vector store +# # (--profile qdrant). Overrides the built-in sqlite-vec / pgvector. +# # Collection and schema are auto-created at startup. Off when unset. +# MIGRATIONS_DIR=/app/migrations +# CRON_INTERVAL_MS=120000 # maintain/sweep + sync cadence (default ~2 min) + +# --- Continuous backup (optional; the Litestream sidecar in docker-compose.yml) --- +# LITESTREAM_ACCESS_KEY_ID= +# LITESTREAM_SECRET_ACCESS_KEY= +# LITESTREAM_ENDPOINT= # e.g. s3.us-west-002.backblazeb2.com (omit for AWS S3) +# LITESTREAM_REGION=us-east-1 + +# --- Queue worker (#977/#1201) --- +# QUEUE_CONCURRENCY=1 # max concurrent job-processing loops per instance (default 1) + +# --- Caddy HTTPS terminator (#1203; requires --profile caddy) --- +# DOMAIN=gittensory.example.com # fully-qualified domain; Caddy auto-obtains a Let's Encrypt cert + +# --- Tailscale sidecar (#1204; requires --profile tailscale) --- +# TS_AUTHKEY= # Tailscale auth key (generate at tailscale.com/admin/settings/keys) +# TS_EXTRA_ARGS= # extra tailscale up flags, e.g. --advertise-tags=tag:self-host + +# --- Self-hosted GitHub Actions runner (#1205; requires --profile runners) --- +# RUNNER_TOKEN= # runner registration token (Settings → Actions → Runners → New) +# RUNNER_REPO_URL=https://github.com/org/repo +# RUNNER_ACCESS_TOKEN= # PAT with repo scope (alternative to RUNNER_TOKEN) +# RUNNER_SCOPE=repo # repo | org | enterprise +# RUNNER_NAME=gittensory-runner +# RUNNER_LABELS=self-hosted,linux + +# --- Grafana (#1206; requires --profile observability) --- +# GRAFANA_ADMIN_PASSWORD=changeme # REQUIRED when using --profile observability; compose fails if unset + +# --- AI review backend (optional; without it reviews run deterministically) --- +# AI_SUMMARIES_ENABLED=true +# AI_PROVIDER=ollama # ollama | openai-compatible | openai | anthropic | claude-code | +# # codex. A COMMA-LIST is a fallback chain, e.g. "anthropic,ollama" +# # (tries each in order until one succeeds). (see #979) +# AI_BASE_URL=http://ollama:11434/v1 # OpenAI-compatible endpoint (Ollama default; or your provider's) +# AI_API_KEY= # generic key for the openai-compatible endpoint +# ANTHROPIC_API_KEY= # for AI_PROVIDER=anthropic (native Messages API, BYOK) +# OPENAI_API_KEY= # for AI_PROVIDER=openai +# AI_MODEL=llama3.1 # the model for your provider (e.g. llama3.1 for Ollama, sonnet +# # for claude-code, gpt-5 for codex). REQUIRED for non-Ollama: +# # without it the adapter falls back to a provider default, never +# # the Cloudflare Workers-AI id the core would otherwise pass. +# AI_EMBED_MODEL=bge-m3 # embedding model for RAG (openai-compatible /embeddings). MUST be +# # 1024-dimensional (e.g. bge-m3 or mxbai-embed-large via Ollama). +# # Used only when RAG is enabled (GITTENSORY_REVIEW_RAG + allowlist). diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml new file mode 100644 index 0000000000..9b4a7f3a66 --- /dev/null +++ b/.github/workflows/release-selfhost.yml @@ -0,0 +1,94 @@ +# Self-host image releases (#980). Cutting a `selfhost-v` tag builds the multi-arch image, pushes it +# to GHCR with version + latest + sha tags (with provenance + SBOM), and opens a GitHub Release. +# +# git tag selfhost-v0.1.0 && git push origin selfhost-v0.1.0 +# +# Pull: docker pull ghcr.io//gittensory-selfhost:0.1.0 +name: release-selfhost + +on: + push: + tags: + - "selfhost-v*" + workflow_dispatch: + inputs: + version: + description: "Version to publish (e.g. 0.1.0)" + required: true + +permissions: + contents: write # create the GitHub Release + packages: write # push to GHCR + +jobs: + release: + runs-on: ubuntu-latest + timeout-minutes: 40 + # Environment gate — requires reviewer approval before a release runs (configure under repo Settings > Environments). + environment: release + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + + - name: Resolve version + id: version + env: + INPUT_VERSION: ${{ github.event.inputs.version }} + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "v=${INPUT_VERSION}" >> "$GITHUB_OUTPUT" + else + echo "v=${GITHUB_REF_NAME#selfhost-v}" >> "$GITHUB_OUTPUT" + fi + + - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Image metadata + id: meta + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5 + with: + images: ghcr.io/${{ github.repository_owner }}/gittensory-selfhost + tags: | + type=raw,value=${{ steps.version.outputs.v }} + type=raw,value=latest + type=sha,format=short + labels: | + org.opencontainers.image.title=gittensory-selfhost + org.opencontainers.image.description=Self-hostable Gittensory review engine + org.opencontainers.image.version=${{ steps.version.outputs.v }} + + - name: Build + push (linux/amd64 + linux/arm64) + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + provenance: true + sbom: true + + - name: GitHub Release + if: github.event_name == 'push' + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 + with: + generate_release_notes: true + body: | + Self-host container image: + + ```bash + docker pull ghcr.io/${{ github.repository_owner }}/gittensory-selfhost:${{ steps.version.outputs.v }} + ``` + + Multi-arch (linux/amd64 + linux/arm64). See [docs/self-hosting.md](docs/self-hosting.md) for setup. + To include the Claude Code / Codex subscription CLIs, build locally with + `--build-arg INSTALL_AI_CLIS=true`. diff --git a/.github/workflows/selfhost.yml b/.github/workflows/selfhost.yml new file mode 100644 index 0000000000..aa791e38a4 --- /dev/null +++ b/.github/workflows/selfhost.yml @@ -0,0 +1,97 @@ +# Self-host stack CI (#980/#982). Provides integration coverage the main CI can't: +# 1. Postgres integration test — needs a real PG service container +# 2. Self-host bundle build validation (build-selfhost.mjs) +# 3. Docker image build + container smoke test (/health, /ready, /metrics) +# Unit tests and typecheck are NOT duplicated here — the main CI validate job covers them. +name: self-host + +on: + push: + branches: [main] + paths: + - "src/selfhost/**" + - "src/server.ts" + - "scripts/build-selfhost.mjs" + - "scripts/register-selfhost.mjs" + - "Dockerfile" + - "docker-compose.yml" + - "migrations/**" + - "test/unit/selfhost-*" + - "test/integration/selfhost-pg*" + - ".github/workflows/selfhost.yml" + pull_request: + paths: + - "src/selfhost/**" + - "src/server.ts" + - "scripts/build-selfhost.mjs" + - "scripts/register-selfhost.mjs" + - "Dockerfile" + - "docker-compose.yml" + - "migrations/**" + - "test/unit/selfhost-*" + - "test/integration/selfhost-pg*" + - ".github/workflows/selfhost.yml" + +# Least privilege — the smoke test only reads the repo; no writes, no packages. +permissions: + contents: read + +jobs: + build-boot: + name: build + boot smoke test + runs-on: ubuntu-latest + timeout-minutes: 20 + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_PASSWORD: devpw + POSTGRES_DB: gittensory + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" --health-interval 5s --health-timeout 5s --health-retries 10 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: "24" + cache: "npm" + + - name: Install deps + run: npm ci --ignore-scripts + + - name: Postgres integration test (real PG) + run: PG_TEST_URL=postgres://postgres:devpw@localhost:5432/gittensory npx vitest run test/integration/selfhost-pg.test.ts + + - name: Build the self-host bundle + run: node scripts/build-selfhost.mjs + + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Build the Docker image + run: | + docker buildx build \ + --cache-from type=gha \ + --cache-to type=gha,mode=max \ + --load \ + -t gittensory:selfhost-ci . + + - name: Boot the container + smoke-test /health, /ready, /metrics, migrations + run: | + docker run -d --name gt -p 8787:8787 gittensory:selfhost-ci + ok=0 + for _ in $(seq 1 30); do + if curl -sf http://127.0.0.1:8787/health >/dev/null; then ok=1; break; fi + sleep 2 + done + if [ "$ok" != "1" ]; then echo "::error::container did not become healthy"; docker logs gt; exit 1; fi + curl -sf http://127.0.0.1:8787/health | grep -q '"status":"ok"' + curl -sf http://127.0.0.1:8787/ready | grep -q '"ok":true' + curl -sf http://127.0.0.1:8787/metrics | grep -q 'gittensory_uptime_seconds' + docker logs gt 2>&1 | grep -q 'selfhost_migrations_applied' + echo "self-host smoke test passed" + docker rm -f gt diff --git a/.gitignore b/.gitignore index 0685cfe229..2ffd37d7f8 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,9 @@ dist-ssr/ .playwright-cli/ output/ .dev.vars +.env +.env.* +!.env.example *.local .DS_Store coverage/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..893b36a4fe --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +# Self-host image for gittensory-api (#980). Runs the SAME Worker handlers on Node via src/server.ts — +# the Cloudflare bindings become self-host adapters (D1 -> node:sqlite, Queue -> in-process). The hosted +# Cloudflare Worker (wrangler) deploy is unaffected. SECRETS ARE NEVER BAKED: supply them at run time via +# the .env file or mounted *_FILE secrets (see docker-compose.yml + .env.example). + +# --- build: install deps + bundle the Node entry -------------------------------------------------------- +# ECR Public Gallery mirrors Docker Official Images with no rate limits and no auth. +FROM public.ecr.aws/docker/library/node:24-slim AS build +WORKDIR /app +COPY package*.json ./ +# --ignore-scripts: no native builds are needed (SQLite is the built-in node:sqlite; @hono/node-server is +# pure JS; esbuild ships its binary as an optional dependency, not a script). +RUN npm ci --ignore-scripts +COPY . . +# --all: bundle every dependency into one self-contained dist/server.mjs, so the runtime image needs no +# node_modules (≈10× smaller). The bundle has zero `cloudflare:*` imports (stubbed at build), so no loader. +RUN node scripts/build-selfhost.mjs --all + +# --- runtime: slim, non-root ---------------------------------------------------------------------------- +FROM public.ecr.aws/docker/library/node:24-slim AS runtime +WORKDIR /app +ENV NODE_ENV=production \ + PLATFORM=self-hosted \ + PORT=8787 \ + DATABASE_PATH=/data/gittensory.sqlite \ + MIGRATIONS_DIR=/app/migrations +COPY --from=build /app/dist ./dist +COPY --from=build /app/migrations ./migrations +# Optional: bake the Claude Code / Codex CLIs so the `claude-code` / `codex` subscription providers (#979) +# work in-image. Build with `--build-arg INSTALL_AI_CLIS=true`. No credentials are baked — operators mint +# CLAUDE_CODE_OAUTH_TOKEN (`claude setup-token`) / codex auth at run time and pass it via the env. +ARG INSTALL_AI_CLIS=false +RUN if [ "$INSTALL_AI_CLIS" = "true" ]; then npm install -g @anthropic-ai/claude-code@2.1.187 @openai/codex@0.142.0 --ignore-scripts; fi +# Optional: enable visual review via an external Chrome sidecar (e.g. `browserless/chrome:latest`). +# Build with `--build-arg INSTALL_VISUAL_REVIEW=true` then set BROWSER_WS_ENDPOINT= at runtime. +ARG INSTALL_VISUAL_REVIEW=false +COPY --from=build /app/package*.json ./ +RUN if [ "$INSTALL_VISUAL_REVIEW" = "true" ]; then npm install puppeteer-core@22.13.1 --ignore-scripts; fi +# Data dir (the SQLite file) — owned by the unprivileged node user; mount a volume here to persist. +RUN mkdir -p /data && chown -R node:node /data /app +USER node +EXPOSE 8787 +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ + CMD node -e "fetch('http://127.0.0.1:'+(process.env.PORT||8787)+'/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" +CMD ["node", "dist/server.mjs"] diff --git a/caddy/Caddyfile b/caddy/Caddyfile new file mode 100644 index 0000000000..197bb96683 --- /dev/null +++ b/caddy/Caddyfile @@ -0,0 +1,34 @@ +# Caddy reverse proxy for gittensory (#980 self-host). +# Activated via: docker compose --profile caddy up +# +# DOMAIN is injected from the DOMAIN env var in docker-compose.yml. +# Set DOMAIN=reviews.yourcompany.com in .env — Caddy fetches a TLS cert from Let's Encrypt automatically. +# For local testing without a domain, set DOMAIN=localhost (self-signed cert, browser will warn). +# +# When using this profile, remove the `ports:` entry from the gittensory service in docker-compose.yml +# so port 8787 is NOT exposed publicly — all traffic should flow through Caddy on 443. + +{$DOMAIN} { + reverse_proxy gittensory:8787 { + # Surface the real client IP to the app (logged in access events). + header_up X-Forwarded-For {remote_host} + header_up X-Real-IP {remote_host} + } + + # Compress responses. + encode zstd gzip + + # Security headers. + header { + Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" + X-Content-Type-Options "nosniff" + X-Frame-Options "DENY" + Referrer-Policy "strict-origin-when-cross-origin" + -Server + } + + log { + output stderr + format json + } +} diff --git a/codecov.yml b/codecov.yml index a70fed5eda..09014bf891 100644 --- a/codecov.yml +++ b/codecov.yml @@ -7,6 +7,10 @@ # # `project` (whole-repo total) is informational only: it is reported as a trend # but never blocks a merge. vitest keeps a loose 90% local backstop separately. +codecov: + # Don't post a verdict until the CI run that produced the report has finished. + require_ci_to_pass: true + coverage: status: patch: @@ -19,9 +23,6 @@ coverage: default: informational: true -# Don't post a verdict until the CI run that produced the report has finished. -require_ci_to_pass: true - comment: layout: "condensed_header, diff, flags, files" require_changes: false @@ -33,3 +34,12 @@ ignore: - "apps/**" - "test/**" - "scripts/**" + # Self-host process entry + build-time stubs: exercised by the Docker build+boot smoke test + # (.github/workflows/selfhost.yml), not unit-coverable without booting a server/subprocess. + - "src/server.ts" + - "src/selfhost/cf-workers-shim.ts" + - "src/selfhost/stubs/**" + # Postgres runtime adapters: validated by the real-Postgres integration test (test/integration/selfhost-pg.ts, + # gated on PG_TEST_URL) + the real-PG boot. The dialect translation itself IS unit-tested (pg-dialect.ts). + - "src/selfhost/pg-adapter.ts" + - "src/selfhost/pg-queue.ts" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000..898bddb0dc --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,264 @@ +# One-command self-host for gittensory (#980): docker compose up --build +# +# SECRETS: copy .env.example → .env, fill in your GitHub App credentials and any optional secrets. +# The file is gitignored. Every value below is a sample placeholder — never commit real secrets. +# +# PROFILES — activate optional services by passing --profile (combine freely): +# +# (none) SQLite single-node stack (default — no flags needed) +# --profile postgres pgvector/pg16 shared database (multi-instance capable) +# --profile pgbouncer PgBouncer connection pooler in front of Postgres +# --profile redis Redis fixed-window rate limiter +# --profile ollama Local Ollama AI backend +# --profile litestream Continuous SQLite backup to S3/B2/R2 via Litestream +# --profile caddy Caddy HTTPS terminator with auto-TLS (set DOMAIN= in .env) +# --profile observability Prometheus + Grafana dashboards (pre-wired to /metrics) +# --profile tailscale Tailscale sidecar — access the stack via your tailnet +# --profile runners GitHub Actions self-hosted runner +# +# Examples: +# docker compose up --build # SQLite, no AI +# docker compose --profile postgres --profile caddy up -d # Postgres + HTTPS +# docker compose --profile observability up -d # add dashboards to anything +# docker compose --profile tailscale --profile runners up -d # tailnet + CI runners + +services: + + # ── Core app (always runs) ───────────────────────────────────────────────── + gittensory: + build: + context: . + restart: unless-stopped + ports: + # Remove this when using the caddy profile — Caddy becomes the public listener. + - "${PORT:-8787}:8787" + env_file: + - path: .env + required: false + environment: + PORT: "8787" + DATABASE_PATH: /data/gittensory.sqlite + # Uncomment the next two lines and activate --profile postgres to use Postgres: + # DATABASE_URL: postgres://gittensory:${POSTGRES_PASSWORD:-CHANGEME}@postgres:5432/gittensory + # PGVECTOR_ENABLED: "true" + # With --profile pgbouncer, route through the pooler instead of postgres directly: + # DATABASE_URL: postgres://gittensory:${POSTGRES_PASSWORD:-CHANGEME}@pgbouncer:5432/gittensory + # Uncomment for Redis rate limiting (--profile redis): + # REDIS_URL: redis://redis:6379 + # Uncomment for Ollama AI (--profile ollama): + # AI_PROVIDER: ollama + # AI_BASE_URL: http://ollama:11434/v1 + volumes: + - gittensory-data:/data + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:8787/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"] + interval: 30s + timeout: 5s + start_period: 20s + retries: 3 + + # ── Postgres (--profile postgres | --profile pgbouncer) ─────────────────── + postgres: + image: pgvector/pgvector:pg16 + restart: unless-stopped + profiles: ["postgres", "pgbouncer"] + environment: + POSTGRES_USER: gittensory + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-CHANGEME} + POSTGRES_DB: gittensory + volumes: + - gittensory-pg:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U gittensory"] + interval: 10s + retries: 5 + + # ── PgBouncer (--profile pgbouncer) ─────────────────────────────────────── + # Transaction-mode pooler — allows hundreds of app clients with a small PG connection cap. + # Set DATABASE_URL in gittensory to: postgres://gittensory:@pgbouncer:5432/gittensory + pgbouncer: + image: edoburu/pgbouncer:latest + restart: unless-stopped + profiles: ["pgbouncer"] + depends_on: + postgres: + condition: service_healthy + environment: + DB_HOST: postgres + DB_PORT: "5432" + DB_NAME: gittensory + DB_USER: gittensory + DB_PASSWORD: ${POSTGRES_PASSWORD:-CHANGEME} + LISTEN_PORT: "5432" + POOL_MODE: transaction + MAX_CLIENT_CONN: "200" + DEFAULT_POOL_SIZE: "20" + AUTH_TYPE: md5 + + # ── Redis (--profile redis) ──────────────────────────────────────────────── + redis: + image: redis:7-alpine + restart: unless-stopped + profiles: ["redis"] + volumes: + - gittensory-redis:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + retries: 5 + + # ── Qdrant (--profile qdrant) ───────────────────────────────────────────── + # Dedicated vector database for RAG — replaces the built-in sqlite-vec / pgvector when + # QDRANT_URL=http://qdrant:6333 is set. Scales to millions of vectors with ANN search. + # REST API: http://localhost:6333 gRPC: localhost:6334 Dashboard: http://localhost:6333/dashboard + qdrant: + image: qdrant/qdrant:latest + restart: unless-stopped + profiles: ["qdrant"] + ports: + - "6333:6333" # REST API + Web UI + - "6334:6334" # gRPC + volumes: + - qdrant-data:/qdrant/storage + # Qdrant's minimal image has no curl/wget/nc — healthcheck omitted. + # The service is ready when the gittensory startup log shows {"event":"selfhost_vectorize","backend":"qdrant"}. + + # ── Ollama (--profile ollama) ────────────────────────────────────────────── + # After `docker compose --profile ollama up -d`, pull a model: + # docker compose exec ollama ollama pull llama3.2 + # Then set AI_PROVIDER=ollama and AI_BASE_URL=http://ollama:11434/v1 in .env. + ollama: + image: ollama/ollama:latest + restart: unless-stopped + profiles: ["ollama"] + volumes: + - ollama-models:/root/.ollama + + # ── Litestream (--profile litestream) ───────────────────────────────────── + # Continuous WAL backup of the SQLite DB to S3/B2/R2. Copy litestream.yml.example + # → litestream.yml and fill in your bucket. Set LITESTREAM_* secrets in .env. + litestream: + image: litestream/litestream:latest + restart: unless-stopped + profiles: ["litestream"] + command: replicate + depends_on: + gittensory: + condition: service_healthy + volumes: + - gittensory-data:/data + - ./litestream.yml:/etc/litestream.yml:ro + environment: + LITESTREAM_ACCESS_KEY_ID: ${LITESTREAM_ACCESS_KEY_ID} + LITESTREAM_SECRET_ACCESS_KEY: ${LITESTREAM_SECRET_ACCESS_KEY} + LITESTREAM_ENDPOINT: ${LITESTREAM_ENDPOINT:-} + LITESTREAM_REGION: ${LITESTREAM_REGION:-us-east-1} + + # ── Caddy (--profile caddy) ──────────────────────────────────────────────── + # Auto-TLS via Let's Encrypt. Set DOMAIN=reviews.yourcompany.com in .env, then + # remove the plain `ports:` entry from the gittensory service above. + caddy: + image: caddy:2-alpine + restart: unless-stopped + profiles: ["caddy"] + ports: + - "80:80" + - "443:443" + - "443:443/udp" # HTTP/3 QUIC + volumes: + - ./caddy/Caddyfile:/etc/caddy/Caddyfile:ro + - caddy-data:/data + - caddy-config:/config + environment: + DOMAIN: ${DOMAIN:-localhost} + depends_on: + gittensory: + condition: service_healthy + + # ── Observability (--profile observability) ──────────────────────────────── + # Prometheus scrapes /metrics; Grafana visualises it. + # Grafana UI: http://localhost:3000 (admin / admin — change on first login). + prometheus: + image: prom/prometheus:latest + restart: unless-stopped + profiles: ["observability"] + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=30d" + + grafana: + image: grafana/grafana:latest + restart: unless-stopped + profiles: ["observability"] + depends_on: [prometheus] + ports: + - "3000:3000" + volumes: + - grafana-data:/var/lib/grafana + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + environment: + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env before using --profile observability} + GF_USERS_ALLOW_SIGN_UP: "false" + + # ── Tailscale (--profile tailscale) ─────────────────────────────────────── + # Joins your tailnet so the stack is accessible via Tailscale IP/hostname — no + # public ports exposed. Generate an auth key at tailscale.com/settings/keys and + # set TS_AUTHKEY= in .env. The gittensory service is reachable at the tailnet IP on port 8787. + tailscale: + image: ghcr.io/tailscale/tailscale:stable + restart: unless-stopped + profiles: ["tailscale"] + hostname: gittensory + cap_add: + - NET_ADMIN + - SYS_MODULE + environment: + TS_AUTHKEY: ${TS_AUTHKEY} + TS_STATE_DIR: /var/lib/tailscale + TS_EXTRA_ARGS: ${TS_EXTRA_ARGS:-} + volumes: + - tailscale-state:/var/lib/tailscale + - /dev/net/tun:/dev/net/tun + network_mode: host # Tailscale needs host networking to advertise the host's address + + # ── Self-hosted GitHub Actions runner (--profile runners) ───────────────── + # Runs `runs-on: self-hosted` jobs on this machine. Set RUNNER_TOKEN= (or ACCESS_TOKEN=) + # and RUNNER_REPO_URL= (e.g. https://github.com/your-org/your-repo) in .env. + # Get a token at: https://github.com///settings/actions/runners/new + runner: + # Pin to a specific version tag — `latest` is mutable. Find a digest via: + # docker pull myoung34/github-runner:ubuntu-22.04 && docker inspect --format='{{index .RepoDigests 0}}' myoung34/github-runner:ubuntu-22.04 + image: myoung34/github-runner:ubuntu-22.04 + restart: unless-stopped + profiles: ["runners"] + environment: + RUNNER_SCOPE: ${RUNNER_SCOPE:-repo} + REPO_URL: ${RUNNER_REPO_URL} + RUNNER_TOKEN: ${RUNNER_TOKEN} + ACCESS_TOKEN: ${RUNNER_ACCESS_TOKEN:-} + RUNNER_NAME: ${RUNNER_NAME:-gittensory-runner} + LABELS: ${RUNNER_LABELS:-self-hosted,linux,x64} + RUNNER_WORKDIR: /tmp/runner + # Docker builds are disabled by default. To enable DinD (Docker-in-Docker) for workflows that + # need `docker build/push`, add a `dind` sidecar service and set DOCKER_HOST here. Mounting + # /var/run/docker.sock grants container-escape risk and is intentionally omitted. + volumes: + - runner-work:/tmp/runner + +volumes: + gittensory-data: + gittensory-pg: + gittensory-redis: + qdrant-data: + ollama-models: + caddy-data: + caddy-config: + prometheus-data: + grafana-data: + tailscale-state: + runner-work: diff --git a/docs/self-hosting.md b/docs/self-hosting.md new file mode 100644 index 0000000000..cb10e47b3a --- /dev/null +++ b/docs/self-hosting.md @@ -0,0 +1,200 @@ +# Self-hosting Gittensory + +Gittensory ships as a Cloudflare Worker, but the **same** review engine runs unchanged on a plain Node +container so you can self-host it next to your own GitHub App. `docker compose up` gives you the full +reviewer — webhooks, the deterministic gate, AI summaries, the maintain/sweep cron, and (optionally) full +maintainer autonomy — backed by a local SQLite database. + +> **How it works (one paragraph).** The Worker's Cloudflare bindings are swapped for self-host adapters and +> nothing else changes: **D1 → `node:sqlite`** (a faithful `D1Database` shim, so Drizzle + every raw query + +> all 56 schema migrations run byte-for-byte the same), **Queue → an in-process FIFO worker** (same +> `processJob`), and the **cron** is a timer that calls the same `scheduled()` handler. The Hono app is served +> with `@hono/node-server`. See [`src/server.ts`](../src/server.ts) and [`src/selfhost/`](../src/selfhost). + +--- + +## 1. Quick start + +```bash +cp .env.example .env # then edit .env — see §3 +docker compose up --build +curl localhost:8787/health # {"status":"ok"} +``` + +On first boot the container creates the SQLite database on the `gittensory-data` volume and applies all 56 +migrations automatically (`{"event":"selfhost_migrations_applied","count":56}` in the logs). Point your +GitHub App's webhook at `https:///v1/github/webhook` (expose port 8787 behind your own TLS). + +**Or use the published image** (multi-arch, ~254 MB) instead of building: + +```bash +docker run -p 8787:8787 --env-file .env -v gittensory-data:/data \ + ghcr.io//gittensory-selfhost:latest # or pin a version, e.g. :0.1.0 +``` + +To run without Docker: + +```bash +npm ci +node scripts/build-selfhost.mjs # external mode (fast local rebuilds) +node --import ./scripts/register-selfhost.mjs dist/server.mjs +``` + +Releases are cut by pushing a `selfhost-v` tag (e.g. `selfhost-v0.1.0`): CI builds the multi-arch +image, pushes it to GHCR with `:`, `:latest`, and `:sha-…` tags (with provenance + SBOM), and opens a +GitHub Release. + +--- + +## 2. Create the GitHub App + +**One-click (recommended):** before setting any GitHub secrets, boot the container and visit **`/setup`**. It +creates the App for you via GitHub's App-manifest flow (correct permissions/events + webhook URL), then writes +the credentials to `/data/gittensory-app.env`. Add those to your `.env`, install the App on your repos, and +restart. `/setup` is disabled once `GITHUB_APP_ID` is set, so it can't rebind a live install. + +**Or manually**, create a GitHub App (the hosted gittensory[bot] is separate) with: + +- **Webhook URL** `https:///v1/github/webhook`, and a **webhook secret** (→ `GITHUB_WEBHOOK_SECRET`). +- **Permissions**: Pull requests (read/write), Contents (read; read/write if you want merge), Issues + (read/write), Checks (read), Metadata (read). Commit statuses (read). +- **Events**: Pull request, Pull request review, Push, Issues, Check suite, Check run, Status. +- Generate a **private key** (→ `GITHUB_APP_PRIVATE_KEY`), and note the **App ID** (→ `GITHUB_APP_ID`) and the + app **slug** (→ `GITHUB_APP_SLUG`). Install the app on the repos you want reviewed. + +--- + +## 3. Configuration + +Everything is environment variables — see [`.env.example`](../.env.example) for the annotated list (it holds +**sample placeholders only; never commit a real `.env`** — it is gitignored). The required core secrets: + +| Variable | What it is | +| --- | --- | +| `GITHUB_APP_ID` / `GITHUB_APP_SLUG` | your GitHub App's id + slug | +| `GITHUB_APP_PRIVATE_KEY` | the App's PKCS#8 private key (or mount `GITHUB_APP_PRIVATE_KEY_FILE`) | +| `GITHUB_WEBHOOK_SECRET` | the webhook secret you set on the App | +| `GITTENSOR_REGISTRY_URL` | registry endpoint (or any reachable placeholder if you don't use the registry) | +| `GITTENSORY_API_TOKEN` / `GITTENSORY_MCP_TOKEN` / `INTERNAL_JOB_TOKEN` | bearer tokens — generate your own (`openssl rand -hex 32`) | + +Runtime knobs: `PORT` (default 8787), `DATABASE_PATH` (default `/data/gittensory.sqlite`), `CRON_INTERVAL_MS` +(default 120000 ≈ the hosted every-2-minutes cron). + +**Secrets via files.** Any `FOO_FILE=/run/secrets/foo` is read into `FOO` at startup (Docker/Compose +secrets, multi-line keys) — an explicit `FOO` always wins. + +--- + +## 4. AI provider (optional) + +Without an AI provider the review still runs fully — deterministic signals, the gate, merge/close decisions — +and only the AI **summary** degrades to "unavailable". To enable AI, set `AI_PROVIDER`: + +| `AI_PROVIDER` | Backend | Extra config | +| --- | --- | --- | +| `ollama` / `openai-compatible` / `openai` | any OpenAI-compatible `/chat/completions` endpoint (Ollama, OpenAI, Groq, Together, OpenRouter, vLLM, Gemini's OpenAI-compat endpoint, …) | `AI_BASE_URL`, `AI_API_KEY` (or `OPENAI_API_KEY`), `AI_MODEL` | +| `anthropic` | **native Anthropic Messages API** (BYOK — bills your API key) | `ANTHROPIC_API_KEY`, `AI_MODEL` (e.g. `claude-sonnet-4-6`) | +| `claude-code` | your **Claude** subscription via the `claude` CLI (read-only, headless) | `CLAUDE_CODE_OAUTH_TOKEN` (from `claude setup-token`), `AI_MODEL` (e.g. `sonnet`) | +| `codex` | your **Codex** subscription via the `codex` CLI | local `codex` auth, `AI_MODEL` (e.g. `gpt-5`) | + +**Fallback chain.** `AI_PROVIDER` accepts a comma-separated list and tries each in order until one succeeds — +e.g. `AI_PROVIDER=anthropic,ollama` uses the Anthropic API first and falls back to a local Ollama model if it +errors. If every provider fails, the AI summary degrades to "unavailable" and the review still runs. + +**Subscription CLIs in the image.** The `claude-code` / `codex` providers need their CLI present. Build the +image with `--build-arg INSTALL_AI_CLIS=true` (or `docker compose build --build-arg INSTALL_AI_CLIS=true`) to +bake them in, then provide `CLAUDE_CODE_OAUTH_TOKEN` / codex auth at run time. No credentials are baked in. + +**Local RAG (retrieval-augmented review).** Self-host ships a SQLite-backed vector store, so RAG works without +Cloudflare Vectorize. Enable it with `GITTENSORY_REVIEW_RAG=true` + the repo in `GITTENSORY_REVIEW_REPOS`, and +point at an **embedding-capable** OpenAI-compatible provider (Ollama) with a **1024-dimensional** model via +`AI_EMBED_MODEL` (e.g. `bge-m3` or `mxbai-embed-large`). Embeddings + chunk vectors are stored in the same +SQLite DB (`_selfhost_vectors`) and queried by cosine similarity. Without an embedding model, RAG degrades to +no-context (the review still runs). + +> **Set `AI_MODEL`.** The core would otherwise hand the adapter a Cloudflare Workers-AI model id +> (`@cf/meta/...`) that Ollama / `claude` / `codex` can't use. The adapter ignores that id in favour of +> `AI_MODEL` (falling back to a provider default), so always set `AI_MODEL` to a real model for your provider. +> The `claude`/`codex` CLIs must be installed and authenticated in the runtime (a CLI-bearing image variant +> is a follow-up); without `AI_MODEL` + a working CLI, the call throws and the review degrades. + +The local-AI default is Ollama: uncomment the `ollama` service in `docker-compose.yml`, set +`AI_PROVIDER=ollama` + `AI_BASE_URL=http://ollama:11434/v1`, then `docker compose exec ollama ollama pull +`. + +**Subscription safety.** The CLI providers run as a read-only subprocess with billable API keys +(`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, …) **scrubbed from the child environment** so a misconfigured CLI +can't silently bill the metered API instead of your subscription. Any error, empty output, or Claude-Code +`is_error` envelope makes the call throw, so the review degrades rather than surfacing an error string as the +model's answer. (Codex is gated/unverified — treat it as best-effort.) + +--- + +## 5. Review modes — advisory vs. full maintainer + +Self-host runs the identical engine, so the behavior is configured exactly as on the hosted product: + +- **Advisory (default).** With Contents write withheld (or autonomy off), Gittensory posts its unified review + comment and check, but never merges or closes — a recommendation engine. +- **Full maintainer.** Grant Contents write and enable per-repo autonomy (merge / close / approve) — the bot + acts on its decisions, gated by the same guardrails (protected-path manual-review globs, owner-PR + no-auto-close, mergeability + green-CI before approve). + +Per-PR capabilities (safety scan, CI/full-file grounding, RAG, unified comment, content lane, self-tune, +parity audit) are the `GITTENSORY_REVIEW_*` flags — every flag defaults **off** and is fully inert until +turned on. Per-repo settings (autonomy, required approvals, protected paths) live in `.gittensory.yml` / +repository settings. The authoritative reference for all of these is +[`docs/review-configuration.md`](./review-configuration.md). + +--- + +## 6. Operations + +- **Endpoints.** + - `GET /health` — binding-free liveness (the container `HEALTHCHECK` uses it). + - `GET /ready` — readiness: returns `503` until the DB answers **and** migrations are applied + (`{"ok":true,"checks":{"db":true,"migrations":true}}`). Use it as your orchestrator's readiness probe. + - `GET /metrics` — Prometheus text: `gittensory_queue_pending` / `_dead`, `gittensory_jobs_*_total` + (enqueued/processed/failed/dead), `gittensory_uptime_seconds`, `gittensory_http_requests_total`. +- **Durable queue.** Jobs are persisted in SQLite (`_selfhost_jobs`), not held in memory — a restart or crash + **re-claims** in-flight work instead of losing it. Failures retry with exponential backoff and dead-letter + after `maxRetries` (visible via `gittensory_queue_dead`). +- **Graceful shutdown.** On `SIGTERM`/`SIGINT` the server stops accepting requests, lets the queue finish its + in-flight job, checkpoints the WAL, and closes the DB before exiting. +- **Logs** are structured JSON (`selfhost_listening`, `selfhost_migrations_applied`, `selfhost_ai_provider`, + `selfhost_queue_recovered`, `selfhost_job_dead`, `selfhost_cron_error`, `selfhost_shutdown`, …). +- **Data + backup.** Everything is the single SQLite file on the `gittensory-data` volume (WAL mode). Back up + by snapshotting the volume or copying the `.sqlite` file. Migrations are idempotent and re-checked at boot. + For **continuous, point-in-time backup**, enable the optional [Litestream](https://litestream.io) sidecar in + `docker-compose.yml` (copy `litestream.yml.example` → `litestream.yml`, set your bucket + credentials); it + streams every change to S3/B2/MinIO/R2. +- **App-level metrics.** Enable `GITTENSORY_REVIEW_OPS=true` for the read-only gate-block anomaly scan and the + bearer-gated `GET /v1/internal/ops/stats` aggregate. + +--- + +## 7. Scaling out — Postgres + Redis (multi-instance) + +The SQLite default is ideal for a single instance. To run **multiple replicas** behind a load balancer, switch +to a shared Postgres + Redis: + +- **`DATABASE_URL=postgres://user:pw@host:5432/db`** — uses Postgres instead of SQLite. The same 56 migrations + apply (translated to Postgres at startup), and the job queue moves to Postgres with `FOR UPDATE SKIP LOCKED` + claiming, so replicas never double-process a job. +- **`REDIS_URL=redis://host:6379`** — a shared fixed-window rate limiter across all replicas. + +Uncomment the `postgres` + `redis` services in `docker-compose.yml`, set the two URLs on the app service, and +scale (`docker compose up --scale gittensory=3`). Postgres is **beta**: the migrations + the exercised query +paths are validated against a real Postgres, but report any dialect edge cases. RAG (the SQLite vector store) +is **not** available on the Postgres backend yet — it degrades to no-context. + +## 8. What is not on self-host + +These are Cloudflare-platform features; they degrade cleanly and the core reviewer is unaffected: + +- **Visual PR capture** (Browser Rendering binding) — off; reviews run text-only. +- **The `/mcp` server** (Durable-Object-backed Agents SDK) — returns `501`. The deterministic API + review + path is unaffected; a native MCP-on-Node port is a follow-up. +- **Distributed rate limiting** (RateLimiter Durable Object) — off by default; set `REDIS_URL` for a + Redis-backed fixed-window limiter (see §7). Otherwise put a reverse proxy / WAF in front. +- **Vectorize-backed RAG** and **R2 audit storage** — inert unless you wire equivalent backends. diff --git a/grafana/dashboards/gittensory.json b/grafana/dashboards/gittensory.json new file mode 100644 index 0000000000..e83163cc21 --- /dev/null +++ b/grafana/dashboards/gittensory.json @@ -0,0 +1,263 @@ +{ + "__inputs": [], + "__requires": [ + { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.0.0" }, + { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } + ], + "annotations": { "list": [] }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "title": "System Health", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "id": 1, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Uptime", + "type": "stat", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gittensory_uptime_seconds", "legendFormat": "uptime" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 10 }, { "color": "red", "value": 50 }] }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "id": 2, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Queue Pending", + "type": "stat", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gittensory_queue_pending", "legendFormat": "pending" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 10 }] }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "id": 3, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Dead-Letter Jobs", + "type": "stat", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gittensory_queue_dead", "legendFormat": "dead" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "id": 6, + "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Total Jobs Processed", + "type": "stat", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gittensory_jobs_processed_total", "legendFormat": "processed" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 }] }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "id": 7, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Webhook Dedups (total)", + "type": "stat", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gittensory_webhook_dedup_total", "legendFormat": "deduped" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "id": 8, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Qdrant Errors (total)", + "type": "stat", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(gittensory_qdrant_errors_total) or vector(0)", "legendFormat": "errors" }] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "id": 101, + "title": "HTTP & Webhooks", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "reqps" } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "id": 4, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "HTTP Request Rate", + "type": "timeseries", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_http_requests_total[2m])", "legendFormat": "requests/s" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_webhook_dedup_total[2m])", "legendFormat": "dedup/s" } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "short" } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "id": 9, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "Queue Depth Over Time", + "type": "timeseries", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gittensory_queue_pending", "legendFormat": "pending" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gittensory_queue_dead", "legendFormat": "dead-letter" } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "id": 102, + "title": "Job Pipeline", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "ops" } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, + "id": 5, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "Job Throughput", + "type": "timeseries", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_jobs_processed_total[2m])", "legendFormat": "processed/s" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_jobs_enqueued_total[2m])", "legendFormat": "enqueued/s" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_jobs_failed_total[2m])", "legendFormat": "failed/s" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_jobs_dead_total[2m])", "legendFormat": "dead/s" } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "lineWidth": 2, "fillOpacity": 10 }, + "unit": "percentunit", + "min": 0, + "max": 1 + } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, + "id": 10, + "options": { "legend": { "calcs": ["mean", "last"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "Job Failure Rate", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "rate(gittensory_jobs_failed_total[5m]) / (rate(gittensory_jobs_processed_total[5m]) + rate(gittensory_jobs_failed_total[5m]) + 0.0001)", + "legendFormat": "failure %" + } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, + "id": 103, + "title": "Vector Store (Qdrant)", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "ops" } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "id": 11, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "Qdrant Query Rate", + "type": "timeseries", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_qdrant_queries_total[2m])", "legendFormat": "queries/s" } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "ops" } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "id": 12, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "title": "Qdrant Upserts & Errors", + "type": "timeseries", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_qdrant_upserts_total[2m])", "legendFormat": "upserts/s" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(gittensory_qdrant_errors_total[2m])", "legendFormat": "errors/s" } + ] + }, + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["gittensory", "self-host"], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "type": "datasource" + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Gittensory Self-Host", + "uid": "gittensory-selfhost", + "version": 2 +} diff --git a/grafana/provisioning/dashboards/provider.yml b/grafana/provisioning/dashboards/provider.yml new file mode 100644 index 0000000000..80cbbbc20f --- /dev/null +++ b/grafana/provisioning/dashboards/provider.yml @@ -0,0 +1,9 @@ +apiVersion: 1 +providers: + - name: gittensory + folder: Gittensory + type: file + disableDeletion: true + editable: false + options: + path: /var/lib/grafana/dashboards diff --git a/grafana/provisioning/datasources/prometheus.yml b/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000000..2d433996f6 --- /dev/null +++ b/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,8 @@ +apiVersion: 1 +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/litestream.yml.example b/litestream.yml.example new file mode 100644 index 0000000000..dc4ae68b0c --- /dev/null +++ b/litestream.yml.example @@ -0,0 +1,14 @@ +# Litestream continuous-backup config for the self-host SQLite DB (SAMPLE — placeholders only). +# Streams /data/gittensory.sqlite to your object store so you can restore to any point in time. +# Copy to litestream.yml, edit the bucket/endpoint, and mount it at /etc/litestream.yml (see docker-compose.yml). +# Docs: https://litestream.io +dbs: + - path: /data/gittensory.sqlite + replicas: + - type: s3 + bucket: your-backup-bucket + path: gittensory + # For S3-compatible stores (Backblaze B2, MinIO, R2) set the endpoint; omit it for AWS S3. + endpoint: ${LITESTREAM_ENDPOINT} + region: ${LITESTREAM_REGION} + # Credentials come from the environment (LITESTREAM_ACCESS_KEY_ID / LITESTREAM_SECRET_ACCESS_KEY). diff --git a/package-lock.json b/package-lock.json index 7e43e83906..f43fadd4a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,11 +15,14 @@ "dependencies": { "@asteasolutions/zod-to-openapi": "^8.5.0", "@cloudflare/puppeteer": "^1.1.0", + "@hono/node-server": "^2.0.6", "@modelcontextprotocol/sdk": "1.29.0", "@octokit/core": "^7.0.6", "agents": "^0.16.2", "drizzle-orm": "^0.45.2", "hono": "^4.12.26", + "ioredis": "^5.11.1", + "pg": "^8.22.0", "yaml": "^2.9.0", "zod": "^4.4.3" }, @@ -27,6 +30,7 @@ "@cloudflare/vitest-pool-workers": "^0.16.17", "@tktco/node-actionlint": "^1.6.0", "@types/node": "^24.13.2", + "@types/pg": "^8.20.0", "@types/pixelmatch": "^5.2.6", "@types/pngjs": "^6.0.5", "@vitest/coverage-v8": "^4.1.9", @@ -1932,12 +1936,12 @@ "license": "MIT" }, "node_modules/@hono/node-server": { - "version": "1.19.14", - "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz", - "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==", + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.0.6.tgz", + "integrity": "sha512-7DeRlKG57JDBNZ5Qj2jwVdgwQy4b0tLubRLl3zCf91/rCf9i7p1V5FtW/yWibm1uUHE493ts9ZXH/7g/LQWl+g==", "license": "MIT", "engines": { - "node": ">=18.14.1" + "node": ">=20" }, "peerDependencies": { "hono": "^4" @@ -2511,6 +2515,12 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@ioredis/commands": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", + "integrity": "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==", + "license": "MIT" + }, "node_modules/@isaacs/fs-minipass": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", @@ -2716,6 +2726,18 @@ } } }, + "node_modules/@modelcontextprotocol/sdk/node_modules/@hono/node-server": { + "version": "1.19.14", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz", + "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==", + "license": "MIT", + "engines": { + "node": ">=18.14.1" + }, + "peerDependencies": { + "hono": "^4" + } + }, "node_modules/@modelcontextprotocol/sdk/node_modules/content-type": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", @@ -5912,6 +5934,18 @@ "undici-types": "~7.18.0" } }, + "node_modules/@types/pg": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.20.0.tgz", + "integrity": "sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, "node_modules/@types/pixelmatch": { "version": "5.2.6", "resolved": "https://registry.npmjs.org/@types/pixelmatch/-/pixelmatch-5.2.6.tgz", @@ -7317,6 +7351,15 @@ "node": ">=6" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz", + "integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/cmdk": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/cmdk/-/cmdk-1.1.1.tgz", @@ -7861,6 +7904,15 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -9915,6 +9967,28 @@ "node": ">=12" } }, + "node_modules/ioredis": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.11.1.tgz", + "integrity": "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.10.0", + "cluster-key-slot": "1.1.1", + "debug": "4.4.3", + "denque": "2.1.0", + "redis-errors": "1.2.0", + "redis-parser": "3.0.0", + "standard-as-callback": "2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ip-address": { "version": "10.2.0", "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", @@ -11754,6 +11828,95 @@ "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", "license": "MIT" }, + "node_modules/pg": { + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz", + "integrity": "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.0", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.15.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz", + "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz", + "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.15.0.tgz", + "integrity": "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -12033,6 +12196,45 @@ "dev": true, "license": "MIT" }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prebuild-install": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", @@ -12643,6 +12845,27 @@ "decimal.js-light": "^2.4.1" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -13257,6 +13480,15 @@ "source-map": "^0.6.0" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sprintf-js": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", @@ -13288,6 +13520,12 @@ "dev": true, "license": "MIT" }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -14631,6 +14869,15 @@ "dev": true, "license": "MIT" }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index b33cce39d7..7f34039a5e 100644 --- a/package.json +++ b/package.json @@ -62,11 +62,14 @@ "dependencies": { "@asteasolutions/zod-to-openapi": "^8.5.0", "@cloudflare/puppeteer": "^1.1.0", + "@hono/node-server": "^2.0.6", "@modelcontextprotocol/sdk": "1.29.0", "@octokit/core": "^7.0.6", "agents": "^0.16.2", "drizzle-orm": "^0.45.2", "hono": "^4.12.26", + "ioredis": "^5.11.1", + "pg": "^8.22.0", "yaml": "^2.9.0", "zod": "^4.4.3" }, @@ -74,6 +77,7 @@ "@cloudflare/vitest-pool-workers": "^0.16.17", "@tktco/node-actionlint": "^1.6.0", "@types/node": "^24.13.2", + "@types/pg": "^8.20.0", "@types/pixelmatch": "^5.2.6", "@types/pngjs": "^6.0.5", "@vitest/coverage-v8": "^4.1.9", diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml new file mode 100644 index 0000000000..e2de3c51c2 --- /dev/null +++ b/prometheus/prometheus.yml @@ -0,0 +1,13 @@ +# Prometheus scrape config for gittensory self-host (#980 observability). +# Activated via: docker compose --profile observability up +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: gittensory + static_configs: + - targets: ["gittensory:8787"] + metrics_path: /metrics + scrape_interval: 15s + scrape_timeout: 10s diff --git a/railway.json b/railway.json new file mode 100644 index 0000000000..f9bde10ee2 --- /dev/null +++ b/railway.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://railway.com/railway.schema.json", + "build": { + "builder": "DOCKERFILE", + "dockerfilePath": "./Dockerfile" + }, + "deploy": { + "healthcheckPath": "/health", + "healthcheckTimeout": 60, + "restartPolicyType": "ON_FAILURE", + "restartPolicyMaxRetries": 5, + "drainingSeconds": 15, + "numReplicas": 1 + } +} diff --git a/scripts/build-selfhost.mjs b/scripts/build-selfhost.mjs new file mode 100644 index 0000000000..a8c9ee1c4f --- /dev/null +++ b/scripts/build-selfhost.mjs @@ -0,0 +1,50 @@ +// Bundle the self-host Node entry (src/server.ts) into dist/server.mjs. +// default → node_modules stay external (resolved at runtime; fast local dev rebuilds). +// --all / SELFHOST_BUNDLE_ALL=1 → bundle EVERYTHING into one self-contained file (the Docker image needs no +// node_modules → a ~10× smaller image). node: builtins stay external (platform:node). +// In both modes the Cloudflare-only specifiers resolve to Node stubs via the plugin (precedence over external), +// so the bundle has zero `cloudflare:*` imports. +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import esbuild from "esbuild"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const bundleAll = process.env.SELFHOST_BUNDLE_ALL === "1" || process.argv.includes("--all"); + +await esbuild.build({ + entryPoints: [resolve(root, "src/server.ts")], + bundle: true, + platform: "node", + format: "esm", + target: "node22", + outfile: resolve(root, "dist/server.mjs"), + // External: nothing (bundle all) vs every package (external). node: builtins are always external on node. + ...(bundleAll ? {} : { packages: "external" }), + // Bundling CJS deps into an ESM output needs require/__dirname/__filename shimmed (some deps call them). + ...(bundleAll + ? { + banner: { + js: [ + "import { createRequire as __createRequire } from 'node:module';", + "import { fileURLToPath as __fileURLToPath } from 'node:url';", + "import { dirname as __pathDirname } from 'node:path';", + "const require = __createRequire(import.meta.url);", + "const __filename = __fileURLToPath(import.meta.url);", + "const __dirname = __pathDirname(__filename);", + ].join("\n"), + }, + } + : {}), + plugins: [ + { + name: "selfhost-stubs", + setup(build) { + // Cloudflare-only modules → Node stubs (their features are inert/degraded on self-host). + build.onResolve({ filter: /^cloudflare:workers$/ }, () => ({ path: resolve(root, "src/selfhost/cf-workers-shim.ts") })); + build.onResolve({ filter: /^@cloudflare\/puppeteer$/ }, () => ({ path: resolve(root, "src/selfhost/stubs/puppeteer.ts") })); + build.onResolve({ filter: /^agents\/mcp$/ }, () => ({ path: resolve(root, "src/selfhost/stubs/agents-mcp.ts") })); + }, + }, + ], + logLevel: "info", +}); diff --git a/scripts/register-selfhost.mjs b/scripts/register-selfhost.mjs new file mode 100644 index 0000000000..a04c36e2a2 --- /dev/null +++ b/scripts/register-selfhost.mjs @@ -0,0 +1,29 @@ +// Self-host module-resolution hooks (run before the app loads). Any `cloudflare:*` import — from gittensory's +// source OR a transitive dep (@cloudflare/puppeteer, the agents SDK / partyserver) — resolves to an in-memory +// stub. These bindings are never USED on self-host (BROWSER/RATE_LIMITER/email absent → the code degrades +// before touching them); the stub only makes the import + any `extends`/named import resolve so Node can load +// the graph. Used as the Docker entry: `node --import ./scripts/register-selfhost.mjs dist/server.mjs`. +import { registerHooks } from "node:module"; + +const STUB_SOURCE = [ + "export class DurableObject { constructor(ctx, env) { this.ctx = ctx; this.env = env; } }", + "export class WorkerEntrypoint { constructor(ctx, env) { this.ctx = ctx; this.env = env; } }", + "export class WorkflowEntrypoint { constructor(ctx, env) { this.ctx = ctx; this.env = env; } }", + "export class RpcTarget {}", + "export class EmailMessage { constructor(from, to, raw) { this.from = from; this.to = to; this.raw = raw; } }", + "export const env = {};", + "export const WorkerVersionMetadata = {};", + "export function connect() { throw new Error('cloudflare:sockets is unavailable on the self-host runtime'); }", + "export default {};", +].join("\n"); + +registerHooks({ + resolve(specifier, context, nextResolve) { + if (specifier.startsWith("cloudflare:")) return { url: `cfstub:${specifier}`, shortCircuit: true }; + return nextResolve(specifier, context); + }, + load(url, context, nextLoad) { + if (url.startsWith("cfstub:")) return { format: "module", shortCircuit: true, source: STUB_SOURCE }; + return nextLoad(url, context); + }, +}); diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts new file mode 100644 index 0000000000..a00a089599 --- /dev/null +++ b/src/selfhost/ai.ts @@ -0,0 +1,284 @@ +// Self-host AI provider (#979). gittensory calls `env.AI.run(model, { messages, max_tokens, temperature })` +// and reads `{ response }`. On self-host we provide an Ai-shaped adapter selected by AI_PROVIDER: +// • ollama / openai-compatible / openai — any OpenAI-compatible /chat/completions endpoint (BYO key) +// • claude-code / codex — a locally-authenticated CLI SUBSCRIPTION, run as a subprocess +// Absent (no AI_PROVIDER) → env.AI is undefined → gittensory's AI summary degrades to "unavailable" and the +// review proceeds deterministically. Every path returns `{ response: string }` (or throws → the caller +// records an error and degrades — never a silent wrong answer). + +interface AiRunOptions { + messages?: Array<{ role: string; content: string }>; + prompt?: string; + text?: string[]; // embedding input — the core's embedTexts passes { text: string[] } + max_tokens?: number; + temperature?: number; +} +/** A chat completion (`response`) or an embedding result (`data`). Both optional: the core reads whichever it + * asked for (extractAiText → `response`, embedTexts → `data`), each defensive about the other being absent. */ +export type AiResult = { response?: string; data?: number[][] }; +export interface SelfHostAi { + run(model: string, options: AiRunOptions): Promise; +} + +function toMessages(options: AiRunOptions): Array<{ role: string; content: string }> { + if (Array.isArray(options.messages)) return options.messages; + return [{ role: "user", content: String(options.prompt ?? "") }]; +} + +/** The core passes a Workers-AI model id (e.g. "@cf/meta/llama-3.1-8b-instruct-fp8-fast") that is meaningless + * off-Workers — handing it to Ollama or `claude --model` fails. Prefer the operator-configured model + * (AI_MODEL / WORKERS_AI_SUMMARY_MODEL), then any non-Workers model the core passed, then a provider default. */ +export function resolveModel(configured: string | undefined, passed: string, providerDefault: string): string { + if (configured && configured.trim()) return configured.trim(); + if (passed && !passed.startsWith("@cf/")) return passed; + return providerDefault; +} + +function configuredModel(env: Record): string | undefined { + return env.AI_MODEL ?? env.WORKERS_AI_SUMMARY_MODEL; +} + +/** OpenAI-compatible endpoint (Ollama's /v1, OpenAI, vLLM, LM Studio, …) — chat + embeddings. */ +export function createOpenAiCompatibleAi(opts: { baseUrl: string; apiKey?: string | undefined; model?: string | undefined; embedModel?: string | undefined }): SelfHostAi { + const base = opts.baseUrl.replace(/\/+$/, ""); + const headers = (): Record => ({ "content-type": "application/json", ...(opts.apiKey ? { authorization: `Bearer ${opts.apiKey}` } : {}) }); + return { + async run(model, options) { + // Embedding request — the core's embedTexts passes { text: string[] }; route to /embeddings (for RAG). + if (Array.isArray(options.text)) { + if (options.text.length === 0) return { data: [] }; + const res = await fetch(`${base}/embeddings`, { + method: "POST", + headers: headers(), + body: JSON.stringify({ model: opts.embedModel ?? "bge-m3", input: options.text }), + signal: AbortSignal.timeout(120_000), + }); + if (!res.ok) throw new Error(`ai_embed_http_${res.status}`); + const json = (await res.json()) as { data?: Array<{ embedding: number[] }> }; + return { data: (json.data ?? []).map((d) => d.embedding) }; + } + const res = await fetch(`${base}/chat/completions`, { + method: "POST", + headers: headers(), + body: JSON.stringify({ model: resolveModel(opts.model, model, "llama3.1"), messages: toMessages(options), max_tokens: options.max_tokens, temperature: options.temperature }), + signal: AbortSignal.timeout(120_000), + }); + if (!res.ok) throw new Error(`ai_http_${res.status}`); + const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }; + return { response: data.choices?.[0]?.message?.content ?? "" }; + }, + }; +} + +/** Native Anthropic Messages API (BYOK — bills your Anthropic API key; distinct from the claude-code + * subscription path). The system message becomes the top-level `system` param; the rest map to user/assistant. */ +export function createAnthropicAi(opts: { apiKey: string; model?: string | undefined; baseUrl?: string | undefined }): SelfHostAi { + const base = (opts.baseUrl ?? "https://api.anthropic.com").replace(/\/+$/, ""); + return { + async run(model, options) { + const msgs = toMessages(options); + const system = + msgs + .filter((m) => m.role === "system") + .map((m) => m.content) + .join("\n\n") || undefined; + const messages = msgs.filter((m) => m.role !== "system").map((m) => ({ role: m.role === "assistant" ? "assistant" : "user", content: m.content })); + const res = await fetch(`${base}/v1/messages`, { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": opts.apiKey, "anthropic-version": "2023-06-01" }, + body: JSON.stringify({ model: resolveModel(opts.model, model, "claude-sonnet-4-6"), max_tokens: options.max_tokens ?? 1024, ...(system ? { system } : {}), messages }), + signal: AbortSignal.timeout(120_000), + }); + if (!res.ok) throw new Error(`anthropic_http_${res.status}`); + const data = (await res.json()) as { content?: Array<{ type: string; text?: string }> }; + return { + response: (data.content ?? []) + .filter((c) => c.type === "text") + .map((c) => c.text ?? "") + .join(""), + }; + }, + }; +} + +// ── Subscription CLI providers (#979) — locally-authenticated `claude` / `codex` as a subprocess ────────── +// SECURITY: the child env DELETES the billable API keys so a misconfigured CLI cannot silently bill the +// metered API instead of using the subscription OAuth token. The CLI runs read-only / no extra tools. Any +// non-zero exit / empty output / error-envelope THROWS so the caller degrades — never a silent answer. +const BILLABLE_KEY_VARS = ["ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "CODEX_API_KEY", "OPENAI_API_KEY"] as const; + +function scrubBillableKeys(parent: Record): Record { + const child = { ...parent }; + for (const k of BILLABLE_KEY_VARS) delete child[k]; + return child; +} + +/** Pull the assistant's final text out of a CLI's JSON output (Claude Code `{result}` or Codex JSONL). */ +export function extractCliText(stdout: string): string { + const trimmed = stdout.trim(); + if (!trimmed) return ""; + const tryParse = (s: string): string => { + try { + const o = JSON.parse(s) as Record; + const text = o.result ?? o.text ?? o.content ?? o.response; + return typeof text === "string" ? text : ""; + } catch { + return ""; + } + }; + const whole = tryParse(trimmed); + if (whole) return whole; + const lines = trimmed.split(/\r?\n/).filter((l) => l.trim()); + for (let i = lines.length - 1; i >= 0; i -= 1) { + const line = lines[i]; + /* v8 ignore next */ // the filter above guarantees a non-empty line; this is a TS undefined-guard only + if (!line) continue; + const t = tryParse(line); + if (t) return t; + } + return ""; +} + +/** Claude Code's `--output-format json` exits 0 even on an API/auth error, returning {is_error:true,result:""}. + * Detect it so the error string is never surfaced as the model's answer. */ +export function claudeErrorStatus(stdout: string): string | null { + try { + const o = JSON.parse(stdout.trim()) as Record; + if (o.is_error === true) return String(o.api_error_status ?? o.subtype ?? "unknown"); + } catch { + /* not a single JSON object — handled by the empty-output guard */ + } + return null; +} + +type SpawnFn = (cmd: string, args: string[], opts: { env: Record; input?: string; timeoutMs: number }) => Promise<{ stdout: string; code: number | null }>; + +async function defaultSpawn(): Promise { + const cp = await import("node:child_process"); + return (cmd, args, o) => + new Promise((resolve, reject) => { + const stdio: ["pipe", "pipe", "pipe"] = ["pipe", "pipe", "pipe"]; + const child = cp.spawn(cmd, args, { env: o.env as NodeJS.ProcessEnv, stdio }); + let stdout = ""; + /* v8 ignore start */ // a 120s subprocess timeout is not unit-testable without a 2-minute wait + const timer = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error("subscription_cli_timeout")); + }, o.timeoutMs); + /* v8 ignore stop */ + child.stdout?.on("data", (d: Buffer) => (stdout += d.toString("utf8"))); + child.on("error", (e) => { + clearTimeout(timer); + reject(e); + }); + child.on("close", (code) => { + clearTimeout(timer); + resolve({ stdout, code }); + }); + if (o.input != null) { + child.stdin?.write(o.input); + child.stdin?.end(); + } + }); +} + +/** Claude Code subscription (CLAUDE_CODE_OAUTH_TOKEN via `claude setup-token`). Headless, read-only, JSON. */ +export function createClaudeCodeAi(parentEnv: Record, spawnImpl?: SpawnFn): SelfHostAi { + return { + async run(model, options) { + const token = parentEnv.CLAUDE_CODE_OAUTH_TOKEN; + if (!token) throw new Error("claude_code_no_oauth_token"); + const env = scrubBillableKeys(parentEnv); + env.CLAUDE_CODE_OAUTH_TOKEN = token; + const prompt = toMessages(options).map((m) => m.content).join("\n\n"); + const spawn = spawnImpl ?? (await defaultSpawn()); + const claudeModel = resolveModel(configuredModel(parentEnv), model, "sonnet"); + const { stdout, code } = await spawn("claude", ["--print", "--output-format", "json", "--model", claudeModel, "--permission-mode", "plan", "--disallowedTools", "Bash,Edit,Write,WebFetch,WebSearch"], { env, input: prompt, timeoutMs: 120_000 }); + if (code !== 0) throw new Error(`claude_code_exit_${code ?? "null"}`); + const errStatus = claudeErrorStatus(stdout); + if (errStatus) throw new Error(`claude_code_error_${errStatus}`); + const text = extractCliText(stdout); + if (!text) throw new Error("claude_code_empty_output"); + return { response: text }; + }, + }; +} + +/** Codex subscription (`codex exec`, auth from ~/.codex/auth.json). Gated/unverified — fail-safe. */ +export function createCodexAi(parentEnv: Record, spawnImpl?: SpawnFn): SelfHostAi { + return { + async run(model, options) { + const env = scrubBillableKeys(parentEnv); + const prompt = toMessages(options).map((m) => m.content).join("\n\n"); + const spawn = spawnImpl ?? (await defaultSpawn()); + const codexModel = resolveModel(configuredModel(parentEnv), model, "gpt-5"); + const { stdout, code } = await spawn("codex", ["exec", "--json", "--sandbox", "read-only", "--ask-for-approval", "never", "--model", codexModel, "--", prompt], { env, timeoutMs: 120_000 }); + if (code !== 0) throw new Error(`codex_exit_${code ?? "null"}`); + const text = extractCliText(stdout); + if (!text) throw new Error("codex_empty_output"); + return { response: text }; + }, + }; +} + +/** Try each provider in order until one returns; if all throw, rethrow the last error so the caller degrades + * (AI summary → "unavailable"; the review still runs deterministically). The fallback chain is what makes a + * BYOK setup robust — e.g. AI_PROVIDER="anthropic,ollama" uses the API first and a local model if it's down. */ +export function createChainAi(providers: Array<{ name: string; ai: SelfHostAi }>): SelfHostAi { + return { + async run(model, options) { + let lastError: unknown = new Error("no_ai_providers"); + for (const p of providers) { + try { + return await p.ai.run(model, options); + } catch (error) { + lastError = error; + console.error(JSON.stringify({ level: "warn", event: "selfhost_ai_provider_failed", provider: p.name, error: error instanceof Error ? error.message : "unknown" })); + } + } + throw lastError instanceof Error ? lastError : new Error("all_ai_providers_failed"); + }, + }; +} + +/** Build one provider adapter by name (BYO credentials read from provider-specific env, then the generic + * AI_API_KEY). Returns undefined when its required credential is missing. */ +export function buildProvider(name: string, env: Record): SelfHostAi | undefined { + switch (name) { + case "ollama": + case "openai-compatible": + case "openai": + return createOpenAiCompatibleAi({ + baseUrl: env.AI_BASE_URL ?? (name === "openai" ? "https://api.openai.com/v1" : "http://localhost:11434/v1"), + apiKey: env.AI_API_KEY ?? env.OPENAI_API_KEY, + model: configuredModel(env), + embedModel: env.AI_EMBED_MODEL, + }); + case "anthropic": { + const apiKey = env.ANTHROPIC_API_KEY ?? env.AI_API_KEY; + return apiKey ? createAnthropicAi({ apiKey, model: configuredModel(env), baseUrl: env.AI_BASE_URL }) : undefined; + } + case "claude-code": + return createClaudeCodeAi(env); + case "codex": + return createCodexAi(env); + default: + return undefined; + } +} + +/** Select the self-host AI provider(s) from AI_PROVIDER. A comma-separated list builds a fallback chain + * (first to succeed wins). Returns undefined when unconfigured or no provider has its credential. */ +export function createSelfHostAi(env: Record): SelfHostAi | undefined { + const raw = (env.AI_PROVIDER ?? "").trim().toLowerCase(); + if (!raw) return undefined; + const providers = raw + .split(",") + .map((s) => s.trim()) + .filter(Boolean) + .map((name) => ({ name, ai: buildProvider(name, env) })) + .filter((p): p is { name: string; ai: SelfHostAi } => Boolean(p.ai)); + if (providers.length === 0) return undefined; + if (providers.length === 1) return providers[0]?.ai; + return createChainAi(providers); +} diff --git a/src/selfhost/audit.ts b/src/selfhost/audit.ts new file mode 100644 index 0000000000..096abea0fa --- /dev/null +++ b/src/selfhost/audit.ts @@ -0,0 +1,32 @@ +// Structured audit log for the self-host runtime (#980). Emits one JSON line per job lifecycle event so +// operators can grep / pipe to their log aggregator (Loki, CloudWatch, Datadog, etc.) without any extra +// setup. Written to process.stdout so it is captured by Docker's default json-file log driver and is +// accessible via `docker compose logs gittensory`. + +export type AuditEventType = "job_complete" | "job_dead" | "job_error"; + +export interface AuditEvent { + event: AuditEventType; + ts: number; // Unix timestamp (ms) + job_id: number | string; + payload_type?: string | undefined; // top-level `type` field from the job payload, if present + latency_ms: number; // wall time from claim to completion/failure + attempts: number; // total attempts consumed (1 = first-try success) + error?: string; // last error message, present for job_dead / job_error +} + +/** Emit a single audit event as a JSON line on stdout. */ +export function logAudit(ev: AuditEvent): void { + process.stdout.write(JSON.stringify({ level: "audit", ...ev }) + "\n"); +} + +/** Extract a `type` label from a raw job payload string without fully parsing it. Returns undefined + * if the payload is not a JSON object or lacks a top-level `type` string. */ +export function extractPayloadType(payload: string): string | undefined { + try { + const o = JSON.parse(payload) as Record; + return typeof o.type === "string" ? o.type : undefined; + } catch { + return undefined; + } +} diff --git a/src/selfhost/cf-workers-shim.ts b/src/selfhost/cf-workers-shim.ts new file mode 100644 index 0000000000..b59af6f08a --- /dev/null +++ b/src/selfhost/cf-workers-shim.ts @@ -0,0 +1,18 @@ +// Minimal stand-in for the `cloudflare:workers` module on the Node self-host runtime. The only import of it +// in the codebase is `DurableObject` (auth/rate-limit.ts → the RateLimiter DO). That DO is NEVER instantiated +// on self-host — env.RATE_LIMITER is undefined, so enforceRateLimit returns null before any DO is touched — +// so this base class only needs to make the import + `extends DurableObject` resolve. The self-host esbuild +// build aliases `cloudflare:workers` to this file (see the Docker build / build:selfhost script). +export class DurableObject { + constructor( + protected ctx?: unknown, + protected env?: E, + ) {} +} +export class WorkerEntrypoint { + constructor( + protected ctx?: unknown, + protected env?: E, + ) {} +} +export class RpcTarget {} diff --git a/src/selfhost/d1-adapter.ts b/src/selfhost/d1-adapter.ts new file mode 100644 index 0000000000..8f77ed3247 --- /dev/null +++ b/src/selfhost/d1-adapter.ts @@ -0,0 +1,123 @@ +// Self-host D1 adapter (#980). A FAITHFUL D1Database implementation over a synchronous SQLite driver, so +// EVERY data-access path in gittensory runs UNCHANGED on a local file: +// • drizzle-orm/d1 (getDb → the ~171 repository call sites) — calls bind/all/run/raw/batch + reads .results +// • the raw `env.DB.prepare(sql).bind(...).all()/.first()/.run()/.batch()` sites +// • the test suite, which uses the same D1 surface +// D1's API is async; the SQLite drivers are sync — sync calls are wrapped in resolved Promises. The driver is +// INJECTED behind the tiny SqliteDriver interface, so this module has no hard SQLite dependency and the +// Cloudflare Worker bundle never imports it. Default driver: node:sqlite (built into Node, no native build). + +/** A uniform sync SQLite primitive both node:sqlite and better-sqlite3 can satisfy via a thin wrapper. `query` + * ALWAYS returns rows (empty for a write) + the write metadata, so the adapter needs no reader-detection. */ +export interface SqliteDriver { + query(sql: string, params: unknown[]): { rows: Record[]; changes: number; lastInsertRowid: number }; + exec(sql: string): void; +} + +function meta(changes = 0, lastRowId = 0): Record { + return { duration: 0, size_after: 0, rows_read: 0, rows_written: changes, last_row_id: lastRowId, changed_db: changes > 0, changes }; +} + +/** One prepared (and optionally bound) statement. bind() returns a fresh instance (D1 statements are immutable + * after bind). The SQLite statement is compiled per execution (drivers cache by SQL text). */ +class Statement { + constructor( + private readonly driver: SqliteDriver, + private readonly sql: string, + private readonly values: unknown[] = [], + ) {} + + bind(...values: unknown[]): Statement { + return new Statement(this.driver, this.sql, values); + } + + /** Sync core used by all()/run() (async wrappers) and batch() (inside a transaction). */ + execSync(): { results: unknown[]; success: boolean; meta: Record } { + const r = this.driver.query(this.sql, this.values); + return { results: r.rows, success: true, meta: meta(r.changes, r.lastInsertRowid) }; + } + + async all(): Promise<{ results: T[]; success: boolean; meta: Record }> { + return this.execSync() as { results: T[]; success: boolean; meta: Record }; + } + + // D1's run() returns the same {results, meta} shape (results empty for a non-returning write). + async run(): Promise<{ results: T[]; success: boolean; meta: Record }> { + return this.execSync() as { results: T[]; success: boolean; meta: Record }; + } + + async first(colName?: string): Promise { + const row = this.driver.query(this.sql, this.values).rows[0]; + if (row == null) return null; + return ((colName != null ? row[colName] : row) ?? null) as T | null; + } + + async raw(): Promise { + // D1 raw() returns each row as an array of column values (column order preserved). + return this.driver.query(this.sql, this.values).rows.map((row) => Object.values(row)) as T[]; + } +} + +/** Wrap a synchronous SQLite driver as a D1Database. */ +export function createD1Adapter(driver: SqliteDriver): D1Database { + const adapter = { + prepare(sql: string) { + return new Statement(driver, sql); + }, + async batch(statements: unknown[]) { + // D1 runs a batch atomically, one result per statement, in order. + const list = statements as Statement[]; + driver.exec("BEGIN"); + try { + const out = list.map((s) => s.execSync()); + driver.exec("COMMIT"); + return out; + } catch (error) { + try { + driver.exec("ROLLBACK"); + } catch { + /* ignore */ + } + throw error; + } + }, + async exec(sql: string) { + driver.exec(sql); // runs one or more statements (used for migrations) + return { count: (sql.match(/;/g) ?? []).length || 1, duration: 0 }; + }, + async dump() { + return new ArrayBuffer(0); // unused by gittensory; present for D1 surface completeness + }, + }; + return adapter as unknown as D1Database; +} + +/** The minimal node:sqlite surface the wrapper uses (DatabaseSync + StatementSync). */ +interface NodeSqliteStatement { + columns(): unknown[]; + all(...params: unknown[]): unknown[]; + get(...params: unknown[]): unknown; + run(...params: unknown[]): { changes: number | bigint; lastInsertRowid: number | bigint }; +} +interface NodeSqliteDatabase { + prepare(sql: string): NodeSqliteStatement; + exec(sql: string): void; +} + +/** Build a SqliteDriver from a node:sqlite DatabaseSync. A statement with zero result columns is a WRITE + * (run → changes); otherwise a READ (all → rows). */ +export function nodeSqliteDriver(db: NodeSqliteDatabase): SqliteDriver { + return { + query(sql, params) { + const stmt = db.prepare(sql); + if (stmt.columns().length > 0) { + return { rows: stmt.all(...params) as Record[], changes: 0, lastInsertRowid: 0 }; + } + const info = stmt.run(...params); + return { rows: [], changes: Number(info.changes), lastInsertRowid: Number(info.lastInsertRowid) }; + }, + exec(sql) { + db.exec(sql); + }, + }; +} diff --git a/src/selfhost/health.ts b/src/selfhost/health.ts new file mode 100644 index 0000000000..26d0359d9a --- /dev/null +++ b/src/selfhost/health.ts @@ -0,0 +1,28 @@ +// Self-host liveness/readiness probes (#982). Liveness is binding-free (the process is up); readiness asserts +// the things a request actually depends on — the DB answers and the schema migrations have been applied. +// Backend-agnostic: runs through the D1 surface, so it works on both the SQLite and Postgres adapters. + +export interface Readiness { + ok: boolean; + checks: Record; +} + +/** Readiness: the DB answers a trivial query and the migrations table shows applied rows. */ +export async function readiness(db: D1Database): Promise { + let dbOk = false; + let migrations = false; + try { + await db.prepare("SELECT 1 AS one").first(); + dbOk = true; + } catch { + /* db down */ + } + try { + const row = await db.prepare("SELECT COUNT(*) AS c FROM _selfhost_migrations").first<{ c: number }>(); + /* v8 ignore next */ // COUNT(*) always returns exactly one row, so the row?./?? 0 guards never fire + migrations = Number(row?.c ?? 0) > 0; + } catch { + /* migrations table missing */ + } + return { ok: dbOk && migrations, checks: { db: dbOk, migrations } }; +} diff --git a/src/selfhost/mcp-server-node.ts b/src/selfhost/mcp-server-node.ts new file mode 100644 index 0000000000..26f093ef2d --- /dev/null +++ b/src/selfhost/mcp-server-node.ts @@ -0,0 +1,37 @@ +// Node-compatible MCP handler (#980). Replaces the Cloudflare Agents SDK `createMcpHandler` (Durable-Object- +// backed) with `WebStandardStreamableHTTPServerTransport` from the MCP SDK, which uses Web Standard APIs and +// runs on Node 18+. Stateless mode: no server-side session state; each HTTP request is self-contained. +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js"; + +type FetchHandler = (req: Request, env?: unknown, ctx?: unknown) => Promise; + +export function createMcpHandler( + server: McpServer, + opts: { route?: string; enableJsonResponse?: boolean } = {}, +): FetchHandler { + return async (req: Request): Promise => { + if (req.method === "OPTIONS") { + return new Response(null, { + status: 204, + headers: { + "access-control-allow-origin": req.headers.get("origin") ?? "*", + "access-control-allow-methods": "GET, POST, DELETE, OPTIONS", + "access-control-allow-headers": "content-type, authorization, mcp-protocol-version, mcp-session-id", + }, + }); + } + const transport = new WebStandardStreamableHTTPServerTransport({ + // sessionIdGenerator omitted → stateless mode (each request is self-contained) + enableJsonResponse: opts.enableJsonResponse ?? true, + }); + await server.connect(transport); + try { + const response = await transport.handleRequest(req); + return response; + } finally { + /* v8 ignore next -- transport.close() only rejects on internal MCP SDK teardown errors */ + await transport.close().catch(() => undefined); + } + }; +} diff --git a/src/selfhost/metrics.ts b/src/selfhost/metrics.ts new file mode 100644 index 0000000000..deae0208a6 --- /dev/null +++ b/src/selfhost/metrics.ts @@ -0,0 +1,49 @@ +// Minimal Prometheus text-format metrics for the self-host runtime (#982 observability). A tiny in-process +// registry — counters (monotonic, incremented at the call site) and gauges (sampled at scrape time via a +// callback, e.g. live queue depth). Rendered at GET /metrics. No deps, no cardinality explosion: callers use +// a small fixed label set. +type Labels = Record; +type GaugeSample = () => number | Promise; + +const counters = new Map(); +const gauges = new Map(); + +function seriesKey(name: string, labels?: Labels): string { + if (!labels || Object.keys(labels).length === 0) return name; + const inner = Object.entries(labels) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([k, v]) => `${k}="${String(v).replace(/"/g, '\\"')}"`) + .join(","); + return `${name}{${inner}}`; +} + +/** Increment a monotonic counter (created on first use). */ +export function incr(name: string, labels?: Labels, by = 1): void { + const k = seriesKey(name, labels); + counters.set(k, (counters.get(k) ?? 0) + by); +} + +/** Register a gauge sampled at scrape time (sync or async). Re-registering replaces the sampler. */ +export function gauge(name: string, sample: GaugeSample): void { + gauges.set(name, sample); +} + +/** Render the registry in Prometheus text exposition format. */ +export async function renderMetrics(): Promise { + const lines: string[] = []; + for (const [k, v] of counters) lines.push(`${k} ${v}`); + for (const [name, sample] of gauges) { + try { + lines.push(`${name} ${await sample()}`); + } catch { + /* a failing sampler must not break the scrape */ + } + } + return `${lines.join("\n")}\n`; +} + +/** Test-only: clear all series. */ +export function resetMetrics(): void { + counters.clear(); + gauges.clear(); +} diff --git a/src/selfhost/migrate.ts b/src/selfhost/migrate.ts new file mode 100644 index 0000000000..b1fc898427 --- /dev/null +++ b/src/selfhost/migrate.ts @@ -0,0 +1,21 @@ +// Apply gittensory's D1 migrations to the self-host SQLite database at startup. The same `migrations/*.sql` +// files Cloudflare applies via `wrangler d1 migrations apply` — they're plain SQLite DDL, so they run as-is +// through the D1 adapter's exec(). Tracked in a `_selfhost_migrations` table so a restart re-applies only the +// new ones (idempotent), mirroring wrangler's migration ledger. +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +export async function runSelfHostMigrations(db: D1Database, dir: string): Promise { + await db.exec("CREATE TABLE IF NOT EXISTS _selfhost_migrations (name TEXT PRIMARY KEY, applied_at TEXT NOT NULL)"); + const existing = await db.prepare("SELECT name FROM _selfhost_migrations").all<{ name: string }>(); + const applied = new Set(existing.results.map((r) => r.name)); + const files = readdirSync(dir).filter((f) => f.endsWith(".sql")).sort(); + let count = 0; + for (const file of files) { + if (applied.has(file)) continue; + await db.exec(readFileSync(join(dir, file), "utf8")); + await db.prepare("INSERT INTO _selfhost_migrations (name, applied_at) VALUES (?, ?)").bind(file, new Date().toISOString()).run(); + count += 1; + } + return count; +} diff --git a/src/selfhost/pg-adapter.ts b/src/selfhost/pg-adapter.ts new file mode 100644 index 0000000000..f5d6d1104d --- /dev/null +++ b/src/selfhost/pg-adapter.ts @@ -0,0 +1,85 @@ +// Postgres-backed D1Database for the self-host Postgres backend (#977). Implements the same D1 surface the +// app + drizzle-orm/d1 use (prepare/bind/all/first/run/raw + batch + exec), translating each SQLite query to +// Postgres (pg-dialect.ts) and running it via node-postgres. A shared Postgres DB makes multi-instance +// self-host possible (vs the single-file SQLite default). +import type { Pool, PoolClient } from "pg"; +import { translateDdl, translateSql } from "./pg-dialect"; + +type Row = Record; +type Runner = Pool | PoolClient; + +class PgStatement { + constructor( + private readonly pool: Pool, + private readonly sql: string, + private readonly params: unknown[] = [], + ) {} + + bind(...params: unknown[]): PgStatement { + return new PgStatement(this.pool, this.sql, params); + } + + private async exec(runner: Runner = this.pool): Promise<{ rows: Row[]; rowCount: number }> { + const res = await runner.query(translateSql(this.sql), this.params as unknown[]); + return { rows: res.rows as Row[], rowCount: res.rowCount ?? 0 }; + } + + async all(): Promise<{ results: T[]; success: true; meta: Record }> { + const { rows, rowCount } = await this.exec(); + return { results: rows as T[], success: true, meta: { rows_read: rowCount, changes: rowCount } }; + } + + async first(colName?: string): Promise { + const { rows } = await this.exec(); + const row = rows[0]; + if (!row) return null; + return (colName ? row[colName] : row) as T; + } + + async run(): Promise<{ success: true; meta: Record }> { + const { rowCount } = await this.exec(); + return { success: true, meta: { changes: rowCount, last_row_id: 0, rows_written: rowCount } }; + } + + async raw(): Promise { + const { rows } = await this.exec(); + return rows.map((r) => Object.values(r)) as T[]; + } + + /** Run this statement on a specific client (used by batch's transaction). */ + async runOn(client: PoolClient): Promise<{ results: Row[]; success: true; meta: Record }> { + const { rows, rowCount } = await this.exec(client); + return { results: rows, success: true, meta: { changes: rowCount } }; + } +} + +export function createPgAdapter(pool: Pool): D1Database { + const adapter = { + prepare: (sql: string) => new PgStatement(pool, sql), + async batch(statements: PgStatement[]) { + const client = await pool.connect(); + try { + await client.query("BEGIN"); + const out: unknown[] = []; + for (const st of statements) out.push(await st.runOn(client)); + await client.query("COMMIT"); + return out; + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } finally { + client.release(); + } + }, + async exec(sql: string) { + // Migrations: no placeholders; translate the DDL functions and run (node-postgres runs the multi-statement + // string in one simple query). + await pool.query(translateDdl(sql)); + return { count: (sql.match(/;/g) ?? []).length || 1, duration: 0 }; + }, + async dump() { + return new ArrayBuffer(0); // unused; present for D1 surface completeness + }, + }; + return adapter as unknown as D1Database; +} diff --git a/src/selfhost/pg-dialect.ts b/src/selfhost/pg-dialect.ts new file mode 100644 index 0000000000..04adf37f0f --- /dev/null +++ b/src/selfhost/pg-dialect.ts @@ -0,0 +1,81 @@ +// SQLite → Postgres SQL dialect translation for the self-host Postgres backend (#977). gittensory's core and +// drizzle-orm/d1 emit SQLite-dialect SQL; this translates the bounded set of SQLite-isms the codebase uses +// (placeholders + a handful of scalar functions + INSERT OR REPLACE/IGNORE) so the SAME queries run on +// Postgres. The timestamp columns are TEXT (ISO strings written by the app), so the datetime/CURRENT_TIMESTAMP +// translations return TEXT in SQLite's format to preserve the existing text-comparison semantics. Validated +// against a real Postgres (all 56 migrations + the runtime query paths). + +// INSERT OR REPLACE needs an explicit conflict target on Postgres; map the (few) tables that use it to their PK. +const REPLACE_CONFLICT_KEYS: Record = { + system_flags: ["key"], + tunables_overrides: ["project"], + tunables_overrides_shadow: ["project"], +}; + +/** Replace `?` placeholders with `$1,$2,…`, skipping any `?` inside single-quoted string literals. */ +export function toNumberedPlaceholders(sql: string): string { + let out = ""; + let n = 0; + let inString = false; + for (const ch of sql) { + if (ch === "'") inString = !inString; + if (ch === "?" && !inString) { + n += 1; + out += `$${n}`; + } else { + out += ch; + } + } + return out; +} + +/** Translate the SQLite scalar functions the codebase uses to Postgres equivalents. */ +export function translateFunctions(sql: string): string { + return ( + sql + // ISO-now (the DEFAULT on TEXT timestamp columns + nowIso parity) + .replace(/strftime\(\s*'%Y-%m-%dT%H:%M:%fZ'\s*,\s*'now'\s*\)/gi, `to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`) + // week / month buckets (stats) + .replace(/strftime\(\s*'%Y-W%W'\s*,\s*([^)]+?)\s*\)/gi, `to_char(($1)::timestamptz, 'YYYY"-W"WW')`) + .replace(/strftime\(\s*'%Y-%m'\s*,\s*([^)]+?)\s*\)/gi, `to_char(($1)::timestamptz, 'YYYY-MM')`) + // datetime('now', ) → TEXT in SQLite's 'YYYY-MM-DD HH:MM:SS' format (TEXT columns compared) + .replace(/datetime\(\s*'now'\s*,\s*([^)]+?)\s*\)/gi, `to_char(now() + ($1)::interval, 'YYYY-MM-DD HH24:MI:SS')`) + .replace(/datetime\(\s*'now'\s*\)/gi, `to_char(now(), 'YYYY-MM-DD HH24:MI:SS')`) + // CURRENT_TIMESTAMP → SQLite's TEXT format (the columns are TEXT) + .replace(/CURRENT_TIMESTAMP/gi, `to_char(now(), 'YYYY-MM-DD HH24:MI:SS')`) + // json_extract(col, '$.key') → (col::jsonb ->> 'key') (single-level paths — all the codebase uses) + .replace(/json_extract\(\s*([^,]+?)\s*,\s*'\$\.([A-Za-z0-9_]+)'\s*\)/gi, `(($1)::jsonb ->> '$2')`) + ); +} + +/** Translate INSERT OR REPLACE / INSERT OR IGNORE to Postgres ON CONFLICT. */ +export function translateInsertOr(sql: string): string { + if (/^\s*INSERT\s+OR\s+IGNORE\s+INTO/i.test(sql)) { + return `${sql.replace(/^(\s*)INSERT\s+OR\s+IGNORE\s+INTO/i, "$1INSERT INTO")} ON CONFLICT DO NOTHING`; + } + const m = /^\s*INSERT\s+OR\s+REPLACE\s+INTO\s+([A-Za-z0-9_]+)\s*\(([^)]+)\)/i.exec(sql); + if (m) { + const table = m[1] as string; + const cols = (m[2] as string).split(",").map((c) => c.trim()); + const pk = REPLACE_CONFLICT_KEYS[table]; + if (!pk) throw new Error(`pg_dialect: INSERT OR REPLACE into '${table}' has no known conflict key`); + const updates = cols + .filter((c) => !pk.includes(c)) + .map((c) => `${c}=excluded.${c}`) + .join(", "); + const base = sql.replace(/^(\s*)INSERT\s+OR\s+REPLACE\s+INTO/i, "$1INSERT INTO"); + return `${base} ON CONFLICT (${pk.join(", ")}) DO UPDATE SET ${updates}`; + } + return sql; +} + +/** Translate a runtime query (SQLite → Postgres). */ +export function translateSql(sql: string): string { + return toNumberedPlaceholders(translateFunctions(translateInsertOr(sql))); +} + +/** Translate a DDL statement (migrations). Column types (TEXT/INTEGER/REAL) are PG-native; only the SQLite + * default expressions need translating. No `?` placeholders in DDL. */ +export function translateDdl(sql: string): string { + return translateFunctions(sql); +} diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts new file mode 100644 index 0000000000..ce21a5cbc8 --- /dev/null +++ b/src/selfhost/pg-queue.ts @@ -0,0 +1,168 @@ +// Postgres-backed durable job queue for multi-instance self-host (#977). Same contract as the SQLite queue +// (persist → restart re-claims, backoff retries, dead-letter) but uses `FOR UPDATE SKIP LOCKED` so multiple +// app instances sharing one Postgres can claim jobs concurrently without double-processing. size()/deadCount() +// are async (the metrics gauges accept async samplers). +import type { Pool } from "pg"; +import { logAudit, extractPayloadType } from "./audit"; +import { incr } from "./metrics"; +import type { JobMessage } from "../types"; + +const TABLE = "_selfhost_jobs"; +const DDL = ` +CREATE TABLE IF NOT EXISTS ${TABLE} ( + id BIGSERIAL PRIMARY KEY, + payload TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + attempts INTEGER NOT NULL DEFAULT 0, + run_after BIGINT NOT NULL DEFAULT 0, + created_at BIGINT NOT NULL, + last_error TEXT +); +CREATE INDEX IF NOT EXISTS ${TABLE}_claim ON ${TABLE}(status, run_after);`; + +export interface PgDurableQueue { + binding: Queue; + init(): Promise; + start(): void; + stop(): Promise; + drain(): Promise; + size(): Promise; + deadCount(): Promise; +} + +interface JobRow { + id: string; + payload: string; + attempts: number; +} + +export interface PgQueueOptions { + maxRetries?: number; + pollIntervalMs?: number; + backoffMs?: (attempt: number) => number; + /** Max concurrent `processOne()` loops. Defaults to QUEUE_CONCURRENCY env var or 1. */ + concurrency?: number; +} + +export function createPgQueue(pool: Pool, consume: (message: JobMessage) => Promise, opts: PgQueueOptions = {}): PgDurableQueue { + const maxRetries = opts.maxRetries ?? 5; + const pollIntervalMs = opts.pollIntervalMs ?? 1000; + const backoff = opts.backoffMs ?? ((attempt: number) => Math.min(60_000, 1000 * 2 ** attempt)); + const concurrency = opts.concurrency ?? Math.max(1, Number(process.env.QUEUE_CONCURRENCY ?? "1")); + + let running = false; + let active = 0; + let timer: ReturnType | null = null; + + async function init(): Promise { + await pool.query(DDL); + const recovered = (await pool.query(`UPDATE ${TABLE} SET status='pending' WHERE status='processing'`)).rowCount ?? 0; + if (recovered) console.log(JSON.stringify({ event: "selfhost_queue_recovered", count: recovered })); + } + + async function enqueue(message: JobMessage, delaySeconds: number): Promise { + const now = Date.now(); + await pool.query(`INSERT INTO ${TABLE} (payload, status, attempts, run_after, created_at) VALUES ($1,'pending',0,$2,$3)`, [JSON.stringify(message), now + delaySeconds * 1000, now]); + incr("gittensory_jobs_enqueued_total"); + void pump(); + } + + async function claimNext(): Promise { + // Atomic, multi-instance-safe: lock + claim one due job, skipping rows another instance already locked. + const res = await pool.query( + `UPDATE ${TABLE} SET status='processing' + WHERE id = (SELECT id FROM ${TABLE} WHERE status='pending' AND run_after<=$1 ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1) + RETURNING id, payload, attempts`, + [Date.now()], + ); + return (res.rows[0] as JobRow | undefined) ?? null; + } + + async function processOne(): Promise { + const job = await claimNext(); + if (!job) return false; + const claimedAt = Date.now(); + let message: JobMessage; + try { + message = JSON.parse(job.payload) as JobMessage; + } catch { + await pool.query(`UPDATE ${TABLE} SET status='dead', last_error='unparseable payload' WHERE id=$1`, [job.id]); + incr("gittensory_jobs_dead_total"); + logAudit({ event: "job_dead", ts: Date.now(), job_id: job.id, latency_ms: Date.now() - claimedAt, attempts: Number(job.attempts) + 1, error: "unparseable payload" }); + return true; + } + try { + await consume(message); + await pool.query(`DELETE FROM ${TABLE} WHERE id=$1`, [job.id]); + incr("gittensory_jobs_processed_total"); + logAudit({ event: "job_complete", ts: Date.now(), job_id: job.id, payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts: Number(job.attempts) + 1 }); + } catch (error) { + const attempts = Number(job.attempts) + 1; + const errMsg = error instanceof Error ? error.message : "unknown error"; + incr("gittensory_jobs_failed_total"); + if (attempts >= maxRetries) { + await pool.query(`UPDATE ${TABLE} SET status='dead', attempts=$1, last_error=$2 WHERE id=$3`, [attempts, errMsg, job.id]); + incr("gittensory_jobs_dead_total"); + console.error(JSON.stringify({ level: "error", event: "selfhost_job_dead", id: job.id, attempts, error: errMsg })); + logAudit({ event: "job_dead", ts: Date.now(), job_id: job.id, payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts, error: errMsg }); + } else { + await pool.query(`UPDATE ${TABLE} SET status='pending', attempts=$1, run_after=$2, last_error=$3 WHERE id=$4`, [attempts, Date.now() + backoff(attempts), errMsg, job.id]); + logAudit({ event: "job_error", ts: Date.now(), job_id: job.id, payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts, error: errMsg }); + } + } + return true; + } + + async function pump(): Promise { + if (active >= concurrency) return; + active++; + try { + while (await processOne()) { + /* drain due jobs */ + } + } finally { + active--; + } + } + + const binding = { + async send(message: JobMessage, options?: { delaySeconds?: number }): Promise { + await enqueue(message, options?.delaySeconds ?? 0); + }, + async sendBatch(messages: Iterable<{ body: JobMessage; delaySeconds?: number }>): Promise { + for (const m of messages) await enqueue(m.body, m.delaySeconds ?? 0); + }, + } as unknown as Queue; + + return { + binding, + init, + start() { + if (running) return; + running = true; + const tick = (): void => { + /* v8 ignore next */ // stop() clears the timer before the next tick can fire with running=false + if (!running) return; + void pump().finally(() => { + if (running) timer = setTimeout(tick, pollIntervalMs); + }); + }; + tick(); + }, + async stop() { + running = false; + if (timer) clearTimeout(timer); + while (active > 0) await new Promise((r) => setTimeout(r, 10)); + }, + async drain() { + while (active > 0) await new Promise((r) => setTimeout(r, 5)); + await pump(); + }, + async size() { + return Number((await pool.query(`SELECT COUNT(*) AS c FROM ${TABLE} WHERE status IN ('pending','processing')`)).rows[0].c); + }, + async deadCount() { + return Number((await pool.query(`SELECT COUNT(*) AS c FROM ${TABLE} WHERE status='dead'`)).rows[0].c); + }, + }; +} diff --git a/src/selfhost/pg-vectorize.ts b/src/selfhost/pg-vectorize.ts new file mode 100644 index 0000000000..14caca85ad --- /dev/null +++ b/src/selfhost/pg-vectorize.ts @@ -0,0 +1,85 @@ +// Postgres-backed Vectorize adapter for the self-host Postgres backend (#980 RAG on Postgres). Implements the +// same Cloudflare `Vectorize` surface (upsert / query / deleteByIds) as the SQLite adapter but backed by a +// pgvector extension table. Cosine similarity is computed by pgvector's `<=>` operator (exact ANN, fast for +// repo-scale corpora). Requires `CREATE EXTENSION IF NOT EXISTS vector` — the init() call issues that DDL. +// +// Enable: set DATABASE_URL to a postgres:// URI and use the pgvector/pgvector:pg16 Docker image. The +// buildPostgresBackend path in server.ts calls init() at startup then injects this adapter as env.VECTORIZE. +import type { Pool } from "pg"; + +const TABLE = "_selfhost_vectors"; + +interface VectorRecord { + id: string; + values: number[]; + namespace?: string; + metadata?: Record; +} +interface QueryOptions { + topK?: number; + namespace?: string; +} +interface Match { + id: string; + score: number; + metadata?: Record; +} + +export async function initPgVectorize(pool: Pool): Promise { + await pool.query("CREATE EXTENSION IF NOT EXISTS vector"); + await pool.query(` + CREATE TABLE IF NOT EXISTS ${TABLE} ( + id TEXT PRIMARY KEY, + namespace TEXT NOT NULL DEFAULT '', + embedding vector, + metadata JSONB + )`); + await pool.query(`CREATE INDEX IF NOT EXISTS ${TABLE}_ns ON ${TABLE}(namespace)`); +} + +export function createPgVectorize(pool: Pool): Vectorize { + const adapter = { + async upsert(vectors: VectorRecord[]): Promise<{ count: number; ids: string[] }> { + for (const v of vectors) { + const embedding = `[${v.values.join(",")}]`; + await pool.query( + `INSERT INTO ${TABLE} (id, namespace, embedding, metadata) + VALUES ($1, $2, $3::vector, $4) + ON CONFLICT(id) DO UPDATE SET namespace=EXCLUDED.namespace, embedding=EXCLUDED.embedding::vector, metadata=EXCLUDED.metadata`, + [v.id, v.namespace ?? "", embedding, v.metadata ? JSON.stringify(v.metadata) : null], + ); + } + return { count: vectors.length, ids: vectors.map((v) => v.id) }; + }, + + async query(vector: number[], opts: QueryOptions): Promise<{ matches: Match[] }> { + const embedding = `[${vector.join(",")}]`; + const topK = opts.topK ?? 12; + const { rows } = opts.namespace + ? await pool.query<{ id: string; score: number; metadata: Record | null }>( + `SELECT id, 1 - (embedding <=> $1::vector) AS score, metadata + FROM ${TABLE} WHERE namespace=$2 + ORDER BY embedding <=> $1::vector LIMIT $3`, + [embedding, opts.namespace, topK], + ) + : await pool.query<{ id: string; score: number; metadata: Record | null }>( + `SELECT id, 1 - (embedding <=> $1::vector) AS score, metadata + FROM ${TABLE} + ORDER BY embedding <=> $1::vector LIMIT $2`, + [embedding, topK], + ); + const matches: Match[] = rows.map((r) => + r.metadata !== null ? { id: r.id, score: Number(r.score), metadata: r.metadata } : { id: r.id, score: Number(r.score) }, + ); + return { matches }; + }, + + async deleteByIds(ids: string[]): Promise<{ count: number }> { + if (ids.length === 0) return { count: 0 }; + const placeholders = ids.map((_, i) => `$${i + 1}`).join(","); + await pool.query(`DELETE FROM ${TABLE} WHERE id IN (${placeholders})`, ids); + return { count: ids.length }; + }, + }; + return adapter as unknown as Vectorize; +} diff --git a/src/selfhost/qdrant-vectorize.ts b/src/selfhost/qdrant-vectorize.ts new file mode 100644 index 0000000000..9621d01362 --- /dev/null +++ b/src/selfhost/qdrant-vectorize.ts @@ -0,0 +1,139 @@ +// Qdrant-backed Vectorize adapter for self-host RAG (#1217). Implements the same Cloudflare +// `Vectorize` surface (upsert / query / deleteByIds) as the SQLite and pgvector adapters but +// backed by a standalone Qdrant REST API. Qdrant provides ANN search, payload filtering by +// namespace, and scales to millions of vectors — making it the recommended vector store for +// production self-host deployments. Enable with QDRANT_URL=http://qdrant:6333 and --profile qdrant. +// +// Qdrant requires UUID or uint64 point IDs. String IDs (e.g. "owner/repo:file:line") are +// mapped to UUIDs via a deterministic SHA-1 hash, with the original ID stored in the payload +// for retrieval. The collection is auto-created at startup via initQdrantCollection(). +// +// Set QDRANT_API_KEY for deployments that require Bearer token authentication (cloud Qdrant, +// production on-prem). Omit for unauthenticated local/dev deployments. +import { createHash } from "node:crypto"; +import { incr } from "./metrics"; + +const DEFAULT_COLLECTION = "gittensory"; +const DEFAULT_DIM = 1024; // bge-m3 / mxbai-embed-large (1024-d); set QDRANT_DIM to override + +interface VectorRecord { + id: string; + values: number[]; + namespace?: string; + metadata?: Record; +} +interface QueryOptions { + topK?: number; + namespace?: string; +} +interface Match { + id: string; + score: number; + metadata?: Record; +} +interface QdrantSearchResult { + result: Array<{ id: string; score: number; payload: Record }>; +} + +/** Maps an arbitrary string ID to a UUID that Qdrant accepts as a point ID. Deterministic. */ +function idToUuid(id: string): string { + const h = createHash("sha1").update(id).digest("hex"); + return `${h.slice(0, 8)}-${h.slice(8, 12)}-${h.slice(12, 16)}-${h.slice(16, 20)}-${h.slice(20, 32)}`; +} + +/** Build fetch headers, including Bearer auth when QDRANT_API_KEY is set. */ +function qdrantHeaders(): Record { + const h: Record = { "content-type": "application/json" }; + if (process.env.QDRANT_API_KEY) h["api-key"] = process.env.QDRANT_API_KEY; + return h; +} + +/** + * Ensures the Qdrant collection exists. Safe to call on every startup — a 409 (already exists) + * is silently ignored. Call this before createQdrantVectorize() when QDRANT_URL is set. + */ +export async function initQdrantCollection(url: string, collection = DEFAULT_COLLECTION, dim = DEFAULT_DIM): Promise { + const base = url.replace(/\/+$/, ""); + const res = await fetch(`${base}/collections/${collection}`, { + method: "PUT", + headers: qdrantHeaders(), + body: JSON.stringify({ vectors: { size: dim, distance: "Cosine" } }), + }); + if (!res.ok && res.status !== 409) { + throw new Error(`Qdrant collection init failed: HTTP ${res.status}`); + } +} + +/** Creates a Vectorize-compatible adapter backed by the Qdrant REST API at `url`. */ +export function createQdrantVectorize(url: string, collection = DEFAULT_COLLECTION): Vectorize { + const base = url.replace(/\/+$/, ""); + + const adapter = { + async upsert(vectors: VectorRecord[]): Promise<{ count: number; ids: string[] }> { + const points = vectors.map((v) => ({ + id: idToUuid(v.id), + vector: v.values, + payload: { _orig_id: v.id, namespace: v.namespace ?? "", ...v.metadata }, + })); + const res = await fetch(`${base}/collections/${collection}/points`, { + method: "PUT", + headers: qdrantHeaders(), + body: JSON.stringify({ points }), + }); + if (!res.ok) { + incr("gittensory_qdrant_errors_total", { op: "upsert" }); + throw new Error(`Qdrant upsert failed: HTTP ${res.status}`); + } + incr("gittensory_qdrant_upserts_total", {}, vectors.length); + return { count: vectors.length, ids: vectors.map((v) => v.id) }; + }, + + async query(vector: number[], opts: QueryOptions): Promise<{ matches: Match[] }> { + const body: Record = { vector, limit: opts.topK ?? 12, with_payload: true }; + if (opts.namespace) { + body.filter = { must: [{ key: "namespace", match: { value: opts.namespace } }] }; + } + let res: Response; + try { + res = await fetch(`${base}/collections/${collection}/points/search`, { + method: "POST", + headers: qdrantHeaders(), + body: JSON.stringify(body), + }); + } catch { + // Qdrant unreachable — degrade gracefully (RAG returns no context rather than crashing) + incr("gittensory_qdrant_errors_total", { op: "query" }); + return { matches: [] }; + } + if (!res.ok) { + incr("gittensory_qdrant_errors_total", { op: "query" }); + return { matches: [] }; + } + incr("gittensory_qdrant_queries_total"); + const data = (await res.json()) as QdrantSearchResult; + const matches: Match[] = data.result.map((r) => { + const { _orig_id, namespace: _ns, ...rest } = r.payload; + const id = typeof _orig_id === "string" ? _orig_id : r.id; + return Object.keys(rest).length > 0 ? { id, score: r.score, metadata: rest } : { id, score: r.score }; + }); + return { matches }; + }, + + async deleteByIds(ids: string[]): Promise<{ count: number }> { + if (ids.length === 0) return { count: 0 }; + const points = ids.map(idToUuid); + const res = await fetch(`${base}/collections/${collection}/points/delete`, { + method: "POST", + headers: qdrantHeaders(), + body: JSON.stringify({ points }), + }); + if (!res.ok) { + incr("gittensory_qdrant_errors_total", { op: "delete" }); + throw new Error(`Qdrant deleteByIds failed: HTTP ${res.status}`); + } + return { count: ids.length }; + }, + }; + + return adapter as unknown as Vectorize; +} diff --git a/src/selfhost/redis-cache.ts b/src/selfhost/redis-cache.ts new file mode 100644 index 0000000000..c3149c789f --- /dev/null +++ b/src/selfhost/redis-cache.ts @@ -0,0 +1,40 @@ +// Redis-backed request-dedup cache for self-host (#1216). Prevents duplicate GitHub webhook +// deliveries from being processed twice — GitHub retries webhooks that receive a non-200 +// response, and each retry carries the same `x-github-delivery` UUID. By caching the delivery +// ID after a successful processing attempt, the server can return 204 immediately on retries +// without re-queuing the job. Activated when REDIS_URL is set alongside --profile redis. +import type { Redis } from "ioredis"; + +export function createRedisCache(redis: Redis) { + return { + async get(key: string): Promise { + return redis.get(key); + }, + async set(key: string, value: string, ttlSeconds: number): Promise { + await redis.set(key, value, "EX", ttlSeconds); + }, + async del(key: string): Promise { + await redis.del(key); + }, + }; +} + +export type RedisCache = ReturnType; + +/** + * Idempotency check for GitHub webhook deliveries. Returns true if the delivery was + * already seen (caller should short-circuit with 204). Marks the delivery as seen + * for `ttlSeconds` (default 5 min — covers GitHub's retry window) on the FIRST call. + * Best-effort: a Redis error is swallowed to avoid blocking webhook processing. + */ +export async function checkAndMarkDelivery(cache: RedisCache, deliveryId: string, ttlSeconds = 300): Promise { + try { + const seen = await cache.get(`delivery:${deliveryId}`); + if (seen) return true; + await cache.set(`delivery:${deliveryId}`, "1", ttlSeconds); + return false; + } catch { + // Redis unavailable → treat as first-time (never block processing on cache failure) + return false; + } +} diff --git a/src/selfhost/redis-ratelimit.ts b/src/selfhost/redis-ratelimit.ts new file mode 100644 index 0000000000..61274aa55d --- /dev/null +++ b/src/selfhost/redis-ratelimit.ts @@ -0,0 +1,44 @@ +// Redis-backed rate limiter for self-host (#977). The Cloudflare deploy uses a RateLimiter Durable Object; +// self-host provides the SAME binding surface (idFromName → get → fetch) backed by a Redis fixed-window +// counter, so `enforceRateLimit` works unchanged and is shared across instances. Without REDIS_URL the binding +// is absent and enforceRateLimit returns null (no limiting) — same as today. +import type { Redis } from "ioredis"; + +interface RateLimitBody { + key?: string; + limit?: number; + windowSeconds?: number; +} + +export function createRedisRateLimiter(redis: Redis): DurableObjectNamespace { + const stub = { + // A DO stub's fetch is called fetch-style: `.fetch(url, init)`. On Workers the runtime builds the Request; + // on Node we construct it ourselves so `.json()` is available. + async fetch(input: RequestInfo | URL, init?: RequestInit): Promise { + const request = input instanceof Request ? input : new Request(input, init); + const body = (await request.json().catch(() => null)) as RateLimitBody | null; + if (!body?.key || !body.limit || !body.windowSeconds) { + return Response.json({ error: "invalid_rate_limit_request" }, { status: 400 }); + } + const k = `ratelimit:${body.key}`; + const count = await redis.incr(k); + if (count === 1) await redis.expire(k, body.windowSeconds); // start the window on first hit + const ttlMs = await redis.pttl(k); + const resetMs = ttlMs > 0 ? ttlMs : body.windowSeconds * 1000; + const allowed = count <= body.limit; + const decision = { + allowed, + limit: body.limit, + remaining: Math.max(body.limit - count, 0), + resetAt: new Date(Date.now() + resetMs).toISOString(), + ...(allowed ? {} : { retryAfterSeconds: Math.max(1, Math.ceil(resetMs / 1000)) }), + }; + return Response.json(decision, { status: allowed ? 200 : 429 }); + }, + }; + const namespace = { + idFromName: (name: string) => ({ toString: () => name }), + get: (_id: unknown) => stub, + }; + return namespace as unknown as DurableObjectNamespace; +} diff --git a/src/selfhost/setup-wizard.ts b/src/selfhost/setup-wizard.ts new file mode 100644 index 0000000000..3c775f43a9 --- /dev/null +++ b/src/selfhost/setup-wizard.ts @@ -0,0 +1,74 @@ +// GitHub App Manifest one-click setup wizard for self-host (#981). On first run (no GITHUB_APP_ID), GET /setup +// renders a form that POSTs an App "manifest" to github.com/settings/apps/new; GitHub creates the App with the +// right permissions/events + webhook URL and redirects back to /setup/callback?code=…, which exchanges the +// code for the App's credentials and writes them to a file the operator loads (then restarts). The routes are +// disabled once an App is configured (server.ts gates on GITHUB_APP_ID), so this can't rebind a live install. + +export interface AppCredentials { + id: number; + slug: string; + webhook_secret: string; + pem: string; + client_id?: string; + client_secret?: string; +} + +/** The GitHub App manifest — permissions + events mirror docs §2 (the manual-setup instructions). */ +export function buildManifest(origin: string, state: string): Record { + const base = origin.replace(/\/+$/, ""); + return { + name: "Gittensory Self-Host", + url: base, + hook_attributes: { url: `${base}/v1/github/webhook` }, + redirect_url: `${base}/setup/callback?state=${encodeURIComponent(state)}`, + public: false, + default_permissions: { + pull_requests: "write", + contents: "write", + issues: "write", + checks: "read", + metadata: "read", + statuses: "read", + }, + default_events: ["pull_request", "pull_request_review", "push", "issues", "check_suite", "check_run", "status"], + }; +} + +/** HTML page that POSTs the manifest to GitHub's App-creation flow (one click). + * `state` is a random CSRF nonce tied to the session via an HttpOnly cookie in the caller. */ +export function renderSetupPage(origin: string, state: string): string { + const manifest = JSON.stringify(buildManifest(origin, state)).replace(/'/g, "'"); + return `Gittensory self-host setup + +

Gittensory self-host setup

+

This creates a GitHub App for your self-host instance. GitHub will redirect back here with the credentials, +which are written to a file for you to load — then restart the container.

+
+ + +
+`; +} + +/** Exchange the temporary manifest code for the App's credentials (id, slug, webhook secret, private key). */ +export async function exchangeManifestCode(code: string, fetchImpl: typeof fetch = fetch): Promise { + const res = await fetchImpl(`https://api.github.com/app-manifests/${encodeURIComponent(code)}/conversions`, { + method: "POST", + headers: { accept: "application/vnd.github+json", "user-agent": "gittensory-selfhost" }, + }); + if (!res.ok) throw new Error(`manifest_exchange_http_${res.status}`); + return (await res.json()) as AppCredentials; +} + +/** Serialize the credentials as .env lines for the operator to load. */ +export function credentialsToEnv(creds: AppCredentials): string { + const lines = [ + `GITHUB_APP_ID=${creds.id}`, + `GITHUB_APP_SLUG=${creds.slug}`, + `GITHUB_WEBHOOK_SECRET=${creds.webhook_secret}`, + `GITHUB_APP_PRIVATE_KEY=${JSON.stringify(creds.pem)}`, + ]; + if (creds.client_id) lines.push(`GITHUB_OAUTH_CLIENT_ID=${creds.client_id}`); + if (creds.client_secret) lines.push(`GITHUB_OAUTH_CLIENT_SECRET=${creds.client_secret}`); + return `${lines.join("\n")}\n`; +} diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts new file mode 100644 index 0000000000..898f35ef91 --- /dev/null +++ b/src/selfhost/sqlite-queue.ts @@ -0,0 +1,168 @@ +// Durable, SQLite-backed job queue for the self-host runtime (#980 reliability). Unlike the in-process FIFO, +// jobs are PERSISTED — a restart (or crash) re-claims anything left in flight instead of losing it. It still +// presents the Cloudflare `Queue` binding surface (send / sendBatch) so the app code is unchanged; only the +// backing store differs. Single-process model: node:sqlite is synchronous + serial, so claim (SELECT→UPDATE) +// is atomic with no row-lock dance. +import type { SqliteDriver } from "./d1-adapter"; +import { logAudit, extractPayloadType } from "./audit"; +import { incr } from "./metrics"; +import type { JobMessage } from "../types"; + +const TABLE = "_selfhost_jobs"; +const DDL = ` +CREATE TABLE IF NOT EXISTS ${TABLE} ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + payload TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + attempts INTEGER NOT NULL DEFAULT 0, + run_after INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + last_error TEXT +); +CREATE INDEX IF NOT EXISTS ${TABLE}_claim ON ${TABLE}(status, run_after);`; + +export interface DurableQueue { + binding: Queue; + start(): void; + stop(): Promise; + drain(): Promise; + size(): number; + deadCount(): number; +} + +interface JobRow { + id: number; + payload: string; + attempts: number; +} + +export interface SqliteQueueOptions { + maxRetries?: number; + pollIntervalMs?: number; + backoffMs?: (attempt: number) => number; + /** Max concurrent `processOne()` loops. Defaults to QUEUE_CONCURRENCY env var or 1. */ + concurrency?: number; +} + +export function createSqliteQueue(driver: SqliteDriver, consume: (message: JobMessage) => Promise, opts: SqliteQueueOptions = {}): DurableQueue { + const maxRetries = opts.maxRetries ?? 5; + const pollIntervalMs = opts.pollIntervalMs ?? 1000; + const backoff = opts.backoffMs ?? ((attempt: number) => Math.min(60_000, 1000 * 2 ** attempt)); + const concurrency = opts.concurrency ?? Math.max(1, Number(process.env.QUEUE_CONCURRENCY ?? "1")); + + driver.exec(DDL); + // Recover jobs a crashed previous run left mid-flight → make them claimable again. + const recovered = driver.query(`UPDATE ${TABLE} SET status='pending' WHERE status='processing'`, []).changes; + if (recovered) console.log(JSON.stringify({ event: "selfhost_queue_recovered", count: recovered })); + + let running = false; + let active = 0; // number of concurrent pump() loops currently draining jobs + let timer: ReturnType | null = null; + + function enqueue(message: JobMessage, delaySeconds: number): void { + const now = Date.now(); + driver.query(`INSERT INTO ${TABLE} (payload, status, attempts, run_after, created_at) VALUES (?, 'pending', 0, ?, ?)`, [JSON.stringify(message), now + delaySeconds * 1000, now]); + incr("gittensory_jobs_enqueued_total"); + void pump(); + } + + function claimNext(): JobRow | null { + const { rows } = driver.query(`SELECT id, payload, attempts FROM ${TABLE} WHERE status='pending' AND run_after<=? ORDER BY id LIMIT 1`, [Date.now()]); + const row = rows[0] as JobRow | undefined; + if (!row) return null; + const { changes } = driver.query(`UPDATE ${TABLE} SET status='processing' WHERE id=? AND status='pending'`, [row.id]); + /* v8 ignore next */ // the no-rows branch is a multi-writer guard; unreachable in the single-process model + return changes ? row : null; + } + + async function processOne(): Promise { + const job = claimNext(); + if (!job) return false; + const claimedAt = Date.now(); + let message: JobMessage; + try { + message = JSON.parse(job.payload) as JobMessage; + } catch { + driver.query(`UPDATE ${TABLE} SET status='dead', last_error='unparseable payload' WHERE id=?`, [job.id]); + incr("gittensory_jobs_dead_total"); + logAudit({ event: "job_dead", ts: Date.now(), job_id: job.id, latency_ms: Date.now() - claimedAt, attempts: job.attempts + 1, error: "unparseable payload" }); + return true; + } + try { + await consume(message); + driver.query(`DELETE FROM ${TABLE} WHERE id=?`, [job.id]); + incr("gittensory_jobs_processed_total"); + logAudit({ event: "job_complete", ts: Date.now(), job_id: job.id, payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts: job.attempts + 1 }); + } catch (error) { + const attempts = job.attempts + 1; + const errMsg = error instanceof Error ? error.message : "unknown error"; + incr("gittensory_jobs_failed_total"); + if (attempts >= maxRetries) { + driver.query(`UPDATE ${TABLE} SET status='dead', attempts=?, last_error=? WHERE id=?`, [attempts, errMsg, job.id]); + incr("gittensory_jobs_dead_total"); + console.error(JSON.stringify({ level: "error", event: "selfhost_job_dead", id: job.id, attempts, error: errMsg })); + logAudit({ event: "job_dead", ts: Date.now(), job_id: job.id, payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts, error: errMsg }); + } else { + driver.query(`UPDATE ${TABLE} SET status='pending', attempts=?, run_after=?, last_error=? WHERE id=?`, [attempts, Date.now() + backoff(attempts), errMsg, job.id]); + logAudit({ event: "job_error", ts: Date.now(), job_id: job.id, payload_type: extractPayloadType(job.payload), latency_ms: Date.now() - claimedAt, attempts, error: errMsg }); + } + } + return true; + } + + // Drains every job that is currently DUE. A retry is rescheduled into the future (run_after > now) so it is + // not re-claimed here — the next poll tick picks it up — which also bounds this loop. Up to `concurrency` + // pump loops may run simultaneously (each claims its own job row, atomic under node:sqlite's serial writes). + async function pump(): Promise { + if (active >= concurrency) return; + active++; + try { + while (await processOne()) { + /* keep draining due jobs */ + } + } finally { + active--; + } + } + + const binding = { + async send(message: JobMessage, options?: { delaySeconds?: number }): Promise { + enqueue(message, options?.delaySeconds ?? 0); + }, + async sendBatch(messages: Iterable<{ body: JobMessage; delaySeconds?: number }>): Promise { + for (const m of messages) enqueue(m.body, m.delaySeconds ?? 0); + }, + } as unknown as Queue; + + return { + binding, + start() { + if (running) return; + running = true; + const tick = (): void => { + /* v8 ignore next */ // stop() clears the timer, so a tick never fires with running=false + if (!running) return; + void pump().finally(() => { + if (running) timer = setTimeout(tick, pollIntervalMs); + }); + }; + tick(); + }, + async stop() { + running = false; + if (timer) clearTimeout(timer); + while (active > 0) await new Promise((r) => setTimeout(r, 10)); // let in-flight pumps finish + }, + async drain() { + // send() fire-and-forgets a pump; wait for any in-flight pumps to settle, then drain to completion. + while (active > 0) await new Promise((r) => setTimeout(r, 5)); + await pump(); + }, + size() { + return Number((driver.query(`SELECT COUNT(*) AS c FROM ${TABLE} WHERE status IN ('pending','processing')`, []).rows[0] as { c: number }).c); + }, + deadCount() { + return Number((driver.query(`SELECT COUNT(*) AS c FROM ${TABLE} WHERE status='dead'`, []).rows[0] as { c: number }).c); + }, + }; +} diff --git a/src/selfhost/stubs/agents-mcp.ts b/src/selfhost/stubs/agents-mcp.ts new file mode 100644 index 0000000000..2bf57c2192 --- /dev/null +++ b/src/selfhost/stubs/agents-mcp.ts @@ -0,0 +1,3 @@ +// Self-host replacement for agents/mcp. The Cloudflare Agents SDK transport is Durable-Object-backed +// (Workers-only); this re-exports the Node-compatible WebStandardStreamableHTTP implementation instead. +export { createMcpHandler } from "../mcp-server-node"; diff --git a/src/selfhost/stubs/puppeteer.ts b/src/selfhost/stubs/puppeteer.ts new file mode 100644 index 0000000000..cc00e61a6c --- /dev/null +++ b/src/selfhost/stubs/puppeteer.ts @@ -0,0 +1,31 @@ +// Self-host replacement for @cloudflare/puppeteer (#980). When BROWSER_WS_ENDPOINT is set, connects to an +// external Chrome-compatible browser (e.g. a `browserless/chrome` sidecar) via puppeteer-core's WebSocket +// connect API — this makes the /gittensory/shot on-demand render endpoint fully functional. When the env var +// is absent, the functions throw so the caller's `if (!env.BROWSER)` guard (in shot.ts) short-circuits first. +// Install: add `puppeteer-core` to package deps + set BROWSER_WS_ENDPOINT (or set INSTALL_VISUAL_REVIEW=true +// in the Dockerfile and point at a `browserless/chrome:latest` sidecar). + +/** Connect to the external browser, using puppeteer-core loaded at runtime (avoids bundling ~20 MB of + * puppeteer's internals when visual review is disabled). Throws a clear error if not installed. */ +async function connectBrowser(): Promise { + const wsEndpoint = process.env.BROWSER_WS_ENDPOINT; + if (!wsEndpoint) throw new Error("browser_rendering_unavailable_on_selfhost: set BROWSER_WS_ENDPOINT to a browserless/chrome ws:// URL"); + try { + // @ts-expect-error -- puppeteer-core is an optional runtime dep (INSTALL_VISUAL_REVIEW=true), not in project deps + const { default: puppeteer } = (await import("puppeteer-core")) as { default: { connect(o: { browserWSEndpoint: string }): unknown } }; + /* v8 ignore next -- only reachable when puppeteer-core is installed (INSTALL_VISUAL_REVIEW=true builds) */ + return puppeteer.connect({ browserWSEndpoint: wsEndpoint }); + } catch (e) { + if (e instanceof Error && e.message.includes("Cannot find package")) { + throw new Error("browser_rendering_unavailable_on_selfhost: install puppeteer-core or build with INSTALL_VISUAL_REVIEW=true"); + } + /* v8 ignore next -- only reachable when puppeteer-core is installed but connect() itself throws */ + throw e; + } +} + +export default { + /** Drop-in for @cloudflare/puppeteer's launch(browserWorker). Ignores the CF binding arg and connects via WS. */ + launch: (_browserWorkerHint: unknown): Promise => connectBrowser(), + connect: (_opts: unknown): Promise => connectBrowser(), +}; diff --git a/src/selfhost/vectorize.ts b/src/selfhost/vectorize.ts new file mode 100644 index 0000000000..8ad8a49a1c --- /dev/null +++ b/src/selfhost/vectorize.ts @@ -0,0 +1,86 @@ +// SQLite-backed Vectorize adapter for self-host RAG (#979). Implements the Cloudflare `Vectorize` binding +// surface (upsert / query / deleteByIds) that gittensory's RAG (reviewVectorAdapter) wraps, backed by a +// SQLite table with brute-force cosine similarity. For a repo's worth of chunks (hundreds–few-thousand +// vectors per namespace) this is fast enough; namespaces (one per repo) keep each query's candidate set +// small. Embeddings come from the OpenAI-compatible AI adapter's /embeddings path (e.g. Ollama bge-m3, 1024-d). +import type { SqliteDriver } from "./d1-adapter"; + +const TABLE = "_selfhost_vectors"; +const DDL = ` +CREATE TABLE IF NOT EXISTS ${TABLE} ( + id TEXT PRIMARY KEY, + namespace TEXT NOT NULL DEFAULT '', + embedding TEXT NOT NULL, + metadata TEXT +); +CREATE INDEX IF NOT EXISTS ${TABLE}_ns ON ${TABLE}(namespace);`; + +interface VectorRecord { + id: string; + values: number[]; + namespace?: string; + metadata?: Record; +} +interface QueryOptions { + topK?: number; + namespace?: string; + returnMetadata?: string; +} +interface Match { + id: string; + score: number; + metadata?: Record; +} + +export function cosineSimilarity(a: number[], b: number[]): number { + let dot = 0; + let na = 0; + let nb = 0; + const n = Math.min(a.length, b.length); + for (let i = 0; i < n; i += 1) { + const x = a[i] as number; + const y = b[i] as number; + dot += x * y; + na += x * x; + nb += y * y; + } + const denom = Math.sqrt(na) * Math.sqrt(nb); + return denom === 0 ? 0 : dot / denom; +} + +export function createSqliteVectorize(driver: SqliteDriver): Vectorize { + driver.exec(DDL); + const adapter = { + async upsert(vectors: VectorRecord[]): Promise<{ count: number; ids: string[] }> { + for (const v of vectors) { + driver.query( + `INSERT INTO ${TABLE} (id, namespace, embedding, metadata) VALUES (?,?,?,?) + ON CONFLICT(id) DO UPDATE SET namespace=excluded.namespace, embedding=excluded.embedding, metadata=excluded.metadata`, + [v.id, v.namespace ?? "", JSON.stringify(v.values), v.metadata ? JSON.stringify(v.metadata) : null], + ); + } + return { count: vectors.length, ids: vectors.map((v) => v.id) }; + }, + async query(vector: number[], opts: QueryOptions): Promise<{ matches: Match[] }> { + const { rows } = opts.namespace + ? driver.query(`SELECT id, embedding, metadata FROM ${TABLE} WHERE namespace=?`, [opts.namespace]) + : driver.query(`SELECT id, embedding, metadata FROM ${TABLE}`, []); + const scored: Match[] = rows.map((r) => { + const values = JSON.parse(r.embedding as string) as number[]; + const metadata = r.metadata ? (JSON.parse(r.metadata as string) as Record) : undefined; + const score = cosineSimilarity(vector, values); + return metadata === undefined ? { id: r.id as string, score } : { id: r.id as string, score, metadata }; + }); + scored.sort((a, b) => b.score - a.score); + return { matches: scored.slice(0, opts.topK ?? 12) }; + }, + async deleteByIds(ids: string[]): Promise<{ count: number }> { + for (let i = 0; i < ids.length; i += 90) { + const batch = ids.slice(i, i + 90); + driver.query(`DELETE FROM ${TABLE} WHERE id IN (${batch.map(() => "?").join(",")})`, batch); + } + return { count: ids.length }; + }, + }; + return adapter as unknown as Vectorize; +} diff --git a/src/server.ts b/src/server.ts new file mode 100644 index 0000000000..97fd351349 --- /dev/null +++ b/src/server.ts @@ -0,0 +1,304 @@ +// Self-host Node entry (#980). Runs gittensory's SAME Worker handlers on Node. Backends are pluggable: +// • DB: SQLite (node:sqlite, default) OR Postgres (DATABASE_URL=postgres://… → shared, multi-instance). +// • Queue: durable SQLite queue OR a Postgres queue (FOR UPDATE SKIP LOCKED). +// • Rate limit: a Redis fixed-window limiter when REDIS_URL is set (else no limiting, as today). +// • RAG vector store: SQLite-only for now (omitted on Postgres → RAG degrades to no-context). +// Serves the Hono app via @hono/node-server, drives the queue with the same processJob, ticks the same +// scheduled handler on a timer, exposes /health /ready /metrics, and shuts down gracefully. The Cloudflare +// Worker (src/index.ts) is untouched — this is a parallel entry the self-host esbuild build bundles. +import { readFileSync, writeFileSync } from "node:fs"; +import { randomUUID } from "node:crypto"; +import { DatabaseSync } from "node:sqlite"; +import { serve } from "@hono/node-server"; +import worker from "./index"; +import { processJob } from "./queue/processors"; +import { createSelfHostAi } from "./selfhost/ai"; +import { credentialsToEnv, exchangeManifestCode, renderSetupPage } from "./selfhost/setup-wizard"; +import { createD1Adapter, nodeSqliteDriver } from "./selfhost/d1-adapter"; +import { readiness } from "./selfhost/health"; +import { gauge, incr, renderMetrics } from "./selfhost/metrics"; +import { runSelfHostMigrations } from "./selfhost/migrate"; +import { createPgAdapter } from "./selfhost/pg-adapter"; +import { createPgQueue } from "./selfhost/pg-queue"; +import { createPgVectorize, initPgVectorize } from "./selfhost/pg-vectorize"; +import { createSqliteQueue } from "./selfhost/sqlite-queue"; +import { createSqliteVectorize } from "./selfhost/vectorize"; +import type { JobMessage } from "./types"; + +/** Resolve `_FILE` env vars (Docker secrets / multi-line keys) into `` at startup. */ +function loadFileSecrets(): void { + for (const key of Object.keys(process.env)) { + if (!key.endsWith("_FILE") || !process.env[key]) continue; + const target = key.slice(0, -"_FILE".length); + if (process.env[target]) continue; // an explicit value wins + try { + process.env[target] = readFileSync(process.env[key] as string, "utf8").trim(); + } catch { + console.error(JSON.stringify({ level: "error", event: "selfhost_secret_file_unreadable", var: key })); + } + } +} + +interface Backend { + db: D1Database; + queue: { binding: Queue; start(): void; stop(): Promise; size(): number | Promise; deadCount(): number | Promise }; + vectorize?: Vectorize; + shutdown(): Promise; +} + +/** Retry a Postgres connection until it succeeds (up to maxWaitMs). Prevents crash-restart loops when + * gittensory starts before Postgres is ready (common in `--profile postgres` compose stacks). */ +async function waitForPostgres(url: string, maxWaitMs = 30_000): Promise { + const pg = (await import("pg")).default; + const start = Date.now(); + let attempt = 0; + while (true) { + const client = new pg.Client({ connectionString: url }); + try { + await client.connect(); + await client.end(); + return; + } catch { + await client.end().catch(() => undefined); + attempt++; + const elapsed = Date.now() - start; + if (elapsed >= maxWaitMs) throw new Error(`Postgres not ready after ${maxWaitMs}ms (${attempt} attempts)`); + const delay = Math.min(2000, 200 * attempt); + console.log(JSON.stringify({ event: "selfhost_pg_wait", attempt, elapsed_ms: elapsed, retry_in_ms: delay })); + await new Promise((r) => setTimeout(r, delay)); + } + } +} + +/** Build the Postgres backend (shared DB + queue) when DATABASE_URL is a postgres:// URL. */ +async function buildPostgresBackend(url: string, consume: (m: JobMessage) => Promise): Promise { + await waitForPostgres(url); + const pg = (await import("pg")).default; + pg.types.setTypeParser(20, (v: string) => Number.parseInt(v, 10)); // int8 (COUNT) → number, like D1 + const pool = new pg.Pool({ connectionString: url }); + const db = createPgAdapter(pool); + const queue = createPgQueue(pool, consume); + await queue.init(); + let vectorize: Vectorize | undefined; + if (process.env.PGVECTOR_ENABLED === "true") { + await initPgVectorize(pool); + vectorize = createPgVectorize(pool); + } + return { + db, + queue, + ...(vectorize ? { vectorize } : {}), + async shutdown() { + await queue.stop(); + await pool.end(); + }, + }; +} + +/** Build the SQLite backend (single file, default). */ +function buildSqliteBackend(consume: (m: JobMessage) => Promise): Backend { + const sqlite = new DatabaseSync(process.env.DATABASE_PATH ?? "/data/gittensory.sqlite"); + sqlite.exec("PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 5000;"); + const driver = nodeSqliteDriver(sqlite as never); + const db = createD1Adapter(driver); + const queue = createSqliteQueue(driver, consume); + const vectorize = createSqliteVectorize(driver); + return { + db, + queue, + vectorize, + async shutdown() { + await queue.stop(); + try { + sqlite.exec("PRAGMA wal_checkpoint(TRUNCATE);"); + sqlite.close(); + } catch { + /* best-effort */ + } + }, + }; +} + +async function main(): Promise { + loadFileSecrets(); + const startedAt = Date.now(); + + // The queue consumer captures `env`, assigned below (the first job only runs once an HTTP/cron event + // arrives, by which point env is set). + let env: Env; + const consume = async (message: JobMessage): Promise => { + await processJob(env, message); + }; + + const databaseUrl = process.env.DATABASE_URL; + const usePostgres = !!databaseUrl && /^postgres(ql)?:\/\//i.test(databaseUrl); + const backend = usePostgres ? await buildPostgresBackend(databaseUrl as string, consume) : buildSqliteBackend(consume); + console.log(JSON.stringify({ event: "selfhost_backend", backend: usePostgres ? "postgres" : "sqlite" })); + + const applied = await runSelfHostMigrations(backend.db, process.env.MIGRATIONS_DIR ?? "migrations"); + console.log(JSON.stringify({ event: "selfhost_migrations_applied", count: applied })); + + const ai = createSelfHostAi(process.env); + if (ai) console.log(JSON.stringify({ event: "selfhost_ai_provider", provider: process.env.AI_PROVIDER })); + + // Redis fixed-window rate limiter + webhook dedup cache (else absent when REDIS_URL is unset). + let rateLimiter: DurableObjectNamespace | undefined; + let webhookCache: import("./selfhost/redis-cache").RedisCache | undefined; + if (process.env.REDIS_URL) { + const { Redis } = await import("ioredis"); + const redisClient = new Redis(process.env.REDIS_URL); + const { createRedisRateLimiter } = await import("./selfhost/redis-ratelimit"); + const { createRedisCache } = await import("./selfhost/redis-cache"); + rateLimiter = createRedisRateLimiter(redisClient); + webhookCache = createRedisCache(redisClient); + console.log(JSON.stringify({ event: "selfhost_rate_limiter", backend: "redis" })); + } + + // Qdrant vector store — overrides the backend's built-in sqlite-vec / pgvector when QDRANT_URL is set. + let vectorizeOverride: Vectorize | undefined; + if (process.env.QDRANT_URL) { + const { createQdrantVectorize, initQdrantCollection } = await import("./selfhost/qdrant-vectorize"); + await initQdrantCollection(process.env.QDRANT_URL); + vectorizeOverride = createQdrantVectorize(process.env.QDRANT_URL); + console.log(JSON.stringify({ event: "selfhost_vectorize", backend: "qdrant" })); + } + + env = { + ...process.env, + DB: backend.db, + JOBS: backend.queue.binding, + AI: ai, + // Qdrant takes priority; falls back to the backend's built-in vectorize (pgvector or sqlite-vec) + ...(vectorizeOverride ? { VECTORIZE: vectorizeOverride } : backend.vectorize ? { VECTORIZE: backend.vectorize } : {}), + ...(rateLimiter ? { RATE_LIMITER: rateLimiter } : {}), + // Visual review: when BROWSER_WS_ENDPOINT is set, expose a truthy BROWSER binding so shot.ts's + // `if (!env.BROWSER) return` guard is bypassed; the puppeteer stub then connects via WS. + ...(process.env.BROWSER_WS_ENDPOINT ? { BROWSER: {} } : {}), + } as unknown as Env; + + gauge("gittensory_queue_pending", () => backend.queue.size()); + gauge("gittensory_queue_dead", () => backend.queue.deadCount()); + gauge("gittensory_uptime_seconds", () => Math.floor((Date.now() - startedAt) / 1000)); + // Pre-initialize job counters to 0 so they appear in the first Prometheus scrape (lazy counters + // created on first use would otherwise cause "No data" in Grafana until the first job event). + for (const c of [ + "gittensory_jobs_enqueued_total", "gittensory_jobs_processed_total", + "gittensory_jobs_failed_total", "gittensory_jobs_dead_total", + "gittensory_http_requests_total", "gittensory_webhook_dedup_total", + "gittensory_qdrant_queries_total", "gittensory_qdrant_upserts_total", + ]) + incr(c, undefined, 0); + + const ctx = { + waitUntil: (p: Promise) => void Promise.resolve(p).catch(() => undefined), + passThroughOnException: () => undefined, + } as unknown as ExecutionContext; + + const port = Number(process.env.PORT ?? 8787); + const server = serve( + { + fetch: async (request: Request) => { + const path = new URL(request.url).pathname; + if (path === "/health") return new Response(JSON.stringify({ status: "ok" }), { headers: { "content-type": "application/json" } }); + if (path === "/ready") { + const r = await readiness(backend.db); + return new Response(JSON.stringify(r), { status: r.ok ? 200 : 503, headers: { "content-type": "application/json" } }); + } + if (path === "/metrics") return new Response(await renderMetrics(), { headers: { "content-type": "text/plain; version=0.0.4" } }); + // First-run GitHub App setup wizard — only while no App is configured (can't rebind a live install). + if ((path === "/setup" || path === "/setup/callback") && !process.env.GITHUB_APP_ID) { + // PUBLIC_API_ORIGIN is required: falling back to request.url.origin would let an attacker spoof + // the Host header and redirect the App-creation callback to an attacker-controlled domain, where + // they could exchange the code for the App private key and webhook secret. + const origin = process.env.PUBLIC_API_ORIGIN; + if (!origin) { + return new Response( + "PUBLIC_API_ORIGIN must be set before using the setup wizard — add it to your .env file", + { status: 400 }, + ); + } + if (path === "/setup") { + // Generate a per-visit CSRF nonce, embed it in the manifest's redirect_url, and bind it to + // this browser session via an HttpOnly cookie so the callback can validate it. + const state = randomUUID(); + return new Response(renderSetupPage(origin, state), { + headers: { + "content-type": "text/html; charset=utf-8", + "Set-Cookie": `setup_state=${state}; Path=/setup; HttpOnly; SameSite=Lax; Max-Age=3600`, + }, + }); + } + const params = new URL(request.url).searchParams; + const code = params.get("code"); + if (!code) return new Response("missing ?code", { status: 400 }); + // Validate the CSRF state: must match the cookie set when /setup was served. + const stateParam = params.get("state"); + const cookieHeader = request.headers.get("cookie") ?? ""; + const cookieState = cookieHeader.split(";").map((c) => c.trim()).find((c) => c.startsWith("setup_state="))?.slice("setup_state=".length); + if (!stateParam || !cookieState || stateParam !== cookieState) { + return new Response("invalid state parameter", { status: 403 }); + } + try { + const creds = await exchangeManifestCode(code); + const outPath = process.env.SETUP_OUTPUT_PATH ?? "/data/gittensory-app.env"; + writeFileSync(outPath, credentialsToEnv(creds), { mode: 0o600 }); + console.log(JSON.stringify({ event: "selfhost_app_created", slug: creds.slug, app_id: creds.id })); + return new Response(`

GitHub App created ✓

Credentials written to ${outPath}. Add them to your .env (or load the file), install the App on your repos, and restart the container.

`, { headers: { "content-type": "text/html; charset=utf-8" } }); + } catch (error) { + return new Response(`setup failed: ${error instanceof Error ? error.message : "error"}`, { status: 500 }); + } + } + incr("gittensory_http_requests_total"); + // Webhook delivery dedup: return 204 immediately for already-processed delivery IDs. + // We mark only AFTER a successful response — failed/rejected webhooks must be retryable. + const isWebhook = webhookCache && path === "/v1/github/webhook" && request.method === "POST"; + const deliveryId = isWebhook ? request.headers.get("x-github-delivery") : null; + if (deliveryId) { + const seen = await webhookCache!.get(`delivery:${deliveryId}`); + if (seen) { + incr("gittensory_webhook_dedup_total"); + return new Response(null, { status: 204 }); + } + } + const response = await worker.fetch(request, env, ctx); + if (deliveryId && response.ok) { + // Best-effort — never block the response on a cache write failure + void webhookCache!.set(`delivery:${deliveryId}`, "1", 300).catch(() => undefined); + } + return response; + }, + port, + }, + () => console.log(JSON.stringify({ event: "selfhost_listening", port })), + ); + + backend.queue.start(); + + // Cron — gittensory ticks ~every 2 minutes; drive the SAME scheduled handler. + const intervalMs = Number(process.env.CRON_INTERVAL_MS ?? 120_000); + const cron = setInterval(() => { + const controller = { scheduledTime: Date.now(), cron: "*/2 * * * *", noRetry: () => undefined } as unknown as ScheduledController; + Promise.resolve(worker.scheduled(controller, env, ctx)).catch((error) => + console.error(JSON.stringify({ level: "error", event: "selfhost_cron_error", error: error instanceof Error ? error.message : "unknown error" })), + ); + }, intervalMs); + + // Graceful shutdown: stop accepting HTTP, let the queue finish, close the backend. + let shuttingDown = false; + const shutdown = async (signal: string): Promise => { + if (shuttingDown) return; + shuttingDown = true; + console.log(JSON.stringify({ event: "selfhost_shutdown", signal })); + clearInterval(cron); + server.close(); + await backend.shutdown(); + process.exit(0); + }; + process.on("SIGTERM", () => void shutdown("SIGTERM")); + process.on("SIGINT", () => void shutdown("SIGINT")); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000000..66b09b0480 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,137 @@ +# Terraform config for a Hetzner Cloud VPS running the gittensory self-host stack. +# Provisions a single server with Docker + Docker Compose pre-installed via cloud-init. +# After provisioning: SSH in, clone the repo, copy .env.example → .env, and run +# `docker compose up -d` (or `docker compose --profile postgres --profile caddy up -d`). + +terraform { + required_version = ">= 1.6" + required_providers { + hcloud = { + source = "hetznercloud/hcloud" + version = "~> 1.49" + } + } +} + +provider "hcloud" { + token = var.hcloud_token +} + +# ── SSH key ──────────────────────────────────────────────────────────────────── +resource "hcloud_ssh_key" "gittensory" { + name = "gittensory-deploy" + public_key = var.ssh_public_key +} + +# ── Firewall ─────────────────────────────────────────────────────────────────── +resource "hcloud_firewall" "gittensory" { + name = "gittensory" + + # SSH — tighten source_ips to your IP range in production + rule { + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.admin_ip_allowlist + } + + # HTTP (Caddy ACME challenge + redirect) + rule { + direction = "in" + protocol = "tcp" + port = "80" + source_ips = ["0.0.0.0/0", "::/0"] + } + + # HTTPS + rule { + direction = "in" + protocol = "tcp" + port = "443" + source_ips = ["0.0.0.0/0", "::/0"] + } + + # HTTP/3 QUIC (used by Caddy when the caddy profile is active) + rule { + direction = "in" + protocol = "udp" + port = "443" + source_ips = ["0.0.0.0/0", "::/0"] + } + + # Direct app access — remove once behind Caddy + rule { + direction = "in" + protocol = "tcp" + port = "8787" + source_ips = var.admin_ip_allowlist + } +} + +# ── Persistent volume for /data (SQLite DB + Litestream WAL) ────────────────── +resource "hcloud_volume" "gittensory_data" { + name = "gittensory-data" + size = var.volume_size_gb + location = var.location + format = "ext4" + automount = false +} + +# ── Server ───────────────────────────────────────────────────────────────────── +resource "hcloud_server" "gittensory" { + name = "gittensory" + server_type = var.server_type + image = "ubuntu-24.04" + location = var.location + ssh_keys = [hcloud_ssh_key.gittensory.id] + firewall_ids = [hcloud_firewall.gittensory.id] + keep_disk = true + + user_data = <<-CLOUDINIT + #cloud-config + package_update: true + package_upgrade: true + + packages: + - ca-certificates + - curl + - gnupg + - git + - jq + + runcmd: + # Install Docker from the official apt repository + - install -m 0755 -d /etc/apt/keyrings + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg + - chmod a+r /etc/apt/keyrings/docker.gpg + - | + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo $VERSION_CODENAME) stable" \ + > /etc/apt/sources.list.d/docker.list + - apt-get update -y + - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + - systemctl enable --now docker + # Mount the attached volume at /data + - mkdir -p /data + - | + DEVICE=$(lsblk -o NAME,SERIAL -dpn | grep $(echo "${hcloud_volume.gittensory_data.linux_device}" | sed 's|/dev/||') | awk '{print $1}') + mount /dev/$$DEVICE /data + - echo "LABEL=gittensory-data /data ext4 defaults 0 2" >> /etc/fstab + # Allow the ubuntu user to run docker without sudo + - usermod -aG docker ubuntu + - echo "cloud-init: gittensory host ready" > /var/log/gittensory-init.log + CLOUDINIT + + labels = { + app = "gittensory" + managed = "terraform" + } +} + +# Attach the volume after the server is created +resource "hcloud_volume_attachment" "gittensory_data" { + server_id = hcloud_server.gittensory.id + volume_id = hcloud_volume.gittensory_data.id + automount = true +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 0000000000..51b930497c --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,19 @@ +output "server_ipv4" { + description = "Public IPv4 address of the gittensory server" + value = hcloud_server.gittensory.ipv4_address +} + +output "server_ipv6" { + description = "Public IPv6 address of the gittensory server" + value = hcloud_server.gittensory.ipv6_address +} + +output "ssh_command" { + description = "SSH command to access the server" + value = "ssh ubuntu@${hcloud_server.gittensory.ipv4_address}" +} + +output "volume_device" { + description = "Linux block device path for the data volume" + value = hcloud_volume.gittensory_data.linux_device +} diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000000..3d1941c3d8 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,34 @@ +variable "hcloud_token" { + description = "Hetzner Cloud API token (generate at console.hetzner.cloud → Security → API Tokens)" + type = string + sensitive = true +} + +variable "ssh_public_key" { + description = "SSH public key content for server access (e.g. file('~/.ssh/id_ed25519.pub'))" + type = string +} + +variable "server_type" { + description = "Hetzner server type. cx22 = 2 vCPU / 4 GB (sufficient for <50 reviews/day). cpx21 = 3 vCPU AMD / 4 GB for heavier load." + type = string + default = "cx22" +} + +variable "location" { + description = "Hetzner datacenter location: nbg1 (Nuremberg), fsn1 (Falkenstein), hel1 (Helsinki), ash (Ashburn VA), sin (Singapore)" + type = string + default = "nbg1" +} + +variable "volume_size_gb" { + description = "Size of the persistent data volume in GB (holds the SQLite DB and Litestream WAL segments)" + type = number + default = 20 +} + +variable "admin_ip_allowlist" { + description = "CIDR ranges allowed to SSH and access the raw app port (8787). Restrict to your IP(s) in production." + type = list(string) + default = ["0.0.0.0/0", "::/0"] +} diff --git a/test/integration/selfhost-pg.test.ts b/test/integration/selfhost-pg.test.ts new file mode 100644 index 0000000000..498a05c08c --- /dev/null +++ b/test/integration/selfhost-pg.test.ts @@ -0,0 +1,58 @@ +// Real-Postgres integration test for the self-host PG backend (#977). Skipped unless PG_TEST_URL is set, so +// CI (no Postgres) skips it; run locally against a real PG: +// docker run -d -e POSTGRES_PASSWORD=devpw -e POSTGRES_DB=gittensory -p 55432:5432 postgres:16 +// PG_TEST_URL=postgres://postgres:devpw@localhost:55432/gittensory npx vitest run test/integration/selfhost-pg.test.ts +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import pg from "pg"; +import { runSelfHostMigrations } from "../../src/selfhost/migrate"; +import { createPgAdapter } from "../../src/selfhost/pg-adapter"; + +const URL = process.env.PG_TEST_URL; +const suite = URL ? describe : describe.skip; + +suite("Postgres backend (#977) — real Postgres", () => { + let pool: pg.Pool; + + beforeAll(async () => { + pg.types.setTypeParser(20, (v: string) => Number.parseInt(v, 10)); // int8 (COUNT) → number, like D1 + pool = new pg.Pool({ connectionString: URL }); + await pool.query("DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;"); + }); + afterAll(async () => { + await pool?.end(); + }); + + it("applies every migration, idempotently", async () => { + const db = createPgAdapter(pool); + const n = await runSelfHostMigrations(db, "migrations"); + expect(n).toBeGreaterThan(50); + expect(await runSelfHostMigrations(db, "migrations")).toBe(0); // idempotent + }); + + it("runs the translated query paths (INSERT OR REPLACE, datetime, json, COUNT→number)", async () => { + const db = createPgAdapter(pool); + // INSERT OR REPLACE → ON CONFLICT upsert (run twice; second must not error) + await db.prepare("INSERT OR REPLACE INTO system_flags (key, value, updated_at) VALUES (?, '1', CURRENT_TIMESTAMP)").bind("rag_enabled").run(); + await db.prepare("INSERT OR REPLACE INTO system_flags (key, value, updated_at) VALUES (?, '0', CURRENT_TIMESTAMP)").bind("rag_enabled").run(); + const flag = await db.prepare("SELECT value FROM system_flags WHERE key=?").bind("rag_enabled").first<{ value: string }>(); + expect(flag?.value).toBe("0"); // upserted + + // datetime('now', ?) compared against a TEXT timestamp column; COUNT(*) must come back as a number + const row = await db.prepare("SELECT COUNT(*) AS n FROM system_flags WHERE updated_at > datetime('now', ?)").bind("-30 days").first<{ n: number }>(); + expect(typeof row?.n).toBe("number"); + expect(row?.n).toBeGreaterThanOrEqual(1); + }); + + it("batch is transactional (rolls back on error)", async () => { + const db = createPgAdapter(pool); + await db.prepare("INSERT OR REPLACE INTO system_flags (key, value, updated_at) VALUES (?, 'x', CURRENT_TIMESTAMP)").bind("batch_probe").run(); + await expect( + db.batch([ + db.prepare("DELETE FROM system_flags WHERE key=?").bind("batch_probe"), + db.prepare("INSERT INTO system_flags (key, value) VALUES (?, ?) , bad-sql").bind("z", "1"), // syntax error → rollback + ]), + ).rejects.toThrow(); + const still = await db.prepare("SELECT COUNT(*) AS n FROM system_flags WHERE key=?").bind("batch_probe").first<{ n: number }>(); + expect(still?.n).toBe(1); // the DELETE rolled back + }); +}); diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts new file mode 100644 index 0000000000..6455e136a1 --- /dev/null +++ b/test/unit/selfhost-ai.test.ts @@ -0,0 +1,294 @@ +import { chmodSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { buildProvider, claudeErrorStatus, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, resolveModel } from "../../src/selfhost/ai"; + +describe("resolveModel (#979 — never leak the Workers-AI default to a self-host backend)", () => { + const WORKERS_DEFAULT = "@cf/meta/llama-3.1-8b-instruct-fp8-fast"; + it("operator-configured model wins over the core's Workers-AI id", () => { + expect(resolveModel("llama3.1", WORKERS_DEFAULT, "x")).toBe("llama3.1"); + }); + it("strips the Workers-AI id and falls back to the provider default", () => { + expect(resolveModel(undefined, WORKERS_DEFAULT, "sonnet")).toBe("sonnet"); + }); + it("passes through a real model the core supplied", () => { + expect(resolveModel(undefined, "gpt-4o", "sonnet")).toBe("gpt-4o"); + }); +}); + +afterEach(() => vi.unstubAllGlobals()); + +type SpawnResult = { stdout: string; code: number | null }; +type StubSpawn = (cmd: string, args: string[], opts: { env: Record; input?: string; timeoutMs: number }) => Promise; + +describe("createOpenAiCompatibleAi (#979)", () => { + it("POSTs to /chat/completions and returns { response }", async () => { + const calls: Array<{ url: string; body: { model: string } }> = []; + vi.stubGlobal("fetch", vi.fn(async (url: string, init: { body: string }) => { + calls.push({ url, body: JSON.parse(init.body) }); + return new Response(JSON.stringify({ choices: [{ message: { content: "hi there" } }] }), { status: 200 }); + })); + const ai = createOpenAiCompatibleAi({ baseUrl: "http://ollama:11434/v1/", apiKey: "k" }); + const out = await ai.run("llama3.1", { messages: [{ role: "user", content: "x" }], max_tokens: 100 }); + expect(out.response).toBe("hi there"); + const first = calls[0]; + expect(first?.url).toBe("http://ollama:11434/v1/chat/completions"); // trailing slash trimmed + expect(first?.body.model).toBe("llama3.1"); + }); + + it("throws on a non-OK response so the caller degrades", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response("err", { status: 500 }))); + await expect(createOpenAiCompatibleAi({ baseUrl: "http://x/v1" }).run("m", { prompt: "p" })).rejects.toThrow(/ai_http_500/); + }); + + it("routes an embedding request ({ text }) to /embeddings and returns { data }", async () => { + let url = ""; + vi.stubGlobal("fetch", vi.fn(async (u: string) => { + url = u; + return new Response(JSON.stringify({ data: [{ embedding: [0.1, 0.2] }, { embedding: [0.3, 0.4] }] }), { status: 200 }); + })); + const out = await createOpenAiCompatibleAi({ baseUrl: "http://o/v1", embedModel: "bge-m3" }).run("@cf/baai/bge-m3", { text: ["a", "b"] }); + expect(url).toBe("http://o/v1/embeddings"); + expect(out).toEqual({ data: [[0.1, 0.2], [0.3, 0.4]] }); + }); + + it("throws on a non-OK embeddings response", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response("e", { status: 502 }))); + await expect(createOpenAiCompatibleAi({ baseUrl: "http://x/v1" }).run("m", { text: ["a"] })).rejects.toThrow(/ai_embed_http_502/); + }); + + it("empty text array returns { data: [] } without a fetch", async () => { + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + const result = await createOpenAiCompatibleAi({ baseUrl: "http://o/v1" }).run("m", { text: [] }); + expect(result).toEqual({ data: [] }); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("undefined prompt falls back to empty string (toMessages ?? guard)", async () => { + let body: { messages: Array<{ role: string; content: string }> } | undefined; + vi.stubGlobal("fetch", vi.fn(async (_u: string, init: { body: string }) => { + body = JSON.parse(init.body) as { messages: Array<{ role: string; content: string }> }; + return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { status: 200 }); + })); + await createOpenAiCompatibleAi({ baseUrl: "http://o/v1" }).run("m", {}); + expect(body?.messages).toEqual([{ role: "user", content: "" }]); + }); +}); + +describe("createSelfHostAi — provider selection", () => { + it("is undefined when AI_PROVIDER is unset", () => { + expect(createSelfHostAi({})).toBeUndefined(); + }); + it("maps ollama/openai-compatible/claude-code/codex to adapters", () => { + expect(typeof createSelfHostAi({ AI_PROVIDER: "ollama", AI_BASE_URL: "http://o/v1" })?.run).toBe("function"); + expect(typeof createSelfHostAi({ AI_PROVIDER: "claude-code" })?.run).toBe("function"); + expect(typeof createSelfHostAi({ AI_PROVIDER: "codex" })?.run).toBe("function"); + expect(createSelfHostAi({ AI_PROVIDER: "nonsense" })).toBeUndefined(); + }); + it("anthropic requires a key; a comma-list builds a fallback chain", () => { + expect(createSelfHostAi({ AI_PROVIDER: "anthropic" })).toBeUndefined(); // no key → dropped + expect(typeof createSelfHostAi({ AI_PROVIDER: "anthropic", ANTHROPIC_API_KEY: "sk-ant" })?.run).toBe("function"); + // "anthropic,ollama" with a key → both build → a chain (a runnable adapter) + expect(typeof createSelfHostAi({ AI_PROVIDER: "anthropic,ollama", ANTHROPIC_API_KEY: "sk-ant" })?.run).toBe("function"); + }); +}); + +describe("createAnthropicAi (#979 native BYOK)", () => { + it("splits the system message and returns the joined text content", async () => { + let sent: { url: string; headers: Record; body: Record } | undefined; + vi.stubGlobal("fetch", vi.fn(async (url: string, init: { headers: Record; body: string }) => { + sent = { url, headers: init.headers, body: JSON.parse(init.body) as Record }; + return new Response(JSON.stringify({ content: [{ type: "text", text: "hi" }, { type: "thinking", text: "ignored" }] }), { status: 200 }); + })); + const out = await createAnthropicAi({ apiKey: "sk-ant", model: "claude-sonnet-4-6" }).run("@cf/ignored", { + messages: [ + { role: "system", content: "be terse" }, + { role: "user", content: "go" }, + ], + max_tokens: 256, + }); + expect(out.response).toBe("hi"); // only text blocks + expect(sent?.url).toBe("https://api.anthropic.com/v1/messages"); + expect(sent?.headers["x-api-key"]).toBe("sk-ant"); + expect(sent?.headers["anthropic-version"]).toBe("2023-06-01"); + expect(sent?.body.system).toBe("be terse"); + expect(sent?.body.model).toBe("claude-sonnet-4-6"); // configured wins over the @cf id + expect(sent?.body.messages).toEqual([{ role: "user", content: "go" }]); + }); + + it("throws on a non-OK response", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response("e", { status: 429 }))); + await expect(createAnthropicAi({ apiKey: "k" }).run("m", { prompt: "x" })).rejects.toThrow(/anthropic_http_429/); + }); +}); + +describe("createChainAi (fallback)", () => { + it("falls through to the next provider on failure, returns the first success", async () => { + const failing = { name: "a", ai: { run: async () => { throw new Error("down"); } } }; + const working = { name: "b", ai: { run: async () => ({ response: "from b" }) } }; + expect((await createChainAi([failing, working]).run("m", { prompt: "x" })).response).toBe("from b"); + }); + it("throws the last error when every provider fails", async () => { + const a = { name: "a", ai: { run: async () => { throw new Error("err-a"); } } }; + const b = { name: "b", ai: { run: async () => { throw new Error("err-b"); } } }; + await expect(createChainAi([a, b]).run("m", { prompt: "x" })).rejects.toThrow(/err-b/); + }); +}); + +describe("branch coverage — defaults + edge inputs", () => { + afterEach(() => vi.unstubAllGlobals()); + + it("chat with no apiKey + empty choices → empty response", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({ choices: [] }), { status: 200 }))); + expect((await createOpenAiCompatibleAi({ baseUrl: "http://o/v1" }).run("m", { prompt: "x" })).response).toBe(""); + }); + it("embed with no data field → empty data", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({}), { status: 200 }))); + expect((await createOpenAiCompatibleAi({ baseUrl: "http://o/v1" }).run("m", { text: ["a"] })).data).toEqual([]); + }); + it("anthropic with no system + missing/empty content → empty response", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({ content: [{ type: "text" }] }), { status: 200 }))); + expect((await createAnthropicAi({ apiKey: "k" }).run("m", { messages: [{ role: "user", content: "x" }] })).response).toBe(""); + }); + it("extractCliText: non-string result falls through to text", () => { + expect(extractCliText(JSON.stringify({ result: 5 }))).toBe(""); + expect(extractCliText(JSON.stringify({ text: "t" }))).toBe("t"); + }); + it("claudeErrorStatus: subtype + unknown fallbacks", () => { + expect(claudeErrorStatus(JSON.stringify({ is_error: true, subtype: "sub" }))).toBe("sub"); + expect(claudeErrorStatus(JSON.stringify({ is_error: true }))).toBe("unknown"); + }); + it("claude/codex with a null exit code", async () => { + const nullExit: StubSpawn = async () => ({ stdout: "", code: null }); + await expect(createClaudeCodeAi({ CLAUDE_CODE_OAUTH_TOKEN: "t" }, nullExit).run("m", { prompt: "x" })).rejects.toThrow(/claude_code_exit_null/); + await expect(createCodexAi({}, nullExit).run("m", { prompt: "x" })).rejects.toThrow(/codex_exit_null/); + }); + it("embed uses the bge-m3 default when no embedModel is set", async () => { + let sentModel = ""; + vi.stubGlobal("fetch", vi.fn(async (_u: string, init: { body: string }) => { + sentModel = JSON.parse(init.body).model; + return new Response(JSON.stringify({ data: [] }), { status: 200 }); + })); + await createOpenAiCompatibleAi({ baseUrl: "http://o/v1" }).run("m", { text: ["a"] }); + expect(sentModel).toBe("bge-m3"); + }); + it("anthropic with no content field → empty response", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({}), { status: 200 }))); + expect((await createAnthropicAi({ apiKey: "k" }).run("m", { prompt: "x" })).response).toBe(""); + }); + it("anthropic maps assistant-role messages to the 'assistant' role", async () => { + let sentMessages: Array<{ role: string; content: string }> | undefined; + vi.stubGlobal("fetch", vi.fn(async (_u: string, init: { body: string }) => { + sentMessages = (JSON.parse(init.body) as { messages: Array<{ role: string; content: string }> }).messages; + return new Response(JSON.stringify({ content: [{ type: "text", text: "hi" }] }), { status: 200 }); + })); + await createAnthropicAi({ apiKey: "k" }).run("m", { + messages: [ + { role: "assistant", content: "prior reply" }, + { role: "user", content: "follow-up" }, + ], + }); + expect(sentMessages).toEqual([ + { role: "assistant", content: "prior reply" }, + { role: "user", content: "follow-up" }, + ]); + }); + it("buildProvider uses provider-specific default base URLs when AI_BASE_URL is unset", () => { + expect(typeof buildProvider("openai", {})?.run).toBe("function"); // defaults to https://api.openai.com/v1 + expect(typeof buildProvider("ollama", {})?.run).toBe("function"); // defaults to http://localhost:11434/v1 + }); + it("extractCliText reads content + response fields", () => { + expect(extractCliText(JSON.stringify({ content: "c" }))).toBe("c"); + expect(extractCliText(JSON.stringify({ response: "r" }))).toBe("r"); + }); + it("chain wraps a non-Error throw", async () => { + const p = { + name: "p", + ai: { + run: async () => { + throw "stringerr"; + }, + }, + }; + await expect(createChainAi([p]).run("m", { prompt: "x" })).rejects.toThrow(/all_ai_providers_failed/); + }); +}); + +describe("subscription CLI helpers + fail-safe", () => { + it("extractCliText pulls the result/text field", () => { + expect(extractCliText(JSON.stringify({ type: "result", result: "ok" }))).toBe("ok"); + expect(extractCliText("")).toBe(""); + }); + it("claudeErrorStatus catches the is_error envelope", () => { + expect(claudeErrorStatus(JSON.stringify({ is_error: true, api_error_status: 401 }))).toBe("401"); + expect(claudeErrorStatus(JSON.stringify({ is_error: false, result: "ok" }))).toBeNull(); + }); + it("Claude Code fails SAFE on an is_error envelope (exits 0) instead of surfacing the error text", async () => { + const stub: StubSpawn = async () => ({ stdout: JSON.stringify({ is_error: true, api_error_status: 401, result: "Failed to authenticate" }), code: 0 }); + await expect(createClaudeCodeAi({ CLAUDE_CODE_OAUTH_TOKEN: "t" }, stub).run("m", { prompt: "x" })).rejects.toThrow(/claude_code_error_401/); + }); + it("Claude Code returns the model text on success and scrubs billable keys", async () => { + let capturedEnv: Record = {}; + const stub: StubSpawn = async (_c, _a, o) => { + capturedEnv = o.env; + return { stdout: JSON.stringify({ type: "result", result: "review text" }), code: 0 }; + }; + const out = await createClaudeCodeAi({ CLAUDE_CODE_OAUTH_TOKEN: "t", ANTHROPIC_API_KEY: "sk-bill" }, stub).run("sonnet", { prompt: "x" }); + expect(out.response).toBe("review text"); + expect(capturedEnv.ANTHROPIC_API_KEY).toBeUndefined(); // scrubbed + expect(capturedEnv.CLAUDE_CODE_OAUTH_TOKEN).toBe("t"); + }); + + it("Codex returns text on success and throws on a non-zero exit", async () => { + const ok: StubSpawn = async () => ({ stdout: JSON.stringify({ type: "result", result: "codex review" }), code: 0 }); + expect((await createCodexAi({}, ok).run("gpt-5", { prompt: "x" })).response).toBe("codex review"); + const bad: StubSpawn = async () => ({ stdout: "", code: 1 }); + await expect(createCodexAi({}, bad).run("gpt-5", { prompt: "x" })).rejects.toThrow(/codex_exit_1/); + }); + + it("drives the REAL subprocess (defaultSpawn) against a fake `claude` on PATH", async () => { + const dir = mkdtempSync(join(tmpdir(), "fakecli-")); + const fake = join(dir, "claude"); + // a minimal stand-in: read the prompt on stdin, emit a Claude-Code-shaped JSON result + writeFileSync(fake, "#!/usr/bin/env node\nlet i='';process.stdin.on('data',d=>i+=d);process.stdin.on('end',()=>process.stdout.write(JSON.stringify({type:'result',result:'OK:'+i.trim()})));\n"); + chmodSync(fake, 0o755); + const origPath = process.env.PATH; + process.env.PATH = `${dir}:${origPath ?? ""}`; + try { + const out = await createClaudeCodeAi({ ...process.env, CLAUDE_CODE_OAUTH_TOKEN: "t" }).run("sonnet", { prompt: "hello" }); + expect(out.response).toBe("OK:hello"); + } finally { + process.env.PATH = origPath; + } + }); + + it("Claude Code throws on no-token / non-zero exit / empty output", async () => { + await expect(createClaudeCodeAi({}).run("m", { prompt: "x" })).rejects.toThrow(/claude_code_no_oauth_token/); + const exit1: StubSpawn = async () => ({ stdout: "", code: 1 }); + await expect(createClaudeCodeAi({ CLAUDE_CODE_OAUTH_TOKEN: "t" }, exit1).run("m", { prompt: "x" })).rejects.toThrow(/claude_code_exit_1/); + const empty: StubSpawn = async () => ({ stdout: "", code: 0 }); + await expect(createClaudeCodeAi({ CLAUDE_CODE_OAUTH_TOKEN: "t" }, empty).run("m", { prompt: "x" })).rejects.toThrow(/claude_code_empty_output/); + }); + + it("Codex throws on empty output", async () => { + const empty: StubSpawn = async () => ({ stdout: "", code: 0 }); + await expect(createCodexAi({}, empty).run("gpt-5", { prompt: "x" })).rejects.toThrow(/codex_empty_output/); + }); + + it("defaultSpawn rejects when the CLI binary is missing (error handler)", async () => { + const origPath = process.env.PATH; + process.env.PATH = "/nonexistent-gittensory-empty"; + try { + await expect(createCodexAi({ ...process.env }).run("gpt-5", { prompt: "x" })).rejects.toThrow(); + } finally { + process.env.PATH = origPath; + } + }); + + it("extractCliText falls back to the last JSON line (JSONL) and is empty when none parse", () => { + expect(extractCliText('not json\n{"result":"x"}')).toBe("x"); + expect(extractCliText("not json\nstill not json")).toBe(""); + }); +}); diff --git a/test/unit/selfhost-audit.test.ts b/test/unit/selfhost-audit.test.ts new file mode 100644 index 0000000000..de9f99bfac --- /dev/null +++ b/test/unit/selfhost-audit.test.ts @@ -0,0 +1,70 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { logAudit, extractPayloadType } from "../../src/selfhost/audit"; + +describe("logAudit", () => { + const written: string[] = []; + + beforeEach(() => { + written.length = 0; + vi.spyOn(process.stdout, "write").mockImplementation((chunk) => { + written.push(String(chunk)); + return true; + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("emits a JSON line with level:audit for job_complete", () => { + logAudit({ event: "job_complete", ts: 1000, job_id: 1, payload_type: "review", latency_ms: 50, attempts: 1 }); + expect(written).toHaveLength(1); + const parsed = JSON.parse(written[0]!) as Record; + expect(parsed).toMatchObject({ level: "audit", event: "job_complete", ts: 1000, job_id: 1, payload_type: "review", latency_ms: 50, attempts: 1 }); + }); + + it("emits a JSON line for job_dead with error field", () => { + logAudit({ event: "job_dead", ts: 2000, job_id: "42", latency_ms: 100, attempts: 5, error: "boom" }); + const parsed = JSON.parse(written[0]!) as Record; + expect(parsed).toMatchObject({ level: "audit", event: "job_dead", error: "boom" }); + expect(parsed.payload_type).toBeUndefined(); + }); + + it("emits a JSON line for job_error", () => { + logAudit({ event: "job_error", ts: 3000, job_id: 2, latency_ms: 10, attempts: 2, error: "transient" }); + const parsed = JSON.parse(written[0]!) as Record; + expect(parsed.event).toBe("job_error"); + expect(parsed.level).toBe("audit"); + }); + + it("output ends with a newline", () => { + logAudit({ event: "job_complete", ts: 0, job_id: 0, latency_ms: 0, attempts: 1 }); + expect(written[0]!).toMatch(/\n$/); + }); +}); + +describe("extractPayloadType", () => { + it("returns the top-level type string", () => { + expect(extractPayloadType(JSON.stringify({ type: "review", other: 1 }))).toBe("review"); + }); + + it("returns undefined when type field is a number", () => { + expect(extractPayloadType(JSON.stringify({ type: 42 }))).toBeUndefined(); + }); + + it("returns undefined when type field is null", () => { + expect(extractPayloadType(JSON.stringify({ type: null }))).toBeUndefined(); + }); + + it("returns undefined when type field is absent", () => { + expect(extractPayloadType(JSON.stringify({ other: "x" }))).toBeUndefined(); + }); + + it("returns undefined for non-JSON input", () => { + expect(extractPayloadType("not-json")).toBeUndefined(); + }); + + it("returns undefined for an empty object", () => { + expect(extractPayloadType("{}")).toBeUndefined(); + }); +}); diff --git a/test/unit/selfhost-d1-adapter.test.ts b/test/unit/selfhost-d1-adapter.test.ts new file mode 100644 index 0000000000..a3248c6a31 --- /dev/null +++ b/test/unit/selfhost-d1-adapter.test.ts @@ -0,0 +1,73 @@ +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, it } from "vitest"; +import { createD1Adapter, nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; + +function makeD1(): D1Database { + const db = new DatabaseSync(":memory:"); + return createD1Adapter(nodeSqliteDriver(db as never)); +} + +describe("createD1Adapter (#980 self-host D1-over-SQLite)", () => { + it("implements the D1 surface faithfully: prepare/bind/all/first/raw on reads", async () => { + const d1 = makeD1(); + await d1.exec("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + await d1.prepare("INSERT INTO t (name) VALUES (?)").bind("a").run(); + await d1.prepare("INSERT INTO t (name) VALUES (?)").bind("b").run(); + + expect((await d1.prepare("SELECT count(*) AS n FROM t").first<{ n: number }>())?.n).toBe(2); + expect((await d1.prepare("SELECT name FROM t WHERE id = ?").bind(1).first<{ name: string }>())?.name).toBe("a"); + expect(await d1.prepare("SELECT name FROM t WHERE id = ?").bind(1).first("name")).toBe("a"); // colName form + expect(await d1.prepare("SELECT * FROM t WHERE id = 99").first()).toBeNull(); // no row → null + + const all = await d1.prepare("SELECT id, name FROM t ORDER BY id").all<{ id: number; name: string }>(); + expect(all.results).toEqual([{ id: 1, name: "a" }, { id: 2, name: "b" }]); + const raw = await d1.prepare("SELECT id, name FROM t ORDER BY id").raw(); + expect(raw).toEqual([[1, "a"], [2, "b"]]); // raw() = arrays of column values + }); + + it("run() reports changes/last_row_id; batch is atomic", async () => { + const d1 = makeD1(); + await d1.exec("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"); + const r = await d1.prepare("INSERT INTO t (name) VALUES (?)").bind("x").run(); + expect(r.meta.changes).toBe(1); + expect(r.meta.last_row_id).toBe(1); + + await d1.batch([ + d1.prepare("INSERT INTO t (name) VALUES (?)").bind("y"), + d1.prepare("INSERT INTO t (name) VALUES (?)").bind("z"), + ]); + expect((await d1.prepare("SELECT count(*) AS n FROM t").first<{ n: number }>())?.n).toBe(3); + }); + + it("batch rolls back entirely on an error (atomicity)", async () => { + const d1 = makeD1(); + await d1.exec("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT UNIQUE)"); + await d1.prepare("INSERT INTO t (name) VALUES (?)").bind("dup").run(); + await expect( + d1.batch([ + d1.prepare("INSERT INTO t (name) VALUES (?)").bind("ok"), + d1.prepare("INSERT INTO t (name) VALUES (?)").bind("dup"), // UNIQUE violation + ]), + ).rejects.toThrow(); + expect((await d1.prepare("SELECT count(*) AS n FROM t").first<{ n: number }>())?.n).toBe(1); // "ok" rolled back + }); + + it("dump() returns an ArrayBuffer (D1 surface completeness)", async () => { + expect(await makeD1().dump()).toBeInstanceOf(ArrayBuffer); + }); + + it("first(colName) returns the named column value", async () => { + const d1 = makeD1(); + await d1.exec("CREATE TABLE t (id INTEGER, x TEXT)"); + await d1.prepare("INSERT INTO t (id, x) VALUES (1, 'val')").run(); + expect(await d1.prepare("SELECT x FROM t").first("x")).toBe("val"); + expect(await d1.prepare("SELECT x FROM t WHERE id=99").first("x")).toBeNull(); // no row → null + }); + + it("first(colName) returns null when the row exists but the column value is NULL", async () => { + const d1 = makeD1(); + await d1.exec("CREATE TABLE t (id INTEGER, x TEXT)"); + await d1.prepare("INSERT INTO t (id, x) VALUES (1, NULL)").run(); + expect(await d1.prepare("SELECT x FROM t WHERE id=1").first("x")).toBeNull(); // row present, value is SQL NULL → null + }); +}); diff --git a/test/unit/selfhost-health.test.ts b/test/unit/selfhost-health.test.ts new file mode 100644 index 0000000000..add268dad0 --- /dev/null +++ b/test/unit/selfhost-health.test.ts @@ -0,0 +1,35 @@ +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, it } from "vitest"; +import { createD1Adapter, nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; +import { readiness } from "../../src/selfhost/health"; + +describe("readiness (#982)", () => { + it("is not ready until the migrations table has applied rows", async () => { + const driver = nodeSqliteDriver(new DatabaseSync(":memory:") as never); + const db = createD1Adapter(driver); + // db answers but no migrations table yet → not ready + expect(await readiness(db)).toEqual({ ok: false, checks: { db: true, migrations: false } }); + // empty migrations table → still not ready + driver.exec("CREATE TABLE _selfhost_migrations (name TEXT, applied_at INTEGER)"); + expect((await readiness(db)).ok).toBe(false); + // an applied migration → ready + driver.query("INSERT INTO _selfhost_migrations (name, applied_at) VALUES (?, ?)", ["0001", 0]); + expect(await readiness(db)).toEqual({ ok: true, checks: { db: true, migrations: true } }); + }); + + it("reports db=false and migrations=false when the SELECT 1 probe throws (db down)", async () => { + const throwingDb = { + prepare: () => ({ + bind: function() { return this; }, + first: () => Promise.reject(new Error("sqlite_io_error")), + all: () => Promise.reject(new Error("sqlite_io_error")), + run: () => Promise.reject(new Error("sqlite_io_error")), + raw: () => Promise.reject(new Error("sqlite_io_error")), + }), + exec: () => Promise.resolve({ results: [], success: true, meta: {} }), + batch: () => Promise.resolve([]), + dump: () => Promise.resolve(new ArrayBuffer(0)), + } as unknown as D1Database; + expect(await readiness(throwingDb)).toEqual({ ok: false, checks: { db: false, migrations: false } }); + }); +}); diff --git a/test/unit/selfhost-mcp-node.test.ts b/test/unit/selfhost-mcp-node.test.ts new file mode 100644 index 0000000000..7a14a06a08 --- /dev/null +++ b/test/unit/selfhost-mcp-node.test.ts @@ -0,0 +1,82 @@ +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { z } from "zod"; +import { describe, expect, it } from "vitest"; +import { createMcpHandler } from "../../src/selfhost/mcp-server-node"; + +const MCP_HEADERS = { "content-type": "application/json", accept: "application/json, text/event-stream" }; + +function makeMcpServer(): McpServer { + const server = new McpServer({ name: "test", version: "0.0.1" }); + server.registerTool("echo", { description: "Echoes the input", inputSchema: { value: z.string() } }, async ({ value }) => ({ + content: [{ type: "text" as const, text: value }], + })); + return server; +} + +function mcpPost(url: string, body: unknown): Request { + return new Request(url, { method: "POST", headers: MCP_HEADERS, body: JSON.stringify(body) }); +} + +describe("createMcpHandler (Node MCP port, #980)", () => { + it("OPTIONS → 204 with CORS headers", async () => { + const handler = createMcpHandler(makeMcpServer(), { enableJsonResponse: true }); + const res = await handler(new Request("http://localhost/mcp", { method: "OPTIONS" })); + expect(res.status).toBe(204); + expect(res.headers.get("access-control-allow-methods")).toContain("POST"); + }); + + it("POST initialize → 200 JSON with serverInfo", async () => { + const res = await createMcpHandler(makeMcpServer(), { enableJsonResponse: true })( + mcpPost("http://localhost/mcp", { jsonrpc: "2.0", id: 1, method: "initialize", params: { protocolVersion: "2024-11-05", capabilities: {}, clientInfo: { name: "test", version: "0.0.0" } } }), + ); + expect(res.status).toBe(200); + const json = (await res.json()) as { result?: { serverInfo?: { name: string } } }; + expect(json.result?.serverInfo?.name).toBe("test"); + }); + + it("POST tools/list → 200 with the registered echo tool", async () => { + const res = await createMcpHandler(makeMcpServer(), { enableJsonResponse: true })( + mcpPost("http://localhost/mcp", { jsonrpc: "2.0", id: 2, method: "tools/list", params: {} }), + ); + expect(res.status).toBe(200); + const json = (await res.json()) as { result?: { tools?: Array<{ name: string }> } }; + expect(json.result?.tools?.map((t) => t.name)).toContain("echo"); + }); + + it("POST tools/call → 200 with the echoed text", async () => { + const res = await createMcpHandler(makeMcpServer(), { enableJsonResponse: true })( + mcpPost("http://localhost/mcp", { jsonrpc: "2.0", id: 3, method: "tools/call", params: { name: "echo", arguments: { value: "hello from self-host" } } }), + ); + expect(res.status).toBe(200); + const json = (await res.json()) as { result?: { content?: Array<{ type: string; text: string }> } }; + expect(json.result?.content?.[0]?.text).toBe("hello from self-host"); + }); + + it("OPTIONS with Origin header echoes the origin in ACAO header", async () => { + const handler = createMcpHandler(makeMcpServer()); + const res = await handler(new Request("http://localhost/mcp", { + method: "OPTIONS", + headers: { origin: "https://example.com" }, + })); + expect(res.status).toBe(204); + expect(res.headers.get("access-control-allow-origin")).toBe("https://example.com"); + }); + + it("handler works without explicit opts (enableJsonResponse defaults to true)", async () => { + // createMcpHandler called with no second arg → opts = {} → enableJsonResponse ?? true + const handler = createMcpHandler(makeMcpServer()); + const res = await handler(mcpPost("http://localhost/mcp", { jsonrpc: "2.0", id: 99, method: "tools/list", params: {} })); + expect(res.status).toBe(200); + }); + + it("each invocation creates a fresh stateless session (no cross-request bleed)", async () => { + // Production code creates a fresh McpServer per request — simulate that here. + const listReq = () => mcpPost("http://localhost/mcp", { jsonrpc: "2.0", id: 4, method: "tools/list", params: {} }); + const [r1, r2] = await Promise.all([ + createMcpHandler(makeMcpServer(), { enableJsonResponse: true })(listReq()), + createMcpHandler(makeMcpServer(), { enableJsonResponse: true })(listReq()), + ]); + expect(r1.status).toBe(200); + expect(r2.status).toBe(200); + }); +}); diff --git a/test/unit/selfhost-metrics.test.ts b/test/unit/selfhost-metrics.test.ts new file mode 100644 index 0000000000..dad9907341 --- /dev/null +++ b/test/unit/selfhost-metrics.test.ts @@ -0,0 +1,38 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { gauge, incr, renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; + +afterEach(() => resetMetrics()); + +describe("metrics registry (#982)", () => { + it("counters accumulate and render", async () => { + incr("c_total"); + incr("c_total", undefined, 2); + expect((await renderMetrics())).toContain("c_total 3"); + }); + + it("renders labels in Prometheus format", async () => { + incr("h_total", { status: "ok" }); + expect((await renderMetrics())).toContain('h_total{status="ok"} 1'); + }); + + it("sorts multiple labels deterministically", async () => { + incr("m_total", { b: "2", a: "1" }); + expect((await renderMetrics())).toContain('m_total{a="1",b="2"} 1'); + }); + + it("gauges sample at scrape time", async () => { + let v = 5; + gauge("g", () => v); + expect((await renderMetrics())).toContain("g 5"); + v = 9; + expect((await renderMetrics())).toContain("g 9"); + }); + + it("a throwing gauge does not break the scrape", async () => { + gauge("bad", () => { + throw new Error("x"); + }); + incr("ok_total"); + expect((await renderMetrics())).toContain("ok_total 1"); + }); +}); diff --git a/test/unit/selfhost-migrate.test.ts b/test/unit/selfhost-migrate.test.ts new file mode 100644 index 0000000000..bf0215e68d --- /dev/null +++ b/test/unit/selfhost-migrate.test.ts @@ -0,0 +1,22 @@ +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, it } from "vitest"; +import { createD1Adapter, nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; +import { runSelfHostMigrations } from "../../src/selfhost/migrate"; + +describe("runSelfHostMigrations (#980)", () => { + it("applies un-applied migrations in order, idempotently", async () => { + const dir = mkdtempSync(join(tmpdir(), "gtmig-")); + writeFileSync(join(dir, "0001_a.sql"), "CREATE TABLE a (id INTEGER);"); + writeFileSync(join(dir, "0002_b.sql"), "CREATE TABLE b (id INTEGER);"); + const db = createD1Adapter(nodeSqliteDriver(new DatabaseSync(":memory:") as never)); + + expect(await runSelfHostMigrations(db, dir)).toBe(2); // both applied + expect(await runSelfHostMigrations(db, dir)).toBe(0); // idempotent — nothing re-applied + + writeFileSync(join(dir, "0003_c.sql"), "CREATE TABLE c (id INTEGER);"); + expect(await runSelfHostMigrations(db, dir)).toBe(1); // only the new one + }); +}); diff --git a/test/unit/selfhost-pg-dialect.test.ts b/test/unit/selfhost-pg-dialect.test.ts new file mode 100644 index 0000000000..c148661c4d --- /dev/null +++ b/test/unit/selfhost-pg-dialect.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; +import { toNumberedPlaceholders, translateDdl, translateFunctions, translateInsertOr, translateSql } from "../../src/selfhost/pg-dialect"; + +describe("pg-dialect (#977 SQLite → Postgres)", () => { + it("numbers placeholders, skipping `?` inside string literals", () => { + expect(toNumberedPlaceholders("SELECT * FROM t WHERE a=? AND b=?")).toBe("SELECT * FROM t WHERE a=$1 AND b=$2"); + expect(toNumberedPlaceholders("SELECT '?' AS lit WHERE a=?")).toBe("SELECT '?' AS lit WHERE a=$1"); + }); + + it("translates datetime/strftime/CURRENT_TIMESTAMP/json to Postgres (text-returning to match SQLite)", () => { + expect(translateFunctions("x > datetime('now', ?)")).toContain("to_char(now() + (?)::interval"); + expect(translateFunctions("datetime('now')")).toContain("to_char(now(),"); + expect(translateFunctions("strftime('%Y-W%W', created_at)")).toContain(`to_char((created_at)::timestamptz, 'YYYY"-W"WW')`); + expect(translateFunctions("strftime('%Y-%m', created_at)")).toContain("'YYYY-MM'"); + expect(translateFunctions("CURRENT_TIMESTAMP")).toContain("to_char(now(),"); + expect(translateFunctions("json_extract(meta, '$.mode')")).toBe("((meta)::jsonb ->> 'mode')"); + }); + + it("translates INSERT OR IGNORE / REPLACE to ON CONFLICT", () => { + expect(translateInsertOr("INSERT OR IGNORE INTO t (a) VALUES (?)")).toBe("INSERT INTO t (a) VALUES (?) ON CONFLICT DO NOTHING"); + const replace = translateInsertOr("INSERT OR REPLACE INTO system_flags (key, value, updated_at) VALUES (?, '1', CURRENT_TIMESTAMP)"); + expect(replace).toContain("INSERT INTO system_flags"); + expect(replace).toContain("ON CONFLICT (key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at"); + expect(() => translateInsertOr("INSERT OR REPLACE INTO unknown_tbl (a) VALUES (?)")).toThrow(/no known conflict key/); + expect(translateInsertOr("SELECT 1")).toBe("SELECT 1"); // passthrough + }); + + it("translateSql composes all passes; translateDdl handles the ISO-now default", () => { + expect(translateSql("SELECT * FROM t WHERE updated_at > datetime('now', ?)")).toMatch(/\$1/); + expect(translateDdl("created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))")).toContain("to_char(now() AT TIME ZONE 'UTC'"); + }); +}); diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts new file mode 100644 index 0000000000..d23a441d26 --- /dev/null +++ b/test/unit/selfhost-pg-queue.test.ts @@ -0,0 +1,216 @@ +// Unit tests for the Postgres-backed job queue (#977). Mocks pg.Pool so no real DB is needed. +// Real-Postgres integration paths (migrations, pg-adapter translation) live in test/integration/selfhost-pg.test.ts. +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { Pool, QueryResult } from "pg"; +import { createPgQueue } from "../../src/selfhost/pg-queue"; +import type { JobMessage } from "../../src/types"; + +const msg = (t: string): JobMessage => ({ type: t }) as unknown as JobMessage; +const typeOf = (m: JobMessage): string => (m as unknown as { type: string }).type; + +type MockFn = { mockResolvedValueOnce(v: unknown): void }; + +interface MockPool { + pool: Pool; + fn: MockFn; + enqueueResult(r: Partial): void; + /** Pre-load a job to be returned by the next RETURNING claim query. */ + enqueueJob(id: string, payload: object, attempts?: number): void; +} + +function makePool(): MockPool { + const results: Partial[] = []; + const fn = vi.fn().mockImplementation(async (sql: unknown) => { + const q = String(sql); + // Claim queries use RETURNING — pop from queue; fall through to empty default otherwise. + if (q.includes("RETURNING")) { + const next = results.shift(); + return next ?? { rows: [], rowCount: 0 }; + } + // COUNT queries need a c column. + if (q.includes("COUNT(*)")) { + return { rows: [{ c: "3" }], rowCount: 1 }; + } + return { rows: [], rowCount: 0 }; + }); + return { + pool: { query: fn } as unknown as Pool, + fn: fn as unknown as MockFn, + enqueueResult(r) { results.push(r); }, + enqueueJob(id, payload, attempts = 0) { + results.push({ rows: [{ id, payload: JSON.stringify(payload), attempts }], rowCount: 1 }); + }, + }; +} + +describe("createPgQueue (durable #977)", () => { + // Suppress audit log stdout noise in tests. + beforeEach(() => { vi.spyOn(process.stdout, "write").mockImplementation(() => true); }); + afterEach(() => { vi.restoreAllMocks(); }); + + it("init() creates the table and recovers stuck-processing jobs", async () => { + const m = makePool(); + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // DDL + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 2 }); // recovery UPDATE + const q = createPgQueue(m.pool, async () => undefined); + await q.init(); + expect(m.pool.query).toHaveBeenCalledTimes(2); + }); + + it("init() handles null rowCount from the recovery query (rowCount ?? 0 nullish arm)", async () => { + const m = makePool(); + m.fn.mockResolvedValueOnce({ rows: [], rowCount: 0 }); // DDL + // pg driver can return null for rowCount on some UPDATE results + m.fn.mockResolvedValueOnce({ rows: [], rowCount: null }); + const q = createPgQueue(m.pool, async () => undefined); + await q.init(); // rowCount=null → ?? 0 → 0 → no recovery log emitted + expect(m.pool.query).toHaveBeenCalledTimes(2); + }); + + it("processes a job successfully (job_complete audit emitted)", async () => { + const m = makePool(); + m.enqueueJob("1", { type: "review" }); + const seen: string[] = []; + const q = createPgQueue(m.pool, async (j) => void seen.push(typeOf(j))); + await q.init(); + await q.drain(); + expect(seen).toEqual(["review"]); + }); + + it("dead-letters an unparseable payload (job_dead audit emitted)", async () => { + const m = makePool(); + // Claim returns a row with bad payload. + m.enqueueResult({ rows: [{ id: "1", payload: "not-json", attempts: 0 }], rowCount: 1 }); + const q = createPgQueue(m.pool, async () => undefined, { maxRetries: 3 }); + await q.init(); + await q.drain(); + // UPDATE dead + then no more rows → pump exits cleanly. + expect(m.pool.query).toHaveBeenCalledWith(expect.stringContaining("status='dead'"), expect.arrayContaining(["1"])); + }); + + it("retries a failing job (job_error audit emitted) then dead-letters at maxRetries (job_dead)", async () => { + const m = makePool(); + // Two attempts: first → retry, second → dead-letter. + m.enqueueJob("1", { type: "t" }, 0); + m.enqueueJob("1", { type: "t" }, 1); // second claim after retry + let calls = 0; + const q = createPgQueue(m.pool, async () => { calls++; throw new Error("fail"); }, { maxRetries: 2, backoffMs: () => 0 }); + await q.init(); + await q.drain(); + await q.drain(); // second drain processes the retried job + expect(calls).toBe(2); + }); + + it("records 'unknown error' when consumer throws a non-Error", async () => { + const m = makePool(); + m.enqueueJob("1", { type: "t" }, 0); + const q = createPgQueue(m.pool, async () => { throw "plain-string"; }, { maxRetries: 1, backoffMs: () => 0 }); + await q.init(); + await q.drain(); + expect(m.pool.query).toHaveBeenCalledWith(expect.stringContaining("status='dead'"), expect.arrayContaining(["unknown error"])); + }); + + it("pump() returns early when active >= concurrency (saturation guard)", async () => { + let concurrent = 0; + let maxConcurrent = 0; + const m = makePool(); + m.enqueueJob("1", { type: "a" }); + m.enqueueJob("2", { type: "b" }); + const q = createPgQueue(m.pool, async () => { + concurrent++; + maxConcurrent = Math.max(maxConcurrent, concurrent); + await new Promise((r) => setTimeout(r, 15)); + concurrent--; + }, { concurrency: 1, pollIntervalMs: 100_000 }); + await q.init(); + await q.binding.send(msg("a")); + await q.binding.send(msg("b")); // second void pump() hits active >= 1 → returns early + await new Promise((r) => setTimeout(r, 60)); + await q.stop(); + expect(maxConcurrent).toBe(1); + }); + + it("concurrency=2 allows two jobs to run simultaneously", async () => { + let concurrent = 0; + let maxConcurrent = 0; + const m = makePool(); + m.enqueueJob("1", { type: "a" }); + m.enqueueJob("2", { type: "b" }); + const q = createPgQueue(m.pool, async () => { + concurrent++; + maxConcurrent = Math.max(maxConcurrent, concurrent); + await new Promise((r) => setTimeout(r, 15)); + concurrent--; + }, { concurrency: 2, pollIntervalMs: 100_000 }); + await q.init(); + await q.binding.send(msg("a")); + await q.binding.send(msg("b")); + await new Promise((r) => setTimeout(r, 60)); + await q.stop(); + expect(maxConcurrent).toBe(2); + }); + + it("start() and stop() run the poll loop", async () => { + const m = makePool(); + m.enqueueJob("1", { type: "ticked" }); + const seen: string[] = []; + const q = createPgQueue(m.pool, async (j) => void seen.push(typeOf(j)), { pollIntervalMs: 10 }); + await q.init(); + q.start(); + for (let i = 0; i < 50 && seen.length === 0; i++) await new Promise((r) => setTimeout(r, 10)); + await q.stop(); + expect(seen).toEqual(["ticked"]); + }); + + it("start() is idempotent", async () => { + const { pool } = makePool(); + const q = createPgQueue(pool, async () => undefined, { pollIntervalMs: 100_000 }); + await q.init(); + q.start(); + q.start(); // second call is a no-op + await q.stop(); + }); + + it("stop() is a no-op when timer is null", async () => { + const { pool } = makePool(); + const q = createPgQueue(pool, async () => undefined); + await q.init(); + await q.stop(); // timer=null → false branch of `if (timer) clearTimeout(timer)` + }); + + it("binding.sendBatch enqueues multiple messages", async () => { + const m = makePool(); + m.enqueueJob("1", { type: "x" }); + m.enqueueJob("2", { type: "y" }); + const seen: string[] = []; + const q = createPgQueue(m.pool, async (j) => void seen.push(typeOf(j))); + await q.init(); + await q.binding.sendBatch([{ body: msg("x") }, { body: msg("y") }]); + await q.drain(); + expect(seen.sort()).toEqual(["x", "y"]); + }); + + it("uses default backoff lambda when backoffMs is not provided", async () => { + // Trigger a retry without providing backoffMs so the default (attempt) => Math.min(60_000, 1000 * 2**attempt) + // is actually called — covering the function body that would otherwise be created but never invoked. + const m = makePool(); + m.enqueueJob("1", { type: "t" }, 0); + const q = createPgQueue(m.pool, async () => { throw new Error("transient"); }, { maxRetries: 5 }); + // No backoffMs → default lambda is used + called when scheduling the retry + await q.init(); + await q.drain(); + expect(m.pool.query).toHaveBeenCalledWith( + expect.stringContaining("status='pending'"), + expect.arrayContaining([1]), + ); + }); + + it("size() and deadCount() return numeric counts", async () => { + const { pool } = makePool(); + // makePool returns { c: "3" } for COUNT queries + const q = createPgQueue(pool, async () => undefined); + await q.init(); + expect(await q.size()).toBe(3); + expect(await q.deadCount()).toBe(3); + }); +}); diff --git a/test/unit/selfhost-pg-vectorize.test.ts b/test/unit/selfhost-pg-vectorize.test.ts new file mode 100644 index 0000000000..e87dfdf0c1 --- /dev/null +++ b/test/unit/selfhost-pg-vectorize.test.ts @@ -0,0 +1,102 @@ +// Unit tests for pg-vectorize (#980 pgvector RAG). Uses a mock pg Pool so no real Postgres is required. +// The integration path (initPgVectorize + real Postgres) is covered by selfhost-pg-queue.test.ts (which +// already spins up Postgres in CI via the pg integration harness). +import { describe, expect, it, vi, beforeEach } from "vitest"; +import { createPgVectorize, initPgVectorize } from "../../src/selfhost/pg-vectorize"; +import type { Pool } from "pg"; + +/** A minimal Pool mock that records queries and returns configurable rows. */ +function makePool(rows: Record[] = []): Pool { + const mock = { + _queries: [] as Array<{ sql: string; params: unknown[] }>, + async query(sql: string, params: unknown[] = []) { + mock._queries.push({ sql: String(sql), params }); + return { rows, rowCount: rows.length }; + }, + }; + return mock as unknown as Pool; +} + +describe("initPgVectorize (#980)", () => { + it("runs CREATE EXTENSION and CREATE TABLE at startup", async () => { + const pool = makePool(); + await initPgVectorize(pool); + const sqls = (pool as unknown as { _queries: Array<{ sql: string }> })._queries.map((q) => q.sql); + expect(sqls.some((s) => s.includes("CREATE EXTENSION IF NOT EXISTS vector"))).toBe(true); + expect(sqls.some((s) => s.includes("CREATE TABLE IF NOT EXISTS"))).toBe(true); + }); +}); + +describe("createPgVectorize (#980 pgvector RAG)", () => { + let pool: Pool & { _queries: Array<{ sql: string; params: unknown[] }> }; + beforeEach(() => { + pool = makePool() as unknown as Pool & { _queries: Array<{ sql: string; params: unknown[] }> }; + }); + + it("upsert generates INSERT … ON CONFLICT with vector literal", async () => { + const v = createPgVectorize(pool); + // pg-vectorize is cast `as unknown as Vectorize` — read internal shape via unknown cast + await v.upsert([{ id: "v1", values: [0.1, 0.2], namespace: "repo1", metadata: { path: "a.ts" } }]); + const q = pool._queries[0]; + expect(q?.sql).toContain("ON CONFLICT(id)"); + expect(q?.sql).toContain("::vector"); + expect(q?.params[0]).toBe("v1"); + expect(q?.params[1]).toBe("repo1"); + expect(q?.params[2]).toBe("[0.1,0.2]"); + }); + + it("upsert uses empty-string namespace when namespace is absent", async () => { + const v = createPgVectorize(pool); + await v.upsert([{ id: "ns-less", values: [1, 0] }]); + expect(pool._queries[0]?.params[1]).toBe(""); + }); + + it("query with namespace adds WHERE namespace= clause", async () => { + const matchPool = makePool([{ id: "v1", score: 0.95, metadata: null }]); + const v = createPgVectorize(matchPool); + const { matches } = await v.query([0.1, 0.2], { topK: 3, namespace: "n1" }); + expect(matches).toHaveLength(1); + expect(matches[0]?.id).toBe("v1"); + expect(matches[0]?.score).toBeCloseTo(0.95); + const q = (matchPool as unknown as { _queries: Array<{ sql: string }> })._queries[0]; + expect(q?.sql).toContain("namespace=$2"); + }); + + it("query without namespace omits the WHERE clause", async () => { + const matchPool = makePool([{ id: "v2", score: 0.8, metadata: null }]); + const v = createPgVectorize(matchPool); + await v.query([0.1, 0.2], { topK: 5 }); + const q = (matchPool as unknown as { _queries: Array<{ sql: string }> })._queries[0]; + expect(q?.sql).not.toContain("namespace="); + }); + + it("query without topK uses the default of 12", async () => { + const matchPool = makePool([{ id: "v4", score: 0.6, metadata: null }]); + const v = createPgVectorize(matchPool); + await v.query([1, 0], {}); // topK omitted → default 12 + const q = (matchPool as unknown as { _queries: Array<{ sql: string; params: unknown[] }> })._queries[0]; + // The LIMIT param should be 12 (the default) + expect(q?.params).toContain(12); + }); + + it("query maps metadata JSONB rows to Match.metadata", async () => { + const matchPool = makePool([{ id: "v3", score: 0.7, metadata: { path: "x.ts" } }]); + const v = createPgVectorize(matchPool); + const { matches } = await v.query([1, 0], { topK: 1, namespace: "n" }); + expect(matches[0]?.metadata?.path).toBe("x.ts"); + }); + + it("deleteByIds with ids sends DELETE … IN (…) with placeholders", async () => { + const v = createPgVectorize(pool); + await v.deleteByIds(["a", "b", "c"]); + const q = pool._queries[0]; + expect(q?.sql).toContain("IN ($1,$2,$3)"); + expect(q?.params).toEqual(["a", "b", "c"]); + }); + + it("deleteByIds with empty array is a no-op (no query issued)", async () => { + const v = createPgVectorize(pool); + await v.deleteByIds([]); + expect(pool._queries).toHaveLength(0); + }); +}); diff --git a/test/unit/selfhost-puppeteer-stub.test.ts b/test/unit/selfhost-puppeteer-stub.test.ts new file mode 100644 index 0000000000..c9a0ad9a4c --- /dev/null +++ b/test/unit/selfhost-puppeteer-stub.test.ts @@ -0,0 +1,38 @@ +// Tests for the self-host puppeteer stub (#980). Verifies the stub throws the right error when +// BROWSER_WS_ENDPOINT is absent and delegates to puppeteer-core when present. +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +describe("selfhost puppeteer stub (#980 visual review)", () => { + let origEndpoint: string | undefined; + beforeEach(() => { origEndpoint = process.env.BROWSER_WS_ENDPOINT; }); + afterEach(() => { + if (origEndpoint === undefined) delete process.env.BROWSER_WS_ENDPOINT; + else process.env.BROWSER_WS_ENDPOINT = origEndpoint; + vi.resetModules(); + }); + + it("launch() throws browser_rendering_unavailable when BROWSER_WS_ENDPOINT is not set", async () => { + delete process.env.BROWSER_WS_ENDPOINT; + const { default: puppeteer } = await import("../../src/selfhost/stubs/puppeteer"); + await expect(puppeteer.launch({})).rejects.toThrow(/browser_rendering_unavailable_on_selfhost/); + }); + + it("connect() throws browser_rendering_unavailable when BROWSER_WS_ENDPOINT is not set", async () => { + delete process.env.BROWSER_WS_ENDPOINT; + const { default: puppeteer } = await import("../../src/selfhost/stubs/puppeteer"); + await expect(puppeteer.connect({})).rejects.toThrow(/browser_rendering_unavailable_on_selfhost/); + }); + + it("launch() throws browser_rendering_unavailable when puppeteer-core is not installed", async () => { + process.env.BROWSER_WS_ENDPOINT = "ws://fake:3000"; + // puppeteer-core is not in this repo's dependencies — the dynamic import naturally throws "Cannot find package". + const { default: puppeteer } = await import("../../src/selfhost/stubs/puppeteer"); + await expect(puppeteer.launch({})).rejects.toThrow(/browser_rendering_unavailable_on_selfhost/); + }); + + it("connect() throws browser_rendering_unavailable when puppeteer-core is not installed", async () => { + process.env.BROWSER_WS_ENDPOINT = "ws://fake:3000"; + const { default: puppeteer } = await import("../../src/selfhost/stubs/puppeteer"); + await expect(puppeteer.connect({})).rejects.toThrow(/browser_rendering_unavailable_on_selfhost/); + }); +}); diff --git a/test/unit/selfhost-qdrant-vectorize.test.ts b/test/unit/selfhost-qdrant-vectorize.test.ts new file mode 100644 index 0000000000..bbfa0ec161 --- /dev/null +++ b/test/unit/selfhost-qdrant-vectorize.test.ts @@ -0,0 +1,241 @@ +import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; +import { createQdrantVectorize, initQdrantCollection } from "../../src/selfhost/qdrant-vectorize"; +import { resetMetrics, renderMetrics } from "../../src/selfhost/metrics"; + +const BASE = "http://qdrant:6333"; + +/** Build a fake fetch that returns the given response for any call. */ +function mockFetch(status: number, body: unknown = {}) { + return vi.fn(async () => new Response(JSON.stringify(body), { status })); +} + +describe("initQdrantCollection (#1217)", () => { + afterEach(() => { vi.restoreAllMocks(); resetMetrics(); }); + + it("PUTs to /collections/ with cosine + size params", async () => { + const fake = mockFetch(200); + vi.stubGlobal("fetch", fake); + await initQdrantCollection(BASE); + expect(fake).toHaveBeenCalledOnce(); + const [url, init] = fake.mock.calls[0] as unknown as [string, RequestInit]; + expect(url).toBe(`${BASE}/collections/gittensory`); + const body = JSON.parse(init.body as string) as { vectors: { size: number; distance: string } }; + expect(body.vectors.distance).toBe("Cosine"); + expect(body.vectors.size).toBe(1024); + }); + + it("ignores a 409 (collection already exists)", async () => { + vi.stubGlobal("fetch", mockFetch(409)); + await expect(initQdrantCollection(BASE)).resolves.not.toThrow(); + }); + + it("throws on any other non-OK status", async () => { + vi.stubGlobal("fetch", mockFetch(500, { error: "server error" })); + await expect(initQdrantCollection(BASE)).rejects.toThrow(/HTTP 500/); + }); + + it("uses a custom collection name and dimension when provided", async () => { + const fake = mockFetch(200); + vi.stubGlobal("fetch", fake); + await initQdrantCollection(BASE, "custom-col", 768); + const [url, init] = fake.mock.calls[0] as unknown as [string, RequestInit]; + expect(url).toContain("custom-col"); + expect((JSON.parse(init.body as string) as { vectors: { size: number } }).vectors.size).toBe(768); + }); +}); + +describe("initQdrantCollection — QDRANT_API_KEY header", () => { + afterEach(() => { vi.restoreAllMocks(); delete process.env.QDRANT_API_KEY; }); + + it("includes api-key header when QDRANT_API_KEY is set", async () => { + process.env.QDRANT_API_KEY = "secret-key"; + const fake = mockFetch(200); + vi.stubGlobal("fetch", fake); + await initQdrantCollection(BASE); + const init = (fake.mock.calls[0] as unknown as [string, RequestInit])[1]; + expect((init.headers as Record)["api-key"]).toBe("secret-key"); + }); + + it("omits api-key header when QDRANT_API_KEY is unset", async () => { + delete process.env.QDRANT_API_KEY; + const fake = mockFetch(200); + vi.stubGlobal("fetch", fake); + await initQdrantCollection(BASE); + const init = (fake.mock.calls[0] as unknown as [string, RequestInit])[1]; + expect((init.headers as Record)["api-key"]).toBeUndefined(); + }); +}); + +describe("createQdrantVectorize (#1217 Qdrant adapter)", () => { + beforeEach(() => { vi.restoreAllMocks(); resetMetrics(); }); + + // ── upsert ──────────────────────────────────────────────────────────────── + + it("upsert PUTs points with uuid-mapped IDs and payload including _orig_id + namespace", async () => { + const fake = mockFetch(200, { status: "ok" }); + vi.stubGlobal("fetch", fake); + const v = createQdrantVectorize(BASE); + const result = await v.upsert([{ id: "repo/file:1", values: [0.1, 0.2], namespace: "ns1", metadata: { path: "a.ts" } }]); + expect(result).toEqual({ count: 1, ids: ["repo/file:1"] }); + const [url, init] = fake.mock.calls[0] as unknown as [string, RequestInit]; + expect(url).toContain("/points"); + const body = JSON.parse(init.body as string) as { points: Array<{ id: string; payload: { _orig_id: string; namespace: string } }> }; + expect(body.points[0]?.payload._orig_id).toBe("repo/file:1"); + expect(body.points[0]?.payload.namespace).toBe("ns1"); + // UUID must match the pattern xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + expect(body.points[0]?.id).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/); + }); + + it("upsert defaults namespace to empty string when absent", async () => { + vi.stubGlobal("fetch", mockFetch(200)); + const v = createQdrantVectorize(BASE); + await v.upsert([{ id: "no-ns", values: [1, 0] }]); + const init = (vi.mocked(fetch).mock.calls[0] as unknown as [string, RequestInit])[1]; + const body = JSON.parse(init.body as string) as { points: Array<{ payload: { namespace: string } }> }; + expect(body.points[0]?.payload.namespace).toBe(""); + }); + + it("upsert throws on a non-OK response and increments error counter", async () => { + vi.stubGlobal("fetch", mockFetch(503)); + const v = createQdrantVectorize(BASE); + await expect(v.upsert([{ id: "x", values: [1] }])).rejects.toThrow(/HTTP 503/); + expect(await renderMetrics()).toContain('gittensory_qdrant_errors_total{op="upsert"}'); + }); + + it("successful upsert increments gittensory_qdrant_upserts_total by vector count", async () => { + vi.stubGlobal("fetch", mockFetch(200)); + const v = createQdrantVectorize(BASE); + await v.upsert([{ id: "a", values: [1] }, { id: "b", values: [0] }]); + const metrics = await renderMetrics(); + expect(metrics).toMatch(/gittensory_qdrant_upserts_total 2/); + }); + + it("same string ID always produces the same UUID (deterministic mapping)", async () => { + vi.stubGlobal("fetch", mockFetch(200)); + const v = createQdrantVectorize(BASE); + await v.upsert([{ id: "stable-id", values: [1] }]); + const body1 = JSON.parse(((vi.mocked(fetch).mock.calls[0] as unknown as [string, RequestInit])[1].body) as string) as { points: Array<{ id: string }> }; + vi.mocked(fetch).mockClear(); + await v.upsert([{ id: "stable-id", values: [1] }]); + const body2 = JSON.parse(((vi.mocked(fetch).mock.calls[0] as unknown as [string, RequestInit])[1].body) as string) as { points: Array<{ id: string }> }; + expect(body1.points[0]?.id).toBe(body2.points[0]?.id); + }); + + // ── query ───────────────────────────────────────────────────────────────── + + it("query POSTs a search request with namespace filter and returns matches with _orig_id restored", async () => { + const qdrantResponse = { + result: [{ id: "some-uuid", score: 0.92, payload: { _orig_id: "repo/f:1", namespace: "ns", path: "f.ts" } }], + }; + vi.stubGlobal("fetch", mockFetch(200, qdrantResponse)); + const v = createQdrantVectorize(BASE); + const { matches } = await v.query([0.5, 0.5], { topK: 5, namespace: "ns" }); + expect(matches).toHaveLength(1); + expect(matches[0]?.id).toBe("repo/f:1"); // _orig_id restored + expect(matches[0]?.score).toBeCloseTo(0.92); + expect(matches[0]?.metadata?.path).toBe("f.ts"); + const init = (vi.mocked(fetch).mock.calls[0] as unknown as [string, RequestInit])[1]; + const body = JSON.parse(init.body as string) as { filter?: { must: Array<{ key: string; match: { value: string } }> } }; + expect(body.filter?.must[0]?.key).toBe("namespace"); + expect(body.filter?.must[0]?.match.value).toBe("ns"); + }); + + it("query without namespace sends no filter", async () => { + vi.stubGlobal("fetch", mockFetch(200, { result: [] })); + const v = createQdrantVectorize(BASE); + await v.query([1, 0], { topK: 10 }); + const body = JSON.parse(((vi.mocked(fetch).mock.calls[0] as unknown as [string, RequestInit])[1].body) as string) as { filter?: unknown }; + expect(body.filter).toBeUndefined(); + }); + + it("query defaults topK to 12 when omitted", async () => { + vi.stubGlobal("fetch", mockFetch(200, { result: [] })); + const v = createQdrantVectorize(BASE); + await v.query([1, 0], {}); + const body = JSON.parse(((vi.mocked(fetch).mock.calls[0] as unknown as [string, RequestInit])[1].body) as string) as { limit: number }; + expect(body.limit).toBe(12); + }); + + it("query returns empty matches when Qdrant is unreachable (network error) and tracks error", async () => { + vi.stubGlobal("fetch", vi.fn(async () => { throw new Error("ECONNREFUSED"); })); + const v = createQdrantVectorize(BASE); + const { matches } = await v.query([1, 0], { topK: 5 }); + expect(matches).toEqual([]); + expect(await renderMetrics()).toContain('gittensory_qdrant_errors_total{op="query"}'); + }); + + it("query returns empty matches on non-OK HTTP response (graceful degrade) and tracks error", async () => { + vi.stubGlobal("fetch", mockFetch(503)); + const v = createQdrantVectorize(BASE); + const { matches } = await v.query([1, 0], { topK: 5 }); + expect(matches).toEqual([]); + expect(await renderMetrics()).toContain('gittensory_qdrant_errors_total{op="query"}'); + }); + + it("successful query increments gittensory_qdrant_queries_total", async () => { + vi.stubGlobal("fetch", mockFetch(200, { result: [] })); + const v = createQdrantVectorize(BASE); + await v.query([1], {}); + await v.query([0], {}); + expect(await renderMetrics()).toMatch(/gittensory_qdrant_queries_total 2/); + }); + + it("query returns match without metadata when payload has no extra fields", async () => { + const qdrantResponse = { + result: [{ id: "uuid-1", score: 0.8, payload: { _orig_id: "plain-id", namespace: "n" } }], + }; + vi.stubGlobal("fetch", mockFetch(200, qdrantResponse)); + const v = createQdrantVectorize(BASE); + const { matches } = await v.query([1], {}); + expect(matches[0]).toEqual({ id: "plain-id", score: 0.8 }); + expect(matches[0]?.metadata).toBeUndefined(); + }); + + it("query falls back to the Qdrant UUID when _orig_id is missing from payload", async () => { + const qdrantResponse = { + result: [{ id: "fallback-uuid", score: 0.5, payload: { namespace: "n" } }], + }; + vi.stubGlobal("fetch", mockFetch(200, qdrantResponse)); + const v = createQdrantVectorize(BASE); + const { matches } = await v.query([1], {}); + expect(matches[0]?.id).toBe("fallback-uuid"); + }); + + // ── deleteByIds ─────────────────────────────────────────────────────────── + + it("deleteByIds POSTs the uuid-mapped IDs and returns the count", async () => { + vi.stubGlobal("fetch", mockFetch(200, { status: "ok" })); + const v = createQdrantVectorize(BASE); + const result = await v.deleteByIds(["id-1", "id-2"]); + expect(result).toEqual({ count: 2 }); + const init = (vi.mocked(fetch).mock.calls[0] as unknown as [string, RequestInit])[1]; + const body = JSON.parse(init.body as string) as { points: string[] }; + expect(body.points).toHaveLength(2); + body.points.forEach((p) => expect(p).toMatch(/^[0-9a-f]{8}-/)); + }); + + it("deleteByIds is a no-op for an empty array (no fetch call)", async () => { + const fake = mockFetch(200); + vi.stubGlobal("fetch", fake); + const v = createQdrantVectorize(BASE); + const result = await v.deleteByIds([]); + expect(result).toEqual({ count: 0 }); + expect(fake).not.toHaveBeenCalled(); + }); + + it("deleteByIds throws on a non-OK response and tracks error", async () => { + vi.stubGlobal("fetch", mockFetch(400)); + const v = createQdrantVectorize(BASE); + await expect(v.deleteByIds(["id"])).rejects.toThrow(/HTTP 400/); + expect(await renderMetrics()).toContain('gittensory_qdrant_errors_total{op="delete"}'); + }); + + it("trailing slash in URL is stripped", async () => { + const fake = mockFetch(200, { result: [] }); + vi.stubGlobal("fetch", fake); + const v = createQdrantVectorize("http://qdrant:6333/"); + await v.query([1], {}); + const [url] = fake.mock.calls[0] as unknown as [string]; + expect(url).not.toContain("//collections"); + }); +}); diff --git a/test/unit/selfhost-redis-cache.test.ts b/test/unit/selfhost-redis-cache.test.ts new file mode 100644 index 0000000000..d350afb5cf --- /dev/null +++ b/test/unit/selfhost-redis-cache.test.ts @@ -0,0 +1,78 @@ +import type { Redis } from "ioredis"; +import { describe, expect, it } from "vitest"; +import { checkAndMarkDelivery, createRedisCache } from "../../src/selfhost/redis-cache"; + +/** Minimal in-memory stand-in for the ioredis methods the cache uses. */ +function fakeRedis(): Redis & { _store: Map } { + const _store = new Map(); + return { + _store, + async get(k: string) { + return _store.get(k) ?? null; + }, + async set(k: string, v: string, _ex: "EX", _ttl: number) { + _store.set(k, v); + return "OK"; + }, + async del(k: string) { + _store.delete(k); + return 1; + }, + } as unknown as Redis & { _store: Map }; +} + +describe("createRedisCache (#1216 webhook dedup cache)", () => { + it("get returns null for a missing key", async () => { + const cache = createRedisCache(fakeRedis()); + expect(await cache.get("missing")).toBeNull(); + }); + + it("set then get returns the stored value", async () => { + const cache = createRedisCache(fakeRedis()); + await cache.set("k", "hello", 60); + expect(await cache.get("k")).toBe("hello"); + }); + + it("del removes the key", async () => { + const r = fakeRedis(); + const cache = createRedisCache(r); + await cache.set("k", "v", 60); + await cache.del("k"); + expect(await cache.get("k")).toBeNull(); + }); +}); + +describe("checkAndMarkDelivery (#1216 webhook idempotency)", () => { + it("returns false (first-time) for a new delivery ID and marks it as seen", async () => { + const cache = createRedisCache(fakeRedis()); + const result = await checkAndMarkDelivery(cache, "delivery-abc", 300); + expect(result).toBe(false); + // second call with the same ID should be a duplicate + const duplicate = await checkAndMarkDelivery(cache, "delivery-abc", 300); + expect(duplicate).toBe(true); + }); + + it("returns true (duplicate) for an already-seen delivery ID", async () => { + const r = fakeRedis(); + r._store.set("delivery:existing-id", "1"); + const cache = createRedisCache(r); + expect(await checkAndMarkDelivery(cache, "existing-id")).toBe(true); + }); + + it("different delivery IDs are tracked independently", async () => { + const cache = createRedisCache(fakeRedis()); + expect(await checkAndMarkDelivery(cache, "id-A")).toBe(false); + expect(await checkAndMarkDelivery(cache, "id-B")).toBe(false); // different ID → first-time + expect(await checkAndMarkDelivery(cache, "id-A")).toBe(true); // id-A seen before + }); + + it("swallows Redis errors and returns false (never blocks processing)", async () => { + const brokenRedis = { + async get() { throw new Error("connection refused"); }, + async set() { throw new Error("connection refused"); }, + } as unknown as Redis; + const cache = createRedisCache(brokenRedis); + // Must not throw — error is swallowed, returns false (first-time / let it through) + expect(await checkAndMarkDelivery(cache, "any-id")).toBe(false); + }); +}); diff --git a/test/unit/selfhost-redis-ratelimit.test.ts b/test/unit/selfhost-redis-ratelimit.test.ts new file mode 100644 index 0000000000..62131ad83e --- /dev/null +++ b/test/unit/selfhost-redis-ratelimit.test.ts @@ -0,0 +1,64 @@ +import type { Redis } from "ioredis"; +import { describe, expect, it } from "vitest"; +import { createRedisRateLimiter } from "../../src/selfhost/redis-ratelimit"; + +/** Minimal in-memory stand-in for the ioredis methods the limiter uses. */ +function fakeRedis(): Redis { + const store = new Map(); + return { + async incr(k: string) { + const v = (store.get(k) ?? 0) + 1; + store.set(k, v); + return v; + }, + async expire() { + return 1; + }, + async pttl() { + return 30_000; + }, + } as unknown as Redis; +} + +describe("createRedisRateLimiter (#977)", () => { + it("allows up to the limit then 429s, exposing a decision", async () => { + const ns = createRedisRateLimiter(fakeRedis()); + const stub = ns.get(ns.idFromName("k")); + const hit = () => stub.fetch("https://rl/check", { method: "POST", body: JSON.stringify({ key: "k", limit: 2, windowSeconds: 60 }) }); + + let res = await hit(); + expect(res.status).toBe(200); + expect(((await res.json()) as { remaining: number }).remaining).toBe(1); + res = await hit(); + expect(res.status).toBe(200); // count 2 == limit → still allowed + res = await hit(); + expect(res.status).toBe(429); // count 3 > limit → blocked + const blocked = (await res.json()) as { allowed: boolean; retryAfterSeconds: number }; + expect(blocked.allowed).toBe(false); + expect(blocked.retryAfterSeconds).toBeGreaterThan(0); + }); + + it("400s on a malformed request", async () => { + const ns = createRedisRateLimiter(fakeRedis()); + const res = await ns.get(ns.idFromName("k")).fetch("https://rl/check", { method: "POST", body: JSON.stringify({}) }); + expect(res.status).toBe(400); + }); + + it("accepts a Request object and handles a missing TTL", async () => { + const noTtl = { + async incr() { + return 1; + }, + async expire() { + return 1; + }, + async pttl() { + return -1; // no expiry set → resetMs falls back to windowSeconds + }, + } as unknown as Redis; + const ns = createRedisRateLimiter(noTtl); + const req = new Request("https://rl/check", { method: "POST", body: JSON.stringify({ key: "k", limit: 5, windowSeconds: 60 }) }); + const res = await ns.get(ns.idFromName("k")).fetch(req); // pass a Request (not url+init) + expect(res.status).toBe(200); + }); +}); diff --git a/test/unit/selfhost-setup-wizard.test.ts b/test/unit/selfhost-setup-wizard.test.ts new file mode 100644 index 0000000000..2e24c197e7 --- /dev/null +++ b/test/unit/selfhost-setup-wizard.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it, vi } from "vitest"; +import { buildManifest, credentialsToEnv, exchangeManifestCode, renderSetupPage } from "../../src/selfhost/setup-wizard"; + +describe("setup-wizard (#981 GitHub App Manifest)", () => { + it("builds a manifest with the webhook + redirect URLs (including CSRF state), permissions, events", () => { + const m = buildManifest("https://gt.example.com/", "test-state-123"); + expect(m.url).toBe("https://gt.example.com"); // trailing slash trimmed + expect((m.hook_attributes as { url: string }).url).toBe("https://gt.example.com/v1/github/webhook"); + expect(m.redirect_url).toBe("https://gt.example.com/setup/callback?state=test-state-123"); + expect((m.default_permissions as Record).pull_requests).toBe("write"); + expect(m.default_events).toContain("pull_request"); + }); + + it("encodes special characters in the state parameter", () => { + const m = buildManifest("https://gt.example.com", "a b+c=d&e"); + expect(m.redirect_url).toContain("state=a%20b%2Bc%3Dd%26e"); + }); + + it("renders a form that POSTs the manifest to GitHub with the CSRF state embedded", () => { + const html = renderSetupPage("https://gt.example.com", "nonce-abc"); + expect(html).toContain('action="https://github.com/settings/apps/new"'); + expect(html).toContain('name="manifest"'); + expect(html).toContain("Gittensory Self-Host"); + expect(html).toContain("nonce-abc"); // state is baked into the manifest value + }); + + it("exchanges the code and serializes credentials to .env lines", async () => { + const fakeFetch = vi.fn( + async () => + new Response(JSON.stringify({ id: 42, slug: "gt-sh", webhook_secret: "whsec", pem: "-----BEGIN-----\nk\n-----END-----", client_id: "cid", client_secret: "csec" }), { status: 200 }), + ) as unknown as typeof fetch; + const creds = await exchangeManifestCode("the-code", fakeFetch); + expect(creds.id).toBe(42); + const env = credentialsToEnv(creds); + expect(env).toContain("GITHUB_APP_ID=42"); + expect(env).toContain("GITHUB_APP_SLUG=gt-sh"); + expect(env).toContain("GITHUB_WEBHOOK_SECRET=whsec"); + expect(env).toContain("GITHUB_OAUTH_CLIENT_ID=cid"); + expect(env).toMatch(/GITHUB_APP_PRIVATE_KEY=".*BEGIN/); + }); + + it("throws on a non-OK exchange", async () => { + const fakeFetch = vi.fn(async () => new Response("e", { status: 422 })) as unknown as typeof fetch; + await expect(exchangeManifestCode("x", fakeFetch)).rejects.toThrow(/manifest_exchange_http_422/); + }); + + it("credentialsToEnv omits optional OAuth lines when client_id / client_secret are absent", () => { + const env = credentialsToEnv({ id: 1, slug: "s", webhook_secret: "w", pem: "k" }); + expect(env).toContain("GITHUB_APP_ID=1"); + expect(env).not.toContain("GITHUB_OAUTH_CLIENT_ID"); + expect(env).not.toContain("GITHUB_OAUTH_CLIENT_SECRET"); + }); +}); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts new file mode 100644 index 0000000000..13e67a690a --- /dev/null +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -0,0 +1,169 @@ +import { DatabaseSync } from "node:sqlite"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; +import { createSqliteQueue } from "../../src/selfhost/sqlite-queue"; +import type { JobMessage } from "../../src/types"; + +function makeDriver(): ReturnType { + return nodeSqliteDriver(new DatabaseSync(":memory:") as never); +} +const msg = (t: string): JobMessage => ({ type: t }) as unknown as JobMessage; +const typeOf = (m: JobMessage): string => (m as unknown as { type: string }).type; + +describe("createSqliteQueue (durable #980)", () => { + // Suppress audit log stdout noise. + beforeEach(() => { vi.spyOn(process.stdout, "write").mockImplementation(() => true); }); + afterEach(() => { vi.restoreAllMocks(); }); + + it("persists + drains FIFO through the consumer", async () => { + const driver = makeDriver(); + const seen: string[] = []; + const q = createSqliteQueue(driver, async (m) => void seen.push(typeOf(m))); + await q.binding.send(msg("a")); + await q.binding.send(msg("b")); + await q.drain(); + expect(seen).toEqual(["a", "b"]); + expect(q.size()).toBe(0); + }); + + it("retries then dead-letters after maxRetries", async () => { + const driver = makeDriver(); + let calls = 0; + const q = createSqliteQueue( + driver, + async () => { + calls += 1; + throw new Error("boom"); + }, + { maxRetries: 3, backoffMs: () => 0 }, + ); + await q.binding.send(msg("x")); + await q.drain(); // backoff 0 → all 3 attempts run within one drain, then dead-lettered + expect(calls).toBe(3); + expect(q.deadCount()).toBe(1); + expect(q.size()).toBe(0); + }); + + it("SURVIVES A RESTART: a fresh queue over the same DB processes a persisted pending job", async () => { + const driver = makeDriver(); + const seen: string[] = []; + const fresh = createSqliteQueue(driver, async (m) => void seen.push(typeOf(m))); // creates the table + // a job left pending on disk by a prior run (insert directly so this instance doesn't auto-process it first) + driver.query("INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at) VALUES (?, 'pending', 0, 0, 0)", [JSON.stringify(msg("persisted"))]); + await fresh.drain(); // the "new process" picks it up + expect(seen).toEqual(["persisted"]); + }); + + it("start() runs the poll loop and processes a job, stop() halts it", async () => { + const driver = makeDriver(); + const seen: string[] = []; + const q = createSqliteQueue(driver, async (m) => void seen.push(typeOf(m)), { pollIntervalMs: 10 }); + q.start(); + await q.binding.send(msg("ticked")); + for (let i = 0; i < 50 && seen.length === 0; i += 1) await new Promise((r) => setTimeout(r, 10)); + await q.stop(); + expect(seen).toEqual(["ticked"]); + }); + + it("recovers a job left 'processing' by a crash", async () => { + const driver = makeDriver(); + createSqliteQueue(driver, async () => undefined); // creates the table + driver.query("INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at) VALUES (?, 'processing', 0, 0, 0)", [JSON.stringify(msg("stuck"))]); + const seen: string[] = []; + const fresh = createSqliteQueue(driver, async (m) => void seen.push(typeOf(m))); + await fresh.drain(); + expect(seen).toEqual(["stuck"]); + }); + + it("records 'unknown error' when a consumer throws a non-Error", async () => { + const q = createSqliteQueue( + makeDriver(), + async () => { + throw "boom-string"; // not an Error instance + }, + { maxRetries: 1, backoffMs: () => 0 }, + ); + await q.binding.send(msg("x")); + await q.drain(); + expect(q.deadCount()).toBe(1); + }); + + it("dead-letters an unparseable payload", async () => { + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + driver.query("INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at) VALUES ('not-json','pending',0,0,0)", []); + await q.drain(); + expect(q.deadCount()).toBe(1); + }); + + it("sendBatch enqueues all; default backoff reschedules a failure into the future", async () => { + const seen: string[] = []; + const q = createSqliteQueue(makeDriver(), async (m) => void seen.push(typeOf(m))); + await q.binding.sendBatch([{ body: msg("a") }, { body: msg("b") }]); + await q.drain(); + expect(seen.sort()).toEqual(["a", "b"]); + + let calls = 0; + const q2 = createSqliteQueue(makeDriver(), async () => { + calls += 1; + throw new Error("x"); + }, { maxRetries: 5 }); // default backoff (~2s) → not re-claimed this drain + await q2.binding.send(msg("f")); + await q2.drain(); + expect(calls).toBe(1); + expect(q2.size()).toBe(1); + }); + + it("stop() is a no-op when start() was never called (timer is null)", async () => { + const q = createSqliteQueue(makeDriver(), async () => undefined); + await q.stop(); // timer=null → the false branch of `if (timer) clearTimeout(timer)` is taken + expect(q.size()).toBe(0); // still usable after a spurious stop() + }); + + it("concurrency=1 saturates after one active pump (active >= concurrency → early return)", async () => { + let concurrent = 0; + let maxConcurrent = 0; + const q = createSqliteQueue(makeDriver(), async () => { + concurrent++; + maxConcurrent = Math.max(maxConcurrent, concurrent); + await new Promise((r) => setTimeout(r, 15)); + concurrent--; + }, { concurrency: 1, pollIntervalMs: 100_000 }); + // sendBatch fires two void pump() calls synchronously; the second sees active=1 >= 1 and returns. + await q.binding.sendBatch([{ body: msg("a") }, { body: msg("b") }]); + await new Promise((r) => setTimeout(r, 60)); + await q.stop(); + expect(maxConcurrent).toBe(1); + expect(q.size()).toBe(0); + }); + + it("concurrency=2 allows two jobs to run simultaneously", async () => { + let concurrent = 0; + let maxConcurrent = 0; + const q = createSqliteQueue(makeDriver(), async () => { + concurrent++; + maxConcurrent = Math.max(maxConcurrent, concurrent); + await new Promise((r) => setTimeout(r, 15)); + concurrent--; + }, { concurrency: 2, pollIntervalMs: 100_000 }); + await q.binding.sendBatch([{ body: msg("a") }, { body: msg("b") }]); + await new Promise((r) => setTimeout(r, 60)); + await q.stop(); + expect(maxConcurrent).toBe(2); + expect(q.size()).toBe(0); + }); + + it("start() is idempotent and stop() waits for an in-flight pump", async () => { + let done = false; + const q = createSqliteQueue(makeDriver(), async () => { + await new Promise((r) => setTimeout(r, 40)); + done = true; + }, { pollIntervalMs: 5 }); + q.start(); + q.start(); // idempotent + await q.binding.send(msg("slow")); + await new Promise((r) => setTimeout(r, 12)); // let the tick claim it + enter the slow consume + await q.stop(); // waits for the in-flight consume to finish + expect(done).toBe(true); + }); +}); diff --git a/test/unit/selfhost-vectorize.test.ts b/test/unit/selfhost-vectorize.test.ts new file mode 100644 index 0000000000..742b6c4f08 --- /dev/null +++ b/test/unit/selfhost-vectorize.test.ts @@ -0,0 +1,85 @@ +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, it } from "vitest"; +import { nodeSqliteDriver } from "../../src/selfhost/d1-adapter"; +import { cosineSimilarity, createSqliteVectorize } from "../../src/selfhost/vectorize"; + +function makeVectorize(): ReturnType { + return createSqliteVectorize(nodeSqliteDriver(new DatabaseSync(":memory:") as never)); +} + +describe("cosineSimilarity", () => { + it("is 1 for identical and 0 for orthogonal vectors", () => { + expect(cosineSimilarity([1, 0], [1, 0])).toBeCloseTo(1); + expect(cosineSimilarity([1, 0], [0, 1])).toBeCloseTo(0); + expect(cosineSimilarity([0, 0], [0, 0])).toBe(0); // zero-norm guard + }); +}); + +describe("createSqliteVectorize (#979 local RAG)", () => { + it("returns the nearest-by-cosine match within a namespace, with metadata + topK", async () => { + const v = makeVectorize(); + await v.upsert([ + { id: "a", values: [1, 0, 0], namespace: "repo1", metadata: { path: "a.ts" } }, + { id: "b", values: [0, 1, 0], namespace: "repo1", metadata: { path: "b.ts" } }, + ]); + const res = await v.query([0.9, 0.1, 0], { topK: 1, namespace: "repo1", returnMetadata: "all" }); + expect(res.matches).toHaveLength(1); + expect(res.matches[0]?.id).toBe("a"); + expect(res.matches[0]?.metadata?.path).toBe("a.ts"); + }); + + it("scopes results by namespace and defaults topK when omitted", async () => { + const v = makeVectorize(); + await v.upsert([ + { id: "x", values: [1, 0], namespace: "n1" }, + { id: "y", values: [1, 0], namespace: "n2" }, + ]); + const res = await v.query([1, 0], { topK: 10, namespace: "n1" }); + expect(res.matches.map((m) => m.id)).toEqual(["x"]); + // topK omitted → default applies (no throw, returns the namespace's match) + const res2 = await v.query([1, 0], { namespace: "n1" }); + expect(res2.matches.map((m) => m.id)).toEqual(["x"]); + }); + + it("returns matches with and without metadata in one query", async () => { + const v = makeVectorize(); + await v.upsert([ + { id: "withMeta", values: [1, 0], namespace: "n", metadata: { path: "p" } }, + { id: "noMeta", values: [0, 1], namespace: "n" }, + ]); + const res = await v.query([1, 1], { topK: 10, namespace: "n" }); + expect(res.matches).toHaveLength(2); + expect(res.matches.some((m) => m.metadata)).toBe(true); + expect(res.matches.some((m) => !m.metadata)).toBe(true); + }); + + it("upsert overwrites by id; deleteByIds removes", async () => { + const v = makeVectorize(); + await v.upsert([{ id: "d", values: [1, 0], namespace: "n", metadata: { path: "old" } }]); + await v.upsert([{ id: "d", values: [0, 1], namespace: "n", metadata: { path: "new" } }]); // overwrite + let res = await v.query([0, 1], { topK: 10, namespace: "n" }); + expect(res.matches[0]?.metadata?.path).toBe("new"); + await v.deleteByIds(["d"]); + res = await v.query([0, 1], { topK: 10, namespace: "n" }); + expect(res.matches).toHaveLength(0); + }); + + it("upsert without a namespace defaults to the empty-string namespace", async () => { + const v = makeVectorize(); + await v.upsert([{ id: "ns-less", values: [1, 0] }]); + // After upserting without namespace, querying with no namespace finds it + const { matches } = await v.query([1, 0], { topK: 1 }); + expect(matches[0]?.id).toBe("ns-less"); + }); + + it("query without a namespace scans the full table", async () => { + const v = makeVectorize(); + await v.upsert([ + { id: "n1", values: [1, 0], namespace: "a" }, + { id: "n2", values: [0, 1], namespace: "b" }, + ]); + const res = await v.query([1, 0], { topK: 10 }); // no namespace → scans all + expect(res.matches.map((m) => m.id)).toContain("n1"); + expect(res.matches.map((m) => m.id)).toContain("n2"); + }); +});