diff --git a/.env.example b/.env.example index 3cf4246534..d558c9ebc5 100644 --- a/.env.example +++ b/.env.example @@ -105,8 +105,15 @@ GITTENSORY_REVIEW_DRAFT=false # (gitignored), UNCOMMENT + fill the required Core secrets in section 2, then add the runtime values below. # Every value here is a SAMPLE placeholder — never commit real secrets. +# PUBLIC_API_ORIGIN=https://reviews.example.com # REQUIRED before the first-run setup wizards (GET /setup and +# # GET /orb/setup). The wizard embeds this origin in the GitHub App +# # manifest's redirect_url; without it the wizard returns 400. Use the +# # EXACT public URL operators browse to (scheme + host [+ port]). +# # Deriving it from the request Host header would let an attacker +# # redirect the App-creation callback, so it must be set explicitly. +# # Not needed once the App credentials are configured. # PORT=8787 -# DATABASE_PATH=/data/gittensory.sqlite # SQLite file on the mounted data volume; all 56 migrations auto-apply +# DATABASE_PATH=/data/gittensory.sqlite # SQLite file on the mounted data volume; all migrations auto-apply # DATABASE_URL= # set to postgres://user:pw@host:5432/db to use Postgres instead of # # SQLite (shared DB → multi-instance). Overrides DATABASE_PATH. # REDIS_URL= # set to redis://host:6379 for distributed rate limiting + webhook dedup @@ -114,6 +121,10 @@ GITTENSORY_REVIEW_DRAFT=false # QDRANT_URL= # set to http://qdrant:6333 to use Qdrant as the RAG vector store # # (--profile qdrant). Overrides the built-in sqlite-vec / pgvector. # # Collection and schema are auto-created at startup. Off when unset. +# QDRANT_API_KEY= # Bearer token for an authenticated Qdrant (cloud / on-prem). Omit for +# # the local --profile qdrant container (unauthenticated). +# QDRANT_DIM=1024 # vector dimension of the collection (default 1024 = bge-m3); set to +# # match your AI_EMBED_MODEL if it differs. # MIGRATIONS_DIR=/app/migrations # CRON_INTERVAL_MS=120000 # maintain/sweep + sync cadence (default ~2 min) @@ -141,7 +152,10 @@ GITTENSORY_REVIEW_DRAFT=false # RUNNER_NAME=gittensory-runner # RUNNER_LABELS=self-hosted,linux -# --- Grafana (#1206; requires --profile observability) --- +# --- Observability: metrics + alerts + logs (#1206; requires --profile observability) --- +# The observability profile starts Prometheus (scrapes /metrics) + Alertmanager (alert rules in +# prometheus/rules/, routing in alertmanager/alertmanager.yml — silent until you fill in a receiver) + +# Loki + Promtail (ship every container's logs to Loki) + Grafana (dashboards for metrics AND logs). # GRAFANA_ADMIN_PASSWORD=changeme # REQUIRED when using --profile observability; compose fails if unset # --- AI review backend (optional; without it reviews run deterministically) --- diff --git a/Dockerfile b/Dockerfile index 893b36a4fe..0e74f0ba96 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,6 +40,9 @@ RUN if [ "$INSTALL_VISUAL_REVIEW" = "true" ]; then npm install puppeteer-core@22 RUN mkdir -p /data && chown -R node:node /data /app USER node EXPOSE 8787 -HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ - CMD node -e "fetch('http://127.0.0.1:'+(process.env.PORT||8787)+'/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" +# Probe /ready (not /health): /health is a liveness stub that returns 200 even when the DB is down, +# whereas /ready returns 503 until the DB answers and migrations are applied. start-period tolerates the +# Postgres cold start (waitForPostgres blocks up to 30s before the HTTP server even binds). +HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ + CMD node -e "fetch('http://127.0.0.1:'+(process.env.PORT||8787)+'/ready').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" CMD ["node", "dist/server.mjs"] diff --git a/alertmanager/alertmanager.yml b/alertmanager/alertmanager.yml new file mode 100644 index 0000000000..a724ca0f0e --- /dev/null +++ b/alertmanager/alertmanager.yml @@ -0,0 +1,122 @@ +# Alertmanager configuration for the gittensory self-host stack (#980 observability). +# Schema: Alertmanager v0.27+ (UTF-8 matchers; `matchers:` as a list of strings). +# Mounted at /etc/alertmanager/alertmanager.yml in the alertmanager container. +# +# OUT OF THE BOX this config is VALID and SILENT: every alert routes to a name-only +# "null" receiver that discards notifications. Nothing pages you until you opt in by +# uncommenting one of the example receivers below and pointing the route at it. This +# means `docker compose --profile observability up -d` always comes up green. + +global: + # How long to wait before declaring a firing alert "resolved" once it stops arriving. + resolve_timeout: 5m + + # ── SMTP defaults (only needed if you enable an email receiver below) ─────── + # Fill these in and they become the defaults for every email_configs block, so you + # don't repeat them per-receiver. Leave commented if you don't use email. + # smtp_from: "alertmanager@example.com" + # smtp_smarthost: "smtp.example.com:587" + # smtp_auth_username: "alertmanager@example.com" + # smtp_auth_password: "CHANGEME" # or smtp_auth_password_file: /etc/alertmanager/smtp_password + # smtp_require_tls: true + + # ── Slack default webhook (only needed for the Slack receiver below) ──────── + # Set this once and slack_configs can omit api_url. Prefer slack_api_url_file to keep + # the secret out of this file. + # slack_api_url: "https://hooks.slack.com/services/T000/B000/XXXX" + # slack_api_url_file: /etc/alertmanager/slack_url + +# ───────────────────────────────────────────────────────────────────────────── +# ROUTING TREE +# The root route catches every alert. With the default config it all flows to the +# `null` receiver. Uncomment the child routes to split by severity once you've added +# real receivers. +# ───────────────────────────────────────────────────────────────────────────── +route: + receiver: "null" # default sink: discards alerts until you change it + + # Group alerts that share these labels into a single notification. + group_by: ["alertname", "severity"] + group_wait: 30s # wait this long to batch the first notification in a group + group_interval: 5m # wait this long before sending an updated batch for a group + repeat_interval: 4h # re-send an unresolved alert at most this often + + # routes: + # # Page-worthy: send critical alerts to the on-call integration. + # - matchers: + # - severity="critical" + # receiver: "oncall-pager" + # # Critical alerts shouldn't wait long to be grouped. + # group_wait: 10s + # repeat_interval: 1h + # continue: true # also let lower routes / inhibition see it + # + # # Everything warning-level goes to a chat channel. + # - matchers: + # - severity="warning" + # receiver: "team-slack" + +# ───────────────────────────────────────────────────────────────────────────── +# RECEIVERS +# A receiver with only a `name` (no integrations) is the canonical "null"/blackhole +# receiver: valid, and it silently drops anything routed to it. Add an integration +# block (slack_configs / email_configs / webhook_configs / pagerduty_configs / …) to a +# receiver to make it notify, then point a route at it. +# ───────────────────────────────────────────────────────────────────────────── +receivers: + - name: "null" + + # ── Slack ─────────────────────────────────────────────────────────────────── + # Create an Incoming Webhook in Slack, then set api_url (or global.slack_api_url). + # - name: "team-slack" + # slack_configs: + # - api_url: "https://hooks.slack.com/services/T000/B000/XXXX" # or omit if global.slack_api_url is set + # channel: "#alerts" + # send_resolved: true + # title: '{{ .CommonLabels.alertname }} ({{ .CommonLabels.severity }})' + # text: >- + # {{ range .Alerts }}*{{ .Annotations.summary }}* + # {{ .Annotations.description }} + # _runbook:_ {{ .Annotations.runbook }} + # {{ end }} + + # ── Email ───────────────────────────────────────────────────────────────────── + # Requires the SMTP globals above (or per-receiver smarthost/auth_* fields). + # - name: "team-email" + # email_configs: + # - to: "oncall@example.com" + # send_resolved: true + # # from / smarthost / auth_username / auth_password inherit from global.smtp_* if set: + # # from: "alertmanager@example.com" + # # smarthost: "smtp.example.com:587" + # # auth_username: "alertmanager@example.com" + # # auth_password: "CHANGEME" + + # ── Generic webhook (PagerDuty bridge, custom handler, etc.) ────────────────── + # POSTs the Alertmanager JSON payload to any HTTP endpoint. + # - name: "oncall-pager" + # webhook_configs: + # - url: "https://your-endpoint.example.com/alerts" + # send_resolved: true + # max_alerts: 0 # 0 = send all alerts in the group, no truncation + +# ───────────────────────────────────────────────────────────────────────────── +# INHIBITION RULES (v0.27+ matcher syntax) +# Mute lower-severity noise while a higher-severity alert for the same scope is firing. +# Uncomment once you have multiple severities routing — keeps a single incident from +# fanning out into a wall of warnings. +# ───────────────────────────────────────────────────────────────────────────── +# inhibit_rules: +# # When a critical fires, silence warnings that share the same alertname. +# - source_matchers: +# - severity="critical" +# target_matchers: +# - severity="warning" +# equal: ["alertname"] +# +# # When the whole target is down, silence its derivative warnings (5xx, latency, queue). +# - source_matchers: +# - alertname="GittensoryTargetDown" +# target_matchers: +# - severity="warning" +# equal: ["job"] diff --git a/docker-compose.yml b/docker-compose.yml index 898bddb0dc..c03aae061e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,17 +9,18 @@ # --profile postgres pgvector/pg16 shared database (multi-instance capable) # --profile pgbouncer PgBouncer connection pooler in front of Postgres # --profile redis Redis fixed-window rate limiter +# --profile qdrant Qdrant vector database for RAG # --profile ollama Local Ollama AI backend # --profile litestream Continuous SQLite backup to S3/B2/R2 via Litestream # --profile caddy Caddy HTTPS terminator with auto-TLS (set DOMAIN= in .env) -# --profile observability Prometheus + Grafana dashboards (pre-wired to /metrics) +# --profile observability Prometheus + Alertmanager + Loki + Promtail + Grafana (pre-wired) # --profile tailscale Tailscale sidecar — access the stack via your tailnet # --profile runners GitHub Actions self-hosted runner # # Examples: # docker compose up --build # SQLite, no AI # docker compose --profile postgres --profile caddy up -d # Postgres + HTTPS -# docker compose --profile observability up -d # add dashboards to anything +# docker compose --profile observability up -d # metrics + logs + dashboards # docker compose --profile tailscale --profile runners up -d # tailnet + CI runners services: @@ -45,16 +46,29 @@ services: # DATABASE_URL: postgres://gittensory:${POSTGRES_PASSWORD:-CHANGEME}@pgbouncer:5432/gittensory # Uncomment for Redis rate limiting (--profile redis): # REDIS_URL: redis://redis:6379 + # Uncomment for Qdrant RAG vector store (--profile qdrant): + # QDRANT_URL: http://qdrant:6333 # Uncomment for Ollama AI (--profile ollama): # AI_PROVIDER: ollama # AI_BASE_URL: http://ollama:11434/v1 volumes: - gittensory-data:/data + depends_on: + # Gate startup on a healthy Qdrant when --profile qdrant is active; required:false means the + # dependency is ignored when qdrant isn't started (default/other profiles). Belt-and-suspenders + # with the app-side waitForQdrant retry in src/server.ts (which also covers a mid-life restart). + qdrant: + condition: service_healthy + required: false healthcheck: - test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:8787/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"] + # Probe /ready (not /health): /health is a liveness stub that is 200 even when the DB is down, + # whereas /ready returns 503 until the DB answers AND migrations are applied — so dependents that + # wait on `condition: service_healthy` only start once the app is truly serving. start_period 60s + # tolerates the Postgres cold start (waitForPostgres blocks up to 30s before the HTTP server binds). + test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:8787/ready').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"] interval: 30s timeout: 5s - start_period: 20s + start_period: 60s retries: 3 # ── Postgres (--profile postgres | --profile pgbouncer) ─────────────────── @@ -96,12 +110,24 @@ services: AUTH_TYPE: md5 # ── Redis (--profile redis) ──────────────────────────────────────────────── + # Ephemeral rate-limiter + 5-min webhook-dedup cache only — losing it is harmless, so persistence is + # fully disabled. --save "" turns off RDB snapshots; --appendonly no turns off AOF. allkeys-lru caps + # memory at 256mb and evicts least-recently-used keys under pressure instead of OOM-killing. No volume: + # nothing here needs to survive a restart. (The empty-string and "no" are quoted so YAML keeps them.) redis: image: redis:7-alpine restart: unless-stopped profiles: ["redis"] - volumes: - - gittensory-redis:/data + command: + - redis-server + - --maxmemory + - 256mb + - --maxmemory-policy + - allkeys-lru + - --save + - "" + - --appendonly + - "no" healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 10s @@ -115,13 +141,31 @@ services: image: qdrant/qdrant:latest restart: unless-stopped profiles: ["qdrant"] + # Ports are bound to LOOPBACK (127.0.0.1), not 0.0.0.0: the app reaches Qdrant over the internal + # docker network (QDRANT_URL=http://qdrant:6333), so the host mapping exists only for the local + # dashboard/debugging. The local container runs UNAUTHENTICATED, and it holds the RAG embeddings + + # PR context — never expose it beyond localhost without setting QDRANT_API_KEY (honored by + # qdrant-vectorize.ts) plus a firewall rule. SSH-forward the port for remote dashboard access. This + # matches the in-network-only posture of Prometheus/Loki/Alertmanager above. ports: - - "6333:6333" # REST API + Web UI - - "6334:6334" # gRPC + - "127.0.0.1:6333:6333" # REST API + Web UI (localhost only) + - "127.0.0.1:6334:6334" # gRPC (localhost only) volumes: - qdrant-data:/qdrant/storage - # Qdrant's minimal image has no curl/wget/nc — healthcheck omitted. - # The service is ready when the gittensory startup log shows {"event":"selfhost_vectorize","backend":"qdrant"}. + # The image is Debian (bash present) but ships no curl/wget/nc. bash's /dev/tcp pseudo-device speaks + # HTTP without an external binary, so we hit Qdrant's k8s-style /readyz probe (200 once it accepts + # traffic). This lets the app's depends_on gate on a serving Qdrant. /readyz is unauthenticated even + # when QDRANT_API_KEY is set, so no secret is needed here. + healthcheck: + test: + - CMD + - bash + - -c + - 'exec 3<>/dev/tcp/127.0.0.1/6333 && printf "GET /readyz HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" >&3 && grep -q "200 OK" <&3' + interval: 10s + timeout: 5s + start_period: 15s + retries: 5 # ── Ollama (--profile ollama) ────────────────────────────────────────────── # After `docker compose --profile ollama up -d`, pull a model: @@ -176,25 +220,42 @@ services: condition: service_healthy # ── Observability (--profile observability) ──────────────────────────────── - # Prometheus scrapes /metrics; Grafana visualises it. - # Grafana UI: http://localhost:3000 (admin / admin — change on first login). + # Prometheus scrapes /metrics; Alertmanager routes alerts; Loki+Promtail collect logs; + # Grafana visualises metrics AND logs. Grafana UI: http://localhost:3000 (admin / $GRAFANA_ADMIN_PASSWORD). prometheus: image: prom/prometheus:latest restart: unless-stopped profiles: ["observability"] volumes: - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./prometheus/rules:/etc/prometheus/rules:ro - prometheus-data:/prometheus command: - "--config.file=/etc/prometheus/prometheus.yml" - "--storage.tsdb.path=/prometheus" - "--storage.tsdb.retention.time=30d" + # Routes Prometheus alerts to your notification channel. Ships SILENT: alerts go to a + # null receiver until you fill in a receiver in alertmanager/alertmanager.yml. + alertmanager: + image: prom/alertmanager:latest + restart: unless-stopped + profiles: ["observability"] + depends_on: [prometheus] + expose: + - "9093" # in-network only; reach the UI/API via `docker compose port` or a tunnel + volumes: + - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro + - alertmanager-data:/alertmanager + command: + - "--config.file=/etc/alertmanager/alertmanager.yml" + - "--storage.path=/alertmanager" + grafana: image: grafana/grafana:latest restart: unless-stopped profiles: ["observability"] - depends_on: [prometheus] + depends_on: [prometheus, loki] ports: - "3000:3000" volumes: @@ -205,6 +266,60 @@ services: GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env before using --profile observability} GF_USERS_ALLOW_SIGN_UP: "false" + # ── Log pipeline (--profile observability) ───────────────────────────────── + # Loki stores logs; Promtail discovers every container in this compose project via the read-only + # docker-proxy (NOT a raw socket) and ships their logs to Loki. Browse in Grafana → Explore → + # Loki, e.g. {service="gittensory"} | json | level="error". + loki: + image: grafana/loki:latest + restart: unless-stopped + profiles: ["observability"] + command: ["-config.file=/etc/loki/loki-config.yml"] + volumes: + - ./loki/loki-config.yml:/etc/loki/loki-config.yml:ro + - loki-data:/loki + # No healthcheck: grafana/loki:latest is distroless (no shell/wget/curl), so an in-container probe + # can't run — a CMD-SHELL check would leave the container stuck "starting". Readiness is observable + # at GET /ready; Promtail retries/backs off until Loki accepts pushes and Grafana retries the + # datasource, so nothing depends on a Docker health gate here. + + # Read-only Docker-API proxy that fronts the socket for Promtail's service discovery. Promtail must + # NOT be given the raw /var/run/docker.sock: a `:ro` socket bind-mount only protects the socket inode, + # it does NOT restrict the Docker API, so socket access is effectively host root (enumerate containers, + # read env/secrets + all container logs, create a privileged container and escape to the host — the + # same risk the `runner` service notes below). This proxy is the ONLY container touching the socket; it + # allows just the read-only `/containers/*` endpoints docker_sd needs (list/inspect/logs) and blocks + # every mutating call (POST=0 → no create/exec/start). Even read-only, GET /containers/{id}/json still + # returns each container's Config.Env, so the proxy is isolated on a dedicated `docker-proxy` network + # shared ONLY with Promtail (publishing no host port is not enough — the default network is reachable by + # every service). No other container can reach :2375 to harvest secrets from peers' environments. + docker-proxy: + image: tecnativa/docker-socket-proxy:0.3.0 + restart: unless-stopped + profiles: ["observability"] + environment: + CONTAINERS: "1" # GET /containers/* (list, inspect, logs) + NETWORKS: "1" # GET /networks/* — docker_sd computes per-target network labels + POST: "0" # deny every mutating call; strictly read-only (no create/exec/start) + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + networks: [docker-proxy] # isolated — only Promtail joins this network + + promtail: + image: grafana/promtail:latest + restart: unless-stopped + profiles: ["observability"] + command: ["-config.file=/etc/promtail/promtail-config.yml"] + depends_on: [loki, docker-proxy] + volumes: + - ./promtail/promtail-config.yml:/etc/promtail/promtail-config.yml:ro + # Persist read positions so restarts don't re-ship the whole log history. + - promtail-data:/tmp/positions + # `default` reaches Loki for pushes; `docker-proxy` reaches the socket proxy for discovery. Promtail is + # the only service on `docker-proxy`, so the inspect endpoint (and the env it exposes) is unreachable + # from any other container. No Docker-socket mount — discovery goes through the proxy's read-only API. + networks: [default, docker-proxy] + # ── Tailscale (--profile tailscale) ─────────────────────────────────────── # Joins your tailnet so the stack is accessible via Tailscale IP/hostname — no # public ports exposed. Generate an auth key at tailscale.com/settings/keys and @@ -253,12 +368,24 @@ services: volumes: gittensory-data: gittensory-pg: - gittensory-redis: qdrant-data: ollama-models: caddy-data: caddy-config: prometheus-data: + alertmanager-data: grafana-data: + loki-data: + promtail-data: tailscale-state: runner-work: + +networks: + # The implicit project network every service joins by default (declared so it can be referenced + # explicitly where a service also needs the isolated proxy network below). + default: + # Isolated network for the Docker-socket proxy. `internal: true` blocks external/internet egress, and + # only docker-proxy + promtail join it — so the proxy's inspect endpoint (which exposes every + # container's Config.Env) is unreachable from any other service. See the docker-proxy service notes. + docker-proxy: + internal: true diff --git a/grafana/dashboards/gittensory.json b/grafana/dashboards/gittensory.json index 46592a5143..774ed9f8a9 100644 --- a/grafana/dashboards/gittensory.json +++ b/grafana/dashboards/gittensory.json @@ -309,6 +309,511 @@ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gittensory_orb_events_recorded_total - gittensory_orb_events_exported_total", "legendFormat": "pending" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "gittensory_orb_events_exported_total", "legendFormat": "exported (cumulative)" } ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 45 + }, + "id": 105, + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 19, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": true, + "enableLogDetails": true, + "prettifyLogMessage": true, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "title": "Service Logs (live)", + "type": "logs", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{service=\"gittensory\"} | json eventf=\"event\", errf=\"error\" | line_format \"{{.eventf}}{{if .errf}} — {{.errf}}{{end}}\"", + "queryType": "range", + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 20, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": true, + "enableLogDetails": true, + "prettifyLogMessage": true, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "title": "Errors & failures", + "type": "logs", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{service=\"gittensory\"} | json eventf=\"event\", errf=\"error\" | errf != \"\" | line_format \"{{.eventf}} — {{.errf}}\"", + "queryType": "range", + "refId": "A" + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 106, + "title": "HTTP Observability", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 56 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "title": "HTTP Request Rate by Status Class", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (status) (rate(gittensory_http_requests_total[5m]))", + "legendFormat": "{{status}}", + "refId": "A" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 56 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "title": "HTTP Latency (p50 / p95 / p99)", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(gittensory_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(gittensory_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(gittensory_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p99", + "refId": "C" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 56 + }, + "id": 23, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "title": "5xx Error Ratio", + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(gittensory_http_requests_total{status=\"5xx\"}[5m])) / clamp_min(sum(rate(gittensory_http_requests_total[5m])), 1e-9)", + "legendFormat": "5xx ratio", + "refId": "A" + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 107, + "title": "Alerts", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "severity" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-background", + "mode": "basic" + } + }, + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "critical": { + "color": "red", + "index": 0 + }, + "warning": { + "color": "yellow", + "index": 1 + }, + "info": { + "color": "blue", + "index": 2 + } + } + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 65 + }, + "id": 24, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "fields": "" + } + }, + "title": "Active Alerts (firing)", + "type": "table", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ALERTS{alertstate=\"firing\"}", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "refId": "A" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "alertstate": true, + "job": true + }, + "indexByName": { + "alertname": 0, + "severity": 1, + "instance": 2, + "summary": 3 + }, + "renameByName": { + "alertname": "Alert", + "severity": "Severity", + "instance": "Instance", + "summary": "Summary" + } + } + } + ] } ], "refresh": "30s", @@ -333,5 +838,5 @@ "timezone": "browser", "title": "Gittensory Self-Host", "uid": "gittensory-selfhost", - "version": 3 + "version": 4 } diff --git a/grafana/provisioning/datasources/loki.yml b/grafana/provisioning/datasources/loki.yml new file mode 100644 index 0000000000..a0b1df64a4 --- /dev/null +++ b/grafana/provisioning/datasources/loki.yml @@ -0,0 +1,16 @@ +# Grafana datasource provisioning — adds Loki alongside the existing Prometheus. +# Prometheus stays the default (isDefault here is omitted -> false), so dashboards +# and Explore default to metrics; pick "Loki" explicitly for logs. +apiVersion: 1 +datasources: + - name: Loki + type: loki + access: proxy + uid: loki + url: http://loki:3100 + isDefault: false + editable: false + jsonData: + # Cap returned lines so a broad query can't hang the browser on a small host. + maxLines: 1000 + timeout: 60 diff --git a/loki/loki-config.yml b/loki/loki-config.yml new file mode 100644 index 0000000000..68b8f2461c --- /dev/null +++ b/loki/loki-config.yml @@ -0,0 +1,101 @@ +# Grafana Loki — monolithic / single-binary config for gittensory self-host. +# Verified against Loki v3.7.x (latest stable, June 2026): TSDB store + schema v13, +# filesystem object store (no S3), compactor-driven retention, structured metadata on. +# +# Activated via: docker compose --profile observability up -d +# All state lives under /loki (the loki-data named volume). + +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + # Keep logs quiet on a small host; bump to debug only when troubleshooting. + log_level: info + # Bounded request sizes for a single small instance. + grpc_server_max_recv_msg_size: 8388608 # 8 MiB + grpc_server_max_send_msg_size: 8388608 # 8 MiB + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + replication_factor: 1 + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +# TSDB is the current Loki index (default since 2.8). schema v13 is required for +# structured metadata (chunk format v4). Do NOT use the stale boltdb-shipper / v11. +schema_config: + configs: + - from: 2024-04-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + tsdb_shipper: + active_index_directory: /loki/tsdb-index + cache_location: /loki/tsdb-cache + filesystem: + directory: /loki/chunks + +# Retention is performed by the compactor (table_manager is deprecated). +# delete_request_store is REQUIRED whenever retention_enabled is true. +compactor: + working_directory: /loki/compactor + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + retention_delete_worker_count: 150 + delete_request_store: filesystem + +limits_config: + # Global retention: 14 days (336h). Minimum allowed is 24h. + retention_period: 336h + # Required for Promtail's structured_metadata stage (level/event metadata). + allow_structured_metadata: true + # Bound cardinality / ingest on a small self-host host. + ingestion_rate_mb: 8 + ingestion_burst_size_mb: 16 + max_streams_per_user: 5000 + max_label_names_per_series: 20 + reject_old_samples: true + reject_old_samples_max_age: 168h # 7d + max_query_length: 721h # ~30d ceiling on a single query + max_query_parallelism: 16 + volume_enabled: true + +querier: + max_concurrent: 4 + +# Disable phone-home analytics on a self-hosted box. +analytics: + reporting_enabled: false + +# Use the embedded cache (no external memcached) on a single small instance. +query_range: + align_queries_with_step: true + cache_results: true + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +# Local ruler dir so Loki doesn't error on a missing rules backend (no rules shipped). +ruler: + storage: + type: local + local: + directory: /loki/rules + rule_path: /loki/rules-tmp + enable_api: true diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index e2de3c51c2..b3d455d172 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -4,6 +4,17 @@ global: scrape_interval: 15s evaluation_interval: 15s +# Alerting + recording rule files. The glob is evaluated inside the container, where +# ./prometheus/rules is mounted at /etc/prometheus/rules (see docker-compose.yml). +rule_files: + - /etc/prometheus/rules/*.yml + +# Ship firing alerts to Alertmanager (reachable by its compose service name on 9093). +alerting: + alertmanagers: + - static_configs: + - targets: ["alertmanager:9093"] + scrape_configs: - job_name: gittensory static_configs: diff --git a/prometheus/rules/alerts.yml b/prometheus/rules/alerts.yml new file mode 100644 index 0000000000..bd421e80be --- /dev/null +++ b/prometheus/rules/alerts.yml @@ -0,0 +1,196 @@ +# Prometheus alerting rules for the gittensory self-host stack (#980 observability). +# +# Mounted into the prometheus container at /etc/prometheus/rules/alerts.yml and loaded +# via the `rule_files: ["/etc/prometheus/rules/*.yml"]` glob in prometheus.yml. +# +# Every rule below is grounded ONLY in metrics the gittensory app actually exports at +# GET /metrics, plus the synthetic `up` metric Prometheus emits per scrape target. +# +# Thresholds are sane defaults for a SMALL single-host self-host. Tune the numbers in +# `expr` / `for` to your traffic — each is commented with what it means and how to adjust. +# +# Severity convention (consumed by Alertmanager routing/inhibition): +# severity: critical → page / wake someone up; the service is down or losing data. +# severity: warning → look soon; degraded but not (yet) an outage. +# +# Annotation convention: +# summary → one-line headline (templated with the offending series' labels). +# description → what tripped + the live value, for the notification body. +# runbook → a short, actionable first-response hint (no external URL needed). + +groups: + # ── Scrape target liveness ──────────────────────────────────────────────── + - name: gittensory-availability + rules: + - alert: GittensoryTargetDown + # `up` is 0 when Prometheus cannot scrape /metrics (process down, crash-loop, + # network partition, or wrong port). Scoped to our job so node/other targets + # don't trip this. 2m absorbs a single missed scrape + a restart. + expr: up{job="gittensory"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "gittensory target {{ $labels.instance }} is down" + description: "Prometheus has failed to scrape {{ $labels.instance }} (job {{ $labels.job }}) for over 2m. The app is unreachable or not serving /metrics." + runbook: "Check `docker compose ps` and `docker compose logs gittensory`. Look for a missing selfhost_listening log line or a crash-loop (restart count climbing)." + + # ── Job queue / worker health ───────────────────────────────────────────── + - name: gittensory-jobs + rules: + - alert: GittensoryHighJobFailureRatio + # Fraction of processed jobs that failed over the last 10m. The `> 0` guard on + # the denominator avoids a 0/0 = NaN (which never fires but is noise in /rules). + # 0.10 = 10% of jobs failing. Raise if your workload has expected transient + # failures; lower if any failure is a real problem. + expr: | + ( + sum(rate(gittensory_jobs_failed_total[10m])) + / + sum(rate(gittensory_jobs_processed_total[10m])) > 0 + ) > 0.10 + for: 15m + labels: + severity: warning + annotations: + summary: "gittensory job failure ratio above 10%" + description: "{{ $value | humanizePercentage }} of jobs processed in the last 10m failed (sustained 15m). Expected: well under 10%." + runbook: "Tail logs for level=error job events (e.g. selfhost_cron_error). A spike usually means a bad upstream (GitHub API / AI provider / DB) or a poison payload — check what changed." + + - alert: GittensoryDeadLetterJobsGrowing + # Jobs exhausted all retries and landed in the dead-letter queue. ANY new dead + # job in 15m is worth a look — these are lost work that won't self-heal. + # `increase(...) > 0` fires on the first dead job in the window. + expr: increase(gittensory_jobs_dead_total[15m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "gittensory dead-letter jobs increasing" + description: "{{ $value | printf \"%.0f\" }} job(s) moved to the dead-letter queue in the last 15m. These exhausted all retries and need manual attention." + runbook: "Inspect gittensory_queue_dead gauge and the dead-letter records. Decide whether to fix-and-requeue or discard. A steady climb points at a systematic failure, not a blip." + + - alert: GittensoryDeadLetterBacklogHigh + # Standing size of the dead-letter queue. Distinct from the rate alert above: + # this catches a large backlog that built up before alerting was watching. + # 50 dead jobs is a generous default for a small host — tune down if you expect ~0. + expr: gittensory_queue_dead > 50 + for: 10m + labels: + severity: warning + annotations: + summary: "gittensory dead-letter backlog above 50" + description: "The dead-letter queue holds {{ $value | printf \"%.0f\" }} jobs (sustained 10m). Work is accumulating with no automatic recovery." + runbook: "Drain or triage the dead-letter queue. If it only ever grows, fix the root cause before requeuing or you'll just re-fill it." + + # ── Queue backlog (live processing pressure) ─────────────────────────────── + - name: gittensory-queue + rules: + - alert: GittensoryQueueBacklogHigh + # Pending (not-yet-processed) jobs. A sustained high pending count means workers + # can't keep up with enqueue rate. 100 is a starting line for a single-node host; + # raise it if your normal steady-state pending depth is higher. + expr: gittensory_queue_pending > 100 + for: 10m + labels: + severity: warning + annotations: + summary: "gittensory queue backlog above 100" + description: "{{ $value | printf \"%.0f\" }} jobs pending for over 10m — the worker is falling behind the enqueue rate." + runbook: "Compare rate(gittensory_jobs_enqueued_total) vs rate(gittensory_jobs_processed_total). If enqueue > processed persistently, the worker is the bottleneck (slow AI/DB, or it's stuck). Check worker logs." + + # ── Qdrant vector backend ────────────────────────────────────────────────── + - name: gittensory-qdrant + rules: + - alert: GittensoryQdrantErrorRateHigh + # Qdrant errors (across upsert/query/delete) relative to total Qdrant traffic. + # `... or vector(0)` keeps the denominator non-empty so the ratio is defined even + # when query/upsert counters haven't been created yet. > 0.05 = 5% error rate. + expr: | + ( + sum(rate(gittensory_qdrant_errors_total[10m])) + / + ( + sum(rate(gittensory_qdrant_queries_total[10m])) + + sum(rate(gittensory_qdrant_upserts_total[10m])) + + sum(rate(gittensory_qdrant_errors_total[10m])) + > 0 + ) + ) > 0.05 + for: 10m + labels: + severity: warning + annotations: + summary: "gittensory Qdrant error rate above 5%" + description: "{{ $value | humanizePercentage }} of Qdrant operations errored over the last 10m (sustained 10m). RAG retrieval/indexing is degraded." + runbook: "Check the qdrant container (`docker compose --profile qdrant ps/logs`) and gittensory_qdrant_errors_total{op=...} to see whether upsert, query, or delete is failing. A reachable-but-erroring Qdrant often means a schema/collection or disk problem." + + # ── Orb usage-event export pipeline ──────────────────────────────────────── + - name: gittensory-orb + rules: + - alert: GittensoryOrbExportErrorRateHigh + # Failed Orb event exports vs total export attempts (errors + successes). + # The `> 0` denominator guard avoids 0/0. Export failures mean billed usage + # events aren't reaching Orb — a revenue/billing-integrity issue, so it warns + # even at a modest 5% rate. + expr: | + ( + sum(rate(gittensory_orb_export_errors_total[15m])) + / + ( + sum(rate(gittensory_orb_events_exported_total[15m])) + + sum(rate(gittensory_orb_export_errors_total[15m])) + > 0 + ) + ) > 0.05 + for: 15m + labels: + severity: warning + annotations: + summary: "gittensory Orb export error rate above 5%" + description: "{{ $value | humanizePercentage }} of Orb usage-event exports failed over the last 15m (sustained 15m). Billable events may not be reaching Orb." + runbook: "Verify Orb API credentials/connectivity and check level=error logs around orb export. Recorded-but-unexported events (gittensory_orb_events_recorded_total vs _exported_total diverging) confirm a stuck exporter." + + # ── HTTP serving health (uses the PLANNED status label + duration histogram) ─ + # NOTE: gittensory_http_requests_total is gaining a status="2xx|3xx|4xx|5xx" label, + # and gittensory_http_request_duration_seconds (a histogram) is being added. Both + # rules below tolerate the metric/label being absent on older data: the ratio uses a + # `> 0` denominator guard, and an absent series simply yields no result (rule stays + # inactive) rather than erroring. They start firing the moment the new metrics exist. + - name: gittensory-http + rules: + - alert: GittensoryHighHttp5xxRatio + # Share of responses that are 5xx. status="5xx" selects only server errors; + # the denominator sums all statuses. If the status label is missing entirely + # (old build), the numerator selector matches nothing → no firing. 0.05 = 5%. + expr: | + ( + sum(rate(gittensory_http_requests_total{status="5xx"}[5m])) + / + sum(rate(gittensory_http_requests_total[5m])) > 0 + ) > 0.05 + for: 10m + labels: + severity: critical + annotations: + summary: "gittensory HTTP 5xx ratio above 5%" + description: "{{ $value | humanizePercentage }} of HTTP responses were 5xx over the last 5m (sustained 10m). The API is throwing server errors." + runbook: "Tail logs for level=error around the request path. Correlate with deploys, DB availability, and AI-provider outages. If 5xx coincides with a target restart, it may be a crash-loop — see GittensoryTargetDown." + + - alert: GittensoryRequestLatencySLOBreach + # p95 request latency from the duration histogram. histogram_quantile over the + # per-le bucket rate gives the 95th percentile end-to-end. 1s p95 is a reasonable + # SLO for a review API; raise for heavier AI-bound endpoints. `by (le)` is + # required for histogram_quantile to work across bucket series. + expr: | + histogram_quantile( + 0.95, + sum(rate(gittensory_http_request_duration_seconds_bucket[5m])) by (le) + ) > 1 + for: 10m + labels: + severity: warning + annotations: + summary: "gittensory p95 request latency above 1s" + description: "p95 HTTP request latency is {{ $value | printf \"%.2f\" }}s over the last 5m (sustained 10m), breaching the 1s SLO." + runbook: "Check whether slowness is queue/DB/AI-bound: correlate with gittensory_queue_pending and Qdrant/AI latency. A rising p95 with flat error rate usually means a saturated dependency, not a bug." diff --git a/promtail/promtail-config.yml b/promtail/promtail-config.yml new file mode 100644 index 0000000000..e46384ffe3 --- /dev/null +++ b/promtail/promtail-config.yml @@ -0,0 +1,77 @@ +# Promtail — scrapes the compose stack's Docker container logs and ships to Loki. +# Verified against the Promtail config schema (Loki release-3.4.x docs; the Promtail +# binary is in LTS and its config is stable against current Loki v3.7.x). +# +# docker_sd_configs talks to the Docker API through the docker-proxy service (a read-only +# Docker-API filter) — NOT a raw socket mount, which would be host-root-equivalent even with :ro. +# It discovers every container in the compose project; we relabel each stream with the compose +# service + container name, then parse our JSON log lines so operators can filter by level and event. + +server: + http_listen_port: 9080 + grpc_listen_port: 0 + log_level: info + +positions: + # Persisted on the promtail-data volume so restarts don't re-ship old logs. + filename: /tmp/positions/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + # Modest backoff/batching for a small host. + batchwait: 1s + batchsize: 1048576 # 1 MiB + backoff_config: + min_period: 500ms + max_period: 5m + max_retries: 10 + +scrape_configs: + - job_name: docker + docker_sd_configs: + # Read-only Docker API exposed by the docker-proxy service (no raw socket in this container). + - host: tcp://docker-proxy:2375 + refresh_interval: 15s + # Only containers in THIS compose project (set automatically by compose). + # Avoids scraping unrelated containers on a shared host. + filters: + - name: label + values: ["com.docker.compose.project"] + + relabel_configs: + # Compose service name -> `service` label (e.g. gittensory, prometheus, loki). + - source_labels: ["__meta_docker_container_label_com_docker_compose_service"] + target_label: "service" + # Compose project -> `compose_project` label. + - source_labels: ["__meta_docker_container_label_com_docker_compose_project"] + target_label: "compose_project" + # Container name -> `container` label (strip the leading "/" Docker prepends). + - source_labels: ["__meta_docker_container_name"] + regex: "/?(.*)" + target_label: "container" + # stdout / stderr -> `stream` label. + - source_labels: ["__meta_docker_container_log_stream"] + target_label: "stream" + # Static job label so streams are easy to select: {job="docker"}. + - target_label: "job" + replacement: "docker" + + pipeline_stages: + # gittensory logs are JSON lines; every line has an "event" field, and only + # error/warn lines set "level". Non-JSON lines (other images) fall through + # untouched because json.expressions on a non-JSON line yields empty values. + - json: + expressions: + level: level + event: event + # Default level to "info" when the field is absent (INFO lines omit it). + - template: + source: level + template: '{{ if .Value }}{{ .Value }}{{ else }}info{{ end }}' + # `level` is low-cardinality -> promote to an index label for fast filtering. + - labels: + level: + # `event` can be many distinct values -> keep it as structured metadata + # (queryable, no stream explosion). Requires allow_structured_metadata in Loki. + - structured_metadata: + event: diff --git a/src/selfhost/metrics.ts b/src/selfhost/metrics.ts index deae0208a6..dde03d487f 100644 --- a/src/selfhost/metrics.ts +++ b/src/selfhost/metrics.ts @@ -1,12 +1,26 @@ // Minimal Prometheus text-format metrics for the self-host runtime (#982 observability). A tiny in-process -// registry — counters (monotonic, incremented at the call site) and gauges (sampled at scrape time via a -// callback, e.g. live queue depth). Rendered at GET /metrics. No deps, no cardinality explosion: callers use -// a small fixed label set. +// registry — counters (monotonic, incremented at the call site), gauges (sampled at scrape time via a +// callback, e.g. live queue depth), and histograms (latency distributions observed at the call site). +// Rendered at GET /metrics. No deps, no cardinality explosion: callers use a small fixed label set. type Labels = Record; type GaugeSample = () => number | Promise; +interface HistogramState { + name: string; + labels: Labels | undefined; + buckets: number[]; // upper bounds (le), ascending + counts: number[]; // cumulative count of observations <= buckets[i] + sum: number; + count: number; +} + const counters = new Map(); const gauges = new Map(); +const histograms = new Map(); + +// Request-latency buckets in seconds (Prometheus convention). Covers sub-ms health checks through +// multi-second webhook processing. Callers may pass their own buckets to observe(). +export const DEFAULT_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]; function seriesKey(name: string, labels?: Labels): string { if (!labels || Object.keys(labels).length === 0) return name; @@ -28,6 +42,22 @@ export function gauge(name: string, sample: GaugeSample): void { gauges.set(name, sample); } +/** Observe a value into a histogram (created on first use). `buckets` must be ascending upper bounds. */ +export function observe(name: string, value: number, labels?: Labels, buckets: number[] = DEFAULT_BUCKETS): void { + const k = seriesKey(name, labels); + let h = histograms.get(k); + if (!h) { + h = { name, labels, buckets, counts: new Array(buckets.length).fill(0), sum: 0, count: 0 }; + histograms.set(k, h); + } + // Cumulative bucketing: bump every bucket whose upper bound is >= the value. + for (let i = 0; i < h.buckets.length; i++) { + if (value <= h.buckets[i]!) h.counts[i]!++; + } + h.sum += value; + h.count += 1; +} + /** Render the registry in Prometheus text exposition format. */ export async function renderMetrics(): Promise { const lines: string[] = []; @@ -39,6 +69,15 @@ export async function renderMetrics(): Promise { /* a failing sampler must not break the scrape */ } } + for (const h of histograms.values()) { + for (let i = 0; i < h.buckets.length; i++) { + lines.push(`${seriesKey(`${h.name}_bucket`, { ...h.labels, le: String(h.buckets[i]) })} ${h.counts[i]}`); + } + // The +Inf bucket equals the total observation count (Prometheus requires it). + lines.push(`${seriesKey(`${h.name}_bucket`, { ...h.labels, le: "+Inf" })} ${h.count}`); + lines.push(`${seriesKey(`${h.name}_sum`, h.labels)} ${h.sum}`); + lines.push(`${seriesKey(`${h.name}_count`, h.labels)} ${h.count}`); + } return `${lines.join("\n")}\n`; } @@ -46,4 +85,5 @@ export async function renderMetrics(): Promise { export function resetMetrics(): void { counters.clear(); gauges.clear(); + histograms.clear(); } diff --git a/src/server.ts b/src/server.ts index 5d3bef42d1..d04c6f9c0c 100644 --- a/src/server.ts +++ b/src/server.ts @@ -17,7 +17,7 @@ import { credentialsToEnv, exchangeManifestCode, renderSetupPage } from "./selfh import { orbEnabled, exportOrbBatch } from "./selfhost/orb-collector"; import { createD1Adapter, nodeSqliteDriver } from "./selfhost/d1-adapter"; import { readiness } from "./selfhost/health"; -import { gauge, incr, renderMetrics } from "./selfhost/metrics"; +import { gauge, incr, observe, renderMetrics } from "./selfhost/metrics"; import { runSelfHostMigrations } from "./selfhost/migrate"; import { createPgAdapter } from "./selfhost/pg-adapter"; import { createPgQueue } from "./selfhost/pg-queue"; @@ -71,6 +71,30 @@ async function waitForPostgres(url: string, maxWaitMs = 30_000): Promise { } } +/** Retry an async readiness operation with backoff until it succeeds (up to maxWaitMs). Prevents a + * crash-restart loop when gittensory starts before a dependency (e.g. Qdrant) is accepting connections — + * Qdrant's init is a single fetch with no retry, so a slow-starting --profile qdrant container would + * otherwise take the whole process down. */ +async function retryUntilReady(name: string, op: () => Promise, maxWaitMs = 30_000): Promise { + const start = Date.now(); + let attempt = 0; + while (true) { + try { + await op(); + return; + } catch (error) { + attempt++; + const elapsed = Date.now() - start; + if (elapsed >= maxWaitMs) { + throw new Error(`${name} not ready after ${maxWaitMs}ms (${attempt} attempts): ${error instanceof Error ? error.message : "unknown error"}`); + } + const delay = Math.min(2000, 200 * attempt); + console.log(JSON.stringify({ event: "selfhost_dependency_wait", dependency: name, attempt, elapsed_ms: elapsed, retry_in_ms: delay })); + await new Promise((r) => setTimeout(r, delay)); + } + } +} + /** Build the Postgres backend (shared DB + queue) when DATABASE_URL is a postgres:// URL. */ async function buildPostgresBackend(url: string, consume: (m: JobMessage) => Promise): Promise { await waitForPostgres(url); @@ -159,7 +183,8 @@ async function main(): Promise { let vectorizeOverride: Vectorize | undefined; if (process.env.QDRANT_URL) { const { createQdrantVectorize, initQdrantCollection } = await import("./selfhost/qdrant-vectorize"); - await initQdrantCollection(process.env.QDRANT_URL); + // Retry until Qdrant accepts the collection PUT — the container may still be booting when we start. + await retryUntilReady("qdrant", () => initQdrantCollection(process.env.QDRANT_URL as string)); vectorizeOverride = createQdrantVectorize(process.env.QDRANT_URL); console.log(JSON.stringify({ event: "selfhost_vectorize", backend: "qdrant" })); } @@ -185,11 +210,14 @@ async function main(): Promise { for (const c of [ "gittensory_jobs_enqueued_total", "gittensory_jobs_processed_total", "gittensory_jobs_failed_total", "gittensory_jobs_dead_total", - "gittensory_http_requests_total", "gittensory_webhook_dedup_total", + "gittensory_webhook_dedup_total", "gittensory_qdrant_queries_total", "gittensory_qdrant_upserts_total", "gittensory_orb_events_exported_total", "gittensory_orb_export_errors_total", ]) incr(c, undefined, 0); + // Seed gittensory_http_requests_total per status class so the breakdown panel has every series from the + // first scrape (keeping the metric consistently labeled — never mix labeled and unlabeled samples). + for (const status of ["2xx", "3xx", "4xx", "5xx"]) incr("gittensory_http_requests_total", { status }, 0); const ctx = { waitUntil: (p: Promise) => void Promise.resolve(p).catch(() => undefined), @@ -250,7 +278,13 @@ async function main(): Promise { return new Response(`setup failed: ${error instanceof Error ? error.message : "error"}`, { status: 500 }); } } - incr("gittensory_http_requests_total"); + // Instrument real app traffic — status-class counter + latency histogram. (Infra endpoints + // /health /ready /metrics and the setup wizard already returned above and are not counted.) + const startedReq = Date.now(); + const record = (status: number): void => { + incr("gittensory_http_requests_total", { status: `${Math.floor(status / 100)}xx` }); + observe("gittensory_http_request_duration_seconds", (Date.now() - startedReq) / 1000); + }; // Webhook delivery dedup: return 204 immediately for already-processed delivery IDs. // We mark only AFTER a successful response — failed/rejected webhooks must be retryable. const isWebhook = webhookCache && path === "/v1/github/webhook" && request.method === "POST"; @@ -259,6 +293,7 @@ async function main(): Promise { const seen = await webhookCache!.get(`delivery:${deliveryId}`); if (seen) { incr("gittensory_webhook_dedup_total"); + record(204); return new Response(null, { status: 204 }); } } @@ -267,6 +302,7 @@ async function main(): Promise { // Best-effort — never block the response on a cache write failure void webhookCache!.set(`delivery:${deliveryId}`, "1", 300).catch(() => undefined); } + record(response.status); return response; }, port, @@ -291,7 +327,7 @@ async function main(): Promise { const runExport = () => exportOrbBatch(backend.db) .then((n) => { if (n > 0) console.log(JSON.stringify({ event: "selfhost_orb_export", exported: n })); }) - .catch(() => undefined); + .catch((error) => console.error(JSON.stringify({ level: "error", event: "selfhost_orb_export_error", error: error instanceof Error ? error.message : "unknown error" }))); void runExport(); // flush any pending events from a previous run at startup setInterval(runExport, 3_600_000); // then hourly } diff --git a/test/unit/selfhost-metrics.test.ts b/test/unit/selfhost-metrics.test.ts index dad9907341..4b899e622a 100644 --- a/test/unit/selfhost-metrics.test.ts +++ b/test/unit/selfhost-metrics.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it } from "vitest"; -import { gauge, incr, renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; +import { gauge, incr, observe, renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; afterEach(() => resetMetrics()); @@ -36,3 +36,48 @@ describe("metrics registry (#982)", () => { expect((await renderMetrics())).toContain("ok_total 1"); }); }); + +describe("histograms (observe)", () => { + it("renders cumulative buckets, +Inf, sum and count (default buckets)", async () => { + observe("rq_seconds", 2); // 2 <= 2.5/5/10 but > 1 + const out = await renderMetrics(); + expect(out).toContain('rq_seconds_bucket{le="1"} 0'); // below the value → not counted + expect(out).toContain('rq_seconds_bucket{le="2.5"} 1'); // first bucket >= value + expect(out).toContain('rq_seconds_bucket{le="+Inf"} 1'); + expect(out).toContain("rq_seconds_sum 2"); + expect(out).toContain("rq_seconds_count 1"); + }); + + it("accumulates across observations into an existing series", async () => { + observe("a_seconds", 0.01); + observe("a_seconds", 0.01); // second observe hits the existing-series branch + const out = await renderMetrics(); + expect(out).toContain('a_seconds_bucket{le="0.005"} 0'); // both observations are above 0.005 + expect(out).toContain('a_seconds_bucket{le="0.01"} 2'); // both <= 0.01 + expect(out).toContain("a_seconds_count 2"); + expect(out).toContain("a_seconds_sum 0.02"); + }); + + it("honors a caller-provided bucket set", async () => { + observe("c_seconds", 7, undefined, [1, 5, 10]); + const out = await renderMetrics(); + expect(out).toContain('c_seconds_bucket{le="5"} 0'); + expect(out).toContain('c_seconds_bucket{le="10"} 1'); + expect(out).toContain('c_seconds_bucket{le="+Inf"} 1'); + expect(out).toContain("c_seconds_sum 7"); + }); + + it("renders labels on every histogram series", async () => { + observe("l_seconds", 0.001, { route: "health" }); + const out = await renderMetrics(); + expect(out).toContain('l_seconds_bucket{le="0.005",route="health"} 1'); + expect(out).toContain('l_seconds_sum{route="health"} 0.001'); + expect(out).toContain('l_seconds_count{route="health"} 1'); + }); + + it("resetMetrics clears histograms", async () => { + observe("z_seconds", 1); + resetMetrics(); + expect(await renderMetrics()).toBe("\n"); + }); +});