diff --git a/.env.example b/.env.example index 9d66eb8fbb..afb47dd356 100644 --- a/.env.example +++ b/.env.example @@ -279,6 +279,7 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # BROWSERLESS_QUEUED=4 # (--profile visual-review) max requests waiting for a session. # BROWSERLESS_TIMEOUT_MS=30000 # (--profile visual-review) per-session timeout. # BROWSERLESS_MEM_LIMIT=2g # (--profile visual-review) container memory cap. +# BROWSERLESS_EXPORTER_MEM_LIMIT=64m # (--profile visual-review) browserless-exporter sidecar cap (#5368) # REVIEW_AUDIT_DIR= # persist visual-review screenshot PNGs to this filesystem path # # instead of re-rendering on demand. Unset = re-render each time. # # Only relevant when BROWSER_WS_ENDPOINT above is set. Ignored diff --git a/docker-compose.yml b/docker-compose.yml index 128af1d23a..8d0c91ad7c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -553,6 +553,40 @@ services: start_period: 15s retries: 5 + # browserless's own /metrics?token= endpoint returns JSON (a rolling history of 5-minute-window + # snapshots), not Prometheus text -- this sidecar polls it and re-exposes the newest snapshot's fields as + # real Prometheus gauges (#5368), same "translate non-Prometheus output" pattern as backup-exporter above. + # Gated on the same --profile visual-review as browserless itself; meaningless without it running. + browserless-exporter: + image: alpine:3.20 + restart: unless-stopped + <<: *default-logging + profiles: ["visual-review"] + depends_on: + browserless: + condition: service_healthy + environment: + BROWSERLESS_METRICS_URL: "http://browserless:3000/metrics" + BROWSERLESS_TOKEN: "${BROWSERLESS_TOKEN:-}" + volumes: + # Directory mount, not a single-file one -- same #reporting-exporter-stale-bind-mount rationale as + # the backup-exporter service above. + - ./scripts:/scripts:ro + expose: + - "9102" + command: + - /bin/sh + - -c + - "apk add --no-cache jq busybox-extras >/dev/null 2>&1 && sh /scripts/browserless-metrics.sh" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9102/metrics | grep -q '^browserless_exporter_last_scrape_success'"] + interval: 30s + retries: 5 + deploy: + resources: + limits: + memory: "${BROWSERLESS_EXPORTER_MEM_LIMIT:-64m}" + # ── REES / review-enrichment (--profile rees) ────────────────────────────── # Self-hosted alternative to pointing REES_URL at a managed/external instance (see the self-hosting docs). # REES runs the heavier PR analysis (dependency CVEs, secret scan, license, EOL runtime, lockfile drift, IaC diff --git a/grafana/dashboards/browserless-metrics.json b/grafana/dashboards/browserless-metrics.json new file mode 100644 index 0000000000..0b224780b7 --- /dev/null +++ b/grafana/dashboards/browserless-metrics.json @@ -0,0 +1,129 @@ +{ + "__inputs": [], + "__requires": [ + { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.0.0" }, + { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } + ], + "annotations": { "list": [] }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "description": "browserless (headless-Chromium visual-review screenshot capture, --profile visual-review, #5368) queue depth, concurrency, and rejection/error/timeout rate, translated from its own JSON /metrics by the browserless-exporter sidecar. Absent/empty unless the visual-review profile is active.", + "panels": [ + { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "title": "Exporter health", "type": "row" }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "FAILED", "color": "red" }, "1": { "text": "OK", "color": "green" } } } + ], + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, + "noValue": "no data" + } + }, + "gridPos": { "h": 5, "w": 8, "x": 0, "y": 1 }, + "id": 2, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Last scrape of browserless's own /metrics", + "description": "0 means the exporter's most recent poll of browserless itself failed (wrong token, browserless down, or network issue) — the panels below are then serving stale data.", + "type": "stat", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "browserless_exporter_last_scrape_success", "legendFormat": "scrape ok", "refId": "A" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "dtdurations" } }, + "gridPos": { "h": 5, "w": 8, "x": 8, "y": 1 }, + "id": 3, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Time since browserless's own sample", + "description": "browserless_sample_timestamp_seconds is the timestamp browserless itself assigned to its newest 5-minute window. A large, growing value here means browserless has stopped reporting new windows even if the exporter's own scrape is succeeding.", + "type": "stat", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "time() - browserless_sample_timestamp_seconds", "legendFormat": "age", "refId": "A" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "ms" } }, + "gridPos": { "h": 5, "w": 8, "x": 16, "y": 1 }, + "id": 4, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, + "title": "Mean session time", + "type": "stat", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "browserless_session_mean_time_ms", "legendFormat": "mean", "refId": "A" }] + }, + + { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, "id": 10, "title": "Queue, concurrency & outcomes", "type": "row" }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 } } }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, + "id": 11, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "title": "Queue depth & concurrency", + "description": "queued: sessions waiting over the configured concurrency limit. running: sessions active at the sample point. max_concurrent: peak concurrent sessions in the window.", + "type": "timeseries", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "browserless_queued", "legendFormat": "queued", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "browserless_running", "legendFormat": "running", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "browserless_max_concurrent", "legendFormat": "max_concurrent", "refId": "C" } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 } } }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, + "id": 12, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "title": "Session outcomes by window", + "description": "Per-5-minute-window counts from browserless's own rolling buffer — a non-zero, sustained rejected/error/timeout/unhealthy count means capture is degrading, not just occasionally flaky.", + "type": "timeseries", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "browserless_successful", "legendFormat": "successful", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "browserless_rejected", "legendFormat": "rejected", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "browserless_errors", "legendFormat": "error", "refId": "C" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "browserless_timedout", "legendFormat": "timedout", "refId": "D" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "browserless_unauthorized", "legendFormat": "unauthorized", "refId": "E" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "browserless_unhealthy", "legendFormat": "unhealthy", "refId": "F" } + ] + }, + + { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 }, "id": 20, "title": "Resource utilization", "type": "row" }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "percentunit", "max": 1, "min": 0 } }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "id": 21, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "title": "CPU utilization", + "type": "timeseries", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "browserless_cpu_ratio", "legendFormat": "cpu", "refId": "A" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "percentunit", "max": 1, "min": 0 } }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "id": 22, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "title": "Memory utilization", + "type": "timeseries", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "browserless_memory_ratio", "legendFormat": "memory", "refId": "A" }] + } + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["gittensory", "browserless", "observability"], + "templating": { + "list": [ + { "current": {}, "hide": 0, "includeAll": false, "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "refresh": 1, "type": "datasource" } + ] + }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Gittensory — Browserless (visual review)", + "uid": "gittensory-browserless", + "version": 1 +} diff --git a/grafana/dashboards/resource-hub.json b/grafana/dashboards/resource-hub.json index c20c3eb4a8..c90e895c83 100644 --- a/grafana/dashboards/resource-hub.json +++ b/grafana/dashboards/resource-hub.json @@ -66,7 +66,7 @@ "gridPos": { "h": 13, "w": 12, "x": 12, "y": 0 }, "options": { "mode": "markdown", - "content": "## 📊 Dashboards\n- **[Upstream PRs & issues (GitHub)](/d/gittensory-github)** — live, accurate census + open-PR triage (GitHub API).\n- **[Reviews & PRs (maintainer)](/d/gittensory-maintainer)** — gittensory's own review activity + reviewed-PR log.\n- **[AI usage](/d/gittensory-ai-usage)** — durable cross-provider ai_usage_events (filterable by provider/feature/model), live Prometheus counters, and Claude Code's own OTEL session telemetry, all in one place.\n- **[Gittensory (infra)](/d/gittensory)** — queue, jobs, HTTP, GitHub API cache/rate limits.\n- **[GPU metrics](/d/gittensory-gpu)** — utilization/VRAM for a self-hosted Ollama GPU box.\n- **[Infra health](/d/gittensory-infra-health)** — host CPU/mem/disk/network (node-exporter), per-container resource usage (cAdvisor), Redis, Qdrant, and whether the observability stack itself is up.\n- **[REES (review-enrichment)](/d/gittensory-rees)** — request outcomes/latency and per-analyzer run/timeout/degrade rates for the optional `--profile rees` service.\n\n## 📈 Metrics & logs\n- **Prometheus** — [targets](http://localhost:9090/targets) · [graph](http://localhost:9090)\n- **Alertmanager** — [alerts](http://localhost:9093)\n- **Loki** — query in [Explore](/explore) (pick the *Loki* datasource), e.g. `{compose_service=\"gittensory\"}`\n- **Sentry** — release/source-map enriched errors. Edit the dashboard link if your project URL differs.\n\n## 🩺 Quick health checks\n| What | Where |\n|---|---|\n| App serving | `GET /ready` → 200 |\n| AI wired | boot log `selfhost_ai_provider` |\n| Embeds wired | boot log `selfhost_embed_provider` |\n| Vectors wired | boot log `selfhost_vectorize` |\n| Token spend | **[AI usage](/d/gittensory-ai-usage)** dashboard |\n\n## 📚 Docs\n- [Maintainer self-hosting](https://gittensory.aethereal.dev/docs/maintainer-self-hosting) — setup, configuration, AI, REES, RAG, operations, and troubleshooting." + "content": "## 📊 Dashboards\n- **[Upstream PRs & issues (GitHub)](/d/gittensory-github)** — live, accurate census + open-PR triage (GitHub API).\n- **[Reviews & PRs (maintainer)](/d/gittensory-maintainer)** — gittensory's own review activity + reviewed-PR log.\n- **[AI usage](/d/gittensory-ai-usage)** — durable cross-provider ai_usage_events (filterable by provider/feature/model), live Prometheus counters, and Claude Code's own OTEL session telemetry, all in one place.\n- **[Gittensory (infra)](/d/gittensory)** — queue, jobs, HTTP, GitHub API cache/rate limits.\n- **[GPU metrics](/d/gittensory-gpu)** — utilization/VRAM for a self-hosted Ollama GPU box.\n- **[Infra health](/d/gittensory-infra-health)** — host CPU/mem/disk/network (node-exporter), per-container resource usage (cAdvisor), Redis, Qdrant, and whether the observability stack itself is up.\n- **[REES (review-enrichment)](/d/gittensory-rees)** — request outcomes/latency and per-analyzer run/timeout/degrade rates for the optional `--profile rees` service.\n- **[Browserless (visual review)](/d/gittensory-browserless)** — queue depth, concurrency, and rejection/error/timeout rate for the optional `--profile visual-review` screenshot service.\n\n## 📈 Metrics & logs\n- **Prometheus** — [targets](http://localhost:9090/targets) · [graph](http://localhost:9090)\n- **Alertmanager** — [alerts](http://localhost:9093)\n- **Loki** — query in [Explore](/explore) (pick the *Loki* datasource), e.g. `{compose_service=\"gittensory\"}`\n- **Sentry** — release/source-map enriched errors. Edit the dashboard link if your project URL differs.\n\n## 🩺 Quick health checks\n| What | Where |\n|---|---|\n| App serving | `GET /ready` → 200 |\n| AI wired | boot log `selfhost_ai_provider` |\n| Embeds wired | boot log `selfhost_embed_provider` |\n| Vectors wired | boot log `selfhost_vectorize` |\n| Token spend | **[AI usage](/d/gittensory-ai-usage)** dashboard |\n\n## 📚 Docs\n- [Maintainer self-hosting](https://gittensory.aethereal.dev/docs/maintainer-self-hosting) — setup, configuration, AI, REES, RAG, operations, and troubleshooting." } } ] diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index a664e2af09..99e9565e08 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -90,6 +90,15 @@ scrape_configs: - targets: ["rees:8080"] scrape_interval: 15s + # browserless queue/concurrency/rejection/error metrics, translated from its own JSON /metrics by the + # browserless-exporter sidecar (--profile visual-review, #5368). Same BusyBox-httpd-wrapper shape as + # gittensory-backup above, so it needs the same explicit scrape protocol fallback. + - job_name: browserless + fallback_scrape_protocol: PrometheusText0.0.4 + static_configs: + - targets: ["browserless-exporter:9102"] + scrape_interval: 30s + # Self-monitoring: the observability stack watching itself (--profile observability, #5366). Every # component here ships its own native Prometheus /metrics — previously scraped by nothing, so a stack # component silently degrading (e.g. Loki dropping ingested lines, Alertmanager failing to notify) had diff --git a/scripts/browserless-metrics.sh b/scripts/browserless-metrics.sh new file mode 100644 index 0000000000..06ad20217a --- /dev/null +++ b/scripts/browserless-metrics.sh @@ -0,0 +1,110 @@ +#!/bin/sh +# Translate browserless's JSON /metrics endpoint into Prometheus text format (#5368). browserless's own +# /metrics?token= returns a rolling history of 5-minute-window snapshots (an array, oldest first) -- +# this exporter polls it, takes only the NEWEST snapshot (the array's last element), and re-exposes its +# fields as Prometheus gauges. Gauges, not counters: each field is a per-window sample from browserless's own +# rolling buffer, not a monotonic total, so it can legitimately go up or down between scrapes. +# Intended for the browserless-exporter compose sidecar; it never touches the app's own state. +set -eu + +URL=${BROWSERLESS_METRICS_URL:-http://browserless:3000/metrics} +TOKEN=${BROWSERLESS_TOKEN:-} +OUT=${BROWSERLESS_METRICS_DIR:-/metrics} +FILE=${BROWSERLESS_METRICS_FILE:-$OUT/metrics} +INTERVAL=${BROWSERLESS_METRICS_INTERVAL_SECONDS:-30} +PORT=${BROWSERLESS_METRICS_PORT:-9102} + +case "$INTERVAL" in + ''|*[!0-9]*) INTERVAL=30 ;; +esac + +write_metrics() { + mkdir -p "$OUT" + tmp="$FILE.tmp" + now=$(date +%s) + raw=$(wget -qO- "$URL?token=$TOKEN" 2>/dev/null || true) + latest=$(printf '%s' "$raw" | jq -c '.[-1] // empty' 2>/dev/null || true) + + { + echo "# HELP browserless_exporter_last_scrape_success Whether the most recent poll of browserless's own /metrics succeeded (1) or failed (0)." + echo "# TYPE browserless_exporter_last_scrape_success gauge" + echo "# HELP browserless_exporter_last_scrape_timestamp_seconds Unix timestamp of the most recent poll attempt, successful or not." + echo "# TYPE browserless_exporter_last_scrape_timestamp_seconds gauge" + + if [ -n "$latest" ]; then + echo "browserless_exporter_last_scrape_success 1" + echo "browserless_exporter_last_scrape_timestamp_seconds $now" + + echo "# HELP browserless_queued Sessions queued (over browserless's concurrency limit) in the most recent 5-minute window." + echo "# TYPE browserless_queued gauge" + echo "browserless_queued $(printf '%s' "$latest" | jq '.queued // 0')" + + echo "# HELP browserless_running Sessions currently running at the time of the most recent window sample." + echo "# TYPE browserless_running gauge" + echo "browserless_running $(printf '%s' "$latest" | jq '.running // 0')" + + echo "# HELP browserless_max_concurrent Peak concurrent sessions observed in the most recent 5-minute window." + echo "# TYPE browserless_max_concurrent gauge" + echo "browserless_max_concurrent $(printf '%s' "$latest" | jq '.maxConcurrent // 0')" + + echo "# HELP browserless_rejected Sessions rejected (over capacity) in the most recent 5-minute window." + echo "# TYPE browserless_rejected gauge" + echo "browserless_rejected $(printf '%s' "$latest" | jq '.rejected // 0')" + + echo "# HELP browserless_errors Sessions that errored in the most recent 5-minute window." + echo "# TYPE browserless_errors gauge" + echo "browserless_errors $(printf '%s' "$latest" | jq '.error // 0')" + + echo "# HELP browserless_timedout Sessions that timed out in the most recent 5-minute window." + echo "# TYPE browserless_timedout gauge" + echo "browserless_timedout $(printf '%s' "$latest" | jq '.timedout // 0')" + + echo "# HELP browserless_unauthorized Unauthorized session requests (bad/missing token) in the most recent 5-minute window." + echo "# TYPE browserless_unauthorized gauge" + echo "browserless_unauthorized $(printf '%s' "$latest" | jq '.unauthorized // 0')" + + echo "# HELP browserless_unhealthy Sessions marked unhealthy in the most recent 5-minute window." + echo "# TYPE browserless_unhealthy gauge" + echo "browserless_unhealthy $(printf '%s' "$latest" | jq '.unhealthy // 0')" + + echo "# HELP browserless_successful Sessions that completed successfully in the most recent 5-minute window." + echo "# TYPE browserless_successful gauge" + echo "browserless_successful $(printf '%s' "$latest" | jq '.successful // 0')" + + echo "# HELP browserless_session_mean_time_ms Mean session duration in milliseconds over the most recent 5-minute window." + echo "# TYPE browserless_session_mean_time_ms gauge" + echo "browserless_session_mean_time_ms $(printf '%s' "$latest" | jq '.meanTime // 0')" + + echo "# HELP browserless_cpu_ratio Host CPU utilization ratio (0-1) sampled at the most recent window." + echo "# TYPE browserless_cpu_ratio gauge" + echo "browserless_cpu_ratio $(printf '%s' "$latest" | jq '.cpu // 0')" + + echo "# HELP browserless_memory_ratio Host memory utilization ratio (0-1) sampled at the most recent window." + echo "# TYPE browserless_memory_ratio gauge" + echo "browserless_memory_ratio $(printf '%s' "$latest" | jq '.memory // 0')" + + echo "# HELP browserless_sample_timestamp_seconds Unix timestamp browserless itself assigned to the most recent window sample." + echo "# TYPE browserless_sample_timestamp_seconds gauge" + echo "browserless_sample_timestamp_seconds $(printf '%s' "$latest" | jq '(.date // 0) / 1000')" + else + echo "browserless_exporter_last_scrape_success 0" + echo "browserless_exporter_last_scrape_timestamp_seconds $now" + fi + } > "$tmp" + mv "$tmp" "$FILE" +} + +if [ "${BROWSERLESS_METRICS_ONCE:-}" = "1" ]; then + write_metrics + exit 0 +fi + +write_metrics +httpd -f -p "$PORT" -h "$OUT" & +server=$! +trap 'kill "$server" 2>/dev/null || true' INT TERM EXIT + +while true; do + sleep "$INTERVAL" + write_metrics +done diff --git a/test/unit/selfhost-compose-resource-limits.test.ts b/test/unit/selfhost-compose-resource-limits.test.ts index 3e58b169d7..501a1e5f8d 100644 --- a/test/unit/selfhost-compose-resource-limits.test.ts +++ b/test/unit/selfhost-compose-resource-limits.test.ts @@ -32,6 +32,7 @@ describe("docker-compose.yml — per-service memory limits (#1828, #2495, #3893) "node-exporter": "${NODE_EXPORTER_MEM_LIMIT:-128m}", cadvisor: "${CADVISOR_MEM_LIMIT:-256m}", "redis-exporter": "${REDIS_EXPORTER_MEM_LIMIT:-64m}", + "browserless-exporter": "${BROWSERLESS_EXPORTER_MEM_LIMIT:-64m}", }; it("caps the core app and every heavyweight optional service with an operator-overridable memory limit", () => { @@ -67,6 +68,7 @@ describe("docker-compose.yml — per-service memory limits (#1828, #2495, #3893) "NODE_EXPORTER_MEM_LIMIT", "CADVISOR_MEM_LIMIT", "REDIS_EXPORTER_MEM_LIMIT", + "BROWSERLESS_EXPORTER_MEM_LIMIT", ]) { expect(env, key).toContain(key); } diff --git a/test/unit/selfhost-grafana-browserless-dashboard.test.ts b/test/unit/selfhost-grafana-browserless-dashboard.test.ts new file mode 100644 index 0000000000..aaa865b9c4 --- /dev/null +++ b/test/unit/selfhost-grafana-browserless-dashboard.test.ts @@ -0,0 +1,79 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +type DashboardTarget = { expr?: string; legendFormat?: string }; +type DashboardPanel = { + id?: number; + title?: string; + type?: string; + description?: string; + datasource?: { type?: string; uid?: string }; + targets?: DashboardTarget[]; +}; +type Dashboard = { + uid: string; + title: string; + tags: string[]; + panels: DashboardPanel[]; +}; + +const dashboardPath = join(process.cwd(), "grafana/dashboards/browserless-metrics.json"); + +function readDashboard(): Dashboard { + return JSON.parse(readFileSync(dashboardPath, "utf8")) as Dashboard; +} + +function allTargets(dashboard = readDashboard()): DashboardTarget[] { + return dashboard.panels.flatMap((panel) => panel.targets ?? []); +} + +describe("Gittensory — Browserless (visual review) dashboard (#5368)", () => { + it("declares the expected uid/title/tags", () => { + const dashboard = readDashboard(); + expect(dashboard.uid).toBe("gittensory-browserless"); + expect(dashboard.title).toBe("Gittensory — Browserless (visual review)"); + expect(dashboard.tags).toEqual(["gittensory", "browserless", "observability"]); + }); + + it("every panel target uses the Prometheus datasource variable, never a hardcoded uid", () => { + const dashboard = readDashboard(); + for (const panel of dashboard.panels) { + if (panel.type === "row") continue; + expect(panel.datasource?.type, panel.title).toBe("prometheus"); + expect(panel.datasource?.uid, panel.title).toBe("${DS_PROMETHEUS}"); + for (const target of panel.targets ?? []) { + expect(target.expr, panel.title).toBeTruthy(); + } + } + }); + + it("uses the real metric names browserless-metrics.sh emits", () => { + const targets = allTargets(); + const expectedMetrics = [ + "browserless_exporter_last_scrape_success", + "browserless_sample_timestamp_seconds", + "browserless_queued", + "browserless_running", + "browserless_max_concurrent", + "browserless_rejected", + "browserless_errors", + "browserless_timedout", + "browserless_unauthorized", + "browserless_unhealthy", + "browserless_successful", + "browserless_session_mean_time_ms", + "browserless_cpu_ratio", + "browserless_memory_ratio", + ]; + for (const metric of expectedMetrics) { + expect(targets.some((t) => t.expr?.includes(metric)), metric).toBe(true); + } + }); + + it("surfaces exporter-scrape health separately from browserless's own sample freshness", () => { + const targets = allTargets(); + expect(targets.some((t) => t.expr === "browserless_exporter_last_scrape_success")).toBe(true); + expect(targets.some((t) => t.expr === "time() - browserless_sample_timestamp_seconds")).toBe(true); + }); +}); diff --git a/test/unit/selfhost-observability-config.test.ts b/test/unit/selfhost-observability-config.test.ts index d4d578fd43..09108994c9 100644 --- a/test/unit/selfhost-observability-config.test.ts +++ b/test/unit/selfhost-observability-config.test.ts @@ -271,4 +271,52 @@ describe("self-host observability trace config", () => { ]), ); }); + + it("translates browserless's JSON /metrics into Prometheus text via a sidecar, gated on --profile visual-review (#5368)", () => { + const compose = record(readYaml("docker-compose.yml")); + const services = record(compose.services); + const browserlessExporter = record(services["browserless-exporter"]); + const prometheus = record(readYaml("prometheus/prometheus.yml")); + const scrapeConfigs = prometheus.scrape_configs as Array>; + + expect(browserlessExporter.image).toBe("alpine:3.20"); + expect(browserlessExporter.profiles).toEqual(["visual-review"]); + expect(browserlessExporter.depends_on?.browserless).toEqual({ condition: "service_healthy" }); + expect(browserlessExporter.environment).toMatchObject({ + BROWSERLESS_METRICS_URL: "http://browserless:3000/metrics", + BROWSERLESS_TOKEN: "${BROWSERLESS_TOKEN:-}", + }); + expect(browserlessExporter.expose).toEqual(["9102"]); + expect(browserlessExporter.command).toEqual([ + "/bin/sh", + "-c", + "apk add --no-cache jq busybox-extras >/dev/null 2>&1 && sh /scripts/browserless-metrics.sh", + ]); + expect(browserlessExporter.healthcheck?.test).toEqual([ + "CMD-SHELL", + "wget -qO- http://127.0.0.1:9102/metrics | grep -q '^browserless_exporter_last_scrape_success'", + ]); + + expect(scrapeConfigs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + job_name: "browserless", + fallback_scrape_protocol: "PrometheusText0.0.4", + static_configs: [{ targets: ["browserless-exporter:9102"] }], + }), + ]), + ); + }); + + it("ships the browserless-metrics.sh translator with the newest-window-wins + scrape-failure-safe shape it claims", () => { + const script = readFileSync(join(process.cwd(), "scripts/browserless-metrics.sh"), "utf8"); + + // Picks the LAST array element (browserless's rolling history is oldest-first), not the first or a merge. + expect(script).toContain("jq -c '.[-1] // empty'"); + // A failed poll must not corrupt/blank the previously-served file: written atomically via tmp+mv, and the + // failure branch still emits a valid, parseable metrics document (just success=0) rather than an empty body. + expect(script).toContain('mv "$tmp" "$FILE"'); + expect(script).toContain("browserless_exporter_last_scrape_success 0"); + expect(script).toContain("browserless_exporter_last_scrape_success 1"); + }); });