From 6d0b06574b4dba6bd34d720f8ed2657d88963353 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:40:25 -0700 Subject: [PATCH 1/2] docs(ops): add runbooks for rate-limit, cache, Qdrant, Orb, and AI provider recovery A prior investigation found the dashboards, alert rules (13 groups, ~21 rules), and metrics for #1943's requested categories were already substantially complete -- the genuine gap was runbook coverage for five specific failure modes an operator can hit but had no documented recovery path for. Adds five new troubleshooting sections, each grounded in the exact metric names, label vocabularies, and alert names the code actually emits (cross- checked against src/selfhost/metrics.ts call sites and prometheus/rules/ alerts.yml, not inferred from the issue text): - GitHub rate-limit responses and admission deferrals, with PromQL to break a spike down by kind/key_scope/job_type and distinguish an expected brief admission hold from a sustained problem. - Low GitHub response-cache hit rate, with a PromQL hit-rate query by endpoint class. - Qdrant/vector-store errors, including the dimension-mismatch-on- embedding-model-change scenario and how to recreate a collection. - Orb export/relay reconciliation, pointing at the export loop's Sentry cron monitor as the fastest first check. - The AI provider circuit breaker (#2540): what it does, that it self-heals in 60s with no manual reset, and how to tell a transient trip from a persistent credential/reachability problem. --- .../docs.self-hosting-troubleshooting.tsx | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx b/apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx index 190d11087d..0fe1f299ad 100644 --- a/apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx +++ b/apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx @@ -121,6 +121,141 @@ rees_analyzer_config_invalid`} docker compose logs gittensory | grep selfhost_job_dead`} /> +

GitHub rate-limit responses or admission deferrals

+

+ Two independent signals cover this:{" "} + gittensory_github_rest_rate_limit_responses_total counts actual 403/429 + responses from GitHub, and the{" "} + gittensory_jobs_rate_limit_admission_deferred_total /{" "} + gittensory_jobs_rate_limit_budget_deferred_total /{" "} + gittensory_jobs_rate_limited_by_type_total counters track jobs the queue itself + held back before making a request, to avoid tripping a limit. All three job-side + counters carry the same three labels — kind (webhook or{" "} + background), key_scope (installation,{" "} + public, global, or other), and job_type{" "} + (the queue job's type, e.g. agent-regate-pr) — so you can break a spike down to + exactly which token pool and which job type is under pressure. +

+

+ A short burst of deferrals is expected and self-resolving: the queue is deliberately trading + a few seconds of delay to avoid a real 429. Treat it as a real problem only once it's + sustained — which is exactly what{" "} + GittensoryGitHubRateLimitResponses (real 403/429s observed) and{" "} + GittensoryQueueRateLimitDeferralsHigh (a sustained deferral rate, not a blip) + are tuned to alert on, rather than firing on every brief admission hold. +

+ +

+ If a single key_scope=installation pool is consistently the bottleneck, the fix + is usually spreading load across more installation tokens (fewer repos per installation) or + raising the GitHub App's own rate-limit tier, not code changes here. +

+ +

Low GitHub response-cache hit rate

+

+ gittensory_github_response_cache_total (REST) and{" "} + gittensory_github_graphql_cache_total (GraphQL) both carry a{" "} + result label — hit, miss, set,{" "} + coalesced, bypassed, or error — and a{" "} + class label identifying the endpoint family. A healthy cache should show most + traffic as hit for endpoints that are read repeatedly in one review/maintenance + pass (PR reads, check-run lookups); a low hit rate on those specific classes, not the + overall average, is the useful signal. +

+ + +

Qdrant / vector-store errors

+

+ gittensory_qdrant_errors_total carries an op label ( + upsert, query, or delete) so you can tell whether + indexing or retrieval is failing. GittensoryQdrantErrorRateHigh fires on a + sustained error ratio, not an isolated blip. +

+ + + +

Orb export or relay problems

+

+ For brokered self-host deployments, gittensory_orb_events_exported_total and{" "} + gittensory_orb_export_errors_total track the hourly outcome-export loop;{" "} + GittensoryOrbExportErrorRateHigh fires on a sustained error ratio there. The + pull-mode relay loop (for installations receiving events outbound from Orb) reports through{" "} + gittensory_orb_relay_drains_total (result=events when it drained + something, result=empty otherwise) and{" "} + gittensory_orb_webhook_total (event + result labels) + for what happened to each relayed event once enqueued locally. +

+

+ If exports are failing but the relay itself looks healthy, the export loop's Sentry + cron monitor (see Self-host operations) is + the fastest way to confirm whether the loop is even running, before digging into the error + counters. +

+ +

AI provider circuit breaker keeps opening

+

+ Each AI provider (self-host AI_PROVIDER entries) has its own circuit breaker: + after 3 consecutive failures it stops attempting real calls to that provider for 60 seconds, + recorded as gittensory_ai_provider_circuit_open_total{'{provider="..."}'}{" "} + (skipped calls) alongside{" "} + gittensory_ai_provider_failures_total{'{provider="..."}'} (real failures). It + self-heals automatically — there is no manual reset — but it will reopen immediately if the + underlying problem is still there. +

+
    +
  • + Search logs for circuit_open: provider "..." to confirm which provider + tripped, and selfhost_ai_provider_failed_in_chain for the real error each + failed attempt hit before the breaker opened. +
    • +
  • + A provider that keeps re-tripping after its cooldown almost always means a persistent + problem, not a transient blip: an expired/invalid API key, a CLI binary missing from the + image (see selfhost_ai_cli_missing at boot), or the endpoint being genuinely + unreachable from the container. +
    • +
  • + GittensoryAiProviderCircuitOpen fires on any circuit-open event in a + 15-minute window — a single trip during a real but brief outage is expected; a rule that + keeps firing across multiple windows points at the persistent case above. +
    • +
+

Grafana traces error or show no data

The trace path is app or smoke process → OTEL collector → Tempo → Grafana. Tempo is only From cc404f7bde731027ee431d0a6637b6c02e7291fd Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 2 Jul 2026 16:11:24 -0700 Subject: [PATCH 2/2] docs(nav): nest self-hosting under Maintainers, fix gate review nits Gate review round on PR #2638 flagged three real content gaps, all fixed: - Route meta description didn't mention the 5 new runbook topics. - Only the REST cache hit-rate PromQL was shown despite documenting both REST and GraphQL cache metrics -- added the GraphQL equivalent. - The Qdrant collection-drop guidance didn't name the fixed collection ("gittensory", hard-coded, not env-configurable) or warn that dropping it temporarily removes ALL indexed RAG context until reindexing completes. Also adds a drift-guard test (mirrors #2556's check-openapi-settings- parity.mjs pattern): every gittensory_*_total metric name and Gittensory* alert name referenced in the troubleshooting doc is cross-checked against the actual source files and prometheus/rules/alerts.yml, so a future rename/removal fails this test instead of the docs silently going stale. Mutation-tested (a deliberately wrong metric name correctly fails it). Separately: the self-hosting docs section was flat (13 pages under one sidebar heading) and sat as its own top-level nav category alongside "Maintainers" -- misleadingly implying self-hosting is an alternative to, rather than a maintainer concern under, "Maintainers". Nests it as 4 sub-categories (setup / integrations / operations / release & security) inside the Maintainers group instead, alongside the existing maintainer pages as a "Hosted app" sub-category. No routes changed, only the nav data model (extended to support one level of subgroups) and its render logic. Verified in a live preview: nested titles render, active-link highlighting and prev/next navigation both correctly span subgroup and group boundaries. --- .../src/components/site/docs-nav.tsx | 142 ++++++++++++------ .../docs.self-hosting-troubleshooting.tsx | 30 ++-- ...fhost-troubleshooting-metric-names.test.ts | 42 ++++++ 3 files changed, 157 insertions(+), 57 deletions(-) create mode 100644 test/unit/docs-selfhost-troubleshooting-metric-names.test.ts diff --git a/apps/gittensory-ui/src/components/site/docs-nav.tsx b/apps/gittensory-ui/src/components/site/docs-nav.tsx index 37bc528fa3..80c4eb76c1 100644 --- a/apps/gittensory-ui/src/components/site/docs-nav.tsx +++ b/apps/gittensory-ui/src/components/site/docs-nav.tsx @@ -3,7 +3,12 @@ import { Link, useRouterState } from "@tanstack/react-router"; import { cn } from "@/lib/utils"; type DocsItem = { to: string; label: string }; -type DocsGroup = { title: string; items: DocsItem[] }; +type DocsSubgroup = { title: string; items: DocsItem[] }; +// A group is either a flat list (`items`) or a nested category/sub-category/step hierarchy +// (`subgroups`) — never both. Self-hosting is deliberately nested UNDER "Maintainers" (a maintainer +// concern: running your own instance) rather than sitting as its own top-level sibling category, and +// its own pages are grouped into sub-categories instead of one long flat list. +type DocsGroup = { title: string } & ({ items: DocsItem[] } | { subgroups: DocsSubgroup[] }); export const docsNav: DocsGroup[] = [ { @@ -21,28 +26,48 @@ export const docsNav: DocsGroup[] = [ }, { title: "Maintainers", - items: [ - { to: "/docs/maintainer-workflow", label: "Maintainer workflow" }, - { to: "/docs/github-app", label: "GitHub App" }, - { to: "/docs/maintainer-install-trust", label: "Maintainer install & trust" }, - ], - }, - { - title: "Self-hosting", - items: [ - { to: "/docs/maintainer-self-hosting", label: "Overview" }, - { to: "/docs/self-hosting-quickstart", label: "Quickstart" }, - { to: "/docs/self-hosting-configuration", label: "Configuration" }, - { to: "/docs/self-hosting-github-app", label: "GitHub App & Orb" }, - { to: "/docs/self-hosting-ai-providers", label: "AI providers" }, - { to: "/docs/self-hosting-rees", label: "REES enrichment" }, - { to: "/docs/self-hosting-rees-analyzers", label: "REES analyzers" }, - { to: "/docs/self-hosting-rag", label: "RAG indexing" }, - { to: "/docs/self-hosting-operations", label: "Operations" }, - { to: "/docs/self-hosting-backup-scaling", label: "Backup & scaling" }, - { to: "/docs/self-hosting-releases", label: "Releases & images" }, - { to: "/docs/self-hosting-security", label: "Security" }, - { to: "/docs/self-hosting-troubleshooting", label: "Troubleshooting" }, + subgroups: [ + { + title: "Hosted app", + items: [ + { to: "/docs/maintainer-workflow", label: "Maintainer workflow" }, + { to: "/docs/github-app", label: "GitHub App" }, + { to: "/docs/maintainer-install-trust", label: "Maintainer install & trust" }, + ], + }, + { + title: "Self-hosting: setup", + items: [ + { to: "/docs/maintainer-self-hosting", label: "Overview" }, + { to: "/docs/self-hosting-quickstart", label: "Quickstart" }, + { to: "/docs/self-hosting-configuration", label: "Configuration" }, + ], + }, + { + title: "Self-hosting: integrations", + items: [ + { to: "/docs/self-hosting-github-app", label: "GitHub App & Orb" }, + { to: "/docs/self-hosting-ai-providers", label: "AI providers" }, + { to: "/docs/self-hosting-rees", label: "REES enrichment" }, + { to: "/docs/self-hosting-rees-analyzers", label: "REES analyzers" }, + { to: "/docs/self-hosting-rag", label: "RAG indexing" }, + ], + }, + { + title: "Self-hosting: operations", + items: [ + { to: "/docs/self-hosting-operations", label: "Operations" }, + { to: "/docs/self-hosting-backup-scaling", label: "Backup & scaling" }, + { to: "/docs/self-hosting-troubleshooting", label: "Troubleshooting" }, + ], + }, + { + title: "Self-hosting: release & security", + items: [ + { to: "/docs/self-hosting-releases", label: "Releases & images" }, + { to: "/docs/self-hosting-security", label: "Security" }, + ], + }, ], }, { @@ -64,6 +89,38 @@ export const docsNav: DocsGroup[] = [ }, ]; +function groupItems(group: DocsGroup): DocsItem[] { + return "items" in group ? group.items : group.subgroups.flatMap((sub) => sub.items); +} + +function DocsItemList({ items, pathname }: { items: DocsItem[]; pathname: string }) { + return ( +

    + {items.map((it) => { + const active = pathname === it.to; + return ( +
  • + + {active && ( + + )} + {it.label} + +
    • + ); + })} +
+ ); +} + export function DocsNav() { const pathname = useRouterState({ select: (s) => s.location.pathname }); return ( @@ -73,29 +130,20 @@ export function DocsNav() {
{group.title}
-
    - {group.items.map((it) => { - const active = pathname === it.to; - return ( -
  • - - {active && ( - - )} - {it.label} - -
    • - ); - })} -
+ {"items" in group ? ( + + ) : ( +
+ {group.subgroups.map((sub) => ( +
+
+ {sub.title} +
+ +
+ ))} +
+ )} ))} @@ -104,7 +152,7 @@ export function DocsNav() { export function DocsPrevNext() { const pathname = useRouterState({ select: (s) => s.location.pathname }); - const flat = docsNav.flatMap((g) => g.items); + const flat = docsNav.flatMap(groupItems); const idx = flat.findIndex((i) => i.to === pathname); const prev = idx > 0 ? flat[idx - 1] : null; const next = idx >= 0 && idx < flat.length - 1 ? flat[idx + 1] : null; diff --git a/apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx b/apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx index 0fe1f299ad..927e9e71e4 100644 --- a/apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx +++ b/apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx @@ -10,13 +10,13 @@ export const Route = createFileRoute("/docs/self-hosting-troubleshooting")({ { name: "description", content: - "Troubleshoot self-hosted Gittensory reviews: webhook delivery, AI unavailable, REES silent, RAG empty, queue stuck, and readiness failures.", + "Troubleshoot self-hosted Gittensory reviews: webhook delivery, AI unavailable, REES silent, RAG empty, queue stuck, GitHub rate limits, Qdrant, Orb, AI provider circuit breakers, and readiness failures.", }, { property: "og:title", content: "Self-host troubleshooting — Gittensory docs" }, { property: "og:description", content: - "Troubleshoot self-hosted Gittensory reviews: webhook delivery, AI unavailable, REES silent, RAG empty, queue stuck, and readiness failures.", + "Troubleshoot self-hosted Gittensory reviews: webhook delivery, AI unavailable, REES silent, RAG empty, queue stuck, GitHub rate limits, Qdrant, Orb, AI provider circuit breakers, and readiness failures.", }, { property: "og:url", content: "/docs/self-hosting-troubleshooting" }, ], @@ -174,10 +174,15 @@ sum(rate(gittensory_github_rest_rate_limit_responses_total[10m]))`}

Qdrant / vector-store errors

@@ -197,16 +202,21 @@ sum by (class) (rate(gittensory_github_response_cache_total[15m]))`} deployment's configuration.
  • - A dimension-mismatch error means the existing collection was created with a different - embedding model than the one currently configured (AI_EMBED_MODEL) — recreate - the collection (drop it in Qdrant and let the next index run recreate it at the current - width) rather than trying to reuse it across embedding models. + A dimension-mismatch error means the existing gittensory collection (the + fixed collection name self-host always uses) was created with a different embedding model + than the one currently configured (AI_EMBED_MODEL). Recreating it — delete + the collection and let the next index run recreate it at the current width — is the fix, + but it temporarily removes ALL indexed RAG context for every repo until re-indexing + completes, so treat it as a deliberate, disruptive step, not a routine one.

    • Orb export or relay problems

    diff --git a/test/unit/docs-selfhost-troubleshooting-metric-names.test.ts b/test/unit/docs-selfhost-troubleshooting-metric-names.test.ts new file mode 100644 index 0000000000..15ec7034e5 --- /dev/null +++ b/test/unit/docs-selfhost-troubleshooting-metric-names.test.ts @@ -0,0 +1,42 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +// Drift guard (#1943 gate review finding): the self-hosting troubleshooting runbooks reference exact +// Prometheus metric names and alert names. If a metric is ever renamed/removed in src/, or an alert is +// renamed/removed in prometheus/rules/alerts.yml, this test fails instead of the docs silently going stale +// — mirrors the same source-of-truth-diff approach as scripts/check-openapi-settings-parity.mjs (#2556). + +const DOC_PATH = "apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx"; +const doc = readFileSync(DOC_PATH, "utf8"); + +// The exact source files that emit every gittensory_*_total metric referenced in the runbooks, per an +// audit against the real incr()/gauge()/observe() call sites (src/selfhost/metrics.ts's API). +const METRIC_SOURCE_FILES = [ + "src/github/client.ts", + "src/github/graphql-cache.ts", + "src/selfhost/queue-common.ts", + "src/selfhost/sqlite-queue.ts", + "src/selfhost/pg-queue.ts", + "src/selfhost/qdrant-vectorize.ts", + "src/selfhost/orb-collector.ts", + "src/selfhost/monitored-work.ts", + "src/selfhost/ai.ts", +]; +const metricSource = METRIC_SOURCE_FILES.map((path) => readFileSync(path, "utf8")).join("\n"); +const alertsSource = readFileSync("prometheus/rules/alerts.yml", "utf8"); + +describe("self-hosting-troubleshooting doc: metric/alert names match source (#1943)", () => { + it("every gittensory_..._total metric name referenced in the doc is actually emitted by the code", () => { + const names = [...new Set([...doc.matchAll(/gittensory_[a-z0-9_]+_total/g)].map((m) => m[0]))]; + expect(names.length).toBeGreaterThan(5); // sanity: the extraction found the runbooks' real content + const missing = names.filter((name) => !metricSource.includes(name)); + expect(missing).toEqual([]); + }); + + it("every GittensoryXxx alert name referenced in the doc exists in prometheus/rules/alerts.yml", () => { + const names = [...new Set([...doc.matchAll(/Gittensory[A-Za-z]+/g)].map((m) => m[0]))]; + expect(names.length).toBeGreaterThan(2); + const missing = names.filter((name) => !alertsSource.includes(`alert: ${name}`)); + expect(missing).toEqual([]); + }); +});