diff --git a/grafana/dashboards/gittensory.json b/grafana/dashboards/gittensory.json index 2a6273197b..4e0a82713f 100644 --- a/grafana/dashboards/gittensory.json +++ b/grafana/dashboards/gittensory.json @@ -3094,6 +3094,94 @@ "legendFormat": "clamped" } ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 213 }, + "id": 162, + "title": "Reviews & Ops Anomalies (#ops-anomaly-metric)", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "lineWidth": 2, "fillOpacity": 10 }, + "unit": "short" + } + }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 214 }, + "id": 163, + "options": { + "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "title": "Ops Anomaly Detections (review burst / review failure burst)", + "description": "runOpsAlerts' hourly scan over gittensory's own outcome data, labeled by kind (review_burst / review_failure_burst) and repo. Any nonzero value here means the scan flagged a stuck-CI finalize loop or retry storm -- see the ops_anomaly structured log for the human-readable detail line.", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum by (kind, repo) (increase(gittensory_ops_anomaly_total[1h])) or vector(0)", + "legendFormat": "{{kind}} {{repo}}", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "lineWidth": 2, "fillOpacity": 10 }, + "unit": "ops" + } + }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 214 }, + "id": 164, + "options": { + "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "title": "Published Review Comments (rate)", + "description": "Published review comments/summaries per repo -- the gittensory review pipeline's actual output rate.", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum by (repo) (rate(gittensory_reviews_published_total[5m])) or vector(0)", + "legendFormat": "{{repo}}", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "lineWidth": 2, "fillOpacity": 10 }, + "unit": "ops" + } + }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 214 }, + "id": 165, + "options": { + "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "title": "Gate Decisions by Conclusion (rate)", + "description": "Rendered gate verdict rate by conclusion (merge/close/hold) -- the \"are we rubber-stamping?\" signal (#reviews-dashboard).", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum by (conclusion) (rate(gittensory_gate_decisions_total[5m])) or vector(0)", + "legendFormat": "{{conclusion}}", + "refId": "A" + } + ] } ], "refresh": "30s", @@ -3118,5 +3206,5 @@ "timezone": "browser", "title": "Gittensory Self-Host", "uid": "gittensory-selfhost", - "version": 8 + "version": 9 } diff --git a/prometheus/rules/alerts.yml b/prometheus/rules/alerts.yml index e4e0d3bc8b..5f3d732445 100644 --- a/prometheus/rules/alerts.yml +++ b/prometheus/rules/alerts.yml @@ -503,6 +503,25 @@ groups: description: "Provider {{ $labels.provider }} has failed repeatedly and its circuit breaker is skipping calls fast during its cooldown (sustained 5m)." runbook: "Check that provider's credentials/reachability (CLI auth for claude-code/codex, or the configured API key/base URL for HTTP providers) via gittensory_ai_provider_failures_total{provider=\"...\"} and recent selfhost_ai_provider_failed logs." + # ── Ops anomaly scan (review burst / review failure burst, #ops-anomaly-metric) ──── + - name: gittensory-ops-anomalies + rules: + - alert: GittensoryOpsAnomalyDetected + # runOpsAlerts (src/review/ops-wire.ts) scans gittensory's own outcome data hourly and increments this + # counter, labeled by kind, when it catches a review burst (a stuck-CI finalize loop or sweep retry + # storm re-publishing the same PR far more than normal iteration ever does) or a review failure burst + # (repeated inconclusive AI-review calls with zero successful publish -- the #3747 incident shape). + # Same absolute-increase style as GittensoryDeadLetterJobsGrowing: any occurrence over the scan's own + # 2h detection window is worth a look, not a rate/ratio. + expr: increase(gittensory_ops_anomaly_total[2h]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "gittensory ops anomaly scan flagged a {{ $labels.kind }} on {{ $labels.repo }}" + description: "{{ $value | printf \"%.0f\" }} {{ $labels.kind }} detection(s) over the last 2h for {{ $labels.repo }} (sustained 5m). Check the ops_anomaly structured log for the full detail line." + runbook: "Tail logs for level=error event=ops_anomaly repo={{ $labels.repo }} for the human-readable anomaly text. A review_burst or review_failure_burst usually means a stuck-CI finalize loop or a sweep retry storm -- see #orb-ci-stuck-repeat / #review-burst-blind-spot." + # ── Host clock sync (#3811) ─────────────────────────────────────────────── - name: gittensory-system-health rules: diff --git a/src/review/ops-wire.ts b/src/review/ops-wire.ts index a3155e980d..e5c75ef864 100644 --- a/src/review/ops-wire.ts +++ b/src/review/ops-wire.ts @@ -31,6 +31,7 @@ // design. This module is READ-ONLY observability: it reports drift; it never changes what blocks a live PR. import { findHottestInconclusiveReviewTargetForRepo, findHottestReviewTargetForRepo, listRepositories, sumByokAiUsageForRepoSince } from "../db/repositories"; +import { incr } from "../selfhost/metrics"; import { isAgentConfigured } from "../settings/autonomy"; import { resolveRepositorySettings } from "../settings/repository-settings"; import { loadGatePrecisionReport, type GatePrecisionReport } from "../services/gate-precision"; @@ -191,6 +192,16 @@ export async function runOpsAlerts(env: Env): Promise> // Structured log = gittensory's notify path (no Discord/operator webhook exists) AND the Sentry path // (level:"error" + an `event` field reaches forwardStructuredLogToSentry). One line per repo. console.error(JSON.stringify({ level: "error", event: "ops_anomaly", repo: repoFullName, at: nowIso(), anomalies })); + // #ops-anomaly-metric: Prometheus counterpart to the log line above so a self-host operator can alert on + // /metrics instead of grepping Workers Logs. Scoped to reviewBurst/reviewFailureBurst -- the two anomalies + // this module exists to catch fast (#orb-ci-stuck-repeat / #review-burst-blind-spot) -- rather than every + // anomaly kind, so the counter stays a precise "stuck-CI/retry-storm" signal, not a catch-all. + if (reviewBurst && reviewBurst.count >= REVIEW_BURST_THRESHOLD) { + incr("gittensory_ops_anomaly_total", { repo: repoFullName, kind: "review_burst" }); + } + if (reviewFailureBurst && reviewFailureBurst.count >= REVIEW_FAILURE_BURST_THRESHOLD) { + incr("gittensory_ops_anomaly_total", { repo: repoFullName, kind: "review_failure_burst" }); + } } catch (error) { console.error(JSON.stringify({ level: "error", event: "ops_anomaly_repo_error", repo: repoFullName, message: errorMessage(error).slice(0, 200) })); } diff --git a/src/selfhost/metrics.ts b/src/selfhost/metrics.ts index 2a1551b460..a52623387a 100644 --- a/src/selfhost/metrics.ts +++ b/src/selfhost/metrics.ts @@ -133,6 +133,7 @@ const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [ ["gittensory_github_pr_files_fetch_total", { help: "GitHub pull-request file fetch attempts.", type: "counter" }], ["gittensory_pr_state_cache_total", { help: "Pull-request state cache outcomes.", type: "counter" }], ["gittensory_ci_state_cache_total", { help: "CI-state snapshot cache outcomes.", type: "counter" }], + ["gittensory_ops_anomaly_total", { help: "Ops anomaly scan detections (review burst / review failure burst), by repo and kind.", type: "counter" }], ]; const metricMeta = new Map(DEFAULT_METRIC_META); @@ -151,6 +152,7 @@ export function setSelfHostedMetricsMode(isSelfHosted: boolean): void { const PRIVATE_REPO_LABEL_METRICS = new Set([ "gittensory_gate_decisions_total", "gittensory_reviews_published_total", + "gittensory_ops_anomaly_total", ]); const ALWAYS_REDACT_REPO_LABEL_METRICS = new Set([ "gittensory_agent_disposition_total", diff --git a/test/unit/ops-wire.test.ts b/test/unit/ops-wire.test.ts index 3ee41faa8c..3714e38bda 100644 --- a/test/unit/ops-wire.test.ts +++ b/test/unit/ops-wire.test.ts @@ -8,6 +8,7 @@ import { type RepoOutcomeSnapshot, runOpsAlerts, } from "../../src/review/ops-wire"; +import { counterValue, resetMetrics, setSelfHostedMetricsMode } from "../../src/selfhost/metrics"; import { createTestEnv } from "../helpers/d1"; // Wrap env.DB.prepare so any SQL matching `pattern` throws, exercising a fail-safe catch; every other @@ -187,7 +188,11 @@ async function seedGateFalsePositiveAnomaly(env: Env, repoFullName: string): Pro } describe("runOpsAlerts — cron path over gittensory's outcome data", () => { - afterEach(() => vi.restoreAllMocks()); + afterEach(() => { + vi.restoreAllMocks(); + resetMetrics(); + setSelfHostedMetricsMode(false); + }); it("emits a structured ops_anomaly log naming the repo + drift on a seeded anomaly, at error level (#orb-ci-stuck-repeat -- so it reaches Sentry)", async () => { const env = createTestEnv(); @@ -224,6 +229,7 @@ describe("runOpsAlerts — cron path over gittensory's outcome data", () => { }); it("detects and reports a review burst end-to-end (a PR published far more review surfaces than normal in the window)", async () => { + setSelfHostedMetricsMode(true); // keep the repo label so the counter assertion can target the exact series const env = createTestEnv(); await seedRegisteredRepo(env, "owner/repo"); for (let i = 0; i < 7; i += 1) { @@ -242,6 +248,9 @@ describe("runOpsAlerts — cron path over gittensory's outcome data", () => { const stats = await computeOpsStats(env); const row = stats.repos.find((r) => r.repoFullName === "owner/repo"); expect(row?.anomalies.some((a) => /review burst/.test(a))).toBe(true); + // #ops-anomaly-metric: the Prometheus counterpart to the log line, labeled by kind (self-host mode preserves + // the repo label so the assertion can target the exact series without relying on cloud-worker redaction). + expect(counterValue("gittensory_ops_anomaly_total", { repo: "owner/repo", kind: "review_burst" })).toBe(1); }); it("detects and reports a review FAILURE burst end-to-end -- reproduces the #3747 incident shape (repeated inconclusive calls, zero publishes) (#review-burst-blind-spot)", async () => { diff --git a/test/unit/selfhost-metrics.test.ts b/test/unit/selfhost-metrics.test.ts index ca5422018b..fddc5d31d1 100644 --- a/test/unit/selfhost-metrics.test.ts +++ b/test/unit/selfhost-metrics.test.ts @@ -120,6 +120,15 @@ describe("metrics registry (#982)", () => { expect(await renderMetrics()).toContain('gittensory_gate_decisions_total{conclusion="hold"} 1'); }); + it("redacts the repository label from the ops anomaly counter but keeps the kind label (#ops-anomaly-metric)", async () => { + incr("gittensory_ops_anomaly_total", { repo: "private-owner/secret-repo", kind: "review_burst" }); + + const out = await renderMetrics(); + expect(out).toContain('gittensory_ops_anomaly_total{kind="review_burst"} 1'); + expect(out).not.toContain("private-owner/secret-repo"); + expect(out).not.toContain('repo="'); + }); + it("preserves repository labels for unrelated metrics", async () => { incr("debug_total", { repo: "public-owner/public-repo" }); expect(await renderMetrics()).toContain('debug_total{repo="public-owner/public-repo"} 1'); @@ -132,10 +141,12 @@ describe("metrics registry (#982)", () => { setSelfHostedMetricsMode(true); incr("gittensory_gate_decisions_total", { repo: "owner/repo", conclusion: "success" }); incr("gittensory_reviews_published_total", { repo: "owner/repo" }); + incr("gittensory_ops_anomaly_total", { repo: "owner/repo", kind: "review_burst" }); const out = await renderMetrics(); expect(out).toContain('gittensory_gate_decisions_total{conclusion="success",repo="owner/repo"} 1'); expect(out).toContain('gittensory_reviews_published_total{repo="owner/repo"} 1'); + expect(out).toContain('gittensory_ops_anomaly_total{kind="review_burst",repo="owner/repo"} 1'); expect(out).toContain('repo="owner/repo"'); });