diff --git a/grafana/dashboards/gittensory.json b/grafana/dashboards/gittensory.json index 9d0751dca6..2a6273197b 100644 --- a/grafana/dashboards/gittensory.json +++ b/grafana/dashboards/gittensory.json @@ -3059,6 +3059,41 @@ "legendFormat": "skew" } ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 209 }, + "id": 161, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "title": "Review-Floor Bypass Attempts (total, #3901)", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_ai_review_onmerge_clamped_total or vector(0)", + "legendFormat": "clamped" + } + ] } ], "refresh": "30s", diff --git a/prometheus/rules/alerts.yml b/prometheus/rules/alerts.yml index 8e7ad3ac7d..e4e0d3bc8b 100644 --- a/prometheus/rules/alerts.yml +++ b/prometheus/rules/alerts.yml @@ -474,6 +474,21 @@ groups: description: "{{ $value | printf \"%.0f\" }} AI review(s) came back inconclusive over the last 30m (sustained 10m). Dual-AI review is repeatedly failing to produce a usable verdict." runbook: "Check provider health and circuit-breaker state (gittensory_ai_provider_failures_total / gittensory_ai_provider_circuit_open_total) and verify AI_PROVIDER credentials are still valid for every configured reviewer." + - alert: GittensoryAiReviewOnMergeFloorBypassAttempted + # A repo's .gittensory.yml tried to loosen the operator's onMerge/combine/reviewer-count floor and + # got clamped back to the operator's own configured minimum (#3901). A governance/config signal, not + # an operational outage, but same absolute-increase style as GittensoryAiProviderCircuitOpen above: + # any occurrence (> 0) over 1h is worth a look, sustained 5m so a single instantaneous blip doesn't + # immediately page. + expr: increase(gittensory_ai_review_onmerge_clamped_total[1h]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "a repo's review-floor clamp fired" + description: "{{ $value | printf \"%.0f\" }} onMerge/combine/reviewer-count clamp event(s) over the last 1h — a repo config tried to loosen below the operator's configured floor." + runbook: "Check which repo's .gittensory.yml requested the loosened setting (gittensory_ai_review_onmerge_clamped_total is labeled by mode) and confirm whether that repo's floor should be adjusted or the request declined." + - alert: GittensoryAiProviderCircuitOpen # A provider's circuit breaker opens after AI_PROVIDER_FAILURE_THRESHOLD consecutive failures and # short-circuits further attempts for a cooldown window -- ANY circuit-open event in 15m means that diff --git a/test/unit/selfhost-grafana-dashboard.test.ts b/test/unit/selfhost-grafana-dashboard.test.ts index 68e582c930..0b6d216985 100644 --- a/test/unit/selfhost-grafana-dashboard.test.ts +++ b/test/unit/selfhost-grafana-dashboard.test.ts @@ -104,6 +104,16 @@ describe("Gittensory Self-Host Grafana dashboard", () => { expect(targets.some((target) => target.expr === "(gittensory_orb_events_recorded_total or vector(0)) - (gittensory_orb_events_exported_total or vector(0))")).toBe(true); }); + it("surfaces the onMerge/combine/reviewer-count floor-clamp counter on a panel and an alert (#3901)", () => { + const dashboard = readDashboard(selfhostDashboardPath); + const targets = dashboard.panels.flatMap((panel) => panel.targets ?? []); + const alerts = readFileSync(selfhostAlertsPath, "utf8"); + + expect(targets.some((target) => target.expr === "gittensory_ai_review_onmerge_clamped_total or vector(0)")).toBe(true); + expect(alerts).toContain("alert: GittensoryAiReviewOnMergeFloorBypassAttempted"); + expect(alerts).toContain("expr: increase(gittensory_ai_review_onmerge_clamped_total[1h]) > 0"); + }); + it("keeps rate-limit alerts grouped by the dashboard label dimensions", () => { const alerts = readFileSync(selfhostAlertsPath, "utf8");