diff --git a/apps/gittensory-ui/src/components/site/app-panels/calibration-card-model.ts b/apps/gittensory-ui/src/components/site/app-panels/calibration-card-model.ts new file mode 100644 index 0000000000..de52c023bb --- /dev/null +++ b/apps/gittensory-ui/src/components/site/app-panels/calibration-card-model.ts @@ -0,0 +1,50 @@ +// Confidence-calibration analytics card model (#2192). UI-side mirror of the Calibration / CalibrationBin +// shapes from src/review/ops.ts surfaced on the operator-dashboard payload. + +export type CalibrationBin = { + label: string; + minConfidence: number; + maxConfidence: number; + sampleSize: number; + keptCount: number; + revertedCount: number; + keptRate: number | null; +}; + +/** Mirror of src/review/ops.ts Calibration for the analytics card. */ +export type GateCalibration = { + currentFloor: number; + mergedCount: number; + revertedCount: number; + keptAvgConfidence: number | null; + revertedMaxConfidence: number | null; + recommendedFloor: number | null; + note: string; + bins: CalibrationBin[]; +}; + +export function formatConfidencePct(value: number | null): string { + return value === null ? "—" : `${Math.round(value * 100)}%`; +} + +export function calibrationHasSamples(calibration: GateCalibration): boolean { + return calibration.bins.some((bin) => bin.sampleSize > 0); +} + +/** Kept-rate curve values for TrendChart — empty-sample bins read as 0 on the chart axis. */ +export function calibrationTrendValues(bins: CalibrationBin[]): number[] { + return bins.map((bin) => (bin.keptRate === null ? 0 : bin.keptRate * 100)); +} + +export function calibrationStatus(calibration: GateCalibration): { + tone: "ready" | "warn" | "info"; + label: string; +} { + if (!calibrationHasSamples(calibration)) { + return { tone: "info", label: "no merge samples" }; + } + if (calibration.recommendedFloor !== null) { + return { tone: "warn", label: "raise confidence floor" }; + } + return { tone: "ready", label: "floor adequate" }; +} diff --git a/apps/gittensory-ui/src/components/site/app-panels/calibration-card.test.tsx b/apps/gittensory-ui/src/components/site/app-panels/calibration-card.test.tsx new file mode 100644 index 0000000000..6ef3eb69d7 --- /dev/null +++ b/apps/gittensory-ui/src/components/site/app-panels/calibration-card.test.tsx @@ -0,0 +1,259 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import { CalibrationCard } from "@/components/site/app-panels/calibration-card"; +import { + calibrationHasSamples, + calibrationStatus, + calibrationTrendValues, + type GateCalibration, +} from "@/components/site/app-panels/calibration-card-model"; + +function emptyBins() { + return [ + { + label: "50–60%", + minConfidence: 0.5, + maxConfidence: 0.6, + sampleSize: 0, + keptCount: 0, + revertedCount: 0, + keptRate: null, + }, + { + label: "60–70%", + minConfidence: 0.6, + maxConfidence: 0.7, + sampleSize: 0, + keptCount: 0, + revertedCount: 0, + keptRate: null, + }, + { + label: "70–80%", + minConfidence: 0.7, + maxConfidence: 0.8, + sampleSize: 0, + keptCount: 0, + revertedCount: 0, + keptRate: null, + }, + { + label: "80–90%", + minConfidence: 0.8, + maxConfidence: 0.9, + sampleSize: 0, + keptCount: 0, + revertedCount: 0, + keptRate: null, + }, + { + label: "90–100%", + minConfidence: 0.9, + maxConfidence: 1, + sampleSize: 0, + keptCount: 0, + revertedCount: 0, + keptRate: null, + }, + ]; +} + +function calibration(overrides: Partial = {}): GateCalibration { + return { + currentFloor: 0.9, + mergedCount: 0, + revertedCount: 0, + keptAvgConfidence: null, + revertedMaxConfidence: null, + recommendedFloor: null, + note: "No reverted auto-merges — the current floor looks adequate.", + bins: emptyBins(), + ...overrides, + }; +} + +describe("calibrationStatus", () => { + it("returns info when every bin is empty (no merge samples arm)", () => { + expect(calibrationStatus(calibration())).toEqual({ tone: "info", label: "no merge samples" }); + expect(calibrationHasSamples(calibration())).toBe(false); + }); + + it("returns warn when recommendedFloor is present (above-current-floor arm)", () => { + expect( + calibrationStatus( + calibration({ + recommendedFloor: 0.94, + bins: [ + { + label: "90–100%", + minConfidence: 0.9, + maxConfidence: 1, + sampleSize: 2, + keptCount: 1, + revertedCount: 1, + keptRate: 0.5, + }, + ], + }), + ), + ).toEqual({ tone: "warn", label: "raise confidence floor" }); + }); + + it("returns ready when there are samples but no floor change is recommended", () => { + expect( + calibrationStatus( + calibration({ + bins: [ + { + label: "90–100%", + minConfidence: 0.9, + maxConfidence: 1, + sampleSize: 3, + keptCount: 3, + revertedCount: 0, + keptRate: 1, + }, + ], + }), + ), + ).toEqual({ tone: "ready", label: "floor adequate" }); + }); +}); + +describe("calibrationTrendValues", () => { + it("maps null keptRate bins to 0 for the chart axis", () => { + expect(calibrationTrendValues(emptyBins())).toEqual([0, 0, 0, 0, 0]); + }); + + it("scales kept rates to percentage points for TrendChart", () => { + expect( + calibrationTrendValues([ + { + label: "80–90%", + minConfidence: 0.8, + maxConfidence: 0.9, + sampleSize: 2, + keptCount: 2, + revertedCount: 0, + keptRate: 1, + }, + { + label: "90–100%", + minConfidence: 0.9, + maxConfidence: 1, + sampleSize: 2, + keptCount: 1, + revertedCount: 1, + keptRate: 0.5, + }, + ]), + ).toEqual([100, 50]); + }); +}); + +describe("CalibrationCard", () => { + it("renders the empty-bins state without the curve or per-bin sparkbars", () => { + render(); + expect(screen.getByText("Confidence calibration")).toBeTruthy(); + expect(screen.getByText("no merge samples")).toBeTruthy(); + expect(screen.getByText("—")).toBeTruthy(); + expect(screen.getByText(/Merge-confidence calibration bins appear once/)).toBeTruthy(); + expect(screen.queryByText("Kept-rate curve by confidence band")).toBeNull(); + }); + + it("renders a single populated bin with kept rate and em dash for empty bins", () => { + render( + , + ); + expect(screen.getByText("floor adequate")).toBeTruthy(); + expect(screen.getByText("100%")).toBeTruthy(); + expect(screen.getByText("Kept-rate curve by confidence band")).toBeTruthy(); + expect(screen.getAllByText("—").length).toBeGreaterThanOrEqual(4); + }); + + it("renders the full curve across multiple bins and surfaces the recommended floor", () => { + render( + , + ); + expect(screen.getByText("raise confidence floor")).toBeTruthy(); + expect(screen.getByText("94%")).toBeTruthy(); + expect(screen.getByText("90%")).toBeTruthy(); + expect(screen.getByText(/Raise confidenceFloor 0.9 → 0.94/)).toBeTruthy(); + expect(screen.getByText("70–80%")).toBeTruthy(); + expect(screen.getByText("90–100%")).toBeTruthy(); + }); +}); diff --git a/apps/gittensory-ui/src/components/site/app-panels/calibration-card.tsx b/apps/gittensory-ui/src/components/site/app-panels/calibration-card.tsx new file mode 100644 index 0000000000..fbc2060eca --- /dev/null +++ b/apps/gittensory-ui/src/components/site/app-panels/calibration-card.tsx @@ -0,0 +1,117 @@ +import { + BoundaryBadge, + MiniSparkbar, + Stat, + StatusPill, +} from "@/components/site/control-primitives"; +import { TrendChart } from "@/components/site/trend-chart"; +import { + calibrationHasSamples, + calibrationStatus, + calibrationTrendValues, + formatConfidencePct, + type GateCalibration, +} from "@/components/site/app-panels/calibration-card-model"; + +/** Analytics card (#2192): confidence-vs-outcome calibration curve from computeCalibration — predicted merge + * confidence bands vs realized kept-rate, plus the recommended confidence floor. Read-only. */ +export function CalibrationCard({ calibration }: { calibration: GateCalibration }) { + const status = calibrationStatus(calibration); + const hasSamples = calibrationHasSamples(calibration); + const trendValues = calibrationTrendValues(calibration.bins); + + return ( +
+
+
+

Confidence calibration

+

+ Predicted merge confidence vs realized kept-rate per bucket. Public-safe aggregate + counts only. +

+
+ {status.label} +
+ +
+ configured confidenceFloor} + /> + + from reverted merges + + + } + /> + terminal merged targets} + /> + human-reverted bot merges} + /> +
+ + {hasSamples ? ( + <> +
+
+ Kept-rate curve by confidence band + + avg kept {formatConfidencePct(calibration.keptAvgConfidence)} + +
+
+ +
+
+ +
+ {calibration.bins.map((bin) => ( +
+
+
{bin.label}
+
+ predicted band · {bin.sampleSize} sample{bin.sampleSize === 1 ? "" : "s"} +
+
+
+
+
+ {formatConfidencePct(bin.keptRate)} +
+
actual kept
+
+ {bin.sampleSize > 0 ? ( + + ) : ( + — + )} +
+
+ ))} +
+ +

{calibration.note}

+ + ) : ( +

+ Merge-confidence calibration bins appear once the gate has auto-merged pull requests with + persisted confidence scores. +

+ )} +
+ ); +} diff --git a/apps/gittensory-ui/src/routes/app.analytics.tsx b/apps/gittensory-ui/src/routes/app.analytics.tsx index 05bf9cdfea..7a26005d6d 100644 --- a/apps/gittensory-ui/src/routes/app.analytics.tsx +++ b/apps/gittensory-ui/src/routes/app.analytics.tsx @@ -15,6 +15,8 @@ import { GatePrecisionCard } from "@/components/site/app-panels/gate-precision-c import type { GateEvalReport } from "@/components/site/app-panels/gate-precision-card-model"; import { CycleTimeCard } from "@/components/site/app-panels/cycle-time-card"; import type { CycleTimeAggregate } from "@/components/site/app-panels/cycle-time-card-model"; +import { CalibrationCard } from "@/components/site/app-panels/calibration-card"; +import type { GateCalibration } from "@/components/site/app-panels/calibration-card-model"; import { ReversalHealthCard } from "@/components/site/app-panels/reversal-health-card"; import type { ReversalHealth } from "@/components/site/app-panels/reversal-health-card-model"; import { AnalyticsCardShell } from "@/components/site/app-panels/analytics-card-shell"; @@ -118,6 +120,7 @@ type OperatorDashboard = { upstreamDrift?: { status?: string; openReportCount?: number } | null; gateEval?: GateEvalReport; cycleTime?: CycleTimeAggregate; + calibration?: GateCalibration; agentHealth?: ReversalHealth; acceptance?: FindingAcceptance; findingsBreakdown?: FindingsBreakdown; @@ -222,6 +225,8 @@ function ProductAnalytics() { /> )} + {data.calibration ? : null} + {data.agentHealth ? : null} diff --git a/src/review/ops.ts b/src/review/ops.ts index 7b20db1225..eca57b5536 100644 --- a/src/review/ops.ts +++ b/src/review/ops.ts @@ -72,6 +72,49 @@ export interface Calibration { /** Per-reasonCode close distribution + how many of each a human REOPENED and the gate did NOT re-merge. */ closesByReason: Array<{ reasonCode: string; closes: number; disputed: number }>; disputedCloseCount: number; + /** Predicted merge-confidence band vs realized kept-rate (not reverted) per bucket. */ + bins: CalibrationBin[]; +} + +/** One confidence band in the calibration curve (#2192). */ +export type CalibrationBin = { + label: string; + minConfidence: number; + maxConfidence: number; + sampleSize: number; + keptCount: number; + revertedCount: number; + /** keptCount / sampleSize; null when sampleSize === 0. */ + keptRate: number | null; +}; + +const CALIBRATION_BIN_EDGES = [0.5, 0.6, 0.7, 0.8, 0.9, 1.0] as const; + +/** Fold merge-confidence samples into fixed calibration bins for the analytics curve card. */ +export function buildCalibrationBins( + samples: ReadonlyArray<{ confidence: number; kept: boolean }>, +): CalibrationBin[] { + const bins: CalibrationBin[] = []; + for (let i = 0; i < CALIBRATION_BIN_EDGES.length - 1; i += 1) { + const min = CALIBRATION_BIN_EDGES[i]!; + const max = CALIBRATION_BIN_EDGES[i + 1]!; + const isLast = i === CALIBRATION_BIN_EDGES.length - 2; + const inBin = samples.filter( + (sample) => sample.confidence >= min && (isLast ? sample.confidence <= max : sample.confidence < max), + ); + const keptCount = inBin.filter((sample) => sample.kept).length; + const sampleSize = inBin.length; + bins.push({ + label: `${Math.round(min * 100)}–${Math.round(max * 100)}%`, + minConfidence: min, + maxConfidence: max, + sampleSize, + keptCount, + revertedCount: sampleSize - keptCount, + keptRate: sampleSize > 0 ? Number((keptCount / sampleSize).toFixed(3)) : null, + }); + } + return bins; } /** The minimal agent-config shape the ops endpoints read. (Subset of reviewbot's AgentConfig.) */ @@ -280,10 +323,13 @@ export async function computeCalibration(env: Env, config: OpsAgentConfig): Prom }; const kept: number[] = []; const rev: number[] = []; + const binSamples: Array<{ confidence: number; kept: boolean }> = []; for (const r of mergedRows.results ?? []) { const c = confidenceOf(r.decision_json); if (c == null) continue; - (reverted.has(r.id) ? rev : kept).push(c); + const isKept = !reverted.has(r.id); + binSamples.push({ confidence: c, kept: isKept }); + (isKept ? kept : rev).push(c); } const avg = (xs: number[]): number | null => (xs.length ? Number((xs.reduce((a, b) => a + b, 0) / xs.length).toFixed(3)) : null); const currentFloor = config.confidenceFloor ?? 0; @@ -305,6 +351,7 @@ export async function computeCalibration(env: Env, config: OpsAgentConfig): Prom note, closesByReason, disputedCloseCount, + bins: buildCalibrationBins(binSamples), }; } diff --git a/src/services/operator-dashboard.ts b/src/services/operator-dashboard.ts index f689429c81..8b591bae40 100644 --- a/src/services/operator-dashboard.ts +++ b/src/services/operator-dashboard.ts @@ -26,7 +26,7 @@ import type { WeeklyValueReport, } from "../types"; import { computeFleetAnalytics, type FleetAnalytics } from "../orb/analytics"; -import { computeAgentHealth, type AgentHealth } from "../review/ops"; +import { computeAgentHealth, computeCalibration, type AgentHealth, type Calibration } from "../review/ops"; import { computeGateEval, type GateEvalReport } from "../review/parity"; import { computeCycleTimeAggregate, type CycleTimeAggregate } from "../review/stats"; import { loadUpstreamStatus, type UpstreamStatus } from "../upstream/ruleset"; @@ -67,6 +67,8 @@ export type OperatorDashboardPayload = { gateEval: GateEvalReport; // PR review cycle-time percentiles (#2194): gate decision → outcome from review_audit; fail-safe empty aggregate. cycleTime: CycleTimeAggregate; + // Confidence-vs-outcome calibration curve (#2192): merge confidence bins + recommended floor from computeCalibration. + calibration: Calibration; // Agent reversal health (#2193): how often humans reopened/reverted bot auto-actions (ops.ts AgentHealth). agentHealth: AgentHealth; }; @@ -94,6 +96,7 @@ export async function buildOperatorDashboardPayload(env: Env): Promise { distribution: [], sampleSize: 0, }); + expect(payload.calibration).toMatchObject({ + currentFloor: 0, + mergedCount: 0, + revertedCount: 0, + recommendedFloor: null, + bins: expect.arrayContaining([ + expect.objectContaining({ label: "90–100%", sampleSize: 0, keptRate: null }), + ]), + }); + expect(payload.agentHealth).toMatchObject({ reversals: 0, reversalRate: 0, diff --git a/test/unit/ops.test.ts b/test/unit/ops.test.ts index 9e007c994c..804ed8f653 100644 --- a/test/unit/ops.test.ts +++ b/test/unit/ops.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { + buildCalibrationBins, computeAgentHealth, computeCalibration, defaultOpsHealthDeps, @@ -83,6 +84,12 @@ describe("computeCalibration", () => { expect(cal.revertedCount).toBe(1); expect(cal.revertedMaxConfidence).toBe(0.92); expect(cal.recommendedFloor).toBe(0.94); // 0.92 + 0.02 + expect(cal.bins.find((bin) => bin.label === "90–100%")).toMatchObject({ + sampleSize: 3, + keptCount: 2, + revertedCount: 1, + keptRate: expect.closeTo(2 / 3, 3), + }); }); it("recommends no change when nothing was reverted", async () => { @@ -106,6 +113,39 @@ describe("computeCalibration", () => { }); }); +describe("buildCalibrationBins", () => { + it("returns five empty bins when there are no confidence samples", () => { + expect(buildCalibrationBins([])).toEqual([ + { label: "50–60%", minConfidence: 0.5, maxConfidence: 0.6, sampleSize: 0, keptCount: 0, revertedCount: 0, keptRate: null }, + { label: "60–70%", minConfidence: 0.6, maxConfidence: 0.7, sampleSize: 0, keptCount: 0, revertedCount: 0, keptRate: null }, + { label: "70–80%", minConfidence: 0.7, maxConfidence: 0.8, sampleSize: 0, keptCount: 0, revertedCount: 0, keptRate: null }, + { label: "80–90%", minConfidence: 0.8, maxConfidence: 0.9, sampleSize: 0, keptCount: 0, revertedCount: 0, keptRate: null }, + { label: "90–100%", minConfidence: 0.9, maxConfidence: 1, sampleSize: 0, keptCount: 0, revertedCount: 0, keptRate: null }, + ]); + }); + + it("folds a single sample into one populated bin", () => { + expect(buildCalibrationBins([{ confidence: 0.95, kept: true }])).toEqual( + expect.arrayContaining([ + { label: "90–100%", minConfidence: 0.9, maxConfidence: 1, sampleSize: 1, keptCount: 1, revertedCount: 0, keptRate: 1 }, + ]), + ); + }); + + it("builds a full kept-rate curve across multiple confidence bands", () => { + const bins = buildCalibrationBins([ + { confidence: 0.75, kept: true }, + { confidence: 0.85, kept: true }, + { confidence: 0.85, kept: false }, + { confidence: 0.95, kept: true }, + { confidence: 0.95, kept: false }, + ]); + expect(bins.find((bin) => bin.label === "70–80%")).toMatchObject({ sampleSize: 1, keptRate: 1 }); + expect(bins.find((bin) => bin.label === "80–90%")).toMatchObject({ sampleSize: 2, keptRate: 0.5 }); + expect(bins.find((bin) => bin.label === "90–100%")).toMatchObject({ sampleSize: 2, keptRate: 0.5 }); + }); +}); + describe("handleInternalCalibration", () => { const cfg: OpsAgentConfig = { slug: "metagraphed", confidenceFloor: 0.9, secrets: { internalSecret: "INTERNAL_SECRET" } }; const env = (extra: Record) => ({ ...calibrationEnv([], []), ...extra }) as unknown as Env;