diff --git a/apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card-model.ts b/apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card-model.ts new file mode 100644 index 0000000000..5e7d30f61e --- /dev/null +++ b/apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card-model.ts @@ -0,0 +1,27 @@ +// Slop-band calibration card model (#2196). UI-side mirror of SlopOutcomeCalibration from src/review/stats.ts. + +export type SlopBand = "clean" | "low" | "elevated" | "high"; + +export type SlopBandCalibrationRow = { + band: SlopBand; + sampleSize: number; + merged: number; + closed: number; + mergeRate: number; +}; + +export type SlopBandCalibration = { + totalResolved: number; + bands: SlopBandCalibrationRow[]; + overallMergeRate: number | null; + discriminates: boolean | null; +}; + +export function formatSlopBandLabel(band: SlopBand): string { + return band.charAt(0).toUpperCase() + band.slice(1); +} + +export function formatMergeRate(rate: number, sampleSize: number): string { + if (sampleSize <= 0) return "—"; + return `${Math.round(rate * 100)}%`; +} diff --git a/apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.test.tsx b/apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.test.tsx new file mode 100644 index 0000000000..60e9b7fdd0 --- /dev/null +++ b/apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.test.tsx @@ -0,0 +1,105 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import { SlopBandCalibrationCard } from "@/components/site/app-panels/slop-band-calibration-card"; +import { + formatMergeRate, + formatSlopBandLabel, + type SlopBandCalibration, +} from "@/components/site/app-panels/slop-band-calibration-card-model"; + +function calibration(overrides: Partial = {}): SlopBandCalibration { + return { + totalResolved: 0, + overallMergeRate: null, + discriminates: null, + bands: [ + { band: "clean", sampleSize: 0, merged: 0, closed: 0, mergeRate: 0 }, + { band: "low", sampleSize: 0, merged: 0, closed: 0, mergeRate: 0 }, + { band: "elevated", sampleSize: 0, merged: 0, closed: 0, mergeRate: 0 }, + { band: "high", sampleSize: 0, merged: 0, closed: 0, mergeRate: 0 }, + ], + ...overrides, + }; +} + +describe("slop-band calibration formatters", () => { + it("capitalizes band labels and renders merge rate or em dash", () => { + expect(formatSlopBandLabel("elevated")).toBe("Elevated"); + expect(formatMergeRate(0.82, 11)).toBe("82%"); + expect(formatMergeRate(0.5, 0)).toBe("—"); + }); +}); + +describe("SlopBandCalibrationCard", () => { + it("renders all four band rows when every band has samples", () => { + render( + , + ); + expect(screen.getByText("Slop-band calibration")).toBeTruthy(); + expect(screen.getByText("Clean")).toBeTruthy(); + expect(screen.getByText("Low")).toBeTruthy(); + expect(screen.getByText("Elevated")).toBeTruthy(); + expect(screen.getByText("High")).toBeTruthy(); + expect(screen.getByText("predictive")).toBeTruthy(); + expect(screen.getByText("Overall merge rate across assessed bands: 75%")).toBeTruthy(); + }); + + it("shows em dash for a single empty band while other bands still render", () => { + render( + , + ); + expect(screen.getAllByText("—")).toHaveLength(2); + expect(screen.getByText("insufficient per-band sample")).toBeTruthy(); + }); + + it("shows inline empty copy and the no-data status pill arm when there are no samples", () => { + render(); + expect(screen.getByText("no samples yet")).toBeTruthy(); + expect( + screen.getByText(/Resolved pull requests with a persisted slop band will appear here/), + ).toBeTruthy(); + }); + + it("surfaces the not-discriminating status arm", () => { + render( + , + ); + expect(screen.getByText("not discriminating")).toBeTruthy(); + }); +}); diff --git a/apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.tsx b/apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.tsx new file mode 100644 index 0000000000..7c97753c9c --- /dev/null +++ b/apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.tsx @@ -0,0 +1,76 @@ +import { MiniSparkbar, Stat, StatusPill } from "@/components/site/control-primitives"; +import { + formatMergeRate, + formatSlopBandLabel, + type SlopBandCalibration, +} from "@/components/site/app-panels/slop-band-calibration-card-model"; + +/** Analytics card (#2196): slop-band predicted severity vs realized merge/close outcomes from the stats feed. + * Renders band labels and aggregate rates only — never raw slop scores. */ +export function SlopBandCalibrationCard({ calibration }: { calibration: SlopBandCalibration }) { + const hasSamples = calibration.totalResolved > 0; + const statusTone = + calibration.discriminates === true + ? "ready" + : calibration.discriminates === false + ? "warn" + : "info"; + const statusLabel = + calibration.discriminates === true + ? "predictive" + : calibration.discriminates === false + ? "not discriminating" + : hasSamples + ? "insufficient per-band sample" + : "no samples yet"; + + return ( +
+
+
+

Slop-band calibration

+

+ Predicted slop band vs realized merge/close outcomes for resolved pull requests. + Public-safe band counts only. +

+
+ {statusLabel} +
+ + {hasSamples ? ( + <> +
+ {calibration.bands.map((band) => ( + + {band.sampleSize} assessed · {band.merged} merged · {band.closed} closed + + } + trend={ + band.sampleSize > 0 ? ( + + ) : undefined + } + /> + ))} +
+ {calibration.overallMergeRate !== null ? ( +

+ Overall merge rate across assessed bands:{" "} + {Math.round(calibration.overallMergeRate * 100)}% +

+ ) : null} + + ) : ( +

+ Resolved pull requests with a persisted slop band will appear here once the gate has + enough outcome history in the analytics window. +

+ )} +
+ ); +} diff --git a/apps/gittensory-ui/src/routes/app.analytics.tsx b/apps/gittensory-ui/src/routes/app.analytics.tsx index 68650d5e1f..229357d512 100644 --- a/apps/gittensory-ui/src/routes/app.analytics.tsx +++ b/apps/gittensory-ui/src/routes/app.analytics.tsx @@ -14,6 +14,8 @@ import { GatePrecisionCard } from "@/components/site/app-panels/gate-precision-c import type { GateEvalReport } from "@/components/site/app-panels/gate-precision-card-model"; import { CycleTimeCard } from "@/components/site/app-panels/cycle-time-card"; import type { CycleTimeAggregate } from "@/components/site/app-panels/cycle-time-card-model"; +import { SlopBandCalibrationCard } from "@/components/site/app-panels/slop-band-calibration-card"; +import type { SlopBandCalibration } from "@/components/site/app-panels/slop-band-calibration-card-model"; import { useApiResource } from "@/lib/api/use-api-resource"; export const Route = createFileRoute("/app/analytics")({ @@ -105,6 +107,7 @@ type OperatorDashboard = { upstreamDrift?: { status?: string; openReportCount?: number } | null; gateEval?: GateEvalReport; cycleTime?: CycleTimeAggregate; + slopBandCalibration?: SlopBandCalibration; }; function ProductAnalytics() { @@ -190,6 +193,10 @@ function ProductAnalytics() { {data.cycleTime ? : null} + {data.slopBandCalibration ? ( + + ) : null} + {data.usageSummary ? ( }; /** PR review cycle-time percentiles (gate decision → outcome) from review_audit (#2194). */ cycleTime: CycleTimeAggregate; + /** Slop-band merge/close calibration over resolved PRs carrying a persisted band (#2196). Bands only — never raw scores. */ + slopBandCalibration: SlopOutcomeCalibration; } /** ms between the gate decision and the resolution; null if implausible (NaN or negative). */ @@ -296,6 +308,32 @@ export async function computeCycleTimeAggregate( } } +function pullRequestResolvedInWindow(pr: PullRequestRecord, fromMs: number): boolean { + const outcomeAt = + pr.mergedAt ?? + pr.closedAt ?? + (pr.state === "closed" || pr.state === "merged" ? pr.updatedAt : null); + if (!outcomeAt) return false; + const ms = new Date(outcomeAt).getTime(); + return Number.isFinite(ms) && ms >= fromMs; +} + +/** Fleet-wide slop-band calibration for the stats feed (#2196). Fail-safe → empty calibration. */ +export async function computeSlopBandCalibrationAggregate( + env: Env, + opts: { days: number; nowMs: number }, +): Promise { + const days = Number.isFinite(opts.days) && opts.days > 0 ? Math.min(opts.days, 730) : 90; + const fromMs = opts.nowMs - days * 86_400_000; + try { + const pullRequests = await listAllPullRequests(env); + const inWindow = pullRequests.filter((pr) => pullRequestResolvedInWindow(pr, fromMs)); + return buildSlopOutcomeCalibration(inWindow); + } catch { + return EMPTY_SLOP_BAND_CALIBRATION; + } +} + /** Fold per-PR persisted minutes into the maintainer aggregate (avg band + total minutes). */ export function aggregateReviewEffort(perPrMinutes: number[]): ReviewEffortAggregate { if (perPrMinutes.length === 0) { @@ -324,7 +362,7 @@ export async function computeStats( const bucketExpr = BUCKET_SQL[bucket] ?? BUCKET_SQL.day; const fromIso = new Date(opts.nowMs - days * 86_400_000).toISOString().slice(0, 10); // YYYY-MM-DD - const [decisionRows, reversalRows, effortRows, cycleTime] = await Promise.all([ + const [decisionRows, reversalRows, effortRows, cycleTime, slopBandCalibration] = await Promise.all([ storage(env).prepare( `SELECT ${bucketExpr} AS bucket, project, COALESCE(verdict, status) AS verdict, COUNT(*) AS n FROM review_targets @@ -359,6 +397,7 @@ export async function computeStats( ).bind(fromIso).all<{ minutes: number }>() .catch(() => ({ results: [] as Array<{ minutes: number }> })), computeCycleTimeAggregate(env, { days, nowMs: opts.nowMs }), + computeSlopBandCalibrationAggregate(env, { days, nowMs: opts.nowMs }), ]); // Non-content gate decisions (incl. SHADOW would-actions) — recorded as `gate_decision` audit rows with @@ -396,6 +435,7 @@ export async function computeStats( recommendations, gateParity: { ...parity, cutoverReady: parity.rows.map((r) => ({ project: r.project, ready: isParityCutoverReady(r) })) }, cycleTime, + slopBandCalibration, }; } diff --git a/src/services/operator-dashboard.ts b/src/services/operator-dashboard.ts index dcaa0a2630..01a0578a50 100644 --- a/src/services/operator-dashboard.ts +++ b/src/services/operator-dashboard.ts @@ -27,7 +27,12 @@ import type { } from "../types"; import { computeFleetAnalytics, type FleetAnalytics } from "../orb/analytics"; import { computeGateEval, type GateEvalReport } from "../review/parity"; -import { computeCycleTimeAggregate, type CycleTimeAggregate } from "../review/stats"; +import { + computeCycleTimeAggregate, + computeSlopBandCalibrationAggregate, + type CycleTimeAggregate, + type SlopOutcomeCalibration, +} from "../review/stats"; import { loadUpstreamStatus, type UpstreamStatus } from "../upstream/ruleset"; import { nowIso } from "../utils/json"; import { buildRecommendationQualityReport, type RecommendationQualityReport } from "./recommendation-quality-report"; @@ -66,6 +71,8 @@ export type OperatorDashboardPayload = { gateEval: GateEvalReport; // PR review cycle-time percentiles (#2194): gate decision → outcome from review_audit; fail-safe empty aggregate. cycleTime: CycleTimeAggregate; + // Slop-band calibration (#2196): predicted band vs realized merge/close outcome; bands only, never raw scores. + slopBandCalibration: SlopOutcomeCalibration; }; const USAGE_WINDOW_DAYS = 7; @@ -91,6 +98,7 @@ export async function buildOperatorDashboardPayload(env: Env): Promise { distribution: [], sampleSize: 0, }); + expect(payload.slopBandCalibration).toEqual({ + totalResolved: 0, + overallMergeRate: null, + discriminates: null, + bands: [ + { band: "clean", sampleSize: 0, merged: 0, closed: 0, mergeRate: 0 }, + { band: "low", sampleSize: 0, merged: 0, closed: 0, mergeRate: 0 }, + { band: "elevated", sampleSize: 0, merged: 0, closed: 0, mergeRate: 0 }, + { band: "high", sampleSize: 0, merged: 0, closed: 0, mergeRate: 0 }, + ], + }); // Empty fleet → instanceCount 0, null precision card ("—"), no-outlier delta. expect(payload.fleetMetrics.instanceCount).toBe(0); expect(payload.metrics).toEqual( diff --git a/test/unit/stats.test.ts b/test/unit/stats.test.ts index 5007f507ce..592cc2b0b2 100644 --- a/test/unit/stats.test.ts +++ b/test/unit/stats.test.ts @@ -4,9 +4,11 @@ import { aggregateCycleTimePercentiles, aggregateReviewEffort, buildCycleTimeDistribution, + computeSlopBandCalibrationAggregate, computeStats, cycleTimeMs, EMPTY_CYCLE_TIME, + EMPTY_SLOP_BAND_CALIBRATION, handleParity, handleStats, isParityCutoverReady, @@ -16,6 +18,7 @@ import { type GateParityRow, type StatsEvalDeps, } from "../../src/review/stats"; +import { updatePullRequestSlopAssessment, upsertPullRequestFromGitHub } from "../../src/db/repositories"; // Stub D1: route by table name — review_audit → reversals, else decision rows. function stubEnv(extra: Record = {}): Env { @@ -86,6 +89,7 @@ describe("computeStats — D1 aggregate for the dashboard", () => { expect(out.cycleTime.sampleSize).toBe(2); expect(out.cycleTime.p50Ms).toBe(300_000); expect(out.cycleTime.distribution.length).toBeGreaterThan(0); + expect(out.slopBandCalibration).toEqual(EMPTY_SLOP_BAND_CALIBRATION); }); it("clamps an absurd window and falls back to a safe bucket", async () => { @@ -423,6 +427,7 @@ describe("computeStats — NaN window + null D1 results (the ?? [] fallbacks)", expect(out.verdicts).toEqual([]); expect(out.reviewEffort).toEqual({ avgBand: null, totalEstimatedMinutes: 0 }); expect(out.cycleTime).toEqual(EMPTY_CYCLE_TIME); + expect(out.slopBandCalibration).toEqual(EMPTY_SLOP_BAND_CALIBRATION); }); }); @@ -561,6 +566,53 @@ describe("cycle-time aggregation (#2194)", () => { }); }); +describe("slop-band calibration aggregation (#2196)", () => { + it("computeSlopBandCalibrationAggregate folds resolved PR bands in the stats window", async () => { + const env = createTestEnv(); + await upsertPullRequestFromGitHub(env, "owner/repo", { + number: 1, + title: "clean merged", + state: "closed", + user: { login: "alice" }, + merged_at: "2026-06-10T12:00:00.000Z", + }); + await updatePullRequestSlopAssessment(env, "owner/repo", 1, { slopRisk: 0, slopBand: "clean" }); + await upsertPullRequestFromGitHub(env, "owner/repo", { + number: 2, + title: "high closed", + state: "closed", + user: { login: "bob" }, + updated_at: "2026-06-11T12:00:00.000Z", + }); + await updatePullRequestSlopAssessment(env, "owner/repo", 2, { slopRisk: 80, slopBand: "high" }); + const agg = await computeSlopBandCalibrationAggregate(env, { days: 90, nowMs: NOW }); + expect(agg.totalResolved).toBe(2); + expect(agg.bands.find((row) => row.band === "clean")).toMatchObject({ sampleSize: 1, merged: 1, closed: 0 }); + expect(agg.bands.find((row) => row.band === "high")).toMatchObject({ sampleSize: 1, merged: 0, closed: 1 }); + }); + + it("computeSlopBandCalibrationAggregate fails safe to EMPTY_SLOP_BAND_CALIBRATION when reads reject", async () => { + const env = { + DB: { + prepare: () => ({ + bind: () => ({ all: async () => { throw new Error("d1 down"); } }), + }), + select: () => { throw new Error("d1 down"); }, + }, + } as unknown as Env; + expect(await computeSlopBandCalibrationAggregate(env, { days: 30, nowMs: NOW })).toEqual( + EMPTY_SLOP_BAND_CALIBRATION, + ); + }); + + it("computeSlopBandCalibrationAggregate defaults non-finite days to 90", async () => { + const env = createTestEnv(); + expect(await computeSlopBandCalibrationAggregate(env, { days: Number.NaN, nowMs: NOW })).toEqual( + EMPTY_SLOP_BAND_CALIBRATION, + ); + }); +}); + describe("isParityCutoverReady — every gate condition", () => { const base: GateParityRow = { project: "p",