From c622b14a630d394cac6b523e940611e9ecf34355 Mon Sep 17 00:00:00 2001 From: Clayton Date: Wed, 8 Jul 2026 08:59:24 -0500 Subject: [PATCH] feat(notifications): add calibration section for maintainer recap (#2243) Co-authored-by: Cursor --- src/services/maintainer-recap-calibration.ts | 80 ++++++++++++++++ .../unit/maintainer-recap-calibration.test.ts | 91 +++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 src/services/maintainer-recap-calibration.ts create mode 100644 test/unit/maintainer-recap-calibration.test.ts diff --git a/src/services/maintainer-recap-calibration.ts b/src/services/maintainer-recap-calibration.ts new file mode 100644 index 0000000000..6a1dc6e17f --- /dev/null +++ b/src/services/maintainer-recap-calibration.ts @@ -0,0 +1,80 @@ +// Maintainer-recap CALIBRATION section (#2243, content slice of the #1963 recap digest). +// +// Pure section builder over a RecapReport projection: surface the ground-truth accuracy signal — how many +// auto-actions humans reversed, and the reversal rate — without any raw score/reward internals. Reuses the +// AgentHealth.reversalRate contract from src/review/alerts.ts:56 (reversals / (merged + closed), 0 when +// nothing auto-acted) and mirrors detectAnomalies' calibration-drift plain-English phrasing at +// src/review/alerts.ts:175 for the drift-present note. +// +// Compatible with the full RecapReport (#2239 / maintainer-recap.ts) once it lands — this file only needs the +// window + totals.{merged,closed,reversals} projection so it can ship independently of the foundation builder. +import { PUBLIC_LOCAL_PATH_SCRUB_PATTERN } from "../signals/redaction"; + +/** Projection of RecapReport used by the calibration section. Structurally compatible with RecapReport.totals. */ +export type CalibrationRecapSource = { + windowDays: number; + totals: { + merged: number; + closed: number; + /** Auto-actions a human overrode in the window (AgentHealth.reversals / RecapReport.totals.reversals). */ + reversals: number; + }; +}; + +/** One titled digest section: structured fields for consumers + ready-to-emit lines for the formatter. */ +export type CalibrationRecapSection = { + title: string; + reversals: number; + /** reversals / (merged + closed) — 0 when nothing auto-acted (alerts.ts AgentHealth.reversalRate). */ + reversalRate: number; + /** Plain-English status line (drift / healthy / nothing-auto-acted). */ + note: string; + lines: string[]; +}; + +/** Public-safe scrub for free text pulled into the section (defense in depth — counts are the only inputs + * today). Mirrors review-recap.ts / weekly-value-report.ts. */ +function sanitizeRecapText(value: string): string { + return value.replace(PUBLIC_LOCAL_PATH_SCRUB_PATTERN, "").slice(0, 240); +} + +/** + * Pure calibration section over a RecapReport projection. + * + * - `reversalRate` = reversals / (merged + closed), **0 when the denominator is 0** (nothing auto-acted). + * - Note arms: drift-present (reversals > 0), healthy (auto-acted + zero reversals), zero-denominator. + */ +export function buildCalibrationRecapSection(report: CalibrationRecapSource): CalibrationRecapSection { + const reversals = report.totals.reversals; + const autoActed = report.totals.merged + report.totals.closed; + // Mirror ops.ts AgentHealth.reversalRate + alerts.ts:56 — zero-denominator stays 0 (not NaN/null). + const reversalRate = autoActed > 0 ? Number((reversals / autoActed).toFixed(3)) : 0; + const ratePct = Math.round(reversalRate * 100); + + let note: string; + if (autoActed === 0) { + // Nothing auto-acted branch — explicit so the digest still carries a calibration section. + note = `Nothing auto-acted in the last ${report.windowDays} day(s) (0 merged + 0 closed) — reversal rate is 0 (no denominator).`; + } else if (reversals > 0) { + // Mirror detectAnomalies calibration-drift phrasing (alerts.ts:175) without floor internals — RecapReport + // does not carry recommendedFloor / revertedMaxConfidence; the rate IS the calibration signal here. + note = `calibration drift: ${reversals} auto-action(s) were human-reverted (reversal-rate ${ratePct}%) over ${autoActed} merged/closed in the last ${report.windowDays} day(s). Consider reviewing confidenceFloor / close-gates for false automations.`; + } else { + note = `Calibration healthy: 0 auto-action(s) reverted over ${autoActed} merged/closed in the last ${report.windowDays} day(s) (reversal-rate 0%).`; + } + + const title = "Calibration"; + const lines = [ + `Reversals: ${reversals}`, + `Reversal rate: ${ratePct}%`, + note, + ].map(sanitizeRecapText); + + return { + title, + reversals, + reversalRate, + note: sanitizeRecapText(note), + lines, + }; +} diff --git a/test/unit/maintainer-recap-calibration.test.ts b/test/unit/maintainer-recap-calibration.test.ts new file mode 100644 index 0000000000..292f44f0ba --- /dev/null +++ b/test/unit/maintainer-recap-calibration.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from "vitest"; +import { + buildCalibrationRecapSection, + type CalibrationRecapSource, +} from "../../src/services/maintainer-recap-calibration"; + +const WINDOW = 7; + +function report(totals: CalibrationRecapSource["totals"], windowDays = WINDOW): CalibrationRecapSource { + return { windowDays, totals }; +} + +describe("buildCalibrationRecapSection (#2243)", () => { + it("emits a drift-present section when humans reversed auto-actions (reversals > 0 arm)", () => { + // 2 reversals over 10 merged+closed ⇒ rate 0.2 — the populated denominator / drift arm. + const section = buildCalibrationRecapSection(report({ merged: 7, closed: 3, reversals: 2 })); + expect(section.title).toBe("Calibration"); + expect(section.reversals).toBe(2); + expect(section.reversalRate).toBe(0.2); + expect(section.note).toMatch(/calibration drift/i); + expect(section.note).toMatch(/2 auto-action\(s\) were human-reverted/); + expect(section.note).toMatch(/reversal-rate 20%/); + expect(section.note).toMatch(/Consider reviewing confidenceFloor/); + expect(section.lines).toEqual([ + "Reversals: 2", + "Reversal rate: 20%", + section.note, + ]); + }); + + it("emits a healthy section when auto-actions resolved with zero reversals (healthy arm)", () => { + // Denominator > 0, reversals === 0 ⇒ healthy calibration (both sides of the reversals > 0 branch). + const section = buildCalibrationRecapSection(report({ merged: 5, closed: 2, reversals: 0 })); + expect(section.reversals).toBe(0); + expect(section.reversalRate).toBe(0); + expect(section.note).toMatch(/Calibration healthy/i); + expect(section.note).toMatch(/0 auto-action\(s\) reverted/); + expect(section.note).toMatch(/reversal-rate 0%/); + expect(section.note).not.toMatch(/calibration drift/i); + expect(section.lines[0]).toBe("Reversals: 0"); + expect(section.lines[1]).toBe("Reversal rate: 0%"); + }); + + it("returns reversalRate 0 when nothing auto-acted (zero-denominator / merged+closed === 0 arm)", () => { + // alerts.ts:56 — "0 when nothing auto-acted"; must NOT divide by zero or emit NaN. + const section = buildCalibrationRecapSection(report({ merged: 0, closed: 0, reversals: 0 })); + expect(section.reversals).toBe(0); + expect(section.reversalRate).toBe(0); + expect(Number.isFinite(section.reversalRate)).toBe(true); + expect(section.note).toMatch(/Nothing auto-acted/); + expect(section.note).toMatch(/no denominator/); + expect(section.lines[1]).toBe("Reversal rate: 0%"); + }); + + it("still reports a finite 0 rate when reversals are present but merged+closed is 0 (zero-denominator with stray count)", () => { + // Defensive: a ledger inconsistency must still take the nothing-auto-acted rate arm (denominator wins). + const section = buildCalibrationRecapSection(report({ merged: 0, closed: 0, reversals: 3 })); + expect(section.reversalRate).toBe(0); + expect(section.reversals).toBe(3); + expect(section.note).toMatch(/Nothing auto-acted/); + expect(section.note).not.toMatch(/calibration drift/i); + }); + + it("covers the merged-only and closed-only denominator arms (both sides of merged + closed)", () => { + const mergedOnly = buildCalibrationRecapSection(report({ merged: 4, closed: 0, reversals: 1 })); + expect(mergedOnly.reversalRate).toBe(0.25); + expect(mergedOnly.note).toMatch(/calibration drift/i); + + const closedOnly = buildCalibrationRecapSection(report({ merged: 0, closed: 4, reversals: 1 })); + expect(closedOnly.reversalRate).toBe(0.25); + expect(closedOnly.note).toMatch(/4 merged\/closed/); + }); + + it("rounds the reversal rate to three decimal places (mirrors ops.ts AgentHealth.reversalRate)", () => { + // 1/3 ⇒ 0.333… → Number((1/3).toFixed(3)) === 0.333; percent line uses Math.round ⇒ 33%. + const section = buildCalibrationRecapSection(report({ merged: 2, closed: 1, reversals: 1 })); + expect(section.reversalRate).toBe(0.333); + expect(section.lines[1]).toBe("Reversal rate: 33%"); + }); + + it("scrubs a local-path leak if one ever reaches the note via windowDays echo (defense-in-depth — note is count-derived today)", () => { + // The note only interpolates numbers + fixed copy today; this pins that EVERY emitted line still + // runs through sanitizeRecapText so a future free-text field cannot leak a path. + const section = buildCalibrationRecapSection(report({ merged: 1, closed: 0, reversals: 0 })); + for (const line of section.lines) { + expect(line).not.toMatch(/\/Users\//); + expect(line).not.toMatch(/C:\\/); + } + expect(section.note).not.toMatch(/\/Users\//); + }); +});