diff --git a/src/api/routes.ts b/src/api/routes.ts index 475902ed15..74caa6eef3 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -319,6 +319,7 @@ import { isFairnessAnalyticsEnabled, resolveFairnessAnalyticsManifestOverride } import { isRagEnabled } from "../review/rag-wire"; import { getPublicStats, isPublicStatsEnabled, resolvePublicStatsManifestOverride } from "../review/public-stats"; import { loadPublicAccuracyTrend } from "../services/public-accuracy-trend"; +import { loadCalibrationTrend } from "../services/rule-calibration-trend"; import { loadPublicReuseRateTrend } from "../services/public-reuse-rate-trend"; import { loadPublicReviewVolumeTrend } from "../services/public-review-volume-trend"; import { buildMaintainerQualityDashboard, isMaintainerQualityDataStale } from "../services/maintainer-quality-dashboard"; @@ -4806,6 +4807,12 @@ export function createApp() { // Fails safe to an empty-but-shaped report when there is no review signal yet. Aggregate counts only. app.get("/v1/internal/calibration", (c) => handleInternalCalibration(c.req.raw, c.env, internalOpsAgentConfig(c.env))); + // Operator calibration trend (#8113): weekly per-rule fired/decided/precision plus backtest-run verdict + // counts, re-bucketed live from audit_events (no cron rollup — see rule-calibration-trend.ts's header). Sibling + // of /v1/internal/calibration above, same INTERNAL_JOB_TOKEN gate via the /v1/internal/* middleware. + // Aggregate counts and rule ids only — no PR content, no raw context. + app.get("/v1/internal/calibration-trend", async (c) => c.json(await loadCalibrationTrend(c.env))); + app.post("/v1/internal/jobs/refresh-registry", async (c) => { const message: JobMessage = { type: "refresh-registry", requestedBy: "api" }; await c.env.JOBS.send(message); diff --git a/src/services/rule-calibration-trend.ts b/src/services/rule-calibration-trend.ts new file mode 100644 index 0000000000..e1c346f38d --- /dev/null +++ b/src/services/rule-calibration-trend.ts @@ -0,0 +1,209 @@ +// Rule/AI-judgment calibration trend (#8113, epic #8082). The fired+override history (#8101/#8104) and the +// persisted backtest runs (#8138/#8139) previously had no aggregate view — only per-PR advisory comments — +// so "is precision for rule X trending up or down" meant manually re-running CLIs. This is the maintainer- +// facing sibling of public-accuracy-trend.ts (#4447): the SAME deliberate no-cron posture (audit_events is +// already durable, so a live weekly re-bucketing recomputes any historical week correctly on every request — +// no rollup copy to drift), served from the /v1/internal/* operator surface, NOT the public stats payload +// (rule-level precision is operator observability, not homepage material). +// +// Precision semantics, deliberately trend-grained: a week's `decided` counts the human override events whose +// own created_at falls in that week (the DECISION week), and precisionPct = confirmed/decided over them. +// This is intentionally NOT computeRulePrecision's per-target fired↔override pairing (that corpus-exact +// pairing needs full event metadata, not day rollups) — the two answer different questions ("how are humans +// judging this rule's calls lately" vs "score this exact corpus") and must not be conflated. +import { safeAll } from "../review/public-stats"; +import { isoWeekStart } from "./public-quality-metrics"; + +export const CALIBRATION_TREND_WEEKS = 8; +/** Below this many decided (confirmed+reversed) verdicts in a week, that week's precision is too noisy to + * report — mirrors MIN_ACCURACY_TREND_SAMPLE's role in the public trend. */ +export const MIN_CALIBRATION_TREND_SAMPLE = 3; + +const RULE_FIRED_EVENT_TYPE_PREFIX = "signal.rule_fired:"; +const HUMAN_OVERRIDE_EVENT_TYPE_PREFIX = "signal.human_override:"; +// Mirrors THRESHOLD_BACKTEST_EVENT_TYPE (src/services/threshold-backtest-run.ts) and the CI writer's +// LOGIC_BACKTEST_EVENT_TYPE (scripts/backtest-logic-check-core.ts) — the same hand-mirrored posture +// scripts/backtest-track-record.ts documents for why the scripts-side constant isn't imported here. +const BACKTEST_RUN_EVENT_TYPES = ["calibration.threshold_backtest_run", "calibration.logic_backtest_run"] as const; + +export type CalibrationRuleTrendWeek = { + /** UTC Monday (YYYY-MM-DD) that starts the bucket. */ + weekStart: string; + fired: number; + confirmed: number | null; + reversed: number | null; + precisionPct: number | null; +}; + +export type CalibrationRuleTrend = { ruleId: string; weeks: CalibrationRuleTrendWeek[] }; + +export type BacktestRunTrendWeek = { + weekStart: string; + runs: number; + regressed: number; + improved: number; + unchanged: number; +}; + +export type CalibrationTrendReport = { + rules: CalibrationRuleTrend[]; + backtestRuns: BacktestRunTrendWeek[]; +}; + +export type FiredDayRow = { ruleId: string; day: string; fired: number }; +export type OverrideDayRow = { ruleId: string; day: string; confirmed: number; reversed: number }; +export type BacktestRunDayRow = { day: string; regressed: number; improved: number; unchanged: number }; + +const MS_PER_WEEK = 7 * 86_400_000; + +function roundPct(value: number): number { + return Math.round(value * 1000) / 10; +} + +/** Week offset of a day row inside the trailing window, or null when the day is unparseable or outside it. */ +function weekOffsetOf(day: string, oldestStartMs: number, weeks: number): number | null { + const dayMs = Date.parse(`${day}T00:00:00.000Z`); + if (!Number.isFinite(dayMs)) return null; + const offset = Math.floor((dayMs - oldestStartMs) / MS_PER_WEEK); + return offset < 0 || offset >= weeks ? null : offset; +} + +/** + * Fold day-granularity calibration rows into `weeks` trailing UTC-Monday buckets ending in the week + * containing `nowMs`. Pure — mirrors buildPublicAccuracyTrend's bucketing shape exactly. Rules are sorted + * by ruleId for byte-stable output; a week with fewer than {@link MIN_CALIBRATION_TREND_SAMPLE} decided + * verdicts reports null confirmed/reversed/precisionPct (unknown stays unknown, never a fake 0 or 100). + */ +export function buildCalibrationTrend( + firedRows: readonly FiredDayRow[], + overrideRows: readonly OverrideDayRow[], + runRows: readonly BacktestRunDayRow[], + nowMs: number, + weeks: number = CALIBRATION_TREND_WEEKS, +): CalibrationTrendReport { + const currentStartMs = Date.parse(isoWeekStart(nowMs)); + const oldestStartMs = currentStartMs - (weeks - 1) * MS_PER_WEEK; + + const ruleBuckets = new Map>(); + const bucketsFor = (ruleId: string) => { + const existing = ruleBuckets.get(ruleId); + if (existing) return existing; + const created = Array.from({ length: weeks }, () => ({ fired: 0, confirmed: 0, reversed: 0 })); + ruleBuckets.set(ruleId, created); + return created; + }; + for (const row of firedRows) { + const offset = weekOffsetOf(row.day, oldestStartMs, weeks); + if (offset === null) continue; + bucketsFor(row.ruleId)[offset]!.fired += row.fired; + } + for (const row of overrideRows) { + const offset = weekOffsetOf(row.day, oldestStartMs, weeks); + if (offset === null) continue; + const bucket = bucketsFor(row.ruleId)[offset]!; + bucket.confirmed += row.confirmed; + bucket.reversed += row.reversed; + } + + const runBuckets = Array.from({ length: weeks }, () => ({ regressed: 0, improved: 0, unchanged: 0 })); + for (const row of runRows) { + const offset = weekOffsetOf(row.day, oldestStartMs, weeks); + if (offset === null) continue; + const bucket = runBuckets[offset]!; + bucket.regressed += row.regressed; + bucket.improved += row.improved; + bucket.unchanged += row.unchanged; + } + + const rules: CalibrationRuleTrend[] = [...ruleBuckets.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([ruleId, buckets]) => ({ + ruleId, + weeks: buckets.map((bucket, offset) => { + const decided = bucket.confirmed + bucket.reversed; + const publishable = decided >= MIN_CALIBRATION_TREND_SAMPLE; + return { + weekStart: isoWeekStart(oldestStartMs + offset * MS_PER_WEEK), + fired: bucket.fired, + confirmed: publishable ? bucket.confirmed : null, + reversed: publishable ? bucket.reversed : null, + precisionPct: publishable ? roundPct(bucket.confirmed / decided) : null, + }; + }), + })); + + const backtestRuns: BacktestRunTrendWeek[] = runBuckets.map((bucket, offset) => ({ + weekStart: isoWeekStart(oldestStartMs + offset * MS_PER_WEEK), + runs: bucket.regressed + bucket.improved + bucket.unchanged, + regressed: bucket.regressed, + improved: bucket.improved, + unchanged: bucket.unchanged, + })); + + return { rules, backtestRuns }; +} + +/** Day-bucketed rule firings — the ruleId is recovered from the event_type suffix (signal-tracking-wire.ts + * folds it into the type: `signal.rule_fired:`). */ +async function loadFiredDayRows(env: Env, sinceIso: string): Promise { + const rows = await safeAll<{ rule_id: string; day: string; n: number }>( + env, + `SELECT substr(event_type, ${RULE_FIRED_EVENT_TYPE_PREFIX.length + 1}) AS rule_id, date(created_at) AS day, COUNT(*) AS n + FROM audit_events + WHERE event_type LIKE '${RULE_FIRED_EVENT_TYPE_PREFIX}%' AND created_at >= ? + GROUP BY rule_id, day`, + sinceIso, + ); + return rows.map((row) => ({ ruleId: row.rule_id, day: row.day, fired: row.n })); +} + +/** Day-bucketed human verdicts, split confirmed/reversed via the recorded `$.verdict` (signal-tracking-wire's + * recordHumanOverride writes it) — bucketed by the override's OWN created_at: this trend reports how humans + * are judging a rule's calls per decision week (see the module doc's precision-semantics note). */ +async function loadOverrideDayRows(env: Env, sinceIso: string): Promise { + const rows = await safeAll<{ rule_id: string; day: string; confirmed: number; reversed: number }>( + env, + `SELECT substr(event_type, ${HUMAN_OVERRIDE_EVENT_TYPE_PREFIX.length + 1}) AS rule_id, date(created_at) AS day, + SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'reversed' THEN 0 ELSE 1 END) AS confirmed, + SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'reversed' THEN 1 ELSE 0 END) AS reversed + FROM audit_events + WHERE event_type LIKE '${HUMAN_OVERRIDE_EVENT_TYPE_PREFIX}%' AND created_at >= ? + GROUP BY rule_id, day`, + sinceIso, + ); + /* v8 ignore next 2 -- SUM(CASE ...) over a GROUP BY always yields a defined integer, never SQL NULL; the ?? 0 + * fallbacks guard a future query-shape change, mirroring loadOrbDayRows' identical note. */ + return rows.map((row) => ({ ruleId: row.rule_id, day: row.day, confirmed: row.confirmed ?? 0, reversed: row.reversed ?? 0 })); +} + +/** Day-bucketed backtest runs across BOTH sibling event types, verdict read from the persisted + * `$.comparison.verdict` (the field backtest-track-record.ts's reader also anchors on). A row whose verdict + * is missing/unrecognized counts as `unchanged` — a malformed run must not vanish from `runs` entirely. */ +async function loadBacktestRunDayRows(env: Env, sinceIso: string): Promise { + const inList = BACKTEST_RUN_EVENT_TYPES.map((eventType) => `'${eventType}'`).join(", "); + const rows = await safeAll<{ day: string; regressed: number; improved: number; unchanged: number }>( + env, + `SELECT date(created_at) AS day, + SUM(CASE WHEN json_extract(metadata_json, '$.comparison.verdict') = 'regressed' THEN 1 ELSE 0 END) AS regressed, + SUM(CASE WHEN json_extract(metadata_json, '$.comparison.verdict') = 'improved' THEN 1 ELSE 0 END) AS improved, + SUM(CASE WHEN json_extract(metadata_json, '$.comparison.verdict') NOT IN ('regressed', 'improved') OR json_extract(metadata_json, '$.comparison.verdict') IS NULL THEN 1 ELSE 0 END) AS unchanged + FROM audit_events + WHERE event_type IN (${inList}) AND created_at >= ? + GROUP BY day`, + sinceIso, + ); + /* v8 ignore next 2 -- same SUM(CASE)-never-NULL note as loadOverrideDayRows above. */ + return rows.map((row) => ({ day: row.day, regressed: row.regressed ?? 0, improved: row.improved ?? 0, unchanged: row.unchanged ?? 0 })); +} + +/** Assemble the calibration trend live from audit_events. Fail-safe: each query degrades to [] on error + * (safeAll), so a single bad query yields under-counted weeks rather than a thrown operator endpoint. */ +export async function loadCalibrationTrend(env: Env, nowMs: number = Date.now()): Promise { + const sinceIso = new Date(Date.parse(isoWeekStart(nowMs)) - (CALIBRATION_TREND_WEEKS - 1) * MS_PER_WEEK).toISOString(); + const [firedRows, overrideRows, runRows] = await Promise.all([ + loadFiredDayRows(env, sinceIso), + loadOverrideDayRows(env, sinceIso), + loadBacktestRunDayRows(env, sinceIso), + ]); + return buildCalibrationTrend(firedRows, overrideRows, runRows, nowMs); +} diff --git a/test/unit/rule-calibration-trend.test.ts b/test/unit/rule-calibration-trend.test.ts new file mode 100644 index 0000000000..a263c7762a --- /dev/null +++ b/test/unit/rule-calibration-trend.test.ts @@ -0,0 +1,198 @@ +import { describe, expect, it } from "vitest"; +import { + CALIBRATION_TREND_WEEKS, + MIN_CALIBRATION_TREND_SAMPLE, + buildCalibrationTrend, + loadCalibrationTrend, + type BacktestRunDayRow, + type FiredDayRow, + type OverrideDayRow, +} from "../../src/services/rule-calibration-trend"; +import { isoWeekStart } from "../../src/services/public-quality-metrics"; +import { createApp } from "../../src/api/routes"; +import { recordAuditEvent } from "../../src/db/repositories"; +import { createTestEnv } from "../helpers/d1"; + +const NOW = Date.parse("2026-07-22T12:00:00.000Z"); +const WEEK_MS = 7 * 86_400_000; +const currentMonday = isoWeekStart(NOW); +const priorMonday = isoWeekStart(NOW - WEEK_MS); + +describe("buildCalibrationTrend (#8113)", () => { + it("buckets fired + override day rows per rule per week and computes precision as confirmed/decided", () => { + const fired: FiredDayRow[] = [ + { ruleId: "linked_issue_scope_mismatch", day: priorMonday, fired: 4 }, + // A second day in the SAME week — must accumulate. + { ruleId: "linked_issue_scope_mismatch", day: priorMonday, fired: 2 }, + ]; + const overrides: OverrideDayRow[] = [{ ruleId: "linked_issue_scope_mismatch", day: priorMonday, confirmed: 3, reversed: 1 }]; + const trend = buildCalibrationTrend(fired, overrides, [], NOW, 2); + expect(trend.rules).toHaveLength(1); + const [rule] = trend.rules; + expect(rule!.ruleId).toBe("linked_issue_scope_mismatch"); + expect(rule!.weeks).toEqual([ + { weekStart: priorMonday, fired: 6, confirmed: 3, reversed: 1, precisionPct: 75 }, + { weekStart: currentMonday, fired: 0, confirmed: null, reversed: null, precisionPct: null }, + ]); + }); + + it("keeps a week's verdict split null below MIN_CALIBRATION_TREND_SAMPLE decided — unknown never fakes 0 or 100", () => { + const overrides: OverrideDayRow[] = [{ ruleId: "duplicate_pr_risk", day: currentMonday, confirmed: MIN_CALIBRATION_TREND_SAMPLE - 1, reversed: 0 }]; + const trend = buildCalibrationTrend([], overrides, [], NOW, 1); + expect(trend.rules[0]!.weeks[0]).toEqual({ weekStart: currentMonday, fired: 0, confirmed: null, reversed: null, precisionPct: null }); + }); + + it("creates a rule bucket from an override-only history (no firings recorded in the window)", () => { + const overrides: OverrideDayRow[] = [{ ruleId: "missing_linked_issue", day: currentMonday, confirmed: 2, reversed: 2 }]; + const trend = buildCalibrationTrend([], overrides, [], NOW, 1); + expect(trend.rules[0]!.weeks[0]!.precisionPct).toBe(50); + expect(trend.rules[0]!.weeks[0]!.fired).toBe(0); + }); + + it("sorts rules by ruleId for byte-stable output", () => { + const fired: FiredDayRow[] = [ + { ruleId: "zeta_rule", day: currentMonday, fired: 1 }, + { ruleId: "alpha_rule", day: currentMonday, fired: 1 }, + ]; + expect(buildCalibrationTrend(fired, [], [], NOW, 1).rules.map((rule) => rule.ruleId)).toEqual(["alpha_rule", "zeta_rule"]); + }); + + it("drops rows outside the window and unparseable days, for every row kind", () => { + const outside = isoWeekStart(NOW - 3 * WEEK_MS); + const future = isoWeekStart(NOW + 2 * WEEK_MS); + const fired: FiredDayRow[] = [ + { ruleId: "r", day: outside, fired: 5 }, + { ruleId: "r", day: "not-a-day", fired: 5 }, + ]; + const overrides: OverrideDayRow[] = [{ ruleId: "r", day: future, confirmed: 5, reversed: 5 }]; + const runs: BacktestRunDayRow[] = [{ day: "junk", regressed: 1, improved: 1, unchanged: 1 }]; + const trend = buildCalibrationTrend(fired, overrides, runs, NOW, 2); + expect(trend.rules).toEqual([]); + expect(trend.backtestRuns.every((week) => week.runs === 0)).toBe(true); + }); + + it("buckets backtest runs per week with verdict counts and a runs total", () => { + const runs: BacktestRunDayRow[] = [ + { day: priorMonday, regressed: 1, improved: 2, unchanged: 0 }, + { day: priorMonday, regressed: 0, improved: 0, unchanged: 3 }, + { day: currentMonday, regressed: 0, improved: 1, unchanged: 0 }, + ]; + const trend = buildCalibrationTrend([], [], runs, NOW, 2); + expect(trend.backtestRuns).toEqual([ + { weekStart: priorMonday, runs: 6, regressed: 1, improved: 2, unchanged: 3 }, + { weekStart: currentMonday, runs: 1, regressed: 0, improved: 1, unchanged: 0 }, + ]); + }); + + it("defaults to CALIBRATION_TREND_WEEKS trailing buckets", () => { + const trend = buildCalibrationTrend([], [], [], NOW); + expect(trend.backtestRuns).toHaveLength(CALIBRATION_TREND_WEEKS); + expect(trend.backtestRuns[0]!.weekStart).toBe(isoWeekStart(NOW - (CALIBRATION_TREND_WEEKS - 1) * WEEK_MS)); + }); +}); + +describe("loadCalibrationTrend (#8113)", () => { + async function seed(env: Env) { + const inWindow = new Date(NOW - WEEK_MS).toISOString(); + await recordAuditEvent(env, { + eventType: "signal.rule_fired:linked_issue_scope_mismatch", + actor: "loopover", + targetKey: "o/r#1", + outcome: "completed", + metadata: { outcome: "unaddressed" }, + createdAt: inWindow, + }); + for (const verdict of ["confirmed", "confirmed", "confirmed", "reversed"]) { + await recordAuditEvent(env, { + eventType: "signal.human_override:linked_issue_scope_mismatch", + actor: "human", + targetKey: "o/r#1", + outcome: "completed", + metadata: { verdict }, + createdAt: inWindow, + }); + } + // One run per sibling event type, plus one with a missing verdict (counts as unchanged, never vanishes). + await recordAuditEvent(env, { + eventType: "calibration.threshold_backtest_run", + actor: "loopover", + targetKey: "o/r#2", + outcome: "completed", + metadata: { comparison: { ruleId: "x", verdict: "regressed" } }, + createdAt: inWindow, + }); + await recordAuditEvent(env, { + eventType: "calibration.logic_backtest_run", + actor: "loopover", + targetKey: "o/r#3", + outcome: "completed", + metadata: { comparison: { ruleId: "x", verdict: "improved" } }, + createdAt: inWindow, + }); + await recordAuditEvent(env, { + eventType: "calibration.logic_backtest_run", + actor: "loopover", + targetKey: "o/r#4", + outcome: "completed", + metadata: {}, + createdAt: inWindow, + }); + // Outside the window — must be excluded by the SQL since-filter. + await recordAuditEvent(env, { + eventType: "signal.rule_fired:linked_issue_scope_mismatch", + actor: "loopover", + targetKey: "o/r#5", + outcome: "completed", + metadata: { outcome: "unaddressed" }, + createdAt: new Date(NOW - (CALIBRATION_TREND_WEEKS + 2) * WEEK_MS).toISOString(), + }); + } + + it("reads fired/override/run history out of audit_events and buckets it", async () => { + const env = createTestEnv(); + await seed(env); + const trend = await loadCalibrationTrend(env, NOW); + const rule = trend.rules.find((entry) => entry.ruleId === "linked_issue_scope_mismatch"); + const priorWeek = rule!.weeks.find((week) => week.weekStart === priorMonday); + expect(priorWeek).toEqual({ weekStart: priorMonday, fired: 1, confirmed: 3, reversed: 1, precisionPct: 75 }); + const runWeek = trend.backtestRuns.find((week) => week.weekStart === priorMonday); + expect(runWeek).toEqual({ weekStart: priorMonday, runs: 3, regressed: 1, improved: 1, unchanged: 1 }); + }); + + it("returns an empty-but-shaped report on a fresh database", async () => { + const trend = await loadCalibrationTrend(createTestEnv(), NOW); + expect(trend.rules).toEqual([]); + expect(trend.backtestRuns).toHaveLength(CALIBRATION_TREND_WEEKS); + expect(trend.backtestRuns.every((week) => week.runs === 0)).toBe(true); + }); +}); + +describe("GET /v1/internal/calibration-trend (#8113)", () => { + it("401s without the internal token (the /v1/internal/* middleware gate)", async () => { + const app = createApp(); + const env = createTestEnv(); + expect((await app.request("/v1/internal/calibration-trend", {}, env)).status).toBe(401); + expect((await app.request("/v1/internal/calibration-trend", { headers: { authorization: "Bearer nope" } }, env)).status).toBe(401); + }); + + it("200s with the report shape and stays aggregate-only (no PR content, no raw context, no private terms)", async () => { + const app = createApp(); + const env = createTestEnv(); + await recordAuditEvent(env, { + eventType: "signal.rule_fired:duplicate_pr_risk", + actor: "loopover", + targetKey: "o/r#9", + outcome: "completed", + metadata: { outcome: "warning", diff: "SECRET-DIFF-CONTENT-MUST-NOT-LEAK" }, + createdAt: new Date().toISOString(), + }); + const res = await app.request("/v1/internal/calibration-trend", { headers: { authorization: `Bearer ${env.INTERNAL_JOB_TOKEN}` } }, env); + expect(res.status).toBe(200); + const body = (await res.json()) as { rules: Array<{ ruleId: string }>; backtestRuns: unknown[] }; + expect(body.rules.map((rule) => rule.ruleId)).toContain("duplicate_pr_risk"); + expect(Array.isArray(body.backtestRuns)).toBe(true); + const raw = JSON.stringify(body); + expect(raw).not.toContain("SECRET-DIFF-CONTENT-MUST-NOT-LEAK"); + expect(raw).not.toMatch(/reward|payout|trust|wallet|hotkey/i); + }); +});