diff --git a/src/review/rule-gate-eval.ts b/src/review/rule-gate-eval.ts index f6d01e2cd9..975dd65c92 100644 --- a/src/review/rule-gate-eval.ts +++ b/src/review/rule-gate-eval.ts @@ -258,3 +258,18 @@ export function rulesBelowClosePrecisionFloor( ): BlendedRuleGateEvalRow[] { return rows.filter((r) => r.wouldClose >= minDecided && r.weightedClosePrecision != null && r.weightedClosePrecision < floor); } + +/** + * The per-(project, ruleCode) counterpart to {@link rulesBelowClosePrecisionFloor}: same floor/minDecided + * check, over an already-fetched {@link RuleGateEvalReport}'s rows instead of the blended ones (no I/O). This + * is the "which rule is broken on which repo" view (#8906) — a rule that looks healthy once pooled across + * every project can still be systematically wrong on ONE of them, diluted out of the blended figure by every + * other project's correct usage of the same rule. + */ +export function projectRulesBelowClosePrecisionFloor( + rows: readonly RuleGateEvalRow[], + floor: number = AUTOTUNE_CLOSE_PRECISION_FLOOR, + minDecided: number = AUTOTUNE_MIN_DECIDED, +): RuleGateEvalRow[] { + return rows.filter((r) => r.wouldClose >= minDecided && r.weightedClosePrecision != null && r.weightedClosePrecision < floor); +} diff --git a/src/services/operator-dashboard.ts b/src/services/operator-dashboard.ts index e1e75ae97c..b80b74ad46 100644 --- a/src/services/operator-dashboard.ts +++ b/src/services/operator-dashboard.ts @@ -34,7 +34,7 @@ import { computeFleetAnalytics, getFleetHealthSummary, type FleetAnalytics, type import { computeAgentHealth, computeCalibration, type AgentHealth, type Calibration } from "../review/ops"; import { computeGateEval, type GateEvalReport } from "../review/parity"; import { computeContributorGateEval, contributorFairnessFlags, computeBlendedContributorGateEval, contributorGlobalFairnessFlags } from "../review/contributor-gate-eval"; -import { computeBlendedRuleGateEval, rulesBelowClosePrecisionFloor } from "../review/rule-gate-eval"; +import { computeRuleGateEval, computeBlendedRuleGateEval, rulesBelowClosePrecisionFloor, projectRulesBelowClosePrecisionFloor } from "../review/rule-gate-eval"; import { computeCycleTimeAggregate, computeFindingAcceptance, type CycleTimeAggregate } from "../review/stats"; import { loadUpstreamStatus, type UpstreamStatus } from "../upstream/ruleset"; import { nowIso } from "../utils/json"; @@ -140,6 +140,7 @@ export async function buildOperatorDashboardPayload( gateEval, contributorGateEval, blendedContributorGateEval, + ruleGateEval, blendedRuleGateEval, cycleTime, calibration, @@ -172,6 +173,9 @@ export async function buildOperatorDashboardPayload( computeContributorGateEval(env, { days: GATE_ANALYTICS_WINDOW_DAYS, nowMs: Date.now() }), // #global-contributor-trust: the SAME data pooled cross-repo into one blended figure per login, same window. computeBlendedContributorGateEval(env, { days: GATE_ANALYTICS_WINDOW_DAYS, nowMs: Date.now() }), + // #8906: per-(project, ruleCode) gate accuracy -- the "which rule is broken on which repo" view that + // computeBlendedRuleGateEval's cross-repo pooling can't provide. Fails safe to an empty report. + computeRuleGateEval(env, { days: GATE_ANALYTICS_WINDOW_DAYS, nowMs: Date.now() }), // #7984: the SAME review_audit data re-aggregated by RULE CODE (pooled cross-repo) instead of by project — // isolates a single systematically-wrong deterministic rule even while its host project's own aggregate // still looks healthy. Fails safe to an empty report on any read error. @@ -214,6 +218,9 @@ export async function buildOperatorDashboardPayload( // #7984: rules whose sample has cleared enough volume to trust but whose weighted close precision sits // below the SAME floor auto-tune.ts's project-level close-precision breaker uses -- pure fold, no extra I/O. const rulesBelowFloor = rulesBelowClosePrecisionFloor(blendedRuleGateEval.rows); + // #8906: the SAME floor check, per (project, ruleCode) instead of pooled -- surfaces a rule that's broken on + // one specific repo even while the blended figure above still looks healthy. + const projectRulesBelowFloor = projectRulesBelowClosePrecisionFloor(ruleGateEval.rows); const installedRepos = repositories.filter((repo: RepositoryRecord) => repo.isInstalled).length; const registeredRepos = repositories.filter((repo: RepositoryRecord) => repo.isRegistered).length; // #1967: map FindingAcceptanceAggregate's field names onto the AcceptanceRateCard's expected shape. @@ -304,6 +311,17 @@ export async function buildOperatorDashboardPayload( value: String(rulesBelowFloor.length), delta: rulesBelowFloor.length > 0 ? rulesBelowFloor.map((r) => r.ruleCode).join(", ") : "no rule below floor", }, + { + // #8906: the per-(project, ruleCode) counterpart to the blended tile above -- names the specific repo + // a rule is broken on, same "name the specific thing" posture (rule codes and repo names are not + // personally identifying, unlike the contributor tiles' logins). + label: "Rules below close-precision floor (per-project)", + value: String(projectRulesBelowFloor.length), + delta: + projectRulesBelowFloor.length > 0 + ? projectRulesBelowFloor.map((r) => `${r.project}:${r.ruleCode}`).join(", ") + : "no rule below floor", + }, ], noiseReduction: [ { diff --git a/test/unit/operator-dashboard.test.ts b/test/unit/operator-dashboard.test.ts index dfe2409cb3..40f840b5c2 100644 --- a/test/unit/operator-dashboard.test.ts +++ b/test/unit/operator-dashboard.test.ts @@ -250,6 +250,38 @@ describe("operator dashboard payload", () => { expect(JSON.stringify(payload)).not.toContain("missing_linked_issue"); // the healthy rule never appears }); + it("surfaces the per-project rule breakdown (#8906), catching a rule broken on ONE repo even while a second repo's healthy usage pools it back above the blended floor", async () => { + const env = createTestEnv(); + const seedClose = async (id: string, project: string, ruleCode: string, truth: "closed" | "merged"): Promise => { + await env.DB.prepare( + `INSERT INTO review_audit (id, project, target_id, event_type, decision, summary, source, created_at) VALUES (?, ?, ?, 'gate_decision', 'close', ?, 'gittensory-native', ?)`, + ) + .bind(`gd-${id}`, project, `${project}#${id}`, ruleCode, new Date().toISOString()) + .run(); + await env.DB.prepare( + `INSERT INTO review_audit (id, project, target_id, event_type, decision, source, created_at) VALUES (?, ?, ?, 'pr_outcome', ?, 'github', ?)`, + ) + .bind(`po-${id}`, project, `${project}#${id}`, truth, new Date().toISOString()) + .run(); + }; + // Broken on ONE repo: 12 closes (clears AUTOTUNE_MIN_DECIDED), every single one later merged -- 0% precision. + for (let i = 1; i <= 12; i++) await seedClose(`bad-${i}`, "metagraphed/broken-repo", "shared_rule", "merged"); + // The SAME rule, healthy on a SECOND repo: 90 closes, all correctly closed -- pools the blended precision to + // 90/102 ≈ 0.882, above AUTOTUNE_CLOSE_PRECISION_FLOOR (0.85), so the blended tile does NOT flag this rule + // even though it is genuinely broken on metagraphed/broken-repo -- exactly the gap #8906 exists to close. + for (let i = 1; i <= 90; i++) await seedClose(`good-${i}`, "metagraphed/healthy-repo", "shared_rule", "closed"); + + const payload = await buildOperatorDashboardPayload(env); + expect(payload.metrics).toEqual( + expect.arrayContaining([expect.objectContaining({ label: "Rules below close-precision floor", value: "0", delta: "no rule below floor" })]), + ); + expect(payload.metrics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ label: "Rules below close-precision floor (per-project)", value: "1", delta: "metagraphed/broken-repo:shared_rule" }), + ]), + ); + }); + it("wires computeFindingAcceptance into the dashboard's acceptance card shape (#1967/#5213)", async () => { const env = createTestEnv(); await env.DB.prepare( diff --git a/test/unit/rule-gate-eval.test.ts b/test/unit/rule-gate-eval.test.ts index 345f3c19b7..004ef6ed77 100644 --- a/test/unit/rule-gate-eval.test.ts +++ b/test/unit/rule-gate-eval.test.ts @@ -3,6 +3,7 @@ import { computeRuleGateEval, computeBlendedRuleGateEval, rulesBelowClosePrecisionFloor, + projectRulesBelowClosePrecisionFloor, type RuleGateEvalRow, type BlendedRuleGateEvalRow, } from "../../src/review/rule-gate-eval"; @@ -383,3 +384,68 @@ describe("rulesBelowClosePrecisionFloor (#7984, #7986's own read)", () => { expect(rulesBelowClosePrecisionFloor(justBelow)).toHaveLength(1); }); }); + +function projectRow(overrides: Partial = {}): RuleGateEvalRow { + return { + project: "acme/widgets", + ruleCode: "rule_a", + wouldMerge: 0, + mergeConfirmed: 0, + mergeFalse: 0, + wouldClose: 0, + closeConfirmed: 0, + closeFalse: 0, + decided: 0, + mergePrecision: null, + closePrecision: null, + weightedMergeConfirmed: 0, + weightedCloseConfirmed: 0, + weightedMergePrecision: null, + weightedClosePrecision: null, + ...overrides, + }; +} + +describe("projectRulesBelowClosePrecisionFloor — per-(project, ruleCode) counterpart (#8906)", () => { + it("flags a rule once its sample clears AUTOTUNE_MIN_DECIDED with a below-floor weighted precision, on the SPECIFIC project it fired on", () => { + const rows = [ + projectRow({ project: "acme/widgets", ruleCode: "surface_lane_reject", wouldClose: 12, closeConfirmed: 0, weightedClosePrecision: 0 }), + ]; + const flagged = projectRulesBelowClosePrecisionFloor(rows, AUTOTUNE_CLOSE_PRECISION_FLOOR, AUTOTUNE_MIN_DECIDED); + expect(flagged).toHaveLength(1); + expect(flagged[0]?.project).toBe("acme/widgets"); + expect(flagged[0]?.ruleCode).toBe("surface_lane_reject"); + }); + + it("does NOT flag a rule with insufficient sample even at 0% precision, mirroring the blended fold's same floor", () => { + const rows = [projectRow({ ruleCode: "rare_rule", wouldClose: 3, closeConfirmed: 0, weightedClosePrecision: 0 })]; + expect(projectRulesBelowClosePrecisionFloor(rows, AUTOTUNE_CLOSE_PRECISION_FLOOR, AUTOTUNE_MIN_DECIDED)).toEqual([]); + }); + + it("does NOT flag a healthy rule with plenty of sample and good precision", () => { + const rows = [projectRow({ ruleCode: "healthy_rule", wouldClose: 40, closeConfirmed: 39, weightedClosePrecision: 39 / 40 })]; + expect(projectRulesBelowClosePrecisionFloor(rows, AUTOTUNE_CLOSE_PRECISION_FLOOR, AUTOTUNE_MIN_DECIDED)).toEqual([]); + }); + + it("does NOT flag a rule whose weightedClosePrecision is null (no decided close verdict at all)", () => { + const rows = [projectRow({ ruleCode: "merge_only_rule", wouldClose: 0, weightedClosePrecision: null })]; + expect(projectRulesBelowClosePrecisionFloor(rows, AUTOTUNE_CLOSE_PRECISION_FLOOR, AUTOTUNE_MIN_DECIDED)).toEqual([]); + }); + + it("defaults floor and minDecided to the SAME constants the blended fold uses", () => { + const atFloor = [projectRow({ ruleCode: "rule_a", wouldClose: 20, weightedClosePrecision: AUTOTUNE_CLOSE_PRECISION_FLOOR })]; + expect(projectRulesBelowClosePrecisionFloor(atFloor)).toEqual([]); + const justBelow = [projectRow({ ruleCode: "rule_a", wouldClose: 20, weightedClosePrecision: AUTOTUNE_CLOSE_PRECISION_FLOOR - 0.01 })]; + expect(projectRulesBelowClosePrecisionFloor(justBelow)).toHaveLength(1); + }); + + it("isolates a rule broken on ONE project even when a second project's healthy rows would pool it away in the blended view", () => { + const rows = [ + projectRow({ project: "acme/broken-repo", ruleCode: "shared_rule", wouldClose: 15, closeConfirmed: 0, weightedClosePrecision: 0 }), + projectRow({ project: "acme/healthy-repo", ruleCode: "shared_rule", wouldClose: 30, closeConfirmed: 30, weightedClosePrecision: 1 }), + ]; + const flagged = projectRulesBelowClosePrecisionFloor(rows, AUTOTUNE_CLOSE_PRECISION_FLOOR, AUTOTUNE_MIN_DECIDED); + expect(flagged).toHaveLength(1); + expect(flagged[0]?.project).toBe("acme/broken-repo"); + }); +});