Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions src/review/rule-gate-eval.ts
Original file line number Diff line number Diff line change
Expand Up @@ -258,3 +258,18 @@ export function rulesBelowClosePrecisionFloor(
): BlendedRuleGateEvalRow[] {
return rows.filter((r) => r.wouldClose >= minDecided && r.weightedClosePrecision != null && r.weightedClosePrecision < floor);
}

/**
* The per-(project, ruleCode) counterpart to {@link rulesBelowClosePrecisionFloor}: same floor/minDecided
* check, over an already-fetched {@link RuleGateEvalReport}'s rows instead of the blended ones (no I/O). This
* is the "which rule is broken on which repo" view (#8906) — a rule that looks healthy once pooled across
* every project can still be systematically wrong on ONE of them, diluted out of the blended figure by every
* other project's correct usage of the same rule.
*/
export function projectRulesBelowClosePrecisionFloor(
rows: readonly RuleGateEvalRow[],
floor: number = AUTOTUNE_CLOSE_PRECISION_FLOOR,
minDecided: number = AUTOTUNE_MIN_DECIDED,
): RuleGateEvalRow[] {
return rows.filter((r) => r.wouldClose >= minDecided && r.weightedClosePrecision != null && r.weightedClosePrecision < floor);
}
20 changes: 19 additions & 1 deletion src/services/operator-dashboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import { computeFleetAnalytics, getFleetHealthSummary, type FleetAnalytics, type
import { computeAgentHealth, computeCalibration, type AgentHealth, type Calibration } from "../review/ops";
import { computeGateEval, type GateEvalReport } from "../review/parity";
import { computeContributorGateEval, contributorFairnessFlags, computeBlendedContributorGateEval, contributorGlobalFairnessFlags } from "../review/contributor-gate-eval";
import { computeBlendedRuleGateEval, rulesBelowClosePrecisionFloor } from "../review/rule-gate-eval";
import { computeRuleGateEval, computeBlendedRuleGateEval, rulesBelowClosePrecisionFloor, projectRulesBelowClosePrecisionFloor } from "../review/rule-gate-eval";
import { computeCycleTimeAggregate, computeFindingAcceptance, type CycleTimeAggregate } from "../review/stats";
import { loadUpstreamStatus, type UpstreamStatus } from "../upstream/ruleset";
import { nowIso } from "../utils/json";
Expand Down Expand Up @@ -140,6 +140,7 @@ export async function buildOperatorDashboardPayload(
gateEval,
contributorGateEval,
blendedContributorGateEval,
ruleGateEval,
blendedRuleGateEval,
cycleTime,
calibration,
Expand Down Expand Up @@ -172,6 +173,9 @@ export async function buildOperatorDashboardPayload(
computeContributorGateEval(env, { days: GATE_ANALYTICS_WINDOW_DAYS, nowMs: Date.now() }),
// #global-contributor-trust: the SAME data pooled cross-repo into one blended figure per login, same window.
computeBlendedContributorGateEval(env, { days: GATE_ANALYTICS_WINDOW_DAYS, nowMs: Date.now() }),
// #8906: per-(project, ruleCode) gate accuracy -- the "which rule is broken on which repo" view that
// computeBlendedRuleGateEval's cross-repo pooling can't provide. Fails safe to an empty report.
computeRuleGateEval(env, { days: GATE_ANALYTICS_WINDOW_DAYS, nowMs: Date.now() }),
// #7984: the SAME review_audit data re-aggregated by RULE CODE (pooled cross-repo) instead of by project —
// isolates a single systematically-wrong deterministic rule even while its host project's own aggregate
// still looks healthy. Fails safe to an empty report on any read error.
Expand Down Expand Up @@ -214,6 +218,9 @@ export async function buildOperatorDashboardPayload(
// #7984: rules whose sample has cleared enough volume to trust but whose weighted close precision sits
// below the SAME floor auto-tune.ts's project-level close-precision breaker uses -- pure fold, no extra I/O.
const rulesBelowFloor = rulesBelowClosePrecisionFloor(blendedRuleGateEval.rows);
// #8906: the SAME floor check, per (project, ruleCode) instead of pooled -- surfaces a rule that's broken on
// one specific repo even while the blended figure above still looks healthy.
const projectRulesBelowFloor = projectRulesBelowClosePrecisionFloor(ruleGateEval.rows);
const installedRepos = repositories.filter((repo: RepositoryRecord) => repo.isInstalled).length;
const registeredRepos = repositories.filter((repo: RepositoryRecord) => repo.isRegistered).length;
// #1967: map FindingAcceptanceAggregate's field names onto the AcceptanceRateCard's expected shape.
Expand Down Expand Up @@ -304,6 +311,17 @@ export async function buildOperatorDashboardPayload(
value: String(rulesBelowFloor.length),
delta: rulesBelowFloor.length > 0 ? rulesBelowFloor.map((r) => r.ruleCode).join(", ") : "no rule below floor",
},
{
// #8906: the per-(project, ruleCode) counterpart to the blended tile above -- names the specific repo
// a rule is broken on, same "name the specific thing" posture (rule codes and repo names are not
// personally identifying, unlike the contributor tiles' logins).
label: "Rules below close-precision floor (per-project)",
value: String(projectRulesBelowFloor.length),
delta:
projectRulesBelowFloor.length > 0
? projectRulesBelowFloor.map((r) => `${r.project}:${r.ruleCode}`).join(", ")
: "no rule below floor",
},
],
noiseReduction: [
{
Expand Down
32 changes: 32 additions & 0 deletions test/unit/operator-dashboard.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,38 @@ describe("operator dashboard payload", () => {
expect(JSON.stringify(payload)).not.toContain("missing_linked_issue"); // the healthy rule never appears
});

it("surfaces the per-project rule breakdown (#8906), catching a rule broken on ONE repo even while a second repo's healthy usage pools it back above the blended floor", async () => {
const env = createTestEnv();
const seedClose = async (id: string, project: string, ruleCode: string, truth: "closed" | "merged"): Promise<void> => {
await env.DB.prepare(
`INSERT INTO review_audit (id, project, target_id, event_type, decision, summary, source, created_at) VALUES (?, ?, ?, 'gate_decision', 'close', ?, 'gittensory-native', ?)`,
)
.bind(`gd-${id}`, project, `${project}#${id}`, ruleCode, new Date().toISOString())
.run();
await env.DB.prepare(
`INSERT INTO review_audit (id, project, target_id, event_type, decision, source, created_at) VALUES (?, ?, ?, 'pr_outcome', ?, 'github', ?)`,
)
.bind(`po-${id}`, project, `${project}#${id}`, truth, new Date().toISOString())
.run();
};
// Broken on ONE repo: 12 closes (clears AUTOTUNE_MIN_DECIDED), every single one later merged -- 0% precision.
for (let i = 1; i <= 12; i++) await seedClose(`bad-${i}`, "metagraphed/broken-repo", "shared_rule", "merged");
// The SAME rule, healthy on a SECOND repo: 90 closes, all correctly closed -- pools the blended precision to
// 90/102 ≈ 0.882, above AUTOTUNE_CLOSE_PRECISION_FLOOR (0.85), so the blended tile does NOT flag this rule
// even though it is genuinely broken on metagraphed/broken-repo -- exactly the gap #8906 exists to close.
for (let i = 1; i <= 90; i++) await seedClose(`good-${i}`, "metagraphed/healthy-repo", "shared_rule", "closed");

const payload = await buildOperatorDashboardPayload(env);
expect(payload.metrics).toEqual(
expect.arrayContaining([expect.objectContaining({ label: "Rules below close-precision floor", value: "0", delta: "no rule below floor" })]),
);
expect(payload.metrics).toEqual(
expect.arrayContaining([
expect.objectContaining({ label: "Rules below close-precision floor (per-project)", value: "1", delta: "metagraphed/broken-repo:shared_rule" }),
]),
);
});

it("wires computeFindingAcceptance into the dashboard's acceptance card shape (#1967/#5213)", async () => {
const env = createTestEnv();
await env.DB.prepare(
Expand Down
66 changes: 66 additions & 0 deletions test/unit/rule-gate-eval.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
computeRuleGateEval,
computeBlendedRuleGateEval,
rulesBelowClosePrecisionFloor,
projectRulesBelowClosePrecisionFloor,
type RuleGateEvalRow,
type BlendedRuleGateEvalRow,
} from "../../src/review/rule-gate-eval";
Expand Down Expand Up @@ -383,3 +384,68 @@ describe("rulesBelowClosePrecisionFloor (#7984, #7986's own read)", () => {
expect(rulesBelowClosePrecisionFloor(justBelow)).toHaveLength(1);
});
});

function projectRow(overrides: Partial<RuleGateEvalRow> = {}): RuleGateEvalRow {
return {
project: "acme/widgets",
ruleCode: "rule_a",
wouldMerge: 0,
mergeConfirmed: 0,
mergeFalse: 0,
wouldClose: 0,
closeConfirmed: 0,
closeFalse: 0,
decided: 0,
mergePrecision: null,
closePrecision: null,
weightedMergeConfirmed: 0,
weightedCloseConfirmed: 0,
weightedMergePrecision: null,
weightedClosePrecision: null,
...overrides,
};
}

describe("projectRulesBelowClosePrecisionFloor — per-(project, ruleCode) counterpart (#8906)", () => {
it("flags a rule once its sample clears AUTOTUNE_MIN_DECIDED with a below-floor weighted precision, on the SPECIFIC project it fired on", () => {
const rows = [
projectRow({ project: "acme/widgets", ruleCode: "surface_lane_reject", wouldClose: 12, closeConfirmed: 0, weightedClosePrecision: 0 }),
];
const flagged = projectRulesBelowClosePrecisionFloor(rows, AUTOTUNE_CLOSE_PRECISION_FLOOR, AUTOTUNE_MIN_DECIDED);
expect(flagged).toHaveLength(1);
expect(flagged[0]?.project).toBe("acme/widgets");
expect(flagged[0]?.ruleCode).toBe("surface_lane_reject");
});

it("does NOT flag a rule with insufficient sample even at 0% precision, mirroring the blended fold's same floor", () => {
const rows = [projectRow({ ruleCode: "rare_rule", wouldClose: 3, closeConfirmed: 0, weightedClosePrecision: 0 })];
expect(projectRulesBelowClosePrecisionFloor(rows, AUTOTUNE_CLOSE_PRECISION_FLOOR, AUTOTUNE_MIN_DECIDED)).toEqual([]);
});

it("does NOT flag a healthy rule with plenty of sample and good precision", () => {
const rows = [projectRow({ ruleCode: "healthy_rule", wouldClose: 40, closeConfirmed: 39, weightedClosePrecision: 39 / 40 })];
expect(projectRulesBelowClosePrecisionFloor(rows, AUTOTUNE_CLOSE_PRECISION_FLOOR, AUTOTUNE_MIN_DECIDED)).toEqual([]);
});

it("does NOT flag a rule whose weightedClosePrecision is null (no decided close verdict at all)", () => {
const rows = [projectRow({ ruleCode: "merge_only_rule", wouldClose: 0, weightedClosePrecision: null })];
expect(projectRulesBelowClosePrecisionFloor(rows, AUTOTUNE_CLOSE_PRECISION_FLOOR, AUTOTUNE_MIN_DECIDED)).toEqual([]);
});

it("defaults floor and minDecided to the SAME constants the blended fold uses", () => {
const atFloor = [projectRow({ ruleCode: "rule_a", wouldClose: 20, weightedClosePrecision: AUTOTUNE_CLOSE_PRECISION_FLOOR })];
expect(projectRulesBelowClosePrecisionFloor(atFloor)).toEqual([]);
const justBelow = [projectRow({ ruleCode: "rule_a", wouldClose: 20, weightedClosePrecision: AUTOTUNE_CLOSE_PRECISION_FLOOR - 0.01 })];
expect(projectRulesBelowClosePrecisionFloor(justBelow)).toHaveLength(1);
});

it("isolates a rule broken on ONE project even when a second project's healthy rows would pool it away in the blended view", () => {
const rows = [
projectRow({ project: "acme/broken-repo", ruleCode: "shared_rule", wouldClose: 15, closeConfirmed: 0, weightedClosePrecision: 0 }),
projectRow({ project: "acme/healthy-repo", ruleCode: "shared_rule", wouldClose: 30, closeConfirmed: 30, weightedClosePrecision: 1 }),
];
const flagged = projectRulesBelowClosePrecisionFloor(rows, AUTOTUNE_CLOSE_PRECISION_FLOOR, AUTOTUNE_MIN_DECIDED);
expect(flagged).toHaveLength(1);
expect(flagged[0]?.project).toBe("acme/broken-repo");
});
});