From 7bcb2314fc1f83b48d98825c25f769ecd2ba487a Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 24 Jun 2026 23:18:02 -0700 Subject: [PATCH 1/2] fix(agent): an AI-judgment gate failure must not close a green-CI PR (#ai-ci-refutation) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live dry-run audit of 44 metagraphed PRs found the AI reviewer false-closing ~60% of clean, green, mergeable contributor PRs: the dual free Workers-AI models share the same hallucination (e.g. inferring a JSON field is "required" from sibling data, or claiming "schema validation fails" / "the test count is wrong" on a PR whose CI is green), and the consensus/split blocker auto-CLOSES the PR. Grounding already feeds the finished CI status to the reviewer, but the model can ignore it — nothing enforced the deterministic ground truth. This makes the disposition enforce it: when the gate FAILED solely because of an AI-judgment blocker (ai_consensus_defect / ai_review_split) and the real CI is GREEN, the AI claim is refuted by the validator → the effective verdict is success → a clean + green PR MERGES. It is NOT routed to manual review (only the guardrail path holds), and it NEVER overrides a deterministic blocker (every blocker must be an AI-judgment code) or a red/unverified CI. The AI concern still surfaces in the review comment as an advisory; it simply no longer auto-closes a green PR. - agent-actions.ts: AI_JUDGMENT_BLOCKER_CODES + an effective `conclusion` that downgrades an AI-only failure to success when ciState==passed; threads through gatePassing, willClose, and the label-reason display (now reports verdict=success; CI green). - processors.ts: thread the gate blocker CODES into the planner, gated under the same grounding + convergence condition as the converged AI review that produces the defects (flag-off / non-convergence repo ⇒ codes omitted ⇒ byte-identical verdict). --- src/queue/processors.ts | 7 ++++ src/settings/agent-actions.ts | 41 +++++++++++++++++++--- test/unit/agent-actions.test.ts | 61 +++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 5 deletions(-) diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 78474e7107..50441144f1 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -788,6 +788,13 @@ async function maybeRunAgentMaintenance( const planned = planAgentMaintenanceActions({ conclusion: gate.conclusion, blockerTitles: gate.blockers.map((blocker) => blocker.title), + // CI-refutation (#ai-ci-refutation): thread the blocker CODES so the planner can suppress an AI-judgment-only + // failure (ai_consensus_defect / ai_review_split) on a green-CI PR — the deterministic validator overrules the + // model hallucination, so a clean+green PR MERGES instead of being false-closed. Gated under the SAME condition + // as the converged AI review that produces these defects (grounding feeds the CI truth to the reviewer; this + // ENFORCES it). Flag-OFF / non-convergence repo ⇒ codes omitted ⇒ the planner's refutation is a no-op + // (byte-identical verdict). The codes are public-safe finding identifiers (no rubric/scoring/reward terms). + ...(isGroundingEnabled(env) && isConvergenceRepoAllowed(env, repoFullName) ? { gateBlockerCodes: gate.blockers.map((blocker) => blocker.code) } : {}), autonomy: settings.autonomy, autoMaintain: settings.autoMaintain, slopGateMinScore: settings.slopGateMinScore, diff --git a/src/settings/agent-actions.ts b/src/settings/agent-actions.ts index e3a04f336b..bb30b4fe63 100644 --- a/src/settings/agent-actions.ts +++ b/src/settings/agent-actions.ts @@ -13,6 +13,19 @@ const DEFAULT_SLOP_GATE_MIN_SCORE = 60; // every action is independently gated by its own autonomy class, and the irreversible ones (merge / close) // demand strong positive signals. +// AI-JUDGMENT blocker codes (#ai-ci-refutation): a gate failure driven by these is the dual-model AI reviewer's +// OPINION, not a deterministic fact. The free Workers-AI pair hallucinates the same plausible-but-wrong defect +// (e.g. inferring a JSON field is "required" from sibling data, or claiming "schema validation fails" / "the test +// count is wrong" on a PR whose CI is GREEN). Grounding already feeds the finished CI status to the reviewer, but +// the model can ignore it — so when the gate FAILED *solely* because of these codes AND the deterministic CI is +// GREEN (the real validator passed), the AI claim is refuted by reality and must NOT auto-close the PR. The AI +// concern stays visible in the review comment as an advisory; the disposition follows the deterministic signals +// (a clean + green PR MERGES). This NEVER routes to manual review (only the guardrail path does), and it NEVER +// overrides a deterministic blocker (secret_leak / duplicate / missing-issue / slop / quality / manifest): it +// applies ONLY when EVERY blocker is one of these AND CI passed. `ai_review_inconclusive` is deliberately EXCLUDED +// — that is a "could not review" HOLD, not a false defect. +const AI_JUDGMENT_BLOCKER_CODES = new Set(["ai_consensus_defect", "ai_review_split"]); + // The bucket labels the layer applies to reflect the gate verdict. Namespaced so a maintainer can filter on // them and they never collide with project labels. export const AGENT_LABEL_READY = "gittensory:ready-to-merge"; @@ -62,6 +75,11 @@ export type PlannedAgentAction = { export type AgentActionPlanInput = { conclusion: GateCheckConclusion; blockerTitles: string[]; + // The gate's blocking finding CODES (parallel to blockerTitles). Used by the CI-refutation rule + // (#ai-ci-refutation): when the gate FAILED solely because of AI-judgment blockers and CI is green, the + // AI claim is refuted by the deterministic validator. Optional/absent (e.g. callers that don't thread it, + // or the refutation gated OFF at the boundary) ⇒ the refutation is a no-op and the verdict is unchanged. + gateBlockerCodes?: string[] | undefined; autonomy: AutonomyPolicy | null | undefined; // Optional so the trigger can pass raw repo settings; both fall back to conservative defaults here. autoMaintain?: AutoMaintainPolicy | undefined; @@ -236,10 +254,23 @@ export function planAgentMaintenanceActions(input: AgentActionPlanInput): Planne // Settle-before-decide: never approve / merge / close on a half-finished CI run. if (input.ciState === "pending") return actions; + // CI-refutation of an AI-judgment-only failure (#ai-ci-refutation). When the gate FAILED *solely* because the + // dual-model AI reviewer flagged a defect (ai_consensus_defect / ai_review_split) but the deterministic CI is + // GREEN, the AI claim is refuted by the real validator → downgrade the verdict to SUCCESS for the disposition so + // a clean+green PR MERGES instead of being false-closed on a model hallucination. Requires EVERY blocker to be an + // AI-judgment code (a mixed failure with any deterministic blocker — duplicate / secret / slop / missing-issue / + // manifest — keeps `failure` and still closes) AND CI === passed (a red/unverified CI is the real signal and is + // never overridden). Empty/absent codes (the rule gated OFF at the boundary, or a non-AI failure) ⇒ no-op, so the + // verdict is byte-identical. The effective `conclusion` drives every gate-verdict decision below; the AI concern + // still surfaces in the review comment (an advisory finding), it just no longer auto-closes a green PR. + const aiJudgmentOnlyFailure = + input.conclusion === "failure" && (input.gateBlockerCodes?.length ?? 0) > 0 && (input.gateBlockerCodes ?? []).every((code) => AI_JUDGMENT_BLOCKER_CODES.has(code)); + const conclusion: GateCheckConclusion = aiJudgmentOnlyFailure && ciPassed ? "success" : input.conclusion; + // Only SUCCESS earns the review-good auto-merge. A NEUTRAL gate flows (no longer silently returns []) but is // NOT auto-merged — it falls through to a HELD + labeled state for review. (Auto-merging a neutral / grace // PR is a separate trust/policy decision, deliberately NOT bundled into the harm-stop.) (#harm-stop) - const gatePassing = input.conclusion === "success"; + const gatePassing = conclusion === "success"; // A changed path matching a hard guardrail forces manual review (suppresses auto-MERGE / auto-approve / auto-close). // Fail SAFE on UNKNOWN paths (#1062): when guardrails are configured but the changed-file set is empty (cache // not yet / no longer populated), we cannot prove the PR doesn't touch a guarded path, so treat it as a hit — @@ -287,7 +318,7 @@ export function planAgentMaintenanceActions(input: AgentActionPlanInput): Planne // have folded in optional / third-party checks and must keep the hard-guardrail manual hold. // (Rebase-if-behind already ran above, so a red CI here is on the latest base — not a stale-base artifact.) (#ci-fail-closes-guarded) const redVerifiedRequiredCi = ciFailed && input.ciRequiredContextsVerified === true; - const willClose = isContributor && acting("close") && (redVerifiedRequiredCi || (!guardrailHit && (ciFailed || input.conclusion === "failure" || isConflict))); + const willClose = isContributor && acting("close") && (redVerifiedRequiredCi || (!guardrailHit && (ciFailed || conclusion === "failure" || isConflict))); // Linked-issue HARD-RULE close (#linked-issue-hard-rules). A DETERMINISTIC verdict about the LINKED ISSUE // (owner-assigned / missing point-label / maintainer-only) — NOT an AI verdict, so there is no hallucination // to guard against: this close fires REGARDLESS of `guardrailHit`. It still only ever closes a CONTRIBUTOR @@ -336,10 +367,10 @@ export function planAgentMaintenanceActions(input: AgentActionPlanInput): Planne const reason = linkedIssueCloseInFlight ? `linked-issue hard rule: ${linkedIssueHardRule?.reason ?? "ineligible linked issue"}` : !reviewGood - ? `verdict=${input.conclusion}${ciReason ? `; ${ciReason}` : ""}` + ? `verdict=${conclusion}${ciReason ? `; ${ciReason}` : ""}` : heldForManualReview - ? `verdict=${input.conclusion}; guarded path → manual review` - : `verdict=${input.conclusion}; CI green`; + ? `verdict=${conclusion}; guarded path → manual review` + : `verdict=${conclusion}; CI green`; if (!hasLabel(input.pr.labels, label)) { actions.push({ actionClass: "label", requiresApproval: approval("label"), reason, label }); } diff --git a/test/unit/agent-actions.test.ts b/test/unit/agent-actions.test.ts index ce4b5f8192..21d7a277f9 100644 --- a/test/unit/agent-actions.test.ts +++ b/test/unit/agent-actions.test.ts @@ -278,6 +278,67 @@ describe("planAgentMaintenanceActions (#778)", () => { }); }); + describe("CI-refutation: an AI-judgment-only gate failure does NOT close a green-CI PR (#ai-ci-refutation)", () => { + const merging = { autonomy: { merge: "auto" as const, approve: "auto" as const, close: "auto" as const, label: "auto" as const }, ciState: "passed" as const, pr: { labels: [], mergeableState: "clean" as const, reviewDecision: "APPROVED" as const } }; + + it("a consensus-defect failure on a green, clean PR MERGES instead of closing (the validator overrules the model)", () => { + const plan = planAgentMaintenanceActions(input({ conclusion: "failure", blockerTitles: ["AI reviewers agree on a likely critical defect"], gateBlockerCodes: ["ai_consensus_defect"], ...merging })); + const cls = classes(plan); + expect(cls).toContain("merge"); + expect(cls).not.toContain("close"); + // Never routed to manual review — the guardrail path is the ONLY manual hold. + expect(plan.find((a) => a.actionClass === "label")?.label).not.toBe(AGENT_LABEL_NEEDS_REVIEW); + }); + + it("a review-split failure on a green PR is refuted too (a single minority model opinion never closes a green PR)", () => { + const cls = classes(planAgentMaintenanceActions(input({ conclusion: "failure", blockerTitles: ["An AI reviewer flagged a likely blocking defect"], gateBlockerCodes: ["ai_review_split"], ...merging }))); + expect(cls).toContain("merge"); + expect(cls).not.toContain("close"); + }); + + it("the refutation reports the EFFECTIVE verdict (verdict=success; CI green), not the contradictory raw failure", () => { + const plan = planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["ai_consensus_defect"], autonomy: { label: "auto" }, ciState: "passed", pr: { labels: [], mergeableState: "clean", reviewDecision: "APPROVED" } })); + const label = plan.find((a) => a.actionClass === "label"); + expect(label?.label).toBe(AGENT_LABEL_READY); + expect(label?.reason).toBe("verdict=success; CI green"); + }); + + it("does NOT refute when CI is RED — a real failing check is the ground truth and still CLOSES", () => { + const cls = classes(planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["ai_consensus_defect"], autonomy: { close: "auto" }, ciState: "failed", pr: { labels: [] } }))); + expect(cls).toContain("close"); + }); + + it("does NOT refute a MIXED failure — any deterministic blocker alongside the AI one still CLOSES", () => { + const cls = classes(planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["ai_consensus_defect", "duplicate_open_pr"], autonomy: { close: "auto" }, ciState: "passed", pr: { labels: [] } }))); + expect(cls).toContain("close"); + }); + + it("does NOT refute a deterministic-only failure (e.g. slop/duplicate) — those close normally on green CI", () => { + const cls = classes(planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["slop_high"], autonomy: { close: "auto" }, ciState: "passed", pr: { labels: [] } }))); + expect(cls).toContain("close"); + }); + + it("does NOT refute ai_review_inconclusive — a 'could not review' hold is not a false defect", () => { + // inconclusive isn't a `failure` conclusion in practice (it HOLDS neutral), but even if a failure carried + // the code, it must NOT be treated as a refutable false positive. + const cls = classes(planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["ai_review_inconclusive"], autonomy: { close: "auto" }, ciState: "passed", pr: { labels: [] } }))); + expect(cls).toContain("close"); + }); + + it("is a no-op when codes are omitted (refutation gated OFF at the boundary) — byte-identical close", () => { + const cls = classes(planAgentMaintenanceActions(input({ conclusion: "failure", blockerTitles: ["AI reviewers agree on a likely critical defect"], autonomy: { close: "auto" }, ciState: "passed", pr: { labels: [] } }))); + expect(cls).toContain("close"); + }); + + it("a guardrail-touching AI-refuted PR is still HELD for the owner (refutation never overrides the guardrail hold)", () => { + const plan = planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["ai_consensus_defect"], autonomy: { merge: "auto", approve: "auto", close: "auto", label: "auto" }, hardGuardrailGlobs: ["src/**"], changedPaths: ["src/index.ts"], ciState: "passed", pr: { labels: [], mergeableState: "clean", reviewDecision: "APPROVED" } })); + const cls = classes(plan); + expect(cls).not.toContain("merge"); + expect(cls).not.toContain("close"); + expect(plan.find((a) => a.actionClass === "label")?.label).toBe(AGENT_LABEL_NEEDS_REVIEW); + }); + }); + describe("owner-PR guard: never auto-close the repo owner's own PRs", () => { it("does NOT auto-close a noisy failing PR authored by the repo owner", () => { const plan = classes(planAgentMaintenanceActions(input({ conclusion: "failure", autonomy: { close: "auto" }, blockerTitles: ["x"], authorIsOwner: true, pr: { labels: [], slopRisk: 95 } }))); From 1af8c0f965cee9c3381ffc3385bf4edea4167f49 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 24 Jun 2026 23:56:33 -0700 Subject: [PATCH 2/2] fix(review): reconcile the public comment with the CI-refutation + share the gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the AI CI-refutation: the disposition (planAgentMaintenanceActions) merges a green-CI PR whose only gate failure was an AI-judgment blocker, but the review COMMENT was still rendered from the raw gate.conclusion=failure — so a merged PR would show a red "blocked/closed" headline + Gate panel row. reconcileGateEvaluationForGreenCi makes the comment render SUCCESS/advisory in that case, matching the action; the specific AI concern still surfaces from the advisory findings as a raised concern under the green verdict. To keep the two surfaces in lock-step and fully unit-testable (no uncovered branch in the processor): - advisory.ts: canonical AI_JUDGMENT_BLOCKER_CODES + isAiJudgmentOnlyFailure + reconcileGateEvaluationForGreenCi(gate, ciState, enabled). agent-actions now imports the shared code set instead of a local copy. - grounding-wire.ts: aiCiRefutationActive(env, repo) — the single grounding+convergence gate both the disposition and the comment use, so they can never disagree. Called as a plain function at each processor site (no inline && / ternary → no integration-only branch). - agent-actions.ts: the refutation is now gated by an explicit aiCiRefutationEnabled flag (unit-tested), and the blocker codes are resolved once so the nullish fallback is covered. - processors.ts: the comment block resolves live CI first, then reconciles the gate ONCE and uses it for both the panel rows and the comment body. --- src/queue/processors.ts | 31 ++++++++----- src/review/grounding-wire.ts | 11 +++++ src/rules/advisory.ts | 34 +++++++++++++++ src/settings/agent-actions.ts | 31 ++++++------- test/unit/agent-actions.test.ts | 23 ++++++---- test/unit/grounding-wiring.test.ts | 19 ++++++++ test/unit/rules.test.ts | 70 ++++++++++++++++++++++++++++++ 7 files changed, 182 insertions(+), 37 deletions(-) diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 50441144f1..f85103c972 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -100,7 +100,7 @@ import { resolveRepoActionMode } from "../github/client"; import { ALL_TYPE_LABELS, resolvePrTypeLabel } from "../settings/pr-type-label"; import { fetchPublicContributorProfile } from "../github/public"; import { refreshRegistry } from "../registry/sync"; -import { buildIssueAdvisory, buildPullRequestAdvisory, evaluateGateCheck, isTestPath } from "../rules/advisory"; +import { buildIssueAdvisory, buildPullRequestAdvisory, evaluateGateCheck, isTestPath, reconcileGateEvaluationForGreenCi } from "../rules/advisory"; import { detectNotificationEvents } from "../notifications/events"; import { deliverNotification, detectIssueWatchEvents, evaluateNotificationEvent } from "../notifications/service"; import { getOrCreateScoringModelSnapshot, refreshScoringModelSnapshot } from "../scoring/model"; @@ -179,7 +179,7 @@ import { resolveRepositorySettings } from "../settings/repository-settings"; import type { LocalBranchAnalysisInput } from "../signals/local-branch"; import { runGittensoryAiReview } from "../services/ai-review"; import { secretLeakFinding } from "../review/safety"; -import { buildReviewGroundingText, checkSummaryText as checkFailureSummaryText, isGroundingEnabled } from "../review/grounding-wire"; +import { aiCiRefutationActive, buildReviewGroundingText, checkSummaryText as checkFailureSummaryText, isGroundingEnabled } from "../review/grounding-wire"; import { buildReviewRagContext, isRagEnabled } from "../review/rag-wire"; import { indexRepo, reindexChangedPaths } from "../review/rag-index"; import { isReputationEnabled, recordReputationOutcome, shouldSkipAiForReputation } from "../review/reputation-wire"; @@ -788,13 +788,15 @@ async function maybeRunAgentMaintenance( const planned = planAgentMaintenanceActions({ conclusion: gate.conclusion, blockerTitles: gate.blockers.map((blocker) => blocker.title), - // CI-refutation (#ai-ci-refutation): thread the blocker CODES so the planner can suppress an AI-judgment-only - // failure (ai_consensus_defect / ai_review_split) on a green-CI PR — the deterministic validator overrules the - // model hallucination, so a clean+green PR MERGES instead of being false-closed. Gated under the SAME condition - // as the converged AI review that produces these defects (grounding feeds the CI truth to the reviewer; this - // ENFORCES it). Flag-OFF / non-convergence repo ⇒ codes omitted ⇒ the planner's refutation is a no-op - // (byte-identical verdict). The codes are public-safe finding identifiers (no rubric/scoring/reward terms). - ...(isGroundingEnabled(env) && isConvergenceRepoAllowed(env, repoFullName) ? { gateBlockerCodes: gate.blockers.map((blocker) => blocker.code) } : {}), + // CI-refutation (#ai-ci-refutation): thread the blocker CODES + the active gate so the planner suppresses an + // AI-judgment-only failure (ai_consensus_defect / ai_review_split) on a green-CI PR — the deterministic + // validator overrules the model hallucination, so a clean+green PR MERGES instead of being false-closed. + // `aiCiRefutationEnabled` is the SAME grounding+convergence gate the public-comment reconciliation uses, passed + // as a single boolean so the refutation condition is unit-tested in the planner and this site carries no branch. + // Enabled=false (non-convergence / grounding-off) ⇒ the refutation is a no-op ⇒ byte-identical verdict. The + // codes are public-safe finding identifiers (no rubric/scoring/reward terms). + gateBlockerCodes: gate.blockers.map((blocker) => blocker.code), + aiCiRefutationEnabled: aiCiRefutationActive(env, repoFullName), autonomy: settings.autonomy, autoMaintain: settings.autoMaintain, slopGateMinScore: settings.slopGateMinScore, @@ -2711,7 +2713,6 @@ async function maybePublishPrPublicSurface( // 3. The `ai_consensus_defect` surfaces exactly ONCE — as the Code-review blocker — never also in the // gate signal row (which renders only the conclusion-derived status text, not the defect string). if (unifiedCommentAllowed && gateEvaluation) { - const { rows, readinessTotal } = buildPublicPrPanelSignalRows({ repo, pr, profile, detection, queueHealth, collisions, preflight, settings, gate: gateEvaluation, duplicateWinnerEnabled }); // FIX B: the unified comment's file count + visual-capture path filter need the real diff — reuse the // shared resolver (one resolve per review; inline-fetches when stored is still empty pre-detail-sync). const unifiedFiles = await getReviewFiles(); @@ -2742,6 +2743,14 @@ async function maybePublishPrPublicSurface( ...(failingDetails.length > 0 ? { failingChecks: failingDetails.map((detail) => detail.name) } : {}), ...(failingDetails.length > 0 ? { failingDetails } : {}), }; + // CI-refutation for the PUBLIC comment (#ai-ci-refutation): when the gate FAILED solely on an AI-judgment + // blocker but the LIVE CI is GREEN, render the comment (headline + Gate panel row) as SUCCESS/advisory so it + // MATCHES the disposition (which merges such a PR) instead of a contradictory red "blocked/closed". Uses the + // SAME grounding+convergence gate as the disposition refutation (a single `aiCiRefutationActive` call so this + // site carries no branch), built AFTER the live CI is resolved and used for BOTH the panel rows and the + // comment body so the two never disagree. Gate OFF ⇒ commentGate === gateEvaluation (byte-identical comment). + const commentGate = reconcileGateEvaluationForGreenCi(gateEvaluation, ciState, aiCiRefutationActive(env, repoFullName)); + const { rows, readinessTotal } = buildPublicPrPanelSignalRows({ repo, pr, profile, detection, queueHealth, collisions, preflight, settings, gate: commentGate, duplicateWinnerEnabled }); // Visual before/after capture (visual-capture port). Fires ONLY when (1) the global flag + per-repo // cutover gate both allow it (screenshotsAllowed) AND (2) the PR touches WEB-VISIBLE files (isVisualPath // — frontend pages / public OG images; backend .ts/.md/.json PRs never qualify). Fully wrapped in @@ -2777,7 +2786,7 @@ async function maybePublishPrPublicSurface( } } deterministicBody = buildUnifiedCommentBody({ - gate: gateEvaluation, + gate: commentGate, ...(aiReview !== undefined ? { aiReview } : {}), advisoryFindings: advisory.findings, panelRows: rows, diff --git a/src/review/grounding-wire.ts b/src/review/grounding-wire.ts index 863e960eb4..94b8c9950c 100644 --- a/src/review/grounding-wire.ts +++ b/src/review/grounding-wire.ts @@ -14,6 +14,7 @@ import { createInstallationToken } from "../github/app"; import type { CheckSummaryRecord, PullRequestFileRecord } from "../types"; import { repoParts } from "../utils/json"; +import { isConvergenceRepoAllowed } from "./cutover-gate"; import { buildGrounding, type FileFetcher, @@ -29,6 +30,16 @@ export function isGroundingEnabled(env: { GITTENSORY_REVIEW_GROUNDING?: string | return /^(1|true|yes|on)$/i.test(env.GITTENSORY_REVIEW_GROUNDING ?? ""); } +/** True when the AI CI-refutation (#ai-ci-refutation) is ACTIVE for this repo: grounding is ON (the converged AI + * review feeds the finished CI status to the reviewer, so enforcing that ground truth is coherent) AND the repo + * is convergence-allowlisted. Centralized so the disposition refutation (agent-actions) and the public-comment + * reconciliation gate on the SAME condition — the merge/close action and the rendered comment can never disagree. + * A single call (not an inline `&&` at the call sites) so the processor carries no branch and this is the one + * place the condition is unit-tested. */ +export function aiCiRefutationActive(env: Env, repoFullName: string): boolean { + return isGroundingEnabled(env) && isConvergenceRepoAllowed(env, repoFullName); +} + /** When ON, both grounding inputs (CI + full files) are gathered; OFF gathers neither. One switch keeps the * flag-OFF path provably byte-identical (no partial grounding). */ function groundingFlags(env: { GITTENSORY_REVIEW_GROUNDING?: string | undefined }): GroundingFlags { diff --git a/src/rules/advisory.ts b/src/rules/advisory.ts index ae4d7d9a07..5524e1d235 100644 --- a/src/rules/advisory.ts +++ b/src/rules/advisory.ts @@ -66,6 +66,40 @@ export type GateCheckEvaluation = { warnings: AdvisoryFinding[]; }; +// AI-JUDGMENT blocker codes (#ai-ci-refutation): a gate failure driven SOLELY by these is the dual-model AI +// reviewer's OPINION, not a deterministic fact. Shared by the disposition refutation (agent-actions) and the +// public-comment reconciliation so the merge/close ACTION and the rendered comment always agree. +// `ai_review_inconclusive` is deliberately EXCLUDED — that is a "could not review" HOLD, not a false defect. +export const AI_JUDGMENT_BLOCKER_CODES = new Set(["ai_consensus_defect", "ai_review_split"]); + +/** True when the gate FAILED *solely* because of AI-judgment blockers (every blocker is an AI-judgment code). + * An empty blocker list is NOT an AI-judgment-only failure (there is nothing to refute). PURE. */ +export function isAiJudgmentOnlyFailure(evaluation: GateCheckEvaluation): boolean { + return evaluation.conclusion === "failure" && evaluation.blockers.length > 0 && evaluation.blockers.every((blocker) => AI_JUDGMENT_BLOCKER_CODES.has(blocker.code)); +} + +/** + * Reconcile a gate evaluation with the deterministic CI for the PUBLIC review comment (#ai-ci-refutation). + * When the gate FAILED solely on an AI-judgment blocker (ai_consensus_defect / ai_review_split) but the real CI + * is GREEN, the AI claim is refuted by the validator — so the comment must render SUCCESS (advisory), matching + * the disposition (planAgentMaintenanceActions merges such a PR) instead of a contradictory red "blocked/closed" + * headline + Gate row. The AI concern stays VISIBLE without double-listing: the specific consensus defect still + * surfaces from the advisory findings as a raised concern under the green verdict, so we only clear the gate's + * hard blockers here. `enabled` is the caller's grounding+convergence gate (passed in so this stays a PURE, + * unit-testable function and the processor carries no branch); `enabled` false, `ciState !== "passed"`, or a + * non-AI-only failure ⇒ the evaluation is returned UNCHANGED. + */ +export function reconcileGateEvaluationForGreenCi(evaluation: GateCheckEvaluation, ciState: "passed" | "failed" | "unverified", enabled: boolean): GateCheckEvaluation { + if (!enabled || ciState !== "passed" || !isAiJudgmentOnlyFailure(evaluation)) return evaluation; + return { + ...evaluation, + conclusion: "success", + title: "Gittensory Gate passed", + summary: "The AI review raised a concern, but the deterministic checks (CI) are green — the concern is advisory, not blocking.", + blockers: [], + }; +} + export function buildRepositoryAdvisory(repo: RepositoryRecord | null, fullName: string): Advisory { const findings: AdvisoryFinding[] = []; if (!repo) { diff --git a/src/settings/agent-actions.ts b/src/settings/agent-actions.ts index bb30b4fe63..7a3f2ee00a 100644 --- a/src/settings/agent-actions.ts +++ b/src/settings/agent-actions.ts @@ -1,5 +1,5 @@ import type { AgentActionClass, AutoMaintainPolicy, AutoMergeMethod, AutonomyPolicy } from "../types"; -import type { GateCheckConclusion } from "../rules/advisory"; +import { AI_JUDGMENT_BLOCKER_CODES, type GateCheckConclusion } from "../rules/advisory"; import { DEFAULT_AUTO_MAINTAIN_POLICY, autonomyRequiresApproval, isActingAutonomyLevel, resolveAutonomy } from "./autonomy"; import { changedPathsHittingGuardrail } from "../signals/change-guardrail"; import { AGENT_LABEL_PENDING_CLOSURE } from "../review/linked-issue-hard-rules"; @@ -13,18 +13,6 @@ const DEFAULT_SLOP_GATE_MIN_SCORE = 60; // every action is independently gated by its own autonomy class, and the irreversible ones (merge / close) // demand strong positive signals. -// AI-JUDGMENT blocker codes (#ai-ci-refutation): a gate failure driven by these is the dual-model AI reviewer's -// OPINION, not a deterministic fact. The free Workers-AI pair hallucinates the same plausible-but-wrong defect -// (e.g. inferring a JSON field is "required" from sibling data, or claiming "schema validation fails" / "the test -// count is wrong" on a PR whose CI is GREEN). Grounding already feeds the finished CI status to the reviewer, but -// the model can ignore it — so when the gate FAILED *solely* because of these codes AND the deterministic CI is -// GREEN (the real validator passed), the AI claim is refuted by reality and must NOT auto-close the PR. The AI -// concern stays visible in the review comment as an advisory; the disposition follows the deterministic signals -// (a clean + green PR MERGES). This NEVER routes to manual review (only the guardrail path does), and it NEVER -// overrides a deterministic blocker (secret_leak / duplicate / missing-issue / slop / quality / manifest): it -// applies ONLY when EVERY blocker is one of these AND CI passed. `ai_review_inconclusive` is deliberately EXCLUDED -// — that is a "could not review" HOLD, not a false defect. -const AI_JUDGMENT_BLOCKER_CODES = new Set(["ai_consensus_defect", "ai_review_split"]); // The bucket labels the layer applies to reflect the gate verdict. Namespaced so a maintainer can filter on // them and they never collide with project labels. @@ -77,9 +65,12 @@ export type AgentActionPlanInput = { blockerTitles: string[]; // The gate's blocking finding CODES (parallel to blockerTitles). Used by the CI-refutation rule // (#ai-ci-refutation): when the gate FAILED solely because of AI-judgment blockers and CI is green, the - // AI claim is refuted by the deterministic validator. Optional/absent (e.g. callers that don't thread it, - // or the refutation gated OFF at the boundary) ⇒ the refutation is a no-op and the verdict is unchanged. + // AI claim is refuted by the deterministic validator. Optional/absent ⇒ the refutation is a no-op. gateBlockerCodes?: string[] | undefined; + // Whether the AI CI-refutation is ACTIVE for this repo (the caller's grounding + convergence gate, passed as a + // single boolean so the refutation condition is fully unit-testable here and the processor carries no branch). + // Absent/false ⇒ the refutation never fires and the verdict is byte-identical to the raw gate. + aiCiRefutationEnabled?: boolean | undefined; autonomy: AutonomyPolicy | null | undefined; // Optional so the trigger can pass raw repo settings; both fall back to conservative defaults here. autoMaintain?: AutoMaintainPolicy | undefined; @@ -263,9 +254,15 @@ export function planAgentMaintenanceActions(input: AgentActionPlanInput): Planne // never overridden). Empty/absent codes (the rule gated OFF at the boundary, or a non-AI failure) ⇒ no-op, so the // verdict is byte-identical. The effective `conclusion` drives every gate-verdict decision below; the AI concern // still surfaces in the review comment (an advisory finding), it just no longer auto-closes a green PR. + // Resolve the blocker codes ONCE (so the nullish fallback is exercised for both a present and an absent list, + // and the checks below carry no further `??` branch). Absent ⇒ [] ⇒ no AI-judgment-only failure. + const gateBlockerCodes = input.gateBlockerCodes ?? []; const aiJudgmentOnlyFailure = - input.conclusion === "failure" && (input.gateBlockerCodes?.length ?? 0) > 0 && (input.gateBlockerCodes ?? []).every((code) => AI_JUDGMENT_BLOCKER_CODES.has(code)); - const conclusion: GateCheckConclusion = aiJudgmentOnlyFailure && ciPassed ? "success" : input.conclusion; + input.aiCiRefutationEnabled === true && input.conclusion === "failure" && gateBlockerCodes.length > 0 && gateBlockerCodes.every((code) => AI_JUDGMENT_BLOCKER_CODES.has(code)); + // The refutation only fires on a GREEN CI (the deterministic validator that overrules the AI). A red/unverified + // CI is the real signal and is never overridden, so the verdict stays as the raw gate `failure`. + const refuteAiFailureOnGreenCi = aiJudgmentOnlyFailure && ciPassed; + const conclusion: GateCheckConclusion = refuteAiFailureOnGreenCi ? "success" : input.conclusion; // Only SUCCESS earns the review-good auto-merge. A NEUTRAL gate flows (no longer silently returns []) but is // NOT auto-merged — it falls through to a HELD + labeled state for review. (Auto-merging a neutral / grace diff --git a/test/unit/agent-actions.test.ts b/test/unit/agent-actions.test.ts index 21d7a277f9..7ad9b6f15e 100644 --- a/test/unit/agent-actions.test.ts +++ b/test/unit/agent-actions.test.ts @@ -279,7 +279,7 @@ describe("planAgentMaintenanceActions (#778)", () => { }); describe("CI-refutation: an AI-judgment-only gate failure does NOT close a green-CI PR (#ai-ci-refutation)", () => { - const merging = { autonomy: { merge: "auto" as const, approve: "auto" as const, close: "auto" as const, label: "auto" as const }, ciState: "passed" as const, pr: { labels: [], mergeableState: "clean" as const, reviewDecision: "APPROVED" as const } }; + const merging = { aiCiRefutationEnabled: true, autonomy: { merge: "auto" as const, approve: "auto" as const, close: "auto" as const, label: "auto" as const }, ciState: "passed" as const, pr: { labels: [], mergeableState: "clean" as const, reviewDecision: "APPROVED" as const } }; it("a consensus-defect failure on a green, clean PR MERGES instead of closing (the validator overrules the model)", () => { const plan = planAgentMaintenanceActions(input({ conclusion: "failure", blockerTitles: ["AI reviewers agree on a likely critical defect"], gateBlockerCodes: ["ai_consensus_defect"], ...merging })); @@ -297,41 +297,46 @@ describe("planAgentMaintenanceActions (#778)", () => { }); it("the refutation reports the EFFECTIVE verdict (verdict=success; CI green), not the contradictory raw failure", () => { - const plan = planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["ai_consensus_defect"], autonomy: { label: "auto" }, ciState: "passed", pr: { labels: [], mergeableState: "clean", reviewDecision: "APPROVED" } })); + const plan = planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["ai_consensus_defect"], aiCiRefutationEnabled: true, autonomy: { label: "auto" }, ciState: "passed", pr: { labels: [], mergeableState: "clean", reviewDecision: "APPROVED" } })); const label = plan.find((a) => a.actionClass === "label"); expect(label?.label).toBe(AGENT_LABEL_READY); expect(label?.reason).toBe("verdict=success; CI green"); }); + it("is GATED by aiCiRefutationEnabled — enabled=false (the converged AI review off) still CLOSES the same PR", () => { + const cls = classes(planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["ai_consensus_defect"], aiCiRefutationEnabled: false, autonomy: { close: "auto" }, ciState: "passed", pr: { labels: [] } }))); + expect(cls).toContain("close"); + }); + it("does NOT refute when CI is RED — a real failing check is the ground truth and still CLOSES", () => { - const cls = classes(planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["ai_consensus_defect"], autonomy: { close: "auto" }, ciState: "failed", pr: { labels: [] } }))); + const cls = classes(planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["ai_consensus_defect"], aiCiRefutationEnabled: true, autonomy: { close: "auto" }, ciState: "failed", pr: { labels: [] } }))); expect(cls).toContain("close"); }); it("does NOT refute a MIXED failure — any deterministic blocker alongside the AI one still CLOSES", () => { - const cls = classes(planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["ai_consensus_defect", "duplicate_open_pr"], autonomy: { close: "auto" }, ciState: "passed", pr: { labels: [] } }))); + const cls = classes(planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["ai_consensus_defect", "duplicate_open_pr"], aiCiRefutationEnabled: true, autonomy: { close: "auto" }, ciState: "passed", pr: { labels: [] } }))); expect(cls).toContain("close"); }); it("does NOT refute a deterministic-only failure (e.g. slop/duplicate) — those close normally on green CI", () => { - const cls = classes(planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["slop_high"], autonomy: { close: "auto" }, ciState: "passed", pr: { labels: [] } }))); + const cls = classes(planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["slop_high"], aiCiRefutationEnabled: true, autonomy: { close: "auto" }, ciState: "passed", pr: { labels: [] } }))); expect(cls).toContain("close"); }); it("does NOT refute ai_review_inconclusive — a 'could not review' hold is not a false defect", () => { // inconclusive isn't a `failure` conclusion in practice (it HOLDS neutral), but even if a failure carried // the code, it must NOT be treated as a refutable false positive. - const cls = classes(planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["ai_review_inconclusive"], autonomy: { close: "auto" }, ciState: "passed", pr: { labels: [] } }))); + const cls = classes(planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["ai_review_inconclusive"], aiCiRefutationEnabled: true, autonomy: { close: "auto" }, ciState: "passed", pr: { labels: [] } }))); expect(cls).toContain("close"); }); - it("is a no-op when codes are omitted (refutation gated OFF at the boundary) — byte-identical close", () => { - const cls = classes(planAgentMaintenanceActions(input({ conclusion: "failure", blockerTitles: ["AI reviewers agree on a likely critical defect"], autonomy: { close: "auto" }, ciState: "passed", pr: { labels: [] } }))); + it("is a no-op when codes are omitted (no AI blockers to refute) — byte-identical close", () => { + const cls = classes(planAgentMaintenanceActions(input({ conclusion: "failure", blockerTitles: ["AI reviewers agree on a likely critical defect"], aiCiRefutationEnabled: true, autonomy: { close: "auto" }, ciState: "passed", pr: { labels: [] } }))); expect(cls).toContain("close"); }); it("a guardrail-touching AI-refuted PR is still HELD for the owner (refutation never overrides the guardrail hold)", () => { - const plan = planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["ai_consensus_defect"], autonomy: { merge: "auto", approve: "auto", close: "auto", label: "auto" }, hardGuardrailGlobs: ["src/**"], changedPaths: ["src/index.ts"], ciState: "passed", pr: { labels: [], mergeableState: "clean", reviewDecision: "APPROVED" } })); + const plan = planAgentMaintenanceActions(input({ conclusion: "failure", gateBlockerCodes: ["ai_consensus_defect"], aiCiRefutationEnabled: true, autonomy: { merge: "auto", approve: "auto", close: "auto", label: "auto" }, hardGuardrailGlobs: ["src/**"], changedPaths: ["src/index.ts"], ciState: "passed", pr: { labels: [], mergeableState: "clean", reviewDecision: "APPROVED" } })); const cls = classes(plan); expect(cls).not.toContain("merge"); expect(cls).not.toContain("close"); diff --git a/test/unit/grounding-wiring.test.ts b/test/unit/grounding-wiring.test.ts index e8689f0cd7..9ca078bc55 100644 --- a/test/unit/grounding-wiring.test.ts +++ b/test/unit/grounding-wiring.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import { runGittensoryAiReview } from "../../src/services/ai-review"; import { runAiReviewForAdvisory } from "../../src/queue/processors"; import { + aiCiRefutationActive, buildCheckAggregate, buildReviewGroundingText, isGroundingEnabled, @@ -87,6 +88,24 @@ describe("isGroundingEnabled", () => { }); }); +describe("aiCiRefutationActive (#ai-ci-refutation gate)", () => { + const env = (grounding: string, repos: string) => ({ GITTENSORY_REVIEW_GROUNDING: grounding, GITTENSORY_REVIEW_REPOS: repos }) as unknown as Env; + const REPO = "JSONbored/metagraphed"; + + it("is ON only when grounding is enabled AND the repo is convergence-allowlisted", () => { + expect(aiCiRefutationActive(env("true", REPO), REPO)).toBe(true); + }); + it("is OFF when grounding is enabled but the repo is NOT allowlisted", () => { + expect(aiCiRefutationActive(env("true", "JSONbored/other"), REPO)).toBe(false); + }); + it("is OFF when grounding is disabled even if the repo is allowlisted (short-circuits before convergence)", () => { + expect(aiCiRefutationActive(env("false", REPO), REPO)).toBe(false); + }); + it("is OFF when both are off", () => { + expect(aiCiRefutationActive(env("", ""), REPO)).toBe(false); + }); +}); + // ── buildCheckAggregate (CI summary source) ────────────────────────────────────────────────────── describe("buildCheckAggregate maps gittensory check summaries → the grounding aggregate", () => { diff --git a/test/unit/rules.test.ts b/test/unit/rules.test.ts index 9b9ec7801e..433876d8d4 100644 --- a/test/unit/rules.test.ts +++ b/test/unit/rules.test.ts @@ -9,6 +9,8 @@ import { firstAddedLineFromPatch, formatCheckRunOutput, formatGateCheckOutput, + isAiJudgmentOnlyFailure, + reconcileGateEvaluationForGreenCi, } from "../../src/rules/advisory"; import type { CollisionReport } from "../../src/signals/engine"; import type { IssueRecord, PullRequestRecord, PullRequestFileRecord, RepositoryRecord } from "../../src/types"; @@ -1076,6 +1078,74 @@ describe("firstAddedLineFromPatch", () => { }); }); +describe("CI-refutation of the public comment gate (#ai-ci-refutation)", () => { + const finding = (code: string): import("../../src/types").AdvisoryFinding => ({ code, severity: "critical", title: `t:${code}`, detail: `d:${code}` }); + const failure = (codes: string[]): import("../../src/rules/advisory").GateCheckEvaluation => ({ + enabled: true, + conclusion: "failure", + title: "Gittensory Gate: blocked", + summary: "A hard blocker was found.", + blockers: codes.map(finding), + warnings: [], + }); + + it("isAiJudgmentOnlyFailure: true only when EVERY blocker is an AI-judgment code", () => { + expect(isAiJudgmentOnlyFailure(failure(["ai_consensus_defect"]))).toBe(true); + expect(isAiJudgmentOnlyFailure(failure(["ai_review_split"]))).toBe(true); + expect(isAiJudgmentOnlyFailure(failure(["ai_consensus_defect", "ai_review_split"]))).toBe(true); + expect(isAiJudgmentOnlyFailure(failure(["ai_consensus_defect", "duplicate_open_pr"]))).toBe(false); + expect(isAiJudgmentOnlyFailure(failure(["slop_high"]))).toBe(false); + expect(isAiJudgmentOnlyFailure(failure(["ai_review_inconclusive"]))).toBe(false); + // An empty blocker list is not a refutable AI-only failure. + expect(isAiJudgmentOnlyFailure({ ...failure([]), conclusion: "failure" })).toBe(false); + // A non-failure conclusion is never AI-only-failure. + expect(isAiJudgmentOnlyFailure({ ...failure(["ai_consensus_defect"]), conclusion: "success" })).toBe(false); + }); + + it("enabled + green CI + AI-judgment-only failure → SUCCESS with cleared blockers (matches the merge disposition)", () => { + const out = reconcileGateEvaluationForGreenCi(failure(["ai_consensus_defect"]), "passed", true); + expect(out.conclusion).toBe("success"); + expect(out.blockers).toEqual([]); + expect(out.title).toBe("Gittensory Gate passed"); + expect(out.summary).toContain("advisory, not blocking"); + }); + + it("enabled + green CI + split-only failure → SUCCESS too", () => { + expect(reconcileGateEvaluationForGreenCi(failure(["ai_review_split"]), "passed", true).conclusion).toBe("success"); + }); + + it("is GATED by `enabled` — enabled=false returns the failure UNCHANGED even on green CI", () => { + const fail = failure(["ai_consensus_defect"]); + expect(reconcileGateEvaluationForGreenCi(fail, "passed", false)).toBe(fail); + }); + + it("does NOT reconcile when CI is red — the real failing check stands", () => { + const fail = failure(["ai_consensus_defect"]); + expect(reconcileGateEvaluationForGreenCi(fail, "failed", true)).toBe(fail); + }); + + it("does NOT reconcile when CI is unverified", () => { + const fail = failure(["ai_consensus_defect"]); + expect(reconcileGateEvaluationForGreenCi(fail, "unverified", true)).toBe(fail); + }); + + it("does NOT reconcile a mixed failure (any deterministic blocker present) even on green CI", () => { + const fail = failure(["ai_consensus_defect", "duplicate_open_pr"]); + expect(reconcileGateEvaluationForGreenCi(fail, "passed", true)).toBe(fail); + expect(reconcileGateEvaluationForGreenCi(fail, "passed", true).conclusion).toBe("failure"); + }); + + it("does NOT reconcile a deterministic-only failure on green CI (e.g. slop)", () => { + const fail = failure(["slop_high"]); + expect(reconcileGateEvaluationForGreenCi(fail, "passed", true)).toBe(fail); + }); + + it("does NOT reconcile a success gate (no-op pass-through)", () => { + const ok = { ...failure([]), conclusion: "success" as const, blockers: [] }; + expect(reconcileGateEvaluationForGreenCi(ok, "passed", true)).toBe(ok); + }); +}); + function emptyCollisions(): CollisionReport { return { repoFullName: "JSONbored/gittensory",