From 7bea99059a810175ffadacad18715e51639917ef Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 26 Jul 2026 05:59:12 -0700 Subject: [PATCH 1/3] =?UTF-8?q?feat(orb):=20distribution-free=20risk-contr?= =?UTF-8?q?ol=20thresholds=20=E2=80=94=20the=20provable-accuracy=20mechani?= =?UTF-8?q?sm?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixed-sequence calibration (Learn-then-Test; Trust or Escalate, ICLR 2025) of the per-arm act/hold threshold over the human-adjudicated labels: sweep lambda in DESCENDING-coverage order, exact Clopper-Pearson upper bound per candidate, stop at the first certification -- then with probability >= 1-delta, P(decision wrong | confidence >= lambda) <= alpha. No distributional assumptions. Separate Neyman-Pearson arms (close alpha=0.015, merge alpha=0.002 -- a wrong merge costs more). Advances #8835. Honesty guards, each pinned by a test: - insufficient labels is a REFUSAL, never a degraded guess: even a zero-error set cannot certify alpha until n >= ln(delta)/ln(1-alpha) (598 clean labels at alpha=0.005) -- and a passing-but-tiny high-confidence clique refuses too - 'uncertain' adjudications are excluded from both sides (the rubric's contract); rule-only decisions (no confidence) cannot join a confidence-thresholded guarantee and are skipped - a stale guarantee is a lie: an under-powered recalibration RETRACTS the published lambda and audits the label burn-down (have/needed) - the coverage-descending sweep direction matters: conservative-first dies on sample-size POWER at small-n candidates, not on errors -- documented in the module Daily recalibration tick (07:00 UTC, flag LOOPOVER_RISK_CONTROL, default OFF, self-host only, stale-queued-job re-check). CONSULT-ONLY deliberately: ai_review_close_confidence already has an automatic writer (backtest-gated knob loosening #8121/#8158), and two auto-writers on one knob need an explicit precedence rule first -- that actuation decision is the tracked remainder on #8835. --- src/env.d.ts | 2 + src/index.ts | 7 ++ src/queue/job-dispatch.ts | 8 ++ src/review/risk-control-wire.ts | 105 ++++++++++++++++++++++ src/review/risk-control.ts | 132 ++++++++++++++++++++++++++++ src/types.ts | 6 ++ test/unit/index.test.ts | 17 ++++ test/unit/risk-control-wire.test.ts | 118 +++++++++++++++++++++++++ test/unit/risk-control.test.ts | 86 ++++++++++++++++++ 9 files changed, 481 insertions(+) create mode 100644 src/review/risk-control-wire.ts create mode 100644 src/review/risk-control.ts create mode 100644 test/unit/risk-control-wire.test.ts create mode 100644 test/unit/risk-control.test.ts diff --git a/src/env.d.ts b/src/env.d.ts index 0ec5ed9585..42b4ab761a 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -504,6 +504,8 @@ declare global { LOOPOVER_REVIEW_SELFTUNE?: string; /** #8830: weekly stratified human-audit sampling of gate decisions (default OFF). */ LOOPOVER_DECISION_AUDIT?: string; + /** #8835: daily distribution-free risk-control recalibration over the audit labels (default OFF). */ + LOOPOVER_RISK_CONTROL?: string; /** Experimental `gittensor` plugin (the `experimental:` manifest block, first key): the operator-level * kill-switch for loopover's original subnet mining-registry/scoring integration, now opt-in rather than * a core dependency. ANDed with the per-repo `.loopover.yml experimental.gittensor` opt-in -- neither diff --git a/src/index.ts b/src/index.ts index 04e44231d2..1b82765e9a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,6 +15,7 @@ import { isPrReconciliationEnabled, resolvePrReconciliationManifestOverride } fr import { isActiveReviewReconciliationEnabled, resolveActiveReviewReconciliationManifestOverride } from "./review/active-review-reconciliation"; import { isRagEnabled } from "./review/rag-wire"; import { isDecisionAuditEnabled } from "./review/decision-audit"; +import { isRiskControlEnabled } from "./review/risk-control-wire"; import { isSelfTuneEnabled } from "./review/selftune-wire"; import { isSatisfactionFloorAutotuneEnabled } from "./services/satisfaction-floor-loosening-run"; import { @@ -298,6 +299,12 @@ async function enqueueScheduledJobs(env: Env, controller: ScheduledController): if (isHourly && scheduledAt.getUTCDay() === 2 && hour === 8 && selfHostedReviews && isDecisionAuditEnabled(env)) { jobs.push({ type: "decision-audit-sample", requestedBy: "schedule" }); } + // Risk-control recalibration (#8835, flag LOOPOVER_RISK_CONTROL): daily fixed-sequence calibration of the + // per-arm act/hold thresholds over the adjudicated labels. 07:00 UTC — its own slot. Enqueued ONLY when + // the flag is ON — flag-OFF (default) this job is never created and the tick is byte-identical. + if (isHourly && hour === 7 && selfHostedReviews && isRiskControlEnabled(env)) { + jobs.push({ type: "risk-control-recalibrate", requestedBy: "schedule" }); + } // Prune expired log/snapshot rows once a day (03:00 UTC) per the conservative RETENTION_POLICY. if (isHourly && hour === 3) { jobs.push({ type: "prune-retention", requestedBy: "schedule" }); diff --git a/src/queue/job-dispatch.ts b/src/queue/job-dispatch.ts index dbb34e20de..0ca00d714c 100644 --- a/src/queue/job-dispatch.ts +++ b/src/queue/job-dispatch.ts @@ -47,6 +47,7 @@ import { syncBrokeredInstalledRepos } from "../orb/installed-repos-sync"; import { incr } from "../selfhost/metrics"; import { generateSignalSnapshots } from "./signal-snapshot"; import { isDecisionAuditEnabled, runDecisionAuditSample } from "../review/decision-audit"; +import { isRiskControlEnabled, runRiskControlRecalibration } from "../review/risk-control-wire"; import { runRetentionPrune } from "./retention"; // The 15 handlers below have no reason to move -- each is only reachable via this dispatcher (or, for // mapWithConcurrency, ALSO used by other still-in-processors.ts code), so they stay put and are exported @@ -244,6 +245,13 @@ export async function processJob(env: Env, message: JobMessage): Promise { console.log(JSON.stringify({ event: "decision_audit_sampled", inserted })); return; } + case "risk-control-recalibrate": { + // #8835: same stale-queued-job posture as its sampling sibling above. + if (!isRiskControlEnabled(env)) return; + const summary = await runRiskControlRecalibration(env); + console.log(JSON.stringify({ event: "risk_control_recalibrated", ...summary })); + return; + } case "generate-weekly-value-report": await generateWeeklyValueReport(env, { variant: message.variant ?? "operator", diff --git a/src/review/risk-control-wire.ts b/src/review/risk-control-wire.ts new file mode 100644 index 0000000000..631616cb78 --- /dev/null +++ b/src/review/risk-control-wire.ts @@ -0,0 +1,105 @@ +// Risk-control recalibration wire (#8835) — the IO around src/review/risk-control.ts's pure math. +// +// Reads (adjudication, decision-time confidence) pairs — human labels from decision_audit_labels (#8830/ +// #8831) joined to the confidence each decision persisted in its decision record (#8834) — runs the +// fixed-sequence calibration PER ARM (Neyman–Pearson: a wrong merge costs more than a wrong close, so each +// arm carries its own α), and publishes the result: a calibrated λ̂ lands in system_flags plus an audit +// event carrying the certified statement; an under-powered set DELETES any stale λ̂ (a stale guarantee is a +// lie) and audits the shortfall so the label-collection burn-down is visible. +// +// DELIBERATELY CONSULT-ONLY in this change: the calibrated λ̂ is not yet wired into the live gate floor, +// because ai_review_close_confidence already has an automatic writer (the backtest-gated knob loosening, +// #8121/#8158) and two auto-writers on one knob need an explicit precedence rule first — that decision is +// tracked on #8835. Flag-gated by LOOPOVER_RISK_CONTROL (default OFF, byte-identical). +import { calibrateActThreshold, type CalibrationPair, type CalibrationResult } from "./risk-control"; +import { recordAuditEvent } from "../db/repositories"; +import { errorMessage, nowIso } from "../utils/json"; + +export function isRiskControlEnabled(env: { LOOPOVER_RISK_CONTROL?: string | undefined }): boolean { + return /^(1|true|yes|on)$/i.test((env.LOOPOVER_RISK_CONTROL ?? "").trim()); +} + +/** Per-arm error budgets (#8835's Neyman–Pearson requirement) and the calibration confidence level. */ +export const RISK_CONTROL_ARMS = [ + { arm: "close" as const, verdict: "close" as const, alpha: 0.015 }, + { arm: "merge" as const, verdict: "merge" as const, alpha: 0.002 }, +]; +export const RISK_CONTROL_DELTA = 0.05; + +/** system_flags key holding one arm's calibrated result (JSON). */ +export function riskControlFlagKey(arm: string): string { + return `riskcontrol:${arm}`; +} + +/** Labeled pairs for one arm: adjudicated correct/incorrect labels (uncertain is EXCLUDED both sides — the + * rubric's contract) joined to the decision-time confidence the record persisted. Rows whose record carries + * no aiConfidence (rule-only decisions) cannot join a confidence-thresholded guarantee and are skipped. */ +export async function loadCalibrationPairs(env: Env, verdict: "close" | "merge"): Promise { + const { results } = await env.DB.prepare( + `SELECT dal.adjudication AS adjudication, dr.record_json AS recordJson + FROM decision_audit_labels dal + JOIN decision_records dr ON dr.repo_full_name || '#' || dr.pull_number = dal.target_id + WHERE dal.status = 'adjudicated' + AND dal.adjudication IN ('correct', 'incorrect') + AND dal.verdict = ?`, + ) + .bind(verdict) + .all<{ adjudication: "correct" | "incorrect"; recordJson: string }>(); + const pairs: CalibrationPair[] = []; + for (const row of results) { + try { + const record = JSON.parse(row.recordJson) as { aiConfidence?: number | null }; + if (typeof record.aiConfidence === "number") { + pairs.push({ confidence: record.aiConfidence, correct: row.adjudication === "correct" }); + } + } catch (error) { + console.warn(JSON.stringify({ event: "risk_control_pair_parse_error", message: errorMessage(error).slice(0, 120) })); + } + } + return pairs; +} + +/** One arm's recalibration: calibrate → publish or retract. Best-effort per arm. */ +async function recalibrateArm(env: Env, arm: string, verdict: "close" | "merge", alpha: number): Promise { + const pairs = await loadCalibrationPairs(env, verdict); + const result = calibrateActThreshold(pairs, alpha, RISK_CONTROL_DELTA); + if (result.status === "calibrated") { + await env.DB.prepare(`INSERT OR REPLACE INTO system_flags (key, value, updated_at) VALUES (?, ?, ?)`) + .bind(riskControlFlagKey(arm), JSON.stringify({ ...result, calibratedAt: nowIso() }), nowIso()) + .run(); + await recordAuditEvent(env, { + eventType: "risk_control_calibrated", + actor: null, + targetKey: `riskcontrol:${arm}`, + outcome: "completed", + detail: `${arm} arm: P(wrong | acted) ≤ ${alpha} guaranteed at ${Math.round(result.coverageAtLambda * 1000) / 10}% coverage (λ=${result.lambda}, n=${result.nAtLambda}, 1−δ=${1 - RISK_CONTROL_DELTA})`, + metadata: { arm, ...result }, + }); + } else { + // A stale guarantee is a lie: retract any previously-published λ̂ the moment the data stops supporting it. + await env.DB.prepare(`DELETE FROM system_flags WHERE key = ?`).bind(riskControlFlagKey(arm)).run(); + await recordAuditEvent(env, { + eventType: "risk_control_insufficient", + actor: null, + targetKey: `riskcontrol:${arm}`, + outcome: "completed", + detail: `${arm} arm: cannot certify α=${alpha} — ${result.have} usable label(s) of ${result.needed} needed`, + metadata: { arm, ...result }, + }); + } + return result; +} + +/** The daily tick: recalibrate every arm. Returns per-arm results for the caller's log line. */ +export async function runRiskControlRecalibration(env: Env): Promise> { + const summary: Record = {}; + for (const { arm, verdict, alpha } of RISK_CONTROL_ARMS) { + try { + summary[arm] = (await recalibrateArm(env, arm, verdict, alpha)).status; + } catch (error) { + console.warn(JSON.stringify({ event: "risk_control_recalibrate_error", arm, message: errorMessage(error).slice(0, 160) })); + summary[arm] = "insufficient_labels"; + } + } + return summary; +} diff --git a/src/review/risk-control.ts b/src/review/risk-control.ts new file mode 100644 index 0000000000..89f378ddb7 --- /dev/null +++ b/src/review/risk-control.ts @@ -0,0 +1,132 @@ +// Distribution-free risk control for the act/hold threshold (#8835, epic #8828 Phase 3) — the mechanism that +// turns "99.5% accurate" from a hope into a guarantee ON THE DECISIONS ACTED. +// +// METHOD: fixed-sequence testing, the Learn-then-Test special case (Angelopoulos et al.; applied to LLM +// judges by Trust or Escalate, ICLR 2025). Sweep the confidence threshold λ from most conservative (1.0) +// downward; at each λ compute the EMPIRICAL error rate over calibration pairs with confidence ≥ λ and take +// an EXACT binomial (Clopper–Pearson) upper confidence bound on it. λ̂ is the smallest λ (largest coverage) +// whose bound — and every more-conservative λ's bound — stays ≤ α. Then, with probability ≥ 1−δ over the +// calibration draw, P(decision wrong | confidence ≥ λ̂) ≤ α. No distributional assumptions. +// +// HONESTY GUARDS, both load-bearing: +// • INSUFFICIENT LABELS IS A REFUSAL, never a degraded guess. Even a zero-error calibration set cannot +// certify α until n ≥ ln(δ)/ln(1−α) (the exact rule-of-three bound) — at α=0.005, δ=0.05 that is 598 +// clean labels. Below it this module refuses and the caller keeps the static floor. +// • `uncertain` adjudications are EXCLUDED from both numerator and denominator (the rubric's contract): +// a genuine judgment call is not evidence about the gate's correctness in either direction. +// +// SEPARATE ARMS (Neyman–Pearson): a wrong merge costs more than a wrong close, so each arm calibrates +// against its own α — never a pooled objective. PURE MODULE: callers own IO. + +/** Exact binomial CDF P(X ≤ k) for X ~ Bin(n, p), via the stable log-pmf recurrence. n here is a label + * count (hundreds), so direct summation is exact enough and allocation-free. */ +function binomialCdf(k: number, n: number, p: number): number { + // Domain note: the only caller is the bisection below, whose midpoints are strictly inside + // (errors/n, 1) — p is never 0 or 1 here, so no boundary guards are needed (and none would be reachable). + let logPmf = n * Math.log(1 - p); // pmf(0) + let cdf = Math.exp(logPmf); + for (let i = 1; i <= k; i += 1) { + logPmf += Math.log((n - i + 1) / i) + Math.log(p) - Math.log(1 - p); + cdf += Math.exp(logPmf); + } + return Math.min(1, cdf); +} + +/** + * Clopper–Pearson UPPER confidence bound for a binomial proportion: the largest p̄ such that observing ≤ + * `errors` failures in `n` trials has probability ≥ δ under Bin(n, p̄). Exact (never anti-conservative), + * found by bisection on the monotone CDF. errors=n returns 1. PURE. + */ +export function clopperPearsonUpperBound(errors: number, n: number, delta: number): number { + if (n <= 0) return 1; + if (errors >= n) return 1; + let lo = errors / n; + let hi = 1; + for (let i = 0; i < 60; i += 1) { + const mid = (lo + hi) / 2; + if (binomialCdf(errors, n, mid) > delta) lo = mid; + else hi = mid; + } + return hi; +} + +/** The exact zero-error sample-size floor: the smallest n where even a CLEAN calibration set can certify α + * at confidence 1−δ (Clopper–Pearson with errors=0 collapses to 1−δ^(1/n) ≤ α). Exported so surfaces can + * say "have 214 of 598 labels" instead of a bare refusal. */ +export function minimumCalibrationLabels(alpha: number, delta: number): number { + return Math.ceil(Math.log(delta) / Math.log(1 - alpha)); +} + +export type CalibrationPair = { + /** The decision-time confidence persisted with the decision record (#8834). */ + confidence: number; + /** The human adjudication: true = the decision was correct. (`uncertain` rows never reach this module.) */ + correct: boolean; +}; + +export type CalibrationResult = + | { + status: "calibrated"; + /** Act when confidence ≥ lambda; hold below. */ + lambda: number; + /** Share of calibration pairs at/above lambda — the coverage the guarantee is earned at. */ + coverageAtLambda: number; + /** Pairs at/above lambda and how many were wrong — the guarantee's own evidence. */ + nAtLambda: number; + errorsAtLambda: number; + /** The certified statement: P(wrong | acted) ≤ alpha with confidence 1−delta. */ + alpha: number; + delta: number; + totalPairs: number; + } + | { status: "insufficient_labels"; needed: number; have: number; alpha: number; delta: number }; + +/** + * Fixed-sequence calibration, walked in DESCENDING-coverage order: candidates are the distinct observed + * confidences ascending, so the FIRST test is the most permissive λ (full calibration set — maximum power, + * maximum coverage) and each subsequent test drops the lowest-confidence stratum. The sweep stops at the + * FIRST λ whose Clopper–Pearson bound certifies α — ordered stopping at the first rejection is what keeps + * the selection valid at level δ without a multiplicity correction (Learn-then-Test, fixed-sequence + * variant). Walking the other way (conservative-first) is a trap: early candidates fail on sample-size + * POWER, not on errors, and monotone stopping would kill the sweep before it ever reached a certifiable λ. + * + * Refuses (never guesses) when the total set is under the zero-error floor, or when no candidate certifies + * — whether from real errors or from a passing-but-tiny high-confidence clique that cannot carry α on its + * own. PURE and deterministic. + */ +export function calibrateActThreshold(pairs: CalibrationPair[], alpha: number, delta: number): CalibrationResult { + const needed = minimumCalibrationLabels(alpha, delta); + if (pairs.length < needed) return { status: "insufficient_labels", needed, have: pairs.length, alpha, delta }; + + const sorted = [...pairs].sort((a, b) => a.confidence - b.confidence); // ascending + const candidates = [...new Set(sorted.map((pair) => pair.confidence))]; // ascending λ = descending coverage + const totalErrors = sorted.filter((pair) => !pair.correct).length; + let dropped = 0; + let droppedErrors = 0; + let index = 0; + let lastTestedN = sorted.length; + for (const lambda of candidates) { + const n = sorted.length - dropped; + const errors = totalErrors - droppedErrors; + lastTestedN = n; + if (n >= needed && clopperPearsonUpperBound(errors, n, delta) <= alpha) { + return { + status: "calibrated", + lambda, + coverageAtLambda: n / sorted.length, + nAtLambda: n, + errorsAtLambda: errors, + alpha, + delta, + totalPairs: sorted.length, + }; + } + // Drop this stratum and test the next, more conservative λ. + while (index < sorted.length && sorted[index]!.confidence <= lambda) { + dropped += 1; + if (!sorted[index]!.correct) droppedErrors += 1; + index += 1; + } + } + return { status: "insufficient_labels", needed, have: lastTestedN, alpha, delta }; +} diff --git a/src/types.ts b/src/types.ts index f77630d950..c62cd2de1c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -151,6 +151,12 @@ export type JobMessage = type: "decision-audit-sample"; requestedBy: "schedule" | "api" | "test"; } + | { + // Risk-control recalibration (#8835, epic #8828): daily fixed-sequence calibration of the per-arm + // act/hold thresholds over the adjudicated labels. Flag-gated by LOOPOVER_RISK_CONTROL. + type: "risk-control-recalibrate"; + requestedBy: "schedule" | "api" | "test"; + } | { type: "generate-weekly-value-report"; requestedBy: "schedule" | "api" | "test"; diff --git a/test/unit/index.test.ts b/test/unit/index.test.ts index b0cba77279..eb0f076786 100644 --- a/test/unit/index.test.ts +++ b/test/unit/index.test.ts @@ -1165,6 +1165,23 @@ describe("worker entrypoint", () => { expect((await send({ LOOPOVER_DECISION_AUDIT: "true", SELFHOST_TRANSIENT_CACHE: undefined }, "2026-06-02T08:00:00.000Z")).some((m) => m.type === "decision-audit-sample")).toBe(false); }); + it("#8835: enqueues the risk-control recalibration daily at 07:00 UTC, flag-ON, on a self-host", async () => { + const send = (over: Record, when: string) => { + const sent: Array = []; + const env = createTestEnv({ + JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue, + ...over, + }); + const waitUntil: Promise[] = []; + return worker.scheduled(controllerFor(when), env, executionContext(waitUntil)).then(() => Promise.all(waitUntil)).then(() => sent); + }; + const selfhost = { SELFHOST_TRANSIENT_CACHE: {} as never, LOOPOVER_RISK_CONTROL: "true" }; + expect((await send(selfhost, "2026-06-03T07:00:00.000Z")).some((m) => m.type === "risk-control-recalibrate")).toBe(true); + expect((await send(selfhost, "2026-06-03T08:00:00.000Z")).some((m) => m.type === "risk-control-recalibrate")).toBe(false); + expect((await send({ ...selfhost, LOOPOVER_RISK_CONTROL: "0" }, "2026-06-03T07:00:00.000Z")).some((m) => m.type === "risk-control-recalibrate")).toBe(false); + expect((await send({ LOOPOVER_RISK_CONTROL: "true", SELFHOST_TRANSIENT_CACHE: undefined }, "2026-06-03T07:00:00.000Z")).some((m) => m.type === "risk-control-recalibrate")).toBe(false); + }); + it("enqueues weekly value report generation during the Monday report window", async () => { const sent: Array = []; const env = createTestEnv({ diff --git a/test/unit/risk-control-wire.test.ts b/test/unit/risk-control-wire.test.ts new file mode 100644 index 0000000000..15ce452970 --- /dev/null +++ b/test/unit/risk-control-wire.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it } from "vitest"; +import { isRiskControlEnabled, loadCalibrationPairs, riskControlFlagKey, runRiskControlRecalibration } from "../../src/review/risk-control-wire"; +import { processJob } from "../../src/queue/processors"; +import { createTestEnv } from "../helpers/d1"; + +// #8835: the IO around the calibration math. What must never drift: uncertain labels excluded, rule-only +// (no-confidence) decisions skipped, per-arm scoping, publish-on-certify, RETRACT-on-insufficient. +async function seedLabeledDecision(env: Env, n: number, verdict: "close" | "merge", adjudication: "correct" | "incorrect" | "uncertain", aiConfidence: number | null): Promise { + await env.DB.prepare( + `INSERT INTO decision_audit_labels (id, project, target_id, verdict, outcome, stratum, rubric_version, sampled_at, status, adjudication, adjudicated_at) + VALUES (?, 'o/r', ?, ?, 'closed', 'close_arm', '1', ?, 'adjudicated', ?, ?)`, + ) + .bind(`audit:o/r#${n}`, `o/r#${n}`, verdict, new Date().toISOString(), adjudication, new Date().toISOString()) + .run(); + await env.DB.prepare( + `INSERT INTO decision_records (id, repo_full_name, pull_number, head_sha, action, reason_code, record_digest, record_json, created_at) + VALUES (?, 'o/r', ?, 'sha', ?, 'r', 'd', ?, ?)`, + ) + .bind(`record:o/r#${n}@sha`, n, verdict, JSON.stringify({ aiConfidence }), new Date().toISOString()) + .run(); +} + +describe("isRiskControlEnabled", () => { + it("default OFF; truthy strings enable", () => { + expect(isRiskControlEnabled({})).toBe(false); + expect(isRiskControlEnabled({ LOOPOVER_RISK_CONTROL: "true" })).toBe(true); + }); +}); + +describe("loadCalibrationPairs", () => { + it("joins adjudicated labels to their record's confidence; excludes uncertain, rule-only, and the other arm", async () => { + const env = createTestEnv(); + await seedLabeledDecision(env, 1, "close", "correct", 0.95); + await seedLabeledDecision(env, 2, "close", "incorrect", 0.6); + await seedLabeledDecision(env, 3, "close", "uncertain", 0.9); // rubric contract: excluded both sides + await seedLabeledDecision(env, 4, "close", "correct", null); // rule-only decision: no confidence to threshold + await seedLabeledDecision(env, 5, "merge", "correct", 0.99); // other arm + const pairs = await loadCalibrationPairs(env, "close"); + expect(pairs.sort((a, b) => a.confidence - b.confidence)).toEqual([ + { confidence: 0.6, correct: false }, + { confidence: 0.95, correct: true }, + ]); + }); +}); + +describe("runRiskControlRecalibration", () => { + it("INSUFFICIENT: retracts any stale flag and audits the shortfall with the burn-down numbers", async () => { + const env = createTestEnv(); + await env.DB.prepare(`INSERT INTO system_flags (key, value) VALUES (?, 'stale')`).bind(riskControlFlagKey("close")).run(); + await seedLabeledDecision(env, 1, "close", "correct", 0.95); + const summary = await runRiskControlRecalibration(env); + expect(summary).toEqual({ close: "insufficient_labels", merge: "insufficient_labels" }); + const flag = await env.DB.prepare(`SELECT value FROM system_flags WHERE key = ?`).bind(riskControlFlagKey("close")).first(); + expect(flag).toBeFalsy(); // a stale guarantee is a lie — retracted + const audit = await env.DB.prepare(`SELECT detail FROM audit_events WHERE event_type = 'risk_control_insufficient' AND target_key = 'riskcontrol:close'`).first<{ detail: string }>(); + expect(audit!.detail).toContain("of 199 needed"); // close-arm alpha 0.015 floor + }); + + it("CALIBRATED: publishes lambda + the certified statement once the close arm clears its floor", async () => { + const env = createTestEnv(); + for (let i = 1; i <= 210; i += 1) await seedLabeledDecision(env, i, "close", "correct", 0.9 + (i % 5) / 100); + const summary = await runRiskControlRecalibration(env); + expect(summary.close).toBe("calibrated"); + expect(summary.merge).toBe("insufficient_labels"); // merge alpha 0.002 needs 1497 — genuinely separate arms + const flag = await env.DB.prepare(`SELECT value FROM system_flags WHERE key = ?`).bind(riskControlFlagKey("close")).first<{ value: string }>(); + const stored = JSON.parse(flag!.value) as { lambda: number; coverageAtLambda: number }; + expect(stored.lambda).toBe(0.9); + expect(stored.coverageAtLambda).toBe(1); + const audit = await env.DB.prepare(`SELECT detail FROM audit_events WHERE event_type = 'risk_control_calibrated'`).first<{ detail: string }>(); + expect(audit!.detail).toContain("P(wrong | acted) ≤ 0.015 guaranteed at 100% coverage"); + }); +}); + +describe("fail-safe arms", () => { + it("an unparseable record_json is skipped with a warn — one bad row never voids the pair set", async () => { + const env = createTestEnv(); + const { vi } = await import("vitest"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + await seedLabeledDecision(env, 1, "close", "correct", 0.95); + await env.DB.prepare("UPDATE decision_records SET record_json = '{broken' WHERE pull_number = 1").run(); + await seedLabeledDecision(env, 2, "close", "incorrect", 0.6); + const pairs = await loadCalibrationPairs(env, "close"); + expect(pairs).toEqual([{ confidence: 0.6, correct: false }]); + expect(warn).toHaveBeenCalled(); + vi.restoreAllMocks(); + }); + + it("a throwing arm recalibration is contained: the OTHER arm still runs and the summary reports insufficient", async () => { + const env = createTestEnv(); + const { vi } = await import("vitest"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const realPrepare = env.DB.prepare.bind(env.DB); + vi.spyOn(env.DB, "prepare").mockImplementation((sql: string) => { + if (sql.includes("dal.verdict = ?")) { + return { bind: () => ({ all: async () => { throw new Error("ledger down"); } }) } as never; + } + return realPrepare(sql); + }); + const summary = await runRiskControlRecalibration(env); + expect(summary).toEqual({ close: "insufficient_labels", merge: "insufficient_labels" }); + expect(warn).toHaveBeenCalled(); + vi.restoreAllMocks(); + }); +}); + +describe("risk-control-recalibrate job dispatch (#8835)", () => { + it("flag-ON runs; flag-OFF (stale queued job) does zero work", async () => { + const on = createTestEnv({ LOOPOVER_RISK_CONTROL: "true" }); + await processJob(on, { type: "risk-control-recalibrate", requestedBy: "test" }); + const audited = await on.DB.prepare(`SELECT COUNT(*) AS n FROM audit_events WHERE event_type LIKE 'risk_control_%'`).first<{ n: number }>(); + expect(audited!.n).toBe(2); // one insufficient audit per arm + + const off = createTestEnv(); + await processJob(off, { type: "risk-control-recalibrate", requestedBy: "test" }); + const none = await off.DB.prepare(`SELECT COUNT(*) AS n FROM audit_events WHERE event_type LIKE 'risk_control_%'`).first<{ n: number }>(); + expect(none!.n).toBe(0); + }); +}); diff --git a/test/unit/risk-control.test.ts b/test/unit/risk-control.test.ts new file mode 100644 index 0000000000..9e7de936df --- /dev/null +++ b/test/unit/risk-control.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from "vitest"; +import { calibrateActThreshold, clopperPearsonUpperBound, minimumCalibrationLabels, type CalibrationPair } from "../../src/review/risk-control"; + +// #8835: the math that turns "99.5%" into a guarantee is pinned exactly — an anti-conservative bound here +// would publish a certainty the labels cannot support, which is the epic's original sin. +describe("clopperPearsonUpperBound", () => { + it("matches the exact zero-error closed form 1 - delta^(1/n)", () => { + for (const [n, delta] of [[10, 0.05], [100, 0.05], [598, 0.05], [59, 0.05]] as const) { + expect(clopperPearsonUpperBound(0, n, delta)).toBeCloseTo(1 - delta ** (1 / n), 6); + } + }); + + it("is monotone: more errors or fewer trials never tightens the bound; edges are honest", () => { + expect(clopperPearsonUpperBound(1, 100, 0.05)).toBeGreaterThan(clopperPearsonUpperBound(0, 100, 0.05)); + expect(clopperPearsonUpperBound(0, 50, 0.05)).toBeGreaterThan(clopperPearsonUpperBound(0, 100, 0.05)); + expect(clopperPearsonUpperBound(5, 5, 0.05)).toBe(1); // all wrong → no upper bound below 1 + expect(clopperPearsonUpperBound(0, 0, 0.05)).toBe(1); // no data → no claim + // Sanity against the rule of three: 0 errors in n gives roughly 3/n at 95%. + expect(clopperPearsonUpperBound(0, 1000, 0.05)).toBeCloseTo(3 / 1000, 3); + }); +}); + +describe("minimumCalibrationLabels", () => { + it("encodes the exact zero-error floor (598 clean labels for alpha=0.005 at 95%)", () => { + expect(minimumCalibrationLabels(0.005, 0.05)).toBe(598); + expect(minimumCalibrationLabels(0.015, 0.05)).toBe(199); + expect(minimumCalibrationLabels(0.002, 0.05)).toBe(1497); + }); +}); + +describe("calibrateActThreshold (fixed-sequence)", () => { + const pair = (confidence: number, correct: boolean): CalibrationPair => ({ confidence, correct }); + + it("REFUSES below the sample-size floor — insufficient labels is never a degraded guess", () => { + const clean = Array.from({ length: 100 }, () => pair(0.99, true)); + const result = calibrateActThreshold(clean, 0.005, 0.05); + expect(result).toMatchObject({ status: "insufficient_labels", needed: 598, have: 100 }); + }); + + it("certifies a clean, large set at full coverage", () => { + const clean = Array.from({ length: 700 }, (_, i) => pair(0.9 + (i % 10) / 100, true)); + const result = calibrateActThreshold(clean, 0.005, 0.05); + expect(result.status).toBe("calibrated"); + if (result.status === "calibrated") { + expect(result.lambda).toBe(0.9); // the least conservative candidate still certifies + expect(result.coverageAtLambda).toBe(1); + expect(result.errorsAtLambda).toBe(0); + } + }); + + it("stops the sweep at the first failing candidate — errors clustered at low confidence RAISE lambda and cut coverage", () => { + // 650 clean pairs at 0.97, then a dirty low-confidence band: the sweep must stop before absorbing it. + const pairs = [ + ...Array.from({ length: 650 }, () => pair(0.97, true)), + ...Array.from({ length: 100 }, (_, i) => pair(0.6, i % 3 !== 0)), // ~33% wrong below the band + ]; + const result = calibrateActThreshold(pairs, 0.005, 0.05); + expect(result.status).toBe("calibrated"); + if (result.status === "calibrated") { + expect(result.lambda).toBe(0.97); + expect(result.nAtLambda).toBe(650); + expect(result.coverageAtLambda).toBeCloseTo(650 / 750, 5); + expect(result.errorsAtLambda).toBe(0); + } + }); + + it("a passing-but-tiny high-confidence clique cannot certify — the prefix itself must clear the floor", () => { + // 50 pristine pairs at 0.99, then errors immediately: the 0.99 prefix passes its bound test... but 50 + // labels cannot certify alpha=0.005, and pretending otherwise is the exact dishonesty this refuses. + const pairs = [...Array.from({ length: 50 }, () => pair(0.99, true)), ...Array.from({ length: 600 }, () => pair(0.5, false))]; + const result = calibrateActThreshold(pairs, 0.005, 0.05); + expect(result).toMatchObject({ status: "insufficient_labels", needed: 598, have: 50 }); + }); + + it("is deterministic and input-order independent", () => { + const base = Array.from({ length: 700 }, (_, i) => pair(0.9 + (i % 10) / 100, i % 400 !== 0)); + const shuffled = [...base].reverse(); + expect(calibrateActThreshold(base, 0.015, 0.05)).toEqual(calibrateActThreshold(shuffled, 0.015, 0.05)); + }); + + it("looser alpha certifies where tighter alpha refuses — the NP arms genuinely differ", () => { + const pairs = Array.from({ length: 250 }, () => pair(0.95, true)); + expect(calibrateActThreshold(pairs, 0.015, 0.05).status).toBe("calibrated"); // close-arm alpha + expect(calibrateActThreshold(pairs, 0.002, 0.05).status).toBe("insufficient_labels"); // merge-arm alpha + }); +}); From e2f08a45539a17604f1c5a42a7133073aa75615b Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 26 Jul 2026 06:16:55 -0700 Subject: [PATCH 2/3] feat(orb): risk-control actuation, per-repo calibration, and the public guarantee Completes #8835 end to end and resolves #8849's precedence question in the same change, per the decided objective (maximum autonomy, minimum standing human involvement): - ACTUATION with the decided precedence: among the AUTOMATIC writers of the AI close-confidence floor, a live calibrated lambda outranks the backtest-gated knob loosening; retraction restores the loosening chain with no human step in either direction. An EXPLICIT per-repo gate.aiReview.closeConfidence still wins over both -- operator config-as-code outranks every automatic writer. The low-confidence hold names its floor's source (calibrated risk-control threshold vs configured floor) so a held contributor sees why. - PER-REPO calibration where a repo's own labels clear the floor (repo-scoped flag keys, independently retractable; the actuation read prefers the repo key, global fallback). - PUBLIC GUARANTEE: the exporter ships live global calibrations in the fleet payload; ingest stores them ONLY for registered instances (the strongest homepage claim must not be plantable via open ingest) and retracts arms the sender stops publishing; fleetAccuracy.guaranteed carries per-arm {alpha, lambda, coveragePct, n}; the hero hint reads 'closes >=98.5% guaranteed at N% coverage' while -- and only while -- a guarantee is live. - BUDGETS as instance-level env config with clamps (close 0.015, merge relaxed 0.005 per the decided objective -- the 0.002 draft needed ~1,497 labels, a year of adjudication for the last 3x of strictness; 0.005 keeps a real 3x asymmetry and is reachable), delta 0.05. Closes #8835. Closes #8849. --- apps/loopover-ui/public/openapi.json | 58 ++++++++ .../site/proof-of-power-stats-model.ts | 1 + .../components/site/proof-of-power-stats.tsx | 2 +- src/env.d.ts | 5 + src/openapi/schemas.ts | 1 + src/orb/ingest.ts | 27 ++++ src/queue/gate-checks.ts | 25 ++-- src/queue/processors.ts | 7 +- src/review/public-stats.ts | 20 +++ src/review/risk-control-wire.ts | 124 ++++++++++++++---- src/rules/advisory.ts | 9 +- src/selfhost/orb-collector.ts | 15 +++ test/integration/orb-ingest.test.ts | 19 +++ test/unit/gate-check-policy.test.ts | 11 ++ test/unit/public-stats.test.ts | 18 +++ test/unit/risk-control-wire.test.ts | 54 +++++++- test/unit/rules.test.ts | 8 ++ test/unit/selfhost-orb-collector.test.ts | 17 +++ 18 files changed, 380 insertions(+), 41 deletions(-) diff --git a/apps/loopover-ui/public/openapi.json b/apps/loopover-ui/public/openapi.json index 60334b6c96..b05be15b6e 100644 --- a/apps/loopover-ui/public/openapi.json +++ b/apps/loopover-ui/public/openapi.json @@ -631,6 +631,63 @@ }, "decidedCount": { "type": "number" + }, + "guaranteed": { + "type": "object", + "properties": { + "close": { + "type": "object", + "nullable": true, + "properties": { + "alpha": { + "type": "number" + }, + "lambda": { + "type": "number" + }, + "coveragePct": { + "type": "number" + }, + "n": { + "type": "number" + } + }, + "required": [ + "alpha", + "lambda", + "coveragePct", + "n" + ] + }, + "merge": { + "type": "object", + "nullable": true, + "properties": { + "alpha": { + "type": "number" + }, + "lambda": { + "type": "number" + }, + "coveragePct": { + "type": "number" + }, + "n": { + "type": "number" + } + }, + "required": [ + "alpha", + "lambda", + "coveragePct", + "n" + ] + } + }, + "required": [ + "close", + "merge" + ] } }, "required": [ @@ -642,6 +699,7 @@ "closePrecisionCiPct", "coveragePct", "decidedCount", + "guaranteed", "instanceCount", "windowDays", "gamingFlagsCaught" diff --git a/apps/loopover-ui/src/components/site/proof-of-power-stats-model.ts b/apps/loopover-ui/src/components/site/proof-of-power-stats-model.ts index f2004785da..14033fefaa 100644 --- a/apps/loopover-ui/src/components/site/proof-of-power-stats-model.ts +++ b/apps/loopover-ui/src/components/site/proof-of-power-stats-model.ts @@ -39,6 +39,7 @@ export type PublicStats = { closePrecisionCiPct?: { lo: number; hi: number } | null; coveragePct?: number | null; decidedCount?: number; + guaranteed?: { close: { alpha: number; lambda: number; coveragePct: number; n: number } | null; merge: { alpha: number; lambda: number; coveragePct: number; n: number } | null }; instanceCount: number; windowDays: number; gamingFlagsCaught: number; diff --git a/apps/loopover-ui/src/components/site/proof-of-power-stats.tsx b/apps/loopover-ui/src/components/site/proof-of-power-stats.tsx index 3813dc9bef..90e24051db 100644 --- a/apps/loopover-ui/src/components/site/proof-of-power-stats.tsx +++ b/apps/loopover-ui/src/components/site/proof-of-power-stats.tsx @@ -168,7 +168,7 @@ export function ProofOfPowerStats({ className }: { className?: string }) { fleetEligible ? // #8829: a bare accuracy scalar at unstated coverage is gameable (holding more raises it), so // the tile names the coverage it was earned at whenever the backend supplies it. - `merge/close calls confirmed by outcome${data.fleetAccuracy.coveragePct != null ? ` · at ${data.fleetAccuracy.coveragePct}% coverage` : ""} · ${intFmt.format(data.fleetAccuracy.instanceCount)} self-hosted instance${data.fleetAccuracy.instanceCount === 1 ? "" : "s"}${data.fleetAccuracy.gamingFlagsCaught > 0 ? ` · ${intFmt.format(data.fleetAccuracy.gamingFlagsCaught)} gaming pattern${data.fleetAccuracy.gamingFlagsCaught === 1 ? "" : "s"} flagged` : ""}` + `merge/close calls confirmed by outcome${data.fleetAccuracy.coveragePct != null ? ` · at ${data.fleetAccuracy.coveragePct}% coverage` : ""}${data.fleetAccuracy.guaranteed?.close ? ` · closes ≥${Math.round((1 - data.fleetAccuracy.guaranteed.close.alpha) * 1000) / 10}% guaranteed at ${data.fleetAccuracy.guaranteed.close.coveragePct}% coverage` : ""} · ${intFmt.format(data.fleetAccuracy.instanceCount)} self-hosted instance${data.fleetAccuracy.instanceCount === 1 ? "" : "s"}${data.fleetAccuracy.gamingFlagsCaught > 0 ? ` · ${intFmt.format(data.fleetAccuracy.gamingFlagsCaught)} gaming pattern${data.fleetAccuracy.gamingFlagsCaught === 1 ? "" : "s"} flagged` : ""}` : totals.reversed > 0 ? `${intFmt.format(totals.reversed)} human-reversed` : "reversal-grounded" diff --git a/src/env.d.ts b/src/env.d.ts index 42b4ab761a..b28857f467 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -506,6 +506,11 @@ declare global { LOOPOVER_DECISION_AUDIT?: string; /** #8835: daily distribution-free risk-control recalibration over the audit labels (default OFF). */ LOOPOVER_RISK_CONTROL?: string; + /** #8835/#8849: per-arm error budgets + calibration confidence — instance-level instrument parameters + * (clamped; defaults 0.015 / 0.005 / 0.05). */ + LOOPOVER_RISK_CONTROL_CLOSE_ALPHA?: string; + LOOPOVER_RISK_CONTROL_MERGE_ALPHA?: string; + LOOPOVER_RISK_CONTROL_DELTA?: string; /** Experimental `gittensor` plugin (the `experimental:` manifest block, first key): the operator-level * kill-switch for loopover's original subnet mining-registry/scoring integration, now opt-in rather than * a core dependency. ANDed with the per-repo `.loopover.yml experimental.gittensor` opt-in -- neither diff --git a/src/openapi/schemas.ts b/src/openapi/schemas.ts index 4ce9a3fbed..b75cc37d54 100644 --- a/src/openapi/schemas.ts +++ b/src/openapi/schemas.ts @@ -141,6 +141,7 @@ export const PublicStatsSchema = z closePrecisionCiPct: z.object({ lo: z.number(), hi: z.number() }).nullable(), coveragePct: z.number().nullable(), decidedCount: z.number(), + guaranteed: z.object({ close: z.object({ alpha: z.number(), lambda: z.number(), coveragePct: z.number(), n: z.number() }).nullable(), merge: z.object({ alpha: z.number(), lambda: z.number(), coveragePct: z.number(), n: z.number() }).nullable() }), instanceCount: z.number(), windowDays: z.number(), gamingFlagsCaught: z.number(), diff --git a/src/orb/ingest.ts b/src/orb/ingest.ts index 65653de7f9..d0570da415 100644 --- a/src/orb/ingest.ts +++ b/src/orb/ingest.ts @@ -208,6 +208,33 @@ export async function handleOrbIngest(body: string, db: D1Database): Promise(); + if (registeredRow?.registered === 1) { + for (const arm of ["close", "merge"]) { + const value = (riskControl as Record)[arm]; + if (value !== undefined && value !== null && typeof value === "object") { + await db + .prepare(`INSERT OR REPLACE INTO system_flags (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)`) + .bind(`riskcontrol:fleet:${arm}`, JSON.stringify(value).slice(0, 2000)) + .run(); + } else { + // The sender no longer publishes this arm — retract the fleet copy too (stale guarantees lie). + await db.prepare(`DELETE FROM system_flags WHERE key = ?`).bind(`riskcontrol:fleet:${arm}`).run(); + } + } + } + } catch { + // best-effort — a calibration hiccup must never fail the outcome batch + } + } + // #8820: day-bucketed reuse counters (optional field; older builds omit it). Every row is // whitelist-validated (strict YYYY-MM-DD day, clamped non-negative counts) and upserted on // (instance_id, day) — the sender re-exports a rolling window each tick, so REPLACE keeps the freshest diff --git a/src/queue/gate-checks.ts b/src/queue/gate-checks.ts index 7997958e2f..4654b49eea 100644 --- a/src/queue/gate-checks.ts +++ b/src/queue/gate-checks.ts @@ -66,10 +66,12 @@ export function gateCheckPolicy( guardrailHit: boolean; guardrailMatches?: ReturnType | undefined; }, - // #8176: the backtest-gated GLOBAL default-override for the AI close-confidence floor, resolved by the - // env-bearing caller (getAiReviewCloseConfidenceOverride — flag-gated + bounds-validated). It only fills - // the DEFAULT: an explicit per-repo `gate.aiReview.closeConfidence` setting always wins below. - aiReviewCloseConfidenceOverride?: number | null, + // #8176/#8849: the AUTOMATIC default-override for the AI close-confidence floor, resolved by the + // env-bearing caller (resolveAutomaticCloseConfidence — a live calibrated λ̂ outranks the backtest-gated + // knob loosening; both are flag-gated + bounds-validated). It only fills the DEFAULT: an explicit per-repo + // `gate.aiReview.closeConfidence` setting always wins below — operator config-as-code outranks every + // automatic writer. Accepts the provenance-carrying shape or (legacy callers/tests) a bare number. + aiReviewCloseConfidenceOverride?: { value: number; calibrated: boolean } | number | null, ) { // `settings` is already the EFFECTIVE config (`.loopover.yml` > DB > defaults), resolved upstream by // resolveRepositorySettings, so the blocker modes here reflect the repo's config file directly. @@ -84,10 +86,17 @@ export function gateCheckPolicy( qualityGateMode: settings.qualityGateMode, qualityGateMinScore: settings.qualityGateMinScore ?? null, aiReviewGateMode: settings.aiReviewMode, - // Calibrated AI close-confidence floor (#7) — config-as-code via `.loopover.yml gate.aiReview.closeConfidence`, - // resolved into settings upstream. When the repo has no explicit setting, the #8176 backtest-gated - // global override (if any) becomes the default; `null` ⇒ advisory.ts applies the 0.93 shipped default. - aiReviewCloseConfidence: settings.aiReviewCloseConfidence ?? aiReviewCloseConfidenceOverride ?? null, + // AI close-confidence floor (#7) — config-as-code via `.loopover.yml gate.aiReview.closeConfidence`, + // resolved into settings upstream. When the repo has no explicit setting, the automatic override chain + // (#8849: calibrated λ̂, else the #8176 backtest loosening) becomes the default; `null` ⇒ advisory.ts + // applies the 0.93 shipped default. + aiReviewCloseConfidence: + settings.aiReviewCloseConfidence ?? + (typeof aiReviewCloseConfidenceOverride === "number" ? aiReviewCloseConfidenceOverride : (aiReviewCloseConfidenceOverride?.value ?? null)), + // #8849: provenance for the low-confidence hold copy — true ONLY when the floor in force actually came + // from a live calibration (an explicit repo setting suppresses it). + aiReviewCloseConfidenceCalibrated: + settings.aiReviewCloseConfidence == null && typeof aiReviewCloseConfidenceOverride === "object" && aiReviewCloseConfidenceOverride !== null && aiReviewCloseConfidenceOverride.calibrated, // Sub-floor AI-judgment disposition (#4603) — DB-backed (dashboard-settable) + `.loopover.yml // gate.aiReview.lowConfidenceDisposition` override, resolved into settings upstream. `null`/undefined ⇒ // advisory.ts applies the "hold_for_review" default. diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 818cd6671f..1206572d28 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -646,6 +646,7 @@ import { } from "../review/outcomes-wire"; import { AI_JUDGMENT_BLOCKER_CODES } from "../rules/advisory"; import { REVIEW_PROMPT_VERSION, REVIEW_SYSTEM_PROMPT } from "../services/ai-review"; +import { resolveAutomaticCloseConfidence } from "../review/risk-control-wire"; import { maybeApplyCloseAuditHoldout } from "../review/close-audit-holdout"; import { buildDecisionRecord, contentDigest, loadDecisionRecordCollapsible, persistDecisionRecord } from "../review/decision-record"; import { neutralHoldReasonCode, nativeGateActionFromConclusion, recordNativeGateDecision } from "../review/parity-wire"; @@ -1548,7 +1549,7 @@ export async function sweepRepoRegate( // unchanged. // #8176: the global close-confidence default-override, resolved once for the sweep (same value the main // webhook path threads; an explicit per-repo setting still wins inside gateCheckPolicy). - const sweepCloseConfidenceOverride = await getAiReviewCloseConfidenceOverride(env, repoFullName); + const sweepCloseConfidenceOverride = await resolveAutomaticCloseConfidence(env, repoFullName, await getAiReviewCloseConfidenceOverride(env, repoFullName)); for (const [index, pr] of candidates.entries()) { const others = openPullRequests.filter( (other) => other.number !== pr.number, @@ -12124,7 +12125,7 @@ async function maybeProcessResolveCommand(env: Env, deliveryId: string, payload: if (!findingRef.ok) { await recordAuditEvent(env, { eventType: "github_app.finding_resolved_skipped", actor: req.actor, targetKey, outcome: "completed", detail: findingRef.reason, metadata: { deliveryId, repoFullName: req.repoFullName, reason: findingRef.reason } }); await recordGithubProductUsage(env, "finding_resolved_skipped", { actor: req.actor, repoFullName: req.repoFullName, targetKey, outcome: "skipped", metadata: { reason: findingRef.reason } }); return true; } const { advisory } = await buildAuthorizedPrActionAdvisory(env, req.repoFullName, pr, settings); await appendPublishedAiReviewFindingsForResolve(env, req.repoFullName, pr, settings.aiReviewMode, advisory); - const gate = evaluateGateCheck(advisory, gateCheckPolicy(settings, null, undefined, pr.slopRisk ?? null, undefined, undefined, await getAiReviewCloseConfidenceOverride(env, req.repoFullName))); + const gate = evaluateGateCheck(advisory, gateCheckPolicy(settings, null, undefined, pr.slopRisk ?? null, undefined, undefined, await resolveAutomaticCloseConfidence(env, req.repoFullName, await getAiReviewCloseConfidenceOverride(env, req.repoFullName)))); const selection = selectWarningsForResolve(gate.warnings, findingRef); if (selection.reason === "finding_not_found") { await recordAuditEvent(env, { eventType: "github_app.finding_resolved_skipped", actor: req.actor, targetKey, outcome: "completed", detail: selection.reason, metadata: { deliveryId, repoFullName: req.repoFullName, reason: selection.reason } }); await recordGithubProductUsage(env, "finding_resolved_skipped", { actor: req.actor, repoFullName: req.repoFullName, targetKey, outcome: "skipped", metadata: { reason: selection.reason } }); return true; } const mode = resolveAgentActionMode({ globalPaused: isGlobalAgentPause(env) || (await isGlobalAgentFrozen(env)), agentPaused: settings.agentPaused, agentDryRun: settings.agentDryRun }); @@ -12355,7 +12356,7 @@ async function maybeProcessExplainCommand(env: Env, deliveryId: string, payload: } const { advisory } = await buildAuthorizedPrActionAdvisory(env, req.repoFullName, pr, settings); await appendPublishedAiReviewFindingsForResolve(env, req.repoFullName, pr, settings.aiReviewMode, advisory); - const gate = evaluateGateCheck(advisory, gateCheckPolicy(settings, null, undefined, pr.slopRisk ?? null, undefined, undefined, await getAiReviewCloseConfidenceOverride(env, req.repoFullName))); + const gate = evaluateGateCheck(advisory, gateCheckPolicy(settings, null, undefined, pr.slopRisk ?? null, undefined, undefined, await resolveAutomaticCloseConfidence(env, req.repoFullName, await getAiReviewCloseConfidenceOverride(env, req.repoFullName)))); const selection = selectWarningsForResolve(gate.warnings, findingRef); if (selection.reason === "finding_not_found") { const notFound = sanitizePublicComment([AGENT_COMMAND_COMMENT_MARKER, "", "> [!NOTE]", `> **No review finding \`${findingRef.findingCode}\` on this PR**`, "> That id is not among this PR's current review findings — re-run `@loopover explain ` with an id from the review summary.", "", "---", loopoverFooter(env)].join("\n")); diff --git a/src/review/public-stats.ts b/src/review/public-stats.ts index 5d7fec1c5b..949cd9e19b 100644 --- a/src/review/public-stats.ts +++ b/src/review/public-stats.ts @@ -256,6 +256,10 @@ export interface PublicStatsPayload { coveragePct: number | null; /** merge + close verdicts behind accuracyPct — the denominator a reader needs to judge the claim. */ decidedCount: number; + /** #8835: the live finite-sample guarantees, per arm, when a registered instance publishes one — + * "P(wrong | acted) ≤ alpha at coveragePct" with the certification's own sample size. Null arms mean no + * guarantee is currently live (insufficient labels, or the instrument retracted it). */ + guaranteed: { close: { alpha: number; lambda: number; coveragePct: number; n: number } | null; merge: { alpha: number; lambda: number; coveragePct: number; n: number } | null }; instanceCount: number; windowDays: number; /** Self-hosted instances currently flagged by computeFleetAnalytics's anti-farming detector @@ -448,6 +452,21 @@ export async function getPublicStats( // for an empty fleet. const fleetAccuracyPct = fleet.fleet.decisionAccuracy === null ? null : Math.round(fleet.fleet.decisionAccuracy * 1000) / 10; + // #8835: live per-arm guarantees, published by a REGISTERED instance's risk-control calibration and + // stored by ingest under riskcontrol:fleet:. Fail-open null — a flags blip hides the guarantee + // rather than fabricating or freezing one. + const readGuarantee = async (arm: string): Promise<{ alpha: number; lambda: number; coveragePct: number; n: number } | null> => { + try { + const row = await env.DB.prepare("SELECT value FROM system_flags WHERE key = ?").bind(`riskcontrol:fleet:${arm}`).first<{ value: string }>(); + if (!row?.value) return null; + const parsed = JSON.parse(row.value) as { alpha?: unknown; lambda?: unknown; coverageAtLambda?: unknown; nAtLambda?: unknown }; + if (typeof parsed.alpha !== "number" || typeof parsed.lambda !== "number" || typeof parsed.coverageAtLambda !== "number" || typeof parsed.nAtLambda !== "number") return null; + return { alpha: parsed.alpha, lambda: parsed.lambda, coveragePct: Math.round(parsed.coverageAtLambda * 1000) / 10, n: parsed.nAtLambda }; + } catch { + return null; + } + }; + const guaranteed = { close: await readGuarantee("close"), merge: await readGuarantee("merge") }; // #8829: intervals/coverage come from the POOLED counts (a median cannot carry a sample size); with one // registered instance — the fleet today — pooled and median views coincide exactly. const pooled = fleet.fleet.pooled; @@ -491,6 +510,7 @@ export async function getPublicStats( closePrecisionCiPct: ciPct(pooled.closeConfirmed, pooled.closeVerdicts), coveragePct: pooled.coverage === null ? null : pct(pooled.coverage), decidedCount: pooledVerdicts, + guaranteed, instanceCount: fleet.instanceCount, windowDays: fleet.windowDays, gamingFlagsCaught: fleet.gamingPatternFlags.length, diff --git a/src/review/risk-control-wire.ts b/src/review/risk-control-wire.ts index 631616cb78..e957ae99b8 100644 --- a/src/review/risk-control-wire.ts +++ b/src/review/risk-control-wire.ts @@ -19,12 +19,30 @@ export function isRiskControlEnabled(env: { LOOPOVER_RISK_CONTROL?: string | und return /^(1|true|yes|on)$/i.test((env.LOOPOVER_RISK_CONTROL ?? "").trim()); } -/** Per-arm error budgets (#8835's Neyman–Pearson requirement) and the calibration confidence level. */ -export const RISK_CONTROL_ARMS = [ - { arm: "close" as const, verdict: "close" as const, alpha: 0.015 }, - { arm: "merge" as const, verdict: "merge" as const, alpha: 0.002 }, -]; -export const RISK_CONTROL_DELTA = 0.05; +/** Parse an instance-level numeric env override with a hard clamp; the default when absent/garbage/outside + * (0, max]. The α/δ budgets are INSTANCE-level instrument parameters (one calibration spans every repo the + * instance reviews), so env — the instance's bootstrap config, like LOOPOVER_RISK_CONTROL itself — is their + * config-as-code home; a per-repo manifest field would imply a per-repo calibration semantics that does not + * exist. */ +export function parseBudget(raw: string | undefined, fallback: number, max: number): number { + const value = Number((raw ?? "").trim()); + if (!Number.isFinite(value) || value <= 0 || value > max) return fallback; + return value; +} + +/** Per-arm error budgets (#8835's Neyman–Pearson requirement) and the calibration confidence level. + * Defaults: close α=0.015 (~199-label floor), merge α=0.005 (~598 — stricter than close by 3x; the earlier + * 0.002 draft needed ~1,497 labels, a year of adjudication for the last 3x of strictness, which contradicts + * the minimal-human-involvement objective this instrument serves). */ +export function riskControlArms(env: Env): Array<{ arm: "close" | "merge"; verdict: "close" | "merge"; alpha: number }> { + return [ + { arm: "close", verdict: "close", alpha: parseBudget(env.LOOPOVER_RISK_CONTROL_CLOSE_ALPHA, 0.015, 0.05) }, + { arm: "merge", verdict: "merge", alpha: parseBudget(env.LOOPOVER_RISK_CONTROL_MERGE_ALPHA, 0.005, 0.05) }, + ]; +} +export function riskControlDelta(env: Env): number { + return parseBudget(env.LOOPOVER_RISK_CONTROL_DELTA, 0.05, 0.2); +} /** system_flags key holding one arm's calibrated result (JSON). */ export function riskControlFlagKey(arm: string): string { @@ -34,17 +52,15 @@ export function riskControlFlagKey(arm: string): string { /** Labeled pairs for one arm: adjudicated correct/incorrect labels (uncertain is EXCLUDED both sides — the * rubric's contract) joined to the decision-time confidence the record persisted. Rows whose record carries * no aiConfidence (rule-only decisions) cannot join a confidence-thresholded guarantee and are skipped. */ -export async function loadCalibrationPairs(env: Env, verdict: "close" | "merge"): Promise { - const { results } = await env.DB.prepare( - `SELECT dal.adjudication AS adjudication, dr.record_json AS recordJson +export async function loadCalibrationPairs(env: Env, verdict: "close" | "merge", project: string | null = null): Promise { + const base = `SELECT dal.adjudication AS adjudication, dr.record_json AS recordJson FROM decision_audit_labels dal JOIN decision_records dr ON dr.repo_full_name || '#' || dr.pull_number = dal.target_id WHERE dal.status = 'adjudicated' AND dal.adjudication IN ('correct', 'incorrect') - AND dal.verdict = ?`, - ) - .bind(verdict) - .all<{ adjudication: "correct" | "incorrect"; recordJson: string }>(); + AND dal.verdict = ?`; + const stmt = project === null ? env.DB.prepare(base).bind(verdict) : env.DB.prepare(`${base} AND LOWER(dal.project) = ?`).bind(verdict, project); + const { results } = await stmt.all<{ adjudication: "correct" | "incorrect"; recordJson: string }>(); const pairs: CalibrationPair[] = []; for (const row of results) { try { @@ -59,43 +75,59 @@ export async function loadCalibrationPairs(env: Env, verdict: "close" | "merge") return pairs; } -/** One arm's recalibration: calibrate → publish or retract. Best-effort per arm. */ -async function recalibrateArm(env: Env, arm: string, verdict: "close" | "merge", alpha: number): Promise { - const pairs = await loadCalibrationPairs(env, verdict); - const result = calibrateActThreshold(pairs, alpha, RISK_CONTROL_DELTA); +/** One arm's recalibration (global when `project` is null, else that repo's own labels): calibrate → + * publish or retract. Best-effort per arm. */ +async function recalibrateArm(env: Env, arm: string, verdict: "close" | "merge", alpha: number, project: string | null): Promise { + const delta = riskControlDelta(env); + const pairs = await loadCalibrationPairs(env, verdict, project); + const result = calibrateActThreshold(pairs, alpha, delta); + const scope = project === null ? arm : `${arm}:${project}`; if (result.status === "calibrated") { await env.DB.prepare(`INSERT OR REPLACE INTO system_flags (key, value, updated_at) VALUES (?, ?, ?)`) - .bind(riskControlFlagKey(arm), JSON.stringify({ ...result, calibratedAt: nowIso() }), nowIso()) + .bind(riskControlFlagKey(scope), JSON.stringify({ ...result, calibratedAt: nowIso() }), nowIso()) .run(); await recordAuditEvent(env, { eventType: "risk_control_calibrated", actor: null, - targetKey: `riskcontrol:${arm}`, + targetKey: `riskcontrol:${scope}`, outcome: "completed", - detail: `${arm} arm: P(wrong | acted) ≤ ${alpha} guaranteed at ${Math.round(result.coverageAtLambda * 1000) / 10}% coverage (λ=${result.lambda}, n=${result.nAtLambda}, 1−δ=${1 - RISK_CONTROL_DELTA})`, + detail: `${scope}: P(wrong | acted) ≤ ${alpha} guaranteed at ${Math.round(result.coverageAtLambda * 1000) / 10}% coverage (λ=${result.lambda}, n=${result.nAtLambda}, 1−δ=${1 - delta})`, metadata: { arm, ...result }, }); } else { // A stale guarantee is a lie: retract any previously-published λ̂ the moment the data stops supporting it. - await env.DB.prepare(`DELETE FROM system_flags WHERE key = ?`).bind(riskControlFlagKey(arm)).run(); + await env.DB.prepare(`DELETE FROM system_flags WHERE key = ?`).bind(riskControlFlagKey(scope)).run(); await recordAuditEvent(env, { eventType: "risk_control_insufficient", actor: null, - targetKey: `riskcontrol:${arm}`, + targetKey: `riskcontrol:${scope}`, outcome: "completed", - detail: `${arm} arm: cannot certify α=${alpha} — ${result.have} usable label(s) of ${result.needed} needed`, + detail: `${scope}: cannot certify α=${alpha} — ${result.have} usable label(s) of ${result.needed} needed`, metadata: { arm, ...result }, }); } return result; } -/** The daily tick: recalibrate every arm. Returns per-arm results for the caller's log line. */ +/** The daily tick: recalibrate every arm globally, then PER-REPO where a repo's own labels clear the floor + * (#8835's "per-repo where sample size permits, global fallback otherwise"). A repo key certifies or is + * retracted independently of the global one; the actuation read prefers the repo key. Returns per-arm + * global statuses for the caller's log line. */ export async function runRiskControlRecalibration(env: Env): Promise> { const summary: Record = {}; - for (const { arm, verdict, alpha } of RISK_CONTROL_ARMS) { + for (const { arm, verdict, alpha } of riskControlArms(env)) { try { - summary[arm] = (await recalibrateArm(env, arm, verdict, alpha)).status; + summary[arm] = (await recalibrateArm(env, arm, verdict, alpha, null)).status; + // Per-repo pass: only repos that have EVER produced a label are considered (the query is the label + // table itself); an under-powered repo is retracted, falling back to the global λ̂ at read time. + const { results } = await env.DB.prepare( + "SELECT DISTINCT project FROM decision_audit_labels WHERE status = 'adjudicated' AND verdict = ?", + ) + .bind(verdict) + .all<{ project: string }>(); + for (const { project } of results) { + await recalibrateArm(env, arm, verdict, alpha, project.toLowerCase()); + } } catch (error) { console.warn(JSON.stringify({ event: "risk_control_recalibrate_error", arm, message: errorMessage(error).slice(0, 160) })); summary[arm] = "insufficient_labels"; @@ -103,3 +135,43 @@ export async function runRiskControlRecalibration(env: Env): Promise { + try { + const keys = repoFullName ? [riskControlFlagKey(`${arm}:${repoFullName.toLowerCase()}`), riskControlFlagKey(arm)] : [riskControlFlagKey(arm)]; + for (const key of keys) { + const row = await env.DB.prepare("SELECT value FROM system_flags WHERE key = ?").bind(key).first<{ value: string }>(); + if (row?.value) { + const parsed = JSON.parse(row.value) as { lambda?: unknown }; + if (typeof parsed.lambda === "number") return parsed.lambda; + } + } + return null; + } catch (error) { + console.warn(JSON.stringify({ event: "risk_control_read_error", arm, message: errorMessage(error).slice(0, 120) })); + return null; + } +} + +/** + * The precedence rule #8849 exists to decide, implemented: among the AUTOMATIC writers of the AI + * close-confidence floor, a live calibrated λ̂ (human-label-backed finite-sample guarantee) outranks the + * backtest-gated knob loosening (#8121/#8158, a throughput optimization under a backtest proxy). Retraction + * of λ̂ automatically restores the loosening chain — no human step in either direction. An EXPLICIT per-repo + * `gate.aiReview.closeConfidence` manifest setting still wins over both downstream (gateCheckPolicy's chain): + * operator config-as-code outranks every automatic writer, in both directions, by standing repo policy. + */ +export async function resolveAutomaticCloseConfidence(env: Env, repoFullName: string | null, knobOverride: number | null): Promise { + if (isRiskControlEnabled(env)) { + const calibrated = await readCalibratedThreshold(env, "close", repoFullName); + if (calibrated !== null) return { value: calibrated, calibrated: true }; + } + return knobOverride !== null ? { value: knobOverride, calibrated: false } : null; +} diff --git a/src/rules/advisory.ts b/src/rules/advisory.ts index 8f108e306a..c6cef3b101 100644 --- a/src/rules/advisory.ts +++ b/src/rules/advisory.ts @@ -57,6 +57,9 @@ export type GateCheckPolicy = { * non-blocker on its own. What varies below the floor is {@link aiReviewLowConfidenceDisposition}: `null`/undefined * ⇒ the 0.93 default. */ aiReviewCloseConfidence?: number | null | undefined; + /** #8849: true when the floor above came from a LIVE risk-control calibration (not a static setting or the + * backtest loosening) — the low-confidence hold names its source so a held contributor sees why. */ + aiReviewCloseConfidenceCalibrated?: boolean | undefined; /** Disposition for a sub-floor `ai_consensus_defect`/`ai_review_split` finding (#4603) — see the type's own doc * comment (`src/types.ts`) for the full semantics of `one_shot` / `hold_for_review` / `advisory_only`. * `null`/undefined ⇒ `hold_for_review` (the shipped default). Only `advisory_only` changes what @@ -231,15 +234,17 @@ export function isAiJudgmentOnlyFailure(evaluation: GateCheckEvaluation): boolea */ export function resolveAiReviewLowConfidenceHold( evaluation: GateCheckEvaluation, - policy: Pick, + policy: Pick, ): { reason: string; comment: string } | undefined { if ((policy.aiReviewLowConfidenceDisposition ?? "hold_for_review") !== "hold_for_review") return undefined; if (!isAiJudgmentOnlyFailure(evaluation)) return undefined; const floor = policy.aiReviewCloseConfidence ?? DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE; const belowFloor = evaluation.blockers.some((blocker) => (blocker.confidence ?? 1) < floor); if (!belowFloor) return undefined; + // #8849: name the calibrated-abstention source when the floor in force is a live risk-control λ̂. + const floorSource = policy.aiReviewCloseConfidenceCalibrated === true ? "calibrated risk-control threshold" : "configured close-confidence floor"; return { - reason: `an AI-reviewer defect finding's confidence is below the configured close-confidence floor (${floor})`, + reason: `an AI-reviewer defect finding's confidence is below the ${floorSource} (${floor})`, comment: "An AI reviewer flagged a likely defect, but its confidence was below this repository's configured close-confidence floor, so this is held for a maintainer to confirm instead of closing automatically. Resolve the flagged defect (see the review notes), or ask a maintainer to override.", }; diff --git a/src/selfhost/orb-collector.ts b/src/selfhost/orb-collector.ts index b178d5cbd1..a7807702b0 100644 --- a/src/selfhost/orb-collector.ts +++ b/src/selfhost/orb-collector.ts @@ -54,6 +54,10 @@ interface OrbExportPayload { instance_id: string; events: FleetEvent[]; health?: { ok: boolean }; + /** #8835/#8849: the instance's live GLOBAL risk-control calibrations (system_flags riskcontrol:), + * when published — counts/α/coverage only, no repo scoping (per-repo keys stay instance-local). The cloud + * stores them ONLY for registered instances and the homepage renders the guarantee while it is live. */ + risk_control?: Record; /** #8820: day-bucketed cache hit/miss aggregates for the public "AI work reused" trend. Counts only — * no repos, no PRs, no content. A rolling window re-sent every tick (the collector upserts per day), * so the field is self-healing and needs no cursor. Omitted when the window has no cache events. */ @@ -244,6 +248,16 @@ export async function exportOrbBatch(db: D1Database, batchSize = 200, fetchFn: t // #8820: the reuse counters ride the same POST as the outcome events (same tick, same signature). Loaded // AFTER the early "nothing to send" return above, so a truly idle tick still costs nothing extra. const reuseCounters = await loadReuseCounters(db, Date.now()); + // #8835: ship the live global calibrations, if any (fail-safe: absent on any read hiccup). + const riskControl: Record = {}; + for (const arm of ["close", "merge"]) { + try { + const row = await db.prepare("SELECT value FROM system_flags WHERE key = ?").bind(`riskcontrol:${arm}`).first<{ value: string }>(); + if (row?.value) riskControl[arm] = JSON.parse(row.value); + } catch { + // a flags blip must never block the outcome export riding the same tick + } + } const payload: OrbExportPayload = { instance_id: instance, @@ -265,6 +279,7 @@ export async function exportOrbBatch(db: D1Database, batchSize = 200, fetchFn: t })), ...(healthOk !== undefined ? { health: { ok: healthOk } } : {}), ...(reuseCounters.length > 0 ? { reuse_counters: reuseCounters } : {}), + ...(Object.keys(riskControl).length > 0 ? { risk_control: riskControl } : {}), }; const body = JSON.stringify(payload); diff --git a/test/integration/orb-ingest.test.ts b/test/integration/orb-ingest.test.ts index 638a0a693a..d3ea369c37 100644 --- a/test/integration/orb-ingest.test.ts +++ b/test/integration/orb-ingest.test.ts @@ -434,4 +434,23 @@ describe("GET /v1/internal/fleet/analytics route", () => { const res = await app.request("/v1/internal/fleet/analytics", {}, createTestEnv()); expect(res.status).toBe(401); }); + + it("stores risk_control calibrations ONLY for REGISTERED senders and retracts absent arms (#8835)", async () => { + const db = new TestD1Database() as unknown as D1Database; + const flag = async () => (await (db as unknown as TestD1Database).prepare("SELECT value FROM system_flags WHERE key='riskcontrol:fleet:close'").first<{ value: string }>())?.value; + const send = (risk_control: unknown) => + handleOrbIngest(JSON.stringify({ instance_id: "inst1", events: [{ repo_hash: "rh", pr_hash: `g${Math.random()}`, outcome: "merged" }], risk_control }), db); + + // Unregistered sender: the strongest homepage claim must not be plantable via open ingest. + await send({ close: { alpha: 0.015, lambda: 0.94, coverageAtLambda: 0.8, nAtLambda: 200 } }); + expect(await flag()).toBeUndefined(); + + await (db as unknown as TestD1Database).prepare("UPDATE orb_instances SET registered = 1 WHERE instance_id = 'inst1'").run(); + await send({ close: { alpha: 0.015, lambda: 0.94, coverageAtLambda: 0.8, nAtLambda: 200 } }); + expect(JSON.parse((await flag())!)).toMatchObject({ lambda: 0.94 }); + + // The sender stops publishing the arm → the fleet copy retracts (a stale guarantee lies). + await send({}); + expect(await flag()).toBeUndefined(); + }); }); diff --git a/test/unit/gate-check-policy.test.ts b/test/unit/gate-check-policy.test.ts index 8255cda29b..c1eed863f9 100644 --- a/test/unit/gate-check-policy.test.ts +++ b/test/unit/gate-check-policy.test.ts @@ -29,6 +29,17 @@ describe("gateCheckPolicy — #8176 global close-confidence default-override", ( expect(gateCheckPolicy(settings({ aiReviewCloseConfidence: 0.97 }), null, undefined, null, undefined, undefined, 0.9).aiReviewCloseConfidence).toBe(0.97); // Neither: null, so advisory.ts applies its shipped default. expect(gateCheckPolicy(settings(), null, undefined, null, undefined, undefined, null).aiReviewCloseConfidence).toBeNull(); + // #8849: the provenance-carrying shape — a calibrated λ̂ sets the floor AND flags its source; an + // explicit repo setting suppresses both (operator config-as-code outranks every automatic writer). + const calibrated = gateCheckPolicy(settings(), null, undefined, null, undefined, undefined, { value: 0.94, calibrated: true }); + expect(calibrated.aiReviewCloseConfidence).toBe(0.94); + expect(calibrated.aiReviewCloseConfidenceCalibrated).toBe(true); + const loosened = gateCheckPolicy(settings(), null, undefined, null, undefined, undefined, { value: 0.9, calibrated: false }); + expect(loosened.aiReviewCloseConfidence).toBe(0.9); + expect(loosened.aiReviewCloseConfidenceCalibrated).toBe(false); + const explicitWins = gateCheckPolicy(settings({ aiReviewCloseConfidence: 0.97 }), null, undefined, null, undefined, undefined, { value: 0.94, calibrated: true }); + expect(explicitWins.aiReviewCloseConfidence).toBe(0.97); + expect(explicitWins.aiReviewCloseConfidenceCalibrated).toBe(false); expect(gateCheckPolicy(settings()).aiReviewCloseConfidence).toBeNull(); }); }); diff --git a/test/unit/public-stats.test.ts b/test/unit/public-stats.test.ts index 1923722ab1..d04f97f366 100644 --- a/test/unit/public-stats.test.ts +++ b/test/unit/public-stats.test.ts @@ -219,6 +219,7 @@ describe("getPublicStats — live aggregate over the review ledger", () => { instanceCount: 0, windowDays: 90, gamingFlagsCaught: 0, + guaranteed: { close: null, merge: null }, }); }); @@ -829,3 +830,20 @@ describe("getPublicStats — live aggregate over the review ledger", () => { expect(out.weekly).toEqual({ reviewed: 0, merged: 0 }); }); }); + +describe("fleetAccuracy.guaranteed (#8835)", () => { + it("publishes a live per-arm guarantee from the fleet flags; malformed or absent flags read null (fail-open)", async () => { + const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "" }); + await env.DB.prepare(`INSERT INTO system_flags (key, value) VALUES ('riskcontrol:fleet:close', ?)`) + .bind(JSON.stringify({ alpha: 0.015, lambda: 0.94, coverageAtLambda: 0.82, nAtLambda: 240 })) + .run(); + await env.DB.prepare(`INSERT INTO system_flags (key, value) VALUES ('riskcontrol:fleet:merge', '{broken')`).run(); + const out = await getPublicStats(env, NOW); + expect(out.fleetAccuracy.guaranteed.close).toEqual({ alpha: 0.015, lambda: 0.94, coveragePct: 82, n: 240 }); + expect(out.fleetAccuracy.guaranteed.merge).toBeNull(); + // A structurally-wrong flag (missing fields) also reads null rather than publishing garbage. + await env.DB.prepare(`UPDATE system_flags SET value = '{"alpha":"high"}' WHERE key = 'riskcontrol:fleet:close'`).run(); + const again = await getPublicStats(env, NOW); + expect(again.fleetAccuracy.guaranteed.close).toBeNull(); + }); +}); diff --git a/test/unit/risk-control-wire.test.ts b/test/unit/risk-control-wire.test.ts index 15ce452970..a1152ef56f 100644 --- a/test/unit/risk-control-wire.test.ts +++ b/test/unit/risk-control-wire.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { isRiskControlEnabled, loadCalibrationPairs, riskControlFlagKey, runRiskControlRecalibration } from "../../src/review/risk-control-wire"; +import { isRiskControlEnabled, loadCalibrationPairs, parseBudget, readCalibratedThreshold, resolveAutomaticCloseConfidence, riskControlArms, riskControlFlagKey, runRiskControlRecalibration } from "../../src/review/risk-control-wire"; import { processJob } from "../../src/queue/processors"; import { createTestEnv } from "../helpers/d1"; @@ -116,3 +116,55 @@ describe("risk-control-recalibrate job dispatch (#8835)", () => { expect(none!.n).toBe(0); }); }); + +describe("actuation precedence (#8849)", () => { + it("a live calibrated λ̂ outranks the knob loosening; retraction restores it; flag-off ignores calibration", async () => { + const env = createTestEnv({ LOOPOVER_RISK_CONTROL: "true" }); + await env.DB.prepare(`INSERT INTO system_flags (key, value) VALUES ('riskcontrol:close', ?)`).bind(JSON.stringify({ lambda: 0.94 })).run(); + expect(await resolveAutomaticCloseConfidence(env, "o/r", 0.9)).toEqual({ value: 0.94, calibrated: true }); + // Retraction → the loosening chain resumes automatically. + await env.DB.prepare(`DELETE FROM system_flags WHERE key = 'riskcontrol:close'`).run(); + expect(await resolveAutomaticCloseConfidence(env, "o/r", 0.9)).toEqual({ value: 0.9, calibrated: false }); + expect(await resolveAutomaticCloseConfidence(env, "o/r", null)).toBeNull(); + // Flag off → calibration is never consulted even when a flag row exists. + const off = createTestEnv(); + await off.DB.prepare(`INSERT INTO system_flags (key, value) VALUES ('riskcontrol:close', ?)`).bind(JSON.stringify({ lambda: 0.94 })).run(); + expect(await resolveAutomaticCloseConfidence(off, "o/r", 0.9)).toEqual({ value: 0.9, calibrated: false }); + }); + + it("readCalibratedThreshold prefers the repo-scoped key, falls back to global, fails OPEN on garbage", async () => { + const env = createTestEnv(); + await env.DB.prepare(`INSERT INTO system_flags (key, value) VALUES ('riskcontrol:close', ?)`).bind(JSON.stringify({ lambda: 0.9 })).run(); + await env.DB.prepare(`INSERT INTO system_flags (key, value) VALUES ('riskcontrol:close:o/r', ?)`).bind(JSON.stringify({ lambda: 0.96 })).run(); + expect(await readCalibratedThreshold(env, "close", "O/R")).toBe(0.96); // repo key, case-insensitive + expect(await readCalibratedThreshold(env, "close", "other/repo")).toBe(0.9); // global fallback + expect(await readCalibratedThreshold(env, "close")).toBe(0.9); // no-repo callers read global directly + await env.DB.prepare(`UPDATE system_flags SET value = '{oops' WHERE key = 'riskcontrol:close:o/r'`).run(); + expect(await readCalibratedThreshold(env, "close", "o/r")).toBeNull(); // garbage fails open, never throws + }); +}); + +describe("env-configurable budgets", () => { + it("clamped parse: defaults on absent/garbage/out-of-range; merge default is the relaxed 0.005", () => { + expect(parseBudget(undefined, 0.015, 0.05)).toBe(0.015); + expect(parseBudget("nope", 0.015, 0.05)).toBe(0.015); + expect(parseBudget("0.2", 0.015, 0.05)).toBe(0.015); // over max + expect(parseBudget("0", 0.015, 0.05)).toBe(0.015); // zero is not a budget + expect(parseBudget("0.01", 0.015, 0.05)).toBe(0.01); + const arms = riskControlArms(createTestEnv({ LOOPOVER_RISK_CONTROL_CLOSE_ALPHA: "0.02" })); + expect(arms.find((a) => a.arm === "close")!.alpha).toBe(0.02); + expect(arms.find((a) => a.arm === "merge")!.alpha).toBe(0.005); + }); +}); + +describe("per-repo calibration (#8835)", () => { + it("publishes a repo-scoped λ̂ when that repo's own labels certify, retractable independently of global", async () => { + const env = createTestEnv({ LOOPOVER_RISK_CONTROL_CLOSE_ALPHA: "0.05" }); // floor ln(.05)/ln(.95) = 59 labels + for (let i = 1; i <= 65; i += 1) await seedLabeledDecision(env, i, "close", "correct", 0.95); + await runRiskControlRecalibration(env); + const repoKey = await env.DB.prepare(`SELECT value FROM system_flags WHERE key = 'riskcontrol:close:o/r'`).first<{ value: string }>(); + expect(JSON.parse(repoKey!.value)).toMatchObject({ lambda: 0.95 }); + const globalKey = await env.DB.prepare(`SELECT value FROM system_flags WHERE key = 'riskcontrol:close'`).first<{ value: string }>(); + expect(JSON.parse(globalKey!.value)).toMatchObject({ lambda: 0.95 }); + }); +}); diff --git a/test/unit/rules.test.ts b/test/unit/rules.test.ts index 534c972eda..8421a25bb6 100644 --- a/test/unit/rules.test.ts +++ b/test/unit/rules.test.ts @@ -533,6 +533,14 @@ describe("advisory rules", () => { const belowFloor = DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE - 0.1; const atFloor = DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE; + it("#8849: the hold names its floor's SOURCE — calibrated risk-control threshold vs configured floor", () => { + const evaluation = failure([finding("ai_consensus_defect", belowFloor)]); + const calibrated = resolveAiReviewLowConfidenceHold(evaluation, { aiReviewCloseConfidenceCalibrated: true }); + expect(calibrated?.reason).toContain("calibrated risk-control threshold"); + const configured = resolveAiReviewLowConfidenceHold(evaluation, { aiReviewCloseConfidenceCalibrated: false }); + expect(configured?.reason).toContain("configured close-confidence floor"); + }); + it("acceptance (1): sub-floor consensus defect + hold_for_review (default) → returns the hold", () => { const evaluation = failure([finding("ai_consensus_defect", belowFloor)]); const hold = resolveAiReviewLowConfidenceHold(evaluation, {}); diff --git a/test/unit/selfhost-orb-collector.test.ts b/test/unit/selfhost-orb-collector.test.ts index 13745187bc..37b13547d7 100644 --- a/test/unit/selfhost-orb-collector.test.ts +++ b/test/unit/selfhost-orb-collector.test.ts @@ -237,6 +237,23 @@ describe("exportOrbBatch() — always-on; reads review_audit, ships anonymized r expect("reuse_counters" in captured!).toBe(false); }); + it("ships live risk-control calibrations in the payload; omits the field when none are published (#8835)", async () => { + const db = makeDb(); + await audit(db, "o/r", 1, "gate_decision", "merge", "2026-02-01T00:00:00Z"); + await audit(db, "o/r", 1, "pr_outcome", "merged", "2026-02-01T01:00:00Z"); + await db.prepare("INSERT INTO system_flags (key, value) VALUES ('riskcontrol:close', ?)").bind(JSON.stringify({ lambda: 0.94, alpha: 0.015 })).run(); + let captured: { risk_control?: Record } | undefined; + await exportOrbBatch(db, 200, async (_u, init) => { captured = JSON.parse(init!.body as string); return new Response(null, { status: 200 }); }); + expect(captured!.risk_control).toEqual({ close: { lambda: 0.94, alpha: 0.015 } }); + + const bare = makeDb(); + await audit(bare, "o/r", 2, "gate_decision", "merge", "2026-02-01T00:00:00Z"); + await audit(bare, "o/r", 2, "pr_outcome", "merged", "2026-02-01T01:00:00Z"); + captured = undefined; + await exportOrbBatch(bare, 200, async (_u, init) => { captured = JSON.parse(init!.body as string); return new Response(null, { status: 200 }); }); + expect("risk_control" in captured!).toBe(false); + }); + it("REGRESSION (#8820): a reversal recorded AFTER a PR was already exported re-exports that PR with the flag", async () => { const db = makeDb(); await audit(db, "o/r", 5, "gate_decision", "close", "2026-02-01T00:00:00Z"); From cf82aca4b930f88f8ad1d52ff44ce41d2073db61 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 26 Jul 2026 06:27:13 -0700 Subject: [PATCH 3/3] fix(orb): scope the calibration-pair join to the latest decision record per PR decision_records keys one row per (PR, head sha), so a PR reviewed across several pushes accumulates several rows; joining labels on target_id alone fanned one adjudicated label into N pairs at different confidences, breaking the one-label-one-trial contract the Clopper-Pearson guarantee depends on. The join now selects the latest record per target (created_at DESC, id DESC tie-break), matching the acted decision the label adjudicates. --- src/review/risk-control-wire.ts | 13 +++++++++++-- test/unit/risk-control-wire.test.ts | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/review/risk-control-wire.ts b/src/review/risk-control-wire.ts index e957ae99b8..4af1aaccea 100644 --- a/src/review/risk-control-wire.ts +++ b/src/review/risk-control-wire.ts @@ -51,11 +51,20 @@ export function riskControlFlagKey(arm: string): string { /** Labeled pairs for one arm: adjudicated correct/incorrect labels (uncertain is EXCLUDED both sides — the * rubric's contract) joined to the decision-time confidence the record persisted. Rows whose record carries - * no aiConfidence (rule-only decisions) cannot join a confidence-thresholded guarantee and are skipped. */ + * no aiConfidence (rule-only decisions) cannot join a confidence-thresholded guarantee and are skipped. + * + * The join scopes to the LATEST decision record per PR (decision_records keys one row per head sha, so a + * PR reviewed across several pushes accumulates several rows): the label adjudicates the decision that was + * ACTED — the latest finalized record, the same latest-row semantics loadDecisionRecordCollapsible renders — + * and a bare target_id join would fan one label out into N pairs with different confidences, breaking the + * one-label-one-trial contract Clopper–Pearson depends on. `id DESC` breaks created_at ties. */ export async function loadCalibrationPairs(env: Env, verdict: "close" | "merge", project: string | null = null): Promise { const base = `SELECT dal.adjudication AS adjudication, dr.record_json AS recordJson FROM decision_audit_labels dal - JOIN decision_records dr ON dr.repo_full_name || '#' || dr.pull_number = dal.target_id + JOIN decision_records dr ON dr.id = ( + SELECT dr2.id FROM decision_records dr2 + WHERE dr2.repo_full_name || '#' || dr2.pull_number = dal.target_id + ORDER BY dr2.created_at DESC, dr2.id DESC LIMIT 1) WHERE dal.status = 'adjudicated' AND dal.adjudication IN ('correct', 'incorrect') AND dal.verdict = ?`; diff --git a/test/unit/risk-control-wire.test.ts b/test/unit/risk-control-wire.test.ts index a1152ef56f..74a5ef5f63 100644 --- a/test/unit/risk-control-wire.test.ts +++ b/test/unit/risk-control-wire.test.ts @@ -41,6 +41,22 @@ describe("loadCalibrationPairs", () => { { confidence: 0.95, correct: true }, ]); }); + + it("a PR with several records (one per head sha) contributes exactly ONE pair — the latest record's confidence", async () => { + const env = createTestEnv(); + await seedLabeledDecision(env, 1, "close", "correct", 0.7); // first review cycle @sha + // Two later cycles on new head shas. Without latest-record scoping this one label fans out into three + // pairs at three different confidences, corrupting the calibration set's one-label-one-trial contract. + for (const [sha, confidence, offsetMs] of [["sha2", 0.8, 60_000], ["sha3", 0.9, 120_000]] as const) { + await env.DB.prepare( + `INSERT INTO decision_records (id, repo_full_name, pull_number, head_sha, action, reason_code, record_digest, record_json, created_at) + VALUES (?, 'o/r', 1, ?, 'close', 'r', 'd', ?, ?)`, + ) + .bind(`record:o/r#1@${sha}`, sha, JSON.stringify({ aiConfidence: confidence }), new Date(Date.now() + offsetMs).toISOString()) + .run(); + } + expect(await loadCalibrationPairs(env, "close")).toEqual([{ confidence: 0.9, correct: true }]); + }); }); describe("runRiskControlRecalibration", () => {