From 27803265f758d95b2acad5f968ce206837f6c697 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 02:50:01 -0700 Subject: [PATCH 1/7] =?UTF-8?q?feat(calibration):=20phase-2=20backfill=20C?= =?UTF-8?q?LI=20=E2=80=94=20retro=20successor=20labels=20+=20raw-context?= =?UTF-8?q?=20re-fetch=20(GitHub=20truth)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass A runs #8166's evaluateSuccessorMatch retrospectively over the phase-1 backfilled close decisions: a bot-closed PR with a merged successor (shared linked issue, or same author reworking a majority of its files, merged inside the 30-day lookback) gets its override row's verdict flipped to reversed with distinct github_successor_scan provenance — the corpus's first organic-shaped negative labels. Pass B re-fetches PR diffs (public repos only) into the phase-1 fired rows' metadata.diff, the exact field the live #8130 capture records, bounded by the same cap. Same discipline as phase 1: pure core + thin IO wrapper, deterministic backfill: ids only (live rows unreachable by construction), idempotent patchers (already-patched rows return null), dry-run default, hard per-run GitHub request budget with a resumable state-file cursor. Advances #8170 --- ...backfill-calibration-corpus-phase2-core.ts | 146 ++++++++ scripts/backfill-calibration-corpus-phase2.ts | 331 ++++++++++++++++++ ...backfill-calibration-corpus-phase2.test.ts | 141 ++++++++ 3 files changed, 618 insertions(+) create mode 100644 scripts/backfill-calibration-corpus-phase2-core.ts create mode 100644 scripts/backfill-calibration-corpus-phase2.ts create mode 100644 test/unit/backfill-calibration-corpus-phase2.test.ts diff --git a/scripts/backfill-calibration-corpus-phase2-core.ts b/scripts/backfill-calibration-corpus-phase2-core.ts new file mode 100644 index 0000000000..88ca87b6c8 --- /dev/null +++ b/scripts/backfill-calibration-corpus-phase2-core.ts @@ -0,0 +1,146 @@ +// Pure core for the phase-2 calibration backfill (#8170, epic #8082): the two GitHub-truth passes phase 1 +// (#8157, backfill-calibration-corpus-core.ts) deliberately deferred. The thin IO wrapper +// (backfill-calibration-corpus-phase2.ts) does every DB/GitHub read and write; everything here is pure. +// +// • Pass A — retro successor scan: run #8166's `evaluateSuccessorMatch` (imported, never re-implemented) +// over historical bot-close decisions vs the merged PRs that followed them. A confirmed match flips the +// phase-1 override row's verdict to `reversed` — the ledger's first organic-shaped negative labels. +// • Pass B — raw-context re-fetch: patch the phase-1 fired rows with the PR diff the live capture (#8130) +// would have recorded (`metadata.diff`, bounded to RAW_CONTEXT_MAX_DIFF_CHARS), public repos only. +// • Conservative + idempotent: borderline successor matches record NOTHING; both patchers return null on +// an already-patched row, so re-runs are no-ops; every patched row carries a distinct provenance tag. +import { evaluateSuccessorMatch, SUPERSEDED_LOOKBACK_MS, type SupersededHeuristics } from "../src/review/reversal-superseded.js"; +import { RAW_CONTEXT_MAX_DIFF_CHARS } from "../src/rules/advisory.js"; +import { BACKFILL_RULE_ID } from "./backfill-calibration-corpus-core.js"; + +/** Distinct provenance for pass A's retro labels — never confusable with phase 1's decision-level rows. */ +export const RETRO_SUCCESSOR_PROVENANCE = "github_successor_scan"; +/** Distinct provenance for pass B's re-fetched raw context. */ +export const RAW_CONTEXT_REFETCH_PROVENANCE = "github_raw_context_refetch"; + +/** A phase-1 backfilled close decision, hydrated with the GitHub truth the wrapper fetched. */ +export type HistoricalCloseSide = { + targetKey: string; + repo: string; + number: number; + /** ISO close time (phase 1's terminal_at) — successors must merge within the #8166 lookback AFTER it. */ + closedAt: string; + authorLogin: string | null; + linkedIssues: readonly number[]; + files: readonly string[]; +}; + +/** A candidate successor: a PR in the same repo that actually merged. */ +export type SuccessorSide = { + number: number; + mergedAt: string; + authorLogin: string | null; + linkedIssues: readonly number[]; + files: readonly string[]; +}; + +export type RetroSuccessorMatch = { + targetKey: string; + supersededBy: number; + heuristics: SupersededHeuristics; +}; + +/** + * Decide which historical closes were superseded by a later merge. Pure: both sides arrive pre-fetched. + * The window is directional — a successor must merge AFTER the close and within {@link SUPERSEDED_LOOKBACK_MS} + * (#8166's own bound) — and the EARLIEST qualifying merge wins so re-runs with more candidates stay stable. + */ +export function matchRetroSuccessors(close: HistoricalCloseSide, successors: readonly SuccessorSide[]): RetroSuccessorMatch | null { + const closedAtMs = Date.parse(close.closedAt); + if (!Number.isFinite(closedAtMs)) return null; + const eligible = successors + .filter((successor) => { + if (successor.number === close.number) return false; + const mergedAtMs = Date.parse(successor.mergedAt); + return Number.isFinite(mergedAtMs) && mergedAtMs > closedAtMs && mergedAtMs - closedAtMs <= SUPERSEDED_LOOKBACK_MS; + }) + .sort((a, b) => (a.mergedAt < b.mergedAt ? -1 : a.mergedAt > b.mergedAt ? 1 : a.number - b.number)); + for (const successor of eligible) { + const heuristics = evaluateSuccessorMatch( + { authorLogin: successor.authorLogin, linkedIssues: successor.linkedIssues, files: successor.files }, + { authorLogin: close.authorLogin, linkedIssues: close.linkedIssues, files: close.files }, + ); + if (heuristics) return { targetKey: close.targetKey, supersededBy: successor.number, heuristics }; + } + return null; +} + +/** The deterministic phase-1 row ids this pass is allowed to touch — live capture rows are never patched. */ +export function backfillOverrideId(targetKey: string): string { + return `backfill:${BACKFILL_RULE_ID}:${targetKey}:override`; +} +export function backfillFiredId(targetKey: string): string { + return `backfill:${BACKFILL_RULE_ID}:${targetKey}:fired`; +} + +/** + * Patch a phase-1 override row's metadata to the retro `reversed` verdict. Returns the new JSON, or null + * when the row is already reversed (idempotent re-run) or does not parse as an object (never guess). + */ +export function patchOverrideMetadataToReversed(metadataJson: string, match: RetroSuccessorMatch): string | null { + const metadata = parseObject(metadataJson); + if (!metadata) return null; + if (metadata.verdict === "reversed") return null; + return JSON.stringify({ + ...metadata, + verdict: "reversed", + retroLabel: { + provenance: RETRO_SUCCESSOR_PROVENANCE, + supersededBy: match.supersededBy, + heuristics: match.heuristics, + }, + }); +} + +/** + * Patch a phase-1 fired row's metadata with the re-fetched PR diff — the field the live #8130 capture + * records for this rule (`metadata.diff`, same bound). Returns null when raw context is already present + * (either captured live or patched by an earlier run), when the diff is empty, or on unparseable metadata. + */ +export function patchFiredMetadataWithDiff(metadataJson: string, diff: string): string | null { + const metadata = parseObject(metadataJson); + if (!metadata) return null; + if (typeof metadata.diff === "string") return null; + const bounded = diff.slice(0, RAW_CONTEXT_MAX_DIFF_CHARS); + if (bounded === "") return null; + return JSON.stringify({ ...metadata, diff: bounded, rawContextProvenance: RAW_CONTEXT_REFETCH_PROVENANCE }); +} + +export type Phase2Report = { + pass: "successors" | "raw-context"; + scanned: number; + patched: number; + alreadyPatched: number; + noMatch: number; + requestsUsed: number; + exhaustedBudget: boolean; + resumeFrom: string | null; +}; + +/** Render the dry-run/apply report #8170 requires before any apply. */ +export function renderPhase2Report(report: Phase2Report, mode: "dry-run" | "apply"): string { + const lines = [ + `Calibration corpus backfill phase 2 (${mode}) — pass ${report.pass}, provenance ${ + report.pass === "successors" ? RETRO_SUCCESSOR_PROVENANCE : RAW_CONTEXT_REFETCH_PROVENANCE + }`, + ` scanned: ${report.scanned} patched: ${report.patched} already-patched: ${report.alreadyPatched} no-match/skipped: ${report.noMatch}`, + ` GitHub requests used: ${report.requestsUsed}${report.exhaustedBudget ? " (budget exhausted — resumable)" : ""}`, + ]; + if (report.resumeFrom) lines.push(` resume from: ${report.resumeFrom} (state file updated)`); + return lines.join("\n"); +} + +function parseObject(json: string): Record | null { + try { + const parsed: unknown = JSON.parse(json); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed as Record; + } catch { + /* corrupt row -- treat as unpatchable, mirroring phase 1's fail-open metadata parse */ + } + return null; +} diff --git a/scripts/backfill-calibration-corpus-phase2.ts b/scripts/backfill-calibration-corpus-phase2.ts new file mode 100644 index 0000000000..d3b09f14e9 --- /dev/null +++ b/scripts/backfill-calibration-corpus-phase2.ts @@ -0,0 +1,331 @@ +#!/usr/bin/env node +// Phase-2 calibration-corpus backfill CLI (#8170, epic #8082) — the two GitHub-truth passes over the rows +// phase 1 (#8157) synthesized. All matching/patching logic lives in backfill-calibration-corpus-phase2-core.ts +// (pure, unit-tested); this file is the thin IO wrapper — mirrors backfill-calibration-corpus.ts's split. +// +// tsx scripts/backfill-calibration-corpus-phase2.ts --pass successors --db loopover [--remote] [--apply] +// tsx scripts/backfill-calibration-corpus-phase2.ts --pass raw-context --db loopover [--remote] [--apply] +// +// Both passes are dry-run by default, resumable (--state-file, default .backfill-phase2-state.json — the +// cursor survives budget exhaustion), and hard-capped on GitHub requests per run (--max-requests, default +// 300). Auth: GITHUB_TOKEN or GH_TOKEN. Pass A flips phase-1 override verdicts to `reversed` where #8166's +// successor heuristics confirm a bot-closed PR was superseded by a merge; pass B patches phase-1 fired rows +// with the PR diff the live #8130 capture records, PUBLIC repos only. Only rows whose ids carry the +// deterministic `backfill:` prefix are ever touched — live capture rows are out of reach by construction. +import { readFileSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { extractLinkedIssueNumbers } from "../src/db/repositories.js"; +import { + backfillFiredId, + backfillOverrideId, + matchRetroSuccessors, + patchFiredMetadataWithDiff, + patchOverrideMetadataToReversed, + renderPhase2Report, + type HistoricalCloseSide, + type Phase2Report, + type SuccessorSide, +} from "./backfill-calibration-corpus-phase2-core.js"; +import { BACKFILL_RULE_ID } from "./backfill-calibration-corpus-core.js"; + +type Pass = "successors" | "raw-context"; +type Args = { db: string; remote: boolean; apply: boolean; pass: Pass; maxRequests: number; stateFile: string }; + +function parseArgs(argv: string[]): Args { + const args: Args = { db: "loopover", remote: false, apply: false, pass: "successors", maxRequests: 300, stateFile: ".backfill-phase2-state.json" }; + for (let i = 0; i < argv.length; i += 1) { + const flag = argv[i]; + if (flag === "--remote") args.remote = true; + else if (flag === "--apply") args.apply = true; + else if (flag === "--db") args.db = argv[++i]!; + else if (flag === "--pass") { + const value = argv[++i]; + if (value !== "successors" && value !== "raw-context") throw new Error(`--pass must be successors or raw-context, got ${value}`); + args.pass = value; + } else if (flag === "--max-requests") args.maxRequests = Number(argv[++i]); + else if (flag === "--state-file") args.stateFile = argv[++i]!; + } + if (!Number.isFinite(args.maxRequests) || args.maxRequests < 1) throw new Error("--max-requests must be a positive number"); + return args; +} + +// Mirrors backfill-calibration-corpus.ts's d1Execute: fail-loud so a partial read/write never passes silently. +function d1Execute(db: string, remote: boolean, sql: string): Array> { + const result = spawnSync("npx", ["wrangler", "d1", "execute", db, remote ? "--remote" : "--local", "--json", "--command", sql], { + encoding: "utf8", + maxBuffer: 256 * 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error(`wrangler d1 execute failed (${result.status}): ${(result.stderr || result.stdout || "").slice(0, 500)}`); + } + const parsed = JSON.parse(result.stdout); + const first = Array.isArray(parsed) ? parsed[0] : parsed; + return first?.results ?? []; +} + +function sqlStringLiteral(value: string): string { + return `'${value.replace(/'/g, "''")}'`; +} + +// ── GitHub IO (budgeted; mirrors check-mcp-release-due.ts's timeout posture) ───────────────────────────── + +const GITHUB_TIMEOUT_MS = 30_000; + +class RequestBudget { + used = 0; + constructor(private readonly max: number) {} + get exhausted(): boolean { + return this.used >= this.max; + } + spend(): void { + this.used += 1; + } +} + +async function githubFetch(budget: RequestBudget, path: string, accept = "application/vnd.github+json"): Promise { + const token = process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN; + if (!token) throw new Error("GITHUB_TOKEN (or GH_TOKEN) is required for the GitHub-truth passes"); + budget.spend(); + return fetch(`https://api.github.com${path}`, { + headers: { authorization: `Bearer ${token}`, accept, "user-agent": "loopover-backfill-phase2" }, + signal: AbortSignal.timeout(GITHUB_TIMEOUT_MS), + }); +} + +async function githubJson(budget: RequestBudget, path: string): Promise { + const response = await githubFetch(budget, path); + if (response.status === 404 || response.status === 410) return null; // deleted repo/PR — skip, never guess + if (!response.ok) throw new Error(`GitHub ${path} failed: ${response.status}`); + return (await response.json()) as T; +} + +type GithubPull = { + number: number; + state: string; + merged_at: string | null; + closed_at: string | null; + body: string | null; + title: string | null; + user: { login?: string } | null; + base: { repo: { private?: boolean } | null } | null; +}; + +async function fetchPullFiles(budget: RequestBudget, repo: string, number: number): Promise { + const files = await githubJson>(budget, `/repos/${repo}/pulls/${number}/files?per_page=100`); + return (files ?? []).map((file) => file.filename ?? "").filter((name) => name !== ""); +} + +function pullLinkedIssues(repo: string, pull: GithubPull): number[] { + return extractLinkedIssueNumbers(`${pull.title ?? ""}\n${pull.body ?? ""}`, repo); +} + +/** Merged PRs in `repo` whose merge could fall inside any pending close's lookback window. Pages + * sort=updated desc and stops once updated_at (an upper bound on merged_at) predates the oldest close. */ +async function fetchMergedSuccessors(budget: RequestBudget, repo: string, oldestClosedAtIso: string): Promise { + const successors: SuccessorSide[] = []; + for (let page = 1; page <= 10 && !budget.exhausted; page += 1) { + const pulls = await githubJson>( + budget, + `/repos/${repo}/pulls?state=closed&sort=updated&direction=desc&per_page=100&page=${page}`, + ); + if (!pulls || pulls.length === 0) break; + for (const pull of pulls) { + if (pull.merged_at) { + successors.push({ + number: pull.number, + mergedAt: pull.merged_at, + authorLogin: pull.user?.login ?? null, + linkedIssues: pullLinkedIssues(repo, pull), + files: [], // fetched lazily only when the author path needs them (cost control) + }); + } + } + if (pulls[pulls.length - 1]!.updated_at < oldestClosedAtIso) break; + } + return successors; +} + +// ── State file (resumable cursor per pass) ─────────────────────────────────────────────────────────────── + +type CursorState = { successorsResumeFrom?: string; rawContextResumeFrom?: string }; + +function readState(path: string): CursorState { + try { + return JSON.parse(readFileSync(path, "utf8")) as CursorState; + } catch { + return {}; + } +} + +// ── Passes ─────────────────────────────────────────────────────────────────────────────────────────────── + +type BackfillRow = { id: string; target_key: string; metadata_json: string; created_at: string }; + +function loadBackfillRows(args: Args, kind: "override" | "fired"): BackfillRow[] { + const rows = d1Execute( + args.db, + args.remote, + `SELECT id, target_key, metadata_json, created_at FROM audit_events WHERE id LIKE 'backfill:${BACKFILL_RULE_ID}:%:${kind}' ORDER BY target_key`, + ); + return rows.filter( + (row): row is BackfillRow => + typeof row.id === "string" && typeof row.target_key === "string" && typeof row.metadata_json === "string" && typeof row.created_at === "string", + ); +} + +function applyMetadataUpdate(args: Args, id: string, metadataJson: string): void { + d1Execute(args.db, args.remote, `UPDATE audit_events SET metadata_json = ${sqlStringLiteral(metadataJson)} WHERE id = ${sqlStringLiteral(id)}`); +} + +function splitTargetKey(targetKey: string): { repo: string; number: number } | null { + const hash = targetKey.lastIndexOf("#"); + if (hash <= 0) return null; + const number = Number(targetKey.slice(hash + 1)); + return Number.isFinite(number) ? { repo: targetKey.slice(0, hash), number } : null; +} + +async function runSuccessorsPass(args: Args, budget: RequestBudget, state: CursorState): Promise { + const report: Phase2Report = { pass: "successors", scanned: 0, patched: 0, alreadyPatched: 0, noMatch: 0, requestsUsed: 0, exhaustedBudget: false, resumeFrom: null }; + const rows = loadBackfillRows(args, "override").filter((row) => !state.successorsResumeFrom || row.target_key > state.successorsResumeFrom); + + const byRepo = new Map(); + for (const row of rows) { + const split = splitTargetKey(row.target_key); + if (!split) continue; + (byRepo.get(split.repo) ?? byRepo.set(split.repo, []).get(split.repo)!).push(row); + } + + outer: for (const [repo, repoRows] of byRepo) { + const oldestClosedAt = repoRows.reduce((min, row) => (row.created_at < min ? row.created_at : min), repoRows[0]!.created_at); + const successors = await fetchMergedSuccessors(budget, repo, oldestClosedAt); + const successorFiles = new Map(); + + for (const row of repoRows) { + if (budget.exhausted) { + report.exhaustedBudget = true; + report.resumeFrom = state.successorsResumeFrom ?? null; + break outer; + } + report.scanned += 1; + const split = splitTargetKey(row.target_key)!; + + const closedPull = await githubJson(budget, `/repos/${repo}/pulls/${split.number}`); + if (!closedPull || closedPull.merged_at) { + // Gone, or actually merged (not a standing bot-close) — not a reversal candidate. + report.noMatch += 1; + state.successorsResumeFrom = row.target_key; + continue; + } + const close: HistoricalCloseSide = { + targetKey: row.target_key, + repo, + number: split.number, + closedAt: closedPull.closed_at ?? row.created_at, + authorLogin: closedPull.user?.login ?? null, + linkedIssues: pullLinkedIssues(repo, closedPull), + files: await fetchPullFiles(budget, repo, split.number), + }; + + // Cheap pass first: the linked-issue path needs no successor files at all. + let match = matchRetroSuccessors(close, successors); + if (!match && close.authorLogin) { + // Author path: hydrate files for same-author successors only, then re-evaluate. + const sameAuthor = successors.filter((successor) => successor.authorLogin?.toLowerCase() === close.authorLogin!.toLowerCase()); + for (const successor of sameAuthor) { + if (budget.exhausted) break; + if (!successorFiles.has(successor.number)) successorFiles.set(successor.number, await fetchPullFiles(budget, repo, successor.number)); + } + match = matchRetroSuccessors( + close, + successors.map((successor) => ({ ...successor, files: successorFiles.get(successor.number) ?? successor.files })), + ); + } + + if (!match) { + report.noMatch += 1; + state.successorsResumeFrom = row.target_key; + continue; + } + const patched = patchOverrideMetadataToReversed(row.metadata_json, match); + if (patched === null) { + report.alreadyPatched += 1; + } else if (args.apply) { + applyMetadataUpdate(args, backfillOverrideId(row.target_key), patched); + report.patched += 1; + } else { + report.patched += 1; // dry-run: counted as "would patch" + } + state.successorsResumeFrom = row.target_key; + } + } + report.requestsUsed = budget.used; + return report; +} + +async function runRawContextPass(args: Args, budget: RequestBudget, state: CursorState): Promise { + const report: Phase2Report = { pass: "raw-context", scanned: 0, patched: 0, alreadyPatched: 0, noMatch: 0, requestsUsed: 0, exhaustedBudget: false, resumeFrom: null }; + const rows = loadBackfillRows(args, "fired").filter((row) => !state.rawContextResumeFrom || row.target_key > state.rawContextResumeFrom); + const repoPrivacy = new Map(); + + for (const row of rows) { + if (budget.exhausted) { + report.exhaustedBudget = true; + report.resumeFrom = state.rawContextResumeFrom ?? null; + break; + } + report.scanned += 1; + const split = splitTargetKey(row.target_key); + if (!split) { + report.noMatch += 1; + continue; + } + + if (!repoPrivacy.has(split.repo)) { + const repoInfo = await githubJson<{ private?: boolean }>(budget, `/repos/${split.repo}`); + repoPrivacy.set(split.repo, repoInfo?.private !== false); // missing repo counts as private — never fetch + } + if (repoPrivacy.get(split.repo)) { + report.noMatch += 1; // private (or gone) repos are out of scope by the issue's own boundary + state.rawContextResumeFrom = row.target_key; + continue; + } + + const diffResponse = await githubFetch(budget, `/repos/${split.repo}/pulls/${split.number}`, "application/vnd.github.v3.diff"); + if (!diffResponse.ok) { + report.noMatch += 1; + state.rawContextResumeFrom = row.target_key; + continue; + } + const patched = patchFiredMetadataWithDiff(row.metadata_json, await diffResponse.text()); + if (patched === null) { + report.alreadyPatched += 1; + } else if (args.apply) { + applyMetadataUpdate(args, backfillFiredId(row.target_key), patched); + report.patched += 1; + } else { + report.patched += 1; + } + state.rawContextResumeFrom = row.target_key; + } + report.requestsUsed = budget.used; + return report; +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + const state = readState(args.stateFile); + const budget = new RequestBudget(args.maxRequests); + + const report = args.pass === "successors" ? await runSuccessorsPass(args, budget, state) : await runRawContextPass(args, budget, state); + writeFileSync(args.stateFile, `${JSON.stringify(state, null, 2)}\n`); + console.log(renderPhase2Report(report, args.apply ? "apply" : "dry-run")); + if (!args.apply) { + console.error("dry-run only — re-run with --apply to write. Patches are idempotent (already-patched rows are skipped)."); + console.error("NOTE: the resume cursor advances in dry-run too — delete the state file before switching to --apply."); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); diff --git a/test/unit/backfill-calibration-corpus-phase2.test.ts b/test/unit/backfill-calibration-corpus-phase2.test.ts new file mode 100644 index 0000000000..bcf5fbf260 --- /dev/null +++ b/test/unit/backfill-calibration-corpus-phase2.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it } from "vitest"; +import { SUPERSEDED_LOOKBACK_MS } from "../../src/review/reversal-superseded"; +import { RAW_CONTEXT_MAX_DIFF_CHARS } from "../../src/rules/advisory"; +import { + backfillFiredId, + backfillOverrideId, + matchRetroSuccessors, + patchFiredMetadataWithDiff, + patchOverrideMetadataToReversed, + renderPhase2Report, + RETRO_SUCCESSOR_PROVENANCE, + RAW_CONTEXT_REFETCH_PROVENANCE, + type HistoricalCloseSide, + type Phase2Report, + type SuccessorSide, +} from "../../scripts/backfill-calibration-corpus-phase2-core"; + +// #8170: the pure core of the phase-2 GitHub-truth backfill. The matching itself is #8166's +// evaluateSuccessorMatch (tested in its own suite); these tests pin the retro windowing, the +// deterministic earliest-successor pick, and both patchers' idempotency. + +const CLOSE_AT = "2026-06-01T00:00:00.000Z"; + +function close(over: Partial = {}): HistoricalCloseSide { + return { + targetKey: "acme/widgets#7", + repo: "acme/widgets", + number: 7, + closedAt: CLOSE_AT, + authorLogin: "alice", + linkedIssues: [42], + files: ["src/a.ts", "src/b.ts"], + ...over, + }; +} + +function successor(over: Partial = {}): SuccessorSide { + return { + number: 9, + mergedAt: "2026-06-02T00:00:00.000Z", + authorLogin: "alice", + linkedIssues: [42], + files: ["src/a.ts", "src/b.ts"], + ...over, + }; +} + +describe("matchRetroSuccessors (#8170)", () => { + it("matches a shared-linked-issue successor inside the lookback window", () => { + const match = matchRetroSuccessors(close(), [successor()]); + expect(match).toMatchObject({ targetKey: "acme/widgets#7", supersededBy: 9 }); + expect(match!.heuristics.sameLinkedIssue).toBe(true); + }); + + it("is directional and bounded: merges before the close, past the lookback, the close itself, and bad timestamps never match", () => { + // Merged BEFORE the close. + expect(matchRetroSuccessors(close(), [successor({ mergedAt: "2026-05-30T00:00:00.000Z" })])).toBeNull(); + // Merged after the 30-day lookback. + const past = new Date(Date.parse(CLOSE_AT) + SUPERSEDED_LOOKBACK_MS + 1).toISOString(); + expect(matchRetroSuccessors(close(), [successor({ mergedAt: past })])).toBeNull(); + // At the boundary exactly: still inside (inclusive). + const boundary = new Date(Date.parse(CLOSE_AT) + SUPERSEDED_LOOKBACK_MS).toISOString(); + expect(matchRetroSuccessors(close(), [successor({ mergedAt: boundary })])).not.toBeNull(); + // The close's own number can never supersede it. + expect(matchRetroSuccessors(close(), [successor({ number: 7 })])).toBeNull(); + // Unparseable timestamps on either side. + expect(matchRetroSuccessors(close({ closedAt: "not-a-date" }), [successor()])).toBeNull(); + expect(matchRetroSuccessors(close(), [successor({ mergedAt: "not-a-date" })])).toBeNull(); + }); + + it("picks the EARLIEST qualifying merge (ties broken by number) so re-runs with more candidates stay stable", () => { + const later = successor({ number: 20, mergedAt: "2026-06-05T00:00:00.000Z" }); + const earlier = successor({ number: 11, mergedAt: "2026-06-03T00:00:00.000Z" }); + const tie = successor({ number: 10, mergedAt: "2026-06-03T00:00:00.000Z" }); + expect(matchRetroSuccessors(close(), [later, earlier, tie])!.supersededBy).toBe(10); + }); + + it("skips non-matching successors and records nothing on a borderline (conservative by #8166's own bar)", () => { + const stranger = successor({ authorLogin: "bob", linkedIssues: [], files: [] }); + expect(matchRetroSuccessors(close({ linkedIssues: [] }), [stranger])).toBeNull(); + // A non-matching earlier successor must not shadow a matching later one. + const matching = successor({ number: 15, mergedAt: "2026-06-09T00:00:00.000Z" }); + expect(matchRetroSuccessors(close(), [stranger, matching])!.supersededBy).toBe(15); + }); +}); + +describe("metadata patchers (#8170)", () => { + const match = { targetKey: "acme/widgets#7", supersededBy: 9, heuristics: { sameLinkedIssue: true, sameAuthorFileOverlap: false, fileOverlapRatio: null } }; + + it("flips a phase-1 confirmed override to reversed with the retro provenance + evidence", () => { + const original = JSON.stringify({ verdict: "confirmed", backfilled: true, provenance: "review_targets_decision_level" }); + const patched = JSON.parse(patchOverrideMetadataToReversed(original, match)!) as Record; + expect(patched.verdict).toBe("reversed"); + expect(patched.backfilled).toBe(true); // phase-1 fields survive + expect(patched.retroLabel).toMatchObject({ provenance: RETRO_SUCCESSOR_PROVENANCE, supersededBy: 9 }); + }); + + it("is idempotent and never guesses: already-reversed and unparseable metadata both return null", () => { + expect(patchOverrideMetadataToReversed(JSON.stringify({ verdict: "reversed" }), match)).toBeNull(); + expect(patchOverrideMetadataToReversed("not-json", match)).toBeNull(); + expect(patchOverrideMetadataToReversed('["array"]', match)).toBeNull(); + }); + + it("patches a fired row with the bounded diff exactly once", () => { + const original = JSON.stringify({ confidence: 0.95, backfilled: true }); + const patched = JSON.parse(patchFiredMetadataWithDiff(original, "diff --git a/x b/x")!) as Record; + expect(patched.diff).toBe("diff --git a/x b/x"); + expect(patched.rawContextProvenance).toBe(RAW_CONTEXT_REFETCH_PROVENANCE); + // Second run: diff present -> null. + expect(patchFiredMetadataWithDiff(JSON.stringify(patched), "different")).toBeNull(); + }); + + it("bounds an oversized diff to the live capture's own cap and refuses empty diffs / bad metadata", () => { + const oversized = "x".repeat(RAW_CONTEXT_MAX_DIFF_CHARS + 5); + const patched = JSON.parse(patchFiredMetadataWithDiff("{}", oversized)!) as { diff: string }; + expect(patched.diff).toHaveLength(RAW_CONTEXT_MAX_DIFF_CHARS); + expect(patchFiredMetadataWithDiff("{}", "")).toBeNull(); + expect(patchFiredMetadataWithDiff("not-json", "diff")).toBeNull(); + }); +}); + +describe("ids + report rendering (#8170)", () => { + it("derives the deterministic phase-1 row ids (the only rows the passes may touch)", () => { + expect(backfillOverrideId("acme/widgets#7")).toBe("backfill:ai_consensus_defect:acme/widgets#7:override"); + expect(backfillFiredId("acme/widgets#7")).toBe("backfill:ai_consensus_defect:acme/widgets#7:fired"); + }); + + it("renders both passes' reports, including the budget-exhausted resumable form", () => { + const base: Phase2Report = { pass: "successors", scanned: 5, patched: 2, alreadyPatched: 1, noMatch: 2, requestsUsed: 42, exhaustedBudget: false, resumeFrom: null }; + const report = renderPhase2Report(base, "dry-run"); + expect(report).toContain(RETRO_SUCCESSOR_PROVENANCE); + expect(report).toContain("scanned: 5"); + const exhausted = renderPhase2Report( + { ...base, pass: "raw-context", exhaustedBudget: true, resumeFrom: "acme/widgets#7" }, + "apply", + ); + expect(exhausted).toContain(RAW_CONTEXT_REFETCH_PROVENANCE); + expect(exhausted).toContain("budget exhausted"); + expect(exhausted).toContain("resume from: acme/widgets#7"); + }); +}); From 2de955773d32d8efd4d1b38115456e1069ff338a Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 03:14:30 -0700 Subject: [PATCH 2/7] fix(calibration): budget-bound the phase-2 successor listing instead of a 10-page cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first full production dry-run returned zero matches because page 10 of the sort=updated listing only reached back to July 5 while the closes under scan ended June 22 — a silently truncated listing is indistinguishable from 'no successors'. Depth is now bounded by the run's request budget (hard page ceiling 200 as a backstop), with the boundary condition unchanged. Advances #8170 --- scripts/backfill-calibration-corpus-phase2.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/backfill-calibration-corpus-phase2.ts b/scripts/backfill-calibration-corpus-phase2.ts index d3b09f14e9..2c2c0a10b4 100644 --- a/scripts/backfill-calibration-corpus-phase2.ts +++ b/scripts/backfill-calibration-corpus-phase2.ts @@ -120,10 +120,13 @@ function pullLinkedIssues(repo: string, pull: GithubPull): number[] { } /** Merged PRs in `repo` whose merge could fall inside any pending close's lookback window. Pages - * sort=updated desc and stops once updated_at (an upper bound on merged_at) predates the oldest close. */ + * sort=updated desc and stops once updated_at (an upper bound on merged_at, so nothing past it can have + * merged inside any window) predates the oldest close. Depth is budget-bound, not page-capped: a busy + * repo's history is deeper than any fixed small cap, and a silently truncated listing looks exactly like + * "no successors" — the first full production dry-run proved that failure mode (#8170). */ async function fetchMergedSuccessors(budget: RequestBudget, repo: string, oldestClosedAtIso: string): Promise { const successors: SuccessorSide[] = []; - for (let page = 1; page <= 10 && !budget.exhausted; page += 1) { + for (let page = 1; page <= 200 && !budget.exhausted; page += 1) { const pulls = await githubJson>( budget, `/repos/${repo}/pulls?state=closed&sort=updated&direction=desc&per_page=100&page=${page}`, From ecdb7ad69da3979450ae293cdbf9b273b3ab38be Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 03:31:12 -0700 Subject: [PATCH 3/7] fix(calibration): time-bound the phase-2 wrapper's wrangler calls so a hung remote execute fails loud Advances #8170 --- scripts/backfill-calibration-corpus-phase2.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/backfill-calibration-corpus-phase2.ts b/scripts/backfill-calibration-corpus-phase2.ts index 2c2c0a10b4..e8cd664980 100644 --- a/scripts/backfill-calibration-corpus-phase2.ts +++ b/scripts/backfill-calibration-corpus-phase2.ts @@ -54,7 +54,10 @@ function d1Execute(db: string, remote: boolean, sql: string): Array Date: Thu, 23 Jul 2026 04:15:19 -0700 Subject: [PATCH 4/7] feat(calibration): per-heuristic breakdown in the pass-A report 295/460 matches on the first full scan is too many to all be reversals: in a duplicate-competition culture a shared-issue match by a different author is usually the gate correctly closing a losing duplicate. The apply decision needs the same-author-rework vs shared-issue-only split. Advances #8170 --- scripts/backfill-calibration-corpus-phase2-core.ts | 8 ++++++++ scripts/backfill-calibration-corpus-phase2.ts | 6 ++++-- test/unit/backfill-calibration-corpus-phase2.test.ts | 4 +++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/scripts/backfill-calibration-corpus-phase2-core.ts b/scripts/backfill-calibration-corpus-phase2-core.ts index 88ca87b6c8..2d93ef89b9 100644 --- a/scripts/backfill-calibration-corpus-phase2-core.ts +++ b/scripts/backfill-calibration-corpus-phase2-core.ts @@ -117,6 +117,11 @@ export type Phase2Report = { patched: number; alreadyPatched: number; noMatch: number; + /** Pass-A heuristic breakdown (#8170's apply decision hinges on it): a SAME-AUTHOR rework merging is + * strong bot-was-wrong evidence; a shared-issue match by a DIFFERENT author is routine duplicate + * competition in this culture — the winner merging does not make closing the loser wrong. */ + matchedSameAuthor: number; + matchedSharedIssueOnly: number; requestsUsed: number; exhaustedBudget: boolean; resumeFrom: string | null; @@ -129,6 +134,9 @@ export function renderPhase2Report(report: Phase2Report, mode: "dry-run" | "appl report.pass === "successors" ? RETRO_SUCCESSOR_PROVENANCE : RAW_CONTEXT_REFETCH_PROVENANCE }`, ` scanned: ${report.scanned} patched: ${report.patched} already-patched: ${report.alreadyPatched} no-match/skipped: ${report.noMatch}`, + ...(report.pass === "successors" + ? [` match heuristics: same-author rework ${report.matchedSameAuthor}, shared-issue-only (different author) ${report.matchedSharedIssueOnly}`] + : []), ` GitHub requests used: ${report.requestsUsed}${report.exhaustedBudget ? " (budget exhausted — resumable)" : ""}`, ]; if (report.resumeFrom) lines.push(` resume from: ${report.resumeFrom} (state file updated)`); diff --git a/scripts/backfill-calibration-corpus-phase2.ts b/scripts/backfill-calibration-corpus-phase2.ts index e8cd664980..68c0d80bda 100644 --- a/scripts/backfill-calibration-corpus-phase2.ts +++ b/scripts/backfill-calibration-corpus-phase2.ts @@ -191,7 +191,7 @@ function splitTargetKey(targetKey: string): { repo: string; number: number } | n } async function runSuccessorsPass(args: Args, budget: RequestBudget, state: CursorState): Promise { - const report: Phase2Report = { pass: "successors", scanned: 0, patched: 0, alreadyPatched: 0, noMatch: 0, requestsUsed: 0, exhaustedBudget: false, resumeFrom: null }; + const report: Phase2Report = { pass: "successors", scanned: 0, patched: 0, alreadyPatched: 0, noMatch: 0, matchedSameAuthor: 0, matchedSharedIssueOnly: 0, requestsUsed: 0, exhaustedBudget: false, resumeFrom: null }; const rows = loadBackfillRows(args, "override").filter((row) => !state.successorsResumeFrom || row.target_key > state.successorsResumeFrom); const byRepo = new Map(); @@ -252,6 +252,8 @@ async function runSuccessorsPass(args: Args, budget: RequestBudget, state: Curso state.successorsResumeFrom = row.target_key; continue; } + if (match.heuristics.sameAuthorFileOverlap) report.matchedSameAuthor += 1; + else report.matchedSharedIssueOnly += 1; const patched = patchOverrideMetadataToReversed(row.metadata_json, match); if (patched === null) { report.alreadyPatched += 1; @@ -269,7 +271,7 @@ async function runSuccessorsPass(args: Args, budget: RequestBudget, state: Curso } async function runRawContextPass(args: Args, budget: RequestBudget, state: CursorState): Promise { - const report: Phase2Report = { pass: "raw-context", scanned: 0, patched: 0, alreadyPatched: 0, noMatch: 0, requestsUsed: 0, exhaustedBudget: false, resumeFrom: null }; + const report: Phase2Report = { pass: "raw-context", scanned: 0, patched: 0, alreadyPatched: 0, noMatch: 0, matchedSameAuthor: 0, matchedSharedIssueOnly: 0, requestsUsed: 0, exhaustedBudget: false, resumeFrom: null }; const rows = loadBackfillRows(args, "fired").filter((row) => !state.rawContextResumeFrom || row.target_key > state.rawContextResumeFrom); const repoPrivacy = new Map(); diff --git a/test/unit/backfill-calibration-corpus-phase2.test.ts b/test/unit/backfill-calibration-corpus-phase2.test.ts index bcf5fbf260..890e132897 100644 --- a/test/unit/backfill-calibration-corpus-phase2.test.ts +++ b/test/unit/backfill-calibration-corpus-phase2.test.ts @@ -126,10 +126,12 @@ describe("ids + report rendering (#8170)", () => { }); it("renders both passes' reports, including the budget-exhausted resumable form", () => { - const base: Phase2Report = { pass: "successors", scanned: 5, patched: 2, alreadyPatched: 1, noMatch: 2, requestsUsed: 42, exhaustedBudget: false, resumeFrom: null }; + const base: Phase2Report = { pass: "successors", scanned: 5, patched: 2, alreadyPatched: 1, noMatch: 2, matchedSameAuthor: 1, matchedSharedIssueOnly: 1, requestsUsed: 42, exhaustedBudget: false, resumeFrom: null }; const report = renderPhase2Report(base, "dry-run"); expect(report).toContain(RETRO_SUCCESSOR_PROVENANCE); expect(report).toContain("scanned: 5"); + // The apply decision hinges on this split (same-author = strong; shared-issue-only = routine duplicate competition). + expect(report).toContain("same-author rework 1, shared-issue-only (different author) 1"); const exhausted = renderPhase2Report( { ...base, pass: "raw-context", exhaustedBudget: true, resumeFrom: "acme/widgets#7" }, "apply", From e89a420f2e90717e24c30365d4a6b3f9bdf9378d Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 04:22:04 -0700 Subject: [PATCH 5/7] =?UTF-8?q?feat(calibration):=20capture=20same-PR=20re?= =?UTF-8?q?opened+merged=20reversals=20in=20pass=20A=20=E2=80=94=20the=20d?= =?UTF-8?q?efinitive=20label=20class?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The operator's correction to the zero-reversals reading: bot/AI-closed PRs HAVE been reopened and merged, but latest-decision-wins erased the earlier close verdicts and the scan skipped merged PRs as 'not a standing close'. A close-verdict PR that itself shows merged_at is a definitive same-PR reversal — no heuristics — labeled under github_same_pr_merged provenance, counted separately from the successor classes in the report. Advances #8170 --- ...backfill-calibration-corpus-phase2-core.ts | 26 ++++++++++++++++++- scripts/backfill-calibration-corpus-phase2.ts | 23 ++++++++++++---- ...backfill-calibration-corpus-phase2.test.ts | 16 ++++++++++-- 3 files changed, 57 insertions(+), 8 deletions(-) diff --git a/scripts/backfill-calibration-corpus-phase2-core.ts b/scripts/backfill-calibration-corpus-phase2-core.ts index 2d93ef89b9..a44e8b6ad3 100644 --- a/scripts/backfill-calibration-corpus-phase2-core.ts +++ b/scripts/backfill-calibration-corpus-phase2-core.ts @@ -15,6 +15,9 @@ import { BACKFILL_RULE_ID } from "./backfill-calibration-corpus-core.js"; /** Distinct provenance for pass A's retro labels — never confusable with phase 1's decision-level rows. */ export const RETRO_SUCCESSOR_PROVENANCE = "github_successor_scan"; +/** Provenance for the strongest retro label: the close-verdict PR ITSELF later merged (the operator + * reopened + merged it) — a definitive same-PR reversal needing no successor heuristics at all. */ +export const RETRO_SAME_PR_MERGED_PROVENANCE = "github_same_pr_merged"; /** Distinct provenance for pass B's re-fetched raw context. */ export const RAW_CONTEXT_REFETCH_PROVENANCE = "github_raw_context_refetch"; @@ -97,6 +100,22 @@ export function patchOverrideMetadataToReversed(metadataJson: string, match: Ret }); } +/** + * Patch a phase-1 override row for the same-PR reversal: GitHub says the close-verdict PR itself MERGED + * (the operator reopened + merged it) — the decision was overridden on its own target, no heuristics + * involved. Same idempotency contract as {@link patchOverrideMetadataToReversed}. + */ +export function patchOverrideMetadataToSamePrMerged(metadataJson: string, mergedAt: string): string | null { + const metadata = parseObject(metadataJson); + if (!metadata) return null; + if (metadata.verdict === "reversed") return null; + return JSON.stringify({ + ...metadata, + verdict: "reversed", + retroLabel: { provenance: RETRO_SAME_PR_MERGED_PROVENANCE, mergedAt }, + }); +} + /** * Patch a phase-1 fired row's metadata with the re-fetched PR diff — the field the live #8130 capture * records for this rule (`metadata.diff`, same bound). Returns null when raw context is already present @@ -122,6 +141,9 @@ export type Phase2Report = { * competition in this culture — the winner merging does not make closing the loser wrong. */ matchedSameAuthor: number; matchedSharedIssueOnly: number; + /** The close-verdict PR itself later merged — definitive reversals, no heuristics (see the operator's + * own reopen-and-merge history; the strongest label class this pass produces). */ + matchedSamePrMerged: number; requestsUsed: number; exhaustedBudget: boolean; resumeFrom: string | null; @@ -135,7 +157,9 @@ export function renderPhase2Report(report: Phase2Report, mode: "dry-run" | "appl }`, ` scanned: ${report.scanned} patched: ${report.patched} already-patched: ${report.alreadyPatched} no-match/skipped: ${report.noMatch}`, ...(report.pass === "successors" - ? [` match heuristics: same-author rework ${report.matchedSameAuthor}, shared-issue-only (different author) ${report.matchedSharedIssueOnly}`] + ? [ + ` match classes: same-PR reopened+merged ${report.matchedSamePrMerged} (definitive), same-author rework ${report.matchedSameAuthor}, shared-issue-only (different author) ${report.matchedSharedIssueOnly}`, + ] : []), ` GitHub requests used: ${report.requestsUsed}${report.exhaustedBudget ? " (budget exhausted — resumable)" : ""}`, ]; diff --git a/scripts/backfill-calibration-corpus-phase2.ts b/scripts/backfill-calibration-corpus-phase2.ts index 68c0d80bda..a248a471b5 100644 --- a/scripts/backfill-calibration-corpus-phase2.ts +++ b/scripts/backfill-calibration-corpus-phase2.ts @@ -21,6 +21,7 @@ import { matchRetroSuccessors, patchFiredMetadataWithDiff, patchOverrideMetadataToReversed, + patchOverrideMetadataToSamePrMerged, renderPhase2Report, type HistoricalCloseSide, type Phase2Report, @@ -191,7 +192,7 @@ function splitTargetKey(targetKey: string): { repo: string; number: number } | n } async function runSuccessorsPass(args: Args, budget: RequestBudget, state: CursorState): Promise { - const report: Phase2Report = { pass: "successors", scanned: 0, patched: 0, alreadyPatched: 0, noMatch: 0, matchedSameAuthor: 0, matchedSharedIssueOnly: 0, requestsUsed: 0, exhaustedBudget: false, resumeFrom: null }; + const report: Phase2Report = { pass: "successors", scanned: 0, patched: 0, alreadyPatched: 0, noMatch: 0, matchedSameAuthor: 0, matchedSharedIssueOnly: 0, matchedSamePrMerged: 0, requestsUsed: 0, exhaustedBudget: false, resumeFrom: null }; const rows = loadBackfillRows(args, "override").filter((row) => !state.successorsResumeFrom || row.target_key > state.successorsResumeFrom); const byRepo = new Map(); @@ -216,9 +217,21 @@ async function runSuccessorsPass(args: Args, budget: RequestBudget, state: Curso const split = splitTargetKey(row.target_key)!; const closedPull = await githubJson(budget, `/repos/${repo}/pulls/${split.number}`); - if (!closedPull || closedPull.merged_at) { - // Gone, or actually merged (not a standing bot-close) — not a reversal candidate. - report.noMatch += 1; + if (!closedPull) { + report.noMatch += 1; // deleted repo/PR — never guess + state.successorsResumeFrom = row.target_key; + continue; + } + if (closedPull.merged_at) { + // The close-verdict PR ITSELF merged: the operator reopened + merged it — a definitive same-PR + // reversal, the strongest label class this pass produces (no heuristics involved). + report.matchedSamePrMerged += 1; + const samePrPatched = patchOverrideMetadataToSamePrMerged(row.metadata_json, closedPull.merged_at); + if (samePrPatched === null) report.alreadyPatched += 1; + else if (args.apply) { + applyMetadataUpdate(args, backfillOverrideId(row.target_key), samePrPatched); + report.patched += 1; + } else report.patched += 1; state.successorsResumeFrom = row.target_key; continue; } @@ -271,7 +284,7 @@ async function runSuccessorsPass(args: Args, budget: RequestBudget, state: Curso } async function runRawContextPass(args: Args, budget: RequestBudget, state: CursorState): Promise { - const report: Phase2Report = { pass: "raw-context", scanned: 0, patched: 0, alreadyPatched: 0, noMatch: 0, matchedSameAuthor: 0, matchedSharedIssueOnly: 0, requestsUsed: 0, exhaustedBudget: false, resumeFrom: null }; + const report: Phase2Report = { pass: "raw-context", scanned: 0, patched: 0, alreadyPatched: 0, noMatch: 0, matchedSameAuthor: 0, matchedSharedIssueOnly: 0, matchedSamePrMerged: 0, requestsUsed: 0, exhaustedBudget: false, resumeFrom: null }; const rows = loadBackfillRows(args, "fired").filter((row) => !state.rawContextResumeFrom || row.target_key > state.rawContextResumeFrom); const repoPrivacy = new Map(); diff --git a/test/unit/backfill-calibration-corpus-phase2.test.ts b/test/unit/backfill-calibration-corpus-phase2.test.ts index 890e132897..4b1ca70197 100644 --- a/test/unit/backfill-calibration-corpus-phase2.test.ts +++ b/test/unit/backfill-calibration-corpus-phase2.test.ts @@ -7,8 +7,10 @@ import { matchRetroSuccessors, patchFiredMetadataWithDiff, patchOverrideMetadataToReversed, + patchOverrideMetadataToSamePrMerged, renderPhase2Report, RETRO_SUCCESSOR_PROVENANCE, + RETRO_SAME_PR_MERGED_PROVENANCE, RAW_CONTEXT_REFETCH_PROVENANCE, type HistoricalCloseSide, type Phase2Report, @@ -101,6 +103,16 @@ describe("metadata patchers (#8170)", () => { expect(patchOverrideMetadataToReversed('["array"]', match)).toBeNull(); }); + it("labels a same-PR reopened+merged decision reversed with its own provenance — the definitive class", () => { + const original = JSON.stringify({ verdict: "confirmed", backfilled: true }); + const patched = JSON.parse(patchOverrideMetadataToSamePrMerged(original, "2026-07-05T00:00:00.000Z")!) as Record; + expect(patched.verdict).toBe("reversed"); + expect(patched.retroLabel).toEqual({ provenance: RETRO_SAME_PR_MERGED_PROVENANCE, mergedAt: "2026-07-05T00:00:00.000Z" }); + // Same idempotency + never-guess contract as the successor patcher. + expect(patchOverrideMetadataToSamePrMerged(JSON.stringify(patched), "later")).toBeNull(); + expect(patchOverrideMetadataToSamePrMerged("not-json", "t")).toBeNull(); + }); + it("patches a fired row with the bounded diff exactly once", () => { const original = JSON.stringify({ confidence: 0.95, backfilled: true }); const patched = JSON.parse(patchFiredMetadataWithDiff(original, "diff --git a/x b/x")!) as Record; @@ -126,12 +138,12 @@ describe("ids + report rendering (#8170)", () => { }); it("renders both passes' reports, including the budget-exhausted resumable form", () => { - const base: Phase2Report = { pass: "successors", scanned: 5, patched: 2, alreadyPatched: 1, noMatch: 2, matchedSameAuthor: 1, matchedSharedIssueOnly: 1, requestsUsed: 42, exhaustedBudget: false, resumeFrom: null }; + const base: Phase2Report = { pass: "successors", scanned: 5, patched: 2, alreadyPatched: 1, noMatch: 2, matchedSameAuthor: 1, matchedSharedIssueOnly: 1, matchedSamePrMerged: 1, requestsUsed: 42, exhaustedBudget: false, resumeFrom: null }; const report = renderPhase2Report(base, "dry-run"); expect(report).toContain(RETRO_SUCCESSOR_PROVENANCE); expect(report).toContain("scanned: 5"); // The apply decision hinges on this split (same-author = strong; shared-issue-only = routine duplicate competition). - expect(report).toContain("same-author rework 1, shared-issue-only (different author) 1"); + expect(report).toContain("same-PR reopened+merged 1 (definitive), same-author rework 1, shared-issue-only (different author) 1"); const exhausted = renderPhase2Report( { ...base, pass: "raw-context", exhaustedBudget: true, resumeFrom: "acme/widgets#7" }, "apply", From 541fa6aea0663e551298a71907eb7bf5414e721a Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 05:32:59 -0700 Subject: [PATCH 6/7] feat(calibration): plan-file applies, class-gated policy, and burst-limit resilience for phase 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Production hardening from running the passes for real: - Scan-once/apply-from-plan: the dry-run emits its matches as a plan file (persisted even on mid-run abort), and --apply --plan-in replays it against any store with ZERO GitHub requests — the cloud D1 and selfhost Postgres applies share one scan instead of re-spending ~2.7k requests each. --pg rides #8171's driver seam for the store side. - Apply policy is class-gated: same_pr_merged + same_author apply by default; shared_issue_only (128 of the 301 production matches — routine duplicate competition, not reversal evidence) is counted and planned but never applied without an explicit --include-shared-issue-only. - GitHub IO survives reality: bounded retries on thrown fetches/5xx, Retry-After honored on 403/429 burst limits (90s default, 5m cap) with ~4 req/s pacing so a stall costs minutes instead of the whole scan, and the wrangler reads are time-bounded so a hung remote execute fails loud. Advances #8170 --- scripts/backfill-calibration-corpus-phase2.ts | 169 +++++++++++++++--- 1 file changed, 149 insertions(+), 20 deletions(-) diff --git a/scripts/backfill-calibration-corpus-phase2.ts b/scripts/backfill-calibration-corpus-phase2.ts index a248a471b5..99b6e3b887 100644 --- a/scripts/backfill-calibration-corpus-phase2.ts +++ b/scripts/backfill-calibration-corpus-phase2.ts @@ -5,6 +5,7 @@ // // tsx scripts/backfill-calibration-corpus-phase2.ts --pass successors --db loopover [--remote] [--apply] // tsx scripts/backfill-calibration-corpus-phase2.ts --pass raw-context --db loopover [--remote] [--apply] +// … --pg postgres://… runs against a self-host Postgres instead (#8171's driver; bare --pg uses DATABASE_URL) // // Both passes are dry-run by default, resumable (--state-file, default .backfill-phase2-state.json — the // cursor survives budget exhaustion), and hard-capped on GitHub requests per run (--max-requests, default @@ -14,6 +15,7 @@ // deterministic `backfill:` prefix are ever touched — live capture rows are out of reach by construction. import { readFileSync, writeFileSync } from "node:fs"; import { spawnSync } from "node:child_process"; +import { openPgDatabase, resolvePgConnection, type PgCliSession } from "./pg-cli.js"; import { extractLinkedIssueNumbers } from "../src/db/repositories.js"; import { backfillFiredId, @@ -30,21 +32,42 @@ import { import { BACKFILL_RULE_ID } from "./backfill-calibration-corpus-core.js"; type Pass = "successors" | "raw-context"; -type Args = { db: string; remote: boolean; apply: boolean; pass: Pass; maxRequests: number; stateFile: string }; +type Args = { + db: string; + remote: boolean; + apply: boolean; + pass: Pass; + maxRequests: number; + stateFile: string; + pgPresent: boolean; + pgValue: string | undefined; + /** Opt-in for the weak class: shared-issue matches by a DIFFERENT author are routine duplicate + * competition in this culture, NOT bot-was-wrong evidence — excluded from apply unless forced. */ + includeSharedIssueOnly: boolean; + planOut: string | undefined; + planIn: string | undefined; +}; function parseArgs(argv: string[]): Args { - const args: Args = { db: "loopover", remote: false, apply: false, pass: "successors", maxRequests: 300, stateFile: ".backfill-phase2-state.json" }; + const args: Args = { db: "loopover", remote: false, apply: false, pass: "successors", maxRequests: 300, stateFile: ".backfill-phase2-state.json", pgPresent: false, pgValue: undefined, includeSharedIssueOnly: false, planOut: undefined, planIn: undefined }; for (let i = 0; i < argv.length; i += 1) { const flag = argv[i]; if (flag === "--remote") args.remote = true; else if (flag === "--apply") args.apply = true; else if (flag === "--db") args.db = argv[++i]!; + else if (flag === "--pg") { + args.pgPresent = true; + if (argv[i + 1] !== undefined && !argv[i + 1]!.startsWith("--")) args.pgValue = argv[++i]; + } else if (flag === "--pass") { const value = argv[++i]; if (value !== "successors" && value !== "raw-context") throw new Error(`--pass must be successors or raw-context, got ${value}`); args.pass = value; } else if (flag === "--max-requests") args.maxRequests = Number(argv[++i]); else if (flag === "--state-file") args.stateFile = argv[++i]!; + else if (flag === "--include-shared-issue-only") args.includeSharedIssueOnly = true; + else if (flag === "--plan-out") args.planOut = argv[++i]; + else if (flag === "--plan-in") args.planIn = argv[++i]; } if (!Number.isFinite(args.maxRequests) || args.maxRequests < 1) throw new Error("--max-requests must be a positive number"); return args; @@ -89,11 +112,48 @@ class RequestBudget { async function githubFetch(budget: RequestBudget, path: string, accept = "application/vnd.github+json"): Promise { const token = process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN; if (!token) throw new Error("GITHUB_TOKEN (or GH_TOKEN) is required for the GitHub-truth passes"); - budget.spend(); - return fetch(`https://api.github.com${path}`, { - headers: { authorization: `Bearer ${token}`, accept, "user-agent": "loopover-backfill-phase2" }, - signal: AbortSignal.timeout(GITHUB_TIMEOUT_MS), - }); + // Two hard-won lessons baked in (#8170's production runs): + // • a single transient network blip must not kill a multi-thousand-request pass ('fetch failed' + // at ~90% through a scan) — bounded retries on thrown fetches and 5xx; + // • GitHub's SECONDARY (burst) limit 403s must be WAITED OUT, not fatal: pace requests to a floor + // interval, and on 403/429 honor Retry-After (default 90s, cap 5 min) before retrying. A paced + // stall costs minutes; a dead run costs the whole scan plus the budget it already spent. + let lastError: unknown; + for (let attempt = 1; attempt <= 5; attempt += 1) { + await pace(); + budget.spend(); + try { + const response = await fetch(`https://api.github.com${path}`, { + headers: { authorization: `Bearer ${token}`, accept, "user-agent": "loopover-backfill-phase2" }, + signal: AbortSignal.timeout(GITHUB_TIMEOUT_MS), + }); + if ((response.status === 403 || response.status === 429) && attempt < 5) { + const retryAfter = Number(response.headers.get("retry-after")); + const waitMs = Math.min(Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter * 1000 : 90_000, 300_000); + console.error(`GitHub ${response.status} on ${path} — waiting ${Math.round(waitMs / 1000)}s for the burst limit (attempt ${attempt}/5)`); + await new Promise((resolve) => setTimeout(resolve, waitMs)); + continue; + } + if (response.status >= 500 && attempt < 5) { + await new Promise((resolve) => setTimeout(resolve, attempt * 2000)); + continue; + } + return response; + } catch (error) { + lastError = error; + if (attempt < 5) await new Promise((resolve) => setTimeout(resolve, attempt * 2000)); + } + } + throw lastError instanceof Error ? lastError : new Error(String(lastError)); +} + +// ~4 req/s ceiling: sequential scans at full speed are what tripped the burst limit across hours. +const REQUEST_FLOOR_MS = 250; +let lastRequestAt = 0; +async function pace(): Promise { + const wait = lastRequestAt + REQUEST_FLOOR_MS - Date.now(); + if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait)); + lastRequestAt = Date.now(); } async function githubJson(budget: RequestBudget, path: string): Promise { @@ -168,10 +228,18 @@ function readState(path: string): CursorState { type BackfillRow = { id: string; target_key: string; metadata_json: string; created_at: string }; -function loadBackfillRows(args: Args, kind: "override" | "fired"): BackfillRow[] { - const rows = d1Execute( - args.db, - args.remote, +// #8171's driver seam: when --pg selected a connection, every read/write below rides the selfhost adapter +// (same dialect translation as the deployed engine); otherwise the wrangler/D1 path is unchanged. +let pgSession: PgCliSession | null = null; + +async function executeSql(args: Args, sql: string): Promise>> { + if (pgSession) return (await pgSession.db.prepare(sql).all>()).results ?? []; + return d1Execute(args.db, args.remote, sql); +} + +async function loadBackfillRows(args: Args, kind: "override" | "fired"): Promise { + const rows = await executeSql( + args, `SELECT id, target_key, metadata_json, created_at FROM audit_events WHERE id LIKE 'backfill:${BACKFILL_RULE_ID}:%:${kind}' ORDER BY target_key`, ); return rows.filter( @@ -180,8 +248,8 @@ function loadBackfillRows(args: Args, kind: "override" | "fired"): BackfillRow[] ); } -function applyMetadataUpdate(args: Args, id: string, metadataJson: string): void { - d1Execute(args.db, args.remote, `UPDATE audit_events SET metadata_json = ${sqlStringLiteral(metadataJson)} WHERE id = ${sqlStringLiteral(id)}`); +async function applyMetadataUpdate(args: Args, id: string, metadataJson: string): Promise { + await executeSql(args, `UPDATE audit_events SET metadata_json = ${sqlStringLiteral(metadataJson)} WHERE id = ${sqlStringLiteral(id)}`); } function splitTargetKey(targetKey: string): { repo: string; number: number } | null { @@ -191,9 +259,49 @@ function splitTargetKey(targetKey: string): { repo: string; number: number } | n return Number.isFinite(number) ? { repo: targetKey.slice(0, hash), number } : null; } +type PlanEntry = + | { class: "same_pr_merged"; targetKey: string; mergedAt: string } + | { class: "same_author" | "shared_issue_only"; targetKey: string; supersededBy: number; heuristics: RetroSuccessorMatch["heuristics"] }; + +/** Replay a previously scanned plan against the selected store — zero GitHub requests. The plan is the + * dry-run's own match output, so cloud and self-host stores (same seeded targets, same deterministic ids) + * apply identically without re-spending the API budget. */ +async function runSuccessorsFromPlan(args: Args): Promise { + const report: Phase2Report = { pass: "successors", scanned: 0, patched: 0, alreadyPatched: 0, noMatch: 0, matchedSameAuthor: 0, matchedSharedIssueOnly: 0, matchedSamePrMerged: 0, requestsUsed: 0, exhaustedBudget: false, resumeFrom: null }; + const plan = JSON.parse(readFileSync(args.planIn!, "utf8")) as PlanEntry[]; + const rowsById = new Map((await loadBackfillRows(args, "override")).map((row) => [row.id, row])); + for (const entry of plan) { + report.scanned += 1; + const row = rowsById.get(backfillOverrideId(entry.targetKey)); + if (!row) { + report.noMatch += 1; + continue; + } + let patched: string | null; + if (entry.class === "same_pr_merged") { + report.matchedSamePrMerged += 1; + patched = patchOverrideMetadataToSamePrMerged(row.metadata_json, entry.mergedAt); + } else { + if (entry.class === "same_author") report.matchedSameAuthor += 1; + else { + report.matchedSharedIssueOnly += 1; + if (!args.includeSharedIssueOnly) continue; // counted, never applied without the explicit opt-in + } + patched = patchOverrideMetadataToReversed(row.metadata_json, { targetKey: entry.targetKey, supersededBy: entry.supersededBy, heuristics: entry.heuristics }); + } + if (patched === null) report.alreadyPatched += 1; + else if (args.apply) { + await applyMetadataUpdate(args, backfillOverrideId(entry.targetKey), patched); + report.patched += 1; + } else report.patched += 1; + } + return report; +} + async function runSuccessorsPass(args: Args, budget: RequestBudget, state: CursorState): Promise { const report: Phase2Report = { pass: "successors", scanned: 0, patched: 0, alreadyPatched: 0, noMatch: 0, matchedSameAuthor: 0, matchedSharedIssueOnly: 0, matchedSamePrMerged: 0, requestsUsed: 0, exhaustedBudget: false, resumeFrom: null }; - const rows = loadBackfillRows(args, "override").filter((row) => !state.successorsResumeFrom || row.target_key > state.successorsResumeFrom); + const rows = (await loadBackfillRows(args, "override")).filter((row) => !state.successorsResumeFrom || row.target_key > state.successorsResumeFrom); + const plan: PlanEntry[] = []; const byRepo = new Map(); for (const row of rows) { @@ -202,6 +310,7 @@ async function runSuccessorsPass(args: Args, budget: RequestBudget, state: Curso (byRepo.get(split.repo) ?? byRepo.set(split.repo, []).get(split.repo)!).push(row); } + try { outer: for (const [repo, repoRows] of byRepo) { const oldestClosedAt = repoRows.reduce((min, row) => (row.created_at < min ? row.created_at : min), repoRows[0]!.created_at); const successors = await fetchMergedSuccessors(budget, repo, oldestClosedAt); @@ -226,10 +335,11 @@ async function runSuccessorsPass(args: Args, budget: RequestBudget, state: Curso // The close-verdict PR ITSELF merged: the operator reopened + merged it — a definitive same-PR // reversal, the strongest label class this pass produces (no heuristics involved). report.matchedSamePrMerged += 1; + plan.push({ class: "same_pr_merged", targetKey: row.target_key, mergedAt: closedPull.merged_at }); const samePrPatched = patchOverrideMetadataToSamePrMerged(row.metadata_json, closedPull.merged_at); if (samePrPatched === null) report.alreadyPatched += 1; else if (args.apply) { - applyMetadataUpdate(args, backfillOverrideId(row.target_key), samePrPatched); + await applyMetadataUpdate(args, backfillOverrideId(row.target_key), samePrPatched); report.patched += 1; } else report.patched += 1; state.successorsResumeFrom = row.target_key; @@ -265,13 +375,19 @@ async function runSuccessorsPass(args: Args, budget: RequestBudget, state: Curso state.successorsResumeFrom = row.target_key; continue; } - if (match.heuristics.sameAuthorFileOverlap) report.matchedSameAuthor += 1; + const matchClass = match.heuristics.sameAuthorFileOverlap ? "same_author" : "shared_issue_only"; + if (matchClass === "same_author") report.matchedSameAuthor += 1; else report.matchedSharedIssueOnly += 1; + plan.push({ class: matchClass, targetKey: row.target_key, supersededBy: match.supersededBy, heuristics: match.heuristics }); + if (matchClass === "shared_issue_only" && !args.includeSharedIssueOnly && args.apply) { + state.successorsResumeFrom = row.target_key; + continue; // counted + planned for the record, never applied without the explicit opt-in + } const patched = patchOverrideMetadataToReversed(row.metadata_json, match); if (patched === null) { report.alreadyPatched += 1; } else if (args.apply) { - applyMetadataUpdate(args, backfillOverrideId(row.target_key), patched); + await applyMetadataUpdate(args, backfillOverrideId(row.target_key), patched); report.patched += 1; } else { report.patched += 1; // dry-run: counted as "would patch" @@ -279,13 +395,18 @@ async function runSuccessorsPass(args: Args, budget: RequestBudget, state: Curso state.successorsResumeFrom = row.target_key; } } + } finally { + // Persisted even when a fetch ultimately fails mid-pass: the plan-so-far plus the cursor make the + // next run a cheap resume instead of a from-scratch re-scan. + if (args.planOut) writeFileSync(args.planOut, `${JSON.stringify(plan, null, 2)}\n`); + } report.requestsUsed = budget.used; return report; } async function runRawContextPass(args: Args, budget: RequestBudget, state: CursorState): Promise { const report: Phase2Report = { pass: "raw-context", scanned: 0, patched: 0, alreadyPatched: 0, noMatch: 0, matchedSameAuthor: 0, matchedSharedIssueOnly: 0, matchedSamePrMerged: 0, requestsUsed: 0, exhaustedBudget: false, resumeFrom: null }; - const rows = loadBackfillRows(args, "fired").filter((row) => !state.rawContextResumeFrom || row.target_key > state.rawContextResumeFrom); + const rows = (await loadBackfillRows(args, "fired")).filter((row) => !state.rawContextResumeFrom || row.target_key > state.rawContextResumeFrom); const repoPrivacy = new Map(); for (const row of rows) { @@ -321,7 +442,7 @@ async function runRawContextPass(args: Args, budget: RequestBudget, state: Curso if (patched === null) { report.alreadyPatched += 1; } else if (args.apply) { - applyMetadataUpdate(args, backfillFiredId(row.target_key), patched); + await applyMetadataUpdate(args, backfillFiredId(row.target_key), patched); report.patched += 1; } else { report.patched += 1; @@ -334,10 +455,18 @@ async function runRawContextPass(args: Args, budget: RequestBudget, state: Curso async function main(): Promise { const args = parseArgs(process.argv.slice(2)); + const pgConnection = resolvePgConnection(args.pgPresent, args.pgValue, process.env.DATABASE_URL); + if (pgConnection) pgSession = openPgDatabase(pgConnection); const state = readState(args.stateFile); const budget = new RequestBudget(args.maxRequests); - const report = args.pass === "successors" ? await runSuccessorsPass(args, budget, state) : await runRawContextPass(args, budget, state); + const report = + args.pass === "successors" + ? args.planIn + ? await runSuccessorsFromPlan(args) + : await runSuccessorsPass(args, budget, state) + : await runRawContextPass(args, budget, state); + await pgSession?.close(); writeFileSync(args.stateFile, `${JSON.stringify(state, null, 2)}\n`); console.log(renderPhase2Report(report, args.apply ? "apply" : "dry-run")); if (!args.apply) { From b894d6d0e7b86e6a6725ffd3c9fe18e8d65a07d7 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 05:33:37 -0700 Subject: [PATCH 7/7] =?UTF-8?q?feat(calibration):=20default=20the=20phase-?= =?UTF-8?q?2=20scan=20to=20~2.4k=20req/hr=20=E2=80=94=20the=20operator's?= =?UTF-8?q?=20token=20pool=20is=20shared?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Advances #8170 --- scripts/backfill-calibration-corpus-phase2.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/backfill-calibration-corpus-phase2.ts b/scripts/backfill-calibration-corpus-phase2.ts index 99b6e3b887..32d885b7a1 100644 --- a/scripts/backfill-calibration-corpus-phase2.ts +++ b/scripts/backfill-calibration-corpus-phase2.ts @@ -147,8 +147,10 @@ async function githubFetch(budget: RequestBudget, path: string, accept = "applic throw lastError instanceof Error ? lastError : new Error(String(lastError)); } -// ~4 req/s ceiling: sequential scans at full speed are what tripped the burst limit across hours. -const REQUEST_FLOOR_MS = 250; +// Default ~0.66 req/s (~2,400/hr): the scan shares the operator's PERSONAL token pool with everything +// else they run, so it deliberately stays under half the primary limit and far from the burst +// heuristics. Override per run with BACKFILL_REQUEST_FLOOR_MS when the pool is otherwise idle. +const REQUEST_FLOOR_MS = Math.max(Number(process.env.BACKFILL_REQUEST_FLOOR_MS) || 1500, 100); let lastRequestAt = 0; async function pace(): Promise { const wait = lastRequestAt + REQUEST_FLOOR_MS - Date.now();