From c59648c9b960fb0654ba28e51165ff095acc7ffa Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 26 Jul 2026 01:56:07 -0700 Subject: [PATCH 1/3] fix(orb): export, ingest, and score the superseded reversal in fleet calibration The fleet pipeline silently dropped reversal_superseded (#8166) at every hop: the self-host exporter's rev CTE only matched reverted/reopened, the collector's ingest whitelist downgraded an unknown flag to 'none', and orb_signals' CHECK constraint would have rejected the value anyway (swallowed by the best-effort insert). Since supersession is the one-shot culture's dominant real reversal shape, the fleet's published reversalRate stayed pinned at 0 and the homepage's reversal-grounded decision accuracy read a degenerate 100%. Advances #8820 (the accuracy-number half; the reuse-rate tile is a separate change). - orb-collector: rev CTE + flag mapping carry 'superseded' (priority reverted > reopened > superseded), regression-tested for the reversal-recorded-after-first-export re-export path - ingest: whitelist 'superseded' - analytics: a superseded close disconfirms closePrecision and counts toward reversalRate exactly like a reopen - migration 0176: rebuild orb_signals with the widened CHECK --- .../0176_orb_signals_superseded_reversal.sql | 37 +++++++++++++++++++ src/orb/analytics.ts | 6 ++- src/orb/ingest.ts | 2 +- src/selfhost/orb-collector.ts | 14 +++++-- test/integration/orb-ingest.test.ts | 4 ++ test/unit/orb-analytics.test.ts | 11 ++++++ test/unit/selfhost-orb-collector.test.ts | 30 +++++++++++++++ 7 files changed, 97 insertions(+), 7 deletions(-) create mode 100644 migrations/0176_orb_signals_superseded_reversal.sql diff --git a/migrations/0176_orb_signals_superseded_reversal.sql b/migrations/0176_orb_signals_superseded_reversal.sql new file mode 100644 index 0000000000..45c3770675 --- /dev/null +++ b/migrations/0176_orb_signals_superseded_reversal.sql @@ -0,0 +1,37 @@ +-- #8820: admit the successor-merge reversal (#8166's reversal_superseded) into the fleet-calibration signal. +-- +-- The exporter/ingest whitelist now carries reversal_flag='superseded', but orb_signals' CHECK constraint +-- (0060) still pins the column to ('none','reopened','reverted') — the ingest's INSERT OR REPLACE would hit +-- the constraint and its best-effort catch would SILENTLY skip the row, so the fleet's published +-- reversalRate stayed pinned at 0 no matter how many supersessions the instances detected. SQLite can't +-- alter a CHECK, so rebuild the table with the widened constraint, preserving existing rows (they are +-- continuously re-exported telemetry, but keeping them avoids a multi-day fleet-metrics blackout while +-- instances re-fill). + +CREATE TABLE orb_signals_new ( + id INTEGER PRIMARY KEY, + instance_id TEXT NOT NULL, -- SHA256(ORB_APP_ID) prefix; one-way, no PII + repo_hash TEXT NOT NULL, -- HMAC(repo, instance secret); collector can't reverse + pr_hash TEXT NOT NULL, -- HMAC(repo#pr, instance secret) + gate_verdict TEXT, -- the prediction: 'merge' | 'close' | 'hold' + outcome TEXT NOT NULL CHECK (outcome IN ('merged', 'closed')), -- realized ground truth + reversal_flag TEXT NOT NULL DEFAULT 'none' CHECK (reversal_flag IN ('none', 'reopened', 'reverted', 'superseded')), + gate_reasoncode_bucket TEXT, -- low-cardinality category, bucketed at source + time_to_close_ms INTEGER, -- decision -> close cycle time (nullable) + decision_timestamp TEXT, -- when the gate decided + outcome_timestamp TEXT, -- when the PR resolved + sent_at TEXT, + received_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE (instance_id, repo_hash, pr_hash) -- dedup unit: one row per PR per instance, upserted +); + +INSERT INTO orb_signals_new (id, instance_id, repo_hash, pr_hash, gate_verdict, outcome, reversal_flag, gate_reasoncode_bucket, time_to_close_ms, decision_timestamp, outcome_timestamp, sent_at, received_at) + SELECT id, instance_id, repo_hash, pr_hash, gate_verdict, outcome, reversal_flag, gate_reasoncode_bucket, time_to_close_ms, decision_timestamp, outcome_timestamp, sent_at, received_at + FROM orb_signals; + +DROP TABLE orb_signals; +ALTER TABLE orb_signals_new RENAME TO orb_signals; + +-- Recreate the indexes the rename does not carry over (same shapes as 0060). +CREATE INDEX IF NOT EXISTS orb_signals_calibration ON orb_signals (instance_id, gate_verdict, outcome, reversal_flag); +CREATE INDEX IF NOT EXISTS orb_signals_instance ON orb_signals (instance_id, received_at); diff --git a/src/orb/analytics.ts b/src/orb/analytics.ts index 3813d9e37b..a2daf731cd 100644 --- a/src/orb/analytics.ts +++ b/src/orb/analytics.ts @@ -107,7 +107,9 @@ export function percentile(sorted: number[], p: number): number | null { } /** Fold the confusion-matrix cells for one instance into accuracy metrics (reversals count as the gate - * being wrong: a reverted merge is a false positive; a reopened close is a false negative). + * being wrong: a reverted merge is a false positive; a reopened OR superseded close is a false negative — + * `superseded` (#8166) is the one-shot culture's dominant "bot was wrong" shape: the closed PR's work later + * merged via a successor PR, so the close is disconfirmed exactly like a literal reopen). * * Exported for the federated bundle export (#1970, src/orb/federated-bundle.ts): a bundle publishes this * instance's own precision for #6481 to compare against the peer median computed here, so both sides MUST use @@ -126,7 +128,7 @@ export function foldInstance(instanceId: string, cells: Cell[]): InstanceMetrics else mergeFalse += c.n; } else if (c.verdict === "close") { wouldClose += c.n; - if (c.outcome === "closed" && c.reversal_flag !== "reopened") closeConfirmed += c.n; + if (c.outcome === "closed" && c.reversal_flag !== "reopened" && c.reversal_flag !== "superseded") closeConfirmed += c.n; else closeFalse += c.n; } } diff --git a/src/orb/ingest.ts b/src/orb/ingest.ts index cafc338a84..a17eec5db8 100644 --- a/src/orb/ingest.ts +++ b/src/orb/ingest.ts @@ -9,7 +9,7 @@ const MAX_HASH_CHARS = 128; const MAX_BUCKET_CHARS = 64; const MAX_VERDICT_CHARS = 32; const VALID_OUTCOMES = new Set(["merged", "closed"]); -const VALID_REVERSALS = new Set(["none", "reopened", "reverted"]); +const VALID_REVERSALS = new Set(["none", "reopened", "reverted", "superseded"]); const MIN_CYCLE_MS = 1_000; // <1s is implausible const MAX_CYCLE_MS = 31_536_000_000; // >1y is implausible diff --git a/src/selfhost/orb-collector.ts b/src/selfhost/orb-collector.ts index e8c42f7774..3720f4f65d 100644 --- a/src/selfhost/orb-collector.ts +++ b/src/selfhost/orb-collector.ts @@ -33,6 +33,7 @@ interface FleetRow { outcome_at: string; reverted: number; // 0|1 reopened: number; // 0|1 + superseded: number; // 0|1 event_at: string; // max(outcome_at, latest reversal time) — the export watermark unit } @@ -41,7 +42,7 @@ interface FleetEvent { pr_hash: string; gate_verdict: string | null; outcome: string; - reversal_flag: "none" | "reopened" | "reverted"; + reversal_flag: "none" | "reopened" | "reverted" | "superseded"; gate_reasoncode_bucket: string; time_to_close_ms: number | null; decision_timestamp: string | null; @@ -125,16 +126,18 @@ const FLEET_QUERY = ` SELECT target_id, MAX(CASE WHEN event_type = 'reversal_reverted' THEN 1 ELSE 0 END) AS reverted, MAX(CASE WHEN event_type = 'reversal_reopened' THEN 1 ELSE 0 END) AS reopened, + MAX(CASE WHEN event_type = 'reversal_superseded' THEN 1 ELSE 0 END) AS superseded, MAX(created_at) AS rev_at FROM review_audit - WHERE event_type IN ('reversal_reverted', 'reversal_reopened') + WHERE event_type IN ('reversal_reverted', 'reversal_reopened', 'reversal_superseded') GROUP BY target_id ) - SELECT project, target_id, verdict, reasoncode, decided_at, outcome, outcome_at, reverted, reopened, event_at + SELECT project, target_id, verdict, reasoncode, decided_at, outcome, outcome_at, reverted, reopened, superseded, event_at FROM ( SELECT gd.project AS project, gd.target_id AS target_id, gd.verdict AS verdict, gd.reasoncode AS reasoncode, gd.decided_at AS decided_at, po.outcome AS outcome, po.outcome_at AS outcome_at, COALESCE(rev.reverted, 0) AS reverted, COALESCE(rev.reopened, 0) AS reopened, + COALESCE(rev.superseded, 0) AS superseded, CASE WHEN rev.rev_at IS NOT NULL AND rev.rev_at > po.outcome_at THEN rev.rev_at ELSE po.outcome_at END AS event_at FROM gd JOIN po ON gd.target_id = po.target_id @@ -202,7 +205,10 @@ export async function exportOrbBatch(db: D1Database, batchSize = 200, fetchFn: t pr_hash: anonymize ? hmacAnonymize(r.target_id, secret) : r.target_id, gate_verdict: r.verdict, outcome: r.outcome, - reversal_flag: r.reverted ? "reverted" : r.reopened ? "reopened" : "none", + // Priority mirrors signal strength: an explicit revert PR beats a reopen beats the successor-merge + // heuristic (#8166's reversal_superseded — the one-shot culture's dominant real "bot was wrong" shape, + // which this export previously DROPPED entirely, silently pinning the fleet's reversalRate at 0). + reversal_flag: r.reverted ? "reverted" : r.reopened ? "reopened" : r.superseded ? "superseded" : "none", gate_reasoncode_bucket: bucketReasonCode(r.reasoncode), time_to_close_ms: cycleTimeMs(r.decided_at, r.outcome_at), decision_timestamp: r.decided_at, diff --git a/test/integration/orb-ingest.test.ts b/test/integration/orb-ingest.test.ts index ae8440d9ce..33d25f3f47 100644 --- a/test/integration/orb-ingest.test.ts +++ b/test/integration/orb-ingest.test.ts @@ -53,10 +53,14 @@ describe("handleOrbIngest()", () => { ev({ pr_hash: "r1", reversal_flag: "reverted" }), ev({ pr_hash: "r2", reversal_flag: "bogus" }), ev({ pr_hash: "r3" }), + // #8820: the successor-merge reversal (#8166) — previously rejected by the whitelist, silently + // downgraded to 'none' and pinning the fleet's published reversalRate at 0. + ev({ pr_hash: "r4", reversal_flag: "superseded" }), ]); expect(await col(db, "r1", "reversal_flag")).toBe("reverted"); expect(await col(db, "r2", "reversal_flag")).toBe("none"); expect(await col(db, "r3", "reversal_flag")).toBe("none"); + expect(await col(db, "r4", "reversal_flag")).toBe("superseded"); }); it("stores gate_reasoncode_bucket string vs null", async () => { diff --git a/test/unit/orb-analytics.test.ts b/test/unit/orb-analytics.test.ts index ad3d5ca5de..96db4d1df7 100644 --- a/test/unit/orb-analytics.test.ts +++ b/test/unit/orb-analytics.test.ts @@ -87,6 +87,17 @@ describe("computeFleetAnalytics()", () => { expect(inst.fnRate).toBeCloseTo(1 / 5); }); + it("a superseded close (#8820) disconfirms closePrecision and counts toward reversalRate, exactly like a reopen", async () => { + const env = createTestEnv(); + await signals(env, "i", 3, { verdict: "close", outcome: "closed", reversal: "none" }); // confirmed + await signals(env, "i", 1, { verdict: "close", outcome: "closed", reversal: "superseded" }); // work merged via a successor PR — the close was wrong + await signals(env, "i", 1, { verdict: "close", outcome: "closed", reversal: "reopened" }); // literal reopen — same treatment + const inst = (await computeFleetAnalytics(env)).instances[0]!; + expect(inst.closePrecision).toBeCloseTo(3 / 5); + expect(inst.fnRate).toBeCloseTo(2 / 5); + expect(inst.reversalRate).toBeCloseTo(2 / 5); + }); + it("null precision when an instance made no merge verdicts", async () => { const env = createTestEnv(); await signals(env, "inst1", 5, { verdict: "close", outcome: "closed" }); diff --git a/test/unit/selfhost-orb-collector.test.ts b/test/unit/selfhost-orb-collector.test.ts index 2929869322..800413d210 100644 --- a/test/unit/selfhost-orb-collector.test.ts +++ b/test/unit/selfhost-orb-collector.test.ts @@ -162,6 +162,36 @@ describe("exportOrbBatch() — always-on; reads review_audit, ships anonymized r expect(flags).toEqual(["reopened", "reverted"]); }); + it("flags reversal_superseded (#8820) — and an explicit reopen outranks the successor heuristic", async () => { + const db = makeDb(); + // The one-shot culture's dominant reversal shape (#8166): bot-closed PR whose work merged via a successor. + await audit(db, "o/r", 3, "gate_decision", "close", "2026-02-01T00:00:00Z"); + await audit(db, "o/r", 3, "pr_outcome", "closed", "2026-02-01T01:00:00Z"); + await audit(db, "o/r", 3, "reversal_superseded", null, "2026-02-01T04:00:00Z"); + // Both signals present → the stronger explicit reopen wins the single exported flag. + await audit(db, "o/r", 4, "gate_decision", "close", "2026-02-01T00:00:00Z"); + await audit(db, "o/r", 4, "pr_outcome", "closed", "2026-02-01T01:30:00Z"); + await audit(db, "o/r", 4, "reversal_reopened", null, "2026-02-01T02:00:00Z"); + await audit(db, "o/r", 4, "reversal_superseded", null, "2026-02-01T03:00:00Z"); + let captured: { events: Array<{ reversal_flag: string }> } | undefined; + await exportOrbBatch(db, 200, async (_u, init) => { captured = JSON.parse(init!.body as string); return new Response(null, { status: 200 }); }); + expect(captured!.events.map((e) => e.reversal_flag).sort()).toEqual(["reopened", "superseded"]); + }); + + it("REGRESSION (#8820): a reversal recorded AFTER a PR was already exported re-exports that PR with the flag", async () => { + const db = makeDb(); + await audit(db, "o/r", 5, "gate_decision", "close", "2026-02-01T00:00:00Z"); + await audit(db, "o/r", 5, "pr_outcome", "closed", "2026-02-01T01:00:00Z"); + const bodies: Array<{ events: Array<{ reversal_flag: string }> }> = []; + const capture = async (_u: RequestInfo | URL, init?: RequestInit) => { bodies.push(JSON.parse(init!.body as string)); return new Response(null, { status: 200 }); }; + expect(await exportOrbBatch(db, 200, capture)).toBe(1); // exported clean, cursor advanced past outcome_at + expect(bodies[0]!.events[0]!.reversal_flag).toBe("none"); + // The successor merge lands LATER — event_at bumps to rev_at, past the cursor, so the row re-exports. + await audit(db, "o/r", 5, "reversal_superseded", null, "2026-02-01T06:00:00Z"); + expect(await exportOrbBatch(db, 200, capture)).toBe(1); + expect(bodies[1]!.events[0]!.reversal_flag).toBe("superseded"); + }); + it("sends raw repo when ORB_ANONYMIZE=false", async () => { process.env.ORB_ANONYMIZE = "false"; const db = makeDb(); From cad3e60cfbfb61321b4337364198a2a9dccbae9e Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 26 Jul 2026 02:04:40 -0700 Subject: [PATCH 2/3] feat(orb): stream live self-host reuse counters into the public AI-work-reused trend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The homepage reuse-rate trend reads cache hit/miss audit events from the cloud worker's own ledger, which froze at the self-host cutover (last event 2026-06-29) — recent weekly buckets fall under the publish floor, so the hero tile renders a dash beside a decaying sparkline while the live signal (133k+ cache events) accrues unexported on the self-hosted instances. Advances #8820 (the reuse-rate half; the accuracy half is the superseded-reversal export on this same branch's sibling commit). - orb-collector: export day-bucketed hit/miss aggregates (counts only, no repos/PRs) over a 70-day rolling window on the same hourly POST; fail-safe when the ledger lacks the table - ingest: validate (strict day format, clamped non-negative counts) and upsert per (instance, day); malformed rows skipped row-by-row - public-reuse-rate-trend: fold counters from REGISTERED instances into the same weekly buckets, unconditional on the own-ledger repo allowlist (parity with the fleet-accuracy fold) - migration 0177: orb_reuse_counters --- migrations/0177_orb_reuse_counters.sql | 17 +++++++ src/orb/ingest.ts | 43 +++++++++++++++++ src/selfhost/orb-collector.ts | 45 ++++++++++++++++++ src/services/public-reuse-rate-trend.ts | 36 ++++++++++++-- test/integration/orb-ingest.test.ts | 31 ++++++++++++ test/unit/public-reuse-rate-trend.test.ts | 22 +++++++++ test/unit/selfhost-orb-collector.test.ts | 58 ++++++++++++++++++++++- 7 files changed, 245 insertions(+), 7 deletions(-) create mode 100644 migrations/0177_orb_reuse_counters.sql diff --git a/migrations/0177_orb_reuse_counters.sql b/migrations/0177_orb_reuse_counters.sql new file mode 100644 index 0000000000..f0e28cfa4a --- /dev/null +++ b/migrations/0177_orb_reuse_counters.sql @@ -0,0 +1,17 @@ +-- #8820 (reuse-rate half): live fleet source for the homepage "AI work reused" trend. +-- +-- The public reuse-rate trend reads github_app.%cache_hit/%cache_miss audit events from THIS worker's own +-- ledger — which froze at the self-host cutover (last event 2026-06-29), so the latest weekly buckets fell +-- under the publish floor and the hero tile rendered a dash next to a decaying sparkline. The live signal +-- (133k+ cache events and growing) accrues on the self-hosted instances; this table receives their +-- day-bucketed, instance-level aggregate counters (counts only — no repos, no PRs, no content), exported on +-- the same hourly tick as orb_signals and folded into the public trend for REGISTERED instances only (the +-- same trust anchor computeFleetAnalytics uses). +CREATE TABLE IF NOT EXISTS orb_reuse_counters ( + instance_id TEXT NOT NULL, + day TEXT NOT NULL, -- YYYY-MM-DD (UTC) + hits INTEGER NOT NULL DEFAULT 0, + misses INTEGER NOT NULL DEFAULT 0, + received_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (instance_id, day) -- senders re-export a rolling window; the upsert keeps the freshest counts +); diff --git a/src/orb/ingest.ts b/src/orb/ingest.ts index a17eec5db8..65653de7f9 100644 --- a/src/orb/ingest.ts +++ b/src/orb/ingest.ts @@ -75,6 +75,23 @@ interface OrbIngestPayload { // #4933: optional -- an older self-host build that hasn't upgraded yet simply omits this, and the // instance's stored health stays whatever it last was (or NULL/unknown on first contact). health?: { ok: boolean }; + // #8820: optional day-bucketed cache hit/miss aggregates for the public "AI work reused" trend (counts + // only). A rolling window re-sent every tick; upserted per (instance, day). Absent from older builds. + reuse_counters?: Array<{ day?: unknown; hits?: unknown; misses?: unknown }>; +} + +/** Rolling-window bound: the sender exports ~70 days (REUSE_COUNTER_WINDOW_DAYS); anything wildly larger is + * a hostile payload padding the loop, not a real export. */ +const MAX_REUSE_COUNTER_DAYS = 400; +const MAX_REUSE_COUNT = 10_000_000; // per-day per-instance ceiling — beyond this is fabrication, not telemetry +const REUSE_DAY_PATTERN = /^\d{4}-\d{2}-\d{2}$/; + +/** Clamp a sender-supplied per-day counter to a plausible non-negative integer; null rejects the row. */ +function clampReuseCount(value: unknown): number | null { + if (typeof value !== "number" || !Number.isFinite(value)) return null; + const rounded = Math.round(value); + if (rounded < 0 || rounded > MAX_REUSE_COUNT) return null; + return rounded; } export type OrbIngestResult = { accepted: number } | { error: string }; @@ -191,5 +208,31 @@ export async function handleOrbIngest(body: string, db: D1Database): Promise; +} + +/** Rolling window the reuse counters cover — the public trend renders 8 weeks; the extra buffer keeps the + * oldest visible bucket complete across week boundaries and export lag. */ +export const REUSE_COUNTER_WINDOW_DAYS = 70; + +/** Day-bucketed reuse counters from the local audit_events ledger. Same event population as + * loadReuseRateDayRows (public-reuse-rate-trend.ts) — the LIKE convention plus ai_review's three + * non-suffix-conforming reuse variants — so the fleet fold can never drift from the own-ledger count. + * substr(created_at, 1, 10) is the portable day bucket (runs on the SQLite AND Postgres backends, and + * tolerates this ledger's mixed 'YYYY-MM-DD hh:mm:ss' / ISO-with-T timestamp formats). */ +const REUSE_COUNTER_QUERY = ` + SELECT substr(created_at, 1, 10) AS day, + SUM(CASE WHEN event_type LIKE 'github_app.%cache_hit' OR event_type IN (${AI_REVIEW_REUSE_EVENT_TYPES.map(() => "?").join(", ")}) THEN 1 ELSE 0 END) AS hits, + SUM(CASE WHEN event_type LIKE 'github_app.%cache_miss' THEN 1 ELSE 0 END) AS misses + FROM audit_events + WHERE (event_type LIKE 'github_app.%cache_hit' OR event_type LIKE 'github_app.%cache_miss' OR event_type IN (${AI_REVIEW_REUSE_EVENT_TYPES.map(() => "?").join(", ")})) + AND created_at >= ? + GROUP BY day`; + +/** Read the rolling reuse-counter window; fail-safe → [] (a counter hiccup must never block the outcome + * export riding the same tick). */ +async function loadReuseCounters(db: D1Database, nowMs: number): Promise> { + const sinceIso = new Date(nowMs - REUSE_COUNTER_WINDOW_DAYS * 86_400_000).toISOString(); + try { + // The rows already carry exactly the export shape; SUM(CASE…) over a GROUP BY never yields SQL NULL, + // so no per-field fallback is needed (a missing table / failed query is the catch below). + const result = await db + .prepare(REUSE_COUNTER_QUERY) + .bind(...AI_REVIEW_REUSE_EVENT_TYPES, ...AI_REVIEW_REUSE_EVENT_TYPES, sinceIso) + .all<{ day: string; hits: number; misses: number }>(); + return result.results; + } catch { + return []; + } } /** Stable instance identifier (hash of the Orb/App ID — no PII). A brokered instance holds no App id, so its @@ -196,6 +236,10 @@ export async function exportOrbBatch(db: D1Database, batchSize = 200, fetchFn: t // otherwise, exactly as before, nothing new means nothing to do. if ((!results || results.length === 0) && healthOk === undefined) return 0; + // #8820: the reuse counters ride the same POST as the outcome events (same tick, same signature). Loaded + // AFTER the early "nothing to send" return above, so a truly idle tick still costs nothing extra. + const reuseCounters = await loadReuseCounters(db, Date.now()); + const payload: OrbExportPayload = { instance_id: instance, /* v8 ignore next -- D1's .all() always returns a `results` array (possibly empty), never omits the field; @@ -215,6 +259,7 @@ export async function exportOrbBatch(db: D1Database, batchSize = 200, fetchFn: t outcome_timestamp: r.outcome_at, })), ...(healthOk !== undefined ? { health: { ok: healthOk } } : {}), + ...(reuseCounters.length > 0 ? { reuse_counters: reuseCounters } : {}), }; const body = JSON.stringify(payload); diff --git a/src/services/public-reuse-rate-trend.ts b/src/services/public-reuse-rate-trend.ts index d8101751ea..bedd5d5677 100644 --- a/src/services/public-reuse-rate-trend.ts +++ b/src/services/public-reuse-rate-trend.ts @@ -26,8 +26,10 @@ export const PUBLIC_REUSE_RATE_TREND_WEEKS = 8; export const MIN_REUSE_RATE_TREND_SAMPLE = 5; /** ai_review reuse events that don't follow the `_cache_hit` suffix convention but are the SAME "avoided a - * redundant AI call" signal -- each one means the review pass reused a prior state instead of re-running. */ -const AI_REVIEW_REUSE_EVENT_TYPES = ["github_app.ai_review_frozen_reuse", "github_app.ai_review_paused_reuse", "github_app.ai_review_one_shot_reuse"] as const; + * redundant AI call" signal -- each one means the review pass reused a prior state instead of re-running. + * Exported for the self-host reuse-counter export (orb-collector.ts, #8820) so both sides count the exact + * same event population -- a drifted copy there would silently skew the published fleet rate. */ +export const AI_REVIEW_REUSE_EVENT_TYPES = ["github_app.ai_review_frozen_reuse", "github_app.ai_review_paused_reuse", "github_app.ai_review_one_shot_reuse"] as const; export type PublicReuseRateTrendWeek = { /** UTC Monday (YYYY-MM-DD) that starts the bucket. */ @@ -104,11 +106,35 @@ async function loadReuseRateDayRows(env: Env, projects: string[], sinceIso: stri return rows.map((row) => ({ day: row.day, hits: row.hits ?? 0, misses: row.misses ?? 0 })); } +/** Day-bucketed reuse counters exported by REGISTERED self-hosted instances (orb_reuse_counters, #8820) -- + * the LIVE side of this trend: the own-ledger audit_events below froze at the self-host cutover, so recent + * weeks otherwise fall under the publish floor and the homepage tile renders a dash. Registration is the + * same trust anchor computeFleetAnalytics uses -- open ingest stores everyone's counters, but an + * unregistered stranger can't move the published rate. Deliberately NOT scoped by the own-ledger repo + * allowlist (counters are instance-level counts only -- no repos to scope by), matching the fleet-accuracy + * fold's own unconditional-regardless-of-allowlist behavior in public-stats.ts. */ +async function loadFleetReuseDayRows(env: Env, sinceIso: string): Promise { + const rows = await safeAll<{ day: string; hits: number; misses: number }>( + env, + `SELECT c.day AS day, SUM(c.hits) AS hits, SUM(c.misses) AS misses + FROM orb_reuse_counters c + JOIN orb_instances i ON i.instance_id = c.instance_id AND i.registered = 1 + WHERE c.day >= ? + GROUP BY c.day`, + sinceIso.slice(0, 10), + ); + /* v8 ignore next -- same guard shape as loadReuseRateDayRows above: SUM over a GROUP BY day of NOT NULL + * integer columns always yields a defined integer, never SQL NULL; kept for defense against a future + * query-shape change. */ + return rows.map((row) => ({ day: row.day, hits: row.hits ?? 0, misses: row.misses ?? 0 })); +} + /** Assemble the public reuse-rate trend from the SAME live audit_events ledger every instrumented capability - * already writes to. */ + * already writes to, plus the registered fleet's exported day counters (#8820) -- buildPublicReuseRateTrend + * sums overlapping days from both sources into the same weekly buckets. */ export async function loadPublicReuseRateTrend(env: Env, nowMs: number = Date.now()): Promise { const projects = publicStatsProjects(env); const sinceIso = new Date(Date.parse(isoWeekStart(nowMs)) - (PUBLIC_REUSE_RATE_TREND_WEEKS - 1) * MS_PER_WEEK).toISOString(); - const dayRows = await loadReuseRateDayRows(env, projects, sinceIso); - return buildPublicReuseRateTrend(dayRows, nowMs); + const [ownRows, fleetRows] = await Promise.all([loadReuseRateDayRows(env, projects, sinceIso), loadFleetReuseDayRows(env, sinceIso)]); + return buildPublicReuseRateTrend([...ownRows, ...fleetRows], nowMs); } diff --git a/test/integration/orb-ingest.test.ts b/test/integration/orb-ingest.test.ts index 33d25f3f47..638a0a693a 100644 --- a/test/integration/orb-ingest.test.ts +++ b/test/integration/orb-ingest.test.ts @@ -100,6 +100,37 @@ describe("handleOrbIngest()", () => { expect(await col(db, "t2", "sent_at")).toBeNull(); }); + it("stores reuse_counters (#8820): valid rows upserted per (instance, day); malformed rows skipped; malformed container ignored", async () => { + const db = makeDb(); + const counterRow = async (day: string) => + (await (db as unknown as TestD1Database).prepare("SELECT hits, misses FROM orb_reuse_counters WHERE instance_id='inst1' AND day=?").bind(day).first<{ hits: number; misses: number }>()) ?? null; + const send = (reuse_counters: unknown) => handleOrbIngest(JSON.stringify({ instance_id: "inst1", events: [ev({ pr_hash: `rc${seq++}` })], reuse_counters }), db); + let seq = 0; + + await send([ + { day: "2026-02-01", hits: 5, misses: 2 }, + { day: "not-a-day", hits: 1, misses: 1 }, // bad day → skipped + { day: "2026-02-02", hits: -1, misses: 0 }, // negative → skipped + { day: "2026-02-03", hits: "many", misses: 0 }, // non-number → skipped + { day: "2026-02-04", hits: 4.6, misses: 10_000_001 }, // over the ceiling → skipped + ]); + expect(await counterRow("2026-02-01")).toEqual({ hits: 5, misses: 2 }); + expect(await counterRow("2026-02-02")).toBeNull(); + expect(await counterRow("2026-02-03")).toBeNull(); + expect(await counterRow("2026-02-04")).toBeNull(); + + // The rolling window re-sends the same day with fresher counts → REPLACE, not a duplicate. + await send([{ day: "2026-02-01", hits: 9, misses: 3 }]); + expect(await counterRow("2026-02-01")).toEqual({ hits: 9, misses: 3 }); + const n = await (db as unknown as TestD1Database).prepare("SELECT COUNT(*) AS n FROM orb_reuse_counters WHERE day='2026-02-01'").first<{ n: number }>(); + expect(n?.n).toBe(1); + + // A malformed container (not an array) is ignored; the outcome batch still lands. + expect(await send({ nope: true })).toEqual({ accepted: 1 }); + // Absent field (older builds) — unchanged behavior. + expect(await ingest(db, [ev({ pr_hash: "plain" })])).toEqual({ accepted: 1 }); + }); + it("UPSERTs on (instance, repo_hash, pr_hash): a re-export updates the freshest outcome (e.g. a later reversal)", async () => { const db = makeDb(); await ingest(db, [ev({ pr_hash: "u1", reversal_flag: "none" })]); diff --git a/test/unit/public-reuse-rate-trend.test.ts b/test/unit/public-reuse-rate-trend.test.ts index 8e4ded106f..8916509974 100644 --- a/test/unit/public-reuse-rate-trend.test.ts +++ b/test/unit/public-reuse-rate-trend.test.ts @@ -130,4 +130,26 @@ describe("loadPublicReuseRateTrend — end-to-end over the real live audit_event for (const week of trend) expect(week).toMatchObject({ hits: 0, misses: 0, reuseRatePct: null }); }); + it("folds REGISTERED fleet instances' day counters into the same weekly buckets; unregistered instances never count (#8820)", async () => { + const env = createTestEnv({ LOOPOVER_PUBLIC_STATS_REPOS: "owner/repo" }); + const thisMonday = isoWeekStart(NOW); + // Own-ledger event in the same week — the fold must SUM both sources, not replace one with the other. + await recordAuditEvent(env, { eventType: "github_app.grounding_cache_hit", targetKey: "owner/repo", outcome: "completed", createdAt: `${thisMonday}T09:00:00.000Z` }); + await env.DB.prepare(`INSERT INTO orb_instances (instance_id, registered) VALUES ('reg-inst', 1), ('stranger', 0)`).run(); + await env.DB.prepare(`INSERT INTO orb_reuse_counters (instance_id, day, hits, misses) VALUES ('reg-inst', ?, 6, 3), ('stranger', ?, 500, 500)`).bind(thisMonday, thisMonday).run(); + + const trend = await loadPublicReuseRateTrend(env, NOW); + const currentWeek = trend[trend.length - 1]; + expect(currentWeek).toMatchObject({ weekStart: thisMonday, hits: 7, misses: 3 }); // 1 own + 6/3 registered; stranger excluded + }); + + it("fleet counters flow even with an EMPTY own-ledger allowlist (#8820) — the frozen-own-ledger deployment shape", async () => { + const env = createTestEnv(); + const thisMonday = isoWeekStart(NOW); + await env.DB.prepare(`INSERT INTO orb_instances (instance_id, registered) VALUES ('reg-inst', 1)`).run(); + await env.DB.prepare(`INSERT INTO orb_reuse_counters (instance_id, day, hits, misses) VALUES ('reg-inst', ?, 8, 2)`).bind(thisMonday).run(); + + const trend = await loadPublicReuseRateTrend(env, NOW); + expect(trend[trend.length - 1]).toMatchObject({ weekStart: thisMonday, hits: 8, misses: 2, reuseRatePct: 80 }); + }); }); diff --git a/test/unit/selfhost-orb-collector.test.ts b/test/unit/selfhost-orb-collector.test.ts index 800413d210..a9da84bbb3 100644 --- a/test/unit/selfhost-orb-collector.test.ts +++ b/test/unit/selfhost-orb-collector.test.ts @@ -4,8 +4,9 @@ import { createD1Adapter, nodeSqliteDriver } from "../../src/selfhost/d1-adapter import { bucketReasonCode, exportOrbBatch, getOrCreateAnonSecret } from "../../src/selfhost/orb-collector"; import { resetMetrics, renderMetrics } from "../../src/selfhost/metrics"; -/** In-memory DB with the review_audit + orb_export_cursor tables the exporter reads. */ -function makeDb(): D1Database { +/** In-memory DB with the review_audit + orb_export_cursor tables the exporter reads. `withAuditEvents: + * false` drops the reuse-counter source table to prove the counter read fails SAFE (export still runs). */ +function makeDb(options: { withAuditEvents?: boolean } = {}): D1Database { const driver = nodeSqliteDriver(new DatabaseSync(":memory:") as never); driver.exec(` CREATE TABLE review_audit ( @@ -24,9 +25,21 @@ function makeDb(): D1Database { updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')) ); `); + if (options.withAuditEvents !== false) { + driver.exec(` + CREATE TABLE audit_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, event_type TEXT NOT NULL, target_key TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')) + ); + `); + } return createD1Adapter(driver); } +async function cacheEvent(db: D1Database, eventType: string, at: string): Promise { + await db.prepare(`INSERT INTO audit_events (event_type, target_key, created_at) VALUES (?, 'o/r#1', ?)`).bind(eventType, at).run(); +} + let seq = 0; async function audit(db: D1Database, project: string, pr: number, eventType: string, decision: string | null, at: string, summary: string | null = null): Promise { await db @@ -178,6 +191,47 @@ describe("exportOrbBatch() — always-on; reads review_audit, ships anonymized r expect(captured!.events.map((e) => e.reversal_flag).sort()).toEqual(["reopened", "superseded"]); }); + it("ships day-bucketed reuse counters alongside the outcome events (#8820), bounded to the rolling window", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-02-10T12:00:00Z")); + try { + const db = makeDb(); + await audit(db, "o/r", 1, "gate_decision", "merge", "2026-02-01T00:00:00Z"); + await audit(db, "o/r", 1, "pr_outcome", "merged", "2026-02-01T01:00:00Z"); + await cacheEvent(db, "github_app.grounding_cache_hit", "2026-02-01T02:00:00Z"); + await cacheEvent(db, "github_app.review_memory_cache_hit", "2026-02-01T03:00:00Z"); + await cacheEvent(db, "github_app.impact_map_cache_miss", "2026-02-01T04:00:00Z"); + await cacheEvent(db, "github_app.ai_review_frozen_reuse", "2026-02-02T00:00:00Z"); // non-suffix reuse variant = hit + await cacheEvent(db, "github_app.grounding_cache_hit", "2020-01-01T00:00:00Z"); // far outside the window + let captured: { reuse_counters?: Array<{ day: string; hits: number; misses: number }> } | undefined; + await exportOrbBatch(db, 200, async (_u, init) => { captured = JSON.parse(init!.body as string); return new Response(null, { status: 200 }); }); + expect(captured!.reuse_counters).toEqual([ + { day: "2026-02-01", hits: 2, misses: 1 }, + { day: "2026-02-02", hits: 1, misses: 0 }, + ]); + } finally { + vi.useRealTimers(); + } + }); + + it("omits reuse_counters when the window has none — and a missing audit_events table fails SAFE (#8820)", async () => { + // No cache events at all → the field is absent, not an empty array. + const clean = makeDb(); + await audit(clean, "o/r", 1, "gate_decision", "merge", "2026-02-01T00:00:00Z"); + await audit(clean, "o/r", 1, "pr_outcome", "merged", "2026-02-01T01:00:00Z"); + let captured: Record | undefined; + await exportOrbBatch(clean, 200, async (_u, init) => { captured = JSON.parse(init!.body as string); return new Response(null, { status: 200 }); }); + expect("reuse_counters" in captured!).toBe(false); + // A deployment whose ledger lacks the table entirely: the counter read degrades to none — the outcome + // export riding the same tick still succeeds. + const bare = makeDb({ withAuditEvents: false }); + await audit(bare, "o/r", 2, "gate_decision", "merge", "2026-02-01T00:00:00Z"); + await audit(bare, "o/r", 2, "pr_outcome", "merged", "2026-02-01T01:00:00Z"); + captured = undefined; + expect(await exportOrbBatch(bare, 200, async (_u, init) => { captured = JSON.parse(init!.body as string); return new Response(null, { status: 200 }); })).toBe(1); + expect("reuse_counters" in captured!).toBe(false); + }); + it("REGRESSION (#8820): a reversal recorded AFTER a PR was already exported re-exports that PR with the flag", async () => { const db = makeDb(); await audit(db, "o/r", 5, "gate_decision", "close", "2026-02-01T00:00:00Z"); From b86c27753237cde42eec3ea527b96483436251ec Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 26 Jul 2026 02:17:23 -0700 Subject: [PATCH 3/3] chore(db): register orb_reuse_counters as a raw-SQL-only table in the drift check --- scripts/check-schema-drift.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/check-schema-drift.ts b/scripts/check-schema-drift.ts index d18d34452b..f04495829f 100644 --- a/scripts/check-schema-drift.ts +++ b/scripts/check-schema-drift.ts @@ -49,6 +49,7 @@ export const RAW_SQL_ONLY_TABLES: Set = new Set([ "orb_instances", "orb_pr_outcomes", "orb_relay_failures", + "orb_reuse_counters", "orb_signals", "orb_webhook_events", "override_audit",