From 9d15e02dfe9744eebaad35bdf203fd2a1624431d Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 07:02:03 -0700 Subject: [PATCH 1/2] =?UTF-8?q?fix(retention):=20dedup=20the=20contributor?= =?UTF-8?q?-intelligence=20snapshot=20types=20=E2=80=94=202026-07=20recurr?= =?UTF-8?q?ence=20of=20#3810?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit signal_snapshots reached D1's absolute 10GB cap again, 61 days into this database's life: the contributor-intelligence writers (the scoring pass in processors.ts) append one ~36KB row per contributor per pass for contributor-evidence-graph / contributor-outcome-history / contributor-strategy — ~6GB in three weeks at current review volume, refilling the cap before the 90-day age window could ever engage. Live writes were one page-split from failing; the phase-2 label apply already did. No reader consumes these types as a series (the canonical latest lives in the dedicated contributor_evidence / contributor_scoring_profiles upsert tables; nothing calls listSignalSnapshots for contributor-*), so the existing latest-only dedup job now covers them — lossless for every actual consumer. contributor-decision-pack stays excluded as the bounded trend series the retention doc records it to be. Production remediation (the incident playbook's chunked per-type manual dedup) executed alongside. --- src/db/retention.ts | 11 +++++++++++ test/unit/retention.test.ts | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/src/db/retention.ts b/src/db/retention.ts index 399b28102b..4a03ce59d6 100644 --- a/src/db/retention.ts +++ b/src/db/retention.ts @@ -104,6 +104,17 @@ export const LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES = [ "repo-doc-refresh-attempt", "repo-focus-manifest", "repo-public-focus-manifest", + // 2026-07-23 recurrence of #3810, new offenders: the contributor-intelligence writers (processors.ts's + // scoring pass) append one ~36KB row PER CONTRIBUTOR PER PASS for these three types — ~6GB in three + // weeks at current review volume, refilling D1's 10GB cap before the 90-day age window could ever + // engage. No reader consumes them as a series (the canonical latest lives in the dedicated + // contributor_evidence / contributor_scoring_profiles upsert tables; nothing calls + // listSignalSnapshots for contributor-* types), so latest-only is lossless for every actual consumer. + // contributor-decision-pack stays EXCLUDED: the retention doc above records it as a bounded + // trend/change series by design, and its volume is a fraction of these three. + "contributor-evidence-graph", + "contributor-outcome-history", + "contributor-strategy", ] as const; /** diff --git a/test/unit/retention.test.ts b/test/unit/retention.test.ts index ef1ec2e49f..0d8f9d4dc7 100644 --- a/test/unit/retention.test.ts +++ b/test/unit/retention.test.ts @@ -162,6 +162,32 @@ describe("dedupeSignalSnapshots", () => { expect(remaining?.id).toBe("s-3"); }); + it("dedupes the contributor-intelligence types to latest-per-contributor (2026-07 recurrence of #3810: ~6GB in three weeks)", async () => { + const env = createTestEnv(); + for (const signalType of ["contributor-evidence-graph", "contributor-outcome-history", "contributor-strategy"] as const) { + await insertSignalSnapshot(env, `${signalType}-old`, signalType, "octocat", "2026-07-01T00:00:00.000Z"); + await insertSignalSnapshot(env, `${signalType}-new`, signalType, "octocat", "2026-07-02T00:00:00.000Z"); + await insertSignalSnapshot(env, `${signalType}-other`, signalType, "hubot", "2026-07-02T00:00:00.000Z"); + } + // decision-pack is a bounded trend series by design — must remain untouched. + await insertSignalSnapshot(env, "pack-1", "contributor-decision-pack", "octocat", "2026-07-01T00:00:00.000Z"); + await insertSignalSnapshot(env, "pack-2", "contributor-decision-pack", "octocat", "2026-07-02T00:00:00.000Z"); + + const results = await dedupeSignalSnapshots(env); + const byType = Object.fromEntries(results.map((r) => [r.signalType, r.deleted])); + expect(byType["contributor-evidence-graph"]).toBe(1); + expect(byType["contributor-outcome-history"]).toBe(1); + expect(byType["contributor-strategy"]).toBe(1); + + const remaining = await env.DB.prepare("SELECT id FROM signal_snapshots ORDER BY id").all<{ id: string }>(); + const ids = (remaining.results ?? []).map((row) => row.id); + expect(ids).toContain("contributor-strategy-new"); + expect(ids).toContain("contributor-strategy-other"); + expect(ids).not.toContain("contributor-strategy-old"); + expect(ids).toContain("pack-1"); // series preserved + expect(ids).toContain("pack-2"); + }); + it("dedupes private and public focus-manifest cache snapshots (regression for storage exhaustion)", async () => { const env = createTestEnv(); await insertSignalSnapshot(env, "private-old", REPO_FOCUS_MANIFEST_SIGNAL, "JSONbored/loopover", "2026-06-01T00:00:00.000Z"); From 3395a9ffb368d0e490b5424b7058c761f2518be6 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 07:17:51 -0700 Subject: [PATCH 2/2] test(selfhost): derive the size-probe placeholder pin from LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pin hardcoded four placeholders; the list just grew to seven (the #3810 recurrence fix) and will grow again — derive it so the guard can never silently desync from the probe's real SQL. --- test/unit/selfhost-d1-size-probe.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/unit/selfhost-d1-size-probe.test.ts b/test/unit/selfhost-d1-size-probe.test.ts index a978b0f928..f8f2813f4a 100644 --- a/test/unit/selfhost-d1-size-probe.test.ts +++ b/test/unit/selfhost-d1-size-probe.test.ts @@ -170,7 +170,10 @@ describe("fetchD1TableRowCount", () => { calls.push({ url: requestUrl(input), method: init?.method ?? "GET" }); const body = JSON.parse(String(init?.body ?? "{}")) as { sql: string; params: string[] }; capturedParams = body.params; - expect(body.sql).toContain("signal_type IN (?1, ?2, ?3, ?4)"); + // Derived from the list itself so growing LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES (as the #3810 + // recurrences keep doing) can never silently desync this pin from the probe's real SQL. + const expectedPlaceholders = LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES.map((_, index) => `?${index + 1}`).join(", "); + expect(body.sql).toContain(`signal_type IN (${expectedPlaceholders})`); return new Response(envelope([{ results: [{ total: 20225, dedup_total: 107, dedup_distinct_keys: 36 }], success: true, meta: {} }])); }; const row = await fetchD1TableRowCount(config, "signal_snapshots", fetchImpl);