diff --git a/packages/loopover-engine/src/calibration/repo-corpus.ts b/packages/loopover-engine/src/calibration/repo-corpus.ts new file mode 100644 index 0000000000..4880c2776b --- /dev/null +++ b/packages/loopover-engine/src/calibration/repo-corpus.ts @@ -0,0 +1,75 @@ +// Per-repo corpus slicing + labeled-density stats (#8215, epic #8211 track B). Every BacktestCase carries +// its repo inside `targetKey` (`owner/repo#N` — host-defined by signal-tracking's own convention), but the +// calibration primitives only ever evaluate globally; per-repo autonomy needs these two pure building blocks +// first: slice a corpus by repo, and know which repos have enough labeled density to evaluate AT ALL under +// the same split + sample-minimum discipline the knob evaluators already apply globally. +// +// Aggregates only in every returned shape (repo names + numbers) — no target keys, no metadata — so a +// consumer can surface these results without re-auditing the public/private boundary. Same purity contract +// as the rest of this module family: no IO, no randomness, no wall-clock reads. + +import type { BacktestCase } from "./backtest-corpus.js"; +import { splitBacktestCorpus } from "./backtest-split.js"; + +/** + * Slice a corpus by repo: the repo is everything before `targetKey`'s LAST `#` (an issue/PR key like + * `owner/repo#123` — the last-`#` parse keeps a `#` inside an owner/repo name from truncating the repo). + * A key with no `#`, or nothing before it, is dropped — never guessed into a slice. Deterministic: + * insertion order follows each repo's first appearance, and case order is preserved within each slice. + */ +export function sliceCorpusByRepo(cases: readonly BacktestCase[]): Map { + const slices = new Map(); + for (const backtestCase of cases) { + const hashIndex = backtestCase.targetKey.lastIndexOf("#"); + if (hashIndex <= 0) continue; // unparseable target key — drop, never guess + const repoFullName = backtestCase.targetKey.slice(0, hashIndex); + let slice = slices.get(repoFullName); + if (slice === undefined) { + slice = []; + slices.set(repoFullName, slice); + } + slice.push(backtestCase); + } + return slices; +} + +/** Aggregate-only labeled-density stats for one repo's slice. `eligible` applies the SAME seeded-split + + * sample-minimum discipline the knob evaluators use: a repo is evaluable only if ITS OWN slice still clears + * both floors after splitting. */ +export type RepoCorpusDensity = { + cases: number; + confirmed: number; + reversed: number; + eligible: boolean; +}; + +/** + * Compute per-repo labeled density over `cases`, keyed by repo (via {@link sliceCorpusByRepo}, so + * unparseable keys are dropped here too). Each repo's `eligible` flag re-runs the deterministic split on + * that repo's OWN slice with the caller's seed/fraction and requires both the visible and held-out sides to + * clear their floors — the exact never-on-noise bar `evaluateKnobLoosening`/`evaluateKnobDrift` apply to the + * global corpus. Pure and deterministic: same corpus + parameters ⇒ same map, in first-appearance order. + */ +export function computeRepoCorpusDensity( + cases: readonly BacktestCase[], + minVisible: number, + minHeldOut: number, + heldOutFraction: number, + splitSeed: string, +): Map { + const densities = new Map(); + for (const [repoFullName, slice] of sliceCorpusByRepo(cases)) { + let confirmed = 0; + for (const backtestCase of slice) { + if (backtestCase.label === "confirmed") confirmed += 1; + } + const { visible, heldOut } = splitBacktestCorpus(slice, heldOutFraction, splitSeed); + densities.set(repoFullName, { + cases: slice.length, + confirmed, + reversed: slice.length - confirmed, + eligible: visible.length >= minVisible && heldOut.length >= minHeldOut, + }); + } + return densities; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index bea83db60d..e3236a63bc 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -175,6 +175,7 @@ export * from "./calibration/backtest-track-record.js"; // same way scripts/backtest-corpus-export.ts already imports BacktestCase. export * from "./calibration/backtest-split.js"; export * from "./calibration/backtest-threshold.js"; +export * from "./calibration/repo-corpus.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/repo-corpus.test.ts b/packages/loopover-engine/test/repo-corpus.test.ts new file mode 100644 index 0000000000..77ff8b950a --- /dev/null +++ b/packages/loopover-engine/test/repo-corpus.test.ts @@ -0,0 +1,49 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { computeRepoCorpusDensity, sliceCorpusByRepo, splitBacktestCorpus, type BacktestCase } from "../dist/index.js"; + +function corpusCase(targetKey: string, label: BacktestCase["label"] = "confirmed"): BacktestCase { + return { + ruleId: "missing_linked_issue", + targetKey, + outcome: "block", + label, + firedAt: "2026-07-01T00:00:00.000Z", + decidedAt: "2026-07-02T00:00:00.000Z", + }; +} + +test("barrel: the public entrypoint re-exports the per-repo corpus primitives (#8215)", () => { + assert.equal(typeof sliceCorpusByRepo, "function"); + assert.equal(typeof computeRepoCorpusDensity, "function"); +}); + +test("sliceCorpusByRepo: slices by the last '#', preserves order, drops unparseable keys", () => { + const slices = sliceCorpusByRepo([ + corpusCase("acme/widgets#1"), + corpusCase("beta/tools#3"), + corpusCase("acme/widgets#2"), + corpusCase("no-hash"), + corpusCase("#5"), + ]); + assert.deepEqual([...slices.keys()], ["acme/widgets", "beta/tools"]); + assert.deepEqual( + slices.get("acme/widgets")!.map((c) => c.targetKey), + ["acme/widgets#1", "acme/widgets#2"], + ); +}); + +test("computeRepoCorpusDensity: per-repo aggregates with the split-floor eligibility bar", () => { + const dense = Array.from({ length: 40 }, (_, i) => corpusCase(`acme/dense#${i + 1}`, i % 2 === 0 ? "reversed" : "confirmed")); + const sparse = [corpusCase("acme/sparse#1"), corpusCase("acme/sparse#2", "reversed")]; + const densities = computeRepoCorpusDensity([...dense, ...sparse], 5, 2, 0.25, "repo-density-test-v1"); + + const denseDensity = densities.get("acme/dense")!; + assert.equal(denseDensity.cases, 40); + assert.equal(denseDensity.confirmed + denseDensity.reversed, 40); + const denseSplit = splitBacktestCorpus(dense, 0.25, "repo-density-test-v1"); + assert.equal(denseDensity.eligible, denseSplit.visible.length >= 5 && denseSplit.heldOut.length >= 2); + + assert.deepEqual(densities.get("acme/sparse"), { cases: 2, confirmed: 1, reversed: 1, eligible: false }); +}); diff --git a/test/unit/repo-corpus-engine.test.ts b/test/unit/repo-corpus-engine.test.ts new file mode 100644 index 0000000000..1d672a736a --- /dev/null +++ b/test/unit/repo-corpus-engine.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; + +// Import the engine SOURCE directly (not the built dist) -- coverage.include lists +// packages/loopover-engine/src/**, so only a source-path import exercises the .ts these branches live in +// (the dist-importing twin in packages/loopover-engine/test/ covers the built barrel for the workspace +// suite). Same pattern as backtest-corpus-engine.test.ts / backtest-split-engine.test.ts. +import { computeRepoCorpusDensity, sliceCorpusByRepo } from "../../packages/loopover-engine/src/calibration/repo-corpus"; +import { splitBacktestCorpus } from "../../packages/loopover-engine/src/calibration/backtest-split"; +import type { BacktestCase } from "../../packages/loopover-engine/src/calibration/backtest-corpus"; + +function corpusCase(targetKey: string, label: BacktestCase["label"] = "confirmed"): BacktestCase { + return { + ruleId: "missing_linked_issue", + targetKey, + outcome: "block", + label, + firedAt: "2026-07-01T00:00:00.000Z", + decidedAt: "2026-07-02T00:00:00.000Z", + }; +} + +describe("sliceCorpusByRepo (#8215)", () => { + it("slices a multi-repo corpus by the last '#', preserving per-slice case order and first-appearance repo order", () => { + const cases = [ + corpusCase("acme/widgets#1"), + corpusCase("acme/gadgets#7"), + corpusCase("acme/widgets#2", "reversed"), + corpusCase("beta/tools#3"), + corpusCase("acme/widgets#3"), + ]; + const slices = sliceCorpusByRepo(cases); + expect([...slices.keys()]).toEqual(["acme/widgets", "acme/gadgets", "beta/tools"]); + expect(slices.get("acme/widgets")!.map((c) => c.targetKey)).toEqual(["acme/widgets#1", "acme/widgets#2", "acme/widgets#3"]); + expect(slices.get("acme/gadgets")).toHaveLength(1); + expect(slices.get("beta/tools")).toHaveLength(1); + }); + + it("keys a repo containing an embedded '#' by the LAST '#', keeping the full repo name intact", () => { + const slices = sliceCorpusByRepo([corpusCase("acme/wid#gets#12")]); + expect([...slices.keys()]).toEqual(["acme/wid#gets"]); + }); + + it("drops unparseable target keys — no '#', or nothing before it — instead of guessing a slice", () => { + const good = corpusCase("acme/widgets#1"); + const slices = sliceCorpusByRepo([corpusCase("no-hash-at-all"), corpusCase("#5"), good]); + expect([...slices.keys()]).toEqual(["acme/widgets"]); + expect(slices.get("acme/widgets")).toEqual([good]); + }); + + it("returns an empty map for an empty corpus", () => { + expect(sliceCorpusByRepo([])).toEqual(new Map()); + }); +}); + +describe("computeRepoCorpusDensity (#8215)", () => { + const SEED = "repo-density-test-v1"; + const FRACTION = 0.25; + + // Probe the real splitter per candidate repo slice so eligibility fixtures are membership-exact rather + // than luck-based — the same probing technique test/unit/loosening-knobs.test.ts uses. + function denseRepoCases(repoFullName: string, count: number): BacktestCase[] { + return Array.from({ length: count }, (_, i) => corpusCase(`${repoFullName}#${i + 1}`, i % 3 === 0 ? "reversed" : "confirmed")); + } + + it("reports aggregate-only per-repo counts and applies the split-based eligibility floors to each repo's OWN slice", () => { + const dense = denseRepoCases("acme/dense", 40); + const sparse = denseRepoCases("acme/sparse", 3); + const densities = computeRepoCorpusDensity([...dense, ...sparse], 5, 2, FRACTION, SEED); + + const denseSplit = splitBacktestCorpus(dense, FRACTION, SEED); + expect(densities.get("acme/dense")).toEqual({ + cases: 40, + confirmed: dense.filter((c) => c.label === "confirmed").length, + reversed: dense.filter((c) => c.label === "reversed").length, + eligible: denseSplit.visible.length >= 5 && denseSplit.heldOut.length >= 2, + }); + expect(densities.get("acme/dense")!.eligible).toBe(true); // 40 cases at 0.25 clears 5/2 for this seed + expect(densities.get("acme/sparse")!.eligible).toBe(false); // 3 cases can never clear the floors + expect(densities.get("acme/sparse")!.cases).toBe(3); + }); + + it("marks a repo ineligible when its held-out side alone misses the floor, even with a large visible side", () => { + const cases = denseRepoCases("acme/lopsided", 40); + const { heldOut } = splitBacktestCorpus(cases, FRACTION, SEED); + const densities = computeRepoCorpusDensity(cases, 1, heldOut.length + 1, FRACTION, SEED); + expect(densities.get("acme/lopsided")!.eligible).toBe(false); + }); + + it("is deterministic per slice: identical corpus + parameters yield the identical map", () => { + const cases = [...denseRepoCases("acme/a", 12), ...denseRepoCases("acme/b", 8)]; + expect(computeRepoCorpusDensity(cases, 3, 1, FRACTION, SEED)).toEqual(computeRepoCorpusDensity(cases, 3, 1, FRACTION, SEED)); + }); + + it("never leaks target keys or metadata — aggregate numbers and the eligibility flag only", () => { + const densities = computeRepoCorpusDensity(denseRepoCases("acme/private", 10), 2, 1, FRACTION, SEED); + for (const [repo, density] of densities) { + expect(repo).toBe("acme/private"); + expect(Object.keys(density).sort()).toEqual(["cases", "confirmed", "eligible", "reversed"]); + expect(JSON.stringify(density)).not.toContain("#"); + } + }); +});