diff --git a/packages/gittensory-miner/lib/replay-task-generation.d.ts b/packages/gittensory-miner/lib/replay-task-generation.d.ts new file mode 100644 index 0000000000..ced38e3253 --- /dev/null +++ b/packages/gittensory-miner/lib/replay-task-generation.d.ts @@ -0,0 +1,110 @@ +export const FORWARD_REF_PLACEHOLDER: string; + +export type RecencyPool = "recent" | "older"; +export const RECENCY_POOLS: readonly RecencyPool[]; + +export type ForwardReference = { + kind: "link" | "hashref" | "sha" | "bare-issue-number"; + value: string | number; +}; + +export type ForwardRefContext = { + knownIssueMax?: number; + knownCommitShas?: string[]; + revealedIssueNumbers?: number[]; +}; + +export type DetectedForwardReferences = { + scrubbable: ForwardReference[]; + unscrubbable: ForwardReference[]; +}; + +export type ScrubResult = { + scrubbed: string; + removed: ForwardReference[]; + residual: ForwardReference[]; +}; + +export type LintResult = { + ok: boolean; + residual: ForwardReference[]; +}; + +export type FreezePointThresholds = { + minPriorCommits?: number; + minRevealedCommits?: number; +}; + +export type FreezePointCandidate = { + repo?: string; + commitT?: string; + priorCommitCount?: number; + revealedCommitCount?: number; + lastActivityAt?: string; + frozenContextTexts?: unknown[]; + revealedGroundTruth?: unknown; +}; + +export type FreezePointSelection = { + eligible: boolean; + reasons: string[]; + priorCommitCount: number; + revealedCommitCount: number; +}; + +export type ReplayTaskOptions = { + thresholds?: FreezePointThresholds; + modelCutoffIso?: string; +}; + +export type ReplayTaskRejected = { + eligible: false; + rejected: "selection" | "unscrubbable_forward_reference"; + reasons?: string[]; + residual?: ForwardReference[]; +}; + +export type ReplayTask = { + eligible: true; + pool: RecencyPool; + frozen: { + repo: string | null; + commitT: string | null; + contextTexts: string[]; + }; + revealed: { + commitCount: number; + groundTruth: unknown; + }; +}; + +export function detectForwardReferences( + text: unknown, + context: ForwardRefContext | null | undefined, +): DetectedForwardReferences; + +export function scrubForwardReferences( + text: unknown, + context: ForwardRefContext | null | undefined, +): ScrubResult; + +export function lintFrozenContext( + texts: unknown, + context: ForwardRefContext | null | undefined, +): LintResult; + +export function selectFreezePoint( + candidate: FreezePointCandidate | null | undefined, + thresholds: FreezePointThresholds | null | undefined, +): FreezePointSelection; + +export function classifyRecencyPool( + candidate: FreezePointCandidate | null | undefined, + options: { modelCutoffIso?: string } | null | undefined, +): RecencyPool; + +export function generateReplayTask( + candidate: FreezePointCandidate | null | undefined, + context: ForwardRefContext | null | undefined, + options: ReplayTaskOptions | null | undefined, +): ReplayTask | ReplayTaskRejected; diff --git a/packages/gittensory-miner/lib/replay-task-generation.js b/packages/gittensory-miner/lib/replay-task-generation.js new file mode 100644 index 0000000000..a861c3855f --- /dev/null +++ b/packages/gittensory-miner/lib/replay-task-generation.js @@ -0,0 +1,202 @@ +// Leakage-safe task generation for the historical-replay calibration harness (#3011). +// +// A frozen snapshot at commit T is only useful for calibration if (a) the freeze point has enough real +// history on both sides to be worth scoring, and (b) nothing in the frozen context lets a replay run infer +// the future by pattern-matching text rather than reasoning. This module selects calibration-worthy freeze +// points, scrubs forward references out of the frozen context, tags each point's recency pool, and returns +// the frozen snapshot and the revealed post-T ground truth as *separate* bundles so the replay pipeline never +// holds both at once. Every function here is pure and deterministic — no clock, no randomness, no IO — so a +// given (candidate, context) always yields an identical task. + +// What a scrubbed-away forward reference is replaced with. A fixed, self-delimiting token so the scrubbed +// text stays readable and the substitution is itself deterministic. +export const FORWARD_REF_PLACEHOLDER = "[redacted-forward-ref]"; + +// Recency pools. Freeze points are mixed across these bands so a judge/planner that has memorized recent +// public history cannot dominate the calibration signal. +export const RECENCY_POOLS = Object.freeze(["recent", "older"]); + +function toIssueNumberSet(values) { + const set = new Set(); + if (Array.isArray(values)) { + for (const value of values) { + if (Number.isInteger(value) && value > 0) set.add(value); + } + } + return set; +} + +function toShaSet(values) { + const set = new Set(); + if (Array.isArray(values)) { + for (const value of values) { + if (typeof value === "string" && /^[0-9a-f]{7,40}$/i.test(value)) set.add(value.toLowerCase()); + } + } + return set; +} + +function resolveContext(context) { + return { + knownIssueMax: + Number.isInteger(context?.knownIssueMax) && context.knownIssueMax >= 0 ? context.knownIssueMax : 0, + knownCommitShas: toShaSet(context?.knownCommitShas), + revealedIssueNumbers: toIssueNumberSet(context?.revealedIssueNumbers), + }; +} + +// Core scanner shared by scrub/detect/lint. Walks a text in a fixed priority order (deep-links first, so an +// issue/PR/commit URL is handled before its inner number/SHA can match a barer pattern) and classifies each +// forward reference as either: +// - scrubbable: a self-delimited token (`#123`, a GitHub issues/pull/commit URL, or a raw commit SHA) that +// resolves only to post-T state and can be safely replaced with the placeholder; or +// - unscrubbable: a *bare* integer that exactly matches a known post-T issue number. A bare number cannot be +// blanket-removed without destroying legitimate pre-T numbers (versions, counts), so it is detected but +// left in place — its presence must fail the freeze point rather than be silently mangled. +function processForwardReferences(rawText, context) { + const resolved = resolveContext(context); + const removed = []; + + const text = typeof rawText === "string" ? rawText : ""; + + // 1. GitHub issue/pull deep-links whose number is after T. + let scrubbed = text.replace( + /https?:\/\/github\.com\/[\w.-]+\/[\w.-]+\/(?:issues|pull)\/(\d+)\b/gi, + (match, digits) => { + if (Number(digits) > resolved.knownIssueMax) { + removed.push({ kind: "link", value: match }); + return FORWARD_REF_PLACEHOLDER; + } + return match; + }, + ); + + // 2. GitHub commit deep-links whose SHA is not in pre-T history. + scrubbed = scrubbed.replace( + /https?:\/\/github\.com\/[\w.-]+\/[\w.-]+\/commit\/([0-9a-f]{7,40})\b/gi, + (match, sha) => { + if (!resolved.knownCommitShas.has(sha.toLowerCase())) { + removed.push({ kind: "link", value: match }); + return FORWARD_REF_PLACEHOLDER; + } + return match; + }, + ); + + // 3. Bare `#123` issue/PR references after T (not already inside a now-removed link). + scrubbed = scrubbed.replace(/(^|[^\w/])#(\d+)\b/g, (match, prefix, digits) => { + if (Number(digits) > resolved.knownIssueMax) { + removed.push({ kind: "hashref", value: `#${digits}` }); + return `${prefix}${FORWARD_REF_PLACEHOLDER}`; + } + return match; + }); + + // 4. Raw commit SHAs not in pre-T history. Require at least one hex letter so a plain decimal number is + // never misread as a SHA — those flow to the bare-issue-number residual check below instead. + scrubbed = scrubbed.replace(/(^|[^\w/#])([0-9a-f]{7,40})\b/gi, (match, prefix, sha) => { + if (!/[a-f]/i.test(sha)) return match; + if (!resolved.knownCommitShas.has(sha.toLowerCase())) { + removed.push({ kind: "sha", value: sha }); + return `${prefix}${FORWARD_REF_PLACEHOLDER}`; + } + return match; + }); + + // Residual: bare integers that name a real post-T issue and so leak the future, but cannot be safely + // auto-removed. Detected against the surviving text — if any remain, the freeze point is not usable as-is. + const residual = []; + if (resolved.revealedIssueNumbers.size > 0) { + for (const bareMatch of scrubbed.matchAll(/(?:^|[^\w#/])(\d+)\b/g)) { + const value = Number(bareMatch[1]); + if (resolved.revealedIssueNumbers.has(value)) { + residual.push({ kind: "bare-issue-number", value }); + } + } + } + + return { scrubbed, removed, residual }; +} + +// Detect forward references in text without modifying it, split by whether they can be safely scrubbed. +export function detectForwardReferences(text, context) { + const { removed, residual } = processForwardReferences(text, context); + return { scrubbable: removed, unscrubbable: residual }; +} + +// Scrub the safely-removable forward references from text, returning the cleaned text, what was removed, and +// any unscrubbable references that remain (a non-empty `residual` means the text still leaks the future). +export function scrubForwardReferences(text, context) { + return processForwardReferences(text, context); +} + +// A freeze point's frozen context is clean iff every provided text scrubs to zero residual forward references. +export function lintFrozenContext(texts, context) { + const list = Array.isArray(texts) ? texts : texts == null ? [] : [texts]; + const residual = []; + for (const text of list) { + residual.push(...processForwardReferences(text, context).residual); + } + return { ok: residual.length === 0, residual }; +} + +// Selection: a freeze point is calibration-worthy only with enough real history on both sides of T. +export function selectFreezePoint(candidate, thresholds) { + const minPriorCommits = Number.isInteger(thresholds?.minPriorCommits) ? thresholds.minPriorCommits : 0; + const minRevealedCommits = Number.isInteger(thresholds?.minRevealedCommits) + ? thresholds.minRevealedCommits + : 0; + const priorCommitCount = Number.isInteger(candidate?.priorCommitCount) ? candidate.priorCommitCount : 0; + const revealedCommitCount = Number.isInteger(candidate?.revealedCommitCount) + ? candidate.revealedCommitCount + : 0; + + const reasons = []; + if (priorCommitCount < minPriorCommits) reasons.push("insufficient_prior_history"); + if (revealedCommitCount < minRevealedCommits) reasons.push("insufficient_revealed_history"); + + return { eligible: reasons.length === 0, reasons, priorCommitCount, revealedCommitCount }; +} + +// Pool provenance: a freeze point whose last activity is at/after the calibration run's model cutoff is +// "recent" (higher memorization risk); everything else, including an unknown date, is "older". ISO-8601 +// timestamps sort lexicographically, so no clock is needed. +export function classifyRecencyPool(candidate, options) { + const modelCutoffIso = typeof options?.modelCutoffIso === "string" ? options.modelCutoffIso : ""; + const lastActivityAt = typeof candidate?.lastActivityAt === "string" ? candidate.lastActivityAt : ""; + if (!modelCutoffIso || !lastActivityAt) return "older"; + return lastActivityAt >= modelCutoffIso ? "recent" : "older"; +} + +// One-shot generator. Applies selection, then scrubs and lints the frozen context, then returns the frozen +// snapshot and the revealed post-T ground truth as SEPARATE bundles — never merged — so a caller persists and +// scopes them independently. An ineligible or un-scrubbable candidate is rejected without producing a task. +export function generateReplayTask(candidate, context, options) { + const selection = selectFreezePoint(candidate, options?.thresholds); + if (!selection.eligible) { + return { eligible: false, rejected: "selection", reasons: selection.reasons }; + } + + const frozenTexts = Array.isArray(candidate?.frozenContextTexts) ? candidate.frozenContextTexts : []; + const lint = lintFrozenContext(frozenTexts, context); + if (!lint.ok) { + return { eligible: false, rejected: "unscrubbable_forward_reference", residual: lint.residual }; + } + + const pool = classifyRecencyPool(candidate, options); + const scrubbedTexts = frozenTexts.map((text) => processForwardReferences(text, context).scrubbed); + + return { + eligible: true, + pool, + frozen: { + repo: typeof candidate?.repo === "string" ? candidate.repo : null, + commitT: typeof candidate?.commitT === "string" ? candidate.commitT : null, + contextTexts: scrubbedTexts, + }, + revealed: { + commitCount: selection.revealedCommitCount, + groundTruth: candidate?.revealedGroundTruth ?? null, + }, + }; +} diff --git a/packages/gittensory-miner/package.json b/packages/gittensory-miner/package.json index 5d107ef464..51b4353a02 100644 --- a/packages/gittensory-miner/package.json +++ b/packages/gittensory-miner/package.json @@ -31,7 +31,7 @@ "lib" ], "scripts": { - "build": "node --check bin/gittensory-miner.js && node --check lib/cli.js && node --check lib/deny-check.js && node --check lib/run-state-cli.js && node --check lib/update-check.js && node --check lib/opportunity-fanout.js && node --check lib/ci-poller.js && node --check lib/run-state.js && node --check lib/deny-hooks.js && node --check lib/event-ledger.js && node --check lib/event-ledger-cli.js && node --check lib/claim-ledger.js && node --check lib/claim-ledger-expiry.js && node --check lib/portfolio-queue.js && node --check lib/portfolio-queue-cli.js && node --check lib/portfolio-discovery.js && node --check lib/opportunity-ranker.js && node --check lib/plan-store.js && node --check lib/plan-store-cli.js && node --check lib/rejection-templates.js && node --check lib/governor-ledger.js && node --check lib/governor-ledger-cli.js && node --check lib/manage-status.js && node --check lib/manage-poll.js && node --check lib/status.js && node --check lib/replay-objective-anchor.js" + "build": "node --check bin/gittensory-miner.js && node --check lib/cli.js && node --check lib/deny-check.js && node --check lib/run-state-cli.js && node --check lib/update-check.js && node --check lib/opportunity-fanout.js && node --check lib/ci-poller.js && node --check lib/run-state.js && node --check lib/deny-hooks.js && node --check lib/event-ledger.js && node --check lib/event-ledger-cli.js && node --check lib/claim-ledger.js && node --check lib/claim-ledger-expiry.js && node --check lib/portfolio-queue.js && node --check lib/portfolio-queue-cli.js && node --check lib/portfolio-discovery.js && node --check lib/opportunity-ranker.js && node --check lib/plan-store.js && node --check lib/plan-store-cli.js && node --check lib/rejection-templates.js && node --check lib/governor-ledger.js && node --check lib/governor-ledger-cli.js && node --check lib/manage-status.js && node --check lib/manage-poll.js && node --check lib/status.js && node --check lib/replay-objective-anchor.js && node --check lib/replay-task-generation.js" }, "dependencies": { "@jsonbored/gittensory-engine": "0.1.0" diff --git a/test/unit/miner-replay-task-generation.test.ts b/test/unit/miner-replay-task-generation.test.ts new file mode 100644 index 0000000000..8fe77a2e50 --- /dev/null +++ b/test/unit/miner-replay-task-generation.test.ts @@ -0,0 +1,191 @@ +import { describe, expect, it } from "vitest"; +import { + FORWARD_REF_PLACEHOLDER, + RECENCY_POOLS, + classifyRecencyPool, + detectForwardReferences, + generateReplayTask, + lintFrozenContext, + scrubForwardReferences, + selectFreezePoint, +} from "../../packages/gittensory-miner/lib/replay-task-generation.js"; + +// Issues 1..100 and one commit SHA existed at T; issues 250/300 are revealed post-T ground truth. +const CONTEXT = { + knownIssueMax: 100, + knownCommitShas: ["abc1234def"], + revealedIssueNumbers: [250, 300], +}; + +describe("gittensory-miner leakage-safe replay task generation (#3011)", () => { + it("exposes a frozen recency-pool vocabulary and a stable placeholder", () => { + expect(Object.isFrozen(RECENCY_POOLS)).toBe(true); + expect(RECENCY_POOLS).toEqual(["recent", "older"]); + expect(FORWARD_REF_PLACEHOLDER).toBe("[redacted-forward-ref]"); + }); + + describe("detectForwardReferences", () => { + it("flags post-T #refs, deep-links, and unknown SHAs as scrubbable; keeps pre-T ones", () => { + const { scrubbable, unscrubbable } = detectForwardReferences( + "closes #300 (see https://github.com/o/r/pull/250) unlike old #42 at abc1234def then c0ffee99", + CONTEXT, + ); + const values = scrubbable.map((ref) => ref.value); + expect(values).toContain("#300"); // > knownIssueMax + expect(values).toContain("https://github.com/o/r/pull/250"); // deep-link > max + expect(values).toContain("c0ffee99"); // unknown SHA + expect(values).not.toContain("#42"); // <= knownIssueMax, pre-T + expect(scrubbable.some((ref) => ref.value === "abc1234def")).toBe(false); // known pre-T SHA kept + expect(unscrubbable).toEqual([]); + }); + + it("flags a bare integer that names a real post-T issue as unscrubbable", () => { + const { scrubbable, unscrubbable } = detectForwardReferences("the tally reached 300 last week", CONTEXT); + expect(scrubbable).toEqual([]); + expect(unscrubbable).toEqual([{ kind: "bare-issue-number", value: 300 }]); + }); + + it("does not misread a plain decimal number as a SHA", () => { + // 12345678 is 8 digits, in [0-9a-f] range, but has no hex letter → it is a number, not a hash. + const { scrubbable } = detectForwardReferences("build 12345678 shipped", { knownCommitShas: [] }); + expect(scrubbable).toEqual([]); + }); + }); + + describe("scrubForwardReferences", () => { + it("replaces every scrubbable forward reference with the placeholder and reports them", () => { + const result = scrubForwardReferences( + "fixed #300 via https://github.com/o/r/pull/250 in deadc0de", + { knownIssueMax: 100, knownCommitShas: [], revealedIssueNumbers: [] }, + ); + expect(result.scrubbed).toBe( + `fixed ${FORWARD_REF_PLACEHOLDER} via ${FORWARD_REF_PLACEHOLDER} in ${FORWARD_REF_PLACEHOLDER}`, + ); + expect(result.removed).toHaveLength(3); + expect(result.residual).toEqual([]); + }); + + it("leaves an unscrubbable bare issue number in place and surfaces it as residual", () => { + const result = scrubForwardReferences("the number 300 leaked here", CONTEXT); + expect(result.scrubbed).toBe("the number 300 leaked here"); // unchanged — cannot safely remove a bare int + expect(result.removed).toEqual([]); + expect(result.residual).toEqual([{ kind: "bare-issue-number", value: 300 }]); + }); + + it("leaves pre-T references untouched", () => { + const result = scrubForwardReferences("see #42 at abc1234def", CONTEXT); + expect(result.scrubbed).toBe("see #42 at abc1234def"); + expect(result.removed).toEqual([]); + }); + + it("coerces a non-string input to an empty scrub", () => { + expect(scrubForwardReferences(null, CONTEXT)).toEqual({ scrubbed: "", removed: [], residual: [] }); + }); + }); + + describe("lintFrozenContext", () => { + it("passes when every text scrubs to zero residual forward references", () => { + const lint = lintFrozenContext(["closes #300", "see https://github.com/o/r/issues/250"], CONTEXT); + expect(lint).toEqual({ ok: true, residual: [] }); + }); + + it("fails when any text carries an unscrubbable forward reference", () => { + const lint = lintFrozenContext(["harmless #42", "leaks 250 in prose"], CONTEXT); + expect(lint.ok).toBe(false); + expect(lint.residual).toEqual([{ kind: "bare-issue-number", value: 250 }]); + }); + }); + + describe("selectFreezePoint", () => { + it("is eligible only when prior and revealed history both clear the thresholds", () => { + const ok = selectFreezePoint( + { priorCommitCount: 50, revealedCommitCount: 10 }, + { minPriorCommits: 10, minRevealedCommits: 5 }, + ); + expect(ok).toEqual({ eligible: true, reasons: [], priorCommitCount: 50, revealedCommitCount: 10 }); + }); + + it("reports each unmet threshold and defaults missing counts to 0", () => { + const result = selectFreezePoint({}, { minPriorCommits: 10, minRevealedCommits: 5 }); + expect(result.eligible).toBe(false); + expect(result.reasons).toEqual(["insufficient_prior_history", "insufficient_revealed_history"]); + }); + }); + + describe("classifyRecencyPool", () => { + it("splits at the model cutoff and defaults unknown dates to 'older'", () => { + const opts = { modelCutoffIso: "2026-01-01T00:00:00Z" }; + expect(classifyRecencyPool({ lastActivityAt: "2026-06-01T00:00:00Z" }, opts)).toBe("recent"); + expect(classifyRecencyPool({ lastActivityAt: "2025-06-01T00:00:00Z" }, opts)).toBe("older"); + expect(classifyRecencyPool({ lastActivityAt: "2026-01-01T00:00:00Z" }, opts)).toBe("recent"); // boundary + expect(classifyRecencyPool({}, opts)).toBe("older"); // unknown activity date + expect(classifyRecencyPool({ lastActivityAt: "2026-06-01T00:00:00Z" }, {})).toBe("older"); // no cutoff + }); + }); + + describe("generateReplayTask", () => { + const eligible = { + repo: "o/r", + commitT: "abc1234def", + priorCommitCount: 50, + revealedCommitCount: 10, + lastActivityAt: "2026-06-01T00:00:00Z", + revealedGroundTruth: { merged: true, approach: "refactor" }, + }; + const options = { + thresholds: { minPriorCommits: 10, minRevealedCommits: 5 }, + modelCutoffIso: "2026-01-01T00:00:00Z", + }; + + it("produces a scrubbed frozen bundle and a SEPARATE revealed bundle for an eligible clean point", () => { + const task = generateReplayTask( + { ...eligible, frozenContextTexts: ["intro references #300 and old #12"] }, + CONTEXT, + options, + ); + if (!task.eligible) throw new Error(`expected eligible task, got ${JSON.stringify(task)}`); + expect(task.pool).toBe("recent"); + expect(task.frozen).toEqual({ + repo: "o/r", + commitT: "abc1234def", + contextTexts: [`intro references ${FORWARD_REF_PLACEHOLDER} and old #12`], + }); + // Ground truth lives only on the revealed side — never merged into the frozen bundle. + expect(task.revealed).toEqual({ commitCount: 10, groundTruth: { merged: true, approach: "refactor" } }); + expect(task.frozen).not.toHaveProperty("groundTruth"); + }); + + it("rejects a candidate that fails selection, without scrubbing", () => { + const task = generateReplayTask( + { priorCommitCount: 2, revealedCommitCount: 1, frozenContextTexts: ["#300"] }, + CONTEXT, + options, + ); + expect(task).toEqual({ + eligible: false, + rejected: "selection", + reasons: ["insufficient_prior_history", "insufficient_revealed_history"], + }); + }); + + it("rejects a candidate whose frozen context has an unscrubbable forward reference", () => { + const task = generateReplayTask( + { ...eligible, frozenContextTexts: ["the tally hit 250 last month"] }, + CONTEXT, + options, + ); + expect(task).toEqual({ + eligible: false, + rejected: "unscrubbable_forward_reference", + residual: [{ kind: "bare-issue-number", value: 250 }], + }); + }); + + it("is deterministic across repeated runs on identical inputs", () => { + const input = { ...eligible, frozenContextTexts: ["closes #300, keeps #7"] }; + expect(generateReplayTask(input, CONTEXT, options)).toEqual( + generateReplayTask(input, CONTEXT, options), + ); + }); + }); +});