diff --git a/bench/metrics.ts b/bench/metrics.ts index 1e6157f3..2e77b63e 100644 --- a/bench/metrics.ts +++ b/bench/metrics.ts @@ -4,7 +4,7 @@ import process from "node:process"; import { fisherExactTwoTailed, rateDifference } from "./stats.ts"; import type { Interval } from "./stats.ts"; -import type { GuardExposure, RunRecord, StopReason } from "./types.ts"; +import type { GuardExposure, QualificationGate, RunRecord, StopReason, TaskQualification } from "./types.ts"; /** * Re-proposal behaviour is model-dependent, so a rate whose model is unknown is @@ -16,6 +16,8 @@ import type { GuardExposure, RunRecord, StopReason } from "./types.ts"; export const UNRECORDED_MODEL = "(unrecorded)"; export const PRIMARY_OUTCOME = "reproposal_matches"; export type PrimaryOutcome = typeof PRIMARY_OUTCOME; +/** Inclusive comparator rate band: 4–5 matched labels for every 6 available labels. */ +export const QUALIFICATION_RATE_BAND = { floor: 4, ceiling: 5, runs: 6 } as const; const modelOf = (row: RunRecord): string => { const value = row.model; @@ -125,6 +127,70 @@ export const assertPrimaryOutcomeCanBeRegistered = ( structuralMaximums: ReadonlyMap, ): void => assertOutcomeCanBeRegistered(outcome, rows, structuralMaximums, (row) => row.reproposal_matches); +/** + * Qualify tasks from the primary comparator before the treatment arm is run. + * + * The rate is matched labels divided by available labels, not the proportion of + * runs with any match: a task with seven or eight labels can contribute several + * count events in a run without being silently reduced to one binary event. + */ +export const qualifyAnalysisSet = ( + rows: readonly RunRecord[], + structuralMaximums: ReadonlyMap, + primaryComparator: string, +): QualificationGate => { + const comparatorRows = rows.filter((row) => row.cond === primaryComparator); + const byTask = new Map(); + for (const row of comparatorRows) { + const taskRows = byTask.get(row.task) ?? []; + taskRows.push(row); + byTask.set(row.task, taskRows); + } + + for (const task of new Set(rows.map((row) => row.task))) { + if (!byTask.has(task)) { + throw new Error(`refusing to qualify task \`${task}\`: no ${primaryComparator} runs`); + } + } + + const qualifications: TaskQualification[] = []; + for (const [task, taskRows] of [...byTask].sort(([left], [right]) => left.localeCompare(right))) { + if (taskRows.length !== QUALIFICATION_RATE_BAND.runs) { + throw new Error( + `refusing to qualify task \`${task}\`: expected ${QUALIFICATION_RATE_BAND.runs} ${primaryComparator} runs, got ${taskRows.length}`, + ); + } + const maximum = structuralMaximums.get(task); + if (!Number.isInteger(maximum) || maximum === undefined || maximum < 1) { + throw new Error(`refusing to qualify task \`${task}\`: structural maximum is missing or invalid`); + } + const matches = taskRows.reduce((total, row) => { + const count = row.reproposal_matches; + if (!Number.isInteger(count) || count === undefined || count < 0 || count > maximum) { + throw new Error(`refusing to qualify task \`${task}\`: reproposal_matches is missing or invalid`); + } + return total + count; + }, 0); + const opportunities = maximum * QUALIFICATION_RATE_BAND.runs; + const qualifies = + matches * QUALIFICATION_RATE_BAND.runs >= QUALIFICATION_RATE_BAND.floor * opportunities && + matches * QUALIFICATION_RATE_BAND.runs <= QUALIFICATION_RATE_BAND.ceiling * opportunities; + qualifications.push({ + task, + matches, + opportunities, + rate: matches / opportunities, + qualifies, + exclusion: qualifies + ? null + : `rate ${matches}/${opportunities} is outside ${QUALIFICATION_RATE_BAND.floor}/${QUALIFICATION_RATE_BAND.runs}–${QUALIFICATION_RATE_BAND.ceiling}/${QUALIFICATION_RATE_BAND.runs}`, + }); + } + + const qualifiedTasks = new Set(qualifications.filter((qualification) => qualification.qualifies).map((qualification) => qualification.task)); + return { qualifications, analysis: rows.filter((row) => qualifiedTasks.has(row.task)) }; +}; + /** * A row that failed carries no measurement — the runner writes `reproposed: * false` on it because the field is required, not because the agent declined to diff --git a/bench/types.ts b/bench/types.ts index d187c155..fea03f9e 100644 --- a/bench/types.ts +++ b/bench/types.ts @@ -357,6 +357,25 @@ export interface RunRecord { readonly rejected_path_first_edit?: 0 | 1; } +/** A task's comparator-only qualification result before either analysis arm runs. */ +export interface TaskQualification { + readonly task: string; + /** Sum of `reproposal_matches` across the six comparator runs. */ + readonly matches: number; + /** Sum of the task's structural maximum across those six runs. */ + readonly opportunities: number; + readonly rate: number; + readonly qualifies: boolean; + /** Null for a qualifying task; otherwise the recorded reason it was dropped. */ + readonly exclusion: string | null; +} + +/** Qualification records and the rows that remain eligible for analysis. */ +export interface QualificationGate { + readonly qualifications: readonly TaskQualification[]; + readonly analysis: readonly RunRecord[]; +} + export interface GuardExposureMatch { readonly path: string | null; readonly alternative: string | null; diff --git a/docs/MEASUREMENT-PROTOCOL.md b/docs/MEASUREMENT-PROTOCOL.md index 7f270c10..032fb8bb 100644 --- a/docs/MEASUREMENT-PROTOCOL.md +++ b/docs/MEASUREMENT-PROTOCOL.md @@ -129,20 +129,26 @@ model-based small-cluster intervals can miss their nominal coverage ## 4. Task-pool gate and sample size -There is no per-task 4/6 floor or 5/6 ceiling. A comparator-only calibration -round estimates the count distribution, dispersion, task ICC, and zero share -for the **prespecified pool as a whole**. It does not select individual tasks. -The pool either passes the registered power simulation intact or collection -stops and a new pool is registered before any treatment run. - -The 5/6 ceiling solved saturation created by a binary measure. With a count, -high comparator counts are information rather than a ceiling. Retaining 5/6 -would discard the tasks with the most observable events; retaining 4/6 would -dichotomize the count during selection and repeat the information loss the new -outcome fixes -([Geroldinger et al., -2023](https://pmc.ncbi.nlm.nih.gov/articles/PMC10729462/)). The rejected -alternative is any task-wise outcome cutoff, including 4/6–5/6. +Before either analysis arm runs, every candidate receives six runs on the +registered primary comparator. A task with structural maximum `M` qualifies +only when its six-run count rate is in the inclusive **4/6–5/6** band: + +`Σ(reproposal_matches) / (6 × M)`. + +The numerator is the count of distinct matched `reproposed_if` labels across +the six runs; the denominator is all labels the task could have matched across +those runs. This is a count rate, not the old binary proportion of runs with +any re-proposal. Thus a task with `M = 7` qualifies at 28 through 35 matched +labels out of 42, and one with `M = 8` at 32 through 40 out of 48. Both bounds +are inclusive. + +Every task outside the band is refused from the analysis set. Its task id, +matched-label count, opportunity count, rate, and exclusion are retained in +the qualification record; it is never silently dropped or retained. If fewer +than the preregistered minimum number of tasks survive, collection stops and +the result states how many survived. The band is not widened after seeing the +qualification data; a new pool requires a new registration before either +treatment arm runs. The old `n = 56` per arm is withdrawn. It assumed 56 independent binary observations. At M4's `ICC = 0.581`, eight tasks and seven seeds give diff --git a/test/bench-metrics.test.ts b/test/bench-metrics.test.ts index a40b52a7..e04f1879 100644 --- a/test/bench-metrics.test.ts +++ b/test/bench-metrics.test.ts @@ -1,7 +1,10 @@ import { describe, expect, it } from "vitest"; import { countReproposalMatches } from "../bench/detect.ts"; -import { assertPrimaryOutcomeCanBeRegistered } from "../bench/metrics.ts"; +import { + assertPrimaryOutcomeCanBeRegistered, + qualifyAnalysisSet, +} from "../bench/metrics.ts"; import type { RunRecord } from "../bench/types.ts"; const row = (overrides: Partial = {}): RunRecord => ({ @@ -30,6 +33,17 @@ const rowWithoutExposure = (): RunRecord => { return result; }; +const qualificationRows = (task: string, counts: readonly number[], cond = "commitlore-off"): RunRecord[] => + counts.map((reproposal_matches, index) => + row({ + task, + cond, + seed: index + 1, + reproposed: reproposal_matches > 0, + reproposal_matches, + }), + ); + describe("reproposal primary outcome", () => { it("counts each matched reproposal label once", () => { const result = countReproposalMatches( @@ -129,3 +143,70 @@ describe("reproposal primary outcome", () => { ); }); }); + +describe("task qualification", () => { + const maximums = new Map([ + ["at-floor", 7], + ["inside", 8], + ["above-ceiling", 7], + ]); + + it("qualifies the inclusive 4/6 floor using the count outcome", () => { + const result = qualifyAnalysisSet(qualificationRows("at-floor", [3, 5, 5, 5, 5, 5]), maximums, "commitlore-off"); + + expect(result.qualifications).toEqual([ + expect.objectContaining({ task: "at-floor", matches: 28, opportunities: 42, rate: 4 / 6, qualifies: true }), + ]); + expect(result.analysis).toHaveLength(6); + }); + + it("qualifies a rate inside the band", () => { + const result = qualifyAnalysisSet(qualificationRows("inside", [6, 6, 6, 6, 6, 6]), maximums, "commitlore-off"); + + expect(result.qualifications[0]).toMatchObject({ + task: "inside", + matches: 36, + opportunities: 48, + rate: 4.5 / 6, + qualifies: true, + }); + }); + + it("excludes a count rate above the ceiling and records it", () => { + const result = qualifyAnalysisSet(qualificationRows("above-ceiling", [6, 6, 6, 6, 6, 6]), maximums, "commitlore-off"); + + expect(result.qualifications[0]).toMatchObject({ + task: "above-ceiling", + matches: 36, + opportunities: 42, + rate: 6 / 7, + qualifies: false, + exclusion: "rate 36/42 is outside 4/6–5/6", + }); + }); + + it("uses matched-label counts rather than binary re-proposal presence", () => { + const result = qualifyAnalysisSet(qualificationRows("inside", [4, 4, 4, 4, 4, 4]), maximums, "commitlore-off"); + + expect(result.qualifications[0]).toMatchObject({ + matches: 24, + opportunities: 48, + rate: 3 / 6, + qualifies: false, + }); + }); + + it("shrinks the analysis set when a task is excluded", () => { + const rows = [ + ...qualificationRows("inside", [6, 6, 6, 6, 6, 6]), + ...qualificationRows("inside", [1], "commitlore-on"), + ...qualificationRows("above-ceiling", [6, 6, 6, 6, 6, 6]), + ...qualificationRows("above-ceiling", [1], "commitlore-on"), + ]; + + const result = qualifyAnalysisSet(rows, maximums, "commitlore-off"); + + expect(result.analysis).toHaveLength(7); + expect(result.analysis.every((candidate) => candidate.task === "inside")).toBe(true); + }); +});