Make task qualification count-rate two-sided

bench/metrics.ts

@@ -4,7 +4,7 @@ import process from "node:process";
 
 import { fisherExactTwoTailed, rateDifference } from "./stats.ts";
 import type { Interval } from "./stats.ts";
-import type { GuardExposure, RunRecord, StopReason } from "./types.ts";
+import type { GuardExposure, QualificationGate, RunRecord, StopReason, TaskQualification } from "./types.ts";
 
 /**
  * Re-proposal behaviour is model-dependent, so a rate whose model is unknown is
@@ -16,6 +16,8 @@ import type { GuardExposure, RunRecord, StopReason } from "./types.ts";
 export const UNRECORDED_MODEL = "(unrecorded)";
 export const PRIMARY_OUTCOME = "reproposal_matches";
 export type PrimaryOutcome = typeof PRIMARY_OUTCOME;
+/** Inclusive comparator rate band: 4–5 matched labels for every 6 available labels. */
+export const QUALIFICATION_RATE_BAND = { floor: 4, ceiling: 5, runs: 6 } as const;
 
 const modelOf = (row: RunRecord): string => {
   const value = row.model;
@@ -125,6 +127,70 @@ export const assertPrimaryOutcomeCanBeRegistered = (
   structuralMaximums: ReadonlyMap<string, number>,
 ): void => assertOutcomeCanBeRegistered(outcome, rows, structuralMaximums, (row) => row.reproposal_matches);
 
+/**
+ * Qualify tasks from the primary comparator before the treatment arm is run.
+ *
+ * The rate is matched labels divided by available labels, not the proportion of
+ * runs with any match: a task with seven or eight labels can contribute several
+ * count events in a run without being silently reduced to one binary event.
+ */
+export const qualifyAnalysisSet = (
+  rows: readonly RunRecord[],
+  structuralMaximums: ReadonlyMap<string, number>,
+  primaryComparator: string,
+): QualificationGate => {
+  const comparatorRows = rows.filter((row) => row.cond === primaryComparator);
+  const byTask = new Map<string, RunRecord[]>();
+  for (const row of comparatorRows) {
+    const taskRows = byTask.get(row.task) ?? [];
+    taskRows.push(row);
+    byTask.set(row.task, taskRows);
+  }
+
+  for (const task of new Set(rows.map((row) => row.task))) {
+    if (!byTask.has(task)) {
+      throw new Error(`refusing to qualify task \`${task}\`: no ${primaryComparator} runs`);
+    }
+  }
+
+  const qualifications: TaskQualification[] = [];
+  for (const [task, taskRows] of [...byTask].sort(([left], [right]) => left.localeCompare(right))) {
+    if (taskRows.length !== QUALIFICATION_RATE_BAND.runs) {
+      throw new Error(
+        `refusing to qualify task \`${task}\`: expected ${QUALIFICATION_RATE_BAND.runs} ${primaryComparator} runs, got ${taskRows.length}`,
+      );
+    }
+    const maximum = structuralMaximums.get(task);
+    if (!Number.isInteger(maximum) || maximum === undefined || maximum < 1) {
+      throw new Error(`refusing to qualify task \`${task}\`: structural maximum is missing or invalid`);
+    }
+    const matches = taskRows.reduce((total, row) => {
+      const count = row.reproposal_matches;
+      if (!Number.isInteger(count) || count === undefined || count < 0 || count > maximum) {
+        throw new Error(`refusing to qualify task \`${task}\`: reproposal_matches is missing or invalid`);
+      }
+      return total + count;
+    }, 0);
+    const opportunities = maximum * QUALIFICATION_RATE_BAND.runs;
+    const qualifies =
+      matches * QUALIFICATION_RATE_BAND.runs >= QUALIFICATION_RATE_BAND.floor * opportunities &&
+      matches * QUALIFICATION_RATE_BAND.runs <= QUALIFICATION_RATE_BAND.ceiling * opportunities;
+    qualifications.push({
+      task,
+      matches,
+      opportunities,
+      rate: matches / opportunities,
+      qualifies,
+      exclusion: qualifies
+        ? null
+        : `rate ${matches}/${opportunities} is outside ${QUALIFICATION_RATE_BAND.floor}/${QUALIFICATION_RATE_BAND.runs}–${QUALIFICATION_RATE_BAND.ceiling}/${QUALIFICATION_RATE_BAND.runs}`,
+    });
+  }
+
+  const qualifiedTasks = new Set(qualifications.filter((qualification) => qualification.qualifies).map((qualification) => qualification.task));
+  return { qualifications, analysis: rows.filter((row) => qualifiedTasks.has(row.task)) };
+};
+
 /**
  * A row that failed carries no measurement — the runner writes `reproposed:
  * false` on it because the field is required, not because the agent declined to

bench/types.ts

@@ -357,6 +357,25 @@ export interface RunRecord {
   readonly rejected_path_first_edit?: 0 | 1;
 }
 
+/** A task's comparator-only qualification result before either analysis arm runs. */
+export interface TaskQualification {
+  readonly task: string;
+  /** Sum of `reproposal_matches` across the six comparator runs. */
+  readonly matches: number;
+  /** Sum of the task's structural maximum across those six runs. */
+  readonly opportunities: number;
+  readonly rate: number;
+  readonly qualifies: boolean;
+  /** Null for a qualifying task; otherwise the recorded reason it was dropped. */
+  readonly exclusion: string | null;
+}
+
+/** Qualification records and the rows that remain eligible for analysis. */
+export interface QualificationGate {
+  readonly qualifications: readonly TaskQualification[];
+  readonly analysis: readonly RunRecord[];
+}
+
 export interface GuardExposureMatch {
   readonly path: string | null;
   readonly alternative: string | null;

docs/MEASUREMENT-PROTOCOL.md

@@ -129,20 +129,26 @@ model-based small-cluster intervals can miss their nominal coverage
 
 ## 4. Task-pool gate and sample size
 
-There is no per-task 4/6 floor or 5/6 ceiling. A comparator-only calibration
-round estimates the count distribution, dispersion, task ICC, and zero share
-for the **prespecified pool as a whole**. It does not select individual tasks.
-The pool either passes the registered power simulation intact or collection
-stops and a new pool is registered before any treatment run.
-
-The 5/6 ceiling solved saturation created by a binary measure. With a count,
-high comparator counts are information rather than a ceiling. Retaining 5/6
-would discard the tasks with the most observable events; retaining 4/6 would
-dichotomize the count during selection and repeat the information loss the new
-outcome fixes
-([Geroldinger et al.,
-2023](https://pmc.ncbi.nlm.nih.gov/articles/PMC10729462/)). The rejected
-alternative is any task-wise outcome cutoff, including 4/6–5/6.
+Before either analysis arm runs, every candidate receives six runs on the
+registered primary comparator. A task with structural maximum `M` qualifies
+only when its six-run count rate is in the inclusive **4/6–5/6** band:
+
+`Σ(reproposal_matches) / (6 × M)`.
+
+The numerator is the count of distinct matched `reproposed_if` labels across
+the six runs; the denominator is all labels the task could have matched across
+those runs. This is a count rate, not the old binary proportion of runs with
+any re-proposal. Thus a task with `M = 7` qualifies at 28 through 35 matched
+labels out of 42, and one with `M = 8` at 32 through 40 out of 48. Both bounds
+are inclusive.
+
+Every task outside the band is refused from the analysis set. Its task id,
+matched-label count, opportunity count, rate, and exclusion are retained in
+the qualification record; it is never silently dropped or retained. If fewer
+than the preregistered minimum number of tasks survive, collection stops and
+the result states how many survived. The band is not widened after seeing the
+qualification data; a new pool requires a new registration before either
+treatment arm runs.
 
 The old `n = 56` per arm is withdrawn. It assumed 56 independent binary
 observations. At M4's `ICC = 0.581`, eight tasks and seven seeds give

test/bench-metrics.test.ts

@@ -1,7 +1,10 @@
 import { describe, expect, it } from "vitest";
 
 import { countReproposalMatches } from "../bench/detect.ts";
-import { assertPrimaryOutcomeCanBeRegistered } from "../bench/metrics.ts";
+import {
+  assertPrimaryOutcomeCanBeRegistered,
+  qualifyAnalysisSet,
+} from "../bench/metrics.ts";
 import type { RunRecord } from "../bench/types.ts";
 
 const row = (overrides: Partial<RunRecord> = {}): RunRecord => ({
@@ -30,6 +33,17 @@ const rowWithoutExposure = (): RunRecord => {
   return result;
 };
 
+const qualificationRows = (task: string, counts: readonly number[], cond = "commitlore-off"): RunRecord[] =>
+  counts.map((reproposal_matches, index) =>
+    row({
+      task,
+      cond,
+      seed: index + 1,
+      reproposed: reproposal_matches > 0,
+      reproposal_matches,
+    }),
+  );
+
 describe("reproposal primary outcome", () => {
   it("counts each matched reproposal label once", () => {
     const result = countReproposalMatches(
@@ -129,3 +143,70 @@ describe("reproposal primary outcome", () => {
     );
   });
 });
+
+describe("task qualification", () => {
+  const maximums = new Map([
+    ["at-floor", 7],
+    ["inside", 8],
+    ["above-ceiling", 7],
+  ]);
+
+  it("qualifies the inclusive 4/6 floor using the count outcome", () => {
+    const result = qualifyAnalysisSet(qualificationRows("at-floor", [3, 5, 5, 5, 5, 5]), maximums, "commitlore-off");
+
+    expect(result.qualifications).toEqual([
+      expect.objectContaining({ task: "at-floor", matches: 28, opportunities: 42, rate: 4 / 6, qualifies: true }),
+    ]);
+    expect(result.analysis).toHaveLength(6);
+  });
+
+  it("qualifies a rate inside the band", () => {
+    const result = qualifyAnalysisSet(qualificationRows("inside", [6, 6, 6, 6, 6, 6]), maximums, "commitlore-off");
+
+    expect(result.qualifications[0]).toMatchObject({
+      task: "inside",
+      matches: 36,
+      opportunities: 48,
+      rate: 4.5 / 6,
+      qualifies: true,
+    });
+  });
+
+  it("excludes a count rate above the ceiling and records it", () => {
+    const result = qualifyAnalysisSet(qualificationRows("above-ceiling", [6, 6, 6, 6, 6, 6]), maximums, "commitlore-off");
+
+    expect(result.qualifications[0]).toMatchObject({
+      task: "above-ceiling",
+      matches: 36,
+      opportunities: 42,
+      rate: 6 / 7,
+      qualifies: false,
+      exclusion: "rate 36/42 is outside 4/6–5/6",
+    });
+  });
+
+  it("uses matched-label counts rather than binary re-proposal presence", () => {
+    const result = qualifyAnalysisSet(qualificationRows("inside", [4, 4, 4, 4, 4, 4]), maximums, "commitlore-off");
+
+    expect(result.qualifications[0]).toMatchObject({
+      matches: 24,
+      opportunities: 48,
+      rate: 3 / 6,
+      qualifies: false,
+    });
+  });
+
+  it("shrinks the analysis set when a task is excluded", () => {
+    const rows = [
+      ...qualificationRows("inside", [6, 6, 6, 6, 6, 6]),
+      ...qualificationRows("inside", [1], "commitlore-on"),
+      ...qualificationRows("above-ceiling", [6, 6, 6, 6, 6, 6]),
+      ...qualificationRows("above-ceiling", [1], "commitlore-on"),
+    ];
+
+    const result = qualifyAnalysisSet(rows, maximums, "commitlore-off");
+
+    expect(result.analysis).toHaveLength(7);
+    expect(result.analysis.every((candidate) => candidate.task === "inside")).toBe(true);
+  });
+});