diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8207cf32..30f5aa0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1603,8 +1603,27 @@ jobs: - name: CDEB evaluator OCI isolation matrix run: npx vitest run test/cdeb-evaluator-oci-matrix.test.ts + # A guard that does not fail under a mutation does not bind its claimed + # property. This deliberately runs separately from `check`: one Vitest + # process per control is expensive, and a missing or inert control must be + # visible as its own failing job rather than disappearing into the broad + # suite. This ratchet fails only when measurement regresses from the recorded + # baseline: its current gaps are thirteen exclusion-index kinds, one inert + # personal-path scan, and one unavailable exclusion-index control. + guard-mutations: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22.23.2' + cache: npm + - run: npm ci + - name: CDEB guard mutations bind their claimed properties + run: node scripts/guard-mutations.mjs + # A fan-in so branch protection can require ONE stable context instead of - # eleven, several of which are matrix-interpolated. + # twelve, several of which are matrix-interpolated. # # `check (22.23.2)` carries the exact pinned floor, and the comment on that # matrix says why: a declared floor must name the exact release CI executes. @@ -1634,6 +1653,7 @@ jobs: - install-script - install-ps1 - oci-matrix + - guard-mutations runs-on: ubuntu-latest steps: - name: every job this gate fans in from succeeded @@ -1647,6 +1667,7 @@ jobs: install-script=${{ needs.install-script.result }} install-ps1=${{ needs.install-ps1.result }} oci-matrix=${{ needs.oci-matrix.result }} + guard-mutations=${{ needs.guard-mutations.result }} run: | set -eu failed=0 diff --git a/bench/cdeb/guards/baseline.json b/bench/cdeb/guards/baseline.json new file mode 100644 index 00000000..3a8021e4 --- /dev/null +++ b/bench/cdeb/guards/baseline.json @@ -0,0 +1,86 @@ +{ + "version": 1, + "properties": [ + { "guard_id": "literature-lock-requires-complete-evidence", "outcome": "bound" }, + { "guard_id": "literature-lock-refuses-circular-justification", "outcome": "bound" }, + { "guard_id": "transition-ledger-refuses-cross-study-row", "outcome": "bound" }, + { "guard_id": "candidate-identity-contract", "outcome": "bound" }, + { + "guard_id": "exclusion-index-blocks-study-id-cdeb-v1", + "outcome": "unavailable", + "reason": "The census matches exclusion rows by value only, so disabling a study-id kind check cannot change the claim that this candidate is ineligible." + }, + { + "guard_id": "exclusion-index-blocks-study-id-cdeb-p-01", + "outcome": "uncovered", + "reason": "The census matches exclusion rows by value only, so this study identity is not represented in the candidate values that the claim would need to make ineligible." + }, + { + "guard_id": "exclusion-index-blocks-study-id-cdeb-fresh-v3", + "outcome": "uncovered", + "reason": "The census matches exclusion rows by value only, so this predecessor study identity is not represented in the candidate values that the claim would need to make ineligible." + }, + { + "guard_id": "exclusion-index-blocks-task-id-pricing-admin-quote", + "outcome": "uncovered", + "reason": "The census matches exclusion rows by value only, so this task identity is not represented in the candidate values that the claim would need to make ineligible." + }, + { + "guard_id": "exclusion-index-blocks-task-id-pending-rm-force", + "outcome": "uncovered", + "reason": "The census matches exclusion rows by value only, so this task identity is not represented in the candidate values that the claim would need to make ineligible." + }, + { + "guard_id": "exclusion-index-blocks-prompt-hash", + "outcome": "uncovered", + "reason": "The census matches exclusion rows by value only, so this prompt artifact is not represented in the candidate values that the claim would need to make ineligible." + }, + { + "guard_id": "exclusion-index-blocks-fixture-hash", + "outcome": "uncovered", + "reason": "The census matches exclusion rows by value only, so this evaluator fixture artifact is not represented in the candidate values that the claim would need to make ineligible." + }, + { + "guard_id": "exclusion-index-blocks-candidate-id", + "outcome": "uncovered", + "reason": "The census can match this candidate identifier by value, but the registry has no mutation that proves the claim makes it ineligible." + }, + { "guard_id": "exclusion-index-blocks-record-id", "outcome": "bound" }, + { + "guard_id": "exclusion-index-blocks-oracle-fixture-hash", + "outcome": "uncovered", + "reason": "The census matches exclusion rows by value only, so this oracle fixture artifact is not represented in the candidate values that the claim would need to make ineligible." + }, + { + "guard_id": "exclusion-index-blocks-randomization-id", + "outcome": "uncovered", + "reason": "The census matches exclusion rows by value only, so this randomization artifact is not represented in the candidate values that the claim would need to make ineligible." + }, + { + "guard_id": "exclusion-index-blocks-trajectory-id", + "outcome": "uncovered", + "reason": "The census matches exclusion rows by value only, so this trajectory artifact is not represented in the candidate values that the claim would need to make ineligible." + }, + { + "guard_id": "exclusion-index-blocks-result-row-id", + "outcome": "uncovered", + "reason": "The census matches exclusion rows by value only, so this result-row artifact is not represented in the candidate values that the claim would need to make ineligible." + }, + { + "guard_id": "exclusion-index-blocks-benchmark-authored-record", + "outcome": "uncovered", + "reason": "The census can match this source-record identity by value, but the registry has no mutation that proves the claim makes its candidate ineligible." + }, + { + "guard_id": "exclusion-index-blocks-publicly-answer-exposed-decision", + "outcome": "uncovered", + "reason": "The census can match this decision's source-record identity by value, but the registry has no mutation that proves the claim makes its candidate ineligible." + }, + { "guard_id": "frozen-bundle-digest-is-verified", "outcome": "bound" }, + { + "guard_id": "personal-paths-are-absent-from-active-material", + "outcome": "inert", + "reason": "The current clean tree still passes when the personal-path scan is bypassed, so the claim has no constructed personal-path control." + } + ] +} diff --git a/bench/cdeb/guards/registry.json b/bench/cdeb/guards/registry.json new file mode 100644 index 00000000..ec553bff --- /dev/null +++ b/bench/cdeb/guards/registry.json @@ -0,0 +1,287 @@ +{ + "version": 1, + "properties": [ + { + "guard_id": "literature-lock-requires-complete-evidence", + "claim": "A study cannot enter LITERATURE_LOCKED unless its source record, resolved claims, and all three independent audit artifacts are present.", + "test_file": "test/cdeb-v3-governance.test.ts", + "test_name": "refuses LITERATURE_LOCKED with 0 sources", + "mutations": [ + { + "mutation_id": "literature-lock-allows-zero-sources", + "file": "bench/cdeb/ledger.ts", + "find": "if (sources === 0) failures.push(\"source-lock sources must be > 0 (measured 0)\");", + "replace": "if (false) failures.push(\"source-lock sources must be > 0 (measured 0)\");", + "must_fail_test": true, + "why": "A study with no locked sources would be allowed to advance." + }, + { + "mutation_id": "literature-lock-allows-zero-claims", + "file": "bench/cdeb/ledger.ts", + "find": "if (claims === 0) failures.push(\"evidence-matrix claims must be > 0 (measured 0)\");", + "replace": "if (false) failures.push(\"evidence-matrix claims must be > 0 (measured 0)\");", + "must_fail_test": true, + "why": "A study with no adjudicated claims would be allowed to advance.", + "test_name": "refuses LITERATURE_LOCKED with 0 claims" + }, + { + "mutation_id": "literature-lock-allows-missing-lit-a", + "file": "bench/cdeb/ledger.ts", + "find": "if (audits.litA === 0) failures.push(\"literature/audits LIT-A artifact must exist (measured 0)\");", + "replace": "if (false) failures.push(\"literature/audits LIT-A artifact must exist (measured 0)\");", + "must_fail_test": true, + "why": "The required independent LIT-A audit could be absent.", + "test_name": "refuses LITERATURE_LOCKED with missing LIT-A audit" + }, + { + "mutation_id": "literature-lock-allows-missing-lit-b", + "file": "bench/cdeb/ledger.ts", + "find": "if (audits.litB === 0) failures.push(\"literature/audits LIT-B artifact must exist (measured 0)\");", + "replace": "if (false) failures.push(\"literature/audits LIT-B artifact must exist (measured 0)\");", + "must_fail_test": true, + "why": "The required independent LIT-B audit could be absent.", + "test_name": "refuses LITERATURE_LOCKED with missing LIT-B audit" + }, + { + "mutation_id": "literature-lock-allows-missing-adjudication", + "file": "bench/cdeb/ledger.ts", + "find": "if (audits.adjudication === 0) failures.push(\"literature/audits adjudication artifact must exist (measured 0)\");", + "replace": "if (false) failures.push(\"literature/audits adjudication artifact must exist (measured 0)\");", + "must_fail_test": true, + "why": "The final adjudication artifact could be absent.", + "test_name": "refuses LITERATURE_LOCKED with missing adjudication audit" + }, + { + "mutation_id": "literature-lock-allows-unresolved-claims", + "file": "bench/cdeb/ledger.ts", + "find": "if (unresolved !== 0) failures.push(`evidence-matrix unresolved claims must be 0 (measured ${unresolved})`);", + "replace": "if (false) failures.push(`evidence-matrix unresolved claims must be 0 (measured ${unresolved})`);", + "must_fail_test": true, + "why": "A study with unresolved evidence would be allowed to advance.", + "test_name": "refuses LITERATURE_LOCKED with an unresolved claim" + } + ] + }, + { + "guard_id": "literature-lock-refuses-circular-justification", + "claim": "A study cannot justify entering a state by claiming that the destination state is already recorded.", + "test_file": "test/cdeb-v3-governance.test.ts", + "test_name": "refuses UNKNOWN and a circular destination check even with real artifact-shaped inputs", + "mutations": [ + { + "mutation_id": "circular-check-is-never-detected", + "file": "bench/cdeb/ledger.ts", + "find": "check.toUpperCase().includes(transition.to),", + "replace": "false,", + "must_fail_test": true, + "why": "A transition whose only stated check is its destination would be accepted." + } + ] + }, + { + "guard_id": "transition-ledger-refuses-cross-study-row", + "claim": "A study's transition ledger cannot accept an artifact from another study.", + "test_file": "test/cdeb-v3-lifecycle.test.ts", + "test_name": "refuses a row from another study and names both study ids", + "mutations": [ + { + "mutation_id": "foreign-transition-study-is-accepted", + "file": "bench/cdeb/ledger.ts", + "find": "if (value.study_id !== expected) {", + "replace": "if (false) {", + "must_fail_test": true, + "why": "A valid-looking transition from a different study could enter this ledger." + } + ] + }, + { + "guard_id": "candidate-identity-contract", + "claim": "Only rows belonging to this v3 study, tied to its frozen repository snapshot, and free of a legacy benchmark identity may enter its candidate registry.", + "test_file": "test/cdeb-v3-census.test.ts", + "test_name": "refuses re-adding benchmark: cdeb-v1 to a v3 row by naming expected and received identity", + "mutations": [ + { + "mutation_id": "legacy-benchmark-identity-is-accepted", + "file": "bench/cdeb/freeze/census.ts", + "find": "if (\"benchmark\" in row) throw identityError(\"benchmark (must be absent)\", \"absent\", row.benchmark);", + "replace": "if (false) throw identityError(\"benchmark (must be absent)\", \"absent\", row.benchmark);", + "must_fail_test": true, + "why": "A legacy benchmark row could be relabelled as a v3 candidate." + }, + { + "mutation_id": "foreign-candidate-study-is-accepted", + "file": "bench/cdeb/freeze/census.ts", + "find": "if (row.study_id !== studyId) throw identityError(\"study_id\", studyId, row.study_id);", + "replace": "if (false) throw identityError(\"study_id\", studyId, row.study_id);", + "must_fail_test": true, + "why": "A candidate belonging to another study could enter this registry.", + "test_name": "refuses a foreign v3 study_id by naming expected and received identity" + }, + { + "mutation_id": "unfrozen-candidate-snapshot-is-accepted", + "file": "bench/cdeb/freeze/census.ts", + "find": "if (row.source_snapshot_sha !== expectedSnapshot) throw identityError(\"source_snapshot_sha\", expectedSnapshot, row.source_snapshot_sha);", + "replace": "if (false) throw identityError(\"source_snapshot_sha\", expectedSnapshot, row.source_snapshot_sha);", + "must_fail_test": true, + "why": "A candidate could be attributed to a different source snapshot.", + "test_name": "refuses a source snapshot that differs from its frozen repository snapshot" + } + ] + }, + { + "guard_id": "exclusion-index-blocks-study-id-cdeb-v1", + "claim": "A candidate associated with the legacy study cdeb-v1 cannot reach the registry as anything but ineligible.", + "test_file": "test/cdeb-v3-census.test.ts", + "test_name": "keeps a Record-Id named by the index as an ineligible visible row", + "mutations": [ + { + "mutation_id": "study-id-kind-is-not-read", + "file": "bench/cdeb/freeze/census.ts", + "find": "entry.kind === \"study-id\"", + "replace": "false", + "must_fail_test": true, + "why": "No study-id-specific comparison exists: the index matcher never reads kind, so this property has no expressible control." + } + ] + }, + { + "guard_id": "exclusion-index-blocks-study-id-cdeb-p-01", + "claim": "A candidate associated with the legacy study cdeb-p-01 cannot reach the registry as anything but ineligible.", + "test_file": "test/cdeb-v3-census.test.ts", + "test_name": "keeps a Record-Id named by the index as an ineligible visible row", + "mutations": [] + }, + { + "guard_id": "exclusion-index-blocks-study-id-cdeb-fresh-v3", + "claim": "A candidate associated with the invalidated predecessor study cannot reach the registry as anything but ineligible.", + "test_file": "test/cdeb-v3-census.test.ts", + "test_name": "keeps a Record-Id named by the index as an ineligible visible row", + "mutations": [] + }, + { + "guard_id": "exclusion-index-blocks-task-id-pricing-admin-quote", + "claim": "A candidate associated with the legacy task pricing-admin-quote cannot reach the registry as anything but ineligible.", + "test_file": "test/cdeb-v3-census.test.ts", + "test_name": "keeps a Record-Id named by the index as an ineligible visible row", + "mutations": [] + }, + { + "guard_id": "exclusion-index-blocks-task-id-pending-rm-force", + "claim": "A candidate associated with the legacy task pending-rm-force cannot reach the registry as anything but ineligible.", + "test_file": "test/cdeb-v3-census.test.ts", + "test_name": "keeps a Record-Id named by the index as an ineligible visible row", + "mutations": [] + }, + { + "guard_id": "exclusion-index-blocks-prompt-hash", + "claim": "A candidate associated with the recorded legacy prompt cannot reach the registry as anything but ineligible.", + "test_file": "test/cdeb-v3-census.test.ts", + "test_name": "keeps a Record-Id named by the index as an ineligible visible row", + "mutations": [] + }, + { + "guard_id": "exclusion-index-blocks-fixture-hash", + "claim": "A candidate associated with the recorded legacy evaluator fixture cannot reach the registry as anything but ineligible.", + "test_file": "test/cdeb-v3-census.test.ts", + "test_name": "keeps a Record-Id named by the index as an ineligible visible row", + "mutations": [] + }, + { + "guard_id": "exclusion-index-blocks-candidate-id", + "claim": "The candidate r-d0004gatecensus cannot reach the registry as anything but ineligible.", + "test_file": "test/cdeb-v3-census.test.ts", + "test_name": "keeps a Record-Id named by the index as an ineligible visible row", + "mutations": [] + }, + { + "guard_id": "exclusion-index-blocks-record-id", + "claim": "A candidate carrying an excluded source record identity cannot reach the registry as anything but ineligible.", + "test_file": "test/cdeb-v3-census.test.ts", + "test_name": "keeps a Record-Id named by the index as an ineligible visible row", + "mutations": [ + { + "mutation_id": "record-id-match-is-disabled", + "file": "bench/cdeb/freeze/census.ts", + "find": "return index.exclusions.filter((entry) => values.has(entry.value));", + "replace": "return [];", + "must_fail_test": true, + "why": "An excluded source record would remain pending instead of becoming visibly ineligible." + } + ] + }, + { + "guard_id": "exclusion-index-blocks-oracle-fixture-hash", + "claim": "A candidate associated with the recorded legacy oracle fixture cannot reach the registry as anything but ineligible.", + "test_file": "test/cdeb-v3-census.test.ts", + "test_name": "keeps a Record-Id named by the index as an ineligible visible row", + "mutations": [] + }, + { + "guard_id": "exclusion-index-blocks-randomization-id", + "claim": "A candidate associated with the recorded legacy randomization block cannot reach the registry as anything but ineligible.", + "test_file": "test/cdeb-v3-census.test.ts", + "test_name": "keeps a Record-Id named by the index as an ineligible visible row", + "mutations": [] + }, + { + "guard_id": "exclusion-index-blocks-trajectory-id", + "claim": "A candidate associated with the recorded legacy trajectory cannot reach the registry as anything but ineligible.", + "test_file": "test/cdeb-v3-census.test.ts", + "test_name": "keeps a Record-Id named by the index as an ineligible visible row", + "mutations": [] + }, + { + "guard_id": "exclusion-index-blocks-result-row-id", + "claim": "A candidate associated with the recorded legacy result row cannot reach the registry as anything but ineligible.", + "test_file": "test/cdeb-v3-census.test.ts", + "test_name": "keeps a Record-Id named by the index as an ineligible visible row", + "mutations": [] + }, + { + "guard_id": "exclusion-index-blocks-benchmark-authored-record", + "claim": "A candidate associated with the recorded benchmark-authored decision cannot reach the registry as anything but ineligible.", + "test_file": "test/cdeb-v3-census.test.ts", + "test_name": "keeps a Record-Id named by the index as an ineligible visible row", + "mutations": [] + }, + { + "guard_id": "exclusion-index-blocks-publicly-answer-exposed-decision", + "claim": "A candidate associated with the publicly answer-exposed decision cannot reach the registry as anything but ineligible.", + "test_file": "test/cdeb-v3-census.test.ts", + "test_name": "keeps a Record-Id named by the index as an ineligible visible row", + "mutations": [] + }, + { + "guard_id": "frozen-bundle-digest-is-verified", + "claim": "A repository bundle whose bytes differ from its frozen identity cannot be materialized for a study run.", + "test_file": "test/cdeb-materializer.test.ts", + "test_name": "refuses a bundle whose bytes do not match the freeze", + "mutations": [ + { + "mutation_id": "tampered-bundle-is-materialized", + "file": "bench/cdeb/freeze/repository-bundle.ts", + "find": "if (actualBundle !== identity.bundle_sha256) {", + "replace": "if (false) {", + "must_fail_test": true, + "why": "Tampered bundle bytes would be handed to the materializer." + } + ] + }, + { + "guard_id": "personal-paths-are-absent-from-active-material", + "claim": "Runnable code and active-study artifacts contain no personal filesystem path.", + "test_file": "test/cdeb-v3-census.test.ts", + "test_name": "forbids personal paths in runnable code and active-study artifacts", + "mutations": [ + { + "mutation_id": "personal-path-scan-is-bypassed", + "file": "test/cdeb-v3-census.test.ts", + "find": "const personalPaths = scanned.filter((path) => readFileSync(path, \"utf8\").includes(\"/Users/\"));", + "replace": "const personalPaths: string[] = [];", + "must_fail_test": true, + "why": "The current clean tree still passes when the scan is bypassed, so this test has no constructed personal-path control." + } + ] + } + ] +} \ No newline at end of file diff --git a/scripts/check-exact-head-ci.mjs b/scripts/check-exact-head-ci.mjs index ef01b141..3ff542a7 100644 --- a/scripts/check-exact-head-ci.mjs +++ b/scripts/check-exact-head-ci.mjs @@ -40,18 +40,18 @@ const CI_WORKFLOW_FILE_PATH = fileURLToPath(new URL(`../${CI_WORKFLOW_PATH}`, im // shell command; without this lock replacing every job body with `true` would // still look like a real successful run. Update deliberately with the CI // workflow when its reviewed job contract changes. -export const EXPECTED_CI_WORKFLOW_SHA256 = '19449197e30bd23e2999da2e0727864c6122c8480abc9c9638c62161dd18ef8a'; +export const EXPECTED_CI_WORKFLOW_SHA256 = 'fc607ffc18f602f23152eef48312bb12e25d2da0af12e09f58396fb375c9b360'; // Fixed rather than inferred from returned jobs: absence must fail rather // than define itself away. `lint` only runs for pull requests and is therefore // deliberately not a member of the push-event release contract. // // That exclusion is about which contexts exist on a main commit, not about -// whether `lint` ran. It is one of the twelve required status checks on the +// whether `lint` ran. It is one of the thirteen required status checks on the // `main` branch protection, so it is evaluated on the pull request's head and // has to pass before anything reaches main; the squash then produces a new -// commit that carries no `lint` context for this gate to find. Eleven here plus -// `lint` is the twelve that protection requires. +// commit that carries no `lint` context for this gate to find. Twelve here plus +// `lint` is the thirteen that protection requires. // // Written down because the shorter version reads as a hole: a reader took it // that way on 2026-08-17 and asked whether main could be pushed unlinted. The @@ -59,7 +59,7 @@ export const EXPECTED_CI_WORKFLOW_SHA256 = '19449197e30bd23e2999da2e0727864c6122 // linted, and saying only the first invites someone to add `lint` to this // list, which would block every release. export const REQUIRED_CHECKS = Object.freeze([ - // `gate` fans in from the eleven below and fails unless every one succeeded. It + // `gate` fans in from the twelve below and fails unless every one succeeded. It // is listed here as well rather than instead: this gate reads the API's job // list, so an entry it does not know about is reported as an unexpected job, // and a `gate` that is not named here would fail every release the moment it @@ -76,6 +76,7 @@ export const REQUIRED_CHECKS = Object.freeze([ 'install-alpine (linux/amd64)', 'install-alpine (linux/arm64)', 'oci-matrix', + 'guard-mutations', ]); class GateError extends Error { diff --git a/scripts/guard-mutations.mjs b/scripts/guard-mutations.mjs new file mode 100644 index 00000000..8991276a --- /dev/null +++ b/scripts/guard-mutations.mjs @@ -0,0 +1,244 @@ +#!/usr/bin/env node +/** + * Mutates each registered guard and requires its named Vitest test to fail. + * + * The outcome is checked against a committed baseline. Known gaps remain + * visible in the complete table, but only a change from that baseline fails: + * regressions make the job red and improvements require the baseline to move. + */ + +import { copyFileSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { dirname, isAbsolute, relative, resolve } from "node:path"; + +const ROOT = resolve(dirname(new URL(import.meta.url).pathname), ".."); +const REGISTRY_PATH = resolve(ROOT, "bench/cdeb/guards/registry.json"); +const BASELINE_PATH = resolve(ROOT, "bench/cdeb/guards/baseline.json"); +const backupRoot = mkdtempSync(resolve(tmpdir(), "commitlore-guard-mutations-")); +let activeRestore = null; + +const restoreActive = () => { + if (activeRestore === null) return; + try { + copyFileSync(activeRestore.backup, activeRestore.target); + } finally { + activeRestore = null; + } +}; + +process.on("exit", restoreActive); +for (const signal of ["SIGINT", "SIGTERM"]) { + process.on(signal, () => { + restoreActive(); + process.exit(128); + }); +} +process.on("uncaughtException", (error) => { + restoreActive(); + throw error; +}); + +const isRecord = (value) => value !== null && typeof value === "object" && !Array.isArray(value); + +const requireString = (value, label) => { + if (typeof value !== "string" || value.length === 0) throw new Error(`${label} must be a non-empty string`); + return value; +}; + +const readRegistry = () => { + const parsed = JSON.parse(readFileSync(REGISTRY_PATH, "utf8")); + if (!isRecord(parsed) || parsed.version !== 1 || !Array.isArray(parsed.properties)) { + throw new Error(`${relative(ROOT, REGISTRY_PATH)} must contain version 1 and a properties array`); + } + return parsed.properties.map((property, propertyIndex) => { + const label = `properties[${propertyIndex}]`; + if (!isRecord(property) || !Array.isArray(property.mutations)) throw new Error(`${label} must be an object with mutations`); + return { + guardId: requireString(property.guard_id, `${label}.guard_id`), + claim: requireString(property.claim, `${label}.claim`), + testFile: requireString(property.test_file, `${label}.test_file`), + testName: requireString(property.test_name, `${label}.test_name`), + mutations: property.mutations.map((mutation, mutationIndex) => { + const mutationLabel = `${label}.mutations[${mutationIndex}]`; + if (!isRecord(mutation) || mutation.must_fail_test !== true) throw new Error(`${mutationLabel}.must_fail_test must be true`); + return { + id: requireString(mutation.mutation_id, `${mutationLabel}.mutation_id`), + file: requireString(mutation.file, `${mutationLabel}.file`), + find: requireString(mutation.find, `${mutationLabel}.find`), + replace: requireString(mutation.replace, `${mutationLabel}.replace`), + why: requireString(mutation.why, `${mutationLabel}.why`), + testName: mutation.test_name === undefined ? undefined : requireString(mutation.test_name, `${mutationLabel}.test_name`), + }; + }), + }; + }); +}; + +const OUTCOMES = new Set(["bound", "inert", "unavailable", "uncovered"]); + +const readBaseline = () => { + const parsed = JSON.parse(readFileSync(BASELINE_PATH, "utf8")); + if (!isRecord(parsed) || parsed.version !== 1 || !Array.isArray(parsed.properties)) { + throw new Error(`${relative(ROOT, BASELINE_PATH)} must contain version 1 and a properties array`); + } + const entries = new Map(); + for (const [index, property] of parsed.properties.entries()) { + const label = `properties[${String(index)}]`; + if (!isRecord(property)) throw new Error(`${label} must be an object`); + const guardId = requireString(property.guard_id, `${label}.guard_id`); + const outcome = requireString(property.outcome, `${label}.outcome`); + if (!OUTCOMES.has(outcome)) throw new Error(`${label}.outcome must be one of ${[...OUTCOMES].join(", ")}`); + if (entries.has(guardId)) throw new Error(`${label}.guard_id duplicates ${guardId}`); + const reason = property.reason === undefined ? undefined : requireString(property.reason, `${label}.reason`); + if (outcome !== "bound" && reason === undefined) throw new Error(`${label}.reason is required for a baseline gap`); + entries.set(guardId, { outcome, reason }); + } + return entries; +}; + +const countOccurrences = (source, find) => { + let count = 0; + let offset = 0; + while (true) { + const found = source.indexOf(find, offset); + if (found === -1) return count; + count += 1; + offset = found + find.length; + } +}; + +const applyMutation = (mutation) => { + const target = resolve(ROOT, mutation.file); + if (!isAbsolute(target) || !target.startsWith(`${ROOT}/`) || !existsSync(target)) { + return { applied: false, reason: `target ${mutation.file} does not exist inside the repository` }; + } + const original = readFileSync(target, "utf8"); + const matches = countOccurrences(original, mutation.find); + if (matches !== 1) { + return { applied: false, reason: `find matched ${String(matches)} times (expected exactly 1)` }; + } + const backup = resolve(backupRoot, `${String(Date.now())}-${mutation.id}`); + copyFileSync(target, backup); + activeRestore = { target, backup }; + try { + const mutated = original.replace(mutation.find, mutation.replace); + writeFileSync(target, mutated, "utf8"); + return { applied: true, target }; + } catch (error) { + restoreActive(); + return { applied: false, reason: `write failed: ${error instanceof Error ? error.message : String(error)}` }; + } +}; + +const runTest = (testFile, testName) => + spawnSync("npx", ["vitest", "run", testFile, "-t", testName], { + cwd: ROOT, + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + }); + +const measurements = []; +let total = 0; +let boundControls = 0; +let inertControls = 0; +let unavailableControls = 0; + +try { + for (const property of readRegistry()) { + if (property.mutations.length === 0) { + measurements.push({ + ...property, + outcome: "uncovered", + detail: "property has zero mutations; no expressible control", + }); + continue; + } + const controlOutcomes = []; + const details = []; + for (const mutation of property.mutations) { + total += 1; + const applied = applyMutation(mutation); + if (!applied.applied) { + unavailableControls += 1; + controlOutcomes.push("unavailable"); + details.push(`${mutation.id}: mutation could not be applied — ${applied.reason}; ${mutation.why}`); + continue; + } + try { + const result = runTest(property.testFile, mutation.testName ?? property.testName); + if (result.error !== undefined) { + unavailableControls += 1; + controlOutcomes.push("unavailable"); + details.push(`${mutation.id}: Vitest could not start — ${result.error.message}; ${mutation.why}`); + } else if (result.status !== 0) { + boundControls += 1; + controlOutcomes.push("bound"); + details.push(`${mutation.id}: mutation applied, test failed — ${mutation.why}`); + } else { + inertControls += 1; + controlOutcomes.push("inert"); + details.push(`${mutation.id}: mutation applied, test passed — ${mutation.why}`); + } + } finally { + restoreActive(); + } + } + const outcome = controlOutcomes.includes("unavailable") + ? "unavailable" + : controlOutcomes.includes("inert") + ? "inert" + : "bound"; + measurements.push({ ...property, outcome, detail: details.join("; ") }); + } +} finally { + restoreActive(); + rmSync(backupRoot, { recursive: true, force: true }); +} + +const baseline = readBaseline(); +const byOutcome = new Map([...OUTCOMES].map((outcome) => [outcome, []])); +for (const measurement of measurements) byOutcome.get(measurement.outcome).push(measurement); + +process.stdout.write("OUTCOME TABLE:\n"); +for (const outcome of ["bound", "inert", "unavailable", "uncovered"]) { + const rows = byOutcome.get(outcome); + process.stdout.write(`${outcome.toUpperCase()} (${String(rows.length)}):\n`); + for (const row of rows) { + const baselineReason = baseline.get(row.guardId)?.reason; + const suffix = baselineReason === undefined ? row.detail : baselineReason; + process.stdout.write(` ${row.guardId}: ${row.claim} — ${suffix}\n`); + } +} +process.stdout.write(`CONTROL SUMMARY: ${String(boundControls)} bound, ${String(inertControls)} inert, ${String(unavailableControls)} unavailable, ${String(byOutcome.get("uncovered").length)} uncovered, ${String(total)} mutations run\n`); + +const failures = []; +const measuredIds = new Set(measurements.map((measurement) => measurement.guardId)); +for (const measurement of measurements) { + const expected = baseline.get(measurement.guardId); + if (expected === undefined) { + if (measurement.outcome === "uncovered") { + failures.push(`REGRESSION: ${measurement.guardId}: new property has zero mutations and is absent from the baseline`); + } else { + failures.push(`BASELINE DISAGREES WITH MEASUREMENT: ${measurement.guardId}: measured ${measurement.outcome}, but the property is absent from the baseline`); + } + continue; + } + if (expected.outcome === measurement.outcome) continue; + if (expected.outcome !== "bound" && measurement.outcome === "bound") { + failures.push(`BASELINE DISAGREES WITH MEASUREMENT: ${measurement.guardId}: baseline records ${expected.outcome}, measurement is bound; tighten the baseline to record the repaired guard`); + } else if (expected.outcome === "bound") { + failures.push(`REGRESSION: BASELINE DISAGREES WITH MEASUREMENT: ${measurement.guardId}: baseline records bound, measurement is ${measurement.outcome}`); + } else { + failures.push(`BASELINE DISAGREES WITH MEASUREMENT: ${measurement.guardId}: baseline records ${expected.outcome}, measurement is ${measurement.outcome}`); + } +} +for (const guardId of baseline.keys()) { + if (!measuredIds.has(guardId)) failures.push(`BASELINE DISAGREES WITH REGISTRY: ${guardId}: baseline property is no longer registered`); +} + +if (failures.length > 0) { + process.stdout.write("RATCHET FAILURES:\n"); + for (const failure of failures) process.stdout.write(` ${failure}\n`); + process.exitCode = 1; +} diff --git a/test/ci-gate.test.ts b/test/ci-gate.test.ts index 840c172c..d9ca6583 100644 --- a/test/ci-gate.test.ts +++ b/test/ci-gate.test.ts @@ -1,6 +1,6 @@ /** * The fan-in gate exists so branch protection can require one stable context - * instead of eleven, several of which are matrix-interpolated: `check (22.23.2)` + * instead of twelve, several of which are matrix-interpolated: `check (22.23.2)` * carries the exact pinned floor, so raising that floor RENAMES the required * context. While admins are exempt a rename is survivable; once they are not, a * renamed context means no commit can satisfy protection and `main` freezes with @@ -69,7 +69,7 @@ describe('the CI fan-in gate', () => { it('lint reports on pull requests, which is the only place protection can see it', () => { // `lint` does not report on `main` commits and is not supposed to: a squash // produces a commit carrying no `lint` context, which is why the release - // gate requires ten jobs and not eleven. It reports on PR heads, and a PR + // gate requires eleven jobs and not twelve. It reports on PR heads, and a PR // head is what branch protection evaluates. // // So the `pull_request` trigger is load-bearing for protection. Remove it