diff --git a/.github/workflows/dogfood-gate.yml b/.github/workflows/dogfood-gate.yml index 444cfed..8823a2a 100644 --- a/.github/workflows/dogfood-gate.yml +++ b/.github/workflows/dogfood-gate.yml @@ -121,7 +121,7 @@ jobs: # Checks for: zero-width spaces, zero-width joiners, BOM, soft hyphens, # non-breaking spaces, null bytes, and other invisible Unicode in source files. set +e - PATTERNS='\xc2\xa0|\xe2\x80\x8b|\xe2\x80\x8c|\xe2\x80\x8d|\xef\xbb\xbf|\xc2\xad|\xe2\x80\x8e|\xe2\x80\x8f|\xe2\x80\xaa|\xe2\x80\xab|\xe2\x80\xac|\xe2\x80\xad|\xe2\x80\xae|\x00' + PATTERNS='(*UTF)[\x00-\x08\x0B\x0C\x0E-\x1F\x{a0}\x{ad}\x{200b}-\x{200f}\x{202a}-\x{202f}\x{2060}\x{2066}-\x{2069}\x{feff}]' find "$GITHUB_WORKSPACE" \ -not -path '*/.git/*' -not -path '*/node_modules/*' \ -not -path '*/.deno/*' -not -path '*/target/*' \ @@ -132,7 +132,7 @@ jobs: -o -name '*.yml' -o -name '*.yaml' -o -name '*.md' -o -name '*.adoc' \ -o -name '*.idr' -o -name '*.zig' -o -name '*.v' -o -name '*.jl' \ -o -name '*.gleam' -o -name '*.hs' -o -name '*.ml' -o -name '*.sh' \) \ - -exec grep -Prl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null + -exec grep -aPrl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null EL_EXIT=$? set -e @@ -141,6 +141,19 @@ jobs: echo "exit_code=$EL_EXIT" >> "$GITHUB_OUTPUT" echo "ready=true" >> "$GITHUB_OUTPUT" + # Blocking subset: C0 controls and NUL only (owner ruling 2026-08-28). + # Invisible Unicode (NBSP/BOM/zero-width) stays ADVISORY - about 2,100 + # estate files carry it as legitimate typography in prose. + blocking=0 + while IFS= read -r bf; do + [ -z "$bf" ] && continue + if grep -qaP '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' "$bf"; then + blocking=$((blocking+1)) + echo "::error file=${bf#$GITHUB_WORKSPACE/}::C0 control characters or NUL bytes - file corruption, blocks the gate" + fi + done < /tmp/empty-lint-results.txt + echo "blocking=$blocking" >> "$GITHUB_OUTPUT" + # Emit annotations for each file with invisible chars while IFS= read -r filepath; do [ -z "$filepath" ] && continue @@ -148,6 +161,21 @@ jobs: echo "::warning file=${REL_PATH}::Invisible Unicode characters detected (zero-width space, BOM, NBSP, etc.)" done < /tmp/empty-lint-results.txt + # Enforce (owner ruling 2026-08-28): C0/NUL corruption BLOCKS; other + # invisible Unicode stays advisory. Enforcement lives inside this step + # so a crash above fails the job directly - counts can never arrive + # empty into a separate check that then passes silently. + if [ "$EL_EXIT" -ne 0 ]; then + echo "::warning::invisible-character scan exited $EL_EXIT - results may be incomplete" + fi + if [ "${blocking:-0}" -gt 0 ]; then + echo "## Empty-linter: BLOCKED - $blocking file(s) with C0/NUL corruption" >> "$GITHUB_STEP_SUMMARY" + echo "::error::$blocking file(s) contain C0 control characters or NUL bytes - corruption, not typography. See file annotations." + exit 1 + elif [ "${FINDINGS:-0}" -gt 0 ]; then + echo "::notice::$FINDINGS file(s) carry invisible Unicode (NBSP/BOM/zero-width) - advisory only" + fi + - name: Write summary run: | if [ "${{ steps.lint.outputs.ready }}" = "true" ]; then diff --git a/docs/developer/INVISIBLE-CHAR-DETECTION.adoc b/docs/developer/INVISIBLE-CHAR-DETECTION.adoc new file mode 100644 index 0000000..5f6445e --- /dev/null +++ b/docs/developer/INVISIBLE-CHAR-DETECTION.adoc @@ -0,0 +1,128 @@ += Invisible Character Detection +:toc: +:toclevels: 3 + +== Overview + +The invisible character detection gate (empty-linter) enforces file hygiene by detecting C0 control characters, NUL bytes, and leading UTF-8 BOM markers that indicate file corruption or encoding issues. + +== Implementation + +=== Location + +The canonical implementation for anvomidaviser is in `.github/workflows/dogfood-gate.yml` (Job 3: empty-lint). + +=== Detection Rules + +==== BLOCKING (fails the CI gate) + +* *C0 Control Characters*: `\x00-\x08`, `\x0B`, `\x0C`, `\x0E-\x1F` +** These indicate file corruption (backspace mangled LaTeX, made workflows unloadable) +** Detection: PCRE byte-wise pattern `\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]` +** Locale-independent (uses exact hex bytes, not character classes) + +* *Leading UTF-8 BOM*: EF BB BF at file position 0 +** Not needed for UTF-8 files (UTF-8 is self-identifying) +** Causes parser issues in some tools +** Detection: byte-wise check `head -c 3 file | od -An -tx1 | tr -d ' '` equals `efbbbf` + +==== ADVISORY (warning only, does not fail) + +* *Invisible Unicode*: NBSP (`\x{a0}`), soft hyphen (`\x{ad}`), zero-width marks, BOM mid-file +** About 2,100 estate files carry these as legitimate typography in prose +** Detection included in main PCRE pattern but not enforced + +=== Pattern Evolution + +[source,bash] +---- +# Before bd73a39 (broken, never matched anything): +PATTERNS='\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]|...' + +# After bd73a39 (locale-independent, consolidated ranges): +PATTERNS='(*UTF)[\x00-\x08\x0B\x0C\x0E-\x1F\x{a0}\x{ad}\x{200b}-\x{200f}\x{202a}-\x{202f}\x{2060}\x{2066}-\x{2069}\x{feff}]' +---- + +The `(*UTF)` prefix enables UTF-8 mode in PCRE while keeping the byte-level C0 detection locale-independent. + +=== Enforcement Strategy + +Per commit b3fbe01, enforcement happens *inside* the scan step: + +1. Main pattern detects all invisible characters +2. Blocking subset re-greps only flagged files for C0/NUL +3. Leading-BOM check uses byte-wise `head | od` on flagged files +4. If scanner crashes, job fails directly (no silent pass) + +This prevents drift between detection and enforcement expressions. + +=== Cross-Repo Synchronization + +NOTE: `stdlib/ByteDetector.affine` and `config.ncl` do not exist in anvomidaviser. The formal/shared implementation lives in the estate-wide iseriser repository and will be updated to match these rules in a separate cross-repo PR. + +== Testing + +=== Test Fixtures + +Test fixtures are in `tests/fixtures/invisible-chars/`: + +* `clean.rs` — No invisible characters (should PASS) +* `legitimate-whitespace.rs` — Tabs, newlines, spaces (should PASS) +* `leading-bom.rs` — UTF-8 BOM at position 0 (should BLOCK) +* `nul-byte.rs` — NUL byte (should BLOCK) +* `backspace.rs` — Backspace C0 control (should BLOCK) +* `vertical-tab.rs` — Vertical tab C0 control (should BLOCK) +* `form-feed.rs` — Form feed C0 control (should BLOCK) +* `escape.rs` — Escape character C0 control (should BLOCK) + +=== Running Tests Locally + +[source,bash] +---- +./tests/test-invisible-char-detection.sh +---- + +This script verifies both C0 detection and leading-BOM detection against all fixtures. + +=== CI Exclusions + +The CI gate excludes test fixtures via: + +[source,bash] +---- +find ... -not -path '*/tests/fixtures/invisible-chars/*' ... +---- + +== Design Rationale + +=== Why Block Leading BOM? + +1. UTF-8 BOM (EF BB BF) is *not* required for UTF-8 files +2. UTF-8 is self-identifying (no BOM needed unlike UTF-16/32) +3. Some tools (parsers, compilers) choke on leading BOM +4. Mid-file BOM stays advisory (may be legitimate in prose/data) + +=== Why Byte-Wise Detection? + +1. *Locale independence*: PCRE character classes like `[:cntrl:]` vary by locale +2. *Precision*: Exact hex bytes ensure consistent behavior across systems +3. *Safety*: No risk of misinterpretation in different environments + +=== Why Separate Detection from Enforcement? + +1. *Full visibility*: Detect everything first (2,100+ files have legit invisible Unicode) +2. *Targeted blocking*: Only fail on proven-dangerous corruption +3. *Fail-safe*: Scanner crash fails the job (no silent pass on empty results) + +== Related Commits + +* `bd73a39`: Make invisible-character PCRE locale-independent +* `b3fbe01`: Enforce C0/NUL corruption in-step; warn on invisible Unicode +* `57ad5da`: Fix invisible-character gate (never matched anything before) +* Current commit: Add leading-BOM byte-wise check + +== References + +* Owner ruling 2026-08-28: C0/NUL/leading-BOM blocks; other invisible Unicode advisory +* Gate-lens census: ~2,100 estate files carry NBSP/zero-width as legit typography +* LaTeX/workflow corruption incidents: Backspace bytes caused silent mangling diff --git a/tests/fixtures/invisible-chars/README.adoc b/tests/fixtures/invisible-chars/README.adoc new file mode 100644 index 0000000..9d61280 --- /dev/null +++ b/tests/fixtures/invisible-chars/README.adoc @@ -0,0 +1,35 @@ += Invisible Character Test Fixtures + +Test files for the invisible character detection gate (empty-linter). + +== File Inventory + +=== Clean Files (Should NOT be flagged) + +* `clean.rs` — No invisible characters at all +* `legitimate-whitespace.rs` — Tabs, newlines, spaces (all legitimate) + +=== Files with C0 Control Characters (BLOCKING - corruption) + +* `nul-byte.rs` — Contains NUL byte (\x00) +* `backspace.rs` — Contains backspace (\x08) +* `vertical-tab.rs` — Contains vertical tab (\x0B) +* `form-feed.rs` — Contains form feed (\x0C) +* `escape.rs` — Contains escape character (\x1B) + +=== Files with Leading BOM (BLOCKING - see CI gate rule) + +* `leading-bom.rs` — UTF-8 BOM at file start (EF BB BF / U+FEFF) + +== Usage + +These fixtures are used to test the invisible character detection in `.github/workflows/dogfood-gate.yml`. + +The gate distinguishes: + +* *BLOCKING*: C0 control characters (\x00-\x08, \x0B, \x0C, \x0E-\x1F) and leading BOM +* *ADVISORY*: Other invisible Unicode (NBSP, zero-width marks, BOM not at file start) + +== Notes + +The leading BOM check enforces byte-wise detection to catch UTF-8 BOM (EF BB BF) at position zero, as it can cause parser issues in some tools and is not needed for UTF-8 files. diff --git a/tests/fixtures/invisible-chars/backspace.rs b/tests/fixtures/invisible-chars/backspace.rs new file mode 100644 index 0000000..3598d52 --- /dev/null +++ b/tests/fixtures/invisible-chars/backspace.rs @@ -0,0 +1,2 @@ +// File with backspace +fn main() {} diff --git a/tests/fixtures/invisible-chars/clean.rs b/tests/fixtures/invisible-chars/clean.rs new file mode 100644 index 0000000..9d988d9 --- /dev/null +++ b/tests/fixtures/invisible-chars/clean.rs @@ -0,0 +1,4 @@ +// Clean Rust file with no invisible characters +fn main() { + println!("Hello, world!"); +} diff --git a/tests/fixtures/invisible-chars/escape.rs b/tests/fixtures/invisible-chars/escape.rs new file mode 100644 index 0000000..2614205 --- /dev/null +++ b/tests/fixtures/invisible-chars/escape.rs @@ -0,0 +1,2 @@ +// File with escape +fn main() {} diff --git a/tests/fixtures/invisible-chars/form-feed.rs b/tests/fixtures/invisible-chars/form-feed.rs new file mode 100644 index 0000000..f89018c --- /dev/null +++ b/tests/fixtures/invisible-chars/form-feed.rs @@ -0,0 +1,2 @@ +// File with form feed +fn main() {} diff --git a/tests/fixtures/invisible-chars/leading-bom.rs b/tests/fixtures/invisible-chars/leading-bom.rs new file mode 100644 index 0000000..ea3041a --- /dev/null +++ b/tests/fixtures/invisible-chars/leading-bom.rs @@ -0,0 +1,4 @@ +// Rust file with leading BOM +fn main() { + println\!("BOM at start"); +} diff --git a/tests/fixtures/invisible-chars/legitimate-whitespace.rs b/tests/fixtures/invisible-chars/legitimate-whitespace.rs new file mode 100644 index 0000000..8b4ad67 --- /dev/null +++ b/tests/fixtures/invisible-chars/legitimate-whitespace.rs @@ -0,0 +1,7 @@ +// File with legitimate whitespace (tabs, newlines, spaces) +fn calculate() -> i32 { + let x = 42; // tab before this comment + let y = 10; + + x + y +} diff --git a/tests/fixtures/invisible-chars/nul-byte.rs b/tests/fixtures/invisible-chars/nul-byte.rs new file mode 100644 index 0000000..1512cf3 Binary files /dev/null and b/tests/fixtures/invisible-chars/nul-byte.rs differ diff --git a/tests/fixtures/invisible-chars/vertical-tab.rs b/tests/fixtures/invisible-chars/vertical-tab.rs new file mode 100644 index 0000000..2cebd1e --- /dev/null +++ b/tests/fixtures/invisible-chars/vertical-tab.rs @@ -0,0 +1,2 @@ +// File with vertical tab +fn main() {} diff --git a/tests/test-invisible-char-detection.sh b/tests/test-invisible-char-detection.sh new file mode 100755 index 0000000..95d1593 --- /dev/null +++ b/tests/test-invisible-char-detection.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Test script for invisible character detection (local verification) +# This tests the logic from .github/workflows/dogfood-gate.yml + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FIXTURES_DIR="${SCRIPT_DIR}/fixtures/invisible-chars" + +echo "Testing invisible character detection..." +echo + +# Test 1: Clean file should not trigger blocking +echo "Test 1: Clean file (should PASS)" +if grep -qaP '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' "${FIXTURES_DIR}/clean.rs"; then + echo " ❌ FAIL: Clean file triggered C0 detection" + exit 1 +fi +if [ "$(head -c 3 "${FIXTURES_DIR}/clean.rs" | od -An -tx1 | tr -d ' ')" = "efbbbf" ]; then + echo " ❌ FAIL: Clean file triggered BOM detection" + exit 1 +fi +echo " ✓ PASS" +echo + +# Test 2: Legitimate whitespace should not trigger blocking +echo "Test 2: Legitimate whitespace (should PASS)" +if grep -qaP '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' "${FIXTURES_DIR}/legitimate-whitespace.rs"; then + echo " ❌ FAIL: Legitimate whitespace triggered C0 detection" + exit 1 +fi +if [ "$(head -c 3 "${FIXTURES_DIR}/legitimate-whitespace.rs" | od -An -tx1 | tr -d ' ')" = "efbbbf" ]; then + echo " ❌ FAIL: Legitimate whitespace triggered BOM detection" + exit 1 +fi +echo " ✓ PASS" +echo + +# Test 3: Leading BOM should trigger blocking +echo "Test 3: Leading BOM (should BLOCK)" +if [ "$(head -c 3 "${FIXTURES_DIR}/leading-bom.rs" | od -An -tx1 | tr -d ' ')" = "efbbbf" ]; then + echo " ✓ PASS: Leading BOM detected" +else + echo " ❌ FAIL: Leading BOM not detected" + exit 1 +fi +echo + +# Test 4: NUL byte should trigger blocking +echo "Test 4: NUL byte (should BLOCK)" +if grep -qaP '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' "${FIXTURES_DIR}/nul-byte.rs"; then + echo " ✓ PASS: NUL byte detected" +else + echo " ❌ FAIL: NUL byte not detected" + exit 1 +fi +echo + +# Test 5: Backspace should trigger blocking +echo "Test 5: Backspace (C0 control, should BLOCK)" +if grep -qaP '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' "${FIXTURES_DIR}/backspace.rs"; then + echo " ✓ PASS: Backspace detected" +else + echo " ❌ FAIL: Backspace not detected" + exit 1 +fi +echo + +# Test 6: Vertical tab should trigger blocking +echo "Test 6: Vertical tab (C0 control, should BLOCK)" +if grep -qaP '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' "${FIXTURES_DIR}/vertical-tab.rs"; then + echo " ✓ PASS: Vertical tab detected" +else + echo " ❌ FAIL: Vertical tab not detected" + exit 1 +fi +echo + +# Test 7: Form feed should trigger blocking +echo "Test 7: Form feed (C0 control, should BLOCK)" +if grep -qaP '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' "${FIXTURES_DIR}/form-feed.rs"; then + echo " ✓ PASS: Form feed detected" +else + echo " ❌ FAIL: Form feed not detected" + exit 1 +fi +echo + +# Test 8: Escape character should trigger blocking +echo "Test 8: Escape character (C0 control, should BLOCK)" +if grep -qaP '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' "${FIXTURES_DIR}/escape.rs"; then + echo " ✓ PASS: Escape character detected" +else + echo " ❌ FAIL: Escape character not detected" + exit 1 +fi +echo + +echo "==========================================" +echo "All tests passed! ✓" +echo "==========================================" +echo +echo "Detection rules verified:" +echo " • C0 control characters (\\x00-\\x08, \\x0B, \\x0C, \\x0E-\\x1F): BLOCKING" +echo " • Leading BOM (EF BB BF at position 0): BLOCKING" +echo " • Legitimate whitespace (tabs, newlines, spaces): ALLOWED" +echo