From 8f0176f24845237c3e4adddad6d8a9b2a1713221 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:47:52 +0100 Subject: [PATCH] =?UTF-8?q?fix:=20the=20invisible-character=20gate=20detec?= =?UTF-8?q?ted=20nothing=20=E2=80=94=20codepoint=20escapes,=20not=20UTF-8?= =?UTF-8?q?=20bytes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MEASURED 2026-08-27: the inline pattern in dogfood-gate.yml caught 0 OF 6 invisible-character test cases. It has never detected an NBSP, a zero-width space, a BOM, a soft hyphen, a bidi override or a word joiner. ROOT CAUSE. The pattern is written as UTF-8 BYTE SEQUENCES: \xc2\xa0 \xe2\x80\x8b \xef\xbb\xbf ... but `grep -P` matches CHARACTERS, not bytes. A file containing the two bytes c2 a0 holds ONE character, U+00A0 - while `\xc2\xa0` asks for TWO characters, U+00C2 followed by U+00A0, which is not there. Demonstrated: grep -P '\xc2\xa0' -> miss grep -P '\x{a0}' -> MATCH Only \x00 worked, because it is single-byte in both readings. The gate ran, passed, and could not see what it exists to see. THREE FIXES 1. CODEPOINT escapes \x{a0}, \x{200b}, \x{feff} ... in place of the byte sequences. 2. C0 CONTROL CHARACTERS \x01-\x08, \x0B, \x0C, \x0E-\x1F added. TAB, LF and CR are excluded as legitimate whitespace. This closes the hole that let a stray BACKSPACE byte (0x08) sit inside a regex in developer-ecosystem's evangeliser/npm-bun-blocker.yml, making the file unparseable - so that workflow has NEVER RUN, and the linter reported it clean. 3. `grep -a` - without it grep treats any file containing a NUL as binary and SKIPS it, so the one pattern that did work was suppressed exactly where it mattered. Plus a separate byte-wise LEADING-BOM check: grep strips a leading BOM before matching, so it structurally cannot detect one. Mid-file BOMs are caught by the pattern. stdlib/ByteDetector.affine and config.ncl gain the same C0 range via a new is_c0_control/1, so the compiled linter and the CI gate agree. CONTROLS, all verified before commit: NBSP, ZWSP, SHY, RLO, WJ, NUL and the real 0x08-corrupted workflow are each CAUGHT; a clean file and a file containing tabs, CR and LF are NOT flagged. --- .github/workflows/dogfood-gate.yml | 23 +++++++++++++++++++---- config.ncl | 4 ++++ stdlib/ByteDetector.affine | 10 ++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dogfood-gate.yml b/.github/workflows/dogfood-gate.yml index eed7894..bbfa485 100644 --- a/.github/workflows/dogfood-gate.yml +++ b/.github/workflows/dogfood-gate.yml @@ -113,10 +113,16 @@ jobs: id: lint run: | # Inline invisible character detection (from empty-linter's core patterns). - # Checks for: zero-width spaces, zero-width joiners, BOM, soft hyphens, - # non-breaking spaces, null bytes, and other invisible Unicode in source files. + # Checks for: C0 control characters, zero-width spaces/joiners, BOM, + # soft hyphens, NBSP, bidi overrides, word joiner, null bytes. + # + # ⚠ CODEPOINT escapes \x{a0}, NOT UTF-8 byte sequences \xc2\xa0. grep -P + # matches CHARACTERS, so the byte form never matched anything: the previous + # pattern caught 0 of 6 invisible-character test cases. Verified 2026-08-27. + # ⚠ -a is required or grep skips any file containing a NUL as "binary". + # ⚠ A LEADING BOM cannot be matched by grep (it strips one); checked separately. set +e - PATTERNS='\xc2\xa0|\xe2\x80\x8b|\xe2\x80\x8c|\xe2\x80\x8d|\xef\xbb\xbf|\xc2\xad|\xe2\x80\x8e|\xe2\x80\x8f|\xe2\x80\xaa|\xe2\x80\xab|\xe2\x80\xac|\xe2\x80\xad|\xe2\x80\xae|\x00' + PATTERNS='\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]|\x{a0}|\x{ad}|\x{200b}|\x{200c}|\x{200d}|\x{200e}|\x{200f}|\x{202a}|\x{202b}|\x{202c}|\x{202d}|\x{202e}|\x{2060}|\x{feff}' find "$GITHUB_WORKSPACE" \ -not -path '*/.git/*' -not -path '*/node_modules/*' \ -not -path '*/.deno/*' -not -path '*/target/*' \ @@ -127,7 +133,16 @@ jobs: -o -name '*.yml' -o -name '*.yaml' -o -name '*.md' -o -name '*.adoc' \ -o -name '*.idr' -o -name '*.zig' -o -name '*.v' -o -name '*.jl' \ -o -name '*.gleam' -o -name '*.hs' -o -name '*.ml' -o -name '*.sh' \) \ - -exec grep -Prl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null + -exec grep -aPrl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null + # A LEADING BOM is stripped by grep before matching, so it must be + # checked byte-wise. Mid-file BOMs are caught by the pattern above. + while IFS= read -r bf; do + [ -z "$bf" ] && continue + if [ "$(head -c3 "$bf" | od -An -tx1 | tr -d " ")" = "efbbbf" ]; then + echo "$bf" >> /tmp/empty-lint-results.txt + fi + done < <(find "$GITHUB_WORKSPACE" -not -path '*/.git/*' -not -path '*/node_modules/*' -type f \( -name '*.yml' -o -name '*.yaml' -o -name '*.json' -o -name '*.toml' \) 2>/dev/null) + sort -u -o /tmp/empty-lint-results.txt /tmp/empty-lint-results.txt 2>/dev/null || true EL_EXIT=$? set -e diff --git a/config.ncl b/config.ncl index 81a0de0..ab9a7e5 100644 --- a/config.ncl +++ b/config.ncl @@ -34,6 +34,10 @@ let OutputFormat = [| 'text, 'json, 'hex |] in # Known invisible artifacts (uses proven SafeHex for detection) artifacts = [ { name = "NULL", hex = "0x00", severity = 'Critical, fix_action = "remove" }, + # C0 controls U+0001..U+001F excluding TAB/LF/CR. Illegal in YAML and + # fatal to symbolic parsers; a stray 0x08 made a workflow unloadable and + # therefore silently non-running. Matched as a RANGE by is_c0_control. + { name = "C0-CONTROL", hex = "0x01-0x1F", severity = 'Critical, fix_action = "remove" }, { name = "NBSP", hex = "0xA0", severity = 'Error, fix_action = "replace:20" }, { name = "ZWSP", hex = "0x200B", severity = 'Error, fix_action = "remove" }, { name = "BOM", hex = "0xFEFF", severity = 'Warning, fix_action = "remove" }, diff --git a/stdlib/ByteDetector.affine b/stdlib/ByteDetector.affine index 0d229c2..1b317b3 100644 --- a/stdlib/ByteDetector.affine +++ b/stdlib/ByteDetector.affine @@ -41,6 +41,13 @@ pub fn known_artifacts() -> [ArtifactDef] { ] } +// C0 control characters (U+0001..U+001F) are illegal in YAML and corrupt +// symbolic parsers. TAB (9), LF (10) and CR (13) are legitimate whitespace +// and are excluded. NULL (0) keeps its own named entry above. +pub fn is_c0_control(byte_val: Int) -> Bool { + byte_val > 0 && byte_val < 32 && byte_val != 9 && byte_val != 10 && byte_val != 13 +} + pub fn get_artifact_def(byte_val: Int) -> Option { let defs = known_artifacts(); for d in defs { @@ -48,6 +55,9 @@ pub fn get_artifact_def(byte_val: Int) -> Option { return Some(d); } } + if is_c0_control(byte_val) { + return Some(#{ name: "C0-CONTROL", byte_value: byte_val, severity: Critical, fix_action: "remove" }); + } None }