From fff1409dac223a12680adb30a68d470b1863c225 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:23:12 +0100 Subject: [PATCH] ci: drop embedded Python YAML parsing; gate inline Python Python is banned estate-wide, yet every Python detector matches `*.py` only, so Python embedded in workflows, scripts and justfiles was invisible. The governance reusable's R5 gate parsed YAML with a `python3 - <<'PY'` heredoc importing PyYAML, against YAML-POLICY Y-1 (read YAML with yq). - governance-reusable.yml R5: ported to bash + yq + jq. Known-answer equality: the old Python step and the new step produce byte-identical output and exit codes on five fixture trees (no dir, empty dir, clean rule, and a mixed tree with hits, a non-mapping, missing patterns, a bad regex, an id-less rule, a numeric id, self-references, changelogs and a missing include). An unparseable rule file now prints one counted R5 error instead of a Python traceback; both exit 1. - tests/test_secret_scanner_canary.sh: step extraction via yq; the extracted script is byte-identical to the Python one, and a reusable without the step still fails the canary. - self-test.yml: drop `pip install pyyaml`. - justfile.template: `python3 -c json.dumps` -> `jq -Rs .` (identical encoding on quotes, backslashes, tabs, ESC and newlines); REGISTRY.a2ml regenerated for the RSR source hash. - scripts/check-inline-python.sh: flags python[3] -c/-m/-/< Claude-Session: https://claude.ai/code/session_01W5CoaksP2Bg21HpDCgFgwS --- .github/workflows/governance-reusable.yml | 123 +++++++------ .github/workflows/self-test.yml | 4 +- .machine_readable/REGISTRY.a2ml | 2 +- .machine_readable/inline-python-allow.txt | 26 +++ .../templates/justfile.template | 4 +- scripts/check-inline-python.sh | 130 +++++++++++++ scripts/tests/check-inline-python-test.sh | 171 ++++++++++++++++++ tests/test_secret_scanner_canary.sh | 38 ++-- 8 files changed, 411 insertions(+), 87 deletions(-) create mode 100644 .machine_readable/inline-python-allow.txt create mode 100755 scripts/check-inline-python.sh create mode 100755 scripts/tests/check-inline-python-test.sh diff --git a/.github/workflows/governance-reusable.yml b/.github/workflows/governance-reusable.yml index 70626f399..f274eafe2 100644 --- a/.github/workflows/governance-reusable.yml +++ b/.github/workflows/governance-reusable.yml @@ -916,70 +916,69 @@ jobs: echo "ℹ️ [R5] no $DIR/ — skipped (repo has not opted in)" exit 0 fi - if ! command -v python3 >/dev/null 2>&1; then - echo "❌ [R5] python3 missing on runner — required for YAML rule parsing" + # Rule files are read with yq, never a hand parser (YAML-POLICY Y-1); this + # gate previously parsed them with Python, which the estate bans. + if ! command -v yq >/dev/null 2>&1 || ! command -v jq >/dev/null 2>&1; then + echo "❌ [R5] yq and jq are required on the runner for YAML rule parsing" exit 2 fi - python3 - <<'PY' - import os, sys, glob, subprocess - try: - import yaml - except ImportError: - sys.exit("❌ [R5] PyYAML not installed on runner; install python3-yaml") - - dir_ = ".github/canonical-references" - files = sorted(glob.glob(f"{dir_}/*.yml") + glob.glob(f"{dir_}/*.yaml")) - if not files: - print(f"ℹ️ [R5] {dir_}/ has no .yml/.yaml rules — skipped") - sys.exit(0) - - total = 0 - for rf in files: - with open(rf, encoding="utf-8") as fh: - cfg = yaml.safe_load(fh) - if not isinstance(cfg, dict): - print(f"❌ [R5] {rf}: top-level must be a mapping"); total += 1; continue - rid = cfg.get("id", os.path.basename(rf)) - desc = cfg.get("description", "") - pats = cfg.get("patterns") or [] - canon = cfg.get("canonical_pointer", "") - scope = (cfg.get("scope") or {}) - includes = scope.get("include") or [] - if not pats or not includes: - print(f"❌ [R5:{rid}] missing patterns or scope.include in {rf}") - total += 1; continue - # exclude self-references - skip = set(["CHANGELOG.md", "CHANGELOG.adoc", rf]) - if canon: skip.add(canon) - rule_hits = 0 - for f_ in includes: - if f_ in skip or not os.path.isfile(f_): - continue - for pat in pats: - r = subprocess.run( - ["grep", "-nE", pat, f_], - capture_output=True, text=True, - ) - if r.returncode == 0: - for line in r.stdout.splitlines(): - ln_no, _, body = line.partition(":") - tail = canon or "drop or move to a canonical source" - print(f"::error file={f_},line={ln_no}::" - f"[R5:{rid}] {body} → route to {tail}") - rule_hits += 1 - elif r.returncode > 1: - print(f"❌ [R5:{rid}] grep error on {f_}: {r.stderr.strip()}") - rule_hits += 1 - if rule_hits: - print(f"❌ [R5:{rid}] {rule_hits} hit(s). {desc}") - total += rule_hits - - if total: - print() - print(f"❌ [R5] {total} canonical-reference drift hit(s) across {len(files)} rule(s).") - sys.exit(1) - print(f"✅ [R5] canonical-reference drift: clean across {len(files)} rule(s).") - PY + mapfile -t files < <(find "$DIR" -maxdepth 1 -type f \( -name '*.yml' -o -name '*.yaml' \) ! -name '.*' | LC_ALL=C sort) + if [ "${#files[@]}" -eq 0 ]; then + echo "ℹ️ [R5] $DIR/ has no .yml/.yaml rules — skipped" + exit 0 + fi + + err=$(mktemp) + trap 'rm -f "$err"' EXIT + total=0 + for rf in "${files[@]}"; do + if ! cfg=$(yq -o json '.' "$rf" 2>&1); then + echo "❌ [R5] $rf: not parseable YAML: $cfg"; total=$((total + 1)); continue + fi + if [ "$(jq -r 'type' <<<"$cfg")" != object ]; then + echo "❌ [R5] $rf: top-level must be a mapping"; total=$((total + 1)); continue + fi + rid=$(jq -r --arg b "$(basename "$rf")" 'if has("id") then .id | tostring else $b end' <<<"$cfg") + desc=$(jq -r '.description // ""' <<<"$cfg") + canon=$(jq -r '.canonical_pointer // ""' <<<"$cfg") + mapfile -t pats < <(jq -r '(.patterns // [])[]' <<<"$cfg") + mapfile -t includes < <(jq -r '((.scope // {}).include // [])[]' <<<"$cfg") + if [ "${#pats[@]}" -eq 0 ] || [ "${#includes[@]}" -eq 0 ]; then + echo "❌ [R5:$rid] missing patterns or scope.include in $rf" + total=$((total + 1)); continue + fi + rule_hits=0 + for f_ in "${includes[@]}"; do + # Self-references are excluded: the changelogs, the rule file, and the + # rule's own canonical pointer. + case "$f_" in CHANGELOG.md|CHANGELOG.adoc|"$rf") continue ;; esac + if [ -n "$canon" ] && [ "$f_" = "$canon" ]; then continue; fi + [ -f "$f_" ] || continue + for pat in "${pats[@]}"; do + out=$(grep -nE -e "$pat" -- "$f_" 2>"$err"); rc=$? + if [ "$rc" -eq 0 ]; then + while IFS= read -r line; do + echo "::error file=$f_,line=${line%%:*}::[R5:$rid] ${line#*:} → route to ${canon:-drop or move to a canonical source}" + rule_hits=$((rule_hits + 1)) + done <<<"$out" + elif [ "$rc" -gt 1 ]; then + echo "❌ [R5:$rid] grep error on $f_: $(<"$err")" + rule_hits=$((rule_hits + 1)) + fi + done + done + if [ "$rule_hits" -gt 0 ]; then + echo "❌ [R5:$rid] $rule_hits hit(s). $desc" + fi + total=$((total + rule_hits)) + done + + if [ "$total" -gt 0 ]; then + echo + echo "❌ [R5] $total canonical-reference drift hit(s) across ${#files[@]} rule(s)." + exit 1 + fi + echo "✅ [R5] canonical-reference drift: clean across ${#files[@]} rule(s)." quality: name: Code quality + docs diff --git a/.github/workflows/self-test.yml b/.github/workflows/self-test.yml index 1b1c52330..9c7af3c5f 100644 --- a/.github/workflows/self-test.yml +++ b/.github/workflows/self-test.yml @@ -42,14 +42,14 @@ jobs: # history is a requirement and not an optimisation. fetch-depth: 0 - # PyYAML is required by the secret-scanner canary. The scorecard + # The secret-scanner canary reads YAML with yq (preinstalled on the + # runner; YAML-POLICY Y-1) -- no Python. The scorecard # grounding tests execute the same checks as registry-verify, including # checks that require ripgrep and xmllint. - name: Install test dependencies run: | sudo apt-get update -qq sudo apt-get install -y --no-install-recommends ripgrep libxml2-utils ruby ruby-minitest - python3 -m pip install --user --quiet pyyaml - name: Run tests/*.sh and scripts/tests/*.sh run: bash scripts/run-shell-test-suite.sh diff --git a/.machine_readable/REGISTRY.a2ml b/.machine_readable/REGISTRY.a2ml index 3cfea210a..008927d55 100644 --- a/.machine_readable/REGISTRY.a2ml +++ b/.machine_readable/REGISTRY.a2ml @@ -207,7 +207,7 @@ name = "RSR — Rhodium Standard Repositories" stream = "governance" home = "rhodium-standard-repositories/" canonical_doc = "rhodium-standard-repositories/README.adoc" -source_hash = "sha256:3c7ccbf889b32ffac7566dcdf93ea9923d1b53f1c6055889f4a330a52c421d85" +source_hash = "sha256:dcd870a92e82159d764a26bf6ca4f37d2cdbf9c418561dc51fc5bc2ab28279f9" route = "the repository-compliance standard every repo is graded against" [[spec]] diff --git a/.machine_readable/inline-python-allow.txt b/.machine_readable/inline-python-allow.txt new file mode 100644 index 000000000..9284085d7 --- /dev/null +++ b/.machine_readable/inline-python-allow.txt @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# inline-python-allow.txt — SHRINK-ONLY ledger for scripts/check-inline-python.sh. +# +# Python is banned estate-wide. Each line names a file that still embeds Python +# (a `python3 -c`/`-m`/`-` call, a `<<'PY'` heredoc, `python3 x.py`, or +# `pip install`) and how many lines of it remain: `path:count`. +# +# The gate fails if a file not listed here gains inline Python, if a count +# GROWS, or if an entry is STALE (the count fell or reached zero). Fixing debt +# therefore means lowering or deleting its line in the same change. Never add +# a line; never raise a count. Regenerate the current set with +# bash scripts/check-inline-python.sh --print +.github/workflows/deed-conformance.yml:3 +2-protocols/axel/Containerfile:1 +rhodium-standard-repositories/Justfile:1 +rhodium-standard-repositories/rhodium-pipeline/Justfile:1 +rhodium-standard-repositories/satellites/cccp/satellites/nextgen-languages/7-tentacles/Justfile:1 +rhodium-standard-repositories/satellites/cccp/satellites/php-aegis/.github/workflows/comprehensive-quality.yml:2 +rhodium-standard-repositories/satellites/cccp/satellites/sanctify-php/.github/workflows/comprehensive-quality.yml:2 +rhodium-standard-repositories/satellites/state.scm/.github/workflows/comprehensive-quality.yml:2 +rhodium-standard-repositories/ux-test-harness/test-repo.sh:2 +scripts/check-allowed-actions.sh:1 +scripts/cicd-census.sh:2 +scripts/tests/apply-branch-gates-test.sh:1 diff --git a/rhodium-standard-repositories/templates/justfile.template b/rhodium-standard-repositories/templates/justfile.template index 016051625..520ccabd5 100644 --- a/rhodium-standard-repositories/templates/justfile.template +++ b/rhodium-standard-repositories/templates/justfile.template @@ -465,8 +465,8 @@ help-me: "timestamp": "$(date -Iseconds 2>/dev/null || date)", "platform": "$(uname -srm 2>/dev/null || echo unknown)", "shell": "$SHELL", - "doctor": $(echo "$DOCTOR_OUT" | python3 -c 'import sys,json; print(json.dumps(sys.stdin.read()))' 2>/dev/null || echo '"(could not encode)"'), - "git_log": $(echo "$GIT_LOG" | python3 -c 'import sys,json; print(json.dumps(sys.stdin.read()))' 2>/dev/null || echo '"(could not encode)"') + "doctor": $(echo "$DOCTOR_OUT" | jq -Rs . 2>/dev/null || echo '"(could not encode)"'), + "git_log": $(echo "$GIT_LOG" | jq -Rs . 2>/dev/null || echo '"(could not encode)"') } ENDJSON # Also plain text for manual use diff --git a/scripts/check-inline-python.sh b/scripts/check-inline-python.sh new file mode 100755 index 000000000..0bf097dcb --- /dev/null +++ b/scripts/check-inline-python.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# check-inline-python.sh — refuse Python embedded in non-Python files. +# +# Python is banned estate-wide, but every Python detector matches by file +# extension (`*.py`). A `python3 - <<'PY'` heredoc inside a workflow, a +# `python3 -c` one-liner in a justfile, or a `pip install` in a Containerfile +# is invisible to them. That gap is how a YAML parser written in Python got +# into the governance reusable after the ban. This gate closes it. +# +# Flagged, on non-comment lines of *.yml, *.yaml, *.sh, *.template, +# Justfile/justfile and Containerfile: +# python[3] -c … python[3] -m … python[3] - (stdin) python[3] <<… +# <<'PY' / <.py pip[3] install +# +# Remaining debt is recorded in a SHRINK-ONLY ledger +# (.machine_readable/inline-python-allow.txt), one `path:count` per line. +# The gate fails when a path not in the ledger has a hit, when a count GROWS, +# and when an entry is STALE (its count dropped): a fix must shrink the ledger +# in the same change, so the ledger can never hide new debt behind old. +# +# Usage: check-inline-python.sh [--root DIR] [--ledger FILE] [--print] +# --print write the current `path:count` set to stdout and exit 0 +# (to regenerate the ledger after a fix). +# Exit: 0 clean / matches ledger, 1 violations, 2 usage error. +set -uo pipefail + +ROOT=. +LEDGER= +PRINT=false +while [ $# -gt 0 ]; do + case "$1" in + --root) ROOT=${2:?--root needs a directory}; shift 2 ;; + --ledger) LEDGER=${2:?--ledger needs a file}; shift 2 ;; + --print) PRINT=true; shift ;; + *) echo "check-inline-python: unknown argument: $1" >&2; exit 2 ;; + esac +done +[ -d "$ROOT" ] || { echo "check-inline-python: no such directory: $ROOT" >&2; exit 2; } +LEDGER=${LEDGER:-$ROOT/.machine_readable/inline-python-allow.txt} + +# One ERE per form, alternated. Kept as data so the test suite can name it. +PATTERN='(^|[^[:alnum:]_.-])python3?[[:space:]]+(-[cm]([[:space:]]|$)|-([[:space:]]|$)|<<|[^[:space:]]+\.py([[:space:]]|$|[;&|)]))' +PATTERN+='|<<-?[[:space:]]*["'"'"']?PY["'"'"']?([[:space:]]|$)' +PATTERN+='|(^|[^[:alnum:]_.-])pip3?[[:space:]]+install([[:space:]]|$)' + +# list_files — prints every in-scope file under $ROOT, relative to it, sorted. +# Tracked files only when $ROOT is a git work tree; otherwise a plain walk. +list_files() { + if git -C "$ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + git -C "$ROOT" ls-files -z + else + (cd "$ROOT" && find . -type d -name .git -prune -o -type f -print0 | sed -z 's|^\./||') + fi | while IFS= read -r -d '' f; do + case "${f##*/}" in + *.yml|*.yaml|*.sh|*.template|Justfile|justfile|Containerfile) printf '%s\n' "$f" ;; + esac + done | LC_ALL=C sort +} + +# count_hits — prints how many non-comment lines of match PATTERN. +count_hits() { + /usr/bin/grep -vE '^[[:space:]]*#' -- "$ROOT/$1" 2>/dev/null | /usr/bin/grep -cE -e "$PATTERN" +} + +# show_hits — prints each offending line as a GitHub ::error annotation. +show_hits() { + /usr/bin/grep -nE -e "$PATTERN" -- "$ROOT/$1" | /usr/bin/grep -vE '^[0-9]+:[[:space:]]*#' | + while IFS= read -r line; do + echo "::error file=$1,line=${line%%:*}::inline Python: ${line#*:}" + done +} + +declare -A actual=() allowed=() +while IFS= read -r f; do + [ -f "$ROOT/$f" ] || continue + n=$(count_hits "$f") + [ "${n:-0}" -gt 0 ] && actual[$f]=$n +done < <(list_files) + +if $PRINT; then + for f in "${!actual[@]}"; do printf '%s:%s\n' "$f" "${actual[$f]}"; done | LC_ALL=C sort + exit 0 +fi + +if [ -f "$LEDGER" ]; then + lineno=0 + while IFS= read -r entry || [ -n "$entry" ]; do + lineno=$((lineno + 1)) + case "$entry" in ''|'#'*) continue ;; esac + if ! [[ "$entry" =~ ^([^[:space:]]+):([1-9][0-9]*)$ ]]; then + echo "::error file=$LEDGER,line=$lineno::malformed ledger entry (want path:count): $entry" + exit 1 + fi + allowed[${BASH_REMATCH[1]}]=${BASH_REMATCH[2]} + done < "$LEDGER" +fi + +fail=0 +total=0 +for f in "${!actual[@]}"; do + n=${actual[$f]}; total=$((total + n)) + want=${allowed[$f]:-0} + if [ "$want" -eq 0 ]; then + echo "❌ new inline Python in $f ($n line(s)) — rewrite it in bash/yq/jq; Python is banned" + show_hits "$f"; fail=$((fail + 1)) + elif [ "$n" -gt "$want" ]; then + echo "❌ inline Python GREW in $f: $n line(s), ledger allows $want — the ledger is shrink-only" + show_hits "$f"; fail=$((fail + 1)) + elif [ "$n" -lt "$want" ]; then + echo "❌ stale ledger entry $f:$want — now $n; shrink the ledger to $f:$n in this change" + fail=$((fail + 1)) + fi +done +for f in "${!allowed[@]}"; do + if [ -z "${actual[$f]:-}" ]; then + echo "❌ stale ledger entry $f:${allowed[$f]} — no inline Python left; delete the line" + fail=$((fail + 1)) + fi +done + +if [ "$fail" -gt 0 ]; then + echo + echo "❌ inline-python: $fail problem(s); $total offending line(s) across ${#actual[@]} file(s)." + exit 1 +fi +echo "✅ inline-python: $total ledgered line(s) across ${#actual[@]} file(s), none new, none grown, none stale." diff --git a/scripts/tests/check-inline-python-test.sh b/scripts/tests/check-inline-python-test.sh new file mode 100755 index 000000000..54433967f --- /dev/null +++ b/scripts/tests/check-inline-python-test.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# check-inline-python-test.sh — fixture suite for scripts/check-inline-python.sh. +# +# Proves three things, each with a planted positive: +# 1. every embedded-Python form is caught, and look-alikes are not; +# 2. the ledger is shrink-only: new path, grown count and stale entry all fail; +# 3. the suite itself can see a regression — two mutants of the gate (stale +# check deleted, `-c` form deleted) must each turn one case green. +# +# Fixture text is assembled from fragments ($PY, $PIP) so this file never +# contains the literal forms and does not appear in the gate's own scan. +# +# Run: bash scripts/tests/check-inline-python-test.sh +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +GATE="${GATE:-$SCRIPT_DIR/../check-inline-python.sh}" +REPO_ROOT="$SCRIPT_DIR/../.." +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +PY="pyth""on3" +PIP="pi""p" +LT="<""<" +pass=0 fail=0 + +# fresh — empties the fixture tree $WORK/t (not a git repo, so the gate walks it). +fresh() { + rm -rf "$WORK/t"; mkdir -p "$WORK/t" +} + +# plant — writes stdin to $WORK/t/, creating parent dirs. +plant() { + mkdir -p "$(dirname "$WORK/t/$1")" + cat > "$WORK/t/$1" +} + +# ledger — writes stdin as the fixture tree's ledger. +ledger() { + plant .machine_readable/inline-python-allow.txt +} + +# expect