From 0e75ad4c5cd636d6a1b4223a8cea49a822cdd342 Mon Sep 17 00:00:00 2001 From: Frank O'Hara Date: Thu, 27 Aug 2026 06:42:44 -0600 Subject: [PATCH 1/2] ci: mutation-testing gate via flawd, report-only while it soaks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Diff-scoped 'flawd ci' on pull requests and a weekly full-depth audit on main, with SARIF into code scanning and the full report as an artifact. Threshold is 0: this is a dogfooding gate being soaked in public before any enforcement — reds here are findings, not blockers, until the score history earns a real threshold. Runners use flawd.ci.toml (same scope and budgets as flawd.toml; native execution backend, the documented hosted-runner setup — an ephemeral VM gains nothing from a per-run Docker image build). Fork PRs are skipped with an explanatory notice because repository secrets are unavailable to them. flawd is installed from the public release mirror with checksum verification and activated from a repo secret. --- .github/workflows/mutation.yml | 109 +++++++++++++++++++++++++++++++++ flawd.ci.toml | 43 +++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 .github/workflows/mutation.yml create mode 100644 flawd.ci.toml diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml new file mode 100644 index 0000000..23155b0 --- /dev/null +++ b/.github/workflows/mutation.yml @@ -0,0 +1,109 @@ +# Mutation testing gate — dogfooding flawd ci, report-only while it soaks. +# +# PRs get a diff-scoped run: mutants from changed source lines, judged by +# this branch's tests. A weekly full-depth run on main feeds the score +# history that will eventually set the enforced threshold. Threshold stays +# 0 (report-only) until the soak holds; the gate is flipped deliberately, +# not by default. +# +# The runner uses flawd.ci.toml (identical scope/budgets to flawd.toml, +# native execution backend — the documented hosted-runner setup). +name: Mutation Testing +on: + pull_request: + schedule: + - cron: "17 5 * * 1" # weekly full audit on main + workflow_dispatch: + +permissions: + contents: read + security-events: write # SARIF upload to code scanning + +env: + FLAWD_VERSION: v0.14.0 # pinned; bump deliberately + +jobs: + mutation: + # Repo secrets are unavailable to fork PRs; the notice job below + # explains the skip instead of red-Xing outside contributors. + if: >- + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 350 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # full history for git merge-base + + - name: Restore cargo + target cache + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: flawd-mutation-${{ runner.os }}-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }} + restore-keys: | + flawd-mutation-${{ runner.os }}- + + - name: Install coverage toolchain + run: | + rustup component add llvm-tools-preview + curl -fsSL "https://github.com/taiki-e/cargo-llvm-cov/releases/latest/download/cargo-llvm-cov-x86_64-unknown-linux-gnu.tar.gz" \ + | tar xzf - -C "$HOME/.cargo/bin" + + - name: Install flawd + run: | + curl -fsSL --retry 3 -o /usr/local/bin/flawd \ + "https://github.com/fixture-dev/flawd-releases/releases/download/${FLAWD_VERSION}/flawd-linux-x86_64" + curl -fsSL --retry 3 -o /tmp/flawd.sha256 \ + "https://github.com/fixture-dev/flawd-releases/releases/download/${FLAWD_VERSION}/flawd-linux-x86_64.sha256" + cd /usr/local/bin && echo "$(cut -d' ' -f1 /tmp/flawd.sha256) flawd" | sha256sum -c - + chmod +x /usr/local/bin/flawd + flawd --version + + - name: Activate flawd + run: flawd activate "${{ secrets.FLAWD_LICENSE_KEY }}" + + - name: Mutation test (PR, diff-scoped) + if: github.event_name == 'pull_request' + run: | + flawd ci --config flawd.ci.toml \ + --diff "${{ github.event.pull_request.base.sha }}" \ + --threshold 0 --out flawd-report + + - name: Mutation test (scheduled, full depth) + if: github.event_name != 'pull_request' + run: | + flawd run --config flawd.ci.toml --ci-mode --yes \ + --baseline-retries 3 --out flawd-report + + - name: Upload SARIF + if: always() + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: flawd-report/results.sarif + + - name: Upload report artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: flawd-report-${{ github.run_id }} + path: flawd-report/ + retention-days: 30 + + mutation-fork-notice: + if: >- + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest + steps: + - name: Explain the skip + run: | + { + echo "### Mutation testing skipped" + echo "Fork PRs cannot access the flawd license secret, so the" + echo "mutation check does not run here. A maintainer pushing the" + echo "branch to this repo will exercise it." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/flawd.ci.toml b/flawd.ci.toml new file mode 100644 index 0000000..d329f29 --- /dev/null +++ b/flawd.ci.toml @@ -0,0 +1,43 @@ +# CI-runner flawd config: identical scope/budgets to flawd.toml, but the +# execution backend is native — GitHub-hosted runners are ephemeral VMs +# with the Rust toolchain preinstalled, where a per-run Docker image build +# costs more than it isolates (measured; see flawd's ci-integration docs). +# Local development keeps the Docker setup in flawd.toml. + +[languages] +enabled = ["rs"] + +[coverage] +inputs = [ + { format = "lcov", path = "coverage/lcov.info" } +] +command = "mkdir -p coverage && cargo llvm-cov --workspace --lcov --output-path coverage/lcov.info" +timeout_seconds = 300 + +[coverage.per_test] +mode = "auto" +granularity = "test_function" +collect_language = "rust" +collect_command = "cargo test --workspace" + +[test] +command = "cargo test" +build_command = "cargo test --no-run" +timeout_seconds = 600 +baseline_required = true + +[mutations] +budget = 400 +per_file_budget = 80 +operators = ["core", "comparisons", "boolean", "returns"] +risky_operators = false + +[pare] +dedupe = true +sampling = "weighted" +cache = true +max_batch_seconds = 300 +max_batch_mutants = 25 + +[runner] +isolation = "native" From 159dea6d990e14eed4b88b8e8ea620b3924a142a Mon Sep 17 00:00:00 2001 From: Frank O'Hara Date: Thu, 27 Aug 2026 06:45:32 -0600 Subject: [PATCH 2/2] =?UTF-8?q?ci:=20guard=20SARIF=20upload=20=E2=80=94=20?= =?UTF-8?q?flawd=20v0.14.0=20writes=20none=20on=20a=20zero-mutant=20diff?= =?UTF-8?q?=20pass?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The next flawd release emits a valid empty SARIF on that path; the guard stays harmless after the version bump. --- .github/workflows/mutation.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index 23155b0..82cd4d5 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -79,8 +79,16 @@ jobs: flawd run --config flawd.ci.toml --ci-mode --yes \ --baseline-retries 3 --out flawd-report - - name: Upload SARIF + # flawd v0.14.0 writes no SARIF on a zero-mutant diff pass (fixed in + # the next release, which emits a valid empty SARIF); guard the upload + # so a trivial pass doesn't fail the job on a missing file. + - name: Check for SARIF + id: sarif if: always() + run: echo "exists=$([ -f flawd-report/results.sarif ] && echo true || echo false)" >> "$GITHUB_OUTPUT" + + - name: Upload SARIF + if: always() && steps.sarif.outputs.exists == 'true' uses: github/codeql-action/upload-sarif@v3 with: sarif_file: flawd-report/results.sarif