From 723600b9dd45ace52c2b345d101a951319be9866 Mon Sep 17 00:00:00 2001 From: Ralph Kuepper Date: Fri, 28 Aug 2026 10:16:21 +0200 Subject: [PATCH 1/2] ci: shard full-tier perry integration tests --- .github/workflows/test.yml | 123 ++++++++++++++++++++++- docs/src/testing/ci-tiers.md | 9 ++ scripts/ci_cargo_test_shard.py | 178 +++++++++++++++++++++++++++++++++ scripts/ci_plan.py | 33 +++++- 4 files changed, 336 insertions(+), 7 deletions(-) create mode 100755 scripts/ci_cargo_test_shard.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b518a5fedf..fe58cfceed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -248,6 +248,10 @@ jobs: if: ${{ !cancelled() }} run: | python3 scripts/ci_plan.py --self-test + python3 scripts/ci_cargo_test_shard.py --self-test + python3 scripts/ci_cargo_test_shard.py --package perry \ + --total-shards "${{ fromJSON(needs.plan.outputs.plan).cargo_test_perry.total }}" \ + --validate python3 scripts/ci_plan.py --table > /tmp/ci-plan-table.md if ! grep -qF -- "$(head -1 /tmp/ci-plan-table.md)" docs/src/testing/ci-tiers.md; then echo "::error::docs/src/testing/ci-tiers.md is missing the tier table header"; exit 1 @@ -1074,8 +1078,16 @@ jobs: # disk doesn't exhaust mid-job. export CARGO_BUILD_JOBS=1 for package in $(printf '%s\n' "$scope" | grep -vx 'perry-runtime'); do - echo "::group::cargo test -p $package" - cargo test -p "$package" + if [ "$package" = "perry" ] && [ "${{ needs.plan.outputs.tier }}" = "full" ]; then + # The full tier's cargo-test-perry matrix owns every perry + # integration target. Keep the bin/unit target here so the + # split is exhaustive without running any target twice. + echo "::group::cargo test --bins -p perry" + cargo test --bins -p perry + else + echo "::group::cargo test -p $package" + cargo test -p "$package" + fi echo "::endgroup::" cargo clean -p "$package" || true find target/debug/deps -maxdepth 1 -type f -perm -111 ! -name '*.so' -delete 2>/dev/null || true @@ -1163,6 +1175,112 @@ jobs: path: ${{ github.workspace }}/.sccache key: sccache-${{ runner.os }}-perry-${{ github.job }}-${{ github.run_id }} + # --------------------------------------------------------------------------- + # Full-tier perry integration tests (#8914) + # + # Release run 33123474583 had 246 `perry` integration-test binaries; the + # serial package invocation was still below their midpoint when its + # 180-minute timeout fired. The ordinary cargo-test job retains the + # perry bin/unit target and every other package; this matrix owns every perry + # integration target exactly once. The helper derives targets from Cargo + # metadata and round-robins the sorted inventory, so newly added suites are + # automatically covered and shard sizes differ by at most one. + # + # `fail-fast: false` preserves evidence from every shard. GitHub aggregates + # the matrix result under this job id, and full-suite-gate needs that result, + # so one failed, timed-out, or cancelled shard blocks a release. + # --------------------------------------------------------------------------- + cargo-test-perry: + name: cargo-test-perry (${{ matrix.shard }}/${{ fromJSON(needs.plan.outputs.plan).cargo_test_perry.total }}) + needs: plan + if: fromJSON(needs.plan.outputs.plan).jobs.cargo_test_perry + runs-on: ubuntu-latest + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON(needs.plan.outputs.plan).cargo_test_perry.shards }} + env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "false" + SCCACHE_DIR: ${{ github.workspace }}/.sccache + SCCACHE_CACHE_SIZE: "12G" + CARGO_INCREMENTAL: "0" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install Rust toolchain + run: rustup toolchain install nightly-2026-08-20 --profile minimal + - uses: ./.github/actions/setup-llvm22 + + - name: Install sccache + uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + + # Every shard starts from the same most-recent compiler-object cache. + # Shards deliberately do not save eight near-duplicate 12G archives: + # cargo-test remains the single main-line writer for this shared prefix. + - name: Restore sccache objects + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ${{ github.workspace }}/.sccache + key: sccache-${{ runner.os }}-perry-${{ github.job }}-${{ github.run_id }}-${{ matrix.shard }} + restore-keys: | + sccache-${{ runner.os }}-perry- + + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + shared-key: "${{ runner.os }}-perry" + save-if: false + + - name: Evict stale auto-opt archives (#5892) + run: | + rm -rf target/perry-auto-* target/debug/libperry_ext_*.a 2>/dev/null || true + + - name: Run perry integration shard ${{ matrix.shard }} + env: + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: "-C linker-features=-lld" + CARGO_PROFILE_TEST_DEBUG: "0" + CARGO_PROFILE_DEV_DEBUG: "0" + SHARD: ${{ matrix.shard }} + TOTAL_SHARDS: ${{ fromJSON(needs.plan.outputs.plan).cargo_test_perry.total }} + run: | + set -euo pipefail + ( + while sleep 60; do + echo "perry integration shard $SHARD/$TOTAL_SHARDS still running at $(date -u +%Y-%m-%dT%H:%M:%SZ)" + done + ) & + shard_heartbeat_pid=$! + trap 'kill "$shard_heartbeat_pid" 2>/dev/null || true' EXIT + + mapfile -t test_targets < <( + python3 scripts/ci_cargo_test_shard.py --package perry \ + --shard "$SHARD" --total-shards "$TOTAL_SHARDS" + ) + if [ "${#test_targets[@]}" -eq 0 ]; then + echo "::error::perry integration shard $SHARD/$TOTAL_SHARDS is empty" + exit 1 + fi + echo "perry integration shard $SHARD/$TOTAL_SHARDS: ${#test_targets[@]} targets" + printf ' %s\n' "${test_targets[@]}" + + # Match cargo-test's shipped panic semantics and provide the static + # archives used by PERRY_NO_AUTO_OPTIMIZE=1 integration fixtures. + cargo build --release -p perry-runtime -p perry-stdlib \ + -p perry-runtime-static -p perry-stdlib-static + export PERRY_RUNTIME_DIR="$PWD/target/release" + export CARGO_BUILD_JOBS=1 + + test_args=() + for test_target in "${test_targets[@]}"; do + test_args+=(--test "$test_target") + done + echo "::group::cargo test -p perry (integration shard $SHARD/$TOTAL_SHARDS)" + # Cargo prints `Running tests/.rs` before each binary, making + # the currently executing integration suite explicit in live logs. + cargo test -p perry "${test_args[@]}" + echo "::endgroup::" + # --------------------------------------------------------------------------- # Scoped e2e: run the integration suites NAMED BY THE DIFF (#5960) # @@ -3571,6 +3689,7 @@ jobs: - check - warnings - cargo-test + - cargo-test-perry - e2e-scoped - windows-build - windows-arm64-build diff --git a/docs/src/testing/ci-tiers.md b/docs/src/testing/ci-tiers.md index a6851459fc..a484489a89 100644 --- a/docs/src/testing/ci-tiers.md +++ b/docs/src/testing/ci-tiers.md @@ -27,6 +27,7 @@ copy is current. | `check` | yes | yes | yes | | `warnings` | yes | yes | yes | | `cargo-test` | yes | yes | yes | +| `cargo-test-perry` | | | yes | | `gap-suite` | 6x fast | 3x fast | 8x full | | `gc-stress` | yes | yes | yes | | `e2e-scoped` | yes | | | @@ -100,6 +101,14 @@ Two specific costs dominated: from main-line runs; PRs restore the newest main-line blob. `cache-warm.yml` is gone: the sweep is the cache-producing build on `main`. +A later release run exposed a separate serial bottleneck: its 246 `perry` +integration-test binaries had not reached their midpoint when `cargo-test` hit +its 180-minute cap. In the full tier, `cargo-test` now retains the `perry` bin/unit +target and every other package while `cargo-test-perry` assigns every `perry` +integration target exactly once across eight deterministic, count-balanced shards. +The shards restore the shared compiler cache but do not upload eight near-duplicate +copies. Their aggregate matrix result is a direct dependency of `full-suite-gate`. + The **satellite gates** (`gc-ratchet`, `gc-root-dominance`, `gc-native-roots`, `gc-moving-witnesses`, `gc-parse-churn-gate`, `gc-ptr-shape-off-witness`, `tls-budget`, `auto-opt-app-patterns`, `eh-transport`, `llvm-inprocess`, `ext-link`, diff --git a/scripts/ci_cargo_test_shard.py b/scripts/ci_cargo_test_shard.py new file mode 100755 index 0000000000..3a8e2adb59 --- /dev/null +++ b/scripts/ci_cargo_test_shard.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Deterministically shard a Cargo package's integration-test targets. + +The full CI tier runs the large `perry` integration-test inventory in a +matrix. This helper derives the inventory from `cargo metadata` instead of a +hand-maintained list, then assigns the sorted target names round-robin across +one-based shards. Every integration target therefore belongs to exactly one +shard, including a target added by the same commit that changes the workflow. + +Examples: + python3 scripts/ci_cargo_test_shard.py --package perry --shard 1 --total-shards 8 + python3 scripts/ci_cargo_test_shard.py --package perry --total-shards 8 --validate + python3 scripts/ci_cargo_test_shard.py --self-test +""" + +from __future__ import annotations + +import argparse +from collections import Counter +import json +import subprocess +import sys + + +def _load_metadata() -> dict: + raw = subprocess.check_output( + ["cargo", "metadata", "--no-deps", "--format-version", "1"] + ) + return json.loads(raw) + + +def integration_targets(metadata: dict, package_name: str) -> list[str]: + """Return every integration-test target for one workspace package.""" + packages = [ + package for package in metadata["packages"] if package["name"] == package_name + ] + if len(packages) != 1: + raise ValueError( + f"expected exactly one package named {package_name!r}, found {len(packages)}" + ) + + targets = sorted( + target["name"] + for target in packages[0]["targets"] + if "test" in target.get("kind", []) + ) + if len(targets) != len(set(targets)): + duplicates = sorted( + name for name, count in Counter(targets).items() if count > 1 + ) + raise ValueError(f"duplicate integration-test targets: {', '.join(duplicates)}") + if not targets: + raise ValueError(f"package {package_name!r} has no integration-test targets") + return targets + + +def shard_targets(targets: list[str], shard: int, total_shards: int) -> list[str]: + """Return a stable, count-balanced one-based round-robin partition.""" + if total_shards < 1: + raise ValueError("total shards must be at least 1") + if not 1 <= shard <= total_shards: + raise ValueError(f"shard must be between 1 and {total_shards}, got {shard}") + return sorted(targets)[shard - 1 :: total_shards] + + +def validate_assignments( + targets: list[str], total_shards: int, package_name: str +) -> None: + if total_shards < 1: + raise ValueError("total shards must be at least 1") + assignments = [ + shard_targets(targets, shard, total_shards) + for shard in range(1, total_shards + 1) + ] + empty = [index + 1 for index, assigned in enumerate(assignments) if not assigned] + if empty: + raise ValueError(f"empty shards: {', '.join(map(str, empty))}") + + assigned_counts = Counter(name for assigned in assignments for name in assigned) + expected_counts = Counter(targets) + if assigned_counts != expected_counts: + missing = sorted((expected_counts - assigned_counts).elements()) + repeated = sorted((assigned_counts - expected_counts).elements()) + raise ValueError( + f"invalid coverage: missing={missing or 'none'}, repeated={repeated or 'none'}" + ) + + sizes = [len(assigned) for assigned in assignments] + if max(sizes) - min(sizes) > 1: + raise ValueError(f"unbalanced shard sizes: {sizes}") + print( + f"{package_name}: {len(targets)} integration targets assigned exactly once " + f"across {total_shards} shards (sizes: {', '.join(map(str, sizes))})" + ) + + +def _self_test() -> int: + metadata = { + "packages": [ + { + "name": "perry", + "targets": [ + {"name": "z_suite", "kind": ["test"]}, + {"name": "perry", "kind": ["bin"]}, + {"name": "a_suite", "kind": ["test"]}, + ], + } + ] + } + if integration_targets(metadata, "perry") != ["a_suite", "z_suite"]: + print("Cargo metadata target discovery drifted", file=sys.stderr) + return 1 + + sample = [f"test_{index:02d}" for index in range(19)] + expected = [ + ["test_00", "test_04", "test_08", "test_12", "test_16"], + ["test_01", "test_05", "test_09", "test_13", "test_17"], + ["test_02", "test_06", "test_10", "test_14", "test_18"], + ["test_03", "test_07", "test_11", "test_15"], + ] + actual = [shard_targets(list(reversed(sample)), shard, 4) for shard in range(1, 5)] + if actual != expected: + print(f"round-robin assignment drifted: {actual!r}", file=sys.stderr) + return 1 + + flattened = [name for assigned in actual for name in assigned] + if Counter(flattened) != Counter(sample): + print("self-test assignment did not cover every target exactly once", file=sys.stderr) + return 1 + + for shard, total in ((0, 4), (5, 4), (1, 0)): + try: + shard_targets(sample, shard, total) + except ValueError: + pass + else: + print( + f"invalid shard {shard}/{total} was accepted", + file=sys.stderr, + ) + return 1 + + print("ci_cargo_test_shard --self-test: deterministic exact coverage holds") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--package", default="perry") + parser.add_argument("--shard", type=int) + parser.add_argument("--total-shards", type=int) + parser.add_argument("--validate", action="store_true") + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + + if args.self_test: + return _self_test() + if args.total_shards is None: + parser.error("--total-shards is required") + if args.validate and args.shard is not None: + parser.error("--validate and --shard are mutually exclusive") + if not args.validate and args.shard is None: + parser.error("--shard is required unless --validate is used") + + try: + targets = integration_targets(_load_metadata(), args.package) + if args.validate: + validate_assignments(targets, args.total_shards, args.package) + else: + for target in shard_targets(targets, args.shard, args.total_shards): + print(target) + except (KeyError, TypeError, ValueError) as exc: + parser.error(str(exc)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci_plan.py b/scripts/ci_plan.py index aff1ebdc54..69d5fb852a 100755 --- a/scripts/ci_plan.py +++ b/scripts/ci_plan.py @@ -76,6 +76,7 @@ "check": ("pr", "sweep", "full"), "warnings": ("pr", "sweep", "full"), "cargo_test": ("pr", "sweep", "full"), + "cargo_test_perry": ("full",), "gap_suite": ("pr", "sweep", "full"), "gc_stress": ("pr", "sweep", "full"), "e2e_scoped": ("pr",), # scoped to the PR's diff; meaningless without one @@ -129,6 +130,14 @@ # `jobs.parity`) merges the shard reports and runs the aggregate-only gates. PARITY_SHARDS = 12 +# The growing `perry` integration inventory cannot finish serially inside +# cargo-test's 180-minute release bound (#8914). Eight round-robin shards put +# the observed low-cache workload below 90 minutes while fitting alongside the +# existing eight gap workers in the org's 20-runner pool. The regular +# `cargo-test` job still covers the perry bin/unit target and every other +# package; this matrix owns only `perry` integration targets. +PERRY_INTEGRATION_SHARDS = 8 + EXTENDED_LABEL = "run-extended-tests" # The nightly (full-tier) cron in test.yml `on.schedule`. Any OTHER cron firing @@ -309,11 +318,15 @@ def plan( "jobs": jobs, "gap": gap, "parity": {"total": PARITY_SHARDS, "shards": list(range(1, PARITY_SHARDS + 1))}, - # cargo-test: a pull_request run scopes to the diff via ci_test_scope.py - # (`--lib --bins` of the affected crates); everything else -- including a - # `workflow_dispatch --tier pr`, which has no PR to read -- runs the full - # workspace with integration suites. - "cargo_test_scope": "pr" if event == "pull_request" else "full", + # cargo-test: the ordinary PR tier scopes to the diff via + # ci_test_scope.py (`--lib --bins` of the affected crates). A labelled + # full-tier PR must run the same complete package set as a nightly or + # release run; workflow dispatch has no PR to scope against either. + "cargo_test_scope": "pr" if tier == "pr" and event == "pull_request" else "full", + "cargo_test_perry": { + "total": PERRY_INTEGRATION_SHARDS, + "shards": list(range(1, PERRY_INTEGRATION_SHARDS + 1)), + }, # gc-stress: the PR subset of the GC x repsel matrix vs the full one. "gc_stress_mode": "pr" if tier == "pr" else "full", } @@ -405,6 +418,7 @@ def check(name: str, cond: bool): check("sweep: windows on", sweep["jobs"]["windows_build"] and sweep["jobs"]["windows_arm64_build"]) check("sweep: binary-size off (macOS report-only, nightly is enough)", not sweep["jobs"]["binary_size"]) check("sweep: parity off", not sweep["jobs"]["parity"]) + check("sweep: perry integration shards off", not sweep["jobs"]["cargo_test_perry"]) check("sweep: e2e-scoped off", not sweep["jobs"]["e2e_scoped"]) check("sweep: 3 fast gap shards", sweep["gap"]["total"] == 3 and sweep["gap"]["mode"] == "fast") check("sweep: cargo-test full", sweep["cargo_test_scope"] == "full") @@ -412,11 +426,20 @@ def check(name: str, cond: bool): full = plan("schedule", "refs/heads/main") check("full: every job except e2e-scoped", all(v for k, v in full["jobs"].items() if k != "e2e_scoped")) + check( + "full: perry integrations sharded", + full["cargo_test_perry"] + == { + "total": PERRY_INTEGRATION_SHARDS, + "shards": list(range(1, PERRY_INTEGRATION_SHARDS + 1)), + }, + ) check("full: 8 auto-optimize gap shards", full["gap"]["total"] == 8 and full["gap"]["mode"] == "full") check("full: parity sharded (6h-cap kill, 2026-08-16)", full["parity"]["total"] >= 2 and full["parity"]["shards"][0] == 1) labelled = plan("pull_request", "refs/pull/1/merge", labels=[EXTENDED_LABEL], changed=["README.md"]) check("labelled PR runs the full tier regardless of scope", labelled["jobs"]["parity"] and labelled["jobs"]["gap_suite"]) + check("labelled full-tier PR runs the full cargo-test scope", labelled["cargo_test_scope"] == "full") snap = plan("workflow_dispatch", "refs/heads/x", tier_input="pr", update_gap_snapshot=True) check("snapshot update: one fast shard", snap["gap"] == {"mode": "fast", "total": 1, "shards": [1], "update_snapshot": True}) From fd35ce5f9abfce4511564df1081a7049bf36cca2 Mon Sep 17 00:00:00 2001 From: Ralph Kuepper Date: Fri, 28 Aug 2026 10:17:29 +0200 Subject: [PATCH 2/2] docs: add CI sharding changeset --- changelog.d/8928-full-cargo-test-shards.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/8928-full-cargo-test-shards.md diff --git a/changelog.d/8928-full-cargo-test-shards.md b/changelog.d/8928-full-cargo-test-shards.md new file mode 100644 index 0000000000..b6babeccfb --- /dev/null +++ b/changelog.d/8928-full-cargo-test-shards.md @@ -0,0 +1 @@ +Shard the full-tier `perry` integration-test inventory across deterministic CI workers so release tests finish within their time budget without dropping coverage.