ci: shard full-tier perry integration tests

.github/workflows/test.yml

@@ -248,6 +248,10 @@ jobs:
         if: ${{ !cancelled() }}
         run: |
           python3 scripts/ci_plan.py --self-test
+          python3 scripts/ci_cargo_test_shard.py --self-test
+          python3 scripts/ci_cargo_test_shard.py --package perry \
+            --total-shards "${{ fromJSON(needs.plan.outputs.plan).cargo_test_perry.total }}" \
+            --validate
           python3 scripts/ci_plan.py --table > /tmp/ci-plan-table.md
           if ! grep -qF -- "$(head -1 /tmp/ci-plan-table.md)" docs/src/testing/ci-tiers.md; then
             echo "::error::docs/src/testing/ci-tiers.md is missing the tier table header"; exit 1
@@ -1074,8 +1078,16 @@ jobs:
             # disk doesn't exhaust mid-job.
             export CARGO_BUILD_JOBS=1
             for package in $(printf '%s\n' "$scope" | grep -vx 'perry-runtime'); do
-              echo "::group::cargo test -p $package"
-              cargo test -p "$package"
+              if [ "$package" = "perry" ] && [ "${{ needs.plan.outputs.tier }}" = "full" ]; then
+                # The full tier's cargo-test-perry matrix owns every perry
+                # integration target. Keep the bin/unit target here so the
+                # split is exhaustive without running any target twice.
+                echo "::group::cargo test --bins -p perry"
+                cargo test --bins -p perry
+              else
+                echo "::group::cargo test -p $package"
+                cargo test -p "$package"
+              fi
               echo "::endgroup::"
               cargo clean -p "$package" || true
               find target/debug/deps -maxdepth 1 -type f -perm -111 ! -name '*.so' -delete 2>/dev/null || true
@@ -1163,6 +1175,112 @@ jobs:
           path: ${{ github.workspace }}/.sccache
           key: sccache-${{ runner.os }}-perry-${{ github.job }}-${{ github.run_id }}
 
+  # ---------------------------------------------------------------------------
+  # Full-tier perry integration tests (#8914)
+  #
+  # Release run 33123474583 had 246 `perry` integration-test binaries; the
+  # serial package invocation was still below their midpoint when its
+  # 180-minute timeout fired. The ordinary cargo-test job retains the
+  # perry bin/unit target and every other package; this matrix owns every perry
+  # integration target exactly once. The helper derives targets from Cargo
+  # metadata and round-robins the sorted inventory, so newly added suites are
+  # automatically covered and shard sizes differ by at most one.
+  #
+  # `fail-fast: false` preserves evidence from every shard. GitHub aggregates
+  # the matrix result under this job id, and full-suite-gate needs that result,
+  # so one failed, timed-out, or cancelled shard blocks a release.
+  # ---------------------------------------------------------------------------
+  cargo-test-perry:
+    name: cargo-test-perry (${{ matrix.shard }}/${{ fromJSON(needs.plan.outputs.plan).cargo_test_perry.total }})
+    needs: plan
+    if: fromJSON(needs.plan.outputs.plan).jobs.cargo_test_perry
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: ${{ fromJSON(needs.plan.outputs.plan).cargo_test_perry.shards }}
+    env:
+      RUSTC_WRAPPER: sccache
+      SCCACHE_GHA_ENABLED: "false"
+      SCCACHE_DIR: ${{ github.workspace }}/.sccache
+      SCCACHE_CACHE_SIZE: "12G"
+      CARGO_INCREMENTAL: "0"
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+
+      - name: Install Rust toolchain
+        run: rustup toolchain install nightly-2026-08-20 --profile minimal
+      - uses: ./.github/actions/setup-llvm22
+
+      - name: Install sccache
+        uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+
+      # Every shard starts from the same most-recent compiler-object cache.
+      # Shards deliberately do not save eight near-duplicate 12G archives:
+      # cargo-test remains the single main-line writer for this shared prefix.
+      - name: Restore sccache objects
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: ${{ github.workspace }}/.sccache
+          key: sccache-${{ runner.os }}-perry-${{ github.job }}-${{ github.run_id }}-${{ matrix.shard }}
+          restore-keys: |
+            sccache-${{ runner.os }}-perry-
+
+      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
+        with:
+          shared-key: "${{ runner.os }}-perry"
+          save-if: false
+
+      - name: Evict stale auto-opt archives (#5892)
+        run: |
+          rm -rf target/perry-auto-* target/debug/libperry_ext_*.a 2>/dev/null || true
+
+      - name: Run perry integration shard ${{ matrix.shard }}
+        env:
+          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: "-C linker-features=-lld"
+          CARGO_PROFILE_TEST_DEBUG: "0"
+          CARGO_PROFILE_DEV_DEBUG: "0"
+          SHARD: ${{ matrix.shard }}
+          TOTAL_SHARDS: ${{ fromJSON(needs.plan.outputs.plan).cargo_test_perry.total }}
+        run: |
+          set -euo pipefail
+          (
+            while sleep 60; do
+              echo "perry integration shard $SHARD/$TOTAL_SHARDS still running at $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+            done
+          ) &
+          shard_heartbeat_pid=$!
+          trap 'kill "$shard_heartbeat_pid" 2>/dev/null || true' EXIT
+
+          mapfile -t test_targets < <(
+            python3 scripts/ci_cargo_test_shard.py --package perry \
+              --shard "$SHARD" --total-shards "$TOTAL_SHARDS"
+          )
+          if [ "${#test_targets[@]}" -eq 0 ]; then
+            echo "::error::perry integration shard $SHARD/$TOTAL_SHARDS is empty"
+            exit 1
+          fi
+          echo "perry integration shard $SHARD/$TOTAL_SHARDS: ${#test_targets[@]} targets"
+          printf '  %s\n' "${test_targets[@]}"
+
+          # Match cargo-test's shipped panic semantics and provide the static
+          # archives used by PERRY_NO_AUTO_OPTIMIZE=1 integration fixtures.
+          cargo build --release -p perry-runtime -p perry-stdlib \
+            -p perry-runtime-static -p perry-stdlib-static
+          export PERRY_RUNTIME_DIR="$PWD/target/release"
+          export CARGO_BUILD_JOBS=1
+
+          test_args=()
+          for test_target in "${test_targets[@]}"; do
+            test_args+=(--test "$test_target")
+          done
+          echo "::group::cargo test -p perry (integration shard $SHARD/$TOTAL_SHARDS)"
+          # Cargo prints `Running tests/<target>.rs` before each binary, making
+          # the currently executing integration suite explicit in live logs.
+          cargo test -p perry "${test_args[@]}"
+          echo "::endgroup::"
+
   # ---------------------------------------------------------------------------
   # Scoped e2e: run the integration suites NAMED BY THE DIFF (#5960)
   #
@@ -3571,6 +3689,7 @@ jobs:
       - check
       - warnings
       - cargo-test
+      - cargo-test-perry
       - e2e-scoped
       - windows-build
       - windows-arm64-build

changelog.d/8928-full-cargo-test-shards.md

@@ -0,0 +1 @@
+Shard the full-tier `perry` integration-test inventory across deterministic CI workers so release tests finish within their time budget without dropping coverage.

docs/src/testing/ci-tiers.md

@@ -27,6 +27,7 @@ copy is current.
 | `check` | yes | yes | yes |
 | `warnings` | yes | yes | yes |
 | `cargo-test` | yes | yes | yes |
+| `cargo-test-perry` |  |  | yes |
 | `gap-suite` | 6x fast | 3x fast | 8x full |
 | `gc-stress` | yes | yes | yes |
 | `e2e-scoped` | yes |  |  |
@@ -100,6 +101,14 @@ Two specific costs dominated:
   from main-line runs; PRs restore the newest main-line blob. `cache-warm.yml` is
   gone: the sweep is the cache-producing build on `main`.
 
+A later release run exposed a separate serial bottleneck: its 246 `perry`
+integration-test binaries had not reached their midpoint when `cargo-test` hit
+its 180-minute cap. In the full tier, `cargo-test` now retains the `perry` bin/unit
+target and every other package while `cargo-test-perry` assigns every `perry`
+integration target exactly once across eight deterministic, count-balanced shards.
+The shards restore the shared compiler cache but do not upload eight near-duplicate
+copies. Their aggregate matrix result is a direct dependency of `full-suite-gate`.
+
 The **satellite gates** (`gc-ratchet`, `gc-root-dominance`, `gc-native-roots`,
 `gc-moving-witnesses`, `gc-parse-churn-gate`, `gc-ptr-shape-off-witness`,
 `tls-budget`, `auto-opt-app-patterns`, `eh-transport`, `llvm-inprocess`, `ext-link`,

scripts/ci_cargo_test_shard.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+"""Deterministically shard a Cargo package's integration-test targets.
+
+The full CI tier runs the large `perry` integration-test inventory in a
+matrix.  This helper derives the inventory from `cargo metadata` instead of a
+hand-maintained list, then assigns the sorted target names round-robin across
+one-based shards.  Every integration target therefore belongs to exactly one
+shard, including a target added by the same commit that changes the workflow.
+
+Examples:
+  python3 scripts/ci_cargo_test_shard.py --package perry --shard 1 --total-shards 8
+  python3 scripts/ci_cargo_test_shard.py --package perry --total-shards 8 --validate
+  python3 scripts/ci_cargo_test_shard.py --self-test
+"""
+
+from __future__ import annotations
+
+import argparse
+from collections import Counter
+import json
+import subprocess
+import sys
+
+
+def _load_metadata() -> dict:
+    raw = subprocess.check_output(
+        ["cargo", "metadata", "--no-deps", "--format-version", "1"]
+    )
+    return json.loads(raw)
+
+
+def integration_targets(metadata: dict, package_name: str) -> list[str]:
+    """Return every integration-test target for one workspace package."""
+    packages = [
+        package for package in metadata["packages"] if package["name"] == package_name
+    ]
+    if len(packages) != 1:
+        raise ValueError(
+            f"expected exactly one package named {package_name!r}, found {len(packages)}"
+        )
+
+    targets = sorted(
+        target["name"]
+        for target in packages[0]["targets"]
+        if "test" in target.get("kind", [])
+    )
+    if len(targets) != len(set(targets)):
+        duplicates = sorted(
+            name for name, count in Counter(targets).items() if count > 1
+        )
+        raise ValueError(f"duplicate integration-test targets: {', '.join(duplicates)}")
+    if not targets:
+        raise ValueError(f"package {package_name!r} has no integration-test targets")
+    return targets
+
+
+def shard_targets(targets: list[str], shard: int, total_shards: int) -> list[str]:
+    """Return a stable, count-balanced one-based round-robin partition."""
+    if total_shards < 1:
+        raise ValueError("total shards must be at least 1")
+    if not 1 <= shard <= total_shards:
+        raise ValueError(f"shard must be between 1 and {total_shards}, got {shard}")
+    return sorted(targets)[shard - 1 :: total_shards]
+
+
+def validate_assignments(
+    targets: list[str], total_shards: int, package_name: str
+) -> None:
+    if total_shards < 1:
+        raise ValueError("total shards must be at least 1")
+    assignments = [
+        shard_targets(targets, shard, total_shards)
+        for shard in range(1, total_shards + 1)
+    ]
+    empty = [index + 1 for index, assigned in enumerate(assignments) if not assigned]
+    if empty:
+        raise ValueError(f"empty shards: {', '.join(map(str, empty))}")
+
+    assigned_counts = Counter(name for assigned in assignments for name in assigned)
+    expected_counts = Counter(targets)
+    if assigned_counts != expected_counts:
+        missing = sorted((expected_counts - assigned_counts).elements())
+        repeated = sorted((assigned_counts - expected_counts).elements())
+        raise ValueError(
+            f"invalid coverage: missing={missing or 'none'}, repeated={repeated or 'none'}"
+        )
+
+    sizes = [len(assigned) for assigned in assignments]
+    if max(sizes) - min(sizes) > 1:
+        raise ValueError(f"unbalanced shard sizes: {sizes}")
+    print(
+        f"{package_name}: {len(targets)} integration targets assigned exactly once "
+        f"across {total_shards} shards (sizes: {', '.join(map(str, sizes))})"
+    )
+
+
+def _self_test() -> int:
+    metadata = {
+        "packages": [
+            {
+                "name": "perry",
+                "targets": [
+                    {"name": "z_suite", "kind": ["test"]},
+                    {"name": "perry", "kind": ["bin"]},
+                    {"name": "a_suite", "kind": ["test"]},
+                ],
+            }
+        ]
+    }
+    if integration_targets(metadata, "perry") != ["a_suite", "z_suite"]:
+        print("Cargo metadata target discovery drifted", file=sys.stderr)
+        return 1
+
+    sample = [f"test_{index:02d}" for index in range(19)]
+    expected = [
+        ["test_00", "test_04", "test_08", "test_12", "test_16"],
+        ["test_01", "test_05", "test_09", "test_13", "test_17"],
+        ["test_02", "test_06", "test_10", "test_14", "test_18"],
+        ["test_03", "test_07", "test_11", "test_15"],
+    ]
+    actual = [shard_targets(list(reversed(sample)), shard, 4) for shard in range(1, 5)]
+    if actual != expected:
+        print(f"round-robin assignment drifted: {actual!r}", file=sys.stderr)
+        return 1
+
+    flattened = [name for assigned in actual for name in assigned]
+    if Counter(flattened) != Counter(sample):
+        print("self-test assignment did not cover every target exactly once", file=sys.stderr)
+        return 1
+
+    for shard, total in ((0, 4), (5, 4), (1, 0)):
+        try:
+            shard_targets(sample, shard, total)
+        except ValueError:
+            pass
+        else:
+            print(
+                f"invalid shard {shard}/{total} was accepted",
+                file=sys.stderr,
+            )
+            return 1
+
+    print("ci_cargo_test_shard --self-test: deterministic exact coverage holds")
+    return 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--package", default="perry")
+    parser.add_argument("--shard", type=int)
+    parser.add_argument("--total-shards", type=int)
+    parser.add_argument("--validate", action="store_true")
+    parser.add_argument("--self-test", action="store_true")
+    args = parser.parse_args()
+
+    if args.self_test:
+        return _self_test()
+    if args.total_shards is None:
+        parser.error("--total-shards is required")
+    if args.validate and args.shard is not None:
+        parser.error("--validate and --shard are mutually exclusive")
+    if not args.validate and args.shard is None:
+        parser.error("--shard is required unless --validate is used")
+
+    try:
+        targets = integration_targets(_load_metadata(), args.package)
+        if args.validate:
+            validate_assignments(targets, args.total_shards, args.package)
+        else:
+            for target in shard_targets(targets, args.shard, args.total_shards):
+                print(target)
+    except (KeyError, TypeError, ValueError) as exc:
+        parser.error(str(exc))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())