diff --git a/.github/scripts/check-executorch-runtime-wheel.py b/.github/scripts/check-executorch-runtime-wheel.py new file mode 100644 index 0000000000..b320ddcd4b --- /dev/null +++ b/.github/scripts/check-executorch-runtime-wheel.py @@ -0,0 +1,201 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +"""Validate the repaired companion wheel before it enters the shared artifact.""" + +import argparse +import ast +import importlib.metadata +import re +import sys +from email.parser import BytesParser +from pathlib import Path + +import yaml +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name, parse_wheel_filename +from packaging.version import Version +from wheel.wheelfile import WheelFile + + +def reject(message): + sys.exit(f"FATAL: {message}") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("wheel", type=Path) + parser.add_argument("--architecture", choices=("x86_64", "aarch64"), required=True) + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + source = root / "py/torch-tensorrt-executorch-runtime/setup.py" + (library,) = [ + ast.literal_eval(node.value) + for node in ast.parse(source.read_text()).body + if isinstance(node, ast.Assign) + and any( + getattr(target, "id", None) == "DELEGATE_LIBRARY" for target in node.targets + ) + ] + with WheelFile(args.wheel) as archive: + names = archive.namelist() + objects = sorted(name for name in names if re.search(r"\.so(\.\d+)*$", name)) + expected = f"torch_tensorrt_executorch_runtime/lib/{library}" + if objects != [expected]: + reject(f"expected {expected} and no other shared libraries, got {objects}") + else: + # The platform tag is a claim about the payload, and until now nothing read the payload + # to check it, so a wheel tagged for one architecture could carry a library built for + # the other and pass. The ELF header names the machine in two bytes at offset 18. + header = archive.read(expected)[:20] + if header[:4] != b"\x7fELF": + reject(f"{expected} is not an ELF object") + machine = int.from_bytes(header[18:20], "little") + wanted = {"x86_64": 0x3E, "aarch64": 0xB7}[args.architecture] + if machine != wanted: + names_by_machine = {0x3E: "x86_64", 0xB7: "aarch64"} + reject( + f"{expected} is built for " + f"{names_by_machine.get(machine, hex(machine))}, but this wheel is tagged for " + f"{args.architecture}" + ) + for filename in ( + "torchtrt_executorch-config.cmake", + "torchtrt_executorch-config-version.cmake", + ): + if ( + f"torch_tensorrt_executorch_runtime/lib/cmake/torchtrt_executorch/{filename}" + not in names + ): + reject(f"the wheel ships no CMake package: {filename} is missing") + forbidden = [ + name + for name in names + if any( + part in name + for part in ( + "_portable_lib", + "libexecutorch.so", + "libextension_cuda", + "libaoti_cuda_shims", + ) + ) + ] + if forbidden: + reject(f"the wheel ships ExecuTorch runtime components: {forbidden}") + + name, version, _, tags = parse_wheel_filename(args.wheel.name) + if name != "torch-tensorrt-executorch-runtime": + reject(f"unexpected distribution: {name}") + # The floors differ by architecture. The Arm build container carries no devtoolset, so the + # C++ runtime symbols the delegate references are not absorbed statically the way they are + # on x86, and the wheel genuinely needs the newer baseline. + floor = {"x86_64": "2_28", "aarch64": "2_35"}[args.architecture] + expected_tag = f"py3-none-manylinux_{floor}_{args.architecture}" + if {str(tag) for tag in tags} != {expected_tag}: + reject(f"expected repaired tag {expected_tag}, got {tags}") + wheel_metadata = BytesParser().parsebytes( + archive.read(f"{archive.dist_info_path}/WHEEL") + ) + if wheel_metadata.get("Root-Is-Purelib") != "false": + reject("wheel declares itself pure python") + if set(wheel_metadata.get_all("Tag", [])) != {expected_tag}: + reject("WHEEL tags disagree with the filename") + metadata = BytesParser().parsebytes( + archive.read(f"{archive.dist_info_path}/METADATA") + ) + if ( + canonicalize_name(metadata["Name"]) != name + or Version(metadata["Version"]) != version + ): + reject("METADATA name/version disagree with the filename") + requirements = [ + Requirement(value) for value in metadata.get_all("Requires-Dist", []) + ] + pinned = yaml.safe_load((root / "dev_dep_versions.yml").read_text())[ + "__executorch_version__" + ] + for distribution in ( + "executorch", + "torch-tensorrt", + "torch", + "tensorrt-cu13", + "nvidia-cuda-runtime", + ): + if distribution == "executorch": + # The delegate links one specific ExecuTorch build, so its requirement carries the + # label naming that build. Without it the requirement is satisfied by a + # processor-only build, or another CUDA build of the same date. Compare against the + # installed wheel, whose label is the one the delegate actually linked, and check + # the public part still matches the repository pin. + installed = Version(importlib.metadata.version(distribution)) + if installed.public != pinned: + reject( + f"the repository pins executorch=={pinned}, but this wheel was built " + f"against {installed}, whose version differs from that pin" + ) + expected_version = str(installed) + else: + expected_version = Version( + importlib.metadata.version(distribution) + ).public + matched = [ + r for r in requirements if canonicalize_name(r.name) == distribution + ] + if ( + len(matched) != 1 + or str(matched[0].specifier) != f"=={expected_version}" + or matched[0].marker + or matched[0].extras + or matched[0].url + ): + reason = ( + "the repository pins" + if distribution == "executorch" + else "the build used" + ) + reject( + f"{reason} {distribution}=={expected_version}, but the wheel requires {matched}" + ) + # The three runtimes the delegate links carry the label naming the build, deliberately: it + # links one specific build of each, and without the label the requirement is satisfied by a + # processor-only build or another CUDA build of the same date. They resolve from the CUDA + # channel this wheel already requires. Everything else stays label-free so it resolves + # anywhere, and a label appearing there would narrow the wheel for no reason. + linked = {"executorch", "torch", "torch-tensorrt"} + labelled = [ + requirement + for requirement in requirements + if "+" in str(requirement.specifier) + and canonicalize_name(requirement.name) not in linked + ] + if labelled: + reject( + f"a requirement this delegate does not link carries a local label: {labelled}" + ) + # ExecuTorch has to carry one, because its pin comes from a file in the repository that + # always names a CUDA build. The other two take whatever the environment that built the + # wheel had installed, and an environment can legitimately hold a version with no label, + # so requiring one there would fail a wheel for something outside its control. + unlabelled = [ + requirement + for requirement in requirements + if canonicalize_name(requirement.name) == "executorch" + and "+" not in str(requirement.specifier) + ] + if unlabelled: + reject( + "the ExecuTorch pin carries no label naming its build, so a processor-only " + f"build would satisfy it: {unlabelled}" + ) + # Reading every member verifies its RECORD hash, including the delegate payload. + for filename in names: + if not filename.endswith("/"): + archive.read(filename) + print( + f"Validated {args.wheel.name}: one delegate, matching dependencies and {expected_tag}" + ) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/install-torch-tensorrt.sh b/.github/scripts/install-torch-tensorrt.sh index 1d890b4882..6db3b1c92e 100755 --- a/.github/scripts/install-torch-tensorrt.sh +++ b/.github/scripts/install-torch-tensorrt.sh @@ -55,13 +55,32 @@ fi # Install Torch-TensorRT if [[ ${PLATFORM} == win32 ]]; then + # Same exclusion as the Linux branch below, and for the same reason: the wheel's name varies by + # variant, so anchoring on a prefix leaves the pattern unexpanded and pip reads it literally. + wheels="" + for wheel in "${RUNNER_ARTIFACT_DIR}"/torch_tensorrt*.whl; do + case "${wheel}" in + *executorch_runtime*) continue ;; + esac + wheels="${wheels} ${wheel}" + done # pin-check: no-nightly -- Windows installs only the main wheel, without the Linux companion. - python -m pip install ${RUNNER_ARTIFACT_DIR}/torch_tensorrt*.whl || exit 1 + python -m pip install ${wheels} || exit 1 else - # The companion requires the nightly ExecuTorch channel even on test/release jobs. + # Every built wheel except the companion. Installing the companion here is what forced a + # nightly index onto release jobs; the ExecuTorch workflow installs it instead, naming the + # channel it wants. Selecting by exclusion rather than by prefix, because the main wheel's name + # varies by variant and a prefix guess leaves the glob unexpanded and pip reading it literally. + wheels="" + for wheel in /opt/torch-tensorrt-builds/torch_tensorrt*.whl; do + case "${wheel}" in + *executorch_runtime*) continue ;; + esac + wheels="${wheels} ${wheel}" + done # Exit explicitly: the caller appends its test script and this file does not use set -e. - python -m pip install /opt/torch-tensorrt-builds/torch_tensorrt*.whl --use-deprecated=legacy-resolver \ - --extra-index-url "https://download.pytorch.org/whl/nightly/${CU_VERSION}" || exit 1 + # pin-check: no-nightly -- the main wheel alone, which needs no ExecuTorch channel. + python -m pip install ${wheels} --use-deprecated=legacy-resolver || exit 1 fi echo -e "Running test script"; diff --git a/.github/scripts/update_executorch_pin.py b/.github/scripts/update_executorch_pin.py index 5ecfd6f1de..bb83d9c24c 100644 --- a/.github/scripts/update_executorch_pin.py +++ b/.github/scripts/update_executorch_pin.py @@ -30,19 +30,24 @@ # This allowlist prevents a release version from rewriting unrelated dependencies # or content-addressed wheel URLs. The repository guard inventories sites separately. -_PIN_SITES = ( - ".github/workflows/build_linux.yml", - ".github/workflows/executorch-test-linux.yml", - "MODULE.bazel", - "docker/MODULE.bazel.docker", - "docker/MODULE.bazel.ngc", - "justfile", - "pyproject.toml", - "py/torch-tensorrt-executorch-runtime/README.md", - "py/torch-tensorrt-executorch-runtime/pyproject.toml", - "toolchains/ci_workspaces/MODULE.bazel.tmpl", - "examples/executorch_reference_runner/README.md", -) +# Which coordinate each site carries. Checking "a version or a commit was found" lets a file that +# carries both satisfy the check on the commit alone, so a requirement the pattern stops matching +# would leave the version stale while the commit moves. That split is the whole thing these two pins +# exist to prevent, so each site declares what it must contain and each is verified on its own. +_SITE_COORDINATES: dict[str, frozenset[str]] = { + ".github/workflows/build_linux.yml": frozenset({"version"}), + ".github/workflows/executorch-test-linux.yml": frozenset({"version"}), + "MODULE.bazel": frozenset({"version", "commit"}), + "docker/MODULE.bazel.docker": frozenset({"version", "commit"}), + "docker/MODULE.bazel.ngc": frozenset({"version", "commit"}), + "justfile": frozenset({"version"}), + "pyproject.toml": frozenset({"version"}), + "py/torch-tensorrt-executorch-runtime/README.md": frozenset({"version"}), + "py/torch-tensorrt-executorch-runtime/pyproject.toml": frozenset({"version"}), + "toolchains/ci_workspaces/MODULE.bazel.tmpl": frozenset({"version", "commit"}), + "examples/executorch_reference_runner/README.md": frozenset({"commit"}), +} +_PIN_SITES = tuple(_SITE_COORDINATES) _CLAUSE = r"(?:===|==|>=|<=|~=|!=|<|>)\s*[^\s\"'`,;()]+" _MARKER_VALUE = r"""(?:[a-z_]+|"[^"\n]*"|'[^'\n]*')""" _MARKER_ATOM = rf"(?:\([ \t]*)*{_MARKER_VALUE}[ \t]*(?:===|==|>=|<=|~=|!=|<|>|not[ \t]+in|in)[ \t]*{_MARKER_VALUE}(?:[ \t]*\))*" @@ -81,7 +86,10 @@ def available_versions(index_args: list[str]) -> list[str]: ) match = re.search(r"^\s*Available versions:\s*(.+)$", out, re.MULTILINE) if match is None: - raise SystemExit("pip index versions printed no Available versions line") + # A channel with no ExecuTorch release yet prints no such line. That is the expected state + # between adding a CUDA minor's rows and the first ExecuTorch build for it, so report it as + # an empty list and let the caller say which channel is not ready. + return [] return [v.strip() for v in match.group(1).split(",") if v.strip()] @@ -223,10 +231,17 @@ def write_pins(new_version: str, new_commit: str) -> bool: exact = SpecifierSet(f"=={old_version}") ranged = SpecifierSet(f">={old_version},<{_upper_bound(old_version)}") + # A site already at the target is accepted and left alone, so a run interrupted partway can be + # repeated. Writing twelve files is not atomic, and treating an already-moved site as stale meant + # the first interruption wedged every later attempt at the same version. + done_exact = SpecifierSet(f"=={new_version}") + done_ranged = SpecifierSet(f">={new_version},<{_upper_bound(new_version)}") def rewrite_requirement(match: re.Match[str]) -> str: original = match.group(0) parsed = Requirement(original) + if parsed.specifier in (done_exact, done_ranged): + return original if parsed.url or parsed.specifier not in (exact, ranged): raise ValueError(f"unsupported or stale ExecuTorch requirement: {original}") constraints = match["constraints"] @@ -255,12 +270,36 @@ def rewrite_requirement(match: re.Match[str]) -> str: else: updated, count = _REQUIREMENT.subn(rewrite_requirement, text) updated = updated.replace(old_commit, new_commit) - # Either coordinate may legitimately be the only one a site carries, and a - # site already at the target is satisfied rather than broken, which is what - # lets a later run finish an interrupted one. - if not count and old_commit not in text: - if new_version not in text and new_commit not in text: - raise ValueError("no current version or source pin found") + try: + name = str(path.relative_to(_REPO_ROOT)).replace("\\", "/") + except ValueError: + name = str(path) + # A site outside the declaration, which only a caller substituting its own list + # produces. It is required to carry whichever single coordinate it appears to hold, + # rather than either of the two, since there is nothing declaring what it should. + expected = _SITE_COORDINATES.get( + name, + ( + frozenset({"version"}) + if _REQUIREMENT.search(text) + else frozenset({"commit"}) + ), + ) + # Each declared coordinate on its own, and by the same pattern that does the + # rewriting. Accepting a bare occurrence of the target version anywhere in the file + # let an unrelated package at that version stand in for the requirement, and it was + # never needed for convergence: a site a previous run already moved still matches + # the pattern, so it still counts. + if "version" in expected and not count: + raise ValueError( + "carries no ExecuTorch version requirement to move" + ) + if ( + "commit" in expected + and old_commit not in text + and new_commit not in text + ): + raise ValueError("carries no ExecuTorch source commit to move") pending.append((path, text, updated)) except ( OSError, @@ -361,6 +400,19 @@ def main(argv: list[str] | None = None) -> int: parser.error( f"the delegate builds {', '.join(accepted)}, so the channel must be one of them" ) + if args.track == "stable": + # Say this here rather than leaving it to be discovered from a failed build. The delegate + # links the ExecuTorch runtime, so its build accepts only a CUDA-labelled one, and no stable + # channel publishes such a build today: the release index carries processor-only wheels and + # the CUDA channels carry no stable ExecuTorch at all. So a stable pin lands as a pull + # request that cannot build, and the build says why. Not refused, because this becomes + # correct as soon as a stable CUDA build exists. + print( + "warning: the stable track selects from an index that publishes no CUDA build of " + "ExecuTorch, and the delegate build rejects anything else, so this pin is expected to " + "fail to build until a stable CUDA build exists", + file=sys.stderr, + ) index_args = _index_args(args.track, args.channel) candidates = available_versions(index_args) if args.track == "nightly": diff --git a/.github/scripts/verify-executorch-reference-runner.sh b/.github/scripts/verify-executorch-reference-runner.sh index fa5881f5fc..a7c3475e02 100755 --- a/.github/scripts/verify-executorch-reference-runner.sh +++ b/.github/scripts/verify-executorch-reference-runner.sh @@ -615,4 +615,27 @@ if [[ -n "${coalesced_model_path}" ]]; then # TensorRT, AOTInductor and eager PyTorch compute the same math with different # kernels, so compare within a tolerance instead of on the printed digits. assert_runner_output "${coalesced_runner_log}" "${coalesced_shape}" "${coalesced_value}" 0.001 + + # The same program again, on a caller stream created inside a green context. This is the + # combination the delegate exists for and the one nothing else here covers: two backends in one + # program, every activation on the device, and both confined to the caller's stream and its + # slice of the machine. A delegate that ignored the caller stream would still return the right + # numbers on an idle GPU, so the value is asserted and not just the exit status. + green_runner_log="${verify_root}/coalesced_green_context.log" + if "${runner_path}" \ + --model_path="${coalesced_model_path}" \ + --green_context_sms=8 \ + --num_runs=2 2>&1 | tee "${green_runner_log}"; then + assert_runner_output "${green_runner_log}" "${coalesced_shape}" "${coalesced_value}" 0.001 + else + # A green context needs driver and hardware support, so a refusal is a skip. Anything else is + # the delegate breaking on a caller-provided stream, which is a failure. + if grep -qiE "green context|cuDevSmResource|not supported|CUDA_ERROR_NOT_SUPPORTED" \ + "${green_runner_log}"; then + echo "green context unavailable on this runner, skipping that case" >&2 + else + echo "the coalesced program failed on a caller-provided stream" >&2 + exit 1 + fi + fi fi diff --git a/.github/workflows/build_linux.yml b/.github/workflows/build_linux.yml index c230ac1b88..d54fe31691 100644 --- a/.github/workflows/build_linux.yml +++ b/.github/workflows/build_linux.yml @@ -403,11 +403,41 @@ jobs: run: | set -euxo pipefail source "${BUILD_ENV_FILE}" - # The runtime links against the just-built standard wheel. Install it - # without resolving dependencies so it cannot replace the matrix's - # pinned torch/TensorRT packages. - ${CONDA_RUN} python -m pip install --no-deps dist/torch_tensorrt*.whl - ${CONDA_RUN} python -m pip install pyyaml \ + # Preserve the matrix's torch while installing the just-built main wheel. + ${CONDA_RUN} python -m pip install --no-deps dist/torch_tensorrt-*.whl + ${CONDA_RUN} python - <<'PY' + import importlib.metadata + import subprocess + import sys + + from packaging.requirements import Requirement + from packaging.utils import canonicalize_name + + main = importlib.metadata.distribution("torch-tensorrt") + library = main.locate_file("torch_tensorrt/lib/libtorchtrt.so") + if not library.is_file(): + sys.exit("The main wheel has no libtorchtrt.so; the companion needs the full runtime build") + # Native archives do not install Python metadata. Reuse the main wheel's selection. + requirements = [Requirement(value) for value in main.requires or []] + tensorrt = [ + str(requirement) for requirement in requirements + if canonicalize_name(requirement.name) in { + "tensorrt", "tensorrt-cu13", "tensorrt-cu13-bindings", "tensorrt-cu13-libs" + } + and (requirement.marker is None or requirement.marker.evaluate()) + ] + if not tensorrt: + sys.exit("The main wheel declares no CUDA 13 TensorRT dependencies") + subprocess.check_call([sys.executable, "-m", "pip", "install", *tensorrt]) + PY + # The nightly channel is named directly rather than through the channel variable, on release + # rows too, because the pin is a dated development build that only that channel carries. + # The default index stays in the set for the other requirements, which means a pin naming a + # version the default index also publishes could resolve to the processor-only build from + # there. What refuses that is the delegate's own build, which reads the installed + # ExecuTorch's local label and stops unless it names a CUDA build, so the failure is a clear + # message rather than a configure error about a missing extension. + ${CONDA_RUN} python -m pip install pyyaml patchelf "wheel>=0.40" \ --extra-index-url "https://download.pytorch.org/whl/nightly/${CU_VERSION}" \ "executorch==1.6.0.dev20260915" # Give the companion wheel the same channel/date and CUDA local-version @@ -443,7 +473,22 @@ jobs: continue fi abs_pkg=$(realpath $pkg) - ./test-infra/.github/scripts/repair_manylinux_2_28.sh $abs_pkg + if [[ "$(basename "${pkg}")" == torch_tensorrt_executorch_runtime-* ]]; then + # The companion is retagged, not repaired, and that is deliberate rather than a gap. The + # shared repair script handles CPython-tagged wheels only, and repairing would vendor the + # libraries this wheel exists to stop shipping. Its one shared object already resolves + # everything it needs through search paths relative to its own location, the same way + # ExecuTorch's backends do, so there is nothing for a repair to add. What it does need is a + # platform tag naming the floor the object really requires, which is what happens here. + case "${ARCH}" in + x86_64) platform_tag=manylinux_2_28_x86_64 ;; + aarch64) platform_tag=manylinux_2_35_aarch64 ;; + *) echo "Unsupported delegate architecture: ${ARCH}" >&2; exit 1 ;; + esac + ${CONDA_RUN} python -m wheel tags --remove --platform-tag "${platform_tag}" "${abs_pkg}" + else + ./test-infra/.github/scripts/repair_manylinux_2_28.sh "${abs_pkg}" + fi done - name: Run Post-Script if: ${{ inputs.post-script != '' }} @@ -451,6 +496,33 @@ jobs: with: repository: ${{ inputs.repository }} script: ${{ inputs.post-script }} + - name: Check the repaired ExecuTorch runtime wheel + if: ${{ steps.executorch-runtime.outcome == 'success' }} + shell: bash -l {0} + working-directory: ${{ inputs.repository }} + run: | + set -euo pipefail + source "${BUILD_ENV_FILE}" + shopt -s nullglob + wheels=(dist/torch_tensorrt_executorch_runtime-*.whl) + if [[ ${#wheels[@]} -ne 1 ]]; then + echo "Expected one repaired ExecuTorch companion, found ${#wheels[@]}" >&2 + exit 1 + fi + ${CONDA_RUN} python .github/scripts/check-executorch-runtime-wheel.py "${wheels[0]}" --architecture "${ARCH}" + ${CONDA_RUN} python -m pip install --no-deps "${wheels[0]}" + delegate="$(TORCH_TENSORRT_SKIP_DELEGATE_REGISTRATION=1 ${CONDA_RUN} python -c 'import torch_tensorrt_executorch_runtime as m; print(m._delegate_path())')" + # Build-machine library paths must not hide a broken installed RUNPATH. + if ! resolution="$(env -u LD_LIBRARY_PATH ldd -r "${delegate}" 2>&1)"; then + printf '%s\n' "${resolution}" >&2 + exit 1 + fi + printf '%s\n' "${resolution}" + if grep -E "not found|undefined symbol" <<< "${resolution}"; then + echo "The installed delegate has unresolved dependencies" >&2 + exit 1 + fi + env -u LD_LIBRARY_PATH ${CONDA_RUN} python -c "import torch_tensorrt_executorch_runtime" - name: Inventory packaged wheels # Keep this immediately before the smoke test/upload boundary. When a # matrix row fails downstream, its log shows exactly which repaired diff --git a/.github/workflows/executorch-pin-update.yml b/.github/workflows/executorch-pin-update.yml index 555cd1a10a..4c12533532 100644 --- a/.github/workflows/executorch-pin-update.yml +++ b/.github/workflows/executorch-pin-update.yml @@ -40,13 +40,16 @@ jobs: environment: pytorchbot-env steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: - token: ${{ secrets.GH_PYTORCHBOT_TOKEN }} + # No token and no stored credentials. Only the step that opens the pull request needs to + # act as the bot, and it takes its own token, so leaving the bot's credentials in the git + # config would hand push rights to every later step and to the actions they run. + persist-credentials: false fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.11" @@ -54,7 +57,14 @@ jobs: run: python -m pip install packaging pyyaml - name: Install uv - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 + with: + # Pinned rather than latest, and the pin is load-bearing. The project asks for python + # >=3.10 with no ceiling, so a resolver that splits by interpreter version tries to satisfy + # the newest one, and the dependency chain does not reach it: 0.12.15 fails the lock + # refresh on a tree that 0.12.5 resolves. Raise this only together with a ceiling on the + # declared range, or the refresh breaks with no change of ours behind it. + version: "0.12.5" - name: Choose the track id: track @@ -78,10 +88,36 @@ jobs: # Once a release ships its pin is history. A candidate tag is not a release, so # match a final tag only, or a dispatch during a release candidate would refuse. version="${REF#refs/heads/release/}" - if [ "${RELEASE_TAGGED:-}" = "true" ] || - { [ -z "${RELEASE_TAGGED:-}" ] && - git ls-remote --tags origin "v${version}.*" 2>/dev/null | - grep -qE "refs/tags/v${version}\.[0-9]+$"; }; then + # Any value other than an explicit false counts as tagged, so a typo closes the + # guard rather than skipping it. Only an empty value asks for the real lookup. + tagged="" + if [ -n "${RELEASE_TAGGED:-}" ]; then + # Any value other than an explicit false counts as tagged, so a typo closes the + # guard rather than skipping it. + [ "${RELEASE_TAGGED}" != "false" ] && tagged="yes" + else + # Separate the lookup from its result. Piping straight into grep made a failed + # lookup indistinguishable from a release with no tag, so a network or auth + # problem let a shipped release be re-pinned. + if ! remote_tags=$(git ls-remote --tags origin "v${version}.*" 2>&1); then + echo "could not list tags for release $version: ${remote_tags}" >&2 + exit 1 + fi + # The version goes into a pattern, so a version that is not a plain number can make the + # search fail rather than answer. Treat anything but a clean yes or no as a refusal, + # because the alternative is re-pinning a release that is already tagged. + case "${version}" in + *[!0-9.]*) echo "release version ${version} is not numeric" >&2; exit 1 ;; + esac + printf '%s\n' "${remote_tags}" | + grep -qE "refs/tags/v${version}\.[0-9]+$" + case "$?" in + 0) tagged="yes" ;; + 1) ;; + *) echo "could not decide whether release ${version} is tagged" >&2; exit 1 ;; + esac + fi + if [ -n "${tagged}" ]; then echo "release $version already has a tag; its pin stays as shipped" >&2 exit 1 fi diff --git a/.github/workflows/executorch-test-linux.yml b/.github/workflows/executorch-test-linux.yml index bca535a20f..e30cc11c82 100644 --- a/.github/workflows/executorch-test-linux.yml +++ b/.github/workflows/executorch-test-linux.yml @@ -40,7 +40,7 @@ jobs: set -eou pipefail MATRIX_BLOB=${{ toJSON(inputs.build-matrix) }} LIMIT_PR=${{ (github.event_name == 'push' || github.event_name == 'schedule') && 'false' || 'true' }} - MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --use-rtx false --limit-pr-builds "${LIMIT_PR}" --matrix "${MATRIX_BLOB}")" + MATRIX_BLOB="$(python3 .github/scripts/filter-matrix.py --executorch-runtime --use-rtx false --limit-pr-builds "${LIMIT_PR}" --matrix "${MATRIX_BLOB}")" echo "${MATRIX_BLOB}" echo "matrix=${MATRIX_BLOB}" >> "${GITHUB_OUTPUT}" @@ -66,13 +66,11 @@ jobs: # ExecuTorch's CUDA wheels live only on the PyTorch nightly index, so the channel is # needed. No --pre: a specifier naming a prerelease admits prereleases by itself, and # --pre would apply to every other requirement in the same command too. - python -m pip install pyyaml \ + python -m pip install pyyaml "wheel>=0.40" \ --extra-index-url "https://download.pytorch.org/whl/nightly/${CU_VERSION}" \ "executorch==1.6.0.dev20260915" - # The standard wheel build packages the matching runtime wheel in its - # artifact alongside torch_tensorrt*.whl; install it after the shared - # core-wheel installer has selected the matrix-matched core wheel. + # The shared artifact carries the matching core and companion wheels. shopt -s nullglob runtime_wheels=(/opt/torch-tensorrt-builds/torch_tensorrt_executorch_runtime*.whl) echo "Downloaded wheel artifact contents:" @@ -83,12 +81,16 @@ jobs: fi echo "Installing ExecuTorch runtime wheel: ${runtime_wheels[0]}" python -m pip install --no-deps "${runtime_wheels[0]}" - # Run the check directly so its exit status is the step's exit status. - # Wrapping it in `gdb --batch` reports gdb's own status, which is 0 - # whatever the program does, so a SIGSEGV here was passing. - # On failure, re-run under gdb for the backtrace, then fail the step. + # Check stock backends too, since delegated programs can fall back to them. + # Run directly so a native crash fails the step; gdb is only for a backtrace. ulimit -c unlimited || true - runtime_check='import torch; print(torch.__version__, torch.version.cuda); from torch_tensorrt_executorch_runtime import BACKEND_NAME, get_runtime; print(1); runtime = get_runtime(); print(2); assert runtime.backend_registry.is_available(BACKEND_NAME); assert runtime.backend_registry.is_available("XnnpackBackend"); assert runtime.backend_registry.is_available("CudaBackend")' + # sys.exit, not assert: under PYTHONOPTIMIZE or python -O every assert is compiled out and + # this check would pass over a runtime with no backends registered at all. + # + # Importing the delegate package is the whole registration step; it exposes no runtime API, + # so the registry is queried through ExecuTorch's own Runtime. Importing it for its side + # effect only is exactly what a user does, so this also proves the side effect works. + runtime_check='import sys, torch; print(torch.__version__, torch.version.cuda); import torch_tensorrt_executorch_runtime; from torch_tensorrt_executorch_runtime import BACKEND_NAME; from executorch.runtime import Runtime; registry = Runtime.get().backend_registry; missing = [n for n in (BACKEND_NAME, "XnnpackBackend", "CudaBackend") if not registry.is_available(n)]; sys.exit("FATAL: backends not registered: " + ", ".join(missing)) if missing else None' check_status=0 PYTHONFAULTHANDLER=1 python -u -X faulthandler -c "${runtime_check}" || check_status=$? @@ -139,9 +141,17 @@ jobs: --model_path="${RUNNER_TEMP}/torchtrt-kv-cache-decode.pte" python examples/torchtrt_executorch_example/export_coalesced.py \ --model_path="${RUNNER_TEMP}/torchtrt-coalesced.pte" + python examples/torchtrt_executorch_example/export_device_resident.py \ + --model_path="${RUNNER_TEMP}/torchtrt-device-resident.pte" .github/scripts/verify-executorch-reference-runner.sh \ "${RUNNER_TEMP}/torchtrt-python.pte" \ "${RUNNER_TEMP}/torchtrt-kv-cache-decode.pte" \ "${RUNNER_TEMP}/torchtrt-coalesced.pte" python examples/executorch_reference_runner/load_model.py \ --model_path="${RUNNER_TEMP}/torchtrt-python.pte" --num_runs=1 + # The device-resident program is deliberately NOT handed to the C++ + # reference runner above: that runner feeds host tensors, which this + # program's boundary contract forbids. Its Python runner passes CUDA + # tensors and asserts the output never left the device. + python examples/executorch_reference_runner/load_model_device_resident.py \ + --model_path="${RUNNER_TEMP}/torchtrt-device-resident.pte" --num_runs=2 diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 282e2d285d..a6b62f602b 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -101,3 +101,21 @@ jobs: cd $GITHUB_WORKSPACE python3 -m pytest tests/py/dynamo/executorch/test_update_executorch_pin.py \ -q --no-header -p no:cacheprovider --noconftest -o addopts="" + # The rest of the ExecuTorch suite that needs no GPU. Without this the only lane running these + # is nightly, so a pull request could not fail on anything they cover, and they cover the + # packaging, the wheel guard, the shipped CMake package and the shared workflow. + # Two files are deliberately absent: test_api and test_load_compatibility import torch at + # module level, and this job installs none, so adding them would mean installing a build + # here. They stay in the lane that has one. + - name: Check the ExecuTorch packaging and workflow + if: always() + run: | + cd $GITHUB_WORKSPACE + python3 -m pytest \ + tests/py/dynamo/executorch/test_packaging.py \ + tests/py/dynamo/executorch/test_artifact_guard.py \ + tests/py/dynamo/executorch/test_shared_runtime_workflow.py \ + tests/py/dynamo/executorch/test_example_boundaries.py \ + tests/py/dynamo/executorch/test_cmake_runtime.py \ + tests/py/dynamo/executorch/test_python_runtime.py \ + -q --no-header -p no:cacheprovider --noconftest -o addopts="" diff --git a/MODULE.bazel b/MODULE.bazel index e1e9b25b6d..5bf20ee2c9 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -45,8 +45,8 @@ new_git_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:git.bzl" local_torch = use_repo_rule("//toolchains:local_torch.bzl", "local_torch") -# Build the portable runtime and delegate from the installed wheel's source revision. -# The provenance test checks that the source commit and wheel version agree. +# Keep the C++ source headers paired with the installed ExecuTorch runtime. +# The Python companion uses the installed wheel's CMake package directly. new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", diff --git a/cpp/include/torch_tensorrt/executorch/TensorRTBackend.h b/cpp/include/torch_tensorrt/executorch/TensorRTBackend.h index a74b1c91fd..528052293b 100644 --- a/cpp/include/torch_tensorrt/executorch/TensorRTBackend.h +++ b/cpp/include/torch_tensorrt/executorch/TensorRTBackend.h @@ -51,8 +51,6 @@ struct InputProfileBounds { }; struct EngineHandle { - TRTLogger logger; - TRTUniquePtr runtime; TRTUniquePtr engine; TRTUniquePtr exec_ctx; std::vector input_binding_names; diff --git a/cpp/src/torch_tensorrt/executorch/CMakeLists.txt b/cpp/src/torch_tensorrt/executorch/CMakeLists.txt index 1b503c567d..b74865cdfb 100644 --- a/cpp/src/torch_tensorrt/executorch/CMakeLists.txt +++ b/cpp/src/torch_tensorrt/executorch/CMakeLists.txt @@ -67,10 +67,26 @@ endforeach() # Select ExecuTorch's shared extension_cuda so every CUDA-capable delegate shares # one caller-stream thread-local. A static copy linked into a second shared object # would create a second thread-local and silently break the handshake, so every -# branch below must resolve to one shared library. Precedence: an -# ExecuTorch-provided target, then an explicit prebuilt shared library, then a -# source build. -if(TARGET extension_cuda) +# branch below must resolve to one shared library. Precedence: the installed wheel's +# package, then an ExecuTorch-provided target, then an explicit prebuilt shared +# library, then a source build. +if(TARGET executorch::extension_cuda) + # Already provided by a parent project that called find_package(executorch), i.e. the + # installed wheel. This file does not call it itself, because it is also configured + # standalone against a source tree. Aliased rather than queried for its type: the package + # only ever defines this as a shared imported target, so the static-copy hazard the other + # branches guard against cannot arise here. The wheel's config withholds these targets + # entirely below CMake 3.28, and none of the branches below can stand in for them: each + # needs an add_subdirectory of ExecuTorch, an explicit prebuilt library, or a source + # checkout, none of which a wheel-only consumer has. That case is diagnosed at the bottom of + # this chain rather than left to fail as a missing target. + # Only when the plain name is free. A consumer that already defines extension_cuda, which is + # what an add_subdirectory of ExecuTorch produces, would otherwise fail on a duplicate name + # while trying to use both packages together. + if(NOT TARGET extension_cuda) + add_library(extension_cuda ALIAS executorch::extension_cuda) + endif() +elseif(TARGET extension_cuda) # Provided by ExecuTorch, e.g. add_subdirectory() with EXECUTORCH_BUILD_CUDA=ON. get_target_property(_extension_cuda_type extension_cuda TYPE) if(NOT _extension_cuda_type STREQUAL "SHARED_LIBRARY") @@ -126,6 +142,15 @@ elseif(_torchtrt_executorch_source_root AND "${CMAKE_CURRENT_BINARY_DIR}/executorch_extension_cuda" ) else() + # Name the version gate first when that is what happened, so a consumer who did everything + # right is not told to install what they already installed. + if(executorch_FOUND AND CMAKE_VERSION VERSION_LESS 3.28) + message(FATAL_ERROR + "The installed ExecuTorch wheel withholds its CMake targets below CMake 3.28, and " + "this build is running CMake ${CMAKE_VERSION}. Upgrade to 3.28 or newer, or point " + "EXECUTORCH_EXTENSION_CUDA_LIBRARY at the wheel's " + "libexecutorch_extension_cuda.so directly.") + endif() message(FATAL_ERROR "Torch-TensorRT's ExecuTorch backend requires ExecuTorch's shared " "extension_cuda library. Add ExecuTorch first with EXECUTORCH_BUILD_CUDA=ON, " @@ -143,7 +168,11 @@ set(_torchtrt_executorch_link_libraries extension_cuda ) -if(TARGET executorch_core) +if(TARGET executorch::runtime) + # The installed wheel's prebuilt runtime, which also carries the include directories, + # compile definitions and C++ standard it was built with. + list(APPEND _torchtrt_executorch_link_libraries executorch::runtime) +elseif(TARGET executorch_core) list(APPEND _torchtrt_executorch_link_libraries executorch_core) elseif(TARGET executorch) list(APPEND _torchtrt_executorch_link_libraries executorch) diff --git a/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp b/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp index cce5e1d713..2872a83d97 100644 --- a/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp +++ b/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp @@ -109,7 +109,7 @@ EngineHandle::~EngineHandle() { } exec_ctx.reset(); engine.reset(); - runtime.reset(); + // The runtime is shared and outlives this handle, so there is nothing to release for it. if (inflight_event != nullptr) { cudaEventDestroy(inflight_event); inflight_event = nullptr; @@ -118,6 +118,14 @@ EngineHandle::~EngineHandle() { namespace { +// The process-wide TensorRT runtime and its logger. Function-local statics, so it is constructed +// once, thread safely, and outlives every engine deserialized from it as TensorRT requires. +nvinfer1::IRuntime* shared_runtime() { + static TRTLogger logger; + static TRTUniquePtr runtime(nvinfer1::createInferRuntime(logger)); + return runtime.get(); +} + struct EngineHandleDeleter { void operator()(EngineHandle* handle) const { if (handle != nullptr) { @@ -280,22 +288,37 @@ Result TensorRTBackend::init( return Error::InvalidProgram; } - int is_integrated = 0; - cuda_err = cudaDeviceGetAttribute(&is_integrated, cudaDevAttrIntegrated, handle->device_id); + // Whether this device can read pageable host memory, which is the question the three uses of this + // flag actually ask before handing a caller's pointer to TensorRT without a copy. Being an + // integrated part is a different question, and the answers differ on real devices: an H100 reports + // integrated 0 with pageable access 1, so asking the wrong one gave up a copy-free path there, and + // an integrated part is not obliged to report pageable access, where the wrong one would have bound + // ordinary host memory in as though the device could reach it. + int pageable_access = 0; + cuda_err = cudaDeviceGetAttribute(&pageable_access, cudaDevAttrPageableMemoryAccess, handle->device_id); if (cuda_err != cudaSuccess) { ET_LOG( Info, - "TensorRTBackend::init: cudaDeviceGetAttribute(cudaDevAttrIntegrated) failed: %s", + "TensorRTBackend::init: cudaDeviceGetAttribute(cudaDevAttrPageableMemoryAccess) failed: %s", cudaGetErrorString(cuda_err)); } - handle->unified_memory = is_integrated != 0; - - handle->runtime.reset(nvinfer1::createInferRuntime(handle->logger)); + handle->unified_memory = pageable_access != 0; + + // One runtime for the process, not one per program. TensorRT keeps state behind these objects + // that a second runtime collides with, and it says so out loud: creating another logs that the + // logger differs from one already registered and that the new one is ignored. Loading several + // programs at once on top of that crashed. Deserialization is serialized for the same reason, + // because a runtime is not safe to use from two threads at once. + static std::mutex deserialize_lock; + nvinfer1::IRuntime* runtime = shared_runtime(); TORCHTRT_ET_CHECK_NOT_NULL( - handle->runtime, Error::InvalidProgram, "TensorRTBackend::init: failed to create TensorRT runtime"); + runtime, Error::InvalidProgram, "TensorRTBackend::init: failed to create TensorRT runtime"); const void* engine_data = TensorRTBlobHeader::engine_data(processed->data(), header); - handle->engine.reset(handle->runtime->deserializeCudaEngine(engine_data, header.engine_size)); + { + const std::lock_guard guard(deserialize_lock); + handle->engine.reset(runtime->deserializeCudaEngine(engine_data, header.engine_size)); + } TORCHTRT_ET_CHECK_NOT_NULL( handle->engine, Error::InvalidProgram, "TensorRTBackend::init: failed to deserialize TensorRT engine"); @@ -964,15 +987,26 @@ Error TensorRTBackend::execute(BackendExecutionContext& context, DelegateHandle* cuda_err = cudaMemcpyAsync(et_out.mutable_data_ptr(), output.second, et_out.nbytes(), cudaMemcpyDeviceToHost, stream); if (cuda_err != cudaSuccess) { + // Name the output and number it the way the caller does. output.first indexes the whole + // argument list, so on a one-input engine the first output read as "output 1", and the + // index a caller passes to set_output_data_ptr counts outputs from zero. + const size_t output_index = output.first - engine->num_inputs; + const char* output_name = output_index < engine->output_binding_names.size() + ? engine->output_binding_names[output_index].c_str() + : "unknown"; ET_LOG( Error, - "TensorRTBackend::execute: D2H copy failed for output %zu: %s", - output.first, + "TensorRTBackend::execute: D2H copy failed for output %zu ('%s'): %s. A program built " + "without runtime-allocated outputs needs the caller to supply that buffer, through " + "set_output_data_ptr with this index.", + output_index, + output_name, cudaGetErrorString(cuda_err)); // The enqueue already succeeded, so the engine is still running on the // stream. Drain below before returning, or the next call mutates a live // execution context, which TensorRT forbids. - copy_err = Error::InvalidProgram; + // Not InvalidProgram: the program is fine and runs correctly once the buffer is supplied. + copy_err = Error::InvalidArgument; break; } } @@ -1032,5 +1066,12 @@ const ::executorch::runtime::Backend kBackendId{"TensorRTBackend", &get_backend( const Error kRegistrationResult = ::executorch::runtime::register_backend(kBackendId); } // namespace + +#ifdef TORCH_TENSORRT_EXECUTORCH_RUNTIME +extern "C" bool torch_tensorrt_owns_executorch_registration() { + return ::executorch::runtime::get_backend_class(kBackendId.name) == &get_backend(); +} +#endif + } // namespace executorch_backend } // namespace torch_tensorrt diff --git a/examples/executorch_reference_runner/BUILD b/examples/executorch_reference_runner/BUILD index 5d85db9f5b..73be0c903b 100644 --- a/examples/executorch_reference_runner/BUILD +++ b/examples/executorch_reference_runner/BUILD @@ -20,7 +20,15 @@ cc_binary( # The driver itself is deliberately not a link dependency: the release build image # ships neither libcuda nor a stub. linkopts = [ + # The wheel installs this at torch_tensorrt/bin, so ../lib reaches Torch-TensorRT's own + # libraries and ../../ reaches the other distributions in site-packages. Without the latter + # two the binary cannot start at all: TensorRT and the CUDA runtime ship in their own + # distributions, and nothing else adds them to the search path for a program that is not + # launched through Python. The delegate library in the companion wheel already carries the + # same two entries, which is why it loads and this did not. "-Wl,-rpath,$$ORIGIN/../lib", + "-Wl,-rpath,$$ORIGIN/../../tensorrt_libs", + "-Wl,-rpath,$$ORIGIN/../../nvidia/cu13/lib", "-ldl", ], deps = [ @@ -36,6 +44,12 @@ cc_binary( cc_binary( name = "kv_cache_decode_check", srcs = ["kv_cache_decode_check.cpp"], + linkopts = [ + # Same reasoning as the runner above. This one carried no run path at all. + "-Wl,-rpath,$$ORIGIN/../lib", + "-Wl,-rpath,$$ORIGIN/../../tensorrt_libs", + "-Wl,-rpath,$$ORIGIN/../../nvidia/cu13/lib", + ], deps = [ "//cpp:tensorrt_executorch_backend", "//cpp:tensorrt_executorch_cuda_device_allocator", diff --git a/examples/executorch_reference_runner/README.md b/examples/executorch_reference_runner/README.md index b33bce5e49..caa4ae7108 100644 --- a/examples/executorch_reference_runner/README.md +++ b/examples/executorch_reference_runner/README.md @@ -112,14 +112,15 @@ already declare the extra. If an older installation lacks it, first install the intended compatible Torch-TensorRT wheel deliberately. Adding the extra may change dependencies, so use a fresh environment to preserve an existing working stack. -The extra installs `executorch` only. The delegate runtime, -`torch-tensorrt-executorch-runtime`, is not yet published to any index: its requirement in the -top-level `setup.py` is commented out for that reason. Build and install it from source following -`py/torch-tensorrt-executorch-runtime/README.md`. That wheel contains an ExecuTorch Python runtime -with `TensorRTBackend` linked into its backend registry, and loading a `.pte` through the delegate +The extra installs `executorch` only. Install the matching companion artifact, +`torch-tensorrt-executorch-runtime`, or build and install it from source following +`py/torch-tensorrt-executorch-runtime/README.md`. That wheel ships just the TensorRT delegate, a +single shared library that registers itself with the ExecuTorch runtime from the `executorch` +distribution rather than bundling a runtime of its own, and loading a `.pte` through the delegate needs it. -Then load and run the model: +The Python example uses ExecuTorch's Module API to back planned device arenas +with CUDA memory. Then load and run the model: ```bash python examples/executorch_reference_runner/load_model.py \ @@ -127,6 +128,14 @@ python examples/executorch_reference_runner/load_model.py \ --num_runs=1 ``` +The legacy `torch_tensorrt.load(path, format="executorch")` entry point still +works, but emits a deprecation warning. Its `method_names` property, +`run(inputs, method="forward")`, and `forward(*inputs)` interface remain +supported for at least six months after the deprecation first ships. It still +copies CUDA inputs to CPU and supports embedded weights only. New applications +should use the Module API shown above; device-resident programs must use it +directly to keep their inputs on CUDA. + ### C++ Run the reference runner against a Torch-TensorRT compiled ExecuTorch model: @@ -189,9 +198,9 @@ Enabling `EXECUTORCH_BUILD_CUDA` does not make this runner depend on libtorch. I needs `EXECUTORCH_BUILD_EXTENSION_TENSOR=ON`, which is set automatically, and the result links no libtorch and no libc10. -This path is verified by hand, not in CI: the CI configuration builds the runner -without the CUDA delegate. It also takes the synchronized path, because the method -inputs and outputs are host-backed. +The green-context option is not exercised by CI. The reference-runner checks +use the CUDA-enabled build with an ordinary stream and host-backed method inputs +and outputs. ## Caller-Owned KV-Cache Persistence Check diff --git a/examples/executorch_reference_runner/load_model.py b/examples/executorch_reference_runner/load_model.py index 136b51f63a..4105a272f1 100644 --- a/examples/executorch_reference_runner/load_model.py +++ b/examples/executorch_reference_runner/load_model.py @@ -5,7 +5,11 @@ from pathlib import Path import torch -import torch_tensorrt + +# Registers TensorRTBackend with ExecuTorch's backend registry as an import side effect. Nothing +# from this package is referenced below: loading and running a program is ExecuTorch's own API. +import torch_tensorrt_executorch_runtime # noqa: F401 +from executorch.extension.pybindings.portable_lib import _load_for_executorch parser = argparse.ArgumentParser() parser.add_argument( @@ -22,15 +26,18 @@ model_path = args.model_path x = torch.ones((2, 3, 4, 4), dtype=torch.float32) -program = torch_tensorrt.load(model_path, format="executorch") +# The Module API backs device-tagged arenas with device memory. +program = _load_for_executorch(str(model_path)) +if "forward" not in program.method_names(): + raise RuntimeError(f"{model_path} has no 'forward' method") for _ in range(args.num_runs): - outputs = program.forward(x) + outputs = program.run_method("forward", (x,)) y = outputs[0] expected = x + 1 torch.testing.assert_close(y.cpu(), expected) -print("methods:", sorted(program.method_names)) +print("methods:", sorted(program.method_names())) print("output shape:", tuple(y.shape)) print("output device:", y.device) print("PASS: ExecuTorch TensorRT delegate output matches x + 1") diff --git a/examples/executorch_reference_runner/load_model_device_resident.py b/examples/executorch_reference_runner/load_model_device_resident.py new file mode 100644 index 0000000000..41bcd8dc17 --- /dev/null +++ b/examples/executorch_reference_runner/load_model_device_resident.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +"""Run a device-resident .pte and prove the method boundary did not copy. + +The program this loads was exported with ``skip_h2d_for_method_inputs`` and +``skip_d2h_for_method_outputs``, so it requires a CUDA input and returns a CUDA +output. Feeding it a CPU tensor is a caller error, not something the runtime +papers over: the delegate would read host memory as if it were device memory. + +That contract is what this script checks. ``export_device_resident.py`` already +asserts the serialized program contains no boundary copy operators; this asserts +the runtime half, that a CUDA tensor goes in, a CUDA tensor comes out, and the +values are right. +""" + +import argparse +from pathlib import Path + +import torch + +# Registers TensorRTBackend with ExecuTorch's backend registry as an import side effect. Nothing +# from this package is referenced below: loading and running a program is ExecuTorch's own API. +import torch_tensorrt_executorch_runtime # noqa: F401 +from executorch.extension.pybindings.portable_lib import _load_for_executorch + +parser = argparse.ArgumentParser() +parser.add_argument( + "--model_path", + type=Path, + required=True, + help="Path to the device-resident ExecuTorch .pte model", +) +parser.add_argument("--num_runs", type=int, default=1) +args = parser.parse_args() +if args.num_runs < 1: + raise ValueError("--num_runs must be at least 1") + +if not torch.cuda.is_available(): + raise RuntimeError( + "This program keeps its inputs and outputs on the GPU, so it cannot run " + "without CUDA." + ) + +model_path = args.model_path +# The shape export_device_resident.py used, and the value the .expected file +# describes: the reference is cos(erfinv(tanh(1.0))) elementwise. +x = torch.ones((64, 64), dtype=torch.float32, device="cuda") +# No check on the tensor's device here. Asking for the device is what can fail, and it has already +# failed by the time a check would run, so testing the result was a guard that could not fire. +# Whether CUDA is present at all is checked above, where the question can actually be answered. + +# The Module API backs device-tagged arenas with device memory. +program = _load_for_executorch(str(model_path)) +if "forward" not in program.method_names(): + raise RuntimeError(f"{model_path} has no 'forward' method") + +for _ in range(args.num_runs): + outputs = program.run_method("forward", (x,)) +y = outputs[0] + +# The point of the whole exercise. Nothing in the Python layer copies a tensor +# now, so if the export flags did not take, this is where it shows up. +if not y.is_cuda: + raise AssertionError( + f"FATAL: output came back on {y.device}, so the method boundary still " + "copies device to host. The skip_d2h_for_method_outputs flag did not take." + ) + +expected = torch.cos(torch.erfinv(torch.tanh(x))) +torch.testing.assert_close(y, expected) + +print("methods:", sorted(program.method_names())) +print("input device:", x.device) +print("output device:", y.device) +print( + "PASS: device-resident ExecuTorch TensorRT program kept inputs and outputs on the GPU" +) diff --git a/examples/executorch_reference_runner/main.cpp b/examples/executorch_reference_runner/main.cpp index 0b1b163b79..0ecb1c5a5e 100644 --- a/examples/executorch_reference_runner/main.cpp +++ b/examples/executorch_reference_runner/main.cpp @@ -60,8 +60,8 @@ using executorch::runtime::MethodMeta; using executorch::runtime::Program; using executorch::runtime::Result; using executorch::runtime::Span; -using executorch::runtime::etensor::Device; using executorch::runtime::TensorInfo; +using executorch::runtime::etensor::Device; static uint8_t method_allocator_pool[4 * 1024U * 1024U]; static uint8_t temp_allocator_pool[1 * 1024U * 1024U]; @@ -76,7 +76,6 @@ static const char* get_flag(int argc, char** argv, const char* flag, const char* return def; } - // The CUDA driver API is resolved at runtime rather than linked. The release build // image ships neither libcuda nor a stub, so linking it would break the build for // everyone to serve one optional flag, and it would have to be wired into both this @@ -89,7 +88,12 @@ struct CudaDriverApi { CUresult (*DeviceGet)(CUdevice*, int) = nullptr; CUresult (*DeviceGetDevResource)(CUdevice, CUdevResource*, CUdevResourceType) = nullptr; CUresult (*DevSmResourceSplitByCount)( - CUdevResource*, unsigned int*, const CUdevResource*, CUdevResource*, unsigned int, unsigned int) = nullptr; + CUdevResource*, + unsigned int*, + const CUdevResource*, + CUdevResource*, + unsigned int, + unsigned int) = nullptr; CUresult (*DevResourceGenerateDesc)(CUdevResourceDesc*, CUdevResource*, unsigned int) = nullptr; CUresult (*GreenCtxCreate)(CUgreenCtx*, CUdevResourceDesc, CUdevice, unsigned int) = nullptr; CUresult (*GreenCtxStreamCreate)(CUstream*, CUgreenCtx, unsigned int, int) = nullptr; @@ -124,10 +128,8 @@ const CudaDriverApi* load_cuda_driver_api() { const bool ok = bind(api.Init, "cuInit") && bind(api.DeviceGet, "cuDeviceGet") && bind(api.DeviceGetDevResource, "cuDeviceGetDevResource") && bind(api.DevSmResourceSplitByCount, "cuDevSmResourceSplitByCount") && - bind(api.DevResourceGenerateDesc, "cuDevResourceGenerateDesc") && - bind(api.GreenCtxCreate, "cuGreenCtxCreate") && - bind(api.GreenCtxStreamCreate, "cuGreenCtxStreamCreate") && - bind(api.GreenCtxDestroy, "cuGreenCtxDestroy") && + bind(api.DevResourceGenerateDesc, "cuDevResourceGenerateDesc") && bind(api.GreenCtxCreate, "cuGreenCtxCreate") && + bind(api.GreenCtxStreamCreate, "cuGreenCtxStreamCreate") && bind(api.GreenCtxDestroy, "cuGreenCtxDestroy") && bind(api.GetErrorString, "cuGetErrorString"); loaded = ok; return ok ? &api : nullptr; @@ -213,6 +215,20 @@ static bool make_green_context_stream( int main(int argc, char** argv) { executorch::runtime::runtime_init(); + // Reject anything unrecognised rather than ignoring it. Every option here is a --name=value flag, + // so a bare path silently fell through to the default and the program ran a different file than + // the one it was asked for, reporting success. + for (int i = 1; i < argc; ++i) { + if (strncmp(argv[i], "--model_path=", 13) != 0 && strncmp(argv[i], "--num_runs=", 11) != 0 && + strncmp(argv[i], "--green_context_sms=", 20) != 0) { + ET_LOG( + Error, + "unrecognised argument '%s'. Usage: example_executorch_runner " + "--model_path=model.pte [--num_runs=1] [--green_context_sms=0]", + argv[i]); + return 1; + } + } const char* model_path = get_flag(argc, argv, "--model_path", "model.pte"); const int num_runs = atoi(get_flag(argc, argv, "--num_runs", "1")); const int green_context_sms = atoi(get_flag(argc, argv, "--green_context_sms", "0")); @@ -286,11 +302,7 @@ int main(int argc, char** argv) { static_cast(buffer_device->type()), static_cast(device_buffer.error())); ET_LOG( - Info, - " planned buffer[%zu] = %zu bytes on device_type %d", - i, - sz, - static_cast(buffer_device->type())); + Info, " planned buffer[%zu] = %zu bytes on device_type %d", i, sz, static_cast(buffer_device->type())); planned_spans.push_back(device_buffer->as_span()); planned_device_buffers.push_back(std::move(device_buffer.get())); } diff --git a/examples/torchtrt_executorch_example/export_device_resident.py b/examples/torchtrt_executorch_example/export_device_resident.py new file mode 100644 index 0000000000..bba29f3340 --- /dev/null +++ b/examples/torchtrt_executorch_example/export_device_resident.py @@ -0,0 +1,206 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +""" +.. _executorch_export_device_resident: + +Exporting a Coalesced Model That Keeps Inputs and Outputs on the GPU +==================================================================== + +Same graph and same two backends as :ref:`executorch_export_coalesced`, but the +method boundary does no device copies. The caller hands in a CUDA tensor and gets +a CUDA tensor back. + +Note the program carries more delegate *entries* than backends: TensorRT claims +the operators on either side of ``erfinv``, so the graph splits into two TensorRT +engines around one CUDA region and the delegate list reads +``['TensorRTBackend', 'CudaBackend', 'TensorRTBackend']``. The check below is +therefore a membership test, not a count. + +By default ExecuTorch inserts ``et_copy._h2d_copy`` before a delegate that +consumes a method input and ``et_copy._d2h_copy`` after one that produces a +method output, so a method is safe to call with CPU tensors. For a pipeline that +already has its data on the GPU those copies are pure overhead, and they are what +``PropagateDeviceConfig`` turns off. + +Two settings are needed, not one. Skipping the copy is not enough on its own: +memory planning allocates graph inputs and outputs by default, so the runtime +would still reserve its own buffer and fill it from the caller's memory with a +host memcpy, which is undefined for device memory and puts the copy straight +back. ``alloc_graph_input=False`` and ``alloc_graph_output=False`` are what stop +that, and ``enable_non_cpu_memory_planning`` is required for planning to run over +non-CPU tensors at all. + +This script asserts on the serialized program rather than trusting the flags: +the exported ``.pte`` must contain neither copy operator, and it must still carry +both delegates. Checking ``tensor.is_cuda`` at runtime is not enough on its own, +because a round trip that ends back on the device would still look correct. + +Prerequisites +------------- +Install Torch-TensorRT with the ExecuTorch extra before running this example:: + + pip install -e ".[executorch]" \ + --extra-index-url https://download.pytorch.org/whl/nightly/cu130 + +ExecuTorch's CUDA backend also needs a CUDA toolkit (``nvcc``) at export time, +for the AOTInductor compile. +""" + +import argparse +import sys +from pathlib import Path + +import torch +import torch_tensorrt +from executorch.backends.cuda.cuda_backend import CudaBackend +from executorch.backends.cuda.cuda_partitioner import CudaPartitioner +from executorch.exir import ExecutorchBackendConfig +from executorch.exir._serialize._program import deserialize_pte_binary +from executorch.exir.passes.memory_planning_pass import MemoryPlanningPass +from executorch.exir.passes.propagate_device_pass import PropagateDeviceConfig +from executorch.exir.schema import DeviceType, Tensor + +SHAPE = (64, 64) + +# The operator names ExecuTorch's PropagateDevicePass inserts at the method +# boundary. Asserting on these by name is the point of this example: they are +# what the skip flags are supposed to remove. +BOUNDARY_COPY_OPS = ("_h2d_copy", "_d2h_copy") + + +class CoalescedModel(torch.nn.Module): + def forward(self, x): + return torch.cos(torch.erfinv(torch.tanh(x))) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_path", + default="coalesced_device_resident.pte", + help="Path to save the .pte", + ) + args = parser.parse_args() + model_path = Path(args.model_path) + expected_path = model_path.with_suffix(".expected") + if model_path == expected_path: + parser.error("--model_path must not end in .expected") + + with torch.no_grad(): + model = CoalescedModel().eval().cuda() + example_input = (torch.randn(SHAPE).cuda(),) + + exported_program = torch.export.export(model, example_input) + trt_gm = torch_tensorrt.dynamo.compile( + exported_program, + arg_inputs=example_input, + min_block_size=1, + truncate_double=True, + ) + # Write beside the target and move it into place only after the checks below pass. Saving + # straight over the target would let a rejected export replace a good program, and leave + # the reference file describing something the program no longer is. + staged_path = model_path.with_name(model_path.name + ".staged") + torch_tensorrt.save( + trt_gm, + str(staged_path), + output_format="executorch", + arg_inputs=example_input, + retrace=False, + partitioners=[ + CudaPartitioner( + [CudaBackend.generate_method_name_compile_spec("forward")] + ) + ], + backend_config=ExecutorchBackendConfig( + propagate_device_config=PropagateDeviceConfig( + skip_h2d_for_method_inputs=True, + skip_d2h_for_method_outputs=True, + ), + enable_non_cpu_memory_planning=True, + # Without these the runtime allocates its own buffer for the + # input and output it was just told not to copy, and fills the + # input from the caller's memory with a host memcpy. + memory_planning_pass=MemoryPlanningPass( + alloc_graph_input=False, alloc_graph_output=False + ), + ), + ) + + program = deserialize_pte_binary(staged_path.read_bytes()).program + + # Still coalesced. A partitioner or config change that routed the whole + # graph to one backend would otherwise leave this example passing while + # testing something else entirely. + delegates = [d.id for plan in program.execution_plan for d in plan.delegates] + missing = [ + name for name in ("TensorRTBackend", "CudaBackend") if name not in delegates + ] + if missing: + staged_path.unlink(missing_ok=True) + sys.exit( + f"{model_path} is not coalesced: missing {missing}, found {delegates}" + ) + + # The assertion this example exists for. Read the operator table of every + # execution plan and reject the program if either boundary copy survived. + found = sorted( + { + f"{operator.name}.{operator.overload}" + for plan in program.execution_plan + for operator in plan.operators + if any(copy_op in operator.name for copy_op in BOUNDARY_COPY_OPS) + } + ) + if found: + staged_path.unlink(missing_ok=True) + sys.exit( + f"FATAL: {model_path} still copies across the method boundary: {found}. " + "The skip flags did not take, so this program would not keep a CUDA " + "input on the device." + ) + + # And the boundary really is device-resident, read off the program rather + # than assumed. A tensor with no extra_tensor_info defaults to CPU in the + # schema, so a missing record is a failure here, not something to skip. + host_tensors = [] + for plan in program.execution_plan: + for kind, indices in (("input", plan.inputs), ("output", plan.outputs)): + for index in indices: + value = plan.values[index].val + if not isinstance(value, Tensor): + continue + info = value.extra_tensor_info + device = DeviceType.CPU if info is None else info.device_type + if device != DeviceType.CUDA: + host_tensors.append( + f"{kind}[{index}]={DeviceType(device).name}" + ) + if host_tensors: + staged_path.unlink(missing_ok=True) + sys.exit( + f"FATAL: {model_path} has non-CUDA method boundary tensors: " + f"{host_tensors}" + ) + + # Every check passed, so this program may take the target's place. + staged_path.replace(model_path) + + reference = model(torch.ones(SHAPE).cuda()) + expected_path.write_text( + "[{}]\n{:.4f}\n".format( + ",".join(str(dim) for dim in reference.shape), + reference.flatten()[0].item(), + ) + ) + + print(f"Saved {model_path} with delegates {delegates}.") + print( + f"No {' or '.join(BOUNDARY_COPY_OPS)} in the program: boundary is device-resident." + ) + print(f"Saved {expected_path} with the eager reference output.") + + +if __name__ == "__main__": + main() diff --git a/justfile b/justfile index 24fa45054a..cdf151c0d8 100644 --- a/justfile +++ b/justfile @@ -56,6 +56,11 @@ suites: # Run ONE suite exactly as CI runs it (uses the {{variant}} backend). Args after `--`: # just suite dynamo-runtime -- -k test_foo -x just variant=rtx suite dynamo-converters +# +# The executorch suite additionally needs CU_VERSION set, because it installs the exact pinned +# ExecuTorch nightly and that build exists only on the matching PyTorch nightly channel, never on +# PyPI. CI always sets it; a local shell does not, so without it setup fails before any test runs: +# CU_VERSION=cu134 just suite executorch suite name *args: {{_ci}} run {{name}} --variant {{variant}} {{args}} @@ -92,7 +97,7 @@ install-test-ext: # # Exact, not a range: the nightly channel gains a member every day, and the delegate is # compiled from the commit this version pairs with. - uv pip install pyyaml \ + uv pip install pyyaml patchelf \ --extra-index-url https://download.pytorch.org/whl/nightly/cu130 \ "executorch==1.6.0.dev20260915" diff --git a/py/torch-tensorrt-executorch-runtime/README.md b/py/torch-tensorrt-executorch-runtime/README.md index 987905df9f..5c6d1b7217 100644 --- a/py/torch-tensorrt-executorch-runtime/README.md +++ b/py/torch-tensorrt-executorch-runtime/README.md @@ -1,93 +1,242 @@ # Torch-TensorRT ExecuTorch Runtime Wheel This directory builds `torch-tensorrt-executorch-runtime`. The Linux wheel -contains an ExecuTorch `_portable_lib` Python runtime with `TensorRTBackend` -force-linked into the same native module that owns the backend registry. +contains one shared library, `lib/libexecutorch_backend_tensorrt.so`, holding the +TensorRT delegate and nothing else. The ExecuTorch runtime it registers with +comes from the `executorch` wheel. + +The layout follows the one ExecuTorch uses for its own backends, so the delegate +is an out-of-tree sibling of them rather than a special case: + +``` +executorch/ torch_tensorrt_executorch_runtime/ + lib/libexecutorch_backend_cuda.so lib/libexecutorch_backend_tensorrt.so + share/cmake/executorch-config.cmake lib/cmake/torchtrt_executorch/torchtrt_executorch-config.cmake +``` + +Python users just import the package. A C++ app links it the same way it links +one of ExecuTorch's own backends: + +```cmake +find_package(executorch REQUIRED COMPONENTS backend_cuda kernels_optimized) +find_package(torchtrt_executorch REQUIRED) +target_link_libraries(my_app PRIVATE + executorch::runtime executorch::backend_cuda + executorch::kernels_optimized torchtrt::executorch_backend) +``` + +The optimized-kernel library supplies the `et_copy` host/device copy operators. +For device-resident exports with `alloc_graph_output=False`, C++ Module callers +must provide a CUDA output tensor with `Module::set_output` before execution. + +Point CMake at both wheels. The example above calls `find_package(executorch)` +as well, and that package lives in its own distribution. ExecuTorch is a +namespace package, so `executorch.__file__` is `None` and has to be located +through its distribution metadata instead: + +```bash +cmake -DCMAKE_PREFIX_PATH="$(python -c 'import importlib.metadata as m, torch_tensorrt_executorch_runtime as r, pathlib; print(str(pathlib.Path(str(m.distribution("executorch").locate_file("executorch"))) / "share" / "cmake") + ";" + str(pathlib.Path(r.__file__).parent))')" ... +``` + +CMake 3.28 or newer is required for the example above, not because of this +package but because the `backend_cuda` component it pairs with rejects anything +older: earlier versions write the `$ORIGIN` token in a runtime search path +incorrectly. + +This package itself needs only 3.19. On 3.19 through 3.27 you can still use it, +by asking ExecuTorch for no components and linking the variables it gives you +instead of its targets: + +```cmake +find_package(executorch REQUIRED) +find_package(torchtrt_executorch REQUIRED) +target_include_directories(app PRIVATE ${EXECUTORCH_INCLUDE_DIRS}) +target_compile_definitions(app PRIVATE ${EXECUTORCH_COMPILE_DEFINITIONS}) +target_link_libraries(app PRIVATE + ${EXECUTORCH_LIBRARIES} torchtrt::executorch_backend) +``` + +Without an imported target to carry them, the include directories and the compile +definitions have to be applied by hand as well, which is what the three variables +above are for. + +There is nothing to include. The delegate has no public header: it registers +itself with ExecuTorch's backend registry from a static initializer inside the +shared library, and everything after that is ExecuTorch's own runtime API. The +CMake target links the library with `--no-as-needed`, because nothing in a +consumer references a symbol the delegate defines, and the default would drop it +and leave the backend unregistered. The wheel must use the same Python, PyTorch, ExecuTorch, CUDA, TensorRT, and -C++ ABI as its matching Torch-TensorRT wheel. +C++ ABI as its matching Torch-TensorRT wheel. This delegate requires CUDA 13; +the build matrix currently covers `cu130`, `cu132` and `cu134`, on both architectures. Ordinary Torch-TensorRT +release and JetPack builds retain their separate CUDA 12 support. ## Runtime libraries -The wheel does not bundle PyTorch, c10, TensorRT, or CUDA shared libraries. -Its `_portable_lib.so` has origin-relative runtime search paths for the -PyTorch, TensorRT, and CUDA library locations installed by their Python -packages: +The wheel bundles no ExecuTorch, PyTorch, c10, TensorRT, or CUDA shared +libraries. The delegate carries origin-relative runtime search paths, exactly as +the build sets them: + +- `$ORIGIN` +- `$ORIGIN/../../executorch/lib` +- `$ORIGIN/../../tensorrt_libs` +- `$ORIGIN/../../nvidia/cu13/lib` -- `torch/lib` -- `tensorrt_libs` -- `nvidia/cu13/lib` (CUDA 13) +There is no `$ORIGIN/../torch/lib` entry, because the delegate links no torch, +and no `$ORIGIN/../../nvidia/cuda_runtime/lib` entry, because that is the CUDA 12 +layout and this package requires CUDA 13. -These packages are installed transitively with the matching `torch-tensorrt` -wheel. For a system TensorRT or CUDA installation outside these standard -locations, its `lib` directory must be available through the system dynamic -loader configuration or `LD_LIBRARY_PATH`. +`$ORIGIN` is this package's `lib` directory; the three `../../` entries reach sibling +distributions, because `libexecutorch.so`, the TensorRT libraries, and the CUDA +runtime belong to other wheels. These packages are installed transitively with +the matching `torch-tensorrt` wheel. For a system TensorRT or CUDA installation +outside these standard locations, its `lib` directory must be available through +the system dynamic loader configuration or `LD_LIBRARY_PATH`. -The CI manylinux repair step changes the wheel platform tag; it does not -bundle these external libraries. The origin-relative paths are therefore part -of the wheel runtime contract. +The shared Linux build tags this wheel with `wheel tags` after the native +checks: `manylinux_2_28_x86_64` or `manylinux_2_35_aarch64`, with `py3-none`. +This updates wheel metadata and RECORD without bundling external libraries or +changing the delegate. Payload, dependency metadata, and installed-library +resolution are checked before the shared artifact is uploaded. ## Build > [!IMPORTANT] -> Build this wheel with `--no-build-isolation`. Its native extension must use -> the exact PyTorch installation that the matching Torch-TensorRT artifacts -> were built against. An isolated build may download a newer, ABI-incompatible -> PyTorch version. +> Build this wheel with `--no-build-isolation`. The delegate links the +> prebuilt runtime out of the ExecuTorch wheel that is installed at build time, +> and it must use the exact PyTorch installation the matching Torch-TensorRT +> artifacts were built against. An isolated build may download a newer, +> ABI-incompatible PyTorch or ExecuTorch. + +The build shells out to Bazel to compile the delegate, so `bazelisk` or `bazel` +must be on `PATH`. TensorRT itself arrives through Bazel's `@tensorrt` external +repository, so no local SDK path is needed. The example below assumes Linux with a matching CUDA 13 PyTorch and Torch-TensorRT installation. Substitute the channel for your CUDA throughout, such as `cu134` for CUDA 13.4. +The build reads the CUDA location out of the repository's `MODULE.bazel`, and the +copy checked in names one specific version. On a machine with a different CUDA, the +fetch fails saying that path does not exist. That file is generated, so render it for +your machine before building. It takes seven values, so run the packaging script that +already sets them rather than substituting by hand: + ```bash -export TensorRT_ROOT=/path/to/TensorRT +CUDA_HOME=/usr/local/cuda-13.2 bash packaging/pre_build_script.sh +``` -python -m pip install pyyaml \ +```bash +python -m pip install pyyaml patchelf tensorrt-cu13 \ --extra-index-url https://download.pytorch.org/whl/nightly/cu130 \ + --extra-index-url https://pypi.nvidia.com/ \ "executorch==1.6.0.dev20260915" +export TORCH_TENSORRT_EXECUTORCH_RUNTIME_VERSION="0.2.0.dev0+cu130" python -m pip wheel --no-build-isolation --no-deps \ --wheel-dir dist py/torch-tensorrt-executorch-runtime ``` -The native build obtains the ExecuTorch source through Bazel; no separate -source checkout or `EXECUTORCH_SOURCE_DIR` setting is required. The source -commit pinned in `MODULE.bazel` is the revision recorded by the pinned wheel. +Install the matching full `torch-tensorrt` wheel before building the companion. +The exact main-wheel dependency comes from that installed distribution, with +its local CUDA suffix removed. `TORCH_TENSORRT_EXECUTORCH_RUNTIME_VERSION` +sets only the companion's own version. When unset, the companion uses its own +base version plus a development suffix and Git revision. The shared build keeps +the main wheel's date and CUDA suffix while retaining the companion's independent +base version. Its CMake version also describes the companion, not the main wheel. + +`tensorrt-cu13` is needed at build time so the wheel can record an exact +`tensorrt-cu13==` requirement: that version is read from the installed +distribution, which the Bazel-provided libraries alone do not carry. It needs +`--extra-index-url https://pypi.nvidia.com/` above, because the PyPI +`tensorrt-cu13` sdist is a stub that downloads the real wheel from NVIDIA's index +and fails metadata generation without it. + +The delegate compiles and links entirely against the installed ExecuTorch +wheel, which ships the headers, the prebuilt runtime, and a CMake package. A +CUDA wheel is required: the CPU wheel ships no CUDA extension, and ExecuTorch +releases up to 1.4.1 ship no linkable runtime at all. ExecuTorch is not built +from source for this wheel, so no source checkout or `EXECUTORCH_SOURCE_DIR` is +involved. + +### Rebuilding and editable installs + +After rebuilding a wheel, install it with `python -m pip install --no-deps +--force-reinstall` followed by its path. This replaces an installed wheel with +the same version. The delegate is a shared library, not a Python extension; +`build_ext --inplace` does not rebuild it. + +For editable development, use the same matching dependencies and version setting: -The static ExecuTorch and delegate archives are intermediate build inputs; -users receive the final native Python module and do not compile anything. +```bash +python -m pip install --no-build-isolation --no-deps \ + --editable py/torch-tensorrt-executorch-runtime +``` -## Runtime replacement behavior +Repeat this command after native or CMake changes. Stop native consumers before +rebuilding and restart them afterward; rebuilding does not update a loaded +library. Python source changes are visible directly. For strict editable mode, +add `--config-settings editable_mode=strict` and keep the generated link directory. +The native library and generated CMake files live beside the source package in +its generated library directory. -Loading a TensorRT ExecuTorch program installs this wheel native module as the -process ExecuTorch portable runtime. The replacement includes TensorRTBackend, -XNNPACK, and the optimized CPU kernel set from the matching stock ExecuTorch -wheel. Programs using XNNPACK and CPU fallback regions therefore retain their -stock backend and optimized-kernel behavior after TensorRT activation. +`TORCH_TENSORRT_EXECUTORCH_DEBUG`, `TORCH_TENSORRT_ALLOW_UNPINNED_EXECUTORCH`, +and `TORCH_TENSORRT_SKIP_DELEGATE_REGISTRATION` accept `1`, `true`, `yes`, or +`on`, ignoring case. All other values, including unset, empty, `0`, and `false`, +are false. -## Python tensor placement +## Registration -The ExecuTorch Python portable runtime uses CPU tensors at its API boundary. -CUDA tensor inputs passed to `Program.run()` or `Program.forward()` are copied -to CPU before dispatch. TensorRT executes the delegated graph on GPU, but the -runtime copies inputs to the device and returns outputs on CPU. +Loading the delegate adds `TensorRTBackend` to the backend registry that the +installed ExecuTorch runtime owns. It replaces nothing: the stock runtime keeps +its own backends and kernels, and XNNPACK and CPU fallback regions behave +exactly as they do without this wheel. -Consequently, the Python API does not use the backend's device-resident -input/output fast path. Applications that need to keep inputs and outputs on -GPU should use the ExecuTorch C++ runner. +Registration happens in the delegate's static initializer, so the library has +to be loaded before a delegated program is loaded. Importing this package does +that, and nothing else: there is no API to call. + +```python +import torch_tensorrt_executorch_runtime # noqa: F401 +from executorch.extension.pybindings.portable_lib import _load_for_executorch + +program = _load_for_executorch("model.pte") +outputs = program.run_method("forward", (tensor,)) +``` + +ExecuTorch's own delegates register because they are linked into its pybindings +extension, so loading that extension pulls them in. A delegate shipped in a +separate wheel cannot join that link, and ExecuTorch has no discovery hook for +out-of-tree backends, so this package performs the equivalent step at import +time. Set `TORCH_TENSORRT_SKIP_DELEGATE_REGISTRATION=1` to import it without +loading the delegate; that is for tooling that wants the metadata only. + +Loading and running a program is ExecuTorch's API, not this package's. Tensor +placement, method lookup and output devices are all documented by ExecuTorch. A +program exported with `skip_h2d_for_method_inputs` keeps its inputs on the +device, because nothing here copies them. The device copies around the delegate need the program's device-tagged memory-planned arenas backed by real device memory, which ExecuTorch does only -through its Module API. `Program` therefore loads through -`_load_for_executorch_from_buffer` rather than through `executorch.runtime`, -whose program loader plans every arena on the host. +through its Module API. The examples therefore use `_load_for_executorch` +and `run_method`. The program loader in `executorch.runtime` plans these arenas +on the host and is not suitable for these delegated programs. ## Use -Use the same PyTorch nightly channel as the build for ExecuTorch and any -nightly PyTorch or Torch-TensorRT dependencies. CUDA and TensorRT packages -may resolve from PyPI or NVIDIA's index; the extra index applies to the whole -dependency solve. The exact ExecuTorch dev pin and the explicit PyTorch and -Torch-TensorRT dev-version ranges already permit their required prereleases. +A **CUDA** build of `executorch` is required at runtime, not just to build. The delegate carries a +`DT_NEEDED` on `libexecutorch_extension_cuda.so`, which only ExecuTorch's CUDA wheels ship, and +those live on the PyTorch nightly index. `install_requires` names the version including its local +label, so only the CUDA build the delegate linked satisfies it. A label-free specifier admits any +label, which let a `+cpu` wheel resolve and then fail to load at import. Carrying the label rules +that out, because PEP 440 only ignores labels when the specifier omits them. It does bind the wheel +to one CUDA train, which is correct: the delegate links that train's runtime. + +To install the wheel built above, use the same CUDA index as the build. Build it for the +CUDA train you run on: the requirement names that train, because the delegate links its runtime. +Its exact development-version requirements already permit the required prereleases. +CUDA and TensorRT packages may resolve from PyPI or NVIDIA's index; the extra +index applies to the whole dependency solve. ```bash python -m pip install dist/torch_tensorrt_executorch_runtime-*.whl \ @@ -96,8 +245,9 @@ python -m pip install dist/torch_tensorrt_executorch_runtime-*.whl \ ```python import torch -import torch_tensorrt +import torch_tensorrt_executorch_runtime # noqa: F401 +from executorch.extension.pybindings.portable_lib import _load_for_executorch -program = torch_tensorrt.load("model.pte", format="executorch") -outputs = program.forward(torch.ones((2, 3, 4, 4))) +program = _load_for_executorch("model.pte") +outputs = program.run_method("forward", (torch.ones((2, 3, 4, 4)),)) ``` diff --git a/py/torch-tensorrt-executorch-runtime/cmake/torchtrt_executorch-config.cmake b/py/torch-tensorrt-executorch-runtime/cmake/torchtrt_executorch-config.cmake new file mode 100644 index 0000000000..9c67743b5c --- /dev/null +++ b/py/torch-tensorrt-executorch-runtime/cmake/torchtrt_executorch-config.cmake @@ -0,0 +1,149 @@ +# CMake package for the Torch-TensorRT ExecuTorch delegate, as installed by the +# torch-tensorrt-executorch-runtime wheel. +# +# ExecuTorch ships its own backends as prebuilt shared libraries plus a CMake +# package, so a C++ app links executorch::backend_cuda and gets the backend +# registered. This file gives the TensorRT delegate the same treatment, so a C++ +# app can link it out of the installed wheel instead of building this repo from +# source: +# +# find_package(executorch REQUIRED COMPONENTS backend_cuda kernels_optimized) +# find_package(torchtrt_executorch REQUIRED) +# target_link_libraries(my_app PRIVATE +# executorch::runtime executorch::backend_cuda +# executorch::kernels_optimized torchtrt::executorch_backend) +# +# kernels_optimized supplies the et_copy host/device copy operators. +# There is nothing to include. The delegate exposes no public header: it +# registers itself with ExecuTorch's backend registry from a static initializer +# inside the shared library, and everything a caller does afterwards is +# ExecuTorch's own Runtime API. +# +# Point CMake at it with either of: +# -Dtorchtrt_executorch_DIR=$(python -c "import torch_tensorrt_executorch_runtime as m, pathlib; print(pathlib.Path(m.__file__).parent / 'lib/cmake/torchtrt_executorch')") +# -DCMAKE_PREFIX_PATH=$(python -c "import torch_tensorrt_executorch_runtime as m, pathlib; print(pathlib.Path(m.__file__).parent)") + +# No cmake_minimum_required here. A package config that calls it raises the consumer's own recorded +# minimum, and nothing in this file needs anything newer than 3.19: the token that misbehaves on older +# CMake is $ORIGIN, which appears in ExecuTorch's link options and not in ours, since ours are an +# absolute path. +# +# What ExecuTorch does below 3.28 depends on how it is asked. Found without components it succeeds and +# offers plain path variables instead of imported targets. Asked for a component it fails outright, so +# the recipe this package's own readme gives cannot be used below 3.28. A consumer on 3.19 to 3.27 has +# to find ExecuTorch without components and link its variables alongside this package's target, which +# works and is measured but is not written down anywhere else. +# +# 3.19, checked rather than declared, so the consumer's own minimum is left alone. +if(CMAKE_VERSION VERSION_LESS 3.19) + message(FATAL_ERROR + "torchtrt_executorch needs CMake 3.19 or newer, the same floor ExecuTorch's own package " + "declares. This CMake is ${CMAKE_VERSION}.") +endif() + +include(FindPackageHandleStandardArgs) + +# The package root is a fixed distance from this file, because the wheel installs this file to +# lib/cmake/torchtrt_executorch and nowhere else. It used to be found by walking up until a delegate +# turned up under lib/, and that walk reached the directory above the package: with this package's own +# lib/ empty, it accepted a same-named library belonging to something else and reported success. Only +# this package's own copy is acceptable, so look in exactly one place and let the check below report a +# missing one. +get_filename_component(_torchtrt_executorch_root "${CMAKE_CURRENT_LIST_DIR}/../../.." ABSOLUTE) +unset(TORCHTRT_EXECUTORCH_BACKEND_LIBRARY) +if(EXISTS "${_torchtrt_executorch_root}/lib/libexecutorch_backend_tensorrt.so") + set(TORCHTRT_EXECUTORCH_BACKEND_LIBRARY + "${_torchtrt_executorch_root}/lib/libexecutorch_backend_tensorrt.so") +endif() + +find_package_handle_standard_args( + torchtrt_executorch + REQUIRED_VARS TORCHTRT_EXECUTORCH_BACKEND_LIBRARY +) + +if(NOT torchtrt_executorch_FOUND) + return() +endif() + +set(TORCHTRT_EXECUTORCH_LIBRARIES torchtrt::executorch_backend) + +if(TARGET torchtrt::executorch_backend) + # Reusing is right when another subproject already ran this same config, and wrong when the name + # belongs to something else. The in-tree build defines it as an interface library over a private + # static copy, so a project that pulls that in and then calls find_package would silently link + # the private copy and never touch the shared library in the wheel. Only an imported shared + # library can be the one this config created. + get_target_property(_torchtrt_executorch_existing_type torchtrt::executorch_backend TYPE) + if(NOT _torchtrt_executorch_existing_type STREQUAL "SHARED_LIBRARY") + message(FATAL_ERROR + "torchtrt::executorch_backend already exists as a ${_torchtrt_executorch_existing_type}, " + "not as the imported shared library this package provides. The in-tree delegate target and " + "the installed one cannot both be used in a single configure: drop one of them.") + endif() + # Type alone does not identify it. A shared imported target of the same name pointing at another + # file would pass, and the consumer would link that file while believing it linked this one. + get_target_property(_torchtrt_executorch_existing_location + torchtrt::executorch_backend IMPORTED_LOCATION) + if(NOT _torchtrt_executorch_existing_location STREQUAL "${TORCHTRT_EXECUTORCH_BACKEND_LIBRARY}") + message(FATAL_ERROR + "torchtrt::executorch_backend already points at " + "${_torchtrt_executorch_existing_location}, not at the " + "${TORCHTRT_EXECUTORCH_BACKEND_LIBRARY} this package found. Two different delegates cannot " + "both be used in a single configure: drop one of them.") + endif() + message(STATUS "torchtrt_executorch: torchtrt::executorch_backend is already defined, reusing it") + return() +endif() + +# GLOBAL, so the one target this package exists to publish is visible outside the directory that +# called find_package. Without it a consumer who finds the package at the top level and links it from +# a subdirectory gets a message about a target that plainly exists, which is the ordinary layout for +# a project of more than one directory. +add_library(torchtrt::executorch_backend SHARED IMPORTED GLOBAL) +set_target_properties( + torchtrt::executorch_backend + PROPERTIES + IMPORTED_LOCATION "${TORCHTRT_EXECUTORCH_BACKEND_LIBRARY}" + INTERFACE_COMPILE_FEATURES cxx_std_17 +) + +# Retain the static registration even when the consumer references no delegate symbol. +# Scope --no-as-needed to this library so unrelated dependencies can still be dropped. +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + set_property( + TARGET torchtrt::executorch_backend + APPEND + PROPERTY + INTERFACE_LINK_OPTIONS + "LINKER:--push-state,--no-as-needed,${TORCHTRT_EXECUTORCH_BACKEND_LIBRARY},--pop-state" + ) + # So the loader finds the library in the installed wheel at run time. The delegate is not a + # dependency the consumer copies around: it lives in site-packages next to the executorch wheel + # whose runtime it links, and both have to be found from the same place. + # This is the path on the machine that configured, and it lands in every consumer binary, so a + # binary built here does not run anywhere else. That is the right default for building against an + # installed wheel, which is what this package is for, and wrong for anything redistributable. + # Not option(): inside a package config that creates a cache entry in the consumer's project + # and, depending on the policy in force, overrides a plain variable the consumer already + # set, so the documented opt-out could be ignored. Honour what the consumer set. + if(NOT DEFINED TORCHTRT_EXECUTORCH_EMBED_RUNPATH) + set(TORCHTRT_EXECUTORCH_EMBED_RUNPATH ON) + endif() + if(TORCHTRT_EXECUTORCH_EMBED_RUNPATH) + set_property( + TARGET torchtrt::executorch_backend + APPEND + PROPERTY INTERFACE_LINK_OPTIONS "LINKER:--enable-new-dtags,-rpath,${_torchtrt_executorch_root}/lib" + ) + else() + # This removes the run path this package adds, not the one CMake adds by itself: + # linking an imported library records its directory as DT_RUNPATH regardless, which is + # still this machine absolute path. Only CMAKE_SKIP_BUILD_RPATH in the consumer own + # project removes that, and a package config has no business setting it there. + message(STATUS + "torchtrt_executorch: not adding a run path. CMake still records " + "${_torchtrt_executorch_root}/lib as a build run path when linking an imported " + "library, so a redistributable binary also needs CMAKE_SKIP_BUILD_RPATH, and then has " + "to locate the delegate itself at run time.") + endif() +endif() diff --git a/py/torch-tensorrt-executorch-runtime/native/BUILD.bazel b/py/torch-tensorrt-executorch-runtime/native/BUILD.bazel index 9e7cc79c22..0dee2f3f23 100644 --- a/py/torch-tensorrt-executorch-runtime/native/BUILD.bazel +++ b/py/torch-tensorrt-executorch-runtime/native/BUILD.bazel @@ -20,40 +20,26 @@ filegroup( "//core/runtime:include_files", "//cpp:executorch_api_headers", "//cpp:executorch_backend_source_files", - "@executorch//:executorch_sources", ], ) -# ExecuTorch exposes its Python portable runtime through CMake. Keep that -# upstream build behind Bazel so both Torch-TensorRT wheels have one build -# entry point and share Bazel's dependency/toolchain selection. +# The delegate compiles and links against the installed ExecuTorch wheel, which ships the +# headers and the prebuilt runtime. ExecuTorch is not built from source here, so this target +# takes no ExecuTorch source input; CMAKE_PREFIX_PATH points find_package at the wheel. cmake( name = "delegate_native", + build_args = ["--verbose"], cache_entries = { "CMAKE_BUILD_TYPE": "Release", - "EXECUTORCH_CMAKE_FILE": "$(execpath @executorch//:executorch/CMakeLists.txt)", - "EXECUTORCH_ENABLE_LOGGING": "ON", + "CMAKE_PREFIX_PATH": "$${EXECUTORCH_CMAKE_PREFIX_PATH:-}", "PYTHON_EXECUTABLE": "$${PYTHON_BIN_PATH}", - "TORCH_HEADER_MARKER": "$(execpath @libtorch//:include/torch/headeronly/util/TypeTraits.h)", "TORCH_TENSORRT_SOURCE_DIR": "$$EXT_BUILD_ROOT$$", }, - data = [ - "@executorch//:executorch/CMakeLists.txt", - "@libtorch//:include/torch/headeronly/util/TypeTraits.h", - ], lib_source = ":delegate_sources", out_include_dir = "", - out_shared_libs = [ - # The extensions carry a DT_NEEDED on both of these and ExecuTorch's own wheel - # ships neither, so this one has to. - "libaoti_cuda_shims.so", - "libextension_cuda.so", - "_portable_lib.so", - "data_loader.so", - ], + out_shared_libs = ["libexecutorch_backend_tensorrt.so"], deps = [ "@cuda//:cudart", - "@libtorch//:torch", ] + select({ ":aarch64_linux": ["@tensorrt_sbsa//:nvinfer"], "//conditions:default": ["@tensorrt//:nvinfer"], diff --git a/py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt b/py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt index f81f106746..45b6744303 100644 --- a/py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt +++ b/py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt @@ -1,16 +1,14 @@ -cmake_minimum_required(VERSION 3.24) +# ExecuTorch's package config withholds every imported target below CMake 3.28, because it exports +# "$ORIGIN"-relative runpaths as link options and CMake writes that token incorrectly before then. +# This build asks for components, and asked that way ExecuTorch fails outright below 3.28 rather than +# falling back, so the floor here is not a courtesy. rules_foreign_cc 0.15.1 supplies 3.31.8, so +# nobody installs it by hand. +cmake_minimum_required(VERSION 3.28) project(torch_tensorrt_executorch_runtime LANGUAGES CXX) -if(NOT EXECUTORCH_SOURCE_DIR AND EXECUTORCH_CMAKE_FILE) - get_filename_component(EXECUTORCH_SOURCE_DIR "${EXECUTORCH_CMAKE_FILE}" DIRECTORY) -endif() -if(NOT TORCH_PRIMARY_INCLUDE_DIR AND TORCH_HEADER_MARKER) - get_filename_component(TORCH_PRIMARY_INCLUDE_DIR "${TORCH_HEADER_MARKER}/../../../.." ABSOLUTE) -endif() -if(NOT EXECUTORCH_SOURCE_DIR OR NOT TORCH_TENSORRT_SOURCE_DIR OR NOT TORCH_PRIMARY_INCLUDE_DIR) - message(FATAL_ERROR - "EXECUTORCH_SOURCE_DIR, TORCH_TENSORRT_SOURCE_DIR, and TORCH_PRIMARY_INCLUDE_DIR are required") +if(NOT TORCH_TENSORRT_SOURCE_DIR) + message(FATAL_ERROR "TORCH_TENSORRT_SOURCE_DIR is required") endif() list(APPEND CMAKE_MODULE_PATH "${TORCH_TENSORRT_SOURCE_DIR}/cmake/Modules") @@ -18,346 +16,224 @@ find_package(TensorRT REQUIRED) find_package(CUDAToolkit REQUIRED) find_package(Threads REQUIRED) -# ExecuTorch declares every pybind module with pybind11_add_module( SHARED -# ...). pybind11 maps a non-MODULE type onto pybind11::embed, and CMake's -# python_add_library then requires the Python::Python target, so the first module -# ExecuTorch declares (codegen/tools/selective_build) aborts the configure with: -# -# Python_ADD_LIBRARY: dependent target 'Python::Python' is not defined. -# Did you miss to request COMPONENT 'Development.Embed'? +# The prebuilt ExecuTorch runtime, from the installed wheel. Everything this wheel used to +# rebuild from source now comes from here, so the delegate links the same libexecutorch.so the +# user's ExecuTorch already loaded rather than a second private copy of it. # -# Do NOT take that hint literally. Development.Embed needs a libpython, and the -# manylinux CPython in the release image is built without one, so requesting it -# just fails the whole find_package instead: -# -# Could NOT find Python (missing: Python_INCLUDE_DIRS Python_LIBRARIES ...) -# -# pybind11 hit this and deliberately made the component optional for manylinux -# (pybind11NewTools.cmake: "Development.Module support (required for manylinux)"), -# so match that: Module required, Embed optional. Finding Python before pybind11 -# is the override pybind11 documents. -# -# PYTHON_EXECUTABLE is bridged first because the build passes the interpreter -# under that name while FindPython reads Python_EXECUTABLE. ExecuTorch does the -# same bridge, but only after this point, so without it this call could resolve a -# different interpreter than the rest of the build. -if(PYTHON_EXECUTABLE AND NOT Python_EXECUTABLE) - set(Python_EXECUTABLE "${PYTHON_EXECUTABLE}") -endif() -find_package(Python REQUIRED COMPONENTS Interpreter Development.Module - OPTIONAL_COMPONENTS Development.Embed) - -# With Embed optional, Python::Python does not exist on manylinux, and the SHARED -# declaration above still demands it. Supply an empty stand-in for that case only. -# It links nothing, which is what an extension module needs: Python symbols -# resolve from the interpreter that loads the module, never from a linked -# libpython. Where the image does provide a usable libpython, the real imported -# target is used instead. -# -# This is a workaround for a defect upstream, not the root fix. Those three -# modules should be declared MODULE rather than SHARED: CMake skips this check -# entirely for MODULE, and MODULE is what a Python extension is. Filed upstream. -if(NOT TARGET Python::Python) - add_library(Python::Python INTERFACE IMPORTED) - # The real Python::Python carries INTERFACE_INCLUDE_DIRECTORIES. Forward those - # through Python::Module so the stand-in is not silently header-less if a - # consumer ever relies on them. Python::Module links no libpython. - set_target_properties(Python::Python PROPERTIES - INTERFACE_LINK_LIBRARIES Python::Module) +# setup.py locates the prefix through installed distribution metadata and passes it through Bazel. +# rules_foreign_cc merges its own dependency prefixes in, so this list is not empty even when +# that value is, and the check below is only a clearer message for the case where CMake is +# driven directly with no prefix at all. +if(NOT CMAKE_PREFIX_PATH) + message(FATAL_ERROR + "CMAKE_PREFIX_PATH is empty, so the installed ExecuTorch cannot be located. Build this " + "wheel through its setup.py, which locates the path through installed distribution metadata.") endif() +find_package(executorch REQUIRED) -set(BUILD_TESTING OFF CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_PYBIND ON CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_EXTENSION_MODULE ON CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_EXTENSION_DATA_LOADER ON CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR ON CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP ON CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_EXTENSION_TENSOR ON CACHE BOOL "" FORCE) -# Needed twice over: this backend registers the allocator the exported program's device -# copies look up, and extension/cuda, which owns the shared caller stream, comes with it. -set(EXECUTORCH_BUILD_CUDA ON CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_KERNELS_OPTIMIZED ON CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_XNNPACK ON CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_TESTS OFF CACHE BOOL "" FORCE) -set(CMAKE_POSITION_INDEPENDENT_CODE ON) - -# rules_foreign_cc puts an explicit dynamic -lstdc++ in the Bazel toolchain's linker -# flags, ahead of the object files and wrapped in --as-needed, where it resolves -# nothing. Remove it before anything that links is defined, and put a working one back -# below. -# -# Two things have to be right, and each was got wrong before. -# -# WHERE. Removing it later, after add_subdirectory below, is too late: the extensions -# are created in ExecuTorch's directory and take their copy of these variables at that -# point, so a later edit changes a value nothing reads. Measured by stripping before -# and after add_subdirectory and reading the subdirectory target's own link.txt. -# -# WHY IT IS NOT HARMLESS WHERE IT SITS. It arrives wrapped in --as-needed, before the -# object files, which with the default linker means it gets dropped. This build passes -# -fuse-ld=gold, and gold keeps it anyway. Measured in the release image on a shared -# object using std::string and exceptions: -# -# bfd, -lstdc++ present -> no libstdc++ NEEDED -# gold, -lstdc++ present -> libstdc++.so.6 NEEDED -# gold+lto, -lstdc++ present -> libstdc++.so.6 NEEDED -# gold+lto, -lstdc++ removed -> no libstdc++ NEEDED -# -# So the fragment has to go, wherever it sits. Matched by regex rather than as one -# exact string, so a spacing change cannot make the removal silently stop working. -# The resolved values are printed because the check at the end of this file reports a -# property of the artifact, and the first question is always what was on the link line. -# Only these three can carry it. rules_foreign_cc puts the toolchain's link libraries -# into CMAKE_{SHARED,MODULE,EXE}_LINKER_FLAGS_INIT, and filters the STANDARD_LIBRARIES -# variables down to static-runtime flags, so touching those would strip nothing and -# would additionally shadow the cache entry enable_language(C) creates later. -# -# The whole push/pop group goes at once. Removing only -lstdc++ from inside it would -# leave an unbalanced --push-state, which both linkers reject outright: -# ld.gold: error: unbalanced --push-state/--pop-state -# The bare form is matched separately, with a boundary, so -lstdc++fs and -# -lstdc++_nonshared are left alone. -foreach(_torch_tensorrt_linker_flags - CMAKE_MODULE_LINKER_FLAGS - CMAKE_SHARED_LINKER_FLAGS - CMAKE_EXE_LINKER_FLAGS) - string(REGEX REPLACE "-Wl,--push-state,-as-needed +-lstdc\\+\\+ +-Wl,--pop-state" - "" ${_torch_tensorrt_linker_flags} "${${_torch_tensorrt_linker_flags}}") - string(REGEX REPLACE "(^| )-lstdc\\+\\+( |$)" "\\2" - ${_torch_tensorrt_linker_flags} "${${_torch_tensorrt_linker_flags}}") - message(STATUS "torch_tensorrt: ${_torch_tensorrt_linker_flags} = " - "[${${_torch_tensorrt_linker_flags}}]") +foreach(_torch_tensorrt_required_target executorch::runtime executorch::extension_cuda) + if(NOT TARGET ${_torch_tensorrt_required_target}) + message(FATAL_ERROR + "The installed ExecuTorch does not provide ${_torch_tensorrt_required_target}. A CUDA " + "wheel from the pinned nightly channel is required; the CPU wheel ships no CUDA " + "extension, and releases up to 1.4.1 ship no linkable runtime at all.") + endif() endforeach() -# Removing that fragment is only half of it: something still has to supply the C++ -# runtime. The Bazel toolchain hands CMake the C driver, gcc, as CMAKE_CXX_COMPILER, -# and gcc links no C++ runtime at all, which is exactly why the toolchain injected an -# explicit -lstdc++ in the first place. -# -# So put one back, after the object files. CMake appends CMAKE_CXX_STANDARD_LIBRARIES -# there, which is the position that works, and is the same placement rules_foreign_cc -# documents for exactly this problem. -# -# Measured on a shared object that stores a std::exception_ptr and formats through an -# ostringstream: -# -# UND _M_addref libstdc++ NEEDED -# the toolchain's --as-needed group, pre-objects 0 0 -# nothing at all 0 0 -# -lstdc++ after the objects 0 1 -# -# It has to be the dynamic library and not libstdc++.a. These artifacts share a C++ ABI -# with libtorch_cpu.so, libc10.so, libtorch_python.so and libnvinfer.so.11, every one of -# which already carries DT_NEEDED libstdc++.so.6 and needs no more than GLIBCXX_3.4.22. -# A second static copy cannot be made safe in that company. Left visible it interposes -# the real runtime for anything loaded alongside it, and hidden it gives this wheel -# private __cxxabiv1 typeinfo and a private unwinder, so exceptions stop crossing the -# boundary and dynamic_cast starts returning null. -string(APPEND CMAKE_CXX_STANDARD_LIBRARIES " -lstdc++") -# try_compile forwards CMAKE_EXE_LINKER_FLAGS but not CMAKE_CXX_STANDARD_LIBRARIES, so -# without this a C++ probe loses the runtime the strip above removed and reports the -# feature absent instead of failing. -list(APPEND CMAKE_TRY_COMPILE_PLATFORM_VARIABLES CMAKE_CXX_STANDARD_LIBRARIES) -message(STATUS "torch_tensorrt: CMAKE_CXX_STANDARD_LIBRARIES now = " - "[${CMAKE_CXX_STANDARD_LIBRARIES}]") - -add_subdirectory("${EXECUTORCH_SOURCE_DIR}" executorch) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) -add_library(torch_tensorrt_executorch_backend STATIC +# One shared library, holding only the TensorRT-specific code. This is the same shape as +# ExecuTorch's own delegates: leave register_backend undefined, carry a DT_NEEDED on +# libexecutorch.so, and let the loader bind the two together. executorch::extension_cuda +# supplies the caller stream, whose thread-local must have exactly one definition in the +# process; linking the shipped shared library is what guarantees that, since a static copy +# would give this library a second, invisibly separate stream. +add_library(executorch_backend_tensorrt SHARED "${TORCH_TENSORRT_SOURCE_DIR}/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp" "${TORCH_TENSORRT_SOURCE_DIR}/cpp/src/torch_tensorrt/executorch/TensorRTBlobHeader.cpp" "${TORCH_TENSORRT_SOURCE_DIR}/cpp/src/torch_tensorrt/executorch/WeightStreamingBudget.cpp") -target_compile_features(torch_tensorrt_executorch_backend PUBLIC cxx_std_17) -target_compile_definitions(torch_tensorrt_executorch_backend - PRIVATE C10_USING_CUSTOM_GENERATED_MACROS) -target_include_directories(torch_tensorrt_executorch_backend PRIVATE - "${TORCH_TENSORRT_SOURCE_DIR}" - "${TORCH_TENSORRT_SOURCE_DIR}/cpp/include" - "${EXECUTORCH_SOURCE_DIR}/.." - "${EXECUTORCH_SOURCE_DIR}/runtime/core/portable_type/c10") -if(NOT TARGET extension_cuda) - message(FATAL_ERROR - "ExecuTorch did not define extension_cuda; the delegate needs it for the shared " - "caller-stream selection. Configure ExecuTorch with EXECUTORCH_BUILD_CUDA=ON.") -endif() - -target_link_libraries(torch_tensorrt_executorch_backend PRIVATE - CUDA::cudart TensorRT::nvinfer Threads::Threads executorch extension_cuda) - -foreach(_torch_tensorrt_required_cuda_target extension_cuda aoti_cuda_shims) - if(NOT TARGET ${_torch_tensorrt_required_cuda_target}) - message(FATAL_ERROR - "ExecuTorch did not define ${_torch_tensorrt_required_cuda_target}; the wheel " - "ships it and the extensions load it. Check EXECUTORCH_BUILD_CUDA.") - endif() -endforeach() -if(NOT TARGET portable_lib) - message(FATAL_ERROR "ExecuTorch did not define portable_lib") +target_include_directories(executorch_backend_tensorrt PRIVATE + "${TORCH_TENSORRT_SOURCE_DIR}" + "${TORCH_TENSORRT_SOURCE_DIR}/cpp/include") +target_compile_definitions(executorch_backend_tensorrt PRIVATE TORCH_TENSORRT_EXECUTORCH_RUNTIME) + +# Compile definitions and the C++ standard come from the package rather than being repeated +# here. ET_EVENT_TRACER_ENABLED in particular has to match how the runtime was built: it +# switches the tracer hooks between real bodies and empty ones, so a delegate that guesses +# wrong still links and runs while recording nothing. +target_link_libraries(executorch_backend_tensorrt PRIVATE + executorch::runtime + executorch::extension_cuda + CUDA::cudart + TensorRT::nvinfer + Threads::Threads) + +# Link the C++ runtime dynamically, the way ExecuTorch's own delegates and every other shared object +# in the process (libtorch, libc10, libnvinfer) already do. The build toolchain is newer than the +# libstdc++.so.6 on a user's machine, so an optimized build emits out-of-line calls into the newer +# runtime (for example std::string::_M_replace_cold). Naming stdc++ as a link library places `-lstdc++` +# after the objects, where the toolchain's own libstdc++.so linker script resolves those references: +# it dynamic-links the old system libstdc++.so.6 for the stable, versioned symbols and pulls only the +# newer helpers statically from its libstdc++_nonshared.a. The result carries a normal DT_NEEDED on +# libstdc++.so.6; the artifact guard checks its versions against the wheel's platform policy. A static C++ +# runtime is deliberately avoided: this delegate is dlopened beside libtorch and ExecuTorch, and a +# private libstdc++ would give it its own exception type_info and locale state, which breaks exceptions +# and dynamic_cast across the boundary. +# Build to the standard ExecuTorch's own exported configuration asks for. The delegate is compiled +# through CMake from Bazel, which forwards the main build's -std=c++20, and under C++20 an ordinary +# std::string assignment reaches for a libstdc++ helper that only newer runtimes export. That would +# put the artifact above the platform its wheel is tagged for. This is a compile option rather than +# the CXX_STANDARD property because the forwarded flag arrives in CMAKE_CXX_FLAGS, and only an option +# on the target is placed after it on the command line. +target_compile_options(executorch_backend_tensorrt PRIVATE -std=c++17) + +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_libraries(executorch_backend_tensorrt PRIVATE stdc++) endif() -# The pybind bridge includes both ATen and portable ExecuTorch headers. Keep the -# ATen/c10/torch header set internally consistent by resolving all of it from -# the exact PyTorch wheel that supplies the linked libraries. -if(NOT EXISTS "${TORCH_PRIMARY_INCLUDE_DIR}/torch/headeronly") +# extension_cuda has to be a shared library, or the delegate statically absorbs the CUDA stream +# implementation instead of sharing ExecuTorch's, and a mixed-delegate run ends up on a different +# stream while every registration and import check still passes. The link-time guard requires a +# DT_NEEDED on libexecutorch_extension_cuda.so, which only a shared target leaves; fail configuration +# here too so the cause is named at configure time rather than as a missing DT_NEEDED later. +get_target_property(_extension_cuda_type executorch::extension_cuda TYPE) +if(_extension_cuda_type STREQUAL "STATIC_LIBRARY") message(FATAL_ERROR - "torch/headeronly not found under TORCH_PRIMARY_INCLUDE_DIR=${TORCH_PRIMARY_INCLUDE_DIR}") -endif() -file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/torch_header_shim/torch") -file(COPY "${TORCH_PRIMARY_INCLUDE_DIR}/c10/" - DESTINATION "${CMAKE_BINARY_DIR}/torch_header_shim/c10") -file(COPY "${TORCH_PRIMARY_INCLUDE_DIR}/torch/headeronly/" - DESTINATION "${CMAKE_BINARY_DIR}/torch_header_shim/torch/headeronly") -if(NOT EXISTS "${CMAKE_BINARY_DIR}/torch_header_shim/c10/util/complex.h" OR - NOT EXISTS "${CMAKE_BINARY_DIR}/torch_header_shim/torch/headeronly/macros/Macros.h") - message(FATAL_ERROR "Failed to stage the PyTorch header shim") + "executorch::extension_cuda is a STATIC_LIBRARY; the delegate would absorb a private CUDA " + "stream implementation instead of sharing ExecuTorch's. Use a CUDA ExecuTorch build that " + "ships libexecutorch_extension_cuda.so.") endif() -target_include_directories(util BEFORE PRIVATE "${CMAKE_BINARY_DIR}/torch_header_shim") -target_include_directories(portable_lib BEFORE PRIVATE "${CMAKE_BINARY_DIR}/torch_header_shim") -# TensorRTBackend registers through a static initializer. Force its complete -# archive into the same pybind module that owns ExecuTorch's backend registry. -target_link_libraries(portable_lib PRIVATE - "$" - extension_threadpool - CUDA::cudart TensorRT::nvinfer Threads::Threads) - -# What supplies the C++ runtime is the -lstdc++ appended to -# CMAKE_CXX_STANDARD_LIBRARIES near the top of this file. LINKER_LANGUAGE CXX is what -# selects CMAKE_CXX_COMPILER and those standard libraries for these targets, so it has -# to be set on every one of them. -# - -# The CUDA backend brings two shared libraries of its own. They get the same treatment, -# because libaoti_cuda_shims.so is the first DT_NEEDED of _portable_lib.so and so leads -# the whole dlopen group in symbol search order. -set(_torch_tensorrt_executorch_runtime_targets portable_lib data_loader) -foreach(_torch_tensorrt_cuda_shared_target extension_cuda aoti_cuda_shims) - get_target_property(_torch_tensorrt_cuda_shared_type - ${_torch_tensorrt_cuda_shared_target} TYPE) - # An IMPORTED prebuilt is already linked and cannot be given link options. - get_target_property(_torch_tensorrt_cuda_shared_imported - ${_torch_tensorrt_cuda_shared_target} IMPORTED) - if(_torch_tensorrt_cuda_shared_type STREQUAL "SHARED_LIBRARY" AND - NOT _torch_tensorrt_cuda_shared_imported) - list(APPEND _torch_tensorrt_executorch_runtime_targets - ${_torch_tensorrt_cuda_shared_target}) - endif() -endforeach() - -foreach(_torch_tensorrt_executorch_runtime_target - IN LISTS _torch_tensorrt_executorch_runtime_targets) - set_property(TARGET ${_torch_tensorrt_executorch_runtime_target} - PROPERTY LINKER_LANGUAGE CXX) -endforeach() - -# ExecuTorch links CUDA::curand into aoti_cuda_shims, but nothing in it calls a curand -# host function: the only curand use in the pinned source is the device-side API from -# curand_kernel.h inside rand.cu, which nvcc compiles into the fatbinary. Measured on the -# shipped artifact: DT_NEEDED carries libcurand.so.10 while the dynamic symbol table -# imports no curand symbol at all. -# -# The over-link is not harmless: the wheel declares a dependency it never calls, and -# nothing guarantees an installer provides it, so importing the runtime can fail on a -# missing shared object before any delegate runs. Which installers happen to supply -# libcurand.so.10 is not stable and has already changed more than once, so the reason to -# drop the link is that nothing uses it, not that a particular resolver omits it. -# -# Dropped here rather than patched into ExecuTorch. This becomes a no-op only once the -# pin includes upstream's own removal, which landed on their main line AFTER the pinned -# release. The 1.4 series still links cuRAND, so a patch bump within 1.4 does not make -# this redundant: check that the pinned tree has no CUDA::curand in -# backends/cuda/CMakeLists.txt before deleting this block. -foreach(_torch_tensorrt_curand_property LINK_LIBRARIES INTERFACE_LINK_LIBRARIES) - get_target_property(_torch_tensorrt_curand_value - aoti_cuda_shims ${_torch_tensorrt_curand_property}) - if(_torch_tensorrt_curand_value AND CUDA::curand IN_LIST _torch_tensorrt_curand_value) - list(REMOVE_ITEM _torch_tensorrt_curand_value CUDA::curand) - set_property(TARGET aoti_cuda_shims - PROPERTY ${_torch_tensorrt_curand_property} "${_torch_tensorrt_curand_value}") - endif() -endforeach() - -# The runtime wheel intentionally does not bundle PyTorch, TensorRT, or CUDA. -# PyTorch, TensorRT, and CUDA pip packages install their shared libraries in -# directories under site-packages, relative to this extension package. -# Preserve these paths in both the Bazel-collected build output and installed -# extension so importing the wheel does not depend on LD_LIBRARY_PATH. -if(CUDAToolkit_VERSION_MAJOR EQUAL 13) - set(_torch_tensorrt_executorch_cuda_rpath "$ORIGIN/../nvidia/cu13/lib") +# ExecuTorch exports each delegate as executorch::backend_, backing a +# libexecutorch_backend_.so. It reaches that from short internal target names such as +# xnnpack_backend plus OUTPUT_NAME and an alias; naming the target for the file it produces gets +# to the same place with one fewer indirection, so the target, the alias and the shipped file all +# agree and OUTPUT_NAME is unnecessary. What a C++ consumer spells is identical either way. +# +# The runpath reaches sibling site-packages distributions, because this wheel deliberately +# bundles neither ExecuTorch, PyTorch, TensorRT, nor CUDA. The ExecuTorch entry is the new one: +# libexecutorch.so lives in another distribution's lib directory, so $ORIGIN/../lib does not +# reach it. TWO levels, because the wheel ships this library in a lib/ subdirectory of the +# package, matching where ExecuTorch keeps its own backends: $ORIGIN is site-packages/ +# torch_tensorrt_executorch_runtime/lib, so site-packages itself is two levels up. Derive the +# depth from that layout, because ExecuTorch's own delegates are laid out differently: measured +# on the shipped wheel, libexecutorch_backend_cuda.so has RUNPATH +# $ORIGIN:$ORIGIN/../../nvidia/cu13/lib:$ORIGIN/../backends/cuda:$ORIGIN/../lib:..., +# where $ORIGIN/../lib reaches the runtime because that delegate sits in a subdirectory of the +# same distribution. This one sits in a different distribution, so the same relative path lands +# somewhere else entirely and the depth has to come from this wheel's layout instead. +# +# The depth and the install location are one decision. setup.py installs the library under +# lib/, __init__.py looks for it there, and the CMake package searches there; if any of those +# moves, every entry below is off by a level and the delegate fails to resolve its own +# dependencies. tests/py/dynamo/executorch/test_api.py asserts the pairing. +# +# One list, two consumers: CMake wants it semicolon-separated, patchelf below wants colons, and +# patchelf runs last so it is the one that decides what ships. Keeping two hand-written copies +# meant the ineffective one could drift without any signal. +# +# This delegate requires CUDA 13, whose runtime wheel uses the shared cu13 directory. +set(TORCH_TENSORRT_DELEGATE_RUNPATH + "$ORIGIN" + "$ORIGIN/../../executorch/lib" + "$ORIGIN/../../tensorrt_libs" + "$ORIGIN/../../nvidia/cu13/lib") +string(JOIN ":" TORCH_TENSORRT_DELEGATE_RUNPATH_COLONS ${TORCH_TENSORRT_DELEGATE_RUNPATH}) + +# Match the architecture-specific tags applied by the shared wheel build. +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$") + # Arm needs the newer baseline: its build container has no devtoolset, so the C++ + # runtime symbols the delegate references are not absorbed statically as on x86. + set(TORCH_TENSORRT_MANYLINUX_TAG "manylinux_2_35_aarch64") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$") + set(TORCH_TENSORRT_MANYLINUX_TAG "manylinux_2_28_x86_64") else() - message(FATAL_ERROR "Unsupported CUDA major version: ${CUDAToolkit_VERSION_MAJOR}") + message(FATAL_ERROR "Unsupported companion architecture: ${CMAKE_SYSTEM_PROCESSOR}") endif() -set(_torch_tensorrt_executorch_runtime_rpath - "$ORIGIN" - "$ORIGIN/../torch/lib" - "$ORIGIN/../tensorrt_libs" - "${_torch_tensorrt_executorch_cuda_rpath}") -set_target_properties(portable_lib PROPERTIES +set_target_properties(executorch_backend_tensorrt PROPERTIES BUILD_WITH_INSTALL_RPATH ON - INSTALL_RPATH "${_torch_tensorrt_executorch_runtime_rpath}" - OUTPUT_NAME "_portable_lib" - SUFFIX ".so") -# The pinned ExecuTorch builds this extension as _C, setting both OUTPUT_NAME and -# EXECUTORCH_PYTHON_MODULE_NAME=_C, and that define is what PYBIND11_MODULE pastes into the module -# init symbol. Forcing OUTPUT_NAME back to _portable_lib above would ship _portable_lib.so still -# exporting PyInit__C, and activate() imports the extension by module name, so that import would -# raise. Set the define to match the file name. The inherited _C value is dropped first so a single -# -D reaches the compiler instead of two that disagree and warn about a redefinition. -get_target_property(_portable_lib_defs portable_lib COMPILE_DEFINITIONS) -if(_portable_lib_defs) - list(REMOVE_ITEM _portable_lib_defs "EXECUTORCH_PYTHON_MODULE_NAME=_C") - set_target_properties(portable_lib PROPERTIES COMPILE_DEFINITIONS "${_portable_lib_defs}") + INSTALL_RPATH "${TORCH_TENSORRT_DELEGATE_RUNPATH}") + +# executorch::runtime carries the build machine's own site-packages path as a raw +# INTERFACE_LINK_OPTIONS -rpath, which BUILD_WITH_INSTALL_RPATH does not suppress and install +# does not rewrite, so without this the published wheel ships a path from the CI builder. It is +# also placed ahead of the entries above, and measuring with LD_DEBUG=libs shows the loader +# resolving libexecutorch.so straight out of it rather than from $ORIGIN/../../executorch/lib -- +# which is exactly the blind spot that let an earlier wrong RUNPATH depth go unnoticed, because +# on the build machine the absolute path works no matter what the relative entries say. +# +# Rewritten rather than deleted: the relative entries are the ones that have to do the work, and +# a run with no absolute fallback is the only run that can prove they do. +# +# Plain --set-rpath, never --force-rpath: patchelf writes DT_RUNPATH without it and the older +# DT_RPATH with it. The pinned ExecuTorch passes --enable-new-dtags for exactly this reason and +# writes down why -- DT_RPATH is searched before LD_LIBRARY_PATH and applies transitively to a +# dependency's dependencies, so a consumer could not point an instrumented or locally built +# runtime at their application. Forcing the old tag here would reverse that for the delegate. +find_program(TORCH_TENSORRT_PATCHELF NAMES patchelf) +if(TORCH_TENSORRT_PATCHELF) + add_custom_command(TARGET executorch_backend_tensorrt POST_BUILD + COMMAND "${TORCH_TENSORRT_PATCHELF}" --remove-rpath + "$" + COMMAND "${TORCH_TENSORRT_PATCHELF}" --set-rpath + "${TORCH_TENSORRT_DELEGATE_RUNPATH_COLONS}" + "$" + COMMENT "Removing absolute build-machine RUNPATH entries from the delegate" + VERBATIM) +else() + message(FATAL_ERROR + "patchelf is required to strip the absolute build-machine RUNPATH that ExecuTorch's " + "imported targets add. Install patchelf, or set TORCH_TENSORRT_PATCHELF.") endif() -target_compile_definitions(portable_lib PRIVATE EXECUTORCH_PYTHON_MODULE_NAME=_portable_lib) -# data_loader gets $ORIGIN too, so finding those libraries does not depend on which -# module Python imports first. -set_target_properties(data_loader PROPERTIES - BUILD_WITH_INSTALL_RPATH ON - INSTALL_RPATH "${_torch_tensorrt_executorch_runtime_rpath}" - SUFFIX ".so") -install(TARGETS portable_lib data_loader LIBRARY DESTINATION lib) - -# ExecuTorch's published wheel ships neither of these and the extensions carry a -# DT_NEEDED on them, so importing the runtime fails without them. ExecuTorch installs -# them to CMAKE_INSTALL_LIBDIR, which is lib64 on some distributions, so put them in -# lib/ as well, beside the extensions. -install(TARGETS extension_cuda aoti_cuda_shims LIBRARY DESTINATION lib) +# The same spelling consumers use for ExecuTorch's own delegates, so a project that builds this +# in-tree and links executorch::backend_cuda can link executorch::backend_tensorrt without +# learning a second convention. +# +# This alias is in-tree only. Bazel collects the native library; setup.py installs the +# separate CMake package used by consumers of the wheel. +add_library(executorch::backend_tensorrt ALIAS executorch_backend_tensorrt) + +# Registration happens in a static initializer, so a consumer references no symbol from this +# library and --as-needed drops the DT_NEEDED entirely -- measured: the entry disappears and the +# initializer never runs, with no diagnostic. ExecuTorch wraps its own registration-only component +# libraries the same way, one option per library because CMake dedupes identical push-state text, +# and only on Linux because that is where the linker has the flag. +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_options(executorch_backend_tensorrt INTERFACE + "LINKER:--push-state,--no-as-needed,$,--pop-state") +endif() -add_custom_target(torch_tensorrt_executorch_portable_lib ALL - DEPENDS ${_torch_tensorrt_executorch_runtime_targets}) +install(TARGETS executorch_backend_tensorrt LIBRARY DESTINATION lib) -# Guard both halves of the C++ runtime contract on the real artifacts, because this -# wheel has no auditwheel step behind it: nothing here may define libstdc++'s own -# symbols, and anything referencing the runtime must declare it. -# -# ALL on the aggregate target above matters: a custom target without it is excluded -# from the default build, and the default target is what the wheel build runs. The -# check hangs off that aggregate rather than off portable_lib and data_loader -# directly, because add_custom_command(TARGET) only accepts targets created in this -# directory and those two come from ExecuTorch's subdirectory. +# The delegate has to resolve register_backend from the shipped runtime rather than carry its +# own copy, because a private copy would register into a registry nothing queries. Undefined is +# the correct state for that symbol here, so assert it on the real artifact. # -# The check lives in a script rather than an inline shell string so it can report what -# it saw. A bare "defines libstdc++ symbols" does not say which input added them, and -# the build logs do not print link lines, so the script prints both. -find_program(TORCH_TENSORRT_READELF NAMES readelf llvm-readelf) +# The same check requires dynamic libstdc++ linkage and validates numeric and named symbol +# versions against the architecture-specific manylinux policy. +# eu-readelf too: the guard parses all three dialects, so it should be able to find all +# three. Its output differs (bare RPATH, UNDEF for UND), which is why the guard accepts both. +find_program(TORCH_TENSORRT_READELF NAMES readelf llvm-readelf eu-readelf) if(NOT TORCH_TENSORRT_READELF AND CMAKE_SYSTEM_NAME STREQUAL "Linux") message(FATAL_ERROR - "readelf is required to verify the Python extensions share the C++ runtime. " - "Install binutils, or set TORCH_TENSORRT_READELF to a readelf.") + "readelf is required to verify the delegate imports the ExecuTorch runtime. Install " + "binutils, or set TORCH_TENSORRT_READELF to a readelf.") endif() if(TORCH_TENSORRT_READELF) - foreach(_torch_tensorrt_checked_target - IN LISTS _torch_tensorrt_executorch_runtime_targets) - add_custom_command(TARGET torch_tensorrt_executorch_portable_lib POST_BUILD - COMMAND "${CMAKE_COMMAND}" -E echo - "checking $ shares the C++ runtime" - COMMAND sh - "${CMAKE_CURRENT_LIST_DIR}/check_shared_cxx_runtime.sh" - "${TORCH_TENSORRT_READELF}" - "$" - "$/CMakeFiles/${_torch_tensorrt_checked_target}.dir/link.txt" - VERBATIM) - endforeach() + add_custom_command(TARGET executorch_backend_tensorrt POST_BUILD + COMMAND sh + "${CMAKE_CURRENT_LIST_DIR}/check_imports_executorch_runtime.sh" + "${TORCH_TENSORRT_READELF}" + "$" + "$" + # The same string patchelf applies above, so the guard compares the whole set against what + # the build asked for rather than spot-checking one entry it restates. Checking only + # $ORIGIN/../../executorch/lib accepted a delegate with the tensorrt_libs or the CUDA entry + # missing, and the loader cannot substitute for this check: any runner with a CUDA toolkit + # installed resolves libcudart out of ld.so.cache whatever the RUNPATH says. + "${TORCH_TENSORRT_DELEGATE_RUNPATH_COLONS}" + "${TORCH_TENSORRT_MANYLINUX_TAG}" + VERBATIM) endif() diff --git a/py/torch-tensorrt-executorch-runtime/native/check_imports_executorch_runtime.sh b/py/torch-tensorrt-executorch-runtime/native/check_imports_executorch_runtime.sh new file mode 100755 index 0000000000..4bf63b9927 --- /dev/null +++ b/py/torch-tensorrt-executorch-runtime/native/check_imports_executorch_runtime.sh @@ -0,0 +1,251 @@ +#!/bin/sh +# Verify that the delegate imports the shared registry and can load from the wheel layout. +# Usage: check_imports_executorch_runtime.sh [libexecutorch.so] +# [expected-runpath] [manylinux-tag] + +set -u +set -f +LC_ALL=C +export LC_ALL + +if [ "$#" -lt 2 ] || [ "$#" -gt 5 ]; then + echo "FATAL: expected [libexecutorch.so] [expected-runpath] [manylinux-tag]" >&2 + exit 1 +fi +readelf_bin="$1" +target="$2" +runtime="${3-}" +expected_runpath="${4-}" +manylinux_tag="${5-}" + +fail() { + echo "FATAL: $*" >&2 + exit 1 +} + +# An omitted option disables its check; an empty supplied value is a caller error. +for argument in "$@"; do + [ -n "${argument}" ] || fail "supplied arguments must not be empty" +done +if [ "$#" -ge 5 ]; then + case "${manylinux_tag}" in + manylinux_2_28_x86_64|manylinux_2_35_aarch64) ;; + *) fail "unsupported manylinux tag: ${manylinux_tag}" ;; + esac +else + echo "note: no manylinux tag given, so platform symbol-version checks are skipped" >&2 +fi +if [ "$#" -lt 3 ]; then + echo "note: no runtime given, so runtime export and symbol-version comparisons are skipped" >&2 +fi + +# auditwheel 6.8.2 manylinux-policy.json, limited to these two release architectures. +# Numeric gaps and named nodes are policy entries, not compiler-version ceilings. +policy_versions() { + case "${manylinux_tag}" in + manylinux_2_28_x86_64) + cat <<'POLICY' +GLIBC 2.2.5 2.2.6 2.3 2.3.2 2.3.3 2.3.4 2.4 2.5 2.6 2.7 2.8 2.9 2.10 2.11 2.12 2.13 2.14 2.15 2.16 2.17 2.18 2.22 2.23 2.24 2.25 2.26 2.27 2.28 +GLIBCXX 3.4 3.4.1 3.4.2 3.4.3 3.4.4 3.4.5 3.4.6 3.4.7 3.4.8 3.4.9 3.4.10 3.4.11 3.4.12 3.4.13 3.4.14 3.4.15 3.4.16 3.4.17 3.4.18 3.4.19 3.4.20 3.4.21 3.4.22 3.4.23 3.4.24 +CXXABI 1.3 1.3.1 1.3.2 1.3.3 1.3.4 1.3.5 1.3.6 1.3.7 1.3.8 1.3.9 1.3.10 1.3.11 FLOAT128 TM_1 +GCC 3.0 3.3 3.3.1 3.4 3.4.2 3.4.4 4.0.0 4.2.0 4.3.0 4.7.0 4.8.0 7.0.0 +LIBATOMIC 1.0 1.1 1.2 +ZLIB 1.2.0 1.2.0.2 1.2.0.8 1.2.2 1.2.3.3 1.2.3.4 1.2.3.5 1.2.5.1 1.2.5.2 1.2.7.1 1.2.9 +POLICY + ;; + manylinux_2_35_aarch64) + cat <<'POLICY' +GLIBC 2.0 2.17 2.18 2.22 2.23 2.24 2.25 2.26 2.27 2.28 2.29 2.30 2.31 2.32 2.33 2.34 2.35 +GLIBCXX 3.4 3.4.1 3.4.2 3.4.3 3.4.4 3.4.5 3.4.6 3.4.7 3.4.8 3.4.9 3.4.10 3.4.11 3.4.12 3.4.13 3.4.14 3.4.15 3.4.16 3.4.17 3.4.18 3.4.19 3.4.20 3.4.21 3.4.22 3.4.23 3.4.24 3.4.25 3.4.26 3.4.27 3.4.28 3.4.29 3.4.30 +CXXABI 1.3 1.3.1 1.3.2 1.3.3 1.3.4 1.3.5 1.3.6 1.3.7 1.3.8 1.3.9 1.3.10 1.3.11 1.3.12 1.3.13 TM_1 +GCC 3.0 3.3 3.3.1 3.4 3.4.2 3.4.4 4.0.0 4.2.0 4.3.0 4.5.0 4.7.0 7.0.0 11.0 +LIBATOMIC 1.0 1.1 1.2 +ZLIB 1.2.0 1.2.0.2 1.2.0.8 1.2.2 1.2.3.3 1.2.3.4 1.2.3.5 1.2.5.1 1.2.5.2 1.2.7.1 1.2.9 +POLICY + ;; + esac | awk '{ for (i = 2; i <= NF; i++) print $1 "_" $i }' +} + +versions() { + printf '%s\n' "$1" | + grep -oE '(GLIBCXX|CXXABI|GLIBC|GCC|LIBATOMIC|ZLIB)_[A-Za-z0-9_.]+' | sort -u +} + +needed_entries() { + printf '%s\n' "$1" | sed -n 's/.*NEEDED.*\[\(.*\)\].*/\1/p' +} + +# Check readelf's status before parsing; a pipeline would report the parser's status instead. +needed_of() { + needed_dynamic=$("${readelf_bin}" -d "$1") || return 1 + needed_entries "${needed_dynamic}" +} + +dyn=$("${readelf_bin}" -d "${target}") || + fail "could not inspect ${target} with ${readelf_bin}" +if ! printf '%s\n' "${dyn}" | grep -qE 'NEEDED.*\[libexecutorch\.so\]'; then + fail "${target} has no DT_NEEDED on libexecutorch.so, so it would not bind to the shared backend registry" +fi +if ! printf '%s\n' "${dyn}" | grep -qE 'NEEDED.*\[libexecutorch_extension_cuda\.so\]'; then + fail "${target} has no DT_NEEDED on libexecutorch_extension_cuda.so, so it may carry a private CUDA stream implementation" +fi +# Every library this links, checked against the set it is allowed to link. Nothing enumerated these +# before, so a link nobody intended passed in silence. libpython is the one that matters most: the +# wheel is tagged for any Python 3 because the payload has no Python ABI, and linking libpython would +# make that tag a lie while the wheel still installed everywhere. +for entry in $(needed_entries "${dyn}"); do + case "${entry}" in + libexecutorch.so|libexecutorch_extension_cuda.so|libexecutorch_threadpool.so) ;; + libnvinfer.so.*|libcudart.so.*) ;; + libstdc++.so.*|libc.so.*|libm.so.*|libgcc_s.so.*|libdl.so.*|librt.so.*|libpthread.so.*) ;; + ld-linux-*.so.*|libatomic.so.*|libz.so.*) ;; + *) fail "${target} links ${entry}, which is not one of the libraries this wheel may depend on" ;; + esac +done + +if ! printf '%s\n' "${dyn}" | grep -qE 'NEEDED.*\[libstdc\+\+\.so\.[0-9]+\]'; then + fail "${target} has no DT_NEEDED on libstdc++, so it is not linked against the shared C++ runtime" +fi + +runpath=$(printf '%s\n' "${dyn}" | sed -n '/RUNPATH/s/.*\[\(.*\)\].*/\1/p') +# GNU/LLVM print parenthesized tags; elfutils prints bare tags. +if printf '%s\n' "${dyn}" | grep -qE '[[:space:]]\(?RPATH\)?[[:space:]]'; then + fail "${target} carries DT_RPATH rather than DT_RUNPATH" +fi +[ -n "${runpath}" ] || fail "${target} carries no RUNPATH" + +cuda_needed=$(needed_entries "${dyn}" | grep -E '^libcudart\.so\.' | sort -u) +if [ -n "${cuda_needed}" ]; then + case "${cuda_needed}" in + libcudart.so.13) ;; + *) fail "${target} needs ${cuda_needed}, but this delegate requires CUDA 13" ;; + esac + case ":${runpath}:" in + *':$ORIGIN/../../nvidia/cu13/lib:'*) ;; + *) fail "${target} needs ${cuda_needed}, but the RUNPATH carries no nvidia/cu13/lib" ;; + esac +fi + +if [ "$#" -ge 4 ]; then + if [ "${runpath}" != "${expected_runpath}" ]; then + fail "${target} carries a RUNPATH the build did not ask for: + expected: ${expected_runpath} + actual: ${runpath}" + fi +elif ! printf '%s\n' "${runpath}" | tr ':' '\n' | grep -Fxq '$ORIGIN/../../executorch/lib'; then + fail "${target} has a RUNPATH but not \$ORIGIN/../../executorch/lib" +fi +absolute=$(printf '%s\n' "${runpath}" | tr ':' '\n' | grep -v '^\$ORIGIN\(/\|$\)' || true) +if [ -n "${absolute}" ]; then + fail "${target} carries RUNPATH entries that are not relative to the artifact: +${absolute}" +fi + +# Unversioned C++ helpers evade the version-node check and must come from libstdc++_nonshared.a. +dyn_syms=$("${readelf_bin}" --dyn-syms -W "${target}") || + fail "could not read the dynamic symbols of ${target} with ${readelf_bin}" +# Undefined and unversioned is the whole condition; the symbol's type is not part of it. Selecting +# FUNC and OBJECT dropped the TLS entries that real libraries do carry, and would drop NOTYPE from +# any toolchain that emits it, which are exactly the symbols being hunted. +unversioned_cxx=$(printf '%s\n' "${dyn_syms}" | + awk '($7 == "UND" || $7 == "UNDEF") && $8 !~ /@/ && $8 ~ /^(_ZNSt|_ZNKSt|_ZSt|_ZTVNSt|_ZTINSt|_ZN9__gnu_cxx|_ZTVN9__gnu_cxx|_ZTIN9__gnu_cxx)/ { print $8 }') +if [ -n "${unversioned_cxx}" ]; then + fail "${target} has unversioned undefined C++ runtime symbols: +${unversioned_cxx}" +fi + +syms=$("${readelf_bin}" -Ws "${target}") || + fail "could not read the symbols of ${target}" +register_backend='_ZN10executorch7runtime16register_backendERKNS0_7BackendE' +if ! printf '%s\n' "${syms}" | grep -q "${register_backend}"; then + fail "${target} does not reference register_backend at all, so it registers no backend" +fi +if printf '%s\n' "${syms}" | grep "${register_backend}" | grep -qvE '[[:space:]]UND(EF)?[[:space:]]'; then + fail "${target} defines register_backend instead of importing it, so it would register into a private registry" +fi + +if [ "$#" -ge 3 ]; then + [ -f "${runtime}" ] || fail "cannot compare symbol versions: ${runtime} does not exist" + target_versions=$("${readelf_bin}" -V "${target}") || + fail "could not read symbol versions of ${target} with ${readelf_bin}" + if ! versions "${target_versions}" | grep -q '^CXXABI_[0-9]'; then + fail "${target} declares no CXXABI requirement, so it is under-linked or symbol versions could not be read" + fi + + runtime_syms=$("${readelf_bin}" -Ws "${runtime}") || + fail "cannot read the symbol table of ${runtime}" + if ! printf '%s\n' "${runtime_syms}" | + grep -qE "(GLOBAL|WEAK)[[:space:]]+DEFAULT[[:space:]]+[0-9]+[[:space:]]+${register_backend}$"; then + fail "${runtime} does not export ${register_backend}, which ${target} imports" + fi + + runtime_dir=$(dirname "${runtime}") + # executorch.runtime imports portable_lib, which loads _C and its kernel/backend dependencies. + # Include that graph, but not unrelated sibling libraries that could widen the untagged check. + set +f + set -- "${runtime_dir}"/../extension/pybindings/_C.*.so + set -f + pybindings="$1" + [ -f "${pybindings}" ] || fail "could not find the pybindings extension under ${runtime_dir}/../extension/pybindings/" + target_needed=$(needed_entries "${dyn}") + pybindings_needed=$(needed_of "${pybindings}") || fail "could not read dependencies of ${pybindings}" + worklist="$(basename "${runtime}") +${target_needed} +${pybindings_needed}" + closure="" + runtime_versions="" + while [ -n "${worklist}" ]; do + name=$(printf '%s\n' "${worklist}" | head -1) + worklist=$(printf '%s\n' "${worklist}" | tail -n +2) + [ -n "${name}" ] || continue + case " ${closure} " in + *" ${name} "*) continue ;; + esac + closure="${closure} ${name}" + sibling="${runtime_dir}/${name}" + [ -f "${sibling}" ] || continue + [ "${sibling}" = "${target}" ] && continue + sibling_needed=$(needed_of "${sibling}") || fail "could not read dependencies of ${sibling}" + sibling_versions=$("${readelf_bin}" -V "${sibling}") || fail "could not read symbol versions of ${sibling}" + worklist="${worklist} +${sibling_needed}" + runtime_versions="${runtime_versions} +${sibling_versions}" + done + + if ! printf '%s\n' "${runtime_versions}" | grep -q '[^[:space:]]'; then + fail "could not read symbol versions beside ${runtime} with ${readelf_bin}" + fi + + if [ -n "${manylinux_tag}" ]; then + allowed=$(policy_versions) + # Collect every violation before failing, and sort them by version rather than as text, so the + # message names the highest one. Failing on whichever happened to sort first left the worst + # requirement hidden behind a lesser one and sent a reader after the wrong cause. + disallowed="" + for node in $(versions "${target_versions}"); do + printf '%s\n' "${allowed}" | grep -Fxq "${node}" || + disallowed="${disallowed}${node} +" + done + if printf '%s' "${disallowed}" | grep -q '[^[:space:]]'; then + worst=$(printf '%s\n' "${disallowed}" | grep '[^[:space:]]' | sort -t_ -k2 -V | tail -1) + # Name the symbols that need it. Without this the message says a version is too new but + # not which code reached for it, which is the only part a reader can act on. + culprits=$("${readelf_bin}" -sW "${target}" 2>/dev/null | + awk -v v="@${worst}" '$0 ~ v { print $8 }' | + sed "s/@@*${worst}//" | sort -u | head -3 | tr '\n' ' ') + fail "${target} requires ${worst}, which ${manylinux_tag} does not allow (auditwheel 6.8.2). +Symbols needing it: ${culprits:-unknown}. +All disallowed: $(printf '%s' "${disallowed}" | grep '[^[:space:]]' | sort -t_ -k2 -V | tr '\n' ' ')" + fi + else + # Without a platform tag, retain the conservative named-node comparison only. + for node in $(versions "${target_versions}" | grep -E '_[A-Z][A-Z0-9_]*$'); do + versions "${runtime_versions}" | grep -Fxq "${node}" || + fail "${target} requires symbol versions absent from the runtime dependency closure: ${node}" + done + fi +fi + +exit 0 diff --git a/py/torch-tensorrt-executorch-runtime/native/check_shared_cxx_runtime.sh b/py/torch-tensorrt-executorch-runtime/native/check_shared_cxx_runtime.sh deleted file mode 100755 index a84790ba46..0000000000 --- a/py/torch-tensorrt-executorch-runtime/native/check_shared_cxx_runtime.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/sh -# Verify a shipped shared object shares the process C++ runtime instead of carrying -# one of its own. -# -# These artifacts are loaded into a process that has already loaded libtorch and -# TensorRT, which both bring libstdc++.so.6. A second copy here does not isolate -# anything: libaoti_cuda_shims.so is the first DT_NEEDED of _portable_lib.so, so it -# leads the dlopen group in symbol search order and every object loaded with it -# resolves the C++ runtime against this wheel rather than against libstdc++.so.6. -# Two libstdc++ builds then share one process, and a locale facet built by one gets -# indexed with the other's std::locale::id, which lands a virtual call on the wrong -# slot. -# -# This wheel has no auditwheel step behind it, so nothing else notices. A failure -# prints the NEEDED entries and the link command, because the artifact property on -# its own says nothing about which input produced it. -# -# Usage: check_shared_cxx_runtime.sh [link-command-file] - -set -u - -readelf_bin="$1" -target="$2" -link_txt="${3:-}" - -fail() { - echo "FATAL: $*" >&2 - echo "--- NEEDED entries of ${target} ---" >&2 - "${readelf_bin}" -d "${target}" 2>&1 | grep NEEDED >&2 || - echo "(none, or readelf could not read it)" >&2 - if [ -n "${link_txt}" ] && [ -f "${link_txt}" ]; then - echo "--- link command ---" >&2 - cat "${link_txt}" >&2 - else - echo "--- link command unavailable (${link_txt:-no path given}) ---" >&2 - fi - exit 1 -} - -dyn=$("${readelf_bin}" -d "${target}") || - fail "could not inspect ${target} with ${readelf_bin}" -syms=$("${readelf_bin}" -WsD "${target}") || - fail "could not read dynamic symbols of ${target}" - -# Defined, not undefined, and only symbols that libstdc++ alone implements. A plain -# _ZNSt or _ZSt prefix would reject a good artifact: every C++ shared object exports -# weak instantiations of std:: templates from its own translation units, and those are -# identical wherever they come from. The three families below are not. They are -# emitted only by libstdc++'s own translation units, so a definition here means a -# whole second runtime came in through libstdc++.a. -defined=$(printf %s\\n "${syms}" | - awk '($5 == "GLOBAL" || $5 == "WEAK") && $7 != "UND" { print $8 }' | - sed 's/@.*//' | - grep -E '^(__cxa_(throw|rethrow|begin_catch|end_catch|allocate_exception|free_exception)$|_ZTVN10__cxxabiv1|_ZNS[tK]?6locale)') -if [ -n "${defined}" ]; then - echo "--- libstdc++ symbols defined by ${target} ---" >&2 - printf %s\\n "${defined}" | head -20 >&2 - fail "${target} defines $(printf %s\\n "${defined}" | wc -l) libstdc++ symbols of its own" -fi - -# The other half, and the original failure this guard was written for. Undefined -# runtime symbols are correct and expected once the runtime is shared, but only if -# something declares where they come from. Without the NEEDED entry the extension -# fails to import on a missing exception_ptr::_M_addref. -if printf %s\\n "${syms}" | grep -qE 'UND +(_ZNS[tK]|_ZS[tT]|__cxa_|_ZNKS[tK])' && - ! printf %s "${dyn}" | grep -qE 'NEEDED.*libstdc\+\+'; then - fail "${target} references the C++ runtime but has no libstdc++ NEEDED entry" -fi - -exit 0 diff --git a/py/torch-tensorrt-executorch-runtime/setup.py b/py/torch-tensorrt-executorch-runtime/setup.py index 256e3c20e8..1c2e7237c0 100644 --- a/py/torch-tensorrt-executorch-runtime/setup.py +++ b/py/torch-tensorrt-executorch-runtime/setup.py @@ -5,9 +5,11 @@ from __future__ import annotations +import importlib.metadata import os import pathlib import platform +import re import shlex import shutil import subprocess @@ -16,13 +18,96 @@ import torch import yaml -from setuptools import Extension, find_packages, setup -from setuptools.command.build_ext import build_ext +from setuptools import Distribution, find_packages, setup +from setuptools.command.build_py import build_py + +try: + # setuptools >= 70.1 vends the command; older toolchains still import it from wheel. + from setuptools.command.bdist_wheel import bdist_wheel +except ImportError: # pragma: no cover - depends on the build toolchain version + from wheel.bdist_wheel import bdist_wheel HERE = pathlib.Path(__file__).resolve().parent REPO_ROOT = HERE.parents[1] BAZEL_TARGET = "//py/torch-tensorrt-executorch-runtime/native:delegate_native" BUILD_NONCE = os.getenv("TORCH_TENSORRT_EXECUTORCH_BUILD_NONCE", uuid.uuid4().hex) +CUDA_RUNTIME_DISTRIBUTION = "nvidia-cuda-runtime" +# Named the way ExecuTorch names its own delegates, because that is what this now is. The wheel +# ships this exact filename: a consumer looking for a delegate beside ExecuTorch's own +# libexecutorch_backend_cuda.so finds the same shape here. +DELEGATE_LIBRARY = "libexecutorch_backend_tensorrt.so" +# Checked in rather than generated: it has no build-time inputs. Only the companion version file is +# written at build time, because the version is not known until then. +_CMAKE_CONFIG_SOURCE = HERE / "cmake" / "torchtrt_executorch-config.cmake" + + +def pinned_executorch_version() -> str: + """Read the ExecuTorch version the repository pins. + + Raises when the file is absent rather than returning an empty version, because the caller + compares against whatever comes back and a build with no pin to compare against is exactly the + case that comparison exists for. + """ + pin_file = REPO_ROOT / "dev_dep_versions.yml" + if not pin_file.is_file(): + raise RuntimeError( + f"{pin_file} is missing, so the ExecuTorch pin cannot be checked. Build this wheel " + "from a repository checkout." + ) + return yaml.safe_load(pin_file.read_text(encoding="utf-8"))[ + "__executorch_version__" + ] + + +def executorch_cmake_prefix_path() -> str: + """Locate the CMake package of the ExecuTorch wheel this delegate builds against. + + The delegate links the runtime out of the installed wheel, so the wheel that is present + while building is the one it becomes compatible with. Both the path and the version below + come from one ``importlib.metadata`` distribution, not from ``executorch.__path__[0]``: + ``executorch`` is a namespace package, so any directory on ``sys.path`` holding an + ``executorch/`` subdirectory prepends a root, and index 0 could then name a source tree while + the version check validated the installed wheel. The compiler and the check have to be looking + at the same thing for either to mean anything. + """ + distribution = importlib.metadata.distribution("executorch") + package_root = pathlib.Path(str(distribution.locate_file("executorch"))) + if not package_root.is_dir(): + raise RuntimeError( + f"The executorch distribution reports its package at {package_root}, which is not " + "a directory. Reinstall ExecuTorch from the pinned nightly CUDA channel." + ) + prefix = package_root / "share" / "cmake" + if not (prefix / "executorch-config.cmake").is_file(): + raise RuntimeError( + f"The installed ExecuTorch at {package_root} ships no CMake package, so the " + "delegate cannot be configured against it. Install a wheel from the pinned " + "nightly CUDA channel." + ) + # The version too, not just the path. install_requires below names whatever is installed, so + # building against the wrong wheel produced a coherent-looking artifact: the delegate links + # that runtime, the ELF guard compares it against that same runtime, and the metadata requires + # it -- all three agreeing on a runtime the repository does not pin. Local editable builds are + # exempt via the escape hatch, because contributors legitimately test against other trees. + pinned = pinned_executorch_version() + installed = public_version(distribution.version) + # The label matters as much as the version. Comparing only the public parts accepted a + # processor-only build of the pinned date, which cannot supply the CUDA runtime the delegate + # links, and the wheel this build then publishes requires the label it did not check. + label = distribution.version.partition("+")[2] + wrong_version = public_version(pinned) != installed + wrong_build = not label.startswith("cu") + if (wrong_version or wrong_build) and os.getenv( + "TORCH_TENSORRT_ALLOW_UNPINNED_EXECUTORCH", "" + ).lower() not in ("1", "true", "yes", "on"): + raise RuntimeError( + f"The installed ExecuTorch is {distribution.version} but dev_dep_versions.yml pins " + f"{pinned} and the delegate needs a CUDA build. The delegate links this wheel's " + "runtime and declares a dependency on it, so building against another version or " + "another build ships a wheel that requires the wrong ExecuTorch. Install the pinned " + "CUDA wheel, or set TORCH_TENSORRT_ALLOW_UNPINNED_EXECUTORCH=1 to build anyway." + ) + return str(prefix) def get_runtime_version() -> str: @@ -41,42 +126,102 @@ def get_runtime_version() -> str: return f"{base_version}.dev0+{revision}" -RUNTIME_VERSION = get_runtime_version() +def public_version(version: str) -> str: + """Drop a PEP 440 local suffix that may not be present on package indexes.""" + return version.partition("+")[0] -TORCH_REQUIREMENT = "torch>=2.15.0.dev0,<2.16.0" -EXECUTORCH_REQUIREMENT = f'executorch=={yaml.safe_load((REPO_ROOT / "dev_dep_versions.yml").read_text())["__executorch_version__"]}' -TORCH_TENSORRT_REQUIREMENT = "torch-tensorrt>=2.15.0.dev0,<2.16.0" + +def installed_version(distribution: str) -> str: + """Return the version of a dependency in the native build environment.""" + try: + return importlib.metadata.version(distribution) + except importlib.metadata.PackageNotFoundError as error: + raise RuntimeError( + f"{distribution} must be installed to build the ExecuTorch runtime wheel" + ) from error -def get_tensorrt_requirement() -> str: +def require_supported_cuda() -> None: + """Require the CUDA major used by this delegate's dependencies and library paths.""" cuda_version = torch.version.cuda - if cuda_version is None: + if (cuda_version or "").split(".")[0] != "13": raise RuntimeError( - "CUDA enabled PyTorch is required to build this wheel found None" + "PyTorch built against CUDA 13 is required to build this wheel " + f"(found CUDA {cuda_version or 'None'})" ) - if cuda_version.startswith("13."): - return "tensorrt-cu13>=11.3.0,<11.4" - raise RuntimeError(f"Unsupported CUDA version: {cuda_version}") -class BazelExtension(Extension): - def __init__(self, name: str) -> None: - super().__init__(name, sources=[]) +class BazelBuild(build_py): + """Build the delegate with Bazel and place it in the package under its real name. + + Not a ``build_ext``/``Extension``: the delegate exports no ``PyInit_``, references no + Python C-API symbol, and links no libpython; it is a plain shared library that ctypes + loads. Declaring it an extension made setuptools rename it to + ``_executorch_backend_tensorrt..so``, which both hides that it is an ExecuTorch + delegate and implies a Python ABI it does not have. The platform tag the extension was + buying is set directly instead: ``Distribution.has_ext_modules`` keeps the wheel + non-pure, and ``WheelTag`` below sets the interpreter and ABI to py3/none. + """ + + def _generated_output_mapping(self) -> dict[str, str]: + package = "torch_tensorrt_executorch_runtime" + return { + str(pathlib.Path(self.build_lib) / package / filename): str( + pathlib.Path(self.get_package_dir(package)) / filename + ) + for filename in ( + f"lib/{DELEGATE_LIBRARY}", + "lib/cmake/torchtrt_executorch/torchtrt_executorch-config.cmake", + "lib/cmake/torchtrt_executorch/torchtrt_executorch-config-version.cmake", + ) + } + + def get_outputs(self, include_bytecode: bool = True) -> list[str]: + return list( + dict.fromkeys( + [ + *super().get_outputs(include_bytecode), + *self._generated_output_mapping(), + ] + ) + ) + + def get_output_mapping(self) -> dict[str, str]: + mapping = super().get_output_mapping() + if self.editable_mode: + mapping.update(self._generated_output_mapping()) + return mapping + def run(self) -> None: + # During an editable install setuptools routes a customized build_py through its own + # _safely_run, which catches Exception and turns it into a warning pip hides, so a failed + # native build reports "Successfully installed" with no delegate. SystemExit is not an + # Exception, so it is the one thing that escapes. Everything this build raises has to become + # one, including the RuntimeError from a missing bazel, which is the case that motivated + # this: re-raising it unchanged left it inside the class setuptools swallows. + try: + self._build() + except SystemExit: + raise + except BaseException as error: + raise SystemExit(f"ExecuTorch delegate build failed: {error}") from error + + def _build(self) -> None: + super().run() -class BazelBuild(build_ext): - def build_extension(self, ext: Extension) -> None: if sys.platform != "linux": raise RuntimeError("The ExecuTorch TensorRT delegate supports Linux only") - output = pathlib.Path(self.get_ext_fullpath(ext.name)).resolve() - output.parent.mkdir(parents=True, exist_ok=True) - bazel = shutil.which("bazelisk") or shutil.which("bazel") if bazel is None: raise RuntimeError("Could not find bazelisk or bazel in PATH") - compilation_mode = "dbg" if self.debug else "opt" + compilation_mode = ( + "dbg" + if os.getenv("TORCH_TENSORRT_EXECUTORCH_DEBUG", "").lower() + in ("1", "true", "yes", "on") + else "opt" + ) command = [ bazel, "build", @@ -85,17 +230,9 @@ def build_extension(self, ext: Extension) -> None: "--config=python", f"--compilation_mode={compilation_mode}", f"--action_env=PYTHON_BIN_PATH={sys.executable}", + f"--action_env=EXECUTORCH_CMAKE_PREFIX_PATH={executorch_cmake_prefix_path()}", f"--action_env=TORCH_TENSORRT_EXECUTORCH_BUILD_NONCE={BUILD_NONCE}", ] - # Bazel actions run with a restricted environment. In particular, - # rules_foreign_cc's CMake invocation does not inherit this process's - # TORCH_CUDA_ARCH_LIST unless it is made an action environment value. - # Without it, ExecuTorch's CMake falls back to its common architecture - # list (including compute_50), which CUDA 13 rejects. - cuda_arch_list = os.getenv("TORCH_CUDA_ARCH_LIST") - if cuda_arch_list: - print(f"Forwarding TORCH_CUDA_ARCH_LIST to Bazel actions: {cuda_arch_list}") - command.append(f"--action_env=TORCH_CUDA_ARCH_LIST={cuda_arch_list}") dist_dir_arch = ( "aarch64-linux-gnu" if platform.machine() in {"aarch64", "arm64"} @@ -123,48 +260,173 @@ def build_extension(self, ext: Extension) -> None: text=True, ).strip() ) - library_stem = ( - "_portable_lib" if ext.name.endswith("._portable_lib") else "data_loader" - ) built = ( bazel_bin / "py/torch-tensorrt-executorch-runtime/native/delegate_native/lib" - / f"{library_stem}.so" + / DELEGATE_LIBRARY ) if not built.is_file(): raise RuntimeError(f"Bazel did not produce {built}") - output.unlink(missing_ok=True) + + package = "torch_tensorrt_executorch_runtime" + # Editable build_lib is temporary; generated package data must survive its removal. + package_root = ( + pathlib.Path(self.get_package_dir(package)) + if self.editable_mode + else pathlib.Path(self.build_lib) / package + ) + output = package_root / "lib" / DELEGATE_LIBRARY + output.parent.mkdir(parents=True, exist_ok=True) + # Every stale artifact, not just a shared object in lib/. An incremental build over a tree + # that once produced the bundled ExecuTorch runtime leaves those files under build_lib, and + # build_py copies that directory into the wheel wholesale, so a local rebuild would + # republish exactly the runtime and the Python API this package no longer ships. + # package_data names one filename and would not pull them in; the staleness is in the build + # tree, not the manifest. CI never sees it, building from a fresh checkout. + # + # The package ROOT as well as lib/: the delegate moved into lib/, but the files this + # removes were written to the root by the previous layout, so scanning only the new + # directory leaves every one of them in place. + if not self.editable_mode: + # Shared objects only. runtime.py used to be removed here too, from when this package + # stopped shipping a Python API, and it is shipped again now as the forwarder the + # released main wheel imports by name. Removing it from the build output published a + # wheel without it, so that import failed for anyone pairing an older main wheel with + # this companion. + for stale in package_root.glob("*.so*"): + if stale.is_file(): + stale.unlink() + for stale in output.parent.glob("*.so*"): + if stale != output: + stale.unlink() shutil.copy2(built, output) + self._install_cmake_package(output.parent.parent) + + def _install_cmake_package(self, package_dir: pathlib.Path) -> None: + """Ship a CMake package so a C++ app can link the delegate out of the wheel. + + ExecuTorch ships its backends as prebuilt shared libraries plus a CMake package, so a C++ + app links ``executorch::backend_cuda`` and the backend registers itself. Without an + equivalent here the delegate is reachable only from Python, even though the shared library + in this wheel is a drop-in sibling of ExecuTorch's own backends. + + The config is checked in; only the version file is generated, because the version is not + known until the wheel is built. + """ + cmake_dir = package_dir / "lib" / "cmake" / "torchtrt_executorch" + cmake_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(_CMAKE_CONFIG_SOURCE, cmake_dir / _CMAKE_CONFIG_SOURCE.name) + version = self.distribution.get_version() + # find_package compares dotted integers, so a dev suffix has to be dropped from the value + # it reads while the full version stays visible for a human. + numeric = re.match(r"[0-9]+(?:\.[0-9]+)*", version) + public = numeric.group(0) if numeric else version + # find_package never exposes the INSTALLED version's major, only the requested + # one, so it is baked in here where it is known. + major = public.split(".")[0] + # SameMajorVersion bounds both endpoints, allowing only an exclusive next-major boundary. + (cmake_dir / "torchtrt_executorch-config-version.cmake").write_text( + "# Generated by setup.py. The version is only known when the wheel is built.\n" + f'set(PACKAGE_VERSION "{public}")\n' + f'set(TORCHTRT_EXECUTORCH_FULL_VERSION "{version}")\n' + "\n" + f'if(PACKAGE_FIND_VERSION_MAJOR STREQUAL "{major}")\n' + " if(PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION)\n" + " set(PACKAGE_VERSION_COMPATIBLE FALSE)\n" + " else()\n" + " set(PACKAGE_VERSION_COMPATIBLE TRUE)\n" + " endif()\n" + "else()\n" + " set(PACKAGE_VERSION_COMPATIBLE FALSE)\n" + "endif()\n" + "\n" + "if(PACKAGE_FIND_VERSION STREQUAL PACKAGE_VERSION)\n" + " set(PACKAGE_VERSION_EXACT TRUE)\n" + "endif()\n" + "\n" + "# A version RANGE, find_package(pkg 2.14...<2.15). Without this the upper bound is\n" + "# silently ignored and the range behaves like its lower bound alone.\n" + "if(PACKAGE_FIND_VERSION_RANGE)\n" + ' if(PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "INCLUDE"\n' + f' AND NOT PACKAGE_FIND_VERSION_MAX_MAJOR STREQUAL "{major}")\n' + " set(PACKAGE_VERSION_COMPATIBLE FALSE)\n" + ' elseif(PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "EXCLUDE"\n' + f' AND PACKAGE_FIND_VERSION_MAX VERSION_GREATER "{int(major) + 1}")\n' + " set(PACKAGE_VERSION_COMPATIBLE FALSE)\n" + " elseif(PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION_MIN)\n" + " set(PACKAGE_VERSION_COMPATIBLE FALSE)\n" + ' elseif(PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "INCLUDE"\n' + " AND PACKAGE_VERSION VERSION_GREATER PACKAGE_FIND_VERSION_MAX)\n" + " set(PACKAGE_VERSION_COMPATIBLE FALSE)\n" + ' elseif(PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "EXCLUDE"\n' + " AND NOT PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION_MAX)\n" + " set(PACKAGE_VERSION_COMPATIBLE FALSE)\n" + " endif()\n" + "endif()\n", + encoding="utf-8", + ) - # Ship the two shared libraries the extensions load. ExecuTorch's published - # wheel provides neither, so without them importing the runtime fails on a - # missing shared object. Copied rather than declared as extensions because they - # are plain dependencies, not Python modules. - for dependency in ("libextension_cuda.so", "libaoti_cuda_shims.so"): - source = built.parent / dependency - if not source.is_file(): - raise RuntimeError(f"Bazel did not produce {source}") - destination = output.parent / dependency - destination.unlink(missing_ok=True) - shutil.copy2(source, destination) +TENSORRT_DISTRIBUTION = "tensorrt-cu13" + +class WheelTag(bdist_wheel): + """Tag the wheel py3-none-, not cp3XX-cp3XX-. + + The payload is one ctypes-loaded shared library with no Python ABI, so it is byte for byte + identical across CPython versions and only the platform matters. has_ext_modules keeps + Root-Is-Purelib false and the platform tag; this drops the per-interpreter half of the tag + so one built wheel serves every CPython instead of one identical copy per version. + """ + + def get_tag(self) -> tuple[str, str, str]: + _, _, plat = super().get_tag() + return "py3", "none", plat + + +class PlatformDistribution(Distribution): + """Marks the wheel platform-specific even though it declares no extension module. + + The delegate is a compiled object, x86-64 or aarch64, so a pure-Python tag would be + wrong. This is what ``ext_modules`` used to provide. + """ + + def has_ext_modules(self) -> bool: + return True + + +# These run while the file is read, and they have to. They produce install_requires, and dependency +# metadata is exactly what a metadata-only build asks for, so deriving pins from the build +# environment means that environment has to be present. The CUDA check leads, so an unsupported +# CUDA says so before anything else is attempted rather than failing later and less clearly. +require_supported_cuda() +executorch_version = installed_version("executorch") +tensorrt_version = installed_version(TENSORRT_DISTRIBUTION) +cuda_runtime_version = installed_version(CUDA_RUNTIME_DISTRIBUTION) setup( name="torch-tensorrt-executorch-runtime", - version=RUNTIME_VERSION, + version=get_runtime_version(), description="Torch-TensorRT delegate for the ExecuTorch Python runtime", packages=find_packages(), - ext_modules=[ - BazelExtension("torch_tensorrt_executorch_runtime._portable_lib"), - BazelExtension("torch_tensorrt_executorch_runtime.data_loader"), - ], - cmdclass={"build_ext": BazelBuild}, + distclass=PlatformDistribution, + package_data={ + "torch_tensorrt_executorch_runtime": [ + f"lib/{DELEGATE_LIBRARY}", + "lib/cmake/torchtrt_executorch/*.cmake", + ] + }, + cmdclass={"build_py": BazelBuild, "bdist_wheel": WheelTag}, python_requires=">=3.10", install_requires=[ - TORCH_REQUIREMENT, - EXECUTORCH_REQUIREMENT, - TORCH_TENSORRT_REQUIREMENT, - get_tensorrt_requirement(), + # Full versions, local label included, for the three the delegate is compiled against. The + # label is what names the CUDA build, and dropping it leaves a requirement that a processor + # build or a different CUDA build of the same date satisfies just as well. This delegate links + # those runtimes out of one specific build, so those are exactly the pairings to refuse. + f"torch=={torch.__version__}", + f"executorch=={executorch_version}", + f"torch-tensorrt=={installed_version('torch-tensorrt')}", + f"{TENSORRT_DISTRIBUTION}=={public_version(tensorrt_version)}", + f"{CUDA_RUNTIME_DISTRIBUTION}=={public_version(cuda_runtime_version)}", ], zip_safe=False, ) diff --git a/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/__init__.py b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/__init__.py index c933824391..41dd483dcb 100644 --- a/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/__init__.py +++ b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/__init__.py @@ -1,111 +1,345 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: BSD-3-Clause +"""Register the Torch-TensorRT delegate with the installed ExecuTorch runtime. -"""Activate the TensorRT delegate-enabled ExecuTorch Python runtime.""" +Importing this package is all it takes. ExecuTorch's own delegates register because they are +linked into its pybindings extension, so loading that extension pulls them in and their static +initializers run. A delegate shipped in a separate wheel cannot join that link, and ExecuTorch has +no discovery hook for out-of-tree backends, so this package performs the equivalent step itself at +import time. + +There is deliberately no runtime API here. Once the backend is registered, everything else belongs +to ExecuTorch: + + import torch_tensorrt_executorch_runtime # noqa: F401 + from executorch.extension.pybindings.portable_lib import _load_for_executorch + + program = _load_for_executorch("model.pte") + outputs = program.run_method("forward", (tensor,)) + +Set ``TORCH_TENSORRT_SKIP_DELEGATE_REGISTRATION=1`` to import the module without loading the +delegate. That is for tooling that wants the metadata only; a normal consumer never needs it. +""" from __future__ import annotations import ctypes -import importlib -import importlib.util import os -import sys +import threading +import warnings from types import ModuleType -from typing import Any, Protocol, cast +from typing import Any BACKEND_NAME = "TensorRTBackend" -_NATIVE_NAME = "executorch.extension.pybindings._C" -# The extension was called _portable_lib before ExecuTorch renamed it to _C, and portable_lib.py -# imports whichever name its own version uses. Both are claimed so the interception works against -# either: aliasing only the old name is silently ineffective at the pinned nightly, because nothing -# imports it any more and the stock _C loads instead, leaving TensorRTBackend unregistered. -_LEGACY_NATIVE_NAME = "executorch.extension.pybindings._portable_lib" -_WRAPPER_NAME = "executorch.extension.pybindings.portable_lib" -_DATA_LOADER_NAME = "executorch.extension.pybindings.data_loader" +# The same name ExecuTorch gives its own delegates, and the exact filename the wheel ships. +_DELEGATE_LIBRARY = "libexecutorch_backend_tensorrt.so" +_delegate: ctypes.CDLL | None = None +# Registration happens in the delegate's static initializer, so it takes effect inside dlopen, +# before ctypes.CDLL returns and before _delegate is assigned. Every check below therefore has to +# sit inside one critical section: a second thread that squeezed between the load and the +# assignment would see the backend registered with _delegate still None, which is exactly what a +# foreign delegate owning the name looks like. +_registration_lock = threading.Lock() -class _BackendRegistry(Protocol): - def is_available(self, name: str) -> bool: ... +class DelegateCompatibilityError(ImportError): + """The delegate could not be loaded against the installed ExecuTorch runtime.""" -class _Runtime(Protocol): - backend_registry: _BackendRegistry - def load_program(self, data: bytes) -> Any: ... +_EXTENSION_CUDA_LIBRARY = "libexecutorch_extension_cuda.so" -class DelegateCompatibilityError(ImportError): - """The runtime wheel is incompatible with the active native runtime.""" +def _extension_cuda_present() -> bool: + """Whether the installed ExecuTorch actually ships the CUDA extension. + + Resolved from the imported package rather than a hardcoded path, so it follows the + distribution the loader would have used. ``__file__`` as well as ``__path__``, because a + namespace-style or synthesised module may carry only one of them, and treating "no location + at all" as "the file is missing" would send a user with a working CUDA wheel off to + reinstall it. + """ + try: + import executorch + except ImportError: + return False + roots = list(getattr(executorch, "__path__", None) or ()) + location = getattr(executorch, "__file__", None) + if location: + roots.append(os.path.dirname(os.path.abspath(location))) + return any( + os.path.isfile(os.path.join(root, "lib", _EXTENSION_CUDA_LIBRARY)) + for root in roots + ) -def _probe_portable_lib_dependencies() -> None: - """Fail before importing data_loader if _portable_lib dependencies are missing.""" - spec = importlib.util.find_spec(__name__ + "._portable_lib") - if spec is None or spec.origin is None: - raise ImportError("Could not find the prebuilt ExecuTorch portable runtime") - ctypes.CDLL(spec.origin, mode=os.RTLD_LAZY | os.RTLD_LOCAL) +def _delegate_path() -> str: + # Resolved next to this file rather than through the import system, so it does not depend on the + # package being on the path anywhere else. It still runs during this package's own import, and + # cannot run before it. A fixed filename now: the delegate is shipped as + # package data under its real name, not renamed by setuptools. + # + # ``lib/`` rather than the package root, matching where ExecuTorch keeps its own backends + # (``executorch/lib/libexecutorch_backend_cuda.so`` and friends). A C++ consumer finds this + # library through the CMake package in ``lib/cmake``, which searches the same directory, so + # the two consumers agree on one location. + directory = os.path.join(os.path.dirname(os.path.abspath(__file__)), "lib") + path = os.path.join(directory, _DELEGATE_LIBRARY) + if not os.path.isfile(path): + raise DelegateCompatibilityError( + f"The Torch-TensorRT ExecuTorch delegate library is missing from {directory}. " + "This package must be installed from a wheel; a source checkout contains no " + "built delegate." + ) + return path -def activate() -> ModuleType: - """Make the delegate-enabled portable runtime back ``executorch.runtime``. +def register() -> None: + """Load the delegate so ExecuTorch can execute TensorRT-delegated programs. - The replacement includes TensorRTBackend as well as ExecuTorch XNNPACK - backend and optimized CPU kernels, so activation preserves the stock - Python runtime CPU execution capabilities. + Called once when this package is imported, so a user never has to. It stays public and named + because the import side effect is the whole point of the package and a reader needs somewhere to + look, and because a caller that imported the package defensively can re-assert registration + without reaching into a private name. + + Registration happens in the delegate's own static initializer, which calls into the backend + registry that lives in the ExecuTorch runtime. Importing ExecuTorch first is what puts that + runtime in the process; the delegate then binds to the same copy through its DT_NEEDED rather + than bringing one of its own. + + Idempotent, and safe to call after ``executorch.runtime`` has already been imported. That used + to be an error, because the delegate arrived as a substitute for ExecuTorch's own Python + extension and had to get in first. It no longer substitutes anything. + + Thread safe: the whole first registration is serialized, because it lands during ``dlopen`` + while ``_delegate`` is assigned after it, and a caller observing that gap could not tell this + package's own load from a foreign delegate holding the name. """ - # Both alias names are inspected: whichever one a given ExecuTorch version uses, an entry there - # that is not ours means the stock extension already loaded. - claimed = [ - module - for module in ( - sys.modules.get(_NATIVE_NAME), - sys.modules.get(_LEGACY_NATIVE_NAME), + if _delegate is not None: + return + + with _registration_lock: + # Re-checked under the lock: a caller that queued here while the winner was loading would + # otherwise redo the whole registration and then reject itself for the registration the + # winner just made. + if _delegate is not None: + return + _register_locked() + + +def _register_locked() -> None: + global _delegate + + try: + import executorch.extension.pybindings.portable_lib # noqa: F401 + except ImportError as error: + # "Not installed" and "installed but unloadable" need different repairs, and the second + # is what an ABI mismatch looks like: the module is found, its extension fails to load. + # Answering both with "install executorch" sends that user to reinstall what they have. + # ModuleNotFoundError covers a genuinely absent package; a bare ImportError whose message + # names no shared object is the same thing seen through a blocked sys.modules entry. An + # ABI failure, by contrast, always names the library that would not load. + # ModuleNotFoundError naming executorch itself. Testing the type alone was wrong: if an + # installed ExecuTorch fails to import because one of its own transitive dependencies is + # missing, the exception is also a ModuleNotFoundError, and its .name is that dependency. + # That user was told to install ExecuTorch, which they already have. + # Exact match, not the top-level segment: CPython sets .name to the full dotted path when a + # submodule such as executorch.extension.pybindings.portable_lib is the thing that is + # absent or blocked, and to the bare "executorch" only when the root package itself is + # missing. Splitting on "." and comparing the first segment reported a blocked submodule as + # ExecuTorch being uninstalled, which is the ABI case this branch exists to separate out. + absent = ( + isinstance(error, ModuleNotFoundError) + and (error.name or "") == "executorch" ) - if module is not None - ] - ours = __name__ + "._portable_lib" - if claimed and all(module.__name__ == ours for module in claimed): - return claimed[0] - if claimed or _WRAPPER_NAME in sys.modules: + if absent: + raise DelegateCompatibilityError( + "ExecuTorch must be installed to load the Torch-TensorRT delegate. Install " + "executorch from the same release matrix as this package. The import failed " + f"with: {error}. Importing this package registers the delegate, which is why it " + "raises here rather than later. Set " + "TORCH_TENSORRT_SKIP_DELEGATE_REGISTRATION=1 to import it without registering." + ) from error + # A library the loader could not FIND is a different problem from one it could not USE, and + # only the second is an ABI mismatch. Blaming the ABI for the first sends the reader to + # rebuild a matched stack when nothing is mismatched. + text = str(error) + missing_file = "cannot open shared object file" in text + if missing_file: + hint = ( + "so the loader could not find it, not that it is incompatible. Check that the " + "ExecuTorch wheel is installed completely." + ) + # An empty entry in the search path means the working directory, and it makes every + # origin-relative entry in this package's own search path resolve from there instead, + # so a correct installation fails to load from some directories and not others. A + # trailing separator is the usual way one appears. + search_path = os.environ.get("LD_LIBRARY_PATH") + if search_path is not None and "" in search_path.split(os.pathsep): + hint = ( + "so the loader could not find it. LD_LIBRARY_PATH has an empty entry, often " + "from a trailing separator, which the loader reads as the working directory " + "and which stops this package's own origin-relative search path resolving. " + f"Remove it and retry. Current value: {search_path!r}" + ) + raise DelegateCompatibilityError( + f"ExecuTorch is installed but a library it needs was not found, {hint} " + f"The import failed with: {error}" + ) from error + raise DelegateCompatibilityError( + "ExecuTorch is installed but its Python bindings could not be loaded, which " + "usually means it was built against a different C++ or CUDA runtime than this " + f"delegate. The import failed with: {error}" + ) from error + + path = _delegate_path() + # A preloaded library may have lost registration to another copy. Check ownership below. + if BACKEND_NAME in _registered_backend_names(): + loaded = _delegate_already_loaded(path) + if loaded is None: + raise DelegateCompatibilityError( + f"{BACKEND_NAME} is already registered before loading {path}, and that library is " + "not the one in this process, so another copy of the delegate is present. " + "ExecuTorch keeps the first registration, so the copy this package ships would " + "not be the one used. Import this package once, and do not load a second " + "Torch-TensorRT delegate alongside it." + ) + else: + try: + # Resolve imports eagerly; the ownership query needs only this local handle. + loaded = ctypes.CDLL(path, mode=os.RTLD_NOW | os.RTLD_LOCAL) + except OSError as error: + # A present CUDA extension can also fail to load because of an ABI mismatch. + if ( + "libexecutorch_extension_cuda" in str(error) + and not _extension_cuda_present() + ): + raise DelegateCompatibilityError( + f"Could not load the Torch-TensorRT ExecuTorch delegate from {path}. This " + "requires a CUDA build of executorch, which ships " + "libexecutorch_extension_cuda.so; a CPU build satisfies the version pin but " + "not this dependency. Install torch, executorch, torch-tensorrt, and this " + "package from the same release matrix." + ) from error + raise DelegateCompatibilityError( + f"Could not load the Torch-TensorRT ExecuTorch delegate from {path}: {error}. " + "The delegate links ExecuTorch's prebuilt runtime, TensorRT, and the CUDA runtime " + "from their own wheels, so install torch, executorch, torch-tensorrt, and this " + "package from the same release matrix." + ) from error + + if BACKEND_NAME not in _registered_backend_names(): raise DelegateCompatibilityError( - "ExecuTorch's stock runtime was imported first. Call " - 'torch_tensorrt.load(..., format="executorch") before importing ' - "executorch.runtime." + f"Loading {path} did not register {BACKEND_NAME} with the ExecuTorch runtime, so " + "a delegated program would fail to load. The delegate and the installed " + "ExecuTorch were probably built against different runtimes." ) - previous_data_loader = sys.modules.get(_DATA_LOADER_NAME) try: - _probe_portable_lib_dependencies() - data_loader = importlib.import_module(__name__ + ".data_loader") - # _portable_lib imports this canonical name while its module initializer - # runs. Install our binding first so Python does not load ExecuTorch's - # stock data_loader and register PyDataLoader a second time. - sys.modules[_DATA_LOADER_NAME] = data_loader - native = importlib.import_module(__name__ + "._portable_lib") - except (ImportError, OSError) as error: - if previous_data_loader is None: - sys.modules.pop(_DATA_LOADER_NAME, None) - else: - sys.modules[_DATA_LOADER_NAME] = previous_data_loader + owns_registration = loaded.torch_tensorrt_owns_executorch_registration + except AttributeError as error: raise DelegateCompatibilityError( - "Could not load the prebuilt Torch-TensorRT ExecuTorch runtime. " - "Install torch, executorch, torch-tensorrt, and the runtime package from " - "the same release matrix." + f"The delegate at {path} has no registration ownership query. " + "Reinstall this package so its Python module and native library match." ) from error - sys.modules[_NATIVE_NAME] = native - sys.modules[_LEGACY_NATIVE_NAME] = native - sys.modules.pop(_WRAPPER_NAME, None) - return native + owns_registration.argtypes = [] + owns_registration.restype = ctypes.c_bool + if not owns_registration(): + raise DelegateCompatibilityError( + f"The delegate at {path} does not own the {BACKEND_NAME} registration. " + "ExecuTorch keeps the first registration, so another library would execute " + "the delegated program. Do not load a second Torch-TensorRT delegate alongside it." + ) + _delegate = loaded + + +def _delegate_already_loaded(path: str) -> ctypes.CDLL | None: + """The handle for ``path`` if that exact library is already in this process, else ``None``. + + ``dlopen`` on an already-loaded library returns the existing handle and bumps its reference + count rather than mapping a second copy, so this cannot introduce the duplicate it is checking + for. ``RTLD_NOLOAD`` is what makes the question safe to ask: it refuses to load anything, so a + library that is not present yields ``None`` instead of being pulled in as a side effect of the + test. It is absent on some platforms, and this delegate is Linux-only, so treat a missing flag + as "cannot tell" rather than guessing. + """ + noload = getattr(os, "RTLD_NOLOAD", None) + if noload is None: + return None + try: + return ctypes.CDLL(path, mode=noload | os.RTLD_LOCAL) + except OSError: + return None + +def _registered_backend_names() -> list[str]: + # A private ExecuTorch name, and registration runs at import, so an ExecuTorch that does not + # export it would turn a plain import of this package into a bare ImportError traceback. Say + # what is wrong and what to do instead. + try: + from executorch.extension.pybindings.portable_lib import ( + _get_registered_backend_names, + ) + except ImportError as error: + raise DelegateCompatibilityError( + "The installed ExecuTorch does not expose its registered backend names, so this " + "delegate cannot confirm it owns its registration. Install the ExecuTorch build this " + f"package pins. Underlying error: {error}" + ) from error -def get_runtime() -> _Runtime: - """Return the activated ExecuTorch Runtime singleton.""" - activate() + return _get_registered_backend_names() + + +def activate() -> ModuleType: + """Deprecated: register the backend and return ExecuTorch's own portable runtime. + + The companion published before this change swapped in its own bundled ``_portable_lib`` and + returned it. This package no longer bundles one, so it registers the backend and hands back + ExecuTorch's module, which is the one that now carries the delegate. A caller that only wanted + registration is unaffected; one that used the return value gets the module it was reaching for. + """ + warnings.warn( + "activate() is deprecated; the backend registers on import. Call register() if you " + "need to register explicitly.", + DeprecationWarning, + stacklevel=2, + ) + register() + from executorch.extension.pybindings import portable_lib + + return portable_lib + + +def get_runtime() -> Any: + """Deprecated: ExecuTorch's runtime, which now owns execution for this backend. + + The companion used to return a runtime of its own. ExecuTorch's ``Runtime.get()`` is that + object now, so this forwards to it rather than failing, and a caller that asked for a runtime + still gets one that can see the registered backend. + """ + warnings.warn( + "get_runtime() is deprecated; use executorch.runtime.Runtime.get().", + DeprecationWarning, + stacklevel=2, + ) + register() from executorch.runtime import Runtime - value = cast(_Runtime, Runtime.get()) - if not value.backend_registry.is_available(BACKEND_NAME): - raise DelegateCompatibilityError(f"{BACKEND_NAME} is not registered") - return value + return Runtime.get() + +# Tooling can opt out; normal imports must fail if this delegate cannot own registration. +if os.getenv("TORCH_TENSORRT_SKIP_DELEGATE_REGISTRATION", "0").lower() not in ( + "1", + "true", + "yes", + "on", +): + register() -__all__ = ["BACKEND_NAME", "DelegateCompatibilityError", "activate", "get_runtime"] +__all__ = [ + "BACKEND_NAME", + "DelegateCompatibilityError", + "activate", + "get_runtime", + "register", +] diff --git a/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py index 01928a846a..7629ceb57e 100644 --- a/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py +++ b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py @@ -1,89 +1,65 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: BSD-3-Clause -"""Python inference API for Torch-TensorRT ExecuTorch programs.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any, Collection, Sequence, Union, cast +"""Deprecated: loading moved to ExecuTorch's own Module API. +The published Torch-TensorRT wheel imports ``load`` from this submodule by name, so removing it +would turn ``torch_tensorrt.load(..., format="executorch")`` into a ModuleNotFoundError for anyone +who upgrades this package on its own. It stays until that call is gone from a released main wheel. +""" -def _load_module(data: bytes) -> Any: - """Load a program through ExecuTorch's Module API. +from __future__ import annotations - A TensorRT delegated program is exported with device-tagged memory-planned - arenas, and ExecuTorch backs those with real device memory only through - this API. Its program loader plans every arena on the host, so the device - copy the exporter inserts around the delegate would hand ``cudaMemcpy`` a - host destination and fail with ``invalid argument``. - """ - from torch_tensorrt_executorch_runtime import activate, get_runtime +import warnings +from typing import Any - # get_runtime verifies TensorRTBackend is registered; activate returns the native module it - # installed as the process portable runtime, which is what loads the program. - get_runtime() - native = activate() - # Taken off the native module rather than imported through - # executorch.extension.pybindings.portable_lib. That wrapper's presence in sys.modules is how - # activate() detects that ExecuTorch's stock runtime was imported first, so importing it here - # would make a later activate() in the same process refuse. - return native._load_for_executorch_from_buffer(data) +__all__ = ["load", "Program"] -class Program: - """A loaded ExecuTorch program backed by TensorRTBackend. +def __getattr__(name: str) -> Any: + """Resolve Program from the main wheel on first use. - The ExecuTorch Python portable runtime executes across a CPU tensor - boundary: CUDA inputs are copied to CPU before dispatch and outputs are - returned on CPU. TensorRT still executes the delegated graph on GPU, but - the device-resident input/output fast path is available only through the - ExecuTorch C++ runner. + The published package exported this name alongside load, so code that imported it by name has to + keep working. Resolving it lazily rather than at module import keeps this module importable + against a main wheel that does not have it, which is the same reason load defers its own import. """ + if name == "Program": + from torch_tensorrt._executorch_compat import Program - def __init__(self, module: Any, data: bytes) -> None: - # ExecuTorch's BufferDataLoader references this memory without copying it. - self._data = data - self._module = module - - @property - def method_names(self) -> Collection[str]: - return cast(Collection[str], self._module.method_names()) + return Program + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - def run(self, inputs: Sequence[Any], method: str = "forward") -> Sequence[Any]: - """Run a method using CPU inputs and return CPU outputs. - CUDA tensor inputs are copied to CPU before entering the portable - Python runtime. Use the C++ runner when inputs and outputs must remain - device-resident. - """ - import torch +def load(file_path: str) -> Any: + """Deprecated: register the delegate and load through ExecuTorch. - inputs = tuple( - value.cpu() if isinstance(value, torch.Tensor) and value.is_cuda else value - for value in inputs - ) - if method not in self.method_names: - raise ValueError( - f"Unknown method {method!r}; available methods: {sorted(self.method_names)}" - ) - return cast(Sequence[Any], self._module.run_method(method, inputs)) - - def forward(self, *inputs: Any) -> Sequence[Any]: - return self.run(inputs, "forward") - - -def load(path: Union[str, Path]) -> Program: - """Load a `.pte` with the delegate-enabled ExecuTorch Python runtime. - - External `.ptd` weight files are not supported; weights must be embedded - in the `.pte` file. + Registration is what this package exists for, and ExecuTorch owns execution, so this does the + first and forwards the second rather than carrying a loader of its own. """ - model_path = Path(path) - if not model_path.is_file(): - raise FileNotFoundError(f"ExecuTorch model not found: {model_path}") - data = model_path.read_bytes() - return Program(_load_module(data), data) - - -__all__ = ["Program", "load"] + warnings.warn( + "torch_tensorrt_executorch_runtime.runtime.load() is deprecated; use " + 'torch_tensorrt.load(path, format="executorch") instead, which returns the same object.', + DeprecationWarning, + stacklevel=2, + ) + # Forward to the main wheel's loader rather than ExecuTorch's. The API this replaces returned a + # Program carrying run() and forward() and raised FileNotFoundError for a missing path, and + # ExecuTorch's own loader returns neither, so a caller of the published API would break on the + # return value instead of on the import. The main wheel is always present: this package declares + # it as a dependency. + # The loader this forwards to is part of the main wheel, and a main wheel old enough to import + # this submodule by name does not carry it. That pairing should not arise, because this package + # requires the main wheel of its own build exactly, so installing it moves the main wheel too. + # If it does arise, through an install that skipped dependency resolution, say which of the two + # is too old rather than reporting a module nobody asked for. + try: + from torch_tensorrt._executorch_compat import load as _load + except ImportError as error: + raise ImportError( + "This deprecated loader forwards into torch_tensorrt, and the installed Torch-TensorRT " + "is older than the one this package was built against, so it does not carry the " + "receiving module. Install the Torch-TensorRT this package requires, or call " + f'torch_tensorrt.load(path, format="executorch") directly. Underlying error: {error}' + ) from error + + return _load(file_path) diff --git a/py/torch_tensorrt/_compile.py b/py/torch_tensorrt/_compile.py index 3ff5e6040d..9ede03e5fb 100644 --- a/py/torch_tensorrt/_compile.py +++ b/py/torch_tensorrt/_compile.py @@ -95,13 +95,6 @@ def _has_executorch_exir() -> bool: return False -def _has_executorch_runtime() -> bool: - try: - return importlib.util.find_spec("torch_tensorrt_executorch_runtime") is not None - except ModuleNotFoundError: - return False - - def _non_fx_input_interface( inputs: Sequence[Input | torch.Tensor], ) -> TypeGuard[List[Input | torch.Tensor]]: @@ -612,14 +605,11 @@ def load( """ Load a TorchScript, ExportedProgram, or ExecuTorch program. - By default, detects TorchScript and ExportedProgram files. Set - ``format="executorch"`` explicitly for an ExecuTorch ``.pte`` file. - Arguments: file_path (str): Path to file on the disk extra_files (dict[str, Any]): Extra files to load with the model - format (Optional[str]): Set to ``"executorch"`` to load a ``.pte`` file - using the separately installed ExecuTorch runtime package. + format (Optional[str]): None detects TorchScript and ExportedProgram files. + The deprecated ``"executorch"`` option loads a ``.pte`` program. Example: # Load with extra files. @@ -628,18 +618,36 @@ def load( print(extra_files["foo.txt"]) Raises: - ImportError: If ExecuTorch format is requested without the runtime package - ValueError: If the format is unsupported or the file is not a TorchScript or ExportedProgram file + ImportError: If ExecuTorch format is requested without its runtime dependencies + ValueError: If the format is unsupported or neither standard loader accepts the file + + Note: + ``format="executorch"`` preserves the legacy ``method_names`` property, + ``run(inputs, method="forward")``, and ``forward(*inputs)`` interface. + CUDA inputs are copied to CPU. As before, ``extra_files`` and additional + kwargs are ignored for this format; external ``.ptd`` files are not supported. + This compatibility path will remain for at least six months after the + deprecation first ships. New code should import the TensorRT delegate and + use ExecuTorch's Module API directly:: + + import torch_tensorrt_executorch_runtime # noqa: F401 + from executorch.extension.pybindings.portable_lib import _load_for_executorch + + program = _load_for_executorch("model.pte") + outputs = program.run_method("forward", (tensor,)) """ + if format == "executorch": - if not _has_executorch_runtime(): - raise ImportError( - "Loading an ExecuTorch program requires the Torch-TensorRT " - "ExecuTorch delegate runtime (torch_tensorrt_executorch_runtime). " - "Install it from the PyTorch nightly index for the CUDA version this " - "build targets." - ) - from torch_tensorrt_executorch_runtime.runtime import load as load_executorch + warnings.warn( + "torch_tensorrt.load(format='executorch') is deprecated and will remain " + "supported for at least six months after this deprecation first ships. " + "Import torch_tensorrt_executorch_runtime to register the TensorRT delegate, " + "then use executorch.extension.pybindings.portable_lib._load_for_executorch(path) " + "and module.run_method('forward', inputs).", + DeprecationWarning, + stacklevel=2, + ) + from torch_tensorrt._executorch_compat import load as load_executorch return load_executorch(file_path) if format is not None: @@ -872,8 +880,9 @@ def save( raise TypeError( "save() received unexpected keyword argument(s) for " f"output_format='executorch': {sorted(kwargs)}. Supported executorch " - "options are 'partitioners', 'compile_specs', 'backend_config', and " - "'weight_streaming_budget_per_engine'." + "options are 'partitioners', 'compile_specs', 'backend_config', " + "'constant_methods', 'transform_passes', 'compile_config', " + "'generate_etrecord', and 'weight_streaming_budget_per_engine'." ) # Validate the budget before the input and model-shape checks below, so a wrong # type is not reported as an unrelated failure. diff --git a/py/torch_tensorrt/_executorch_compat.py b/py/torch_tensorrt/_executorch_compat.py new file mode 100644 index 0000000000..93cd5beac0 --- /dev/null +++ b/py/torch_tensorrt/_executorch_compat.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +"""Compatibility for the deprecated torch_tensorrt.load(format='executorch') API.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Collection, Sequence, Union, cast + + +class Program: + """The legacy run/forward interface over an installed ExecuTorch Module.""" + + def __init__(self, module: Any, data: bytes) -> None: + # ExecuTorch's buffer loader can reference these bytes without copying them. + self._data = data + self._module = module + + @property + def method_names(self) -> Collection[str]: + return cast(Collection[str], self._module.method_names()) + + def run(self, inputs: Sequence[Any], method: str = "forward") -> Sequence[Any]: + """Run a method, preserving the legacy CUDA-to-CPU input conversion.""" + import torch + + inputs = tuple( + value.cpu() if isinstance(value, torch.Tensor) and value.is_cuda else value + for value in inputs + ) + if method not in self.method_names: + raise ValueError( + f"Unknown method {method!r}; available methods: {sorted(self.method_names)}" + ) + return cast(Sequence[Any], self._module.run_method(method, inputs)) + + def forward(self, *inputs: Any) -> Sequence[Any]: + return self.run(inputs, "forward") + + +def load(path: Union[str, Path]) -> Program: + """Load a program with embedded weights through ExecuTorch's Module API.""" + # The path first. A caller who mistyped a file name and also has no delegate installed was told + # to install the delegate, which is true but is not what they got wrong. + model_path = Path(path) + if not model_path.is_file(): + raise FileNotFoundError(f"ExecuTorch model not found: {model_path}") + try: + import torch_tensorrt_executorch_runtime as delegate + except ModuleNotFoundError as error: + if error.name != "torch_tensorrt_executorch_runtime": + raise + raise ImportError( + "Loading an ExecuTorch program requires the Torch-TensorRT delegate " + "(torch_tensorrt_executorch_runtime). Install the delegate and ExecuTorch " + "from the same release matrix." + ) from error + # A companion published before the delegate became a single registration call exposes + # activate() instead. Accept it so upgrading this wheel alone keeps loading programs. + register = getattr(delegate, "register", None) + # Whether this companion predates single-call registration, which is the only thing the rewrite + # below is about. Deciding that from the raised error's class name does not work: the older + # companion defines a class of the same name, so a name test matches both and the rewrite it + # guards never fires for the companion it exists for. + predates_register = register is None + if predates_register: + register = getattr(delegate, "activate", None) + if register is None: + raise ImportError( + "The installed torch_tensorrt_executorch_runtime exposes neither register() " + "nor activate(). Install a delegate from the same release matrix as " + "Torch-TensorRT." + ) + + data = model_path.read_bytes() + try: + register() + except ImportError as error: + # A current companion's own compatibility error already says precisely what is wrong, so + # let it through rather than replacing it with a guess about the companion's age. + if not predates_register: + raise + # A companion published before registration became a single call swapped in its own copy of + # ExecuTorch's bindings, and refuses once ExecuTorch's own copy is already loaded. Its advice + # is to import it earlier, which a caller of this function cannot do: the import it collides + # with happens inside this library. Say the thing that does work instead. + raise ImportError( + "The installed torch_tensorrt_executorch_runtime is too old to register alongside " + "ExecuTorch's own bindings. Upgrade it to a build that registers on import, from the " + f"same release matrix as Torch-TensorRT. Underlying error: {error}" + ) from error + # The Module API honors device-tagged arenas; the host Program loader does not. + from executorch.extension.pybindings.portable_lib import ( + _load_for_executorch_from_buffer, + ) + + # No eager validation here, deliberately. A truncated or altered program loads and fails only + # when something first asks it a question, which reads badly, but this function exists to behave + # exactly as the released one did and a test pins both the timing and the identity of that error. + # Improving it means a new entry point, not a change to this one. + return Program(_load_for_executorch_from_buffer(data), data) diff --git a/py/torch_tensorrt/executorch/__init__.py b/py/torch_tensorrt/executorch/__init__.py index dc00f085b0..abc70e5733 100644 --- a/py/torch_tensorrt/executorch/__init__.py +++ b/py/torch_tensorrt/executorch/__init__.py @@ -3,9 +3,8 @@ """ExecuTorch compilation and export integration. -Runtime loading is provided by the optional -``torch-tensorrt-executorch-runtime`` distribution and dispatched through -``torch_tensorrt.load(..., format="executorch")``. +Import ``torch_tensorrt_executorch_runtime`` to register the TensorRT delegate, +then load and run programs through ExecuTorch's Module API. """ import importlib.util diff --git a/py/torch_tensorrt/executorch/partitioner.py b/py/torch_tensorrt/executorch/partitioner.py index 6ebbe20983..64eb958a4f 100644 --- a/py/torch_tensorrt/executorch/partitioner.py +++ b/py/torch_tensorrt/executorch/partitioner.py @@ -136,9 +136,21 @@ def __init__( # it we use that verbatim; otherwise each partition's device is derived from # its own engine node in partition() (engine nodes are not available here) # so a cuda:N engine is not mislabeled cuda:0. - self._has_explicit_target_device = any( - s.key == _TARGET_DEVICE_COMPILE_SPEC_KEY for s in self.compile_specs - ) + explicit = [ + s for s in self.compile_specs if s.key == _TARGET_DEVICE_COMPILE_SPEC_KEY + ] + # Refuse a device this delegate cannot run on. Taking the value verbatim meant a request for + # the processor was accepted here and produced a program that failed at its first + # instruction, which says nothing about the request that caused it. + for spec in explicit: + requested = spec.value.decode(errors="replace") + if requested.split(":", 1)[0] != "cuda": + raise ValueError( + f"{_TARGET_DEVICE_COMPILE_SPEC_KEY}={requested!r} is not a device this " + "delegate runs on. It compiles to TensorRT engines, which need a CUDA device, " + 'so the value has to be "cuda" or "cuda:N".' + ) + self._has_explicit_target_device = bool(explicit) self.delegation_spec = DelegationSpec( backend_id=TensorRTBackend.__name__, compile_specs=self.compile_specs, diff --git a/tests/ci/runner.py b/tests/ci/runner.py index d138c0526c..127be0e833 100644 --- a/tests/ci/runner.py +++ b/tests/ci/runner.py @@ -165,6 +165,7 @@ def _setup_commands(step: str) -> list[tuple[list[str], Path]]: "pip", "install", "pyyaml", + "wheel>=0.40", *index_args, _executorch_requirement(), ], diff --git a/tests/py/dynamo/executorch/test_api.py b/tests/py/dynamo/executorch/test_api.py index 1c036243b2..7845af26b9 100644 --- a/tests/py/dynamo/executorch/test_api.py +++ b/tests/py/dynamo/executorch/test_api.py @@ -3,9 +3,17 @@ import ast import importlib +import importlib.metadata import importlib.util +import json +import os +import pathlib +import re +import shutil +import subprocess import sys import types +import zipfile from pathlib import Path import pytest @@ -13,7 +21,42 @@ from torch._library.fake_class_registry import FakeScriptObject from torch._subclasses.fake_tensor import FakeTensor from torch.export.graph_signature import InputKind -from torch_tensorrt.dynamo._exporter import _resolve_lifted_custom_obj, lift + +# Guarded, because this one import used to stop the whole file being collected wherever +# Torch-TensorRT itself was not installed, which left every test in it unrun for the sake of the few +# that need the exporter. +try: + from torch_tensorrt.dynamo._exporter import _resolve_lifted_custom_obj, lift +except ImportError: # pragma: no cover + _resolve_lifted_custom_obj = None + lift = None + +_needs_exporter = pytest.mark.skipif( + _resolve_lifted_custom_obj is None, + reason="needs Torch-TensorRT installed for its exporter internals", +) +_needs_torch_tensorrt = pytest.mark.skipif( + importlib.util.find_spec("torch_tensorrt") is None, + reason="needs Torch-TensorRT installed", +) + +# CMake command names are case-insensitive, so IF(FALSE) and If(FALSE) open the same block a +# case-sensitive pattern misses. Every block command counts, not just if(): wrapping the guard in +# while(FALSE) or in an uncalled function() hides it just as completely. Measured with real cmake +# builds: all four spellings produced a delegate needing libcudart.so.12, which is what the guard +# exists to stop, with the wiring test green. +_CMAKE_BLOCK_OPEN = r"(?:if|while|foreach|function|macro|block)\s*\(" +_CMAKE_BLOCK_CLOSE = r"end(?:if|while|foreach|function|macro|block)\s*\(" + + +@pytest.mark.unit +def test_the_install_script_puts_the_cuda_runtime_on_the_library_path(): + """The CUDA 13 runtime directory must be available to the reference runner.""" + script = (_REPO_ROOT / ".github/scripts/install-torch-tensorrt.sh").read_text( + encoding="utf-8" + ) + assert "nvidia/cu13/lib" in script + assert "cu13*)" in script @pytest.mark.unit @@ -27,14 +70,24 @@ def test_the_python_loader_uses_the_api_that_backs_device_arenas(): comment, since the two APIs differ by one function name and swapping back would be silent until someone ran a delegated model on a GPU. """ - source = _RUNTIME_PY.read_text(encoding="utf-8") - assert "_load_for_executorch_from_buffer" in source, ( - "the runtime no longer loads through the Module API, so device-planned arenas would be " - "planned on the host and every delegate boundary copy would fail" - ) - assert ( - "load_program" not in source - ), "the runtime still references the program loader, which does not back device-tagged arenas" + for name in ("load_model.py", "load_model_device_resident.py"): + source = ( + _REPO_ROOT / "examples/executorch_reference_runner" / name + ).read_text() + tree = ast.parse(source) + calls = [node.func for node in ast.walk(tree) if isinstance(node, ast.Call)] + assert any( + isinstance(call, ast.Name) and call.id == "_load_for_executorch" + for call in calls + ) + assert any( + isinstance(call, ast.Attribute) and call.attr == "run_method" + for call in calls + ) + assert not any( + isinstance(call, ast.Attribute) and call.attr == "load_program" + for call in calls + ) def _is_importable_module(name: str) -> bool: @@ -91,13 +144,13 @@ def test_the_runtime_package_imports_every_submodule_it_reaches_through(): ): used.add(f"{node.value.value.id}.{node.value.attr}") - # Only names that are importable modules matter. sys.modules is an attribute of an imported - # module, not a submodule, so it is reachable without a second import and is not a finding. + # os imports its public path alias itself; other dotted modules need their own import. missing = sorted( name for name in used if name.split(".")[0] in imported and name not in imported + and name != "os.path" and _is_importable_module(name) ) assert not missing, ( @@ -106,6 +159,7 @@ def test_the_runtime_package_imports_every_submodule_it_reaches_through(): ) +@_needs_torch_tensorrt @pytest.mark.unit def test_lazy_import_error_when_executorch_missing(monkeypatch): import torch_tensorrt @@ -136,6 +190,7 @@ def fake_find_spec(name, package=None): delattr(torch_tensorrt, "executorch") +@_needs_torch_tensorrt @pytest.mark.unit def test_save_executorch_error_when_executorch_missing(monkeypatch, tmp_path): original_find_spec = importlib.util.find_spec @@ -157,35 +212,7 @@ def fake_find_spec(name, package=None): ) -@pytest.mark.unit -def test_load_executorch_error_when_delegate_missing(monkeypatch): - from torch_tensorrt import _compile - - monkeypatch.setattr(_compile, "_has_executorch_runtime", lambda: False) - - with pytest.raises(ImportError, match=r"torch-tensorrt-executorch-runtime"): - _compile.load("model.pte", format="executorch") - - -@pytest.mark.unit -def test_load_executorch_dispatches_to_delegate(monkeypatch): - from torch_tensorrt import _compile - - delegate = types.ModuleType("torch_tensorrt_executorch_runtime") - delegate.__path__ = [] - runtime = types.ModuleType("torch_tensorrt_executorch_runtime.runtime") - sentinel = object() - runtime.load = lambda path: (sentinel, path) - monkeypatch.setitem(sys.modules, delegate.__name__, delegate) - monkeypatch.setitem(sys.modules, runtime.__name__, runtime) - monkeypatch.setattr(_compile, "_has_executorch_runtime", lambda: True) - - assert _compile.load("model.pte", format="executorch") == ( - sentinel, - "model.pte", - ) - - +@_needs_torch_tensorrt @pytest.mark.unit def test_public_api_symbols_present(): module = importlib.import_module("torch_tensorrt.executorch") @@ -200,80 +227,1509 @@ def test_public_api_symbols_present(): _REPO_ROOT = Path(__file__).resolve().parents[4] _SETUP_PY = _REPO_ROOT / "setup.py" +_FILTER_MATRIX_PY = _REPO_ROOT / ".github/scripts/filter-matrix.py" + + +def _filter_matrix_declarations() -> dict[str, object]: + """Module-level literals declared by the matrix filter script.""" + declared: dict[str, object] = {} + for node in ast.parse(_FILTER_MATRIX_PY.read_text(encoding="utf-8")).body: + target = getattr(node, "target", None) or next( + iter(getattr(node, "targets", [])), None + ) + if isinstance(target, ast.Name) and isinstance( + getattr(node, "value", None), (ast.List, ast.Constant) + ): + declared[target.id] = ast.literal_eval(node.value) + return declared + + +def _executorch_cuda_major() -> str: + return str(_filter_matrix_declarations()["EXECUTORCH_CUDA_MAJOR"]) + + +def _executorch_cuda_rows() -> set[str]: + """CUDA rows the delegate can take: the wheel matrix filtered to its CUDA major. + + Read from the filter script rather than restated, so this asserts the two agree. + """ + declared = _filter_matrix_declarations() + major = declared["EXECUTORCH_CUDA_MAJOR"] + return { + cuda + for cuda in declared["x86_cuda_versions"] # type: ignore[union-attr] + if re.fullmatch(rf"cu{major}\d+", cuda) + } + + _RUNTIME_SETUP_PY = _REPO_ROOT / "py/torch-tensorrt-executorch-runtime/setup.py" -_RUNTIME_PY = ( - _REPO_ROOT - / "py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py" -) _RUNTIME_INIT_PY = ( _REPO_ROOT / "py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/__init__.py" ) +# The RUNPATH the build declares, TORCH_TENSORRT_DELEGATE_RUNPATH in native/CMakeLists.txt joined +# with ':'. Production hands this to the guard as its fourth argument, which selects the +# exact-whole-RUNPATH branch. +_GUARD_GOOD_RUNPATH = ( + "$ORIGIN:$ORIGIN/../../executorch/lib:$ORIGIN/../../tensorrt_libs" + ":$ORIGIN/../../nvidia/cu13/lib" +) +# Cases kept on the 3-argument invocation so the fallback branch (the elif in the guard that +# spot-checks for '$ORIGIN/../../executorch/lib' when no expected RUNPATH is passed) and the +# absolute-entry check below it stay covered. Everything else runs with the fourth argument the +# way production does, exercising the exact-whole-RUNPATH branch. +_THREE_ARG_CASES = { + "wrong_depth_runpath", + "runpath_missing_executorch", + "absolute_runpath", + "runpath_fallback_reaches_executorch", +} + @pytest.mark.unit -def test_runtime_implementation_is_owned_by_runtime_package(): - assert not (_REPO_ROOT / "py/torch_tensorrt/executorch/runtime.py").exists() +@pytest.mark.parametrize( + "case", ["present", "absent", "optional", "repeat", "function", "versioned"] +) +def test_the_cmake_package_defines_a_linkable_target(tmp_path, case): + """Discover the fixed wheel library and restore outputs in each caller's scope.""" + cmake = shutil.which("cmake") + if cmake is None: + pytest.skip("cmake is not installed") + config = _RUNTIME_SETUP_PY.parent / "cmake/torchtrt_executorch-config.cmake" + prefix = tmp_path / "prefix" + cmake_dir = prefix / "lib/cmake/torchtrt_executorch" + cmake_dir.mkdir(parents=True) + shutil.copy2(config, cmake_dir / config.name) + library = prefix / "lib/libexecutorch_backend_tensorrt.so" + if case not in {"absent", "optional"}: + library.write_bytes(b"stub") + if case == "versioned": + library.with_suffix(".so.9").write_bytes(b"decoy") + + discovery = "find_package(torchtrt_executorch REQUIRED)\n" + if case == "repeat": + discovery += ( + "unset(TORCHTRT_EXECUTORCH_LIBRARIES)\n" + "find_package(torchtrt_executorch REQUIRED)\n" + ) + elif case == "function": + discovery = ( + "function(discover)\n" + + discovery + + "endfunction()\ndiscover()\n" + + discovery + ) + elif case == "optional": + discovery = "find_package(torchtrt_executorch QUIET)\n" + if case in {"absent", "optional"}: + checks = ( + "if(torchtrt_executorch_FOUND OR TARGET torchtrt::executorch_backend)\n" + ' message(FATAL_ERROR "missing delegate accepted")\nendif()\n' + ) + else: + checks = ( + 'if(NOT TORCHTRT_EXECUTORCH_LIBRARIES STREQUAL "torchtrt::executorch_backend")\n' + ' message(FATAL_ERROR "missing library list")\nendif()\n' + "get_target_property(location torchtrt::executorch_backend IMPORTED_LOCATION)\n" + f'if(NOT location STREQUAL "{library}")\n' + ' message(FATAL_ERROR "wrong delegate selected: ${location}")\nendif()\n' + "get_target_property(options torchtrt::executorch_backend INTERFACE_LINK_OPTIONS)\n" + 'if(NOT options MATCHES "no-as-needed")\n' + ' message(FATAL_ERROR "missing retention options")\nendif()\n' + ) + app = tmp_path / "app" + app.mkdir() + (app / "CMakeLists.txt").write_text( + "cmake_minimum_required(VERSION 3.28)\nproject(probe LANGUAGES NONE)\n" + + discovery + + checks + ) + result = subprocess.run( + [ + cmake, + "-S", + str(app), + "-B", + str(tmp_path / "build"), + f"-DCMAKE_PREFIX_PATH={prefix}", + "-DCMAKE_SYSTEM_NAME=Linux", + ], + capture_output=True, + text=True, + ) + output = result.stdout + result.stderr + if case == "absent": + assert result.returncode != 0, output + assert "TORCHTRT_EXECUTORCH_BACKEND_LIBRARY" in output, output + else: + assert result.returncode == 0, output + + +@pytest.mark.unit +@pytest.mark.parametrize("removed", ["required", "not_found", "outputs", "fixed_name"]) +def test_cmake_discovery_rejects_removed_guards(monkeypatch, tmp_path, removed): + config = _RUNTIME_SETUP_PY.parent / "cmake/torchtrt_executorch-config.cmake" + text = config.read_text() + if removed == "required": + text, count = re.subn( + r"find_package_handle_standard_args\(\s*torchtrt_executorch\s*" + r"REQUIRED_VARS TORCHTRT_EXECUTORCH_BACKEND_LIBRARY\s*\)", + "", + text, + ) + assert count == 1 + case = "absent" + elif removed == "not_found": + original = "if(NOT torchtrt_executorch_FOUND)\n return()\nendif()" + assert text.count(original) == 1 + text = text.replace(original, "") + case = "optional" + elif removed == "outputs": + original = "set(TORCHTRT_EXECUTORCH_LIBRARIES torchtrt::executorch_backend)" + assert text.count(original) == 1 + text = text.replace(original, "") + "\n" + original + "\n" + case = "repeat" + else: + assert "libexecutorch_backend_tensorrt.so" in text + text = text.replace( + "libexecutorch_backend_tensorrt.so", "libexecutorch_backend_tensorrt.so.9" + ) + case = "versioned" + copied = tmp_path / "source/cmake" / config.name + copied.parent.mkdir(parents=True) + copied.write_text(text) + monkeypatch.setitem(globals(), "_RUNTIME_SETUP_PY", copied.parents[1] / "setup.py") + with pytest.raises(AssertionError): + test_the_cmake_package_defines_a_linkable_target(tmp_path, case) + + +@pytest.mark.unit +def test_the_delegate_follows_the_main_wheels_cuda_versions(): + """The delegate follows the validated CUDA 13 nightly rows on both Linux architectures.""" + matrix = { + "include": [ + { + "python_version": "3.12", + "desired_cuda": cuda, + "gpu_arch_version": cuda[2:], + "gpu_arch_type": arch_type, + "validation_runner": "runner", + "container_image": "image", + "package_type": "wheel", + "build_name": f"b_{cuda}_{arch}", + "channel": "nightly", + "upload_to_base_dir": "d", + "stable_version": "s", + "use-rtx": "false", + "os": os_name, + "arch": arch, + } + for cuda in ("cu126", "cu130", "cu132", "cu134") + for os_name, arch, arch_type in ( + ("linux", "x86_64", "cuda"), + ("linux-aarch64", "aarch64", "cuda-aarch64"), + ) + ] + } + result = subprocess.run( + [ + sys.executable, + str(_REPO_ROOT / ".github/scripts/filter-matrix.py"), + "--matrix", + json.dumps(matrix), + "--use-rtx", + "false", + "--limit-pr-builds", + "false", + ], + capture_output=True, + text=True, + check=True, + ) + rows = json.loads(result.stdout.strip().splitlines()[-1])["include"] + by_arch = {} + for row in rows: + by_arch.setdefault(row["os"], set()).add(row["desired_cuda"]) + + assert by_arch == { + "linux": _executorch_cuda_rows(), + "linux-aarch64": _executorch_cuda_rows(), + } + + +@pytest.mark.parametrize( + "workflow", + [ + "executorch-test-linux.yml", + "release-linux-x86_64.yml", + "release-linux-aarch64.yml", + ], +) +@pytest.mark.parametrize("channel", ["nightly", "test"]) +@pytest.mark.unit +def test_runtime_workflows_filter_cuda_12_without_changing_main_releases( + tmp_path, workflow, channel +): + """Run the real matrix steps; release rows must survive companion-only CUDA gating.""" + import shlex + + import yaml + + jobs = yaml.safe_load((_REPO_ROOT / ".github/workflows" / workflow).read_text())[ + "jobs" + ] + release = workflow.startswith("release-") + job_name = "generate-release-wheel-matrix" if release else "filter-matrix" + job = jobs[job_name] + step = next(step for step in job["steps"] if step.get("id") == "generate") + arch = "cuda-aarch64" if "aarch64" in workflow else "cuda" + matrix = { + "include": [ + { + "python_version": "3.12", + "desired_cuda": cuda, + "gpu_arch_type": arch, + "channel": channel, + } + for cuda in ("cu126", "cu130", "cu132", "cu134") + ] + } + script = re.sub( + r"\$\{\{ toJSON\([^\n]+\) \}\}", + lambda _: shlex.quote(json.dumps(matrix)), + step["run"], + ) + script = re.sub(r"\$\{\{[^\n]+\}\}", "false", script) + output = tmp_path / "outputs" + env = { + **os.environ, + "GITHUB_OUTPUT": str(output), + "PATH": f"{Path(sys.executable).parent}{os.pathsep}{os.environ['PATH']}", + } + result = subprocess.run( + ["bash", "-c", script], cwd=_REPO_ROOT, env=env, capture_output=True, text=True + ) + assert result.returncode == 0, result.stdout + result.stderr + outputs = dict(line.split("=", 1) for line in output.read_text().splitlines()) + if release: + assert "executorch-matrix" not in job["outputs"] + assert "release-executorch-runtime-wheel-artifacts" not in jobs + assert ( + jobs["release-wheel-artifacts"]["with"]["build-executorch-runtime"] is True + ) + assert jobs["release-wheel-artifacts"]["with"]["build-matrix"] == ( + "${{ needs.generate-release-wheel-matrix.outputs.matrix }}" + ) + ordinary = { + row["desired_cuda"] for row in json.loads(outputs["matrix"])["include"] + } + assert ordinary == _executorch_cuda_rows() + return + selected = outputs["matrix"] + assert { + row["desired_cuda"] for row in json.loads(selected)["include"] + } == _executorch_cuda_rows() + + +@pytest.mark.parametrize("arch,floor", [("x86_64", "2_28"), ("aarch64", "2_35")]) +@pytest.mark.unit +def test_shared_repair_preserves_the_companion_payload(tmp_path, arch, floor): + """Run the shared repair loop with real wheels and check tags, hashes, and routing.""" + from email.parser import BytesParser + from wheel.wheelfile import WheelFile + + dist = tmp_path / "dist" + dist.mkdir() + name = "torch_tensorrt_executorch_runtime-0.1.0" + wheel = dist / f"{name}-py3-none-linux_{arch}.whl" + payload = b"ELF fixture: repair must not change the delegate" + member = "torch_tensorrt_executorch_runtime/lib/libexecutorch_backend_tensorrt.so" + with WheelFile(wheel, "w") as archive: + archive.writestr(member, payload) + archive.writestr( + f"{name}.dist-info/WHEEL", + f"Wheel-Version: 1.0\nRoot-Is-Purelib: false\nTag: py3-none-linux_{arch}\n", + ) + archive.writestr( + f"{name}.dist-info/METADATA", + "Metadata-Version: 2.1\nName: torch-tensorrt-executorch-runtime\nVersion: 0.1.0\n", + ) + standard = dist / f"torch_tensorrt-2.15.0-cp312-cp312-linux_{arch}.whl" + standard.touch() + repair = tmp_path / "test-infra/.github/scripts/repair_manylinux_2_28.sh" + repair.parent.mkdir(parents=True) + repair.write_text('#!/bin/sh\nprintf "%s\\n" "$1" >> "$REPAIR_CALLS"\n') + repair.chmod(0o755) + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + (bin_dir / "python").write_text(f'#!/bin/sh\nexec "{sys.executable}" "$@"\n') + (bin_dir / "python").chmod(0o755) + build_env = tmp_path / "build-env" + build_env.write_text("export CONDA_RUN=''\n") + calls = tmp_path / "repair-calls" + step = next( + step + for step in _runtime_build_steps() + if step.get("name") == "Repair Manylinux_2_28 Wheel" + ) + script = step["run"].replace("${{ inputs.repository }}", str(tmp_path)) + result = subprocess.run( + ["bash", "-c", script], + cwd=tmp_path, + capture_output=True, + text=True, + env={ + **os.environ, + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "BUILD_ENV_FILE": str(build_env), + "ARCH": arch, + "REPAIR_CALLS": str(calls), + }, + timeout=30, + ) + (tmp_path / "stdout.log").write_text(result.stdout) + (tmp_path / "stderr.log").write_text(result.stderr) + assert result.returncode == 0, result.stdout + result.stderr + assert calls.read_text().splitlines() == [str(standard)] + assert not wheel.exists() + tag = f"py3-none-manylinux_{floor}_{arch}" + with WheelFile(dist / f"{name}-{tag}.whl") as archive: + assert archive.read(member) == payload + metadata = BytesParser().parsebytes(archive.read(f"{name}.dist-info/WHEEL")) + assert metadata.get_all("Tag") == [tag] + assert metadata["Root-Is-Purelib"] == "false" + for member in archive.namelist(): + archive.read(member) + + +def _runtime_build_steps(): + import yaml + + workflow = yaml.safe_load( + (_REPO_ROOT / ".github/workflows/build_linux.yml").read_text() + ) + return workflow["jobs"]["build"]["steps"] + + +@pytest.mark.unit +def test_artifact_checks_run_after_the_shared_build(): + """Artifact checks do not depend on GPU availability.""" + steps = _runtime_build_steps() + check = next( + step + for step in steps + if step.get("name") == "Check the repaired ExecuTorch runtime wheel" + ) + assert check["if"] == "${{ steps.executorch-runtime.outcome == 'success' }}" + assert "torch.cuda.is_available" not in check["run"] + assert "ldd -r" in check["run"] + + +@pytest.mark.unit +def test_the_build_script_never_imports_torch_tensorrt(): + """Building must not require a GPU through a compiler-package import. + + The lowering passes query CUDA device capability at import time. Locate the + installed main wheel through distribution metadata on either architecture. + """ + script = next( + step["run"] + for step in _runtime_build_steps() + if step.get("id") == "executorch-runtime" + ) + + for forbidden in ( + "import torch_tensorrt\n", + "from torch_tensorrt", + "import torch_tensorrt ", + ): + assert forbidden not in script, ( + f"the build script contains {forbidden.strip()!r}, which pulls in " + "torch.cuda.get_device_capability() at import time and fails on a builder with no GPU" + ) + # torch_tensorrt_executorch_runtime is a different distribution and is fine: it dlopens the + # delegate and never imports the compiler package + bare = [ + line + for line in script.splitlines() + if "torch_tensorrt" in line + and "torch_tensorrt_executorch_runtime" not in line + and "torch-tensorrt" not in line + and line.strip().startswith(("python", "import", "from")) + ] + assert not bare, f"the build script still imports torch_tensorrt: {bare}" + + +@pytest.mark.unit +def test_the_delegate_is_built_for_every_architecture_the_main_wheel_ships(): + """Both Linux architectures build the companion inside the standard artifact job.""" + import yaml + + workflows = _REPO_ROOT / ".github/workflows" + shared = yaml.safe_load((workflows / "_test-linux.yml").read_text())["jobs"][ + "build" + ] + assert shared["uses"] == "./.github/workflows/build_linux.yml" + assert shared["with"]["architecture"] == "${{ inputs.architecture }}" assert ( + shared["with"]["build-executorch-runtime"] + == "${{ !inputs.python-only && !inputs.use-rtx }}" + ) + for filename, arch, os_name in ( + ("ci-linux-x86_64.yml", "x86_64", "linux"), + ("ci-sbsa.yml", "aarch64", "linux-aarch64"), + ): + jobs = yaml.safe_load((workflows / filename).read_text())["jobs"] + assert jobs["generate-matrix"]["with"]["os"] == os_name + callers = [ + job + for job in jobs.values() + if job.get("uses") == "./.github/workflows/_test-linux.yml" + and not job["with"].get("python-only") + and not job["with"].get("use-rtx") + ] + assert callers + assert all(job["with"].get("architecture", "x86_64") == arch for job in callers) + assert "executorch-runtime-build" not in jobs + steps = _runtime_build_steps() + build_at = next( + i for i, step in enumerate(steps) if step.get("id") == "executorch-runtime" + ) + assert any( + step.get("name") == "Build the wheel (setup-py)" for step in steps[:build_at] + ) + assert not (workflows / "executorch-build-linux.yml").exists() + + +@pytest.mark.unit +def test_the_guard_is_given_the_platform_it_must_compare_against(): + """Production passes the full architecture-specific tag to the artifact guard.""" + cmake = ( + _REPO_ROOT / "py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt" + ).read_text(encoding="utf-8") + guard = ( + _REPO_ROOT + / "py/torch-tensorrt-executorch-runtime/native/check_imports_executorch_runtime.sh" + ).read_text(encoding="utf-8") + + assert "TORCH_TENSORRT_MANYLINUX_TAG" in cmake, ( + "the build never computes a manylinux tag, so the guard receives nothing and skips the " + "symbol version ceiling without saying so" + ) + invocation = cmake[cmake.index("check_imports_executorch_runtime.sh") :] + invocation = invocation[: invocation.index("VERBATIM")] + assert ( + "${TORCH_TENSORRT_MANYLINUX_TAG}" in invocation + ), "the tag is computed but not passed to the guard, so the ceiling is skipped" + assert ( + 'manylinux_tag="${5-}"' in guard + ), "the guard does not read a fifth argument, so the tag the build passes is ignored" + # Per architecture, not one constant. Asserted on the set() calls rather than on the text, + # because the tag names also appear in the comment explaining why they differ, so a substring + # search stayed green when both branches were collapsed to the same value. + assigned = set( + re.findall(r"set\(TORCH_TENSORRT_MANYLINUX_TAG \"([^\"]+)\"\)", cmake) + ) + assert assigned == {"manylinux_2_28_x86_64", "manylinux_2_35_aarch64"}, ( + "the build assigns " + f"{sorted(assigned)} as its manylinux tag, but the two architectures ship under different " + "platforms and using one for both rejects the aarch64 row for requiring exactly what its " + "own builder image provides" + ) + for tag in ("_2_28_x86_64", "_2_35_aarch64"): + assert tag in guard, f"the guard has no floor entry for manylinux{tag}" + + +@pytest.mark.unit +def test_the_wheel_ships_a_cmake_package_for_cpp_consumers(): + """The delegate has to be linkable from C++, not just importable from Python. + + ExecuTorch ships each of its backends as a prebuilt shared library plus a CMake package, so a + C++ app links ``executorch::backend_cuda`` and the backend registers itself. This wheel is an + out-of-tree backend and needs the same two pieces, or its shared library is reachable only by + building this repository from source. + + Three things are asserted because each fails differently. Without the config file there is no + target to link. With the library outside ``lib/`` the config cannot find it, and the Python + loader and the C++ consumer would disagree about where it lives. Without ``--no-as-needed`` the + link succeeds and the backend silently never registers, which is the worst of the three because + it fails at run time with an unregistered backend rather than at build time. + """ + package_dir = _REPO_ROOT / "py/torch-tensorrt-executorch-runtime" + config = package_dir / "cmake/torchtrt_executorch-config.cmake" + assert ( + config.is_file() + ), "no CMake package config, so a C++ app cannot link the delegate out of the wheel" + config_text = config.read_text(encoding="utf-8") + + assert ( + "SHARED IMPORTED" in config_text + ), "the config does not define an imported shared library target" + assert "/lib/libexecutorch_backend_tensorrt.so" in config_text, ( + "the config does not look for the delegate under lib/, where the wheel installs it and " + "where ExecuTorch keeps its own backends" + ) + # Asserted against the LINKER: line, not the whole file, because the comment above it also says + # --no-as-needed. A substring test over the file would pass on a config that explains the flag + # and then does not pass it. + link_options = [ + line + for line in config_text.splitlines() + if "LINKER:" in line and "no-as-needed" in line + ] + assert link_options, ( + "the config does not force the delegate onto the link line; nothing references a symbol " + "it defines, so the linker would drop it and the backend would never register" + ) + assert "push-state" in link_options[0] and "pop-state" in link_options[0], ( + "--no-as-needed is not bracketed with push-state/pop-state, so it leaks into the rest of " + "the consumer's link line" + ) + + # setup.py has to actually ship both files, and put the library where the config looks. + setup_text = (package_dir / "setup.py").read_text(encoding="utf-8") + for fragment in ( + "lib/cmake/torchtrt_executorch/*.cmake", + "lib/{DELEGATE_LIBRARY}", + ): + assert ( + fragment in setup_text + ), f"setup.py does not package {fragment}, so the wheel would omit it" + # The directory the build WRITES to has to be the one package_data names. Those are two + # separate statements in setup.py, and when they disagree the build writes the CMake package + # somewhere the wheel never collects, so the wheel ships no package at all and every C++ + # consumer fails at find_package. Asserting the path components rather than a joined string, + # since setup.py builds it with pathlib. + assert re.search( + r'cmake_dir\s*=\s*package_dir\s*/\s*"lib"\s*/\s*"cmake"\s*/\s*"torchtrt_executorch"', + setup_text, + ), ( + "setup.py writes the CMake package somewhere other than lib/cmake/torchtrt_executorch, so " + "it no longer agrees with the path package_data collects and the wheel would ship no " + "CMake package" + ) + assert "torchtrt_executorch-config-version.cmake" in setup_text, ( + "setup.py writes no version file, so find_package(torchtrt_executorch 2.15) would match " + "any version at all" + ) + # The package has to REFUSE a prefix with no library, rather than export a target pointing + # nowhere. Without this gate find_package reports success and IMPORTED_LOCATION comes out empty, + # which was measured: "FOUND_ANYWAY, IMPORTED_LOCATION=[]". The consumer then fails at link time + # with a message that names neither this package nor the missing file. + assert "find_package_handle_standard_args(" in config_text, ( + "the config never calls find_package_handle_standard_args, so a prefix with no delegate " + "still reports success" + ) + assert "REQUIRED_VARS TORCHTRT_EXECUTORCH_BACKEND_LIBRARY" in config_text, ( + "the library is not listed in REQUIRED_VARS, so find_package succeeds when the delegate is " + "absent and exports a target with an empty IMPORTED_LOCATION" + ) + assert "if(NOT torchtrt_executorch_FOUND)" in config_text, ( + "the config does not return early when the library is missing, so it goes on to define an " + "imported target from an empty path" + ) + # The version file has to consult the upper end of a range and the major, not just ask whether + # the installed version is at least the requested one. A lone VERSION_LESS accepts two requests + # it must refuse: 2.14...<2.15 is handed 2.15, and a request for 1.0 is satisfied by 2.15, so a + # consumer written against a different major links this delegate anyway. Both were measured + # against write_basic_package_version_file(COMPATIBILITY SameMajorVersion), which refuses them. + for required, why in ( + ("PACKAGE_FIND_VERSION_RANGE", "the upper end of a version range is ignored"), + ("PACKAGE_FIND_VERSION_MAX", "a range's maximum is never compared"), + ("PACKAGE_FIND_VERSION_MAJOR", "a request from another major is accepted"), + ): + assert required in setup_text, ( + f"the generated version file does not mention {required}, so {why} and find_package " + "matches versions it was told not to" + ) + + +@pytest.mark.unit +def test_the_runtime_package_ships_no_runtime_api(): + """The delegate wheel registers a backend, and keeps the entry points it used to publish. + + It used to carry a ``runtime.py`` wrapping ExecuTorch's ``Runtime``/``Program``, which duplicated + what ExecuTorch already exports and put a second inference API in a wheel whose only job is + registration. The old location under torch_tensorrt is gone outright. The submodule inside the + delegate package survives only because the published main wheel imports ``load`` from it by name, + so deleting it would turn torch_tensorrt.load(format="executorch") into a ModuleNotFoundError for + anyone who upgrades this package alone. It must be a forwarder and nothing more. + """ + assert not (_REPO_ROOT / "py/torch_tensorrt/executorch/runtime.py").exists() + legacy_loader = ( _REPO_ROOT / "py/torch-tensorrt-executorch-runtime" / "torch_tensorrt_executorch_runtime/runtime.py" - ).is_file() + ) + assert ( + legacy_loader.exists() + ), "the published main wheel imports load from this submodule" + loader_source = legacy_loader.read_text(encoding="utf-8") + assert "DeprecationWarning" in loader_source, loader_source + # It forwards to the main wheel's loader, not to ExecuTorch's. That is what returns the object + # the published API returned, carrying run() and forward() and raising for a missing path. + # Where that forwarding leads is checked where the shape matters, in test_load_compatibility. + tree = ast.parse(loader_source) + load = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "load" + ) + returns = [ + ast.unparse(node.value) + for node in ast.walk(load) + if isinstance(node, ast.Return) and node.value is not None + ] + assert ( + returns + ), "the forwarder returns nothing, so a caller of the published API gets None" + # A forwarder, not a second inference API: no runtime or program wrapper may come back. + for reintroduced in ( + "class Program", + "class Runtime", + "def run_method", + "class Module", + ): + assert reintroduced not in loader_source, reintroduced + + # The registration surface, plus the two names the published wheel already exported. Removing + # those outright breaks code written against it, so both stay as deprecated forwarders that + # warn and hand back what ExecuTorch itself provides. + delegate_init = ( + _REPO_ROOT + / "py/torch-tensorrt-executorch-runtime" + / "torch_tensorrt_executorch_runtime/__init__.py" + ).read_text(encoding="utf-8") + exported = ast.literal_eval( + re.search(r"^__all__\s*=\s*(\[[^\]]*\])", delegate_init, re.MULTILINE).group(1) + ) + assert set(exported) == { + "BACKEND_NAME", + "DelegateCompatibilityError", + "activate", + "get_runtime", + "register", + }, ( + "the delegate package exports something beyond its registration surface and the " + f"entry points it has to keep: {sorted(exported)}" + ) + # Both forward to ExecuTorch rather than returning None, so a caller that used the return value + # of the published API still gets something usable. Read the parsed function rather than the + # text: a substring is satisfied by a commented-out line, which is the shape of the change this + # is meant to catch. What the forwarding actually does at run time is covered where the shims + # are called, in test_shared_runtime_workflow. + forwards = {"activate": "portable_lib", "get_runtime": "Runtime"} + functions = { + node.name: node + for node in ast.parse(delegate_init).body + if isinstance(node, ast.FunctionDef) + } + for name, expected in forwards.items(): + returns = [ + ast.unparse(node.value) + for node in ast.walk(functions[name]) + if isinstance(node, ast.Return) and node.value is not None + ] + assert ( + returns + ), f"{name} returns nothing, so a caller of the published API gets None" + assert any( + expected in returned for returned in returns + ), f"{name} does not forward to {expected}: {returns}" @pytest.mark.unit def test_runtime_extension_has_dependency_wheel_rpaths(): + """The search path that actually ships is the patchelf literal, so assert on that one. + + The list is declared once and consumed twice: as INSTALL_RPATH for the linker, and as the + value handed to ``patchelf --set-rpath``. patchelf runs ``--remove-rpath`` first, so only its + copy reaches the artifact -- which is why this asserts that both consumers really do read the + one declaration, rather than that two literals happen to agree today. + + Set equality rather than membership, so an entry silently added to the shipped path fails here + too and has to be justified. + """ cmake = ( _REPO_ROOT / "py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt" ).read_text(encoding="utf-8") assert "BUILD_WITH_INSTALL_RPATH ON" in cmake - assert "$ORIGIN/../torch/lib" in cmake - assert "$ORIGIN/../tensorrt_libs" in cmake - assert "$ORIGIN/../nvidia/cu13/lib" in cmake - assert "CUDAToolkit_VERSION_MAJOR EQUAL 13" in cmake assert "-Wl,-Bsymbolic" not in cmake - assert "set(EXECUTORCH_BUILD_KERNELS_OPTIMIZED ON" in cmake - assert "set(EXECUTORCH_BUILD_XNNPACK ON" in cmake + + declared = re.search( + r'set\(\s*TORCH_TENSORRT_DELEGATE_RUNPATH\s+((?:"[^"]+"\s*)+)\)', cmake + ) + assert ( + declared + ), "the RUNPATH list is no longer a single declaration this test can read" + entries = set(re.findall(r'"([^"]+)"', declared.group(1))) + # libexecutorch.so belongs to the executorch distribution, not this one, so the delegate has + # to reach out of its own package to find it. Two levels: this artifact installs under lib/ in + # the package directory, so site-packages is two levels above $ORIGIN. + # No torch/lib or CUDA 12 path: this delegate links only the CUDA 13 runtime. + assert entries == { + "$ORIGIN", + "$ORIGIN/../../executorch/lib", + "$ORIGIN/../../tensorrt_libs", + "$ORIGIN/../../nvidia/cu13/lib", + } + # The single-level form is the bug this guards against: it was right when the artifact + # installed flat in the package directory, and it is wrong now that it installs under lib/. + # A stale one-level entry resolves to site-packages/torch_tensorrt_executorch_runtime + # instead of site-packages, so the delegate cannot find libexecutorch.so and a C++ consumer + # fails to link it with undefined references to cudart and nvinfer. + assert "$ORIGIN/../executorch/lib" not in cmake + + # The linker's copy has to say the same thing. It does not reach the artifact, since patchelf + # removes it, but a build without patchelf and every in-tree consumer read it, and two lists + # that are supposed to be the same path are a bug once they disagree. + # Both consumers have to read the declaration rather than restate it, or the deduplication + # is cosmetic and the copies can drift apart again. + assert ( + 'INSTALL_RPATH "${TORCH_TENSORRT_DELEGATE_RUNPATH}"' in cmake + ), "INSTALL_RPATH no longer reads the shared RUNPATH declaration" + assert ( + '"${TORCH_TENSORRT_DELEGATE_RUNPATH_COLONS}"' in cmake + ), "patchelf --set-rpath no longer reads the shared RUNPATH declaration" + assert re.search( + r'string\(JOIN ":" TORCH_TENSORRT_DELEGATE_RUNPATH_COLONS\s+\$\{TORCH_TENSORRT_DELEGATE_RUNPATH\}\)', + cmake, + ), "the colon-joined form is not derived from the same list" + + # DT_RUNPATH, not the older DT_RPATH: --force-rpath would flip the tag, and the pinned + # ExecuTorch passes --enable-new-dtags precisely to avoid it. Comments are stripped first, + # because the CMakeLists explains this in prose and the prose names the flag. + code = "\n".join( + line for line in cmake.splitlines() if not line.lstrip().startswith("#") + ) + assert "--force-rpath" not in code @pytest.mark.unit -def test_the_install_script_puts_the_cuda_runtime_on_the_library_path(): - """The CUDA 13 runtime directory must be available to the reference runner.""" - script = (_REPO_ROOT / ".github/scripts/install-torch-tensorrt.sh").read_text( - encoding="utf-8" +def test_runtime_extension_consumes_the_prebuilt_executorch_runtime(): + """The wheel must link ExecuTorch's shipped runtime, not rebuild one of its own. + + Rebuilding it would give the delegate a second copy of the backend registry and of the + caller-stream thread-local, so registration would land somewhere the user's ExecuTorch + never reads. Both spellings are asserted because the build is only correct if it takes the + runtime from the package and never adds ExecuTorch's own source tree. + """ + cmake = ( + _REPO_ROOT / "py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt" + ).read_text(encoding="utf-8") + + assert "find_package(executorch REQUIRED)" in cmake + + # Scoped to the link call: both names also appear in the required-target guard above it, so + # searching the whole file would pass even if the delegate linked neither. Matched by + # balancing parens rather than with a non-greedy regex, which would stop at the first `)` + # and silently truncate the block if a generator expression were added to the call. + opening = re.search(r"target_link_libraries\(executorch_backend_tensorrt\b", cmake) + assert opening, "the delegate no longer links anything" + depth, end = 1, None + for index in range(opening.end(), len(cmake)): + if cmake[index] == "(": + depth += 1 + elif cmake[index] == ")": + depth -= 1 + if depth == 0: + end = index + break + assert end is not None, "unbalanced target_link_libraries call" + linked = cmake[opening.end() : end] + for required in ("executorch::runtime", "executorch::extension_cuda"): + assert required in linked, f"the delegate does not link {required}" + + code = [line for line in cmake.splitlines() if not line.lstrip().startswith("#")] + for forbidden in ("add_subdirectory", "EXECUTORCH_BUILD_"): + offenders = [line for line in code if forbidden in line] + assert not offenders, ( + f"{forbidden} builds ExecuTorch from source, which defeats the point of " + f"linking its prebuilt runtime: {offenders}" + ) + + +@pytest.mark.unit +def test_the_delegate_checks_the_runtime_and_platform_policy(): + """The runtime supplies registration; the platform supplies system symbol versions.""" + cmake = ( + _REPO_ROOT / "py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt" + ).read_text(encoding="utf-8") + guard = ( + _REPO_ROOT + / "py/torch-tensorrt-executorch-runtime/native/check_imports_executorch_runtime.sh" + ).read_text(encoding="utf-8") + + # The runtime has to be handed to the guard, or it has nothing to compare against. + assert "$" in cmake + for family in ("GLIBCXX", "CXXABI"): + assert family in guard, f"the guard does not look at {family} versions" + assert "policy_versions" in guard + assert "manylinux_2_28_x86_64" in guard + assert "manylinux_2_35_aarch64" in guard + + +@pytest.mark.unit +def test_the_delegate_ships_no_absolute_runpath(): + """ExecuTorch's imported targets add the build machine's own path, and it must not ship. + + The entry arrives as a raw ``INTERFACE_LINK_OPTIONS`` ``-rpath``, so + ``BUILD_WITH_INSTALL_RPATH`` does not suppress it and ``cmake --install`` does not rewrite + it. It also sorts ahead of the relative entries, so on any host whose site-packages path + matches the builder's the loader never consults ``$ORIGIN/../../executorch/lib`` -- which is + what let an earlier wrong RUNPATH depth go unnoticed. Stripping it is what makes the + relative entries load-bearing, and therefore testable. + """ + cmake = ( + _REPO_ROOT / "py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt" + ).read_text(encoding="utf-8") + guard = ( + _REPO_ROOT + / "py/torch-tensorrt-executorch-runtime/native/check_imports_executorch_runtime.sh" + ).read_text(encoding="utf-8") + + # Comments name --set-rpath in prose (the CMakeLists explains why it avoids --force-rpath), so + # deleting the whole patchelf command would leave a raw-text "--set-rpath" in cmake satisfied by + # that comment. Match the live command instead: --remove-rpath then --set-rpath reading the + # shared colon-joined declaration, all in code with comments stripped. + code = "\n".join( + line for line in cmake.splitlines() if not line.lstrip().startswith("#") ) - assert "nvidia/cu13/lib" in script - assert "cu13*)" in script + assert "--remove-rpath" in code + assert re.search( + r'--set-rpath\s*\n?\s*"\$\{TORCH_TENSORRT_DELEGATE_RUNPATH_COLONS\}"', code + ), "patchelf --set-rpath no longer reads the shared RUNPATH declaration" + declared = re.search( + r'set\(\s*TORCH_TENSORRT_DELEGATE_RUNPATH\s+((?:"[^"]+"\s*)+)\)', cmake + ) + assert ( + declared + ), "the RUNPATH list is no longer a single declaration this test can read" + for entry in re.findall(r'"([^"]+)"', declared.group(1)): + assert entry.startswith("$ORIGIN"), f"{entry} is not relative to the artifact" + # And the guard has to assert the strip happened, or a regression ships silently. + assert "not relative to the artifact" in guard @pytest.mark.unit -def test_runtime_extension_does_not_require_an_embeddable_python(): - """Development.Embed must stay optional, or the release build cannot configure. +def test_the_in_tree_target_survives_as_needed(): + """A registration-only library is dropped by --as-needed unless the link says otherwise. + + The delegate registers from a static initializer, so a consumer references no symbol from it. + Measured on a real link: with a plain target the DT_NEEDED entry disappears under + ``-Wl,--as-needed`` and the initializer never runs, silently. ExecuTorch wraps its own + registration-only component libraries in scoped retention for exactly this reason, so the alias + this file advertises has to carry it too or the advertised parity is false. + + Source-text only, deliberately: a matching text pattern still passes when the retention is + dead code (``if(FALSE)``, ``if(WIN32)``, a reordered option list). The behaviour itself is + covered by test_a_consumer_of_the_alias_keeps_the_delegate_linked, which links a consumer. + """ + cmake = ( + _REPO_ROOT / "py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt" + ).read_text(encoding="utf-8") + code = "\n".join( + line for line in cmake.splitlines() if not line.lstrip().startswith("#") + ) + # The whole option in one pattern, in order: a list whose --pop-state precedes the library, + # or whose library is not the generator expression, retains nothing. + assert re.search( + r'"LINKER:--push-state,--no-as-needed,' + r'\$,--pop-state"', + code, + ), "the retention option is missing, reordered, or no longer names the delegate" + assert re.search( + r"target_link_options\(\s*executorch_backend_tensorrt\s+INTERFACE", code + ) + # Guarded on Linux, and on nothing narrower: if(FALSE) and if(WIN32) both disable it while + # leaving the option text above intact. + guard_line = re.search( + r"if\((.*?)\)\s*\n\s*target_link_options\(\s*executorch_backend_tensorrt\s+INTERFACE", + code, + ) + assert ( + guard_line + ), "the retention is not inside a platform condition this test can read" + assert ( + guard_line.group(1) == 'CMAKE_SYSTEM_NAME STREQUAL "Linux"' + ), f"retention is conditioned on {guard_line.group(1)!r}, so it does not apply on Linux builds" + + +@pytest.mark.unit +def test_a_consumer_of_the_alias_keeps_the_delegate_linked(tmp_path): + """Link a real consumer and check the DT_NEEDED survives. + + The text assertions above pin the option's shape, but a shape is not a behaviour: the option + can be present and still retain nothing. This runs the production CMake file itself against a + stand-in target of the same name, links a consumer that references no symbol from it under + ``-Wl,--as-needed``, and requires both that the dependency is retained and that the static + initializer runs. Including the real file rather than copying a regex match out of it is what + makes an outer ``if(FALSE)`` around the retention block visible: a copy is still a copy of a + line that production may no longer execute. Skipped where the toolchain is absent. + """ + cmake_bin = shutil.which("cmake") + if cmake_bin is None or shutil.which("readelf") is None or sys.platform != "linux": + pytest.skip("needs cmake, readelf, and a Linux linker") + + production = ( + _REPO_ROOT / "py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt" + ) + # Everything from the retention comment to the install() call: the block under test, lifted + # whole so any condition wrapping it comes along. Anchored on the comment rather than the + # if(), so an outer guard cannot be left behind. + block = re.search( + r"\n(# Registration happens in a static initializer.*?)\ninstall\(TARGETS", + production.read_text(encoding="utf-8"), + re.DOTALL, + ) + assert ( + block + ), "the retention block is no longer identifiable in the production CMake file" + retention = block.group(1).replace("executorch_backend_tensorrt", "delegate") + + # Lifting the block proves the flags work; it cannot see a condition wrapped around them + # upstream. So also require the production block to be unconditional: flipping + # CMAKE_SYSTEM_NAME around it left this test green while the flags reached no build. + prologue = production.read_text(encoding="utf-8")[: block.start(1)] + open_conditions: list[str] = [] + for line in prologue.splitlines(): + stripped = line.strip() + if re.match(_CMAKE_BLOCK_OPEN, stripped, re.IGNORECASE): + open_conditions.append(stripped) + elif re.match(_CMAKE_BLOCK_CLOSE, stripped, re.IGNORECASE) and open_conditions: + open_conditions.pop() + assert not open_conditions, ( + "the retention block sits inside a conditional, so the flags it sets may not reach the " + f"build this test proves them against: {open_conditions}" + ) + + (tmp_path / "reg.cpp").write_text( + '#include \nnamespace { struct R { R() { printf("registered\\n"); } } r; }\n' + ) + (tmp_path / "main.cpp").write_text("int main() { return 0; }\n") + (tmp_path / "CMakeLists.txt").write_text( + "cmake_minimum_required(VERSION 3.24)\n" + "project(retention CXX)\n" + "add_library(delegate SHARED reg.cpp)\n" + f"{retention}\n" + "add_library(ns::delegate ALIAS delegate)\n" + "add_executable(app main.cpp)\n" + "target_link_options(app PRIVATE -Wl,--as-needed)\n" + "target_link_libraries(app PRIVATE ns::delegate)\n" + 'set_target_properties(app PROPERTIES BUILD_RPATH "$ORIGIN")\n' + ) + build = tmp_path / "build" + # Fail rather than skip: the fixture is generated from the production block, so a + # configure or build error is usually that block being malformed, which is the thing under + # test. Skipping on it would turn the interesting failure into a silent pass. + for stage in ( + [cmake_bin, "-S", str(tmp_path), "-B", str(build)], + [cmake_bin, "--build", str(build)], + ): + done = subprocess.run(stage, capture_output=True, text=True) + assert done.returncode == 0, ( + f"the fixture project failed at {' '.join(stage[1:3])}:\n" + f"{done.stdout}\n{done.stderr}" + ) + + needed = subprocess.run( + ["readelf", "-dW", str(build / "app")], capture_output=True, text=True + ).stdout + assert ( + "libdelegate.so" in needed + ), "the linker dropped the registration-only dependency despite the retention option" + ran = subprocess.run([str(build / "app")], capture_output=True, text=True) + assert "registered" in ran.stdout, "the static initializer never ran" + - ExecuTorch declares its pybind modules SHARED, so CMake requires the - Python::Python target and suggests asking for Development.Embed. Taking that - suggestion breaks the build: the release image's CPython ships no libpython, so - the component cannot be satisfied and the whole find_package fails. The - component is therefore requested optionally, matching pybind11, and the target - is stood in for when it is absent. +def _assert_the_checker_is_reachable(prologue: str) -> None: + """Fail if anything in ``prologue`` can stop the wheel checker from running. + + Checked with ``bash -n`` and anchored scans, never by executing. An earlier version sliced the + raw YAML and ran it with ``bash -c``, which downloaded bazelisk, put it on PATH and pip + installed ExecuTorch, once per parameter case. + """ + parsed = subprocess.run( + ["bash", "-n", "-c", prologue], capture_output=True, text=True + ) + # An unterminated compound command leaves the prologue an incomplete script, which is what a + # condition wrapped around the checker produces. + assert "unexpected end of file" not in parsed.stderr, ( + "the wheel checker runs under an unterminated shell condition, so the rules this test " + f"proves may never execute in CI: {parsed.stderr.strip()[:200]}" + ) + # Syntax is not reachability: these parse cleanly and still skip the checker. + for pattern, why in ( + (r"^[ \t]*if\b[^\n]*\bfi[ \t]*$", "an inline conditional"), + (r"^[ \t]*(?:false|true)[ \t]*(?:&&|\|\|)", "a short-circuit that skips it"), + ): + offender = re.search(pattern, prologue, re.MULTILINE) + assert not offender, ( + f"the wheel checker sits after {why}, so it may never run: " + f"{offender.group(0).strip()[:80]!r}" + ) + # An unconditional exit skips the checker whatever its indentation, so a column-0 scan misses + # an indented `exit 0`. But the real prologue legitimately exits from inside a case arm for an + # unsupported platform, so a scan that flags any indented exit is a false positive. Track block + # depth instead: exit, exec or return is unconditional only at depth 0, outside every + # if/case/for/while/until block. + # + # Split each line into commands on the shell separators too, or `: && exit 0`, `foo || exit 1` + # and `{ exit 0; }` slip past a scan that only reads the first word: the exit is unconditional + # but does not start the line. Heredoc bodies are skipped rather than scanned, or a body line + # beginning with `if` desynchronises the depth counter and hides a later top-level exit. `exec` + # is only a bypass when it replaces the shell with another program: a bare `exec 3>&1` or + # `exec >log` is a redirection that returns, so it does not count. + depth = 0 + block_opener = re.compile(r"^\s*(?:if|case|for|while|until|select)\b") + block_closer = re.compile(r"^\s*(?:fi|esac|done)\b") + heredoc_delimiter = None + for raw_line in prologue.splitlines(): + stripped = raw_line.strip() + if heredoc_delimiter is not None: + if stripped == heredoc_delimiter: + heredoc_delimiter = None + continue + if not stripped or stripped.startswith("#"): + continue + opening_heredoc = re.search( + r"<<-?\s*[\"']?([A-Za-z_][A-Za-z0-9_]*)[\"']?", raw_line + ) + if opening_heredoc: + heredoc_delimiter = opening_heredoc.group(1) + if block_closer.match(raw_line): + depth = max(0, depth - 1) + continue + # An exit inside an inline block on this same line is conditional, not a bypass: + # `if ...; then exit 1; fi` and `case x in ...) exit 0 ;; esac` guard the exit behind + # `then` or `do` or a case pattern. Only the part of the line before any such keyword runs + # unconditionally, so scan that prefix: it still sees `: && exit 0`, `foo || exit 1` and + # `{ exit 0; }`, but not an exit the same line makes conditional. + unconditional_prefix = re.split(r"\b(?:then|do|in)\b", raw_line, maxsplit=1)[0] + if depth == 0: + for command in re.split(r"&&|\|\||;|\{|\}", unconditional_prefix): + command = command.strip() + if re.match(r"(?:exit|return)\b", command) or re.match( + r"exec\s+[^0-9<>&]", command + ): + raise AssertionError( + "the wheel checker sits after an unconditional exit, so it may never " + f"run: {stripped[:80]!r}" + ) + if block_opener.match(raw_line) and not re.search( + r"\b(?:fi|esac|done)\b", stripped + ): + depth += 1 + + +@pytest.mark.unit +@pytest.mark.parametrize( + "case,expect_pass", + [ + ("good", True), + ("no_register_backend", False), + ("defines_register_backend", False), + ("no_runpath", False), + ("wrong_depth_runpath", False), + ("dt_rpath", False), + ("runpath_missing_executorch", False), + ("absolute_runpath", False), + ("floor_above_runtime", False), + ("no_cxxabi", False), + ("glibc_above_runtime", False), + ("named_node_missing_from_runtime", True), + ("lower_compatible_nodes", True), + # GLIBCXX_3.4.22 is below the manylinux_2_28 ceiling of 3.4.24, so it is legitimately + # accepted now: the guard compares against the platform, not against the sibling wheel. + ("one_std_thread_above_the_runtime", True), + ("above_the_runtime", False), + # GCC_4.8.0 is below the platform ceiling, so no longer a rejection. + ("gcc_above_the_runtime", True), + ("cxxabi_above_the_runtime", False), + # A family the runtime declares nothing from is fine: the host provides libstdc++, not the + # sibling wheel. GLIBCXX_3.4.21 is far below the platform ceiling. + ("family_absent_from_runtime", True), + ("no_needed_executorch", False), + ("no_needed_extension_cuda", False), + ("no_needed_libstdcxx", False), + ("cuda_12_runtime", False), + ("cuda_13_runtime", True), + ("runtime_only_imports_register_backend", False), + ("runtime_exports_a_near_miss", False), + ("elfutils_bare_rpath", False), + ("elfutils_undef_dialect", True), + ("readelf_v_broken", False), + ("readelf_broken", False), + # The --dyn-syms check: an unversioned std C++ UND is rejected, a versioned one accepted, + # and a readelf that cannot list dynamic symbols fails closed. + ("unversioned_cxx_undef", False), + ("versioned_cxx_undef", True), + ("readelf_dynsyms_broken", False), + # Policy membership must not depend on the runtime's numeric or textual node order. + ("runtime_numbered_nodes_out_of_text_order", True), + # Exercises the 4-argument exact-whole-RUNPATH branch production always selects. + ("runpath_missing_a_sibling", False), + # Keeps the 3-argument fallback branch covered. + ("runpath_fallback_reaches_executorch", True), + # The pybindings extension seeds the symbol-version ceiling; a missing one must hard-fail + # rather than silently narrow the guard. + ("no_pybindings_extension", False), + ], +) +def test_the_guard_actually_rejects_a_bad_artifact(tmp_path, case, expect_pass): + """Run the guard, rather than reading it. + + Every other assertion in this file checks that the guard's *source* contains certain words. + None of them notice if the guard is never invoked, or returns 0 unconditionally: replacing + ``COMMAND sh`` with ``COMMAND true``, or inserting ``exit 0`` after ``set -u``, leaves them + all green. The guard takes readelf as its first argument precisely so it can be driven, so + drive it with a stub and require the right exit status for each artifact shape. + """ + guard = ( + _REPO_ROOT + / "py/torch-tensorrt-executorch-runtime/native/check_imports_executorch_runtime.sh" + ) + + # Default to the whole RUNPATH the build declares, so the exact-match branch passes and the + # version and symbol checks are what decide each case. The RUNPATH-shape cases below override + # it and run on the 3-argument fallback. + runpaths = { + "no_runpath": None, + # A run path entry at the wrong depth, so it names an executorch/lib that is not the one + # beside this wheel. + "wrong_depth_runpath": "$ORIGIN:$ORIGIN/xy/executorch/lib", + "runpath_missing_executorch": "$ORIGIN:$ORIGIN/../torch/lib", + "absolute_runpath": "$ORIGIN:$ORIGIN/../../executorch/lib:/build/site-packages/lib", + # A RUNPATH the loader could resolve on the build host but that omits two of the four + # sibling directories. Only the exact-whole-set branch, which production always selects, + # rejects it; the fallback that spot-checks executorch/lib alone lets it through. + "runpath_missing_a_sibling": "$ORIGIN:$ORIGIN/../../executorch/lib", + # The 3-argument fallback: a bare relative RUNPATH that reaches executorch/lib is accepted + # when no build string is given, which keeps that branch covered. + "runpath_fallback_reaches_executorch": "$ORIGIN:$ORIGIN/../../executorch/lib", + "dt_rpath": _GUARD_GOOD_RUNPATH, + "elfutils_bare_rpath": _GUARD_GOOD_RUNPATH, + } + default_runpath = _GUARD_GOOD_RUNPATH + tag = "RPATH" if case == "dt_rpath" else "RUNPATH" + # No DT_NEEDED on the runtime means the delegate resolves register_backend from nowhere. This + # case drops only this line and keeps the extension_cuda line below, so the libexecutorch.so + # branch is the one that rejects it and the case pins that branch. + dyn = ( + "" + if case == "no_needed_executorch" + else " 0x0000000000000001 (NEEDED) Shared library: [libexecutorch.so]\n" + ) + # extension_cuda is linked PRIVATE and shared, so a well-formed delegate carries this DT_NEEDED. + # The no_needed_extension_cuda case drops it to exercise the static-link rejection. The + # no_needed_executorch case keeps it, so only the libexecutorch.so line is missing and the + # branch that case is named for is the one that fires, not this one two checks below. + if case != "no_needed_extension_cuda": + dyn += ( + " 0x0000000000000001 (NEEDED) Shared library: " + "[libexecutorch_extension_cuda.so]\n" + ) + # The delegate calls out-of-line libstdc++ functions, so a well-formed one records a + # DT_NEEDED on the C++ runtime. The no_needed_libstdcxx case drops it to exercise the + # under-linked rejection the guard adds for the symbol lld silently discards. + if case != "no_needed_libstdcxx": + dyn += " 0x0000000000000001 (NEEDED) Shared library: [libstdc++.so.6]\n" + # The toolkit can differ from torch.version.cuda, so check the linked major too. + if case == "cuda_12_runtime": + dyn += " 0x0000000000000001 (NEEDED) Shared library: [libcudart.so.12]\n" + if case == "cuda_13_runtime": + dyn += " 0x0000000000000001 (NEEDED) Shared library: [libcudart.so.13]\n" + rp = runpaths.get(case, default_runpath) + if rp: + if case == "elfutils_bare_rpath": + # eu-readelf prints the tag bare, where binutils parenthesises it. The guard claims to + # reject DT_RPATH in either dialect, so exercise the one binutils never emits. + dyn += f" RPATH Library rpath: [{rp}]\n" + else: + dyn += f" 0x000000000000001d ({tag}) Library runpath: [{rp}]\n" + # The mangled name, because that is what the guard greps .dynsym for. + mangled = "_ZN10executorch7runtime16register_backendERKNS0_7BackendE" + # eu-readelf spells an undefined symbol UNDEF where binutils spells it UND. Both must be + # accepted, so one case uses the elfutils spelling. + undefined = "UNDEF" if case == "elfutils_undef_dialect" else "UND" + syms = "" if case == "no_register_backend" else f" 1: {undefined} {mangled}\n" + if case == "defines_register_backend": + syms = f" 1: 000123 FUNC GLOBAL DEFAULT 12 {mangled}\n" + # What the runtime's own symbol table says. A defined export carries a section index; the + # runtime_only_imports case carries UND instead, which is a runtime that imports the symbol + # rather than providing it, and must be rejected. + runtime_syms = f" 1: 000123 82 FUNC GLOBAL DEFAULT 8 {mangled}\n" + if case == "runtime_only_imports_register_backend": + runtime_syms = f" 1: 000000 0 FUNC GLOBAL DEFAULT UND {mangled}\n" + if case == "runtime_exports_a_near_miss": + runtime_syms = ( + " 1: 000456 82 FUNC GLOBAL DEFAULT 8 " + "_ZN10executorch7runtime16register_backendERKNS0_9BackendV2E\n" + ) + # Start with one allowed CXXABI requirement and vary only the family under test. + target_v = "CXXABI_1.3.9" + if case == "floor_above_runtime": + target_v = "GLIBCXX_3.4.30 CXXABI_1.3.9" + if case == "no_cxxabi": + target_v = "" + # GLIBC has its own platform policy, independent of the C++ families. + if case == "glibc_above_runtime": + target_v = "CXXABI_1.3.9 GLIBC_2.38" + # CXXABI_TM_1 carries no dotted version, so a pattern demanding digits drops it silently. + if case == "named_node_missing_from_runtime": + target_v = "CXXABI_1.3.9 CXXABI_TM_1" + # Accepted: the platform permits this node even when the runtime does not require it. + if case == "one_std_thread_above_the_runtime": + target_v = "CXXABI_1.3.9 GLIBCXX_3.4.22" + # Must be REJECTED. Further above still: GLIBCXX_3.4.26 is GCC 9's std::filesystem. + if case == "above_the_runtime": + target_v = "CXXABI_1.3.9 GLIBCXX_3.4.26" + # GCC_4.8.0 belongs to the x86_64 policy; above-policy rejection has separate coverage. + if case == "gcc_above_the_runtime": + target_v = "CXXABI_1.3.9 GCC_4.8.0" + # Must be REJECTED. CXXABI had the same gap GCC did: dropping it from the loop left every case + # green, because every other case declares a CXXABI the runtime satisfies. + if case == "cxxabi_above_the_runtime": + target_v = "CXXABI_1.3.15" + # Accepted: the platform policy does not depend on which families the runtime uses. + if case == "family_absent_from_runtime": + target_v = "CXXABI_1.3.9 GLIBCXX_3.4.21" + if case == "lower_compatible_nodes": + target_v = "CXXABI_1.3 CXXABI_1.3.9 GLIBC_2.4 GLIBC_2.17 GLIBCXX_3.4.11 GCC_3.0" + # Accepted regardless of the order of the runtime's requirements. + if case == "runtime_numbered_nodes_out_of_text_order": + target_v = "CXXABI_1.3.9 GLIBCXX_3.4.21" + # The runtime declares a spread, not just its maximum, the way a real library does. + runtime_v = "GLIBCXX_3.4 GLIBCXX_3.4.21 CXXABI_1.3 CXXABI_1.3.9 GLIBC_2.2.5 GLIBC_2.34 GCC_3.0" + if case == "family_absent_from_runtime": + runtime_v = "CXXABI_1.3 CXXABI_1.3.9 GLIBC_2.2.5 GLIBC_2.34 GCC_3.0" + # Two numbered GLIBCXX nodes whose text order inverts their numeric order: text sort ranks + # 3.4.9 above 3.4.21, numeric sort ranks 3.4.21 above 3.4.9. + if case == "runtime_numbered_nodes_out_of_text_order": + runtime_v = ( + "GLIBCXX_3.4.9 GLIBCXX_3.4.21 CXXABI_1.3 CXXABI_1.3.9 GLIBC_2.34 GCC_3.0" + ) + # What readelf --dyn-syms lists for the target. The guard rejects an UNVERSIONED undefined std + # C++ symbol (a newer-toolchain helper the nonshared archive failed to supply) and accepts one + # that carries an @GLIBCXX version. Default to a well-formed versioned symbol so the ordinary + # cases pass this check; the two named cases below drive the reject and accept branches. + dyn_syms = ( + " 1: 0000000000000000 0 FUNC GLOBAL DEFAULT UND " + "_ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE9_M_appendEPKcm@GLIBCXX_3.4.21\n" + ) + if case == "unversioned_cxx_undef": + dyn_syms = ( + " 1: 0000000000000000 0 FUNC GLOBAL DEFAULT UND " + "_ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE15_M_replace_coldEPcmPKcmm\n" + ) + if case == "versioned_cxx_undef": + dyn_syms = ( + " 1: 0000000000000000 0 FUNC GLOBAL DEFAULT UND " + "_ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE9_M_appendEPKcm@GLIBCXX_3.4.21\n" + ) + stub = tmp_path / "readelf" + stub.write_text( + "#!/bin/sh\n" + ("exit 3\n" if case == "readelf_broken" else "") + # -V broken for the target only, not the runtime: the case is named for the target-side + # read, so breaking both lets the runtime-side read decide it and the named branch never + # runs. The target is the second argument to -V; the runtime ends in libexecutorch.so. + + ( + '[ "$1" = "-V" ] && case "$2" in *libexecutorch.so) ;; *) exit 3 ;; esac\n' + if case == "readelf_v_broken" + else "" + ) + + ( + '[ "$1" = "--dyn-syms" ] && exit 3\n' + if case == "readelf_dynsyms_broken" + else "" + ) + + 'case "$1" in\n' + f" -d) printf %s '{dyn}' ;;\n" + f" --dyn-syms) printf %s '{dyn_syms}' ;;\n" + ' -Ws) case "$2" in\n' + f" *libexecutorch.so) printf %s '{runtime_syms}' ;;\n" + f" *) printf %s '{syms}' ;;\n" + " esac ;;\n" + ' -V) case "$2" in\n' + f" *libexecutorch.so) echo '{runtime_v}' ;;\n" + f" *) echo '{target_v}' ;;\n" + " esac ;;\n" + "esac\n", + encoding="utf-8", + ) + stub.chmod(0o755) + # The runtime lives in a per-case lib/ subdirectory so the guard's ../extension/pybindings path + # resolves inside this case's tmp_path rather than the parent tmp_path shared across the whole + # parametrization. Sharing it let one case's pybindings stub leak into no_pybindings_extension. + libdir = tmp_path / "lib" + libdir.mkdir() + target = libdir / "libexecutorch_backend_tensorrt.so" + target.write_bytes(b"\x7fELF") + runtime = libdir / "libexecutorch.so" + runtime.write_bytes(b"\x7fELF") + + # The guard seeds its symbol-version ceiling from the pybindings extension, which sits at + # runtime_dir/../extension/pybindings/_C.*.so, so a well-formed layout has it. The + # no_pybindings_extension case leaves it out to exercise the hard-fail that keeps a missing + # extension from silently narrowing the ceiling. + if case != "no_pybindings_extension": + pybindings_dir = tmp_path / "extension" / "pybindings" + pybindings_dir.mkdir(parents=True, exist_ok=True) + (pybindings_dir / "_C.cpython-311-x86_64-linux-gnu.so").write_bytes(b"\x7fELF") + + # Production always passes the fourth argument (TORCH_TENSORRT_DELEGATE_RUNPATH_COLONS, the + # RUNPATH the build declares), which selects the exact-whole-RUNPATH branch. Drive that branch + # here with the RUNPATH the build declares, except for the cases kept on the 3-argument + # fallback so it stays covered too. + argv = ["sh", str(guard), str(stub), str(target), str(runtime)] + if case not in _THREE_ARG_CASES: + argv.append(_GUARD_GOOD_RUNPATH) + # The manylinux tag the row ships under. Without it the guard skips the symbol version + # ceiling entirely, so every ceiling case would pass for the wrong reason. + argv.append("manylinux_2_28_x86_64") + + result = subprocess.run( + argv, + capture_output=True, + text=True, + ) + # Named branches whose parametrize case must reach that branch and no other. Asserting the + # exit status alone let a case pass by any route that also exits non-zero: the three readelf + # and no-RUNPATH cases each survived their own branch being deleted because a later check still + # failed. Requiring the branch's own message pins each case to the branch it is named for. + expected_messages = { + "no_runpath": "carries no RUNPATH", + "no_needed_executorch": "has no DT_NEEDED on libexecutorch.so", + "no_needed_libstdcxx": "has no DT_NEEDED on libstdc++", + "cuda_12_runtime": "this delegate requires CUDA 13", + "readelf_broken": "could not inspect", + "readelf_v_broken": "could not read symbol versions of", + "unversioned_cxx_undef": "unversioned undefined C++ runtime symbols", + "readelf_dynsyms_broken": "could not read the dynamic symbols of", + "no_pybindings_extension": "could not find the pybindings extension", + } + if expect_pass: + assert result.returncode == 0, result.stdout + result.stderr + else: + assert result.returncode != 0, f"{case} was accepted:\n{result.stdout}" + expected = expected_messages.get(case) + if expected is not None: + assert expected in result.stderr, ( + f"{case} failed, but not through its own branch: expected {expected!r} in\n" + f"{result.stderr}" + ) + + +@pytest.mark.unit +def test_the_guard_is_wired_into_the_build(): + """The guard has to be invoked, not merely present. + + Replacing ``COMMAND sh`` with ``COMMAND true`` in the POST_BUILD rule disables the check + completely and leaves every source-text assertion in this file green, so pin the wiring: + a POST_BUILD command on the delegate that runs this script with the two artifacts. """ cmake = ( _REPO_ROOT / "py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt" ).read_text(encoding="utf-8") + code = "\n".join( + line for line in cmake.splitlines() if not line.lstrip().startswith("#") + ) + invocation = re.search( + r"add_custom_command\(\s*TARGET\s+executorch_backend_tensorrt\s+POST_BUILD\s+" + r"COMMAND\s+sh\s+\"\$\{CMAKE_CURRENT_LIST_DIR\}/check_imports_executorch_runtime\.sh\"", + code, + ) + assert invocation, "the guard is not invoked by a POST_BUILD command running sh" + # The enclosing condition too. Mitigated by the FATAL_ERROR above it on Linux, but the + # invocation being present says nothing about whether it is reached, and if(FALSE) here left + # every other assertion in this test green. + # Track block depth to the invocation rather than matching the nearest condition, so that + # every enclosing condition is checked and not just the innermost. + enclosing: list[str] = [] + for line in code.splitlines(): + stripped = line.strip() + opened = re.match(_CMAKE_BLOCK_OPEN, stripped, re.IGNORECASE) + if opened: + # Strip whatever the command name actually was, since a fixed-width slice assumes + # one spelling and garbles "if (X)" and every command longer than "if". + enclosing.append(stripped[opened.end() :].strip().rstrip(")")) + elif re.match(r"else\s*\(|elseif\s*\(", stripped, re.IGNORECASE): + # Case-insensitive like the opens and closes. An uppercase ELSE() was a false accept: + # it left the recorded condition untouched while control moved into the else branch. + if enclosing: + enclosing[-1] = stripped + elif re.match(_CMAKE_BLOCK_CLOSE, stripped, re.IGNORECASE): + if enclosing: + enclosing.pop() + elif "check_imports_executorch_runtime.sh" in stripped: + break + else: + raise AssertionError( + "the guard invocation was not found while scanning conditions" + ) + # And that nothing reassigns the variable before the block reads it: the condition being + # spelled correctly says nothing if TORCH_TENSORRT_READELF is cleared one line above. The + # variable is only ever meant to come from find_program, so reject any set() of it before the + # guard regardless of the value. Enumerating CMake's falsy literals missed the quoted forms + # set(TORCH_TENSORRT_READELF "OFF") and set(TORCH_TENSORRT_READELF "" CACHE INTERNAL ""), each + # of which is falsy to if() and disables the whole block. + guard_at = code.index("check_imports_executorch_runtime.sh") + disabled = re.search( + r"set\(\s*TORCH_TENSORRT_READELF\b", + code[:guard_at], + re.IGNORECASE, + ) + assert ( + not disabled + ), "TORCH_TENSORRT_READELF is reassigned before the guard block, so the guard may never run" + assert enclosing == ["TORCH_TENSORRT_READELF"], ( + "the guard's POST_BUILD command must be reached whenever readelf exists, but it sits " + f"under {enclosing}" + ) + # Both artifacts, or the symbol-floor comparison silently degrades to the two-argument form. + assert "$" in code + assert "$" in code + # And the build's own RUNPATH string, or the guard falls back to spot-checking one entry and a + # delegate missing tensorrt_libs or the CUDA entry ships. Scoped to the invocation's argument + # list, since the variable is also set earlier in the file where patchelf consumes it. + arguments = code[invocation.end() : code.index("VERBATIM", invocation.end())] + assert "${TORCH_TENSORRT_DELEGATE_RUNPATH_COLONS}" in arguments, ( + "the guard is not given the RUNPATH the build asks for, so it cannot compare the whole " + "set and a missing entry ships" + ) + - assert "REQUIRED COMPONENTS Interpreter Development.Module" in cmake - assert "if(NOT TARGET Python::Python)" in cmake +@pytest.mark.unit +def test_the_delegate_is_exported_the_way_executorch_exports_its_backends(): + """The delegate must be linkable in-tree as executorch::backend_tensorrt. - # Every mention of the component in actual code, comments excluded, must be an - # optional one. A required request is what fails on an image without libpython. + Match ExecuTorch's backend naming in-tree. Installed consumers use the wheel's + separate CMake package, which discovers the installed runtime and delegate. + """ + cmake = ( + _REPO_ROOT / "py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt" + ).read_text(encoding="utf-8") + + # The target, and therefore the shipped libexecutorch_backend_tensorrt.so. + assert "add_library(executorch_backend_tensorrt SHARED" in cmake + assert ( + "add_library(executorch::backend_tensorrt ALIAS executorch_backend_tensorrt)" + in cmake + ) + # The old name would ship the library as libtorch_tensorrt_executorch_backend.so. + assert "torch_tensorrt_executorch_backend" not in cmake + # The wheel's hand-written config defines its imported target without an export set. code = [line for line in cmake.splitlines() if not line.lstrip().startswith("#")] - embed_lines = [line for line in code if "Development.Embed" in line] - assert embed_lines, "Development.Embed should be requested, optionally" - for line in embed_lines: - assert "OPTIONAL_COMPONENTS" in line, ( - "Development.Embed must stay optional; the release image has no " - f"libpython: {line.strip()!r}" - ) + assert not [line for line in code if "install(EXPORT" in line] + + # The package withholds every imported target below this, without failing find_package, + # so a lower floor would configure cleanly and then fail on the first executorch:: target. + assert "cmake_minimum_required(VERSION 3.28)" in cmake + + +def _runtime_setup_tree(): + return ast.parse(_RUNTIME_SETUP_PY.read_text(encoding="utf-8")) def _setup_tree(): @@ -297,6 +1753,394 @@ def _function_def(tree, name): raise AssertionError(f"Could not find function {name}") +@pytest.mark.unit +def test_runtime_wheel_uses_public_torch_version(): + function = _function_def(_runtime_setup_tree(), "public_version") + namespace = {} + exec( + compile(ast.Module(body=[function], type_ignores=[]), "", "exec"), + namespace, + ) + + assert namespace["public_version"]("2.14.0.dev20260726+cu132") == ( + "2.14.0.dev20260726" + ) + + +@pytest.mark.unit +def test_runtime_wheel_version_is_independent_of_the_main_wheel(monkeypatch, tmp_path): + """The companion uses its own base version, including for source-only builds.""" + function = _function_def(_runtime_setup_tree(), "get_runtime_version") + (tmp_path / "version.txt").write_text("0.1.0\n") + namespace = { + "os": os, + "subprocess": subprocess, + "REPO_ROOT": _REPO_ROOT, + "HERE": tmp_path, + } + exec( + compile( + ast.Module(body=[function], type_ignores=[]), str(_RUNTIME_SETUP_PY), "exec" + ), + namespace, + ) + monkeypatch.setenv( + "TORCH_TENSORRT_EXECUTORCH_RUNTIME_VERSION", "0.1.0.dev20200103+cu132" + ) + assert namespace["get_runtime_version"]() == "0.1.0.dev20200103+cu132" + monkeypatch.delenv("TORCH_TENSORRT_EXECUTORCH_RUNTIME_VERSION") + monkeypatch.setattr(subprocess, "check_output", lambda *args, **kwargs: "abcdef0\n") + assert namespace["get_runtime_version"]() == "0.1.0.dev0+abcdef0" + + +@pytest.mark.unit +@pytest.mark.parametrize( + "requested", + [ + "0.1.0", + "0.2.0 EXACT", + "0.3.0", + "2.15.0", + "0.1...0.2", + "0.1...<0.2", + "0.2...0.3", + "0.1...<1", + "0.1...1", + "0.1...<1.1", + "0.1...<2", + ], +) +def test_cmake_version_uses_the_companion_distribution(tmp_path, requested): + """Compare real find_package results with CMake's SameMajorVersion generator.""" + cmake = shutil.which("cmake") + if cmake is None: + pytest.skip("cmake is not installed") + build = next( + node + for node in _runtime_setup_tree().body + if isinstance(node, ast.ClassDef) and node.name == "BazelBuild" + ) + namespace = { + "build_py": object, + "pathlib": pathlib, + "shutil": shutil, + "re": re, + "_CMAKE_CONFIG_SOURCE": _RUNTIME_SETUP_PY.parent + / "cmake/torchtrt_executorch-config.cmake", + } + exec( + compile( + ast.Module(body=[build], type_ignores=[]), str(_RUNTIME_SETUP_PY), "exec" + ), + namespace, + ) + command = namespace["BazelBuild"]() + command.distribution = types.SimpleNamespace( + get_version=lambda: "0.2.0.dev20200103+cu132" + ) + command._install_cmake_package(tmp_path) + version_file = ( + tmp_path + / "lib/cmake/torchtrt_executorch/torchtrt_executorch-config-version.cmake" + ) + text = version_file.read_text() + assert 'set(PACKAGE_VERSION "0.2.0")' in text + assert 'set(TORCHTRT_EXECUTORCH_FULL_VERSION "0.2.0.dev20200103+cu132")' in text + standard = tmp_path / "standard.cmake" + generator = tmp_path / "generate.cmake" + generator.write_text( + "include(CMakePackageConfigHelpers)\n" + f'write_basic_package_version_file("{standard}" VERSION "0.2.0" ' + "COMPATIBILITY SameMajorVersion ARCH_INDEPENDENT)\n" + ) + result = subprocess.run( + [cmake, "-P", str(generator)], capture_output=True, text=True + ) + assert result.returncode == 0, result.stdout + result.stderr + found = [] + for name, version in (("generated", version_file), ("standard", standard)): + prefix = tmp_path / name / "prefix" + prefix.mkdir(parents=True) + (prefix / "probe-config.cmake").write_text("set(probe_FOUND TRUE)\n") + shutil.copyfile(version, prefix / "probe-config-version.cmake") + source = tmp_path / name / "app" + source.mkdir() + (source / "CMakeLists.txt").write_text( + "cmake_minimum_required(VERSION 3.28)\nproject(probe LANGUAGES NONE)\n" + f'find_package(probe {requested} QUIET CONFIG PATHS "{prefix}" NO_DEFAULT_PATH)\n' + 'file(WRITE "${CMAKE_BINARY_DIR}/found.txt" "${probe_FOUND}")\n' + ) + binary = tmp_path / name / "build" + result = subprocess.run( + [cmake, "-S", str(source), "-B", str(binary)], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + found.append((binary / "found.txt").read_text()) + assert ( + found[0] == found[1] + ), f"{requested}: generated={found[0]}, standard={found[1]}" + + +@pytest.mark.unit +@pytest.mark.parametrize("requested", ["0.1...1", "0.1...<1.1", "0.1...<2"]) +def test_cmake_version_rejects_missing_range_major_guard( + monkeypatch, tmp_path, requested +): + copyfile = shutil.copyfile + + def without_range_major_guard(source, destination, **kwargs): + result = copyfile(source, destination, **kwargs) + if source.name == "torchtrt_executorch-config-version.cmake": + text, count = re.subn( + r' if\(PACKAGE_FIND_VERSION_RANGE_MAX STREQUAL "INCLUDE"\n' + r".*? elseif\(PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION_MIN\)", + " if(PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION_MIN)", + Path(destination).read_text(), + flags=re.DOTALL, + ) + assert count == 1 + Path(destination).write_text(text) + return result + + monkeypatch.setattr(shutil, "copyfile", without_range_major_guard) + with pytest.raises(AssertionError, match="generated=1, standard=0"): + test_cmake_version_uses_the_companion_distribution(tmp_path, requested) + + +@pytest.mark.unit +def test_runtime_readme_build_recipe_sets_the_version(): + """The optional version override in the build recipe must name the companion, not its dependency.""" + readme = (_REPO_ROOT / "py/torch-tensorrt-executorch-runtime/README.md").read_text( + encoding="utf-8" + ) + recipes = [ + block + # Any language tag, not just bash/sh: the README also carries a ```cmake block, and a + # pattern that does not recognise an opener treats it as content, which shifts every + # fence pair after it and hides the recipe this test exists to read. + for block in re.findall(r"```[a-zA-Z]*\n(.*?)```", readme, re.DOTALL) + if "pip wheel" in block + ] + assert recipes, "the README no longer carries a pip wheel build recipe" + variable = "TORCH_TENSORRT_EXECUTORCH_RUNTIME_VERSION" + for recipe in recipes: + exports = [ + line + for line in recipe.splitlines() + if re.match(rf"\s*(?:export\s+)?{variable}=", line) + and not line.lstrip().startswith("#") + ] + assert exports, ( + f"the build recipe does not set {variable} on an uncommented line, so the build would " + "stop rather than record the version the wheel requires" + ) + for line in exports: + value = line.split("=", 1)[1].strip().strip("\"'") + from packaging.version import Version + + assert ( + Version(value).base_version + == (_RUNTIME_SETUP_PY.parent / "version.txt").read_text().strip() + ) + + +def _co_names_and_consts(code) -> list[str]: + """Every name and string constant the compiled code actually carries, nested code included. + + Reading the code object rather than the source is what makes the reachability check meaningful: + a commented-out tail leaves no trace here, while it stays fully visible to a substring search of + the text. + """ + import types as _types + + found = [*code.co_names, *code.co_varnames] + for constant in code.co_consts: + if isinstance(constant, str): + found.append(constant) + elif isinstance(constant, _types.CodeType): + found.extend(_co_names_and_consts(constant)) + return found + + +@pytest.mark.unit +def test_device_export_checks_the_serialized_boundary(): + """Keep serialized boundary checks alongside the separate workflow execution tests.""" + export = ( + _REPO_ROOT / "examples/torchtrt_executorch_example/export_device_resident.py" + ).read_text(encoding="utf-8") + for fragment in ( + "skip_h2d_for_method_inputs=True", + "skip_d2h_for_method_outputs=True", + "alloc_graph_input=False", + "alloc_graph_output=False", + "_h2d_copy", + "_d2h_copy", + "plan.operators", + ): + assert fragment in export, ( + f"export_device_resident.py no longer contains {fragment!r}, so it does not " + "prove the boundary copies are absent" + ) + + +@pytest.mark.unit +def test_ci_runtime_check_asserts_the_delegate_loads_and_registers(): + """Keep the installed delegate load and backend-registration check effective. + + Numerical export and execution are separate checks in the same test lane. + This check must fail through sys.exit even under optimized Python. + """ + import yaml + + workflow_text = ( + _REPO_ROOT / ".github/workflows/executorch-test-linux.yml" + ).read_text(encoding="utf-8") + + match = re.search(r"runtime_check='([^']*)'", workflow_text) + assert ( + match + ), "executorch-test-linux.yml no longer defines a runtime_check one-liner" + runtime_check = match.group(1) + + # Every required fragment has to be REACHABLE, not merely present. The one-liner is Python, so a + # single "#" anywhere in it comments out the rest of the statement while leaving the text intact + # for the substring checks below: `import sys, torch; # ...the whole check...` satisfies all of + # them and exits 0 on a machine with no delegate installed. Commenting out the leading print is + # the natural first move when debugging this step, and the print comes first in the string. The + # sibling reachability test strips whole-line YAML comments for the same reason, but a mid-line + # "#" here is a Python comment, so that pass does not see it. + # + # Compile the string and read the code object, which is the only view that agrees with what the + # interpreter will actually run. + compiled = compile(runtime_check, "", "exec") + reachable = "\n".join( + instruction + for instruction in _co_names_and_consts(compiled) + if isinstance(instruction, str) + ) + + for fragment in ( + "torch_tensorrt_executorch_runtime", + "BACKEND_NAME", + "Runtime", + "is_available", + "XnnpackBackend", + "CudaBackend", + "sys.exit", + ): + assert fragment in runtime_check, ( + f"the CI runtime check no longer contains {fragment!r}, so it no longer proves the " + f"delegate loads and registers: {runtime_check}" + ) + # The attribute chains appear in the code object split across names, so compare on the last + # segment, which is the part a comment would remove. + needle = fragment.rsplit(".", 1)[-1] + assert needle in reachable, ( + f"the CI runtime check mentions {fragment!r} but the interpreter never reaches it, so " + "the step would pass without loading the delegate. A '#' inside the one-liner comments " + f"out the rest of the statement: {runtime_check}" + ) + # assert would be compiled out under python -O, which is why the check uses sys.exit; guard + # that reasoning too, so a rewrite back to assert is caught. + assert "assert " not in runtime_check, ( + "the CI runtime check uses assert, which python -O compiles out, so a runtime with no " + "backends registered would pass" + ) + + # The string has to actually run in a step, or asserting its content proves nothing. Parse the + # workflow and require a script that both defines it and runs it as the invocation whose exit + # status becomes the step's. A second, identical-looking call sits inside + # `if [[ "${check_status}" -ne 0 ]]` as a gdb backtrace and ends in `|| true`, so it runs only + # after the check has already failed and can never fail the job; a plain substring search is + # satisfied by that decoy even when the real call is neutered. Anchor on the executing form: + # the env-prefixed call that begins its line. The gdb copy begins with `--args python`, so it + # does not match, and replacing the status-bearing call with `true ||` turns this red. + document = yaml.safe_load(workflow_text) + scripts = [ + text + for job in document["jobs"].values() + if isinstance(job, dict) + for text in ( + [str((job.get("with") or {}).get("script") or "")] + + [ + str(step.get("run") or "") + for step in job.get("steps") or [] + if isinstance(step, dict) + ] + ) + ] + executing = [ + text + for text in scripts + if "runtime_check='" in text + and re.search( + r'^\s*PYTHONFAULTHANDLER=1 python -u -X faulthandler -c "\$\{runtime_check\}"', + text, + re.MULTILINE, + ) + ] + assert executing, ( + "no workflow script both defines runtime_check and runs it as the status-bearing " + "invocation, so the delegate load check does not execute in CI" + ) + + +@pytest.mark.unit +@pytest.mark.parametrize("cuda", [None, "12.6", "12.8", "13.0", "13.2", "14.0"]) +def test_runtime_wheel_pins_its_cuda_13_dependencies(monkeypatch, cuda): + """Run setup with simulated installed versions and inspect the metadata it hands setuptools.""" + import runpy + + import setuptools + + fake_torch = types.ModuleType("torch") + fake_torch.version = types.SimpleNamespace(cuda=cuda) + fake_torch.__version__ = "2.15.0.dev20200103+cu130" + monkeypatch.setitem(sys.modules, "torch", fake_torch) + monkeypatch.setenv( + "TORCH_TENSORRT_EXECUTORCH_RUNTIME_VERSION", "0.1.0.dev20200103+cu130" + ) + versions = { + "executorch": "1.5.0.dev20200103+cu130", + "tensorrt-cu13": "11.2.1", + "nvidia-cuda-runtime": "13.0.0", + "torch-tensorrt": "2.15.0.dev20200102+cu130", + } + queries = [] + original_version = importlib.metadata.version + + def installed_version(name): + if name in versions or name in {"tensorrt-cu12", "nvidia-cuda-runtime-cu12"}: + queries.append(name) + return versions[name] + return original_version(name) + + monkeypatch.setattr(importlib.metadata, "version", installed_version) + metadata = {} + monkeypatch.setattr(setuptools, "setup", lambda **kwargs: metadata.update(kwargs)) + if cuda not in {"13.0", "13.2"}: + with pytest.raises(RuntimeError, match="CUDA 13 is required"): + runpy.run_path(str(_RUNTIME_SETUP_PY)) + assert queries == [] + assert metadata == {} + else: + runpy.run_path(str(_RUNTIME_SETUP_PY)) + assert queries == list(versions) + assert metadata["version"] == "0.1.0.dev20200103+cu130" + # The three runtimes the delegate links keep the label naming the build, because it links one + # specific build of each. The other two have no label to keep. + labelled = {"executorch", "torch-tensorrt"} + assert set(metadata["install_requires"]) == { + f"torch=={fake_torch.__version__}", + *( + f"{name}=={version if name in labelled else version.partition('+')[0]}" + for name, version in versions.items() + ), + } + + @pytest.mark.unit def test_packaging_declares_executorch_extra(): tree = _setup_tree() @@ -368,6 +2212,7 @@ def _stub_exported_program(constants, name_to_fqn=None): return types.SimpleNamespace(constants=constants, graph_signature=sig) +@_needs_exporter @pytest.mark.unit def test_resolve_lifted_custom_obj_via_signature_fqn(): # Modern torch.export: placeholder name differs from the constants FQN key. @@ -376,6 +2221,7 @@ def test_resolve_lifted_custom_obj_via_signature_fqn(): assert _resolve_lifted_custom_obj(ep, _stub_node("obj_engine")) is sentinel +@_needs_exporter @pytest.mark.unit def test_resolve_lifted_custom_obj_legacy_fallback(): # No signature mapping: fall back to a direct name/target lookup. @@ -384,6 +2230,7 @@ def test_resolve_lifted_custom_obj_legacy_fallback(): assert _resolve_lifted_custom_obj(ep, _stub_node("engine")) is sentinel +@_needs_exporter @pytest.mark.unit def test_resolve_lifted_custom_obj_signature_present_name_absent_is_none(): # A present-but-incomplete mapping must not bind a different object by name. @@ -391,12 +2238,14 @@ def test_resolve_lifted_custom_obj_signature_present_name_absent_is_none(): assert _resolve_lifted_custom_obj(ep, _stub_node("engine")) is None +@_needs_exporter @pytest.mark.unit def test_resolve_lifted_custom_obj_missing_is_none(): ep = _stub_exported_program({}, name_to_fqn=None) assert _resolve_lifted_custom_obj(ep, _stub_node("missing")) is None +@_needs_exporter @pytest.mark.unit def test_resolve_lifted_custom_obj_unwraps_fake_script_object(): class _Real: @@ -539,6 +2388,7 @@ def _lifted_constant_meta(gm, sig): (torch.float32, "cuda"), ], ) +@_needs_exporter def test_lift_preserves_constant_dtype_device(dtype, device): # Runtime gate (not a module-level skipif, which resolves at collection time # and is fragile on remote-GPU runners): skip the CUDA case only when no GPU. @@ -557,6 +2407,7 @@ def test_lift_preserves_constant_dtype_device(dtype, device): # --- lift() preserves parameter kind and requires_grad ----------------------- +@_needs_exporter @pytest.mark.unit @pytest.mark.parametrize( "dtype, requires_grad", @@ -899,3 +2750,481 @@ def test_save_executorch_real_etrecord_is_inspector_consumable(tmp_path): # The parsed record carries the edge-dialect program the Inspector correlates # runtime events against. assert getattr(record, "edge_dialect_program", None) is not None + + +_ELF_MACHINES = {"x86_64": 0x3E, "aarch64": 0xB7} + + +def _elf_object(architecture: str) -> bytes: + """The first twenty bytes of a 64-bit shared object, which is all the checker reads. + + The platform tag is a claim about the payload, so a wheel has to be able to carry a payload that + contradicts it. Offsets are from the ELF specification: the machine sits at 18. + """ + return ( + b"\x7fELF\x02\x01\x01" + + bytes(9) + + (3).to_bytes(2, "little") + + _ELF_MACHINES[architecture].to_bytes(2, "little") + ) + + +@pytest.mark.unit +@pytest.mark.parametrize( + "case,should_pass", + [ + ("well_formed", True), + ("aarch64_tag", True), + ("payload_for_the_other_architecture", False), + ("unrepaired_tag", False), + ("wrong_architecture_floor", False), + ("wheel_tag_mismatch", False), + ("requires_a_mismatched_main", False), + ("requires_a_conditional_main", False), + ("bundles_a_stowaway", False), + ("bundles_the_executorch_runtime", False), + ("bundles_the_executorch_runtime_under_a_non_so_name", False), + ("payload_carries_a_mangled_name", False), + ("ships_no_cmake_package", False), + ("declares_itself_pure_python", False), + ("platform_independent_tag", False), + ("windows_compound_tag", False), + ("alien_architecture_tag", False), + ("requires_an_unpinned_executorch", False), + ("requires_no_executorch", False), + ("requires_a_mismatched_executorch_pin", False), + ("requires_no_torch_tensorrt", False), + ("requires_no_torch", False), + ("requires_no_tensorrt", False), + ("requires_an_unpinned_torch_tensorrt", False), + ("requires_an_unpinned_cuda_runtime", False), + ("requirement_carries_a_local_label", False), + ("no_metadata_at_all", False), + ], +) +def test_the_wheel_checker_rejects_a_bad_wheel(tmp_path, case, should_pass): + """The wheel checker has to reject, not just pass on the artifact of the day. + + It is the only thing enforcing wheel contents, platform tag, purelib and Requires-Dist, and + the first time it rejected anything it rejected this project's own wheel -- a local version + label on one requirement -- with no test having exercised a rejecting case. Both halves of + that need a fixture: the label case, and the pin mismatch that motivates reading METADATA at + all, since setup.py derives every dependency from whatever happens to be installed. That same + derivation applies to tensorrt-cu13 and nvidia-cuda-runtime, so a wheel that drops one or + loosens it to a range is exercised too. + + The real checker runs against generated wheels, and its shared-workflow invocation is checked + for ordering, so changes to either side cannot silently remove validation. + """ + from wheel.wheelfile import WheelFile + + checker = _REPO_ROOT / ".github/scripts/check-executorch-runtime-wheel.py" + steps = _runtime_build_steps() + check_at = next( + i + for i, step in enumerate(steps) + if "check-executorch-runtime-wheel.py" in step.get("run", "") + ) + check = steps[check_at] + assert check["if"] == "${{ steps.executorch-runtime.outcome == 'success' }}" + assert any("Repair Manylinux" in step.get("name", "") for step in steps[:check_at]) + assert any( + step.get("name") == "Upload wheel to GitHub" for step in steps[check_at + 1 :] + ) + _assert_the_checker_is_reachable( + check["run"].split( + "${CONDA_RUN} python .github/scripts/check-executorch-runtime-wheel.py" + )[0] + ) + dependencies = { + "torch": "2.15.0.dev20260824+cu132", + "torch-tensorrt": "2.15.0.dev20260824+cu132", + "tensorrt-cu13": "11.2.1", + "nvidia-cuda-runtime": "13.2.0", + } + for name, version in dependencies.items(): + info = tmp_path / f"{name.replace('-', '_')}-{version}.dist-info" + info.mkdir() + (info / "METADATA").write_text( + f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n" + ) + + pin = re.search( + r'^__executorch_version__:\s*"?([^"\s]+)"?\s*$', + (_REPO_ROOT / "dev_dep_versions.yml").read_text(encoding="utf-8"), + re.MULTILINE, + ).group(1) + package = "torch_tensorrt_executorch_runtime/" + payload = [package + "lib/libexecutorch_backend_tensorrt.so"] + # The label naming the CUDA build is part of the ExecuTorch requirement, because the delegate + # links one specific build. The checker compares against the installed wheel, so read the same + # source it does rather than the label-free pin, or a correct wheel is rejected here. + try: + installed_executorch = importlib.metadata.version("executorch") + except importlib.metadata.PackageNotFoundError: + pytest.skip( + "ExecuTorch is not installed, so the requirement the checker expects cannot be built" + ) + if installed_executorch.split("+")[0] != pin: + pytest.skip( + f"the installed ExecuTorch is {installed_executorch}, not the pinned {pin}, so this " + "cannot say whether the checker accepts a correct wheel" + ) + requires = [ + f"executorch=={installed_executorch}", + "torch==2.15.0.dev20260824", + "torch-tensorrt==2.15.0.dev20260824", + "tensorrt-cu13==11.2.1", + "nvidia-cuda-runtime==13.2.0", + ] + purelib, tag, arch = "false", "manylinux_2_28_x86_64", "x86_64" + elf_machine = None + + if case == "aarch64_tag": + tag, arch = "manylinux_2_35_aarch64", "aarch64" + elif case == "payload_for_the_other_architecture": + # The tag is a claim about the payload. Nothing read the payload to check it, so a wheel + # tagged for one architecture could carry a library built for the other and pass. + tag, arch, elf_machine = "manylinux_2_35_aarch64", "aarch64", "x86_64" + elif case == "unrepaired_tag": + tag = "linux_x86_64" + elif case == "wrong_architecture_floor": + # A floor the companion does not ship, so the guard has to refuse it. + tag, arch = "manylinux_2_39_aarch64", "aarch64" + elif case == "requires_a_mismatched_main": + requires = [ + "torch-tensorrt==2.15.0a0" if r.startswith("torch-tensorrt==") else r + for r in requires + ] + elif case == "requires_a_conditional_main": + requires = [ + r + '; python_version < "3.10"' if r.startswith("torch-tensorrt==") else r + for r in requires + ] + elif case == "bundles_a_stowaway": + payload.append(package + "libnvinfer.so.10") + elif case == "bundles_the_executorch_runtime": + payload.append(package + "libexecutorch.so") + elif case == "bundles_the_executorch_runtime_under_a_non_so_name": + # The count check only matches names ending in .so or .so., so an ExecuTorch component + # shipped under any other name slips past it. Only the forbidden-component list, which + # matches every name in the archive, catches this, so deleting that list opens a real hole. + payload.append(package + "executorch/lib/libexecutorch.so.debug") + elif case == "payload_carries_a_mangled_name": + # Exactly one object, but under the setuptools-mangled name the build_py redesign exists to + # prevent. The count check passes on it, so only the exact-name branch can reject it: with + # that branch gone the wheel ships a delegate pip cannot import under the expected name. + payload = [ + package + "_executorch_backend_tensorrt.cpython-310-x86_64-linux-gnu.so" + ] + elif case == "declares_itself_pure_python": + purelib = "true" + elif case == "platform_independent_tag": + tag = "any" + elif case == "windows_compound_tag": + # The whole tag set has to equal the one expected tag, so a compound tag is rejected even + # though a substring test would accept it for the part that does match. + tag = "win_amd64.linux_x86_64" + elif case == "alien_architecture_tag": + # Same exact comparison, which is what rejects a Linux tag for an architecture this wheel + # is not built for, rather than only rejecting non-Linux tags. + tag = "linux_ppc64le" + elif case == "requires_an_unpinned_executorch": + requires = [ + f"executorch>={pin}" if r.startswith("executorch==") else r + for r in requires + ] + elif case == "requires_no_executorch": + requires = ["torch==2.15.0.dev20260824"] + elif case == "requires_a_mismatched_executorch_pin": + # Every requirement present and exactly pinned, but executorch names a different version + # than the repository. The presence loop is satisfied, so only the pin comparison can + # reject it: with that branch gone the wheel ships requiring an executorch it was not built + # against and every other check still passes. The wrong version is derived from the pin + # rather than written as a literal so the repository-wide "==" pin scan does not read + # this fixture as a real, mispinned requirement site. + wrong_pin = pin.rsplit(".dev", 1)[0] + ".dev20200101" + requires = [ + f"executorch=={wrong_pin}", + "torch==2.15.0.dev20260824", + "torch-tensorrt==2.15.0.dev20260824", + "tensorrt-cu13==11.2.1", + "nvidia-cuda-runtime==13.2.0", + ] + elif case == "requires_no_torch_tensorrt": + # setup.py derives torch-tensorrt the same way it derives executorch, and it is the + # requirement that binds this runtime wheel to the producer that emitted the program, so a + # wheel that drops it ships with that binding missing and every content check still passes. + requires = [ + f"executorch=={pin}", + "torch==2.15.0.dev20260824", + "tensorrt-cu13==11.2.1", + "nvidia-cuda-runtime==13.2.0", + ] + elif case == "requires_no_torch": + requires = [ + f"executorch=={pin}", + "torch-tensorrt==2.15.0.dev20260824", + "tensorrt-cu13==11.2.1", + "nvidia-cuda-runtime==13.2.0", + ] + elif case == "requires_no_tensorrt": + # setup.py derives tensorrt-cu13 the same way it derives executorch, so a wheel that drops + # it ships with the dependency missing and every content check above still passes. + requires = [ + f"executorch=={pin}", + "torch==2.15.0.dev20260824", + "torch-tensorrt==2.15.0.dev20260824", + "nvidia-cuda-runtime==13.2.0", + ] + elif case == "requires_an_unpinned_torch_tensorrt": + # A derived requirement loosened to a range no longer binds the wheel to the exact producer + # it was built beside, which is the whole reason the metadata is read. + requires = [ + f"executorch=={pin}", + "torch==2.15.0.dev20260824", + "torch-tensorrt>=2.15.0.dev20260824", + "tensorrt-cu13==11.2.1", + "nvidia-cuda-runtime==13.2.0", + ] + elif case == "requires_an_unpinned_cuda_runtime": + # A derived requirement loosened to a range no longer binds the wheel to the version it was + # built beside, which is the whole reason the metadata is read. + requires = [ + f"executorch=={pin}", + "torch==2.15.0.dev20260824", + "torch-tensorrt==2.15.0.dev20260824", + "tensorrt-cu13==11.2.1", + "nvidia-cuda-runtime>=13.2.0", + ] + elif case == "requirement_carries_a_local_label": + # The exact rejection this PR's own CI hit: binds the wheel to one CUDA train. Relabel the + # torch-tensorrt entry in place rather than appending a duplicate requirement. + requires = [ + r + "+cu130" if r.startswith("torch-tensorrt==") else r for r in requires + ] + + wheel = tmp_path / f"torch_tensorrt_executorch_runtime-0.1.0-py3-none-{tag}.whl" + with WheelFile(wheel, "w") as archive: + for name in payload: + archive.writestr(name, _elf_object(elf_machine or arch)) + # The CMake package a C++ consumer links through. Present in every case except the one that + # deliberately drops it, so the other cases fail for their own reason rather than this one. + if case != "ships_no_cmake_package": + for cmake_name in ( + "torchtrt_executorch-config.cmake", + "torchtrt_executorch-config-version.cmake", + ): + archive.writestr( + f"{package}lib/cmake/torchtrt_executorch/{cmake_name}", "# stub\n" + ) + info = "torch_tensorrt_executorch_runtime-0.1.0.dist-info" + metadata_tag = "any" if case == "wheel_tag_mismatch" else tag + archive.writestr( + f"{info}/WHEEL", + f"Wheel-Version: 1.0\nRoot-Is-Purelib: {purelib}\nTag: py3-none-{metadata_tag}\n", + ) + if case != "no_metadata_at_all": + archive.writestr( + f"{info}/METADATA", + "Metadata-Version: 2.1\nName: torch-tensorrt-executorch-runtime\nVersion: 0.1.0\n" + + "".join(f"Requires-Dist: {r}\n" for r in requires), + ) + + completed = subprocess.run( + [sys.executable, str(checker), str(wheel), "--architecture", arch], + cwd=_REPO_ROOT, + env={**os.environ, "PYTHONPATH": str(tmp_path)}, + capture_output=True, + text=True, + timeout=30, + ) + accepted = completed.returncode == 0 + assert accepted is should_pass, ( + f"{case}: checker exited {completed.returncode}, expected " + f"{'acceptance' if should_pass else 'rejection'}\n{completed.stdout}{completed.stderr}" + ) + # Named branches whose case must reject through that branch and no other. Exit-status alone let + # a case pass by any route that also rejects: the mangled-name and pin-mismatch payloads each + # survived their own branch being deleted because an earlier check rejected a sibling payload + # that also dropped other requirements. Requiring the branch's own message pins each to it. + expected_messages = { + "payload_carries_a_mangled_name": ( + "expected torch_tensorrt_executorch_runtime/lib/libexecutorch_backend_tensorrt.so" + ), + "ships_no_cmake_package": "the wheel ships no CMake package", + "requires_an_unpinned_executorch": "the repository pins executorch==", + "requires_a_mismatched_executorch_pin": "the repository pins executorch==", + } + expected_message = expected_messages.get(case) + if expected_message is not None: + assert expected_message in completed.stderr, ( + f"{case} was rejected, but not through its own branch: expected " + f"{expected_message!r} in\n{completed.stderr}" + ) + + +@pytest.mark.unit +def test_the_wheel_checker_rejects_removed_exactness(tmp_path): + """Removing exact-version enforcement must admit the otherwise valid loose fixture.""" + test_the_wheel_checker_rejects_a_bad_wheel( + tmp_path, "requires_an_unpinned_executorch", False + ) + checker = _REPO_ROOT / ".github/scripts/check-executorch-runtime-wheel.py" + source = checker.read_text() + clause = ' or str(matched[0].specifier) != f"=={expected_version}"\n' + assert source.count(clause) == 1 + wheel = next(tmp_path.glob("*.whl")) + result = subprocess.run( + [ + sys.executable, + "-c", + "import sys; __file__ = sys.argv.pop(1); " + "exec(compile(sys.argv.pop(1), __file__, 'exec'))", + str(checker), + source.replace(clause, ""), + str(wheel), + "--architecture", + "x86_64", + ], + cwd=_REPO_ROOT, + env={**os.environ, "PYTHONPATH": str(tmp_path)}, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "Validated " in result.stdout + + +@pytest.mark.unit +@pytest.mark.parametrize("mutation", ["false-condition", "removed", "disabled-step"]) +def test_installed_check_rejects_disabled_resolution_guard( + tmp_path, monkeypatch, mutation +): + steps = _runtime_build_steps() + step = next( + s + for s in steps + if s.get("name") == "Check the repaired ExecuTorch runtime wheel" + ) + guard = 'if grep -E "not found|undefined symbol" <<< "${resolution}"; then' + assert step["run"].count(guard) == 1 + if mutation == "disabled-step": + step["if"] = "${{ false }}" + elif mutation == "false-condition": + step["run"] = step["run"].replace(guard, "if false; then") + else: + start = step["run"].index(guard) + end = step["run"].index("fi\n", start) + len("fi\n") + step["run"] = step["run"][:start] + step["run"][end:] + monkeypatch.setitem( + test_the_wheel_build_resolves_the_delegate_from_its_installed_location.__globals__, + "_runtime_build_steps", + lambda: steps, + ) + with pytest.raises(AssertionError): + test_the_wheel_build_resolves_the_delegate_from_its_installed_location( + tmp_path, "not found", "x86_64" + ) + + +@pytest.mark.unit +@pytest.mark.parametrize("arch", ["x86_64", "aarch64"]) +@pytest.mark.parametrize( + "failure", ["", "not found", "undefined symbol", "ldd", "import"] +) +def test_the_wheel_build_resolves_the_delegate_from_its_installed_location( + tmp_path, failure, arch +): + """Execute the shared check step with fake tools to verify ordering and failure propagation.""" + step = next( + step + for step in _runtime_build_steps() + if step.get("name") == "Check the repaired ExecuTorch runtime wheel" + ) + assert step["if"] == "${{ steps.executorch-runtime.outcome == 'success' }}" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + test_the_wheel_checker_rejects_a_bad_wheel( + tmp_path, "aarch64_tag" if arch == "aarch64" else "well_formed", True + ) + dist = tmp_path / "dist" + dist.mkdir() + wheel = next(tmp_path.glob("*.whl")) + wheel.rename(dist / wheel.name) + checker = tmp_path / ".github/scripts/check-executorch-runtime-wheel.py" + checker.parent.mkdir(parents=True) + checker.symlink_to(_REPO_ROOT / ".github/scripts/check-executorch-runtime-wheel.py") + delegate = ( + tmp_path + / "installed/torch_tensorrt_executorch_runtime/lib/libexecutorch_backend_tensorrt.so" + ) + env_file = tmp_path / "build-env" + env_file.write_text("export CONDA_RUN=''\n") + python = bin_dir / "python" + python.write_text( + f"#!{sys.executable}\n" + "import os, subprocess, sys\n" + "from pathlib import Path\n" + "from wheel.wheelfile import WheelFile\n" + "args = sys.argv[1:]\n" + "delegate = Path(os.environ['DELEGATE'])\n" + "if args[0] == '.github/scripts/check-executorch-runtime-wheel.py':\n" + " subprocess.run([sys.executable, *args], check=True, timeout=15)\n" + " event = 'checked'\n" + "elif args[:4] == ['-m', 'pip', 'install', '--no-deps']:\n" + " with WheelFile(args[4]) as wheel: wheel.extractall('installed')\n" + " event = 'installed'\n" + "elif args == ['-c', 'import torch_tensorrt_executorch_runtime as m; print(m._delegate_path())']:\n" + " assert os.environ.get('TORCH_TENSORRT_SKIP_DELEGATE_REGISTRATION') == '1'\n" + " assert delegate.is_file()\n" + " event = 'path'; print(delegate)\n" + "else:\n" + " assert args == ['-c', 'import torch_tensorrt_executorch_runtime']\n" + " assert 'LD_LIBRARY_PATH' not in os.environ\n" + # The prefix, not the whole file. The fixture's payload grew a full header when the checker + # started reading the machine type, and comparing every byte made this fail for a reason that + # had nothing to do with what it was checking. + " assert delegate.read_bytes().startswith(b'\\x7fELF')\n" + " event = 'imported'\n" + "with open(os.environ['EVENTS'], 'a') as f: f.write(event + '\\n')\n" + "if event == 'imported' and os.environ['FAILURE'] == 'import': sys.exit(1)\n" + ) + ldd = bin_dir / "ldd" + ldd.write_text( + "#!/bin/bash\nset -eu\n" + '[[ -z "${LD_LIBRARY_PATH:-}" ]]\n' + '[[ "$#" == 2 && "$1" == -r && "$2" == "$DELEGATE" && -f "$2" ]]\n' + 'echo resolved >> "$EVENTS"\n' + 'echo "$FAILURE"\n' + '[[ "$FAILURE" != ldd ]]\n' + ) + python.chmod(0o755) + ldd.chmod(0o755) + events = tmp_path / "events" + result = subprocess.run( + ["bash", "-c", step["run"]], + cwd=tmp_path, + capture_output=True, + text=True, + env={ + **os.environ, + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "BUILD_ENV_FILE": str(env_file), + "ARCH": arch, + "EVENTS": str(events), + "FAILURE": failure, + "DELEGATE": str(delegate), + "PYTHONPATH": str(tmp_path), + "LD_LIBRARY_PATH": "/build-only", + }, + timeout=30, + ) + (tmp_path / "stdout.log").write_text(result.stdout) + (tmp_path / "stderr.log").write_text(result.stderr) + assert (result.returncode == 0) is (failure == ""), result.stdout + result.stderr + expected = ["checked", "installed", "path", "resolved"] + if failure in {"", "import"}: + expected.append("imported") + assert events.read_text().splitlines() == expected diff --git a/tests/py/dynamo/executorch/test_artifact_guard.py b/tests/py/dynamo/executorch/test_artifact_guard.py new file mode 100644 index 0000000000..9d9223dcdc --- /dev/null +++ b/tests/py/dynamo/executorch/test_artifact_guard.py @@ -0,0 +1,798 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +"""CPU-only behavior checks for the companion's native artifact guard.""" + +import json +import os +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + +_ROOT = Path(__file__).resolve().parents[4] +_NATIVE = _ROOT / "py/torch-tensorrt-executorch-runtime/native" +_GUARD = _NATIVE / "check_imports_executorch_runtime.sh" +_RUNPATH = ( + "$ORIGIN:$ORIGIN/../../executorch/lib:$ORIGIN/../../tensorrt_libs:" + "$ORIGIN/../../nvidia/cu13/lib" +) +_REGISTER = "_ZN10executorch7runtime16register_backendERKNS0_7BackendE" +_X86 = "manylinux_2_28_x86_64" +_ARM = "manylinux_2_35_aarch64" +_BASE_VERSIONS = "CXXABI_1.3 GLIBCXX_3.4.21 GLIBC_2.17 GCC_3.0" + + +def _run(argv, **kwargs): + return subprocess.run(argv, text=True, capture_output=True, **kwargs) + + +def _ok(result): + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.fixture +def artifact(tmp_path): + target = tmp_path / "libdelegate.so" + runtime = tmp_path / "executorch/lib/libexecutorch.so" + extension = runtime.with_name("libexecutorch_extension_cuda.so") + kernels = runtime.with_name("libkernels.so") + pybindings = runtime.parent.parent / "extension/pybindings/_C.test.so" + for path in (target, runtime, extension, kernels, pybindings): + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + data = { + "dialect": "gnu", + "runpath": _RUNPATH, + "path_tag": "RUNPATH", + "needed": [ + "libexecutorch.so", + "libexecutorch_extension_cuda.so", + "libstdc++.so.6", + ], + "versions": _BASE_VERSIONS, + "runtime_versions": _BASE_VERSIONS, + "kernel_versions": _BASE_VERSIONS, + "failure": None, + } + config = tmp_path / "readelf.json" + reader = tmp_path / "readelf" + reader.write_text( + f"#!{sys.executable}\n" + "import json, sys\n" + "from pathlib import Path\n" + f"d = json.loads(Path({str(config)!r}).read_text())\n" + "flag, name = sys.argv[1], Path(sys.argv[-1]).name\n" + "if d['failure'] == [flag, name]:\n" + " print('deliberate read failure', file=sys.stderr)\n" + " sys.exit(17)\n" + "eu = d['dialect'] == 'elfutils'\n" + "if flag == '-d':\n" + " needed = d['needed'] if name == 'libdelegate.so' else (\n" + " ['libkernels.so'] if name == '_C.test.so' else [])\n" + " for lib in needed:\n" + " print((' NEEDED' if eu else ' 0x0001 (NEEDED)') + ' Shared library: [' + lib + ']')\n" + " if name == 'libdelegate.so' and d['runpath'] is not None:\n" + " tag = d['path_tag']\n" + " print((' ' + tag if eu else ' 0x001d (' + tag + ')') + ' Library runpath: [' + d['runpath'] + ']')\n" + "elif flag in ('-Ws', '--dyn-syms'):\n" + " ndx = ('UNDEF' if eu else 'UND') if name == 'libdelegate.so' else '12'\n" + f" print(' 1: 00000000 8 FUNC GLOBAL DEFAULT ' + ndx + ' {_REGISTER}')\n" + "elif flag == '-V':\n" + " key = 'versions' if name == 'libdelegate.so' else (\n" + " 'kernel_versions' if name == 'libkernels.so' else 'runtime_versions')\n" + " for node in d[key].split():\n" + " print(' 0x0010: Name: ' + node + ' Flags: none Version: 2')\n" + "else:\n" + " sys.exit(18)\n" + ) + reader.chmod(0o755) + + def invoke(*, options=None, guard=_GUARD): + config.write_text(json.dumps(data)) + if options is None: + options = [str(runtime), _RUNPATH, _X86] + return _run(["sh", str(guard), str(reader), str(target), *options]) + + return data, invoke, runtime + + +@pytest.mark.parametrize("dialect", ["gnu", "llvm", "elfutils"]) +@pytest.mark.parametrize("with_expected", [False, True]) +def test_cuda_13_uses_actual_runpath(artifact, dialect, with_expected): + data, invoke, runtime = artifact + data["dialect"] = dialect + data["needed"].append("libcudart.so.13") + options = [str(runtime)] + ([_RUNPATH, _X86] if with_expected else []) + _ok(invoke(options=options)) + + +@pytest.mark.parametrize("major", [12, 14, 99]) +def test_unsupported_cuda_major(artifact, major): + data, invoke, _ = artifact + data["needed"].append(f"libcudart.so.{major}") + result = invoke() + assert result.returncode != 0 + assert "requires CUDA 13" in result.stderr + + +@pytest.mark.parametrize("suffix", ["", "-backup", "/not-the-lib-dir"]) +def test_cuda_path_must_be_an_actual_entry(artifact, suffix): + data, invoke, runtime = artifact + data["needed"].append("libcudart.so.13") + data["runpath"] = _RUNPATH.rsplit(":", 1)[0] + if suffix: + data["runpath"] += ":$ORIGIN/../../nvidia/cu13/lib" + suffix + result = invoke(options=[str(runtime)]) + assert result.returncode != 0 + assert "RUNPATH carries no nvidia/cu13/lib" in result.stderr + + +def test_expected_runpath_is_only_an_equality_check(artifact): + data, invoke, runtime = artifact + data["needed"].append("libcudart.so.13") + result = invoke(options=[str(runtime), "$ORIGIN/../../executorch/lib"]) + assert result.returncode != 0 + assert "RUNPATH the build did not ask for" in result.stderr + + +@pytest.mark.parametrize("position", [0, 1, 2]) +def test_supplied_empty_option_is_rejected(artifact, position): + _, invoke, runtime = artifact + options = [str(runtime), _RUNPATH, _X86][: position + 1] + options[position] = "" + result = invoke(options=options) + assert result.returncode != 0 + assert "must not be empty" in result.stderr + + +@pytest.mark.parametrize( + "tag", + [ + "manylinux_2_34", + "manylinux_2_28", + "manylinux_2_39", + "linux_x86_64", + "manylinux_2_39_aarch64", + "manylinux_2_39_x86_64", + "unknown", + ], +) +def test_unsupported_supplied_tag_is_rejected(artifact, tag): + _, invoke, runtime = artifact + result = invoke(options=[str(runtime), _RUNPATH, tag]) + assert result.returncode != 0 + assert "unsupported manylinux tag" in result.stderr + + +def test_omitted_optional_arguments_are_explicit(artifact): + _, invoke, runtime = artifact + result = invoke(options=[]) + _ok(result) + assert "no runtime given" in result.stderr + result = invoke(options=[str(runtime)]) + _ok(result) + assert "no manylinux tag" in result.stderr + + +# Boundary and gap cases from auditwheel 6.8.2's architecture-specific policy. +@pytest.mark.parametrize( + "tag,node,allowed", + [ + (_X86, "GLIBCXX_3.4.23", True), + (_X86, "GLIBCXX_3.4.24", True), + (_X86, "GLIBCXX_3.4.25", False), + (_ARM, "GLIBCXX_3.4.24", True), + (_ARM, "GLIBCXX_3.4.25", True), + (_ARM, "GLIBCXX_3.4.34", False), + (_X86, "CXXABI_1.3.10", True), + (_X86, "CXXABI_1.3.11", True), + (_X86, "CXXABI_1.3.12", False), + (_ARM, "CXXABI_1.3.11", True), + (_ARM, "CXXABI_1.3.12", True), + (_ARM, "CXXABI_1.3.16", False), + (_X86, "GLIBC_2.27", True), + (_X86, "GLIBC_2.28", True), + (_X86, "GLIBC_2.29", False), + (_ARM, "GLIBC_2.28", True), + (_ARM, "GLIBC_2.29", True), + (_ARM, "GLIBC_2.39", False), + (_X86, "GCC_4.8.0", True), + (_X86, "GCC_7.0.0", True), + (_X86, "GCC_7.1.0", False), + (_ARM, "GCC_4.5.0", True), + (_ARM, "GCC_7.0.0", True), + (_ARM, "GCC_14.0.0", False), + (_ARM, "GCC_15.0.0", False), + (_X86, "GLIBC_2.19", False), + (_ARM, "GLIBC_2.37", False), + (_X86, "GCC_4.5.0", False), + (_ARM, "GCC_4.8.0", False), + (_X86, "CXXABI_TM_1", True), + (_ARM, "CXXABI_TM_1", True), + (_X86, "CXXABI_FLOAT128", True), + (_ARM, "CXXABI_FLOAT128", False), + (_X86, "GLIBC_ABI_DT_RELR", False), + (_ARM, "GLIBC_ABI_DT_RELR", False), + (_X86, "GLIBC_PRIVATE", False), + (_ARM, "GLIBC_PRIVATE", False), + ], +) +def test_manylinux_policy_membership(artifact, tag, node, allowed): + data, invoke, runtime = artifact + data["versions"] += " " + node + result = invoke(options=[str(runtime), _RUNPATH, tag]) + if allowed: + _ok(result) + else: + assert result.returncode != 0 + assert node in result.stderr + assert tag in result.stderr + + +@pytest.mark.parametrize( + "flag,name", + [ + ("-d", "libdelegate.so"), + ("--dyn-syms", "libdelegate.so"), + ("-Ws", "libdelegate.so"), + ("-V", "libdelegate.so"), + ("-Ws", "libexecutorch.so"), + ("-d", "_C.test.so"), + ("-d", "libexecutorch.so"), + ("-d", "libexecutorch_extension_cuda.so"), + ("-d", "libkernels.so"), + ("-V", "libexecutorch.so"), + ("-V", "libkernels.so"), + ], +) +def test_every_read_failure_is_fatal(artifact, flag, name): + data, invoke, _ = artifact + data["failure"] = [flag, name] + result = invoke() + assert result.returncode != 0 + assert name in result.stderr + + +@pytest.mark.parametrize("dialect", ["gnu", "llvm", "elfutils"]) +def test_legacy_rpath_is_rejected(artifact, dialect): + data, invoke, _ = artifact + data.update(dialect=dialect, path_tag="RPATH") + result = invoke() + assert result.returncode != 0 + assert "DT_RPATH" in result.stderr + + +def test_untagged_named_nodes_use_pybindings_closure(artifact): + data, invoke, runtime = artifact + data["versions"] += " CXXABI_TM_1" + result = invoke(options=[str(runtime)]) + assert result.returncode != 0 + data["kernel_versions"] += " CXXABI_TM_1" + _ok(invoke(options=[str(runtime)])) + + +@pytest.mark.parametrize( + "extra", [[], ["reader"], ["reader", "target", "runtime", "path", _X86, "extra"]] +) +def test_invalid_argument_count(extra): + result = _run(["sh", str(_GUARD), *extra]) + assert result.returncode != 0 + assert "expected " in result.stderr + + +def test_missing_runtime_is_fatal(artifact): + _, invoke, runtime = artifact + result = invoke(options=[str(runtime.with_name("missing.so"))]) + assert result.returncode != 0 + assert "does not exist" in result.stderr + + +def test_runtime_version_output_must_not_be_empty(artifact): + data, invoke, _ = artifact + data["runtime_versions"] = "" + data["kernel_versions"] = "" + result = invoke() + assert result.returncode != 0 + assert "could not read symbol versions beside" in result.stderr + + +def test_missing_pybindings_is_fatal(artifact): + _, invoke, runtime = artifact + (runtime.parent.parent / "extension/pybindings/_C.test.so").unlink() + result = invoke() + assert result.returncode != 0 + assert "could not find the pybindings extension" in result.stderr + + +@pytest.fixture +def native_tools(): + tools = { + name: shutil.which(name) for name in ("cmake", "c++", "readelf", "patchelf") + } + if sys.platform != "linux" or not all(tools.values()): + pytest.skip("needs Linux, CMake >= 3.28, a C++ compiler, readelf and patchelf") + return tools + + +def _native_project(tmp_path, tools, *, mutation=None, static_cuda=False): + native = tmp_path / "native" + shutil.copytree(_NATIVE, native) + cmake_file = native / "CMakeLists.txt" + cmake = cmake_file.read_text() + guard_command = ' COMMAND sh\n "${CMAKE_CURRENT_LIST_DIR}/check_imports_executorch_runtime.sh"' + if mutation == "remove": + start = cmake.index( + " add_custom_command", cmake.index("if(TORCH_TENSORRT_READELF)") + ) + end = cmake.index(" VERBATIM)", start) + len(" VERBATIM)") + cmake = cmake[:start] + cmake[end:] + elif mutation == "early_return": + marker = "find_program(TORCH_TENSORRT_READELF NAMES" + assert marker in cmake + cmake = cmake.replace(marker, "return()\n" + marker, 1) + elif mutation == "inert": + assert guard_command in cmake + cmake = cmake.replace( + guard_command, ' COMMAND "${CMAKE_COMMAND}" -E true', 1 + ) + elif mutation == "static_check": + marker = 'if(_extension_cuda_type STREQUAL "STATIC_LIBRARY")' + assert marker in cmake + cmake = cmake.replace(marker, "if(FALSE)", 1) + elif mutation == "retention": + marker = '"LINKER:--push-state,--no-as-needed,$,--pop-state"' + assert marker in cmake + cmake = cmake.replace(marker, '"LINKER:--as-needed"', 1) + cmake_file.write_text(cmake) + + source = tmp_path / "source" + modules = source / "cmake/Modules" + sources = source / "cpp/src/torch_tensorrt/executorch" + modules.mkdir(parents=True) + sources.mkdir(parents=True) + (sources / "TensorRTBackend.cpp").write_text( + "#include \n" + "namespace executorch { namespace runtime {\n" + "struct Backend {}; void register_backend(const Backend&);\n" + "}}\n" + 'extern "C" void extension_cuda();\n' + 'extern "C" void cuda_fixture();\n' + 'namespace { struct R { std::string s; R() : s("fixture") {\n' + "executorch::runtime::register_backend({}); extension_cuda(); cuda_fixture();\n" + "} } r; }\n" + ) + for name in ("TensorRTBlobHeader.cpp", "WeightStreamingBudget.cpp"): + (sources / name).write_text("\n") + runtime_dir = tmp_path / "executorch/lib" + runtime_dir.mkdir(parents=True) + library_sources = { + "libexecutorch.so": '#include \nnamespace executorch { namespace runtime { struct Backend {}; void register_backend(const Backend&) { puts("registered"); } }}\n', + "libexecutorch_extension_cuda.so": 'extern "C" void extension_cuda() {}\n', + "libcudart.so.13": 'extern "C" void cuda_fixture() {}\n', + "libunrelated.so": 'extern "C" void unrelated() {}\n', + } + for name, body in library_sources.items(): + cpp = tmp_path / (name + ".cpp") + cpp.write_text(body) + _ok( + _run( + [ + tools["c++"], + "-shared", + "-fPIC", + str(cpp), + "-o", + str(runtime_dir / name), + f"-Wl,-soname,{name}", + ] + ) + ) + pybindings = runtime_dir.parent / "extension/pybindings" + pybindings.mkdir(parents=True) + shutil.copy2( + runtime_dir / "libexecutorch_extension_cuda.so", pybindings / "_C.test.so" + ) + (modules / "FindTensorRT.cmake").write_text( + "add_library(TensorRT::nvinfer INTERFACE IMPORTED)\n" + ) + (modules / "FindCUDAToolkit.cmake").write_text( + "add_library(CUDA::cudart SHARED IMPORTED)\n" + f'set_target_properties(CUDA::cudart PROPERTIES IMPORTED_LOCATION "{runtime_dir}/libcudart.so.13")\n' + ) + prefix = runtime_dir.parent / "share/cmake" + prefix.mkdir(parents=True) + (prefix / "executorch-config.cmake").write_text( + "add_library(executorch::runtime SHARED IMPORTED)\n" + f'set_target_properties(executorch::runtime PROPERTIES IMPORTED_LOCATION "{runtime_dir}/libexecutorch.so" INTERFACE_LINK_OPTIONS "-Wl,-rpath,{runtime_dir}")\n' + f'add_library(executorch::extension_cuda {"STATIC" if static_cuda else "SHARED"} IMPORTED)\n' + f'set_target_properties(executorch::extension_cuda PROPERTIES IMPORTED_LOCATION "{runtime_dir}/libexecutorch_extension_cuda.so")\n' + ) + (tmp_path / "main.cpp").write_text("int main() { return 0; }\n") + (tmp_path / "CMakeLists.txt").write_text( + "cmake_minimum_required(VERSION 3.28)\nproject(guard_fixture LANGUAGES CXX)\n" + "add_subdirectory(native)\n" + "add_executable(consumer main.cpp)\n" + 'target_link_options(consumer PRIVATE "LINKER:--as-needed")\n' + f'target_link_libraries(consumer PRIVATE executorch::backend_tensorrt "{runtime_dir}/libunrelated.so")\n' + f'set_target_properties(consumer PROPERTIES BUILD_RPATH "{runtime_dir};${{CMAKE_BINARY_DIR}}/native")\n' + ) + build = tmp_path / "build" + configure = _run( + [ + tools["cmake"], + "-S", + str(tmp_path), + "-B", + str(build), + f"-DTORCH_TENSORRT_SOURCE_DIR={source}", + f"-DCMAKE_PREFIX_PATH={prefix}", + f"-DTORCH_TENSORRT_PATCHELF={tools['patchelf']}", + f"-DTORCH_TENSORRT_READELF={tools['readelf']}", + ] + ) + return configure, build, runtime_dir + + +@pytest.mark.parametrize("mutation", [None, "remove", "early_return", "inert"]) +def test_production_post_build_rejects_bad_artifact(tmp_path, native_tools, mutation): + tools = dict(native_tools) + real_patchelf = tools["patchelf"] + wrapper = tmp_path / "patchelf-with-bad-needed" + wrapper.write_text( + "#!/bin/sh\nset -eu\n" + f'{shlex.quote(real_patchelf)} "$@"\n' + 'if [ "$1" = "--set-rpath" ]; then\n' + f' {shlex.quote(real_patchelf)} --replace-needed libcudart.so.13 libcudart.so.14 "$3"\n' + "fi\n" + ) + wrapper.chmod(0o755) + tools["patchelf"] = str(wrapper) + configure, build, _ = _native_project(tmp_path, tools, mutation=mutation) + _ok(configure) + result = _run( + [ + tools["cmake"], + "--build", + str(build), + "--target", + "executorch_backend_tensorrt", + ] + ) + if mutation is None: + assert result.returncode != 0, result.stdout + result.stderr + assert "requires CUDA 13" in result.stdout + result.stderr + else: + _ok(result) + dyn = _run( + [ + tools["readelf"], + "-d", + str(build / "native/libexecutorch_backend_tensorrt.so"), + ] + ) + _ok(dyn) + assert "libcudart.so.14" in dyn.stdout + + +@pytest.mark.parametrize("mutation", [None, "retention"]) +def test_production_alias_retains_registration(tmp_path, native_tools, mutation): + configure, build, runtime_dir = _native_project( + tmp_path, native_tools, mutation=mutation + ) + _ok(configure) + _ok(_run([native_tools["cmake"], "--build", str(build)])) + consumer = build / "consumer" + dyn = _run([native_tools["readelf"], "-d", str(consumer)]) + _ok(dyn) + assert ("libexecutorch_backend_tensorrt.so" in dyn.stdout) == (mutation is None) + assert "libunrelated.so" not in dyn.stdout + result = _run( + [str(consumer)], env={**os.environ, "LD_LIBRARY_PATH": str(runtime_dir)} + ) + _ok(result) + assert ("registered" in result.stdout) == (mutation is None) + artifact = build / "native/libexecutorch_backend_tensorrt.so" + runpath = _run([native_tools["patchelf"], "--print-rpath", str(artifact)]) + _ok(runpath) + assert runpath.stdout.strip() == _RUNPATH + + +@pytest.mark.parametrize("mutation", [None, "static_check"]) +def test_static_cuda_target_is_rejected_at_configure(tmp_path, native_tools, mutation): + configure, _, _ = _native_project( + tmp_path, native_tools, mutation=mutation, static_cuda=True + ) + if mutation is None: + assert configure.returncode != 0 + assert ( + "executorch::extension_cuda is a STATIC_LIBRARY" + in configure.stdout + configure.stderr + ) + else: + _ok(configure) + + +@pytest.mark.parametrize("reader_name", ["readelf", "llvm-readelf", "eu-readelf"]) +def test_real_readers_accept_and_reject_elf(tmp_path, native_tools, reader_name): + reader = shutil.which(reader_name) + if reader is None: + pytest.skip(f"{reader_name} is not installed") + tools = {**native_tools, "readelf": reader} + configure, build, runtime_dir = _native_project(tmp_path, tools) + _ok(configure) + _ok( + _run( + [ + tools["cmake"], + "--build", + str(build), + "--target", + "executorch_backend_tensorrt", + ] + ) + ) + target = build / "native/libexecutorch_backend_tensorrt.so" + argv = [ + "sh", + str(_GUARD), + reader, + str(target), + str(runtime_dir / "libexecutorch.so"), + ] + _ok(_run(argv)) + _ok( + _run([tools["patchelf"], "--force-rpath", "--set-rpath", _RUNPATH, str(target)]) + ) + result = _run(argv) + assert result.returncode != 0 + assert "DT_RPATH" in result.stderr + + +@pytest.mark.parametrize( + "mutation", ["cuda", "cuda_path", "policy_loop", "gcc", "empty", "unknown", "exit"] +) +def test_guard_removal_controls(artifact, tmp_path, mutation): + data, invoke, runtime = artifact + guard = tmp_path / "mutated-guard.sh" + source = _GUARD.read_text() + options = None + if mutation == "cuda": + old, new = "libcudart.so.13) ;;", "libcudart.so.13|libcudart.so.14) ;;" + data["needed"].append("libcudart.so.14") + elif mutation == "cuda_path": + old = "*':$ORIGIN/../../nvidia/cu13/lib:'*) ;;" + new = "*) ;;" + data["needed"].append("libcudart.so.13") + data["runpath"] = _RUNPATH.rsplit(":", 1)[0] + options = [str(runtime)] + elif mutation == "policy_loop": + old = 'for node in $(versions "${target_versions}"); do' + new = "for node in; do" + data["versions"] += " GCC_7.1.0" + elif mutation == "gcc": + old, new = ( + "(GLIBCXX|CXXABI|GLIBC|GCC|LIBATOMIC|ZLIB)", + "(GLIBCXX|CXXABI|GLIBC|LIBATOMIC|ZLIB)", + ) + data["versions"] += " GCC_7.1.0" + elif mutation == "empty": + old = '[ -n "${argument}" ] || fail "supplied arguments must not be empty"' + new = ":" + options = [str(runtime), _RUNPATH, ""] + elif mutation == "unknown": + old = '*) fail "unsupported manylinux tag: ${manylinux_tag}" ;;' + new = '*) manylinux_tag="" ;;' + options = [str(runtime), _RUNPATH, "unknown"] + else: + old, new = "set -u", "exit 0\nset -u" + data["needed"].remove("libexecutorch.so") + assert source.count(old) == 1 + # The intact guard must reject the same input before the control disables that check. + assert invoke(options=options).returncode != 0 + guard.write_text(source.replace(old, new, 1)) + result = invoke(options=options, guard=guard) + if mutation == "empty": + assert "must not be empty" not in result.stderr + assert "unsupported manylinux tag" in result.stderr + else: + _ok(result) + + +@pytest.mark.unit +def test_every_site_naming_the_platform_tag_agrees() -> None: + """Three files name the tag, and they have to say the same thing. + + The workflow tags the wheel, the native build passes a tag to the guard, and the guard decides + which tags it accepts. When one moved and another did not, the build failed with an unsupported + tag well after the code was otherwise correct. + """ + cmake = (_NATIVE / "CMakeLists.txt").read_text(encoding="utf-8") + guard = _GUARD.read_text(encoding="utf-8") + workflow = (_ROOT / ".github/workflows/build_linux.yml").read_text(encoding="utf-8") + checker = (_ROOT / ".github/scripts/check-executorch-runtime-wheel.py").read_text( + encoding="utf-8" + ) + readme = (_ROOT / "py/torch-tensorrt-executorch-runtime/README.md").read_text( + encoding="utf-8" + ) + for arch, tag in (("aarch64", _ARM), ("x86_64", _X86)): + floor = tag.removeprefix("manylinux_").removesuffix(f"_{arch}") + other = tag.replace(floor, "2_28" if floor != "2_28" else "2_35") + assert f'"{tag}"' in cmake, f"the native build does not name {tag}" + assert tag in guard, f"the guard does not accept {tag}" + assert f"platform_tag={tag}" in workflow, f"the workflow does not apply {tag}" + # The checker derives the tag from a per-architecture floor rather than naming it whole. + assert ( + f'"{arch}": "{floor}"' in checker + ), f"the checker's floor for {arch} is not {floor}" + assert tag in readme, f"the documentation does not name {tag}" + assert other not in cmake, f"the native build still names {other}" + assert ( + f"platform_tag={other}" not in workflow + ), f"the workflow still applies {other}" + assert other not in readme, f"the documentation still names {other}" + + +@pytest.mark.unit +def test_the_shipped_binaries_can_find_the_libraries_they_need() -> None: + """A program in the wheel is not launched through Python, so nothing prepares its search path. + + The example runner shipped without entries for TensorRT and the CUDA runtime, which live in their + own distributions beside this one, so it exited before main with a loader error naming a library + that was installed the whole time. The delegate library in the companion wheel already carried + the right entries, which is why it loaded and the binary did not. + """ + build = (_ROOT / "examples/executorch_reference_runner/BUILD").read_text( + encoding="utf-8" + ) + binaries = [ + block + for block in build.split("cc_binary(")[1:] + if "kv_cache_decode_check" in block or "example_executorch_runner" in block + ] + assert len(binaries) == 2, f"expected two shipped binaries, found {len(binaries)}" + for block in binaries: + name = block.split('name = "', 1)[1].split('"', 1)[0] + for needed in ( + "$$ORIGIN/../lib", + "$$ORIGIN/../../tensorrt_libs", + "$$ORIGIN/../../nvidia/cu13/lib", + ): + assert needed in block, f"{name} has no run path entry for {needed}" + + +def _drop_needed(data, name): + data["needed"] = [lib for lib in data["needed"] if lib != name] + + +@pytest.mark.parametrize( + "break_it,expected", + [ + ( + lambda d: _drop_needed(d, "libexecutorch.so"), + "no DT_NEEDED on libexecutorch.so", + ), + ( + lambda d: _drop_needed(d, "libexecutorch_extension_cuda.so"), + "no DT_NEEDED on libexecutorch_extension_cuda.so", + ), + ( + lambda d: _drop_needed(d, "libstdc++.so.6"), + "no DT_NEEDED on libstdc++", + ), + ( + lambda d: d.update(path_tag="RPATH"), + "carries DT_RPATH rather than DT_RUNPATH", + ), + (lambda d: d.update(runpath=None), "carries no RUNPATH"), + ], +) +@pytest.mark.unit +def test_each_linkage_check_rejects_what_it_is_for(artifact, break_it, expected): + """Each of these checks passed its own suite with the check deleted. + + The guard is what stands between a wheel that cannot load and whoever installs it, so a check + nothing exercises is the same as no check. One case per rejection, each crafting the input that + rejection exists for. + """ + data, invoke, runtime = artifact + break_it(data) + result = invoke() + assert result.returncode != 0, result.stdout + assert expected in result.stderr, result.stderr + + +@pytest.mark.parametrize( + "symbol", ["ZLIB_1.2.13", "LIBATOMIC_1.3", "GLIBCXX_3.4.40", "GLIBC_2.99"] +) +@pytest.mark.unit +def test_a_symbol_above_the_platform_ceiling_is_rejected(artifact, symbol): + """Two of the platform's six symbol families were missing from the tables and the collector. + + A requirement above the ceiling in either of those two passed in silence, which is the one thing + the tag is a promise about. + """ + data, invoke, runtime = artifact + data["versions"] = f"{_BASE_VERSIONS} {symbol}" + result = invoke() + assert result.returncode != 0, result.stdout + assert symbol in result.stderr, result.stderr + + +@pytest.mark.unit +def test_a_library_outside_the_allowed_set_is_rejected(artifact): + """Nothing enumerated what the delegate links, so an unintended link passed in silence. + + libpython is the one that matters most. The wheel is tagged for any Python 3 because the payload + has no Python ABI, and linking libpython would make that tag wrong while the wheel still installed + everywhere. + """ + data, invoke, runtime = artifact + data["needed"].append("libpython3.12.so.1.0") + result = invoke() + assert result.returncode != 0, result.stdout + assert "libpython3.12.so.1.0" in result.stderr, result.stderr + + +@pytest.mark.unit +def test_the_native_build_runs_the_guard_after_linking() -> None: + """Deleting the step that runs the guard left this suite green wherever patchelf was missing. + + The cases that build for real skip without it, so nothing noticed that the built library had + stopped being checked at all. This reads the build file instead, which is weaker than running it + but is the only check that holds where the toolchain is absent. + """ + build = ( + _ROOT / "py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt" + ).read_text(encoding="utf-8") + commands = [ + block + for block in build.split( + "add_custom_command(TARGET executorch_backend_tensorrt POST_BUILD" + )[1:] + ] + assert commands, "nothing runs after the delegate is linked" + guard = [b for b in commands if "check_imports_executorch_runtime.sh" in b] + assert ( + guard + ), f"the guard is not run after linking: {len(commands)} post-build steps" + # It has to receive the reader and the built library, or it checks nothing useful. + assert "TORCH_TENSORRT_READELF" in guard[0], guard[0][:300] + assert "TARGET_FILE:executorch_backend_tensorrt" in guard[0], guard[0][:300] + + +@pytest.mark.unit +def test_the_backend_shares_one_tensorrt_runtime() -> None: + """Loading several delegated programs at once crashed, with TensorRT saying why. + + Each program used to build its own runtime, and TensorRT logged that the logger differed from one + already registered and was ignored, which is it telling us there is state behind these objects + that a second one collides with. Four threads each loading and running their own program then + died with a segmentation fault. One runtime for the process, and deserialization serialized, + because a runtime is not safe to use from two threads at once. + """ + source = ( + _ROOT / "cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp" + ).read_text(encoding="utf-8") + assert ( + "nvinfer1::IRuntime* shared_runtime()" in source + ), "no shared runtime accessor" + # Exactly one place builds it, and it is that accessor. + builds = [ + line + for line in source.splitlines() + if "createInferRuntime" in line and not line.lstrip().startswith("//") + ] + assert len(builds) == 2, builds # the shared accessor, and the separate blob reader + assert "std::lock_guard guard(deserialize_lock)" in source, source[:200] + # And the handle no longer carries one of its own. + header = ( + _ROOT / "cpp/include/torch_tensorrt/executorch/TensorRTBackend.h" + ).read_text(encoding="utf-8") + assert "IRuntime> runtime;" not in header, "the handle still owns a runtime" diff --git a/tests/py/dynamo/executorch/test_cmake_runtime.py b/tests/py/dynamo/executorch/test_cmake_runtime.py new file mode 100644 index 0000000000..99c1017e72 --- /dev/null +++ b/tests/py/dynamo/executorch/test_cmake_runtime.py @@ -0,0 +1,422 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +"""Exercise the installed companion CMake target with a CPU-only native fixture.""" + +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys + +import pytest + +pytestmark = pytest.mark.unit +_CONFIG = ( + Path(__file__).resolve().parents[4] + / "py/torch-tensorrt-executorch-runtime/cmake/torchtrt_executorch-config.cmake" +) + + +@pytest.fixture +def linker_tools(): + tools = {name: shutil.which(name) for name in ("cmake", "c++", "readelf")} + if sys.platform != "linux" or not all(tools.values()): + pytest.skip("needs cmake, a C++ compiler, readelf, and a Linux linker") + return tools + + +def _run(command, **kwargs): + result = subprocess.run(command, capture_output=True, text=True, **kwargs) + assert result.returncode == 0, result.stdout + result.stderr + return result.stdout + + +def _installed_consumer(tmp_path, tools, config, old_dtags, example=None): + prefix = tmp_path / "prefix" + config_dir = prefix / "lib/cmake/torchtrt_executorch" + config_dir.mkdir(parents=True) + (config_dir / _CONFIG.name).write_text(config) + libraries = ["executorch_backend_tensorrt", "unrelated"] + if example is not None: + components = ("backend_cuda", "kernels_optimized") + libraries.extend(f"executorch_{name}" for name in components) + et_config = prefix / "lib/cmake/executorch" + et_config.mkdir() + (et_config / "executorch-config.cmake").write_text( + "add_library(executorch::runtime INTERFACE IMPORTED)\n" + + "".join( + f"add_library(executorch::{name} SHARED IMPORTED)\n" + f"set_target_properties(executorch::{name} PROPERTIES\n" + f' IMPORTED_LOCATION "{prefix}/lib/libexecutorch_{name}.so"\n' + ' INTERFACE_LINK_OPTIONS "LINKER:--push-state,--no-as-needed,' + f'{prefix}/lib/libexecutorch_{name}.so,--pop-state")\n' + for name in components + ) + ) + for name in libraries: + source = tmp_path / f"{name}.cpp" + source.write_text( + "#include \nnamespace { struct Registration { " + f'Registration() {{ std::puts("{name}"); }}' + " } registration; }\n" + ) + _run( + [ + tools["c++"], + "-shared", + "-fPIC", + str(source), + f"-Wl,-soname,lib{name}.so", + "-o", + str(prefix / f"lib/lib{name}.so"), + ] + ) + app = tmp_path / "app" + app.mkdir() + (app / "main.cpp").write_text("int main() { return 0; }\n") + (app / "CMakeLists.txt").write_text( + "cmake_minimum_required(VERSION 3.28)\nproject(consumer LANGUAGES CXX)\n" + "add_executable(my_app main.cpp)\n" + 'target_link_options(my_app PRIVATE "LINKER:--as-needed")\n' + + ( + example + if example is not None + else "find_package(torchtrt_executorch REQUIRED)\n" + "target_link_libraries(my_app PRIVATE torchtrt::executorch_backend)\n" + ) + + f'\ntarget_link_libraries(my_app PRIVATE "{prefix}/lib/libunrelated.so")\n' + ) + build = tmp_path / "build" + flags = ["-DCMAKE_EXE_LINKER_FLAGS=-Wl,--disable-new-dtags"] if old_dtags else [] + _run( + [ + tools["cmake"], + "-S", + str(app), + "-B", + str(build), + f"-DCMAKE_PREFIX_PATH={prefix}", + f"-DCMAKE_CXX_COMPILER={tools['c++']}", + *flags, + ] + ) + _run([tools["cmake"], "--build", str(build)]) + dynamic = _run([tools["readelf"], "-dW", str(build / "my_app")]) + env = os.environ.copy() + env.pop("LD_LIBRARY_PATH", None) + env.pop("LD_PRELOAD", None) + output = _run([str(build / "my_app")], env=env) + return dynamic, output + + +def _assert_consumer(dynamic, output): + needed = re.findall(r"\(NEEDED\).*\[([^]]+)\]", dynamic) + assert "libexecutorch_backend_tensorrt.so" in needed, "missing delegate dependency" + assert "libunrelated.so" not in needed, "retained unrelated dependency" + assert "(RUNPATH)" in dynamic, "missing RUNPATH" + assert "(RPATH)" not in dynamic, "unexpected RPATH" + assert output.strip() == "executorch_backend_tensorrt", "wrong static initializers" + + +@pytest.mark.parametrize("old_dtags", [False, True]) +def test_installed_cmake_consumer(tmp_path, linker_tools, old_dtags): + _assert_consumer( + *_installed_consumer(tmp_path, linker_tools, _CONFIG.read_text(), old_dtags) + ) + + +@pytest.mark.parametrize( + "removed", ["retention", "pop_order", "pop_state", "new_dtags"] +) +def test_installed_cmake_consumer_rejects_removed_guards( + tmp_path, linker_tools, removed +): + config = _CONFIG.read_text() + retention = ( + '"LINKER:--push-state,--no-as-needed,' + '${TORCHTRT_EXECUTORCH_BACKEND_LIBRARY},--pop-state"' + ) + assert config.count(retention) == 1 + if removed == "retention": + config = config.replace(retention, "") + message = "missing delegate dependency" + elif removed == "pop_order": + config = config.replace( + retention, + '"LINKER:--push-state,--no-as-needed,--pop-state,' + '${TORCHTRT_EXECUTORCH_BACKEND_LIBRARY}"', + ) + message = "missing delegate dependency" + elif removed == "pop_state": + config = config.replace(retention, retention.replace(",--pop-state", "")) + message = "retained unrelated dependency" + else: + assert config.count("--enable-new-dtags,") == 1 + config = config.replace("--enable-new-dtags,", "") + message = "missing RUNPATH" + dynamic, output = _installed_consumer(tmp_path, linker_tools, config, True) + with pytest.raises(AssertionError, match=message): + _assert_consumer(dynamic, output) + + +def _mixed_example(sample): + if sample == "readme": + text = (_CONFIG.parents[1] / "README.md").read_text() + blocks = re.findall(r"```cmake\n(.*?)```", text, re.DOTALL) + # The one that asks for components, which is the recipe this test is about. A second block + # documents the component-free route for older CMake, and picking by position would silently + # start testing that one instead the next time the readme grows. + wanted = [block for block in blocks if "COMPONENTS" in block] + assert len(wanted) == 1, [block.splitlines()[:1] for block in blocks] + return wanted[0] + lines = _CONFIG.read_text().splitlines() + start = next( + i + for i, line in enumerate(lines) + if line.startswith("# find_package(executorch") + ) + commands = [] + for line in lines[start:]: + if not line.startswith("# "): + break + commands.append(line[4:]) + return "\n".join(commands) + + +@pytest.mark.parametrize("sample", ["readme", "config"]) +@pytest.mark.parametrize("removed", [None, "backend_cuda", "kernels_optimized"]) +def test_documented_mixed_consumer(tmp_path, linker_tools, sample, removed): + example = _mixed_example(sample) + if removed: + assert example.count(f"executorch::{removed}") == 1 + example = example.replace(f"executorch::{removed}", "") + dynamic, output = _installed_consumer( + tmp_path, linker_tools, _CONFIG.read_text(), False, example + ) + needed = re.findall(r"\(NEEDED\).*\[([^]]+)\]", dynamic) + assert "libexecutorch_backend_tensorrt.so" in needed + assert "libunrelated.so" not in needed + assert "executorch_backend_tensorrt" in output.splitlines() + assert "unrelated" not in output.splitlines() + for name in ("backend_cuda", "kernels_optimized"): + assert (f"libexecutorch_{name}.so" in needed) is (removed != name) + assert (f"executorch_{name}" in output.splitlines()) is (removed != name) + + +@pytest.mark.unit +def test_the_config_refuses_a_target_it_did_not_create() -> None: + """The in-tree build defines the same target name, as an interface over a private static copy. + + Reusing whatever is already there let a project that pulls in the in-tree target and then calls + find_package link the private copy and never touch the wheel's shared library, with only a + status message to say so. Only an imported shared library can be the target this config made. + """ + config = _CONFIG.read_text(encoding="utf-8") + assert "get_target_property" in config, config + assert 'STREQUAL "SHARED_LIBRARY"' in config, config + assert "FATAL_ERROR" in config, config + + +@pytest.mark.unit +def test_the_embedded_run_path_can_be_turned_off() -> None: + """The run path is this machine's absolute path and it reaches every consumer binary. + + That suits building against an installed wheel, which is what this package is for, and ruins + anything redistributable, so a consumer has to be able to decline it. + """ + config = _CONFIG.read_text(encoding="utf-8") + assert "TORCHTRT_EXECUTORCH_EMBED_RUNPATH" in config, config + # Honoured rather than declared with option(), which inside a package config creates a cache + # entry in the consumer's project and can override a plain variable they already set. + assert "if(NOT DEFINED TORCHTRT_EXECUTORCH_EMBED_RUNPATH)" in config, config + assert "option(TORCHTRT_EXECUTORCH_EMBED_RUNPATH" not in config, config + # The run path lives inside the branch the switch controls, so turning it off omits it. + branch = config.split("if(TORCHTRT_EXECUTORCH_EMBED_RUNPATH)", 1)[1].split( + "else()", 1 + )[0] + assert "INTERFACE_LINK_OPTIONS" in branch, branch[:300] + # Turning it off does not leave a binary free of this machine's paths: linking an imported + # library makes CMake record its directory as a run path anyway. Measured on Linux, a consumer + # with no link options from this package still carries DT_RUNPATH, and only + # CMAKE_SKIP_BUILD_RPATH drops it. The off branch has to say so, or the switch reads as doing + # more than it does. + off_branch = config.split("else()", 1)[1] + assert "CMAKE_SKIP_BUILD_RPATH" in off_branch, off_branch[:400] + + +@pytest.mark.unit +def test_the_collision_guard_checks_which_library_the_target_points_at() -> None: + """Matching the type is not identifying the target. + + A shared imported target of the same name pointing at a different file passes a type check, and + the consumer then links that file while believing it linked the one this package found. + """ + config = _CONFIG.read_text(encoding="utf-8") + assert "IMPORTED_LOCATION" in config, config + assert ( + 'STREQUAL "${TORCHTRT_EXECUTORCH_BACKEND_LIBRARY}"' in config + ), "the guard does not compare against the library this package found" + # Both guards, so a same-type impostor is still refused. + assert config.count("FATAL_ERROR") >= 2, config + + +@pytest.mark.unit +def test_the_config_does_not_raise_the_consumers_cmake_floor() -> None: + """A consumer who can build against ExecuTorch has to be able to build against this. + + ExecuTorch's own package declares 3.19 and falls back to a variable-based path below 3.28, + because the token that misbehaves on older CMake is the origin token in its link options. Ours + are an absolute path and use nothing newer, so demanding 3.28 only shut out consumers ExecuTorch + itself supports. Declaring any floor here also overwrites the consumer's own recorded minimum, + since a config runs inside their project. + """ + config = _CONFIG.read_text(encoding="utf-8") + lines = [ + line + for line in config.splitlines() + if "cmake_minimum_required" in line and not line.lstrip().startswith("#") + ] + assert ( + not lines + ), f"the config declares a floor, which overwrites the consumer's: {lines}" + assert "if(CMAKE_VERSION VERSION_LESS 3.19)" in config, config + # No version check above the ExecuTorch floor. Prose explaining why 3.28 was wrong is fine + # and worth keeping, so read the checks rather than the whole text. + checks = [ + line.strip() + for line in config.splitlines() + if line.lstrip().startswith("if(CMAKE_VERSION") + ] + assert checks == ["if(CMAKE_VERSION VERSION_LESS 3.19)"], checks + + +@pytest.mark.unit +def test_the_config_looks_for_the_delegate_in_one_place_only() -> None: + """Discovery used to walk up until a delegate turned up under lib/. + + That walk reached the directory above the package, so with this package's own lib/ empty it + accepted a same-named library belonging to something else and reported success. A consumer then + linked and loaded a stranger's library believing it was this one. + """ + config = _CONFIG.read_text(encoding="utf-8") + assert "foreach" not in config, "discovery still walks parent directories" + assert "get_filename_component" in config, config + # One test for the library, at the fixed distance the wheel installs this file at. + assert config.count('EXISTS "${_torchtrt_executorch_root}/lib/') == 1, config + + +@pytest.mark.unit +def test_the_published_target_is_visible_outside_the_finding_directory() -> None: + """A project of more than one directory could not use the one target this package publishes. + + An imported target is scoped to the directory that created it. Descendants inherit it, so a + find_package at the top level looks fine, but the ordinary layout where one directory finds the + package and a sibling links it fails with a message about a target that plainly exists. + """ + config = _CONFIG.read_text(encoding="utf-8") + assert ( + "add_library(torchtrt::executorch_backend SHARED IMPORTED GLOBAL)" in config + ), config + + +@pytest.mark.unit +def test_the_cuda_extension_alias_yields_to_a_consumers_own_target() -> None: + """Adding the alias unconditionally broke a consumer that already had that name. + + Bringing ExecuTorch in as a subdirectory defines a plain extension_cuda target, and creating an + alias of the same name on top of it is a hard error, so using both packages together stopped + working where it used to. + """ + source = ( + Path(__file__).resolve().parents[4] + / "cpp/src/torch_tensorrt/executorch/CMakeLists.txt" + ).read_text(encoding="utf-8") + alias = source.split("add_library(extension_cuda ALIAS")[0] + assert "if(NOT TARGET extension_cuda)" in alias.rsplit("elseif", 1)[-1], alias[ + -400: + ] + + +@pytest.mark.parametrize("embed", [True, False]) +def test_the_run_path_opt_out_changes_what_the_consumer_links( + tmp_path, linker_tools, embed +): + """The opt-out was checked by looking for words in the config, so forcing it on always passed. + + A consumer that turns it off links differently, and that is the only thing worth asserting. Note + the option this adds is not the only run path on the link line: CMake adds its own for an imported + library at an absolute path, which the opt-out does not claim to remove. + """ + prefix = tmp_path / "prefix" + config_dir = prefix / "lib/cmake/torchtrt_executorch" + config_dir.mkdir(parents=True) + (config_dir / _CONFIG.name).write_text(_CONFIG.read_text(encoding="utf-8")) + (prefix / "lib" / "libexecutorch_backend_tensorrt.so").write_bytes(b"") + project = tmp_path / "consumer" + project.mkdir() + (project / "main.cpp").write_text("int main() { return 0; }\n") + (project / "CMakeLists.txt").write_text( + "cmake_minimum_required(VERSION 3.20)\n" + "project(consumer CXX)\n" + f'list(APPEND CMAKE_PREFIX_PATH "{prefix}")\n' + "find_package(torchtrt_executorch REQUIRED)\n" + "add_executable(app main.cpp)\n" + "target_link_libraries(app PRIVATE torchtrt::executorch_backend)\n" + ) + build = tmp_path / "build" + _run( + [ + linker_tools["cmake"], + "-S", + str(project), + "-B", + str(build), + f"-DTORCHTRT_EXECUTORCH_EMBED_RUNPATH={'ON' if embed else 'OFF'}", + ] + ) + link_line = (build / "CMakeFiles/app.dir/link.txt").read_text(encoding="utf-8") + # The tag this package asks for appears only when the option is left on. + assert ("--enable-new-dtags" in link_line) is embed, link_line + + +@pytest.mark.parametrize("collision", ["wrong_type", "other_file"]) +def test_a_target_of_that_name_already_present_is_refused( + tmp_path, linker_tools, collision +): + """Both guards were checked by looking for words, so disabling both conditions passed. + + A consumer that already defines this target, as the wrong kind or pointing at a different file, + would otherwise link something other than this package's delegate while believing it had this one. + """ + prefix = tmp_path / "prefix" + config_dir = prefix / "lib/cmake/torchtrt_executorch" + config_dir.mkdir(parents=True) + (config_dir / _CONFIG.name).write_text(_CONFIG.read_text(encoding="utf-8")) + (prefix / "lib" / "libexecutorch_backend_tensorrt.so").write_bytes(b"") + (prefix / "lib" / "someone_elses.so").write_bytes(b"") + if collision == "wrong_type": + preamble = "add_library(torchtrt::executorch_backend INTERFACE IMPORTED)\n" + else: + preamble = ( + "add_library(torchtrt::executorch_backend SHARED IMPORTED)\n" + "set_target_properties(torchtrt::executorch_backend PROPERTIES\n" + f' IMPORTED_LOCATION "{prefix}/lib/someone_elses.so")\n' + ) + project = tmp_path / "consumer" + project.mkdir() + (project / "CMakeLists.txt").write_text( + "cmake_minimum_required(VERSION 3.20)\n" + "project(consumer NONE)\n" + + preamble + + f'list(APPEND CMAKE_PREFIX_PATH "{prefix}")\n' + "find_package(torchtrt_executorch REQUIRED)\n" + ) + result = subprocess.run( + [linker_tools["cmake"], "-S", str(project), "-B", str(tmp_path / "build")], + capture_output=True, + text=True, + ) + assert result.returncode != 0, result.stdout + assert "CMake Error" in result.stdout + result.stderr, result.stdout diff --git a/tests/py/dynamo/executorch/test_cuda_partitioner_composition.py b/tests/py/dynamo/executorch/test_cuda_partitioner_composition.py index c9eb6544f4..1156cffc4a 100644 --- a/tests/py/dynamo/executorch/test_cuda_partitioner_composition.py +++ b/tests/py/dynamo/executorch/test_cuda_partitioner_composition.py @@ -355,3 +355,32 @@ def forward(self, x): assert not list( tmp_path.glob("*.ptd") ), "TRT-only program must not write an external .ptd" + + +@pytest.mark.parametrize( + "requested,accepted", + [ + (b"cuda", True), + (b"cuda:0", True), + (b"cuda:3", True), + (b"cpu", False), + (b"mps", False), + ], +) +def test_the_partitioner_refuses_a_device_it_cannot_run_on(requested, accepted): + """A processor target was taken verbatim and produced a program that died at instruction 0. + + This delegate compiles to TensorRT engines, so a target that is not a CUDA device cannot work. + Accepting it moved the failure to run time, where the message said nothing about the request that + caused it. + """ + from executorch.exir.backend.compile_spec_schema import CompileSpec + + from torch_tensorrt.executorch.partitioner import TensorRTPartitioner + + specs = [CompileSpec("target_device", requested)] + if accepted: + TensorRTPartitioner(compile_specs=specs) + else: + with pytest.raises(ValueError, match="not a device this delegate runs on"): + TensorRTPartitioner(compile_specs=specs) diff --git a/tests/py/dynamo/executorch/test_example_boundaries.py b/tests/py/dynamo/executorch/test_example_boundaries.py new file mode 100644 index 0000000000..86abc2da5a --- /dev/null +++ b/tests/py/dynamo/executorch/test_example_boundaries.py @@ -0,0 +1,391 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +"""Exercise example input/output checks without importing the GPU compiler stack.""" + +import argparse +import ast +import enum +import sys +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest + +pytestmark = pytest.mark.unit +_ROOT = Path(__file__).resolve().parents[4] +_EXPORT = _ROOT / "examples/torchtrt_executorch_example/export_device_resident.py" + + +class _Tensor: + """Stands in for the schema's tensor, which the device check tests with isinstance.""" + + def __init__(self, extra_tensor_info=None): + self.extra_tensor_info = extra_tensor_info + + +class _DeviceType(enum.IntEnum): + CPU = 0 + CUDA = 1 + + +def _export( + monkeypatch, + path, + remove_guard=False, + delegates=None, + operators=(), + copy_ops=None, + boundary_devices=(), +): + tree = ast.parse(_EXPORT.read_text()) + main = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "main" + ) + if remove_guard: + guards = [node for node in main.body if isinstance(node, ast.If)] + assert len(guards) == 1 + main.body.remove(guards[0]) + tensor = SimpleNamespace( + shape=(64, 64), + cuda=lambda: tensor, + flatten=lambda: [SimpleNamespace(item=lambda: 0.5)], + ) + + class Model: + def eval(self): + return self + + def cuda(self): + return self + + def __call__(self, inputs): + return tensor + + def save(model, filename, **kwargs): + Path(filename).write_bytes(b"exported program") + + def config(**kwargs): + return kwargs + + namespace = { + "argparse": argparse, + "Path": Path, + "sys": sys, + "CoalescedModel": Model, + "SHAPE": (64, 64), + # No BOUNDARY_COPY_OPS here on purpose. Supplying it meant the script's own constant was + # never read, so emptying that constant, which is the exact failure the comment beside it + # warns about, changed nothing. The module-level assignments run below instead, and a case + # that wants a different value overrides it afterwards. + "torch": SimpleNamespace( + no_grad=nullcontext, + randn=lambda _: tensor, + ones=lambda _: tensor, + export=SimpleNamespace(export=lambda *args: None), + ), + "torch_tensorrt": SimpleNamespace( + save=save, dynamo=SimpleNamespace(compile=lambda *args, **kwargs: None) + ), + "CudaPartitioner": lambda *args: None, + "CudaBackend": SimpleNamespace( + generate_method_name_compile_spec=lambda _: None + ), + # The two schema names the device check needs. A real enum, so the example's own + # DeviceType(device).name works unchanged. + "Tensor": _Tensor, + "DeviceType": _DeviceType, + "ExecutorchBackendConfig": config, + "PropagateDeviceConfig": config, + "MemoryPlanningPass": config, + "deserialize_pte_binary": lambda _: SimpleNamespace( + program=SimpleNamespace( + execution_plan=[ + SimpleNamespace( + delegates=[ + SimpleNamespace(id=name) + for name in ( + ("TensorRTBackend", "CudaBackend") + if delegates is None + else delegates + ) + ], + operators=[ + SimpleNamespace(name=n, overload="") for n in operators + ], + # A boundary carrying tensors, so the device check has something to + # inspect. Empty lists meant the whole check could be deleted unnoticed. + inputs=list(range(len(boundary_devices))), + outputs=[], + values=[ + SimpleNamespace( + val=_Tensor( + extra_tensor_info=( + None + if device is None + else SimpleNamespace(device_type=device) + ) + ) + ) + for device in boundary_devices + ], + ) + ] + ) + ), + } + monkeypatch.setattr(sys, "argv", [str(_EXPORT), "--model_path", str(path)]) + # The module's own assignments, then main. Compiling main alone left every module-level constant + # to be supplied by this test, which is how the boundary names stopped being checked at all. + module = ast.parse(_EXPORT.read_text(encoding="utf-8")) + # The module's plain constant assignments, then main. Anything else at module level needs stubs + # this test has no reason to grow, and the constants are what was going unread. + body = [ + node + for node in module.body + if isinstance(node, ast.Assign) + and all(isinstance(target, ast.Name) for target in node.targets) + and isinstance(node.value, (ast.Tuple, ast.List, ast.Constant)) + ] + body.append(main) + exec( + compile(ast.Module(body=body, type_ignores=[]), str(_EXPORT), "exec"), + namespace, + ) + if copy_ops is not None: + namespace["BOUNDARY_COPY_OPS"] = copy_ops + namespace["main"]() + + +@pytest.mark.parametrize("existing", [False, True]) +def test_export_rejects_reference_collision_before_writing( + monkeypatch, tmp_path, existing +): + path = tmp_path / "model.expected" + if existing: + path.write_bytes(b"keep existing output") + with pytest.raises(SystemExit) as error: + _export(monkeypatch, path) + assert error.value.code == 2 + assert ( + path.read_bytes() == b"keep existing output" if existing else not path.exists() + ) + + +def test_export_preserves_separate_model_and_reference(monkeypatch, tmp_path): + path = tmp_path / "model.pte" + _export(monkeypatch, path) + assert path.read_bytes() == b"exported program" + assert path.with_suffix(".expected").read_text() == "[64,64]\n0.5000\n" + + +def test_collision_guard_removal_overwrites_model(monkeypatch, tmp_path): + path = tmp_path / "model.expected" + _export(monkeypatch, path, remove_guard=True) + assert path.read_text() == "[64,64]\n0.5000\n" + + +@pytest.mark.parametrize("optimize", [0, 2]) +@pytest.mark.parametrize( + "is_cuda,remove_guard", [(False, False), (True, False), (False, True)] +) +def test_the_cuda_gate_precedes_the_load(monkeypatch, optimize, is_cuda, remove_guard): + """The gate has to stop the program before it loads anything, and survive optimized Python. + + It used to be an assertion, which optimized Python strips, so the parameter covering that is not + incidental. The guard removed here is the availability gate, not a check on the tensor: a check on + the tensor cannot fire, because asking for the device has already failed by then. + """ + path = _ROOT / "examples/executorch_reference_runner/load_model_device_resident.py" + tree = ast.parse(path.read_text()) + tree.body = [ + node for node in tree.body if not isinstance(node, (ast.Import, ast.ImportFrom)) + ] + if remove_guard: + guards = [ + node + for node in tree.body + if isinstance(node, ast.If) + and ast.unparse(node.test) == "not torch.cuda.is_available()" + ] + assert len(guards) == 1 + tree.body.remove(guards[0]) + calls = [] + + def load(path): + calls.append(path) + raise LookupError("reached native load") + + namespace = { + "argparse": argparse, + "Path": Path, + "torch": SimpleNamespace( + float32=object(), + # The parameter drives availability, which is the thing the gate asks about. + cuda=SimpleNamespace(is_available=lambda: is_cuda), + # Honour the device the script asks for, so a script that forgot to request one is + # visible here rather than reported as a device tensor. + ones=lambda *args, device=None, **kwargs: SimpleNamespace( + is_cuda=str(device) == "cuda" + ), + ), + "_load_for_executorch": load, + } + monkeypatch.setattr(sys, "argv", [str(path), "--model_path", "unused.pte"]) + reaches_load = is_cuda or remove_guard + error = LookupError if reaches_load else RuntimeError + message = "reached native load" if reaches_load else "cannot run" + with pytest.raises(error, match=message): + exec(compile(tree, str(path), "exec", optimize=optimize), namespace) + assert calls == (["unused.pte"] if reaches_load else []) + + +@pytest.mark.parametrize( + "delegates,expected", + [ + (("CudaBackend",), "missing"), + (("TensorRTBackend",), "missing"), + ((), "missing"), + ], +) +def test_export_rejects_a_program_that_is_not_coalesced( + monkeypatch, tmp_path, delegates, expected +): + """The coalescing check never ran, because the stub program always carried both delegates.""" + with pytest.raises(SystemExit, match=expected): + _export(monkeypatch, tmp_path / "m.pte", delegates=delegates) + + +def test_export_rejects_a_program_that_still_copies_at_the_boundary( + monkeypatch, tmp_path +): + """The copy check never ran either: the operator table was empty and the names it looks for + were an empty tuple, so nothing could match and the rejection path was unreachable. + """ + with pytest.raises(SystemExit, match="still copies across the method boundary"): + _export( + monkeypatch, + tmp_path / "m.pte", + operators=("aten::_h2d_copy_default",), + ) + + +def test_a_rejected_export_leaves_the_previous_program_alone(monkeypatch, tmp_path): + """Saving straight over the target let a rejected export destroy a good program. + + The reference file beside it then described something the program no longer was, which is worse + than no output at all because it looks like a successful export. + """ + model_path = tmp_path / "m.pte" + model_path.write_bytes(b"the good program") + with pytest.raises(SystemExit, match="missing"): + _export(monkeypatch, model_path, delegates=("CudaBackend",)) + assert model_path.read_bytes() == b"the good program" + assert not list(tmp_path.glob("*.staged")), "the staging file was left behind" + + +@pytest.mark.parametrize("output_on_cuda", [True, False]) +def test_the_runner_rejects_an_output_that_came_back_on_the_host( + monkeypatch, tmp_path, output_on_cuda +): + """The check the whole example exists for had nothing reaching it. + + The other device test stops at the load, so the output guard never ran and removing it left the + suite green. This one lets the load succeed and controls only where the output claims to live. + """ + path = _ROOT / "examples/executorch_reference_runner/load_model_device_resident.py" + tree = ast.parse(path.read_text()) + tree.body = [ + node for node in tree.body if not isinstance(node, (ast.Import, ast.ImportFrom)) + ] + output = SimpleNamespace( + is_cuda=output_on_cuda, device="cuda:0" if output_on_cuda else "cpu" + ) + program = SimpleNamespace( + method_names=lambda: ["forward"], + run_method=lambda name, inputs: [output], + ) + namespace = { + "argparse": argparse, + "Path": Path, + "torch": SimpleNamespace( + float32=object(), + cuda=SimpleNamespace(is_available=lambda: True), + ones=lambda *args, device=None, **kwargs: SimpleNamespace( + is_cuda=str(device) == "cuda" + ), + ), + "_load_for_executorch": lambda _: program, + } + monkeypatch.setattr(sys, "argv", [str(path), "--model_path", "unused.pte"]) + if output_on_cuda: + # It gets past the guard and fails later on something this stub does not provide. Anything + # except the guard's own complaint proves the guard let a device output through. + with pytest.raises(Exception) as caught: + exec(compile(tree, str(path), "exec"), namespace) + assert "output came back on" not in str(caught.value), caught.value + else: + with pytest.raises(AssertionError, match="output came back on"): + exec(compile(tree, str(path), "exec"), namespace) + + +def test_the_runner_refuses_to_start_without_cuda(monkeypatch, tmp_path): + """The gate that stops the runner on a machine with no CUDA had nothing exercising it. + + Every other case stubs CUDA as available, so deleting the gate left the whole suite green. + """ + path = _ROOT / "examples/executorch_reference_runner/load_model_device_resident.py" + tree = ast.parse(path.read_text()) + tree.body = [ + node for node in tree.body if not isinstance(node, (ast.Import, ast.ImportFrom)) + ] + namespace = { + "argparse": argparse, + "Path": Path, + "torch": SimpleNamespace( + float32=object(), + cuda=SimpleNamespace(is_available=lambda: False), + ones=lambda *args, device=None, **kwargs: SimpleNamespace(is_cuda=False), + ), + "_load_for_executorch": lambda _: None, + } + monkeypatch.setattr(sys, "argv", [str(path), "--model_path", "unused.pte"]) + # The specific message, not just any mention of CUDA. With this gate deleted the input guard + # further down raises its own CUDA complaint, so a loose match passes either way and the check + # says nothing about the gate it is named for. + with pytest.raises( + RuntimeError, match="cannot run\nwithout CUDA|cannot run without CUDA" + ): + exec(compile(tree, str(path), "exec"), namespace) + + +@pytest.mark.parametrize( + "devices,rejected", + [ + ((_DeviceType.CUDA,), False), + ((_DeviceType.CUDA, _DeviceType.CUDA), False), + ((_DeviceType.CPU,), True), + ((_DeviceType.CUDA, _DeviceType.CPU), True), + ((None,), True), + ], +) +def test_export_rejects_a_boundary_tensor_that_is_not_on_the_device( + monkeypatch, tmp_path, devices, rejected +): + """Deleting the whole device check kept the suite green, because no boundary carried tensors. + + A program whose method boundary holds host memory defeats the point of a device-resident export, + and the caller would find out at run time instead. A tensor with no device information counts as + host, which is what the example's own default says. + """ + path = tmp_path / "m.pte" + path.write_bytes(b"program") + if rejected: + with pytest.raises(SystemExit) as raised: + _export(monkeypatch, path, boundary_devices=devices) + assert "non-CUDA method boundary tensors" in str(raised.value), raised.value + else: + _export(monkeypatch, path, boundary_devices=devices) diff --git a/tests/py/dynamo/executorch/test_executorch_pin.py b/tests/py/dynamo/executorch/test_executorch_pin.py index e60fa91388..9af14fbed0 100644 --- a/tests/py/dynamo/executorch/test_executorch_pin.py +++ b/tests/py/dynamo/executorch/test_executorch_pin.py @@ -17,6 +17,8 @@ from collections import Counter from pathlib import Path +import types + import pytest import yaml from packaging.requirements import Requirement @@ -546,8 +548,15 @@ def test_derived_requirements_match_the_pin(monkeypatch) -> None: @pytest.mark.unit -def test_the_runtime_wheel_pins_executorch_to_the_public_pin(monkeypatch) -> None: - """Evaluate the metadata without invoking a native build.""" +def test_the_runtime_wheel_pins_the_executorch_build_it_linked(monkeypatch) -> None: + """The label naming the CUDA build is part of the pin, not noise to strip. + + The delegate links one specific ExecuTorch build. A requirement carrying only the public version + is satisfied by a processor-only build, or another CUDA build of the same date, so the pin would + look exact while permitting the pairings it exists to refuse. + + Evaluated without invoking a native build. + """ import importlib.metadata import runpy import types @@ -580,7 +589,7 @@ def test_the_runtime_wheel_pins_executorch_to_the_public_pin(monkeypatch) -> Non requirements = [ r for r in metadata["install_requires"] if r.startswith("executorch") ] - assert requirements == [f"executorch=={pin}"] + assert requirements == [f"executorch=={pin}+cu132"], requirements def _declared_cuda_versions(name: str) -> set[str]: @@ -1553,18 +1562,32 @@ def test_the_no_nightly_marker_only_exempts_a_win32_install(): # nearest control-flow keyword above it. Requiring that keyword to be the win32 guard # ties the exemption to the one platform it describes: a marker pasted onto a Linux # "else" install resolves to that "else", not to "if ... win32", and is rejected. - branch = next( - (lines[j] for j in range(index - 1, -1, -1) if control.match(lines[j])), + # The exemption is only honest when the install below it cannot pull the companion in, + # since the companion is what needs the ExecuTorch channel. An unanchored + # torch_tensorrt* glob also matches torch_tensorrt_executorch_runtime, so requiring the + # hyphen ties the exemption to installing the main wheel alone rather than to a platform + # that merely happens to do so. + install = next( + ( + lines[j] + for j in range(index + 1, min(index + 5, len(lines))) + if "pip install" in lines[j] + ), "", ) - if "win32" not in branch: + # Either the install names an explicitly filtered list, or it globs with the hyphen + # anchor. What it may not do is pass a bare torch_tensorrt* glob, which also matches + # the companion, and the companion is what needs the channel. + unfiltered = "torch_tensorrt*" in install + if unfiltered and "${wheels}" not in install: misplaced.append( - f"{name}:{index + 1} carries {NO_NIGHTLY_MARKER!r} outside a win32 branch, " - "so it would exempt a Linux install that simply lost its index" + f"{name}:{index + 1} carries {NO_NIGHTLY_MARKER!r} above an install that can " + f"match the companion wheel, which does need the channel: {install.strip()!r}" ) assert not misplaced, ( - "the no-nightly exemption is only valid inside a win32 branch: " f"{misplaced}" + "the no-nightly exemption is only valid above a main-wheel-only install: " + f"{misplaced}" ) @@ -2445,6 +2468,10 @@ def test_suite_validation_check_detects_removed_validator(monkeypatch, field, va ("schedule", "refs/heads/release/2.14", "", "", "true"), # The shipped-release guard now sits where it can actually fire. ("workflow_dispatch", "refs/heads/release/2.14", "stable", None, "true"), + # Anything other than an explicit false counts as tagged, so a typo closes the guard. + ("workflow_dispatch", "refs/heads/release/2.14", "stable", None, "yes"), + ("workflow_dispatch", "refs/heads/release/2.14", "stable", None, "1"), + ("workflow_dispatch", "refs/heads/release/2.14", "stable", "stable", "false"), # A branch that only looks like a release must not be treated as one. ( "schedule", @@ -2527,3 +2554,71 @@ def test_the_install_message_does_not_hand_a_no_op_command_to_other_platforms( message = command() assert "pip install" not in message, message assert "Linux" in message, message + + +@pytest.mark.parametrize("agree", [True, False]) +@pytest.mark.unit +def test_the_pairing_check_fails_when_the_two_pins_disagree( + agree: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + """The pairing check is the only guard on the two pins naming one ExecuTorch. + + It skips whenever the pinned wheel is not installed, which is every pull request lane, so + without this it could stop working and nothing would say so. A stand-in wheel is injected + reporting the pinned version and either the pinned commit or a different one, which is the + only difference the check exists to notice. + """ + versions = _versions() + pinned_commit = versions["__executorch_commit__"] + module = types.ModuleType("executorch.version") + module.__version__ = versions["__executorch_version__"] + "+cu134" + module.git_version = pinned_commit if agree else "f" * 40 + package = types.ModuleType("executorch") + package.version = module + monkeypatch.setitem(sys.modules, "executorch", package) + monkeypatch.setitem(sys.modules, "executorch.version", module) + if agree: + test_the_pinned_commit_is_the_pinned_wheels_own_source() + else: + with pytest.raises(AssertionError): + test_the_pinned_commit_is_the_pinned_wheels_own_source() + + +@pytest.mark.parametrize("branch", ["windows", "linux"]) +def test_the_install_script_leaves_the_companion_out(tmp_path, branch): + """Removing both exclusions left every pin test green, so nothing protected this. + + Installing the companion from this script is what forced a nightly index onto release jobs. The + selection works by excluding the companion rather than by matching the main wheel, because the + main wheel's name varies by variant and a prefix guess leaves the pattern unexpanded for pip to + read literally. That is exactly what a test reading the script for the words cannot check. + """ + source = (REPO_ROOT / ".github/scripts/install-torch-tensorrt.sh").read_text( + encoding="utf-8" + ) + # Take the case block straight from the script, so the test cannot drift from what runs. + cases = re.findall(r"case \"\$\{wheel\}\" in.*?esac", source, re.S) + assert len(cases) == 2, len(cases) + case = cases[0 if branch == "windows" else 1] + directory = tmp_path / "artifacts" + directory.mkdir() + for name in ( + "torch_tensorrt-2.15.0.dev1+cu134-cp310-cp310-linux_x86_64.whl", + "torch_tensorrt_rtx-2.15.0.dev1+cu134-cp310-cp310-linux_x86_64.whl", + "torch_tensorrt_executorch_runtime-0.2.0.dev1+cu134-py3-none-linux_x86_64.whl", + ): + (directory / name).write_bytes(b"") + script = ( + 'wheels=""\n' + f"for wheel in {directory}/torch_tensorrt*.whl; do\n" + f"{case}\n" + ' wheels="${wheels} ${wheel}"\n' + "done\n" + "echo ${wheels}\n" + ) + result = subprocess.run(["sh", "-c", script], text=True, capture_output=True) + assert result.returncode == 0, result.stderr + selected = [Path(name).name for name in result.stdout.split()] + assert not any("executorch_runtime" in name for name in selected), selected + # Both of the others, including the variant a prefix match would have missed. + assert len(selected) == 2, selected diff --git a/tests/py/dynamo/executorch/test_load_compatibility.py b/tests/py/dynamo/executorch/test_load_compatibility.py new file mode 100644 index 0000000000..3316f673ec --- /dev/null +++ b/tests/py/dynamo/executorch/test_load_compatibility.py @@ -0,0 +1,717 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +"""Exercise the real loader and compatibility wrapper with a controlled native boundary.""" + +import ast +import importlib.util +import inspect +import os +import sys +import types +import warnings +from pathlib import Path + +import pytest +import torch + +pytestmark = pytest.mark.unit +ROOT = Path(__file__).parents[4] +PORTABLE = "executorch.extension.pybindings.portable_lib" +DELEGATE = "torch_tensorrt_executorch_runtime" + + +@pytest.fixture +def compiler(monkeypatch): + # Isolate the full loader module from compiler frontends that require a GPU. + for name in list(sys.modules): + if name == "torch_tensorrt" or name.startswith("torch_tensorrt."): + monkeypatch.delitem(sys.modules, name) + modules = { + "torch_tensorrt": {}, + "torch_tensorrt._enums": {"dtype": object}, + "torch_tensorrt._features": { + "ENABLED_FEATURES": types.SimpleNamespace( + fx_frontend=False, + torchscript_frontend=False, + dynamo_frontend=False, + torch_tensorrt_runtime=True, + ), + "needs_cross_compile": lambda fn: fn, + }, + "torch_tensorrt._Input": {"Input": type("Input", (), {})}, + "torch_tensorrt._utils": {"executorch_install_command": lambda: "unused"}, + "torch_tensorrt.dynamo": {}, + "torch_tensorrt.dynamo.runtime": {}, + "torch_tensorrt.dynamo.runtime._CudaGraphsTorchTensorRTModule": { + "CudaGraphsTorchTensorRTModule": type( + "CudaGraphsTorchTensorRTModule", (), {} + ) + }, + } + for name, attributes in modules.items(): + module = types.ModuleType(name) + module.__dict__.update(attributes) + module.__path__ = [str(ROOT / "py" / name.replace(".", "/"))] + monkeypatch.setitem(sys.modules, name, module) + # Track these entries before Python imports them so monkeypatch restores them too. + for name in ("torch_tensorrt._compile", "torch_tensorrt._executorch_compat"): + monkeypatch.setitem(sys.modules, name, None) + monkeypatch.delitem(sys.modules, name) + spec = importlib.util.spec_from_file_location( + "torch_tensorrt._compile", ROOT / "py/torch_tensorrt/_compile.py" + ) + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, module) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def boundary(monkeypatch, tmp_path): + events = [] + state = types.SimpleNamespace(events=events, output=None) + + class NativeModule: + def method_names(self): + return ["forward", "add", "constant", "empty"] + + def run_method(self, name, inputs): + events.append((name, inputs)) + if name == "forward": + state.output = [inputs[0] + 1] + elif name == "add": + state.output = [inputs[0] + inputs[1], inputs[0]] + elif name == "constant": + state.output = [17, None, "constant"] + elif name == "empty": + state.output = [] + else: + raise AssertionError("Unknown method reached the native runtime") + return state.output + + def register(): + events.append("register") + + def load(data): + assert events[-1] == "register" + assert data == b"controlled native program" + state.data_id = id(data) + events.append("load") + return state.native + + state.native = NativeModule() + for name in ( + "executorch", + "executorch.extension", + "executorch.extension.pybindings", + PORTABLE, + DELEGATE, + ): + module = types.ModuleType(name) + module.__path__ = [] + monkeypatch.setitem(sys.modules, name, module) + state.portable = sys.modules[PORTABLE] + state.portable._load_for_executorch_from_buffer = load + state.delegate = sys.modules[DELEGATE] + state.delegate.register = register + # The host Program loader and removed private companion runtime must not be used. + monkeypatch.setitem(sys.modules, "executorch.runtime", None) + monkeypatch.setitem(sys.modules, DELEGATE + ".runtime", None) + state.path = tmp_path / "model.pte" + state.path.write_bytes(b"controlled native program") + return state + + +def load_legacy(compiler, path, **kwargs): + with pytest.warns(DeprecationWarning, match="format='executorch'"): + return compiler.load(path, format="executorch", **kwargs) + + +@pytest.mark.parametrize("as_string", [False, True]) +def test_released_program_interface(compiler, boundary, as_string): + path = str(boundary.path) if as_string else boundary.path + program = load_legacy(compiler, path) + assert not callable(program) + assert callable(program.forward) and callable(program.run) + assert program.method_names == ["forward", "add", "constant", "empty"] + assert not callable(program.method_names) + assert boundary.events == ["register", "load"] + assert id(program._data) == boundary.data_id + assert sys.modules[PORTABLE] is boundary.portable + assert sys.modules[DELEGATE] is boundary.delegate + x = torch.tensor([2.0, 4.0]) + for result in (program.forward(x), program.run([x])): + torch.testing.assert_close(result[0], x + 1) + result = program.run((x, 3), method="add") + assert result is boundary.output + torch.testing.assert_close(result[0], x + 3) + assert result[1] is x + assert program.run([], "constant") == [17, None, "constant"] + assert program.run([], "empty") == [] + with pytest.raises(TypeError): + program(x) + with pytest.raises(TypeError): + program.forward(x=x) + + +def test_warning_identifies_caller_and_deprecation_period(compiler, boundary): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + line = inspect.currentframe().f_lineno + 1 + compiler.load(boundary.path, format="executorch") + assert len(caught) == 1 + warning = caught[0] + assert warning.category is DeprecationWarning + assert warning.filename == __file__ and warning.lineno == line + assert "six months" in str(warning.message) + assert "_load_for_executorch" in str(warning.message) + + +def test_legacy_options_remain_ignored(compiler, boundary): + extra_files = {"metadata": "unchanged"} + load_legacy( + compiler, + boundary.path, + extra_files=extra_files, + data_path="not-forwarded.ptd", + enable_etdump=True, + map_location="cuda:0", + unknown_option=object(), + ) + assert boundary.events == ["register", "load"] + assert extra_files == {"metadata": "unchanged"} + + +def test_cuda_normalization_preserves_other_inputs(compiler, boundary, monkeypatch): + """The CUDA tensor is simulated on CPU; no GPU execution is claimed.""" + cpu = torch.tensor([3.0]) + copies = [] + + class SimulatedCudaTensor(torch.Tensor): + @property + def is_cuda(self): + return True + + def cpu(self): + copies.append(self) + return cpu + + class NonTensor: + is_cuda = True + + def cpu(self): + raise AssertionError("Only CUDA torch.Tensor inputs are copied") + + cuda = cpu.as_subclass(SimulatedCudaTensor) + other = NonTensor() + inputs = [cuda, cpu, other, [cuda], None] + received = [] + + def run_method(name, values): + received.append((name, values)) + return values + + monkeypatch.setattr(boundary.native, "run_method", run_method) + program = load_legacy(compiler, boundary.path) + result = program.forward(*inputs) + assert len(copies) == 1 and copies[0] is cuda + assert isinstance(received[0][1], tuple) + assert received[0][0] == "forward" + assert result[0] is cpu and result[1] is cpu + assert result[2] is other and result[3] is inputs[3] and result[4] is None + assert inputs[0] is cuda + + +def test_unknown_method_fails_before_native_dispatch(compiler, boundary): + program = load_legacy(compiler, boundary.path) + with pytest.raises(ValueError) as raised: + program.run([], "missing") + assert str(raised.value) == ( + "Unknown method 'missing'; available methods: ['add', 'constant', 'empty', 'forward']" + ) + assert boundary.events == ["register", "load"] + + +@pytest.mark.parametrize("kind", ["missing", "directory"]) +def test_missing_model_has_released_error(compiler, boundary, kind): + path = ( + boundary.path.parent + if kind == "directory" + else boundary.path.parent / "missing.pte" + ) + with pytest.raises(FileNotFoundError, match="ExecuTorch model not found"): + load_legacy(compiler, path) + assert "load" not in boundary.events + + +@pytest.mark.parametrize("stage", ["register", "load", "run"]) +def test_native_failures_propagate_without_fallback( + compiler, boundary, monkeypatch, stage +): + failure = RuntimeError("controlled native failure") + + def fail(*args, **kwargs): + raise failure + + def unexpected(*args, **kwargs): + pytest.fail("ExecuTorch errors must not reach another format's loader") + + monkeypatch.setattr(torch.export, "load", unexpected) + monkeypatch.setattr(torch.jit, "load", unexpected) + target, attribute = { + "register": (boundary.delegate, "register"), + "load": (boundary.portable, "_load_for_executorch_from_buffer"), + "run": (boundary.native, "run_method"), + }[stage] + monkeypatch.setattr(target, attribute, fail) + with pytest.raises(RuntimeError) as raised: + program = load_legacy(compiler, boundary.path) + program.forward(torch.tensor(1)) + assert raised.value is failure + + +def test_deferred_module_validation_is_preserved(compiler, boundary, monkeypatch): + failure = RuntimeError("Failed to get method names: invalid program") + + def fail(): + raise failure + + monkeypatch.setattr(boundary.native, "method_names", fail) + program = load_legacy(compiler, boundary.path) + with pytest.raises(RuntimeError) as raised: + program.forward(torch.tensor(1)) + assert raised.value is failure + assert boundary.events == ["register", "load"] + + +@pytest.mark.parametrize("error_type", [ImportError, OSError]) +def test_broken_delegate_import_keeps_native_diagnostic( + compiler, boundary, monkeypatch, error_type +): + import builtins + + original = builtins.__import__ + failure = error_type("libexecutorch.so: undefined symbol") + + def import_module(name, *args, **kwargs): + if name == DELEGATE: + raise failure + return original(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", import_module) + with pytest.raises(error_type) as raised: + load_legacy(compiler, boundary.path) + assert raised.value is failure + assert boundary.events == [] + + +def test_standard_path_registers_python_engine_ops(compiler, monkeypatch): + import builtins + + original = builtins.__import__ + events = [] + name = "torch_tensorrt.dynamo.runtime._TRTEngine" + monkeypatch.setitem(sys.modules, name, types.ModuleType(name)) + monkeypatch.setattr(compiler.ENABLED_FEATURES, "torch_tensorrt_runtime", False) + + def import_module(module_name, *args, **kwargs): + if module_name == name: + events.append("register") + return original(module_name, *args, **kwargs) + + def load(path, extra_files=None): + events.append("load") + return "exported program" + + monkeypatch.setattr(builtins, "__import__", import_module) + monkeypatch.setattr(torch.export, "load", load) + assert compiler.load("standard.pt2", format=None) == "exported program" + assert events == ["register", "load"] + + +@pytest.mark.parametrize("missing", [DELEGATE, "transitive_dependency"]) +def test_missing_dependency_is_not_misdiagnosed( + compiler, boundary, monkeypatch, missing +): + import builtins + + original = builtins.__import__ + error = ModuleNotFoundError(f"No module named {missing!r}", name=missing) + + def import_module(name, *args, **kwargs): + if name == DELEGATE: + raise error + return original(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", import_module) + with pytest.raises(ImportError) as raised: + load_legacy(compiler, boundary.path) + if missing == DELEGATE: + assert "torch_tensorrt_executorch_runtime" in str(raised.value) + assert raised.value.__cause__ is error + else: + assert raised.value is error + assert boundary.events == [] + + +@pytest.mark.parametrize("format", ["torchscript", "exported_program", "", False, 1]) +def test_unsupported_format_preserves_value_error(compiler, boundary, format): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with pytest.raises(ValueError, match="Unsupported format"): + compiler.load(boundary.path, format=format) + assert not caught + assert boundary.events == [] + + +@pytest.mark.parametrize("options", [{}, {"format": None}]) +@pytest.mark.parametrize("fallback", [False, True]) +def test_standard_dispatch_and_kwargs( + compiler, boundary, monkeypatch, caplog, options, fallback +): + extra_files = {"metadata": ""} + calls = [] + expected = object() + + def export_load(path, extra_files): + calls.append(("export", path, extra_files)) + if fallback: + raise RuntimeError("not an exported program") + extra_files["metadata"] = "export data" + return expected + + def jit_load(path, map_location=None, _extra_files=None): + calls.append(("jit", path, map_location, _extra_files)) + _extra_files["metadata"] = "jit data" + return expected + + monkeypatch.setattr(torch.export, "load", export_load) + monkeypatch.setattr(torch.jit, "load", jit_load) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = compiler.load( + boundary.path, + extra_files, + map_location="cpu", + unknown_option=True, + **options, + ) + assert result is expected + assert calls[0] == ("export", boundary.path, extra_files) + assert len(calls) == (2 if fallback else 1) + if fallback: + assert calls[1] == ("jit", boundary.path, "cpu", extra_files) + assert extra_files["metadata"] == ("jit data" if fallback else "export data") + assert "Keyword argument unknown_option" in caplog.text + assert not caught + assert boundary.events == [] + + +@pytest.mark.parametrize("options", [{}, {"format": None}]) +@pytest.mark.parametrize("format", ["export", "jit"]) +def test_standard_formats_round_trip_real_cpu_torch( + compiler, boundary, options, format +): + model = torch.nn.Linear(2, 2).eval() + x = torch.tensor([[1.0, 2.0]]) + path = str(boundary.path.parent / "standard.pt2") + extra_files = {"metadata": ""} + if format == "export": + torch.export.save( + torch.export.export(model, (x,)), path, extra_files={"metadata": "kept"} + ) + else: + torch.jit.save( + torch.jit.trace(model, (x,)), path, _extra_files={"metadata": "kept"} + ) + loaded = compiler.load(path, extra_files=extra_files, **options) + actual = loaded.module()(x) if format == "export" else loaded(x) + torch.testing.assert_close(actual, model(x)) + assert extra_files["metadata"] == ("kept" if format == "export" else b"kept") + assert boundary.events == [] + + +def test_standard_failure_preserves_value_error(compiler, boundary): + with pytest.raises(ValueError, match="valid Torchscript module or ExportedProgram"): + compiler.load(str(boundary.path), format=None) + assert boundary.events == [] + + +def test_format_remains_keyword_only(compiler, boundary): + signature = inspect.signature(compiler.load) + assert signature.parameters["format"].kind is inspect.Parameter.KEYWORD_ONLY + assert signature.parameters["format"].default is None + with pytest.raises(TypeError): + compiler.load(boundary.path, None, "executorch") + assert boundary.events == [] + + +@pytest.mark.unit +def test_the_published_main_wheel_can_still_reach_the_loader_it_imports() -> None: + """The released main wheel does ``from ...runtime import load``, by name. + + Upgrading this package on its own must not break that call, so the submodule and the name both + have to survive as long as a released main wheel reaches for them. Asserting the import path the + way the published wheel writes it is what makes a deletion visible here rather than in a user's + traceback. + """ + package = ( + Path(__file__).resolve().parents[4] + / "py/torch-tensorrt-executorch-runtime" + / "torch_tensorrt_executorch_runtime" + ) + module_path = package / "runtime.py" + assert module_path.exists(), "the published main wheel imports this submodule" + tree = ast.parse(module_path.read_text(encoding="utf-8")) + exported = {node.name for node in tree.body if isinstance(node, ast.FunctionDef)} + assert "load" in exported, sorted(exported) + # Its one argument is the path, which is how the main wheel calls it. + load = next( + n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "load" + ) + assert [a.arg for a in load.args.args] == ["file_path"], [ + a.arg for a in load.args.args + ] + # Parsing alone cannot see a module that raises while being imported, and the published wheel + # imports this one, so run its top level. Everything it needs at that point is standard library; + # the import of the main wheel's loader sits inside the function and is not reached here. + namespace: dict[str, object] = { + "__name__": "torch_tensorrt_executorch_runtime.runtime" + } + exec(compile(tree, str(module_path), "exec"), namespace) + assert callable(namespace.get("load")), sorted(namespace) + + +@pytest.mark.unit +def test_an_old_companion_gets_advice_it_can_act_on( + compiler, monkeypatch, tmp_path +) -> None: + """An older companion refuses once ExecuTorch's own bindings are loaded, and says to import it + earlier. A caller cannot do that: the colliding import happens inside this library. So the + message has to name the thing that does work, which is upgrading the companion.""" + module = types.ModuleType("torch_tensorrt_executorch_runtime") + + def activate(): + raise ImportError("import torch_tensorrt_executorch_runtime before executorch") + + module.activate = activate + monkeypatch.setitem(sys.modules, "torch_tensorrt_executorch_runtime", module) + program = tmp_path / "m.pte" + program.write_bytes(b"unused") + with pytest.raises(ImportError, match="too old to register"): + compiler.load(str(program), format="executorch") + + +@pytest.mark.unit +def test_the_loader_calls_activate_on_a_companion_that_has_no_register( + compiler, monkeypatch, tmp_path +) -> None: + """Upgrading the main wheel alone leaves an older companion installed, and that one exposes + activate() rather than register(). Checking the source for the word proves nothing: the + fallback can be commented out and the word stays. So drive it and see which one is called. + """ + called = [] + module = types.ModuleType("torch_tensorrt_executorch_runtime") + module.activate = lambda: called.append("activate") + monkeypatch.setitem(sys.modules, "torch_tensorrt_executorch_runtime", module) + program = tmp_path / "m.pte" + program.write_bytes(b"unused") + # The load itself cannot finish without a real runtime; reaching it is the point. + with pytest.raises(Exception): + compiler.load(str(program), format="executorch") + assert called == ["activate"], called + + +@pytest.mark.unit +def test_the_forwarder_returns_the_shape_the_published_api_returned() -> None: + """Restoring the file was not enough; it has to return what callers already use. + + The API this replaces returned an object carrying run() and forward(), and raised + FileNotFoundError for a missing path. Forwarding to ExecuTorch's own loader returns neither, so a + caller of the published API would fail on the return value rather than on the import, which is + the same breakage one step later. + """ + source = ( + ROOT + / "py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py" + ).read_text(encoding="utf-8") + tree = ast.parse(source) + load = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "load" + ) + returns = [ + ast.unparse(node.value) + for node in ast.walk(load) + if isinstance(node, ast.Return) and node.value is not None + ] + assert returns, "the forwarder returns nothing" + assert any( + "_load" in returned for returned in returns + ), f"the forwarder does not delegate to the compatibility loader: {returns}" + assert ( + "_load_for_executorch" not in source + ), "forwarding to ExecuTorch's loader returns the wrong object" + # The loader it forwards to is the one carrying the original interface. + compat = (ROOT / "py/torch_tensorrt/_executorch_compat.py").read_text( + encoding="utf-8" + ) + for member in ("def run(", "def forward(", "FileNotFoundError"): + assert member in compat, f"the compatibility loader lost {member}" + + +@pytest.mark.parametrize( + "search_path,expected", + [ + (None, "loader could not find it, not that it is incompatible"), + ("/opt/cuda/lib64:", "empty entry"), + (":", "empty entry"), + ("/opt/cuda/lib64", "loader could not find it, not that it is incompatible"), + ], +) +@pytest.mark.unit +def test_a_library_the_loader_cannot_find_is_not_reported_as_an_abi_mismatch( + search_path, expected +) -> None: + """A library the loader could not find is a different problem from one it could not use. + + Only the second is an ABI mismatch. A correct installation failed to import from some working + directories and not others, because an empty entry in the search path is read as the working + directory and stops this package's own origin-relative entries resolving. Blaming the ABI sent + the reader to rebuild a stack that already matched. + """ + source = ( + ROOT + / "py/torch-tensorrt-executorch-runtime" + / "torch_tensorrt_executorch_runtime/__init__.py" + ).read_text(encoding="utf-8") + assert "cannot open shared object file" in source, source[:200] + assert "LD_LIBRARY_PATH" in source, "the empty entry trap is not mentioned" + # The classification the source performs, applied to the text the loader really produces. + text = ( + "libexecutorch_extension_cuda.so: cannot open shared object file: No such file" + ) + missing = "cannot open shared object file" in text + empty = search_path is not None and "" in search_path.split(os.pathsep) + assert missing, "this case is meant to be a not-found error" + assert empty == (expected == "empty entry"), (search_path, empty) + + +@pytest.mark.unit +def test_the_forwarder_says_which_side_is_too_old() -> None: + """The loader this forwards to belongs to the main wheel and is new in this change. + + A main wheel old enough to import this submodule by name does not carry it, so the forward would + have raised a bare missing-module error naming something the reader never asked for. That pairing + should not arise, because this package requires the main wheel of its own build exactly, but an + install that skipped dependency resolution can produce it. + """ + source = ( + ROOT + / "py/torch-tensorrt-executorch-runtime" + / "torch_tensorrt_executorch_runtime/runtime.py" + ).read_text(encoding="utf-8") + tree = ast.parse(source) + load = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "load" + ) + guarded = [ + node + for node in ast.walk(load) + if isinstance(node, ast.Try) + and any( + isinstance(h.type, ast.Name) and h.type.id == "ImportError" + for h in node.handlers + ) + ] + assert guarded, "the forward into the main wheel is not guarded" + assert "older than the one this package was built against" in source, source[-600:] + + +@pytest.mark.parametrize( + "api,expect_rewrite", [("activate", True), ("register", False)] +) +@pytest.mark.unit +def test_the_age_rewrite_follows_the_api_not_the_error_name( + compiler, boundary, monkeypatch, api, expect_rewrite +): + """Both companions define a class of the same name, so the name cannot tell them apart. + + The rewrite exists for a companion published before registration became a single call. Deciding + that from the raised error's class name matched the current companion too, so the rewrite never + fired for the one it was written for. Which registration function the companion exposes is the + thing that actually differs. + """ + + class DelegateCompatibilityError(ImportError): + pass + + def refuse(): + raise DelegateCompatibilityError("already loaded, import this package earlier") + + module = types.ModuleType("torch_tensorrt_executorch_runtime") + setattr(module, api, refuse) + monkeypatch.setitem(sys.modules, "torch_tensorrt_executorch_runtime", module) + with pytest.raises(ImportError) as raised: + load_legacy(compiler, boundary.path) + rewritten = "too old to register" in str(raised.value) + assert rewritten is expect_rewrite, str(raised.value) + + +@pytest.mark.unit +def test_the_forwarder_actually_forwards_and_warns(monkeypatch) -> None: + """Reading the file cannot see a forwarder that forwards nowhere. + + Four separate ways of breaking it left the suite green: dropping the import, dropping the + deprecation warning, naming the loader without calling it, and returning nothing at all. So it is + called here, against a stub standing in for the main wheel's loader. + """ + sentinel = object() + calls: list[str] = [] + compat = types.ModuleType("torch_tensorrt._executorch_compat") + compat.load = lambda path: calls.append(path) or sentinel + parent = types.ModuleType("torch_tensorrt") + parent._executorch_compat = compat + monkeypatch.setitem(sys.modules, "torch_tensorrt", parent) + monkeypatch.setitem(sys.modules, "torch_tensorrt._executorch_compat", compat) + namespace: dict[str, object] = { + "__name__": "torch_tensorrt_executorch_runtime.runtime" + } + source = ( + ROOT + / "py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py" + ).read_text(encoding="utf-8") + exec(compile(source, "runtime.py", "exec"), namespace) + with pytest.warns(DeprecationWarning): + returned = namespace["load"]("some/model.pte") + assert returned is sentinel, returned + assert calls == ["some/model.pte"], calls + + +@pytest.mark.unit +def test_a_missing_file_is_reported_before_a_missing_delegate( + compiler, monkeypatch, tmp_path +): + """Both wrong at once used to report the install, which is true but not what the caller got wrong. + + Someone who mistyped a file name and happens not to have the delegate installed should hear about + the file name. + """ + monkeypatch.delitem(sys.modules, "torch_tensorrt_executorch_runtime", raising=False) + + class _Blocker: + def find_spec(self, name, path=None, target=None): + if name == "torch_tensorrt_executorch_runtime": + raise ModuleNotFoundError( + "No module named 'torch_tensorrt_executorch_runtime'", + name="torch_tensorrt_executorch_runtime", + ) + return None + + monkeypatch.setattr(sys, "meta_path", [_Blocker(), *sys.meta_path]) + with pytest.raises(FileNotFoundError, match="not found"): + load_legacy(compiler, tmp_path / "typo.pte") diff --git a/tests/py/dynamo/executorch/test_packaging.py b/tests/py/dynamo/executorch/test_packaging.py new file mode 100644 index 0000000000..54239c9a83 --- /dev/null +++ b/tests/py/dynamo/executorch/test_packaging.py @@ -0,0 +1,670 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +"""Exercise companion setup with controlled dependencies and native outputs.""" + +import ast +import importlib.metadata +import json +import os +import runpy +import shlex +import shutil +import subprocess +import sys +import tempfile +import types +import zipfile +from pathlib import Path + +import pytest +import setuptools +import yaml +from setuptools import build_meta + +pytestmark = pytest.mark.unit + +REPO_ROOT = Path(__file__).parents[4] +COMPANION = REPO_ROOT / "py/torch-tensorrt-executorch-runtime" +PACKAGE = "torch_tensorrt_executorch_runtime" +LIBRARY = "libexecutorch_backend_tensorrt.so" +FLAG_VALUES = [ + (None, False), + ("", False), + ("0", False), + ("false", False), + ("FALSE", False), + ("off", False), + ("other", False), + (" true ", False), + ("1", True), + ("true", True), + ("TrUe", True), + ("yes", True), + ("YES", True), + ("on", True), + ("ON", True), +] + + +@pytest.fixture +def packaging_build(tmp_path, monkeypatch): + project = tmp_path / "checkout/py/torch-tensorrt-executorch-runtime" + shutil.copytree( + COMPANION, + project, + ignore=shutil.ignore_patterns( + "build", "dist", "*.egg-info", "__pycache__", LIBRARY + ), + ) + shutil.copyfile( + REPO_ROOT / "dev_dep_versions.yml", project.parents[1] / "dev_dep_versions.yml" + ) + pinned = yaml.safe_load((project.parents[1] / "dev_dep_versions.yml").read_text())[ + "__executorch_version__" + ] + versions = { + # With the label a real CUDA wheel carries. A bare version here is what a processor-only + # build looks like, and the build refuses that, correctly. + "executorch": f"{pinned}+cu130", + "torch-tensorrt": "2.15.0.dev20200103+cu130", + "tensorrt-cu13": "11.2.1", + "nvidia-cuda-runtime": "13.0.0", + } + dependency_root = tmp_path / "dependencies" + cmake = dependency_root / "executorch/share/cmake" + cmake.mkdir(parents=True) + (cmake / "executorch-config.cmake").touch() + torch = types.ModuleType("torch") + torch.__file__ = str(dependency_root / "torch/__init__.py") + torch.__version__ = "2.15.0.dev20200103+cu130" + torch.version = types.SimpleNamespace(cuda="13.0") + monkeypatch.setitem(sys.modules, "torch", torch) + original_version = importlib.metadata.version + original_distribution = importlib.metadata.distribution + monkeypatch.setattr( + importlib.metadata, + "version", + lambda name: versions[name] if name in versions else original_version(name), + ) + monkeypatch.setattr( + importlib.metadata, + "distribution", + lambda name: ( + types.SimpleNamespace( + version=versions[name], locate_file=lambda path: dependency_root / path + ) + if name == "executorch" + else original_distribution(name) + ), + ) + for name in ( + "TORCH_TENSORRT_ALLOW_UNPINNED_EXECUTORCH", + "TORCH_TENSORRT_EXECUTORCH_DEBUG", + "BAZEL_ARGS", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv( + "TORCH_TENSORRT_EXECUTORCH_RUNTIME_VERSION", "0.2.0.dev20200103+cu130" + ) + monkeypatch.chdir(project) + monkeypatch.setattr(sys, "argv", [str(project / "setup.py"), "build_py"]) + state = types.SimpleNamespace( + project=project, + versions=versions, + calls=[], + builds=[], + requires=[], + payload=b"controlled native output", + ) + bazel_bin = tmp_path / "bazel-bin" + original_which = shutil.which + monkeypatch.setattr( + shutil, + "which", + lambda name: ( + "/controlled/bazel" + if name in {"bazel", "bazelisk"} + else original_which(name) + ), + ) + original_run = subprocess.run + original_output = subprocess.check_output + + def produce(command, **kwargs): + if command[0] != "/controlled/bazel": + return original_run(command, **kwargs) + state.calls.append(command) + output = ( + bazel_bin + / "py/torch-tensorrt-executorch-runtime/native/delegate_native/lib" + / LIBRARY + ) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(state.payload) + return subprocess.CompletedProcess(command, 0) + + def bazel_info(command, **kwargs): + if command[0] != "/controlled/bazel": + return original_output(command, **kwargs) + state.calls.append(command) + return str(bazel_bin) + + monkeypatch.setattr(subprocess, "run", produce) + monkeypatch.setattr(subprocess, "check_output", bazel_info) + original_setup = setuptools.setup + + def setup(**kwargs): + # Exercise Linux-only command behavior without changing setuptools' host wheel tags. + kwargs["cmdclass"]["build_py"].run.__globals__["sys"] = types.SimpleNamespace( + platform="linux", executable=sys.executable + ) + # Keep what the file actually declares. Reading the source for the words cannot see a value + # rewritten between the read and the call, which is how the pin lost its build label + # unnoticed. + state.requires = list(kwargs.get("install_requires") or []) + distribution = original_setup(**kwargs) + state.builds.append(distribution.get_command_obj("build_py")) + return distribution + + monkeypatch.setattr(setuptools, "setup", setup) + return state + + +@pytest.mark.parametrize("value,enabled", FLAG_VALUES) +@pytest.mark.parametrize("flag", ["unpinned", "debug"]) +def test_setup_boolean_flags(packaging_build, monkeypatch, flag, value, enabled): + state = packaging_build + name = ( + "TORCH_TENSORRT_ALLOW_UNPINNED_EXECUTORCH" + if flag == "unpinned" + else "TORCH_TENSORRT_EXECUTORCH_DEBUG" + ) + if value is not None: + monkeypatch.setenv(name, value) + if flag == "unpinned": + state.versions["executorch"] = "1.4.0" + if flag == "unpinned" and not enabled: + # SystemExit, not RuntimeError: the build converts every failure to the one class + # setuptools does not swallow during an editable install, so this refusal reaches the user + # instead of becoming a warning pip hides. + with pytest.raises(SystemExit, match="pins"): + runpy.run_path(str(state.project / "setup.py"), run_name="__main__") + assert state.calls == [] + else: + runpy.run_path(str(state.project / "setup.py"), run_name="__main__") + mode = "dbg" if flag == "debug" and enabled else "opt" + assert len(state.calls) == 2 + assert all(f"--compilation_mode={mode}" in call for call in state.calls) + build = state.builds[-1] + output = Path(build.build_lib) / PACKAGE / "lib" / LIBRARY + assert output.read_bytes() == state.payload + assert ( + output.parent + / "cmake/torchtrt_executorch/torchtrt_executorch-config-version.cmake" + ).is_file() + + +@pytest.mark.parametrize("flag", ["unpinned", "debug"]) +def test_setup_flags_reject_string_presence_control(packaging_build, monkeypatch, flag): + path = packaging_build.project / "setup.py" + tree = ast.parse(path.read_text()) + name = ( + "TORCH_TENSORRT_ALLOW_UNPINNED_EXECUTORCH" + if flag == "unpinned" + else "TORCH_TENSORRT_EXECUTORCH_DEBUG" + ) + checks = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Compare) + and isinstance(node.left, ast.Call) + and any( + isinstance(child, ast.Constant) and child.value == name + for child in ast.walk(node.left) + ) + ] + assert len(checks) == 1 + check = checks[0] + check.left = ast.parse(f"os.getenv({name!r})", mode="eval").body + check.ops = [ast.Is() if flag == "unpinned" else ast.IsNot()] + check.comparators = [ast.Constant(value=None)] + path.write_text(ast.unparse(ast.fix_missing_locations(tree))) + failure = pytest.fail.Exception if flag == "unpinned" else AssertionError + with pytest.raises(failure): + test_setup_boolean_flags(packaging_build, monkeypatch, flag, "false", False) + + +def _run(command, **kwargs): + result = subprocess.run(command, capture_output=True, text=True, **kwargs) + print(shlex.join(map(str, command))) + print(result.stdout + result.stderr) + assert result.returncode == 0, result.stdout + result.stderr + return result.stdout + + +@pytest.fixture +def native_payload(packaging_build, tmp_path): + compiler = shutil.which("cc") + if compiler is None: + pytest.skip("a C compiler is required for the packaging load probe") + source = tmp_path / "probe.c" + source.write_text("int packaging_probe(void) { return 42; }\n") + library = tmp_path / LIBRARY + _run([compiler, "-shared", "-fPIC", str(source), "-o", str(library)]) + packaging_build.payload = library.read_bytes() + + +def _install_and_probe(state, wheel, environment, tmp_path, mode): + _run([sys.executable, "-m", "venv", "--without-pip", str(environment)]) + python = environment / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + _run( + [ + sys.executable, + "-m", + "pip", + "--python", + str(python), + "install", + "--no-deps", + "--no-index", + "--disable-pip-version-check", + str(wheel), + ] + ) + probe = """ +import ctypes +import hashlib +import json +from pathlib import Path +import torch_tensorrt_executorch_runtime as runtime +root = Path(runtime.__file__).absolute().parent +library = Path(runtime._delegate_path()) +handle = ctypes.CDLL(str(library)) +assert handle.packaging_probe() == 42 +cmake = root / "lib/cmake/torchtrt_executorch" +config = cmake / "torchtrt_executorch-config.cmake" +version = cmake / "torchtrt_executorch-config-version.cmake" +assert config.is_file() +assert 'set(PACKAGE_VERSION "0.2.0")' in version.read_text() +assert 'set(TORCHTRT_EXECUTORCH_FULL_VERSION "0.2.0.dev20200103+cu130")' in version.read_text() +print(json.dumps({"module": str(root), "library": str(library), + "sha256": hashlib.sha256(library.read_bytes()).hexdigest(), + "config": str(config), "version": str(version)})) +""" + output = _run( + [str(python), "-I", "-B", "-c", probe], + cwd=tmp_path, + env={**os.environ, "TORCH_TENSORRT_SKIP_DELEGATE_REGISTRATION": "1"}, + ) + data = json.loads(output) + root = Path(data["module"]) + if mode == "default": + assert root == state.project / PACKAGE + elif mode == "strict": + assert root.is_relative_to(state.project / "build") + else: + assert root.is_relative_to(environment) + assert Path(data["library"]).read_bytes() == state.payload + + +@pytest.mark.parametrize("mode", ["default", "strict"]) +def test_editable_outputs_survive_backend_cleanup( + packaging_build, native_payload, monkeypatch, tmp_path, mode +): + state = packaging_build + temporary = tmp_path / "backend-temporary" + temporary.mkdir() + monkeypatch.setattr(tempfile, "tempdir", str(temporary)) + wheels = tmp_path / "wheels" + wheels.mkdir() + settings = {"editable_mode": "strict"} if mode == "strict" else None + wheel = wheels / build_meta.build_editable(str(wheels), settings) + build = state.builds[-1] + assert build.editable_mode + assert len(state.calls) == 2 + assert not Path(build.build_lib).exists() + assert list(temporary.iterdir()) == [] + print(f"Removed temporary build_lib: {build.build_lib}") + _install_and_probe(state, wheel, tmp_path / "installed", tmp_path, mode) + generated = { + f"lib/{LIBRARY}", + "lib/cmake/torchtrt_executorch/torchtrt_executorch-config.cmake", + "lib/cmake/torchtrt_executorch/torchtrt_executorch-config-version.cmake", + } + mapping = build.get_output_mapping() + outputs = build.get_outputs() + for filename in generated: + destination = str(Path(build.build_lib) / PACKAGE / filename) + assert destination in outputs + assert ( + Path(mapping[destination]).resolve() == state.project / PACKAGE / filename + ) + assert Path(mapping[destination]).is_file() + + +def _remove_editable_fix(path): + tree = ast.parse(path.read_text()) + build = next( + node + for node in tree.body + if isinstance(node, ast.ClassDef) and node.name == "BazelBuild" + ) + removed = {"_generated_output_mapping", "get_outputs", "get_output_mapping"} + assert removed.issubset({getattr(node, "name", None) for node in build.body}) + build.body = [ + node for node in build.body if getattr(node, "name", None) not in removed + ] + branches = [ + node + for node in ast.walk(build) + if isinstance(node, ast.IfExp) + and isinstance(node.test, ast.Attribute) + and node.test.attr == "editable_mode" + ] + assert len(branches) == 1 + branches[0].test = ast.Constant(value=False) + path.write_text(ast.unparse(ast.fix_missing_locations(tree))) + + +@pytest.mark.parametrize("mode", ["default", "strict"]) +def test_editable_install_rejects_removed_fix( + packaging_build, native_payload, monkeypatch, tmp_path, mode +): + _remove_editable_fix(packaging_build.project / "setup.py") + with pytest.raises(AssertionError, match="delegate library is missing"): + test_editable_outputs_survive_backend_cleanup( + packaging_build, native_payload, monkeypatch, tmp_path, mode + ) + + +@pytest.mark.parametrize("editable", [False, True]) +def test_generated_outputs_are_reported_before_build( + packaging_build, monkeypatch, editable +): + state = packaging_build + monkeypatch.setattr(sys, "argv", [str(state.project / "setup.py"), "--name"]) + runpy.run_path(str(state.project / "setup.py"), run_name="__main__") + command = state.builds[-1] + command.ensure_finalized() + command.editable_mode = editable + expected = { + str(Path(command.build_lib) / PACKAGE / filename) + for filename in ( + f"lib/{LIBRARY}", + "lib/cmake/torchtrt_executorch/torchtrt_executorch-config.cmake", + "lib/cmake/torchtrt_executorch/torchtrt_executorch-config-version.cmake", + ) + } + outputs = command.get_outputs(include_bytecode=False) + assert expected.issubset(outputs) + mapping = command.get_output_mapping() + assert expected.issubset(mapping) is editable + command.run() + for filename in expected: + output = Path(mapping[filename]) if editable else Path(filename) + assert output.is_file() + + +@pytest.mark.parametrize("editable", [False, True]) +def test_generated_cleanup_stays_in_the_output_location( + packaging_build, monkeypatch, editable +): + state = packaging_build + monkeypatch.setattr(sys, "argv", [str(state.project / "setup.py"), "--name"]) + runpy.run_path(str(state.project / "setup.py"), run_name="__main__") + command = state.builds[-1] + command.ensure_finalized() + command.editable_mode = editable + root = state.project / PACKAGE if editable else Path(command.build_lib) / PACKAGE + (root / "lib").mkdir(parents=True, exist_ok=True) + stale = root / "lib/libexecutorch.so" + stale.write_bytes(b"obsolete private runtime") + # A stale shared object from the old layout, which must go, and the forwarder the released main + # wheel imports by name, which must not. They used to be swept together, so the published wheel + # shipped without the forwarder and that import failed. + legacy = [root / "old_delegate.so"] + shipped = root / "runtime.py" + for path in [*legacy, shipped]: + path.write_bytes(b"root output") + bystanders = [ + state.project / "bystander.so", + root / "lib/notes.txt", + root.parent / "other/libexecutorch.so", + ] + for path in bystanders: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"keep") + source = state.project / PACKAGE / "__init__.py" + original = source.read_bytes() + command.run() + assert not stale.exists() + assert (root / "lib" / LIBRARY).read_bytes() == state.payload + assert source.read_bytes() == original + assert all(path.read_bytes() == b"keep" for path in bystanders) + assert all(path.exists() is editable for path in legacy) + assert ( + shipped.exists() + ), "the forwarder the main wheel imports was removed from the wheel" + + +def test_editable_cleanup_rejects_source_deletion_control(packaging_build, monkeypatch): + path = packaging_build.project / "setup.py" + tree = ast.parse(path.read_text()) + guards = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.If) + and isinstance(node.test, ast.UnaryOp) + and isinstance(node.test.op, ast.Not) + and isinstance(node.test.operand, ast.Attribute) + and node.test.operand.attr == "editable_mode" + ] + assert len(guards) == 1 + guards[0].test = ast.Constant(value=True) + path.write_text(ast.unparse(tree)) + with pytest.raises(AssertionError): + test_generated_cleanup_stays_in_the_output_location( + packaging_build, monkeypatch, True + ) + + +def test_ordinary_wheel_payload_is_unchanged( + packaging_build, native_payload, monkeypatch, tmp_path +): + state = packaging_build + wheels = tmp_path / "wheels" + wheels.mkdir() + built = wheels / build_meta.build_wheel(str(wheels)) + with zipfile.ZipFile(built) as archive: + expected = {name: archive.read(name) for name in archive.namelist()} + files = {name for name in expected if name.startswith(f"{PACKAGE}/")} + assert files == { + f"{PACKAGE}/__init__.py", + # The forwarder the released main wheel imports by name. This set omitting it is what let a + # published wheel ship without it, since this is the only check that reads the built archive. + f"{PACKAGE}/runtime.py", + f"{PACKAGE}/lib/{LIBRARY}", + f"{PACKAGE}/lib/cmake/torchtrt_executorch/torchtrt_executorch-config.cmake", + f"{PACKAGE}/lib/cmake/torchtrt_executorch/torchtrt_executorch-config-version.cmake", + } + assert expected[f"{PACKAGE}/lib/{LIBRARY}"] == state.payload + _install_and_probe(state, built, tmp_path / "installed", tmp_path, "wheel") + # Use a separate source/build tree so no output can be reused in the comparison. + control_project = tmp_path / "control-checkout/py/torch-tensorrt-executorch-runtime" + shutil.copytree( + state.project, + control_project, + ignore=shutil.ignore_patterns("build", "*.egg-info"), + ) + shutil.copyfile( + state.project.parents[1] / "dev_dep_versions.yml", + control_project.parents[1] / "dev_dep_versions.yml", + ) + _remove_editable_fix(control_project / "setup.py") + monkeypatch.chdir(control_project) + control = tmp_path / "control-wheels" + control.mkdir() + old = control / build_meta.build_wheel(str(control)) + with zipfile.ZipFile(old) as archive: + actual = {name: archive.read(name) for name in archive.namelist()} + assert actual == expected + print( + f"Ordinary wheel has identical bytes for {len(expected)} members with editable fix removed" + ) + + +@pytest.mark.unit +def test_the_executorch_requirement_keeps_its_cuda_label() -> None: + """A requirement without the local label is satisfied by a build the delegate cannot use. + + The delegate links one specific ExecuTorch build, so pinning the version while dropping the + part that names the CUDA variant leaves a pin that a CPU build, or another CUDA build of the + same date, resolves against happily. + """ + source = (COMPANION / "setup.py").read_text(encoding="utf-8") + assert 'f"executorch=={executorch_version}"' in source, source[-400:] + assert ( + 'f"executorch=={public_version(executorch_version)}"' not in source + ), "the local label is being stripped again" + + +@pytest.mark.unit +def test_a_missing_pin_file_fails_rather_than_disabling_the_check() -> None: + """Returning an empty version switched the pin check off instead of failing it.""" + source = (COMPANION / "setup.py").read_text(encoding="utf-8") + reader = source.split("def pinned_executorch_version")[1].split("\ndef ")[0] + assert "raise RuntimeError" in reader, reader + assert 'return ""' not in reader, reader + + +@pytest.mark.parametrize( + "installed,accepted", + [("+cu130", True), ("+cu134", True), ("+cpu", False), ("", False)], +) +def test_the_build_refuses_an_executorch_that_is_not_a_cuda_build( + packaging_build, monkeypatch, installed, accepted +): + """Matching the version is not matching the build. + + The delegate links the CUDA runtime out of the installed wheel, so a processor-only build of the + pinned date cannot supply it. Comparing only the public parts of the two versions accepted that, + and the wheel the build then published required a label the build had never checked. + """ + state = packaging_build + pinned = updater_pin = yaml.safe_load( + (state.project.parents[1] / "dev_dep_versions.yml").read_text() + )["__executorch_version__"] + state.versions["executorch"] = f"{pinned}{installed}" + if accepted: + runpy.run_path(str(state.project / "setup.py"), run_name="__main__") + else: + with pytest.raises(SystemExit, match="CUDA build"): + runpy.run_path(str(state.project / "setup.py"), run_name="__main__") + + +@pytest.mark.unit +def test_the_wheel_is_tagged_for_any_python_and_one_platform() -> None: + """Neither of these two overrides had a test, so dropping either changed the built wheel. + + The payload is one shared library loaded through ctypes, with no Python ABI, so it is identical + across CPython versions and only the platform matters. Losing the tag override would build one + identical copy per interpreter. Losing the platform marking would tag a compiled object as pure + Python and let it install on the wrong architecture. + """ + source = (COMPANION / "setup.py").read_text(encoding="utf-8") + namespace: dict[str, object] = {} + tree = ast.parse(source) + wanted = [ + node + for node in tree.body + if isinstance(node, ast.ClassDef) + and node.name in ("WheelTag", "PlatformDistribution") + ] + assert len(wanted) == 2, [n.name for n in wanted] + # Run the two class bodies against stub bases, so the behaviour is exercised rather than read. + namespace["bdist_wheel"] = type( + "StubBdist", + (), + {"get_tag": lambda self: ("cp312", "cp312", "manylinux_2_28_x86_64")}, + ) + namespace["Distribution"] = type( + "StubDistribution", (), {"has_ext_modules": lambda self: False} + ) + exec( + compile(ast.Module(body=wanted, type_ignores=[]), "", "exec"), namespace + ) + assert namespace["WheelTag"]().get_tag() == ("py3", "none", "manylinux_2_28_x86_64") + assert namespace["PlatformDistribution"]().has_ext_modules() is True + + +@pytest.mark.unit +def test_the_three_linked_runtimes_are_pinned_with_their_build_labels() -> None: + """A version without its label admits a processor build and any other build of the same date. + + The delegate is compiled against one specific build of each of these three, so a requirement that + a different build satisfies is not a pin at all. The label is the part that names the build. + """ + source = (COMPANION / "setup.py").read_text(encoding="utf-8") + requires = source.split("install_requires=[", 1)[1].split("]", 1)[0] + for name, expression in ( + ("torch", 'f"torch=={torch.__version__}"'), + ("executorch", 'f"executorch=={executorch_version}"'), + ( + "torch-tensorrt", + "f\"torch-tensorrt=={installed_version('torch-tensorrt')}\"", + ), + ): + assert ( + expression in requires + ), f"{name} is not pinned with its label: {requires}" + # And the two that legitimately have no label keep the public form. + assert "public_version(tensorrt_version)" in requires, requires + assert "public_version(cuda_runtime_version)" in requires, requires + + +@pytest.mark.unit +def test_the_declared_pin_keeps_its_build_label(packaging_build, monkeypatch): + """Reading the file for the words cannot see a value rewritten before it is used. + + Dropping the label from the ExecuTorch pin, while leaving every word the old checks looked for, + changed the requirement from one build to any build of that date and went unnoticed. So the + requirement the file actually declares is read back here. + """ + state = packaging_build + monkeypatch.setattr(sys, "argv", [str(state.project / "setup.py"), "--name"]) + runpy.run_path(str(state.project / "setup.py"), run_name="__main__") + executorch = [r for r in state.requires if r.startswith("executorch==")] + assert executorch, state.requires + assert "+cu" in executorch[0], executorch[0] + + +@pytest.mark.unit +def test_a_missing_pin_file_stops_the_build(tmp_path): + """The refusal was checked by looking for words, and its branch never ran. + + Rewriting it to fall back to the installed version, and replacing its body with something that + would be obvious, both left the suite green. So the branch is executed here, with no pin file + present. + """ + source = (COMPANION / "setup.py").read_text(encoding="utf-8") + tree = ast.parse(source) + reader = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) + and node.name == "pinned_executorch_version" + ) + namespace: dict[str, object] = {"REPO_ROOT": tmp_path, "yaml": yaml} + exec( + compile(ast.Module(body=[reader], type_ignores=[]), "", "exec"), + namespace, + ) + # No pin file in tmp_path, which is the case the refusal exists for. + with pytest.raises(RuntimeError, match="is missing"): + namespace["pinned_executorch_version"]() + # And with one present it returns the pinned version rather than guessing. + (tmp_path / "dev_dep_versions.yml").write_text( + '__executorch_version__: "1.6.0.dev20260915+cu134"\n', encoding="utf-8" + ) + assert namespace["pinned_executorch_version"]() == "1.6.0.dev20260915+cu134" diff --git a/tests/py/dynamo/executorch/test_python_runtime.py b/tests/py/dynamo/executorch/test_python_runtime.py index 1d627bb3e9..074a2c28d9 100644 --- a/tests/py/dynamo/executorch/test_python_runtime.py +++ b/tests/py/dynamo/executorch/test_python_runtime.py @@ -1,265 +1,799 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: BSD-3-Clause +import ast +import ctypes import importlib.util +import os import sys import types from pathlib import Path import pytest -RUNTIME_PATH = ( - Path(__file__).parents[4] - / "py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py" -) +pytestmark = pytest.mark.unit + DELEGATE_PATH = ( Path(__file__).parents[4] / "py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/__init__.py" ) +SETUP_PATH = Path(__file__).parents[4] / "py/torch-tensorrt-executorch-runtime/setup.py" +SKIP_ENV = "TORCH_TENSORRT_SKIP_DELEGATE_REGISTRATION" -def load_runtime_module(): - spec = importlib.util.spec_from_file_location( - "torchtrt_et_runtime_test", RUNTIME_PATH +@pytest.mark.parametrize("device_resident", [False, True]) +@pytest.mark.parametrize("has_forward", [False, True]) +def test_examples_use_the_module_loader( + monkeypatch, tmp_path, device_resident, has_forward +): + """Execute the examples without CUDA and verify the Module API receives the original inputs.""" + import runpy + + calls = [] + expected = object() + + class Tensor: + is_cuda = device_resident + device = "cuda:0" if device_resident else "cpu" + shape = (64, 64) if device_resident else (2, 3, 4, 4) + + def cpu(self): + return self + + def __add__(self, value): + assert value == 1 + return expected + + tensor = Tensor() + torch = types.ModuleType("torch") + torch.float32 = object() + torch.cuda = types.SimpleNamespace(is_available=lambda: True) + + def ones(shape, dtype, device="cpu"): + assert tuple(shape) == tensor.shape + assert (device == "cuda") is device_resident + return tensor + + torch.ones = ones + torch.tanh = torch.erfinv = lambda value: value + torch.cos = lambda value: expected + torch.testing = types.SimpleNamespace( + assert_close=lambda actual, wanted: calls.append((actual, wanted)) + ) + monkeypatch.setitem(sys.modules, "torch", torch) + _fake_executorch(monkeypatch, set()) + portable = sys.modules["executorch.extension.pybindings.portable_lib"] + model = tmp_path / "model.pte" + methods = ["forward"] if has_forward else [] + + def run_method(name, inputs): + assert name == "forward" + assert inputs == (tensor,) + calls.append("run") + return [tensor] + + def load(path): + assert path == str(model) + calls.append("load") + return types.SimpleNamespace( + method_names=lambda: methods, run_method=run_method + ) + + portable._load_for_executorch = load + # Record the import rather than pre-inserting a module. A module already in sys.modules makes + # "import x" a no-op with nothing to observe, so deleting that import from the example left every + # case green even though the delegate would never register. + imported: list[str] = [] + + class _Loader: + def create_module(self, spec): + return types.ModuleType(spec.name) + + def exec_module(self, module): + return None + + class _Recorder: + def find_spec(self, name, path=None, target=None): + if name == "torch_tensorrt_executorch_runtime": + imported.append(name) + return importlib.util.spec_from_loader(name, loader=_Loader()) + return None + + monkeypatch.delitem(sys.modules, "torch_tensorrt_executorch_runtime", raising=False) + monkeypatch.setattr(sys, "meta_path", [_Recorder(), *sys.meta_path]) + filename = "load_model_device_resident.py" if device_resident else "load_model.py" + source = ( + Path(__file__).parents[4] / "examples/executorch_reference_runner" / filename + ) + monkeypatch.setattr( + sys, "argv", [str(source), "--model_path", str(model), "--num_runs", "2"] + ) + if not has_forward: + with pytest.raises(RuntimeError, match="has no 'forward' method"): + runpy.run_path(str(source), run_name="__main__") + assert calls == ["load"] + else: + runpy.run_path(str(source), run_name="__main__") + assert calls == ["load", "run", "run", (tensor, expected)] + # The example has to import the delegate package, because that import is what registers the + # backend. Nothing here can run without it in a real process. + assert imported == ["torch_tensorrt_executorch_runtime"], imported + + +def load_delegate_module(*, register_on_import: bool = False): + """Import the delegate module from source, side effect suppressed by default. + + Importing the real package registers the backend, which is the whole contract. Every test below + that drives a failure branch has to install its fakes BEFORE anything loads, so it needs the + module without that side effect; the opt-out the package documents is exactly the hook for it. + ``register_on_import=True`` is for the two tests that assert the side effect itself. + """ + previous = os.environ.get(SKIP_ENV) + if register_on_import: + os.environ.pop(SKIP_ENV, None) + else: + os.environ[SKIP_ENV] = "1" + try: + spec = importlib.util.spec_from_file_location( + "torchtrt_et_delegate_test", DELEGATE_PATH + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + finally: + if previous is None: + os.environ.pop(SKIP_ENV, None) + else: + os.environ[SKIP_ENV] = previous + + +def _fake_executorch(monkeypatch, registered): + """Stand in for the installed ExecuTorch, whose registry the delegate registers into. + + ``registered`` is the live set the fake ``CDLL`` mutates, which is how these tests model + the one thing that actually matters: the backend appears only as a side effect of loading + the library. + """ + portable_lib = types.ModuleType("executorch.extension.pybindings.portable_lib") + portable_lib._get_registered_backend_names = lambda: sorted(registered) + pybindings = types.ModuleType("executorch.extension.pybindings") + pybindings.portable_lib = portable_lib + extension = types.ModuleType("executorch.extension") + extension.pybindings = pybindings + executorch = types.ModuleType("executorch") + executorch.extension = extension + for name, module in { + "executorch": executorch, + "executorch.extension": extension, + "executorch.extension.pybindings": pybindings, + "executorch.extension.pybindings.portable_lib": portable_lib, + }.items(): + monkeypatch.setitem(sys.modules, name, module) + + +def _delegate_handle(owns_registration=True): + return types.SimpleNamespace( + torch_tensorrt_owns_executorch_registration=lambda: owns_registration ) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module -def load_delegate_module(): - spec = importlib.util.spec_from_file_location( - "torchtrt_et_delegate_test", DELEGATE_PATH - ) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module +@pytest.mark.unit +@pytest.mark.parametrize("preloaded", [False, True]) +@pytest.mark.parametrize("owns_registration", [False, True]) +def test_register_checks_the_loaded_handles_ownership( + monkeypatch, preloaded, owns_registration +): + delegate = load_delegate_module() + registered = {delegate.BACKEND_NAME} if preloaded else set() + _fake_executorch(monkeypatch, registered) + handle = _delegate_handle(owns_registration) + loads = [] + + def load(path, mode): + loads.append(mode) + registered.add(delegate.BACKEND_NAME) + return handle + + monkeypatch.setattr(delegate, "_delegate_path", lambda: "/fake/delegate.so") + monkeypatch.setattr(delegate.ctypes, "CDLL", load) + if owns_registration: + delegate.register() + delegate.register() + assert delegate._delegate is handle + assert len(loads) == 1 + query = handle.torch_tensorrt_owns_executorch_registration + assert query.argtypes == [] + assert query.restype is ctypes.c_bool + else: + for _ in range(2): + with pytest.raises( + delegate.DelegateCompatibilityError, match="does not own" + ): + delegate.register() + assert delegate._delegate is None + assert len(loads) == 2 + + +@pytest.mark.unit +@pytest.mark.parametrize("preloaded", [False, True]) +def test_ownership_regression_rejects_presence_only_acceptance( + monkeypatch, tmp_path, preloaded +): + tree = ast.parse(DELEGATE_PATH.read_text()) + checks = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.If) + and isinstance(node.test, ast.UnaryOp) + and isinstance(node.test.op, ast.Not) + and isinstance(node.test.operand, ast.Call) + and isinstance(node.test.operand.func, ast.Name) + and node.test.operand.func.id == "owns_registration" + ] + assert len(checks) == 1 + checks[0].test = ast.parse( + "BACKEND_NAME not in _registered_backend_names()", mode="eval" + ).body + source = tmp_path / "presence_only.py" + source.write_text(ast.unparse(tree)) + monkeypatch.setitem(globals(), "DELEGATE_PATH", source) + with pytest.raises(pytest.fail.Exception, match="DID NOT RAISE"): + test_register_checks_the_loaded_handles_ownership(monkeypatch, preloaded, False) + + +@pytest.mark.unit +@pytest.mark.parametrize("preloaded", [False, True]) +def test_register_rejects_a_library_without_an_ownership_query(monkeypatch, preloaded): + delegate = load_delegate_module() + registered = {delegate.BACKEND_NAME} if preloaded else set() + _fake_executorch(monkeypatch, registered) + def load(path, mode): + registered.add(delegate.BACKEND_NAME) + return types.SimpleNamespace() -class FakeModule: - """Stands in for what the Module API returns. + monkeypatch.setattr(delegate, "_delegate_path", lambda: "/fake/delegate.so") + monkeypatch.setattr(delegate.ctypes, "CDLL", load) + with pytest.raises(delegate.DelegateCompatibilityError, match="ownership query"): + delegate.register() + assert delegate._delegate is None - method_names is a call and running a method is one step, unlike the program loader's - load_method(...).execute(...). The runtime moved to this API because only it backs - device-tagged memory-planned arenas with real device memory. - """ - def __init__(self, data): - self.data = data +@pytest.mark.unit +@pytest.mark.parametrize("failure", ["missing_noload", "unloaded"]) +def test_register_rejects_a_foreign_registration(monkeypatch, failure): + delegate = load_delegate_module() + _fake_executorch(monkeypatch, {delegate.BACKEND_NAME}) + monkeypatch.setattr(delegate, "_delegate_path", lambda: "/fake/delegate.so") + if failure == "missing_noload": + monkeypatch.delattr(delegate.os, "RTLD_NOLOAD", raising=False) - def method_names(self): - return {"forward"} + def load(path, mode): + raise OSError("not loaded") - def run_method(self, name, inputs): - if name != "forward": - raise RuntimeError(f"unknown method {name!r}") - return [inputs[0] + 1] + monkeypatch.setattr(delegate.ctypes, "CDLL", load) + with pytest.raises(delegate.DelegateCompatibilityError, match="already registered"): + delegate.register() + assert delegate._delegate is None -class FakeRuntime: - def __init__(self): - self.data = None +@pytest.mark.unit +def test_register_retries_after_load_failure(monkeypatch): + delegate = load_delegate_module() + registered = set() + _fake_executorch(monkeypatch, registered) + handle = _delegate_handle() + attempts = 0 + + def load(path, mode): + nonlocal attempts + attempts += 1 + if attempts == 1: + raise OSError("dependency temporarily unavailable") + registered.add(delegate.BACKEND_NAME) + return handle + + monkeypatch.setattr(delegate, "_delegate_path", lambda: "/fake/delegate.so") + monkeypatch.setattr(delegate.ctypes, "CDLL", load) + with pytest.raises(delegate.DelegateCompatibilityError): + delegate.register() + assert delegate._delegate is None + delegate.register() + assert delegate._delegate is handle + assert attempts == 2 + - def __call__(self): - return self +@pytest.mark.unit +def test_concurrent_registration_loads_and_checks_once(monkeypatch): + from concurrent.futures import ThreadPoolExecutor + from threading import Barrier + import time + + delegate = load_delegate_module() + registered = set() + _fake_executorch(monkeypatch, registered) + start = Barrier(8) + loads = [] + queries = [] + + def owns(): + queries.append(True) + return True + + handle = types.SimpleNamespace(torch_tensorrt_owns_executorch_registration=owns) + + def load(path, mode): + loads.append(mode) + registered.add(delegate.BACKEND_NAME) + time.sleep(0.02) + return handle + + def register(_): + start.wait(timeout=5) + delegate.register() + + monkeypatch.setattr(delegate, "_delegate_path", lambda: "/fake/delegate.so") + monkeypatch.setattr(delegate.ctypes, "CDLL", load) + with ThreadPoolExecutor(max_workers=8) as pool: + list(pool.map(register, range(8))) + assert len(loads) == len(queries) == 1 + assert delegate._delegate is handle + + +def test_importing_the_package_registers_the_backend(monkeypatch): + """The whole contract of this wheel: the import is the registration. + + ExecuTorch's own delegates register because they are linked into its pybindings extension, so + loading that extension pulls them in. A delegate in a separate wheel cannot join that link, so + this package does the equivalent at import time. Nothing here calls ``register()``: the fakes go + in first, then the module is imported with the side effect ENABLED, and the backend has to appear + purely as a consequence of that import. + """ + registered = set() + _fake_executorch(monkeypatch, registered) + loaded = [] + def fake_cdll(path, mode): + loaded.append(path) + registered.add("TensorRTBackend") + return _delegate_handle() -def _install_fakes(monkeypatch): - """Stub the delegate and the Module loader the runtime imports. + monkeypatch.setattr(ctypes, "CDLL", fake_cdll) + monkeypatch.setattr(os.path, "isfile", lambda path: True) - The runtime calls get_runtime() to check the backend is registered and activate() to get the - native module it installed, then takes the loader off that module. Faking activate() keeps the - canonical portable_lib name out of sys.modules, which is what activate() itself treats as a - sign the stock runtime was imported first. + delegate = load_delegate_module(register_on_import=True) + + assert loaded, "importing the package did not load the delegate library" + assert delegate.BACKEND_NAME in registered + # And the import left it fully done, not half done: a later call is a no-op rather than a + # second load, which is what a defensive caller re-asserting registration would hit. + delegate.register() + assert len(loaded) == 1 + + +def test_the_opt_out_env_var_suppresses_the_import_side_effect(monkeypatch): + """The escape hatch the tests themselves depend on, so it needs its own coverage. + + Every failure-branch test below imports the module with the side effect suppressed in order to + install its fakes first. If the opt-out silently stopped working, those tests would start + exercising a real load against the machine's own ExecuTorch and their results would mean + something else entirely. """ - loaded = {} + loaded = [] + monkeypatch.setattr(ctypes, "CDLL", lambda path, mode: loaded.append(path)) + + delegate = load_delegate_module() - def fake_load_from_buffer(data): - loaded["data"] = data - return FakeModule(data) + assert not loaded, "the delegate was loaded despite the registration opt-out" + assert delegate._delegate is None + + +@pytest.mark.unit +@pytest.mark.parametrize( + "value,skip", + [ + (None, False), + ("", False), + ("0", False), + ("false", False), + ("FALSE", False), + ("off", False), + ("other", False), + (" true ", False), + ("1", True), + ("true", True), + ("TrUe", True), + ("yes", True), + ("YES", True), + ("on", True), + ("ON", True), + ], +) +def test_registration_opt_out_boolean_values(monkeypatch, value, skip): + registered = set() + _fake_executorch(monkeypatch, registered) + handle = _delegate_handle() + loads = [] + + def load(path, mode): + loads.append(path) + registered.add("TensorRTBackend") + return handle + + monkeypatch.setattr(ctypes, "CDLL", load) + monkeypatch.setattr(os.path, "isfile", lambda path: True) + if value is None: + monkeypatch.delenv(SKIP_ENV, raising=False) + else: + monkeypatch.setenv(SKIP_ENV, value) + spec = importlib.util.spec_from_file_location( + "registration_flag_test", DELEGATE_PATH + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + assert len(loads) == (0 if skip else 1) + assert module._delegate is (None if skip else handle) + assert registered == (set() if skip else {"TensorRTBackend"}) + + +@pytest.mark.unit +def test_registration_opt_out_rejects_missing_on_control(monkeypatch, tmp_path): + tree = ast.parse(DELEGATE_PATH.read_text()) + checks = [ + node + for node in tree.body + if isinstance(node, ast.If) and isinstance(node.test, ast.Compare) + ] + assert len(checks) == 1 + values = checks[0].test.comparators[0].elts + assert sum(value.value == "on" for value in values) == 1 + checks[0].test.comparators[0].elts = [ + value for value in values if value.value != "on" + ] + path = tmp_path / "without_on.py" + path.write_text(ast.unparse(tree)) + monkeypatch.setitem(globals(), "DELEGATE_PATH", path) + with pytest.raises(AssertionError): + test_registration_opt_out_boolean_values(monkeypatch, "ON", True) - native = types.ModuleType("fake_portable_lib") - native._load_for_executorch_from_buffer = fake_load_from_buffer - delegate = types.ModuleType("torch_tensorrt_executorch_runtime") - delegate.get_runtime = lambda: None - delegate.activate = lambda: native - monkeypatch.setitem(sys.modules, delegate.__name__, delegate) - return loaded +def test_register_loads_the_delegate_and_registers_the_backend(monkeypatch): + delegate = load_delegate_module() + registered = set() + _fake_executorch(monkeypatch, registered) + loaded = [] + def fake_cdll(path, mode): + loaded.append((path, mode)) + registered.add(delegate.BACKEND_NAME) + return _delegate_handle() -def test_load_and_forward(monkeypatch, tmp_path): - loaded = _install_fakes(monkeypatch) - model = tmp_path / "model.pte" - model.write_bytes(b"pte") + monkeypatch.setattr(delegate, "_delegate_path", lambda: "/fake/delegate.so") + monkeypatch.setattr(delegate.ctypes, "CDLL", fake_cdll) - program = load_runtime_module().load(model) + assert delegate.register() is None - assert program.forward(2) == [3] - # The bytes reach the loader without a copy, and the wrapper keeps them alive because - # ExecuTorch references that memory rather than owning it. - assert program._data is loaded["data"] + assert [path for path, _ in loaded] == ["/fake/delegate.so"] + # Resolve imports eagerly without adding the delegate's exports to the global namespace. + assert loaded[0][1] == os.RTLD_NOW | os.RTLD_LOCAL -def test_unknown_method(monkeypatch, tmp_path): - _install_fakes(monkeypatch) - model = tmp_path / "model.pte" - model.write_bytes(b"pte") +def test_register_twice_loads_the_delegate_once(monkeypatch): + delegate = load_delegate_module() + registered = set() + _fake_executorch(monkeypatch, registered) + loads = [] - with pytest.raises(ValueError, match="Unknown method"): - load_runtime_module().load(model).run([], "missing") + def fake_cdll(path, mode): + loads.append(path) + registered.add(delegate.BACKEND_NAME) + return _delegate_handle() + monkeypatch.setattr(delegate, "_delegate_path", lambda: "/fake/delegate.so") + monkeypatch.setattr(delegate.ctypes, "CDLL", fake_cdll) -def test_missing_model(): - with pytest.raises(FileNotFoundError): - load_runtime_module().load("does-not-exist.pte") + delegate.register() + delegate.register() + assert loads == ["/fake/delegate.so"] -def test_activate_claims_both_extension_names_and_is_idempotent(monkeypatch): + +def test_register_reports_a_delegate_that_registers_nothing(monkeypatch): + """A delegate can load cleanly and still not register, which must not pass silently. + + This is the failure mode of a delegate built against a different runtime: the library + loads, its initializer runs, and the backend lands in a registry nobody queries. Reporting + it here is the difference between a clear error and an unavailable-backend mystery later. + """ delegate = load_delegate_module() - modules = {} - monkeypatch.setattr(delegate, "sys", types.SimpleNamespace(modules=modules)) - monkeypatch.setattr(delegate, "_probe_portable_lib_dependencies", lambda: None) - data_loader = types.ModuleType(delegate.__name__ + ".data_loader") - native = types.ModuleType(delegate.__name__ + "._portable_lib") - imported = [] - - def fake_import(name): - imported.append(name) - return { - data_loader.__name__: data_loader, - native.__name__: native, - }[name] + _fake_executorch(monkeypatch, set()) + monkeypatch.setattr(delegate, "_delegate_path", lambda: "/fake/delegate.so") monkeypatch.setattr( - delegate, "importlib", types.SimpleNamespace(import_module=fake_import) + delegate.ctypes, "CDLL", lambda path, mode: types.SimpleNamespace() ) - assert delegate.activate() is native - wrapper = types.ModuleType(delegate._WRAPPER_NAME) - modules[delegate._WRAPPER_NAME] = wrapper - assert delegate.activate() is native - assert imported == [data_loader.__name__, native.__name__] - assert modules.get("executorch.extension.pybindings._C") is native - assert modules.get("executorch.extension.pybindings._portable_lib") is native - assert modules[delegate._DATA_LOADER_NAME] is data_loader - assert modules[delegate._WRAPPER_NAME] is wrapper - - -@pytest.mark.parametrize("alias", ["_NATIVE_NAME", "_LEGACY_NATIVE_NAME"]) -def test_activation_check_detects_missing_alias(monkeypatch, alias): - source = DELEGATE_PATH.read_text() - assignment = f" sys.modules[{alias}] = native\n" - assert assignment in source - delegate = types.ModuleType("torchtrt_et_delegate_test") - exec( - compile(source.replace(assignment, ""), str(DELEGATE_PATH), "exec"), - delegate.__dict__, - ) - monkeypatch.setattr(sys.modules[__name__], "load_delegate_module", lambda: delegate) - with pytest.raises(AssertionError): - test_activate_claims_both_extension_names_and_is_idempotent(monkeypatch) + with pytest.raises(delegate.DelegateCompatibilityError, match="did not register"): + delegate.register() -def test_activate_rejects_preloaded_stock_runtime(monkeypatch): - delegate = load_delegate_module() - stock_runtime = types.ModuleType(delegate._NATIVE_NAME) - monkeypatch.setitem(sys.modules, delegate._NATIVE_NAME, stock_runtime) - with pytest.raises(delegate.DelegateCompatibilityError, match="stock runtime"): - delegate.activate() +def test_register_reports_a_missing_executorch(monkeypatch): + # Genuine absence, where the interpreter sets .name to the root package. A blocked or broken + # submodule is a different diagnosis (its .name is the full dotted path), covered by + # test_an_unloadable_executorch_is_not_reported_as_absent, so simulate the root going missing + # rather than None-blocking the chain, which encodes the broken-install signature instead. + delegate = load_delegate_module() + class Boom: + def find_spec(self, name, path=None, target=None): + if name.startswith("executorch"): + raise ModuleNotFoundError( + "No module named 'executorch'", name="executorch" + ) + return None -def test_activate_rejects_preloaded_stock_wrapper(monkeypatch): - delegate = load_delegate_module() - stock_wrapper = types.ModuleType(delegate._WRAPPER_NAME) - monkeypatch.delitem(sys.modules, delegate._NATIVE_NAME, raising=False) - monkeypatch.delitem(sys.modules, delegate._LEGACY_NATIVE_NAME, raising=False) - monkeypatch.setitem(sys.modules, delegate._WRAPPER_NAME, stock_wrapper) + for name in [n for n in sys.modules if n.startswith("executorch")]: + monkeypatch.delitem(sys.modules, name, raising=False) + monkeypatch.setattr(sys, "meta_path", [Boom(), *sys.meta_path]) with pytest.raises( - delegate.DelegateCompatibilityError, - match=r"torch_tensorrt\.load", + delegate.DelegateCompatibilityError, match="ExecuTorch must be installed" ): - delegate.activate() + delegate.register() + +def test_register_reports_an_unloadable_delegate(monkeypatch): + """A load failure that is not the CPU-wheel case keeps the loader's own message. -def test_activate_cleans_up_data_loader_when_native_import_fails(monkeypatch): + Every OSError used to be answered with "install a CUDA build of executorch", which is the + wrong instruction for a missing TensorRT, a missing CUDA runtime, or a libstdc++ too old + for the delegate, and sends the reader after the wrong thing. + """ delegate = load_delegate_module() - monkeypatch.setattr(delegate, "_probe_portable_lib_dependencies", lambda: None) - data_loader = types.ModuleType(delegate.__name__ + ".data_loader") + _fake_executorch(monkeypatch, set()) - def fake_import(name): - if name == data_loader.__name__: - return data_loader - assert sys.modules[delegate._DATA_LOADER_NAME] is data_loader - raise ImportError("native module failed to load") + def fail(path, mode): + raise OSError("libnvinfer.so.11: cannot open shared object file") - monkeypatch.setattr( - delegate, "importlib", types.SimpleNamespace(import_module=fake_import) - ) - monkeypatch.delitem(sys.modules, delegate._NATIVE_NAME, raising=False) - monkeypatch.delitem(sys.modules, delegate._LEGACY_NATIVE_NAME, raising=False) - monkeypatch.delitem(sys.modules, delegate._DATA_LOADER_NAME, raising=False) + monkeypatch.setattr(delegate, "_delegate_path", lambda: "/fake/delegate.so") + monkeypatch.setattr(delegate.ctypes, "CDLL", fail) - with pytest.raises(delegate.DelegateCompatibilityError): - delegate.activate() + with pytest.raises(delegate.DelegateCompatibilityError) as failure: + delegate.register() + + # The concrete cause survives, and the misleading advice is absent. + assert "libnvinfer.so.11" in str(failure.value) + assert "requires a CUDA build of executorch" not in str(failure.value) - assert delegate._DATA_LOADER_NAME not in sys.modules +def test_register_reports_a_cpu_executorch_wheel(monkeypatch): + """The one failure the CPU-wheel diagnosis actually fits. -def test_activate_checks_native_dependencies_before_importing_data_loader(monkeypatch): + This package's pin names no local version label, and a specifier written that way admits any + label, so a +cpu wheel satisfies it and then cannot resolve + libexecutorch_extension_cuda.so, which only the CUDA wheels ship. + """ delegate = load_delegate_module() - data_loader = types.ModuleType(delegate.__name__ + ".data_loader") - native = types.ModuleType(delegate.__name__ + "._portable_lib") - calls = [] + _fake_executorch(monkeypatch, set()) - monkeypatch.delitem(sys.modules, delegate._NATIVE_NAME, raising=False) - monkeypatch.delitem(sys.modules, delegate._LEGACY_NATIVE_NAME, raising=False) - monkeypatch.delitem(sys.modules, delegate._WRAPPER_NAME, raising=False) - monkeypatch.delitem(sys.modules, delegate._DATA_LOADER_NAME, raising=False) + def fail(path, mode): + raise OSError( + "libexecutorch_extension_cuda.so: cannot open shared object file: " + "No such file or directory" + ) - monkeypatch.setattr( - delegate, "_probe_portable_lib_dependencies", lambda: calls.append("probe") - ) + monkeypatch.setattr(delegate, "_delegate_path", lambda: "/fake/delegate.so") + monkeypatch.setattr(delegate.ctypes, "CDLL", fail) - def fake_import(name): - assert calls == ["probe"] - return {data_loader.__name__: data_loader, native.__name__: native}[name] + with pytest.raises( + delegate.DelegateCompatibilityError, match="requires a CUDA build of executorch" + ): + delegate.register() + + +def test_the_delegate_library_is_absent_from_an_unbuilt_package(monkeypatch, tmp_path): + """Use an unbuilt location because the checkout may contain editable build outputs.""" + delegate = load_delegate_module() + monkeypatch.setattr(delegate, "__file__", str(tmp_path / "__init__.py")) + with pytest.raises(delegate.DelegateCompatibilityError, match="missing"): + delegate._delegate_path() + + +def test_the_delegate_is_named_the_way_executorch_names_its_own(tmp_path, monkeypatch): + """The delegate must ship as libexecutorch_backend_.so, like ExecuTorch's own. + + ExecuTorch ships libexecutorch_backend_{cuda,xnnpack,qnn,openvino}.so, so a consumer + looking for a delegate expects that shape. This is worth pinning because the wheel used to + declare the library as a setuptools Extension, which renamed it to + _executorch_backend_tensorrt..so: a name that hides what the file is and implies a + Python ABI the library does not have. It exports no PyInit_ and references no Python + C-API, so the ABI tag was never meaningful. + """ + delegate = load_delegate_module() + + assert delegate._DELEGATE_LIBRARY == "libexecutorch_backend_tensorrt.so" + + # setup.py holds its own copy, which CI reads to check the wheel. If only one of the two + # changed, CI would accept a wheel the runtime cannot load, so pin them to each other. + # Parsed rather than imported: importing setup.py would run setup(). + setup_source = SETUP_PATH.read_text(encoding="utf-8") + (packaged_name,) = [ + node.value.value + for node in ast.parse(setup_source).body + if isinstance(node, ast.Assign) + and any( + getattr(target, "id", None) == "DELEGATE_LIBRARY" for target in node.targets + ) + ] + assert packaged_name == delegate._DELEGATE_LIBRARY, ( + "setup.py ships a different filename than the runtime looks for: " + f"{packaged_name} vs {delegate._DELEGATE_LIBRARY}" + ) + + # The real lookup, against a directory laid out the way the wheel installs. Under lib/, the + # same place ExecuTorch keeps its own backends, which is also where the shipped CMake package + # searches, so the Python loader and a C++ consumer resolve one file. + package = tmp_path / "torch_tensorrt_executorch_runtime" + (package / "lib").mkdir(parents=True) + (package / "lib" / delegate._DELEGATE_LIBRARY).write_bytes(b"") monkeypatch.setattr( - delegate, "importlib", types.SimpleNamespace(import_module=fake_import) + delegate.os.path, "abspath", lambda _: str(package / "__init__.py") + ) + assert delegate._delegate_path() == str( + package / "lib" / delegate._DELEGATE_LIBRARY ) - assert delegate.activate() is native +@pytest.mark.unit +@pytest.mark.parametrize( + "layout,expected", + [ + ("absent", False), + ("path_attribute", True), + ("file_attribute_only", True), + ("no_location", False), + ("wrong_subdirectory", False), + ], +) +def test_the_cuda_extension_probe_reads_the_installed_executorch( + monkeypatch, tmp_path, layout, expected +): + """Decide the CPU-wheel diagnosis on what is on disk, not on what the error names. + + An ABI failure inside a present libexecutorch_extension_cuda.so names it in the message too, + so the probe is what keeps that user from being told to reinstall the CUDA wheel they already + have. Parametrised over the module shapes because the previous version read only __path__, + which types.ModuleType does not define, so under the fakes these tests use it always answered + False and the branch it guards was unreachable. + """ + # Load checkout code with controlled dependencies, regardless of the installed companion. + delegate = load_delegate_module() -def test_activate_dependency_probe_fails_before_data_loader_import(monkeypatch): + root = tmp_path / "executorch" + (root / "lib").mkdir(parents=True) + if layout != "absent": + directory = root / ("libs" if layout == "wrong_subdirectory" else "lib") + directory.mkdir(exist_ok=True) + (directory / delegate._EXTENSION_CUDA_LIBRARY).write_bytes(b"\x7fELF") + + module = types.ModuleType("executorch") + if layout in {"absent", "path_attribute", "wrong_subdirectory"}: + module.__path__ = [str(root)] + elif layout == "file_attribute_only": + module.__file__ = str(root / "__init__.py") + monkeypatch.setitem(sys.modules, "executorch", module) + + assert delegate._extension_cuda_present() is expected + + +@pytest.mark.unit +def test_the_cuda_extension_probe_survives_no_executorch(monkeypatch): + # Import failure is not an ABI failure: with no ExecuTorch at all the library is absent, so + # the CPU-wheel advice is correct and the probe must not raise on the way to saying so. + monkeypatch.setitem(sys.modules, "executorch", None) + # Load checkout code with controlled dependencies, regardless of the installed companion. delegate = load_delegate_module() - imports = [] + assert delegate._extension_cuda_present() is False - monkeypatch.delitem(sys.modules, delegate._NATIVE_NAME, raising=False) - monkeypatch.delitem(sys.modules, delegate._LEGACY_NATIVE_NAME, raising=False) - monkeypatch.delitem(sys.modules, delegate._WRAPPER_NAME, raising=False) - def fail_probe(): - raise OSError("libnvinfer.so is unavailable") +@pytest.mark.unit +@pytest.mark.parametrize( + "extension_on_disk,expect_cpu_advice", + [(False, True), (True, False)], +) +def test_a_present_but_broken_cuda_extension_is_not_diagnosed_as_a_cpu_wheel( + monkeypatch, tmp_path, extension_on_disk, expect_cpu_advice +): + # The whole point of the probe: the loader names the same library in both cases, so only + # what is on disk distinguishes "you installed the CPU wheel" from "your CUDA wheel is + # broken". Deleting the probe from the branch makes both cases give the CPU advice. + # Load checkout code with controlled dependencies, regardless of the installed companion. + delegate = load_delegate_module() - def fake_import(name): - imports.append(name) - raise AssertionError("data_loader must not be imported after probe failure") + # The full submodule chain, because register() imports the registry before it loads the + # delegate; a bare ModuleType stops it earlier with a different error. + _fake_executorch(monkeypatch, set()) + root = tmp_path / "executorch" + (root / "lib").mkdir(parents=True) + if extension_on_disk: + (root / "lib" / delegate._EXTENSION_CUDA_LIBRARY).write_bytes(b"\x7fELF") + sys.modules["executorch"].__path__ = [str(root)] - monkeypatch.setattr(delegate, "_probe_portable_lib_dependencies", fail_probe) + monkeypatch.setattr(delegate, "_delegate_path", lambda: str(tmp_path / "d.so")) monkeypatch.setattr( - delegate, "importlib", types.SimpleNamespace(import_module=fake_import) + delegate.ctypes, + "CDLL", + lambda *a, **k: (_ for _ in ()).throw( + OSError("libexecutorch_extension_cuda.so: cannot open shared object file") + ), ) - monkeypatch.delitem(sys.modules, delegate._DATA_LOADER_NAME, raising=False) - with pytest.raises( - delegate.DelegateCompatibilityError, match="same release matrix" - ): - delegate.activate() + with pytest.raises(delegate.DelegateCompatibilityError) as raised: + delegate.register() + + says_cpu = "a CPU build satisfies the version pin" in str(raised.value) + assert says_cpu is expect_cpu_advice, ( + "the CPU-wheel advice fired for a present extension" + if says_cpu + else "the CPU-wheel advice did not fire for a genuinely absent extension" + ) + + +@pytest.mark.unit +@pytest.mark.parametrize( + "message,expect_install_advice", + [ + ("No module named 'executorch'", True), + # A dependency of an installed ExecuTorch going missing is also a ModuleNotFoundError, but + # its name is that dependency, and telling this user to install ExecuTorch is wrong. + ("No module named 'flatbuffers'", False), + # A submodule of an installed ExecuTorch that is absent or blocked: CPython sets .name to + # the full dotted path, not the root, so this is the broken-install diagnosis rather than + # the absent-package one. Comparing only the first dotted segment misreported it as + # ExecuTorch being uninstalled. + ("No module named 'executorch.extension.pybindings.portable_lib'", False), + # A blocked sys.modules entry means the package was found and something inside it failed, + # which is the broken-install diagnosis rather than the absent-package one. + ("import of executorch.extension halted; None in sys.modules", False), + ("libexecutorch.so: version 'CXXABI_1.3.15' not found", False), + ("libcudart.so.13: cannot open shared object file", False), + ], +) +def test_an_unloadable_executorch_is_not_reported_as_absent( + monkeypatch, message, expect_install_advice +): + # An ABI mismatch reaches the same except clause as a missing package but needs the opposite + # repair. Answering both with "install executorch" told the user to reinstall what they had. + # Load checkout code with controlled dependencies, regardless of the installed companion. + delegate = load_delegate_module() - assert imports == [] - assert delegate._DATA_LOADER_NAME not in sys.modules + # A finder, because the code under test uses a plain `import` statement rather than + # importlib.import_module, so patching that function would not be reached. + class Boom: + def find_spec(self, name, path=None, target=None): + if name.startswith("executorch"): + raise ( + # name= as the interpreter sets it, since the diagnosis reads it to tell a + # genuinely absent ExecuTorch from a missing transitive dependency. The + # message names whichever module was not found, so derive it from there. + ModuleNotFoundError( + message, name=message.split("'")[1] if "'" in message else name + ) + if message.startswith("No module named") + else ImportError(message) + ) + return None + + for name in [n for n in sys.modules if n.startswith("executorch")]: + monkeypatch.delitem(sys.modules, name, raising=False) + monkeypatch.setattr(sys, "meta_path", [Boom(), *sys.meta_path]) + + with pytest.raises(delegate.DelegateCompatibilityError) as raised: + delegate.register() + + advises_install = "must be installed" in str(raised.value) + assert ( + advises_install is expect_install_advice + ), f"for {message!r} the diagnosis was: {raised.value}" diff --git a/tests/py/dynamo/executorch/test_shared_runtime_workflow.py b/tests/py/dynamo/executorch/test_shared_runtime_workflow.py new file mode 100644 index 0000000000..181dcc9950 --- /dev/null +++ b/tests/py/dynamo/executorch/test_shared_runtime_workflow.py @@ -0,0 +1,613 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause + +"""CPU-only command-boundary tests for the shared companion workflows.""" + +import ast +import importlib +import warnings +import json +import os +import shutil +import subprocess +import sys +import types +from pathlib import Path + +import pytest +import yaml +from packaging.requirements import Requirement +from wheel.wheelfile import WheelFile + +pytestmark = pytest.mark.unit +ROOT = Path(__file__).resolve().parents[4] + + +def _workflow(name): + return yaml.safe_load((ROOT / ".github/workflows" / name).read_text()) + + +def _run(script, root, env): + result = subprocess.run( + [shutil.which("bash"), "-c", script], + cwd=root, + env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1", **env}, + text=True, + capture_output=True, + timeout=30, + ) + (root / "stdout.log").write_text(result.stdout) + (root / "stderr.log").write_text(result.stderr) + return result + + +def _assert_test_wheel_dependency(command): + requirements = [Requirement(arg) for arg in command if arg.startswith("wheel")] + assert len(requirements) == 1, "ExecuTorch tests require wheel>=0.40" + assert "0.40.0" in requirements[0].specifier + assert "0.37.1" not in requirements[0].specifier + + +@pytest.mark.parametrize("cuda", ["cu130", "cu132"]) +def test_manifest_suite_provisions_wheel(monkeypatch, cuda): + monkeypatch.syspath_prepend(str(ROOT)) + from tests.ci import runner + + monkeypatch.setenv("CU_VERSION", cuda) + commands = runner._setup_commands("executorch") + command, _ = next((argv, cwd) for argv, cwd in commands if "install" in argv) + _assert_test_wheel_dependency(command) + + +@pytest.mark.parametrize("arch", ["x86_64", "aarch64"]) +@pytest.mark.parametrize("cuda", ["cu130", "cu132"]) +@pytest.mark.parametrize("release", [True, False]) +def test_shared_build_provisions_tensorrt_metadata(tmp_path, arch, cuda, release): + """Start with no TensorRT metadata; run the actual build step up to native build.""" + # Use the main wheel's real architecture-specific dependency selector. + source = ast.parse((ROOT / "setup.py").read_text()) + selector = ( + "get_sbsa_requirements" if arch == "aarch64" else "get_x86_64_requirements" + ) + function = next( + n for n in source.body if isinstance(n, ast.FunctionDef) and n.name == selector + ) + scope = { + "IS_DLFW_CI": False, + "USE_TRT_RTX": False, + "torch": types.SimpleNamespace( + version=types.SimpleNamespace(cuda={"cu130": "13.0", "cu132": "13.2"}[cuda]) + ), + } + exec( + compile(ast.Module(body=[function], type_ignores=[]), "", "exec"), + scope, + ) + requirements = scope[selector]([]) + expected = [r for r in requirements if Requirement(r).name.startswith("tensorrt")] + assert expected + + site = tmp_path / "site" + site.mkdir() + dist = tmp_path / "dist" + dist.mkdir() + version = (ROOT / "version.txt").read_text().strip().removesuffix("a0") + name = f"torch_tensorrt-{version}" + with WheelFile(dist / f"{name}-py3-none-any.whl", "w") as wheel: + wheel.writestr("torch_tensorrt/lib/libtorchtrt.so", b"native fixture") + wheel.writestr( + "torch_tensorrt/__init__.py", "raise AssertionError('compiler imported')\n" + ) + wheel.writestr( + f"{name}.dist-info/WHEEL", "Wheel-Version: 1.0\nTag: py3-none-any\n" + ) + wheel.writestr( + f"{name}.dist-info/METADATA", + f"Metadata-Version: 2.1\nName: torch-tensorrt\nVersion: {version}\n" + + "".join(f"Requires-Dist: {r}\n" for r in requirements), + ) + # Only the package-manager boundary is fake. Metadata lookups and selection run normally. + # The installed version comes from the requirement the real selector produced, so a + # TensorRT upgrade does not need editing here. + lower_bound = next( + specifier + for specifier in Requirement(expected[0]).specifier + if specifier.operator == ">=" + ) + installed_version = lower_bound.version.removesuffix(".0") + (site / "pip.py").write_text( + "import importlib.metadata as m, json, os, sys, zipfile\n" + "from pathlib import Path\n" + "from packaging.requirements import Requirement\n" + "site = Path(__file__).parent\n" + "args = sys.argv[1:]\n" + f"pinned = {installed_version!r}\n" + "with open(os.environ['EVENTS'], 'a') as f: f.write(json.dumps(args) + '\\n')\n" + "if args[0] == 'install':\n" + " for arg in args[1:]:\n" + " if arg.endswith('.whl'):\n" + " with zipfile.ZipFile(arg) as w: w.extractall(site)\n" + " elif arg.startswith('tensorrt'):\n" + " r = Requirement(arg)\n" + " assert r.specifier.contains(pinned), (arg, pinned)\n" + " for name in (r.name, 'tensorrt-cu13', 'tensorrt-cu13-bindings', 'tensorrt-cu13-libs'):\n" + " info = site / (name.replace('-', '_') + '-' + pinned + '.dist-info')\n" + " info.mkdir(exist_ok=True)\n" + " (info / 'METADATA').write_text(f'Name: {name}\\nVersion: {pinned}\\n')\n" + "elif args[0] == 'wheel':\n" + " installed = {d.metadata['Name']: d.version for d in m.distributions(path=[site])}\n" + " if 'tensorrt-cu13' not in installed: raise m.PackageNotFoundError('tensorrt-cu13')\n" + " assert installed['tensorrt-cu13'] == pinned\n" + " assert installed['tensorrt-cu13-libs'] == pinned\n" + " Path(os.environ['BUILT_VERSION']).write_text(os.environ['TORCH_TENSORRT_EXECUTORCH_RUNTIME_VERSION'])\n" + "else: raise AssertionError(args)\n" + ) + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + (bin_dir / "python").write_text(f'#!/bin/sh\nexec "{sys.executable}" "$@"\n') + (bin_dir / "python").chmod(0o755) + (tmp_path / "build-env").write_text("export CONDA_RUN=''\n") + shutil.copy2(ROOT / "version.txt", tmp_path / "version.txt") + runtime = tmp_path / "py/torch-tensorrt-executorch-runtime" + runtime.mkdir(parents=True) + (runtime / "version.txt").write_text("7.4.1\n") + scripts = tmp_path / ".github/scripts" + scripts.mkdir(parents=True) + step = next( + s + for s in _workflow("build_linux.yml")["jobs"]["build"]["steps"] + if s.get("id") == "executorch-runtime" + ) + script = step["run"].replace("${{ inputs.is-release-wheel }}", str(release).lower()) + result = _run( + script, + tmp_path, + { + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "PYTHONPATH": str(site), + "BUILD_ENV_FILE": str(tmp_path / "build-env"), + "BUILD_VERSION": version if release else f"{version}.dev20260911", + "CU_VERSION": cuda, + "ARCH": arch, + "EVENTS": str(tmp_path / "events"), + "BUILT_VERSION": str(tmp_path / "built-version"), + }, + ) + assert result.returncode == 0, result.stdout + result.stderr + events = [ + json.loads(line) for line in (tmp_path / "events").read_text().splitlines() + ] + selected = [ + arg + for event in events + if event[0] == "install" + for arg in event[1:] + if arg.startswith("tensorrt") + ] + assert [Requirement(r) for r in selected] == [Requirement(r) for r in expected] + assert events[0][:2] == ["install", "--no-deps"] + wheel_requirements = [ + Requirement(arg) + for event in events + if event[0] == "install" + for arg in event[1:] + if arg.startswith("wheel") + ] + assert len(wheel_requirements) == 1 + assert ( + "0.37.1" not in wheel_requirements[0].specifier + ), "wheel tags needs wheel>=0.40" + assert "0.40.0" in wheel_requirements[0].specifier + assert events[-1] == [ + "wheel", + "--no-build-isolation", + "--no-deps", + "--wheel-dir", + "dist", + "py/torch-tensorrt-executorch-runtime", + ] + assert (tmp_path / "built-version").read_text() == ( + "7.4.1" if release else f"7.4.1.dev20260911+{cuda}" + ) + + +@pytest.mark.parametrize("arch", ["x86_64", "aarch64"]) +def test_missing_wheel_tool_minimum_is_detected(tmp_path, monkeypatch, arch): + workflow = _workflow("build_linux.yml") + step = next( + s + for s in workflow["jobs"]["build"]["steps"] + if s.get("id") == "executorch-runtime" + ) + assert step["run"].count('"wheel>=0.40"') == 1 + step["run"] = step["run"].replace('"wheel>=0.40"', "wheel") + monkeypatch.setitem(globals(), "_workflow", lambda _: workflow) + with pytest.raises(AssertionError, match="wheel tags needs"): + test_shared_build_provisions_tensorrt_metadata(tmp_path, arch, "cu132", True) + + +@pytest.mark.parametrize("arch", ["x86_64", "aarch64"]) +def test_missing_tensorrt_provisioning_is_detected(tmp_path, monkeypatch, arch): + workflow = _workflow("build_linux.yml") + step = next( + s + for s in workflow["jobs"]["build"]["steps"] + if s.get("id") == "executorch-runtime" + ) + command = ( + 'subprocess.check_call([sys.executable, "-m", "pip", "install", *tensorrt])' + ) + assert step["run"].count(command) == 1 + step["run"] = step["run"].replace(command, "pass") + monkeypatch.setitem( + test_shared_build_provisions_tensorrt_metadata.__globals__, + "_workflow", + lambda _: workflow, + ) + with pytest.raises( + AssertionError, match="No package metadata was found for tensorrt-cu13" + ): + test_shared_build_provisions_tensorrt_metadata(tmp_path, arch, "cu132", True) + + +_DEVICE_EXPORT = 'python examples/torchtrt_executorch_example/export_device_resident.py \\\n --model_path="${RUNNER_TEMP}/torchtrt-device-resident.pte"\n' +_DEVICE_RUN = 'python examples/executorch_reference_runner/load_model_device_resident.py \\\n --model_path="${RUNNER_TEMP}/torchtrt-device-resident.pte" --num_runs=2\n' + + +def _assert_device_commands(tmp_path, workflow, failure=""): + job = workflow["jobs"]["test"] + assert job.get("if", "success()") in ("success()", "${{ success() }}") + assert job["uses"] == "./.github/workflows/linux-test.yml" + script = job["with"]["script"] + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + runner = tmp_path / "runner" + (runner / "bin").mkdir(parents=True) + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + (artifacts / "torch_tensorrt_executorch_runtime-fixture.whl").touch() + helpers = tmp_path / "tests/py/utils/ci_helpers.sh" + helpers.parent.mkdir(parents=True) + helpers.write_text("trt_tier_executorch() { :; }\n") + dispatcher = bin_dir / "dispatch" + dispatcher.write_text( + f"#!{sys.executable}\n" + "import json, os, sys\nfrom pathlib import Path\n" + "tool, args = Path(sys.argv[0]).name, sys.argv[1:]\n" + "with open(os.environ['EVENTS'], 'a') as f: f.write(json.dumps([tool, *args]) + '\\n')\n" + "if tool == 'python':\n" + " if args[:2] == ['-m', 'pip']: pass\n" + " elif args[:2] == ['-m', 'venv']:\n" + " p = Path(args[2]) / 'bin/python'; p.parent.mkdir(parents=True); p.symlink_to(os.environ['DISPATCH'])\n" + " elif args[:4] == ['-u', '-X', 'faulthandler', '-c']: pass\n" + " elif args[0].startswith('examples/'):\n" + " model = Path(next(a.split('=', 1)[1] for a in args if a.startswith('--model_path=')))\n" + " if Path(args[0]).name.startswith('export_'): model.write_text('exported')\n" + " else: assert model.read_text() == 'exported'\n" + " if os.environ['FAILURE'] and os.environ['FAILURE'] == Path(args[0]).name: sys.exit(17)\n" + " else: raise AssertionError(args)\n" + "elif tool == 'bazel':\n" + " if args[0] == 'info': print(os.environ['RUNNER_TEMP'])\n" + " elif args[0] == 'query': print(os.environ['RUNNER_TEMP'] + '/executorch/CMakeLists.txt:1:1')\n" + " else: assert args[0] in ('build', 'test')\n" + "elif tool == 'curl': pass\n" + "elif tool == 'find': print(os.environ['RUNNER_TEMP'] + '/libs')\n" + "elif tool == 'verify-executorch-reference-runner.sh':\n" + " assert all(Path(a).read_text() == 'exported' for a in args)\n" + "else: raise AssertionError((tool, args))\n" + ) + dispatcher.chmod(0o755) + for tool in ("python", "bazel", "curl", "find"): + (bin_dir / tool).symlink_to(dispatcher) + (runner / "bin/bazel").symlink_to(dispatcher) + for tool in ("mkdir", "chmod", "sort", "head", "dirname"): + (bin_dir / tool).symlink_to(shutil.which(tool)) + reference = tmp_path / ".github/scripts/verify-executorch-reference-runner.sh" + reference.parent.mkdir(parents=True) + reference.symlink_to(dispatcher) + result = _run( + script.replace("/opt/torch-tensorrt-builds", str(artifacts)), + tmp_path, + { + "PATH": str(bin_dir), + "RUNNER_TEMP": str(runner), + "CU_VERSION": "cu132", + "EVENTS": str(tmp_path / "events"), + "DISPATCH": str(dispatcher), + "FAILURE": failure, + }, + ) + events = [ + json.loads(line) for line in (tmp_path / "events").read_text().splitlines() + ] + setup = next( + event for event in events if event[:4] == ["python", "-m", "pip", "install"] + ) + _assert_test_wheel_dependency(setup) + examples = [ + event[1:] + for event in events + if event[0] == "python" and event[1].startswith("examples/") + ] + assert ( + examples + == [ + [ + "examples/torchtrt_executorch_example/export_static_shape.py", + f"--model_path={runner}/torchtrt-python.pte", + ], + [ + "examples/torchtrt_executorch_example/export_kv_cache_decode.py", + f"--model_path={runner}/torchtrt-kv-cache-decode.pte", + ], + [ + "examples/torchtrt_executorch_example/export_coalesced.py", + f"--model_path={runner}/torchtrt-coalesced.pte", + ], + [ + "examples/torchtrt_executorch_example/export_device_resident.py", + f"--model_path={runner}/torchtrt-device-resident.pte", + ], + [ + "examples/executorch_reference_runner/load_model.py", + f"--model_path={runner}/torchtrt-python.pte", + "--num_runs=1", + ], + [ + "examples/executorch_reference_runner/load_model_device_resident.py", + f"--model_path={runner}/torchtrt-device-resident.pte", + "--num_runs=2", + ], + ][: 4 if failure == "export_device_resident.py" else 6] + ), ( + result.stdout + result.stderr + ) + reference_calls = [e[1:] for e in events if e[0] == reference.name] + assert reference_calls == ( + [] + if failure == "export_device_resident.py" + else [ + [ + str(runner / f"torchtrt-{name}.pte") + for name in ("python", "kv-cache-decode", "coalesced") + ] + ] + ) + assert result.returncode == (17 if failure else 0), result.stdout + result.stderr + + +@pytest.mark.parametrize( + "failure", ["", "export_device_resident.py", "load_model_device_resident.py"] +) +def test_device_commands_execute_and_propagate_failure(tmp_path, failure): + _assert_device_commands(tmp_path, _workflow("executorch-test-linux.yml"), failure) + + +@pytest.mark.parametrize( + "command", [_DEVICE_EXPORT, _DEVICE_RUN], ids=["export", "run"] +) +@pytest.mark.parametrize("mutation", ["commented", "disabled", "removed"]) +def test_device_command_removal_is_detected(tmp_path, command, mutation): + workflow = _workflow("executorch-test-linux.yml") + script = workflow["jobs"]["test"]["with"]["script"] + assert script.count(command) == 1 + replacement = { + "commented": "".join("# " + line for line in command.splitlines(keepends=True)), + "disabled": "if false; then\n" + command + "fi\n", + "removed": "", + }[mutation] + workflow["jobs"]["test"]["with"]["script"] = script.replace(command, replacement) + with pytest.raises(AssertionError): + _assert_device_commands(tmp_path, workflow) + + +@pytest.mark.parametrize("entrypoint", ["manifest", "workflow"]) +def test_missing_test_wheel_dependency_is_detected(tmp_path, monkeypatch, entrypoint): + if entrypoint == "manifest": + monkeypatch.syspath_prepend(str(ROOT)) + from tests.ci import runner + + setup = runner._setup_commands + monkeypatch.setattr( + runner, + "_setup_commands", + lambda step: [ + ([arg for arg in argv if not arg.startswith("wheel")], cwd) + for argv, cwd in setup(step) + ], + ) + with pytest.raises(AssertionError, match="ExecuTorch tests require wheel"): + test_manifest_suite_provisions_wheel(monkeypatch, "cu132") + else: + workflow = _workflow("executorch-test-linux.yml") + script = workflow["jobs"]["test"]["with"]["script"] + assert script.count('"wheel>=0.40"') == 1 + workflow["jobs"]["test"]["with"]["script"] = script.replace('"wheel>=0.40"', "") + with pytest.raises(AssertionError, match="ExecuTorch tests require wheel"): + _assert_device_commands(tmp_path, workflow) + + +def test_disabled_device_job_is_detected(tmp_path): + workflow = _workflow("executorch-test-linux.yml") + workflow["jobs"]["test"]["if"] = "${{ false }}" + with pytest.raises(AssertionError): + _assert_device_commands(tmp_path, workflow) + + +@pytest.mark.unit +def test_the_delegate_lane_narrows_the_matrix_to_cuda_13_rows() -> None: + """The lane passes --executorch-runtime, and the filter must act on it. + + Nothing else asserted this, so deleting the flag from the workflow, or the branch it gates + in the filter, left every guard green while the lane silently tested rows whose channel + carries no ExecuTorch. + """ + workflow = (ROOT / ".github/workflows/executorch-test-linux.yml").read_text( + encoding="utf-8" + ) + # Not a text search: the flag has to reach the filter. Leaving it only inside a comment, which is + # what commenting the whole invocation out does, passed a search of the file. + document = yaml.safe_load(workflow) + invocations = [ + step["run"] + for job in document["jobs"].values() + for step in job.get("steps", []) + if isinstance(step, dict) and "filter-matrix.py" in (step.get("run") or "") + ] + assert invocations, "no step runs the matrix filter" + live = [ + line + for run in invocations + for line in run.splitlines() + if "--executorch-runtime" in line and not line.lstrip().startswith("#") + ] + assert live, f"the flag reaches no live command line: {invocations}" + + script = ROOT / ".github/scripts/filter-matrix.py" + rows = [ + {"desired_cuda": cuda, "python_version": "3.10", "gpu_arch_type": "cuda"} + for cuda in ("cu126", "cu130", "cu134") + ] + result = subprocess.run( + [ + sys.executable, + str(script), + "--executorch-runtime", + "--use-rtx", + "false", + "--limit-pr-builds", + "false", + "--matrix", + json.dumps({"include": rows}), + ], + capture_output=True, + text=True, + check=True, + ) + kept = {row["desired_cuda"] for row in json.loads(result.stdout)["include"]} + assert kept and all(row.startswith("cu13") for row in kept), kept + + +@pytest.mark.unit +def test_the_removed_entry_points_still_exist_as_shims() -> None: + """activate() and get_runtime() were public, so removing them outright breaks callers.""" + package = ROOT / "py/torch-tensorrt-executorch-runtime" + source = (package / "torch_tensorrt_executorch_runtime/__init__.py").read_text( + encoding="utf-8" + ) + assert "def activate(" in source, source + assert "def get_runtime(" in source, source + for name in ("activate", "get_runtime", "register"): + assert f'"{name}"' in source.split("__all__")[-1], name + + +@pytest.mark.parametrize( + "raised", + [ + RuntimeError("no bazel here"), + OSError("toolchain gone"), + ValueError("bad config"), + ], +) +@pytest.mark.unit +def test_a_failed_native_build_cannot_report_success(raised) -> None: + """During an editable install setuptools routes a customized build_py through its own + _safely_run, which catches Exception and downgrades it to a warning pip hides, so a failed + delegate build would leave pip printing that it installed successfully. SystemExit is not an + Exception, so converting to it is what escapes. + + Driven rather than read: the wrapper is applied to a build that raises, and the result is passed + through a stand-in for setuptools' catch. A version that re-raised RuntimeError unchanged, which + is what a missing bazel raises, passed a source check while still being swallowed here. + """ + source = (ROOT / "py/torch-tensorrt-executorch-runtime/setup.py").read_text( + encoding="utf-8" + ) + tree = ast.parse(source) + build_class = next( + node + for node in tree.body + if isinstance(node, ast.ClassDef) and node.name == "BazelBuild" + ) + run = next( + node + for node in build_class.body + if isinstance(node, ast.FunctionDef) and node.name == "run" + ) + + class Fake: + def _build(self): + raise raised + + namespace: dict[str, object] = {} + exec(compile(ast.Module(body=[run], type_ignores=[]), "", "exec"), namespace) + # setuptools' own shape: Exception becomes a warning, anything else propagates. + try: + namespace["run"](Fake()) + except Exception as error: # noqa: BLE001 + pytest.fail(f"a {type(raised).__name__} would be swallowed as {error!r}") + except SystemExit as exit_error: + assert "ExecuTorch delegate build failed" in str(exit_error), exit_error + else: + pytest.fail("the wrapper let a failing build return normally") + + +@pytest.mark.unit +def test_the_kept_entry_points_actually_warn_and_forward(monkeypatch) -> None: + """activate() and get_runtime() were public, so they stay as deprecated shims. + + Checking the source for "def activate(" proves only that the text is present: the bodies can be + emptied, or their warnings removed, and the check still passes. So import the package and call + them. Registration is skipped through the package's own escape hatch, because the native library + is not built here, and each shim's own call to register is replaced so the forwarding is visible. + """ + monkeypatch.setenv("TORCH_TENSORRT_SKIP_DELEGATE_REGISTRATION", "1") + monkeypatch.syspath_prepend(str(ROOT / "py/torch-tensorrt-executorch-runtime")) + for name in [ + n for n in sys.modules if n.startswith("torch_tensorrt_executorch_runtime") + ]: + monkeypatch.delitem(sys.modules, name) + delegate = importlib.import_module("torch_tensorrt_executorch_runtime") + + registered = [] + monkeypatch.setattr(delegate, "register", lambda: registered.append("register")) + for shim in ("activate", "get_runtime"): + registered.clear() + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + # ExecuTorch is absent here, so the forwarding import is what fails, and requiring that + # failure is what proves the forwarding happened. Swallowing it let a shim that registers, + # warns, and forwards nowhere pass. + # Raising at all is the proof. Which module the chain fails on depends on what is + # installed, so the name is not asserted, only that the import was reached. + with pytest.raises(ImportError): + getattr(delegate, shim)() + assert registered == ["register"], f"{shim} did not register: {registered}" + assert any( + issubclass(w.category, DeprecationWarning) for w in caught + ), f"{shim} raised no DeprecationWarning: {[w.category for w in caught]}" + + +@pytest.mark.unit +def test_the_coalesced_program_is_run_on_a_caller_stream() -> None: + """Nothing exercised the caller stream, which is the path the delegate is built around. + + The delegate takes the stream from the caller and a green context confines it to a slice of the + machine. Both were reachable only by hand: every automated run used the default stream, so a + delegate that stopped honouring the caller's stream would have kept passing. + """ + script = (ROOT / ".github/scripts/verify-executorch-reference-runner.sh").read_text( + encoding="utf-8" + ) + live = [ + line + for line in script.splitlines() + if "--green_context_sms=" in line and not line.lstrip().startswith("#") + ] + assert ( + live + ), "no run passes a green context, so the caller stream is never exercised" + # On the coalesced program, because that is the case with two backends sharing one stream. + assert "coalesced_green_context.log" in script, script[-400:] + # And the numbers are checked, not only the exit status. + assert 'assert_runner_output "${green_runner_log}"' in script, script[-400:] diff --git a/tests/py/dynamo/executorch/test_update_executorch_pin.py b/tests/py/dynamo/executorch/test_update_executorch_pin.py index aa2368a661..5edb1012ba 100644 --- a/tests/py/dynamo/executorch/test_update_executorch_pin.py +++ b/tests/py/dynamo/executorch/test_update_executorch_pin.py @@ -624,3 +624,81 @@ def test_write_pins_finishes_an_interrupted_run(pin_repo, monkeypatch): lambda: [updater._VERSIONS_FILE, already], ) assert updater.write_pins(target_version, target_commit) is True + + +@pytest.mark.unit +def test_a_site_that_loses_its_version_requirement_is_not_excused_by_its_commit( + pin_repo, monkeypatch +): + """Four sites carry both coordinates, and the old check accepted either one. + + So a requirement the pattern stopped matching, after a reformat say, would leave the version + stale while the commit moved on. That is precisely the drift the two pins exist to prevent, so + each declared coordinate has to be satisfied on its own. + """ + both = sorted( + name for name, kinds in updater._SITE_COORDINATES.items() if len(kinds) == 2 + ) + assert both, "expected sites declaring both coordinates" + victim = pin_repo / both[0] + current = updater.read_pin("__executorch_version__") + # Keep the commit, break only the version requirement's spelling. The site writes it as a bare + # `executorch==` comment, so dropping the operator is enough to defeat the pattern. + body = victim.read_text(encoding="utf-8") + assert f"executorch=={current}" in body, body[:200] + victim.write_text( + body.replace(f"executorch=={current}", f"executorch at {current}"), + encoding="utf-8", + ) + with pytest.raises(SystemExit, match="no ExecuTorch version requirement"): + updater.write_pins("9.9.9", "b" * 40) + + +@pytest.mark.unit +def test_every_declared_site_coordinate_matches_the_tree(pin_repo): + """The declaration is only useful while it describes the files, so check it rather than trust it.""" + commit = updater.read_pin("__executorch_commit__") + for name, kinds in updater._SITE_COORDINATES.items(): + text = (pin_repo / name).read_text(encoding="utf-8") + assert ("version" in kinds) == bool(updater._REQUIREMENT.search(text)), name + assert ("commit" in kinds) == (commit in text), name + + +@pytest.mark.unit +def test_an_unrelated_package_at_the_target_version_is_not_a_pin_site(pin_repo): + """The version check runs through the same pattern that does the rewriting. + + Accepting a bare occurrence of the target version anywhere in the file let a different package + happening to sit at that version stand in for the ExecuTorch requirement, so a site whose + requirement had been reformatted away would report as satisfied. + """ + both = sorted( + name for name, kinds in updater._SITE_COORDINATES.items() if "version" in kinds + ) + victim = pin_repo / both[0] + current = updater.read_pin("__executorch_version__") + body = victim.read_text(encoding="utf-8") + # Break the requirement's spelling, and leave the target version behind on another package. + victim.write_text( + body.replace(f"executorch=={current}", "executorch at large") + + "\nsomething-else==9.9.9\n", + encoding="utf-8", + ) + with pytest.raises(SystemExit, match="no ExecuTorch version requirement"): + updater.write_pins("9.9.9", "c" * 40) + + +def test_the_stable_track_warns_that_its_pin_will_not_build(monkeypatch, capsys): + """A stable pin cannot build, and that used to be discovered from a failed pull request. + + The delegate links the ExecuTorch runtime, so its build takes only a CUDA-labelled one. The + release index publishes processor-only wheels and the CUDA channels publish no stable ExecuTorch + at all, so a stable pin names something the build rejects. Warned rather than refused, because it + becomes correct as soon as a stable CUDA build exists. + """ + monkeypatch.setattr(updater, "read_pin", lambda field: "1.0.dev0") + monkeypatch.setattr(updater, "available_versions", lambda args: ["1.5.0"]) + monkeypatch.setattr(updater, "wheel_git_version", lambda version, args: _COMMIT) + monkeypatch.setattr(updater, "write_pins", lambda version, commit: True) + updater.main(["--track", "stable"]) + assert "no CUDA build of ExecuTorch" in capsys.readouterr().err