diff --git a/scripts/test262_subset.py b/scripts/test262_subset.py index 81077cff52..235c86becc 100755 --- a/scripts/test262_subset.py +++ b/scripts/test262_subset.py @@ -59,8 +59,10 @@ rejected, the compile rejection is *correct* and lands in `pass` instead.) - skip — couldn't assemble / needs an unsupported flag or `$262` host - API. Excluded from the parity verdict — never charged against - Perry. + API, OR compiled but then hit a construct the AOT model can't + evaluate at runtime (runtime-string `eval`/`new Function`, + runtime-computed `import()`, `.wasm` import — #5593). Excluded + from the parity verdict — never charged against Perry. Usage ----- @@ -136,6 +138,17 @@ r"\$262\b|detachArrayBuffer|createRealm|evalScript|IsHTMLDDA|\bagent\." ) +# Cases that *compile* but, when run, reach a construct the AOT model +# categorically cannot evaluate — a runtime-string `eval()` / `new Function()`, +# a runtime-computed dynamic `import()`, or a `.wasm` import. Perry defers these +# to a runtime throw carrying this sentinel (`PERRY_ALLOW_EVAL=1` only const- +# folds *constant-string* eval, so the runtime-string probes — e.g. Annex B +# B.3.3 sloppy block-function hoisting checked via `eval` — still defer). This +# is a permanent AOT limitation, not a language-parity gap, so such a case +# buckets as `skip` (excluded from the verdict) rather than being charged as a +# `runtime-fail` (#5593). +_AOT_DEFER = re.compile(r"cannot run in an ahead-of-time compiled binary") + # Frontmatter flags that make a case un-runnable as a plain script under this # differential. `module` needs ESM loader semantics; the agent/Can-block flags # need a multi-realm host. @@ -502,6 +515,10 @@ def judge_one(case): return (rel, cat, "pass", "", False, False) return (rel, cat, "diff", first_line(p_out), False, False) if node_clean and not perry_clean: + if _AOT_DEFER.search(p_out): + return (rel, cat, "skip", + "AOT-deferred runtime eval/import (#5593)", + False, False) return (rel, cat, "runtime-fail", first_line(p_out), False, False) if not node_clean and not perry_clean: diff --git a/test-compat/test262/README.md b/test-compat/test262/README.md index 443106c1a4..9b58060486 100644 --- a/test-compat/test262/README.md +++ b/test-compat/test262/README.md @@ -110,15 +110,41 @@ runner buckets by Perry-vs-Node **agreement**: - `compile-fail` — Perry refused to compile a case Node ran clean. (When Node *also* rejected, Perry's compile rejection is the correct answer and lands in `pass`.) -- `skip` — couldn't assemble (missing include) or needs an - unsupported flag / `$262` host API. Excluded from the - parity verdict — never charged against Perry. +- `skip` — couldn't assemble (missing include), needs an + unsupported flag / `$262` host API, or *compiled but then* + hit a construct the AOT model categorically can't evaluate + at runtime — a runtime-string `eval()` / `new Function()`, + a runtime-computed dynamic `import()`, or a `.wasm` import + (Perry throws `… cannot run in an ahead-of-time compiled + binary`; see the AOT-eval note below, #5593). Excluded from + the parity verdict — never charged against Perry. `parity_pct = pass / (pass + diff + runtime-fail + compile-fail)`. The report also records `negative_agreements` (how many of the passes are both-runtimes-correctly-rejected) so the language-correctness signal and the negative-rejection signal stay legible. +### Runtime-string `eval` / `new Function` — a permanent AOT limitation (#5593) + +Perry compiles ahead-of-time: there is no interpreter in the produced binary, +so a `eval()` / `new Function()` whose **source is only known at runtime** +cannot be evaluated. `PERRY_ALLOW_EVAL=1` (set in the compile env above) makes +Perry const-fold the cases whose argument is a *constant string* — those run — +but the rest are compiled to a deferred site that throws +`eval() cannot run in an ahead-of-time compiled binary (file:line)` if it is +ever reached. The same deferral covers `new Function(body)`, a runtime-computed +dynamic `import()`, and a `.wasm` import. + +In Test262 this hits a cluster of Annex B cases — `annexB/language/eval-code/ +direct/*` (B.3.3 sloppy block-function hoisting, *probed* via a runtime `eval`) +and a few `annexB/built-ins/RegExp/*` — roughly 72 cases as of the 2026-06-23 +sweep. These are **out of scope by construction**, not a language-parity gap: +they require an interpreter Perry deliberately doesn't ship. The runner detects +the sentinel error in the compiled binary's output and buckets such a case as +`skip` (excluded from the parity verdict) instead of charging it as a +`runtime-fail`. This is a deliberate **WONTFIX**: closing it would mean +embedding a runtime JS interpreter, which is a non-goal for an AOT compiler. + ## Files - `features-applicable.txt` — curated allow-list of feature tags.