From 3fc6e0422ac9e6838a2c6f65ba6d6595c432b9e5 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sun, 19 Jul 2026 10:59:48 +0800 Subject: [PATCH 01/26] docs(plan): add short-function hot-loop inline plan --- ...jit_short_function_hot_loop_inline_plan.md | 606 ++++++++++++++++++ 1 file changed, 606 insertions(+) create mode 100644 plans/pd_vm_jit_short_function_hot_loop_inline_plan.md diff --git a/plans/pd_vm_jit_short_function_hot_loop_inline_plan.md b/plans/pd_vm_jit_short_function_hot_loop_inline_plan.md new file mode 100644 index 00000000..ec60a639 --- /dev/null +++ b/plans/pd_vm_jit_short_function_hot_loop_inline_plan.md @@ -0,0 +1,606 @@ +# pd-vm JIT Short-Function Hot-Loop Inlining Implementation Plan + +**Goal:** Inline hot, short RustScript function-item calls into caller traces while preserving real callable semantics and restoring exact VM state on every cold exit, guard miss, interruption, and deoptimization. + +**Architecture:** Add per-call-site target profiling and a conservative inline eligibility pass, then let the trace recorder maintain a bounded virtual frame stack across eligible `CallValue`/`Ret` pairs. Normal execution maps arguments, locals, captures, and return values directly in SSA without entering a physical VM frame. Each exit carries virtual-frame snapshots so the existing cold Rust recovery path can reconstruct the physical caller/callee frame chain only when native execution actually leaves the trace. + +**Tech Stack:** Rust, pd-vm bytecode and callable metadata, trace JIT SSA, Cranelift native lowering, inherited-state ABI, Criterion, Cargo workspace tests. + +--- + +## 1. Motivation and Current State + +The completed inherited-state trace-link work removed repeated full frame restoration between compatible native traces. The remaining sort workload still crosses a `CallValue` boundary for short script functions and therefore retains native call/return bookkeeping that the old flattened compiler path did not pay. + +The final inherited-state gate recorded in [`pd_vm_call_overhead_reduction_plan.md`](pd_vm_call_overhead_reduction_plan.md) was: + +| Endpoint | Four-run median | +|---|---:| +| crates.io `pd-vm 0.23.1` | `72.091 ms` | +| inherited-state candidate | `134.495 ms` | +| ratio | `1.866x` | + +The current recorder terminates a trace at every dynamic script call: + +- [`src/vm/jit/recorder.rs`](../src/vm/jit/recorder.rs) lowers `DecodedOp::CallValue` to `SsaTerminator::CallValue` and ends recording. +- [`src/vm/jit/native/lower.rs`](../src/vm/jit/native/lower.rs) materializes the call exit and invokes the inherited frame-enter helper. +- callee `Ret` leaves the physical frame through the inherited frame-leave path before the caller continuation can run. +- [`src/vm/jit/trace.rs`](../src/vm/jit/trace.rs) profiles hot loop entries and trace exits, but does not yet profile call-site target identity. + +The target shape is the LuaJIT-style model already identified in the call-overhead plan: a hot caller trace records through a guarded script call, keeps the callee as a virtual frame, records through `Ret`, and reconstructs interpreter frames only on a true exit. + +## 2. Scope + +### 2.1 Phase-1 supported calls + +Phase 1 admits only calls that satisfy every rule below: + +- caller is a root-frame hot loop trace +- callable resolves to one immutable `RootCallableBinding` +- target is `CallableTarget::ScriptFunction` +- callable kind is `FunctionItem` +- callable environment is absent +- arity exactly matches the prototype +- callee has one bytecode `Ret` and no fallthrough beyond its `FunctionRegion` +- no recursive edge +- no nested script `CallValue` +- no yielding host import +- no backward branch in the callee +- decoded callee length is at most 32 instructions +- touched callee-local footprint is at most 8 slots, excluding untouched global-layout slots +- combined caller and inlined-callee trace length remains within `JitConfig::max_trace_len` + +Specialized non-yielding builtins already supported by the SSA recorder may remain inside an eligible leaf. Mutable collection operations are enabled only after virtual-frame deopt restoration and ownership tests pass. + +### 2.2 Later extensions + +After phase 1 meets correctness and performance gates: + +1. allow inline depth 2 for caller → leaf → leaf +2. admit monomorphic no-capture dynamic callables with an exact identity guard +3. admit immutable copied captures +4. admit simple forward branches +5. consider a two-entry polymorphic inline cache only when profiling proves a meaningful workload + +### 2.3 Non-goals + +- no source-level call flattening +- no change to interpreter callable semantics +- no inlining of recursive, yielding, polymorphic, or mutable-capture calls in phase 1 +- no unbounded bytecode cloning +- no new VM-visible opcode +- no fast path that skips callable arity, depth, ownership, fuel, epoch, or invalidation semantics +- no warm-path Rust helper per inlined static function-item call + +## 3. Correctness Invariants + +1. **Callable identity:** an inlined call executes only the profiled/proven prototype. A target change takes the existing `CallValue` path. +2. **Frame semantics:** every cold exit can reconstruct the same `ExecutionFrame`, operand stack, locals, continuation IP, prototype ID, and active IP that the interpreter would have at that bytecode point. +3. **Ownership:** each owned heap value has one owner transfer. Borrowed values are never released. A deopt packet cannot classify `BoxHeapPtr` as an inherited owned value. +4. **Capture semantics:** phase 1 accepts no environment. Later captured-call support must guard exact environment identity and preserve capture-cell aliasing. +5. **Depth semantics:** virtual inline depth counts toward `max_script_call_depth`. Phase 1 root-only admission makes the physical depth known; later non-root admission requires a native depth guard. +6. **Interruption:** fuel and epoch exits preserve the same resume IP and do not repeat or skip a side effect. +7. **Mutation ordering:** a collection mutation visible before deopt remains visible exactly once after restoration. +8. **Invalidation:** mode changes, reset, callable release, trace invalidation, or program replacement clear all call-site profiles and inline target references. +9. **Bounded growth:** inline depth, callee instruction count, touched locals, total trace length, compile time, and machine-code growth all have hard caps. +10. **Cold-path reuse:** existing full Rust recovery remains authoritative for unsupported calls and failed guards. + +## 4. Data Model + +Create [`src/vm/jit/inline.rs`](../src/vm/jit/inline.rs) and register it from [`src/vm/jit/mod.rs`](../src/vm/jit/mod.rs). + +```rust +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub(crate) struct CallSiteKey { + pub(crate) caller_frame_key: u64, + pub(crate) call_ip: usize, +} + +#[derive(Clone, Debug)] +pub(crate) struct CallSiteProfile { + pub(crate) prototype_id: u32, + pub(crate) observations: u64, + pub(crate) mismatches: u64, + pub(crate) monomorphic: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum InlineRejectReason { + NonRootCaller, + UnknownTarget, + PolymorphicTarget, + HostTarget, + CapturedCallable, + ArityMismatch, + Recursive, + NestedScriptCall, + YieldingCall, + BackwardBranch, + MultipleReturns, + TooManyInstructions, + TooManyTouchedLocals, + TraceBudgetExceeded, +} + +#[derive(Clone, Debug)] +pub(crate) struct InlineCandidate { + pub(crate) prototype_id: u32, + pub(crate) entry_ip: usize, + pub(crate) end_ip: usize, + pub(crate) parameter_slots: Vec, + pub(crate) touched_locals: Vec, + pub(crate) decoded_instruction_count: usize, +} +``` + +Phase-1 target proof uses `Program::root_callable_bindings` plus the callable value's SSA `source_local`. It must not rely only on `ValueType::Callable`. + +For later dynamic targets, retain a strong `SharedCallable` in the profile/trace and compare exact callable identity. Do not persist a naked environment pointer that can be reused after deallocation. + +Extend [`src/vm/jit/ir.rs`](../src/vm/jit/ir.rs) so exits can carry zero or more virtual frames, outermost to innermost: + +```rust +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct VirtualFrameSnapshot { + pub(crate) prototype_id: u32, + pub(crate) call_ip: usize, + pub(crate) return_ip: usize, + pub(crate) resume_ip: usize, + pub(crate) operand_stack: Vec, + pub(crate) locals: Vec, + pub(crate) dirty_locals: Vec, +} +``` + +The exact field placement may be adjusted to avoid duplicating existing `SsaExit` data, but one exit must describe the complete physical frame chain required by recovery. + +## 5. Task Plan + +### Task 1: Add Call-Site Profiling and Metrics + +**Objective:** Measure actual script-call target stability before changing trace shape. + +**Files:** +- Create: `src/vm/jit/inline.rs` +- Modify: `src/vm/jit/mod.rs` +- Modify: `src/vm/jit/trace.rs` +- Modify: `src/vm/mod.rs` +- Modify: `src/vm/native/bridge.rs` +- Test: `tests/jit/jit_tests.rs` +- Test: `tests/jit/perf_tests.rs` + +**Steps:** + +1. Add RED tests proving profiles are keyed by `(caller_frame_key, call_ip)`, not only bytecode IP. +2. Add RED tests for monomorphic, target-change, reset, and invalidation behavior. +3. Record the actual `prototype_id` immediately after `CallValue` callable validation and before physical frame entry. +4. Record observations from both interpreter call entry and `pd_vm_native_enter_call_value` so a native caller still trains the profile. +5. Clear profiles on `TraceJitEngine::clear`, JIT mode changes, program replacement, and `Vm::reset_for_reuse` where current trace state is invalidated. +6. Extend `JitMetrics` and JIT dump output with: + - `script_call_observations` + - `monomorphic_call_sites` + - `polymorphic_call_sites` + - `inline_attempts` + - `inline_successes` + - `inline_rejects` by reason + - `inline_deopts` + - `inline_virtual_frame_restores` +7. Keep metrics diagnostic-only; no admission decision may depend on a counter that is absent after deserialization or reset. + +**Focused commands:** + +```bash +cargo test -p pd-vm --features cranelift-jit --test jit_tests call_site_profile -- --nocapture +cargo test -p pd-vm --features cranelift-jit --test jit_tests trace_jit_executes_call_value_natively_inside_loop -- --nocapture +``` + +**Expected:** profile tests pass; existing script-call behavior and trace counts remain unchanged. + +**Commit:** + +```bash +git add src/vm/jit src/vm/mod.rs src/vm/native/bridge.rs tests/jit + +git commit -m "feat(jit): profile script call targets" +``` + +### Task 2: Implement Conservative Inline Eligibility Analysis + +**Objective:** Resolve immutable root function items and reject every unsupported callee before recorder mutation. + +**Files:** +- Modify: `src/vm/jit/inline.rs` +- Modify: `src/vm/jit/recorder.rs` +- Test: `src/vm/jit/recorder.rs` +- Test: `tests/jit/jit_tests.rs` + +**Steps:** + +1. Add table-driven RED tests for every `InlineRejectReason`. +2. Decode the target `ScriptFunction` region without changing the active recorder cursor. +3. Count decoded instructions and touched local slots; do not use `CallablePrototype::frame_local_count` as the leaf-local footprint because callable frames use the merged global slot layout. +4. Reject backward branches, region escapes, recursive target IDs, nested `CallValue`, yielding calls, malformed immediates, and multiple returns. +5. Allow only builtin calls already accepted by `emit_specialized_builtin_call`; all other calls reject the candidate. +6. Resolve phase-1 targets only when the callable SSA value has a `source_local` matching an immutable `RootCallableBinding`. +7. Cache immutable candidate analysis by `prototype_id`; clear it with the program/JIT engine. + +**Focused commands:** + +```bash +cargo test -p pd-vm --lib vm::jit::recorder::tests::inline -- --nocapture +cargo test -p pd-vm --features cranelift-jit --test jit_tests inline_eligibility -- --nocapture +``` + +**Expected:** analysis tests pass and no trace is inlined yet. + +**Commit:** + +```bash +git add src/vm/jit/inline.rs src/vm/jit/recorder.rs tests/jit/jit_tests.rs + +git commit -m "feat(jit): classify inline script callees" +``` + +### Task 3: Add Virtual-Frame Snapshots to SSA Exits + +**Objective:** Represent deopt state inside an inlined callee without creating a physical frame on the normal path. + +**Files:** +- Modify: `src/vm/jit/ir.rs` +- Modify: `src/vm/jit/deopt.rs` +- Modify: `src/vm/jit/recorder.rs` +- Modify: `src/vm/jit/native/lower.rs` +- Modify: `src/vm/native/mod.rs` +- Modify: `src/vm/native/bridge.rs` +- Test: `src/vm/jit/ir.rs` +- Test: `src/vm/jit/deopt.rs` +- Test: `tests/jit/jit_tests.rs` + +**Steps:** + +1. Add RED verifier tests for malformed frame order, invalid local counts, bad continuation IPs, unsupported ownership classes, and snapshot values missing from an exit input list. +2. Extend `SsaExit` with virtual-frame snapshots and include snapshot values in `exit_inputs` deterministically. +3. Reuse inherited-state representation classification for scalar, pointer, tagged borrowed, and tagged owned values. +4. Reject `BoxHeapPtr` transfer in a virtual snapshot, matching current side-entry ownership restrictions. +5. Add a cold bridge entry that: + - restores the physical caller through existing sparse/full exit logic + - validates the callable and prototype + - creates the physical callee frame with the existing callable-entry semantics + - replaces callee stack/locals with the snapshot materializations + - sets `Vm.ip` to the inlined callee exit IP +6. Restore multiple virtual frames outermost to innermost for the future depth-2 extension, even though phase 1 emits one frame. +7. Make validation atomic: reject all metadata before mutating VM stack, locals, capture cells, or frame vectors. + +**Required RED/GREEN cases:** + +- scalar local deopt inside callee +- owned string/array/map local deopt +- borrowed collection value deopt +- dirty callee local plus clean caller local +- deopt after one visible collection mutation +- invalid packet leaves VM state unchanged +- reset after restored frame releases every owner once + +**Focused commands:** + +```bash +cargo test -p pd-vm --lib vm::jit::deopt::tests -- --nocapture +cargo test -p pd-vm --lib vm::native::bridge::tests -- --nocapture +cargo test -p pd-vm --features cranelift-jit --test jit_tests inline_deopt -- --nocapture +``` + +**Expected:** snapshots verify and restore correctly; ordinary exits remain byte-for-byte compatible with zero virtual frames. + +**Commit:** + +```bash +git add src/vm/jit/ir.rs src/vm/jit/deopt.rs src/vm/jit/recorder.rs \ + src/vm/jit/native/lower.rs src/vm/native tests/jit/jit_tests.rs + +git commit -m "feat(jit): restore virtual inline frames on deopt" +``` + +### Task 4: Record Through a Pure Static Leaf Call and Return + +**Objective:** Keep one eligible no-capture leaf call inside the caller SSA trace. + +**Files:** +- Modify: `src/vm/jit/recorder.rs` +- Modify: `src/vm/jit/inline.rs` +- Modify: `src/vm/jit/ir.rs` +- Test: `src/vm/jit/recorder.rs` +- Test: `tests/jit/jit_tests.rs` + +**Steps:** + +1. Add RED tests for a root loop calling a branchless arithmetic leaf. +2. Replace the single symbolic `Frame` with a bounded recorder frame stack. +3. At an eligible `CallValue`: + - pop the callable and arguments from the caller symbolic stack + - create a virtual callee frame + - map arguments to `CallablePrototype::parameter_slots` + - initialize touched non-parameter locals to `Null` + - jump the recorder cursor to `ScriptFunction::entry_ip` + - retain caller `resume_ip` in the virtual frame +4. Record callee instructions into the same SSA graph. +5. At callee `Ret`: + - move the return value to the caller symbolic stack + - emit ownership cleanup for callee temporaries/locals + - restore the caller recorder cursor to `resume_ip` +6. Continue recording to the caller loop backedge. +7. Annotate `op_names` with `inline_call:` and `inline_ret` for diagnostics. +8. Keep old `SsaTerminator::CallValue` behavior for every rejection. + +**Focused commands:** + +```bash +cargo test -p pd-vm --features cranelift-jit --test jit_tests trace_jit_inlines_static_leaf -- --nocapture +cargo test -p pd-vm --features cranelift-jit --test jit_tests trace_jit_inline_guard_exit_restores_callee -- --nocapture +``` + +**Expected:** arithmetic leaf calls show one caller trace containing `inline_call` and no warm-path frame-enter/frame-leave helper. + +**Commit:** + +```bash +git add src/vm/jit tests/jit/jit_tests.rs + +git commit -m "feat(jit): inline static leaf calls in root traces" +``` + +### Task 5: Lower Inline Frames and Preserve Ownership + +**Objective:** Generate native code for inline call/return without physical frame helpers on the normal path. + +**Files:** +- Modify: `src/vm/jit/native/lower.rs` +- Modify: `src/vm/jit/native/mod.rs` +- Modify: `src/vm/native/offsets.rs` +- Modify: `src/vm/native/bridge.rs` +- Test: `src/vm/jit/native/lower.rs` +- Test: `src/vm/jit/native/mod.rs` +- Test: `tests/jit/jit_tests.rs` + +**Steps:** + +1. Add RED native tests that count frame-enter, frame-leave, frame-state, and sparse-restore calls for an inline leaf loop. +2. Lower virtual argument/local values as ordinary SSA values; do not write `Vm.locals` on the normal path. +3. Transfer owned return values from callee to caller and release dead callee owners exactly once. +4. Route only virtual-frame exits to the cold restore bridge from Task 3. +5. Preserve inherited-state direct links after the caller loop backedge. +6. Verify generated code contains no call to: + - `enter_call_value_inherited` + - `leave_frame_inherited` + - `frame_state` + for the normal inline call/return edge. +7. Include inline code size and compile time in native telemetry. + +**Focused commands:** + +```bash +cargo test -p pd-vm --lib vm::jit::native -- --nocapture +cargo test -p pd-vm --features cranelift-jit --test jit_tests inline_ownership -- --nocapture +cargo test -p pd-vm --features cranelift-jit --test jit_tests inline_frame_helpers -- --nocapture +``` + +**Expected:** helper counts are zero on the normal inline edge; drop-contract and deopt tests pass. + +**Commit:** + +```bash +git add src/vm/jit/native src/vm/native tests/jit/jit_tests.rs + +git commit -m "feat(jit): lower virtual leaf call frames" +``` + +### Task 6: Enable the Mutable Collection Leaf Needed by Sort + +**Objective:** Inline the short collection-manipulation function used by the sort workload without changing alias or COW behavior. + +**Files:** +- Modify: `src/vm/jit/inline.rs` +- Modify: `src/vm/jit/recorder.rs` +- Modify: `src/vm/jit/native/lower.rs` +- Test: `tests/jit/jit_tests.rs` +- Test: `tests/jit/perf_tests.rs` + +**Steps:** + +1. Add RED tests for a no-capture `swap`-shaped leaf using array get/set operations. +2. Cover distinct array, shared/COW array, same-index swap, out-of-bounds guard failure, and alias-visible mutation. +3. Ensure every guard before and after the first mutation has a virtual-frame snapshot at the exact callee IP. +4. Verify a deopt after the first mutation does not repeat that mutation when the interpreter resumes. +5. Add a bounded ignored characterization test that reports: + - inline call count + - inline deopt count + - enter/leave helper count + - frame-state count + - sparse-restore count + - native direct links + - trace count + - machine-code bytes +6. Keep the eligibility scanner restricted to existing specialized collection operations; generic host/builtin boundaries still reject. + +**Focused commands:** + +```bash +cargo test -p pd-vm --features cranelift-jit --test jit_tests inline_array_swap -- --nocapture +cargo test -p pd-vm --release --features cranelift-jit --test jit_tests perf_jit_inline_short_leaf -- --ignored --nocapture +``` + +**Expected:** sorted output and COW/drop semantics match the interpreter; the hot swap edge has no physical frame helper on the normal path. + +**Commit:** + +```bash +git add src/vm/jit tests/jit/jit_tests.rs tests/jit/perf_tests.rs + +git commit -m "feat(jit): inline short collection leaf calls" +``` + +### Task 7: Add Guarded Monomorphic Dynamic Calls and Depth 2 + +**Objective:** Extend only after static leaf inlining meets the sort gate. + +**Files:** +- Modify: `src/vm/jit/inline.rs` +- Modify: `src/vm/jit/recorder.rs` +- Modify: `src/vm/jit/ir.rs` +- Modify: `src/vm/jit/native/lower.rs` +- Modify: `src/vm/jit/trace.rs` +- Test: `tests/jit/jit_tests.rs` + +**Steps:** + +1. Gate this task on Task 6 performance evidence. +2. Retain the exact `SharedCallable` target in the trace/profile to prevent pointer reuse. +3. Emit one target-identity guard before entering a dynamic inline frame. +4. Send guard miss to the original caller `CallValue` IP before any callee side effect. +5. Admit only `env == None` in the first dynamic step. +6. Add depth-2 virtual snapshots and reject depth 3. +7. Add recursion and mutual-recursion rejection tests. +8. Add mutable callable rebind and target-change tests; the changed target must execute through the existing call boundary. +9. Add non-root depth-limit guard before permitting later non-root inline support. + +**Focused commands:** + +```bash +cargo test -p pd-vm --features cranelift-jit --test jit_tests inline_monomorphic -- --nocapture +cargo test -p pd-vm --features cranelift-jit --test jit_tests inline_target_change -- --nocapture +cargo test -p pd-vm --features cranelift-jit --test jit_tests inline_depth -- --nocapture +``` + +**Expected:** exact target guards preserve semantics; static call sites remain helper-free. + +**Commit:** + +```bash +git add src/vm/jit tests/jit/jit_tests.rs + +git commit -m "feat(jit): guard monomorphic inline calls" +``` + +### Task 8: Full Verification and Alternating A/B Gate + +**Objective:** Prove correctness across targets and retain the optimization only when structural and wall-clock gates pass. + +**Files:** +- Modify: `plans/pd_vm_jit_short_function_hot_loop_inline_plan.md` with measured results +- Modify only if needed: `tests/jit/perf_tests.rs` + +**Correctness commands:** + +```bash +cargo fmt --all -- --check +RUSTFLAGS="-Dwarnings" cargo clippy --workspace --all-targets --all-features +cargo test --workspace +cargo test -p pd-vm --features cranelift-jit --test jit_tests +cargo test -p pd-vm --release --features cranelift-jit --test jit_tests -- --ignored --nocapture +``` + +Expected JIT baseline before new tests: `128 passed`, `0 failed`, `11 ignored`; the counts increase only by the added inline tests. + +**Cross-platform gate:** + +- Ubuntu CI +- macOS CI +- Windows CI +- strict Rust lint +- VS Code extension checks + +All required checks must pass before merge. + +**Sort A/B setup:** + +- benchmark repository: `/mnt/TEMP/workspace/script-bench-rs` +- benchmark revision: `6fd2a61f0a2164c40f841d585ac7dbae55362b15` +- workload: `Sort Rust objects`, 10,000 objects +- baseline A: crates.io `pd-vm 0.23.1` +- baseline B: pre-inline master `938174e` +- candidate: inline branch +- CPU affinity: CPU 11, with CPU 3 as a reproduction check +- Criterion sample size: 10 +- run order: A/B/candidate/candidate/B/A, then reverse on the next round +- minimum: four valid runs per endpoint + +Run each isolated export with: + +```bash +taskset -c 11 cargo bench --bench rustscript_jit --features pd_vm_jit -- "Sort Rust objects" +``` + +The baseline and candidate exports must have separate target directories and lockfiles. The benchmark repository itself must remain unmodified. + +**Mandatory structural gates:** + +- sorted output validation passes on every run +- eligible hot call sites report `inline_successes > 0` +- normal inline edge reports zero frame-enter/frame-leave helper calls +- virtual-frame restore occurs only on cold exits +- helper calls attributable to the short script call fall by at least 10x +- interpreter fallback remains zero on the measured native path +- no growth in owned-value drop mismatches + +**Mandatory performance gates:** + +- sort candidate median is at least 20% below pre-inline master median on the same CPU +- sort candidate median is at most `1.25x` crates.io `0.23.1` +- non-call tight-loop, array-builtin, map-builtin, AES, and IFFT JIT medians regress by no more than 5% after alternating reruns +- JIT compile time grows by no more than 20% for the sort program +- installed native code size grows by no more than 30% + +**Stretch gate:** sort candidate at or below `1.10x` crates.io `0.23.1`. + +If the mandatory sort gate misses, stop feature widening. Record call-site profile, inline/deopt counts, helper counts, trace/code size, and compile time, then identify the largest remaining cost before enabling captures, PICs, deeper inline, or larger callees. + +**Final plan update and commit:** + +```bash +git add plans/pd_vm_jit_short_function_hot_loop_inline_plan.md tests/jit/perf_tests.rs + +git commit -m "docs(plan): record short-call inline gates" +``` + +## 6. Test Matrix + +| Case | Inline? | Required result | +|---|---:|---| +| root loop → no-capture arithmetic leaf | yes | one caller trace, exact result | +| root loop → no-capture array swap leaf | yes after Task 6 | exact COW and mutation semantics | +| root loop → captured closure | no in phase 1 | existing `CallValue` path | +| root loop → yielding host call leaf | no | existing wait/yield behavior | +| root loop → recursive leaf | no | depth semantics unchanged | +| target changes after trace install | guard miss / fallback | new target executes | +| out-of-bounds inside inline callee | deopt | exact callee IP and error | +| fuel expires at inline boundary | deopt/yield | no replayed side effect | +| epoch deadline expires | deopt/yield | resumable exact state | +| reset after inline execution | yes | profiles/traces cleared, owners released once | +| JIT invalidation | yes | inline target references released | +| branchless callee exceeds 32 ops | no | reject reason recorded | +| touched locals exceed 8 | no | reject reason recorded | +| nested eligible leaf at depth 2 | later yes | two virtual snapshots restore correctly | +| nested depth 3 | no | existing physical call path | + +## 7. Rollback Boundaries + +Each stage is independently removable: + +1. profiling has no execution effect +2. eligibility analysis has no execution effect +3. zero-frame exits preserve existing deopt encoding +4. disabling inline admission restores `SsaTerminator::CallValue` +5. cold virtual restore is unreachable when inline admission is off +6. mutable collection admission has its own eligibility switch +7. dynamic target guards and depth 2 remain separately gated + +Keep one internal configuration constant for emergency disablement during development. Remove it only after all performance and CI gates pass. + +## 8. Definition of Done + +- phase-1 eligible leaf calls remain in the caller trace through `Ret` +- warm execution performs no physical frame-enter/frame-leave helper for the inline edge +- every guard, error, fuel exit, epoch exit, and invalidation path restores exact VM state +- static function-item, mutable collection, target-change, depth, ownership, and reset tests pass +- strict clippy, workspace tests, focused JIT tests, ignored perf suite, and cross-platform CI pass +- alternating sort A/B meets all mandatory structural and performance gates +- measured results and any retained scope limits are written back into this plan From 4455e4efc7ddbac16aa037baaac6b4006c5db646 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sun, 19 Jul 2026 17:36:20 +0800 Subject: [PATCH 02/26] feat(jit): profile script call targets --- src/lib.rs | 8 +-- src/vm/jit/inline.rs | 92 +++++++++++++++++++++++++++++ src/vm/jit/mod.rs | 2 + src/vm/jit/runtime.rs | 7 +++ src/vm/jit/trace.rs | 128 ++++++++++++++++++++++++++++++++++++++++ src/vm/mod.rs | 19 +++++- src/vm/native/bridge.rs | 2 +- tests/jit/jit_tests.rs | 36 +++++++++++ 8 files changed, 286 insertions(+), 8 deletions(-) create mode 100644 src/vm/jit/inline.rs diff --git a/src/lib.rs b/src/lib.rs index 833ad637..fbdfcff6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,8 +9,8 @@ pub mod debugger; #[cfg(feature = "runtime")] pub mod jit { pub use crate::vm::jit::{ - JitAttempt, JitConfig, JitExitProfile, JitMetrics, JitNyiDoc, JitNyiReason, JitSnapshot, - JitTrace, JitTraceTerminal, TraceJitEngine, + JitAttempt, JitCallSiteProfile, JitConfig, JitExitProfile, JitMetrics, JitNyiDoc, + JitNyiReason, JitSnapshot, JitTrace, JitTraceTerminal, TraceJitEngine, }; } #[cfg(feature = "cli")] @@ -81,8 +81,8 @@ pub use debugger::{ }; #[cfg(feature = "runtime")] pub use jit::{ - JitAttempt, JitConfig, JitExitProfile, JitMetrics, JitNyiDoc, JitNyiReason, JitSnapshot, - JitTrace, JitTraceTerminal, TraceJitEngine, + JitAttempt, JitCallSiteProfile, JitConfig, JitExitProfile, JitMetrics, JitNyiDoc, JitNyiReason, + JitSnapshot, JitTrace, JitTraceTerminal, TraceJitEngine, }; #[cfg(feature = "runtime")] pub use vm::diagnostics::render_vm_error; diff --git a/src/vm/jit/inline.rs b/src/vm/jit/inline.rs new file mode 100644 index 00000000..93e38857 --- /dev/null +++ b/src/vm/jit/inline.rs @@ -0,0 +1,92 @@ +use std::collections::HashMap; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub(crate) struct CallSiteKey { + pub(crate) caller_frame_key: u64, + pub(crate) call_ip: usize, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct CallSiteProfile { + prototype_id: u32, + observations: u64, + mismatches: u64, + monomorphic: bool, +} + +impl CallSiteProfile { + fn new(prototype_id: u32) -> Self { + Self { + prototype_id, + observations: 1, + mismatches: 0, + monomorphic: true, + } + } + + fn observe(&mut self, prototype_id: u32) { + self.observations = self.observations.saturating_add(1); + if prototype_id != self.prototype_id { + self.mismatches = self.mismatches.saturating_add(1); + self.monomorphic = false; + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct JitCallSiteProfile { + pub caller_frame_key: u64, + pub call_ip: usize, + pub prototype_id: u32, + pub observations: u64, + pub mismatches: u64, + pub monomorphic: bool, +} + +pub(crate) fn observe_script_call_target( + profiles: &mut HashMap, + key: CallSiteKey, + prototype_id: u32, +) { + match profiles.entry(key) { + std::collections::hash_map::Entry::Occupied(mut entry) => { + entry.get_mut().observe(prototype_id); + } + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(CallSiteProfile::new(prototype_id)); + } + } +} + +pub(crate) fn call_site_profiles( + profiles: &HashMap, +) -> Vec { + let mut snapshot = profiles + .iter() + .map(|(key, profile)| JitCallSiteProfile { + caller_frame_key: key.caller_frame_key, + call_ip: key.call_ip, + prototype_id: profile.prototype_id, + observations: profile.observations, + mismatches: profile.mismatches, + monomorphic: profile.monomorphic, + }) + .collect::>(); + snapshot.sort_unstable_by_key(|profile| (profile.caller_frame_key, profile.call_ip)); + snapshot +} + +pub(crate) fn call_site_metric_summary( + profiles: &HashMap, +) -> (u64, u64, u64) { + profiles.values().fold( + (0u64, 0u64, 0u64), + |(observations, monomorphic, polymorphic), profile| { + ( + observations.saturating_add(profile.observations), + monomorphic.saturating_add(u64::from(profile.monomorphic)), + polymorphic.saturating_add(u64::from(!profile.monomorphic)), + ) + }, + ) +} diff --git a/src/vm/jit/mod.rs b/src/vm/jit/mod.rs index bbae02a7..0afe9be2 100644 --- a/src/vm/jit/mod.rs +++ b/src/vm/jit/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod deopt; +pub(crate) mod inline; pub(crate) mod ir; pub(crate) mod liveness; pub(crate) mod native; @@ -7,6 +8,7 @@ pub(crate) mod region; pub(crate) mod runtime; pub(crate) mod trace; +pub use inline::JitCallSiteProfile; pub(crate) use runtime::NativeTrace; pub use trace::{ JitAttempt, JitConfig, JitExitProfile, JitMetrics, JitNyiDoc, JitNyiReason, JitSnapshot, diff --git a/src/vm/jit/runtime.rs b/src/vm/jit/runtime.rs index d8c03970..6152094c 100644 --- a/src/vm/jit/runtime.rs +++ b/src/vm/jit/runtime.rs @@ -783,6 +783,10 @@ impl Vm { self.jit.exit_profiles() } + pub fn jit_call_site_profiles(&self) -> Vec { + self.jit.call_site_profiles() + } + pub fn jit_native_code_bytes(&self) -> usize { self.native_traces .iter() @@ -1634,6 +1638,9 @@ impl Vm { native_loop_back_count: self.jit_native_loop_back_count, helper_fallback_count: self.jit_helper_fallback_count, native_trace_exec_count: self.native_trace_exec_count, + script_call_observations: 0, + monomorphic_call_sites: 0, + polymorphic_call_sites: 0, } } diff --git a/src/vm/jit/trace.rs b/src/vm/jit/trace.rs index 31b02902..66a2528a 100644 --- a/src/vm/jit/trace.rs +++ b/src/vm/jit/trace.rs @@ -5,6 +5,10 @@ use crate::vm::native::ROOT_FRAME_KEY; use crate::vm::{OpCode, Program}; use super::deopt::{SideTraceImport, SideTraceImportError}; +use super::inline::{ + CallSiteKey, CallSiteProfile, JitCallSiteProfile, call_site_metric_summary, call_site_profiles, + observe_script_call_target, +}; use super::ir::{SsaExitId, SsaMaterialization, SsaTrace, SsaValueId, SsaValueRepr}; use super::liveness::{boxed_load_site_count, boxed_store_site_count}; use super::recorder::{RecordedTrace, TraceRecordError, record_trace_with_local_count}; @@ -253,6 +257,9 @@ pub struct JitMetrics { pub native_loop_back_count: u64, pub helper_fallback_count: u64, pub native_trace_exec_count: u64, + pub script_call_observations: u64, + pub monomorphic_call_sites: u64, + pub polymorphic_call_sites: u64, } impl JitMetrics { @@ -298,6 +305,7 @@ pub struct TraceJitEngine { non_yielding_host_imports: Vec, traces: Vec, trace_exit_profiles: HashMap, + call_site_profiles: HashMap, region_generation: u64, published_region: Option, failed_region_exits: HashSet, @@ -323,6 +331,7 @@ impl TraceJitEngine { non_yielding_host_imports: Vec::new(), traces: Vec::new(), trace_exit_profiles: HashMap::new(), + call_site_profiles: HashMap::new(), region_generation: 0, published_region: None, failed_region_exits: HashSet::new(), @@ -336,6 +345,30 @@ impl TraceJitEngine { &self.config } + pub(crate) fn observe_script_call_target( + &mut self, + caller_frame_key: u64, + call_ip: usize, + prototype_id: u32, + ) { + observe_script_call_target( + &mut self.call_site_profiles, + CallSiteKey { + caller_frame_key, + call_ip, + }, + prototype_id, + ); + } + + pub fn call_site_profiles(&self) -> Vec { + call_site_profiles(&self.call_site_profiles) + } + + pub(crate) fn clear_call_site_profiles(&mut self) { + self.call_site_profiles.clear(); + } + pub fn set_config(&mut self, config: JitConfig) { self.invalidate_regions(); self.config = config; @@ -345,6 +378,7 @@ impl TraceJitEngine { self.loop_headers = None; self.traces.clear(); self.trace_exit_profiles.clear(); + self.call_site_profiles.clear(); self.callable_side_exit_streaks.clear(); self.blocked_callable_frames.clear(); self.attempts.clear(); @@ -362,6 +396,7 @@ impl TraceJitEngine { self.loop_headers = None; self.traces.clear(); self.trace_exit_profiles.clear(); + self.call_site_profiles.clear(); self.callable_side_exit_streaks.clear(); self.blocked_callable_frames.clear(); self.attempts.clear(); @@ -830,6 +865,12 @@ impl TraceJitEngine { self.trace_exit_profiles.len() )); out.push_str(&format!(" compile attempts: {}\n", self.attempts.len())); + out.push_str(&format!( + " script calls: observations={} monomorphic_sites={} polymorphic_sites={}\n", + metrics.script_call_observations, + metrics.monomorphic_call_sites, + metrics.polymorphic_call_sites + )); out.push_str(&format!( " boxed value sites: loads={} stores={}\n", metrics.boxed_load_site_count, metrics.boxed_store_site_count @@ -1032,6 +1073,11 @@ impl TraceJitEngine { } fn aggregate_metrics(&self, mut runtime_metrics: JitMetrics) -> JitMetrics { + let (observations, monomorphic, polymorphic) = + call_site_metric_summary(&self.call_site_profiles); + runtime_metrics.script_call_observations = observations; + runtime_metrics.monomorphic_call_sites = monomorphic; + runtime_metrics.polymorphic_call_sites = polymorphic; for trace in &self.traces { runtime_metrics.boxed_load_site_count = runtime_metrics .boxed_load_site_count @@ -1179,6 +1225,88 @@ mod tests { ValueType, }; + #[test] + fn call_site_profiles_distinguish_caller_frames_and_target_changes() { + let mut engine = TraceJitEngine::new(JitConfig::default()); + + engine.observe_script_call_target(7, 41, 2); + engine.observe_script_call_target(7, 41, 2); + engine.observe_script_call_target(8, 41, 3); + engine.observe_script_call_target(7, 41, 4); + + assert_eq!( + engine.call_site_profiles(), + vec![ + JitCallSiteProfile { + caller_frame_key: 7, + call_ip: 41, + prototype_id: 2, + observations: 3, + mismatches: 1, + monomorphic: false, + }, + JitCallSiteProfile { + caller_frame_key: 8, + call_ip: 41, + prototype_id: 3, + observations: 1, + mismatches: 0, + monomorphic: true, + }, + ] + ); + } + + #[test] + fn call_site_profiles_clear_when_engine_config_changes() { + let mut engine = TraceJitEngine::new(JitConfig::default()); + engine.observe_script_call_target(7, 41, 2); + + engine.set_config(JitConfig { + enabled: false, + ..JitConfig::default() + }); + + assert!(engine.call_site_profiles().is_empty()); + } + + #[test] + fn call_site_profiles_clear_when_host_import_capabilities_change() { + let mut engine = TraceJitEngine::new(JitConfig::default()); + engine.observe_script_call_target(7, 41, 2); + + assert!(engine.set_non_yielding_host_imports(vec![true])); + + assert!(engine.call_site_profiles().is_empty()); + } + + #[test] + fn call_site_metrics_count_observations_and_site_classes() { + let mut engine = TraceJitEngine::new(JitConfig::default()); + engine.observe_script_call_target(7, 41, 2); + engine.observe_script_call_target(7, 41, 2); + engine.observe_script_call_target(8, 41, 3); + engine.observe_script_call_target(7, 41, 4); + + let metrics = engine.snapshot(JitMetrics::default()).metrics; + assert_eq!(metrics.script_call_observations, 4); + assert_eq!(metrics.monomorphic_call_sites, 1); + assert_eq!(metrics.polymorphic_call_sites, 1); + } + + #[test] + fn jit_dump_reports_call_site_profile_metrics() { + let mut engine = TraceJitEngine::new(JitConfig::default()); + engine.observe_script_call_target(7, 41, 2); + engine.observe_script_call_target(7, 41, 4); + + let dump = engine.dump_text(None, JitMetrics::default()); + assert!( + dump.contains("script calls: observations=2 monomorphic_sites=0 polymorphic_sites=1"), + "dump:\n{dump}" + ); + } + fn test_trace(id: usize, frame_key: u64, has_yielding_call: bool, ssa: SsaTrace) -> JitTrace { JitTrace { id, diff --git a/src/vm/mod.rs b/src/vm/mod.rs index 07135f9a..9f368960 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -929,6 +929,7 @@ impl Vm { self.io_state = crate::builtins::runtime::IoState::default(); self.map_iterators.clear(); self.jit.reset_runtime_backoff(); + self.jit.clear_call_site_profiles(); self.clear_interpreter_metrics(); } @@ -1238,7 +1239,11 @@ impl Vm { Ok(Value::Callable(callable)) } - fn execute_call_value(&mut self, argc: u8) -> VmResult { + fn execute_call_value( + &mut self, + argc: u8, + call_site_ip: Option, + ) -> VmResult { let operand_count = argc as usize + 1; if self.stack.len() < operand_count { return Err(VmError::StackUnderflow); @@ -1274,6 +1279,13 @@ impl Vm { match prototype.target { CallableTarget::ScriptFunction(function_id) => { + if let Some(call_ip) = call_site_ip { + self.jit.observe_script_call_target( + self.active_frame_key(), + call_ip, + callable.prototype_id, + ); + } if self.call_depth >= self.max_script_call_depth { return Err(VmError::CallStackOverflow { limit: self.max_script_call_depth, @@ -2578,8 +2590,9 @@ impl Vm { } x if x == OpCode::CallValue as u8 => { + let call_ip = self.ip.saturating_sub(1); let argc = self.read_u8()?; - return self.execute_call_value(argc); + return self.execute_call_value(argc, Some(call_ip)); } other => return Err(VmError::InvalidOpcode(other)), } @@ -2804,7 +2817,7 @@ impl Vm { self.stack.push(callable); self.stack.extend_from_slice(args); self.host_return = None; - let outcome = match self.execute_call_value(argc) { + let outcome = match self.execute_call_value(argc, None) { Ok(outcome) => outcome, Err(error) => { self.abort_host_invocation(stack_base, frame_count); diff --git a/src/vm/native/bridge.rs b/src/vm/native/bridge.rs index 7be6bd25..effd7c0f 100644 --- a/src/vm/native/bridge.rs +++ b/src/vm/native/bridge.rs @@ -929,7 +929,7 @@ fn native_enter_call_value( return Err(VmError::BytecodeBounds); } vm.ip = resume_ip; - let status = match vm.execute_call_value(argc)? { + let status = match vm.execute_call_value(argc, Some(call_ip))? { ExecOutcome::Continue => STATUS_LINKED_CONTINUE, ExecOutcome::Halted => STATUS_HALTED, ExecOutcome::Yielded => STATUS_YIELDED, diff --git a/tests/jit/jit_tests.rs b/tests/jit/jit_tests.rs index 9ea4b236..d24cbd2d 100644 --- a/tests/jit/jit_tests.rs +++ b/tests/jit/jit_tests.rs @@ -6298,6 +6298,13 @@ fn trace_jit_executes_call_value_natively_inside_loop() { ); assert_eq!(vm.stack(), &[Value::Int(48)]); let snapshot = vm.jit_snapshot(); + assert_eq!(snapshot.metrics.script_call_observations, 16); + assert_eq!(snapshot.metrics.monomorphic_call_sites, 1); + assert_eq!(snapshot.metrics.polymorphic_call_sites, 0); + let call_sites = vm.jit_call_site_profiles(); + assert_eq!(call_sites.len(), 1, "{}", vm.dump_jit_info()); + assert_eq!(call_sites[0].observations, 16); + assert!(call_sites[0].monomorphic); assert!( snapshot.traces.iter().any(|trace| { trace.terminal == JitTraceTerminal::CallValue @@ -6315,6 +6322,35 @@ fn trace_jit_executes_call_value_natively_inside_loop() { assert!(vm.dump_jit_info().contains("interpreter fallbacks: 0")); } +#[test] +fn trace_jit_call_site_profiles_clear_on_vm_reuse() { + let source = r#" + fn add_one(value: int) -> int { value + 1 } + let mut i = 0; + while i < 3 { + i = add_one(i); + } + i; + "#; + let compiled = compile_source(source).expect("call-site profile source should compile"); + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_jit_config(JitConfig { + enabled: false, + hot_loop_threshold: 64, + max_trace_len: 512, + }); + + assert_eq!(vm.run().expect("first profile run"), VmStatus::Halted); + assert_eq!(vm.jit_snapshot().metrics.script_call_observations, 3); + + vm.reset_for_reuse(); + + assert_eq!(vm.jit_snapshot().metrics.script_call_observations, 0); + assert!(vm.jit_call_site_profiles().is_empty()); + assert_eq!(vm.run().expect("second profile run"), VmStatus::Halted); + assert_eq!(vm.jit_snapshot().metrics.script_call_observations, 3); +} + #[test] fn trace_jit_call_value_waits_and_resumes_host_callable_without_replay() { if !native_jit_supported() { From 080d81886a9b7467376e80028a10e21faf4fce85 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sun, 19 Jul 2026 18:10:22 +0800 Subject: [PATCH 03/26] feat(jit): classify inline script callees --- src/vm/jit/inline.rs | 379 +++++++++++++++++++++++++++++++++++++++++ src/vm/jit/recorder.rs | 15 ++ src/vm/jit/trace.rs | 1 + 3 files changed, 395 insertions(+) diff --git a/src/vm/jit/inline.rs b/src/vm/jit/inline.rs index 93e38857..9d50ed70 100644 --- a/src/vm/jit/inline.rs +++ b/src/vm/jit/inline.rs @@ -1,5 +1,229 @@ use std::collections::HashMap; +use crate::builtins::BuiltinFunction; +use crate::vm::native::ROOT_FRAME_KEY; +use crate::{CallableKind, CallableTarget, OpCode, Program}; + +pub(crate) const MAX_INLINE_INSTRUCTIONS: usize = 32; +pub(crate) const MAX_INLINE_TOUCHED_LOCALS: usize = 8; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub(crate) enum InlineRejectReason { + NonRootCaller, + UnknownTarget, + #[allow(dead_code)] + PolymorphicTarget, + HostTarget, + CapturedCallable, + ArityMismatch, + Recursive, + NestedScriptCall, + YieldingCall, + BackwardBranch, + MultipleReturns, + TooManyInstructions, + TooManyTouchedLocals, + TraceBudgetExceeded, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct InlineCandidate { + pub(crate) prototype_id: u32, + pub(crate) entry_ip: usize, + pub(crate) end_ip: usize, + pub(crate) parameter_slots: Vec, + pub(crate) touched_locals: Vec, + pub(crate) decoded_instruction_count: usize, +} + +pub(crate) fn classify_static_inline_candidate( + program: &Program, + caller_frame_key: u64, + caller_prototype_id: Option, + source_local: Option, + argc: u8, + remaining_trace_budget: usize, +) -> Result { + if caller_frame_key != ROOT_FRAME_KEY { + return Err(InlineRejectReason::NonRootCaller); + } + let source_local = source_local.ok_or(InlineRejectReason::UnknownTarget)?; + let binding = program + .root_callable_bindings + .iter() + .find(|binding| binding.local_slot == u16::from(source_local)) + .ok_or(InlineRejectReason::UnknownTarget)?; + if caller_prototype_id == Some(binding.prototype_id) { + return Err(InlineRejectReason::Recursive); + } + let prototype = program + .callable_prototypes + .get(binding.prototype_id as usize) + .ok_or(InlineRejectReason::UnknownTarget)?; + if prototype.kind != CallableKind::FunctionItem + || !prototype.capture_slots.is_empty() + || !prototype.capture_source_slots.is_empty() + || !prototype.capture_modes.is_empty() + { + return Err(InlineRejectReason::CapturedCallable); + } + if prototype.arity != argc || prototype.parameter_slots.len() != usize::from(argc) { + return Err(InlineRejectReason::ArityMismatch); + } + let CallableTarget::ScriptFunction(function_id) = prototype.target else { + return Err(InlineRejectReason::HostTarget); + }; + let function = program + .script_functions + .get(function_id as usize) + .ok_or(InlineRejectReason::UnknownTarget)?; + let entry_ip = function.entry_ip as usize; + let end_ip = function.end_ip as usize; + if entry_ip >= end_ip || end_ip > program.code.len() { + return Err(InlineRejectReason::UnknownTarget); + } + + let (decoded_instruction_count, touched_locals) = + scan_inline_region(program, entry_ip, end_ip)?; + if decoded_instruction_count > remaining_trace_budget { + return Err(InlineRejectReason::TraceBudgetExceeded); + } + Ok(InlineCandidate { + prototype_id: binding.prototype_id, + entry_ip, + end_ip, + parameter_slots: prototype.parameter_slots.clone(), + touched_locals, + decoded_instruction_count, + }) +} + +fn scan_inline_region( + program: &Program, + entry_ip: usize, + end_ip: usize, +) -> Result<(usize, Vec), InlineRejectReason> { + let mut ip = entry_ip; + let mut instruction_count = 0usize; + let mut return_count = 0usize; + let mut touched_locals = Vec::::new(); + + while ip < end_ip { + let instruction_ip = ip; + let opcode = OpCode::try_from( + *program + .code + .get(ip) + .ok_or(InlineRejectReason::UnknownTarget)?, + ) + .map_err(|_| InlineRejectReason::UnknownTarget)?; + ip = ip.saturating_add(1); + instruction_count = instruction_count.saturating_add(1); + if instruction_count > MAX_INLINE_INSTRUCTIONS { + return Err(InlineRejectReason::TooManyInstructions); + } + + match opcode { + OpCode::Ret => { + return_count = return_count.saturating_add(1); + if return_count > 1 || ip != end_ip { + return Err(InlineRejectReason::MultipleReturns); + } + } + OpCode::Ldloc | OpCode::Stloc => { + let local = *program + .code + .get(ip) + .ok_or(InlineRejectReason::UnknownTarget)?; + ip = ip.saturating_add(1); + let local = u16::from(local); + if !touched_locals.contains(&local) { + touched_locals.push(local); + if touched_locals.len() > MAX_INLINE_TOUCHED_LOCALS { + return Err(InlineRejectReason::TooManyTouchedLocals); + } + } + } + OpCode::Ldc => { + let index = + read_u32(&program.code, &mut ip).ok_or(InlineRejectReason::UnknownTarget)?; + if program.constants.get(index as usize).is_none() { + return Err(InlineRejectReason::UnknownTarget); + } + } + OpCode::Br | OpCode::Brfalse => { + let target = read_u32(&program.code, &mut ip) + .ok_or(InlineRejectReason::UnknownTarget)? + as usize; + if target <= instruction_ip { + return Err(InlineRejectReason::BackwardBranch); + } + if target < entry_ip || target >= end_ip { + return Err(InlineRejectReason::UnknownTarget); + } + } + OpCode::CallValue => return Err(InlineRejectReason::NestedScriptCall), + OpCode::Call => { + let index = + read_u16(&program.code, &mut ip).ok_or(InlineRejectReason::UnknownTarget)?; + let argc = + read_u8(&program.code, &mut ip).ok_or(InlineRejectReason::UnknownTarget)?; + let Some(builtin) = BuiltinFunction::from_call_index(index) else { + return Err(InlineRejectReason::YieldingCall); + }; + if !builtin.accepts_arity(argc) { + return Err(InlineRejectReason::UnknownTarget); + } + } + OpCode::Nop + | OpCode::Add + | OpCode::Sub + | OpCode::Mul + | OpCode::Div + | OpCode::Neg + | OpCode::Ceq + | OpCode::Clt + | OpCode::Cgt + | OpCode::Pop + | OpCode::Dup + | OpCode::Shl + | OpCode::Shr + | OpCode::Mod + | OpCode::And + | OpCode::Or + | OpCode::Not + | OpCode::Lshr => {} + } + if ip > end_ip { + return Err(InlineRejectReason::UnknownTarget); + } + } + + if return_count != 1 { + return Err(InlineRejectReason::MultipleReturns); + } + touched_locals.sort_unstable(); + Ok((instruction_count, touched_locals)) +} + +fn read_u8(code: &[u8], ip: &mut usize) -> Option { + let value = *code.get(*ip)?; + *ip = ip.saturating_add(1); + Some(value) +} + +fn read_u16(code: &[u8], ip: &mut usize) -> Option { + let bytes = code.get(*ip..ip.saturating_add(2))?; + *ip = ip.saturating_add(2); + Some(u16::from_le_bytes(bytes.try_into().ok()?)) +} + +fn read_u32(code: &[u8], ip: &mut usize) -> Option { + let bytes = code.get(*ip..ip.saturating_add(4))?; + *ip = ip.saturating_add(4); + Some(u32::from_le_bytes(bytes.try_into().ok()?)) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub(crate) struct CallSiteKey { pub(crate) caller_frame_key: u64, @@ -90,3 +314,158 @@ pub(crate) fn call_site_metric_summary( }, ) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{CallablePrototype, FunctionRegion, RootCallableBinding, ScriptFunction}; + + fn function_item_program(code: Vec) -> Program { + Program::new(Vec::new(), code.clone()).with_callable_metadata( + vec![ScriptFunction { + entry_ip: 0, + end_ip: code.len() as u32, + }], + vec![CallablePrototype { + kind: CallableKind::FunctionItem, + target: CallableTarget::ScriptFunction(0), + arity: 2, + frame_local_count: 2, + parameter_slots: vec![0, 1], + capture_source_slots: Vec::new(), + capture_slots: Vec::new(), + capture_modes: Vec::new(), + self_slot: None, + schema: None, + }], + vec![FunctionRegion { + start_ip: 0, + end_ip: code.len() as u32, + prototype_id: Some(0), + }], + vec![RootCallableBinding { + local_slot: 0, + prototype_id: 0, + }], + ) + } + + fn classify(program: &Program) -> Result { + classify_static_inline_candidate(program, ROOT_FRAME_KEY, None, Some(0), 2, 64) + } + + fn arithmetic_leaf_code() -> Vec { + vec![ + OpCode::Ldloc as u8, + 0, + OpCode::Ldloc as u8, + 1, + OpCode::Add as u8, + OpCode::Ret as u8, + ] + } + + #[test] + fn inline_eligibility_accepts_short_static_arithmetic_leaf() { + let candidate = classify(&function_item_program(arithmetic_leaf_code())).unwrap(); + assert_eq!(candidate.prototype_id, 0); + assert_eq!(candidate.entry_ip, 0); + assert_eq!(candidate.parameter_slots, vec![0, 1]); + assert_eq!(candidate.touched_locals, vec![0, 1]); + assert_eq!(candidate.decoded_instruction_count, 4); + } + + #[test] + fn inline_eligibility_rejects_unsupported_metadata_and_call_shapes() { + let base = function_item_program(arithmetic_leaf_code()); + assert_eq!( + classify_static_inline_candidate(&base, 0, None, Some(0), 2, 64), + Err(InlineRejectReason::NonRootCaller) + ); + assert_eq!( + classify_static_inline_candidate(&base, ROOT_FRAME_KEY, None, None, 2, 64), + Err(InlineRejectReason::UnknownTarget) + ); + assert_eq!( + classify_static_inline_candidate(&base, ROOT_FRAME_KEY, None, Some(0), 1, 64), + Err(InlineRejectReason::ArityMismatch) + ); + assert_eq!( + classify_static_inline_candidate(&base, ROOT_FRAME_KEY, Some(0), Some(0), 2, 64), + Err(InlineRejectReason::Recursive) + ); + + let mut host = base.clone(); + host.callable_prototypes[0].target = CallableTarget::HostImport(0); + assert_eq!(classify(&host), Err(InlineRejectReason::HostTarget)); + + let mut captured = base; + captured.callable_prototypes[0].kind = CallableKind::Closure; + captured.callable_prototypes[0].capture_slots.push(1); + assert_eq!( + classify(&captured), + Err(InlineRejectReason::CapturedCallable) + ); + } + + #[test] + fn inline_eligibility_rejects_unsafe_regions_and_hard_limits() { + let nested = function_item_program(vec![OpCode::CallValue as u8, 0, OpCode::Ret as u8]); + assert_eq!(classify(&nested), Err(InlineRejectReason::NestedScriptCall)); + + let yielding = function_item_program(vec![OpCode::Call as u8, 0, 0, 0, OpCode::Ret as u8]); + assert_eq!(classify(&yielding), Err(InlineRejectReason::YieldingCall)); + + let backward = function_item_program(vec![ + OpCode::Nop as u8, + OpCode::Br as u8, + 0, + 0, + 0, + 0, + OpCode::Ret as u8, + ]); + assert_eq!(classify(&backward), Err(InlineRejectReason::BackwardBranch)); + + let multiple_returns = function_item_program(vec![OpCode::Ret as u8, OpCode::Ret as u8]); + assert_eq!( + classify(&multiple_returns), + Err(InlineRejectReason::MultipleReturns) + ); + + let mut long = vec![OpCode::Nop as u8; MAX_INLINE_INSTRUCTIONS]; + long.push(OpCode::Ret as u8); + assert_eq!( + classify(&function_item_program(long)), + Err(InlineRejectReason::TooManyInstructions) + ); + + let mut locals = Vec::new(); + for local in 0..=MAX_INLINE_TOUCHED_LOCALS as u8 { + locals.extend([OpCode::Ldloc as u8, local, OpCode::Pop as u8]); + } + locals.push(OpCode::Ret as u8); + assert_eq!( + classify(&function_item_program(locals)), + Err(InlineRejectReason::TooManyTouchedLocals) + ); + + let base = function_item_program(arithmetic_leaf_code()); + assert_eq!( + classify_static_inline_candidate(&base, ROOT_FRAME_KEY, None, Some(0), 2, 3), + Err(InlineRejectReason::TraceBudgetExceeded) + ); + } + + #[test] + fn inline_eligibility_uses_touched_slots_for_merged_frames() { + let mut program = function_item_program(arithmetic_leaf_code()); + program.callable_prototypes[0].frame_local_count = 200; + let candidate = classify(&program).unwrap(); + assert_eq!(candidate.touched_locals, vec![0, 1]); + assert_ne!( + program.callable_prototypes[0].frame_local_count, + candidate.touched_locals.len() + ); + } +} diff --git a/src/vm/jit/recorder.rs b/src/vm/jit/recorder.rs index ce1dae97..1d2416f2 100644 --- a/src/vm/jit/recorder.rs +++ b/src/vm/jit/recorder.rs @@ -5,6 +5,7 @@ use crate::vm::{OpCode, Program, Value, ValueType, checked_int_div}; use super::JitTraceTerminal; use super::deopt::materialize_ssa_values; +use super::inline::classify_static_inline_candidate; use super::ir::{ SsaBranchTarget, SsaInstKind, SsaMaterialization, SsaTerminator, SsaTrace, SsaTraceBuilder, SsaValue, SsaValueRepr, @@ -721,6 +722,7 @@ pub(crate) fn record_trace( ) -> Result { record_trace_with_local_count( program, + crate::vm::native::ROOT_FRAME_KEY, root_ip, entry_stack_depth, program.local_count, @@ -730,8 +732,10 @@ pub(crate) fn record_trace( ) } +#[allow(clippy::too_many_arguments)] pub(crate) fn record_trace_with_local_count( program: &Program, + caller_frame_key: u64, root_ip: usize, entry_stack_depth: usize, local_count: usize, @@ -1197,6 +1201,17 @@ pub(crate) fn record_trace_with_local_count( if frame.stack.len() < usize::from(argc) + 1 { return Err(TraceRecordError::StackUnderflow); } + let callable = frame.stack[frame.stack.len() - usize::from(argc) - 1]; + let caller_prototype_id = (caller_frame_key != crate::vm::native::ROOT_FRAME_KEY) + .then_some(caller_frame_key as u32); + let _ = classify_static_inline_candidate( + program, + caller_frame_key, + caller_prototype_id, + callable.info.source_local, + argc, + max_trace_len.saturating_sub(cursor.recorded_ops), + ); op_names.push("call_value".to_string()); let exit = builder.add_exit( ip, diff --git a/src/vm/jit/trace.rs b/src/vm/jit/trace.rs index 66a2528a..08742d85 100644 --- a/src/vm/jit/trace.rs +++ b/src/vm/jit/trace.rs @@ -1005,6 +1005,7 @@ impl TraceJitEngine { }; let recorded = record_trace_with_local_count( program, + key.frame_key, key.root_ip, key.stack_depth, local_count, From 2039f7541fffd42f53f03386921bcbcec1a55893 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sun, 19 Jul 2026 18:22:43 +0800 Subject: [PATCH 04/26] feat(jit): restore virtual inline frames on deopt --- src/vm/jit/deopt.rs | 62 ++++++++++++ src/vm/jit/ir.rs | 198 ++++++++++++++++++++++++++++++++++++++ src/vm/native/bridge.rs | 203 ++++++++++++++++++++++++++++++++++++++- src/vm/native/codegen.rs | 16 +++ src/vm/native/mod.rs | 21 ++-- 5 files changed, 487 insertions(+), 13 deletions(-) diff --git a/src/vm/jit/deopt.rs b/src/vm/jit/deopt.rs index d28cd88b..2c8a3a98 100644 --- a/src/vm/jit/deopt.rs +++ b/src/vm/jit/deopt.rs @@ -41,6 +41,25 @@ pub(crate) fn exit_inputs(exit: &SsaExit) -> Vec { out.push(value); } } + for frame in &exit.virtual_frames { + let dirty_locals = frame + .locals + .iter() + .zip(&frame.dirty_locals) + .filter_map(|(materialization, dirty)| dirty.then_some(materialization)); + for materialization in frame.operand_stack.iter().chain(dirty_locals) { + let value = match materialization { + SsaMaterialization::Value(value) + | SsaMaterialization::BoxInt(value) + | SsaMaterialization::BoxFloat(value) + | SsaMaterialization::BoxBool(value) => *value, + SsaMaterialization::BoxHeapPtr { value, .. } => *value, + }; + if !out.contains(&value) { + out.push(value); + } + } + } out } @@ -250,4 +269,47 @@ mod tests { }) ); } + + #[test] + fn exit_inputs_include_virtual_frame_values_once_in_frame_order() { + use crate::vm::jit::ir::VirtualFrameSnapshot; + + let mut builder = SsaTraceBuilder::new(0, 0); + let entry = builder.entry(); + let caller = builder + .append_param(entry, SsaValueRepr::Tagged, "caller") + .unwrap(); + let callee_stack = builder + .append_param(entry, SsaValueRepr::I64, "callee_stack") + .unwrap(); + let callee_local = builder + .append_param(entry, SsaValueRepr::Bool, "callee_local") + .unwrap(); + let exit_id = builder.add_exit_with_virtual_frames( + 20, + vec![SsaMaterialization::Value(caller.id)], + Vec::new(), + Vec::new(), + vec![VirtualFrameSnapshot { + prototype_id: 1, + call_ip: 10, + return_ip: 12, + resume_ip: 20, + operand_stack: vec![SsaMaterialization::BoxInt(callee_stack.id)], + locals: vec![ + SsaMaterialization::Value(caller.id), + SsaMaterialization::BoxBool(callee_local.id), + ], + dirty_locals: vec![true, true], + }], + ); + builder + .set_terminator(entry, SsaTerminator::Exit { exit: exit_id }) + .unwrap(); + let trace = builder.finish(); + assert_eq!( + exit_inputs(&trace.exits[0]), + vec![caller.id, callee_stack.id, callee_local.id] + ); + } } diff --git a/src/vm/jit/ir.rs b/src/vm/jit/ir.rs index 8f856331..8797927e 100644 --- a/src/vm/jit/ir.rs +++ b/src/vm/jit/ir.rs @@ -500,6 +500,17 @@ pub(crate) enum SsaMaterialization { BoxHeapPtr { value: SsaValueId, tag: ValueType }, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct VirtualFrameSnapshot { + pub(crate) prototype_id: u32, + pub(crate) call_ip: usize, + pub(crate) return_ip: usize, + pub(crate) resume_ip: usize, + pub(crate) operand_stack: Vec, + pub(crate) locals: Vec, + pub(crate) dirty_locals: Vec, +} + #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct SsaExit { pub(crate) id: SsaExitId, @@ -507,6 +518,7 @@ pub(crate) struct SsaExit { pub(crate) stack: Vec, pub(crate) locals: Vec, pub(crate) dirty_locals: Vec, + pub(crate) virtual_frames: Vec, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -652,6 +664,41 @@ impl SsaTrace { for materialization in exit.stack.iter().chain(exit.locals.iter()) { verify_materialization(materialization, &value_reprs)?; } + let mut outer_resume_ip = None; + for (frame_index, frame) in exit.virtual_frames.iter().enumerate() { + if frame.dirty_locals.len() != frame.locals.len() { + return Err(SsaVerifyError::VirtualFrameDirtyLocalLengthMismatch { + exit: exit.id, + frame: frame_index, + locals: frame.locals.len(), + dirty_locals: frame.dirty_locals.len(), + }); + } + if frame.call_ip >= frame.return_ip { + return Err(SsaVerifyError::InvalidVirtualFrameContinuation { + exit: exit.id, + frame: frame_index, + call_ip: frame.call_ip, + return_ip: frame.return_ip, + }); + } + if outer_resume_ip.is_some_and(|resume_ip| resume_ip != frame.call_ip) { + return Err(SsaVerifyError::InvalidVirtualFrameOrder { + exit: exit.id, + frame: frame_index, + }); + } + for materialization in frame.operand_stack.iter().chain(&frame.locals) { + if matches!(materialization, SsaMaterialization::BoxHeapPtr { .. }) { + return Err(SsaVerifyError::UnsupportedVirtualFrameOwnership { + exit: exit.id, + frame: frame_index, + }); + } + verify_materialization(materialization, &value_reprs)?; + } + outer_resume_ip = Some(frame.resume_ip); + } } Ok(()) @@ -756,6 +803,26 @@ pub(crate) enum SsaVerifyError { locals: usize, dirty_locals: usize, }, + VirtualFrameDirtyLocalLengthMismatch { + exit: SsaExitId, + frame: usize, + locals: usize, + dirty_locals: usize, + }, + InvalidVirtualFrameContinuation { + exit: SsaExitId, + frame: usize, + call_ip: usize, + return_ip: usize, + }, + InvalidVirtualFrameOrder { + exit: SsaExitId, + frame: usize, + }, + UnsupportedVirtualFrameOwnership { + exit: SsaExitId, + frame: usize, + }, } pub(crate) struct SsaTraceBuilder { @@ -842,6 +909,17 @@ impl SsaTraceBuilder { stack: Vec, locals: Vec, dirty_locals: Vec, + ) -> SsaExitId { + self.add_exit_with_virtual_frames(exit_ip, stack, locals, dirty_locals, Vec::new()) + } + + pub(crate) fn add_exit_with_virtual_frames( + &mut self, + exit_ip: usize, + stack: Vec, + locals: Vec, + dirty_locals: Vec, + virtual_frames: Vec, ) -> SsaExitId { let id = SsaExitId::new(self.trace.exits.len() as u32); self.trace.exits.push(SsaExit { @@ -850,6 +928,7 @@ impl SsaTraceBuilder { stack, locals, dirty_locals, + virtual_frames, }); id } @@ -1320,4 +1399,123 @@ mod tests { }) ); } + + fn trace_with_virtual_frame(frame: VirtualFrameSnapshot) -> (SsaTrace, SsaExitId) { + let mut builder = SsaTraceBuilder::new(1, 0); + let entry = builder.entry(); + let exit = builder.add_exit_with_virtual_frames( + 20, + Vec::new(), + Vec::new(), + Vec::new(), + vec![frame], + ); + builder + .set_terminator(entry, SsaTerminator::Exit { exit }) + .unwrap(); + (builder.finish(), exit) + } + + #[test] + fn verifier_accepts_scalar_virtual_frame_snapshot() { + let mut builder = SsaTraceBuilder::new(1, 0); + let entry = builder.entry(); + let local = builder + .append_param(entry, SsaValueRepr::I64, "inline_local") + .unwrap(); + let exit = builder.add_exit_with_virtual_frames( + 20, + Vec::new(), + Vec::new(), + Vec::new(), + vec![VirtualFrameSnapshot { + prototype_id: 3, + call_ip: 10, + return_ip: 12, + resume_ip: 20, + operand_stack: Vec::new(), + locals: vec![SsaMaterialization::BoxInt(local.id)], + dirty_locals: vec![true], + }], + ); + builder + .set_terminator(entry, SsaTerminator::Exit { exit }) + .unwrap(); + assert_eq!(builder.finish().verify(), Ok(())); + } + + #[test] + fn verifier_rejects_malformed_virtual_frame_metadata() { + let (trace, exit) = trace_with_virtual_frame(VirtualFrameSnapshot { + prototype_id: 3, + call_ip: 12, + return_ip: 12, + resume_ip: 20, + operand_stack: Vec::new(), + locals: Vec::new(), + dirty_locals: Vec::new(), + }); + assert_eq!( + trace.verify(), + Err(SsaVerifyError::InvalidVirtualFrameContinuation { + exit, + frame: 0, + call_ip: 12, + return_ip: 12, + }) + ); + + let (trace, exit) = trace_with_virtual_frame(VirtualFrameSnapshot { + prototype_id: 3, + call_ip: 10, + return_ip: 12, + resume_ip: 20, + operand_stack: Vec::new(), + locals: Vec::new(), + dirty_locals: vec![true], + }); + assert_eq!( + trace.verify(), + Err(SsaVerifyError::VirtualFrameDirtyLocalLengthMismatch { + exit, + frame: 0, + locals: 0, + dirty_locals: 1, + }) + ); + } + + #[test] + fn verifier_rejects_heap_pointer_transfer_in_virtual_frame() { + let mut builder = SsaTraceBuilder::new(1, 0); + let entry = builder.entry(); + let value = builder + .append_param(entry, SsaValueRepr::HeapPtr(ValueType::Array), "array") + .unwrap(); + let exit = builder.add_exit_with_virtual_frames( + 20, + Vec::new(), + Vec::new(), + Vec::new(), + vec![VirtualFrameSnapshot { + prototype_id: 3, + call_ip: 10, + return_ip: 12, + resume_ip: 20, + operand_stack: Vec::new(), + locals: vec![SsaMaterialization::BoxHeapPtr { + value: value.id, + tag: ValueType::Array, + }], + dirty_locals: vec![true], + }], + ); + builder + .set_terminator(entry, SsaTerminator::Exit { exit }) + .unwrap(); + assert_eq!( + builder.finish().verify(), + Err(SsaVerifyError::UnsupportedVirtualFrameOwnership { exit, frame: 0 }) + ); + } } diff --git a/src/vm/native/bridge.rs b/src/vm/native/bridge.rs index effd7c0f..ff18adb7 100644 --- a/src/vm/native/bridge.rs +++ b/src/vm/native/bridge.rs @@ -1,9 +1,9 @@ #![allow(dead_code)] use crate::builtins::BuiltinFunction; -use crate::bytecode::{Value, ValueType, VmMap}; +use crate::bytecode::{CallableKind, CallableTarget, Value, ValueType, VmMap}; use crate::vm::{ - CallOutcome, CallReturn, ExecOutcome, FrameContinuation, HostCallExecOutcome, NumericValue, Vm, - VmError, VmHostFunction, VmResult, logical_shr_i64, + CallOutcome, CallReturn, ExecOutcome, ExecutionFrame, FrameContinuation, HostCallExecOutcome, + NumericValue, Vm, VmError, VmHostFunction, VmResult, logical_shr_i64, }; use std::cell::RefCell; use std::sync::Arc; @@ -328,6 +328,10 @@ pub(crate) fn restore_active_sparse_exit_state_entry_address() -> usize { pd_vm_native_restore_active_sparse_exit_state as *const () as usize } +pub(crate) fn restore_virtual_frame_entry_address() -> usize { + pd_vm_native_restore_virtual_frame as *const () as usize +} + pub(crate) fn map_has_entry_address() -> usize { pd_vm_native_map_has as *const () as usize } @@ -1218,6 +1222,99 @@ pub(crate) extern "C" fn pd_vm_native_restore_active_sparse_exit_state( }) } +#[allow(clippy::too_many_arguments)] +pub(crate) extern "C" fn pd_vm_native_restore_virtual_frame( + vm: *mut Vm, + prototype_id: u32, + call_ip: usize, + return_ip: usize, + resume_ip: usize, + stack_src: *const Value, + stack_len: usize, + locals_src: *const Value, + locals_len: usize, +) -> i32 { + run_step(vm, "restore_virtual_frame", |vm| { + if stack_len != 0 && stack_src.is_null() { + return Err(VmError::JitNative( + "virtual frame restore received null stack buffer".to_string(), + )); + } + if locals_len != 0 && locals_src.is_null() { + return Err(VmError::JitNative( + "virtual frame restore received null locals buffer".to_string(), + )); + } + if vm.call_depth >= vm.max_script_call_depth { + return Err(VmError::CallStackOverflow { + limit: vm.max_script_call_depth, + }); + } + let prototype = vm + .program + .callable_prototypes + .get(prototype_id as usize) + .ok_or(VmError::InvalidCallablePrototype(prototype_id))?; + if prototype.kind != CallableKind::FunctionItem + || !prototype.capture_slots.is_empty() + || !prototype.capture_source_slots.is_empty() + || !prototype.capture_modes.is_empty() + { + return Err(VmError::InvalidFrameState( + "virtual frame prototype is not an environment-free function item", + )); + } + let CallableTarget::ScriptFunction(function_id) = prototype.target else { + return Err(VmError::InvalidFrameState( + "virtual frame prototype is not a script function", + )); + }; + let function = vm + .program + .script_functions + .get(function_id as usize) + .ok_or(VmError::InvalidCallablePrototype(prototype_id))?; + if locals_len != prototype.frame_local_count { + return Err(VmError::InvalidFrameState( + "virtual frame local count does not match prototype", + )); + } + if call_ip.saturating_add(2) != return_ip + || vm.program.code.get(call_ip).copied() != Some(crate::OpCode::CallValue as u8) + || return_ip > vm.program.code.len() + || resume_ip < function.entry_ip as usize + || resume_ip >= function.end_ip as usize + { + return Err(VmError::InvalidFrameState( + "virtual frame continuation metadata is invalid", + )); + } + + let operand_stack_base = vm.stack.len(); + let local_base = vm.locals.len(); + vm.stack.reserve(stack_len); + vm.locals.reserve(locals_len); + for index in 0..stack_len { + vm.stack + .push(unsafe { std::ptr::read(stack_src.add(index)) }); + } + for index in 0..locals_len { + vm.locals + .push(unsafe { std::ptr::read(locals_src.add(index)) }); + } + vm.execution_frames.push(ExecutionFrame { + continuation: FrameContinuation::ResumeBytecode { return_ip }, + operand_stack_base, + local_base, + local_count: locals_len, + prototype_id: Some(prototype_id), + }); + vm.call_depth = vm.script_frame_depth(); + vm.ip = resume_ip; + Ok(STATUS_CONTINUE) + }) +} + pub(crate) extern "C" fn pd_vm_native_map_has(repr_ptr: *mut u8, key: *const Value) -> i32 { if repr_ptr.is_null() || key.is_null() { store_bridge_error(VmError::JitNative( @@ -1832,8 +1929,108 @@ pub(crate) extern "C" fn pd_vm_native_step(vm: *mut Vm, op: i64, a: i64, b: i64, #[cfg(test)] mod tests { use super::*; + use crate::{CallablePrototype, FunctionRegion, Program, RootCallableBinding, ScriptFunction}; use std::mem::{ManuallyDrop, MaybeUninit}; + fn virtual_frame_program() -> Program { + Program::new( + Vec::new(), + vec![crate::OpCode::CallValue as u8, 0, crate::OpCode::Ret as u8], + ) + .with_callable_metadata( + vec![ScriptFunction { + entry_ip: 2, + end_ip: 3, + }], + vec![CallablePrototype { + kind: CallableKind::FunctionItem, + target: CallableTarget::ScriptFunction(0), + arity: 0, + frame_local_count: 1, + parameter_slots: Vec::new(), + capture_source_slots: Vec::new(), + capture_slots: Vec::new(), + capture_modes: Vec::new(), + self_slot: None, + schema: None, + }], + vec![FunctionRegion { + start_ip: 2, + end_ip: 3, + prototype_id: Some(0), + }], + vec![RootCallableBinding { + local_slot: 0, + prototype_id: 0, + }], + ) + } + + #[test] + fn virtual_frame_restore_is_atomic_for_invalid_metadata() { + let mut vm = Vm::new(virtual_frame_program()); + let locals = [Value::Int(7)]; + let before = ( + vm.ip, + vm.stack.len(), + vm.locals.len(), + vm.execution_frames.len(), + vm.call_depth, + ); + let status = pd_vm_native_restore_virtual_frame( + &mut vm, + 99, + 0, + 2, + 2, + std::ptr::null(), + 0, + locals.as_ptr(), + locals.len(), + ); + assert_eq!(status, STATUS_ERROR); + assert_eq!( + before, + ( + vm.ip, + vm.stack.len(), + vm.locals.len(), + vm.execution_frames.len(), + vm.call_depth, + ) + ); + let _ = take_bridge_error(); + } + + #[test] + fn virtual_frame_restore_builds_script_frame_from_materialized_values() { + let mut vm = Vm::new(virtual_frame_program()); + let mut locals = ManuallyDrop::new(vec![Value::Int(7)]); + let status = pd_vm_native_restore_virtual_frame( + &mut vm, + 0, + 0, + 2, + 2, + std::ptr::null(), + 0, + locals.as_mut_ptr(), + locals.len(), + ); + assert_eq!(status, STATUS_CONTINUE); + assert_eq!(vm.ip, 2); + assert_eq!(vm.call_depth, 1); + assert_eq!(vm.execution_frames.len(), 2); + assert_eq!(vm.locals.last(), Some(&Value::Int(7))); + let frame = vm.execution_frames.last().unwrap(); + assert_eq!(frame.prototype_id, Some(0)); + assert_eq!(frame.local_count, 1); + assert_eq!( + frame.continuation, + FrameContinuation::ResumeBytecode { return_ip: 2 } + ); + } + #[test] fn jit_trace_exit_status_round_trips_reserved_range_boundaries() { for exit_id in [0, 1, 7, 255, STATUS_JIT_TRACE_EXIT_MAX_ID] { diff --git a/src/vm/native/codegen.rs b/src/vm/native/codegen.rs index d3decbb7..72d71f48 100644 --- a/src/vm/native/codegen.rs +++ b/src/vm/native/codegen.rs @@ -431,6 +431,22 @@ pub(crate) fn sparse_restore_exit_signature( sig } +#[cfg(feature = "cranelift-jit")] +#[allow(dead_code)] +pub(crate) fn restore_virtual_frame_signature( + pointer_type: cranelift_codegen::ir::Type, + call_conv: cranelift_codegen::isa::CallConv, +) -> Signature { + let mut sig = Signature::new(call_conv); + sig.params.push(AbiParam::new(pointer_type)); + sig.params.push(AbiParam::new(types::I32)); + for _ in 0..7 { + sig.params.push(AbiParam::new(pointer_type)); + } + sig.returns.push(AbiParam::new(types::I32)); + sig +} + #[cfg(feature = "cranelift-jit")] pub(crate) fn entry_signature( pointer_type: cranelift_codegen::ir::Type, diff --git a/src/vm/native/mod.rs b/src/vm/native/mod.rs index 49b87e34..828ce34f 100644 --- a/src/vm/native/mod.rs +++ b/src/vm/native/mod.rs @@ -24,12 +24,13 @@ pub(crate) use bridge::{ non_yielding_scalar_host_call_entry_address, regex_match_entry_address, regex_replace_entry_address, restore_active_exit_state_entry_address, restore_active_sparse_exit_state_entry_address, restore_exit_state_entry_address, - restore_sparse_exit_state_entry_address, shared_array_from_buffer_entry_address, - shared_bytes_from_buffer_entry_address, shared_string_from_buffer_entry_address, - store_bridge_error, string_contains_entry_address, string_lower_ascii_entry_address, - string_replace_literal_entry_address, string_split_literal_entry_address, take_bridge_error, - to_string_entry_address, type_of_entry_address, value_eq_entry_address, - value_len_entry_address, write_heap_value_to_slot_entry_address, zero_bytes_entry_address, + restore_sparse_exit_state_entry_address, restore_virtual_frame_entry_address, + shared_array_from_buffer_entry_address, shared_bytes_from_buffer_entry_address, + shared_string_from_buffer_entry_address, store_bridge_error, string_contains_entry_address, + string_lower_ascii_entry_address, string_replace_literal_entry_address, + string_split_literal_entry_address, take_bridge_error, to_string_entry_address, + type_of_entry_address, value_eq_entry_address, value_len_entry_address, + write_heap_value_to_slot_entry_address, zero_bytes_entry_address, }; #[cfg(feature = "cranelift-jit")] pub(crate) use codegen::{ @@ -41,10 +42,10 @@ pub(crate) use codegen::{ map_iter_next_signature, map_iter_take_signature, map_set_signature, non_yielding_host_call_signature, non_yielding_i64_host_call_signature, non_yielding_scalar_host_call_signature, pack_shared_signature, regex_match_signature, - regex_replace_signature, restore_exit_signature, sparse_restore_exit_signature, - string_binary_transform_signature, string_contains_signature, string_replace_signature, - string_unary_transform_signature, value_eq_signature, value_len_signature, - value_slot_signature, + regex_replace_signature, restore_exit_signature, restore_virtual_frame_signature, + sparse_restore_exit_signature, string_binary_transform_signature, string_contains_signature, + string_replace_signature, string_unary_transform_signature, value_eq_signature, + value_len_signature, value_slot_signature, }; pub(crate) use exec::{ExecutableBuffer, prepare_for_execution}; pub(crate) use layout::{ From 2bccf38e63d1d3a5d46192bd2378967afb448148 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sun, 19 Jul 2026 19:17:25 +0800 Subject: [PATCH 05/26] feat(jit): inline bounded static leaf calls --- src/vm/jit/deopt.rs | 13 +- src/vm/jit/inline.rs | 8 +- src/vm/jit/ir.rs | 19 +- src/vm/jit/native/lower.rs | 122 +++++++++++- src/vm/jit/recorder.rs | 392 +++++++++++++++++++++++++++++++------ src/vm/jit/region.rs | 3 +- src/vm/jit/runtime.rs | 3 + src/vm/jit/trace.rs | 33 ++++ tests/jit/jit_tests.rs | 149 +++++++++++++- 9 files changed, 672 insertions(+), 70 deletions(-) diff --git a/src/vm/jit/deopt.rs b/src/vm/jit/deopt.rs index 2c8a3a98..e60f5954 100644 --- a/src/vm/jit/deopt.rs +++ b/src/vm/jit/deopt.rs @@ -42,12 +42,7 @@ pub(crate) fn exit_inputs(exit: &SsaExit) -> Vec { } } for frame in &exit.virtual_frames { - let dirty_locals = frame - .locals - .iter() - .zip(&frame.dirty_locals) - .filter_map(|(materialization, dirty)| dirty.then_some(materialization)); - for materialization in frame.operand_stack.iter().chain(dirty_locals) { + for materialization in frame.operand_stack.iter().chain(&frame.locals) { let value = match materialization { SsaMaterialization::Value(value) | SsaMaterialization::BoxInt(value) @@ -78,6 +73,7 @@ pub(crate) enum SideTraceImportError { ExitIpMismatch { parent: usize, child: usize }, StackDepthMismatch { parent: usize, child: usize }, LocalCountMismatch { parent: usize, child: usize }, + VirtualFramesUnsupported { count: usize }, InvalidChildEntry, } @@ -91,6 +87,11 @@ pub(crate) fn side_trace_import( .iter() .find(|exit| exit.id == parent_exit) .ok_or(SideTraceImportError::UnknownParentExit(parent_exit))?; + if !exit.virtual_frames.is_empty() { + return Err(SideTraceImportError::VirtualFramesUnsupported { + count: exit.virtual_frames.len(), + }); + } if exit.exit_ip != child.root_ip { return Err(SideTraceImportError::ExitIpMismatch { parent: exit.exit_ip, diff --git a/src/vm/jit/inline.rs b/src/vm/jit/inline.rs index 9d50ed70..dddbcd1f 100644 --- a/src/vm/jit/inline.rs +++ b/src/vm/jit/inline.rs @@ -67,7 +67,13 @@ pub(crate) fn classify_static_inline_candidate( { return Err(InlineRejectReason::CapturedCallable); } - if prototype.arity != argc || prototype.parameter_slots.len() != usize::from(argc) { + if prototype.arity != argc + || prototype.parameter_slots.len() != usize::from(argc) + || prototype + .parameter_slots + .iter() + .any(|slot| usize::from(*slot) >= prototype.frame_local_count) + { return Err(InlineRejectReason::ArityMismatch); } let CallableTarget::ScriptFunction(function_id) = prototype.target else { diff --git a/src/vm/jit/ir.rs b/src/vm/jit/ir.rs index 8797927e..b20d41a6 100644 --- a/src/vm/jit/ir.rs +++ b/src/vm/jit/ir.rs @@ -102,6 +102,9 @@ pub(crate) struct SsaBlockParam { #[derive(Clone, Debug, PartialEq)] pub(crate) enum SsaInstKind { Constant(Value), + CloneTagged { + input: SsaValueId, + }, UnboxInt { input: SsaValueId, }, @@ -390,7 +393,8 @@ impl SsaInstKind { Self::Constant(_) => Vec::new(), Self::HostCall { args, .. } => args.clone(), - Self::UnboxInt { input } + Self::CloneTagged { input } + | Self::UnboxInt { input } | Self::UnboxFloat { input } | Self::UnboxBool { input } | Self::UnboxHeapPtr { input, .. } @@ -760,6 +764,18 @@ impl SsaTrace { let _ = writeln!(&mut out, " stack=[{}]", stack); let _ = writeln!(&mut out, " locals=[{}]", locals); let _ = writeln!(&mut out, " dirty_locals=[{}]", dirty_locals); + for (index, frame) in exit.virtual_frames.iter().enumerate() { + let _ = writeln!( + &mut out, + " virtual_frame{index}=prototype{} call_ip={} return_ip={} resume_ip={} stack={} locals={}", + frame.prototype_id, + frame.call_ip, + frame.return_ip, + frame.resume_ip, + frame.operand_stack.len(), + frame.locals.len(), + ); + } } out } @@ -1104,6 +1120,7 @@ fn verify_materialization( fn render_inst_kind(kind: &SsaInstKind) -> String { match kind { SsaInstKind::Constant(value) => format!("const {value:?}"), + SsaInstKind::CloneTagged { input } => format!("clone_tagged {input}"), SsaInstKind::UnboxInt { input } => format!("unbox_int {input}"), SsaInstKind::UnboxFloat { input } => format!("unbox_float {input}"), SsaInstKind::UnboxBool { input } => format!("unbox_bool {input}"), diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index bee4b6ce..d6b6db06 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -27,7 +27,8 @@ use crate::vm::native::{ non_yielding_i64_host_call_signature, non_yielding_scalar_host_call_entry_address, non_yielding_scalar_host_call_signature, pack_shared_signature, regex_match_entry_address, regex_match_signature, regex_replace_entry_address, regex_replace_signature, - restore_active_sparse_exit_state_entry_address, shared_array_from_buffer_entry_address, + restore_active_sparse_exit_state_entry_address, restore_virtual_frame_entry_address, + restore_virtual_frame_signature, shared_array_from_buffer_entry_address, shared_bytes_from_buffer_entry_address, shared_string_from_buffer_entry_address, sparse_restore_exit_signature, string_binary_transform_signature, string_contains_entry_address, string_contains_signature, string_lower_ascii_entry_address, @@ -602,6 +603,7 @@ fn try_compile_ssa_trace( let array_set_sig = array_set_signature(pointer_type, call_conv); let map_set_sig = map_set_signature(pointer_type, call_conv); let sparse_restore_exit_sig = sparse_restore_exit_signature(pointer_type, call_conv); + let restore_virtual_frame_sig = restore_virtual_frame_signature(pointer_type, call_conv); let frame_state_sig = frame_state_signature(pointer_type, call_conv); let leave_frame_sig = leave_frame_inherited_signature(pointer_type, call_conv); let enter_call_value_sig = enter_call_value_inherited_signature(pointer_type, call_conv); @@ -683,6 +685,7 @@ fn try_compile_ssa_trace( array_set_ref: b.import_signature(array_set_sig), map_set_ref: b.import_signature(map_set_sig), sparse_restore_exit_ref: b.import_signature(sparse_restore_exit_sig), + restore_virtual_frame_ref: b.import_signature(restore_virtual_frame_sig), leave_frame_ref: b.import_signature(leave_frame_sig), enter_call_value_ref: b.import_signature(enter_call_value_sig), @@ -706,6 +709,7 @@ fn try_compile_ssa_trace( array_set: array_set_entry_address(), map_set: map_set_entry_address(), sparse_restore_exit: restore_active_sparse_exit_state_entry_address(), + restore_virtual_frame: restore_virtual_frame_entry_address(), leave_frame: leave_frame_inherited_entry_address(), enter_call_value: enter_call_value_inherited_entry_address(), @@ -1165,6 +1169,7 @@ struct SsaDeoptHelperRefs { array_set_ref: cranelift_codegen::ir::SigRef, map_set_ref: cranelift_codegen::ir::SigRef, sparse_restore_exit_ref: cranelift_codegen::ir::SigRef, + restore_virtual_frame_ref: cranelift_codegen::ir::SigRef, leave_frame_ref: cranelift_codegen::ir::SigRef, enter_call_value_ref: cranelift_codegen::ir::SigRef, @@ -1190,6 +1195,7 @@ struct SsaDeoptHelperAddrs { array_set: usize, map_set: usize, sparse_restore_exit: usize, + restore_virtual_frame: usize, leave_frame: usize, enter_call_value: usize, @@ -1312,6 +1318,7 @@ fn ssa_trace_supported(ssa: &SsaTrace) -> bool { if !matches!( inst.kind, SsaInstKind::Constant(_) + | SsaInstKind::CloneTagged { .. } | SsaInstKind::UnboxHeapPtr { .. } | SsaInstKind::UnboxInt { .. } | SsaInstKind::UnboxFloat { .. } @@ -1663,7 +1670,8 @@ fn borrowed_array_get_outputs(ssa: &SsaTrace) -> BTreeSet { fn ssa_inst_requires_owned_value_slot(kind: &SsaInstKind) -> bool { matches!( kind, - SsaInstKind::ArrayGet { .. } + SsaInstKind::CloneTagged { .. } + | SsaInstKind::ArrayGet { .. } | SsaInstKind::ArraySet { .. } | SsaInstKind::ArrayPush { .. } | SsaInstKind::MapGet { .. } @@ -2105,6 +2113,28 @@ fn lower_ssa_inst( })?; iconst_ptr_from_addr(b, pointer_type, addr)? } + SsaInstKind::CloneTagged { input } => { + if value_reprs.get(input) != Some(&SsaValueRepr::Tagged) { + return Err(VmError::JitNative( + "SSA clone-tagged input must be tagged".to_string(), + )); + } + let out = owned_value_temp_slot_addr( + b, + pointer_type, + owned_value_temps, + SsaTempValueSlotKey::Output(output.id), + )?; + ssa_call_status_helper( + b, + exit_block, + pointer_type, + helper_refs.clone_value_ref, + helper_addrs.clone_value, + &[out, values[input]], + )?; + out + } SsaInstKind::UnboxInt { input } => { let input = *values .get(input) @@ -4675,7 +4705,8 @@ fn lower_ssa_exit_block( }; let mut moved_owned_values = BTreeSet::new(); - let inline_owned_restore = entry_stack_depth == 0 + let inline_owned_restore = exit.virtual_frames.is_empty() + && entry_stack_depth == 0 && exit.stack.is_empty() && exit .locals @@ -4838,7 +4869,8 @@ fn lower_ssa_exit_block( ip_val, ], )?; - if matches!(action, SsaExitAction::TraceExit { .. }) { + restore_virtual_frames(b, materialize_ctx, vm_ptr, &exit.virtual_frames)?; + if matches!(action, SsaExitAction::TraceExit { .. }) && exit.virtual_frames.is_empty() { write_inherited_state_packet(b, ctx, exit)?; } let status = ssa_exit_action_status( @@ -4855,6 +4887,88 @@ fn lower_ssa_exit_block( Ok(()) } +fn restore_virtual_frames( + b: &mut FunctionBuilder, + materialize_ctx: SsaMaterializeCtx<'_>, + vm_ptr: cranelift_codegen::ir::Value, + frames: &[crate::vm::jit::ir::VirtualFrameSnapshot], +) -> VmResult<()> { + let pointer_type = materialize_ctx.pointer_type; + let value_size = materialize_ctx.value_layout.size; + let null_ptr = b.ins().iconst(pointer_type, 0); + for frame in frames { + let stack_ptr = + ssa_alloc_value_buffer(b, pointer_type, frame.operand_stack.len(), value_size)?; + for (index, materialization) in frame.operand_stack.iter().enumerate() { + let dst = ssa_value_buffer_slot_addr( + b, + pointer_type, + stack_ptr, + index, + value_size, + "virtual stack", + )?; + ssa_materialize_slot(b, materialize_ctx, materialization, dst, "virtual stack")?; + } + let locals_ptr = ssa_alloc_value_buffer(b, pointer_type, frame.locals.len(), value_size)?; + for (index, materialization) in frame.locals.iter().enumerate() { + let dst = ssa_value_buffer_slot_addr( + b, + pointer_type, + locals_ptr, + index, + value_size, + "virtual local", + )?; + ssa_materialize_slot(b, materialize_ctx, materialization, dst, "virtual local")?; + } + let prototype_id = b.ins().iconst(types::I32, i64::from(frame.prototype_id)); + let call_ip = iconst_usize(b, pointer_type, frame.call_ip, "virtual call ip")?; + let return_ip = iconst_usize(b, pointer_type, frame.return_ip, "virtual return ip")?; + let resume_ip = iconst_usize(b, pointer_type, frame.resume_ip, "virtual resume ip")?; + let stack_len = iconst_usize( + b, + pointer_type, + frame.operand_stack.len(), + "virtual stack length", + )?; + let locals_len = + iconst_usize(b, pointer_type, frame.locals.len(), "virtual locals length")?; + ssa_call_status_helper( + b, + materialize_ctx.exit_block, + pointer_type, + materialize_ctx.deopt_refs.restore_virtual_frame_ref, + materialize_ctx.deopt_addrs.restore_virtual_frame, + &[ + vm_ptr, + prototype_id, + call_ip, + return_ip, + resume_ip, + stack_ptr.unwrap_or(null_ptr), + stack_len, + locals_ptr.unwrap_or(null_ptr), + locals_len, + ], + )?; + } + Ok(()) +} + +fn iconst_usize( + b: &mut FunctionBuilder, + pointer_type: cranelift_codegen::ir::Type, + value: usize, + label: &'static str, +) -> VmResult { + Ok(b.ins().iconst( + pointer_type, + i64::try_from(value) + .map_err(|_| VmError::JitNative(format!("SSA {label} out of range")))?, + )) +} + fn ssa_materialize_slot( b: &mut FunctionBuilder, ctx: SsaMaterializeCtx<'_>, diff --git a/src/vm/jit/recorder.rs b/src/vm/jit/recorder.rs index 1d2416f2..282947c9 100644 --- a/src/vm/jit/recorder.rs +++ b/src/vm/jit/recorder.rs @@ -5,10 +5,10 @@ use crate::vm::{OpCode, Program, Value, ValueType, checked_int_div}; use super::JitTraceTerminal; use super::deopt::materialize_ssa_values; -use super::inline::classify_static_inline_candidate; +use super::inline::{InlineCandidate, classify_static_inline_candidate}; use super::ir::{ SsaBranchTarget, SsaInstKind, SsaMaterialization, SsaTerminator, SsaTrace, SsaTraceBuilder, - SsaValue, SsaValueRepr, + SsaValue, SsaValueRepr, VirtualFrameSnapshot, }; #[derive(Clone, Debug, PartialEq)] @@ -269,6 +269,14 @@ struct SymbolicFrame { dirty_locals: Vec, } +#[derive(Clone, Debug, PartialEq)] +struct InlineRecorderFrame { + candidate: InlineCandidate, + call_ip: usize, + return_ip: usize, + caller: SymbolicFrame, +} + #[derive(Clone, Debug, PartialEq, Eq)] struct LoopHeaderPlan { stack_reprs: Vec, @@ -745,6 +753,7 @@ pub(crate) fn record_trace_with_local_count( ) -> Result { let loop_header_plan = infer_loop_header_plan( program, + caller_frame_key, root_ip, entry_stack_depth, local_count, @@ -896,6 +905,7 @@ pub(crate) fn record_trace_with_local_count( let mut has_call = false; let mut has_yielding_call = false; let mut has_useful_native_computation = false; + let mut inline_frame: Option = None; loop { let Some(decoded) = cursor.next()? else { @@ -906,13 +916,33 @@ pub(crate) fn record_trace_with_local_count( match decoded { DecodedOp::Nop { .. } => op_names.push("nop".to_string()), DecodedOp::Ret { ip } => { + if let Some(inlined) = inline_frame.take() { + let result = match frame.stack.pop() { + Some(result) => result, + None => { + let value = builder + .append_value_inst( + current_block, + ip, + SsaValueRepr::Tagged, + SsaInstKind::Constant(Value::Null), + ) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + SymbolicValue { + value, + info: ValueInfo::tagged_typed(ValueType::Null), + } + } + }; + op_names.push("inline_ret".to_string()); + frame = inlined.caller; + frame.push(result); + cursor.jump_to(inlined.return_ip)?; + has_useful_native_computation = true; + continue; + } op_names.push("ret".to_string()); - let exit = builder.add_exit( - ip, - materialize_stack(&frame.stack), - materialize_locals(&frame.locals), - frame.dirty_locals.clone(), - ); + let exit = add_symbolic_exit(&mut builder, ip, &frame, inline_frame.as_ref()); builder .set_terminator(current_block, SsaTerminator::Return { exit }) .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; @@ -1041,11 +1071,11 @@ pub(crate) fn record_trace_with_local_count( got: frame.stack.len(), }); } - let exit = builder.add_exit( + let exit = add_symbolic_exit( + &mut builder, fallthrough_ip, - materialize_stack(&frame.stack), - materialize_locals(&frame.locals), - frame.dirty_locals.clone(), + &frame, + inline_frame.as_ref(), ); builder .set_terminator( @@ -1069,14 +1099,18 @@ pub(crate) fn record_trace_with_local_count( if condition.info.const_bool == Some(false) { op_names.push("guard_true".to_string()); - let exit = builder.add_exit( + let exit = add_symbolic_exit( + &mut builder, fallthrough_ip, - materialize_stack(&frame.stack), - materialize_locals(&frame.locals), - frame.dirty_locals.clone(), + &frame, + inline_frame.as_ref(), ); - let (next_block, next_frame, args) = - continue_with_frame(&mut builder, &frame, "guard")?; + let (next_block, next_frame, args) = continue_with_inline_frame( + &mut builder, + &frame, + &mut inline_frame, + "guard", + )?; builder .set_terminator( current_block, @@ -1099,14 +1133,18 @@ pub(crate) fn record_trace_with_local_count( if prefer_join_path { op_names.push("guard_true".to_string()); - let exit = builder.add_exit( + let exit = add_symbolic_exit( + &mut builder, fallthrough_ip, - materialize_stack(&frame.stack), - materialize_locals(&frame.locals), - frame.dirty_locals.clone(), + &frame, + inline_frame.as_ref(), ); - let (next_block, next_frame, args) = - continue_with_frame(&mut builder, &frame, "guard")?; + let (next_block, next_frame, args) = continue_with_inline_frame( + &mut builder, + &frame, + &mut inline_frame, + "guard", + )?; builder .set_terminator( current_block, @@ -1128,14 +1166,9 @@ pub(crate) fn record_trace_with_local_count( } op_names.push("guard_false".to_string()); - let exit = builder.add_exit( - target, - materialize_stack(&frame.stack), - materialize_locals(&frame.locals), - frame.dirty_locals.clone(), - ); + let exit = add_symbolic_exit(&mut builder, target, &frame, inline_frame.as_ref()); let (next_block, next_frame, args) = - continue_with_frame(&mut builder, &frame, "guard")?; + continue_with_inline_frame(&mut builder, &frame, &mut inline_frame, "guard")?; builder .set_terminator( current_block, @@ -1178,12 +1211,8 @@ pub(crate) fn record_trace_with_local_count( } if target < cursor.ip() { op_names.push("jump_ip".to_string()); - let exit = builder.add_exit( - target, - materialize_stack(&frame.stack), - materialize_locals(&frame.locals), - frame.dirty_locals.clone(), - ); + let exit = + add_symbolic_exit(&mut builder, target, &frame, inline_frame.as_ref()); builder .set_terminator(current_block, SsaTerminator::Exit { exit }) .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; @@ -1204,7 +1233,7 @@ pub(crate) fn record_trace_with_local_count( let callable = frame.stack[frame.stack.len() - usize::from(argc) - 1]; let caller_prototype_id = (caller_frame_key != crate::vm::native::ROOT_FRAME_KEY) .then_some(caller_frame_key as u32); - let _ = classify_static_inline_candidate( + let candidate = classify_static_inline_candidate( program, caller_frame_key, caller_prototype_id, @@ -1212,13 +1241,71 @@ pub(crate) fn record_trace_with_local_count( argc, max_trace_len.saturating_sub(cursor.recorded_ops), ); + let inline_reject_reason = candidate.as_ref().err().copied(); + if inline_frame.is_none() + && let Ok(candidate) = candidate + { + let operand_base = frame.stack.len() - usize::from(argc) - 1; + let mut operands = frame.stack.split_off(operand_base); + let _callable = operands.remove(0); + let prototype = &program.callable_prototypes[candidate.prototype_id as usize]; + let null = builder + .append_value_inst( + current_block, + ip, + SsaValueRepr::Tagged, + SsaInstKind::Constant(Value::Null), + ) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + let null = SymbolicValue { + value: null, + info: ValueInfo::tagged_typed(ValueType::Null), + }; + let mut callee_locals = vec![null; prototype.frame_local_count]; + for binding in &program.root_callable_bindings { + let slot = usize::from(binding.local_slot); + if slot < callee_locals.len() && slot < frame.locals.len() { + callee_locals[slot] = frame.locals[slot]; + } + } + for (slot, mut argument) in candidate.parameter_slots.iter().zip(operands) { + if argument.info.repr == SsaValueRepr::Tagged { + let cloned = builder + .append_value_inst( + current_block, + ip, + SsaValueRepr::Tagged, + SsaInstKind::CloneTagged { + input: argument.value.id, + }, + ) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + argument.value = cloned; + } + callee_locals[usize::from(*slot)] = argument; + } + op_names.push(format!("inline_call:{}", candidate.prototype_id)); + let caller = std::mem::replace( + &mut frame, + SymbolicFrame::new(Vec::new(), callee_locals), + ); + inline_frame = Some(InlineRecorderFrame { + candidate: candidate.clone(), + call_ip: ip, + return_ip: resume_ip, + caller, + }); + cursor.jump_to(candidate.entry_ip)?; + has_call = true; + continue; + } + if let Some(reason) = inline_reject_reason { + op_names.push(format!("inline_reject:{reason:?}")); + } else if inline_frame.is_some() { + op_names.push("inline_reject:NestedCallable".to_string()); + } op_names.push("call_value".to_string()); - let exit = builder.add_exit( - ip, - materialize_stack(&frame.stack), - materialize_locals(&frame.locals), - frame.dirty_locals.clone(), - ); + let exit = add_symbolic_exit(&mut builder, ip, &frame, inline_frame.as_ref()); builder .set_terminator( current_block, @@ -1341,12 +1428,7 @@ pub(crate) fn record_trace_with_local_count( has_call = true; has_yielding_call |= yields; op_names.push("call".to_string()); - let exit = builder.add_exit( - ip, - materialize_stack(&frame.stack), - materialize_locals(&frame.locals), - frame.dirty_locals.clone(), - ); + let exit = add_symbolic_exit(&mut builder, ip, &frame, inline_frame.as_ref()); builder .set_terminator(current_block, SsaTerminator::Exit { exit }) .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; @@ -1371,8 +1453,10 @@ pub(crate) fn record_trace_with_local_count( }) } +#[allow(clippy::too_many_arguments)] fn infer_loop_header_plan( program: &Program, + caller_frame_key: u64, root_ip: usize, entry_stack_depth: usize, local_count: usize, @@ -1384,6 +1468,7 @@ fn infer_loop_header_plan( let mut frame = AnalysisFrame::new(program, entry_stack_depth, local_count, entry_local_types); let mut entry_use = vec![EntryUseState::Untouched; local_count]; let mut local_written = vec![false; local_count]; + let mut inline_frame: Option<(AnalysisFrame, usize)> = None; loop { let Some(decoded) = cursor.next()? else { @@ -1392,7 +1477,18 @@ fn infer_loop_header_plan( match decoded { DecodedOp::Nop { .. } => {} - DecodedOp::Ret { .. } => return Ok(None), + DecodedOp::Ret { .. } => { + let Some((mut caller, return_ip)) = inline_frame.take() else { + return Ok(None); + }; + let result = frame + .stack + .pop() + .unwrap_or_else(|| ValueInfo::tagged_typed(ValueType::Null)); + caller.push(result); + frame = caller; + cursor.jump_to(return_ip)?; + } DecodedOp::Ldc { index, .. } => { let constant = program.constants.get(index as usize).ok_or_else(|| { TraceRecordError::UnsupportedTrace(format!( @@ -1402,7 +1498,8 @@ fn infer_loop_header_plan( frame.push(ValueInfo::from_constant(constant)); } DecodedOp::Ldloc { index, .. } => { - if let Some(state) = entry_use.get_mut(index as usize) + if inline_frame.is_none() + && let Some(state) = entry_use.get_mut(index as usize) && matches!(*state, EntryUseState::Untouched) { *state = EntryUseState::ReadBeforeWrite; @@ -1410,10 +1507,13 @@ fn infer_loop_header_plan( frame.push(frame.local(index)?.sourced_from(index)) } DecodedOp::Stloc { index, .. } => { - if let Some(written) = local_written.get_mut(index as usize) { + if inline_frame.is_none() + && let Some(written) = local_written.get_mut(index as usize) + { *written = true; } - if let Some(state) = entry_use.get_mut(index as usize) + if inline_frame.is_none() + && let Some(state) = entry_use.get_mut(index as usize) && matches!(*state, EntryUseState::Untouched) { *state = EntryUseState::WrittenBeforeRead; @@ -1558,7 +1658,51 @@ fn infer_loop_header_plan( _ => ValueInfo::tagged_typed(return_type), }); } - DecodedOp::Call { .. } | DecodedOp::CallValue { .. } => return Ok(None), + DecodedOp::CallValue { + argc, resume_ip, .. + } => { + if inline_frame.is_some() || frame.stack.len() < usize::from(argc) + 1 { + return Ok(None); + } + let callable = frame.stack[frame.stack.len() - usize::from(argc) - 1]; + let caller_prototype_id = (caller_frame_key != crate::vm::native::ROOT_FRAME_KEY) + .then_some(caller_frame_key as u32); + let Ok(candidate) = classify_static_inline_candidate( + program, + caller_frame_key, + caller_prototype_id, + callable.source_local, + argc, + max_trace_len.saturating_sub(cursor.recorded_ops), + ) else { + return Ok(None); + }; + let prototype = &program.callable_prototypes[candidate.prototype_id as usize]; + let operand_base = frame.stack.len() - usize::from(argc) - 1; + let mut operands = frame.stack.split_off(operand_base); + let _callable = operands.remove(0); + let null = ValueInfo::tagged_typed(ValueType::Null); + let mut callee_locals = vec![null; prototype.frame_local_count]; + for binding in &program.root_callable_bindings { + let slot = usize::from(binding.local_slot); + if slot < callee_locals.len() && slot < frame.locals.len() { + callee_locals[slot] = frame.locals[slot]; + } + } + for (slot, argument) in candidate.parameter_slots.iter().zip(operands) { + callee_locals[usize::from(*slot)] = argument; + } + let caller = std::mem::replace( + &mut frame, + AnalysisFrame { + stack: Vec::new(), + locals: callee_locals, + }, + ); + inline_frame = Some((caller, resume_ip)); + cursor.jump_to(candidate.entry_ip)?; + } + DecodedOp::Call { .. } => return Ok(None), DecodedOp::Brfalse { target, fallthrough_ip, @@ -4473,6 +4617,48 @@ fn load_constant( Ok(SymbolicValue { value, info }) } +fn continue_with_inline_frame( + builder: &mut SsaTraceBuilder, + frame: &SymbolicFrame, + inline_frame: &mut Option, + label_prefix: &str, +) -> Result< + ( + super::ir::SsaBlockId, + SymbolicFrame, + Vec, + ), + TraceRecordError, +> { + let Some(inlined) = inline_frame.as_mut() else { + return continue_with_frame(builder, frame, label_prefix); + }; + let callee_stack_len = frame.stack.len(); + let callee_local_len = frame.locals.len(); + let combined = SymbolicFrame { + stack: frame + .stack + .iter() + .chain(&inlined.caller.stack) + .copied() + .collect(), + locals: frame + .locals + .iter() + .chain(&inlined.caller.locals) + .copied() + .collect(), + dirty_locals: Vec::new(), + }; + let (block, mut next, args) = continue_with_frame(builder, &combined, label_prefix)?; + let caller_stack = next.stack.split_off(callee_stack_len); + let caller_locals = next.locals.split_off(callee_local_len); + inlined.caller.stack = caller_stack; + inlined.caller.locals = caller_locals; + next.dirty_locals = frame.dirty_locals.clone(); + Ok((block, next, args)) +} + fn continue_with_frame( builder: &mut SsaTraceBuilder, frame: &SymbolicFrame, @@ -4558,6 +4744,37 @@ fn materialize_locals(locals: &[SymbolicValue]) -> Vec { materialize_ssa_values(locals.iter().map(|value| value.value)) } +fn add_symbolic_exit( + builder: &mut SsaTraceBuilder, + exit_ip: usize, + frame: &SymbolicFrame, + inline_frame: Option<&InlineRecorderFrame>, +) -> super::ir::SsaExitId { + let Some(inline_frame) = inline_frame else { + return builder.add_exit( + exit_ip, + materialize_stack(&frame.stack), + materialize_locals(&frame.locals), + frame.dirty_locals.clone(), + ); + }; + builder.add_exit_with_virtual_frames( + inline_frame.call_ip, + materialize_stack(&inline_frame.caller.stack), + materialize_locals(&inline_frame.caller.locals), + inline_frame.caller.dirty_locals.clone(), + vec![VirtualFrameSnapshot { + prototype_id: inline_frame.candidate.prototype_id, + call_ip: inline_frame.call_ip, + return_ip: inline_frame.return_ip, + resume_ip: exit_ip, + operand_stack: materialize_stack(&frame.stack), + locals: materialize_locals(&frame.locals), + dirty_locals: frame.dirty_locals.clone(), + }], + ) +} + fn build_loop_header_plan( stack: &[ValueInfo], locals: &[ValueInfo], @@ -4693,7 +4910,10 @@ fn read_u32(code: &[u8], ip: &mut usize) -> Option { #[cfg(test)] mod tests { use super::*; - use crate::{BytecodeBuilder, Value}; + use crate::{ + BytecodeBuilder, CallableKind, CallablePrototype, CallableTarget, FunctionRegion, + RootCallableBinding, ScriptFunction, Value, + }; fn patch_branch_target(code: &mut [u8], instr_ip: u32, target: u32) { let start = instr_ip as usize + 1; @@ -4957,4 +5177,64 @@ mod tests { Some(SsaTerminator::Jump { .. }) )); } + + #[test] + fn inline_static_leaf_records_call_and_return_inside_root_loop() { + let mut bc = BytecodeBuilder::new(); + let root_ip = bc.position(); + bc.ldloc(0); + bc.ldc(0); + bc.call_value(1); + bc.pop(); + bc.br(root_ip); + let function_entry = bc.position(); + bc.ldloc(1); + bc.ldc(0); + bc.add(); + bc.ret(); + let function_end = bc.position(); + + let program = Program::new(vec![Value::Int(1)], bc.finish()) + .with_local_count(2) + .with_callable_metadata( + vec![ScriptFunction { + entry_ip: function_entry, + end_ip: function_end, + }], + vec![CallablePrototype { + kind: CallableKind::FunctionItem, + target: CallableTarget::ScriptFunction(0), + arity: 1, + frame_local_count: 2, + parameter_slots: vec![1], + capture_source_slots: Vec::new(), + capture_slots: Vec::new(), + capture_modes: Vec::new(), + self_slot: None, + schema: None, + }], + vec![FunctionRegion { + start_ip: function_entry, + end_ip: function_end, + prototype_id: Some(0), + }], + vec![RootCallableBinding { + local_slot: 0, + prototype_id: 0, + }], + ); + + let recorded = record_trace(&program, root_ip as usize, 0, 64, &[]).unwrap(); + assert_eq!(recorded.terminal, JitTraceTerminal::LoopBack); + assert!(recorded.op_names.iter().any(|name| name == "inline_call:0")); + assert!(recorded.op_names.iter().any(|name| name == "inline_ret")); + assert!(!recorded.op_names.iter().any(|name| name == "call_value")); + assert!( + recorded + .ssa + .blocks + .iter() + .all(|block| !matches!(block.terminator, Some(SsaTerminator::CallValue { .. }))) + ); + } } diff --git a/src/vm/jit/region.rs b/src/vm/jit/region.rs index e23fbbd4..a735c93d 100644 --- a/src/vm/jit/region.rs +++ b/src/vm/jit/region.rs @@ -211,7 +211,8 @@ fn remap_inst_inputs( one!(arg); } } - SsaInstKind::UnboxInt { input } + SsaInstKind::CloneTagged { input } + | SsaInstKind::UnboxInt { input } | SsaInstKind::UnboxFloat { input } | SsaInstKind::UnboxBool { input } | SsaInstKind::UnboxHeapPtr { input, .. } diff --git a/src/vm/jit/runtime.rs b/src/vm/jit/runtime.rs index 6152094c..6f1f1deb 100644 --- a/src/vm/jit/runtime.rs +++ b/src/vm/jit/runtime.rs @@ -1641,6 +1641,9 @@ impl Vm { script_call_observations: 0, monomorphic_call_sites: 0, polymorphic_call_sites: 0, + inline_attempts: 0, + inline_successes: 0, + inline_rejections: 0, } } diff --git a/src/vm/jit/trace.rs b/src/vm/jit/trace.rs index 08742d85..459bf8e7 100644 --- a/src/vm/jit/trace.rs +++ b/src/vm/jit/trace.rs @@ -260,6 +260,9 @@ pub struct JitMetrics { pub script_call_observations: u64, pub monomorphic_call_sites: u64, pub polymorphic_call_sites: u64, + pub inline_attempts: u64, + pub inline_successes: u64, + pub inline_rejections: u64, } impl JitMetrics { @@ -871,6 +874,23 @@ impl TraceJitEngine { metrics.monomorphic_call_sites, metrics.polymorphic_call_sites )); + out.push_str(&format!( + " inline: attempts={} successes={} rejections={}\n", + metrics.inline_attempts, metrics.inline_successes, metrics.inline_rejections + )); + let mut inline_reject_reasons = std::collections::BTreeMap::<&str, u64>::new(); + for reason in self.traces.iter().flat_map(|trace| { + trace + .op_names + .iter() + .filter_map(|op| op.strip_prefix("inline_reject:")) + }) { + let count = inline_reject_reasons.entry(reason).or_default(); + *count = count.saturating_add(1); + } + for (reason, count) in inline_reject_reasons { + out.push_str(&format!(" reject {reason}: {count}\n")); + } out.push_str(&format!( " boxed value sites: loads={} stores={}\n", metrics.boxed_load_site_count, metrics.boxed_store_site_count @@ -1080,6 +1100,19 @@ impl TraceJitEngine { runtime_metrics.monomorphic_call_sites = monomorphic; runtime_metrics.polymorphic_call_sites = polymorphic; for trace in &self.traces { + for op in &trace.op_names { + if op.starts_with("inline_call:") { + runtime_metrics.inline_attempts = + runtime_metrics.inline_attempts.saturating_add(1); + runtime_metrics.inline_successes = + runtime_metrics.inline_successes.saturating_add(1); + } else if op.starts_with("inline_reject:") { + runtime_metrics.inline_attempts = + runtime_metrics.inline_attempts.saturating_add(1); + runtime_metrics.inline_rejections = + runtime_metrics.inline_rejections.saturating_add(1); + } + } runtime_metrics.boxed_load_site_count = runtime_metrics .boxed_load_site_count .saturating_add(boxed_load_site_count(&trace.ssa)); diff --git a/tests/jit/jit_tests.rs b/tests/jit/jit_tests.rs index d24cbd2d..c74aaf44 100644 --- a/tests/jit/jit_tests.rs +++ b/tests/jit/jit_tests.rs @@ -2490,7 +2490,9 @@ fn trace_jit_reports_exact_parent_exit_profiles() { .traces .iter() .enumerate() - .filter(|(_, trace)| trace.frame_key != u64::MAX) + .filter(|(_, trace)| { + trace.frame_key != u64::MAX || trace.ssa_text().contains("virtual_frame") + }) .any(|(parent_trace_id, _)| { let parent_profiles = exit_profiles .iter() @@ -6301,6 +6303,7 @@ fn trace_jit_executes_call_value_natively_inside_loop() { assert_eq!(snapshot.metrics.script_call_observations, 16); assert_eq!(snapshot.metrics.monomorphic_call_sites, 1); assert_eq!(snapshot.metrics.polymorphic_call_sites, 0); + assert!(snapshot.metrics.inline_rejections > 0); let call_sites = vm.jit_call_site_profiles(); assert_eq!(call_sites.len(), 1, "{}", vm.dump_jit_info()); assert_eq!(call_sites[0].observations, 16); @@ -6322,6 +6325,150 @@ fn trace_jit_executes_call_value_natively_inside_loop() { assert!(vm.dump_jit_info().contains("interpreter fallbacks: 0")); } +#[test] +fn trace_jit_inlines_static_leaf_in_root_loop() { + if !native_jit_supported() { + return; + } + let source = r#" + fn add_one(value: int) -> int { value + 1 } + let mut i = 0; + while i < 100 { + i = add_one(i); + } + i; + "#; + let compiled = compile_source(source).expect("static leaf loop should compile"); + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + + assert_eq!( + vm.run().expect("static leaf loop should run"), + VmStatus::Halted + ); + assert_eq!(vm.stack(), &[Value::Int(100)]); + let snapshot = vm.jit_snapshot(); + assert!(snapshot.metrics.inline_successes > 0); + assert!( + snapshot.traces.iter().any(|trace| { + trace.terminal == JitTraceTerminal::LoopBack + && trace.op_names.iter().any(|name| name == "inline_call:0") + && trace.op_names.iter().any(|name| name == "inline_ret") + && trace.executions > 0 + }), + "expected static leaf inline trace: {}", + vm.dump_jit_info() + ); + assert!(!any_trace_op(&snapshot, "call_value")); + assert!(vm.dump_jit_info().contains("interpreter fallbacks: 0")); +} + +#[test] +fn trace_jit_inlines_array_swap_leaf() { + if !native_jit_supported() { + return; + } + let source = r#" + fn swap(values: [int], left: int, right: int) -> int { + let temporary = values[left]; + values[left] = values[right]; + values[right] = temporary; + temporary + } + let values: [int] = [1, 2]; + let mut i = 0; + while i < 100 { + i = i + swap(values, 0, 1) * 0 + 1; + } + values[0] * 10 + values[1]; + "#; + let compiled = compile_source(source).expect("array swap inline source should compile"); + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + + let status = vm + .run() + .unwrap_or_else(|error| panic!("{error:?}\n{}", vm.dump_jit_info())); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(12)], "{}", vm.dump_jit_info()); + let snapshot = vm.jit_snapshot(); + assert!( + any_trace_op(&snapshot, "inline_call:0"), + "{}", + vm.dump_jit_info() + ); + assert!( + any_trace_op(&snapshot, "array_get"), + "{}", + vm.dump_jit_info() + ); + assert!( + any_trace_op(&snapshot, "array_set"), + "{}", + vm.dump_jit_info() + ); + assert!(!any_trace_op(&snapshot, "call_value")); +} + +#[test] +fn trace_jit_inline_guard_exit_restores_callee() { + if !native_jit_supported() { + return; + } + let source = r#" + fn classify(value: int) -> int { + let mut out = 1; + if value < 2 { + out = 2; + } + out + } + let mut i = 0; + let mut result = 0; + while i < 4 { + result = classify(i); + i = i + 1; + } + result; + "#; + let compiled = compile_source(source).expect("guarded inline source should compile"); + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_jit_native_bridge_stats_enabled(true); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + + let result = vm.run(); + assert_eq!( + result.unwrap_or_else(|error| panic!("{error:?}\n{}", vm.dump_jit_info())), + VmStatus::Halted + ); + assert_eq!(vm.stack(), &[Value::Int(1)]); + assert!( + any_trace_op(&vm.jit_snapshot(), "inline_call:0"), + "{}", + vm.dump_jit_info() + ); + let bridge_hits = vm.jit_native_bridge_stats_snapshot(); + assert!( + bridge_hits + .iter() + .any(|(name, hits)| *name == "restore_virtual_frame" && *hits > 0), + "expected virtual frame restore: {}", + vm.dump_jit_info() + ); +} + #[test] fn trace_jit_call_site_profiles_clear_on_vm_reuse() { let source = r#" From a71ed48a2888b990d75d58cf579e9991db68f121 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sun, 19 Jul 2026 21:17:16 +0800 Subject: [PATCH 06/26] perf(vm): reduce callable-frame hot path overhead --- src/bytecode.rs | 31 ++++++++++++++-- src/vm/mod.rs | 71 ++++++++++++++++++++++++++++++------- src/vm/native/bridge.rs | 23 +++++++----- src/vm/superinstructions.rs | 45 ++++++++++++++++++----- tests/jit/jit_tests.rs | 29 +++++++++++++++ 5 files changed, 167 insertions(+), 32 deletions(-) diff --git a/src/bytecode.rs b/src/bytecode.rs index 75dc2183..84e993e7 100644 --- a/src/bytecode.rs +++ b/src/bytecode.rs @@ -539,6 +539,7 @@ pub struct HostImport { pub(crate) struct DecodedInstructionData { pub(crate) ldc_values: Box<[Option]>, pub(crate) jump_targets: Box<[Option]>, + pub(crate) valid_jump_targets: Box<[bool]>, pub(crate) local_indices: Box<[Option]>, } @@ -546,6 +547,7 @@ impl DecodedInstructionData { fn build(program: &Program) -> Self { let mut ldc_values = vec![None; program.code.len()]; let mut jump_targets = vec![None; program.code.len()]; + let mut valid_jump_targets = vec![false; program.code.len()]; let mut local_indices = vec![None; program.code.len()]; let mut ip = 0usize; while ip < program.code.len() { @@ -562,8 +564,32 @@ impl DecodedInstructionData { } } OpCode::Br | OpCode::Brfalse => { - if let Some(target) = read_u32_at(&program.code, ip + 1) { - jump_targets[ip] = Some(target as usize); + if let Some(target) = + read_u32_at(&program.code, ip + 1).map(|target| target as usize) + { + jump_targets[ip] = Some(target); + if target >= program.code.len() { + ip = ip.saturating_add(1 + opcode.operand_len()); + continue; + } + let source_owner = program + .function_regions + .iter() + .find(|region| { + region.start_ip as usize <= ip && ip < region.end_ip as usize + }) + .and_then(|region| region.prototype_id); + let target_owner = program + .function_regions + .iter() + .find(|region| { + region.start_ip as usize <= target + && target < region.end_ip as usize + }) + .and_then(|region| region.prototype_id); + if source_owner == target_owner { + valid_jump_targets[ip] = true; + } } } OpCode::Ldloc | OpCode::Stloc => { @@ -578,6 +604,7 @@ impl DecodedInstructionData { Self { ldc_values: ldc_values.into_boxed_slice(), jump_targets: jump_targets.into_boxed_slice(), + valid_jump_targets: valid_jump_targets.into_boxed_slice(), local_indices: local_indices.into_boxed_slice(), } } diff --git a/src/vm/mod.rs b/src/vm/mod.rs index 9f368960..e61319cd 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -1041,6 +1041,7 @@ impl Vm { .unwrap_or(crate::vm::native::ROOT_FRAME_KEY) } + #[inline(always)] pub(super) fn active_local_base(&self) -> usize { self.execution_frames .last() @@ -1087,6 +1088,9 @@ impl Vm { #[inline(always)] fn load_local_value(&self, index: u8) -> VmResult { let absolute = self.absolute_local_index(index)?; + if self.capture_cells.is_empty() { + return Ok(self.locals[absolute].clone()); + } if let Some(cell) = self.capture_cells.get(&absolute) { return cell .lock() @@ -1099,6 +1103,13 @@ impl Vm { #[inline(always)] pub(super) fn local_numeric_value(&self, index: u8) -> Option { let absolute = self.absolute_local_index(index).ok()?; + if self.capture_cells.is_empty() { + return match self.locals.get(absolute)? { + Value::Int(value) => Some(NumericValue::Int(*value)), + Value::Float(value) => Some(NumericValue::Float(*value)), + _ => None, + }; + } let captured = self .capture_cells .get(&absolute) @@ -1905,6 +1916,25 @@ impl Vm { value: Value, ) -> VmResult<()> { let absolute = self.absolute_local_index(index)?; + self.store_local_absolute_with_drop_contract(absolute, index, value) + } + + #[inline(always)] + pub(super) fn store_local_absolute_with_drop_contract( + &mut self, + absolute: usize, + index: u8, + value: Value, + ) -> VmResult<()> { + if self.capture_cells.is_empty() { + let slot = self + .locals + .get_mut(absolute) + .ok_or(VmError::InvalidLocal(index))?; + let previous = std::mem::replace(slot, value); + self.drop_value_with_contract(previous); + return Ok(()); + } if let Some(cell) = self.capture_cells.get(&absolute).cloned() { if Self::value_references_capture_cell(&value, &cell, &mut HashSet::new())? { return Err(VmError::InvalidFrameState( @@ -2031,15 +2061,33 @@ impl Vm { .execution_frames .last() .and_then(|frame| frame.prototype_id); - let target_prototype = self - .program - .function_regions - .iter() - .find(|region| { - region.start_ip as usize <= target && target < region.end_ip as usize - }) - .and_then(|region| region.prototype_id); - if active_prototype != target_prototype { + let target_is_valid = if let Some(prototype_id) = active_prototype + && !self.has_aot_program() + { + self.program + .callable_prototypes + .get(prototype_id as usize) + .and_then(|prototype| match prototype.target { + CallableTarget::ScriptFunction(function_id) => { + self.program.script_functions.get(function_id as usize) + } + CallableTarget::HostImport(_) => None, + }) + .is_some_and(|function| { + function.entry_ip as usize <= target && target < function.end_ip as usize + }) + } else { + let target_prototype = self + .program + .function_regions + .iter() + .find(|region| { + region.start_ip as usize <= target && target < region.end_ip as usize + }) + .and_then(|region| region.prototype_id); + active_prototype == target_prototype + }; + if !target_is_valid { return Err(VmError::InvalidBranchTarget { target }); } } @@ -2277,7 +2325,6 @@ impl Vm { opcode: u8, allow_superinstructions: bool, ) -> VmResult { - let allow_superinstructions = allow_superinstructions && self.call_depth == 0; match opcode { x if x == OpCode::Nop as u8 => {} x if x == OpCode::Ret as u8 => return self.complete_active_frame(), @@ -2546,9 +2593,7 @@ impl Vm { } else { self.read_u8()? }; - if self.call_depth == 0 - && self.try_fuse_scalar_sequence(index, allow_superinstructions)? - { + if self.try_fuse_scalar_sequence(index, allow_superinstructions)? { return Ok(ExecOutcome::Continue); } let value = self.load_local_value(index)?; diff --git a/src/vm/native/bridge.rs b/src/vm/native/bridge.rs index ff18adb7..846f43ac 100644 --- a/src/vm/native/bridge.rs +++ b/src/vm/native/bridge.rs @@ -1171,7 +1171,6 @@ pub(crate) extern "C" fn pd_vm_native_restore_active_sparse_exit_state( .last() .map(|frame| frame.local_count) .unwrap_or(vm.locals.len()); - let mut validated_indices = Vec::with_capacity(dirty_local_count); for compact_index in 0..dirty_local_count { let local_index = unsafe { *dirty_local_indices.add(compact_index) }; let local_index_usize = usize::try_from(local_index).map_err(|_| { @@ -1184,18 +1183,19 @@ pub(crate) extern "C" fn pd_vm_native_restore_active_sparse_exit_state( "native active sparse exit restore local index {local_index} out of range for {local_count} active locals" ))); } - let local_index = u8::try_from(local_index).map_err(|_| { + u8::try_from(local_index).map_err(|_| { VmError::JitNative( "native active sparse exit restore local index exceeds VM local range" .to_string(), ) })?; - if validated_indices.contains(&local_index) { - return Err(VmError::JitNative(format!( - "native active sparse exit restore received duplicate local index {local_index}" - ))); + for prior_index in 0..compact_index { + if unsafe { *dirty_local_indices.add(prior_index) } == local_index { + return Err(VmError::JitNative(format!( + "native active sparse exit restore received duplicate local index {local_index}" + ))); + } } - validated_indices.push(local_index); } let stack_base = vm.active_operand_stack_base(); @@ -1212,7 +1212,14 @@ pub(crate) extern "C" fn pd_vm_native_restore_active_sparse_exit_state( vm.stack.push(value); } - for (compact_index, local_index) in validated_indices.into_iter().enumerate() { + for compact_index in 0..dirty_local_count { + let local_index = unsafe { *dirty_local_indices.add(compact_index) }; + let local_index = u8::try_from(local_index).map_err(|_| { + VmError::JitNative( + "native active sparse exit restore local index exceeds VM local range" + .to_string(), + ) + })?; let value = unsafe { std::ptr::read(dirty_local_values.add(compact_index)) }; vm.store_local_with_drop_contract(local_index, value)?; } diff --git a/src/vm/superinstructions.rs b/src/vm/superinstructions.rs index 72191ebc..2453be23 100644 --- a/src/vm/superinstructions.rs +++ b/src/vm/superinstructions.rs @@ -27,10 +27,15 @@ impl ScalarValue { impl Vm { #[inline(always)] - fn local_scalar_value_with_hint(&mut self, index: u8) -> Option { + fn local_scalar_value_with_hint( + &mut self, + local_base: usize, + index: u8, + ) -> Option { + let absolute = local_base.checked_add(index as usize)?; match self.local_type_hint(index) { ValueType::Int => { - let value = match self.locals.get(index as usize)? { + let value = match self.locals.get(absolute)? { Value::Int(value) => *value, _ => return None, }; @@ -38,7 +43,7 @@ impl Vm { Some(ScalarValue::Int(value)) } ValueType::Float => { - let value = match self.locals.get(index as usize)? { + let value = match self.locals.get(absolute)? { Value::Float(value) => *value, _ => return None, }; @@ -68,6 +73,15 @@ impl Vm { .and_then(|target| *target) } + #[inline(always)] + pub(super) fn decoded_jump_target_is_valid_at(&self, opcode_ip: usize) -> bool { + self.decoded_instruction_data + .valid_jump_targets + .get(opcode_ip) + .copied() + .unwrap_or(false) + } + #[inline(always)] pub(super) fn decoded_local_index_at(&self, opcode_ip: usize) -> Option { self.decoded_instruction_data @@ -85,7 +99,8 @@ impl Vm { if !allow_superinstructions { return Ok(false); } - let Some(initial) = self.local_scalar_value_with_hint(src) else { + let local_base = self.active_local_base(); + let Some(initial) = self.local_scalar_value_with_hint(local_base, src) else { return Ok(false); }; let mut cursor = self.ip; @@ -120,7 +135,7 @@ impl Vm { let Some(index) = self.decoded_local_index_at(cursor) else { return Ok(false); }; - let Some(value) = self.local_scalar_value_with_hint(index) else { + let Some(value) = self.local_scalar_value_with_hint(local_base, index) else { return Ok(false); }; if stack_len == stack.len() { @@ -211,7 +226,13 @@ impl Vm { .take() .expect("result scalar should exist") .into_value(); - self.store_local_with_drop_contract(dst, value)?; + let absolute = + local_base + .checked_add(dst as usize) + .ok_or(VmError::InvalidFrameState( + "superinstruction local index overflow", + ))?; + self.store_local_absolute_with_drop_contract(absolute, dst, value)?; self.record_scalar_superinstruction(); self.ip = cursor + 2; return Ok(true); @@ -220,10 +241,12 @@ impl Vm { if stack_len != 2 { return Ok(false); } - if self.program.code.get(cursor + 1).copied() != Some(OpCode::Brfalse as u8) { + let jump_opcode_ip = cursor + 1; + if self.program.code.get(jump_opcode_ip).copied() != Some(OpCode::Brfalse as u8) + { return Ok(false); } - let Some(target) = self.decoded_jump_target_at(cursor + 1) else { + let Some(target) = self.decoded_jump_target_at(jump_opcode_ip) else { return Ok(false); }; let rhs = stack[stack_len - 1] @@ -253,7 +276,11 @@ impl Vm { }; self.ip = cursor + 6; if !condition { - self.jump_to(target)?; + if self.decoded_jump_target_is_valid_at(jump_opcode_ip) { + self.ip = target; + } else { + self.jump_to(target)?; + } } self.record_scalar_superinstruction(); return Ok(true); diff --git a/tests/jit/jit_tests.rs b/tests/jit/jit_tests.rs index c74aaf44..b9f79c15 100644 --- a/tests/jit/jit_tests.rs +++ b/tests/jit/jit_tests.rs @@ -6325,6 +6325,35 @@ fn trace_jit_executes_call_value_natively_inside_loop() { assert!(vm.dump_jit_info().contains("interpreter fallbacks: 0")); } +#[test] +fn interpreter_superinstructions_use_script_frame_local_base() { + let source = r#" + fn count() -> int { + let mut i = 0; + while i < 100 { + i = i + 1; + } + i + } + count(); + "#; + let compiled = + compile_source(source).expect("script-frame superinstruction source should compile"); + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_jit_config(JitConfig { + enabled: false, + ..JitConfig::default() + }); + + assert_eq!(vm.run().unwrap(), VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(100)]); + assert!( + vm.interpreter_metrics_snapshot() + .scalar_superinstruction_count + > 0 + ); +} + #[test] fn trace_jit_inlines_static_leaf_in_root_loop() { if !native_jit_supported() { From ec3e1c999d930cea0db61f97f21788e9de4fe95f Mon Sep 17 00:00:00 2001 From: fffonion Date: Sun, 19 Jul 2026 23:17:18 +0800 Subject: [PATCH 07/26] perf(vm): streamline interpreter and exit hot paths --- src/vm/host.rs | 36 ++++++++++++ src/vm/mod.rs | 124 ++++++++++++++++++++++++++++++++++++---- src/vm/native/bridge.rs | 25 +++++--- 3 files changed, 166 insertions(+), 19 deletions(-) diff --git a/src/vm/host.rs b/src/vm/host.rs index 83f9ee4b..ecca479a 100644 --- a/src/vm/host.rs +++ b/src/vm/host.rs @@ -994,6 +994,20 @@ impl Vm { .get(usize::from(index)) .map(|import| import.return_type); let resolved_index = self.resolve_call_target(index, argc_u8)?; + if let Some(function) = + self.host_functions + .get(resolved_index as usize) + .and_then(|function| match function { + VmHostFunction::ArgsStaticNonYielding(function) => Some(*function), + _ => None, + }) + { + return self.execute_static_non_yielding_args_host_function( + function, + argc, + expected_return_type, + ); + } if self.bound_host_function_uses_args_slice(resolved_index)? { self.execute_bound_args_host_function( resolved_index, @@ -1537,6 +1551,28 @@ impl Vm { )) } + #[inline(always)] + fn execute_static_non_yielding_args_host_function( + &mut self, + function: StaticHostArgsFunction, + argc: usize, + expected_return_type: Option, + ) -> VmResult { + let arg_start = self + .stack + .len() + .checked_sub(argc) + .ok_or(VmError::StackUnderflow)?; + self.call_depth += 1; + let outcome = function(&self.stack[arg_start..]); + self.call_depth = self.call_depth.saturating_sub(1); + let value = require_non_yielding_host_value(outcome?)?; + let value = validate_non_yielding_host_value(value, expected_return_type)?; + self.stack.truncate(arg_start); + self.stack.push(value); + Ok(HostCallExecOutcome::Returned) + } + pub(super) fn execute_bound_args_host_function( &mut self, resolved_index: u16, diff --git a/src/vm/mod.rs b/src/vm/mod.rs index e61319cd..b24cba49 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -346,6 +346,8 @@ pub struct Vm { call_depth: usize, max_script_call_depth: usize, execution_frames: Vec, + active_local_base_cache: usize, + active_operand_stack_base_cache: usize, host_return: Option, queued_callables: VecDeque, completed_callable_results: VecDeque, @@ -675,6 +677,8 @@ impl Vm { call_depth: 0, max_script_call_depth: DEFAULT_MAX_SCRIPT_CALL_DEPTH, execution_frames: vec![ExecutionFrame::root(local_count)], + active_local_base_cache: 0, + active_operand_stack_base_cache: 0, host_return: None, queued_callables: VecDeque::new(), completed_callable_results: VecDeque::new(), @@ -918,6 +922,8 @@ impl Vm { self.execution_frames.clear(); self.execution_frames .push(ExecutionFrame::root(self.program.local_count)); + self.active_local_base_cache = 0; + self.active_operand_stack_base_cache = 0; self.host_return = None; self.queued_callables.clear(); self.completed_callable_results.clear(); @@ -1019,10 +1025,7 @@ impl Vm { #[inline(always)] pub(super) fn active_operand_stack_base(&self) -> usize { - self.execution_frames - .last() - .map(|frame| frame.operand_stack_base) - .unwrap_or(0) + self.active_operand_stack_base_cache } #[inline(always)] @@ -1043,10 +1046,7 @@ impl Vm { #[inline(always)] pub(super) fn active_local_base(&self) -> usize { - self.execution_frames - .last() - .map(|frame| frame.local_base) - .unwrap_or(0) + self.active_local_base_cache } pub(super) fn active_local_types(&self) -> Vec { @@ -1091,13 +1091,22 @@ impl Vm { if self.capture_cells.is_empty() { return Ok(self.locals[absolute].clone()); } + self.load_local_value_with_captures(absolute, index) + } + + #[cold] + #[inline(never)] + fn load_local_value_with_captures(&self, absolute: usize, index: u8) -> VmResult { if let Some(cell) = self.capture_cells.get(&absolute) { return cell .lock() .map(|value| value.clone()) .map_err(|_| VmError::InvalidFrameState("capture cell lock is poisoned")); } - Ok(self.locals[absolute].clone()) + self.locals + .get(absolute) + .cloned() + .ok_or(VmError::InvalidLocal(index)) } #[inline(always)] @@ -1110,6 +1119,12 @@ impl Vm { _ => None, }; } + self.local_numeric_value_with_captures(absolute) + } + + #[cold] + #[inline(never)] + fn local_numeric_value_with_captures(&self, absolute: usize) -> Option { let captured = self .capture_cells .get(&absolute) @@ -1412,6 +1427,8 @@ impl Vm { local_count, prototype_id: Some(callable.prototype_id), }); + self.active_local_base_cache = local_base; + self.active_operand_stack_base_cache = operand_stack_base; self.call_depth = self.script_frame_depth(); self.ip = function.entry_ip as usize; self.charge_interrupt_tick()?; @@ -1439,6 +1456,16 @@ impl Vm { .execution_frames .pop() .ok_or(VmError::InvalidFrameState("missing active frame"))?; + self.active_local_base_cache = self + .execution_frames + .last() + .map(|frame| frame.local_base) + .unwrap_or(0); + self.active_operand_stack_base_cache = self + .execution_frames + .last() + .map(|frame| frame.operand_stack_base) + .unwrap_or(0); if self.stack.len() < frame.operand_stack_base { return Err(VmError::InvalidFrameState( "operand stack is below the active frame base", @@ -1935,6 +1962,17 @@ impl Vm { self.drop_value_with_contract(previous); return Ok(()); } + self.store_local_with_captures(absolute, index, value) + } + + #[cold] + #[inline(never)] + fn store_local_with_captures( + &mut self, + absolute: usize, + index: u8, + value: Value, + ) -> VmResult<()> { if let Some(cell) = self.capture_cells.get(&absolute).cloned() { if Self::value_references_capture_cell(&value, &cell, &mut HashSet::new())? { return Err(VmError::InvalidFrameState( @@ -2172,6 +2210,39 @@ impl Vm { result } + fn run_fast_interpreter(&mut self, allow_jit: bool) -> VmResult> { + loop { + if self.ip >= self.program.code.len() { + return Err(VmError::BytecodeBounds); + } + let opcode = self.read_u8()?; + let outcome = self.execute_interpreter_instruction(opcode, true)?; + match outcome { + ExecOutcome::Continue => {} + ExecOutcome::Halted => { + self.last_yield_reason = None; + return Ok(Some(VmStatus::Halted)); + } + ExecOutcome::Yielded => { + if self.last_yield_reason.is_none() { + self.last_yield_reason = Some(VmYieldReason::Host); + } + return Ok(Some(VmStatus::Yielded)); + } + ExecOutcome::Waiting(op_id) => { + self.last_yield_reason = None; + return Ok(Some(VmStatus::Waiting(op_id))); + } + } + if (opcode == OpCode::Call as u8 || opcode == OpCode::CallValue as u8) + && (self.interruption_enabled() + || (allow_jit && (self.jit_config().enabled || self.has_aot_program()))) + { + return Ok(None); + } + } + } + fn run_internal_impl( &mut self, mut debugger: Option<&mut crate::debugger::Debugger>, @@ -2186,6 +2257,16 @@ impl Vm { return Ok(status); } self.last_yield_reason = None; + if self.epoch_rearm_pending { + self.rearm_epoch_after_yield_if_needed(); + } + if debugger.is_none() + && !self.interruption_enabled() + && (!allow_jit || (!self.jit_config().enabled && !self.has_aot_program())) + && let Some(status) = self.run_fast_interpreter(allow_jit)? + { + return Ok(status); + } loop { if self.epoch_rearm_pending { @@ -2320,6 +2401,7 @@ impl Vm { } } + #[inline(always)] pub(super) fn execute_interpreter_instruction( &mut self, opcode: u8, @@ -2563,7 +2645,11 @@ impl Vm { } else { self.read_u32()? as usize }; - self.jump_to(target)?; + if self.decoded_jump_target_is_valid_at(opcode_ip) { + self.ip = target; + } else { + self.jump_to(target)?; + } } x if x == OpCode::Brfalse as u8 => { let opcode_ip = self.ip - 1; @@ -2575,7 +2661,11 @@ impl Vm { }; let condition = self.pop_bool()?; if !condition { - self.jump_to(target)?; + if self.decoded_jump_target_is_valid_at(opcode_ip) { + self.ip = target; + } else { + self.jump_to(target)?; + } } } x if x == OpCode::Pop as u8 => { @@ -2822,6 +2912,8 @@ impl Vm { self.capture_cells.clear(); self.clear_locals_with_drop_contract(); self.execution_frames.clear(); + self.active_local_base_cache = 0; + self.active_operand_stack_base_cache = 0; self.call_depth = 0; self.host_return = None; self.waiting_host_op = None; @@ -2936,6 +3028,16 @@ impl Vm { } } } + self.active_local_base_cache = self + .execution_frames + .last() + .map(|frame| frame.local_base) + .unwrap_or(0); + self.active_operand_stack_base_cache = self + .execution_frames + .last() + .map(|frame| frame.operand_stack_base) + .unwrap_or(0); while self.stack.len() > stack_base { if let Some(value) = self.stack.pop() { self.drop_value_with_contract(value); diff --git a/src/vm/native/bridge.rs b/src/vm/native/bridge.rs index 846f43ac..4099f47c 100644 --- a/src/vm/native/bridge.rs +++ b/src/vm/native/bridge.rs @@ -1171,6 +1171,7 @@ pub(crate) extern "C" fn pd_vm_native_restore_active_sparse_exit_state( .last() .map(|frame| frame.local_count) .unwrap_or(vm.locals.len()); + let mut seen_local_indices = [0u64; 4]; for compact_index in 0..dirty_local_count { let local_index = unsafe { *dirty_local_indices.add(compact_index) }; let local_index_usize = usize::try_from(local_index).map_err(|_| { @@ -1183,19 +1184,20 @@ pub(crate) extern "C" fn pd_vm_native_restore_active_sparse_exit_state( "native active sparse exit restore local index {local_index} out of range for {local_count} active locals" ))); } - u8::try_from(local_index).map_err(|_| { + let local_index = u8::try_from(local_index).map_err(|_| { VmError::JitNative( "native active sparse exit restore local index exceeds VM local range" .to_string(), ) })?; - for prior_index in 0..compact_index { - if unsafe { *dirty_local_indices.add(prior_index) } == local_index { - return Err(VmError::JitNative(format!( - "native active sparse exit restore received duplicate local index {local_index}" - ))); - } + let word = usize::from(local_index) / 64; + let bit = 1u64 << (local_index % 64); + if seen_local_indices[word] & bit != 0 { + return Err(VmError::JitNative(format!( + "native active sparse exit restore received duplicate local index {local_index}" + ))); } + seen_local_indices[word] |= bit; } let stack_base = vm.active_operand_stack_base(); @@ -1224,7 +1226,10 @@ pub(crate) extern "C" fn pd_vm_native_restore_active_sparse_exit_state( vm.store_local_with_drop_contract(local_index, value)?; } - vm.jump_to(ip)?; + if ip >= vm.program.code.len() { + return Err(VmError::InvalidBranchTarget { target: ip }); + } + vm.ip = ip; Ok(STATUS_CONTINUE) }) } @@ -1316,6 +1321,8 @@ pub(crate) extern "C" fn pd_vm_native_restore_virtual_frame( local_count: locals_len, prototype_id: Some(prototype_id), }); + vm.active_local_base_cache = local_base; + vm.active_operand_stack_base_cache = operand_stack_base; vm.call_depth = vm.script_frame_depth(); vm.ip = resume_ip; Ok(STATUS_CONTINUE) @@ -2082,6 +2089,8 @@ mod tests { local_count: 3, prototype_id: Some(7), }); + vm.active_local_base_cache = 2; + vm.active_operand_stack_base_cache = 1; vm.call_depth = 1; let mut state = MaybeUninit::::uninit(); From 9f5f98311cf4a500a870c7326b168d3d657f12c6 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sun, 19 Jul 2026 23:29:53 +0800 Subject: [PATCH 08/26] perf(jit): trust active sparse exit metadata --- src/vm/native/bridge.rs | 44 +++++------------------------------------ 1 file changed, 5 insertions(+), 39 deletions(-) diff --git a/src/vm/native/bridge.rs b/src/vm/native/bridge.rs index 4099f47c..848a4097 100644 --- a/src/vm/native/bridge.rs +++ b/src/vm/native/bridge.rs @@ -1166,39 +1166,9 @@ pub(crate) extern "C" fn pd_vm_native_restore_active_sparse_exit_state( )); } - let local_count = vm - .execution_frames - .last() - .map(|frame| frame.local_count) - .unwrap_or(vm.locals.len()); - let mut seen_local_indices = [0u64; 4]; - for compact_index in 0..dirty_local_count { - let local_index = unsafe { *dirty_local_indices.add(compact_index) }; - let local_index_usize = usize::try_from(local_index).map_err(|_| { - VmError::JitNative( - "native active sparse exit restore local index out of range".to_string(), - ) - })?; - if local_index_usize >= local_count { - return Err(VmError::JitNative(format!( - "native active sparse exit restore local index {local_index} out of range for {local_count} active locals" - ))); - } - let local_index = u8::try_from(local_index).map_err(|_| { - VmError::JitNative( - "native active sparse exit restore local index exceeds VM local range" - .to_string(), - ) - })?; - let word = usize::from(local_index) / 64; - let bit = 1u64 << (local_index % 64); - if seen_local_indices[word] & bit != 0 { - return Err(VmError::JitNative(format!( - "native active sparse exit restore received duplicate local index {local_index}" - ))); - } - seen_local_indices[word] |= bit; - } + // This entry point is emitted only by the native lowering pipeline. Dirty local + // indices are compile-time metadata whose range and uniqueness are guaranteed + // while the sparse exit metadata is built. let stack_base = vm.active_operand_stack_base(); if stack_base > vm.stack.len() { @@ -1216,12 +1186,8 @@ pub(crate) extern "C" fn pd_vm_native_restore_active_sparse_exit_state( for compact_index in 0..dirty_local_count { let local_index = unsafe { *dirty_local_indices.add(compact_index) }; - let local_index = u8::try_from(local_index).map_err(|_| { - VmError::JitNative( - "native active sparse exit restore local index exceeds VM local range" - .to_string(), - ) - })?; + debug_assert!(u8::try_from(local_index).is_ok()); + let local_index = local_index as u8; let value = unsafe { std::ptr::read(dirty_local_values.add(compact_index)) }; vm.store_local_with_drop_contract(local_index, value)?; } From bc8f6fb14d9eb3a9b6bbc0ee22cfb3841a98eeac Mon Sep 17 00:00:00 2001 From: fffonion Date: Sun, 19 Jul 2026 23:40:00 +0800 Subject: [PATCH 09/26] perf(jit): restore capture-free exits directly --- src/vm/native/bridge.rs | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/vm/native/bridge.rs b/src/vm/native/bridge.rs index 848a4097..0e410143 100644 --- a/src/vm/native/bridge.rs +++ b/src/vm/native/bridge.rs @@ -1184,12 +1184,26 @@ pub(crate) extern "C" fn pd_vm_native_restore_active_sparse_exit_state( vm.stack.push(value); } - for compact_index in 0..dirty_local_count { - let local_index = unsafe { *dirty_local_indices.add(compact_index) }; - debug_assert!(u8::try_from(local_index).is_ok()); - let local_index = local_index as u8; - let value = unsafe { std::ptr::read(dirty_local_values.add(compact_index)) }; - vm.store_local_with_drop_contract(local_index, value)?; + if vm.capture_cells.is_empty() { + let local_base = vm.active_local_base(); + for compact_index in 0..dirty_local_count { + let local_index = unsafe { *dirty_local_indices.add(compact_index) } as usize; + debug_assert!(local_index < 256); + let absolute = local_base + local_index; + debug_assert!(absolute < vm.locals.len()); + let value = unsafe { std::ptr::read(dirty_local_values.add(compact_index)) }; + let slot = unsafe { vm.locals.get_unchecked_mut(absolute) }; + let previous = std::mem::replace(slot, value); + vm.drop_value_with_contract(previous); + } + } else { + for compact_index in 0..dirty_local_count { + let local_index = unsafe { *dirty_local_indices.add(compact_index) }; + debug_assert!(u8::try_from(local_index).is_ok()); + let local_index = local_index as u8; + let value = unsafe { std::ptr::read(dirty_local_values.add(compact_index)) }; + vm.store_local_with_drop_contract(local_index, value)?; + } } if ip >= vm.program.code.len() { From 4e8bc4b70bec6f61a71a3c7062a1b90d5b2e9849 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sun, 19 Jul 2026 23:50:12 +0800 Subject: [PATCH 10/26] perf(jit): inline sparse exit drop handling --- src/vm/native/bridge.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/vm/native/bridge.rs b/src/vm/native/bridge.rs index 0e410143..bfa4a0f1 100644 --- a/src/vm/native/bridge.rs +++ b/src/vm/native/bridge.rs @@ -1186,6 +1186,7 @@ pub(crate) extern "C" fn pd_vm_native_restore_active_sparse_exit_state( if vm.capture_cells.is_empty() { let local_base = vm.active_local_base(); + let count_drop_events = vm.drop_contract_events_enabled; for compact_index in 0..dirty_local_count { let local_index = unsafe { *dirty_local_indices.add(compact_index) } as usize; debug_assert!(local_index < 256); @@ -1194,7 +1195,9 @@ pub(crate) extern "C" fn pd_vm_native_restore_active_sparse_exit_state( let value = unsafe { std::ptr::read(dirty_local_values.add(compact_index)) }; let slot = unsafe { vm.locals.get_unchecked_mut(absolute) }; let previous = std::mem::replace(slot, value); - vm.drop_value_with_contract(previous); + if count_drop_events { + vm.count_value_drop_contract(&previous); + } } } else { for compact_index in 0..dirty_local_count { From c2bec7ffc0d814bc0bcd37058f68eba191cb3f2e Mon Sep 17 00:00:00 2001 From: fffonion Date: Mon, 20 Jul 2026 00:28:51 +0800 Subject: [PATCH 11/26] perf(jit): restore heap exits inline --- src/vm/jit/native/lower.rs | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index d6b6db06..581f97a2 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -4726,7 +4726,7 @@ fn lower_ssa_exit_block( .contains_key(&SsaTempValueSlotKey::Output(*value)) && moved_owned_values.insert(*value) } - SsaMaterialization::BoxHeapPtr { .. } => false, + SsaMaterialization::BoxHeapPtr { .. } => true, } }); if inline_owned_restore { @@ -4755,15 +4755,24 @@ fn lower_ssa_exit_block( })?, ); let dst_addr = ssa_value_addr(b, pointer_type, vm_locals_ptr, index, layout.value.size); - clear_owned_value_temp_slot(b, pointer_type, deopt_refs, deopt_addrs, dst_addr)?; - if let SsaMaterialization::Value(value) = materialization { - let src = *exit_values.get(value).ok_or_else(|| { - VmError::JitNative("SSA exit tagged local value missing".to_string()) - })?; - ssa_copy_value_bytes(b, src, dst_addr, layout.value.size); - ssa_store_tag(b, layout.value, src, layout.value.null_tag); + if matches!(materialization, SsaMaterialization::BoxHeapPtr { .. }) { + let temp_slot = ssa_create_value_stack_slot(b, layout.value.size)?; + let temp_addr = b.ins().stack_addr(pointer_type, temp_slot, 0); + ssa_materialize_slot(b, materialize_ctx, materialization, temp_addr, "local")?; + clear_owned_value_temp_slot(b, pointer_type, deopt_refs, deopt_addrs, dst_addr)?; + ssa_copy_value_bytes(b, temp_addr, dst_addr, layout.value.size); + ssa_store_tag(b, layout.value, temp_addr, layout.value.null_tag); } else { - ssa_materialize_slot(b, materialize_ctx, materialization, dst_addr, "local")?; + clear_owned_value_temp_slot(b, pointer_type, deopt_refs, deopt_addrs, dst_addr)?; + if let SsaMaterialization::Value(value) = materialization { + let src = *exit_values.get(value).ok_or_else(|| { + VmError::JitNative("SSA exit tagged local value missing".to_string()) + })?; + ssa_copy_value_bytes(b, src, dst_addr, layout.value.size); + ssa_store_tag(b, layout.value, src, layout.value.null_tag); + } else { + ssa_materialize_slot(b, materialize_ctx, materialization, dst_addr, "local")?; + } } } let ip_val = b.ins().iconst( From c0922472b792f9d62c066a7fca0d1a881c26f8c5 Mon Sep 17 00:00:00 2001 From: fffonion Date: Mon, 20 Jul 2026 00:47:49 +0800 Subject: [PATCH 12/26] perf(jit): clone borrowed exits inline --- src/vm/jit/native/lower.rs | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index 581f97a2..98a1eb2c 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -4704,6 +4704,12 @@ fn lower_ssa_exit_block( deopt_addrs, }; + let mut tagged_local_counts = HashMap::::new(); + for (materialization, dirty) in exit.locals.iter().zip(&exit.dirty_locals) { + if *dirty && let SsaMaterialization::Value(value) = materialization { + *tagged_local_counts.entry(*value).or_default() += 1; + } + } let mut moved_owned_values = BTreeSet::new(); let inline_owned_restore = exit.virtual_frames.is_empty() && entry_stack_depth == 0 @@ -4721,10 +4727,14 @@ fn lower_ssa_exit_block( | SsaMaterialization::BoxBool(_) | SsaMaterialization::BoxFloat(_) => true, SsaMaterialization::Value(value) => { - owned_value_temps - .slots - .contains_key(&SsaTempValueSlotKey::Output(*value)) - && moved_owned_values.insert(*value) + if tagged_local_counts.get(value) == Some(&1) + && owned_value_temps + .slots + .contains_key(&SsaTempValueSlotKey::Output(*value)) + { + moved_owned_values.insert(*value); + } + true } SsaMaterialization::BoxHeapPtr { .. } => true, } @@ -4755,7 +4765,12 @@ fn lower_ssa_exit_block( })?, ); let dst_addr = ssa_value_addr(b, pointer_type, vm_locals_ptr, index, layout.value.size); - if matches!(materialization, SsaMaterialization::BoxHeapPtr { .. }) { + let clone_before_clear = match materialization { + SsaMaterialization::BoxHeapPtr { .. } => true, + SsaMaterialization::Value(value) => !moved_owned_values.contains(value), + _ => false, + }; + if clone_before_clear { let temp_slot = ssa_create_value_stack_slot(b, layout.value.size)?; let temp_addr = b.ins().stack_addr(pointer_type, temp_slot, 0); ssa_materialize_slot(b, materialize_ctx, materialization, temp_addr, "local")?; From 8f7a4abf6d04944c0ee621a46ceee72953237d86 Mon Sep 17 00:00:00 2001 From: fffonion Date: Mon, 20 Jul 2026 01:10:08 +0800 Subject: [PATCH 13/26] perf(jit): trust admitted inherited links --- src/vm/jit/native/lower.rs | 57 +++----------------------------------- 1 file changed, 4 insertions(+), 53 deletions(-) diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index 98a1eb2c..9bb07f3e 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -836,7 +836,6 @@ fn try_compile_ssa_trace( let normal_entry = b.create_block(); if entry_ssa_block.params.len() <= MAX_INHERITED_ENTRY_VALUES { - let inherited_candidate = b.create_block(); let inherited_entry = b.create_block(); let active = b.ins().load( pointer_type, @@ -845,59 +844,11 @@ fn try_compile_ssa_trace( INHERITED_STATE_ACTIVE_OFFSET, ); let has_inherited_state = b.ins().icmp_imm(IntCC::NotEqual, active, 0); - b.ins().brif( - has_inherited_state, - inherited_candidate, - &[], - normal_entry, - &[], - ); - - b.switch_to_block(inherited_candidate); - let inherited_frame_key = b.ins().load( - pointer_type, - MemFlags::new(), - inherited_state_ptr, - INHERITED_STATE_FRAME_KEY_OFFSET, - ); - let expected_frame_key = b.ins().iconst(pointer_type, trace.frame_key as i64); - let frame_matches = b - .ins() - .icmp(IntCC::Equal, inherited_frame_key, expected_frame_key); - let inherited_target_ip = b.ins().load( - pointer_type, - MemFlags::new(), - inherited_state_ptr, - INHERITED_STATE_TARGET_IP_OFFSET, - ); - let expected_target_ip = b.ins().iconst( - pointer_type, - i64::try_from(trace.root_ip).map_err(|_| { - VmError::JitNative("SSA inherited target ip exceeds i64".to_string()) - })?, - ); - let target_matches = - b.ins() - .icmp(IntCC::Equal, inherited_target_ip, expected_target_ip); - let inherited_value_count = b.ins().load( - pointer_type, - MemFlags::new(), - inherited_state_ptr, - INHERITED_STATE_VALUE_COUNT_OFFSET, - ); - let expected_value_count = b.ins().iconst( - pointer_type, - i64::try_from(entry_ssa_block.params.len()).map_err(|_| { - VmError::JitNative("SSA inherited value count exceeds i64".to_string()) - })?, - ); - let value_count_matches = - b.ins() - .icmp(IntCC::Equal, inherited_value_count, expected_value_count); - let metadata_matches = b.ins().band(frame_matches, target_matches); - let metadata_matches = b.ins().band(metadata_matches, value_count_matches); + // Active inherited packets are created only by admitted native links. Link + // publication already validates frame, target, and entry shape, and invalidation + // clears the slot before the packet can be reused. b.ins() - .brif(metadata_matches, inherited_entry, &[], normal_entry, &[]); + .brif(has_inherited_state, inherited_entry, &[], normal_entry, &[]); b.switch_to_block(inherited_entry); let active_stack_base = b.ins().load( From dd1db5f3ad7f5671ba330a4bfcfbdfd3a4018ae7 Mon Sep 17 00:00:00 2001 From: fffonion Date: Mon, 20 Jul 2026 02:13:14 +0800 Subject: [PATCH 14/26] fix(jit): preserve owned values across native writes --- src/vm/jit/native/lower.rs | 57 +++++++++++++++++++++++++------------- src/vm/native/bridge.rs | 36 ++++++++++++++++++++++++ src/vm/native/mod.rs | 18 ++++++------ tests/jit/jit_tests.rs | 57 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 29 deletions(-) diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index 9bb07f3e..f077fc09 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -27,16 +27,16 @@ use crate::vm::native::{ non_yielding_i64_host_call_signature, non_yielding_scalar_host_call_entry_address, non_yielding_scalar_host_call_signature, pack_shared_signature, regex_match_entry_address, regex_match_signature, regex_replace_entry_address, regex_replace_signature, - restore_active_sparse_exit_state_entry_address, restore_virtual_frame_entry_address, - restore_virtual_frame_signature, shared_array_from_buffer_entry_address, - shared_bytes_from_buffer_entry_address, shared_string_from_buffer_entry_address, - sparse_restore_exit_signature, string_binary_transform_signature, - string_contains_entry_address, string_contains_signature, string_lower_ascii_entry_address, - string_replace_literal_entry_address, string_replace_signature, - string_split_literal_entry_address, string_unary_transform_signature, to_string_entry_address, - type_of_entry_address, value_eq_entry_address, value_eq_signature, value_len_entry_address, - value_len_signature, value_slot_signature, write_heap_value_to_slot_entry_address, - zero_bytes_entry_address, + replace_value_in_slot_entry_address, restore_active_sparse_exit_state_entry_address, + restore_virtual_frame_entry_address, restore_virtual_frame_signature, + shared_array_from_buffer_entry_address, shared_bytes_from_buffer_entry_address, + shared_string_from_buffer_entry_address, sparse_restore_exit_signature, + string_binary_transform_signature, string_contains_entry_address, string_contains_signature, + string_lower_ascii_entry_address, string_replace_literal_entry_address, + string_replace_signature, string_split_literal_entry_address, string_unary_transform_signature, + to_string_entry_address, type_of_entry_address, value_eq_entry_address, value_eq_signature, + value_len_entry_address, value_len_signature, value_slot_signature, + write_heap_value_to_slot_entry_address, zero_bytes_entry_address, }; use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; use cranelift_codegen::ir::immediates::Ieee64; @@ -667,7 +667,8 @@ fn try_compile_ssa_trace( }; let frame_state_ref = b.import_signature(frame_state_sig); let deopt_refs = SsaDeoptHelperRefs { - clone_value_ref: b.import_signature(clone_value_sig), + clone_value_ref: b.import_signature(clone_value_sig.clone()), + replace_value_ref: b.import_signature(clone_value_sig), value_eq_ref: b.import_signature(value_eq_sig), value_len_ref: b.import_signature(value_len_sig), non_yielding_host_call_ref: b.import_signature(non_yielding_host_call_sig), @@ -693,6 +694,7 @@ fn try_compile_ssa_trace( }; let deopt_addrs = SsaDeoptHelperAddrs { clone_value: clone_value_to_slot_entry_address(), + replace_value: replace_value_in_slot_entry_address(), value_eq: value_eq_entry_address(), value_len: value_len_entry_address(), non_yielding_host_call: non_yielding_host_call_entry_address(), @@ -1104,6 +1106,7 @@ enum SsaExitAction { #[derive(Clone, Copy)] struct SsaDeoptHelperRefs { clone_value_ref: cranelift_codegen::ir::SigRef, + replace_value_ref: cranelift_codegen::ir::SigRef, value_eq_ref: cranelift_codegen::ir::SigRef, value_len_ref: cranelift_codegen::ir::SigRef, non_yielding_host_call_ref: cranelift_codegen::ir::SigRef, @@ -1130,6 +1133,7 @@ struct SsaDeoptHelperRefs { #[derive(Clone, Copy)] struct SsaDeoptHelperAddrs { clone_value: usize, + replace_value: usize, value_eq: usize, value_len: usize, non_yielding_host_call: usize, @@ -2080,8 +2084,8 @@ fn lower_ssa_inst( b, exit_block, pointer_type, - helper_refs.clone_value_ref, - helper_addrs.clone_value, + helper_refs.replace_value_ref, + helper_addrs.replace_value, &[out, values[input]], )?; out @@ -4701,6 +4705,7 @@ fn lower_ssa_exit_block( active_local_base, layout.value.size, ); + let mut cloned_local_addrs = HashMap::new(); for (local_index, materialization) in exit.locals .iter() @@ -4709,13 +4714,6 @@ fn lower_ssa_exit_block( exit.dirty_locals[local_index].then_some((local_index, materialization)) }) { - let index = b.ins().iconst( - pointer_type, - i64::try_from(local_index).map_err(|_| { - VmError::JitNative("SSA dirty local index out of range".to_string()) - })?, - ); - let dst_addr = ssa_value_addr(b, pointer_type, vm_locals_ptr, index, layout.value.size); let clone_before_clear = match materialization { SsaMaterialization::BoxHeapPtr { .. } => true, SsaMaterialization::Value(value) => !moved_owned_values.contains(value), @@ -4725,6 +4723,25 @@ fn lower_ssa_exit_block( let temp_slot = ssa_create_value_stack_slot(b, layout.value.size)?; let temp_addr = b.ins().stack_addr(pointer_type, temp_slot, 0); ssa_materialize_slot(b, materialize_ctx, materialization, temp_addr, "local")?; + cloned_local_addrs.insert(local_index, temp_addr); + } + } + for (local_index, materialization) in + exit.locals + .iter() + .enumerate() + .filter_map(|(local_index, materialization)| { + exit.dirty_locals[local_index].then_some((local_index, materialization)) + }) + { + let index = b.ins().iconst( + pointer_type, + i64::try_from(local_index).map_err(|_| { + VmError::JitNative("SSA dirty local index out of range".to_string()) + })?, + ); + let dst_addr = ssa_value_addr(b, pointer_type, vm_locals_ptr, index, layout.value.size); + if let Some(temp_addr) = cloned_local_addrs.get(&local_index).copied() { clear_owned_value_temp_slot(b, pointer_type, deopt_refs, deopt_addrs, dst_addr)?; ssa_copy_value_bytes(b, temp_addr, dst_addr, layout.value.size); ssa_store_tag(b, layout.value, temp_addr, layout.value.null_tag); diff --git a/src/vm/native/bridge.rs b/src/vm/native/bridge.rs index bfa4a0f1..94740ca6 100644 --- a/src/vm/native/bridge.rs +++ b/src/vm/native/bridge.rs @@ -264,6 +264,10 @@ pub(crate) fn clone_value_to_slot_entry_address() -> usize { pd_vm_native_clone_value_to_slot as *const () as usize } +pub(crate) fn replace_value_in_slot_entry_address() -> usize { + pd_vm_native_replace_value_in_slot as *const () as usize +} + pub(crate) fn init_null_value_slot_entry_address() -> usize { pd_vm_native_init_null_value_slot as *const () as usize } @@ -636,6 +640,25 @@ pub(crate) extern "C" fn pd_vm_native_clone_value_to_slot( STATUS_CONTINUE } +pub(crate) extern "C" fn pd_vm_native_replace_value_in_slot( + dst: *mut Value, + src: *const Value, +) -> i32 { + if dst.is_null() || src.is_null() { + store_bridge_error(VmError::JitNative( + "native replace-value helper received null slot pointer".to_string(), + )); + return STATUS_ERROR; + } + + unsafe { + let replacement = (*src).clone(); + let previous = std::ptr::replace(dst, replacement); + drop(previous); + } + STATUS_CONTINUE +} + pub(crate) extern "C" fn pd_vm_native_init_null_value_slot(dst: *mut Value) -> i32 { if dst.is_null() { store_bridge_error(VmError::JitNative( @@ -2421,4 +2444,17 @@ mod tests { assert_eq!(result.get(&Value::Int(2)), Some(&Value::Int(20))); assert!(!Arc::ptr_eq(&result, &alias)); } + + #[test] + fn replace_value_slot_clones_before_dropping_an_aliasing_destination() { + let shared = Arc::new(vec![Value::Int(1)]); + let mut slot = Value::Array(shared.clone()); + let slot_ptr = &mut slot as *mut Value; + + let status = pd_vm_native_replace_value_in_slot(slot_ptr, slot_ptr); + + assert_eq!(status, STATUS_CONTINUE); + assert_eq!(Arc::strong_count(&shared), 2); + assert!(matches!(slot, Value::Array(_))); + } } diff --git a/src/vm/native/mod.rs b/src/vm/native/mod.rs index 828ce34f..9bf3ca3b 100644 --- a/src/vm/native/mod.rs +++ b/src/vm/native/mod.rs @@ -22,15 +22,15 @@ pub(crate) use bridge::{ map_iter_take_key_entry_address, map_iter_take_value_entry_address, map_set_entry_address, non_yielding_host_call_entry_address, non_yielding_i64_host_call_entry_address, non_yielding_scalar_host_call_entry_address, regex_match_entry_address, - regex_replace_entry_address, restore_active_exit_state_entry_address, - restore_active_sparse_exit_state_entry_address, restore_exit_state_entry_address, - restore_sparse_exit_state_entry_address, restore_virtual_frame_entry_address, - shared_array_from_buffer_entry_address, shared_bytes_from_buffer_entry_address, - shared_string_from_buffer_entry_address, store_bridge_error, string_contains_entry_address, - string_lower_ascii_entry_address, string_replace_literal_entry_address, - string_split_literal_entry_address, take_bridge_error, to_string_entry_address, - type_of_entry_address, value_eq_entry_address, value_len_entry_address, - write_heap_value_to_slot_entry_address, zero_bytes_entry_address, + regex_replace_entry_address, replace_value_in_slot_entry_address, + restore_active_exit_state_entry_address, restore_active_sparse_exit_state_entry_address, + restore_exit_state_entry_address, restore_sparse_exit_state_entry_address, + restore_virtual_frame_entry_address, shared_array_from_buffer_entry_address, + shared_bytes_from_buffer_entry_address, shared_string_from_buffer_entry_address, + store_bridge_error, string_contains_entry_address, string_lower_ascii_entry_address, + string_replace_literal_entry_address, string_split_literal_entry_address, take_bridge_error, + to_string_entry_address, type_of_entry_address, value_eq_entry_address, + value_len_entry_address, write_heap_value_to_slot_entry_address, zero_bytes_entry_address, }; #[cfg(feature = "cranelift-jit")] pub(crate) use codegen::{ diff --git a/tests/jit/jit_tests.rs b/tests/jit/jit_tests.rs index b9f79c15..cdf7a566 100644 --- a/tests/jit/jit_tests.rs +++ b/tests/jit/jit_tests.rs @@ -3342,6 +3342,63 @@ fn trace_jit_restores_tagged_heap_locals_on_ssa_exit() { ); } +#[test] +fn trace_jit_snapshots_borrowed_tagged_locals_before_exit_writes() { + if !native_jit_supported() { + return; + } + + let mut bc = BytecodeBuilder::new(); + bc.ldc(0); + bc.stloc(0); + bc.ldc(1); + bc.stloc(1); + bc.ldc(2); + bc.stloc(2); + let root_ip = bc.position(); + bc.ldloc(2); + bc.ldc(4); + bc.clt(); + let guard_ip = bc.position(); + bc.brfalse(0); + bc.ldloc(0); + bc.ldloc(1); + bc.stloc(0); + bc.stloc(1); + bc.ldloc(2); + bc.ldc(3); + bc.add(); + bc.stloc(2); + bc.br(root_ip); + let exit_ip = bc.position(); + bc.ldloc(1); + bc.ret(); + + let mut code = bc.finish(); + patch_branch_target(&mut code, guard_ip, exit_ip); + let program = Program::new( + vec![ + Value::string("left"), + Value::string("right"), + Value::Int(0), + Value::Int(1), + Value::Int(5), + ], + code, + ) + .with_local_count(3); + let mut vm = Vm::new(program); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + + assert_eq!(vm.run().expect("vm should run"), VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::string("left")]); + assert!(vm.jit_native_exec_count() > 0, "{}", vm.dump_jit_info()); +} + #[test] fn trace_jit_restores_array_and_map_locals_on_ssa_exit() { if !native_jit_supported() { From 289cc3c0d347863c994d891c175acde05c81e8bd Mon Sep 17 00:00:00 2001 From: fffonion Date: Mon, 20 Jul 2026 02:24:24 +0800 Subject: [PATCH 15/26] fix(jit): guard inlined callable identity --- src/vm/jit/inline.rs | 9 +++++--- src/vm/jit/recorder.rs | 52 +++++++++++++++++++++++++++++++++++++++++- tests/jit/jit_tests.rs | 45 ++++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 4 deletions(-) diff --git a/src/vm/jit/inline.rs b/src/vm/jit/inline.rs index dddbcd1f..3fb616b6 100644 --- a/src/vm/jit/inline.rs +++ b/src/vm/jit/inline.rs @@ -48,11 +48,14 @@ pub(crate) fn classify_static_inline_candidate( return Err(InlineRejectReason::NonRootCaller); } let source_local = source_local.ok_or(InlineRejectReason::UnknownTarget)?; - let binding = program + let mut bindings = program .root_callable_bindings .iter() - .find(|binding| binding.local_slot == u16::from(source_local)) - .ok_or(InlineRejectReason::UnknownTarget)?; + .filter(|binding| binding.local_slot == u16::from(source_local)); + let binding = bindings.next().ok_or(InlineRejectReason::UnknownTarget)?; + if bindings.next().is_some() { + return Err(InlineRejectReason::PolymorphicTarget); + } if caller_prototype_id == Some(binding.prototype_id) { return Err(InlineRejectReason::Recursive); } diff --git a/src/vm/jit/recorder.rs b/src/vm/jit/recorder.rs index 282947c9..53858378 100644 --- a/src/vm/jit/recorder.rs +++ b/src/vm/jit/recorder.rs @@ -1,5 +1,6 @@ -use std::fmt; +use std::{fmt, sync::Arc}; +use crate::CallableValue; use crate::builtins::BuiltinFunction; use crate::vm::{OpCode, Program, Value, ValueType, checked_int_div}; @@ -1245,6 +1246,55 @@ pub(crate) fn record_trace_with_local_count( if inline_frame.is_none() && let Ok(candidate) = candidate { + let prototype = &program.callable_prototypes[candidate.prototype_id as usize]; + let expected_callable = builder + .append_value_inst( + current_block, + ip, + SsaValueRepr::Tagged, + SsaInstKind::Constant(Value::Callable(Arc::new(CallableValue { + prototype_id: candidate.prototype_id, + kind: prototype.kind, + env: None, + }))), + ) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + let callable_matches = builder + .append_value_inst( + current_block, + ip, + SsaValueRepr::Bool, + SsaInstKind::ValueCmpEq { + lhs: callable.value.id, + rhs: expected_callable.id, + }, + ) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + let identity_exit = + add_symbolic_exit(&mut builder, ip, &frame, inline_frame.as_ref()); + let (guarded_block, guarded_frame, guard_args) = continue_with_inline_frame( + &mut builder, + &frame, + &mut inline_frame, + "inline_callable_identity", + )?; + builder + .set_terminator( + current_block, + SsaTerminator::BranchBool { + condition: callable_matches.id, + if_true: SsaBranchTarget::Block { + target: guarded_block, + args: guard_args, + }, + if_false: SsaBranchTarget::Exit(identity_exit), + }, + ) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + current_block = guarded_block; + frame = guarded_frame; + op_names.push("inline_callable_identity_guard".to_string()); + let operand_base = frame.stack.len() - usize::from(argc) - 1; let mut operands = frame.stack.split_off(operand_base); let _callable = operands.remove(0); diff --git a/tests/jit/jit_tests.rs b/tests/jit/jit_tests.rs index cdf7a566..f76d6117 100644 --- a/tests/jit/jit_tests.rs +++ b/tests/jit/jit_tests.rs @@ -6453,6 +6453,51 @@ fn trace_jit_inlines_static_leaf_in_root_loop() { assert!(vm.dump_jit_info().contains("interpreter fallbacks: 0")); } +#[test] +fn trace_jit_guards_static_inline_callable_identity() { + if !native_jit_supported() { + return; + } + let source = r#" + fn add_one(value: int) -> int { value + 1 } + fn add_ten(value: int) -> int { value + 10 } + let mut i = 0; + let mut total = 0; + while i < 100 { + total = add_one(total); + i = i + 1; + } + total; + "#; + let compiled = compile_source(source).expect("callable identity source should compile"); + let bindings = compiled.program.root_callable_bindings.clone(); + let replaced_slot = bindings.first().expect("add_one binding").local_slot; + let replacement_id = bindings.get(1).expect("add_ten binding").prototype_id; + let replacement_kind = compiled.program.callable_prototypes[replacement_id as usize].kind; + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_local( + u8::try_from(replaced_slot).expect("root callable slot should fit u8"), + Value::Callable(Arc::new(vm::CallableValue { + prototype_id: replacement_id, + kind: replacement_kind, + env: None, + })), + ) + .expect("callable replacement should succeed"); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + + assert_eq!( + vm.run().expect("identity guard source should run"), + VmStatus::Halted + ); + assert_eq!(vm.stack(), &[Value::Int(1_000)]); + assert!(vm.jit_native_exec_count() > 0, "{}", vm.dump_jit_info()); +} + #[test] fn trace_jit_inlines_array_swap_leaf() { if !native_jit_supported() { From 47152c62534c76960971edb5cabea56c10e4970e Mon Sep 17 00:00:00 2001 From: fffonion Date: Mon, 20 Jul 2026 02:43:33 +0800 Subject: [PATCH 16/26] fix(jit): guard inlined callable schemas --- src/vm/jit/inline.rs | 1 + src/vm/jit/ir.rs | 8 +++ src/vm/jit/native/lower.rs | 24 +++++++ src/vm/jit/recorder.rs | 136 +++++++++++++++++++++++++++++++++++-- src/vm/jit/region.rs | 1 + tests/jit/jit_tests.rs | 54 +++++++++++++++ 6 files changed, 220 insertions(+), 4 deletions(-) diff --git a/src/vm/jit/inline.rs b/src/vm/jit/inline.rs index 3fb616b6..0a0cc1e0 100644 --- a/src/vm/jit/inline.rs +++ b/src/vm/jit/inline.rs @@ -16,6 +16,7 @@ pub(crate) enum InlineRejectReason { HostTarget, CapturedCallable, ArityMismatch, + SchemaUnproven, Recursive, NestedScriptCall, YieldingCall, diff --git a/src/vm/jit/ir.rs b/src/vm/jit/ir.rs index b20d41a6..8a7c2808 100644 --- a/src/vm/jit/ir.rs +++ b/src/vm/jit/ir.rs @@ -105,6 +105,10 @@ pub(crate) enum SsaInstKind { CloneTagged { input: SsaValueId, }, + ValueIsType { + input: SsaValueId, + tag: ValueType, + }, UnboxInt { input: SsaValueId, }, @@ -394,6 +398,7 @@ impl SsaInstKind { Self::HostCall { args, .. } => args.clone(), Self::CloneTagged { input } + | Self::ValueIsType { input, .. } | Self::UnboxInt { input } | Self::UnboxFloat { input } | Self::UnboxBool { input } @@ -1121,6 +1126,9 @@ fn render_inst_kind(kind: &SsaInstKind) -> String { match kind { SsaInstKind::Constant(value) => format!("const {value:?}"), SsaInstKind::CloneTagged { input } => format!("clone_tagged {input}"), + SsaInstKind::ValueIsType { input, tag } => { + format!("value_is_type {input}, {tag:?}") + } SsaInstKind::UnboxInt { input } => format!("unbox_int {input}"), SsaInstKind::UnboxFloat { input } => format!("unbox_float {input}"), SsaInstKind::UnboxBool { input } => format!("unbox_bool {input}"), diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index f077fc09..ae555726 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -1274,6 +1274,7 @@ fn ssa_trace_supported(ssa: &SsaTrace) -> bool { inst.kind, SsaInstKind::Constant(_) | SsaInstKind::CloneTagged { .. } + | SsaInstKind::ValueIsType { .. } | SsaInstKind::UnboxHeapPtr { .. } | SsaInstKind::UnboxInt { .. } | SsaInstKind::UnboxFloat { .. } @@ -2090,6 +2091,29 @@ fn lower_ssa_inst( )?; out } + SsaInstKind::ValueIsType { input, tag } => { + let input = *values.get(input).ok_or_else(|| { + VmError::JitNative("SSA type predicate input missing".to_string()) + })?; + let expected_tag = match tag { + ValueType::Null => layout.value.null_tag, + ValueType::Int => layout.value.int_tag, + ValueType::Float => layout.value.float_tag, + ValueType::Bool => layout.value.bool_tag, + ValueType::String => layout.value.string_tag, + ValueType::Bytes => layout.value.bytes_tag, + ValueType::Array => layout.value.array_tag, + ValueType::Map => layout.value.map_tag, + ValueType::Callable | ValueType::Unknown => { + return Err(VmError::JitNative(format!( + "unsupported SSA type predicate tag {tag:?}" + ))); + } + }; + let actual_tag = ssa_load_tag_i32(b, layout.value, input); + b.ins() + .icmp_imm(IntCC::Equal, actual_tag, i64::from(expected_tag)) + } SsaInstKind::UnboxInt { input } => { let input = *values .get(input) diff --git a/src/vm/jit/recorder.rs b/src/vm/jit/recorder.rs index 53858378..8801b096 100644 --- a/src/vm/jit/recorder.rs +++ b/src/vm/jit/recorder.rs @@ -2,14 +2,15 @@ use std::{fmt, sync::Arc}; use crate::CallableValue; use crate::builtins::BuiltinFunction; +use crate::compiler::TypeSchema; use crate::vm::{OpCode, Program, Value, ValueType, checked_int_div}; use super::JitTraceTerminal; use super::deopt::materialize_ssa_values; -use super::inline::{InlineCandidate, classify_static_inline_candidate}; +use super::inline::{InlineCandidate, InlineRejectReason, classify_static_inline_candidate}; use super::ir::{ SsaBranchTarget, SsaInstKind, SsaMaterialization, SsaTerminator, SsaTrace, SsaTraceBuilder, - SsaValue, SsaValueRepr, VirtualFrameSnapshot, + SsaValue, SsaValueId, SsaValueRepr, VirtualFrameSnapshot, }; #[derive(Clone, Debug, PartialEq)] @@ -263,6 +264,99 @@ struct SymbolicValue { info: ValueInfo, } +fn inline_schema_guard_type(schema: &TypeSchema) -> Option> { + match schema { + TypeSchema::Unknown | TypeSchema::GenericParam(_) => Some(None), + TypeSchema::Int => Some(Some(ValueType::Int)), + TypeSchema::Float => Some(Some(ValueType::Float)), + TypeSchema::Bool => Some(Some(ValueType::Bool)), + TypeSchema::String => Some(Some(ValueType::String)), + TypeSchema::Bytes => Some(Some(ValueType::Bytes)), + TypeSchema::Named(_, _) | TypeSchema::Map(_) | TypeSchema::Object(_) => { + Some(Some(ValueType::Map)) + } + TypeSchema::Array(_) | TypeSchema::ArrayTuple(_) | TypeSchema::ArrayTupleRest { .. } => { + Some(Some(ValueType::Array)) + } + TypeSchema::Null + | TypeSchema::Number + | TypeSchema::Optional(_) + | TypeSchema::Callable { .. } => None, + } +} + +fn inline_argument_schemas_supported( + arguments: &[SymbolicValue], + schema: Option<&TypeSchema>, +) -> bool { + let Some(TypeSchema::Callable { params, .. }) = schema else { + return schema.is_none(); + }; + params.len() == arguments.len() + && params.iter().zip(arguments).all(|(schema, argument)| { + let Some(guard_type) = inline_schema_guard_type(schema) else { + return false; + }; + match (guard_type, argument.info.repr) { + (None, _) | (Some(_), SsaValueRepr::Tagged) => true, + (Some(ValueType::Int), SsaValueRepr::I64) + | (Some(ValueType::Float), SsaValueRepr::F64) + | (Some(ValueType::Bool), SsaValueRepr::Bool) => true, + (Some(expected), SsaValueRepr::HeapPtr(actual)) => expected == actual, + _ => false, + } + }) +} + +fn append_inline_argument_schema_guards( + builder: &mut SsaTraceBuilder, + block: super::ir::SsaBlockId, + ip: usize, + arguments: &[SymbolicValue], + schema: Option<&TypeSchema>, +) -> Result, TraceRecordError> { + let Some(TypeSchema::Callable { params, .. }) = schema else { + return Ok(None); + }; + let mut guard = None; + for (schema, argument) in params.iter().zip(arguments) { + let Some(Some(expected)) = inline_schema_guard_type(schema) else { + continue; + }; + if argument.info.repr != SsaValueRepr::Tagged { + continue; + } + let predicate = builder + .append_value_inst( + block, + ip, + SsaValueRepr::Bool, + SsaInstKind::ValueIsType { + input: argument.value.id, + tag: expected, + }, + ) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + guard = Some(if let Some(previous) = guard { + builder + .append_value_inst( + block, + ip, + SsaValueRepr::Bool, + SsaInstKind::BoolAnd { + lhs: previous, + rhs: predicate.id, + }, + ) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))? + .id + } else { + predicate.id + }); + } + Ok(guard) +} + #[derive(Clone, Debug, PartialEq)] struct SymbolicFrame { stack: Vec, @@ -1241,12 +1335,30 @@ pub(crate) fn record_trace_with_local_count( callable.info.source_local, argc, max_trace_len.saturating_sub(cursor.recorded_ops), - ); + ) + .and_then(|candidate| { + let prototype = &program.callable_prototypes[candidate.prototype_id as usize]; + let argument_start = frame.stack.len() - usize::from(argc); + inline_argument_schemas_supported( + &frame.stack[argument_start..], + prototype.schema.as_ref(), + ) + .then_some(candidate) + .ok_or(InlineRejectReason::SchemaUnproven) + }); let inline_reject_reason = candidate.as_ref().err().copied(); if inline_frame.is_none() && let Ok(candidate) = candidate { let prototype = &program.callable_prototypes[candidate.prototype_id as usize]; + let argument_start = frame.stack.len() - usize::from(argc); + let schema_guard = append_inline_argument_schema_guards( + &mut builder, + current_block, + ip, + &frame.stack[argument_start..], + prototype.schema.as_ref(), + )?; let expected_callable = builder .append_value_inst( current_block, @@ -1270,6 +1382,22 @@ pub(crate) fn record_trace_with_local_count( }, ) .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + let inline_guard = if let Some(schema_guard) = schema_guard { + builder + .append_value_inst( + current_block, + ip, + SsaValueRepr::Bool, + SsaInstKind::BoolAnd { + lhs: schema_guard, + rhs: callable_matches.id, + }, + ) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))? + .id + } else { + callable_matches.id + }; let identity_exit = add_symbolic_exit(&mut builder, ip, &frame, inline_frame.as_ref()); let (guarded_block, guarded_frame, guard_args) = continue_with_inline_frame( @@ -1282,7 +1410,7 @@ pub(crate) fn record_trace_with_local_count( .set_terminator( current_block, SsaTerminator::BranchBool { - condition: callable_matches.id, + condition: inline_guard, if_true: SsaBranchTarget::Block { target: guarded_block, args: guard_args, diff --git a/src/vm/jit/region.rs b/src/vm/jit/region.rs index a735c93d..97c34596 100644 --- a/src/vm/jit/region.rs +++ b/src/vm/jit/region.rs @@ -212,6 +212,7 @@ fn remap_inst_inputs( } } SsaInstKind::CloneTagged { input } + | SsaInstKind::ValueIsType { input, .. } | SsaInstKind::UnboxInt { input } | SsaInstKind::UnboxFloat { input } | SsaInstKind::UnboxBool { input } diff --git a/tests/jit/jit_tests.rs b/tests/jit/jit_tests.rs index f76d6117..23274bc4 100644 --- a/tests/jit/jit_tests.rs +++ b/tests/jit/jit_tests.rs @@ -6498,6 +6498,60 @@ fn trace_jit_guards_static_inline_callable_identity() { assert!(vm.jit_native_exec_count() > 0, "{}", vm.dump_jit_info()); } +#[test] +fn trace_jit_preserves_inline_callable_argument_schema_checks() { + if !native_jit_supported() { + return; + } + let source = r#" + fn ignore(value: int) -> int { 1 } + let mut i = 0; + let value: int = 7; + while i < 100 { + i = i + ignore(value); + } + i; + "#; + let compiled = compile_source(source).expect("callable schema source should compile"); + let value_slot = compiled + .program + .debug + .as_ref() + .expect("debug metadata") + .locals + .iter() + .find(|local| local.name == "value") + .expect("value local") + .index; + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_fuel_check_interval(1).expect("fuel interval"); + vm.set_fuel(1); + loop { + assert_eq!( + vm.run().expect("initialization step should run"), + VmStatus::Yielded + ); + if vm.locals().get(value_slot as usize) == Some(&Value::Int(7)) { + break; + } + vm.recharge_fuel(1).expect("fuel recharge"); + } + vm.set_local(value_slot, Value::Bool(true)) + .expect("dynamic value replacement should succeed"); + vm.clear_fuel(); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + + let error = vm.run().expect_err("invalid callable argument must fail"); + assert!( + matches!(error, vm::VmError::TypeMismatch("callable argument schema")), + "unexpected error: {error:?}" + ); +} + #[test] fn trace_jit_inlines_array_swap_leaf() { if !native_jit_supported() { From 4cbc0c494b1836e9e1560f58d032f06d1d46b117 Mon Sep 17 00:00:00 2001 From: fffonion Date: Mon, 20 Jul 2026 02:48:19 +0800 Subject: [PATCH 17/26] fix(jit): remap virtual frames in fused regions --- src/vm/jit/region.rs | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/vm/jit/region.rs b/src/vm/jit/region.rs index 97c34596..cce7defa 100644 --- a/src/vm/jit/region.rs +++ b/src/vm/jit/region.rs @@ -179,6 +179,11 @@ fn offset_exit(mut exit: SsaExit, value_offset: u32, exit_offset: u32) -> VmResu for materialization in exit.stack.iter_mut().chain(&mut exit.locals) { offset_materialization(materialization, value_offset)?; } + for frame in &mut exit.virtual_frames { + for materialization in frame.operand_stack.iter_mut().chain(&mut frame.locals) { + offset_materialization(materialization, value_offset)?; + } + } Ok(exit) } @@ -506,7 +511,7 @@ fn offset_exit_id(exit: SsaExitId, offset: u32) -> VmResult { mod tests { use super::*; use crate::vm::jit::JitTraceTerminal; - use crate::vm::jit::ir::{SsaTerminator, SsaTraceBuilder}; + use crate::vm::jit::ir::{SsaTerminator, SsaTraceBuilder, VirtualFrameSnapshot}; fn test_trace(id: usize, root_ip: usize, exit_ip: usize) -> (JitTrace, SsaExitId) { let mut ssa = SsaTraceBuilder::new(root_ip, 0); @@ -532,6 +537,43 @@ mod tests { ) } + #[test] + fn offset_exit_remaps_virtual_frame_materializations() { + let exit = SsaExit { + id: SsaExitId::new(0), + exit_ip: 42, + stack: vec![SsaMaterialization::Value(SsaValueId::new(1))], + locals: vec![SsaMaterialization::BoxInt(SsaValueId::new(2))], + dirty_locals: vec![true], + virtual_frames: vec![VirtualFrameSnapshot { + prototype_id: 3, + call_ip: 7, + return_ip: 9, + resume_ip: 11, + operand_stack: vec![SsaMaterialization::Value(SsaValueId::new(3))], + locals: vec![SsaMaterialization::BoxHeapPtr { + value: SsaValueId::new(4), + tag: crate::ValueType::Array, + }], + dirty_locals: vec![true], + }], + }; + + let remapped = offset_exit(exit, 10, 2).expect("exit remap should succeed"); + assert_eq!(remapped.id, SsaExitId::new(2)); + assert_eq!( + remapped.virtual_frames[0].operand_stack, + vec![SsaMaterialization::Value(SsaValueId::new(13))] + ); + assert_eq!( + remapped.virtual_frames[0].locals, + vec![SsaMaterialization::BoxHeapPtr { + value: SsaValueId::new(14), + tag: crate::ValueType::Array, + }] + ); + } + #[test] fn two_trace_region_remaps_child_ids_and_preserves_exit_identity() { let (parent, parent_exit) = test_trace(3, 0, 12); From de0f159c7f218e1ed13823de0a78b9d116d4fd65 Mon Sep 17 00:00:00 2001 From: fffonion Date: Mon, 20 Jul 2026 03:37:38 +0800 Subject: [PATCH 18/26] perf(jit): specialize inline targets at trace entry --- src/vm/jit/recorder.rs | 98 ++++++++++++++++-------------------------- src/vm/jit/trace.rs | 10 +++-- src/vm/mod.rs | 17 +++++++- 3 files changed, 60 insertions(+), 65 deletions(-) diff --git a/src/vm/jit/recorder.rs b/src/vm/jit/recorder.rs index 8801b096..e5c09269 100644 --- a/src/vm/jit/recorder.rs +++ b/src/vm/jit/recorder.rs @@ -1,6 +1,5 @@ -use std::{fmt, sync::Arc}; +use std::fmt; -use crate::CallableValue; use crate::builtins::BuiltinFunction; use crate::compiler::TypeSchema; use crate::vm::{OpCode, Program, Value, ValueType, checked_int_div}; @@ -830,6 +829,7 @@ pub(crate) fn record_trace( entry_stack_depth, program.local_count, None, + None, max_trace_len, non_yielding_host_imports, ) @@ -843,6 +843,7 @@ pub(crate) fn record_trace_with_local_count( entry_stack_depth: usize, local_count: usize, entry_local_types: Option<&[ValueType]>, + entry_callable_prototypes: Option<&[Option]>, max_trace_len: usize, non_yielding_host_imports: &[bool], ) -> Result { @@ -1337,6 +1338,17 @@ pub(crate) fn record_trace_with_local_count( max_trace_len.saturating_sub(cursor.recorded_ops), ) .and_then(|candidate| { + if let Some(prototypes) = entry_callable_prototypes { + let source_local = callable + .info + .source_local + .ok_or(InlineRejectReason::UnknownTarget)?; + if prototypes.get(usize::from(source_local)).copied().flatten() + != Some(candidate.prototype_id) + { + return Err(InlineRejectReason::PolymorphicTarget); + } + } let prototype = &program.callable_prototypes[candidate.prototype_id as usize]; let argument_start = frame.stack.len() - usize::from(argc); inline_argument_schemas_supported( @@ -1359,69 +1371,33 @@ pub(crate) fn record_trace_with_local_count( &frame.stack[argument_start..], prototype.schema.as_ref(), )?; - let expected_callable = builder - .append_value_inst( - current_block, - ip, - SsaValueRepr::Tagged, - SsaInstKind::Constant(Value::Callable(Arc::new(CallableValue { - prototype_id: candidate.prototype_id, - kind: prototype.kind, - env: None, - }))), - ) - .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; - let callable_matches = builder - .append_value_inst( - current_block, - ip, - SsaValueRepr::Bool, - SsaInstKind::ValueCmpEq { - lhs: callable.value.id, - rhs: expected_callable.id, - }, - ) - .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; - let inline_guard = if let Some(schema_guard) = schema_guard { + if let Some(schema_guard) = schema_guard { + let schema_exit = + add_symbolic_exit(&mut builder, ip, &frame, inline_frame.as_ref()); + let (guarded_block, guarded_frame, guard_args) = + continue_with_inline_frame( + &mut builder, + &frame, + &mut inline_frame, + "inline_callable_schema", + )?; builder - .append_value_inst( + .set_terminator( current_block, - ip, - SsaValueRepr::Bool, - SsaInstKind::BoolAnd { - lhs: schema_guard, - rhs: callable_matches.id, + SsaTerminator::BranchBool { + condition: schema_guard, + if_true: SsaBranchTarget::Block { + target: guarded_block, + args: guard_args, + }, + if_false: SsaBranchTarget::Exit(schema_exit), }, ) - .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))? - .id - } else { - callable_matches.id - }; - let identity_exit = - add_symbolic_exit(&mut builder, ip, &frame, inline_frame.as_ref()); - let (guarded_block, guarded_frame, guard_args) = continue_with_inline_frame( - &mut builder, - &frame, - &mut inline_frame, - "inline_callable_identity", - )?; - builder - .set_terminator( - current_block, - SsaTerminator::BranchBool { - condition: inline_guard, - if_true: SsaBranchTarget::Block { - target: guarded_block, - args: guard_args, - }, - if_false: SsaBranchTarget::Exit(identity_exit), - }, - ) - .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; - current_block = guarded_block; - frame = guarded_frame; - op_names.push("inline_callable_identity_guard".to_string()); + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + current_block = guarded_block; + frame = guarded_frame; + op_names.push("inline_callable_schema_guard".to_string()); + } let operand_base = frame.stack.len() - usize::from(argc) - 1; let mut operands = frame.stack.split_off(operand_base); diff --git a/src/vm/jit/trace.rs b/src/vm/jit/trace.rs index 459bf8e7..fb067e4a 100644 --- a/src/vm/jit/trace.rs +++ b/src/vm/jit/trace.rs @@ -417,7 +417,7 @@ impl TraceJitEngine { stack_depth: usize, program: &Program, ) -> Option { - self.observe_hot_entry_with_local_types(frame_key, ip, stack_depth, None, program) + self.observe_hot_entry_with_local_types(frame_key, ip, stack_depth, None, None, program) } pub(crate) fn observe_hot_entry_with_local_types( @@ -426,6 +426,7 @@ impl TraceJitEngine { ip: usize, stack_depth: usize, entry_local_types: Option<&[crate::ValueType]>, + entry_callable_prototypes: Option<&[Option]>, program: &Program, ) -> Option { if !self.config.enabled @@ -461,7 +462,7 @@ impl TraceJitEngine { let result = if self.config.hot_loop_threshold == 0 { Err(JitNyiReason::HotLoopThresholdZero) } else { - self.compile_trace(program, key, entry_local_types) + self.compile_trace(program, key, entry_local_types, entry_callable_prototypes) }; self.finish_attempt(key, line, result) } @@ -639,6 +640,7 @@ impl TraceJitEngine { entry_local_types: Option<&[crate::ValueType]>, program: &Program, ) -> Option { + let entry_callable_prototypes = None; if !self.config.enabled || !native_jit_supported() || self.callable_frame_is_blocked(frame_key) @@ -670,7 +672,7 @@ impl TraceJitEngine { let result = if self.config.hot_loop_threshold == 0 { Err(JitNyiReason::HotLoopThresholdZero) } else { - self.compile_trace(program, key, entry_local_types) + self.compile_trace(program, key, entry_local_types, entry_callable_prototypes) }; self.finish_attempt(key, line, result) } @@ -1008,6 +1010,7 @@ impl TraceJitEngine { program: &Program, key: TraceEntryKey, entry_local_types: Option<&[crate::ValueType]>, + entry_callable_prototypes: Option<&[Option]>, ) -> Result { let local_count = if key.frame_key == ROOT_FRAME_KEY { program.local_count @@ -1030,6 +1033,7 @@ impl TraceJitEngine { key.stack_depth, local_count, entry_local_types, + entry_callable_prototypes, self.config.max_trace_len, &self.non_yielding_host_imports, ) diff --git a/src/vm/mod.rs b/src/vm/mod.rs index b24cba49..98033524 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -2317,12 +2317,24 @@ impl Vm { let stack_depth = self.active_operand_stack_len(); let entry_local_types = (frame_key != crate::vm::native::ROOT_FRAME_KEY) .then(|| self.active_local_types()); + let entry_callable_prototypes = + (frame_key == crate::vm::native::ROOT_FRAME_KEY).then(|| { + self.locals + .iter() + .take(self.program.local_count) + .map(|value| match value { + Value::Callable(callable) => Some(callable.prototype_id), + _ => None, + }) + .collect::>() + }); let program = &self.program; self.jit.observe_hot_entry_with_local_types( frame_key, self.ip, stack_depth, entry_local_types.as_deref(), + entry_callable_prototypes.as_deref(), program, ) }; @@ -2753,7 +2765,10 @@ impl Vm { } pub fn set_local(&mut self, index: u8, value: Value) -> VmResult<()> { - self.store_local_with_drop_contract(index, value) + self.store_local_with_drop_contract(index, value)?; + let config = *self.jit.config(); + self.jit.set_config(config); + Ok(()) } pub fn program(&self) -> &Program { From e5458f9013538891fd321a0c0682717b174b43fa Mon Sep 17 00:00:00 2001 From: fffonion Date: Mon, 20 Jul 2026 05:16:51 +0800 Subject: [PATCH 19/26] perf(jit): rebuild inherited entries from active frame state --- src/vm/jit/native/lower.rs | 166 ++----------------------------------- 1 file changed, 8 insertions(+), 158 deletions(-) diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index ae555726..cd4899bc 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -59,13 +59,9 @@ static CRANELIFT_TAIL_ISA: OnceLock> = OnceLock:: const MAX_INHERITED_ENTRY_VALUES: usize = 256; const INHERITED_STATE_ACTIVE_OFFSET: i32 = 0; -const INHERITED_STATE_FRAME_KEY_OFFSET: i32 = 8; const INHERITED_STATE_STACK_BASE_OFFSET: i32 = 16; const INHERITED_STATE_LOCAL_BASE_OFFSET: i32 = 24; const INHERITED_STATE_DYNAMIC_TARGET_OFFSET: i32 = 32; -const INHERITED_STATE_TARGET_IP_OFFSET: i32 = 40; -const INHERITED_STATE_VALUE_COUNT_OFFSET: i32 = 48; -const INHERITED_STATE_VALUES_OFFSET: i32 = 56; type TaggedConstants = (Box<[Value]>, HashMap); @@ -865,11 +861,16 @@ fn try_compile_ssa_trace( inherited_state_ptr, INHERITED_STATE_LOCAL_BASE_OFFSET, ); - let inherited_args = build_inherited_entry_args( + let inherited_args = build_entry_args( &mut b, - inherited_state_ptr, + vm_ptr, pointer_type, - entry_ssa_block.params.len(), + layout, + offsets, + active_stack_base, + active_local_base, + ssa.entry_stack_depth, + entry_local_count, )?; let inactive = b.ins().iconst(pointer_type, 0); b.ins().store( @@ -1797,40 +1798,6 @@ fn ssa_backedge_targets( targets } -fn build_inherited_entry_args( - b: &mut FunctionBuilder, - inherited_state_ptr: cranelift_codegen::ir::Value, - pointer_type: cranelift_codegen::ir::Type, - entry_value_count: usize, -) -> VmResult> { - if entry_value_count > MAX_INHERITED_ENTRY_VALUES { - return Err(VmError::JitNative( - "SSA inherited entry value count exceeds packet capacity".to_string(), - )); - } - (0..entry_value_count) - .map(|index| { - let byte_offset = index - .checked_mul(std::mem::size_of::()) - .and_then(|offset| { - usize::try_from(INHERITED_STATE_VALUES_OFFSET) - .ok() - .and_then(|base| base.checked_add(offset)) - }) - .and_then(|offset| i32::try_from(offset).ok()) - .ok_or_else(|| { - VmError::JitNative("SSA inherited entry packet offset exceeds i32".to_string()) - })?; - Ok(b.ins().load( - pointer_type, - MemFlags::new(), - inherited_state_ptr, - byte_offset, - )) - }) - .collect() -} - #[allow(clippy::too_many_arguments)] fn build_entry_args( b: &mut FunctionBuilder, @@ -4487,19 +4454,6 @@ fn write_inherited_state_packet( ctx: SsaLowerCtx<'_>, exit: &crate::vm::jit::ir::SsaExit, ) -> VmResult<()> { - let inactive = b.ins().iconst(ctx.pointer_type, 0); - b.ins().store( - MemFlags::new(), - inactive, - ctx.inherited_state_ptr, - INHERITED_STATE_ACTIVE_OFFSET, - ); - b.ins().store( - MemFlags::new(), - inactive, - ctx.inherited_state_ptr, - INHERITED_STATE_DYNAMIC_TARGET_OFFSET, - ); let entry_value_count = exit .stack .len() @@ -4509,13 +4463,6 @@ fn write_inherited_state_packet( return Ok(()); } - let frame_key = b.ins().iconst(ctx.pointer_type, ctx.frame_key as i64); - b.ins().store( - MemFlags::new(), - frame_key, - ctx.inherited_state_ptr, - INHERITED_STATE_FRAME_KEY_OFFSET, - ); b.ins().store( MemFlags::new(), ctx.active_stack_base, @@ -4528,103 +4475,6 @@ fn write_inherited_state_packet( ctx.inherited_state_ptr, INHERITED_STATE_LOCAL_BASE_OFFSET, ); - let target_ip = b.ins().iconst( - ctx.pointer_type, - i64::try_from(exit.exit_ip) - .map_err(|_| VmError::JitNative("SSA inherited target ip exceeds i64".to_string()))?, - ); - b.ins().store( - MemFlags::new(), - target_ip, - ctx.inherited_state_ptr, - INHERITED_STATE_TARGET_IP_OFFSET, - ); - let value_count = b.ins().iconst( - ctx.pointer_type, - i64::try_from(entry_value_count) - .map_err(|_| VmError::JitNative("SSA inherited value count exceeds i64".to_string()))?, - ); - b.ins().store( - MemFlags::new(), - value_count, - ctx.inherited_state_ptr, - INHERITED_STATE_VALUE_COUNT_OFFSET, - ); - let stack_ptr = b.ins().load( - ctx.pointer_type, - MemFlags::new(), - ctx.vm_ptr, - ctx.offsets.stack_ptr, - ); - let stack_base_ptr = ssa_value_addr( - b, - ctx.pointer_type, - stack_ptr, - ctx.active_stack_base, - ctx.layout.value.size, - ); - let locals_ptr = b.ins().load( - ctx.pointer_type, - MemFlags::new(), - ctx.vm_ptr, - ctx.offsets.locals_ptr, - ); - let locals_base_ptr = ssa_value_addr( - b, - ctx.pointer_type, - locals_ptr, - ctx.active_local_base, - ctx.layout.value.size, - ); - for index in 0..entry_value_count { - let value_addr = if index < exit.stack.len() { - let stack_index = b.ins().iconst( - ctx.pointer_type, - i64::try_from(index).map_err(|_| { - VmError::JitNative("SSA inherited stack index exceeds i64".to_string()) - })?, - ); - ssa_value_addr( - b, - ctx.pointer_type, - stack_base_ptr, - stack_index, - ctx.layout.value.size, - ) - } else { - let local_index = index - exit.stack.len(); - let local_index = b.ins().iconst( - ctx.pointer_type, - i64::try_from(local_index).map_err(|_| { - VmError::JitNative("SSA inherited local index exceeds i64".to_string()) - })?, - ); - ssa_value_addr( - b, - ctx.pointer_type, - locals_base_ptr, - local_index, - ctx.layout.value.size, - ) - }; - let packet_offset = index - .checked_mul(std::mem::size_of::()) - .and_then(|offset| { - usize::try_from(INHERITED_STATE_VALUES_OFFSET) - .ok() - .and_then(|base| base.checked_add(offset)) - }) - .and_then(|offset| i32::try_from(offset).ok()) - .ok_or_else(|| { - VmError::JitNative("SSA inherited packet offset exceeds i32".to_string()) - })?; - b.ins().store( - MemFlags::new(), - value_addr, - ctx.inherited_state_ptr, - packet_offset, - ); - } let active = b.ins().iconst(ctx.pointer_type, 1); b.ins().store( MemFlags::new(), From e5a52327be4695c90a7a170444ec6f7ad20e30bd Mon Sep 17 00:00:00 2001 From: fffonion Date: Mon, 20 Jul 2026 05:40:28 +0800 Subject: [PATCH 20/26] fix(jit): invalidate native traces on local mutation --- src/vm/mod.rs | 2 +- tests/jit/jit_tests.rs | 49 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/vm/mod.rs b/src/vm/mod.rs index 98033524..aafb7d33 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -2767,7 +2767,7 @@ impl Vm { pub fn set_local(&mut self, index: u8, value: Value) -> VmResult<()> { self.store_local_with_drop_contract(index, value)?; let config = *self.jit.config(); - self.jit.set_config(config); + self.set_jit_config(config); Ok(()) } diff --git a/tests/jit/jit_tests.rs b/tests/jit/jit_tests.rs index 23274bc4..fc4f33fb 100644 --- a/tests/jit/jit_tests.rs +++ b/tests/jit/jit_tests.rs @@ -6498,6 +6498,55 @@ fn trace_jit_guards_static_inline_callable_identity() { assert!(vm.jit_native_exec_count() > 0, "{}", vm.dump_jit_info()); } +#[test] +fn trace_jit_invalidates_native_inline_after_callable_local_replacement() { + if !native_jit_supported() { + return; + } + let source = r#" + fn add_one(value: int) -> int { value + 1 } + fn add_ten(value: int) -> int { value + 10 } + let mut i = 0; + let mut total = 0; + while i < 100 { + total = add_one(total); + i = i + 1; + } + total; + "#; + let compiled = compile_source(source).expect("callable invalidation source should compile"); + let bindings = compiled.program.root_callable_bindings.clone(); + let replaced_slot = bindings.first().expect("add_one binding").local_slot; + let replacement_id = bindings.get(1).expect("add_ten binding").prototype_id; + let replacement_kind = compiled.program.callable_prototypes[replacement_id as usize].kind; + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + + assert_eq!(vm.run().expect("initial inline run"), VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(100)]); + assert!(vm.jit_native_trace_count() > 0, "{}", vm.dump_jit_info()); + + vm.reset_for_reuse(); + vm.set_local( + u8::try_from(replaced_slot).expect("root callable slot should fit u8"), + Value::Callable(Arc::new(vm::CallableValue { + prototype_id: replacement_id, + kind: replacement_kind, + env: None, + })), + ) + .expect("callable replacement should succeed"); + assert_eq!(vm.jit_native_trace_count(), 0); + + assert_eq!(vm.run().expect("replacement inline run"), VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(1_000)]); + assert!(vm.jit_native_exec_count() > 0, "{}", vm.dump_jit_info()); +} + #[test] fn trace_jit_preserves_inline_callable_argument_schema_checks() { if !native_jit_supported() { From c60d9fd52e773d70eace993dde030ad42bac7190 Mon Sep 17 00:00:00 2001 From: fffonion Date: Mon, 20 Jul 2026 05:49:12 +0800 Subject: [PATCH 21/26] fix(jit): prove inline targets on exit traces --- src/vm/jit/recorder.rs | 38 +++++++++++++++++++++++++++----------- src/vm/jit/runtime.rs | 11 +++++++++++ src/vm/jit/trace.rs | 4 ++-- src/vm/mod.rs | 32 ++++++++++++++++++++++---------- 4 files changed, 62 insertions(+), 23 deletions(-) diff --git a/src/vm/jit/recorder.rs b/src/vm/jit/recorder.rs index e5c09269..4c374490 100644 --- a/src/vm/jit/recorder.rs +++ b/src/vm/jit/recorder.rs @@ -1338,16 +1338,16 @@ pub(crate) fn record_trace_with_local_count( max_trace_len.saturating_sub(cursor.recorded_ops), ) .and_then(|candidate| { - if let Some(prototypes) = entry_callable_prototypes { - let source_local = callable - .info - .source_local - .ok_or(InlineRejectReason::UnknownTarget)?; - if prototypes.get(usize::from(source_local)).copied().flatten() - != Some(candidate.prototype_id) - { - return Err(InlineRejectReason::PolymorphicTarget); - } + let prototypes = entry_callable_prototypes + .ok_or(InlineRejectReason::UnknownTarget)?; + let source_local = callable + .info + .source_local + .ok_or(InlineRejectReason::UnknownTarget)?; + if prototypes.get(usize::from(source_local)).copied().flatten() + != Some(candidate.prototype_id) + { + return Err(InlineRejectReason::PolymorphicTarget); } let prototype = &program.callable_prototypes[candidate.prototype_id as usize]; let argument_start = frame.stack.len() - usize::from(argc); @@ -5378,7 +5378,23 @@ mod tests { }], ); - let recorded = record_trace(&program, root_ip as usize, 0, 64, &[]).unwrap(); + let unproven = record_trace(&program, root_ip as usize, 0, 64, &[]).unwrap(); + assert_eq!(unproven.terminal, JitTraceTerminal::CallValue); + assert!(!unproven.op_names.iter().any(|name| name == "inline_call:0")); + + let callable_prototypes = [Some(0), None]; + let recorded = record_trace_with_local_count( + &program, + crate::vm::native::ROOT_FRAME_KEY, + root_ip as usize, + 0, + program.local_count, + None, + Some(&callable_prototypes), + 64, + &[], + ) + .unwrap(); assert_eq!(recorded.terminal, JitTraceTerminal::LoopBack); assert!(recorded.op_names.iter().any(|name| name == "inline_call:0")); assert!(recorded.op_names.iter().any(|name| name == "inline_ret")); diff --git a/src/vm/jit/runtime.rs b/src/vm/jit/runtime.rs index 6f1f1deb..703f59c9 100644 --- a/src/vm/jit/runtime.rs +++ b/src/vm/jit/runtime.rs @@ -297,12 +297,14 @@ impl Vm { if next_trace_id.is_none() && !self.jit.callable_frame_is_blocked(frame_key) { let entry_local_types = (frame_key != ROOT_FRAME_KEY).then(|| self.active_local_types()); + let entry_callable_prototypes = self.active_local_callable_prototypes(); let program = &self.program; next_trace_id = self.jit.observe_exit_entry_with_local_types( frame_key, ip, stack_depth, entry_local_types.as_deref(), + entry_callable_prototypes.as_deref(), program, ); } @@ -458,12 +460,15 @@ impl Vm { { let entry_local_types = (frame_key != ROOT_FRAME_KEY).then(|| self.active_local_types()); + let entry_callable_prototypes = + self.active_local_callable_prototypes(); let program = &self.program; next_trace_id = self.jit.observe_exit_entry_with_local_types( frame_key, ip, stack_depth, entry_local_types.as_deref(), + entry_callable_prototypes.as_deref(), program, ); } @@ -1123,12 +1128,15 @@ impl Vm { next_trace_id = { let entry_local_types = (frame_key != ROOT_FRAME_KEY) .then(|| self.active_local_types()); + let entry_callable_prototypes = + self.active_local_callable_prototypes(); let program = &self.program; self.jit.observe_exit_entry_with_local_types( frame_key, ip, stack_depth, entry_local_types.as_deref(), + entry_callable_prototypes.as_deref(), program, ) }; @@ -1194,12 +1202,15 @@ impl Vm { next_trace_id = { let entry_local_types = (frame_key != ROOT_FRAME_KEY).then(|| self.active_local_types()); + let entry_callable_prototypes = + self.active_local_callable_prototypes(); let program = &self.program; self.jit.observe_exit_entry_with_local_types( frame_key, ip, stack_depth, entry_local_types.as_deref(), + entry_callable_prototypes.as_deref(), program, ) }; diff --git a/src/vm/jit/trace.rs b/src/vm/jit/trace.rs index fb067e4a..ecf2d867 100644 --- a/src/vm/jit/trace.rs +++ b/src/vm/jit/trace.rs @@ -629,7 +629,7 @@ impl TraceJitEngine { stack_depth: usize, program: &Program, ) -> Option { - self.observe_exit_entry_with_local_types(frame_key, ip, stack_depth, None, program) + self.observe_exit_entry_with_local_types(frame_key, ip, stack_depth, None, None, program) } pub(crate) fn observe_exit_entry_with_local_types( @@ -638,9 +638,9 @@ impl TraceJitEngine { ip: usize, stack_depth: usize, entry_local_types: Option<&[crate::ValueType]>, + entry_callable_prototypes: Option<&[Option]>, program: &Program, ) -> Option { - let entry_callable_prototypes = None; if !self.config.enabled || !native_jit_supported() || self.callable_frame_is_blocked(frame_key) diff --git a/src/vm/mod.rs b/src/vm/mod.rs index aafb7d33..9eedd390 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -1066,6 +1066,27 @@ impl Vm { .collect() } + pub(super) fn active_local_callable_prototypes(&self) -> Option>> { + let base = self.active_local_base(); + let mut prototypes = Vec::with_capacity(self.locals.len().saturating_sub(base)); + for (offset, value) in self.locals[base..].iter().enumerate() { + let prototype_id = if let Some(cell) = self.capture_cells.get(&(base + offset)) { + let value = cell.lock().ok()?; + match &*value { + Value::Callable(callable) => Some(callable.prototype_id), + _ => None, + } + } else { + match value { + Value::Callable(callable) => Some(callable.prototype_id), + _ => None, + } + }; + prototypes.push(prototype_id); + } + Some(prototypes) + } + fn script_frame_depth(&self) -> usize { self.execution_frames .iter() @@ -2318,16 +2339,7 @@ impl Vm { let entry_local_types = (frame_key != crate::vm::native::ROOT_FRAME_KEY) .then(|| self.active_local_types()); let entry_callable_prototypes = - (frame_key == crate::vm::native::ROOT_FRAME_KEY).then(|| { - self.locals - .iter() - .take(self.program.local_count) - .map(|value| match value { - Value::Callable(callable) => Some(callable.prototype_id), - _ => None, - }) - .collect::>() - }); + self.active_local_callable_prototypes(); let program = &self.program; self.jit.observe_hot_entry_with_local_types( frame_key, From 9c0f376878c3405a1e793867c42602f57d4b8367 Mon Sep 17 00:00:00 2001 From: fffonion Date: Mon, 20 Jul 2026 06:17:18 +0800 Subject: [PATCH 22/26] fix(jit): avoid inherited state with shared captures --- src/vm/jit/recorder.rs | 4 +- src/vm/jit/runtime.rs | 65 ++++++++++++++----------------- src/vm/mod.rs | 36 ++++++++++++++++- src/vm/native/bridge.rs | 18 +++++++-- tests/jit/jit_tests.rs | 86 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 166 insertions(+), 43 deletions(-) diff --git a/src/vm/jit/recorder.rs b/src/vm/jit/recorder.rs index 4c374490..1209248d 100644 --- a/src/vm/jit/recorder.rs +++ b/src/vm/jit/recorder.rs @@ -1338,8 +1338,8 @@ pub(crate) fn record_trace_with_local_count( max_trace_len.saturating_sub(cursor.recorded_ops), ) .and_then(|candidate| { - let prototypes = entry_callable_prototypes - .ok_or(InlineRejectReason::UnknownTarget)?; + let prototypes = + entry_callable_prototypes.ok_or(InlineRejectReason::UnknownTarget)?; let source_local = callable .info .source_local diff --git a/src/vm/jit/runtime.rs b/src/vm/jit/runtime.rs index 703f59c9..964b6190 100644 --- a/src/vm/jit/runtime.rs +++ b/src/vm/jit/runtime.rs @@ -278,6 +278,17 @@ pub(crate) extern "C" fn pd_vm_native_resume_linked_trace(vm: *mut Vm) -> i32 { } impl Vm { + fn compiled_trace_for_active_entry(&self) -> Option { + if self.active_frame_has_shared_capture_cells() { + return None; + } + self.jit.compiled_trace_for_entry( + self.active_frame_key(), + self.ip, + self.active_operand_stack_len(), + ) + } + #[cfg(any( all( target_arch = "x86_64", @@ -291,10 +302,11 @@ impl Vm { let ip = self.ip; let frame_key = self.active_frame_key(); let stack_depth = self.active_operand_stack_len(); - let mut next_trace_id = self - .jit - .compiled_trace_for_entry(frame_key, ip, stack_depth); - if next_trace_id.is_none() && !self.jit.callable_frame_is_blocked(frame_key) { + let mut next_trace_id = self.compiled_trace_for_active_entry(); + if next_trace_id.is_none() + && !self.active_frame_has_shared_capture_cells() + && !self.jit.callable_frame_is_blocked(frame_key) + { let entry_local_types = (frame_key != ROOT_FRAME_KEY).then(|| self.active_local_types()); let entry_callable_prototypes = self.active_local_callable_prototypes(); @@ -392,11 +404,7 @@ impl Vm { let next_trace_id = if has_yielding_call { None } else { - self.jit.compiled_trace_for_entry( - self.active_frame_key(), - self.ip, - self.active_operand_stack_len(), - ) + self.compiled_trace_for_active_entry() }; if let Some(next_trace_id) = next_trace_id && next_trace_id != current_trace_id @@ -449,19 +457,16 @@ impl Vm { self.block_jit_callable_frame(current_trace_id); return Ok(native::STATUS_LINKED_CONTINUE); } - if !has_yielding_call { + if !has_yielding_call && !self.active_frame_has_shared_capture_cells() { let ip = self.ip; let frame_key = self.active_frame_key(); let stack_depth = self.active_operand_stack_len(); - let mut next_trace_id = - self.jit - .compiled_trace_for_entry(frame_key, ip, stack_depth); + let mut next_trace_id = self.compiled_trace_for_active_entry(); if next_trace_id.is_none() && !self.jit.callable_frame_is_blocked(frame_key) { let entry_local_types = (frame_key != ROOT_FRAME_KEY).then(|| self.active_local_types()); - let entry_callable_prototypes = - self.active_local_callable_prototypes(); + let entry_callable_prototypes = self.active_local_callable_prototypes(); let program = &self.program; next_trace_id = self.jit.observe_exit_entry_with_local_types( frame_key, @@ -1036,11 +1041,7 @@ impl Vm { let next_trace_id = if has_yielding_call { None } else { - self.jit.compiled_trace_for_entry( - self.active_frame_key(), - self.ip, - self.active_operand_stack_len(), - ) + self.compiled_trace_for_active_entry() }; if let Some(next_trace_id) = next_trace_id && next_trace_id != current_trace_id @@ -1116,13 +1117,11 @@ impl Vm { self.block_jit_callable_frame(current_trace_id); return Ok(ExecOutcome::Continue); } - if !has_yielding_call { + if !has_yielding_call && !self.active_frame_has_shared_capture_cells() { let ip = self.ip; let frame_key = self.active_frame_key(); let stack_depth = self.active_operand_stack_len(); - let mut next_trace_id = - self.jit - .compiled_trace_for_entry(frame_key, ip, stack_depth); + let mut next_trace_id = self.compiled_trace_for_active_entry(); if next_trace_id.is_none() && !self.jit.callable_frame_is_blocked(frame_key) { next_trace_id = { @@ -1192,18 +1191,18 @@ impl Vm { } native::STATUS_HALTED => return Ok(ExecOutcome::Halted), native::STATUS_LINKED_CONTINUE => { + if self.active_frame_has_shared_capture_cells() { + return Ok(ExecOutcome::Continue); + } let ip = self.ip; let frame_key = self.active_frame_key(); let stack_depth = self.active_operand_stack_len(); - let mut next_trace_id = - self.jit - .compiled_trace_for_entry(frame_key, ip, stack_depth); + let mut next_trace_id = self.compiled_trace_for_active_entry(); if next_trace_id.is_none() && !self.jit.callable_frame_is_blocked(frame_key) { next_trace_id = { let entry_local_types = (frame_key != ROOT_FRAME_KEY).then(|| self.active_local_types()); - let entry_callable_prototypes = - self.active_local_callable_prototypes(); + let entry_callable_prototypes = self.active_local_callable_prototypes(); let program = &self.program; self.jit.observe_exit_entry_with_local_types( frame_key, @@ -1569,14 +1568,10 @@ impl Vm { } pub(crate) fn jit_native_inherited_target(&self) -> usize { - if !self.jit_native_direct_links_enabled { + if !self.jit_native_direct_links_enabled || self.active_frame_has_shared_capture_cells() { return 0; } - let Some(trace_id) = self.jit.compiled_trace_for_entry( - self.active_frame_key(), - self.ip, - self.active_operand_stack_len(), - ) else { + let Some(trace_id) = self.compiled_trace_for_active_entry() else { return 0; }; self.native_traces diff --git a/src/vm/mod.rs b/src/vm/mod.rs index 9eedd390..7625ab17 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -1087,6 +1087,38 @@ impl Vm { Some(prototypes) } + pub(super) fn active_frame_has_shared_capture_cells(&self) -> bool { + if self.capture_cells.is_empty() { + return false; + } + let Some(frame) = self.execution_frames.last() else { + return false; + }; + let base = frame.local_base; + if let Some(prototype_id) = frame.prototype_id { + let Some(prototype) = self.program.callable_prototypes.get(prototype_id as usize) + else { + return true; + }; + return prototype + .capture_slots + .iter() + .zip(&prototype.capture_modes) + .any(|(slot, mode)| { + matches!( + mode, + crate::CaptureBindingMode::Borrow | crate::CaptureBindingMode::BorrowMut + ) && self + .capture_cells + .contains_key(&base.saturating_add(usize::from(*slot))) + }); + } + let end = base.saturating_add(frame.local_count); + self.capture_cells + .keys() + .any(|absolute| base <= *absolute && *absolute < end) + } + fn script_frame_depth(&self) -> usize { self.execution_frames .iter() @@ -2330,6 +2362,7 @@ impl Vm { && self.jit_config().enabled && self.builtin_overrides.is_empty() && !self.drop_contract_events_enabled() + && !self.active_frame_has_shared_capture_cells() { let frame_key = self.active_frame_key(); let trace_id = if self.jit.callable_frame_is_blocked(frame_key) { @@ -2338,8 +2371,7 @@ impl Vm { let stack_depth = self.active_operand_stack_len(); let entry_local_types = (frame_key != crate::vm::native::ROOT_FRAME_KEY) .then(|| self.active_local_types()); - let entry_callable_prototypes = - self.active_local_callable_prototypes(); + let entry_callable_prototypes = self.active_local_callable_prototypes(); let program = &self.program; self.jit.observe_hot_entry_with_local_types( frame_key, diff --git a/src/vm/native/bridge.rs b/src/vm/native/bridge.rs index 94740ca6..9a118e10 100644 --- a/src/vm/native/bridge.rs +++ b/src/vm/native/bridge.rs @@ -962,8 +962,13 @@ fn native_enter_call_value( ExecOutcome::Yielded => STATUS_YIELDED, ExecOutcome::Waiting(_) => STATUS_WAITING, }; - if status == STATUS_LINKED_CONTINUE && !inherited_state.is_null() { - write_inherited_state_packet(vm, inherited_state)?; + if status == STATUS_LINKED_CONTINUE { + if vm.active_frame_has_shared_capture_cells() { + return Ok(STATUS_CONTINUE); + } + if !inherited_state.is_null() { + write_inherited_state_packet(vm, inherited_state)?; + } } Ok(status) } @@ -1001,8 +1006,13 @@ fn native_leave_frame(vm: &mut Vm, ret_ip: i64, inherited_state: *mut u8) -> VmR ExecOutcome::Yielded => STATUS_YIELDED, ExecOutcome::Waiting(_) => STATUS_WAITING, }; - if status == STATUS_LINKED_CONTINUE && !inherited_state.is_null() { - write_inherited_state_packet(vm, inherited_state)?; + if status == STATUS_LINKED_CONTINUE { + if vm.active_frame_has_shared_capture_cells() { + return Ok(STATUS_CONTINUE); + } + if !inherited_state.is_null() { + write_inherited_state_packet(vm, inherited_state)?; + } } Ok(status) } diff --git a/tests/jit/jit_tests.rs b/tests/jit/jit_tests.rs index fc4f33fb..aa0fb42e 100644 --- a/tests/jit/jit_tests.rs +++ b/tests/jit/jit_tests.rs @@ -6547,6 +6547,92 @@ fn trace_jit_invalidates_native_inline_after_callable_local_replacement() { assert!(vm.jit_native_exec_count() > 0, "{}", vm.dump_jit_info()); } +#[test] +fn trace_jit_skips_frames_with_shared_capture_cells() { + if !native_jit_supported() { + return; + } + let source = r#" + let mut count = 0; + let recurse = |depth| if depth == 0 => { + count = count + 1; + count + } else => { + recurse(depth - 1); + count + }; + let mut i = 0; + let mut total = 0; + while i < 100 { + total = total + recurse(0) + count; + i = i + 1; + } + total; + "#; + let compiled = + compile_source(source).expect("shared capture continuation source should compile"); + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + + assert_eq!( + vm.run().expect("shared capture continuation run"), + VmStatus::Halted + ); + assert_eq!(vm.stack(), &[Value::Int(10_100)], "{}", vm.dump_jit_info()); + assert_eq!(vm.jit_native_exec_count(), 0, "{}", vm.dump_jit_info()); + assert_eq!( + vm.jit_native_active_direct_link_slot_count(), + 0, + "{}", + vm.dump_jit_info() + ); +} + +#[test] +fn trace_jit_native_return_does_not_link_into_shared_capture_caller() { + if !native_jit_supported() { + return; + } + let source = r#" + let mut shared = 40; + let capture = || &shared; + shared = 41; + fn work() { + let mut i = 0; + while i < 100 { + i = i + 1; + } + 2 + } + let before = capture(); + let result = work(); + before + shared + result; + "#; + let compiled = compile_source(source).expect("captured caller source should compile"); + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + + assert_eq!(vm.run().expect("captured caller run"), VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(84)], "{}", vm.dump_jit_info()); + assert!(vm.jit_native_exec_count() > 0, "{}", vm.dump_jit_info()); + assert!( + vm.jit_snapshot() + .traces + .iter() + .all(|trace| trace.frame_key != u64::MAX), + "root frame with shared captures must remain interpreted: {}", + vm.dump_jit_info() + ); +} + #[test] fn trace_jit_preserves_inline_callable_argument_schema_checks() { if !native_jit_supported() { From 38ba34796da39ce917971bb2f87b3e36b31771f9 Mon Sep 17 00:00:00 2001 From: fffonion Date: Mon, 20 Jul 2026 06:56:56 +0800 Subject: [PATCH 23/26] fix(jit): track nested shared capture slots --- src/vm/mod.rs | 48 +++++++++++++++++++++++------------------- tests/jit/jit_tests.rs | 47 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 22 deletions(-) diff --git a/src/vm/mod.rs b/src/vm/mod.rs index 7625ab17..1eaeae24 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -336,6 +336,7 @@ pub struct Vm { stack: Vec, locals: Vec, capture_cells: HashMap, + shared_capture_slots: HashSet, operand_type_hints: Option>, decoded_instruction_data: Arc, host_functions: Vec, @@ -667,6 +668,7 @@ impl Vm { stack: Vec::new(), locals: vec![Value::Null; local_count], capture_cells: HashMap::new(), + shared_capture_slots: HashSet::new(), operand_type_hints, decoded_instruction_data, host_functions: Vec::new(), @@ -913,6 +915,7 @@ impl Vm { self.clear_epoch_deadline(); self.clear_stack_with_drop_contract(); self.capture_cells.clear(); + self.shared_capture_slots.clear(); self.clear_locals_with_drop_contract(); self.owned_callables.clear(); self.locals.resize(self.program.local_count, Value::Null); @@ -1088,34 +1091,16 @@ impl Vm { } pub(super) fn active_frame_has_shared_capture_cells(&self) -> bool { - if self.capture_cells.is_empty() { + if self.shared_capture_slots.is_empty() { return false; } let Some(frame) = self.execution_frames.last() else { return false; }; let base = frame.local_base; - if let Some(prototype_id) = frame.prototype_id { - let Some(prototype) = self.program.callable_prototypes.get(prototype_id as usize) - else { - return true; - }; - return prototype - .capture_slots - .iter() - .zip(&prototype.capture_modes) - .any(|(slot, mode)| { - matches!( - mode, - crate::CaptureBindingMode::Borrow | crate::CaptureBindingMode::BorrowMut - ) && self - .capture_cells - .contains_key(&base.saturating_add(usize::from(*slot))) - }); - } let end = base.saturating_add(frame.local_count); - self.capture_cells - .keys() + self.shared_capture_slots + .iter() .any(|absolute| base <= *absolute && *absolute < end) } @@ -1236,6 +1221,7 @@ impl Drop for Vm { self.cancel_waiting_host_op(); self.clear_stack_with_drop_contract(); self.capture_cells.clear(); + self.shared_capture_slots.clear(); self.clear_locals_with_drop_contract(); crate::builtins::runtime::close_all_handles(self); } @@ -1292,6 +1278,7 @@ impl Vm { .entry(absolute) .or_insert_with(|| Arc::new(Mutex::new(value))) .clone(); + self.shared_capture_slots.insert(absolute); self.locals[absolute] = cell .lock() .map_err(|_| VmError::InvalidFrameState("capture cell lock is poisoned"))? @@ -1444,7 +1431,12 @@ impl Vm { "callable environment layout mismatch", )); } - for (slot, cell) in prototype.capture_slots.iter().zip(cells.iter()) { + for ((slot, mode), cell) in prototype + .capture_slots + .iter() + .zip(&prototype.capture_modes) + .zip(cells.iter()) + { let relative = *slot as usize; if relative >= local_count { return Err(VmError::InvalidFrameState( @@ -1460,6 +1452,13 @@ impl Vm { .clone(); if prototype.self_slot != Some(*slot) { self.capture_cells.insert(absolute, cell.clone()); + if matches!( + mode, + crate::CaptureBindingMode::Borrow + | crate::CaptureBindingMode::BorrowMut + ) { + self.shared_capture_slots.insert(absolute); + } } } } @@ -1544,6 +1543,8 @@ impl Vm { let frame_end = frame.local_base.saturating_add(frame.local_count); self.capture_cells .retain(|absolute, _| *absolute < frame.local_base || *absolute >= frame_end); + self.shared_capture_slots + .retain(|absolute| *absolute < frame.local_base || *absolute >= frame_end); } if !matches!(frame.continuation, FrameContinuation::Halt) { @@ -2969,6 +2970,7 @@ impl Vm { self.draining_queued_callables = false; self.clear_stack_with_drop_contract(); self.capture_cells.clear(); + self.shared_capture_slots.clear(); self.clear_locals_with_drop_contract(); self.execution_frames.clear(); self.active_local_base_cache = 0; @@ -3080,6 +3082,8 @@ impl Vm { let frame_end = frame.local_base.saturating_add(frame.local_count); self.capture_cells .retain(|absolute, _| *absolute < frame.local_base || *absolute >= frame_end); + self.shared_capture_slots + .retain(|absolute| *absolute < frame.local_base || *absolute >= frame_end); if frame.local_base <= self.locals.len() { let drained = self.locals.drain(frame.local_base..).collect::>(); for value in drained { diff --git a/tests/jit/jit_tests.rs b/tests/jit/jit_tests.rs index aa0fb42e..785200ad 100644 --- a/tests/jit/jit_tests.rs +++ b/tests/jit/jit_tests.rs @@ -6592,6 +6592,53 @@ fn trace_jit_skips_frames_with_shared_capture_cells() { ); } +#[test] +fn trace_jit_skips_callable_frame_with_nested_shared_capture() { + if !native_jit_supported() { + return; + } + let source = r#" + fn outer(limit: int) -> int { + let mut count = 0; + let recurse = |depth| if depth == 0 => { + count = count + 1; + count + } else => { + recurse(depth - 1); + count + }; + let mut i = 0; + let mut total = 0; + while i < limit { + total = total + recurse(0) + count; + i = i + 1; + } + total + } + outer(100); + "#; + let compiled = compile_source(source).expect("nested shared capture source should compile"); + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + + assert_eq!( + vm.run().expect("nested shared capture run"), + VmStatus::Halted + ); + assert_eq!(vm.stack(), &[Value::Int(10_100)], "{}", vm.dump_jit_info()); + assert_eq!(vm.jit_native_exec_count(), 0, "{}", vm.dump_jit_info()); + assert_eq!( + vm.jit_native_active_direct_link_slot_count(), + 0, + "{}", + vm.dump_jit_info() + ); +} + #[test] fn trace_jit_native_return_does_not_link_into_shared_capture_caller() { if !native_jit_supported() { From d7d24a425cabccb55fc253b2c823eb5b80c38c33 Mon Sep 17 00:00:00 2001 From: fffonion Date: Mon, 20 Jul 2026 17:09:35 +0800 Subject: [PATCH 24/26] fix(vm): read captured scalar locals from shared cells --- src/vm/superinstructions.rs | 33 ++++++++++----------------------- tests/jit/jit_tests.rs | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/src/vm/superinstructions.rs b/src/vm/superinstructions.rs index 2453be23..47c8e581 100644 --- a/src/vm/superinstructions.rs +++ b/src/vm/superinstructions.rs @@ -27,33 +27,20 @@ impl ScalarValue { impl Vm { #[inline(always)] - fn local_scalar_value_with_hint( - &mut self, - local_base: usize, - index: u8, - ) -> Option { - let absolute = local_base.checked_add(index as usize)?; - match self.local_type_hint(index) { - ValueType::Int => { - let value = match self.locals.get(absolute)? { - Value::Int(value) => *value, - _ => return None, - }; + fn local_scalar_value_with_hint(&mut self, index: u8) -> Option { + let value = self.local_numeric_value(index)?; + match (self.local_type_hint(index), value) { + (ValueType::Int, NumericValue::Int(value)) => { self.record_local_type_hint_hit(); Some(ScalarValue::Int(value)) } - ValueType::Float => { - let value = match self.locals.get(absolute)? { - Value::Float(value) => *value, - _ => return None, - }; + (ValueType::Float, NumericValue::Float(value)) => { self.record_local_type_hint_hit(); Some(ScalarValue::Float(value)) } - _ => self.local_numeric_value(index).map(|value| match value { - NumericValue::Int(value) => ScalarValue::Int(value), - NumericValue::Float(value) => ScalarValue::Float(value), - }), + (ValueType::Int | ValueType::Float, _) => None, + (_, NumericValue::Int(value)) => Some(ScalarValue::Int(value)), + (_, NumericValue::Float(value)) => Some(ScalarValue::Float(value)), } } @@ -100,7 +87,7 @@ impl Vm { return Ok(false); } let local_base = self.active_local_base(); - let Some(initial) = self.local_scalar_value_with_hint(local_base, src) else { + let Some(initial) = self.local_scalar_value_with_hint(src) else { return Ok(false); }; let mut cursor = self.ip; @@ -135,7 +122,7 @@ impl Vm { let Some(index) = self.decoded_local_index_at(cursor) else { return Ok(false); }; - let Some(value) = self.local_scalar_value_with_hint(local_base, index) else { + let Some(value) = self.local_scalar_value_with_hint(index) else { return Ok(false); }; if stack_len == stack.len() { diff --git a/tests/jit/jit_tests.rs b/tests/jit/jit_tests.rs index 785200ad..cca4c18c 100644 --- a/tests/jit/jit_tests.rs +++ b/tests/jit/jit_tests.rs @@ -6411,6 +6411,38 @@ fn interpreter_superinstructions_use_script_frame_local_base() { ); } +#[test] +fn interpreter_superinstructions_read_nested_shared_capture_cells() { + let source = r#" + fn outer() -> int { + let mut count = 0; + fn set_count() -> int { + count = 5; + count + } + set_count(); + count = count + 1; + count + } + outer(); + "#; + let compiled = + compile_source(source).expect("shared-capture superinstruction source should compile"); + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_jit_config(JitConfig { + enabled: false, + ..JitConfig::default() + }); + + assert_eq!(vm.run().unwrap(), VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(6)]); + assert!( + vm.interpreter_metrics_snapshot() + .scalar_superinstruction_count + > 0 + ); +} + #[test] fn trace_jit_inlines_static_leaf_in_root_loop() { if !native_jit_supported() { From dc5147af8b16d0fc1567b611ec544b7aec2fcab0 Mon Sep 17 00:00:00 2001 From: fffonion Date: Mon, 20 Jul 2026 18:50:07 +0800 Subject: [PATCH 25/26] fix(jit): restore inline failures and callable guards --- src/vm/jit/ir.rs | 14 ++ src/vm/jit/native/lower.rs | 90 ++++++-- src/vm/jit/recorder.rs | 417 ++++++++++++++++++++++++++++++++++--- src/vm/jit/region.rs | 25 ++- src/vm/jit/runtime.rs | 24 ++- src/vm/jit/trace.rs | 194 ++++++++++++++++- tests/jit/jit_tests.rs | 171 +++++++++++++++ 7 files changed, 882 insertions(+), 53 deletions(-) diff --git a/src/vm/jit/ir.rs b/src/vm/jit/ir.rs index 8a7c2808..5b3e50f6 100644 --- a/src/vm/jit/ir.rs +++ b/src/vm/jit/ir.rs @@ -497,6 +497,7 @@ impl SsaInstKind { pub(crate) struct SsaInst { pub(crate) ip: usize, pub(crate) output: Option, + pub(crate) failure_exit: Option, pub(crate) kind: SsaInstKind, } @@ -638,6 +639,11 @@ impl SsaTrace { scope.insert(param.value.id, param.value.repr); } for inst in &block.insts { + if let Some(failure_exit) = inst.failure_exit + && !exit_ids.contains(&failure_exit) + { + return Err(SsaVerifyError::UnknownExit(failure_exit)); + } for input in inst.kind.inputs() { if !scope.contains_key(&input) { return Err(SsaVerifyError::UseBeforeDef { @@ -849,6 +855,7 @@ pub(crate) enum SsaVerifyError { pub(crate) struct SsaTraceBuilder { trace: SsaTrace, next_value: u32, + current_failure_exit: Option, } impl SsaTraceBuilder { @@ -868,6 +875,7 @@ impl SsaTraceBuilder { exits: Vec::new(), }, next_value: 0, + current_failure_exit: None, } } @@ -916,14 +924,20 @@ impl SsaTraceBuilder { kind: SsaInstKind, ) -> VmResult { let output = self.alloc_value(repr); + let failure_exit = self.current_failure_exit; self.block_mut(block)?.insts.push(SsaInst { ip, output: Some(output), + failure_exit, kind, }); Ok(output) } + pub(crate) fn set_current_failure_exit(&mut self, exit: Option) { + self.current_failure_exit = exit; + } + pub(crate) fn add_exit( &mut self, exit_ip: usize, diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index cd4899bc..1462dc9c 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -941,6 +941,7 @@ fn try_compile_ssa_trace( borrowed_array_gets: &borrowed_array_gets, value_reprs: &value_reprs, tagged_constant_addrs: &tagged_constant_addrs, + exit_specs: &exit_specs, interrupt_settings, interrupt_ops_to_advance, }; @@ -1235,6 +1236,7 @@ struct SsaLowerCtx<'a> { borrowed_array_gets: &'a BTreeSet, value_reprs: &'a HashMap, tagged_constant_addrs: &'a HashMap, + exit_specs: &'a HashMap, interrupt_settings: Option, interrupt_ops_to_advance: u32, } @@ -1262,6 +1264,7 @@ struct SsaBoxCtx { struct SsaConcatOp { output_id: SsaValueId, ip: usize, + failure_exit: Option, lhs: cranelift_codegen::ir::Value, rhs: cranelift_codegen::ir::Value, result_tag: u32, @@ -1995,7 +1998,6 @@ fn lower_ssa_inst( exit_block, pointer_type, layout, - offsets, heap_refs, heap_addrs, string_refs, @@ -2104,7 +2106,7 @@ fn lower_ssa_inst( b.ins().jump(cont, &[]); b.switch_to_block(fail); - ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, inst.ip)?; + ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values)?; b.switch_to_block(cont); out @@ -2132,7 +2134,7 @@ fn lower_ssa_inst( b.ins().jump(cont, &[]); b.switch_to_block(fail); - ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, inst.ip)?; + ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values)?; b.switch_to_block(cont); out @@ -2161,7 +2163,7 @@ fn lower_ssa_inst( b.ins().jump(cont, &[]); b.switch_to_block(fail); - ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, inst.ip)?; + ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values)?; b.switch_to_block(cont); out @@ -2183,7 +2185,7 @@ fn lower_ssa_inst( b.ins().jump(cont, &[]); b.switch_to_block(fail); - ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, inst.ip)?; + ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values)?; b.switch_to_block(cont); out @@ -2637,7 +2639,7 @@ fn lower_ssa_inst( ); b.switch_to_block(fail); - ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, inst.ip)?; + ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values)?; b.switch_to_block(cont); out @@ -2671,7 +2673,7 @@ fn lower_ssa_inst( b.ins().jump(cont, &[]); b.switch_to_block(fail); - ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, inst.ip)?; + ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values)?; b.switch_to_block(cont); out @@ -2844,9 +2846,11 @@ fn lower_ssa_inst( SsaInstKind::StringConcat { lhs, rhs } => ssa_inline_concat( b, ctx, + values, SsaConcatOp { output_id: output.id, ip: inst.ip, + failure_exit: inst.failure_exit, lhs: values[lhs], rhs: values[rhs], result_tag: layout.value.string_tag, @@ -2856,9 +2860,11 @@ fn lower_ssa_inst( SsaInstKind::BytesConcat { lhs, rhs } => ssa_inline_concat( b, ctx, + values, SsaConcatOp { output_id: output.id, ip: inst.ip, + failure_exit: inst.failure_exit, lhs: values[lhs], rhs: values[rhs], result_tag: layout.value.bytes_tag, @@ -2989,7 +2995,7 @@ fn lower_ssa_inst( b.ins().jump(cont, &[]); b.switch_to_block(fail); - ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, inst.ip)?; + ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values)?; b.switch_to_block(cont); out @@ -3075,7 +3081,7 @@ fn lower_ssa_inst( b.ins().jump(cont, &[]); b.switch_to_block(fail); - ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, inst.ip)?; + ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values)?; b.switch_to_block(cont); out @@ -3162,7 +3168,7 @@ fn lower_ssa_inst( b.ins().jump(cont, &[]); b.switch_to_block(fail); - ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, inst.ip)?; + ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values)?; b.switch_to_block(cont); out @@ -3269,7 +3275,7 @@ fn lower_ssa_inst( b.ins().jump(done, &[]); b.switch_to_block(fail); - ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, inst.ip)?; + ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values)?; b.switch_to_block(done); out @@ -3461,7 +3467,7 @@ fn lower_ssa_inst( jump_with_status(b, exit_block, status); b.switch_to_block(miss_block); - ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, inst.ip)?; + ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values)?; b.switch_to_block(cont); out @@ -3771,7 +3777,7 @@ fn lower_ssa_inst( b.ins().jump(cont, &[]); b.switch_to_block(fail); - ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, inst.ip)?; + ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values)?; b.switch_to_block(cont); out @@ -3807,7 +3813,7 @@ fn lower_ssa_inst( b.ins().jump(cont, &[]); b.switch_to_block(fail); - ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, inst.ip)?; + ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values)?; b.switch_to_block(cont); out @@ -3840,7 +3846,7 @@ fn lower_ssa_inst( b.ins().jump(cont, &[]); b.switch_to_block(fail); - ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, inst.ip)?; + ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values)?; b.switch_to_block(cont); out @@ -3868,7 +3874,7 @@ fn lower_ssa_inst( b.ins().jump(cont, &[]); b.switch_to_block(fail); - ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, inst.ip)?; + ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values)?; b.switch_to_block(cont); out @@ -3896,7 +3902,7 @@ fn lower_ssa_inst( b.ins().jump(cont, &[]); b.switch_to_block(fail); - ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, inst.ip)?; + ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values)?; b.switch_to_block(cont); out @@ -5098,14 +5104,12 @@ fn ssa_index_in_range( fn ssa_inline_concat( b: &mut FunctionBuilder, ctx: SsaLowerCtx<'_>, + values: &HashMap, op: SsaConcatOp, ) -> VmResult { let SsaLowerCtx { - vm_ptr, - exit_block, pointer_type, layout, - offsets, heap_refs, heap_addrs, helper_refs, @@ -5116,6 +5120,7 @@ fn ssa_inline_concat( let SsaConcatOp { output_id, ip, + failure_exit, lhs, rhs, result_tag, @@ -5213,7 +5218,7 @@ fn ssa_inline_concat( b.ins().jump(cont, &[]); b.switch_to_block(fail); - ssa_emit_trace_exit_status(b, vm_ptr, exit_block, pointer_type, offsets, ip)?; + ssa_emit_failure_exit(b, ctx, failure_exit, ip, values)?; b.switch_to_block(cont); Ok(out) @@ -5698,6 +5703,49 @@ fn ssa_utf8_char_width( b.ins().select(lt_80, one, wide) } +fn ssa_emit_failure_exit( + b: &mut FunctionBuilder, + ctx: SsaLowerCtx<'_>, + failure_exit: Option, + ip: usize, + values: &HashMap, +) -> VmResult<()> { + if let Some(failure_exit) = failure_exit { + let spec = ctx.exit_specs.get(&failure_exit).ok_or_else(|| { + VmError::JitNative(format!( + "SSA instruction failure exit {} lowering missing", + failure_exit.raw() + )) + })?; + let args = spec + .inputs + .iter() + .map(|value| { + values + .get(value) + .copied() + .map(BlockArg::Value) + .ok_or_else(|| { + VmError::JitNative(format!( + "SSA instruction failure exit input {} missing", + value.raw() + )) + }) + }) + .collect::>>()?; + b.ins().jump(spec.trace_exit_block, &args); + return Ok(()); + } + ssa_emit_trace_exit_status( + b, + ctx.vm_ptr, + ctx.exit_block, + ctx.pointer_type, + ctx.offsets, + ip, + ) +} + fn ssa_emit_trace_exit_status( b: &mut FunctionBuilder, vm_ptr: cranelift_codegen::ir::Value, diff --git a/src/vm/jit/recorder.rs b/src/vm/jit/recorder.rs index 1209248d..9b8474a5 100644 --- a/src/vm/jit/recorder.rs +++ b/src/vm/jit/recorder.rs @@ -16,6 +16,7 @@ use super::ir::{ pub(crate) struct RecordedTrace { pub(crate) has_call: bool, pub(crate) has_yielding_call: bool, + pub(crate) entry_callable_guards: Vec<(u8, u32)>, pub(crate) op_names: Vec, pub(crate) ssa: SsaTrace, pub(crate) terminal: JitTraceTerminal, @@ -187,7 +188,9 @@ impl ValueInfo { } fn sourced_from(mut self, local: u8) -> Self { - self.source_local = Some(local); + if self.source_local.is_none() { + self.source_local = Some(local); + } self } } @@ -277,10 +280,8 @@ fn inline_schema_guard_type(schema: &TypeSchema) -> Option> { TypeSchema::Array(_) | TypeSchema::ArrayTuple(_) | TypeSchema::ArrayTupleRest { .. } => { Some(Some(ValueType::Array)) } - TypeSchema::Null - | TypeSchema::Number - | TypeSchema::Optional(_) - | TypeSchema::Callable { .. } => None, + TypeSchema::Null => Some(Some(ValueType::Null)), + TypeSchema::Number | TypeSchema::Optional(_) | TypeSchema::Callable { .. } => None, } } @@ -288,10 +289,11 @@ fn inline_argument_schemas_supported( arguments: &[SymbolicValue], schema: Option<&TypeSchema>, ) -> bool { - let Some(TypeSchema::Callable { params, .. }) = schema else { + let Some(TypeSchema::Callable { params, result }) = schema else { return schema.is_none(); }; - params.len() == arguments.len() + inline_schema_guard_type(result).is_some() + && params.len() == arguments.len() && params.iter().zip(arguments).all(|(schema, argument)| { let Some(guard_type) = inline_schema_guard_type(schema) else { return false; @@ -356,6 +358,58 @@ fn append_inline_argument_schema_guards( Ok(guard) } +fn append_inline_result_schema_guard( + builder: &mut SsaTraceBuilder, + block: super::ir::SsaBlockId, + ip: usize, + result: SymbolicValue, + schema: Option<&TypeSchema>, +) -> Result, TraceRecordError> { + let Some(TypeSchema::Callable { result: schema, .. }) = schema else { + return Ok(None); + }; + let Some(guard_type) = inline_schema_guard_type(schema) else { + return Err(TraceRecordError::UnsupportedTrace( + "inline callable return schema is not guardable".to_string(), + )); + }; + let Some(expected) = guard_type else { + return Ok(None); + }; + let statically_matches = match result.info.repr { + SsaValueRepr::I64 => expected == ValueType::Int, + SsaValueRepr::F64 => expected == ValueType::Float, + SsaValueRepr::Bool => expected == ValueType::Bool, + SsaValueRepr::HeapPtr(actual) => expected == actual, + SsaValueRepr::Tagged => { + let predicate = builder + .append_value_inst( + block, + ip, + SsaValueRepr::Bool, + SsaInstKind::ValueIsType { + input: result.value.id, + tag: expected, + }, + ) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + return Ok(Some(predicate.id)); + } + }; + if statically_matches { + return Ok(None); + } + let predicate = builder + .append_value_inst( + block, + ip, + SsaValueRepr::Bool, + SsaInstKind::Constant(Value::Bool(false)), + ) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + Ok(Some(predicate.id)) +} + #[derive(Clone, Debug, PartialEq)] struct SymbolicFrame { stack: Vec, @@ -497,6 +551,39 @@ enum DecodedOp { } impl DecodedOp { + fn ip(self) -> usize { + match self { + Self::Nop { ip } + | Self::Ret { ip } + | Self::Ldc { ip, .. } + | Self::Ldloc { ip, .. } + | Self::Stloc { ip, .. } + | Self::Pop { ip } + | Self::Dup { ip } + | Self::Neg { ip } + | Self::Not { ip } + | Self::BinOp { ip, .. } + | Self::Compare { ip, .. } + | Self::Brfalse { ip, .. } + | Self::Br { ip, .. } + | Self::Call { ip, .. } + | Self::CallValue { ip, .. } => ip, + } + } + + fn may_need_inline_failure_exit(self) -> bool { + matches!( + self, + Self::Ret { .. } + | Self::Neg { .. } + | Self::Not { .. } + | Self::BinOp { .. } + | Self::Compare { .. } + | Self::Brfalse { .. } + | Self::Call { .. } + ) + } + fn is_useful_native_computation(self) -> bool { match self { Self::Nop { .. } @@ -1001,6 +1088,7 @@ pub(crate) fn record_trace_with_local_count( let mut has_call = false; let mut has_yielding_call = false; let mut has_useful_native_computation = false; + let mut entry_callable_guards = Vec::new(); let mut inline_frame: Option = None; loop { @@ -1008,28 +1096,81 @@ pub(crate) fn record_trace_with_local_count( break; }; let is_useful_native_computation = decoded.is_useful_native_computation(); + let instruction_failure_exit = if decoded.may_need_inline_failure_exit() { + inline_frame + .as_ref() + .map(|inlined| add_symbolic_exit(&mut builder, decoded.ip(), &frame, Some(inlined))) + } else { + None + }; + builder.set_current_failure_exit(instruction_failure_exit); match decoded { DecodedOp::Nop { .. } => op_names.push("nop".to_string()), DecodedOp::Ret { ip } => { - if let Some(inlined) = inline_frame.take() { - let result = match frame.stack.pop() { - Some(result) => result, - None => { - let value = builder - .append_value_inst( - current_block, - ip, - SsaValueRepr::Tagged, - SsaInstKind::Constant(Value::Null), - ) - .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; - SymbolicValue { - value, - info: ValueInfo::tagged_typed(ValueType::Null), - } - } - }; + if inline_frame.is_some() { + if frame.stack.is_empty() { + let value = builder + .append_value_inst( + current_block, + ip, + SsaValueRepr::Tagged, + SsaInstKind::Constant(Value::Null), + ) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + frame.push(SymbolicValue { + value, + info: ValueInfo::tagged_typed(ValueType::Null), + }); + } + let result = *frame.stack.last().expect("inline result ensured above"); + let prototype_id = inline_frame + .as_ref() + .expect("inline frame checked above") + .candidate + .prototype_id; + let return_guard = append_inline_result_schema_guard( + &mut builder, + current_block, + ip, + result, + program.callable_prototypes[prototype_id as usize] + .schema + .as_ref(), + )?; + if let Some(condition) = return_guard { + let failure_exit = instruction_failure_exit.ok_or_else(|| { + TraceRecordError::InvalidIr( + "inline return schema guard is missing its virtual-frame exit" + .to_string(), + ) + })?; + let (guarded_block, guarded_frame, guarded_args) = + continue_with_inline_frame( + &mut builder, + &frame, + &mut inline_frame, + "inline_return_schema", + )?; + builder + .set_terminator( + current_block, + SsaTerminator::BranchBool { + condition, + if_true: super::ir::SsaBranchTarget::Block { + target: guarded_block, + args: guarded_args, + }, + if_false: super::ir::SsaBranchTarget::Exit(failure_exit), + }, + ) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + current_block = guarded_block; + frame = guarded_frame; + op_names.push("inline_return_schema_guard".to_string()); + } + let inlined = inline_frame.take().expect("inline frame checked above"); + let result = frame.pop()?; op_names.push("inline_ret".to_string()); frame = inlined.caller; frame.push(result); @@ -1362,6 +1503,17 @@ pub(crate) fn record_trace_with_local_count( if inline_frame.is_none() && let Ok(candidate) = candidate { + let source_local = + callable + .info + .source_local + .ok_or(TraceRecordError::UnsupportedTrace( + "inline candidate lost callable source local".to_string(), + ))?; + let entry_guard = (source_local, candidate.prototype_id); + if !entry_callable_guards.contains(&entry_guard) { + entry_callable_guards.push(entry_guard); + } let prototype = &program.callable_prototypes[candidate.prototype_id as usize]; let argument_start = frame.stack.len() - usize::from(argc); let schema_guard = append_inline_argument_schema_guards( @@ -1594,6 +1746,21 @@ pub(crate) fn record_trace_with_local_count( } let terminal = terminal.ok_or(TraceRecordError::MissingTerminal)?; + if loop_header_plan.is_some() + && entry_callable_guards.iter().any(|(local, _)| { + let local = usize::from(*local); + frame.dirty_locals.get(local).copied().unwrap_or(false) + || inline_frame + .as_ref() + .and_then(|inlined| inlined.caller.dirty_locals.get(local)) + .copied() + .unwrap_or(false) + }) + { + return Err(TraceRecordError::UnsupportedTrace( + "inline callable source local is mutated by the native loop".to_string(), + )); + } let ssa = builder.finish(); ssa.verify() .map_err(|err| TraceRecordError::InvalidIr(format!("{err:?}")))?; @@ -1601,6 +1768,7 @@ pub(crate) fn record_trace_with_local_count( Ok(RecordedTrace { has_call, has_yielding_call, + entry_callable_guards, op_names, ssa, terminal, @@ -5332,6 +5500,205 @@ mod tests { )); } + #[test] + fn inline_static_leaf_rejects_loop_that_mutates_guarded_callable_source() { + let mut bc = BytecodeBuilder::new(); + let root_ip = bc.position(); + bc.ldloc(0); + bc.call_value(0); + bc.pop(); + bc.ldloc(1); + bc.stloc(0); + bc.br(root_ip); + let first_entry = bc.position(); + bc.ldc(0); + bc.ret(); + let first_end = bc.position(); + let second_entry = bc.position(); + bc.ldc(1); + bc.ret(); + let second_end = bc.position(); + let program = Program::new(vec![Value::Int(1), Value::Int(2)], bc.finish()) + .with_local_count(2) + .with_callable_metadata( + vec![ + ScriptFunction { + entry_ip: first_entry, + end_ip: first_end, + }, + ScriptFunction { + entry_ip: second_entry, + end_ip: second_end, + }, + ], + vec![ + CallablePrototype { + kind: CallableKind::FunctionItem, + target: CallableTarget::ScriptFunction(0), + arity: 0, + frame_local_count: 1, + parameter_slots: Vec::new(), + capture_source_slots: Vec::new(), + capture_slots: Vec::new(), + capture_modes: Vec::new(), + self_slot: None, + schema: None, + }, + CallablePrototype { + kind: CallableKind::FunctionItem, + target: CallableTarget::ScriptFunction(1), + arity: 0, + frame_local_count: 1, + parameter_slots: Vec::new(), + capture_source_slots: Vec::new(), + capture_slots: Vec::new(), + capture_modes: Vec::new(), + self_slot: None, + schema: None, + }, + ], + vec![ + FunctionRegion { + start_ip: first_entry, + end_ip: first_end, + prototype_id: Some(0), + }, + FunctionRegion { + start_ip: second_entry, + end_ip: second_end, + prototype_id: Some(1), + }, + ], + vec![ + RootCallableBinding { + local_slot: 0, + prototype_id: 0, + }, + RootCallableBinding { + local_slot: 1, + prototype_id: 1, + }, + ], + ); + + let error = record_trace_with_local_count( + &program, + crate::vm::native::ROOT_FRAME_KEY, + root_ip as usize, + 0, + 2, + None, + Some(&[Some(0), Some(1)]), + 64, + &[], + ) + .expect_err("mutated callable source must reject native loop"); + assert!( + matches!(error, TraceRecordError::UnsupportedTrace(ref reason) if reason.contains("callable source local")), + "unexpected error: {error:?}" + ); + } + + #[test] + fn inline_static_leaf_tracks_callable_source_across_local_copy() { + let mut bc = BytecodeBuilder::new(); + let root_ip = bc.position(); + bc.ldloc(1); + bc.stloc(0); + bc.ldloc(0); + bc.call_value(0); + bc.pop(); + bc.br(root_ip); + let first_entry = bc.position(); + bc.ldc(0); + bc.ret(); + let first_end = bc.position(); + let second_entry = bc.position(); + bc.ldc(1); + bc.ret(); + let second_end = bc.position(); + + let program = Program::new(vec![Value::Int(1), Value::Int(2)], bc.finish()) + .with_local_count(2) + .with_callable_metadata( + vec![ + ScriptFunction { + entry_ip: first_entry, + end_ip: first_end, + }, + ScriptFunction { + entry_ip: second_entry, + end_ip: second_end, + }, + ], + vec![ + CallablePrototype { + kind: CallableKind::FunctionItem, + target: CallableTarget::ScriptFunction(0), + arity: 0, + frame_local_count: 2, + parameter_slots: Vec::new(), + capture_source_slots: Vec::new(), + capture_slots: Vec::new(), + capture_modes: Vec::new(), + self_slot: None, + schema: None, + }, + CallablePrototype { + kind: CallableKind::FunctionItem, + target: CallableTarget::ScriptFunction(1), + arity: 0, + frame_local_count: 2, + parameter_slots: Vec::new(), + capture_source_slots: Vec::new(), + capture_slots: Vec::new(), + capture_modes: Vec::new(), + self_slot: None, + schema: None, + }, + ], + vec![ + FunctionRegion { + start_ip: first_entry, + end_ip: first_end, + prototype_id: Some(0), + }, + FunctionRegion { + start_ip: second_entry, + end_ip: second_end, + prototype_id: Some(1), + }, + ], + vec![ + RootCallableBinding { + local_slot: 0, + prototype_id: 0, + }, + RootCallableBinding { + local_slot: 1, + prototype_id: 1, + }, + ], + ); + let callable_prototypes = [Some(0), Some(1)]; + + let recorded = record_trace_with_local_count( + &program, + crate::vm::native::ROOT_FRAME_KEY, + root_ip as usize, + 0, + program.local_count, + None, + Some(&callable_prototypes), + 64, + &[], + ) + .expect("copied callable trace"); + + assert!(recorded.op_names.iter().any(|name| name == "inline_call:1")); + assert!(!recorded.op_names.iter().any(|name| name == "inline_call:0")); + } + #[test] fn inline_static_leaf_records_call_and_return_inside_root_loop() { let mut bc = BytecodeBuilder::new(); diff --git a/src/vm/jit/region.rs b/src/vm/jit/region.rs index cce7defa..aea8c35b 100644 --- a/src/vm/jit/region.rs +++ b/src/vm/jit/region.rs @@ -166,7 +166,7 @@ fn offset_block( param.value.id = offset_value_id(param.value.id, value_offset)?; } for inst in &mut block.insts { - offset_inst(inst, value_offset)?; + offset_inst(inst, value_offset, exit_offset)?; } if let Some(terminator) = &mut block.terminator { offset_terminator(terminator, value_offset, block_offset, exit_offset)?; @@ -187,10 +187,13 @@ fn offset_exit(mut exit: SsaExit, value_offset: u32, exit_offset: u32) -> VmResu Ok(exit) } -fn offset_inst(inst: &mut SsaInst, value_offset: u32) -> VmResult<()> { +fn offset_inst(inst: &mut SsaInst, value_offset: u32, exit_offset: u32) -> VmResult<()> { if let Some(output) = &mut inst.output { output.id = offset_value_id(output.id, value_offset)?; } + if let Some(failure_exit) = &mut inst.failure_exit { + *failure_exit = offset_exit_id(*failure_exit, exit_offset)?; + } remap_inst_inputs(&mut inst.kind, |value| offset_value_id(value, value_offset)) } @@ -528,6 +531,7 @@ mod tests { start_line: None, has_call: false, has_yielding_call: false, + entry_callable_guards: Vec::new(), op_names: vec![format!("trace{id}")], terminal: JitTraceTerminal::BranchExit, executions: 0, @@ -537,6 +541,23 @@ mod tests { ) } + #[test] + fn offset_inst_remaps_failure_exit() { + let mut inst = SsaInst { + ip: 7, + output: Some(crate::vm::jit::ir::SsaValue { + id: SsaValueId::new(1), + repr: crate::vm::jit::ir::SsaValueRepr::I64, + }), + failure_exit: Some(SsaExitId::new(0)), + kind: SsaInstKind::Constant(crate::Value::Int(1)), + }; + + offset_inst(&mut inst, 10, 2).expect("instruction remap should succeed"); + assert_eq!(inst.output.expect("output").id, SsaValueId::new(11)); + assert_eq!(inst.failure_exit, Some(SsaExitId::new(2))); + } + #[test] fn offset_exit_remaps_virtual_frame_materializations() { let exit = SsaExit { diff --git a/src/vm/jit/runtime.rs b/src/vm/jit/runtime.rs index 964b6190..30168b21 100644 --- a/src/vm/jit/runtime.rs +++ b/src/vm/jit/runtime.rs @@ -282,10 +282,12 @@ impl Vm { if self.active_frame_has_shared_capture_cells() { return None; } - self.jit.compiled_trace_for_entry( + let entry_callable_prototypes = self.active_local_callable_prototypes(); + self.jit.compiled_trace_for_entry_with_callables( self.active_frame_key(), self.ip, self.active_operand_stack_len(), + entry_callable_prototypes.as_deref(), ) } @@ -377,6 +379,7 @@ impl Vm { } self.jit.mark_trace_executed(current_trace_id); let mut trace_exit_key = None; + let mut instruction_failure_exit = false; let status = if let Some(exit_id) = native::decode_jit_trace_exit_status(status) { let key = if let Some(region_exit_keys) = &exit_keys { *region_exit_keys.get(&exit_id).ok_or_else(|| { @@ -390,6 +393,7 @@ impl Vm { exit_id: SsaExitId::new(exit_id), } }; + instruction_failure_exit = self.jit.trace_exit_is_instruction_failure(key); self.jit .record_trace_exit(key) .map_err(|err| VmError::JitNative(err.message()))?; @@ -442,6 +446,9 @@ impl Vm { } native::STATUS_TRACE_EXIT => { self.jit_trace_exit_count = self.jit_trace_exit_count.saturating_add(1); + if instruction_failure_exit { + return Ok(native::STATUS_LINKED_CONTINUE); + } if !has_yielding_call && terminal == JitTraceTerminal::LoopBack && self.ip == root_ip @@ -583,6 +590,9 @@ impl Vm { slot_id: u32, child_trace_id: usize, ) -> VmResult<()> { + if self.jit.trace_has_entry_callable_guards(child_trace_id) { + return Ok(()); + } if !self.jit_native_direct_cross_frame_enabled { let parent_frame_key = self .jit @@ -645,6 +655,13 @@ impl Vm { if self.jit_native_direct_links_enabled && !self.jit_native_direct_region_fallback { return; } + if self + .jit + .trace_has_entry_callable_guards(key.parent_trace_id) + || self.jit.trace_has_entry_callable_guards(child_trace_id) + { + return; + } let Some(candidate) = self.jit.region_candidate(key, child_trace_id) else { return; }; @@ -1014,6 +1031,7 @@ impl Vm { } self.jit.mark_trace_executed(current_trace_id); let mut trace_exit_key = None; + let mut instruction_failure_exit = false; let status = if let Some(exit_id) = native::decode_jit_trace_exit_status(status) { let key = if let Some(region_exit_keys) = &exit_keys { *region_exit_keys.get(&exit_id).ok_or_else(|| { @@ -1027,6 +1045,7 @@ impl Vm { exit_id: SsaExitId::new(exit_id), } }; + instruction_failure_exit = self.jit.trace_exit_is_instruction_failure(key); self.jit .record_trace_exit(key) .map_err(|err| VmError::JitNative(err.message()))?; @@ -1094,6 +1113,9 @@ impl Vm { } native::STATUS_TRACE_EXIT => { self.jit_trace_exit_count = self.jit_trace_exit_count.saturating_add(1); + if instruction_failure_exit { + return Ok(ExecOutcome::Continue); + } if self.jit.trace_clone(current_trace_id).is_some_and(|trace| { trace.op_names.last().map(String::as_str) == Some("callable_boundary") }) { diff --git a/src/vm/jit/trace.rs b/src/vm/jit/trace.rs index ecf2d867..43afd16f 100644 --- a/src/vm/jit/trace.rs +++ b/src/vm/jit/trace.rs @@ -193,6 +193,7 @@ pub struct JitTrace { pub start_line: Option, pub has_call: bool, pub has_yielding_call: bool, + pub(crate) entry_callable_guards: Vec<(u8, u32)>, pub op_names: Vec, pub terminal: JitTraceTerminal, pub executions: u64, @@ -440,7 +441,9 @@ impl TraceJitEngine { root_ip: ip, stack_depth, }; - if let Some(trace_id) = self.compiled_trace_for_key(key) { + if let Some(trace_id) = + self.compiled_trace_for_key_matching_callables(key, entry_callable_prototypes) + { return Some(trace_id); } if !self.is_loop_header(program, ip) @@ -652,7 +655,9 @@ impl TraceJitEngine { root_ip: ip, stack_depth, }; - if let Some(trace_id) = self.compiled_trace_for_key(key) { + if let Some(trace_id) = + self.compiled_trace_for_key_matching_callables(key, entry_callable_prototypes) + { return Some(trace_id); } if !self.blocked_entries.is_empty() && self.blocked_entries.contains(&key) { @@ -700,6 +705,40 @@ impl TraceJitEngine { }) } + pub(crate) fn compiled_trace_for_entry_with_callables( + &self, + frame_key: u64, + ip: usize, + stack_depth: usize, + entry_callable_prototypes: Option<&[Option]>, + ) -> Option { + self.compiled_trace_for_key_matching_callables( + TraceEntryKey { + frame_key, + root_ip: ip, + stack_depth, + }, + entry_callable_prototypes, + ) + } + + pub(crate) fn trace_has_entry_callable_guards(&self, trace_id: usize) -> bool { + self.traces + .get(trace_id) + .is_some_and(|trace| !trace.entry_callable_guards.is_empty()) + } + + pub(crate) fn trace_exit_is_instruction_failure(&self, key: TraceExitKey) -> bool { + self.traces.get(key.parent_trace_id).is_some_and(|trace| { + trace + .ssa + .blocks + .iter() + .flat_map(|block| &block.insts) + .any(|inst| inst.failure_exit == Some(key.exit_id)) + }) + } + pub fn mark_trace_executed(&mut self, trace_id: usize) { if let Some(trace) = self.traces.get_mut(trace_id) { trace.executions = trace.executions.saturating_add(1); @@ -1059,6 +1098,32 @@ impl TraceJitEngine { ) } + fn compiled_trace_for_key_matching_callables( + &self, + key: TraceEntryKey, + entry_callable_prototypes: Option<&[Option]>, + ) -> Option { + self.compiled_by_ip.get(key.root_ip)?.iter().find_map( + |(frame_key, stack_depth, trace_id)| { + if *frame_key != key.frame_key || *stack_depth != key.stack_depth { + return None; + } + let trace = self.traces.get(*trace_id)?; + trace + .entry_callable_guards + .iter() + .all(|(local, prototype_id)| { + entry_callable_prototypes + .and_then(|prototypes| prototypes.get(usize::from(*local))) + .copied() + .flatten() + == Some(*prototype_id) + }) + .then_some(*trace_id) + }, + ) + } + fn insert_compiled_trace(&mut self, key: TraceEntryKey, trace_id: usize) { if self.compiled_by_ip.len() <= key.root_ip { self.compiled_by_ip.resize_with(key.root_ip + 1, Vec::new); @@ -1146,6 +1211,7 @@ fn build_jit_trace( start_line, has_call: recorded.has_call, has_yielding_call: recorded.has_yielding_call, + entry_callable_guards: recorded.entry_callable_guards, op_names: recorded.op_names, terminal: recorded.terminal, executions: 0, @@ -1259,8 +1325,8 @@ mod tests { SsaExitId, SsaMaterialization, SsaTerminator, SsaTrace, SsaTraceBuilder, SsaValueRepr, }; use crate::{ - BytecodeBuilder, CallableKind, CallablePrototype, CallableTarget, ScriptFunction, Value, - ValueType, + BytecodeBuilder, CallableKind, CallablePrototype, CallableTarget, FunctionRegion, + RootCallableBinding, ScriptFunction, Value, ValueType, }; #[test] @@ -1354,6 +1420,7 @@ mod tests { start_line: None, has_call: false, has_yielding_call, + entry_callable_guards: Vec::new(), op_names: Vec::new(), terminal: JitTraceTerminal::BranchExit, executions: 0, @@ -1709,6 +1776,7 @@ mod tests { start_line: None, has_call: false, has_yielding_call: false, + entry_callable_guards: Vec::new(), op_names: Vec::new(), terminal: JitTraceTerminal::BranchExit, executions: 0, @@ -1767,6 +1835,7 @@ mod tests { start_line: None, has_call: false, has_yielding_call: false, + entry_callable_guards: Vec::new(), op_names: Vec::new(), terminal: JitTraceTerminal::BranchExit, executions: 0, @@ -1789,6 +1858,123 @@ mod tests { assert!(engine.trace_exit_profiles.is_empty()); } + #[test] + fn cached_inline_trace_requires_matching_entry_callable_prototype() { + if !native_jit_supported() { + return; + } + let mut bc = BytecodeBuilder::new(); + let root_ip = bc.position(); + bc.ldloc(0); + bc.call_value(0); + bc.pop(); + bc.br(root_ip); + let first_entry = bc.position(); + bc.ldc(0); + bc.ret(); + let first_end = bc.position(); + let second_entry = bc.position(); + bc.ldc(1); + bc.ret(); + let second_end = bc.position(); + let program = Program::new(vec![Value::Int(1), Value::Int(2)], bc.finish()) + .with_local_count(1) + .with_callable_metadata( + vec![ + ScriptFunction { + entry_ip: first_entry, + end_ip: first_end, + }, + ScriptFunction { + entry_ip: second_entry, + end_ip: second_end, + }, + ], + vec![ + CallablePrototype { + kind: CallableKind::FunctionItem, + target: CallableTarget::ScriptFunction(0), + arity: 0, + frame_local_count: 1, + parameter_slots: Vec::new(), + capture_source_slots: Vec::new(), + capture_slots: Vec::new(), + capture_modes: Vec::new(), + self_slot: None, + schema: None, + }, + CallablePrototype { + kind: CallableKind::FunctionItem, + target: CallableTarget::ScriptFunction(1), + arity: 0, + frame_local_count: 1, + parameter_slots: Vec::new(), + capture_source_slots: Vec::new(), + capture_slots: Vec::new(), + capture_modes: Vec::new(), + self_slot: None, + schema: None, + }, + ], + vec![ + FunctionRegion { + start_ip: first_entry, + end_ip: first_end, + prototype_id: Some(0), + }, + FunctionRegion { + start_ip: second_entry, + end_ip: second_end, + prototype_id: Some(1), + }, + ], + vec![RootCallableBinding { + local_slot: 0, + prototype_id: 0, + }], + ); + let mut engine = TraceJitEngine::new(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 64, + }); + + let first = engine + .observe_hot_entry_with_local_types( + ROOT_FRAME_KEY, + root_ip as usize, + 0, + None, + Some(&[Some(0)]), + &program, + ) + .expect("first inline trace"); + assert!( + engine.traces[first] + .op_names + .iter() + .any(|name| name == "inline_call:0") + ); + + let replacement = engine + .observe_hot_entry_with_local_types( + ROOT_FRAME_KEY, + root_ip as usize, + 0, + None, + Some(&[Some(1)]), + &program, + ) + .expect("replacement trace"); + assert_ne!(replacement, first); + assert!( + !engine.traces[replacement] + .op_names + .iter() + .any(|name| name == "inline_call:0") + ); + } + #[test] fn scan_loop_headers_finds_backward_targets() { let mut bc = BytecodeBuilder::new(); diff --git a/tests/jit/jit_tests.rs b/tests/jit/jit_tests.rs index cca4c18c..cffa2f3a 100644 --- a/tests/jit/jit_tests.rs +++ b/tests/jit/jit_tests.rs @@ -6766,6 +6766,177 @@ fn trace_jit_preserves_inline_callable_argument_schema_checks() { ); } +#[test] +fn trace_jit_inline_instruction_failure_restores_callee_frame() { + if !native_jit_supported() { + return; + } + let source = r#" + fn get(values: [int], index: int) -> int { values[index] } + let values: [int] = [10, 20]; + let mut i = 0; + let mut sink = 0; + while i < 100 { + sink = get(values, i); + i = i + 1; + } + i + sink * 0; + "#; + let compiled = compile_source(source).expect("inline failure source should compile"); + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + + let error = vm + .run() + .expect_err("out-of-range inline array get must fail"); + assert!( + matches!(error, vm::VmError::HostError(ref message) if message == "array index 2 out of bounds"), + "unexpected error: {error:?}\n{}", + vm.dump_jit_info() + ); + assert!( + any_trace_op(&vm.jit_snapshot(), "inline_call:0"), + "{}", + vm.dump_jit_info() + ); +} + +#[test] +fn trace_jit_inline_unbox_failure_matches_interpreter_error() { + if !native_jit_supported() { + return; + } + let source = r#" + fn add_one(values: [int]) -> int { values[0] + 1 } + let values: [int] = [7]; + let mut i = 0; + let mut sink = 0; + while i < 100 { + sink = add_one(values); + i = i + 1; + } + i + sink * 0; + "#; + let compiled = compile_source(source).expect("inline unbox failure source should compile"); + let values_slot = compiled + .program + .debug + .as_ref() + .expect("debug metadata") + .locals + .iter() + .find(|local| local.name == "values") + .expect("values local") + .index; + let local_count = compiled.locals; + let prepare = |program: Program| { + let mut vm = Vm::new(program.with_local_count(local_count)); + vm.set_fuel_check_interval(1).expect("fuel interval"); + vm.set_fuel(1); + loop { + assert_eq!( + vm.run().expect("initialization step should run"), + VmStatus::Yielded + ); + if vm.locals().get(values_slot as usize) == Some(&Value::array(vec![Value::Int(7)])) { + break; + } + vm.recharge_fuel(1).expect("fuel recharge"); + } + vm.set_local(values_slot, Value::array(vec![Value::Bool(true)])) + .expect("dynamic array replacement should succeed"); + vm.clear_fuel(); + vm + }; + + let mut interpreted = prepare(compiled.program.clone()); + let expected = interpreted + .run() + .expect_err("interpreter must reject the invalid array element"); + let mut vm = prepare(compiled.program); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + + let actual = vm + .run() + .expect_err("inline unbox failure must return the interpreter error"); + assert_eq!(format!("{actual:?}"), format!("{expected:?}")); + assert!( + any_trace_op(&vm.jit_snapshot(), "inline_call:0"), + "{}", + vm.dump_jit_info() + ); +} + +#[test] +fn trace_jit_preserves_inline_callable_return_schema_checks() { + if !native_jit_supported() { + return; + } + let source = r#" + fn first(values: [int]) -> int { values[0] } + let values: [int] = [7]; + let mut i = 0; + let mut sink = 0; + while i < 100 { + sink = first(values); + i = i + 1; + } + i + sink * 0; + "#; + let compiled = compile_source(source).expect("return schema source should compile"); + let values_slot = compiled + .program + .debug + .as_ref() + .expect("debug metadata") + .locals + .iter() + .find(|local| local.name == "values") + .expect("values local") + .index; + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_fuel_check_interval(1).expect("fuel interval"); + vm.set_fuel(1); + loop { + assert_eq!( + vm.run().expect("initialization step should run"), + VmStatus::Yielded + ); + if vm.locals().get(values_slot as usize) == Some(&Value::array(vec![Value::Int(7)])) { + break; + } + vm.recharge_fuel(1).expect("fuel recharge"); + } + vm.set_local(values_slot, Value::array(vec![Value::Bool(true)])) + .expect("dynamic array replacement should succeed"); + vm.clear_fuel(); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + + let error = vm.run().expect_err("invalid inline result must fail"); + assert!( + matches!(error, vm::VmError::TypeMismatch("callable return schema")), + "unexpected error: {error:?}\n{}", + vm.dump_jit_info() + ); + assert!( + any_trace_op(&vm.jit_snapshot(), "inline_call:0"), + "{}", + vm.dump_jit_info() + ); +} + #[test] fn trace_jit_inlines_array_swap_leaf() { if !native_jit_supported() { From 2267fa2ee10baa35390af94859877175e6f7d231 Mon Sep 17 00:00:00 2001 From: fffonion Date: Mon, 20 Jul 2026 19:28:41 +0800 Subject: [PATCH 26/26] fix(jit): deopt inline helper failures --- src/builtins/runtime/core.rs | 2 +- src/vm/jit/native/lower.rs | 96 ++++++----- src/vm/jit/recorder.rs | 2 - src/vm/mod.rs | 21 ++- src/vm/native/bridge.rs | 306 +++++++++++++++++++++++++++++------ src/vm/native/mod.rs | 6 +- src/vm/tests.rs | 40 +++++ tests/jit/jit_tests.rs | 61 ++++++- 8 files changed, 432 insertions(+), 102 deletions(-) diff --git a/src/builtins/runtime/core.rs b/src/builtins/runtime/core.rs index 4abd9338..6a47e996 100644 --- a/src/builtins/runtime/core.rs +++ b/src/builtins/runtime/core.rs @@ -713,7 +713,7 @@ pub(crate) fn builtin_set_map_shared_impl( entries } -fn ensure_supported_map_key(key: &Value) -> VmResult<()> { +pub(crate) fn ensure_supported_map_key(key: &Value) -> VmResult<()> { if matches!(key, Value::Callable(_)) { return Err(VmError::HostError( "callable values are not supported as map keys".to_string(), diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index 1462dc9c..ca4c9794 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -5,38 +5,39 @@ use super::super::runtime::resume_linked_trace_entry_address; use super::{NativeCompileProfile, TraceLoweringKind}; use crate::vm::jit::deopt::exit_inputs; use crate::vm::jit::ir::{ - SsaBlockId, SsaBranchTarget, SsaExitId, SsaInstKind, SsaMaterialization, SsaTerminator, - SsaTrace, SsaValueId, SsaValueRepr, + SsaBlockId, SsaBranchTarget, SsaExitId, SsaInst, SsaInstKind, SsaMaterialization, + SsaTerminator, SsaTrace, SsaValueId, SsaValueRepr, }; use crate::vm::native::{ ExecutableBuffer, HeapIntrinsicAddrs, HeapIntrinsicRefs, NativeFrameState, NativeInterruptMode, NativeInterruptSettings, NativeStackLayout, STATUS_CONTINUE, STATUS_ERROR, STATUS_OUT_OF_FUEL, STATUS_TRACE_EXIT, alloc_buffer_signature, alloc_byte_buffer_entry_address, alloc_value_buffer_entry_address, array_push_entry_address, array_set_entry_address, - array_set_signature, box_heap_value_signature, checked_add_i32, clear_value_slot_entry_address, - clone_value_signature, clone_value_to_slot_entry_address, collection_get_signature, - collection_predicate_signature, copy_bytes_entry_address, copy_bytes_signature, - detect_native_stack_layout, enter_call_value_inherited_entry_address, - enter_call_value_inherited_signature, entry_signature, frame_state_entry_address, - frame_state_signature, free_buffer_signature, jump_with_status, - leave_frame_inherited_entry_address, leave_frame_inherited_signature, map_get_entry_address, - map_has_entry_address, map_iter_next_entry_address, map_iter_next_signature, - map_iter_take_key_entry_address, map_iter_take_signature, map_iter_take_value_entry_address, - map_set_entry_address, map_set_signature, non_yielding_host_call_entry_address, - non_yielding_host_call_signature, non_yielding_i64_host_call_entry_address, - non_yielding_i64_host_call_signature, non_yielding_scalar_host_call_entry_address, - non_yielding_scalar_host_call_signature, pack_shared_signature, regex_match_entry_address, - regex_match_signature, regex_replace_entry_address, regex_replace_signature, - replace_value_in_slot_entry_address, restore_active_sparse_exit_state_entry_address, - restore_virtual_frame_entry_address, restore_virtual_frame_signature, - shared_array_from_buffer_entry_address, shared_bytes_from_buffer_entry_address, - shared_string_from_buffer_entry_address, sparse_restore_exit_signature, - string_binary_transform_signature, string_contains_entry_address, string_contains_signature, - string_lower_ascii_entry_address, string_replace_literal_entry_address, - string_replace_signature, string_split_literal_entry_address, string_unary_transform_signature, - to_string_entry_address, type_of_entry_address, value_eq_entry_address, value_eq_signature, - value_len_entry_address, value_len_signature, value_slot_signature, - write_heap_value_to_slot_entry_address, zero_bytes_entry_address, + array_set_signature, box_heap_value_signature, checked_add_i32, + clear_bridge_error_entry_address, clear_value_slot_entry_address, clone_value_signature, + clone_value_to_slot_entry_address, collection_get_signature, collection_predicate_signature, + copy_bytes_entry_address, copy_bytes_signature, detect_native_stack_layout, + enter_call_value_inherited_entry_address, enter_call_value_inherited_signature, + entry_signature, frame_state_entry_address, frame_state_signature, free_buffer_signature, + jump_with_status, leave_frame_inherited_entry_address, leave_frame_inherited_signature, + map_get_entry_address, map_has_entry_address, map_iter_next_entry_address, + map_iter_next_signature, map_iter_take_key_entry_address, map_iter_take_signature, + map_iter_take_value_entry_address, map_set_entry_address, map_set_signature, + non_yielding_host_call_entry_address, non_yielding_host_call_signature, + non_yielding_i64_host_call_entry_address, non_yielding_i64_host_call_signature, + non_yielding_scalar_host_call_entry_address, non_yielding_scalar_host_call_signature, + pack_shared_signature, regex_match_entry_address, regex_match_signature, + regex_replace_entry_address, regex_replace_signature, replace_value_in_slot_entry_address, + restore_active_sparse_exit_state_entry_address, restore_virtual_frame_entry_address, + restore_virtual_frame_signature, shared_array_from_buffer_entry_address, + shared_bytes_from_buffer_entry_address, shared_string_from_buffer_entry_address, + sparse_restore_exit_signature, string_binary_transform_signature, + string_contains_entry_address, string_contains_signature, string_lower_ascii_entry_address, + string_replace_literal_entry_address, string_replace_signature, + string_split_literal_entry_address, string_unary_transform_signature, to_string_entry_address, + type_of_entry_address, value_eq_entry_address, value_eq_signature, value_len_entry_address, + value_len_signature, value_slot_signature, write_heap_value_to_slot_entry_address, + zero_bytes_entry_address, }; use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; use cranelift_codegen::ir::immediates::Ieee64; @@ -598,6 +599,7 @@ fn try_compile_ssa_trace( let array_push_sig = collection_get_signature(pointer_type, call_conv); let array_set_sig = array_set_signature(pointer_type, call_conv); let map_set_sig = map_set_signature(pointer_type, call_conv); + let clear_bridge_error_sig = Signature::new(call_conv); let sparse_restore_exit_sig = sparse_restore_exit_signature(pointer_type, call_conv); let restore_virtual_frame_sig = restore_virtual_frame_signature(pointer_type, call_conv); let frame_state_sig = frame_state_signature(pointer_type, call_conv); @@ -672,6 +674,7 @@ fn try_compile_ssa_trace( .import_signature(non_yielding_scalar_host_call_sig), non_yielding_i64_host_call_ref: b.import_signature(non_yielding_i64_host_call_sig), clear_value_slot_ref: b.import_signature(value_slot_sig), + clear_bridge_error_ref: b.import_signature(clear_bridge_error_sig), box_heap_value_ref: b.import_signature(box_heap_value_sig), map_has_ref: b.import_signature(map_has_sig), map_get_ref: b.import_signature(map_get_sig), @@ -697,6 +700,7 @@ fn try_compile_ssa_trace( non_yielding_scalar_host_call: non_yielding_scalar_host_call_entry_address(), non_yielding_i64_host_call: non_yielding_i64_host_call_entry_address(), clear_value_slot: clear_value_slot_entry_address(), + clear_bridge_error: clear_bridge_error_entry_address(), box_heap_value: write_heap_value_to_slot_entry_address(), map_has: map_has_entry_address(), map_get: map_get_entry_address(), @@ -1115,6 +1119,7 @@ struct SsaDeoptHelperRefs { non_yielding_scalar_host_call_ref: cranelift_codegen::ir::SigRef, non_yielding_i64_host_call_ref: cranelift_codegen::ir::SigRef, clear_value_slot_ref: cranelift_codegen::ir::SigRef, + clear_bridge_error_ref: cranelift_codegen::ir::SigRef, box_heap_value_ref: cranelift_codegen::ir::SigRef, map_has_ref: cranelift_codegen::ir::SigRef, map_get_ref: cranelift_codegen::ir::SigRef, @@ -1142,6 +1147,7 @@ struct SsaDeoptHelperAddrs { non_yielding_scalar_host_call: usize, non_yielding_i64_host_call: usize, clear_value_slot: usize, + clear_bridge_error: usize, box_heap_value: usize, map_has: usize, map_get: usize, @@ -2211,7 +2217,7 @@ fn lower_ssa_inst( b.ins().brif(ok, success, &[], fail, &[]); b.switch_to_block(fail); - jump_with_status(b, exit_block, status); + ssa_emit_helper_failure_exit(b, ctx, inst, values, status)?; b.switch_to_block(success); b.ins().stack_load(types::I64, out_slot, 0) @@ -2715,7 +2721,7 @@ fn lower_ssa_inst( b.ins().brif(error, failed, &[], matched, &[]); b.switch_to_block(failed); let status = b.ins().iconst(types::I32, STATUS_ERROR as i64); - jump_with_status(b, exit_block, status); + ssa_emit_helper_failure_exit(b, ctx, inst, values, status)?; b.switch_to_block(matched); b.ins().icmp_imm(IntCC::NotEqual, raw, 0) } @@ -2743,7 +2749,7 @@ fn lower_ssa_inst( b.ins().brif(error, failed, &[], replaced, &[]); b.switch_to_block(failed); let status = b.ins().iconst(types::I32, STATUS_ERROR as i64); - jump_with_status(b, exit_block, status); + ssa_emit_helper_failure_exit(b, ctx, inst, values, status)?; b.switch_to_block(replaced); let out = owned_value_temp_slot_addr( b, @@ -3344,7 +3350,7 @@ fn lower_ssa_inst( b.ins().brif(is_error, fail, &[], cont, &[]); b.switch_to_block(fail); - jump_with_status(b, exit_block, status); + ssa_emit_helper_failure_exit(b, ctx, inst, values, status)?; b.switch_to_block(cont); out @@ -3392,7 +3398,7 @@ fn lower_ssa_inst( b.ins().brif(is_error, fail, &[], cont, &[]); b.switch_to_block(fail); - jump_with_status(b, exit_block, status); + ssa_emit_helper_failure_exit(b, ctx, inst, values, status)?; b.switch_to_block(cont); out @@ -3464,7 +3470,7 @@ fn lower_ssa_inst( .brif(is_status_error, error_block, &[], miss_block, &[]); b.switch_to_block(error_block); - jump_with_status(b, exit_block, status); + ssa_emit_helper_failure_exit(b, ctx, inst, values, status)?; b.switch_to_block(miss_block); ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values)?; @@ -3509,7 +3515,7 @@ fn lower_ssa_inst( b.ins().brif(is_error, fail, &[], cont, &[]); b.switch_to_block(fail); - jump_with_status(b, exit_block, status); + ssa_emit_helper_failure_exit(b, ctx, inst, values, status)?; b.switch_to_block(cont); b.ins().icmp_imm(IntCC::NotEqual, status, 0) @@ -3529,7 +3535,7 @@ fn lower_ssa_inst( .icmp_imm(IntCC::Equal, status, i64::from(STATUS_ERROR)); b.ins().brif(is_error, fail, &[], cont, &[]); b.switch_to_block(fail); - jump_with_status(b, exit_block, status); + ssa_emit_helper_failure_exit(b, ctx, inst, values, status)?; b.switch_to_block(cont); b.ins().icmp_imm(IntCC::NotEqual, status, 0) } @@ -3563,7 +3569,7 @@ fn lower_ssa_inst( .icmp_imm(IntCC::Equal, status, i64::from(STATUS_ERROR)); b.ins().brif(is_error, fail, &[], cont, &[]); b.switch_to_block(fail); - jump_with_status(b, exit_block, status); + ssa_emit_helper_failure_exit(b, ctx, inst, values, status)?; b.switch_to_block(cont); out } @@ -3631,7 +3637,7 @@ fn lower_ssa_inst( b.ins().brif(is_error, fail, &[], cont, &[]); b.switch_to_block(fail); - jump_with_status(b, exit_block, status); + ssa_emit_helper_failure_exit(b, ctx, inst, values, status)?; b.switch_to_block(cont); out @@ -5703,6 +5709,24 @@ fn ssa_utf8_char_width( b.ins().select(lt_80, one, wide) } +fn ssa_emit_helper_failure_exit( + b: &mut FunctionBuilder, + ctx: SsaLowerCtx<'_>, + inst: &SsaInst, + values: &HashMap, + status: cranelift_codegen::ir::Value, +) -> VmResult<()> { + if inst.failure_exit.is_some() { + let helper_ptr = + iconst_ptr_from_addr(b, ctx.pointer_type, ctx.helper_addrs.clear_bridge_error)?; + b.ins() + .call_indirect(ctx.helper_refs.clear_bridge_error_ref, helper_ptr, &[]); + return ssa_emit_failure_exit(b, ctx, inst.failure_exit, inst.ip, values); + } + jump_with_status(b, ctx.exit_block, status); + Ok(()) +} + fn ssa_emit_failure_exit( b: &mut FunctionBuilder, ctx: SsaLowerCtx<'_>, diff --git a/src/vm/jit/recorder.rs b/src/vm/jit/recorder.rs index 9b8474a5..642823d0 100644 --- a/src/vm/jit/recorder.rs +++ b/src/vm/jit/recorder.rs @@ -1167,7 +1167,6 @@ pub(crate) fn record_trace_with_local_count( .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; current_block = guarded_block; frame = guarded_frame; - op_names.push("inline_return_schema_guard".to_string()); } let inlined = inline_frame.take().expect("inline frame checked above"); let result = frame.pop()?; @@ -1548,7 +1547,6 @@ pub(crate) fn record_trace_with_local_count( .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; current_block = guarded_block; frame = guarded_frame; - op_names.push("inline_callable_schema_guard".to_string()); } let operand_base = frame.stack.len() - usize::from(argc) - 1; diff --git a/src/vm/mod.rs b/src/vm/mod.rs index 1eaeae24..150da557 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -635,6 +635,17 @@ fn hash_type_schema(schema: &crate::compiler::TypeSchema, state: &mut impl Hashe } } +fn inline_compatible_callable_prototype(value: &Value) -> Option { + match value { + Value::Callable(callable) + if callable.kind == crate::CallableKind::FunctionItem && callable.env.is_none() => + { + Some(callable.prototype_id) + } + _ => None, + } +} + impl Vm { pub fn new(program: Program) -> Self { Self::new_shared_with_jit_config(Arc::new(program), jit::JitConfig::default()) @@ -1075,15 +1086,9 @@ impl Vm { for (offset, value) in self.locals[base..].iter().enumerate() { let prototype_id = if let Some(cell) = self.capture_cells.get(&(base + offset)) { let value = cell.lock().ok()?; - match &*value { - Value::Callable(callable) => Some(callable.prototype_id), - _ => None, - } + inline_compatible_callable_prototype(&value) } else { - match value { - Value::Callable(callable) => Some(callable.prototype_id), - _ => None, - } + inline_compatible_callable_prototype(value) }; prototypes.push(prototype_id); } diff --git a/src/vm/native/bridge.rs b/src/vm/native/bridge.rs index 9a118e10..ff692ba3 100644 --- a/src/vm/native/bridge.rs +++ b/src/vm/native/bridge.rs @@ -112,6 +112,10 @@ pub(crate) fn clear_bridge_error() { GENERIC_BRIDGE_ERROR.with(|slot| *slot.borrow_mut() = None); } +pub(crate) extern "C" fn pd_vm_native_clear_bridge_error() { + clear_bridge_error(); +} + pub(crate) fn take_bridge_error() -> Option { GENERIC_BRIDGE_ERROR.with(|slot| slot.borrow_mut().take()) } @@ -276,6 +280,10 @@ pub(crate) fn clear_value_slot_entry_address() -> usize { pd_vm_native_clear_value_slot as *const () as usize } +pub(crate) fn clear_bridge_error_entry_address() -> usize { + pd_vm_native_clear_bridge_error as *const () as usize +} + pub(crate) fn value_eq_entry_address() -> usize { pd_vm_native_value_eq as *const () as usize } @@ -1353,8 +1361,13 @@ pub(crate) extern "C" fn pd_vm_native_map_has(repr_ptr: *mut u8, key: *const Val return STATUS_ERROR; } + let key = unsafe { &*key }; + if let Err(err) = crate::builtins::runtime::core::ensure_supported_map_key(key) { + store_bridge_error(err); + return STATUS_ERROR; + } let entries = unsafe { arc_from_repr_ptr::(repr_ptr) }; - let present = entries.get(unsafe { &*key }).is_some(); + let present = entries.get(key).is_some(); std::mem::forget(entries); i32::from(present) } @@ -1371,8 +1384,13 @@ pub(crate) extern "C" fn pd_vm_native_map_get( return STATUS_ERROR; } + let key = unsafe { &*key }; + if let Err(err) = crate::builtins::runtime::core::ensure_supported_map_key(key) { + store_bridge_error(err); + return STATUS_ERROR; + } let entries = unsafe { arc_from_repr_ptr::(repr_ptr) }; - let Some(value) = entries.get(unsafe { &*key }) else { + let Some(value) = entries.get(key) else { std::mem::forget(entries); return 0; }; @@ -1482,26 +1500,50 @@ pub(crate) extern "C" fn pd_vm_native_array_set( return STATUS_ERROR; } - let array = unsafe { std::ptr::replace(array, Value::Null) }; - let value = unsafe { (&*value).clone() }; - let result = match array { + let validated_index = match unsafe { &*array } { Value::Array(values) => { - crate::builtins::runtime::core::builtin_set_array_shared_impl(values, index, value) - .map(Value::Array) + if index < 0 { + store_bridge_error(VmError::HostError( + "array index must be non-negative".to_string(), + )); + return STATUS_ERROR; + } + let Ok(index) = usize::try_from(index) else { + store_bridge_error(VmError::HostError("array index overflow".to_string())); + return STATUS_ERROR; + }; + if index > values.len() { + store_bridge_error(VmError::HostError(format!( + "array index {index} out of bounds" + ))); + return STATUS_ERROR; + } + index } - _ => Err(VmError::TypeMismatch("array")), - }; - match result { - Ok(result) => { - let previous = unsafe { std::ptr::replace(dst, result) }; - drop(previous); - STATUS_CONTINUE + _ => { + store_bridge_error(VmError::TypeMismatch("array")); + return STATUS_ERROR; } - Err(err) => { - store_bridge_error(err); - STATUS_ERROR + }; + + let value = unsafe { (&*value).clone() }; + let mut values = match unsafe { std::ptr::replace(array, Value::Null) } { + Value::Array(values) => values, + other => { + unsafe { std::ptr::write(array, other) }; + store_bridge_error(VmError::TypeMismatch("array")); + return STATUS_ERROR; } + }; + let values_mut = Arc::make_mut(&mut values); + if validated_index == values_mut.len() { + values_mut.push(value); + } else { + values_mut[validated_index] = value; } + let previous = unsafe { std::ptr::replace(dst, Value::Array(values)) }; + drop(previous); + STATUS_CONTINUE } pub(crate) extern "C" fn pd_vm_native_map_set( @@ -1517,26 +1559,30 @@ pub(crate) extern "C" fn pd_vm_native_map_set( return STATUS_ERROR; } - let map = unsafe { std::ptr::replace(map, Value::Null) }; + if !matches!(unsafe { &*map }, Value::Map(_)) { + store_bridge_error(VmError::TypeMismatch("map")); + return STATUS_ERROR; + } let key = unsafe { (&*key).clone() }; let value = unsafe { (&*value).clone() }; - let result = match map { - Value::Map(entries) => Ok(Value::Map( - crate::builtins::runtime::core::builtin_set_map_shared_impl(entries, key, value), - )), - _ => Err(VmError::TypeMismatch("map")), - }; - match result { - Ok(result) => { - let previous = unsafe { std::ptr::replace(dst, result) }; - drop(previous); - STATUS_CONTINUE - } - Err(err) => { - store_bridge_error(err); - STATUS_ERROR - } + if let Err(err) = crate::builtins::runtime::core::ensure_supported_map_key(&key) { + store_bridge_error(err); + return STATUS_ERROR; } + let entries = match unsafe { std::ptr::replace(map, Value::Null) } { + Value::Map(entries) => entries, + other => { + unsafe { std::ptr::write(map, other) }; + store_bridge_error(VmError::TypeMismatch("map")); + return STATUS_ERROR; + } + }; + let result = Value::Map(crate::builtins::runtime::core::builtin_set_map_shared_impl( + entries, key, value, + )); + let previous = unsafe { std::ptr::replace(dst, result) }; + drop(previous); + STATUS_CONTINUE } pub(crate) extern "C" fn pd_vm_native_array_push( @@ -1551,25 +1597,24 @@ pub(crate) extern "C" fn pd_vm_native_array_push( return STATUS_ERROR; } - let array = unsafe { std::ptr::replace(array, Value::Null) }; + if !matches!(unsafe { &*array }, Value::Array(_)) { + store_bridge_error(VmError::TypeMismatch("array")); + return STATUS_ERROR; + } let value = unsafe { (&*value).clone() }; - let result = match array { - Value::Array(values) => Ok(Value::Array( - crate::builtins::runtime::core::builtin_array_push_shared_impl(values, value), - )), - _ => Err(VmError::TypeMismatch("array")), - }; - match result { - Ok(result) => { - let previous = unsafe { std::ptr::replace(dst, result) }; - drop(previous); - STATUS_CONTINUE - } - Err(err) => { - store_bridge_error(err); - STATUS_ERROR + let values = match unsafe { std::ptr::replace(array, Value::Null) } { + Value::Array(values) => values, + other => { + unsafe { std::ptr::write(array, other) }; + store_bridge_error(VmError::TypeMismatch("array")); + return STATUS_ERROR; } - } + }; + let result = + Value::Array(crate::builtins::runtime::core::builtin_array_push_shared_impl(values, value)); + let previous = unsafe { std::ptr::replace(dst, result) }; + drop(previous); + STATUS_CONTINUE } pub(crate) extern "C" fn pd_vm_native_non_yielding_host_call( @@ -2412,6 +2457,45 @@ mod tests { assert!(!Arc::ptr_eq(&result, &alias)); } + #[test] + fn typed_array_set_failure_preserves_owned_input() { + clear_bridge_error(); + let shared = Arc::new(vec![Value::Int(10), Value::Int(20)]); + let mut input = Value::Array(shared.clone()); + let replacement = Value::Int(99); + let mut output = Value::Null; + + let status = pd_vm_native_array_set(&mut output, &mut input, 3, &replacement); + + assert_eq!(status, STATUS_ERROR); + assert_eq!(input, Value::Array(shared)); + assert_eq!(output, Value::Null); + assert!(matches!( + take_bridge_error(), + Some(VmError::HostError(message)) if message == "array index 3 out of bounds" + )); + } + + #[test] + fn typed_array_set_clones_aliasing_value_before_transfer() { + let shared = Arc::new(vec![Value::Int(10)]); + let mut input = Value::Array(shared.clone()); + let input_ptr = &mut input as *mut Value; + let mut output = Value::Null; + + let status = pd_vm_native_array_set(&mut output, input_ptr, 0, input_ptr); + + assert_eq!(status, STATUS_CONTINUE); + assert_eq!(input, Value::Null); + let Value::Array(result) = output else { + panic!("expected array result"); + }; + let Value::Array(nested) = &result[0] else { + panic!("expected aliased array value"); + }; + assert!(Arc::ptr_eq(nested, &shared)); + } + #[test] fn typed_array_push_detaches_shared_input() { let shared = Arc::new(vec![Value::Int(10)]); @@ -2432,6 +2516,26 @@ mod tests { assert!(!Arc::ptr_eq(&result, &alias)); } + #[test] + fn typed_array_push_clones_aliasing_value_before_transfer() { + let shared = Arc::new(vec![Value::Int(10)]); + let mut input = Value::Array(shared.clone()); + let input_ptr = &mut input as *mut Value; + let mut output = Value::Null; + + let status = pd_vm_native_array_push(&mut output, input_ptr, input_ptr); + + assert_eq!(status, STATUS_CONTINUE); + assert_eq!(input, Value::Null); + let Value::Array(result) = output else { + panic!("expected array result"); + }; + let Value::Array(nested) = &result[1] else { + panic!("expected aliased array value"); + }; + assert!(Arc::ptr_eq(nested, &shared)); + } + #[test] fn typed_map_set_detaches_shared_input() { let shared = Arc::new(VmMap::from(vec![(Value::Int(1), Value::Int(10))])); @@ -2455,6 +2559,106 @@ mod tests { assert!(!Arc::ptr_eq(&result, &alias)); } + #[test] + fn typed_map_lookup_rejects_callable_key() { + clear_bridge_error(); + let entries = Arc::new(VmMap::default()); + let repr_ptr = arc_repr_word(&entries) as *mut u8; + let callable = Value::Callable(Arc::new(crate::CallableValue { + prototype_id: 0, + kind: CallableKind::FunctionItem, + env: None, + })); + let mut output = Value::Null; + + assert_eq!(pd_vm_native_map_has(repr_ptr, &callable), STATUS_ERROR); + assert!(matches!( + take_bridge_error(), + Some(VmError::HostError(message)) + if message == "callable values are not supported as map keys" + )); + + assert_eq!( + pd_vm_native_map_get(&mut output, repr_ptr, &callable), + STATUS_ERROR + ); + assert_eq!(output, Value::Null); + assert!(matches!( + take_bridge_error(), + Some(VmError::HostError(message)) + if message == "callable values are not supported as map keys" + )); + assert_eq!(Arc::strong_count(&entries), 1); + } + + #[test] + fn typed_map_set_rejects_callable_key_without_consuming_input() { + clear_bridge_error(); + let shared = Arc::new(VmMap::default()); + let mut input = Value::Map(shared.clone()); + let callable = Value::Callable(Arc::new(crate::CallableValue { + prototype_id: 0, + kind: CallableKind::FunctionItem, + env: None, + })); + let value = Value::Int(1); + let mut output = Value::Null; + + let status = pd_vm_native_map_set(&mut output, &mut input, &callable, &value); + + assert_eq!(status, STATUS_ERROR); + assert_eq!(input, Value::Map(shared)); + assert_eq!(output, Value::Null); + assert!(matches!( + take_bridge_error(), + Some(VmError::HostError(message)) + if message == "callable values are not supported as map keys" + )); + } + + #[test] + fn typed_map_set_clones_aliasing_key_before_transfer() { + let shared = Arc::new(VmMap::default()); + let mut input = Value::Map(shared.clone()); + let input_ptr = &mut input as *mut Value; + let value = Value::Int(7); + let mut output = Value::Null; + + let status = pd_vm_native_map_set(&mut output, input_ptr, input_ptr, &value); + + assert_eq!(status, STATUS_CONTINUE); + assert_eq!(input, Value::Null); + let Value::Map(result) = output else { + panic!("expected map result"); + }; + let Some((Value::Map(key), stored)) = result.iter().next() else { + panic!("expected aliased map key"); + }; + assert!(Arc::ptr_eq(key, &shared)); + assert_eq!(stored, &value); + } + + #[test] + fn typed_map_set_clones_aliasing_value_before_transfer() { + let shared = Arc::new(VmMap::default()); + let mut input = Value::Map(shared.clone()); + let input_ptr = &mut input as *mut Value; + let key = Value::Int(1); + let mut output = Value::Null; + + let status = pd_vm_native_map_set(&mut output, input_ptr, &key, input_ptr); + + assert_eq!(status, STATUS_CONTINUE); + assert_eq!(input, Value::Null); + let Value::Map(result) = output else { + panic!("expected map result"); + }; + let Some(Value::Map(nested)) = result.get(&key) else { + panic!("expected aliased map value"); + }; + assert!(Arc::ptr_eq(nested, &shared)); + } + #[test] fn replace_value_slot_clones_before_dropping_an_aliasing_destination() { let shared = Arc::new(vec![Value::Int(1)]); diff --git a/src/vm/native/mod.rs b/src/vm/native/mod.rs index 9bf3ca3b..bbc415e7 100644 --- a/src/vm/native/mod.rs +++ b/src/vm/native/mod.rs @@ -12,9 +12,9 @@ pub(crate) use bridge::{ active_local_base_entry_address, active_stack_base_entry_address, alloc_byte_buffer_entry_address, alloc_value_buffer_entry_address, aot_call_boundary_interrupt_entry_address, array_push_entry_address, array_set_entry_address, - clear_bridge_error, clear_value_slot_entry_address, clone_value_to_slot_entry_address, - collection_set_entry_address, copy_bytes_entry_address, decode_jit_trace_exit_status, - encode_jit_trace_exit_status, enter_call_value_entry_address, + clear_bridge_error, clear_bridge_error_entry_address, clear_value_slot_entry_address, + clone_value_to_slot_entry_address, collection_set_entry_address, copy_bytes_entry_address, + decode_jit_trace_exit_status, encode_jit_trace_exit_status, enter_call_value_entry_address, enter_call_value_inherited_entry_address, frame_state_entry_address, helper_entry_address, helper_entry_offset, init_null_value_slot_entry_address, interrupt_helper_entry_address, interrupt_helper_entry_offset, leave_frame_entry_address, leave_frame_inherited_entry_address, diff --git a/src/vm/tests.rs b/src/vm/tests.rs index 817da2af..acf11f32 100644 --- a/src/vm/tests.rs +++ b/src/vm/tests.rs @@ -54,6 +54,46 @@ fn shared_capture_cell_rejects_callable_ownership_cycle() { assert_eq!(vm.locals()[0], Value::Null); } +#[test] +fn inline_callable_identity_requires_capture_free_function_item_state() { + let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8]).with_local_count(1)); + let malformed_environment = Arc::new(crate::CallableEnvironment { + cells: Mutex::new(vec![Arc::new(Mutex::new(Value::Int(7)))]), + }); + vm.set_local( + 0, + Value::Callable(Arc::new(crate::CallableValue { + prototype_id: 42, + kind: crate::CallableKind::FunctionItem, + env: Some(malformed_environment), + })), + ) + .expect("install malformed callable"); + assert_eq!(vm.active_local_callable_prototypes(), Some(vec![None])); + + vm.set_local( + 0, + Value::Callable(Arc::new(crate::CallableValue { + prototype_id: 42, + kind: crate::CallableKind::Closure, + env: None, + })), + ) + .expect("install closure-shaped callable"); + assert_eq!(vm.active_local_callable_prototypes(), Some(vec![None])); + + vm.set_local( + 0, + Value::Callable(Arc::new(crate::CallableValue { + prototype_id: 42, + kind: crate::CallableKind::FunctionItem, + env: None, + })), + ) + .expect("install inline-compatible callable"); + assert_eq!(vm.active_local_callable_prototypes(), Some(vec![Some(42)])); +} + #[test] fn callable_operand_type_hint_roundtrips() { let packed = pack_operand_types(ValueType::Callable, ValueType::Callable); diff --git a/tests/jit/jit_tests.rs b/tests/jit/jit_tests.rs index cffa2f3a..af1f80e9 100644 --- a/tests/jit/jit_tests.rs +++ b/tests/jit/jit_tests.rs @@ -6930,11 +6930,22 @@ fn trace_jit_preserves_inline_callable_return_schema_checks() { "unexpected error: {error:?}\n{}", vm.dump_jit_info() ); + let snapshot = vm.jit_snapshot(); assert!( - any_trace_op(&vm.jit_snapshot(), "inline_call:0"), + any_trace_op(&snapshot, "inline_call:0"), "{}", vm.dump_jit_info() ); + assert!( + !any_trace_op(&snapshot, "inline_return_schema_guard"), + "synthetic return guards must not advance bytecode interruption accounting: {}", + vm.dump_jit_info() + ); + assert!( + !any_trace_op(&snapshot, "inline_callable_schema_guard"), + "synthetic argument guards must not advance bytecode interruption accounting: {}", + vm.dump_jit_info() + ); } #[test] @@ -6988,6 +6999,54 @@ fn trace_jit_inlines_array_swap_leaf() { assert!(!any_trace_op(&snapshot, "call_value")); } +#[test] +fn trace_jit_inline_array_set_failure_restores_callee_frame() { + if !native_jit_supported() { + return; + } + let source = r#" + fn write(values: [int], index: int) -> int { + values[index] = 99; + values[0] + } + let values: [int] = [10, 20]; + let mut i = 0; + let mut sink = 0; + while i < 100 { + sink = write(values, i); + i = i + 1; + } + i + sink * 0; + "#; + let compiled = compile_source(source).expect("inline array-set failure source should compile"); + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + + let error = vm + .run() + .expect_err("out-of-range inline array set must fail"); + assert!( + matches!(error, vm::VmError::HostError(ref message) if message == "array index 3 out of bounds"), + "unexpected error: {error:?}\n{}", + vm.dump_jit_info() + ); + let snapshot = vm.jit_snapshot(); + assert!( + any_trace_op(&snapshot, "inline_call:0"), + "{}", + vm.dump_jit_info() + ); + assert!( + any_trace_op(&snapshot, "array_set"), + "{}", + vm.dump_jit_info() + ); +} + #[test] fn trace_jit_inline_guard_exit_restores_callee() { if !native_jit_supported() {