From dadbf6fa64d3ce55a4387f9d06b516cb03b5dc18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 11 Aug 2026 11:27:58 +0200 Subject: [PATCH] perf(codegen): stamp a pointer-free shape's typed layout into the allocation header Four allocation benchmarks sat within a 6% band at 2.50-2.65x node, which is the signature of one shared per-allocation cost rather than four problems. Profiling the band (200M-allocation variants, two samples each, agreeing within 1.5pp) found it: `js_gc_declare_typed_shape_layout` was 30% of `churn_alloc` and `push_cls`, and it spent that re-deriving per OBJECT a fact that is a property of the SHAPE. #7510's memo had already reduced the map round-trip to a direct-mapped probe; what remained was the probe itself, a type-table lookup, a field-count compare, and the cross-crate call. For a shape whose pointer mask is statically EMPTY the answer is a constant: `GC_LAYOUT_POINTER_FREE | GC_OBJ_TYPED_LAYOUT_INTACT`. The inline-bump `new` path already writes a packed `GcHeader` constant that carries the state half, so the intact bit is folded into the same store and the call disappears. What survives is the one half that depends on the recycled ADDRESS rather than the shape - clearing a previous tenant's per-object record - now a one-argument `js_gc_forget_object_layout` behind a `PERRY_PER_OBJECT_LAYOUTS_ANY` test whose `0` state proves every thread's per-object tables empty. Two smaller levers in the same band: * `js_ctor_return_override` was called per construction to answer a question that is `undefined` for every constructor without an explicit `return` - 8% of `churn_alloc`. `JSValue::is_undefined` is `bits == TAG_UNDEFINED`, so one 64-bit compare decides it inline and the runtime call stays on the cold arm, where derived-constructor TypeErrors and object returns still need it. * A `new` in a function the hot-loop-callee pre-pass admitted is a `new` in a loop one frame out, so it takes the inline bump too. `cycles.ts`'s `makeCycle` is the shape: 5 statements, hence `alwaysinline` and hence never `inlinehint`, so the existing gate read the one flag it could not have. Measured on the quiet M1 mini, best-of-5, outputs byte-identical to node with exit 0 verified for all 27 programs: | bench | before | after | |---|--:|--:| | churn | 0.4217 | 0.2900 | | churn_alloc | 0.3720 | 0.2409 | | push_cls | 0.3665 | 0.2368 | `churn_alloc` goes 18.6 -> 12.0 ns per allocation (node is 7.1 on the same shape). `gc-handoff/bench/alloc_declare_{pf,ptr}.ts` isolate it: identical programs differing only in whether the second field's declared type makes the pointer mask non-empty. Soundness: the collector's view is bit-identical. `heap_payload_slot_selection` skips a `GC_LAYOUT_POINTER_FREE` payload without consulting any map, and the pre-existing path also reached `POINTER_FREE` for an empty pointer mask. A later pointer store still downgrades - with no descriptor to classify against, `layout_note_slot` falls through to its generic pointer-mask branch, which mints a per-object mask and flips the state to `SIDE_MASK`, needing no descriptor at all. A pointer-BEARING shape keeps the full runtime declare, because its `SIDE_MASK` state means the tracer reads a mask and that call is what installs it. --- .../7834-alloc-band-typed-shape-bake.md | 86 ++++ crates/perry-codegen/src/codegen/function.rs | 6 + crates/perry-codegen/src/function.rs | 13 + crates/perry-codegen/src/gc_call_effects.rs | 3 + crates/perry-codegen/src/lower_call/mod.rs | 2 + crates/perry-codegen/src/lower_call/new.rs | 28 +- .../perry-codegen/src/lower_call/new_alloc.rs | 102 ++++- .../src/lower_call/new_helpers.rs | 60 +++ .../src/lower_call/typed_shape_bake_tests.rs | 393 ++++++++++++++++++ .../src/lower_call/typed_shape_init.rs | 103 ++++- crates/perry-codegen/src/root_reload.rs | 1 + .../perry-codegen/src/runtime_decls/arrays.rs | 4 + .../src/runtime_decls/objects.rs | 5 + crates/perry-runtime/src/gc/layout_tables.rs | 85 +++- 14 files changed, 856 insertions(+), 35 deletions(-) create mode 100644 changelog.d/7834-alloc-band-typed-shape-bake.md create mode 100644 crates/perry-codegen/src/lower_call/typed_shape_bake_tests.rs diff --git a/changelog.d/7834-alloc-band-typed-shape-bake.md b/changelog.d/7834-alloc-band-typed-shape-bake.md new file mode 100644 index 0000000000..76b601c8d0 --- /dev/null +++ b/changelog.d/7834-alloc-band-typed-shape-bake.md @@ -0,0 +1,86 @@ +### Allocation: the typed-shape layout is a property of the shape, so it is now a constant in the header + +Four allocation benchmarks sat inside a **6% band** at 2.50–2.65× node — object literals +(`churn`, `churn_alloc`), class instances (`push_cls`), cyclic graphs (`cycles`). Four +structurally different shapes do not land in a 6% band by coincidence; one shared +per-allocation cost does. + +Symbolicated profiles of the 200M-allocation variants (two samples per program, agreeing +within 1.5 pp) found it. `js_gc_declare_typed_shape_layout` was **30% of `churn_alloc` and +`push_cls`**, and it spent that re-deriving *per object* a fact that is a property of the +*shape*. #7510's memo had already collapsed the map round-trip to a direct-mapped probe; +what remained was the probe, a type-table lookup, a field-count compare, and the cross-crate +call itself. GC pause time, by contrast, is **3.5%** — the cost is that Perry performed nine +out-of-line operations per allocation where V8 performs a bump-pointer and a write barrier. + +For a shape whose pointer mask is **statically empty**, the canonical layout is the constant +`GC_LAYOUT_POINTER_FREE | GC_OBJ_TYPED_LAYOUT_INTACT`. The inline-bump `new` path already +emits a packed `GcHeader` constant carrying the state half, so the intact bit is folded into +that same store and the call is not emitted. What survives is the one half that depends on +the recycled **address** rather than the shape — clearing whatever per-object record a +previous tenant left — as a one-argument `js_gc_forget_object_layout` behind an inline +`PERRY_PER_OBJECT_LAYOUTS_ANY` test. That global is a process-wide mirror of the per-thread +emptiness flag, maintained by an armed-thread count; its `0` state proves every thread's +per-object tables empty, and it is now also the first test inside `layout_forget_object` +itself, replacing a Darwin `_tlv_get_addr` call with a static load on the disarmed path for +every caller including object death. + +Two smaller levers in the same band: + +- **`js_ctor_return_override`** was called on every construction to answer a question that is + `undefined` for every constructor without an explicit `return` — 8% of `churn_alloc`, where + the synthesized object-literal constructor's only `ret` is the `TAG_UNDEFINED` constant. + `JSValue::is_undefined` is `bits == TAG_UNDEFINED`, so one 64-bit compare decides it inline. + The runtime call stays on the cold arm, where derived-constructor `TypeError`s, object + returns, arguments objects and arrays still need it. +- **A `new` in a hot-loop callee is a `new` in a loop**, one frame out, so it now takes the + inline bump as well. `cycles.ts`'s `makeCycle` is the shape that needed this: 5 statements, + therefore `alwaysinline`, therefore *never* `inlinehint` — so the site gate was reading the + one flag that could not be set for the hottest function in the program. The signal is + `collect_hot_loop_callees` directly (≥1 in-loop call site AND ≤4 module-wide call sites), + which is the same anti-bloat bound the loop arm already accepts. + +Measured on the quiet M1 mini, best-of-5, with exit code 0 and byte-identical output verified +for all 27 corpus programs before timing: + +| bench | before | after | +|---|--:|--:| +| `churn` | 0.4217 | **0.2900** (−31%) | +| `churn_alloc` | 0.3720 | **0.2409** (−35%) | +| `push_cls` | 0.3665 | **0.2368** (−35%) | + +`churn_alloc` goes **18.6 → 12.0 ns per allocation**; node is 7.1 ns on the same shape. +Nothing else in the 19-benchmark corpus moves outside noise. + +`gc-handoff/bench/alloc_declare_pf.ts` and `alloc_declare_ptr.ts` isolate the cause with a +control: the same program, same allocation count, same runtime stores, differing only in +whether the second field's *declared type* makes the pointer mask non-empty. Before, both +arms pay the declare and their times match (0.9044 / 0.8647); after, only the control does +(0.5802 / 0.7845). Both move by the shared return-override lever (1.6 ns/alloc); the extra +4.1 ns/alloc on the pointer-free arm is the layout declare itself. + +**Soundness.** The collector's view is bit-identical: `heap_payload_slot_selection` skips a +`GC_LAYOUT_POINTER_FREE` payload without consulting any map, and the pre-existing path also +reached `POINTER_FREE` for an empty pointer mask. A later pointer store still downgrades — +with no descriptor to classify against, `layout_note_slot` falls through to its generic +pointer-mask branch, which mints a per-object mask and flips the state to `SIDE_MASK`, a +branch that needs no descriptor at all. A pointer-**bearing** shape keeps the full runtime +declare and must: `SIDE_MASK` means the tracer reads a mask, and that call is what installs +the shared `SHAPE_LAYOUTS` descriptor the mask lives in. (Installing it once at module init +is not a substitute: the `keys_array` lives in the longlived arena and can be relocated by +old-page defrag, and today's design survives that only by re-installing on the next +construction.) + +One hypothesis was refuted rather than assumed: "INTACT set with no descriptor installed" is +*not* a wrong-answer hazard. A JS number's NaN box **is** its double bits, so the raw-f64 +claim never changes the storage; every site that writes a slot raw re-proves the value finite +inline; and every read that could treat those bits as a machine double is itself +value-guarded. Verified directly — `(p as any).a = true` followed by `p.a + 1` through a +warmed monomorphic accessor prints node's `2` on both arms, byte-identical. + +Tests: `crates/perry-codegen/src/lower_call/typed_shape_bake_tests.rs` — three IR-census +ratchets of the "assert the subject was live" kind, one per direction (pointer-free bakes, +pointer-bearing keeps the declare, `undefined` completion takes the inline arm). They earned +that description: the first version **failed**, because codegen had scalar-replaced the +probe's `new` and there was no allocation left to assert about. The escape is now explicit +and commented, for the next person who writes a probe on this path. diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index af8230201e..0b35a863a9 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -426,6 +426,12 @@ pub(super) fn compile_function( // functions (the hint would be redundant) and async/generator forms. // Try-containing functions are ordinary inline candidates since #7302 // (invoke-EH removed the setjmp-era noinline requirement). + // #7834: record the raw admission, before the `inline_hint` window narrows + // it. `lower_call/new_alloc.rs` uses this to decide the inline-bump + // allocation, which wants "is this code hot" and not "may LLVM's inline + // threshold move" — an `alwaysinline` callee is excluded from the latter + // and is the hottest possible case for the former. + lf.hot_loop_callee = cross_module.hot_loop_callees.contains(&f.id); if !lf.force_inline && inline_hot_small_enabled() && (INLINE_HOT_SMALL_MIN..=inline_hot_small_size_cap()).contains(&f.body.len()) diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index b21c145757..689bc77050 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -36,6 +36,18 @@ pub struct LlFunction { /// inline-hot-small heuristic in `codegen/function.rs`. `alwaysinline` /// already implies the hint, so the two are never emitted together. pub inline_hint: bool, + /// #7834: `collectors::collect_hot_loop_callees` admitted this function — + /// it has at least one direct call site inside a LOOP and at most + /// `inline_hot_small_max_call_sites` call sites in the whole module. + /// + /// Not the same question as [`Self::inline_hint`], which is that set + /// INTERSECTED with a body-length window and with "not already + /// `alwaysinline`". A ≤8-statement function is `alwaysinline` and therefore + /// never hinted, yet it is exactly the shape whose body ends up executing + /// once per loop iteration — `cycles.ts`'s `makeCycle`. Sites that want + /// "is this code hot?" rather than "should LLVM's threshold move?" read + /// this. + pub hot_loop_callee: bool, /// Invoke-EH (#7302): this function contains landing pads (Itanium) or /// funclet pads (SEH), so its `define` line must carry /// `personality ptr @` — `perry_eh_personality` on Mach-O/ELF, @@ -215,6 +227,7 @@ impl LlFunction { linkage: String::new(), force_inline: false, inline_hint: false, + hot_loop_callee: false, personality: None, blocks: Vec::new(), block_counter: 0, diff --git a/crates/perry-codegen/src/gc_call_effects.rs b/crates/perry-codegen/src/gc_call_effects.rs index fb0742e6e4..ced28ad696 100644 --- a/crates/perry-codegen/src/gc_call_effects.rs +++ b/crates/perry-codegen/src/gc_call_effects.rs @@ -83,6 +83,9 @@ pub(crate) fn classify_direct_callee(name: &str) -> GcCallEffect { | "js_gc_note_slot_layout_aware" | "js_gc_init_typed_shape_layout" | "js_gc_declare_typed_shape_layout" + // #7834: `layout_forget_object` behind a null check — two thread-local + // side-table removals, no allocation and no re-entry. + | "js_gc_forget_object_layout" // `typed_feedback.rs`: counters/registries only. This intentionally // does not include feedback wrappers that perform the actual object // get/set operation. diff --git a/crates/perry-codegen/src/lower_call/mod.rs b/crates/perry-codegen/src/lower_call/mod.rs index 4c4b967f5a..f6b90a862e 100644 --- a/crates/perry-codegen/src/lower_call/mod.rs +++ b/crates/perry-codegen/src/lower_call/mod.rs @@ -71,6 +71,8 @@ mod scalar_method; /// module header for why the default build cannot fault on them. #[cfg(test)] mod timer_rooting_tests; +#[cfg(test)] +mod typed_shape_bake_tests; /// #7510: which of the two typed-shape layout entry points a `new` site emits, /// and where. Split out of `new.rs` to keep it under the 2000-line cap. mod typed_shape_init; diff --git a/crates/perry-codegen/src/lower_call/new.rs b/crates/perry-codegen/src/lower_call/new.rs index ce25c7dfb2..3ff62aa12c 100644 --- a/crates/perry-codegen/src/lower_call/new.rs +++ b/crates/perry-codegen/src/lower_call/new.rs @@ -500,7 +500,8 @@ fn lower_new_impl_inner<'a>( // #7615 slice 8: the field-count computation and the three-arm instance // allocation moved verbatim to `new_alloc.rs` (see its header for why). - let obj_handle = super::new_alloc::emit_instance_alloc(ctx, class_name, class); + let alloc = super::new_alloc::emit_instance_alloc(ctx, class_name, class); + let obj_handle = alloc.handle; // #7154: root the instance for the duration of the constructor body. // // Until now the instance existed ONLY as an SSA register while that body @@ -532,7 +533,7 @@ fn lower_new_impl_inner<'a>( // // Before the instance root's push, so the handle this names is the one the // allocator returned: nothing between here and there can collect. - emit_typed_shape_layout_declare(ctx, class_name, &obj_handle); + emit_typed_shape_layout_declare(ctx, class_name, &obj_handle, alloc.typed_layout_baked); let instance = { let protected = construction_runs_user_code(ctx, class_name); Instance { @@ -669,16 +670,8 @@ fn lower_new_impl_inner<'a>( || class.extends_name.is_some() || class.native_extends.is_some() || class.extends_expr.is_some(); - let is_derived_lit = if is_derived { "1" } else { "0" }; - let final_box = ctx.block().call( - DOUBLE, - "js_ctor_return_override", - &[ - (DOUBLE, &obj_box), - (DOUBLE, &ctor_ret), - (crate::types::I32, is_derived_lit), - ], - ); + let final_box = + super::new_helpers::emit_ctor_return_override(ctx, &obj_box, &ctor_ret, is_derived); return Ok(final_box); } if let Some(save) = &saved_new_target { @@ -1560,16 +1553,7 @@ fn lower_new_impl_inner<'a>( } ctx.current_block = after_idx; let raw = ctx.block().load(DOUBLE, &ret.result_slot); - let is_derived = if ret.is_derived { "1" } else { "0" }; - ctx.block().call( - DOUBLE, - "js_ctor_return_override", - &[ - (DOUBLE, &obj_box), - (DOUBLE, &raw), - (crate::types::I32, is_derived), - ], - ) + super::new_helpers::emit_ctor_return_override(ctx, &obj_box, &raw, ret.is_derived) } else { obj_box }; diff --git a/crates/perry-codegen/src/lower_call/new_alloc.rs b/crates/perry-codegen/src/lower_call/new_alloc.rs index 2ba0e0057d..b744a6040c 100644 --- a/crates/perry-codegen/src/lower_call/new_alloc.rs +++ b/crates/perry-codegen/src/lower_call/new_alloc.rs @@ -42,9 +42,42 @@ use crate::types::{I32, I64, I8, PTR}; /// scan-outward-past-switch-frames logic uses. A `new` inside a bare `switch` /// is therefore correctly treated as not-in-a-loop. fn new_site_is_in_loop(ctx: &FnCtx<'_>) -> bool { - ctx.loop_targets + if ctx + .loop_targets .iter() .any(|(continue_label, _, _)| !continue_label.is_empty()) + { + return true; + } + // #7834: a `new` in a function the hot-loop-callee pre-pass admitted is a + // `new` in a loop, one frame out. + // + // The gate below this comment is about SPEED-vs-SIZE, and + // `collect_hot_loop_callees` answers exactly the question the loop test + // does — is this site hot enough to be worth ~268 bytes — with the + // anti-bloat backstop already attached: it admits only a function that + // (a) has a direct call site inside a loop and (b) has at most + // `inline_hot_small_max_call_sites` (4) direct call sites in the whole + // module. So the added code is bounded by 4 × (news in the function), + // which is the same order the loop arm already accepts. + // + // Deliberately NOT `func.inline_hint`: that is this set intersected with a + // 9..=20-statement window and with "not already `alwaysinline`", and the + // functions this needs most fall out of BOTH. `makeCycle` is 5 statements, + // so it is `alwaysinline` and never hinted — while being the single + // hottest function in the program. + // + // `cycles.ts` is the shape that needs it: `makeCycle` is called 10M times + // from `main`'s loop and allocates two `Cell`s, but its own body has no + // loop, so both allocations took the outlined + // `js_object_alloc_class_inline_keys` — 22% of the program's samples, plus + // a further 5% in `arena_alloc`'s inline-state sync, for work the inline + // bump does in eight stores. + // + // Reading `func.hot_loop_callee` here is well-ordered: `codegen/function.rs` + // sets it from `cross_module.hot_loop_callees` before the entry block is + // created and before any expression is lowered. + ctx.func.hot_loop_callee } /// Emit the instance allocation for `new (...)` and return the raw @@ -62,7 +95,37 @@ fn new_site_is_in_loop(ctx: &FnCtx<'_>) -> bool { /// emission, which is the `RootedGroup::adopt_emitted` push that roots it for /// the constructor body; nothing between the allocator call and that push can /// collect. -pub(super) fn emit_instance_alloc(ctx: &mut FnCtx<'_>, class_name: &str, class: &Class) -> String { +/// What [`emit_instance_alloc`] produced: the instance's user pointer, plus +/// whether the allocation already stamped this class's canonical typed-shape +/// layout into the object's `GcHeader` constant (#7834). +pub(super) struct InstanceAlloc { + pub(super) handle: String, + /// `true` ⟹ the header already reads `GC_LAYOUT_POINTER_FREE | + /// GC_OBJ_TYPED_LAYOUT_INTACT`, so the construction site owes the runtime + /// only the address-dependent half of `js_gc_declare_typed_shape_layout` + /// (clearing a recycled address's stale per-object record). + pub(super) typed_layout_baked: bool, +} + +pub(super) fn emit_instance_alloc( + ctx: &mut FnCtx<'_>, + class_name: &str, + class: &Class, +) -> InstanceAlloc { + let mut typed_layout_baked = false; + let handle = emit_instance_alloc_inner(ctx, class_name, class, &mut typed_layout_baked); + InstanceAlloc { + handle, + typed_layout_baked, + } +} + +fn emit_instance_alloc_inner( + ctx: &mut FnCtx<'_>, + class_name: &str, + class: &Class, + typed_layout_baked: &mut bool, +) -> String { // Compute total field count including inherited parent fields. // The runtime allocates at least 8 inline slots regardless, so this // mostly matters for shapes >8 fields. @@ -296,8 +359,41 @@ pub(super) fn emit_instance_alloc(ctx: &mut FnCtx<'_>, class_name: &str, class: // `js_gc_note_slot_layout` so the GC sees real pointer-bearing // slots regardless of this initial tag. const GC_LAYOUT_POINTER_FREE: u64 = 0x4000; + /// `GC_OBJ_TYPED_LAYOUT_INTACT` — the bit + /// `class_field_inline_guard` requires before it will read or write + /// a raw-f64 slot directly. Runtime-side name: + /// `gc::layout::GC_OBJ_TYPED_LAYOUT_INTACT`. + const GC_OBJ_TYPED_LAYOUT_INTACT: u64 = 0x1000; const OBJECT_TYPE_REGULAR: u64 = 1; + // #7834: when this class's canonical layout is declarable at + // allocation AND its pointer mask is statically empty, the state + // this header already carries (`GC_LAYOUT_POINTER_FREE`) is the + // FINAL one, and the only thing `js_gc_declare_typed_shape_layout` + // would add per instance is the intact bit. Stamping it into the + // same constant store removes the call: on `churn_alloc` / + // `push_cls` that call was ~30% of the program, almost all of it + // re-deriving per object a fact that is a property of the SHAPE + // (see `gc::shape_install`'s module docs — the memo already reduced + // the map round-trip to a direct-mapped probe, and what is left is + // that probe, the type-table lookup, and the call itself). + // + // Requires `field_count == slot_count`: that mismatch is the one + // case `init_typed_shape_layout` answers by DOWNGRADING + // (`layout_set_typed_unknown`), and a constant cannot express "it + // depends". Computed here, before `ctx.block()` takes its mutable + // borrow. + *typed_layout_baked = super::typed_shape_init::layout_pointer_free_at_allocation( + ctx, + class_name, + field_count, + ); + let typed_intact_bits = if *typed_layout_baked { + GC_OBJ_TYPED_LAYOUT_INTACT + } else { + 0 + }; + let alloc_field_count = std::cmp::max(field_count as u64, MIN_FIELD_SLOTS); let payload_size = object_header_size + alloc_field_count * FIELD_SLOT_SIZE; // Round the whole allocation up to FIELD_SLOT_SIZE (8). The inline @@ -410,7 +506,7 @@ pub(super) fn emit_instance_alloc(ctx: &mut FnCtx<'_>, class_name: &str, class: // bits 32..63 = size (u32) let gc_packed: u64 = GC_TYPE_OBJECT | (GC_FLAG_ARENA << 8) - | (GC_LAYOUT_POINTER_FREE << 16) + | ((GC_LAYOUT_POINTER_FREE | typed_intact_bits) << 16) | ((total_size as u64) << 32); // GC_STORE_AUDIT(INIT): inline headers initialize freshly allocated unpublished object storage. blk.store(I64, &gc_packed.to_string(), &raw); diff --git a/crates/perry-codegen/src/lower_call/new_helpers.rs b/crates/perry-codegen/src/lower_call/new_helpers.rs index be98b4d0c2..a72f86af0c 100644 --- a/crates/perry-codegen/src/lower_call/new_helpers.rs +++ b/crates/perry-codegen/src/lower_call/new_helpers.rs @@ -585,3 +585,63 @@ pub(crate) fn ctor_chain_uses_new_target(ctx: &FnCtx<'_>, class: &Class) -> bool } false } + +/// ECMAScript constructor return-override, with the `undefined` case decided +/// **inline** (#7834). +/// +/// Every `new` site has to apply the spec's return-override rule: a +/// constructor that returns an Object yields that object, `undefined` yields +/// the implicit `this`, and any other primitive yields `this` for a base +/// constructor or throws for a derived one. `js_ctor_return_override` decides +/// all three. +/// +/// The overwhelmingly common case is the one it can answer from a single +/// 64-bit compare: a constructor with no `return ` at all completes as +/// `undefined`, and the runtime's answer for `undefined` is *exactly* +/// `this_val` — `JSValue::is_undefined` is `bits == TAG_UNDEFINED`, and +/// `constructor_return_overrides_this(undefined)` is `false` at its first +/// `is_pointer()` test. Emitting that compare here turns a cross-crate call +/// into a never-taken branch: measured at 8.2% of `churn_alloc` and `push_cls`, +/// where the synthesized object-literal constructor's only `ret` is the +/// `TAG_UNDEFINED` constant. +/// +/// The slow arm is byte-for-byte the previous emission, so derived-constructor +/// `TypeError`s, `return new Promise(…)`, arguments objects and arrays all keep +/// the runtime's answer. Skipping the call cannot lose a relocation either: +/// `js_ctor_return_override` is in `root_reload`'s no-reload set, so no live +/// value's address depends on having made it. +pub(super) fn emit_ctor_return_override( + ctx: &mut FnCtx<'_>, + obj_box: &str, + ctor_ret: &str, + is_derived: bool, +) -> String { + let override_idx = ctx.new_block("ctor_ret.override"); + let merge_idx = ctx.new_block("ctor_ret.merge"); + let override_label = ctx.block_label(override_idx); + let merge_label = ctx.block_label(merge_idx); + let this_pred_label = ctx.block_label(ctx.current_block); + { + let blk = ctx.block(); + let bits = blk.bitcast_double_to_i64(ctor_ret); + let is_undef = blk.icmp_eq(crate::types::I64, &bits, crate::nanbox::TAG_UNDEFINED_I64); + blk.cond_br(&is_undef, &merge_label, &override_label); + } + ctx.current_block = override_idx; + let is_derived_lit = if is_derived { "1" } else { "0" }; + let overridden = ctx.block().call( + DOUBLE, + "js_ctor_return_override", + &[(DOUBLE, obj_box), (DOUBLE, ctor_ret), (I32, is_derived_lit)], + ); + let override_pred_label = ctx.block_label(ctx.current_block); + ctx.block().br(&merge_label); + ctx.current_block = merge_idx; + ctx.block().phi( + DOUBLE, + &[ + (obj_box, &this_pred_label), + (&overridden, &override_pred_label), + ], + ) +} diff --git a/crates/perry-codegen/src/lower_call/typed_shape_bake_tests.rs b/crates/perry-codegen/src/lower_call/typed_shape_bake_tests.rs new file mode 100644 index 0000000000..8d29b40781 --- /dev/null +++ b/crates/perry-codegen/src/lower_call/typed_shape_bake_tests.rs @@ -0,0 +1,393 @@ +//! #7834: the at-allocation typed-shape layout, folded into the header +//! constant — and the `undefined` return-override decided inline. +//! +//! Both are IR-census tests, and both are the "assert the subject was live" +//! kind (CLAUDE.md). An optimisation whose predicate quietly answers `false` +//! everywhere still compiles, still prints the right answer, and shows up in no +//! other test — `js_gc_declare_typed_shape_layout` was 30% of `churn_alloc` and +//! nothing but a profile said so. +//! +//! ## What the positive asserts +//! +//! For a class whose pointer mask is statically EMPTY, the canonical layout is +//! the constant `GC_LAYOUT_POINTER_FREE | GC_OBJ_TYPED_LAYOUT_INTACT`, so the +//! inline-bump path stamps it into the packed `GcHeader` store it was already +//! emitting and drops the per-instance call. What survives is the one half that +//! depends on the recycled ADDRESS rather than on the shape — clearing a +//! previous tenant's per-object record — behind a `PERRY_PER_OBJECT_LAYOUTS_ANY` +//! test whose `0` state proves every thread's tables empty. +//! +//! ## What the negative asserts +//! +//! A pointer-BEARING shape keeps the full runtime declare. It has to: its state +//! is `GC_LAYOUT_SIDE_MASK`, which means the collector consults a mask, and the +//! shared `SHAPE_LAYOUTS` descriptor that mask lives in is installed by exactly +//! that call. Baking `SIDE_MASK` into the header without it would hand the +//! tracer a masked object with no mask. +//! +//! ## Why the pointer-free bake needs no descriptor +//! +//! `heap_payload_slot_selection` skips a `GC_LAYOUT_POINTER_FREE` payload +//! outright, without consulting any map — so the collector's view is +//! bit-identical to the pre-#7834 one, which also reached `POINTER_FREE` for an +//! empty pointer mask. And a later pointer store still downgrades: with no +//! descriptor to classify against, `layout_note_slot` falls through to its +//! generic pointer-mask branch, which mints a per-object mask and flips the +//! state to `SIDE_MASK`. That branch needs no descriptor at all. + +use crate::{compile_module, AppMetadata, CompileOptions}; +use perry_hir::types::Type; +use perry_hir::{ + BinaryOp, Class, ClassField, CompareOp, Expr, Function, Module, ModuleInitKind, Param, Stmt, + UpdateOp, +}; + +/// The six-argument per-instance declare this ticket removes. +const DECLARE_CALL: &str = "call void @js_gc_declare_typed_shape_layout("; +/// The one-argument address-only remainder that replaces it. +const FORGET_CALL: &str = "call void @js_gc_forget_object_layout("; +/// The process-global emptiness proof the remainder is gated on. +const ANY_GLOBAL: &str = "@PERRY_PER_OBJECT_LAYOUTS_ANY"; + +/// The packed `GcHeader` word the inline bump writes for a two-`number`-field +/// class, WITH the baked layout: +/// +/// ```text +/// obj_type GC_TYPE_OBJECT = 0x02 bits 0..7 +/// gc_flags GC_FLAG_ARENA = 0x02 bits 8..15 +/// _reserved GC_LAYOUT_POINTER_FREE | INTACT = 0x5000 bits 16..31 +/// size 8 + 32 + max(2,4)*8 = 72 bits 32..63 +/// ``` +const BAKED_HEADER_WORD: &str = "store i64 310579823106,"; +/// The same word WITHOUT `GC_OBJ_TYPED_LAYOUT_INTACT` (0x1000 << 16 less) — +/// what the pointer-bearing class still writes. +const UNBAKED_HEADER_WORD: &str = "store i64 310311387650,"; + +fn ir_opts() -> CompileOptions { + CompileOptions { + target: None, + is_entry_module: true, + non_entry_module_prefixes: Vec::new(), + nextjs_path_init_modules: Vec::new(), + import_function_prefixes: std::collections::HashMap::new(), + import_function_ffi_aliases: std::collections::HashMap::new(), + import_function_origin_names: std::collections::HashMap::new(), + import_function_v8_specifiers: std::collections::HashMap::new(), + import_function_node_submodule: std::collections::HashMap::new(), + namespace_node_submodules: std::collections::HashMap::new(), + namespace_v8_specifiers: std::collections::HashMap::new(), + namespace_member_prefixes: std::collections::HashMap::new(), + namespace_member_origin_names: std::collections::HashMap::new(), + emit_ir_only: true, + verify_native_regions: false, + disable_buffer_fast_path: false, + namespace_imports: Vec::new(), + namespace_member_nested: Vec::new(), + imported_classes: Vec::new(), + imported_enums: Vec::new(), + imported_async_funcs: std::collections::HashSet::new(), + type_aliases: std::collections::HashMap::new(), + imported_func_param_counts: std::collections::HashMap::new(), + imported_func_has_rest: std::collections::HashSet::new(), + imported_func_synthetic_arguments: std::collections::HashSet::new(), + imported_func_return_types: std::collections::HashMap::new(), + imported_vars: std::collections::HashSet::new(), + output_type: "executable".to_string(), + needs_stdlib: false, + needs_ui: false, + needs_geisterhand: false, + geisterhand_port: 7676, + enabled_features: Vec::new(), + native_module_init_names: Vec::new(), + js_module_specifiers: Vec::new(), + bundled_extensions: Vec::new(), + native_library_functions: Vec::new(), + i18n_table: None, + fast_math: false, + fp_contract_mode: crate::FpContractMode::Off, + app_metadata: AppMetadata::default(), + namespace_entries: Vec::new(), + dynamic_import_path_to_prefix: std::collections::HashMap::new(), + deferred_module_prefixes: std::collections::HashSet::new(), + module_init_deps: Vec::new(), + is_dynamic_import_target: false, + debug_locations: false, + module_source: None, + debug_source_line_offset: 0, + } +} + +fn field(name: &str, ty: Type) -> ClassField { + ClassField { + name: name.to_string(), + key_expr: None, + ty, + init: None, + is_private: false, + is_readonly: false, + decorators: Vec::new(), + } +} + +const A_ID: u32 = 1; +const B_ID: u32 = 2; +const I_ID: u32 = 7; + +/// `constructor(a, b) { this. = a; this. = b }` — the maximal +/// param-assigned prologue `ctor_prologue_param_assigned_fields` admits, which +/// is what makes the layout declarable at allocation at all (#7510). +fn two_field_ctor(f0: &str, f1: &str, f1_ty: Type) -> Function { + Function { + id: 900, + name: "constructor".to_string(), + type_params: Vec::new(), + params: vec![ + Param { + id: A_ID, + name: "a".to_string(), + ty: Type::Number, + default: None, + decorators: Vec::new(), + is_rest: false, + arguments_object: None, + }, + Param { + id: B_ID, + name: "b".to_string(), + ty: f1_ty, + default: None, + decorators: Vec::new(), + is_rest: false, + arguments_object: None, + }, + ], + return_type: Type::Void, + body: vec![ + Stmt::Expr(Expr::PropertySet { + object: Box::new(Expr::This), + property: f0.to_string(), + value: Box::new(Expr::LocalGet(A_ID)), + }), + Stmt::Expr(Expr::PropertySet { + object: Box::new(Expr::This), + property: f1.to_string(), + value: Box::new(Expr::LocalGet(B_ID)), + }), + ], + is_async: false, + is_generator: false, + is_strict: false, + is_exported: false, + captures: Vec::new(), + decorators: Vec::new(), + was_plain_async: false, + was_unrolled: false, + } +} + +fn two_field_class(name: &str, f1_ty: Type) -> Class { + Class { + id: 404, + name: name.to_string(), + type_params: Vec::new(), + extends: None, + extends_name: None, + native_extends: None, + extends_expr: None, + heritage_lexically_shadowed: false, + fields: vec![field("a", Type::Number), field("b", f1_ty.clone())], + constructor: Some(two_field_ctor("a", "b", f1_ty)), + methods: Vec::new(), + getters: Vec::new(), + setters: Vec::new(), + static_accessor_names: Vec::new(), + static_accessor_fn_ids: Vec::new(), + computed_members: Vec::new(), + static_fields: Vec::new(), + static_methods: Vec::new(), + decorators: Vec::new(), + is_exported: false, + aliases: Vec::new(), + is_nested: false, + alloc_width_hint: 0, + specialized_from: None, + } +} + +/// `for (let i = 0; i < 1000; i++) { const x = new (i, ); }` +/// +/// The loop is load-bearing: `new_site_is_in_loop` is what selects the inline +/// bump allocator, and only the inline bump has a packed header constant to +/// fold the layout into. An outlined `js_object_alloc_class_inline_keys` site +/// keeps the runtime declare, by design. +fn loop_new_module(name: &str, f1_ty: Type, second: Expr) -> Module { + let mut m = Module::new("typed_shape_bake.ts"); + m.classes = vec![two_field_class(name, f1_ty)]; + m.init = vec![Stmt::For { + init: Some(Box::new(Stmt::Let { + id: I_ID, + name: "i".to_string(), + ty: Type::Any, + mutable: true, + init: Some(Expr::Integer(0)), + })), + condition: Some(Expr::Compare { + op: CompareOp::Lt, + left: Box::new(Expr::LocalGet(I_ID)), + right: Box::new(Expr::Integer(1000)), + }), + update: Some(Expr::Update { + id: I_ID, + op: UpdateOp::Increment, + prefix: false, + }), + body: vec![ + Stmt::Let { + id: 20, + name: "x".to_string(), + ty: Type::Named(name.to_string()), + mutable: false, + init: Some(Expr::New { + class_name: name.to_string(), + args: vec![ + Expr::Binary { + op: BinaryOp::Add, + left: Box::new(Expr::LocalGet(I_ID)), + right: Box::new(Expr::Integer(1)), + }, + second, + ], + type_args: Vec::new(), + byte_offset: 0, + cap_args_appended: 0, + }), + }, + // ESCAPE. Without this the instance never leaves the iteration and + // is scalar-replaced away — the test would then assert about an + // allocation the compiler deleted, and would pass on a build where + // the bake does nothing. (`collectors/escape_news.rs`; the same trap + // is called out in the campaign's measurement protocol.) + Stmt::Expr(Expr::Call { + callee: Box::new(Expr::FuncRef(10)), + args: vec![Expr::LocalGet(20)], + type_args: Vec::new(), + byte_offset: 0, + }), + ], + }]; + m.functions = vec![Function { + id: 10, + name: "sink".to_string(), + type_params: Vec::new(), + params: vec![Param { + id: 30, + name: "p".to_string(), + ty: Type::Named(name.to_string()), + default: None, + decorators: Vec::new(), + is_rest: false, + arguments_object: None, + }], + return_type: Type::Number, + body: vec![Stmt::Return(Some(Expr::PropertyGet { + object: Box::new(Expr::LocalGet(30)), + property: "a".to_string(), + byte_offset: 0, + }))], + is_async: false, + is_generator: false, + is_strict: false, + is_exported: true, + captures: Vec::new(), + decorators: Vec::new(), + was_plain_async: false, + was_unrolled: false, + }]; + m.init_kind = ModuleInitKind::Eager; + m +} + +fn emit(m: &Module) -> String { + String::from_utf8(compile_module(m, ir_opts()).unwrap()).expect("LLVM IR should be UTF-8") +} + +/// `class Pair { a: number; b: number }` — pointer mask statically empty. +#[test] +fn a_pointer_free_shape_bakes_its_layout_into_the_header_constant() { + let ir = emit(&loop_new_module("Pair", Type::Number, Expr::Integer(2))); + assert!( + ir.contains(BAKED_HEADER_WORD), + "the inline-bump header constant does not carry \ + GC_OBJ_TYPED_LAYOUT_INTACT, so the bake did not fire and every \ + construction still pays the runtime declare:\n{ir}" + ); + assert!( + !ir.contains(DECLARE_CALL), + "the per-instance `js_gc_declare_typed_shape_layout` is still emitted \ + for a pointer-free shape — this is the 30% of `churn_alloc` the \ + ticket removes:\n{ir}" + ); + assert!( + ir.contains(FORGET_CALL) && ir.contains(ANY_GLOBAL), + "the address-dependent half must survive, gated on the global \ + emptiness proof: a recycled address can carry a previous tenant's \ + per-object mask, and `layout_note_slot` would then OR the new \ + object's pointer bits into it:\n{ir}" + ); +} + +/// `class Link { a: number; b: Link | null }` — the control. One declared type +/// differs; everything else about the program is identical. +#[test] +fn a_pointer_bearing_shape_keeps_the_runtime_declare() { + let ir = emit(&loop_new_module( + "Link", + Type::Union(vec![Type::Named("Link".to_string()), Type::Null]), + Expr::Null, + )); + assert!( + ir.contains(DECLARE_CALL), + "a SIDE_MASK shape MUST keep the declare — it is the only thing that \ + installs the shared `SHAPE_LAYOUTS` descriptor the tracer's mask \ + lookup reads:\n{ir}" + ); + assert!( + ir.contains(UNBAKED_HEADER_WORD) && !ir.contains(BAKED_HEADER_WORD), + "the header constant must NOT claim GC_OBJ_TYPED_LAYOUT_INTACT for a \ + shape whose descriptor is installed at runtime:\n{ir}" + ); + assert!( + !ir.contains(FORGET_CALL), + "the standalone forget is only for the baked path; the declare already \ + performs it:\n{ir}" + ); +} + +/// The return-override's `undefined` arm, decided inline. +/// +/// Asserted together with the surviving call: the change is "answer the common +/// case with one compare", never "stop applying the spec rule". A constructor +/// that returns an object, an arguments object or an array — and a derived +/// constructor that returns a primitive, which must throw — all still route to +/// the runtime. +#[test] +fn an_undefined_constructor_completion_takes_the_inline_arm() { + let ir = emit(&loop_new_module("Pair", Type::Number, Expr::Integer(2))); + assert!( + ir.contains("ctor_ret.merge"), + "the inline `undefined` arm was not emitted, so every construction \ + still calls `js_ctor_return_override` (8% of `churn_alloc`):\n{ir}" + ); + assert!( + ir.contains(", 9222246136947933185") && ir.contains("ctor_ret.override"), + "the inline arm must be exactly the TAG_UNDEFINED bit compare — \ + `JSValue::is_undefined` is `bits == TAG_UNDEFINED`, which is what \ + makes returning `this` here equal to what the runtime returns:\n{ir}" + ); + assert!( + ir.contains("call double @js_ctor_return_override("), + "the runtime call must survive on the cold arm; without it a \ + constructor returning an object would be ignored and a derived one \ + returning a primitive would not throw:\n{ir}" + ); +} diff --git a/crates/perry-codegen/src/lower_call/typed_shape_init.rs b/crates/perry-codegen/src/lower_call/typed_shape_init.rs index e30f822506..4b29accb8c 100644 --- a/crates/perry-codegen/src/lower_call/typed_shape_init.rs +++ b/crates/perry-codegen/src/lower_call/typed_shape_init.rs @@ -44,6 +44,40 @@ pub(super) fn layout_declared_at_allocation(ctx: &FnCtx<'_>, class_name: &str) - }) } +/// #7834: is `class_name`'s at-allocation declaration expressible as a +/// **constant** — the state `GC_LAYOUT_POINTER_FREE | GC_OBJ_TYPED_LAYOUT_INTACT` +/// stamped straight into the inline-bump path's packed `GcHeader` store? +/// +/// Three conditions, and each maps to one branch of +/// `gc::layout::init_typed_shape_layout` that would otherwise decide it at +/// runtime, per instance: +/// +/// 1. [`layout_declared_at_allocation`] — the declare form is what would have +/// been emitted at all, so the fresh-slot proof is already discharged. +/// 2. The pointer mask is **statically empty**, so the state is +/// `GC_LAYOUT_POINTER_FREE` and the shape needs no `SHAPE_LAYOUTS` +/// descriptor: with no pointer-bearing slot there is nothing for a mask to +/// select, and `heap_payload_slot_selection` skips the payload outright. +/// 3. `field_count == slot_count` — the runtime's one *downgrading* branch +/// (`layout_set_typed_unknown`), which a constant cannot express. +/// +/// What is deliberately NOT folded in is `layout_forget_object`: it depends on +/// the recycled ADDRESS, not on the shape. The caller emits it separately, +/// behind the `PERRY_PER_OBJECT_LAYOUTS_ANY` gate. +pub(super) fn layout_pointer_free_at_allocation( + ctx: &FnCtx<'_>, + class_name: &str, + field_count: u32, +) -> bool { + if !layout_declared_at_allocation(ctx, class_name) { + return false; + } + let Some(typed_layout) = resolve_typed_layout(ctx, class_name) else { + return false; + }; + typed_layout.pointer_mask_words.is_empty() && typed_layout.slot_count == field_count +} + /// Emit the `js_gc_declare_typed_shape_layout` call that registers a **freshly /// allocated** instance's layout, before its constructor runs, so the /// constructor's own field stores can pass the intact-bit guard (#7510/#7512). @@ -51,14 +85,26 @@ pub(super) fn layout_declared_at_allocation(ctx: &FnCtx<'_>, class_name: &str) - /// No-op unless [`layout_declared_at_allocation`] holds. **Must be emitted /// while the instance's slots are still the allocator's fill** — that is the /// runtime contract, and it is not checkable from the runtime side. +/// +/// #7834: when the allocation already stamped the layout into the header +/// constant (`typed_layout_baked`), the shape half of this call is already +/// done and only the address half is left — clearing whatever per-object +/// record a previous tenant of this recycled address left behind. That is +/// gated on a process-global emptiness proof, so the steady state is one +/// never-taken branch instead of a six-argument runtime call. pub(super) fn emit_typed_shape_layout_declare( ctx: &mut FnCtx<'_>, class_name: &str, obj_handle: &str, + typed_layout_baked: bool, ) { if !layout_declared_at_allocation(ctx, class_name) { return; } + if typed_layout_baked { + emit_gated_forget_object_layout(ctx, obj_handle); + return; + } emit_typed_shape_layout_call( ctx, class_name, @@ -67,6 +113,31 @@ pub(super) fn emit_typed_shape_layout_declare( ); } +/// `if (PERRY_PER_OBJECT_LAYOUTS_ANY) js_gc_forget_object_layout(obj);` +/// +/// The load is **volatile** for the same reason +/// `PERRY_CLASS_FIELD_INLINE_GUARD_DISABLED`'s is: the runtime flips this byte +/// mid-execution (both ways), and LLVM must not hoist a stale `0` out of the +/// loop this construction sits in. A never-taken branch per allocation is the +/// entire steady-state cost. +fn emit_gated_forget_object_layout(ctx: &mut FnCtx<'_>, obj_handle: &str) { + let call_idx = ctx.new_block("layout_forget.armed"); + let done_idx = ctx.new_block("layout_forget.done"); + let call_label = ctx.block_label(call_idx); + let done_label = ctx.block_label(done_idx); + { + let blk = ctx.block(); + let any = blk.load_volatile(crate::types::I8, "@PERRY_PER_OBJECT_LAYOUTS_ANY"); + let armed = blk.icmp_ne(crate::types::I8, &any, "0"); + blk.cond_br(&armed, &call_label, &done_label); + } + ctx.current_block = call_idx; + ctx.block() + .call_void("js_gc_forget_object_layout", &[(I64, obj_handle)]); + ctx.block().br(&done_label); + ctx.current_block = done_idx; +} + /// Emit the `js_gc_init_typed_shape_layout` call that registers the freshly /// constructed instance's raw-f64 / pointer slot masks with the GC so the /// typed-feedback class-field fast path engages. Must run AFTER the constructor @@ -93,6 +164,27 @@ pub(super) fn emit_typed_shape_layout_init( emit_typed_shape_layout_call(ctx, class_name, obj_handle, "js_gc_init_typed_shape_layout"); } +/// The class's canonical typed slot layout, resolved the way both emitters +/// need it. +/// +/// Refs #5094: prefer the prefix-disambiguated chain so slot/word counts agree +/// with the mask globals emitted in `compile_module` (same-named cross-module +/// parents mis-resolve in the name-keyed walk). Returns `None` when the class +/// has no keys global, which is the same condition +/// [`emit_typed_shape_layout_call`] bails on. +fn resolve_typed_layout( + ctx: &FnCtx<'_>, + class_name: &str, +) -> Option { + ctx.class_keys_globals.get(class_name)?; + Some( + ctx.class_init_chains + .get(class_name) + .map(|chain| crate::typed_shape::class_typed_layout_from_chain(chain)) + .unwrap_or_else(|| crate::typed_shape::class_typed_layout(ctx.classes, class_name)), + ) +} + /// The shared operand build. Both entry points take the identical six-argument /// signature, so the only thing that varies is the callee name. fn emit_typed_shape_layout_call( @@ -104,14 +196,9 @@ fn emit_typed_shape_layout_call( let Some(keys_global_name) = ctx.class_keys_globals.get(class_name).cloned() else { return; }; - // Refs #5094: prefer the prefix-disambiguated chain so slot/word counts - // agree with the mask globals emitted in compile_module (same-named - // cross-module parents mis-resolve in the name-keyed walk). - let typed_layout = ctx - .class_init_chains - .get(class_name) - .map(|chain| crate::typed_shape::class_typed_layout_from_chain(chain)) - .unwrap_or_else(|| crate::typed_shape::class_typed_layout(ctx.classes, class_name)); + let Some(typed_layout) = resolve_typed_layout(ctx, class_name) else { + return; + }; let slot_count_str = typed_layout.slot_count.to_string(); let raw_mask_word_count_str = typed_layout.raw_f64_mask_words.len().to_string(); let pointer_mask_word_count_str = typed_layout.pointer_mask_words.len().to_string(); diff --git a/crates/perry-codegen/src/root_reload.rs b/crates/perry-codegen/src/root_reload.rs index 1bd7cd6175..0360797c8f 100644 --- a/crates/perry-codegen/src/root_reload.rs +++ b/crates/perry-codegen/src/root_reload.rs @@ -182,6 +182,7 @@ const NON_COLLECTING: &[&str] = &[ // layout / barrier bookkeeping "js_gc_init_typed_shape_layout", "js_gc_declare_typed_shape_layout", + "js_gc_forget_object_layout", "js_gc_layout_note_slot", "js_write_barrier", "js_write_barrier_root_nanbox", diff --git a/crates/perry-codegen/src/runtime_decls/arrays.rs b/crates/perry-codegen/src/runtime_decls/arrays.rs index 7872a736fe..7105c6fe54 100644 --- a/crates/perry-codegen/src/runtime_decls/arrays.rs +++ b/crates/perry-codegen/src/runtime_decls/arrays.rs @@ -179,6 +179,10 @@ pub fn declare_phase_b_arrays(module: &mut LlModule) { VOID, &[I64, I32, PTR, I32, PTR, I32], ); + // #7834: the address-dependent half of the declare, on its own. Emitted + // behind a `PERRY_PER_OBJECT_LAYOUTS_ANY` test by a construction site whose + // shape half is already baked into the inline-bump header constant. + module.declare_function("js_gc_forget_object_layout", VOID, &[I64]); // Array methods (Phase B.12). // - js_array_pop_f64(arr) -> f64 (last element, NaN if empty) // - js_array_join(arr, sep) -> *mut StringHeader (i64) diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index b6f33ff310..4fbe78c468 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -29,6 +29,11 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // when it is non-zero (descriptors / typed-feedback in use). Defined in // perry-runtime as `PERRY_CLASS_FIELD_INLINE_GUARD_DISABLED`. module.add_external_global("PERRY_CLASS_FIELD_INLINE_GUARD_DISABLED", I8); + // #7834: process-global "some thread holds a per-object layout record". + // `0` proves both per-object side tables are empty everywhere, so a + // construction site can skip `js_gc_forget_object_layout` outright. + // perry-runtime: `gc::layout_tables::PERRY_PER_OBJECT_LAYOUTS_ANY`. + module.add_external_global("PERRY_PER_OBJECT_LAYOUTS_ANY", I8); // Sticky summary of indexed Array/Object prototype pollution and custom // Array [[Prototype]] installation. Normal compiled programs read this // byte directly in the inline plain-array index guard. diff --git a/crates/perry-runtime/src/gc/layout_tables.rs b/crates/perry-runtime/src/gc/layout_tables.rs index cb3a151e57..b004630446 100644 --- a/crates/perry-runtime/src/gc/layout_tables.rs +++ b/crates/perry-runtime/src/gc/layout_tables.rs @@ -380,7 +380,76 @@ pub(in crate::gc) fn per_object_layouts_maybe_nonempty() -> bool { /// through the wrappers below. #[inline(always)] pub(in crate::gc) fn mark_per_object_layouts_nonempty() { - hot_per_object_layout_hint().nonempty.set(true); + let hint = hot_per_object_layout_hint(); + if !hint.nonempty.replace(true) { + per_object_layouts_global_arm(); + } +} + +/// Process-global mirror of [`PER_OBJECT_LAYOUTS_NONEMPTY`], ORed over every +/// thread, exported so **generated code** can test it with one load (#7834). +/// +/// `layout_forget_object` is a runtime call on every inline-bump construction, +/// and on a monomorphic workload every one of those calls returns immediately +/// having proved emptiness. The proof itself is thread-local, so codegen could +/// not read it: a `_tlv_get_addr` from generated code costs more than the call +/// it would replace. This byte is the same proof in a plain `static`, so the +/// construction site becomes `load i8` + a never-taken branch. +/// +/// `0` is a proof that **no thread** holds a per-object layout record, and so +/// that no recycled address can carry a stale one. `1` is only a hint — the +/// call it gates re-tests the thread-local flag and the address filter, exactly +/// as it always did. +/// +/// Maintained by a count of *armed threads* rather than a sticky set, so a +/// workload that arms the regime transiently (a `tree`-shaped phase that later +/// drops every per-object record) returns to the cheap path. A thread that +/// exits while armed leaks its count, which can only leave this stuck at `1` — +/// the conservative direction, costing the pre-#7834 call and nothing else. +#[no_mangle] +pub static PERRY_PER_OBJECT_LAYOUTS_ANY: std::sync::atomic::AtomicU8 = + std::sync::atomic::AtomicU8::new(0); + +/// How many threads currently have a non-empty per-object layout table. +static PER_OBJECT_LAYOUT_ARMED_THREADS: std::sync::atomic::AtomicIsize = + std::sync::atomic::AtomicIsize::new(0); + +#[inline(never)] +fn per_object_layouts_global_arm() { + use std::sync::atomic::Ordering; + PER_OBJECT_LAYOUT_ARMED_THREADS.fetch_add(1, Ordering::Relaxed); + PERRY_PER_OBJECT_LAYOUTS_ANY.store(1, Ordering::Release); +} + +#[inline(never)] +fn per_object_layouts_global_disarm() { + use std::sync::atomic::Ordering; + if PER_OBJECT_LAYOUT_ARMED_THREADS.fetch_sub(1, Ordering::Relaxed) <= 1 { + // Re-read rather than trusting the returned value: another thread may + // have armed between the decrement and here, and the store must not + // clear a live arm. A lost race leaves the byte at `1` with the count + // at `0`, which is the conservative direction (see the doc above). + if PER_OBJECT_LAYOUT_ARMED_THREADS.load(Ordering::Relaxed) <= 0 { + PERRY_PER_OBJECT_LAYOUTS_ANY.store(0, Ordering::Release); + } + } +} + +/// The generated-code entry point for [`layout_forget_object`] (#7834). +/// +/// An inline-bump `new` site that baked its layout state into the header +/// constant still has to clear whatever a previous tenant of the recycled +/// address left in the per-object tables — that is the one part of +/// `js_gc_declare_typed_shape_layout` which depends on the address rather than +/// on the shape. Codegen emits this call behind a +/// [`PERRY_PER_OBJECT_LAYOUTS_ANY`] test, so it runs only in the armed regime. +#[no_mangle] +pub extern "C" fn js_gc_forget_object_layout(obj: u64) { + let user_ptr = super::layout::strip_nanbox_user_ptr(obj); + if user_ptr == 0 { + return; + } + layout_forget_object(user_ptr); } /// Re-establish the flag after a removal emptied one map: clear it once the @@ -396,7 +465,9 @@ pub(in crate::gc) fn refresh_per_object_layouts_flag(touched_map_emptied: bool) return; } if hot_layout_slot_masks().borrow().is_empty() && hot_typed_layouts().borrow().is_empty() { - hot_per_object_layout_hint().nonempty.set(false); + if hot_per_object_layout_hint().nonempty.replace(false) { + per_object_layouts_global_disarm(); + } // Both maps are empty, so every bit is now stale. Clearing here is what // makes the filter's occupancy track LIVE entries rather than every // entry the program has ever created. @@ -534,6 +605,16 @@ pub(in crate::gc) fn transfer_per_object_slot_mask(old_user: usize, new_user: us /// pre-#7510 path unchanged, and re-arms the flag on the way out. #[inline] pub(in crate::gc) fn layout_forget_object(user_ptr: usize) { + // #7834: the process-global mirror first, because reading it is a plain + // static load while `hot_per_object_layout_hint()` is a thread-local — and + // on Darwin a thread-local access is an out-of-line `_tlv_get_addr` call. + // `0` proves every thread's tables are empty, which is the steady state of + // every monomorphic workload, so the disarmed path now costs one load and + // one branch instead of a call. (Measured as 6% of `cycles`, whose + // pointer-bearing shape keeps the full runtime declare.) + if PERRY_PER_OBJECT_LAYOUTS_ANY.load(std::sync::atomic::Ordering::Acquire) == 0 { + return; + } // ONE hot-slot resolution for both halves of the guard: the flag (cheap, // and false for the overwhelming majority of workloads) and then the // address filter (what rescues a workload with an immortal record).