diff --git a/README.md b/README.md index 37bf8ccc..9ec5dcbd 100644 --- a/README.md +++ b/README.md @@ -761,6 +761,9 @@ frontend/source patterns may still reach a `Bridge/Exit` path even when an SSA i | `len(map)` | Bridge/Exit | Inline | | `get(map)` | Bridge/Exit | Helper | | `has(map)` | Bridge/Exit | Helper | +| `string_contains`, `string_replace_literal`, `string_lower_ascii`, `string_split_literal` | Bridge/Exit | Helper | +| `re::match`, `re::replace` | Bridge/Exit | Helper | +| Dynamic `type`, `to_string`, equality, and `len` | Bridge/Exit | Helper (known `type` / string `to_string` cases are folded) | | All other builtins | Bridge/Exit | Bridge/Exit or NYI, depending on trace shape | | Host imports | Bridge/Exit | Bridge/Exit or branch-exit trace, never inline | diff --git a/src/builtins/runtime/core.rs b/src/builtins/runtime/core.rs index 8fb1ff7e..747cbe9d 100644 --- a/src/builtins/runtime/core.rs +++ b/src/builtins/runtime/core.rs @@ -425,7 +425,7 @@ pub(super) fn builtin_type_of_impl(value: VmValueRef<'_>) -> String { /// Convert a value into a display string. #[pd_host_function(name = "__to_string")] -pub(super) fn builtin_to_string_impl(value: VmValueRef<'_>) -> String { +pub(crate) fn builtin_to_string_impl(value: VmValueRef<'_>) -> String { render_value_for_display(value) } diff --git a/src/builtins/runtime/regex.rs b/src/builtins/runtime/regex.rs index a3885ddc..b335cf53 100644 --- a/src/builtins/runtime/regex.rs +++ b/src/builtins/runtime/regex.rs @@ -107,6 +107,10 @@ pub(super) fn builtin_re_match(vm: &mut Vm, pattern: &str, text: &str) -> VmResu Ok(regex.is_match(text)) } +pub(crate) fn native_re_match(vm: &mut Vm, pattern: &str, text: &str) -> VmResult { + builtin_re_match_impl(vm, pattern, text) +} + /// Returns the first substring matched by a regular expression. #[pd_host_function(name = "re::find")] pub(super) fn builtin_re_find(vm: &mut Vm, pattern: &str, text: &str) -> VmResult> { @@ -126,6 +130,15 @@ pub(super) fn builtin_re_replace( Ok(regex.replace_all(text, replacement).into_owned()) } +pub(crate) fn native_re_replace( + vm: &mut Vm, + pattern: &str, + text: &str, + replacement: &str, +) -> VmResult { + builtin_re_replace_impl(vm, pattern, text, replacement) +} + /// Splits a string on regular-expression matches. #[pd_host_function(name = "re::split")] pub(super) fn builtin_re_split(vm: &mut Vm, pattern: &str, text: &str) -> VmResult { diff --git a/src/vm/jit/ir.rs b/src/vm/jit/ir.rs index 5fb78eae..498db899 100644 --- a/src/vm/jit/ir.rs +++ b/src/vm/jit/ir.rs @@ -107,6 +107,9 @@ pub(crate) enum SsaInstKind { input: SsaValueId, tag: ValueType, }, + ValueLen { + value: SsaValueId, + }, StringLen { text: SsaValueId, }, @@ -139,6 +142,15 @@ pub(crate) enum SsaInstKind { text: SsaValueId, needle: SsaValueId, }, + RegexMatch { + pattern: SsaValueId, + text: SsaValueId, + }, + RegexReplace { + pattern: SsaValueId, + text: SsaValueId, + replacement: SsaValueId, + }, StringReplaceLiteral { text: SsaValueId, needle: SsaValueId, @@ -147,6 +159,12 @@ pub(crate) enum SsaInstKind { StringLowerAscii { text: SsaValueId, }, + TypeOf { + value: SsaValueId, + }, + ToString { + value: SsaValueId, + }, StringSplitLiteral { text: SsaValueId, delimiter: SsaValueId, @@ -331,6 +349,10 @@ pub(crate) enum SsaInstKind { lhs: SsaValueId, rhs: SsaValueId, }, + ValueCmpEq { + lhs: SsaValueId, + rhs: SsaValueId, + }, IntCmpLt { lhs: SsaValueId, rhs: SsaValueId, @@ -358,6 +380,7 @@ impl SsaInstKind { | Self::UnboxFloat { input } | Self::UnboxBool { input } | Self::UnboxHeapPtr { input, .. } + | Self::ValueLen { value: input } | Self::StringLen { text: input } | Self::BytesLen { bytes: input } | Self::ArrayLen { array: input } @@ -379,12 +402,19 @@ impl SsaInstKind { Self::BytesGet { bytes, index } => vec![*bytes, *index], Self::BytesHas { bytes, index } => vec![*bytes, *index], Self::StringContains { text, needle } => vec![*text, *needle], + Self::RegexMatch { pattern, text } => vec![*pattern, *text], + Self::RegexReplace { + pattern, + text, + replacement, + } => vec![*pattern, *text, *replacement], Self::StringReplaceLiteral { text, needle, replacement, } => vec![*text, *needle, *replacement], Self::StringLowerAscii { text } => vec![*text], + Self::TypeOf { value } | Self::ToString { value } => vec![*value], Self::StringSplitLiteral { text, delimiter } => vec![*text, *delimiter], Self::StringConcat { lhs, rhs } | Self::BytesConcat { lhs, rhs } => vec![*lhs, *rhs], Self::BytesFromArrayU8 { array } => vec![*array], @@ -422,6 +452,7 @@ impl SsaInstKind { | Self::FloatCmpLt { lhs, rhs } | Self::FloatCmpGt { lhs, rhs } | Self::IntCmpEq { lhs, rhs } + | Self::ValueCmpEq { lhs, rhs } | Self::IntCmpLt { lhs, rhs } | Self::IntCmpGt { lhs, rhs } => vec![*lhs, *rhs], Self::IntAddImm { lhs, .. } @@ -975,6 +1006,7 @@ fn render_inst_kind(kind: &SsaInstKind) -> String { SsaInstKind::UnboxFloat { input } => format!("unbox_float {input}"), SsaInstKind::UnboxBool { input } => format!("unbox_bool {input}"), SsaInstKind::UnboxHeapPtr { input, tag } => format!("unbox_ptr {input}, {tag:?}"), + SsaInstKind::ValueLen { value } => format!("value_len {value}"), SsaInstKind::StringLen { text } => format!("string_len {text}"), SsaInstKind::BytesLen { bytes } => format!("bytes_len {bytes}"), SsaInstKind::StringSlice { @@ -993,12 +1025,24 @@ fn render_inst_kind(kind: &SsaInstKind) -> String { SsaInstKind::StringContains { text, needle } => { format!("string_contains {text}, {needle}") } + SsaInstKind::RegexMatch { pattern, text } => { + format!("regex_match {pattern}, {text}") + } + SsaInstKind::RegexReplace { + pattern, + text, + replacement, + } => format!("regex_replace {pattern}, {text}, {replacement}"), SsaInstKind::StringReplaceLiteral { text, needle, replacement, - } => format!("string_replace_literal {text}, {needle}, {replacement}"), + } => { + format!("string_replace_literal {text}, {needle}, {replacement}") + } SsaInstKind::StringLowerAscii { text } => format!("string_lower_ascii {text}"), + SsaInstKind::TypeOf { value } => format!("type_of {value}"), + SsaInstKind::ToString { value } => format!("to_string {value}"), SsaInstKind::StringSplitLiteral { text, delimiter } => { format!("string_split_literal {text}, {delimiter}") } @@ -1057,6 +1101,7 @@ fn render_inst_kind(kind: &SsaInstKind) -> String { SsaInstKind::FloatCmpLt { lhs, rhs } => format!("fcmp_lt {lhs}, {rhs}"), SsaInstKind::FloatCmpGt { lhs, rhs } => format!("fcmp_gt {lhs}, {rhs}"), SsaInstKind::IntCmpEq { lhs, rhs } => format!("icmp_eq {lhs}, {rhs}"), + SsaInstKind::ValueCmpEq { lhs, rhs } => format!("value_eq {lhs}, {rhs}"), SsaInstKind::IntCmpLt { lhs, rhs } => format!("icmp_lt {lhs}, {rhs}"), SsaInstKind::IntCmpLtImm { lhs, imm } => format!("icmp_lt_imm {lhs}, {imm}"), SsaInstKind::IntCmpGt { lhs, rhs } => format!("icmp_gt {lhs}, {rhs}"), diff --git a/src/vm/jit/native/lower.rs b/src/vm/jit/native/lower.rs index fc09e844..5185cebb 100644 --- a/src/vm/jit/native/lower.rs +++ b/src/vm/jit/native/lower.rs @@ -20,12 +20,15 @@ use crate::vm::native::{ map_iter_next_signature, map_iter_take_key_entry_address, map_iter_take_signature, map_iter_take_value_entry_address, map_set_entry_address, map_set_signature, non_yielding_host_call_entry_address, non_yielding_host_call_signature, pack_shared_signature, - restore_sparse_exit_state_entry_address, shared_array_from_buffer_entry_address, - shared_bytes_from_buffer_entry_address, shared_string_from_buffer_entry_address, - sparse_restore_exit_signature, string_binary_transform_signature, - string_contains_entry_address, string_contains_signature, string_lower_ascii_entry_address, - string_replace_literal_entry_address, string_replace_signature, - string_split_literal_entry_address, string_unary_transform_signature, value_slot_signature, + regex_match_entry_address, regex_match_signature, regex_replace_entry_address, + regex_replace_signature, restore_sparse_exit_state_entry_address, + shared_array_from_buffer_entry_address, shared_bytes_from_buffer_entry_address, + shared_string_from_buffer_entry_address, sparse_restore_exit_signature, + string_binary_transform_signature, string_contains_entry_address, string_contains_signature, + string_lower_ascii_entry_address, string_replace_literal_entry_address, + string_replace_signature, string_split_literal_entry_address, string_unary_transform_signature, + to_string_entry_address, type_of_entry_address, value_eq_entry_address, value_eq_signature, + value_len_entry_address, value_len_signature, value_slot_signature, write_heap_value_to_slot_entry_address, zero_bytes_entry_address, }; use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; @@ -100,6 +103,8 @@ fn try_compile_ssa_trace( let clone_value_sig = clone_value_signature(pointer_type, call_conv); let non_yielding_host_call_sig = non_yielding_host_call_signature(pointer_type, call_conv); let value_slot_sig = value_slot_signature(pointer_type, call_conv); + let value_eq_sig = value_eq_signature(pointer_type, call_conv); + let value_len_sig = value_len_signature(pointer_type, call_conv); let box_heap_value_sig = box_heap_value_signature(pointer_type, call_conv); let alloc_buffer_sig = alloc_buffer_signature(pointer_type, call_conv); let free_buffer_sig = free_buffer_signature(pointer_type, call_conv); @@ -115,6 +120,8 @@ fn try_compile_ssa_trace( let sparse_restore_exit_sig = sparse_restore_exit_signature(pointer_type, call_conv); let resume_linked_trace_sig = entry_signature(pointer_type, call_conv); let string_contains_sig = string_contains_signature(pointer_type, call_conv); + let regex_match_sig = regex_match_signature(pointer_type, call_conv); + let regex_replace_sig = regex_replace_signature(pointer_type, call_conv); let string_lower_sig = string_unary_transform_signature(pointer_type, call_conv); let string_replace_sig = string_replace_signature(pointer_type, call_conv); let string_split_sig = string_binary_transform_signature(pointer_type, call_conv); @@ -150,18 +157,28 @@ fn try_compile_ssa_trace( }; let string_refs = SsaStringHelperRefs { contains_ref: b.import_signature(string_contains_sig), + regex_match_ref: b.import_signature(regex_match_sig), + regex_replace_ref: b.import_signature(regex_replace_sig), replace_ref: b.import_signature(string_replace_sig), - lower_ascii_ref: b.import_signature(string_lower_sig), + lower_ascii_ref: b.import_signature(string_lower_sig.clone()), + type_of_ref: b.import_signature(string_lower_sig.clone()), + to_string_ref: b.import_signature(string_lower_sig), split_literal_ref: b.import_signature(string_split_sig), }; let string_addrs = SsaStringHelperAddrs { contains: string_contains_entry_address(), + regex_match: regex_match_entry_address(), + regex_replace: regex_replace_entry_address(), replace_literal: string_replace_literal_entry_address(), lower_ascii: string_lower_ascii_entry_address(), + type_of: type_of_entry_address(), + to_string: to_string_entry_address(), split_literal: string_split_literal_entry_address(), }; let deopt_refs = SsaDeoptHelperRefs { clone_value_ref: b.import_signature(clone_value_sig), + value_eq_ref: b.import_signature(value_eq_sig), + value_len_ref: b.import_signature(value_len_sig), non_yielding_host_call_ref: b.import_signature(non_yielding_host_call_sig), clear_value_slot_ref: b.import_signature(value_slot_sig), box_heap_value_ref: b.import_signature(box_heap_value_sig), @@ -178,6 +195,8 @@ fn try_compile_ssa_trace( }; let deopt_addrs = SsaDeoptHelperAddrs { clone_value: clone_value_to_slot_entry_address(), + value_eq: value_eq_entry_address(), + value_len: value_len_entry_address(), non_yielding_host_call: non_yielding_host_call_entry_address(), clear_value_slot: clear_value_slot_entry_address(), box_heap_value: write_heap_value_to_slot_entry_address(), @@ -425,6 +444,8 @@ struct SsaExitLowering { #[derive(Clone, Copy)] struct SsaDeoptHelperRefs { clone_value_ref: cranelift_codegen::ir::SigRef, + value_eq_ref: cranelift_codegen::ir::SigRef, + value_len_ref: cranelift_codegen::ir::SigRef, non_yielding_host_call_ref: cranelift_codegen::ir::SigRef, clear_value_slot_ref: cranelift_codegen::ir::SigRef, box_heap_value_ref: cranelift_codegen::ir::SigRef, @@ -443,6 +464,8 @@ struct SsaDeoptHelperRefs { #[derive(Clone, Copy)] struct SsaDeoptHelperAddrs { clone_value: usize, + value_eq: usize, + value_len: usize, non_yielding_host_call: usize, clear_value_slot: usize, box_heap_value: usize, @@ -461,16 +484,24 @@ struct SsaDeoptHelperAddrs { #[derive(Clone, Copy)] struct SsaStringHelperRefs { contains_ref: cranelift_codegen::ir::SigRef, + regex_match_ref: cranelift_codegen::ir::SigRef, + regex_replace_ref: cranelift_codegen::ir::SigRef, replace_ref: cranelift_codegen::ir::SigRef, lower_ascii_ref: cranelift_codegen::ir::SigRef, + type_of_ref: cranelift_codegen::ir::SigRef, + to_string_ref: cranelift_codegen::ir::SigRef, split_literal_ref: cranelift_codegen::ir::SigRef, } #[derive(Clone, Copy)] struct SsaStringHelperAddrs { contains: usize, + regex_match: usize, + regex_replace: usize, replace_literal: usize, lower_ascii: usize, + type_of: usize, + to_string: usize, split_literal: usize, } @@ -547,6 +578,7 @@ fn ssa_trace_supported(ssa: &SsaTrace) -> bool { | SsaInstKind::UnboxInt { .. } | SsaInstKind::UnboxFloat { .. } | SsaInstKind::UnboxBool { .. } + | SsaInstKind::ValueLen { .. } | SsaInstKind::StringLen { .. } | SsaInstKind::BytesLen { .. } | SsaInstKind::StringSlice { .. } @@ -555,8 +587,12 @@ fn ssa_trace_supported(ssa: &SsaTrace) -> bool { | SsaInstKind::BytesGet { .. } | SsaInstKind::BytesHas { .. } | SsaInstKind::StringContains { .. } + | SsaInstKind::RegexMatch { .. } + | SsaInstKind::RegexReplace { .. } | SsaInstKind::StringReplaceLiteral { .. } | SsaInstKind::StringLowerAscii { .. } + | SsaInstKind::TypeOf { .. } + | SsaInstKind::ToString { .. } | SsaInstKind::StringSplitLiteral { .. } | SsaInstKind::StringConcat { .. } | SsaInstKind::BytesConcat { .. } @@ -604,6 +640,7 @@ fn ssa_trace_supported(ssa: &SsaTrace) -> bool { | SsaInstKind::FloatCmpLt { .. } | SsaInstKind::FloatCmpGt { .. } | SsaInstKind::IntCmpEq { .. } + | SsaInstKind::ValueCmpEq { .. } | SsaInstKind::IntCmpLt { .. } | SsaInstKind::IntCmpLtImm { .. } | SsaInstKind::IntCmpGt { .. } @@ -834,8 +871,11 @@ fn ssa_inst_requires_owned_value_slot(kind: &SsaInstKind) -> bool { | SsaInstKind::StringSlice { .. } | SsaInstKind::BytesSlice { .. } | SsaInstKind::StringGet { .. } + | SsaInstKind::RegexReplace { .. } | SsaInstKind::StringReplaceLiteral { .. } | SsaInstKind::StringLowerAscii { .. } + | SsaInstKind::TypeOf { .. } + | SsaInstKind::ToString { .. } | SsaInstKind::StringSplitLiteral { .. } | SsaInstKind::BytesFromArrayU8 { .. } | SsaInstKind::BytesToArrayU8 { .. } @@ -1221,6 +1261,32 @@ fn lower_ssa_inst( b.switch_to_block(cont); out } + SsaInstKind::ValueLen { value } => { + let value = values[value]; + let out_slot = b.create_sized_stack_slot(StackSlotData::new( + StackSlotKind::ExplicitSlot, + std::mem::size_of::() as u32, + std::mem::align_of::().trailing_zeros() as u8, + )); + let out = b.ins().stack_addr(pointer_type, out_slot, 0); + let helper_ptr = iconst_ptr_from_addr(b, pointer_type, helper_addrs.value_len)?; + let call = b + .ins() + .call_indirect(helper_refs.value_len_ref, helper_ptr, &[value, out]); + let status = b.inst_results(call)[0]; + let success = b.create_block(); + let fail = b.create_block(); + let ok = b + .ins() + .icmp_imm(IntCC::Equal, status, i64::from(STATUS_CONTINUE)); + b.ins().brif(ok, success, &[], fail, &[]); + + b.switch_to_block(fail); + jump_with_status(b, exit_block, status); + + b.switch_to_block(success); + b.ins().stack_load(types::I64, out_slot, 0) + } SsaInstKind::StringLen { text } => { let string_data = ssa_load_heap_data_ptr(b, layout.value, values[text]); let bytes_ptr = b.ins().load( @@ -1702,6 +1768,64 @@ fn lower_ssa_inst( ssa_call_string_contains(b, pointer_type, string_refs, string_addrs, text, needle)?; b.ins().icmp_imm(IntCC::NotEqual, raw, 0) } + SsaInstKind::RegexMatch { pattern, text } => { + let pattern = values[pattern]; + let text = values[text]; + let raw = ssa_call_regex_match( + b, + pointer_type, + string_refs, + string_addrs, + vm_ptr, + pattern, + text, + )?; + let error = b.ins().icmp_imm(IntCC::SignedLessThan, raw, 0); + let failed = b.create_block(); + let matched = b.create_block(); + b.ins().brif(error, failed, &[], matched, &[]); + b.switch_to_block(failed); + let status = b.ins().iconst(types::I32, STATUS_ERROR as i64); + jump_with_status(b, exit_block, status); + b.switch_to_block(matched); + b.ins().icmp_imm(IntCC::NotEqual, raw, 0) + } + SsaInstKind::RegexReplace { + pattern, + text, + replacement, + } => { + let pattern = values[pattern]; + let text = values[text]; + let replacement = values[replacement]; + let out_raw = ssa_call_regex_replace( + b, + pointer_type, + string_refs, + string_addrs, + vm_ptr, + pattern, + text, + replacement, + )?; + let error = b.ins().icmp_imm(IntCC::Equal, out_raw, 0); + let failed = b.create_block(); + let replaced = b.create_block(); + b.ins().brif(error, failed, &[], replaced, &[]); + b.switch_to_block(failed); + let status = b.ins().iconst(types::I32, STATUS_ERROR as i64); + jump_with_status(b, exit_block, status); + b.switch_to_block(replaced); + let out = owned_value_temp_slot_addr( + b, + pointer_type, + owned_value_temps, + SsaTempValueSlotKey::Output(output.id), + )?; + clear_owned_value_temp_slot(b, pointer_type, helper_refs, helper_addrs, out)?; + ssa_store_heap_ptr_in_value(b, layout.value, out, layout.value.string_tag, out_raw); + out + } SsaInstKind::StringReplaceLiteral { text, needle, @@ -1743,6 +1867,32 @@ fn lower_ssa_inst( ssa_store_heap_ptr_in_value(b, layout.value, out, layout.value.string_tag, out_raw); out } + SsaInstKind::TypeOf { value } => { + let value = values[value]; + let out = owned_value_temp_slot_addr( + b, + pointer_type, + owned_value_temps, + SsaTempValueSlotKey::Output(output.id), + )?; + let out_raw = ssa_call_type_of(b, pointer_type, string_refs, string_addrs, value)?; + clear_owned_value_temp_slot(b, pointer_type, helper_refs, helper_addrs, out)?; + ssa_store_heap_ptr_in_value(b, layout.value, out, layout.value.string_tag, out_raw); + out + } + SsaInstKind::ToString { value } => { + let value = values[value]; + let out = owned_value_temp_slot_addr( + b, + pointer_type, + owned_value_temps, + SsaTempValueSlotKey::Output(output.id), + )?; + let out_raw = ssa_call_to_string(b, pointer_type, string_refs, string_addrs, value)?; + clear_owned_value_temp_slot(b, pointer_type, helper_refs, helper_addrs, out)?; + ssa_store_heap_ptr_in_value(b, layout.value, out, layout.value.string_tag, out_raw); + out + } SsaInstKind::StringSplitLiteral { text, delimiter } => { let text = values[text]; let delimiter = values[delimiter]; @@ -2688,6 +2838,17 @@ fn lower_ssa_inst( b.ins().fcmp(FloatCC::GreaterThan, values[lhs], values[rhs]) } SsaInstKind::IntCmpEq { lhs, rhs } => b.ins().icmp(IntCC::Equal, values[lhs], values[rhs]), + SsaInstKind::ValueCmpEq { lhs, rhs } => { + let raw = ssa_call_value_eq( + b, + pointer_type, + helper_refs, + helper_addrs, + values[lhs], + values[rhs], + )?; + b.ins().icmp_imm(IntCC::NotEqual, raw, 0) + } SsaInstKind::IntCmpLt { lhs, rhs } => { b.ins() .icmp(IntCC::SignedLessThan, values[lhs], values[rhs]) @@ -3217,6 +3378,21 @@ fn ssa_value_buffer_slot_addr( Ok(ssa_value_addr(b, pointer_type, base_ptr, index, value_size)) } +fn ssa_call_value_eq( + b: &mut FunctionBuilder, + pointer_type: cranelift_codegen::ir::Type, + helper_refs: SsaDeoptHelperRefs, + helper_addrs: SsaDeoptHelperAddrs, + lhs: cranelift_codegen::ir::Value, + rhs: cranelift_codegen::ir::Value, +) -> VmResult { + let helper_ptr = iconst_ptr_from_addr(b, pointer_type, helper_addrs.value_eq)?; + let call = b + .ins() + .call_indirect(helper_refs.value_eq_ref, helper_ptr, &[lhs, rhs]); + Ok(b.inst_results(call)[0]) +} + fn ssa_call_status_helper( b: &mut FunctionBuilder, exit_block: Block, @@ -3753,6 +3929,44 @@ fn ssa_call_string_contains( Ok(b.inst_results(call)[0]) } +fn ssa_call_regex_match( + b: &mut FunctionBuilder, + pointer_type: cranelift_codegen::ir::Type, + string_refs: SsaStringHelperRefs, + string_addrs: SsaStringHelperAddrs, + vm_ptr: cranelift_codegen::ir::Value, + pattern: cranelift_codegen::ir::Value, + text: cranelift_codegen::ir::Value, +) -> VmResult { + let helper_ptr = iconst_ptr_from_addr(b, pointer_type, string_addrs.regex_match)?; + let call = b.ins().call_indirect( + string_refs.regex_match_ref, + helper_ptr, + &[vm_ptr, pattern, text], + ); + Ok(b.inst_results(call)[0]) +} + +#[allow(clippy::too_many_arguments)] +fn ssa_call_regex_replace( + b: &mut FunctionBuilder, + pointer_type: cranelift_codegen::ir::Type, + string_refs: SsaStringHelperRefs, + string_addrs: SsaStringHelperAddrs, + vm_ptr: cranelift_codegen::ir::Value, + pattern: cranelift_codegen::ir::Value, + text: cranelift_codegen::ir::Value, + replacement: cranelift_codegen::ir::Value, +) -> VmResult { + let helper_ptr = iconst_ptr_from_addr(b, pointer_type, string_addrs.regex_replace)?; + let call = b.ins().call_indirect( + string_refs.regex_replace_ref, + helper_ptr, + &[vm_ptr, pattern, text, replacement], + ); + Ok(b.inst_results(call)[0]) +} + fn ssa_call_string_replace_literal( b: &mut FunctionBuilder, pointer_type: cranelift_codegen::ir::Type, @@ -3785,6 +3999,34 @@ fn ssa_call_string_lower_ascii( Ok(b.inst_results(call)[0]) } +fn ssa_call_type_of( + b: &mut FunctionBuilder, + pointer_type: cranelift_codegen::ir::Type, + string_refs: SsaStringHelperRefs, + string_addrs: SsaStringHelperAddrs, + value: cranelift_codegen::ir::Value, +) -> VmResult { + let helper_ptr = iconst_ptr_from_addr(b, pointer_type, string_addrs.type_of)?; + let call = b + .ins() + .call_indirect(string_refs.type_of_ref, helper_ptr, &[value]); + Ok(b.inst_results(call)[0]) +} + +fn ssa_call_to_string( + b: &mut FunctionBuilder, + pointer_type: cranelift_codegen::ir::Type, + string_refs: SsaStringHelperRefs, + string_addrs: SsaStringHelperAddrs, + value: cranelift_codegen::ir::Value, +) -> VmResult { + let helper_ptr = iconst_ptr_from_addr(b, pointer_type, string_addrs.to_string)?; + let call = b + .ins() + .call_indirect(string_refs.to_string_ref, helper_ptr, &[value]); + Ok(b.inst_results(call)[0]) +} + fn ssa_call_string_split_literal( b: &mut FunctionBuilder, pointer_type: cranelift_codegen::ir::Type, diff --git a/src/vm/jit/recorder.rs b/src/vm/jit/recorder.rs index ac8b424b..7bcdea6b 100644 --- a/src/vm/jit/recorder.rs +++ b/src/vm/jit/recorder.rs @@ -87,6 +87,7 @@ struct ValueInfo { const_float: Option, const_bool: Option, known_type: Option, + force_value_eq: bool, source_local: Option, } @@ -98,6 +99,7 @@ impl ValueInfo { const_float: None, const_bool: None, known_type: None, + force_value_eq: false, source_local: None, } } @@ -109,6 +111,7 @@ impl ValueInfo { const_float: None, const_bool: None, known_type: Some(known_type), + force_value_eq: false, source_local: None, } } @@ -120,6 +123,7 @@ impl ValueInfo { const_float: None, const_bool: None, known_type: Some(ValueType::Int), + force_value_eq: false, source_local: None, } } @@ -131,6 +135,7 @@ impl ValueInfo { const_float: value, const_bool: None, known_type: Some(ValueType::Float), + force_value_eq: false, source_local: None, } } @@ -142,6 +147,7 @@ impl ValueInfo { const_float: None, const_bool: value, known_type: Some(ValueType::Bool), + force_value_eq: false, source_local: None, } } @@ -153,6 +159,7 @@ impl ValueInfo { const_float: None, const_bool: None, known_type: Some(tag), + force_value_eq: false, source_local: None, } } @@ -170,6 +177,12 @@ impl ValueInfo { } } + fn type_name() -> Self { + let mut info = Self::tagged_typed(ValueType::String); + info.force_value_eq = true; + info + } + fn sourced_from(mut self, local: u8) -> Self { self.source_local = Some(local); self @@ -457,6 +470,7 @@ enum HeapContainerKind { #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum SpecializedBuiltinKind { + ValueLen, StringLen, BytesLen, StringSlice, @@ -465,8 +479,14 @@ enum SpecializedBuiltinKind { BytesGet, BytesHas, StringContains, + RegexMatch, + RegexReplace, StringReplaceLiteral, StringLowerAscii, + TypeOf, + TypeOfKnown(ValueType), + ToString, + ToStringIdentity, StringSplitLiteral, StringConcat, BytesConcat, @@ -729,6 +749,7 @@ pub(crate) fn record_trace( const_float: None, const_bool: None, known_type: loop_plan.stack_known_types[index], + force_value_eq: false, source_local: None, }, }); @@ -786,6 +807,7 @@ pub(crate) fn record_trace( const_float: None, const_bool: None, known_type: loop_plan.local_known_types[local], + force_value_eq: false, source_local: None, }, }); @@ -915,19 +937,27 @@ pub(crate) fn record_trace( DecodedOp::Compare { ip, opcode } => { let rhs = frame.pop()?; let lhs = frame.pop()?; - let (name, out) = - match select_numeric_compare(program, ip, opcode, lhs.info, rhs.info)? { - NumericCompareKind::Int(kind) => { - let lhs = ensure_int(&mut builder, current_block, ip, lhs)?; - let rhs = ensure_int(&mut builder, current_block, ip, rhs)?; - emit_int_compare(&mut builder, current_block, ip, kind, lhs, rhs)? - } - NumericCompareKind::Float(kind) => { - let lhs = ensure_float(&mut builder, current_block, ip, lhs)?; - let rhs = ensure_float(&mut builder, current_block, ip, rhs)?; - emit_float_compare(&mut builder, current_block, ip, kind, lhs, rhs)? - } - }; + let numeric = select_numeric_compare(program, ip, opcode, lhs.info, rhs.info); + let (name, out) = match numeric { + Ok(NumericCompareKind::Int(kind)) => { + let lhs = ensure_int(&mut builder, current_block, ip, lhs)?; + let rhs = ensure_int(&mut builder, current_block, ip, rhs)?; + emit_int_compare(&mut builder, current_block, ip, kind, lhs, rhs)? + } + Ok(NumericCompareKind::Float(kind)) => { + let lhs = ensure_float(&mut builder, current_block, ip, lhs)?; + let rhs = ensure_float(&mut builder, current_block, ip, rhs)?; + emit_float_compare(&mut builder, current_block, ip, kind, lhs, rhs)? + } + Err(_) + if opcode == OpCode::Ceq as u8 + && lhs.info.repr == SsaValueRepr::Tagged + && rhs.info.repr == SsaValueRepr::Tagged => + { + emit_value_eq(&mut builder, current_block, ip, lhs, rhs)? + } + Err(err) => return Err(err), + }; op_names.push(name.to_string()); frame.push(out); } @@ -1123,13 +1153,14 @@ pub(crate) fn record_trace( .iter() .all(|arg| arg.info.source_local != Some(local)) }); - if let Some(kind) = select_specialized_builtin_kind( + let specialized_kind = select_specialized_builtin_kind( program, ip, builtin, args[0].info, container_was_moved, - ) { + ); + if let Some(kind) = specialized_kind { let (name, out) = emit_specialized_builtin_call( &mut builder, current_block, @@ -1323,19 +1354,27 @@ fn infer_loop_header_plan( DecodedOp::Compare { ip, opcode } => { let rhs = frame.pop()?; let lhs = frame.pop()?; - match select_numeric_compare(program, ip, opcode, lhs, rhs)? { - NumericCompareKind::Int(kind) => { + match select_numeric_compare(program, ip, opcode, lhs, rhs) { + Ok(NumericCompareKind::Int(kind)) => { let lhs = expect_int_info(lhs)?; let rhs = expect_int_info(rhs)?; validate_int_compare_operands(program, ip, kind, lhs, rhs)?; frame.push(result_info_for_int_compare(kind, lhs, rhs)); } - NumericCompareKind::Float(kind) => { + Ok(NumericCompareKind::Float(kind)) => { let lhs = expect_float_info(lhs)?; let rhs = expect_float_info(rhs)?; validate_float_compare_operands(program, ip, kind, lhs, rhs)?; frame.push(result_info_for_float_compare(kind, lhs, rhs)); } + Err(_) + if opcode == OpCode::Ceq as u8 + && lhs.repr == SsaValueRepr::Tagged + && rhs.repr == SsaValueRepr::Tagged => + { + frame.push(ValueInfo::bool(None)); + } + Err(err) => return Err(err), } } DecodedOp::Call { @@ -1514,6 +1553,39 @@ fn select_numeric_binop( lhs: ValueInfo, rhs: ValueInfo, ) -> Result { + if lhs.repr == SsaValueRepr::I64 && rhs.repr == SsaValueRepr::I64 { + let kind = match opcode { + x if x == OpCode::Add as u8 => IntBinOpKind::Add, + x if x == OpCode::Sub as u8 => IntBinOpKind::Sub, + x if x == OpCode::Mul as u8 => IntBinOpKind::Mul, + x if x == OpCode::Div as u8 => IntBinOpKind::Div, + x if x == OpCode::Mod as u8 => IntBinOpKind::Mod, + x if x == OpCode::Shl as u8 => IntBinOpKind::Shl, + x if x == OpCode::Shr as u8 => IntBinOpKind::Shr, + x if x == OpCode::Lshr as u8 => IntBinOpKind::Lshr, + _ => { + return Err(TraceRecordError::UnsupportedTrace( + "SSA recorder expected a numeric binop opcode".to_string(), + )); + } + }; + return Ok(NumericBinOpKind::Int(kind)); + } + if lhs.repr == SsaValueRepr::F64 && rhs.repr == SsaValueRepr::F64 { + let kind = match opcode { + x if x == OpCode::Add as u8 => FloatBinOpKind::Add, + x if x == OpCode::Sub as u8 => FloatBinOpKind::Sub, + x if x == OpCode::Mul as u8 => FloatBinOpKind::Mul, + x if x == OpCode::Div as u8 => FloatBinOpKind::Div, + x if x == OpCode::Mod as u8 => FloatBinOpKind::Mod, + _ => { + return Err(TraceRecordError::UnsupportedTrace( + "SSA recorder expected a numeric binop opcode".to_string(), + )); + } + }; + return Ok(NumericBinOpKind::Float(kind)); + } let operand_types = operand_types(program, ip); let observed_concat = observed_concat_binop_kind(lhs, rhs); let int_like = matches!(lhs.repr, SsaValueRepr::I64 | SsaValueRepr::Tagged) @@ -1716,6 +1788,31 @@ fn select_numeric_compare( lhs: ValueInfo, rhs: ValueInfo, ) -> Result { + if lhs.force_value_eq || rhs.force_value_eq { + return Err(TraceRecordError::UnsupportedTrace( + "SSA recorder requires value equality for known non-numeric operands".to_string(), + )); + } + if lhs.repr == SsaValueRepr::I64 && rhs.repr == SsaValueRepr::I64 { + return match opcode { + x if x == OpCode::Ceq as u8 => Ok(NumericCompareKind::Int(IntCompareKind::Eq)), + x if x == OpCode::Clt as u8 => Ok(NumericCompareKind::Int(IntCompareKind::Lt)), + x if x == OpCode::Cgt as u8 => Ok(NumericCompareKind::Int(IntCompareKind::Gt)), + _ => Err(TraceRecordError::UnsupportedTrace( + "SSA recorder expected a numeric comparison opcode".to_string(), + )), + }; + } + if lhs.repr == SsaValueRepr::F64 && rhs.repr == SsaValueRepr::F64 { + return match opcode { + x if x == OpCode::Ceq as u8 => Ok(NumericCompareKind::Float(FloatCompareKind::Eq)), + x if x == OpCode::Clt as u8 => Ok(NumericCompareKind::Float(FloatCompareKind::Lt)), + x if x == OpCode::Cgt as u8 => Ok(NumericCompareKind::Float(FloatCompareKind::Gt)), + _ => Err(TraceRecordError::UnsupportedTrace( + "SSA recorder expected a numeric comparison opcode".to_string(), + )), + }; + } let operand_types = operand_types(program, ip); let int_like = matches!(lhs.repr, SsaValueRepr::I64 | SsaValueRepr::Tagged) && matches!(rhs.repr, SsaValueRepr::I64 | SsaValueRepr::Tagged); @@ -2294,6 +2391,33 @@ fn emit_bool_not( )) } +fn emit_value_eq( + builder: &mut SsaTraceBuilder, + block: super::ir::SsaBlockId, + ip: usize, + lhs: SymbolicValue, + rhs: SymbolicValue, +) -> Result<(&'static str, SymbolicValue), TraceRecordError> { + let result = builder + .append_value_inst( + block, + ip, + SsaValueRepr::Bool, + SsaInstKind::ValueCmpEq { + lhs: lhs.value.id, + rhs: rhs.value.id, + }, + ) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + Ok(( + "value_eq", + SymbolicValue { + value: result, + info: ValueInfo::bool(None), + }, + )) +} + fn emit_int_compare( builder: &mut SsaTraceBuilder, block: super::ir::SsaBlockId, @@ -2533,6 +2657,28 @@ fn select_specialized_builtin_kind( container_was_moved: bool, ) -> Option { match builtin { + BuiltinFunction::ReMatch => return Some(SpecializedBuiltinKind::RegexMatch), + BuiltinFunction::ReReplace => return Some(SpecializedBuiltinKind::RegexReplace), + BuiltinFunction::StringContains => return Some(SpecializedBuiltinKind::StringContains), + BuiltinFunction::StringLowerAscii => { + return Some(SpecializedBuiltinKind::StringLowerAscii); + } + BuiltinFunction::TypeOf => { + return Some(match container.known_type { + Some(value_type) => SpecializedBuiltinKind::TypeOfKnown(value_type), + None => SpecializedBuiltinKind::TypeOf, + }); + } + BuiltinFunction::ToString => { + return Some(if container.known_type == Some(ValueType::String) { + SpecializedBuiltinKind::ToStringIdentity + } else { + SpecializedBuiltinKind::ToString + }); + } + BuiltinFunction::StringSplitLiteral => { + return Some(SpecializedBuiltinKind::StringSplitLiteral); + } BuiltinFunction::MapIterNext => return Some(SpecializedBuiltinKind::MapIterNext), BuiltinFunction::MapIterTakeKey => return Some(SpecializedBuiltinKind::MapIterTakeKey), BuiltinFunction::MapIterTakeValue => { @@ -2548,6 +2694,7 @@ fn select_specialized_builtin_kind( | BuiltinFunction::Get | BuiltinFunction::Has | BuiltinFunction::Concat + | BuiltinFunction::StringReplaceLiteral | BuiltinFunction::Set | BuiltinFunction::ArrayPush ) { @@ -2560,7 +2707,10 @@ fn select_specialized_builtin_kind( } } else { observed_kind - }?; + }; + let Some(container_kind) = container_kind else { + return (builtin == BuiltinFunction::Len).then_some(SpecializedBuiltinKind::ValueLen); + }; match (builtin, container_kind) { (BuiltinFunction::Len, HeapContainerKind::String) => { @@ -2578,18 +2728,9 @@ fn select_specialized_builtin_kind( } (BuiltinFunction::Get, HeapContainerKind::Bytes) => Some(SpecializedBuiltinKind::BytesGet), (BuiltinFunction::Has, HeapContainerKind::Bytes) => Some(SpecializedBuiltinKind::BytesHas), - (BuiltinFunction::StringContains, HeapContainerKind::String) => { - Some(SpecializedBuiltinKind::StringContains) - } (BuiltinFunction::StringReplaceLiteral, HeapContainerKind::String) => { Some(SpecializedBuiltinKind::StringReplaceLiteral) } - (BuiltinFunction::StringLowerAscii, HeapContainerKind::String) => { - Some(SpecializedBuiltinKind::StringLowerAscii) - } - (BuiltinFunction::StringSplitLiteral, HeapContainerKind::String) => { - Some(SpecializedBuiltinKind::StringSplitLiteral) - } (BuiltinFunction::Concat, HeapContainerKind::String) => { Some(SpecializedBuiltinKind::StringConcat) } @@ -2626,6 +2767,11 @@ fn analyze_specialized_builtin_call( kind: SpecializedBuiltinKind, ) -> Result<&'static str, TraceRecordError> { match kind { + SpecializedBuiltinKind::ValueLen => { + let _ = frame.pop()?; + frame.push(ValueInfo::int(None)); + Ok("value_len") + } SpecializedBuiltinKind::StringLen => { let _ = frame.pop()?; frame.push(ValueInfo::int(None)); @@ -2674,6 +2820,19 @@ fn analyze_specialized_builtin_call( frame.push(ValueInfo::bool(None)); Ok("string_contains") } + SpecializedBuiltinKind::RegexMatch => { + let _ = frame.pop()?; + let _ = frame.pop()?; + frame.push(ValueInfo::bool(None)); + Ok("regex_match") + } + SpecializedBuiltinKind::RegexReplace => { + let _ = frame.pop()?; + let _ = frame.pop()?; + let _ = frame.pop()?; + frame.push(ValueInfo::tagged_typed(ValueType::String)); + Ok("regex_replace") + } SpecializedBuiltinKind::StringReplaceLiteral => { let _ = frame.pop()?; let _ = frame.pop()?; @@ -2686,6 +2845,22 @@ fn analyze_specialized_builtin_call( frame.push(ValueInfo::tagged_typed(ValueType::String)); Ok("string_lower_ascii") } + SpecializedBuiltinKind::TypeOf | SpecializedBuiltinKind::TypeOfKnown(_) => { + let _ = frame.pop()?; + frame.push(ValueInfo::type_name()); + Ok("type_of") + } + SpecializedBuiltinKind::ToString | SpecializedBuiltinKind::ToStringIdentity => { + let _ = frame.pop()?; + frame.push(ValueInfo::tagged_typed(ValueType::String)); + Ok( + if matches!(kind, SpecializedBuiltinKind::ToStringIdentity) { + "to_string_identity" + } else { + "to_string" + }, + ) + } SpecializedBuiltinKind::StringSplitLiteral => { let _ = frame.pop()?; let _ = frame.pop()?; @@ -2794,6 +2969,24 @@ fn emit_specialized_builtin_call( kind: SpecializedBuiltinKind, ) -> Result<(&'static str, SymbolicValue), TraceRecordError> { match kind { + SpecializedBuiltinKind::ValueLen => { + let value = frame.pop()?; + let out = builder + .append_value_inst( + block, + ip, + SsaValueRepr::I64, + SsaInstKind::ValueLen { + value: value.value.id, + }, + ) + .map(|value| SymbolicValue { + value, + info: ValueInfo::int(None), + }) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + Ok(("value_len", out)) + } SpecializedBuiltinKind::StringLen => { let text = frame.pop()?; let text = ensure_heap_ptr( @@ -3010,6 +3203,78 @@ fn emit_specialized_builtin_call( .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; Ok(("string_contains", out)) } + SpecializedBuiltinKind::RegexMatch => { + let text = ensure_heap_ptr( + builder, + block, + ip, + frame.pop()?, + HeapContainerKind::String.value_type(), + )?; + let pattern = ensure_heap_ptr( + builder, + block, + ip, + frame.pop()?, + HeapContainerKind::String.value_type(), + )?; + let out = builder + .append_value_inst( + block, + ip, + SsaValueRepr::Bool, + SsaInstKind::RegexMatch { + pattern: pattern.value.id, + text: text.value.id, + }, + ) + .map(|value| SymbolicValue { + value, + info: ValueInfo::bool(None), + }) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + Ok(("regex_match", out)) + } + SpecializedBuiltinKind::RegexReplace => { + let replacement = ensure_heap_ptr( + builder, + block, + ip, + frame.pop()?, + HeapContainerKind::String.value_type(), + )?; + let text = ensure_heap_ptr( + builder, + block, + ip, + frame.pop()?, + HeapContainerKind::String.value_type(), + )?; + let pattern = ensure_heap_ptr( + builder, + block, + ip, + frame.pop()?, + HeapContainerKind::String.value_type(), + )?; + let out = builder + .append_value_inst( + block, + ip, + SsaValueRepr::Tagged, + SsaInstKind::RegexReplace { + pattern: pattern.value.id, + text: text.value.id, + replacement: replacement.value.id, + }, + ) + .map(|value| SymbolicValue { + value, + info: ValueInfo::tagged_typed(ValueType::String), + }) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + Ok(("regex_replace", out)) + } SpecializedBuiltinKind::StringReplaceLiteral => { let replacement = ensure_heap_ptr( builder, @@ -3074,6 +3339,70 @@ fn emit_specialized_builtin_call( .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; Ok(("string_lower_ascii", out)) } + SpecializedBuiltinKind::TypeOfKnown(value_type) => { + let _ = frame.pop()?; + let type_name = match value_type { + ValueType::Null => "null", + ValueType::Int => "int", + ValueType::Float => "float", + ValueType::Bool => "bool", + ValueType::String => "string", + ValueType::Bytes => "bytes", + ValueType::Array => "array", + ValueType::Map => "map", + ValueType::Unknown => { + return Err(TraceRecordError::UnsupportedTrace( + "type_of known specialization received unknown type".to_string(), + )); + } + }; + let constant = Value::string(type_name); + let info = ValueInfo::type_name(); + let value = builder + .append_value_inst(block, ip, info.repr, SsaInstKind::Constant(constant)) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + Ok(("type_of", SymbolicValue { value, info })) + } + SpecializedBuiltinKind::ToStringIdentity => { + let value = frame.pop()?; + Ok(("to_string_identity", value)) + } + SpecializedBuiltinKind::ToString => { + let value = frame.pop()?; + let out = builder + .append_value_inst( + block, + ip, + SsaValueRepr::Tagged, + SsaInstKind::ToString { + value: value.value.id, + }, + ) + .map(|value| SymbolicValue { + value, + info: ValueInfo::tagged_typed(ValueType::String), + }) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + Ok(("to_string", out)) + } + SpecializedBuiltinKind::TypeOf => { + let value = frame.pop()?; + let out = builder + .append_value_inst( + block, + ip, + SsaValueRepr::Tagged, + SsaInstKind::TypeOf { + value: value.value.id, + }, + ) + .map(|value| SymbolicValue { + value, + info: ValueInfo::type_name(), + }) + .map_err(|err| TraceRecordError::InvalidIr(err.to_string()))?; + Ok(("type_of", out)) + } SpecializedBuiltinKind::StringSplitLiteral => { let delimiter = ensure_heap_ptr( builder, @@ -3653,6 +3982,9 @@ fn validate_int_operands( lhs: ValueInfo, rhs: ValueInfo, ) -> Result<(), TraceRecordError> { + if lhs.repr == SsaValueRepr::I64 && rhs.repr == SsaValueRepr::I64 { + return Ok(()); + } let explicit = operand_types(program, ip); let has_evidence = lhs.repr == SsaValueRepr::I64 || rhs.repr == SsaValueRepr::I64 @@ -3688,6 +4020,9 @@ fn validate_int_compare_operands( lhs: ValueInfo, rhs: ValueInfo, ) -> Result<(), TraceRecordError> { + if lhs.repr == SsaValueRepr::I64 && rhs.repr == SsaValueRepr::I64 { + return Ok(()); + } let explicit = operand_types(program, ip); let has_evidence = lhs.repr == SsaValueRepr::I64 || rhs.repr == SsaValueRepr::I64 @@ -3723,6 +4058,9 @@ fn validate_float_operands( lhs: ValueInfo, rhs: ValueInfo, ) -> Result<(), TraceRecordError> { + if lhs.repr == SsaValueRepr::F64 && rhs.repr == SsaValueRepr::F64 { + return Ok(()); + } let explicit = operand_types(program, ip); let has_evidence = lhs.repr == SsaValueRepr::F64 || rhs.repr == SsaValueRepr::F64 @@ -3985,6 +4323,7 @@ fn continue_with_frame( const_float: None, const_bool: None, known_type: value.info.known_type, + force_value_eq: value.info.force_value_eq, source_local: None, }, }); @@ -4007,6 +4346,7 @@ fn continue_with_frame( const_float: None, const_bool: None, known_type: value.info.known_type, + force_value_eq: value.info.force_value_eq, source_local: None, }, }); @@ -4173,6 +4513,49 @@ mod tests { code[start..start + 4].copy_from_slice(&target.to_le_bytes()); } + #[test] + fn unknown_len_container_uses_checked_value_len_specialization() { + let program = Program::new(Vec::new(), Vec::new()); + assert_eq!( + select_specialized_builtin_kind( + &program, + 0, + BuiltinFunction::Len, + ValueInfo::tagged(), + false, + ), + Some(SpecializedBuiltinKind::ValueLen), + ); + } + + #[test] + fn unboxed_numeric_compare_ignores_stale_operand_type_hint() { + let program = Program::new(Vec::new(), Vec::new()).with_type_map(crate::TypeMap { + operand_types: std::collections::HashMap::from([( + 7, + (ValueType::Null, ValueType::Int), + )]), + ..crate::TypeMap::default() + }); + + let lhs = ValueInfo::int(None); + let rhs = ValueInfo::int(None); + assert_eq!( + select_numeric_compare(&program, 7, OpCode::Clt as u8, lhs, rhs) + .expect("unboxed integer compare should override stale hint"), + NumericCompareKind::Int(IntCompareKind::Lt), + ); + validate_int_compare_operands(&program, 7, IntCompareKind::Lt, lhs, rhs) + .expect("unboxed integer validator should override stale hint"); + assert_eq!( + select_numeric_binop(&program, 7, OpCode::Add as u8, lhs, rhs) + .expect("unboxed integer binop should override stale hint"), + NumericBinOpKind::Int(IntBinOpKind::Add), + ); + validate_int_operands(&program, 7, IntBinOpKind::Add, lhs, rhs) + .expect("unboxed integer binop validator should override stale hint"); + } + #[test] fn records_linear_local_increment_trace_directly() { let mut bc = BytecodeBuilder::new(); diff --git a/src/vm/jit/trace.rs b/src/vm/jit/trace.rs index 2dc4c0d8..55097972 100644 --- a/src/vm/jit/trace.rs +++ b/src/vm/jit/trace.rs @@ -128,6 +128,12 @@ impl JitTrace { .filter(|dirty| **dirty) .count() as u64 } + + pub fn terminal_call_exit_ip(&self) -> Option { + (self.op_names.last().map(String::as_str) == Some("call")) + .then(|| self.ssa.exits.last().map(|exit| exit.exit_ip)) + .flatten() + } } #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/src/vm/native/bridge.rs b/src/vm/native/bridge.rs index 74927436..a9816d36 100644 --- a/src/vm/native/bridge.rs +++ b/src/vm/native/bridge.rs @@ -203,6 +203,14 @@ pub(crate) fn string_contains_entry_address() -> usize { pd_vm_native_string_contains as *const () as usize } +pub(crate) fn regex_match_entry_address() -> usize { + pd_vm_native_regex_match as *const () as usize +} + +pub(crate) fn regex_replace_entry_address() -> usize { + pd_vm_native_regex_replace as *const () as usize +} + pub(crate) fn string_replace_literal_entry_address() -> usize { pd_vm_native_string_replace_literal as *const () as usize } @@ -211,6 +219,14 @@ pub(crate) fn string_lower_ascii_entry_address() -> usize { pd_vm_native_string_lower_ascii as *const () as usize } +pub(crate) fn type_of_entry_address() -> usize { + pd_vm_native_type_of as *const () as usize +} + +pub(crate) fn to_string_entry_address() -> usize { + pd_vm_native_to_string as *const () as usize +} + pub(crate) fn string_split_literal_entry_address() -> usize { pd_vm_native_string_split_literal as *const () as usize } @@ -231,6 +247,10 @@ pub(crate) fn value_eq_entry_address() -> usize { pd_vm_native_value_eq as *const () as usize } +pub(crate) fn value_len_entry_address() -> usize { + pd_vm_native_value_len as *const () as usize +} + pub(crate) fn write_heap_value_to_slot_entry_address() -> usize { pd_vm_native_write_heap_value_to_slot as *const () as usize } @@ -404,6 +424,59 @@ pub(crate) extern "C" fn pd_vm_native_string_contains( ) } +pub(crate) extern "C" fn pd_vm_native_regex_match( + vm: *mut Vm, + pattern_ptr: *mut u8, + text_ptr: *mut u8, +) -> i32 { + let Some(vm_ref) = (unsafe { vm.as_mut() }) else { + store_bridge_error(VmError::JitNative( + "native regex-match helper received null vm pointer".to_string(), + )); + return STATUS_ERROR; + }; + let pattern = unsafe { std::mem::ManuallyDrop::new(arc_from_repr_ptr::(pattern_ptr)) }; + let text = unsafe { std::mem::ManuallyDrop::new(arc_from_repr_ptr::(text_ptr)) }; + match crate::builtins::runtime::regex::native_re_match(vm_ref, pattern.as_str(), text.as_str()) + { + Ok(matched) => i32::from(matched), + Err(err) => { + store_bridge_error(err); + STATUS_ERROR + } + } +} + +pub(crate) extern "C" fn pd_vm_native_regex_replace( + vm: *mut Vm, + pattern_ptr: *mut u8, + text_ptr: *mut u8, + replacement_ptr: *mut u8, +) -> *mut u8 { + let Some(vm_ref) = (unsafe { vm.as_mut() }) else { + store_bridge_error(VmError::JitNative( + "native regex-replace helper received null vm pointer".to_string(), + )); + return std::ptr::null_mut(); + }; + let pattern = unsafe { std::mem::ManuallyDrop::new(arc_from_repr_ptr::(pattern_ptr)) }; + let text = unsafe { std::mem::ManuallyDrop::new(arc_from_repr_ptr::(text_ptr)) }; + let replacement = + unsafe { std::mem::ManuallyDrop::new(arc_from_repr_ptr::(replacement_ptr)) }; + match crate::builtins::runtime::regex::native_re_replace( + vm_ref, + pattern.as_str(), + text.as_str(), + replacement.as_str(), + ) { + Ok(replaced) => arc_into_repr_ptr(Arc::new(replaced)), + Err(err) => { + store_bridge_error(err); + std::ptr::null_mut() + } + } +} + pub(crate) extern "C" fn pd_vm_native_string_replace_literal( text_ptr: *mut u8, needle_ptr: *mut u8, @@ -413,6 +486,9 @@ pub(crate) extern "C" fn pd_vm_native_string_replace_literal( let needle = unsafe { std::mem::ManuallyDrop::new(arc_from_repr_ptr::(needle_ptr)) }; let replacement = unsafe { std::mem::ManuallyDrop::new(arc_from_repr_ptr::(replacement_ptr)) }; + if !needle.is_empty() && !text.contains(needle.as_str()) { + return arc_into_repr_ptr(Arc::clone(&*text)); + } arc_into_repr_ptr(Arc::new( crate::builtins::runtime::core::builtin_string_replace_literal_impl( text.as_str(), @@ -429,6 +505,27 @@ pub(crate) extern "C" fn pd_vm_native_string_lower_ascii(text_ptr: *mut u8) -> * )) } +pub(crate) extern "C" fn pd_vm_native_type_of(value_ptr: *const Value) -> *mut u8 { + let name = match unsafe { &*value_ptr } { + Value::Null => "null", + Value::Int(_) => "int", + Value::Float(_) => "float", + Value::Bool(_) => "bool", + Value::String(_) => "string", + Value::Bytes(_) => "bytes", + Value::Array(_) => "array", + Value::Map(_) => "map", + }; + arc_into_repr_ptr(Arc::new(name.to_string())) +} + +pub(crate) extern "C" fn pd_vm_native_to_string(value_ptr: *const Value) -> *mut u8 { + let value = unsafe { &*value_ptr }; + arc_into_repr_ptr(Arc::new( + crate::builtins::runtime::core::builtin_to_string_impl(value), + )) +} + pub(crate) extern "C" fn pd_vm_native_string_split_literal( text_ptr: *mut u8, delimiter_ptr: *mut u8, @@ -501,6 +598,31 @@ pub(crate) extern "C" fn pd_vm_native_value_eq(lhs: *const Value, rhs: *const Va i32::from(unsafe { *lhs == *rhs }) } +pub(crate) extern "C" fn pd_vm_native_value_len(value: *const Value, out: *mut i64) -> i32 { + if value.is_null() || out.is_null() { + store_bridge_error(VmError::JitNative( + "native value-len helper received null pointer".to_string(), + )); + return STATUS_ERROR; + } + let len = match unsafe { &*value } { + Value::String(value) => value.chars().count(), + Value::Bytes(value) => value.len(), + Value::Array(value) => value.len(), + Value::Map(value) => value.len(), + _ => { + store_bridge_error(VmError::TypeMismatch("string, bytes, array, or map")); + return STATUS_ERROR; + } + }; + let Ok(len) = i64::try_from(len) else { + store_bridge_error(VmError::IntegerOverflow("len")); + return STATUS_ERROR; + }; + unsafe { out.write(len) }; + STATUS_CONTINUE +} + pub(crate) extern "C" fn pd_vm_native_write_heap_value_to_slot( dst: *mut Value, repr_ptr: *mut u8, diff --git a/src/vm/native/codegen.rs b/src/vm/native/codegen.rs index 258b40eb..7768260e 100644 --- a/src/vm/native/codegen.rs +++ b/src/vm/native/codegen.rs @@ -74,6 +74,33 @@ pub(crate) fn string_contains_signature( sig } +#[cfg(feature = "cranelift-jit")] +pub(crate) fn regex_match_signature( + pointer_type: cranelift_codegen::ir::Type, + call_conv: cranelift_codegen::isa::CallConv, +) -> Signature { + let mut sig = Signature::new(call_conv); + sig.params.push(AbiParam::new(pointer_type)); + sig.params.push(AbiParam::new(pointer_type)); + sig.params.push(AbiParam::new(pointer_type)); + sig.returns.push(AbiParam::new(types::I32)); + sig +} + +#[cfg(feature = "cranelift-jit")] +pub(crate) fn regex_replace_signature( + pointer_type: cranelift_codegen::ir::Type, + call_conv: cranelift_codegen::isa::CallConv, +) -> Signature { + let mut sig = Signature::new(call_conv); + sig.params.push(AbiParam::new(pointer_type)); + sig.params.push(AbiParam::new(pointer_type)); + sig.params.push(AbiParam::new(pointer_type)); + sig.params.push(AbiParam::new(pointer_type)); + sig.returns.push(AbiParam::new(pointer_type)); + sig +} + #[cfg(feature = "cranelift-jit")] pub(crate) fn string_unary_transform_signature( pointer_type: cranelift_codegen::ir::Type, @@ -264,6 +291,18 @@ pub(crate) fn value_eq_signature( sig } +#[cfg(feature = "cranelift-jit")] +pub(crate) fn value_len_signature( + pointer_type: cranelift_codegen::ir::Type, + call_conv: cranelift_codegen::isa::CallConv, +) -> Signature { + let mut sig = Signature::new(call_conv); + sig.params.push(AbiParam::new(pointer_type)); + sig.params.push(AbiParam::new(pointer_type)); + sig.returns.push(AbiParam::new(types::I32)); + sig +} + #[cfg(feature = "cranelift-jit")] pub(crate) fn box_heap_value_signature( pointer_type: cranelift_codegen::ir::Type, diff --git a/src/vm/native/mod.rs b/src/vm/native/mod.rs index 51014d53..0501cd92 100644 --- a/src/vm/native/mod.rs +++ b/src/vm/native/mod.rs @@ -17,12 +17,13 @@ pub(crate) use bridge::{ interrupt_helper_entry_offset, map_get_entry_address, map_has_entry_address, map_iter_next_entry_address, map_iter_take_key_entry_address, map_iter_take_value_entry_address, map_set_entry_address, non_yielding_host_call_entry_address, - restore_exit_state_entry_address, restore_sparse_exit_state_entry_address, - shared_array_from_buffer_entry_address, shared_bytes_from_buffer_entry_address, - shared_string_from_buffer_entry_address, store_bridge_error, string_contains_entry_address, - string_lower_ascii_entry_address, string_replace_literal_entry_address, - string_split_literal_entry_address, take_bridge_error, value_eq_entry_address, - write_heap_value_to_slot_entry_address, zero_bytes_entry_address, + regex_match_entry_address, regex_replace_entry_address, restore_exit_state_entry_address, + restore_sparse_exit_state_entry_address, shared_array_from_buffer_entry_address, + shared_bytes_from_buffer_entry_address, shared_string_from_buffer_entry_address, + store_bridge_error, string_contains_entry_address, string_lower_ascii_entry_address, + string_replace_literal_entry_address, string_split_literal_entry_address, take_bridge_error, + to_string_entry_address, type_of_entry_address, value_eq_entry_address, + value_len_entry_address, write_heap_value_to_slot_entry_address, zero_bytes_entry_address, }; #[cfg(feature = "cranelift-jit")] pub(crate) use codegen::{ @@ -30,9 +31,10 @@ pub(crate) use codegen::{ collection_get_signature, collection_mutation_signature, collection_predicate_signature, copy_bytes_signature, entry_signature, free_buffer_signature, helper_signature, jump_with_status, map_iter_next_signature, map_iter_take_signature, map_set_signature, - non_yielding_host_call_signature, pack_shared_signature, restore_exit_signature, - sparse_restore_exit_signature, string_binary_transform_signature, string_contains_signature, - string_replace_signature, string_unary_transform_signature, value_eq_signature, + non_yielding_host_call_signature, pack_shared_signature, regex_match_signature, + regex_replace_signature, restore_exit_signature, sparse_restore_exit_signature, + string_binary_transform_signature, string_contains_signature, string_replace_signature, + string_unary_transform_signature, value_eq_signature, value_len_signature, value_slot_signature, }; pub(crate) use exec::{ExecutableBuffer, prepare_for_execution}; diff --git a/tests/jit/jit_tests.rs b/tests/jit/jit_tests.rs index c07dc983..b2f043ea 100644 --- a/tests/jit/jit_tests.rs +++ b/tests/jit/jit_tests.rs @@ -1230,16 +1230,11 @@ fn trace_jit_preserves_local_move_semantics_across_fuel_yields() { ); if native_jit_supported() { - let snapshot = vm.jit_snapshot(); assert!( - snapshot - .attempts - .iter() - .any(|attempt| attempt.result.is_err()), - "string-equality loop should record an NYI attempt, dump:\n{}", + vm.jit_native_exec_count() > 0, + "string-equality loop should use native value_eq, dump:\n{}", vm.dump_jit_info() ); - assert_eq!(vm.jit_native_exec_count(), 0); } } @@ -1453,16 +1448,11 @@ fn trace_jit_preserves_local_move_semantics_across_epoch_yields() { Some(&Value::Int(50)), "move-heavy loop should preserve final result across epoch yields" ); - let snapshot = vm.jit_snapshot(); assert!( - snapshot - .attempts - .iter() - .any(|attempt| attempt.result.is_err()), - "string-equality loop should record an NYI attempt, dump:\n{}", + vm.jit_native_exec_count() > 0, + "string-equality loop should use native value_eq, dump:\n{}", vm.dump_jit_info() ); - assert_eq!(vm.jit_native_exec_count(), 0); } #[test] @@ -2903,7 +2893,12 @@ fn trace_jit_supports_float_and_string_loops_through_ssa() { assert_eq!(string_status, VmStatus::Halted); assert_eq!(string_vm.stack(), &[Value::string("xxx")]); let string_snapshot = string_vm.jit_snapshot(); - assert_native_ssa_call_boundary_trace(&string_vm, &string_snapshot, "string add loop"); + assert_native_ssa_specialized_trace( + &string_vm, + &string_snapshot, + "string add loop", + &["type_of", "to_string_identity", "string_concat"], + ); } #[test] @@ -3095,12 +3090,17 @@ fn trace_jit_supports_string_call_boundary_exits() { assert_eq!(status, VmStatus::Halted); assert_eq!(vm.stack(), &[Value::string("xxxxxx")]); let snapshot = vm.jit_snapshot(); - assert_native_ssa_call_boundary_trace(&vm, &snapshot, "string concat loop"); + assert_native_ssa_specialized_trace( + &vm, + &snapshot, + "string concat loop", + &["type_of", "to_string_identity", "string_concat"], + ); let bridge_hits = vm.jit_native_bridge_stats_snapshot(); assert!( bridge_hits.iter().all(|(_, count)| *count == 0), - "string concat loop should not need native helper bridges for call-boundary execution, bridge hits: {bridge_hits:?}\n{}", + "string concat loop should not need native helper bridges, bridge hits: {bridge_hits:?}\n{}", vm.dump_jit_info() ); } @@ -4630,3 +4630,184 @@ fn trace_jit_specializes_literal_string_builtins_without_call_boundary() { vm.dump_jit_info() ); } + +#[test] +fn trace_jit_specializes_loop_carried_string_builtins() { + if !native_jit_supported() { + return; + } + let source = r#" + let values: [string] = ["abc"]; + let mut text: string = (&values)[0]; + let mut i = 0; + let mut found = false; + let mut same = false; + while i < 8 { + same = (&text) == "abc"; + found = string_contains(&text, "a"); + text = string_replace_literal(text, "x", "x"); + i = i + 1; + } + found; + same; + text; + "#; + let compiled = + compile_source(source).expect("loop-carried string builtin compile should succeed"); + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + assert_eq!( + vm.run() + .expect("loop-carried string builtin jit should run"), + VmStatus::Halted + ); + assert_eq!( + vm.stack(), + &[Value::Bool(true), Value::Bool(true), Value::string("abc")] + ); + let snapshot = vm.jit_snapshot(); + assert_native_ssa_specialized_trace( + &vm, + &snapshot, + "loop-carried string builtin", + &["value_eq", "string_contains", "string_replace_literal"], + ); +} + +#[test] +fn trace_jit_preserves_dynamic_concat_type_guards() { + if !native_jit_supported() { + return; + } + let source = r#" + fn encode_map(values: map) -> string { + let keys = (&values).keys; + let mut out = ""; + for i in 0..keys.length { + let key: string = (&keys)[i]; + out = out + key + "=" + (&values)[key] + "\n"; + } + out + } + let values: map = { "a": "one", "b": "two" }; + let mut i = 0; + let mut out = ""; + while i < 8 { + out = encode_map(&values); + i = i + 1; + } + string_contains(&out, "a=one"); + "#; + let compiled = compile_source(source).expect("dynamic concat fixture should compile"); + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + let status = vm.run().unwrap_or_else(|error| { + panic!( + "dynamic concat fixture failed at ip {} stack={:?}: {error:?}\n{}", + vm.ip(), + vm.stack(), + vm.dump_jit_info(), + ) + }); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Bool(true)]); + let snapshot = vm.jit_snapshot(); + assert!( + snapshot.traces.iter().any(|trace| { + !trace.has_call + && ["type_of", "to_string_identity", "string_concat"] + .iter() + .all(|required| trace.op_names().iter().any(|op| op == required)) + }), + "dynamic concat inner trace should specialize type dispatch without calls:\n{}", + vm.dump_jit_info(), + ); +} + +#[test] +fn trace_jit_folds_known_type_of_guards_after_map_get() { + if !native_jit_supported() { + return; + } + let source = r#" + let values = { "key": "value", "number": 1 }; + let keys = ["key", "key"]; + let mut i = 0; + let mut matched = false; + while i < 8 { + let key = (&keys)[i % 2]; + matched = type((&values)[key]) == "string"; + i = i + 1; + } + matched; + "#; + let compiled = compile_source(source).expect("known type guard fixture should compile"); + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + assert_eq!( + vm.run().expect("known type guard fixture should run"), + VmStatus::Halted + ); + assert_eq!(vm.stack(), &[Value::Bool(true)]); + let snapshot = vm.jit_snapshot(); + assert_native_ssa_specialized_trace( + &vm, + &snapshot, + "known type guard after map get", + &["map_get", "type_of", "value_eq"], + ); +} + +#[test] +fn trace_jit_specializes_regex_builtins_without_call_boundary() { + if !native_jit_supported() { + return; + } + let source = r#" + use re; + let mut i = 0; + let mut matched = false; + let mut replaced = ""; + while i < 8 { + matched = re::match("(?i)^rustscript$", "RustScript"); + replaced = re::replace("\\s+", "a b", ""); + i = i + 1; + } + matched; + replaced; + "#; + let compiled = compile_source(source).expect("regex match compile should succeed"); + let mut vm = Vm::new(compiled.program.with_local_count(compiled.locals)); + vm.set_jit_config(JitConfig { + enabled: true, + hot_loop_threshold: 1, + max_trace_len: 512, + }); + assert_eq!( + vm.run().expect("regex match jit should run"), + VmStatus::Halted + ); + assert_eq!(vm.stack(), &[Value::Bool(true), Value::string("ab")]); + let snapshot = vm.jit_snapshot(); + assert_native_ssa_specialized_trace( + &vm, + &snapshot, + "regex builtin loop", + &["regex_match", "regex_replace"], + ); + assert_eq!(vm.regex_cache_entry_count(), 2); + assert_eq!(vm.regex_cache_compile_count(), 2); + assert!(vm.regex_cache_hit_count() >= 14); +}