Skip to content

JIT: Fix value profiling in optimized instrumented tiers #134160

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
EgorBo merged 3 commits into from
Sep 18, 2026
Merged

JIT: Fix value profiling in optimized instrumented tiers #134160

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b81dae7
JIT: Fix value profiling in optimized instrumented tiers
EgorBo Sep 17, 2026
0cf1ece
JIT: Intrinsify primitive Buffer.Memmove<T> calls
EgorBo Sep 17, 2026
48da544
JIT: Raise ARM64 profiled memmove limit to 64 bytes
EgorBo Sep 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/coreclr/jit/compiler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10820,7 +10820,11 @@ class Compiler
// and it works better for small sizes.
if ((type == UnrollKind::ProfiledMemcmp) || (type == UnrollKind::ProfiledMemmove))
{
#ifdef TARGET_ARM64
threshold = maxRegSize * (type == UnrollKind::ProfiledMemmove ? 4 : 2);
#else
threshold = maxRegSize * 2;
#endif
}

return threshold;
Expand Down
105 changes: 83 additions & 22 deletions src/coreclr/jit/importercalls.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1553,6 +1553,25 @@ var_types Compiler::impImportCall(OPCODE opcode,
}

DONE_CALL:
// Collect value profiles in optimized instrumented tiers too, before wrapping inline candidates.
if (opts.IsInstrumented() && JitConfig.JitProfileValues() && call->IsCall() && call->AsCall()->IsSpecialIntrinsic())
{
const NamedIntrinsic ni = lookupNamedIntrinsic(call->AsCall()->gtCallMethHnd);
if ((ni == NI_System_SpanHelpers_Memmove) || (ni == NI_System_SpanHelpers_SequenceEqual))
Comment thread
EgorBo marked this conversation as resolved.
{
assert(!call->AsCall()->IsGuardedDevirtualizationCandidate());

// Reuse inline-candidate info: it derives from the probe info and shares the same union slot.
HandleHistogramProfileCandidateInfo* pInfo =
call->AsCall()->IsInlineCandidate() ? call->AsCall()->GetSingleInlineCandidateInfo()
: new (this, CMK_Inlining) HandleHistogramProfileCandidateInfo;
pInfo->ilOffset = rawILOffset;
pInfo->probeIndex = 0;
call->AsCall()->gtHandleHistogramProfileCandidateInfo = pInfo;
compCurBB->SetFlags(BBF_HAS_VALUE_PROFILE);
}
}

// Push or append the result of the call
if (callRetTyp == TYP_VOID)
{
Expand All @@ -1565,20 +1584,10 @@ var_types Compiler::impImportCall(OPCODE opcode,
else if (JitConfig.JitProfileValues() && call->IsCall() &&
call->AsCall()->IsSpecialIntrinsic(this, NI_System_SpanHelpers_Memmove))
{
if (opts.IsOptimizedWithProfile())
if (opts.IsOptimizedWithProfile() && !opts.IsInstrumented())
{
call = impDuplicateWithProfiledArg(call->AsCall(), rawILOffset);
}
else if (opts.IsInstrumented())
{
// We might want to instrument it for optimized versions too, but we don't currently.
HandleHistogramProfileCandidateInfo* pInfo =
new (this, CMK_Inlining) HandleHistogramProfileCandidateInfo;
pInfo->ilOffset = rawILOffset;
pInfo->probeIndex = 0;
call->AsCall()->gtHandleHistogramProfileCandidateInfo = pInfo;
compCurBB->SetFlags(BBF_HAS_VALUE_PROFILE);
}
impAppendTree(call, CHECK_SPILL_ALL, impCurStmtDI);
}
else
Expand Down Expand Up @@ -1721,7 +1730,7 @@ var_types Compiler::impImportCall(OPCODE opcode,
if (JitConfig.JitProfileValues() && call->IsCall() &&
call->AsCall()->IsSpecialIntrinsic(this, NI_System_SpanHelpers_SequenceEqual))
{
if (opts.IsOptimizedWithProfile())
if (opts.IsOptimizedWithProfile() && !opts.IsInstrumented())
{
call = impDuplicateWithProfiledArg(call->AsCall(), rawILOffset);
if (call->OperIs(GT_QMARK))
Expand All @@ -1732,16 +1741,6 @@ var_types Compiler::impImportCall(OPCODE opcode,
call = gtNewLclvNode(tmp, call->TypeGet());
}
}
else if (opts.IsInstrumented())
{
// We might want to instrument it for optimized versions too, but we don't currently.
HandleHistogramProfileCandidateInfo* pInfo =
new (this, CMK_Inlining) HandleHistogramProfileCandidateInfo;
pInfo->ilOffset = rawILOffset;
pInfo->probeIndex = 0;
call->AsCall()->gtHandleHistogramProfileCandidateInfo = pInfo;
compCurBB->SetFlags(BBF_HAS_VALUE_PROFILE);
}
}
}

Expand Down Expand Up @@ -3694,6 +3693,7 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd,
case NI_System_Type_get_TypeHandle:
case NI_System_RuntimeType_get_TypeHandle:
case NI_System_RuntimeTypeHandle_ToIntPtr:
case NI_System_Buffer_Memmove:

// This one is not simple, but it will help us
// to avoid some unnecessary boxing
Expand Down Expand Up @@ -5521,6 +5521,60 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd,
break;
}

case NI_System_Buffer_Memmove:
{
// Convert Buffer.Memmove<T>(ref T dst, ref T src, count) to
// SpanHelpers.Memmove(ref byte dst, ref byte src, count * sizeof(T)).
if (sig->sigInst.methInstCount != 1)
{
break;
}

const CorInfoType primitiveType =
info.compCompHnd->getTypeForPrimitiveValueClass(sig->sigInst.methInst[0]);
if (primitiveType == CORINFO_TYPE_UNDEF)
Comment thread
EgorBo marked this conversation as resolved.
{
break;
}

const var_types elementType = JITtype2varType(primitiveType);
if (!varTypeIsArithmetic(elementType))
{
break;
}

// TODO: Rename CORINFO_HELP_MEMCPY to CORINFO_HELP_MEMMOVE to reflect its overlap-safe semantics.
CORINFO_METHOD_HANDLE memmoveHnd = NO_METHOD_HANDLE;
info.compCompHnd->getHelperFtn(CORINFO_HELP_MEMCPY, nullptr, &memmoveHnd);
if (memmoveHnd == NO_METHOD_HANDLE)
{
break;
}

assert(sig->numArgs == 3);
assert(sig->retType == CORINFO_TYPE_VOID);

GenTree* length = impImplicitIorI4Cast(impPopStack().val, TYP_I_IMPL, /* zeroExtend */ true);
GenTree* source = impPopStack().val;
GenTree* destination = impPopStack().val;
const unsigned elementSize = genTypeSize(elementType);
if (elementSize != 1)
{
length =
gtFoldExpr(gtNewOperNode(GT_MUL, TYP_I_IMPL, length, gtNewIconNode(elementSize, TYP_I_IMPL)));
}

// Keep the byte-length probe at this call site, rather than inside the generic wrapper.
GenTreeCall* memmove = gtNewUserCallNode(memmoveHnd, TYP_VOID, impCurStmtDI);
memmove->gtArgs.PushBack(this, NewCallArg::Primitive(destination));
memmove->gtArgs.PushBack(this, NewCallArg::Primitive(source));
memmove->gtArgs.PushBack(this, NewCallArg::Primitive(length));
memmove->gtCallMoreFlags |= GTF_CALL_M_SPECIAL_INTRINSIC;
gtUpdateNodeSideEffects(memmove);
retNode = memmove;
break;
}

case NI_System_Text_UTF8Encoding_UTF8EncodingSealed_ReadUtf8:
case NI_System_SpanHelpers_SequenceEqual:
case NI_System_SpanHelpers_ClearWithoutReferences:
Expand Down Expand Up @@ -11749,6 +11803,13 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method)
result = NI_System_BitConverter_Int64BitsToDouble;
}
}
else if (strcmp(className, "Buffer") == 0)
{
if (strcmp(methodName, "Memmove") == 0)
{
result = NI_System_Buffer_Memmove;
}
}
break;
}

Expand Down
1 change: 1 addition & 0 deletions src/coreclr/jit/namedintrinsiclist.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ enum NamedIntrinsic : unsigned short
NI_System_BitConverter_Int64BitsToDouble,
NI_System_BitConverter_SingleToInt32Bits,

NI_System_Buffer_Memmove,
NI_System_SpanHelpers_Memmove,

NI_System_Half_FusedMultiplyAdd,
Expand Down
Loading