JIT: Type implicit byref args as TYP_I_IMPL

docs/design/coreclr/botr/clr-abi.md

@@ -141,7 +141,7 @@ Structures (value types in IL parlance) that are declared to be passed by value
 
 System V AMD64, x86 and ARM32 do not use this convention. For JIT generated code, it follows the native ABI where the passed-in reference is a pointer to a compiler generated temp local on the stack.
 
-Since .NET 12, implicit-byref argument storage must be outside the GC heap on all architectures that use this convention. Runtime callers may use explicitly GC-protected native memory instead of the stack. The caller is responsible for making a writable copy as required by by-value semantics and for reporting any GC references in that copy. Callees may omit write barriers when modifying the argument. This does not apply to explicit byref parameters or the `this` pointer of a value type, and does not remove aliasing caused by taking the argument's address within the callee. Implicit-byref argument pointers are still represented and reported as GC byrefs.
+Since .NET 12, implicit-byref argument storage must be outside the GC heap on all architectures that use this convention. Runtime callers may use explicitly GC-protected native memory instead of the stack. The caller is responsible for making a writable copy as required by by-value semantics and for reporting any GC references in that copy. Callees may omit write barriers when modifying the argument and need not report implicit-byref argument pointers to the GC. This does not apply to explicit byref parameters or the `this` pointer of a value type, and does not remove aliasing caused by taking the argument's address within the callee.
 
 ## RISC-V only: structs passed/returned according to hardware floating-point calling convention
 

src/coreclr/jit/compiler.h

@@ -1146,7 +1146,7 @@ class LclVarDsc
     ClassLayout* GetLayout() const
     {
 #if FEATURE_IMPLICIT_BYREFS
-        assert(varTypeIsStruct(TypeGet()) || (lvIsImplicitByRef && TypeIs(TYP_BYREF)));
+        assert(varTypeIsStruct(TypeGet()) || (lvIsImplicitByRef && TypeIs(TYP_I_IMPL, TYP_BYREF)));
 #else
         assert(varTypeIsStruct(TypeGet()));
 #endif
@@ -4905,7 +4905,8 @@ class Compiler
 
     bool lvaIsArgAccessedViaVarArgsCookie(unsigned lclNum);
 
-    bool lvaIsImplicitByRefLocal(unsigned lclNum) const;
+    bool      lvaIsImplicitByRefLocal(unsigned lclNum) const;
+    var_types lvaGetImplicitByRefParamType();
     bool lvaIsLocalImplicitlyAccessedByRef(unsigned lclNum) const;
 
     // If the local is a TYP_STRUCT, get/set a class handle describing it

src/coreclr/jit/fgdiagnostic.cpp

@@ -3378,7 +3378,7 @@ void Compiler::fgDebugCheckFlagsAndTypes(GenTree* tree, BasicBlock* block)
         case GT_STORE_LCL_FLD:
             assert((tree->gtFlags & GTF_VAR_DEF) != 0);
             if (!fgImplicitByRefLclFldsStale || !tree->OperIs(GT_STORE_LCL_FLD) ||
-                !lvaGetDesc(tree->AsLclFld())->TypeIs(TYP_BYREF) ||
+                !lvaGetDesc(tree->AsLclFld())->TypeIs(lvaGetImplicitByRefParamType()) ||
                 !lvaIsImplicitByRefLocal(tree->AsLclFld()->GetLclNum()))
             {
                 assert(((tree->gtFlags & GTF_VAR_USEASG) != 0) == tree->IsPartialLclFld(this));

src/coreclr/jit/flowgraph.cpp

@@ -927,7 +927,11 @@ bool Compiler::fgAddrCouldBeNull(GenTree* addr)
             return !addr->IsBoxedValue();
 
         case GT_LCL_VAR:
-            return !lvaIsImplicitByRefLocal(addr->AsLclVar()->GetLclNum());
+        {
+            // Implicit byrefs and return buffers always point to caller-allocated storage.
+            const unsigned lclNum = addr->AsLclVar()->GetLclNum();
+            return !lvaIsImplicitByRefLocal(lclNum) && (lclNum != impInlineRoot()->info.compRetBuffArg);
+        }
 
         case GT_COMMA:
             return fgAddrCouldBeNull(addr->AsOp()->gtOp2);
@@ -1020,6 +1024,12 @@ bool Compiler::fgAddrCouldBeHeap(GenTree* addr)
         return false;
     }
 
+    if (op->OperIs(GT_LCL_VAR) && lvaIsImplicitByRefLocal(op->AsLclVar()->GetLclNum()))
+    {
+        // Implicit byrefs are known to not be on the heap
+        return false;
+    }
+
     return true;
 }
 

src/coreclr/jit/gcinfo.cpp

@@ -265,8 +265,8 @@ GCInfo::WriteBarrierForm GCInfo::gcIsWriteBarrierCandidate(GenTreeStoreInd* stor
 // gcWriteBarrierFormFromTargetAddress: Get the write barrier form from address.
 //
 // This method deconstructs "tgtAddr" to find out if it is "based on" a TYP_REF
-// address, allowing an unchecked barrier to be used, or an address of a local,
-// in which case no barrier is needed.
+// address, allowing an unchecked barrier to be used, or a non-heap address (e.g.
+// a local or an implicit byref), in which case no barrier is needed.
 //
 // Arguments:
 //    tgtAddr - The target address of the store
@@ -276,9 +276,9 @@ GCInfo::WriteBarrierForm GCInfo::gcIsWriteBarrierCandidate(GenTreeStoreInd* stor
 //
 GCInfo::WriteBarrierForm GCInfo::gcWriteBarrierFormFromTargetAddress(GenTree* tgtAddr)
 {
-    if (tgtAddr->OperIs(GT_LCL_ADDR))
+    if (!m_compiler->fgAddrCouldBeHeap(tgtAddr->gtSkipReloadOrCopy()))
     {
-        // No need for a GC barrier when writing to a local variable.
+        // No need for a GC barrier when writing to a local variable or other non-heap storage.
         return GCInfo::WBF_NoBarrier;
     }
 

src/coreclr/jit/gentree.cpp

@@ -10330,7 +10330,7 @@ GenTreeLclVar* Compiler::gtNewLclvNode(unsigned lnum, var_types type DEBUGARG(IL
     if (varTypeIsStruct(type))
     {
         // Make an exception for implicit by-ref parameters during global morph, since
-        // their lvType has been updated to byref but their appearances have not yet all
+        // their lvType has been updated to a pointer but their appearances have not yet all
         // been rewritten and so may have struct type still.
         LclVarDsc* varDsc = lvaGetDesc(lnum);
 
@@ -10340,7 +10340,7 @@ GenTreeLclVar* Compiler::gtNewLclvNode(unsigned lnum, var_types type DEBUGARG(IL
         simd12ToSimd16Widening = (type == TYP_SIMD16) && (varDsc->lvType == TYP_SIMD12);
 #endif // FEATURE_SIMD
         assert((type == varDsc->lvType) || simd12ToSimd16Widening ||
-               (lvaIsImplicitByRefLocal(lnum) && fgGlobalMorph && (varDsc->lvType == TYP_BYREF)));
+               (lvaIsImplicitByRefLocal(lnum) && fgGlobalMorph && (varDsc->lvType == lvaGetImplicitByRefParamType())));
     }
     GenTreeLclVar* node = new (this, GT_LCL_VAR) GenTreeLclVar(GT_LCL_VAR, type, lnum DEBUGARG(offs));
 
@@ -10376,10 +10376,10 @@ GenTreeLclVar* Compiler::gtNewLclLNode(unsigned lnum, var_types type DEBUGARG(IL
     if (varTypeIsStruct(type))
     {
         // Make an exception for implicit by-ref parameters during global morph, since
-        // their lvType has been updated to byref but their appearances have not yet all
+        // their lvType has been updated to a pointer but their appearances have not yet all
         // been rewritten and so may have struct type still.
-        assert(type == lvaTable[lnum].lvType ||
-               (lvaIsImplicitByRefLocal(lnum) && fgGlobalMorph && (lvaTable[lnum].lvType == TYP_BYREF)));
+        assert(type == lvaTable[lnum].lvType || (lvaIsImplicitByRefLocal(lnum) && fgGlobalMorph &&
+                                                 (lvaTable[lnum].lvType == lvaGetImplicitByRefParamType())));
     }
     // This local variable node may later get transformed into a large node
     assert(GenTree::s_gtNodeSizes[LargeOpOpcode()] > GenTree::s_gtNodeSizes[GT_LCL_VAR]);

src/coreclr/jit/importer.cpp

@@ -12294,8 +12294,8 @@ bool Compiler::impFoldAwaitedTopOfStack()
 //
 // Remarks:
 //   The memory pointed to by implicit byrefs is owned by the callee but
-//   usually exists on the caller's frame (or on the heap for some reflection
-//   invoke scenarios). This function helps catch situations where the caller
+//   usually exists on the caller's frame (or in GC-protected native memory for
+//   some runtime invoke scenarios). This function helps catch situations where the caller
 //   reads from the memory after the invocation, for example due to a bug in
 //   the JIT's own last-use copy elision for implicit byrefs.
 //

src/coreclr/jit/lclvars.cpp

@@ -2492,11 +2492,12 @@ bool Compiler::lvaIsArgAccessedViaVarArgsCookie(unsigned lclNum)
 // lvaIsImplicitByRefLocal: Is the local an "implicit byref" parameter?
 //
 // We term structs passed via pointers to shadow copies "implicit byrefs".
-// They are used on Windows x64, ARM64, LoongArch64 and RISC-V; see
+// They are used on Windows x64, ARM64, LoongArch64, RISC-V and WebAssembly; see
 // "By-value value types passed by reference" in clr-abi.md for the exact rules.
 //
 // The shadow copies must be outside the GC heap, so stores into them do not
-// require write barriers. The caller is responsible for GC reporting their contents.
+// require write barriers and the pointers need not be GC reported (see
+// lvaGetImplicitByRefParamType). The caller is responsible for GC reporting their contents.
 //
 // Arguments:
 //    lclNum - The local in question
@@ -2512,13 +2513,27 @@ bool Compiler::lvaIsImplicitByRefLocal(unsigned lclNum) const
     {
         assert(varDsc->lvIsParam);
 
-        assert(varTypeIsStruct(varDsc) || varDsc->TypeIs(TYP_BYREF));
+        assert(varTypeIsStruct(varDsc) || varDsc->TypeIs(TYP_I_IMPL, TYP_BYREF));
         return true;
     }
 #endif // FEATURE_IMPLICIT_BYREFS
     return false;
 }
 
+//------------------------------------------------------------------------
+// lvaGetImplicitByRefParamType: Get the type implicit byref parameters are
+//    retyped to by fgRetypeImplicitByRefArgs.
+//
+// Return Value:
+//    TYP_I_IMPL since the storage is never on the GC heap. Async methods use
+//    TYP_BYREF so that derived addresses are not kept live across suspension
+//    points, as the storage is different after resumption.
+//
+var_types Compiler::lvaGetImplicitByRefParamType()
+{
+    return impInlineRoot()->compIsAsync() ? TYP_BYREF : TYP_I_IMPL;
+}
+
 //------------------------------------------------------------------------
 // lvaIsLocalImplicitlyAccessedByRef: Will this local be accessed indirectly?
 //

src/coreclr/jit/morph.cpp

@@ -3356,8 +3356,8 @@ GenTree* Compiler::fgMorphExpandImplicitByRefArg(GenTreeLclVarCommon* lclNode)
     {
         // The SIMD transformation to coalesce contiguous references to SIMD vector fields will re-invoke
         // the traversal to mark address-taken locals. So, we may encounter a tree that has already been
-        // transformed to TYP_BYREF. If we do, leave it as-is.
-        if (lclNode->OperIs(GT_LCL_VAR) && lclNode->TypeIs(TYP_BYREF))
+        // transformed. If we do, leave it as-is.
+        if (lclNode->OperIs(GT_LCL_VAR) && lclNode->TypeIs(varDsc->TypeGet()))
         {
             return nullptr;
         }
@@ -3419,7 +3419,10 @@ GenTree* Compiler::fgMorphExpandImplicitByRefArg(GenTreeLclVarCommon* lclNode)
     JITDUMP("\nRewriting an implicit by-ref parameter reference:\n");
     DISPTREE(lclNode);
 
-    lclNode->ChangeType(TYP_BYREF);
+    const var_types ptrType = lvaGetImplicitByRefParamType();
+    assert(lvaGetDesc(newLclNum)->TypeIs(ptrType));
+
+    lclNode->ChangeType(ptrType);
     lclNode->ChangeOper(GT_LCL_VAR);
     lclNode->SetLclNum(newLclNum);
     lclNode->SetAllEffectsFlags(GTF_EMPTY); // Implicit by-ref parameters cannot be address-exposed.
@@ -3432,7 +3435,7 @@ GenTree* Compiler::fgMorphExpandImplicitByRefArg(GenTreeLclVarCommon* lclNode)
     GenTree* addrNode = lclNode;
     if (offset != 0)
     {
-        addrNode = gtNewOperNode(GT_ADD, TYP_BYREF, addrNode, gtNewIconNode(offset, TYP_I_IMPL));
+        addrNode = gtNewOperNode(GT_ADD, ptrType, addrNode, gtNewIconNode(offset, TYP_I_IMPL));
     }
 
     // Note: currently, we have to conservatively treat all indirections off of implicit byrefs
@@ -9077,9 +9080,8 @@ GenTree* Compiler::fgOptimizeEqualityComparisonWithConst(GenTreeOp* cmp)
         }
 
         GenTree* andOpOp1 = andOp->gtGetOp1();
-        // Note the operand's type does not have to match the AND's; morph can retype nodes to
-        // TYP_BYREF, e. g. when rewriting references to implicit byref parameters. Such operands
-        // cannot be narrowed, but can still be cast to TYP_INT below.
+        // Note the operand's type does not have to match the AND's; e. g. it can be TYP_BYREF.
+        // Such operands cannot be narrowed, but can still be cast to TYP_INT below.
         //
         // Now we narrow the first operand of AND to int.
         if (andOpOp1->TypeIs(TYP_LONG) && optNarrowTree(andOpOp1, TYP_LONG, TYP_INT, ValueNumPair(), false))
@@ -15590,7 +15592,7 @@ PhaseStatus Compiler::fgRetypeImplicitByRefArgs()
                     // The first BB should already be a valid insertion point,
                     // which is a precondition for this phase when optimizing.
                     assert(fgFirstBB->bbPreds == nullptr);
-                    GenTree* addr  = gtNewLclvNode(lclNum, TYP_BYREF);
+                    GenTree* addr  = gtNewLclvNode(lclNum, lvaGetImplicitByRefParamType());
                     GenTree* data  = varDsc->TypeIs(TYP_STRUCT) ? gtNewBlkIndir(varDsc->GetLayout(), addr)
                                                                 : gtNewIndir(varDsc->TypeGet(), addr);
                     GenTree* store = gtNewStoreLclVarNode(newLclNum, data);
@@ -15661,8 +15663,8 @@ PhaseStatus Compiler::fgRetypeImplicitByRefArgs()
                 assert(varDsc->lvFieldLclStart == 0);
             }
 
-            // Since the parameter in this position is really a pointer, its type is TYP_BYREF.
-            varDsc->lvType = TYP_BYREF;
+            // The parameter in this position is really a pointer to storage outside the GC heap.
+            varDsc->lvType = lvaGetImplicitByRefParamType();
 
             // The struct parameter may have had its address taken, but the pointer parameter
             // cannot -- any uses of the struct parameter's address are uses of the pointer
@@ -15678,7 +15680,8 @@ PhaseStatus Compiler::fgRetypeImplicitByRefArgs()
 
             if (verbose)
             {
-                printf("Changing the lvType for struct parameter V%02d to TYP_BYREF.\n", lclNum);
+                printf("Changing the lvType for struct parameter V%02d to %s.\n", lclNum,
+                       varTypeName(varDsc->TypeGet()));
             }
 #endif // DEBUG
         }

src/coreclr/jit/scopeinfo.cpp

@@ -404,21 +404,12 @@ void CodeGenInterface::siVarLoc::siFillStackVarLoc(
         case TYP_MASK:
 #endif // FEATURE_MASKED_HW_INTRINSICS
 #if FEATURE_IMPLICIT_BYREFS
-            // In the AMD64 ABI we are supposed to pass a struct by reference when its
-            // size is not 1, 2, 4 or 8 bytes in size. During fgMorph, the compiler modifies
-            // the IR to comply with the ABI and therefore changes the type of the lclVar
-            // that holds the struct from TYP_STRUCT to TYP_BYREF but it gives us a hint that
-            // this is still a struct by setting the lvIsImplicitByref flag.
-            // The same is true for ARM64 and structs > 16 bytes.
-            //
-            // See lvaSetStruct for further detail.
-            //
-            // Now, the VM expects a special enum for these type of local vars: VLT_STK_BYREF
-            // to accommodate for this situation.
+            // Implicit byref parameters are retyped by morph from structs to pointers
+            // to the struct storage; describe that indirection to the debugger.
             if (varDsc->lvIsImplicitByRef)
             {
                 assert(varDsc->lvIsParam);
-                assert(varDsc->lvType == TYP_BYREF);
+                assert(varDsc->TypeIs(TYP_I_IMPL, TYP_BYREF));
                 this->vlType = VLT_STK_BYREF;
             }
             else