[LoongArch64] Add atomic instruction implementation for LoongArch64

src/coreclr/System.Private.CoreLib/src/System/Threading/Interlocked.CoreCLR.cs

@@ -52,7 +52,7 @@ public static long Decrement(ref long location) =>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int Exchange(ref int location1, int value)
         {
-#if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64
+#if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 || TARGET_LOONGARCH64
             return Exchange(ref location1, value); // Must expand intrinsic
 #else
             if (Unsafe.IsNullRef(ref location1))
@@ -73,7 +73,7 @@ public static int Exchange(ref int location1, int value)
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static long Exchange(ref long location1, long value)
         {
-#if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64
+#if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 || TARGET_LOONGARCH64
             return Exchange(ref location1, value); // Must expand intrinsic
 #else
             if (Unsafe.IsNullRef(ref location1))
@@ -116,7 +116,7 @@ public static long Exchange(ref long location1, long value)
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int CompareExchange(ref int location1, int value, int comparand)
         {
-#if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64
+#if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 || TARGET_LOONGARCH64
             return CompareExchange(ref location1, value, comparand); // Must expand intrinsic
 #else
             if (Unsafe.IsNullRef(ref location1))
@@ -137,7 +137,7 @@ public static int CompareExchange(ref int location1, int value, int comparand)
         [Intrinsic]
         internal static unsafe int CompareExchange(int* location1, int value, int comparand)
         {
-#if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64
+#if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 || TARGET_LOONGARCH64
             return CompareExchange(location1, value, comparand); // Must expand intrinsic
 #else
             Debug.Assert(location1 != null);
@@ -158,7 +158,7 @@ internal static unsafe int CompareExchange(int* location1, int value, int compar
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static long CompareExchange(ref long location1, long value, long comparand)
         {
-#if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64
+#if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 || TARGET_LOONGARCH64
             return CompareExchange(ref location1, value, comparand); // Must expand intrinsic
 #else
             if (Unsafe.IsNullRef(ref location1))
@@ -212,7 +212,7 @@ public static long Add(ref long location1, long value) =>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static int ExchangeAdd(ref int location1, int value)
         {
-#if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64
+#if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 || TARGET_LOONGARCH64
             return ExchangeAdd(ref location1, value); // Must expand intrinsic
 #else
             if (Unsafe.IsNullRef(ref location1))
@@ -228,7 +228,7 @@ private static int ExchangeAdd(ref int location1, int value)
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static long ExchangeAdd(ref long location1, long value)
         {
-#if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64
+#if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 || TARGET_LOONGARCH64
             return ExchangeAdd(ref location1, value); // Must expand intrinsic
 #else
             if (Unsafe.IsNullRef(ref location1))

src/coreclr/inc/clrconfigvalues.h

@@ -719,6 +719,9 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableRiscV64Zba,             W("EnableRiscV64
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableRiscV64Zbb,             W("EnableRiscV64Zbb"),          1, "Allows RiscV64 Zbb hardware intrinsics to be disabled")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableRiscV64Zbs,             W("EnableRiscV64Zbs"),          1, "Allows RiscV64 Zbs hardware intrinsics to be disabled")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableRiscV64Zicond,          W("EnableRiscV64Zicond"),       1, "Allows RiscV64 Zicond hardware intrinsics to be disabled")
+#elif defined(TARGET_LOONGARCH64)
+RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableLoongArch64LAM_BH,      W("EnableLoongArch64LAM_BH"),   1, "Allows LoongArch64 LAM_BH+ hardware intrinsics to be disabled")
+RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableLoongArch64LAM_CAS,     W("EnableLoongArch64LAM_CAS"),  1, "Allows LoongArch64 LAM_CAS+ hardware intrinsics to be disabled")
 #endif
 
 ///

src/coreclr/inc/corinfoinstructionset.h

@@ -57,6 +57,10 @@ enum CORINFO_InstructionSet
     InstructionSet_SveSha3_Arm64=38,
     InstructionSet_SveSm4_Arm64=39,
 #endif // TARGET_ARM64
+#ifdef TARGET_LOONGARCH64
+    InstructionSet_LAM_BH=1,
+    InstructionSet_LAM_CAS=2,
+#endif // TARGET_LOONGARCH64
 #ifdef TARGET_RISCV64
     InstructionSet_RiscV64Base=1,
     InstructionSet_Zba=2,
@@ -290,6 +294,8 @@ struct CORINFO_InstructionSetFlags
         if (HasInstructionSet(InstructionSet_SveSm4))
             AddInstructionSet(InstructionSet_SveSm4_Arm64);
 #endif // TARGET_ARM64
+#ifdef TARGET_LOONGARCH64
+#endif // TARGET_LOONGARCH64
 #ifdef TARGET_RISCV64
 #endif // TARGET_RISCV64
 #ifdef TARGET_WASM
@@ -450,6 +456,8 @@ inline CORINFO_InstructionSetFlags EnsureInstructionSetFlagsAreValid(CORINFO_Ins
         if (resultflags.HasInstructionSet(InstructionSet_SveSm4) && !resultflags.HasInstructionSet(InstructionSet_Sm4))
             resultflags.RemoveInstructionSet(InstructionSet_SveSm4);
 #endif // TARGET_ARM64
+#ifdef TARGET_LOONGARCH64
+#endif // TARGET_LOONGARCH64
 #ifdef TARGET_RISCV64
         if (resultflags.HasInstructionSet(InstructionSet_Zbb) && !resultflags.HasInstructionSet(InstructionSet_RiscV64Base))
             resultflags.RemoveInstructionSet(InstructionSet_Zbb);
@@ -752,6 +760,12 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet)
         case InstructionSet_Cssc :
             return "Cssc";
 #endif // TARGET_ARM64
+#ifdef TARGET_LOONGARCH64
+        case InstructionSet_LAM_BH :
+            return "LAM_BH";
+        case InstructionSet_LAM_CAS :
+            return "LAM_CAS";
+#endif // TARGET_LOONGARCH64
 #ifdef TARGET_RISCV64
         case InstructionSet_RiscV64Base :
             return "RiscV64Base";
@@ -968,6 +982,10 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst
         case READYTORUN_INSTRUCTION_SveSm4: return InstructionSet_SveSm4;
         case READYTORUN_INSTRUCTION_Cssc: return InstructionSet_Cssc;
 #endif // TARGET_ARM64
+#ifdef TARGET_LOONGARCH64
+        case READYTORUN_INSTRUCTION_LAM_BH: return InstructionSet_LAM_BH;
+        case READYTORUN_INSTRUCTION_LAM_CAS: return InstructionSet_LAM_CAS;
+#endif // TARGET_LOONGARCH64
 #ifdef TARGET_RISCV64
         case READYTORUN_INSTRUCTION_RiscV64Base: return InstructionSet_RiscV64Base;
         case READYTORUN_INSTRUCTION_Zba: return InstructionSet_Zba;

src/coreclr/inc/jiteeversionguid.h

@@ -37,11 +37,11 @@
 
 #include <minipal/guid.h>
 
-constexpr GUID JITEEVersionIdentifier = { /* 65743063-e8fa-41d4-9496-c436974c00f5 */
-    0x65743063,
-    0xe8fa,
-    0x41d4,
-    {0x94, 0x96, 0xc4, 0x36, 0x97, 0x4c, 0x00, 0xf5}
+constexpr GUID JITEEVersionIdentifier = { /* 7f943c0c-aad7-4ef5-b7e7-c07d05321f8f */
+    0x7f943c0c,
+    0xaad7,
+    0x4ef5,
+    {0xb7, 0xe7, 0xc0, 0x7d, 0x05, 0x32, 0x1f, 0x8f}
   };
 
 #endif // JIT_EE_VERSIONING_GUID_H

src/coreclr/inc/readytoruninstructionset.h

@@ -102,6 +102,9 @@ enum ReadyToRunInstructionSet
     READYTORUN_INSTRUCTION_PackedSimd=92,
     READYTORUN_INSTRUCTION_Cssc=93,
     READYTORUN_INSTRUCTION_Zicond=94,
+    READYTORUN_INSTRUCTION_LAM_BH=95,
+    READYTORUN_INSTRUCTION_LAM_CAS=96,
+
 };
 
 #endif // READYTORUNINSTRUCTIONSET_H

src/coreclr/jit/codegenloongarch64.cpp

@@ -2167,14 +2167,74 @@ void CodeGen::genNonLocalJmp(GenTreeUnOp* tree)
 }
 
 //------------------------------------------------------------------------
-// genLockedInstructions: Generate code for a GT_XADD or GT_XCHG node.
+// genLockedInstructions: Generate code for a GT_XADD, GT_XAND, GT_XORR or GT_XCHG node.
 //
 // Arguments:
-//    treeNode - the GT_XADD/XCHG node
+//    treeNode - the GT_XADD/XAND/XORR/XCHG node
 //
 void CodeGen::genLockedInstructions(GenTreeOp* treeNode)
 {
-    NYI("unimplemented on LOONGARCH64 yet");
+    GenTree*  data      = treeNode->AsOp()->gtOp2;
+    GenTree*  addr      = treeNode->AsOp()->gtOp1;
+    regNumber dataReg   = !data->isContained() ? data->GetRegNum() : REG_R0;
+    regNumber addrReg   = addr->GetRegNum();
+    regNumber targetReg = treeNode->GetRegNum();
+
+    genConsumeAddress(addr);
+    genConsumeRegs(data);
+
+    assert(treeNode->OperIs(GT_XCHG) || !varTypeIsSmall(treeNode->TypeGet()));
+
+    emitter* emit     = GetEmitter();
+    emitAttr dataSize = emitActualTypeSize(data);
+    bool     is4      = (dataSize == EA_4BYTE);
+
+    instruction ins = INS_none;
+    if (!varTypeIsSmall(treeNode->TypeGet()))
+    {
+        switch (treeNode->gtOper)
+        {
+            case GT_XORR:
+                ins = is4 ? INS_amor_db_w : INS_amor_db_d;
+                break;
+            case GT_XAND:
+                ins = is4 ? INS_amand_db_w : INS_amand_db_d;
+                break;
+            case GT_XCHG:
+                ins = is4 ? INS_amswap_db_w : INS_amswap_db_d;
+                break;
+            case GT_XADD:
+                ins = is4 ? INS_amadd_db_w : INS_amadd_db_d;
+                break;
+            default:
+                noway_assert(!"Unexpected treeNode->gtOper");
+        }
+        emit->emitIns_R_R_R(ins, dataSize, targetReg, dataReg, addrReg);
+    }
+    else
+    {
+        // Smalltypes only support atomic instructions for GT_XCHG
+        assert(!treeNode->OperIs(GT_XORR, GT_XAND, GT_XADD));
+        // When LAM_BH is not supported, it will back to the managed implementation or throws PlatformNotSupported.
+        assert(m_compiler->opts.compSupportsISA.HasInstructionSet(InstructionSet_LAM_BH));
+        if (varTypeIsByte(treeNode->TypeGet()))
+        {
+            ins = INS_amswap_db_b;
+        }
+        else if (varTypeIsShort(treeNode->TypeGet()))
+        {
+            ins = INS_amswap_db_h;
+        }
+        emit->emitIns_R_R_R(ins, dataSize, targetReg, dataReg, addrReg);
+    }
+
+    if (varTypeIsSmall(treeNode->TypeGet()) && varTypeIsUnsigned(treeNode->TypeGet()))
+    {
+        int imm1 = varTypeIsByte(treeNode->TypeGet()) ? 7 : 15;
+        emit->emitIns_R_R_I_I(INS_bstrpick_d, dataSize, targetReg, targetReg, imm1, 0);
+    }
+
+    genProduceReg(treeNode);
 }
 
 //------------------------------------------------------------------------
@@ -2185,7 +2245,98 @@ void CodeGen::genLockedInstructions(GenTreeOp* treeNode)
 //
 void CodeGen::genCodeForCmpXchg(GenTreeCmpXchg* treeNode)
 {
-    NYI("unimplemented on LOONGARCH64 yet");
+    assert(treeNode->OperIs(GT_CMPXCHG));
+
+    GenTree* addr      = treeNode->Addr();      // arg1
+    GenTree* data      = treeNode->Data();      // arg2
+    GenTree* comparand = treeNode->Comparand(); // arg3
+
+    regNumber targetReg    = treeNode->GetRegNum();
+    regNumber dataReg      = data->GetRegNum();
+    regNumber addrReg      = addr->GetRegNum();
+    regNumber comparandReg = !comparand->isContained() ? comparand->GetRegNum() : REG_R0;
+
+    genConsumeAddress(addr);
+    genConsumeRegs(data);
+    genConsumeRegs(comparand);
+
+    emitter* emit     = GetEmitter();
+    emitAttr dataSize = emitActualTypeSize(data);
+    bool     is4      = (dataSize == EA_4BYTE);
+    if (m_compiler->opts.compSupportsISA.HasInstructionSet(InstructionSet_LAM_CAS))
+    {
+        // amcas_db use the comparand as the target reg
+        emit->emitIns_R_R(INS_mov, dataSize, targetReg, comparandReg);
+
+        // Catch case we destroyed data or address before use
+        noway_assert((addrReg != targetReg) || (targetReg == comparandReg));
+        noway_assert((dataReg != targetReg) || (targetReg == comparandReg));
+
+        instruction ins = INS_none;
+        ins             = is4 ? INS_amcas_db_w : INS_amcas_db_d;
+        if (varTypeIsByte(treeNode->TypeGet()))
+        {
+            ins = INS_amcas_db_b;
+        }
+        else if (varTypeIsShort(treeNode->TypeGet()))
+        {
+            ins = INS_amcas_db_h;
+        }
+        GetEmitter()->emitIns_R_R_R(ins, dataSize, targetReg, dataReg, addrReg);
+    }
+    else
+    {
+        assert(!varTypeIsSmall(treeNode->TypeGet()));
+        regNumber exResultReg = internalRegisters.Extract(treeNode, RBM_ALLINT);
+        // Register allocator should have extended the lifetimes of all input and internal registers
+        // They should all be different
+        noway_assert(addrReg != targetReg);
+        noway_assert(dataReg != targetReg);
+        noway_assert(comparandReg != targetReg);
+        noway_assert(addrReg != dataReg);
+        noway_assert(targetReg != REG_NA);
+        noway_assert(exResultReg != REG_NA);
+        noway_assert(exResultReg != targetReg);
+
+        assert(addr->isUsedFromReg());
+        assert(!comparand->isUsedFromMemory());
+
+        // Store exclusive unpredictable cases must be avoided
+        noway_assert(exResultReg != dataReg);
+        noway_assert(exResultReg != addrReg);
+
+        // NOTE: `genConsumeAddress` marks consumed register as not a GC pointer, assuming the input
+        // registers die at the first generated instruction. However, here the input registers are reused,
+        // so mark the location register as a GC pointer until code generation for this node is finished.
+        gcInfo.gcMarkRegPtrVal(addrReg, addr->TypeGet());
+
+        BasicBlock* labelRetry = genCreateTempLabel();
+        BasicBlock* fail       = genCreateTempLabel();
+
+        if (is4)
+        {
+            // INS_bne is 64 bit comparison, high bits may be contaminated.
+            emit->emitIns_R_R_I(INS_slli_w, dataSize, comparandReg, comparandReg, 0x0);
+        }
+        instGen_MemoryBarrier();
+        genDefineTempLabel(labelRetry);
+        emit->emitIns_R_R_I(is4 ? INS_ll_w : INS_ll_d, dataSize, targetReg, addrReg, 0); // load original value
+        emit->emitIns_J_cond_la(INS_bne, fail, targetReg, comparandReg);                 // fail if doesn’t match
+        emit->emitIns_R_R(INS_mov, dataSize, exResultReg, dataReg);
+        emit->emitIns_R_R_I(is4 ? INS_sc_w : INS_sc_d, dataSize, exResultReg, addrReg, 0); // try to update
+        emit->emitIns_J_cond_la(INS_beq, labelRetry, exResultReg, REG_R0);                 // retry if update failed
+        genDefineTempLabel(fail);
+        instGen_MemoryBarrier();
+        gcInfo.gcMarkRegSetNpt(addr->gtGetRegMask());
+    }
+
+    if (varTypeIsSmall(treeNode->TypeGet()) && varTypeIsUnsigned(treeNode->TypeGet()))
+    {
+        int imm1 = varTypeIsByte(treeNode->TypeGet()) ? 7 : 15;
+        GetEmitter()->emitIns_R_R_I_I(INS_bstrpick_d, dataSize, targetReg, targetReg, imm1, 0);
+    }
+
+    genProduceReg(treeNode);
 }
 
 static inline bool isImmed(GenTree* treeNode)
@@ -4205,6 +4356,8 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
 
         case GT_XCHG:
         case GT_XADD:
+        case GT_XORR:
+        case GT_XAND:
             genLockedInstructions(treeNode->AsOp());
             break;
 

src/coreclr/jit/compiler.h

@@ -10795,7 +10795,7 @@ class Compiler
     // on which the function is executed (except for CoreLib, where there are special rules)
     bool compExactlyDependsOn(CORINFO_InstructionSet isa) const
     {
-#if defined(TARGET_XARCH) || defined(TARGET_ARM64) || defined(TARGET_RISCV64)
+#if defined(TARGET_XARCH) || defined(TARGET_ARM64) || defined(TARGET_RISCV64) || defined(TARGET_LOONGARCH64)
         if ((opts.compSupportsISAReported.HasInstructionSet(isa)) == false)
         {
             if (notifyInstructionSetUsage(isa, (opts.compSupportsISA.HasInstructionSet(isa))))

src/coreclr/jit/emitloongarch64.cpp

@@ -1557,17 +1557,33 @@ void emitter::emitIns_R_R_R(
         code |= (reg2 /*& 0x1f*/) << 5;  // rj
         code |= (reg3 /*& 0x1f*/) << 10; // rk
     }
-    else if ((INS_amswap_w <= ins) && (ins <= INS_ammin_db_du))
+    else if ((INS_amcas_b <= ins) && (ins <= INS_ammin_db_du))
     {
 #ifdef DEBUG
         switch (ins)
         {
+            case INS_amcas_b:
+            case INS_amcas_h:
+            case INS_amcas_w:
+            case INS_amcas_d:
+            case INS_amcas_db_b:
+            case INS_amcas_db_h:
+            case INS_amcas_db_w:
+            case INS_amcas_db_d:
+            case INS_amswap_b:
+            case INS_amswap_h:
             case INS_amswap_w:
             case INS_amswap_d:
+            case INS_amswap_db_b:
+            case INS_amswap_db_h:
             case INS_amswap_db_w:
             case INS_amswap_db_d:
+            case INS_amadd_b:
+            case INS_amadd_h:
             case INS_amadd_w:
             case INS_amadd_d:
+            case INS_amadd_db_b:
+            case INS_amadd_db_h:
             case INS_amadd_db_w:
             case INS_amadd_db_d:
             case INS_amand_w:
@@ -4239,6 +4255,11 @@ void emitter::emitDisInsName(code_t code, const BYTE* addr, instrDesc* id)
                 printf("0x%x, %s, %s\n", regd, RegNames[regj], RegNames[(code >> 10) & 0x1f]);
                 return;
             }
+            if ((INS_amcas_b <= ins) && (ins <= INS_ammin_db_du))
+            {
+                printf("%s, %s, %s\n", RegNames[regd], RegNames[(code >> 10) & 0x1f], RegNames[regj]);
+                return;
+            }
             printf("%s, %s, %s\n", RegNames[regd], RegNames[regj], RegNames[(code >> 10) & 0x1f]);
             return;
         }