JIT: recognition of ADC/SBB, MULX/ADCX/ADOX patterns for BigInteger, Decimal, UInt128

src/coreclr/inc/clrconfigvalues.h

@@ -667,6 +667,7 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableHWIntrinsic,            W("EnableHWIntri
 
 #if defined(TARGET_AMD64) || defined(TARGET_X86)
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVX,                    W("EnableAVX"),                 1, "Allows AVX and dependent hardware intrinsics to be disabled")
+RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableADX,                    W("EnableADX"),                 1, "Allows ADX carry-chain instructions to be disabled")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVX2,                   W("EnableAVX2"),                1, "Allows AVX2, BMI1, BMI2, F16C, FMA, LZCNT, MOVBE and dependent hardware intrinsics to be disabled")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVX512,                 W("EnableAVX512"),              1, "Allows AVX512 F+BW+CD+DQ+VL and dependent hardware intrinsics to be disabled")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVX512BMM,              W("EnableAVX512BMM"),           1, "Allows AVX512BMM and dependent hardware intrinsics to be disabled")

src/coreclr/inc/corinfoinstructionset.h

@@ -118,6 +118,7 @@ enum CORINFO_InstructionSet
     InstructionSet_SHA_X64=44,
     InstructionSet_WAITPKG_X64=45,
     InstructionSet_X86Serialize_X64=46,
+    InstructionSet_ADX=47,
 #endif // TARGET_AMD64
 #ifdef TARGET_X86
     InstructionSet_X86Base=1,
@@ -166,6 +167,7 @@ enum CORINFO_InstructionSet
     InstructionSet_SHA_X64=44,
     InstructionSet_WAITPKG_X64=45,
     InstructionSet_X86Serialize_X64=46,
+    InstructionSet_ADX=47,
 #endif // TARGET_X86
 
 };
@@ -879,6 +881,8 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet)
             return "AVXVNNIINT";
         case InstructionSet_AVXVNNIINT_V512 :
             return "AVXVNNIINT_V512";
+        case InstructionSet_ADX :
+            return "ADX";
 #endif // TARGET_AMD64
 #ifdef TARGET_X86
         case InstructionSet_X86Base :
@@ -941,6 +945,8 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet)
             return "AVXVNNIINT";
         case InstructionSet_AVXVNNIINT_V512 :
             return "AVXVNNIINT_V512";
+        case InstructionSet_ADX :
+            return "ADX";
 #endif // TARGET_X86
 
         default:
@@ -1063,6 +1069,7 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst
         case READYTORUN_INSTRUCTION_AvxVnniInt8_V512: return InstructionSet_AVXVNNIINT_V512;
         case READYTORUN_INSTRUCTION_AvxVnniInt16: return InstructionSet_AVXVNNIINT;
         case READYTORUN_INSTRUCTION_AvxVnniInt16_V512: return InstructionSet_AVXVNNIINT_V512;
+        case READYTORUN_INSTRUCTION_Adx: return InstructionSet_ADX;
 #endif // TARGET_AMD64
 #ifdef TARGET_X86
         case READYTORUN_INSTRUCTION_X86Base: return InstructionSet_X86Base;
@@ -1133,6 +1140,7 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst
         case READYTORUN_INSTRUCTION_AvxVnniInt8_V512: return InstructionSet_AVXVNNIINT_V512;
         case READYTORUN_INSTRUCTION_AvxVnniInt16: return InstructionSet_AVXVNNIINT;
         case READYTORUN_INSTRUCTION_AvxVnniInt16_V512: return InstructionSet_AVXVNNIINT_V512;
+        case READYTORUN_INSTRUCTION_Adx: return InstructionSet_ADX;
 #endif // TARGET_X86
 
         default:

src/coreclr/inc/jiteeversionguid.h

@@ -37,11 +37,11 @@
 
 #include <minipal/guid.h>
 
-constexpr GUID JITEEVersionIdentifier = { /* fbbaf45f-5b0e-4767-b962-5084c8caae77 */
-    0xfbbaf45f,
-    0x5b0e,
-    0x4767,
-    {0xb9, 0x62, 0x50, 0x84, 0xc8, 0xca, 0xae, 0x77}
+constexpr GUID JITEEVersionIdentifier = { /* 539ba29b-25d8-4b34-895a-01f513a3683a */
+    0x539ba29b,
+    0x25d8,
+    0x4b34,
+    {0x89, 0x5a, 0x01, 0xf5, 0x13, 0xa3, 0x68, 0x3a}
   };
 
 #endif // JIT_EE_VERSIONING_GUID_H

src/coreclr/inc/readytoruninstructionset.h

@@ -103,6 +103,7 @@ enum ReadyToRunInstructionSet
     READYTORUN_INSTRUCTION_Cssc=93,
     READYTORUN_INSTRUCTION_Zicond=94,
     READYTORUN_INSTRUCTION_Fp16=95,
+    READYTORUN_INSTRUCTION_Adx=96,
 
 };
 

src/coreclr/jit/codegenarm64.cpp

@@ -2691,6 +2691,25 @@ void CodeGen::genCodeForBinary(GenTreeOp* tree)
     var_types        targetType = tree->TypeGet();
     emitter*         emit       = GetEmitter();
 
+    if (oper == GT_ADD_BORROW)
+    {
+        assert(!tree->gtSetFlags() && tree->gtGetOp2()->IsIntegralConst(0));
+        emit->emitIns_R_R_COND(INS_cinc, emitActualTypeSize(tree), targetReg, tree->gtGetOp1()->GetRegNum(),
+                               INS_COND_LO);
+        genProduceReg(tree);
+        return;
+    }
+    if (oper == GT_ADD_CARRY || oper == GT_SUB_BORROW)
+    {
+        instruction ins = oper == GT_SUB_BORROW ? (tree->gtSetFlags() ? INS_sbcs : INS_sbc)
+                                                : (tree->gtSetFlags() ? INS_adcs : INS_adc);
+        emit->emitIns_R_R_R(ins, emitActualTypeSize(tree), targetReg,
+                            tree->gtGetOp1()->isContainedIntOrIImmed() ? REG_ZR : tree->gtGetOp1()->GetRegNum(),
+                            tree->gtGetOp2()->isContainedIntOrIImmed() ? REG_ZR : tree->gtGetOp2()->GetRegNum());
+        genProduceReg(tree);
+        return;
+    }
+
     assert(tree->OperIs(GT_ADD, GT_SUB, GT_MUL, GT_DIV, GT_UDIV, GT_AND, GT_AND_NOT, GT_OR, GT_OR_NOT, GT_XOR,
                         GT_XOR_NOT));
 
@@ -2700,6 +2719,16 @@ void CodeGen::genCodeForBinary(GenTreeOp* tree)
     // The arithmetic node must be sitting in a register (since it's not contained)
     assert(targetReg != REG_NA);
 
+    if (tree->IsFunnelShift())
+    {
+        assert(!tree->gtSetFlags());
+        emit->emitIns_R_R_R_I(INS_extr, emitTypeSize(tree), targetReg, op2->gtGetOp1()->GetRegNum(),
+                              op1->gtGetOp1()->GetRegNum(),
+                              static_cast<unsigned>(op1->gtGetOp2()->AsIntCon()->IconValue()));
+        genProduceReg(tree);
+        return;
+    }
+
     // Handles combined operations: 'madd', 'msub'
     if (op2->OperIs(GT_MUL) && op2->isContained())
     {

src/coreclr/jit/codegenarmarch.cpp

@@ -232,6 +232,11 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
         case GT_SUB_HI:
 #endif // !defined(TARGET_64BIT)
 
+#ifdef TARGET_ARM64
+        case GT_SUB_BORROW:
+        case GT_ADD_BORROW:
+        case GT_ADD_CARRY:
+#endif
         case GT_ADD:
         case GT_SUB:
         case GT_MUL:

src/coreclr/jit/codegenxarch.cpp

@@ -921,6 +921,7 @@ void CodeGen::genCodeForDivMod(GenTreeOp* treeNode)
     emit->emitInsBinary(ins, size, treeNode, divisor);
 
     // DIV/IDIV instructions always store the quotient in RAX and the remainder in RDX.
+    assert(!treeNode->IsDivRemPair() || (targetReg == REG_RAX));
     // Move the result to the desired register, if necessary
     if (oper == GT_DIV || oper == GT_UDIV)
     {
@@ -951,6 +952,9 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode)
 {
 #ifdef DEBUG
     bool isValidOper = treeNode->OperIs(GT_ADD, GT_SUB);
+#ifdef TARGET_AMD64
+    isValidOper |= treeNode->OperIs(GT_ADD_CARRY, GT_SUB_BORROW, GT_ADD_BORROW, GT_ADCX, GT_ADOX);
+#endif
     if (varTypeIsFloating(treeNode->TypeGet()))
     {
         isValidOper |= treeNode->OperIs(GT_MUL, GT_DIV);
@@ -975,6 +979,18 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode)
     GenTree* op1 = treeNode->gtGetOp1();
     GenTree* op2 = treeNode->gtGetOp2();
 
+    if (treeNode->IsFunnelShift())
+    {
+        regNumber lo = op1->gtGetOp1()->GetRegNum();
+        regNumber hi = op2->gtGetOp1()->GetRegNum();
+        assert(targetReg != hi);
+        inst_Mov(targetType, targetReg, lo, /* canSkip */ true);
+        inst_RV_RV_IV(INS_shrd, emitTypeSize(targetType), targetReg, hi,
+                      static_cast<unsigned>(op1->gtGetOp2()->AsIntCon()->IconValue()));
+        genProduceReg(treeNode);
+        return;
+    }
+
     bool eligibleForNDD = false;
 
     // Commutative operations can mark op1 as contained or reg-optional to generate "op reg, memop/immed"
@@ -1098,7 +1114,8 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode)
     assert(!varTypeIsFloating(treeNode));
 
     // try to use an inc or dec
-    if (oper == GT_ADD && src->isContainedIntOrIImmed() && !treeNode->gtOverflowEx())
+    if (oper == GT_ADD && src->isContainedIntOrIImmed() && !treeNode->gtOverflowEx() &&
+        ((treeNode->gtFlags & GTF_ADD_CARRY_FLAGS) == 0))
     {
         if (src->IsIntegralConst(1))
         {
@@ -1940,6 +1957,13 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
         case GT_SUB_HI:
 #endif // !defined(TARGET_64BIT)
 
+#ifdef TARGET_AMD64
+        case GT_SUB_BORROW:
+        case GT_ADD_BORROW:
+        case GT_ADD_CARRY:
+        case GT_ADCX:
+        case GT_ADOX:
+#endif
         case GT_ADD:
         case GT_SUB:
             genCodeForBinary(treeNode->AsOp());
@@ -2080,6 +2104,58 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
             genCodeForJTrue(treeNode->AsOp());
             break;
 
+#ifdef TARGET_AMD64
+        case GT_ADX_SEED:
+        {
+            // TEST clears CF and OF without a temporary register or changing SP.
+            GetEmitter()->emitIns_R_R(INS_test, EA_4BYTE, REG_ESP, REG_ESP);
+            break;
+        }
+        case GT_ADX_DRAIN:
+        {
+            genConsumeOperands(treeNode->AsOp());
+            regNumber target = treeNode->GetRegNum();
+            regNumber zero   = internalRegisters.GetSingle(treeNode);
+            inst_Mov(TYP_LONG, target, treeNode->gtGetOp1()->GetRegNum(), true);
+            // A MOV is required: XOR would destroy the carries being drained.
+            GetEmitter()->emitIns_R_I(INS_mov, EA_4BYTE, zero, 0);
+            GetEmitter()->emitIns_R_R(INS_adcx, EA_8BYTE, target, zero);
+            GetEmitter()->emitIns_R_R(INS_adox, EA_8BYTE, target, zero);
+            genProduceReg(treeNode);
+            break;
+        }
+        case GT_JCMP:
+        {
+            genConsumeOperands(treeNode->AsOp());
+            GenTree* count = treeNode->gtGetOp1();
+            // Enregistered local uses can retain their source register here.
+            // Copy the count to RCX when necessary. The countdown's 32-bit LEA
+            // (or its 32-bit reload/copy) has already cleared the upper half when
+            // this move is skipped; bare JRCXZ tests all of RCX, not just ECX.
+            GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(count), REG_ECX, count->GetRegNum(), true);
+            assert(treeNode->gtGetOp2()->IsIntegralConst(0) && treeNode->gtGetOp2()->isContained());
+            // JRCXZ preserves both carry flags but has only a rel8 encoding.
+            // Skip a JMP to reach the loop: two branch instructions per backedge.
+            GenCondition::Code condition = treeNode->AsOpCC()->gtCondition.GetCode();
+            assert(condition == GenCondition::EQ || condition == GenCondition::NE);
+            BasicBlock* done = genCreateTempLabel();
+            if (condition == GenCondition::NE)
+            {
+                GetEmitter()->emitIns_J(INS_jrcxz, done, true);
+                GetEmitter()->emitIns_J(INS_jmp, m_compiler->compCurBB->GetTrueTarget());
+            }
+            else
+            {
+                BasicBlock* taken = genCreateTempLabel();
+                GetEmitter()->emitIns_J(INS_jrcxz, taken, true);
+                GetEmitter()->emitIns_J(INS_jmp, done, true);
+                genDefineTempLabel(taken);
+                GetEmitter()->emitIns_J(INS_jmp, m_compiler->compCurBB->GetTrueTarget());
+            }
+            genDefineTempLabel(done);
+            break;
+        }
+#endif
         case GT_JCC:
             genCodeForJcc(treeNode->AsCC());
             break;
@@ -4496,6 +4572,7 @@ void CodeGen::genRangeCheck(GenTree* oper)
 void CodeGen::genCodeForPhysReg(GenTreePhysReg* tree)
 {
     assert(tree->OperIs(GT_PHYSREG));
+    assert(!tree->IsDivRemPair() || ((tree->GetRegNum() == REG_RDX) && (tree->gtSrcReg == REG_RDX)));
 
     var_types targetType = tree->TypeGet();
     regNumber targetReg  = tree->GetRegNum();
@@ -4537,6 +4614,19 @@ instruction CodeGen::genGetInsForOper(genTreeOps oper, var_types type)
 
     switch (oper)
     {
+#ifdef TARGET_AMD64
+        case GT_ADCX:
+            return INS_adcx;
+        case GT_ADOX:
+            return INS_adox;
+        case GT_SUB_BORROW:
+            ins = INS_sbb;
+            break;
+        case GT_ADD_BORROW:
+        case GT_ADD_CARRY:
+            ins = INS_adc;
+            break;
+#endif
         case GT_ADD:
             ins = INS_add;
             break;

src/coreclr/jit/compiler.h

@@ -8353,6 +8353,9 @@ class Compiler
     bool optMakeLoopDownwardsCounted(ScalarEvolutionContext& scevContext,
                                      FlowGraphNaturalLoop*   loop,
                                      PerLoopInfo*            loopLocals);
+#if defined(TARGET_AMD64) || defined(TARGET_ARM64)
+    void optFoldFourLimbOffsets(ScalarEvolutionContext& scevContext, BasicBlock* block);
+#endif
     bool optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevContext,
                                          FlowGraphNaturalLoop*   loop,
                                          BasicBlock*             exiting,

src/coreclr/jit/emitarm64.cpp

@@ -516,7 +516,9 @@ void emitter::emitInsSanityCheck(instrDesc* id)
             }
             else
             {
-                assert(isGeneralRegister(id->idReg3()));
+                assert(isGeneralRegister(id->idReg3()) || (((id->idIns() == INS_adc) || (id->idIns() == INS_adcs) ||
+                                                            (id->idIns() == INS_sbc) || (id->idIns() == INS_sbcs)) &&
+                                                           (id->idReg3() == REG_ZR)));
             }
             assert(insOptsNone(id->idInsOpt()));
             break;
@@ -6138,10 +6140,6 @@ void emitter::emitIns_R_R_R(instruction     ins,
         case INS_lsr:
         case INS_asr:
         case INS_ror:
-        case INS_adc:
-        case INS_adcs:
-        case INS_sbc:
-        case INS_sbcs:
         case INS_udiv:
         case INS_sdiv:
         case INS_mneg:
@@ -6174,6 +6172,18 @@ void emitter::emitIns_R_R_R(instruction     ins,
             fmt = IF_DR_3A;
             break;
 
+        case INS_adc:
+        case INS_adcs:
+        case INS_sbc:
+        case INS_sbcs:
+            assert(insOptsNone(opt));
+            assert(isValidGeneralDatasize(size));
+            assert(isGeneralRegisterOrZR(reg1));
+            assert(isGeneralRegisterOrZR(reg2));
+            assert(isGeneralRegisterOrZR(reg3));
+            fmt = IF_DR_3A;
+            break;
+
         case INS_add:
         case INS_sub:
             if (isVectorRegister(reg1))