diff --git a/src/coreclr/inc/clrconfigvalues.h b/src/coreclr/inc/clrconfigvalues.h index d6badce75704c0..89e70bc6edd97d 100644 --- a/src/coreclr/inc/clrconfigvalues.h +++ b/src/coreclr/inc/clrconfigvalues.h @@ -667,6 +667,7 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableHWIntrinsic, W("EnableHWIntri #if defined(TARGET_AMD64) || defined(TARGET_X86) RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVX, W("EnableAVX"), 1, "Allows AVX and dependent hardware intrinsics to be disabled") +RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableADX, W("EnableADX"), 1, "Allows ADX carry-chain instructions to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVX2, W("EnableAVX2"), 1, "Allows AVX2, BMI1, BMI2, F16C, FMA, LZCNT, MOVBE and dependent hardware intrinsics to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVX512, W("EnableAVX512"), 1, "Allows AVX512 F+BW+CD+DQ+VL and dependent hardware intrinsics to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVX512BMM, W("EnableAVX512BMM"), 1, "Allows AVX512BMM and dependent hardware intrinsics to be disabled") diff --git a/src/coreclr/inc/corinfoinstructionset.h b/src/coreclr/inc/corinfoinstructionset.h index 2f347572ee34d4..0e86fed61c92c0 100644 --- a/src/coreclr/inc/corinfoinstructionset.h +++ b/src/coreclr/inc/corinfoinstructionset.h @@ -118,6 +118,7 @@ enum CORINFO_InstructionSet InstructionSet_SHA_X64=44, InstructionSet_WAITPKG_X64=45, InstructionSet_X86Serialize_X64=46, + InstructionSet_ADX=47, #endif // TARGET_AMD64 #ifdef TARGET_X86 InstructionSet_X86Base=1, @@ -166,6 +167,7 @@ enum CORINFO_InstructionSet InstructionSet_SHA_X64=44, InstructionSet_WAITPKG_X64=45, InstructionSet_X86Serialize_X64=46, + InstructionSet_ADX=47, #endif // TARGET_X86 }; @@ -879,6 +881,8 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet) return "AVXVNNIINT"; case InstructionSet_AVXVNNIINT_V512 : return "AVXVNNIINT_V512"; + case InstructionSet_ADX : + return "ADX"; #endif // TARGET_AMD64 #ifdef TARGET_X86 case InstructionSet_X86Base : @@ -941,6 +945,8 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet) return "AVXVNNIINT"; case InstructionSet_AVXVNNIINT_V512 : return "AVXVNNIINT_V512"; + case InstructionSet_ADX : + return "ADX"; #endif // TARGET_X86 default: @@ -1063,6 +1069,7 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst case READYTORUN_INSTRUCTION_AvxVnniInt8_V512: return InstructionSet_AVXVNNIINT_V512; case READYTORUN_INSTRUCTION_AvxVnniInt16: return InstructionSet_AVXVNNIINT; case READYTORUN_INSTRUCTION_AvxVnniInt16_V512: return InstructionSet_AVXVNNIINT_V512; + case READYTORUN_INSTRUCTION_Adx: return InstructionSet_ADX; #endif // TARGET_AMD64 #ifdef TARGET_X86 case READYTORUN_INSTRUCTION_X86Base: return InstructionSet_X86Base; @@ -1133,6 +1140,7 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst case READYTORUN_INSTRUCTION_AvxVnniInt8_V512: return InstructionSet_AVXVNNIINT_V512; case READYTORUN_INSTRUCTION_AvxVnniInt16: return InstructionSet_AVXVNNIINT; case READYTORUN_INSTRUCTION_AvxVnniInt16_V512: return InstructionSet_AVXVNNIINT_V512; + case READYTORUN_INSTRUCTION_Adx: return InstructionSet_ADX; #endif // TARGET_X86 default: diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index a1559f43510c20..847663cada9877 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -37,11 +37,11 @@ #include -constexpr GUID JITEEVersionIdentifier = { /* fbbaf45f-5b0e-4767-b962-5084c8caae77 */ - 0xfbbaf45f, - 0x5b0e, - 0x4767, - {0xb9, 0x62, 0x50, 0x84, 0xc8, 0xca, 0xae, 0x77} +constexpr GUID JITEEVersionIdentifier = { /* 539ba29b-25d8-4b34-895a-01f513a3683a */ + 0x539ba29b, + 0x25d8, + 0x4b34, + {0x89, 0x5a, 0x01, 0xf5, 0x13, 0xa3, 0x68, 0x3a} }; #endif // JIT_EE_VERSIONING_GUID_H diff --git a/src/coreclr/inc/readytoruninstructionset.h b/src/coreclr/inc/readytoruninstructionset.h index d2851e91577f1e..ddcfb2af193445 100644 --- a/src/coreclr/inc/readytoruninstructionset.h +++ b/src/coreclr/inc/readytoruninstructionset.h @@ -103,6 +103,7 @@ enum ReadyToRunInstructionSet READYTORUN_INSTRUCTION_Cssc=93, READYTORUN_INSTRUCTION_Zicond=94, READYTORUN_INSTRUCTION_Fp16=95, + READYTORUN_INSTRUCTION_Adx=96, }; diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 588afa49dadfb5..69f07c93be9d0e 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -2691,6 +2691,25 @@ void CodeGen::genCodeForBinary(GenTreeOp* tree) var_types targetType = tree->TypeGet(); emitter* emit = GetEmitter(); + if (oper == GT_ADD_BORROW) + { + assert(!tree->gtSetFlags() && tree->gtGetOp2()->IsIntegralConst(0)); + emit->emitIns_R_R_COND(INS_cinc, emitActualTypeSize(tree), targetReg, tree->gtGetOp1()->GetRegNum(), + INS_COND_LO); + genProduceReg(tree); + return; + } + if (oper == GT_ADD_CARRY || oper == GT_SUB_BORROW) + { + instruction ins = oper == GT_SUB_BORROW ? (tree->gtSetFlags() ? INS_sbcs : INS_sbc) + : (tree->gtSetFlags() ? INS_adcs : INS_adc); + emit->emitIns_R_R_R(ins, emitActualTypeSize(tree), targetReg, + tree->gtGetOp1()->isContainedIntOrIImmed() ? REG_ZR : tree->gtGetOp1()->GetRegNum(), + tree->gtGetOp2()->isContainedIntOrIImmed() ? REG_ZR : tree->gtGetOp2()->GetRegNum()); + genProduceReg(tree); + return; + } + assert(tree->OperIs(GT_ADD, GT_SUB, GT_MUL, GT_DIV, GT_UDIV, GT_AND, GT_AND_NOT, GT_OR, GT_OR_NOT, GT_XOR, GT_XOR_NOT)); @@ -2700,6 +2719,16 @@ void CodeGen::genCodeForBinary(GenTreeOp* tree) // The arithmetic node must be sitting in a register (since it's not contained) assert(targetReg != REG_NA); + if (tree->IsFunnelShift()) + { + assert(!tree->gtSetFlags()); + emit->emitIns_R_R_R_I(INS_extr, emitTypeSize(tree), targetReg, op2->gtGetOp1()->GetRegNum(), + op1->gtGetOp1()->GetRegNum(), + static_cast(op1->gtGetOp2()->AsIntCon()->IconValue())); + genProduceReg(tree); + return; + } + // Handles combined operations: 'madd', 'msub' if (op2->OperIs(GT_MUL) && op2->isContained()) { diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 37cca461351f49..3e845f985f2b84 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -232,6 +232,11 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) case GT_SUB_HI: #endif // !defined(TARGET_64BIT) +#ifdef TARGET_ARM64 + case GT_SUB_BORROW: + case GT_ADD_BORROW: + case GT_ADD_CARRY: +#endif case GT_ADD: case GT_SUB: case GT_MUL: diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 36c4bcfd21f72f..59baa9c7deefd4 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -921,6 +921,7 @@ void CodeGen::genCodeForDivMod(GenTreeOp* treeNode) emit->emitInsBinary(ins, size, treeNode, divisor); // DIV/IDIV instructions always store the quotient in RAX and the remainder in RDX. + assert(!treeNode->IsDivRemPair() || (targetReg == REG_RAX)); // Move the result to the desired register, if necessary if (oper == GT_DIV || oper == GT_UDIV) { @@ -951,6 +952,9 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) { #ifdef DEBUG bool isValidOper = treeNode->OperIs(GT_ADD, GT_SUB); +#ifdef TARGET_AMD64 + isValidOper |= treeNode->OperIs(GT_ADD_CARRY, GT_SUB_BORROW, GT_ADD_BORROW, GT_ADCX, GT_ADOX); +#endif if (varTypeIsFloating(treeNode->TypeGet())) { isValidOper |= treeNode->OperIs(GT_MUL, GT_DIV); @@ -975,6 +979,18 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) GenTree* op1 = treeNode->gtGetOp1(); GenTree* op2 = treeNode->gtGetOp2(); + if (treeNode->IsFunnelShift()) + { + regNumber lo = op1->gtGetOp1()->GetRegNum(); + regNumber hi = op2->gtGetOp1()->GetRegNum(); + assert(targetReg != hi); + inst_Mov(targetType, targetReg, lo, /* canSkip */ true); + inst_RV_RV_IV(INS_shrd, emitTypeSize(targetType), targetReg, hi, + static_cast(op1->gtGetOp2()->AsIntCon()->IconValue())); + genProduceReg(treeNode); + return; + } + bool eligibleForNDD = false; // Commutative operations can mark op1 as contained or reg-optional to generate "op reg, memop/immed" @@ -1098,7 +1114,8 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) assert(!varTypeIsFloating(treeNode)); // try to use an inc or dec - if (oper == GT_ADD && src->isContainedIntOrIImmed() && !treeNode->gtOverflowEx()) + if (oper == GT_ADD && src->isContainedIntOrIImmed() && !treeNode->gtOverflowEx() && + ((treeNode->gtFlags & GTF_ADD_CARRY_FLAGS) == 0)) { if (src->IsIntegralConst(1)) { @@ -1940,6 +1957,13 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) case GT_SUB_HI: #endif // !defined(TARGET_64BIT) +#ifdef TARGET_AMD64 + case GT_SUB_BORROW: + case GT_ADD_BORROW: + case GT_ADD_CARRY: + case GT_ADCX: + case GT_ADOX: +#endif case GT_ADD: case GT_SUB: genCodeForBinary(treeNode->AsOp()); @@ -2080,6 +2104,58 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) genCodeForJTrue(treeNode->AsOp()); break; +#ifdef TARGET_AMD64 + case GT_ADX_SEED: + { + // TEST clears CF and OF without a temporary register or changing SP. + GetEmitter()->emitIns_R_R(INS_test, EA_4BYTE, REG_ESP, REG_ESP); + break; + } + case GT_ADX_DRAIN: + { + genConsumeOperands(treeNode->AsOp()); + regNumber target = treeNode->GetRegNum(); + regNumber zero = internalRegisters.GetSingle(treeNode); + inst_Mov(TYP_LONG, target, treeNode->gtGetOp1()->GetRegNum(), true); + // A MOV is required: XOR would destroy the carries being drained. + GetEmitter()->emitIns_R_I(INS_mov, EA_4BYTE, zero, 0); + GetEmitter()->emitIns_R_R(INS_adcx, EA_8BYTE, target, zero); + GetEmitter()->emitIns_R_R(INS_adox, EA_8BYTE, target, zero); + genProduceReg(treeNode); + break; + } + case GT_JCMP: + { + genConsumeOperands(treeNode->AsOp()); + GenTree* count = treeNode->gtGetOp1(); + // Enregistered local uses can retain their source register here. + // Copy the count to RCX when necessary. The countdown's 32-bit LEA + // (or its 32-bit reload/copy) has already cleared the upper half when + // this move is skipped; bare JRCXZ tests all of RCX, not just ECX. + GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(count), REG_ECX, count->GetRegNum(), true); + assert(treeNode->gtGetOp2()->IsIntegralConst(0) && treeNode->gtGetOp2()->isContained()); + // JRCXZ preserves both carry flags but has only a rel8 encoding. + // Skip a JMP to reach the loop: two branch instructions per backedge. + GenCondition::Code condition = treeNode->AsOpCC()->gtCondition.GetCode(); + assert(condition == GenCondition::EQ || condition == GenCondition::NE); + BasicBlock* done = genCreateTempLabel(); + if (condition == GenCondition::NE) + { + GetEmitter()->emitIns_J(INS_jrcxz, done, true); + GetEmitter()->emitIns_J(INS_jmp, m_compiler->compCurBB->GetTrueTarget()); + } + else + { + BasicBlock* taken = genCreateTempLabel(); + GetEmitter()->emitIns_J(INS_jrcxz, taken, true); + GetEmitter()->emitIns_J(INS_jmp, done, true); + genDefineTempLabel(taken); + GetEmitter()->emitIns_J(INS_jmp, m_compiler->compCurBB->GetTrueTarget()); + } + genDefineTempLabel(done); + break; + } +#endif case GT_JCC: genCodeForJcc(treeNode->AsCC()); break; @@ -4496,6 +4572,7 @@ void CodeGen::genRangeCheck(GenTree* oper) void CodeGen::genCodeForPhysReg(GenTreePhysReg* tree) { assert(tree->OperIs(GT_PHYSREG)); + assert(!tree->IsDivRemPair() || ((tree->GetRegNum() == REG_RDX) && (tree->gtSrcReg == REG_RDX))); var_types targetType = tree->TypeGet(); regNumber targetReg = tree->GetRegNum(); @@ -4537,6 +4614,19 @@ instruction CodeGen::genGetInsForOper(genTreeOps oper, var_types type) switch (oper) { +#ifdef TARGET_AMD64 + case GT_ADCX: + return INS_adcx; + case GT_ADOX: + return INS_adox; + case GT_SUB_BORROW: + ins = INS_sbb; + break; + case GT_ADD_BORROW: + case GT_ADD_CARRY: + ins = INS_adc; + break; +#endif case GT_ADD: ins = INS_add; break; diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index ffacc9f0671743..3fcf4f5d2b504f 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -8353,6 +8353,9 @@ class Compiler bool optMakeLoopDownwardsCounted(ScalarEvolutionContext& scevContext, FlowGraphNaturalLoop* loop, PerLoopInfo* loopLocals); +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + void optFoldFourLimbOffsets(ScalarEvolutionContext& scevContext, BasicBlock* block); +#endif bool optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevContext, FlowGraphNaturalLoop* loop, BasicBlock* exiting, diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index df354db94fe56e..ecf21930c94ae2 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -516,7 +516,9 @@ void emitter::emitInsSanityCheck(instrDesc* id) } else { - assert(isGeneralRegister(id->idReg3())); + assert(isGeneralRegister(id->idReg3()) || (((id->idIns() == INS_adc) || (id->idIns() == INS_adcs) || + (id->idIns() == INS_sbc) || (id->idIns() == INS_sbcs)) && + (id->idReg3() == REG_ZR))); } assert(insOptsNone(id->idInsOpt())); break; @@ -6138,10 +6140,6 @@ void emitter::emitIns_R_R_R(instruction ins, case INS_lsr: case INS_asr: case INS_ror: - case INS_adc: - case INS_adcs: - case INS_sbc: - case INS_sbcs: case INS_udiv: case INS_sdiv: case INS_mneg: @@ -6174,6 +6172,18 @@ void emitter::emitIns_R_R_R(instruction ins, fmt = IF_DR_3A; break; + case INS_adc: + case INS_adcs: + case INS_sbc: + case INS_sbcs: + assert(insOptsNone(opt)); + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegisterOrZR(reg1)); + assert(isGeneralRegisterOrZR(reg2)); + assert(isGeneralRegisterOrZR(reg3)); + fmt = IF_DR_3A; + break; + case INS_add: case INS_sub: if (isVectorRegister(reg1)) diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index fa33e803f4273d..365f01f7638242 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -2691,7 +2691,7 @@ bool emitter::TakesRexWPrefix(const instrDesc* id) const // so we never need it if ((ins != INS_push) && (ins != INS_pop) && (ins != INS_movq) && (ins != INS_movzx) && (ins != INS_push_hide) && (ins != INS_pop_hide) && (ins != INS_ret) && (ins != INS_call) && (ins != INS_tail_i_jmp) && - !((ins >= INS_i_jmp) && (ins <= INS_l_jg))) + (ins != INS_jrcxz) && !((ins >= INS_i_jmp) && (ins <= INS_l_jg))) { return true; } @@ -3912,9 +3912,9 @@ unsigned emitter::emitGetAdjustedSize(instrDesc* id, code_t code) const #endif // TARGET_AMD64 else { - if (ins == INS_crc32) + if (IsLegacyMap38Instruction(ins)) { - // Adjust code size for CRC32 that has 4-byte opcode but does not use SSE38 or EES3A encoding. + // Account for the mandatory prefix of legacy map-38 instructions. adjustedSize++; } @@ -5269,7 +5269,10 @@ inline UNATIVE_OFFSET emitter::emitInsSizeRR(instrDesc* id) if ((code & 0xFF00) != 0) { - sz += (IsSimdInstruction(ins) || TakesEvexPrefix(id)) ? emitInsSize(id, code, includeRexPrefixSize) : 5; + // Legacy map-38 instructions have their mandatory prefix counted above. + sz += (IsSimdInstruction(ins) || IsLegacyMap38Instruction(ins) || TakesEvexPrefix(id)) + ? emitInsSize(id, code, includeRexPrefixSize) + : 5; } else { @@ -10846,6 +10849,12 @@ void emitter::emitIns_J(instruction ins, BasicBlock* dst, bool keepShort, bool i #endif } + if (ins == INS_jrcxz) + { + // Only used to skip one unconditional jump to a nearby temporary label. + noway_assert(keepShort && !id->idjKeepLong); + sz = JCC_SIZE_SMALL; + } id->idCodeSize(sz); dispIns(id); @@ -14209,7 +14218,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) } // Special case emitting AVX instructions - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if ((ins == INS_crc32) && (size > EA_1BYTE)) { @@ -14251,7 +14260,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) dst += emitOutputRexOrSimdPrefixIfNeeded(ins, dst, code); - if (UseSimdEncoding() && (ins != INS_crc32)) + if (UseSimdEncoding() && !IsLegacyMap38Instruction(ins)) { // Emit last opcode byte // TODO-XArch-CQ: Right now support 4-byte opcode instructions only @@ -14466,7 +14475,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) // The address is of the form "[disp]" // On x86 - disp is relative to zero // On Amd64 - disp is relative to RIP - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { dst += emitOutputByte(dst, code | 0x05); } @@ -14529,7 +14538,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) else { #ifdef TARGET_X86 - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { dst += emitOutputByte(dst, code | 0x05); } @@ -14546,7 +14555,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) noway_assert((int)dsp == dsp); // This requires, specifying a SIB byte after ModRM byte. - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { dst += emitOutputByte(dst, code | 0x04); } @@ -14563,7 +14572,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) case REG_EBP: { - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspInByte) { @@ -14605,7 +14614,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) case REG_ESP: { - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspIsZero) { @@ -14663,7 +14672,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) default: { - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { // Put the register in the opcode code |= insEncodeReg012(id, reg, EA_PTRSIZE, nullptr); @@ -14742,7 +14751,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) regByte = insEncodeReg012(id, reg, EA_PTRSIZE, nullptr) | insEncodeReg345(id, rgx, EA_PTRSIZE, nullptr) | insSSval(mul); - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspIsZero && (reg != REG_EBP)) { @@ -14802,7 +14811,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) regByte = insEncodeReg012(id, REG_EBP, EA_PTRSIZE, nullptr) | insEncodeReg345(id, rgx, EA_PTRSIZE, nullptr) | insSSval(mul); - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { dst += emitOutputByte(dst, code | 0x04); } @@ -14831,7 +14840,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) // The address is "[reg+rgx+dsp]" regByte = insEncodeReg012(id, reg, EA_PTRSIZE, nullptr) | insEncodeReg345(id, rgx, EA_PTRSIZE, nullptr); - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspIsZero && (reg != REG_EBP)) { @@ -15101,7 +15110,7 @@ BYTE* emitter::emitOutputSV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) } // Special case emitting AVX instructions - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if ((ins == INS_crc32) && (size > EA_1BYTE)) { @@ -15130,7 +15139,7 @@ BYTE* emitter::emitOutputSV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) dst += emitOutputRexOrSimdPrefixIfNeeded(ins, dst, code); - if (UseSimdEncoding() && (ins != INS_crc32)) + if (UseSimdEncoding() && !IsLegacyMap38Instruction(ins)) { // Emit last opcode byte // TODO-XArch-CQ: Right now support 4-byte opcode instructions only @@ -15333,7 +15342,7 @@ BYTE* emitter::emitOutputSV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) if (EBPbased) { // EBP-based variable: does the offset fit in a byte? - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspInByte) { @@ -15384,7 +15393,7 @@ BYTE* emitter::emitOutputSV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) #endif // !FEATURE_FIXED_OUT_ARGS // Does the offset fit in a byte? - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspIsZero) { @@ -15673,7 +15682,7 @@ BYTE* emitter::emitOutputCV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) #endif // TARGET_X86 // Special case emitting AVX instructions - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if ((ins == INS_crc32) && (size > EA_1BYTE)) { @@ -15702,7 +15711,7 @@ BYTE* emitter::emitOutputCV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) dst += emitOutputRexOrSimdPrefixIfNeeded(ins, dst, code); - if (UseVEXEncoding() && (ins != INS_crc32)) + if (UseVEXEncoding() && !IsLegacyMap38Instruction(ins)) { // Emit last opcode byte // TODO-XArch-CQ: Right now support 4-byte opcode instructions only @@ -16463,8 +16472,8 @@ BYTE* emitter::emitOutputRR(BYTE* dst, instrDesc* id) #endif // TARGET_AMD64 } #ifdef FEATURE_HW_INTRINSICS - else if ((ins == INS_bsf) || (ins == INS_bsr) || (ins == INS_crc32) || (ins == INS_lzcnt) || (ins == INS_popcnt) || - (ins == INS_tzcnt) + else if ((ins == INS_bsf) || (ins == INS_bsr) || IsLegacyMap38Instruction(ins) || (ins == INS_lzcnt) || + (ins == INS_popcnt) || (ins == INS_tzcnt) #ifdef TARGET_AMD64 || (ins == INS_lzcnt_apx) || (ins == INS_tzcnt_apx) || (ins == INS_popcnt_apx) || (ins == INS_crc32_apx) #endif // TARGET_AMD64 @@ -17403,6 +17412,12 @@ BYTE* emitter::emitOutputLJ(insGroup* ig, BYTE* dst, instrDesc* i) jmp = true; break; + case INS_jrcxz: + noway_assert(id->idjShort && !id->idjKeepLong); + ssz = lsz = JCC_SIZE_SMALL; + jmp = true; + break; + case INS_jmp: ssz = JMP_SIZE_SMALL; lsz = JMP_SIZE_LARGE; @@ -18941,7 +18956,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) break; } - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { // Special case 4-byte AVX instructions as the // regcode position conflicts with the opcode byte @@ -19180,7 +19195,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) { code = insCodeRM(ins); - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { // Special case 4-byte AVX instructions as the // regcode position conflicts with the opcode byte @@ -19436,7 +19451,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) { code = insCodeRM(ins); - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { // Special case 4-byte AVX instructions as the // regcode position conflicts with the opcode byte diff --git a/src/coreclr/jit/emitxarch.h b/src/coreclr/jit/emitxarch.h index 51c0cca338f005..eaa1dfb110b2ff 100644 --- a/src/coreclr/jit/emitxarch.h +++ b/src/coreclr/jit/emitxarch.h @@ -116,6 +116,10 @@ code_t insEncodeOpreg(const instrDesc* id, regNumber reg, emitAttr size); unsigned insSSval(unsigned scale); static bool IsSSEInstruction(instruction ins); +static bool IsLegacyMap38Instruction(instruction ins) +{ + return ins == INS_crc32 || ins == INS_adcx || ins == INS_adox; +} static bool IsSSEOrAVXInstruction(instruction ins); static bool IsAvx512OnlyInstruction(instruction ins); static bool IsKMOVInstruction(instruction ins); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 3700c917e7d2e8..1ac666ef6485d4 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -12008,6 +12008,39 @@ void Compiler::gtUpdateNodeSideEffects(GenTree* tree) }); } +//------------------------------------------------------------------------ +// IsFunnelShift: Whether lowering contained two complementary shifts in an OR. +// +// Return Value: +// True for the canonical (lo >>> count) | (hi << (width - count)) form. +// +bool GenTree::IsFunnelShift() const +{ +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + bool isFunnelShift = OperIs(GT_OR) && gtGetOp1()->OperIs(GT_RSZ) && gtGetOp1()->isContained() && + gtGetOp2()->OperIs(GT_LSH) && gtGetOp2()->isContained(); +#ifdef DEBUG + if (isFunnelShift) + { + // TryContainFunnelShift is the only path that contains both shifts. It validates + // their widths and complementary immediate counts before establishing this shape. + assert(TypeIs(TYP_INT, TYP_LONG)); + assert(gtGetOp1()->TypeGet() == TypeGet()); + assert(gtGetOp2()->TypeGet() == TypeGet()); + assert(gtGetOp1()->gtGetOp2()->IsCnsIntOrI()); + assert(gtGetOp2()->gtGetOp2()->IsCnsIntOrI()); + ssize_t width = genTypeSize(TypeGet()) * BITS_PER_BYTE; + ssize_t count = gtGetOp1()->gtGetOp2()->AsIntCon()->IconValue(); + assert((count > 0) && (count < width)); + assert(gtGetOp2()->gtGetOp2()->AsIntCon()->IconValue() == width - count); + } +#endif + return isFunnelShift; +#else + return false; +#endif +} + bool GenTree::gtSetFlags() const { return (gtFlags & GTF_SET_FLAGS) != 0; @@ -12061,6 +12094,9 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) case GT_JMP: case GT_JCC: case GT_SETCC: +#ifdef TARGET_AMD64 + case GT_ADX_SEED: +#endif case GT_NO_OP: case GT_START_NONGC: case GT_START_PREEMPTGC: @@ -12083,6 +12119,9 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) // Standard unary operators case GT_STORE_LCL_VAR: +#ifdef TARGET_AMD64 + case GT_ADX_DRAIN: +#endif case GT_STORE_LCL_FLD: case GT_NOT: case GT_NEG: @@ -14294,6 +14333,10 @@ void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack) } break; +#ifdef TARGET_AMD64 + case GT_ADX_SEED: + break; +#endif case GT_PHYSREG: printf(" %s", getRegName(tree->AsPhysReg()->gtSrcReg)); break; diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 60477341da53e0..5ace23683f7519 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -498,6 +498,7 @@ enum GenTreeFlags : unsigned GTF_ADDRMODE_NO_CSE = 0x80000000, // GT_ADD/GT_MUL/GT_LSH/GT_CAST -- Do not CSE this node only, forms complex // addressing mode + GTF_ADD_CARRY_FLAGS = 0x04000000, // GT_ADD -- CF is consumed; INC/DEC cannot implement this add GTF_MUL_64RSLT = 0x40000000, // GT_MUL -- produce 64-bit result GTF_RELOP_NAN_UN = 0x80000000, // GT_ -- Is branch taken if ops are NaN? @@ -528,6 +529,8 @@ enum GenTreeFlags : unsigned GTF_DIV_MOD_NO_OVERFLOW = 0x40000000, // GT_DIV, GT_MOD -- Div or mod definitely does not overflow. + GTF_DIV_REM_PAIR = 0x08000000, // GT_[U]DIV, GT_PHYSREG -- adjacent quotient/remainder definitions on xarch. + GTF_ARRLEN_NONFAULTING = 0x20000000, // GT_ARR_LENGTH -- An array length operation that cannot fault. Same as GT_IND_NONFAULTING. GTF_MDARRLEN_NONFAULTING = 0x20000000, // GT_MDARR_LENGTH -- An MD array length operation that cannot fault. Same as GT_IND_NONFAULTING. @@ -537,6 +540,7 @@ enum GenTreeFlags : unsigned GTF_ALLOCOBJ_EMPTY_STATIC = 0x80000000, // GT_ALLOCOBJ -- allocation site is part of an empty static pattern #ifdef FEATURE_HW_INTRINSICS + GTF_HW_MULX = 0x08000000, // GT_HWINTRINSIC -- unsigned BigMul in a matched multiply-carry chain uses MULX GTF_HW_EM_OP = 0x10000000, // GT_HWINTRINSIC -- node is used as an operand to an embedded mask GTF_HW_USER_CALL = 0x20000000, // GT_HWINTRINSIC -- node is implemented via a user call #endif // FEATURE_HW_INTRINSICS @@ -1861,6 +1865,18 @@ struct GenTree bool OperConsumesFlags() const { +#ifdef TARGET_AMD64 + if (OperIs(GT_ADCX, GT_ADOX, GT_ADX_DRAIN)) + { + return true; + } +#endif +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + if (OperIs(GT_ADD_CARRY, GT_SUB_BORROW, GT_ADD_BORROW)) + { + return true; + } +#endif #if !defined(TARGET_64BIT) if (OperIs(GT_ADD_HI, GT_SUB_HI)) { @@ -2466,6 +2482,12 @@ struct GenTree bool gtOverflow() const; bool gtOverflowEx() const; bool gtSetFlags() const; + bool IsFunnelShift() const; + + bool IsDivRemPair() const + { + return OperIs(GT_DIV, GT_UDIV, GT_PHYSREG) && ((gtFlags & GTF_DIV_REM_PAIR) != 0); + } #ifdef DEBUG static int gtDispFlags(GenTreeFlags flags, GenTreeDebugFlags debugFlags); @@ -9528,6 +9550,8 @@ struct GenTreeOpCC : public GenTreeOp { #ifdef TARGET_ARM64 assert(OperIs(GT_SELECTCC, GT_SELECT_INCCC, GT_SELECT_INVCC, GT_SELECT_NEGCC)); +#elif defined(TARGET_AMD64) + assert(OperIs(GT_SELECTCC, GT_JCMP)); #else assert(OperIs(GT_SELECTCC)); #endif diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index 622a9643e7db0b..9f638d92665198 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -391,6 +391,21 @@ GTNODE(RELOAD , GenTreeCopyOrReload,0,0,GTK_UNOP|DBK_NOTHIR) GTNODE(IL_OFFSET , GenTreeILOffset ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTHIR) // marks an IL offset for debugging purposes GTNODE(RECORD_ASYNC_RESUME, GenTreeVal ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTHIR) // record native offset for async resumption info +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) +#ifdef TARGET_AMD64 +// Independent unsigned carry chains, introduced only after whole-loop validation. +GTNODE(ADCX , GenTreeOp ,1,0,GTK_BINOP|DBK_NOTHIR) +GTNODE(ADOX , GenTreeOp ,1,0,GTK_BINOP|DBK_NOTHIR) +GTNODE(ADX_SEED , GenTree ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTHIR) +GTNODE(ADX_DRAIN , GenTreeOp ,0,0,GTK_UNOP|DBK_NOTHIR) +#endif +// Adds two operands and the carry flag. Only introduced during lowering. +GTNODE(ADD_CARRY , GenTreeOp ,1,0,GTK_BINOP|DBK_NOTHIR) +// Subtract op2 and borrow; add op2 and subtraction borrow to op1 (op2 is zero on arm64). +GTNODE(SUB_BORROW , GenTreeOp ,0,0,GTK_BINOP|DBK_NOTHIR) +GTNODE(ADD_BORROW , GenTreeOp ,0,0,GTK_BINOP|DBK_NOTHIR) +#endif + /*****************************************************************************/ #undef GTNODE /*****************************************************************************/ diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index d84f1dfa732107..d1150a9d1e97b6 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -973,6 +973,7 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = { { NI_Illegal, NI_Illegal }, // SHA_X64 { NI_Illegal, NI_Illegal }, // WAITPKG_X64 { NI_Illegal, NI_Illegal }, // X86Serialize_X64 + { NI_Illegal, NI_Illegal }, // ADX #elif defined (TARGET_ARM64) { FIRST_NI_ArmBase, LAST_NI_ArmBase }, // ArmBase { FIRST_NI_AdvSimd, LAST_NI_AdvSimd }, // AdvSimd diff --git a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp index a2362695a6716b..f0a4ce975da303 100644 --- a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp @@ -2596,6 +2596,15 @@ void CodeGen::genX86BaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions) emitAttr attr = emitTypeSize(baseType); + if ((node->gtFlags & GTF_HW_MULX) != 0) + { + assert(!rmOp->isUsedFromReg() || (rmOp->GetRegNum() != REG_EDX) || (regOp->GetRegNum() == REG_EDX)); + emit->emitIns_Mov(INS_mov, attr, REG_EDX, regOp->GetRegNum(), /* canSkip */ true); + inst_RV_RV_TT(INS_mulx, attr, node->GetRegByIndex(1), node->GetRegByIndex(0), rmOp, + /* isRMW */ false, INS_OPTS_NONE); + break; + } + // If rmOp is already in EAX, use that as implicit operand if (rmOp->isUsedFromReg() && rmOp->GetRegNum() == REG_EAX) { diff --git a/src/coreclr/jit/inductionvariableopts.cpp b/src/coreclr/jit/inductionvariableopts.cpp index cbc9751324ebb3..55cb41c14d643e 100644 --- a/src/coreclr/jit/inductionvariableopts.cpp +++ b/src/coreclr/jit/inductionvariableopts.cpp @@ -1187,6 +1187,65 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte } bool checkProfitability = !compStressCompile(STRESS_DOWNWARDS_COUNTED_LOOPS, 50); +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + bool preferCarryCountdown = false; + // A countdown can keep carry or borrow flags live across arithmetic loops, + // even when their addressing IV remains useful after the loop. + // Leave trip-count legality to SCEV below and the carry proof to lowering. + if (checkProfitability && removableLocals.Height() == 0 && compHndBBtabCount == 0 && loop->GetHeader() == exiting && + exiting->GetTrueTarget() == exiting) + { + unsigned products = 0; + unsigned carries = 0; + unsigned nodes = 0; + unsigned differences = 0; + for (Statement* stmt : exiting->Statements()) + { + for (GenTree* node : stmt->TreeList()) + { + if (++nodes > 512) + { + break; + } + if (node->OperIs(GT_HWINTRINSIC) && + node->AsHWIntrinsic()->GetHWIntrinsicId() == +#ifdef TARGET_AMD64 + NI_X86Base_X64_BigMul && +#else + NI_ArmBase_Arm64_MultiplyHigh && +#endif + node->AsHWIntrinsic()->GetSimdBaseType() == TYP_ULONG) + { + products++; + } + if (node->OperIs(GT_LT, GT_GT, GT_GE, GT_LE) && (node->gtFlags & GTF_UNSIGNED) != 0) + { + carries++; + } + if (node->OperIs(GT_SUB) && !node->gtOverflow() && node->TypeIs(TYP_INT, TYP_LONG)) + { + differences++; + } + } + if (nodes > 512) + { + break; + } + } + bool multiply = products >= 2 && products <= 4 && carries == 2 * products; +#ifdef TARGET_AMD64 + multiply = multiply && compOpportunisticallyDependsOn(InstructionSet_ADX) && + compOpportunisticallyDependsOn(InstructionSet_AVX2); +#endif + bool subtract = products == 0 && differences == 2 && carries == 2; + if (nodes <= 512 && (multiply || subtract)) + { + JITDUMP(" Keeping a countdown for a potential carry/borrow loop\n"); + checkProfitability = false; + preferCarryCountdown = true; + } + } +#endif if (checkProfitability && (removableLocals.Height() <= 0)) { JITDUMP(" Found no potentially removable locals when making this loop downwards counted\n"); @@ -1199,8 +1258,65 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte return false; } - Scev* backedgeCount = scevContext.ComputeExitNotTakenCount(exiting); - if (backedgeCount == nullptr) + Scev* backedgeCount = scevContext.ComputeExitNotTakenCount(exiting); + GenTree* tripCountNode = nullptr; +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + // SCEV's general trip-count materialization currently handles only unit + // strides. Recognize the bounded four-lane case: i starts at zero, steps + // by four, and continues while i < length - 3 for a nonnegative length. + // Its trip count is length / 4; the induction variable cannot overflow. + if (backedgeCount == nullptr && preferCarryCountdown && cond->OperIs(GT_LT) && !cond->IsUnsigned() && + cond->gtGetOp1()->TypeIs(TYP_INT) && cond->gtGetOp2()->TypeIs(TYP_INT)) + { + Scev* iv = scevContext.Analyze(exiting, cond->gtGetOp1()); + if (iv != nullptr) + { + iv = scevContext.Simplify(iv); + } + Scev* bound = scevContext.Analyze(exiting, cond->gtGetOp2()); + int64_t start, step; + if (iv != nullptr && iv->OperIs(ScevOper::AddRec) && bound != nullptr && bound->IsInvariant() && + ((ScevAddRec*)iv)->Start->GetConstantValue(this, &start) && start == 4 && + ((ScevAddRec*)iv)->Step->GetConstantValue(this, &step) && step == 4) + { + auto resolve = [this](GenTree* node) { + for (unsigned depth = 0; depth < 8 && node->OperIs(GT_LCL_VAR); depth++) + { + GenTreeLclVar* local = node->AsLclVar(); + if (!local->HasSsaName()) + { + break; + } + GenTreeLclVarCommon* def = lvaGetDesc(local)->GetPerSsaData(local->GetSsaNum())->GetDefNode(); + if (def == nullptr || def->GetLclNum() != local->GetLclNum()) + { + break; + } + node = def->Data(); + } + return node; + }; + GenTree* limit = resolve(cond->gtGetOp2()); + if (limit->OperIs(GT_ADD) && limit->gtGetOp2()->IsIntegralConst(-3) && + IntegralRange::ForNode(resolve(limit->gtGetOp1()), this).IsNonNegative()) + { + ValueNum limitVN = vnStore->VNLiberalNormalValue(cond->gtGetOp2()->gtVNPair); + ValueNum positive = vnStore->VNForFunc(TYP_INT, VNF_GT, limitVN, vnStore->VNForIntCon(0)); + if (scevContext.EvaluateRelop(positive) == RelopEvaluationResult::True) + { + GenTree* limitNode = scevContext.Materialize(bound); + if (limitNode != nullptr) + { + GenTree* length = gtNewOperNode(GT_ADD, TYP_INT, limitNode, gtNewIconNode(3)); + tripCountNode = gtNewOperNode(GT_RSZ, TYP_INT, length, gtNewIconNode(2)); + optFoldFourLimbOffsets(scevContext, exiting); + } + } + } + } + } +#endif + if (backedgeCount == nullptr && tripCountNode == nullptr) { JITDUMP(" Could not compute backedge count -- not a counted loop\n"); return false; @@ -1213,9 +1329,12 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte // to add one to the computed backedge count, giving us the trip count of // the loop. We do not need to worry about overflow here (even with // wraparound we have the right behavior). - Scev* tripCount = scevContext.Simplify( - scevContext.NewBinop(ScevOper::Add, backedgeCount, scevContext.NewConstant(backedgeCount->Type, 1))); - GenTree* tripCountNode = scevContext.Materialize(tripCount); + if (tripCountNode == nullptr) + { + Scev* tripCount = scevContext.Simplify( + scevContext.NewBinop(ScevOper::Add, backedgeCount, scevContext.NewConstant(backedgeCount->Type, 1))); + tripCountNode = scevContext.Materialize(tripCount); + } if (tripCountNode == nullptr) { JITDUMP(" Could not materialize trip count into IR\n"); @@ -1239,9 +1358,9 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte exitOp = GT_NE; } - GenTree* negOne = tripCount->TypeIs(TYP_LONG) ? gtNewLconNode(-1) : gtNewIconNode(-1, tripCount->Type); - GenTree* decremented = - gtNewOperNode(GT_ADD, tripCount->Type, gtNewLclVarNode(tripCountLcl, tripCount->Type), negOne); + var_types countType = tripCountNode->TypeGet(); + GenTree* negOne = countType == TYP_LONG ? gtNewLconNode(-1) : gtNewIconNode(-1, countType); + GenTree* decremented = gtNewOperNode(GT_ADD, countType, gtNewLclVarNode(tripCountLcl, countType), negOne); store = gtNewTempStore(tripCountLcl, decremented); @@ -1253,8 +1372,8 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte // Update the test. cond->SetOper(exitOp); - cond->AsOp()->gtOp1 = gtNewLclVarNode(tripCountLcl, tripCount->Type); - cond->AsOp()->gtOp2 = gtNewZeroConNode(tripCount->Type); + cond->AsOp()->gtOp1 = gtNewLclVarNode(tripCountLcl, countType); + cond->AsOp()->gtOp2 = gtNewZeroConNode(countType); gtSetStmtInfo(jtrueStmt); fgSetStmtSeq(jtrueStmt); @@ -1267,6 +1386,171 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte return true; } +//------------------------------------------------------------------------ +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) +// optFoldFourLimbOffsets: Share a widened integer index among the lane addresses +// of a proven four-limb loop. The caller proves i starts at zero, steps by four, +// and executes only while i < length - 3, for a nonnegative signed int length. +// Thus i + k cannot wrap for k in [0, 3]. Managed bases remain unchanged. +void Compiler::optFoldFourLimbOffsets(ScalarEvolutionContext& scevContext, BasicBlock* block) +{ + auto resolve = [this](GenTree* node) { + for (unsigned depth = 0; depth < 8; depth++) + { + if (node->OperIs(GT_COMMA)) + { + node = node->gtGetOp2(); + continue; + } + if (!node->OperIs(GT_LCL_VAR)) + { + break; + } + GenTreeLclVar* local = node->AsLclVar(); + if (!local->HasSsaName()) + { + break; + } + GenTreeLclVarCommon* def = lvaGetDesc(local)->GetPerSsaData(local->GetSsaNum())->GetDefNode(); + if ((def == nullptr) || (def->GetLclNum() != local->GetLclNum()) || def->Data()->OperIs(GT_PHI)) + { + break; + } + node = def->Data(); + } + return node; + }; + + struct Address + { + GenTreeOp* Node; + Statement* Stmt; + unsigned Offset; + }; + jitstd::vector
addresses(getAllocator(CMK_LoopOpt)); + jitstd::vector stores(getAllocator(CMK_LoopOpt)); + GenTreeLclVar* index = nullptr; + Statement* firstStmt = nullptr; + for (Statement* stmt : block->Statements()) + { + if (stmt->IsPhiDefnStmt()) + { + continue; + } + if (firstStmt == nullptr) + { + firstStmt = stmt; + } + for (GenTree* node : stmt->TreeList()) + { + if (node->OperIsLocalStore()) + { + stores.push_back(node->AsLclVarCommon()->GetLclNum()); + } + if (!node->OperIs(GT_ADD) || !node->TypeIs(TYP_BYREF) || !node->gtGetOp1()->TypeIs(TYP_BYREF)) + { + continue; + } + GenTree* scaled = resolve(node->gtGetOp2()); + if (!scaled->OperIs(GT_LSH) || !scaled->TypeIs(TYP_LONG) || !scaled->gtGetOp2()->IsIntegralConst(3)) + { + continue; + } + GenTree* widened = resolve(scaled->gtGetOp1()); + if (!widened->OperIs(GT_CAST) || !widened->TypeIs(TYP_LONG) || !widened->IsUnsigned() || + widened->gtOverflow()) + { + continue; + } + GenTree* base = resolve(widened->AsCast()->CastOp()); + unsigned offset = 0; + if (base->OperIs(GT_ADD) && base->TypeIs(TYP_INT) && !base->gtOverflow() && base->gtGetOp2()->IsCnsIntOrI()) + { + ssize_t value = base->gtGetOp2()->AsIntCon()->IconValue(); + if ((value < 0) || (value > 3)) + { + continue; + } + offset = (unsigned)value; + base = base->gtGetOp1(); + } + if (!base->OperIs(GT_LCL_VAR) || !base->TypeIs(TYP_INT) || !base->AsLclVar()->HasSsaName()) + { + continue; + } + GenTreeLclVar* local = base->AsLclVar(); + if (lvaGetDesc(local)->IsAddressExposed() || lvaGetDesc(local)->lvIsStructField) + { + continue; + } + bool overwritten = false; + for (unsigned store : stores) + { + overwritten |= store == local->GetLclNum(); + } + if (overwritten || ((index != nullptr) && ((index->GetLclNum() != local->GetLclNum()) || + (index->GetSsaNum() != local->GetSsaNum())))) + { + continue; + } + Scev* recurrence = scevContext.Analyze(block, local); + int64_t start, step; + if ((recurrence == nullptr) || !recurrence->OperIs(ScevOper::AddRec) || + !((ScevAddRec*)recurrence)->Start->GetConstantValue(this, &start) || (start != 0) || + !((ScevAddRec*)recurrence)->Step->GetConstantValue(this, &step) || (step != 4)) + { + continue; + } + index = local; + addresses.push_back({node->AsOp(), stmt, offset}); + } + } + if (addresses.size() < 4) + { + return; + } + + unsigned temp = lvaGrabTemp(false DEBUGARG("Shared four-limb integer index")); + GenTree* widened = gtNewCastNode(TYP_LONG, gtCloneExpr(index), true, TYP_LONG); + Statement* initialization = fgNewStmtFromTree(gtNewTempStore(temp, widened)); + fgInsertStmtBefore(block, firstStmt, initialization); + gtSetStmtInfo(initialization); + fgSetStmtSeq(initialization); +#ifdef TARGET_ARM64 + // A64 has no base+index+displacement addressing mode. Share native-sized + // lane indices instead, avoiding a separate zero extension after each i+k. + unsigned laneTemps[4] = {temp, BAD_VAR_NUM, BAD_VAR_NUM, BAD_VAR_NUM}; + for (unsigned lane = 1; lane < 4; lane++) + { + laneTemps[lane] = lvaGrabTemp(false DEBUGARG("Four-limb lane index")); + GenTree* value = gtNewOperNode(GT_ADD, TYP_LONG, gtNewLclvNode(temp, TYP_LONG), gtNewLconNode(lane)); + Statement* init = fgNewStmtFromTree(gtNewTempStore(laneTemps[lane], value)); + fgInsertStmtBefore(block, firstStmt, init); + gtSetStmtInfo(init); + fgSetStmtSeq(init); + } +#endif + for (const Address& address : addresses) + { +#ifdef TARGET_ARM64 + GenTree* offset = + gtNewOperNode(GT_LSH, TYP_LONG, gtNewLclvNode(laneTemps[address.Offset], TYP_LONG), gtNewIconNode(3)); +#else + GenTree* scaled = gtNewOperNode(GT_LSH, TYP_LONG, gtNewLclvNode(temp, TYP_LONG), gtNewIconNode(3)); + GenTree* offset = gtNewOperNode(GT_ADD, TYP_LONG, scaled, gtNewLconNode(address.Offset * 8)); +#endif + // Preserve embedded CSE definitions and any other effects of the original offset. + GenTree* effects = nullptr; + gtExtractSideEffList(address.Node->gtOp2, &effects); + address.Node->gtOp2 = effects == nullptr ? offset : gtNewOperNode(GT_COMMA, TYP_LONG, effects, offset); + gtSetStmtInfo(address.Stmt); + fgSetStmtSeq(address.Stmt); + } + JITDUMP("Shared widened index V%02u for %u four-limb addresses in " FMT_BB "\n", temp, (unsigned)addresses.size(), + block->bbNum); +} +#endif // defined(TARGET_AMD64) || defined(TARGET_ARM64) + //------------------------------------------------------------------------ // optCanAndShouldChangeExitTest: // Check if the exit test can be rephrased to a downwards counted exit test diff --git a/src/coreclr/jit/instrsxarch.h b/src/coreclr/jit/instrsxarch.h index fcd6e9b3e06f6b..ba7d09541eeb65 100644 --- a/src/coreclr/jit/instrsxarch.h +++ b/src/coreclr/jit/instrsxarch.h @@ -1289,6 +1289,10 @@ INST3(sar, "sar", IUM_RW, 0x0038D2, BAD_CODE, INST3(sar_1, "sar", IUM_RW, 0x0038D0, 0x0038D0, 0x0038D0, 1C, 2X, INS_TT_NONE, Writes_OF | Writes_SF | Writes_ZF | Undefined_AF | Writes_PF | Writes_CF | INS_FLAGS_HasWBit | Encoding_REX2 | INS_FLAGS_HasNDD | INS_FLAGS_HasNF) INST3(sar_N, "sar", IUM_RW, 0x0038C0, 0x0038C0, BAD_CODE, 1C, 2X, INS_TT_NONE, Undefined_OF | Writes_SF | Writes_ZF | Undefined_AF | Writes_PF | Writes_CF | INS_FLAGS_HasWBit | Encoding_REX2 | INS_FLAGS_HasNDD | INS_FLAGS_HasNF) +// ADX carry chains. Keep these at the end of the three-encoding instruction table. +INST3(adcx, "adcx", IUM_RW, BAD_CODE, BAD_CODE, PSSE38(0x66, 0xF6), 1C, 2X, INS_TT_NONE, Reads_CF | Writes_CF) +INST3(adox, "adox", IUM_RW, BAD_CODE, BAD_CODE, PSSE38(0xF3, 0xF6), 1C, 2X, INS_TT_NONE, Reads_OF | Writes_OF) + // id nm um mr mi lat tp tt flags INST2(ret, "ret", IUM_RD, 0x0000C3, 0x0000C2, ILLEGAL, ILLEGAL, INS_TT_NONE, INS_FLAGS_None) INST2(loop, "loop", IUM_RD, BAD_CODE, 0x0000E2, ILLEGAL, ILLEGAL, INS_TT_NONE, INS_FLAGS_None) @@ -1409,6 +1413,9 @@ INST0(l_jg, "jg", IUM_RD, 0x008F0F, INST0(align, "align", IUM_RD, BAD_CODE, ILLEGAL, ILLEGAL, INS_TT_NONE, INS_FLAGS_None) INST0(data16, "data16", IUM_RD, 0x000066, ZERO, 4X, INS_TT_NONE, INS_FLAGS_None) +// Flags-preserving loop branch. +INST0(jrcxz, "jrcxz", IUM_RD, 0x0000E3, BRANCH_COND, 2X, INS_TT_NONE, INS_FLAGS_None) + /*****************************************************************************/ #undef INST0 #undef INST1 diff --git a/src/coreclr/jit/liveness.cpp b/src/coreclr/jit/liveness.cpp index 9198ab6c529d0e..b54c6fb80748d9 100644 --- a/src/coreclr/jit/liveness.cpp +++ b/src/coreclr/jit/liveness.cpp @@ -2461,6 +2461,9 @@ void Liveness::ComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VAR case GT_STORE_BLK: case GT_JCMP: case GT_JTEST: +#ifdef TARGET_AMD64 + case GT_ADX_SEED: +#endif case GT_JCC: case GT_JTRUE: case GT_RETURN: @@ -2653,6 +2656,13 @@ bool Liveness::TryRemoveNonLocalLIR(GenTree* node, LIR::Range* blockR } assert(!node->OperIsLocal()); + // A paired divide also defines RDX for the following physical-register + // capture. That implicit use must survive even when the quotient is dead + // and the division has been proven non-throwing. + if (node->OperIs(GT_DIV, GT_UDIV) && node->IsDivRemPair()) + { + return false; + } if (!node->IsValue() || node->IsUnusedValue()) { // We are only interested in avoiding the removal of nodes with direct side effects @@ -2669,7 +2679,7 @@ bool Liveness::TryRemoveNonLocalLIR(GenTree* node, LIR::Range* blockR return GenTree::VisitResult::Continue; }); - if (node->OperConsumesFlags() && node->gtPrev->gtSetFlags()) + if (node->OperConsumesFlags() && (node->gtPrev != nullptr) && node->gtPrev->gtSetFlags()) { node->gtPrev->gtFlags &= ~GTF_SET_FLAGS; } diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index bbd00eb1fa752c..5ce802839c337b 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -306,6 +306,10 @@ bool Lowering::IsSafeToMarkRegOptional(GenTree* parentNode, GenTree* childNode) void Lowering::LowerRange(GenTree* firstNode, GenTree* lastNode) { assert(lastNode != nullptr); +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + // A partial walk can have pre-existing flag consumers outside its range. + m_blockMayHaveSetCC = true; +#endif // Multiple possible behaviors of LowerNode are possible: // 1. The node being lowered may be removed @@ -375,6 +379,14 @@ GenTree* Lowering::LowerNode(GenTree* node) case GT_ADD: { +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + // Morph represents subtraction of a constant as addition of its negation. + if (m_compiler->opts.OptimizationEnabled() && node->TypeIs(TYP_INT, TYP_LONG) && + node->gtGetOp2()->IsCnsIntOrI() && (node->gtGetOp2()->AsIntCon()->IconValue() < 0)) + { + m_lastSubtractionBlock = m_block; + } +#endif GenTree* next = LowerAdd(node->AsOp()); if (next != nullptr) { @@ -383,6 +395,14 @@ GenTree* Lowering::LowerNode(GenTree* node) } break; +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + case GT_SUB_BORROW: + case GT_ADD_BORROW: + case GT_ADD_CARRY: + ContainCheckBinary(node->AsOp()); + break; +#endif + #if !defined(TARGET_64BIT) case GT_ADD_LO: case GT_ADD_HI: @@ -396,6 +416,17 @@ GenTree* Lowering::LowerNode(GenTree* node) { if (m_compiler->opts.OptimizationEnabled()) { +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + if (node->OperIs(GT_SUB)) + { + m_lastSubtractionBlock = m_block; + GenTree* next = node->gtNext; + if (TryLowerAddCarry(node->AsOp())) + { + return next; + } + } +#endif GenTree* nextNode = nullptr; if (node->OperIs(GT_AND) && TryLowerAndNegativeOne(node->AsOp(), &nextNode)) { @@ -722,6 +753,12 @@ GenTree* Lowering::LowerNode(GenTree* node) break; #endif // defined(TARGET_WASM) +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + case GT_SETCC: + m_blockMayHaveSetCC = true; + break; +#endif + default: break; } @@ -4649,6 +4686,9 @@ GenTree* Lowering::OptimizeConstCompare(GenTree* cmp) } } GenTreeCC* setcc = m_compiler->gtNewCC(GT_SETCC, cmp->TypeGet(), cmpCondition); +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + m_blockMayHaveSetCC = true; +#endif BlockRange().InsertAfter(op1, setcc); use.ReplaceWith(setcc); @@ -4669,6 +4709,13 @@ GenTree* Lowering::OptimizeConstCompare(GenTree* cmp) // GenTree* Lowering::LowerCompare(GenTree* cmp) { +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + GenTree* next = cmp->gtNext; + if (m_compiler->opts.OptimizationEnabled() && TryLowerCarryCompare(cmp->AsOp())) + { + return next; + } +#endif #if LOWER_DECOMPOSE_LONGS if (cmp->gtGetOp1()->TypeIs(TYP_LONG)) { @@ -4974,6 +5021,12 @@ bool Lowering::TryLowerConditionToFlagsNode(GenTree* parent, { assert((condition->gtPrev->gtFlags & GTF_SET_FLAGS) != 0); GenTree* flagsDef = condition->gtPrev; + // A value-producing flags definition can have intervening LIR users. Moving + // it next to the condition consumer would move its value past those uses. + if (flagsDef->IsValue() && !flagsDef->IsUnusedValue()) + { + return false; + } #if defined(TARGET_ARM64) || defined(TARGET_AMD64) && !defined(TARGET_WASM) // CCMP is a flag producing node that also consumes flags, so find the // "root" of the flags producers and move the entire range. @@ -5071,10 +5124,9 @@ GenTreeCC* Lowering::LowerNodeCC(GenTree* node, GenCondition condition) if (next->OperIs(GT_JTRUE)) { // If the instruction immediately following 'relop', i.e. 'next' is a conditional branch, - // it should always have 'relop' as its 'op1'. If it doesn't, then we have improperly - // constructed IL (the setting of a condition code should always immediately precede its - // use, since the JIT doesn't track dataflow for condition codes). Still, if it happens - // it's not our problem, it simply means that `node` is not used and can be removed. + // it should have 'relop' as its 'op1'. Only convert that use to JCC. + // JCC can be separated from its flags producer by flag-preserving nodes; + // the JIT does not otherwise track dataflow for condition codes. if (next->AsUnOp()->gtGetOp1() == relop) { assert(relop->OperIsCompare()); @@ -5093,6 +5145,9 @@ GenTreeCC* Lowering::LowerNodeCC(GenTree* node, GenCondition condition) if (BlockRange().TryGetUse(relop, &use)) { cc = m_compiler->gtNewCC(GT_SETCC, TYP_INT, condition); +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + m_blockMayHaveSetCC = true; +#endif BlockRange().InsertAfter(node, cc); use.ReplaceWith(cc); } @@ -7944,6 +7999,13 @@ bool Lowering::TryCreateAddrMode(GenTree* addr, bool isContainable, GenTree* par // GenTree* Lowering::LowerAdd(GenTreeOp* node) { +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + GenTree* next = node->gtNext; + if (m_compiler->opts.OptimizationEnabled() && TryLowerAddCarry(node)) + { + return next; + } +#endif if (varTypeIsIntegralOrI(node->TypeGet())) { GenTree* op1 = node->gtGetOp1(); @@ -8765,6 +8827,9 @@ GenTree* Lowering::LowerSignedDivOrMod(GenTree* node) // void Lowering::LowerDivOrMod(GenTreeOp* divMod) { +#ifdef TARGET_XARCH + TryLowerDivRem(divMod); +#endif ContainCheckDivOrMod(divMod); } #endif // !TARGET_WASM @@ -8873,6 +8938,77 @@ void Lowering::TryRemoveShiftRotateMask(GenTreeOp* op) } } +//------------------------------------------------------------------------ +// TryContainFunnelShift: Combine complementary constant shifts of two values. +// +// Arguments: +// node - The binary node to check. +// +// Return Value: +// True if the node can be emitted as SHRD or EXTR. +// +bool Lowering::TryContainFunnelShift(GenTreeOp* node) +{ +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + if (!node->OperIs(GT_OR)) + { + return false; + } + if (node->IsFunnelShift()) + { + return true; + } + + if (!m_compiler->opts.OptimizationEnabled() || !node->TypeIs(TYP_INT, TYP_LONG) || node->gtSetFlags()) + { + return false; + } + + GenTree* right = node->gtGetOp1(); + GenTree* left = node->gtGetOp2(); + if (right->OperIs(GT_LSH)) + { + std::swap(right, left); + } + + if (!right->OperIs(GT_RSZ) || !left->OperIs(GT_LSH) || right->gtSetFlags() || left->gtSetFlags() || + !right->gtGetOp2()->IsCnsIntOrI() || !left->gtGetOp2()->IsCnsIntOrI() || + (right->TypeGet() != node->TypeGet()) || (left->TypeGet() != node->TypeGet())) + { + return false; + } + + ssize_t width = genTypeSize(node) * BITS_PER_BYTE; + ssize_t rightCount = right->gtGetOp2()->AsIntCon()->IconValue(); + ssize_t leftCount = left->gtGetOp2()->AsIntCon()->IconValue(); + if ((rightCount <= 0) || (rightCount >= width) || (leftCount != width - rightCount)) + { + return false; + } + + GenTree* lo = right->gtGetOp1(); + GenTree* hi = left->gtGetOp1(); + // Keep memory accesses at their original positions, and do not extend a local + // read past a redefinition. Both input values must be available in registers. + if (lo->isContained() || hi->isContained() || !IsInvariantInRange(lo, node) || !IsInvariantInRange(hi, node)) + { + return false; + } + + lo->ClearRegOptional(); + hi->ClearRegOptional(); + node->gtOp1 = right; + node->gtOp2 = left; + MakeSrcContained(node, right); + MakeSrcContained(node, left); + MakeSrcContained(right, right->gtGetOp2()); + MakeSrcContained(left, left->gtGetOp2()); + return true; +#else + return false; +#endif +} + //------------------------------------------------------------------------ // LowerShift: Lower shift nodes // @@ -9640,6 +9776,9 @@ void Lowering::LowerBlock(BasicBlock* block) assert(block->isEmpty() || block->IsLIR()); m_block = block; +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + m_blockMayHaveSetCC = false; +#endif #ifdef TARGET_ARM64 m_blockIndirs.Reset(); m_ffrTrashed = true; @@ -9667,6 +9806,22 @@ void Lowering::LowerBlock(BasicBlock* block) // void Lowering::AfterLowerBlocks() { +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + // Post-processing revisits blocks and inserts SETCCs before their consumers. + m_blockMayHaveSetCC = true; + if (m_compiler->opts.OptimizationEnabled() && m_hasAddCarry) + { + if (LowerFullAdders()) + { + LowerCarryChains(); + } +#ifdef TARGET_AMD64 + LowerMultiplyCarryLoops(); +#else + LowerArm64MultiplyCarryLoops(); +#endif + } +#endif } #endif // !TARGET_WASM @@ -9920,6 +10075,11 @@ void Lowering::ContainCheckNode(GenTree* node) case GT_ADD: case GT_SUB: +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + case GT_SUB_BORROW: + case GT_ADD_BORROW: + case GT_ADD_CARRY: +#endif #if !defined(TARGET_64BIT) case GT_ADD_LO: case GT_ADD_HI: @@ -13072,6 +13232,7 @@ bool Lowering::TryLowerAndOrToCCMP(GenTreeOp* tree, GenTree** next) ContainCheckConditionalCompare(ccmp); tree->SetOper(GT_SETCC); + m_blockMayHaveSetCC = true; tree->AsCC()->gtCondition = cond2; JITDUMP("Conversion was legal. Result:\n"); @@ -13390,3 +13551,2940 @@ void Lowering::SetFramePointerFromArgSpaceSize() m_compiler->codeGen->setFramePointerRequired(true); } } + +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) +//------------------------------------------------------------------------ +// IsCarryLocalDeadOnExit: Prove that the current local value cannot be read +// after leaving block. Lowering cannot use stale bbLiveOut sets. Walk current +// successors up to a full definition, including EH paths and parent aliases. +// Results are not cached across IR/CFG mutations. Budget exhaustion declines +// optimization; implicit OSR state and JMP argument uses are excluded. +// During the main lowering walk successors may not yet be lowered. This relies +// on lowering preserving observations of existing tracked scalar locals: it +// cannot introduce an earlier read of a killed value or turn its full def into +// a partial def. Promoted parents are not scalar carry locals and are excluded. +bool Lowering::IsCarryLocalDeadOnExit(unsigned local, BasicBlock* block) +{ + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (dsc->lvPromoted || dsc->IsAddressExposed() || m_compiler->opts.IsOSR() || + (m_compiler->compJmpOpUsed && dsc->lvIsParam) || + (m_compiler->lvaKeepAliveAndReportThis() && (local == m_compiler->info.compThisArg))) + { + return false; + } + unsigned visitedWords = m_compiler->fgBBNumMax / (sizeof(size_t) * 8) + 1; + if (visitedWords > m_carryLocalScanBudget) + { + return false; + } + m_carryLocalScanBudget -= visitedWords; + BitVecTraits traits(m_compiler->fgBBNumMax + 1, m_compiler); + BitVec visited = BitVecOps::MakeEmpty(&traits); + jitstd::vector pending(m_compiler->getAllocator(CMK_Lower)); + bool exhausted = false; + auto enqueue = [&](BasicBlock* successor) { + if (m_carryLocalScanBudget == 0) + { + exhausted = true; + return BasicBlockVisit::Abort; + } + m_carryLocalScanBudget--; + if (BitVecOps::IsMember(&traits, visited, successor->bbNum)) + { + return BasicBlockVisit::Continue; + } + BitVecOps::AddElemD(&traits, visited, successor->bbNum); + pending.push_back(successor); + return BasicBlockVisit::Continue; + }; + block->VisitRegularSuccs(m_compiler, enqueue); + block->VisitEHSuccs(m_compiler, enqueue); + for (unsigned index = 0; index < pending.size(); index++) + { + if (m_carryLocalScanBudget == 0) + { + return false; + } + m_carryLocalScanBudget--; + BasicBlock* current = pending[index]; + // A throw can reach the handler before any definition in this block. + current->VisitEHSuccs(m_compiler, enqueue); + bool killed = false; + for (GenTree* node : LIR::AsRange(current)) + { + if (m_carryLocalScanBudget == 0) + { + return false; + } + m_carryLocalScanBudget--; + if (!node->OperIsLocal()) + { + continue; + } + unsigned reference = node->AsLclVarCommon()->GetLclNum(); + if ((reference != local) && !(dsc->lvIsStructField && (reference == dsc->lvParentLcl))) + { + continue; + } + if (node->OperIs(GT_STORE_LCL_VAR)) + { + killed = true; + break; + } + // Reads, addresses and partial stores cannot prove the old value dead. + return false; + } + if (!killed) + { + current->VisitRegularSuccs(m_compiler, enqueue); + } + } + return !exhausted; +} + +//------------------------------------------------------------------------ +// FindCarryLoopEntry: Find the sole predecessor other than the loop itself. +BasicBlock* Lowering::FindCarryLoopEntry(BasicBlock* block) +{ + BasicBlock* entry = nullptr; + for (FlowEdge* edge : block->PredEdges()) + { + BasicBlock* pred = edge->getSourceBlock(); + if (pred != block) + { + if (entry != nullptr) + { + return nullptr; + } + entry = pred; + } + } + return entry; +} + +//------------------------------------------------------------------------ +// FindDeadCarryConstantStore: Identify a constant/store pair that can be +// removed from a flags-live range. Prove the local unobserved throughout the +// current method, excluding aliases and implicit uses absent from the LIR scan. +GenTree* Lowering::FindDeadCarryConstantStore(GenTree* constant) +{ + assert(constant->OperIs(GT_CNS_INT) && !constant->isContained()); + LIR::Use use; + if (!BlockRange().TryGetUse(constant, &use) || !use.User()->OperIs(GT_STORE_LCL_VAR)) + { + return nullptr; + } + unsigned local = use.User()->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (dsc->IsAddressExposed() || dsc->lvIsStructField || dsc->lvPromoted || m_compiler->opts.IsOSR() || + (m_compiler->compJmpOpUsed && dsc->lvIsParam) || + (m_compiler->lvaKeepAliveAndReportThis() && local == m_compiler->info.compThisArg)) + { + return nullptr; + } + for (BasicBlock* block : m_compiler->Blocks()) + { + for (GenTree* node : LIR::AsRange(block)) + { + if (m_carryUseScanBudget == 0) + { + return nullptr; + } + m_carryUseScanBudget--; + if (node->OperIsLocal() && node->AsLclVarCommon()->GetLclNum() == local && !node->OperIs(GT_STORE_LCL_VAR)) + { + return nullptr; + } + } + } + return use.User(); +} + +//------------------------------------------------------------------------ +// FindCarryLocalStore: Find the nearest full definition before a local read, +// within the fixed local search allowance. This does not prove single use or +// deadness on exit; callers establish those when deleting the definition. +// Keep short searches independent of the budget for extended carry-use scans. +GenTree* Lowering::FindCarryLocalStore(GenTree* read) +{ + assert(read->OperIs(GT_LCL_VAR)); + unsigned local = read->AsLclVar()->GetLclNum(); + GenTree* node = read->gtPrev; + for (unsigned remaining = 64; (node != nullptr) && (remaining != 0); node = node->gtPrev, remaining--) + { + if (node->OperIs(GT_STORE_LCL_VAR) && (node->AsLclVar()->GetLclNum() == local)) + { + return node; + } + } + return nullptr; +} + +//------------------------------------------------------------------------ +// TryLowerCarryCompare: Reuse arithmetic flags for unsigned carry/borrow comparisons. +// Match sum < operand, difference > minuend, or a < b after computing a-b. +// The arithmetic result can be forwarded or stored to a local in this block. +bool Lowering::TryLowerCarryCompare(GenTreeOp* cmp) +{ + auto match = [&](unsigned mode) -> bool { + // 0: sum comparison, 1: difference comparison, 2: subtraction operands, + // 3: negated value compared with zero (the borrow from 0 - value). + bool subtract = mode != 0; + bool negate = mode == 3; + + if (negate ? (!cmp->OperIs(GT_EQ, GT_NE) || !cmp->gtOp2->IsIntegralConst(0)) + : (!cmp->OperIs(GT_LT, GT_GT, GT_GE, GT_LE) || !cmp->IsUnsigned())) + { + return false; + } + + bool reversed = cmp->OperIs(GT_GT, GT_LE) != (mode == 1); + GenTree* sum = reversed ? cmp->gtOp2 : cmp->gtOp1; + GenTree* operand = reversed ? cmp->gtOp1 : cmp->gtOp2; + if ((!sum->OperIs(GT_LCL_VAR, negate ? GT_NEG : (subtract ? GT_SUB : GT_ADD)) && + !(mode == 1 && sum->OperIs(GT_ADD))) || + !sum->TypeIs(TYP_INT, TYP_LONG) || !operand->OperIs(GT_LCL_VAR, GT_CNS_INT) || + (sum->TypeGet() != operand->TypeGet())) + { + return false; + } + + GenTree* add = sum; + if (mode == 2) + { + if (!sum->OperIs(GT_LCL_VAR)) + { + return false; + } + // a < b can reuse an already-computed a-b, as in UInt128 subtraction + // and subtract-multiply. Compare the reaching operand values as well. + add = cmp->gtPrev; + unsigned budget = 64; + for (; add != nullptr && budget != 0; add = add->gtPrev, budget--) + { + if (m_carryUseScanBudget == 0) + { + return false; + } + m_carryUseScanBudget--; + if (add->OperIs(GT_SUB) && GenTree::Compare(add->gtGetOp1(), sum) && + GenTree::Compare(add->gtGetOp2(), operand) && IsInvariantInRange(add->gtGetOp1(), cmp) && + IsInvariantInRange(add->gtGetOp2(), cmp)) + { + break; + } + } + if (add == nullptr || budget == 0) + { + return false; + } + } + else if (sum->OperIs(GT_LCL_VAR)) + { + if (m_compiler->lvaGetDesc(sum->AsLclVar()->GetLclNum())->IsAddressExposed()) + { + return false; + } + GenTree* def = FindCarryLocalStore(sum); + if (def == nullptr) + { + return false; + } + add = def->gtGetOp1(); + } + bool negatedConstant = (mode == 1) && add->OperIs(GT_ADD) && add->gtGetOp2()->IsCnsIntOrI() && + !add->gtGetOp2()->AsIntCon()->IsIconHandle() && + (add->gtGetOp2()->AsIntCon()->IconValue() < 0) && + (add->gtGetOp2()->AsIntCon()->IconValue() != INT64_MIN) && + (add->gtGetOp2()->AsIntCon()->IconValue() != INT32_MIN); + if ((!add->OperIs(negate ? GT_NEG : (subtract ? GT_SUB : GT_ADD)) && !negatedConstant) || + (!negate && add->gtOverflow()) || add->gtSetFlags() || add->isContained() || + (add->TypeGet() != sum->TypeGet())) + { + return false; + } +#ifdef TARGET_ARM64 + // MNEG does not set flags, even when its containing NEG requests them. + if (negate && add->gtGetOp1()->OperIs(GT_MUL)) + { + return false; + } +#endif + + GenTree* matched = negate ? add->gtGetOp1() : nullptr; + for (GenTree* input : add->Operands()) + { + if ((mode != 1 || input == add->gtGetOp1()) && GenTree::Compare(input, operand) && + IsInvariantInRange(input, cmp)) + { + matched = input; + break; + } + } + LIR::Use use; + if ((matched == nullptr) || !BlockRange().TryGetUse(cmp, &use)) + { + return false; + } + + // A select needs flags at the select itself. Materializing them here can + // introduce SETCC and TEST when the sum is still used or flags are clobbered. + // Preserve the existing compare-to-select lowering in that case. + if (use.User()->OperIs(GT_SELECT)) + { + return false; + } + + GenCondition condition = cmp->OperIs(GT_LT, GT_GT, GT_NE) ? GenCondition::C : GenCondition::NC; +#ifdef TARGET_ARM64 + if (subtract) + { + condition = GenCondition::Reverse(condition); + } +#endif + bool branch = use.User()->OperIs(GT_JTRUE); + if (branch) + { +#ifdef TARGET_AMD64 + // Unlike a carry chain, a branch saves only a comparison. Avoid sacrificing LEA + // or perturbing allocation in register-heavy methods for that small saving. + // LAST - FIRST excludes SP from the inclusive integer-register range. + unsigned intRegCount = m_compiler->get_REG_INT_LAST() - REG_INT_FIRST; + if (negatedConstant || (m_compiler->lvaTrackedCount > 2 * intRegCount)) + { + return false; + } +#endif + if (use.User() != cmp->gtNext) + { + return false; + } + // Keep the branch at the end of the block, and only cross nodes that + // cannot overwrite flags. JCC need not be adjacent to its flags producer. + // Constants may become XOR on x64. + for (GenTree* node = add->gtNext; node != cmp; node = node->gtNext) + { + if ((node == nullptr) || + ((node != operand) && !node->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR, GT_IL_OFFSET))) + { + return false; + } + } + } + + JITDUMP("Replacing carry comparison [%06u] with flags from [%06u]\n", cmp->gtTreeID, add->gtTreeID); + if (negatedConstant) + { + // Restore SUB so its flags describe the borrow, rather than the carry + // from adding the two's-complement constant. The result is unchanged. + add->ChangeOper(GT_SUB); + GenTreeIntCon* constant = add->gtGetOp2()->AsIntCon(); + constant->SetIconValue(-constant->IconValue()); + } + add->gtFlags |= GTF_SET_FLAGS; + if (add->OperIs(GT_ADD)) + { + add->gtFlags |= GTF_ADD_CARRY_FLAGS; + } + for (GenTree* input : add->Operands()) + { + input->ClearContained(); + input->ClearRegOptional(); + } + if (negate) + { + ContainCheckNode(add); + } + else + { + ContainCheckBinary(add->AsOp()); + } + if (branch) + { + GenTree* jump = use.User(); + jump->ChangeOper(GT_JCC); + jump->AsCC()->gtCondition = condition; + } + else + { + GenTreeCC* carry = m_compiler->gtNewCC(GT_SETCC, TYP_INT, condition); + m_blockMayHaveSetCC = true; + BlockRange().InsertAfter(add, carry); + use.ReplaceWith(carry); + } + if (sum == add) + { + add->SetUnusedValue(); + } + else + { + BlockRange().Remove(sum); + } + BlockRange().Remove(operand); + BlockRange().Remove(cmp); + return true; + }; + return match(0) || ((m_lastSubtractionBlock == m_block) && (match(1) || match(2))) || match(3); +} + +#ifdef TARGET_AMD64 +//------------------------------------------------------------------------ +// ReplaceCarryAddressAdd: Replace a validated base-plus-constant ADD with LEA +// so the address calculation preserves both carry flags. +void Lowering::ReplaceCarryAddressAdd(GenTree* add) +{ + assert(add->OperIs(GT_ADD) && !add->gtOverflow() && !add->gtSetFlags()); + GenTree* offset = add->gtGetOp2(); + assert(offset->IsIntCnsFitsInI32()); + GenTree* lea = new (m_compiler, GT_LEA) + GenTreeAddrMode(add->TypeGet(), add->gtGetOp1(), nullptr, 1, (int)offset->AsIntCon()->IconValue()); + LIR::Use use; + bool found = BlockRange().TryGetUse(add, &use); + assert(found); + BlockRange().InsertBefore(add, lea); + use.ReplaceWith(lea); + BlockRange().Remove(add); + BlockRange().Remove(offset); +} + +//------------------------------------------------------------------------ +// LowerMultiplyCarryLoops: Keep both unsigned carry chains of scalar and +// unrolled widening multiply-accumulates in flags around counted loops. +// Memory containment checks interference; no bounds check is removed. +void Lowering::LowerMultiplyCarryLoops() +{ + if (m_compiler->compHndBBtabCount != 0 || !m_compiler->compOpportunisticallyDependsOn(InstructionSet_ADX) || + !m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + return; + } + + BasicBlock* savedBlock = m_block; + + // MULX is required while CF/OF are live: ordinary MUL overwrites both flags. + // Snapshot blocks since successful matches split edges. + jitstd::vector loops(m_compiler->getAllocator(CMK_Lower)); + for (BasicBlock* block : m_compiler->Blocks()) + { + loops.push_back(block); + } + for (BasicBlock* block : loops) + { + m_block = block; + bool hasAdx = TryLowerMultiplyCarryChain(); + if (block->KindIs(BBJ_COND) && block->GetTrueTarget() == block && block->GetFalseTarget() != block) + { + TryLowerMultiplyCarryBackedge(); + hasAdx |= TryLowerMultiplyCarryLoop(); + LowerMultiplyCarryMultiplier(); + } + if (hasAdx) + { + LowerMultiplyCarryLoads(); + } + } + m_block = savedBlock; +} + +//------------------------------------------------------------------------ +// LowerMultiplyCarryLoads: Forward single-use load temporaries into the memory +// operand of MULX and ADCX/ADOX after the carry patterns have been matched. +void Lowering::LowerMultiplyCarryLoads() +{ + for (GenTree* node : BlockRange()) + { + GenTree** operand = nullptr; + if (node->OperIs(GT_ADCX, GT_ADOX)) + { + operand = &node->AsOp()->gtOp2; + } + else if (node->OperIs(GT_HWINTRINSIC) && (node->AsHWIntrinsic()->GetHWIntrinsicId() == NI_X86Base_X64_BigMul) && + (node->AsHWIntrinsic()->GetSimdBaseType() == TYP_ULONG) && ((node->gtFlags & GTF_HW_MULX) != 0)) + { + operand = &node->AsHWIntrinsic()->Op(2); + } + if ((operand == nullptr) || !(*operand)->OperIs(GT_LCL_VAR)) + { + continue; + } + + GenTree* read = *operand; + unsigned local = read->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (!dsc->lvTracked || dsc->IsAddressExposed()) + { + continue; + } + + GenTree* store = nullptr; + bool valid = true; + for (GenTree* reference : BlockRange()) + { + if (m_carryUseScanBudget == 0) + { + return; + } + --m_carryUseScanBudget; + // A promoted field's value can also be observed or changed through its parent. + if (dsc->lvIsStructField && reference->OperIsLocal() && + (reference->AsLclVarCommon()->GetLclNum() == dsc->lvParentLcl)) + { + valid = false; + break; + } + if ((reference == read) && (store == nullptr)) + { + valid = false; + break; + } + if (reference->OperIsLocal() && (reference->AsLclVarCommon()->GetLclNum() == local)) + { + if (reference->OperIs(GT_STORE_LCL_VAR) && (store == nullptr)) + { + store = reference; + } + else if (reference != read) + { + valid = false; + break; + } + } + } + if (!valid || (store == nullptr)) + { + continue; + } + GenTree* load = store->gtGetOp1(); + if (!load->OperIs(GT_IND) || !load->TypeIs(TYP_LONG) || ((load->gtFlags & GTF_IND_VOLATILE) != 0) || + !IsSafeToContainMem(node, store, load)) + { + continue; + } + + if (!IsCarryLocalDeadOnExit(local, m_block)) + { + continue; + } + + JITDUMP("Containing load [%06u] from V%02u in [%06u]\n", load->gtTreeID, local, node->gtTreeID); + *operand = load; + // Both removed nodes precede the current iterator. The load stays in + // LIR; containment delays its execution only across the checked range. + BlockRange().Remove(store); + BlockRange().Remove(read); + MakeSrcContained(node, load); + } +} + +//------------------------------------------------------------------------ +// LowerMultiplyCarryMultiplier: Give a flags-preserving multiply loop its own +// copy of an invariant multiplier so its fixed RDX uses can share a register +// assignment independently of the incoming argument and the remainder loop. +void Lowering::LowerMultiplyCarryMultiplier() +{ + if (!BlockRange().LastNode()->OperIs(GT_JCMP)) + { + return; + } + + BasicBlock* entry = FindCarryLoopEntry(m_block); + if ((entry == nullptr) || !entry->KindIs(BBJ_ALWAYS) || (entry->GetTarget() != m_block)) + { + return; + } + // These loop transforms create a dedicated entry block containing only the flags seed. + // Insert before the seed so even a zero-initializing copy cannot interfere with the flags. + if ((LIR::AsRange(entry).FirstNode() != LIR::AsRange(entry).LastNode()) || + (LIR::AsRange(entry).FirstNode() == nullptr) || !LIR::AsRange(entry).FirstNode()->OperIs(GT_ADX_SEED)) + { + return; + } + + unsigned multiplier = BAD_VAR_NUM; + jitstd::vector products(m_compiler->getAllocator(CMK_Lower)); + for (GenTree* node : BlockRange()) + { + if (node->OperIs(GT_HWINTRINSIC)) + { + GenTreeHWIntrinsic* product = node->AsHWIntrinsic(); + if ((product->GetHWIntrinsicId() != NI_X86Base_X64_BigMul) || (product->GetSimdBaseType() != TYP_ULONG) || + ((product->gtFlags & GTF_HW_MULX) == 0) || !product->Op(1)->OperIs(GT_LCL_VAR) || + !product->Op(1)->TypeIs(TYP_LONG)) + { + return; + } + unsigned local = product->Op(1)->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (!dsc->lvTracked || dsc->IsAddressExposed() || dsc->lvIsStructField || + ((multiplier != BAD_VAR_NUM) && (multiplier != local))) + { + return; + } + multiplier = local; + products.push_back(product); + } + } + if (products.empty()) + { + return; + } + for (GenTree* node : BlockRange()) + { + if (node->OperIs(GT_STORE_LCL_VAR, GT_STORE_LCL_FLD) && (node->AsLclVarCommon()->GetLclNum() == multiplier)) + { + return; + } + } + + unsigned temp = m_compiler->lvaGrabTemp(true DEBUGARG("ADX loop multiplier")); + JITDUMP("Copying invariant multiplier V%02u to V%02u on entry to " FMT_BB "\n", multiplier, temp, m_block->bbNum); + m_compiler->lvaGetDesc(temp)->lvType = TYP_LONG; + GenTree* read = m_compiler->gtNewLclvNode(multiplier, TYP_LONG); + GenTree* store = m_compiler->gtNewStoreLclVarNode(temp, read); + LIR::AsRange(entry).InsertAtBeginning(read, store); + for (GenTreeHWIntrinsic* product : products) + { + GenTree* oldRead = product->Op(1); + GenTree* newRead = m_compiler->gtNewLclvNode(temp, TYP_LONG); + BlockRange().InsertBefore(oldRead, newRead); + product->Op(1) = newRead; + BlockRange().Remove(oldRead); + } +} + +//------------------------------------------------------------------------ +// TryLowerMultiplyCarryChain: Keep CF and OF across a bounded sequence of +// widening multiply-accumulates in one block. Seed after any preceding checks +// and drain before the final high word is consumed, so checks and loop control +// outside the sequence need not preserve either flag. No memory access moves. +// Returns true if an ADX chain was created; a scalar MULX-only match returns false. +bool Lowering::TryLowerMultiplyCarryChain() +{ + const unsigned maxLanes = 4; + struct Lane + { + GenTree* product; + GenTree* productStore; + GenTree* adc[2]; + GenTree* add[2]; + GenTree* low; + GenTree* destination; + GenTree* partial; + GenTree* carry; + } lanes[maxLanes] = {}; + + unsigned laneCount = 0; + unsigned adcCount = 0; + unsigned nodeCount = 0; + for (GenTree* node : BlockRange()) + { + if (++nodeCount > 512) + { + return false; + } + if (node->OperIs(GT_HWINTRINSIC)) + { + if (laneCount == maxLanes || adcCount != laneCount * 2 || + node->AsHWIntrinsic()->GetHWIntrinsicId() != NI_X86Base_X64_BigMul || + node->AsHWIntrinsic()->GetSimdBaseType() != TYP_ULONG) + { + return false; + } + lanes[laneCount++].product = node; + } + if (node->OperIs(GT_ADD_CARRY)) + { + if (laneCount == 0 || adcCount >= laneCount * 2 || !node->TypeIs(TYP_LONG) || node->gtSetFlags() || + !node->gtGetOp1()->OperIs(GT_LCL_VAR) || !node->gtGetOp2()->IsIntegralConst(0)) + { + return false; + } + lanes[adcCount / 2].adc[adcCount % 2] = node; + adcCount++; + } + } + if (laneCount == 0 || adcCount != laneCount * 2) + { + return false; + } + + // Changed intermediate sums/high words may have only one reader and must + // not escape this block. Requiring the store first also proves availability. + for (unsigned i = 0; i < laneCount; i++) + { + Lane& lane = lanes[i]; + LIR::Use productUse; + if (!BlockRange().TryGetUse(lane.product, &productUse) || !productUse.User()->OperIs(GT_STORE_LCL_VAR)) + { + return false; + } + lane.productStore = productUse.User(); + unsigned productLocal = lane.productStore->AsLclVar()->GetLclNum(); + LclVarDsc* fields = m_compiler->lvaGetDesc(productLocal); + if (!fields->lvPromoted || fields->lvFieldCnt != 2) + { + return false; + } + unsigned lowLocal = fields->lvFieldLclStart; + unsigned highLocal = lowLocal + 1; + for (unsigned j = 0; j < 2; j++) + { + GenTree* add = lane.adc[j]->gtPrev; + if (add == nullptr || !add->OperIs(GT_ADD) || !add->TypeIs(TYP_LONG) || add->gtOverflow() || + !add->gtSetFlags() || !add->gtGetOp1()->OperIs(GT_LCL_VAR) || !add->gtGetOp2()->OperIs(GT_LCL_VAR)) + { + return false; + } + lane.add[j] = add; + } + if (lane.adc[0]->gtGetOp1()->AsLclVar()->GetLclNum() != highLocal) + { + return false; + } + GenTree* highStore = FindMultiplyCarryStore(lane.adc[1]->gtGetOp1()); + if (highStore == nullptr || highStore->gtGetOp1() != lane.adc[0]) + { + return false; + } + lane.low = lane.add[0]->gtGetOp1(); + lane.destination = lane.add[0]->gtGetOp2(); + if (lane.low->AsLclVar()->GetLclNum() != lowLocal) + { + std::swap(lane.low, lane.destination); + } + if (lane.low->AsLclVar()->GetLclNum() != lowLocal) + { + return false; + } + lane.partial = lane.add[1]->gtGetOp1(); + lane.carry = lane.add[1]->gtGetOp2(); + GenTree* partialStore = FindMultiplyCarryStore(lane.partial); + if (partialStore == nullptr || partialStore->gtGetOp1() != lane.add[0]) + { + std::swap(lane.partial, lane.carry); + partialStore = FindMultiplyCarryStore(lane.partial); + } + if (partialStore == nullptr || partialStore->gtGetOp1() != lane.add[0]) + { + return false; + } + if (i != 0) + { + GenTree* carryStore = FindMultiplyCarryStore(lane.carry); + if (carryStore == nullptr || carryStore->gtGetOp1() != lanes[i - 1].adc[1]) + { + return false; + } + } + + // The two field reads must refer to this product, not to earlier or + // intervening assignments. A parent/field alias must not escape us. + bool defined = false; + for (GenTree* node : BlockRange()) + { + if (node == lane.productStore) + { + defined = true; + } + if ((node == lane.low || node == lane.adc[0]->gtGetOp1()) && !defined) + { + return false; + } + if (node->OperIsLocal()) + { + unsigned local = node->AsLclVarCommon()->GetLclNum(); + if ((local == lowLocal && node != lane.low) || + (local == highLocal && node != lane.adc[0]->gtGetOp1()) || + (local == productLocal && node != lane.productStore)) + { + return false; + } + } + } + + // Move only the carry read, from the second addition to the first. + // Unlike intermediate carries, the initial value may have other users. + unsigned carryLocal = lane.carry->AsLclVar()->GetLclNum(); + if (m_compiler->lvaGetDesc(carryLocal)->IsAddressExposed()) + { + return false; + } + for (GenTree* node = lane.add[0]; node != lane.carry; node = node->gtNext) + { + if (node == nullptr || (node->OperIsLocalStore() && node->AsLclVarCommon()->GetLclNum() == carryLocal)) + { + return false; + } + } + } + + if (laneCount == 1) + { + // The product feeds both halves of this proven multiply-accumulate. + // Keep MULX's independent outputs for scalar tails without introducing + // an ADX seed/drain pair for a single lane. + lanes[0].product->gtFlags |= GTF_HW_MULX; + return false; + } + + GenTree* first = lanes[0].add[0]; + GenTree* last = lanes[laneCount - 1].adc[1]; + jitstd::vector addresses(m_compiler->getAllocator(CMK_Lower)); + for (GenTree* node = first; node != last->gtNext; node = node->gtNext) + { + bool matched = false; + for (unsigned i = 0; i < laneCount; i++) + { + Lane& lane = lanes[i]; + matched |= node == lane.product || node == lane.add[0] || node == lane.add[1] || node == lane.adc[0] || + node == lane.adc[1]; + } + if (matched) + { + continue; + } + switch (node->OperGet()) + { + case GT_LCL_VAR: + case GT_STORE_LCL_VAR: + case GT_IL_OFFSET: + case GT_LEA: + break; + case GT_CAST: + if (node->gtOverflow() || !node->TypeIs(TYP_LONG) || !node->gtGetOp1()->TypeIs(TYP_INT)) + { + return false; + } + break; + case GT_IND: + case GT_STOREIND: + if (!node->TypeIs(TYP_LONG) || (node->gtFlags & GTF_IND_VOLATILE) != 0) + { + return false; + } + break; + case GT_CNS_INT: + if (!node->isContained()) + { + return false; + } + break; + case GT_ADD: + if (node->gtSetFlags() || node->gtOverflow() || !node->TypeIs(TYP_INT, TYP_LONG, TYP_BYREF) || + !node->gtGetOp1()->OperIs(GT_LCL_VAR) || !node->gtGetOp2()->IsIntCnsFitsInI32()) + { + return false; + } + addresses.push_back(node); + break; + default: + return false; + } + } + + JITDUMP("Keeping multiply-accumulate CF and OF chains across %u lanes in " FMT_BB "\n", laneCount, m_block->bbNum); + GenTree* initialize = new (m_compiler, GT_ADX_SEED) GenTree(GT_ADX_SEED, TYP_VOID); + BlockRange().InsertBefore(first, initialize); + for (unsigned i = 0; i < laneCount; i++) + { + Lane& lane = lanes[i]; + // Every product in the matched chain must preserve both live carry flags. + lane.product->gtFlags |= GTF_HW_MULX; + BlockRange().Remove(lane.carry); + BlockRange().InsertBefore(lane.add[0], lane.carry); + lane.add[0]->AsOp()->gtOp1 = lane.carry; + lane.add[0]->AsOp()->gtOp2 = lane.destination; + lane.add[1]->AsOp()->gtOp1 = lane.partial; + lane.add[1]->AsOp()->gtOp2 = lane.low; + lane.add[0]->ChangeOper(GT_ADCX); + lane.add[1]->ChangeOper(GT_ADOX); + for (GenTree* adc : lane.adc) + { + LIR::Use use; + bool found = BlockRange().TryGetUse(adc, &use); + assert(found); + GenTree* replacement = adc->gtGetOp1(); + if (adc == last) + { + replacement = m_compiler->gtNewOperNode(GT_ADX_DRAIN, TYP_LONG, replacement); + BlockRange().InsertBefore(adc, replacement); + } + use.ReplaceWith(replacement); + BlockRange().Remove(adc->gtGetOp2()); + BlockRange().Remove(adc); + } + } + for (GenTree* add : addresses) + { + ReplaceCarryAddressAdd(add); + } + return true; +} + +//------------------------------------------------------------------------ +// FindMultiplyCarryStore: Find a local's sole store and sole read in this block. +// Ordinary intermediates require a preceding store and cannot be live out. +// A loop carry may be read before its store and observed on the exit edge. +// No nodes are moved by this helper; callers check the transformed range. +GenTree* Lowering::FindMultiplyCarryStore(GenTree* read, bool loopCarry) +{ + if (!read->OperIs(GT_LCL_VAR)) + { + return nullptr; + } + unsigned local = read->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (!dsc->lvTracked || dsc->IsAddressExposed()) + { + return nullptr; + } + GenTree* store = nullptr; + unsigned reads = 0; + for (GenTree* node : BlockRange()) + { + if (dsc->lvIsStructField && node->OperIsLocal() && node->AsLclVarCommon()->GetLclNum() == dsc->lvParentLcl) + { + return nullptr; + } + if (node == read && store == nullptr && !loopCarry) + { + return nullptr; + } + if (node->OperIsLocal() && node->AsLclVarCommon()->GetLclNum() == local) + { + if (node->OperIs(GT_STORE_LCL_VAR)) + { + if (store != nullptr) + { + return nullptr; + } + store = node; + } + else if (node != read || ++reads != 1) + { + return nullptr; + } + } + } + if ((reads != 1) || (store == nullptr) || (!loopCarry && !IsCarryLocalDeadOnExit(local, m_block))) + { + return nullptr; + } + return store; +} + +//------------------------------------------------------------------------ +// TryLowerMultiplyCarryBackedge: Extend a proven in-block ADX chain around +// a countdown loop. Only the full-word carry may escape, and its two pending +// flags are materialized on the exit edge before any outside observer. +void Lowering::TryLowerMultiplyCarryBackedge() +{ + BasicBlock* block = m_block; + BasicBlock* entry = FindCarryLoopEntry(block); + if (entry == nullptr) + { + return; + } + + GenTree* branch = BlockRange().LastNode(); + GenTree* cmp = branch->gtPrev; + if (!branch->OperIs(GT_JCC) || branch->AsCC()->gtCondition.GetCode() != GenCondition::NE || cmp == nullptr || + !cmp->OperIs(GT_CMP) || !cmp->gtGetOp1()->OperIs(GT_LCL_VAR) || !cmp->gtGetOp1()->TypeIs(TYP_INT) || + !cmp->gtGetOp2()->IsIntegralConst(0)) + { + return; + } + + GenTree* initialize = nullptr; + GenTree* drain = nullptr; + GenTree* first = nullptr; + for (GenTree* node : BlockRange()) + { + if (node->OperIs(GT_ADX_SEED)) + { + if (initialize != nullptr) + { + return; + } + initialize = node; + } + if (node->OperIs(GT_ADX_DRAIN)) + { + if (drain != nullptr) + { + return; + } + drain = node; + } + if (node->OperIs(GT_ADCX) && first == nullptr) + { + first = node; + } + } + if (initialize == nullptr || drain == nullptr || first == nullptr || !first->gtGetOp1()->OperIs(GT_LCL_VAR)) + { + return; + } + // Resolve only a single, preceding definition with a single use. Changed + // intermediate values must not be observable outside this loop. + GenTree* carryRead = first->gtGetOp1(); + GenTree* carryStore = FindMultiplyCarryStore(carryRead, true); + if (carryStore == nullptr || !carryStore->gtGetOp1()->OperIs(GT_LCL_VAR)) + { + return; + } + GenTree* finalHighStore = FindMultiplyCarryStore(carryStore->gtGetOp1()); + if (finalHighStore == nullptr || finalHighStore->gtGetOp1() != drain) + { + return; + } + GenTree* countRead = cmp->gtGetOp1(); + unsigned countLocal = countRead->AsLclVar()->GetLclNum(); + GenTree* countStore = nullptr; + for (GenTree* node = cmp->gtPrev; node != nullptr; node = node->gtPrev) + { + if (node->OperIs(GT_STORE_LCL_VAR) && node->AsLclVar()->GetLclNum() == countLocal) + { + countStore = node; + break; + } + } + if (countStore == nullptr) + { + return; + } + GenTree* decrement = countStore->gtGetOp1(); + if (!decrement->OperIs(GT_ADD) || !decrement->TypeIs(TYP_INT) || decrement->gtSetFlags() || + decrement->gtOverflow() || !decrement->gtGetOp1()->OperIs(GT_LCL_VAR) || + decrement->gtGetOp1()->AsLclVar()->GetLclNum() != countLocal || !decrement->gtGetOp2()->IsIntegralConst(-1)) + { + return; + } + + jitstd::vector addresses(m_compiler->getAllocator(CMK_Lower)); + jitstd::vector deadNodes(m_compiler->getAllocator(CMK_Lower)); + bool sawCarryRead = false; + for (GenTree* node : BlockRange()) + { + if (node == carryRead) + { + sawCarryRead = true; + } + if (node == carryStore && !sawCarryRead) + { + return; + } + if (node == initialize || node == drain || node == cmp || node == branch) + { + continue; + } + switch (node->OperGet()) + { + case GT_ADCX: + case GT_ADOX: + case GT_LCL_VAR: + case GT_STORE_LCL_VAR: + case GT_IL_OFFSET: + case GT_LEA: + break; + case GT_HWINTRINSIC: + if (node->AsHWIntrinsic()->GetHWIntrinsicId() != NI_X86Base_X64_BigMul || + node->AsHWIntrinsic()->GetSimdBaseType() != TYP_ULONG || (node->gtFlags & GTF_HW_MULX) == 0) + { + return; + } + break; + case GT_CAST: + if (node->gtOverflow() || !node->TypeIs(TYP_LONG) || !node->gtGetOp1()->TypeIs(TYP_INT)) + { + return; + } + break; + case GT_IND: + case GT_STOREIND: + if (!node->TypeIs(TYP_LONG) || (node->gtFlags & GTF_IND_VOLATILE) != 0) + { + return; + } + break; + case GT_CNS_INT: + if (!node->isContained()) + { + GenTree* deadStore = FindDeadCarryConstantStore(node); + if (deadStore == nullptr) + { + return; + } + deadNodes.push_back(node); + deadNodes.push_back(deadStore); + } + break; + + case GT_ADD: + if (node->gtSetFlags() || node->gtOverflow() || !node->TypeIs(TYP_INT, TYP_LONG, TYP_BYREF) || + !node->gtGetOp1()->OperIs(GT_LCL_VAR) || !node->gtGetOp2()->IsIntCnsFitsInI32()) + { + return; + } + addresses.push_back(node); + break; + default: + return; + } + } + + JITDUMP("Keeping unrolled multiply-accumulate CF and OF across the backedge of " FMT_BB "\n", block->bbNum); + BasicBlock* seed = m_compiler->fgSplitEdge(entry, block); + BasicBlock* finish = m_compiler->fgSplitEdge(block, block->GetFalseTarget()); + m_compiler->fgInvalidateDfsTree(); + BlockRange().Remove(initialize); + LIR::AsRange(seed).InsertAtEnd(initialize); + finalHighStore->AsOp()->gtOp1 = drain->gtGetOp1(); + BlockRange().Remove(drain); + unsigned carryLocal = carryRead->AsLclVar()->GetLclNum(); + GenTree* finalRead = m_compiler->gtNewLclvNode(carryLocal, TYP_LONG); + drain->AsOp()->gtOp1 = finalRead; + GenTree* finalStore = m_compiler->gtNewStoreLclVarNode(carryLocal, drain); + LIR::AsRange(finish).InsertAtEnd(finalRead, drain, finalStore); + for (GenTree* dead : deadNodes) + { + BlockRange().Remove(dead); + } + for (GenTree* add : addresses) + { + ReplaceCarryAddressAdd(add); + } + // The countdown is integer arithmetic, but must use LEA too: DEC would + // destroy OF before the next iteration consumes the ADOX carry. + assert(countStore->gtGetOp1()->OperIs(GT_LEA)); + GenTree* jump = + new (m_compiler, GT_JCMP) GenTreeOpCC(GT_JCMP, TYP_VOID, GenCondition::NE, countRead, cmp->gtGetOp2()); + jump->gtFlags |= GTF_ORDER_SIDEEFF; + BlockRange().InsertBefore(branch, jump); + BlockRange().Remove(cmp); + BlockRange().Remove(branch); +} + +//------------------------------------------------------------------------ +// TryLowerMultiplyCarryLoop: Match two low-word ADD/high-word ADC pairs +// following one unsigned widening multiply. The full-word recurrence is +// carry' = productHigh + CF + OF. +// ADCX adds the old high word and destination; ADOX adds the product low word. +// Both flags start at zero and are drained into the final high word on exit. +// Returns true if an ADX loop was created. +bool Lowering::TryLowerMultiplyCarryLoop() +{ + BasicBlock* block = m_block; + BasicBlock* entry = FindCarryLoopEntry(block); + if (entry == nullptr) + { + return false; + } + + GenTree* branch = BlockRange().LastNode(); + GenTree* cmp = branch->gtPrev; + if (!branch->OperIs(GT_JCC) || branch->AsCC()->gtCondition.GetCode() != GenCondition::NE || cmp == nullptr || + !cmp->OperIs(GT_CMP) || !cmp->gtGetOp1()->OperIs(GT_LCL_VAR) || !cmp->gtGetOp1()->TypeIs(TYP_INT) || + !cmp->gtGetOp2()->IsIntegralConst(0)) + { + return false; + } + + GenTree* adcs[2] = {}; + GenTree* product = nullptr; + GenTree* productStore = nullptr; + unsigned count = 0; + unsigned adcCount = 0; + for (GenTree* node : BlockRange()) + { + if (++count > 256) + { + return false; + } + if (node->OperIs(GT_ADD_CARRY)) + { + if (adcCount == 2 || !node->TypeIs(TYP_LONG) || node->gtSetFlags() || + !node->gtGetOp1()->OperIs(GT_LCL_VAR) || !node->gtGetOp2()->IsIntegralConst(0)) + { + return false; + } + adcs[adcCount++] = node; + } + if (node->OperIs(GT_HWINTRINSIC)) + { + if (product != nullptr || node->AsHWIntrinsic()->GetHWIntrinsicId() != NI_X86Base_X64_BigMul || + node->AsHWIntrinsic()->GetSimdBaseType() != TYP_ULONG) + { + return false; + } + product = node; + } + } + if (adcCount != 2 || product == nullptr) + { + return false; + } + LIR::Use productUse; + if (!BlockRange().TryGetUse(product, &productUse) || !productUse.User()->OperIs(GT_STORE_LCL_VAR)) + { + return false; + } + productStore = productUse.User(); + LclVarDsc* fields = m_compiler->lvaGetDesc(productStore->AsLclVar()->GetLclNum()); + if (!fields->lvPromoted || fields->lvFieldCnt != 2) + { + return false; + } + unsigned lowLocal = fields->lvFieldLclStart; + unsigned highLocal = lowLocal + 1; + + // Resolve only a single, preceding definition with a single use. Changed + // intermediate values must not be observable outside this loop. + GenTree* first = adcs[0]->gtPrev; + GenTree* second = adcs[1]->gtPrev; + if (first == nullptr || second == nullptr || !first->OperIs(GT_ADD) || !second->OperIs(GT_ADD) || + !first->TypeIs(TYP_LONG) || !second->TypeIs(TYP_LONG) || first->gtOverflow() || second->gtOverflow() || + !first->gtSetFlags() || !second->gtSetFlags() || !first->gtGetOp1()->OperIs(GT_LCL_VAR) || + !first->gtGetOp2()->OperIs(GT_LCL_VAR) || !second->gtGetOp1()->OperIs(GT_LCL_VAR) || + !second->gtGetOp2()->OperIs(GT_LCL_VAR) || adcs[0]->gtGetOp1()->AsLclVar()->GetLclNum() != highLocal) + { + return false; + } + GenTree* highStore = FindMultiplyCarryStore(adcs[1]->gtGetOp1()); + if (highStore == nullptr || highStore->gtGetOp1() != adcs[0]) + { + return false; + } + GenTree* lowRead = first->gtGetOp1(); + GenTree* destination = first->gtGetOp2(); + if (lowRead->AsLclVar()->GetLclNum() != lowLocal) + { + std::swap(lowRead, destination); + } + if (lowRead->AsLclVar()->GetLclNum() != lowLocal) + { + return false; + } + GenTree* partial = second->gtGetOp1(); + GenTree* carryRead = second->gtGetOp2(); + GenTree* partialStore = FindMultiplyCarryStore(partial); + if (partialStore == nullptr || partialStore->gtGetOp1() != first) + { + std::swap(partial, carryRead); + partialStore = FindMultiplyCarryStore(partial); + } + if (partialStore == nullptr || partialStore->gtGetOp1() != first) + { + return false; + } + GenTree* carryStore = FindMultiplyCarryStore(carryRead, true); + if (carryStore == nullptr || !carryStore->gtGetOp1()->OperIs(GT_LCL_VAR)) + { + return false; + } + GenTree* finalHighStore = FindMultiplyCarryStore(carryStore->gtGetOp1()); + if (finalHighStore == nullptr || finalHighStore->gtGetOp1() != adcs[1]) + { + return false; + } + + GenTree* countRead = cmp->gtGetOp1(); + unsigned countLocal = countRead->AsLclVar()->GetLclNum(); + GenTree* countStore = nullptr; + for (GenTree* node = cmp->gtPrev; node != nullptr; node = node->gtPrev) + { + if (node->OperIs(GT_STORE_LCL_VAR) && node->AsLclVar()->GetLclNum() == countLocal) + { + countStore = node; + break; + } + } + if (countStore == nullptr) + { + return false; + } + GenTree* decrement = countStore->gtGetOp1(); + if (!decrement->OperIs(GT_ADD) || !decrement->TypeIs(TYP_INT) || decrement->gtSetFlags() || + decrement->gtOverflow() || !decrement->gtGetOp1()->OperIs(GT_LCL_VAR) || + decrement->gtGetOp1()->AsLclVar()->GetLclNum() != countLocal || !decrement->gtGetOp2()->IsIntegralConst(-1)) + { + return false; + } + + jitstd::vector addresses(m_compiler->getAllocator(CMK_Lower)); + jitstd::vector deadNodes(m_compiler->getAllocator(CMK_Lower)); + bool beforeFirst = true; + bool sawCarryRead = false; + bool sawCarryStore = false; + bool productDefined = false; + for (GenTree* node : BlockRange()) + { + if (node == productStore) + { + productDefined = true; + } + if ((node == lowRead || node == adcs[0]->gtGetOp1()) && !productDefined) + { + return false; + } + if (node->OperIsLocal()) + { + unsigned local = node->AsLclVarCommon()->GetLclNum(); + // The tuple fields must still be exactly this product's low/high + // words, with no intervening field assignment or extra observer. + if (((local == lowLocal) && (node != lowRead)) || ((local == highLocal) && (node != adcs[0]->gtGetOp1())) || + ((local == productStore->AsLclVar()->GetLclNum()) && (node != productStore))) + { + return false; + } + } + if (node == first) + { + beforeFirst = false; + } + if (node == carryRead) + { + sawCarryRead = true; + } + if (node == carryStore) + { + sawCarryStore = true; + if (!sawCarryRead) + { + return false; + } + } + if (node == product && !beforeFirst) + { + return false; + } + if (node == first || node == second || node == adcs[0] || node == adcs[1] || node == product || node == cmp || + node == branch) + { + continue; + } + switch (node->OperGet()) + { + case GT_LCL_VAR: + case GT_STORE_LCL_VAR: + case GT_IL_OFFSET: + case GT_LEA: + break; + case GT_IND: + case GT_STOREIND: + if (!node->TypeIs(TYP_LONG) || (node->gtFlags & GTF_IND_VOLATILE) != 0) + { + + return false; + } + break; + case GT_CNS_INT: + if (!node->isContained()) + { + GenTree* deadStore = FindDeadCarryConstantStore(node); + if (deadStore == nullptr) + { + return false; + } + deadNodes.push_back(node); + deadNodes.push_back(deadStore); + } + break; + case GT_ADD: + if (node->gtSetFlags() || node->gtOverflow() || !node->TypeIs(TYP_INT, TYP_LONG, TYP_BYREF) || + !node->gtGetOp1()->OperIs(GT_LCL_VAR) || !node->gtGetOp2()->IsIntCnsFitsInI32()) + { + + return false; + } + addresses.push_back(node); + break; + default: + + return false; + } + } + if (!sawCarryStore) + { + return false; + } + + product->gtFlags |= GTF_HW_MULX; + JITDUMP("Keeping multiply-accumulate CF and OF chains around " FMT_BB "\n", block->bbNum); + BasicBlock* exit = block->GetFalseTarget(); + BasicBlock* seed = m_compiler->fgSplitEdge(entry, block); + BasicBlock* finish = m_compiler->fgSplitEdge(block, exit); + m_compiler->fgInvalidateDfsTree(); + GenTree* initialize = new (m_compiler, GT_ADX_SEED) GenTree(GT_ADX_SEED, TYP_VOID); + LIR::AsRange(seed).InsertAtEnd(initialize); + unsigned carryLocal = carryRead->AsLclVar()->GetLclNum(); + GenTree* finalRead = m_compiler->gtNewLclvNode(carryLocal, TYP_LONG); + GenTree* drain = m_compiler->gtNewOperNode(GT_ADX_DRAIN, TYP_LONG, finalRead); + GenTree* finalStore = m_compiler->gtNewStoreLclVarNode(carryLocal, drain); + LIR::AsRange(finish).InsertAtEnd(finalRead, drain, finalStore); + + for (GenTree* dead : deadNodes) + { + BlockRange().Remove(dead); + } + BlockRange().Remove(carryRead); + BlockRange().InsertBefore(first, carryRead); + first->AsOp()->gtOp1 = carryRead; + first->AsOp()->gtOp2 = destination; + second->AsOp()->gtOp1 = partial; + second->AsOp()->gtOp2 = lowRead; + first->ChangeOper(GT_ADCX); + second->ChangeOper(GT_ADOX); + for (GenTree* adc : adcs) + { + LIR::Use use; + bool found = BlockRange().TryGetUse(adc, &use); + assert(found); + use.ReplaceWith(adc->gtGetOp1()); + BlockRange().Remove(adc->gtGetOp2()); + BlockRange().Remove(adc); + } + for (GenTree* add : addresses) + { + ReplaceCarryAddressAdd(add); + } + // The countdown is integer arithmetic, but must use LEA too: DEC would + // destroy OF before the next iteration consumes the ADOX carry. + assert(countStore->gtGetOp1()->OperIs(GT_LEA)); + GenTree* jump = + new (m_compiler, GT_JCMP) GenTreeOpCC(GT_JCMP, TYP_VOID, GenCondition::NE, countRead, cmp->gtGetOp2()); + jump->gtFlags |= GTF_ORDER_SIDEEFF; + BlockRange().InsertBefore(branch, jump); + BlockRange().Remove(cmp); + BlockRange().Remove(branch); + return true; +} +#endif // TARGET_AMD64 + +//------------------------------------------------------------------------ +// LowerFullAdders: Recognize full adders and subtractors after their carry/borrow comparisons have been lowered. +// Prove the loop-carried bit by induction over every definition of its local: constants are 0/1 and +// arithmetic definitions are other recognized full adders/subtractors. Boolean definitions +// (including widened comparisons in carry/borrow chains) are also accepted as bits. +// Returns true when a full adder or subtractor was folded. +bool Lowering::LowerFullAdders() +{ + // Bound the whole-method definition analysis and the number of candidates. + unsigned nodeCount = 0; + for (BasicBlock* block : m_compiler->Blocks()) + { + for (GenTree* node : LIR::AsRange(block)) + { + if (++nodeCount > 2048) + { + return false; + } + } + } + struct FullAdder + { + BasicBlock* block; + GenTree* output; + GenTree* first; + GenTree* second; + GenTree* firstStore; + GenTree* firstRead; + GenTree* carryStore; + GenTree* carryRead; + GenTree* carryCast; + GenTree* carryCC; + GenTree* input; + unsigned carryLocal; + }; + jitstd::vector adders(m_compiler->getAllocator(CMK_Lower)); + + for (BasicBlock* block : m_compiler->Blocks()) + { + m_block = block; + for (GenTree* output : LIR::AsRange(block)) + { + if (!output->OperIs(GT_ADD_CARRY, GT_ADD_BORROW) || output->gtSetFlags() || + !output->gtGetOp2()->IsIntegralConst(0) || !output->gtGetOp1()->OperIs(GT_LCL_VAR, GT_SETCC)) + { + continue; + } + bool subtract = output->OperIs(GT_ADD_BORROW); + GenTree* second = output->gtPrev; + if ((second == nullptr) || !second->OperIs(subtract ? GT_SUB : GT_ADD) || second->gtOverflow() || + !second->gtSetFlags() || !second->gtGetOp1()->OperIs(GT_LCL_VAR) || + !second->gtGetOp2()->OperIs(GT_LCL_VAR)) + { + continue; + } + + // Require a single store and a single read of each intermediate. + auto findStore = [&](GenTree* read) -> GenTree* { + unsigned local = read->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (!dsc->lvTracked || dsc->IsAddressExposed()) + { + return nullptr; + } + GenTree* store = nullptr; + int reads = 0; + for (GenTree* node : LIR::AsRange(block)) + { + if (dsc->lvIsStructField && node->OperIsLocal() && + node->AsLclVarCommon()->GetLclNum() == dsc->lvParentLcl) + { + return nullptr; + } + if (node->OperIsLocal() && node->AsLclVarCommon()->GetLclNum() == local) + { + if (node->OperIs(GT_STORE_LCL_VAR) && store == nullptr) + { + store = node; + } + else if ((node == read) && (store != nullptr)) + { + reads++; + } + else + { + return nullptr; + } + } + } + return (reads == 1) && IsCarryLocalDeadOnExit(local, block) ? store : nullptr; + }; + GenTree* carryRead = nullptr; + GenTree* carryStore = nullptr; + GenTree* cc = output->gtGetOp1(); + if (cc->OperIs(GT_LCL_VAR)) + { + carryRead = cc; + carryStore = findStore(carryRead); + if (carryStore == nullptr) + { + continue; + } + cc = carryStore->gtGetOp1(); + } + GenTree* cast = nullptr; + if (cc->OperIs(GT_CAST) && !cc->gtOverflow() && cc->TypeIs(TYP_LONG)) + { + cast = cc; + cc = cast->gtGetOp1(); + } + GenCondition condition = GenCondition::C; +#ifdef TARGET_ARM64 + if (subtract) + { + condition = GenCondition::NC; + } +#endif + if (!cc->OperIs(GT_SETCC) || (cc->AsCC()->gtCondition.GetCode() != condition.GetCode())) + { + continue; + } + GenTree* first = cc->gtPrev; + if (first == nullptr || !first->OperIs(subtract ? GT_SUB : GT_ADD) || !first->gtSetFlags() || + first->gtOverflow() || first->TypeGet() != second->TypeGet() || second->TypeGet() != output->TypeGet()) + { + continue; + } + + for (unsigned i = 0; i < (subtract ? 1u : 2u); i++) + { + GenTree* read = i == 0 ? second->gtGetOp1() : second->gtGetOp2(); + GenTree* input = i == 0 ? second->gtGetOp2() : second->gtGetOp1(); + GenTree* store = findStore(read); + if (store == nullptr || store->gtGetOp1() != first) + { + continue; + } + unsigned local = input->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (dsc->lvIsParam || dsc->IsAddressExposed()) + { + continue; + } + LIR::Use use; + if (!BlockRange().TryGetUse(output, &use) || !use.User()->OperIs(GT_STORE_LCL_VAR) || + use.User()->AsLclVar()->GetLclNum() != local) + { + continue; + } + if (adders.size() == 32) + { + return false; + } + adders.push_back( + {block, output, first, second, store, read, carryStore, carryRead, cast, cc, input, local}); + break; + } + } + } + + // Validate every candidate before changing any definitions. + jitstd::vector proven(m_compiler->getAllocator(CMK_Lower)); + jitstd::vector examined(m_compiler->getAllocator(CMK_Lower)); + bool changed = false; + for (const FullAdder& adder : adders) + { + bool alreadyExamined = false; + for (unsigned local : examined) + { + alreadyExamined |= local == adder.carryLocal; + } + if (alreadyExamined) + { + continue; + } + examined.push_back(adder.carryLocal); + bool valid = true; + bool initialized = false; + LclVarDsc* carryDsc = m_compiler->lvaGetDesc(adder.carryLocal); + for (BasicBlock* block : m_compiler->Blocks()) + { + for (GenTree* node : LIR::AsRange(block)) + { + if (carryDsc->lvIsStructField && node->OperIsLocal() && + node->AsLclVarCommon()->GetLclNum() == carryDsc->lvParentLcl) + { + valid = false; + break; + } + if (!node->OperIsLocal() || node->AsLclVarCommon()->GetLclNum() != adder.carryLocal) + { + continue; + } + if (node->OperIs(GT_LCL_VAR)) + { + continue; + } + if (!node->OperIs(GT_STORE_LCL_VAR)) + { + valid = false; + break; + } + GenTree* value = node->gtGetOp1(); + if (value->IsIntegralConst(0) || value->IsIntegralConst(1)) + { + initialized = true; + continue; + } + { + // The first limb of a straight-line chain initializes carry or borrow + // from a comparison. Subtraction tails can also redefine borrow as + // (limb == 0). Both are proven bits, including after widening. + GenTree* bit = value; + if (bit->OperIs(GT_CAST) && !bit->gtOverflow() && bit->TypeIs(TYP_LONG) && + bit->gtGetOp1()->TypeIs(TYP_INT)) + { + bit = bit->gtGetOp1(); + } + if (bit->OperIs(GT_SETCC) || bit->OperIsCompare()) + { + initialized = true; + continue; + } + } + bool fullAdder = false; + for (const FullAdder& definition : adders) + { + if (definition.carryLocal == adder.carryLocal && definition.output == value) + { + fullAdder = true; + } + } + if (!fullAdder) + { + valid = false; + } + if (!valid) + { + break; + } + } + if (!valid) + { + break; + } + } + if (valid && initialized) + { + proven.push_back(adder.carryLocal); + } + } + + for (const FullAdder& adder : adders) + { + bool valid = false; + for (unsigned local : proven) + { + if (local == adder.carryLocal) + { + valid = true; + } + } + if (!valid) + { + continue; + } + m_block = adder.block; + changed = true; + JITDUMP("Folding full adder [%06u] with one-bit carry V%02u\n", adder.second->gtTreeID, adder.carryLocal); + bool subtract = adder.first->OperIs(GT_SUB); + GenCondition condition = GenCondition::C; + GenTree* carryConstant = adder.output->gtGetOp2(); + carryConstant->ClearContained(); + carryConstant->ClearRegOptional(); + adder.input->ClearContained(); + adder.input->ClearRegOptional(); +#ifdef TARGET_AMD64 + // Adding -1 sets CF exactly when the proven carry bit is 1. + carryConstant->AsIntCon()->SetIconValue(-1); + GenTree* restore = m_compiler->gtNewOperNode(GT_ADD, adder.input->TypeGet(), adder.input, carryConstant); + restore->SetUnusedValue(); +#else + // Addition needs C = carry; subtraction needs C = !borrow. + carryConstant->AsIntCon()->SetIconValue(subtract ? 0 : 1); + GenTree* restore = m_compiler->gtNewOperNode(GT_CMP, TYP_VOID, subtract ? carryConstant : adder.input, + subtract ? adder.input : carryConstant); + if (subtract) + { + condition = GenCondition::NC; + } +#endif + restore->gtFlags |= GTF_SET_FLAGS; +#ifdef TARGET_AMD64 + restore->gtFlags |= GTF_ADD_CARRY_FLAGS; +#endif + BlockRange().Remove(carryConstant); + BlockRange().InsertBefore(adder.second, carryConstant); + BlockRange().InsertBefore(adder.second, restore); +#ifdef TARGET_ARM64 + if (!subtract) +#endif + { + MakeSrcContained(restore, carryConstant); + } + adder.second->ChangeOper(subtract ? GT_SUB_BORROW : GT_ADD_CARRY); + adder.second->AsOp()->gtOp1 = adder.first->gtGetOp1(); + adder.second->AsOp()->gtOp2 = adder.first->gtGetOp2(); + for (GenTree* operand : adder.second->Operands()) + { + operand->ClearContained(); + operand->ClearRegOptional(); + } +#ifdef TARGET_ARM64 + // CSET already defines the entire register as 0 or 1. + GenTree* cc = m_compiler->gtNewCC(GT_SETCC, adder.output->TypeGet(), condition); + GenTree* carryCast = nullptr; +#else + GenTree* cc = m_compiler->gtNewCC(GT_SETCC, TYP_INT, condition); + GenTree* carryCast = adder.carryCast; + assert((carryCast != nullptr) || adder.output->TypeIs(TYP_INT)); +#endif + BlockRange().InsertAfter(adder.second, cc); + if (adder.carryCast != nullptr) + { + BlockRange().Remove(adder.carryCast); + } + if (carryCast != nullptr) + { + carryCast->AsCast()->CastOp() = cc; + BlockRange().InsertAfter(cc, carryCast); + } + LIR::Use use; + bool found = BlockRange().TryGetUse(adder.output, &use); + assert(found); + GenTree* outputStore = use.User(); + use.ReplaceWith(carryCast != nullptr ? carryCast : cc); + for (GenTree* node : {adder.first, adder.firstStore, adder.firstRead, adder.carryStore, adder.carryRead, + adder.carryCC, adder.output}) + { + if (node != nullptr) + { + BlockRange().Remove(node); + } + } + ContainCheckBinary(adder.second->AsOp()); + TryLowerCarryLoop(adder.second, restore, cc, carryCast, outputStore, adder.carryLocal); + } + return changed; +} + +//------------------------------------------------------------------------ +// LowerCarryChains: Fold final-limb consumers, then remove single-use carry +// materialization and restoration between arithmetic stages in one block. +// The latter step only removes nodes; it does not move memory accesses. +// +void Lowering::LowerCarryChains() +{ + unsigned budget = 2048; + for (BasicBlock* block : m_compiler->Blocks()) + { + m_block = block; + for (GenTree* node = BlockRange().FirstNode(); node != nullptr;) + { + if (budget-- == 0) + { + return; + } + GenTree* next = node->gtNext; + // Full-adder lowering creates SETCCs after the ordinary lowering + // walk. Let the last limb consume such a carry directly as well. + if (node->OperIs(GT_ADD, GT_SUB)) + { + TryLowerAddCarry(node->AsOp()); + } + node = next; + } + + for (GenTree* consumer : BlockRange()) + { + if (!consumer->OperIs(GT_ADD_CARRY, GT_SUB_BORROW) || !consumer->gtSetFlags()) + { + continue; + } + bool subtract = consumer->OperIs(GT_SUB_BORROW); + GenTree* restore = consumer->gtPrev; + unsigned restoreBudget = 64; + // Folding the final limb can insert its operand reads between a + // stage's restore and arithmetic. These register copies do not + // change the restored flags. + while (restore != nullptr && restoreBudget != 0 && !restore->gtSetFlags() && + !restore->OperConsumesFlags() && + (restore->OperIs(GT_IL_OFFSET) || + (restore->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR) && restore->TypeIs(TYP_INT, TYP_LONG, TYP_BYREF)))) + { + restore = restore->gtPrev; + restoreBudget--; + } + if (restore == nullptr || restoreBudget == 0 || !restore->gtSetFlags()) + { + continue; + } + GenTree* read; + GenTree* constant; +#ifdef TARGET_AMD64 + if (!restore->OperIs(GT_ADD) || !restore->IsUnusedValue() || restore->gtOverflow()) + { + continue; + } + read = restore->gtGetOp1(); + constant = restore->gtGetOp2(); + if (!constant->IsIntegralConst(-1)) + { + continue; + } +#else + if (!restore->OperIs(GT_CMP)) + { + continue; + } + read = subtract ? restore->gtGetOp2() : restore->gtGetOp1(); + constant = subtract ? restore->gtGetOp1() : restore->gtGetOp2(); + if (!constant->IsIntegralConst(subtract ? 0 : 1)) + { + continue; + } +#endif + if (!read->OperIs(GT_LCL_VAR)) + { + continue; + } + unsigned local = read->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (!dsc->lvTracked || dsc->IsAddressExposed() || dsc->lvIsStructField || + !IsCarryLocalDeadOnExit(local, block)) + { + continue; + } + GenTree* store = FindCarryLocalStore(read); + if (store == nullptr) + { + continue; + } + GenTree* cc = store->gtGetOp1(); + GenTree* cast = nullptr; + if (cc->OperIs(GT_CAST) && !cc->gtOverflow() && cc->CastToType() == TYP_LONG && + cc->gtGetOp1()->TypeIs(TYP_INT)) + { + cast = cc; + cc = cc->gtGetOp1(); + } + GenCondition condition = GenCondition::C; +#ifdef TARGET_ARM64 + if (subtract) + { + condition = GenCondition::NC; + } +#endif + if (!cc->OperIs(GT_SETCC) || cc->AsCC()->gtCondition.GetCode() != condition.GetCode()) + { + continue; + } + GenTree* producer = cc->gtPrev; + if (producer == nullptr || !producer->gtSetFlags() || + (producer->OperIs(GT_ADD, GT_SUB) && producer->gtOverflow()) || + !(subtract ? producer->OperIs(GT_SUB, GT_SUB_BORROW) : producer->OperIs(GT_ADD, GT_ADD_CARRY))) + { + continue; + } + + // Only register copies and IL markers may cross the flags edge. + // In particular, constants, address arithmetic and loads are not + // assumed flags-preserving after containment and register allocation. + GenTree* crossing = producer->gtNext; + unsigned scan = 64; + for (; crossing != nullptr && crossing != restore && scan != 0; crossing = crossing->gtNext, scan--) + { + if (crossing == cc || crossing == cast || crossing == constant) + { + continue; + } + if (crossing->gtSetFlags() || crossing->OperConsumesFlags() || + !(crossing->OperIs(GT_IL_OFFSET) || (crossing->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR) && + crossing->TypeIs(TYP_INT, TYP_LONG, TYP_BYREF)))) + { + break; + } + } + if (crossing != restore) + { + continue; + } + + // The definition may be removed only if this is its sole read up + // to the next definition, and it cannot escape through a live-out. + GenTree* use = store->gtNext; + scan = 64; + for (; use != nullptr && scan != 0; use = use->gtNext, scan--) + { + if (use->OperIsLocal() && use->AsLclVarCommon()->GetLclNum() == local) + { + if (use->OperIs(GT_STORE_LCL_VAR)) + { + break; + } + if (use != read) + { + break; + } + } + } + if (scan == 0 || (use != nullptr && !use->OperIs(GT_STORE_LCL_VAR))) + { + continue; + } + JITDUMP("Forwarding carry flags from [%06u] to [%06u]\n", producer->gtTreeID, consumer->gtTreeID); + for (GenTree* removed : {cc, cast, store, read, constant, restore}) + { + if (removed != nullptr) + { + BlockRange().Remove(removed); + } + } + } + } +} + +//------------------------------------------------------------------------ +// TryLowerCarryLoop: Keep carry or borrow in flags around a simple counted loop. More general control flow +// retains the explicit carry/borrow representation produced by LowerFullAdders. +void Lowering::TryLowerCarryLoop( + GenTree* sum, GenTree* restore, GenTree* cc, GenTree* cast, GenTree* store, unsigned carryLocal) +{ + BasicBlock* block = m_block; + if (m_compiler->compHndBBtabCount != 0 || !block->KindIs(BBJ_COND) || + (block->GetTrueTarget() != block && block->GetFalseTarget() != block) || + (block->GetTrueTarget() == block->GetFalseTarget())) + { + return; + } + BasicBlock* entry = FindCarryLoopEntry(block); + if (entry == nullptr) + { + return; + } + GenTree* branch = BlockRange().LastNode(); +#ifdef TARGET_AMD64 + GenTree* cmp = branch->gtPrev; + if (!branch->OperIs(GT_JCC) || cmp == nullptr || !cmp->OperIs(GT_CMP) || !cmp->gtGetOp1()->OperIs(GT_LCL_VAR) || + !cmp->gtGetOp2()->IsIntegralConst(0)) + { + return; + } + GenCondition::Code condition = branch->AsCC()->gtCondition.GetCode(); +#else + // CBZ/CBNZ test the counter without changing NZCV. + GenTree* cmp = branch; + if (!branch->OperIs(GT_JCMP) || !cmp->gtGetOp1()->OperIs(GT_LCL_VAR) || !cmp->gtGetOp2()->IsIntegralConst(0)) + { + return; + } + GenCondition::Code condition = branch->AsOpCC()->gtCondition.GetCode(); +#endif + if (condition != GenCondition::EQ && condition != GenCondition::NE) + { + return; + } + unsigned countLocal = cmp->gtGetOp1()->AsLclVar()->GetLclNum(); + GenTree* countStore = cmp->gtPrev; + while (countStore != nullptr && !countStore->OperIs(GT_STORE_LCL_VAR)) + { + countStore = countStore->gtPrev; + } + if (countStore == nullptr || countStore->AsLclVar()->GetLclNum() != countLocal) + { + return; + } + GenTree* decrement = countStore->gtGetOp1(); + if (!decrement->OperIs(GT_ADD, GT_SUB) || decrement->gtSetFlags() || decrement->gtOverflow() || + !decrement->gtGetOp1()->OperIs(GT_LCL_VAR) || decrement->gtGetOp1()->AsLclVar()->GetLclNum() != countLocal || + !decrement->gtGetOp2()->IsIntegralConst(decrement->OperIs(GT_ADD) ? -1 : 1)) + { + return; + } + + bool sumBeforeDecrement = false; + for (GenTree* node = sum->gtNext; node != nullptr; node = node->gtNext) + { + if (node == decrement) + { + sumBeforeDecrement = true; + break; + } + } + if (!sumBeforeDecrement) + { + return; + } + + // Permit only instructions that preserve CF, plus additions we can turn + // into LEA and the loop decrement. In particular, reject calls, bounds + // checks, write barriers, and any other flag consumers or producers. + jitstd::vector addresses(m_compiler->getAllocator(CMK_Lower)); + jitstd::vector deadStores(m_compiler->getAllocator(CMK_Lower)); + for (GenTree* node : BlockRange()) + { + if (node == sum || node == restore || node == restore->gtGetOp1() || node == restore->gtGetOp2() || + node == cc || node == cast || node == cmp || node == decrement) + { + continue; + } + if (node->OperIsLocal() && node->AsLclVarCommon()->GetLclNum() == carryLocal && node != restore->gtGetOp1() && + node != restore->gtGetOp2() && node != store) + { + return; + } + switch (node->OperGet()) + { + case GT_LCL_VAR: + case GT_STORE_LCL_VAR: + case GT_IL_OFFSET: + case GT_LEA: + break; + case GT_CNS_INT: + if (!node->isContained() && node != cmp->gtGetOp2() && node != decrement->gtGetOp2()) + { + GenTree* deadStore = FindDeadCarryConstantStore(node); + if (deadStore == nullptr) + { + return; + } + deadStores.push_back(node); + deadStores.push_back(deadStore); + } + break; + case GT_IND: + case GT_STOREIND: + if (!node->AsIndir()->TypeIs(TYP_INT, TYP_LONG) || (node->gtFlags & GTF_IND_VOLATILE) != 0) + { + return; + } + break; + case GT_CAST: + // Widening an index preserves flags on both targets. The + // addressing IV can remain live for a subsequent borrow tail. + if (node->gtOverflow() || !node->TypeIs(TYP_LONG) || !node->gtGetOp1()->TypeIs(TYP_INT)) + { + return; + } + break; +#ifdef TARGET_ARM64 + case GT_LSH: + case GT_BFIZ: + // Unlike x64 shifts, A64 index scaling preserves NZCV. + if (node->gtSetFlags() || !node->TypeIs(TYP_INT, TYP_LONG)) + { + return; + } + break; +#endif + case GT_JCC: +#ifdef TARGET_ARM64 + case GT_JCMP: +#endif + if (node != branch) + { + return; + } + break; + case GT_ADD: + if (node->gtSetFlags() || node->gtOverflow() || !node->TypeIs(TYP_INT, TYP_LONG, TYP_BYREF)) + { + return; + } +#ifdef TARGET_ARM64 + if (node->TypeIs(TYP_LONG, TYP_BYREF) && node->gtGetOp1()->OperIs(GT_LCL_VAR) && + node->gtGetOp2()->OperIs(GT_LSH) && node->gtGetOp2()->TypeIs(TYP_LONG) && + node->gtGetOp2()->gtGetOp1()->TypeIs(TYP_LONG) && + node->gtGetOp2()->gtGetOp2()->OperIs(GT_CNS_INT) && + node->gtGetOp2()->gtGetOp2()->AsIntCon()->IconValue() >= 0 && + node->gtGetOp2()->gtGetOp2()->AsIntCon()->IconValue() <= 3) + { + addresses.push_back(node); + break; + } +#endif + if (!node->gtGetOp1()->OperIs(GT_LCL_VAR) || !node->gtGetOp2()->OperIs(GT_LCL_VAR, GT_CNS_INT)) + { +#ifdef TARGET_ARM64 + break; // An ordinary A64 ADD also preserves flags. +#else + return; +#endif + } + if (node->gtGetOp2()->OperIs(GT_CNS_INT) && !FitsIn(node->gtGetOp2()->AsIntCon()->IconValue())) + { + return; + } + addresses.push_back(node); + break; + default: + return; + } + } + + JITDUMP("Keeping full-adder carry in flags around " FMT_BB "\n", block->bbNum); + BasicBlock* exit = block->GetTrueTarget() == block ? block->GetFalseTarget() : block->GetTrueTarget(); + BasicBlock* seed = m_compiler->fgSplitEdge(entry, block); + BasicBlock* finish = m_compiler->fgSplitEdge(block, exit); + m_compiler->fgInvalidateDfsTree(); + for (GenTree* node : {restore->gtGetOp1(), restore->gtGetOp2(), restore}) + { + BlockRange().Remove(node); + LIR::AsRange(seed).InsertAtEnd(node); + } + for (GenTree* node : {cc, cast, store}) + { + if (node == nullptr) + { + continue; + } + BlockRange().Remove(node); + LIR::AsRange(finish).InsertAtEnd(node); + } + for (GenTree* node : deadStores) + { + BlockRange().Remove(node); + } + for (GenTree* address : addresses) + { + GenTree* base = address->gtGetOp1(); + GenTree* index = address->gtGetOp2(); + int offset = 0; + unsigned scale = 1; +#ifdef TARGET_ARM64 + if (index->OperIs(GT_LSH)) + { + GenTree* shift = index; + scale = 1u << shift->gtGetOp2()->AsIntCon()->IconValue(); + index = shift->gtGetOp1(); + BlockRange().Remove(shift->gtGetOp2()); + BlockRange().Remove(shift); + } +#endif + if (index->OperIs(GT_CNS_INT)) + { + offset = (int)index->AsIntCon()->IconValue(); + BlockRange().Remove(index); + index = nullptr; + } + GenTree* lea = new (m_compiler, GT_LEA) GenTreeAddrMode(address->TypeGet(), base, index, scale, offset); + base->ClearContained(); + base->ClearRegOptional(); + if (index != nullptr) + { + index->ClearContained(); + index->ClearRegOptional(); + } + LIR::Use use; + bool found = BlockRange().TryGetUse(address, &use); + assert(found); + BlockRange().InsertBefore(address, lea); + use.ReplaceWith(lea); + BlockRange().Remove(address); + } +#ifdef TARGET_ARM64 + // Strength reduction may advance the byte offset before computing the sum. + // Delay an independent offset update until after the stores so their address + // can use the same offset register as the loads. + GenTree* lastStore = nullptr; + for (GenTree* node : BlockRange()) + { + if (node->OperIs(GT_STOREIND)) + { + lastStore = node; + } + } + jitstd::vector updates(m_compiler->getAllocator(CMK_Lower)); + for (GenTree* node = BlockRange().FirstNode(); node != nullptr && node != lastStore; node = node->gtNext) + { + if (!node->OperIs(GT_STORE_LCL_VAR) || !node->TypeIs(TYP_LONG) || !node->gtGetOp1()->OperIs(GT_LEA)) + { + continue; + } + GenTreeAddrMode* address = node->gtGetOp1()->AsAddrMode(); + unsigned local = node->AsLclVar()->GetLclNum(); + if (address->Index() != nullptr || address->Base() == nullptr || !address->Base()->OperIs(GT_LCL_VAR) || + address->Base()->AsLclVar()->GetLclNum() != local || m_compiler->lvaGetDesc(local)->IsAddressExposed()) + { + continue; + } + updates.push_back(node); + } + if (lastStore != nullptr) + { + for (GenTree* update : updates) + { + bool isClosed; + LIR::ReadOnlyRange range = BlockRange().GetTreeRange(update, &isClosed); + if (isClosed && IsRangeInvariantInRange(range.FirstNode(), update, lastStore->gtNext, nullptr)) + { + LIR::Range moved = BlockRange().Remove(range.FirstNode(), update); + BlockRange().InsertAfter(lastStore, std::move(moved)); + } + } + } +#endif + + // Expose single-use load/address temporaries to containment now that the + // carry comparisons no longer require their original local values. + auto forward = [&](GenTree* parent, GenTree* read, genTreeOps oper) { + if (!read->OperIs(GT_LCL_VAR)) + { + return; + } + unsigned local = read->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (!dsc->lvTracked || dsc->IsAddressExposed() || !IsCarryLocalDeadOnExit(local, block)) + { + return; + } + GenTree* definition = nullptr; + for (GenTree* node : BlockRange()) + { + if ((node == read) && (definition == nullptr)) + { + return; + } + if (!node->OperIsLocal() || node->AsLclVarCommon()->GetLclNum() != local || node == read) + { + continue; + } + if (definition != nullptr || !node->OperIs(GT_STORE_LCL_VAR)) + { + return; + } + definition = node; + } + if (definition == nullptr) + { + return; + } + GenTree* value = definition->gtGetOp1(); + if (!value->OperIs(oper) || !IsInvariantInRange(value, parent)) + { + return; + } + LIR::Use use; + bool found = BlockRange().TryGetUse(read, &use); + assert(found); + use.ReplaceWith(value); + BlockRange().Remove(read); + BlockRange().Remove(definition); + }; + forward(sum, sum->gtGetOp2(), GT_IND); + ContainCheckBinary(sum->AsOp()); + for (GenTree* node : BlockRange()) + { + if (node->OperIs(GT_STOREIND)) + { + forward(node, node->gtGetOp1(), GT_LEA); + ContainCheckStoreIndir(node->AsStoreInd()); + } + } + +#ifdef TARGET_AMD64 + // ADD(-1) selects DEC: update ZF for the branch while preserving CF. + if (decrement->OperIs(GT_SUB)) + { + decrement->ChangeOper(GT_ADD); + decrement->gtGetOp2()->AsIntCon()->SetIconValue(-1); + } + decrement->gtFlags |= GTF_SET_FLAGS; + decrement->gtFlags &= ~GTF_ADD_CARRY_FLAGS; + ContainCheckBinary(decrement->AsOp()); + BlockRange().Remove(cmp->gtGetOp1()); + BlockRange().Remove(cmp->gtGetOp2()); + BlockRange().Remove(cmp); +#endif +} + +//------------------------------------------------------------------------ +// IsCarryOperandAvailable: Check whether an operand is available at a flags producer, +// or can be moved there without changing its value. Bound the scans to keep lowering linear. +bool Lowering::IsCarryOperandAvailable(GenTree* operand, GenTree* producer, GenTree** moveStart) +{ + *moveStart = nullptr; + GenTree* node = producer->gtPrev; + for (int budget = 64; (node != nullptr) && (budget > 0); node = node->gtPrev, budget--) + { + if (node == operand) + { + return true; + } + } + + if (operand->OperIs(GT_IND)) + { + // Move the load together with its address calculation. A closed range + // ensures that no intermediate value has a use left behind. Check all + // crossed nodes for aliasing, exception ordering, and local definitions. + bool isClosed; + LIR::ReadOnlyRange range = BlockRange().GetTreeRange(operand, &isClosed); + if (!isClosed) + { + return false; + } + + SideEffectSet effects; + node = operand; + int budget = 64; + for (; (node != nullptr) && (node != producer) && (budget > 0); node = node->gtPrev, budget--) + { + effects.AddNode(m_compiler, node); + if (node == range.FirstNode()) + { + break; + } + } + if ((node != range.FirstNode()) || (node == producer)) + { + return false; + } + + for (node = node->gtPrev; (node != nullptr) && (budget > 0); node = node->gtPrev, budget--) + { + if (effects.InterferesWith(m_compiler, node, true)) + { + return false; + } + if (node == producer) + { + *moveStart = range.FirstNode(); + return true; + } + } + return false; + } + + if (operand->OperIs(GT_CAST) && !operand->gtOverflow() && operand->TypeIs(TYP_LONG) && + operand->AsCast()->CastOp()->TypeIs(TYP_INT)) + { + GenTree* input = operand->AsCast()->CastOp(); + GenTree* moveInput; + if (!IsCarryOperandAvailable(input, producer, &moveInput)) + { + return false; + } + if (moveInput != nullptr) + { + // Move the input together with its widening cast. The recursive + // check proved a local/constant invariant, or a load's closed tree + // safe to move. Adjacency keeps the cast within that same range. + if (!input->OperIs(GT_LCL_VAR, GT_CNS_INT, GT_IND) || (input->gtNext != operand)) + { + return false; + } + *moveStart = moveInput; + return true; + } + } + else if (!operand->OperIs(GT_LCL_VAR, GT_CNS_INT) || + (operand->OperIs(GT_LCL_VAR) && + m_compiler->lvaGetDesc(operand->AsLclVar()->GetLclNum())->IsAddressExposed())) + { + return false; + } + + unsigned local = BAD_VAR_NUM; + unsigned parent = BAD_VAR_NUM; + if (operand->OperIs(GT_LCL_VAR)) + { + local = operand->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (dsc->lvIsStructField) + { + parent = dsc->lvParentLcl; + if (m_compiler->lvaGetDesc(parent)->IsAddressExposed()) + { + return false; + } + } + } + + node = producer->gtNext; + for (int budget = 64; (node != nullptr) && (budget > 0); node = node->gtNext, budget--) + { + if (node == operand) + { + *moveStart = operand; + return true; + } + // A promoted field can also be overwritten through its parent local. + if (local != BAD_VAR_NUM && node->OperIs(GT_STORE_LCL_VAR, GT_STORE_LCL_FLD) && + ((node->AsLclVarCommon()->GetLclNum() == local) || (node->AsLclVarCommon()->GetLclNum() == parent))) + { + return false; + } + } + return false; +} + +//------------------------------------------------------------------------ +// TryLowerAddCarry: Fold a materialized carry or borrow into arithmetic. Keep the flag consumer +// immediately after its producer, moving only operands whose values are available there. +bool Lowering::TryLowerAddCarry(GenTreeOp* add) +{ + // Comparison recognition must still run: it can create the first SETCC. + // Only folding an existing carry can skip the local-definition searches. + if (!m_blockMayHaveSetCC) + { + return false; + } + if (!add->TypeIs(TYP_INT, TYP_LONG) || add->gtOverflow() || add->gtSetFlags() || add->IsUnusedValue()) + { + return false; + } + + for (unsigned index = add->OperIs(GT_SUB) ? 1 : 0; index < 2; index++) + { + GenTree* value = index == 0 ? add->gtOp1 : add->gtOp2; + GenTree* other = index == 0 ? add->gtOp2 : add->gtOp1; + GenTree* carry = value; + GenTree* store = nullptr; + if (value->OperIs(GT_LCL_VAR)) + { + unsigned lclNum = value->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(lclNum); + if (!dsc->lvTracked || dsc->IsAddressExposed()) + { + continue; + } + + store = FindCarryLocalStore(value); + if (store == nullptr) + { + continue; + } + + carry = store->gtGetOp1(); + } + GenTree* cast = nullptr; + if (carry->OperIs(GT_CAST) && !carry->gtOverflow() && carry->CastToType() == TYP_LONG && + carry->AsCast()->CastOp()->TypeIs(TYP_INT)) + { + cast = carry; + carry = carry->AsCast()->CastOp(); + } + if (!carry->OperIs(GT_SETCC)) + { + continue; + } + + GenTree* producer = carry->gtPrev; + if ((producer == nullptr) || !producer->OperIs(GT_ADD, GT_ADD_CARRY, GT_SUB, GT_SUB_BORROW, GT_NEG) || + !producer->gtSetFlags()) + { + continue; + } + + bool borrow = producer->OperIs(GT_SUB, GT_SUB_BORROW, GT_NEG); + GenCondition condition = GenCondition::C; +#ifdef TARGET_ARM64 + if (borrow) + { + condition = GenCondition::NC; + } +#endif + if (carry->AsCC()->gtCondition.GetCode() != condition.GetCode() || (add->OperIs(GT_SUB) && !borrow)) + { + continue; + } + + // A direct SETCC operand has one use by the LIR value-node invariant. + if (store != nullptr) + { + if (!IsCarryLocalDeadOnExit(value->AsLclVar()->GetLclNum(), m_block)) + { + continue; + } + // Only spend the extended scan budget on a recognized carry. Keep + // the original local scan allowance, and share the extra work over + // the lowering phase so large unrolled blocks do not make this quadratic. + GenTree* node = store->gtNext; + unsigned scanned = 0; + bool singleUse = true; + for (; node != nullptr; node = node->gtNext, scanned++) + { + if (scanned >= 64) + { + if (m_carryUseScanBudget == 0) + { + break; + } + m_carryUseScanBudget--; + } + if (node->OperIsLocal() && (node != value) && + (node->AsLclVarCommon()->GetLclNum() == value->AsLclVar()->GetLclNum())) + { + singleUse = false; + break; + } + } + if (!singleUse || (node != nullptr)) + { + continue; + } + } + + bool combine = + other->OperIs(add->OperIs(GT_SUB) ? GT_SUB : GT_ADD) && !other->gtOverflow() && !other->gtSetFlags(); + bool negate = add->OperIs(GT_SUB) && other->OperIs(GT_NEG) && !other->gtSetFlags(); +#ifdef TARGET_ARM64 + // CINC handles high + borrow; arbitrary binary combinations need a + // separate addition and are left to the existing lowering. + if (borrow && add->OperIs(GT_ADD) && combine) + { + continue; + } +#endif + GenTree* operands[] = {(combine || negate) ? other->gtGetOp1() : other, combine ? other->gtGetOp2() : nullptr}; + GenTree* moveStart[2] = {}; + if (!IsCarryOperandAvailable(operands[0], producer, &moveStart[0]) || + (combine && !IsCarryOperandAvailable(operands[1], producer, &moveStart[1]))) + { + continue; + } + + // Inserting ADC/SBB changes flags. No later consumer may observe the + // original producer's flags before the next flags definition. + bool hasOtherFlagsConsumer = false; + for (GenTree* node = carry->gtNext; node != nullptr; node = node->gtNext) + { + if (node->OperConsumesFlags()) + { + hasOtherFlagsConsumer = true; + break; + } + if (node->gtSetFlags()) + { + break; + } + } + if (hasOtherFlagsConsumer) + { + continue; + } + + JITDUMP("Folding carry [%06u] into addition [%06u]\n", carry->gtTreeID, add->gtTreeID); + if (!combine) + { + operands[1] = m_compiler->gtNewZeroConNode(add->TypeGet()); + BlockRange().InsertBefore(producer, operands[1]); + } + for (unsigned i = 0; i < 2; i++) + { + if (moveStart[i] != nullptr) + { + LIR::Range range = BlockRange().Remove(moveStart[i], operands[i]); + BlockRange().InsertBefore(producer, std::move(range)); + } + operands[i]->ClearContained(); + operands[i]->ClearRegOptional(); + } + if (negate) + { + // -(high) - borrow is 0 - high - borrow. Keep the zero before the + // flags producer, since zeroing a register can overwrite flags. + std::swap(operands[0], operands[1]); + } + if (combine || negate) + { + BlockRange().Remove(other); + } + m_hasAddCarry = true; + add->ChangeOper(add->OperIs(GT_SUB) ? GT_SUB_BORROW : (borrow ? GT_ADD_BORROW : GT_ADD_CARRY)); + add->gtOp1 = operands[0]; + add->gtOp2 = operands[1]; + BlockRange().Remove(add); + BlockRange().InsertAfter(producer, add); + BlockRange().Remove(carry); + if (cast != nullptr) + { + BlockRange().Remove(cast); + } + if (store != nullptr) + { + BlockRange().Remove(store); + BlockRange().Remove(value); + } + ContainCheckBinary(add); + return true; + } + return false; +} +#endif // defined(TARGET_AMD64) || defined(TARGET_ARM64) + +#ifdef TARGET_ARM64 +//------------------------------------------------------------------------ +// LowerArm64MultiplyCarryLoops: Keep the last carry of each multiply-accumulate +// in NZCV until the next limb, including across a counted loop's backedge. +void Lowering::LowerArm64MultiplyCarryLoops() +{ + if (m_compiler->compHndBBtabCount != 0) + { + return; + } + BasicBlock* savedBlock = m_block; + for (BasicBlock* block : m_compiler->Blocks()) + { + if (block->KindIs(BBJ_COND) && block->GetTrueTarget() == block && block->GetFalseTarget() != block) + { + m_block = block; + TryLowerArm64MultiplyCarryLoop(); + } + } + m_block = savedBlock; +} + +//------------------------------------------------------------------------ +// TryLowerArm64MultiplyCarryLoop: Match one to four full-width unsigned products, +// each followed by two low ADD/high ADC pairs. Represent the incoming carry as +// a register plus C, and use ADCS/ADC/ADDS per limb. The final ADDS leaves C for +// the next limb. No memory access moves, and all intervening nodes must preserve +// flags. Only the final loop carry may be live out; drain it on the exit edge. +// For B = 2^64, a*b + destination + carry <= B^2-1. Thus the outgoing +// high word plus C fits in one limb, preserving this representation by induction. +void Lowering::TryLowerArm64MultiplyCarryLoop() +{ + BasicBlock* block = m_block; + GenTree* branch = BlockRange().LastNode(); + if (!branch->OperIs(GT_JCMP) || branch->AsOpCC()->gtCondition.GetCode() != GenCondition::NE || + !branch->gtGetOp1()->OperIs(GT_LCL_VAR) || !branch->gtGetOp1()->TypeIs(TYP_INT) || + !branch->gtGetOp2()->IsIntegralConst(0)) + { + return; + } + BasicBlock* entry = FindCarryLoopEntry(block); + if (entry == nullptr) + { + return; + } + + struct Lane + { + GenTree* lowProduct; + GenTree* highProduct; + GenTree* add[2]; + GenTree* adc[2]; + GenTree* low; + GenTree* destination; + GenTree* partial; + GenTree* carry; + } lanes[4] = {}; + unsigned laneCount = 0; + unsigned adcCount = 0; + unsigned nodes = 0; + for (GenTree* node : BlockRange()) + { + if (++nodes > 512) + { + return; + } + if (node->OperIs(GT_HWINTRINSIC)) + { + if (laneCount == ArrLen(lanes) || adcCount != laneCount * 2 || + node->AsHWIntrinsic()->GetHWIntrinsicId() != NI_ArmBase_Arm64_MultiplyHigh || + node->AsHWIntrinsic()->GetSimdBaseType() != TYP_ULONG) + { + return; + } + lanes[laneCount++].highProduct = node; + } + if (node->OperIs(GT_ADD_CARRY)) + { + if (laneCount == 0 || adcCount >= laneCount * 2 || !node->TypeIs(TYP_LONG) || node->gtSetFlags() || + !node->gtGetOp1()->OperIs(GT_LCL_VAR) || !node->gtGetOp2()->IsIntegralConst(0)) + { + return; + } + lanes[adcCount / 2].adc[adcCount % 2] = node; + adcCount++; + } + } + if (laneCount == 0 || adcCount != laneCount * 2) + { + return; + } + + // Require one definition and one read in this block. For the loop-carried + // local alone, the read precedes the definition and the value is live out. + auto findStore = [&](GenTree* read, bool loopCarry = false) -> GenTree* { + if (!read->OperIs(GT_LCL_VAR)) + { + return nullptr; + } + unsigned local = read->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (!dsc->lvTracked || dsc->IsAddressExposed() || (!loopCarry && !IsCarryLocalDeadOnExit(local, block))) + { + return nullptr; + } + GenTree* store = nullptr; + for (GenTree* node : BlockRange()) + { + if (dsc->lvIsStructField && node->OperIsLocal() && node->AsLclVarCommon()->GetLclNum() == dsc->lvParentLcl) + { + return nullptr; + } + if (node == read && ((store == nullptr && !loopCarry) || (store != nullptr && loopCarry))) + { + return nullptr; + } + if (!node->OperIsLocal() || node->AsLclVarCommon()->GetLclNum() != local || node == read) + { + continue; + } + if (!node->OperIs(GT_STORE_LCL_VAR) || store != nullptr) + { + return nullptr; + } + store = node; + } + return store; + }; + auto definition = [&](GenTree* read) -> GenTree* { + for (unsigned depth = 0; depth < 4 && read->OperIs(GT_LCL_VAR); depth++) + { + GenTree* store = findStore(read); + if (store == nullptr) + { + return nullptr; + } + read = store->gtGetOp1(); + } + return read; + }; + + for (unsigned i = 0; i < laneCount; i++) + { + Lane& lane = lanes[i]; + for (unsigned j = 0; j < 2; j++) + { + GenTree* add = lane.adc[j]->gtPrev; + if (add == nullptr || !add->OperIs(GT_ADD) || !add->TypeIs(TYP_LONG) || add->gtOverflow() || + !add->gtSetFlags() || !add->gtGetOp1()->OperIs(GT_LCL_VAR) || !add->gtGetOp2()->OperIs(GT_LCL_VAR)) + { + return; + } + lane.add[j] = add; + } + if (definition(lane.adc[0]->gtGetOp1()) != lane.highProduct || + definition(lane.adc[1]->gtGetOp1()) != lane.adc[0]) + { + return; + } + lane.low = lane.add[0]->gtGetOp1(); + lane.destination = lane.add[0]->gtGetOp2(); + lane.lowProduct = definition(lane.low); + if (lane.lowProduct == nullptr || !lane.lowProduct->OperIs(GT_MUL)) + { + std::swap(lane.low, lane.destination); + lane.lowProduct = definition(lane.low); + } + if (lane.lowProduct == nullptr || !lane.lowProduct->OperIs(GT_MUL) || lane.lowProduct->gtOverflow() || + lane.lowProduct->gtSetFlags() || !lane.lowProduct->TypeIs(TYP_LONG)) + { + return; + } + // Both halves must be the same unsigned product. Loads have already + // been stored to locals; do not move or duplicate them across aliases. + GenTree* left = lane.lowProduct->gtGetOp1(); + GenTree* right = lane.lowProduct->gtGetOp2(); + GenTreeHWIntrinsic* high = lane.highProduct->AsHWIntrinsic(); + if (!left->OperIs(GT_LCL_VAR) || !right->OperIs(GT_LCL_VAR) || !GenTree::Compare(left, high->Op(1)) || + !GenTree::Compare(right, high->Op(2)) || m_compiler->lvaGetDesc(left->AsLclVar())->IsAddressExposed() || + m_compiler->lvaGetDesc(right->AsLclVar())->IsAddressExposed()) + { + return; + } + // Compare the reaching values at the operand reads, in either order. + // Either half can be evaluated first; neither product is relocated. + for (unsigned operand = 1; operand <= 2; operand++) + { + GenTree* input = operand == 1 ? left : right; + GenTree* other = high->Op(operand); + LclVarDsc* dsc = m_compiler->lvaGetDesc(input->AsLclVar()); + bool seenRead = false; + for (GenTree* node : BlockRange()) + { + if (node == input || node == other) + { + if (seenRead) + { + break; + } + seenRead = true; + } + if (seenRead && node->OperIsLocal()) + { + unsigned local = node->AsLclVarCommon()->GetLclNum(); + if ((node->OperIsLocalStore() && local == input->AsLclVar()->GetLclNum()) || + (dsc->lvIsStructField && local == dsc->lvParentLcl)) + { + return; + } + } + } + } + lane.partial = lane.add[1]->gtGetOp1(); + lane.carry = lane.add[1]->gtGetOp2(); + if (definition(lane.partial) != lane.add[0]) + { + std::swap(lane.partial, lane.carry); + } + if (definition(lane.partial) != lane.add[0] || (i != 0 && definition(lane.carry) != lanes[i - 1].adc[1]) || + m_compiler->lvaGetDesc(lane.carry->AsLclVar())->IsAddressExposed()) + { + return; + } + unsigned carryLocal = lane.carry->AsLclVar()->GetLclNum(); + for (GenTree* node = lane.add[0]; node != lane.carry; node = node->gtNext) + { + if (node == nullptr || (node->OperIsLocalStore() && node->AsLclVarCommon()->GetLclNum() == carryLocal)) + { + return; + } + } + } + GenTree* carryStore = findStore(lanes[0].carry, true); + if (carryStore == nullptr || definition(carryStore->gtGetOp1()) != lanes[laneCount - 1].adc[1]) + { + return; + } + + for (GenTree* node : BlockRange()) + { + bool matched = node == branch; + for (unsigned i = 0; i < laneCount; i++) + { + Lane& lane = lanes[i]; + matched |= node == lane.lowProduct || node == lane.highProduct || node == lane.add[0] || + node == lane.add[1] || node == lane.adc[0] || node == lane.adc[1]; + } + if (matched) + { + continue; + } + if (node->gtSetFlags() || node->OperConsumesFlags()) + { + return; + } + switch (node->OperGet()) + { + case GT_LCL_VAR: + case GT_STORE_LCL_VAR: + case GT_IL_OFFSET: + case GT_CNS_INT: + case GT_LEA: + break; + case GT_IND: + case GT_STOREIND: + if (!node->TypeIs(TYP_LONG) || (node->gtFlags & GTF_IND_VOLATILE) != 0) + { + return; + } + break; + case GT_CAST: + if (node->gtOverflow() || !node->TypeIs(TYP_LONG) || !node->gtGetOp1()->TypeIs(TYP_INT)) + { + return; + } + break; + case GT_ADD: + case GT_SUB: + if (node->gtOverflow() || !node->TypeIs(TYP_INT, TYP_LONG, TYP_BYREF)) + { + return; + } + break; + default: + return; + } + } + + JITDUMP("Keeping ARM64 multiply-accumulate carry in NZCV across %u lanes around " FMT_BB "\n", laneCount, + block->bbNum); + BasicBlock* seed = m_compiler->fgSplitEdge(entry, block); + BasicBlock* finish = m_compiler->fgSplitEdge(block, block->GetFalseTarget()); + m_compiler->fgInvalidateDfsTree(); + GenTree* zero = m_compiler->gtNewLconNode(0); + GenTree* one = m_compiler->gtNewLconNode(1); + GenTree* clearCarry = m_compiler->gtNewOperNode(GT_CMP, TYP_VOID, zero, one); + clearCarry->gtFlags |= GTF_SET_FLAGS; + LIR::AsRange(seed).InsertAtEnd(zero, one, clearCarry); + LIR::ReadOnlyRange seedRange(zero, clearCarry); + LowerRange(seed, seedRange); + + unsigned carryLocal = lanes[0].carry->AsLclVar()->GetLclNum(); + GenTree* finalRead = m_compiler->gtNewLclvNode(carryLocal, TYP_LONG); + GenTree* finalZero = m_compiler->gtNewLconNode(0); + GenTree* drain = m_compiler->gtNewOperNode(GT_ADD_CARRY, TYP_LONG, finalRead, finalZero); + GenTree* finalStore = m_compiler->gtNewStoreLclVarNode(carryLocal, drain); + LIR::AsRange(finish).InsertAtEnd(finalRead, finalZero, drain, finalStore); + LIR::ReadOnlyRange finishRange(finalRead, finalStore); + LowerRange(finish, finishRange); + + for (unsigned i = 0; i < laneCount; i++) + { + Lane& lane = lanes[i]; + BlockRange().Remove(lane.carry); + BlockRange().InsertBefore(lane.add[0], lane.carry); + lane.add[0]->AsOp()->gtOp1 = lane.low; + lane.add[0]->AsOp()->gtOp2 = lane.carry; + lane.add[0]->ChangeOper(GT_ADD_CARRY); + lane.add[1]->AsOp()->gtOp1 = lane.partial; + lane.add[1]->AsOp()->gtOp2 = lane.destination; + LIR::Use use; + bool found = BlockRange().TryGetUse(lane.adc[1], &use); + assert(found); + use.ReplaceWith(lane.adc[1]->gtGetOp1()); + BlockRange().Remove(lane.adc[1]->gtGetOp2()); + BlockRange().Remove(lane.adc[1]); + ContainCheckBinary(lane.add[0]->AsOp()); + ContainCheckBinary(lane.add[1]->AsOp()); + } +} +#endif // TARGET_ARM64 diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 06913e3a8e5b5a..2522ef0c036482 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -167,6 +167,32 @@ class Lowering final : public Phase #endif GenTree* OptimizeConstCompare(GenTree* cmp); GenTree* LowerCompare(GenTree* cmp); +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + BasicBlock* FindCarryLoopEntry(BasicBlock* block); + GenTree* FindDeadCarryConstantStore(GenTree* constant); + GenTree* FindCarryLocalStore(GenTree* read); + bool IsCarryLocalDeadOnExit(unsigned local, BasicBlock* block); + bool TryLowerCarryCompare(GenTreeOp* cmp); + bool TryLowerAddCarry(GenTreeOp* add); + bool LowerFullAdders(); + void LowerCarryChains(); +#ifdef TARGET_AMD64 + void ReplaceCarryAddressAdd(GenTree* add); + GenTree* FindMultiplyCarryStore(GenTree* read, bool loopCarry = false); + void LowerMultiplyCarryLoops(); + bool TryLowerMultiplyCarryLoop(); + bool TryLowerMultiplyCarryChain(); + void TryLowerMultiplyCarryBackedge(); + void LowerMultiplyCarryMultiplier(); + void LowerMultiplyCarryLoads(); +#else + void LowerArm64MultiplyCarryLoops(); + void TryLowerArm64MultiplyCarryLoop(); +#endif + void TryLowerCarryLoop( + GenTree* sum, GenTree* restore, GenTree* cc, GenTree* cast, GenTree* store, unsigned carryLocal); + bool IsCarryOperandAvailable(GenTree* operand, GenTree* producer, GenTree** moveStart); +#endif GenTree* LowerJTrue(GenTreeOp* jtrue); #ifdef TARGET_RISCV64 GenTree* LowerSavedIntegerCompare(GenTree* cmp); @@ -427,6 +453,7 @@ class Lowering final : public Phase bool TryLowerConstIntDivOrMod(GenTree* node, GenTree** nextNode); GenTree* LowerSignedDivOrMod(GenTree* node); void LowerDivOrMod(GenTreeOp* divMod); + void TryLowerDivRem(GenTreeOp* div); void LowerBlockStoreCommon(GenTreeBlk* blkNode); void LowerBlockStoreAsHelperCall(GenTreeBlk* blkNode); void LowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blkNode); @@ -488,6 +515,7 @@ class Lowering final : public Phase void LowerShift(GenTreeOp* shift); void TryRemoveShiftRotateMask(GenTreeOp* op); bool TryFoldBinop(GenTreeOp* node); + bool TryContainFunnelShift(GenTreeOp* node); #ifdef FEATURE_HW_INTRINSICS GenTree* LowerHWIntrinsic(GenTreeHWIntrinsic* node); void LowerHWIntrinsicCC(GenTreeHWIntrinsic* node, NamedIntrinsic newIntrinsicId, GenCondition condition); @@ -661,6 +689,15 @@ class Lowering final : public Phase mutable SideEffectSet m_scratchSideEffects; // SideEffectSet used for IsSafeToContainMem and isRMWIndirCandidate BasicBlock* m_block; +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + // Conservative during partial/post-lowering walks; reset for each full block walk. + bool m_blockMayHaveSetCC = true; + bool m_hasAddCarry = false; + BasicBlock* m_lastSubtractionBlock = nullptr; + unsigned m_carryLocalScanBudget = 65536; // Node/block budget for current-IR dead-on-exit proofs. + unsigned m_carryUseScanBudget = 4096; // Extra node visits beyond the bounded local carry-use scans. +#endif + #ifdef FEATURE_FIXED_OUT_ARGS unsigned m_outgoingArgSpaceSize = 0; #endif diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index beb9089ab319d5..c7f2e4f95a5a2a 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -2809,9 +2809,28 @@ void Lowering::ContainCheckIndir(GenTreeIndir* indirNode) // void Lowering::ContainCheckBinary(GenTreeOp* node) { + if (TryContainFunnelShift(node)) + { + return; + } GenTree* op1 = node->gtGetOp1(); GenTree* op2 = node->gtGetOp2(); +#ifdef TARGET_ARM64 + if (node->OperIs(GT_ADD_CARRY, GT_SUB_BORROW, GT_ADD_BORROW)) + { + if (node->OperIs(GT_SUB_BORROW) && op1->IsIntegralConst(0)) + { + MakeSrcContained(node, op1); + } + if (op2->IsIntegralConst(0)) + { + MakeSrcContained(node, op2); + } + return; + } +#endif + if (CheckImmedAndMakeContained(node, op2)) { return; diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index f544824aa0044b..7e7d3f70be5643 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -8063,6 +8063,178 @@ void Lowering::ContainCheckMul(GenTreeOp* node) } } +//------------------------------------------------------------------------ +// TryLowerDivRem: Reuse the remainder of a divide for x - (x / y) * y. +// +// Arguments: +// div - The division producing the quotient. +// +// Notes: +// Keep the division (and its exceptions) in place. Capture RDX immediately +// after it, before any quotient use or intervening code can overwrite it. +// The pair's fixed definitions must not be reassigned by LSRA. +// +void Lowering::TryLowerDivRem(GenTreeOp* div) +{ + if (!m_compiler->opts.OptimizationEnabled() || !div->OperIs(GT_DIV, GT_UDIV) || !varTypeIsIntOrI(div) || + div->IsDivRemPair()) + { + return; + } + + // The inclusive integer-register range includes SP, which is not allocatable. + // Exclude it from this rough capacity estimate; other reservations (such as FP) vary by method. +#ifdef TARGET_AMD64 + unsigned intRegCount = m_compiler->get_REG_INT_LAST() - REG_INT_FIRST; +#else + unsigned intRegCount = REG_INT_COUNT - 1; +#endif + // Conservatively leave large, register-heavy methods to the existing sequence. Keeping two + // fixed-register results can increase spills well beyond the multiply/subtract being removed. + // Make this decision for the whole method so partially pairing its divides does not produce + // unstable allocation tradeoffs between otherwise similar paths. + if (m_compiler->lvaTrackedCount > 2 * intRegCount) + { + return; + } + + GenTree* dividend = div->gtGetOp1(); + GenTree* divisor = div->gtGetOp2(); + if (!dividend->OperIs(GT_LCL_VAR) || !divisor->OperIs(GT_LCL_VAR)) + { + return; + } + + GenTree* quotientStore = div->gtNext; + if ((quotientStore == nullptr) || !quotientStore->OperIs(GT_STORE_LCL_VAR) || (quotientStore->gtGetOp1() != div)) + { + quotientStore = nullptr; + } + else if (m_compiler->lvaGetDesc(quotientStore->AsLclVarCommon())->IsAddressExposed()) + { + return; + } + + unsigned budget = 64; + for (GenTree* node = div->gtNext; (node != nullptr) && (budget-- != 0); node = node->gtNext) + { + if ((quotientStore != nullptr) && (node != quotientStore) && node->OperIs(GT_STORE_LCL_VAR) && + (node->AsLclVarCommon()->GetLclNum() == quotientStore->AsLclVarCommon()->GetLclNum())) + { + return; + } + + if (!node->OperIs(GT_SUB) || node->gtOverflow() || node->gtSetFlags() || (node->TypeGet() != div->TypeGet())) + { + continue; + } + + GenTree* original = node->gtGetOp1(); + GenTree* mul = node->gtGetOp2(); + if (!original->OperIs(GT_LCL_VAR) || !GenTree::Compare(original, dividend) || !mul->OperIs(GT_MUL) || + mul->gtOverflow() || mul->gtSetFlags() || (mul->TypeGet() != div->TypeGet())) + { + continue; + } + + GenTree* quotient = mul->gtGetOp1(); + GenTree* factor = mul->gtGetOp2(); + if (quotient->OperIs(GT_LCL_VAR) && GenTree::Compare(quotient, divisor)) + { + std::swap(quotient, factor); + } + + if (!factor->OperIs(GT_LCL_VAR) || !GenTree::Compare(factor, divisor)) + { + continue; + } + + if (quotientStore != nullptr) + { + if (!quotient->OperIs(GT_LCL_VAR) || + (quotient->AsLclVarCommon()->GetLclNum() != quotientStore->AsLclVarCommon()->GetLclNum()) || + (quotient->TypeGet() != div->TypeGet())) + { + continue; + } + } + else if (quotient != div) + { + continue; + } + + // Either copy of an input can precede the other in LIR. Check both + // reads so assignments embedded in an expression cannot change it. + if (!IsInvariantInRange(dividend, node) || !IsInvariantInRange(divisor, node) || + !IsInvariantInRange(original, node) || !IsInvariantInRange(factor, node)) + { + return; + } + + LIR::Use use; + if (!BlockRange().TryGetUse(node, &use)) + { + return; + } + + GenTree* remainder = m_compiler->gtNewPhysRegNode(REG_RDX, div->TypeGet()); + remainder->gtFlags |= GTF_DIV_REM_PAIR; + div->gtFlags |= GTF_DIV_REM_PAIR; + // Reuse a local destination when it can be defined at the divide. Introducing another + // temporary for an immediately stored remainder needlessly perturbs register allocation. + // Do not cross an exception: a handler may observe the destination's old value. + GenTree* store = use.User(); + bool reuseStore = store->OperIs(GT_STORE_LCL_VAR) && + !m_compiler->lvaGetDesc(store->AsLclVarCommon())->IsAddressExposed() && + !m_compiler->lvaGetDesc(store->AsLclVarCommon())->lvIsStructField; + if (reuseStore) + { + unsigned local = store->AsLclVarCommon()->GetLclNum(); + unsigned storeBudget = 64; + for (GenTree* between = div->gtNext; between != store; between = between->gtNext) + { + if ((between == nullptr) || (storeBudget-- == 0) || + ((between->gtFlags & (GTF_CALL | GTF_EXCEPT)) != 0) || + ((between != original) && between->OperIsLocal() && + (between->AsLclVarCommon()->GetLclNum() == local))) + { + reuseStore = false; + break; + } + } + } + if (reuseStore) + { + store->AsOp()->gtOp1 = remainder; + BlockRange().Remove(store); + } + else + { + unsigned temp = m_compiler->lvaGrabTemp(true DEBUGARG("division remainder")); + store = m_compiler->gtNewTempStore(temp, remainder); + GenTree* read = m_compiler->gtNewLclvNode(temp, div->TypeGet()); + BlockRange().InsertBefore(node, read); + use.ReplaceWith(read); + } + BlockRange().InsertAfter(div, remainder, store); + + BlockRange().Remove(original); + BlockRange().Remove(factor); + if (quotient != div) + { + BlockRange().Remove(quotient); + } + else + { + div->SetUnusedValue(); + } + BlockRange().Remove(mul); + BlockRange().Remove(node); + JITDUMP("Reusing the hardware remainder of division [%06u]\n", div->gtTreeID); + return; + } +} + //------------------------------------------------------------------------ // ContainCheckDivOrMod: determine which operands of a div/mod should be contained. // @@ -8585,6 +8757,11 @@ void Lowering::ContainCheckBinary(GenTreeOp* node) { assert(node->OperIsBinary()); + if (TryContainFunnelShift(node)) + { + return; + } + if (varTypeIsFloating(node)) { assert(node->OperIs(GT_ADD, GT_SUB)); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index faf1b4b10c8c96..aa1bd94a3d3cb4 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -8982,6 +8982,15 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) { GenTree* lastNode = LIR::AsRange(block).LastNode(); +#ifdef TARGET_AMD64 + if (lastNode->OperIs(GT_JCMP)) + { + // The flag-preserving zero test consumes RCX even when the local + // itself remains in another register. Edge resolution must not + // restore an outgoing value into RCX before the branch. + consumedRegs |= SRBM_ECX; + } +#endif if (lastNode->OperIs(GT_JTRUE, GT_JCMP, GT_JTEST)) { assert(!lastNode->OperIs(GT_JTRUE) || !lastNode->gtGetOp1()->isContained()); @@ -13753,6 +13762,53 @@ SingleTypeRegSet LinearScan::RegisterSelection::select(Interval* } } +#ifdef TARGET_AMD64 + // Carry arithmetic can constrain the multiplier to RDX and a flags-preserving loop count to RCX. + // A preference that encounters one of these fixed references before the next use can force a spill. + // At a definition, prefer a free register that covers the range when one is available instead. + // Keep the original preferences if none of the alternatives can cover it. + if (linearScan->m_hasCarryArithmetic && currentInterval->isLocalVar && !found && + RefTypeIsDef(refPosition->refType) && !refPosition->isFixedRegRef) + { + SingleTypeRegSet coveringRegisters = RBM_NONE; + SingleTypeRegSet conflictingPreferences = RBM_NONE; + for (SingleTypeRegSet remaining = freeCandidates; remaining != RBM_NONE;) + { + regNumber reg = genFirstRegNumFromMask(remaining, regType); + SingleTypeRegSet bit = genSingleTypeRegMask(reg); + remaining &= ~bit; + + LsraLocation fixedLocation = linearScan->getNextFixedRef(reg, regType); + if ((fixedLocation == rangeEndLocation) && rangeEndRefPosition->isFixedRefOfReg(reg)) + { + fixedLocation++; + } + if (fixedLocation <= rangeEndLocation) + { + // Register kills (for example, at calls) are handled by the existing callee-save preferences. + RefPosition* fixedRef = linearScan->getRegisterRecord(reg)->getNextRefPosition(); + if ((fixedRef != nullptr) && (fixedRef->refType == RefTypeFixedReg) && + (fixedRef->nodeLocation == fixedLocation)) + { + conflictingPreferences |= preferences & bit; + } + } + else if (linearScan->getNextIntervalRef(reg, regType) > rangeEndLocation) + { + coveringRegisters |= bit; + } + } + if ((conflictingPreferences != RBM_NONE) && (coveringRegisters != RBM_NONE)) + { + preferences &= ~conflictingPreferences; + if ((preferences & freeCandidates) == RBM_NONE) + { + preferences |= coveringRegisters; + } + } + } +#endif // TARGET_AMD64 + #define IF_FOUND_GOTO_DONE \ if (found) \ goto Selection_Done; diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 3541e1a2309349..2468d141c02e28 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1147,6 +1147,7 @@ class LinearScan : public RegAllocInterface void applyCalleeSaveHeuristics(RefPosition* rp); void checkConflictingDefUse(RefPosition* rp); + void addFixedRefsForPropagatedDefs(); void associateRefPosWithInterval(RefPosition* rp); @@ -1602,6 +1603,8 @@ class LinearScan : public RegAllocInterface IntervalList intervals; + bool m_hasCarryArithmetic = false; + RegRecord physRegs[REG_COUNT]; // Map from tracked variable index to Interval*. @@ -1672,6 +1675,7 @@ class LinearScan : public RegAllocInterface // Ordered list of RefPositions RefPositionList refPositions; + bool m_hasUnmodeledFixedDefs = false; // Head of linked list of RefTypeKill ref positions RefPosition* killHead; diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 7856d4302d8aa3..e3a97cb07af3fe 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -943,6 +943,11 @@ int LinearScan::BuildNode(GenTree* tree) assert(dstCount == 0); break; + case GT_SUB_BORROW: + case GT_ADD_BORROW: + case GT_ADD_CARRY: + m_hasCarryArithmetic = true; + FALLTHROUGH; case GT_ADD: case GT_SUB: if (varTypeIsFloating(tree->TypeGet())) diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 0df524b0a9abda..9d606749539d91 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -268,6 +268,11 @@ void LinearScan::resolveConflictingDefAndUse(Interval* interval, RefPosition* de // to change the register assignments to RDX and RAX respectively. bool canChangeDef = !defRefPosition->treeNode->IsMultiRegNode(); + // A lowered divide/remainder pair defines RAX followed immediately by RDX. + // Redirecting either definition could overwrite the other hardware result + // before the physical-register capture has taken place. + canChangeDef &= !defRefPosition->treeNode->IsDivRemPair(); + // Avoid changing the def reg away from its assignment if that register is // currently busy. The reason is that we have a number of places in LSRA // that assume that BuildDef(tree, SRBM_REG) means that SRBM_REG will be @@ -417,6 +422,10 @@ void LinearScan::checkConflictingDefUse(RefPosition* useRP) if (!isSingleRegister(newAssignment) || !theInterval->hasInterferingUses) { defRP->registerAssignment = newAssignment; + if (isSingleRegister(newAssignment) && !defRP->isFixedRegRef && !theInterval->isConstant) + { + m_hasUnmodeledFixedDefs = true; + } } } else @@ -2776,6 +2785,11 @@ void LinearScan::buildIntervals() actualRegistersMask = regMaskTP(~RBM_NONE, ~0); } + if (m_hasUnmodeledFixedDefs && m_hasCarryArithmetic) + { + addFixedRefsForPropagatedDefs(); + } + #ifdef DEBUG // Make sure we don't have any blocks that were not visited for (BasicBlock* const block : m_compiler->Blocks()) @@ -2793,6 +2807,72 @@ void LinearScan::buildIntervals() #endif // DEBUG } +//------------------------------------------------------------------------ +// addFixedRefsForPropagatedDefs: Make inherited single-register constraints +// visible to allocations that precede the constrained definition. +// +// Notes: +// Runs after reference construction, when all use constraints are known. +// Maintains the global and physical-register reference lists in one pass. +// Limited to methods with lowered carry arithmetic. Applying the additional +// constraints to general call argument setup can introduce spills. +// Constants retain their existing register-reuse handling. +// +void LinearScan::addFixedRefsForPropagatedDefs() +{ + // A tree-temp definition can inherit a single-register constraint from its + // use after its RefPosition has been built. Model that constraint in the + // physical register's timeline too, so earlier allocations see the conflict. + // Insert in both ordered lists without moving existing RefPositions. + RefPosition* previousRegRef[REG_COUNT] = {}; + INDEBUG(unsigned refNumber = 0;) + for (auto it = refPositions.begin(); it != refPositions.end(); ++it) + { + RefPosition& ref = *it; + if (ref.isPhysRegRef) + { + previousRegRef[ref.getReg()->regNum] = &ref; + } + // Keep constant definitions under the existing constant-reuse handling. + // A physical fixed reference would discard an already-matching constant + // before its definition gets a chance to reuse the register. + else if ((ref.refType == RefTypeDef) && !ref.isFixedRegRef && !ref.getInterval()->isLocalVar && + !ref.getInterval()->isConstant && isSingleRegister(ref.registerAssignment)) + { + regNumber reg = genRegNumFromMask(ref.registerAssignment, ref.getInterval()->registerType); + RegRecord* record = getRegisterRecord(reg); + RefPosition* previous = previousRegRef[reg]; + if ((previous == nullptr) || (previous->nodeLocation != ref.nodeLocation) || + (previous->refType != RefTypeFixedReg)) + { + auto fixedIt = + refPositions.emplace(it, ref.bbNum, ref.nodeLocation, nullptr, RefTypeFixedReg DEBUG_ARG(nullptr)); + RefPosition* fixed = &*fixedIt; + fixed->setReg(record); + if (previous == nullptr) + { + fixed->nextRefPosition = record->firstRefPosition; + record->firstRefPosition = fixed; + } + else + { + fixed->nextRefPosition = previous->nextRefPosition; + previous->nextRefPosition = fixed; + } + if (fixed->nextRefPosition == nullptr) + { + record->lastRefPosition = fixed; + record->recentRefPosition = fixed; + } + previousRegRef[reg] = fixed; + INDEBUG(fixed->rpNum = refNumber++;) + } + ref.isFixedRegRef = true; + } + INDEBUG(ref.rpNum = refNumber++;) + } +} + //------------------------------------------------------------------------ // buildInitialParamDef: Build the initial definition for a parameter. // diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 2695fe7a850d82..f50b30f9310078 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -238,6 +238,23 @@ int LinearScan::BuildNode(GenTree* tree) srcCount = 1; break; +#ifdef TARGET_AMD64 + case GT_ADX_SEED: + srcCount = 0; + break; + case GT_ADX_DRAIN: + srcCount = BuildOperandUses(tree->gtGetOp1(), lowGprRegs); + buildInternalIntRegisterDefForNode(tree, lowGprRegs); + setInternalRegsDelayFree = true; + buildInternalRegisterUses(); + BuildDef(tree, lowGprRegs); + break; + case GT_JCMP: + assert(tree->gtGetOp2()->IsIntegralConst(0) && tree->gtGetOp2()->isContained()); + m_hasCarryArithmetic = true; + srcCount = BuildOperandUses(tree->gtGetOp1(), SRBM_ECX); + break; +#endif case GT_JCC: srcCount = 0; assert(dstCount == 0); @@ -266,6 +283,30 @@ int LinearScan::BuildNode(GenTree* tree) BuildDef(tree, allByteRegs()); break; + case GT_PHYSREG: + if (tree->IsDivRemPair()) + { + assert(tree->AsPhysReg()->gtSrcReg == REG_RDX); +#ifdef DEBUG + // Sequencing may move IL markers between the two definitions. + // They emit no register-writing instructions. + GenTree* producer = tree->gtPrev; + while ((producer != nullptr) && producer->OperIs(GT_IL_OFFSET)) + { + producer = producer->gtPrev; + } + assert((producer != nullptr) && producer->IsDivRemPair()); + assert(producer->OperIs(GT_DIV, GT_UDIV)); +#endif + srcCount = 0; + BuildDef(tree, SRBM_RDX); + } + else + { + srcCount = BuildSimple(tree); + } + break; + case GT_SELECT: assert(dstCount == 1); srcCount = BuildSelect(tree->AsConditional()); @@ -309,6 +350,19 @@ int LinearScan::BuildNode(GenTree* tree) case GT_ADD_HI: case GT_SUB_LO: case GT_SUB_HI: +#endif +#ifdef TARGET_AMD64 + case GT_ADCX: + case GT_ADOX: + m_hasCarryArithmetic = true; + srcCount = BuildBinaryUses(tree->AsOp(), lowGprRegs); + BuildDef(tree, lowGprRegs); + break; + case GT_SUB_BORROW: + case GT_ADD_BORROW: + case GT_ADD_CARRY: + m_hasCarryArithmetic = true; + FALLTHROUGH; #endif case GT_ADD: case GT_SUB: @@ -318,6 +372,16 @@ int LinearScan::BuildNode(GenTree* tree) case GT_BIT_SET: case GT_BIT_CLEAR: case GT_BIT_INVERT: + if (tree->IsFunnelShift()) + { + // SHRD overwrites the low input. Keep the high input live until + // after the result is defined so the initial copy cannot clobber it. + tgtPrefUse = BuildUse(tree->gtGetOp1()->gtGetOp1(), lowGprRegs); + setDelayFree(BuildUse(tree->gtGetOp2()->gtGetOp1(), lowGprRegs)); + srcCount = 2; + BuildDef(tree, lowGprRegs); + break; + } srcCount = BuildBinaryUses(tree->AsOp()); assert(dstCount == 1); BuildDef(tree); @@ -2393,6 +2457,18 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou SingleTypeRegSet apxAwareRegCandidates = ForceLowGprForApxIfNeeded(op1, RBM_NONE, canHWIntrinsicUseApxRegs); + if ((intrinsicTree->gtFlags & GTF_HW_MULX) != 0) + { + // MULX allows both halves to stay in independently allocated registers. + srcCount = BuildOperandUses(op1, SRBM_EDX); + srcCount += BuildOperandUses(op2, ForceLowGprForApxIfNeeded(op2, availableIntRegs & ~SRBM_EDX, + canHWIntrinsicUseApxRegs)); + BuildDef(intrinsicTree, apxAwareRegCandidates, 0); + BuildDef(intrinsicTree, apxAwareRegCandidates, 1); + buildUses = false; + break; + } + // mulEAX always uses EAX; if one operand is contained, force the other op into EAX. // Otherwise don't force any register: the second parameter may already happen to be in EAX, // in which case codegen will use it as the implicit operand. diff --git a/src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs b/src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs index c38cf1e66a61f7..a01d89d4a5ecb8 100644 --- a/src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs +++ b/src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs @@ -81,9 +81,12 @@ private static class XArchIntrinsicConstants public const int WaitPkg = (1 << 16); public const int X86Serialize = (1 << 17); public const int Avx512Bmm = (1 << 18); // NativeAOT does not currently consume this here. + public const int Adx = (1 << 19); public static void AddToBuilder(InstructionSetSupportBuilder builder, int flags) { + if ((flags & Adx) != 0) + builder.AddSupportedInstructionSet("adx"); if ((flags & Avx) != 0) builder.AddSupportedInstructionSet("avx"); if ((flags & Avx2) != 0) @@ -149,6 +152,7 @@ public static int FromInstructionSet(InstructionSet instructionSet) InstructionSet.X64_AVX => Avx, InstructionSet.X64_AVX_X64 => Avx, + InstructionSet.X64_ADX => Adx, InstructionSet.X64_AVX2 => Avx2, InstructionSet.X64_AVX2_X64 => Avx2, diff --git a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSet.cs b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSet.cs index 4c5574d6870c32..d8f77cf948178c 100644 --- a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSet.cs +++ b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSet.cs @@ -106,5 +106,6 @@ public enum ReadyToRunInstructionSet Cssc = 93, Zicond = 94, Fp16 = 95, + Adx = 96, } } diff --git a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSetHelper.cs b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSetHelper.cs index 632e3053189a72..91b2955fb587ca 100644 --- a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSetHelper.cs +++ b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSetHelper.cs @@ -144,6 +144,7 @@ public static class ReadyToRunInstructionSetHelper case InstructionSet.X64_VectorT512: return ReadyToRunInstructionSet.VectorT512; case InstructionSet.X64_AVXVNNIINT: return ReadyToRunInstructionSet.AvxVnniInt8; case InstructionSet.X64_AVXVNNIINT_V512: return ReadyToRunInstructionSet.AvxVnniInt8_V512; + case InstructionSet.X64_ADX: return ReadyToRunInstructionSet.Adx; default: throw new Exception("Unknown instruction set"); } @@ -199,6 +200,7 @@ public static class ReadyToRunInstructionSetHelper case InstructionSet.X86_VectorT512: return ReadyToRunInstructionSet.VectorT512; case InstructionSet.X86_AVXVNNIINT: return ReadyToRunInstructionSet.AvxVnniInt8; case InstructionSet.X86_AVXVNNIINT_V512: return ReadyToRunInstructionSet.AvxVnniInt8_V512; + case InstructionSet.X86_ADX: return ReadyToRunInstructionSet.Adx; default: throw new Exception("Unknown instruction set"); } diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs index 870544e2156425..466756f42b02da 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs @@ -112,6 +112,7 @@ public enum InstructionSet X64_SHA_X64 = InstructionSet_X64.SHA_X64, X64_WAITPKG_X64 = InstructionSet_X64.WAITPKG_X64, X64_X86Serialize_X64 = InstructionSet_X64.X86Serialize_X64, + X64_ADX = InstructionSet_X64.ADX, X86_X86Base = InstructionSet_X86.X86Base, X86_AVX = InstructionSet_X86.AVX, X86_AVX2 = InstructionSet_X86.AVX2, @@ -158,6 +159,7 @@ public enum InstructionSet X86_SHA_X64 = InstructionSet_X86.SHA_X64, X86_WAITPKG_X64 = InstructionSet_X86.WAITPKG_X64, X86_X86Serialize_X64 = InstructionSet_X86.X86Serialize_X64, + X86_ADX = InstructionSet_X86.ADX, } public enum InstructionSet_ARM64 { @@ -276,6 +278,7 @@ public enum InstructionSet_X64 SHA_X64 = 44, WAITPKG_X64 = 45, X86Serialize_X64 = 46, + ADX = 47, } public enum InstructionSet_X86 @@ -328,6 +331,7 @@ public enum InstructionSet_X86 SHA_X64 = 44, WAITPKG_X64 = 45, X86Serialize_X64 = 46, + ADX = 47, } public unsafe struct InstructionSetFlags : IEnumerable @@ -1268,6 +1272,7 @@ public static IEnumerable ArchitectureToValidInstructionSets yield return new InstructionSetInfo("avxvnniint_v512", "AvxVnniInt8_V512", InstructionSet.X64_AVXVNNIINT_V512, true); yield return new InstructionSetInfo("avxvnniint", "AvxVnniInt16", InstructionSet.X64_AVXVNNIINT, true); yield return new InstructionSetInfo("avxvnniint_v512", "AvxVnniInt16_V512", InstructionSet.X64_AVXVNNIINT_V512, true); + yield return new InstructionSetInfo("adx", "", InstructionSet.X64_ADX, true); break; case TargetArchitecture.X86: @@ -1342,6 +1347,7 @@ public static IEnumerable ArchitectureToValidInstructionSets yield return new InstructionSetInfo("avxvnniint_v512", "AvxVnniInt8_V512", InstructionSet.X86_AVXVNNIINT_V512, true); yield return new InstructionSetInfo("avxvnniint", "AvxVnniInt16", InstructionSet.X86_AVXVNNIINT, true); yield return new InstructionSetInfo("avxvnniint_v512", "AvxVnniInt16_V512", InstructionSet.X86_AVXVNNIINT_V512, true); + yield return new InstructionSetInfo("adx", "", InstructionSet.X86_ADX, true); break; } } diff --git a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt index a70cd7d9991687..4a261ff1040dc4 100644 --- a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt +++ b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt @@ -26,7 +26,7 @@ ; DO NOT CHANGE R2R NUMERIC VALUES OF THE EXISTING SETS. Changing R2R numeric values definitions would be R2R format breaking change. ; The ISA definitions should also be mapped to `hwintrinsicIsaRangeArray` in hwintrinsic.cpp. -; NEXT_AVAILABLE_R2R_BIT = 96 +; NEXT_AVAILABLE_R2R_BIT = 97 ; Definition of X86 instruction sets definearch ,X86 ,32Bit ,X64, X64, X86 @@ -206,6 +206,11 @@ definearch ,X64 ,64Bit ,X64, X64, X86 copyinstructionsets,X86 ,X64 +; Append new instruction sets here to preserve existing JIT ISA numbers. +; ADX is selected by JIT pattern recognition, so it has no managed intrinsic class name. +instructionset ,X86 , ,Adx ,96 ,ADX ,adx +instructionset ,X64 , ,Adx ,96 ,ADX ,adx + ; Definition of Arm64 instruction sets definearch ,ARM64 ,64Bit ,Arm64, Arm64, Arm diff --git a/src/coreclr/vm/codeman.cpp b/src/coreclr/vm/codeman.cpp index dcfdcfcf0d78e0..16fe0c6b058541 100644 --- a/src/coreclr/vm/codeman.cpp +++ b/src/coreclr/vm/codeman.cpp @@ -1546,6 +1546,11 @@ void EEJitManager::SetCpuInfo() CPUCompileFlags.Set(InstructionSet_AVX); } + if (((cpuFeatures & XArchIntrinsicConstants_Adx) != 0) && CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_EnableADX)) + { + CPUCompileFlags.Set(InstructionSet_ADX); + } + if (((cpuFeatures & XArchIntrinsicConstants_Avx2) != 0) && CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_EnableAVX2)) { CPUCompileFlags.Set(InstructionSet_AVX2); diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.DivRem.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.DivRem.cs index b62e1877fe46b0..ea18e70f8fe0a5 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.DivRem.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.DivRem.cs @@ -1,7 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; namespace System.Numerics.Tensors { @@ -77,7 +79,15 @@ public static void DivRem(T x, ReadOnlySpan y, Span quotientDestination /// Math.DivRem(x, y) private readonly struct DivRemOperator : IBinaryInputBinaryOutput where T : IBinaryInteger { - public static bool Vectorizable => true; + // On x64, 64-bit integer vector division is scalarized. Calling + // DivRem per element preserves each divide's hardware remainder + // instead of reconstructing it with a vector multiply/subtract. + public static bool Vectorizable + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => !X86Base.X64.IsSupported || + (typeof(T) != typeof(long) && typeof(T) != typeof(ulong)); + } public static (T, T) Invoke(T x, T y) => T.DivRem(x, y); diff --git a/src/libraries/System.Private.CoreLib/src/System/Decimal.DecCalc.cs b/src/libraries/System.Private.CoreLib/src/System/Decimal.DecCalc.cs index a0adc37bc6d996..23cd9ef8861d62 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Decimal.DecCalc.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Decimal.DecCalc.cs @@ -188,6 +188,24 @@ private static (uint Quotient, uint Remainder) Div64By32(ulong dividend, uint de } } + // Keep the limbs separate so x86/x64 can use them directly as the + // widening divide inputs. The incoming high limb must be below den. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static (uint Quotient, uint Remainder) Div64By32(uint low, uint high, uint den) + { + if (X86.X86Base.IsSupported) + { + return X86.X86Base.DivRem(low, high, den); + } + + ulong dividend = ((ulong)high << 32) | low; + uint quotient = (uint)(dividend / den); + // high < den guarantees that the quotient fits in uint. The remainder is + // below den and fits in uint too, so computing dividend - quotient * den + // modulo 2^32 gives the exact remainder using only the low limb. + return (quotient, low - quotient * den); + } + /// /// Do full divide, yielding 96-bit result and 32-bit remainder. /// @@ -200,6 +218,7 @@ private static uint Div96By32(ref Buf12 bufNum, uint den) if (X86.X86Base.IsSupported) { uint remainder = 0; + uint quotient; if (bufNum.U2 != 0) goto Div3Word; @@ -210,11 +229,14 @@ private static uint Div96By32(ref Buf12 bufNum, uint den) bufNum.U1 = 0; goto Div1Word; Div3Word: - (bufNum.U2, remainder) = X86.X86Base.DivRem(bufNum.U2, remainder, den); + (quotient, remainder) = X86.X86Base.DivRem(bufNum.U2, remainder, den); + bufNum.U2 = quotient; Div2Word: - (bufNum.U1, remainder) = X86.X86Base.DivRem(bufNum.U1, remainder, den); + (quotient, remainder) = X86.X86Base.DivRem(bufNum.U1, remainder, den); + bufNum.U1 = quotient; Div1Word: - (bufNum.U0, remainder) = X86.X86Base.DivRem(bufNum.U0, remainder, den); + (quotient, remainder) = X86.X86Base.DivRem(bufNum.U0, remainder, den); + bufNum.U0 = quotient; return remainder; } else @@ -237,7 +259,8 @@ private static uint Div96By32(ref Buf12 bufNum, uint den) tmp = bufNum.Low64; if (tmp == 0) return 0; - (bufNum.Low64, rem) = Math.DivRem(tmp, den); + (div, rem) = Math.DivRem(tmp, den); + bufNum.Low64 = div; return (uint)rem; } } @@ -477,6 +500,35 @@ private static uint Div128By96(ref Buf16 bufNum, ref Buf12 bufDen) // Compute full remainder, rem = dividend - (quo * divisor). // ulong prod1; +#if TARGET_64BIT + // A 64-by-32-bit product has at most 32 high bits. Keep that + // result wide so the subtraction can consume the low borrow directly. + ulong prod2 = Math.BigMul(bufDen.Low64, quo, out prod1); + ulong low = bufNum.Low64; + ulong num = low - prod1; + // Keep the high word signed so its sign records the full-width + // borrow. Both subtraction and correction stay within Int64. + long high = (long)remainder - (long)prod2; + high -= (num > low) ? 1L : 0L; + + if (high < 0) + { + // The quotient estimate is at most two too large. Add the + // divisor back, including the low word's carry, until nonnegative. + prod1 = bufDen.Low64; + do + { + quo--; + num += prod1; + high += den; + high += (num < prod1) ? 1L : 0L; + } while (high < 0); + } + + bufNum.Low64 = num; + bufNum.U2 = (uint)high; + return quo; +#else uint prod2 = (uint)Math.BigMul(bufDen.Low64, quo, out prod1); ulong num = bufNum.Low64 - prod1; remainder -= (uint)prod2; @@ -521,6 +573,7 @@ private static uint Div128By96(ref Buf16 bufNum, ref Buf12 bufDen) bufNum.Low64 = num; bufNum.U2 = remainder; return quo; +#endif } /// @@ -935,12 +988,12 @@ private static int SearchScale(ulong resMidLo, uint resHi, int scale) /// Returns false if there is an overflow private static bool Add32To96(ref Buf12 bufNum, uint value) { - if ((bufNum.Low64 += value) < value) - { - if (++bufNum.U2 == 0) - return false; - } - return true; + ulong low = bufNum.Low64 + value; + ulong high = bufNum.U2; + high += low < value ? 1UL : 0UL; + bufNum.Low64 = low; + bufNum.U2 = (uint)high; + return high <= uint.MaxValue; } /// @@ -1237,6 +1290,14 @@ internal static void DecAddSub(ref DecCalc d1, ref DecCalc d2, bool sign) // Signs differ - subtract // low64 = d1Low64 - d2.Low64; +#if TARGET_64BIT + // The signed high word retains the full-width borrow. + long difference = (long)d1High - d2.High; + difference -= low64 > d1Low64 ? 1L : 0L; + high = (uint)difference; + if (difference < 0) + goto SignFlip; +#else high = d1High - d2.High; // Propagate carry @@ -1249,12 +1310,21 @@ internal static void DecAddSub(ref DecCalc d1, ref DecCalc d2, bool sign) } else if (high > d1High) goto SignFlip; +#endif } else { // Signs are the same - add // low64 = d1Low64 + d2.Low64; +#if TARGET_64BIT + // Keep the 97th bit until the overflow/rescaling check. + ulong sum = (ulong)d1High + d2.High; + sum += low64 < d1Low64 ? 1UL : 0UL; + high = (uint)sum; + if (sum > uint.MaxValue) + goto AlignedScale; +#else high = d1High + d2.High; // Propagate carry @@ -1267,6 +1337,7 @@ internal static void DecAddSub(ref DecCalc d1, ref DecCalc d2, bool sign) } else if (high < d1High) goto AlignedScale; +#endif } goto ReturnResult; } @@ -1508,7 +1579,7 @@ internal static void VarDecMul(ref DecCalc d1, ref DecCalc d2) { // At least one operand has bits set in the upper 64 bits. // - // Compute and accumulate the 9 partial products into a + // Compute and accumulate the four partial products into a // 192-bit (3*64bit) result. // // [l-hi][l-lo] left high32, low64 @@ -1531,10 +1602,22 @@ internal static void VarDecMul(ref DecCalc d1, ref DecCalc d2) // hi64 will never overflow since the result will always fit in 192 (2*96) bits ulong hi64 = Math.BigMul(d1.High, d2.High); - // Do crosswise multiplications between upper 32bit and lower 64 bits + // Do crosswise multiplications between upper 32bit and lower 64 bits. +#if TARGET_64BIT + // Consume d1.High first so MULX need not preserve it across the other product. + ulong crossHigh = Math.BigMul(d2.Low64, d1.High, out tmp); + mid64 += tmp; + // Add the high product and carry together after the low addition, + // so the JIT can combine them into an add-with-carry instruction. + hi64 = hi64 + crossHigh + ((mid64 < tmp) ? 1UL : 0UL); + + crossHigh = Math.BigMul(d1.Low64, d2.High, out tmp); + mid64 += tmp; + hi64 = hi64 + crossHigh + ((mid64 < tmp) ? 1UL : 0UL); +#else + // Keep conditional increments when 64-bit arithmetic is decomposed into 32-bit operations. hi64 += Math.BigMul(d1.Low64, d2.High, out tmp); mid64 += tmp; - // propagate carry, can be simplified if https://github.com/dotnet/runtime/issues/48247 is done if (mid64 < tmp) ++hi64; @@ -1542,6 +1625,7 @@ internal static void VarDecMul(ref DecCalc d1, ref DecCalc d2) mid64 += tmp; if (mid64 < tmp) ++hi64; +#endif bufProd.Mid64 = mid64; bufProd.High64 = hi64; @@ -2252,8 +2336,19 @@ internal static void VarDecMod(ref DecCalc d1, ref DecCalc d2) { uint den = d2.Low; ulong tmp = ((ulong)d1.High << 32) | d1.Mid; - tmp = ((tmp % den) << 32) | d1.Low; - d1.Low64 = tmp % den; + if (X86.X86Base.IsSupported) + { + // The first remainder is below den, so the final quotient + // fits in 32 bits and the limbs can feed a widening divide. + uint remainder = (uint)(tmp % den); + (_, remainder) = Div64By32(d1.Low, remainder, den); + d1.Low64 = remainder; + } + else + { + tmp = ((tmp % den) << 32) | d1.Low; + d1.Low64 = tmp % den; + } d1.High = 0; } else @@ -2433,7 +2528,6 @@ internal static void InternalRound(ref DecCalc d, uint scale, MidpointRounding m { power = UInt32Powers10[(int)scale]; - // TODO: https://github.com/dotnet/runtime/issues/5213 uint n = d.uhi; if (n == 0) { @@ -2445,25 +2539,34 @@ internal static void InternalRound(ref DecCalc d, uint scale, MidpointRounding m remainder = 0; goto checkRemainder; } +#if TARGET_64BIT + (ulong quotient, ulong rem) = Math.DivRem(tmp, power); + d.Low64 = quotient; + remainder = (uint)rem; +#else + // Keep the remainder calculation narrow on 32-bit targets. ulong div = tmp / power; d.Low64 = div; remainder = (uint)(tmp - div * power); +#endif } else { - uint q; - (d.uhi, remainder) = Math.DivRem(n, power); + (uint highQuotient, remainder) = Math.DivRem(n, power); + d.uhi = highQuotient; + // Each remainder is less than power, so the next quotient + // fits in 32 bits. Reuse the widening divide's remainder. n = d.umid; if ((n | remainder) != 0) { - d.umid = q = (uint)((((ulong)remainder << 32) | n) / power); - remainder = n - q * power; + (uint quotient, remainder) = Div64By32(n, remainder, power); + d.umid = quotient; } n = d.ulo; if ((n | remainder) != 0) { - d.ulo = q = (uint)((((ulong)remainder << 32) | n) / power); - remainder = n - q * power; + (uint quotient, remainder) = Div64By32(n, remainder, power); + d.ulo = quotient; } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Int128.cs b/src/libraries/System.Private.CoreLib/src/System/Int128.cs index 6b9116968224fe..0dc630f1205958 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Int128.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Int128.cs @@ -701,8 +701,27 @@ public static implicit operator Int128(sbyte value) /// public static (Int128 Quotient, Int128 Remainder) DivRem(Int128 left, Int128 right) { +#if TARGET_64BIT + if ((right == -1) && (left == MinValue)) + { + ThrowHelper.ThrowOverflowException(); + } + + bool leftNegative = IsNegative(left); + bool rightNegative = IsNegative(right); + UInt128 dividend = (UInt128)(leftNegative ? -left : left); + UInt128 divisor = (UInt128)(rightNegative ? -right : right); + (UInt128 quotient, UInt128 remainder) = UInt128.DivRem(dividend, divisor); + + // The unsigned divide already computed the remainder. Its sign follows + // the dividend, independently of the quotient's sign. + return ((leftNegative != rightNegative) ? -(Int128)quotient : (Int128)quotient, + leftNegative ? -(Int128)remainder : (Int128)remainder); +#else + // Retain the smaller form when 128-bit arithmetic is decomposed into 32-bit limbs. Int128 quotient = left / right; return (quotient, left - (quotient * right)); +#endif } /// @@ -1119,10 +1138,14 @@ public static Int128 Log2(Int128 value) /// public static Int128 operator %(Int128 left, Int128 right) +#if TARGET_64BIT + => DivRem(left, right).Remainder; +#else { Int128 quotient = left / right; return left - (quotient * right); } +#endif // // IMultiplicativeIdentity diff --git a/src/libraries/System.Private.CoreLib/src/System/Number.BigInteger.cs b/src/libraries/System.Private.CoreLib/src/System/Number.BigInteger.cs index 50656f9dcff6b2..e06b9acd4e51b6 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Number.BigInteger.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Number.BigInteger.cs @@ -686,13 +686,13 @@ public static uint HeuristicDivide(scoped ref BigInteger dividend, scoped ref re dividend._length = divisorLength; } - // If the dividend is still larger than the divisor, we overshot our estimate quotient. To correct, - // we increment the quotient and subtract one more divisor from the dividend (Because we guaranteed the error range). + // If the remainder is still at least the divisor, the estimated quotient was too small. + // Increment the quotient and subtract one more divisor; the estimate is at most one below the true quotient. if (Compare(ref dividend, in divisor) >= 0) { quotient++; - // dividend -= divisor. This is the cold correction path (only on overshoot), so it + // dividend -= divisor. This is the cold correction path (only on an underestimate), so it // reuses the shared SubWithBorrow leaf, which inlines into this local loop. nuint borrow = 0; for (int i = 0; i < divisorLength; i++) diff --git a/src/libraries/System.Private.CoreLib/src/System/Number.DecimalIeee754.cs b/src/libraries/System.Private.CoreLib/src/System/Number.DecimalIeee754.cs index 2e784e4731c33f..9aa479b8c23896 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Number.DecimalIeee754.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Number.DecimalIeee754.cs @@ -2446,6 +2446,48 @@ private static void WideMultiply(TValue left, TValue right, out TValue h return; } + if (typeof(TValue) == typeof(UInt128)) + { + UInt128 a = UInt128.CreateTruncating(left); + UInt128 b = UInt128.CreateTruncating(right); + ulong p00High = Math.BigMul(a.Lower, b.Lower, out ulong p00Low); + + // Preserve the common small-coefficient path: its full product + // needs only one widening multiply. + if ((a.Upper | b.Upper) == 0) + { + high = TValue.Zero; + low = TValue.CreateTruncating(new UInt128(p00High, p00Low)); + return; + } + + ulong p01High = Math.BigMul(a.Lower, b.Upper, out ulong p01Low); + ulong p10High = Math.BigMul(a.Upper, b.Lower, out ulong p10Low); + ulong p11High = Math.BigMul(a.Upper, b.Upper, out ulong p11Low); + + // Accumulate the four exact partial products in base 2^64. + // Each comparison produces a one-bit carry; the two carries + // from the second word can sum to two in the third word. + ulong word1 = p00High + p01Low; + ulong carry1 = (word1 < p00High) ? 1UL : 0UL; + ulong previous = word1; + word1 += p10Low; + carry1 += (word1 < previous) ? 1UL : 0UL; + + ulong word2 = p01High + p10High; + ulong carry2 = (word2 < p01High) ? 1UL : 0UL; + previous = word2; + word2 += p11Low; + carry2 += (word2 < previous) ? 1UL : 0UL; + previous = word2; + word2 += carry1; + carry2 += (word2 < previous) ? 1UL : 0UL; + + high = TValue.CreateTruncating(new UInt128(p11High + carry2, word2)); + low = TValue.CreateTruncating(new UInt128(word1, p00Low)); + return; + } + int bits = TValue.Zero.GetByteCount() * 8; int half = bits / 2; TValue lowMask = (TValue.One << half) - TValue.One; @@ -2570,13 +2612,30 @@ private static void AlignWideToCommonExponent(ref TValue high, private static void WideAdd(TValue leftHigh, TValue leftLow, TValue rightHigh, TValue rightLow, out TValue high, out TValue low) where TValue : unmanaged, IBinaryInteger { - low = leftLow + rightLow; - high = leftHigh + rightHigh; - - if (low < leftLow) - { - high += TValue.One; +#if TARGET_64BIT + if (typeof(TValue) == typeof(UInt128)) + { + UInt128 a0 = UInt128.CreateTruncating(leftLow); + UInt128 a1 = UInt128.CreateTruncating(leftHigh); + UInt128 b0 = UInt128.CreateTruncating(rightLow); + UInt128 b1 = UInt128.CreateTruncating(rightHigh); + // Feed each one-bit carry directly into the next native limb. + nuint w0 = BigIntegerCalculator.AddWithCarry((nuint)a0.Lower, (nuint)b0.Lower, 0, out nuint c); + nuint w1 = BigIntegerCalculator.AddWithCarry((nuint)a0.Upper, (nuint)b0.Upper, c, out c); + nuint w2 = BigIntegerCalculator.AddWithCarry((nuint)a1.Lower, (nuint)b1.Lower, c, out c); + nuint w3 = (nuint)a1.Upper + (nuint)b1.Upper + c; + high = TValue.CreateTruncating(new UInt128(w3, w2)); + low = TValue.CreateTruncating(new UInt128(w1, w0)); + return; } +#endif + + // Keep the low result and one-bit carry in locals so the JIT can + // consume the addition flags before writing the output limbs. + TValue sum = leftLow + rightLow; + TValue carry = sum < leftLow ? TValue.One : TValue.Zero; + high = (leftHigh + rightHigh) + carry; + low = sum; } /// @@ -2597,14 +2656,30 @@ private static int WideCompare(TValue leftHigh, TValue leftLow, TValue r private static void WideSubtract(TValue leftHigh, TValue leftLow, TValue rightHigh, TValue rightLow, out TValue high, out TValue low) where TValue : unmanaged, IBinaryInteger { - high = leftHigh - rightHigh; - - if (leftLow < rightLow) - { - high -= TValue.One; +#if TARGET_64BIT + if (typeof(TValue) == typeof(UInt128)) + { + UInt128 a0 = UInt128.CreateTruncating(leftLow); + UInt128 a1 = UInt128.CreateTruncating(leftHigh); + UInt128 b0 = UInt128.CreateTruncating(rightLow); + UInt128 b1 = UInt128.CreateTruncating(rightHigh); + // As with addition, the borrow remains a bit across all four limbs. + nuint w0 = BigIntegerCalculator.SubWithBorrow((nuint)a0.Lower, (nuint)b0.Lower, 0, out nuint b); + nuint w1 = BigIntegerCalculator.SubWithBorrow((nuint)a0.Upper, (nuint)b0.Upper, b, out b); + nuint w2 = BigIntegerCalculator.SubWithBorrow((nuint)a1.Lower, (nuint)b1.Lower, b, out b); + nuint w3 = (nuint)a1.Upper - (nuint)b1.Upper - b; + high = TValue.CreateTruncating(new UInt128(w3, w2)); + low = TValue.CreateTruncating(new UInt128(w1, w0)); + return; } +#endif - low = leftLow - rightLow; + // Compute the low subtraction before consuming its borrow in the + // high limb. Delay output stores until both results are available. + TValue difference = leftLow - rightLow; + TValue borrow = leftLow < rightLow ? TValue.One : TValue.Zero; + high = (leftHigh - rightHigh) - borrow; + low = difference; } /// @@ -2675,6 +2750,47 @@ private static TValue WideDivideByPow10(ref TValue high, ref TValue low, return remainder; } +#if TARGET_64BIT + if (typeof(TValue) == typeof(uint)) + { + ulong value = ((ulong)uint.CreateTruncating(high) << 32) | uint.CreateTruncating(low); + (ulong quotient, ulong remainder) = Math.DivRem(value, uint.CreateTruncating(divisor)); + high = TValue.CreateTruncating(quotient >> 32); + low = TValue.CreateTruncating(quotient); + return TValue.CreateTruncating(remainder); + } +#endif + + // A native widening divide consumes an entire limb and returns its + // remainder. Start with ordinary division; each following high + // input is then a remainder strictly below the divisor. + if (typeof(TValue) == typeof(ulong) && Runtime.Intrinsics.X86.X86Base.X64.IsSupported) + { + ulong d = ulong.CreateTruncating(divisor); + (ulong upper, ulong remainder) = Math.DivRem(ulong.CreateTruncating(high), d); + ulong lower = (ulong)BigIntegerCalculator.DivRem((nuint)remainder, (nuint)ulong.CreateTruncating(low), (nuint)d, out nuint nativeRemainder); + high = TValue.CreateTruncating(upper); + low = TValue.CreateTruncating(lower); + return TValue.CreateTruncating(nativeRemainder); + } + + if (typeof(TValue) == typeof(UInt128) && Runtime.Intrinsics.X86.X86Base.X64.IsSupported) + { + // SinglePassPow10 bounds this divisor by 10^19, so it fits in + // one ulong even though the dividend occupies four ulongs. + UInt128 h = UInt128.CreateTruncating(high); + UInt128 l = UInt128.CreateTruncating(low); + UInt128 d = UInt128.CreateTruncating(divisor); + Debug.Assert(d.Upper == 0); + (ulong digit3, ulong remainder) = Math.DivRem(h.Upper, d.Lower); + nuint digit2 = BigIntegerCalculator.DivRem((nuint)remainder, (nuint)h.Lower, (nuint)d.Lower, out nuint nativeRemainder); + nuint digit1 = BigIntegerCalculator.DivRem(nativeRemainder, (nuint)l.Upper, (nuint)d.Lower, out nativeRemainder); + nuint digit0 = BigIntegerCalculator.DivRem(nativeRemainder, (nuint)l.Lower, (nuint)d.Lower, out nativeRemainder); + high = TValue.CreateTruncating(new UInt128(digit3, digit2)); + low = TValue.CreateTruncating(new UInt128(digit1, digit0)); + return TValue.CreateTruncating(nativeRemainder); + } + int bits = TValue.Zero.GetByteCount() * 8; int half = bits / 2; TValue lowMask = (TValue.One << half) - TValue.One; diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/BigIntegerCalculator.Shared.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/BigIntegerCalculator.Shared.cs index c573a23c4f3e1a..261494a97bcb7e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/BigIntegerCalculator.Shared.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/BigIntegerCalculator.Shared.cs @@ -184,13 +184,18 @@ internal static void DivideByPowerOfTwo(ReadOnlySpan left, int exponent, /// /// Performs widening addition of two limbs plus a carry-in, returning the sum and carry-out. - /// On 64-bit: uses 128-bit arithmetic. On 32-bit: uses 64-bit arithmetic. + /// On 64-bit: uses limb additions and unsigned carry comparisons. On 32-bit: uses 64-bit arithmetic. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static nuint AddWithCarry(nuint a, nuint b, nuint carryIn, out nuint carryOut) { if (nint.Size == 8) { + // Keep each unsigned comparison tied to its own addition. Supporting JITs can reuse the + // addition's flags and, when carryIn is proven to be 0 or 1, fold both steps into ADC/ADCS. + // For a limb chain, initialize carry to zero and feed carryOut directly into the next limb. + // Then c1 and c2 cannot both be one, so carryOut remains a bit. An arbitrary full-limb + // carryIn can instead produce carryOut == 2 and must not be treated as a single carry flag. nuint sum1 = a + b; nuint c1 = (sum1 < a) ? 1 : (nuint)0; nuint sum2 = sum1 + carryIn; @@ -208,7 +213,8 @@ internal static nuint AddWithCarry(nuint a, nuint b, nuint carryIn, out nuint ca /// /// Performs widening subtraction of two limbs with a borrow-in, returning the difference and borrow-out. - /// borrowOut is 0 (no borrow) or 1 (borrow occurred). + /// When borrowIn is 0 or 1, borrowOut is also 0 (no borrow) or 1 (borrow occurred). + /// An arbitrary full-limb borrowIn can produce borrowOut == 2. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static nuint SubWithBorrow(nuint a, nuint b, nuint borrowIn, out nuint borrowOut) @@ -226,7 +232,7 @@ internal static nuint SubWithBorrow(nuint a, nuint b, nuint borrowIn, out nuint else { long diff = (long)a - (long)b - (long)borrowIn; - borrowOut = (uint)(-(int)(diff >> 32)); // 0 or 1 + borrowOut = (uint)(-(int)(diff >> 32)); // 0 or 1 for a one-bit borrowIn; otherwise up to 2 return (uint)diff; } } @@ -237,12 +243,30 @@ internal static nuint SubWithBorrow(nuint a, nuint b, nuint borrowIn, out nuint [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static nuint DivRem(nuint hi, nuint lo, nuint divisor, out nuint remainder) { + Debug.Assert(hi < divisor || divisor == 0); + // Callers ensure hi < divisor, so the quotient fits in one limb. + // Use the widening divide before splitting into smaller divisions; + // it produces both the quotient and remainder in one instruction. +#pragma warning disable SYSLIB5004 // X86Base.DivRem is experimental + if (X86Base.X64.IsSupported) + { + (ulong q, ulong r) = X86Base.X64.DivRem(lo, hi, divisor); + remainder = (nuint)r; + return (nuint)q; + } + + if (nint.Size == 4 && X86Base.IsSupported) + { + (uint q, uint r) = X86Base.DivRem((uint)lo, (uint)hi, (uint)divisor); + remainder = r; + return q; + } +#pragma warning restore SYSLIB5004 + if (nint.Size == 8) { // Compute (hi * 2^64 + lo) / divisor. // hi < divisor is guaranteed by callers, so quotient fits in 64 bits. - Debug.Assert(hi < (ulong)divisor || divisor == 0); - if (hi == 0) { (ulong q, ulong r) = Math.DivRem(lo, (ulong)divisor); @@ -267,18 +291,9 @@ internal static nuint DivRem(nuint hi, nuint lo, nuint divisor, out nuint remain } { -#pragma warning disable SYSLIB5004 // X86Base.DivRem is experimental - if (X86Base.X64.IsSupported) - { - (ulong q, ulong r) = X86Base.X64.DivRem(lo, hi, divisor); - remainder = (nuint)r; - return (nuint)q; - } -#pragma warning restore SYSLIB5004 - UInt128 value = ((UInt128)(ulong)hi << 64) | (ulong)lo; - UInt128 digit = value / (ulong)divisor; - remainder = (nuint)(ulong)(value - digit * (ulong)divisor); + (UInt128 digit, UInt128 rem) = UInt128.DivRem(value, (ulong)divisor); + remainder = (nuint)(ulong)rem; return (nuint)(ulong)digit; } } @@ -295,6 +310,8 @@ internal static nuint DivRem(nuint hi, nuint lo, nuint divisor, out nuint remain /// Multiply by scalar: result[0..left.Length] = left * multiplier. /// Returns the carry out. Unrolled by 4 on 64-bit. /// Unlike MulAdd1, this writes to result rather than accumulating. + /// Widen before multiplying, store the low limb, and pass the high limb to the next product. + /// This carry is a full limb, unlike the one-bit carry used by an addition chain. /// internal static nuint Mul1(Span result, ReadOnlySpan left, nuint multiplier) { @@ -345,22 +362,29 @@ internal static nuint Mul1(Span result, ReadOnlySpan left, nuint m /// /// Fused multiply-accumulate by scalar: result[0..left.Length] += left * multiplier. - /// Returns the carry out. Unrolled by 4 on 64-bit to overlap multiply latencies. + /// Returns the carry out. Processes four limbs per iteration on 64-bit platforms. /// internal static nuint MulAdd1(Span result, ReadOnlySpan left, nuint multiplier) { - Debug.Assert(result.Length >= left.Length); - int length = left.Length; + // Give both spans the same bound so the loop checks cover every lane. This slice validates + // the destination once; Debug.Assert alone would not establish the bound in release builds. + result = result.Slice(0, length); int i = 0; nuint carry = 0; if (nint.Size == 8) { - // Unroll by 4: mulx has 3-5 cycle latency but 1 cycle throughput, - // so issuing 4 multiplies allows the CPU to pipeline them while - // carry chains complete sequentially behind. - for (; i + 3 < length; i += 4) + // Unroll by four to expose independent products while propagating carry between limbs. + // Widen before multiplication and keep both additions in UInt128: even the maximum product + // plus a destination limb and a full-limb carry fits. Store each low limb and feed its high + // limb directly into the next product; the last high limb also feeds the next iteration. + // Keep the i < length - 3 loop bound so the JIT can prove all four accesses are in range. + // On x64 with BMI2 and ADX, the JIT can use MULX with ADCX/ADOX and keep both carry + // flags live across loop iterations, materializing the carry before the remainder loop. + // ARM64 can similarly preserve C with MUL/UMULH/ADCS/ADC/ADDS. Let the JIT derive the + // countdown from this ascending loop; a separate index and countdown can obscure the proof. + for (; i < length - 3; i += 4) { UInt128 p0 = (UInt128)(ulong)left[i] * (ulong)multiplier + (ulong)result[i] + (ulong)carry; result[i] = (nuint)(ulong)p0; @@ -515,11 +539,14 @@ public static void Add(ReadOnlySpan left, ReadOnlySpan right, Span } } - // Establish cross-span length relationships so the JIT can - // elide bounds checks for left[i] and bits[i] in the loop. + // Establish cross-span length relationships so the JIT can elide bounds checks for left[i] + // and bits[i]. These executed checks, unlike Debug.Assert, establish the bounds in release builds. _ = left[right.Length - 1]; _ = bits[right.Length]; + // The zero seed and AddWithCarry recurrence prove that carry stays a bit. Keep the helper + // inlineable and use the same index for all spans: supporting JITs can keep CF (x64) or C + // (ARM64) live across a derived countdown loop and materialize carry only when leaving it. nuint carry = 0; for (int i = 0; i < right.Length; i++) @@ -591,6 +618,8 @@ public static void AddSelf(Span left, ReadOnlySpan right) _ = left[right.Length - 1]; } + // As in Add, keep the one-bit carry recurrence in this ascending loop. The final carry + // is still needed by the tail below, even when the JIT carries it in flags inside this loop. for (; i < right.Length; i++) { left[i] = AddWithCarry(left[i], right[i], carry, out carry); @@ -1772,10 +1801,8 @@ private static bool DivideGuessTooBig(nuint q, nuint valHi1, nuint valHi0, nuint chkLoHi = nuint.BigMul(divLo, q, out nuint chkLoLo); chkHiLo += chkLoHi; - if (chkHiLo < chkLoHi) - { - chkHiHi++; - } + // Add the carry explicitly so the JIT can reuse the low addition's flags. + chkHiHi += (chkHiLo < chkLoHi) ? (nuint)1 : 0; return (chkHiHi > valHi1) || ((chkHiHi == valHi1) && ((chkHiLo > valHi0) || ((chkHiLo == valHi0) && (chkLoLo > valLo)))); diff --git a/src/libraries/System.Private.CoreLib/src/System/UInt128.cs b/src/libraries/System.Private.CoreLib/src/System/UInt128.cs index df6f7d19a6081a..97047b261a1025 100644 --- a/src/libraries/System.Private.CoreLib/src/System/UInt128.cs +++ b/src/libraries/System.Private.CoreLib/src/System/UInt128.cs @@ -838,10 +838,8 @@ public static (UInt128 Quotient, UInt128 Remainder) DivRem(UInt128 left, UInt128 ulong chkLoHi = Math.BigMul(divLo, q, out ulong chkLo); ulong chkMi = chkHiLo + chkLoHi; - if (chkMi < chkLoHi) - { - chkHi++; - } + // Add the carry explicitly so the JIT can reuse the low addition's flags. + chkHi += (chkMi < chkLoHi) ? 1UL : 0UL; UInt128 divisor = new UInt128(divHi, divLo); UInt128 chkMiLo = new UInt128(chkMi, chkLo); @@ -985,11 +983,19 @@ public static (UInt128 Quotient, UInt128 Remainder) DivRem(UInt128 left, UInt128 Debug.Assert(new UInt128(valHi, valMiLo) < new UInt128(chkHi, chkLo)); q--; +#if TARGET_64BIT + // Subtract the low word first and consume its borrow in the + // high word, allowing SUB/SBB or SUBS/SBC without a branch. + ulong difference = chkLo - divisor; + chkHi -= (difference > chkLo) ? 1U : 0U; + chkLo = difference; +#else if (chkLo < divisor) { chkHi--; } chkLo -= divisor; +#endif } ulong remainder = valMiLo - chkLo; @@ -1412,12 +1418,14 @@ public static UInt128 BigMul(UInt128 left, UInt128 right, out UInt128 lower) ulong bl = right._lower; ulong bh = right._upper; - UInt128 mull = Math.BigMul(al, bl); - UInt128 t = Math.BigMul(ah, bl) + mull._upper; - UInt128 tl = Math.BigMul(al, bh) + t._lower; + // Group products with the same first operand: MULX uses a fixed + // register for that operand, while the other operand can stay live. + UInt128 mull = Math.BigMul(bl, al); + UInt128 t = Math.BigMul(bl, ah) + mull._upper; + UInt128 tl = Math.BigMul(bh, al) + t._lower; lower = new UInt128(tl._lower, mull._lower); - return Math.BigMul(ah, bh) + t._upper + tl._upper; + return Math.BigMul(bh, ah) + t._upper + tl._upper; } // diff --git a/src/libraries/System.Private.CoreLib/src/System/UIntPtr.cs b/src/libraries/System.Private.CoreLib/src/System/UIntPtr.cs index 5343dc70dea2bd..f2bf581248b7e7 100644 --- a/src/libraries/System.Private.CoreLib/src/System/UIntPtr.cs +++ b/src/libraries/System.Private.CoreLib/src/System/UIntPtr.cs @@ -184,6 +184,7 @@ public static nuint MinValue /// The integer to multiply with . /// The lower half of the full product. /// The upper half of the full product. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static nuint BigMul(nuint left, nuint right, out nuint lower) { #if TARGET_64BIT diff --git a/src/libraries/System.Runtime.Numerics/src/System/Numerics/BigIntegerCalculator.PowMod.cs b/src/libraries/System.Runtime.Numerics/src/System/Numerics/BigIntegerCalculator.PowMod.cs index 9dbb64a65ea084..ffaf82cda2dcb9 100644 --- a/src/libraries/System.Runtime.Numerics/src/System/Numerics/BigIntegerCalculator.PowMod.cs +++ b/src/libraries/System.Runtime.Numerics/src/System/Numerics/BigIntegerCalculator.PowMod.cs @@ -857,15 +857,28 @@ private static int MontgomeryReduce(Span value, ReadOnlySpan modul nuint m = value[i] * n0inv; nuint carry = 0; - for (int j = 0; j < k; j++) + if (nint.Size == 8) { - if (nint.Size == 8) + // Keep both spans the same length and index them with j, rather than value[i + j]. + // The slice checks the whole row once, allowing the JIT to eliminate per-limb bounds checks. + // Use the ordinary ascending loop: the JIT can derive a countdown that preserves carry flags; + // a separate hand-written countdown and index can obscure the bounds proof. + // This multiply-add shape lets supporting JITs keep carry flags live across the backedge + // (MULX/ADCX/ADOX on x64, MUL/UMULH/ADCS/ADC/ADDS on ARM64), without unrolling or a helper call. + Span row = value.Slice(i, k); + for (int j = 0; j < row.Length; j++) { - UInt128 p = (UInt128)m * modulus[j] + value[i + j] + carry; - value[i + j] = (nuint)(ulong)p; + // Widen before multiplying and keep both additions in UInt128. The incoming carry is + // a full limb, not a single bit: product + row[j] + carry fits in 128 bits. Store the low + // limb and feed the high limb directly into the next iteration; preserve it on loop exit. + UInt128 p = (UInt128)(ulong)modulus[j] * (ulong)m + (ulong)row[j] + (ulong)carry; + row[j] = (nuint)(ulong)p; carry = (nuint)(ulong)(p >> 64); } - else + } + else + { + for (int j = 0; j < k; j++) { ulong p = (ulong)m * modulus[j] + value[i + j] + carry; value[i + j] = (uint)p; diff --git a/src/native/minipal/cpufeatures.c b/src/native/minipal/cpufeatures.c index ed7d7612779842..7b4083050d9a5e 100644 --- a/src/native/minipal/cpufeatures.c +++ b/src/native/minipal/cpufeatures.c @@ -349,6 +349,11 @@ int minipal_getcpufeatures(void) { __cpuidex(cpuidInfo, 0x00000007, 0x00000000); + if ((cpuidInfo[CPUID_EBX] & (1 << 19)) != 0) // ADX + { + result |= XArchIntrinsicConstants_Adx; + } + if ((result & XArchIntrinsicConstants_Avx) != 0) { if (((cpuidInfo[CPUID_EBX] & (1 << 5)) != 0) && // AVX2 diff --git a/src/native/minipal/cpufeatures.h b/src/native/minipal/cpufeatures.h index 31786bc641971b..9476dede581c7d 100644 --- a/src/native/minipal/cpufeatures.h +++ b/src/native/minipal/cpufeatures.h @@ -31,6 +31,7 @@ #define XArchIntrinsicConstants_WaitPkg (1 << 16) #define XArchIntrinsicConstants_X86Serialize (1 << 17) #define XArchIntrinsicConstants_Avx512Bmm (1 << 18) +#define XArchIntrinsicConstants_Adx (1 << 19) #endif // HOST_X86 || HOST_AMD64 #if defined(HOST_ARM64) diff --git a/src/tests/JIT/opt/Add/ArithmeticCodegen.cs b/src/tests/JIT/opt/Add/ArithmeticCodegen.cs new file mode 100644 index 00000000000000..4acfc0bbf3ee78 --- /dev/null +++ b/src/tests/JIT/opt/Add/ArithmeticCodegen.cs @@ -0,0 +1,292 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Threading; +using Xunit; + +public class ArithmeticCodegenTests +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong FoldMultiply(ulong left, ulong right) + { + // Independent products do not require flags-preserving multiplication. + // X64: {{^ +}}mul {{.*}} + // X64-NOT: mulx + // X64: ret + ulong high = Math.BigMul(left, right, out ulong low); + return high ^ low; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong FoldMultiplyWithCarry(ulong left, ulong right, ulong addend, out ulong high) + { + // An unrelated carry in the same block must not force this product to MULX. + // X64-NOT: mulx + // X64: {{^ +}}mul {{.*}} + // X64-NOT: mulx + // X64: {{^ +}}adc {{.*}} + // X64-NOT: mulx + // X64: ret + ulong productHigh = Math.BigMul(left, right, out ulong productLow); + ulong sum = unchecked(left + addend); + high = unchecked(right + (sum < left ? 1UL : 0UL)); + return productHigh ^ productLow ^ sum; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool DecrementIsZero(ref int value) + { + // ZF-only consumers can still use the shorter DEC instruction. + // X64: {{^ +}}dec + // X64-NEXT: sete + return Interlocked.Decrement(ref value) == 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong IncrementWithCarry(ulong value, out ulong high) + { + // INC would preserve stale CF instead of producing the overflow bit. + // X64: {{^ +}}add {{.*}}, 1 + // X64-NOT: {{^ +}}inc + // X64: ret + ulong result = unchecked(value + 1); + high = result < value ? 1UL : 0UL; + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong StoredRemainder(ulong value, ulong divisor, out ulong quotient) + { + ulong q = value / divisor; + ulong remainder = value - q * divisor; + quotient = q; + return remainder; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static (ulong, ulong, ulong) DestinationRead(ulong value, ulong divisor, ulong remainder) + { + ulong quotient = value / divisor; + ulong oldRemainder = remainder; + remainder = value - quotient * divisor; + return (quotient, remainder, oldRemainder); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ReplaceDividend(ulong value, ulong divisor) + { + ulong quotient = value / divisor; + value = value - quotient * divisor; + return value; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void MayThrow(bool fail) + { + if (fail) + { + throw new InvalidOperationException(); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong RemainderAcrossException(ulong value, ulong divisor, ulong remainder, bool fail) + { + try + { + ulong quotient = value / divisor; + MayThrow(fail); + remainder = value - quotient * divisor; + } + catch (InvalidOperationException) + { + return remainder; + } + return remainder; + } + + private static class CarryState + { + public static readonly ulong Mask; + static CarryState() => Mask = 0x123456789ABCDEF0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ObserveCarry(ulong carry) => carry; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong CarryAcrossHelper(ulong left, ulong right, ulong high, bool observe) + { + // Generic static initialization may split the block before lowering. + // The carry is consumed here and remains observable in successor blocks. + ulong mask = CarryState.Mask; + ulong low = unchecked(left + right); + ulong carry = low < left ? 1UL : 0UL; + high = unchecked(high + carry); + if (observe) + { + high ^= ObserveCarry(carry) + mask; + } + return low ^ high ^ carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong CarryAcrossException(ulong left, ulong right, ulong high, bool fail) + { + ulong carry = 123; + try + { + ulong low = unchecked(left + right); + carry = low < left ? 1UL : 0UL; + high = unchecked(high + carry); + MayThrow(fail); + return low ^ high; + } + catch (InvalidOperationException) + { + return carry; + } + } + + private struct LimbPair + { + public ulong Low; + public ulong High; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong CarryAcrossStructStore(ulong left, ulong right, LimbPair replacement) + { + LimbPair value = new LimbPair { Low = left, High = right }; + ulong sum = unchecked(value.Low + right); + ulong carry = sum < value.Low ? 1UL : 0UL; + // Reading High must observe the replacement, even if the field is promoted. + value = replacement; + return sum ^ unchecked(value.High + carry) ^ value.Low; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong BorrowAcrossStructStore(ulong left, ulong right, LimbPair replacement) + { + LimbPair value = new LimbPair { Low = left, High = right }; + ulong difference = unchecked(value.Low - right); + ulong borrow = value.Low < right ? 1UL : 0UL; + value = replacement; + return difference ^ unchecked(value.High - borrow) ^ value.Low; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong CarryAcrossFieldAlias(ulong left, ulong right, ulong high) + { + LimbPair value = new LimbPair { Low = left, High = right }; + ref ulong alias = ref value.High; + ulong sum = unchecked(value.Low + right); + ulong carry = sum < value.Low ? 1UL : 0UL; + alias = high; + return sum ^ unchecked(value.High + carry); + } + + [Fact] + public static void TestEntryPoint() + { + ulong[] edges = { 0, 1, 2, uint.MaxValue, 1UL << 32, 1UL << 63, ulong.MaxValue - 1, ulong.MaxValue }; + foreach (ulong left in edges) + { + foreach (ulong right in edges) + { + BigInteger product = (BigInteger)left * right; + ulong folded = (ulong)(product & ulong.MaxValue) ^ (ulong)(product >> 64); + if (FoldMultiply(left, right) != folded) + { + throw new Exception("FoldMultiply"); + } + foreach (ulong addend in edges) + { + LimbPair replacement = new LimbPair { Low = right, High = addend }; + ulong lowSum = unchecked(left + right); + ulong carry = (ulong)(((BigInteger)left + right) >> 64); + ulong difference = unchecked(left - right); + ulong borrow = left < right ? 1UL : 0UL; + if (CarryAcrossStructStore(left, right, replacement) != + (lowSum ^ unchecked(addend + carry) ^ right) || + BorrowAcrossStructStore(left, right, replacement) != + (difference ^ unchecked(addend - borrow) ^ right) || + CarryAcrossFieldAlias(left, right, addend) != (lowSum ^ unchecked(addend + carry))) + { + throw new Exception("Carry/borrow operand observed across struct or field store"); + } + ulong actual = FoldMultiplyWithCarry(left, right, addend, out ulong mixedHigh); + BigInteger sum = (BigInteger)left + addend; + ulong expected = (ulong)(product >> 64) ^ (ulong)(product & ulong.MaxValue) ^ + (ulong)(sum & ulong.MaxValue); + if (actual != expected || mixedHigh != unchecked(right + (ulong)(sum >> 64))) + { + throw new Exception("Independent multiply with carry"); + } + } + foreach (bool observe in new[] { false, true }) + { + BigInteger sum = (BigInteger)left + right; + ulong carry = (ulong)(sum >> 64); + ulong highPart = unchecked(left + carry); + if (observe) + { + highPart ^= carry + 0x123456789ABCDEF0; + } + ulong expected = (ulong)(sum & ulong.MaxValue) ^ highPart ^ carry; + if (CarryAcrossHelper(left, right, left, observe) != expected || + CarryAcrossHelper(left, right, left, observe) != expected) + { + throw new Exception("Carry observed across helper expansion and branches"); + } + } + ulong expectedCarry = (ulong)(((BigInteger)left + right) >> 64); + if (CarryAcrossException(left, right, left, true) != expectedCarry || + CarryAcrossException(left, right, left, false) != + (unchecked(left + right) ^ unchecked(left + expectedCarry))) + { + throw new Exception("Carry observed by exception handler"); + } + if (right != 0) + { + ulong rem = StoredRemainder(left, right, out ulong quotient); + var observed = DestinationRead(left, right, 0x123456789ABCDEF0); + if (rem != left % right || quotient != left / right || + observed != (quotient, rem, 0x123456789ABCDEF0UL) || ReplaceDividend(left, right) != rem) + { + throw new Exception("Stored remainder"); + } + } + } + if (RemainderAcrossException(left, 7, 123, true) != 123 || + RemainderAcrossException(left, 7, 123, false) != left % 7) + { + throw new Exception("Remainder exception ordering"); + } + if (IncrementWithCarry(left, out ulong high) != unchecked(left + 1) || + high != (left == ulong.MaxValue ? 1UL : 0UL)) + { + throw new Exception("Carry flag"); + } + } + foreach (int start in new[] { int.MinValue, -1, 0, 1, 2, int.MaxValue }) + { + int value = start; + if (DecrementIsZero(ref value) != (start == 1) || value != unchecked(start - 1)) + { + throw new Exception("Zero flag"); + } + } + try + { + StoredRemainder(1, 0, out _); + throw new Exception("Missing division exception"); + } + catch (DivideByZeroException) + { + } + } +} diff --git a/src/tests/JIT/opt/Add/ArithmeticCodegen.csproj b/src/tests/JIT/opt/Add/ArithmeticCodegen.csproj new file mode 100644 index 00000000000000..d1892ed3a4be23 --- /dev/null +++ b/src/tests/JIT/opt/Add/ArithmeticCodegen.csproj @@ -0,0 +1,12 @@ + + + true + None + True + + + + + + + diff --git a/src/tests/JIT/opt/Add/Borrow.cs b/src/tests/JIT/opt/Add/Borrow.cs new file mode 100644 index 00000000000000..22daf33c427454 --- /dev/null +++ b/src/tests/JIT/opt/Add/Borrow.cs @@ -0,0 +1,350 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Xunit; + +public class BorrowTests +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 Sub128(UInt128 a, UInt128 b) + { + // X64: sub + // X64: sbb + // ARM64: subs + // ARM64: sbc + return a - b; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Int128 SubSigned128(Int128 a, Int128 b) + { + // X64: sub + // X64: sbb + // ARM64: subs + // ARM64: sbc + return unchecked(a - b); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Borrow(ulong a, ulong b, ulong high, out ulong difference) + { + // X64: sub + // X64: adc + // ARM64: subs + // ARM64: cinc + ulong value = a - b; + ulong result = high + (value > a ? 1UL : 0); + difference = value; + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong BorrowOnly(ulong a, ulong b, ulong high, out ulong difference) + { + // X64: sbb + // ARM64: sbc {{.*}}, xzr + ulong value = a - b; + ulong result = high - (value > a ? 1UL : 0); + difference = value; + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Branch(ulong a, ulong b, out ulong difference) + { + difference = a - b; + if (difference > a) + { + return 13; + } + return 29; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Select(ulong a, ulong b, ulong x, ulong y, out ulong difference) + { + difference = a - b; + return difference > a ? x : y; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong Sub(ulong a, ulong b, ulong borrow, out ulong next) + { + ulong d = a - b; + ulong b1 = d > a ? 1UL : 0; + ulong r = d - borrow; + next = b1 + (r > d ? 1UL : 0); + return r; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong LimbLoop(Span result, ReadOnlySpan left, ReadOnlySpan right) + { + // X64: sbb + // X64-NOT: setb + // X64: jne + // X64: setb + // ARM64: sbcs + // ARM64-NOT: cset + // ARM64: cbnz + // ARM64: cset + result = result.Slice(0, left.Length); + right = right.Slice(0, left.Length); + ulong borrow = 0; + for (int i = 0; i < left.Length; i++) + { + result[i] = Sub(left[i], right[i], borrow, out borrow); + } + return borrow; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint LimbLoop32(Span result, ReadOnlySpan left, ReadOnlySpan right) + { + result = result.Slice(0, left.Length); + right = right.Slice(0, left.Length); + uint borrow = 0; + for (int i = 0; i < left.Length; i++) + { + uint d = left[i] - right[i]; + uint b1 = d > left[i] ? 1U : 0; + uint r = d - borrow; + borrow = b1 + (r > d ? 1U : 0); + result[i] = r; + } + return borrow; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ArbitraryBorrow(ulong a, ulong b, ulong carry, out ulong next) => Sub(a, b, carry, out next); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong VolatileLoop(Span result, ReadOnlySpan right) + { + ulong borrow = 0; + for (int i = 0; i < right.Length; i++) + { + result[i] = Sub(System.Threading.Volatile.Read(ref result[i]), right[i], borrow, out borrow); + } + return borrow; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong AcrossCall(ulong a, ulong b, ulong high) + { + ulong d = a - b; + ulong borrow = d > a ? 1UL : 0; + return Opaque(high) + borrow; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Opaque(ulong value) => ~value; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong SubMul(Span result, ReadOnlySpan right, ulong multiplier) + { + // X64: mul{{x| }} + // X64: sub + // X64-NEXT: adc + // ARM64: umulh + // ARM64: subs + // ARM64-NEXT: cinc + ulong carry = 0; + int i = 0; + for (; i + 3 < right.Length; i += 4) + { + UInt128 product0 = (UInt128)right[i + 0] * multiplier + carry; + ulong low0 = (ulong)product0; + ulong high0 = (ulong)(product0 >> 64); + ulong original0 = result[i + 0]; + result[i + 0] = original0 - low0; + high0 += original0 < low0 ? 1UL : 0; + UInt128 product1 = (UInt128)right[i + 1] * multiplier + high0; + ulong low1 = (ulong)product1; + ulong high1 = (ulong)(product1 >> 64); + ulong original1 = result[i + 1]; + result[i + 1] = original1 - low1; + high1 += original1 < low1 ? 1UL : 0; + UInt128 product2 = (UInt128)right[i + 2] * multiplier + high1; + ulong low2 = (ulong)product2; + ulong high2 = (ulong)(product2 >> 64); + ulong original2 = result[i + 2]; + result[i + 2] = original2 - low2; + high2 += original2 < low2 ? 1UL : 0; + UInt128 product3 = (UInt128)right[i + 3] * multiplier + high2; + ulong low3 = (ulong)product3; + ulong high3 = (ulong)(product3 >> 64); + ulong original3 = result[i + 3]; + result[i + 3] = original3 - low3; + high3 += original3 < low3 ? 1UL : 0; + carry = high3; + } + for (; i < right.Length; i++) + { + UInt128 product = (UInt128)right[i] * multiplier + carry; + ulong low = (ulong)product; + ulong high = (ulong)(product >> 64); + ulong original = result[i]; + result[i] = original - low; + high += original < low ? 1UL : 0; + carry = high; + } + return carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool CompareRight(ulong a, ulong b) => unchecked(a - b) > b; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong CheckedSub(ulong a, ulong b) => checked(a - b); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong LimbLoopWithTail(Span left, ReadOnlySpan right) + { + // X64: sbb + // X64-NOT: setb + // X64: jne + // X64: setb + // ARM64: sbcs + // ARM64-NOT: cset + // ARM64: cbnz + // ARM64: cset + if (right.Length != 0) _ = left[right.Length - 1]; + ulong borrow = 0; + int i = 0; + for (; i < right.Length; i++) + { + left[i] = Sub(left[i], right[i], borrow, out borrow); + } + for (; borrow != 0 && i < left.Length; i++) + { + ulong value = left[i]; + left[i] = value - borrow; + borrow = value == 0 ? 1UL : 0; + } + return borrow; + } + + [Fact] + public static void TestEntryPoint() + { + Random random = new Random(125803); + ulong[] boundary = { 0, 1, 2, 0x7FFFFFFFFFFFFFFF, 0x8000000000000000, ulong.MaxValue - 1, ulong.MaxValue }; + foreach (ulong a in boundary) + foreach (ulong b in boundary) + foreach (ulong c in boundary) + { + Check(a, b, c, ~c); + } + for (int trial = 0; trial < 10000; trial++) + { + ulong a = (ulong)random.NextInt64() << 1 | (uint)random.Next(2); + ulong b = (ulong)random.NextInt64() << 1 | (uint)random.Next(2); + Check(a, b, (ulong)random.NextInt64(), (ulong)random.NextInt64()); + } + for (int trial = 0; trial < 512; trial++) + { + int n = trial % 65; + int source = 2, rhs = n + 4; + int dest = (trial % 4) switch { 0 => 2 * n + 6, 1 => 2, 2 => 3, _ => 1 }; + ulong[] data = new ulong[3 * n + 8]; + random.NextBytes(MemoryMarshal.AsBytes(data.AsSpan())); + if (trial % 7 == 0) Array.Clear(data, source, n); + ulong[] expected = (ulong[])data.Clone(); + ulong borrow = 0; + for (int i = 0; i < n; i++) + { + BigInteger d = (BigInteger)expected[source + i] - expected[rhs + i] - borrow; + expected[dest + i] = (ulong)(d & ulong.MaxValue); + borrow = d.Sign < 0 ? 1UL : 0; + } + ulong[] multiply = (ulong[])data.Clone(); + ulong[] expectedMultiply = (ulong[])data.Clone(); + ulong multiplier = trial % 3 == 0 ? ulong.MaxValue : data[0]; + ulong highCarry = 0; + for (int i = 0; i < n; i++) + { + BigInteger product = (BigInteger)expectedMultiply[source + i] * multiplier + highCarry; + BigInteger difference = (BigInteger)expectedMultiply[dest + i] - (product & ulong.MaxValue); + expectedMultiply[dest + i] = (ulong)(difference & ulong.MaxValue); + highCarry = (ulong)(product >> 64) + (difference.Sign < 0 ? 1UL : 0); + } + ulong actualHigh = SubMul(multiply.AsSpan(dest, n), multiply.AsSpan(source, n), multiplier); + if (actualHigh != highCarry || !multiply.AsSpan().SequenceEqual(expectedMultiply)) throw new Exception("SubMul"); + + ulong actual = LimbLoop(data.AsSpan(dest, n), data.AsSpan(source, n), data.AsSpan(rhs, n)); + if (actual != borrow || !data.AsSpan().SequenceEqual(expected)) throw new Exception("LimbLoop"); + + ulong[] tail = new ulong[n + 3], tailExpected = new ulong[n + 3]; + random.NextBytes(MemoryMarshal.AsBytes(tail.AsSpan())); + if (trial % 3 == 0) Array.Clear(tail); + borrow = 0; + for (int i = 0; i < tail.Length; i++) + { + BigInteger d = (BigInteger)tail[i] - (i < n ? data[rhs + i] : 0UL) - borrow; + tailExpected[i] = (ulong)(d & ulong.MaxValue); + borrow = d.Sign < 0 ? 1UL : 0; + } + actual = LimbLoopWithTail(tail, data.AsSpan(rhs, n)); + if (actual != borrow || !tail.AsSpan().SequenceEqual(tailExpected)) throw new Exception("LimbLoopWithTail"); + + uint[] left32 = new uint[n], right32 = new uint[n], result32 = new uint[n]; + random.NextBytes(MemoryMarshal.AsBytes(left32.AsSpan())); + random.NextBytes(MemoryMarshal.AsBytes(right32.AsSpan())); + uint actual32 = LimbLoop32(result32, left32, right32); + long borrow32 = 0; + for (int i = 0; i < n; i++) + { + long d = (long)left32[i] - right32[i] - borrow32; + if (result32[i] != unchecked((uint)d)) throw new Exception("LimbLoop32 value"); + borrow32 = d < 0 ? 1 : 0; + } + if (actual32 != borrow32) throw new Exception("LimbLoop32 borrow"); + + ulong[] volatileResult = new ulong[n], volatileExpected = new ulong[n]; + borrow = 0; + for (int i = 0; i < n; i++) + { + BigInteger d = -(BigInteger)data[rhs + i] - borrow; + volatileExpected[i] = (ulong)(d & ulong.MaxValue); + borrow = d.Sign < 0 ? 1UL : 0; + } + actual = VolatileLoop(volatileResult, data.AsSpan(rhs, n)); + if (actual != borrow || !volatileResult.AsSpan().SequenceEqual(volatileExpected)) throw new Exception("VolatileLoop"); + } + } + + private static void Check(ulong a, ulong b, ulong c, ulong d) + { + if (CompareRight(a, b) != (unchecked(a - b) > b)) throw new Exception("CompareRight"); + try + { + ulong checkedResult = CheckedSub(a, b); + if (a < b || checkedResult != a - b) throw new Exception("CheckedSub"); + } + catch (OverflowException) + { + if (a >= b) throw; + } + ulong bit = a < b ? 1UL : 0; + ulong high = Borrow(a, b, c, out ulong diff); + if (diff != unchecked(a - b) || high != unchecked(c + bit)) throw new Exception("Borrow"); + high = BorrowOnly(a, b, c, out diff); + if (diff != unchecked(a - b) || high != unchecked(c - bit)) throw new Exception("BorrowOnly"); + UInt128 x = ((UInt128)c << 64) | a, y = ((UInt128)d << 64) | b; + BigInteger expected = ((BigInteger)x - (BigInteger)y) & ((BigInteger.One << 128) - 1); + if ((BigInteger)Sub128(x, y) != expected) throw new Exception("Sub128"); + if ((BigInteger)unchecked((UInt128)SubSigned128((Int128)x, (Int128)y)) != expected) throw new Exception("SubSigned128"); + if (Branch(a, b, out diff) != (bit != 0 ? 13UL : 29UL) || diff != unchecked(a - b)) throw new Exception("Branch"); + if (Select(a, b, c, d, out diff) != (bit != 0 ? c : d) || diff != unchecked(a - b)) throw new Exception("Select"); + ulong result = ArbitraryBorrow(a, b, c, out ulong next); + BigInteger wide = (BigInteger)a - b - c; + ulong expectedBorrow = (ulong)(-(wide >> 64)); + if (result != (ulong)(wide & ulong.MaxValue) || next != expectedBorrow) throw new Exception("ArbitraryBorrow"); + if (AcrossCall(a, b, c) != unchecked(~c + bit)) throw new Exception("AcrossCall"); + } +} diff --git a/src/tests/JIT/opt/Add/Borrow.csproj b/src/tests/JIT/opt/Add/Borrow.csproj new file mode 100644 index 00000000000000..ba455bd85b0c01 --- /dev/null +++ b/src/tests/JIT/opt/Add/Borrow.csproj @@ -0,0 +1,14 @@ + + + true + None + True + + + + true + + + + + diff --git a/src/tests/JIT/opt/Add/Carry.cs b/src/tests/JIT/opt/Add/Carry.cs new file mode 100644 index 00000000000000..691c19ffca7bf3 --- /dev/null +++ b/src/tests/JIT/opt/Add/Carry.cs @@ -0,0 +1,1099 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using Xunit; + +public static class CarryTests +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Count64(ulong a, ulong b, ulong count, out ulong sum) + { + // X64: add + // X64-NEXT: adc + // ARM64: adds + // ARM64-NEXT: adc + ulong s = a + b; + count += s < a ? 1UL : 0UL; + sum = s; + return count; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint Count32(uint a, uint b, uint count, out uint sum) + { + // X64: add + // X64-NEXT: adc + // ARM64: adds + // ARM64-NEXT: adc + uint s = a + b; + count += b > s ? 1U : 0U; + sum = s; + return count; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Increment(ulong a, ulong count, out ulong sum) + { + // X64-NOT: {{^ +}}inc + // X64: add + // X64-NEXT: adc + // X64-NOT: {{^ +}}inc + ulong s = a + 1; + count += s < a ? 1UL : 0UL; + sum = s; + return count; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong AddWithCarry(ulong a, ulong b, ulong carryIn, out ulong carryOut) + { + ulong t = a + b; + ulong s = t + carryIn; + carryOut = (t < a ? 1UL : 0UL) + (s < t ? 1UL : 0UL); + return s; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 Add128(ulong aLo, ulong aHi, ulong bLo, ulong bHi) + { + // X64: add + // X64: adc + // ARM64: adds + // ARM64: adc + ulong lo = aLo + bLo; + ulong hi = aHi + bHi + (lo < aLo ? 1UL : 0UL); + return ((UInt128)hi << 64) | lo; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 DirectAdd128(UInt128 a, UInt128 b) + { + // X64: add + // X64-NEXT: adc + // ARM64: adds + // ARM64-NEXT: adc + return a + b; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong AddMemoryAfterStore(ulong a, ulong b, ref ulong source, ref ulong destination, ulong replacement) + { + // X64: add + // X64-NEXT: setb + // ARM64: adds + // ARM64-NEXT: cset + ulong lo = a + b; + ulong carry = lo < a ? 1UL : 0UL; + destination = replacement; + return source + carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 MultiplyAdd(ulong a, ulong b, ulong existing, ulong carry) + { + // X64: add + // X64-NEXT: adc + // X64-NEXT: add + // X64-NEXT: adc + // ARM64: adds + // ARM64-NEXT: adc + // ARM64-NEXT: adds + // ARM64-NEXT: adc + return (UInt128)a * b + existing + carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong CountColumn(ulong a, ulong b, ulong c, ulong d, out ulong sum) + { + ulong s0 = a + b; + ulong count = s0 < a ? 1UL : 0UL; + ulong s1 = s0 + c; + count += s1 < s0 ? 1UL : 0UL; + ulong s2 = s1 + d; + count += s2 < s1 ? 1UL : 0UL; + sum = s2; + return count; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool Branch(ulong a, ulong b, out ulong sum) + { + // X64: add + // X64-NOT: cmp + // X64: j{{b|ae}} + // ARM64: adds + // ARM64-NOT: cmp + // ARM64: b{{lo|hs}} + ulong s = a + b; + if (s < a) + { + sum = s; + return true; + } + sum = s + 1; + return false; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Signed(long a, long b) + { + long sum = a + b; + return sum < a ? 1UL : 0UL; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Reload(ulong a, ulong b, ref ulong sum, ref ulong alias) + { + sum = a + b; + alias = 0; + return sum < a ? 1UL : 0UL; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Checked(ulong a, ulong b) + { + ulong sum = checked(a + b); + return sum < a ? 1UL : 0UL; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 TwoConsumers(ulong a, ulong b, ulong count) + { + // X64: add + // X64-NEXT: setb + // ARM64: adds + // ARM64-NEXT: cset + ulong sum = a + b; + ulong carry = sum < a ? 1UL : 0UL; + return ((UInt128)unchecked(count + carry) << 64) | carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong AcrossCall(ulong a, ulong b, ulong count) + { + ulong sum = a + b; + Mutate(ref count); + return count + (sum < a ? 1UL : 0UL); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void Mutate(ref ulong value) => value = ~value; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static nuint AddLimbWithCarry(nuint a, nuint b, nuint carryIn, out nuint carryOut) + { + if (nint.Size == 8) + { + nuint sum1 = a + b; + nuint c1 = (sum1 < a) ? 1 : (nuint)0; + nuint sum2 = sum1 + carryIn; + nuint c2 = (sum2 < sum1) ? 1 : (nuint)0; + carryOut = c1 + c2; + return sum2; + } + else + { + ulong sum = (ulong)a + b + carryIn; + carryOut = (uint)(sum >> 32); + return (uint)sum; + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void AddLimbLoop(ReadOnlySpan left, ReadOnlySpan right, Span bits) + { + // X64: adc {{.*}}, qword ptr + // X64-NEXT: mov + // X64-NEXT: lea + // X64-NEXT: dec + // X64-NEXT: jne + // ARM64: ldr + // ARM64-NEXT: ldr + // ARM64-NEXT: adcs + // ARM64-NEXT: str + // ARM64-NEXT: add + // ARM64-NEXT: sub + // ARM64-NEXT: cbnz + // Establish cross-span length relationships so the JIT can + // elide bounds checks for left[i] and bits[i] in the loop. + _ = left[right.Length - 1]; + _ = bits[right.Length]; + + nuint carry = 0; + + for (int i = 0; i < right.Length; i++) + { + bits[i] = AddLimbWithCarry(left[i], right[i], carry, out carry); + } + + bits[right.Length] = carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void AddLimbLoopWithCarry(ReadOnlySpan left, ReadOnlySpan right, Span bits, nuint initialCarry) + { + // Establish cross-span length relationships so the JIT can + // elide bounds checks for left[i] and bits[i] in the loop. + _ = left[right.Length - 1]; + _ = bits[right.Length]; + + nuint carry = initialCarry; + + for (int i = 0; i < right.Length; i++) + { + bits[i] = AddLimbWithCarry(left[i], right[i], carry, out carry); + } + + bits[right.Length] = carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void AddLimbLoopDiscardCarry(ReadOnlySpan left, ReadOnlySpan right, Span bits) + { + // Establish cross-span length relationships so the JIT can + // elide bounds checks for left[i] and bits[i] in the loop. + _ = left[right.Length - 1]; + _ = bits[right.Length]; + + nuint carry = 0; + + for (int i = 0; i < right.Length; i++) + { + bits[i] = AddLimbWithCarry(left[i], right[i], carry, out carry); + } + + } + + private static int s_loopSignal; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void AddLimbLoopVolatile(ReadOnlySpan left, ReadOnlySpan right, Span bits) + { + // Establish cross-span length relationships so the JIT can + // elide bounds checks for left[i] and bits[i] in the loop. + _ = left[right.Length - 1]; + _ = bits[right.Length]; + + nuint carry = 0; + + for (int i = 0; i < right.Length; i++) + { + bits[i] = AddLimbWithCarry(left[i], right[i], carry, out carry); + System.Threading.Volatile.Write(ref s_loopSignal, i); + } + + bits[right.Length] = carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void AddLimbLoopOne(ReadOnlySpan left, ReadOnlySpan right, Span bits) + { + // Establish cross-span length relationships so the JIT can + // elide bounds checks for left[i] and bits[i] in the loop. + _ = left[right.Length - 1]; + _ = bits[right.Length]; + + nuint carry = 1; + + for (int i = 0; i < right.Length; i++) + { + bits[i] = AddLimbWithCarry(left[i], right[i], carry, out carry); + } + + bits[right.Length] = carry; + } + private static BigInteger LimbsToBigInteger(ReadOnlySpan limbs) + { + BigInteger value = 0; + for (int i = limbs.Length - 1; i >= 0; i--) + value = (value << (IntPtr.Size * 8)) | (ulong)limbs[i]; + return value; + } + + private static void CheckLimbLoops() + { + Random random = new Random(125799); + byte[] bytes = new byte[8]; + for (int test = 0; test < 1000; test++) + { + int length = 1 + test % 129; + nuint[] left = new nuint[length + 1]; + nuint[] right = new nuint[length]; + nuint[] result = new nuint[length + 1]; + for (int i = 0; i < length; i++) + { + random.NextBytes(bytes); + left[i] = test % 4 == 0 ? nuint.MaxValue : (nuint)BitConverter.ToUInt64(bytes); + random.NextBytes(bytes); + right[i] = test % 4 == 0 ? nuint.MaxValue : (nuint)BitConverter.ToUInt64(bytes); + } + BigInteger expected = LimbsToBigInteger(left.AsSpan(0, length)) + LimbsToBigInteger(right); + AddLimbLoop(left.AsSpan(0, length), right, result); + if (LimbsToBigInteger(result) != expected) throw new Exception("AddLimbLoop"); + AddLimbLoopOne(left.AsSpan(0, length), right, result); + if (LimbsToBigInteger(result) != expected + 1) throw new Exception("AddLimbLoopOne"); + AddLimbLoopVolatile(left.AsSpan(0, length), right, result); + if (LimbsToBigInteger(result) != expected || s_loopSignal != length - 1) throw new Exception("AddLimbLoopVolatile"); + AddLimbLoopDiscardCarry(left.AsSpan(0, length), right, result); + if (LimbsToBigInteger(result.AsSpan(0, length)) != (expected & ((BigInteger.One << (length * IntPtr.Size * 8)) - 1))) + throw new Exception("AddLimbLoopDiscardCarry"); + foreach (nuint carry in new nuint[] { 0, 1, 2, nuint.MaxValue }) + { + AddLimbLoopWithCarry(left.AsSpan(0, length), right, result, carry); + if (LimbsToBigInteger(result) != expected + (ulong)carry) throw new Exception("AddLimbLoopWithCarry"); + } + AddLimbLoop(left.AsSpan(0, length), right, left); + if (LimbsToBigInteger(left) != expected) throw new Exception("AddLimbLoop alias"); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong SelectCarry(ulong a, ulong b, ulong x, ulong y, out ulong sum) + { + // X64-NOT: {{^ +}}set + // X64: cmov + // ARM64-NOT: {{^ +}}cset + // ARM64: cs{{el|inc}} + ulong s = unchecked(a + b); + sum = s; + return s >= a ? x + 1 : y; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void AddLimbLoop32(ReadOnlySpan left, ReadOnlySpan right, Span bits) + { + // X64: adc {{.*}}, dword ptr + // X64-NEXT: mov + // X64-NEXT: lea + // X64-NEXT: dec + // X64-NEXT: jne + // ARM64: adcs w + // ARM64-NEXT: str w + // ARM64-NEXT: add + // ARM64-NEXT: sub + // ARM64-NEXT: cbnz + _ = left[right.Length - 1]; + _ = bits[right.Length]; + uint carry = 0; + for (int i = 0; i < right.Length; i++) + { + uint a = left[i]; + uint sum1 = unchecked(a + right[i]); + uint c1 = sum1 < a ? 1U : 0U; + uint sum2 = unchecked(sum1 + carry); + uint c2 = sum2 < sum1 ? 1U : 0U; + carry = c1 + c2; + bits[i] = sum2; + } + bits[right.Length] = carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void AddLimbLoop32WithIndex(ReadOnlySpan left, ReadOnlySpan right, Span bits, Span indices) + { + // X64: adc {{.*}}, dword ptr + // X64: lea e + // ARM64: adcs w + // ARM64: add w + _ = left[right.Length - 1]; + _ = bits[right.Length]; + _ = indices[right.Length - 1]; + uint index = 0; + uint carry = 0; + for (int i = 0; i < right.Length; i++) + { + uint a = left[i]; + uint sum1 = unchecked(a + right[i]); + uint c1 = sum1 < a ? 1U : 0U; + uint sum2 = unchecked(sum1 + carry); + uint c2 = sum2 < sum1 ? 1U : 0U; + carry = c1 + c2; + bits[i] = sum2; + index = unchecked(index + 7); + indices[i] = index; + } + bits[right.Length] = carry; + } + + private static BigInteger Limbs32ToBigInteger(ReadOnlySpan limbs) + { + BigInteger value = 0; + for (int i = limbs.Length - 1; i >= 0; i--) + { + value = (value << 32) | limbs[i]; + } + return value; + } + + private static void CheckLimbLoops32() + { + Random random = new Random(125799); + for (int trial = 0; trial < 1000; trial++) + { + int length = 1 + trial % 129; + uint[] left = new uint[length + 1]; + uint[] right = new uint[length + 1]; + uint[] result = new uint[length + 1]; + uint[] indices = new uint[length]; + for (int i = 0; i < length; i++) + { + left[i] = trial % 4 == 0 ? uint.MaxValue : (uint)random.NextInt64(1L << 32); + right[i] = trial % 4 == 1 ? uint.MaxValue : (uint)random.NextInt64(1L << 32); + } + BigInteger expected = Limbs32ToBigInteger(left) + Limbs32ToBigInteger(right); + AddLimbLoop32WithIndex(left.AsSpan(0, length), right.AsSpan(0, length), result, indices); + if (Limbs32ToBigInteger(result) != expected) throw new Exception("AddLimbLoop32WithIndex"); + for (int i = 0; i < length; i++) + { + if (indices[i] != (uint)(7 * (i + 1))) throw new Exception("AddLimbLoop32WithIndex index"); + } + AddLimbLoop32(left.AsSpan(0, length), right.AsSpan(0, length), result); + if (Limbs32ToBigInteger(result) != expected) throw new Exception("AddLimbLoop32"); + AddLimbLoop32(left.AsSpan(0, length), right.AsSpan(0, length), left); + if (Limbs32ToBigInteger(left) != expected) throw new Exception("AddLimbLoop32 alias left"); + left = (uint[])result.Clone(); + expected = Limbs32ToBigInteger(left.AsSpan(0, length)) + Limbs32ToBigInteger(right); + AddLimbLoop32(left.AsSpan(0, length), right.AsSpan(0, length), right); + if (Limbs32ToBigInteger(right) != expected) throw new Exception("AddLimbLoop32 alias right"); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool NoCarry(ulong a, ulong b) + { + // X64: add + // X64-NEXT: setae + // ARM64: adds + // ARM64-NEXT: cset {{.*}}, lo + return a <= unchecked(a + b); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int ZeroOr(int x, int y) + { + // X64: {{^ +}}or + // X64-NOT: {{^ +}}set + // X64-NOT: test + // X64: cmov + return (x | y) == 0 ? 1 : 2; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int ZeroAdd(int x, int y) + { + // X64: add + // X64-NOT: {{^ +}}set + // X64-NOT: test + // X64: cmov + return unchecked(x + y) == 0 ? 1 : 2; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int ZeroShift(int x) + { + // X64: shl + // X64-NOT: {{^ +}}set + // X64-NOT: test + // X64: cmov + return (x << 3) == 0 ? 1 : 2; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static UInt128 WideningProduct(ulong a, ulong b) + { + if (System.Runtime.Intrinsics.Arm.ArmBase.Arm64.IsSupported) + { + ulong high = System.Runtime.Intrinsics.Arm.ArmBase.Arm64.MultiplyHigh(a, b); + return ((UInt128)high << 64) | unchecked(a * b); + } + return Math.BigMul(a, b); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static nuint UnrolledMultiplyAdd(Span result, ReadOnlySpan left, nuint multiplier) + { + // X64-NOT: {{^ +}}setb + // X64: mul{{x| }} + // X64-NOT: {{^ +}}setb + // X64: adc + // X64-NOT: {{^ +}}setb + // X64: ret + // ARM64-NOT: {{^ +}}cset + // ARM64: umulh + // ARM64-NOT: {{^ +}}cset + // ARM64: adc + // ARM64-NOT: {{^ +}}cset + // ARM64: ret + + int length = left.Length; + result = result.Slice(0, length); + int i = 0; + nuint carry = 0; + + if (nint.Size == 8) + { + // Unroll by 4: mulx has 3-5 cycle latency but 1 cycle throughput, + // so issuing 4 multiplies allows the CPU to pipeline them while + // carry chains complete sequentially behind. + for (; i < length - 3; i += 4) + { + UInt128 p0 = WideningProduct((ulong)left[i], (ulong)multiplier) + (ulong)result[i] + (ulong)carry; + result[i] = (nuint)(ulong)p0; + + UInt128 p1 = WideningProduct((ulong)left[i + 1], (ulong)multiplier) + (ulong)result[i + 1] + (ulong)(p0 >> 64); + result[i + 1] = (nuint)(ulong)p1; + + UInt128 p2 = WideningProduct((ulong)left[i + 2], (ulong)multiplier) + (ulong)result[i + 2] + (ulong)(p1 >> 64); + result[i + 2] = (nuint)(ulong)p2; + + UInt128 p3 = WideningProduct((ulong)left[i + 3], (ulong)multiplier) + (ulong)result[i + 3] + (ulong)(p2 >> 64); + result[i + 3] = (nuint)(ulong)p3; + + carry = (nuint)(ulong)(p3 >> 64); + } + + for (; i < length; i++) + { + UInt128 product = WideningProduct((ulong)left[i], (ulong)multiplier) + (ulong)result[i] + (ulong)carry; + result[i] = (nuint)(ulong)product; + carry = (nuint)(ulong)(product >> 64); + } + } + else + { + for (; i < length; i++) + { + ulong product = (ulong)left[i] * multiplier + + result[i] + carry; + result[i] = (uint)product; + carry = (uint)(product >> 32); + } + } + + return carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static long SignedWideningProduct(long a, long b, out long low) => Math.BigMul(a, b, out low); + + private static void CheckUnrolledMultiplyAdd() + { + Random random = new Random(80674); + for (int trial = 0; trial < 1000; trial++) + { + int length = 1 + trial % 65; + nuint[] left = new nuint[length]; + nuint[] result = new nuint[length]; + for (int i = 0; i < length; i++) + { + left[i] = trial % 4 == 0 ? nuint.MaxValue : (nuint)random.NextInt64(); + result[i] = trial % 4 == 0 ? nuint.MaxValue : (nuint)random.NextInt64(); + } + nuint multiplier = trial % 4 == 0 ? nuint.MaxValue : (nuint)random.NextInt64(); + BigInteger expected = LimbsToBigInteger(left) * (ulong)multiplier + LimbsToBigInteger(result); + nuint carry = UnrolledMultiplyAdd(result, left, multiplier); + if (LimbsToBigInteger(result) + ((BigInteger)(ulong)carry << (length * IntPtr.Size * 8)) != expected) + throw new Exception("UnrolledMultiplyAdd"); + expected = LimbsToBigInteger(left) * ((BigInteger)(ulong)multiplier + 1); + carry = UnrolledMultiplyAdd(left, left, multiplier); + if (LimbsToBigInteger(left) + ((BigInteger)(ulong)carry << (length * IntPtr.Size * 8)) != expected) + throw new Exception("UnrolledMultiplyAdd alias"); + long a = unchecked((long)(ulong)multiplier); + long b = trial % 2 == 0 ? long.MinValue : random.NextInt64(); + long high = SignedWideningProduct(a, b, out long low); + if (((BigInteger)high << 64) + (ulong)low != (BigInteger)a * b) + throw new Exception("SignedWideningProduct"); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ScalarMultiplyAdd(Span result, ReadOnlySpan left, ulong multiplier, ulong carry) + { + // X64: mul{{x| }} + // X64: ad{{cx|d}} + // X64: ad{{ox|c}} + // ARM64: umulh + // ARM64: adc + for (int i = 0; i < left.Length; i++) + { + UInt128 product = WideningProduct(left[i], multiplier) + result[i] + carry; + result[i] = (ulong)product; + carry = (ulong)(product >> 64); + } + return carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong VolatileMultiplyAdd(Span result, ReadOnlySpan left, ulong multiplier, ulong carry) + { + // X64-NOT: adcx + // X64-NOT: adox + // X64: ret + // Volatile loads must not become part of an ADX loop. + for (int i = 0; i < left.Length; i++) + { + UInt128 product = WideningProduct(left[i], multiplier) + System.Threading.Volatile.Read(ref result[i]) + carry; + result[i] = (ulong)product; + carry = (ulong)(product >> 64); + } + return carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong GuardedUnrolledMultiplyAdd(Span result, ReadOnlySpan left, ulong multiplier, ulong carry) + { + // X64: mul{{x| }} + // X64: ad{{cx|d}} + // X64: ad{{ox|c}} + // ARM64: umulh + // ARM64: adcs + // ARM64-NEXT: adc + // ARM64-NEXT: adds + // ARM64: adcs + // ARM64-NEXT: adc + // ARM64-NEXT: adds + // ARM64: adcs + // ARM64-NEXT: adc + // ARM64-NEXT: adds + // ARM64: adcs + // ARM64-NEXT: adc + // ARM64-NEXT: adds + // ARM64: cbnz + // ARM64: adc + int length = left.Length; + result = result.Slice(0, length); + int i = 0; + for (; i < length - 3; i += 4) + { + UInt128 p0 = WideningProduct(left[i], multiplier) + result[i] + carry; + result[i] = (ulong)p0; + UInt128 p1 = WideningProduct(left[i + 1], multiplier) + result[i + 1] + (ulong)(p0 >> 64); + result[i + 1] = (ulong)p1; + UInt128 p2 = WideningProduct(left[i + 2], multiplier) + result[i + 2] + (ulong)(p1 >> 64); + result[i + 2] = (ulong)p2; + UInt128 p3 = WideningProduct(left[i + 3], multiplier) + result[i + 3] + (ulong)(p2 >> 64); + result[i + 3] = (ulong)p3; + carry = (ulong)(p3 >> 64); + } + for (; i < left.Length; i++) + { + UInt128 product = WideningProduct(left[i], multiplier) + result[i] + carry; + result[i] = (ulong)product; + carry = (ulong)(product >> 64); + } + return carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ObservableCarryUnrolledMultiplyAdd(Span result, ReadOnlySpan left, ulong multiplier, ulong carry, out ulong observed) + { + // The intermediate high word is observable and must include both carries. + // ARM64-NOT: adcs + // ARM64: ret + observed = 0; + int length = left.Length; + result = result.Slice(0, length); + int i = 0; + for (; i < length - 3; i += 4) + { + UInt128 p0 = WideningProduct(left[i], multiplier) + result[i] + carry; + result[i] = (ulong)p0; + observed = (ulong)(p0 >> 64); + UInt128 p1 = WideningProduct(left[i + 1], multiplier) + result[i + 1] + (ulong)(p0 >> 64); + result[i + 1] = (ulong)p1; + UInt128 p2 = WideningProduct(left[i + 2], multiplier) + result[i + 2] + (ulong)(p1 >> 64); + result[i + 2] = (ulong)p2; + UInt128 p3 = WideningProduct(left[i + 3], multiplier) + result[i + 3] + (ulong)(p2 >> 64); + result[i + 3] = (ulong)p3; + carry = (ulong)(p3 >> 64); + } + for (; i < left.Length; i++) + { + UInt128 product = WideningProduct(left[i], multiplier) + result[i] + carry; + result[i] = (ulong)product; + carry = (ulong)(product >> 64); + } + return carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong InterveningStoreUnrolledMultiplyAdd(Span result, ReadOnlySpan left, ulong multiplier, ulong carry) + { + // X64: mul{{x| }} + // X64: ad{{cx|d}} + // X64: ad{{ox|c}} + // ARM64: umulh + // ARM64: adc + int length = left.Length; + result = result.Slice(0, length); + int i = 0; + for (; i < length - 3; i += 4) + { + // The saved load must remain before this potentially aliasing store. + ulong saved = result[i]; + result[i + 1] = multiplier; + UInt128 p0 = WideningProduct(left[i], multiplier) + saved + carry; + result[i] = (ulong)p0; + UInt128 p1 = WideningProduct(left[i + 1], multiplier) + result[i + 1] + (ulong)(p0 >> 64); + result[i + 1] = (ulong)p1; + UInt128 p2 = WideningProduct(left[i + 2], multiplier) + result[i + 2] + (ulong)(p1 >> 64); + result[i + 2] = (ulong)p2; + UInt128 p3 = WideningProduct(left[i + 3], multiplier) + result[i + 3] + (ulong)(p2 >> 64); + result[i + 3] = (ulong)p3; + carry = (ulong)(p3 >> 64); + } + for (; i < left.Length; i++) + { + UInt128 product = WideningProduct(left[i], multiplier) + result[i] + carry; + result[i] = (ulong)product; + carry = (ulong)(product >> 64); + } + return carry; + } + + private static void CheckObservableCarry() + { + Random random = new Random(125802); + for (int trial = 0; trial < 256; trial++) + { + int length = trial % 66; + ulong[] left = new ulong[length]; + ulong[] result = new ulong[length]; + random.NextBytes(System.Runtime.InteropServices.MemoryMarshal.AsBytes(left.AsSpan())); + random.NextBytes(System.Runtime.InteropServices.MemoryMarshal.AsBytes(result.AsSpan())); + ulong multiplier = trial % 3 == 0 ? ulong.MaxValue : (ulong)random.NextInt64(); + ulong carry = trial % 2 == 0 ? ulong.MaxValue : 0; + ulong[] expected = (ulong[])result.Clone(); + ulong expectedCarry = carry; + ulong expectedObserved = 0; + for (int i = 0; i < length; i++) + { + BigInteger product = (BigInteger)left[i] * multiplier + expected[i] + expectedCarry; + expected[i] = (ulong)(product & ulong.MaxValue); + expectedCarry = (ulong)(product >> 64); + if ((i & 3) == 0 && i < length - 3) + { + expectedObserved = expectedCarry; + } + } + ulong actualCarry = ObservableCarryUnrolledMultiplyAdd(result, left, multiplier, carry, out ulong observed); + if (actualCarry != expectedCarry || observed != expectedObserved || !result.AsSpan().SequenceEqual(expected)) + { + throw new Exception("ObservableCarryUnrolledMultiplyAdd"); + } + } + } + private static void CheckInterveningStore() + { + Random random = new Random(125801); + for (int trial = 0; trial < 256; trial++) + { + int length = trial % 66; + int source = 2; + int destination = (trial % 4) switch { 0 => length + 4, 1 => 2, 2 => 3, _ => 1 }; + ulong[] data = new ulong[length * 2 + 8]; + random.NextBytes(System.Runtime.InteropServices.MemoryMarshal.AsBytes(data.AsSpan())); + ulong multiplier = trial % 3 == 0 ? ulong.MaxValue : data[0]; + ulong carry = trial % 2 == 0 ? ulong.MaxValue : 0; + ulong[] expected = (ulong[])data.Clone(); + ulong expectedCarry = carry; + for (int i = 0; i < length; i++) + { + ulong saved = expected[destination + i]; + if (((i & 3) == 0) && (i < length - 3)) + { + expected[destination + i + 1] = multiplier; + } + BigInteger product = (BigInteger)expected[source + i] * multiplier + saved + expectedCarry; + expected[destination + i] = (ulong)(product & ulong.MaxValue); + expectedCarry = (ulong)(product >> 64); + } + ulong actualCarry = InterveningStoreUnrolledMultiplyAdd(data.AsSpan(destination, length), + data.AsSpan(source, length), multiplier, carry); + if (actualCarry != expectedCarry || !data.AsSpan().SequenceEqual(expected)) + { + throw new Exception("InterveningStoreUnrolledMultiplyAdd"); + } + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ChangingMultiplierUnrolledMultiplyAdd(Span result, ReadOnlySpan left, ulong multiplier, ulong carry) + { + // The multiplier changes on the backedge and must not be hoisted. + // X64: mul{{x| }} + // X64: ad{{cx|d}} + // X64: ad{{ox|c}} + // ARM64: umulh + // ARM64: adc + int length = left.Length; + result = result.Slice(0, length); + int i = 0; + for (; i < length - 3; i += 4) + { + UInt128 p0 = WideningProduct(left[i], multiplier) + result[i] + carry; + result[i] = (ulong)p0; + UInt128 p1 = WideningProduct(left[i + 1], multiplier) + result[i + 1] + (ulong)(p0 >> 64); + result[i + 1] = (ulong)p1; + UInt128 p2 = WideningProduct(left[i + 2], multiplier) + result[i + 2] + (ulong)(p1 >> 64); + result[i + 2] = (ulong)p2; + UInt128 p3 = WideningProduct(left[i + 3], multiplier) + result[i + 3] + (ulong)(p2 >> 64); + result[i + 3] = (ulong)p3; + carry = (ulong)(p3 >> 64); + multiplier = unchecked(multiplier + 1); + } + for (; i < left.Length; i++) + { + UInt128 product = WideningProduct(left[i], multiplier) + result[i] + carry; + result[i] = (ulong)product; + carry = (ulong)(product >> 64); + } + return carry; + } + + private static void CheckChangingMultiplier() + { + Random random = new Random(125800); + for (int trial = 0; trial < 256; trial++) + { + int length = trial % 66; + ulong[] left = new ulong[length]; + ulong[] result = new ulong[length]; + random.NextBytes(System.Runtime.InteropServices.MemoryMarshal.AsBytes(left.AsSpan())); + random.NextBytes(System.Runtime.InteropServices.MemoryMarshal.AsBytes(result.AsSpan())); + ulong multiplier = trial % 3 == 0 ? ulong.MaxValue : (ulong)random.NextInt64(); + ulong carry = trial % 2 == 0 ? ulong.MaxValue : 0; + ulong[] expected = (ulong[])result.Clone(); + ulong expectedMultiplier = multiplier; + ulong expectedCarry = carry; + for (int i = 0; i < length; i++) + { + BigInteger product = (BigInteger)left[i] * expectedMultiplier + expected[i] + expectedCarry; + expected[i] = (ulong)(product & ulong.MaxValue); + expectedCarry = (ulong)(product >> 64); + if ((i & 3) == 3) + { + expectedMultiplier = unchecked(expectedMultiplier + 1); + } + } + ulong actualCarry = ChangingMultiplierUnrolledMultiplyAdd(result, left, multiplier, carry); + if (actualCarry != expectedCarry || !result.AsSpan().SequenceEqual(expected)) + { + throw new Exception("ChangingMultiplierUnrolledMultiplyAdd"); + } + } + } + + private static void CheckScalarMultiplyAdd() + { + Random random = new Random(125799); + BigInteger mask = (BigInteger.One << 64) - 1; + for (int trial = 0; trial < 12000; trial++) + { + int length = trial % 129; + int source = 2; + int destination = (trial % 4) switch { 0 => length + 4, 1 => 2, 2 => 3, _ => 1 }; + ulong[] input = new ulong[length * 2 + 8]; + random.NextBytes(System.Runtime.InteropServices.MemoryMarshal.AsBytes(input.AsSpan())); + if (trial % 13 == 0) + Array.Fill(input, ulong.MaxValue); + if (trial % 17 == 0) + Array.Fill(input, 0UL); + ulong multiplier = trial % 7 == 0 ? ulong.MaxValue : input[0]; + ulong initialCarry = trial % 11 == 0 ? ulong.MaxValue : input[^1]; + ulong[] expected = (ulong[])input.Clone(); + BigInteger expectedCarry = initialCarry; + // The sequential oracle also covers overlap in either direction. + for (int i = 0; i < length; i++) + { + BigInteger product = (BigInteger)expected[source + i] * multiplier + expected[destination + i] + expectedCarry; + expected[destination + i] = (ulong)(product & mask); + expectedCarry = product >> 64; + } + for (int variant = 0; variant < 3; variant++) + { + ulong[] actual = (ulong[])input.Clone(); + ulong carry = variant switch + { + 0 => ScalarMultiplyAdd(actual.AsSpan(destination, length), actual.AsSpan(source, length), multiplier, initialCarry), + 1 => VolatileMultiplyAdd(actual.AsSpan(destination, length), actual.AsSpan(source, length), multiplier, initialCarry), + _ => GuardedUnrolledMultiplyAdd(actual.AsSpan(destination, length), actual.AsSpan(source, length), multiplier, initialCarry) + }; + if (carry != (ulong)expectedCarry || !actual.AsSpan().SequenceEqual(expected)) + throw new Exception($"ScalarMultiplyAdd trial {trial}, variant {variant}"); + } + } + + // A short destination must still throw after storing only the valid prefix. + for (int length = 0; length < 8; length++) + { + ulong[] left = new ulong[length + 1]; + ulong[] result = new ulong[length + 2]; + Array.Fill(left, ulong.MaxValue); + Array.Fill(result, ulong.MaxValue); + ulong[] expected = (ulong[])result.Clone(); + BigInteger carry = ulong.MaxValue; + for (int i = 0; i < length; i++) + { + BigInteger product = (BigInteger)left[i] * ulong.MaxValue + expected[i] + carry; + expected[i] = (ulong)(product & mask); + carry = product >> 64; + } + try + { + ScalarMultiplyAdd(result.AsSpan(0, length), left, ulong.MaxValue, ulong.MaxValue); + throw new Exception("Missing bounds exception"); + } + catch (IndexOutOfRangeException) + { + if (!result.AsSpan().SequenceEqual(expected)) + throw new Exception("MultiplyAdd exception store order"); + } + } + + // The upfront slice must reject a short destination before any stores. + for (int length = 0; length < 12; length++) + { + ulong[] left = new ulong[length + 4]; + ulong[] result = new ulong[length + 4]; + Array.Fill(left, ulong.MaxValue); + Array.Fill(result, ulong.MaxValue); + ulong[] expected = (ulong[])result.Clone(); + try + { + GuardedUnrolledMultiplyAdd(result.AsSpan(0, length), left, ulong.MaxValue, ulong.MaxValue); + throw new Exception("Missing guarded bounds exception"); + } + catch (ArgumentOutOfRangeException) + { + if (!result.AsSpan().SequenceEqual(expected)) + throw new Exception("Guarded multiply-add exception store order"); + } + } + } + + [Fact] + public static void TestEntryPoint() + { + CheckLimbLoops(); + CheckUnrolledMultiplyAdd(); + CheckChangingMultiplier(); + CheckInterveningStore(); + CheckObservableCarry(); + CheckScalarMultiplyAdd(); + CheckLimbLoops32(); + ulong[] values = { 0, 1, 2, uint.MaxValue, 1UL << 32, (1UL << 63) - 1, 1UL << 63, ulong.MaxValue - 1, ulong.MaxValue }; + foreach (ulong a in values) + { + foreach (ulong b in values) + { + foreach (ulong c in values) + { + Check(a, b, c, ~c); + } + } + } + + Random random = new Random(80674); + byte[] bytes = new byte[32]; + for (int i = 0; i < 10000; i++) + { + random.NextBytes(bytes); + Check(BitConverter.ToUInt64(bytes, 0), BitConverter.ToUInt64(bytes, 8), + BitConverter.ToUInt64(bytes, 16), BitConverter.ToUInt64(bytes, 24)); + } + + ulong shared = 42; + if (Reload(1, 2, ref shared, ref shared) != 1 || shared != 0) + { + throw new Exception("Aliased reload"); + } + try + { + Checked(ulong.MaxValue, 1); + throw new Exception("Missing overflow exception"); + } + catch (OverflowException) + { + } + } + + private static void Check(ulong a, ulong b, ulong c, ulong d) + { + int x = (int)a; + int y = (int)b; + if (ZeroOr(x, y) != ((x | y) == 0 ? 1 : 2) || + ZeroAdd(x, y) != (unchecked(x + y) == 0 ? 1 : 2) || + ZeroShift(x) != ((x << 3) == 0 ? 1 : 2)) + { + throw new Exception("Zero flags"); + } + UInt128 wide = (UInt128)a + b; + if (Count64(a, b, c, out ulong sum) != unchecked(c + (ulong)(wide >> 64)) || sum != (ulong)wide) + throw new Exception("Count64"); + + if (NoCarry(a, b) != (wide <= ulong.MaxValue)) + { + throw new Exception("NoCarry"); + } + ulong selected = SelectCarry(a, b, c, d, out ulong selectedSum); + if (selectedSum != (ulong)wide || selected != (wide <= ulong.MaxValue ? unchecked(c + 1) : d)) + { + throw new Exception("SelectCarry"); + } + ulong bit = (ulong)(wide >> 64); + if (TwoConsumers(a, b, c) != (((UInt128)unchecked(c + bit) << 64) | bit)) + throw new Exception("TwoConsumers"); + + if (AcrossCall(a, b, c) != unchecked(~c + bit)) + throw new Exception("AcrossCall"); + + ulong wide32 = (ulong)(uint)a + (uint)b; + if (Count32((uint)a, (uint)b, (uint)c, out uint sum32) != unchecked((uint)c + (uint)(wide32 >> 32)) || sum32 != (uint)wide32) + throw new Exception("Count32"); + + wide = (UInt128)a + 1; + if (Increment(a, c, out sum) != unchecked(c + (ulong)(wide >> 64)) || sum != (ulong)wide) + throw new Exception("Increment"); + + wide = (UInt128)a + b + c; + if (AddWithCarry(a, b, c, out ulong carry) != (ulong)wide || carry != (ulong)(wide >> 64)) + throw new Exception("AddWithCarry"); + + UInt128 left = ((UInt128)c << 64) | a; + UInt128 right = ((UInt128)d << 64) | b; + if (Add128(a, c, b, d) != unchecked(left + right)) + throw new Exception("Add128"); + + BigInteger sum128 = ((BigInteger)left + (BigInteger)right) & ((BigInteger.One << 128) - 1); + if ((BigInteger)DirectAdd128(left, right) != sum128) + throw new Exception("DirectAdd128"); + + ulong memory = c; + if (AddMemoryAfterStore(a, b, ref memory, ref memory, d) != unchecked(d + bit) || memory != d) + throw new Exception("AddMemoryAfterStore alias"); + ulong destination = 0; + if (AddMemoryAfterStore(a, b, ref memory, ref destination, c) != unchecked(d + bit) || destination != c) + throw new Exception("AddMemoryAfterStore distinct"); + + BigInteger product = (BigInteger)a * b + c + d; + if ((BigInteger)MultiplyAdd(a, b, c, d) != product) + throw new Exception("MultiplyAdd"); + + wide = (UInt128)a + b + c + d; + if (CountColumn(a, b, c, d, out sum) != (ulong)(wide >> 64) || sum != (ulong)wide) + throw new Exception("CountColumn"); + + wide = (UInt128)a + b; + bool overflow = wide > ulong.MaxValue; + if (Branch(a, b, out sum) != overflow || sum != unchecked((ulong)wide + (overflow ? 0UL : 1UL))) + throw new Exception("Branch"); + + long signedSum = unchecked((long)a + (long)b); + if (Signed((long)a, (long)b) != (signedSum < (long)a ? 1UL : 0UL)) + throw new Exception("Signed"); + } +} diff --git a/src/tests/JIT/opt/Add/Carry.csproj b/src/tests/JIT/opt/Add/Carry.csproj new file mode 100644 index 00000000000000..3c2866959f4fff --- /dev/null +++ b/src/tests/JIT/opt/Add/Carry.csproj @@ -0,0 +1,14 @@ + + + true + None + True + + + + true + + + + + diff --git a/src/tests/JIT/opt/Add/CarryChains.cs b/src/tests/JIT/opt/Add/CarryChains.cs new file mode 100644 index 00000000000000..620dc9263ef224 --- /dev/null +++ b/src/tests/JIT/opt/Add/CarryChains.cs @@ -0,0 +1,140 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using Xunit; + +public class CarryChainTests +{ + public struct Words + { + public ulong A, B, C, D; + public Words(ulong a, ulong b, ulong c, ulong d) { A = a; B = b; C = c; D = d; } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong Add(ulong a, ulong b, ulong carry, out ulong next) + { + ulong sum = a + b; + ulong c1 = sum < a ? 1UL : 0; + ulong result = sum + carry; + next = c1 + (result < sum ? 1UL : 0); + return result; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong Sub(ulong a, ulong b, ulong borrow, out ulong next) + { + ulong difference = a - b; + ulong b1 = difference > a ? 1UL : 0; + ulong result = difference - borrow; + next = b1 + (result > difference ? 1UL : 0); + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Words AddChain(Words a, Words b) + { + ulong w0 = Add(a.A, b.A, 0, out ulong carry); + ulong w1 = Add(a.B, b.B, carry, out carry); + ulong w2 = Add(a.C, b.C, carry, out carry); + ulong w3 = a.D + b.D + carry; + return new Words(w0, w1, w2, w3); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Words SubChain(Words a, Words b) + { + ulong w0 = Sub(a.A, b.A, 0, out ulong borrow); + ulong w1 = Sub(a.B, b.B, borrow, out borrow); + ulong w2 = Sub(a.C, b.C, borrow, out borrow); + ulong w3 = a.D - b.D - borrow; + return new Words(w0, w1, w2, w3); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Clobber(ulong value) => (value * 37) ^ (value >> 3); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Words Interrupted(Words a, Words b, out ulong observed) + { + ulong w0 = Add(a.A, b.A, 0, out ulong carry); + observed = Clobber(carry); + ulong w1 = Add(a.B, b.B, carry, out carry); + ulong w2 = Add(a.C, b.C, carry, out carry); + return new Words(w0, w1, w2, a.D + b.D + carry); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Words ExtraConsumer(Words a, Words b, out ulong observed) + { + ulong w0 = Sub(a.A, b.A, 0, out ulong borrow); + ulong saved = borrow; + ulong w1 = Sub(a.B, b.B, borrow, out borrow); + ulong w2 = Sub(a.C, b.C, borrow, out borrow); + observed = saved; + return new Words(w0, w1, w2, a.D - b.D - borrow); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Words FullLimbCarry(Words a, Words b, ulong carry) + { + ulong w0 = Add(a.A, b.A, carry, out carry); + ulong w1 = Add(a.B, b.B, carry, out carry); + ulong w2 = Add(a.C, b.C, carry, out carry); + return new Words(w0, w1, w2, a.D + b.D + carry); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Words TwoCarrySeed(Words a, Words b) + { + // Each comparison is a bit, but their sum need not be. This local + // must not acquire a one-bit proof merely because its inputs have one. + ulong carry = (a.A + b.A < a.A ? 1UL : 0) + (a.B + b.B < a.B ? 1UL : 0); + ulong w0 = Add(a.A, b.A, carry, out carry); + ulong w1 = Add(a.B, b.B, carry, out carry); + ulong w2 = Add(a.C, b.C, carry, out carry); + return new Words(w0, w1, w2, a.D + b.D + carry); + } + + private static BigInteger Value(Words value) => value.A | ((BigInteger)value.B << 64) | + ((BigInteger)value.C << 128) | ((BigInteger)value.D << 192); + + [Fact] + public static void TestEntryPoint() + { + Random random = new Random(80674); + byte[] bytes = new byte[64]; + BigInteger mask = (BigInteger.One << 256) - 1; + void Equal(Words actual, BigInteger expected) + { + if (Value(actual) != (expected & mask)) throw new Exception("Incorrect carry chain"); + } + for (int i = 0; i < 3000; i++) + { + random.NextBytes(bytes); + Words a = new Words(BitConverter.ToUInt64(bytes, 0), BitConverter.ToUInt64(bytes, 8), BitConverter.ToUInt64(bytes, 16), BitConverter.ToUInt64(bytes, 24)); + Words b = new Words(BitConverter.ToUInt64(bytes, 32), BitConverter.ToUInt64(bytes, 40), BitConverter.ToUInt64(bytes, 48), BitConverter.ToUInt64(bytes, 56)); + if (i < 4) + { + a = new Words(ulong.MaxValue, ulong.MaxValue, ulong.MaxValue, ulong.MaxValue); + b = i == 0 ? a : new Words(1, 0, 0, 0); + if (i == 2) a = default; + if (i == 3) b = default; + } + BigInteger x = Value(a), y = Value(b); + Equal(AddChain(a, b), x + y); + Equal(SubChain(a, b), x - y); + Equal(Interrupted(a, b, out ulong observed), x + y); + if (observed != Clobber(a.A + b.A < a.A ? 1UL : 0)) throw new Exception("Lost carry consumer"); + Equal(ExtraConsumer(a, b, out observed), x - y); + if (observed != (a.A < b.A ? 1UL : 0)) throw new Exception("Lost borrow consumer"); + Equal(FullLimbCarry(a, b, 2), x + y + 2); + Equal(FullLimbCarry(a, b, ulong.MaxValue), x + y + ulong.MaxValue); + ulong seed = (a.A + b.A < a.A ? 1UL : 0) + (a.B + b.B < a.B ? 1UL : 0); + Equal(TwoCarrySeed(a, b), x + y + seed); + } + } +} diff --git a/src/tests/JIT/opt/Add/CarryChains.csproj b/src/tests/JIT/opt/Add/CarryChains.csproj new file mode 100644 index 00000000000000..5914015efe533e --- /dev/null +++ b/src/tests/JIT/opt/Add/CarryChains.csproj @@ -0,0 +1,12 @@ + + + true + None + True + + + + + + + diff --git a/src/tests/JIT/opt/Add/DecimalMultiply.cs b/src/tests/JIT/opt/Add/DecimalMultiply.cs new file mode 100644 index 00000000000000..e5960a287c1596 --- /dev/null +++ b/src/tests/JIT/opt/Add/DecimalMultiply.cs @@ -0,0 +1,122 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using Xunit; + +public class DecimalMultiplyTests +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static decimal Multiply(decimal a, decimal b) => a * b; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static decimal Square(decimal value) => value * value; + + private static readonly BigInteger MaxCoefficient = (BigInteger.One << 96) - 1; + + private static BigInteger Coefficient(int[] bits) => + ((BigInteger)(uint)bits[2] << 64) | ((BigInteger)(uint)bits[1] << 32) | (uint)bits[0]; + + private static decimal Expected(decimal a, decimal b) + { + int[] left = decimal.GetBits(a); + int[] right = decimal.GetBits(b); + BigInteger product = Coefficient(left) * Coefficient(right); + int scale = ((left[3] >> 16) & 0xFF) + ((right[3] >> 16) & 0xFF); + bool negative = ((left[3] ^ right[3]) & int.MinValue) != 0; + + // Round the exact product directly, avoiding double rounding as digits + // are removed to fit decimal's 96-bit coefficient and maximum scale. + for (int dropped = Math.Max(0, scale - 28); dropped <= scale; dropped++) + { + BigInteger divisor = BigInteger.Pow(10, dropped); + BigInteger coefficient = BigInteger.DivRem(product, divisor, out BigInteger remainder); + int comparison = (remainder * 2).CompareTo(divisor); + if (comparison > 0 || (comparison == 0 && !coefficient.IsEven)) + { + coefficient++; + } + if (coefficient <= MaxCoefficient) + { + return new decimal((int)(uint)(coefficient & uint.MaxValue), + (int)(uint)((coefficient >> 32) & uint.MaxValue), + (int)(uint)(coefficient >> 64), negative, (byte)(scale - dropped)); + } + } + throw new OverflowException(); + } + + private static void Check(decimal a, decimal b, bool square = false) + { + decimal expected = 0; + bool overflow = false; + try + { + expected = Expected(a, b); + } + catch (OverflowException) + { + overflow = true; + } + try + { + decimal actual = square ? Square(a) : Multiply(a, b); + if (overflow || actual != expected) + { + throw new Exception("Decimal multiplication mismatch"); + } + } + catch (OverflowException) + { + if (!overflow) + { + throw; + } + } + } + + [Fact] + public static void TestEntryPoint() + { + decimal[] edges = + { + 0m, 1m, -1m, 0.5m, uint.MaxValue, ulong.MaxValue, + decimal.MaxValue, decimal.MinValue, + new decimal(-1, -1, -1, false, 28), + new decimal(-1, -1, -1, true, 28), + new decimal(1, 0, 0, false, 28), + new decimal(3, 0, 0, false, 28), + new decimal(1, 0, 1, false, 14), + }; + foreach (decimal a in edges) + { + Check(a, a, square: true); + foreach (decimal b in edges) + { + Check(a, b); + } + } + + var random = new Random(80674); + byte[] bytes = new byte[24]; + for (int i = 0; i < 3000; i++) + { + random.NextBytes(bytes); + int leftMid = i % 4 == 0 ? 0 : BitConverter.ToInt32(bytes, 4); + int leftHigh = i % 3 == 0 || i % 4 == 0 ? 0 : BitConverter.ToInt32(bytes, 8); + int rightMid = i % 5 == 0 ? 0 : BitConverter.ToInt32(bytes, 16); + int rightHigh = i % 3 == 0 || i % 5 == 0 ? 0 : BitConverter.ToInt32(bytes, 20); + decimal a = new decimal(BitConverter.ToInt32(bytes, 0), leftMid, leftHigh, + i % 2 == 0, (byte)random.Next(29)); + decimal b = new decimal(BitConverter.ToInt32(bytes, 12), rightMid, rightHigh, + i % 7 == 0, (byte)random.Next(29)); + Check(a, b); + if (i % 17 == 0) + { + Check(a, a, square: true); + } + } + } +} diff --git a/src/tests/JIT/opt/Add/DecimalMultiply.csproj b/src/tests/JIT/opt/Add/DecimalMultiply.csproj new file mode 100644 index 00000000000000..09bf8b9fc33972 --- /dev/null +++ b/src/tests/JIT/opt/Add/DecimalMultiply.csproj @@ -0,0 +1,12 @@ + + + true + None + True + + + + + + + diff --git a/src/tests/JIT/opt/Add/DecimalWide.cs b/src/tests/JIT/opt/Add/DecimalWide.cs new file mode 100644 index 00000000000000..d12ae92008d075 --- /dev/null +++ b/src/tests/JIT/opt/Add/DecimalWide.cs @@ -0,0 +1,90 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Reflection; +using Xunit; + +public class DecimalWideTests +{ + private static void Check(int width, int maximumPower) where T : IBinaryInteger + { + Type number = typeof(object).Assembly.GetType("System.Number", throwOnError: true)!; + MethodInfo Method(string name) => number.GetMethod(name, BindingFlags.Static | BindingFlags.NonPublic)!.MakeGenericMethod(typeof(T)); + MethodInfo multiply = Method("WideMultiply"); + MethodInfo add = Method("WideAdd"); + MethodInfo subtract = Method("WideSubtract"); + MethodInfo divide = Method("WideDivideByPow10"); + BigInteger mask = (BigInteger.One << width) - 1; + Random random = new Random(125799); + byte[] bytes = new byte[width / 4]; + + BigInteger Next() + { + random.NextBytes(bytes); + return new BigInteger(bytes, isUnsigned: true); + } + + T Limb(BigInteger value) => T.CreateChecked(value & mask); + + void Equal(object actual, BigInteger expected) + { + if (BigInteger.CreateChecked((T)actual) != expected) + { + throw new Exception($"Incorrect {typeof(T)} wide arithmetic result: expected {expected}, got {actual}"); + } + } + + for (int i = 0; i < 1000; i++) + { + BigInteger x = Next(); + BigInteger y = Next(); + // Include all-one limbs, zero, and carry/borrow across limb boundaries. + if (i < 4) + { + x = i == 0 ? 0 : (BigInteger.One << (2 * width)) - 1; + y = i < 2 ? x : BigInteger.One << ((i - 2) * width); + } + + object[] product = { Limb(x), Limb(y), T.Zero, T.Zero }; + multiply.Invoke(null, product); + BigInteger expected = (x & mask) * (y & mask); + Equal(product[2], expected >> width); + Equal(product[3], expected & mask); + + // The private addition helper's contract excludes high-limb overflow. + BigInteger a = x >> 1; + BigInteger b = y >> 1; + object[] sum = { Limb(a >> width), Limb(a), Limb(b >> width), Limb(b), T.Zero, T.Zero }; + add.Invoke(null, sum); + expected = a + b; + Equal(sum[4], expected >> width); + Equal(sum[5], expected & mask); + + a = BigInteger.Max(x, y); + b = BigInteger.Min(x, y); + object[] difference = { Limb(a >> width), Limb(a), Limb(b >> width), Limb(b), T.Zero, T.Zero }; + subtract.Invoke(null, difference); + expected = a - b; + Equal(difference[4], expected >> width); + Equal(difference[5], expected & mask); + + BigInteger divisor = BigInteger.Pow(10, i % (maximumPower + 1)); + object[] quotient = { Limb(x >> width), Limb(x), Limb(divisor) }; + object remainder = divide.Invoke(null, quotient)!; + expected = BigInteger.DivRem(x, divisor, out BigInteger expectedRemainder); + Equal(quotient[0], expected >> width); + Equal(quotient[1], expected & mask); + Equal(remainder, expectedRemainder); + } + } + + [Fact] + public static void TestEntryPoint() + { + Check(32, 4); + Check(64, 9); + Check(128, 19); + } +} diff --git a/src/tests/JIT/opt/Add/DecimalWide.csproj b/src/tests/JIT/opt/Add/DecimalWide.csproj new file mode 100644 index 00000000000000..ba355ba38b6f66 --- /dev/null +++ b/src/tests/JIT/opt/Add/DecimalWide.csproj @@ -0,0 +1,12 @@ + + + true + None + True + + + + + + + diff --git a/src/tests/JIT/opt/Add/Division.cs b/src/tests/JIT/opt/Add/Division.cs new file mode 100644 index 00000000000000..3764b78f1523af --- /dev/null +++ b/src/tests/JIT/opt/Add/Division.cs @@ -0,0 +1,681 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Reflection; +using System.Runtime.CompilerServices; +using Xunit; + +public class DivisionTests +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong RemainderNonZero(ulong value, ulong divisor) + { + // X64: div + // X64: ret + divisor |= 1; + return Math.DivRem(value, divisor).Remainder; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong AddWithHighMemory(ulong low, ulong value, ref uint high) + { + // X64: add + // X64-NEXT: adc + // ARM64: adds + // ARM64-NEXT: adc + ulong sum = low + value; + ulong upper = high; + return upper + (sum < low ? 1UL : 0UL); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void IncrementHigh(ref uint high) => high++; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong AddWithChangedHighMemory(ulong low, ulong value, ref uint high) + { + ulong sum = low + value; + IncrementHigh(ref high); + ulong upper = high; + return upper + (sum < low ? 1UL : 0UL); + } + + private static void CheckDecimalArithmetic() + { + BigInteger max = (BigInteger.One << 96) - 1; + (BigInteger Coefficient, int Scale) Parts(decimal value) + { + int[] bits = decimal.GetBits(value); + BigInteger coefficient = (uint)bits[0] | ((BigInteger)(uint)bits[1] << 32) | + ((BigInteger)(uint)bits[2] << 64); + return (bits[3] < 0 ? -coefficient : coefficient, (bits[3] >> 16) & 255); + } + + decimal Make(BigInteger coefficient, int scale) => new decimal( + (int)(uint)(BigInteger.Abs(coefficient) & uint.MaxValue), + (int)(uint)((BigInteger.Abs(coefficient) >> 32) & uint.MaxValue), + (int)(uint)(BigInteger.Abs(coefficient) >> 64), coefficient.Sign < 0, (byte)scale); + + void Check(decimal a, decimal b) + { + var (ca, sa) = Parts(a); + var (cb, sb) = Parts(b); + int scale = Math.Max(sa, sb); + ca *= BigInteger.Pow(10, scale - sa); + cb *= BigInteger.Pow(10, scale - sb); + foreach (bool subtract in new[] { false, true }) + { + BigInteger exact = subtract ? ca - cb : ca + cb; + bool fits = false; + for (int drop = 0; drop <= scale; drop++) + { + BigInteger divisor = BigInteger.Pow(10, drop); + BigInteger q = BigInteger.DivRem(BigInteger.Abs(exact), divisor, out BigInteger r); + if (r * 2 > divisor || (r * 2 == divisor && !q.IsEven)) + { + q++; + } + if (q <= max) + { + decimal expected = Make(exact.Sign < 0 ? -q : q, scale - drop); + decimal actual = subtract ? a - b : a + b; + if (actual != expected) + { + throw new Exception("Decimal addition/subtraction oracle"); + } + fits = true; + break; + } + } + if (!fits) + { + Throws(() => { _ = subtract ? a - b : a + b; }); + } + } + if (cb != 0) + { + BigInteger remainder = ca % cb; + int remainderScale = scale; + while (BigInteger.Abs(remainder) > max && remainderScale > 0 && remainder % 10 == 0) + { + remainder /= 10; + remainderScale--; + } + decimal actualRemainder = a % b; + decimal expectedRemainder = Make(remainder, remainderScale); + if (actualRemainder != expectedRemainder) + { + throw new Exception($"Decimal remainder oracle: {a} % {b} = {actualRemainder}, expected {expectedRemainder}"); + } + } + } + + decimal[] edges = { 0m, 1m, -1m, decimal.MaxValue, decimal.MinValue, + new decimal(-1, -1, 0, false, 0), new decimal(0, 0, 1, false, 0), + new decimal(-1, -1, -1, false, 1), new decimal(-1, -1, -1, true, 28), + 0.0000000000000000000000000001m, 0.5m, 1.5m, 10m, uint.MaxValue }; + foreach (decimal a in edges) + { + foreach (decimal b in edges) + { + Check(a, b); + } + } + Random random = new Random(789); + byte[] bytes = new byte[24]; + for (int i = 0; i < 1500; i++) + { + random.NextBytes(bytes); + int scale = random.Next(29); + decimal a = new decimal(BitConverter.ToInt32(bytes, 0), BitConverter.ToInt32(bytes, 4), + BitConverter.ToInt32(bytes, 8), (i & 1) != 0, (byte)scale); + decimal b = new decimal(BitConverter.ToInt32(bytes, 12), i % 3 == 0 ? 0 : BitConverter.ToInt32(bytes, 16), + i % 3 == 0 ? 0 : BitConverter.ToInt32(bytes, 20), (i & 2) != 0, + (byte)(i % 2 == 0 ? scale : random.Next(29))); + Check(a, b); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static long SubtractWithWidenedHigh(ulong low, ulong subLow, uint high, ulong subHigh, out ulong result) + { + // X64: sub + // X64-NEXT: sbb + // ARM64: subs + // ARM64-NEXT: sbc + ulong difference = low - subLow; + long upper = (long)high - (long)subHigh; + upper -= difference > low ? 1L : 0L; + result = difference; + return upper; + } + + private static void CheckDecimalDivisionHelpers() + { + Type calculator = typeof(decimal).GetNestedType("DecCalc", BindingFlags.NonPublic); + Type buf12 = calculator.GetNestedType("Buf12", BindingFlags.NonPublic); + Type buf16 = calculator.GetNestedType("Buf16", BindingFlags.NonPublic); + MethodInfo divide96 = calculator.GetMethod("Div96By32", BindingFlags.NonPublic | BindingFlags.Static); + MethodInfo divide128 = calculator.GetMethod("Div128By96", BindingFlags.NonPublic | BindingFlags.Static); + MethodInfo add32 = calculator.GetMethod("Add32To96", BindingFlags.NonPublic | BindingFlags.Static); + + object Pack(Type type, BigInteger value, int limbs) + { + object buffer = Activator.CreateInstance(type); + for (int limb = 0; limb < limbs; limb++) + { + type.GetField($"U{limb}").SetValue(buffer, (uint)(value & uint.MaxValue)); + value >>= 32; + } + return buffer; + } + + BigInteger Unpack(object buffer, int limbs) + { + BigInteger value = 0; + for (int limb = limbs - 1; limb >= 0; limb--) + { + value = (value << 32) | (uint)buffer.GetType().GetField($"U{limb}").GetValue(buffer); + } + return value; + } + + int correctionsSeen = 0; + void Check128(BigInteger numerator, BigInteger denominator) + { + BigInteger expected = BigInteger.DivRem(numerator, denominator, out BigInteger remainder); + BigInteger estimate = (numerator >> 64) / (denominator >> 64); + int corrections = (int)(estimate - expected); + if (corrections < 0 || corrections > 2) + { + throw new Exception("Invalid normalized division test vector"); + } + correctionsSeen |= 1 << corrections; + object[] args = { Pack(buf16, numerator, 4), Pack(buf12, denominator, 3) }; + uint quotient = (uint)divide128.Invoke(null, args); + if (quotient != expected || Unpack(args[0], 3) != remainder) + { + throw new Exception($"Decimal 128/96 division, corrections {corrections}"); + } + } + + BigInteger normalized = (BigInteger.One << 95) | ulong.MaxValue; + foreach (uint q in new uint[] { 0, 1, 2, 0x7FFFFFFF, 0xFFFFFFFD }) + { + foreach (BigInteger remainder in new[] { BigInteger.Zero, BigInteger.One, normalized - 1 }) + { + Check128(normalized * q + remainder, normalized); + } + } + + Random random = new Random(456); + byte[] bytes = new byte[16]; + for (int i = 0; i < 1500; i++) + { + random.NextBytes(bytes); + BigInteger numerator = new BigInteger(bytes, isUnsigned: true); + random.NextBytes(bytes); + BigInteger denominator = (new BigInteger(bytes, isUnsigned: true) & ((BigInteger.One << 96) - 1)) | + (BigInteger.One << 95); + // The helper requires the highest dividend limb to be below the + // highest divisor limb, so its quotient estimate fits in 32 bits. + Check128(numerator % ((denominator >> 64) << 96), denominator); + + uint divisor = (i % 4) switch { 0 => 1U, 1 => uint.MaxValue, 2 => 10U, _ => (uint)random.Next(1, int.MaxValue) }; + numerator &= (BigInteger.One << 96) - 1; + if (i % 3 == 0) + { + numerator &= ulong.MaxValue; + } + BigInteger expected = BigInteger.DivRem(numerator, divisor, out BigInteger remainder); + object[] args = { Pack(buf12, numerator, 3), divisor }; + uint actualRemainder = (uint)divide96.Invoke(null, args); + if (Unpack(args[0], 3) != expected || actualRemainder != remainder) + { + throw new Exception("Decimal 96/32 division"); + } + BigInteger max = (BigInteger.One << 96) - 1; + if (i % 5 == 0) + { + numerator = max - i % 3; + } + args = new object[] { Pack(buf12, numerator, 3), divisor }; + bool fits = (bool)add32.Invoke(null, args); + BigInteger sum = numerator + divisor; + if (fits != (sum <= max) || Unpack(args[0], 3) != (sum & max)) + { + throw new Exception("Decimal add32 carry/overflow"); + } + } + if (correctionsSeen != 7) + { + throw new Exception("Decimal tests must exercise zero, one, and two quotient corrections"); + } + } + + private static void CheckDecimalRounding() + { + Random random = new Random(123); + byte[] bytes = new byte[12]; + for (int i = 0; i < 1200; i++) + { + random.NextBytes(bytes); + BigInteger coefficient = new BigInteger(bytes, isUnsigned: true); + if (i % 3 == 0) + { + coefficient &= ulong.MaxValue; + } + int scale = i % 29; + int decimals = random.Next(scale + 1); + BigInteger divisor = BigInteger.Pow(10, scale - decimals); + // Include exact midpoints, their neighbors, zero, and maximal limbs. + if (i % 7 == 0) + { + coefficient = divisor / 2 + i % 3 - 1; + coefficient = BigInteger.Max(coefficient, BigInteger.Zero); + } + else if (i % 11 == 0) + { + coefficient = (BigInteger.One << 96) - 1; + } + bool negative = (i & 1) != 0; + decimal value = new decimal((int)(uint)(coefficient & uint.MaxValue), + (int)(uint)((coefficient >> 32) & uint.MaxValue), + (int)(uint)(coefficient >> 64), negative, (byte)scale); + BigInteger quotient = BigInteger.DivRem(coefficient, divisor, out BigInteger remainder); + foreach (MidpointRounding mode in Enum.GetValues()) + { + bool increment = mode switch + { + MidpointRounding.ToEven => remainder * 2 > divisor || + (remainder * 2 == divisor && !quotient.IsEven), + MidpointRounding.AwayFromZero => remainder * 2 >= divisor, + MidpointRounding.ToNegativeInfinity => negative && !remainder.IsZero, + MidpointRounding.ToPositiveInfinity => !negative && !remainder.IsZero, + _ => false, + }; + BigInteger expected = quotient + (increment ? 1 : 0); + int[] bits = decimal.GetBits(decimal.Round(value, decimals, mode)); + BigInteger actual = (uint)bits[0] | ((BigInteger)(uint)bits[1] << 32) | + ((BigInteger)(uint)bits[2] << 64); + int expectedFlags = (decimals << 16) | (negative ? int.MinValue : 0); + if (actual != expected || bits[3] != expectedFlags) + { + throw new Exception($"Decimal rounding: vector {i}, mode {mode}"); + } + } + } + } + + private static int s_calls; + private delegate nuint WideningDivide(nuint hi, nuint lo, nuint divisor, out nuint remainder); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Unsigned(ulong x, ulong y, out ulong remainder) + { + // X64: div + // X64-NOT: imul + // X64-NOT: sub + // X64: ret + ulong q = x / y; + remainder = x - q * y; + return q; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static long Signed(long x, long y, out long remainder) + { + // X64: idiv + // X64-NOT: imul + // X64-NOT: sub + // X64: ret + long q = x / y; + remainder = x - y * q; + return q; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint Unsigned32(uint x, uint y, out uint remainder) + { + // X64: div + // X64-NOT: imul + // X64: ret + // X86: div + // X86-NOT: imul + // X86: ret + uint q = x / y; + remainder = x - q * y; + return q; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int Signed32(int x, int y, out int remainder) + { + // X64: idiv + // X64-NOT: imul + // X64: ret + // X86: idiv + // X86-NOT: imul + // X86: ret + int q = x / y; + remainder = x - q * y; + return q; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong RemainderOnly(ulong x, ulong y) + { + return x - (x / y) * y; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong InlinedRemainder(ulong x, ulong y) => Math.DivRem(x, y).Remainder; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Touch(ulong x) => x + 3; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong AcrossCall(ulong x, ulong y) + { + ulong q = x / y; + ulong z = Touch(x); + return z ^ q ^ (x - q * y); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong KeepQuotient(ulong x, ulong y, ulong z, out ulong remainder) + { + ulong q = x / y; + s_calls++; + remainder = x - q * y; + return q ^ z; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Changed(ulong x, ulong y) + { + ulong q = x / y; + x += 17; + y += 3; + q += 2; + return x - q * y; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Aliased(ulong[] values, ulong y) + { + ulong q = values[0] / y; + values[0] += 17; + return values[0] - q * y; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void Mutate(ref ulong x) => x += 17; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ChangedByRef(ulong x, ulong y) + { + ulong q = x / y; + Mutate(ref x); + return x - q * y; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong QuotientChangedByRef(ulong x, ulong y) + { + ulong q = x / y; + Mutate(ref q); + return x - q * y; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong EmbeddedAssignment(ulong x, ulong y) => x - ((x += 17) / y) * y; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong CheckedProduct(ulong x, ulong y, ulong z) + { + ulong q = x / y; + return checked(x - checked(q * z)); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 Unsigned128(UInt128 x, UInt128 y, out UInt128 r) + { + var result = UInt128.DivRem(x, y); + r = result.Remainder; + return result.Quotient; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Int128 Signed128(Int128 x, Int128 y, out Int128 r) + { + var result = Int128.DivRem(x, y); + r = result.Remainder; + return result.Quotient; + } + + private static void Check128(UInt128 a, UInt128 b) + { + if (b == 0) + { + return; + } + UInt128 q = Unsigned128(a, b, out UInt128 r); + if ((BigInteger)q * (BigInteger)b + (BigInteger)r != (BigInteger)a || r >= b) + { + throw new Exception("UInt128 division"); + } + + Int128 x = (Int128)a, y = (Int128)b; + if (x == Int128.MinValue && y == -1) + { + return; + } + Int128 sq = Signed128(x, y, out Int128 sr); + if ((BigInteger)sq * (BigInteger)y + (BigInteger)sr != (BigInteger)x || + BigInteger.Abs((BigInteger)sr) >= BigInteger.Abs((BigInteger)y) || + (sr != 0 && Int128.IsNegative(sr) != Int128.IsNegative(x)) || x % y != sr || x / y != sq) + { + throw new Exception("Int128 division"); + } + } + + private static void CheckScalar(ulong x, ulong y) + { + if (RemainderNonZero(x, y) != x % (y | 1)) + { + throw new Exception("The unused quotient must not remove the remainder's division"); + } + uint memoryHigh = (uint)y; + ulong carry = (ulong)(((BigInteger)x + y) >> 64); + if (AddWithHighMemory(x, y, ref memoryHigh) != (ulong)memoryHigh + carry) + { + throw new Exception("Carry with widened memory input"); + } + ulong changed = AddWithChangedHighMemory(x, y, ref memoryHigh); + if (changed != (ulong)(uint)(y + 1) + carry || memoryHigh != (uint)(y + 1)) + { + throw new Exception("Carry must observe the intervening memory write"); + } + long high = SubtractWithWidenedHigh(x, y, (uint)y, x, out ulong low); + BigInteger expected = ((((BigInteger)(uint)y << 64) | x) - (((BigInteger)x << 64) | y)) & + ((BigInteger.One << 128) - 1); + if ((((BigInteger)(ulong)high << 64) | low) != expected) + { + throw new Exception("Subtraction with widened high word"); + } + if (y == 0) + { + return; + } + ulong q = Unsigned(x, y, out ulong r); + if ((BigInteger)q * y + r != x || r >= y || RemainderOnly(x, y) != r || + InlinedRemainder(x, y) != r || AcrossCall(x, y) != ((x + 3) ^ q ^ r)) + { + throw new Exception("unsigned scalar"); + } + int calls = s_calls; + if (KeepQuotient(x, y, ~x, out ulong r2) != (q ^ ~x) || r2 != r || s_calls != calls + 1) + { + throw new Exception("quotient lifetime"); + } + if (Changed(x, y) != unchecked((x + 17) - (q + 2) * (y + 3)) || + Aliased(new[] { x }, y) != unchecked(x + 17 - q * y) || + ChangedByRef(x, y) != unchecked(x + 17 - q * y) || + QuotientChangedByRef(x, y) != unchecked(x - (q + 17) * y) || + EmbeddedAssignment(x, y) != unchecked(x - ((x + 17) / y) * y)) + { + throw new Exception("changed operands"); + } + + long sx = (long)x, sy = (long)y; + if (!(sx == long.MinValue && sy == -1)) + { + long sq = Signed(sx, sy, out long sr); + if ((BigInteger)sq * sy + sr != sx || BigInteger.Abs(sr) >= BigInteger.Abs(sy) || + (sr != 0 && (sr < 0) != (sx < 0))) + { + throw new Exception("signed scalar"); + } + } + + uint ux = (uint)x, uy = (uint)y; + if (uy == 0) + { + return; + } + uint uq = Unsigned32(ux, uy, out uint ur); + if ((ulong)uq * uy + ur != ux || ur >= uy) + { + throw new Exception("uint division"); + } + int ix = (int)x, iy = (int)y; + if (!(ix == int.MinValue && iy == -1)) + { + int iq = Signed32(ix, iy, out int ir); + if ((long)iq * iy + ir != ix || Math.Abs((long)ir) >= Math.Abs((long)iy) || + (ir != 0 && (ir < 0) != (ix < 0))) + { + throw new Exception("int division"); + } + } + } + + private static void Throws(Action action) where T : Exception + { + try + { + action(); + } + catch (T) + { + return; + } + throw new Exception($"Expected {typeof(T)}"); + } + + [Fact] + public static void TestEntryPoint() + { + ulong[] edges = { 0, 1, 2, 3, uint.MaxValue, (ulong)uint.MaxValue + 1, 1UL << 63, (1UL << 63) - 1, ulong.MaxValue }; + foreach (ulong x in edges) + { + foreach (ulong y in edges) + { + CheckScalar(x, y); + Check128(x, y); + Check128(((UInt128)x << 64) | y, ((UInt128)y << 64) | x); + } + } + Throws(() => Unsigned(123, 0, out _)); + Throws(() => Signed(123, 0, out _)); + Throws(() => Unsigned32(123, 0, out _)); + Throws(() => Signed32(123, 0, out _)); + int callsBeforeException = s_calls; + Throws(() => KeepQuotient(123, 0, 456, out _)); + if (s_calls != callsBeforeException) + { + throw new Exception("division exception order"); + } + Throws(() => Signed(long.MinValue, -1, out _)); + Throws(() => Signed32(int.MinValue, -1, out _)); + Throws(() => Signed128(123, 0, out _)); + Throws(() => Signed128(Int128.MinValue, -1, out _)); + Throws(() => { _ = Int128.MinValue % -1; }); + Throws(() => CheckedProduct(ulong.MaxValue, 1, 2)); + + Type calculator = typeof(BigInteger).Assembly.GetType("System.Numerics.BigIntegerCalculator", throwOnError: true); + MethodInfo widening = calculator.GetMethod("DivRem", BindingFlags.Static | BindingFlags.NonPublic, + binder: null, new[] { typeof(nuint), typeof(nuint), typeof(nuint), typeof(nuint).MakeByRefType() }, modifiers: null); + WideningDivide divide = widening.CreateDelegate(); + Throws(() => divide(0, 123, 0, out _)); + foreach (ulong edge in edges) + { + nuint d = (nuint)edge; + if (d == 0) + { + continue; + } + foreach (nuint hi in new nuint[] { 0, d - 1 }) + { + foreach (nuint lo in new nuint[] { 0, 1, nuint.MaxValue }) + { + nuint q = divide(hi, lo, d, out nuint r); + BigInteger value = ((BigInteger)(ulong)hi << (IntPtr.Size * 8)) | (ulong)lo; + if ((BigInteger)(ulong)q * (ulong)d + (ulong)r != value || r >= d) + { + throw new Exception("widening division"); + } + } + } + } + + CheckDecimalRounding(); + CheckDecimalDivisionHelpers(); + CheckDecimalArithmetic(); + + Random random = new Random(42); + byte[] bytes = new byte[32]; + for (int i = 0; i < 3000; i++) + { + random.NextBytes(bytes); + ulong lo = BitConverter.ToUInt64(bytes, 0), hi = BitConverter.ToUInt64(bytes, 8); + ulong dlo = BitConverter.ToUInt64(bytes, 16), dhi = BitConverter.ToUInt64(bytes, 24); + CheckScalar(lo, dlo); + UInt128 a = ((UInt128)hi << 64) | lo; + Check128(a, ((UInt128)dhi << 64) | dlo); + Check128(a, dlo); + } + + // Exercise both the single-limb and multi-limb BigInteger division paths. + for (int length = 1; length <= 160; length++) + { + byte[] input = new byte[length * IntPtr.Size]; + random.NextBytes(input); + BigInteger a = new BigInteger(input, isUnsigned: true); + foreach (ulong d in edges) + { + if (d == 0) + { + continue; + } + BigInteger q = BigInteger.DivRem(a, d, out BigInteger r); + if (q * d + r != a || r < 0 || r >= d) + { + throw new Exception("BigInteger limb division"); + } + } + byte[] divisor = new byte[Math.Max(1, input.Length / 2)]; + random.NextBytes(divisor); + divisor[0] |= 1; + BigInteger b = new BigInteger(divisor, isUnsigned: true); + BigInteger quotient = BigInteger.DivRem(a, b, out BigInteger remainder); + if (quotient * b + remainder != a || remainder < 0 || remainder >= b) + { + throw new Exception("BigInteger multi-limb division"); + } + } + } +} diff --git a/src/tests/JIT/opt/Add/Division.csproj b/src/tests/JIT/opt/Add/Division.csproj new file mode 100644 index 00000000000000..e927ded2fbb6d3 --- /dev/null +++ b/src/tests/JIT/opt/Add/Division.csproj @@ -0,0 +1,14 @@ + + + true + None + True + + + + true + + + + + diff --git a/src/tests/JIT/opt/Add/FunnelShift.cs b/src/tests/JIT/opt/Add/FunnelShift.cs new file mode 100644 index 00000000000000..c154264192fb5e --- /dev/null +++ b/src/tests/JIT/opt/Add/FunnelShift.cs @@ -0,0 +1,182 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using Xunit; + +public class FunnelShiftTests +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Shift1(ulong lo, ulong hi) + { + // X64: shrd {{.*}}, 1 + // ARM64: extr {{.*}}, #1 + return (lo >> 1) | (hi << 63); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Shift17(ulong lo, ulong hi) + { + // X64: shrd {{.*}}, 17 + // ARM64: extr {{.*}}, #17 + return (hi << 47) | (lo >> 17); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Shift32(ulong lo, ulong hi) + { + // X64: shrd {{.*}}, 32 + // ARM64: extr {{.*}}, #32 + return (lo >> 32) | (hi << 32); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Shift63(ulong lo, ulong hi) + { + // X64: shrd {{.*}}, 63 + // ARM64: extr {{.*}}, #63 + return (lo >> 63) | (hi << 1); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint Shift13(uint lo, uint hi) + { + // X64: shrd {{.*}}, 13 + // ARM64: extr {{.*}}, #13 + return (lo >> 13) | (hi << 19); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint Shift31(uint lo, uint hi) + { + // X64: shrd {{.*}}, 31 + // ARM64: extr {{.*}}, #31 + return (hi << 1) | (lo >> 31); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Masked(ulong lo, ulong hi) => (lo >> 65) | (hi << -1); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong KeepInputs(ulong lo, ulong hi, out ulong result) + { + result = (lo >> 17) | (hi << 47); + return lo ^ hi; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool IsZero(ulong lo, ulong hi) => ((lo >> 17) | (hi << 47)) == 0; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Overlap(ulong lo, ulong hi) => (lo >> 17) | (hi << 46); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Gap(ulong lo, ulong hi) => (lo >> 17) | (hi << 48); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ZeroCount(ulong lo, ulong hi) => (lo >> 64) | (hi << 64); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static long Signed(long lo, long hi) => (lo >> 17) | (hi << 47); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Redefine(ulong lo, ulong hi) => (lo >> 17) | ((lo = hi) << 47); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Aliased(ulong[] values) + { + ulong shifted = values[0] >> 17; + values[0] = ~values[0]; + return shifted | (values[0] << 47); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 SquareStep(ulong a, ulong b, ulong old, UInt128 carry, out ulong low) + { + // X64: shrd {{.*}}, 1 + // X64: shrd {{.*}}, 63 + // ARM64: extr {{.*}}, #1 + // ARM64: extr {{.*}}, #63 + UInt128 digit1 = old + carry; + UInt128 digit2 = (UInt128)a * b; + low = (ulong)(digit1 + (digit2 << 1)); + return (digit2 + (digit1 >> 1)) >> 63; + } + + // Construct the expected result one bit at a time, independently of the + // complementary-shift pattern under test. + private static ulong Expected(ulong lo, ulong hi, int count, int width = 64) + { + ulong result = 0; + for (int bit = 0; bit < width; bit++) + { + int sourceBit = bit + count; + ulong source = sourceBit < width ? lo : hi; + result |= ((source >> (sourceBit % width)) & 1) << bit; + } + return result; + } + + private static void Check(ulong lo, ulong hi) + { + if (Shift1(lo, hi) != Expected(lo, hi, 1) || + Shift17(lo, hi) != Expected(lo, hi, 17) || + Shift32(lo, hi) != Expected(lo, hi, 32) || + Shift63(lo, hi) != Expected(lo, hi, 63) || + Shift13((uint)lo, (uint)hi) != Expected((uint)lo, (uint)hi, 13, 32) || + Shift31((uint)lo, (uint)hi) != Expected((uint)lo, (uint)hi, 31, 32) || + Masked(lo, hi) != Expected(lo, hi, 1) || + KeepInputs(lo, hi, out ulong result) != (lo ^ hi) || result != Expected(lo, hi, 17) || + IsZero(lo, hi) != (Expected(lo, hi, 17) == 0) || + Overlap(lo, hi) != ((lo >> 17) | (hi << 46)) || + Gap(lo, hi) != ((lo >> 17) | (hi << 48)) || + ZeroCount(lo, hi) != (lo | hi) || + Signed((long)lo, (long)hi) != (((long)lo >> 17) | ((long)hi << 47)) || + Redefine(lo, hi) != Expected(lo, hi, 17) || + Aliased(new[] { lo }) != Expected(lo, ~lo, 17)) + { + throw new Exception($"Funnel shift failed: {lo:X16}, {hi:X16}"); + } + + // The square kernel carries 65 bits between limbs. Use a wider oracle + // so the doubled product's 129th bit is retained independently. + UInt128 carry = ((UInt128)(hi & 1) << 64) | lo; + BigInteger full = (BigInteger)lo * hi * 2 + hi + (BigInteger)carry; + UInt128 nextCarry = SquareStep(lo, hi, hi, carry, out ulong low); + if (low != (ulong)(full & ulong.MaxValue) || nextCarry != (UInt128)(full >> 64)) + { + throw new Exception($"Square carry failed: {lo:X16}, {hi:X16}"); + } + } + + [Fact] + public static void TestEntryPoint() + { + ulong[] edges = { 0, 1, ulong.MaxValue, 1UL << 63, (1UL << 63) - 1, 0xAAAAAAAAAAAAAAAA, 0x5555555555555555 }; + foreach (ulong lo in edges) + { + foreach (ulong hi in edges) + { + Check(lo, hi); + } + } + + for (int lo = 0; lo < 64; lo++) + { + for (int hi = 0; hi < 64; hi++) + { + Check(1UL << lo, 1UL << hi); + } + } + + Random random = new Random(42); + byte[] bytes = new byte[16]; + for (int i = 0; i < 2000; i++) + { + random.NextBytes(bytes); + Check(BitConverter.ToUInt64(bytes, 0), BitConverter.ToUInt64(bytes, 8)); + } + } +} diff --git a/src/tests/JIT/opt/Add/FunnelShift.csproj b/src/tests/JIT/opt/Add/FunnelShift.csproj new file mode 100644 index 00000000000000..49fbd76be0ee73 --- /dev/null +++ b/src/tests/JIT/opt/Add/FunnelShift.csproj @@ -0,0 +1,14 @@ + + + true + None + True + + + + true + + + + + diff --git a/src/tests/JIT/opt/Add/WideArithmetic.cs b/src/tests/JIT/opt/Add/WideArithmetic.cs new file mode 100644 index 00000000000000..022f866b2b92f5 --- /dev/null +++ b/src/tests/JIT/opt/Add/WideArithmetic.cs @@ -0,0 +1,246 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using Xunit; + +public class WideArithmeticTests +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 Increment(UInt128 value) => value + 1; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 Decrement(UInt128 value) + { + // X64: sub + // X64-NEXT: sbb + // ARM64: subs + // ARM64-NEXT: sbc + return value - 1; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Int128 Negate(Int128 value) + { + // X64: neg + // X64-NEXT: sbb + // ARM64: negs + // ARM64-NEXT: sbc {{.*}}, xzr, + return -value; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong SubtractSeven(ulong value, ulong high, out ulong low) + { + // X64: sub + // X64-NOT: cmp + // X64: sbb + // ARM64: subs + // ARM64-NEXT: sbc + // ARM64-NEXT: str {{x[0-9]+}}, [{{x[0-9]+}}] + // ARM64-NOT: mov + ulong difference = value - 7; + ulong result = high - (difference > value ? 1UL : 0UL); + low = difference; + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint SubtractSeven32(uint value, uint high, out uint low) + { + // X64: sub + // X64-NOT: cmp + // X64: sbb + // ARM64: subs + // ARM64-NEXT: sbc + // ARM64-NEXT: str {{w[0-9]+}}, [{{x[0-9]+}}] + // ARM64-NOT: mov + uint difference = value - 7; + uint result = high - (difference > value ? 1U : 0U); + low = difference; + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong SubtractHalfRange(ulong value, ulong high, out ulong low) + { + ulong difference = value - (1UL << 63); + ulong result = high - (difference > value ? 1UL : 0UL); + low = difference; + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ComplementBorrow(ulong value, out ulong low) + { + ulong difference = value - 7; + ulong result = difference <= value ? 1UL : 0UL; + low = difference; + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool NegatedProductIsZero(long a, long b, out long product) + { + // The returned condition must not displace the still-live product. + // X64: imul + // X64-NEXT: neg + // X64-NEXT: setae + // X64-NEXT: movzx + // X64-NEXT: mov {{qword ptr \[[^]]+\]}}, + // A contained ARM64 MNEG must not be used as a flags producer. + long value = -(a * b); + bool zero = value == 0; + product = value; + return zero; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong CorrectQuotient(ulong high, ulong low, ulong extra, out ulong sum) + { + // X64: add + // X64-NEXT: adc + // ARM64: adds + // ARM64-NEXT: adc + ulong middle = low + extra; + high += middle < extra ? 1UL : 0UL; + sum = middle; + return high; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool ClearAndReturnFalse(out object value) + { + // The return constant can reuse the register already zeroed for the store. + // X64: xor + // X64-NEXT: mov + // X64-NOT: xor + // X64: ret + value = null; + return false; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 Multiply(UInt128 a, UInt128 b, out UInt128 low) => UInt128.BigMul(a, b, out low); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Int128 MultiplySigned(Int128 a, Int128 b, out Int128 low) => Int128.BigMul(a, b, out low); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Int128 CheckedAdd(Int128 a, Int128 b) => checked(a + b); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Int128 CheckedSubtract(Int128 a, Int128 b) => checked(a - b); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Int128 CheckedNegate(Int128 value) => checked(-value); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Int128 CheckedMultiply(Int128 a, Int128 b) => checked(a * b); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint Negate32(uint value, uint high, out uint low) + { + uint difference = 0 - value; + uint result = (0 - high) - (difference != 0 ? 1U : 0U); + low = difference; + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 CheckedDecrement(UInt128 value) => checked(value - 1); + + private static void Require(bool condition) + { + if (!condition) + { + throw new Exception("Wide arithmetic mismatch"); + } + } + + private static void CheckOverflow(Func operation, BigInteger expected) + { + bool overflow = expected < (BigInteger)Int128.MinValue || expected > (BigInteger)Int128.MaxValue; + try + { + Int128 actual = operation(); + Require(!overflow && (BigInteger)actual == expected); + } + catch (OverflowException) + { + Require(overflow); + } + } + + [Fact] + public static void TestEntryPoint() + { + ulong[] edges = { 0, 1, 6, 7, 8, uint.MaxValue, 1UL << 32, 1UL << 63, ulong.MaxValue - 1, ulong.MaxValue }; + var random = new Random(80674); + byte[] bytes = new byte[32]; + BigInteger mask = (BigInteger.One << 128) - 1; + Require(!ClearAndReturnFalse(out object cleared) && cleared == null); + for (int i = 0; i < 2500; i++) + { + random.NextBytes(bytes); + UInt128 a = ((UInt128)BitConverter.ToUInt64(bytes, 0) << 64) | BitConverter.ToUInt64(bytes, 8); + UInt128 b = ((UInt128)BitConverter.ToUInt64(bytes, 16) << 64) | BitConverter.ToUInt64(bytes, 24); + if (i < edges.Length * edges.Length) + { + a = ((UInt128)edges[i / edges.Length] << 64) | edges[i % edges.Length]; + b = UInt128.MaxValue - a; + } + BigInteger product = (BigInteger)a * (BigInteger)b; + UInt128 high = Multiply(a, b, out UInt128 low); + Require((BigInteger)high == product >> 128 && (BigInteger)low == (product & mask)); + product = (BigInteger)(Int128)a * (BigInteger)(Int128)b; + Int128 signedHigh = MultiplySigned((Int128)a, (Int128)b, out Int128 signedLow); + Require((BigInteger)signedHigh == product >> 128 && (BigInteger)(UInt128)signedLow == (product & mask)); + Require((BigInteger)Increment(a) == (((BigInteger)a + 1) & mask)); + Require((BigInteger)Decrement(a) == (((BigInteger)a - 1) & mask)); + Require((BigInteger)(UInt128)Negate((Int128)a) == (-(BigInteger)a & mask)); + CheckOverflow(() => CheckedAdd((Int128)a, (Int128)b), (BigInteger)(Int128)a + (BigInteger)(Int128)b); + CheckOverflow(() => CheckedSubtract((Int128)a, (Int128)b), (BigInteger)(Int128)a - (BigInteger)(Int128)b); + CheckOverflow(() => CheckedNegate((Int128)a), -(BigInteger)(Int128)a); + if (i < edges.Length * edges.Length) + { + CheckOverflow(() => CheckedMultiply((Int128)a, (Int128)b), product); + } + try + { + Require(CheckedDecrement(a) == Decrement(a) && a != 0); + } + catch (OverflowException) + { + Require(a == 0); + } + ulong x = (ulong)a, h = (ulong)(a >> 64); + ulong pair32 = ((ulong)(uint)h << 32) | (uint)x; + uint negatedHigh = Negate32((uint)x, (uint)h, out uint negatedLow); + Require((((ulong)negatedHigh << 32) | negatedLow) == unchecked(0UL - pair32)); + Require(SubtractSeven(x, h, out ulong l) == unchecked(h - (x < 7 ? 1UL : 0)) && l == unchecked(x - 7)); + Require(SubtractSeven32((uint)x, (uint)h, out uint l32) == unchecked((uint)h - ((uint)x < 7 ? 1U : 0)) && l32 == unchecked((uint)x - 7)); + Require(SubtractHalfRange(x, h, out l) == unchecked(h - (x < (1UL << 63) ? 1UL : 0)) && l == unchecked(x - (1UL << 63))); + Require(ComplementBorrow(x, out l) == (x >= 7 ? 1UL : 0) && l == unchecked(x - 7)); + bool zero = NegatedProductIsZero((long)x, (long)h, out long negated); + Require(negated == unchecked(-((long)x * (long)h)) && zero == (negated == 0)); + UInt128 sum = (UInt128)x + h; + Require(CorrectQuotient((ulong)b, x, h, out l) == unchecked((ulong)b + (ulong)(sum >> 64)) && l == (ulong)sum); + + // Exercise the calculator's multiply-by-limb and quotient correction + // through public APIs, using a known quotient and remainder. + int bits = IntPtr.Size * 8; + BigInteger divisor = ((BigInteger)a << bits) + (ulong)b + 1; + BigInteger quotient = (ulong)(nuint)x; + BigInteger remainder = divisor >> 1; + BigInteger dividend = divisor * quotient + remainder; + Require(BigInteger.DivRem(dividend, divisor, out BigInteger actualRemainder) == quotient && actualRemainder == remainder); + if (b != 0) + { + Require((BigInteger)(a / b) == (BigInteger)a / (BigInteger)b && (BigInteger)(a % b) == (BigInteger)a % (BigInteger)b); + } + } + } +} diff --git a/src/tests/JIT/opt/Add/WideArithmetic.csproj b/src/tests/JIT/opt/Add/WideArithmetic.csproj new file mode 100644 index 00000000000000..0b16b3cff8348c --- /dev/null +++ b/src/tests/JIT/opt/Add/WideArithmetic.csproj @@ -0,0 +1,14 @@ + + + true + None + True + + + + true + + + + +