From af9cdfc2c6edfb2fafd90becf566b4d5b829779a Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 16 Sep 2026 08:47:41 +0100 Subject: [PATCH 1/4] Optimize wide integer arithmetic on x64 and ARM64 Recognize unsigned carry and borrow idioms expressed through ordinary arithmetic without adding managed APIs. Lower eligible straight-line and counted-loop chains to ADD/ADC and SUB/SBB on x64 and ADDS/ADCS and SUBS/SBCS on ARM64. Preserve flags across loop backedges and materialize carry or borrow only where needed. Recognize full-width multiply-accumulate chains. Add ADX feature detection and select MULX/ADCX/ADOX for proven products on supported x64 targets; use MUL/UMULH with carry arithmetic on ARM64. Keep unrelated BigMul nodes on their existing path. Preserve managed byref tracking and fallback paths, and refine register constraints, operand reuse, addressing and loop control to avoid redundant carry spills and loads. Seed ADX without a temporary register and preserve both carry flags through LEA countdowns and JRCXZ. Use ordinary DEC selection for loops that need to preserve CF alone. Combine complementary constant shifts into SHRD or EXTR. Reuse the hardware remainder for eligible xarch quotient/remainder expressions while preserving exception behavior and observable destination values. Scope fixed-register propagation to methods with carry arithmetic and apply conservative profitability guards. Retain existing select lowering and zero-flag INC/DEC optimizations when carry transforms would add overhead. Expose these patterns in Int128, UInt128, Decimal, IEEE decimal helpers and BigIntegerCalculator. Use matching span bounds for limb loops, retain MulAdd1 unrolling, and express Montgomery reduction as a scalar carry loop. Preserve architecture-specific alternatives and 32-bit arithmetic paths. Use scalar long/ulong Tensor DivRem on x64 to retain the hardware remainder. Document the source idioms, carry-width requirements and internal helper preconditions. Widening division requires a quotient that fits in one limb; span slicing validates destination capacity before entering limb loops. Prove deleted carry-local values dead from current control flow instead of relying on liveness computed before lowering. Include exception successors and promoted-parent aliases, exclude unsupported implicit uses, and bound analysis work. Use per-query visited bitsets without caching across IR or control-flow mutations. Skip carry-folding searches until a SETCC may be available while keeping comparison recognition enabled. Share local-use, loop-entry, dead-store and address-rewrite helpers. Preserve memory ordering, flags consumers, LIR use ordering, value lifetimes and control-flow analysis validity, and require distinct loop exits. Add correctness and disassembly coverage for carry/borrow chains, wide products, decimal rounding, division and funnel shifts. Cover boundary and randomized inputs, struct and field aliases, calls, exceptions, volatile accesses, arbitrary carry inputs, generic helper expansion and exception- handler observations. Validation: - Windows x64 and all cross-target Checked JIT builds; Linux ARM64 Checked runtime and JIT builds. - x64 arithmetic suites with expensive JIT checks, register stress, AVX2 disabled and hardware intrinsics disabled. - Docker/QEMU ARM64 arithmetic suites with expensive checks, register stress and hardware intrinsics disabled; targeted x64/ARM64 disassembly checks. - Actual BigInteger MulAdd1 checks cover 4,000 random, boundary and overlap cases per run. Its x64 implementation is 344 bytes and 98 instructions, including the ADX loop and MULX scalar tail. - IEEE Decimal32/64/128 differential checks cover 31,500 arithmetic results, matching the comparison baseline byte-for-byte, including under stress. - Full Windows x64 SuperPMI validation: 980,044 contexts across nine overlapping collections, 218 known missing contexts and zero other failures. Incremental assembly comparisons found no code-size regressions; the final comparison improved one Decimal128 method by nine bytes. - Changed-line formatting and git diff --check passed. Compilation replay and code size do not establish execution equivalence or throughput gains. No full assembly comparison against the parent was rerun for the final tree, and no ARM64 hardware timing or full ARM64 SuperPMI sweep was performed. --- src/coreclr/inc/clrconfigvalues.h | 1 + src/coreclr/inc/corinfoinstructionset.h | 8 + src/coreclr/inc/jiteeversionguid.h | 10 +- src/coreclr/inc/readytoruninstructionset.h | 1 + src/coreclr/jit/codegenarm64.cpp | 29 + src/coreclr/jit/codegenarmarch.cpp | 5 + src/coreclr/jit/codegenxarch.cpp | 92 +- src/coreclr/jit/compiler.h | 3 + src/coreclr/jit/emitarm64.cpp | 20 +- src/coreclr/jit/emitxarch.cpp | 67 +- src/coreclr/jit/emitxarch.h | 4 + src/coreclr/jit/gentree.cpp | 26 + src/coreclr/jit/gentree.h | 24 + src/coreclr/jit/gtlist.h | 15 + src/coreclr/jit/hwintrinsic.cpp | 1 + src/coreclr/jit/hwintrinsiccodegenxarch.cpp | 9 + src/coreclr/jit/inductionvariableopts.cpp | 304 +- src/coreclr/jit/instrsxarch.h | 7 + src/coreclr/jit/liveness.cpp | 12 +- src/coreclr/jit/lower.cpp | 3105 ++++++++++++++++- src/coreclr/jit/lower.h | 37 + src/coreclr/jit/lowerarmarch.cpp | 19 + src/coreclr/jit/lowerxarch.cpp | 177 + src/coreclr/jit/lsra.cpp | 56 + src/coreclr/jit/lsra.h | 4 + src/coreclr/jit/lsraarm64.cpp | 5 + src/coreclr/jit/lsrabuild.cpp | 80 + src/coreclr/jit/lsraxarch.cpp | 76 + .../Compiler/HardwareIntrinsicHelpers.cs | 4 + .../Runtime/ReadyToRunInstructionSet.cs | 1 + .../Runtime/ReadyToRunInstructionSetHelper.cs | 2 + .../JitInterface/CorInfoInstructionSet.cs | 6 + .../ThunkGenerator/InstructionSetDesc.txt | 6 +- src/coreclr/vm/codeman.cpp | 5 + .../netcore/TensorPrimitives.DivRem.cs | 12 +- .../src/System/Decimal.DecCalc.cs | 144 +- .../src/System/Int128.cs | 23 + .../src/System/Number.BigInteger.cs | 6 +- .../src/System/Number.DecimalIeee754.cs | 140 +- .../Numerics/BigIntegerCalculator.Shared.cs | 85 +- .../src/System/UInt128.cs | 24 +- .../src/System/UIntPtr.cs | 1 + .../Numerics/BigIntegerCalculator.PowMod.cs | 23 +- src/native/minipal/cpufeatures.c | 5 + src/native/minipal/cpufeatures.h | 1 + src/tests/JIT/opt/Add/ArithmeticCodegen.cs | 292 ++ .../JIT/opt/Add/ArithmeticCodegen.csproj | 12 + src/tests/JIT/opt/Add/Borrow.cs | 347 ++ src/tests/JIT/opt/Add/Borrow.csproj | 14 + src/tests/JIT/opt/Add/Carry.cs | 1099 ++++++ src/tests/JIT/opt/Add/Carry.csproj | 14 + src/tests/JIT/opt/Add/CarryChains.cs | 140 + src/tests/JIT/opt/Add/CarryChains.csproj | 12 + src/tests/JIT/opt/Add/DecimalMultiply.cs | 119 + src/tests/JIT/opt/Add/DecimalMultiply.csproj | 12 + src/tests/JIT/opt/Add/DecimalWide.cs | 90 + src/tests/JIT/opt/Add/DecimalWide.csproj | 12 + src/tests/JIT/opt/Add/Division.cs | 681 ++++ src/tests/JIT/opt/Add/Division.csproj | 14 + src/tests/JIT/opt/Add/FunnelShift.cs | 182 + src/tests/JIT/opt/Add/FunnelShift.csproj | 14 + src/tests/JIT/opt/Add/WideArithmetic.cs | 243 ++ src/tests/JIT/opt/Add/WideArithmetic.csproj | 14 + 63 files changed, 7863 insertions(+), 133 deletions(-) create mode 100644 src/tests/JIT/opt/Add/ArithmeticCodegen.cs create mode 100644 src/tests/JIT/opt/Add/ArithmeticCodegen.csproj create mode 100644 src/tests/JIT/opt/Add/Borrow.cs create mode 100644 src/tests/JIT/opt/Add/Borrow.csproj create mode 100644 src/tests/JIT/opt/Add/Carry.cs create mode 100644 src/tests/JIT/opt/Add/Carry.csproj create mode 100644 src/tests/JIT/opt/Add/CarryChains.cs create mode 100644 src/tests/JIT/opt/Add/CarryChains.csproj create mode 100644 src/tests/JIT/opt/Add/DecimalMultiply.cs create mode 100644 src/tests/JIT/opt/Add/DecimalMultiply.csproj create mode 100644 src/tests/JIT/opt/Add/DecimalWide.cs create mode 100644 src/tests/JIT/opt/Add/DecimalWide.csproj create mode 100644 src/tests/JIT/opt/Add/Division.cs create mode 100644 src/tests/JIT/opt/Add/Division.csproj create mode 100644 src/tests/JIT/opt/Add/FunnelShift.cs create mode 100644 src/tests/JIT/opt/Add/FunnelShift.csproj create mode 100644 src/tests/JIT/opt/Add/WideArithmetic.cs create mode 100644 src/tests/JIT/opt/Add/WideArithmetic.csproj diff --git a/src/coreclr/inc/clrconfigvalues.h b/src/coreclr/inc/clrconfigvalues.h index d6badce75704c0..89e70bc6edd97d 100644 --- a/src/coreclr/inc/clrconfigvalues.h +++ b/src/coreclr/inc/clrconfigvalues.h @@ -667,6 +667,7 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableHWIntrinsic, W("EnableHWIntri #if defined(TARGET_AMD64) || defined(TARGET_X86) RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVX, W("EnableAVX"), 1, "Allows AVX and dependent hardware intrinsics to be disabled") +RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableADX, W("EnableADX"), 1, "Allows ADX carry-chain instructions to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVX2, W("EnableAVX2"), 1, "Allows AVX2, BMI1, BMI2, F16C, FMA, LZCNT, MOVBE and dependent hardware intrinsics to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVX512, W("EnableAVX512"), 1, "Allows AVX512 F+BW+CD+DQ+VL and dependent hardware intrinsics to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVX512BMM, W("EnableAVX512BMM"), 1, "Allows AVX512BMM and dependent hardware intrinsics to be disabled") diff --git a/src/coreclr/inc/corinfoinstructionset.h b/src/coreclr/inc/corinfoinstructionset.h index 2f347572ee34d4..0e86fed61c92c0 100644 --- a/src/coreclr/inc/corinfoinstructionset.h +++ b/src/coreclr/inc/corinfoinstructionset.h @@ -118,6 +118,7 @@ enum CORINFO_InstructionSet InstructionSet_SHA_X64=44, InstructionSet_WAITPKG_X64=45, InstructionSet_X86Serialize_X64=46, + InstructionSet_ADX=47, #endif // TARGET_AMD64 #ifdef TARGET_X86 InstructionSet_X86Base=1, @@ -166,6 +167,7 @@ enum CORINFO_InstructionSet InstructionSet_SHA_X64=44, InstructionSet_WAITPKG_X64=45, InstructionSet_X86Serialize_X64=46, + InstructionSet_ADX=47, #endif // TARGET_X86 }; @@ -879,6 +881,8 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet) return "AVXVNNIINT"; case InstructionSet_AVXVNNIINT_V512 : return "AVXVNNIINT_V512"; + case InstructionSet_ADX : + return "ADX"; #endif // TARGET_AMD64 #ifdef TARGET_X86 case InstructionSet_X86Base : @@ -941,6 +945,8 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet) return "AVXVNNIINT"; case InstructionSet_AVXVNNIINT_V512 : return "AVXVNNIINT_V512"; + case InstructionSet_ADX : + return "ADX"; #endif // TARGET_X86 default: @@ -1063,6 +1069,7 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst case READYTORUN_INSTRUCTION_AvxVnniInt8_V512: return InstructionSet_AVXVNNIINT_V512; case READYTORUN_INSTRUCTION_AvxVnniInt16: return InstructionSet_AVXVNNIINT; case READYTORUN_INSTRUCTION_AvxVnniInt16_V512: return InstructionSet_AVXVNNIINT_V512; + case READYTORUN_INSTRUCTION_Adx: return InstructionSet_ADX; #endif // TARGET_AMD64 #ifdef TARGET_X86 case READYTORUN_INSTRUCTION_X86Base: return InstructionSet_X86Base; @@ -1133,6 +1140,7 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst case READYTORUN_INSTRUCTION_AvxVnniInt8_V512: return InstructionSet_AVXVNNIINT_V512; case READYTORUN_INSTRUCTION_AvxVnniInt16: return InstructionSet_AVXVNNIINT; case READYTORUN_INSTRUCTION_AvxVnniInt16_V512: return InstructionSet_AVXVNNIINT_V512; + case READYTORUN_INSTRUCTION_Adx: return InstructionSet_ADX; #endif // TARGET_X86 default: diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index 529831e7e5aada..847663cada9877 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -37,11 +37,11 @@ #include -constexpr GUID JITEEVersionIdentifier = { /* d3368fe9-d21b-4c5c-9786-49f591047bf7 */ - 0xd3368fe9, - 0xd21b, - 0x4c5c, - {0x97, 0x86, 0x49, 0xf5, 0x91, 0x04, 0x7b, 0xf7} +constexpr GUID JITEEVersionIdentifier = { /* 539ba29b-25d8-4b34-895a-01f513a3683a */ + 0x539ba29b, + 0x25d8, + 0x4b34, + {0x89, 0x5a, 0x01, 0xf5, 0x13, 0xa3, 0x68, 0x3a} }; #endif // JIT_EE_VERSIONING_GUID_H diff --git a/src/coreclr/inc/readytoruninstructionset.h b/src/coreclr/inc/readytoruninstructionset.h index d2851e91577f1e..ddcfb2af193445 100644 --- a/src/coreclr/inc/readytoruninstructionset.h +++ b/src/coreclr/inc/readytoruninstructionset.h @@ -103,6 +103,7 @@ enum ReadyToRunInstructionSet READYTORUN_INSTRUCTION_Cssc=93, READYTORUN_INSTRUCTION_Zicond=94, READYTORUN_INSTRUCTION_Fp16=95, + READYTORUN_INSTRUCTION_Adx=96, }; diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 588afa49dadfb5..ade39e4a2c434f 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -2691,6 +2691,25 @@ void CodeGen::genCodeForBinary(GenTreeOp* tree) var_types targetType = tree->TypeGet(); emitter* emit = GetEmitter(); + if (oper == GT_ADD_BORROW) + { + assert(!tree->gtSetFlags() && tree->gtGetOp2()->IsIntegralConst(0)); + emit->emitIns_R_R_COND(INS_cinc, emitActualTypeSize(tree), targetReg, tree->gtGetOp1()->GetRegNum(), + INS_COND_LO); + genProduceReg(tree); + return; + } + if (oper == GT_ADD_CARRY || oper == GT_SUB_BORROW) + { + instruction ins = oper == GT_SUB_BORROW ? (tree->gtSetFlags() ? INS_sbcs : INS_sbc) + : (tree->gtSetFlags() ? INS_adcs : INS_adc); + emit->emitIns_R_R_R(ins, emitActualTypeSize(tree), targetReg, + tree->gtGetOp1()->isContainedIntOrIImmed() ? REG_ZR : tree->gtGetOp1()->GetRegNum(), + tree->gtGetOp2()->isContainedIntOrIImmed() ? REG_ZR : tree->gtGetOp2()->GetRegNum()); + genProduceReg(tree); + return; + } + assert(tree->OperIs(GT_ADD, GT_SUB, GT_MUL, GT_DIV, GT_UDIV, GT_AND, GT_AND_NOT, GT_OR, GT_OR_NOT, GT_XOR, GT_XOR_NOT)); @@ -2700,6 +2719,16 @@ void CodeGen::genCodeForBinary(GenTreeOp* tree) // The arithmetic node must be sitting in a register (since it's not contained) assert(targetReg != REG_NA); + if (tree->IsFunnelShift()) + { + assert(!tree->gtSetFlags()); + emit->emitIns_R_R_R_I(INS_extr, emitTypeSize(tree), targetReg, op2->gtGetOp1()->GetRegNum(), + op1->gtGetOp1()->GetRegNum(), + static_cast(op1->gtGetOp2()->AsIntCon()->IconValue())); + genProduceReg(tree); + return; + } + // Handles combined operations: 'madd', 'msub' if (op2->OperIs(GT_MUL) && op2->isContained()) { diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 37cca461351f49..3e845f985f2b84 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -232,6 +232,11 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) case GT_SUB_HI: #endif // !defined(TARGET_64BIT) +#ifdef TARGET_ARM64 + case GT_SUB_BORROW: + case GT_ADD_BORROW: + case GT_ADD_CARRY: +#endif case GT_ADD: case GT_SUB: case GT_MUL: diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 36c4bcfd21f72f..59baa9c7deefd4 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -921,6 +921,7 @@ void CodeGen::genCodeForDivMod(GenTreeOp* treeNode) emit->emitInsBinary(ins, size, treeNode, divisor); // DIV/IDIV instructions always store the quotient in RAX and the remainder in RDX. + assert(!treeNode->IsDivRemPair() || (targetReg == REG_RAX)); // Move the result to the desired register, if necessary if (oper == GT_DIV || oper == GT_UDIV) { @@ -951,6 +952,9 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) { #ifdef DEBUG bool isValidOper = treeNode->OperIs(GT_ADD, GT_SUB); +#ifdef TARGET_AMD64 + isValidOper |= treeNode->OperIs(GT_ADD_CARRY, GT_SUB_BORROW, GT_ADD_BORROW, GT_ADCX, GT_ADOX); +#endif if (varTypeIsFloating(treeNode->TypeGet())) { isValidOper |= treeNode->OperIs(GT_MUL, GT_DIV); @@ -975,6 +979,18 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) GenTree* op1 = treeNode->gtGetOp1(); GenTree* op2 = treeNode->gtGetOp2(); + if (treeNode->IsFunnelShift()) + { + regNumber lo = op1->gtGetOp1()->GetRegNum(); + regNumber hi = op2->gtGetOp1()->GetRegNum(); + assert(targetReg != hi); + inst_Mov(targetType, targetReg, lo, /* canSkip */ true); + inst_RV_RV_IV(INS_shrd, emitTypeSize(targetType), targetReg, hi, + static_cast(op1->gtGetOp2()->AsIntCon()->IconValue())); + genProduceReg(treeNode); + return; + } + bool eligibleForNDD = false; // Commutative operations can mark op1 as contained or reg-optional to generate "op reg, memop/immed" @@ -1098,7 +1114,8 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) assert(!varTypeIsFloating(treeNode)); // try to use an inc or dec - if (oper == GT_ADD && src->isContainedIntOrIImmed() && !treeNode->gtOverflowEx()) + if (oper == GT_ADD && src->isContainedIntOrIImmed() && !treeNode->gtOverflowEx() && + ((treeNode->gtFlags & GTF_ADD_CARRY_FLAGS) == 0)) { if (src->IsIntegralConst(1)) { @@ -1940,6 +1957,13 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) case GT_SUB_HI: #endif // !defined(TARGET_64BIT) +#ifdef TARGET_AMD64 + case GT_SUB_BORROW: + case GT_ADD_BORROW: + case GT_ADD_CARRY: + case GT_ADCX: + case GT_ADOX: +#endif case GT_ADD: case GT_SUB: genCodeForBinary(treeNode->AsOp()); @@ -2080,6 +2104,58 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) genCodeForJTrue(treeNode->AsOp()); break; +#ifdef TARGET_AMD64 + case GT_ADX_SEED: + { + // TEST clears CF and OF without a temporary register or changing SP. + GetEmitter()->emitIns_R_R(INS_test, EA_4BYTE, REG_ESP, REG_ESP); + break; + } + case GT_ADX_DRAIN: + { + genConsumeOperands(treeNode->AsOp()); + regNumber target = treeNode->GetRegNum(); + regNumber zero = internalRegisters.GetSingle(treeNode); + inst_Mov(TYP_LONG, target, treeNode->gtGetOp1()->GetRegNum(), true); + // A MOV is required: XOR would destroy the carries being drained. + GetEmitter()->emitIns_R_I(INS_mov, EA_4BYTE, zero, 0); + GetEmitter()->emitIns_R_R(INS_adcx, EA_8BYTE, target, zero); + GetEmitter()->emitIns_R_R(INS_adox, EA_8BYTE, target, zero); + genProduceReg(treeNode); + break; + } + case GT_JCMP: + { + genConsumeOperands(treeNode->AsOp()); + GenTree* count = treeNode->gtGetOp1(); + // Enregistered local uses can retain their source register here. + // Copy the count to RCX when necessary. The countdown's 32-bit LEA + // (or its 32-bit reload/copy) has already cleared the upper half when + // this move is skipped; bare JRCXZ tests all of RCX, not just ECX. + GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(count), REG_ECX, count->GetRegNum(), true); + assert(treeNode->gtGetOp2()->IsIntegralConst(0) && treeNode->gtGetOp2()->isContained()); + // JRCXZ preserves both carry flags but has only a rel8 encoding. + // Skip a JMP to reach the loop: two branch instructions per backedge. + GenCondition::Code condition = treeNode->AsOpCC()->gtCondition.GetCode(); + assert(condition == GenCondition::EQ || condition == GenCondition::NE); + BasicBlock* done = genCreateTempLabel(); + if (condition == GenCondition::NE) + { + GetEmitter()->emitIns_J(INS_jrcxz, done, true); + GetEmitter()->emitIns_J(INS_jmp, m_compiler->compCurBB->GetTrueTarget()); + } + else + { + BasicBlock* taken = genCreateTempLabel(); + GetEmitter()->emitIns_J(INS_jrcxz, taken, true); + GetEmitter()->emitIns_J(INS_jmp, done, true); + genDefineTempLabel(taken); + GetEmitter()->emitIns_J(INS_jmp, m_compiler->compCurBB->GetTrueTarget()); + } + genDefineTempLabel(done); + break; + } +#endif case GT_JCC: genCodeForJcc(treeNode->AsCC()); break; @@ -4496,6 +4572,7 @@ void CodeGen::genRangeCheck(GenTree* oper) void CodeGen::genCodeForPhysReg(GenTreePhysReg* tree) { assert(tree->OperIs(GT_PHYSREG)); + assert(!tree->IsDivRemPair() || ((tree->GetRegNum() == REG_RDX) && (tree->gtSrcReg == REG_RDX))); var_types targetType = tree->TypeGet(); regNumber targetReg = tree->GetRegNum(); @@ -4537,6 +4614,19 @@ instruction CodeGen::genGetInsForOper(genTreeOps oper, var_types type) switch (oper) { +#ifdef TARGET_AMD64 + case GT_ADCX: + return INS_adcx; + case GT_ADOX: + return INS_adox; + case GT_SUB_BORROW: + ins = INS_sbb; + break; + case GT_ADD_BORROW: + case GT_ADD_CARRY: + ins = INS_adc; + break; +#endif case GT_ADD: ins = INS_add; break; diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 5e0dbba6988618..7e96f04429a98d 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -8350,6 +8350,9 @@ class Compiler bool optMakeLoopDownwardsCounted(ScalarEvolutionContext& scevContext, FlowGraphNaturalLoop* loop, PerLoopInfo* loopLocals); +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + void optFoldFourLimbOffsets(ScalarEvolutionContext& scevContext, BasicBlock* block); +#endif bool optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevContext, FlowGraphNaturalLoop* loop, BasicBlock* exiting, diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index df354db94fe56e..ecf21930c94ae2 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -516,7 +516,9 @@ void emitter::emitInsSanityCheck(instrDesc* id) } else { - assert(isGeneralRegister(id->idReg3())); + assert(isGeneralRegister(id->idReg3()) || (((id->idIns() == INS_adc) || (id->idIns() == INS_adcs) || + (id->idIns() == INS_sbc) || (id->idIns() == INS_sbcs)) && + (id->idReg3() == REG_ZR))); } assert(insOptsNone(id->idInsOpt())); break; @@ -6138,10 +6140,6 @@ void emitter::emitIns_R_R_R(instruction ins, case INS_lsr: case INS_asr: case INS_ror: - case INS_adc: - case INS_adcs: - case INS_sbc: - case INS_sbcs: case INS_udiv: case INS_sdiv: case INS_mneg: @@ -6174,6 +6172,18 @@ void emitter::emitIns_R_R_R(instruction ins, fmt = IF_DR_3A; break; + case INS_adc: + case INS_adcs: + case INS_sbc: + case INS_sbcs: + assert(insOptsNone(opt)); + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegisterOrZR(reg1)); + assert(isGeneralRegisterOrZR(reg2)); + assert(isGeneralRegisterOrZR(reg3)); + fmt = IF_DR_3A; + break; + case INS_add: case INS_sub: if (isVectorRegister(reg1)) diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index fa33e803f4273d..2241d85af3c2f1 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -2691,7 +2691,7 @@ bool emitter::TakesRexWPrefix(const instrDesc* id) const // so we never need it if ((ins != INS_push) && (ins != INS_pop) && (ins != INS_movq) && (ins != INS_movzx) && (ins != INS_push_hide) && (ins != INS_pop_hide) && (ins != INS_ret) && (ins != INS_call) && (ins != INS_tail_i_jmp) && - !((ins >= INS_i_jmp) && (ins <= INS_l_jg))) + (ins != INS_jrcxz) && !((ins >= INS_i_jmp) && (ins <= INS_l_jg))) { return true; } @@ -3912,9 +3912,9 @@ unsigned emitter::emitGetAdjustedSize(instrDesc* id, code_t code) const #endif // TARGET_AMD64 else { - if (ins == INS_crc32) + if (IsLegacyMap38Instruction(ins)) { - // Adjust code size for CRC32 that has 4-byte opcode but does not use SSE38 or EES3A encoding. + // Account for the mandatory prefix of legacy map-38 instructions. adjustedSize++; } @@ -5269,7 +5269,10 @@ inline UNATIVE_OFFSET emitter::emitInsSizeRR(instrDesc* id) if ((code & 0xFF00) != 0) { - sz += (IsSimdInstruction(ins) || TakesEvexPrefix(id)) ? emitInsSize(id, code, includeRexPrefixSize) : 5; + // ADX uses a four-byte opcode encoding; its mandatory prefix was counted above. + sz += (IsSimdInstruction(ins) || (ins == INS_adcx) || (ins == INS_adox) || TakesEvexPrefix(id)) + ? emitInsSize(id, code, includeRexPrefixSize) + : 5; } else { @@ -10846,6 +10849,12 @@ void emitter::emitIns_J(instruction ins, BasicBlock* dst, bool keepShort, bool i #endif } + if (ins == INS_jrcxz) + { + // Only used to skip one unconditional jump to a nearby temporary label. + noway_assert(keepShort && !id->idjKeepLong); + sz = JCC_SIZE_SMALL; + } id->idCodeSize(sz); dispIns(id); @@ -14209,7 +14218,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) } // Special case emitting AVX instructions - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if ((ins == INS_crc32) && (size > EA_1BYTE)) { @@ -14251,7 +14260,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) dst += emitOutputRexOrSimdPrefixIfNeeded(ins, dst, code); - if (UseSimdEncoding() && (ins != INS_crc32)) + if (UseSimdEncoding() && !IsLegacyMap38Instruction(ins)) { // Emit last opcode byte // TODO-XArch-CQ: Right now support 4-byte opcode instructions only @@ -14466,7 +14475,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) // The address is of the form "[disp]" // On x86 - disp is relative to zero // On Amd64 - disp is relative to RIP - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { dst += emitOutputByte(dst, code | 0x05); } @@ -14529,7 +14538,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) else { #ifdef TARGET_X86 - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { dst += emitOutputByte(dst, code | 0x05); } @@ -14546,7 +14555,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) noway_assert((int)dsp == dsp); // This requires, specifying a SIB byte after ModRM byte. - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { dst += emitOutputByte(dst, code | 0x04); } @@ -14563,7 +14572,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) case REG_EBP: { - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspInByte) { @@ -14605,7 +14614,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) case REG_ESP: { - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspIsZero) { @@ -14663,7 +14672,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) default: { - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { // Put the register in the opcode code |= insEncodeReg012(id, reg, EA_PTRSIZE, nullptr); @@ -14742,7 +14751,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) regByte = insEncodeReg012(id, reg, EA_PTRSIZE, nullptr) | insEncodeReg345(id, rgx, EA_PTRSIZE, nullptr) | insSSval(mul); - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspIsZero && (reg != REG_EBP)) { @@ -14802,7 +14811,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) regByte = insEncodeReg012(id, REG_EBP, EA_PTRSIZE, nullptr) | insEncodeReg345(id, rgx, EA_PTRSIZE, nullptr) | insSSval(mul); - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { dst += emitOutputByte(dst, code | 0x04); } @@ -14831,7 +14840,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) // The address is "[reg+rgx+dsp]" regByte = insEncodeReg012(id, reg, EA_PTRSIZE, nullptr) | insEncodeReg345(id, rgx, EA_PTRSIZE, nullptr); - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspIsZero && (reg != REG_EBP)) { @@ -15101,7 +15110,7 @@ BYTE* emitter::emitOutputSV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) } // Special case emitting AVX instructions - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if ((ins == INS_crc32) && (size > EA_1BYTE)) { @@ -15130,7 +15139,7 @@ BYTE* emitter::emitOutputSV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) dst += emitOutputRexOrSimdPrefixIfNeeded(ins, dst, code); - if (UseSimdEncoding() && (ins != INS_crc32)) + if (UseSimdEncoding() && !IsLegacyMap38Instruction(ins)) { // Emit last opcode byte // TODO-XArch-CQ: Right now support 4-byte opcode instructions only @@ -15333,7 +15342,7 @@ BYTE* emitter::emitOutputSV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) if (EBPbased) { // EBP-based variable: does the offset fit in a byte? - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspInByte) { @@ -15384,7 +15393,7 @@ BYTE* emitter::emitOutputSV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) #endif // !FEATURE_FIXED_OUT_ARGS // Does the offset fit in a byte? - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspIsZero) { @@ -15673,7 +15682,7 @@ BYTE* emitter::emitOutputCV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) #endif // TARGET_X86 // Special case emitting AVX instructions - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if ((ins == INS_crc32) && (size > EA_1BYTE)) { @@ -15702,7 +15711,7 @@ BYTE* emitter::emitOutputCV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) dst += emitOutputRexOrSimdPrefixIfNeeded(ins, dst, code); - if (UseVEXEncoding() && (ins != INS_crc32)) + if (UseVEXEncoding() && !IsLegacyMap38Instruction(ins)) { // Emit last opcode byte // TODO-XArch-CQ: Right now support 4-byte opcode instructions only @@ -16463,8 +16472,8 @@ BYTE* emitter::emitOutputRR(BYTE* dst, instrDesc* id) #endif // TARGET_AMD64 } #ifdef FEATURE_HW_INTRINSICS - else if ((ins == INS_bsf) || (ins == INS_bsr) || (ins == INS_crc32) || (ins == INS_lzcnt) || (ins == INS_popcnt) || - (ins == INS_tzcnt) + else if ((ins == INS_bsf) || (ins == INS_bsr) || IsLegacyMap38Instruction(ins) || (ins == INS_lzcnt) || + (ins == INS_popcnt) || (ins == INS_tzcnt) #ifdef TARGET_AMD64 || (ins == INS_lzcnt_apx) || (ins == INS_tzcnt_apx) || (ins == INS_popcnt_apx) || (ins == INS_crc32_apx) #endif // TARGET_AMD64 @@ -17403,6 +17412,12 @@ BYTE* emitter::emitOutputLJ(insGroup* ig, BYTE* dst, instrDesc* i) jmp = true; break; + case INS_jrcxz: + noway_assert(id->idjShort && !id->idjKeepLong); + ssz = lsz = JCC_SIZE_SMALL; + jmp = true; + break; + case INS_jmp: ssz = JMP_SIZE_SMALL; lsz = JMP_SIZE_LARGE; @@ -18941,7 +18956,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) break; } - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { // Special case 4-byte AVX instructions as the // regcode position conflicts with the opcode byte @@ -19180,7 +19195,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) { code = insCodeRM(ins); - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { // Special case 4-byte AVX instructions as the // regcode position conflicts with the opcode byte @@ -19436,7 +19451,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) { code = insCodeRM(ins); - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { // Special case 4-byte AVX instructions as the // regcode position conflicts with the opcode byte diff --git a/src/coreclr/jit/emitxarch.h b/src/coreclr/jit/emitxarch.h index 51c0cca338f005..eaa1dfb110b2ff 100644 --- a/src/coreclr/jit/emitxarch.h +++ b/src/coreclr/jit/emitxarch.h @@ -116,6 +116,10 @@ code_t insEncodeOpreg(const instrDesc* id, regNumber reg, emitAttr size); unsigned insSSval(unsigned scale); static bool IsSSEInstruction(instruction ins); +static bool IsLegacyMap38Instruction(instruction ins) +{ + return ins == INS_crc32 || ins == INS_adcx || ins == INS_adox; +} static bool IsSSEOrAVXInstruction(instruction ins); static bool IsAvx512OnlyInstruction(instruction ins); static bool IsKMOVInstruction(instruction ins); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 3700c917e7d2e8..88918dcd510180 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -12008,6 +12008,22 @@ void Compiler::gtUpdateNodeSideEffects(GenTree* tree) }); } +//------------------------------------------------------------------------ +// IsFunnelShift: Whether lowering contained two complementary shifts in an OR. +// +// Return Value: +// True for the canonical (lo >>> count) | (hi << (width - count)) form. +// +bool GenTree::IsFunnelShift() const +{ +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + return OperIs(GT_OR) && gtGetOp1()->OperIs(GT_RSZ) && gtGetOp1()->isContained() && + gtGetOp2()->OperIs(GT_LSH) && gtGetOp2()->isContained(); +#else + return false; +#endif +} + bool GenTree::gtSetFlags() const { return (gtFlags & GTF_SET_FLAGS) != 0; @@ -12061,6 +12077,9 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) case GT_JMP: case GT_JCC: case GT_SETCC: +#ifdef TARGET_AMD64 + case GT_ADX_SEED: +#endif case GT_NO_OP: case GT_START_NONGC: case GT_START_PREEMPTGC: @@ -12083,6 +12102,9 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) // Standard unary operators case GT_STORE_LCL_VAR: +#ifdef TARGET_AMD64 + case GT_ADX_DRAIN: +#endif case GT_STORE_LCL_FLD: case GT_NOT: case GT_NEG: @@ -14294,6 +14316,10 @@ void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack) } break; +#ifdef TARGET_AMD64 + case GT_ADX_SEED: + break; +#endif case GT_PHYSREG: printf(" %s", getRegName(tree->AsPhysReg()->gtSrcReg)); break; diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 60477341da53e0..5ace23683f7519 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -498,6 +498,7 @@ enum GenTreeFlags : unsigned GTF_ADDRMODE_NO_CSE = 0x80000000, // GT_ADD/GT_MUL/GT_LSH/GT_CAST -- Do not CSE this node only, forms complex // addressing mode + GTF_ADD_CARRY_FLAGS = 0x04000000, // GT_ADD -- CF is consumed; INC/DEC cannot implement this add GTF_MUL_64RSLT = 0x40000000, // GT_MUL -- produce 64-bit result GTF_RELOP_NAN_UN = 0x80000000, // GT_ -- Is branch taken if ops are NaN? @@ -528,6 +529,8 @@ enum GenTreeFlags : unsigned GTF_DIV_MOD_NO_OVERFLOW = 0x40000000, // GT_DIV, GT_MOD -- Div or mod definitely does not overflow. + GTF_DIV_REM_PAIR = 0x08000000, // GT_[U]DIV, GT_PHYSREG -- adjacent quotient/remainder definitions on xarch. + GTF_ARRLEN_NONFAULTING = 0x20000000, // GT_ARR_LENGTH -- An array length operation that cannot fault. Same as GT_IND_NONFAULTING. GTF_MDARRLEN_NONFAULTING = 0x20000000, // GT_MDARR_LENGTH -- An MD array length operation that cannot fault. Same as GT_IND_NONFAULTING. @@ -537,6 +540,7 @@ enum GenTreeFlags : unsigned GTF_ALLOCOBJ_EMPTY_STATIC = 0x80000000, // GT_ALLOCOBJ -- allocation site is part of an empty static pattern #ifdef FEATURE_HW_INTRINSICS + GTF_HW_MULX = 0x08000000, // GT_HWINTRINSIC -- unsigned BigMul in a matched multiply-carry chain uses MULX GTF_HW_EM_OP = 0x10000000, // GT_HWINTRINSIC -- node is used as an operand to an embedded mask GTF_HW_USER_CALL = 0x20000000, // GT_HWINTRINSIC -- node is implemented via a user call #endif // FEATURE_HW_INTRINSICS @@ -1861,6 +1865,18 @@ struct GenTree bool OperConsumesFlags() const { +#ifdef TARGET_AMD64 + if (OperIs(GT_ADCX, GT_ADOX, GT_ADX_DRAIN)) + { + return true; + } +#endif +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + if (OperIs(GT_ADD_CARRY, GT_SUB_BORROW, GT_ADD_BORROW)) + { + return true; + } +#endif #if !defined(TARGET_64BIT) if (OperIs(GT_ADD_HI, GT_SUB_HI)) { @@ -2466,6 +2482,12 @@ struct GenTree bool gtOverflow() const; bool gtOverflowEx() const; bool gtSetFlags() const; + bool IsFunnelShift() const; + + bool IsDivRemPair() const + { + return OperIs(GT_DIV, GT_UDIV, GT_PHYSREG) && ((gtFlags & GTF_DIV_REM_PAIR) != 0); + } #ifdef DEBUG static int gtDispFlags(GenTreeFlags flags, GenTreeDebugFlags debugFlags); @@ -9528,6 +9550,8 @@ struct GenTreeOpCC : public GenTreeOp { #ifdef TARGET_ARM64 assert(OperIs(GT_SELECTCC, GT_SELECT_INCCC, GT_SELECT_INVCC, GT_SELECT_NEGCC)); +#elif defined(TARGET_AMD64) + assert(OperIs(GT_SELECTCC, GT_JCMP)); #else assert(OperIs(GT_SELECTCC)); #endif diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index 622a9643e7db0b..9f638d92665198 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -391,6 +391,21 @@ GTNODE(RELOAD , GenTreeCopyOrReload,0,0,GTK_UNOP|DBK_NOTHIR) GTNODE(IL_OFFSET , GenTreeILOffset ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTHIR) // marks an IL offset for debugging purposes GTNODE(RECORD_ASYNC_RESUME, GenTreeVal ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTHIR) // record native offset for async resumption info +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) +#ifdef TARGET_AMD64 +// Independent unsigned carry chains, introduced only after whole-loop validation. +GTNODE(ADCX , GenTreeOp ,1,0,GTK_BINOP|DBK_NOTHIR) +GTNODE(ADOX , GenTreeOp ,1,0,GTK_BINOP|DBK_NOTHIR) +GTNODE(ADX_SEED , GenTree ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTHIR) +GTNODE(ADX_DRAIN , GenTreeOp ,0,0,GTK_UNOP|DBK_NOTHIR) +#endif +// Adds two operands and the carry flag. Only introduced during lowering. +GTNODE(ADD_CARRY , GenTreeOp ,1,0,GTK_BINOP|DBK_NOTHIR) +// Subtract op2 and borrow; add op2 and subtraction borrow to op1 (op2 is zero on arm64). +GTNODE(SUB_BORROW , GenTreeOp ,0,0,GTK_BINOP|DBK_NOTHIR) +GTNODE(ADD_BORROW , GenTreeOp ,0,0,GTK_BINOP|DBK_NOTHIR) +#endif + /*****************************************************************************/ #undef GTNODE /*****************************************************************************/ diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index d84f1dfa732107..d1150a9d1e97b6 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -973,6 +973,7 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = { { NI_Illegal, NI_Illegal }, // SHA_X64 { NI_Illegal, NI_Illegal }, // WAITPKG_X64 { NI_Illegal, NI_Illegal }, // X86Serialize_X64 + { NI_Illegal, NI_Illegal }, // ADX #elif defined (TARGET_ARM64) { FIRST_NI_ArmBase, LAST_NI_ArmBase }, // ArmBase { FIRST_NI_AdvSimd, LAST_NI_AdvSimd }, // AdvSimd diff --git a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp index a2362695a6716b..f0a4ce975da303 100644 --- a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp @@ -2596,6 +2596,15 @@ void CodeGen::genX86BaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions) emitAttr attr = emitTypeSize(baseType); + if ((node->gtFlags & GTF_HW_MULX) != 0) + { + assert(!rmOp->isUsedFromReg() || (rmOp->GetRegNum() != REG_EDX) || (regOp->GetRegNum() == REG_EDX)); + emit->emitIns_Mov(INS_mov, attr, REG_EDX, regOp->GetRegNum(), /* canSkip */ true); + inst_RV_RV_TT(INS_mulx, attr, node->GetRegByIndex(1), node->GetRegByIndex(0), rmOp, + /* isRMW */ false, INS_OPTS_NONE); + break; + } + // If rmOp is already in EAX, use that as implicit operand if (rmOp->isUsedFromReg() && rmOp->GetRegNum() == REG_EAX) { diff --git a/src/coreclr/jit/inductionvariableopts.cpp b/src/coreclr/jit/inductionvariableopts.cpp index cbc9751324ebb3..55cb41c14d643e 100644 --- a/src/coreclr/jit/inductionvariableopts.cpp +++ b/src/coreclr/jit/inductionvariableopts.cpp @@ -1187,6 +1187,65 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte } bool checkProfitability = !compStressCompile(STRESS_DOWNWARDS_COUNTED_LOOPS, 50); +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + bool preferCarryCountdown = false; + // A countdown can keep carry or borrow flags live across arithmetic loops, + // even when their addressing IV remains useful after the loop. + // Leave trip-count legality to SCEV below and the carry proof to lowering. + if (checkProfitability && removableLocals.Height() == 0 && compHndBBtabCount == 0 && loop->GetHeader() == exiting && + exiting->GetTrueTarget() == exiting) + { + unsigned products = 0; + unsigned carries = 0; + unsigned nodes = 0; + unsigned differences = 0; + for (Statement* stmt : exiting->Statements()) + { + for (GenTree* node : stmt->TreeList()) + { + if (++nodes > 512) + { + break; + } + if (node->OperIs(GT_HWINTRINSIC) && + node->AsHWIntrinsic()->GetHWIntrinsicId() == +#ifdef TARGET_AMD64 + NI_X86Base_X64_BigMul && +#else + NI_ArmBase_Arm64_MultiplyHigh && +#endif + node->AsHWIntrinsic()->GetSimdBaseType() == TYP_ULONG) + { + products++; + } + if (node->OperIs(GT_LT, GT_GT, GT_GE, GT_LE) && (node->gtFlags & GTF_UNSIGNED) != 0) + { + carries++; + } + if (node->OperIs(GT_SUB) && !node->gtOverflow() && node->TypeIs(TYP_INT, TYP_LONG)) + { + differences++; + } + } + if (nodes > 512) + { + break; + } + } + bool multiply = products >= 2 && products <= 4 && carries == 2 * products; +#ifdef TARGET_AMD64 + multiply = multiply && compOpportunisticallyDependsOn(InstructionSet_ADX) && + compOpportunisticallyDependsOn(InstructionSet_AVX2); +#endif + bool subtract = products == 0 && differences == 2 && carries == 2; + if (nodes <= 512 && (multiply || subtract)) + { + JITDUMP(" Keeping a countdown for a potential carry/borrow loop\n"); + checkProfitability = false; + preferCarryCountdown = true; + } + } +#endif if (checkProfitability && (removableLocals.Height() <= 0)) { JITDUMP(" Found no potentially removable locals when making this loop downwards counted\n"); @@ -1199,8 +1258,65 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte return false; } - Scev* backedgeCount = scevContext.ComputeExitNotTakenCount(exiting); - if (backedgeCount == nullptr) + Scev* backedgeCount = scevContext.ComputeExitNotTakenCount(exiting); + GenTree* tripCountNode = nullptr; +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + // SCEV's general trip-count materialization currently handles only unit + // strides. Recognize the bounded four-lane case: i starts at zero, steps + // by four, and continues while i < length - 3 for a nonnegative length. + // Its trip count is length / 4; the induction variable cannot overflow. + if (backedgeCount == nullptr && preferCarryCountdown && cond->OperIs(GT_LT) && !cond->IsUnsigned() && + cond->gtGetOp1()->TypeIs(TYP_INT) && cond->gtGetOp2()->TypeIs(TYP_INT)) + { + Scev* iv = scevContext.Analyze(exiting, cond->gtGetOp1()); + if (iv != nullptr) + { + iv = scevContext.Simplify(iv); + } + Scev* bound = scevContext.Analyze(exiting, cond->gtGetOp2()); + int64_t start, step; + if (iv != nullptr && iv->OperIs(ScevOper::AddRec) && bound != nullptr && bound->IsInvariant() && + ((ScevAddRec*)iv)->Start->GetConstantValue(this, &start) && start == 4 && + ((ScevAddRec*)iv)->Step->GetConstantValue(this, &step) && step == 4) + { + auto resolve = [this](GenTree* node) { + for (unsigned depth = 0; depth < 8 && node->OperIs(GT_LCL_VAR); depth++) + { + GenTreeLclVar* local = node->AsLclVar(); + if (!local->HasSsaName()) + { + break; + } + GenTreeLclVarCommon* def = lvaGetDesc(local)->GetPerSsaData(local->GetSsaNum())->GetDefNode(); + if (def == nullptr || def->GetLclNum() != local->GetLclNum()) + { + break; + } + node = def->Data(); + } + return node; + }; + GenTree* limit = resolve(cond->gtGetOp2()); + if (limit->OperIs(GT_ADD) && limit->gtGetOp2()->IsIntegralConst(-3) && + IntegralRange::ForNode(resolve(limit->gtGetOp1()), this).IsNonNegative()) + { + ValueNum limitVN = vnStore->VNLiberalNormalValue(cond->gtGetOp2()->gtVNPair); + ValueNum positive = vnStore->VNForFunc(TYP_INT, VNF_GT, limitVN, vnStore->VNForIntCon(0)); + if (scevContext.EvaluateRelop(positive) == RelopEvaluationResult::True) + { + GenTree* limitNode = scevContext.Materialize(bound); + if (limitNode != nullptr) + { + GenTree* length = gtNewOperNode(GT_ADD, TYP_INT, limitNode, gtNewIconNode(3)); + tripCountNode = gtNewOperNode(GT_RSZ, TYP_INT, length, gtNewIconNode(2)); + optFoldFourLimbOffsets(scevContext, exiting); + } + } + } + } + } +#endif + if (backedgeCount == nullptr && tripCountNode == nullptr) { JITDUMP(" Could not compute backedge count -- not a counted loop\n"); return false; @@ -1213,9 +1329,12 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte // to add one to the computed backedge count, giving us the trip count of // the loop. We do not need to worry about overflow here (even with // wraparound we have the right behavior). - Scev* tripCount = scevContext.Simplify( - scevContext.NewBinop(ScevOper::Add, backedgeCount, scevContext.NewConstant(backedgeCount->Type, 1))); - GenTree* tripCountNode = scevContext.Materialize(tripCount); + if (tripCountNode == nullptr) + { + Scev* tripCount = scevContext.Simplify( + scevContext.NewBinop(ScevOper::Add, backedgeCount, scevContext.NewConstant(backedgeCount->Type, 1))); + tripCountNode = scevContext.Materialize(tripCount); + } if (tripCountNode == nullptr) { JITDUMP(" Could not materialize trip count into IR\n"); @@ -1239,9 +1358,9 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte exitOp = GT_NE; } - GenTree* negOne = tripCount->TypeIs(TYP_LONG) ? gtNewLconNode(-1) : gtNewIconNode(-1, tripCount->Type); - GenTree* decremented = - gtNewOperNode(GT_ADD, tripCount->Type, gtNewLclVarNode(tripCountLcl, tripCount->Type), negOne); + var_types countType = tripCountNode->TypeGet(); + GenTree* negOne = countType == TYP_LONG ? gtNewLconNode(-1) : gtNewIconNode(-1, countType); + GenTree* decremented = gtNewOperNode(GT_ADD, countType, gtNewLclVarNode(tripCountLcl, countType), negOne); store = gtNewTempStore(tripCountLcl, decremented); @@ -1253,8 +1372,8 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte // Update the test. cond->SetOper(exitOp); - cond->AsOp()->gtOp1 = gtNewLclVarNode(tripCountLcl, tripCount->Type); - cond->AsOp()->gtOp2 = gtNewZeroConNode(tripCount->Type); + cond->AsOp()->gtOp1 = gtNewLclVarNode(tripCountLcl, countType); + cond->AsOp()->gtOp2 = gtNewZeroConNode(countType); gtSetStmtInfo(jtrueStmt); fgSetStmtSeq(jtrueStmt); @@ -1267,6 +1386,171 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte return true; } +//------------------------------------------------------------------------ +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) +// optFoldFourLimbOffsets: Share a widened integer index among the lane addresses +// of a proven four-limb loop. The caller proves i starts at zero, steps by four, +// and executes only while i < length - 3, for a nonnegative signed int length. +// Thus i + k cannot wrap for k in [0, 3]. Managed bases remain unchanged. +void Compiler::optFoldFourLimbOffsets(ScalarEvolutionContext& scevContext, BasicBlock* block) +{ + auto resolve = [this](GenTree* node) { + for (unsigned depth = 0; depth < 8; depth++) + { + if (node->OperIs(GT_COMMA)) + { + node = node->gtGetOp2(); + continue; + } + if (!node->OperIs(GT_LCL_VAR)) + { + break; + } + GenTreeLclVar* local = node->AsLclVar(); + if (!local->HasSsaName()) + { + break; + } + GenTreeLclVarCommon* def = lvaGetDesc(local)->GetPerSsaData(local->GetSsaNum())->GetDefNode(); + if ((def == nullptr) || (def->GetLclNum() != local->GetLclNum()) || def->Data()->OperIs(GT_PHI)) + { + break; + } + node = def->Data(); + } + return node; + }; + + struct Address + { + GenTreeOp* Node; + Statement* Stmt; + unsigned Offset; + }; + jitstd::vector
addresses(getAllocator(CMK_LoopOpt)); + jitstd::vector stores(getAllocator(CMK_LoopOpt)); + GenTreeLclVar* index = nullptr; + Statement* firstStmt = nullptr; + for (Statement* stmt : block->Statements()) + { + if (stmt->IsPhiDefnStmt()) + { + continue; + } + if (firstStmt == nullptr) + { + firstStmt = stmt; + } + for (GenTree* node : stmt->TreeList()) + { + if (node->OperIsLocalStore()) + { + stores.push_back(node->AsLclVarCommon()->GetLclNum()); + } + if (!node->OperIs(GT_ADD) || !node->TypeIs(TYP_BYREF) || !node->gtGetOp1()->TypeIs(TYP_BYREF)) + { + continue; + } + GenTree* scaled = resolve(node->gtGetOp2()); + if (!scaled->OperIs(GT_LSH) || !scaled->TypeIs(TYP_LONG) || !scaled->gtGetOp2()->IsIntegralConst(3)) + { + continue; + } + GenTree* widened = resolve(scaled->gtGetOp1()); + if (!widened->OperIs(GT_CAST) || !widened->TypeIs(TYP_LONG) || !widened->IsUnsigned() || + widened->gtOverflow()) + { + continue; + } + GenTree* base = resolve(widened->AsCast()->CastOp()); + unsigned offset = 0; + if (base->OperIs(GT_ADD) && base->TypeIs(TYP_INT) && !base->gtOverflow() && base->gtGetOp2()->IsCnsIntOrI()) + { + ssize_t value = base->gtGetOp2()->AsIntCon()->IconValue(); + if ((value < 0) || (value > 3)) + { + continue; + } + offset = (unsigned)value; + base = base->gtGetOp1(); + } + if (!base->OperIs(GT_LCL_VAR) || !base->TypeIs(TYP_INT) || !base->AsLclVar()->HasSsaName()) + { + continue; + } + GenTreeLclVar* local = base->AsLclVar(); + if (lvaGetDesc(local)->IsAddressExposed() || lvaGetDesc(local)->lvIsStructField) + { + continue; + } + bool overwritten = false; + for (unsigned store : stores) + { + overwritten |= store == local->GetLclNum(); + } + if (overwritten || ((index != nullptr) && ((index->GetLclNum() != local->GetLclNum()) || + (index->GetSsaNum() != local->GetSsaNum())))) + { + continue; + } + Scev* recurrence = scevContext.Analyze(block, local); + int64_t start, step; + if ((recurrence == nullptr) || !recurrence->OperIs(ScevOper::AddRec) || + !((ScevAddRec*)recurrence)->Start->GetConstantValue(this, &start) || (start != 0) || + !((ScevAddRec*)recurrence)->Step->GetConstantValue(this, &step) || (step != 4)) + { + continue; + } + index = local; + addresses.push_back({node->AsOp(), stmt, offset}); + } + } + if (addresses.size() < 4) + { + return; + } + + unsigned temp = lvaGrabTemp(false DEBUGARG("Shared four-limb integer index")); + GenTree* widened = gtNewCastNode(TYP_LONG, gtCloneExpr(index), true, TYP_LONG); + Statement* initialization = fgNewStmtFromTree(gtNewTempStore(temp, widened)); + fgInsertStmtBefore(block, firstStmt, initialization); + gtSetStmtInfo(initialization); + fgSetStmtSeq(initialization); +#ifdef TARGET_ARM64 + // A64 has no base+index+displacement addressing mode. Share native-sized + // lane indices instead, avoiding a separate zero extension after each i+k. + unsigned laneTemps[4] = {temp, BAD_VAR_NUM, BAD_VAR_NUM, BAD_VAR_NUM}; + for (unsigned lane = 1; lane < 4; lane++) + { + laneTemps[lane] = lvaGrabTemp(false DEBUGARG("Four-limb lane index")); + GenTree* value = gtNewOperNode(GT_ADD, TYP_LONG, gtNewLclvNode(temp, TYP_LONG), gtNewLconNode(lane)); + Statement* init = fgNewStmtFromTree(gtNewTempStore(laneTemps[lane], value)); + fgInsertStmtBefore(block, firstStmt, init); + gtSetStmtInfo(init); + fgSetStmtSeq(init); + } +#endif + for (const Address& address : addresses) + { +#ifdef TARGET_ARM64 + GenTree* offset = + gtNewOperNode(GT_LSH, TYP_LONG, gtNewLclvNode(laneTemps[address.Offset], TYP_LONG), gtNewIconNode(3)); +#else + GenTree* scaled = gtNewOperNode(GT_LSH, TYP_LONG, gtNewLclvNode(temp, TYP_LONG), gtNewIconNode(3)); + GenTree* offset = gtNewOperNode(GT_ADD, TYP_LONG, scaled, gtNewLconNode(address.Offset * 8)); +#endif + // Preserve embedded CSE definitions and any other effects of the original offset. + GenTree* effects = nullptr; + gtExtractSideEffList(address.Node->gtOp2, &effects); + address.Node->gtOp2 = effects == nullptr ? offset : gtNewOperNode(GT_COMMA, TYP_LONG, effects, offset); + gtSetStmtInfo(address.Stmt); + fgSetStmtSeq(address.Stmt); + } + JITDUMP("Shared widened index V%02u for %u four-limb addresses in " FMT_BB "\n", temp, (unsigned)addresses.size(), + block->bbNum); +} +#endif // defined(TARGET_AMD64) || defined(TARGET_ARM64) + //------------------------------------------------------------------------ // optCanAndShouldChangeExitTest: // Check if the exit test can be rephrased to a downwards counted exit test diff --git a/src/coreclr/jit/instrsxarch.h b/src/coreclr/jit/instrsxarch.h index fcd6e9b3e06f6b..ba7d09541eeb65 100644 --- a/src/coreclr/jit/instrsxarch.h +++ b/src/coreclr/jit/instrsxarch.h @@ -1289,6 +1289,10 @@ INST3(sar, "sar", IUM_RW, 0x0038D2, BAD_CODE, INST3(sar_1, "sar", IUM_RW, 0x0038D0, 0x0038D0, 0x0038D0, 1C, 2X, INS_TT_NONE, Writes_OF | Writes_SF | Writes_ZF | Undefined_AF | Writes_PF | Writes_CF | INS_FLAGS_HasWBit | Encoding_REX2 | INS_FLAGS_HasNDD | INS_FLAGS_HasNF) INST3(sar_N, "sar", IUM_RW, 0x0038C0, 0x0038C0, BAD_CODE, 1C, 2X, INS_TT_NONE, Undefined_OF | Writes_SF | Writes_ZF | Undefined_AF | Writes_PF | Writes_CF | INS_FLAGS_HasWBit | Encoding_REX2 | INS_FLAGS_HasNDD | INS_FLAGS_HasNF) +// ADX carry chains. Keep these at the end of the three-encoding instruction table. +INST3(adcx, "adcx", IUM_RW, BAD_CODE, BAD_CODE, PSSE38(0x66, 0xF6), 1C, 2X, INS_TT_NONE, Reads_CF | Writes_CF) +INST3(adox, "adox", IUM_RW, BAD_CODE, BAD_CODE, PSSE38(0xF3, 0xF6), 1C, 2X, INS_TT_NONE, Reads_OF | Writes_OF) + // id nm um mr mi lat tp tt flags INST2(ret, "ret", IUM_RD, 0x0000C3, 0x0000C2, ILLEGAL, ILLEGAL, INS_TT_NONE, INS_FLAGS_None) INST2(loop, "loop", IUM_RD, BAD_CODE, 0x0000E2, ILLEGAL, ILLEGAL, INS_TT_NONE, INS_FLAGS_None) @@ -1409,6 +1413,9 @@ INST0(l_jg, "jg", IUM_RD, 0x008F0F, INST0(align, "align", IUM_RD, BAD_CODE, ILLEGAL, ILLEGAL, INS_TT_NONE, INS_FLAGS_None) INST0(data16, "data16", IUM_RD, 0x000066, ZERO, 4X, INS_TT_NONE, INS_FLAGS_None) +// Flags-preserving loop branch. +INST0(jrcxz, "jrcxz", IUM_RD, 0x0000E3, BRANCH_COND, 2X, INS_TT_NONE, INS_FLAGS_None) + /*****************************************************************************/ #undef INST0 #undef INST1 diff --git a/src/coreclr/jit/liveness.cpp b/src/coreclr/jit/liveness.cpp index 9198ab6c529d0e..b54c6fb80748d9 100644 --- a/src/coreclr/jit/liveness.cpp +++ b/src/coreclr/jit/liveness.cpp @@ -2461,6 +2461,9 @@ void Liveness::ComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VAR case GT_STORE_BLK: case GT_JCMP: case GT_JTEST: +#ifdef TARGET_AMD64 + case GT_ADX_SEED: +#endif case GT_JCC: case GT_JTRUE: case GT_RETURN: @@ -2653,6 +2656,13 @@ bool Liveness::TryRemoveNonLocalLIR(GenTree* node, LIR::Range* blockR } assert(!node->OperIsLocal()); + // A paired divide also defines RDX for the following physical-register + // capture. That implicit use must survive even when the quotient is dead + // and the division has been proven non-throwing. + if (node->OperIs(GT_DIV, GT_UDIV) && node->IsDivRemPair()) + { + return false; + } if (!node->IsValue() || node->IsUnusedValue()) { // We are only interested in avoiding the removal of nodes with direct side effects @@ -2669,7 +2679,7 @@ bool Liveness::TryRemoveNonLocalLIR(GenTree* node, LIR::Range* blockR return GenTree::VisitResult::Continue; }); - if (node->OperConsumesFlags() && node->gtPrev->gtSetFlags()) + if (node->OperConsumesFlags() && (node->gtPrev != nullptr) && node->gtPrev->gtSetFlags()) { node->gtPrev->gtFlags &= ~GTF_SET_FLAGS; } diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index bbd00eb1fa752c..01bf6dd18df42d 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -306,6 +306,10 @@ bool Lowering::IsSafeToMarkRegOptional(GenTree* parentNode, GenTree* childNode) void Lowering::LowerRange(GenTree* firstNode, GenTree* lastNode) { assert(lastNode != nullptr); +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + // A partial walk can have pre-existing flag consumers outside its range. + m_blockMayHaveSetCC = true; +#endif // Multiple possible behaviors of LowerNode are possible: // 1. The node being lowered may be removed @@ -375,6 +379,14 @@ GenTree* Lowering::LowerNode(GenTree* node) case GT_ADD: { +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + // Morph represents subtraction of a constant as addition of its negation. + if (m_compiler->opts.OptimizationEnabled() && node->TypeIs(TYP_INT, TYP_LONG) && + node->gtGetOp2()->IsCnsIntOrI() && (node->gtGetOp2()->AsIntCon()->IconValue() < 0)) + { + m_lastSubtractionBlock = m_block; + } +#endif GenTree* next = LowerAdd(node->AsOp()); if (next != nullptr) { @@ -383,6 +395,14 @@ GenTree* Lowering::LowerNode(GenTree* node) } break; +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + case GT_SUB_BORROW: + case GT_ADD_BORROW: + case GT_ADD_CARRY: + ContainCheckBinary(node->AsOp()); + break; +#endif + #if !defined(TARGET_64BIT) case GT_ADD_LO: case GT_ADD_HI: @@ -396,6 +416,17 @@ GenTree* Lowering::LowerNode(GenTree* node) { if (m_compiler->opts.OptimizationEnabled()) { +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + if (node->OperIs(GT_SUB)) + { + m_lastSubtractionBlock = m_block; + GenTree* next = node->gtNext; + if (TryLowerAddCarry(node->AsOp())) + { + return next; + } + } +#endif GenTree* nextNode = nullptr; if (node->OperIs(GT_AND) && TryLowerAndNegativeOne(node->AsOp(), &nextNode)) { @@ -722,6 +753,12 @@ GenTree* Lowering::LowerNode(GenTree* node) break; #endif // defined(TARGET_WASM) +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + case GT_SETCC: + m_blockMayHaveSetCC = true; + break; +#endif + default: break; } @@ -4649,6 +4686,9 @@ GenTree* Lowering::OptimizeConstCompare(GenTree* cmp) } } GenTreeCC* setcc = m_compiler->gtNewCC(GT_SETCC, cmp->TypeGet(), cmpCondition); +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + m_blockMayHaveSetCC = true; +#endif BlockRange().InsertAfter(op1, setcc); use.ReplaceWith(setcc); @@ -4669,6 +4709,13 @@ GenTree* Lowering::OptimizeConstCompare(GenTree* cmp) // GenTree* Lowering::LowerCompare(GenTree* cmp) { +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + GenTree* next = cmp->gtNext; + if (m_compiler->opts.OptimizationEnabled() && TryLowerCarryCompare(cmp->AsOp())) + { + return next; + } +#endif #if LOWER_DECOMPOSE_LONGS if (cmp->gtGetOp1()->TypeIs(TYP_LONG)) { @@ -4974,6 +5021,12 @@ bool Lowering::TryLowerConditionToFlagsNode(GenTree* parent, { assert((condition->gtPrev->gtFlags & GTF_SET_FLAGS) != 0); GenTree* flagsDef = condition->gtPrev; + // A value-producing flags definition can have intervening LIR users. Moving + // it next to the condition consumer would move its value past those uses. + if (flagsDef->IsValue() && !flagsDef->IsUnusedValue()) + { + return false; + } #if defined(TARGET_ARM64) || defined(TARGET_AMD64) && !defined(TARGET_WASM) // CCMP is a flag producing node that also consumes flags, so find the // "root" of the flags producers and move the entire range. @@ -5071,10 +5124,9 @@ GenTreeCC* Lowering::LowerNodeCC(GenTree* node, GenCondition condition) if (next->OperIs(GT_JTRUE)) { // If the instruction immediately following 'relop', i.e. 'next' is a conditional branch, - // it should always have 'relop' as its 'op1'. If it doesn't, then we have improperly - // constructed IL (the setting of a condition code should always immediately precede its - // use, since the JIT doesn't track dataflow for condition codes). Still, if it happens - // it's not our problem, it simply means that `node` is not used and can be removed. + // it should have 'relop' as its 'op1'. Only convert that use to JCC. + // JCC can be separated from its flags producer by flag-preserving nodes; + // the JIT does not otherwise track dataflow for condition codes. if (next->AsUnOp()->gtGetOp1() == relop) { assert(relop->OperIsCompare()); @@ -5093,6 +5145,9 @@ GenTreeCC* Lowering::LowerNodeCC(GenTree* node, GenCondition condition) if (BlockRange().TryGetUse(relop, &use)) { cc = m_compiler->gtNewCC(GT_SETCC, TYP_INT, condition); +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + m_blockMayHaveSetCC = true; +#endif BlockRange().InsertAfter(node, cc); use.ReplaceWith(cc); } @@ -7944,6 +7999,13 @@ bool Lowering::TryCreateAddrMode(GenTree* addr, bool isContainable, GenTree* par // GenTree* Lowering::LowerAdd(GenTreeOp* node) { +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + GenTree* next = node->gtNext; + if (m_compiler->opts.OptimizationEnabled() && TryLowerAddCarry(node)) + { + return next; + } +#endif if (varTypeIsIntegralOrI(node->TypeGet())) { GenTree* op1 = node->gtGetOp1(); @@ -8765,6 +8827,9 @@ GenTree* Lowering::LowerSignedDivOrMod(GenTree* node) // void Lowering::LowerDivOrMod(GenTreeOp* divMod) { +#ifdef TARGET_XARCH + TryLowerDivRem(divMod); +#endif ContainCheckDivOrMod(divMod); } #endif // !TARGET_WASM @@ -8873,6 +8938,77 @@ void Lowering::TryRemoveShiftRotateMask(GenTreeOp* op) } } +//------------------------------------------------------------------------ +// TryContainFunnelShift: Combine complementary constant shifts of two values. +// +// Arguments: +// node - The binary node to check. +// +// Return Value: +// True if the node can be emitted as SHRD or EXTR. +// +bool Lowering::TryContainFunnelShift(GenTreeOp* node) +{ +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + if (!node->OperIs(GT_OR)) + { + return false; + } + if (node->IsFunnelShift()) + { + return true; + } + + if (!m_compiler->opts.OptimizationEnabled() || !node->TypeIs(TYP_INT, TYP_LONG) || node->gtSetFlags()) + { + return false; + } + + GenTree* right = node->gtGetOp1(); + GenTree* left = node->gtGetOp2(); + if (right->OperIs(GT_LSH)) + { + std::swap(right, left); + } + + if (!right->OperIs(GT_RSZ) || !left->OperIs(GT_LSH) || right->gtSetFlags() || left->gtSetFlags() || + !right->gtGetOp2()->IsCnsIntOrI() || !left->gtGetOp2()->IsCnsIntOrI() || + (right->TypeGet() != node->TypeGet()) || (left->TypeGet() != node->TypeGet())) + { + return false; + } + + ssize_t width = genTypeSize(node) * BITS_PER_BYTE; + ssize_t rightCount = right->gtGetOp2()->AsIntCon()->IconValue(); + ssize_t leftCount = left->gtGetOp2()->AsIntCon()->IconValue(); + if ((rightCount <= 0) || (rightCount >= width) || (leftCount != width - rightCount)) + { + return false; + } + + GenTree* lo = right->gtGetOp1(); + GenTree* hi = left->gtGetOp1(); + // Keep memory accesses at their original positions, and do not extend a local + // read past a redefinition. Both input values must be available in registers. + if (lo->isContained() || hi->isContained() || !IsInvariantInRange(lo, node) || !IsInvariantInRange(hi, node)) + { + return false; + } + + lo->ClearRegOptional(); + hi->ClearRegOptional(); + node->gtOp1 = right; + node->gtOp2 = left; + MakeSrcContained(node, right); + MakeSrcContained(node, left); + MakeSrcContained(right, right->gtGetOp2()); + MakeSrcContained(left, left->gtGetOp2()); + return true; +#else + return false; +#endif +} + //------------------------------------------------------------------------ // LowerShift: Lower shift nodes // @@ -9640,6 +9776,9 @@ void Lowering::LowerBlock(BasicBlock* block) assert(block->isEmpty() || block->IsLIR()); m_block = block; +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + m_blockMayHaveSetCC = false; +#endif #ifdef TARGET_ARM64 m_blockIndirs.Reset(); m_ffrTrashed = true; @@ -9667,6 +9806,22 @@ void Lowering::LowerBlock(BasicBlock* block) // void Lowering::AfterLowerBlocks() { +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + // Post-processing revisits blocks and inserts SETCCs before their consumers. + m_blockMayHaveSetCC = true; + if (m_compiler->opts.OptimizationEnabled() && m_hasAddCarry) + { + if (LowerFullAdders()) + { + LowerCarryChains(); + } +#ifdef TARGET_AMD64 + LowerMultiplyCarryLoops(); +#else + LowerArm64MultiplyCarryLoops(); +#endif + } +#endif } #endif // !TARGET_WASM @@ -9920,6 +10075,11 @@ void Lowering::ContainCheckNode(GenTree* node) case GT_ADD: case GT_SUB: +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + case GT_SUB_BORROW: + case GT_ADD_BORROW: + case GT_ADD_CARRY: +#endif #if !defined(TARGET_64BIT) case GT_ADD_LO: case GT_ADD_HI: @@ -13072,6 +13232,7 @@ bool Lowering::TryLowerAndOrToCCMP(GenTreeOp* tree, GenTree** next) ContainCheckConditionalCompare(ccmp); tree->SetOper(GT_SETCC); + m_blockMayHaveSetCC = true; tree->AsCC()->gtCondition = cond2; JITDUMP("Conversion was legal. Result:\n"); @@ -13390,3 +13551,2939 @@ void Lowering::SetFramePointerFromArgSpaceSize() m_compiler->codeGen->setFramePointerRequired(true); } } + +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) +//------------------------------------------------------------------------ +// IsCarryLocalDeadOnExit: Prove that the current local value cannot be read +// after leaving block. Lowering cannot use stale bbLiveOut sets. Walk current +// successors up to a full definition, including EH paths and parent aliases. +// Results are not cached across IR/CFG mutations. Budget exhaustion declines +// optimization; implicit OSR state and JMP argument uses are excluded. +// During the main lowering walk successors may not yet be lowered. This relies +// on lowering preserving observations of existing tracked scalar locals: it +// cannot introduce an earlier read of a killed value or turn its full def into +// a partial def. Promoted parents are not scalar carry locals and are excluded. +bool Lowering::IsCarryLocalDeadOnExit(unsigned local, BasicBlock* block) +{ + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (dsc->lvPromoted || dsc->IsAddressExposed() || m_compiler->opts.IsOSR() || + (m_compiler->compJmpOpUsed && dsc->lvIsParam) || + (m_compiler->lvaKeepAliveAndReportThis() && (local == m_compiler->info.compThisArg))) + { + return false; + } + unsigned visitedWords = m_compiler->fgBBNumMax / (sizeof(size_t) * 8) + 1; + if (visitedWords > m_carryLocalScanBudget) + { + return false; + } + m_carryLocalScanBudget -= visitedWords; + BitVecTraits traits(m_compiler->fgBBNumMax + 1, m_compiler); + BitVec visited = BitVecOps::MakeEmpty(&traits); + jitstd::vector pending(m_compiler->getAllocator(CMK_Lower)); + bool exhausted = false; + auto enqueue = [&](BasicBlock* successor) { + if (m_carryLocalScanBudget == 0) + { + exhausted = true; + return BasicBlockVisit::Abort; + } + m_carryLocalScanBudget--; + if (BitVecOps::IsMember(&traits, visited, successor->bbNum)) + { + return BasicBlockVisit::Continue; + } + BitVecOps::AddElemD(&traits, visited, successor->bbNum); + pending.push_back(successor); + return BasicBlockVisit::Continue; + }; + block->VisitRegularSuccs(m_compiler, enqueue); + block->VisitEHSuccs(m_compiler, enqueue); + for (unsigned index = 0; index < pending.size(); index++) + { + if (m_carryLocalScanBudget == 0) + { + return false; + } + m_carryLocalScanBudget--; + BasicBlock* current = pending[index]; + // A throw can reach the handler before any definition in this block. + current->VisitEHSuccs(m_compiler, enqueue); + bool killed = false; + for (GenTree* node : LIR::AsRange(current)) + { + if (m_carryLocalScanBudget == 0) + { + return false; + } + m_carryLocalScanBudget--; + if (!node->OperIsLocal()) + { + continue; + } + unsigned reference = node->AsLclVarCommon()->GetLclNum(); + if ((reference != local) && !(dsc->lvIsStructField && (reference == dsc->lvParentLcl))) + { + continue; + } + if (node->OperIs(GT_STORE_LCL_VAR)) + { + killed = true; + break; + } + // Reads, addresses and partial stores cannot prove the old value dead. + return false; + } + if (!killed) + { + current->VisitRegularSuccs(m_compiler, enqueue); + } + } + return !exhausted; +} + +//------------------------------------------------------------------------ +// FindCarryLoopEntry: Find the sole predecessor other than the loop itself. +BasicBlock* Lowering::FindCarryLoopEntry(BasicBlock* block) +{ + BasicBlock* entry = nullptr; + for (FlowEdge* edge : block->PredEdges()) + { + BasicBlock* pred = edge->getSourceBlock(); + if (pred != block) + { + if (entry != nullptr) + { + return nullptr; + } + entry = pred; + } + } + return entry; +} + +//------------------------------------------------------------------------ +// FindDeadCarryConstantStore: Identify a constant/store pair that can be +// removed from a flags-live range. Prove the local unobserved throughout the +// current method, excluding aliases and implicit uses absent from the LIR scan. +GenTree* Lowering::FindDeadCarryConstantStore(GenTree* constant) +{ + assert(constant->OperIs(GT_CNS_INT) && !constant->isContained()); + LIR::Use use; + if (!BlockRange().TryGetUse(constant, &use) || !use.User()->OperIs(GT_STORE_LCL_VAR)) + { + return nullptr; + } + unsigned local = use.User()->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (dsc->IsAddressExposed() || dsc->lvIsStructField || dsc->lvPromoted || m_compiler->opts.IsOSR() || + (m_compiler->compJmpOpUsed && dsc->lvIsParam) || + (m_compiler->lvaKeepAliveAndReportThis() && local == m_compiler->info.compThisArg)) + { + return nullptr; + } + for (BasicBlock* block : m_compiler->Blocks()) + { + for (GenTree* node : LIR::AsRange(block)) + { + if (m_carryUseScanBudget == 0) + { + return nullptr; + } + m_carryUseScanBudget--; + if (node->OperIsLocal() && node->AsLclVarCommon()->GetLclNum() == local && !node->OperIs(GT_STORE_LCL_VAR)) + { + return nullptr; + } + } + } + return use.User(); +} + +//------------------------------------------------------------------------ +// FindCarryLocalStore: Find the nearest full definition before a local read, +// within the fixed local search allowance. This does not prove single use or +// deadness on exit; callers establish those when deleting the definition. +// Keep short searches independent of the budget for extended carry-use scans. +GenTree* Lowering::FindCarryLocalStore(GenTree* read) +{ + assert(read->OperIs(GT_LCL_VAR)); + unsigned local = read->AsLclVar()->GetLclNum(); + GenTree* node = read->gtPrev; + for (unsigned remaining = 64; (node != nullptr) && (remaining != 0); node = node->gtPrev, remaining--) + { + if (node->OperIs(GT_STORE_LCL_VAR) && (node->AsLclVar()->GetLclNum() == local)) + { + return node; + } + } + return nullptr; +} + +//------------------------------------------------------------------------ +// TryLowerCarryCompare: Reuse arithmetic flags for unsigned carry/borrow comparisons. +// Match sum < operand, difference > minuend, or a < b after computing a-b. +// The arithmetic result can be forwarded or stored to a local in this block. +bool Lowering::TryLowerCarryCompare(GenTreeOp* cmp) +{ + auto match = [&](unsigned mode) -> bool { + // 0: sum comparison, 1: difference comparison, 2: subtraction operands, + // 3: negated value compared with zero (the borrow from 0 - value). + bool subtract = mode != 0; + bool negate = mode == 3; + + if (negate ? (!cmp->OperIs(GT_EQ, GT_NE) || !cmp->gtOp2->IsIntegralConst(0)) + : (!cmp->OperIs(GT_LT, GT_GT, GT_GE, GT_LE) || !cmp->IsUnsigned())) + { + return false; + } + + bool reversed = cmp->OperIs(GT_GT, GT_LE) != (mode == 1); + GenTree* sum = reversed ? cmp->gtOp2 : cmp->gtOp1; + GenTree* operand = reversed ? cmp->gtOp1 : cmp->gtOp2; + if ((!sum->OperIs(GT_LCL_VAR, negate ? GT_NEG : (subtract ? GT_SUB : GT_ADD)) && + !(mode == 1 && sum->OperIs(GT_ADD))) || + !sum->TypeIs(TYP_INT, TYP_LONG) || !operand->OperIs(GT_LCL_VAR, GT_CNS_INT) || + (sum->TypeGet() != operand->TypeGet())) + { + return false; + } + + GenTree* add = sum; + if (mode == 2) + { + if (!sum->OperIs(GT_LCL_VAR)) + { + return false; + } + // a < b can reuse an already-computed a-b, as in UInt128 subtraction + // and subtract-multiply. Compare the reaching operand values as well. + add = cmp->gtPrev; + unsigned budget = 64; + for (; add != nullptr && budget != 0; add = add->gtPrev, budget--) + { + if (m_carryUseScanBudget == 0) + { + return false; + } + m_carryUseScanBudget--; + if (add->OperIs(GT_SUB) && GenTree::Compare(add->gtGetOp1(), sum) && + GenTree::Compare(add->gtGetOp2(), operand) && IsInvariantInRange(add->gtGetOp1(), cmp) && + IsInvariantInRange(add->gtGetOp2(), cmp)) + { + break; + } + } + if (add == nullptr || budget == 0) + { + return false; + } + } + else if (sum->OperIs(GT_LCL_VAR)) + { + if (m_compiler->lvaGetDesc(sum->AsLclVar()->GetLclNum())->IsAddressExposed()) + { + return false; + } + GenTree* def = FindCarryLocalStore(sum); + if (def == nullptr) + { + return false; + } + add = def->gtGetOp1(); + } + bool negatedConstant = (mode == 1) && add->OperIs(GT_ADD) && add->gtGetOp2()->IsCnsIntOrI() && + !add->gtGetOp2()->AsIntCon()->IsIconHandle() && + (add->gtGetOp2()->AsIntCon()->IconValue() < 0) && + (add->gtGetOp2()->AsIntCon()->IconValue() != INT64_MIN) && + (add->gtGetOp2()->AsIntCon()->IconValue() != INT32_MIN); + if ((!add->OperIs(negate ? GT_NEG : (subtract ? GT_SUB : GT_ADD)) && !negatedConstant) || + (!negate && add->gtOverflow()) || add->gtSetFlags() || add->isContained() || + (add->TypeGet() != sum->TypeGet())) + { + return false; + } +#ifdef TARGET_ARM64 + // MNEG does not set flags, even when its containing NEG requests them. + if (negate && add->gtGetOp1()->OperIs(GT_MUL)) + { + return false; + } +#endif + + GenTree* matched = negate ? add->gtGetOp1() : nullptr; + for (GenTree* input : add->Operands()) + { + if ((mode != 1 || input == add->gtGetOp1()) && GenTree::Compare(input, operand) && + IsInvariantInRange(input, cmp)) + { + matched = input; + break; + } + } + LIR::Use use; + if ((matched == nullptr) || !BlockRange().TryGetUse(cmp, &use)) + { + return false; + } + + // A select needs flags at the select itself. Materializing them here can + // introduce SETCC and TEST when the sum is still used or flags are clobbered. + // Preserve the existing compare-to-select lowering in that case. + if (use.User()->OperIs(GT_SELECT)) + { + return false; + } + + GenCondition condition = cmp->OperIs(GT_LT, GT_GT, GT_NE) ? GenCondition::C : GenCondition::NC; +#ifdef TARGET_ARM64 + if (subtract) + { + condition = GenCondition::Reverse(condition); + } +#endif + bool branch = use.User()->OperIs(GT_JTRUE); + if (branch) + { +#ifdef TARGET_AMD64 + // Unlike a carry chain, a branch saves only a comparison. Avoid sacrificing LEA + // or perturbing allocation in register-heavy methods for that small saving. + unsigned intRegCount = m_compiler->get_REG_INT_LAST() - REG_INT_FIRST; + if (negatedConstant || (m_compiler->lvaTrackedCount > 2 * intRegCount)) + { + return false; + } +#endif + if (use.User() != cmp->gtNext) + { + return false; + } + // Keep the branch at the end of the block, and only cross nodes that + // cannot overwrite flags. JCC need not be adjacent to its flags producer. + // Constants may become XOR on x64. + for (GenTree* node = add->gtNext; node != cmp; node = node->gtNext) + { + if ((node == nullptr) || + ((node != operand) && !node->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR, GT_IL_OFFSET))) + { + return false; + } + } + } + + JITDUMP("Replacing carry comparison [%06u] with flags from [%06u]\n", cmp->gtTreeID, add->gtTreeID); + if (negatedConstant) + { + // Restore SUB so its flags describe the borrow, rather than the carry + // from adding the two's-complement constant. The result is unchanged. + add->ChangeOper(GT_SUB); + GenTreeIntCon* constant = add->gtGetOp2()->AsIntCon(); + constant->SetIconValue(-constant->IconValue()); + } + add->gtFlags |= GTF_SET_FLAGS; + if (add->OperIs(GT_ADD)) + { + add->gtFlags |= GTF_ADD_CARRY_FLAGS; + } + for (GenTree* input : add->Operands()) + { + input->ClearContained(); + input->ClearRegOptional(); + } + if (negate) + { + ContainCheckNode(add); + } + else + { + ContainCheckBinary(add->AsOp()); + } + if (branch) + { + GenTree* jump = use.User(); + jump->ChangeOper(GT_JCC); + jump->AsCC()->gtCondition = condition; + } + else + { + GenTreeCC* carry = m_compiler->gtNewCC(GT_SETCC, TYP_INT, condition); + m_blockMayHaveSetCC = true; + BlockRange().InsertAfter(add, carry); + use.ReplaceWith(carry); + } + if (sum == add) + { + add->SetUnusedValue(); + } + else + { + BlockRange().Remove(sum); + } + BlockRange().Remove(operand); + BlockRange().Remove(cmp); + return true; + }; + return match(0) || ((m_lastSubtractionBlock == m_block) && (match(1) || match(2))) || match(3); +} + +#ifdef TARGET_AMD64 +//------------------------------------------------------------------------ +// ReplaceCarryAddressAdd: Replace a validated base-plus-constant ADD with LEA +// so the address calculation preserves both carry flags. +void Lowering::ReplaceCarryAddressAdd(GenTree* add) +{ + assert(add->OperIs(GT_ADD) && !add->gtOverflow() && !add->gtSetFlags()); + GenTree* offset = add->gtGetOp2(); + assert(offset->IsIntCnsFitsInI32()); + GenTree* lea = new (m_compiler, GT_LEA) + GenTreeAddrMode(add->TypeGet(), add->gtGetOp1(), nullptr, 1, (int)offset->AsIntCon()->IconValue()); + LIR::Use use; + bool found = BlockRange().TryGetUse(add, &use); + assert(found); + BlockRange().InsertBefore(add, lea); + use.ReplaceWith(lea); + BlockRange().Remove(add); + BlockRange().Remove(offset); +} + +//------------------------------------------------------------------------ +// LowerMultiplyCarryLoops: Keep both unsigned carry chains of scalar and +// unrolled widening multiply-accumulates in flags around counted loops. +// Memory containment checks interference; no bounds check is removed. +void Lowering::LowerMultiplyCarryLoops() +{ + if (m_compiler->compHndBBtabCount != 0 || !m_compiler->compOpportunisticallyDependsOn(InstructionSet_ADX) || + !m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + return; + } + + BasicBlock* savedBlock = m_block; + + // MULX is required while CF/OF are live: ordinary MUL overwrites both flags. + // Snapshot blocks since successful matches split edges. + jitstd::vector loops(m_compiler->getAllocator(CMK_Lower)); + for (BasicBlock* block : m_compiler->Blocks()) + { + loops.push_back(block); + } + for (BasicBlock* block : loops) + { + m_block = block; + bool hasAdx = TryLowerMultiplyCarryChain(); + if (block->KindIs(BBJ_COND) && block->GetTrueTarget() == block && block->GetFalseTarget() != block) + { + TryLowerMultiplyCarryBackedge(); + hasAdx |= TryLowerMultiplyCarryLoop(); + LowerMultiplyCarryMultiplier(); + } + if (hasAdx) + { + LowerMultiplyCarryLoads(); + } + } + m_block = savedBlock; +} + +//------------------------------------------------------------------------ +// LowerMultiplyCarryLoads: Forward single-use load temporaries into the memory +// operand of MULX and ADCX/ADOX after the carry patterns have been matched. +void Lowering::LowerMultiplyCarryLoads() +{ + for (GenTree* node : BlockRange()) + { + GenTree** operand = nullptr; + if (node->OperIs(GT_ADCX, GT_ADOX)) + { + operand = &node->AsOp()->gtOp2; + } + else if (node->OperIs(GT_HWINTRINSIC) && (node->AsHWIntrinsic()->GetHWIntrinsicId() == NI_X86Base_X64_BigMul) && + (node->AsHWIntrinsic()->GetSimdBaseType() == TYP_ULONG) && ((node->gtFlags & GTF_HW_MULX) != 0)) + { + operand = &node->AsHWIntrinsic()->Op(2); + } + if ((operand == nullptr) || !(*operand)->OperIs(GT_LCL_VAR)) + { + continue; + } + + GenTree* read = *operand; + unsigned local = read->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (!dsc->lvTracked || dsc->IsAddressExposed()) + { + continue; + } + + GenTree* store = nullptr; + bool valid = true; + for (GenTree* reference : BlockRange()) + { + if (m_carryUseScanBudget == 0) + { + return; + } + --m_carryUseScanBudget; + // A promoted field's value can also be observed or changed through its parent. + if (dsc->lvIsStructField && reference->OperIsLocal() && + (reference->AsLclVarCommon()->GetLclNum() == dsc->lvParentLcl)) + { + valid = false; + break; + } + if ((reference == read) && (store == nullptr)) + { + valid = false; + break; + } + if (reference->OperIsLocal() && (reference->AsLclVarCommon()->GetLclNum() == local)) + { + if (reference->OperIs(GT_STORE_LCL_VAR) && (store == nullptr)) + { + store = reference; + } + else if (reference != read) + { + valid = false; + break; + } + } + } + if (!valid || (store == nullptr)) + { + continue; + } + GenTree* load = store->gtGetOp1(); + if (!load->OperIs(GT_IND) || !load->TypeIs(TYP_LONG) || ((load->gtFlags & GTF_IND_VOLATILE) != 0) || + !IsSafeToContainMem(node, store, load)) + { + continue; + } + + if (!IsCarryLocalDeadOnExit(local, m_block)) + { + continue; + } + + JITDUMP("Containing load [%06u] from V%02u in [%06u]\n", load->gtTreeID, local, node->gtTreeID); + *operand = load; + // Both removed nodes precede the current iterator. The load stays in + // LIR; containment delays its execution only across the checked range. + BlockRange().Remove(store); + BlockRange().Remove(read); + MakeSrcContained(node, load); + } +} + +//------------------------------------------------------------------------ +// LowerMultiplyCarryMultiplier: Give a flags-preserving multiply loop its own +// copy of an invariant multiplier so its fixed RDX uses can share a register +// assignment independently of the incoming argument and the remainder loop. +void Lowering::LowerMultiplyCarryMultiplier() +{ + if (!BlockRange().LastNode()->OperIs(GT_JCMP)) + { + return; + } + + BasicBlock* entry = FindCarryLoopEntry(m_block); + if ((entry == nullptr) || !entry->KindIs(BBJ_ALWAYS) || (entry->GetTarget() != m_block)) + { + return; + } + // These loop transforms create a dedicated entry block containing only the flags seed. + // Insert before the seed so even a zero-initializing copy cannot interfere with the flags. + if ((LIR::AsRange(entry).FirstNode() != LIR::AsRange(entry).LastNode()) || + (LIR::AsRange(entry).FirstNode() == nullptr) || !LIR::AsRange(entry).FirstNode()->OperIs(GT_ADX_SEED)) + { + return; + } + + unsigned multiplier = BAD_VAR_NUM; + jitstd::vector products(m_compiler->getAllocator(CMK_Lower)); + for (GenTree* node : BlockRange()) + { + if (node->OperIs(GT_HWINTRINSIC)) + { + GenTreeHWIntrinsic* product = node->AsHWIntrinsic(); + if ((product->GetHWIntrinsicId() != NI_X86Base_X64_BigMul) || (product->GetSimdBaseType() != TYP_ULONG) || + ((product->gtFlags & GTF_HW_MULX) == 0) || !product->Op(1)->OperIs(GT_LCL_VAR) || + !product->Op(1)->TypeIs(TYP_LONG)) + { + return; + } + unsigned local = product->Op(1)->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (!dsc->lvTracked || dsc->IsAddressExposed() || dsc->lvIsStructField || + ((multiplier != BAD_VAR_NUM) && (multiplier != local))) + { + return; + } + multiplier = local; + products.push_back(product); + } + } + if (products.empty()) + { + return; + } + for (GenTree* node : BlockRange()) + { + if (node->OperIs(GT_STORE_LCL_VAR, GT_STORE_LCL_FLD) && (node->AsLclVarCommon()->GetLclNum() == multiplier)) + { + return; + } + } + + unsigned temp = m_compiler->lvaGrabTemp(true DEBUGARG("ADX loop multiplier")); + JITDUMP("Copying invariant multiplier V%02u to V%02u on entry to " FMT_BB "\n", multiplier, temp, m_block->bbNum); + m_compiler->lvaGetDesc(temp)->lvType = TYP_LONG; + GenTree* read = m_compiler->gtNewLclvNode(multiplier, TYP_LONG); + GenTree* store = m_compiler->gtNewStoreLclVarNode(temp, read); + LIR::AsRange(entry).InsertAtBeginning(read, store); + for (GenTreeHWIntrinsic* product : products) + { + GenTree* oldRead = product->Op(1); + GenTree* newRead = m_compiler->gtNewLclvNode(temp, TYP_LONG); + BlockRange().InsertBefore(oldRead, newRead); + product->Op(1) = newRead; + BlockRange().Remove(oldRead); + } +} + +//------------------------------------------------------------------------ +// TryLowerMultiplyCarryChain: Keep CF and OF across a bounded sequence of +// widening multiply-accumulates in one block. Seed after any preceding checks +// and drain before the final high word is consumed, so checks and loop control +// outside the sequence need not preserve either flag. No memory access moves. +// Returns true if an ADX chain was created; a scalar MULX-only match returns false. +bool Lowering::TryLowerMultiplyCarryChain() +{ + const unsigned maxLanes = 4; + struct Lane + { + GenTree* product; + GenTree* productStore; + GenTree* adc[2]; + GenTree* add[2]; + GenTree* low; + GenTree* destination; + GenTree* partial; + GenTree* carry; + } lanes[maxLanes] = {}; + + unsigned laneCount = 0; + unsigned adcCount = 0; + unsigned nodeCount = 0; + for (GenTree* node : BlockRange()) + { + if (++nodeCount > 512) + { + return false; + } + if (node->OperIs(GT_HWINTRINSIC)) + { + if (laneCount == maxLanes || adcCount != laneCount * 2 || + node->AsHWIntrinsic()->GetHWIntrinsicId() != NI_X86Base_X64_BigMul || + node->AsHWIntrinsic()->GetSimdBaseType() != TYP_ULONG) + { + return false; + } + lanes[laneCount++].product = node; + } + if (node->OperIs(GT_ADD_CARRY)) + { + if (laneCount == 0 || adcCount >= laneCount * 2 || !node->TypeIs(TYP_LONG) || node->gtSetFlags() || + !node->gtGetOp1()->OperIs(GT_LCL_VAR) || !node->gtGetOp2()->IsIntegralConst(0)) + { + return false; + } + lanes[adcCount / 2].adc[adcCount % 2] = node; + adcCount++; + } + } + if (laneCount == 0 || adcCount != laneCount * 2) + { + return false; + } + + // Changed intermediate sums/high words may have only one reader and must + // not escape this block. Requiring the store first also proves availability. + for (unsigned i = 0; i < laneCount; i++) + { + Lane& lane = lanes[i]; + LIR::Use productUse; + if (!BlockRange().TryGetUse(lane.product, &productUse) || !productUse.User()->OperIs(GT_STORE_LCL_VAR)) + { + return false; + } + lane.productStore = productUse.User(); + unsigned productLocal = lane.productStore->AsLclVar()->GetLclNum(); + LclVarDsc* fields = m_compiler->lvaGetDesc(productLocal); + if (!fields->lvPromoted || fields->lvFieldCnt != 2) + { + return false; + } + unsigned lowLocal = fields->lvFieldLclStart; + unsigned highLocal = lowLocal + 1; + for (unsigned j = 0; j < 2; j++) + { + GenTree* add = lane.adc[j]->gtPrev; + if (add == nullptr || !add->OperIs(GT_ADD) || !add->TypeIs(TYP_LONG) || add->gtOverflow() || + !add->gtSetFlags() || !add->gtGetOp1()->OperIs(GT_LCL_VAR) || !add->gtGetOp2()->OperIs(GT_LCL_VAR)) + { + return false; + } + lane.add[j] = add; + } + if (lane.adc[0]->gtGetOp1()->AsLclVar()->GetLclNum() != highLocal) + { + return false; + } + GenTree* highStore = FindMultiplyCarryStore(lane.adc[1]->gtGetOp1()); + if (highStore == nullptr || highStore->gtGetOp1() != lane.adc[0]) + { + return false; + } + lane.low = lane.add[0]->gtGetOp1(); + lane.destination = lane.add[0]->gtGetOp2(); + if (lane.low->AsLclVar()->GetLclNum() != lowLocal) + { + std::swap(lane.low, lane.destination); + } + if (lane.low->AsLclVar()->GetLclNum() != lowLocal) + { + return false; + } + lane.partial = lane.add[1]->gtGetOp1(); + lane.carry = lane.add[1]->gtGetOp2(); + GenTree* partialStore = FindMultiplyCarryStore(lane.partial); + if (partialStore == nullptr || partialStore->gtGetOp1() != lane.add[0]) + { + std::swap(lane.partial, lane.carry); + partialStore = FindMultiplyCarryStore(lane.partial); + } + if (partialStore == nullptr || partialStore->gtGetOp1() != lane.add[0]) + { + return false; + } + if (i != 0) + { + GenTree* carryStore = FindMultiplyCarryStore(lane.carry); + if (carryStore == nullptr || carryStore->gtGetOp1() != lanes[i - 1].adc[1]) + { + return false; + } + } + + // The two field reads must refer to this product, not to earlier or + // intervening assignments. A parent/field alias must not escape us. + bool defined = false; + for (GenTree* node : BlockRange()) + { + if (node == lane.productStore) + { + defined = true; + } + if ((node == lane.low || node == lane.adc[0]->gtGetOp1()) && !defined) + { + return false; + } + if (node->OperIsLocal()) + { + unsigned local = node->AsLclVarCommon()->GetLclNum(); + if ((local == lowLocal && node != lane.low) || + (local == highLocal && node != lane.adc[0]->gtGetOp1()) || + (local == productLocal && node != lane.productStore)) + { + return false; + } + } + } + + // Move only the carry read, from the second addition to the first. + // Unlike intermediate carries, the initial value may have other users. + unsigned carryLocal = lane.carry->AsLclVar()->GetLclNum(); + if (m_compiler->lvaGetDesc(carryLocal)->IsAddressExposed()) + { + return false; + } + for (GenTree* node = lane.add[0]; node != lane.carry; node = node->gtNext) + { + if (node == nullptr || (node->OperIsLocalStore() && node->AsLclVarCommon()->GetLclNum() == carryLocal)) + { + return false; + } + } + } + + if (laneCount == 1) + { + // The product feeds both halves of this proven multiply-accumulate. + // Keep MULX's independent outputs for scalar tails without introducing + // an ADX seed/drain pair for a single lane. + lanes[0].product->gtFlags |= GTF_HW_MULX; + return false; + } + + GenTree* first = lanes[0].add[0]; + GenTree* last = lanes[laneCount - 1].adc[1]; + jitstd::vector addresses(m_compiler->getAllocator(CMK_Lower)); + for (GenTree* node = first; node != last->gtNext; node = node->gtNext) + { + bool matched = false; + for (unsigned i = 0; i < laneCount; i++) + { + Lane& lane = lanes[i]; + matched |= node == lane.product || node == lane.add[0] || node == lane.add[1] || node == lane.adc[0] || + node == lane.adc[1]; + } + if (matched) + { + continue; + } + switch (node->OperGet()) + { + case GT_LCL_VAR: + case GT_STORE_LCL_VAR: + case GT_IL_OFFSET: + case GT_LEA: + break; + case GT_CAST: + if (node->gtOverflow() || !node->TypeIs(TYP_LONG) || !node->gtGetOp1()->TypeIs(TYP_INT)) + { + return false; + } + break; + case GT_IND: + case GT_STOREIND: + if (!node->TypeIs(TYP_LONG) || (node->gtFlags & GTF_IND_VOLATILE) != 0) + { + return false; + } + break; + case GT_CNS_INT: + if (!node->isContained()) + { + return false; + } + break; + case GT_ADD: + if (node->gtSetFlags() || node->gtOverflow() || !node->TypeIs(TYP_INT, TYP_LONG, TYP_BYREF) || + !node->gtGetOp1()->OperIs(GT_LCL_VAR) || !node->gtGetOp2()->IsIntCnsFitsInI32()) + { + return false; + } + addresses.push_back(node); + break; + default: + return false; + } + } + + JITDUMP("Keeping multiply-accumulate CF and OF chains across %u lanes in " FMT_BB "\n", laneCount, m_block->bbNum); + GenTree* initialize = new (m_compiler, GT_ADX_SEED) GenTree(GT_ADX_SEED, TYP_VOID); + BlockRange().InsertBefore(first, initialize); + for (unsigned i = 0; i < laneCount; i++) + { + Lane& lane = lanes[i]; + // Every product in the matched chain must preserve both live carry flags. + lane.product->gtFlags |= GTF_HW_MULX; + BlockRange().Remove(lane.carry); + BlockRange().InsertBefore(lane.add[0], lane.carry); + lane.add[0]->AsOp()->gtOp1 = lane.carry; + lane.add[0]->AsOp()->gtOp2 = lane.destination; + lane.add[1]->AsOp()->gtOp1 = lane.partial; + lane.add[1]->AsOp()->gtOp2 = lane.low; + lane.add[0]->ChangeOper(GT_ADCX); + lane.add[1]->ChangeOper(GT_ADOX); + for (GenTree* adc : lane.adc) + { + LIR::Use use; + bool found = BlockRange().TryGetUse(adc, &use); + assert(found); + GenTree* replacement = adc->gtGetOp1(); + if (adc == last) + { + replacement = m_compiler->gtNewOperNode(GT_ADX_DRAIN, TYP_LONG, replacement); + BlockRange().InsertBefore(adc, replacement); + } + use.ReplaceWith(replacement); + BlockRange().Remove(adc->gtGetOp2()); + BlockRange().Remove(adc); + } + } + for (GenTree* add : addresses) + { + ReplaceCarryAddressAdd(add); + } + return true; +} + +//------------------------------------------------------------------------ +// FindMultiplyCarryStore: Find a local's sole store and sole read in this block. +// Ordinary intermediates require a preceding store and cannot be live out. +// A loop carry may be read before its store and observed on the exit edge. +// No nodes are moved by this helper; callers check the transformed range. +GenTree* Lowering::FindMultiplyCarryStore(GenTree* read, bool loopCarry) +{ + if (!read->OperIs(GT_LCL_VAR)) + { + return nullptr; + } + unsigned local = read->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (!dsc->lvTracked || dsc->IsAddressExposed()) + { + return nullptr; + } + GenTree* store = nullptr; + unsigned reads = 0; + for (GenTree* node : BlockRange()) + { + if (dsc->lvIsStructField && node->OperIsLocal() && node->AsLclVarCommon()->GetLclNum() == dsc->lvParentLcl) + { + return nullptr; + } + if (node == read && store == nullptr && !loopCarry) + { + return nullptr; + } + if (node->OperIsLocal() && node->AsLclVarCommon()->GetLclNum() == local) + { + if (node->OperIs(GT_STORE_LCL_VAR)) + { + if (store != nullptr) + { + return nullptr; + } + store = node; + } + else if (node != read || ++reads != 1) + { + return nullptr; + } + } + } + if ((reads != 1) || (store == nullptr) || (!loopCarry && !IsCarryLocalDeadOnExit(local, m_block))) + { + return nullptr; + } + return store; +} + +//------------------------------------------------------------------------ +// TryLowerMultiplyCarryBackedge: Extend a proven in-block ADX chain around +// a countdown loop. Only the full-word carry may escape, and its two pending +// flags are materialized on the exit edge before any outside observer. +void Lowering::TryLowerMultiplyCarryBackedge() +{ + BasicBlock* block = m_block; + BasicBlock* entry = FindCarryLoopEntry(block); + if (entry == nullptr) + { + return; + } + + GenTree* branch = BlockRange().LastNode(); + GenTree* cmp = branch->gtPrev; + if (!branch->OperIs(GT_JCC) || branch->AsCC()->gtCondition.GetCode() != GenCondition::NE || cmp == nullptr || + !cmp->OperIs(GT_CMP) || !cmp->gtGetOp1()->OperIs(GT_LCL_VAR) || !cmp->gtGetOp1()->TypeIs(TYP_INT) || + !cmp->gtGetOp2()->IsIntegralConst(0)) + { + return; + } + + GenTree* initialize = nullptr; + GenTree* drain = nullptr; + GenTree* first = nullptr; + for (GenTree* node : BlockRange()) + { + if (node->OperIs(GT_ADX_SEED)) + { + if (initialize != nullptr) + { + return; + } + initialize = node; + } + if (node->OperIs(GT_ADX_DRAIN)) + { + if (drain != nullptr) + { + return; + } + drain = node; + } + if (node->OperIs(GT_ADCX) && first == nullptr) + { + first = node; + } + } + if (initialize == nullptr || drain == nullptr || first == nullptr || !first->gtGetOp1()->OperIs(GT_LCL_VAR)) + { + return; + } + // Resolve only a single, preceding definition with a single use. Changed + // intermediate values must not be observable outside this loop. + GenTree* carryRead = first->gtGetOp1(); + GenTree* carryStore = FindMultiplyCarryStore(carryRead, true); + if (carryStore == nullptr || !carryStore->gtGetOp1()->OperIs(GT_LCL_VAR)) + { + return; + } + GenTree* finalHighStore = FindMultiplyCarryStore(carryStore->gtGetOp1()); + if (finalHighStore == nullptr || finalHighStore->gtGetOp1() != drain) + { + return; + } + GenTree* countRead = cmp->gtGetOp1(); + unsigned countLocal = countRead->AsLclVar()->GetLclNum(); + GenTree* countStore = nullptr; + for (GenTree* node = cmp->gtPrev; node != nullptr; node = node->gtPrev) + { + if (node->OperIs(GT_STORE_LCL_VAR) && node->AsLclVar()->GetLclNum() == countLocal) + { + countStore = node; + break; + } + } + if (countStore == nullptr) + { + return; + } + GenTree* decrement = countStore->gtGetOp1(); + if (!decrement->OperIs(GT_ADD) || !decrement->TypeIs(TYP_INT) || decrement->gtSetFlags() || + decrement->gtOverflow() || !decrement->gtGetOp1()->OperIs(GT_LCL_VAR) || + decrement->gtGetOp1()->AsLclVar()->GetLclNum() != countLocal || !decrement->gtGetOp2()->IsIntegralConst(-1)) + { + return; + } + + jitstd::vector addresses(m_compiler->getAllocator(CMK_Lower)); + jitstd::vector deadNodes(m_compiler->getAllocator(CMK_Lower)); + bool sawCarryRead = false; + for (GenTree* node : BlockRange()) + { + if (node == carryRead) + { + sawCarryRead = true; + } + if (node == carryStore && !sawCarryRead) + { + return; + } + if (node == initialize || node == drain || node == cmp || node == branch) + { + continue; + } + switch (node->OperGet()) + { + case GT_ADCX: + case GT_ADOX: + case GT_LCL_VAR: + case GT_STORE_LCL_VAR: + case GT_IL_OFFSET: + case GT_LEA: + break; + case GT_HWINTRINSIC: + if (node->AsHWIntrinsic()->GetHWIntrinsicId() != NI_X86Base_X64_BigMul || + node->AsHWIntrinsic()->GetSimdBaseType() != TYP_ULONG || (node->gtFlags & GTF_HW_MULX) == 0) + { + return; + } + break; + case GT_CAST: + if (node->gtOverflow() || !node->TypeIs(TYP_LONG) || !node->gtGetOp1()->TypeIs(TYP_INT)) + { + return; + } + break; + case GT_IND: + case GT_STOREIND: + if (!node->TypeIs(TYP_LONG) || (node->gtFlags & GTF_IND_VOLATILE) != 0) + { + return; + } + break; + case GT_CNS_INT: + if (!node->isContained()) + { + GenTree* deadStore = FindDeadCarryConstantStore(node); + if (deadStore == nullptr) + { + return; + } + deadNodes.push_back(node); + deadNodes.push_back(deadStore); + } + break; + + case GT_ADD: + if (node->gtSetFlags() || node->gtOverflow() || !node->TypeIs(TYP_INT, TYP_LONG, TYP_BYREF) || + !node->gtGetOp1()->OperIs(GT_LCL_VAR) || !node->gtGetOp2()->IsIntCnsFitsInI32()) + { + return; + } + addresses.push_back(node); + break; + default: + return; + } + } + + JITDUMP("Keeping unrolled multiply-accumulate CF and OF across the backedge of " FMT_BB "\n", block->bbNum); + BasicBlock* seed = m_compiler->fgSplitEdge(entry, block); + BasicBlock* finish = m_compiler->fgSplitEdge(block, block->GetFalseTarget()); + m_compiler->fgInvalidateDfsTree(); + BlockRange().Remove(initialize); + LIR::AsRange(seed).InsertAtEnd(initialize); + finalHighStore->AsOp()->gtOp1 = drain->gtGetOp1(); + BlockRange().Remove(drain); + unsigned carryLocal = carryRead->AsLclVar()->GetLclNum(); + GenTree* finalRead = m_compiler->gtNewLclvNode(carryLocal, TYP_LONG); + drain->AsOp()->gtOp1 = finalRead; + GenTree* finalStore = m_compiler->gtNewStoreLclVarNode(carryLocal, drain); + LIR::AsRange(finish).InsertAtEnd(finalRead, drain, finalStore); + for (GenTree* dead : deadNodes) + { + BlockRange().Remove(dead); + } + for (GenTree* add : addresses) + { + ReplaceCarryAddressAdd(add); + } + // The countdown is integer arithmetic, but must use LEA too: DEC would + // destroy OF before the next iteration consumes the ADOX carry. + assert(countStore->gtGetOp1()->OperIs(GT_LEA)); + GenTree* jump = + new (m_compiler, GT_JCMP) GenTreeOpCC(GT_JCMP, TYP_VOID, GenCondition::NE, countRead, cmp->gtGetOp2()); + jump->gtFlags |= GTF_ORDER_SIDEEFF; + BlockRange().InsertBefore(branch, jump); + BlockRange().Remove(cmp); + BlockRange().Remove(branch); +} + +//------------------------------------------------------------------------ +// TryLowerMultiplyCarryLoop: Match two low-word ADD/high-word ADC pairs +// following one unsigned widening multiply. The full-word recurrence is +// carry' = productHigh + CF + OF. +// ADCX adds the old high word and destination; ADOX adds the product low word. +// Both flags start at zero and are drained into the final high word on exit. +// Returns true if an ADX loop was created. +bool Lowering::TryLowerMultiplyCarryLoop() +{ + BasicBlock* block = m_block; + BasicBlock* entry = FindCarryLoopEntry(block); + if (entry == nullptr) + { + return false; + } + + GenTree* branch = BlockRange().LastNode(); + GenTree* cmp = branch->gtPrev; + if (!branch->OperIs(GT_JCC) || branch->AsCC()->gtCondition.GetCode() != GenCondition::NE || cmp == nullptr || + !cmp->OperIs(GT_CMP) || !cmp->gtGetOp1()->OperIs(GT_LCL_VAR) || !cmp->gtGetOp1()->TypeIs(TYP_INT) || + !cmp->gtGetOp2()->IsIntegralConst(0)) + { + return false; + } + + GenTree* adcs[2] = {}; + GenTree* product = nullptr; + GenTree* productStore = nullptr; + unsigned count = 0; + unsigned adcCount = 0; + for (GenTree* node : BlockRange()) + { + if (++count > 256) + { + return false; + } + if (node->OperIs(GT_ADD_CARRY)) + { + if (adcCount == 2 || !node->TypeIs(TYP_LONG) || node->gtSetFlags() || + !node->gtGetOp1()->OperIs(GT_LCL_VAR) || !node->gtGetOp2()->IsIntegralConst(0)) + { + return false; + } + adcs[adcCount++] = node; + } + if (node->OperIs(GT_HWINTRINSIC)) + { + if (product != nullptr || node->AsHWIntrinsic()->GetHWIntrinsicId() != NI_X86Base_X64_BigMul || + node->AsHWIntrinsic()->GetSimdBaseType() != TYP_ULONG) + { + return false; + } + product = node; + } + } + if (adcCount != 2 || product == nullptr) + { + return false; + } + LIR::Use productUse; + if (!BlockRange().TryGetUse(product, &productUse) || !productUse.User()->OperIs(GT_STORE_LCL_VAR)) + { + return false; + } + productStore = productUse.User(); + LclVarDsc* fields = m_compiler->lvaGetDesc(productStore->AsLclVar()->GetLclNum()); + if (!fields->lvPromoted || fields->lvFieldCnt != 2) + { + return false; + } + unsigned lowLocal = fields->lvFieldLclStart; + unsigned highLocal = lowLocal + 1; + + // Resolve only a single, preceding definition with a single use. Changed + // intermediate values must not be observable outside this loop. + GenTree* first = adcs[0]->gtPrev; + GenTree* second = adcs[1]->gtPrev; + if (first == nullptr || second == nullptr || !first->OperIs(GT_ADD) || !second->OperIs(GT_ADD) || + !first->TypeIs(TYP_LONG) || !second->TypeIs(TYP_LONG) || first->gtOverflow() || second->gtOverflow() || + !first->gtSetFlags() || !second->gtSetFlags() || !first->gtGetOp1()->OperIs(GT_LCL_VAR) || + !first->gtGetOp2()->OperIs(GT_LCL_VAR) || !second->gtGetOp1()->OperIs(GT_LCL_VAR) || + !second->gtGetOp2()->OperIs(GT_LCL_VAR) || adcs[0]->gtGetOp1()->AsLclVar()->GetLclNum() != highLocal) + { + return false; + } + GenTree* highStore = FindMultiplyCarryStore(adcs[1]->gtGetOp1()); + if (highStore == nullptr || highStore->gtGetOp1() != adcs[0]) + { + return false; + } + GenTree* lowRead = first->gtGetOp1(); + GenTree* destination = first->gtGetOp2(); + if (lowRead->AsLclVar()->GetLclNum() != lowLocal) + { + std::swap(lowRead, destination); + } + if (lowRead->AsLclVar()->GetLclNum() != lowLocal) + { + return false; + } + GenTree* partial = second->gtGetOp1(); + GenTree* carryRead = second->gtGetOp2(); + GenTree* partialStore = FindMultiplyCarryStore(partial); + if (partialStore == nullptr || partialStore->gtGetOp1() != first) + { + std::swap(partial, carryRead); + partialStore = FindMultiplyCarryStore(partial); + } + if (partialStore == nullptr || partialStore->gtGetOp1() != first) + { + return false; + } + GenTree* carryStore = FindMultiplyCarryStore(carryRead, true); + if (carryStore == nullptr || !carryStore->gtGetOp1()->OperIs(GT_LCL_VAR)) + { + return false; + } + GenTree* finalHighStore = FindMultiplyCarryStore(carryStore->gtGetOp1()); + if (finalHighStore == nullptr || finalHighStore->gtGetOp1() != adcs[1]) + { + return false; + } + + GenTree* countRead = cmp->gtGetOp1(); + unsigned countLocal = countRead->AsLclVar()->GetLclNum(); + GenTree* countStore = nullptr; + for (GenTree* node = cmp->gtPrev; node != nullptr; node = node->gtPrev) + { + if (node->OperIs(GT_STORE_LCL_VAR) && node->AsLclVar()->GetLclNum() == countLocal) + { + countStore = node; + break; + } + } + if (countStore == nullptr) + { + return false; + } + GenTree* decrement = countStore->gtGetOp1(); + if (!decrement->OperIs(GT_ADD) || !decrement->TypeIs(TYP_INT) || decrement->gtSetFlags() || + decrement->gtOverflow() || !decrement->gtGetOp1()->OperIs(GT_LCL_VAR) || + decrement->gtGetOp1()->AsLclVar()->GetLclNum() != countLocal || !decrement->gtGetOp2()->IsIntegralConst(-1)) + { + return false; + } + + jitstd::vector addresses(m_compiler->getAllocator(CMK_Lower)); + jitstd::vector deadNodes(m_compiler->getAllocator(CMK_Lower)); + bool beforeFirst = true; + bool sawCarryRead = false; + bool sawCarryStore = false; + bool productDefined = false; + for (GenTree* node : BlockRange()) + { + if (node == productStore) + { + productDefined = true; + } + if ((node == lowRead || node == adcs[0]->gtGetOp1()) && !productDefined) + { + return false; + } + if (node->OperIsLocal()) + { + unsigned local = node->AsLclVarCommon()->GetLclNum(); + // The tuple fields must still be exactly this product's low/high + // words, with no intervening field assignment or extra observer. + if (((local == lowLocal) && (node != lowRead)) || ((local == highLocal) && (node != adcs[0]->gtGetOp1())) || + ((local == productStore->AsLclVar()->GetLclNum()) && (node != productStore))) + { + return false; + } + } + if (node == first) + { + beforeFirst = false; + } + if (node == carryRead) + { + sawCarryRead = true; + } + if (node == carryStore) + { + sawCarryStore = true; + if (!sawCarryRead) + { + return false; + } + } + if (node == product && !beforeFirst) + { + return false; + } + if (node == first || node == second || node == adcs[0] || node == adcs[1] || node == product || node == cmp || + node == branch) + { + continue; + } + switch (node->OperGet()) + { + case GT_LCL_VAR: + case GT_STORE_LCL_VAR: + case GT_IL_OFFSET: + case GT_LEA: + break; + case GT_IND: + case GT_STOREIND: + if (!node->TypeIs(TYP_LONG) || (node->gtFlags & GTF_IND_VOLATILE) != 0) + { + + return false; + } + break; + case GT_CNS_INT: + if (!node->isContained()) + { + GenTree* deadStore = FindDeadCarryConstantStore(node); + if (deadStore == nullptr) + { + return false; + } + deadNodes.push_back(node); + deadNodes.push_back(deadStore); + } + break; + case GT_ADD: + if (node->gtSetFlags() || node->gtOverflow() || !node->TypeIs(TYP_INT, TYP_LONG, TYP_BYREF) || + !node->gtGetOp1()->OperIs(GT_LCL_VAR) || !node->gtGetOp2()->IsIntCnsFitsInI32()) + { + + return false; + } + addresses.push_back(node); + break; + default: + + return false; + } + } + if (!sawCarryStore) + { + return false; + } + + product->gtFlags |= GTF_HW_MULX; + JITDUMP("Keeping multiply-accumulate CF and OF chains around " FMT_BB "\n", block->bbNum); + BasicBlock* exit = block->GetFalseTarget(); + BasicBlock* seed = m_compiler->fgSplitEdge(entry, block); + BasicBlock* finish = m_compiler->fgSplitEdge(block, exit); + m_compiler->fgInvalidateDfsTree(); + GenTree* initialize = new (m_compiler, GT_ADX_SEED) GenTree(GT_ADX_SEED, TYP_VOID); + LIR::AsRange(seed).InsertAtEnd(initialize); + unsigned carryLocal = carryRead->AsLclVar()->GetLclNum(); + GenTree* finalRead = m_compiler->gtNewLclvNode(carryLocal, TYP_LONG); + GenTree* drain = m_compiler->gtNewOperNode(GT_ADX_DRAIN, TYP_LONG, finalRead); + GenTree* finalStore = m_compiler->gtNewStoreLclVarNode(carryLocal, drain); + LIR::AsRange(finish).InsertAtEnd(finalRead, drain, finalStore); + + for (GenTree* dead : deadNodes) + { + BlockRange().Remove(dead); + } + BlockRange().Remove(carryRead); + BlockRange().InsertBefore(first, carryRead); + first->AsOp()->gtOp1 = carryRead; + first->AsOp()->gtOp2 = destination; + second->AsOp()->gtOp1 = partial; + second->AsOp()->gtOp2 = lowRead; + first->ChangeOper(GT_ADCX); + second->ChangeOper(GT_ADOX); + for (GenTree* adc : adcs) + { + LIR::Use use; + bool found = BlockRange().TryGetUse(adc, &use); + assert(found); + use.ReplaceWith(adc->gtGetOp1()); + BlockRange().Remove(adc->gtGetOp2()); + BlockRange().Remove(adc); + } + for (GenTree* add : addresses) + { + ReplaceCarryAddressAdd(add); + } + // The countdown is integer arithmetic, but must use LEA too: DEC would + // destroy OF before the next iteration consumes the ADOX carry. + assert(countStore->gtGetOp1()->OperIs(GT_LEA)); + GenTree* jump = + new (m_compiler, GT_JCMP) GenTreeOpCC(GT_JCMP, TYP_VOID, GenCondition::NE, countRead, cmp->gtGetOp2()); + jump->gtFlags |= GTF_ORDER_SIDEEFF; + BlockRange().InsertBefore(branch, jump); + BlockRange().Remove(cmp); + BlockRange().Remove(branch); + return true; +} +#endif // TARGET_AMD64 + +//------------------------------------------------------------------------ +// LowerFullAdders: Recognize full adders and subtractors after their carry/borrow comparisons have been lowered. +// Prove the loop-carried bit by induction over every definition of its local: constants are 0/1 and +// arithmetic definitions are other recognized full adders/subtractors. Boolean definitions +// (including widened comparisons in carry/borrow chains) are also accepted as bits. +// Returns true when a full adder or subtractor was folded. +bool Lowering::LowerFullAdders() +{ + // Bound the whole-method definition analysis and the number of candidates. + unsigned nodeCount = 0; + for (BasicBlock* block : m_compiler->Blocks()) + { + for (GenTree* node : LIR::AsRange(block)) + { + if (++nodeCount > 2048) + { + return false; + } + } + } + struct FullAdder + { + BasicBlock* block; + GenTree* output; + GenTree* first; + GenTree* second; + GenTree* firstStore; + GenTree* firstRead; + GenTree* carryStore; + GenTree* carryRead; + GenTree* carryCast; + GenTree* carryCC; + GenTree* input; + unsigned carryLocal; + }; + jitstd::vector adders(m_compiler->getAllocator(CMK_Lower)); + + for (BasicBlock* block : m_compiler->Blocks()) + { + m_block = block; + for (GenTree* output : LIR::AsRange(block)) + { + if (!output->OperIs(GT_ADD_CARRY, GT_ADD_BORROW) || output->gtSetFlags() || + !output->gtGetOp2()->IsIntegralConst(0) || !output->gtGetOp1()->OperIs(GT_LCL_VAR, GT_SETCC)) + { + continue; + } + bool subtract = output->OperIs(GT_ADD_BORROW); + GenTree* second = output->gtPrev; + if ((second == nullptr) || !second->OperIs(subtract ? GT_SUB : GT_ADD) || second->gtOverflow() || + !second->gtSetFlags() || !second->gtGetOp1()->OperIs(GT_LCL_VAR) || + !second->gtGetOp2()->OperIs(GT_LCL_VAR)) + { + continue; + } + + // Require a single store and a single read of each intermediate. + auto findStore = [&](GenTree* read) -> GenTree* { + unsigned local = read->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (!dsc->lvTracked || dsc->IsAddressExposed()) + { + return nullptr; + } + GenTree* store = nullptr; + int reads = 0; + for (GenTree* node : LIR::AsRange(block)) + { + if (dsc->lvIsStructField && node->OperIsLocal() && + node->AsLclVarCommon()->GetLclNum() == dsc->lvParentLcl) + { + return nullptr; + } + if (node->OperIsLocal() && node->AsLclVarCommon()->GetLclNum() == local) + { + if (node->OperIs(GT_STORE_LCL_VAR) && store == nullptr) + { + store = node; + } + else if ((node == read) && (store != nullptr)) + { + reads++; + } + else + { + return nullptr; + } + } + } + return (reads == 1) && IsCarryLocalDeadOnExit(local, block) ? store : nullptr; + }; + GenTree* carryRead = nullptr; + GenTree* carryStore = nullptr; + GenTree* cc = output->gtGetOp1(); + if (cc->OperIs(GT_LCL_VAR)) + { + carryRead = cc; + carryStore = findStore(carryRead); + if (carryStore == nullptr) + { + continue; + } + cc = carryStore->gtGetOp1(); + } + GenTree* cast = nullptr; + if (cc->OperIs(GT_CAST) && !cc->gtOverflow() && cc->TypeIs(TYP_LONG)) + { + cast = cc; + cc = cast->gtGetOp1(); + } + GenCondition condition = GenCondition::C; +#ifdef TARGET_ARM64 + if (subtract) + { + condition = GenCondition::NC; + } +#endif + if (!cc->OperIs(GT_SETCC) || (cc->AsCC()->gtCondition.GetCode() != condition.GetCode())) + { + continue; + } + GenTree* first = cc->gtPrev; + if (first == nullptr || !first->OperIs(subtract ? GT_SUB : GT_ADD) || !first->gtSetFlags() || + first->gtOverflow() || first->TypeGet() != second->TypeGet() || second->TypeGet() != output->TypeGet()) + { + continue; + } + + for (unsigned i = 0; i < (subtract ? 1u : 2u); i++) + { + GenTree* read = i == 0 ? second->gtGetOp1() : second->gtGetOp2(); + GenTree* input = i == 0 ? second->gtGetOp2() : second->gtGetOp1(); + GenTree* store = findStore(read); + if (store == nullptr || store->gtGetOp1() != first) + { + continue; + } + unsigned local = input->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (dsc->lvIsParam || dsc->IsAddressExposed()) + { + continue; + } + LIR::Use use; + if (!BlockRange().TryGetUse(output, &use) || !use.User()->OperIs(GT_STORE_LCL_VAR) || + use.User()->AsLclVar()->GetLclNum() != local) + { + continue; + } + if (adders.size() == 32) + { + return false; + } + adders.push_back( + {block, output, first, second, store, read, carryStore, carryRead, cast, cc, input, local}); + break; + } + } + } + + // Validate every candidate before changing any definitions. + jitstd::vector proven(m_compiler->getAllocator(CMK_Lower)); + jitstd::vector examined(m_compiler->getAllocator(CMK_Lower)); + bool changed = false; + for (const FullAdder& adder : adders) + { + bool alreadyExamined = false; + for (unsigned local : examined) + { + alreadyExamined |= local == adder.carryLocal; + } + if (alreadyExamined) + { + continue; + } + examined.push_back(adder.carryLocal); + bool valid = true; + bool initialized = false; + LclVarDsc* carryDsc = m_compiler->lvaGetDesc(adder.carryLocal); + for (BasicBlock* block : m_compiler->Blocks()) + { + for (GenTree* node : LIR::AsRange(block)) + { + if (carryDsc->lvIsStructField && node->OperIsLocal() && + node->AsLclVarCommon()->GetLclNum() == carryDsc->lvParentLcl) + { + valid = false; + break; + } + if (!node->OperIsLocal() || node->AsLclVarCommon()->GetLclNum() != adder.carryLocal) + { + continue; + } + if (node->OperIs(GT_LCL_VAR)) + { + continue; + } + if (!node->OperIs(GT_STORE_LCL_VAR)) + { + valid = false; + break; + } + GenTree* value = node->gtGetOp1(); + if (value->IsIntegralConst(0) || value->IsIntegralConst(1)) + { + initialized = true; + continue; + } + { + // The first limb of a straight-line chain initializes carry or borrow + // from a comparison. Subtraction tails can also redefine borrow as + // (limb == 0). Both are proven bits, including after widening. + GenTree* bit = value; + if (bit->OperIs(GT_CAST) && !bit->gtOverflow() && bit->TypeIs(TYP_LONG) && + bit->gtGetOp1()->TypeIs(TYP_INT)) + { + bit = bit->gtGetOp1(); + } + if (bit->OperIs(GT_SETCC) || bit->OperIsCompare()) + { + initialized = true; + continue; + } + } + bool fullAdder = false; + for (const FullAdder& definition : adders) + { + if (definition.carryLocal == adder.carryLocal && definition.output == value) + { + fullAdder = true; + } + } + if (!fullAdder) + { + valid = false; + } + if (!valid) + { + break; + } + } + if (!valid) + { + break; + } + } + if (valid && initialized) + { + proven.push_back(adder.carryLocal); + } + } + + for (const FullAdder& adder : adders) + { + bool valid = false; + for (unsigned local : proven) + { + if (local == adder.carryLocal) + { + valid = true; + } + } + if (!valid) + { + continue; + } + m_block = adder.block; + changed = true; + JITDUMP("Folding full adder [%06u] with one-bit carry V%02u\n", adder.second->gtTreeID, adder.carryLocal); + bool subtract = adder.first->OperIs(GT_SUB); + GenCondition condition = GenCondition::C; + GenTree* carryConstant = adder.output->gtGetOp2(); + carryConstant->ClearContained(); + carryConstant->ClearRegOptional(); + adder.input->ClearContained(); + adder.input->ClearRegOptional(); +#ifdef TARGET_AMD64 + // Adding -1 sets CF exactly when the proven carry bit is 1. + carryConstant->AsIntCon()->SetIconValue(-1); + GenTree* restore = m_compiler->gtNewOperNode(GT_ADD, adder.input->TypeGet(), adder.input, carryConstant); + restore->SetUnusedValue(); +#else + // Addition needs C = carry; subtraction needs C = !borrow. + carryConstant->AsIntCon()->SetIconValue(subtract ? 0 : 1); + GenTree* restore = m_compiler->gtNewOperNode(GT_CMP, TYP_VOID, subtract ? carryConstant : adder.input, + subtract ? adder.input : carryConstant); + if (subtract) + { + condition = GenCondition::NC; + } +#endif + restore->gtFlags |= GTF_SET_FLAGS; +#ifdef TARGET_AMD64 + restore->gtFlags |= GTF_ADD_CARRY_FLAGS; +#endif + BlockRange().Remove(carryConstant); + BlockRange().InsertBefore(adder.second, carryConstant); + BlockRange().InsertBefore(adder.second, restore); +#ifdef TARGET_ARM64 + if (!subtract) +#endif + { + MakeSrcContained(restore, carryConstant); + } + adder.second->ChangeOper(subtract ? GT_SUB_BORROW : GT_ADD_CARRY); + adder.second->AsOp()->gtOp1 = adder.first->gtGetOp1(); + adder.second->AsOp()->gtOp2 = adder.first->gtGetOp2(); + for (GenTree* operand : adder.second->Operands()) + { + operand->ClearContained(); + operand->ClearRegOptional(); + } +#ifdef TARGET_ARM64 + // CSET already defines the entire register as 0 or 1. + GenTree* cc = m_compiler->gtNewCC(GT_SETCC, adder.output->TypeGet(), condition); + GenTree* carryCast = nullptr; +#else + GenTree* cc = m_compiler->gtNewCC(GT_SETCC, TYP_INT, condition); + GenTree* carryCast = adder.carryCast; + assert((carryCast != nullptr) || adder.output->TypeIs(TYP_INT)); +#endif + BlockRange().InsertAfter(adder.second, cc); + if (adder.carryCast != nullptr) + { + BlockRange().Remove(adder.carryCast); + } + if (carryCast != nullptr) + { + carryCast->AsCast()->CastOp() = cc; + BlockRange().InsertAfter(cc, carryCast); + } + LIR::Use use; + bool found = BlockRange().TryGetUse(adder.output, &use); + assert(found); + GenTree* outputStore = use.User(); + use.ReplaceWith(carryCast != nullptr ? carryCast : cc); + for (GenTree* node : {adder.first, adder.firstStore, adder.firstRead, adder.carryStore, adder.carryRead, + adder.carryCC, adder.output}) + { + if (node != nullptr) + { + BlockRange().Remove(node); + } + } + ContainCheckBinary(adder.second->AsOp()); + TryLowerCarryLoop(adder.second, restore, cc, carryCast, outputStore, adder.carryLocal); + } + return changed; +} + +//------------------------------------------------------------------------ +// LowerCarryChains: Fold final-limb consumers, then remove single-use carry +// materialization and restoration between arithmetic stages in one block. +// The latter step only removes nodes; it does not move memory accesses. +// +void Lowering::LowerCarryChains() +{ + unsigned budget = 2048; + for (BasicBlock* block : m_compiler->Blocks()) + { + m_block = block; + for (GenTree* node = BlockRange().FirstNode(); node != nullptr;) + { + if (budget-- == 0) + { + return; + } + GenTree* next = node->gtNext; + // Full-adder lowering creates SETCCs after the ordinary lowering + // walk. Let the last limb consume such a carry directly as well. + if (node->OperIs(GT_ADD, GT_SUB)) + { + TryLowerAddCarry(node->AsOp()); + } + node = next; + } + + for (GenTree* consumer : BlockRange()) + { + if (!consumer->OperIs(GT_ADD_CARRY, GT_SUB_BORROW) || !consumer->gtSetFlags()) + { + continue; + } + bool subtract = consumer->OperIs(GT_SUB_BORROW); + GenTree* restore = consumer->gtPrev; + unsigned restoreBudget = 64; + // Folding the final limb can insert its operand reads between a + // stage's restore and arithmetic. These register copies do not + // change the restored flags. + while (restore != nullptr && restoreBudget != 0 && !restore->gtSetFlags() && + !restore->OperConsumesFlags() && + (restore->OperIs(GT_IL_OFFSET) || + (restore->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR) && restore->TypeIs(TYP_INT, TYP_LONG, TYP_BYREF)))) + { + restore = restore->gtPrev; + restoreBudget--; + } + if (restore == nullptr || restoreBudget == 0 || !restore->gtSetFlags()) + { + continue; + } + GenTree* read; + GenTree* constant; +#ifdef TARGET_AMD64 + if (!restore->OperIs(GT_ADD) || !restore->IsUnusedValue() || restore->gtOverflow()) + { + continue; + } + read = restore->gtGetOp1(); + constant = restore->gtGetOp2(); + if (!constant->IsIntegralConst(-1)) + { + continue; + } +#else + if (!restore->OperIs(GT_CMP)) + { + continue; + } + read = subtract ? restore->gtGetOp2() : restore->gtGetOp1(); + constant = subtract ? restore->gtGetOp1() : restore->gtGetOp2(); + if (!constant->IsIntegralConst(subtract ? 0 : 1)) + { + continue; + } +#endif + if (!read->OperIs(GT_LCL_VAR)) + { + continue; + } + unsigned local = read->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (!dsc->lvTracked || dsc->IsAddressExposed() || dsc->lvIsStructField || + !IsCarryLocalDeadOnExit(local, block)) + { + continue; + } + GenTree* store = FindCarryLocalStore(read); + if (store == nullptr) + { + continue; + } + GenTree* cc = store->gtGetOp1(); + GenTree* cast = nullptr; + if (cc->OperIs(GT_CAST) && !cc->gtOverflow() && cc->CastToType() == TYP_LONG && + cc->gtGetOp1()->TypeIs(TYP_INT)) + { + cast = cc; + cc = cc->gtGetOp1(); + } + GenCondition condition = GenCondition::C; +#ifdef TARGET_ARM64 + if (subtract) + { + condition = GenCondition::NC; + } +#endif + if (!cc->OperIs(GT_SETCC) || cc->AsCC()->gtCondition.GetCode() != condition.GetCode()) + { + continue; + } + GenTree* producer = cc->gtPrev; + if (producer == nullptr || !producer->gtSetFlags() || + (producer->OperIs(GT_ADD, GT_SUB) && producer->gtOverflow()) || + !(subtract ? producer->OperIs(GT_SUB, GT_SUB_BORROW) : producer->OperIs(GT_ADD, GT_ADD_CARRY))) + { + continue; + } + + // Only register copies and IL markers may cross the flags edge. + // In particular, constants, address arithmetic and loads are not + // assumed flags-preserving after containment and register allocation. + GenTree* crossing = producer->gtNext; + unsigned scan = 64; + for (; crossing != nullptr && crossing != restore && scan != 0; crossing = crossing->gtNext, scan--) + { + if (crossing == cc || crossing == cast || crossing == constant) + { + continue; + } + if (crossing->gtSetFlags() || crossing->OperConsumesFlags() || + !(crossing->OperIs(GT_IL_OFFSET) || (crossing->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR) && + crossing->TypeIs(TYP_INT, TYP_LONG, TYP_BYREF)))) + { + break; + } + } + if (crossing != restore) + { + continue; + } + + // The definition may be removed only if this is its sole read up + // to the next definition, and it cannot escape through a live-out. + GenTree* use = store->gtNext; + scan = 64; + for (; use != nullptr && scan != 0; use = use->gtNext, scan--) + { + if (use->OperIsLocal() && use->AsLclVarCommon()->GetLclNum() == local) + { + if (use->OperIs(GT_STORE_LCL_VAR)) + { + break; + } + if (use != read) + { + break; + } + } + } + if (scan == 0 || (use != nullptr && !use->OperIs(GT_STORE_LCL_VAR))) + { + continue; + } + JITDUMP("Forwarding carry flags from [%06u] to [%06u]\n", producer->gtTreeID, consumer->gtTreeID); + for (GenTree* removed : {cc, cast, store, read, constant, restore}) + { + if (removed != nullptr) + { + BlockRange().Remove(removed); + } + } + } + } +} + +//------------------------------------------------------------------------ +// TryLowerCarryLoop: Keep carry or borrow in flags around a simple counted loop. More general control flow +// retains the explicit carry/borrow representation produced by LowerFullAdders. +void Lowering::TryLowerCarryLoop( + GenTree* sum, GenTree* restore, GenTree* cc, GenTree* cast, GenTree* store, unsigned carryLocal) +{ + BasicBlock* block = m_block; + if (m_compiler->compHndBBtabCount != 0 || !block->KindIs(BBJ_COND) || + (block->GetTrueTarget() != block && block->GetFalseTarget() != block) || + (block->GetTrueTarget() == block->GetFalseTarget())) + { + return; + } + BasicBlock* entry = FindCarryLoopEntry(block); + if (entry == nullptr) + { + return; + } + GenTree* branch = BlockRange().LastNode(); +#ifdef TARGET_AMD64 + GenTree* cmp = branch->gtPrev; + if (!branch->OperIs(GT_JCC) || cmp == nullptr || !cmp->OperIs(GT_CMP) || !cmp->gtGetOp1()->OperIs(GT_LCL_VAR) || + !cmp->gtGetOp2()->IsIntegralConst(0)) + { + return; + } + GenCondition::Code condition = branch->AsCC()->gtCondition.GetCode(); +#else + // CBZ/CBNZ test the counter without changing NZCV. + GenTree* cmp = branch; + if (!branch->OperIs(GT_JCMP) || !cmp->gtGetOp1()->OperIs(GT_LCL_VAR) || !cmp->gtGetOp2()->IsIntegralConst(0)) + { + return; + } + GenCondition::Code condition = branch->AsOpCC()->gtCondition.GetCode(); +#endif + if (condition != GenCondition::EQ && condition != GenCondition::NE) + { + return; + } + unsigned countLocal = cmp->gtGetOp1()->AsLclVar()->GetLclNum(); + GenTree* countStore = cmp->gtPrev; + while (countStore != nullptr && !countStore->OperIs(GT_STORE_LCL_VAR)) + { + countStore = countStore->gtPrev; + } + if (countStore == nullptr || countStore->AsLclVar()->GetLclNum() != countLocal) + { + return; + } + GenTree* decrement = countStore->gtGetOp1(); + if (!decrement->OperIs(GT_ADD, GT_SUB) || decrement->gtSetFlags() || decrement->gtOverflow() || + !decrement->gtGetOp1()->OperIs(GT_LCL_VAR) || decrement->gtGetOp1()->AsLclVar()->GetLclNum() != countLocal || + !decrement->gtGetOp2()->IsIntegralConst(decrement->OperIs(GT_ADD) ? -1 : 1)) + { + return; + } + + bool sumBeforeDecrement = false; + for (GenTree* node = sum->gtNext; node != nullptr; node = node->gtNext) + { + if (node == decrement) + { + sumBeforeDecrement = true; + break; + } + } + if (!sumBeforeDecrement) + { + return; + } + + // Permit only instructions that preserve CF, plus additions we can turn + // into LEA and the loop decrement. In particular, reject calls, bounds + // checks, write barriers, and any other flag consumers or producers. + jitstd::vector addresses(m_compiler->getAllocator(CMK_Lower)); + jitstd::vector deadStores(m_compiler->getAllocator(CMK_Lower)); + for (GenTree* node : BlockRange()) + { + if (node == sum || node == restore || node == restore->gtGetOp1() || node == restore->gtGetOp2() || + node == cc || node == cast || node == cmp || node == decrement) + { + continue; + } + if (node->OperIsLocal() && node->AsLclVarCommon()->GetLclNum() == carryLocal && node != restore->gtGetOp1() && + node != restore->gtGetOp2() && node != store) + { + return; + } + switch (node->OperGet()) + { + case GT_LCL_VAR: + case GT_STORE_LCL_VAR: + case GT_IL_OFFSET: + case GT_LEA: + break; + case GT_CNS_INT: + if (!node->isContained() && node != cmp->gtGetOp2() && node != decrement->gtGetOp2()) + { + GenTree* deadStore = FindDeadCarryConstantStore(node); + if (deadStore == nullptr) + { + return; + } + deadStores.push_back(node); + deadStores.push_back(deadStore); + } + break; + case GT_IND: + case GT_STOREIND: + if (!node->AsIndir()->TypeIs(TYP_INT, TYP_LONG) || (node->gtFlags & GTF_IND_VOLATILE) != 0) + { + return; + } + break; + case GT_CAST: + // Widening an index preserves flags on both targets. The + // addressing IV can remain live for a subsequent borrow tail. + if (node->gtOverflow() || !node->TypeIs(TYP_LONG) || !node->gtGetOp1()->TypeIs(TYP_INT)) + { + return; + } + break; +#ifdef TARGET_ARM64 + case GT_LSH: + case GT_BFIZ: + // Unlike x64 shifts, A64 index scaling preserves NZCV. + if (node->gtSetFlags() || !node->TypeIs(TYP_INT, TYP_LONG)) + { + return; + } + break; +#endif + case GT_JCC: +#ifdef TARGET_ARM64 + case GT_JCMP: +#endif + if (node != branch) + { + return; + } + break; + case GT_ADD: + if (node->gtSetFlags() || node->gtOverflow() || !node->TypeIs(TYP_INT, TYP_LONG, TYP_BYREF)) + { + return; + } +#ifdef TARGET_ARM64 + if (node->TypeIs(TYP_LONG, TYP_BYREF) && node->gtGetOp1()->OperIs(GT_LCL_VAR) && + node->gtGetOp2()->OperIs(GT_LSH) && node->gtGetOp2()->TypeIs(TYP_LONG) && + node->gtGetOp2()->gtGetOp1()->TypeIs(TYP_LONG) && + node->gtGetOp2()->gtGetOp2()->OperIs(GT_CNS_INT) && + node->gtGetOp2()->gtGetOp2()->AsIntCon()->IconValue() >= 0 && + node->gtGetOp2()->gtGetOp2()->AsIntCon()->IconValue() <= 3) + { + addresses.push_back(node); + break; + } +#endif + if (!node->gtGetOp1()->OperIs(GT_LCL_VAR) || !node->gtGetOp2()->OperIs(GT_LCL_VAR, GT_CNS_INT)) + { +#ifdef TARGET_ARM64 + break; // An ordinary A64 ADD also preserves flags. +#else + return; +#endif + } + if (node->gtGetOp2()->OperIs(GT_CNS_INT) && !FitsIn(node->gtGetOp2()->AsIntCon()->IconValue())) + { + return; + } + addresses.push_back(node); + break; + default: + return; + } + } + + JITDUMP("Keeping full-adder carry in flags around " FMT_BB "\n", block->bbNum); + BasicBlock* exit = block->GetTrueTarget() == block ? block->GetFalseTarget() : block->GetTrueTarget(); + BasicBlock* seed = m_compiler->fgSplitEdge(entry, block); + BasicBlock* finish = m_compiler->fgSplitEdge(block, exit); + m_compiler->fgInvalidateDfsTree(); + for (GenTree* node : {restore->gtGetOp1(), restore->gtGetOp2(), restore}) + { + BlockRange().Remove(node); + LIR::AsRange(seed).InsertAtEnd(node); + } + for (GenTree* node : {cc, cast, store}) + { + if (node == nullptr) + { + continue; + } + BlockRange().Remove(node); + LIR::AsRange(finish).InsertAtEnd(node); + } + for (GenTree* node : deadStores) + { + BlockRange().Remove(node); + } + for (GenTree* address : addresses) + { + GenTree* base = address->gtGetOp1(); + GenTree* index = address->gtGetOp2(); + int offset = 0; + unsigned scale = 1; +#ifdef TARGET_ARM64 + if (index->OperIs(GT_LSH)) + { + GenTree* shift = index; + scale = 1u << shift->gtGetOp2()->AsIntCon()->IconValue(); + index = shift->gtGetOp1(); + BlockRange().Remove(shift->gtGetOp2()); + BlockRange().Remove(shift); + } +#endif + if (index->OperIs(GT_CNS_INT)) + { + offset = (int)index->AsIntCon()->IconValue(); + BlockRange().Remove(index); + index = nullptr; + } + GenTree* lea = new (m_compiler, GT_LEA) GenTreeAddrMode(address->TypeGet(), base, index, scale, offset); + base->ClearContained(); + base->ClearRegOptional(); + if (index != nullptr) + { + index->ClearContained(); + index->ClearRegOptional(); + } + LIR::Use use; + bool found = BlockRange().TryGetUse(address, &use); + assert(found); + BlockRange().InsertBefore(address, lea); + use.ReplaceWith(lea); + BlockRange().Remove(address); + } +#ifdef TARGET_ARM64 + // Strength reduction may advance the byte offset before computing the sum. + // Delay an independent offset update until after the stores so their address + // can use the same offset register as the loads. + GenTree* lastStore = nullptr; + for (GenTree* node : BlockRange()) + { + if (node->OperIs(GT_STOREIND)) + { + lastStore = node; + } + } + jitstd::vector updates(m_compiler->getAllocator(CMK_Lower)); + for (GenTree* node = BlockRange().FirstNode(); node != nullptr && node != lastStore; node = node->gtNext) + { + if (!node->OperIs(GT_STORE_LCL_VAR) || !node->TypeIs(TYP_LONG) || !node->gtGetOp1()->OperIs(GT_LEA)) + { + continue; + } + GenTreeAddrMode* address = node->gtGetOp1()->AsAddrMode(); + unsigned local = node->AsLclVar()->GetLclNum(); + if (address->Index() != nullptr || address->Base() == nullptr || !address->Base()->OperIs(GT_LCL_VAR) || + address->Base()->AsLclVar()->GetLclNum() != local || m_compiler->lvaGetDesc(local)->IsAddressExposed()) + { + continue; + } + updates.push_back(node); + } + if (lastStore != nullptr) + { + for (GenTree* update : updates) + { + bool isClosed; + LIR::ReadOnlyRange range = BlockRange().GetTreeRange(update, &isClosed); + if (isClosed && IsRangeInvariantInRange(range.FirstNode(), update, lastStore->gtNext, nullptr)) + { + LIR::Range moved = BlockRange().Remove(range.FirstNode(), update); + BlockRange().InsertAfter(lastStore, std::move(moved)); + } + } + } +#endif + + // Expose single-use load/address temporaries to containment now that the + // carry comparisons no longer require their original local values. + auto forward = [&](GenTree* parent, GenTree* read, genTreeOps oper) { + if (!read->OperIs(GT_LCL_VAR)) + { + return; + } + unsigned local = read->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (!dsc->lvTracked || dsc->IsAddressExposed() || !IsCarryLocalDeadOnExit(local, block)) + { + return; + } + GenTree* definition = nullptr; + for (GenTree* node : BlockRange()) + { + if ((node == read) && (definition == nullptr)) + { + return; + } + if (!node->OperIsLocal() || node->AsLclVarCommon()->GetLclNum() != local || node == read) + { + continue; + } + if (definition != nullptr || !node->OperIs(GT_STORE_LCL_VAR)) + { + return; + } + definition = node; + } + if (definition == nullptr) + { + return; + } + GenTree* value = definition->gtGetOp1(); + if (!value->OperIs(oper) || !IsInvariantInRange(value, parent)) + { + return; + } + LIR::Use use; + bool found = BlockRange().TryGetUse(read, &use); + assert(found); + use.ReplaceWith(value); + BlockRange().Remove(read); + BlockRange().Remove(definition); + }; + forward(sum, sum->gtGetOp2(), GT_IND); + ContainCheckBinary(sum->AsOp()); + for (GenTree* node : BlockRange()) + { + if (node->OperIs(GT_STOREIND)) + { + forward(node, node->gtGetOp1(), GT_LEA); + ContainCheckStoreIndir(node->AsStoreInd()); + } + } + +#ifdef TARGET_AMD64 + // ADD(-1) selects DEC: update ZF for the branch while preserving CF. + if (decrement->OperIs(GT_SUB)) + { + decrement->ChangeOper(GT_ADD); + decrement->gtGetOp2()->AsIntCon()->SetIconValue(-1); + } + decrement->gtFlags |= GTF_SET_FLAGS; + decrement->gtFlags &= ~GTF_ADD_CARRY_FLAGS; + ContainCheckBinary(decrement->AsOp()); + BlockRange().Remove(cmp->gtGetOp1()); + BlockRange().Remove(cmp->gtGetOp2()); + BlockRange().Remove(cmp); +#endif +} + +//------------------------------------------------------------------------ +// IsCarryOperandAvailable: Check whether an operand is available at a flags producer, +// or can be moved there without changing its value. Bound the scans to keep lowering linear. +bool Lowering::IsCarryOperandAvailable(GenTree* operand, GenTree* producer, GenTree** moveStart) +{ + *moveStart = nullptr; + GenTree* node = producer->gtPrev; + for (int budget = 64; (node != nullptr) && (budget > 0); node = node->gtPrev, budget--) + { + if (node == operand) + { + return true; + } + } + + if (operand->OperIs(GT_IND)) + { + // Move the load together with its address calculation. A closed range + // ensures that no intermediate value has a use left behind. Check all + // crossed nodes for aliasing, exception ordering, and local definitions. + bool isClosed; + LIR::ReadOnlyRange range = BlockRange().GetTreeRange(operand, &isClosed); + if (!isClosed) + { + return false; + } + + SideEffectSet effects; + node = operand; + int budget = 64; + for (; (node != nullptr) && (node != producer) && (budget > 0); node = node->gtPrev, budget--) + { + effects.AddNode(m_compiler, node); + if (node == range.FirstNode()) + { + break; + } + } + if ((node != range.FirstNode()) || (node == producer)) + { + return false; + } + + for (node = node->gtPrev; (node != nullptr) && (budget > 0); node = node->gtPrev, budget--) + { + if (effects.InterferesWith(m_compiler, node, true)) + { + return false; + } + if (node == producer) + { + *moveStart = range.FirstNode(); + return true; + } + } + return false; + } + + if (operand->OperIs(GT_CAST) && !operand->gtOverflow() && operand->TypeIs(TYP_LONG) && + operand->AsCast()->CastOp()->TypeIs(TYP_INT)) + { + GenTree* input = operand->AsCast()->CastOp(); + GenTree* moveInput; + if (!IsCarryOperandAvailable(input, producer, &moveInput)) + { + return false; + } + if (moveInput != nullptr) + { + // Move the input together with its widening cast. The recursive + // check proved a local/constant invariant, or a load's closed tree + // safe to move. Adjacency keeps the cast within that same range. + if (!input->OperIs(GT_LCL_VAR, GT_CNS_INT, GT_IND) || (input->gtNext != operand)) + { + return false; + } + *moveStart = moveInput; + return true; + } + } + else if (!operand->OperIs(GT_LCL_VAR, GT_CNS_INT) || + (operand->OperIs(GT_LCL_VAR) && + m_compiler->lvaGetDesc(operand->AsLclVar()->GetLclNum())->IsAddressExposed())) + { + return false; + } + + unsigned local = BAD_VAR_NUM; + unsigned parent = BAD_VAR_NUM; + if (operand->OperIs(GT_LCL_VAR)) + { + local = operand->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (dsc->lvIsStructField) + { + parent = dsc->lvParentLcl; + if (m_compiler->lvaGetDesc(parent)->IsAddressExposed()) + { + return false; + } + } + } + + node = producer->gtNext; + for (int budget = 64; (node != nullptr) && (budget > 0); node = node->gtNext, budget--) + { + if (node == operand) + { + *moveStart = operand; + return true; + } + // A promoted field can also be overwritten through its parent local. + if (local != BAD_VAR_NUM && node->OperIs(GT_STORE_LCL_VAR, GT_STORE_LCL_FLD) && + ((node->AsLclVarCommon()->GetLclNum() == local) || (node->AsLclVarCommon()->GetLclNum() == parent))) + { + return false; + } + } + return false; +} + +//------------------------------------------------------------------------ +// TryLowerAddCarry: Fold a materialized carry or borrow into arithmetic. Keep the flag consumer +// immediately after its producer, moving only operands whose values are available there. +bool Lowering::TryLowerAddCarry(GenTreeOp* add) +{ + // Comparison recognition must still run: it can create the first SETCC. + // Only folding an existing carry can skip the local-definition searches. + if (!m_blockMayHaveSetCC) + { + return false; + } + if (!add->TypeIs(TYP_INT, TYP_LONG) || add->gtOverflow() || add->gtSetFlags() || add->IsUnusedValue()) + { + return false; + } + + for (unsigned index = add->OperIs(GT_SUB) ? 1 : 0; index < 2; index++) + { + GenTree* value = index == 0 ? add->gtOp1 : add->gtOp2; + GenTree* other = index == 0 ? add->gtOp2 : add->gtOp1; + GenTree* carry = value; + GenTree* store = nullptr; + if (value->OperIs(GT_LCL_VAR)) + { + unsigned lclNum = value->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(lclNum); + if (!dsc->lvTracked || dsc->IsAddressExposed()) + { + continue; + } + + store = FindCarryLocalStore(value); + if (store == nullptr) + { + continue; + } + + carry = store->gtGetOp1(); + } + GenTree* cast = nullptr; + if (carry->OperIs(GT_CAST) && !carry->gtOverflow() && carry->CastToType() == TYP_LONG && + carry->AsCast()->CastOp()->TypeIs(TYP_INT)) + { + cast = carry; + carry = carry->AsCast()->CastOp(); + } + if (!carry->OperIs(GT_SETCC)) + { + continue; + } + + GenTree* producer = carry->gtPrev; + if ((producer == nullptr) || !producer->OperIs(GT_ADD, GT_ADD_CARRY, GT_SUB, GT_SUB_BORROW, GT_NEG) || + !producer->gtSetFlags()) + { + continue; + } + + bool borrow = producer->OperIs(GT_SUB, GT_SUB_BORROW, GT_NEG); + GenCondition condition = GenCondition::C; +#ifdef TARGET_ARM64 + if (borrow) + { + condition = GenCondition::NC; + } +#endif + if (carry->AsCC()->gtCondition.GetCode() != condition.GetCode() || (add->OperIs(GT_SUB) && !borrow)) + { + continue; + } + + // A direct SETCC operand has one use by the LIR value-node invariant. + if (store != nullptr) + { + if (!IsCarryLocalDeadOnExit(value->AsLclVar()->GetLclNum(), m_block)) + { + continue; + } + // Only spend the extended scan budget on a recognized carry. Keep + // the original local scan allowance, and share the extra work over + // the lowering phase so large unrolled blocks do not make this quadratic. + GenTree* node = store->gtNext; + unsigned scanned = 0; + bool singleUse = true; + for (; node != nullptr; node = node->gtNext, scanned++) + { + if (scanned >= 64) + { + if (m_carryUseScanBudget == 0) + { + break; + } + m_carryUseScanBudget--; + } + if (node->OperIsLocal() && (node != value) && + (node->AsLclVarCommon()->GetLclNum() == value->AsLclVar()->GetLclNum())) + { + singleUse = false; + break; + } + } + if (!singleUse || (node != nullptr)) + { + continue; + } + } + + bool combine = + other->OperIs(add->OperIs(GT_SUB) ? GT_SUB : GT_ADD) && !other->gtOverflow() && !other->gtSetFlags(); + bool negate = add->OperIs(GT_SUB) && other->OperIs(GT_NEG) && !other->gtSetFlags(); +#ifdef TARGET_ARM64 + // CINC handles high + borrow; arbitrary binary combinations need a + // separate addition and are left to the existing lowering. + if (borrow && add->OperIs(GT_ADD) && combine) + { + continue; + } +#endif + GenTree* operands[] = {(combine || negate) ? other->gtGetOp1() : other, combine ? other->gtGetOp2() : nullptr}; + GenTree* moveStart[2] = {}; + if (!IsCarryOperandAvailable(operands[0], producer, &moveStart[0]) || + (combine && !IsCarryOperandAvailable(operands[1], producer, &moveStart[1]))) + { + continue; + } + + // Inserting ADC/SBB changes flags. No later consumer may observe the + // original producer's flags before the next flags definition. + bool hasOtherFlagsConsumer = false; + for (GenTree* node = carry->gtNext; node != nullptr; node = node->gtNext) + { + if (node->OperConsumesFlags()) + { + hasOtherFlagsConsumer = true; + break; + } + if (node->gtSetFlags()) + { + break; + } + } + if (hasOtherFlagsConsumer) + { + continue; + } + + JITDUMP("Folding carry [%06u] into addition [%06u]\n", carry->gtTreeID, add->gtTreeID); + if (!combine) + { + operands[1] = m_compiler->gtNewZeroConNode(add->TypeGet()); + BlockRange().InsertBefore(producer, operands[1]); + } + for (unsigned i = 0; i < 2; i++) + { + if (moveStart[i] != nullptr) + { + LIR::Range range = BlockRange().Remove(moveStart[i], operands[i]); + BlockRange().InsertBefore(producer, std::move(range)); + } + operands[i]->ClearContained(); + operands[i]->ClearRegOptional(); + } + if (negate) + { + // -(high) - borrow is 0 - high - borrow. Keep the zero before the + // flags producer, since zeroing a register can overwrite flags. + std::swap(operands[0], operands[1]); + } + if (combine || negate) + { + BlockRange().Remove(other); + } + m_hasAddCarry = true; + add->ChangeOper(add->OperIs(GT_SUB) ? GT_SUB_BORROW : (borrow ? GT_ADD_BORROW : GT_ADD_CARRY)); + add->gtOp1 = operands[0]; + add->gtOp2 = operands[1]; + BlockRange().Remove(add); + BlockRange().InsertAfter(producer, add); + BlockRange().Remove(carry); + if (cast != nullptr) + { + BlockRange().Remove(cast); + } + if (store != nullptr) + { + BlockRange().Remove(store); + BlockRange().Remove(value); + } + ContainCheckBinary(add); + return true; + } + return false; +} +#endif // defined(TARGET_AMD64) || defined(TARGET_ARM64) + +#ifdef TARGET_ARM64 +//------------------------------------------------------------------------ +// LowerArm64MultiplyCarryLoops: Keep the last carry of each multiply-accumulate +// in NZCV until the next limb, including across a counted loop's backedge. +void Lowering::LowerArm64MultiplyCarryLoops() +{ + if (m_compiler->compHndBBtabCount != 0) + { + return; + } + BasicBlock* savedBlock = m_block; + for (BasicBlock* block : m_compiler->Blocks()) + { + if (block->KindIs(BBJ_COND) && block->GetTrueTarget() == block && block->GetFalseTarget() != block) + { + m_block = block; + TryLowerArm64MultiplyCarryLoop(); + } + } + m_block = savedBlock; +} + +//------------------------------------------------------------------------ +// TryLowerArm64MultiplyCarryLoop: Match one to four full-width unsigned products, +// each followed by two low ADD/high ADC pairs. Represent the incoming carry as +// a register plus C, and use ADCS/ADC/ADDS per limb. The final ADDS leaves C for +// the next limb. No memory access moves, and all intervening nodes must preserve +// flags. Only the final loop carry may be live out; drain it on the exit edge. +// For B = 2^64, a*b + destination + carry <= B^2-1. Thus the outgoing +// high word plus C fits in one limb, preserving this representation by induction. +void Lowering::TryLowerArm64MultiplyCarryLoop() +{ + BasicBlock* block = m_block; + GenTree* branch = BlockRange().LastNode(); + if (!branch->OperIs(GT_JCMP) || branch->AsOpCC()->gtCondition.GetCode() != GenCondition::NE || + !branch->gtGetOp1()->OperIs(GT_LCL_VAR) || !branch->gtGetOp1()->TypeIs(TYP_INT) || + !branch->gtGetOp2()->IsIntegralConst(0)) + { + return; + } + BasicBlock* entry = FindCarryLoopEntry(block); + if (entry == nullptr) + { + return; + } + + struct Lane + { + GenTree* lowProduct; + GenTree* highProduct; + GenTree* add[2]; + GenTree* adc[2]; + GenTree* low; + GenTree* destination; + GenTree* partial; + GenTree* carry; + } lanes[4] = {}; + unsigned laneCount = 0; + unsigned adcCount = 0; + unsigned nodes = 0; + for (GenTree* node : BlockRange()) + { + if (++nodes > 512) + { + return; + } + if (node->OperIs(GT_HWINTRINSIC)) + { + if (laneCount == ArrLen(lanes) || adcCount != laneCount * 2 || + node->AsHWIntrinsic()->GetHWIntrinsicId() != NI_ArmBase_Arm64_MultiplyHigh || + node->AsHWIntrinsic()->GetSimdBaseType() != TYP_ULONG) + { + return; + } + lanes[laneCount++].highProduct = node; + } + if (node->OperIs(GT_ADD_CARRY)) + { + if (laneCount == 0 || adcCount >= laneCount * 2 || !node->TypeIs(TYP_LONG) || node->gtSetFlags() || + !node->gtGetOp1()->OperIs(GT_LCL_VAR) || !node->gtGetOp2()->IsIntegralConst(0)) + { + return; + } + lanes[adcCount / 2].adc[adcCount % 2] = node; + adcCount++; + } + } + if (laneCount == 0 || adcCount != laneCount * 2) + { + return; + } + + // Require one definition and one read in this block. For the loop-carried + // local alone, the read precedes the definition and the value is live out. + auto findStore = [&](GenTree* read, bool loopCarry = false) -> GenTree* { + if (!read->OperIs(GT_LCL_VAR)) + { + return nullptr; + } + unsigned local = read->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (!dsc->lvTracked || dsc->IsAddressExposed() || (!loopCarry && !IsCarryLocalDeadOnExit(local, block))) + { + return nullptr; + } + GenTree* store = nullptr; + for (GenTree* node : BlockRange()) + { + if (dsc->lvIsStructField && node->OperIsLocal() && node->AsLclVarCommon()->GetLclNum() == dsc->lvParentLcl) + { + return nullptr; + } + if (node == read && ((store == nullptr && !loopCarry) || (store != nullptr && loopCarry))) + { + return nullptr; + } + if (!node->OperIsLocal() || node->AsLclVarCommon()->GetLclNum() != local || node == read) + { + continue; + } + if (!node->OperIs(GT_STORE_LCL_VAR) || store != nullptr) + { + return nullptr; + } + store = node; + } + return store; + }; + auto definition = [&](GenTree* read) -> GenTree* { + for (unsigned depth = 0; depth < 4 && read->OperIs(GT_LCL_VAR); depth++) + { + GenTree* store = findStore(read); + if (store == nullptr) + { + return nullptr; + } + read = store->gtGetOp1(); + } + return read; + }; + + for (unsigned i = 0; i < laneCount; i++) + { + Lane& lane = lanes[i]; + for (unsigned j = 0; j < 2; j++) + { + GenTree* add = lane.adc[j]->gtPrev; + if (add == nullptr || !add->OperIs(GT_ADD) || !add->TypeIs(TYP_LONG) || add->gtOverflow() || + !add->gtSetFlags() || !add->gtGetOp1()->OperIs(GT_LCL_VAR) || !add->gtGetOp2()->OperIs(GT_LCL_VAR)) + { + return; + } + lane.add[j] = add; + } + if (definition(lane.adc[0]->gtGetOp1()) != lane.highProduct || + definition(lane.adc[1]->gtGetOp1()) != lane.adc[0]) + { + return; + } + lane.low = lane.add[0]->gtGetOp1(); + lane.destination = lane.add[0]->gtGetOp2(); + lane.lowProduct = definition(lane.low); + if (lane.lowProduct == nullptr || !lane.lowProduct->OperIs(GT_MUL)) + { + std::swap(lane.low, lane.destination); + lane.lowProduct = definition(lane.low); + } + if (lane.lowProduct == nullptr || !lane.lowProduct->OperIs(GT_MUL) || lane.lowProduct->gtOverflow() || + lane.lowProduct->gtSetFlags() || !lane.lowProduct->TypeIs(TYP_LONG)) + { + return; + } + // Both halves must be the same unsigned product. Loads have already + // been stored to locals; do not move or duplicate them across aliases. + GenTree* left = lane.lowProduct->gtGetOp1(); + GenTree* right = lane.lowProduct->gtGetOp2(); + GenTreeHWIntrinsic* high = lane.highProduct->AsHWIntrinsic(); + if (!left->OperIs(GT_LCL_VAR) || !right->OperIs(GT_LCL_VAR) || !GenTree::Compare(left, high->Op(1)) || + !GenTree::Compare(right, high->Op(2)) || m_compiler->lvaGetDesc(left->AsLclVar())->IsAddressExposed() || + m_compiler->lvaGetDesc(right->AsLclVar())->IsAddressExposed()) + { + return; + } + // Compare the reaching values at the operand reads, in either order. + // Either half can be evaluated first; neither product is relocated. + for (unsigned operand = 1; operand <= 2; operand++) + { + GenTree* input = operand == 1 ? left : right; + GenTree* other = high->Op(operand); + LclVarDsc* dsc = m_compiler->lvaGetDesc(input->AsLclVar()); + bool seenRead = false; + for (GenTree* node : BlockRange()) + { + if (node == input || node == other) + { + if (seenRead) + { + break; + } + seenRead = true; + } + if (seenRead && node->OperIsLocal()) + { + unsigned local = node->AsLclVarCommon()->GetLclNum(); + if ((node->OperIsLocalStore() && local == input->AsLclVar()->GetLclNum()) || + (dsc->lvIsStructField && local == dsc->lvParentLcl)) + { + return; + } + } + } + } + lane.partial = lane.add[1]->gtGetOp1(); + lane.carry = lane.add[1]->gtGetOp2(); + if (definition(lane.partial) != lane.add[0]) + { + std::swap(lane.partial, lane.carry); + } + if (definition(lane.partial) != lane.add[0] || (i != 0 && definition(lane.carry) != lanes[i - 1].adc[1]) || + m_compiler->lvaGetDesc(lane.carry->AsLclVar())->IsAddressExposed()) + { + return; + } + unsigned carryLocal = lane.carry->AsLclVar()->GetLclNum(); + for (GenTree* node = lane.add[0]; node != lane.carry; node = node->gtNext) + { + if (node == nullptr || (node->OperIsLocalStore() && node->AsLclVarCommon()->GetLclNum() == carryLocal)) + { + return; + } + } + } + GenTree* carryStore = findStore(lanes[0].carry, true); + if (carryStore == nullptr || definition(carryStore->gtGetOp1()) != lanes[laneCount - 1].adc[1]) + { + return; + } + + for (GenTree* node : BlockRange()) + { + bool matched = node == branch; + for (unsigned i = 0; i < laneCount; i++) + { + Lane& lane = lanes[i]; + matched |= node == lane.lowProduct || node == lane.highProduct || node == lane.add[0] || + node == lane.add[1] || node == lane.adc[0] || node == lane.adc[1]; + } + if (matched) + { + continue; + } + if (node->gtSetFlags() || node->OperConsumesFlags()) + { + return; + } + switch (node->OperGet()) + { + case GT_LCL_VAR: + case GT_STORE_LCL_VAR: + case GT_IL_OFFSET: + case GT_CNS_INT: + case GT_LEA: + break; + case GT_IND: + case GT_STOREIND: + if (!node->TypeIs(TYP_LONG) || (node->gtFlags & GTF_IND_VOLATILE) != 0) + { + return; + } + break; + case GT_CAST: + if (node->gtOverflow() || !node->TypeIs(TYP_LONG) || !node->gtGetOp1()->TypeIs(TYP_INT)) + { + return; + } + break; + case GT_ADD: + case GT_SUB: + if (node->gtOverflow() || !node->TypeIs(TYP_INT, TYP_LONG, TYP_BYREF)) + { + return; + } + break; + default: + return; + } + } + + JITDUMP("Keeping ARM64 multiply-accumulate carry in NZCV across %u lanes around " FMT_BB "\n", laneCount, + block->bbNum); + BasicBlock* seed = m_compiler->fgSplitEdge(entry, block); + BasicBlock* finish = m_compiler->fgSplitEdge(block, block->GetFalseTarget()); + m_compiler->fgInvalidateDfsTree(); + GenTree* zero = m_compiler->gtNewLconNode(0); + GenTree* one = m_compiler->gtNewLconNode(1); + GenTree* clearCarry = m_compiler->gtNewOperNode(GT_CMP, TYP_VOID, zero, one); + clearCarry->gtFlags |= GTF_SET_FLAGS; + LIR::AsRange(seed).InsertAtEnd(zero, one, clearCarry); + LIR::ReadOnlyRange seedRange(zero, clearCarry); + LowerRange(seed, seedRange); + + unsigned carryLocal = lanes[0].carry->AsLclVar()->GetLclNum(); + GenTree* finalRead = m_compiler->gtNewLclvNode(carryLocal, TYP_LONG); + GenTree* finalZero = m_compiler->gtNewLconNode(0); + GenTree* drain = m_compiler->gtNewOperNode(GT_ADD_CARRY, TYP_LONG, finalRead, finalZero); + GenTree* finalStore = m_compiler->gtNewStoreLclVarNode(carryLocal, drain); + LIR::AsRange(finish).InsertAtEnd(finalRead, finalZero, drain, finalStore); + LIR::ReadOnlyRange finishRange(finalRead, finalStore); + LowerRange(finish, finishRange); + + for (unsigned i = 0; i < laneCount; i++) + { + Lane& lane = lanes[i]; + BlockRange().Remove(lane.carry); + BlockRange().InsertBefore(lane.add[0], lane.carry); + lane.add[0]->AsOp()->gtOp1 = lane.low; + lane.add[0]->AsOp()->gtOp2 = lane.carry; + lane.add[0]->ChangeOper(GT_ADD_CARRY); + lane.add[1]->AsOp()->gtOp1 = lane.partial; + lane.add[1]->AsOp()->gtOp2 = lane.destination; + LIR::Use use; + bool found = BlockRange().TryGetUse(lane.adc[1], &use); + assert(found); + use.ReplaceWith(lane.adc[1]->gtGetOp1()); + BlockRange().Remove(lane.adc[1]->gtGetOp2()); + BlockRange().Remove(lane.adc[1]); + ContainCheckBinary(lane.add[0]->AsOp()); + ContainCheckBinary(lane.add[1]->AsOp()); + } +} +#endif // TARGET_ARM64 diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 06913e3a8e5b5a..2522ef0c036482 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -167,6 +167,32 @@ class Lowering final : public Phase #endif GenTree* OptimizeConstCompare(GenTree* cmp); GenTree* LowerCompare(GenTree* cmp); +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + BasicBlock* FindCarryLoopEntry(BasicBlock* block); + GenTree* FindDeadCarryConstantStore(GenTree* constant); + GenTree* FindCarryLocalStore(GenTree* read); + bool IsCarryLocalDeadOnExit(unsigned local, BasicBlock* block); + bool TryLowerCarryCompare(GenTreeOp* cmp); + bool TryLowerAddCarry(GenTreeOp* add); + bool LowerFullAdders(); + void LowerCarryChains(); +#ifdef TARGET_AMD64 + void ReplaceCarryAddressAdd(GenTree* add); + GenTree* FindMultiplyCarryStore(GenTree* read, bool loopCarry = false); + void LowerMultiplyCarryLoops(); + bool TryLowerMultiplyCarryLoop(); + bool TryLowerMultiplyCarryChain(); + void TryLowerMultiplyCarryBackedge(); + void LowerMultiplyCarryMultiplier(); + void LowerMultiplyCarryLoads(); +#else + void LowerArm64MultiplyCarryLoops(); + void TryLowerArm64MultiplyCarryLoop(); +#endif + void TryLowerCarryLoop( + GenTree* sum, GenTree* restore, GenTree* cc, GenTree* cast, GenTree* store, unsigned carryLocal); + bool IsCarryOperandAvailable(GenTree* operand, GenTree* producer, GenTree** moveStart); +#endif GenTree* LowerJTrue(GenTreeOp* jtrue); #ifdef TARGET_RISCV64 GenTree* LowerSavedIntegerCompare(GenTree* cmp); @@ -427,6 +453,7 @@ class Lowering final : public Phase bool TryLowerConstIntDivOrMod(GenTree* node, GenTree** nextNode); GenTree* LowerSignedDivOrMod(GenTree* node); void LowerDivOrMod(GenTreeOp* divMod); + void TryLowerDivRem(GenTreeOp* div); void LowerBlockStoreCommon(GenTreeBlk* blkNode); void LowerBlockStoreAsHelperCall(GenTreeBlk* blkNode); void LowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blkNode); @@ -488,6 +515,7 @@ class Lowering final : public Phase void LowerShift(GenTreeOp* shift); void TryRemoveShiftRotateMask(GenTreeOp* op); bool TryFoldBinop(GenTreeOp* node); + bool TryContainFunnelShift(GenTreeOp* node); #ifdef FEATURE_HW_INTRINSICS GenTree* LowerHWIntrinsic(GenTreeHWIntrinsic* node); void LowerHWIntrinsicCC(GenTreeHWIntrinsic* node, NamedIntrinsic newIntrinsicId, GenCondition condition); @@ -661,6 +689,15 @@ class Lowering final : public Phase mutable SideEffectSet m_scratchSideEffects; // SideEffectSet used for IsSafeToContainMem and isRMWIndirCandidate BasicBlock* m_block; +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + // Conservative during partial/post-lowering walks; reset for each full block walk. + bool m_blockMayHaveSetCC = true; + bool m_hasAddCarry = false; + BasicBlock* m_lastSubtractionBlock = nullptr; + unsigned m_carryLocalScanBudget = 65536; // Node/block budget for current-IR dead-on-exit proofs. + unsigned m_carryUseScanBudget = 4096; // Extra node visits beyond the bounded local carry-use scans. +#endif + #ifdef FEATURE_FIXED_OUT_ARGS unsigned m_outgoingArgSpaceSize = 0; #endif diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index d4c691af0a033f..2dba7c5dfa0b36 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -2791,9 +2791,28 @@ void Lowering::ContainCheckIndir(GenTreeIndir* indirNode) // void Lowering::ContainCheckBinary(GenTreeOp* node) { + if (TryContainFunnelShift(node)) + { + return; + } GenTree* op1 = node->gtGetOp1(); GenTree* op2 = node->gtGetOp2(); +#ifdef TARGET_ARM64 + if (node->OperIs(GT_ADD_CARRY, GT_SUB_BORROW, GT_ADD_BORROW)) + { + if (node->OperIs(GT_SUB_BORROW) && op1->IsIntegralConst(0)) + { + MakeSrcContained(node, op1); + } + if (op2->IsIntegralConst(0)) + { + MakeSrcContained(node, op2); + } + return; + } +#endif + if (CheckImmedAndMakeContained(node, op2)) { return; diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index f544824aa0044b..c8fb5e1dc71ebf 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -8063,6 +8063,178 @@ void Lowering::ContainCheckMul(GenTreeOp* node) } } +//------------------------------------------------------------------------ +// TryLowerDivRem: Reuse the remainder of a divide for x - (x / y) * y. +// +// Arguments: +// div - The division producing the quotient. +// +// Notes: +// Keep the division (and its exceptions) in place. Capture RDX immediately +// after it, before any quotient use or intervening code can overwrite it. +// The pair's fixed definitions must not be reassigned by LSRA. +// +void Lowering::TryLowerDivRem(GenTreeOp* div) +{ + if (!m_compiler->opts.OptimizationEnabled() || !div->OperIs(GT_DIV, GT_UDIV) || + !varTypeIsIntOrI(div) || div->IsDivRemPair()) + { + return; + } + +#ifdef TARGET_AMD64 + unsigned intRegCount = m_compiler->get_REG_INT_LAST() - REG_INT_FIRST; +#else + unsigned intRegCount = REG_INT_COUNT - 1; +#endif + // Conservatively leave large, register-heavy methods to the existing sequence. Keeping two + // fixed-register results can increase spills well beyond the multiply/subtract being removed. + // Make this decision for the whole method so partially pairing its divides does not produce + // unstable allocation tradeoffs between otherwise similar paths. + if (m_compiler->lvaTrackedCount > 2 * intRegCount) + { + return; + } + + GenTree* dividend = div->gtGetOp1(); + GenTree* divisor = div->gtGetOp2(); + if (!dividend->OperIs(GT_LCL_VAR) || !divisor->OperIs(GT_LCL_VAR)) + { + return; + } + + GenTree* quotientStore = div->gtNext; + if ((quotientStore == nullptr) || !quotientStore->OperIs(GT_STORE_LCL_VAR) || + (quotientStore->gtGetOp1() != div)) + { + quotientStore = nullptr; + } + else if (m_compiler->lvaGetDesc(quotientStore->AsLclVarCommon())->IsAddressExposed()) + { + return; + } + + unsigned budget = 64; + for (GenTree* node = div->gtNext; (node != nullptr) && (budget-- != 0); node = node->gtNext) + { + if ((quotientStore != nullptr) && (node != quotientStore) && node->OperIs(GT_STORE_LCL_VAR) && + (node->AsLclVarCommon()->GetLclNum() == quotientStore->AsLclVarCommon()->GetLclNum())) + { + return; + } + + if (!node->OperIs(GT_SUB) || node->gtOverflow() || node->gtSetFlags() || + (node->TypeGet() != div->TypeGet())) + { + continue; + } + + GenTree* original = node->gtGetOp1(); + GenTree* mul = node->gtGetOp2(); + if (!original->OperIs(GT_LCL_VAR) || !GenTree::Compare(original, dividend) || !mul->OperIs(GT_MUL) || + mul->gtOverflow() || mul->gtSetFlags() || (mul->TypeGet() != div->TypeGet())) + { + continue; + } + + GenTree* quotient = mul->gtGetOp1(); + GenTree* factor = mul->gtGetOp2(); + if (quotient->OperIs(GT_LCL_VAR) && GenTree::Compare(quotient, divisor)) + { + std::swap(quotient, factor); + } + + if (!factor->OperIs(GT_LCL_VAR) || !GenTree::Compare(factor, divisor)) + { + continue; + } + + if (quotientStore != nullptr) + { + if (!quotient->OperIs(GT_LCL_VAR) || + (quotient->AsLclVarCommon()->GetLclNum() != quotientStore->AsLclVarCommon()->GetLclNum()) || + (quotient->TypeGet() != div->TypeGet())) + { + continue; + } + } + else if (quotient != div) + { + continue; + } + + // Either copy of an input can precede the other in LIR. Check both + // reads so assignments embedded in an expression cannot change it. + if (!IsInvariantInRange(dividend, node) || !IsInvariantInRange(divisor, node) || + !IsInvariantInRange(original, node) || !IsInvariantInRange(factor, node)) + { + return; + } + + LIR::Use use; + if (!BlockRange().TryGetUse(node, &use)) + { + return; + } + + GenTree* remainder = m_compiler->gtNewPhysRegNode(REG_RDX, div->TypeGet()); + remainder->gtFlags |= GTF_DIV_REM_PAIR; + div->gtFlags |= GTF_DIV_REM_PAIR; + // Reuse a local destination when it can be defined at the divide. Introducing another + // temporary for an immediately stored remainder needlessly perturbs register allocation. + // Do not cross an exception: a handler may observe the destination's old value. + GenTree* store = use.User(); + bool reuseStore = store->OperIs(GT_STORE_LCL_VAR) && + !m_compiler->lvaGetDesc(store->AsLclVarCommon())->IsAddressExposed() && + !m_compiler->lvaGetDesc(store->AsLclVarCommon())->lvIsStructField; + if (reuseStore) + { + unsigned local = store->AsLclVarCommon()->GetLclNum(); + unsigned storeBudget = 64; + for (GenTree* between = div->gtNext; between != store; between = between->gtNext) + { + if ((between == nullptr) || (storeBudget-- == 0) || + ((between->gtFlags & (GTF_CALL | GTF_EXCEPT)) != 0) || + ((between != original) && between->OperIsLocal() && + (between->AsLclVarCommon()->GetLclNum() == local))) + { + reuseStore = false; + break; + } + } + } + if (reuseStore) + { + store->AsOp()->gtOp1 = remainder; + BlockRange().Remove(store); + } + else + { + unsigned temp = m_compiler->lvaGrabTemp(true DEBUGARG("division remainder")); + store = m_compiler->gtNewTempStore(temp, remainder); + GenTree* read = m_compiler->gtNewLclvNode(temp, div->TypeGet()); + BlockRange().InsertBefore(node, read); + use.ReplaceWith(read); + } + BlockRange().InsertAfter(div, remainder, store); + + BlockRange().Remove(original); + BlockRange().Remove(factor); + if (quotient != div) + { + BlockRange().Remove(quotient); + } + else + { + div->SetUnusedValue(); + } + BlockRange().Remove(mul); + BlockRange().Remove(node); + JITDUMP("Reusing the hardware remainder of division [%06u]\n", div->gtTreeID); + return; + } +} + //------------------------------------------------------------------------ // ContainCheckDivOrMod: determine which operands of a div/mod should be contained. // @@ -8585,6 +8757,11 @@ void Lowering::ContainCheckBinary(GenTreeOp* node) { assert(node->OperIsBinary()); + if (TryContainFunnelShift(node)) + { + return; + } + if (varTypeIsFloating(node)) { assert(node->OperIs(GT_ADD, GT_SUB)); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index faf1b4b10c8c96..aa1bd94a3d3cb4 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -8982,6 +8982,15 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) { GenTree* lastNode = LIR::AsRange(block).LastNode(); +#ifdef TARGET_AMD64 + if (lastNode->OperIs(GT_JCMP)) + { + // The flag-preserving zero test consumes RCX even when the local + // itself remains in another register. Edge resolution must not + // restore an outgoing value into RCX before the branch. + consumedRegs |= SRBM_ECX; + } +#endif if (lastNode->OperIs(GT_JTRUE, GT_JCMP, GT_JTEST)) { assert(!lastNode->OperIs(GT_JTRUE) || !lastNode->gtGetOp1()->isContained()); @@ -13753,6 +13762,53 @@ SingleTypeRegSet LinearScan::RegisterSelection::select(Interval* } } +#ifdef TARGET_AMD64 + // Carry arithmetic can constrain the multiplier to RDX and a flags-preserving loop count to RCX. + // A preference that encounters one of these fixed references before the next use can force a spill. + // At a definition, prefer a free register that covers the range when one is available instead. + // Keep the original preferences if none of the alternatives can cover it. + if (linearScan->m_hasCarryArithmetic && currentInterval->isLocalVar && !found && + RefTypeIsDef(refPosition->refType) && !refPosition->isFixedRegRef) + { + SingleTypeRegSet coveringRegisters = RBM_NONE; + SingleTypeRegSet conflictingPreferences = RBM_NONE; + for (SingleTypeRegSet remaining = freeCandidates; remaining != RBM_NONE;) + { + regNumber reg = genFirstRegNumFromMask(remaining, regType); + SingleTypeRegSet bit = genSingleTypeRegMask(reg); + remaining &= ~bit; + + LsraLocation fixedLocation = linearScan->getNextFixedRef(reg, regType); + if ((fixedLocation == rangeEndLocation) && rangeEndRefPosition->isFixedRefOfReg(reg)) + { + fixedLocation++; + } + if (fixedLocation <= rangeEndLocation) + { + // Register kills (for example, at calls) are handled by the existing callee-save preferences. + RefPosition* fixedRef = linearScan->getRegisterRecord(reg)->getNextRefPosition(); + if ((fixedRef != nullptr) && (fixedRef->refType == RefTypeFixedReg) && + (fixedRef->nodeLocation == fixedLocation)) + { + conflictingPreferences |= preferences & bit; + } + } + else if (linearScan->getNextIntervalRef(reg, regType) > rangeEndLocation) + { + coveringRegisters |= bit; + } + } + if ((conflictingPreferences != RBM_NONE) && (coveringRegisters != RBM_NONE)) + { + preferences &= ~conflictingPreferences; + if ((preferences & freeCandidates) == RBM_NONE) + { + preferences |= coveringRegisters; + } + } + } +#endif // TARGET_AMD64 + #define IF_FOUND_GOTO_DONE \ if (found) \ goto Selection_Done; diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 3541e1a2309349..2468d141c02e28 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1147,6 +1147,7 @@ class LinearScan : public RegAllocInterface void applyCalleeSaveHeuristics(RefPosition* rp); void checkConflictingDefUse(RefPosition* rp); + void addFixedRefsForPropagatedDefs(); void associateRefPosWithInterval(RefPosition* rp); @@ -1602,6 +1603,8 @@ class LinearScan : public RegAllocInterface IntervalList intervals; + bool m_hasCarryArithmetic = false; + RegRecord physRegs[REG_COUNT]; // Map from tracked variable index to Interval*. @@ -1672,6 +1675,7 @@ class LinearScan : public RegAllocInterface // Ordered list of RefPositions RefPositionList refPositions; + bool m_hasUnmodeledFixedDefs = false; // Head of linked list of RefTypeKill ref positions RefPosition* killHead; diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 7856d4302d8aa3..e3a97cb07af3fe 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -943,6 +943,11 @@ int LinearScan::BuildNode(GenTree* tree) assert(dstCount == 0); break; + case GT_SUB_BORROW: + case GT_ADD_BORROW: + case GT_ADD_CARRY: + m_hasCarryArithmetic = true; + FALLTHROUGH; case GT_ADD: case GT_SUB: if (varTypeIsFloating(tree->TypeGet())) diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 0df524b0a9abda..9d606749539d91 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -268,6 +268,11 @@ void LinearScan::resolveConflictingDefAndUse(Interval* interval, RefPosition* de // to change the register assignments to RDX and RAX respectively. bool canChangeDef = !defRefPosition->treeNode->IsMultiRegNode(); + // A lowered divide/remainder pair defines RAX followed immediately by RDX. + // Redirecting either definition could overwrite the other hardware result + // before the physical-register capture has taken place. + canChangeDef &= !defRefPosition->treeNode->IsDivRemPair(); + // Avoid changing the def reg away from its assignment if that register is // currently busy. The reason is that we have a number of places in LSRA // that assume that BuildDef(tree, SRBM_REG) means that SRBM_REG will be @@ -417,6 +422,10 @@ void LinearScan::checkConflictingDefUse(RefPosition* useRP) if (!isSingleRegister(newAssignment) || !theInterval->hasInterferingUses) { defRP->registerAssignment = newAssignment; + if (isSingleRegister(newAssignment) && !defRP->isFixedRegRef && !theInterval->isConstant) + { + m_hasUnmodeledFixedDefs = true; + } } } else @@ -2776,6 +2785,11 @@ void LinearScan::buildIntervals() actualRegistersMask = regMaskTP(~RBM_NONE, ~0); } + if (m_hasUnmodeledFixedDefs && m_hasCarryArithmetic) + { + addFixedRefsForPropagatedDefs(); + } + #ifdef DEBUG // Make sure we don't have any blocks that were not visited for (BasicBlock* const block : m_compiler->Blocks()) @@ -2793,6 +2807,72 @@ void LinearScan::buildIntervals() #endif // DEBUG } +//------------------------------------------------------------------------ +// addFixedRefsForPropagatedDefs: Make inherited single-register constraints +// visible to allocations that precede the constrained definition. +// +// Notes: +// Runs after reference construction, when all use constraints are known. +// Maintains the global and physical-register reference lists in one pass. +// Limited to methods with lowered carry arithmetic. Applying the additional +// constraints to general call argument setup can introduce spills. +// Constants retain their existing register-reuse handling. +// +void LinearScan::addFixedRefsForPropagatedDefs() +{ + // A tree-temp definition can inherit a single-register constraint from its + // use after its RefPosition has been built. Model that constraint in the + // physical register's timeline too, so earlier allocations see the conflict. + // Insert in both ordered lists without moving existing RefPositions. + RefPosition* previousRegRef[REG_COUNT] = {}; + INDEBUG(unsigned refNumber = 0;) + for (auto it = refPositions.begin(); it != refPositions.end(); ++it) + { + RefPosition& ref = *it; + if (ref.isPhysRegRef) + { + previousRegRef[ref.getReg()->regNum] = &ref; + } + // Keep constant definitions under the existing constant-reuse handling. + // A physical fixed reference would discard an already-matching constant + // before its definition gets a chance to reuse the register. + else if ((ref.refType == RefTypeDef) && !ref.isFixedRegRef && !ref.getInterval()->isLocalVar && + !ref.getInterval()->isConstant && isSingleRegister(ref.registerAssignment)) + { + regNumber reg = genRegNumFromMask(ref.registerAssignment, ref.getInterval()->registerType); + RegRecord* record = getRegisterRecord(reg); + RefPosition* previous = previousRegRef[reg]; + if ((previous == nullptr) || (previous->nodeLocation != ref.nodeLocation) || + (previous->refType != RefTypeFixedReg)) + { + auto fixedIt = + refPositions.emplace(it, ref.bbNum, ref.nodeLocation, nullptr, RefTypeFixedReg DEBUG_ARG(nullptr)); + RefPosition* fixed = &*fixedIt; + fixed->setReg(record); + if (previous == nullptr) + { + fixed->nextRefPosition = record->firstRefPosition; + record->firstRefPosition = fixed; + } + else + { + fixed->nextRefPosition = previous->nextRefPosition; + previous->nextRefPosition = fixed; + } + if (fixed->nextRefPosition == nullptr) + { + record->lastRefPosition = fixed; + record->recentRefPosition = fixed; + } + previousRegRef[reg] = fixed; + INDEBUG(fixed->rpNum = refNumber++;) + } + ref.isFixedRegRef = true; + } + INDEBUG(ref.rpNum = refNumber++;) + } +} + //------------------------------------------------------------------------ // buildInitialParamDef: Build the initial definition for a parameter. // diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 2695fe7a850d82..3fad7e0e65796c 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -238,6 +238,23 @@ int LinearScan::BuildNode(GenTree* tree) srcCount = 1; break; +#ifdef TARGET_AMD64 + case GT_ADX_SEED: + srcCount = 0; + break; + case GT_ADX_DRAIN: + srcCount = BuildOperandUses(tree->gtGetOp1(), lowGprRegs); + buildInternalIntRegisterDefForNode(tree, lowGprRegs); + setInternalRegsDelayFree = true; + buildInternalRegisterUses(); + BuildDef(tree, lowGprRegs); + break; + case GT_JCMP: + assert(tree->gtGetOp2()->IsIntegralConst(0) && tree->gtGetOp2()->isContained()); + m_hasCarryArithmetic = true; + srcCount = BuildOperandUses(tree->gtGetOp1(), SRBM_ECX); + break; +#endif case GT_JCC: srcCount = 0; assert(dstCount == 0); @@ -266,6 +283,30 @@ int LinearScan::BuildNode(GenTree* tree) BuildDef(tree, allByteRegs()); break; + case GT_PHYSREG: + if (tree->IsDivRemPair()) + { + assert(tree->AsPhysReg()->gtSrcReg == REG_RDX); +#ifdef DEBUG + // Sequencing may move IL markers between the two definitions. + // They emit no register-writing instructions. + GenTree* producer = tree->gtPrev; + while ((producer != nullptr) && producer->OperIs(GT_IL_OFFSET)) + { + producer = producer->gtPrev; + } + assert((producer != nullptr) && producer->IsDivRemPair()); + assert(producer->OperIs(GT_DIV, GT_UDIV)); +#endif + srcCount = 0; + BuildDef(tree, SRBM_RDX); + } + else + { + srcCount = BuildSimple(tree); + } + break; + case GT_SELECT: assert(dstCount == 1); srcCount = BuildSelect(tree->AsConditional()); @@ -309,6 +350,19 @@ int LinearScan::BuildNode(GenTree* tree) case GT_ADD_HI: case GT_SUB_LO: case GT_SUB_HI: +#endif +#ifdef TARGET_AMD64 + case GT_ADCX: + case GT_ADOX: + m_hasCarryArithmetic = true; + srcCount = BuildBinaryUses(tree->AsOp(), lowGprRegs); + BuildDef(tree, lowGprRegs); + break; + case GT_SUB_BORROW: + case GT_ADD_BORROW: + case GT_ADD_CARRY: + m_hasCarryArithmetic = true; + FALLTHROUGH; #endif case GT_ADD: case GT_SUB: @@ -318,6 +372,16 @@ int LinearScan::BuildNode(GenTree* tree) case GT_BIT_SET: case GT_BIT_CLEAR: case GT_BIT_INVERT: + if (tree->IsFunnelShift()) + { + // SHRD overwrites the low input. Keep the high input live until + // after the result is defined so the initial copy cannot clobber it. + tgtPrefUse = BuildUse(tree->gtGetOp1()->gtGetOp1(), lowGprRegs); + setDelayFree(BuildUse(tree->gtGetOp2()->gtGetOp1(), lowGprRegs)); + srcCount = 2; + BuildDef(tree, lowGprRegs); + break; + } srcCount = BuildBinaryUses(tree->AsOp()); assert(dstCount == 1); BuildDef(tree); @@ -2393,6 +2457,18 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou SingleTypeRegSet apxAwareRegCandidates = ForceLowGprForApxIfNeeded(op1, RBM_NONE, canHWIntrinsicUseApxRegs); + if ((intrinsicTree->gtFlags & GTF_HW_MULX) != 0) + { + // MULX allows both halves to stay in independently allocated registers. + srcCount = BuildOperandUses(op1, SRBM_EDX); + srcCount += BuildOperandUses(op2, ForceLowGprForApxIfNeeded(op2, availableIntRegs & ~SRBM_EDX, + canHWIntrinsicUseApxRegs)); + BuildDef(intrinsicTree, apxAwareRegCandidates, 0); + BuildDef(intrinsicTree, apxAwareRegCandidates, 1); + buildUses = false; + break; + } + // mulEAX always uses EAX; if one operand is contained, force the other op into EAX. // Otherwise don't force any register: the second parameter may already happen to be in EAX, // in which case codegen will use it as the implicit operand. diff --git a/src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs b/src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs index c38cf1e66a61f7..a01d89d4a5ecb8 100644 --- a/src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs +++ b/src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs @@ -81,9 +81,12 @@ private static class XArchIntrinsicConstants public const int WaitPkg = (1 << 16); public const int X86Serialize = (1 << 17); public const int Avx512Bmm = (1 << 18); // NativeAOT does not currently consume this here. + public const int Adx = (1 << 19); public static void AddToBuilder(InstructionSetSupportBuilder builder, int flags) { + if ((flags & Adx) != 0) + builder.AddSupportedInstructionSet("adx"); if ((flags & Avx) != 0) builder.AddSupportedInstructionSet("avx"); if ((flags & Avx2) != 0) @@ -149,6 +152,7 @@ public static int FromInstructionSet(InstructionSet instructionSet) InstructionSet.X64_AVX => Avx, InstructionSet.X64_AVX_X64 => Avx, + InstructionSet.X64_ADX => Adx, InstructionSet.X64_AVX2 => Avx2, InstructionSet.X64_AVX2_X64 => Avx2, diff --git a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSet.cs b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSet.cs index 4c5574d6870c32..d8f77cf948178c 100644 --- a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSet.cs +++ b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSet.cs @@ -106,5 +106,6 @@ public enum ReadyToRunInstructionSet Cssc = 93, Zicond = 94, Fp16 = 95, + Adx = 96, } } diff --git a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSetHelper.cs b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSetHelper.cs index 632e3053189a72..91b2955fb587ca 100644 --- a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSetHelper.cs +++ b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSetHelper.cs @@ -144,6 +144,7 @@ public static class ReadyToRunInstructionSetHelper case InstructionSet.X64_VectorT512: return ReadyToRunInstructionSet.VectorT512; case InstructionSet.X64_AVXVNNIINT: return ReadyToRunInstructionSet.AvxVnniInt8; case InstructionSet.X64_AVXVNNIINT_V512: return ReadyToRunInstructionSet.AvxVnniInt8_V512; + case InstructionSet.X64_ADX: return ReadyToRunInstructionSet.Adx; default: throw new Exception("Unknown instruction set"); } @@ -199,6 +200,7 @@ public static class ReadyToRunInstructionSetHelper case InstructionSet.X86_VectorT512: return ReadyToRunInstructionSet.VectorT512; case InstructionSet.X86_AVXVNNIINT: return ReadyToRunInstructionSet.AvxVnniInt8; case InstructionSet.X86_AVXVNNIINT_V512: return ReadyToRunInstructionSet.AvxVnniInt8_V512; + case InstructionSet.X86_ADX: return ReadyToRunInstructionSet.Adx; default: throw new Exception("Unknown instruction set"); } diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs index 870544e2156425..466756f42b02da 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs @@ -112,6 +112,7 @@ public enum InstructionSet X64_SHA_X64 = InstructionSet_X64.SHA_X64, X64_WAITPKG_X64 = InstructionSet_X64.WAITPKG_X64, X64_X86Serialize_X64 = InstructionSet_X64.X86Serialize_X64, + X64_ADX = InstructionSet_X64.ADX, X86_X86Base = InstructionSet_X86.X86Base, X86_AVX = InstructionSet_X86.AVX, X86_AVX2 = InstructionSet_X86.AVX2, @@ -158,6 +159,7 @@ public enum InstructionSet X86_SHA_X64 = InstructionSet_X86.SHA_X64, X86_WAITPKG_X64 = InstructionSet_X86.WAITPKG_X64, X86_X86Serialize_X64 = InstructionSet_X86.X86Serialize_X64, + X86_ADX = InstructionSet_X86.ADX, } public enum InstructionSet_ARM64 { @@ -276,6 +278,7 @@ public enum InstructionSet_X64 SHA_X64 = 44, WAITPKG_X64 = 45, X86Serialize_X64 = 46, + ADX = 47, } public enum InstructionSet_X86 @@ -328,6 +331,7 @@ public enum InstructionSet_X86 SHA_X64 = 44, WAITPKG_X64 = 45, X86Serialize_X64 = 46, + ADX = 47, } public unsafe struct InstructionSetFlags : IEnumerable @@ -1268,6 +1272,7 @@ public static IEnumerable ArchitectureToValidInstructionSets yield return new InstructionSetInfo("avxvnniint_v512", "AvxVnniInt8_V512", InstructionSet.X64_AVXVNNIINT_V512, true); yield return new InstructionSetInfo("avxvnniint", "AvxVnniInt16", InstructionSet.X64_AVXVNNIINT, true); yield return new InstructionSetInfo("avxvnniint_v512", "AvxVnniInt16_V512", InstructionSet.X64_AVXVNNIINT_V512, true); + yield return new InstructionSetInfo("adx", "", InstructionSet.X64_ADX, true); break; case TargetArchitecture.X86: @@ -1342,6 +1347,7 @@ public static IEnumerable ArchitectureToValidInstructionSets yield return new InstructionSetInfo("avxvnniint_v512", "AvxVnniInt8_V512", InstructionSet.X86_AVXVNNIINT_V512, true); yield return new InstructionSetInfo("avxvnniint", "AvxVnniInt16", InstructionSet.X86_AVXVNNIINT, true); yield return new InstructionSetInfo("avxvnniint_v512", "AvxVnniInt16_V512", InstructionSet.X86_AVXVNNIINT_V512, true); + yield return new InstructionSetInfo("adx", "", InstructionSet.X86_ADX, true); break; } } diff --git a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt index a70cd7d9991687..811caee60723ce 100644 --- a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt +++ b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt @@ -26,7 +26,7 @@ ; DO NOT CHANGE R2R NUMERIC VALUES OF THE EXISTING SETS. Changing R2R numeric values definitions would be R2R format breaking change. ; The ISA definitions should also be mapped to `hwintrinsicIsaRangeArray` in hwintrinsic.cpp. -; NEXT_AVAILABLE_R2R_BIT = 96 +; NEXT_AVAILABLE_R2R_BIT = 97 ; Definition of X86 instruction sets definearch ,X86 ,32Bit ,X64, X64, X86 @@ -206,6 +206,10 @@ definearch ,X64 ,64Bit ,X64, X64, X86 copyinstructionsets,X86 ,X64 +; Append new instruction sets here to preserve existing JIT ISA numbers. +instructionset ,X86 , ,Adx ,96 ,ADX ,adx +instructionset ,X64 , ,Adx ,96 ,ADX ,adx + ; Definition of Arm64 instruction sets definearch ,ARM64 ,64Bit ,Arm64, Arm64, Arm diff --git a/src/coreclr/vm/codeman.cpp b/src/coreclr/vm/codeman.cpp index dcfdcfcf0d78e0..16fe0c6b058541 100644 --- a/src/coreclr/vm/codeman.cpp +++ b/src/coreclr/vm/codeman.cpp @@ -1546,6 +1546,11 @@ void EEJitManager::SetCpuInfo() CPUCompileFlags.Set(InstructionSet_AVX); } + if (((cpuFeatures & XArchIntrinsicConstants_Adx) != 0) && CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_EnableADX)) + { + CPUCompileFlags.Set(InstructionSet_ADX); + } + if (((cpuFeatures & XArchIntrinsicConstants_Avx2) != 0) && CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_EnableAVX2)) { CPUCompileFlags.Set(InstructionSet_AVX2); diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.DivRem.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.DivRem.cs index b62e1877fe46b0..ea18e70f8fe0a5 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.DivRem.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.DivRem.cs @@ -1,7 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; namespace System.Numerics.Tensors { @@ -77,7 +79,15 @@ public static void DivRem(T x, ReadOnlySpan y, Span quotientDestination /// Math.DivRem(x, y) private readonly struct DivRemOperator : IBinaryInputBinaryOutput where T : IBinaryInteger { - public static bool Vectorizable => true; + // On x64, 64-bit integer vector division is scalarized. Calling + // DivRem per element preserves each divide's hardware remainder + // instead of reconstructing it with a vector multiply/subtract. + public static bool Vectorizable + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => !X86Base.X64.IsSupported || + (typeof(T) != typeof(long) && typeof(T) != typeof(ulong)); + } public static (T, T) Invoke(T x, T y) => T.DivRem(x, y); diff --git a/src/libraries/System.Private.CoreLib/src/System/Decimal.DecCalc.cs b/src/libraries/System.Private.CoreLib/src/System/Decimal.DecCalc.cs index a0adc37bc6d996..79c6ae902c8bad 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Decimal.DecCalc.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Decimal.DecCalc.cs @@ -188,6 +188,21 @@ private static (uint Quotient, uint Remainder) Div64By32(ulong dividend, uint de } } + // Keep the limbs separate so x86/x64 can use them directly as the + // widening divide inputs. The incoming high limb must be below den. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static (uint Quotient, uint Remainder) Div64By32(uint low, uint high, uint den) + { + if (X86.X86Base.IsSupported) + { + return X86.X86Base.DivRem(low, high, den); + } + + ulong dividend = ((ulong)high << 32) | low; + uint quotient = (uint)(dividend / den); + return (quotient, low - quotient * den); + } + /// /// Do full divide, yielding 96-bit result and 32-bit remainder. /// @@ -200,6 +215,7 @@ private static uint Div96By32(ref Buf12 bufNum, uint den) if (X86.X86Base.IsSupported) { uint remainder = 0; + uint quotient; if (bufNum.U2 != 0) goto Div3Word; @@ -210,11 +226,14 @@ private static uint Div96By32(ref Buf12 bufNum, uint den) bufNum.U1 = 0; goto Div1Word; Div3Word: - (bufNum.U2, remainder) = X86.X86Base.DivRem(bufNum.U2, remainder, den); + (quotient, remainder) = X86.X86Base.DivRem(bufNum.U2, remainder, den); + bufNum.U2 = quotient; Div2Word: - (bufNum.U1, remainder) = X86.X86Base.DivRem(bufNum.U1, remainder, den); + (quotient, remainder) = X86.X86Base.DivRem(bufNum.U1, remainder, den); + bufNum.U1 = quotient; Div1Word: - (bufNum.U0, remainder) = X86.X86Base.DivRem(bufNum.U0, remainder, den); + (quotient, remainder) = X86.X86Base.DivRem(bufNum.U0, remainder, den); + bufNum.U0 = quotient; return remainder; } else @@ -237,7 +256,8 @@ private static uint Div96By32(ref Buf12 bufNum, uint den) tmp = bufNum.Low64; if (tmp == 0) return 0; - (bufNum.Low64, rem) = Math.DivRem(tmp, den); + (div, rem) = Math.DivRem(tmp, den); + bufNum.Low64 = div; return (uint)rem; } } @@ -477,6 +497,35 @@ private static uint Div128By96(ref Buf16 bufNum, ref Buf12 bufDen) // Compute full remainder, rem = dividend - (quo * divisor). // ulong prod1; +#if TARGET_64BIT + // A 64-by-32-bit product has at most 32 high bits. Keep that + // result wide so the subtraction can consume the low borrow directly. + ulong prod2 = Math.BigMul(bufDen.Low64, quo, out prod1); + ulong low = bufNum.Low64; + ulong num = low - prod1; + // Keep the high word signed so its sign records the full-width + // borrow. Both subtraction and correction stay within Int64. + long high = (long)remainder - (long)prod2; + high -= (num > low) ? 1L : 0L; + + if (high < 0) + { + // The quotient estimate is at most two too large. Add the + // divisor back, including the low word's carry, until nonnegative. + prod1 = bufDen.Low64; + do + { + quo--; + num += prod1; + high += den; + high += (num < prod1) ? 1L : 0L; + } while (high < 0); + } + + bufNum.Low64 = num; + bufNum.U2 = (uint)high; + return quo; +#else uint prod2 = (uint)Math.BigMul(bufDen.Low64, quo, out prod1); ulong num = bufNum.Low64 - prod1; remainder -= (uint)prod2; @@ -521,6 +570,7 @@ private static uint Div128By96(ref Buf16 bufNum, ref Buf12 bufDen) bufNum.Low64 = num; bufNum.U2 = remainder; return quo; +#endif } /// @@ -935,12 +985,12 @@ private static int SearchScale(ulong resMidLo, uint resHi, int scale) /// Returns false if there is an overflow private static bool Add32To96(ref Buf12 bufNum, uint value) { - if ((bufNum.Low64 += value) < value) - { - if (++bufNum.U2 == 0) - return false; - } - return true; + ulong low = bufNum.Low64 + value; + ulong high = bufNum.U2; + high += low < value ? 1UL : 0UL; + bufNum.Low64 = low; + bufNum.U2 = (uint)high; + return high <= uint.MaxValue; } /// @@ -1237,6 +1287,14 @@ internal static void DecAddSub(ref DecCalc d1, ref DecCalc d2, bool sign) // Signs differ - subtract // low64 = d1Low64 - d2.Low64; +#if TARGET_64BIT + // The signed high word retains the full-width borrow. + long difference = (long)d1High - d2.High; + difference -= low64 > d1Low64 ? 1L : 0L; + high = (uint)difference; + if (difference < 0) + goto SignFlip; +#else high = d1High - d2.High; // Propagate carry @@ -1249,12 +1307,21 @@ internal static void DecAddSub(ref DecCalc d1, ref DecCalc d2, bool sign) } else if (high > d1High) goto SignFlip; +#endif } else { // Signs are the same - add // low64 = d1Low64 + d2.Low64; +#if TARGET_64BIT + // Keep the 97th bit until the overflow/rescaling check. + ulong sum = (ulong)d1High + d2.High; + sum += low64 < d1Low64 ? 1UL : 0UL; + high = (uint)sum; + if (sum > uint.MaxValue) + goto AlignedScale; +#else high = d1High + d2.High; // Propagate carry @@ -1267,6 +1334,7 @@ internal static void DecAddSub(ref DecCalc d1, ref DecCalc d2, bool sign) } else if (high < d1High) goto AlignedScale; +#endif } goto ReturnResult; } @@ -1508,7 +1576,7 @@ internal static void VarDecMul(ref DecCalc d1, ref DecCalc d2) { // At least one operand has bits set in the upper 64 bits. // - // Compute and accumulate the 9 partial products into a + // Compute and accumulate the four partial products into a // 192-bit (3*64bit) result. // // [l-hi][l-lo] left high32, low64 @@ -1531,10 +1599,22 @@ internal static void VarDecMul(ref DecCalc d1, ref DecCalc d2) // hi64 will never overflow since the result will always fit in 192 (2*96) bits ulong hi64 = Math.BigMul(d1.High, d2.High); - // Do crosswise multiplications between upper 32bit and lower 64 bits + // Do crosswise multiplications between upper 32bit and lower 64 bits. +#if TARGET_64BIT + // Consume d1.High first so MULX need not preserve it across the other product. + ulong crossHigh = Math.BigMul(d2.Low64, d1.High, out tmp); + mid64 += tmp; + // Add the high product and carry together after the low addition, + // so the JIT can combine them into an add-with-carry instruction. + hi64 = hi64 + crossHigh + ((mid64 < tmp) ? 1UL : 0UL); + + crossHigh = Math.BigMul(d1.Low64, d2.High, out tmp); + mid64 += tmp; + hi64 = hi64 + crossHigh + ((mid64 < tmp) ? 1UL : 0UL); +#else + // Keep conditional increments when 64-bit arithmetic is decomposed into 32-bit operations. hi64 += Math.BigMul(d1.Low64, d2.High, out tmp); mid64 += tmp; - // propagate carry, can be simplified if https://github.com/dotnet/runtime/issues/48247 is done if (mid64 < tmp) ++hi64; @@ -1542,6 +1622,7 @@ internal static void VarDecMul(ref DecCalc d1, ref DecCalc d2) mid64 += tmp; if (mid64 < tmp) ++hi64; +#endif bufProd.Mid64 = mid64; bufProd.High64 = hi64; @@ -2252,8 +2333,19 @@ internal static void VarDecMod(ref DecCalc d1, ref DecCalc d2) { uint den = d2.Low; ulong tmp = ((ulong)d1.High << 32) | d1.Mid; - tmp = ((tmp % den) << 32) | d1.Low; - d1.Low64 = tmp % den; + if (X86.X86Base.IsSupported) + { + // The first remainder is below den, so the final quotient + // fits in 32 bits and the limbs can feed a widening divide. + uint remainder = (uint)(tmp % den); + (_, remainder) = Div64By32(d1.Low, remainder, den); + d1.Low64 = remainder; + } + else + { + tmp = ((tmp % den) << 32) | d1.Low; + d1.Low64 = tmp % den; + } d1.High = 0; } else @@ -2433,7 +2525,6 @@ internal static void InternalRound(ref DecCalc d, uint scale, MidpointRounding m { power = UInt32Powers10[(int)scale]; - // TODO: https://github.com/dotnet/runtime/issues/5213 uint n = d.uhi; if (n == 0) { @@ -2445,25 +2536,34 @@ internal static void InternalRound(ref DecCalc d, uint scale, MidpointRounding m remainder = 0; goto checkRemainder; } +#if TARGET_64BIT + (ulong quotient, ulong rem) = Math.DivRem(tmp, power); + d.Low64 = quotient; + remainder = (uint)rem; +#else + // Keep the remainder calculation narrow on 32-bit targets. ulong div = tmp / power; d.Low64 = div; remainder = (uint)(tmp - div * power); +#endif } else { - uint q; - (d.uhi, remainder) = Math.DivRem(n, power); + (uint highQuotient, remainder) = Math.DivRem(n, power); + d.uhi = highQuotient; + // Each remainder is less than power, so the next quotient + // fits in 32 bits. Reuse the widening divide's remainder. n = d.umid; if ((n | remainder) != 0) { - d.umid = q = (uint)((((ulong)remainder << 32) | n) / power); - remainder = n - q * power; + (uint quotient, remainder) = Div64By32(n, remainder, power); + d.umid = quotient; } n = d.ulo; if ((n | remainder) != 0) { - d.ulo = q = (uint)((((ulong)remainder << 32) | n) / power); - remainder = n - q * power; + (uint quotient, remainder) = Div64By32(n, remainder, power); + d.ulo = quotient; } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Int128.cs b/src/libraries/System.Private.CoreLib/src/System/Int128.cs index 6b9116968224fe..0dc630f1205958 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Int128.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Int128.cs @@ -701,8 +701,27 @@ public static implicit operator Int128(sbyte value) /// public static (Int128 Quotient, Int128 Remainder) DivRem(Int128 left, Int128 right) { +#if TARGET_64BIT + if ((right == -1) && (left == MinValue)) + { + ThrowHelper.ThrowOverflowException(); + } + + bool leftNegative = IsNegative(left); + bool rightNegative = IsNegative(right); + UInt128 dividend = (UInt128)(leftNegative ? -left : left); + UInt128 divisor = (UInt128)(rightNegative ? -right : right); + (UInt128 quotient, UInt128 remainder) = UInt128.DivRem(dividend, divisor); + + // The unsigned divide already computed the remainder. Its sign follows + // the dividend, independently of the quotient's sign. + return ((leftNegative != rightNegative) ? -(Int128)quotient : (Int128)quotient, + leftNegative ? -(Int128)remainder : (Int128)remainder); +#else + // Retain the smaller form when 128-bit arithmetic is decomposed into 32-bit limbs. Int128 quotient = left / right; return (quotient, left - (quotient * right)); +#endif } /// @@ -1119,10 +1138,14 @@ public static Int128 Log2(Int128 value) /// public static Int128 operator %(Int128 left, Int128 right) +#if TARGET_64BIT + => DivRem(left, right).Remainder; +#else { Int128 quotient = left / right; return left - (quotient * right); } +#endif // // IMultiplicativeIdentity diff --git a/src/libraries/System.Private.CoreLib/src/System/Number.BigInteger.cs b/src/libraries/System.Private.CoreLib/src/System/Number.BigInteger.cs index 50656f9dcff6b2..e06b9acd4e51b6 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Number.BigInteger.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Number.BigInteger.cs @@ -686,13 +686,13 @@ public static uint HeuristicDivide(scoped ref BigInteger dividend, scoped ref re dividend._length = divisorLength; } - // If the dividend is still larger than the divisor, we overshot our estimate quotient. To correct, - // we increment the quotient and subtract one more divisor from the dividend (Because we guaranteed the error range). + // If the remainder is still at least the divisor, the estimated quotient was too small. + // Increment the quotient and subtract one more divisor; the estimate is at most one below the true quotient. if (Compare(ref dividend, in divisor) >= 0) { quotient++; - // dividend -= divisor. This is the cold correction path (only on overshoot), so it + // dividend -= divisor. This is the cold correction path (only on an underestimate), so it // reuses the shared SubWithBorrow leaf, which inlines into this local loop. nuint borrow = 0; for (int i = 0; i < divisorLength; i++) diff --git a/src/libraries/System.Private.CoreLib/src/System/Number.DecimalIeee754.cs b/src/libraries/System.Private.CoreLib/src/System/Number.DecimalIeee754.cs index 2e784e4731c33f..9aa479b8c23896 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Number.DecimalIeee754.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Number.DecimalIeee754.cs @@ -2446,6 +2446,48 @@ private static void WideMultiply(TValue left, TValue right, out TValue h return; } + if (typeof(TValue) == typeof(UInt128)) + { + UInt128 a = UInt128.CreateTruncating(left); + UInt128 b = UInt128.CreateTruncating(right); + ulong p00High = Math.BigMul(a.Lower, b.Lower, out ulong p00Low); + + // Preserve the common small-coefficient path: its full product + // needs only one widening multiply. + if ((a.Upper | b.Upper) == 0) + { + high = TValue.Zero; + low = TValue.CreateTruncating(new UInt128(p00High, p00Low)); + return; + } + + ulong p01High = Math.BigMul(a.Lower, b.Upper, out ulong p01Low); + ulong p10High = Math.BigMul(a.Upper, b.Lower, out ulong p10Low); + ulong p11High = Math.BigMul(a.Upper, b.Upper, out ulong p11Low); + + // Accumulate the four exact partial products in base 2^64. + // Each comparison produces a one-bit carry; the two carries + // from the second word can sum to two in the third word. + ulong word1 = p00High + p01Low; + ulong carry1 = (word1 < p00High) ? 1UL : 0UL; + ulong previous = word1; + word1 += p10Low; + carry1 += (word1 < previous) ? 1UL : 0UL; + + ulong word2 = p01High + p10High; + ulong carry2 = (word2 < p01High) ? 1UL : 0UL; + previous = word2; + word2 += p11Low; + carry2 += (word2 < previous) ? 1UL : 0UL; + previous = word2; + word2 += carry1; + carry2 += (word2 < previous) ? 1UL : 0UL; + + high = TValue.CreateTruncating(new UInt128(p11High + carry2, word2)); + low = TValue.CreateTruncating(new UInt128(word1, p00Low)); + return; + } + int bits = TValue.Zero.GetByteCount() * 8; int half = bits / 2; TValue lowMask = (TValue.One << half) - TValue.One; @@ -2570,13 +2612,30 @@ private static void AlignWideToCommonExponent(ref TValue high, private static void WideAdd(TValue leftHigh, TValue leftLow, TValue rightHigh, TValue rightLow, out TValue high, out TValue low) where TValue : unmanaged, IBinaryInteger { - low = leftLow + rightLow; - high = leftHigh + rightHigh; - - if (low < leftLow) - { - high += TValue.One; +#if TARGET_64BIT + if (typeof(TValue) == typeof(UInt128)) + { + UInt128 a0 = UInt128.CreateTruncating(leftLow); + UInt128 a1 = UInt128.CreateTruncating(leftHigh); + UInt128 b0 = UInt128.CreateTruncating(rightLow); + UInt128 b1 = UInt128.CreateTruncating(rightHigh); + // Feed each one-bit carry directly into the next native limb. + nuint w0 = BigIntegerCalculator.AddWithCarry((nuint)a0.Lower, (nuint)b0.Lower, 0, out nuint c); + nuint w1 = BigIntegerCalculator.AddWithCarry((nuint)a0.Upper, (nuint)b0.Upper, c, out c); + nuint w2 = BigIntegerCalculator.AddWithCarry((nuint)a1.Lower, (nuint)b1.Lower, c, out c); + nuint w3 = (nuint)a1.Upper + (nuint)b1.Upper + c; + high = TValue.CreateTruncating(new UInt128(w3, w2)); + low = TValue.CreateTruncating(new UInt128(w1, w0)); + return; } +#endif + + // Keep the low result and one-bit carry in locals so the JIT can + // consume the addition flags before writing the output limbs. + TValue sum = leftLow + rightLow; + TValue carry = sum < leftLow ? TValue.One : TValue.Zero; + high = (leftHigh + rightHigh) + carry; + low = sum; } /// @@ -2597,14 +2656,30 @@ private static int WideCompare(TValue leftHigh, TValue leftLow, TValue r private static void WideSubtract(TValue leftHigh, TValue leftLow, TValue rightHigh, TValue rightLow, out TValue high, out TValue low) where TValue : unmanaged, IBinaryInteger { - high = leftHigh - rightHigh; - - if (leftLow < rightLow) - { - high -= TValue.One; +#if TARGET_64BIT + if (typeof(TValue) == typeof(UInt128)) + { + UInt128 a0 = UInt128.CreateTruncating(leftLow); + UInt128 a1 = UInt128.CreateTruncating(leftHigh); + UInt128 b0 = UInt128.CreateTruncating(rightLow); + UInt128 b1 = UInt128.CreateTruncating(rightHigh); + // As with addition, the borrow remains a bit across all four limbs. + nuint w0 = BigIntegerCalculator.SubWithBorrow((nuint)a0.Lower, (nuint)b0.Lower, 0, out nuint b); + nuint w1 = BigIntegerCalculator.SubWithBorrow((nuint)a0.Upper, (nuint)b0.Upper, b, out b); + nuint w2 = BigIntegerCalculator.SubWithBorrow((nuint)a1.Lower, (nuint)b1.Lower, b, out b); + nuint w3 = (nuint)a1.Upper - (nuint)b1.Upper - b; + high = TValue.CreateTruncating(new UInt128(w3, w2)); + low = TValue.CreateTruncating(new UInt128(w1, w0)); + return; } +#endif - low = leftLow - rightLow; + // Compute the low subtraction before consuming its borrow in the + // high limb. Delay output stores until both results are available. + TValue difference = leftLow - rightLow; + TValue borrow = leftLow < rightLow ? TValue.One : TValue.Zero; + high = (leftHigh - rightHigh) - borrow; + low = difference; } /// @@ -2675,6 +2750,47 @@ private static TValue WideDivideByPow10(ref TValue high, ref TValue low, return remainder; } +#if TARGET_64BIT + if (typeof(TValue) == typeof(uint)) + { + ulong value = ((ulong)uint.CreateTruncating(high) << 32) | uint.CreateTruncating(low); + (ulong quotient, ulong remainder) = Math.DivRem(value, uint.CreateTruncating(divisor)); + high = TValue.CreateTruncating(quotient >> 32); + low = TValue.CreateTruncating(quotient); + return TValue.CreateTruncating(remainder); + } +#endif + + // A native widening divide consumes an entire limb and returns its + // remainder. Start with ordinary division; each following high + // input is then a remainder strictly below the divisor. + if (typeof(TValue) == typeof(ulong) && Runtime.Intrinsics.X86.X86Base.X64.IsSupported) + { + ulong d = ulong.CreateTruncating(divisor); + (ulong upper, ulong remainder) = Math.DivRem(ulong.CreateTruncating(high), d); + ulong lower = (ulong)BigIntegerCalculator.DivRem((nuint)remainder, (nuint)ulong.CreateTruncating(low), (nuint)d, out nuint nativeRemainder); + high = TValue.CreateTruncating(upper); + low = TValue.CreateTruncating(lower); + return TValue.CreateTruncating(nativeRemainder); + } + + if (typeof(TValue) == typeof(UInt128) && Runtime.Intrinsics.X86.X86Base.X64.IsSupported) + { + // SinglePassPow10 bounds this divisor by 10^19, so it fits in + // one ulong even though the dividend occupies four ulongs. + UInt128 h = UInt128.CreateTruncating(high); + UInt128 l = UInt128.CreateTruncating(low); + UInt128 d = UInt128.CreateTruncating(divisor); + Debug.Assert(d.Upper == 0); + (ulong digit3, ulong remainder) = Math.DivRem(h.Upper, d.Lower); + nuint digit2 = BigIntegerCalculator.DivRem((nuint)remainder, (nuint)h.Lower, (nuint)d.Lower, out nuint nativeRemainder); + nuint digit1 = BigIntegerCalculator.DivRem(nativeRemainder, (nuint)l.Upper, (nuint)d.Lower, out nativeRemainder); + nuint digit0 = BigIntegerCalculator.DivRem(nativeRemainder, (nuint)l.Lower, (nuint)d.Lower, out nativeRemainder); + high = TValue.CreateTruncating(new UInt128(digit3, digit2)); + low = TValue.CreateTruncating(new UInt128(digit1, digit0)); + return TValue.CreateTruncating(nativeRemainder); + } + int bits = TValue.Zero.GetByteCount() * 8; int half = bits / 2; TValue lowMask = (TValue.One << half) - TValue.One; diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/BigIntegerCalculator.Shared.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/BigIntegerCalculator.Shared.cs index c573a23c4f3e1a..261494a97bcb7e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/BigIntegerCalculator.Shared.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/BigIntegerCalculator.Shared.cs @@ -184,13 +184,18 @@ internal static void DivideByPowerOfTwo(ReadOnlySpan left, int exponent, /// /// Performs widening addition of two limbs plus a carry-in, returning the sum and carry-out. - /// On 64-bit: uses 128-bit arithmetic. On 32-bit: uses 64-bit arithmetic. + /// On 64-bit: uses limb additions and unsigned carry comparisons. On 32-bit: uses 64-bit arithmetic. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static nuint AddWithCarry(nuint a, nuint b, nuint carryIn, out nuint carryOut) { if (nint.Size == 8) { + // Keep each unsigned comparison tied to its own addition. Supporting JITs can reuse the + // addition's flags and, when carryIn is proven to be 0 or 1, fold both steps into ADC/ADCS. + // For a limb chain, initialize carry to zero and feed carryOut directly into the next limb. + // Then c1 and c2 cannot both be one, so carryOut remains a bit. An arbitrary full-limb + // carryIn can instead produce carryOut == 2 and must not be treated as a single carry flag. nuint sum1 = a + b; nuint c1 = (sum1 < a) ? 1 : (nuint)0; nuint sum2 = sum1 + carryIn; @@ -208,7 +213,8 @@ internal static nuint AddWithCarry(nuint a, nuint b, nuint carryIn, out nuint ca /// /// Performs widening subtraction of two limbs with a borrow-in, returning the difference and borrow-out. - /// borrowOut is 0 (no borrow) or 1 (borrow occurred). + /// When borrowIn is 0 or 1, borrowOut is also 0 (no borrow) or 1 (borrow occurred). + /// An arbitrary full-limb borrowIn can produce borrowOut == 2. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static nuint SubWithBorrow(nuint a, nuint b, nuint borrowIn, out nuint borrowOut) @@ -226,7 +232,7 @@ internal static nuint SubWithBorrow(nuint a, nuint b, nuint borrowIn, out nuint else { long diff = (long)a - (long)b - (long)borrowIn; - borrowOut = (uint)(-(int)(diff >> 32)); // 0 or 1 + borrowOut = (uint)(-(int)(diff >> 32)); // 0 or 1 for a one-bit borrowIn; otherwise up to 2 return (uint)diff; } } @@ -237,12 +243,30 @@ internal static nuint SubWithBorrow(nuint a, nuint b, nuint borrowIn, out nuint [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static nuint DivRem(nuint hi, nuint lo, nuint divisor, out nuint remainder) { + Debug.Assert(hi < divisor || divisor == 0); + // Callers ensure hi < divisor, so the quotient fits in one limb. + // Use the widening divide before splitting into smaller divisions; + // it produces both the quotient and remainder in one instruction. +#pragma warning disable SYSLIB5004 // X86Base.DivRem is experimental + if (X86Base.X64.IsSupported) + { + (ulong q, ulong r) = X86Base.X64.DivRem(lo, hi, divisor); + remainder = (nuint)r; + return (nuint)q; + } + + if (nint.Size == 4 && X86Base.IsSupported) + { + (uint q, uint r) = X86Base.DivRem((uint)lo, (uint)hi, (uint)divisor); + remainder = r; + return q; + } +#pragma warning restore SYSLIB5004 + if (nint.Size == 8) { // Compute (hi * 2^64 + lo) / divisor. // hi < divisor is guaranteed by callers, so quotient fits in 64 bits. - Debug.Assert(hi < (ulong)divisor || divisor == 0); - if (hi == 0) { (ulong q, ulong r) = Math.DivRem(lo, (ulong)divisor); @@ -267,18 +291,9 @@ internal static nuint DivRem(nuint hi, nuint lo, nuint divisor, out nuint remain } { -#pragma warning disable SYSLIB5004 // X86Base.DivRem is experimental - if (X86Base.X64.IsSupported) - { - (ulong q, ulong r) = X86Base.X64.DivRem(lo, hi, divisor); - remainder = (nuint)r; - return (nuint)q; - } -#pragma warning restore SYSLIB5004 - UInt128 value = ((UInt128)(ulong)hi << 64) | (ulong)lo; - UInt128 digit = value / (ulong)divisor; - remainder = (nuint)(ulong)(value - digit * (ulong)divisor); + (UInt128 digit, UInt128 rem) = UInt128.DivRem(value, (ulong)divisor); + remainder = (nuint)(ulong)rem; return (nuint)(ulong)digit; } } @@ -295,6 +310,8 @@ internal static nuint DivRem(nuint hi, nuint lo, nuint divisor, out nuint remain /// Multiply by scalar: result[0..left.Length] = left * multiplier. /// Returns the carry out. Unrolled by 4 on 64-bit. /// Unlike MulAdd1, this writes to result rather than accumulating. + /// Widen before multiplying, store the low limb, and pass the high limb to the next product. + /// This carry is a full limb, unlike the one-bit carry used by an addition chain. /// internal static nuint Mul1(Span result, ReadOnlySpan left, nuint multiplier) { @@ -345,22 +362,29 @@ internal static nuint Mul1(Span result, ReadOnlySpan left, nuint m /// /// Fused multiply-accumulate by scalar: result[0..left.Length] += left * multiplier. - /// Returns the carry out. Unrolled by 4 on 64-bit to overlap multiply latencies. + /// Returns the carry out. Processes four limbs per iteration on 64-bit platforms. /// internal static nuint MulAdd1(Span result, ReadOnlySpan left, nuint multiplier) { - Debug.Assert(result.Length >= left.Length); - int length = left.Length; + // Give both spans the same bound so the loop checks cover every lane. This slice validates + // the destination once; Debug.Assert alone would not establish the bound in release builds. + result = result.Slice(0, length); int i = 0; nuint carry = 0; if (nint.Size == 8) { - // Unroll by 4: mulx has 3-5 cycle latency but 1 cycle throughput, - // so issuing 4 multiplies allows the CPU to pipeline them while - // carry chains complete sequentially behind. - for (; i + 3 < length; i += 4) + // Unroll by four to expose independent products while propagating carry between limbs. + // Widen before multiplication and keep both additions in UInt128: even the maximum product + // plus a destination limb and a full-limb carry fits. Store each low limb and feed its high + // limb directly into the next product; the last high limb also feeds the next iteration. + // Keep the i < length - 3 loop bound so the JIT can prove all four accesses are in range. + // On x64 with BMI2 and ADX, the JIT can use MULX with ADCX/ADOX and keep both carry + // flags live across loop iterations, materializing the carry before the remainder loop. + // ARM64 can similarly preserve C with MUL/UMULH/ADCS/ADC/ADDS. Let the JIT derive the + // countdown from this ascending loop; a separate index and countdown can obscure the proof. + for (; i < length - 3; i += 4) { UInt128 p0 = (UInt128)(ulong)left[i] * (ulong)multiplier + (ulong)result[i] + (ulong)carry; result[i] = (nuint)(ulong)p0; @@ -515,11 +539,14 @@ public static void Add(ReadOnlySpan left, ReadOnlySpan right, Span } } - // Establish cross-span length relationships so the JIT can - // elide bounds checks for left[i] and bits[i] in the loop. + // Establish cross-span length relationships so the JIT can elide bounds checks for left[i] + // and bits[i]. These executed checks, unlike Debug.Assert, establish the bounds in release builds. _ = left[right.Length - 1]; _ = bits[right.Length]; + // The zero seed and AddWithCarry recurrence prove that carry stays a bit. Keep the helper + // inlineable and use the same index for all spans: supporting JITs can keep CF (x64) or C + // (ARM64) live across a derived countdown loop and materialize carry only when leaving it. nuint carry = 0; for (int i = 0; i < right.Length; i++) @@ -591,6 +618,8 @@ public static void AddSelf(Span left, ReadOnlySpan right) _ = left[right.Length - 1]; } + // As in Add, keep the one-bit carry recurrence in this ascending loop. The final carry + // is still needed by the tail below, even when the JIT carries it in flags inside this loop. for (; i < right.Length; i++) { left[i] = AddWithCarry(left[i], right[i], carry, out carry); @@ -1772,10 +1801,8 @@ private static bool DivideGuessTooBig(nuint q, nuint valHi1, nuint valHi0, nuint chkLoHi = nuint.BigMul(divLo, q, out nuint chkLoLo); chkHiLo += chkLoHi; - if (chkHiLo < chkLoHi) - { - chkHiHi++; - } + // Add the carry explicitly so the JIT can reuse the low addition's flags. + chkHiHi += (chkHiLo < chkLoHi) ? (nuint)1 : 0; return (chkHiHi > valHi1) || ((chkHiHi == valHi1) && ((chkHiLo > valHi0) || ((chkHiLo == valHi0) && (chkLoLo > valLo)))); diff --git a/src/libraries/System.Private.CoreLib/src/System/UInt128.cs b/src/libraries/System.Private.CoreLib/src/System/UInt128.cs index df6f7d19a6081a..97047b261a1025 100644 --- a/src/libraries/System.Private.CoreLib/src/System/UInt128.cs +++ b/src/libraries/System.Private.CoreLib/src/System/UInt128.cs @@ -838,10 +838,8 @@ public static (UInt128 Quotient, UInt128 Remainder) DivRem(UInt128 left, UInt128 ulong chkLoHi = Math.BigMul(divLo, q, out ulong chkLo); ulong chkMi = chkHiLo + chkLoHi; - if (chkMi < chkLoHi) - { - chkHi++; - } + // Add the carry explicitly so the JIT can reuse the low addition's flags. + chkHi += (chkMi < chkLoHi) ? 1UL : 0UL; UInt128 divisor = new UInt128(divHi, divLo); UInt128 chkMiLo = new UInt128(chkMi, chkLo); @@ -985,11 +983,19 @@ public static (UInt128 Quotient, UInt128 Remainder) DivRem(UInt128 left, UInt128 Debug.Assert(new UInt128(valHi, valMiLo) < new UInt128(chkHi, chkLo)); q--; +#if TARGET_64BIT + // Subtract the low word first and consume its borrow in the + // high word, allowing SUB/SBB or SUBS/SBC without a branch. + ulong difference = chkLo - divisor; + chkHi -= (difference > chkLo) ? 1U : 0U; + chkLo = difference; +#else if (chkLo < divisor) { chkHi--; } chkLo -= divisor; +#endif } ulong remainder = valMiLo - chkLo; @@ -1412,12 +1418,14 @@ public static UInt128 BigMul(UInt128 left, UInt128 right, out UInt128 lower) ulong bl = right._lower; ulong bh = right._upper; - UInt128 mull = Math.BigMul(al, bl); - UInt128 t = Math.BigMul(ah, bl) + mull._upper; - UInt128 tl = Math.BigMul(al, bh) + t._lower; + // Group products with the same first operand: MULX uses a fixed + // register for that operand, while the other operand can stay live. + UInt128 mull = Math.BigMul(bl, al); + UInt128 t = Math.BigMul(bl, ah) + mull._upper; + UInt128 tl = Math.BigMul(bh, al) + t._lower; lower = new UInt128(tl._lower, mull._lower); - return Math.BigMul(ah, bh) + t._upper + tl._upper; + return Math.BigMul(bh, ah) + t._upper + tl._upper; } // diff --git a/src/libraries/System.Private.CoreLib/src/System/UIntPtr.cs b/src/libraries/System.Private.CoreLib/src/System/UIntPtr.cs index 5343dc70dea2bd..f2bf581248b7e7 100644 --- a/src/libraries/System.Private.CoreLib/src/System/UIntPtr.cs +++ b/src/libraries/System.Private.CoreLib/src/System/UIntPtr.cs @@ -184,6 +184,7 @@ public static nuint MinValue /// The integer to multiply with . /// The lower half of the full product. /// The upper half of the full product. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static nuint BigMul(nuint left, nuint right, out nuint lower) { #if TARGET_64BIT diff --git a/src/libraries/System.Runtime.Numerics/src/System/Numerics/BigIntegerCalculator.PowMod.cs b/src/libraries/System.Runtime.Numerics/src/System/Numerics/BigIntegerCalculator.PowMod.cs index 9dbb64a65ea084..ffaf82cda2dcb9 100644 --- a/src/libraries/System.Runtime.Numerics/src/System/Numerics/BigIntegerCalculator.PowMod.cs +++ b/src/libraries/System.Runtime.Numerics/src/System/Numerics/BigIntegerCalculator.PowMod.cs @@ -857,15 +857,28 @@ private static int MontgomeryReduce(Span value, ReadOnlySpan modul nuint m = value[i] * n0inv; nuint carry = 0; - for (int j = 0; j < k; j++) + if (nint.Size == 8) { - if (nint.Size == 8) + // Keep both spans the same length and index them with j, rather than value[i + j]. + // The slice checks the whole row once, allowing the JIT to eliminate per-limb bounds checks. + // Use the ordinary ascending loop: the JIT can derive a countdown that preserves carry flags; + // a separate hand-written countdown and index can obscure the bounds proof. + // This multiply-add shape lets supporting JITs keep carry flags live across the backedge + // (MULX/ADCX/ADOX on x64, MUL/UMULH/ADCS/ADC/ADDS on ARM64), without unrolling or a helper call. + Span row = value.Slice(i, k); + for (int j = 0; j < row.Length; j++) { - UInt128 p = (UInt128)m * modulus[j] + value[i + j] + carry; - value[i + j] = (nuint)(ulong)p; + // Widen before multiplying and keep both additions in UInt128. The incoming carry is + // a full limb, not a single bit: product + row[j] + carry fits in 128 bits. Store the low + // limb and feed the high limb directly into the next iteration; preserve it on loop exit. + UInt128 p = (UInt128)(ulong)modulus[j] * (ulong)m + (ulong)row[j] + (ulong)carry; + row[j] = (nuint)(ulong)p; carry = (nuint)(ulong)(p >> 64); } - else + } + else + { + for (int j = 0; j < k; j++) { ulong p = (ulong)m * modulus[j] + value[i + j] + carry; value[i + j] = (uint)p; diff --git a/src/native/minipal/cpufeatures.c b/src/native/minipal/cpufeatures.c index ed7d7612779842..7b4083050d9a5e 100644 --- a/src/native/minipal/cpufeatures.c +++ b/src/native/minipal/cpufeatures.c @@ -349,6 +349,11 @@ int minipal_getcpufeatures(void) { __cpuidex(cpuidInfo, 0x00000007, 0x00000000); + if ((cpuidInfo[CPUID_EBX] & (1 << 19)) != 0) // ADX + { + result |= XArchIntrinsicConstants_Adx; + } + if ((result & XArchIntrinsicConstants_Avx) != 0) { if (((cpuidInfo[CPUID_EBX] & (1 << 5)) != 0) && // AVX2 diff --git a/src/native/minipal/cpufeatures.h b/src/native/minipal/cpufeatures.h index 31786bc641971b..9476dede581c7d 100644 --- a/src/native/minipal/cpufeatures.h +++ b/src/native/minipal/cpufeatures.h @@ -31,6 +31,7 @@ #define XArchIntrinsicConstants_WaitPkg (1 << 16) #define XArchIntrinsicConstants_X86Serialize (1 << 17) #define XArchIntrinsicConstants_Avx512Bmm (1 << 18) +#define XArchIntrinsicConstants_Adx (1 << 19) #endif // HOST_X86 || HOST_AMD64 #if defined(HOST_ARM64) diff --git a/src/tests/JIT/opt/Add/ArithmeticCodegen.cs b/src/tests/JIT/opt/Add/ArithmeticCodegen.cs new file mode 100644 index 00000000000000..4acfc0bbf3ee78 --- /dev/null +++ b/src/tests/JIT/opt/Add/ArithmeticCodegen.cs @@ -0,0 +1,292 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Threading; +using Xunit; + +public class ArithmeticCodegenTests +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong FoldMultiply(ulong left, ulong right) + { + // Independent products do not require flags-preserving multiplication. + // X64: {{^ +}}mul {{.*}} + // X64-NOT: mulx + // X64: ret + ulong high = Math.BigMul(left, right, out ulong low); + return high ^ low; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong FoldMultiplyWithCarry(ulong left, ulong right, ulong addend, out ulong high) + { + // An unrelated carry in the same block must not force this product to MULX. + // X64-NOT: mulx + // X64: {{^ +}}mul {{.*}} + // X64-NOT: mulx + // X64: {{^ +}}adc {{.*}} + // X64-NOT: mulx + // X64: ret + ulong productHigh = Math.BigMul(left, right, out ulong productLow); + ulong sum = unchecked(left + addend); + high = unchecked(right + (sum < left ? 1UL : 0UL)); + return productHigh ^ productLow ^ sum; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool DecrementIsZero(ref int value) + { + // ZF-only consumers can still use the shorter DEC instruction. + // X64: {{^ +}}dec + // X64-NEXT: sete + return Interlocked.Decrement(ref value) == 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong IncrementWithCarry(ulong value, out ulong high) + { + // INC would preserve stale CF instead of producing the overflow bit. + // X64: {{^ +}}add {{.*}}, 1 + // X64-NOT: {{^ +}}inc + // X64: ret + ulong result = unchecked(value + 1); + high = result < value ? 1UL : 0UL; + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong StoredRemainder(ulong value, ulong divisor, out ulong quotient) + { + ulong q = value / divisor; + ulong remainder = value - q * divisor; + quotient = q; + return remainder; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static (ulong, ulong, ulong) DestinationRead(ulong value, ulong divisor, ulong remainder) + { + ulong quotient = value / divisor; + ulong oldRemainder = remainder; + remainder = value - quotient * divisor; + return (quotient, remainder, oldRemainder); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ReplaceDividend(ulong value, ulong divisor) + { + ulong quotient = value / divisor; + value = value - quotient * divisor; + return value; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void MayThrow(bool fail) + { + if (fail) + { + throw new InvalidOperationException(); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong RemainderAcrossException(ulong value, ulong divisor, ulong remainder, bool fail) + { + try + { + ulong quotient = value / divisor; + MayThrow(fail); + remainder = value - quotient * divisor; + } + catch (InvalidOperationException) + { + return remainder; + } + return remainder; + } + + private static class CarryState + { + public static readonly ulong Mask; + static CarryState() => Mask = 0x123456789ABCDEF0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ObserveCarry(ulong carry) => carry; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong CarryAcrossHelper(ulong left, ulong right, ulong high, bool observe) + { + // Generic static initialization may split the block before lowering. + // The carry is consumed here and remains observable in successor blocks. + ulong mask = CarryState.Mask; + ulong low = unchecked(left + right); + ulong carry = low < left ? 1UL : 0UL; + high = unchecked(high + carry); + if (observe) + { + high ^= ObserveCarry(carry) + mask; + } + return low ^ high ^ carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong CarryAcrossException(ulong left, ulong right, ulong high, bool fail) + { + ulong carry = 123; + try + { + ulong low = unchecked(left + right); + carry = low < left ? 1UL : 0UL; + high = unchecked(high + carry); + MayThrow(fail); + return low ^ high; + } + catch (InvalidOperationException) + { + return carry; + } + } + + private struct LimbPair + { + public ulong Low; + public ulong High; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong CarryAcrossStructStore(ulong left, ulong right, LimbPair replacement) + { + LimbPair value = new LimbPair { Low = left, High = right }; + ulong sum = unchecked(value.Low + right); + ulong carry = sum < value.Low ? 1UL : 0UL; + // Reading High must observe the replacement, even if the field is promoted. + value = replacement; + return sum ^ unchecked(value.High + carry) ^ value.Low; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong BorrowAcrossStructStore(ulong left, ulong right, LimbPair replacement) + { + LimbPair value = new LimbPair { Low = left, High = right }; + ulong difference = unchecked(value.Low - right); + ulong borrow = value.Low < right ? 1UL : 0UL; + value = replacement; + return difference ^ unchecked(value.High - borrow) ^ value.Low; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong CarryAcrossFieldAlias(ulong left, ulong right, ulong high) + { + LimbPair value = new LimbPair { Low = left, High = right }; + ref ulong alias = ref value.High; + ulong sum = unchecked(value.Low + right); + ulong carry = sum < value.Low ? 1UL : 0UL; + alias = high; + return sum ^ unchecked(value.High + carry); + } + + [Fact] + public static void TestEntryPoint() + { + ulong[] edges = { 0, 1, 2, uint.MaxValue, 1UL << 32, 1UL << 63, ulong.MaxValue - 1, ulong.MaxValue }; + foreach (ulong left in edges) + { + foreach (ulong right in edges) + { + BigInteger product = (BigInteger)left * right; + ulong folded = (ulong)(product & ulong.MaxValue) ^ (ulong)(product >> 64); + if (FoldMultiply(left, right) != folded) + { + throw new Exception("FoldMultiply"); + } + foreach (ulong addend in edges) + { + LimbPair replacement = new LimbPair { Low = right, High = addend }; + ulong lowSum = unchecked(left + right); + ulong carry = (ulong)(((BigInteger)left + right) >> 64); + ulong difference = unchecked(left - right); + ulong borrow = left < right ? 1UL : 0UL; + if (CarryAcrossStructStore(left, right, replacement) != + (lowSum ^ unchecked(addend + carry) ^ right) || + BorrowAcrossStructStore(left, right, replacement) != + (difference ^ unchecked(addend - borrow) ^ right) || + CarryAcrossFieldAlias(left, right, addend) != (lowSum ^ unchecked(addend + carry))) + { + throw new Exception("Carry/borrow operand observed across struct or field store"); + } + ulong actual = FoldMultiplyWithCarry(left, right, addend, out ulong mixedHigh); + BigInteger sum = (BigInteger)left + addend; + ulong expected = (ulong)(product >> 64) ^ (ulong)(product & ulong.MaxValue) ^ + (ulong)(sum & ulong.MaxValue); + if (actual != expected || mixedHigh != unchecked(right + (ulong)(sum >> 64))) + { + throw new Exception("Independent multiply with carry"); + } + } + foreach (bool observe in new[] { false, true }) + { + BigInteger sum = (BigInteger)left + right; + ulong carry = (ulong)(sum >> 64); + ulong highPart = unchecked(left + carry); + if (observe) + { + highPart ^= carry + 0x123456789ABCDEF0; + } + ulong expected = (ulong)(sum & ulong.MaxValue) ^ highPart ^ carry; + if (CarryAcrossHelper(left, right, left, observe) != expected || + CarryAcrossHelper(left, right, left, observe) != expected) + { + throw new Exception("Carry observed across helper expansion and branches"); + } + } + ulong expectedCarry = (ulong)(((BigInteger)left + right) >> 64); + if (CarryAcrossException(left, right, left, true) != expectedCarry || + CarryAcrossException(left, right, left, false) != + (unchecked(left + right) ^ unchecked(left + expectedCarry))) + { + throw new Exception("Carry observed by exception handler"); + } + if (right != 0) + { + ulong rem = StoredRemainder(left, right, out ulong quotient); + var observed = DestinationRead(left, right, 0x123456789ABCDEF0); + if (rem != left % right || quotient != left / right || + observed != (quotient, rem, 0x123456789ABCDEF0UL) || ReplaceDividend(left, right) != rem) + { + throw new Exception("Stored remainder"); + } + } + } + if (RemainderAcrossException(left, 7, 123, true) != 123 || + RemainderAcrossException(left, 7, 123, false) != left % 7) + { + throw new Exception("Remainder exception ordering"); + } + if (IncrementWithCarry(left, out ulong high) != unchecked(left + 1) || + high != (left == ulong.MaxValue ? 1UL : 0UL)) + { + throw new Exception("Carry flag"); + } + } + foreach (int start in new[] { int.MinValue, -1, 0, 1, 2, int.MaxValue }) + { + int value = start; + if (DecrementIsZero(ref value) != (start == 1) || value != unchecked(start - 1)) + { + throw new Exception("Zero flag"); + } + } + try + { + StoredRemainder(1, 0, out _); + throw new Exception("Missing division exception"); + } + catch (DivideByZeroException) + { + } + } +} diff --git a/src/tests/JIT/opt/Add/ArithmeticCodegen.csproj b/src/tests/JIT/opt/Add/ArithmeticCodegen.csproj new file mode 100644 index 00000000000000..d1892ed3a4be23 --- /dev/null +++ b/src/tests/JIT/opt/Add/ArithmeticCodegen.csproj @@ -0,0 +1,12 @@ + + + true + None + True + + + + + + + diff --git a/src/tests/JIT/opt/Add/Borrow.cs b/src/tests/JIT/opt/Add/Borrow.cs new file mode 100644 index 00000000000000..ad2166a49bb3ee --- /dev/null +++ b/src/tests/JIT/opt/Add/Borrow.cs @@ -0,0 +1,347 @@ +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Xunit; + +public class BorrowTests +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 Sub128(UInt128 a, UInt128 b) + { + // X64: sub + // X64: sbb + // ARM64: subs + // ARM64: sbc + return a - b; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Int128 SubSigned128(Int128 a, Int128 b) + { + // X64: sub + // X64: sbb + // ARM64: subs + // ARM64: sbc + return unchecked(a - b); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Borrow(ulong a, ulong b, ulong high, out ulong difference) + { + // X64: sub + // X64: adc + // ARM64: subs + // ARM64: cinc + ulong value = a - b; + ulong result = high + (value > a ? 1UL : 0); + difference = value; + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong BorrowOnly(ulong a, ulong b, ulong high, out ulong difference) + { + // X64: sbb + // ARM64: sbc {{.*}}, xzr + ulong value = a - b; + ulong result = high - (value > a ? 1UL : 0); + difference = value; + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Branch(ulong a, ulong b, out ulong difference) + { + difference = a - b; + if (difference > a) + { + return 13; + } + return 29; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Select(ulong a, ulong b, ulong x, ulong y, out ulong difference) + { + difference = a - b; + return difference > a ? x : y; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong Sub(ulong a, ulong b, ulong borrow, out ulong next) + { + ulong d = a - b; + ulong b1 = d > a ? 1UL : 0; + ulong r = d - borrow; + next = b1 + (r > d ? 1UL : 0); + return r; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong LimbLoop(Span result, ReadOnlySpan left, ReadOnlySpan right) + { + // X64: sbb + // X64-NOT: setb + // X64: jne + // X64: setb + // ARM64: sbcs + // ARM64-NOT: cset + // ARM64: cbnz + // ARM64: cset + result = result.Slice(0, left.Length); + right = right.Slice(0, left.Length); + ulong borrow = 0; + for (int i = 0; i < left.Length; i++) + { + result[i] = Sub(left[i], right[i], borrow, out borrow); + } + return borrow; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint LimbLoop32(Span result, ReadOnlySpan left, ReadOnlySpan right) + { + result = result.Slice(0, left.Length); + right = right.Slice(0, left.Length); + uint borrow = 0; + for (int i = 0; i < left.Length; i++) + { + uint d = left[i] - right[i]; + uint b1 = d > left[i] ? 1U : 0; + uint r = d - borrow; + borrow = b1 + (r > d ? 1U : 0); + result[i] = r; + } + return borrow; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ArbitraryBorrow(ulong a, ulong b, ulong carry, out ulong next) => Sub(a, b, carry, out next); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong VolatileLoop(Span result, ReadOnlySpan right) + { + ulong borrow = 0; + for (int i = 0; i < right.Length; i++) + { + result[i] = Sub(System.Threading.Volatile.Read(ref result[i]), right[i], borrow, out borrow); + } + return borrow; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong AcrossCall(ulong a, ulong b, ulong high) + { + ulong d = a - b; + ulong borrow = d > a ? 1UL : 0; + return Opaque(high) + borrow; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Opaque(ulong value) => ~value; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong SubMul(Span result, ReadOnlySpan right, ulong multiplier) + { + // X64: mul{{x| }} + // X64: sub + // X64-NEXT: adc + // ARM64: umulh + // ARM64: subs + // ARM64-NEXT: cinc + ulong carry = 0; + int i = 0; + for (; i + 3 < right.Length; i += 4) + { + UInt128 product0 = (UInt128)right[i + 0] * multiplier + carry; + ulong low0 = (ulong)product0; + ulong high0 = (ulong)(product0 >> 64); + ulong original0 = result[i + 0]; + result[i + 0] = original0 - low0; + high0 += original0 < low0 ? 1UL : 0; + UInt128 product1 = (UInt128)right[i + 1] * multiplier + high0; + ulong low1 = (ulong)product1; + ulong high1 = (ulong)(product1 >> 64); + ulong original1 = result[i + 1]; + result[i + 1] = original1 - low1; + high1 += original1 < low1 ? 1UL : 0; + UInt128 product2 = (UInt128)right[i + 2] * multiplier + high1; + ulong low2 = (ulong)product2; + ulong high2 = (ulong)(product2 >> 64); + ulong original2 = result[i + 2]; + result[i + 2] = original2 - low2; + high2 += original2 < low2 ? 1UL : 0; + UInt128 product3 = (UInt128)right[i + 3] * multiplier + high2; + ulong low3 = (ulong)product3; + ulong high3 = (ulong)(product3 >> 64); + ulong original3 = result[i + 3]; + result[i + 3] = original3 - low3; + high3 += original3 < low3 ? 1UL : 0; + carry = high3; + } + for (; i < right.Length; i++) + { + UInt128 product = (UInt128)right[i] * multiplier + carry; + ulong low = (ulong)product; + ulong high = (ulong)(product >> 64); + ulong original = result[i]; + result[i] = original - low; + high += original < low ? 1UL : 0; + carry = high; + } + return carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool CompareRight(ulong a, ulong b) => unchecked(a - b) > b; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong CheckedSub(ulong a, ulong b) => checked(a - b); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong LimbLoopWithTail(Span left, ReadOnlySpan right) + { + // X64: sbb + // X64-NOT: setb + // X64: jne + // X64: setb + // ARM64: sbcs + // ARM64-NOT: cset + // ARM64: cbnz + // ARM64: cset + if (right.Length != 0) _ = left[right.Length - 1]; + ulong borrow = 0; + int i = 0; + for (; i < right.Length; i++) + { + left[i] = Sub(left[i], right[i], borrow, out borrow); + } + for (; borrow != 0 && i < left.Length; i++) + { + ulong value = left[i]; + left[i] = value - borrow; + borrow = value == 0 ? 1UL : 0; + } + return borrow; + } + + [Fact] + public static void TestEntryPoint() + { + Random random = new Random(125803); + ulong[] boundary = { 0, 1, 2, 0x7FFFFFFFFFFFFFFF, 0x8000000000000000, ulong.MaxValue - 1, ulong.MaxValue }; + foreach (ulong a in boundary) + foreach (ulong b in boundary) + foreach (ulong c in boundary) + { + Check(a, b, c, ~c); + } + for (int trial = 0; trial < 10000; trial++) + { + ulong a = (ulong)random.NextInt64() << 1 | (uint)random.Next(2); + ulong b = (ulong)random.NextInt64() << 1 | (uint)random.Next(2); + Check(a, b, (ulong)random.NextInt64(), (ulong)random.NextInt64()); + } + for (int trial = 0; trial < 512; trial++) + { + int n = trial % 65; + int source = 2, rhs = n + 4; + int dest = (trial % 4) switch { 0 => 2 * n + 6, 1 => 2, 2 => 3, _ => 1 }; + ulong[] data = new ulong[3 * n + 8]; + random.NextBytes(MemoryMarshal.AsBytes(data.AsSpan())); + if (trial % 7 == 0) Array.Clear(data, source, n); + ulong[] expected = (ulong[])data.Clone(); + ulong borrow = 0; + for (int i = 0; i < n; i++) + { + BigInteger d = (BigInteger)expected[source + i] - expected[rhs + i] - borrow; + expected[dest + i] = (ulong)(d & ulong.MaxValue); + borrow = d.Sign < 0 ? 1UL : 0; + } + ulong[] multiply = (ulong[])data.Clone(); + ulong[] expectedMultiply = (ulong[])data.Clone(); + ulong multiplier = trial % 3 == 0 ? ulong.MaxValue : data[0]; + ulong highCarry = 0; + for (int i = 0; i < n; i++) + { + BigInteger product = (BigInteger)expectedMultiply[source + i] * multiplier + highCarry; + BigInteger difference = (BigInteger)expectedMultiply[dest + i] - (product & ulong.MaxValue); + expectedMultiply[dest + i] = (ulong)(difference & ulong.MaxValue); + highCarry = (ulong)(product >> 64) + (difference.Sign < 0 ? 1UL : 0); + } + ulong actualHigh = SubMul(multiply.AsSpan(dest, n), multiply.AsSpan(source, n), multiplier); + if (actualHigh != highCarry || !multiply.AsSpan().SequenceEqual(expectedMultiply)) throw new Exception("SubMul"); + + ulong actual = LimbLoop(data.AsSpan(dest, n), data.AsSpan(source, n), data.AsSpan(rhs, n)); + if (actual != borrow || !data.AsSpan().SequenceEqual(expected)) throw new Exception("LimbLoop"); + + ulong[] tail = new ulong[n + 3], tailExpected = new ulong[n + 3]; + random.NextBytes(MemoryMarshal.AsBytes(tail.AsSpan())); + if (trial % 3 == 0) Array.Clear(tail); + borrow = 0; + for (int i = 0; i < tail.Length; i++) + { + BigInteger d = (BigInteger)tail[i] - (i < n ? data[rhs + i] : 0UL) - borrow; + tailExpected[i] = (ulong)(d & ulong.MaxValue); + borrow = d.Sign < 0 ? 1UL : 0; + } + actual = LimbLoopWithTail(tail, data.AsSpan(rhs, n)); + if (actual != borrow || !tail.AsSpan().SequenceEqual(tailExpected)) throw new Exception("LimbLoopWithTail"); + + uint[] left32 = new uint[n], right32 = new uint[n], result32 = new uint[n]; + random.NextBytes(MemoryMarshal.AsBytes(left32.AsSpan())); + random.NextBytes(MemoryMarshal.AsBytes(right32.AsSpan())); + uint actual32 = LimbLoop32(result32, left32, right32); + long borrow32 = 0; + for (int i = 0; i < n; i++) + { + long d = (long)left32[i] - right32[i] - borrow32; + if (result32[i] != unchecked((uint)d)) throw new Exception("LimbLoop32 value"); + borrow32 = d < 0 ? 1 : 0; + } + if (actual32 != borrow32) throw new Exception("LimbLoop32 borrow"); + + ulong[] volatileResult = new ulong[n], volatileExpected = new ulong[n]; + borrow = 0; + for (int i = 0; i < n; i++) + { + BigInteger d = -(BigInteger)data[rhs + i] - borrow; + volatileExpected[i] = (ulong)(d & ulong.MaxValue); + borrow = d.Sign < 0 ? 1UL : 0; + } + actual = VolatileLoop(volatileResult, data.AsSpan(rhs, n)); + if (actual != borrow || !volatileResult.AsSpan().SequenceEqual(volatileExpected)) throw new Exception("VolatileLoop"); + } + } + + private static void Check(ulong a, ulong b, ulong c, ulong d) + { + if (CompareRight(a, b) != (unchecked(a - b) > b)) throw new Exception("CompareRight"); + try + { + ulong checkedResult = CheckedSub(a, b); + if (a < b || checkedResult != a - b) throw new Exception("CheckedSub"); + } + catch (OverflowException) + { + if (a >= b) throw; + } + ulong bit = a < b ? 1UL : 0; + ulong high = Borrow(a, b, c, out ulong diff); + if (diff != unchecked(a - b) || high != unchecked(c + bit)) throw new Exception("Borrow"); + high = BorrowOnly(a, b, c, out diff); + if (diff != unchecked(a - b) || high != unchecked(c - bit)) throw new Exception("BorrowOnly"); + UInt128 x = ((UInt128)c << 64) | a, y = ((UInt128)d << 64) | b; + BigInteger expected = ((BigInteger)x - (BigInteger)y) & ((BigInteger.One << 128) - 1); + if ((BigInteger)Sub128(x, y) != expected) throw new Exception("Sub128"); + if ((BigInteger)unchecked((UInt128)SubSigned128((Int128)x, (Int128)y)) != expected) throw new Exception("SubSigned128"); + if (Branch(a, b, out diff) != (bit != 0 ? 13UL : 29UL) || diff != unchecked(a - b)) throw new Exception("Branch"); + if (Select(a, b, c, d, out diff) != (bit != 0 ? c : d) || diff != unchecked(a - b)) throw new Exception("Select"); + ulong result = ArbitraryBorrow(a, b, c, out ulong next); + BigInteger wide = (BigInteger)a - b - c; + ulong expectedBorrow = (ulong)(-(wide >> 64)); + if (result != (ulong)(wide & ulong.MaxValue) || next != expectedBorrow) throw new Exception("ArbitraryBorrow"); + if (AcrossCall(a, b, c) != unchecked(~c + bit)) throw new Exception("AcrossCall"); + } +} diff --git a/src/tests/JIT/opt/Add/Borrow.csproj b/src/tests/JIT/opt/Add/Borrow.csproj new file mode 100644 index 00000000000000..ba455bd85b0c01 --- /dev/null +++ b/src/tests/JIT/opt/Add/Borrow.csproj @@ -0,0 +1,14 @@ + + + true + None + True + + + + true + + + + + diff --git a/src/tests/JIT/opt/Add/Carry.cs b/src/tests/JIT/opt/Add/Carry.cs new file mode 100644 index 00000000000000..691c19ffca7bf3 --- /dev/null +++ b/src/tests/JIT/opt/Add/Carry.cs @@ -0,0 +1,1099 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using Xunit; + +public static class CarryTests +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Count64(ulong a, ulong b, ulong count, out ulong sum) + { + // X64: add + // X64-NEXT: adc + // ARM64: adds + // ARM64-NEXT: adc + ulong s = a + b; + count += s < a ? 1UL : 0UL; + sum = s; + return count; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint Count32(uint a, uint b, uint count, out uint sum) + { + // X64: add + // X64-NEXT: adc + // ARM64: adds + // ARM64-NEXT: adc + uint s = a + b; + count += b > s ? 1U : 0U; + sum = s; + return count; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Increment(ulong a, ulong count, out ulong sum) + { + // X64-NOT: {{^ +}}inc + // X64: add + // X64-NEXT: adc + // X64-NOT: {{^ +}}inc + ulong s = a + 1; + count += s < a ? 1UL : 0UL; + sum = s; + return count; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong AddWithCarry(ulong a, ulong b, ulong carryIn, out ulong carryOut) + { + ulong t = a + b; + ulong s = t + carryIn; + carryOut = (t < a ? 1UL : 0UL) + (s < t ? 1UL : 0UL); + return s; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 Add128(ulong aLo, ulong aHi, ulong bLo, ulong bHi) + { + // X64: add + // X64: adc + // ARM64: adds + // ARM64: adc + ulong lo = aLo + bLo; + ulong hi = aHi + bHi + (lo < aLo ? 1UL : 0UL); + return ((UInt128)hi << 64) | lo; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 DirectAdd128(UInt128 a, UInt128 b) + { + // X64: add + // X64-NEXT: adc + // ARM64: adds + // ARM64-NEXT: adc + return a + b; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong AddMemoryAfterStore(ulong a, ulong b, ref ulong source, ref ulong destination, ulong replacement) + { + // X64: add + // X64-NEXT: setb + // ARM64: adds + // ARM64-NEXT: cset + ulong lo = a + b; + ulong carry = lo < a ? 1UL : 0UL; + destination = replacement; + return source + carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 MultiplyAdd(ulong a, ulong b, ulong existing, ulong carry) + { + // X64: add + // X64-NEXT: adc + // X64-NEXT: add + // X64-NEXT: adc + // ARM64: adds + // ARM64-NEXT: adc + // ARM64-NEXT: adds + // ARM64-NEXT: adc + return (UInt128)a * b + existing + carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong CountColumn(ulong a, ulong b, ulong c, ulong d, out ulong sum) + { + ulong s0 = a + b; + ulong count = s0 < a ? 1UL : 0UL; + ulong s1 = s0 + c; + count += s1 < s0 ? 1UL : 0UL; + ulong s2 = s1 + d; + count += s2 < s1 ? 1UL : 0UL; + sum = s2; + return count; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool Branch(ulong a, ulong b, out ulong sum) + { + // X64: add + // X64-NOT: cmp + // X64: j{{b|ae}} + // ARM64: adds + // ARM64-NOT: cmp + // ARM64: b{{lo|hs}} + ulong s = a + b; + if (s < a) + { + sum = s; + return true; + } + sum = s + 1; + return false; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Signed(long a, long b) + { + long sum = a + b; + return sum < a ? 1UL : 0UL; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Reload(ulong a, ulong b, ref ulong sum, ref ulong alias) + { + sum = a + b; + alias = 0; + return sum < a ? 1UL : 0UL; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Checked(ulong a, ulong b) + { + ulong sum = checked(a + b); + return sum < a ? 1UL : 0UL; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 TwoConsumers(ulong a, ulong b, ulong count) + { + // X64: add + // X64-NEXT: setb + // ARM64: adds + // ARM64-NEXT: cset + ulong sum = a + b; + ulong carry = sum < a ? 1UL : 0UL; + return ((UInt128)unchecked(count + carry) << 64) | carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong AcrossCall(ulong a, ulong b, ulong count) + { + ulong sum = a + b; + Mutate(ref count); + return count + (sum < a ? 1UL : 0UL); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void Mutate(ref ulong value) => value = ~value; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static nuint AddLimbWithCarry(nuint a, nuint b, nuint carryIn, out nuint carryOut) + { + if (nint.Size == 8) + { + nuint sum1 = a + b; + nuint c1 = (sum1 < a) ? 1 : (nuint)0; + nuint sum2 = sum1 + carryIn; + nuint c2 = (sum2 < sum1) ? 1 : (nuint)0; + carryOut = c1 + c2; + return sum2; + } + else + { + ulong sum = (ulong)a + b + carryIn; + carryOut = (uint)(sum >> 32); + return (uint)sum; + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void AddLimbLoop(ReadOnlySpan left, ReadOnlySpan right, Span bits) + { + // X64: adc {{.*}}, qword ptr + // X64-NEXT: mov + // X64-NEXT: lea + // X64-NEXT: dec + // X64-NEXT: jne + // ARM64: ldr + // ARM64-NEXT: ldr + // ARM64-NEXT: adcs + // ARM64-NEXT: str + // ARM64-NEXT: add + // ARM64-NEXT: sub + // ARM64-NEXT: cbnz + // Establish cross-span length relationships so the JIT can + // elide bounds checks for left[i] and bits[i] in the loop. + _ = left[right.Length - 1]; + _ = bits[right.Length]; + + nuint carry = 0; + + for (int i = 0; i < right.Length; i++) + { + bits[i] = AddLimbWithCarry(left[i], right[i], carry, out carry); + } + + bits[right.Length] = carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void AddLimbLoopWithCarry(ReadOnlySpan left, ReadOnlySpan right, Span bits, nuint initialCarry) + { + // Establish cross-span length relationships so the JIT can + // elide bounds checks for left[i] and bits[i] in the loop. + _ = left[right.Length - 1]; + _ = bits[right.Length]; + + nuint carry = initialCarry; + + for (int i = 0; i < right.Length; i++) + { + bits[i] = AddLimbWithCarry(left[i], right[i], carry, out carry); + } + + bits[right.Length] = carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void AddLimbLoopDiscardCarry(ReadOnlySpan left, ReadOnlySpan right, Span bits) + { + // Establish cross-span length relationships so the JIT can + // elide bounds checks for left[i] and bits[i] in the loop. + _ = left[right.Length - 1]; + _ = bits[right.Length]; + + nuint carry = 0; + + for (int i = 0; i < right.Length; i++) + { + bits[i] = AddLimbWithCarry(left[i], right[i], carry, out carry); + } + + } + + private static int s_loopSignal; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void AddLimbLoopVolatile(ReadOnlySpan left, ReadOnlySpan right, Span bits) + { + // Establish cross-span length relationships so the JIT can + // elide bounds checks for left[i] and bits[i] in the loop. + _ = left[right.Length - 1]; + _ = bits[right.Length]; + + nuint carry = 0; + + for (int i = 0; i < right.Length; i++) + { + bits[i] = AddLimbWithCarry(left[i], right[i], carry, out carry); + System.Threading.Volatile.Write(ref s_loopSignal, i); + } + + bits[right.Length] = carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void AddLimbLoopOne(ReadOnlySpan left, ReadOnlySpan right, Span bits) + { + // Establish cross-span length relationships so the JIT can + // elide bounds checks for left[i] and bits[i] in the loop. + _ = left[right.Length - 1]; + _ = bits[right.Length]; + + nuint carry = 1; + + for (int i = 0; i < right.Length; i++) + { + bits[i] = AddLimbWithCarry(left[i], right[i], carry, out carry); + } + + bits[right.Length] = carry; + } + private static BigInteger LimbsToBigInteger(ReadOnlySpan limbs) + { + BigInteger value = 0; + for (int i = limbs.Length - 1; i >= 0; i--) + value = (value << (IntPtr.Size * 8)) | (ulong)limbs[i]; + return value; + } + + private static void CheckLimbLoops() + { + Random random = new Random(125799); + byte[] bytes = new byte[8]; + for (int test = 0; test < 1000; test++) + { + int length = 1 + test % 129; + nuint[] left = new nuint[length + 1]; + nuint[] right = new nuint[length]; + nuint[] result = new nuint[length + 1]; + for (int i = 0; i < length; i++) + { + random.NextBytes(bytes); + left[i] = test % 4 == 0 ? nuint.MaxValue : (nuint)BitConverter.ToUInt64(bytes); + random.NextBytes(bytes); + right[i] = test % 4 == 0 ? nuint.MaxValue : (nuint)BitConverter.ToUInt64(bytes); + } + BigInteger expected = LimbsToBigInteger(left.AsSpan(0, length)) + LimbsToBigInteger(right); + AddLimbLoop(left.AsSpan(0, length), right, result); + if (LimbsToBigInteger(result) != expected) throw new Exception("AddLimbLoop"); + AddLimbLoopOne(left.AsSpan(0, length), right, result); + if (LimbsToBigInteger(result) != expected + 1) throw new Exception("AddLimbLoopOne"); + AddLimbLoopVolatile(left.AsSpan(0, length), right, result); + if (LimbsToBigInteger(result) != expected || s_loopSignal != length - 1) throw new Exception("AddLimbLoopVolatile"); + AddLimbLoopDiscardCarry(left.AsSpan(0, length), right, result); + if (LimbsToBigInteger(result.AsSpan(0, length)) != (expected & ((BigInteger.One << (length * IntPtr.Size * 8)) - 1))) + throw new Exception("AddLimbLoopDiscardCarry"); + foreach (nuint carry in new nuint[] { 0, 1, 2, nuint.MaxValue }) + { + AddLimbLoopWithCarry(left.AsSpan(0, length), right, result, carry); + if (LimbsToBigInteger(result) != expected + (ulong)carry) throw new Exception("AddLimbLoopWithCarry"); + } + AddLimbLoop(left.AsSpan(0, length), right, left); + if (LimbsToBigInteger(left) != expected) throw new Exception("AddLimbLoop alias"); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong SelectCarry(ulong a, ulong b, ulong x, ulong y, out ulong sum) + { + // X64-NOT: {{^ +}}set + // X64: cmov + // ARM64-NOT: {{^ +}}cset + // ARM64: cs{{el|inc}} + ulong s = unchecked(a + b); + sum = s; + return s >= a ? x + 1 : y; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void AddLimbLoop32(ReadOnlySpan left, ReadOnlySpan right, Span bits) + { + // X64: adc {{.*}}, dword ptr + // X64-NEXT: mov + // X64-NEXT: lea + // X64-NEXT: dec + // X64-NEXT: jne + // ARM64: adcs w + // ARM64-NEXT: str w + // ARM64-NEXT: add + // ARM64-NEXT: sub + // ARM64-NEXT: cbnz + _ = left[right.Length - 1]; + _ = bits[right.Length]; + uint carry = 0; + for (int i = 0; i < right.Length; i++) + { + uint a = left[i]; + uint sum1 = unchecked(a + right[i]); + uint c1 = sum1 < a ? 1U : 0U; + uint sum2 = unchecked(sum1 + carry); + uint c2 = sum2 < sum1 ? 1U : 0U; + carry = c1 + c2; + bits[i] = sum2; + } + bits[right.Length] = carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void AddLimbLoop32WithIndex(ReadOnlySpan left, ReadOnlySpan right, Span bits, Span indices) + { + // X64: adc {{.*}}, dword ptr + // X64: lea e + // ARM64: adcs w + // ARM64: add w + _ = left[right.Length - 1]; + _ = bits[right.Length]; + _ = indices[right.Length - 1]; + uint index = 0; + uint carry = 0; + for (int i = 0; i < right.Length; i++) + { + uint a = left[i]; + uint sum1 = unchecked(a + right[i]); + uint c1 = sum1 < a ? 1U : 0U; + uint sum2 = unchecked(sum1 + carry); + uint c2 = sum2 < sum1 ? 1U : 0U; + carry = c1 + c2; + bits[i] = sum2; + index = unchecked(index + 7); + indices[i] = index; + } + bits[right.Length] = carry; + } + + private static BigInteger Limbs32ToBigInteger(ReadOnlySpan limbs) + { + BigInteger value = 0; + for (int i = limbs.Length - 1; i >= 0; i--) + { + value = (value << 32) | limbs[i]; + } + return value; + } + + private static void CheckLimbLoops32() + { + Random random = new Random(125799); + for (int trial = 0; trial < 1000; trial++) + { + int length = 1 + trial % 129; + uint[] left = new uint[length + 1]; + uint[] right = new uint[length + 1]; + uint[] result = new uint[length + 1]; + uint[] indices = new uint[length]; + for (int i = 0; i < length; i++) + { + left[i] = trial % 4 == 0 ? uint.MaxValue : (uint)random.NextInt64(1L << 32); + right[i] = trial % 4 == 1 ? uint.MaxValue : (uint)random.NextInt64(1L << 32); + } + BigInteger expected = Limbs32ToBigInteger(left) + Limbs32ToBigInteger(right); + AddLimbLoop32WithIndex(left.AsSpan(0, length), right.AsSpan(0, length), result, indices); + if (Limbs32ToBigInteger(result) != expected) throw new Exception("AddLimbLoop32WithIndex"); + for (int i = 0; i < length; i++) + { + if (indices[i] != (uint)(7 * (i + 1))) throw new Exception("AddLimbLoop32WithIndex index"); + } + AddLimbLoop32(left.AsSpan(0, length), right.AsSpan(0, length), result); + if (Limbs32ToBigInteger(result) != expected) throw new Exception("AddLimbLoop32"); + AddLimbLoop32(left.AsSpan(0, length), right.AsSpan(0, length), left); + if (Limbs32ToBigInteger(left) != expected) throw new Exception("AddLimbLoop32 alias left"); + left = (uint[])result.Clone(); + expected = Limbs32ToBigInteger(left.AsSpan(0, length)) + Limbs32ToBigInteger(right); + AddLimbLoop32(left.AsSpan(0, length), right.AsSpan(0, length), right); + if (Limbs32ToBigInteger(right) != expected) throw new Exception("AddLimbLoop32 alias right"); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool NoCarry(ulong a, ulong b) + { + // X64: add + // X64-NEXT: setae + // ARM64: adds + // ARM64-NEXT: cset {{.*}}, lo + return a <= unchecked(a + b); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int ZeroOr(int x, int y) + { + // X64: {{^ +}}or + // X64-NOT: {{^ +}}set + // X64-NOT: test + // X64: cmov + return (x | y) == 0 ? 1 : 2; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int ZeroAdd(int x, int y) + { + // X64: add + // X64-NOT: {{^ +}}set + // X64-NOT: test + // X64: cmov + return unchecked(x + y) == 0 ? 1 : 2; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int ZeroShift(int x) + { + // X64: shl + // X64-NOT: {{^ +}}set + // X64-NOT: test + // X64: cmov + return (x << 3) == 0 ? 1 : 2; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static UInt128 WideningProduct(ulong a, ulong b) + { + if (System.Runtime.Intrinsics.Arm.ArmBase.Arm64.IsSupported) + { + ulong high = System.Runtime.Intrinsics.Arm.ArmBase.Arm64.MultiplyHigh(a, b); + return ((UInt128)high << 64) | unchecked(a * b); + } + return Math.BigMul(a, b); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static nuint UnrolledMultiplyAdd(Span result, ReadOnlySpan left, nuint multiplier) + { + // X64-NOT: {{^ +}}setb + // X64: mul{{x| }} + // X64-NOT: {{^ +}}setb + // X64: adc + // X64-NOT: {{^ +}}setb + // X64: ret + // ARM64-NOT: {{^ +}}cset + // ARM64: umulh + // ARM64-NOT: {{^ +}}cset + // ARM64: adc + // ARM64-NOT: {{^ +}}cset + // ARM64: ret + + int length = left.Length; + result = result.Slice(0, length); + int i = 0; + nuint carry = 0; + + if (nint.Size == 8) + { + // Unroll by 4: mulx has 3-5 cycle latency but 1 cycle throughput, + // so issuing 4 multiplies allows the CPU to pipeline them while + // carry chains complete sequentially behind. + for (; i < length - 3; i += 4) + { + UInt128 p0 = WideningProduct((ulong)left[i], (ulong)multiplier) + (ulong)result[i] + (ulong)carry; + result[i] = (nuint)(ulong)p0; + + UInt128 p1 = WideningProduct((ulong)left[i + 1], (ulong)multiplier) + (ulong)result[i + 1] + (ulong)(p0 >> 64); + result[i + 1] = (nuint)(ulong)p1; + + UInt128 p2 = WideningProduct((ulong)left[i + 2], (ulong)multiplier) + (ulong)result[i + 2] + (ulong)(p1 >> 64); + result[i + 2] = (nuint)(ulong)p2; + + UInt128 p3 = WideningProduct((ulong)left[i + 3], (ulong)multiplier) + (ulong)result[i + 3] + (ulong)(p2 >> 64); + result[i + 3] = (nuint)(ulong)p3; + + carry = (nuint)(ulong)(p3 >> 64); + } + + for (; i < length; i++) + { + UInt128 product = WideningProduct((ulong)left[i], (ulong)multiplier) + (ulong)result[i] + (ulong)carry; + result[i] = (nuint)(ulong)product; + carry = (nuint)(ulong)(product >> 64); + } + } + else + { + for (; i < length; i++) + { + ulong product = (ulong)left[i] * multiplier + + result[i] + carry; + result[i] = (uint)product; + carry = (uint)(product >> 32); + } + } + + return carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static long SignedWideningProduct(long a, long b, out long low) => Math.BigMul(a, b, out low); + + private static void CheckUnrolledMultiplyAdd() + { + Random random = new Random(80674); + for (int trial = 0; trial < 1000; trial++) + { + int length = 1 + trial % 65; + nuint[] left = new nuint[length]; + nuint[] result = new nuint[length]; + for (int i = 0; i < length; i++) + { + left[i] = trial % 4 == 0 ? nuint.MaxValue : (nuint)random.NextInt64(); + result[i] = trial % 4 == 0 ? nuint.MaxValue : (nuint)random.NextInt64(); + } + nuint multiplier = trial % 4 == 0 ? nuint.MaxValue : (nuint)random.NextInt64(); + BigInteger expected = LimbsToBigInteger(left) * (ulong)multiplier + LimbsToBigInteger(result); + nuint carry = UnrolledMultiplyAdd(result, left, multiplier); + if (LimbsToBigInteger(result) + ((BigInteger)(ulong)carry << (length * IntPtr.Size * 8)) != expected) + throw new Exception("UnrolledMultiplyAdd"); + expected = LimbsToBigInteger(left) * ((BigInteger)(ulong)multiplier + 1); + carry = UnrolledMultiplyAdd(left, left, multiplier); + if (LimbsToBigInteger(left) + ((BigInteger)(ulong)carry << (length * IntPtr.Size * 8)) != expected) + throw new Exception("UnrolledMultiplyAdd alias"); + long a = unchecked((long)(ulong)multiplier); + long b = trial % 2 == 0 ? long.MinValue : random.NextInt64(); + long high = SignedWideningProduct(a, b, out long low); + if (((BigInteger)high << 64) + (ulong)low != (BigInteger)a * b) + throw new Exception("SignedWideningProduct"); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ScalarMultiplyAdd(Span result, ReadOnlySpan left, ulong multiplier, ulong carry) + { + // X64: mul{{x| }} + // X64: ad{{cx|d}} + // X64: ad{{ox|c}} + // ARM64: umulh + // ARM64: adc + for (int i = 0; i < left.Length; i++) + { + UInt128 product = WideningProduct(left[i], multiplier) + result[i] + carry; + result[i] = (ulong)product; + carry = (ulong)(product >> 64); + } + return carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong VolatileMultiplyAdd(Span result, ReadOnlySpan left, ulong multiplier, ulong carry) + { + // X64-NOT: adcx + // X64-NOT: adox + // X64: ret + // Volatile loads must not become part of an ADX loop. + for (int i = 0; i < left.Length; i++) + { + UInt128 product = WideningProduct(left[i], multiplier) + System.Threading.Volatile.Read(ref result[i]) + carry; + result[i] = (ulong)product; + carry = (ulong)(product >> 64); + } + return carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong GuardedUnrolledMultiplyAdd(Span result, ReadOnlySpan left, ulong multiplier, ulong carry) + { + // X64: mul{{x| }} + // X64: ad{{cx|d}} + // X64: ad{{ox|c}} + // ARM64: umulh + // ARM64: adcs + // ARM64-NEXT: adc + // ARM64-NEXT: adds + // ARM64: adcs + // ARM64-NEXT: adc + // ARM64-NEXT: adds + // ARM64: adcs + // ARM64-NEXT: adc + // ARM64-NEXT: adds + // ARM64: adcs + // ARM64-NEXT: adc + // ARM64-NEXT: adds + // ARM64: cbnz + // ARM64: adc + int length = left.Length; + result = result.Slice(0, length); + int i = 0; + for (; i < length - 3; i += 4) + { + UInt128 p0 = WideningProduct(left[i], multiplier) + result[i] + carry; + result[i] = (ulong)p0; + UInt128 p1 = WideningProduct(left[i + 1], multiplier) + result[i + 1] + (ulong)(p0 >> 64); + result[i + 1] = (ulong)p1; + UInt128 p2 = WideningProduct(left[i + 2], multiplier) + result[i + 2] + (ulong)(p1 >> 64); + result[i + 2] = (ulong)p2; + UInt128 p3 = WideningProduct(left[i + 3], multiplier) + result[i + 3] + (ulong)(p2 >> 64); + result[i + 3] = (ulong)p3; + carry = (ulong)(p3 >> 64); + } + for (; i < left.Length; i++) + { + UInt128 product = WideningProduct(left[i], multiplier) + result[i] + carry; + result[i] = (ulong)product; + carry = (ulong)(product >> 64); + } + return carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ObservableCarryUnrolledMultiplyAdd(Span result, ReadOnlySpan left, ulong multiplier, ulong carry, out ulong observed) + { + // The intermediate high word is observable and must include both carries. + // ARM64-NOT: adcs + // ARM64: ret + observed = 0; + int length = left.Length; + result = result.Slice(0, length); + int i = 0; + for (; i < length - 3; i += 4) + { + UInt128 p0 = WideningProduct(left[i], multiplier) + result[i] + carry; + result[i] = (ulong)p0; + observed = (ulong)(p0 >> 64); + UInt128 p1 = WideningProduct(left[i + 1], multiplier) + result[i + 1] + (ulong)(p0 >> 64); + result[i + 1] = (ulong)p1; + UInt128 p2 = WideningProduct(left[i + 2], multiplier) + result[i + 2] + (ulong)(p1 >> 64); + result[i + 2] = (ulong)p2; + UInt128 p3 = WideningProduct(left[i + 3], multiplier) + result[i + 3] + (ulong)(p2 >> 64); + result[i + 3] = (ulong)p3; + carry = (ulong)(p3 >> 64); + } + for (; i < left.Length; i++) + { + UInt128 product = WideningProduct(left[i], multiplier) + result[i] + carry; + result[i] = (ulong)product; + carry = (ulong)(product >> 64); + } + return carry; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong InterveningStoreUnrolledMultiplyAdd(Span result, ReadOnlySpan left, ulong multiplier, ulong carry) + { + // X64: mul{{x| }} + // X64: ad{{cx|d}} + // X64: ad{{ox|c}} + // ARM64: umulh + // ARM64: adc + int length = left.Length; + result = result.Slice(0, length); + int i = 0; + for (; i < length - 3; i += 4) + { + // The saved load must remain before this potentially aliasing store. + ulong saved = result[i]; + result[i + 1] = multiplier; + UInt128 p0 = WideningProduct(left[i], multiplier) + saved + carry; + result[i] = (ulong)p0; + UInt128 p1 = WideningProduct(left[i + 1], multiplier) + result[i + 1] + (ulong)(p0 >> 64); + result[i + 1] = (ulong)p1; + UInt128 p2 = WideningProduct(left[i + 2], multiplier) + result[i + 2] + (ulong)(p1 >> 64); + result[i + 2] = (ulong)p2; + UInt128 p3 = WideningProduct(left[i + 3], multiplier) + result[i + 3] + (ulong)(p2 >> 64); + result[i + 3] = (ulong)p3; + carry = (ulong)(p3 >> 64); + } + for (; i < left.Length; i++) + { + UInt128 product = WideningProduct(left[i], multiplier) + result[i] + carry; + result[i] = (ulong)product; + carry = (ulong)(product >> 64); + } + return carry; + } + + private static void CheckObservableCarry() + { + Random random = new Random(125802); + for (int trial = 0; trial < 256; trial++) + { + int length = trial % 66; + ulong[] left = new ulong[length]; + ulong[] result = new ulong[length]; + random.NextBytes(System.Runtime.InteropServices.MemoryMarshal.AsBytes(left.AsSpan())); + random.NextBytes(System.Runtime.InteropServices.MemoryMarshal.AsBytes(result.AsSpan())); + ulong multiplier = trial % 3 == 0 ? ulong.MaxValue : (ulong)random.NextInt64(); + ulong carry = trial % 2 == 0 ? ulong.MaxValue : 0; + ulong[] expected = (ulong[])result.Clone(); + ulong expectedCarry = carry; + ulong expectedObserved = 0; + for (int i = 0; i < length; i++) + { + BigInteger product = (BigInteger)left[i] * multiplier + expected[i] + expectedCarry; + expected[i] = (ulong)(product & ulong.MaxValue); + expectedCarry = (ulong)(product >> 64); + if ((i & 3) == 0 && i < length - 3) + { + expectedObserved = expectedCarry; + } + } + ulong actualCarry = ObservableCarryUnrolledMultiplyAdd(result, left, multiplier, carry, out ulong observed); + if (actualCarry != expectedCarry || observed != expectedObserved || !result.AsSpan().SequenceEqual(expected)) + { + throw new Exception("ObservableCarryUnrolledMultiplyAdd"); + } + } + } + private static void CheckInterveningStore() + { + Random random = new Random(125801); + for (int trial = 0; trial < 256; trial++) + { + int length = trial % 66; + int source = 2; + int destination = (trial % 4) switch { 0 => length + 4, 1 => 2, 2 => 3, _ => 1 }; + ulong[] data = new ulong[length * 2 + 8]; + random.NextBytes(System.Runtime.InteropServices.MemoryMarshal.AsBytes(data.AsSpan())); + ulong multiplier = trial % 3 == 0 ? ulong.MaxValue : data[0]; + ulong carry = trial % 2 == 0 ? ulong.MaxValue : 0; + ulong[] expected = (ulong[])data.Clone(); + ulong expectedCarry = carry; + for (int i = 0; i < length; i++) + { + ulong saved = expected[destination + i]; + if (((i & 3) == 0) && (i < length - 3)) + { + expected[destination + i + 1] = multiplier; + } + BigInteger product = (BigInteger)expected[source + i] * multiplier + saved + expectedCarry; + expected[destination + i] = (ulong)(product & ulong.MaxValue); + expectedCarry = (ulong)(product >> 64); + } + ulong actualCarry = InterveningStoreUnrolledMultiplyAdd(data.AsSpan(destination, length), + data.AsSpan(source, length), multiplier, carry); + if (actualCarry != expectedCarry || !data.AsSpan().SequenceEqual(expected)) + { + throw new Exception("InterveningStoreUnrolledMultiplyAdd"); + } + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ChangingMultiplierUnrolledMultiplyAdd(Span result, ReadOnlySpan left, ulong multiplier, ulong carry) + { + // The multiplier changes on the backedge and must not be hoisted. + // X64: mul{{x| }} + // X64: ad{{cx|d}} + // X64: ad{{ox|c}} + // ARM64: umulh + // ARM64: adc + int length = left.Length; + result = result.Slice(0, length); + int i = 0; + for (; i < length - 3; i += 4) + { + UInt128 p0 = WideningProduct(left[i], multiplier) + result[i] + carry; + result[i] = (ulong)p0; + UInt128 p1 = WideningProduct(left[i + 1], multiplier) + result[i + 1] + (ulong)(p0 >> 64); + result[i + 1] = (ulong)p1; + UInt128 p2 = WideningProduct(left[i + 2], multiplier) + result[i + 2] + (ulong)(p1 >> 64); + result[i + 2] = (ulong)p2; + UInt128 p3 = WideningProduct(left[i + 3], multiplier) + result[i + 3] + (ulong)(p2 >> 64); + result[i + 3] = (ulong)p3; + carry = (ulong)(p3 >> 64); + multiplier = unchecked(multiplier + 1); + } + for (; i < left.Length; i++) + { + UInt128 product = WideningProduct(left[i], multiplier) + result[i] + carry; + result[i] = (ulong)product; + carry = (ulong)(product >> 64); + } + return carry; + } + + private static void CheckChangingMultiplier() + { + Random random = new Random(125800); + for (int trial = 0; trial < 256; trial++) + { + int length = trial % 66; + ulong[] left = new ulong[length]; + ulong[] result = new ulong[length]; + random.NextBytes(System.Runtime.InteropServices.MemoryMarshal.AsBytes(left.AsSpan())); + random.NextBytes(System.Runtime.InteropServices.MemoryMarshal.AsBytes(result.AsSpan())); + ulong multiplier = trial % 3 == 0 ? ulong.MaxValue : (ulong)random.NextInt64(); + ulong carry = trial % 2 == 0 ? ulong.MaxValue : 0; + ulong[] expected = (ulong[])result.Clone(); + ulong expectedMultiplier = multiplier; + ulong expectedCarry = carry; + for (int i = 0; i < length; i++) + { + BigInteger product = (BigInteger)left[i] * expectedMultiplier + expected[i] + expectedCarry; + expected[i] = (ulong)(product & ulong.MaxValue); + expectedCarry = (ulong)(product >> 64); + if ((i & 3) == 3) + { + expectedMultiplier = unchecked(expectedMultiplier + 1); + } + } + ulong actualCarry = ChangingMultiplierUnrolledMultiplyAdd(result, left, multiplier, carry); + if (actualCarry != expectedCarry || !result.AsSpan().SequenceEqual(expected)) + { + throw new Exception("ChangingMultiplierUnrolledMultiplyAdd"); + } + } + } + + private static void CheckScalarMultiplyAdd() + { + Random random = new Random(125799); + BigInteger mask = (BigInteger.One << 64) - 1; + for (int trial = 0; trial < 12000; trial++) + { + int length = trial % 129; + int source = 2; + int destination = (trial % 4) switch { 0 => length + 4, 1 => 2, 2 => 3, _ => 1 }; + ulong[] input = new ulong[length * 2 + 8]; + random.NextBytes(System.Runtime.InteropServices.MemoryMarshal.AsBytes(input.AsSpan())); + if (trial % 13 == 0) + Array.Fill(input, ulong.MaxValue); + if (trial % 17 == 0) + Array.Fill(input, 0UL); + ulong multiplier = trial % 7 == 0 ? ulong.MaxValue : input[0]; + ulong initialCarry = trial % 11 == 0 ? ulong.MaxValue : input[^1]; + ulong[] expected = (ulong[])input.Clone(); + BigInteger expectedCarry = initialCarry; + // The sequential oracle also covers overlap in either direction. + for (int i = 0; i < length; i++) + { + BigInteger product = (BigInteger)expected[source + i] * multiplier + expected[destination + i] + expectedCarry; + expected[destination + i] = (ulong)(product & mask); + expectedCarry = product >> 64; + } + for (int variant = 0; variant < 3; variant++) + { + ulong[] actual = (ulong[])input.Clone(); + ulong carry = variant switch + { + 0 => ScalarMultiplyAdd(actual.AsSpan(destination, length), actual.AsSpan(source, length), multiplier, initialCarry), + 1 => VolatileMultiplyAdd(actual.AsSpan(destination, length), actual.AsSpan(source, length), multiplier, initialCarry), + _ => GuardedUnrolledMultiplyAdd(actual.AsSpan(destination, length), actual.AsSpan(source, length), multiplier, initialCarry) + }; + if (carry != (ulong)expectedCarry || !actual.AsSpan().SequenceEqual(expected)) + throw new Exception($"ScalarMultiplyAdd trial {trial}, variant {variant}"); + } + } + + // A short destination must still throw after storing only the valid prefix. + for (int length = 0; length < 8; length++) + { + ulong[] left = new ulong[length + 1]; + ulong[] result = new ulong[length + 2]; + Array.Fill(left, ulong.MaxValue); + Array.Fill(result, ulong.MaxValue); + ulong[] expected = (ulong[])result.Clone(); + BigInteger carry = ulong.MaxValue; + for (int i = 0; i < length; i++) + { + BigInteger product = (BigInteger)left[i] * ulong.MaxValue + expected[i] + carry; + expected[i] = (ulong)(product & mask); + carry = product >> 64; + } + try + { + ScalarMultiplyAdd(result.AsSpan(0, length), left, ulong.MaxValue, ulong.MaxValue); + throw new Exception("Missing bounds exception"); + } + catch (IndexOutOfRangeException) + { + if (!result.AsSpan().SequenceEqual(expected)) + throw new Exception("MultiplyAdd exception store order"); + } + } + + // The upfront slice must reject a short destination before any stores. + for (int length = 0; length < 12; length++) + { + ulong[] left = new ulong[length + 4]; + ulong[] result = new ulong[length + 4]; + Array.Fill(left, ulong.MaxValue); + Array.Fill(result, ulong.MaxValue); + ulong[] expected = (ulong[])result.Clone(); + try + { + GuardedUnrolledMultiplyAdd(result.AsSpan(0, length), left, ulong.MaxValue, ulong.MaxValue); + throw new Exception("Missing guarded bounds exception"); + } + catch (ArgumentOutOfRangeException) + { + if (!result.AsSpan().SequenceEqual(expected)) + throw new Exception("Guarded multiply-add exception store order"); + } + } + } + + [Fact] + public static void TestEntryPoint() + { + CheckLimbLoops(); + CheckUnrolledMultiplyAdd(); + CheckChangingMultiplier(); + CheckInterveningStore(); + CheckObservableCarry(); + CheckScalarMultiplyAdd(); + CheckLimbLoops32(); + ulong[] values = { 0, 1, 2, uint.MaxValue, 1UL << 32, (1UL << 63) - 1, 1UL << 63, ulong.MaxValue - 1, ulong.MaxValue }; + foreach (ulong a in values) + { + foreach (ulong b in values) + { + foreach (ulong c in values) + { + Check(a, b, c, ~c); + } + } + } + + Random random = new Random(80674); + byte[] bytes = new byte[32]; + for (int i = 0; i < 10000; i++) + { + random.NextBytes(bytes); + Check(BitConverter.ToUInt64(bytes, 0), BitConverter.ToUInt64(bytes, 8), + BitConverter.ToUInt64(bytes, 16), BitConverter.ToUInt64(bytes, 24)); + } + + ulong shared = 42; + if (Reload(1, 2, ref shared, ref shared) != 1 || shared != 0) + { + throw new Exception("Aliased reload"); + } + try + { + Checked(ulong.MaxValue, 1); + throw new Exception("Missing overflow exception"); + } + catch (OverflowException) + { + } + } + + private static void Check(ulong a, ulong b, ulong c, ulong d) + { + int x = (int)a; + int y = (int)b; + if (ZeroOr(x, y) != ((x | y) == 0 ? 1 : 2) || + ZeroAdd(x, y) != (unchecked(x + y) == 0 ? 1 : 2) || + ZeroShift(x) != ((x << 3) == 0 ? 1 : 2)) + { + throw new Exception("Zero flags"); + } + UInt128 wide = (UInt128)a + b; + if (Count64(a, b, c, out ulong sum) != unchecked(c + (ulong)(wide >> 64)) || sum != (ulong)wide) + throw new Exception("Count64"); + + if (NoCarry(a, b) != (wide <= ulong.MaxValue)) + { + throw new Exception("NoCarry"); + } + ulong selected = SelectCarry(a, b, c, d, out ulong selectedSum); + if (selectedSum != (ulong)wide || selected != (wide <= ulong.MaxValue ? unchecked(c + 1) : d)) + { + throw new Exception("SelectCarry"); + } + ulong bit = (ulong)(wide >> 64); + if (TwoConsumers(a, b, c) != (((UInt128)unchecked(c + bit) << 64) | bit)) + throw new Exception("TwoConsumers"); + + if (AcrossCall(a, b, c) != unchecked(~c + bit)) + throw new Exception("AcrossCall"); + + ulong wide32 = (ulong)(uint)a + (uint)b; + if (Count32((uint)a, (uint)b, (uint)c, out uint sum32) != unchecked((uint)c + (uint)(wide32 >> 32)) || sum32 != (uint)wide32) + throw new Exception("Count32"); + + wide = (UInt128)a + 1; + if (Increment(a, c, out sum) != unchecked(c + (ulong)(wide >> 64)) || sum != (ulong)wide) + throw new Exception("Increment"); + + wide = (UInt128)a + b + c; + if (AddWithCarry(a, b, c, out ulong carry) != (ulong)wide || carry != (ulong)(wide >> 64)) + throw new Exception("AddWithCarry"); + + UInt128 left = ((UInt128)c << 64) | a; + UInt128 right = ((UInt128)d << 64) | b; + if (Add128(a, c, b, d) != unchecked(left + right)) + throw new Exception("Add128"); + + BigInteger sum128 = ((BigInteger)left + (BigInteger)right) & ((BigInteger.One << 128) - 1); + if ((BigInteger)DirectAdd128(left, right) != sum128) + throw new Exception("DirectAdd128"); + + ulong memory = c; + if (AddMemoryAfterStore(a, b, ref memory, ref memory, d) != unchecked(d + bit) || memory != d) + throw new Exception("AddMemoryAfterStore alias"); + ulong destination = 0; + if (AddMemoryAfterStore(a, b, ref memory, ref destination, c) != unchecked(d + bit) || destination != c) + throw new Exception("AddMemoryAfterStore distinct"); + + BigInteger product = (BigInteger)a * b + c + d; + if ((BigInteger)MultiplyAdd(a, b, c, d) != product) + throw new Exception("MultiplyAdd"); + + wide = (UInt128)a + b + c + d; + if (CountColumn(a, b, c, d, out sum) != (ulong)(wide >> 64) || sum != (ulong)wide) + throw new Exception("CountColumn"); + + wide = (UInt128)a + b; + bool overflow = wide > ulong.MaxValue; + if (Branch(a, b, out sum) != overflow || sum != unchecked((ulong)wide + (overflow ? 0UL : 1UL))) + throw new Exception("Branch"); + + long signedSum = unchecked((long)a + (long)b); + if (Signed((long)a, (long)b) != (signedSum < (long)a ? 1UL : 0UL)) + throw new Exception("Signed"); + } +} diff --git a/src/tests/JIT/opt/Add/Carry.csproj b/src/tests/JIT/opt/Add/Carry.csproj new file mode 100644 index 00000000000000..3c2866959f4fff --- /dev/null +++ b/src/tests/JIT/opt/Add/Carry.csproj @@ -0,0 +1,14 @@ + + + true + None + True + + + + true + + + + + diff --git a/src/tests/JIT/opt/Add/CarryChains.cs b/src/tests/JIT/opt/Add/CarryChains.cs new file mode 100644 index 00000000000000..620dc9263ef224 --- /dev/null +++ b/src/tests/JIT/opt/Add/CarryChains.cs @@ -0,0 +1,140 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using Xunit; + +public class CarryChainTests +{ + public struct Words + { + public ulong A, B, C, D; + public Words(ulong a, ulong b, ulong c, ulong d) { A = a; B = b; C = c; D = d; } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong Add(ulong a, ulong b, ulong carry, out ulong next) + { + ulong sum = a + b; + ulong c1 = sum < a ? 1UL : 0; + ulong result = sum + carry; + next = c1 + (result < sum ? 1UL : 0); + return result; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong Sub(ulong a, ulong b, ulong borrow, out ulong next) + { + ulong difference = a - b; + ulong b1 = difference > a ? 1UL : 0; + ulong result = difference - borrow; + next = b1 + (result > difference ? 1UL : 0); + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Words AddChain(Words a, Words b) + { + ulong w0 = Add(a.A, b.A, 0, out ulong carry); + ulong w1 = Add(a.B, b.B, carry, out carry); + ulong w2 = Add(a.C, b.C, carry, out carry); + ulong w3 = a.D + b.D + carry; + return new Words(w0, w1, w2, w3); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Words SubChain(Words a, Words b) + { + ulong w0 = Sub(a.A, b.A, 0, out ulong borrow); + ulong w1 = Sub(a.B, b.B, borrow, out borrow); + ulong w2 = Sub(a.C, b.C, borrow, out borrow); + ulong w3 = a.D - b.D - borrow; + return new Words(w0, w1, w2, w3); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Clobber(ulong value) => (value * 37) ^ (value >> 3); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Words Interrupted(Words a, Words b, out ulong observed) + { + ulong w0 = Add(a.A, b.A, 0, out ulong carry); + observed = Clobber(carry); + ulong w1 = Add(a.B, b.B, carry, out carry); + ulong w2 = Add(a.C, b.C, carry, out carry); + return new Words(w0, w1, w2, a.D + b.D + carry); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Words ExtraConsumer(Words a, Words b, out ulong observed) + { + ulong w0 = Sub(a.A, b.A, 0, out ulong borrow); + ulong saved = borrow; + ulong w1 = Sub(a.B, b.B, borrow, out borrow); + ulong w2 = Sub(a.C, b.C, borrow, out borrow); + observed = saved; + return new Words(w0, w1, w2, a.D - b.D - borrow); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Words FullLimbCarry(Words a, Words b, ulong carry) + { + ulong w0 = Add(a.A, b.A, carry, out carry); + ulong w1 = Add(a.B, b.B, carry, out carry); + ulong w2 = Add(a.C, b.C, carry, out carry); + return new Words(w0, w1, w2, a.D + b.D + carry); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Words TwoCarrySeed(Words a, Words b) + { + // Each comparison is a bit, but their sum need not be. This local + // must not acquire a one-bit proof merely because its inputs have one. + ulong carry = (a.A + b.A < a.A ? 1UL : 0) + (a.B + b.B < a.B ? 1UL : 0); + ulong w0 = Add(a.A, b.A, carry, out carry); + ulong w1 = Add(a.B, b.B, carry, out carry); + ulong w2 = Add(a.C, b.C, carry, out carry); + return new Words(w0, w1, w2, a.D + b.D + carry); + } + + private static BigInteger Value(Words value) => value.A | ((BigInteger)value.B << 64) | + ((BigInteger)value.C << 128) | ((BigInteger)value.D << 192); + + [Fact] + public static void TestEntryPoint() + { + Random random = new Random(80674); + byte[] bytes = new byte[64]; + BigInteger mask = (BigInteger.One << 256) - 1; + void Equal(Words actual, BigInteger expected) + { + if (Value(actual) != (expected & mask)) throw new Exception("Incorrect carry chain"); + } + for (int i = 0; i < 3000; i++) + { + random.NextBytes(bytes); + Words a = new Words(BitConverter.ToUInt64(bytes, 0), BitConverter.ToUInt64(bytes, 8), BitConverter.ToUInt64(bytes, 16), BitConverter.ToUInt64(bytes, 24)); + Words b = new Words(BitConverter.ToUInt64(bytes, 32), BitConverter.ToUInt64(bytes, 40), BitConverter.ToUInt64(bytes, 48), BitConverter.ToUInt64(bytes, 56)); + if (i < 4) + { + a = new Words(ulong.MaxValue, ulong.MaxValue, ulong.MaxValue, ulong.MaxValue); + b = i == 0 ? a : new Words(1, 0, 0, 0); + if (i == 2) a = default; + if (i == 3) b = default; + } + BigInteger x = Value(a), y = Value(b); + Equal(AddChain(a, b), x + y); + Equal(SubChain(a, b), x - y); + Equal(Interrupted(a, b, out ulong observed), x + y); + if (observed != Clobber(a.A + b.A < a.A ? 1UL : 0)) throw new Exception("Lost carry consumer"); + Equal(ExtraConsumer(a, b, out observed), x - y); + if (observed != (a.A < b.A ? 1UL : 0)) throw new Exception("Lost borrow consumer"); + Equal(FullLimbCarry(a, b, 2), x + y + 2); + Equal(FullLimbCarry(a, b, ulong.MaxValue), x + y + ulong.MaxValue); + ulong seed = (a.A + b.A < a.A ? 1UL : 0) + (a.B + b.B < a.B ? 1UL : 0); + Equal(TwoCarrySeed(a, b), x + y + seed); + } + } +} diff --git a/src/tests/JIT/opt/Add/CarryChains.csproj b/src/tests/JIT/opt/Add/CarryChains.csproj new file mode 100644 index 00000000000000..5914015efe533e --- /dev/null +++ b/src/tests/JIT/opt/Add/CarryChains.csproj @@ -0,0 +1,12 @@ + + + true + None + True + + + + + + + diff --git a/src/tests/JIT/opt/Add/DecimalMultiply.cs b/src/tests/JIT/opt/Add/DecimalMultiply.cs new file mode 100644 index 00000000000000..bd22abe3e49bf4 --- /dev/null +++ b/src/tests/JIT/opt/Add/DecimalMultiply.cs @@ -0,0 +1,119 @@ +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using Xunit; + +public class DecimalMultiplyTests +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static decimal Multiply(decimal a, decimal b) => a * b; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static decimal Square(decimal value) => value * value; + + private static readonly BigInteger MaxCoefficient = (BigInteger.One << 96) - 1; + + private static BigInteger Coefficient(int[] bits) => + ((BigInteger)(uint)bits[2] << 64) | ((BigInteger)(uint)bits[1] << 32) | (uint)bits[0]; + + private static decimal Expected(decimal a, decimal b) + { + int[] left = decimal.GetBits(a); + int[] right = decimal.GetBits(b); + BigInteger product = Coefficient(left) * Coefficient(right); + int scale = ((left[3] >> 16) & 0xFF) + ((right[3] >> 16) & 0xFF); + bool negative = ((left[3] ^ right[3]) & int.MinValue) != 0; + + // Round the exact product directly, avoiding double rounding as digits + // are removed to fit decimal's 96-bit coefficient and maximum scale. + for (int dropped = Math.Max(0, scale - 28); dropped <= scale; dropped++) + { + BigInteger divisor = BigInteger.Pow(10, dropped); + BigInteger coefficient = BigInteger.DivRem(product, divisor, out BigInteger remainder); + int comparison = (remainder * 2).CompareTo(divisor); + if (comparison > 0 || (comparison == 0 && !coefficient.IsEven)) + { + coefficient++; + } + if (coefficient <= MaxCoefficient) + { + return new decimal((int)(uint)(coefficient & uint.MaxValue), + (int)(uint)((coefficient >> 32) & uint.MaxValue), + (int)(uint)(coefficient >> 64), negative, (byte)(scale - dropped)); + } + } + throw new OverflowException(); + } + + private static void Check(decimal a, decimal b, bool square = false) + { + decimal expected = 0; + bool overflow = false; + try + { + expected = Expected(a, b); + } + catch (OverflowException) + { + overflow = true; + } + try + { + decimal actual = square ? Square(a) : Multiply(a, b); + if (overflow || actual != expected) + { + throw new Exception("Decimal multiplication mismatch"); + } + } + catch (OverflowException) + { + if (!overflow) + { + throw; + } + } + } + + [Fact] + public static void TestEntryPoint() + { + decimal[] edges = + { + 0m, 1m, -1m, 0.5m, uint.MaxValue, ulong.MaxValue, + decimal.MaxValue, decimal.MinValue, + new decimal(-1, -1, -1, false, 28), + new decimal(-1, -1, -1, true, 28), + new decimal(1, 0, 0, false, 28), + new decimal(3, 0, 0, false, 28), + new decimal(1, 0, 1, false, 14), + }; + foreach (decimal a in edges) + { + Check(a, a, square: true); + foreach (decimal b in edges) + { + Check(a, b); + } + } + + var random = new Random(80674); + byte[] bytes = new byte[24]; + for (int i = 0; i < 3000; i++) + { + random.NextBytes(bytes); + int leftMid = i % 4 == 0 ? 0 : BitConverter.ToInt32(bytes, 4); + int leftHigh = i % 3 == 0 || i % 4 == 0 ? 0 : BitConverter.ToInt32(bytes, 8); + int rightMid = i % 5 == 0 ? 0 : BitConverter.ToInt32(bytes, 16); + int rightHigh = i % 3 == 0 || i % 5 == 0 ? 0 : BitConverter.ToInt32(bytes, 20); + decimal a = new decimal(BitConverter.ToInt32(bytes, 0), leftMid, leftHigh, + i % 2 == 0, (byte)random.Next(29)); + decimal b = new decimal(BitConverter.ToInt32(bytes, 12), rightMid, rightHigh, + i % 7 == 0, (byte)random.Next(29)); + Check(a, b); + if (i % 17 == 0) + { + Check(a, a, square: true); + } + } + } +} diff --git a/src/tests/JIT/opt/Add/DecimalMultiply.csproj b/src/tests/JIT/opt/Add/DecimalMultiply.csproj new file mode 100644 index 00000000000000..09bf8b9fc33972 --- /dev/null +++ b/src/tests/JIT/opt/Add/DecimalMultiply.csproj @@ -0,0 +1,12 @@ + + + true + None + True + + + + + + + diff --git a/src/tests/JIT/opt/Add/DecimalWide.cs b/src/tests/JIT/opt/Add/DecimalWide.cs new file mode 100644 index 00000000000000..d12ae92008d075 --- /dev/null +++ b/src/tests/JIT/opt/Add/DecimalWide.cs @@ -0,0 +1,90 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Reflection; +using Xunit; + +public class DecimalWideTests +{ + private static void Check(int width, int maximumPower) where T : IBinaryInteger + { + Type number = typeof(object).Assembly.GetType("System.Number", throwOnError: true)!; + MethodInfo Method(string name) => number.GetMethod(name, BindingFlags.Static | BindingFlags.NonPublic)!.MakeGenericMethod(typeof(T)); + MethodInfo multiply = Method("WideMultiply"); + MethodInfo add = Method("WideAdd"); + MethodInfo subtract = Method("WideSubtract"); + MethodInfo divide = Method("WideDivideByPow10"); + BigInteger mask = (BigInteger.One << width) - 1; + Random random = new Random(125799); + byte[] bytes = new byte[width / 4]; + + BigInteger Next() + { + random.NextBytes(bytes); + return new BigInteger(bytes, isUnsigned: true); + } + + T Limb(BigInteger value) => T.CreateChecked(value & mask); + + void Equal(object actual, BigInteger expected) + { + if (BigInteger.CreateChecked((T)actual) != expected) + { + throw new Exception($"Incorrect {typeof(T)} wide arithmetic result: expected {expected}, got {actual}"); + } + } + + for (int i = 0; i < 1000; i++) + { + BigInteger x = Next(); + BigInteger y = Next(); + // Include all-one limbs, zero, and carry/borrow across limb boundaries. + if (i < 4) + { + x = i == 0 ? 0 : (BigInteger.One << (2 * width)) - 1; + y = i < 2 ? x : BigInteger.One << ((i - 2) * width); + } + + object[] product = { Limb(x), Limb(y), T.Zero, T.Zero }; + multiply.Invoke(null, product); + BigInteger expected = (x & mask) * (y & mask); + Equal(product[2], expected >> width); + Equal(product[3], expected & mask); + + // The private addition helper's contract excludes high-limb overflow. + BigInteger a = x >> 1; + BigInteger b = y >> 1; + object[] sum = { Limb(a >> width), Limb(a), Limb(b >> width), Limb(b), T.Zero, T.Zero }; + add.Invoke(null, sum); + expected = a + b; + Equal(sum[4], expected >> width); + Equal(sum[5], expected & mask); + + a = BigInteger.Max(x, y); + b = BigInteger.Min(x, y); + object[] difference = { Limb(a >> width), Limb(a), Limb(b >> width), Limb(b), T.Zero, T.Zero }; + subtract.Invoke(null, difference); + expected = a - b; + Equal(difference[4], expected >> width); + Equal(difference[5], expected & mask); + + BigInteger divisor = BigInteger.Pow(10, i % (maximumPower + 1)); + object[] quotient = { Limb(x >> width), Limb(x), Limb(divisor) }; + object remainder = divide.Invoke(null, quotient)!; + expected = BigInteger.DivRem(x, divisor, out BigInteger expectedRemainder); + Equal(quotient[0], expected >> width); + Equal(quotient[1], expected & mask); + Equal(remainder, expectedRemainder); + } + } + + [Fact] + public static void TestEntryPoint() + { + Check(32, 4); + Check(64, 9); + Check(128, 19); + } +} diff --git a/src/tests/JIT/opt/Add/DecimalWide.csproj b/src/tests/JIT/opt/Add/DecimalWide.csproj new file mode 100644 index 00000000000000..ba355ba38b6f66 --- /dev/null +++ b/src/tests/JIT/opt/Add/DecimalWide.csproj @@ -0,0 +1,12 @@ + + + true + None + True + + + + + + + diff --git a/src/tests/JIT/opt/Add/Division.cs b/src/tests/JIT/opt/Add/Division.cs new file mode 100644 index 00000000000000..3764b78f1523af --- /dev/null +++ b/src/tests/JIT/opt/Add/Division.cs @@ -0,0 +1,681 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Reflection; +using System.Runtime.CompilerServices; +using Xunit; + +public class DivisionTests +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong RemainderNonZero(ulong value, ulong divisor) + { + // X64: div + // X64: ret + divisor |= 1; + return Math.DivRem(value, divisor).Remainder; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong AddWithHighMemory(ulong low, ulong value, ref uint high) + { + // X64: add + // X64-NEXT: adc + // ARM64: adds + // ARM64-NEXT: adc + ulong sum = low + value; + ulong upper = high; + return upper + (sum < low ? 1UL : 0UL); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void IncrementHigh(ref uint high) => high++; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong AddWithChangedHighMemory(ulong low, ulong value, ref uint high) + { + ulong sum = low + value; + IncrementHigh(ref high); + ulong upper = high; + return upper + (sum < low ? 1UL : 0UL); + } + + private static void CheckDecimalArithmetic() + { + BigInteger max = (BigInteger.One << 96) - 1; + (BigInteger Coefficient, int Scale) Parts(decimal value) + { + int[] bits = decimal.GetBits(value); + BigInteger coefficient = (uint)bits[0] | ((BigInteger)(uint)bits[1] << 32) | + ((BigInteger)(uint)bits[2] << 64); + return (bits[3] < 0 ? -coefficient : coefficient, (bits[3] >> 16) & 255); + } + + decimal Make(BigInteger coefficient, int scale) => new decimal( + (int)(uint)(BigInteger.Abs(coefficient) & uint.MaxValue), + (int)(uint)((BigInteger.Abs(coefficient) >> 32) & uint.MaxValue), + (int)(uint)(BigInteger.Abs(coefficient) >> 64), coefficient.Sign < 0, (byte)scale); + + void Check(decimal a, decimal b) + { + var (ca, sa) = Parts(a); + var (cb, sb) = Parts(b); + int scale = Math.Max(sa, sb); + ca *= BigInteger.Pow(10, scale - sa); + cb *= BigInteger.Pow(10, scale - sb); + foreach (bool subtract in new[] { false, true }) + { + BigInteger exact = subtract ? ca - cb : ca + cb; + bool fits = false; + for (int drop = 0; drop <= scale; drop++) + { + BigInteger divisor = BigInteger.Pow(10, drop); + BigInteger q = BigInteger.DivRem(BigInteger.Abs(exact), divisor, out BigInteger r); + if (r * 2 > divisor || (r * 2 == divisor && !q.IsEven)) + { + q++; + } + if (q <= max) + { + decimal expected = Make(exact.Sign < 0 ? -q : q, scale - drop); + decimal actual = subtract ? a - b : a + b; + if (actual != expected) + { + throw new Exception("Decimal addition/subtraction oracle"); + } + fits = true; + break; + } + } + if (!fits) + { + Throws(() => { _ = subtract ? a - b : a + b; }); + } + } + if (cb != 0) + { + BigInteger remainder = ca % cb; + int remainderScale = scale; + while (BigInteger.Abs(remainder) > max && remainderScale > 0 && remainder % 10 == 0) + { + remainder /= 10; + remainderScale--; + } + decimal actualRemainder = a % b; + decimal expectedRemainder = Make(remainder, remainderScale); + if (actualRemainder != expectedRemainder) + { + throw new Exception($"Decimal remainder oracle: {a} % {b} = {actualRemainder}, expected {expectedRemainder}"); + } + } + } + + decimal[] edges = { 0m, 1m, -1m, decimal.MaxValue, decimal.MinValue, + new decimal(-1, -1, 0, false, 0), new decimal(0, 0, 1, false, 0), + new decimal(-1, -1, -1, false, 1), new decimal(-1, -1, -1, true, 28), + 0.0000000000000000000000000001m, 0.5m, 1.5m, 10m, uint.MaxValue }; + foreach (decimal a in edges) + { + foreach (decimal b in edges) + { + Check(a, b); + } + } + Random random = new Random(789); + byte[] bytes = new byte[24]; + for (int i = 0; i < 1500; i++) + { + random.NextBytes(bytes); + int scale = random.Next(29); + decimal a = new decimal(BitConverter.ToInt32(bytes, 0), BitConverter.ToInt32(bytes, 4), + BitConverter.ToInt32(bytes, 8), (i & 1) != 0, (byte)scale); + decimal b = new decimal(BitConverter.ToInt32(bytes, 12), i % 3 == 0 ? 0 : BitConverter.ToInt32(bytes, 16), + i % 3 == 0 ? 0 : BitConverter.ToInt32(bytes, 20), (i & 2) != 0, + (byte)(i % 2 == 0 ? scale : random.Next(29))); + Check(a, b); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static long SubtractWithWidenedHigh(ulong low, ulong subLow, uint high, ulong subHigh, out ulong result) + { + // X64: sub + // X64-NEXT: sbb + // ARM64: subs + // ARM64-NEXT: sbc + ulong difference = low - subLow; + long upper = (long)high - (long)subHigh; + upper -= difference > low ? 1L : 0L; + result = difference; + return upper; + } + + private static void CheckDecimalDivisionHelpers() + { + Type calculator = typeof(decimal).GetNestedType("DecCalc", BindingFlags.NonPublic); + Type buf12 = calculator.GetNestedType("Buf12", BindingFlags.NonPublic); + Type buf16 = calculator.GetNestedType("Buf16", BindingFlags.NonPublic); + MethodInfo divide96 = calculator.GetMethod("Div96By32", BindingFlags.NonPublic | BindingFlags.Static); + MethodInfo divide128 = calculator.GetMethod("Div128By96", BindingFlags.NonPublic | BindingFlags.Static); + MethodInfo add32 = calculator.GetMethod("Add32To96", BindingFlags.NonPublic | BindingFlags.Static); + + object Pack(Type type, BigInteger value, int limbs) + { + object buffer = Activator.CreateInstance(type); + for (int limb = 0; limb < limbs; limb++) + { + type.GetField($"U{limb}").SetValue(buffer, (uint)(value & uint.MaxValue)); + value >>= 32; + } + return buffer; + } + + BigInteger Unpack(object buffer, int limbs) + { + BigInteger value = 0; + for (int limb = limbs - 1; limb >= 0; limb--) + { + value = (value << 32) | (uint)buffer.GetType().GetField($"U{limb}").GetValue(buffer); + } + return value; + } + + int correctionsSeen = 0; + void Check128(BigInteger numerator, BigInteger denominator) + { + BigInteger expected = BigInteger.DivRem(numerator, denominator, out BigInteger remainder); + BigInteger estimate = (numerator >> 64) / (denominator >> 64); + int corrections = (int)(estimate - expected); + if (corrections < 0 || corrections > 2) + { + throw new Exception("Invalid normalized division test vector"); + } + correctionsSeen |= 1 << corrections; + object[] args = { Pack(buf16, numerator, 4), Pack(buf12, denominator, 3) }; + uint quotient = (uint)divide128.Invoke(null, args); + if (quotient != expected || Unpack(args[0], 3) != remainder) + { + throw new Exception($"Decimal 128/96 division, corrections {corrections}"); + } + } + + BigInteger normalized = (BigInteger.One << 95) | ulong.MaxValue; + foreach (uint q in new uint[] { 0, 1, 2, 0x7FFFFFFF, 0xFFFFFFFD }) + { + foreach (BigInteger remainder in new[] { BigInteger.Zero, BigInteger.One, normalized - 1 }) + { + Check128(normalized * q + remainder, normalized); + } + } + + Random random = new Random(456); + byte[] bytes = new byte[16]; + for (int i = 0; i < 1500; i++) + { + random.NextBytes(bytes); + BigInteger numerator = new BigInteger(bytes, isUnsigned: true); + random.NextBytes(bytes); + BigInteger denominator = (new BigInteger(bytes, isUnsigned: true) & ((BigInteger.One << 96) - 1)) | + (BigInteger.One << 95); + // The helper requires the highest dividend limb to be below the + // highest divisor limb, so its quotient estimate fits in 32 bits. + Check128(numerator % ((denominator >> 64) << 96), denominator); + + uint divisor = (i % 4) switch { 0 => 1U, 1 => uint.MaxValue, 2 => 10U, _ => (uint)random.Next(1, int.MaxValue) }; + numerator &= (BigInteger.One << 96) - 1; + if (i % 3 == 0) + { + numerator &= ulong.MaxValue; + } + BigInteger expected = BigInteger.DivRem(numerator, divisor, out BigInteger remainder); + object[] args = { Pack(buf12, numerator, 3), divisor }; + uint actualRemainder = (uint)divide96.Invoke(null, args); + if (Unpack(args[0], 3) != expected || actualRemainder != remainder) + { + throw new Exception("Decimal 96/32 division"); + } + BigInteger max = (BigInteger.One << 96) - 1; + if (i % 5 == 0) + { + numerator = max - i % 3; + } + args = new object[] { Pack(buf12, numerator, 3), divisor }; + bool fits = (bool)add32.Invoke(null, args); + BigInteger sum = numerator + divisor; + if (fits != (sum <= max) || Unpack(args[0], 3) != (sum & max)) + { + throw new Exception("Decimal add32 carry/overflow"); + } + } + if (correctionsSeen != 7) + { + throw new Exception("Decimal tests must exercise zero, one, and two quotient corrections"); + } + } + + private static void CheckDecimalRounding() + { + Random random = new Random(123); + byte[] bytes = new byte[12]; + for (int i = 0; i < 1200; i++) + { + random.NextBytes(bytes); + BigInteger coefficient = new BigInteger(bytes, isUnsigned: true); + if (i % 3 == 0) + { + coefficient &= ulong.MaxValue; + } + int scale = i % 29; + int decimals = random.Next(scale + 1); + BigInteger divisor = BigInteger.Pow(10, scale - decimals); + // Include exact midpoints, their neighbors, zero, and maximal limbs. + if (i % 7 == 0) + { + coefficient = divisor / 2 + i % 3 - 1; + coefficient = BigInteger.Max(coefficient, BigInteger.Zero); + } + else if (i % 11 == 0) + { + coefficient = (BigInteger.One << 96) - 1; + } + bool negative = (i & 1) != 0; + decimal value = new decimal((int)(uint)(coefficient & uint.MaxValue), + (int)(uint)((coefficient >> 32) & uint.MaxValue), + (int)(uint)(coefficient >> 64), negative, (byte)scale); + BigInteger quotient = BigInteger.DivRem(coefficient, divisor, out BigInteger remainder); + foreach (MidpointRounding mode in Enum.GetValues()) + { + bool increment = mode switch + { + MidpointRounding.ToEven => remainder * 2 > divisor || + (remainder * 2 == divisor && !quotient.IsEven), + MidpointRounding.AwayFromZero => remainder * 2 >= divisor, + MidpointRounding.ToNegativeInfinity => negative && !remainder.IsZero, + MidpointRounding.ToPositiveInfinity => !negative && !remainder.IsZero, + _ => false, + }; + BigInteger expected = quotient + (increment ? 1 : 0); + int[] bits = decimal.GetBits(decimal.Round(value, decimals, mode)); + BigInteger actual = (uint)bits[0] | ((BigInteger)(uint)bits[1] << 32) | + ((BigInteger)(uint)bits[2] << 64); + int expectedFlags = (decimals << 16) | (negative ? int.MinValue : 0); + if (actual != expected || bits[3] != expectedFlags) + { + throw new Exception($"Decimal rounding: vector {i}, mode {mode}"); + } + } + } + } + + private static int s_calls; + private delegate nuint WideningDivide(nuint hi, nuint lo, nuint divisor, out nuint remainder); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Unsigned(ulong x, ulong y, out ulong remainder) + { + // X64: div + // X64-NOT: imul + // X64-NOT: sub + // X64: ret + ulong q = x / y; + remainder = x - q * y; + return q; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static long Signed(long x, long y, out long remainder) + { + // X64: idiv + // X64-NOT: imul + // X64-NOT: sub + // X64: ret + long q = x / y; + remainder = x - y * q; + return q; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint Unsigned32(uint x, uint y, out uint remainder) + { + // X64: div + // X64-NOT: imul + // X64: ret + // X86: div + // X86-NOT: imul + // X86: ret + uint q = x / y; + remainder = x - q * y; + return q; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int Signed32(int x, int y, out int remainder) + { + // X64: idiv + // X64-NOT: imul + // X64: ret + // X86: idiv + // X86-NOT: imul + // X86: ret + int q = x / y; + remainder = x - q * y; + return q; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong RemainderOnly(ulong x, ulong y) + { + return x - (x / y) * y; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong InlinedRemainder(ulong x, ulong y) => Math.DivRem(x, y).Remainder; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Touch(ulong x) => x + 3; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong AcrossCall(ulong x, ulong y) + { + ulong q = x / y; + ulong z = Touch(x); + return z ^ q ^ (x - q * y); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong KeepQuotient(ulong x, ulong y, ulong z, out ulong remainder) + { + ulong q = x / y; + s_calls++; + remainder = x - q * y; + return q ^ z; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Changed(ulong x, ulong y) + { + ulong q = x / y; + x += 17; + y += 3; + q += 2; + return x - q * y; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Aliased(ulong[] values, ulong y) + { + ulong q = values[0] / y; + values[0] += 17; + return values[0] - q * y; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void Mutate(ref ulong x) => x += 17; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ChangedByRef(ulong x, ulong y) + { + ulong q = x / y; + Mutate(ref x); + return x - q * y; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong QuotientChangedByRef(ulong x, ulong y) + { + ulong q = x / y; + Mutate(ref q); + return x - q * y; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong EmbeddedAssignment(ulong x, ulong y) => x - ((x += 17) / y) * y; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong CheckedProduct(ulong x, ulong y, ulong z) + { + ulong q = x / y; + return checked(x - checked(q * z)); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 Unsigned128(UInt128 x, UInt128 y, out UInt128 r) + { + var result = UInt128.DivRem(x, y); + r = result.Remainder; + return result.Quotient; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Int128 Signed128(Int128 x, Int128 y, out Int128 r) + { + var result = Int128.DivRem(x, y); + r = result.Remainder; + return result.Quotient; + } + + private static void Check128(UInt128 a, UInt128 b) + { + if (b == 0) + { + return; + } + UInt128 q = Unsigned128(a, b, out UInt128 r); + if ((BigInteger)q * (BigInteger)b + (BigInteger)r != (BigInteger)a || r >= b) + { + throw new Exception("UInt128 division"); + } + + Int128 x = (Int128)a, y = (Int128)b; + if (x == Int128.MinValue && y == -1) + { + return; + } + Int128 sq = Signed128(x, y, out Int128 sr); + if ((BigInteger)sq * (BigInteger)y + (BigInteger)sr != (BigInteger)x || + BigInteger.Abs((BigInteger)sr) >= BigInteger.Abs((BigInteger)y) || + (sr != 0 && Int128.IsNegative(sr) != Int128.IsNegative(x)) || x % y != sr || x / y != sq) + { + throw new Exception("Int128 division"); + } + } + + private static void CheckScalar(ulong x, ulong y) + { + if (RemainderNonZero(x, y) != x % (y | 1)) + { + throw new Exception("The unused quotient must not remove the remainder's division"); + } + uint memoryHigh = (uint)y; + ulong carry = (ulong)(((BigInteger)x + y) >> 64); + if (AddWithHighMemory(x, y, ref memoryHigh) != (ulong)memoryHigh + carry) + { + throw new Exception("Carry with widened memory input"); + } + ulong changed = AddWithChangedHighMemory(x, y, ref memoryHigh); + if (changed != (ulong)(uint)(y + 1) + carry || memoryHigh != (uint)(y + 1)) + { + throw new Exception("Carry must observe the intervening memory write"); + } + long high = SubtractWithWidenedHigh(x, y, (uint)y, x, out ulong low); + BigInteger expected = ((((BigInteger)(uint)y << 64) | x) - (((BigInteger)x << 64) | y)) & + ((BigInteger.One << 128) - 1); + if ((((BigInteger)(ulong)high << 64) | low) != expected) + { + throw new Exception("Subtraction with widened high word"); + } + if (y == 0) + { + return; + } + ulong q = Unsigned(x, y, out ulong r); + if ((BigInteger)q * y + r != x || r >= y || RemainderOnly(x, y) != r || + InlinedRemainder(x, y) != r || AcrossCall(x, y) != ((x + 3) ^ q ^ r)) + { + throw new Exception("unsigned scalar"); + } + int calls = s_calls; + if (KeepQuotient(x, y, ~x, out ulong r2) != (q ^ ~x) || r2 != r || s_calls != calls + 1) + { + throw new Exception("quotient lifetime"); + } + if (Changed(x, y) != unchecked((x + 17) - (q + 2) * (y + 3)) || + Aliased(new[] { x }, y) != unchecked(x + 17 - q * y) || + ChangedByRef(x, y) != unchecked(x + 17 - q * y) || + QuotientChangedByRef(x, y) != unchecked(x - (q + 17) * y) || + EmbeddedAssignment(x, y) != unchecked(x - ((x + 17) / y) * y)) + { + throw new Exception("changed operands"); + } + + long sx = (long)x, sy = (long)y; + if (!(sx == long.MinValue && sy == -1)) + { + long sq = Signed(sx, sy, out long sr); + if ((BigInteger)sq * sy + sr != sx || BigInteger.Abs(sr) >= BigInteger.Abs(sy) || + (sr != 0 && (sr < 0) != (sx < 0))) + { + throw new Exception("signed scalar"); + } + } + + uint ux = (uint)x, uy = (uint)y; + if (uy == 0) + { + return; + } + uint uq = Unsigned32(ux, uy, out uint ur); + if ((ulong)uq * uy + ur != ux || ur >= uy) + { + throw new Exception("uint division"); + } + int ix = (int)x, iy = (int)y; + if (!(ix == int.MinValue && iy == -1)) + { + int iq = Signed32(ix, iy, out int ir); + if ((long)iq * iy + ir != ix || Math.Abs((long)ir) >= Math.Abs((long)iy) || + (ir != 0 && (ir < 0) != (ix < 0))) + { + throw new Exception("int division"); + } + } + } + + private static void Throws(Action action) where T : Exception + { + try + { + action(); + } + catch (T) + { + return; + } + throw new Exception($"Expected {typeof(T)}"); + } + + [Fact] + public static void TestEntryPoint() + { + ulong[] edges = { 0, 1, 2, 3, uint.MaxValue, (ulong)uint.MaxValue + 1, 1UL << 63, (1UL << 63) - 1, ulong.MaxValue }; + foreach (ulong x in edges) + { + foreach (ulong y in edges) + { + CheckScalar(x, y); + Check128(x, y); + Check128(((UInt128)x << 64) | y, ((UInt128)y << 64) | x); + } + } + Throws(() => Unsigned(123, 0, out _)); + Throws(() => Signed(123, 0, out _)); + Throws(() => Unsigned32(123, 0, out _)); + Throws(() => Signed32(123, 0, out _)); + int callsBeforeException = s_calls; + Throws(() => KeepQuotient(123, 0, 456, out _)); + if (s_calls != callsBeforeException) + { + throw new Exception("division exception order"); + } + Throws(() => Signed(long.MinValue, -1, out _)); + Throws(() => Signed32(int.MinValue, -1, out _)); + Throws(() => Signed128(123, 0, out _)); + Throws(() => Signed128(Int128.MinValue, -1, out _)); + Throws(() => { _ = Int128.MinValue % -1; }); + Throws(() => CheckedProduct(ulong.MaxValue, 1, 2)); + + Type calculator = typeof(BigInteger).Assembly.GetType("System.Numerics.BigIntegerCalculator", throwOnError: true); + MethodInfo widening = calculator.GetMethod("DivRem", BindingFlags.Static | BindingFlags.NonPublic, + binder: null, new[] { typeof(nuint), typeof(nuint), typeof(nuint), typeof(nuint).MakeByRefType() }, modifiers: null); + WideningDivide divide = widening.CreateDelegate(); + Throws(() => divide(0, 123, 0, out _)); + foreach (ulong edge in edges) + { + nuint d = (nuint)edge; + if (d == 0) + { + continue; + } + foreach (nuint hi in new nuint[] { 0, d - 1 }) + { + foreach (nuint lo in new nuint[] { 0, 1, nuint.MaxValue }) + { + nuint q = divide(hi, lo, d, out nuint r); + BigInteger value = ((BigInteger)(ulong)hi << (IntPtr.Size * 8)) | (ulong)lo; + if ((BigInteger)(ulong)q * (ulong)d + (ulong)r != value || r >= d) + { + throw new Exception("widening division"); + } + } + } + } + + CheckDecimalRounding(); + CheckDecimalDivisionHelpers(); + CheckDecimalArithmetic(); + + Random random = new Random(42); + byte[] bytes = new byte[32]; + for (int i = 0; i < 3000; i++) + { + random.NextBytes(bytes); + ulong lo = BitConverter.ToUInt64(bytes, 0), hi = BitConverter.ToUInt64(bytes, 8); + ulong dlo = BitConverter.ToUInt64(bytes, 16), dhi = BitConverter.ToUInt64(bytes, 24); + CheckScalar(lo, dlo); + UInt128 a = ((UInt128)hi << 64) | lo; + Check128(a, ((UInt128)dhi << 64) | dlo); + Check128(a, dlo); + } + + // Exercise both the single-limb and multi-limb BigInteger division paths. + for (int length = 1; length <= 160; length++) + { + byte[] input = new byte[length * IntPtr.Size]; + random.NextBytes(input); + BigInteger a = new BigInteger(input, isUnsigned: true); + foreach (ulong d in edges) + { + if (d == 0) + { + continue; + } + BigInteger q = BigInteger.DivRem(a, d, out BigInteger r); + if (q * d + r != a || r < 0 || r >= d) + { + throw new Exception("BigInteger limb division"); + } + } + byte[] divisor = new byte[Math.Max(1, input.Length / 2)]; + random.NextBytes(divisor); + divisor[0] |= 1; + BigInteger b = new BigInteger(divisor, isUnsigned: true); + BigInteger quotient = BigInteger.DivRem(a, b, out BigInteger remainder); + if (quotient * b + remainder != a || remainder < 0 || remainder >= b) + { + throw new Exception("BigInteger multi-limb division"); + } + } + } +} diff --git a/src/tests/JIT/opt/Add/Division.csproj b/src/tests/JIT/opt/Add/Division.csproj new file mode 100644 index 00000000000000..e927ded2fbb6d3 --- /dev/null +++ b/src/tests/JIT/opt/Add/Division.csproj @@ -0,0 +1,14 @@ + + + true + None + True + + + + true + + + + + diff --git a/src/tests/JIT/opt/Add/FunnelShift.cs b/src/tests/JIT/opt/Add/FunnelShift.cs new file mode 100644 index 00000000000000..c154264192fb5e --- /dev/null +++ b/src/tests/JIT/opt/Add/FunnelShift.cs @@ -0,0 +1,182 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using Xunit; + +public class FunnelShiftTests +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Shift1(ulong lo, ulong hi) + { + // X64: shrd {{.*}}, 1 + // ARM64: extr {{.*}}, #1 + return (lo >> 1) | (hi << 63); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Shift17(ulong lo, ulong hi) + { + // X64: shrd {{.*}}, 17 + // ARM64: extr {{.*}}, #17 + return (hi << 47) | (lo >> 17); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Shift32(ulong lo, ulong hi) + { + // X64: shrd {{.*}}, 32 + // ARM64: extr {{.*}}, #32 + return (lo >> 32) | (hi << 32); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Shift63(ulong lo, ulong hi) + { + // X64: shrd {{.*}}, 63 + // ARM64: extr {{.*}}, #63 + return (lo >> 63) | (hi << 1); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint Shift13(uint lo, uint hi) + { + // X64: shrd {{.*}}, 13 + // ARM64: extr {{.*}}, #13 + return (lo >> 13) | (hi << 19); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint Shift31(uint lo, uint hi) + { + // X64: shrd {{.*}}, 31 + // ARM64: extr {{.*}}, #31 + return (hi << 1) | (lo >> 31); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Masked(ulong lo, ulong hi) => (lo >> 65) | (hi << -1); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong KeepInputs(ulong lo, ulong hi, out ulong result) + { + result = (lo >> 17) | (hi << 47); + return lo ^ hi; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool IsZero(ulong lo, ulong hi) => ((lo >> 17) | (hi << 47)) == 0; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Overlap(ulong lo, ulong hi) => (lo >> 17) | (hi << 46); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Gap(ulong lo, ulong hi) => (lo >> 17) | (hi << 48); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ZeroCount(ulong lo, ulong hi) => (lo >> 64) | (hi << 64); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static long Signed(long lo, long hi) => (lo >> 17) | (hi << 47); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Redefine(ulong lo, ulong hi) => (lo >> 17) | ((lo = hi) << 47); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Aliased(ulong[] values) + { + ulong shifted = values[0] >> 17; + values[0] = ~values[0]; + return shifted | (values[0] << 47); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 SquareStep(ulong a, ulong b, ulong old, UInt128 carry, out ulong low) + { + // X64: shrd {{.*}}, 1 + // X64: shrd {{.*}}, 63 + // ARM64: extr {{.*}}, #1 + // ARM64: extr {{.*}}, #63 + UInt128 digit1 = old + carry; + UInt128 digit2 = (UInt128)a * b; + low = (ulong)(digit1 + (digit2 << 1)); + return (digit2 + (digit1 >> 1)) >> 63; + } + + // Construct the expected result one bit at a time, independently of the + // complementary-shift pattern under test. + private static ulong Expected(ulong lo, ulong hi, int count, int width = 64) + { + ulong result = 0; + for (int bit = 0; bit < width; bit++) + { + int sourceBit = bit + count; + ulong source = sourceBit < width ? lo : hi; + result |= ((source >> (sourceBit % width)) & 1) << bit; + } + return result; + } + + private static void Check(ulong lo, ulong hi) + { + if (Shift1(lo, hi) != Expected(lo, hi, 1) || + Shift17(lo, hi) != Expected(lo, hi, 17) || + Shift32(lo, hi) != Expected(lo, hi, 32) || + Shift63(lo, hi) != Expected(lo, hi, 63) || + Shift13((uint)lo, (uint)hi) != Expected((uint)lo, (uint)hi, 13, 32) || + Shift31((uint)lo, (uint)hi) != Expected((uint)lo, (uint)hi, 31, 32) || + Masked(lo, hi) != Expected(lo, hi, 1) || + KeepInputs(lo, hi, out ulong result) != (lo ^ hi) || result != Expected(lo, hi, 17) || + IsZero(lo, hi) != (Expected(lo, hi, 17) == 0) || + Overlap(lo, hi) != ((lo >> 17) | (hi << 46)) || + Gap(lo, hi) != ((lo >> 17) | (hi << 48)) || + ZeroCount(lo, hi) != (lo | hi) || + Signed((long)lo, (long)hi) != (((long)lo >> 17) | ((long)hi << 47)) || + Redefine(lo, hi) != Expected(lo, hi, 17) || + Aliased(new[] { lo }) != Expected(lo, ~lo, 17)) + { + throw new Exception($"Funnel shift failed: {lo:X16}, {hi:X16}"); + } + + // The square kernel carries 65 bits between limbs. Use a wider oracle + // so the doubled product's 129th bit is retained independently. + UInt128 carry = ((UInt128)(hi & 1) << 64) | lo; + BigInteger full = (BigInteger)lo * hi * 2 + hi + (BigInteger)carry; + UInt128 nextCarry = SquareStep(lo, hi, hi, carry, out ulong low); + if (low != (ulong)(full & ulong.MaxValue) || nextCarry != (UInt128)(full >> 64)) + { + throw new Exception($"Square carry failed: {lo:X16}, {hi:X16}"); + } + } + + [Fact] + public static void TestEntryPoint() + { + ulong[] edges = { 0, 1, ulong.MaxValue, 1UL << 63, (1UL << 63) - 1, 0xAAAAAAAAAAAAAAAA, 0x5555555555555555 }; + foreach (ulong lo in edges) + { + foreach (ulong hi in edges) + { + Check(lo, hi); + } + } + + for (int lo = 0; lo < 64; lo++) + { + for (int hi = 0; hi < 64; hi++) + { + Check(1UL << lo, 1UL << hi); + } + } + + Random random = new Random(42); + byte[] bytes = new byte[16]; + for (int i = 0; i < 2000; i++) + { + random.NextBytes(bytes); + Check(BitConverter.ToUInt64(bytes, 0), BitConverter.ToUInt64(bytes, 8)); + } + } +} diff --git a/src/tests/JIT/opt/Add/FunnelShift.csproj b/src/tests/JIT/opt/Add/FunnelShift.csproj new file mode 100644 index 00000000000000..49fbd76be0ee73 --- /dev/null +++ b/src/tests/JIT/opt/Add/FunnelShift.csproj @@ -0,0 +1,14 @@ + + + true + None + True + + + + true + + + + + diff --git a/src/tests/JIT/opt/Add/WideArithmetic.cs b/src/tests/JIT/opt/Add/WideArithmetic.cs new file mode 100644 index 00000000000000..29d8e0937b73ef --- /dev/null +++ b/src/tests/JIT/opt/Add/WideArithmetic.cs @@ -0,0 +1,243 @@ +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using Xunit; + +public class WideArithmeticTests +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 Increment(UInt128 value) => value + 1; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 Decrement(UInt128 value) + { + // X64: sub + // X64-NEXT: sbb + // ARM64: subs + // ARM64-NEXT: sbc + return value - 1; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Int128 Negate(Int128 value) + { + // X64: neg + // X64-NEXT: sbb + // ARM64: negs + // ARM64-NEXT: sbc {{.*}}, xzr, + return -value; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong SubtractSeven(ulong value, ulong high, out ulong low) + { + // X64: sub + // X64-NOT: cmp + // X64: sbb + // ARM64: subs + // ARM64-NEXT: sbc + // ARM64-NEXT: str {{x[0-9]+}}, [{{x[0-9]+}}] + // ARM64-NOT: mov + ulong difference = value - 7; + ulong result = high - (difference > value ? 1UL : 0UL); + low = difference; + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint SubtractSeven32(uint value, uint high, out uint low) + { + // X64: sub + // X64-NOT: cmp + // X64: sbb + // ARM64: subs + // ARM64-NEXT: sbc + // ARM64-NEXT: str {{w[0-9]+}}, [{{x[0-9]+}}] + // ARM64-NOT: mov + uint difference = value - 7; + uint result = high - (difference > value ? 1U : 0U); + low = difference; + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong SubtractHalfRange(ulong value, ulong high, out ulong low) + { + ulong difference = value - (1UL << 63); + ulong result = high - (difference > value ? 1UL : 0UL); + low = difference; + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong ComplementBorrow(ulong value, out ulong low) + { + ulong difference = value - 7; + ulong result = difference <= value ? 1UL : 0UL; + low = difference; + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool NegatedProductIsZero(long a, long b, out long product) + { + // The returned condition must not displace the still-live product. + // X64: imul + // X64-NEXT: neg + // X64-NEXT: setae + // X64-NEXT: movzx + // X64-NEXT: mov {{qword ptr \[[^]]+\]}}, + // A contained ARM64 MNEG must not be used as a flags producer. + long value = -(a * b); + bool zero = value == 0; + product = value; + return zero; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong CorrectQuotient(ulong high, ulong low, ulong extra, out ulong sum) + { + // X64: add + // X64-NEXT: adc + // ARM64: adds + // ARM64-NEXT: adc + ulong middle = low + extra; + high += middle < extra ? 1UL : 0UL; + sum = middle; + return high; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool ClearAndReturnFalse(out object value) + { + // The return constant can reuse the register already zeroed for the store. + // X64: xor + // X64-NEXT: mov + // X64-NOT: xor + // X64: ret + value = null; + return false; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 Multiply(UInt128 a, UInt128 b, out UInt128 low) => UInt128.BigMul(a, b, out low); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Int128 MultiplySigned(Int128 a, Int128 b, out Int128 low) => Int128.BigMul(a, b, out low); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Int128 CheckedAdd(Int128 a, Int128 b) => checked(a + b); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Int128 CheckedSubtract(Int128 a, Int128 b) => checked(a - b); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Int128 CheckedNegate(Int128 value) => checked(-value); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Int128 CheckedMultiply(Int128 a, Int128 b) => checked(a * b); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static uint Negate32(uint value, uint high, out uint low) + { + uint difference = 0 - value; + uint result = (0 - high) - (difference != 0 ? 1U : 0U); + low = difference; + return result; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static UInt128 CheckedDecrement(UInt128 value) => checked(value - 1); + + private static void Require(bool condition) + { + if (!condition) + { + throw new Exception("Wide arithmetic mismatch"); + } + } + + private static void CheckOverflow(Func operation, BigInteger expected) + { + bool overflow = expected < (BigInteger)Int128.MinValue || expected > (BigInteger)Int128.MaxValue; + try + { + Int128 actual = operation(); + Require(!overflow && (BigInteger)actual == expected); + } + catch (OverflowException) + { + Require(overflow); + } + } + + [Fact] + public static void TestEntryPoint() + { + ulong[] edges = { 0, 1, 6, 7, 8, uint.MaxValue, 1UL << 32, 1UL << 63, ulong.MaxValue - 1, ulong.MaxValue }; + var random = new Random(80674); + byte[] bytes = new byte[32]; + BigInteger mask = (BigInteger.One << 128) - 1; + Require(!ClearAndReturnFalse(out object cleared) && cleared == null); + for (int i = 0; i < 2500; i++) + { + random.NextBytes(bytes); + UInt128 a = ((UInt128)BitConverter.ToUInt64(bytes, 0) << 64) | BitConverter.ToUInt64(bytes, 8); + UInt128 b = ((UInt128)BitConverter.ToUInt64(bytes, 16) << 64) | BitConverter.ToUInt64(bytes, 24); + if (i < edges.Length * edges.Length) + { + a = ((UInt128)edges[i / edges.Length] << 64) | edges[i % edges.Length]; + b = UInt128.MaxValue - a; + } + BigInteger product = (BigInteger)a * (BigInteger)b; + UInt128 high = Multiply(a, b, out UInt128 low); + Require((BigInteger)high == product >> 128 && (BigInteger)low == (product & mask)); + product = (BigInteger)(Int128)a * (BigInteger)(Int128)b; + Int128 signedHigh = MultiplySigned((Int128)a, (Int128)b, out Int128 signedLow); + Require((BigInteger)signedHigh == product >> 128 && (BigInteger)(UInt128)signedLow == (product & mask)); + Require((BigInteger)Increment(a) == (((BigInteger)a + 1) & mask)); + Require((BigInteger)Decrement(a) == (((BigInteger)a - 1) & mask)); + Require((BigInteger)(UInt128)Negate((Int128)a) == (-(BigInteger)a & mask)); + CheckOverflow(() => CheckedAdd((Int128)a, (Int128)b), (BigInteger)(Int128)a + (BigInteger)(Int128)b); + CheckOverflow(() => CheckedSubtract((Int128)a, (Int128)b), (BigInteger)(Int128)a - (BigInteger)(Int128)b); + CheckOverflow(() => CheckedNegate((Int128)a), -(BigInteger)(Int128)a); + if (i < edges.Length * edges.Length) + { + CheckOverflow(() => CheckedMultiply((Int128)a, (Int128)b), product); + } + try + { + Require(CheckedDecrement(a) == Decrement(a) && a != 0); + } + catch (OverflowException) + { + Require(a == 0); + } + ulong x = (ulong)a, h = (ulong)(a >> 64); + ulong pair32 = ((ulong)(uint)h << 32) | (uint)x; + uint negatedHigh = Negate32((uint)x, (uint)h, out uint negatedLow); + Require((((ulong)negatedHigh << 32) | negatedLow) == unchecked(0UL - pair32)); + Require(SubtractSeven(x, h, out ulong l) == unchecked(h - (x < 7 ? 1UL : 0)) && l == unchecked(x - 7)); + Require(SubtractSeven32((uint)x, (uint)h, out uint l32) == unchecked((uint)h - ((uint)x < 7 ? 1U : 0)) && l32 == unchecked((uint)x - 7)); + Require(SubtractHalfRange(x, h, out l) == unchecked(h - (x < (1UL << 63) ? 1UL : 0)) && l == unchecked(x - (1UL << 63))); + Require(ComplementBorrow(x, out l) == (x >= 7 ? 1UL : 0) && l == unchecked(x - 7)); + bool zero = NegatedProductIsZero((long)x, (long)h, out long negated); + Require(negated == unchecked(-((long)x * (long)h)) && zero == (negated == 0)); + UInt128 sum = (UInt128)x + h; + Require(CorrectQuotient((ulong)b, x, h, out l) == unchecked((ulong)b + (ulong)(sum >> 64)) && l == (ulong)sum); + + // Exercise the calculator's multiply-by-limb and quotient correction + // through public APIs, using a known quotient and remainder. + int bits = IntPtr.Size * 8; + BigInteger divisor = ((BigInteger)a << bits) + (ulong)b + 1; + BigInteger quotient = (ulong)(nuint)x; + BigInteger remainder = divisor >> 1; + BigInteger dividend = divisor * quotient + remainder; + Require(BigInteger.DivRem(dividend, divisor, out BigInteger actualRemainder) == quotient && actualRemainder == remainder); + if (b != 0) + { + Require((BigInteger)(a / b) == (BigInteger)a / (BigInteger)b && (BigInteger)(a % b) == (BigInteger)a % (BigInteger)b); + } + } + } +} diff --git a/src/tests/JIT/opt/Add/WideArithmetic.csproj b/src/tests/JIT/opt/Add/WideArithmetic.csproj new file mode 100644 index 00000000000000..0b16b3cff8348c --- /dev/null +++ b/src/tests/JIT/opt/Add/WideArithmetic.csproj @@ -0,0 +1,14 @@ + + + true + None + True + + + + true + + + + + From 8592a85ffe7544d472b4636eb966f75dbc352e62 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 16 Sep 2026 09:42:05 +0100 Subject: [PATCH 2/4] Apply JIT formatting corrections --- src/coreclr/jit/codegenarm64.cpp | 4 ++-- src/coreclr/jit/gentree.cpp | 4 ++-- src/coreclr/jit/lower.cpp | 4 ++-- src/coreclr/jit/lowerxarch.cpp | 10 ++++------ src/coreclr/jit/lsraxarch.cpp | 6 +++--- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index ade39e4a2c434f..69f07c93be9d0e 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -2723,8 +2723,8 @@ void CodeGen::genCodeForBinary(GenTreeOp* tree) { assert(!tree->gtSetFlags()); emit->emitIns_R_R_R_I(INS_extr, emitTypeSize(tree), targetReg, op2->gtGetOp1()->GetRegNum(), - op1->gtGetOp1()->GetRegNum(), - static_cast(op1->gtGetOp2()->AsIntCon()->IconValue())); + op1->gtGetOp1()->GetRegNum(), + static_cast(op1->gtGetOp2()->AsIntCon()->IconValue())); genProduceReg(tree); return; } diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 88918dcd510180..44fd5d7ef8f387 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -12017,8 +12017,8 @@ void Compiler::gtUpdateNodeSideEffects(GenTree* tree) bool GenTree::IsFunnelShift() const { #if defined(TARGET_AMD64) || defined(TARGET_ARM64) - return OperIs(GT_OR) && gtGetOp1()->OperIs(GT_RSZ) && gtGetOp1()->isContained() && - gtGetOp2()->OperIs(GT_LSH) && gtGetOp2()->isContained(); + return OperIs(GT_OR) && gtGetOp1()->OperIs(GT_RSZ) && gtGetOp1()->isContained() && gtGetOp2()->OperIs(GT_LSH) && + gtGetOp2()->isContained(); #else return false; #endif diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 01bf6dd18df42d..e35f8de07c7cfb 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -13906,7 +13906,7 @@ bool Lowering::TryLowerCarryCompare(GenTreeOp* cmp) } else { - GenTreeCC* carry = m_compiler->gtNewCC(GT_SETCC, TYP_INT, condition); + GenTreeCC* carry = m_compiler->gtNewCC(GT_SETCC, TYP_INT, condition); m_blockMayHaveSetCC = true; BlockRange().InsertAfter(add, carry); use.ReplaceWith(carry); @@ -15138,7 +15138,7 @@ bool Lowering::LowerFullAdders() examined.push_back(adder.carryLocal); bool valid = true; bool initialized = false; - LclVarDsc* carryDsc = m_compiler->lvaGetDesc(adder.carryLocal); + LclVarDsc* carryDsc = m_compiler->lvaGetDesc(adder.carryLocal); for (BasicBlock* block : m_compiler->Blocks()) { for (GenTree* node : LIR::AsRange(block)) diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index c8fb5e1dc71ebf..f17685f6999c2a 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -8076,8 +8076,8 @@ void Lowering::ContainCheckMul(GenTreeOp* node) // void Lowering::TryLowerDivRem(GenTreeOp* div) { - if (!m_compiler->opts.OptimizationEnabled() || !div->OperIs(GT_DIV, GT_UDIV) || - !varTypeIsIntOrI(div) || div->IsDivRemPair()) + if (!m_compiler->opts.OptimizationEnabled() || !div->OperIs(GT_DIV, GT_UDIV) || !varTypeIsIntOrI(div) || + div->IsDivRemPair()) { return; } @@ -8104,8 +8104,7 @@ void Lowering::TryLowerDivRem(GenTreeOp* div) } GenTree* quotientStore = div->gtNext; - if ((quotientStore == nullptr) || !quotientStore->OperIs(GT_STORE_LCL_VAR) || - (quotientStore->gtGetOp1() != div)) + if ((quotientStore == nullptr) || !quotientStore->OperIs(GT_STORE_LCL_VAR) || (quotientStore->gtGetOp1() != div)) { quotientStore = nullptr; } @@ -8123,8 +8122,7 @@ void Lowering::TryLowerDivRem(GenTreeOp* div) return; } - if (!node->OperIs(GT_SUB) || node->gtOverflow() || node->gtSetFlags() || - (node->TypeGet() != div->TypeGet())) + if (!node->OperIs(GT_SUB) || node->gtOverflow() || node->gtSetFlags() || (node->TypeGet() != div->TypeGet())) { continue; } diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 3fad7e0e65796c..f50b30f9310078 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -251,8 +251,8 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_JCMP: assert(tree->gtGetOp2()->IsIntegralConst(0) && tree->gtGetOp2()->isContained()); - m_hasCarryArithmetic = true; - srcCount = BuildOperandUses(tree->gtGetOp1(), SRBM_ECX); + m_hasCarryArithmetic = true; + srcCount = BuildOperandUses(tree->gtGetOp1(), SRBM_ECX); break; #endif case GT_JCC: @@ -355,7 +355,7 @@ int LinearScan::BuildNode(GenTree* tree) case GT_ADCX: case GT_ADOX: m_hasCarryArithmetic = true; - srcCount = BuildBinaryUses(tree->AsOp(), lowGprRegs); + srcCount = BuildBinaryUses(tree->AsOp(), lowGprRegs); BuildDef(tree, lowGprRegs); break; case GT_SUB_BORROW: From 2073c37b9c4d9563c5e595a4b488435198b17522 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 16 Sep 2026 10:30:10 +0100 Subject: [PATCH 3/4] Clarify arithmetic lowering invariants and unify legacy opcode sizing --- src/coreclr/jit/emitxarch.cpp | 4 ++-- src/coreclr/jit/gentree.cpp | 21 +++++++++++++++++-- src/coreclr/jit/lower.cpp | 1 + src/coreclr/jit/lowerxarch.cpp | 2 ++ .../ThunkGenerator/InstructionSetDesc.txt | 1 + 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 2241d85af3c2f1..365f01f7638242 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -5269,8 +5269,8 @@ inline UNATIVE_OFFSET emitter::emitInsSizeRR(instrDesc* id) if ((code & 0xFF00) != 0) { - // ADX uses a four-byte opcode encoding; its mandatory prefix was counted above. - sz += (IsSimdInstruction(ins) || (ins == INS_adcx) || (ins == INS_adox) || TakesEvexPrefix(id)) + // Legacy map-38 instructions have their mandatory prefix counted above. + sz += (IsSimdInstruction(ins) || IsLegacyMap38Instruction(ins) || TakesEvexPrefix(id)) ? emitInsSize(id, code, includeRexPrefixSize) : 5; } diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 44fd5d7ef8f387..1ac666ef6485d4 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -12017,8 +12017,25 @@ void Compiler::gtUpdateNodeSideEffects(GenTree* tree) bool GenTree::IsFunnelShift() const { #if defined(TARGET_AMD64) || defined(TARGET_ARM64) - return OperIs(GT_OR) && gtGetOp1()->OperIs(GT_RSZ) && gtGetOp1()->isContained() && gtGetOp2()->OperIs(GT_LSH) && - gtGetOp2()->isContained(); + bool isFunnelShift = OperIs(GT_OR) && gtGetOp1()->OperIs(GT_RSZ) && gtGetOp1()->isContained() && + gtGetOp2()->OperIs(GT_LSH) && gtGetOp2()->isContained(); +#ifdef DEBUG + if (isFunnelShift) + { + // TryContainFunnelShift is the only path that contains both shifts. It validates + // their widths and complementary immediate counts before establishing this shape. + assert(TypeIs(TYP_INT, TYP_LONG)); + assert(gtGetOp1()->TypeGet() == TypeGet()); + assert(gtGetOp2()->TypeGet() == TypeGet()); + assert(gtGetOp1()->gtGetOp2()->IsCnsIntOrI()); + assert(gtGetOp2()->gtGetOp2()->IsCnsIntOrI()); + ssize_t width = genTypeSize(TypeGet()) * BITS_PER_BYTE; + ssize_t count = gtGetOp1()->gtGetOp2()->AsIntCon()->IconValue(); + assert((count > 0) && (count < width)); + assert(gtGetOp2()->gtGetOp2()->AsIntCon()->IconValue() == width - count); + } +#endif + return isFunnelShift; #else return false; #endif diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index e35f8de07c7cfb..5ce802839c337b 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -13848,6 +13848,7 @@ bool Lowering::TryLowerCarryCompare(GenTreeOp* cmp) #ifdef TARGET_AMD64 // Unlike a carry chain, a branch saves only a comparison. Avoid sacrificing LEA // or perturbing allocation in register-heavy methods for that small saving. + // LAST - FIRST excludes SP from the inclusive integer-register range. unsigned intRegCount = m_compiler->get_REG_INT_LAST() - REG_INT_FIRST; if (negatedConstant || (m_compiler->lvaTrackedCount > 2 * intRegCount)) { diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index f17685f6999c2a..7e7d3f70be5643 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -8082,6 +8082,8 @@ void Lowering::TryLowerDivRem(GenTreeOp* div) return; } + // The inclusive integer-register range includes SP, which is not allocatable. + // Exclude it from this rough capacity estimate; other reservations (such as FP) vary by method. #ifdef TARGET_AMD64 unsigned intRegCount = m_compiler->get_REG_INT_LAST() - REG_INT_FIRST; #else diff --git a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt index 811caee60723ce..4a261ff1040dc4 100644 --- a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt +++ b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt @@ -207,6 +207,7 @@ definearch ,X64 ,64Bit ,X64, X64, X86 copyinstructionsets,X86 ,X64 ; Append new instruction sets here to preserve existing JIT ISA numbers. +; ADX is selected by JIT pattern recognition, so it has no managed intrinsic class name. instructionset ,X86 , ,Adx ,96 ,ADX ,adx instructionset ,X64 , ,Adx ,96 ,ADX ,adx From 623d7e9c42bd2339aa5eb1cc6fef55479049606f Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 16 Sep 2026 11:11:37 +0100 Subject: [PATCH 4/4] Document decimal remainder arithmetic and add test license headers --- .../System.Private.CoreLib/src/System/Decimal.DecCalc.cs | 3 +++ src/tests/JIT/opt/Add/Borrow.cs | 3 +++ src/tests/JIT/opt/Add/DecimalMultiply.cs | 3 +++ src/tests/JIT/opt/Add/WideArithmetic.cs | 3 +++ 4 files changed, 12 insertions(+) diff --git a/src/libraries/System.Private.CoreLib/src/System/Decimal.DecCalc.cs b/src/libraries/System.Private.CoreLib/src/System/Decimal.DecCalc.cs index 79c6ae902c8bad..23cd9ef8861d62 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Decimal.DecCalc.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Decimal.DecCalc.cs @@ -200,6 +200,9 @@ private static (uint Quotient, uint Remainder) Div64By32(uint low, uint high, ui ulong dividend = ((ulong)high << 32) | low; uint quotient = (uint)(dividend / den); + // high < den guarantees that the quotient fits in uint. The remainder is + // below den and fits in uint too, so computing dividend - quotient * den + // modulo 2^32 gives the exact remainder using only the low limb. return (quotient, low - quotient * den); } diff --git a/src/tests/JIT/opt/Add/Borrow.cs b/src/tests/JIT/opt/Add/Borrow.cs index ad2166a49bb3ee..22daf33c427454 100644 --- a/src/tests/JIT/opt/Add/Borrow.cs +++ b/src/tests/JIT/opt/Add/Borrow.cs @@ -1,3 +1,6 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + using System; using System.Numerics; using System.Runtime.CompilerServices; diff --git a/src/tests/JIT/opt/Add/DecimalMultiply.cs b/src/tests/JIT/opt/Add/DecimalMultiply.cs index bd22abe3e49bf4..e5960a287c1596 100644 --- a/src/tests/JIT/opt/Add/DecimalMultiply.cs +++ b/src/tests/JIT/opt/Add/DecimalMultiply.cs @@ -1,3 +1,6 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + using System; using System.Numerics; using System.Runtime.CompilerServices; diff --git a/src/tests/JIT/opt/Add/WideArithmetic.cs b/src/tests/JIT/opt/Add/WideArithmetic.cs index 29d8e0937b73ef..022f866b2b92f5 100644 --- a/src/tests/JIT/opt/Add/WideArithmetic.cs +++ b/src/tests/JIT/opt/Add/WideArithmetic.cs @@ -1,3 +1,6 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + using System; using System.Numerics; using System.Runtime.CompilerServices;