From d870e25e4abaffd6d687dac42fcc15b6f3638bc1 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 15 Apr 2022 11:46:47 -0700 Subject: [PATCH 1/4] Arm64: Have CpBlkUnroll and InitBlkUnroll use SIMD registers Do not restrict SIMD registers only for memory that are 16B aligned. Motivation: https://godbolt.org/z/eb53xPvYT --- src/coreclr/jit/codegenarmarch.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 7fee192a419f27..54045cd3a6fa21 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -2674,7 +2674,7 @@ void CodeGen::genCodeForInitBlkUnroll(GenTreeBlk* node) // The following condition prevents using 16-byte stores when dstRegAddrAlignment is: // 1) unknown (i.e. dstReg is neither FP nor SP) or // 2) non-zero (i.e. dstRegAddr is not 16-byte aligned). - const bool hasAvailableSimdReg = isDstRegAddrAlignmentKnown && (size > FP_REGSIZE_BYTES); + const bool hasAvailableSimdReg = /*isDstRegAddrAlignmentKnown &&*/ (size > FP_REGSIZE_BYTES); const bool canUse16ByteWideInstrs = hasAvailableSimdReg && (dstRegAddrAlignment == 0) && helper.CanEncodeAllOffsets(FP_REGSIZE_BYTES); @@ -2914,8 +2914,8 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* node) // known and the block size is larger than a single SIMD register size (i.e. when using SIMD instructions can // be profitable). - const bool canUse16ByteWideInstrs = isSrcRegAddrAlignmentKnown && isDstRegAddrAlignmentKnown && - (size >= 2 * FP_REGSIZE_BYTES) && (srcRegAddrAlignment == dstRegAddrAlignment); + const bool canUse16ByteWideInstrs = /*isSrcRegAddrAlignmentKnown && isDstRegAddrAlignmentKnown &&*/ + (size >= 2 * FP_REGSIZE_BYTES) /*&& (srcRegAddrAlignment == dstRegAddrAlignment)*/; bool shouldUse16ByteWideInstrs = false; From c96bcf6feb9bc6b48940679af03e0af3d2745999 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 18 Apr 2022 10:17:37 -0700 Subject: [PATCH 2/4] also adjust the LSRA for SIMD registers --- src/coreclr/jit/lsraarmarch.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index efa0de76761aa7..af971db8b2cc77 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -212,7 +212,7 @@ int LinearScan::BuildCall(GenTreeCall* call) // the target. We do not handle these constraints on the same // refposition too well so we help ourselves a bit here by forcing the // null check with LR. - regMaskTP candidates = call->IsFastTailCall() ? RBM_LR : 0; + regMaskTP candidates = call->IsFastTailCall() ? RBM_LR : RBM_NONE; buildInternalIntRegisterDefForNode(call, candidates); } #endif // TARGET_ARM @@ -634,7 +634,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) const bool isDstRegAddrAlignmentKnown = dstAddr->OperIsLocalAddr(); - if (isDstRegAddrAlignmentKnown && (size > FP_REGSIZE_BYTES)) + if (/*isDstRegAddrAlignmentKnown && */(size > FP_REGSIZE_BYTES)) { // For larger block sizes CodeGen can choose to use 16-byte SIMD instructions. buildInternalFloatRegisterDefForNode(blkNode, internalFloatRegCandidates()); @@ -713,7 +713,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) // CodeGen can use 16-byte SIMD ldp/stp for larger block sizes // only when both source and destination base address registers have known alignment. // This is the case, when both registers are either sp or fp. - bool canUse16ByteWideInstrs = isSrcAddrLocal && isDstAddrLocal && (size >= 2 * FP_REGSIZE_BYTES); + bool canUse16ByteWideInstrs = /*isSrcAddrLocal && isDstAddrLocal &&*/ (size >= 2 * FP_REGSIZE_BYTES); // Note that the SIMD registers allocation is speculative - LSRA doesn't know at this point // whether CodeGen will use SIMD registers (i.e. if such instruction sequence will be more optimal). From 25a379121c42fc3a6a2c58d5c5297bf9b3998c74 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 18 Apr 2022 22:46:33 -0700 Subject: [PATCH 3/4] cleanup --- src/coreclr/jit/codegenarmarch.cpp | 22 +++------------------- src/coreclr/jit/lsraarmarch.cpp | 9 +++------ 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 54045cd3a6fa21..4332b1c758eb6c 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -2639,7 +2639,6 @@ void CodeGen::genCodeForInitBlkUnroll(GenTreeBlk* node) regNumber dstReg = dstAddrBaseReg; int dstRegAddrAlignment = 0; - bool isDstRegAddrAlignmentKnown = false; if (dstLclNum != BAD_VAR_NUM) { @@ -2648,7 +2647,6 @@ void CodeGen::genCodeForInitBlkUnroll(GenTreeBlk* node) dstReg = fpBased ? REG_FPBASE : REG_SPBASE; dstRegAddrAlignment = fpBased ? (genSPtoFPdelta() % 16) : 0; - isDstRegAddrAlignmentKnown = true; helper.SetDstOffset(baseAddr + dstOffset); } @@ -2670,11 +2668,7 @@ void CodeGen::genCodeForInitBlkUnroll(GenTreeBlk* node) bool shouldUse16ByteWideInstrs = false; - // Store operations that cross a 16-byte boundary reduce bandwidth or incur additional latency. - // The following condition prevents using 16-byte stores when dstRegAddrAlignment is: - // 1) unknown (i.e. dstReg is neither FP nor SP) or - // 2) non-zero (i.e. dstRegAddr is not 16-byte aligned). - const bool hasAvailableSimdReg = /*isDstRegAddrAlignmentKnown &&*/ (size > FP_REGSIZE_BYTES); + const bool hasAvailableSimdReg = (size > FP_REGSIZE_BYTES); const bool canUse16ByteWideInstrs = hasAvailableSimdReg && (dstRegAddrAlignment == 0) && helper.CanEncodeAllOffsets(FP_REGSIZE_BYTES); @@ -2825,10 +2819,7 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* node) #ifdef TARGET_ARM64 CopyBlockUnrollHelper helper(srcOffset, dstOffset, size); - regNumber srcReg = srcAddrBaseReg; - int srcRegAddrAlignment = 0; - bool isSrcRegAddrAlignmentKnown = false; if (srcLclNum != BAD_VAR_NUM) { @@ -2836,15 +2827,11 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* node) const int baseAddr = compiler->lvaFrameAddress(srcLclNum, &fpBased); srcReg = fpBased ? REG_FPBASE : REG_SPBASE; - srcRegAddrAlignment = fpBased ? (genSPtoFPdelta() % 16) : 0; - isSrcRegAddrAlignmentKnown = true; helper.SetSrcOffset(baseAddr + srcOffset); } - regNumber dstReg = dstAddrBaseReg; - int dstRegAddrAlignment = 0; - bool isDstRegAddrAlignmentKnown = false; + regNumber dstReg = dstAddrBaseReg; if (dstLclNum != BAD_VAR_NUM) { @@ -2852,8 +2839,6 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* node) const int baseAddr = compiler->lvaFrameAddress(dstLclNum, &fpBased); dstReg = fpBased ? REG_FPBASE : REG_SPBASE; - dstRegAddrAlignment = fpBased ? (genSPtoFPdelta() % 16) : 0; - isDstRegAddrAlignmentKnown = true; helper.SetDstOffset(baseAddr + dstOffset); } @@ -2914,8 +2899,7 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* node) // known and the block size is larger than a single SIMD register size (i.e. when using SIMD instructions can // be profitable). - const bool canUse16ByteWideInstrs = /*isSrcRegAddrAlignmentKnown && isDstRegAddrAlignmentKnown &&*/ - (size >= 2 * FP_REGSIZE_BYTES) /*&& (srcRegAddrAlignment == dstRegAddrAlignment)*/; + const bool canUse16ByteWideInstrs = (size >= 2 * FP_REGSIZE_BYTES); bool shouldUse16ByteWideInstrs = false; diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index af971db8b2cc77..7ce916d0a305c6 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -632,9 +632,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) buildInternalIntRegisterDefForNode(blkNode); } - const bool isDstRegAddrAlignmentKnown = dstAddr->OperIsLocalAddr(); - - if (/*isDstRegAddrAlignmentKnown && */(size > FP_REGSIZE_BYTES)) + if (size > FP_REGSIZE_BYTES) { // For larger block sizes CodeGen can choose to use 16-byte SIMD instructions. buildInternalFloatRegisterDefForNode(blkNode, internalFloatRegCandidates()); @@ -710,10 +708,9 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) ((srcAddrOrFill != nullptr) && srcAddrOrFill->OperIsLocalAddr()); const bool isDstAddrLocal = dstAddr->OperIsLocalAddr(); - // CodeGen can use 16-byte SIMD ldp/stp for larger block sizes - // only when both source and destination base address registers have known alignment. + // CodeGen can use 16-byte SIMD ldp/stp for larger block sizes. // This is the case, when both registers are either sp or fp. - bool canUse16ByteWideInstrs = /*isSrcAddrLocal && isDstAddrLocal &&*/ (size >= 2 * FP_REGSIZE_BYTES); + bool canUse16ByteWideInstrs = (size >= 2 * FP_REGSIZE_BYTES); // Note that the SIMD registers allocation is speculative - LSRA doesn't know at this point // whether CodeGen will use SIMD registers (i.e. if such instruction sequence will be more optimal). From a678a02a1b7edf0cb28b4075914d88617115e61e Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 18 Apr 2022 22:48:02 -0700 Subject: [PATCH 4/4] jit formatting --- src/coreclr/jit/codegenarmarch.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 4332b1c758eb6c..9b621a1a816d1c 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -2637,16 +2637,16 @@ void CodeGen::genCodeForInitBlkUnroll(GenTreeBlk* node) srcReg = REG_ZR; } - regNumber dstReg = dstAddrBaseReg; - int dstRegAddrAlignment = 0; + regNumber dstReg = dstAddrBaseReg; + int dstRegAddrAlignment = 0; if (dstLclNum != BAD_VAR_NUM) { bool fpBased; const int baseAddr = compiler->lvaFrameAddress(dstLclNum, &fpBased); - dstReg = fpBased ? REG_FPBASE : REG_SPBASE; - dstRegAddrAlignment = fpBased ? (genSPtoFPdelta() % 16) : 0; + dstReg = fpBased ? REG_FPBASE : REG_SPBASE; + dstRegAddrAlignment = fpBased ? (genSPtoFPdelta() % 16) : 0; helper.SetDstOffset(baseAddr + dstOffset); } @@ -2819,14 +2819,14 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* node) #ifdef TARGET_ARM64 CopyBlockUnrollHelper helper(srcOffset, dstOffset, size); - regNumber srcReg = srcAddrBaseReg; + regNumber srcReg = srcAddrBaseReg; if (srcLclNum != BAD_VAR_NUM) { bool fpBased; const int baseAddr = compiler->lvaFrameAddress(srcLclNum, &fpBased); - srcReg = fpBased ? REG_FPBASE : REG_SPBASE; + srcReg = fpBased ? REG_FPBASE : REG_SPBASE; helper.SetSrcOffset(baseAddr + srcOffset); } @@ -2838,7 +2838,7 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* node) bool fpBased; const int baseAddr = compiler->lvaFrameAddress(dstLclNum, &fpBased); - dstReg = fpBased ? REG_FPBASE : REG_SPBASE; + dstReg = fpBased ? REG_FPBASE : REG_SPBASE; helper.SetDstOffset(baseAddr + dstOffset); }