From 5d33b4910285285deaa80217de73cb83cdf474d6 Mon Sep 17 00:00:00 2001 From: Prashanth Govindarajan Date: Thu, 30 Jul 2020 18:09:34 -0700 Subject: [PATCH 01/14] Arm64 intrinsics for GetIndexOfFirstNonAsciiByte --- .../src/System/Text/ASCIIUtility.cs | 134 +++++++++++++++--- 1 file changed, 115 insertions(+), 19 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index 76075a5e66dc48..d03a2f75d82685 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -42,6 +42,28 @@ private static bool AllCharsInUInt64AreAscii(ulong value) return (value & ~0x007F007F_007F007Ful) == 0; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int GetIndexOfFirstNonAsciiByteInLane(Vector128 value, Vector128 bitmask) + { + if (!AdvSimd.Arm64.IsSupported || !BitConverter.IsLittleEndian) + { + throw new PlatformNotSupportedException(); + } + + // extractedBits[i] = (value[i] >> 7) & (1 << (12 * (i % 2))); + Vector128 mostSignificantBitIsSet = AdvSimd.ShiftRightArithmetic(value.AsSByte(), 7).AsByte(); + Vector128 extractedBits = AdvSimd.And(mostSignificantBitIsSet, bitmask); + + // collapse mask to lower bits + extractedBits = AdvSimd.Arm64.AddPairwise(extractedBits, extractedBits); + ulong mask = extractedBits.AsUInt64().ToScalar(); + + // calculate the index + int index = BitOperations.TrailingZeroCount(mask) >> 2; + Debug.Assert((mask != 0) ? index < 16 : index >= 16); + return index; + } + /// /// Given a DWORD which represents two packed chars in machine-endian order, /// iff the first char (in machine-endian order) is ASCII. @@ -67,8 +89,8 @@ public static unsafe nuint GetIndexOfFirstNonAsciiByte(byte* pBuffer, nuint buff // pmovmskb which we know are optimized, and (b) we can avoid downclocking the processor while // this method is running. - return (Sse2.IsSupported) - ? GetIndexOfFirstNonAsciiByte_Sse2(pBuffer, bufferLength) + return (Sse2.IsSupported || AdvSimd.Arm64.IsSupported && BitConverter.IsLittleEndian) + ? GetIndexOfFirstNonAsciiByte_Sse2OrArm64(pBuffer, bufferLength) : GetIndexOfFirstNonAsciiByte_Default(pBuffer, bufferLength); } @@ -215,15 +237,29 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Default(byte* pBuffer, n goto Finish; } - private static unsafe nuint GetIndexOfFirstNonAsciiByte_Sse2(byte* pBuffer, nuint bufferLength) + private static bool ContainsNonAsciiByte(uint currentMask) + { + if (Sse2.IsSupported) + { + return currentMask != 0; + } + else + { + return currentMask < 16; + } + } + + private static unsafe nuint GetIndexOfFirstNonAsciiByte_Sse2OrArm64(byte* pBuffer, nuint bufferLength) { // JIT turns the below into constants uint SizeOfVector128 = (uint)Unsafe.SizeOf>(); nuint MaskOfAllBitsInVector128 = (nuint)(SizeOfVector128 - 1); - Debug.Assert(Sse2.IsSupported, "Should've been checked by caller."); - Debug.Assert(BitConverter.IsLittleEndian, "SSE2 assumes little-endian."); + Debug.Assert(Sse2.IsSupported || AdvSimd.Arm64.IsSupported, "Sse2 or AdvSimd64 required."); + Debug.Assert(BitConverter.IsLittleEndian, "This SSE2/Arm64 implementation assumes little-endian."); + + Vector128 bitmask = Vector128.Create((ushort)0x1001).AsByte(); uint currentMask, secondMask; byte* pOriginalBuffer = pBuffer; @@ -240,9 +276,20 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Sse2(byte* pBuffer, nuin // Read the first vector unaligned. - currentMask = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load + if (Sse2.IsSupported) + { + currentMask = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load + } + else if (AdvSimd.Arm64.IsSupported) + { + currentMask = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load + } + else + { + throw new PlatformNotSupportedException(); + } - if (currentMask != 0) + if (ContainsNonAsciiByte(currentMask)) { goto FoundNonAsciiDataInCurrentMask; } @@ -281,13 +328,28 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Sse2(byte* pBuffer, nuin do { - Vector128 firstVector = Sse2.LoadAlignedVector128(pBuffer); - Vector128 secondVector = Sse2.LoadAlignedVector128(pBuffer + SizeOfVector128); + if (Sse2.IsSupported) + { + Vector128 firstVector = Sse2.LoadAlignedVector128(pBuffer); + Vector128 secondVector = Sse2.LoadAlignedVector128(pBuffer + SizeOfVector128); - currentMask = (uint)Sse2.MoveMask(firstVector); - secondMask = (uint)Sse2.MoveMask(secondVector); + currentMask = (uint)Sse2.MoveMask(firstVector); + secondMask = (uint)Sse2.MoveMask(secondVector); + } + else if (AdvSimd.Arm64.IsSupported) + { + Vector128 firstVector = AdvSimd.LoadVector128(pBuffer); + Vector128 secondVector = AdvSimd.LoadVector128(pBuffer + SizeOfVector128); - if ((currentMask | secondMask) != 0) + currentMask = (uint)GetIndexOfFirstNonAsciiByteInLane(firstVector, bitmask); + secondMask = (uint)GetIndexOfFirstNonAsciiByteInLane(secondVector, bitmask); + } + else + { + throw new PlatformNotSupportedException(); + } + + if (ContainsNonAsciiByte(currentMask | secondMask)) { goto FoundNonAsciiDataInInnerLoop; } @@ -313,8 +375,20 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Sse2(byte* pBuffer, nuin // At least one full vector's worth of data remains, so we can safely read it. // Remember, at this point pBuffer is still aligned. - currentMask = (uint)Sse2.MoveMask(Sse2.LoadAlignedVector128(pBuffer)); - if (currentMask != 0) + if (Sse2.IsSupported) + { + currentMask = (uint)Sse2.MoveMask(Sse2.LoadAlignedVector128(pBuffer)); + } + else if (AdvSimd.Arm64.IsSupported) + { + currentMask = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); + } + else + { + throw new PlatformNotSupportedException(); + } + + if (ContainsNonAsciiByte(currentMask)) { goto FoundNonAsciiDataInCurrentMask; } @@ -332,8 +406,20 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Sse2(byte* pBuffer, nuin pBuffer += (bufferLength & MaskOfAllBitsInVector128) - SizeOfVector128; - currentMask = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load - if (currentMask != 0) + if (Sse2.IsSupported) + { + currentMask = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load + } + else if (AdvSimd.Arm64.IsSupported) + { + currentMask = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load + } + else + { + throw new PlatformNotSupportedException(); + } + + if (ContainsNonAsciiByte(currentMask)) { goto FoundNonAsciiDataInCurrentMask; } @@ -342,7 +428,6 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Sse2(byte* pBuffer, nuin } Finish: - return (nuint)pBuffer - (nuint)pOriginalBuffer; // and we're done! FoundNonAsciiDataInInnerLoop: @@ -351,7 +436,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Sse2(byte* pBuffer, nuin // instead be the second mask. If so, skip the entire first mask and drain ASCII bytes // from the second mask. - if (currentMask == 0) + if (!ContainsNonAsciiByte(currentMask)) { pBuffer += SizeOfVector128; currentMask = secondMask; @@ -364,7 +449,18 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Sse2(byte* pBuffer, nuin // available, we'll fall back to a normal loop. Debug.Assert(currentMask != 0, "Shouldn't be here unless we see non-ASCII data."); - pBuffer += (uint)BitOperations.TrailingZeroCount(currentMask); + if (Sse2.IsSupported) + { + pBuffer += (uint)BitOperations.TrailingZeroCount(currentMask); + } + else if (AdvSimd.Arm64.IsSupported) + { + pBuffer += currentMask; + } + else + { + throw new PlatformNotSupportedException(); + } goto Finish; From 9d4182cb9f0624d5d0190c000c2972565cd6f818 Mon Sep 17 00:00:00 2001 From: Prashanth Govindarajan Date: Thu, 30 Jul 2020 19:35:37 -0700 Subject: [PATCH 02/14] sq --- .../src/System/Text/ASCIIUtility.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index d03a2f75d82685..916765d35c50fe 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -90,7 +90,7 @@ public static unsafe nuint GetIndexOfFirstNonAsciiByte(byte* pBuffer, nuint buff // this method is running. return (Sse2.IsSupported || AdvSimd.Arm64.IsSupported && BitConverter.IsLittleEndian) - ? GetIndexOfFirstNonAsciiByte_Sse2OrArm64(pBuffer, bufferLength) + ? GetIndexOfFirstNonAsciiByte_Intrinsified(pBuffer, bufferLength) : GetIndexOfFirstNonAsciiByte_Default(pBuffer, bufferLength); } @@ -249,7 +249,7 @@ private static bool ContainsNonAsciiByte(uint currentMask) } } - private static unsafe nuint GetIndexOfFirstNonAsciiByte_Sse2OrArm64(byte* pBuffer, nuint bufferLength) + private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuffer, nuint bufferLength) { // JIT turns the below into constants @@ -259,7 +259,9 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Sse2OrArm64(byte* pBuffe Debug.Assert(Sse2.IsSupported || AdvSimd.Arm64.IsSupported, "Sse2 or AdvSimd64 required."); Debug.Assert(BitConverter.IsLittleEndian, "This SSE2/Arm64 implementation assumes little-endian."); - Vector128 bitmask = Vector128.Create((ushort)0x1001).AsByte(); + Vector128 bitmask = BitConverter.IsLittleEndian ? + Vector128.Create((ushort)0x1001).AsByte() : + Vector128.Create((ushort)0x0110).AsByte(); uint currentMask, secondMask; byte* pOriginalBuffer = pBuffer; From d22d29c09a09d715027c366e77f66d7d1aaa964f Mon Sep 17 00:00:00 2001 From: Prashanth Govindarajan Date: Thu, 30 Jul 2020 19:50:09 -0700 Subject: [PATCH 03/14] sq --- .../System.Private.CoreLib/src/System/Text/ASCIIUtility.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index 916765d35c50fe..54d29d054f2748 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -450,7 +450,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff // Tzcnt is the correct operation to count the number of zero bits quickly. If this instruction isn't // available, we'll fall back to a normal loop. - Debug.Assert(currentMask != 0, "Shouldn't be here unless we see non-ASCII data."); + Debug.Assert(ContainsNonAsciiByte(currentMask), "Shouldn't be here unless we see non-ASCII data."); if (Sse2.IsSupported) { pBuffer += (uint)BitOperations.TrailingZeroCount(currentMask); From 3dca0007baadd70e8ebeab13497e7e6be38eb75b Mon Sep 17 00:00:00 2001 From: Prashanth Govindarajan Date: Thu, 30 Jul 2020 20:37:27 -0700 Subject: [PATCH 04/14] sq --- .../System.Private.CoreLib/src/System/Text/ASCIIUtility.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index 54d29d054f2748..e8b33cbe435dc9 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -43,7 +43,7 @@ private static bool AllCharsInUInt64AreAscii(ulong value) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetIndexOfFirstNonAsciiByteInLane(Vector128 value, Vector128 bitmask) + private static int GetIndexOfFirstNonAsciiByteInLane(Vector128 value, Vector128 bitmask) { if (!AdvSimd.Arm64.IsSupported || !BitConverter.IsLittleEndian) { From 49d87f0ecaebc34b1e1aab543b66db85537753f4 Mon Sep 17 00:00:00 2001 From: Prashanth Govindarajan Date: Thu, 30 Jul 2020 20:41:34 -0700 Subject: [PATCH 05/14] sq --- .../System.Private.CoreLib/src/System/Text/ASCIIUtility.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index e8b33cbe435dc9..6b782e74f6fdac 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -243,10 +243,14 @@ private static bool ContainsNonAsciiByte(uint currentMask) { return currentMask != 0; } - else + else if (AdvSimd.IsSupported) { return currentMask < 16; } + else + { + throw new PlatformNotSupportedException(); + } } private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuffer, nuint bufferLength) From 6d774192ce0c5fce80539704d95af21a29ea51e3 Mon Sep 17 00:00:00 2001 From: Prashanth Govindarajan Date: Thu, 30 Jul 2020 20:45:07 -0700 Subject: [PATCH 06/14] Last bit of formatting --- .../System.Private.CoreLib/src/System/Text/ASCIIUtility.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index 6b782e74f6fdac..d3038629d98dde 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -264,8 +264,8 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff Debug.Assert(BitConverter.IsLittleEndian, "This SSE2/Arm64 implementation assumes little-endian."); Vector128 bitmask = BitConverter.IsLittleEndian ? - Vector128.Create((ushort)0x1001).AsByte() : - Vector128.Create((ushort)0x0110).AsByte(); + Vector128.Create((ushort)0x1001).AsByte() : + Vector128.Create((ushort)0x0110).AsByte(); uint currentMask, secondMask; byte* pOriginalBuffer = pBuffer; From 8f259e649c178f83eb7a99a91efc5fffb0e16fb4 Mon Sep 17 00:00:00 2001 From: Prashanth Govindarajan Date: Fri, 31 Jul 2020 15:09:43 -0700 Subject: [PATCH 07/14] Fix bug --- .../System.Private.CoreLib/src/System/Text/ASCIIUtility.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index d3038629d98dde..1e285a0daf1e5e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -355,7 +355,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff throw new PlatformNotSupportedException(); } - if (ContainsNonAsciiByte(currentMask | secondMask)) + if (ContainsNonAsciiByte(currentMask) || ContainsNonAsciiByte(secondMask)) { goto FoundNonAsciiDataInInnerLoop; } From 9dd727c48ccba5bf335f330eb7962bb63e833eee Mon Sep 17 00:00:00 2001 From: Prashanth Govindarajan Date: Fri, 31 Jul 2020 15:58:40 -0700 Subject: [PATCH 08/14] Better naming --- .../src/System/Text/ASCIIUtility.cs | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index 1e285a0daf1e5e..6ec4f071dbea42 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -237,15 +237,15 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Default(byte* pBuffer, n goto Finish; } - private static bool ContainsNonAsciiByte(uint currentMask) + private static bool ContainsNonAsciiByte(uint currentSseMaskOrAdvSimdIndex) { if (Sse2.IsSupported) { - return currentMask != 0; + return currentSseMaskOrAdvSimdIndex != 0; } else if (AdvSimd.IsSupported) { - return currentMask < 16; + return currentSseMaskOrAdvSimdIndex < 16; } else { @@ -267,7 +267,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff Vector128.Create((ushort)0x1001).AsByte() : Vector128.Create((ushort)0x0110).AsByte(); - uint currentMask, secondMask; + uint currentSseMaskOrAdvSimdIndex, secondSseMaskOrAdvSimdIndex; byte* pOriginalBuffer = pBuffer; // This method is written such that control generally flows top-to-bottom, avoiding @@ -284,18 +284,18 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff if (Sse2.IsSupported) { - currentMask = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load + currentSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load } else if (AdvSimd.Arm64.IsSupported) { - currentMask = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load + currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load } else { throw new PlatformNotSupportedException(); } - if (ContainsNonAsciiByte(currentMask)) + if (ContainsNonAsciiByte(currentSseMaskOrAdvSimdIndex)) { goto FoundNonAsciiDataInCurrentMask; } @@ -339,23 +339,23 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff Vector128 firstVector = Sse2.LoadAlignedVector128(pBuffer); Vector128 secondVector = Sse2.LoadAlignedVector128(pBuffer + SizeOfVector128); - currentMask = (uint)Sse2.MoveMask(firstVector); - secondMask = (uint)Sse2.MoveMask(secondVector); + currentSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(firstVector); + secondSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(secondVector); } else if (AdvSimd.Arm64.IsSupported) { Vector128 firstVector = AdvSimd.LoadVector128(pBuffer); Vector128 secondVector = AdvSimd.LoadVector128(pBuffer + SizeOfVector128); - currentMask = (uint)GetIndexOfFirstNonAsciiByteInLane(firstVector, bitmask); - secondMask = (uint)GetIndexOfFirstNonAsciiByteInLane(secondVector, bitmask); + currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(firstVector, bitmask); + secondSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(secondVector, bitmask); } else { throw new PlatformNotSupportedException(); } - if (ContainsNonAsciiByte(currentMask) || ContainsNonAsciiByte(secondMask)) + if (ContainsNonAsciiByte(currentSseMaskOrAdvSimdIndex) || ContainsNonAsciiByte(secondSseMaskOrAdvSimdIndex)) { goto FoundNonAsciiDataInInnerLoop; } @@ -383,18 +383,18 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff if (Sse2.IsSupported) { - currentMask = (uint)Sse2.MoveMask(Sse2.LoadAlignedVector128(pBuffer)); + currentSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(Sse2.LoadAlignedVector128(pBuffer)); } else if (AdvSimd.Arm64.IsSupported) { - currentMask = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); + currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); } else { throw new PlatformNotSupportedException(); } - if (ContainsNonAsciiByte(currentMask)) + if (ContainsNonAsciiByte(currentSseMaskOrAdvSimdIndex)) { goto FoundNonAsciiDataInCurrentMask; } @@ -414,18 +414,18 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff if (Sse2.IsSupported) { - currentMask = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load + currentSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load } else if (AdvSimd.Arm64.IsSupported) { - currentMask = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load + currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load } else { throw new PlatformNotSupportedException(); } - if (ContainsNonAsciiByte(currentMask)) + if (ContainsNonAsciiByte(currentSseMaskOrAdvSimdIndex)) { goto FoundNonAsciiDataInCurrentMask; } @@ -442,10 +442,10 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff // instead be the second mask. If so, skip the entire first mask and drain ASCII bytes // from the second mask. - if (!ContainsNonAsciiByte(currentMask)) + if (!ContainsNonAsciiByte(currentSseMaskOrAdvSimdIndex)) { pBuffer += SizeOfVector128; - currentMask = secondMask; + currentSseMaskOrAdvSimdIndex = secondSseMaskOrAdvSimdIndex; } FoundNonAsciiDataInCurrentMask: @@ -454,14 +454,14 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff // Tzcnt is the correct operation to count the number of zero bits quickly. If this instruction isn't // available, we'll fall back to a normal loop. - Debug.Assert(ContainsNonAsciiByte(currentMask), "Shouldn't be here unless we see non-ASCII data."); + Debug.Assert(ContainsNonAsciiByte(currentSseMaskOrAdvSimdIndex), "Shouldn't be here unless we see non-ASCII data."); if (Sse2.IsSupported) { - pBuffer += (uint)BitOperations.TrailingZeroCount(currentMask); + pBuffer += (uint)BitOperations.TrailingZeroCount(currentSseMaskOrAdvSimdIndex); } else if (AdvSimd.Arm64.IsSupported) { - pBuffer += currentMask; + pBuffer += currentSseMaskOrAdvSimdIndex; } else { From b8014d943a46a7fb5a207fcea2e6fb5c5566e8ab Mon Sep 17 00:00:00 2001 From: Prashanth Govindarajan Date: Thu, 6 Aug 2020 11:22:51 -0700 Subject: [PATCH 09/14] Compiler complains --- .../src/System/Text/ASCIIUtility.cs | 105 +++++++++++------- 1 file changed, 67 insertions(+), 38 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index 6ec4f071dbea42..2cbd7456631cd0 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -267,7 +267,8 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff Vector128.Create((ushort)0x1001).AsByte() : Vector128.Create((ushort)0x0110).AsByte(); - uint currentSseMaskOrAdvSimdIndex, secondSseMaskOrAdvSimdIndex; + uint currentSseMask, secondSseMask; + uint currentAdvSimdIndex, secondAdvSimdIndex; byte* pOriginalBuffer = pBuffer; // This method is written such that control generally flows top-to-bottom, avoiding @@ -284,22 +285,25 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff if (Sse2.IsSupported) { - currentSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load + currentSseMask = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load + if (ContainsNonAsciiByte(currentSseMask)) + { + goto FoundNonAsciiDataInCurrentMask; + } } else if (AdvSimd.Arm64.IsSupported) { - currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load + currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load + if (ContainsNonAsciiByte(currentAdvSimdIndex)) + { + goto FoundNonAsciiDataInCurrentMask; + } } else { throw new PlatformNotSupportedException(); } - if (ContainsNonAsciiByte(currentSseMaskOrAdvSimdIndex)) - { - goto FoundNonAsciiDataInCurrentMask; - } - // If we have less than 32 bytes to process, just go straight to the final unaligned // read. There's no need to mess with the loop logic in the middle of this method. @@ -339,27 +343,30 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff Vector128 firstVector = Sse2.LoadAlignedVector128(pBuffer); Vector128 secondVector = Sse2.LoadAlignedVector128(pBuffer + SizeOfVector128); - currentSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(firstVector); - secondSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(secondVector); + currentSseMask = (uint)Sse2.MoveMask(firstVector); + secondSseMask = (uint)Sse2.MoveMask(secondVector); + if (ContainsNonAsciiByte(currentSseMask | secondSseMask)) + { + goto FoundNonAsciiDataInInnerLoop; + } } else if (AdvSimd.Arm64.IsSupported) { Vector128 firstVector = AdvSimd.LoadVector128(pBuffer); Vector128 secondVector = AdvSimd.LoadVector128(pBuffer + SizeOfVector128); - currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(firstVector, bitmask); - secondSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(secondVector, bitmask); + currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(firstVector, bitmask); + secondAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(secondVector, bitmask); + if (ContainsNonAsciiByte(currentAdvSimdIndex) || ContainsNonAsciiByte(secondAdvSimdIndex)) + { + goto FoundNonAsciiDataInInnerLoop; + } } else { throw new PlatformNotSupportedException(); } - if (ContainsNonAsciiByte(currentSseMaskOrAdvSimdIndex) || ContainsNonAsciiByte(secondSseMaskOrAdvSimdIndex)) - { - goto FoundNonAsciiDataInInnerLoop; - } - pBuffer += 2 * SizeOfVector128; } while (pBuffer <= pFinalVectorReadPos); } @@ -383,22 +390,25 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff if (Sse2.IsSupported) { - currentSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(Sse2.LoadAlignedVector128(pBuffer)); + currentSseMask = (uint)Sse2.MoveMask(Sse2.LoadAlignedVector128(pBuffer)); + if (ContainsNonAsciiByte(currentSseMask)) + { + goto FoundNonAsciiDataInCurrentMask; + } } else if (AdvSimd.Arm64.IsSupported) { - currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); + currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); + if (ContainsNonAsciiByte(currentAdvSimdIndex)) + { + goto FoundNonAsciiDataInCurrentMask; + } } else { throw new PlatformNotSupportedException(); } - if (ContainsNonAsciiByte(currentSseMaskOrAdvSimdIndex)) - { - goto FoundNonAsciiDataInCurrentMask; - } - IncrementCurrentOffsetBeforeFinalUnalignedVectorRead: pBuffer += SizeOfVector128; @@ -414,22 +424,27 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff if (Sse2.IsSupported) { - currentSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load + currentSseMask = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load + if (ContainsNonAsciiByte(currentSseMask)) + { + goto FoundNonAsciiDataInCurrentMask; + } + } else if (AdvSimd.Arm64.IsSupported) { - currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load + currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load + if (ContainsNonAsciiByte(currentAdvSimdIndex)) + { + goto FoundNonAsciiDataInCurrentMask; + } + } else { throw new PlatformNotSupportedException(); } - if (ContainsNonAsciiByte(currentSseMaskOrAdvSimdIndex)) - { - goto FoundNonAsciiDataInCurrentMask; - } - pBuffer += SizeOfVector128; } @@ -442,26 +457,40 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff // instead be the second mask. If so, skip the entire first mask and drain ASCII bytes // from the second mask. - if (!ContainsNonAsciiByte(currentSseMaskOrAdvSimdIndex)) + if (Sse2.IsSupported) { - pBuffer += SizeOfVector128; - currentSseMaskOrAdvSimdIndex = secondSseMaskOrAdvSimdIndex; + if (!ContainsNonAsciiByte(currentSseMask)) + { + pBuffer += SizeOfVector128; + currentSseMask = secondSseMask; + } + } + else if (AdvSimd.IsSupported) + { + if (!ContainsNonAsciiByte(currentAdvSimdIndex)) + { + pBuffer += SizeOfVector128; + currentAdvSimdIndex = secondAdvSimdIndex; + } + } + else + { + throw new PlatformNotSupportedException(); } - FoundNonAsciiDataInCurrentMask: // The mask contains - from the LSB - a 0 for each ASCII byte we saw, and a 1 for each non-ASCII byte. // Tzcnt is the correct operation to count the number of zero bits quickly. If this instruction isn't // available, we'll fall back to a normal loop. - Debug.Assert(ContainsNonAsciiByte(currentSseMaskOrAdvSimdIndex), "Shouldn't be here unless we see non-ASCII data."); + Debug.Assert(ContainsNonAsciiByte(currentSseMask), "Shouldn't be here unless we see non-ASCII data."); if (Sse2.IsSupported) { - pBuffer += (uint)BitOperations.TrailingZeroCount(currentSseMaskOrAdvSimdIndex); + pBuffer += (uint)BitOperations.TrailingZeroCount(currentSseMask); } else if (AdvSimd.Arm64.IsSupported) { - pBuffer += currentSseMaskOrAdvSimdIndex; + pBuffer += currentAdvSimdIndex; } else { From 0152f82beb5d644d7e5ecd102711d87f6408f141 Mon Sep 17 00:00:00 2001 From: Prashanth Govindarajan Date: Thu, 6 Aug 2020 16:18:11 -0700 Subject: [PATCH 10/14] Fix compile error --- .../src/System/Text/ASCIIUtility.cs | 74 ++++++++++--------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index 2cbd7456631cd0..a86c0433478b82 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -237,20 +237,24 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Default(byte* pBuffer, n goto Finish; } - private static bool ContainsNonAsciiByte(uint currentSseMaskOrAdvSimdIndex) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool ContainsNonAsciiByte_Sse2(uint sseMask) { - if (Sse2.IsSupported) - { - return currentSseMaskOrAdvSimdIndex != 0; - } - else if (AdvSimd.IsSupported) + if (!Sse2.IsSupported) { - return currentSseMaskOrAdvSimdIndex < 16; + throw new PlatformNotSupportedException(); } - else + return sseMask != 0; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool ContainsNonAsciiByte_AdvSimd(uint advSimdIndex) + { + if (!AdvSimd.IsSupported) { throw new PlatformNotSupportedException(); } + return advSimdIndex < 16; } private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuffer, nuint bufferLength) @@ -267,8 +271,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff Vector128.Create((ushort)0x1001).AsByte() : Vector128.Create((ushort)0x0110).AsByte(); - uint currentSseMask, secondSseMask; - uint currentAdvSimdIndex, secondAdvSimdIndex; + uint currentSseMaskOrAdvSimdIndex, secondSseMaskOrAdvSimdIndex; byte* pOriginalBuffer = pBuffer; // This method is written such that control generally flows top-to-bottom, avoiding @@ -285,16 +288,16 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff if (Sse2.IsSupported) { - currentSseMask = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load - if (ContainsNonAsciiByte(currentSseMask)) + currentSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load + if (ContainsNonAsciiByte_Sse2(currentSseMaskOrAdvSimdIndex)) { goto FoundNonAsciiDataInCurrentMask; } } else if (AdvSimd.Arm64.IsSupported) { - currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load - if (ContainsNonAsciiByte(currentAdvSimdIndex)) + currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load + if (ContainsNonAsciiByte_AdvSimd(currentSseMaskOrAdvSimdIndex)) { goto FoundNonAsciiDataInCurrentMask; } @@ -343,9 +346,9 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff Vector128 firstVector = Sse2.LoadAlignedVector128(pBuffer); Vector128 secondVector = Sse2.LoadAlignedVector128(pBuffer + SizeOfVector128); - currentSseMask = (uint)Sse2.MoveMask(firstVector); - secondSseMask = (uint)Sse2.MoveMask(secondVector); - if (ContainsNonAsciiByte(currentSseMask | secondSseMask)) + currentSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(firstVector); + secondSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(secondVector); + if (ContainsNonAsciiByte_Sse2(currentSseMaskOrAdvSimdIndex | secondSseMaskOrAdvSimdIndex)) { goto FoundNonAsciiDataInInnerLoop; } @@ -355,9 +358,9 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff Vector128 firstVector = AdvSimd.LoadVector128(pBuffer); Vector128 secondVector = AdvSimd.LoadVector128(pBuffer + SizeOfVector128); - currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(firstVector, bitmask); - secondAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(secondVector, bitmask); - if (ContainsNonAsciiByte(currentAdvSimdIndex) || ContainsNonAsciiByte(secondAdvSimdIndex)) + currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(firstVector, bitmask); + secondSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(secondVector, bitmask); + if (ContainsNonAsciiByte_AdvSimd(currentSseMaskOrAdvSimdIndex) || ContainsNonAsciiByte_AdvSimd(secondSseMaskOrAdvSimdIndex)) { goto FoundNonAsciiDataInInnerLoop; } @@ -390,16 +393,16 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff if (Sse2.IsSupported) { - currentSseMask = (uint)Sse2.MoveMask(Sse2.LoadAlignedVector128(pBuffer)); - if (ContainsNonAsciiByte(currentSseMask)) + currentSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(Sse2.LoadAlignedVector128(pBuffer)); + if (ContainsNonAsciiByte_Sse2(currentSseMaskOrAdvSimdIndex)) { goto FoundNonAsciiDataInCurrentMask; } } else if (AdvSimd.Arm64.IsSupported) { - currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); - if (ContainsNonAsciiByte(currentAdvSimdIndex)) + currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); + if (ContainsNonAsciiByte_AdvSimd(currentSseMaskOrAdvSimdIndex)) { goto FoundNonAsciiDataInCurrentMask; } @@ -424,8 +427,8 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff if (Sse2.IsSupported) { - currentSseMask = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load - if (ContainsNonAsciiByte(currentSseMask)) + currentSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load + if (ContainsNonAsciiByte_Sse2(currentSseMaskOrAdvSimdIndex)) { goto FoundNonAsciiDataInCurrentMask; } @@ -433,8 +436,8 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff } else if (AdvSimd.Arm64.IsSupported) { - currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load - if (ContainsNonAsciiByte(currentAdvSimdIndex)) + currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load + if (ContainsNonAsciiByte_AdvSimd(currentSseMaskOrAdvSimdIndex)) { goto FoundNonAsciiDataInCurrentMask; } @@ -459,18 +462,18 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff if (Sse2.IsSupported) { - if (!ContainsNonAsciiByte(currentSseMask)) + if (!ContainsNonAsciiByte_Sse2(currentSseMaskOrAdvSimdIndex)) { pBuffer += SizeOfVector128; - currentSseMask = secondSseMask; + currentSseMaskOrAdvSimdIndex = secondSseMaskOrAdvSimdIndex; } } else if (AdvSimd.IsSupported) { - if (!ContainsNonAsciiByte(currentAdvSimdIndex)) + if (!ContainsNonAsciiByte_AdvSimd(currentSseMaskOrAdvSimdIndex)) { pBuffer += SizeOfVector128; - currentAdvSimdIndex = secondAdvSimdIndex; + currentSseMaskOrAdvSimdIndex = secondSseMaskOrAdvSimdIndex; } } else @@ -483,14 +486,15 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff // Tzcnt is the correct operation to count the number of zero bits quickly. If this instruction isn't // available, we'll fall back to a normal loop. - Debug.Assert(ContainsNonAsciiByte(currentSseMask), "Shouldn't be here unless we see non-ASCII data."); if (Sse2.IsSupported) { - pBuffer += (uint)BitOperations.TrailingZeroCount(currentSseMask); + Debug.Assert(ContainsNonAsciiByte_Sse2(currentSseMaskOrAdvSimdIndex), "Shouldn't be here unless we see non-ASCII data."); + pBuffer += (uint)BitOperations.TrailingZeroCount(currentSseMaskOrAdvSimdIndex); } else if (AdvSimd.Arm64.IsSupported) { - pBuffer += currentAdvSimdIndex; + Debug.Assert(ContainsNonAsciiByte_AdvSimd(currentSseMaskOrAdvSimdIndex), "Shouldn't be here unless we see non-ASCII data."); + pBuffer += currentSseMaskOrAdvSimdIndex; } else { From 0370ec94e4844cc549de4e196de81b6096e11627 Mon Sep 17 00:00:00 2001 From: Prashanth Govindarajan Date: Mon, 10 Aug 2020 14:35:23 -0700 Subject: [PATCH 11/14] Address nit --- .../src/System/Text/ASCIIUtility.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index a86c0433478b82..ee51e7635b8668 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -43,7 +43,7 @@ private static bool AllCharsInUInt64AreAscii(ulong value) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetIndexOfFirstNonAsciiByteInLane(Vector128 value, Vector128 bitmask) + private static int GetIndexOfFirstNonAsciiByteInLane_AdvSimd(Vector128 value, Vector128 bitmask) { if (!AdvSimd.Arm64.IsSupported || !BitConverter.IsLittleEndian) { @@ -296,7 +296,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff } else if (AdvSimd.Arm64.IsSupported) { - currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load + currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load if (ContainsNonAsciiByte_AdvSimd(currentSseMaskOrAdvSimdIndex)) { goto FoundNonAsciiDataInCurrentMask; @@ -358,8 +358,8 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff Vector128 firstVector = AdvSimd.LoadVector128(pBuffer); Vector128 secondVector = AdvSimd.LoadVector128(pBuffer + SizeOfVector128); - currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(firstVector, bitmask); - secondSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(secondVector, bitmask); + currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(firstVector, bitmask); + secondSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(secondVector, bitmask); if (ContainsNonAsciiByte_AdvSimd(currentSseMaskOrAdvSimdIndex) || ContainsNonAsciiByte_AdvSimd(secondSseMaskOrAdvSimdIndex)) { goto FoundNonAsciiDataInInnerLoop; @@ -401,7 +401,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff } else if (AdvSimd.Arm64.IsSupported) { - currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); + currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(AdvSimd.LoadVector128(pBuffer), bitmask); if (ContainsNonAsciiByte_AdvSimd(currentSseMaskOrAdvSimdIndex)) { goto FoundNonAsciiDataInCurrentMask; @@ -436,7 +436,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff } else if (AdvSimd.Arm64.IsSupported) { - currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load + currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load if (ContainsNonAsciiByte_AdvSimd(currentSseMaskOrAdvSimdIndex)) { goto FoundNonAsciiDataInCurrentMask; From c3008eba4791fc848e7d4f7a1b958b2c0ef2ad77 Mon Sep 17 00:00:00 2001 From: Prashanth Govindarajan Date: Mon, 10 Aug 2020 16:20:29 -0700 Subject: [PATCH 12/14] Different variables --- .../src/System/Text/ASCIIUtility.cs | 57 ++++++++++--------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index ee51e7635b8668..837ffcdb9063aa 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -240,6 +240,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Default(byte* pBuffer, n [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool ContainsNonAsciiByte_Sse2(uint sseMask) { + Debug.Assert(sseMask != uint.MaxValue); if (!Sse2.IsSupported) { throw new PlatformNotSupportedException(); @@ -250,6 +251,7 @@ private static bool ContainsNonAsciiByte_Sse2(uint sseMask) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool ContainsNonAsciiByte_AdvSimd(uint advSimdIndex) { + Debug.Assert(advSimdIndex != uint.MaxValue); if (!AdvSimd.IsSupported) { throw new PlatformNotSupportedException(); @@ -271,7 +273,8 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff Vector128.Create((ushort)0x1001).AsByte() : Vector128.Create((ushort)0x0110).AsByte(); - uint currentSseMaskOrAdvSimdIndex, secondSseMaskOrAdvSimdIndex; + uint currentSseMask = uint.MaxValue, secondSseMask = uint.MaxValue; + uint currentAdvSimdIndex = uint.MaxValue, secondAdvSimdIndex = uint.MaxValue; byte* pOriginalBuffer = pBuffer; // This method is written such that control generally flows top-to-bottom, avoiding @@ -288,16 +291,16 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff if (Sse2.IsSupported) { - currentSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load - if (ContainsNonAsciiByte_Sse2(currentSseMaskOrAdvSimdIndex)) + currentSseMask = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load + if (ContainsNonAsciiByte_Sse2(currentSseMask)) { goto FoundNonAsciiDataInCurrentMask; } } else if (AdvSimd.Arm64.IsSupported) { - currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load - if (ContainsNonAsciiByte_AdvSimd(currentSseMaskOrAdvSimdIndex)) + currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load + if (ContainsNonAsciiByte_AdvSimd(currentAdvSimdIndex)) { goto FoundNonAsciiDataInCurrentMask; } @@ -346,9 +349,9 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff Vector128 firstVector = Sse2.LoadAlignedVector128(pBuffer); Vector128 secondVector = Sse2.LoadAlignedVector128(pBuffer + SizeOfVector128); - currentSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(firstVector); - secondSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(secondVector); - if (ContainsNonAsciiByte_Sse2(currentSseMaskOrAdvSimdIndex | secondSseMaskOrAdvSimdIndex)) + currentSseMask = (uint)Sse2.MoveMask(firstVector); + secondSseMask = (uint)Sse2.MoveMask(secondVector); + if (ContainsNonAsciiByte_Sse2(currentSseMask | secondSseMask)) { goto FoundNonAsciiDataInInnerLoop; } @@ -358,9 +361,9 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff Vector128 firstVector = AdvSimd.LoadVector128(pBuffer); Vector128 secondVector = AdvSimd.LoadVector128(pBuffer + SizeOfVector128); - currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(firstVector, bitmask); - secondSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(secondVector, bitmask); - if (ContainsNonAsciiByte_AdvSimd(currentSseMaskOrAdvSimdIndex) || ContainsNonAsciiByte_AdvSimd(secondSseMaskOrAdvSimdIndex)) + currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(firstVector, bitmask); + secondAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(secondVector, bitmask); + if (ContainsNonAsciiByte_AdvSimd(currentAdvSimdIndex) || ContainsNonAsciiByte_AdvSimd(secondAdvSimdIndex)) { goto FoundNonAsciiDataInInnerLoop; } @@ -393,16 +396,16 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff if (Sse2.IsSupported) { - currentSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(Sse2.LoadAlignedVector128(pBuffer)); - if (ContainsNonAsciiByte_Sse2(currentSseMaskOrAdvSimdIndex)) + currentSseMask = (uint)Sse2.MoveMask(Sse2.LoadAlignedVector128(pBuffer)); + if (ContainsNonAsciiByte_Sse2(currentSseMask)) { goto FoundNonAsciiDataInCurrentMask; } } else if (AdvSimd.Arm64.IsSupported) { - currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(AdvSimd.LoadVector128(pBuffer), bitmask); - if (ContainsNonAsciiByte_AdvSimd(currentSseMaskOrAdvSimdIndex)) + currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(AdvSimd.LoadVector128(pBuffer), bitmask); + if (ContainsNonAsciiByte_AdvSimd(currentAdvSimdIndex)) { goto FoundNonAsciiDataInCurrentMask; } @@ -427,8 +430,8 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff if (Sse2.IsSupported) { - currentSseMaskOrAdvSimdIndex = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load - if (ContainsNonAsciiByte_Sse2(currentSseMaskOrAdvSimdIndex)) + currentSseMask = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load + if (ContainsNonAsciiByte_Sse2(currentSseMask)) { goto FoundNonAsciiDataInCurrentMask; } @@ -436,8 +439,8 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff } else if (AdvSimd.Arm64.IsSupported) { - currentSseMaskOrAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load - if (ContainsNonAsciiByte_AdvSimd(currentSseMaskOrAdvSimdIndex)) + currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load + if (ContainsNonAsciiByte_AdvSimd(currentAdvSimdIndex)) { goto FoundNonAsciiDataInCurrentMask; } @@ -462,18 +465,18 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff if (Sse2.IsSupported) { - if (!ContainsNonAsciiByte_Sse2(currentSseMaskOrAdvSimdIndex)) + if (!ContainsNonAsciiByte_Sse2(currentSseMask)) { pBuffer += SizeOfVector128; - currentSseMaskOrAdvSimdIndex = secondSseMaskOrAdvSimdIndex; + currentSseMask = secondSseMask; } } else if (AdvSimd.IsSupported) { - if (!ContainsNonAsciiByte_AdvSimd(currentSseMaskOrAdvSimdIndex)) + if (!ContainsNonAsciiByte_AdvSimd(currentAdvSimdIndex)) { pBuffer += SizeOfVector128; - currentSseMaskOrAdvSimdIndex = secondSseMaskOrAdvSimdIndex; + currentAdvSimdIndex = secondSseMask; } } else @@ -488,13 +491,13 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff if (Sse2.IsSupported) { - Debug.Assert(ContainsNonAsciiByte_Sse2(currentSseMaskOrAdvSimdIndex), "Shouldn't be here unless we see non-ASCII data."); - pBuffer += (uint)BitOperations.TrailingZeroCount(currentSseMaskOrAdvSimdIndex); + Debug.Assert(ContainsNonAsciiByte_Sse2(currentSseMask), "Shouldn't be here unless we see non-ASCII data."); + pBuffer += (uint)BitOperations.TrailingZeroCount(currentSseMask); } else if (AdvSimd.Arm64.IsSupported) { - Debug.Assert(ContainsNonAsciiByte_AdvSimd(currentSseMaskOrAdvSimdIndex), "Shouldn't be here unless we see non-ASCII data."); - pBuffer += currentSseMaskOrAdvSimdIndex; + Debug.Assert(ContainsNonAsciiByte_AdvSimd(currentAdvSimdIndex), "Shouldn't be here unless we see non-ASCII data."); + pBuffer += currentAdvSimdIndex; } else { From 6c1288596dc61ddb349f8bc4c7d720ad6be1009d Mon Sep 17 00:00:00 2001 From: Prashanth Govindarajan Date: Mon, 10 Aug 2020 16:45:18 -0700 Subject: [PATCH 13/14] sq --- .../System.Private.CoreLib/src/System/Text/ASCIIUtility.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index 837ffcdb9063aa..748f8358807550 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -476,7 +476,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff if (!ContainsNonAsciiByte_AdvSimd(currentAdvSimdIndex)) { pBuffer += SizeOfVector128; - currentAdvSimdIndex = secondSseMask; + currentAdvSimdIndex = secondAdvSimdIndex; } } else From 576af36ccafe19615521291d9e36a78325bd1ee1 Mon Sep 17 00:00:00 2001 From: Prashanth Govindarajan Date: Wed, 12 Aug 2020 12:48:56 -0700 Subject: [PATCH 14/14] Nits --- .../src/System/Text/ASCIIUtility.cs | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index 748f8358807550..c1ad8b87781ab5 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -241,10 +241,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Default(byte* pBuffer, n private static bool ContainsNonAsciiByte_Sse2(uint sseMask) { Debug.Assert(sseMask != uint.MaxValue); - if (!Sse2.IsSupported) - { - throw new PlatformNotSupportedException(); - } + Debug.Assert(Sse2.IsSupported); return sseMask != 0; } @@ -252,10 +249,7 @@ private static bool ContainsNonAsciiByte_Sse2(uint sseMask) private static bool ContainsNonAsciiByte_AdvSimd(uint advSimdIndex) { Debug.Assert(advSimdIndex != uint.MaxValue); - if (!AdvSimd.IsSupported) - { - throw new PlatformNotSupportedException(); - } + Debug.Assert(AdvSimd.IsSupported); return advSimdIndex < 16; } @@ -294,7 +288,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff currentSseMask = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load if (ContainsNonAsciiByte_Sse2(currentSseMask)) { - goto FoundNonAsciiDataInCurrentMask; + goto FoundNonAsciiDataInCurrentChunk; } } else if (AdvSimd.Arm64.IsSupported) @@ -302,7 +296,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load if (ContainsNonAsciiByte_AdvSimd(currentAdvSimdIndex)) { - goto FoundNonAsciiDataInCurrentMask; + goto FoundNonAsciiDataInCurrentChunk; } } else @@ -399,7 +393,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff currentSseMask = (uint)Sse2.MoveMask(Sse2.LoadAlignedVector128(pBuffer)); if (ContainsNonAsciiByte_Sse2(currentSseMask)) { - goto FoundNonAsciiDataInCurrentMask; + goto FoundNonAsciiDataInCurrentChunk; } } else if (AdvSimd.Arm64.IsSupported) @@ -407,7 +401,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(AdvSimd.LoadVector128(pBuffer), bitmask); if (ContainsNonAsciiByte_AdvSimd(currentAdvSimdIndex)) { - goto FoundNonAsciiDataInCurrentMask; + goto FoundNonAsciiDataInCurrentChunk; } } else @@ -433,7 +427,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff currentSseMask = (uint)Sse2.MoveMask(Sse2.LoadVector128(pBuffer)); // unaligned load if (ContainsNonAsciiByte_Sse2(currentSseMask)) { - goto FoundNonAsciiDataInCurrentMask; + goto FoundNonAsciiDataInCurrentChunk; } } @@ -442,7 +436,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff currentAdvSimdIndex = (uint)GetIndexOfFirstNonAsciiByteInLane_AdvSimd(AdvSimd.LoadVector128(pBuffer), bitmask); // unaligned load if (ContainsNonAsciiByte_AdvSimd(currentAdvSimdIndex)) { - goto FoundNonAsciiDataInCurrentMask; + goto FoundNonAsciiDataInCurrentChunk; } } @@ -483,14 +477,14 @@ private static unsafe nuint GetIndexOfFirstNonAsciiByte_Intrinsified(byte* pBuff { throw new PlatformNotSupportedException(); } - FoundNonAsciiDataInCurrentMask: + FoundNonAsciiDataInCurrentChunk: - // The mask contains - from the LSB - a 0 for each ASCII byte we saw, and a 1 for each non-ASCII byte. - // Tzcnt is the correct operation to count the number of zero bits quickly. If this instruction isn't - // available, we'll fall back to a normal loop. if (Sse2.IsSupported) { + // The mask contains - from the LSB - a 0 for each ASCII byte we saw, and a 1 for each non-ASCII byte. + // Tzcnt is the correct operation to count the number of zero bits quickly. If this instruction isn't + // available, we'll fall back to a normal loop. Debug.Assert(ContainsNonAsciiByte_Sse2(currentSseMask), "Shouldn't be here unless we see non-ASCII data."); pBuffer += (uint)BitOperations.TrailingZeroCount(currentSseMask); }