diff --git a/src/libraries/System.Collections/src/System/Collections/BitArray.cs b/src/libraries/System.Collections/src/System/Collections/BitArray.cs index 1c6240f4311319..f368c91ace6a62 100644 --- a/src/libraries/System.Collections/src/System/Collections/BitArray.cs +++ b/src/libraries/System.Collections/src/System/Collections/BitArray.cs @@ -6,8 +6,6 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.X86; -using System.Runtime.Intrinsics.Arm; namespace System.Collections { @@ -118,10 +116,6 @@ public BitArray(byte[] bytes) _version = 0; } - private const uint Vector128ByteCount = 16; - private const uint Vector128IntCount = 4; - private const uint Vector256ByteCount = 32; - private const uint Vector256IntCount = 8; public unsafe BitArray(bool[] values) { ArgumentNullException.ThrowIfNull(values); @@ -144,7 +138,7 @@ public unsafe BitArray(bool[] values) if (Vector256.IsHardwareAccelerated) { - for (; (i + Vector256ByteCount) <= (uint)values.Length; i += Vector256ByteCount) + for (; (i + Vector256.Count) <= (uint)values.Length; i += (uint)Vector256.Count) { Vector256 vector = Vector256.LoadUnsafe(ref value, i); Vector256 isFalse = Vector256.Equals(vector, Vector256.Zero); @@ -155,13 +149,13 @@ public unsafe BitArray(bool[] values) } else if (Vector128.IsHardwareAccelerated) { - for (; (i + Vector128ByteCount * 2u) <= (uint)values.Length; i += Vector128ByteCount * 2u) + for (; (i + Vector128.Count * 2u) <= (uint)values.Length; i += (uint)Vector128.Count * 2u) { Vector128 lowerVector = Vector128.LoadUnsafe(ref value, i); Vector128 lowerIsFalse = Vector128.Equals(lowerVector, Vector128.Zero); uint lowerResult = lowerIsFalse.ExtractMostSignificantBits(); - Vector128 upperVector = Vector128.LoadUnsafe(ref value, i + Vector128ByteCount); + Vector128 upperVector = Vector128.LoadUnsafe(ref value, i + (uint)Vector128.Count); Vector128 upperIsFalse = Vector128.Equals(upperVector, Vector128.Zero); uint upperResult = upperIsFalse.ExtractMostSignificantBits(); @@ -345,7 +339,7 @@ public unsafe BitArray And(BitArray value) if (Vector256.IsHardwareAccelerated) { - for (; i < (uint)count - (Vector256IntCount - 1u); i += Vector256IntCount) + for (; i < (uint)count - (Vector256.Count - 1u); i += (uint)Vector256.Count) { Vector256 result = Vector256.LoadUnsafe(ref left, i) & Vector256.LoadUnsafe(ref right, i); result.StoreUnsafe(ref left, i); @@ -353,7 +347,7 @@ public unsafe BitArray And(BitArray value) } else if (Vector128.IsHardwareAccelerated) { - for (; i < (uint)count - (Vector128IntCount - 1u); i += Vector128IntCount) + for (; i < (uint)count - (Vector128.Count - 1u); i += (uint)Vector128.Count) { Vector128 result = Vector128.LoadUnsafe(ref left, i) & Vector128.LoadUnsafe(ref right, i); result.StoreUnsafe(ref left, i); @@ -411,7 +405,7 @@ public unsafe BitArray Or(BitArray value) if (Vector256.IsHardwareAccelerated) { - for (; i < (uint)count - (Vector256IntCount - 1u); i += Vector256IntCount) + for (; i < (uint)count - (Vector256.Count - 1u); i += (uint)Vector256.Count) { Vector256 result = Vector256.LoadUnsafe(ref left, i) | Vector256.LoadUnsafe(ref right, i); result.StoreUnsafe(ref left, i); @@ -419,7 +413,7 @@ public unsafe BitArray Or(BitArray value) } else if (Vector128.IsHardwareAccelerated) { - for (; i < (uint)count - (Vector128IntCount - 1u); i += Vector128IntCount) + for (; i < (uint)count - (Vector128.Count - 1u); i += (uint)Vector128.Count) { Vector128 result = Vector128.LoadUnsafe(ref left, i) | Vector128.LoadUnsafe(ref right, i); result.StoreUnsafe(ref left, i); @@ -477,7 +471,7 @@ public unsafe BitArray Xor(BitArray value) if (Vector256.IsHardwareAccelerated) { - for (; i < (uint)count - (Vector256IntCount - 1u); i += Vector256IntCount) + for (; i < (uint)count - (Vector256.Count - 1u); i += (uint)Vector256.Count) { Vector256 result = Vector256.LoadUnsafe(ref left, i) ^ Vector256.LoadUnsafe(ref right, i); result.StoreUnsafe(ref left, i); @@ -485,7 +479,7 @@ public unsafe BitArray Xor(BitArray value) } else if (Vector128.IsHardwareAccelerated) { - for (; i < (uint)count - (Vector128IntCount - 1u); i += Vector128IntCount) + for (; i < (uint)count - (Vector128.Count - 1u); i += (uint)Vector128.Count) { Vector128 result = Vector128.LoadUnsafe(ref left, i) ^ Vector128.LoadUnsafe(ref right, i); result.StoreUnsafe(ref left, i); @@ -535,7 +529,7 @@ public unsafe BitArray Not() if (Vector256.IsHardwareAccelerated) { - for (; i < (uint)count - (Vector256IntCount - 1u); i += Vector256IntCount) + for (; i < (uint)count - (Vector256.Count - 1u); i += (uint)Vector256.Count) { Vector256 result = ~Vector256.LoadUnsafe(ref value, i); result.StoreUnsafe(ref value, i); @@ -543,7 +537,7 @@ public unsafe BitArray Not() } else if (Vector128.IsHardwareAccelerated) { - for (; i < (uint)count - (Vector128IntCount - 1u); i += Vector128IntCount) + for (; i < (uint)count - (Vector128.Count - 1u); i += (uint)Vector128.Count) { Vector128 result = ~Vector128.LoadUnsafe(ref value, i); result.StoreUnsafe(ref value, i); @@ -806,115 +800,64 @@ public unsafe void CopyTo(Array array, int index) } uint i = 0; + int[] thisArray = m_array; + uint thisLength = (uint)m_length; - if (m_length < BitsPerInt32) + if (thisLength < BitsPerInt32) goto LessThan32; - // The mask used when shuffling a single int into Vector128/256. - // On little endian machines, the lower 8 bits of int belong in the first byte, next lower 8 in the second and so on. - // We place the bytes that contain the bits to its respective byte so that we can mask out only the relevant bits later. - Vector128 lowerShuffleMask_CopyToBoolArray = Vector128.Create(0, 0x01010101_01010101).AsByte(); - Vector128 upperShuffleMask_CopyToBoolArray = Vector128.Create(0x02020202_02020202, 0x03030303_03030303).AsByte(); - - if (Avx2.IsSupported) + if (Vector256.IsHardwareAccelerated) { - Vector256 shuffleMask = Vector256.Create(lowerShuffleMask_CopyToBoolArray, upperShuffleMask_CopyToBoolArray); Vector256 bitMask = Vector256.Create(0x80402010_08040201).AsByte(); Vector256 ones = Vector256.Create((byte)1); - fixed (bool* destination = &boolArray[index]) - { - for (; (i + Vector256ByteCount) <= (uint)m_length; i += Vector256ByteCount) - { - int bits = m_array[i / (uint)BitsPerInt32]; - Vector256 scalar = Vector256.Create(bits); - Vector256 shuffled = Avx2.Shuffle(scalar.AsByte(), shuffleMask); - Vector256 extracted = Avx2.And(shuffled, bitMask); - - // The extracted bits can be anywhere between 0 and 255, so we normalise the value to either 0 or 1 - // to ensure compatibility with "C# bool" (0 for false, 1 for true, rest undefined) - Vector256 normalized = Avx2.Min(extracted, ones); - Avx.Store((byte*)destination + i, normalized); - } - } - } - else if (Ssse3.IsSupported) - { - Vector128 lowerShuffleMask = lowerShuffleMask_CopyToBoolArray; - Vector128 upperShuffleMask = upperShuffleMask_CopyToBoolArray; - Vector128 ones = Vector128.Create((byte)1); - Vector128 bitMask128 = BitConverter.IsLittleEndian ? - Vector128.Create(0x80402010_08040201).AsByte() : - Vector128.Create(0x01020408_10204080).AsByte(); + ref byte destination = ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(boolArray), index)); - fixed (bool* destination = &boolArray[index]) + for (; i <= thisLength - Vector256.Count; i += (uint)Vector256.Count) { - for (; (i + Vector128ByteCount * 2u) <= (uint)m_length; i += Vector128ByteCount * 2u) - { - int bits = m_array[i / (uint)BitsPerInt32]; - Vector128 scalar = Vector128.CreateScalarUnsafe(bits); - - Vector128 shuffledLower = Ssse3.Shuffle(scalar.AsByte(), lowerShuffleMask); - Vector128 extractedLower = Sse2.And(shuffledLower, bitMask128); - Vector128 normalizedLower = Sse2.Min(extractedLower, ones); - Sse2.Store((byte*)destination + i, normalizedLower); - - Vector128 shuffledHigher = Ssse3.Shuffle(scalar.AsByte(), upperShuffleMask); - Vector128 extractedHigher = Sse2.And(shuffledHigher, bitMask128); - Vector128 normalizedHigher = Sse2.Min(extractedHigher, ones); - Sse2.Store((byte*)destination + i + Vector128.Count, normalizedHigher); - } + int bits = thisArray[i / (uint)BitsPerInt32]; + Vector256 scalar = Vector256.Create(bits); + Vector256 shuffled = Vector256.Shuffle(scalar.AsByte(), Vector256.Create(0x0000_0000_0000_0000, 0x0101_0101_0101_0101, 0x12121212_12121212, 0x13131313_13131313).AsByte()); + Vector256 extracted = shuffled & bitMask; + + // The extracted bits can be anywhere between 0 and 255, so we normalise the value to either 0 or 1 + // to ensure compatibility with "C# bool" (0 for false, 1 for true, rest undefined) + Vector256 normalized = Vector256.Min(extracted, ones); + normalized.StoreUnsafe(ref destination, i); } } - else if (AdvSimd.Arm64.IsSupported) + else if (Vector128.IsHardwareAccelerated) { Vector128 ones = Vector128.Create((byte)1); Vector128 bitMask128 = BitConverter.IsLittleEndian ? Vector128.Create(0x80402010_08040201).AsByte() : Vector128.Create(0x01020408_10204080).AsByte(); - fixed (bool* destination = &boolArray[index]) + ref byte destination = ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(boolArray), index)); + + for (; i <= thisLength - Vector128.Count * 2u; i += (uint)Vector128.Count * 2u) { - for (; (i + Vector128ByteCount * 2u) <= (uint)m_length; i += Vector128ByteCount * 2u) - { - int bits = m_array[i / (uint)BitsPerInt32]; - // Same logic as SSSE3 path, except we do not have Shuffle instruction. - // (TableVectorLookup could be an alternative - dotnet/runtime#1277) - // Instead we use chained ZIP1/2 instructions: - // (A0 is the byte containing LSB, A3 is the byte containing MSB) - // bits (on Big endian) - A3 A2 A1 A0 - // bits (Little endian) / Byte reversal - A0 A1 A2 A3 - // v1 = Vector128.Create - A0 A1 A2 A3 A0 A1 A2 A3 A0 A1 A2 A3 A0 A1 A2 A3 - // v2 = ZipLow(v1, v1) - A0 A0 A1 A1 A2 A2 A3 A3 A0 A0 A1 A1 A2 A2 A3 A3 - // v3 = ZipLow(v2, v2) - A0 A0 A0 A0 A1 A1 A1 A1 A2 A2 A2 A2 A3 A3 A3 A3 - // shuffledLower = ZipLow(v3, v3) - A0 A0 A0 A0 A0 A0 A0 A0 A1 A1 A1 A1 A1 A1 A1 A1 - // shuffledHigher = ZipHigh(v3, v3) - A2 A2 A2 A2 A2 A2 A2 A2 A3 A3 A3 A3 A3 A3 A3 A3 - if (!BitConverter.IsLittleEndian) - { - bits = BinaryPrimitives.ReverseEndianness(bits); - } - Vector128 vector = Vector128.Create(bits).AsByte(); - vector = AdvSimd.Arm64.ZipLow(vector, vector); - vector = AdvSimd.Arm64.ZipLow(vector, vector); - - Vector128 shuffledLower = AdvSimd.Arm64.ZipLow(vector, vector); - Vector128 extractedLower = AdvSimd.And(shuffledLower, bitMask128); - Vector128 normalizedLower = AdvSimd.Min(extractedLower, ones); - - Vector128 shuffledHigher = AdvSimd.Arm64.ZipHigh(vector, vector); - Vector128 extractedHigher = AdvSimd.And(shuffledHigher, bitMask128); - Vector128 normalizedHigher = AdvSimd.Min(extractedHigher, ones); - - AdvSimd.Arm64.StorePair((byte*)destination + i, normalizedLower, normalizedHigher); - } + int bits = thisArray[i / (uint)BitsPerInt32]; + Vector128 scalar = Vector128.CreateScalarUnsafe(bits); + + // they masks need to be consts for optimal codegen (see https://github.com/dotnet/runtime/issues/72793) + Vector128 shuffledLower = Vector128.Shuffle(scalar.AsByte(), Vector128.Create((byte)0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1)); + Vector128 extractedLower = shuffledLower & bitMask128; + Vector128 normalizedLower = Vector128.Min(extractedLower, ones); + normalizedLower.StoreUnsafe(ref destination, i); + + Vector128 shuffledHigher = Vector128.Shuffle(scalar.AsByte(), Vector128.Create((byte)2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3)); + Vector128 extractedHigher = shuffledHigher & bitMask128; + Vector128 normalizedHigher = Vector128.Min(extractedHigher, ones); + normalizedHigher.StoreUnsafe(ref destination, (nuint)(i + Vector128.Count)); } } LessThan32: - for (; i < (uint)m_length; i++) + for (; i < thisLength; i++) { int elementIndex = Div32Rem((int)i, out int extraBits); - boolArray[(uint)index + i] = ((m_array[elementIndex] >> extraBits) & 0x00000001) != 0; + boolArray[(uint)index + i] = ((thisArray[elementIndex] >> extraBits) & 0x00000001) != 0; } } else