Skip to content

port BitArray.CopyTo to xplat hardware intrinsics #72788

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
adamsitnik wants to merge 10 commits into from
Closed

port BitArray.CopyTo to xplat hardware intrinsics #72788

Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
15f6a0b
port BitArray.CopyTo to cross platform APIs
adamsitnik Jul 25, 2022
c64ed21
remove ARM code path
adamsitnik Jul 25, 2022
e402e8a
Merge remote-tracking branch 'upstream/main' into bitArrayCopyTo
adamsitnik Jul 25, 2022
1a345d4
remove unnecessary usings
adamsitnik Jul 25, 2022
918baf1
restore Avx2 code path as Vector256.Shuffle does not produce optimal …
adamsitnik Jul 26, 2022
7965d15
optimize Vector128 codepath to avoid regression
adamsitnik Jul 26, 2022
7d4757c
address code review feedback: use Vector128<int>.Count instead of a p…
adamsitnik Jul 26, 2022
2ea70af
address code review feedback: access array as local variable rather t…
adamsitnik Jul 26, 2022
5aca325
address code review feedback: optimize loop
adamsitnik Jul 26, 2022
bf41da6
use magic bitmask provided by Tanner and port Avx2 code to Vector256 …
adamsitnik Jul 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 45 additions & 102 deletions src/libraries/System.Collections/src/System/Collections/BitArray.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
using System.Runtime.Intrinsics.Arm;

namespace System.Collections
{
Expand Down Expand Up @@ -118,10 +116,6 @@ public BitArray(byte[] bytes)
_version = 0;
}

private const uint Vector128ByteCount = 16;
private const uint Vector128IntCount = 4;
private const uint Vector256ByteCount = 32;
private const uint Vector256IntCount = 8;
public unsafe BitArray(bool[] values)
{
ArgumentNullException.ThrowIfNull(values);
Expand All @@ -144,7 +138,7 @@ public unsafe BitArray(bool[] values)

if (Vector256.IsHardwareAccelerated)
{
for (; (i + Vector256ByteCount) <= (uint)values.Length; i += Vector256ByteCount)
for (; (i + Vector256<byte>.Count) <= (uint)values.Length; i += (uint)Vector256<byte>.Count)
Comment thread
adamsitnik marked this conversation as resolved.
{
Vector256<byte> vector = Vector256.LoadUnsafe(ref value, i);
Vector256<byte> isFalse = Vector256.Equals(vector, Vector256<byte>.Zero);
Expand All @@ -155,13 +149,13 @@ public unsafe BitArray(bool[] values)
}
else if (Vector128.IsHardwareAccelerated)
{
for (; (i + Vector128ByteCount * 2u) <= (uint)values.Length; i += Vector128ByteCount * 2u)
for (; (i + Vector128<byte>.Count * 2u) <= (uint)values.Length; i += (uint)Vector128<byte>.Count * 2u)
{
Vector128<byte> lowerVector = Vector128.LoadUnsafe(ref value, i);
Vector128<byte> lowerIsFalse = Vector128.Equals(lowerVector, Vector128<byte>.Zero);
uint lowerResult = lowerIsFalse.ExtractMostSignificantBits();

Vector128<byte> upperVector = Vector128.LoadUnsafe(ref value, i + Vector128ByteCount);
Vector128<byte> upperVector = Vector128.LoadUnsafe(ref value, i + (uint)Vector128<byte>.Count);
Vector128<byte> upperIsFalse = Vector128.Equals(upperVector, Vector128<byte>.Zero);
uint upperResult = upperIsFalse.ExtractMostSignificantBits();

Expand Down Expand Up @@ -345,15 +339,15 @@ public unsafe BitArray And(BitArray value)

if (Vector256.IsHardwareAccelerated)
{
for (; i < (uint)count - (Vector256IntCount - 1u); i += Vector256IntCount)
for (; i < (uint)count - (Vector256<int>.Count - 1u); i += (uint)Vector256<int>.Count)
{
Vector256<int> result = Vector256.LoadUnsafe(ref left, i) & Vector256.LoadUnsafe(ref right, i);
result.StoreUnsafe(ref left, i);
}
}
else if (Vector128.IsHardwareAccelerated)
{
for (; i < (uint)count - (Vector128IntCount - 1u); i += Vector128IntCount)
for (; i < (uint)count - (Vector128<int>.Count - 1u); i += (uint)Vector128<int>.Count)
{
Vector128<int> result = Vector128.LoadUnsafe(ref left, i) & Vector128.LoadUnsafe(ref right, i);
result.StoreUnsafe(ref left, i);
Expand Down Expand Up @@ -411,15 +405,15 @@ public unsafe BitArray Or(BitArray value)

if (Vector256.IsHardwareAccelerated)
{
for (; i < (uint)count - (Vector256IntCount - 1u); i += Vector256IntCount)
for (; i < (uint)count - (Vector256<int>.Count - 1u); i += (uint)Vector256<int>.Count)
{
Vector256<int> result = Vector256.LoadUnsafe(ref left, i) | Vector256.LoadUnsafe(ref right, i);
result.StoreUnsafe(ref left, i);
}
}
else if (Vector128.IsHardwareAccelerated)
{
for (; i < (uint)count - (Vector128IntCount - 1u); i += Vector128IntCount)
for (; i < (uint)count - (Vector128<int>.Count - 1u); i += (uint)Vector128<int>.Count)
{
Vector128<int> result = Vector128.LoadUnsafe(ref left, i) | Vector128.LoadUnsafe(ref right, i);
result.StoreUnsafe(ref left, i);
Expand Down Expand Up @@ -477,15 +471,15 @@ public unsafe BitArray Xor(BitArray value)

if (Vector256.IsHardwareAccelerated)
{
for (; i < (uint)count - (Vector256IntCount - 1u); i += Vector256IntCount)
for (; i < (uint)count - (Vector256<int>.Count - 1u); i += (uint)Vector256<int>.Count)
{
Vector256<int> result = Vector256.LoadUnsafe(ref left, i) ^ Vector256.LoadUnsafe(ref right, i);
result.StoreUnsafe(ref left, i);
}
}
else if (Vector128.IsHardwareAccelerated)
{
for (; i < (uint)count - (Vector128IntCount - 1u); i += Vector128IntCount)
for (; i < (uint)count - (Vector128<int>.Count - 1u); i += (uint)Vector128<int>.Count)
{
Vector128<int> result = Vector128.LoadUnsafe(ref left, i) ^ Vector128.LoadUnsafe(ref right, i);
result.StoreUnsafe(ref left, i);
Expand Down Expand Up @@ -535,15 +529,15 @@ public unsafe BitArray Not()

if (Vector256.IsHardwareAccelerated)
{
for (; i < (uint)count - (Vector256IntCount - 1u); i += Vector256IntCount)
for (; i < (uint)count - (Vector256<int>.Count - 1u); i += (uint)Vector256<int>.Count)
{
Vector256<int> result = ~Vector256.LoadUnsafe(ref value, i);
result.StoreUnsafe(ref value, i);
}
}
else if (Vector128.IsHardwareAccelerated)
{
for (; i < (uint)count - (Vector128IntCount - 1u); i += Vector128IntCount)
for (; i < (uint)count - (Vector128<int>.Count - 1u); i += (uint)Vector128<int>.Count)
{
Vector128<int> result = ~Vector128.LoadUnsafe(ref value, i);
result.StoreUnsafe(ref value, i);
Expand Down Expand Up @@ -806,115 +800,64 @@ public unsafe void CopyTo(Array array, int index)
}

uint i = 0;
int[] thisArray = m_array;
uint thisLength = (uint)m_length;

if (m_length < BitsPerInt32)
if (thisLength < BitsPerInt32)
goto LessThan32;

// The mask used when shuffling a single int into Vector128/256.
// On little endian machines, the lower 8 bits of int belong in the first byte, next lower 8 in the second and so on.
// We place the bytes that contain the bits to its respective byte so that we can mask out only the relevant bits later.
Vector128<byte> lowerShuffleMask_CopyToBoolArray = Vector128.Create(0, 0x01010101_01010101).AsByte();
Vector128<byte> upperShuffleMask_CopyToBoolArray = Vector128.Create(0x02020202_02020202, 0x03030303_03030303).AsByte();

if (Avx2.IsSupported)
if (Vector256.IsHardwareAccelerated)
{
Vector256<byte> shuffleMask = Vector256.Create(lowerShuffleMask_CopyToBoolArray, upperShuffleMask_CopyToBoolArray);
Vector256<byte> bitMask = Vector256.Create(0x80402010_08040201).AsByte();
Vector256<byte> ones = Vector256.Create((byte)1);

fixed (bool* destination = &boolArray[index])
{
for (; (i + Vector256ByteCount) <= (uint)m_length; i += Vector256ByteCount)
{
int bits = m_array[i / (uint)BitsPerInt32];
Vector256<int> scalar = Vector256.Create(bits);
Vector256<byte> shuffled = Avx2.Shuffle(scalar.AsByte(), shuffleMask);
Vector256<byte> extracted = Avx2.And(shuffled, bitMask);

// The extracted bits can be anywhere between 0 and 255, so we normalise the value to either 0 or 1
// to ensure compatibility with "C# bool" (0 for false, 1 for true, rest undefined)
Vector256<byte> normalized = Avx2.Min(extracted, ones);
Avx.Store((byte*)destination + i, normalized);
}
}
}
else if (Ssse3.IsSupported)
{
Vector128<byte> lowerShuffleMask = lowerShuffleMask_CopyToBoolArray;
Vector128<byte> upperShuffleMask = upperShuffleMask_CopyToBoolArray;
Vector128<byte> ones = Vector128.Create((byte)1);
Vector128<byte> bitMask128 = BitConverter.IsLittleEndian ?
Vector128.Create(0x80402010_08040201).AsByte() :
Vector128.Create(0x01020408_10204080).AsByte();
ref byte destination = ref Unsafe.As<bool, byte>(ref Unsafe.Add<bool>(ref MemoryMarshal.GetArrayDataReference<bool>(boolArray), index));

fixed (bool* destination = &boolArray[index])
for (; i <= thisLength - Vector256<byte>.Count; i += (uint)Vector256<byte>.Count)
{
for (; (i + Vector128ByteCount * 2u) <= (uint)m_length; i += Vector128ByteCount * 2u)
{
int bits = m_array[i / (uint)BitsPerInt32];
Vector128<int> scalar = Vector128.CreateScalarUnsafe(bits);

Vector128<byte> shuffledLower = Ssse3.Shuffle(scalar.AsByte(), lowerShuffleMask);
Vector128<byte> extractedLower = Sse2.And(shuffledLower, bitMask128);
Vector128<byte> normalizedLower = Sse2.Min(extractedLower, ones);
Sse2.Store((byte*)destination + i, normalizedLower);

Vector128<byte> shuffledHigher = Ssse3.Shuffle(scalar.AsByte(), upperShuffleMask);
Vector128<byte> extractedHigher = Sse2.And(shuffledHigher, bitMask128);
Vector128<byte> normalizedHigher = Sse2.Min(extractedHigher, ones);
Sse2.Store((byte*)destination + i + Vector128<byte>.Count, normalizedHigher);
}
int bits = thisArray[i / (uint)BitsPerInt32];
Vector256<int> scalar = Vector256.Create(bits);
Vector256<byte> shuffled = Vector256.Shuffle(scalar.AsByte(), Vector256.Create(0x0000_0000_0000_0000, 0x0101_0101_0101_0101, 0x12121212_12121212, 0x13131313_13131313).AsByte());
Vector256<byte> extracted = shuffled & bitMask;

// The extracted bits can be anywhere between 0 and 255, so we normalise the value to either 0 or 1
// to ensure compatibility with "C# bool" (0 for false, 1 for true, rest undefined)
Vector256<byte> normalized = Vector256.Min(extracted, ones);
normalized.StoreUnsafe(ref destination, i);
}
}
else if (AdvSimd.Arm64.IsSupported)
else if (Vector128.IsHardwareAccelerated)
{
Vector128<byte> ones = Vector128.Create((byte)1);
Vector128<byte> bitMask128 = BitConverter.IsLittleEndian ?
Vector128.Create(0x80402010_08040201).AsByte() :
Vector128.Create(0x01020408_10204080).AsByte();

fixed (bool* destination = &boolArray[index])
ref byte destination = ref Unsafe.As<bool, byte>(ref Unsafe.Add<bool>(ref MemoryMarshal.GetArrayDataReference<bool>(boolArray), index));

for (; i <= thisLength - Vector128<byte>.Count * 2u; i += (uint)Vector128<byte>.Count * 2u)
{
for (; (i + Vector128ByteCount * 2u) <= (uint)m_length; i += Vector128ByteCount * 2u)
{
int bits = m_array[i / (uint)BitsPerInt32];
// Same logic as SSSE3 path, except we do not have Shuffle instruction.
// (TableVectorLookup could be an alternative - dotnet/runtime#1277)
// Instead we use chained ZIP1/2 instructions:
// (A0 is the byte containing LSB, A3 is the byte containing MSB)
// bits (on Big endian) - A3 A2 A1 A0
// bits (Little endian) / Byte reversal - A0 A1 A2 A3
// v1 = Vector128.Create - A0 A1 A2 A3 A0 A1 A2 A3 A0 A1 A2 A3 A0 A1 A2 A3
// v2 = ZipLow(v1, v1) - A0 A0 A1 A1 A2 A2 A3 A3 A0 A0 A1 A1 A2 A2 A3 A3
// v3 = ZipLow(v2, v2) - A0 A0 A0 A0 A1 A1 A1 A1 A2 A2 A2 A2 A3 A3 A3 A3
// shuffledLower = ZipLow(v3, v3) - A0 A0 A0 A0 A0 A0 A0 A0 A1 A1 A1 A1 A1 A1 A1 A1
// shuffledHigher = ZipHigh(v3, v3) - A2 A2 A2 A2 A2 A2 A2 A2 A3 A3 A3 A3 A3 A3 A3 A3
if (!BitConverter.IsLittleEndian)
{
bits = BinaryPrimitives.ReverseEndianness(bits);
}
Vector128<byte> vector = Vector128.Create(bits).AsByte();
vector = AdvSimd.Arm64.ZipLow(vector, vector);
vector = AdvSimd.Arm64.ZipLow(vector, vector);

Vector128<byte> shuffledLower = AdvSimd.Arm64.ZipLow(vector, vector);
Vector128<byte> extractedLower = AdvSimd.And(shuffledLower, bitMask128);
Vector128<byte> normalizedLower = AdvSimd.Min(extractedLower, ones);

Vector128<byte> shuffledHigher = AdvSimd.Arm64.ZipHigh(vector, vector);
Vector128<byte> extractedHigher = AdvSimd.And(shuffledHigher, bitMask128);
Vector128<byte> normalizedHigher = AdvSimd.Min(extractedHigher, ones);

AdvSimd.Arm64.StorePair((byte*)destination + i, normalizedLower, normalizedHigher);
}
int bits = thisArray[i / (uint)BitsPerInt32];
Vector128<int> scalar = Vector128.CreateScalarUnsafe(bits);

// they masks need to be consts for optimal codegen (see https://github.com/dotnet/runtime/issues/72793)
Vector128<byte> shuffledLower = Vector128.Shuffle(scalar.AsByte(), Vector128.Create((byte)0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1));
Vector128<byte> extractedLower = shuffledLower & bitMask128;
Vector128<byte> normalizedLower = Vector128.Min(extractedLower, ones);
normalizedLower.StoreUnsafe(ref destination, i);

Vector128<byte> shuffledHigher = Vector128.Shuffle(scalar.AsByte(), Vector128.Create((byte)2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3));
Vector128<byte> extractedHigher = shuffledHigher & bitMask128;
Vector128<byte> normalizedHigher = Vector128.Min(extractedHigher, ones);
normalizedHigher.StoreUnsafe(ref destination, (nuint)(i + Vector128<byte>.Count));
}
}

LessThan32:
for (; i < (uint)m_length; i++)
for (; i < thisLength; i++)
{
int elementIndex = Div32Rem((int)i, out int extraBits);
boolArray[(uint)index + i] = ((m_array[elementIndex] >> extraBits) & 0x00000001) != 0;
boolArray[(uint)index + i] = ((thisArray[elementIndex] >> extraBits) & 0x00000001) != 0;
}
}
else
Expand Down