diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBooleanUnaryOperator.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBooleanUnaryOperator.cs index 45c33f35e7c1ab..f1a7e5c978dd82 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBooleanUnaryOperator.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBooleanUnaryOperator.cs @@ -5,13 +5,19 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; namespace System.Numerics.Tensors { public static unsafe partial class TensorPrimitives { /// Unary operator that produces a Boolean result for each element. - /// For vector-based methods, the Boolean result is either all-bits-set or zero. + /// + /// For vector-based methods, the Boolean result is either all-bits-set or zero. + /// An operator may also have a threshold form (): its result is then a single unsigned comparison of a + /// key derived from the element's bits against a constant, which lets fold a + /// block of vectors with one unsigned minimum or maximum per vector and compare once per block. + /// private interface IBooleanUnaryOperator { static abstract bool Vectorizable { get; } @@ -19,8 +25,89 @@ private interface IBooleanUnaryOperator static abstract Vector128 Invoke(Vector128 x); static abstract Vector256 Invoke(Vector256 x); static abstract Vector512 Invoke(Vector512 x); + + /// + /// Whether, reading the bits of the key and as unsigned integers of the element size, Invoke(x) + /// is Key(x) < ThresholdBits when and Key(x) > ThresholdBits otherwise. + /// + static virtual bool HasThresholdForm => false; + + /// Whether the operator is true for keys below the threshold rather than for keys above it. + static virtual bool TrueBelowThreshold => throw new NotSupportedException(); + + /// The threshold of the threshold form, as the bits of an unsigned integer of the element size. + static virtual ulong ThresholdBits => throw new NotSupportedException(); + + /// The key of the threshold form: the element's bits, transformed so that the operator is a single comparison of them. + static virtual Vector128 Key(Vector128 x) => throw new NotSupportedException(); + /// + static virtual Vector256 Key(Vector256 x) => throw new NotSupportedException(); + /// + static virtual Vector512 Key(Vector512 x) => throw new NotSupportedException(); + } + + /// The bits of the positive infinity of , or . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong PositiveInfinityBits() + { + Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); + return typeof(T) == typeof(float) ? + BitConverter.SingleToUInt32Bits(float.PositiveInfinity) : + BitConverter.DoubleToUInt64Bits(double.PositiveInfinity); + } + + /// The bits of the smallest positive normal value of , or . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong SmallestNormalBits() + { + Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); + return typeof(T) == typeof(float) ? 0x0080_0000u : 0x0010_0000_0000_0000ul; + } + + /// The sign bit of , a primitive signed integer, or , as an unsigned integer of the element size. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong SignBit() => 1ul << ((sizeof(T) * 8) - 1); + + /// Subtracts from every element's bits, read as unsigned integers of the element size ( or ). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 SubtractBits(Vector128 x, ulong bits) + { + Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); + return typeof(T) == typeof(float) ? + (x.AsUInt32() - Vector128.Create((uint)bits)).As() : + (x.AsUInt64() - Vector128.Create(bits)).As(); + } + + /// Subtracts from every element's bits, read as unsigned integers of the element size ( or ). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 SubtractBits(Vector256 x, ulong bits) + { + Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); + return typeof(T) == typeof(float) ? + (x.AsUInt32() - Vector256.Create((uint)bits)).As() : + (x.AsUInt64() - Vector256.Create(bits)).As(); + } + + /// Subtracts from every element's bits, read as unsigned integers of the element size ( or ). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 SubtractBits(Vector512 x, ulong bits) + { + Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); + return typeof(T) == typeof(float) ? + (x.AsUInt32() - Vector512.Create((uint)bits)).As() : + (x.AsUInt64() - Vector512.Create(bits)).As(); } + /// + /// Combines the results of into an Any/All decision. The vectorized loops of + /// fold the per-vector results of a block of vectors into an accumulator + /// and decide once per block whether the block contains an element which settles the result, in which case they exit with + /// !DefaultResult. At 128 and 256 bits the accumulator is a result itself, the fold of the block's results with + /// Accumulate starting from the result that equals DefaultResult, so ShouldEarlyExit decides. At 512 bits + /// the operator's comparison leaves its result in a mask register, which a select consumes as it is (a masked blend) whereas a + /// bitwise operation would first have to expand the mask into a vector, so the accumulator starts from all bits set and + /// ClearSettled clears the lanes whose result settles the aggregation, which a zero lane then signals. + /// private interface IAnyAllAggregator { static abstract bool DefaultResult { get; } @@ -28,6 +115,14 @@ private interface IAnyAllAggregator static abstract bool ShouldEarlyExit(Vector128 result); static abstract bool ShouldEarlyExit(Vector256 result); static abstract bool ShouldEarlyExit(Vector512 result); + + /// Folds an operator result into the aggregation of the results folded so far, a result itself: the OR of Any results, the AND of All results. + static abstract Vector128 Accumulate(Vector128 accumulator, Vector128 result); + /// + static abstract Vector256 Accumulate(Vector256 accumulator, Vector256 result); + + /// Clears the lanes of whose lane of settles the aggregation, that is, for which would hold. + static abstract Vector512 ClearSettled(Vector512 accumulator, Vector512 result); } private readonly struct AnyAggregator : IAnyAllAggregator @@ -39,6 +134,13 @@ private interface IAnyAllAggregator public static bool ShouldEarlyExit(Vector128 result) => Vector128.AnyWhereAllBitsSet(result); public static bool ShouldEarlyExit(Vector256 result) => Vector256.AnyWhereAllBitsSet(result); public static bool ShouldEarlyExit(Vector512 result) => Vector512.AnyWhereAllBitsSet(result); + + public static Vector128 Accumulate(Vector128 accumulator, Vector128 result) => accumulator | result; + public static Vector256 Accumulate(Vector256 accumulator, Vector256 result) => accumulator | result; + + // A lane where the operator was true is cleared: the select's constant is the zero vector, which costs no instruction, + // and the JIT folds the selection of the other lanes into a zero-masking move under the inverted comparison. + public static Vector512 ClearSettled(Vector512 accumulator, Vector512 result) => Vector512.ConditionalSelect(result, Vector512.Zero, accumulator); } private readonly struct AllAggregator : IAnyAllAggregator @@ -47,22 +149,38 @@ private interface IAnyAllAggregator public static bool ShouldEarlyExit(bool result) => !result; - public static bool ShouldEarlyExit(Vector128 result) => - typeof(T) == typeof(float) ? Vector128.EqualsAny(result.AsUInt32(), Vector128.Zero) : - typeof(T) == typeof(double) ? Vector128.EqualsAny(result.AsUInt64(), Vector128.Zero) : - Vector128.EqualsAny(result, Vector128.Zero); + public static bool ShouldEarlyExit(Vector128 result) => AnyLaneZero(result); + public static bool ShouldEarlyExit(Vector256 result) => AnyLaneZero(result); + public static bool ShouldEarlyExit(Vector512 result) => AnyLaneZero(result); - public static bool ShouldEarlyExit(Vector256 result) => - typeof(T) == typeof(float) ? Vector256.EqualsAny(result.AsUInt32(), Vector256.Zero) : - typeof(T) == typeof(double) ? Vector256.EqualsAny(result.AsUInt64(), Vector256.Zero) : - Vector256.EqualsAny(result, Vector256.Zero); + public static Vector128 Accumulate(Vector128 accumulator, Vector128 result) => accumulator & result; + public static Vector256 Accumulate(Vector256 accumulator, Vector256 result) => accumulator & result; - public static bool ShouldEarlyExit(Vector512 result) => - typeof(T) == typeof(float) ? Vector512.EqualsAny(result.AsUInt32(), Vector512.Zero) : - typeof(T) == typeof(double) ? Vector512.EqualsAny(result.AsUInt64(), Vector512.Zero) : - Vector512.EqualsAny(result, Vector512.Zero); + // A lane where the operator was false (its result is zero) is cleared (see AnyAggregator). + public static Vector512 ClearSettled(Vector512 accumulator, Vector512 result) => Vector512.ConditionalSelect(result, accumulator, Vector512.Zero); } + /// Whether any lane of is zero. For the floating-point types the lanes are compared as integers: a lane of an operator result or of an accumulator is either all bits set (a NaN) or zero. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool AnyLaneZero(Vector128 vector) => + typeof(T) == typeof(float) ? Vector128.EqualsAny(vector.AsUInt32(), Vector128.Zero) : + typeof(T) == typeof(double) ? Vector128.EqualsAny(vector.AsUInt64(), Vector128.Zero) : + Vector128.EqualsAny(vector, Vector128.Zero); + + /// Whether any lane of is zero. For the floating-point types the lanes are compared as integers: a lane of an operator result or of an accumulator is either all bits set (a NaN) or zero. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool AnyLaneZero(Vector256 vector) => + typeof(T) == typeof(float) ? Vector256.EqualsAny(vector.AsUInt32(), Vector256.Zero) : + typeof(T) == typeof(double) ? Vector256.EqualsAny(vector.AsUInt64(), Vector256.Zero) : + Vector256.EqualsAny(vector, Vector256.Zero); + + /// Whether any lane of is zero. For the floating-point types the lanes are compared as integers: a lane of an operator result or of an accumulator is either all bits set (a NaN) or zero. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool AnyLaneZero(Vector512 vector) => + typeof(T) == typeof(float) ? Vector512.EqualsAny(vector.AsUInt32(), Vector512.Zero) : + typeof(T) == typeof(double) ? Vector512.EqualsAny(vector.AsUInt64(), Vector512.Zero) : + Vector512.EqualsAny(vector, Vector512.Zero); + private static bool All(ReadOnlySpan x) where TOperator : struct, IBooleanUnaryOperator => AggregateAnyAll>(x); @@ -71,110 +189,600 @@ private static bool Any(ReadOnlySpan x) where TOperator : struct, IBooleanUnaryOperator => AggregateAnyAll>(x); + /// Vectors per block in the vectorized paths of . Must be even. + private const int AnyAllBlockVectors = 32; + private static bool AggregateAnyAll(ReadOnlySpan x) where TOperator : struct, IBooleanUnaryOperator where TAnyAll : struct, IAnyAllAggregator { Debug.Assert(!x.IsEmpty); + if (TOperator.HasThresholdForm) + { + // The keys are folded as unsigned integers of the element size. Their 64-bit minimum and maximum are single instructions + // only with AVX-512; elsewhere the operator's own comparison is cheaper. + if (sizeof(T) == 1) return AggregateAnyAllThreshold(x); + if (sizeof(T) == 2) return AggregateAnyAllThreshold(x); + if (sizeof(T) == 4) return AggregateAnyAllThreshold(x); + if (sizeof(T) == 8 && Avx512F.VL.IsSupported) return AggregateAnyAllThreshold(x); + } + + if (Vector512.IsHardwareAccelerated && TOperator.Vectorizable && Vector512.IsSupported && x.Length >= Vector512.Count) + { + return AggregateAnyAllVectorized512(x); + } + + if (Vector256.IsHardwareAccelerated && TOperator.Vectorizable && Vector256.IsSupported && x.Length >= Vector256.Count) + { + return AggregateAnyAllVectorized256(x); + } + + if (Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128.IsSupported && x.Length >= Vector128.Count) + { + return AggregateAnyAllVectorized128(x); + } + + return AggregateAnyAllScalar(x); + } + + /// The scalar path of , used when vectorization is not supported or the input is too small to vectorize. + private static bool AggregateAnyAllScalar(ReadOnlySpan x) + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { + ref T xRef = ref MemoryMarshal.GetReference(x); + for (int i = 0; i < x.Length; i++) + { + if (TAnyAll.ShouldEarlyExit(TOperator.Invoke(Unsafe.Add(ref xRef, i)))) + { + return !TAnyAll.DefaultResult; + } + } + + return TAnyAll.DefaultResult; + } + + /// The 512-bit path of : the whole vectors in blocks, then one final vector that overlaps the last whole one. + /// + /// Every block of up to vectors is folded into two independent accumulators with no branch on the + /// data, and the exit decision is made once per block, so a hit is detected after at most one block of extra reads. + /// Blocks are visited in order and the whole input lies within the span, so the result is the same as with a test per vector. + /// The accumulators start from all bits set and have the lanes of the settling results cleared (see ). + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per aggregation; its own inlining budget keeps the operator and the aggregator inlined + private static bool AggregateAnyAllVectorized512(ReadOnlySpan x) + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { + Debug.Assert(Vector512.IsHardwareAccelerated && TOperator.Vectorizable && Vector512.IsSupported); + Debug.Assert(x.Length >= Vector512.Count); + ref T xRef = ref MemoryMarshal.GetReference(x); - int i = 0, oneVectorFromEnd; + nuint length = (uint)x.Length; + nuint oneVectorFromEnd = length - (uint)Vector512.Count; + nuint i = 0; - if (Vector512.IsHardwareAccelerated && TOperator.Vectorizable && Vector512.IsSupported) + // Whole blocks: two accumulators, one decision per block. + nuint blockLength = (uint)(AnyAllBlockVectors * Vector512.Count); + if (length >= blockLength) { - oneVectorFromEnd = x.Length - Vector512.Count; - if (i <= oneVectorFromEnd) + nuint oneBlockFromEnd = length - blockLength; + do { - // Loop handling one vector at a time. + Vector512 accumulator0 = Vector512.AllBitsSet; + Vector512 accumulator1 = Vector512.AllBitsSet; + nuint blockEnd = i + blockLength; do { - if (TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, (uint)i)))) - { - return !TAnyAll.DefaultResult; - } - - i += Vector512.Count; + accumulator0 = TAnyAll.ClearSettled(accumulator0, TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, i))); + accumulator1 = TAnyAll.ClearSettled(accumulator1, TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, i + (uint)Vector512.Count))); + i += (uint)(2 * Vector512.Count); } - while (i <= oneVectorFromEnd); + while (i < blockEnd); - // Handle any remaining elements with a final vector. - if (i != x.Length && - TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, (uint)(x.Length - Vector512.Count))))) + if (AnyLaneZero(accumulator0 & accumulator1)) { return !TAnyAll.DefaultResult; } + } + while (i <= oneBlockFromEnd); + } - return TAnyAll.DefaultResult; + // The remaining whole vectors, fewer than a block. + if (i <= oneVectorFromEnd) + { + Vector512 accumulator = Vector512.AllBitsSet; + do + { + accumulator = TAnyAll.ClearSettled(accumulator, TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, i))); + i += (uint)Vector512.Count; } + while (i <= oneVectorFromEnd); + + if (AnyLaneZero(accumulator)) + { + return !TAnyAll.DefaultResult; + } + } + + // Handle any remaining elements with a final vector. + if (i != length && + TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, oneVectorFromEnd)))) + { + return !TAnyAll.DefaultResult; } - if (Vector256.IsHardwareAccelerated && TOperator.Vectorizable && Vector256.IsSupported) + return TAnyAll.DefaultResult; + } + + /// The 256-bit path of : the whole vectors in blocks, then one final vector that overlaps the last whole one. + /// + /// Every block of up to vectors is folded into two independent accumulators with no branch on the + /// data, and the exit decision is made once per block, so a hit is detected after at most one block of extra reads. + /// Blocks are visited in order and the whole input lies within the span, so the result is the same as with a test per vector. + /// The accumulators are results themselves: the fold of the block's results, starting from the result that equals the default + /// (see ). + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per aggregation; its own inlining budget keeps the operator and the aggregator inlined + private static bool AggregateAnyAllVectorized256(ReadOnlySpan x) + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { + Debug.Assert(Vector256.IsHardwareAccelerated && TOperator.Vectorizable && Vector256.IsSupported); + Debug.Assert(x.Length >= Vector256.Count); + + ref T xRef = ref MemoryMarshal.GetReference(x); + nuint length = (uint)x.Length; + nuint oneVectorFromEnd = length - (uint)Vector256.Count; + nuint i = 0; + + Vector256 defaultResult = TAnyAll.DefaultResult ? Vector256.AllBitsSet : Vector256.Zero; + + // Whole blocks: two accumulators, one decision per block. + nuint blockLength = (uint)(AnyAllBlockVectors * Vector256.Count); + if (length >= blockLength) { - oneVectorFromEnd = x.Length - Vector256.Count; - if (i <= oneVectorFromEnd) + nuint oneBlockFromEnd = length - blockLength; + do { - // Loop handling one vector at a time. + Vector256 accumulator0 = defaultResult; + Vector256 accumulator1 = defaultResult; + nuint blockEnd = i + blockLength; do { - if (TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)i)))) - { - return !TAnyAll.DefaultResult; - } + accumulator0 = TAnyAll.Accumulate(accumulator0, TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, i))); + accumulator1 = TAnyAll.Accumulate(accumulator1, TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, i + (uint)Vector256.Count))); + i += (uint)(2 * Vector256.Count); + } + while (i < blockEnd); - i += Vector256.Count; + if (TAnyAll.ShouldEarlyExit(TAnyAll.Accumulate(accumulator0, accumulator1))) + { + return !TAnyAll.DefaultResult; } - while (i <= oneVectorFromEnd); + } + while (i <= oneBlockFromEnd); + } - // Handle any remaining elements with a final vector. - if (i != x.Length && - TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256.Count))))) + // The remaining whole vectors, fewer than a block. + if (i <= oneVectorFromEnd) + { + Vector256 accumulator = defaultResult; + do + { + accumulator = TAnyAll.Accumulate(accumulator, TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, i))); + i += (uint)Vector256.Count; + } + while (i <= oneVectorFromEnd); + + if (TAnyAll.ShouldEarlyExit(accumulator)) + { + return !TAnyAll.DefaultResult; + } + } + + // Handle any remaining elements with a final vector. + if (i != length && + TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, oneVectorFromEnd)))) + { + return !TAnyAll.DefaultResult; + } + + return TAnyAll.DefaultResult; + } + + /// The 128-bit path of : the whole vectors in blocks, then one final vector that overlaps the last whole one. + /// + /// Every block of up to vectors is folded into two independent accumulators with no branch on the + /// data, and the exit decision is made once per block, so a hit is detected after at most one block of extra reads. + /// Blocks are visited in order and the whole input lies within the span, so the result is the same as with a test per vector. + /// The accumulators are results themselves: the fold of the block's results, starting from the result that equals the default + /// (see ). + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per aggregation; its own inlining budget keeps the operator and the aggregator inlined + private static bool AggregateAnyAllVectorized128(ReadOnlySpan x) + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { + Debug.Assert(Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128.IsSupported); + Debug.Assert(x.Length >= Vector128.Count); + + ref T xRef = ref MemoryMarshal.GetReference(x); + nuint length = (uint)x.Length; + nuint oneVectorFromEnd = length - (uint)Vector128.Count; + nuint i = 0; + + Vector128 defaultResult = TAnyAll.DefaultResult ? Vector128.AllBitsSet : Vector128.Zero; + + // Whole blocks: two accumulators, one decision per block. + nuint blockLength = (uint)(AnyAllBlockVectors * Vector128.Count); + if (length >= blockLength) + { + nuint oneBlockFromEnd = length - blockLength; + do + { + Vector128 accumulator0 = defaultResult; + Vector128 accumulator1 = defaultResult; + nuint blockEnd = i + blockLength; + do + { + accumulator0 = TAnyAll.Accumulate(accumulator0, TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, i))); + accumulator1 = TAnyAll.Accumulate(accumulator1, TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, i + (uint)Vector128.Count))); + i += (uint)(2 * Vector128.Count); + } + while (i < blockEnd); + + if (TAnyAll.ShouldEarlyExit(TAnyAll.Accumulate(accumulator0, accumulator1))) { return !TAnyAll.DefaultResult; } + } + while (i <= oneBlockFromEnd); + } - return TAnyAll.DefaultResult; + // The remaining whole vectors, fewer than a block. + if (i <= oneVectorFromEnd) + { + Vector128 accumulator = defaultResult; + do + { + accumulator = TAnyAll.Accumulate(accumulator, TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, i))); + i += (uint)Vector128.Count; + } + while (i <= oneVectorFromEnd); + + if (TAnyAll.ShouldEarlyExit(accumulator)) + { + return !TAnyAll.DefaultResult; } } - if (Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128.IsSupported) + // Handle any remaining elements with a final vector. + if (i != length && + TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, oneVectorFromEnd)))) + { + return !TAnyAll.DefaultResult; + } + + return TAnyAll.DefaultResult; + } + + /// + /// for an operator with a threshold form, whose keys are folded as + /// , the unsigned integer of the element size. + /// + private static bool AggregateAnyAllThreshold(ReadOnlySpan x) + where TKey : unmanaged, IBinaryInteger + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { + Debug.Assert(TOperator.HasThresholdForm); + Debug.Assert(sizeof(TKey) == sizeof(T)); + Debug.Assert(!x.IsEmpty); + + // The fold is chosen by type rather than by a branch on the direction so that the loops contain a single use of each key, + // which the JIT then folds into the fold instruction's memory operand. + return ThresholdFoldsMax() ? + AggregateAnyAllThreshold, TOperator, TAnyAll>(x) : + AggregateAnyAllThreshold, TOperator, TAnyAll>(x); + } + + /// + /// Whether the threshold form of aggregated by folds the keys of a block + /// with their maximum rather than their minimum. Any looks for an element for which the operator holds and All for one for which it + /// does not: with the operator true below the threshold, those are a key below it (the minimum decides) and a key at or above it (the + /// maximum decides); with the operator true above the threshold, it is the other way round. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool ThresholdFoldsMax() + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator => + TOperator.TrueBelowThreshold == TAnyAll.DefaultResult; + + /// + /// with the fold of the keys, , chosen: + /// or over (see ). + /// + private static bool AggregateAnyAllThreshold(ReadOnlySpan x) + where TKey : unmanaged, IBinaryInteger + where TFold : struct, IBinaryOperator + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { + Debug.Assert(typeof(TFold) == (ThresholdFoldsMax() ? typeof(MaxOperator) : typeof(MinOperator))); + + if (Vector512.IsHardwareAccelerated && TOperator.Vectorizable && Vector512.IsSupported && x.Length >= Vector512.Count) + { + return AggregateAnyAllThreshold512(x); + } + + if (Vector256.IsHardwareAccelerated && TOperator.Vectorizable && Vector256.IsSupported && x.Length >= Vector256.Count) + { + return AggregateAnyAllThreshold256(x); + } + + if (Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128.IsSupported && x.Length >= Vector128.Count) { - oneVectorFromEnd = x.Length - Vector128.Count; - if (i <= oneVectorFromEnd) + return AggregateAnyAllThreshold128(x); + } + + return AggregateAnyAllScalar(x); + } + + /// The 512-bit path of : the whole vectors in blocks, then one final vector that overlaps the last whole one. + /// + /// The shape of , except that a block is folded with the unsigned + /// minimum or maximum of the elements' keys, one instruction per vector with no comparison, and the fold is compared against the + /// threshold once per block. + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per aggregation; its own inlining budget keeps the operator inlined + private static bool AggregateAnyAllThreshold512(ReadOnlySpan x) + where TKey : unmanaged, IBinaryInteger + where TFold : struct, IBinaryOperator + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { + Debug.Assert(Vector512.IsHardwareAccelerated && TOperator.Vectorizable && Vector512.IsSupported); + Debug.Assert(x.Length >= Vector512.Count); + + ref T xRef = ref MemoryMarshal.GetReference(x); + nuint length = (uint)x.Length; + nuint oneVectorFromEnd = length - (uint)Vector512.Count; + nuint i = 0; + + Vector512 identity = ThresholdFoldsMax() ? Vector512.Zero : Vector512.AllBitsSet; + Vector512 threshold = Vector512.Create(TKey.CreateTruncating(TOperator.ThresholdBits)); + + // Whole blocks: two accumulators, one decision per block. + nuint blockLength = (uint)(AnyAllBlockVectors * Vector512.Count); + if (length >= blockLength) + { + nuint oneBlockFromEnd = length - blockLength; + do { - // Loop handling one vector at a time. + Vector512 accumulator0 = identity; + Vector512 accumulator1 = identity; + nuint blockEnd = i + blockLength; do { - if (TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)i)))) - { - return !TAnyAll.DefaultResult; - } + accumulator0 = TFold.Invoke(accumulator0, TOperator.Key(Vector512.LoadUnsafe(ref xRef, i)).As()); + accumulator1 = TFold.Invoke(accumulator1, TOperator.Key(Vector512.LoadUnsafe(ref xRef, i + (uint)Vector512.Count)).As()); + i += (uint)(2 * Vector512.Count); + } + while (i < blockEnd); - i += Vector128.Count; + if (Settles(TFold.Invoke(accumulator0, accumulator1), threshold)) + { + return !TAnyAll.DefaultResult; } - while (i <= oneVectorFromEnd); + } + while (i <= oneBlockFromEnd); + } + + // The remaining whole vectors, fewer than a block. + if (i <= oneVectorFromEnd) + { + Vector512 accumulator = identity; + do + { + accumulator = TFold.Invoke(accumulator, TOperator.Key(Vector512.LoadUnsafe(ref xRef, i)).As()); + i += (uint)Vector512.Count; + } + while (i <= oneVectorFromEnd); - // Handle any remaining elements with a final vector. - if (i != x.Length && - TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128.Count))))) + if (Settles(accumulator, threshold)) + { + return !TAnyAll.DefaultResult; + } + } + + // Handle any remaining elements with a final vector. + if (i != length && + Settles(TOperator.Key(Vector512.LoadUnsafe(ref xRef, oneVectorFromEnd)).As(), threshold)) + { + return !TAnyAll.DefaultResult; + } + + return TAnyAll.DefaultResult; + + // Whether the keys folded into the accumulator include one that settles the result: for Any a key on the operator's side of + // the threshold, for All a key on the other side (or on the threshold). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static bool Settles(Vector512 accumulator, Vector512 threshold) => + TAnyAll.DefaultResult ? + (TOperator.TrueBelowThreshold ? Vector512.GreaterThanOrEqualAny(accumulator, threshold) : Vector512.LessThanOrEqualAny(accumulator, threshold)) : + (TOperator.TrueBelowThreshold ? Vector512.LessThanAny(accumulator, threshold) : Vector512.GreaterThanAny(accumulator, threshold)); + } + + /// The 256-bit path of : the whole vectors in blocks, then one final vector that overlaps the last whole one. + /// + /// The shape of , except that a block is folded with the unsigned + /// minimum or maximum of the elements' keys, one instruction per vector with no comparison, and the fold is compared against the + /// threshold once per block. + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per aggregation; its own inlining budget keeps the operator inlined + private static bool AggregateAnyAllThreshold256(ReadOnlySpan x) + where TKey : unmanaged, IBinaryInteger + where TFold : struct, IBinaryOperator + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { + Debug.Assert(Vector256.IsHardwareAccelerated && TOperator.Vectorizable && Vector256.IsSupported); + Debug.Assert(x.Length >= Vector256.Count); + + ref T xRef = ref MemoryMarshal.GetReference(x); + nuint length = (uint)x.Length; + nuint oneVectorFromEnd = length - (uint)Vector256.Count; + nuint i = 0; + + Vector256 identity = ThresholdFoldsMax() ? Vector256.Zero : Vector256.AllBitsSet; + Vector256 threshold = Vector256.Create(TKey.CreateTruncating(TOperator.ThresholdBits)); + + // Whole blocks: two accumulators, one decision per block. + nuint blockLength = (uint)(AnyAllBlockVectors * Vector256.Count); + if (length >= blockLength) + { + nuint oneBlockFromEnd = length - blockLength; + do + { + Vector256 accumulator0 = identity; + Vector256 accumulator1 = identity; + nuint blockEnd = i + blockLength; + do + { + accumulator0 = TFold.Invoke(accumulator0, TOperator.Key(Vector256.LoadUnsafe(ref xRef, i)).As()); + accumulator1 = TFold.Invoke(accumulator1, TOperator.Key(Vector256.LoadUnsafe(ref xRef, i + (uint)Vector256.Count)).As()); + i += (uint)(2 * Vector256.Count); + } + while (i < blockEnd); + + if (Settles(TFold.Invoke(accumulator0, accumulator1), threshold)) { return !TAnyAll.DefaultResult; } + } + while (i <= oneBlockFromEnd); + } - return TAnyAll.DefaultResult; + // The remaining whole vectors, fewer than a block. + if (i <= oneVectorFromEnd) + { + Vector256 accumulator = identity; + do + { + accumulator = TFold.Invoke(accumulator, TOperator.Key(Vector256.LoadUnsafe(ref xRef, i)).As()); + i += (uint)Vector256.Count; + } + while (i <= oneVectorFromEnd); + + if (Settles(accumulator, threshold)) + { + return !TAnyAll.DefaultResult; } } - while (i < x.Length) + // Handle any remaining elements with a final vector. + if (i != length && + Settles(TOperator.Key(Vector256.LoadUnsafe(ref xRef, oneVectorFromEnd)).As(), threshold)) { - if (TAnyAll.ShouldEarlyExit(TOperator.Invoke(Unsafe.Add(ref xRef, i)))) + return !TAnyAll.DefaultResult; + } + + return TAnyAll.DefaultResult; + + // Whether the keys folded into the accumulator include one that settles the result: for Any a key on the operator's side of + // the threshold, for All a key on the other side (or on the threshold). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static bool Settles(Vector256 accumulator, Vector256 threshold) => + TAnyAll.DefaultResult ? + (TOperator.TrueBelowThreshold ? Vector256.GreaterThanOrEqualAny(accumulator, threshold) : Vector256.LessThanOrEqualAny(accumulator, threshold)) : + (TOperator.TrueBelowThreshold ? Vector256.LessThanAny(accumulator, threshold) : Vector256.GreaterThanAny(accumulator, threshold)); + } + + /// The 128-bit path of : the whole vectors in blocks, then one final vector that overlaps the last whole one. + /// + /// The shape of , except that a block is folded with the unsigned + /// minimum or maximum of the elements' keys, one instruction per vector with no comparison, and the fold is compared against the + /// threshold once per block. + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per aggregation; its own inlining budget keeps the operator inlined + private static bool AggregateAnyAllThreshold128(ReadOnlySpan x) + where TKey : unmanaged, IBinaryInteger + where TFold : struct, IBinaryOperator + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { + Debug.Assert(Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128.IsSupported); + Debug.Assert(x.Length >= Vector128.Count); + + ref T xRef = ref MemoryMarshal.GetReference(x); + nuint length = (uint)x.Length; + nuint oneVectorFromEnd = length - (uint)Vector128.Count; + nuint i = 0; + + Vector128 identity = ThresholdFoldsMax() ? Vector128.Zero : Vector128.AllBitsSet; + Vector128 threshold = Vector128.Create(TKey.CreateTruncating(TOperator.ThresholdBits)); + + // Whole blocks: two accumulators, one decision per block. + nuint blockLength = (uint)(AnyAllBlockVectors * Vector128.Count); + if (length >= blockLength) + { + nuint oneBlockFromEnd = length - blockLength; + do + { + Vector128 accumulator0 = identity; + Vector128 accumulator1 = identity; + nuint blockEnd = i + blockLength; + do + { + accumulator0 = TFold.Invoke(accumulator0, TOperator.Key(Vector128.LoadUnsafe(ref xRef, i)).As()); + accumulator1 = TFold.Invoke(accumulator1, TOperator.Key(Vector128.LoadUnsafe(ref xRef, i + (uint)Vector128.Count)).As()); + i += (uint)(2 * Vector128.Count); + } + while (i < blockEnd); + + if (Settles(TFold.Invoke(accumulator0, accumulator1), threshold)) + { + return !TAnyAll.DefaultResult; + } + } + while (i <= oneBlockFromEnd); + } + + // The remaining whole vectors, fewer than a block. + if (i <= oneVectorFromEnd) + { + Vector128 accumulator = identity; + do + { + accumulator = TFold.Invoke(accumulator, TOperator.Key(Vector128.LoadUnsafe(ref xRef, i)).As()); + i += (uint)Vector128.Count; + } + while (i <= oneVectorFromEnd); + + if (Settles(accumulator, threshold)) { return !TAnyAll.DefaultResult; } + } - i++; + // Handle any remaining elements with a final vector. + if (i != length && + Settles(TOperator.Key(Vector128.LoadUnsafe(ref xRef, oneVectorFromEnd)).As(), threshold)) + { + return !TAnyAll.DefaultResult; } return TAnyAll.DefaultResult; + + // Whether the keys folded into the accumulator include one that settles the result: for Any a key on the operator's side of + // the threshold, for All a key on the other side (or on the threshold). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static bool Settles(Vector128 accumulator, Vector128 threshold) => + TAnyAll.DefaultResult ? + (TOperator.TrueBelowThreshold ? Vector128.GreaterThanOrEqualAny(accumulator, threshold) : Vector128.LessThanOrEqualAny(accumulator, threshold)) : + (TOperator.TrueBelowThreshold ? Vector128.LessThanAny(accumulator, threshold) : Vector128.GreaterThanAny(accumulator, threshold)); } /// Performs an element-wise operation on and writes the results to . diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsFinite.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsFinite.cs index 3c1e22bfb16097..52a374d6256bc2 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsFinite.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsFinite.cs @@ -70,6 +70,14 @@ private static bool AlwaysFinite() => public static Vector128 Invoke(Vector128 x) => Vector128.IsFinite(x); public static Vector256 Invoke(Vector256 x) => Vector256.IsFinite(x); public static Vector512 Invoke(Vector512 x) => Vector512.IsFinite(x); + + // A finite value is one whose absolute bit pattern lies below that of infinity. + public static bool HasThresholdForm => typeof(T) == typeof(float) || typeof(T) == typeof(double); + public static bool TrueBelowThreshold => true; + public static ulong ThresholdBits => PositiveInfinityBits(); + public static Vector128 Key(Vector128 x) => Vector128.Abs(x); + public static Vector256 Key(Vector256 x) => Vector256.Abs(x); + public static Vector512 Key(Vector512 x) => Vector512.Abs(x); } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNaN.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNaN.cs index 7e38263b25a347..2c5217ad4c69b1 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNaN.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNaN.cs @@ -68,6 +68,14 @@ private static bool MayBeNaN() => public static Vector128 Invoke(Vector128 x) => Vector128.IsNaN(x); public static Vector256 Invoke(Vector256 x) => Vector256.IsNaN(x); public static Vector512 Invoke(Vector512 x) => Vector512.IsNaN(x); + + // A NaN is a value whose absolute bit pattern lies above that of infinity. + public static bool HasThresholdForm => typeof(T) == typeof(float) || typeof(T) == typeof(double); + public static bool TrueBelowThreshold => false; + public static ulong ThresholdBits => PositiveInfinityBits(); + public static Vector128 Key(Vector128 x) => Vector128.Abs(x); + public static Vector256 Key(Vector256 x) => Vector256.Abs(x); + public static Vector512 Key(Vector512 x) => Vector512.Abs(x); } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNegative.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNegative.cs index 3ed5de79558d1a..fb4be418108b6e 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNegative.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNegative.cs @@ -72,6 +72,14 @@ private static bool MayBeNegative() => public static Vector128 Invoke(Vector128 x) => Vector128.IsNegative(x); public static Vector256 Invoke(Vector256 x) => Vector256.IsNegative(x); public static Vector512 Invoke(Vector512 x) => Vector512.IsNegative(x); + + // A negative value is one whose sign bit is set: whose bits lie above the values with the sign bit clear. + public static bool HasThresholdForm => MayBeNegative() && (IsPrimitiveBinaryInteger() || typeof(T) == typeof(float) || typeof(T) == typeof(double)); + public static bool TrueBelowThreshold => false; + public static ulong ThresholdBits => SignBit() - 1; + public static Vector128 Key(Vector128 x) => x; + public static Vector256 Key(Vector256 x) => x; + public static Vector512 Key(Vector512 x) => x; } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNormal.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNormal.cs index 1fc78f2bf87de4..be510e8491ebc2 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNormal.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNormal.cs @@ -54,6 +54,15 @@ public static bool IsNormalAny(ReadOnlySpan x) public static Vector256 Invoke(Vector256 x) => Vector256.IsNormal(x); public static Vector512 Invoke(Vector512 x) => Vector512.IsNormal(x); + + // A normal value is one whose absolute bit pattern lies between those of the smallest normal value (inclusive) and infinity (exclusive), + // the same range test as the operator's, which the subtraction turns into a single comparison: a smaller pattern wraps around. + public static bool HasThresholdForm => typeof(T) == typeof(float) || typeof(T) == typeof(double); + public static bool TrueBelowThreshold => true; + public static ulong ThresholdBits => PositiveInfinityBits() - SmallestNormalBits(); + public static Vector128 Key(Vector128 x) => SubtractBits(Vector128.Abs(x), SmallestNormalBits()); + public static Vector256 Key(Vector256 x) => SubtractBits(Vector256.Abs(x), SmallestNormalBits()); + public static Vector512 Key(Vector512 x) => SubtractBits(Vector512.Abs(x), SmallestNormalBits()); } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsPositive.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsPositive.cs index 187b57c1b37bc0..f03fd2b1dc0c8d 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsPositive.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsPositive.cs @@ -50,6 +50,14 @@ public static bool IsPositiveAny(ReadOnlySpan x) public static Vector128 Invoke(Vector128 x) => Vector128.IsPositive(x); public static Vector256 Invoke(Vector256 x) => Vector256.IsPositive(x); public static Vector512 Invoke(Vector512 x) => Vector512.IsPositive(x); + + // A positive value is one whose sign bit is clear: whose bits lie below the sign bit. + public static bool HasThresholdForm => MayBeNegative() && (IsPrimitiveBinaryInteger() || typeof(T) == typeof(float) || typeof(T) == typeof(double)); + public static bool TrueBelowThreshold => true; + public static ulong ThresholdBits => SignBit(); + public static Vector128 Key(Vector128 x) => x; + public static Vector256 Key(Vector256 x) => x; + public static Vector512 Key(Vector512 x) => x; } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsRealNumber.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsRealNumber.cs index daadd47a83ab3b..71d29578e05642 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsRealNumber.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsRealNumber.cs @@ -68,6 +68,14 @@ private static bool AlwaysReal() => public static Vector128 Invoke(Vector128 x) => ~Vector128.IsNaN(x); public static Vector256 Invoke(Vector256 x) => ~Vector256.IsNaN(x); public static Vector512 Invoke(Vector512 x) => ~Vector512.IsNaN(x); + + // A real number is any value but a NaN: one whose absolute bit pattern lies below that of the first NaN, the one after infinity. + public static bool HasThresholdForm => typeof(T) == typeof(float) || typeof(T) == typeof(double); + public static bool TrueBelowThreshold => true; + public static ulong ThresholdBits => PositiveInfinityBits() + 1; + public static Vector128 Key(Vector128 x) => Vector128.Abs(x); + public static Vector256 Key(Vector256 x) => Vector256.Abs(x); + public static Vector512 Key(Vector512 x) => Vector512.Abs(x); } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsSubnormal.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsSubnormal.cs index f821ac491ef922..0a799dc971953d 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsSubnormal.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsSubnormal.cs @@ -57,6 +57,15 @@ private static bool MayBeSubnormal() => public static Vector128 Invoke(Vector128 x) => Vector128.IsSubnormal(x); public static Vector256 Invoke(Vector256 x) => Vector256.IsSubnormal(x); public static Vector512 Invoke(Vector512 x) => Vector512.IsSubnormal(x); + + // A subnormal value is one whose absolute bit pattern lies between one (inclusive) and that of the smallest normal value (exclusive), + // the same range test as the operator's, which the subtraction turns into a single comparison: zero wraps around. + public static bool HasThresholdForm => typeof(T) == typeof(float) || typeof(T) == typeof(double); + public static bool TrueBelowThreshold => true; + public static ulong ThresholdBits => SmallestNormalBits() - 1; + public static Vector128 Key(Vector128 x) => SubtractBits(Vector128.Abs(x), 1); + public static Vector256 Key(Vector256 x) => SubtractBits(Vector256.Abs(x), 1); + public static Vector512 Key(Vector512 x) => SubtractBits(Vector512.Abs(x), 1); } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsZero.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsZero.cs index eaca0c55dc73cf..00e7feb17fb0b9 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsZero.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsZero.cs @@ -50,6 +50,14 @@ public static bool IsZeroAny(ReadOnlySpan x) public static Vector128 Invoke(Vector128 x) => Vector128.Equals(x, Vector128.Zero); public static Vector256 Invoke(Vector256 x) => Vector256.Equals(x, Vector256.Zero); public static Vector512 Invoke(Vector512 x) => Vector512.Equals(x, Vector512.Zero); + + // Zero is the value whose bits, for the floating-point types apart from the sign, are below one. + public static bool HasThresholdForm => IsPrimitiveBinaryInteger() || typeof(T) == typeof(float) || typeof(T) == typeof(double); + public static bool TrueBelowThreshold => true; + public static ulong ThresholdBits => 1; + public static Vector128 Key(Vector128 x) => typeof(T) == typeof(float) || typeof(T) == typeof(double) ? Vector128.Abs(x) : x; + public static Vector256 Key(Vector256 x) => typeof(T) == typeof(float) || typeof(T) == typeof(double) ? Vector256.Abs(x) : x; + public static Vector512 Key(Vector512 x) => typeof(T) == typeof(float) || typeof(T) == typeof(double) ? Vector512.Abs(x) : x; } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs index 1e7249bd3fc751..d1462c91c2cebe 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; @@ -119,6 +120,9 @@ public static void Max(ReadOnlySpan x, T y, Span destination) public static T Invoke(Vector512 x) => HorizontalAggregate>(x); } + /// Vectors per block in the vectorized paths of . + private const int MinMaxBlockVectors = 32; + /// /// This is the same as /// with an identity transform, except it early exits on NaN. @@ -139,219 +143,803 @@ private static T MinMaxCore(ReadOnlySpan x) if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && x.Length >= Vector512.Count) { - ref T xRef = ref MemoryMarshal.GetReference(x); + return MinMaxVectorized512(x); + } - // Load the first vector as the initial set of results, and bail immediately - // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector512 result = Vector512.LoadUnsafe(ref xRef, 0); - Vector512 current; + if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && x.Length >= Vector256.Count) + { + return MinMaxVectorized256(x); + } - Vector512 nanMask; - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && x.Length >= Vector128.Count) + { + return MinMaxVectorized128(x); + } + + // Scalar path used when either vectorization is not supported or the input is too small to vectorize. + T curResult = x[0]; + if (T.IsNaN(curResult)) + { + return curResult; + } + + for (int i = 1; i < x.Length; i++) + { + T current = x[i]; + if (T.IsNaN(current)) { - // Check for NaNs - nanMask = Vector512.IsNaN(result); - if (nanMask != Vector512.Zero) - { - return result.GetElement(IndexOfFirstMatch(nanMask)); - } + return current; } - int oneVectorFromEnd = x.Length - Vector512.Count; - int i = Vector512.Count; + curResult = TMinMaxOperator.Invoke(curResult, current); + } + + return curResult; + } + + /// + /// Whether is or over + /// or : the reductions whose blocks are reduced with the native vector minimum/maximum + /// (, one instruction) plus masks that restore the IEEE 754:2019 rules once per block, + /// rather than with the operator itself (, several dependent instructions). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsNativeMinMax() => + (typeof(T) == typeof(float) && (typeof(TMinMaxOperator) == typeof(MinOperator) || typeof(TMinMaxOperator) == typeof(MaxOperator))) || + (typeof(T) == typeof(double) && (typeof(TMinMaxOperator) == typeof(MinOperator) || typeof(TMinMaxOperator) == typeof(MaxOperator))); + + /// Whether is a minimum rather than a maximum. Only meaningful when . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsNativeMin() => + typeof(TMinMaxOperator) == typeof(MinOperator) || typeof(TMinMaxOperator) == typeof(MinOperator); + + /// + /// Restores the IEEE 754:2019 `minimum`/`maximum` result of a block reduced with the native vector minimum/maximum, which does not + /// order the signed zeros. The native operation never returns a value below (above) every input, so if the block minimum (maximum) + /// compares equal to zero and the block holds no NaN, no element is negative (positive), and an element whose sign bit is set (clear) + /// can only be -0 (+0). + /// + /// The result of the native reduction of a block that contains no NaN. + /// For a minimum, whether any element of the block has its sign bit set; for a maximum, whether any has it clear. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T FixUpNativeSignedZero(T blockResult, bool anyOppositeSignBit) + where T : INumberBase + { + Debug.Assert(IsNativeMinMax()); + Debug.Assert(!T.IsNaN(blockResult)); + + if (anyOppositeSignBit && blockResult == T.Zero) + { + return IsNativeMin() ? -T.Zero : T.Zero; + } - // Aggregate additional vectors into the result as long as there's at least one full vector left to process. - while (i <= oneVectorFromEnd) + return blockResult; + } + + /// Min(x, y) with no NaN or signed-zero guarantees: the hardware minimum where there is one. Used for the horizontal reductions of the native blocks. + private readonly struct MinNativeOperator : IBinaryOperator + where T : INumberBase + { + public static bool Vectorizable => true; + + public static T Invoke(T x, T y) => Vector128.MinNative(Vector128.CreateScalar(x), Vector128.CreateScalar(y)).ToScalar(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) => Vector128.MinNative(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) => Vector256.MinNative(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) => Vector512.MinNative(x, y); + } + + /// Max(x, y) with no NaN or signed-zero guarantees: the hardware maximum where there is one. Used for the horizontal reductions of the native blocks. + private readonly struct MaxNativeOperator : IBinaryOperator + where T : INumberBase + { + public static bool Vectorizable => true; + + public static T Invoke(T x, T y) => Vector128.MaxNative(Vector128.CreateScalar(x), Vector128.CreateScalar(y)).ToScalar(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) => Vector128.MaxNative(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) => Vector256.MaxNative(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) => Vector512.MaxNative(x, y); + } + + /// The 512-bit path of : a block reduction of the whole vectors followed by one final vector that overlaps the last whole one. + /// + /// Every block of up to vectors is reduced with two independent accumulators and a single NaN decision + /// (the OR of the elements' NaN masks), so the hot loop contains no branch on the data. Blocks are visited in order and the running + /// result is only combined after a block has been checked, so the first NaN of the input is the one returned. + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per reduction; its own inlining budget keeps the block reduction and the horizontal aggregates inlined + private static T MinMaxVectorized512(ReadOnlySpan x) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(Vector512.IsHardwareAccelerated && Vector512.IsSupported); + Debug.Assert(x.Length >= Vector512.Count); + + ref T xRef = ref MemoryMarshal.GetReference(x); + int length = x.Length; + int blockSize = MinMaxBlockVectors * Vector512.Count; + int wholeVectorsLength = length - (length % Vector512.Count); + + // Reduce the whole vectors block by block. Starting from the first element is harmless: it is part of the first block, + // and combining an element with a result that already accounts for it does not change that result. + T result = xRef; + for (int i = 0; i < wholeVectorsLength; i += blockSize) + { + int blockLength = Math.Min(blockSize, wholeVectorsLength - i); + T blockResult; + bool anyNaN; + + if (IsNativeMinMax()) { - // Load the next vector, and early exit on NaN. - current = Vector512.LoadUnsafe(ref xRef, (uint)i); + blockResult = BlockReduceNative512(ref Unsafe.Add(ref xRef, i), blockLength, out anyNaN, out bool anyOppositeSignBit); + if (anyNaN) + { + return FirstNaN512(ref Unsafe.Add(ref xRef, i), blockLength); + } + blockResult = FixUpNativeSignedZero(blockResult, anyOppositeSignBit); + } + else + { + blockResult = BlockReduce512(ref Unsafe.Add(ref xRef, i), blockLength, out anyNaN); if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - // Check for NaNs - nanMask = ~Vector512.Equals(current, current); - if (nanMask != Vector512.Zero) + if (anyNaN) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return FirstNaN512(ref Unsafe.Add(ref xRef, i), blockLength); } } - - result = TMinMaxOperator.Invoke(result, current); - i += Vector512.Count; } - // If any elements remain, handle them in one final vector. - if (i != x.Length) + result = TMinMaxOperator.Invoke(result, blockResult); + } + + // If any elements remain, handle them in one final vector. Its NaN check does not change which NaN is returned: + // a NaN among the elements it revisits would already have been returned by their block. + if (wholeVectorsLength != length) + { + Vector512 last = Vector512.LoadUnsafe(ref xRef, (uint)(length - Vector512.Count)); + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - current = Vector512.LoadUnsafe(ref xRef, (uint)(x.Length - Vector512.Count)); + Vector512 nanMask = Vector512.IsNaN(last); + if (nanMask != Vector512.Zero) + { + return last.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + result = TMinMaxOperator.Invoke(result, TMinMaxOperator.Invoke(last)); + } + return result; + } + + /// Reduces the elements at , a whole number of 512-bit vectors, with . + /// The first element of the block. + /// The number of elements in the block, a positive multiple of . + /// + /// For and , whether any element is NaN, in which case the returned value is meaningless. + /// It is gathered from the elements rather than from the result because the Number operators discard NaN. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T BlockReduce512(ref T xRef, int length, out bool anyNaN) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(length >= Vector512.Count && length % Vector512.Count == 0); + + Vector512 acc1 = Vector512.LoadUnsafe(ref xRef); + Vector512 nan = Vector512.IsNaN(acc1); + int i = Vector512.Count; + + if (length >= 2 * Vector512.Count) + { + Vector512 acc2 = Vector512.LoadUnsafe(ref xRef, (uint)i); + nan |= Vector512.IsNaN(acc2); + i += Vector512.Count; + + int twoVectorsFromEnd = length - (2 * Vector512.Count); + while (i <= twoVectorsFromEnd) + { + Vector512 current1 = Vector512.LoadUnsafe(ref xRef, (uint)i); + Vector512 current2 = Vector512.LoadUnsafe(ref xRef, (uint)(i + Vector512.Count)); + acc1 = TMinMaxOperator.Invoke(acc1, current1); + acc2 = TMinMaxOperator.Invoke(acc2, current2); if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - // Check for NaNs - nanMask = ~Vector512.Equals(current, current); - if (nanMask != Vector512.Zero) - { - return current.GetElement(IndexOfFirstMatch(nanMask)); - } + nan |= Vector512.IsNaN(current1) | Vector512.IsNaN(current2); } - result = TMinMaxOperator.Invoke(result, current); + i += 2 * Vector512.Count; } - // Aggregate the lanes in the vector to create the final scalar result. - return TMinMaxOperator.Invoke(result); + acc1 = TMinMaxOperator.Invoke(acc1, acc2); } - if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && x.Length >= Vector256.Count) + if (i != length) { - ref T xRef = ref MemoryMarshal.GetReference(x); + Vector512 current = Vector512.LoadUnsafe(ref xRef, (uint)i); + acc1 = TMinMaxOperator.Invoke(acc1, current); + nan |= Vector512.IsNaN(current); + } - // Load the first vector as the initial set of results, and bail immediately - // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector256 result = Vector256.LoadUnsafe(ref xRef, 0); - Vector256 current; + anyNaN = (typeof(T) == typeof(float) || typeof(T) == typeof(double)) && nan != Vector512.Zero; + return TMinMaxOperator.Invoke(acc1); + } - Vector256 nanMask; - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + /// + /// Reduces the elements at , a whole number of 512-bit vectors, with the native vector + /// minimum/maximum, gathering alongside the two facts that let the caller restore the IEEE 754:2019 result (see ). + /// + /// The first element of the block. + /// The number of elements in the block, a positive multiple of . + /// Whether any element is NaN, in which case the returned value is meaningless. + /// For a minimum, whether any element has its sign bit set; for a maximum, whether any has it clear. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T BlockReduceNative512(ref T xRef, int length, out bool anyNaN, out bool anyOppositeSignBit) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(IsNativeMinMax()); + Debug.Assert(length >= Vector512.Count && length % Vector512.Count == 0); + + // For a minimum, OR the elements together so that any set sign bit survives; for a maximum, AND them so that any clear one does. + Vector512 acc1 = Vector512.LoadUnsafe(ref xRef); + Vector512 nan = Vector512.IsNaN(acc1); + Vector512 signBits = acc1; + int i = Vector512.Count; + + if (length >= 2 * Vector512.Count) + { + Vector512 acc2 = Vector512.LoadUnsafe(ref xRef, (uint)i); + nan |= Vector512.IsNaN(acc2); + signBits = IsNativeMin() ? signBits | acc2 : signBits & acc2; + i += Vector512.Count; + + int twoVectorsFromEnd = length - (2 * Vector512.Count); + while (i <= twoVectorsFromEnd) { - // Check for NaNs - nanMask = ~Vector256.Equals(result, result); - if (nanMask != Vector256.Zero) + Vector512 current1 = Vector512.LoadUnsafe(ref xRef, (uint)i); + Vector512 current2 = Vector512.LoadUnsafe(ref xRef, (uint)(i + Vector512.Count)); + if (IsNativeMin()) + { + acc1 = Vector512.MinNative(acc1, current1); + acc2 = Vector512.MinNative(acc2, current2); + signBits |= current1 | current2; + } + else { - return result.GetElement(IndexOfFirstMatch(nanMask)); + acc1 = Vector512.MaxNative(acc1, current1); + acc2 = Vector512.MaxNative(acc2, current2); + signBits &= current1 & current2; } + + nan |= Vector512.IsNaN(current1) | Vector512.IsNaN(current2); + i += 2 * Vector512.Count; + } + + acc1 = IsNativeMin() ? Vector512.MinNative(acc1, acc2) : Vector512.MaxNative(acc1, acc2); + } + + if (i != length) + { + Vector512 current = Vector512.LoadUnsafe(ref xRef, (uint)i); + if (IsNativeMin()) + { + acc1 = Vector512.MinNative(acc1, current); + signBits |= current; + } + else + { + acc1 = Vector512.MaxNative(acc1, current); + signBits &= current; } - int oneVectorFromEnd = x.Length - Vector256.Count; - int i = Vector256.Count; + nan |= Vector512.IsNaN(current); + } + + anyNaN = nan != Vector512.Zero; + anyOppositeSignBit = IsNativeMin() ? + signBits.ExtractMostSignificantBits() != 0 : + (~signBits).ExtractMostSignificantBits() != 0; + + return IsNativeMin() ? + HorizontalAggregate>(acc1) : + HorizontalAggregate>(acc1); + } + + /// Returns the first NaN among the elements at , a whole number of 512-bit vectors that contains one. + [MethodImpl(MethodImplOptions.NoInlining)] // cold: called at most once per reduction; keeps the caller within the inlining budget + private static T FirstNaN512(ref T xRef, int length) + where T : INumberBase + { + Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); + Debug.Assert(length >= Vector512.Count && length % Vector512.Count == 0); + + for (int i = 0; i < length; i += Vector512.Count) + { + Vector512 current = Vector512.LoadUnsafe(ref xRef, (uint)i); + Vector512 nanMask = Vector512.IsNaN(current); + if (nanMask != Vector512.Zero) + { + return current.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + Debug.Fail("The block must contain a NaN."); + return default!; + } + + /// The 256-bit path of : a block reduction of the whole vectors followed by one final vector that overlaps the last whole one. + /// + /// Every block of up to vectors is reduced with two independent accumulators and a single NaN decision + /// (the OR of the elements' NaN masks), so the hot loop contains no branch on the data. Blocks are visited in order and the running + /// result is only combined after a block has been checked, so the first NaN of the input is the one returned. + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per reduction; its own inlining budget keeps the block reduction and the horizontal aggregates inlined + private static T MinMaxVectorized256(ReadOnlySpan x) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(Vector256.IsHardwareAccelerated && Vector256.IsSupported); + Debug.Assert(x.Length >= Vector256.Count); + + ref T xRef = ref MemoryMarshal.GetReference(x); + int length = x.Length; + int blockSize = MinMaxBlockVectors * Vector256.Count; + int wholeVectorsLength = length - (length % Vector256.Count); + + // Reduce the whole vectors block by block. Starting from the first element is harmless: it is part of the first block, + // and combining an element with a result that already accounts for it does not change that result. + T result = xRef; + for (int i = 0; i < wholeVectorsLength; i += blockSize) + { + int blockLength = Math.Min(blockSize, wholeVectorsLength - i); + T blockResult; + bool anyNaN; - // Aggregate additional vectors into the result as long as there's at least one full vector left to process. - while (i <= oneVectorFromEnd) + if (IsNativeMinMax()) { - // Load the next vector, and early exit on NaN. - current = Vector256.LoadUnsafe(ref xRef, (uint)i); + blockResult = BlockReduceNative256(ref Unsafe.Add(ref xRef, i), blockLength, out anyNaN, out bool anyOppositeSignBit); + if (anyNaN) + { + return FirstNaN256(ref Unsafe.Add(ref xRef, i), blockLength); + } + blockResult = FixUpNativeSignedZero(blockResult, anyOppositeSignBit); + } + else + { + blockResult = BlockReduce256(ref Unsafe.Add(ref xRef, i), blockLength, out anyNaN); if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - // Check for NaNs - nanMask = ~Vector256.Equals(current, current); - if (nanMask != Vector256.Zero) + if (anyNaN) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return FirstNaN256(ref Unsafe.Add(ref xRef, i), blockLength); } } - - result = TMinMaxOperator.Invoke(result, current); - i += Vector256.Count; } - // If any elements remain, handle them in one final vector. - if (i != x.Length) + result = TMinMaxOperator.Invoke(result, blockResult); + } + + // If any elements remain, handle them in one final vector. Its NaN check does not change which NaN is returned: + // a NaN among the elements it revisits would already have been returned by their block. + if (wholeVectorsLength != length) + { + Vector256 last = Vector256.LoadUnsafe(ref xRef, (uint)(length - Vector256.Count)); + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - current = Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256.Count)); + Vector256 nanMask = Vector256.IsNaN(last); + if (nanMask != Vector256.Zero) + { + return last.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + result = TMinMaxOperator.Invoke(result, TMinMaxOperator.Invoke(last)); + } + + return result; + } + /// Reduces the elements at , a whole number of 256-bit vectors, with . + /// The first element of the block. + /// The number of elements in the block, a positive multiple of . + /// + /// For and , whether any element is NaN, in which case the returned value is meaningless. + /// It is gathered from the elements rather than from the result because the Number operators discard NaN. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T BlockReduce256(ref T xRef, int length, out bool anyNaN) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(length >= Vector256.Count && length % Vector256.Count == 0); + Vector256 acc1 = Vector256.LoadUnsafe(ref xRef); + Vector256 nan = Vector256.IsNaN(acc1); + int i = Vector256.Count; + + if (length >= 2 * Vector256.Count) + { + Vector256 acc2 = Vector256.LoadUnsafe(ref xRef, (uint)i); + nan |= Vector256.IsNaN(acc2); + i += Vector256.Count; + + int twoVectorsFromEnd = length - (2 * Vector256.Count); + while (i <= twoVectorsFromEnd) + { + Vector256 current1 = Vector256.LoadUnsafe(ref xRef, (uint)i); + Vector256 current2 = Vector256.LoadUnsafe(ref xRef, (uint)(i + Vector256.Count)); + acc1 = TMinMaxOperator.Invoke(acc1, current1); + acc2 = TMinMaxOperator.Invoke(acc2, current2); if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - // Check for NaNs - nanMask = ~Vector256.Equals(current, current); - if (nanMask != Vector256.Zero) - { - return current.GetElement(IndexOfFirstMatch(nanMask)); - } + nan |= Vector256.IsNaN(current1) | Vector256.IsNaN(current2); } - result = TMinMaxOperator.Invoke(result, current); + i += 2 * Vector256.Count; } - // Aggregate the lanes in the vector to create the final scalar result. - return TMinMaxOperator.Invoke(result); + acc1 = TMinMaxOperator.Invoke(acc1, acc2); } - if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && x.Length >= Vector128.Count) + if (i != length) { - ref T xRef = ref MemoryMarshal.GetReference(x); + Vector256 current = Vector256.LoadUnsafe(ref xRef, (uint)i); + acc1 = TMinMaxOperator.Invoke(acc1, current); + nan |= Vector256.IsNaN(current); + } - // Load the first vector as the initial set of results, and bail immediately - // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector128 result = Vector128.LoadUnsafe(ref xRef, 0); - Vector128 current; + anyNaN = (typeof(T) == typeof(float) || typeof(T) == typeof(double)) && nan != Vector256.Zero; + return TMinMaxOperator.Invoke(acc1); + } - Vector128 nanMask; - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + /// + /// Reduces the elements at , a whole number of 256-bit vectors, with the native vector + /// minimum/maximum, gathering alongside the two facts that let the caller restore the IEEE 754:2019 result (see ). + /// + /// The first element of the block. + /// The number of elements in the block, a positive multiple of . + /// Whether any element is NaN, in which case the returned value is meaningless. + /// For a minimum, whether any element has its sign bit set; for a maximum, whether any has it clear. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T BlockReduceNative256(ref T xRef, int length, out bool anyNaN, out bool anyOppositeSignBit) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(IsNativeMinMax()); + Debug.Assert(length >= Vector256.Count && length % Vector256.Count == 0); + + // For a minimum, OR the elements together so that any set sign bit survives; for a maximum, AND them so that any clear one does. + Vector256 acc1 = Vector256.LoadUnsafe(ref xRef); + Vector256 nan = Vector256.IsNaN(acc1); + Vector256 signBits = acc1; + int i = Vector256.Count; + + if (length >= 2 * Vector256.Count) + { + Vector256 acc2 = Vector256.LoadUnsafe(ref xRef, (uint)i); + nan |= Vector256.IsNaN(acc2); + signBits = IsNativeMin() ? signBits | acc2 : signBits & acc2; + i += Vector256.Count; + + int twoVectorsFromEnd = length - (2 * Vector256.Count); + while (i <= twoVectorsFromEnd) { - // Check for NaNs - nanMask = Vector128.IsNaN(result); - if (nanMask != Vector128.Zero) + Vector256 current1 = Vector256.LoadUnsafe(ref xRef, (uint)i); + Vector256 current2 = Vector256.LoadUnsafe(ref xRef, (uint)(i + Vector256.Count)); + if (IsNativeMin()) + { + acc1 = Vector256.MinNative(acc1, current1); + acc2 = Vector256.MinNative(acc2, current2); + signBits |= current1 | current2; + } + else { - return result.GetElement(IndexOfFirstMatch(nanMask)); + acc1 = Vector256.MaxNative(acc1, current1); + acc2 = Vector256.MaxNative(acc2, current2); + signBits &= current1 & current2; } + + nan |= Vector256.IsNaN(current1) | Vector256.IsNaN(current2); + i += 2 * Vector256.Count; } - int oneVectorFromEnd = x.Length - Vector128.Count; - int i = Vector128.Count; + acc1 = IsNativeMin() ? Vector256.MinNative(acc1, acc2) : Vector256.MaxNative(acc1, acc2); + } - // Aggregate additional vectors into the result as long as there's at least one full vector left to process. - while (i <= oneVectorFromEnd) + if (i != length) + { + Vector256 current = Vector256.LoadUnsafe(ref xRef, (uint)i); + if (IsNativeMin()) + { + acc1 = Vector256.MinNative(acc1, current); + signBits |= current; + } + else { - // Load the next vector, and early exit on NaN. - current = Vector128.LoadUnsafe(ref xRef, (uint)i); + acc1 = Vector256.MaxNative(acc1, current); + signBits &= current; + } + + nan |= Vector256.IsNaN(current); + } + + anyNaN = nan != Vector256.Zero; + anyOppositeSignBit = IsNativeMin() ? + signBits.ExtractMostSignificantBits() != 0 : + (~signBits).ExtractMostSignificantBits() != 0; + return IsNativeMin() ? + HorizontalAggregate>(acc1) : + HorizontalAggregate>(acc1); + } + + /// Returns the first NaN among the elements at , a whole number of 256-bit vectors that contains one. + [MethodImpl(MethodImplOptions.NoInlining)] // cold: called at most once per reduction; keeps the caller within the inlining budget + private static T FirstNaN256(ref T xRef, int length) + where T : INumberBase + { + Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); + Debug.Assert(length >= Vector256.Count && length % Vector256.Count == 0); + + for (int i = 0; i < length; i += Vector256.Count) + { + Vector256 current = Vector256.LoadUnsafe(ref xRef, (uint)i); + Vector256 nanMask = Vector256.IsNaN(current); + if (nanMask != Vector256.Zero) + { + return current.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + Debug.Fail("The block must contain a NaN."); + return default!; + } + + /// The 128-bit path of : a block reduction of the whole vectors followed by one final vector that overlaps the last whole one. + /// + /// Every block of up to vectors is reduced with two independent accumulators and a single NaN decision + /// (the OR of the elements' NaN masks), so the hot loop contains no branch on the data. Blocks are visited in order and the running + /// result is only combined after a block has been checked, so the first NaN of the input is the one returned. + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per reduction; its own inlining budget keeps the block reduction and the horizontal aggregates inlined + private static T MinMaxVectorized128(ReadOnlySpan x) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(Vector128.IsHardwareAccelerated && Vector128.IsSupported); + Debug.Assert(x.Length >= Vector128.Count); + + ref T xRef = ref MemoryMarshal.GetReference(x); + int length = x.Length; + int blockSize = MinMaxBlockVectors * Vector128.Count; + int wholeVectorsLength = length - (length % Vector128.Count); + + // Reduce the whole vectors block by block. Starting from the first element is harmless: it is part of the first block, + // and combining an element with a result that already accounts for it does not change that result. + T result = xRef; + for (int i = 0; i < wholeVectorsLength; i += blockSize) + { + int blockLength = Math.Min(blockSize, wholeVectorsLength - i); + T blockResult; + bool anyNaN; + + if (IsNativeMinMax()) + { + blockResult = BlockReduceNative128(ref Unsafe.Add(ref xRef, i), blockLength, out anyNaN, out bool anyOppositeSignBit); + if (anyNaN) + { + return FirstNaN128(ref Unsafe.Add(ref xRef, i), blockLength); + } + + blockResult = FixUpNativeSignedZero(blockResult, anyOppositeSignBit); + } + else + { + blockResult = BlockReduce128(ref Unsafe.Add(ref xRef, i), blockLength, out anyNaN); if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - // Check for NaNs - nanMask = Vector128.IsNaN(current); - if (nanMask != Vector128.Zero) + if (anyNaN) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return FirstNaN128(ref Unsafe.Add(ref xRef, i), blockLength); } } - - result = TMinMaxOperator.Invoke(result, current); - i += Vector128.Count; } - // If any elements remain, handle them in one final vector. - if (i != x.Length) + result = TMinMaxOperator.Invoke(result, blockResult); + } + + // If any elements remain, handle them in one final vector. Its NaN check does not change which NaN is returned: + // a NaN among the elements it revisits would already have been returned by their block. + if (wholeVectorsLength != length) + { + Vector128 last = Vector128.LoadUnsafe(ref xRef, (uint)(length - Vector128.Count)); + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - current = Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128.Count)); + Vector128 nanMask = Vector128.IsNaN(last); + if (nanMask != Vector128.Zero) + { + return last.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + result = TMinMaxOperator.Invoke(result, TMinMaxOperator.Invoke(last)); + } + + return result; + } + /// Reduces the elements at , a whole number of 128-bit vectors, with . + /// The first element of the block. + /// The number of elements in the block, a positive multiple of . + /// + /// For and , whether any element is NaN, in which case the returned value is meaningless. + /// It is gathered from the elements rather than from the result because the Number operators discard NaN. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T BlockReduce128(ref T xRef, int length, out bool anyNaN) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(length >= Vector128.Count && length % Vector128.Count == 0); + + Vector128 acc1 = Vector128.LoadUnsafe(ref xRef); + Vector128 nan = Vector128.IsNaN(acc1); + int i = Vector128.Count; + + if (length >= 2 * Vector128.Count) + { + Vector128 acc2 = Vector128.LoadUnsafe(ref xRef, (uint)i); + nan |= Vector128.IsNaN(acc2); + i += Vector128.Count; + + int twoVectorsFromEnd = length - (2 * Vector128.Count); + while (i <= twoVectorsFromEnd) + { + Vector128 current1 = Vector128.LoadUnsafe(ref xRef, (uint)i); + Vector128 current2 = Vector128.LoadUnsafe(ref xRef, (uint)(i + Vector128.Count)); + acc1 = TMinMaxOperator.Invoke(acc1, current1); + acc2 = TMinMaxOperator.Invoke(acc2, current2); if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - // Check for NaNs - nanMask = Vector128.IsNaN(current); - if (nanMask != Vector128.Zero) - { - return current.GetElement(IndexOfFirstMatch(nanMask)); - } + nan |= Vector128.IsNaN(current1) | Vector128.IsNaN(current2); } - result = TMinMaxOperator.Invoke(result, current); + i += 2 * Vector128.Count; } - // Aggregate the lanes in the vector to create the final scalar result. - return TMinMaxOperator.Invoke(result); + acc1 = TMinMaxOperator.Invoke(acc1, acc2); } - // Scalar path used when either vectorization is not supported or the input is too small to vectorize. - T curResult = x[0]; - if (T.IsNaN(curResult)) + if (i != length) { - return curResult; + Vector128 current = Vector128.LoadUnsafe(ref xRef, (uint)i); + acc1 = TMinMaxOperator.Invoke(acc1, current); + nan |= Vector128.IsNaN(current); } - for (int i = 1; i < x.Length; i++) + anyNaN = (typeof(T) == typeof(float) || typeof(T) == typeof(double)) && nan != Vector128.Zero; + return TMinMaxOperator.Invoke(acc1); + } + + /// + /// Reduces the elements at , a whole number of 128-bit vectors, with the native vector + /// minimum/maximum, gathering alongside the two facts that let the caller restore the IEEE 754:2019 result (see ). + /// + /// The first element of the block. + /// The number of elements in the block, a positive multiple of . + /// Whether any element is NaN, in which case the returned value is meaningless. + /// For a minimum, whether any element has its sign bit set; for a maximum, whether any has it clear. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T BlockReduceNative128(ref T xRef, int length, out bool anyNaN, out bool anyOppositeSignBit) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(IsNativeMinMax()); + Debug.Assert(length >= Vector128.Count && length % Vector128.Count == 0); + + // For a minimum, OR the elements together so that any set sign bit survives; for a maximum, AND them so that any clear one does. + Vector128 acc1 = Vector128.LoadUnsafe(ref xRef); + Vector128 nan = Vector128.IsNaN(acc1); + Vector128 signBits = acc1; + int i = Vector128.Count; + + if (length >= 2 * Vector128.Count) { - T current = x[i]; - if (T.IsNaN(current)) + Vector128 acc2 = Vector128.LoadUnsafe(ref xRef, (uint)i); + nan |= Vector128.IsNaN(acc2); + signBits = IsNativeMin() ? signBits | acc2 : signBits & acc2; + i += Vector128.Count; + + int twoVectorsFromEnd = length - (2 * Vector128.Count); + while (i <= twoVectorsFromEnd) { - return current; + Vector128 current1 = Vector128.LoadUnsafe(ref xRef, (uint)i); + Vector128 current2 = Vector128.LoadUnsafe(ref xRef, (uint)(i + Vector128.Count)); + if (IsNativeMin()) + { + acc1 = Vector128.MinNative(acc1, current1); + acc2 = Vector128.MinNative(acc2, current2); + signBits |= current1 | current2; + } + else + { + acc1 = Vector128.MaxNative(acc1, current1); + acc2 = Vector128.MaxNative(acc2, current2); + signBits &= current1 & current2; + } + + nan |= Vector128.IsNaN(current1) | Vector128.IsNaN(current2); + i += 2 * Vector128.Count; } - curResult = TMinMaxOperator.Invoke(curResult, current); + acc1 = IsNativeMin() ? Vector128.MinNative(acc1, acc2) : Vector128.MaxNative(acc1, acc2); } - return curResult; + if (i != length) + { + Vector128 current = Vector128.LoadUnsafe(ref xRef, (uint)i); + if (IsNativeMin()) + { + acc1 = Vector128.MinNative(acc1, current); + signBits |= current; + } + else + { + acc1 = Vector128.MaxNative(acc1, current); + signBits &= current; + } + + nan |= Vector128.IsNaN(current); + } + + anyNaN = nan != Vector128.Zero; + anyOppositeSignBit = IsNativeMin() ? + signBits.ExtractMostSignificantBits() != 0 : + (~signBits).ExtractMostSignificantBits() != 0; + + return IsNativeMin() ? + HorizontalAggregate>(acc1) : + HorizontalAggregate>(acc1); + } + + /// Returns the first NaN among the elements at , a whole number of 128-bit vectors that contains one. + [MethodImpl(MethodImplOptions.NoInlining)] // cold: called at most once per reduction; keeps the caller within the inlining budget + private static T FirstNaN128(ref T xRef, int length) + where T : INumberBase + { + Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); + Debug.Assert(length >= Vector128.Count && length % Vector128.Count == 0); + + for (int i = 0; i < length; i += Vector128.Count) + { + Vector128 current = Vector128.LoadUnsafe(ref xRef, (uint)i); + Vector128 nanMask = Vector128.IsNaN(current); + if (nanMask != Vector128.Zero) + { + return current.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + Debug.Fail("The block must contain a NaN."); + return default!; } } } diff --git a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs index 295955d9497ea6..97cdb73056deda 100644 --- a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs +++ b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs @@ -3115,6 +3115,56 @@ public void ScalarSpanDestination_ThrowsForOverlappingInputsWithOutputs(ScalarSp } #endregion + #region MinNumber/MaxNumber + // The multi-block coverage of Min_LongLengths_* for the Number variants, which share the block-reduction core. + [Fact] + public void MinNumber_LongLengths() => + AssertMinMaxLongLengths(TensorPrimitives.MinNumber, T.MinNumber); + + [Fact] + public void MaxNumber_LongLengths() => + AssertMinMaxLongLengths(TensorPrimitives.MaxNumber, T.MaxNumber); + + [Fact] + public void MinNumber_LongLengths_MinimumAtBlockBoundaries() => + AssertMinMaxLongLengthsValues(TensorPrimitives.MinNumber, fill: ConvertFromSingle(2), best: ConvertFromSingle(1)); + + [Fact] + public void MaxNumber_LongLengths_MaximumAtBlockBoundaries() => + AssertMinMaxLongLengthsValues(TensorPrimitives.MaxNumber, fill: ConvertFromSingle(1), best: ConvertFromSingle(2)); + + [Fact] + public void MinNumber_LongLengths_Negative0LesserThanPositive0() => + AssertMinMaxLongLengthsValues(TensorPrimitives.MinNumber, fill: Zero, best: NegativeZero); + + [Fact] + public void MaxNumber_LongLengths_Positive0GreaterThanNegative0() => + AssertMinMaxLongLengthsValues(TensorPrimitives.MaxNumber, fill: NegativeZero, best: Zero); + + [Fact] + public void MinNumber_LongLengths_AllZero() => + AssertMinMaxLongLengthsAllZero(TensorPrimitives.MinNumber); + + [Fact] + public void MaxNumber_LongLengths_AllZero() => + AssertMinMaxLongLengthsAllZero(TensorPrimitives.MaxNumber); + + // The vectorized float/double paths return the first NaN of the input, as Min/Max do; this pins that behavior across block boundaries. + [Fact] + public void MinNumber_LongLengths_FirstNaNReturned() + { + if (typeof(T) != typeof(float) && typeof(T) != typeof(double)) return; + AssertMinMaxLongLengthsFirstNaN(TensorPrimitives.MinNumber, fill: ConvertFromSingle(1), better: ConvertFromSingle(-1)); + } + + [Fact] + public void MaxNumber_LongLengths_FirstNaNReturned() + { + if (typeof(T) != typeof(float) && typeof(T) != typeof(double)) return; + AssertMinMaxLongLengthsFirstNaN(TensorPrimitives.MaxNumber, fill: ConvertFromSingle(1), better: ConvertFromSingle(2)); + } + #endregion + #region IsXx public static IEnumerable SpanDestinationIsFunctionsToTest() { @@ -3263,6 +3313,349 @@ public void SpanDestionIsAny_AllLengths(SpanIsAllAnyDelegate tensorPrimitivesMet Assert.Equal(expected, actual); }); } + + // The vectorized Any/All paths decide once per block of vectors. These place the element that settles the result at every + // block, vector and ragged boundary of long inputs, where the block loop, the ragged vectors and the final overlapping vector meet. + private static IEnumerable IsAnyAllLongPositions(int tensorLength) + { + var positions = new List { 0, 1, tensorLength / 2, tensorLength - 2, tensorLength - 1 }; + foreach (int vectorBytes in new[] { 16, 32, 64 }) + { + int vector = vectorBytes / sizeof(T), block = 32 * vector; + int afterLastWholeVector = tensorLength - (tensorLength % vector), afterLastWholeBlock = tensorLength - (tensorLength % block); + positions.AddRange([vector - 1, vector, vector + 1, block - 1, block, block + 1, (2 * block) - 1, 2 * block, + afterLastWholeVector - 1, afterLastWholeVector, afterLastWholeBlock - 1, afterLastWholeBlock]); + } + + return positions.Where(i => i >= 0 && i < tensorLength).Distinct(); + } + + /// + /// Filled with the aggregation must return for every long length; with + /// placed at any boundary position, alone or together with one at the end, it must return the opposite. + /// + private void AssertIsAnyAllLongLengths(SpanIsAllAnyDelegate tensorPrimitivesMethod, T fill, T hit, bool expectedWithoutHit) + { + Assert.All(s_minMaxLongLengths, tensorLength => + { + using BoundedMemory x = CreateTensor(tensorLength); + x.Span.Fill(fill); + Assert.Equal(expectedWithoutHit, tensorPrimitivesMethod(x)); + + foreach (int position in IsAnyAllLongPositions(tensorLength)) + { + x[position] = hit; + Assert.Equal(!expectedWithoutHit, tensorPrimitivesMethod(x)); + + x[tensorLength - 1] = hit; + Assert.Equal(!expectedWithoutHit, tensorPrimitivesMethod(x)); + + x[position] = fill; + x[tensorLength - 1] = fill; + } + }); + } + + [Fact] + public void IsNaNAny_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNaNAny, fill: ConvertFromSingle(1), hit: ConvertFromSingle(float.NaN), expectedWithoutHit: false); + } + + [Fact] + public void IsNaNAll_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNaNAll, fill: ConvertFromSingle(float.NaN), hit: ConvertFromSingle(1), expectedWithoutHit: true); + } + + [Fact] + public void IsFiniteAll_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAll, fill: ConvertFromSingle(1), hit: ConvertFromSingle(float.PositiveInfinity), expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAll, fill: ConvertFromSingle(-1), hit: ConvertFromSingle(float.NaN), expectedWithoutHit: true); + } + + [Fact] + public void IsFiniteAny_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAny, fill: ConvertFromSingle(float.NegativeInfinity), hit: Zero, expectedWithoutHit: false); + } + + [Fact] + public void IsNegativeAny_LongLengths() + { + if (!HasNegativeValues) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNegativeAny, fill: ConvertFromSingle(1), hit: ConvertFromSingle(-1), expectedWithoutHit: false); + } + + [Fact] + public void IsPositiveAny_LongLengths() + { + if (!HasNegativeValues) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsPositiveAny, fill: ConvertFromSingle(-1), hit: ConvertFromSingle(1), expectedWithoutHit: false); + } + + [Fact] + public void IsZeroAll_LongLengths() => + AssertIsAnyAllLongLengths(TensorPrimitives.IsZeroAll, fill: Zero, hit: ConvertFromSingle(1), expectedWithoutHit: true); + + [Fact] + public void IsZeroAny_LongLengths() => + AssertIsAnyAllLongLengths(TensorPrimitives.IsZeroAny, fill: ConvertFromSingle(1), hit: Zero, expectedWithoutHit: false); + + // The vectorized Any/All paths of the classifications below decide from the bits of the elements (an unsigned minimum or maximum + // of the block's bit patterns compared against a threshold) rather than from the operator, so these place the values whose + // bits lie next to the thresholds: the sign bit alone (-0), the smallest subnormal, the largest subnormal, the smallest normal, + // the largest finite value, infinity and the NaN whose bits follow it. + [Fact] + public void IsNegativeAll_LongLengths() + { + if (!HasNegativeValues) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNegativeAll, fill: ConvertFromSingle(-1), hit: ConvertFromSingle(1), expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNegativeAll, fill: MinValue, hit: Zero, expectedWithoutHit: true); + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNegativeAll, fill: NegativeZero, hit: Zero, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNegativeAll, fill: -FirstNaN, hit: FirstNaN, expectedWithoutHit: true); + } + + [Fact] + public void IsNegativeAny_LongLengths_SignBitOnly() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNegativeAny, fill: Zero, hit: NegativeZero, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNegativeAny, fill: FirstNaN, hit: -FirstNaN, expectedWithoutHit: false); + } + + [Fact] + public void IsPositiveAll_LongLengths() + { + if (!HasNegativeValues) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsPositiveAll, fill: ConvertFromSingle(1), hit: ConvertFromSingle(-1), expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsPositiveAll, fill: Zero, hit: MinValue, expectedWithoutHit: true); + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsPositiveAll, fill: Zero, hit: NegativeZero, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsPositiveAll, fill: FirstNaN, hit: -FirstNaN, expectedWithoutHit: true); + } + + [Fact] + public void IsPositiveAny_LongLengths_SignBitOnly() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsPositiveAny, fill: NegativeZero, hit: Zero, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsPositiveAny, fill: -FirstNaN, hit: FirstNaN, expectedWithoutHit: false); + } + + [Fact] + public void IsZeroAll_LongLengths_SignBitAndEpsilon() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsZeroAll, fill: NegativeZero, hit: SmallestSubnormal, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsZeroAll, fill: Zero, hit: -SmallestSubnormal, expectedWithoutHit: true); + } + + [Fact] + public void IsZeroAny_LongLengths_SignBitAndEpsilon() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsZeroAny, fill: SmallestSubnormal, hit: NegativeZero, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsZeroAny, fill: NaN, hit: Zero, expectedWithoutHit: false); + } + + [Fact] + public void IsNaNAny_LongLengths_Boundaries() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNaNAny, fill: PositiveInfinity, hit: FirstNaN, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNaNAny, fill: -PositiveInfinity, hit: -FirstNaN, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNaNAny, fill: T.MaxValue, hit: -NaN, expectedWithoutHit: false); + } + + [Fact] + public void IsNaNAll_LongLengths_Boundaries() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNaNAll, fill: FirstNaN, hit: PositiveInfinity, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNaNAll, fill: -NaN, hit: -PositiveInfinity, expectedWithoutHit: true); + } + + [Fact] + public void IsFiniteAll_LongLengths_Boundaries() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAll, fill: T.MaxValue, hit: -PositiveInfinity, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAll, fill: MinValue, hit: FirstNaN, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAll, fill: SmallestSubnormal, hit: -NaN, expectedWithoutHit: true); + } + + [Fact] + public void IsFiniteAny_LongLengths_Boundaries() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAny, fill: NaN, hit: T.MaxValue, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAny, fill: PositiveInfinity, hit: MinValue, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAny, fill: FirstNaN, hit: NegativeZero, expectedWithoutHit: false); + } + + [Fact] + public void IsRealNumberAll_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsRealNumberAll, fill: ConvertFromSingle(1), hit: FirstNaN, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsRealNumberAll, fill: PositiveInfinity, hit: NaN, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsRealNumberAll, fill: -PositiveInfinity, hit: -NaN, expectedWithoutHit: true); + } + + [Fact] + public void IsRealNumberAny_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsRealNumberAny, fill: NaN, hit: PositiveInfinity, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsRealNumberAny, fill: FirstNaN, hit: -PositiveInfinity, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsRealNumberAny, fill: -NaN, hit: ConvertFromSingle(1), expectedWithoutHit: false); + } + + [Fact] + public void IsNormalAll_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAll, fill: ConvertFromSingle(1), hit: Zero, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAll, fill: SmallestNormal, hit: LargestSubnormal, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAll, fill: -SmallestNormal, hit: -LargestSubnormal, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAll, fill: T.MaxValue, hit: PositiveInfinity, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAll, fill: MinValue, hit: NaN, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAll, fill: ConvertFromSingle(-1), hit: NegativeZero, expectedWithoutHit: true); + } + + [Fact] + public void IsNormalAny_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAny, fill: Zero, hit: ConvertFromSingle(1), expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAny, fill: LargestSubnormal, hit: SmallestNormal, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAny, fill: -LargestSubnormal, hit: -SmallestNormal, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAny, fill: PositiveInfinity, hit: T.MaxValue, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAny, fill: NaN, hit: MinValue, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAny, fill: NegativeZero, hit: ConvertFromSingle(-1), expectedWithoutHit: false); + } + + [Fact] + public void IsSubnormalAll_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAll, fill: SmallestSubnormal, hit: Zero, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAll, fill: -SmallestSubnormal, hit: NegativeZero, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAll, fill: LargestSubnormal, hit: SmallestNormal, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAll, fill: -LargestSubnormal, hit: -SmallestNormal, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAll, fill: SmallestSubnormal, hit: NaN, expectedWithoutHit: true); + } + + [Fact] + public void IsSubnormalAny_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAny, fill: Zero, hit: SmallestSubnormal, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAny, fill: NegativeZero, hit: -SmallestSubnormal, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAny, fill: SmallestNormal, hit: LargestSubnormal, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAny, fill: -SmallestNormal, hit: -LargestSubnormal, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAny, fill: NaN, hit: SmallestSubnormal, expectedWithoutHit: false); + } + + /// The value of a floating-point with the given bits. + private static T FromBits(ulong bits) => + Unsafe.SizeOf() == 2 ? Unsafe.BitCast((ushort)bits) : + Unsafe.SizeOf() == 4 ? Unsafe.BitCast((uint)bits) : + Unsafe.BitCast(bits); + + private static (ulong PositiveInfinity, ulong SmallestNormal) FloatingPointBits => + Unsafe.SizeOf() == 2 ? (0x7C00ul, 0x0400ul) : + Unsafe.SizeOf() == 4 ? (0x7F80_0000ul, 0x0080_0000ul) : + (0x7FF0_0000_0000_0000ul, 0x0010_0000_0000_0000ul); + + private static T PositiveInfinity => FromBits(FloatingPointBits.PositiveInfinity); + /// The NaN whose bits follow those of positive infinity. + private static T FirstNaN => FromBits(FloatingPointBits.PositiveInfinity + 1); + private static T SmallestNormal => FromBits(FloatingPointBits.SmallestNormal); + private static T LargestSubnormal => FromBits(FloatingPointBits.SmallestNormal - 1); + private static T SmallestSubnormal => FromBits(1); + + public static IEnumerable IsAnyAllFunctionsToTest() + { + yield return Create(TensorPrimitives.IsCanonicalAny, TensorPrimitives.IsCanonicalAll, T.IsCanonical); + yield return Create(TensorPrimitives.IsComplexNumberAny, TensorPrimitives.IsComplexNumberAll, T.IsComplexNumber); + yield return Create(TensorPrimitives.IsEvenIntegerAny, TensorPrimitives.IsEvenIntegerAll, T.IsEvenInteger); + yield return Create(TensorPrimitives.IsFiniteAny, TensorPrimitives.IsFiniteAll, T.IsFinite); + yield return Create(TensorPrimitives.IsImaginaryNumberAny, TensorPrimitives.IsImaginaryNumberAll, T.IsImaginaryNumber); + yield return Create(TensorPrimitives.IsInfinityAny, TensorPrimitives.IsInfinityAll, T.IsInfinity); + yield return Create(TensorPrimitives.IsIntegerAny, TensorPrimitives.IsIntegerAll, T.IsInteger); + yield return Create(TensorPrimitives.IsNaNAny, TensorPrimitives.IsNaNAll, T.IsNaN); + yield return Create(TensorPrimitives.IsNegativeAny, TensorPrimitives.IsNegativeAll, T.IsNegative); + yield return Create(TensorPrimitives.IsNegativeInfinityAny, TensorPrimitives.IsNegativeInfinityAll, T.IsNegativeInfinity); + yield return Create(TensorPrimitives.IsNormalAny, TensorPrimitives.IsNormalAll, T.IsNormal); + yield return Create(TensorPrimitives.IsOddIntegerAny, TensorPrimitives.IsOddIntegerAll, T.IsOddInteger); + yield return Create(TensorPrimitives.IsPositiveAny, TensorPrimitives.IsPositiveAll, T.IsPositive); + yield return Create(TensorPrimitives.IsPositiveInfinityAny, TensorPrimitives.IsPositiveInfinityAll, T.IsPositiveInfinity); + yield return Create(TensorPrimitives.IsRealNumberAny, TensorPrimitives.IsRealNumberAll, T.IsRealNumber); + yield return Create(TensorPrimitives.IsSubnormalAny, TensorPrimitives.IsSubnormalAll, T.IsSubnormal); + yield return Create(TensorPrimitives.IsZeroAny, TensorPrimitives.IsZeroAll, T.IsZero); + + static object[] Create(SpanIsAllAnyDelegate anyMethod, SpanIsAllAnyDelegate allMethod, Func predicate) + => new object[] { anyMethod, allMethod, predicate }; + } + + /// + /// Checks Any and All against the scalar predicate for long spans filled with each special value, and for a span filled with one + /// special value in which another one is placed at every block, vector and ragged boundary. + /// + [Theory] + [MemberData(nameof(IsAnyAllFunctionsToTest))] + public void IsAnyAll_SpecialValues(SpanIsAllAnyDelegate anyMethod, SpanIsAllAnyDelegate allMethod, Func predicate) + { + List values = [Zero, One, NegativeOne, MinValue, T.MaxValue, ConvertFromSingle(2), ConvertFromSingle(3), .. GetSpecialValues()]; + if (IsFloatingPoint) + { + values.AddRange([NegativeZero, FirstNaN, -FirstNaN, SmallestNormal, -SmallestNormal, LargestSubnormal, -LargestSubnormal, SmallestSubnormal, -SmallestSubnormal]); + } + + Assert.All(s_minMaxLongLengths, tensorLength => + { + using BoundedMemory x = CreateTensor(tensorLength); + foreach (T value in values) + { + x.Span.Fill(value); + Assert.Equal(predicate(value), anyMethod(x)); + Assert.Equal(predicate(value), allMethod(x)); + } + }); + + // Two lengths that cover several blocks of every vector width, with a ragged end. + Assert.All(new[] { 1025, 4097 }, tensorLength => + { + using BoundedMemory x = CreateTensor(tensorLength); + foreach (T fill in values) + { + foreach (T hit in values) + { + if (predicate(hit) == predicate(fill)) + { + continue; + } + + x.Span.Fill(fill); + foreach (int position in IsAnyAllLongPositions(tensorLength)) + { + x[position] = hit; + Assert.True(anyMethod(x)); + Assert.False(allMethod(x)); + x[position] = fill; + } + } + } + }); + } #endregion #region HammingDistance diff --git a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs index f428af07f79d64..f627412a2c9e99 100644 --- a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs +++ b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs @@ -1825,6 +1825,31 @@ public void Max_TwoTensors_ThrowsForOverlappingInputsWithOutputs() AssertExtensions.Throws("destination", () => Max(array.AsSpan(1, 2), array.AsSpan(4, 2), array.AsSpan(3, 2))); AssertExtensions.Throws("destination", () => Max(array.AsSpan(1, 2), array.AsSpan(4, 2), array.AsSpan(5, 2))); } + + // The same multi-block coverage as Min_LongLengths_* for the other reductions that share the block-reduction core. + [Fact] + public void Max_LongLengths() => + AssertMinMaxLongLengths(Max, Max); + + [Fact] + public void Max_LongLengths_MaximumAtBlockBoundaries() => + AssertMinMaxLongLengthsValues(Max, fill: ConvertFromSingle(1), best: ConvertFromSingle(2)); + + [Fact] + public void Max_LongLengths_FirstNaNReturned() => + AssertMinMaxLongLengthsFirstNaN(Max, fill: ConvertFromSingle(1), better: ConvertFromSingle(2)); // a larger value in an earlier block must not beat the NaN + + [Fact] + public void Max_LongLengths_Positive0GreaterThanNegative0() => + AssertMinMaxLongLengthsValues(Max, fill: NegativeZero, best: Zero); + + [Fact] + public void Max_LongLengths_AllZero() => + AssertMinMaxLongLengthsAllZero(Max); + + [Fact] + public void Max_LongLengths_TwoDistinctNaNs_ReturnsOneOfThem() => + AssertMinMaxLongLengthsTwoDistinctNaNs(Max, fill: ConvertFromSingle(2)); #endregion #region MaxMagnitude @@ -2010,6 +2035,36 @@ public void MaxMagnitude_TwoTensors_ThrowsForOverlappingInputsWithOutputs() AssertExtensions.Throws("destination", () => MaxMagnitude(array.AsSpan(1, 2), array.AsSpan(4, 2), array.AsSpan(3, 2))); AssertExtensions.Throws("destination", () => MaxMagnitude(array.AsSpan(1, 2), array.AsSpan(4, 2), array.AsSpan(5, 2))); } + + [Fact] + public void MaxMagnitude_LongLengths() => + AssertMinMaxLongLengths(MaxMagnitude, MaxMagnitude); + + [Fact] + public void MaxMagnitude_LongLengths_MaximumAtBlockBoundaries() + { + AssertMinMaxLongLengthsValues(MaxMagnitude, fill: ConvertFromSingle(1), best: ConvertFromSingle(2)); + if (HasNegativeValues) + { + AssertMinMaxLongLengthsValues(MaxMagnitude, fill: ConvertFromSingle(1), best: ConvertFromSingle(-2)); + } + } + + [Fact] + public void MaxMagnitude_LongLengths_FirstNaNReturned() => + AssertMinMaxLongLengthsFirstNaN(MaxMagnitude, fill: ConvertFromSingle(1), better: ConvertFromSingle(-2)); + + [Fact] + public void MaxMagnitude_LongLengths_Positive0GreaterThanNegative0() => + AssertMinMaxLongLengthsValues(MaxMagnitude, fill: NegativeZero, best: Zero); + + [Fact] + public void MaxMagnitude_LongLengths_AllZero() => + AssertMinMaxLongLengthsAllZero(MaxMagnitude); + + [Fact] + public void MaxMagnitude_LongLengths_TwoDistinctNaNs_ReturnsOneOfThem() => + AssertMinMaxLongLengthsTwoDistinctNaNs(MaxMagnitude, fill: ConvertFromSingle(-2)); #endregion #region Min @@ -2195,6 +2250,197 @@ public void Min_TwoTensors_ThrowsForOverlappingInputsWithOutputs() AssertExtensions.Throws("destination", () => Min(array.AsSpan(1, 2), array.AsSpan(4, 2), array.AsSpan(3, 2))); AssertExtensions.Throws("destination", () => Min(array.AsSpan(1, 2), array.AsSpan(4, 2), array.AsSpan(5, 2))); } + + // Lengths that span several blocks of the block reduction behind Min/Max/MinMagnitude/MaxMagnitude/MinNumber/MaxNumber + // (Helpers.TensorLengths stops at 256). + protected static readonly int[] s_minMaxLongLengths = [255, 256, 257, 511, 512, 513, 1023, 1024, 1025, 2047, 2048, 2049, 4097, 65539]; + + protected static IEnumerable MinMaxLongPositions(int tensorLength) => + new[] { 0, 1, 15, 16, 17, 255, 256, 257, 511, 512, 513, tensorLength / 2, tensorLength - 2, tensorLength - 1 }.Where(i => i < tensorLength).Distinct(); + + protected delegate T MinMaxReduction(ReadOnlySpan x); + + /// Whether represents -1 (the magnitude tests place a negative value only then). + protected bool HasNegativeValues => !ConvertFromSingle(-1).Equals(Zero) && Min(ConvertFromSingle(-1), Zero).Equals(ConvertFromSingle(-1)); + + /// A NaN whose payload identifies it, so that a test can tell which NaN of the input was returned. + protected T NaNWithPayload(int payload) => ConvertFromSingle(BitConverter.Int32BitsToSingle(0x7FC00000 | payload)); + + /// Asserts bit-for-bit equality, which unlike tells -0 from +0 and one NaN payload from another. + protected static void AssertEqualBits(T expected, T actual) + { + if (typeof(T) == typeof(float)) + { + Assert.Equal(BitConverter.SingleToInt32Bits((float)(object)expected), BitConverter.SingleToInt32Bits((float)(object)actual)); + } + else if (typeof(T) == typeof(double)) + { + Assert.Equal(BitConverter.DoubleToInt64Bits((double)(object)expected), BitConverter.DoubleToInt64Bits((double)(object)actual)); + } + else if (typeof(T) == typeof(Half)) + { + Assert.Equal(BitConverter.HalfToInt16Bits((Half)(object)expected), BitConverter.HalfToInt16Bits((Half)(object)actual)); + } + else + { + Assert.Equal(expected, actual); + } + } + + /// The reduction of random data of every long length must equal the scalar fold with the operator. + protected void AssertMinMaxLongLengths(MinMaxReduction reduction, Func scalar) + { + Assert.All(s_minMaxLongLengths, tensorLength => + { + using BoundedMemory x = CreateAndFillTensor(tensorLength); + T expected = x[0]; + for (int i = 1; i < tensorLength; i++) + { + expected = scalar(expected, x[i]); + } + + AssertEqualBits(expected, reduction(x.Span)); + }); + } + + /// Fills with and places at each block-boundary position; the reduction must return bit for bit. + protected void AssertMinMaxLongLengthsValues(MinMaxReduction reduction, T fill, T best) + { + Assert.All(s_minMaxLongLengths, tensorLength => + { + foreach (int position in MinMaxLongPositions(tensorLength)) + { + using BoundedMemory x = CreateTensor(tensorLength); + x.Span.Fill(fill); + x[position] = best; + AssertEqualBits(best, reduction(x.Span)); + } + }); + } + + /// + /// A NaN at any block-boundary position must be returned although a better value sits in an earlier block and a second NaN at the end; + /// for float and double it must be that first NaN, payload included. + /// + protected void AssertMinMaxLongLengthsFirstNaN(MinMaxReduction reduction, T fill, T better) + { + if (!IsFloatingPoint) return; + + Assert.All(s_minMaxLongLengths, tensorLength => + { + foreach (int position in MinMaxLongPositions(tensorLength)) + { + using BoundedMemory x = CreateTensor(tensorLength); + x.Span.Fill(fill); + x[0] = better; + x[tensorLength - 1] = NaNWithPayload(2); + x[position] = NaNWithPayload(1); + + T actual = reduction(x.Span); + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + AssertEqualBits(NaNWithPayload(1), actual); + } + else + { + Assert.Equal(ConvertFromSingle(float.NaN), actual); + } + } + }); + } + + /// + /// A NaN whose payload survives narrowing to (the low 13 bits of a single's payload are truncated), + /// so that two of them stay distinguishable through every floating-point . + /// + protected T NaNWithWidePayload(int payload) => ConvertFromSingle(BitConverter.Int32BitsToSingle(0x7FC00000 | (payload << 13))); + + /// + /// Two NaNs with different payloads at block-boundary positions must reduce to one of those two NaNs, bit for bit. + /// Which one is only specified for float and double (the first); for Half the choice depends on how the lanes pair up, + /// but the result must still be an input NaN and not a canonical NaN or a value. + /// + protected void AssertMinMaxLongLengthsTwoDistinctNaNs(MinMaxReduction reduction, T fill) + { + if (!IsFloatingPoint) return; + + T first = NaNWithWidePayload(1), second = NaNWithWidePayload(2); + Assert.NotEqual(ToBits(first), ToBits(second)); + + Assert.All(s_minMaxLongLengths, tensorLength => + { + int[] positions = MinMaxLongPositions(tensorLength).ToArray(); + foreach (int i in positions) + { + foreach (int j in positions) + { + if (i == j) continue; + + using BoundedMemory x = CreateTensor(tensorLength); + x.Span.Fill(fill); + x[i] = first; + x[j] = second; + + T actual = reduction(x.Span); + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + AssertEqualBits(i < j ? first : second, actual); + } + else + { + long bits = ToBits(actual); + Assert.True(bits == ToBits(first) || bits == ToBits(second), + $"Length {tensorLength}, NaNs at {i} and {j}: got 0x{bits:X}, expected 0x{ToBits(first):X} or 0x{ToBits(second):X}"); + } + } + } + }); + } + + /// The bit pattern of a floating-point value, for comparisons that must tell NaN payloads apart. + protected static long ToBits(T value) => + typeof(T) == typeof(float) ? BitConverter.SingleToInt32Bits((float)(object)value) : + typeof(T) == typeof(double) ? BitConverter.DoubleToInt64Bits((double)(object)value) : + typeof(T) == typeof(Half) ? BitConverter.HalfToInt16Bits((Half)(object)value) : + typeof(T) == typeof(NFloat) ? BitConverter.DoubleToInt64Bits((double)(NFloat)(object)value) : + throw new NotSupportedException(typeof(T).Name); + + /// All +0 must reduce to +0 and all -0 to -0, bit for bit. + protected void AssertMinMaxLongLengthsAllZero(MinMaxReduction reduction) + { + Assert.All(s_minMaxLongLengths, tensorLength => + { + using BoundedMemory x = CreateTensor(tensorLength); + x.Span.Fill(Zero); + AssertEqualBits(Zero, reduction(x.Span)); + x.Span.Fill(NegativeZero); + AssertEqualBits(NegativeZero, reduction(x.Span)); + }); + } + + [Fact] + public void Min_LongLengths() => + AssertMinMaxLongLengths(Min, Min); + + [Fact] + public void Min_LongLengths_MinimumAtBlockBoundaries() => + AssertMinMaxLongLengthsValues(Min, fill: ConvertFromSingle(2), best: ConvertFromSingle(1)); + + [Fact] + public void Min_LongLengths_FirstNaNReturned() => + AssertMinMaxLongLengthsFirstNaN(Min, fill: ConvertFromSingle(1), better: ConvertFromSingle(-1)); // a smaller value in an earlier block must not beat the NaN + + [Fact] + public void Min_LongLengths_Negative0LesserThanPositive0() => + AssertMinMaxLongLengthsValues(Min, fill: Zero, best: NegativeZero); + + [Fact] + public void Min_LongLengths_AllZero() => + AssertMinMaxLongLengthsAllZero(Min); + + [Fact] + public void Min_LongLengths_TwoDistinctNaNs_ReturnsOneOfThem() => + AssertMinMaxLongLengthsTwoDistinctNaNs(Min, fill: ConvertFromSingle(-1)); // a smaller value must not beat either NaN #endregion #region MinMagnitude @@ -2378,6 +2624,36 @@ public void MinMagnitude_TwoTensors_ThrowsForOverlappingInputsWithOutputs() AssertExtensions.Throws("destination", () => MinMagnitude(array.AsSpan(1, 2), array.AsSpan(4, 2), array.AsSpan(3, 2))); AssertExtensions.Throws("destination", () => MinMagnitude(array.AsSpan(1, 2), array.AsSpan(4, 2), array.AsSpan(5, 2))); } + + [Fact] + public void MinMagnitude_LongLengths() => + AssertMinMaxLongLengths(MinMagnitude, MinMagnitude); + + [Fact] + public void MinMagnitude_LongLengths_MinimumAtBlockBoundaries() + { + AssertMinMaxLongLengthsValues(MinMagnitude, fill: ConvertFromSingle(2), best: ConvertFromSingle(1)); + if (HasNegativeValues) + { + AssertMinMaxLongLengthsValues(MinMagnitude, fill: ConvertFromSingle(2), best: ConvertFromSingle(-1)); + } + } + + [Fact] + public void MinMagnitude_LongLengths_FirstNaNReturned() => + AssertMinMaxLongLengthsFirstNaN(MinMagnitude, fill: ConvertFromSingle(2), better: ConvertFromSingle(-1)); + + [Fact] + public void MinMagnitude_LongLengths_Negative0LesserThanPositive0() => + AssertMinMaxLongLengthsValues(MinMagnitude, fill: Zero, best: NegativeZero); + + [Fact] + public void MinMagnitude_LongLengths_AllZero() => + AssertMinMaxLongLengthsAllZero(MinMagnitude); + + [Fact] + public void MinMagnitude_LongLengths_TwoDistinctNaNs_ReturnsOneOfThem() => + AssertMinMaxLongLengthsTwoDistinctNaNs(MinMagnitude, fill: ConvertFromSingle(-1)); #endregion #region Multiply