diff --git a/src/libraries/System.Linq/src/System/Linq/Max.cs b/src/libraries/System.Linq/src/System/Linq/Max.cs index 3a78d56dc35e6e..ec7c1735d604ce 100644 --- a/src/libraries/System.Linq/src/System/Linq/Max.cs +++ b/src/libraries/System.Linq/src/System/Linq/Max.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Numerics; +using System.Runtime.Intrinsics; namespace System.Linq { @@ -129,7 +130,39 @@ private static T MaxFloat(this IEnumerable source) where T : struct, IFloa return span[^1]; } - for (value = span[i]; (uint)i < (uint)span.Length; i++) + value = span[i]; + + // Only worth vectorizing when a vector holds at least four elements: the NaN lanes + // have to be replaced before the comparison, and with two elements per vector that + // costs as much as it saves. + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && + Vector128.Count >= 4 && span.Length - i >= Vector128.Count * 2) + { + Vector128 negativeInfinity = Vector128.Create(T.NegativeInfinity); + Vector128 best = Vector128.Create(value); + ReadOnlySpan remaining = span.Slice(i); + + while (remaining.Length >= Vector128.Count) + { + // A NaN is never the maximum here, and Vector128.Max would propagate it. + Vector128 current = Vector128.Create(remaining); + best = Vector128.Max(best, Vector128.ConditionalSelect(Vector128.Equals(current, current), current, negativeInfinity)); + remaining = remaining.Slice(Vector128.Count); + } + + i = span.Length - remaining.Length; + + for (int lane = 0; lane < Vector128.Count; lane++) + { + T candidate = best.GetElement(lane); + if (candidate > value) + { + value = candidate; + } + } + } + + for (; (uint)i < (uint)span.Length; i++) { if (span[i] > value) { @@ -137,6 +170,19 @@ private static T MaxFloat(this IEnumerable source) where T : struct, IFloa } } + // Negative and positive zero compare equal, so the reduction may have kept either + // one, while the sequential walk keeps the first of two equal values. + if (value == T.Zero) + { + foreach (T element in span) + { + if (element == T.Zero) + { + return element; + } + } + } + return value; } diff --git a/src/libraries/System.Linq/src/System/Linq/Min.cs b/src/libraries/System.Linq/src/System/Linq/Min.cs index 7cdc7abcda1664..3a349dae6e5e4d 100644 --- a/src/libraries/System.Linq/src/System/Linq/Min.cs +++ b/src/libraries/System.Linq/src/System/Linq/Min.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Numerics; +using System.Runtime.Intrinsics; namespace System.Linq { @@ -87,6 +88,93 @@ private static T MinIntegerEnumerator(IEnumerable source) where T : struct public static double? Min(this IEnumerable source) => MinFloat(source); + private static T MinFloat(ReadOnlySpan span) where T : struct, IFloatingPointIeee754 + { + T value; + int i = 1; + + // A vector's worth at a time, bailing out to the sequential walk as soon as a NaN + // appears, since the first NaN is the result and the walk already reports it. + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && span.Length >= Vector128.Count * 2) + { + Vector128 best = Vector128.Create(span); + + if (~Vector128.Equals(best, best) == Vector128.Zero) + { + ReadOnlySpan remaining = span.Slice(Vector128.Count); + Vector128 nanFound = Vector128.Zero; + + // Accumulating the NaN lanes and testing once keeps the loop branchless. + while (remaining.Length >= Vector128.Count) + { + Vector128 current = Vector128.Create(remaining); + nanFound |= ~Vector128.Equals(current, current); + best = Vector128.Min(best, current); + remaining = remaining.Slice(Vector128.Count); + } + + int index = span.Length - remaining.Length; + + if (nanFound == Vector128.Zero) + { + value = best.GetElement(0); + for (int lane = 1; lane < Vector128.Count; lane++) + { + T candidate = best.GetElement(lane); + if (candidate < value) + { + value = candidate; + } + } + + for (i = index; (uint)i < (uint)span.Length; i++) + { + T current = span[i]; + if (current < value) + { + value = current; + } + else if (T.IsNaN(current)) + { + return current; + } + } + + // Negative and positive zero compare equal, so the reduction may have kept + // either one, while the sequential walk keeps the first of two equal values. + if (value == T.Zero) + { + foreach (T element in span) + { + if (element == T.Zero) + { + return element; + } + } + } + + return value; + } + } + } + + value = span[0]; + for (; (uint)i < (uint)span.Length; i++) + { + T current = span[i]; + if (current < value) + { + value = current; + } + else if (T.IsNaN(current)) + { + return current; + } + } + + return value; + } + private static T MinFloat(this IEnumerable source) where T : struct, IFloatingPointIeee754 { T value; @@ -103,21 +191,7 @@ private static T MinFloat(this IEnumerable source) where T : struct, IFloa ThrowHelper.ThrowNoElementsException(); } - value = span[0]; - for (int i = 1; (uint)i < (uint)span.Length; i++) - { - T current = span[i]; - if (current < value) - { - value = current; - } - else if (T.IsNaN(current)) - { - return current; - } - } - - return value; + return MinFloat(span); } using IEnumerator e = source.GetEnumerator(); diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.MinMax.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.MinMax.cs index 04344cfc23f08b..6e2b48cea96eb1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.MinMax.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.MinMax.cs @@ -38,6 +38,8 @@ public static partial class MemoryExtensions if (typeof(T) == typeof(nint)) return MinMaxInteger>(span); if (typeof(T) == typeof(Int128)) return MinMaxInteger>(span); if (typeof(T) == typeof(UInt128)) return MinMaxInteger>(span); + if (typeof(T) == typeof(float)) return MinFloat(span); + if (typeof(T) == typeof(double)) return MinFloat(span); return MinMax(span, Comparer.Default); } @@ -91,6 +93,8 @@ public static partial class MemoryExtensions if (typeof(T) == typeof(nint)) return MinMaxInteger>(span); if (typeof(T) == typeof(Int128)) return MinMaxInteger>(span); if (typeof(T) == typeof(UInt128)) return MinMaxInteger>(span); + if (typeof(T) == typeof(float)) return MaxFloat(span); + if (typeof(T) == typeof(double)) return MaxFloat(span); return MinMax(span, Comparer.Default); } @@ -221,6 +225,169 @@ private interface IMinMaxCalc where T : struct, IBinaryInteger public static Vector512 Compare(Vector512 left, Vector512 right) => Vector512.Max(left, right); } + private static TOuter MinFloat(this ReadOnlySpan span) + where TInner : struct, IFloatingPointIeee754 => + Unsafe.BitCast(MinFloat(Unsafe.BitCast, ReadOnlySpan>(span))); + + private static TOuter MaxFloat(this ReadOnlySpan span) + where TInner : struct, IFloatingPointIeee754 => + Unsafe.BitCast(MaxFloat(Unsafe.BitCast, ReadOnlySpan>(span))); + + /// + /// Comparer{T}.Default orders NaN below every value and treats negative and positive zero + /// as equal, keeping whichever of the two the span holds first. Both are preserved here. + /// + private static T MinFloat(this ReadOnlySpan span) where T : struct, IFloatingPointIeee754 + { + if (span.IsEmpty) + { + ThrowHelper.ThrowInvalidOperationException(ExceptionResource.InvalidOperation_NoElements); + } + + T value; + int i = 1; + + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && span.Length >= Vector128.Count * 2) + { + Vector128 best = Vector128.Create(span); + Vector128 nanFound = ~Vector128.Equals(best, best); + ReadOnlySpan remaining = span.Slice(Vector128.Count); + + while (remaining.Length >= Vector128.Count) + { + Vector128 current = Vector128.Create(remaining); + nanFound |= ~Vector128.Equals(current, current); + best = Vector128.Min(best, current); + remaining = remaining.Slice(Vector128.Count); + } + + i = span.Length - remaining.Length; + + if (nanFound != Vector128.Zero) + { + foreach (T element in span) + { + if (T.IsNaN(element)) + { + return element; + } + } + } + + value = best.GetElement(0); + for (int lane = 1; lane < Vector128.Count; lane++) + { + T candidate = best.GetElement(lane); + if (candidate < value) + { + value = candidate; + } + } + } + else + { + value = span[0]; + } + + for (; i < span.Length; i++) + { + T current = span[i]; + if (T.IsNaN(current)) + { + return current; + } + + if (current < value) + { + value = current; + } + } + + if (value == T.Zero) + { + foreach (T element in span) + { + if (element == T.Zero) + { + return element; + } + } + } + + return value; + } + + /// + private static T MaxFloat(this ReadOnlySpan span) where T : struct, IFloatingPointIeee754 + { + if (span.IsEmpty) + { + ThrowHelper.ThrowInvalidOperationException(ExceptionResource.InvalidOperation_NoElements); + } + + int i = 0; + while (i < span.Length && T.IsNaN(span[i])) + { + i++; + } + + if (i == span.Length) + { + return span[0]; + } + + T value = span[i]; + i++; + + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && span.Length - i >= Vector128.Count * 2) + { + Vector128 negativeInfinity = Vector128.Create(T.NegativeInfinity); + Vector128 best = Vector128.Create(value); + ReadOnlySpan remaining = span.Slice(i); + + while (remaining.Length >= Vector128.Count) + { + // A NaN is never the maximum here, and Vector128.Max would propagate it. + Vector128 current = Vector128.Create(remaining); + best = Vector128.Max(best, Vector128.ConditionalSelect(Vector128.Equals(current, current), current, negativeInfinity)); + remaining = remaining.Slice(Vector128.Count); + } + + i = span.Length - remaining.Length; + + for (int lane = 0; lane < Vector128.Count; lane++) + { + T candidate = best.GetElement(lane); + if (candidate > value) + { + value = candidate; + } + } + } + + for (; i < span.Length; i++) + { + T current = span[i]; + if (current > value) + { + value = current; + } + } + + if (value == T.Zero) + { + foreach (T element in span) + { + if (element == T.Zero) + { + return element; + } + } + } + + return value; + } + private static TOuter MinMaxInteger(this ReadOnlySpan span) where TInner : struct, IBinaryInteger where TMinMax : IMinMaxCalc =>