Skip to content

Vectorize Min and Max over spans of float and double #134409

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tahakocal wants to merge 9 commits into
base: main
Choose a base branch
from
+303 −16
Open

Vectorize Min and Max over spans of float and double #134409

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
acfb7e9
Compare a vector at a time when taking the minimum of a float span
tahakocal Sep 22, 2026
b4b160c
Compare a vector at a time when taking the maximum of a float span
tahakocal Sep 22, 2026
71d6ab9
Compare vectors instead of calling the comparer for float spans in Mi…
tahakocal Sep 22, 2026
a4dc15f
Load the vectors through the span instead of an unsafe reference
tahakocal Sep 22, 2026
62d6417
Load the vectors through the span instead of an unsafe reference
tahakocal Sep 22, 2026
73a88ba
Load the vectors through the span instead of an unsafe reference
tahakocal Sep 22, 2026
ad34356
Merge branches 'perf/linq-min-float' and 'perf/linq-max-float' into p…
tahakocal Sep 22, 2026
58612ce
Walk the span forward so the vector loads need no bounds check
tahakocal Sep 22, 2026
b9e1987
Accumulate the NaN lanes instead of testing them every iteration
tahakocal Sep 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 47 additions & 1 deletion src/libraries/System.Linq/src/System/Linq/Max.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System.Collections.Generic;
using System.Numerics;
using System.Runtime.Intrinsics;

namespace System.Linq
{
Expand Down Expand Up @@ -129,14 +130,59 @@ private static T MaxFloat<T>(this IEnumerable<T> source) where T : struct, IFloa
return span[^1];
}

for (value = span[i]; (uint)i < (uint)span.Length; i++)
value = span[i];

// Only worth vectorizing when a vector holds at least four elements: the NaN lanes
// have to be replaced before the comparison, and with two elements per vector that
// costs as much as it saves.
if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported &&
Vector128<T>.Count >= 4 && span.Length - i >= Vector128<T>.Count * 2)
{
Vector128<T> negativeInfinity = Vector128.Create(T.NegativeInfinity);
Vector128<T> best = Vector128.Create(value);
ReadOnlySpan<T> remaining = span.Slice(i);

while (remaining.Length >= Vector128<T>.Count)
{
// A NaN is never the maximum here, and Vector128.Max would propagate it.
Vector128<T> current = Vector128.Create(remaining);
best = Vector128.Max(best, Vector128.ConditionalSelect(Vector128.Equals(current, current), current, negativeInfinity));
remaining = remaining.Slice(Vector128<T>.Count);
}

i = span.Length - remaining.Length;

for (int lane = 0; lane < Vector128<T>.Count; lane++)
{
T candidate = best.GetElement(lane);
if (candidate > value)
{
value = candidate;
}
}
}

for (; (uint)i < (uint)span.Length; i++)
{
if (span[i] > value)
{
value = span[i];
}
}

// Negative and positive zero compare equal, so the reduction may have kept either
// one, while the sequential walk keeps the first of two equal values.
if (value == T.Zero)
{
foreach (T element in span)
{
if (element == T.Zero)
{
return element;
}
}
}

return value;
}

Expand Down
104 changes: 89 additions & 15 deletions src/libraries/System.Linq/src/System/Linq/Min.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System.Collections.Generic;
using System.Numerics;
using System.Runtime.Intrinsics;

namespace System.Linq
{
Expand Down Expand Up @@ -87,6 +88,93 @@ private static T MinIntegerEnumerator<T>(IEnumerable<T> source) where T : struct

public static double? Min(this IEnumerable<double?> source) => MinFloat(source);

private static T MinFloat<T>(ReadOnlySpan<T> span) where T : struct, IFloatingPointIeee754<T>
{
T value;
int i = 1;

// A vector's worth at a time, bailing out to the sequential walk as soon as a NaN
// appears, since the first NaN is the result and the walk already reports it.
if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && span.Length >= Vector128<T>.Count * 2)
{
Vector128<T> best = Vector128.Create(span);

if (~Vector128.Equals(best, best) == Vector128<T>.Zero)
{
ReadOnlySpan<T> remaining = span.Slice(Vector128<T>.Count);
Vector128<T> nanFound = Vector128<T>.Zero;

// Accumulating the NaN lanes and testing once keeps the loop branchless.
while (remaining.Length >= Vector128<T>.Count)
{
Vector128<T> current = Vector128.Create(remaining);
nanFound |= ~Vector128.Equals(current, current);
best = Vector128.Min(best, current);
remaining = remaining.Slice(Vector128<T>.Count);
}

int index = span.Length - remaining.Length;

if (nanFound == Vector128<T>.Zero)
{
value = best.GetElement(0);
for (int lane = 1; lane < Vector128<T>.Count; lane++)
{
T candidate = best.GetElement(lane);
if (candidate < value)
{
value = candidate;
}
}

for (i = index; (uint)i < (uint)span.Length; i++)
{
T current = span[i];
if (current < value)
{
value = current;
}
else if (T.IsNaN(current))
{
return current;
}
}

// Negative and positive zero compare equal, so the reduction may have kept
// either one, while the sequential walk keeps the first of two equal values.
if (value == T.Zero)
{
foreach (T element in span)
{
if (element == T.Zero)
{
return element;
}
}
}

return value;
}
}
}

value = span[0];
for (; (uint)i < (uint)span.Length; i++)
{
T current = span[i];
if (current < value)
{
value = current;
}
else if (T.IsNaN(current))
{
return current;
}
}

return value;
}

private static T MinFloat<T>(this IEnumerable<T> source) where T : struct, IFloatingPointIeee754<T>
{
T value;
Expand All @@ -103,21 +191,7 @@ private static T MinFloat<T>(this IEnumerable<T> source) where T : struct, IFloa
ThrowHelper.ThrowNoElementsException();
}

value = span[0];
for (int i = 1; (uint)i < (uint)span.Length; i++)
{
T current = span[i];
if (current < value)
{
value = current;
}
else if (T.IsNaN(current))
{
return current;
}
}

return value;
return MinFloat(span);
}

using IEnumerator<T> e = source.GetEnumerator();
Expand Down
167 changes: 167 additions & 0 deletions src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.MinMax.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ public static partial class MemoryExtensions
if (typeof(T) == typeof(nint)) return MinMaxInteger<T, nint, MinCalc<nint>>(span);
if (typeof(T) == typeof(Int128)) return MinMaxInteger<T, Int128, MinCalc<Int128>>(span);
if (typeof(T) == typeof(UInt128)) return MinMaxInteger<T, UInt128, MinCalc<UInt128>>(span);
if (typeof(T) == typeof(float)) return MinFloat<T, float>(span);
if (typeof(T) == typeof(double)) return MinFloat<T, double>(span);

return MinMax<T, MinDirection>(span, Comparer<T>.Default);
}
Expand Down Expand Up @@ -91,6 +93,8 @@ public static partial class MemoryExtensions
if (typeof(T) == typeof(nint)) return MinMaxInteger<T, nint, MaxCalc<nint>>(span);
if (typeof(T) == typeof(Int128)) return MinMaxInteger<T, Int128, MaxCalc<Int128>>(span);
if (typeof(T) == typeof(UInt128)) return MinMaxInteger<T, UInt128, MaxCalc<UInt128>>(span);
if (typeof(T) == typeof(float)) return MaxFloat<T, float>(span);
if (typeof(T) == typeof(double)) return MaxFloat<T, double>(span);

return MinMax<T, MaxDirection>(span, Comparer<T>.Default);
}
Expand Down Expand Up @@ -221,6 +225,169 @@ private interface IMinMaxCalc<T> where T : struct, IBinaryInteger<T>
public static Vector512<T> Compare(Vector512<T> left, Vector512<T> right) => Vector512.Max(left, right);
}

private static TOuter MinFloat<TOuter, TInner>(this ReadOnlySpan<TOuter> span)
where TInner : struct, IFloatingPointIeee754<TInner> =>
Unsafe.BitCast<TInner, TOuter>(MinFloat<TInner>(Unsafe.BitCast<ReadOnlySpan<TOuter>, ReadOnlySpan<TInner>>(span)));

private static TOuter MaxFloat<TOuter, TInner>(this ReadOnlySpan<TOuter> span)
where TInner : struct, IFloatingPointIeee754<TInner> =>
Unsafe.BitCast<TInner, TOuter>(MaxFloat<TInner>(Unsafe.BitCast<ReadOnlySpan<TOuter>, ReadOnlySpan<TInner>>(span)));

/// <remarks>
/// Comparer{T}.Default orders NaN below every value and treats negative and positive zero
/// as equal, keeping whichever of the two the span holds first. Both are preserved here.
/// </remarks>
private static T MinFloat<T>(this ReadOnlySpan<T> span) where T : struct, IFloatingPointIeee754<T>
{
if (span.IsEmpty)
{
ThrowHelper.ThrowInvalidOperationException(ExceptionResource.InvalidOperation_NoElements);
}

T value;
int i = 1;

if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && span.Length >= Vector128<T>.Count * 2)
{
Vector128<T> best = Vector128.Create(span);
Vector128<T> nanFound = ~Vector128.Equals(best, best);
ReadOnlySpan<T> remaining = span.Slice(Vector128<T>.Count);

while (remaining.Length >= Vector128<T>.Count)
{
Vector128<T> current = Vector128.Create(remaining);
nanFound |= ~Vector128.Equals(current, current);
best = Vector128.Min(best, current);
remaining = remaining.Slice(Vector128<T>.Count);
}

i = span.Length - remaining.Length;

if (nanFound != Vector128<T>.Zero)
{
foreach (T element in span)
{
if (T.IsNaN(element))
{
return element;
}
}
}

value = best.GetElement(0);
for (int lane = 1; lane < Vector128<T>.Count; lane++)
{
T candidate = best.GetElement(lane);
if (candidate < value)
{
value = candidate;
}
}
}
else
{
value = span[0];
}

for (; i < span.Length; i++)
{
T current = span[i];
if (T.IsNaN(current))
{
return current;
}

if (current < value)
{
value = current;
}
}

if (value == T.Zero)
{
foreach (T element in span)
{
if (element == T.Zero)
{
return element;
}
}
}

return value;
}

/// <inheritdoc cref="MinFloat{T}(ReadOnlySpan{T})"/>
private static T MaxFloat<T>(this ReadOnlySpan<T> span) where T : struct, IFloatingPointIeee754<T>
{
if (span.IsEmpty)
{
ThrowHelper.ThrowInvalidOperationException(ExceptionResource.InvalidOperation_NoElements);
}

int i = 0;
while (i < span.Length && T.IsNaN(span[i]))
{
i++;
}

if (i == span.Length)
{
return span[0];
}

T value = span[i];
i++;

if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && span.Length - i >= Vector128<T>.Count * 2)
{
Vector128<T> negativeInfinity = Vector128.Create(T.NegativeInfinity);
Vector128<T> best = Vector128.Create(value);
ReadOnlySpan<T> remaining = span.Slice(i);

while (remaining.Length >= Vector128<T>.Count)
{
// A NaN is never the maximum here, and Vector128.Max would propagate it.
Vector128<T> current = Vector128.Create(remaining);
best = Vector128.Max(best, Vector128.ConditionalSelect(Vector128.Equals(current, current), current, negativeInfinity));
remaining = remaining.Slice(Vector128<T>.Count);
}

i = span.Length - remaining.Length;

for (int lane = 0; lane < Vector128<T>.Count; lane++)
{
T candidate = best.GetElement(lane);
if (candidate > value)
{
value = candidate;
}
}
}

for (; i < span.Length; i++)
{
T current = span[i];
if (current > value)
{
value = current;
}
}

if (value == T.Zero)
{
foreach (T element in span)
{
if (element == T.Zero)
{
return element;
}
}
}

return value;
}

private static TOuter MinMaxInteger<TOuter, TInner, TMinMax>(this ReadOnlySpan<TOuter> span)
where TInner : struct, IBinaryInteger<TInner>
where TMinMax : IMinMaxCalc<TInner> =>
Expand Down
Loading