From b8ab80321e25cdb148680f77f0772800896c1633 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 17 Sep 2026 11:17:12 -0700 Subject: [PATCH 1/2] Fix NLS ordinal casing of supplementary characters to agree with OrdinalIgnoreCase Under Windows NLS, ordinal casing (ToUpperOrdinal/ToLowerOrdinal) uses LCMapStringEx while ordinal comparison uses CompareStringOrdinal. These two Windows APIs disagree for some supplementary scalars: LCMapStringEx maps the Deseret upper/lower pairs (for example U+10428 to U+10400), but CompareStringOrdinal with IgnoreCase treats those pairs as unequal. That broke the canonicalization contract where ToUpperOrdinal(a) equals ToUpperOrdinal(b) if and only if OrdinalIgnoreCase considers a and b equal. The fix is NLS only and adds no casing tables. After casing, changed supplementary pairs are validated against CompareStringOrdinal and restored to their original scalar when the mapping moved them outside their OrdinalIgnoreCase class. ASCII keeps its fast path, pure BMP input adds a single vectorized surrogate scan, and unchanged surrogate runs are skipped in bulk. ICU mode is untouched. --- .../src/System/Globalization/Ordinal.cs | 117 +++++++++++++++++- .../src/System/Text/Rune.cs | 10 +- .../Globalization/OrdinalCasingTests.cs | 67 +++++++++- 3 files changed, 188 insertions(+), 6 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs index 459e771fc3d6af..4a5dd7423c51c0 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs @@ -8,6 +8,7 @@ using System.Runtime.InteropServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; +using System.Text; using System.Text.Unicode; namespace System.Globalization @@ -701,7 +702,7 @@ internal static int ToUpperOrdinal(ReadOnlySpan source, Span destina if (GlobalizationMode.UseNls) { - TextInfo.Invariant.ChangeCaseToUpper(source, destination); // this is the best so far for NLS. + ChangeCaseNlsOrdinal(source, destination, toUpper: true); return source.Length; } @@ -727,7 +728,7 @@ internal static int ToLowerOrdinal(ReadOnlySpan source, Span destina if (GlobalizationMode.UseNls) { - TextInfo.Invariant.ChangeCaseToLower(source, destination); // this is the best so far for NLS. + ChangeCaseNlsOrdinal(source, destination, toUpper: false); PreserveOrdinalLowerCasingClass(source, destination); return source.Length; } @@ -736,6 +737,118 @@ internal static int ToLowerOrdinal(ReadOnlySpan source, Span destina return source.Length; } + private static void ChangeCaseNlsOrdinal(ReadOnlySpan source, Span destination, bool toUpper) + { + Debug.Assert(GlobalizationMode.UseNls); + + OperationStatus operationStatus = toUpper + ? Ascii.ToUpper(source, destination, out int charsConsumed) + : Ascii.ToLower(source, destination, out charsConsumed); + + if (operationStatus != OperationStatus.InvalidData) + { + Debug.Assert(operationStatus == OperationStatus.Done); + return; + } + + source = source.Slice(charsConsumed); + destination = destination.Slice(charsConsumed); + + if (toUpper) + { + TextInfo.Invariant.ChangeCaseToUpper(source, destination); + } + else + { + TextInfo.Invariant.ChangeCaseToLower(source, destination); + } + + PreserveNlsOrdinalCasingClass(source, destination); + } + + private static void PreserveNlsOrdinalCasingClass(ReadOnlySpan source, Span destination) + { + Debug.Assert(GlobalizationMode.UseNls); + Debug.Assert(source.Length == destination.Length); + + int i = source.IndexOfAnyInRange('\uD800', '\uDBFF'); + if (i < 0) + { + return; + } + + while (i < source.Length - 1) + { + // source[i] is a high surrogate located by the scan below. + if (char.IsLowSurrogate(source[i + 1])) + { + if (source[i] != destination[i] || source[i + 1] != destination[i + 1]) + { + // The supplementary scalar was changed by casing. Restore the original pair + // only when the cased value moved outside its OrdinalIgnoreCase class. + if (CompareInfo.NlsCompareStringOrdinalIgnoreCase( + ref MemoryMarshal.GetReference(source.Slice(i)), 2, + ref MemoryMarshal.GetReference(destination.Slice(i)), 2) != 0) + { + destination[i] = source[i]; + destination[i + 1] = source[i + 1]; + } + + i += 2; + } + else + { + // Unchanged pair. Skip the whole unchanged run in one vectorized step so + // surrogate-dense input (for example emoji) does not pay per-pair scanning. + i += source.Slice(i).CommonPrefixLength(destination.Slice(i)); + + // A changed pair whose high surrogate matches (for example Deseret, where + // both scalars share the same high surrogate) stops the run on its low + // surrogate. Step back so the scan re-finds the start of that pair. + if (i < source.Length && char.IsLowSurrogate(source[i]) && char.IsHighSurrogate(source[i - 1])) + { + i--; + } + } + } + else + { + i++; + } + + int next = source.Slice(i).IndexOfAnyInRange('\uD800', '\uDBFF'); + if (next < 0) + { + return; + } + + i += next; + } + } + + internal static uint PreserveNlsOrdinalCasingClass(uint source, uint destination) + { + Debug.Assert(GlobalizationMode.UseNls); + Debug.Assert(source > char.MaxValue); + Debug.Assert(destination > char.MaxValue); + + if (source == destination) + { + return destination; + } + + Span sourceChars = stackalloc char[2]; + Span destinationChars = stackalloc char[2]; + UnicodeUtility.GetUtf16SurrogatesFromSupplementaryPlaneScalar(source, out sourceChars[0], out sourceChars[1]); + UnicodeUtility.GetUtf16SurrogatesFromSupplementaryPlaneScalar(destination, out destinationChars[0], out destinationChars[1]); + + return CompareInfo.NlsCompareStringOrdinalIgnoreCase( + ref MemoryMarshal.GetReference(sourceChars), sourceChars.Length, + ref MemoryMarshal.GetReference(destinationChars), destinationChars.Length) == 0 + ? destination + : source; + } + // The only BMP scalars whose simple invariant/NLS lower mapping moves them out of their ordinal // upper-casing class. Each one is its own ordinal upper-casing (ToUpperOrdinal(c) == c) yet lowers to a // different letter, so simple lowering would break OrdinalIgnoreCase consistency for them. The ICU ordinal diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Rune.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Rune.cs index 61673fc1da8d51..af873fb5dec362 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Rune.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Rune.cs @@ -1611,7 +1611,10 @@ public static Rune ToUpperOrdinal(Rune value) } // Supplementary characters use the same simple scalar mapping as OrdinalIgnoreCase comparisons. - return UnsafeCreate(CharUnicodeInfo.ToUpper(value._value)); + uint upper = CharUnicodeInfo.ToUpper(value._value); + return UnsafeCreate(GlobalizationMode.UseNls + ? Ordinal.PreserveNlsOrdinalCasingClass(value._value, upper) + : upper); } /// @@ -1631,7 +1634,10 @@ public static Rune ToLowerOrdinal(Rune value) return UnsafeCreate(TextInfo.ToLowerOrdinal((char)value._value)); } - return UnsafeCreate(CharUnicodeInfo.ToLower(value._value)); + uint lower = CharUnicodeInfo.ToLower(value._value); + return UnsafeCreate(GlobalizationMode.UseNls + ? Ordinal.PreserveNlsOrdinalCasingClass(value._value, lower) + : lower); } #endif diff --git a/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs b/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs index c77815b2b397d8..d949b7f2abb3f2 100644 --- a/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs +++ b/src/libraries/System.Runtime/tests/System.Globalization.Tests/System/Globalization/OrdinalCasingTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; +using System.Diagnostics; using System.Text; using Microsoft.DotNet.RemoteExecutor; using Xunit; @@ -336,7 +337,11 @@ public static IEnumerable RuneUpperData() yield return new object[] { new Rune('5'), new Rune('5') }; yield return new object[] { new Rune('\u03B1'), new Rune('\u0391') }; // greek yield return new object[] { new Rune('\u212A'), new Rune('\u212A') }; // KELVIN SIGN is not folded by ordinal upper - yield return new object[] { new Rune(0x10428), new Rune(0x10400) }; // DESERET small ee -> capital long i + yield return new object[] + { + new Rune(0x10428), + new Rune(PlatformDetection.IsNlsGlobalization ? 0x10428 : 0x10400) + }; } public static IEnumerable RuneLowerData() @@ -345,7 +350,11 @@ public static IEnumerable RuneLowerData() yield return new object[] { new Rune('5'), new Rune('5') }; yield return new object[] { new Rune('\u0391'), new Rune('\u03B1') }; // greek yield return new object[] { new Rune('\u212A'), new Rune('\u212A') }; // KELVIN SIGN stays itself (not OIC equal to 'k') - yield return new object[] { new Rune(0x10400), new Rune(0x10428) }; // DESERET capital -> small + yield return new object[] + { + new Rune(0x10400), + new Rune(PlatformDetection.IsNlsGlobalization ? 0x10400 : 0x10428) + }; } [Theory] @@ -372,6 +381,60 @@ public void Rune_ToLowerOrdinal(Rune input, Rune expected) } } + [PlatformSpecific(TestPlatforms.Windows)] + [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + public void OrdinalCasing_SupplementaryCharacters_AgreeWithOrdinalIgnoreCaseInNlsMode() + { + var startInfo = new ProcessStartInfo(); + TestEnvironment.ClearGlobalizationEnvironmentVars(startInfo.Environment); + startInfo.Environment.Add("DOTNET_SYSTEM_GLOBALIZATION_USENLS", "true"); + + RemoteExecutor.Invoke(static () => + { + const int DeseretCapitalStart = 0x10400; + const int DeseretSmallEnd = 0x1044F; + int count = DeseretSmallEnd - DeseretCapitalStart + 1; + var values = new string[count]; + var upperValues = new string[count]; + var lowerValues = new string[count]; + Span upperDestination = stackalloc char[2]; + Span lowerDestination = stackalloc char[2]; + + for (int i = 0; i < count; i++) + { + Rune rune = new Rune(DeseretCapitalStart + i); + string value = rune.ToString(); + values[i] = value; + upperValues[i] = value.ToUpperOrdinal(); + lowerValues[i] = value.ToLowerOrdinal(); + + Assert.Equal(value, upperValues[i]); + Assert.Equal(value, lowerValues[i]); + Assert.Equal(rune, Rune.ToUpperOrdinal(rune)); + Assert.Equal(rune, Rune.ToLowerOrdinal(rune)); + + Assert.Equal(2, value.AsSpan().ToUpperOrdinal(upperDestination)); + Assert.Equal(2, value.AsSpan().ToLowerOrdinal(lowerDestination)); + Assert.True(value.AsSpan().SequenceEqual(upperDestination)); + Assert.True(value.AsSpan().SequenceEqual(lowerDestination)); + } + + for (int i = 0; i < count; i++) + { + for (int j = 0; j < count; j++) + { + bool ordinalIgnoreCaseEqual = string.Equals(values[i], values[j], StringComparison.OrdinalIgnoreCase); + Assert.Equal(ordinalIgnoreCaseEqual, string.Equals(upperValues[i], upperValues[j], StringComparison.Ordinal)); + Assert.Equal(ordinalIgnoreCaseEqual, string.Equals(lowerValues[i], lowerValues[j], StringComparison.Ordinal)); + } + } + + Assert.Equal("\U00010428X\U00010429", "\U00010428x\U00010429".ToUpperOrdinal()); + Assert.Equal("\U00010400x\U00010401", "\U00010400X\U00010401".ToLowerOrdinal()); + Assert.Equal("\uD800\U00010428", "\uD800\U00010428".ToUpperOrdinal()); + }, new RemoteInvokeOptions { StartInfo = startInfo }).Dispose(); + } + [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] public void OrdinalIgnoreCase_Equivalence_HoldsInInvariantMode() { From ef8e22a75ade6099adb6def5d6eef46eac1a1ef8 Mon Sep 17 00:00:00 2001 From: Tarek Mahmoud Sayed Date: Thu, 17 Sep 2026 11:50:01 -0700 Subject: [PATCH 2/2] Allow oversized destination buffers and drop redundant comments --- .../src/System/Globalization/Ordinal.cs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs index 4a5dd7423c51c0..0f9334d409848a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs @@ -769,7 +769,9 @@ private static void ChangeCaseNlsOrdinal(ReadOnlySpan source, Span d private static void PreserveNlsOrdinalCasingClass(ReadOnlySpan source, Span destination) { Debug.Assert(GlobalizationMode.UseNls); - Debug.Assert(source.Length == destination.Length); + Debug.Assert(destination.Length >= source.Length); + + destination = destination.Slice(0, source.Length); int i = source.IndexOfAnyInRange('\uD800', '\uDBFF'); if (i < 0) @@ -779,13 +781,10 @@ private static void PreserveNlsOrdinalCasingClass(ReadOnlySpan source, Spa while (i < source.Length - 1) { - // source[i] is a high surrogate located by the scan below. if (char.IsLowSurrogate(source[i + 1])) { if (source[i] != destination[i] || source[i + 1] != destination[i + 1]) { - // The supplementary scalar was changed by casing. Restore the original pair - // only when the cased value moved outside its OrdinalIgnoreCase class. if (CompareInfo.NlsCompareStringOrdinalIgnoreCase( ref MemoryMarshal.GetReference(source.Slice(i)), 2, ref MemoryMarshal.GetReference(destination.Slice(i)), 2) != 0) @@ -798,13 +797,8 @@ ref MemoryMarshal.GetReference(destination.Slice(i)), 2) != 0) } else { - // Unchanged pair. Skip the whole unchanged run in one vectorized step so - // surrogate-dense input (for example emoji) does not pay per-pair scanning. i += source.Slice(i).CommonPrefixLength(destination.Slice(i)); - // A changed pair whose high surrogate matches (for example Deseret, where - // both scalars share the same high surrogate) stops the run on its low - // surrogate. Step back so the scan re-finds the start of that pair. if (i < source.Length && char.IsLowSurrogate(source[i]) && char.IsHighSurrogate(source[i - 1])) { i--;