diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index 62e69dd3b3663..1963af0025c7e 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -861,7 +861,7 @@ - + @@ -1811,6 +1811,7 @@ + @@ -1827,4 +1828,4 @@ - \ No newline at end of file + diff --git a/src/libraries/System.Private.CoreLib/src/System/Index.cs b/src/libraries/System.Private.CoreLib/src/System/Index.cs index 8f7c12eaac7ed..1da3a691c7c76 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Index.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Index.cs @@ -140,11 +140,15 @@ public override string ToString() private string ToStringFromEnd() { +#if !NETSTANDARD2_0 Span span = stackalloc char[11]; // 1 for ^ and 10 for longest possible uint value bool formatted = ((uint)Value).TryFormat(span.Slice(1), out int charsWritten); Debug.Assert(formatted); span[0] = '^'; return new string(span.Slice(0, charsWritten + 1)); +#else + return '^' + Value.ToString(); +#endif } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/BitOperations.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/BitOperations.cs index 5510659216d30..2506043f6f721 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/BitOperations.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/BitOperations.cs @@ -6,7 +6,9 @@ using System.Runtime.InteropServices; using System.Runtime.Intrinsics.X86; +#if SYSTEM_PRIVATE_CORELIB using Internal.Runtime.CompilerServices; +#endif // Some routines inspired by the Stanford Bit Twiddling Hacks by Sean Eron Anderson: // http://graphics.stanford.edu/~seander/bithacks.html @@ -18,7 +20,12 @@ namespace System.Numerics /// The methods use hardware intrinsics when available on the underlying platform, /// otherwise they use optimized software fallbacks. /// - public static class BitOperations +#if SYSTEM_PRIVATE_CORELIB + public +#else + internal +#endif + static class BitOperations { // C# no-alloc optimization that directly wraps the data section of the dll (similar to string constants) // https://github.com/dotnet/roslyn/pull/24621 diff --git a/src/libraries/System.Private.CoreLib/src/System/Range.cs b/src/libraries/System.Private.CoreLib/src/System/Range.cs index e8ed4564ac695..f2c5493818e65 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Range.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Range.cs @@ -5,6 +5,10 @@ using System.Diagnostics; using System.Runtime.CompilerServices; +#if NETSTANDARD2_0 +using System.Numerics.Hashing; +#endif + namespace System { /// Represent a range has start and end indexes. @@ -47,12 +51,17 @@ value is Range r && /// Returns the hash code for this instance. public override int GetHashCode() { +#if !NETSTANDARD2_0 return HashCode.Combine(Start.GetHashCode(), End.GetHashCode()); +#else + return HashHelpers.Combine(Start.GetHashCode(), End.GetHashCode()); +#endif } /// Converts the value of the current Range object to its equivalent string representation. public override string ToString() { +#if !NETSTANDARD2_0 Span span = stackalloc char[2 + (2 * 11)]; // 2 for "..", then for each index 1 for '^' and 10 for longest possible uint int pos = 0; @@ -77,6 +86,9 @@ public override string ToString() pos += charsWritten; return new string(span.Slice(0, pos)); +#else + return Start.ToString() + ".." + End.ToString(); +#endif } /// Create a Range object starting from start index to the end of the collection. diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index b1f59d3b46090..c0bdbf03f7059 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -7,9 +7,13 @@ using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; + +#if SYSTEM_PRIVATE_CORELIB using Internal.Runtime.CompilerServices; +#endif #pragma warning disable SA1121 // explicitly using type aliases instead of built-in types +#if SYSTEM_PRIVATE_CORELIB #if TARGET_64BIT using nint = System.Int64; using nuint = System.UInt64; @@ -17,18 +21,22 @@ using nint = System.Int32; using nuint = System.UInt32; #endif // TARGET_64BIT +#else +using nint = System.Int64; // https://github.com/dotnet/runtime/issues/33575 - use long/ulong outside of corelib until the compiler supports it +using nuint = System.UInt64; +#endif namespace System.Text { internal static partial class ASCIIUtility { -#if DEBUG +#if DEBUG && SYSTEM_PRIVATE_CORELIB static ASCIIUtility() { Debug.Assert(sizeof(nint) == IntPtr.Size && nint.MinValue < 0, "nint is defined incorrectly."); Debug.Assert(sizeof(nuint) == IntPtr.Size && nuint.MinValue == 0, "nuint is defined incorrectly."); } -#endif // DEBUG +#endif // DEBUG && SYSTEM_PRIVATE_CORELIB [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool AllBytesInUInt64AreAscii(ulong value) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Rune.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Rune.cs index fd3ac737eae97..3a703d661ff78 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Rune.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Rune.cs @@ -20,6 +20,10 @@ namespace System.Text [DebuggerDisplay("{DebuggerDisplay,nq}")] public readonly struct Rune : IComparable, IEquatable { + private const char HighSurrogateStart = '\ud800'; + private const char LowSurrogateStart = '\udc00'; + private const int HighSurrogateRange = 0x3FF; + private const byte IsWhiteSpaceFlag = 0x80; private const byte IsLetterOrDigitFlag = 0x40; private const byte UnicodeCategoryMask = 0x1F; @@ -175,6 +179,7 @@ private Rune(uint scalarValue, bool unused) /// public int Value => (int)_value; +#if SYSTEM_PRIVATE_CORELIB private static Rune ChangeCaseCultureAware(Rune rune, TextInfo textInfo, bool toUpper) { Debug.Assert(!GlobalizationMode.Invariant, "This should've been checked by the caller."); @@ -209,6 +214,42 @@ private static Rune ChangeCaseCultureAware(Rune rune, TextInfo textInfo, bool to return UnsafeCreate(UnicodeUtility.GetScalarFromUtf16SurrogatePair(modified[0], modified[1])); } } +#else + private static Rune ChangeCaseCultureAware(Rune rune, CultureInfo culture, bool toUpper) + { + Debug.Assert(!GlobalizationMode.Invariant, "This should've been checked by the caller."); + Debug.Assert(culture != null, "This should've been checked by the caller."); + + Span original = stackalloc char[2]; // worst case scenario = 2 code units (for a surrogate pair) + Span modified = stackalloc char[2]; // case change should preserve UTF-16 code unit count + + int charCount = rune.EncodeToUtf16(original); + original = original.Slice(0, charCount); + modified = modified.Slice(0, charCount); + + if (toUpper) + { + MemoryExtensions.ToUpper(original, modified, culture); + } + else + { + MemoryExtensions.ToLower(original, modified, culture); + } + + // We use simple case folding rules, which disallows moving between the BMP and supplementary + // planes when performing a case conversion. The helper methods which reconstruct a Rune + // contain debug asserts for this condition. + + if (rune.IsBmp) + { + return UnsafeCreate(modified[0]); + } + else + { + return UnsafeCreate(UnicodeUtility.GetScalarFromUtf16SurrogatePair(modified[0], modified[1])); + } + } +#endif public int CompareTo(Rune other) => _value.CompareTo(other._value); @@ -827,6 +868,7 @@ private static int ReadRuneFromString(string input, int index) /// public override string ToString() { +#if SYSTEM_PRIVATE_CORELIB if (IsBmp) { return string.CreateFromChar((char)_value); @@ -836,6 +878,18 @@ public override string ToString() UnicodeUtility.GetUtf16SurrogatesFromSupplementaryPlaneScalar(_value, out char high, out char low); return string.CreateFromChar(high, low); } +#else + if (IsBmp) + { + return ((char)_value).ToString(); + } + else + { + Span buffer = stackalloc char[2]; + UnicodeUtility.GetUtf16SurrogatesFromSupplementaryPlaneScalar(_value, out buffer[0], out buffer[1]); + return buffer.ToString(); + } +#endif } /// @@ -865,17 +919,17 @@ public static bool TryCreate(char highSurrogate, char lowSurrogate, out Rune res // First, extend both to 32 bits, then calculate the offset of // each candidate surrogate char from the start of its range. - uint highSurrogateOffset = (uint)highSurrogate - CharUnicodeInfo.HIGH_SURROGATE_START; - uint lowSurrogateOffset = (uint)lowSurrogate - CharUnicodeInfo.LOW_SURROGATE_START; + uint highSurrogateOffset = (uint)highSurrogate - HighSurrogateStart; + uint lowSurrogateOffset = (uint)lowSurrogate - LowSurrogateStart; // This is a single comparison which allows us to check both for validity at once since // both the high surrogate range and the low surrogate range are the same length. // If the comparison fails, we call to a helper method to throw the correct exception message. - if ((highSurrogateOffset | lowSurrogateOffset) <= CharUnicodeInfo.HIGH_SURROGATE_RANGE) + if ((highSurrogateOffset | lowSurrogateOffset) <= HighSurrogateRange) { // The 0x40u << 10 below is to account for uuuuu = wwww + 1 in the surrogate encoding. - result = UnsafeCreate((highSurrogateOffset << 10) + ((uint)lowSurrogate - CharUnicodeInfo.LOW_SURROGATE_START) + (0x40u << 10)); + result = UnsafeCreate((highSurrogateOffset << 10) + ((uint)lowSurrogate - LowSurrogateStart) + (0x40u << 10)); return true; } else @@ -1070,7 +1124,15 @@ public static double GetNumericValue(Rune value) else { // not an ASCII char; fall back to globalization table +#if SYSTEM_PRIVATE_CORELIB return CharUnicodeInfo.GetNumericValue(value.Value); +#else + if (value.IsBmp) + { + return CharUnicodeInfo.GetNumericValue((char)value._value); + } + return CharUnicodeInfo.GetNumericValue(value.ToString(), 0); +#endif } } @@ -1089,7 +1151,15 @@ public static UnicodeCategory GetUnicodeCategory(Rune value) private static UnicodeCategory GetUnicodeCategoryNonAscii(Rune value) { Debug.Assert(!value.IsAscii, "Shouldn't use this non-optimized code path for ASCII characters."); +#if !NETSTANDARD2_0 return CharUnicodeInfo.GetUnicodeCategory(value.Value); +#else + if (value.IsBmp) + { + return CharUnicodeInfo.GetUnicodeCategory((char)value._value); + } + return CharUnicodeInfo.GetUnicodeCategory(value.ToString(), 0); +#endif } // Returns true iff this Unicode category represents a letter @@ -1240,7 +1310,12 @@ public static bool IsWhiteSpace(Rune value) // Only BMP code points can be white space, so only call into CharUnicodeInfo // if the incoming value is within the BMP. - return value.IsBmp && CharUnicodeInfo.GetIsWhiteSpace((char)value._value); + return value.IsBmp && +#if SYSTEM_PRIVATE_CORELIB + CharUnicodeInfo.GetIsWhiteSpace((char)value._value); +#else + char.IsWhiteSpace((char)value._value); +#endif } public static Rune ToLower(Rune value, CultureInfo culture) @@ -1259,7 +1334,11 @@ public static Rune ToLower(Rune value, CultureInfo culture) return ToLowerInvariant(value); } - return ChangeCaseCultureAware(value, culture!.TextInfo, toUpper: false); +#if SYSTEM_PRIVATE_CORELIB + return ChangeCaseCultureAware(value, culture.TextInfo, toUpper: false); +#else + return ChangeCaseCultureAware(value, culture, toUpper: false); +#endif } public static Rune ToLowerInvariant(Rune value) @@ -1283,7 +1362,11 @@ public static Rune ToLowerInvariant(Rune value) // Non-ASCII data requires going through the case folding tables. +#if SYSTEM_PRIVATE_CORELIB return ChangeCaseCultureAware(value, TextInfo.Invariant, toUpper: false); +#else + return ChangeCaseCultureAware(value, CultureInfo.InvariantCulture, toUpper: false); +#endif } public static Rune ToUpper(Rune value, CultureInfo culture) @@ -1302,7 +1385,11 @@ public static Rune ToUpper(Rune value, CultureInfo culture) return ToUpperInvariant(value); } - return ChangeCaseCultureAware(value, culture!.TextInfo, toUpper: true); +#if SYSTEM_PRIVATE_CORELIB + return ChangeCaseCultureAware(value, culture.TextInfo, toUpper: true); +#else + return ChangeCaseCultureAware(value, culture, toUpper: true); +#endif } public static Rune ToUpperInvariant(Rune value) @@ -1326,7 +1413,11 @@ public static Rune ToUpperInvariant(Rune value) // Non-ASCII data requires going through the case folding tables. +#if SYSTEM_PRIVATE_CORELIB return ChangeCaseCultureAware(value, TextInfo.Invariant, toUpper: true); +#else + return ChangeCaseCultureAware(value, CultureInfo.InvariantCulture, toUpper: true); +#endif } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.Validation.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.Validation.cs index be733006d599a..8d23f74cfe89b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.Validation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.Validation.cs @@ -5,10 +5,15 @@ using System.Diagnostics; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; +using System.Runtime.CompilerServices; using System.Numerics; + +#if SYSTEM_PRIVATE_CORELIB using Internal.Runtime.CompilerServices; +#endif #pragma warning disable SA1121 // explicitly using type aliases instead of built-in types +#if SYSTEM_PRIVATE_CORELIB #if TARGET_64BIT using nint = System.Int64; using nuint = System.UInt64; @@ -16,18 +21,22 @@ using nint = System.Int32; using nuint = System.UInt32; #endif // TARGET_64BIT +#else +using nint = System.Int64; // https://github.com/dotnet/runtime/issues/33575 - use long/ulong outside of corelib until the compiler supports it +using nuint = System.UInt64; +#endif namespace System.Text.Unicode { internal static unsafe partial class Utf16Utility { -#if DEBUG +#if DEBUG && SYSTEM_PRIVATE_CORELIB static Utf16Utility() { Debug.Assert(sizeof(nint) == IntPtr.Size && nint.MinValue < 0, "nint is defined incorrectly."); Debug.Assert(sizeof(nuint) == IntPtr.Size && nuint.MinValue == 0, "nuint is defined incorrectly."); } -#endif // DEBUG +#endif // DEBUG && SYSTEM_PRIVATE_CORELIB // Returns &inputBuffer[inputLength] if the input buffer is valid. /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8.cs index 1904df7dbf5d5..7992cd52a0f79 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8.cs @@ -4,12 +4,21 @@ using System.Buffers; using System.Diagnostics; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; + +#if SYSTEM_PRIVATE_CORELIB using Internal.Runtime.CompilerServices; +#endif namespace System.Text.Unicode { - public static class Utf8 +#if SYSTEM_PRIVATE_CORELIB + public +#else + internal +#endif + static class Utf8 { /* * OperationStatus-based APIs for transcoding of chunked data. diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Helpers.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Helpers.cs index 5cdf7574e82a3..4021195ff0646 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Helpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Helpers.cs @@ -6,7 +6,10 @@ using System.Diagnostics; using System.Numerics; using System.Runtime.CompilerServices; + +#if SYSTEM_PRIVATE_CORELIB using Internal.Runtime.CompilerServices; +#endif namespace System.Text.Unicode { diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs index b9d7fd305b609..a1288394d7045 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs @@ -6,11 +6,16 @@ using System.Buffers.Binary; using System.Diagnostics; using System.Numerics; +using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; + +#if SYSTEM_PRIVATE_CORELIB using Internal.Runtime.CompilerServices; +#endif #pragma warning disable SA1121 // explicitly using type aliases instead of built-in types +#if SYSTEM_PRIVATE_CORELIB #if TARGET_64BIT using nint = System.Int64; using nuint = System.UInt64; @@ -18,12 +23,16 @@ using nint = System.Int32; using nuint = System.UInt32; #endif // TARGET_64BIT +#else +using nint = System.Int64; // https://github.com/dotnet/runtime/issues/33575 - use long/ulong outside of corelib until the compiler supports it +using nuint = System.UInt64; +#endif namespace System.Text.Unicode { internal static unsafe partial class Utf8Utility { -#if DEBUG +#if DEBUG && SYSTEM_PRIVATE_CORELIB static Utf8Utility() { Debug.Assert(sizeof(nint) == IntPtr.Size && nint.MinValue < 0, "nint is defined incorrectly."); @@ -31,7 +40,7 @@ static Utf8Utility() _ValidateAdditionalNIntDefinitions(); } -#endif // DEBUG +#endif // DEBUG && SYSTEM_PRIVATE_CORELIB // On method return, pInputBufferRemaining and pOutputBufferRemaining will both point to where // the next byte would have been consumed from / the next char would have been written to. diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Validation.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Validation.cs index 9f721d5408db5..137b6775c8944 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Validation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Validation.cs @@ -5,9 +5,14 @@ using System.Diagnostics; using System.Numerics; using System.Runtime.Intrinsics.X86; +using System.Runtime.CompilerServices; + +#if SYSTEM_PRIVATE_CORELIB using Internal.Runtime.CompilerServices; +#endif #pragma warning disable SA1121 // explicitly using type aliases instead of built-in types +#if SYSTEM_PRIVATE_CORELIB #if TARGET_64BIT using nint = System.Int64; using nuint = System.UInt64; @@ -15,18 +20,22 @@ using nint = System.Int32; using nuint = System.UInt32; #endif // TARGET_64BIT +#else +using nint = System.Int64; // https://github.com/dotnet/runtime/issues/33575 - use long/ulong outside of corelib until the compiler supports it +using nuint = System.UInt64; +#endif namespace System.Text.Unicode { internal static unsafe partial class Utf8Utility { -#if DEBUG +#if DEBUG && SYSTEM_PRIVATE_CORELIB private static void _ValidateAdditionalNIntDefinitions() { Debug.Assert(sizeof(nint) == IntPtr.Size && nint.MinValue < 0, "nint is defined incorrectly."); Debug.Assert(sizeof(nuint) == IntPtr.Size && nuint.MinValue == 0, "nuint is defined incorrectly."); } -#endif // DEBUG +#endif // DEBUG && SYSTEM_PRIVATE_CORELIB // Returns &inputBuffer[inputLength] if the input buffer is valid. /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.WhiteSpace.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.WhiteSpace.CoreLib.cs similarity index 100% rename from src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.WhiteSpace.cs rename to src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.WhiteSpace.CoreLib.cs diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.WhiteSpace.NonCoreLib.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.WhiteSpace.NonCoreLib.cs new file mode 100644 index 0000000000000..214f01a66e908 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.WhiteSpace.NonCoreLib.cs @@ -0,0 +1,119 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace System.Text.Unicode +{ + internal static partial class Utf8Utility + { + /// + /// Returns the index in where the first non-whitespace character + /// appears, or the input length if the data contains only whitespace characters. + /// + public static int GetIndexOfFirstNonWhiteSpaceChar(ReadOnlySpan utf8Data) + { + // This method is optimized for the case where the input data is ASCII, and if the + // data does need to be trimmed it's likely that only a relatively small number of + // bytes will be trimmed. + + int i = 0; + int length = utf8Data.Length; + + while (i < length) + { + // Very quick check: see if the byte is in the range [ 21 .. 7F ]. + // If so, we can skip the more expensive logic later in this method. + + if ((sbyte)utf8Data[i] > (sbyte)0x20) + { + break; + } + + uint possibleAsciiByte = utf8Data[i]; + if (UnicodeUtility.IsAsciiCodePoint(possibleAsciiByte)) + { + // The simple comparison failed. Let's read the actual byte value, + // and if it's ASCII we can delegate to Rune's inlined method + // implementation. + + if (Rune.IsWhiteSpace(new Rune(possibleAsciiByte))) + { + i++; + continue; + } + } + else + { + // Not ASCII data. Go back to the slower "decode the entire scalar" + // code path, then compare it against our Unicode tables. + + Rune.DecodeFromUtf8(utf8Data.Slice(i), out Rune decodedRune, out int bytesConsumed); + if (Rune.IsWhiteSpace(decodedRune)) + { + i += bytesConsumed; + continue; + } + } + + break; // If we got here, we saw a non-whitespace subsequence. + } + + return i; + } + + /// + /// Returns the index in where the trailing whitespace sequence + /// begins, or 0 if the data contains only whitespace characters, or the span length if the + /// data does not end with any whitespace characters. + /// + public static int GetIndexOfTrailingWhiteSpaceSequence(ReadOnlySpan utf8Data) + { + // This method is optimized for the case where the input data is ASCII, and if the + // data does need to be trimmed it's likely that only a relatively small number of + // bytes will be trimmed. + + int length = utf8Data.Length; + + while (length > 0) + { + // Very quick check: see if the byte is in the range [ 21 .. 7F ]. + // If so, we can skip the more expensive logic later in this method. + + if ((sbyte)utf8Data[length - 1] > (sbyte)0x20) + { + break; + } + + uint possibleAsciiByte = utf8Data[length - 1]; + if (UnicodeUtility.IsAsciiCodePoint(possibleAsciiByte)) + { + // The simple comparison failed. Let's read the actual byte value, + // and if it's ASCII we can delegate to Rune's inlined method + // implementation. + + if (Rune.IsWhiteSpace(new Rune(possibleAsciiByte))) + { + length--; + continue; + } + } + else + { + // Not ASCII data. Go back to the slower "decode the entire scalar" + // code path, then compare it against our Unicode tables. + + Rune.DecodeLastFromUtf8(utf8Data.Slice(0, length), out Rune decodedRune, out int bytesConsumed); + if (Rune.IsWhiteSpace(decodedRune)) + { + length -= bytesConsumed; + continue; + } + } + + break; // If we got here, we saw a non-whitespace subsequence. + } + + return length; + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.cs index bb6853b072e58..78de994a2091d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.cs @@ -7,7 +7,10 @@ using System.IO; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; + +#if SYSTEM_PRIVATE_CORELIB using Internal.Runtime.CompilerServices; +#endif namespace System.Text.Unicode { @@ -22,7 +25,11 @@ internal static partial class Utf8Utility /// /// The UTF-8 representation of . /// +#if !NETSTANDARD2_0 private static ReadOnlySpan ReplacementCharSequence => new byte[] { 0xEF, 0xBF, 0xBD }; +#else + private static readonly byte[] ReplacementCharSequence = new byte[] { 0xEF, 0xBF, 0xBD }; +#endif /// /// Returns the byte index in where the first invalid UTF-8 sequence begins, @@ -83,6 +90,7 @@ public static Utf8String ValidateAndFixupUtf8String(Utf8String value) // (The faster implementation is in the dev/utf8string_bak branch currently.) MemoryStream memStream = new MemoryStream(); +#if !NETSTANDARD2_0 memStream.Write(valueAsBytes.Slice(0, idxOfFirstInvalidData)); valueAsBytes = valueAsBytes.Slice(idxOfFirstInvalidData); @@ -101,6 +109,37 @@ public static Utf8String ValidateAndFixupUtf8String(Utf8String value) valueAsBytes = valueAsBytes.Slice(bytesConsumed); } while (!valueAsBytes.IsEmpty); +#else + if (!MemoryMarshal.TryGetArray(value.AsMemoryBytes(), out ArraySegment valueArraySegment)) + { + Debug.Fail("Utf8String on netstandard should always be backed by an array."); + } + + memStream.Write(valueArraySegment.Array, valueArraySegment.Offset, idxOfFirstInvalidData); + + valueArraySegment = new ArraySegment( + valueArraySegment.Array, + idxOfFirstInvalidData, + valueArraySegment.Count - idxOfFirstInvalidData); + do + { + if (Rune.DecodeFromUtf8(valueArraySegment, out _, out int bytesConsumed) == OperationStatus.Done) + { + // Valid scalar value - copy data as-is to MemoryStream + memStream.Write(valueArraySegment.Array, valueArraySegment.Offset, bytesConsumed); + } + else + { + // Invalid scalar value - copy U+FFFD to MemoryStream + memStream.Write(ReplacementCharSequence, 0, ReplacementCharSequence.Length); + } + + valueArraySegment = new ArraySegment( + valueArraySegment.Array, + valueArraySegment.Offset + bytesConsumed, + valueArraySegment.Count - bytesConsumed); + } while (valueArraySegment.Count > 0); +#endif bool success = memStream.TryGetBuffer(out ArraySegment memStreamBuffer); Debug.Assert(success, "Couldn't get underlying MemoryStream buffer."); diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.Comparison.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.Comparison.cs index 3abd979f43389..72f20864f09e9 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.Comparison.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.Comparison.cs @@ -68,7 +68,11 @@ public bool Contains(Rune value, StringComparison comparison) { // TODO_UTF8STRING: Optimize me to avoid allocations. +#if !NETSTANDARD2_0 return this.ToString().Contains(value.ToString(), comparison); +#else + return this.ToString().IndexOf(value.ToString(), comparison) >= 0; +#endif } /// @@ -88,7 +92,11 @@ public bool Contains(Utf8Span value, StringComparison comparison) { // TODO_UTF8STRING: Optimize me to avoid allocations. +#if !NETSTANDARD2_0 return this.ToString().Contains(value.ToString(), comparison); +#else + return this.ToString().IndexOf(value.ToString(), comparison) >= 0; +#endif } /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.Conversion.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.Conversion.cs index 48754b0097b3c..51f0840e10412 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.Conversion.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.Conversion.cs @@ -43,7 +43,7 @@ public int Normalize(Span destination, NormalizationForm normalizationForm { // TODO_UTF8STRING: Reduce allocations in this code path. - ReadOnlySpan normalized = this.ToString().Normalize(normalizationForm); + ReadOnlySpan normalized = this.ToString().Normalize(normalizationForm).AsSpan(); OperationStatus status = Utf8.FromUtf16(normalized, destination, out int _, out int bytesWritten, replaceInvalidSequences: false, isFinalBlock: true); Debug.Assert(status == OperationStatus.Done || status == OperationStatus.DestinationTooSmall, "Normalize shouldn't have produced malformed Unicode string."); @@ -81,7 +81,7 @@ public unsafe char[] ToCharArray() Debug.Assert(pbUtf8Invalid == pbUtf8 + this.Length, "Invalid UTF-8 data seen in buffer."); char[] asUtf16 = new char[this.Length + utf16CodeUnitCountAdjustment]; - fixed (char* pbUtf16 = &MemoryMarshal.GetArrayDataReference(asUtf16)) + fixed (char* pbUtf16 = asUtf16) { OperationStatus status = Utf8Utility.TranscodeToUtf16(pbUtf8, this.Length, pbUtf16, asUtf16.Length, out byte* pbUtf8End, out char* pchUtf16End); Debug.Assert(status == OperationStatus.Done, "The buffer changed out from under us unexpectedly?"); @@ -158,7 +158,7 @@ public int ToLower(Span destination, CultureInfo culture) ThrowHelper.ThrowArgumentNullException(ExceptionArgument.culture); } - ReadOnlySpan asLower = this.ToString().ToLower(culture); + ReadOnlySpan asLower = this.ToString().ToLower(culture).AsSpan(); OperationStatus status = Utf8.FromUtf16(asLower, destination, out int _, out int bytesWritten, replaceInvalidSequences: false, isFinalBlock: true); Debug.Assert(status == OperationStatus.Done || status == OperationStatus.DestinationTooSmall, "ToLower shouldn't have produced malformed Unicode string."); @@ -206,7 +206,7 @@ public int ToLowerInvariant(Span destination) { // TODO_UTF8STRING: Avoid intermediate allocations. - ReadOnlySpan asLowerInvariant = this.ToString().ToLowerInvariant(); + ReadOnlySpan asLowerInvariant = this.ToString().ToLowerInvariant().AsSpan(); OperationStatus status = Utf8.FromUtf16(asLowerInvariant, destination, out int _, out int bytesWritten, replaceInvalidSequences: false, isFinalBlock: true); Debug.Assert(status == OperationStatus.Done || status == OperationStatus.DestinationTooSmall, "ToLowerInvariant shouldn't have produced malformed Unicode string."); @@ -262,7 +262,7 @@ public int ToUpper(Span destination, CultureInfo culture) ThrowHelper.ThrowArgumentNullException(ExceptionArgument.culture); } - ReadOnlySpan asUpper = this.ToString().ToUpper(culture); + ReadOnlySpan asUpper = this.ToString().ToUpper(culture).AsSpan(); OperationStatus status = Utf8.FromUtf16(asUpper, destination, out int _, out int bytesWritten, replaceInvalidSequences: false, isFinalBlock: true); Debug.Assert(status == OperationStatus.Done || status == OperationStatus.DestinationTooSmall, "ToUpper shouldn't have produced malformed Unicode string."); @@ -310,7 +310,7 @@ public int ToUpperInvariant(Span destination) { // TODO_UTF8STRING: Avoid intermediate allocations. - ReadOnlySpan asUpperInvariant = this.ToString().ToUpperInvariant(); + ReadOnlySpan asUpperInvariant = this.ToString().ToUpperInvariant().AsSpan(); OperationStatus status = Utf8.FromUtf16(asUpperInvariant, destination, out int _, out int bytesWritten, replaceInvalidSequences: false, isFinalBlock: true); Debug.Assert(status == OperationStatus.Done || status == OperationStatus.DestinationTooSmall, "ToUpperInvariant shouldn't have produced malformed Unicode string."); diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.Manipulation.cs index f679b14da76d0..2317de908ab37 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.Manipulation.cs @@ -431,7 +431,11 @@ internal readonly bool DeconstructHelper(in Utf8Span source, out Utf8Span firstI if (SearchRune >= 0) { +#if NETCOREAPP3_0 + wasMatchFound = searchSpan.TryFind(new Rune((uint)SearchRune), out matchRange); +#else wasMatchFound = searchSpan.TryFind(Rune.UnsafeCreate((uint)SearchRune), out matchRange); +#endif } else { diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.Searching.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.Searching.cs index 6be6e2173cfad..ea295dfa26c4d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.Searching.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.Searching.cs @@ -53,7 +53,7 @@ public bool TryFind(char value, StringComparison comparisonType, out Range range } else { - string.CheckStringComparison(comparisonType); + CheckStringComparison(comparisonType); // Surrogate chars can't exist in well-formed UTF-8 data - bail immediately. @@ -185,7 +185,7 @@ public bool TryFind(Utf8Span value, out Range range) private unsafe bool TryFind(Utf8Span value, StringComparison comparisonType, out Range range, bool fromBeginning) { - string.CheckStringComparison(comparisonType); + CheckStringComparison(comparisonType); if (value.IsEmpty) { @@ -211,7 +211,7 @@ private unsafe bool TryFind(Utf8Span value, StringComparison comparisonType, out } CompareInfo compareInfo = default!; // will be overwritten if it matters - CompareOptions compareOptions = string.GetCaseCompareOfComparisonCulture(comparisonType); + CompareOptions compareOptions = GetCaseCompareOfComparisonCulture(comparisonType); if (GlobalizationMode.Invariant) { @@ -221,7 +221,7 @@ private unsafe bool TryFind(Utf8Span value, StringComparison comparisonType, out // TODO_UTF8STRING: We should take advantage of the property described above to avoid the UTF-16 // transcoding step entirely. - if (compareOptions != CompareOptions.None) + if (compareOptions == CompareOptions.None) { return (fromBeginning) ? TryFind(value, out range) @@ -239,7 +239,11 @@ private unsafe bool TryFind(Utf8Span value, StringComparison comparisonType, out case StringComparison.OrdinalIgnoreCase: // TODO_UTF8STRING: Can probably optimize this case. +#if SYSTEM_PRIVATE_CORELIB compareInfo = CompareInfo.Invariant; +#else + compareInfo = CultureInfo.InvariantCulture.CompareInfo; +#endif break; case StringComparison.CurrentCulture: @@ -249,7 +253,11 @@ private unsafe bool TryFind(Utf8Span value, StringComparison comparisonType, out default: Debug.Assert(comparisonType == StringComparison.InvariantCulture || comparisonType == StringComparison.InvariantCultureIgnoreCase); +#if SYSTEM_PRIVATE_CORELIB compareInfo = CompareInfo.Invariant; +#else + compareInfo = CultureInfo.InvariantCulture.CompareInfo; +#endif break; } } @@ -261,6 +269,7 @@ private unsafe bool TryFind(Utf8Span value, StringComparison comparisonType, out int idx, matchLength; +#if SYSTEM_PRIVATE_CORELIB if (GlobalizationMode.Invariant) { // If we got here, it meant we're doing an OrdinalIgnoreCase comparison. @@ -274,6 +283,19 @@ private unsafe bool TryFind(Utf8Span value, StringComparison comparisonType, out { idx = compareInfo.IndexOf(thisTranscodedToUtf16, otherTranscodedToUtf16, 0, thisTranscodedToUtf16.Length, compareOptions, &matchLength, fromBeginning); } +#else + Debug.Assert(comparisonType == StringComparison.OrdinalIgnoreCase); + + if (fromBeginning) + { + idx = compareInfo.IndexOf(thisTranscodedToUtf16, otherTranscodedToUtf16, 0, thisTranscodedToUtf16.Length, compareOptions); + } + else + { + idx = compareInfo.LastIndexOf(thisTranscodedToUtf16, otherTranscodedToUtf16, thisTranscodedToUtf16.Length, thisTranscodedToUtf16.Length, compareOptions); + } + matchLength = otherTranscodedToUtf16.Length; +#endif if (idx < 0) { @@ -290,7 +312,11 @@ private unsafe bool TryFind(Utf8Span value, StringComparison comparisonType, out // follow Unicode full case folding semantics and might also normalize characters like // digraphs. +#if SYSTEM_PRIVATE_CORELIB fixed (char* pThisTranscodedToUtf16 = &thisTranscodedToUtf16.GetRawStringData()) +#else + fixed (char* pThisTranscodedToUtf16 = thisTranscodedToUtf16) +#endif { // First, we need to convert the UTF-16 'idx' to its UTF-8 equivalent. @@ -367,7 +393,7 @@ public bool TryFindLast(char value, StringComparison comparisonType, out Range r } else { - string.CheckStringComparison(comparisonType); + CheckStringComparison(comparisonType); // Surrogate chars can't exist in well-formed UTF-8 data - bail immediately. @@ -501,5 +527,50 @@ public bool TryFindLast(Utf8Span value, out Range range) /// The search is performed using the specified . /// public bool TryFindLast(Utf8Span value, StringComparison comparisonType, out Range range) => TryFind(value, comparisonType, out range, fromBeginning: false); + + private static void CheckStringComparison(StringComparison comparisonType) + { +#if SYSTEM_PRIVATE_CORELIB + string.CheckStringComparison(comparisonType); +#else + // Single comparison to check if comparisonType is within [CurrentCulture .. OrdinalIgnoreCase] + if ((uint)comparisonType > (uint)StringComparison.OrdinalIgnoreCase) + { + ThrowHelper.ThrowArgumentException(SR.NotSupported_StringComparison, ExceptionArgument.comparisonType); + } + + // There's no API that would allow getting the correct match length + // for other StringComparisons. + if (comparisonType != StringComparison.Ordinal && + comparisonType != StringComparison.OrdinalIgnoreCase) + { + ThrowHelper.ThrowNotSupportedException(SR.Utf8Span_TryFindOnlySupportsOrdinal); + } +#endif + } + + private static CompareOptions GetCaseCompareOfComparisonCulture(StringComparison comparisonType) + { +#if SYSTEM_PRIVATE_CORELIB + return string.GetCaseCompareOfComparisonCulture(comparisonType); +#else + Debug.Assert((uint)comparisonType <= (uint)StringComparison.OrdinalIgnoreCase); + + // Culture enums can be & with CompareOptions.IgnoreCase 0x01 to extract if IgnoreCase or CompareOptions.None 0x00 + // + // CompareOptions.None 0x00 + // CompareOptions.IgnoreCase 0x01 + // + // StringComparison.CurrentCulture: 0x00 + // StringComparison.InvariantCulture: 0x02 + // StringComparison.Ordinal 0x04 + // + // StringComparison.CurrentCultureIgnoreCase: 0x01 + // StringComparison.InvariantCultureIgnoreCase: 0x03 + // StringComparison.OrdinalIgnoreCase 0x05 + + return (CompareOptions)((int)comparisonType & (int)CompareOptions.IgnoreCase); +#endif + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.cs index 5c9ba2c589a59..eb6cdb6527d41 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Utf8Span.cs @@ -8,11 +8,15 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text.Unicode; + +#if SYSTEM_PRIVATE_CORELIB using Internal.Runtime.CompilerServices; +#endif #pragma warning disable 0809 //warning CS0809: Obsolete member 'Utf8Span.Equals(object)' overrides non-obsolete member 'object.Equals(object)' #pragma warning disable SA1121 // explicitly using type aliases instead of built-in types +#if SYSTEM_PRIVATE_CORELIB #if TARGET_64BIT using nint = System.Int64; using nuint = System.UInt64; @@ -20,6 +24,10 @@ using nint = System.Int32; using nuint = System.UInt32; #endif +#else +using nint = System.Int64; // https://github.com/dotnet/runtime/issues/33575 - use long/ulong outside of corelib until the compiler supports it +using nuint = System.UInt64; +#endif namespace System.Text { @@ -84,7 +92,11 @@ public Utf8Span this[Range range] Utf8String.ThrowImproperStringSplit(); } +#if SYSTEM_PRIVATE_CORELIB return UnsafeCreateWithoutValidation(new ReadOnlySpan(ref newRef, length)); +#else + return UnsafeCreateWithoutValidation(Bytes.Slice(offset, length)); +#endif } } @@ -117,7 +129,11 @@ internal ref byte DangerousGetMutableReference(nuint index) // Allow retrieving references to just past the end of the span (but shouldn't dereference this). Debug.Assert(index <= (uint)Length, "Caller should've performed bounds checking."); +#if SYSTEM_PRIVATE_CORELIB return ref Unsafe.AddByteOffset(ref DangerousGetMutableReference(), index); +#else + return ref Unsafe.AddByteOffset(ref DangerousGetMutableReference(), (IntPtr)index); +#endif } public bool IsEmptyOrWhiteSpace() => (Utf8Utility.GetIndexOfFirstNonWhiteSpaceChar(Bytes) == Length); @@ -156,7 +172,11 @@ public override int GetHashCode() // UTF-8 textual data, not over arbitrary binary sequences. ulong seed = Marvin.DefaultSeed; +#if SYSTEM_PRIVATE_CORELIB return Marvin.ComputeHash32(ref MemoryMarshal.GetReference(Bytes), (uint)Length /* in bytes */, (uint)seed, (uint)(seed >> 32)); +#else + return Marvin.ComputeHash32(Bytes, seed); +#endif } public int GetHashCode(StringComparison comparison) @@ -225,7 +245,22 @@ public override string ToString() // TODO_UTF8STRING: Since we know the underlying data is immutable, well-formed UTF-8, // we can perform transcoding using an optimized code path that skips all safety checks. +#if !NETSTANDARD2_0 return Encoding.UTF8.GetString(Bytes); +#else + if (IsEmpty) + { + return string.Empty; + } + + unsafe + { + fixed (byte* pBytes = Bytes) + { + return Encoding.UTF8.GetString(pBytes, Length); + } + } +#endif } /// @@ -253,13 +288,28 @@ internal unsafe string ToStringNoReplacement() int utf16CharCount = Length + utf16CodeUnitCountAdjustment; Debug.Assert(utf16CharCount <= Length && utf16CharCount >= 0); +#if !NETSTANDARD2_0 // TODO_UTF8STRING: Can we call string.FastAllocate directly? - return string.Create(utf16CharCount, (pbData: (IntPtr)pData, cbData: Length), (chars, state) => { OperationStatus status = Utf8.ToUtf16(new ReadOnlySpan((byte*)state.pbData, state.cbData), chars, out _, out _, replaceInvalidSequences: false); Debug.Assert(status == OperationStatus.Done, "Did somebody mutate this Utf8String instance unexpectedly?"); }); +#else + char[] buffer = ArrayPool.Shared.Rent(utf16CharCount); + try + { + fixed (char* pBuffer = buffer) + { + Encoding.UTF8.GetChars(pData, Length, pBuffer, utf16CharCount); + return new string(pBuffer, 0, utf16CharCount); + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } +#endif } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Utf8StringComparer.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Utf8StringComparer.cs index 6a264b4cc02e0..0911fd77725a5 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Utf8StringComparer.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Utf8StringComparer.cs @@ -57,8 +57,13 @@ public static Utf8StringComparer FromComparison(StringComparison comparisonType) private sealed class CultureAwareComparer : Utf8StringComparer { +#if SYSTEM_PRIVATE_CORELIB internal static readonly CultureAwareComparer Invariant = new CultureAwareComparer(CompareInfo.Invariant, CompareOptions.None); internal static readonly CultureAwareComparer InvariantIgnoreCase = new CultureAwareComparer(CompareInfo.Invariant, CompareOptions.IgnoreCase); +#else + internal static readonly CultureAwareComparer Invariant = new CultureAwareComparer(CultureInfo.InvariantCulture.CompareInfo, CompareOptions.None); + internal static readonly CultureAwareComparer InvariantIgnoreCase = new CultureAwareComparer(CultureInfo.InvariantCulture.CompareInfo, CompareOptions.IgnoreCase); +#endif private readonly CompareInfo _compareInfo; private readonly CompareOptions _options; diff --git a/src/libraries/System.Private.CoreLib/src/System/Utf8Extensions.CoreLib.cs b/src/libraries/System.Private.CoreLib/src/System/Utf8Extensions.CoreLib.cs new file mode 100644 index 0000000000000..7f0d20d70816c --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Utf8Extensions.CoreLib.cs @@ -0,0 +1,129 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Runtime.CompilerServices; + +namespace System +{ + public static partial class Utf8Extensions + { + /// Creates a new over the portion of the target . + /// The target . + /// Returns default when is null. + public static ReadOnlyMemory AsMemory(this Utf8String? text) + { + if (text is null) + return default; + + return new ReadOnlyMemory(text, 0, text.Length); + } + + /// Creates a new over the portion of the target . + /// The target . + /// The index at which to begin this slice. + /// Returns default when is null. + /// + /// Thrown when the specified index is not in range (<0 or >text.Length). + /// + public static ReadOnlyMemory AsMemory(this Utf8String? text, int start) + { + if (text is null) + { + if (start != 0) + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start); + return default; + } + + if ((uint)start > (uint)text.Length) + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start); + + return new ReadOnlyMemory(text, start, text.Length - start); + } + + /// Creates a new over the portion of the target . + /// The target . + /// The index at which to begin this slice. + public static ReadOnlyMemory AsMemory(this Utf8String? text, Index startIndex) + { + if (text is null) + { + if (!startIndex.Equals(Index.Start)) + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.text); + + return default; + } + + int actualIndex = startIndex.GetOffset(text.Length); + if ((uint)actualIndex > (uint)text.Length) + ThrowHelper.ThrowArgumentOutOfRangeException(); + + return new ReadOnlyMemory(text, actualIndex, text.Length - actualIndex); + } + + /// Creates a new over the portion of the target . + /// The target . + /// The index at which to begin this slice. + /// The desired length for the slice (exclusive). + /// Returns default when is null. + /// + /// Thrown when the specified index or is not in range. + /// + public static ReadOnlyMemory AsMemory(this Utf8String? text, int start, int length) + { + if (text is null) + { + if (start != 0 || length != 0) + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start); + return default; + } + +#if TARGET_64BIT + // See comment in Span.Slice for how this works. + if ((ulong)(uint)start + (ulong)(uint)length > (ulong)(uint)text.Length) + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start); +#else + if ((uint)start > (uint)text.Length || (uint)length > (uint)(text.Length - start)) + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start); +#endif + + return new ReadOnlyMemory(text, start, length); + } + + /// Creates a new over the portion of the target . + /// The target . + /// The range used to indicate the start and length of the sliced string. + public static ReadOnlyMemory AsMemory(this Utf8String? text, Range range) + { + if (text is null) + { + Index startIndex = range.Start; + Index endIndex = range.End; + + if (!startIndex.Equals(Index.Start) || !endIndex.Equals(Index.Start)) + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.text); + + return default; + } + + (int start, int length) = range.GetOffsetAndLength(text.Length); + return new ReadOnlyMemory(text, start, length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ReadOnlySpan CreateSpan(Utf8String text) => + new ReadOnlySpan(ref text.DangerousGetMutableReference(), text.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ReadOnlySpan CreateSpan(Utf8String text, int start) => + new ReadOnlySpan(ref text.DangerousGetMutableReference(start), text.Length - start); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ReadOnlySpan CreateSpan(Utf8String text, int start, int length) => + new ReadOnlySpan(ref text.DangerousGetMutableReference(start), length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ReadOnlyMemory CreateMemoryBytes(Utf8String text, int start, int length) => + new ReadOnlyMemory(text, start, length); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Utf8Extensions.cs b/src/libraries/System.Private.CoreLib/src/System/Utf8Extensions.cs index 9a7d37b885034..2613506abc402 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Utf8Extensions.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Utf8Extensions.cs @@ -9,7 +9,7 @@ namespace System { - public static class Utf8Extensions + public static partial class Utf8Extensions { /// /// Projects as a . @@ -30,7 +30,7 @@ public static ReadOnlySpan AsBytes(this Utf8String? text) if (text is null) return default; - return new ReadOnlySpan(ref text.DangerousGetMutableReference(), text.Length); + return CreateSpan(text); } /// @@ -55,7 +55,7 @@ public static ReadOnlySpan AsBytes(this Utf8String? text, int start) if ((uint)start > (uint)text.Length) ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start); - return new ReadOnlySpan(ref text.DangerousGetMutableReference(start), text.Length - start); + return CreateSpan(text, start); } /// @@ -87,7 +87,7 @@ public static ReadOnlySpan AsBytes(this Utf8String? text, int start, int l ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start); #endif - return new ReadOnlySpan(ref text.DangerousGetMutableReference(start), length); + return CreateSpan(text, start, length); } /// @@ -138,7 +138,7 @@ public static Utf8Span AsSpan(this Utf8String? text, int start) Utf8String.ThrowImproperStringSplit(); } - return Utf8Span.UnsafeCreateWithoutValidation(new ReadOnlySpan(ref text.DangerousGetMutableReference(start), text.Length - start)); + return Utf8Span.UnsafeCreateWithoutValidation(CreateSpan(text, start)); } /// @@ -183,109 +183,7 @@ public static Utf8Span AsSpan(this Utf8String? text, int start, int length) Utf8String.ThrowImproperStringSplit(); } - return Utf8Span.UnsafeCreateWithoutValidation(new ReadOnlySpan(ref text.DangerousGetMutableReference(start), length)); - } - - /// Creates a new over the portion of the target . - /// The target . - /// Returns default when is null. - public static ReadOnlyMemory AsMemory(this Utf8String? text) - { - if (text is null) - return default; - - return new ReadOnlyMemory(text, 0, text.Length); - } - - /// Creates a new over the portion of the target . - /// The target . - /// The index at which to begin this slice. - /// Returns default when is null. - /// - /// Thrown when the specified index is not in range (<0 or >text.Length). - /// - public static ReadOnlyMemory AsMemory(this Utf8String? text, int start) - { - if (text is null) - { - if (start != 0) - ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start); - return default; - } - - if ((uint)start > (uint)text.Length) - ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start); - - return new ReadOnlyMemory(text, start, text.Length - start); - } - - /// Creates a new over the portion of the target . - /// The target . - /// The index at which to begin this slice. - public static ReadOnlyMemory AsMemory(this Utf8String? text, Index startIndex) - { - if (text is null) - { - if (!startIndex.Equals(Index.Start)) - ThrowHelper.ThrowArgumentNullException(ExceptionArgument.text); - - return default; - } - - int actualIndex = startIndex.GetOffset(text.Length); - if ((uint)actualIndex > (uint)text.Length) - ThrowHelper.ThrowArgumentOutOfRangeException(); - - return new ReadOnlyMemory(text, actualIndex, text.Length - actualIndex); - } - - /// Creates a new over the portion of the target . - /// The target . - /// The index at which to begin this slice. - /// The desired length for the slice (exclusive). - /// Returns default when is null. - /// - /// Thrown when the specified index or is not in range. - /// - public static ReadOnlyMemory AsMemory(this Utf8String? text, int start, int length) - { - if (text is null) - { - if (start != 0 || length != 0) - ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start); - return default; - } - -#if TARGET_64BIT - // See comment in Span.Slice for how this works. - if ((ulong)(uint)start + (ulong)(uint)length > (ulong)(uint)text.Length) - ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start); -#else - if ((uint)start > (uint)text.Length || (uint)length > (uint)(text.Length - start)) - ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start); -#endif - - return new ReadOnlyMemory(text, start, length); - } - - /// Creates a new over the portion of the target . - /// The target . - /// The range used to indicate the start and length of the sliced string. - public static ReadOnlyMemory AsMemory(this Utf8String? text, Range range) - { - if (text is null) - { - Index startIndex = range.Start; - Index endIndex = range.End; - - if (!startIndex.Equals(Index.Start) || !endIndex.Equals(Index.Start)) - ThrowHelper.ThrowArgumentNullException(ExceptionArgument.text); - - return default; - } - - (int start, int length) = range.GetOffsetAndLength(text.Length); - return new ReadOnlyMemory(text, start, length); + return Utf8Span.UnsafeCreateWithoutValidation(CreateSpan(text, start, length)); } /// Creates a new over the portion of the target . @@ -296,7 +194,7 @@ public static ReadOnlyMemory AsMemoryBytes(this Utf8String? text) if (text is null) return default; - return new ReadOnlyMemory(text, 0, text.Length); + return CreateMemoryBytes(text, 0, text.Length); } /// Creates a new over the portion of the target . @@ -318,7 +216,7 @@ public static ReadOnlyMemory AsMemoryBytes(this Utf8String? text, int star if ((uint)start > (uint)text.Length) ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start); - return new ReadOnlyMemory(text, start, text.Length - start); + return CreateMemoryBytes(text, start, text.Length - start); } /// Creates a new over the portion of the target . @@ -338,7 +236,7 @@ public static ReadOnlyMemory AsMemoryBytes(this Utf8String? text, Index st if ((uint)actualIndex > (uint)text.Length) ThrowHelper.ThrowArgumentOutOfRangeException(); - return new ReadOnlyMemory(text, actualIndex, text.Length - actualIndex); + return CreateMemoryBytes(text, actualIndex, text.Length - actualIndex); } /// Creates a new over the portion of the target . @@ -367,7 +265,7 @@ public static ReadOnlyMemory AsMemoryBytes(this Utf8String? text, int star ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start); #endif - return new ReadOnlyMemory(text, start, length); + return CreateMemoryBytes(text, start, length); } /// Creates a new over the portion of the target . @@ -387,7 +285,7 @@ public static ReadOnlyMemory AsMemoryBytes(this Utf8String? text, Range ra } (int start, int length) = range.GetOffsetAndLength(text.Length); - return new ReadOnlyMemory(text, start, length); + return CreateMemoryBytes(text, start, length); } /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Utf8String.Comparison.cs b/src/libraries/System.Private.CoreLib/src/System/Utf8String.Comparison.cs index 2d5007e76293b..273df67eae1cc 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Utf8String.Comparison.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Utf8String.Comparison.cs @@ -58,7 +58,7 @@ public static bool AreEquivalent(Utf8String? utf8Text, string? utf16Text) return false; } - return AreEquivalentOrdinalSkipShortCircuitingChecks(utf8Text.AsBytes(), utf16Text); + return AreEquivalentOrdinalSkipShortCircuitingChecks(utf8Text.AsBytes(), utf16Text.AsSpan()); } /// @@ -172,9 +172,14 @@ public bool Contains(Rune value) Span runeBytes = stackalloc byte[Utf8Utility.MaxBytesPerScalar]; int runeBytesWritten = value.EncodeToUtf8(runeBytes); +#if SYSTEM_PRIVATE_CORELIB return SpanHelpers.IndexOf( ref DangerousGetMutableReference(), Length, ref MemoryMarshal.GetReference(runeBytes), runeBytesWritten) >= 0; +#else + return GetSpan() + .IndexOf(runeBytes.Slice(0, runeBytesWritten)) >= 0; +#endif } /// @@ -185,7 +190,11 @@ public bool Contains(Rune value, StringComparison comparison) { // TODO_UTF8STRING: Optimize me to avoid allocations. +#if !NETSTANDARD2_0 return ToString().Contains(value.ToString(), comparison); +#else + return ToString().IndexOf(value.ToString(), comparison) >= 0; +#endif } /// @@ -215,7 +224,11 @@ public bool Contains(Utf8String value, StringComparison comparison) // TODO_UTF8STRING: Optimize me to avoid allocations. +#if !NETSTANDARD2_0 return ToString().Contains(value.ToString(), comparison); +#else + return ToString().IndexOf(value.ToString(), comparison) >= 0; +#endif } /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Utf8String.Construction.cs b/src/libraries/System.Private.CoreLib/src/System/Utf8String.Construction.cs index aedea4d935f03..5ef0f5cfbafa9 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Utf8String.Construction.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Utf8String.Construction.cs @@ -48,7 +48,11 @@ public static bool TryCreateFrom(ReadOnlySpan buffer, [NotNullWhen(true)] // Create and populate the Utf8String instance. Utf8String newString = FastAllocateSkipZeroInit(buffer.Length); +#if SYSTEM_PRIVATE_CORELIB Buffer.Memmove(ref newString.DangerousGetMutableReference(), ref MemoryMarshal.GetReference(buffer), (uint)buffer.Length); +#else + buffer.CopyTo(newString.DangerousGetMutableSpan()); +#endif // Now perform validation. // Reminder: Perform validation over the copy, not over the source. @@ -111,7 +115,11 @@ public static Utf8String CreateFromRelaxed(ReadOnlySpan buffer) // Create and populate the Utf8String instance. Utf8String newString = FastAllocateSkipZeroInit(buffer.Length); +#if SYSTEM_PRIVATE_CORELIB Buffer.Memmove(ref newString.DangerousGetMutableReference(), ref MemoryMarshal.GetReference(buffer), (uint)buffer.Length); +#else + buffer.CopyTo(newString.DangerousGetMutableSpan()); +#endif // Now perform validation & fixup. @@ -256,6 +264,96 @@ internal static Utf8String CreateFromRune(Rune value) return newString; } +#if !SYSTEM_PRIVATE_CORELIB + // Returns 'null' if the input buffer does not represent well-formed UTF-16 data and 'replaceInvalidSequences' is false. + private static byte[]? CreateBufferFromUtf16Common(ReadOnlySpan value, bool replaceInvalidSequences) + { + // Shortcut: Since we expect most strings to be small-ish, first try a one-pass + // operation where we transcode directly on to the stack and then copy the validated + // data into the new Utf8String instance. It's still O(n), but it should have a smaller + // constant factor than a typical "count + transcode" combo. + + OperationStatus status; + byte[] newBuffer; + + if (value.Length <= MAX_STACK_TRANSCODE_CHAR_COUNT /* in chars */) + { + if (value.IsEmpty) + { + return Utf8String.Empty._bytes; + } + + Span scratch = stackalloc byte[MAX_STACK_TRANSCODE_CHAR_COUNT * MAX_UTF8_BYTES_PER_UTF16_CHAR]; // largest possible expansion, as explained below + status = Utf8.FromUtf16(value, scratch, out _, out int scratchBytesWritten, replaceInvalidSequences); + Debug.Assert(status == OperationStatus.Done || status == OperationStatus.InvalidData); + + if (status == OperationStatus.InvalidData) + { + return null; + } + + // At this point we know transcoding succeeded, so the original input data was well-formed. + // We'll memcpy the scratch buffer into the new Utf8String instance, which is very fast. + + newBuffer = new byte[scratchBytesWritten + 1]; // null-terminated + scratch.Slice(0, scratchBytesWritten).CopyTo(newBuffer); + return newBuffer; + } + + // First, determine how many UTF-8 bytes we'll need in order to represent this data. + // This also checks the input data for well-formedness. + + long utf8CodeUnitCountAdjustment; + + unsafe + { + fixed (char* pChars = &MemoryMarshal.GetReference(value)) + { + if (Utf16Utility.GetPointerToFirstInvalidChar(pChars, value.Length, out utf8CodeUnitCountAdjustment, out int _) != (pChars + (uint)value.Length)) + { + return null; + } + } + } + + // The max possible expansion transcoding UTF-16 to UTF-8 is that each input char corresponds + // to 3 UTF-8 bytes. This is most common in CJK languages. Since the input buffer could be + // up to int.MaxValue elements in length, we need to use a 64-bit value to hold the total + // required UTF-8 byte length. However, the VM places restrictions on how large a Utf8String + // instance can be, and the maximum allowed element count is just under int.MaxValue. (This + // mirrors the restrictions already in place for System.String.) The VM will throw an + // OutOfMemoryException if anybody tries to create a Utf8String instance larger than that, + // so if we detect any sort of overflow we'll end up passing int.MaxValue down to the allocation + // routine. This normalizes the OutOfMemoryException the caller sees. + + long totalUtf8BytesRequired = (uint)value.Length + utf8CodeUnitCountAdjustment; + if (totalUtf8BytesRequired >= int.MaxValue) + { + totalUtf8BytesRequired = int.MaxValue - 1; + } + + // We can get away with FastAllocateSkipZeroInit here because we're not going to return the + // new Utf8String instance to the caller if we don't overwrite every byte of the buffer. + + newBuffer = new byte[(int)totalUtf8BytesRequired + 1]; // null-terminated + + // Now transcode the UTF-16 input into the newly allocated Utf8String's buffer. We can't call the + // "skip validation" transcoder because the caller could've mutated the input buffer between the + // initial counting step and the transcoding step below. + + status = Utf8.FromUtf16(value, newBuffer.AsSpan(0, newBuffer.Length - 1), out _, out int bytesWritten, replaceInvalidSequences: false); + if (status != OperationStatus.Done || bytesWritten != newBuffer.Length - 1) + { + // Did somebody mutate our input buffer? Shouldn't be any other way this could happen. + + return null; + } + + return newBuffer; + } +#endif // !SYSTEM_PRIVATE_CORELIB + +#if !NETSTANDARD2_0 /// /// Creates a new instance, allowing the provided delegate to populate the /// instance data of the returned object. @@ -348,37 +446,6 @@ public static Utf8String CreateRelaxed(int length, TState state, SpanAct return Utf8Utility.ValidateAndFixupUtf8String(newString); } - /// - /// Creates a new instance populated with a copy of the provided contents. - /// Please see remarks for important safety information about this method. - /// - /// The contents to copy to the new . - /// - /// This factory method can be used as an optimization to skip the validation step that the - /// constructors normally perform. The contract of this method requires that - /// contain only well-formed UTF-8 data, as - /// contractually guarantees that it contains only well-formed UTF-8 data, and runtime instability - /// could occur if a caller violates this guarantee. - /// - public static Utf8String UnsafeCreateWithoutValidation(ReadOnlySpan utf8Contents) - { - if (utf8Contents.IsEmpty) - { - return Empty; // special-case empty input - } - - // Create and populate the Utf8String instance. - - Utf8String newString = FastAllocateSkipZeroInit(utf8Contents.Length); - utf8Contents.CopyTo(newString.DangerousGetMutableSpan()); - - // The line below is removed entirely in release builds. - - Debug.Assert(Utf8Utility.IsWellFormedUtf8(newString.AsBytes()), "Buffer contained ill-formed UTF-8 data."); - - return newString; - } - /// /// Creates a new instance, allowing the provided delegate to populate the /// instance data of the returned object. Please see remarks for important safety information about @@ -424,6 +491,39 @@ public static Utf8String UnsafeCreateWithoutValidation(int length, TStat return newString; } +#endif // !NETSTANDARD2_0 + + /// + /// Creates a new instance populated with a copy of the provided contents. + /// Please see remarks for important safety information about this method. + /// + /// The contents to copy to the new . + /// + /// This factory method can be used as an optimization to skip the validation step that the + /// constructors normally perform. The contract of this method requires that + /// contain only well-formed UTF-8 data, as + /// contractually guarantees that it contains only well-formed UTF-8 data, and runtime instability + /// could occur if a caller violates this guarantee. + /// + public static Utf8String UnsafeCreateWithoutValidation(ReadOnlySpan utf8Contents) + { + if (utf8Contents.IsEmpty) + { + return Empty; // special-case empty input + } + + // Create and populate the Utf8String instance. + + Utf8String newString = FastAllocateSkipZeroInit(utf8Contents.Length); + utf8Contents.CopyTo(newString.DangerousGetMutableSpan()); + // TODO_UTF8STRING: Zero-init was skipped above, but we didn't null-terminate the string + + // The line below is removed entirely in release builds. + + Debug.Assert(Utf8Utility.IsWellFormedUtf8(newString.AsBytes()), "Buffer contained ill-formed UTF-8 data."); + + return newString; + } /* * HELPER METHODS diff --git a/src/libraries/System.Private.CoreLib/src/System/Utf8String.Enumeration.cs b/src/libraries/System.Private.CoreLib/src/System/Utf8String.Enumeration.cs index 2ae5ced65f567..9b2ea648539b2 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Utf8String.Enumeration.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Utf8String.Enumeration.cs @@ -8,15 +8,6 @@ using System.Diagnostics; using System.Text; -#pragma warning disable SA1121 // explicitly using type aliases instead of built-in types -#if TARGET_64BIT -using nint = System.Int64; -using nuint = System.UInt64; -#else -using nint = System.Int32; -using nuint = System.UInt32; -#endif - namespace System { public sealed partial class Utf8String diff --git a/src/libraries/System.Private.CoreLib/src/System/Utf8String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/Utf8String.Manipulation.cs index 85acec8d16f90..ac66e0e0668a3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Utf8String.Manipulation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Utf8String.Manipulation.cs @@ -10,7 +10,10 @@ using System.Runtime.InteropServices; using System.Text; using System.Text.Unicode; + +#if SYSTEM_PRIVATE_CORELIB using Internal.Runtime.CompilerServices; +#endif namespace System { @@ -62,7 +65,12 @@ private Utf8String InternalSubstring(int startIndex, int length) } Utf8String newString = FastAllocateSkipZeroInit(length); +#if SYSTEM_PRIVATE_CORELIB Buffer.Memmove(ref newString.DangerousGetMutableReference(), ref this.DangerousGetMutableReference(startIndex), (uint)length); +#else + this.GetSpan().Slice(startIndex, length).CopyTo(newString.DangerousGetMutableSpan()); +#endif + return newString; } @@ -90,7 +98,11 @@ private Utf8String InternalSubstringWithoutCorrectnessChecks(int startIndex, int else { Utf8String newString = FastAllocateSkipZeroInit(length); +#if SYSTEM_PRIVATE_CORELIB Buffer.Memmove(ref newString.DangerousGetMutableReference(), ref this.DangerousGetMutableReference(startIndex), (uint)length); +#else + this.GetSpan().Slice(startIndex, length).CopyTo(newString.DangerousGetMutableSpan()); +#endif return newString; } } @@ -644,7 +656,11 @@ internal readonly bool DeconstructHelper(in Utf8Span source, out Utf8Span firstI int searchRune = SearchRune; // local copy so as to avoid struct tearing if (searchRune >= 0) { +#if NETCOREAPP3_0 + wasMatchFound = searchSpan.TryFind(new Rune((uint)searchRune), out matchRange); +#else wasMatchFound = searchSpan.TryFind(Rune.UnsafeCreate((uint)searchRune), out matchRange); +#endif } else { @@ -739,3 +755,14 @@ public void Deconstruct(out Utf8String before, out Utf8String? after) } } } + +#if !SYSTEM_PRIVATE_CORELIB +namespace System.Diagnostics +{ + [AttributeUsage(AttributeTargets.Class | AttributeTargets.Method | AttributeTargets.Constructor | AttributeTargets.Struct, Inherited = false)] + internal sealed class StackTraceHiddenAttribute : Attribute + { + public StackTraceHiddenAttribute() { } + } +} +#endif diff --git a/src/libraries/System.Private.CoreLib/src/System/Utf8String.cs b/src/libraries/System.Private.CoreLib/src/System/Utf8String.cs index 7d9511d6f818d..e6cba4d2e4c2b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Utf8String.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Utf8String.cs @@ -8,9 +8,13 @@ using System.Runtime.CompilerServices; using System.Text; using System.Text.Unicode; + +#if SYSTEM_PRIVATE_CORELIB using Internal.Runtime.CompilerServices; +#endif #pragma warning disable SA1121 // explicitly using type aliases instead of built-in types +#if SYSTEM_PRIVATE_CORELIB #if TARGET_64BIT using nint = System.Int64; using nuint = System.UInt64; @@ -18,6 +22,10 @@ using nint = System.Int32; using nuint = System.UInt32; #endif +#else +using nint = System.Int64; // https://github.com/dotnet/runtime/issues/33575 - use long/ulong outside of corelib until the compiler supports it +using nuint = System.UInt64; +#endif namespace System { @@ -122,7 +130,11 @@ internal ref byte DangerousGetMutableReference(nuint index) // Allow retrieving references to the null terminator. Debug.Assert(index <= (uint)Length, "Caller should've performed bounds checking."); +#if SYSTEM_PRIVATE_CORELIB return ref Unsafe.AddByteOffset(ref DangerousGetMutableReference(), index); +#else + return ref Unsafe.AddByteOffset(ref DangerousGetMutableReference(), (IntPtr)index); +#endif } /// @@ -149,7 +161,11 @@ public bool Equals(Utf8String? value) return !(value is null) && this.Length == value.Length +#if SYSTEM_PRIVATE_CORELIB && SpanHelpers.SequenceEqual(ref this.DangerousGetMutableReference(), ref value.DangerousGetMutableReference(), (uint)Length); +#else + && this.GetSpan().SequenceEqual(value.GetSpan()); +#endif } /// @@ -174,7 +190,11 @@ public static bool Equals(Utf8String? left, Utf8String? right) return !(left is null) && !(right is null) && left.Length == right.Length +#if SYSTEM_PRIVATE_CORELIB && SpanHelpers.SequenceEqual(ref left.DangerousGetMutableReference(), ref right.DangerousGetMutableReference(), (uint)left.Length); +#else + && left.GetSpan().SequenceEqual(right.GetSpan()); +#endif } /// @@ -196,7 +216,11 @@ public override int GetHashCode() // TODO_UTF8STRING: Consider whether this should use a different seed than String.GetHashCode. ulong seed = Marvin.DefaultSeed; +#if SYSTEM_PRIVATE_CORELIB return Marvin.ComputeHash32(ref DangerousGetMutableReference(), (uint)_length /* in bytes */, (uint)seed, (uint)(seed >> 32)); +#else + return Marvin.ComputeHash32(_bytes, seed); +#endif } /// @@ -250,44 +274,22 @@ public override string ToString() { // TODO_UTF8STRING: Optimize the call below, potentially by avoiding the two-pass. +#if !NETSTANDARD2_0 return Encoding.UTF8.GetString(this.AsBytesSkipNullCheck()); - } - - /// - /// Converts this instance to a . - /// - /// - /// This routine throws if the underlying instance - /// contains invalid UTF-8 data. - /// - internal unsafe string ToStringNoReplacement() - { - // TODO_UTF8STRING: Optimize the call below, potentially by avoiding the two-pass. - - int utf16CharCount; +#else + if (Length == 0) + { + return string.Empty; + } - fixed (byte* pData = &_firstByte) + unsafe { - byte* pFirstInvalidByte = Utf8Utility.GetPointerToFirstInvalidByte(pData, this.Length, out int utf16CodeUnitCountAdjustment, out _); - if (pFirstInvalidByte != pData + (uint)this.Length) + fixed (byte* pBytes = this.AsBytesSkipNullCheck()) { - // Saw bad UTF-8 data. - // TODO_UTF8STRING: Throw a better exception below? - - ThrowHelper.ThrowInvalidOperationException(); + return Encoding.UTF8.GetString(pBytes, Length); } - - utf16CharCount = this.Length + utf16CodeUnitCountAdjustment; - Debug.Assert(utf16CharCount <= this.Length && utf16CharCount >= 0); } - - // TODO_UTF8STRING: Can we call string.FastAllocate directly? - - return string.Create(utf16CharCount, this, (chars, thisObj) => - { - OperationStatus status = Utf8.ToUtf16(thisObj.AsBytes(), chars, out _, out _, replaceInvalidSequences: false); - Debug.Assert(status == OperationStatus.Done, "Did somebody mutate this Utf8String instance unexpectedly?"); - }); +#endif } } } diff --git a/src/libraries/System.Utf8String.Experimental/pkg/System.Utf8String.Experimental.pkgproj b/src/libraries/System.Utf8String.Experimental/pkg/System.Utf8String.Experimental.pkgproj index 53280e1329e0f..1577c85ef1902 100644 --- a/src/libraries/System.Utf8String.Experimental/pkg/System.Utf8String.Experimental.pkgproj +++ b/src/libraries/System.Utf8String.Experimental/pkg/System.Utf8String.Experimental.pkgproj @@ -3,7 +3,7 @@ - netcoreapp5.0; + net461;netcoreapp2.0;uap10.0.16299;$(AllXamarinFrameworks) diff --git a/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.Forwards.cs b/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.Forwards.cs new file mode 100644 index 0000000000000..865608e4ac863 --- /dev/null +++ b/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.Forwards.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#if !NETSTANDARD2_0 +[assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.Index))] +[assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.Range))] + +#if !NETSTANDARD2_1 +[assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.Text.Rune))] +#endif // !NETSTANDARD2_1 + +#endif // !NETSTANDARD2_0 diff --git a/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.Range.cs b/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.Range.cs new file mode 100644 index 0000000000000..58cadf8efc353 --- /dev/null +++ b/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.Range.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// ------------------------------------------------------------------------------ +// Changes to this file must follow the https://aka.ms/api-review process. +// ------------------------------------------------------------------------------ + +namespace System +{ + public readonly partial struct Index : System.IEquatable + { + private readonly int _dummyPrimitive; + public Index(int value, bool fromEnd = false) { throw null; } + public static System.Index End { get { throw null; } } + public bool IsFromEnd { get { throw null; } } + public static System.Index Start { get { throw null; } } + public int Value { get { throw null; } } + public bool Equals(System.Index other) { throw null; } + public override bool Equals(object? value) { throw null; } + public static System.Index FromEnd(int value) { throw null; } + public static System.Index FromStart(int value) { throw null; } + public override int GetHashCode() { throw null; } + public int GetOffset(int length) { throw null; } + public static implicit operator System.Index(int value) { throw null; } + public override string ToString() { throw null; } + } + public readonly partial struct Range : System.IEquatable + { + private readonly int _dummyPrimitive; + public Range(System.Index start, System.Index end) { throw null; } + public static System.Range All { get { throw null; } } + public System.Index End { get { throw null; } } + public System.Index Start { get { throw null; } } + public static System.Range EndAt(System.Index end) { throw null; } + public override bool Equals(object? value) { throw null; } + public bool Equals(System.Range other) { throw null; } + public override int GetHashCode() { throw null; } + public (int Offset, int Length) GetOffsetAndLength(int length) { throw null; } + public static System.Range StartAt(System.Index start) { throw null; } + public override string ToString() { throw null; } + } +} diff --git a/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.Rune.cs b/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.Rune.cs new file mode 100644 index 0000000000000..0cf292727a56f --- /dev/null +++ b/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.Rune.cs @@ -0,0 +1,76 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// ------------------------------------------------------------------------------ +// Changes to this file must follow the https://aka.ms/api-review process. +// ------------------------------------------------------------------------------ + +namespace System.Text +{ + public readonly partial struct Rune : System.IComparable, System.IEquatable + { + private readonly int _dummyPrimitive; + public Rune(char ch) { throw null; } + public Rune(char highSurrogate, char lowSurrogate) { throw null; } + public Rune(int value) { throw null; } + [System.CLSCompliantAttribute(false)] + public Rune(uint value) { throw null; } + public bool IsAscii { get { throw null; } } + public bool IsBmp { get { throw null; } } + public int Plane { get { throw null; } } + public static System.Text.Rune ReplacementChar { get { throw null; } } + public int Utf16SequenceLength { get { throw null; } } + public int Utf8SequenceLength { get { throw null; } } + public int Value { get { throw null; } } + public int CompareTo(System.Text.Rune other) { throw null; } + public static System.Buffers.OperationStatus DecodeFromUtf16(System.ReadOnlySpan source, out System.Text.Rune result, out int charsConsumed) { throw null; } + public static System.Buffers.OperationStatus DecodeFromUtf8(System.ReadOnlySpan source, out System.Text.Rune result, out int bytesConsumed) { throw null; } + public static System.Buffers.OperationStatus DecodeLastFromUtf16(System.ReadOnlySpan source, out System.Text.Rune result, out int charsConsumed) { throw null; } + public static System.Buffers.OperationStatus DecodeLastFromUtf8(System.ReadOnlySpan source, out System.Text.Rune value, out int bytesConsumed) { throw null; } + public int EncodeToUtf16(System.Span destination) { throw null; } + public int EncodeToUtf8(System.Span destination) { throw null; } + public override bool Equals(object? obj) { throw null; } + public bool Equals(System.Text.Rune other) { throw null; } + public override int GetHashCode() { throw null; } + public static double GetNumericValue(System.Text.Rune value) { throw null; } + public static System.Text.Rune GetRuneAt(string input, int index) { throw null; } + public static System.Globalization.UnicodeCategory GetUnicodeCategory(System.Text.Rune value) { throw null; } + public static bool IsControl(System.Text.Rune value) { throw null; } + public static bool IsDigit(System.Text.Rune value) { throw null; } + public static bool IsLetter(System.Text.Rune value) { throw null; } + public static bool IsLetterOrDigit(System.Text.Rune value) { throw null; } + public static bool IsLower(System.Text.Rune value) { throw null; } + public static bool IsNumber(System.Text.Rune value) { throw null; } + public static bool IsPunctuation(System.Text.Rune value) { throw null; } + public static bool IsSeparator(System.Text.Rune value) { throw null; } + public static bool IsSymbol(System.Text.Rune value) { throw null; } + public static bool IsUpper(System.Text.Rune value) { throw null; } + public static bool IsValid(int value) { throw null; } + [System.CLSCompliantAttribute(false)] + public static bool IsValid(uint value) { throw null; } + public static bool IsWhiteSpace(System.Text.Rune value) { throw null; } + public static bool operator ==(System.Text.Rune left, System.Text.Rune right) { throw null; } + public static explicit operator System.Text.Rune(char ch) { throw null; } + public static explicit operator System.Text.Rune(int value) { throw null; } + [System.CLSCompliantAttribute(false)] + public static explicit operator System.Text.Rune(uint value) { throw null; } + public static bool operator >(System.Text.Rune left, System.Text.Rune right) { throw null; } + public static bool operator >=(System.Text.Rune left, System.Text.Rune right) { throw null; } + public static bool operator !=(System.Text.Rune left, System.Text.Rune right) { throw null; } + public static bool operator <(System.Text.Rune left, System.Text.Rune right) { throw null; } + public static bool operator <=(System.Text.Rune left, System.Text.Rune right) { throw null; } + public static System.Text.Rune ToLower(System.Text.Rune value, System.Globalization.CultureInfo culture) { throw null; } + public static System.Text.Rune ToLowerInvariant(System.Text.Rune value) { throw null; } + public override string ToString() { throw null; } + public static System.Text.Rune ToUpper(System.Text.Rune value, System.Globalization.CultureInfo culture) { throw null; } + public static System.Text.Rune ToUpperInvariant(System.Text.Rune value) { throw null; } + public static bool TryCreate(char highSurrogate, char lowSurrogate, out System.Text.Rune result) { throw null; } + public static bool TryCreate(char ch, out System.Text.Rune result) { throw null; } + public static bool TryCreate(int value, out System.Text.Rune result) { throw null; } + [System.CLSCompliantAttribute(false)] + public static bool TryCreate(uint value, out System.Text.Rune result) { throw null; } + public bool TryEncodeToUtf16(System.Span destination, out int charsWritten) { throw null; } + public bool TryEncodeToUtf8(System.Span destination, out int bytesWritten) { throw null; } + public static bool TryGetRuneAt(string input, int index, out System.Text.Rune value) { throw null; } + } +} diff --git a/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.cs b/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.cs index a6a2339b633fb..4a906ba12ca44 100644 --- a/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.cs +++ b/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.cs @@ -54,11 +54,6 @@ public static partial class Utf8Extensions public static System.ReadOnlySpan AsBytes(this System.Utf8String? text) { throw null; } public static System.ReadOnlySpan AsBytes(this System.Utf8String? text, int start) { throw null; } public static System.ReadOnlySpan AsBytes(this System.Utf8String? text, int start, int length) { throw null; } - public static System.ReadOnlyMemory AsMemory(this System.Utf8String? text) { throw null; } - public static System.ReadOnlyMemory AsMemory(this System.Utf8String? text, System.Index startIndex) { throw null; } - public static System.ReadOnlyMemory AsMemory(this System.Utf8String? text, int start) { throw null; } - public static System.ReadOnlyMemory AsMemory(this System.Utf8String? text, int start, int length) { throw null; } - public static System.ReadOnlyMemory AsMemory(this System.Utf8String? text, System.Range range) { throw null; } public static System.ReadOnlyMemory AsMemoryBytes(this System.Utf8String? text) { throw null; } public static System.ReadOnlyMemory AsMemoryBytes(this System.Utf8String? text, System.Index startIndex) { throw null; } public static System.ReadOnlyMemory AsMemoryBytes(this System.Utf8String? text, int start) { throw null; } @@ -96,10 +91,14 @@ public Utf8String(string value) { } public bool Contains(System.Text.Rune value, System.StringComparison comparison) { throw null; } public bool Contains(System.Utf8String value) { throw null; } public bool Contains(System.Utf8String value, System.StringComparison comparison) { throw null; } +#if !NETSTANDARD2_0 public static System.Utf8String Create(int length, TState state, System.Buffers.SpanAction action) { throw null; } +#endif public static System.Utf8String CreateFromRelaxed(System.ReadOnlySpan buffer) { throw null; } public static System.Utf8String CreateFromRelaxed(System.ReadOnlySpan buffer) { throw null; } +#if !NETSTANDARD2_0 public static System.Utf8String CreateRelaxed(int length, TState state, System.Buffers.SpanAction action) { throw null; } +#endif public bool EndsWith(char value) { throw null; } public bool EndsWith(char value, System.StringComparison comparison) { throw null; } public bool EndsWith(System.Text.Rune value) { throw null; } @@ -170,7 +169,9 @@ public Utf8String(string value) { } public bool TryFindLast(System.Utf8String value, out System.Range range) { throw null; } public bool TryFindLast(System.Utf8String value, System.StringComparison comparisonType, out System.Range range) { throw null; } public static System.Utf8String UnsafeCreateWithoutValidation(System.ReadOnlySpan utf8Contents) { throw null; } +#if !NETSTANDARD2_0 public static System.Utf8String UnsafeCreateWithoutValidation(int length, TState state, System.Buffers.SpanAction action) { throw null; } +#endif public readonly partial struct ByteEnumerable : System.Collections.Generic.IEnumerable { private readonly object _dummy; @@ -277,7 +278,6 @@ public Utf8StringContent(System.Utf8String content) { } public Utf8StringContent(System.Utf8String content, string? mediaType) { } protected override System.Threading.Tasks.Task CreateContentReadStreamAsync() { throw null; } protected override System.Threading.Tasks.Task SerializeToStreamAsync(System.IO.Stream stream, System.Net.TransportContext? context) { throw null; } - protected override System.Threading.Tasks.Task SerializeToStreamAsync(System.IO.Stream stream, System.Net.TransportContext? context, System.Threading.CancellationToken cancellationToken) { throw null; } protected override bool TryComputeLength(out long length) { throw null; } } } diff --git a/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.csproj b/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.csproj index 8356230578e6c..7932a85d40138 100644 --- a/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.csproj +++ b/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.csproj @@ -1,15 +1,36 @@ - + true $(NoWarn);0809;0618 - $(NetCoreAppCurrent) + netstandard2.0;netstandard2.1;netcoreapp3.0;$(NetCoreAppCurrent) enable + - + + + + + + + + + + + + + + + + + + + + + diff --git a/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.netcoreapp5.0.cs b/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.netcoreapp5.0.cs new file mode 100644 index 0000000000000..d65563d272f25 --- /dev/null +++ b/src/libraries/System.Utf8String.Experimental/ref/System.Utf8String.Experimental.netcoreapp5.0.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// ------------------------------------------------------------------------------ +// Changes to this file must follow the https://aka.ms/api-review process. +// ------------------------------------------------------------------------------ + +namespace System +{ + public static partial class Utf8Extensions + { + public static System.ReadOnlyMemory AsMemory(this System.Utf8String? text) { throw null; } + public static System.ReadOnlyMemory AsMemory(this System.Utf8String? text, System.Index startIndex) { throw null; } + public static System.ReadOnlyMemory AsMemory(this System.Utf8String? text, int start) { throw null; } + public static System.ReadOnlyMemory AsMemory(this System.Utf8String? text, int start, int length) { throw null; } + public static System.ReadOnlyMemory AsMemory(this System.Utf8String? text, System.Range range) { throw null; } + } +} +namespace System.Net.Http +{ + public sealed partial class Utf8StringContent : System.Net.Http.HttpContent + { + protected override System.Threading.Tasks.Task SerializeToStreamAsync(System.IO.Stream stream, System.Net.TransportContext? context, System.Threading.CancellationToken cancellationToken) { throw null; } + } +} diff --git a/src/libraries/System.Utf8String.Experimental/src/Resources/Strings.resx b/src/libraries/System.Utf8String.Experimental/src/Resources/Strings.resx index 1af7de150c99c..a02a1757a6902 100644 --- a/src/libraries/System.Utf8String.Experimental/src/Resources/Strings.resx +++ b/src/libraries/System.Utf8String.Experimental/src/Resources/Strings.resx @@ -117,4 +117,52 @@ System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + Index was out of range. Must be non-negative and less than the size of the collection. + + + Non-negative number required. + + + UTF-16 surrogate code points (U+D800..U+DFFF) are disallowed. + + + Argument cannot be an empty span. + + + Argument cannot be null or empty. + + + Cannot extract a Unicode scalar value from the specified index in the input. + + + Destination is too short. + + + Illegal enum value: {0}. + + + The string must be null-terminated. + + + The string comparison type passed in is currently not supported. + + + Cannot call Utf8Span.Equals(object). Use Equals(Utf8Span) or operator == instead. + + + UTF-8 searching only supports StringComparison Ordinal and OrdinalIgnoreCase on this platform. + + + The callback populated its buffer with ill-formed UTF-8 data. Callbacks are required to populate the buffer only with well-formed UTF-8 data. + + + Cannot create the desired substring because it would split a multi-byte UTF-8 subsequence. + + + The input buffer contained ill-formed UTF-16 data. + + + The input buffer contained ill-formed UTF-8 data. + \ No newline at end of file diff --git a/src/libraries/System.Utf8String.Experimental/src/System.Utf8String.Experimental.csproj b/src/libraries/System.Utf8String.Experimental/src/System.Utf8String.Experimental.csproj index fb6e385f0988b..dd5adc8a69195 100644 --- a/src/libraries/System.Utf8String.Experimental/src/System.Utf8String.Experimental.csproj +++ b/src/libraries/System.Utf8String.Experimental/src/System.Utf8String.Experimental.csproj @@ -1,15 +1,174 @@ - + true - true - $(NetCoreAppCurrent)-Windows_NT;$(NetCoreAppCurrent)-Unix + + + $(NoWarn);CS3019;CS0162 + true + netstandard2.0;netstandard2.1;netcoreapp3.0;$(NetCoreAppCurrent)-Windows_NT;$(NetCoreAppCurrent)-Unix enable + $(DefineContants);FEATURE_UTF8STRING - + + + System\Index.cs + + + System\Numerics\BitOperations.cs + + + System\Numerics\Hashing\HashHelpers.cs + + + System\Range.cs + + + System\Text\Rune.cs + + + System\Text\Unicode\Utf8.cs + + + + + System\Numerics\BitOperations.cs + + + System\Text\Rune.cs + + + System\Text\Unicode\Utf8.cs + + + + + + + + + + Common\System\Marvin.cs + + + Common\System\NotImplemented.cs + + + System\Char8.cs + + + System\Text\TrimType.cs + + + System\Text\UnicodeDebug.cs + + + System\Text\UnicodeUtility.cs + + + System\Text\Unicode\Utf16Utility.cs + + + System\Text\Unicode\Utf16Utility.Validation.cs + + + System\Text\Utf8StringComparer.cs + + + System\Text\ASCIIUtility.cs + + + System\Text\ASCIIUtility.Helpers.cs + + + System\Text\Unicode\Utf8Utility.cs + + + System\Text\Unicode\Utf8Utility.Helpers.cs + + + System\Text\Unicode\Utf8Utility.Validation.cs + + + System\Text\Unicode\Utf8Utility.Transcoding.cs + + + System\Text\Unicode\Utf8Utility.WhiteSpace.NonCoreLib.cs + + + System\Utf8StringSplitOptions.cs + + + System\Utf8Extensions.cs + + + System\Utf8String.cs + + + System\Utf8String.Comparison.cs + + + System\Utf8String.Construction.cs + + + System\Utf8String.Conversion.cs + + + System\Utf8String.Enumeration.cs + + + System\Utf8String.Manipulation.cs + + + System\Utf8String.Searching.cs + + + System\Text\Utf8Span.cs + + + System\Text\Utf8Span.Comparison.cs + + + System\Text\Utf8Span.Conversion.cs + + + System\Text\Utf8Span.Enumeration.cs + + + System\Text\Utf8Span.Manipulation.cs + + + System\Text\Utf8Span.Searching.cs + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/libraries/System.Utf8String.Experimental/src/System/Globalization/GlobalizationMode.cs b/src/libraries/System.Utf8String.Experimental/src/System/Globalization/GlobalizationMode.cs new file mode 100644 index 0000000000000..2f653ee75e970 --- /dev/null +++ b/src/libraries/System.Utf8String.Experimental/src/System/Globalization/GlobalizationMode.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace System.Globalization +{ + internal static partial class GlobalizationMode + { + internal static bool Invariant { get; } = false; // TODO: should we enable this? + } +} diff --git a/src/libraries/System.Utf8String.Experimental/src/System/IO/Utf8StringStream.cs b/src/libraries/System.Utf8String.Experimental/src/System/IO/Utf8StringStream.cs index 39bf7ebf469b3..39577742f56a1 100644 --- a/src/libraries/System.Utf8String.Experimental/src/System/IO/Utf8StringStream.cs +++ b/src/libraries/System.Utf8String.Experimental/src/System/IO/Utf8StringStream.cs @@ -57,7 +57,11 @@ public override int Read(byte[] buffer, int offset, int count) return Read(new Span(buffer, offset, count)); } - public override int Read(Span buffer) + public +#if !NETSTANDARD2_0 + override +#endif + int Read(Span buffer) { ReadOnlySpan contentToWrite = _content.AsBytes(_position); if (buffer.Length < contentToWrite.Length) @@ -76,10 +80,12 @@ public override Task ReadAsync(byte[] buffer, int offset, int count, Cancel return Task.FromResult(Read(new Span(buffer, offset, count))); } +#if !NETSTANDARD2_0 public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) { return new ValueTask(Read(buffer.Span)); } +#endif public override int ReadByte() { @@ -122,12 +128,15 @@ public override long Seek(long offset, SeekOrigin origin) public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); +#if !NETSTANDARD2_0 public override void Write(ReadOnlySpan buffer) => throw new NotSupportedException(); +#endif public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => throw new NotSupportedException(); +#if !NETSTANDARD2_0 public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) => throw new NotSupportedException(); - +#endif public override void WriteByte(byte value) => throw new NotSupportedException(); } } diff --git a/src/libraries/System.Utf8String.Experimental/src/System/Net/Http/Utf8StringContent.cs b/src/libraries/System.Utf8String.Experimental/src/System/Net/Http/Utf8StringContent.cs index 5f7ab00395b15..3db8f03c8fc18 100644 --- a/src/libraries/System.Utf8String.Experimental/src/System/Net/Http/Utf8StringContent.cs +++ b/src/libraries/System.Utf8String.Experimental/src/System/Net/Http/Utf8StringContent.cs @@ -2,8 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System.Buffers; using System.IO; using System.Net.Http.Headers; +using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; @@ -40,11 +42,34 @@ public Utf8StringContent(Utf8String content, string? mediaType) protected override Task CreateContentReadStreamAsync() => Task.FromResult(new Utf8StringStream(_content)); +#if NETSTANDARD2_0 + protected async override Task SerializeToStreamAsync(Stream stream, TransportContext? context) + { + ReadOnlyMemory buffer = _content.AsMemoryBytes(); + if (MemoryMarshal.TryGetArray(buffer, out ArraySegment array)) + { + await stream.WriteAsync(array.Array, array.Offset, array.Count).ConfigureAwait(false); + } + else + { + byte[] localBuffer = ArrayPool.Shared.Rent(buffer.Length); + buffer.Span.CopyTo(localBuffer); + + await stream.WriteAsync(localBuffer, 0, buffer.Length).ConfigureAwait(false); + + ArrayPool.Shared.Return(localBuffer); + } + } +#elif NETSTANDARD2_1 || NETCOREAPP3_0 + protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context) => + stream.WriteAsync(_content.AsMemoryBytes()).AsTask(); +#else protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context) => SerializeToStreamAsync(stream, context, default); protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context, CancellationToken cancellationToken) => stream.WriteAsync(_content.AsMemoryBytes(), cancellationToken).AsTask(); +#endif protected override bool TryComputeLength(out long length) { diff --git a/src/libraries/System.Utf8String.Experimental/src/System/Runtime/Intrinsics/Intrinsics.Shims.cs b/src/libraries/System.Utf8String.Experimental/src/System/Runtime/Intrinsics/Intrinsics.Shims.cs new file mode 100644 index 0000000000000..12fba80e1a98f --- /dev/null +++ b/src/libraries/System.Utf8String.Experimental/src/System/Runtime/Intrinsics/Intrinsics.Shims.cs @@ -0,0 +1,105 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace System.Runtime.Intrinsics +{ + internal static class Vector128 + { + public static Vector128 Create(short value) => throw new PlatformNotSupportedException(); + public static Vector128 Create(ushort value) => throw new PlatformNotSupportedException(); + public static Vector128 CreateScalarUnsafe(ulong value) => throw new PlatformNotSupportedException(); + public static Vector128 AsByte(this Vector128 vector) where T : struct => throw new PlatformNotSupportedException(); + public static Vector128 AsInt16(this Vector128 vector) where T : struct => throw new PlatformNotSupportedException(); + public static Vector128 AsUInt16(this Vector128 vector) where T : struct => throw new PlatformNotSupportedException(); + public static Vector128 AsUInt32(this Vector128 vector) where T : struct => throw new PlatformNotSupportedException(); + public static Vector128 AsUInt64(this Vector128 vector) where T : struct => throw new PlatformNotSupportedException(); + public static T GetElement(this Vector128 vector, int index) where T : struct => throw new PlatformNotSupportedException(); + } + internal readonly struct Vector128 + where T : struct + { + public static Vector128 Zero => throw new PlatformNotSupportedException(); + public static int Count => throw new PlatformNotSupportedException(); + } +} + +namespace System.Runtime.Intrinsics.X86 +{ + internal abstract class Bmi1 + { + public abstract class X64 + { + public const bool IsSupported = false; + public static ulong TrailingZeroCount(ulong value) => throw new PlatformNotSupportedException(); + } + public const bool IsSupported = false; + public static uint TrailingZeroCount(uint value) => throw new PlatformNotSupportedException(); + } + internal abstract class Lzcnt + { + public abstract class X64 + { + public const bool IsSupported = false; + public static ulong LeadingZeroCount(ulong value) => throw new PlatformNotSupportedException(); + } + public const bool IsSupported = false; + public static uint LeadingZeroCount(uint value) => throw new PlatformNotSupportedException(); + } + internal abstract class Popcnt + { + public abstract class X64 + { + public const bool IsSupported = false; + public static ulong PopCount(ulong value) => throw new PlatformNotSupportedException(); + } + public const bool IsSupported = false; + public static uint PopCount(uint value) => throw new PlatformNotSupportedException(); + } + + internal abstract class Sse2 + { + public abstract class X64 + { + public const bool IsSupported = false; + public static Vector128 ConvertScalarToVector128UInt64(ulong value) => throw new PlatformNotSupportedException(); + public static ulong ConvertToUInt64(Vector128 value) => throw new PlatformNotSupportedException(); + } + public const bool IsSupported = false; + public static Vector128 Add(Vector128 left, Vector128 right) => throw new PlatformNotSupportedException(); + public static Vector128 AddSaturate(Vector128 left, Vector128 right) => throw new PlatformNotSupportedException(); + public static Vector128 AndNot(Vector128 left, Vector128 right) => throw new PlatformNotSupportedException(); + public static Vector128 CompareGreaterThan(Vector128 left, Vector128 right) => throw new PlatformNotSupportedException(); + public static Vector128 CompareLessThan(Vector128 left, Vector128 right) => throw new PlatformNotSupportedException(); + public static Vector128 ConvertScalarToVector128UInt32(uint value) => throw new PlatformNotSupportedException(); + public static uint ConvertToUInt32(Vector128 value) => throw new PlatformNotSupportedException(); + public static unsafe Vector128 LoadAlignedVector128(byte* address) => throw new PlatformNotSupportedException(); + public static unsafe Vector128 LoadAlignedVector128(ushort* address) => throw new PlatformNotSupportedException(); + public static unsafe Vector128 LoadVector128(byte* address) => throw new PlatformNotSupportedException(); + public static unsafe Vector128 LoadVector128(short* address) => throw new PlatformNotSupportedException(); + public static unsafe Vector128 LoadVector128(ushort* address) => throw new PlatformNotSupportedException(); + public static int MoveMask(Vector128 value) => throw new PlatformNotSupportedException(); + public static Vector128 Or(Vector128 left, Vector128 right) => throw new PlatformNotSupportedException(); + public static Vector128 Or(Vector128 left, Vector128 right) => throw new PlatformNotSupportedException(); + public static Vector128 PackUnsignedSaturate(Vector128 left, Vector128 right) => throw new PlatformNotSupportedException(); + public static Vector128 ShiftRightLogical(Vector128 value, byte count) => throw new PlatformNotSupportedException(); + public static unsafe void Store(byte* address, Vector128 source) => throw new PlatformNotSupportedException(); + public static unsafe void StoreAligned(byte* address, Vector128 source) => throw new PlatformNotSupportedException(); + public static unsafe void StoreScalar(ulong* address, Vector128 source) => throw new PlatformNotSupportedException(); + public static Vector128 Subtract(Vector128 left, Vector128 right) => throw new PlatformNotSupportedException(); + public static Vector128 UnpackHigh(Vector128 left, Vector128 right) => throw new PlatformNotSupportedException(); + public static Vector128 UnpackLow(Vector128 left, Vector128 right) => throw new PlatformNotSupportedException(); + } + + internal abstract class Sse41 + { + public abstract class X64 + { + public const bool IsSupported = false; + } + public const bool IsSupported = false; + public static Vector128 Min(Vector128 left, Vector128 right) => throw new PlatformNotSupportedException(); + public static bool TestZ(Vector128 left, Vector128 right) => throw new PlatformNotSupportedException(); + public static bool TestZ(Vector128 left, Vector128 right) => throw new PlatformNotSupportedException(); + } +} diff --git a/src/libraries/System.Utf8String.Experimental/src/System/ThrowHelper.cs b/src/libraries/System.Utf8String.Experimental/src/System/ThrowHelper.cs new file mode 100644 index 0000000000000..4d3a949111697 --- /dev/null +++ b/src/libraries/System.Utf8String.Experimental/src/System/ThrowHelper.cs @@ -0,0 +1,95 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +namespace System +{ + internal static class ThrowHelper + { + [DoesNotReturn] + internal static void ThrowArgumentException(string resource, ExceptionArgument argument) + { + throw new ArgumentException(resource, argument.ToString()); + } + + [DoesNotReturn] + internal static void ThrowArgumentNullException(ExceptionArgument argument) { throw CreateArgumentNullException(argument); } + [MethodImpl(MethodImplOptions.NoInlining)] + private static Exception CreateArgumentNullException(ExceptionArgument argument) { return new ArgumentNullException(argument.ToString()); } + + [DoesNotReturn] + internal static void ThrowArgumentOutOfRangeException() { throw new ArgumentOutOfRangeException(); } + + internal static void ThrowArgumentOutOfRangeException(ExceptionArgument argument) { throw CreateArgumentOutOfRangeException(argument); } + [MethodImpl(MethodImplOptions.NoInlining)] + private static Exception CreateArgumentOutOfRangeException(ExceptionArgument argument) { return new ArgumentOutOfRangeException(argument.ToString()); } + + [DoesNotReturn] + internal static void ThrowValueArgumentOutOfRange_NeedNonNegNumException() + { + throw GetArgumentOutOfRangeException(ExceptionArgument.value, + SR.ArgumentOutOfRange_NeedNonNegNum); + } + + [DoesNotReturn] + internal static void ThrowLengthArgumentOutOfRange_ArgumentOutOfRange_NeedNonNegNum() + { + throw GetArgumentOutOfRangeException(ExceptionArgument.length, + SR.ArgumentOutOfRange_NeedNonNegNum); + } + + [DoesNotReturn] + internal static void ThrowInvalidOperationException() { throw CreateInvalidOperationException(); } + [MethodImpl(MethodImplOptions.NoInlining)] + private static Exception CreateInvalidOperationException() { return new InvalidOperationException(); } + + [DoesNotReturn] + internal static void ThrowArgumentException_DestinationTooShort() + { + throw new ArgumentException(SR.Argument_DestinationTooShort, "destination"); + } + + [DoesNotReturn] + internal static void ThrowArgumentException_CannotExtractScalar(ExceptionArgument argument) + { + throw new ArgumentException(SR.Argument_CannotExtractScalar, argument.ToString()); + } + + internal static void ThrowArgumentOutOfRange_IndexException() + { + throw GetArgumentOutOfRangeException(ExceptionArgument.index, + SR.ArgumentOutOfRange_Index); + } + + private static ArgumentOutOfRangeException GetArgumentOutOfRangeException(ExceptionArgument argument, string resource) + { + return new ArgumentOutOfRangeException(argument.ToString(), resource); + } + + [DoesNotReturn] + internal static void ThrowNotSupportedException(string message) + { + throw new NotSupportedException(message); + } + } + + // + // The convention for this enum is using the argument name as the enum name + // + internal enum ExceptionArgument + { + action, + ch, + comparisonType, + culture, + index, + input, + length, + start, + text, + value, + } +} diff --git a/src/libraries/System.Utf8String.Experimental/src/System/Utf8Extensions.Portable.cs b/src/libraries/System.Utf8String.Experimental/src/System/Utf8Extensions.Portable.cs new file mode 100644 index 0000000000000..84a03a37d84a5 --- /dev/null +++ b/src/libraries/System.Utf8String.Experimental/src/System/Utf8Extensions.Portable.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Runtime.CompilerServices; + +namespace System +{ + public static partial class Utf8Extensions + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ReadOnlySpan CreateSpan(Utf8String text) => text.GetSpan(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ReadOnlySpan CreateSpan(Utf8String text, int start) => + text.GetSpan().Slice(start); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ReadOnlySpan CreateSpan(Utf8String text, int start, int length) => + text.GetSpan().Slice(start, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ReadOnlyMemory CreateMemoryBytes(Utf8String text, int start, int length) => + text.CreateMemoryBytes(start, length); + } +} diff --git a/src/libraries/System.Utf8String.Experimental/src/System/Utf8String.Portable.cs b/src/libraries/System.Utf8String.Experimental/src/System/Utf8String.Portable.cs new file mode 100644 index 0000000000000..edbe70c2011a7 --- /dev/null +++ b/src/libraries/System.Utf8String.Experimental/src/System/Utf8String.Portable.cs @@ -0,0 +1,240 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.ComponentModel; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text.Unicode; + +namespace System +{ + public sealed partial class Utf8String + { + private readonly byte[] _bytes; + + /// + /// Returns the length (in UTF-8 code units, or s) of this instance. + /// + public int Length => _bytes.Length - 1; // -1 because the bytes are always null-terminated + + public Utf8String(ReadOnlySpan value) + { + _bytes = InitializeBuffer(value); + } + + public Utf8String(byte[] value, int startIndex, int length) + { + if (value is null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.value); + } + + _bytes = InitializeBuffer(new ReadOnlySpan(value, startIndex, length)); + } + + [CLSCompliant(false)] + public unsafe Utf8String(byte* value) + { + if (value is null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.value); + } + + _bytes = InitializeBuffer(new ReadOnlySpan(value, strlen(value))); + } + + public Utf8String(ReadOnlySpan value) + { + _bytes = InitializeBuffer(value); + } + + public Utf8String(char[] value, int startIndex, int length) + { + if (value is null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.value); + } + + _bytes = InitializeBuffer(new ReadOnlySpan(value, startIndex, length)); + } + + [CLSCompliant(false)] + public unsafe Utf8String(char* value) + { + if (value == null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.value); + } + + _bytes = InitializeBuffer(new ReadOnlySpan(value, wcslen(value))); + } + + public Utf8String(string value) + { + if (value is null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.value); + } + + _bytes = InitializeBuffer(value.AsSpan()); + } + + private static byte[] InitializeBuffer(ReadOnlySpan value) + { + if (value.IsEmpty) + { + return Empty._bytes; + } + + // Create and populate the Utf8String buffer. + + byte[] newBuffer = AllocateBuffer(value.Length); + value.CopyTo(newBuffer); + + // Now perform validation. + // Reminder: Perform validation over the copy, not over the source. + + if (!Utf8Utility.IsWellFormedUtf8(newBuffer)) + { + throw new ArgumentException( + message: SR.Utf8String_InputContainedMalformedUtf8, + paramName: nameof(value)); + } + + return newBuffer; + } + + private static byte[] InitializeBuffer(ReadOnlySpan value) + { + byte[]? newBuffer = CreateBufferFromUtf16Common(value, replaceInvalidSequences: false); + + if (newBuffer is null) + { + // Input buffer contained invalid UTF-16 data. + + throw new ArgumentException( + message: SR.Utf8String_InputContainedMalformedUtf16, + paramName: nameof(value)); + } + + return newBuffer; + } + + // This should only be called from FastAllocate + private Utf8String(byte[] bytes) + { + _bytes = bytes; + } + + /// + /// Returns a mutable reference to the first byte of this + /// (or the null terminator if the string is empty). + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ref byte DangerousGetMutableReference() => + ref MemoryMarshal.GetReference(_bytes.AsSpan()); + + /// + /// Returns a mutable that can be used to populate this + /// instance. Only to be used during construction. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Span DangerousGetMutableSpan() + { + Debug.Assert(Length > 0, $"This should only ever be called on a non-empty {nameof(Utf8String)}."); + return _bytes.AsSpan(0, Length); + } + + /// + /// Returns a for this + /// instance. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ReadOnlySpan GetSpan() => _bytes.AsSpan(0, Length); + + /// + /// Gets an immutable reference that can be used in a statement. The resulting + /// reference can be pinned and used as a null-terminated LPCUTF8STR. + /// + /// + /// If this instance is empty, returns a reference to the null terminator. + /// + [EditorBrowsable(EditorBrowsableState.Never)] // for compiler use only + public ref readonly byte GetPinnableReference() => ref _bytes.AsSpan().GetPinnableReference(); + + /// + /// Similar to , but skips the null check on the input. + /// Throws a if the input is null. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ReadOnlySpan AsBytesSkipNullCheck() => GetSpan(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ReadOnlyMemory CreateMemoryBytes(int start, int length) => + _bytes.AsMemory(start, length); + + /// + /// Creates a new zero-initialized instance of the specified length. Actual storage allocated is "length + 1" bytes + /// because instances are null-terminated. + /// + /// + /// The implementation of this method checks its input argument for overflow. + /// + private static Utf8String FastAllocate(int length) + { + // just simulate a "fast allocate", since this is portable + return new Utf8String(AllocateBuffer(length)); + } + + private static byte[] AllocateBuffer(int length) + { + Debug.Assert(length > 0); + + if (length == int.MaxValue) + { + // Ensure we don't overflow below. The VM will throw an OutOfMemoryException + // if we try to create a byte[] this large anyway. + length = int.MaxValue - 1; + } + + // Actual storage allocated is "length + 1" bytes because instances are null-terminated. + return new byte[length + 1]; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static unsafe int wcslen(char* ptr) + { + // IndexOf processes memory in aligned chunks, and thus it won't crash even if it accesses memory beyond the null terminator. + int length = new ReadOnlySpan(ptr, int.MaxValue).IndexOf('\0'); + if (length < 0) + { + ThrowMustBeNullTerminatedString(); + } + + return length; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe int strlen(byte* ptr) + { + // IndexOf processes memory in aligned chunks, and thus it won't crash even if it accesses memory beyond the null terminator. + int length = new ReadOnlySpan(ptr, int.MaxValue).IndexOf((byte)'\0'); + if (length < 0) + { + ThrowMustBeNullTerminatedString(); + } + + return length; + } + + [DoesNotReturn] + private static void ThrowMustBeNullTerminatedString() + { + throw new ArgumentException(SR.Arg_MustBeNullTerminatedString); + } + } +} diff --git a/src/libraries/System.Utf8String.Experimental/tests/System.Utf8String.Experimental.Tests.csproj b/src/libraries/System.Utf8String.Experimental/tests/System.Utf8String.Experimental.Tests.csproj index 669fe610acf22..e571bbc5877b6 100644 --- a/src/libraries/System.Utf8String.Experimental/tests/System.Utf8String.Experimental.Tests.csproj +++ b/src/libraries/System.Utf8String.Experimental/tests/System.Utf8String.Experimental.Tests.csproj @@ -1,8 +1,8 @@ - + true true - $(NetCoreAppCurrent) + $(NetCoreAppCurrent);$(NetFrameworkCurrent) true true @@ -38,4 +38,13 @@ + + + + + + + + + \ No newline at end of file diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/BoundedUtf8Span.cs b/src/libraries/System.Utf8String.Experimental/tests/System/BoundedUtf8Span.cs index d8a1c4641dbb4..8eda11d8b6b0d 100644 --- a/src/libraries/System.Utf8String.Experimental/tests/System/BoundedUtf8Span.cs +++ b/src/libraries/System.Utf8String.Experimental/tests/System/BoundedUtf8Span.cs @@ -17,6 +17,11 @@ public sealed class BoundedUtf8Span : IDisposable { private readonly BoundedMemory _boundedMemory; + public BoundedUtf8Span(string utf16Data, PoisonPagePlacement placement = PoisonPagePlacement.After) + : this(utf16Data.AsSpan(), placement) + { + } + public BoundedUtf8Span(ReadOnlySpan utf16Data, PoisonPagePlacement placement = PoisonPagePlacement.After) : this(u8(utf16Data.ToString()).AsBytes(), placement) { diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/MemoryTests.cs b/src/libraries/System.Utf8String.Experimental/tests/System/MemoryTests.cs index 8e7086a66f1b1..665bc978285c5 100644 --- a/src/libraries/System.Utf8String.Experimental/tests/System/MemoryTests.cs +++ b/src/libraries/System.Utf8String.Experimental/tests/System/MemoryTests.cs @@ -14,24 +14,6 @@ namespace System.Tests [SkipOnMono("The features from System.Utf8String.Experimental namespace are experimental.")] public partial class MemoryTests { - [Fact] - public static void MemoryMarshal_TryGetArrayOfByte_Utf8String() - { - ReadOnlyMemory rom = u8("Hello").AsMemoryBytes(); - - Assert.False(MemoryMarshal.TryGetArray(rom, out ArraySegment segment)); - Assert.True(default(ArraySegment).Equals(segment)); - } - - [Fact] - public static void MemoryMarshal_TryGetArrayOfChar8_Utf8String() - { - ReadOnlyMemory rom = u8("Hello").AsMemory(); - - Assert.False(MemoryMarshal.TryGetArray(rom, out ArraySegment segment)); - Assert.True(default(ArraySegment).Equals(segment)); - } - [Fact] public static unsafe void MemoryOfByte_WithUtf8String_Pin() { @@ -56,30 +38,6 @@ public static void MemoryOfByte_WithUtf8String_ToString() Assert.Equal("System.Memory[5]", Unsafe.As, Memory>(ref rom).ToString()); } - [Fact] - public static unsafe void MemoryOfChar8_WithUtf8String_Pin() - { - Utf8String theString = u8("Hello"); - ReadOnlyMemory rom = theString.AsMemory(); - MemoryHandle memHandle = default; - try - { - memHandle = Unsafe.As, Memory>(ref rom).Pin(); - Assert.True(memHandle.Pointer == Unsafe.AsPointer(ref Unsafe.AsRef(in theString.GetPinnableReference()))); - } - finally - { - memHandle.Dispose(); - } - } - - [Fact] - public static void MemoryOfChar8_WithUtf8String_ToString() - { - ReadOnlyMemory rom = u8("Hello").AsMemory(); - Assert.Equal("Hello", Unsafe.As, Memory>(ref rom).ToString()); - } - [Fact] public static unsafe void ReadOnlyMemoryOfByte_WithUtf8String_Pin() { @@ -103,29 +61,6 @@ public static void ReadOnlyMemoryOfByte_WithUtf8String_ToString() Assert.Equal("System.ReadOnlyMemory[5]", u8("Hello").AsMemoryBytes().ToString()); } - [Fact] - public static unsafe void ReadOnlyMemoryOfChar8_WithUtf8String_Pin() - { - Utf8String theString = u8("Hello"); - ReadOnlyMemory rom = theString.AsMemory(); - MemoryHandle memHandle = default; - try - { - memHandle = rom.Pin(); - Assert.True(memHandle.Pointer == Unsafe.AsPointer(ref Unsafe.AsRef(in theString.GetPinnableReference()))); - } - finally - { - memHandle.Dispose(); - } - } - - [Fact] - public static void ReadOnlyMemoryOfChar8_WithUtf8String_ToString() - { - Assert.Equal("Hello", u8("Hello").AsMemory().ToString()); - } - [Fact] public static void ReadOnlySpanOfByte_ToString() { @@ -133,25 +68,11 @@ public static void ReadOnlySpanOfByte_ToString() Assert.Equal("System.ReadOnlySpan[2]", span.ToString()); } - [Fact] - public static void ReadOnlySpanOfChar8_ToString() - { - ReadOnlySpan span = stackalloc Char8[] { (Char8)'H', (Char8)'i' }; - Assert.Equal("Hi", span.ToString()); - } - [Fact] public static void SpanOfByte_ToString() { Span span = stackalloc byte[] { (byte)'H', (byte)'i' }; Assert.Equal("System.Span[2]", span.ToString()); } - - [Fact] - public static void SpanOfChar8_ToString() - { - Span span = stackalloc Char8[] { (Char8)'H', (Char8)'i' }; - Assert.Equal("Hi", span.ToString()); - } } } diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/MemoryTests.netcoreapp.cs b/src/libraries/System.Utf8String.Experimental/tests/System/MemoryTests.netcoreapp.cs new file mode 100644 index 0000000000000..4a080cdc6b19e --- /dev/null +++ b/src/libraries/System.Utf8String.Experimental/tests/System/MemoryTests.netcoreapp.cs @@ -0,0 +1,95 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Xunit; + +using static System.Tests.Utf8TestUtilities; + +namespace System.Tests +{ + public partial class MemoryTests + { + [Fact] + public static void MemoryMarshal_TryGetArrayOfByte_Utf8String() + { + ReadOnlyMemory rom = u8("Hello").AsMemoryBytes(); + + Assert.False(MemoryMarshal.TryGetArray(rom, out ArraySegment segment)); + Assert.True(default(ArraySegment).Equals(segment)); + } + + [Fact] + public static void MemoryMarshal_TryGetArrayOfChar8_Utf8String() + { + ReadOnlyMemory rom = u8("Hello").AsMemory(); + + Assert.False(MemoryMarshal.TryGetArray(rom, out ArraySegment segment)); + Assert.True(default(ArraySegment).Equals(segment)); + } + + [Fact] + public static unsafe void MemoryOfChar8_WithUtf8String_Pin() + { + Utf8String theString = u8("Hello"); + ReadOnlyMemory rom = theString.AsMemory(); + MemoryHandle memHandle = default; + try + { + memHandle = Unsafe.As, Memory>(ref rom).Pin(); + Assert.True(memHandle.Pointer == Unsafe.AsPointer(ref Unsafe.AsRef(in theString.GetPinnableReference()))); + } + finally + { + memHandle.Dispose(); + } + } + + [Fact] + public static void MemoryOfChar8_WithUtf8String_ToString() + { + ReadOnlyMemory rom = u8("Hello").AsMemory(); + Assert.Equal("Hello", Unsafe.As, Memory>(ref rom).ToString()); + } + + [Fact] + public static unsafe void ReadOnlyMemoryOfChar8_WithUtf8String_Pin() + { + Utf8String theString = u8("Hello"); + ReadOnlyMemory rom = theString.AsMemory(); + MemoryHandle memHandle = default; + try + { + memHandle = rom.Pin(); + Assert.True(memHandle.Pointer == Unsafe.AsPointer(ref Unsafe.AsRef(in theString.GetPinnableReference()))); + } + finally + { + memHandle.Dispose(); + } + } + + [Fact] + public static void ReadOnlyMemoryOfChar8_WithUtf8String_ToString() + { + Assert.Equal("Hello", u8("Hello").AsMemory().ToString()); + } + + [Fact] + public static void ReadOnlySpanOfChar8_ToString() + { + ReadOnlySpan span = stackalloc Char8[] { (Char8)'H', (Char8)'i' }; + Assert.Equal("Hi", span.ToString()); + } + + [Fact] + public static void SpanOfChar8_ToString() + { + Span span = stackalloc Char8[] { (Char8)'H', (Char8)'i' }; + Assert.Equal("Hi", span.ToString()); + } + } +} diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/MemoryTests.netfx.cs b/src/libraries/System.Utf8String.Experimental/tests/System/MemoryTests.netfx.cs new file mode 100644 index 0000000000000..8238c0a08be56 --- /dev/null +++ b/src/libraries/System.Utf8String.Experimental/tests/System/MemoryTests.netfx.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Runtime.InteropServices; +using Xunit; + +using static System.Tests.Utf8TestUtilities; + +namespace System.Tests +{ + public partial class MemoryTests + { + [Fact] + public static void MemoryMarshal_TryGetArrayOfByte_Utf8String() + { + ReadOnlyMemory rom = u8("Hello").AsMemoryBytes(); + + Assert.True(MemoryMarshal.TryGetArray(rom, out ArraySegment segment)); + Assert.NotNull(segment.Array); + Assert.Equal(0, segment.Offset); + Assert.Equal(5, segment.Count); + } + + [Fact] + public static void ReadOnlySpanOfChar8_ToString() + { + // unable to override ReadOnlySpan.ToString on netfx + + ReadOnlySpan span = stackalloc Char8[] { (Char8)'H', (Char8)'i' }; + Assert.Equal("System.ReadOnlySpan[2]", span.ToString()); + } + + [Fact] + public static void SpanOfChar8_ToString() + { + // unable to override Span.ToString on netfx + + Span span = stackalloc Char8[] { (Char8)'H', (Char8)'i' }; + Assert.Equal("System.Span[2]", span.ToString()); + } + } +} diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/RangeEqualityComparer.cs b/src/libraries/System.Utf8String.Experimental/tests/System/RangeEqualityComparer.cs index 8cc393896345b..7cd1ba3c726cd 100644 --- a/src/libraries/System.Utf8String.Experimental/tests/System/RangeEqualityComparer.cs +++ b/src/libraries/System.Utf8String.Experimental/tests/System/RangeEqualityComparer.cs @@ -32,7 +32,11 @@ public bool Equals(Range x, Range y) public int GetHashCode(Range obj) { (int offset, int length) = obj.GetOffsetAndLength(_length); +#if NETCOREAPP return HashCode.Combine(offset, length); +#else + return Tuple.Create(offset, length).GetHashCode(); +#endif } } } diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/ReflectionTests.cs b/src/libraries/System.Utf8String.Experimental/tests/System/ReflectionTests.cs index e45e8ddca07cc..5f5d9d71fab1a 100644 --- a/src/libraries/System.Utf8String.Experimental/tests/System/ReflectionTests.cs +++ b/src/libraries/System.Utf8String.Experimental/tests/System/ReflectionTests.cs @@ -24,14 +24,5 @@ public static void ActivatorCreateInstance_CannotCallParameterlessCtor() { Assert.Throws(() => Activator.CreateInstance(typeof(Utf8String))); } - - [Fact] - public static void FormatterServices_GetUninitializedObject_Throws() - { - // Like String, shouldn't be able to create an uninitialized Utf8String. - - Assert.Throws(() => FormatterServices.GetSafeUninitializedObject(typeof(Utf8String))); - Assert.Throws(() => FormatterServices.GetUninitializedObject(typeof(Utf8String))); - } } } diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/ReflectionTests.netcoreapp.cs b/src/libraries/System.Utf8String.Experimental/tests/System/ReflectionTests.netcoreapp.cs new file mode 100644 index 0000000000000..4dc36558d6c1e --- /dev/null +++ b/src/libraries/System.Utf8String.Experimental/tests/System/ReflectionTests.netcoreapp.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Runtime.Serialization; +using Xunit; + +namespace System.Tests +{ + public partial class ReflectionTests + { + [Fact] + public static void FormatterServices_GetUninitializedObject_Throws() + { + // Like String, shouldn't be able to create an uninitialized Utf8String. + + Assert.Throws(() => FormatterServices.GetSafeUninitializedObject(typeof(Utf8String))); + Assert.Throws(() => FormatterServices.GetUninitializedObject(typeof(Utf8String))); + } + } +} diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8ExtensionsTests.cs b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8ExtensionsTests.cs index 5b15679b006be..39b7a91a6cf81 100644 --- a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8ExtensionsTests.cs +++ b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8ExtensionsTests.cs @@ -24,20 +24,12 @@ public unsafe void AsBytes_FromSpan_Default() // Next, an empty but non-default span should become an empty but non-default span. Assert.True(new ReadOnlySpan((void*)0x12345, 0) == new ReadOnlySpan((void*)0x12345, 0).AsBytes()); - - // Finally, a span wrapping data should become a span wrapping that same data. - - Utf8String theString = u8("Hello"); - Assert.True(MemoryMarshal.CreateReadOnlySpan(ref Unsafe.AsRef(in theString.GetPinnableReference()), 5) == (theString.AsMemory().Span).AsBytes()); } [Fact] public void AsBytes_FromUtf8String() { Assert.True(default(ReadOnlySpan) == ((Utf8String)null).AsBytes()); - - Utf8String theString = u8("Hello"); - Assert.True(MemoryMarshal.CreateReadOnlySpan(ref Unsafe.AsRef(in theString.GetPinnableReference()), 5) == theString.AsBytes()); } [Fact] @@ -76,56 +68,18 @@ public void AsBytes_FromUtf8String_WithStartAndLength_ArgOutOfRange() } [Fact] - public void AsMemory_FromUtf8String() - { - Assert.True(default(ReadOnlyMemory).Equals(((Utf8String)null).AsMemory())); - - Utf8String theString = u8("Hello"); - Assert.True(MemoryMarshal.CreateReadOnlySpan(ref Unsafe.As(ref Unsafe.AsRef(in theString.GetPinnableReference())), 5) == theString.AsMemory().Span); - } - - [Fact] - public void AsMemory_FromUtf8String_WithStart() - { - Assert.True(default(ReadOnlyMemory).Equals(((Utf8String)null).AsMemory(0))); - Assert.True(u8("Hello").AsMemory(5).IsEmpty); - - SpanAssert.Equal(new Char8[] { (Char8)'e', (Char8)'l', (Char8)'l', (Char8)'o' }, u8("Hello").AsMemory(1).Span); - } - - [Fact] - public void AsMemory_FromUtf8String_WithStart_ArgOutOfRange() - { - Assert.Throws("start", () => ((Utf8String)null).AsMemory(1)); - Assert.Throws("start", () => u8("Hello").AsMemory(-1)); - Assert.Throws("start", () => u8("Hello").AsMemory(6)); - } - - [Fact] - public void AsMemory_FromUtf8String_WithStartAndLength() - { - Assert.True(default(ReadOnlyMemory).Equals(((Utf8String)null).AsMemory(0, 0))); - Assert.True(u8("Hello").AsMemory(5, 0).IsEmpty); - - SpanAssert.Equal(new Char8[] { (Char8)'e', (Char8)'l', (Char8)'l' }, u8("Hello").AsMemory(1, 3).Span); - } - - [Fact] - public void AsMemory_FromUtf8String_WithStartAndLength_ArgOutOfRange() - { - Assert.Throws("start", () => ((Utf8String)null).AsMemory(0, 1)); - Assert.Throws("start", () => ((Utf8String)null).AsMemory(1, 0)); - Assert.Throws("start", () => u8("Hello").AsMemory(5, 1)); - Assert.Throws("start", () => u8("Hello").AsMemory(4, -2)); - } - - [Fact] - public void AsMemoryBytes_FromUtf8String() + public unsafe void AsMemoryBytes_FromUtf8String() { Assert.True(default(ReadOnlyMemory).Equals(((Utf8String)null).AsMemoryBytes())); Utf8String theString = u8("Hello"); - Assert.True(MemoryMarshal.CreateReadOnlySpan(ref Unsafe.AsRef(in theString.GetPinnableReference()), 5) == theString.AsMemoryBytes().Span); + fixed (byte* pTheString = theString) + { + fixed (byte* pTheStringAsMemoryBytes = theString.AsMemoryBytes().Span) + { + Assert.True(pTheString == pTheStringAsMemoryBytes); + } + } } [Fact] diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8ExtensionsTests.netcoreapp.cs b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8ExtensionsTests.netcoreapp.cs new file mode 100644 index 0000000000000..9833e922e391b --- /dev/null +++ b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8ExtensionsTests.netcoreapp.cs @@ -0,0 +1,77 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Xunit; + +using static System.Tests.Utf8TestUtilities; + +namespace System.Tests +{ + public partial class Utf8ExtensionsTests + { + [Fact] + public void AsBytes_FromSpan_Default_netcoreapp() + { + // a span wrapping data should become a span wrapping that same data. + + Utf8String theString = u8("Hello"); + + Assert.True(MemoryMarshal.CreateReadOnlySpan(ref Unsafe.AsRef(in theString.GetPinnableReference()), 5) == (theString.AsMemory().Span).AsBytes()); + } + + [Fact] + public void AsBytes_FromUtf8String_netcoreapp() + { + Utf8String theString = u8("Hello"); + Assert.True(MemoryMarshal.CreateReadOnlySpan(ref Unsafe.AsRef(in theString.GetPinnableReference()), 5) == theString.AsBytes()); + } + + [Fact] + public void AsMemory_FromUtf8String() + { + Assert.True(default(ReadOnlyMemory).Equals(((Utf8String)null).AsMemory())); + + Utf8String theString = u8("Hello"); + Assert.True(MemoryMarshal.CreateReadOnlySpan(ref Unsafe.As(ref Unsafe.AsRef(in theString.GetPinnableReference())), 5) == theString.AsMemory().Span); + } + + [Fact] + public void AsMemory_FromUtf8String_WithStart() + { + Assert.True(default(ReadOnlyMemory).Equals(((Utf8String)null).AsMemory(0))); + Assert.True(u8("Hello").AsMemory(5).IsEmpty); + + SpanAssert.Equal(new Char8[] { (Char8)'e', (Char8)'l', (Char8)'l', (Char8)'o' }, u8("Hello").AsMemory(1).Span); + } + + [Fact] + public void AsMemory_FromUtf8String_WithStart_ArgOutOfRange() + { + Assert.Throws("start", () => ((Utf8String)null).AsMemory(1)); + Assert.Throws("start", () => u8("Hello").AsMemory(-1)); + Assert.Throws("start", () => u8("Hello").AsMemory(6)); + } + + [Fact] + public void AsMemory_FromUtf8String_WithStartAndLength() + { + Assert.True(default(ReadOnlyMemory).Equals(((Utf8String)null).AsMemory(0, 0))); + Assert.True(u8("Hello").AsMemory(5, 0).IsEmpty); + + SpanAssert.Equal(new Char8[] { (Char8)'e', (Char8)'l', (Char8)'l' }, u8("Hello").AsMemory(1, 3).Span); + } + + [Fact] + public void AsMemory_FromUtf8String_WithStartAndLength_ArgOutOfRange() + { + Assert.Throws("start", () => ((Utf8String)null).AsMemory(0, 1)); + Assert.Throws("start", () => ((Utf8String)null).AsMemory(1, 0)); + Assert.Throws("start", () => u8("Hello").AsMemory(5, 1)); + Assert.Throws("start", () => u8("Hello").AsMemory(4, -2)); + } + } +} diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8SpanTests.Conversion.cs b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8SpanTests.Conversion.cs index bd18fdd177fbf..4057a855e9edd 100644 --- a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8SpanTests.Conversion.cs +++ b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8SpanTests.Conversion.cs @@ -42,7 +42,7 @@ public static void Normalize(string utf16Source, string utf16Expected, Normaliza { byte[] dest = new byte[bufferLength]; Assert.Equal(utf8Normalized.Length, utf8Source.Normalize(dest, normalizationForm)); - Utf8Span normalizedSpan = Utf8Span.UnsafeCreateWithoutValidation(dest[..utf8Normalized.Length]); + Utf8Span normalizedSpan = Utf8Span.UnsafeCreateWithoutValidation(dest.AsSpan().Slice(0, utf8Normalized.Length)); Assert.True(utf8Normalized.AsSpan() == normalizedSpan); // ordinal equality Assert.True(normalizedSpan.IsNormalized(normalizationForm)); } @@ -159,7 +159,7 @@ static void RunTest(string testData, string expected, CultureInfo culture) foreach (int bufferSize in new[] { expectedUtf8.Length, expectedUtf8.Length + 1 }) { - byte[] buffer = new byte[expectedUtf8.Length]; + byte[] buffer = new byte[bufferSize]; if (culture is null) { @@ -170,7 +170,7 @@ static void RunTest(string testData, string expected, CultureInfo culture) Assert.Equal(expectedUtf8.Length, inputSpan.ToUpper(buffer, culture)); } - Assert.True(expectedUtf8.AsBytes().SequenceEqual(buffer)); + Assert.True(expectedUtf8.AsBytes().SequenceEqual(buffer.AsSpan(0, expectedUtf8.Length))); } } diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8SpanTests.Searching.cs b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8SpanTests.Searching.cs index a46da8a6c29db..36b1842f48a7e 100644 --- a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8SpanTests.Searching.cs +++ b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8SpanTests.Searching.cs @@ -96,57 +96,71 @@ public static void TryFind_Char_WithComparison(ustring source, char searchTerm, CultureInfo.CurrentCulture = currentCulture; } - // First, search forward - - bool wasFound = searchSpan.TryFind(searchTerm, comparison, out Range actualForwardMatch); - Assert.Equal(expectedForwardMatch.HasValue, wasFound); - - if (wasFound) - { - AssertRangesEqual(searchSpan.Length, expectedForwardMatch.Value, actualForwardMatch); - } - - // Also check Contains / StartsWith / SplitOn - - Assert.Equal(wasFound, searchSpan.Contains(searchTerm, comparison)); - Assert.Equal(wasFound && searchSpan.Bytes[..actualForwardMatch.Start].IsEmpty, searchSpan.StartsWith(searchTerm, comparison)); - - (var before, var after) = searchSpan.SplitOn(searchTerm, comparison); - if (wasFound) - { - Assert.True(searchSpan.Bytes[..actualForwardMatch.Start] == before.Bytes); // check for referential equality - Assert.True(searchSpan.Bytes[actualForwardMatch.End..] == after.Bytes); // check for referential equality - } - else - { - Assert.True(searchSpan.Bytes == before.Bytes); // check for reference equality - Assert.True(after.IsNull()); - } - - // Now search backward - - wasFound = searchSpan.TryFindLast(searchTerm, comparison, out Range actualBackwardMatch); - Assert.Equal(expectedBackwardMatch.HasValue, wasFound); - - if (wasFound) + if (IsTryFindSupported(comparison)) { - AssertRangesEqual(searchSpan.Length, expectedBackwardMatch.Value, actualBackwardMatch); - } - - // Also check EndsWith / SplitOnLast - - Assert.Equal(wasFound && searchSpan.Bytes[actualBackwardMatch.End..].IsEmpty, searchSpan.EndsWith(searchTerm, comparison)); - - (before, after) = searchSpan.SplitOnLast(searchTerm, comparison); - if (wasFound) - { - Assert.True(searchSpan.Bytes[..actualBackwardMatch.Start] == before.Bytes); // check for referential equality - Assert.True(searchSpan.Bytes[actualBackwardMatch.End..] == after.Bytes); // check for referential equality + // First, search forward + + bool wasFound = searchSpan.TryFind(searchTerm, comparison, out Range actualForwardMatch); + Assert.Equal(expectedForwardMatch.HasValue, wasFound); + + if (wasFound) + { + AssertRangesEqual(searchSpan.Length, expectedForwardMatch.Value, actualForwardMatch); + } + + // Also check Contains / StartsWith / SplitOn + + Assert.Equal(wasFound, searchSpan.Contains(searchTerm, comparison)); + Assert.Equal(wasFound && searchSpan.Bytes[..actualForwardMatch.Start].IsEmpty, searchSpan.StartsWith(searchTerm, comparison)); + + (var before, var after) = searchSpan.SplitOn(searchTerm, comparison); + if (wasFound) + { + Assert.True(searchSpan.Bytes[..actualForwardMatch.Start] == before.Bytes); // check for referential equality + Assert.True(searchSpan.Bytes[actualForwardMatch.End..] == after.Bytes); // check for referential equality + } + else + { + Assert.True(searchSpan.Bytes == before.Bytes); // check for reference equality + Assert.True(after.IsNull()); + } + + // Now search backward + + wasFound = searchSpan.TryFindLast(searchTerm, comparison, out Range actualBackwardMatch); + Assert.Equal(expectedBackwardMatch.HasValue, wasFound); + + if (wasFound) + { + AssertRangesEqual(searchSpan.Length, expectedBackwardMatch.Value, actualBackwardMatch); + } + + // Also check EndsWith / SplitOnLast + + Assert.Equal(wasFound && searchSpan.Bytes[actualBackwardMatch.End..].IsEmpty, searchSpan.EndsWith(searchTerm, comparison)); + + (before, after) = searchSpan.SplitOnLast(searchTerm, comparison); + if (wasFound) + { + Assert.True(searchSpan.Bytes[..actualBackwardMatch.Start] == before.Bytes); // check for referential equality + Assert.True(searchSpan.Bytes[actualBackwardMatch.End..] == after.Bytes); // check for referential equality + } + else + { + Assert.True(searchSpan.Bytes == before.Bytes); // check for reference equality + Assert.True(after.IsNull()); + } } else { - Assert.True(searchSpan.Bytes == before.Bytes); // check for reference equality - Assert.True(after.IsNull()); + Assert.Throws(() => boundedSpan.Span.TryFind(searchTerm, comparison, out var _)); + Assert.Throws(() => boundedSpan.Span.TryFindLast(searchTerm, comparison, out var _)); + Assert.Throws(() => boundedSpan.Span.SplitOn(searchTerm, comparison)); + Assert.Throws(() => boundedSpan.Span.SplitOnLast(searchTerm, comparison)); + + Assert.Equal(expectedForwardMatch.HasValue, searchSpan.Contains(searchTerm, comparison)); + Assert.Equal(expectedForwardMatch.HasValue && searchSpan.Bytes[..expectedForwardMatch.Value.Start].IsEmpty, searchSpan.StartsWith(searchTerm, comparison)); + Assert.Equal(expectedBackwardMatch.HasValue && searchSpan.Bytes[expectedBackwardMatch.Value.End..].IsEmpty, searchSpan.EndsWith(searchTerm, comparison)); } }); } @@ -229,57 +243,71 @@ public static void TryFind_Rune_WithComparison(ustring source, Rune searchTerm, CultureInfo.CurrentCulture = currentCulture; } - // First, search forward - - bool wasFound = searchSpan.TryFind(searchTerm, comparison, out Range actualForwardMatch); - Assert.Equal(expectedForwardMatch.HasValue, wasFound); - - if (wasFound) - { - AssertRangesEqual(searchSpan.Length, expectedForwardMatch.Value, actualForwardMatch); - } - - // Also check Contains / StartsWith / SplitOn - - Assert.Equal(wasFound, searchSpan.Contains(searchTerm, comparison)); - Assert.Equal(wasFound && searchSpan.Bytes[..actualForwardMatch.Start].IsEmpty, searchSpan.StartsWith(searchTerm, comparison)); - - (var before, var after) = searchSpan.SplitOn(searchTerm, comparison); - if (wasFound) - { - Assert.True(searchSpan.Bytes[..actualForwardMatch.Start] == before.Bytes); // check for referential equality - Assert.True(searchSpan.Bytes[actualForwardMatch.End..] == after.Bytes); // check for referential equality - } - else - { - Assert.True(searchSpan.Bytes == before.Bytes); // check for reference equality - Assert.True(after.IsNull()); - } - - // Now search backward - - wasFound = searchSpan.TryFindLast(searchTerm, comparison, out Range actualBackwardMatch); - Assert.Equal(expectedBackwardMatch.HasValue, wasFound); - - if (wasFound) + if (IsTryFindSupported(comparison)) { - AssertRangesEqual(searchSpan.Length, expectedBackwardMatch.Value, actualBackwardMatch); - } - - // Also check EndsWith / SplitOnLast - - Assert.Equal(wasFound && searchSpan.Bytes[actualBackwardMatch.End..].IsEmpty, searchSpan.EndsWith(searchTerm, comparison)); - - (before, after) = searchSpan.SplitOnLast(searchTerm, comparison); - if (wasFound) - { - Assert.True(searchSpan.Bytes[..actualBackwardMatch.Start] == before.Bytes); // check for referential equality - Assert.True(searchSpan.Bytes[actualBackwardMatch.End..] == after.Bytes); // check for referential equality + // First, search forward + + bool wasFound = searchSpan.TryFind(searchTerm, comparison, out Range actualForwardMatch); + Assert.Equal(expectedForwardMatch.HasValue, wasFound); + + if (wasFound) + { + AssertRangesEqual(searchSpan.Length, expectedForwardMatch.Value, actualForwardMatch); + } + + // Also check Contains / StartsWith / SplitOn + + Assert.Equal(wasFound, searchSpan.Contains(searchTerm, comparison)); + Assert.Equal(wasFound && searchSpan.Bytes[..actualForwardMatch.Start].IsEmpty, searchSpan.StartsWith(searchTerm, comparison)); + + (var before, var after) = searchSpan.SplitOn(searchTerm, comparison); + if (wasFound) + { + Assert.True(searchSpan.Bytes[..actualForwardMatch.Start] == before.Bytes); // check for referential equality + Assert.True(searchSpan.Bytes[actualForwardMatch.End..] == after.Bytes); // check for referential equality + } + else + { + Assert.True(searchSpan.Bytes == before.Bytes); // check for reference equality + Assert.True(after.IsNull()); + } + + // Now search backward + + wasFound = searchSpan.TryFindLast(searchTerm, comparison, out Range actualBackwardMatch); + Assert.Equal(expectedBackwardMatch.HasValue, wasFound); + + if (wasFound) + { + AssertRangesEqual(searchSpan.Length, expectedBackwardMatch.Value, actualBackwardMatch); + } + + // Also check EndsWith / SplitOnLast + + Assert.Equal(wasFound && searchSpan.Bytes[actualBackwardMatch.End..].IsEmpty, searchSpan.EndsWith(searchTerm, comparison)); + + (before, after) = searchSpan.SplitOnLast(searchTerm, comparison); + if (wasFound) + { + Assert.True(searchSpan.Bytes[..actualBackwardMatch.Start] == before.Bytes); // check for referential equality + Assert.True(searchSpan.Bytes[actualBackwardMatch.End..] == after.Bytes); // check for referential equality + } + else + { + Assert.True(searchSpan.Bytes == before.Bytes); // check for reference equality + Assert.True(after.IsNull()); + } } else { - Assert.True(searchSpan.Bytes == before.Bytes); // check for reference equality - Assert.True(after.IsNull()); + Assert.Throws(() => boundedSpan.Span.TryFind(searchTerm, comparison, out var _)); + Assert.Throws(() => boundedSpan.Span.TryFindLast(searchTerm, comparison, out var _)); + Assert.Throws(() => boundedSpan.Span.SplitOn(searchTerm, comparison)); + Assert.Throws(() => boundedSpan.Span.SplitOnLast(searchTerm, comparison)); + + Assert.Equal(expectedForwardMatch.HasValue, searchSpan.Contains(searchTerm, comparison)); + Assert.Equal(expectedForwardMatch.HasValue && searchSpan.Bytes[..expectedForwardMatch.Value.Start].IsEmpty, searchSpan.StartsWith(searchTerm, comparison)); + Assert.Equal(expectedBackwardMatch.HasValue && searchSpan.Bytes[expectedBackwardMatch.Value.End..].IsEmpty, searchSpan.EndsWith(searchTerm, comparison)); } }); } @@ -362,57 +390,71 @@ public static void TryFind_Utf8Span_WithComparison(ustring source, ustring searc CultureInfo.CurrentCulture = currentCulture; } - // First, search forward - - bool wasFound = searchSpan.TryFind(searchTerm, comparison, out Range actualForwardMatch); - Assert.Equal(expectedForwardMatch.HasValue, wasFound); - - if (wasFound) - { - AssertRangesEqual(searchSpan.Length, expectedForwardMatch.Value, actualForwardMatch); - } - - // Also check Contains / StartsWith / SplitOn - - Assert.Equal(wasFound, searchSpan.Contains(searchTerm, comparison)); - Assert.Equal(wasFound && searchSpan.Bytes[..actualForwardMatch.Start].IsEmpty, searchSpan.StartsWith(searchTerm, comparison)); - - (var before, var after) = searchSpan.SplitOn(searchTerm, comparison); - if (wasFound) - { - Assert.True(searchSpan.Bytes[..actualForwardMatch.Start] == before.Bytes); // check for referential equality - Assert.True(searchSpan.Bytes[actualForwardMatch.End..] == after.Bytes); // check for referential equality - } - else - { - Assert.True(searchSpan.Bytes == before.Bytes); // check for reference equality - Assert.True(after.IsNull()); - } - - // Now search backward - - wasFound = searchSpan.TryFindLast(searchTerm, comparison, out Range actualBackwardMatch); - Assert.Equal(expectedBackwardMatch.HasValue, wasFound); - - if (wasFound) - { - AssertRangesEqual(searchSpan.Length, expectedBackwardMatch.Value, actualBackwardMatch); - } - - // Also check EndsWith / SplitOnLast - - Assert.Equal(wasFound && searchSpan.Bytes[actualBackwardMatch.End..].IsEmpty, searchSpan.EndsWith(searchTerm, comparison)); - - (before, after) = searchSpan.SplitOnLast(searchTerm, comparison); - if (wasFound) + if (IsTryFindSupported(comparison)) { - Assert.True(searchSpan.Bytes[..actualBackwardMatch.Start] == before.Bytes); // check for referential equality - Assert.True(searchSpan.Bytes[actualBackwardMatch.End..] == after.Bytes); // check for referential equality + // First, search forward + + bool wasFound = searchSpan.TryFind(searchTerm, comparison, out Range actualForwardMatch); + Assert.Equal(expectedForwardMatch.HasValue, wasFound); + + if (wasFound) + { + AssertRangesEqual(searchSpan.Length, expectedForwardMatch.Value, actualForwardMatch); + } + + // Also check Contains / StartsWith / SplitOn + + Assert.Equal(wasFound, searchSpan.Contains(searchTerm, comparison)); + Assert.Equal(wasFound && searchSpan.Bytes[..actualForwardMatch.Start].IsEmpty, searchSpan.StartsWith(searchTerm, comparison)); + + (var before, var after) = searchSpan.SplitOn(searchTerm, comparison); + if (wasFound) + { + Assert.True(searchSpan.Bytes[..actualForwardMatch.Start] == before.Bytes); // check for referential equality + Assert.True(searchSpan.Bytes[actualForwardMatch.End..] == after.Bytes); // check for referential equality + } + else + { + Assert.True(searchSpan.Bytes == before.Bytes); // check for reference equality + Assert.True(after.IsNull()); + } + + // Now search backward + + wasFound = searchSpan.TryFindLast(searchTerm, comparison, out Range actualBackwardMatch); + Assert.Equal(expectedBackwardMatch.HasValue, wasFound); + + if (wasFound) + { + AssertRangesEqual(searchSpan.Length, expectedBackwardMatch.Value, actualBackwardMatch); + } + + // Also check EndsWith / SplitOnLast + + Assert.Equal(wasFound && searchSpan.Bytes[actualBackwardMatch.End..].IsEmpty, searchSpan.EndsWith(searchTerm, comparison)); + + (before, after) = searchSpan.SplitOnLast(searchTerm, comparison); + if (wasFound) + { + Assert.True(searchSpan.Bytes[..actualBackwardMatch.Start] == before.Bytes); // check for referential equality + Assert.True(searchSpan.Bytes[actualBackwardMatch.End..] == after.Bytes); // check for referential equality + } + else + { + Assert.True(searchSpan.Bytes == before.Bytes); // check for reference equality + Assert.True(after.IsNull()); + } } else { - Assert.True(searchSpan.Bytes == before.Bytes); // check for reference equality - Assert.True(after.IsNull()); + Assert.Throws(() => boundedSpan.Span.TryFind(searchTerm, comparison, out var _)); + Assert.Throws(() => boundedSpan.Span.TryFindLast(searchTerm, comparison, out var _)); + Assert.Throws(() => boundedSpan.Span.SplitOn(searchTerm, comparison)); + Assert.Throws(() => boundedSpan.Span.SplitOnLast(searchTerm, comparison)); + + Assert.Equal(expectedForwardMatch.HasValue, searchSpan.Contains(searchTerm, comparison)); + Assert.Equal(expectedForwardMatch.HasValue && searchSpan.Bytes[..expectedForwardMatch.Value.Start].IsEmpty, searchSpan.StartsWith(searchTerm, comparison)); + Assert.Equal(expectedBackwardMatch.HasValue && searchSpan.Bytes[expectedBackwardMatch.Value.End..].IsEmpty, searchSpan.EndsWith(searchTerm, comparison)); } }); } diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8SpanTests.TestData.cs b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8SpanTests.TestData.cs index e5ee9b31e8c18..10c13247ca12d 100644 --- a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8SpanTests.TestData.cs +++ b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8SpanTests.TestData.cs @@ -125,7 +125,7 @@ private static bool TryParseSearchTermAsRune(object searchTerm, out Rune parsed) } else if (searchTerm is string str) { - if (Rune.DecodeFromUtf16(str, out parsed, out int charsConsumed) == OperationStatus.Done + if (Rune.DecodeFromUtf16(str.AsSpan(), out parsed, out int charsConsumed) == OperationStatus.Done && charsConsumed == str.Length) { return true; @@ -161,7 +161,7 @@ private static bool TryParseSearchTermAsUtf8String(object searchTerm, out ustrin } else if (searchTerm is string str) { - if (ustring.TryCreateFrom(str, out parsed)) + if (ustring.TryCreateFrom(str.AsSpan(), out parsed)) { return true; } diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8SpanTests.cs b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8SpanTests.cs index 93148cb652951..ab1b528d49674 100644 --- a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8SpanTests.cs +++ b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8SpanTests.cs @@ -287,7 +287,7 @@ public static void ToCharsTest(string expected) { using BoundedMemory boundedMemory = BoundedMemory.Allocate(i); Assert.Equal(expected.Length, span.ToChars(boundedMemory.Span)); - Assert.True(boundedMemory.Span.Slice(0, expected.Length).SequenceEqual(expected)); + Assert.True(boundedMemory.Span.Slice(0, expected.Length).SequenceEqual(expected.AsSpan())); } } diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8StringTests.Ctor.cs b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8StringTests.Ctor.cs index 4b5e278faa5d6..6776d5abf4e88 100644 --- a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8StringTests.Ctor.cs +++ b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8StringTests.Ctor.cs @@ -17,7 +17,7 @@ public unsafe partial class Utf8StringTests public static void Ctor_ByteArrayOffset_Empty_ReturnsEmpty() { byte[] inputData = new byte[] { (byte)'H', (byte)'e', (byte)'l', (byte)'l', (byte)'o' }; - Assert.Same(Utf8String.Empty, new Utf8String(inputData, 3, 0)); + AssertSameAsEmpty(new Utf8String(inputData, 3, 0)); } [Fact] @@ -67,7 +67,7 @@ public static void Ctor_BytePointer_Empty_ReturnsEmpty() using (BoundedMemory boundedMemory = BoundedMemory.AllocateFromExistingData(inputData)) { - Assert.Same(Utf8String.Empty, new Utf8String((byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(boundedMemory.Span)))); + AssertSameAsEmpty(new Utf8String((byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(boundedMemory.Span)))); } } @@ -96,7 +96,7 @@ public static void Ctor_BytePointer_InvalidData_Throws() [Fact] public static void Ctor_ByteSpan_Empty_ReturnsEmpty() { - Assert.Same(Utf8String.Empty, new Utf8String(ReadOnlySpan.Empty)); + AssertSameAsEmpty(new Utf8String(ReadOnlySpan.Empty)); } [Fact] @@ -121,7 +121,7 @@ public static void Ctor_ByteSpan_InvalidData_Throws() public static void Ctor_CharArrayOffset_Empty_ReturnsEmpty() { char[] inputData = "H\U00012345ello".ToCharArray(); // ok to have an empty slice in the middle of a multi-byte subsequence - Assert.Same(Utf8String.Empty, new Utf8String(inputData, 3, 0)); + AssertSameAsEmpty(new Utf8String(inputData, 3, 0)); } [Fact] @@ -171,7 +171,7 @@ public static void Ctor_CharPointer_Empty_ReturnsEmpty() using (BoundedMemory boundedMemory = BoundedMemory.AllocateFromExistingData(inputData)) { - Assert.Same(Utf8String.Empty, new Utf8String((char*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(boundedMemory.Span)))); + AssertSameAsEmpty(new Utf8String((char*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(boundedMemory.Span)))); } } @@ -200,7 +200,7 @@ public static void Ctor_CharPointer_InvalidData_Throws() [Fact] public static void Ctor_CharSpan_Empty_ReturnsEmpty() { - Assert.Same(Utf8String.Empty, new Utf8String(ReadOnlySpan.Empty)); + AssertSameAsEmpty(new Utf8String(ReadOnlySpan.Empty)); } [Fact] @@ -231,7 +231,7 @@ public static void Ctor_String_Null_Throws() [Fact] public static void Ctor_String_Empty_ReturnsEmpty() { - Assert.Same(Utf8String.Empty, new Utf8String(string.Empty)); + AssertSameAsEmpty(new Utf8String(string.Empty)); } [Fact] @@ -240,6 +240,13 @@ public static void Ctor_String_ValidData_ReturnsOriginalContents() Assert.Equal(u8("Hello"), new Utf8String("Hello")); } + [Fact] + public static void Ctor_String_Long_ReturnsOriginalContents() + { + string longString = new string('a', 500); + Assert.Equal(u8(longString), new Utf8String(longString)); + } + [Fact] public static void Ctor_String_InvalidData_Throws() { @@ -255,76 +262,11 @@ public static void Ctor_NonValidating_FromByteSpan() Assert.Equal(u8("xyz"), actual); } - [Fact] - public static void Ctor_NonValidating_FromDelegate() - { - object expectedState = new object(); - SpanAction spanAction = (span, actualState) => - { - Assert.Same(expectedState, actualState); - Assert.NotEqual(0, span.Length); // shouldn't have been called for a zero-length span - - for (int i = 0; i < span.Length; i++) - { - Assert.Equal(0, span[i]); // should've been zero-inited - span[i] = (byte)('a' + (i % 26)); // writes "abc...xyzabc...xyz..." - } - }; - - ArgumentException exception = Assert.Throws(() => Utf8String.UnsafeCreateWithoutValidation(-1, expectedState, spanAction)); - Assert.Equal("length", exception.ParamName); - - exception = Assert.Throws(() => Utf8String.UnsafeCreateWithoutValidation(10, expectedState, action: null)); - Assert.Equal("action", exception.ParamName); - - Assert.Same(Utf8String.Empty, Utf8String.UnsafeCreateWithoutValidation(0, expectedState, spanAction)); - - Assert.Equal(u8("abcde"), Utf8String.UnsafeCreateWithoutValidation(5, expectedState, spanAction)); - } - - [Fact] - public static void Ctor_Validating_FromDelegate() - { - object expectedState = new object(); - SpanAction spanAction = (span, actualState) => - { - Assert.Same(expectedState, actualState); - Assert.NotEqual(0, span.Length); // shouldn't have been called for a zero-length span - - for (int i = 0; i < span.Length; i++) - { - Assert.Equal(0, span[i]); // should've been zero-inited - span[i] = (byte)('a' + (i % 26)); // writes "abc...xyzabc...xyz..." - } - }; - - ArgumentException exception = Assert.Throws(() => Utf8String.Create(-1, expectedState, spanAction)); - Assert.Equal("length", exception.ParamName); - - exception = Assert.Throws(() => Utf8String.Create(10, expectedState, action: null)); - Assert.Equal("action", exception.ParamName); - - Assert.Same(Utf8String.Empty, Utf8String.Create(0, expectedState, spanAction)); - - Assert.Equal(u8("abcde"), Utf8String.Create(5, expectedState, spanAction)); - } - - [Fact] - public static void Ctor_Validating_FromDelegate_ThrowsIfDelegateProvidesInvalidData() - { - SpanAction spanAction = (span, actualState) => - { - span[0] = 0xFF; // never a valid UTF-8 byte - }; - - Assert.Throws(() => Utf8String.Create(10, new object(), spanAction)); - } - [Fact] public static void Ctor_CreateFromRelaxed_Utf16() { Assert.Same(Utf8String.Empty, Utf8String.CreateFromRelaxed(ReadOnlySpan.Empty)); - Assert.Equal(u8("xy\uFFFDz"), Utf8String.CreateFromRelaxed("xy\ud800z")); + Assert.Equal(u8("xy\uFFFDz"), Utf8String.CreateFromRelaxed("xy\ud800z".AsSpan())); } [Fact] @@ -334,33 +276,6 @@ public static void Ctor_CreateFromRelaxed_Utf8() Assert.Equal(u8("xy\uFFFDz"), Utf8String.CreateFromRelaxed(new byte[] { (byte)'x', (byte)'y', 0xF4, 0x80, 0x80, (byte)'z' })); } - [Fact] - public static void Ctor_CreateRelaxed_FromDelegate() - { - object expectedState = new object(); - SpanAction spanAction = (span, actualState) => - { - Assert.Same(expectedState, actualState); - Assert.NotEqual(0, span.Length); // shouldn't have been called for a zero-length span - - for (int i = 0; i < span.Length; i++) - { - Assert.Equal(0, span[i]); // should've been zero-inited - span[i] = 0xFF; // never a valid UTF-8 byte - } - }; - - ArgumentException exception = Assert.Throws(() => Utf8String.CreateRelaxed(-1, expectedState, spanAction)); - Assert.Equal("length", exception.ParamName); - - exception = Assert.Throws(() => Utf8String.CreateRelaxed(10, expectedState, action: null)); - Assert.Equal("action", exception.ParamName); - - Assert.Same(Utf8String.Empty, Utf8String.CreateRelaxed(0, expectedState, spanAction)); - - Assert.Equal(u8("\uFFFD\uFFFD"), Utf8String.CreateRelaxed(2, expectedState, spanAction)); - } - [Fact] public static void Ctor_TryCreateFrom_Utf8() { @@ -399,18 +314,32 @@ public static void Ctor_TryCreateFrom_Utf16() // Well-formed ASCII contents - Assert.True(Utf8String.TryCreateFrom("Hello", out value)); + Assert.True(Utf8String.TryCreateFrom("Hello".AsSpan(), out value)); Assert.Equal(u8("Hello"), value); // Well-formed non-ASCII contents - Assert.True(Utf8String.TryCreateFrom("\U0001F47D", out value)); // U+1F47D EXTRATERRESTRIAL ALIEN + Assert.True(Utf8String.TryCreateFrom("\U0001F47D".AsSpan(), out value)); // U+1F47D EXTRATERRESTRIAL ALIEN Assert.Equal(u8("\U0001F47D"), value); // Ill-formed contents - Assert.False(Utf8String.TryCreateFrom("\uD800x", out value)); + Assert.False(Utf8String.TryCreateFrom("\uD800x".AsSpan(), out value)); Assert.Null(value); } + + private static void AssertSameAsEmpty(Utf8String value) + { +#if NETFRAMEWORK + // When OOB, we can't change the actual object returned from a constructor. + // So just assert the underlying "_bytes" is the same. + Assert.Equal(0, value.Length); + Assert.True(Unsafe.AreSame( + ref Unsafe.AsRef(in Utf8String.Empty.GetPinnableReference()), + ref Unsafe.AsRef(in value.GetPinnableReference()))); +#else + Assert.Same(Utf8String.Empty, value); +#endif + } } } diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8StringTests.Ctor.netcoreapp.cs b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8StringTests.Ctor.netcoreapp.cs new file mode 100644 index 0000000000000..5012fbe9435e3 --- /dev/null +++ b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8StringTests.Ctor.netcoreapp.cs @@ -0,0 +1,108 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Xunit; + +using static System.Tests.Utf8TestUtilities; + +namespace System.Tests +{ + public unsafe partial class Utf8StringTests + { + [Fact] + public static void Ctor_NonValidating_FromDelegate() + { + object expectedState = new object(); + SpanAction spanAction = (span, actualState) => + { + Assert.Same(expectedState, actualState); + Assert.NotEqual(0, span.Length); // shouldn't have been called for a zero-length span + + for (int i = 0; i < span.Length; i++) + { + Assert.Equal(0, span[i]); // should've been zero-inited + span[i] = (byte)('a' + (i % 26)); // writes "abc...xyzabc...xyz..." + } + }; + + ArgumentException exception = Assert.Throws(() => Utf8String.UnsafeCreateWithoutValidation(-1, expectedState, spanAction)); + Assert.Equal("length", exception.ParamName); + + exception = Assert.Throws(() => Utf8String.UnsafeCreateWithoutValidation(10, expectedState, action: null)); + Assert.Equal("action", exception.ParamName); + + Assert.Same(Utf8String.Empty, Utf8String.UnsafeCreateWithoutValidation(0, expectedState, spanAction)); + + Assert.Equal(u8("abcde"), Utf8String.UnsafeCreateWithoutValidation(5, expectedState, spanAction)); + } + + [Fact] + public static void Ctor_Validating_FromDelegate() + { + object expectedState = new object(); + SpanAction spanAction = (span, actualState) => + { + Assert.Same(expectedState, actualState); + Assert.NotEqual(0, span.Length); // shouldn't have been called for a zero-length span + + for (int i = 0; i < span.Length; i++) + { + Assert.Equal(0, span[i]); // should've been zero-inited + span[i] = (byte)('a' + (i % 26)); // writes "abc...xyzabc...xyz..." + } + }; + + ArgumentException exception = Assert.Throws(() => Utf8String.Create(-1, expectedState, spanAction)); + Assert.Equal("length", exception.ParamName); + + exception = Assert.Throws(() => Utf8String.Create(10, expectedState, action: null)); + Assert.Equal("action", exception.ParamName); + + Assert.Same(Utf8String.Empty, Utf8String.Create(0, expectedState, spanAction)); + + Assert.Equal(u8("abcde"), Utf8String.Create(5, expectedState, spanAction)); + } + + [Fact] + public static void Ctor_Validating_FromDelegate_ThrowsIfDelegateProvidesInvalidData() + { + SpanAction spanAction = (span, actualState) => + { + span[0] = 0xFF; // never a valid UTF-8 byte + }; + + Assert.Throws(() => Utf8String.Create(10, new object(), spanAction)); + } + + [Fact] + public static void Ctor_CreateRelaxed_FromDelegate() + { + object expectedState = new object(); + SpanAction spanAction = (span, actualState) => + { + Assert.Same(expectedState, actualState); + Assert.NotEqual(0, span.Length); // shouldn't have been called for a zero-length span + + for (int i = 0; i < span.Length; i++) + { + Assert.Equal(0, span[i]); // should've been zero-inited + span[i] = 0xFF; // never a valid UTF-8 byte + } + }; + + ArgumentException exception = Assert.Throws(() => Utf8String.CreateRelaxed(-1, expectedState, spanAction)); + Assert.Equal("length", exception.ParamName); + + exception = Assert.Throws(() => Utf8String.CreateRelaxed(10, expectedState, action: null)); + Assert.Equal("action", exception.ParamName); + + Assert.Same(Utf8String.Empty, Utf8String.CreateRelaxed(0, expectedState, spanAction)); + + Assert.Equal(u8("\uFFFD\uFFFD"), Utf8String.CreateRelaxed(2, expectedState, spanAction)); + } + } +} diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8StringTests.Searching.cs b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8StringTests.Searching.cs index 573ffd34f8149..245877de18b0d 100644 --- a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8StringTests.Searching.cs +++ b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8StringTests.Searching.cs @@ -104,57 +104,71 @@ public static void TryFind_Char_WithComparison(ustring source, char searchTerm, CultureInfo.CurrentCulture = currentCulture; } - // First, search forward - - bool wasFound = source.TryFind(searchTerm, comparison, out Range actualForwardMatch); - Assert.Equal(expectedForwardMatch.HasValue, wasFound); - - if (wasFound) - { - AssertRangesEqual(source.Length, expectedForwardMatch.Value, actualForwardMatch); - } - - // Also check Contains / StartsWith / SplitOn - - Assert.Equal(wasFound, source.Contains(searchTerm, comparison)); - Assert.Equal(wasFound && source[..actualForwardMatch.Start].Length == 0, source.StartsWith(searchTerm, comparison)); - - (var before, var after) = source.SplitOn(searchTerm, comparison); - if (wasFound) - { - Assert.Equal(source[..actualForwardMatch.Start], before); - Assert.Equal(source[actualForwardMatch.End..], after); - } - else - { - Assert.Same(source, before); // check for reference equality - Assert.Null(after); - } - - // Now search backward - - wasFound = source.TryFindLast(searchTerm, comparison, out Range actualBackwardMatch); - Assert.Equal(expectedBackwardMatch.HasValue, wasFound); - - if (wasFound) + if (IsTryFindSupported(comparison)) { - AssertRangesEqual(source.Length, expectedBackwardMatch.Value, actualBackwardMatch); - } - - // Also check EndsWith / SplitOnLast - - Assert.Equal(wasFound && source[actualBackwardMatch.End..].Length == 0, source.EndsWith(searchTerm, comparison)); - - (before, after) = source.SplitOnLast(searchTerm, comparison); - if (wasFound) - { - Assert.Equal(source[..actualBackwardMatch.Start], before); - Assert.Equal(source[actualBackwardMatch.End..], after); + // First, search forward + + bool wasFound = source.TryFind(searchTerm, comparison, out Range actualForwardMatch); + Assert.Equal(expectedForwardMatch.HasValue, wasFound); + + if (wasFound) + { + AssertRangesEqual(source.Length, expectedForwardMatch.Value, actualForwardMatch); + } + + // Also check Contains / StartsWith / SplitOn + + Assert.Equal(wasFound, source.Contains(searchTerm, comparison)); + Assert.Equal(wasFound && source[..actualForwardMatch.Start].Length == 0, source.StartsWith(searchTerm, comparison)); + + (var before, var after) = source.SplitOn(searchTerm, comparison); + if (wasFound) + { + Assert.Equal(source[..actualForwardMatch.Start], before); + Assert.Equal(source[actualForwardMatch.End..], after); + } + else + { + Assert.Same(source, before); // check for reference equality + Assert.Null(after); + } + + // Now search backward + + wasFound = source.TryFindLast(searchTerm, comparison, out Range actualBackwardMatch); + Assert.Equal(expectedBackwardMatch.HasValue, wasFound); + + if (wasFound) + { + AssertRangesEqual(source.Length, expectedBackwardMatch.Value, actualBackwardMatch); + } + + // Also check EndsWith / SplitOnLast + + Assert.Equal(wasFound && source[actualBackwardMatch.End..].Length == 0, source.EndsWith(searchTerm, comparison)); + + (before, after) = source.SplitOnLast(searchTerm, comparison); + if (wasFound) + { + Assert.Equal(source[..actualBackwardMatch.Start], before); + Assert.Equal(source[actualBackwardMatch.End..], after); + } + else + { + Assert.Same(source, before); // check for reference equality + Assert.Null(after); + } } else { - Assert.Same(source, before); // check for reference equality - Assert.Null(after); + Assert.Throws(() => source.TryFind(searchTerm, comparison, out var _)); + Assert.Throws(() => source.TryFindLast(searchTerm, comparison, out var _)); + Assert.Throws(() => source.SplitOn(searchTerm, comparison)); + Assert.Throws(() => source.SplitOnLast(searchTerm, comparison)); + + Assert.Equal(expectedForwardMatch.HasValue, source.Contains(searchTerm, comparison)); + Assert.Equal(expectedForwardMatch.HasValue && source[..expectedForwardMatch.Value.Start].Length == 0, source.StartsWith(searchTerm, comparison)); + Assert.Equal(expectedBackwardMatch.HasValue && source[expectedBackwardMatch.Value.End..].Length == 0, source.EndsWith(searchTerm, comparison)); } }); } @@ -243,57 +257,71 @@ public static void TryFind_Rune_WithComparison(ustring source, Rune searchTerm, CultureInfo.CurrentCulture = currentCulture; } - // First, search forward - - bool wasFound = source.TryFind(searchTerm, comparison, out Range actualForwardMatch); - Assert.Equal(expectedForwardMatch.HasValue, wasFound); - - if (wasFound) - { - AssertRangesEqual(source.Length, expectedForwardMatch.Value, actualForwardMatch); - } - - // Also check Contains / StartsWith / SplitOn - - Assert.Equal(wasFound, source.Contains(searchTerm, comparison)); - Assert.Equal(wasFound && source[..actualForwardMatch.Start].Length == 0, source.StartsWith(searchTerm, comparison)); - - (var before, var after) = source.SplitOn(searchTerm, comparison); - if (wasFound) - { - Assert.Equal(source[..actualForwardMatch.Start], before); - Assert.Equal(source[actualForwardMatch.End..], after); - } - else - { - Assert.Same(source, before); // check for reference equality - Assert.Null(after); - } - - // Now search backward - - wasFound = source.TryFindLast(searchTerm, comparison, out Range actualBackwardMatch); - Assert.Equal(expectedBackwardMatch.HasValue, wasFound); - - if (wasFound) + if (IsTryFindSupported(comparison)) { - AssertRangesEqual(source.Length, expectedBackwardMatch.Value, actualBackwardMatch); - } - - // Also check EndsWith / SplitOnLast - - Assert.Equal(wasFound && source[actualBackwardMatch.End..].Length == 0, source.EndsWith(searchTerm, comparison)); - - (before, after) = source.SplitOnLast(searchTerm, comparison); - if (wasFound) - { - Assert.Equal(source[..actualBackwardMatch.Start], before); - Assert.Equal(source[actualBackwardMatch.End..], after); + // First, search forward + + bool wasFound = source.TryFind(searchTerm, comparison, out Range actualForwardMatch); + Assert.Equal(expectedForwardMatch.HasValue, wasFound); + + if (wasFound) + { + AssertRangesEqual(source.Length, expectedForwardMatch.Value, actualForwardMatch); + } + + // Also check Contains / StartsWith / SplitOn + + Assert.Equal(wasFound, source.Contains(searchTerm, comparison)); + Assert.Equal(wasFound && source[..actualForwardMatch.Start].Length == 0, source.StartsWith(searchTerm, comparison)); + + (var before, var after) = source.SplitOn(searchTerm, comparison); + if (wasFound) + { + Assert.Equal(source[..actualForwardMatch.Start], before); + Assert.Equal(source[actualForwardMatch.End..], after); + } + else + { + Assert.Same(source, before); // check for reference equality + Assert.Null(after); + } + + // Now search backward + + wasFound = source.TryFindLast(searchTerm, comparison, out Range actualBackwardMatch); + Assert.Equal(expectedBackwardMatch.HasValue, wasFound); + + if (wasFound) + { + AssertRangesEqual(source.Length, expectedBackwardMatch.Value, actualBackwardMatch); + } + + // Also check EndsWith / SplitOnLast + + Assert.Equal(wasFound && source[actualBackwardMatch.End..].Length == 0, source.EndsWith(searchTerm, comparison)); + + (before, after) = source.SplitOnLast(searchTerm, comparison); + if (wasFound) + { + Assert.Equal(source[..actualBackwardMatch.Start], before); + Assert.Equal(source[actualBackwardMatch.End..], after); + } + else + { + Assert.Same(source, before); // check for reference equality + Assert.Null(after); + } } else { - Assert.Same(source, before); // check for reference equality - Assert.Null(after); + Assert.Throws(() =>source.TryFind(searchTerm, comparison, out var _)); + Assert.Throws(() =>source.TryFindLast(searchTerm, comparison, out var _)); + Assert.Throws(() =>source.SplitOn(searchTerm, comparison)); + Assert.Throws(() =>source.SplitOnLast(searchTerm, comparison)); + + Assert.Equal(expectedForwardMatch.HasValue, source.Contains(searchTerm, comparison)); + Assert.Equal(expectedForwardMatch.HasValue && source[..expectedForwardMatch.Value.Start].Length == 0, source.StartsWith(searchTerm, comparison)); + Assert.Equal(expectedBackwardMatch.HasValue && source[expectedBackwardMatch.Value.End..].Length == 0, source.EndsWith(searchTerm, comparison)); } }); } @@ -382,57 +410,71 @@ public static void TryFind_Utf8String_WithComparison(ustring source, ustring sea CultureInfo.CurrentCulture = currentCulture; } - // First, search forward - - bool wasFound = source.TryFind(searchTerm, comparison, out Range actualForwardMatch); - Assert.Equal(expectedForwardMatch.HasValue, wasFound); - - if (wasFound) - { - AssertRangesEqual(source.Length, expectedForwardMatch.Value, actualForwardMatch); - } - - // Also check Contains / StartsWith / SplitOn - - Assert.Equal(wasFound, source.Contains(searchTerm, comparison)); - Assert.Equal(wasFound && source[..actualForwardMatch.Start].Length == 0, source.StartsWith(searchTerm, comparison)); - - (var before, var after) = source.SplitOn(searchTerm, comparison); - if (wasFound) - { - Assert.Equal(source[..actualForwardMatch.Start], before); - Assert.Equal(source[actualForwardMatch.End..], after); - } - else - { - Assert.Same(source, before); // check for reference equality - Assert.Null(after); - } - - // Now search backward - - wasFound = source.TryFindLast(searchTerm, comparison, out Range actualBackwardMatch); - Assert.Equal(expectedBackwardMatch.HasValue, wasFound); - - if (wasFound) - { - AssertRangesEqual(source.Length, expectedBackwardMatch.Value, actualBackwardMatch); - } - - // Also check EndsWith / SplitOnLast - - Assert.Equal(wasFound && source[actualBackwardMatch.End..].Length == 0, source.EndsWith(searchTerm, comparison)); - - (before, after) = source.SplitOnLast(searchTerm, comparison); - if (wasFound) + if (IsTryFindSupported(comparison)) { - Assert.Equal(source[..actualBackwardMatch.Start], before); - Assert.Equal(source[actualBackwardMatch.End..], after); + // First, search forward + + bool wasFound = source.TryFind(searchTerm, comparison, out Range actualForwardMatch); + Assert.Equal(expectedForwardMatch.HasValue, wasFound); + + if (wasFound) + { + AssertRangesEqual(source.Length, expectedForwardMatch.Value, actualForwardMatch); + } + + // Also check Contains / StartsWith / SplitOn + + Assert.Equal(wasFound, source.Contains(searchTerm, comparison)); + Assert.Equal(wasFound && source[..actualForwardMatch.Start].Length == 0, source.StartsWith(searchTerm, comparison)); + + (var before, var after) = source.SplitOn(searchTerm, comparison); + if (wasFound) + { + Assert.Equal(source[..actualForwardMatch.Start], before); + Assert.Equal(source[actualForwardMatch.End..], after); + } + else + { + Assert.Same(source, before); // check for reference equality + Assert.Null(after); + } + + // Now search backward + + wasFound = source.TryFindLast(searchTerm, comparison, out Range actualBackwardMatch); + Assert.Equal(expectedBackwardMatch.HasValue, wasFound); + + if (wasFound) + { + AssertRangesEqual(source.Length, expectedBackwardMatch.Value, actualBackwardMatch); + } + + // Also check EndsWith / SplitOnLast + + Assert.Equal(wasFound && source[actualBackwardMatch.End..].Length == 0, source.EndsWith(searchTerm, comparison)); + + (before, after) = source.SplitOnLast(searchTerm, comparison); + if (wasFound) + { + Assert.Equal(source[..actualBackwardMatch.Start], before); + Assert.Equal(source[actualBackwardMatch.End..], after); + } + else + { + Assert.Same(source, before); // check for reference equality + Assert.Null(after); + } } else { - Assert.Same(source, before); // check for reference equality - Assert.Null(after); + Assert.Throws(() => source.TryFind(searchTerm, comparison, out var _)); + Assert.Throws(() => source.TryFindLast(searchTerm, comparison, out var _)); + Assert.Throws(() => source.SplitOn(searchTerm, comparison)); + Assert.Throws(() => source.SplitOnLast(searchTerm, comparison)); + + Assert.Equal(expectedForwardMatch.HasValue, source.Contains(searchTerm, comparison)); + Assert.Equal(expectedForwardMatch.HasValue && source[..expectedForwardMatch.Value.Start].Length == 0, source.StartsWith(searchTerm, comparison)); + Assert.Equal(expectedBackwardMatch.HasValue && source[expectedBackwardMatch.Value.End..].Length == 0, source.EndsWith(searchTerm, comparison)); } }); } diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8StringTests.cs b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8StringTests.cs index 12b8ad587d488..260382383a68a 100644 --- a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8StringTests.cs +++ b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8StringTests.cs @@ -234,7 +234,12 @@ public static void IsNullOrWhiteSpace(string input, bool expected) [Fact] public static void ToByteArray_Empty() { +#if NETFRAMEWORK + // An empty Span.ToArray doesn't return Array.Empty on netfx + Assert.Equal(Array.Empty(), Utf8String.Empty.ToByteArray()); +#else Assert.Same(Array.Empty(), Utf8String.Empty.ToByteArray()); +#endif } [Fact] diff --git a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8TestUtilities.cs b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8TestUtilities.cs index 591dd8f5f0668..b1a5ba73db81b 100644 --- a/src/libraries/System.Utf8String.Experimental/tests/System/Utf8TestUtilities.cs +++ b/src/libraries/System.Utf8String.Experimental/tests/System/Utf8TestUtilities.cs @@ -33,12 +33,17 @@ public unsafe static bool IsNull(this Utf8Span span) return Unsafe.AreSame(ref Unsafe.AsRef(null), ref MemoryMarshal.GetReference(span.Bytes)); } + /// + /// Parses an expression of the form "a..b" and returns a . + /// + public static Range ParseRangeExpr(string expression) => ParseRangeExpr(expression.AsSpan()); + /// /// Parses an expression of the form "a..b" and returns a . /// public static Range ParseRangeExpr(ReadOnlySpan expression) { - int idxOfDots = expression.IndexOf("..", StringComparison.Ordinal); + int idxOfDots = expression.IndexOf("..".AsSpan(), StringComparison.Ordinal); if (idxOfDots < 0) { goto Error; @@ -57,7 +62,7 @@ public static Range ParseRangeExpr(ReadOnlySpan expression) firstPart = firstPart[1..]; } - if (!int.TryParse(firstPart, NumberStyles.AllowLeadingWhite | NumberStyles.AllowTrailingWhite, CultureInfo.InvariantCulture, out int startIndex)) + if (!int.TryParse(firstPart.ToString(), NumberStyles.AllowLeadingWhite | NumberStyles.AllowTrailingWhite, CultureInfo.InvariantCulture, out int startIndex)) { goto Error; } @@ -78,7 +83,7 @@ public static Range ParseRangeExpr(ReadOnlySpan expression) secondPart = secondPart[1..]; } - if (!int.TryParse(secondPart, NumberStyles.AllowLeadingWhite | NumberStyles.AllowTrailingWhite, CultureInfo.InvariantCulture, out int endIndex)) + if (!int.TryParse(secondPart.ToString(), NumberStyles.AllowLeadingWhite | NumberStyles.AllowTrailingWhite, CultureInfo.InvariantCulture, out int endIndex)) { goto Error; } @@ -147,14 +152,14 @@ public static Utf8String u8(string str) MemoryStream memStream = new MemoryStream(); - Span utf8Bytes = stackalloc byte[4]; // 4 UTF-8 code units is the largest any scalar value can be encoded as + byte[] utf8Bytes = new byte[4]; // 4 UTF-8 code units is the largest any scalar value can be encoded as int index = 0; while (index < str.Length) { if (Rune.TryGetRuneAt(str, index, out Rune value) && value.TryEncodeToUtf8(utf8Bytes, out int bytesWritten)) { - memStream.Write(utf8Bytes.Slice(0, bytesWritten)); + memStream.Write(utf8Bytes, 0, bytesWritten); index += value.Utf16SequenceLength; } else @@ -195,5 +200,10 @@ public static bool IsEmpty(this Range range, int length) (_, int actualLength) = range.GetOffsetAndLength(length); return (actualLength == 0); } + + public static bool IsTryFindSupported(StringComparison comparison) => + !PlatformDetection.IsNetFramework || + comparison == StringComparison.Ordinal || + comparison == StringComparison.OrdinalIgnoreCase; } }