Add Base64url encoding/decoding

src/libraries/System.Memory/tests/Base64/Base64DecoderUnitTests.cs

@@ -1,7 +1,6 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
@@ -273,6 +272,9 @@ public void BasicDecodingWithFinalBlockTrueKnownInputDone(string inputString, in
 
         [Theory]
         [InlineData("A", 0, 0)]
+        [InlineData("A===", 0, 0)]
+        [InlineData("A==", 0, 0)]
+        [InlineData("A=", 0, 0)]
         [InlineData("AQ", 0, 0)]
         [InlineData("AQI", 0, 0)]
         [InlineData("AQIDBA", 4, 3)]
@@ -285,16 +287,18 @@ public void BasicDecodingWithFinalBlockTrueKnownInputInvalid(string inputString,
             Assert.Equal(OperationStatus.InvalidData, Base64.DecodeFromUtf8(source, decodedBytes, out int consumed, out int decodedByteCount));
             Assert.Equal(expectedConsumed, consumed);
             Assert.Equal(expectedWritten, decodedByteCount); // expectedWritten == decodedBytes.Length
-            Assert.True(Base64TestHelper.VerifyDecodingCorrectness(expectedConsumed, decodedBytes.Length, source, decodedBytes));
+            Assert.True(Base64TestHelper.VerifyDecodingCorrectness(expectedConsumed, expectedWritten, source, decodedBytes));
         }
 
         [Theory]
         [InlineData("\u00ecz/T", 0, 0)]                                              // scalar code-path
         [InlineData("z/Ta123\u00ec", 4, 3)]
         [InlineData("\u00ecz/TpH7sqEkerqMweH1uSw==", 0, 0)]                          // Vector128 code-path
-        [InlineData("z/TpH7sqEkerqMweH1uSw\u00ec==", 20, 15)]
-        [InlineData("\u00ecz/TpH7sqEkerqMweH1uSw1a5ebaAF9xa8B0ze1wet4epo==", 0, 0)]  // Vector256 / AVX code-path
+        [InlineData("z/TpH7sqEkerqMweH1uSw\u5948==", 20, 15)]
+        [InlineData("\u5948/TpH7sqEkerqMweH1uSw1a5ebaAF9xa8B0ze1wet4epo==", 0, 0)]  // Vector256 / AVX code-path
         [InlineData("z/TpH7sqEkerqMweH1uSw1a5ebaAF9xa8B0ze1wet4epo\u00ec==", 44, 33)]
+        [InlineData("\u5948z+T/H7sqEkerqMweH1uSw1a5ebaAF9xa8B0ze1wet4epo01234567890123456789012345678901234567890123456789==", 0, 0)]  // Vector512 / Avx512Vbmi code-path
+        [InlineData("z/T+H7sqEkerqMweH1uSw1a5ebaAF9xa8B0ze1wet4epo01234567890123456789012345678901234567890123456789\u5948==", 92, 69)]
         public void BasicDecodingNonAsciiInputInvalid(string inputString, int expectedConsumed, int expectedWritten)
         {
             Span<byte> source = Encoding.UTF8.GetBytes(inputString);
@@ -749,19 +753,5 @@ public void BasicDecodingWithExtraWhitespaceShouldBeCountedInConsumedBytes(strin
             Assert.Equal(expectedWritten, decodedByteCount);
             Assert.True(Base64TestHelper.VerifyDecodingCorrectness(expectedConsumed, expectedWritten, source, decodedBytes));
         }
-
-        public static IEnumerable<object[]> BasicDecodingWithExtraWhitespaceShouldBeCountedInConsumedBytes_MemberData()
-        {
-            var r = new Random(42);
-            for (int i = 0; i < 5; i++)
-            {
-                yield return new object[] { "AQ==" + new string(r.GetItems<char>(" \n\t\r", i)), 4 + i, 1 };
-            }
-
-            foreach (string s in new[] { "MTIz", "M TIz", "MT Iz", "MTI z", "MTIz ", "M    TI   z", "M T I Z " })
-            {
-                yield return new object[] { s + s + s + s, s.Length * 4, 12 };
-            }
-        }
     }
 }

src/libraries/System.Memory/tests/Base64/Base64TestBase.cs

@@ -107,5 +107,19 @@ public static IEnumerable<object[]> StringsOnlyWithCharsToBeIgnored()
 
             string GetRepeatedChar(char charToInsert, int numberOfTimesToInsert) => new string(charToInsert, numberOfTimesToInsert);
         }
+
+        public static IEnumerable<object[]> BasicDecodingWithExtraWhitespaceShouldBeCountedInConsumedBytes_MemberData()
+        {
+            var r = new Random(42);
+            for (int i = 0; i < 5; i++)
+            {
+                yield return new object[] { "AQ==" + new string(r.GetItems<char>(" \n\t\r", i)), 4 + i, 1 };
+            }
+
+            foreach (string s in new[] { "MTIz", "M TIz", "MT Iz", "MTI z", "MTIz ", "M    TI   z", "M T I Z " })
+            {
+                yield return new object[] { s + s + s + s, s.Length * 4, 12 };
+            }
+        }
     }
 }

src/libraries/System.Memory/tests/Base64/Base64TestHelper.cs

@@ -24,12 +24,23 @@ public static class Base64TestHelper
             52, 53, 54, 55, 56, 57, 43, 47          //4..9, +, /
         };
 
+        public static readonly byte[] s_urlEncodingMap = {
+            65, 66, 67, 68, 69, 70, 71, 72,         //A..H
+            73, 74, 75, 76, 77, 78, 79, 80,         //I..P
+            81, 82, 83, 84, 85, 86, 87, 88,         //Q..X
+            89, 90, 97, 98, 99, 100, 101, 102,      //Y..Z, a..f
+            103, 104, 105, 106, 107, 108, 109, 110, //g..n
+            111, 112, 113, 114, 115, 116, 117, 118, //o..v
+            119, 120, 121, 122, 48, 49, 50, 51,     //w..z, 0..3
+            52, 53, 54, 55, 56, 57, 45, 95          //4..9, -, _
+        };
+
         // Pre-computing this table using a custom string(s_characters) and GenerateDecodingMapAndVerify (found in tests)
         public static readonly sbyte[] s_decodingMap = {
             -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
             -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
             -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63,         //62 is placed at index 43 (for +), 63 at index 47 (for /)
-            52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1,         //52-61 are placed at index 48-57 (for 0-9), 64 at index 61 (for =)
+            52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1,         //52-61 are placed at index 48-57 (for 0-9)
             -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
             15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1,         //0-25 are placed at index 65-90 (for A-Z)
             -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
@@ -44,9 +55,29 @@ public static class Base64TestHelper
             -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
         };
 
+        public static readonly sbyte[] s_urlDecodingMap = {
+            -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+            -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+            -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1,         //62 is placed at index 45 (for -), 63 at index 95 (for _)
+            52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1,         //52-61 are placed at index 48-57 (for 0-9)
+            -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
+            15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, 63,         //0-25 are placed at index 65-90 (for A-Z)
+            -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
+            41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1,         //26-51 are placed at index 97-122 (for a-z)
+            -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,         // Bytes over 122 ('z') are invalid and cannot be decoded
+            -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,         // Hence, padding the map with 255, which indicates invalid input
+            -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+            -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+            -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+            -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+            -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+            -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+        };
+
         public static bool IsByteToBeIgnored(byte charByte) => charByte is (byte)' ' or (byte)'\t' or (byte)'\r' or (byte)'\n';
 
         public const byte EncodingPad = (byte)'=';      // '=', for padding
+        public const byte UrlEncodingPad = (byte)'%';   // '%', for url padding
         public const sbyte InvalidByte = -1;            // Designating -1 for invalid bytes in the decoding map
 
         public static byte[] InvalidBytes
@@ -60,6 +91,17 @@ public static byte[] InvalidBytes
             }
         }
 
+        public static byte[] UrlInvalidBytes
+        {
+            get
+            {
+                int[] indices = s_urlDecodingMap.FindAllIndexOf(InvalidByte);
+                // Workaround for indices.Cast<byte>().ToArray() since it throws
+                // InvalidCastException: Unable to cast object of type 'System.Int32' to type 'System.Byte'
+                return indices.Select(i => (byte)i).ToArray();
+            }
+        }
+
         internal static void InitializeBytes(Span<byte> bytes, int seed = 100)
         {
             var rnd = new Random(seed);
@@ -79,6 +121,26 @@ internal static void InitializeDecodableBytes(Span<byte> bytes, int seed = 100)
             }
         }
 
+        internal static void InitializeUrlDecodableChars(Span<char> bytes, int seed = 100)
+        {
+            var rnd = new Random(seed);
+            for (int i = 0; i < bytes.Length; i++)
+            {
+                int index = (byte)rnd.Next(0, s_urlEncodingMap.Length);
+                bytes[i] = (char)s_urlEncodingMap[index];
+            }
+        }
+
+        internal static void InitializeUrlDecodableBytes(Span<byte> bytes, int seed = 100)
+        {
+            var rnd = new Random(seed);
+            for (int i = 0; i < bytes.Length; i++)
+            {
+                int index = (byte)rnd.Next(0, s_urlEncodingMap.Length);
+                bytes[i] = s_urlEncodingMap[index];
+            }
+        }
+
         [Fact]
         public static void GenerateEncodingMapAndVerify()
         {
@@ -112,16 +174,34 @@ public static int[] FindAllIndexOf<T>(this IEnumerable<T> values, T valueToFind)
 
         public static bool VerifyEncodingCorrectness(int expectedConsumed, int expectedWritten, Span<byte> source, Span<byte> encodedBytes)
         {
-            string expectedText = Convert.ToBase64String(source.Slice(0, expectedConsumed).ToArray());
-            string encodedText = Encoding.ASCII.GetString(encodedBytes.Slice(0, expectedWritten).ToArray());
+            string expectedText = Convert.ToBase64String(source.Slice(0, expectedConsumed));
+            string encodedText = Encoding.ASCII.GetString(encodedBytes.Slice(0, expectedWritten));
+            return expectedText.Equals(encodedText);
+        }
+
+        public static bool VerifyUrlEncodingCorrectness(int expectedConsumed, int expectedWritten, Span<byte> source, Span<byte> encodedBytes)
+        {
+            string expectedText = Convert.ToBase64String(source.Slice(0, expectedConsumed))
+                .Replace('+', '-').Replace('/', '_').TrimEnd('=');
+            string encodedText = Encoding.ASCII.GetString(encodedBytes.Slice(0, expectedWritten));
             return expectedText.Equals(encodedText);
         }
 
         public static bool VerifyDecodingCorrectness(int expectedConsumed, int expectedWritten, Span<byte> source, Span<byte> decodedBytes)
         {
-            string sourceString = Encoding.ASCII.GetString(source.Slice(0, expectedConsumed).ToArray());
+            string sourceString = Encoding.ASCII.GetString(source.Slice(0, expectedConsumed));
             byte[] expectedBytes = Convert.FromBase64String(sourceString);
             return expectedBytes.AsSpan().SequenceEqual(decodedBytes.Slice(0, expectedWritten));
         }
+
+        public static bool VerifyUrlDecodingCorrectness(int expectedConsumed, int expectedWritten, Span<byte> source, Span<byte> decodedBytes)
+        {
+            string sourceString = Encoding.ASCII.GetString(source.Slice(0, expectedConsumed));
+            string padded = sourceString.Length % 4 == 0 ? sourceString :
+                sourceString.PadRight(sourceString.Length + (4 - sourceString.Length % 4), '=');
+            string base64 = padded.Replace('_', '/').Replace('-', '+').Replace('%', '=');
+            byte[] expectedBytes = Convert.FromBase64String(base64);
+            return expectedBytes.AsSpan().SequenceEqual(decodedBytes.Slice(0, expectedWritten));
+        }
     }
 }

src/libraries/System.Memory/tests/Base64/Base64ValidationUnitTests.cs

@@ -69,6 +69,7 @@ public void BasicValidationInvalidInputLengthBytes()
                 } while (numBytes % 4 == 0);    // ensure we have a invalid length
 
                 Span<byte> source = new byte[numBytes];
+                Base64TestHelper.InitializeDecodableBytes(source, numBytes);
 
                 Assert.False(Base64.IsValid(source));
                 Assert.False(Base64.IsValid(source, out int decodedLength));
@@ -88,10 +89,16 @@ public void BasicValidationInvalidInputLengthChars()
                     numBytes = rnd.Next(100, 1000 * 1000);
                 } while (numBytes % 4 == 0);    // ensure we have a invalid length
 
-                Span<char> source = new char[numBytes];
+                Span<byte> source = new byte[numBytes];
+                Base64TestHelper.InitializeDecodableBytes(source, numBytes);
+                Span<char> chars = source
+                    .ToArray()
+                    .Select(Convert.ToChar)
+                    .ToArray()
+                    .AsSpan();
 
-                Assert.False(Base64.IsValid(source));
-                Assert.False(Base64.IsValid(source, out int decodedLength));
+                Assert.False(Base64.IsValid(chars));
+                Assert.False(Base64.IsValid(chars, out int decodedLength));
                 Assert.Equal(0, decodedLength);
             }
         }
@@ -267,7 +274,7 @@ public void InvalidSizeBytes(string utf8WithByteToBeIgnored)
         [InlineData("Y")]
         public void InvalidSizeChars(string utf8WithByteToBeIgnored)
         {
-            byte[] utf8BytesWithByteToBeIgnored = UTF8Encoding.UTF8.GetBytes(utf8WithByteToBeIgnored);
+            ReadOnlySpan<char> utf8BytesWithByteToBeIgnored = utf8WithByteToBeIgnored;
 
             Assert.False(Base64.IsValid(utf8BytesWithByteToBeIgnored));
             Assert.False(Base64.IsValid(utf8BytesWithByteToBeIgnored, out int decodedLength));
@@ -329,10 +336,10 @@ public void InvalidBase64Bytes(string utf8WithByteToBeIgnored)
         [InlineData("    a    ")]
         public void InvalidBase64Chars(string utf8WithByteToBeIgnored)
         {
-            byte[] utf8BytesWithByteToBeIgnored = UTF8Encoding.UTF8.GetBytes(utf8WithByteToBeIgnored);
+            ReadOnlySpan<char> utf8CharsWithCharToBeIgnored = utf8WithByteToBeIgnored;
 
-            Assert.False(Base64.IsValid(utf8BytesWithByteToBeIgnored));
-            Assert.False(Base64.IsValid(utf8BytesWithByteToBeIgnored, out int decodedLength));
+            Assert.False(Base64.IsValid(utf8CharsWithCharToBeIgnored));
+            Assert.False(Base64.IsValid(utf8CharsWithCharToBeIgnored, out int decodedLength));
             Assert.Equal(0, decodedLength);
         }
     }