[browser] [wasm] Request Streaming upload (#91295)

Co-authored-by: pavelsavara <[email protected]>

src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.Cancellation.cs

@@ -28,7 +28,6 @@ public HttpClientHandler_Cancellation_Test(ITestOutputHelper output) : base(outp
         [Theory]
         [InlineData(false, CancellationMode.Token)]
         [InlineData(true, CancellationMode.Token)]
-        [ActiveIssue("https://github.com/dotnet/runtime/issues/36634", TestPlatforms.Browser)] // out of memory
         public async Task PostAsync_CancelDuringRequestContentSend_TaskCanceledQuickly(bool chunkedTransfer, CancellationMode mode)
         {
             if (LoopbackServerFactory.Version >= HttpVersion20.Value && chunkedTransfer)
@@ -42,6 +41,12 @@ public async Task PostAsync_CancelDuringRequestContentSend_TaskCanceledQuickly(b
                 return;
             }
 
+            if (PlatformDetection.IsBrowser && LoopbackServerFactory.Version < HttpVersion20.Value)
+            {
+                // Browser request streaming is only supported on HTTP/2 or higher
+                return;
+            }
+
             var serverRelease = new TaskCompletionSource<bool>();
             await LoopbackServerFactory.CreateClientAndServerAsync(async uri =>
             {
@@ -58,6 +63,13 @@ await LoopbackServerFactory.CreateClientAndServerAsync(async uri =>
                         req.Content = new ByteAtATimeContent(int.MaxValue, waitToSend.Task, contentSending, millisecondDelayBetweenBytes: 1);
                         req.Headers.TransferEncodingChunked = chunkedTransfer;
 
+                        if (PlatformDetection.IsBrowser)
+                        {
+#if !NETFRAMEWORK
+                            req.Options.Set(new HttpRequestOptionsKey<bool>("WebAssemblyEnableStreamingRequest"), true);
+#endif
+                        }
+
                         Task<HttpResponseMessage> resp = client.SendAsync(TestAsync, req, HttpCompletionOption.ResponseHeadersRead, cts.Token);
                         waitToSend.SetResult(true);
                         await Task.WhenAny(contentSending.Task, resp);

src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.cs

@@ -1886,16 +1886,30 @@ await connection.ReadRequestHeaderAndSendCustomResponseAsync(
         }
 
         [Theory]
-        [InlineData(false)]
-        [InlineData(true)]
-        public async Task PostAsync_ThrowFromContentCopy_RequestFails(bool syncFailure)
+        [InlineData(false, false)]
+        [InlineData(false, true)]
+        [InlineData(true, false)]
+        [InlineData(true, true)]
+        public async Task PostAsync_ThrowFromContentCopy_RequestFails(bool syncFailure, bool enableWasmStreaming)
         {
             if (UseVersion == HttpVersion30)
             {
                 // TODO: Make this version-indepdendent
                 return;
             }
 
+            if (enableWasmStreaming && !PlatformDetection.IsBrowser)
+            {
+                // enableWasmStreaming makes only sense on Browser platform
+                return;
+            }
+
+            if (enableWasmStreaming && PlatformDetection.IsBrowser && UseVersion < HttpVersion20.Value)
+            {
+                // Browser request streaming is only supported on HTTP/2 or higher
+                return;
+            }
+
             await LoopbackServer.CreateServerAsync(async (server, uri) =>
             {
                 Task responseTask = server.AcceptConnectionAsync(async connection =>
@@ -1914,8 +1928,20 @@ await LoopbackServer.CreateServerAsync(async (server, uri) =>
                         canReadFunc: () => true,
                         readFunc: (buffer, offset, count) => throw error,
                         readAsyncFunc: (buffer, offset, count, cancellationToken) => syncFailure ? throw error : Task.Delay(1).ContinueWith<int>(_ => throw error)));
+                    var request = new HttpRequestMessage(HttpMethod.Post, uri);
+                    request.Content = content;
+
+                    if (PlatformDetection.IsBrowser)
+                    {
+                        if (enableWasmStreaming)
+                        {
+#if !NETFRAMEWORK
+                            request.Options.Set(new HttpRequestOptionsKey<bool>("WebAssemblyEnableStreamingRequest"), true);
+#endif
+                        }
+                    }
 
-                    Assert.Same(error, await Assert.ThrowsAsync<FormatException>(() => client.PostAsync(uri, content)));
+                    Assert.Same(error, await Assert.ThrowsAsync<FormatException>(() => client.SendAsync(request)));
                 }
             });
         }

src/libraries/Common/tests/System/Net/Http/ResponseStreamTest.cs

@@ -229,9 +229,147 @@ await client.GetAsync(remoteServer.EchoUri, HttpCompletionOption.ResponseHeaders
         }
 
 #if NETCOREAPP
-        [OuterLoop]
+
         [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsBrowser))]
         public async Task BrowserHttpHandler_Streaming()
+        {
+            var WebAssemblyEnableStreamingRequestKey = new HttpRequestOptionsKey<bool>("WebAssemblyEnableStreamingRequest");
+            var WebAssemblyEnableStreamingResponseKey = new HttpRequestOptionsKey<bool>("WebAssemblyEnableStreamingResponse");
+
+            var req = new HttpRequestMessage(HttpMethod.Post, Configuration.Http.RemoteHttp2Server.BaseUri + "echobody.ashx");
+
+            req.Options.Set(WebAssemblyEnableStreamingRequestKey, true);
+            req.Options.Set(WebAssemblyEnableStreamingResponseKey, true);
+
+            byte[] body = new byte[1024 * 1024];
+            Random.Shared.NextBytes(body);
+
+            int readOffset = 0;
+            req.Content = new StreamContent(new DelegateStream(
+                readAsyncFunc: async (buffer, offset, count, cancellationToken) =>
+                {
+                    await Task.Delay(1);
+                    if (readOffset < body.Length)
+                    {
+                        int send = Math.Min(body.Length - readOffset, count);
+                        body.AsSpan(readOffset, send).CopyTo(buffer.AsSpan(offset, send));
+                        readOffset += send;
+                        return send;
+                    }
+                    return 0;
+                }));
+
+            using (HttpClient client = CreateHttpClientForRemoteServer(Configuration.Http.RemoteHttp2Server))
+            // we need to switch off Response buffering of default ResponseContentRead option
+            using (HttpResponseMessage response = await client.SendAsync(req, HttpCompletionOption.ResponseHeadersRead))
+            {
+                Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+                // Streaming requests can't set Content-Length
+                Assert.False(response.Headers.Contains("X-HttpRequest-Headers-ContentLength"));
+                // Streaming response uses StreamContent
+                Assert.Equal(typeof(StreamContent), response.Content.GetType());
+
+                var stream = await response.Content.ReadAsStreamAsync();
+                Assert.Equal("ReadOnlyStream", stream.GetType().Name);
+                var buffer = new byte[1024 * 1024];
+                int totalCount = 0;
+                int fetchedCount = 0;
+                do
+                {
+                    fetchedCount = await stream.ReadAsync(buffer, 0, buffer.Length);
+                    Assert.True(body.AsSpan(totalCount, fetchedCount).SequenceEqual(buffer.AsSpan(0, fetchedCount)));
+                    totalCount += fetchedCount;
+                } while (fetchedCount != 0);
+                Assert.Equal(body.Length, totalCount);
+            }
+        }
+
+        [OuterLoop]
+        [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsBrowser))]
+        [InlineData(true)]
+        [InlineData(false)]
+        public async Task BrowserHttpHandler_StreamingRequest(bool useStringContent)
+        {
+            var WebAssemblyEnableStreamingRequestKey = new HttpRequestOptionsKey<bool>("WebAssemblyEnableStreamingRequest");
+
+            var req = new HttpRequestMessage(HttpMethod.Post, Configuration.Http.Http2RemoteVerifyUploadServer);
+
+            req.Options.Set(WebAssemblyEnableStreamingRequestKey, true);
+
+            int size;
+            if (useStringContent)
+            {
+                string bodyContent = "Hello World";
+                size = bodyContent.Length;
+                req.Content = new StringContent(bodyContent);
+            }
+            else
+            {
+                size = 1500 * 1024 * 1024;
+                int remaining = size;
+                req.Content = new StreamContent(new DelegateStream(
+                    readAsyncFunc: (buffer, offset, count, cancellationToken) =>
+                    {
+                        if (remaining > 0)
+                        {
+                            int send = Math.Min(remaining, count);
+                            buffer.AsSpan(offset, send).Fill(65);
+                            remaining -= send;
+                            return Task.FromResult(send);
+                        }
+                        return Task.FromResult(0);
+                    }));
+            }
+
+            req.Content.Headers.Add("Content-MD5-Skip", "browser");
+
+            using (HttpClient client = CreateHttpClientForRemoteServer(Configuration.Http.RemoteHttp2Server))
+            using (HttpResponseMessage response = await client.SendAsync(req))
+            {
+                Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+                Assert.Equal(size.ToString(), Assert.Single(response.Headers.GetValues("X-HttpRequest-Body-Length")));
+                // Streaming requests can't set Content-Length
+                Assert.Equal(useStringContent, response.Headers.Contains("X-HttpRequest-Headers-ContentLength"));
+                if (useStringContent)
+                {
+                    Assert.Equal(size.ToString(), Assert.Single(response.Headers.GetValues("X-HttpRequest-Headers-ContentLength")));
+                }
+            }
+        }
+
+        // Duplicate of PostAsync_ThrowFromContentCopy_RequestFails using remote server
+        [OuterLoop]
+        [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsBrowser))]
+        [InlineData(false)]
+        [InlineData(true)]
+        public async Task BrowserHttpHandler_StreamingRequest_ThrowFromContentCopy_RequestFails(bool syncFailure)
+        {
+            var WebAssemblyEnableStreamingRequestKey = new HttpRequestOptionsKey<bool>("WebAssemblyEnableStreamingRequest");
+
+            var req = new HttpRequestMessage(HttpMethod.Post, Configuration.Http.Http2RemoteEchoServer);
+
+            req.Options.Set(WebAssemblyEnableStreamingRequestKey, true);
+
+            Exception error = new FormatException();
+            var content = new StreamContent(new DelegateStream(
+                canSeekFunc: () => true,
+                lengthFunc: () => 12345678,
+                positionGetFunc: () => 0,
+                canReadFunc: () => true,
+                readFunc: (buffer, offset, count) => throw error,
+                readAsyncFunc: (buffer, offset, count, cancellationToken) => syncFailure ? throw error : Task.Delay(1).ContinueWith<int>(_ => throw error)));
+
+            req.Content = content;
+
+            using (HttpClient client = CreateHttpClientForRemoteServer(Configuration.Http.RemoteHttp2Server))
+            {
+                Assert.Same(error, await Assert.ThrowsAsync<FormatException>(() => client.SendAsync(req)));
+            }
+        }
+
+        [OuterLoop]
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsBrowser))]
+        public async Task BrowserHttpHandler_StreamingResponse()
         {
             var WebAssemblyEnableStreamingResponseKey = new HttpRequestOptionsKey<bool>("WebAssemblyEnableStreamingResponse");
 
@@ -244,6 +382,7 @@ public async Task BrowserHttpHandler_Streaming()
             // we need to switch off Response buffering of default ResponseContentRead option
             using (HttpResponseMessage response = await client.SendAsync(req, HttpCompletionOption.ResponseHeadersRead))
             {
+                // Streaming response uses StreamContent
                 Assert.Equal(typeof(StreamContent), response.Content.GetType());
 
                 Assert.Equal("application/octet-stream", response.Content.Headers.ContentType.MediaType);

src/libraries/Common/tests/System/Net/Prerequisites/NetCoreServer/GenericHandler.cs

@@ -88,6 +88,11 @@ public async Task Invoke(HttpContext context)
                 await LargeResponseHandler.InvokeAsync(context);
                 return;
             }
+            if (path.Equals(new PathString("/echobody.ashx")))
+            {
+                await EchoBodyHandler.InvokeAsync(context);
+                return;
+            }
 
             // Default handling.
             await EchoHandler.InvokeAsync(context);

src/libraries/Common/tests/System/Net/Prerequisites/NetCoreServer/Handlers/EchoBodyHandler.cs

@@ -0,0 +1,38 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Http.Features;
+
+namespace NetCoreServer
+{
+    public class EchoBodyHandler
+    {
+        public static async Task InvokeAsync(HttpContext context)
+        {
+            context.Features.Get<IHttpMaxRequestBodySizeFeature>().MaxRequestBodySize = null;
+
+            // Report back original request method verb.
+            context.Response.Headers["X-HttpRequest-Method"] = context.Request.Method;
+
+            // Report back original entity-body related request headers.
+            string contentLength = context.Request.Headers["Content-Length"];
+            if (!string.IsNullOrEmpty(contentLength))
+            {
+                context.Response.Headers["X-HttpRequest-Headers-ContentLength"] = contentLength;
+            }
+
+            string transferEncoding = context.Request.Headers["Transfer-Encoding"];
+            if (!string.IsNullOrEmpty(transferEncoding))
+            {
+                context.Response.Headers["X-HttpRequest-Headers-TransferEncoding"] = transferEncoding;
+            }
+
+            context.Response.StatusCode = 200;
+            context.Response.ContentType = context.Request.ContentType;
+            await context.Request.Body.CopyToAsync(context.Response.Body);
+        }
+    }
+}

src/libraries/Common/tests/System/Net/Prerequisites/NetCoreServer/Handlers/VerifyUploadHandler.cs

@@ -6,13 +6,16 @@
 using System.Security.Cryptography;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Http.Features;
 
 namespace NetCoreServer
 {
     public class VerifyUploadHandler
     {
         public static async Task InvokeAsync(HttpContext context)
         {
+            context.Features.Get<IHttpMaxRequestBodySizeFeature>().MaxRequestBodySize = null;
+
             // Report back original request method verb.
             context.Response.Headers["X-HttpRequest-Method"] = context.Request.Method;
 
@@ -29,12 +32,15 @@ public static async Task InvokeAsync(HttpContext context)
                 context.Response.Headers["X-HttpRequest-Headers-TransferEncoding"] = transferEncoding;
             }
 
-            // Get request body.
-            byte[] requestBodyBytes = await ReadAllRequestBytesAsync(context);
+            // Compute MD5 hash of received request body.
+            (byte[] md5Bytes, int bodyLength) = await ComputeMD5HashRequestBodyAsync(context);
+
+            // Report back the actual body length.
+            context.Response.Headers["X-HttpRequest-Body-Length"] = bodyLength.ToString();
 
-            // Skip MD5 checksum for empty request body 
+            // Skip MD5 checksum for empty request body
             // or for requests which opt to skip it due to [ActiveIssue("https://github.com/dotnet/runtime/issues/37669", TestPlatforms.Browser)]
-            if (requestBodyBytes.Length == 0 || !string.IsNullOrEmpty(context.Request.Headers["Content-MD5-Skip"]))
+            if (bodyLength == 0 || !string.IsNullOrEmpty(context.Request.Headers["Content-MD5-Skip"]))
             {
                 context.Response.StatusCode = 200;
                 return;
@@ -49,13 +55,7 @@ public static async Task InvokeAsync(HttpContext context)
                 return;
             }
 
-            // Compute MD5 hash of received request body.
-            string actualHash;
-            using (MD5 md5 = MD5.Create())
-            {
-                byte[] hash = md5.ComputeHash(requestBodyBytes);
-                actualHash = Convert.ToBase64String(hash);
-            }
+            string actualHash = Convert.ToBase64String(md5Bytes);
 
             if (expectedHash == actualHash)
             {
@@ -66,21 +66,22 @@ public static async Task InvokeAsync(HttpContext context)
                 context.Response.StatusCode = 400;
                 context.Response.SetStatusDescription("Received request body fails MD5 checksum");
             }
-
         }
 
-        private static async Task<byte[]> ReadAllRequestBytesAsync(HttpContext context)
+        private static async Task<(byte[] MD5Hash, int BodyLength)> ComputeMD5HashRequestBodyAsync(HttpContext context)
         {
             Stream requestStream = context.Request.Body;
             byte[] buffer = new byte[16 * 1024];
-            using (MemoryStream ms = new MemoryStream())
+            using (MD5 md5 = MD5.Create())
             {
-                int read;
+                int read, size = 0;
                 while ((read = await requestStream.ReadAsync(buffer, 0, buffer.Length)) > 0)
                 {
-                    ms.Write(buffer, 0, read);
+                    size += read;
+                    md5.TransformBlock(buffer, 0, read, buffer, 0);
                 }
-                return ms.ToArray();
+                md5.TransformFinalBlock(buffer, 0, read);
+                return (md5.Hash, size);
             }
         }
     }

src/libraries/Common/tests/System/Net/Prerequisites/NetCoreServer/NetCoreServer.csproj

@@ -26,6 +26,7 @@
   <ItemGroup>
     <Compile Include="Handlers\DeflateHandler.cs" />
     <Compile Include="Handlers\EchoHandler.cs" />
+    <Compile Include="Handlers\EchoBodyHandler.cs" />
     <Compile Include="Handlers\EchoWebSocketHandler.cs" />
     <Compile Include="Handlers\EchoWebSocketHeadersHandler.cs" />
     <Compile Include="Handlers\EmptyContentHandler.cs" />