Allow users to pass in HttpClient to use for file upload operations (A…

…zure#3293)

Since we don't target any version of .NET core in our device SDK, there is no way for us to create an HttpClient that sets the connection lease timeout for the user by default. Instead, we will allow users to pass in an HttpClient that we will use. This allows .NET core users to construct an HTTP client with their desired connection lease timeout, and give it to the SDK to use.

For .NET standard, net472, and .NET 5+ cases, the SDK can set the connection lease timeout for the user by default.

On the service SDK side, .NET standard, net472, .NET 5+ and .NET core 2.1+ cases will create an HttpClient with the connection lease timeout set by default.

common/src/service/HttpClientHelper.cs

@@ -10,8 +10,10 @@
 using System.Net;
 using System.Net.Http;
 using System.Net.Http.Headers;
+using System.Security.Cryptography.X509Certificates;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.Azure.Amqp.Transport;
 using Microsoft.Azure.Devices.Common;
 using Microsoft.Azure.Devices.Common.Exceptions;
 using Microsoft.Azure.Devices.Common.Extensions;
@@ -885,26 +887,39 @@ internal static HttpClient CreateDefaultClient(IWebProxy webProxy, Uri baseUri,
 
         internal static HttpMessageHandler CreateDefaultHttpMessageHandler(IWebProxy webProxy, Uri baseUri, int connectionLeaseTimeoutMilliseconds)
         {
-#pragma warning disable CA2000 // Dispose objects before losing scope (object is returned by this method, so the caller is responsible for disposing it)
-#if NETCOREAPP && !NETCOREAPP2_0 && !NETCOREAPP1_0 && !NETCOREAPP1_1
-            // SocketsHttpHandler is only available in netcoreapp2.1 and onwards
-            var httpMessageHandler = new SocketsHttpHandler();
-            httpMessageHandler.SslOptions.EnabledSslProtocols = TlsVersions.Instance.Preferred;
+            HttpMessageHandler httpMessageHandler = null;
+
+#if NETCOREAPP2_1_OR_GREATER || NET5_0_OR_GREATER
+            var socketsHandler = new SocketsHttpHandler();
+            socketsHandler.SslOptions.EnabledSslProtocols = TlsVersions.Instance.Preferred;
+
+            if (!TlsVersions.Instance.CertificateRevocationCheck)
+            {
+                socketsHandler.SslOptions.CertificateRevocationCheckMode = X509RevocationMode.NoCheck;
+            }
+
+            if (webProxy != DefaultWebProxySettings.Instance)
+            {
+                socketsHandler.UseProxy = webProxy != null;
+                socketsHandler.Proxy = webProxy;
+            }
+
+            httpMessageHandler = socketsHandler;
 #else
-            var httpMessageHandler = new HttpClientHandler();
+            var httpClientHandler = new HttpClientHandler();
 #if !NET451
-            httpMessageHandler.SslProtocols = TlsVersions.Instance.Preferred;
-            httpMessageHandler.CheckCertificateRevocationList = TlsVersions.Instance.CertificateRevocationCheck;
+            httpClientHandler.SslProtocols = TlsVersions.Instance.Preferred;
+            httpClientHandler.CheckCertificateRevocationList = TlsVersions.Instance.CertificateRevocationCheck;
 #endif
-#endif
-#pragma warning restore CA2000 // Dispose objects before losing scope
-
             if (webProxy != DefaultWebProxySettings.Instance)
             {
-                httpMessageHandler.UseProxy = webProxy != null;
-                httpMessageHandler.Proxy = webProxy;
+                httpClientHandler.UseProxy = webProxy != null;
+                httpClientHandler.Proxy = webProxy;
             }
 
+            httpMessageHandler = httpClientHandler;
+#endif
+
             ServicePointHelpers.SetLimits(httpMessageHandler, baseUri, connectionLeaseTimeoutMilliseconds);
 
             return httpMessageHandler;

e2e/test/iothub/FileUploadE2ETests.cs

@@ -4,9 +4,13 @@
 using System;
 using System.IO;
 using System.Net;
+using System.Net.Http;
 using System.Security.Cryptography.X509Certificates;
+using System.Threading;
 using System.Threading.Tasks;
+using FluentAssertions;
 using Microsoft.Azure.Devices.Client;
+using Microsoft.Azure.Devices.Client.Exceptions;
 using Microsoft.Azure.Devices.Client.Transport;
 using Microsoft.Azure.Devices.E2ETests.Helpers;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
@@ -127,6 +131,42 @@ public async Task FileUpload_SmallFile_Http_GranularSteps_Proxy()
             await UploadFileGranularAsync(fileStreamSource, filename, fileUploadTransportSettings).ConfigureAwait(false);
         }
 
+        // File upload requests can be configured to use a user-provided HttpClient
+        [TestMethod]
+        public async Task FileUpload_UsesCustomHttpClient()
+        {
+            using TestDevice testDevice =
+                await TestDevice.GetTestDeviceAsync(_devicePrefix, TestDeviceType.Sasl).ConfigureAwait(false);
+
+            using var CustomHttpMessageHandler = new CustomHttpMessageHandler();
+            var fileUploadSettings = new Http1TransportSettings()
+            {
+                // This HttpClient should throw a NotImplementedException whenever it makes an HTTP
+                // request
+                HttpClient = new HttpClient(CustomHttpMessageHandler),
+            };
+
+            var clientOptions = new ClientOptions()
+            {
+                FileUploadTransportSettings = fileUploadSettings,
+            };
+
+            using var deviceClient =
+                DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, clientOptions);
+
+            await deviceClient.OpenAsync().ConfigureAwait(false);
+
+            var request = new FileUploadSasUriRequest()
+            {
+                BlobName = "someBlobName",
+            };
+            var ex = await Assert.ThrowsExceptionAsync<IotHubException>(
+                            async () => await deviceClient.GetFileUploadSasUriAsync(request).ConfigureAwait(false));
+
+            ex.InnerException.Should().BeOfType<NotImplementedException>(
+                "The provided custom HttpMessageHandler throws NotImplementedException when making any HTTP request");
+        }
+
         private async Task UploadFileGranularAsync(Stream source, string filename, Http1TransportSettings fileUploadTransportSettings, bool useX509auth = false)
         {
             using TestDevice testDevice = await TestDevice.GetTestDeviceAsync(
@@ -265,6 +305,14 @@ private static async Task<string> GetTestFileNameAsync(int fileSize)
             return filePath;
         }
 
+        private class CustomHttpMessageHandler : HttpMessageHandler
+        {
+            protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+            {
+                throw new NotImplementedException("Deliberately not implemented for test purposes");
+            }
+        }
+
         [ClassCleanup]
         public static void CleanupCertificates()
         {

e2e/test/iothub/messaging/MessageSendE2ETests.cs

@@ -3,9 +3,12 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Net;
+using System.Net.Http;
 using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
 using FluentAssertions;
 using Microsoft.Azure.Devices.Client;
@@ -416,6 +419,31 @@ public async Task Message_DeviceSendMessageWayOverAllowedSize_Http()
             await SendSingleMessage(TestDeviceType.Sasl, Client.TransportType.Http1, OverlyExceedAllowedMessageSizeInBytes).ConfigureAwait(false);
         }
 
+        [TestMethod]
+        [Timeout(TestTimeoutMilliseconds)]
+        public async Task Message_DeviceSendSingleWithCustomHttpClient_Http()
+        {
+            using var httpMessageHandler = new CustomHttpMessageHandler();
+            var httpTransportSettings = new Http1TransportSettings()
+            {
+                HttpClient = new HttpClient(httpMessageHandler)
+            };
+            var transportSettings = new ITransportSettings[] { httpTransportSettings };
+
+            using TestDevice testDevice =
+                await TestDevice.GetTestDeviceAsync(_devicePrefix).ConfigureAwait(false);
+
+            using DeviceClient deviceClient = testDevice.CreateDeviceClient(transportSettings);
+
+            await deviceClient.OpenAsync().ConfigureAwait(false);
+            using var message = new Client.Message();
+            var ex = await Assert.ThrowsExceptionAsync<IotHubException>(
+                async () => await deviceClient.SendEventAsync(message).ConfigureAwait(false));
+
+            ex.InnerException.Should().BeOfType<NotImplementedException>(
+                "The provided custom HttpMessageHandler throws NotImplementedException when making any HTTP request");
+        }
+
         private async Task SendSingleMessage(TestDeviceType type, Client.TransportType transport, int messageSize = 0)
         {
             using TestDevice testDevice = await TestDevice.GetTestDeviceAsync(_devicePrefix, type).ConfigureAwait(false);
@@ -532,5 +560,13 @@ public static Client.Message ComposeD2cTestMessageOfSpecifiedSize(int messageSiz
 
             return message;
         }
+
+        private class CustomHttpMessageHandler : HttpMessageHandler
+        {
+            protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+            {
+                throw new NotImplementedException("Deliberately not implemented for test purposes");
+            }
+        }
     }
 }

iothub/device/src/ClientSettings/Http1TransportSettings.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Net;
+using System.Net.Http;
 using System.Security.Cryptography.X509Certificates;
 using Microsoft.Azure.Devices.Shared;
 
@@ -47,5 +48,20 @@ public TransportType GetTransportType()
 
         /// <inheritdoc/>
         public IWebProxy Proxy { get; set; }
+
+        /// <summary>
+        /// The HTTP client to use for all HTTP operations.
+        /// </summary>
+        /// <remarks>
+        /// If not provided, an HTTP client will be created for you based on the other settings provided.
+        /// <para>
+        /// This HTTP client instance will be disposed when the device/module client using it is disposed.
+        /// </para>
+        /// <para>
+        /// If provided, all other HTTP-specific settings (such as proxy, SSL protocols, and certificate revocation check)
+        /// on this class will be ignored and must be specified on this HttpClient instance.
+        /// </para>
+        /// </remarks>
+        public HttpClient HttpClient { get; set; }
     }
 }

iothub/device/src/ModuleClient.cs

@@ -36,6 +36,7 @@ public class ModuleClient : IDisposable
         private const string DeviceMethodUriFormat = "/twins/{0}/methods?" + ClientApiVersionHelper.ApiVersionQueryStringLatest;
         private readonly bool _isAnEdgeModule;
         private readonly ICertificateValidator _certValidator;
+        private HttpTransportHandler _httpTransportHandler;
 
         internal InternalClient InternalClient { get; private set; }
 
@@ -514,6 +515,7 @@ public void Dispose()
         {
             Dispose(true);
             GC.SuppressFinalize(this);
+            _httpTransportHandler?.Dispose();
         }
 
 #if !NET451 && !NET472 && !NETSTANDARD2_0
@@ -827,57 +829,66 @@ private async Task<MethodResponse> InvokeMethodAsync(Uri uri, MethodRequest meth
             {
                 throw new ObjectDisposedException("IoT client", DefaultDelegatingHandler.ClientDisposedMessage);
             }
-            methodRequest.ThrowIfNull(nameof(methodRequest));
 
-            HttpClientHandler httpClientHandler = null;
-            Func<object, X509Certificate, X509Chain, SslPolicyErrors, bool> customCertificateValidation = _certValidator.GetCustomCertificateValidation();
+            methodRequest.ThrowIfNull(nameof(methodRequest));
 
-            try
+            if (_httpTransportHandler == null)
             {
+                Func<object, X509Certificate, X509Chain, SslPolicyErrors, bool> customCertificateValidation = _certValidator.GetCustomCertificateValidation();
+
+                // The HTTP message handlers created in this block are disposed when this client is
+                // disposed.
+#pragma warning disable CA2000 // Dispose objects before losing scope
+#if !NET451
+                var httpMessageHandler = new HttpClientHandler();
+#else
+                var httpMessageHandler = new WebRequestHandler();
+#endif
+#pragma warning restore CA2000 // Dispose objects before losing scope
+
                 if (customCertificateValidation != null)
                 {
                     TlsVersions.Instance.SetLegacyAcceptableVersions();
 
 #if !NET451
-                    httpClientHandler = new HttpClientHandler
-                    {
-                        ServerCertificateCustomValidationCallback = customCertificateValidation,
-                        SslProtocols = TlsVersions.Instance.Preferred,
-                    };
+                    httpMessageHandler.ServerCertificateCustomValidationCallback = customCertificateValidation;
+                    httpMessageHandler.SslProtocols = TlsVersions.Instance.Preferred;
 #else
-                    httpClientHandler = new WebRequestHandler();
-                    ((WebRequestHandler)httpClientHandler).ServerCertificateValidationCallback = (sender, certificate, chain, errors) =>
+                    httpMessageHandler.ServerCertificateValidationCallback = (sender, certificate, chain, errors) =>
                     {
                         return customCertificateValidation(sender, certificate, chain, errors);
                     };
 #endif
                 }
 
-                var context = new PipelineContext()
+                // We need to add the certificate to the httpTransport if DeviceAuthenticationWithX509Certificate
+                if (InternalClient.Certificate != null)
+                {
+                    httpMessageHandler.ClientCertificates.Add(InternalClient.Certificate);
+                }
+
+                // Note that this client is ignoring any HttpTransportSettings that the user may have
+                // provided. This is because the kinds of settings a user would want to override
+                // aren't as applicable for this particular operation.
+                var transportSettings = new Http1TransportSettings()
+                {
+                    HttpClient = new HttpClient(httpMessageHandler, true)
+                };
+
+                var context = new PipelineContext
                 {
                     ProductInfo = new ProductInfo
                     {
                         Extra = InternalClient.ProductInfo
                     }
                 };
 
-                var transportSettings = new Http1TransportSettings();
-                //We need to add the certificate to the httpTransport if DeviceAuthenticationWithX509Certificate
-                if (InternalClient.Certificate != null)
-                {
-                    transportSettings.ClientCertificate = InternalClient.Certificate;
-                }
-
-                using var httpTransport = new HttpTransportHandler(context, InternalClient.IotHubConnectionString, transportSettings, httpClientHandler);
-                var methodInvokeRequest = new MethodInvokeRequest(methodRequest.Name, methodRequest.DataAsJson, methodRequest.ResponseTimeout, methodRequest.ConnectionTimeout);
-                MethodInvokeResponse result = await httpTransport.InvokeMethodAsync(methodInvokeRequest, uri, cancellationToken).ConfigureAwait(false);
-
-                return new MethodResponse(Encoding.UTF8.GetBytes(result.GetPayloadAsJson()), result.Status);
-            }
-            finally
-            {
-                httpClientHandler?.Dispose();
+                _httpTransportHandler = new HttpTransportHandler(context, InternalClient.IotHubConnectionString, transportSettings);
             }
+
+            var methodInvokeRequest = new MethodInvokeRequest(methodRequest.Name, methodRequest.DataAsJson, methodRequest.ResponseTimeout, methodRequest.ConnectionTimeout);
+            MethodInvokeResponse result = await _httpTransportHandler.InvokeMethodAsync(methodInvokeRequest, uri, cancellationToken).ConfigureAwait(false);
+            return new MethodResponse(Encoding.UTF8.GetBytes(result.GetPayloadAsJson()), result.Status);
         }
 
         private static Uri GetDeviceMethodUri(string deviceId)