From d418b3fd577b9bf1fd6ac8ccc54cba08b6e08a87 Mon Sep 17 00:00:00 2001 From: Ben Bartholomew <70723971+ben-bartholomew@users.noreply.github.com> Date: Tue, 16 Jul 2024 11:52:14 -0400 Subject: [PATCH 1/8] #2084 Apply default config file paths in `GetMergedOcelotJson` when providing the `folder` argument of `AddOcelot` (#2120) * Adding unit test first * Fixing default global config file not being found in folder * Adding PR trait to test * Backing out whitespace changes * Code review by @raman-m * Create Configuration feature folder and move test classes * Adjust namespace and review what we have * Acceptance tests for #2084 user scenario --------- Co-authored-by: Raman Maksimchuk --- .../ConfigurationBuilderExtensions.cs | 7 +- .../ConfigurationInConsulTests.cs | 346 +++++++++--------- .../Configuration/ConfigurationMergeTests.cs | 117 ++++++ .../ConfigurationReloadTests.cs | 2 +- .../ConfigurationMergeTests.cs | 68 ---- test/Ocelot.AcceptanceTests/Steps.cs | 50 ++- .../ConfigurationBuilderExtensionsTests.cs | 37 +- 7 files changed, 364 insertions(+), 263 deletions(-) rename test/Ocelot.AcceptanceTests/{ => Configuration}/ConfigurationInConsulTests.cs (97%) create mode 100644 test/Ocelot.AcceptanceTests/Configuration/ConfigurationMergeTests.cs rename test/Ocelot.AcceptanceTests/{ => Configuration}/ConfigurationReloadTests.cs (98%) delete mode 100644 test/Ocelot.AcceptanceTests/ConfigurationMergeTests.cs diff --git a/src/Ocelot/DependencyInjection/ConfigurationBuilderExtensions.cs b/src/Ocelot/DependencyInjection/ConfigurationBuilderExtensions.cs index 214e6e0c5..9101f5b52 100644 --- a/src/Ocelot/DependencyInjection/ConfigurationBuilderExtensions.cs +++ b/src/Ocelot/DependencyInjection/ConfigurationBuilderExtensions.cs @@ -91,6 +91,7 @@ public static IConfigurationBuilder AddOcelot(this IConfigurationBuilder builder string primaryConfigFile = null, string globalConfigFile = null, string environmentConfigFile = null, bool? optional = null, bool? reloadOnChange = null) // optional injections { var json = GetMergedOcelotJson(folder, env, null, primaryConfigFile, globalConfigFile, environmentConfigFile); + primaryConfigFile ??= Path.Join(folder, PrimaryConfigFile); // if not specified, merge & write back to the same folder return ApplyMergeOcelotJsonOption(builder, mergeTo, json, primaryConfigFile, optional, reloadOnChange); } @@ -106,7 +107,7 @@ private static string GetMergedOcelotJson(string folder, IWebHostEnvironment env FileConfiguration fileConfiguration = null, string primaryFile = null, string globalFile = null, string environmentFile = null) { var envName = string.IsNullOrEmpty(env?.EnvironmentName) ? "Development" : env.EnvironmentName; - environmentFile ??= string.Format(EnvironmentConfigFile, envName); + environmentFile ??= Path.Join(folder, string.Format(EnvironmentConfigFile, envName)); var reg = SubConfigRegex(); var environmentFileInfo = new FileInfo(environmentFile); var files = new DirectoryInfo(folder) @@ -117,8 +118,8 @@ private static string GetMergedOcelotJson(string folder, IWebHostEnvironment env .ToArray(); fileConfiguration ??= new FileConfiguration(); - primaryFile ??= PrimaryConfigFile; - globalFile ??= GlobalConfigFile; + primaryFile ??= Path.Join(folder, PrimaryConfigFile); + globalFile ??= Path.Join(folder, GlobalConfigFile); var primaryFileInfo = new FileInfo(primaryFile); var globalFileInfo = new FileInfo(globalFile); foreach (var file in files) diff --git a/test/Ocelot.AcceptanceTests/ConfigurationInConsulTests.cs b/test/Ocelot.AcceptanceTests/Configuration/ConfigurationInConsulTests.cs similarity index 97% rename from test/Ocelot.AcceptanceTests/ConfigurationInConsulTests.cs rename to test/Ocelot.AcceptanceTests/Configuration/ConfigurationInConsulTests.cs index c889f353e..ca2da9360 100644 --- a/test/Ocelot.AcceptanceTests/ConfigurationInConsulTests.cs +++ b/test/Ocelot.AcceptanceTests/Configuration/ConfigurationInConsulTests.cs @@ -7,176 +7,176 @@ using Newtonsoft.Json; using Ocelot.Configuration.File; using System.Text; - -namespace Ocelot.AcceptanceTests -{ - public class ConfigurationInConsulTests : IDisposable - { - private IHost _builder; - private readonly Steps _steps; - private IHost _fakeConsulBuilder; - private FileConfiguration _config; - private readonly List _consulServices; - - public ConfigurationInConsulTests() - { - _consulServices = new List(); - _steps = new Steps(); - } - - [Fact] - public void should_return_response_200_with_simple_url_when_using_jsonserialized_cache() - { - var consulPort = PortFinder.GetRandomPort(); - var servicePort = PortFinder.GetRandomPort(); - - var configuration = new FileConfiguration - { - Routes = new List - { - new() - { - DownstreamPathTemplate = "/", - DownstreamScheme = "http", - DownstreamHostAndPorts = new List - { - new() - { - Host = "localhost", - Port = servicePort, - }, - }, - UpstreamPathTemplate = "/", - UpstreamHttpMethod = new List { "Get" }, - }, - }, - GlobalConfiguration = new FileGlobalConfiguration - { - ServiceDiscoveryProvider = new FileServiceDiscoveryProvider - { - Scheme = "http", - Host = "localhost", - Port = consulPort, - }, - }, - }; - - var fakeConsulServiceDiscoveryUrl = $"http://localhost:{consulPort}"; - - this.Given(x => GivenThereIsAFakeConsulServiceDiscoveryProvider(fakeConsulServiceDiscoveryUrl, string.Empty)) - .And(x => x.GivenThereIsAServiceRunningOn($"http://localhost:{servicePort}", string.Empty, 200, "Hello from Laura")) - .And(x => _steps.GivenThereIsAConfiguration(configuration)) - .And(x => _steps.GivenOcelotIsRunningUsingConsulToStoreConfigAndJsonSerializedCache()) - .When(x => _steps.WhenIGetUrlOnTheApiGateway("/")) - .Then(x => _steps.ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) - .And(x => _steps.ThenTheResponseBodyShouldBe("Hello from Laura")) - .BDDfy(); - } - - private void GivenThereIsAFakeConsulServiceDiscoveryProvider(string url, string serviceName) - { - _fakeConsulBuilder = Host.CreateDefaultBuilder() - .ConfigureWebHost(webBuilder => - { - webBuilder.UseUrls(url) - .UseKestrel() - .UseContentRoot(Directory.GetCurrentDirectory()) - .UseIISIntegration() - .UseUrls(url) - .Configure(app => - { - app.Run(async context => - { - if (context.Request.Method.ToLower() == "get" && context.Request.Path.Value == "/v1/kv/InternalConfiguration") - { - var json = JsonConvert.SerializeObject(_config); - - var bytes = Encoding.UTF8.GetBytes(json); - - var base64 = Convert.ToBase64String(bytes); - - var kvp = new FakeConsulGetResponse(base64); - - await context.Response.WriteJsonAsync(new[] { kvp }); - } - else if (context.Request.Method.ToLower() == "put" && context.Request.Path.Value == "/v1/kv/InternalConfiguration") - { - try - { - var reader = new StreamReader(context.Request.Body); - - // Synchronous operations are disallowed. Call ReadAsync or set AllowSynchronousIO to true instead. - // var json = reader.ReadToEnd(); - var json = await reader.ReadToEndAsync(); - - _config = JsonConvert.DeserializeObject(json); - - var response = JsonConvert.SerializeObject(true); - - await context.Response.WriteAsync(response); - } - catch (Exception e) - { - Console.WriteLine(e); - throw; - } - } - else if (context.Request.Path.Value == $"/v1/health/service/{serviceName}") - { - await context.Response.WriteJsonAsync(_consulServices); - } - }); - }); - }).Build(); - - _fakeConsulBuilder.Start(); - } - - public class FakeConsulGetResponse - { - public FakeConsulGetResponse(string value) - { - Value = value; - } - - public int CreateIndex => 100; - public int ModifyIndex => 200; - public int LockIndex => 200; - public string Key => "InternalConfiguration"; - public int Flags => 0; - public string Value { get; } - public string Session => "adf4238a-882b-9ddc-4a9d-5b6758e4159e"; - } - - private void GivenThereIsAServiceRunningOn(string url, string basePath, int statusCode, string responseBody) - { - _builder = Host.CreateDefaultBuilder() - .ConfigureWebHost(webBuilder => - { - webBuilder.UseUrls(url) - .UseKestrel() - .UseContentRoot(Directory.GetCurrentDirectory()) - .UseIISIntegration() - .UseUrls(url) - .Configure(app => - { - app.UsePathBase(basePath); - app.Run(async context => - { - context.Response.StatusCode = statusCode; - await context.Response.WriteAsync(responseBody); - }); - }); - }) - .Build(); - - _builder.Start(); - } - - public void Dispose() - { - _builder?.Dispose(); - _steps.Dispose(); - } - } -} + +namespace Ocelot.AcceptanceTests.Configuration +{ + public class ConfigurationInConsulTests : IDisposable + { + private IHost _builder; + private readonly Steps _steps; + private IHost _fakeConsulBuilder; + private FileConfiguration _config; + private readonly List _consulServices; + + public ConfigurationInConsulTests() + { + _consulServices = new List(); + _steps = new Steps(); + } + + [Fact] + public void should_return_response_200_with_simple_url_when_using_jsonserialized_cache() + { + var consulPort = PortFinder.GetRandomPort(); + var servicePort = PortFinder.GetRandomPort(); + + var configuration = new FileConfiguration + { + Routes = new List + { + new() + { + DownstreamPathTemplate = "/", + DownstreamScheme = "http", + DownstreamHostAndPorts = new List + { + new() + { + Host = "localhost", + Port = servicePort, + }, + }, + UpstreamPathTemplate = "/", + UpstreamHttpMethod = new List { "Get" }, + }, + }, + GlobalConfiguration = new FileGlobalConfiguration + { + ServiceDiscoveryProvider = new FileServiceDiscoveryProvider + { + Scheme = "http", + Host = "localhost", + Port = consulPort, + }, + }, + }; + + var fakeConsulServiceDiscoveryUrl = $"http://localhost:{consulPort}"; + + this.Given(x => GivenThereIsAFakeConsulServiceDiscoveryProvider(fakeConsulServiceDiscoveryUrl, string.Empty)) + .And(x => x.GivenThereIsAServiceRunningOn($"http://localhost:{servicePort}", string.Empty, 200, "Hello from Laura")) + .And(x => _steps.GivenThereIsAConfiguration(configuration)) + .And(x => _steps.GivenOcelotIsRunningUsingConsulToStoreConfigAndJsonSerializedCache()) + .When(x => _steps.WhenIGetUrlOnTheApiGateway("/")) + .Then(x => _steps.ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) + .And(x => _steps.ThenTheResponseBodyShouldBe("Hello from Laura")) + .BDDfy(); + } + + private void GivenThereIsAFakeConsulServiceDiscoveryProvider(string url, string serviceName) + { + _fakeConsulBuilder = Host.CreateDefaultBuilder() + .ConfigureWebHost(webBuilder => + { + webBuilder.UseUrls(url) + .UseKestrel() + .UseContentRoot(Directory.GetCurrentDirectory()) + .UseIISIntegration() + .UseUrls(url) + .Configure(app => + { + app.Run(async context => + { + if (context.Request.Method.ToLower() == "get" && context.Request.Path.Value == "/v1/kv/InternalConfiguration") + { + var json = JsonConvert.SerializeObject(_config); + + var bytes = Encoding.UTF8.GetBytes(json); + + var base64 = Convert.ToBase64String(bytes); + + var kvp = new FakeConsulGetResponse(base64); + + await context.Response.WriteJsonAsync(new[] { kvp }); + } + else if (context.Request.Method.ToLower() == "put" && context.Request.Path.Value == "/v1/kv/InternalConfiguration") + { + try + { + var reader = new StreamReader(context.Request.Body); + + // Synchronous operations are disallowed. Call ReadAsync or set AllowSynchronousIO to true instead. + // var json = reader.ReadToEnd(); + var json = await reader.ReadToEndAsync(); + + _config = JsonConvert.DeserializeObject(json); + + var response = JsonConvert.SerializeObject(true); + + await context.Response.WriteAsync(response); + } + catch (Exception e) + { + Console.WriteLine(e); + throw; + } + } + else if (context.Request.Path.Value == $"/v1/health/service/{serviceName}") + { + await context.Response.WriteJsonAsync(_consulServices); + } + }); + }); + }).Build(); + + _fakeConsulBuilder.Start(); + } + + public class FakeConsulGetResponse + { + public FakeConsulGetResponse(string value) + { + Value = value; + } + + public int CreateIndex => 100; + public int ModifyIndex => 200; + public int LockIndex => 200; + public string Key => "InternalConfiguration"; + public int Flags => 0; + public string Value { get; } + public string Session => "adf4238a-882b-9ddc-4a9d-5b6758e4159e"; + } + + private void GivenThereIsAServiceRunningOn(string url, string basePath, int statusCode, string responseBody) + { + _builder = Host.CreateDefaultBuilder() + .ConfigureWebHost(webBuilder => + { + webBuilder.UseUrls(url) + .UseKestrel() + .UseContentRoot(Directory.GetCurrentDirectory()) + .UseIISIntegration() + .UseUrls(url) + .Configure(app => + { + app.UsePathBase(basePath); + app.Run(async context => + { + context.Response.StatusCode = statusCode; + await context.Response.WriteAsync(responseBody); + }); + }); + }) + .Build(); + + _builder.Start(); + } + + public void Dispose() + { + _builder?.Dispose(); + _steps.Dispose(); + } + } +} diff --git a/test/Ocelot.AcceptanceTests/Configuration/ConfigurationMergeTests.cs b/test/Ocelot.AcceptanceTests/Configuration/ConfigurationMergeTests.cs new file mode 100644 index 000000000..7648abf25 --- /dev/null +++ b/test/Ocelot.AcceptanceTests/Configuration/ConfigurationMergeTests.cs @@ -0,0 +1,117 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Ocelot.Configuration.File; +using Ocelot.Configuration.Repository; +using Ocelot.DependencyInjection; +using System.Runtime.CompilerServices; + +namespace Ocelot.AcceptanceTests.Configuration; + +public sealed class ConfigurationMergeTests : Steps +{ + private readonly FileConfiguration _initialGlobalConfig; + private readonly string _globalConfigFileName; + + public ConfigurationMergeTests() : base() + { + _initialGlobalConfig = new(); + _globalConfigFileName = $"{TestID}-{ConfigurationBuilderExtensions.GlobalConfigFile}"; + Files.Add(_globalConfigFileName); + } + + [Theory] + [Trait("Bug", "1216")] + [Trait("Feat", "1227")] + [InlineData(MergeOcelotJson.ToFile, true)] + [InlineData(MergeOcelotJson.ToMemory, false)] + public void ShouldRunWithGlobalConfigMerged_WithExplicitGlobalConfigFileParameter(MergeOcelotJson where, bool fileExist) + { + Arrange(); + + // Act + StartOcelot((context, config) => config + .AddOcelot(_initialGlobalConfig, context.HostingEnvironment, where, _ocelotConfigFileName, _globalConfigFileName, null, false, false)); + + // Assert + TheOcelotPrimaryConfigFileExists(fileExist); + ThenGlobalConfigurationHasBeenMerged(); + } + + [Theory] + [Trait("Bug", "2084")] + [InlineData(MergeOcelotJson.ToFile, true)] + [InlineData(MergeOcelotJson.ToMemory, false)] + public void ShouldRunWithGlobalConfigMerged_WithImplicitGlobalConfigFileParameter(MergeOcelotJson where, bool fileExist) + { + Arrange(); + var globalConfig = _initialGlobalConfig; + globalConfig.Routes.Clear(); + var routeAConfig = GivenConfiguration(GetRoute("A")); + var routeBConfig = GivenConfiguration(GetRoute("B")); + var environmentConfig = GivenConfiguration(GetRoute("Env")); + environmentConfig.GlobalConfiguration = null; + var folder = "GatewayConfiguration-" + TestID; + Folders.Add(Directory.CreateDirectory(folder).FullName); + var globalPath = Path.Combine(folder, ConfigurationBuilderExtensions.GlobalConfigFile); + var routeAPath = Path.Combine(folder, string.Format(ConfigurationBuilderExtensions.EnvironmentConfigFile, "A")); + var routeBPath = Path.Combine(folder, string.Format(ConfigurationBuilderExtensions.EnvironmentConfigFile, "B")); + var environmentPath = Path.Combine(folder, string.Format(ConfigurationBuilderExtensions.EnvironmentConfigFile, "Env")); + GivenThereIsAConfiguration(globalConfig, globalPath); + GivenThereIsAConfiguration(routeAConfig, routeAPath); + GivenThereIsAConfiguration(routeBConfig, routeBPath); + GivenThereIsAConfiguration(environmentConfig, environmentPath); + + // Act + StartOcelot((context, config) => config + .AddOcelot(folder, context.HostingEnvironment, where) // overloaded version from the user's scenario + .AddJsonFile(environmentPath), + "Env"); + + // Assert + TheOcelotPrimaryConfigFileExists(false); + ThenGlobalConfigurationHasBeenMerged(); + + var actualLocation = Path.Combine(folder, ConfigurationBuilderExtensions.PrimaryConfigFile); + File.Exists(actualLocation).ShouldBe(fileExist); + + var repository = _ocelotServer.Services.GetService().ShouldNotBeNull(); + var response = repository.Get().ShouldNotBeNull(); + response.IsError.ShouldBeFalse(); + var internalConfig = response.Data.ShouldNotBeNull(); + + // Assert Arrange() setup + internalConfig.RequestId.ShouldBe(nameof(ShouldRunWithGlobalConfigMerged_WithImplicitGlobalConfigFileParameter)); + internalConfig.ServiceProviderConfiguration.ConfigurationKey.ShouldBe(nameof(ShouldRunWithGlobalConfigMerged_WithImplicitGlobalConfigFileParameter)); + } + + private void Arrange([CallerMemberName] string testName = null) + { + _initialGlobalConfig.GlobalConfiguration.RequestIdKey = testName; + _initialGlobalConfig.GlobalConfiguration.ServiceDiscoveryProvider.ConfigurationKey = testName; + } + + private void TheOcelotPrimaryConfigFileExists(bool expected) + => File.Exists(_ocelotConfigFileName).ShouldBe(expected); + + private void ThenGlobalConfigurationHasBeenMerged([CallerMemberName] string testName = null) + { + var config = _ocelotServer.Services.GetService().ShouldNotBeNull(); + var actual = config["GlobalConfiguration:RequestIdKey"]; + actual.ShouldNotBeNull().ShouldBe(testName); + actual = config["GlobalConfiguration:ServiceDiscoveryProvider:ConfigurationKey"]; + actual.ShouldNotBeNull().ShouldBe(testName); + } + + private static FileRoute GetRoute(string suffix, [CallerMemberName] string testName = null) => new() + { + DownstreamScheme = nameof(FileRoute.DownstreamScheme) + suffix, + DownstreamPathTemplate = "/" + suffix, + Key = testName + suffix, + UpstreamPathTemplate = "/" + suffix, + UpstreamHttpMethod = new() { nameof(FileRoute.UpstreamHttpMethod) + suffix }, + DownstreamHostAndPorts = new() + { + new(nameof(FileHostAndPort.Host) + suffix, 80), + }, + }; +} diff --git a/test/Ocelot.AcceptanceTests/ConfigurationReloadTests.cs b/test/Ocelot.AcceptanceTests/Configuration/ConfigurationReloadTests.cs similarity index 98% rename from test/Ocelot.AcceptanceTests/ConfigurationReloadTests.cs rename to test/Ocelot.AcceptanceTests/Configuration/ConfigurationReloadTests.cs index 07d8cd546..7d9037fc2 100644 --- a/test/Ocelot.AcceptanceTests/ConfigurationReloadTests.cs +++ b/test/Ocelot.AcceptanceTests/Configuration/ConfigurationReloadTests.cs @@ -1,7 +1,7 @@ using Ocelot.Configuration.ChangeTracking; using Ocelot.Configuration.File; -namespace Ocelot.AcceptanceTests +namespace Ocelot.AcceptanceTests.Configuration { [Collection(nameof(SequentialTests))] public sealed class ConfigurationReloadTests : IDisposable diff --git a/test/Ocelot.AcceptanceTests/ConfigurationMergeTests.cs b/test/Ocelot.AcceptanceTests/ConfigurationMergeTests.cs deleted file mode 100644 index 3836dcbf6..000000000 --- a/test/Ocelot.AcceptanceTests/ConfigurationMergeTests.cs +++ /dev/null @@ -1,68 +0,0 @@ -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Ocelot.Configuration.File; -using Ocelot.DependencyInjection; -using System.Runtime.CompilerServices; - -namespace Ocelot.AcceptanceTests; - -[Trait("PR", "1227")] -[Trait("Issue", "1216")] -public sealed class ConfigurationMergeTests : Steps -{ - private readonly FileConfiguration _globalConfig; - private readonly string _globalConfigFileName; - - public ConfigurationMergeTests() : base() - { - _globalConfig = new(); - _globalConfigFileName = $"{TestID}-{ConfigurationBuilderExtensions.GlobalConfigFile}"; - } - - protected override void DeleteOcelotConfig(params string[] files) => base.DeleteOcelotConfig(_globalConfigFileName); - - [Fact] - public void Should_run_with_global_config_merged_to_memory() - { - Arrange(); - - // Act - GivenOcelotIsRunningMergedConfig(MergeOcelotJson.ToMemory); - - // Assert - TheOcelotPrimaryConfigFileExists(false); - Assert(); - } - - [Fact] - public void Should_run_with_global_config_merged_to_file() - { - Arrange(); - - // Act - GivenOcelotIsRunningMergedConfig(MergeOcelotJson.ToFile); - - // Assert - TheOcelotPrimaryConfigFileExists(true); - Assert(); - } - - private void GivenOcelotIsRunningMergedConfig(MergeOcelotJson mergeTo) - => StartOcelot((context, config) => config.AddOcelot(_globalConfig, context.HostingEnvironment, mergeTo, _ocelotConfigFileName, _globalConfigFileName, null, false, false)); - - private void TheOcelotPrimaryConfigFileExists(bool expected) - => File.Exists(_ocelotConfigFileName).ShouldBe(expected); - - private void Arrange([CallerMemberName] string testName = null) - { - _globalConfig.GlobalConfiguration.RequestIdKey = testName; - } - - private void Assert([CallerMemberName] string testName = null) - { - var config = _ocelotServer.Services.GetService(); - config.ShouldNotBeNull(); - var actual = config["GlobalConfiguration:RequestIdKey"]; - actual.ShouldNotBeNull().ShouldBe(testName); - } -} diff --git a/test/Ocelot.AcceptanceTests/Steps.cs b/test/Ocelot.AcceptanceTests/Steps.cs index 9b1e4b927..4591e9b22 100644 --- a/test/Ocelot.AcceptanceTests/Steps.cs +++ b/test/Ocelot.AcceptanceTests/Steps.cs @@ -58,9 +58,13 @@ public Steps() { _random = new Random(); _testId = Guid.NewGuid(); - _ocelotConfigFileName = $"{_testId:N}-{ConfigurationBuilderExtensions.PrimaryConfigFile}"; + _ocelotConfigFileName = $"{_testId:N}-{ConfigurationBuilderExtensions.PrimaryConfigFile}"; + Files = new() { _ocelotConfigFileName }; + Folders = new(); } + protected List Files { get; } + protected List Folders { get; } protected string TestID { get => _testId.ToString("N"); } protected static string DownstreamUrl(int port) => $"{Uri.UriSchemeHttp}://localhost:{port}"; @@ -167,16 +171,20 @@ public async Task StartFakeOcelotWithWebSocketsWithConsul() await _ocelotHost.StartAsync(); } - public void GivenThereIsAConfiguration(FileConfiguration fileConfiguration) - { - var jsonConfiguration = JsonConvert.SerializeObject(fileConfiguration, Formatting.Indented); - File.WriteAllText(_ocelotConfigFileName, jsonConfiguration); + public void GivenThereIsAConfiguration(FileConfiguration fileConfiguration) + => GivenThereIsAConfiguration(fileConfiguration, _ocelotConfigFileName); + + public void GivenThereIsAConfiguration(FileConfiguration from, string toFile) + { + toFile ??= _ocelotConfigFileName; + var jsonConfiguration = JsonConvert.SerializeObject(from, Formatting.Indented); + File.WriteAllText(toFile, jsonConfiguration); + Files.Add(toFile); // register for disposing } - protected virtual void DeleteOcelotConfig(params string[] files) + protected virtual void DeleteFiles() { - var allFiles = files.Append(_ocelotConfigFileName); - foreach (var file in allFiles) + foreach (var file in Files) { if (!File.Exists(file)) { @@ -193,6 +201,25 @@ protected virtual void DeleteOcelotConfig(params string[] files) } } } + + protected virtual void DeleteFolders() + { + foreach (var folder in Folders) + { + try + { + var f = new DirectoryInfo(folder); + if (f.Exists && f.FullName != AppContext.BaseDirectory) + { + f.Delete(true); + } + } + catch (Exception e) + { + Console.WriteLine(e); + } + } + } public void ThenTheResponseBodyHeaderIs(string key, string value) { @@ -218,7 +245,7 @@ public void GivenOcelotIsRunning() StartOcelot((_, config) => config.AddJsonFile(_ocelotConfigFileName, false, false)); } - protected void StartOcelot(Action configureAddOcelot) + protected void StartOcelot(Action configureAddOcelot, string environmentName = null) { _webHostBuilder = new WebHostBuilder(); @@ -234,7 +261,7 @@ protected void StartOcelot(Action }) .ConfigureServices(WithAddOcelot) .Configure(WithUseOcelot) - .UseEnvironment(nameof(AcceptanceTests)); + .UseEnvironment(environmentName ?? nameof(AcceptanceTests)); _ocelotServer = new TestServer(_webHostBuilder); _ocelotClient = _ocelotServer.CreateClient(); @@ -1166,7 +1193,8 @@ protected virtual void Dispose(bool disposing) _ocelotClient?.Dispose(); _ocelotServer?.Dispose(); _ocelotHost?.Dispose(); - DeleteOcelotConfig(); + DeleteFiles(); + DeleteFolders(); } _disposedValue = true; diff --git a/test/Ocelot.UnitTests/DependencyInjection/ConfigurationBuilderExtensionsTests.cs b/test/Ocelot.UnitTests/DependencyInjection/ConfigurationBuilderExtensionsTests.cs index f0e322fa8..caa8dfefd 100644 --- a/test/Ocelot.UnitTests/DependencyInjection/ConfigurationBuilderExtensionsTests.cs +++ b/test/Ocelot.UnitTests/DependencyInjection/ConfigurationBuilderExtensionsTests.cs @@ -139,19 +139,34 @@ public void Should_merge_files_with_null_environment() TheOcelotPrimaryConfigFileExists(false); } - private void GivenCombinedFileConfigurationObject() + [Fact] + [Trait("Bug", "2084")] + public void Should_use_relative_path_for_global_config() + { + // Arrange + GivenMultipleConfigurationFiles(TestID); + + // Act + WhenIAddOcelotConfigurationWithDefaultFilePaths(TestID); + + // Assert + var config = ThenTheConfigsAreMergedAndAddedInApplicationConfiguration(false); + config.ShouldNotBeNull().GlobalConfiguration.RequestIdKey.ShouldBe(nameof(Should_use_relative_path_for_global_config)); + } + + private void GivenCombinedFileConfigurationObject([CallerMemberName] string testName = null) { _combinedFileConfiguration = new FileConfiguration { - GlobalConfiguration = GetFileGlobalConfigurationData(), + GlobalConfiguration = GetFileGlobalConfigurationData(testName), Routes = GetServiceARoutes().Concat(GetServiceBRoutes()).Concat(GetEnvironmentSpecificRoutes()).ToList(), Aggregates = GetFileAggregatesRouteData(), }; } - private void GivenMultipleConfigurationFiles(string folder, bool withEnvironment = false) + private void GivenMultipleConfigurationFiles(string folder, bool withEnvironment = false, [CallerMemberName] string testName = null) { - _globalConfig = new() { GlobalConfiguration = GetFileGlobalConfigurationData() }; + _globalConfig = new() { GlobalConfiguration = GetFileGlobalConfigurationData(testName) }; _routeA = new() { Routes = GetServiceARoutes() }; _routeB = new() { Routes = GetServiceBRoutes() }; _aggregate = new() { Aggregates = GetFileAggregatesRouteData() }; @@ -178,7 +193,7 @@ private void GivenMultipleConfigurationFiles(string folder, bool withEnvironment } } - private static FileGlobalConfiguration GetFileGlobalConfigurationData() => new() + private static FileGlobalConfiguration GetFileGlobalConfigurationData(string requestIdKey = null) => new() { BaseUrl = "BaseUrl", RateLimitOptions = new() @@ -196,7 +211,7 @@ private void GivenMultipleConfigurationFiles(string folder, bool withEnvironment Port = 80, Type = "Type", }, - RequestIdKey = "RequestIdKey", + RequestIdKey = requestIdKey ?? "RequestIdKey", }; private static List GetFileAggregatesRouteData() => new() @@ -246,7 +261,14 @@ private void WhenIAddOcelotConfiguration(string folder, MergeOcelotJson mergeOce .Build(); } - private void ThenTheConfigsAreMergedAndAddedInApplicationConfiguration(bool useCombinedConfig) + private void WhenIAddOcelotConfigurationWithDefaultFilePaths(string folder, MergeOcelotJson mergeOcelotJson = MergeOcelotJson.ToFile) + { + _configRoot = new ConfigurationBuilder() + .AddOcelot(folder, _hostingEnvironment.Object, mergeOcelotJson, optional: false, reloadOnChange: false) + .Build(); + } + + private FileConfiguration ThenTheConfigsAreMergedAndAddedInApplicationConfiguration(bool useCombinedConfig) { var fc = (FileConfiguration)_configRoot.Get(typeof(FileConfiguration)); @@ -281,6 +303,7 @@ private void ThenTheConfigsAreMergedAndAddedInApplicationConfiguration(bool useC fc.Routes.ShouldContain(x => x.UpstreamHost == (useCombinedConfig ? _combinedFileConfiguration.Routes[2].UpstreamHost : _routeB.Routes[1].UpstreamHost)); fc.Aggregates.Count.ShouldBe(useCombinedConfig ? _combinedFileConfiguration.Aggregates.Count :_aggregate.Aggregates.Count); + return fc; } private void NotContainsEnvSpecificConfig() From b4e21c47106acc8577862203356c3e33280a2985 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 16:48:55 +0300 Subject: [PATCH 2/8] Bump Steeltoe.Discovery.Eureka from 3.2.5 to 3.2.8 in /src/Ocelot.Provider.Eureka (#2122) * Bump Steeltoe.Discovery.Eureka in /src/Ocelot.Provider.Eureka Bumps [Steeltoe.Discovery.Eureka](https://github.com/SteeltoeOSS/Steeltoe) from 3.2.5 to 3.2.8. - [Release notes](https://github.com/SteeltoeOSS/Steeltoe/releases) - [Changelog](https://github.com/SteeltoeOSS/Steeltoe/blob/main/Steeltoe.Release.ruleset) - [Commits](https://github.com/SteeltoeOSS/Steeltoe/compare/3.2.5...3.2.8) --- updated-dependencies: - dependency-name: Steeltoe.Discovery.Eureka dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Bump Steeltoe.Discovery.ClientCore from 3.2.5 to 3.2.8 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Raman Maksimchuk --- src/Ocelot.Provider.Eureka/Ocelot.Provider.Eureka.csproj | 4 ++-- test/Ocelot.AcceptanceTests/Ocelot.AcceptanceTests.csproj | 2 +- test/Ocelot.UnitTests/Ocelot.UnitTests.csproj | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Ocelot.Provider.Eureka/Ocelot.Provider.Eureka.csproj b/src/Ocelot.Provider.Eureka/Ocelot.Provider.Eureka.csproj index b7d578440..826126199 100644 --- a/src/Ocelot.Provider.Eureka/Ocelot.Provider.Eureka.csproj +++ b/src/Ocelot.Provider.Eureka/Ocelot.Provider.Eureka.csproj @@ -31,8 +31,8 @@ - - + + all diff --git a/test/Ocelot.AcceptanceTests/Ocelot.AcceptanceTests.csproj b/test/Ocelot.AcceptanceTests/Ocelot.AcceptanceTests.csproj index c997d07c5..c2b3e01bf 100644 --- a/test/Ocelot.AcceptanceTests/Ocelot.AcceptanceTests.csproj +++ b/test/Ocelot.AcceptanceTests/Ocelot.AcceptanceTests.csproj @@ -69,7 +69,7 @@ - + diff --git a/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj b/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj index 5cab0b487..bb3a32301 100644 --- a/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj +++ b/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj @@ -70,7 +70,7 @@ - + From 19a8e2f8b3e773fbe962a56fc2e7067b6b19132b Mon Sep 17 00:00:00 2001 From: Roman <61905975+antikorol@users.noreply.github.com> Date: Wed, 7 Aug 2024 16:44:22 +0300 Subject: [PATCH 3/8] #2110 Review load balancing and independent fetching the list of services in `Kube` provider (#2111) * Move the creation of the services list from the class field to the method, to prevent modification list from different threads * Early return after data checking * Add unit test for concurrent get list of services * Add logging for invalid service configuration error in RoundRobin load balancer * Code review by @raman-m * Workaround for mistakes made during acceptance testing of load balancing versus service discovery, where tests designed for parallel requests were mistakenly executed sequentially. This resulted in load balancers being loaded by sequential `HttpClient` calls, which was a significant oversight. * Let's DRY StickySessionsTests * Add acceptance tests, but... RoundRobin is not actually RoundRobin :grin: -> :laughing: * Independent static indexing iterators per route via service names * Stabilize `CookieStickySessions` load balancer. Review tests after refactoring of `RoundRobin` load balancer * Refactor Lease operation for load balancing. Review LeastConnection load balancer * Leasing mechanism in Round Robin load balancer * Acceptance tests, final version * Apply Retry pattern for K8s endpoint integration * Fix IDE warnings and messages * Follow suggestions and fix issues from code review by @ggnaegi * Bump KubeClient from 2.4.10 to 2.5.8 * Fix warnings * Final version of `Retry` pattern --------- Co-authored-by: Raman Maksimchuk --- src/Ocelot.Provider.Kubernetes/Kube.cs | 40 +- .../KubeServiceCreator.cs | 8 +- .../Ocelot.Provider.Kubernetes.csproj | 4 +- .../Infrastructure/DesignPatterns/Retry.cs | 115 +++++ .../LoadBalancers/CookieStickySessions.cs | 116 +++-- .../CookieStickySessionsCreator.cs | 7 +- .../LoadBalancer/LoadBalancers/Lease.cs | 69 ++- .../LoadBalancers/LeastConnection.cs | 168 ++----- .../LoadBalancer/LoadBalancers/RoundRobin.cs | 135 ++++- .../LoadBalancers/RoundRobinCreator.cs | 16 +- src/Ocelot/Values/ServiceHostAndPort.cs | 64 ++- .../CancelRequestTests.cs | 141 ++---- .../Configuration/ConfigurationReloadTests.cs | 8 +- .../LoadBalancerTests.cs | 408 +++++++-------- .../Properties/GlobalSuppressions.cs | 9 + .../ConsulServiceDiscoveryTests.cs | 90 +++- .../KubernetesServiceDiscoveryTests.cs | 474 +++++++++++++++--- test/Ocelot.AcceptanceTests/Steps.cs | 161 +++--- .../StickySessionsTests.cs | 390 +++++--------- test/Ocelot.Testing/FileRouteExtensions.cs | 12 + test/Ocelot.Testing/Ocelot.Testing.csproj | 6 +- test/Ocelot.UnitTests/Kubernetes/KubeTests.cs | 272 +++++----- .../LoadBalancer/CookieStickySessionsTests.cs | 422 ++++++++-------- .../LoadBalancer/LeastConnectionTests.cs | 37 +- .../LoadBalancer/RoundRobinTests.cs | 176 +++++-- 25 files changed, 1970 insertions(+), 1378 deletions(-) create mode 100644 src/Ocelot/Infrastructure/DesignPatterns/Retry.cs create mode 100644 test/Ocelot.AcceptanceTests/Properties/GlobalSuppressions.cs create mode 100644 test/Ocelot.Testing/FileRouteExtensions.cs diff --git a/src/Ocelot.Provider.Kubernetes/Kube.cs b/src/Ocelot.Provider.Kubernetes/Kube.cs index 5350f43b9..39dca0b41 100644 --- a/src/Ocelot.Provider.Kubernetes/Kube.cs +++ b/src/Ocelot.Provider.Kubernetes/Kube.cs @@ -1,20 +1,24 @@ using KubeClient.Models; +using Ocelot.Infrastructure.DesignPatterns; using Ocelot.Logging; using Ocelot.Provider.Kubernetes.Interfaces; using Ocelot.Values; namespace Ocelot.Provider.Kubernetes; -/// -/// Default Kubernetes service discovery provider. -/// +/// Default Kubernetes service discovery provider. +/// +/// +/// NuGet: KubeClient +/// GitHub: dotnet-kube-client +/// +/// public class Kube : IServiceDiscoveryProvider { private readonly KubeRegistryConfiguration _configuration; private readonly IOcelotLogger _logger; private readonly IKubeApiClient _kubeApi; private readonly IKubeServiceBuilder _serviceBuilder; - private readonly List _services; public Kube( KubeRegistryConfiguration configuration, @@ -26,28 +30,32 @@ public Kube( _logger = factory.CreateLogger(); _kubeApi = kubeApi; _serviceBuilder = serviceBuilder; - _services = new(); } public virtual async Task> GetAsync() { - var endpoint = await _kubeApi - .ResourceClient(client => new EndPointClientV1(client)) - .GetAsync(_configuration.KeyOfServiceInK8s, _configuration.KubeNamespace); + var endpoint = await Retry.OperationAsync(GetEndpoint, CheckErroneousState, logger: _logger); - _services.Clear(); - if (endpoint?.Subsets.Count != 0) + if (CheckErroneousState(endpoint)) { - _services.AddRange(BuildServices(_configuration, endpoint)); - } - else - { - _logger.LogWarning(() => $"K8s Namespace:{_configuration.KubeNamespace}, Service:{_configuration.KeyOfServiceInK8s}; Unable to use: it is invalid. Address must contain host only e.g. localhost and port must be greater than 0!"); + _logger.LogWarning(() => GetMessage($"Unable to use bad result returned by {nameof(Kube)} integration endpoint because the final result is invalid/unknown after multiple retries!")); + return new(0); } - return _services; + return BuildServices(_configuration, endpoint) + .ToList(); } + private Task GetEndpoint() => _kubeApi + .ResourceClient(client => new EndPointClientV1(client)) + .GetAsync(_configuration.KeyOfServiceInK8s, _configuration.KubeNamespace); + + private bool CheckErroneousState(EndpointsV1 endpoint) + => (endpoint?.Subsets?.Count ?? 0) == 0; // null or count is zero + + private string GetMessage(string message) + => $"{nameof(Kube)} provider. Namespace:{_configuration.KubeNamespace}, Service:{_configuration.KeyOfServiceInK8s}; {message}"; + protected virtual IEnumerable BuildServices(KubeRegistryConfiguration configuration, EndpointsV1 endpoint) => _serviceBuilder.BuildServices(configuration, endpoint); } diff --git a/src/Ocelot.Provider.Kubernetes/KubeServiceCreator.cs b/src/Ocelot.Provider.Kubernetes/KubeServiceCreator.cs index 3d51159c3..021c3e37f 100644 --- a/src/Ocelot.Provider.Kubernetes/KubeServiceCreator.cs +++ b/src/Ocelot.Provider.Kubernetes/KubeServiceCreator.cs @@ -7,12 +7,10 @@ namespace Ocelot.Provider.Kubernetes; public class KubeServiceCreator : IKubeServiceCreator { - private readonly IOcelotLogger _logger; - public KubeServiceCreator(IOcelotLoggerFactory factory) { ArgumentNullException.ThrowIfNull(factory); - _logger = factory.CreateLogger(); + Logger = factory.CreateLogger(); } public virtual IEnumerable Create(KubeRegistryConfiguration configuration, EndpointsV1 endpoint, EndpointSubsetV1 subset) @@ -34,6 +32,8 @@ public virtual IEnumerable CreateInstance(KubeRegistryConfiguration con return new Service[] { instance }; } + protected IOcelotLogger Logger { get; } + protected virtual string GetServiceName(KubeRegistryConfiguration configuration, EndpointsV1 endpoint, EndpointSubsetV1 subset, EndpointAddressV1 address) => endpoint.Metadata?.Name; @@ -46,7 +46,7 @@ protected virtual ServiceHostAndPort GetServiceHostAndPort(KubeRegistryConfigura : ports.FirstOrDefault(portNameToScheme); portV1 ??= new(); portV1.Name ??= configuration.Scheme ?? string.Empty; - _logger.LogDebug(() => $"K8s service with key '{configuration.KeyOfServiceInK8s}' and address {address.Ip}; Detected port is {portV1.Name}:{portV1.Port}. Total {ports.Count} ports of [{string.Join(',', ports.Select(p => p.Name))}]."); + Logger.LogDebug(() => $"K8s service with key '{configuration.KeyOfServiceInK8s}' and address {address.Ip}; Detected port is {portV1.Name}:{portV1.Port}. Total {ports.Count} ports of [{string.Join(',', ports.Select(p => p.Name))}]."); return new ServiceHostAndPort(address.Ip, portV1.Port, portV1.Name); } diff --git a/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj b/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj index 375b89535..f45dd07dd 100644 --- a/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj +++ b/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj @@ -29,8 +29,8 @@ - - + + all diff --git a/src/Ocelot/Infrastructure/DesignPatterns/Retry.cs b/src/Ocelot/Infrastructure/DesignPatterns/Retry.cs new file mode 100644 index 000000000..be402e114 --- /dev/null +++ b/src/Ocelot/Infrastructure/DesignPatterns/Retry.cs @@ -0,0 +1,115 @@ +using Ocelot.Logging; + +namespace Ocelot.Infrastructure.DesignPatterns; + +/// +/// Basic Retry pattern for stabilizing integrated services. +/// +/// Docs: +/// +/// Microsoft Learn | Retry pattern +/// +/// +public static class Retry +{ + public const int DefaultRetryTimes = 3; + public const int DefaultWaitTimeMilliseconds = 25; + + private static string GetMessage(T operation, int retryNo, string message) + where T : Delegate + => $"Ocelot {nameof(Retry)} strategy for the operation of '{operation.GetType()}' type -> {nameof(Retry)} No {retryNo}: {message}"; + + /// + /// Retry a synchronous operation when an exception occurs or predicate is true, then delay and retry again. + /// + /// Type of the result of the sync operation. + /// Required Func-delegate of the operation. + /// Predicate to check, optionally. + /// Number of retries. + /// Waiting time in milliseconds. + /// Concrete logger from upper context. + /// A value as the result of the sync operation. + public static TResult Operation( + Func operation, + Predicate predicate = null, + int retryTimes = DefaultRetryTimes, int waitTime = DefaultWaitTimeMilliseconds, + IOcelotLogger logger = null) + { + for (int n = 1; n < retryTimes; n++) + { + TResult result; + try + { + result = operation.Invoke(); + } + catch (Exception e) + { + logger?.LogError(() => GetMessage(operation, n, $"Caught exception of the {e.GetType()} type -> Message: {e.Message}."), e); + Thread.Sleep(waitTime); + continue; // the result is unknown, so continue to retry + } + + // Apply predicate for known result + if (predicate?.Invoke(result) == true) + { + logger?.LogWarning(() => GetMessage(operation, n, $"The predicate has identified erroneous state in the returned result. For further details, implement logging of the result's value or properties within the predicate method.")); + Thread.Sleep(waitTime); + continue; // on erroneous state + } + + // Happy path + return result; + } + + // Last retry should generate native exception or other erroneous state(s) + logger?.LogDebug(() => GetMessage(operation, retryTimes, $"Retrying lastly...")); + return operation.Invoke(); // also final result must be analyzed in the upper context + } + + /// + /// Retry an asynchronous operation when an exception occurs or predicate is true, then delay and retry again. + /// + /// Type of the result of the async operation. + /// Required Func-delegate of the operation. + /// Predicate to check, optionally. + /// Number of retries. + /// Waiting time in milliseconds. + /// Concrete logger from upper context. + /// A value as the result of the async operation. + public static async Task OperationAsync( + Func> operation, // required operation delegate + Predicate predicate = null, // optional retry predicate for the result + int retryTimes = DefaultRetryTimes, int waitTime = DefaultWaitTimeMilliseconds, // retrying options + IOcelotLogger logger = null) // static injections + { + for (int n = 1; n < retryTimes; n++) + { + TResult result; + try + { + result = await operation?.Invoke(); + } + catch (Exception e) + { + logger?.LogError(() => GetMessage(operation, n, $"Caught exception of the {e.GetType()} type -> Message: {e.Message}."), e); + await Task.Delay(waitTime); + continue; // the result is unknown, so continue to retry + } + + // Apply predicate for known result + if (predicate?.Invoke(result) == true) + { + logger?.LogWarning(() => GetMessage(operation, n, $"The predicate has identified erroneous state in the returned result. For further details, implement logging of the result's value or properties within the predicate method.")); + await Task.Delay(waitTime); + continue; // on erroneous state + } + + // Happy path + return result; + } + + // Last retry should generate native exception or other erroneous state(s) + logger?.LogDebug(() => GetMessage(operation, retryTimes, $"Retrying lastly...")); + return await operation?.Invoke(); // also final result must be analyzed in the upper context + } +} diff --git a/src/Ocelot/LoadBalancer/LoadBalancers/CookieStickySessions.cs b/src/Ocelot/LoadBalancer/LoadBalancers/CookieStickySessions.cs index dd2c6d185..a135855a4 100644 --- a/src/Ocelot/LoadBalancer/LoadBalancers/CookieStickySessions.cs +++ b/src/Ocelot/LoadBalancer/LoadBalancers/CookieStickySessions.cs @@ -1,85 +1,83 @@ using Microsoft.AspNetCore.Http; using Ocelot.Infrastructure; +using Ocelot.Middleware; using Ocelot.Responses; using Ocelot.Values; -namespace Ocelot.LoadBalancer.LoadBalancers +namespace Ocelot.LoadBalancer.LoadBalancers; + +public class CookieStickySessions : ILoadBalancer { - public class CookieStickySessions : ILoadBalancer + private readonly int _keyExpiryInMs; + private readonly string _cookieName; + private readonly ILoadBalancer _loadBalancer; + private readonly IBus _bus; + + private static readonly object Locker = new(); + private static readonly Dictionary Stored = new(); // TODO Inject instead of static sharing + + public CookieStickySessions(ILoadBalancer loadBalancer, string cookieName, int keyExpiryInMs, IBus bus) { - private readonly int _keyExpiryInMs; - private readonly string _key; - private readonly ILoadBalancer _loadBalancer; - private readonly ConcurrentDictionary _stored; - private readonly IBus _bus; - private readonly object _lock = new(); + _bus = bus; + _cookieName = cookieName; + _keyExpiryInMs = keyExpiryInMs; + _loadBalancer = loadBalancer; + _bus.Subscribe(CheckExpiry); + } - public CookieStickySessions(ILoadBalancer loadBalancer, string key, int keyExpiryInMs, IBus bus) + private void CheckExpiry(StickySession sticky) + { + // TODO Get test coverage for this + lock (Locker) { - _bus = bus; - _key = key; - _keyExpiryInMs = keyExpiryInMs; - _loadBalancer = loadBalancer; - _stored = new ConcurrentDictionary(); - _bus.Subscribe(ss => + if (!Stored.TryGetValue(sticky.Key, out var session) || session.Expiry >= DateTime.UtcNow) { - //todo - get test coverage for this. - if (_stored.TryGetValue(ss.Key, out var stickySession)) - { - lock (_lock) - { - if (stickySession.Expiry < DateTime.UtcNow) - { - _stored.TryRemove(stickySession.Key, out _); - _loadBalancer.Release(stickySession.HostAndPort); - } - } - } - }); + return; + } + + Stored.Remove(session.Key); + _loadBalancer.Release(session.HostAndPort); } + } - public async Task> Lease(HttpContext httpContext) + public Task> Lease(HttpContext httpContext) + { + var route = httpContext.Items.DownstreamRoute(); + var serviceName = route.LoadBalancerKey; + var cookie = httpContext.Request.Cookies[_cookieName]; + var key = $"{serviceName}:{cookie}"; // strong key name because of static store + lock (Locker) { - var key = httpContext.Request.Cookies[_key]; - - lock (_lock) + if (!string.IsNullOrEmpty(key) && Stored.TryGetValue(key, out StickySession cached)) { - if (!string.IsNullOrEmpty(key) && _stored.ContainsKey(key)) - { - var cached = _stored[key]; - - var updated = new StickySession(cached.HostAndPort, DateTime.UtcNow.AddMilliseconds(_keyExpiryInMs), key); - - _stored[key] = updated; - - _bus.Publish(updated, _keyExpiryInMs); - - return new OkResponse(updated.HostAndPort); - } + var updated = new StickySession(cached.HostAndPort, DateTime.UtcNow.AddMilliseconds(_keyExpiryInMs), key); + Update(key, updated); + return Task.FromResult>(new OkResponse(updated.HostAndPort)); } - var next = await _loadBalancer.Lease(httpContext); - + // There is no value in the store, so lease it now! + var next = _loadBalancer.Lease(httpContext).GetAwaiter().GetResult(); // unfortunately the operation must be synchronous if (next.IsError) { - return new ErrorResponse(next.Errors); - } - - lock (_lock) - { - if (!string.IsNullOrEmpty(key) && !_stored.ContainsKey(key)) - { - var ss = new StickySession(next.Data, DateTime.UtcNow.AddMilliseconds(_keyExpiryInMs), key); - _stored[key] = ss; - _bus.Publish(ss, _keyExpiryInMs); - } + return Task.FromResult>(new ErrorResponse(next.Errors)); } - return new OkResponse(next.Data); + var ss = new StickySession(next.Data, DateTime.UtcNow.AddMilliseconds(_keyExpiryInMs), key); + Update(key, ss); + return Task.FromResult>(new OkResponse(next.Data)); } + } - public void Release(ServiceHostAndPort hostAndPort) + protected void Update(string key, StickySession value) + { + lock (Locker) { + Stored[key] = value; + _bus.Publish(value, _keyExpiryInMs); } } + + public void Release(ServiceHostAndPort hostAndPort) + { + } } diff --git a/src/Ocelot/LoadBalancer/LoadBalancers/CookieStickySessionsCreator.cs b/src/Ocelot/LoadBalancer/LoadBalancers/CookieStickySessionsCreator.cs index 2994bdbdc..15b6836da 100644 --- a/src/Ocelot/LoadBalancer/LoadBalancers/CookieStickySessionsCreator.cs +++ b/src/Ocelot/LoadBalancer/LoadBalancers/CookieStickySessionsCreator.cs @@ -9,10 +9,11 @@ public class CookieStickySessionsCreator : ILoadBalancerCreator { public Response Create(DownstreamRoute route, IServiceDiscoveryProvider serviceProvider) { - var loadBalancer = new RoundRobin(async () => await serviceProvider.GetAsync()); + var options = route.LoadBalancerOptions; + var loadBalancer = new RoundRobin(serviceProvider.GetAsync, route.LoadBalancerKey); var bus = new InMemoryBus(); - return new OkResponse(new CookieStickySessions(loadBalancer, route.LoadBalancerOptions.Key, - route.LoadBalancerOptions.ExpiryInMs, bus)); + return new OkResponse( + new CookieStickySessions(loadBalancer, options.Key, options.ExpiryInMs, bus)); } public string Type => nameof(CookieStickySessions); diff --git a/src/Ocelot/LoadBalancer/LoadBalancers/Lease.cs b/src/Ocelot/LoadBalancer/LoadBalancers/Lease.cs index 632837b3e..46d120e4c 100644 --- a/src/Ocelot/LoadBalancer/LoadBalancers/Lease.cs +++ b/src/Ocelot/LoadBalancer/LoadBalancers/Lease.cs @@ -1,16 +1,63 @@ using Ocelot.Values; -namespace Ocelot.LoadBalancer.LoadBalancers +namespace Ocelot.LoadBalancer.LoadBalancers; + +public struct Lease : IEquatable { - public class Lease + public Lease() + { + HostAndPort = null; + Connections = 0; + } + + public Lease(Lease from) + { + HostAndPort = from.HostAndPort; + Connections = from.Connections; + } + + public Lease(ServiceHostAndPort hostAndPort) + { + HostAndPort = hostAndPort; + Connections = 0; + } + + public Lease(ServiceHostAndPort hostAndPort, int connections) { - public Lease(ServiceHostAndPort hostAndPort, int connections) - { - HostAndPort = hostAndPort; - Connections = connections; - } - - public ServiceHostAndPort HostAndPort { get; } - public int Connections { get; } + HostAndPort = hostAndPort; + Connections = connections; } -} + + public ServiceHostAndPort HostAndPort { get; } + public int Connections { get; set; } + + public static Lease Null => new(); + + public override readonly string ToString() => $"({HostAndPort}+{Connections})"; + public override readonly int GetHashCode() => HostAndPort.GetHashCode(); + public override readonly bool Equals(object obj) => obj is Lease l && this == l; + public readonly bool Equals(Lease other) => this == other; + + /// Checks equality of two leases. + /// + /// Override default implementation of because we want to ignore the property. + /// Microsoft Learn | .NET | C# Docs: + /// + /// Equality operators + /// System.Object.Equals method + /// IEquatable<T>.Equals(T) Method + /// ValueType.Equals(Object) Method + /// + /// + /// First operand. + /// Second operand. + /// if both operands are equal; otherwise, . + public static bool operator ==(Lease x, Lease y) => x.HostAndPort == y.HostAndPort; // ignore -> x.Connections == y.Connections; + public static bool operator !=(Lease x, Lease y) => !(x == y); + + public static bool operator ==(ServiceHostAndPort h, Lease l) => h == l.HostAndPort; + public static bool operator !=(ServiceHostAndPort h, Lease l) => !(h == l); + + public static bool operator ==(Lease l, ServiceHostAndPort h) => l.HostAndPort == h; + public static bool operator !=(Lease l, ServiceHostAndPort h) => !(l == h); +} diff --git a/src/Ocelot/LoadBalancer/LoadBalancers/LeastConnection.cs b/src/Ocelot/LoadBalancer/LoadBalancers/LeastConnection.cs index 261f3fe73..bede3ba8e 100644 --- a/src/Ocelot/LoadBalancer/LoadBalancers/LeastConnection.cs +++ b/src/Ocelot/LoadBalancer/LoadBalancers/LeastConnection.cs @@ -2,140 +2,80 @@ using Ocelot.Responses; using Ocelot.Values; -namespace Ocelot.LoadBalancer.LoadBalancers +namespace Ocelot.LoadBalancer.LoadBalancers; + +public class LeastConnection : ILoadBalancer { - public class LeastConnection : ILoadBalancer + private readonly Func>> _services; + private readonly List _leases; + private readonly string _serviceName; + private static readonly object SyncLock = new(); + + public LeastConnection(Func>> services, string serviceName) { - private readonly Func>> _services; - private readonly List _leases; - private readonly string _serviceName; - private static readonly object SyncLock = new(); + _services = services; + _serviceName = serviceName; + _leases = new List(); + } - public LeastConnection(Func>> services, string serviceName) + public async Task> Lease(HttpContext httpContext) + { + var services = await _services.Invoke(); + if ((services?.Count ?? 0) == 0) { - _services = services; - _serviceName = serviceName; - _leases = new List(); + return new ErrorResponse(new ServicesAreNullError($"Services were null/empty in {nameof(LeastConnection)} for '{_serviceName}' during {nameof(Lease)} operation!")); } - public async Task> Lease(HttpContext httpContext) + lock (SyncLock) { - var services = await _services.Invoke(); - - if (services == null) - { - return new ErrorResponse(new ServicesAreNullError($"services were null for {_serviceName}")); - } - - if (!services.Any()) - { - return new ErrorResponse(new ServicesAreEmptyError($"services were empty for {_serviceName}")); - } - - lock (SyncLock) - { - //todo - maybe this should be moved somewhere else...? Maybe on a repeater on seperate thread? loop every second and update or something? - UpdateServices(services); + //todo - maybe this should be moved somewhere else...? Maybe on a repeater on seperate thread? loop every second and update or something? + UpdateLeasing(services); - var leaseWithLeastConnections = GetLeaseWithLeastConnections(); - - _leases.Remove(leaseWithLeastConnections); - - leaseWithLeastConnections = AddConnection(leaseWithLeastConnections); - - _leases.Add(leaseWithLeastConnections); - - return new OkResponse(new ServiceHostAndPort(leaseWithLeastConnections.HostAndPort.DownstreamHost, leaseWithLeastConnections.HostAndPort.DownstreamPort)); - } + Lease wanted = GetLeaseWithLeastConnections(); + _ = Update(ref wanted, true); + return new OkResponse(new(wanted.HostAndPort)); } + } - public void Release(ServiceHostAndPort hostAndPort) + public void Release(ServiceHostAndPort hostAndPort) + { + lock (SyncLock) { - lock (SyncLock) + var matchingLease = _leases.Find(l => l == hostAndPort); + if (matchingLease != LoadBalancers.Lease.Null) { - var matchingLease = _leases.FirstOrDefault(l => l.HostAndPort.DownstreamHost == hostAndPort.DownstreamHost - && l.HostAndPort.DownstreamPort == hostAndPort.DownstreamPort); - - if (matchingLease != null) - { - var replacementLease = new Lease(hostAndPort, matchingLease.Connections - 1); - - _leases.Remove(matchingLease); - - _leases.Add(replacementLease); - } + _ = Update(ref matchingLease, false); } } + } - private static Lease AddConnection(Lease lease) - { - return new Lease(lease.HostAndPort, lease.Connections + 1); - } + private int Update(ref Lease item, bool increase) + { + var index = _leases.IndexOf(item); + _ = increase ? item.Connections++ : item.Connections--; + _leases[index] = item; // write the value back to the position + return index; + } - private Lease GetLeaseWithLeastConnections() - { - //now get the service with the least connections? - Lease leaseWithLeastConnections = null; + private Lease GetLeaseWithLeastConnections() + { + var min = _leases.Min(l => l.Connections); + return _leases.Find(l => l.Connections == min); + } - for (var i = 0; i < _leases.Count; i++) - { - if (i == 0) - { - leaseWithLeastConnections = _leases[i]; - } - else - { - if (_leases[i].Connections < leaseWithLeastConnections.Connections) - { - leaseWithLeastConnections = _leases[i]; - } - } - } + private void UpdateLeasing(List services) + { + if (_leases.Count > 0) + { + _leases.RemoveAll(l => !services.Exists(s => s.HostAndPort == l)); - return leaseWithLeastConnections; + services.Where(s => !_leases.Exists(l => l == s.HostAndPort)) + .ToList() + .ForEach(s => _leases.Add(new(s.HostAndPort, 0))); } - - private Response UpdateServices(List services) + else { - if (_leases.Count > 0) - { - var leasesToRemove = new List(); - - foreach (var lease in _leases) - { - var match = services.FirstOrDefault(s => s.HostAndPort.DownstreamHost == lease.HostAndPort.DownstreamHost - && s.HostAndPort.DownstreamPort == lease.HostAndPort.DownstreamPort); - - if (match == null) - { - leasesToRemove.Add(lease); - } - } - - foreach (var lease in leasesToRemove) - { - _leases.Remove(lease); - } - - foreach (var service in services) - { - var exists = _leases.FirstOrDefault(l => l.HostAndPort.DownstreamHost == service.HostAndPort.DownstreamHost && l.HostAndPort.DownstreamPort == service.HostAndPort.DownstreamPort); - - if (exists == null) - { - _leases.Add(new Lease(service.HostAndPort, 0)); - } - } - } - else - { - foreach (var service in services) - { - _leases.Add(new Lease(service.HostAndPort, 0)); - } - } - - return new OkResponse(); + services.ForEach(s => _leases.Add(new(s.HostAndPort))); } } } diff --git a/src/Ocelot/LoadBalancer/LoadBalancers/RoundRobin.cs b/src/Ocelot/LoadBalancer/LoadBalancers/RoundRobin.cs index 834f01e4d..9c087f77a 100644 --- a/src/Ocelot/LoadBalancer/LoadBalancers/RoundRobin.cs +++ b/src/Ocelot/LoadBalancer/LoadBalancers/RoundRobin.cs @@ -2,43 +2,132 @@ using Ocelot.Responses; using Ocelot.Values; -namespace Ocelot.LoadBalancer.LoadBalancers +namespace Ocelot.LoadBalancer.LoadBalancers; + +public class RoundRobin : ILoadBalancer { - public class RoundRobin : ILoadBalancer + private readonly Func>> _servicesDelegate; + private readonly string _serviceName; + private readonly List _leasing; + + public RoundRobin(Func>> services, string serviceName) { - private readonly Func>> _servicesDelegate; - private readonly object _lock = new(); + _servicesDelegate = services; + _serviceName = serviceName; + _leasing = new(); + } - private int _last; + private static readonly Dictionary LastIndices = new(); + protected static readonly object SyncRoot = new(); - public RoundRobin(Func>> services) + public event EventHandler Leased; + protected virtual void OnLeased(LeaseEventArgs e) => Leased?.Invoke(this, e); + + public virtual async Task> Lease(HttpContext httpContext) + { + var services = await _servicesDelegate?.Invoke() ?? new List(); + if (services.Count == 0) { - _servicesDelegate = services; + return new ErrorResponse(new ServicesAreEmptyError($"There were no services in {nameof(RoundRobin)} for '{_serviceName}' during {nameof(Lease)} operation!")); } - public async Task> Lease(HttpContext httpContext) + lock (SyncRoot) { - var services = await _servicesDelegate?.Invoke() ?? new List(); - - if (services?.Count != 0) + var readMe = CaptureState(services, out int count); + if (!TryScanNext(readMe, out Service next, out int index)) { - lock (_lock) - { - if (_last >= services.Count) - { - _last = 0; - } - - var next = services[_last++]; - return new OkResponse(next.HostAndPort); - } + return new ErrorResponse(new ServicesAreNullError($"The service at index {index} was null in {nameof(RoundRobin)} for {_serviceName} during the {nameof(Lease)} operation. Total services count: {count}.")); } - return new ErrorResponse(new ServicesAreEmptyError($"There were no services in {nameof(RoundRobin)} during {nameof(Lease)} operation.")); + ProcessLeasing(readMe, next, index); // Happy path: Lease now + return new OkResponse(next.HostAndPort); } + } - public void Release(ServiceHostAndPort hostAndPort) + public virtual void Release(ServiceHostAndPort hostAndPort) { } + + /// Capture the count value because another thread might modify the list. + /// Mutable collection of services. + /// Captured count value. + /// Captured collection as a object. + private static Service[] CaptureState(List services, out int count) + { + // Capture the count value because another thread might modify the list + count = services.Count; + var readMe = new Service[count]; + services.CopyTo(readMe); + return readMe; + } + + /// Scan for the next online service instance which must be healthy. + /// Read-only collection. + /// The next online service to return. + /// The index of the next service to return. + /// if found next online service; otherwise . + private bool TryScanNext(Service[] readme, out Service next, out int index) + { + int length = readme.Length, stop = length; + LastIndices.TryGetValue(_serviceName, out int last); + if (last >= length) + { + last = 0; + } + + next = null; + index = last; + + // Scan for the next service instance + // TODO Check real health status + while (next?.HostAndPort == null && stop-- > 0) { + index = last; + next = readme[last]; + LastIndices[_serviceName] = (++last < length) ? last : 0; } + + return next != null; } + + private void ProcessLeasing(Service[] readme, Service next, int index) + { + UpdateLeasing(readme); + Lease wanted = GetLease(next); + _ = Update(ref wanted, true); // perform counting based on Connections + OnLeased(new(wanted, next, index)); + } + + private int Update(ref Lease item, bool increase) + { + var index = _leasing.IndexOf(item); + _ = increase ? item.Connections++ : item.Connections--; + _leasing[index] = item; // write the value back to the position + return index; + } + + private Lease GetLease(Service @for) => _leasing.Find(l => l == @for.HostAndPort); + + private void UpdateLeasing(IList services) + { + // Don't remove leasing data of old services, so keep data during life time of the load balancer + // _leasing.RemoveAll(l => services.All(s => s?.HostAndPort != l)); + var newLeases = services + .Where(s => s != null && !_leasing.Exists(l => l == s.HostAndPort)) + .Select(s => new Lease(s.HostAndPort)) + .ToArray(); // capture leasing state and produce new collection + _leasing.AddRange(newLeases); + } +} + +public class LeaseEventArgs : EventArgs +{ + public LeaseEventArgs(Lease lease, Service service, int serviceIndex) + { + Lease = lease; + Service = service; + ServiceIndex = serviceIndex; + } + + public Lease Lease { get; } + public Service Service { get; } + public int ServiceIndex { get; } } diff --git a/src/Ocelot/LoadBalancer/LoadBalancers/RoundRobinCreator.cs b/src/Ocelot/LoadBalancer/LoadBalancers/RoundRobinCreator.cs index e0c0e1a81..057fa95e6 100644 --- a/src/Ocelot/LoadBalancer/LoadBalancers/RoundRobinCreator.cs +++ b/src/Ocelot/LoadBalancer/LoadBalancers/RoundRobinCreator.cs @@ -2,15 +2,15 @@ using Ocelot.Responses; using Ocelot.ServiceDiscovery.Providers; -namespace Ocelot.LoadBalancer.LoadBalancers +namespace Ocelot.LoadBalancer.LoadBalancers; + +public class RoundRobinCreator : ILoadBalancerCreator { - public class RoundRobinCreator : ILoadBalancerCreator + public Response Create(DownstreamRoute route, IServiceDiscoveryProvider serviceProvider) { - public Response Create(DownstreamRoute route, IServiceDiscoveryProvider serviceProvider) - { - return new OkResponse(new RoundRobin(async () => await serviceProvider.GetAsync())); - } - - public string Type => nameof(RoundRobin); + var loadBalancer = new RoundRobin(serviceProvider.GetAsync, route.ServiceName); + return new OkResponse(loadBalancer); } + + public string Type => nameof(RoundRobin); } diff --git a/src/Ocelot/Values/ServiceHostAndPort.cs b/src/Ocelot/Values/ServiceHostAndPort.cs index fff7edba9..38890df57 100644 --- a/src/Ocelot/Values/ServiceHostAndPort.cs +++ b/src/Ocelot/Values/ServiceHostAndPort.cs @@ -1,20 +1,54 @@ -namespace Ocelot.Values +namespace Ocelot.Values; + +public class ServiceHostAndPort : IEquatable { - public class ServiceHostAndPort + public ServiceHostAndPort(ServiceHostAndPort from) { - public ServiceHostAndPort(string downstreamHost, int downstreamPort) - { - DownstreamHost = downstreamHost?.Trim('/'); - DownstreamPort = downstreamPort; - } + DownstreamHost = from.DownstreamHost; + DownstreamPort = from.DownstreamPort; + Scheme = from.Scheme; + } - public ServiceHostAndPort(string downstreamHost, int downstreamPort, string scheme) - : this(downstreamHost, downstreamPort) => Scheme = scheme; + public ServiceHostAndPort(string downstreamHost, int downstreamPort) + { + DownstreamHost = downstreamHost?.Trim('/'); + DownstreamPort = downstreamPort; + } - public string DownstreamHost { get; } + public ServiceHostAndPort(string downstreamHost, int downstreamPort, string scheme) + : this(downstreamHost, downstreamPort) => Scheme = scheme; - public int DownstreamPort { get; } - - public string Scheme { get; } - } -} + public string DownstreamHost { get; } + public int DownstreamPort { get; } + public string Scheme { get; } + + public override string ToString() + => $"{Scheme}:{DownstreamHost}:{DownstreamPort}"; + public override int GetHashCode() + => Tuple.Create(Scheme, DownstreamHost, DownstreamPort).GetHashCode(); + + public bool Equals(ServiceHostAndPort other) => this == other; + public override bool Equals(object obj) + => obj != null && obj is ServiceHostAndPort o && this == o; + + /// Checks equality of two hosts. + /// Microsoft Learn | .NET | C# Docs: + /// + /// Equality operators + /// System.Object.Equals method + /// IEquatable<T>.Equals(T) Method + /// + /// + /// Left operand. + /// Right operand. + /// if both operands are equal; otherwise, . + public static bool operator ==(ServiceHostAndPort l, ServiceHostAndPort r) + => (((object)l) == null || ((object)r) == null) + ? Equals(l, r) + : l.DownstreamHost == r.DownstreamHost && l.DownstreamPort == r.DownstreamPort && l.Scheme == r.Scheme; + + public static bool operator !=(ServiceHostAndPort l, ServiceHostAndPort r) + => (((object)l) == null || ((object)r) == null) + ? !Equals(l, r) + : !(l == r); +} diff --git a/test/Ocelot.AcceptanceTests/CancelRequestTests.cs b/test/Ocelot.AcceptanceTests/CancelRequestTests.cs index 984160591..8533f4183 100644 --- a/test/Ocelot.AcceptanceTests/CancelRequestTests.cs +++ b/test/Ocelot.AcceptanceTests/CancelRequestTests.cs @@ -1,107 +1,86 @@ using Microsoft.AspNetCore.Http; -using Ocelot.Configuration.File; -using Shouldly; -using System; -using System.Collections.Generic; -using System.Net; -using System.Threading.Tasks; -using TestStack.BDDfy; -using Xunit; namespace Ocelot.AcceptanceTests; -public class CancelRequestTests : IDisposable +public sealed class CancelRequestTests : Steps, IDisposable { - private const int SERVICE_WORK_TIME = 5_000; - private const int MAX_WAITING_TIME = 60_000; - - private readonly Steps _steps; private readonly ServiceHandler _serviceHandler; - private readonly Notifier _serviceWorkStartedNotifier; - private readonly Notifier _serviceWorkStoppedNotifier; - - private bool _cancelExceptionThrown; public CancelRequestTests() { - _steps = new Steps(); _serviceHandler = new ServiceHandler(); - _serviceWorkStartedNotifier = new Notifier("service work started notifier"); - _serviceWorkStoppedNotifier = new Notifier("service work finished notifier"); + } + + public override void Dispose() + { + _serviceHandler?.Dispose(); + base.Dispose(); } [Fact] - public void Should_abort_service_work_when_cancelling_the_request() + public async Task ShouldAbortServiceWork_WhenCancellingTheRequest() { + // Arrange var port = PortFinder.GetRandomPort(); - - var configuration = new FileConfiguration + var route = GivenDefaultRoute(port); + var configuration = GivenConfiguration(route); + var started = new Notifier("service work started notifier"); + var stopped = new Notifier("service work finished notifier"); + GivenThereIsAServiceRunningOn(DownstreamUrl(port), started, stopped); + GivenThereIsAConfiguration(configuration); + GivenOcelotIsRunning(); + + // Act: Initialize + var getting = WhenIGetUrl("/"); + var canceling = WhenIWaitForNotification(started).ContinueWith(Cancel); + Exception ex = null; + + // Act + try { - Routes = new List - { - new() - { - DownstreamPathTemplate = "/", - DownstreamHostAndPorts = new List - { - new() - { - Host = "localhost", - Port = port, - }, - }, - DownstreamScheme = "http", - UpstreamPathTemplate = "/", - UpstreamHttpMethod = new List { "Get" }, - }, - }, - }; + await Task.WhenAll(getting, canceling); + } + catch (Exception e) + { + ex = e; + } - this.Given(x => x.GivenThereIsAServiceRunningOn($"http://localhost:{port}")) - .And(x => _steps.GivenThereIsAConfiguration(configuration)) - .And(x => _steps.GivenOcelotIsRunning()) - .When(x => _steps.WhenIGetUrlOnTheApiGatewayAndDontWait("/")) - .And(x => WhenIWaitForNotification(_serviceWorkStartedNotifier)) - .And(x => _steps.WhenICancelTheRequest()) - .And(x => WhenIWaitForNotification(_serviceWorkStoppedNotifier)) - .Then(x => x.ThenOcelotClientRequestIsCanceled()) - .BDDfy(); + // Assert + started.NotificationSent.ShouldBeTrue(); + stopped.NotificationSent.ShouldBeFalse(); +#if NET8_0_OR_GREATER + ex.ShouldNotBeNull().ShouldBeOfType(); +#else + ex.ShouldNotBeNull().ShouldBeOfType(); +#endif } - private void GivenThereIsAServiceRunningOn(string baseUrl) + private Task Cancel(Task t) => Task.Run(_ocelotClient.CancelPendingRequests); + + private void GivenThereIsAServiceRunningOn(string baseUrl, Notifier startedNotifier, Notifier stoppedNotifier) { _serviceHandler.GivenThereIsAServiceRunningOn(baseUrl, async context => { - try - { - var response = string.Empty; - - _serviceWorkStartedNotifier.NotificationSent = true; - await Task.Delay(SERVICE_WORK_TIME, context.RequestAborted); + startedNotifier.NotificationSent = true; + await Task.Delay(SERVICE_WORK_TIME, context.RequestAborted); - context.Response.StatusCode = (int)HttpStatusCode.OK; - await context.Response.WriteAsync(response); - } - catch (TaskCanceledException) - { - _cancelExceptionThrown = true; - } - finally - { - _serviceWorkStoppedNotifier.NotificationSent = true; - } + context.Response.StatusCode = (int)HttpStatusCode.OK; + await context.Response.WriteAsync("OK"); + stoppedNotifier.NotificationSent = true; }); } + private const int SERVICE_WORK_TIME = 1_000; + private const int WAITING_TIME = 50; + private const int MAX_WAITING_TIME = 10_000; + private static async Task WhenIWaitForNotification(Notifier notifier) { int waitingTime = 0; while (!notifier.NotificationSent) { - var waitingInterval = 50; - await Task.Delay(waitingInterval); - waitingTime += waitingInterval; - + await Task.Delay(WAITING_TIME); + waitingTime += WAITING_TIME; if (waitingTime > MAX_WAITING_TIME) { throw new TimeoutException(notifier.Name + $" did not sent notification within {MAX_WAITING_TIME / 1000} second(s)."); @@ -109,25 +88,9 @@ private static async Task WhenIWaitForNotification(Notifier notifier) } } - private void ThenOcelotClientRequestIsCanceled() - { - _serviceWorkStartedNotifier.NotificationSent.ShouldBeTrue(); - _serviceWorkStoppedNotifier.NotificationSent.ShouldBeTrue(); - - _cancelExceptionThrown.ShouldBeTrue(); - } - - public void Dispose() - { - _serviceHandler?.Dispose(); - _steps.Dispose(); - GC.SuppressFinalize(this); - } - class Notifier { public Notifier(string name) => Name = name; - public bool NotificationSent { get; set; } public string Name { get; set; } } diff --git a/test/Ocelot.AcceptanceTests/Configuration/ConfigurationReloadTests.cs b/test/Ocelot.AcceptanceTests/Configuration/ConfigurationReloadTests.cs index 7d9037fc2..d416c6c45 100644 --- a/test/Ocelot.AcceptanceTests/Configuration/ConfigurationReloadTests.cs +++ b/test/Ocelot.AcceptanceTests/Configuration/ConfigurationReloadTests.cs @@ -47,7 +47,7 @@ public void should_not_reload_config_on_change() this.Given(x => _steps.GivenThereIsAConfiguration(_initialConfig)) .And(x => _steps.GivenOcelotIsRunningReloadingConfig(false)) .And(x => _steps.GivenThereIsAConfiguration(_anotherConfig)) - .And(x => _steps.GivenIWait(MillisecondsToWaitForChangeToken)) + .And(x => Steps.GivenIWait(MillisecondsToWaitForChangeToken)) .And(x => _steps.ThenConfigShouldBe(_initialConfig)) .BDDfy(); } @@ -59,7 +59,7 @@ public void should_trigger_change_token_on_change() .And(x => _steps.GivenOcelotIsRunningReloadingConfig(true)) .And(x => _steps.GivenIHaveAChangeToken()) .And(x => _steps.GivenThereIsAConfiguration(_anotherConfig)) - .And(x => _steps.GivenIWait(MillisecondsToWaitForChangeToken)) + .And(x => Steps.GivenIWait(MillisecondsToWaitForChangeToken)) .Then(x => _steps.TheChangeTokenShouldBeActive(true)) .BDDfy(); } @@ -70,9 +70,9 @@ public void should_not_trigger_change_token_with_no_change() this.Given(x => _steps.GivenThereIsAConfiguration(_initialConfig)) .And(x => _steps.GivenOcelotIsRunningReloadingConfig(false)) .And(x => _steps.GivenIHaveAChangeToken()) - .And(x => _steps.GivenIWait(MillisecondsToWaitForChangeToken)) // Wait for prior activation to expire. + .And(x => Steps.GivenIWait(MillisecondsToWaitForChangeToken)) // Wait for prior activation to expire. .And(x => _steps.GivenThereIsAConfiguration(_anotherConfig)) - .And(x => _steps.GivenIWait(MillisecondsToWaitForChangeToken)) + .And(x => Steps.GivenIWait(MillisecondsToWaitForChangeToken)) .Then(x => _steps.TheChangeTokenShouldBeActive(false)) .BDDfy(); } diff --git a/test/Ocelot.AcceptanceTests/LoadBalancerTests.cs b/test/Ocelot.AcceptanceTests/LoadBalancerTests.cs index f882868d6..104cdaaea 100644 --- a/test/Ocelot.AcceptanceTests/LoadBalancerTests.cs +++ b/test/Ocelot.AcceptanceTests/LoadBalancerTests.cs @@ -6,261 +6,269 @@ using Ocelot.ServiceDiscovery.Providers; using Ocelot.Values; -namespace Ocelot.AcceptanceTests +namespace Ocelot.AcceptanceTests; + +public sealed class LoadBalancerTests : IDisposable { - public class LoadBalancerTests : IDisposable - { - private readonly Steps _steps; - private int _counterOne; - private int _counterTwo; - private static readonly object SyncLock = new(); - private readonly ServiceHandler _serviceHandler; + private readonly Steps _steps; + private int _counterOne; + private int _counterTwo; + private static readonly object SyncLock = new(); + private readonly ServiceHandler _serviceHandler; - public LoadBalancerTests() - { - _serviceHandler = new ServiceHandler(); - _steps = new Steps(); - } + public LoadBalancerTests() + { + _serviceHandler = new ServiceHandler(); + _steps = new Steps(); + } - [Fact] - public void should_load_balance_request_with_least_connection() - { - var portOne = PortFinder.GetRandomPort(); - var portTwo = PortFinder.GetRandomPort(); + [Fact] + public void Should_load_balance_request_with_least_connection() + { + var portOne = PortFinder.GetRandomPort(); + var portTwo = PortFinder.GetRandomPort(); - var downstreamServiceOneUrl = $"http://localhost:{portOne}"; - var downstreamServiceTwoUrl = $"http://localhost:{portTwo}"; + var downstreamServiceOneUrl = $"http://localhost:{portOne}"; + var downstreamServiceTwoUrl = $"http://localhost:{portTwo}"; - var configuration = new FileConfiguration - { - Routes = new List + var configuration = new FileConfiguration + { + Routes = new List + { + new() { - new() + DownstreamPathTemplate = "/", + DownstreamScheme = "http", + UpstreamPathTemplate = "/", + UpstreamHttpMethod = new List { "Get" }, + LoadBalancerOptions = new FileLoadBalancerOptions { Type = nameof(LeastConnection) }, + DownstreamHostAndPorts = new List { - DownstreamPathTemplate = "/", - DownstreamScheme = "http", - UpstreamPathTemplate = "/", - UpstreamHttpMethod = new List { "Get" }, - LoadBalancerOptions = new FileLoadBalancerOptions { Type = nameof(LeastConnection) }, - DownstreamHostAndPorts = new List + new() + { + Host = "localhost", + Port = portOne, + }, + new() { - new() - { - Host = "localhost", - Port = portOne, - }, - new() - { - Host = "localhost", - Port = portTwo, - }, + Host = "localhost", + Port = portTwo, }, }, }, - GlobalConfiguration = new FileGlobalConfiguration(), - }; + }, + GlobalConfiguration = new FileGlobalConfiguration(), + }; - this.Given(x => x.GivenProductServiceOneIsRunning(downstreamServiceOneUrl, 200)) - .And(x => x.GivenProductServiceTwoIsRunning(downstreamServiceTwoUrl, 200)) - .And(x => _steps.GivenThereIsAConfiguration(configuration)) - .And(x => _steps.GivenOcelotIsRunning()) - .When(x => _steps.WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 50)) - .Then(x => x.ThenTheTwoServicesShouldHaveBeenCalledTimes(50)) - .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(24, 26)) - .BDDfy(); - } + this.Given(x => x.GivenProductServiceOneIsRunning(downstreamServiceOneUrl, 200)) + .And(x => x.GivenProductServiceTwoIsRunning(downstreamServiceTwoUrl, 200)) + .And(x => _steps.GivenThereIsAConfiguration(configuration)) + .And(x => _steps.GivenOcelotIsRunning()) + .When(x => _steps.WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 50)) + .Then(x => x.ThenTheTwoServicesShouldHaveBeenCalledTimes(50)) - [Fact] - public void should_load_balance_request_with_round_robin() - { - var downstreamPortOne = PortFinder.GetRandomPort(); - var downstreamPortTwo = PortFinder.GetRandomPort(); - var downstreamServiceOneUrl = $"http://localhost:{downstreamPortOne}"; - var downstreamServiceTwoUrl = $"http://localhost:{downstreamPortTwo}"; + // Quite risky assertion because the actual values based on health checks and threading + //.And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(24, 26)) + .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(1, 49)) + .BDDfy(); + } - var configuration = new FileConfiguration - { - Routes = new List + [Fact] + public void Should_load_balance_request_with_round_robin() + { + var downstreamPortOne = PortFinder.GetRandomPort(); + var downstreamPortTwo = PortFinder.GetRandomPort(); + var downstreamServiceOneUrl = $"http://localhost:{downstreamPortOne}"; + var downstreamServiceTwoUrl = $"http://localhost:{downstreamPortTwo}"; + + var configuration = new FileConfiguration + { + Routes = new List + { + new() { - new() + DownstreamPathTemplate = "/", + DownstreamScheme = "http", + UpstreamPathTemplate = "/", + UpstreamHttpMethod = new List { "Get" }, + LoadBalancerOptions = new FileLoadBalancerOptions { Type = nameof(RoundRobin) }, + DownstreamHostAndPorts = new List { - DownstreamPathTemplate = "/", - DownstreamScheme = "http", - UpstreamPathTemplate = "/", - UpstreamHttpMethod = new List { "Get" }, - LoadBalancerOptions = new FileLoadBalancerOptions { Type = nameof(RoundRobin) }, - DownstreamHostAndPorts = new List + new() + { + Host = "localhost", + Port = downstreamPortOne, + }, + new() { - new() - { - Host = "localhost", - Port = downstreamPortOne, - }, - new() - { - Host = "localhost", - Port = downstreamPortTwo, - }, + Host = "localhost", + Port = downstreamPortTwo, }, }, }, - GlobalConfiguration = new FileGlobalConfiguration(), - }; + }, + GlobalConfiguration = new FileGlobalConfiguration(), + }; - this.Given(x => x.GivenProductServiceOneIsRunning(downstreamServiceOneUrl, 200)) - .And(x => x.GivenProductServiceTwoIsRunning(downstreamServiceTwoUrl, 200)) - .And(x => _steps.GivenThereIsAConfiguration(configuration)) - .And(x => _steps.GivenOcelotIsRunning()) - .When(x => _steps.WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 50)) - .Then(x => x.ThenTheTwoServicesShouldHaveBeenCalledTimes(50)) - .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(24, 26)) - .BDDfy(); - } + this.Given(x => x.GivenProductServiceOneIsRunning(downstreamServiceOneUrl, 200)) + .And(x => x.GivenProductServiceTwoIsRunning(downstreamServiceTwoUrl, 200)) + .And(x => _steps.GivenThereIsAConfiguration(configuration)) + .And(x => _steps.GivenOcelotIsRunning()) + .When(x => _steps.WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 50)) + .Then(x => x.ThenTheTwoServicesShouldHaveBeenCalledTimes(50)) - [Fact] - public void should_load_balance_request_with_custom_load_balancer() - { - var downstreamPortOne = PortFinder.GetRandomPort(); - var downstreamPortTwo = PortFinder.GetRandomPort(); - var downstreamServiceOneUrl = $"http://localhost:{downstreamPortOne}"; - var downstreamServiceTwoUrl = $"http://localhost:{downstreamPortTwo}"; + // Quite risky assertion because the actual values based on health checks and threading + //.And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(24, 26)) + .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(1, 49)) + .BDDfy(); + } - var configuration = new FileConfiguration - { - Routes = new List + [Fact] + public void Should_load_balance_request_with_custom_load_balancer() + { + var downstreamPortOne = PortFinder.GetRandomPort(); + var downstreamPortTwo = PortFinder.GetRandomPort(); + var downstreamServiceOneUrl = $"http://localhost:{downstreamPortOne}"; + var downstreamServiceTwoUrl = $"http://localhost:{downstreamPortTwo}"; + + var configuration = new FileConfiguration + { + Routes = new List + { + new() { - new() + DownstreamPathTemplate = "/", + DownstreamScheme = "http", + UpstreamPathTemplate = "/", + UpstreamHttpMethod = new List { "Get" }, + LoadBalancerOptions = new FileLoadBalancerOptions { Type = nameof(CustomLoadBalancer) }, + DownstreamHostAndPorts = new List { - DownstreamPathTemplate = "/", - DownstreamScheme = "http", - UpstreamPathTemplate = "/", - UpstreamHttpMethod = new List { "Get" }, - LoadBalancerOptions = new FileLoadBalancerOptions { Type = nameof(CustomLoadBalancer) }, - DownstreamHostAndPorts = new List + new() + { + Host = "localhost", + Port = downstreamPortOne, + }, + new() { - new() - { - Host = "localhost", - Port = downstreamPortOne, - }, - new() - { - Host = "localhost", - Port = downstreamPortTwo, - }, + Host = "localhost", + Port = downstreamPortTwo, }, }, }, - GlobalConfiguration = new FileGlobalConfiguration(), - }; + }, + GlobalConfiguration = new FileGlobalConfiguration(), + }; - Func loadBalancerFactoryFunc = (serviceProvider, route, serviceDiscoveryProvider) => new CustomLoadBalancer(serviceDiscoveryProvider.GetAsync); + Func loadBalancerFactoryFunc = (serviceProvider, route, serviceDiscoveryProvider) => new CustomLoadBalancer(serviceDiscoveryProvider.GetAsync); - this.Given(x => x.GivenProductServiceOneIsRunning(downstreamServiceOneUrl, 200)) - .And(x => x.GivenProductServiceTwoIsRunning(downstreamServiceTwoUrl, 200)) - .And(x => _steps.GivenThereIsAConfiguration(configuration)) - .And(x => _steps.GivenOcelotIsRunningWithCustomLoadBalancer(loadBalancerFactoryFunc)) - .When(x => _steps.WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 50)) - .Then(x => x.ThenTheTwoServicesShouldHaveBeenCalledTimes(50)) - .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(24, 26)) - .BDDfy(); - } + this.Given(x => x.GivenProductServiceOneIsRunning(downstreamServiceOneUrl, 200)) + .And(x => x.GivenProductServiceTwoIsRunning(downstreamServiceTwoUrl, 200)) + .And(x => _steps.GivenThereIsAConfiguration(configuration)) + .And(x => _steps.GivenOcelotIsRunningWithCustomLoadBalancer(loadBalancerFactoryFunc)) + .When(x => _steps.WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 50)) + .Then(x => x.ThenTheTwoServicesShouldHaveBeenCalledTimes(50)) - private class CustomLoadBalancer : ILoadBalancer - { - private readonly Func>> _services; - private readonly object _lock = new(); + // Quite risky assertion because the actual values based on health checks and threading + //.And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(24, 26)) + .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(1, 49)) + .BDDfy(); + } - private int _last; + private class CustomLoadBalancer : ILoadBalancer + { + private readonly Func>> _services; + private readonly object _lock = new(); - public CustomLoadBalancer(Func>> services) - { - _services = services; - } + private int _last; + + public CustomLoadBalancer(Func>> services) + { + _services = services; + } - public async Task> Lease(HttpContext httpContext) + public async Task> Lease(HttpContext httpContext) + { + var services = await _services(); + lock (_lock) { - var services = await _services(); - lock (_lock) + if (_last >= services.Count) { - if (_last >= services.Count) - { - _last = 0; - } - - var next = services[_last]; - _last++; - return new OkResponse(next.HostAndPort); + _last = 0; } - } - public void Release(ServiceHostAndPort hostAndPort) - { + var next = services[_last]; + _last++; + return new OkResponse(next.HostAndPort); } } - private void ThenBothServicesCalledRealisticAmountOfTimes(int bottom, int top) + public void Release(ServiceHostAndPort hostAndPort) { - _counterOne.ShouldBeInRange(bottom, top); - _counterOne.ShouldBeInRange(bottom, top); } + } - private void ThenTheTwoServicesShouldHaveBeenCalledTimes(int expected) - { - var total = _counterOne + _counterTwo; - total.ShouldBe(expected); - } + private void ThenBothServicesCalledRealisticAmountOfTimes(int bottom, int top) + { + _counterOne.ShouldBeInRange(bottom, top); + _counterTwo.ShouldBeInRange(bottom, top); + } + + private void ThenTheTwoServicesShouldHaveBeenCalledTimes(int expected) + { + var total = _counterOne + _counterTwo; + total.ShouldBe(expected); + } - private void GivenProductServiceOneIsRunning(string url, int statusCode) + private void GivenProductServiceOneIsRunning(string url, int statusCode) + { + _serviceHandler.GivenThereIsAServiceRunningOn(url, async context => { - _serviceHandler.GivenThereIsAServiceRunningOn(url, async context => + try { - try - { - string response; - lock (SyncLock) - { - _counterOne++; - response = _counterOne.ToString(); - } - - context.Response.StatusCode = statusCode; - await context.Response.WriteAsync(response); - } - catch (Exception exception) + string response; + lock (SyncLock) { - await context.Response.WriteAsync(exception.StackTrace); + _counterOne++; + response = _counterOne.ToString(); } - }); - } - private void GivenProductServiceTwoIsRunning(string url, int statusCode) - { - _serviceHandler.GivenThereIsAServiceRunningOn(url, async context => + context.Response.StatusCode = statusCode; + await context.Response.WriteAsync(response); + } + catch (Exception exception) { - try - { - string response; - lock (SyncLock) - { - _counterTwo++; - response = _counterTwo.ToString(); - } + await context.Response.WriteAsync(exception.StackTrace); + } + }); + } - context.Response.StatusCode = statusCode; - await context.Response.WriteAsync(response); - } - catch (Exception exception) + private void GivenProductServiceTwoIsRunning(string url, int statusCode) + { + _serviceHandler.GivenThereIsAServiceRunningOn(url, async context => + { + try + { + string response; + lock (SyncLock) { - await context.Response.WriteAsync(exception.StackTrace); + _counterTwo++; + response = _counterTwo.ToString(); } - }); - } - public void Dispose() - { - _serviceHandler?.Dispose(); - _steps.Dispose(); - } + context.Response.StatusCode = statusCode; + await context.Response.WriteAsync(response); + } + catch (Exception exception) + { + await context.Response.WriteAsync(exception.StackTrace); + } + }); + } + + public void Dispose() + { + _serviceHandler?.Dispose(); + _steps.Dispose(); } } diff --git a/test/Ocelot.AcceptanceTests/Properties/GlobalSuppressions.cs b/test/Ocelot.AcceptanceTests/Properties/GlobalSuppressions.cs new file mode 100644 index 000000000..903242484 --- /dev/null +++ b/test/Ocelot.AcceptanceTests/Properties/GlobalSuppressions.cs @@ -0,0 +1,9 @@ +// This file is used by Code Analysis to maintain SuppressMessage +// attributes that are applied to this project. +// Project-level suppressions either have no target or are given +// a specific target and scoped to a namespace, type, member, etc. + +using System.Diagnostics.CodeAnalysis; + +[assembly: SuppressMessage("StyleCop.CSharp.ReadabilityRules", "SA1132:Do not combine fields", Justification = "Has no much sense in test projects", Scope = "namespaceanddescendants", Target = "~N:Ocelot.AcceptanceTests")] +[assembly: SuppressMessage("StyleCop.CSharp.LayoutRules", "SA1513:Closing brace should be followed by blank line", Justification = "Has no much sense in test projects", Scope = "namespaceanddescendants", Target = "~N:Ocelot.AcceptanceTests")] diff --git a/test/Ocelot.AcceptanceTests/ServiceDiscovery/ConsulServiceDiscoveryTests.cs b/test/Ocelot.AcceptanceTests/ServiceDiscovery/ConsulServiceDiscoveryTests.cs index b98f93c0d..dce57ba0f 100644 --- a/test/Ocelot.AcceptanceTests/ServiceDiscovery/ConsulServiceDiscoveryTests.cs +++ b/test/Ocelot.AcceptanceTests/ServiceDiscovery/ConsulServiceDiscoveryTests.cs @@ -1,6 +1,7 @@ using Consul; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Net.Http.Headers; using Newtonsoft.Json; using Ocelot.Configuration.File; using Ocelot.DependencyInjection; @@ -13,7 +14,10 @@ namespace Ocelot.AcceptanceTests.ServiceDiscovery; -public sealed class ConsulServiceDiscoveryTests : Steps, IDisposable +/// +/// Tests for the provider. +/// +public sealed partial class ConsulServiceDiscoveryTests : Steps, IDisposable { private readonly List _consulServices; private readonly List _consulNodes; @@ -63,18 +67,23 @@ public void Should_use_consul_service_discovery_and_load_balance_request() .And(x => GivenOcelotIsRunningWithConsul()) .When(x => WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 50)) .Then(x => x.ThenTheTwoServicesShouldHaveBeenCalledTimes(50)) - .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(24, 26)) + + // Quite risky assertion because the actual values based on health checks and threading + .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(1, 49)) //(24, 26)) .BDDfy(); } + private static readonly string[] VersionV1Tags = new[] { "version-v1" }; + private static readonly string[] GetVsOptionsMethods = new[] { "Get", "Options" }; + [Fact] public void Should_handle_request_to_consul_for_downstream_service_and_make_request() { const string serviceName = "web"; var consulPort = PortFinder.GetRandomPort(); var servicePort = PortFinder.GetRandomPort(); - var serviceEntryOne = GivenServiceEntry(servicePort, "localhost", "web_90_0_2_224_8080", new[] { "version-v1" }, serviceName); - var route = GivenRoute("/api/home", "/home", serviceName, httpMethods: new[] { "Get", "Options" }); + var serviceEntryOne = GivenServiceEntry(servicePort, "localhost", "web_90_0_2_224_8080", VersionV1Tags, serviceName); + var route = GivenRoute("/api/home", "/home", serviceName, httpMethods: GetVsOptionsMethods); var configuration = GivenServiceDiscovery(consulPort, route); this.Given(x => x.GivenThereIsAServiceRunningOn(DownstreamUrl(servicePort), "/api/home", HttpStatusCode.OK, "Hello from Laura")) .And(x => x.GivenThereIsAFakeConsulServiceDiscoveryProvider(DownstreamUrl(consulPort))) @@ -93,7 +102,7 @@ public void Should_handle_request_to_consul_for_downstream_service_and_make_requ const string serviceName = "web"; var consulPort = PortFinder.GetRandomPort(); var servicePort = PortFinder.GetRandomPort(); - var serviceEntry = GivenServiceEntry(servicePort, "localhost", "web_90_0_2_224_8080", new[] { "version-v1" }, serviceName); + var serviceEntry = GivenServiceEntry(servicePort, "localhost", "web_90_0_2_224_8080", VersionV1Tags, serviceName); var configuration = GivenServiceDiscovery(consulPort); configuration.GlobalConfiguration.DownstreamScheme = "http"; @@ -137,7 +146,9 @@ public void Should_use_consul_service_discovery_and_load_balance_request_no_re_r .And(x => GivenOcelotIsRunningWithConsul()) .When(x => WhenIGetUrlOnTheApiGatewayMultipleTimes($"/{serviceName}/", 50)) .Then(x => x.ThenTheTwoServicesShouldHaveBeenCalledTimes(50)) - .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(24, 26)) + + // Quite risky assertion because the actual values based on health checks and threading + .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(1, 49)) //(24, 26)) .BDDfy(); } @@ -148,8 +159,8 @@ public void Should_use_token_to_make_request_to_consul() const string token = "abctoken"; var consulPort = PortFinder.GetRandomPort(); var servicePort = PortFinder.GetRandomPort(); - var serviceEntry = GivenServiceEntry(servicePort, "localhost", "web_90_0_2_224_8080", new[] { "version-v1" }, serviceName); - var route = GivenRoute("/api/home", "/home", serviceName, httpMethods: new[] { "Get", "Options" }); + var serviceEntry = GivenServiceEntry(servicePort, "localhost", "web_90_0_2_224_8080", VersionV1Tags, serviceName); + var route = GivenRoute("/api/home", "/home", serviceName, httpMethods: GetVsOptionsMethods); var configuration = GivenServiceDiscovery(consulPort, route); configuration.GlobalConfiguration.ServiceDiscoveryProvider.Token = token; @@ -185,7 +196,7 @@ public void Should_send_request_to_service_after_it_becomes_available_in_consul( .And(x => GivenOcelotIsRunningWithConsul()) .And(x => WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 10)) .And(x => x.ThenTheTwoServicesShouldHaveBeenCalledTimes(10)) - .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(4, 6)) + .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(1, 9)) //(4, 6)) .And(x => WhenIRemoveAService(serviceEntry2)) .And(x => GivenIResetCounters()) .And(x => WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 10)) @@ -194,7 +205,9 @@ public void Should_send_request_to_service_after_it_becomes_available_in_consul( .And(x => GivenIResetCounters()) .When(x => WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 10)) .Then(x => x.ThenTheTwoServicesShouldHaveBeenCalledTimes(10)) - .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(4, 6)) + + // Quite risky assertion because the actual values based on health checks and threading + .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(1, 9)) //(4, 6)) .BDDfy(); } @@ -204,8 +217,8 @@ public void Should_handle_request_to_poll_consul_for_downstream_service_and_make const string serviceName = "web"; var consulPort = PortFinder.GetRandomPort(); var servicePort = PortFinder.GetRandomPort(); - var serviceEntry = GivenServiceEntry(servicePort, "localhost", $"web_90_0_2_224_{servicePort}", new[] { "version-v1" }, serviceName); - var route = GivenRoute("/api/home", "/home", serviceName, httpMethods: new[] { "Get", "Options" }); + var serviceEntry = GivenServiceEntry(servicePort, "localhost", $"web_90_0_2_224_{servicePort}", VersionV1Tags, serviceName); + var route = GivenRoute("/api/home", "/home", serviceName, httpMethods: GetVsOptionsMethods); var configuration = GivenServiceDiscovery(consulPort, route); var sd = configuration.GlobalConfiguration.ServiceDiscoveryProvider; @@ -238,6 +251,7 @@ public void Should_use_consul_service_discovery_based_on_upstream_host(string lo // UpstreamHost is used to determine which ServiceName to use when making a request to Consul (e.g. Host: us-shop goes to product-us) const string serviceNameUS = "product-us"; const string serviceNameEU = "product-eu"; + string[] tagsUS = new[] { "US" }, tagsEU = new[] { "EU" }; var consulPort = PortFinder.GetRandomPort(); var servicePortUS = PortFinder.GetRandomPort(); var servicePortEU = PortFinder.GetRandomPort(); @@ -247,34 +261,37 @@ public void Should_use_consul_service_discovery_based_on_upstream_host(string lo var publicUrlEU = $"http://{upstreamHostEU}"; const string responseBodyUS = "Phone chargers with US plug"; const string responseBodyEU = "Phone chargers with EU plug"; - var serviceEntryUS = GivenServiceEntry(servicePortUS, serviceName: serviceNameUS, tags: new[] { "US" }); - var serviceEntryEU = GivenServiceEntry(servicePortEU, serviceName: serviceNameEU, tags: new[] { "EU" }); + var serviceEntryUS = GivenServiceEntry(servicePortUS, serviceName: serviceNameUS, tags: tagsUS); + var serviceEntryEU = GivenServiceEntry(servicePortEU, serviceName: serviceNameEU, tags: tagsEU); var routeUS = GivenRoute("/products", "/", serviceNameUS, loadBalancerType, upstreamHostUS); var routeEU = GivenRoute("/products", "/", serviceNameEU, loadBalancerType, upstreamHostEU); var configuration = GivenServiceDiscovery(consulPort, routeUS, routeEU); + bool isStickySession = loadBalancerType == nameof(CookieStickySessions); + var sessionCookieUS = isStickySession ? new CookieHeaderValue(routeUS.LoadBalancerOptions.Key, Guid.NewGuid().ToString()) : null; + var sessionCookieEU = isStickySession ? new CookieHeaderValue(routeEU.LoadBalancerOptions.Key, Guid.NewGuid().ToString()) : null; // Ocelot request for http://us-shop/ should find 'product-us' in Consul, call /products and return "Phone chargers with US plug" // Ocelot request for http://eu-shop/ should find 'product-eu' in Consul, call /products and return "Phone chargers with EU plug" this.Given(x => x._serviceHandler.GivenThereIsAServiceRunningOn(DownstreamUrl(servicePortUS), "/products", MapGet("/products", responseBodyUS))) - .And(x => x._serviceHandler2.GivenThereIsAServiceRunningOn(DownstreamUrl(servicePortEU), "/products", MapGet("/products", responseBodyEU))) + .Given(x => x._serviceHandler2.GivenThereIsAServiceRunningOn(DownstreamUrl(servicePortEU), "/products", MapGet("/products", responseBodyEU))) .And(x => x.GivenThereIsAFakeConsulServiceDiscoveryProvider(DownstreamUrl(consulPort))) .And(x => x.GivenTheServicesAreRegisteredWithConsul(serviceEntryUS, serviceEntryEU)) .And(x => GivenThereIsAConfiguration(configuration)) .And(x => GivenOcelotIsRunningWithConsul(publicUrlUS, publicUrlEU)) - .When(x => WhenIGetUrlOnTheApiGateway(publicUrlUS), "When I get US shop for the first time") + .When(x => x.WhenIGetUrl(publicUrlUS, sessionCookieUS), "When I get US shop for the first time") .Then(x => x.ThenConsulShouldHaveBeenCalledTimes(1)) .And(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) .And(x => ThenTheResponseBodyShouldBe(responseBodyUS)) - .When(x => WhenIGetUrlOnTheApiGateway(publicUrlEU), "When I get EU shop for the first time") + .When(x => x.WhenIGetUrl(publicUrlEU, sessionCookieEU), "When I get EU shop for the first time") .Then(x => x.ThenConsulShouldHaveBeenCalledTimes(2)) .And(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) .And(x => ThenTheResponseBodyShouldBe(responseBodyEU)) - .When(x => WhenIGetUrlOnTheApiGateway(publicUrlUS), "When I get US shop again") - .Then(x => x.ThenConsulShouldHaveBeenCalledTimes(3)) + .When(x => x.WhenIGetUrl(publicUrlUS, sessionCookieUS), "When I get US shop again") + .Then(x => x.ThenConsulShouldHaveBeenCalledTimes(isStickySession ? 2 : 3)) // sticky sessions use cache, so Consul shouldn't be called .And(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) .And(x => ThenTheResponseBodyShouldBe(responseBodyUS)) - .When(x => WhenIGetUrlOnTheApiGateway(publicUrlEU), "When I get EU shop again") - .Then(x => x.ThenConsulShouldHaveBeenCalledTimes(4)) + .When(x => x.WhenIGetUrl(publicUrlEU, sessionCookieEU), "When I get EU shop again") + .Then(x => x.ThenConsulShouldHaveBeenCalledTimes(isStickySession ? 2 : 4)) // sticky sessions use cache, so Consul shouldn't be called .And(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) .And(x => ThenTheResponseBodyShouldBe(responseBodyEU)) .BDDfy(); @@ -285,6 +302,7 @@ public void Should_use_consul_service_discovery_based_on_upstream_host(string lo public void Should_return_service_address_by_overridden_service_builder_when_there_is_a_node() { const string serviceName = "OpenTestService"; + string[] methods = new[] { HttpMethods.Post, HttpMethods.Get }; var consulPort = PortFinder.GetRandomPort(); var servicePort = PortFinder.GetRandomPort(); // 9999 var serviceEntry = GivenServiceEntry(servicePort, @@ -293,7 +311,7 @@ public void Should_return_service_address_by_overridden_service_builder_when_the tags: new[] { serviceName }); var serviceNode = new Node() { Name = "n1" }; // cornerstone of the bug serviceEntry.Node = serviceNode; - var route = GivenRoute("/api/{url}", "/open/{url}", serviceName, httpMethods: new[] { "POST", "GET" }); + var route = GivenRoute("/api/{url}", "/open/{url}", serviceName, httpMethods: methods); var configuration = GivenServiceDiscovery(consulPort, route); this.Given(x => x.GivenThereIsAServiceRunningOn(DownstreamUrl(servicePort), "/api/home", HttpStatusCode.OK, "Hello from Raman")) @@ -341,7 +359,7 @@ public MyConsulServiceBuilder(Func configurationFac }, }; - private static FileRoute GivenRoute(string downstream = null, string upstream = null, [CallerMemberName] string serviceName = null, string loadBalancerType = null, string upstreamHost = null, string[] httpMethods = null) => new() + private FileRoute GivenRoute(string downstream = null, string upstream = null, [CallerMemberName] string serviceName = null, string loadBalancerType = null, string upstreamHost = null, string[] httpMethods = null) => new() { DownstreamPathTemplate = downstream ?? "/", DownstreamScheme = Uri.UriSchemeHttp, @@ -349,7 +367,12 @@ public MyConsulServiceBuilder(Func configurationFac UpstreamHttpMethod = httpMethods != null ? new(httpMethods) : new() { HttpMethods.Get }, UpstreamHost = upstreamHost, ServiceName = serviceName, - LoadBalancerOptions = new() { Type = loadBalancerType ?? nameof(LeastConnection) }, + LoadBalancerOptions = new() + { + Type = loadBalancerType ?? nameof(LeastConnection), + Key = serviceName, + Expiry = 60_000, + }, }; private static FileConfiguration GivenServiceDiscovery(int consulPort, params FileRoute[] routes) @@ -365,6 +388,14 @@ private static FileConfiguration GivenServiceDiscovery(int consulPort, params Fi return config; } + private void WhenIGetUrl(string url, CookieHeaderValue cookie) + { + var t = cookie != null + ? WhenIGetUrlOnTheApiGateway(url, cookie) + : WhenIGetUrl(url); + _response = t.Result; + } + private void ThenTheTokenIs(string token) { _receivedToken.ShouldBe(token); @@ -396,7 +427,7 @@ private void GivenIResetCounters() private void ThenBothServicesCalledRealisticAmountOfTimes(int bottom, int top) { _counterOne.ShouldBeInRange(bottom, top); - _counterOne.ShouldBeInRange(bottom, top); + _counterTwo.ShouldBeInRange(bottom, top); } private void ThenTheTwoServicesShouldHaveBeenCalledTimes(int expected) @@ -408,6 +439,13 @@ private void ThenTheTwoServicesShouldHaveBeenCalledTimes(int expected) private void GivenTheServicesAreRegisteredWithConsul(params ServiceEntry[] serviceEntries) => _consulServices.AddRange(serviceEntries); private void GivenTheServiceNodesAreRegisteredWithConsul(params Node[] nodes) => _consulNodes.AddRange(nodes); +#if NET7_0_OR_GREATER + [GeneratedRegex("/v1/health/service/(?[^/]+)")] + private static partial Regex ServiceNameRegex(); +#else + private static readonly Regex ServiceNameRegexVar = new("/v1/health/service/(?[^/]+)"); + private static Regex ServiceNameRegex() => ServiceNameRegexVar; +#endif private void GivenThereIsAFakeConsulServiceDiscoveryProvider(string url) { _consulHandler.GivenThereIsAServiceRunningOn(url, async context => @@ -418,7 +456,7 @@ private void GivenThereIsAFakeConsulServiceDiscoveryProvider(string url) } // Parse the request path to get the service name - var pathMatch = Regex.Match(context.Request.Path.Value, "/v1/health/service/(?[^/]+)"); + var pathMatch = ServiceNameRegex().Match(context.Request.Path.Value); if (pathMatch.Success) { _counterConsul++; diff --git a/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs b/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs index 5ca22da6e..99fe18f14 100644 --- a/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs +++ b/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs @@ -4,11 +4,18 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Newtonsoft.Json; +using Ocelot.Configuration; using Ocelot.Configuration.File; using Ocelot.DependencyInjection; using Ocelot.LoadBalancer.LoadBalancers; +using Ocelot.Logging; using Ocelot.Provider.Kubernetes; +using Ocelot.Provider.Kubernetes.Interfaces; +using Ocelot.ServiceDiscovery.Providers; +using Ocelot.Values; +using System.Collections.Concurrent; using System.Runtime.CompilerServices; +using System.Text; namespace Ocelot.AcceptanceTests.ServiceDiscovery; @@ -16,13 +23,13 @@ public sealed class KubernetesServiceDiscoveryTests : Steps, IDisposable { private readonly string _kubernetesUrl; private readonly IKubeApiClient _clientFactory; - private readonly ServiceHandler _serviceHandler; + private readonly List _serviceHandlers; private readonly ServiceHandler _kubernetesHandler; private string _receivedToken; public KubernetesServiceDiscoveryTests() { - _kubernetesUrl = DownstreamUrl(PortFinder.GetRandomPort()); //5567 + _kubernetesUrl = DownstreamUrl(PortFinder.GetRandomPort()); var option = new KubeClientOptions { ApiEndPoint = new Uri(_kubernetesUrl), @@ -31,13 +38,13 @@ public KubernetesServiceDiscoveryTests() AllowInsecure = true, }; _clientFactory = KubeApiClient.Create(option); - _serviceHandler = new ServiceHandler(); - _kubernetesHandler = new ServiceHandler(); + _serviceHandlers = new(); + _kubernetesHandler = new(); } public override void Dispose() { - _serviceHandler.Dispose(); + _serviceHandlers.ForEach(handler => handler?.Dispose()); _kubernetesHandler.Dispose(); base.Dispose(); } @@ -48,31 +55,21 @@ public void ShouldReturnServicesFromK8s() const string namespaces = nameof(KubernetesServiceDiscoveryTests); const string serviceName = nameof(ShouldReturnServicesFromK8s); var servicePort = PortFinder.GetRandomPort(); - var downstreamUrl = DownstreamUrl(servicePort); + var downstreamUrl = LoopbackLocalhostUrl(servicePort); var downstream = new Uri(downstreamUrl); - var subsetV1 = new EndpointSubsetV1(); - subsetV1.Addresses.Add(new() - { - Ip = Dns.GetHostAddresses(downstream.Host).Select(x => x.ToString()).First(a => a.Contains('.')), - Hostname = downstream.Host, - }); - subsetV1.Ports.Add(new() - { - Name = downstream.Scheme, - Port = servicePort, - }); + var subsetV1 = GivenSubsetAddress(downstream); var endpoints = GivenEndpoints(subsetV1); var route = GivenRouteWithServiceName(namespaces); var configuration = GivenKubeConfiguration(namespaces, route); var downstreamResponse = serviceName; - this.Given(x => GivenK8sProductServiceOneIsRunning(downstreamUrl, downstreamResponse)) - .And(x => GivenThereIsAFakeKubernetesProvider(serviceName, namespaces, endpoints)) - .And(x => GivenThereIsAConfiguration(configuration)) - .And(x => x.GivenOcelotIsRunningWithKubernetes()) - .When(x => WhenIGetUrlOnTheApiGateway("/")) - .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) - .And(_ => ThenTheResponseBodyShouldBe(downstreamResponse)) - .And(_ => ThenTheTokenIs("Bearer txpc696iUhbVoudg164r93CxDTrKRVWG")) + this.Given(x => x.GivenK8sProductServiceIsRunning(downstreamUrl, downstreamResponse)) + .And(x => x.GivenThereIsAFakeKubernetesProvider(endpoints, serviceName, namespaces)) + .And(_ => GivenThereIsAConfiguration(configuration)) + .And(_ => GivenOcelotIsRunningWithServices(WithKubernetes)) + .When(_ => WhenIGetUrlOnTheApiGateway("/")) + .Then(_ => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) + .And(_ => ThenTheResponseBodyShouldBe($"1:{downstreamResponse}")) + .And(x => x.ThenTheTokenIs("Bearer txpc696iUhbVoudg164r93CxDTrKRVWG")) .BDDfy(); } @@ -85,27 +82,18 @@ public void ShouldReturnServicesByPortNameAsDownstreamScheme(string downstreamSc const string serviceName = "example-web"; const string namespaces = "default"; var servicePort = PortFinder.GetRandomPort(); - var downstreamUrl = DownstreamUrl(servicePort); + var downstreamUrl = LoopbackLocalhostUrl(servicePort); var downstream = new Uri(downstreamUrl); + var subsetV1 = GivenSubsetAddress(downstream); - var subsetV1 = new EndpointSubsetV1(); - subsetV1.Addresses.Add(new() - { - Ip = Dns.GetHostAddresses(downstream.Host).Select(x => x.ToString()).First(a => a.Contains('.')), - Hostname = downstream.Host, - }); - subsetV1.Ports.Add(new() + // Ports[0] -> port(https, 443) + // Ports[1] -> port(http, not 80) + subsetV1.Ports.Insert(0, new() { Name = "https", // This service instance is offline -> BadGateway Port = 443, }); - subsetV1.Ports.Add(new() - { - Name = downstream.Scheme, // http, should be real scheme - Port = downstream.Port, // not 80, should be real port - }); var endpoints = GivenEndpoints(subsetV1); - var route = GivenRouteWithServiceName(namespaces); route.DownstreamPathTemplate = "/{url}"; route.DownstreamScheme = downstreamScheme; // !!! Warning !!! Select port by name as scheme @@ -113,18 +101,106 @@ public void ShouldReturnServicesByPortNameAsDownstreamScheme(string downstreamSc route.ServiceName = serviceName; // "example-web" var configuration = GivenKubeConfiguration(namespaces, route); - this.Given(x => GivenK8sProductServiceOneIsRunning(downstreamUrl, nameof(ShouldReturnServicesByPortNameAsDownstreamScheme))) - .And(x => GivenThereIsAFakeKubernetesProvider(serviceName, namespaces, endpoints)) - .And(x => GivenThereIsAConfiguration(configuration)) - .And(x => x.GivenOcelotIsRunningWithKubernetes()) - .When(x => WhenIGetUrlOnTheApiGateway("/api/example/1")) - .Then(x => ThenTheStatusCodeShouldBe(statusCode)) + this.Given(x => x.GivenK8sProductServiceIsRunning(downstreamUrl, nameof(ShouldReturnServicesByPortNameAsDownstreamScheme))) + .And(x => x.GivenThereIsAFakeKubernetesProvider(endpoints, serviceName, namespaces)) + .And(_ => GivenThereIsAConfiguration(configuration)) + .And(_ => GivenOcelotIsRunningWithServices(WithKubernetes)) + .When(_ => WhenIGetUrlOnTheApiGateway("/api/example/1")) + .Then(_ => ThenTheStatusCodeShouldBe(statusCode)) .And(_ => ThenTheResponseBodyShouldBe(downstreamScheme == "http" - ? nameof(ShouldReturnServicesByPortNameAsDownstreamScheme) : string.Empty)) - .And(_ => ThenTheTokenIs("Bearer txpc696iUhbVoudg164r93CxDTrKRVWG")) + ? "1:" + nameof(ShouldReturnServicesByPortNameAsDownstreamScheme) + : string.Empty)) + .And(x => x.ThenTheTokenIs("Bearer txpc696iUhbVoudg164r93CxDTrKRVWG")) .BDDfy(); } + [Theory] + [Trait("Bug", "2110")] + [InlineData(1, 30)] + [InlineData(2, 50)] + [InlineData(3, 50)] + [InlineData(4, 50)] + [InlineData(5, 50)] + [InlineData(6, 99)] + [InlineData(7, 99)] + [InlineData(8, 99)] + [InlineData(9, 999)] + [InlineData(10, 999)] + public void ShouldHighlyLoadOnStableKubeProvider_WithRoundRobinLoadBalancing(int totalServices, int totalRequests) + { + const int ZeroGeneration = 0; + var (endpoints, servicePorts) = ArrangeHighLoadOnKubeProviderAndRoundRobinBalancer(totalServices); + GivenThereIsAFakeKubernetesProvider(endpoints); // stable, services will not be removed from the list + + HighlyLoadOnKubeProviderAndRoundRobinBalancer(totalRequests, ZeroGeneration); + + int bottom = totalRequests / totalServices, + top = totalRequests - (bottom * totalServices) + bottom; + ThenAllServicesCalledRealisticAmountOfTimes(bottom, top); + ThenServiceCountersShouldMatchLeasingCounters(servicePorts); + } + + [Theory] + [Trait("Bug", "2110")] + [InlineData(5, 50, 1)] + [InlineData(5, 50, 2)] + [InlineData(5, 50, 3)] + [InlineData(5, 50, 4)] + public void ShouldHighlyLoadOnUnstableKubeProvider_WithRoundRobinLoadBalancing(int totalServices, int totalRequests, int k8sGeneration) + { + int failPerThreads = (totalRequests / k8sGeneration) - 1; // k8sGeneration means number of offline services + var (endpoints, servicePorts) = ArrangeHighLoadOnKubeProviderAndRoundRobinBalancer(totalServices); + GivenThereIsAFakeKubernetesProvider(endpoints, false, k8sGeneration, failPerThreads); // false means unstable, k8sGeneration services will be removed from the list + + HighlyLoadOnKubeProviderAndRoundRobinBalancer(totalRequests, k8sGeneration); + + int bottom = _roundRobinAnalyzer.BottomOfConnections(), + top = _roundRobinAnalyzer.TopOfConnections(); + ThenAllServicesCalledRealisticAmountOfTimes(bottom, top); // with unstable checkings + ThenServiceCountersShouldMatchLeasingCounters(servicePorts); + } + + private (EndpointsV1 Endpoints, int[] ServicePorts) ArrangeHighLoadOnKubeProviderAndRoundRobinBalancer( + int totalServices, + [CallerMemberName] string serviceName = nameof(ArrangeHighLoadOnKubeProviderAndRoundRobinBalancer)) + { + const string namespaces = nameof(KubernetesServiceDiscoveryTests); + var servicePorts = Enumerable.Repeat(0, totalServices) + .Select(_ => PortFinder.GetRandomPort()) + .ToArray(); + var downstreamUrls = servicePorts + .Select(port => LoopbackLocalhostUrl(port, Array.IndexOf(servicePorts, port))) + .ToList(); // based on localhost aka loopback network interface + var downstreams = downstreamUrls.Select(url => new Uri(url)) + .ToList(); + var downstreamResponses = downstreams + .Select(ds => $"{serviceName}:{ds.Host}:{ds.Port}") + .ToList(); + var subset = new EndpointSubsetV1(); + downstreams.ForEach(ds => GivenSubsetAddress(ds, subset)); + var endpoints = GivenEndpoints(subset, serviceName); // totalServices service instances with different ports + var route = GivenRouteWithServiceName(namespaces, serviceName, nameof(RoundRobinAnalyzer)); // !!! + var configuration = GivenKubeConfiguration(namespaces, route); + GivenMultipleK8sProductServicesAreRunning(downstreamUrls, downstreamResponses); + GivenThereIsAConfiguration(configuration); + GivenOcelotIsRunningWithServices(WithKubernetesAndRoundRobin); + return (endpoints, servicePorts); + } + + private void HighlyLoadOnKubeProviderAndRoundRobinBalancer(int totalRequests, int k8sGenerationNo) + { + // Act + WhenIGetUrlOnTheApiGatewayMultipleTimes("/", totalRequests); // load by X parallel requests + + // Assert + _k8sCounter.ShouldBeGreaterThanOrEqualTo(totalRequests); // integration endpoint called times + _k8sServiceGeneration.ShouldBe(k8sGenerationNo); + ThenAllStatusCodesShouldBe(HttpStatusCode.OK); + ThenAllServicesShouldHaveBeenCalledTimes(totalRequests); + _roundRobinAnalyzer.ShouldNotBeNull().Analyze(); + _roundRobinAnalyzer.HasManyServiceGenerations(k8sGenerationNo).ShouldBeTrue(); + } + private void ThenTheTokenIs(string token) { _receivedToken.ShouldBe(token); @@ -146,16 +222,35 @@ private EndpointsV1 GivenEndpoints(EndpointSubsetV1 subset, [CallerMemberName] s return e; } - private FileRoute GivenRouteWithServiceName(string serviceNamespace, [CallerMemberName] string serviceName = null) => new() + private static EndpointSubsetV1 GivenSubsetAddress(Uri downstream, EndpointSubsetV1 subset = null) { - DownstreamPathTemplate = "/", - DownstreamScheme = Uri.UriSchemeHttp, - UpstreamPathTemplate = "/", - UpstreamHttpMethod = new() { HttpMethods.Get }, - ServiceName = serviceName, - ServiceNamespace = serviceNamespace, - LoadBalancerOptions = new() { Type = nameof(LeastConnection) }, - }; + subset ??= new(); + subset.Addresses.Add(new() + { + Ip = Dns.GetHostAddresses(downstream.Host).Select(x => x.ToString()).First(a => a.Contains('.')), // 127.0.0.1 + Hostname = downstream.Host, + }); + subset.Ports.Add(new() + { + Name = downstream.Scheme, + Port = downstream.Port, + }); + return subset; + } + + private FileRoute GivenRouteWithServiceName(string serviceNamespace, + [CallerMemberName] string serviceName = null, + string loadBalancerType = nameof(LeastConnection)) + => new() + { + DownstreamPathTemplate = "/", + DownstreamScheme = null, // the scheme should not be defined in service discovery scenarios by default, only ServiceName + UpstreamPathTemplate = "/", + UpstreamHttpMethod = new() { HttpMethods.Get }, + ServiceName = serviceName, // !!! + ServiceNamespace = serviceNamespace, + LoadBalancerOptions = new() { Type = loadBalancerType }, + }; private FileConfiguration GivenKubeConfiguration(string serviceNamespace, params FileRoute[] routes) { @@ -173,11 +268,39 @@ private FileConfiguration GivenKubeConfiguration(string serviceNamespace, params return configuration; } - private void GivenThereIsAFakeKubernetesProvider(string serviceName, string namespaces, EndpointsV1 endpoints) - => _kubernetesHandler.GivenThereIsAServiceRunningOn(_kubernetesUrl, async context => + private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, + [CallerMemberName] string serviceName = nameof(KubernetesServiceDiscoveryTests), string namespaces = nameof(KubernetesServiceDiscoveryTests)) + => GivenThereIsAFakeKubernetesProvider(endpoints, true, 0, 0, serviceName, namespaces); + + private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, bool isStable, int offlineServicesNo, int offlinePerThreads, + [CallerMemberName] string serviceName = nameof(KubernetesServiceDiscoveryTests), string namespaces = nameof(KubernetesServiceDiscoveryTests)) + { + _k8sCounter = 0; + _kubernetesHandler.GivenThereIsAServiceRunningOn(_kubernetesUrl, async context => { + await Task.Delay(Random.Shared.Next(1, 10)); // emulate integration delay up to 10 milliseconds if (context.Request.Path.Value == $"/api/v1/namespaces/{namespaces}/endpoints/{serviceName}") { + // Each offlinePerThreads-th request to integrated K8s endpoint should fail + lock (K8sCounterLocker) + { + _k8sCounter++; + var subset = endpoints.Subsets[0]; + if (!isStable && _k8sCounter % offlinePerThreads == 0 && _k8sCounter >= offlinePerThreads) + { + while (offlineServicesNo-- > 0) + { + int index = subset.Addresses.Count - 1; // Random.Shared.Next(0, subset.Addresses.Count - 1); + subset.Addresses.RemoveAt(index); + subset.Ports.RemoveAt(index); + } + + _k8sServiceGeneration++; + } + + endpoints.Metadata.Generation = _k8sServiceGeneration; + } + if (context.Request.Headers.TryGetValue("Authorization", out var values)) { _receivedToken = values.First(); @@ -188,25 +311,236 @@ private void GivenThereIsAFakeKubernetesProvider(string serviceName, string name await context.Response.WriteAsync(json); } }); + } + + private void WithKubernetes(IServiceCollection services) => services + .AddOcelot().AddKubernetes() + .Services.RemoveAll().AddSingleton(_clientFactory); + + private void WithKubernetesAndRoundRobin(IServiceCollection services) => services + .AddOcelot().AddKubernetes() + .AddCustomLoadBalancer(GetRoundRobinAnalyzer) + .Services + .RemoveAll().AddSingleton(_clientFactory) + .RemoveAll().AddSingleton(); + + private RoundRobinAnalyzer _roundRobinAnalyzer; + private RoundRobinAnalyzer GetRoundRobinAnalyzer(DownstreamRoute route, IServiceDiscoveryProvider provider) + { + lock (K8sCounterLocker) + { + return _roundRobinAnalyzer ??= new RoundRobinAnalyzer(provider.GetAsync, route.ServiceName); + } + } + + private static readonly object ServiceCountersLocker = new(); + private Dictionary _serviceCounters; + + private static readonly object K8sCounterLocker = new(); + private int _k8sCounter, _k8sServiceGeneration; + + private void GivenK8sProductServiceIsRunning(string url, string response) + { + _serviceHandlers.Add(new()); // allocate single instance + _serviceCounters = new(); // single counter + GivenK8sProductServiceIsRunning(url, response, 0); + _serviceCounters[0] = 0; + } + + private void GivenMultipleK8sProductServicesAreRunning(List urls, List responses) + { + urls.ForEach(_ => _serviceHandlers.Add(new())); // allocate multiple instances + _serviceCounters = new(urls.Count); // multiple counters + for (int i = 0; i < urls.Count; i++) + { + GivenK8sProductServiceIsRunning(urls[i], responses[i], i); + _serviceCounters[i] = 0; + } + } - private void GivenOcelotIsRunningWithKubernetes() - => GivenOcelotIsRunningWithServices(s => + private void GivenK8sProductServiceIsRunning(string url, string response, int handlerIndex) + { + var serviceHandler = _serviceHandlers[handlerIndex]; + serviceHandler.GivenThereIsAServiceRunningOn(url, async context => { - s.AddOcelot().AddKubernetes(); - s.RemoveAll().AddSingleton(_clientFactory); + await Task.Delay(Random.Shared.Next(5, 15)); // emulate integration delay up to 15 milliseconds + int count = 0; + lock (ServiceCountersLocker) + { + count = ++_serviceCounters[handlerIndex]; + } + + context.Response.StatusCode = (int)HttpStatusCode.OK; + var threadResponse = string.Concat(count, ':', response); + await context.Response.WriteAsync(threadResponse ?? ((int)HttpStatusCode.OK).ToString()); }); + } + + private void ThenAllServicesShouldHaveBeenCalledTimes(int expected) + { + var sortedByIndex = _serviceCounters.OrderBy(_ => _.Key).Select(_ => _.Value).ToArray(); + var customMessage = $"All values are [{string.Join(',', sortedByIndex)}]"; + _serviceCounters.Sum(_ => _.Value).ShouldBe(expected, customMessage); + _roundRobinAnalyzer.Events.Count.ShouldBe(expected); + } + + private void ThenAllServicesCalledRealisticAmountOfTimes(int bottom, int top) + { + var sortedByIndex = _serviceCounters.OrderBy(_ => _.Key).Select(_ => _.Value).ToArray(); + var customMessage = $"{nameof(bottom)}: {bottom}\n {nameof(top)}: {top}\n All values are [{string.Join(',', sortedByIndex)}]"; + int sum = 0, totalSum = _serviceCounters.Sum(_ => _.Value); - private void GivenK8sProductServiceOneIsRunning(string url, string response) - => _serviceHandler.GivenThereIsAServiceRunningOn(url, async context => + // Last services cannot be called at all, zero counters + for (int i = 0; i < _serviceCounters.Count && sum < totalSum; i++) + { + int actual = _serviceCounters[i]; + actual.ShouldBeInRange(bottom, top, customMessage); + sum += actual; + } + } + + private void ThenServiceCountersShouldMatchLeasingCounters(int[] ports) + { + var leasingCounters = _roundRobinAnalyzer.GetHostCounters(); + for (int i = 0; i < ports.Length; i++) { - try + var host = leasingCounters.Keys.FirstOrDefault(k => k.DownstreamPort == ports[i]); + if (host != null) // leasing info/counters can be absent because of offline service instance with exact port in unstable scenario { - context.Response.StatusCode = (int)HttpStatusCode.OK; - await context.Response.WriteAsync(response ?? nameof(HttpStatusCode.OK)); + int counter1 = _serviceCounters[i]; + int counter2 = leasingCounters[host]; + counter1.ShouldBe(counter2, $"Port: {ports[i]}\n Host: {host}"); } - catch (Exception exception) + } + } +} + +internal class FakeKubeServiceCreator : KubeServiceCreator +{ + public FakeKubeServiceCreator(IOcelotLoggerFactory factory) : base(factory) { } + + protected override ServiceHostAndPort GetServiceHostAndPort(KubeRegistryConfiguration configuration, EndpointsV1 endpoint, EndpointSubsetV1 subset, EndpointAddressV1 address) + { + //return base.GetServiceHostAndPort(configuration, endpoint, subset, address); + var ports = subset.Ports; + var index = subset.Addresses.IndexOf(address); + var portV1 = ports[index]; + Logger.LogDebug(() => $"K8s service with key '{configuration.KeyOfServiceInK8s}' and address {address.Ip}; Detected port is {portV1.Name}:{portV1.Port}. Total {ports.Count} ports of [{string.Join(',', ports.Select(p => p.Name))}]."); + return new ServiceHostAndPort(address.Ip, portV1.Port, portV1.Name); + } + + protected override IEnumerable GetServiceTags(KubeRegistryConfiguration configuration, EndpointsV1 endpoint, EndpointSubsetV1 subset, EndpointAddressV1 address) + { + var tags = base.GetServiceTags(configuration, endpoint, subset, address) + .ToList(); + long gen = endpoint.Metadata.Generation ?? 0L; + tags.Add($"{nameof(endpoint.Metadata.Generation)}:{gen}"); + return tags; + } +} + +internal class RoundRobinAnalyzer : RoundRobin, ILoadBalancer +{ + public readonly ConcurrentBag Events = new(); + + public RoundRobinAnalyzer(Func>> services, string serviceName) + : base(services, serviceName) + { + this.Leased += Me_Leased; + } + + private void Me_Leased(object sender, LeaseEventArgs e) => Events.Add(e); + + public const string GenerationPrefix = nameof(EndpointsV1.Metadata.Generation) + ":"; + + public object Analyze() + { + var allGenerations = Events + .Select(e => e.Service.Tags.FirstOrDefault(t => t.StartsWith(GenerationPrefix))) + .Distinct().ToArray(); + var allIndices = Events.Select(e => e.ServiceIndex) + .Distinct().ToArray(); + + Dictionary> eventsPerGeneration = new(); + foreach (var generation in allGenerations) + { + var l = Events.Where(e => e.Service.Tags.Contains(generation)).ToList(); + eventsPerGeneration.Add(generation, l); + } + + Dictionary> generationIndices = new(); + foreach (var generation in allGenerations) + { + var l = eventsPerGeneration[generation].Select(e => e.ServiceIndex).Distinct().ToList(); + generationIndices.Add(generation, l); + } + + Dictionary> generationLeases = new(); + foreach (var generation in allGenerations) + { + var l = eventsPerGeneration[generation].Select(e => e.Lease).ToList(); + generationLeases.Add(generation, l); + } + + Dictionary> generationHosts = new(); + foreach (var generation in allGenerations) + { + var l = eventsPerGeneration[generation].Select(e => e.Lease.HostAndPort).Distinct().ToList(); + generationHosts.Add(generation, l); + } + + Dictionary> generationLeasesWithMaxConnections = new(); + foreach (var generation in allGenerations) + { + List leases = new(); + var uniqueHosts = generationHosts[generation]; + foreach (var host in uniqueHosts) { - await context.Response.WriteAsync(exception.StackTrace); + int max = generationLeases[generation].Where(l => l == host).Max(l => l.Connections); + Lease wanted = generationLeases[generation].Find(l => l == host && l.Connections == max); + leases.Add(wanted); } - }); + + leases = leases.OrderBy(l => l.HostAndPort.DownstreamPort).ToList(); + generationLeasesWithMaxConnections.Add(generation, leases); + } + + return generationLeasesWithMaxConnections; + } + + public bool HasManyServiceGenerations(int maxGeneration) + { + int[] generations = new int[maxGeneration + 1]; + string[] tags = new string[maxGeneration + 1]; + for (int i = 0; i < generations.Length; i++) + { + generations[i] = i; + tags[i] = GenerationPrefix + i; + } + + var all = Events + .Select(e => e.Service.Tags.FirstOrDefault(t => t.StartsWith(GenerationPrefix))) + .Distinct().ToArray(); + return all.All(tags.Contains); + } + + public Dictionary GetHostCounters() + { + var hosts = Events.Select(e => e.Lease.HostAndPort).Distinct().ToList(); + return Events + .GroupBy(e => e.Lease.HostAndPort) + .ToDictionary(g => g.Key, g => g.Max(e => e.Lease.Connections)); + } + + public int BottomOfConnections() + { + var hostCounters = GetHostCounters(); + return hostCounters.Min(_ => _.Value); + } + + public int TopOfConnections() + { + var hostCounters = GetHostCounters(); + return hostCounters.Max(_ => _.Value); + } } diff --git a/test/Ocelot.AcceptanceTests/Steps.cs b/test/Ocelot.AcceptanceTests/Steps.cs index 4591e9b22..501e741d4 100644 --- a/test/Ocelot.AcceptanceTests/Steps.cs +++ b/test/Ocelot.AcceptanceTests/Steps.cs @@ -2,6 +2,7 @@ using IdentityServer4.AccessTokenValidation; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.TestHost; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -27,9 +28,9 @@ using Ocelot.Tracing.OpenTracing; using Serilog; using Serilog.Core; +using System.Collections.Concurrent; using System.IO.Compression; using System.Net.Http.Headers; -using System.Security.Policy; using System.Text; using static Ocelot.AcceptanceTests.HttpDelegatingHandlersTests; using ConfigurationBuilder = Microsoft.Extensions.Configuration.ConfigurationBuilder; @@ -42,7 +43,8 @@ public class Steps : IDisposable { protected TestServer _ocelotServer; protected HttpClient _ocelotClient; - private HttpResponseMessage _response; + protected HttpResponseMessage _response; + protected ConcurrentDictionary _parallelResponses; private HttpContent _postContent; private BearerToken _token; public string RequestIdKey = "OcRequestId"; @@ -58,20 +60,32 @@ public Steps() { _random = new Random(); _testId = Guid.NewGuid(); - _ocelotConfigFileName = $"{_testId:N}-{ConfigurationBuilderExtensions.PrimaryConfigFile}"; + _ocelotConfigFileName = $"{_testId:N}-{ConfigurationBuilderExtensions.PrimaryConfigFile}"; Files = new() { _ocelotConfigFileName }; Folders = new(); + _parallelResponses = new(); } protected List Files { get; } protected List Folders { get; } protected string TestID { get => _testId.ToString("N"); } + protected static FileHostAndPort Localhost(int port) => new("localhost", port); protected static string DownstreamUrl(int port) => $"{Uri.UriSchemeHttp}://localhost:{port}"; + protected static string LoopbackLocalhostUrl(int port, int loopbackIndex = 0) => $"{Uri.UriSchemeHttp}://127.0.0.{++loopbackIndex}:{port}"; protected static FileConfiguration GivenConfiguration(params FileRoute[] routes) => new() { Routes = new(routes), + }; + + protected static FileRoute GivenDefaultRoute(int port) => new() + { + DownstreamPathTemplate = "/", + DownstreamHostAndPorts = new() { Localhost(port) }, + DownstreamScheme = Uri.UriSchemeHttp, + UpstreamPathTemplate = "/", + UpstreamHttpMethod = new() { HttpMethods.Get }, }; public async Task ThenConfigShouldBe(FileConfiguration fileConfig) @@ -171,11 +185,11 @@ public async Task StartFakeOcelotWithWebSocketsWithConsul() await _ocelotHost.StartAsync(); } - public void GivenThereIsAConfiguration(FileConfiguration fileConfiguration) + public void GivenThereIsAConfiguration(FileConfiguration fileConfiguration) => GivenThereIsAConfiguration(fileConfiguration, _ocelotConfigFileName); public void GivenThereIsAConfiguration(FileConfiguration from, string toFile) - { + { toFile ??= _ocelotConfigFileName; var jsonConfiguration = JsonConvert.SerializeObject(from, Formatting.Indented); File.WriteAllText(toFile, jsonConfiguration); @@ -214,12 +228,12 @@ protected virtual void DeleteFolders() f.Delete(true); } } - catch (Exception e) - { - Console.WriteLine(e); - } + catch (Exception e) + { + Console.WriteLine(e); + } } - } + } public void ThenTheResponseBodyHeaderIs(string key, string value) { @@ -476,7 +490,7 @@ public void GivenOcelotIsRunningUsingJsonSerializedCache() _ocelotClient = _ocelotServer.CreateClient(); } - internal void GivenIWait(int wait) => Thread.Sleep(wait); + public static void GivenIWait(int wait) => Thread.Sleep(wait); public void GivenOcelotIsRunningWithMiddlewareBeforePipeline(Func callback) { @@ -619,13 +633,33 @@ public void GivenOcelotIsRunningWithGlobalHandlersRegisteredInDi(FakeDepen _ocelotServer = new TestServer(_webHostBuilder); _ocelotClient = _ocelotServer.CreateClient(); - } - - internal void GivenIAddCookieToMyRequest(string cookie) - { - _ocelotClient.DefaultRequestHeaders.Add("Set-Cookie", cookie); - } - + } + + // # + // # Cookies helpers + // # + public void GivenIAddCookieToMyRequest(string cookie) + => _ocelotClient.DefaultRequestHeaders.Add("Set-Cookie", cookie); + public async Task WhenIGetUrlOnTheApiGatewayWithCookie(string url, string cookie, string value) + => _response = await WhenIGetUrlOnTheApiGateway(url, cookie, value); + public async Task WhenIGetUrlOnTheApiGatewayWithCookie(string url, CookieHeaderValue cookie) + => _response = await WhenIGetUrlOnTheApiGateway(url, cookie); + + public Task WhenIGetUrlOnTheApiGateway(string url, string cookie, string value) + { + var header = new CookieHeaderValue(cookie, value); + return WhenIGetUrlOnTheApiGateway(url, header); + } + + public Task WhenIGetUrlOnTheApiGateway(string url, CookieHeaderValue cookie) + { + var requestMessage = new HttpRequestMessage(HttpMethod.Get, url); + requestMessage.Headers.Add("Cookie", cookie.ToString()); + return _ocelotClient.SendAsync(requestMessage); + } + + // END of Cookies helpers + /// /// This is annoying cos it should be in the constructor but we need to set up the file before calling startup so its a step. /// @@ -812,15 +846,11 @@ public void GivenOcelotIsRunningWithEureka() public void GivenOcelotIsRunningWithPolly() => GivenOcelotIsRunningWithServices(WithPolly); public static void WithPolly(IServiceCollection services) => services.AddOcelot().AddPolly(); - public void WhenIGetUrlOnTheApiGateway(string url) - { - _response = _ocelotClient.GetAsync(url).Result; - } + public void WhenIGetUrlOnTheApiGateway(string url) + => _response = _ocelotClient.GetAsync(url).Result; - public void WhenIGetUrlOnTheApiGatewayAndDontWait(string url) - { - _ocelotClient.GetAsync(url); - } + public Task WhenIGetUrl(string url) + => _ocelotClient.GetAsync(url); public void WhenIGetUrlWithBodyOnTheApiGateway(string url, string body) { @@ -845,11 +875,6 @@ public void WhenIGetUrlWithFormOnTheApiGateway(string url, string name, IEnumera _response = _ocelotClient.SendAsync(request).Result; } - public void WhenICancelTheRequest() - { - _ocelotClient.CancelPendingRequests(); - } - public void WhenIGetUrlOnTheApiGateway(string url, HttpContent content) { var httpRequestMessage = new HttpRequestMessage(HttpMethod.Get, url) { Content = content }; @@ -862,62 +887,37 @@ public void WhenIPostUrlOnTheApiGateway(string url, HttpContent content) _response = _ocelotClient.SendAsync(httpRequestMessage).Result; } - public void WhenIGetUrlOnTheApiGateway(string url, string cookie, string value) - { - var request = _ocelotServer.CreateRequest(url); - request.And(x => { x.Headers.Add("Cookie", new CookieHeaderValue(cookie, value).ToString()); }); - var response = request.GetAsync().Result; - _response = response; - } - public void GivenIAddAHeader(string key, string value) { _ocelotClient.DefaultRequestHeaders.TryAddWithoutValidation(key, value); } - public void WhenIGetUrlOnTheApiGatewayMultipleTimes(string url, int times) - { - var tasks = new Task[times]; - - for (var i = 0; i < times; i++) - { - var urlCopy = url; - tasks[i] = GetForServiceDiscoveryTest(urlCopy); - Thread.Sleep(_random.Next(40, 60)); - } - - Task.WaitAll(tasks); - } - - public void WhenIGetUrlOnTheApiGatewayMultipleTimes(string url, int times, string cookie, string value) + public Task[] WhenIGetUrlOnTheApiGatewayMultipleTimes(string url, int times) { var tasks = new Task[times]; - + _parallelResponses = new(times, times); for (var i = 0; i < times; i++) { - tasks[i] = GetForServiceDiscoveryTest(url, cookie, value); - Thread.Sleep(_random.Next(40, 60)); + tasks[i] = GetParallelResponse(url, i); + _parallelResponses[i] = null; } - Task.WaitAll(tasks); - } - - private async Task GetForServiceDiscoveryTest(string url, string cookie, string value) - { - var request = _ocelotServer.CreateRequest(url); - request.And(x => { x.Headers.Add("Cookie", new CookieHeaderValue(cookie, value).ToString()); }); - var response = await request.GetAsync(); - var content = await response.Content.ReadAsStringAsync(); - var count = int.Parse(content); - count.ShouldBeGreaterThan(0); + Task.WaitAll(tasks); + return tasks; } - private async Task GetForServiceDiscoveryTest(string url) + private async Task GetParallelResponse(string url, int threadIndex) { - var response = await _ocelotClient.GetAsync(url); - var content = await response.Content.ReadAsStringAsync(); - var count = int.Parse(content); - count.ShouldBeGreaterThan(0); + var response = await _ocelotClient.GetAsync(url); + + //Thread.Sleep(_random.Next(40, 60)); + //var content = await response.Content.ReadAsStringAsync(); + //var counterValue = content.Contains(':') + // ? content.Split(':')[0] // let the first fragment is counter value + // : content; + //int count = int.Parse(counterValue); + //count.ShouldBeGreaterThan(0); + _parallelResponses[threadIndex] = response; } public void WhenIGetUrlOnTheApiGatewayMultipleTimesForRateLimit(string url, int times) @@ -980,10 +980,11 @@ public void ThenTheContentLengthIs(int expected) _response.Content.Headers.ContentLength.ShouldBe(expected); } - public void ThenTheStatusCodeShouldBe(HttpStatusCode expectedHttpStatusCode) - { - _response.StatusCode.ShouldBe(expectedHttpStatusCode); - } + public void ThenTheStatusCodeShouldBe(HttpStatusCode expected) + => _response.StatusCode.ShouldBe(expected); + + public void ThenAllStatusCodesShouldBe(HttpStatusCode expected) + => _parallelResponses.ShouldAllBe(response => response.Value.StatusCode == expected); public void ThenTheStatusCodeShouldBe(int expectedHttpStatusCode) { @@ -1193,6 +1194,12 @@ protected virtual void Dispose(bool disposing) _ocelotClient?.Dispose(); _ocelotServer?.Dispose(); _ocelotHost?.Dispose(); + _response?.Dispose(); + foreach (var response in _parallelResponses) + { + response.Value?.Dispose(); + } + DeleteFiles(); DeleteFolders(); } diff --git a/test/Ocelot.AcceptanceTests/StickySessionsTests.cs b/test/Ocelot.AcceptanceTests/StickySessionsTests.cs index 6592ec83b..cc7435e19 100644 --- a/test/Ocelot.AcceptanceTests/StickySessionsTests.cs +++ b/test/Ocelot.AcceptanceTests/StickySessionsTests.cs @@ -1,289 +1,157 @@ using Microsoft.AspNetCore.Http; using Ocelot.Configuration.File; +using Ocelot.LoadBalancer.LoadBalancers; +using System.Runtime.CompilerServices; -namespace Ocelot.AcceptanceTests +namespace Ocelot.AcceptanceTests; + +public sealed class StickySessionsTests : Steps, IDisposable { - public class StickySessionsTests : IDisposable + private readonly int[] _counters; + private static readonly object SyncLock = new(); + private readonly ServiceHandler[] _handlers; + + public StickySessionsTests() : base() { - private readonly Steps _steps; - private int _counterOne; - private int _counterTwo; - private static readonly object SyncLock = new(); - private readonly ServiceHandler _serviceHandler; + _counters = new int[2]; + _handlers = new ServiceHandler[2]; + } - public StickySessionsTests() + public override void Dispose() + { + foreach (var handler in _handlers) { - _serviceHandler = new ServiceHandler(); - _steps = new Steps(); + handler?.Dispose(); } - [Fact] - public void should_use_same_downstream_host() - { - var downstreamPortOne = PortFinder.GetRandomPort(); - var downstreamPortTwo = PortFinder.GetRandomPort(); - var downstreamServiceOneUrl = $"http://localhost:{downstreamPortOne}"; - var downstreamServiceTwoUrl = $"http://localhost:{downstreamPortTwo}"; - - var configuration = new FileConfiguration - { - Routes = new List - { - new() - { - DownstreamPathTemplate = "/", - DownstreamScheme = "http", - UpstreamPathTemplate = "/", - UpstreamHttpMethod = new List { "Get" }, - LoadBalancerOptions = new FileLoadBalancerOptions - { - Type = "CookieStickySessions", - Key = "sessionid", - Expiry = 300000, - }, - DownstreamHostAndPorts = new List - { - new() - { - Host = "localhost", - Port = downstreamPortOne, - }, - new() - { - Host = "localhost", - Port = downstreamPortTwo, - }, - }, - }, - }, - }; - - this.Given(x => x.GivenProductServiceOneIsRunning(downstreamServiceOneUrl, 200)) - .And(x => x.GivenProductServiceTwoIsRunning(downstreamServiceTwoUrl, 200)) - .And(x => _steps.GivenThereIsAConfiguration(configuration)) - .And(x => _steps.GivenOcelotIsRunning()) - .When(x => _steps.WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 10, "sessionid", "123")) - .Then(x => x.ThenTheFirstServiceIsCalled(10)) - .Then(x => x.ThenTheSecondServiceIsCalled(0)) - .BDDfy(); - } + base.Dispose(); + } - [Fact] - public void should_use_different_downstream_host_for_different_re_route() - { - var downstreamPortOne = PortFinder.GetRandomPort(); - var downstreamPortTwo = PortFinder.GetRandomPort(); - var downstreamServiceOneUrl = $"http://localhost:{downstreamPortOne}"; - var downstreamServiceTwoUrl = $"http://localhost:{downstreamPortTwo}"; + [Fact] + public void ShouldUseSameDownstreamHost_ForSingleRouteWithHighLoad() + { + var port1 = PortFinder.GetRandomPort(); + var port2 = PortFinder.GetRandomPort(); + var route = GivenRoute("/") + .WithHosts(Localhost(port1), Localhost(port2)); + var cookieName = route.LoadBalancerOptions.Key; + var configuration = GivenConfiguration(route); + + this.Given(x => x.GivenProductServiceIsRunning(0, DownstreamUrl(port1))) + .Given(x => x.GivenProductServiceIsRunning(1, DownstreamUrl(port2))) + .And(_ => GivenThereIsAConfiguration(configuration)) + .And(_ => GivenOcelotIsRunning()) + .When(x => x.WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 10, cookieName, Guid.NewGuid().ToString())) + .Then(x => x.ThenServiceShouldHaveBeenCalledTimes(0, 10)) // RoundRobin should return first service with port1 + .Then(x => x.ThenServiceShouldHaveBeenCalledTimes(1, 0)) + .BDDfy(); + } - var configuration = new FileConfiguration - { - Routes = new List - { - new() - { - DownstreamPathTemplate = "/", - DownstreamScheme = "http", - UpstreamPathTemplate = "/", - UpstreamHttpMethod = new List { "Get" }, - LoadBalancerOptions = new FileLoadBalancerOptions - { - Type = "CookieStickySessions", - Key = "sessionid", - Expiry = 300000, - }, - DownstreamHostAndPorts = new List - { - new() - { - Host = "localhost", - Port = downstreamPortOne, - }, - new() - { - Host = "localhost", - Port = downstreamPortTwo, - }, - }, - }, - new() - { - DownstreamPathTemplate = "/", - DownstreamScheme = "http", - UpstreamPathTemplate = "/test", - UpstreamHttpMethod = new List { "Get" }, - LoadBalancerOptions = new FileLoadBalancerOptions - { - Type = "CookieStickySessions", - Key = "bestid", - Expiry = 300000, - }, - DownstreamHostAndPorts = new List - { - new() - { - Host = "localhost", - Port = downstreamPortTwo, - }, - new() - { - Host = "localhost", - Port = downstreamPortOne, - }, - }, - }, - }, - }; + [Fact] + public void ShouldUseDifferentDownstreamHost_ForDoubleRoutesWithDifferentCookies() + { + var port1 = PortFinder.GetRandomPort(); + var port2 = PortFinder.GetRandomPort(); + var route1 = GivenRoute("/") + .WithHosts(Localhost(port1), Localhost(port2)); + var cookieName = route1.LoadBalancerOptions.Key; + var route2 = GivenRoute("/test", cookieName + "bestid") + .WithHosts(Localhost(port2), Localhost(port1)); + var configuration = GivenConfiguration(route1, route2); + + this.Given(x => x.GivenProductServiceIsRunning(0, DownstreamUrl(port1))) + .Given(x => x.GivenProductServiceIsRunning(1, DownstreamUrl(port2))) + .And(_ => GivenThereIsAConfiguration(configuration)) + .And(_ => GivenOcelotIsRunning()) + .When(_ => WhenIGetUrlOnTheApiGatewayWithCookie("/", cookieName, "123")) // both cookies should have different values + .When(_ => WhenIGetUrlOnTheApiGatewayWithCookie("/test", cookieName + "bestid", "123")) // stick by cookie value + .Then(x => x.ThenServiceShouldHaveBeenCalledTimes(0, 1)) + .Then(x => x.ThenServiceShouldHaveBeenCalledTimes(1, 1)) + .BDDfy(); + } - this.Given(x => x.GivenProductServiceOneIsRunning(downstreamServiceOneUrl, 200)) - .And(x => x.GivenProductServiceTwoIsRunning(downstreamServiceTwoUrl, 200)) - .And(x => _steps.GivenThereIsAConfiguration(configuration)) - .And(x => _steps.GivenOcelotIsRunning()) - .When(x => _steps.WhenIGetUrlOnTheApiGateway("/", "sessionid", "123")) - .When(x => _steps.WhenIGetUrlOnTheApiGateway("/test", "bestid", "123")) - .Then(x => x.ThenTheFirstServiceIsCalled(1)) - .Then(x => x.ThenTheSecondServiceIsCalled(1)) - .BDDfy(); - } + [Fact] + public void ShouldUseSameDownstreamHost_ForDifferentRoutesWithSameCookie() + { + var port1 = PortFinder.GetRandomPort(); + var port2 = PortFinder.GetRandomPort(); + var route1 = GivenRoute("/") + .WithHosts(Localhost(port1), Localhost(port2)); + var cookieName = route1.LoadBalancerOptions.Key; + var route2 = GivenRoute("/test", cookieName) + .WithHosts(Localhost(port2), Localhost(port1)); + var configuration = GivenConfiguration(route1, route2); + + this.Given(x => x.GivenProductServiceIsRunning(0, DownstreamUrl(port1))) + .Given(x => x.GivenProductServiceIsRunning(1, DownstreamUrl(port2))) + .And(_ => GivenThereIsAConfiguration(configuration)) + .And(_ => GivenOcelotIsRunning()) + .When(_ => WhenIGetUrlOnTheApiGatewayWithCookie("/", cookieName, "123")) + .When(_ => WhenIGetUrlOnTheApiGatewayWithCookie("/test", cookieName, "123")) + .Then(x => x.ThenServiceShouldHaveBeenCalledTimes(0, 2)) + .Then(x => x.ThenServiceShouldHaveBeenCalledTimes(1, 0)) + .BDDfy(); + } - [Fact] - public void should_use_same_downstream_host_for_different_re_route() + private static FileRoute GivenRoute(string upstream, [CallerMemberName] string cookieName = null) => new() + { + DownstreamPathTemplate = "/", + DownstreamScheme = Uri.UriSchemeHttp, + UpstreamPathTemplate = upstream ?? "/", + UpstreamHttpMethod = new() { HttpMethods.Get }, + LoadBalancerOptions = new() { - var downstreamPortOne = PortFinder.GetRandomPort(); - var downstreamPortTwo = PortFinder.GetRandomPort(); - var downstreamServiceOneUrl = $"http://localhost:{downstreamPortOne}"; - var downstreamServiceTwoUrl = $"http://localhost:{downstreamPortTwo}"; - - var configuration = new FileConfiguration - { - Routes = new List - { - new() - { - DownstreamPathTemplate = "/", - DownstreamScheme = "http", - UpstreamPathTemplate = "/", - UpstreamHttpMethod = new List { "Get" }, - LoadBalancerOptions = new FileLoadBalancerOptions - { - Type = "CookieStickySessions", - Key = "sessionid", - Expiry = 300000, - }, - DownstreamHostAndPorts = new List - { - new() - { - Host = "localhost", - Port = downstreamPortOne, - }, - new() - { - Host = "localhost", - Port = downstreamPortTwo, - }, - }, - }, - new() - { - DownstreamPathTemplate = "/", - DownstreamScheme = "http", - UpstreamPathTemplate = "/test", - UpstreamHttpMethod = new List { "Get" }, - LoadBalancerOptions = new FileLoadBalancerOptions - { - Type = "CookieStickySessions", - Key = "sessionid", - Expiry = 300000, - }, - DownstreamHostAndPorts = new List - { - new() - { - Host = "localhost", - Port = downstreamPortTwo, - }, - new() - { - Host = "localhost", - Port = downstreamPortOne, - }, - }, - }, - }, - }; - - this.Given(x => x.GivenProductServiceOneIsRunning(downstreamServiceOneUrl, 200)) - .And(x => x.GivenProductServiceTwoIsRunning(downstreamServiceTwoUrl, 200)) - .And(x => _steps.GivenThereIsAConfiguration(configuration)) - .And(x => _steps.GivenOcelotIsRunning()) - .When(x => _steps.WhenIGetUrlOnTheApiGateway("/", "sessionid", "123")) - .When(x => _steps.WhenIGetUrlOnTheApiGateway("/test", "sessionid", "123")) - .Then(x => x.ThenTheFirstServiceIsCalled(2)) - .Then(x => x.ThenTheSecondServiceIsCalled(0)) - .BDDfy(); - } + Type = nameof(CookieStickySessions), + Key = cookieName, // !!! + Expiry = 300000, + }, + }; - private void ThenTheFirstServiceIsCalled(int expected) + private Task WhenIGetUrlOnTheApiGatewayMultipleTimes(string url, int times, string cookie, string value) + { + var tasks = new Task[times]; + for (var i = 0; i < times; i++) { - _counterOne.ShouldBe(expected); + tasks[i] = GetParallelTask(url, cookie, value); } - private void ThenTheSecondServiceIsCalled(int expected) - { - _counterTwo.ShouldBe(expected); - } + return Task.WhenAll(tasks); + } - private void GivenProductServiceOneIsRunning(string url, int statusCode) - { - _serviceHandler.GivenThereIsAServiceRunningOn(url, async context => - { - try - { - string response; - lock (SyncLock) - { - _counterOne++; - response = _counterOne.ToString(); - } + private async Task GetParallelTask(string url, string cookie, string value) + { + var response = await WhenIGetUrlOnTheApiGateway(url, cookie, value); + var content = await response.Content.ReadAsStringAsync(); + var count = int.Parse(content); + count.ShouldBeGreaterThan(0); + } - context.Response.StatusCode = statusCode; - await context.Response.WriteAsync(response); - } - catch (Exception exception) - { - await context.Response.WriteAsync(exception.StackTrace); - } - }); - } + private void ThenServiceShouldHaveBeenCalledTimes(int index, int times) + { + _counters[index].ShouldBe(times); + } - private void GivenProductServiceTwoIsRunning(string url, int statusCode) + private void GivenProductServiceIsRunning(int index, string url) + { + _handlers[index] = new(); + _handlers[index].GivenThereIsAServiceRunningOn(url, async context => { - _serviceHandler.GivenThereIsAServiceRunningOn(url, async context => + try { - try + string response; + lock (SyncLock) { - string response; - lock (SyncLock) - { - _counterTwo++; - response = _counterTwo.ToString(); - } - - context.Response.StatusCode = statusCode; - await context.Response.WriteAsync(response); + _counters[index]++; + response = _counters[index].ToString(); } - catch (Exception exception) - { - await context.Response.WriteAsync(exception.StackTrace); - } - }); - } - public void Dispose() - { - _serviceHandler?.Dispose(); - _steps.Dispose(); - } + context.Response.StatusCode = (int)HttpStatusCode.OK; + await context.Response.WriteAsync(response); + } + catch (Exception exception) + { + await context.Response.WriteAsync(exception.StackTrace); + } + }); } } diff --git a/test/Ocelot.Testing/FileRouteExtensions.cs b/test/Ocelot.Testing/FileRouteExtensions.cs new file mode 100644 index 000000000..06c47ce20 --- /dev/null +++ b/test/Ocelot.Testing/FileRouteExtensions.cs @@ -0,0 +1,12 @@ +using Ocelot.Configuration.File; + +namespace Ocelot.Testing; + +public static class FileRouteExtensions +{ + public static FileRoute WithHosts(this FileRoute route, params FileHostAndPort[] hosts) + { + route.DownstreamHostAndPorts.AddRange(hosts); + return route; + } +} diff --git a/test/Ocelot.Testing/Ocelot.Testing.csproj b/test/Ocelot.Testing/Ocelot.Testing.csproj index fa27745f4..b7a4f4253 100644 --- a/test/Ocelot.Testing/Ocelot.Testing.csproj +++ b/test/Ocelot.Testing/Ocelot.Testing.csproj @@ -1,4 +1,4 @@ - + 0.0.0-dev @@ -7,4 +7,8 @@ enable + + + + diff --git a/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs b/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs index 213a25f65..1b3a3f416 100644 --- a/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs +++ b/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs @@ -7,149 +7,171 @@ using Ocelot.Logging; using Ocelot.Provider.Kubernetes; using Ocelot.Provider.Kubernetes.Interfaces; +using Ocelot.Testing; using Ocelot.Values; +using System.Runtime.CompilerServices; -namespace Ocelot.UnitTests.Kubernetes +namespace Ocelot.UnitTests.Kubernetes; + +public class KubeTests { - public class KubeTests : IDisposable + private readonly Mock _factory; + private readonly Mock _logger; + + public KubeTests() { - private IWebHost _fakeKubeBuilder; - private readonly Kube _provider; - private EndpointsV1 _endpointEntries; - private readonly string _serviceName; - private readonly string _namespaces; - private readonly int _port; - private readonly string _kubeHost; - private readonly string _fakekubeServiceDiscoveryUrl; - private List _services; - private string _receivedToken; - private readonly Mock _factory; - private readonly Mock _logger; - private readonly IKubeApiClient _clientFactory; - private readonly Mock _serviceBuilder; - - public KubeTests() - { - _serviceName = "test"; - _namespaces = "dev"; - _port = 5567; - _kubeHost = "localhost"; - _fakekubeServiceDiscoveryUrl = $"{Uri.UriSchemeHttp}://{_kubeHost}:{_port}"; - _endpointEntries = new(); - _factory = new(); - - var option = new KubeClientOptions - { - ApiEndPoint = new Uri(_fakekubeServiceDiscoveryUrl), - AccessToken = "txpc696iUhbVoudg164r93CxDTrKRVWG", - AuthStrategy = KubeAuthStrategy.BearerToken, - AllowInsecure = true, - }; - - _clientFactory = KubeApiClient.Create(option); - _logger = new(); - _factory.Setup(x => x.CreateLogger()).Returns(_logger.Object); - var config = new KubeRegistryConfiguration - { - KeyOfServiceInK8s = _serviceName, - KubeNamespace = _namespaces, - }; - _serviceBuilder = new(); - _provider = new Kube(config, _factory.Object, _clientFactory, _serviceBuilder.Object); - } + _factory = new(); + _logger = new(); + _factory.Setup(x => x.CreateLogger()).Returns(_logger.Object); + } - [Fact] - public void Should_return_service_from_k8s() - { - // Arrange - var token = "Bearer txpc696iUhbVoudg164r93CxDTrKRVWG"; - var endPointEntryOne = new EndpointsV1 - { - Kind = "endpoint", - ApiVersion = "1.0", - Metadata = new ObjectMetaV1 - { - Name = nameof(Should_return_service_from_k8s), - Namespace = "dev", - }, - }; - var endpointSubsetV1 = new EndpointSubsetV1(); - endpointSubsetV1.Addresses.Add(new EndpointAddressV1 - { - Ip = "127.0.0.1", - Hostname = "localhost", - }); - endpointSubsetV1.Ports.Add(new EndpointPortV1 + [Fact] + public async Task Should_return_service_from_k8s() + { + // Arrange + var given = GivenClientAndProvider(out var serviceBuilder); + serviceBuilder.Setup(x => x.BuildServices(It.IsAny(), It.IsAny())) + .Returns(new Service[] { new(nameof(Should_return_service_from_k8s), new("localhost", 80), string.Empty, string.Empty, Array.Empty()) }); + + var endpoints = GivenEndpoints(); + using var kubernetes = GivenThereIsAFakeKubeServiceDiscoveryProvider( + given.ClientOptions.ApiEndPoint.ToString(), + given.ProviderOptions.KubeNamespace, + given.ProviderOptions.KeyOfServiceInK8s, + endpoints, + out Lazy receivedToken); + + // Act + var services = await given.Provider.GetAsync(); + + // Assert + services.ShouldNotBeNull().Count.ShouldBe(1); + receivedToken.Value.ShouldBe($"Bearer {nameof(Should_return_service_from_k8s)}"); + } + + [Fact] + [Trait("Bug", "2110")] + public async Task Should_return_single_service_from_k8s_during_concurrent_calls() + { + // Arrange + var given = GivenClientAndProvider(out var serviceBuilder); + var manualResetEvent = new ManualResetEvent(false); + serviceBuilder.Setup(x => x.BuildServices(It.IsAny(), It.IsAny())) + .Returns(() => { - Port = 80, + manualResetEvent.WaitOne(); + return new Service[] { new(nameof(Should_return_single_service_from_k8s_during_concurrent_calls), new("localhost", 80), string.Empty, string.Empty, Array.Empty()) }; }); - endPointEntryOne.Subsets.Add(endpointSubsetV1); - _serviceBuilder.Setup(x => x.BuildServices(It.IsAny(), It.IsAny())) - .Returns(new Service[] { new(nameof(Should_return_service_from_k8s), new("localhost", 80), string.Empty, string.Empty, new string[0]) }); - GivenThereIsAFakeKubeServiceDiscoveryProvider(_fakekubeServiceDiscoveryUrl, _serviceName, _namespaces); - GivenTheServicesAreRegisteredWithKube(endPointEntryOne); - - // Act - WhenIGetTheServices(); - - // Assert - ThenTheCountIs(1); - ThenTheTokenIs(token); - } - private void ThenTheTokenIs(string token) - { - _receivedToken.ShouldBe(token); - } + var endpoints = GivenEndpoints(); + using var kubernetes = GivenThereIsAFakeKubeServiceDiscoveryProvider( + given.ClientOptions.ApiEndPoint.ToString(), + given.ProviderOptions.KubeNamespace, + given.ProviderOptions.KeyOfServiceInK8s, + endpoints, + out Lazy receivedToken); + + // Act + var services = new List(); + async Task WhenIGetTheServices() => services = await given.Provider.GetAsync(); + var getServiceTasks = Task.WhenAll( + WhenIGetTheServices(), + WhenIGetTheServices()); + manualResetEvent.Set(); + await getServiceTasks; + + // Assert + receivedToken.Value.ShouldBe($"Bearer {nameof(Should_return_single_service_from_k8s_during_concurrent_calls)}"); + services.ShouldNotBeNull().Count.ShouldBe(1); + services.ShouldAllBe(s => s != null); + } - private void ThenTheCountIs(int count) + private (IKubeApiClient Client, KubeClientOptions ClientOptions, Kube Provider, KubeRegistryConfiguration ProviderOptions) + GivenClientAndProvider(out Mock serviceBuilder, string namespaces = null, [CallerMemberName] string serviceName = null) + { + namespaces ??= nameof(KubeTests); + var kubePort = PortFinder.GetRandomPort(); + serviceName ??= "test" + kubePort; + var kubeEndpointUrl = $"{Uri.UriSchemeHttp}://localhost:{kubePort}"; + var options = new KubeClientOptions { - _services.Count.ShouldBe(count); - } + ApiEndPoint = new Uri(kubeEndpointUrl), + AccessToken = serviceName, // "txpc696iUhbVoudg164r93CxDTrKRVWG", + AuthStrategy = KubeAuthStrategy.BearerToken, + AllowInsecure = true, + }; + IKubeApiClient client = KubeApiClient.Create(options); - private void WhenIGetTheServices() + var config = new KubeRegistryConfiguration { - _services = _provider.GetAsync().GetAwaiter().GetResult(); - } + KeyOfServiceInK8s = serviceName, + KubeNamespace = namespaces, + }; + serviceBuilder = new(); + var provider = new Kube(config, _factory.Object, client, serviceBuilder.Object); + return (client, options, provider, config); + } - private void GivenTheServicesAreRegisteredWithKube(EndpointsV1 endpointEntries) + private EndpointsV1 GivenEndpoints( + string namespaces = nameof(KubeTests), + [CallerMemberName] string serviceName = "test") + { + var endpoints = new EndpointsV1 { - _endpointEntries = endpointEntries; - } + Kind = "endpoint", + ApiVersion = "1.0", + Metadata = new ObjectMetaV1 + { + Name = serviceName, + Namespace = namespaces, + }, + }; + var subset = new EndpointSubsetV1(); + subset.Addresses.Add(new EndpointAddressV1 + { + Ip = "127.0.0.1", + Hostname = "localhost", + }); + subset.Ports.Add(new EndpointPortV1 + { + Port = 80, + }); + endpoints.Subsets.Add(subset); + return endpoints; + } + + private IWebHost GivenThereIsAFakeKubeServiceDiscoveryProvider(string url, string namespaces, string serviceName, + EndpointsV1 endpointEntries, out Lazy receivedToken) + { + var token = string.Empty; + receivedToken = new(() => token); - private void GivenThereIsAFakeKubeServiceDiscoveryProvider(string url, string serviceName, string namespaces) + Task ProcessKubernetesRequest(HttpContext context) { - _fakeKubeBuilder = new WebHostBuilder() - .UseUrls(url) - .UseKestrel() - .UseContentRoot(Directory.GetCurrentDirectory()) - .UseIISIntegration() - .UseUrls(url) - .Configure(app => + if (context.Request.Path.Value == $"/api/v1/namespaces/{namespaces}/endpoints/{serviceName}") + { + if (context.Request.Headers.TryGetValue("Authorization", out var values)) { - app.Run(async context => - { - if (context.Request.Path.Value == $"/api/v1/namespaces/{namespaces}/endpoints/{serviceName}") - { - if (context.Request.Headers.TryGetValue("Authorization", out var values)) - { - _receivedToken = values.First(); - } - - var json = JsonConvert.SerializeObject(_endpointEntries); - context.Response.Headers.Append("Content-Type", "application/json"); - await context.Response.WriteAsync(json); - } - }); - }) - .Build(); - - _fakeKubeBuilder.Start(); - } + token = values.First(); + } - public void Dispose() - { - _fakeKubeBuilder?.Dispose(); + var json = JsonConvert.SerializeObject(endpointEntries); + context.Response.Headers.Append("Content-Type", "application/json"); + return context.Response.WriteAsync(json); + } + + return Task.CompletedTask; } + + var host = new WebHostBuilder() + .UseUrls(url) + .UseKestrel() + .UseContentRoot(Directory.GetCurrentDirectory()) + .UseIISIntegration() + .UseUrls(url) + .Configure(app => app.Run(ProcessKubernetesRequest)) + .Build(); + host.Start(); + return host; } } diff --git a/test/Ocelot.UnitTests/LoadBalancer/CookieStickySessionsTests.cs b/test/Ocelot.UnitTests/LoadBalancer/CookieStickySessionsTests.cs index 71c4d0517..c13155e46 100644 --- a/test/Ocelot.UnitTests/LoadBalancer/CookieStickySessionsTests.cs +++ b/test/Ocelot.UnitTests/LoadBalancer/CookieStickySessionsTests.cs @@ -1,256 +1,272 @@ using Microsoft.AspNetCore.Http; +using Ocelot.Configuration.Builder; using Ocelot.Infrastructure; using Ocelot.LoadBalancer.LoadBalancers; +using Ocelot.Middleware; using Ocelot.Responses; using Ocelot.UnitTests.Responder; using Ocelot.Values; using System.Collections; +using System.Runtime.CompilerServices; -namespace Ocelot.UnitTests.LoadBalancer +namespace Ocelot.UnitTests.LoadBalancer; + +public sealed class CookieStickySessionsTests : UnitTest { - public class CookieStickySessionsTests : UnitTest - { - private readonly CookieStickySessions _stickySessions; - private readonly Mock _loadBalancer; - private readonly int _defaultExpiryInMs; - private Response _result; - private Response _firstHostAndPort; - private Response _secondHostAndPort; - private readonly FakeBus _bus; - private HttpContext _httpContext; - - public CookieStickySessionsTests() - { - _httpContext = new DefaultHttpContext(); - _bus = new FakeBus(); - _loadBalancer = new Mock(); - _defaultExpiryInMs = 0; - _stickySessions = new CookieStickySessions(_loadBalancer.Object, "sessionid", _defaultExpiryInMs, _bus); - } + private readonly CookieStickySessions _stickySessions; + private readonly Mock _loadBalancer; + private readonly int _defaultExpiryInMs; + private Response _result; + private Response _firstHostAndPort; + private Response _secondHostAndPort; + private readonly FakeBus _bus; + private readonly HttpContext _httpContext; + + public CookieStickySessionsTests() + { + _httpContext = new DefaultHttpContext(); + _bus = new FakeBus(); + _loadBalancer = new Mock(); + _defaultExpiryInMs = 0; + _stickySessions = new CookieStickySessions(_loadBalancer.Object, "sessionid", _defaultExpiryInMs, _bus); + } - [Fact] - public void should_expire_sticky_session() - { - this.Given(_ => GivenTheLoadBalancerReturns()) - .And(_ => GivenTheDownstreamRequestHasSessionId("321")) - .And(_ => GivenIHackAMessageInWithAPastExpiry()) - .And(_ => WhenILease()) - .When(_ => WhenTheMessagesAreProcessed()) - .Then(_ => ThenTheLoadBalancerIsCalled()) - .BDDfy(); - } + private void Arrange([CallerMemberName] string serviceName = null) + { + var route = new DownstreamRouteBuilder() + .WithLoadBalancerKey(serviceName) + .Build(); + _httpContext.Items.UpsertDownstreamRoute(route); + } - [Fact] - public void should_return_host_and_port() - { - this.Given(_ => GivenTheLoadBalancerReturns()) - .When(_ => WhenILease()) - .Then(_ => ThenTheHostAndPortIsNotNull()) - .BDDfy(); - } + [Fact] + public async Task Should_expire_sticky_session() + { + Arrange(); + GivenTheLoadBalancerReturns(); + GivenTheDownstreamRequestHasSessionId("321"); + GivenIHackAMessageInWithAPastExpiry(); + await WhenILease(); + WhenTheMessagesAreProcessed(); + ThenTheLoadBalancerIsCalled(); + } - [Fact] - public void should_return_same_host_and_port() - { - this.Given(_ => GivenTheLoadBalancerReturnsSequence()) - .And(_ => GivenTheDownstreamRequestHasSessionId("321")) - .When(_ => WhenILeaseTwiceInARow()) - .Then(_ => ThenTheFirstAndSecondResponseAreTheSame()) - .And(_ => ThenTheStickySessionWillTimeout()) - .BDDfy(); - } + [Fact] + public async Task Should_return_host_and_port() + { + Arrange(); + GivenTheLoadBalancerReturns(); + GivenTheDownstreamRequestHasSessionId("321"); + await WhenILease(); + ThenTheHostAndPortIsNotNull(); + } - [Fact] - public void should_return_different_host_and_port_if_load_balancer_does() - { - this.Given(_ => GivenTheLoadBalancerReturnsSequence()) - .When(_ => WhenIMakeTwoRequetsWithDifferentSessionValues()) - .Then(_ => ThenADifferentHostAndPortIsReturned()) - .BDDfy(); - } + [Fact] + public async Task Should_return_same_host_and_port() + { + Arrange(); + GivenTheLoadBalancerReturnsSequence(); + GivenTheDownstreamRequestHasSessionId("321"); + await WhenILeaseTwiceInARow(); + ThenTheFirstAndSecondResponseAreTheSame(); + ThenTheStickySessionWillTimeout(); + } - [Fact] - public void should_return_error() - { - this.Given(_ => GivenTheLoadBalancerReturnsError()) - .When(_ => WhenILease()) - .Then(_ => ThenAnErrorIsReturned()) - .BDDfy(); - } + [Fact] + public async Task Should_return_different_host_and_port_if_load_balancer_does() + { + Arrange(); + GivenTheLoadBalancerReturnsSequence(); + await WhenIMakeTwoRequetsWithDifferentSessionValues(); + ThenADifferentHostAndPortIsReturned(); + } - [Fact] - public void should_release() - { - _stickySessions.Release(new ServiceHostAndPort(string.Empty, 0)); - } + [Fact] + public async Task Should_return_error() + { + Arrange(); + GivenTheLoadBalancerReturnsError(); + await WhenILease(); + ThenAnErrorIsReturned(); + } - private void ThenTheLoadBalancerIsCalled() - { - _loadBalancer.Verify(x => x.Release(It.IsAny()), Times.Once); - } + [Fact] + public void Should_release() + { + _stickySessions.Release(new ServiceHostAndPort(string.Empty, 0)); + } - private void WhenTheMessagesAreProcessed() - { - _bus.Process(); - } + private void ThenTheLoadBalancerIsCalled() + { + _loadBalancer.Verify(x => x.Release(It.IsAny()), Times.Once); + } - private void GivenIHackAMessageInWithAPastExpiry() - { - var hostAndPort = new ServiceHostAndPort("999", 999); - _bus.Publish(new StickySession(hostAndPort, DateTime.UtcNow.AddDays(-1), "321"), 0); - } + private void WhenTheMessagesAreProcessed() + { + _bus.Process(); + } - private void ThenAnErrorIsReturned() - { - _result.IsError.ShouldBeTrue(); - } + private void GivenIHackAMessageInWithAPastExpiry() + { + var hostAndPort = new ServiceHostAndPort("999", 999); + _bus.Publish(new StickySession(hostAndPort, DateTime.UtcNow.AddDays(-1), "321"), 0); + } - private void GivenTheLoadBalancerReturnsError() - { - _loadBalancer - .Setup(x => x.Lease(It.IsAny())) - .ReturnsAsync(new ErrorResponse(new AnyError())); - } + private void ThenAnErrorIsReturned() + { + _result.IsError.ShouldBeTrue(); + } - private void ThenADifferentHostAndPortIsReturned() - { - _firstHostAndPort.Data.DownstreamHost.ShouldBe("one"); - _firstHostAndPort.Data.DownstreamPort.ShouldBe(80); - _secondHostAndPort.Data.DownstreamHost.ShouldBe("two"); - _secondHostAndPort.Data.DownstreamPort.ShouldBe(80); - } + private void GivenTheLoadBalancerReturnsError() + { + _loadBalancer + .Setup(x => x.Lease(It.IsAny())) + .ReturnsAsync(new ErrorResponse(new AnyError())); + } - private async Task WhenIMakeTwoRequetsWithDifferentSessionValues() - { - var contextOne = new DefaultHttpContext(); - var cookiesOne = new FakeCookies(); - cookiesOne.AddCookie("sessionid", "321"); - contextOne.Request.Cookies = cookiesOne; - var contextTwo = new DefaultHttpContext(); - var cookiesTwo = new FakeCookies(); - cookiesTwo.AddCookie("sessionid", "123"); - contextTwo.Request.Cookies = cookiesTwo; - _firstHostAndPort = await _stickySessions.Lease(contextOne); - _secondHostAndPort = await _stickySessions.Lease(contextTwo); - } + private void ThenADifferentHostAndPortIsReturned() + { + _firstHostAndPort.Data.DownstreamHost.ShouldBe("one"); + _firstHostAndPort.Data.DownstreamPort.ShouldBe(80); + _secondHostAndPort.Data.DownstreamHost.ShouldBe("two"); + _secondHostAndPort.Data.DownstreamPort.ShouldBe(80); + } - private void GivenTheLoadBalancerReturnsSequence() - { - _loadBalancer - .SetupSequence(x => x.Lease(It.IsAny())) - .ReturnsAsync(new OkResponse(new ServiceHostAndPort("one", 80))) - .ReturnsAsync(new OkResponse(new ServiceHostAndPort("two", 80))); - } + private async Task WhenIMakeTwoRequetsWithDifferentSessionValues([CallerMemberName] string serviceName = null) + { + var contextOne = new DefaultHttpContext(); + var cookiesOne = new FakeCookies(); + cookiesOne.AddCookie("sessionid", "321"); + contextOne.Request.Cookies = cookiesOne; + var route = new DownstreamRouteBuilder() + .WithLoadBalancerKey(serviceName) + .Build(); + contextOne.Items.UpsertDownstreamRoute(route); + + var contextTwo = new DefaultHttpContext(); + var cookiesTwo = new FakeCookies(); + cookiesTwo.AddCookie("sessionid", "123"); + contextTwo.Request.Cookies = cookiesTwo; + contextTwo.Items.UpsertDownstreamRoute(route); + + _firstHostAndPort = await _stickySessions.Lease(contextOne); + _secondHostAndPort = await _stickySessions.Lease(contextTwo); + } - private void ThenTheFirstAndSecondResponseAreTheSame() - { - _firstHostAndPort.Data.DownstreamHost.ShouldBe(_secondHostAndPort.Data.DownstreamHost); - _firstHostAndPort.Data.DownstreamPort.ShouldBe(_secondHostAndPort.Data.DownstreamPort); - } + private void GivenTheLoadBalancerReturnsSequence() + { + _loadBalancer + .SetupSequence(x => x.Lease(It.IsAny())) + .ReturnsAsync(new OkResponse(new ServiceHostAndPort("one", 80))) + .ReturnsAsync(new OkResponse(new ServiceHostAndPort("two", 80))); + } - private async Task WhenILeaseTwiceInARow() - { - _firstHostAndPort = await _stickySessions.Lease(_httpContext); - _secondHostAndPort = await _stickySessions.Lease(_httpContext); - } + private void ThenTheFirstAndSecondResponseAreTheSame() + { + _firstHostAndPort.Data.DownstreamHost.ShouldBe(_secondHostAndPort.Data.DownstreamHost); + _firstHostAndPort.Data.DownstreamPort.ShouldBe(_secondHostAndPort.Data.DownstreamPort); + } - private void GivenTheDownstreamRequestHasSessionId(string value) - { - var context = new DefaultHttpContext(); - var cookies = new FakeCookies(); - cookies.AddCookie("sessionid", value); - context.Request.Cookies = cookies; - _httpContext = context; - } + private async Task WhenILeaseTwiceInARow() + { + _firstHostAndPort = await _stickySessions.Lease(_httpContext); + _secondHostAndPort = await _stickySessions.Lease(_httpContext); + } - private void GivenTheLoadBalancerReturns() - { - _loadBalancer - .Setup(x => x.Lease(It.IsAny())) - .ReturnsAsync(new OkResponse(new ServiceHostAndPort(string.Empty, 80))); - } + private void GivenTheDownstreamRequestHasSessionId(string value) + { + var cookies = new FakeCookies(); + cookies.AddCookie("sessionid", value); + _httpContext.Request.Cookies = cookies; + } - private async Task WhenILease() - { - _result = await _stickySessions.Lease(_httpContext); - } + private void GivenTheLoadBalancerReturns() + { + _loadBalancer + .Setup(x => x.Lease(It.IsAny())) + .ReturnsAsync(new OkResponse(new ServiceHostAndPort(string.Empty, 80))); + } - private void ThenTheHostAndPortIsNotNull() - { - _result.Data.ShouldNotBeNull(); - } + private async Task WhenILease() + { + _result = await _stickySessions.Lease(_httpContext); + } - private void ThenTheStickySessionWillTimeout() - { - _bus.Messages.Count.ShouldBe(2); - } + private void ThenTheHostAndPortIsNotNull() + { + _result.Data.ShouldNotBeNull(); } - internal class FakeCookies : IRequestCookieCollection + private void ThenTheStickySessionWillTimeout() { - private readonly Dictionary _cookies = new(); + _bus.Messages.Count.ShouldBe(2); + } +} - public string this[string key] => _cookies[key]; +internal class FakeCookies : IRequestCookieCollection +{ + private readonly Dictionary _cookies = new(); - public int Count => _cookies.Count; + public string this[string key] => _cookies[key]; - public ICollection Keys => _cookies.Keys; + public int Count => _cookies.Count; - public void AddCookie(string key, string value) - { - _cookies[key] = value; - } + public ICollection Keys => _cookies.Keys; - public bool ContainsKey(string key) - { - return _cookies.ContainsKey(key); - } + public void AddCookie(string key, string value) + { + _cookies[key] = value; + } - public IEnumerator> GetEnumerator() - { - return _cookies.GetEnumerator(); - } + public bool ContainsKey(string key) + { + return _cookies.ContainsKey(key); + } - public bool TryGetValue(string key, out string value) - { - return _cookies.TryGetValue(key, out value); - } + public IEnumerator> GetEnumerator() + { + return _cookies.GetEnumerator(); + } - IEnumerator IEnumerable.GetEnumerator() - { - return _cookies.GetEnumerator(); - } + public bool TryGetValue(string key, out string value) + { + return _cookies.TryGetValue(key, out value); } - internal class FakeBus : IBus + IEnumerator IEnumerable.GetEnumerator() { - public FakeBus() - { - Messages = new List(); - Subscriptions = new List>(); - } + return _cookies.GetEnumerator(); + } +} - public List Messages { get; } - public List> Subscriptions { get; } +internal class FakeBus : IBus +{ + public FakeBus() + { + Messages = new List(); + Subscriptions = new List>(); + } - public void Subscribe(Action action) - { - Subscriptions.Add(action); - } + public List Messages { get; } + public List> Subscriptions { get; } - public void Publish(T message, int delay) - { - Messages.Add(message); - } + public void Subscribe(Action action) + { + Subscriptions.Add(action); + } - public void Process() + public void Publish(T message, int delay) + { + Messages.Add(message); + } + + public void Process() + { + foreach (var message in Messages) { - foreach (var message in Messages) + foreach (var subscription in Subscriptions) { - foreach (var subscription in Subscriptions) - { - subscription(message); - } + subscription(message); } } } diff --git a/test/Ocelot.UnitTests/LoadBalancer/LeastConnectionTests.cs b/test/Ocelot.UnitTests/LoadBalancer/LeastConnectionTests.cs index 368e866e6..c0ee460ca 100644 --- a/test/Ocelot.UnitTests/LoadBalancer/LeastConnectionTests.cs +++ b/test/Ocelot.UnitTests/LoadBalancer/LeastConnectionTests.cs @@ -1,4 +1,5 @@ using Microsoft.AspNetCore.Http; +using Ocelot.Errors; using Ocelot.LoadBalancer.LoadBalancers; using Ocelot.Responses; using Ocelot.Values; @@ -21,7 +22,7 @@ public LeastConnectionTests() } [Fact] - public async Task should_be_able_to_lease_and_release_concurrently() + public async Task Should_be_able_to_lease_and_release_concurrently() { var serviceName = "products"; @@ -45,7 +46,7 @@ public async Task should_be_able_to_lease_and_release_concurrently() } [Fact] - public async Task should_handle_service_returning_to_available() + public async Task Should_handle_service_returning_to_available() { var serviceName = "products"; @@ -98,7 +99,7 @@ private async Task LeaseDelayAndRelease() } [Fact] - public void should_get_next_url() + public void Should_get_next_url() { var serviceName = "products"; @@ -117,7 +118,7 @@ public void should_get_next_url() } [Fact] - public async Task should_serve_from_service_with_least_connections() + public async Task Should_serve_from_service_with_least_connections() { var serviceName = "products"; @@ -145,7 +146,7 @@ public async Task should_serve_from_service_with_least_connections() } [Fact] - public async Task should_build_connections_per_service() + public async Task Should_build_connections_per_service() { var serviceName = "products"; @@ -176,7 +177,7 @@ public async Task should_build_connections_per_service() } [Fact] - public async Task should_release_connection() + public async Task Should_release_connection() { var serviceName = "products"; @@ -214,7 +215,7 @@ public async Task should_release_connection() } [Fact] - public void should_return_error_if_services_are_null() + public void Should_return_error_if_services_are_null() { var serviceName = "products"; @@ -222,12 +223,12 @@ public void should_return_error_if_services_are_null() this.Given(x => x.GivenAHostAndPort(hostAndPort)) .And(x => x.GivenTheLoadBalancerStarts(null, serviceName)) .When(x => x.WhenIGetTheNextHostAndPort()) - .Then(x => x.ThenServiceAreNullErrorIsReturned()) + .Then(x => x.ThenErrorIsReturned()) .BDDfy(); } [Fact] - public void should_return_error_if_services_are_empty() + public void Should_return_error_if_services_are_empty() { var serviceName = "products"; @@ -235,20 +236,15 @@ public void should_return_error_if_services_are_empty() this.Given(x => x.GivenAHostAndPort(hostAndPort)) .And(x => x.GivenTheLoadBalancerStarts(new List(), serviceName)) .When(x => x.WhenIGetTheNextHostAndPort()) - .Then(x => x.ThenServiceAreEmptyErrorIsReturned()) + .Then(x => x.ThenErrorIsReturned()) .BDDfy(); } - private void ThenServiceAreNullErrorIsReturned() + private void ThenErrorIsReturned() + where TError : Error { _result.IsError.ShouldBeTrue(); - _result.Errors[0].ShouldBeOfType(); - } - - private void ThenServiceAreEmptyErrorIsReturned() - { - _result.IsError.ShouldBeTrue(); - _result.Errors[0].ShouldBeOfType(); + _result.Errors[0].ShouldBeOfType(); } private void GivenTheLoadBalancerStarts(List services, string serviceName) @@ -257,11 +253,6 @@ private void GivenTheLoadBalancerStarts(List services, string serviceNa _leastConnection = new LeastConnection(() => Task.FromResult(_services), serviceName); } - private void WhenTheLoadBalancerStarts(List services, string serviceName) - { - GivenTheLoadBalancerStarts(services, serviceName); - } - private void GivenAHostAndPort(ServiceHostAndPort hostAndPort) { _hostAndPort = hostAndPort; diff --git a/test/Ocelot.UnitTests/LoadBalancer/RoundRobinTests.cs b/test/Ocelot.UnitTests/LoadBalancer/RoundRobinTests.cs index af55d65aa..1eb8e9957 100644 --- a/test/Ocelot.UnitTests/LoadBalancer/RoundRobinTests.cs +++ b/test/Ocelot.UnitTests/LoadBalancer/RoundRobinTests.cs @@ -2,66 +2,154 @@ using Ocelot.LoadBalancer.LoadBalancers; using Ocelot.Responses; using Ocelot.Values; +using System.Collections.Generic; using System.Diagnostics; +using System.Runtime.CompilerServices; -namespace Ocelot.UnitTests.LoadBalancer +namespace Ocelot.UnitTests.LoadBalancer; + +public class RoundRobinTests : UnitTest { - public class RoundRobinTests : UnitTest - { - private readonly RoundRobin _roundRobin; - private readonly List _services; - private Response _hostAndPort; - private readonly HttpContext _httpContext; + private readonly HttpContext _httpContext; - public RoundRobinTests() - { - _httpContext = new DefaultHttpContext(); - _services = new List - { - new("product", new ServiceHostAndPort("127.0.0.1", 5000), string.Empty, string.Empty, Array.Empty()), - new("product", new ServiceHostAndPort("127.0.0.1", 5001), string.Empty, string.Empty, Array.Empty()), - new("product", new ServiceHostAndPort("127.0.0.1", 5001), string.Empty, string.Empty, Array.Empty()), - }; + public RoundRobinTests() + { + _httpContext = new DefaultHttpContext(); + } - _roundRobin = new RoundRobin(() => Task.FromResult(_services)); - } + [Fact] + public void Lease_LoopThroughIndexRangeOnce_ShouldGetNextAddress() + { + var services = GivenServices(); + var roundRobin = GivenLoadBalancer(services); + WhenIGetTheNextAddress(roundRobin).Data.ShouldNotBeNull().ShouldBe(services[0].HostAndPort); + WhenIGetTheNextAddress(roundRobin).Data.ShouldNotBeNull().ShouldBe(services[1].HostAndPort); + WhenIGetTheNextAddress(roundRobin).Data.ShouldNotBeNull().ShouldBe(services[2].HostAndPort); + } - [Fact] - public void should_get_next_address() + [Fact] + [Trait("Feat", "336")] + public void Lease_LoopThroughIndexRangeIndefinitelyButOneSecond_ShouldGoBackToFirstAddressAfterFinishedLast() + { + var services = GivenServices(); + var roundRobin = GivenLoadBalancer(services); + var stopWatch = Stopwatch.StartNew(); + while (stopWatch.ElapsedMilliseconds < 1000) { - this.Given(x => x.GivenIGetTheNextAddress()) - .Then(x => x.ThenTheNextAddressIndexIs(0)) - .Given(x => x.GivenIGetTheNextAddress()) - .Then(x => x.ThenTheNextAddressIndexIs(1)) - .Given(x => x.GivenIGetTheNextAddress()) - .Then(x => x.ThenTheNextAddressIndexIs(2)) - .BDDfy(); + WhenIGetTheNextAddress(roundRobin).Data.ShouldNotBeNull().ShouldBe(services[0].HostAndPort); + WhenIGetTheNextAddress(roundRobin).Data.ShouldNotBeNull().ShouldBe(services[1].HostAndPort); + WhenIGetTheNextAddress(roundRobin).Data.ShouldNotBeNull().ShouldBe(services[2].HostAndPort); } + } - [Fact] - public async Task should_go_back_to_first_address_after_finished_last() - { - var stopWatch = Stopwatch.StartNew(); + [Fact] + [Trait("Bug", "2110")] + public void Lease_SelectedServiceIsNull_ShouldReturnError() + { + var invalidServices = new List { null }; + var roundRobin = GivenLoadBalancer(invalidServices); + var response = WhenIGetTheNextAddress(roundRobin); + ThenServicesAreNullErrorIsReturned(response); + } - while (stopWatch.ElapsedMilliseconds < 1000) - { - var address = await _roundRobin.Lease(_httpContext); - address.Data.ShouldBe(_services[0].HostAndPort); - address = await _roundRobin.Lease(_httpContext); - address.Data.ShouldBe(_services[1].HostAndPort); - address = await _roundRobin.Lease(_httpContext); - address.Data.ShouldBe(_services[2].HostAndPort); - } + //[InlineData(1, 10)] + //[InlineData(2, 50)] + //[InlineData(3, 50)] + //[InlineData(4, 50)] + //[InlineData(5, 50)] + //[InlineData(3, 100)] + //[InlineData(4, 100)] + //[InlineData(7, 100)] + [InlineData(3, 100)] + [Theory] + [Trait("Feat", "2110")] + public void Lease_LoopThroughIndexRangeIndefinitelyUnderHighLoad_ShouldDistributeIndexValuesUniformly(int totalServices, int totalThreads) + { + // Arrange + const bool ReturnServicesNotImmediately = false; + var services = GivenServices(totalServices); + var roundRobin = GivenLoadBalancer(services, ReturnServicesNotImmediately); + int bottom = totalThreads / totalServices, + top = totalThreads - (bottom * totalServices) + bottom; + + // Act + var responses = WhenICallLeaseFromMultipleThreads(roundRobin, totalThreads); + var counters = CountServices(services, responses); + + // Assert + responses.ShouldNotBeNull(); + responses.Length.ShouldBe(totalThreads); + + var message = $"All values are [{string.Join(',', counters)}]"; + counters.Sum().ShouldBe(totalThreads, message); + + message = $"{nameof(bottom)}: {bottom}\n\t{nameof(top)}: {top}\n\tAll values are [{string.Join(',', counters)}]"; + counters.ShouldAllBe(counter => bottom <= counter && counter <= top, message); + } + + private static int[] CountServices(List services, Response[] responses) + { + var counters = new int[services.Count]; + var firstPort = services[0].HostAndPort.DownstreamPort; + foreach (var response in responses) + { + var idx = response.Data.DownstreamPort - firstPort; + counters[idx]++; } - private void GivenIGetTheNextAddress() + return counters; + } + + private Response[] WhenICallLeaseFromMultipleThreads(RoundRobin roundRobin, int times) + { + var tasks = new Task[times]; // allocate N-times threads as Task + var parallelResponses = new Response[times]; + for (var i = 0; i < times; i++) { - _hostAndPort = _roundRobin.Lease(_httpContext).Result; + tasks[i] = GetParallelResponse(parallelResponses, roundRobin, i); } - private void ThenTheNextAddressIndexIs(int index) + Task.WaitAll(tasks); // load by N-times threads + return parallelResponses; + } + + private async Task GetParallelResponse(Response[] responses, RoundRobin roundRobin, int threadIndex) + { + responses[threadIndex] = await WhenIGetTheNextAddressAsync(roundRobin); + } + + private static List GivenServices(int total = 3, [CallerMemberName] string serviceName = null) + { + var list = new List(total); + for (int i = 1; i <= total; i++) { - _hostAndPort.Data.ShouldBe(_services[index].HostAndPort); + list.Add(new(serviceName, new ServiceHostAndPort("127.0.0." + i, 5000 + i), string.Empty, string.Empty, Array.Empty())); } + + return list; + } + + private static RoundRobin GivenLoadBalancer(List services, bool immediately = true, [CallerMemberName] string serviceName = null) + { + return new( + () => + { + int leasingDelay = immediately ? 0 : Random.Shared.Next(5, 15); + Thread.Sleep(leasingDelay); + return Task.FromResult(services); + }, + serviceName); + } + + private Response WhenIGetTheNextAddress(RoundRobin roundRobin) + => roundRobin.Lease(_httpContext).Result; + private Task> WhenIGetTheNextAddressAsync(RoundRobin roundRobin) + => roundRobin.Lease(_httpContext); + + private static void ThenServicesAreNullErrorIsReturned(Response response) + { + response.ShouldNotBeNull().Data.ShouldBeNull(); + response.IsError.ShouldBeTrue(); + response.Errors[0].ShouldBeOfType(); } } From c502e4fe1d065d3a0abdd3fea23efdb42020ca09 Mon Sep 17 00:00:00 2001 From: Paul Roy Date: Sat, 14 Sep 2024 18:47:59 +0200 Subject: [PATCH 4/8] Downgrade the Warning to Information on missing `Content-Length` header in `MultiplexingMiddleware` (#2146) * fix: downgrade the warning to information on missing content-length header * chore: add route name to logs * test: fixing multiplexing middleware tests * Code review by @raman-m --------- Co-authored-by: Paul Roy Co-authored-by: Raman Maksimchuk --- .../PollyQoSResiliencePipelineProvider.cs | 13 ++++-------- src/Ocelot/Configuration/DownstreamRoute.cs | 8 ++++++- .../Multiplexer/MultiplexingMiddleware.cs | 21 ++++++++++--------- .../ServiceDiscoveryProviderFactory.cs | 3 +-- .../MultiplexingMiddlewareTests.cs | 5 ++++- 5 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/Ocelot.Provider.Polly/PollyQoSResiliencePipelineProvider.cs b/src/Ocelot.Provider.Polly/PollyQoSResiliencePipelineProvider.cs index a0000c62d..6ec9b5d53 100644 --- a/src/Ocelot.Provider.Polly/PollyQoSResiliencePipelineProvider.cs +++ b/src/Ocelot.Provider.Polly/PollyQoSResiliencePipelineProvider.cs @@ -38,11 +38,7 @@ public PollyQoSResiliencePipelineProvider( }; protected virtual HashSet ServerErrorCodes { get; } = DefaultServerErrorCodes; - - protected virtual string GetRouteName(DownstreamRoute route) - => string.IsNullOrWhiteSpace(route.ServiceName) - ? route.UpstreamPathTemplate?.Template ?? route.DownstreamPathTemplate?.Value ?? string.Empty - : route.ServiceName; + protected virtual string GetRouteName(DownstreamRoute route) => route.Name(); /// /// Gets Polly V8 resilience pipeline (applies QoS feature) for the route. @@ -57,9 +53,8 @@ public ResiliencePipeline GetResiliencePipeline(DownstreamR return ResiliencePipeline.Empty; // shortcut -> No QoS } - var currentRouteName = GetRouteName(route); return _registry.GetOrAddPipeline( - key: new OcelotResiliencePipelineKey(currentRouteName), + key: new OcelotResiliencePipelineKey(GetRouteName(route)), configure: (builder) => ConfigureStrategies(builder, route)); } @@ -78,7 +73,7 @@ protected virtual ResiliencePipelineBuilder ConfigureCircui } var options = route.QosOptions; - var info = $"Circuit Breaker for Route: {GetRouteName(route)}: "; + var info = $"Circuit Breaker for the route: {GetRouteName(route)}: "; var strategyOptions = new CircuitBreakerStrategyOptions { FailureRatio = 0.8, @@ -127,7 +122,7 @@ protected virtual ResiliencePipelineBuilder ConfigureTimeou Timeout = TimeSpan.FromMilliseconds(options.TimeoutValue), OnTimeout = _ => { - _logger.LogInformation($"Timeout for Route: {GetRouteName(route)}"); + _logger.LogInformation(() => $"Timeout for the route: {GetRouteName(route)}"); return ValueTask.CompletedTask; }, }; diff --git a/src/Ocelot/Configuration/DownstreamRoute.cs b/src/Ocelot/Configuration/DownstreamRoute.cs index 0241f9b61..818390a02 100644 --- a/src/Ocelot/Configuration/DownstreamRoute.cs +++ b/src/Ocelot/Configuration/DownstreamRoute.cs @@ -91,7 +91,6 @@ public DownstreamRoute( public string ServiceName { get; } public string ServiceNamespace { get; } public HttpHandlerOptions HttpHandlerOptions { get; } - public bool UseServiceDiscovery { get; } public bool EnableEndpointEndpointRateLimiting { get; } public QoSOptions QosOptions { get; } public string DownstreamScheme { get; } @@ -130,6 +129,13 @@ public DownstreamRoute( /// public HttpVersionPolicy DownstreamHttpVersionPolicy { get; } public Dictionary UpstreamHeaders { get; } + public bool UseServiceDiscovery { get; } public MetadataOptions MetadataOptions { get; } + + /// Gets the route name depending on whether the service discovery mode is enabled or disabled. + /// A object with the name. + public string Name() => string.IsNullOrEmpty(ServiceName) && !UseServiceDiscovery + ? UpstreamPathTemplate?.Template ?? DownstreamPathTemplate?.Value ?? "?" + : string.Join(':', ServiceNamespace, ServiceName, UpstreamPathTemplate?.Template); } } diff --git a/src/Ocelot/Multiplexer/MultiplexingMiddleware.cs b/src/Ocelot/Multiplexer/MultiplexingMiddleware.cs index 43a98fcd3..c148eb9b9 100644 --- a/src/Ocelot/Multiplexer/MultiplexingMiddleware.cs +++ b/src/Ocelot/Multiplexer/MultiplexingMiddleware.cs @@ -183,7 +183,7 @@ private IEnumerable> ProcessRouteWithComplexAggregation(Aggreg /// The cloned Http context. private async Task ProcessRouteAsync(HttpContext sourceContext, DownstreamRoute route, List placeholders = null) { - var newHttpContext = await CreateThreadContextAsync(sourceContext); + var newHttpContext = await CreateThreadContextAsync(sourceContext, route); CopyItemsToNewContext(newHttpContext, sourceContext, placeholders); newHttpContext.Items.UpsertDownstreamRoute(route); @@ -200,17 +200,18 @@ private static void CopyItemsToNewContext(HttpContext target, HttpContext source target.Items.SetIInternalConfiguration(source.Items.IInternalConfiguration()); target.Items.UpsertTemplatePlaceholderNameAndValues(placeholders ?? source.Items.TemplatePlaceholderNameAndValues()); - } - + } + /// /// Creates a new HttpContext based on the source. /// - /// The base http context. + /// The base http context. + /// Downstream route. /// The cloned context. - protected virtual async Task CreateThreadContextAsync(HttpContext source) + protected virtual async Task CreateThreadContextAsync(HttpContext source, DownstreamRoute route) { var from = source.Request; - var bodyStream = await CloneRequestBodyAsync(from, source.RequestAborted); + var bodyStream = await CloneRequestBodyAsync(from, route, source.RequestAborted); var target = new DefaultHttpContext { Request = @@ -245,7 +246,7 @@ protected virtual async Task CreateThreadContextAsync(HttpContext s // Once the downstream request is completed and the downstream response has been read, the downstream response object can dispose of the body's Stream object target.Response.RegisterForDisposeAsync(bodyStream); // manage Stream lifetime by HttpResponse object return target; - } + } protected virtual Task MapAsync(HttpContext httpContext, Route route, List contexts) { @@ -258,12 +259,12 @@ protected virtual Task MapAsync(HttpContext httpContext, Route route, List CloneRequestBodyAsync(HttpRequest request, CancellationToken aborted) + protected virtual async Task CloneRequestBodyAsync(HttpRequest request, DownstreamRoute route, CancellationToken aborted) { request.EnableBuffering(); if (request.Body.Position != 0) { - Logger.LogWarning("Ocelot does not support body copy without stream in initial position 0"); + Logger.LogWarning(() => $"Ocelot does not support body copy without stream in initial position 0 for the route {route.Name()}."); return request.Body; } @@ -276,7 +277,7 @@ protected virtual async Task CloneRequestBodyAsync(HttpRequest request, } else { - Logger.LogWarning("Aggregation does not support body copy without Content-Length header!"); + Logger.LogInformation(() => $"Aggregation does not support body copy without Content-Length header, skipping body copy for the route {route.Name()}."); } return targetBuffer; diff --git a/src/Ocelot/ServiceDiscovery/ServiceDiscoveryProviderFactory.cs b/src/Ocelot/ServiceDiscovery/ServiceDiscoveryProviderFactory.cs index b47e4d921..c42493a37 100644 --- a/src/Ocelot/ServiceDiscovery/ServiceDiscoveryProviderFactory.cs +++ b/src/Ocelot/ServiceDiscovery/ServiceDiscoveryProviderFactory.cs @@ -25,8 +25,7 @@ public Response Get(ServiceProviderConfiguration serv { if (route.UseServiceDiscovery) { - var routeName = route.UpstreamPathTemplate?.Template ?? route.ServiceName ?? string.Empty; - _logger.LogInformation(() => $"The {nameof(DownstreamRoute.UseServiceDiscovery)} mode of the route '{routeName}' is enabled."); + _logger.LogInformation(() => $"The {nameof(DownstreamRoute.UseServiceDiscovery)} mode of the route '{route.Name()}' is enabled."); return GetServiceDiscoveryProvider(serviceConfig, route); } diff --git a/test/Ocelot.UnitTests/Multiplexing/MultiplexingMiddlewareTests.cs b/test/Ocelot.UnitTests/Multiplexing/MultiplexingMiddlewareTests.cs index b94c247ff..acff4ae9a 100644 --- a/test/Ocelot.UnitTests/Multiplexing/MultiplexingMiddlewareTests.cs +++ b/test/Ocelot.UnitTests/Multiplexing/MultiplexingMiddlewareTests.cs @@ -63,12 +63,14 @@ public void should_not_multiplex() [Trait("Bug", "1396")] public async Task CreateThreadContextAsync_CopyUser_ToTarget() { + var route = new DownstreamRouteBuilder().Build(); + // Arrange GivenUser("test", "Copy", nameof(CreateThreadContextAsync_CopyUser_ToTarget)); // Act var method = _middleware.GetType().GetMethod("CreateThreadContextAsync", BindingFlags.NonPublic | BindingFlags.Instance); - var actual = await (Task)method.Invoke(_middleware, new object[] { _httpContext }); + var actual = await (Task)method.Invoke(_middleware, new object[] { _httpContext, route }); // Assert AssertUsers(actual); @@ -234,6 +236,7 @@ public async Task Should_Call_CloneRequestBodyAsync_Each_Time_Per_Requests(int n mock.Protected().Verify>("CloneRequestBodyAsync", numberOfRoutes > 1 ? Times.Exactly(numberOfRoutes) : Times.Never(), ItExpr.IsAny(), + ItExpr.IsAny(), ItExpr.IsAny()); } From 8e66be75fdeb8e0c8d12f9de95748d9aa0ba11ae Mon Sep 17 00:00:00 2001 From: Emmanuel Ferdman Date: Sat, 14 Sep 2024 20:15:02 +0300 Subject: [PATCH 5/8] Correct the broken link to the GraphQL sample's `README.md` (#2149) Signed-off-by: Emmanuel Ferdman Co-authored-by: Raman Maksimchuk --- docs/features/graphql.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/graphql.rst b/docs/features/graphql.rst index 93007f0ba..bd1ae12fe 100644 --- a/docs/features/graphql.rst +++ b/docs/features/graphql.rst @@ -12,7 +12,7 @@ We wanted to show how easy it is to integrate the `GraphQL for .NET `_. Using a combination of the `graphql-dotnet `_ project and Ocelot :doc:`../features/delegatinghandlers` features, this is pretty easy to do. However we do not intend to integrate more closely with **GraphQL** at the moment. -Check out the samples `README.md `_ and that should give you enough instruction on how to do this! +Check out the samples `README.md `_ and that should give you enough instruction on how to do this! Future ------ From 58d87c9273c1252a9da64a8900b080c96511d480 Mon Sep 17 00:00:00 2001 From: Finn <26823828+int0x81@users.noreply.github.com> Date: Fri, 20 Sep 2024 13:59:45 +0200 Subject: [PATCH 6/8] =?UTF-8?q?#2116=20Escaping=20unsafe=20pattern=20value?= =?UTF-8?q?s=20of=20`Regex`=20constructor=20=E2=80=8B=E2=80=8Bderived=20fr?= =?UTF-8?q?om=20URL=20query=20parameter=20values=20containing=20special=20?= =?UTF-8?q?`Regex`=20chars=20(#2150)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * regex escape handling for url templates * refactored regex method to lamda version * Quick code review by @raman-m * added acceptance test for url regex bug * moved acceptance test to routing tests * Convert to theory: define 2 test cases --------- Co-authored-by: Raman Maksimchuk --- .../DownstreamUrlCreatorMiddleware.cs | 4 +- .../Routing/RoutingTests.cs | 19 ++++++++++ .../DownstreamUrlCreatorMiddlewareTests.cs | 37 +++++++++++++++++++ 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/src/Ocelot/DownstreamUrlCreator/Middleware/DownstreamUrlCreatorMiddleware.cs b/src/Ocelot/DownstreamUrlCreator/Middleware/DownstreamUrlCreatorMiddleware.cs index dc7fb312e..2a5380c27 100644 --- a/src/Ocelot/DownstreamUrlCreator/Middleware/DownstreamUrlCreatorMiddleware.cs +++ b/src/Ocelot/DownstreamUrlCreator/Middleware/DownstreamUrlCreatorMiddleware.cs @@ -124,8 +124,8 @@ private static void RemoveQueryStringParametersThatHaveBeenUsedInTemplate(Downst foreach (var nAndV in templatePlaceholderNameAndValues) { var name = nAndV.Name.Trim(OpeningBrace, ClosingBrace); - - var rgx = new Regex($@"\b{name}={nAndV.Value}\b"); + var value = Regex.Escape(nAndV.Value); // to ensure a placeholder value containing special Regex characters from URL query parameters is safely used in a Regex constructor, it's necessary to escape the value + var rgx = new Regex($@"\b{name}={value}\b"); if (rgx.IsMatch(downstreamRequest.Query)) { diff --git a/test/Ocelot.AcceptanceTests/Routing/RoutingTests.cs b/test/Ocelot.AcceptanceTests/Routing/RoutingTests.cs index 388e690d1..61bc3fa12 100644 --- a/test/Ocelot.AcceptanceTests/Routing/RoutingTests.cs +++ b/test/Ocelot.AcceptanceTests/Routing/RoutingTests.cs @@ -1,5 +1,6 @@ using Microsoft.AspNetCore.Http; using Ocelot.Configuration.File; +using System.Web; namespace Ocelot.AcceptanceTests.Routing { @@ -1167,6 +1168,24 @@ public void should_fix_issue_271() .BDDfy(); } + [Theory] + [Trait("Bug", "2116")] + [InlineData("debug()")] // no query + [InlineData("debug%28%29")] // debug() + public void Should_change_downstream_path_by_upstream_path_when_path_contains_malicious_characters(string path) + { + var port = PortFinder.GetRandomPort(); + var configuration = GivenDefaultConfiguration(port, "/api/{path}", "/routed/api/{path}"); + var decodedDownstreamUrlPath = $"/routed/api/{HttpUtility.UrlDecode(path)}"; + this.Given(x => x.GivenThereIsAServiceRunningOn($"http://localhost:{port}", decodedDownstreamUrlPath, HttpStatusCode.OK, string.Empty)) + .And(x => _steps.GivenThereIsAConfiguration(configuration)) + .And(x => _steps.GivenOcelotIsRunning()) + .When(x => _steps.WhenIGetUrlOnTheApiGateway($"/api/{path}")) // should be encoded + .Then(x => _steps.ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) + .And(x => ThenTheDownstreamUrlPathShouldBe(decodedDownstreamUrlPath)) + .BDDfy(); + } + private void GivenThereIsAServiceRunningOn(string baseUrl, string basePath, HttpStatusCode statusCode, string responseBody) { _serviceHandler.GivenThereIsAServiceRunningOn(baseUrl, basePath, async context => diff --git a/test/Ocelot.UnitTests/DownstreamUrlCreator/DownstreamUrlCreatorMiddlewareTests.cs b/test/Ocelot.UnitTests/DownstreamUrlCreator/DownstreamUrlCreatorMiddlewareTests.cs index 8a05f8b13..b75be54a5 100644 --- a/test/Ocelot.UnitTests/DownstreamUrlCreator/DownstreamUrlCreatorMiddlewareTests.cs +++ b/test/Ocelot.UnitTests/DownstreamUrlCreator/DownstreamUrlCreatorMiddlewareTests.cs @@ -611,6 +611,43 @@ public void Should_map_when_query_parameters_has_same_names_with_placeholder() ThenTheQueryStringIs($"?roleId={roleid}&{everything}"); } + [Theory] + [Trait("Bug", "2116")] + [InlineData("api/debug()")] // no query + [InlineData("api/debug%28%29")] // debug() + public void ShouldNotFailToHandleUrlWithSpecialRegexChars(string urlPath) + { + // Arrange + var withGetMethod = new List { "Get" }; + var downstreamRoute = new DownstreamRouteBuilder() + .WithUpstreamPathTemplate(new UpstreamPathTemplateBuilder() + .WithOriginalValue("/routed/api/{path}") + .Build()) + .WithDownstreamPathTemplate("/api/{path}") + .WithUpstreamHttpMethod(withGetMethod) + .WithDownstreamScheme(Uri.UriSchemeHttp) + .Build(); + GivenTheDownStreamRouteIs(new DownstreamRouteHolder( + new List + { + new("{path}", urlPath), + }, + new RouteBuilder().WithDownstreamRoute(downstreamRoute) + .WithUpstreamHttpMethod(withGetMethod) + .Build() + )); + GivenTheDownstreamRequestUriIs($"http://localhost:5000/{urlPath}"); + GivenTheServiceProviderConfigIs(new ServiceProviderConfigurationBuilder().Build()); + GivenTheUrlReplacerWillReturn($"routed/{urlPath}"); + + // Act + WhenICallTheMiddleware(); + + // Assert + ThenTheDownstreamRequestUriIs($"http://localhost:5000/routed/{urlPath}"); + Assert.Equal((int)HttpStatusCode.OK, _httpContext.Response.StatusCode); + } + private void GivenTheServiceProviderConfigIs(ServiceProviderConfiguration config) { var configuration = new InternalConfiguration(null, null, config, null, null, null, null, null, null, null); From 09f2b1afe15f4d7f70f7175ca41ada5bfaaf1c6d Mon Sep 17 00:00:00 2001 From: Raman Maksimchuk Date: Thu, 3 Oct 2024 11:48:02 +0300 Subject: [PATCH 7/8] #2119 Review load balancing (2nd round) and redesign `DefaultConsulServiceBuilder` with `ConsulProviderFactory` refactoring to make it thread safe and friendly (#2151) * Review tests * History of Service Discovery testing: add traits * LoadBalancer traits * #2119 Steps to Reproduce * Reuse service handlers of `ConcurrentSteps` * Reuse service counters of `ConcurrentSteps` * Add LoadBalancer namespace and move classes * Move `Lease` * Move `LeaseEventArgs` * Analyze load balancers aka `ILoadBalancerAnalyzer` interface objects * Prefer using named local methods as delegates over anonymous methods for awesome call stack, ensuring the delegate's typed result matches the typed balancer's creator. Additionally, employ an IServiceProvider workaround. * Review load balancing. Assert service & leasing counters as concurrent step. Final version of acceptance test. * Fixed naming violation for asynchronous methods: `Lease` -> `LeaseAsync` * Fix ugly reflection issue of dymanic detection in favor of static type property * Propagate the `ConsulRegistryConfiguration` object through `HttpContext` in the scoped version of the default service builder, utilizing the injected `IHttpContextAccessor` object. Update `ConsulProviderFactory`. Update docs. Update tests. * Add tests from clean experiment * Final review of the tests * Review `IHttpContextAccessor` logic. Convert anonymous delegates to named ones in placeholders processing * Tried to enhance more, but failed --- ReleaseNotes.md | 7 + docs/features/servicediscovery.rst | 10 +- src/Ocelot.Provider.Consul/Consul.cs | 17 +- .../ConsulClientFactory.cs | 7 +- .../ConsulProviderFactory.cs | 38 +- .../ConsulRegistryConfiguration.cs | 3 +- .../DefaultConsulServiceBuilder.cs | 33 +- .../Interfaces/IConsulServiceBuilder.cs | 3 +- .../OcelotBuilderExtensions.cs | 5 +- .../DependencyInjection/OcelotBuilder.cs | 87 ++-- src/Ocelot/Infrastructure/Placeholders.cs | 121 ++--- .../RequestData/HttpDataRepository.cs | 22 +- .../LoadBalancer/{LoadBalancers => }/Lease.cs | 126 ++--- src/Ocelot/LoadBalancer/LeaseEventArgs.cs | 17 + .../LoadBalancers/CookieStickySessions.cs | 6 +- .../LoadBalancers/ILoadBalancer.cs | 9 +- .../LoadBalancers/LeastConnection.cs | 21 +- .../LoadBalancers/LeastConnectionCreator.cs | 7 +- .../LoadBalancers/LoadBalancerHouse.cs | 41 +- .../LoadBalancers/NoLoadBalancer.cs | 6 +- .../LoadBalancer/LoadBalancers/RoundRobin.cs | 22 +- .../LoadBalancers/RoundRobinCreator.cs | 6 +- .../Middleware/LoadBalancingMiddleware.cs | 2 +- .../Providers/ConfigurationServiceProvider.cs | 19 +- .../Ocelot.AcceptanceTests/ConcurrentSteps.cs | 271 ++++++++++ .../LoadBalancer/ILoadBalancerAnalyzer.cs | 18 + .../LoadBalancer/LeastConnectionAnalyzer.cs | 28 + .../LeastConnectionAnalyzerCreator.cs | 22 + .../LoadBalancer/LoadBalancerAnalyzer.cs | 118 +++++ .../LoadBalancer/LoadBalancerTests.cs | 129 +++++ .../LoadBalancer/RoundRobinAnalyzer.cs | 31 ++ .../LoadBalancer/RoundRobinAnalyzerCreator.cs | 22 + .../LoadBalancerTests.cs | 274 ---------- .../Properties/GlobalSuppressions.cs | 1 + .../ConsulServiceDiscoveryTests.cs | 487 ++++++++++-------- .../EurekaServiceDiscoveryTests.cs | 3 +- .../KubernetesServiceDiscoveryTests.cs | 235 +-------- test/Ocelot.AcceptanceTests/ServiceHandler.cs | 16 +- test/Ocelot.AcceptanceTests/Steps.cs | 115 +---- .../StickySessionsTests.cs | 3 + test/Ocelot.Testing/PortFinder.cs | 33 +- test/Ocelot.UnitTests/Consul/ConsulTests.cs | 7 +- .../DefaultConsulServiceBuilderTests.cs | 28 +- .../Consul/ProviderFactoryTests.cs | 15 +- .../DependencyInjection/OcelotBuilderTests.cs | 16 +- test/Ocelot.UnitTests/Kubernetes/KubeTests.cs | 1 + .../OcelotBuilderExtensionsTests.cs | 3 +- .../Kubernetes/PollKubeTests.cs | 3 +- .../LoadBalancer/CookieStickySessionsTests.cs | 16 +- ...elegateInvokingLoadBalancerCreatorTests.cs | 12 +- .../LoadBalancer/LeastConnectionTests.cs | 40 +- .../LoadBalancer/LoadBalancerFactoryTests.cs | 48 +- .../LoadBalancer/LoadBalancerHouseTests.cs | 24 +- .../LoadBalancerMiddlewareTests.cs | 6 +- .../LoadBalancer/NoLoadBalancerTests.cs | 2 +- .../LoadBalancer/RoundRobinTests.cs | 4 +- 56 files changed, 1421 insertions(+), 1245 deletions(-) rename src/Ocelot/LoadBalancer/{LoadBalancers => }/Lease.cs (92%) create mode 100644 src/Ocelot/LoadBalancer/LeaseEventArgs.cs create mode 100644 test/Ocelot.AcceptanceTests/ConcurrentSteps.cs create mode 100644 test/Ocelot.AcceptanceTests/LoadBalancer/ILoadBalancerAnalyzer.cs create mode 100644 test/Ocelot.AcceptanceTests/LoadBalancer/LeastConnectionAnalyzer.cs create mode 100644 test/Ocelot.AcceptanceTests/LoadBalancer/LeastConnectionAnalyzerCreator.cs create mode 100644 test/Ocelot.AcceptanceTests/LoadBalancer/LoadBalancerAnalyzer.cs create mode 100644 test/Ocelot.AcceptanceTests/LoadBalancer/LoadBalancerTests.cs create mode 100644 test/Ocelot.AcceptanceTests/LoadBalancer/RoundRobinAnalyzer.cs create mode 100644 test/Ocelot.AcceptanceTests/LoadBalancer/RoundRobinAnalyzerCreator.cs delete mode 100644 test/Ocelot.AcceptanceTests/LoadBalancerTests.cs diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 78f65140a..91cbe6cad 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -1 +1,8 @@ Technical release, version {0} + +### Breaking changes + +- The `ILoadBalancer` interface: The `Lease` method was renamed to `LeaseAsync`. + Interface FQN: `Ocelot.LoadBalancer.LoadBalancers.ILoadBalancer` + Method FQN: `Ocelot.LoadBalancer.LoadBalancers.ILoadBalancer.LeaseAsync` +- TO BE Written \ No newline at end of file diff --git a/docs/features/servicediscovery.rst b/docs/features/servicediscovery.rst index a4e4f96c5..ed2915a63 100644 --- a/docs/features/servicediscovery.rst +++ b/docs/features/servicediscovery.rst @@ -246,10 +246,12 @@ However, the quickest and most streamlined approach is to inherit directly from public class MyConsulServiceBuilder : DefaultConsulServiceBuilder { - public MyConsulServiceBuilder(Func configurationFactory, IConsulClientFactory clientFactory, IOcelotLoggerFactory loggerFactory) - : base(configurationFactory, clientFactory, loggerFactory) { } + public MyConsulServiceBuilder(IHttpContextAccessor contextAccessor, IConsulClientFactory clientFactory, IOcelotLoggerFactory loggerFactory) + : base(contextAccessor, clientFactory, loggerFactory) { } + // I want to use the agent service IP address as the downstream hostname - protected override string GetDownstreamHost(ServiceEntry entry, Node node) => entry.Service.Address; + protected override string GetDownstreamHost(ServiceEntry entry, Node node) + => entry.Service.Address; } **Second**, we must inject the new behavior into DI, as demonstrated in the Ocelot versus Consul setup: @@ -543,7 +545,7 @@ But you can leave this ``Type`` option for compatibility between both designs. .. _KV Store: https://developer.hashicorp.com/consul/docs/dynamic-app-config/kv .. _3 seconds TTL: https://github.com/search?q=repo%3AThreeMammals%2FOcelot+TimeSpan.FromSeconds%283%29&type=code .. _catalog nodes: https://developer.hashicorp.com/consul/api-docs/catalog#list-nodes -.. _the acceptance test: https://github.com/search?q=repo%3AThreeMammals%2FOcelot+Should_return_service_address_by_overridden_service_builder_when_there_is_a_node&type=code +.. _the acceptance test: https://github.com/search?q=repo%3AThreeMammals%2FOcelot+ShouldReturnServiceAddressByOverriddenServiceBuilderWhenThereIsANode&type=code .. _346: https://github.com/ThreeMammals/Ocelot/issues/346 .. _909: https://github.com/ThreeMammals/Ocelot/pull/909 .. _954: https://github.com/ThreeMammals/Ocelot/issues/954 diff --git a/src/Ocelot.Provider.Consul/Consul.cs b/src/Ocelot.Provider.Consul/Consul.cs index 27b5b4422..9be0128e0 100644 --- a/src/Ocelot.Provider.Consul/Consul.cs +++ b/src/Ocelot.Provider.Consul/Consul.cs @@ -33,21 +33,16 @@ public virtual async Task> GetAsync() var entries = entriesTask.Result.Response ?? Array.Empty(); var nodes = nodesTask.Result.Response ?? Array.Empty(); - var services = new List(); - - if (entries.Length != 0) - { - _logger.LogDebug(() => $"{nameof(Consul)} Provider: Found total {entries.Length} service entries for '{_configuration.KeyOfServiceInConsul}' service."); - _logger.LogDebug(() => $"{nameof(Consul)} Provider: Found total {nodes.Length} catalog nodes."); - var collection = BuildServices(entries, nodes); - services.AddRange(collection); - } - else + if (entries.Length == 0) { _logger.LogWarning(() => $"{nameof(Consul)} Provider: No service entries found for '{_configuration.KeyOfServiceInConsul}' service!"); + return new(); } - return services; + _logger.LogDebug(() => $"{nameof(Consul)} Provider: Found total {entries.Length} service entries for '{_configuration.KeyOfServiceInConsul}' service."); + _logger.LogDebug(() => $"{nameof(Consul)} Provider: Found total {nodes.Length} catalog nodes."); + return BuildServices(entries, nodes) + .ToList(); } protected virtual IEnumerable BuildServices(ServiceEntry[] entries, Node[] nodes) diff --git a/src/Ocelot.Provider.Consul/ConsulClientFactory.cs b/src/Ocelot.Provider.Consul/ConsulClientFactory.cs index f7c5c0c0c..fdece1d99 100644 --- a/src/Ocelot.Provider.Consul/ConsulClientFactory.cs +++ b/src/Ocelot.Provider.Consul/ConsulClientFactory.cs @@ -4,10 +4,15 @@ namespace Ocelot.Provider.Consul; public class ConsulClientFactory : IConsulClientFactory { + // TODO We need this overloaded method -> + //public IConsulClient Get(ServiceProviderConfiguration config) public IConsulClient Get(ConsulRegistryConfiguration config) => new ConsulClient(c => OverrideConfig(c, config)); - private static void OverrideConfig(ConsulClientConfiguration to, ConsulRegistryConfiguration from) + // TODO -> + //private static void OverrideConfig(ConsulClientConfiguration to, ServiceProviderConfiguration from) + // Factory which consumes concrete types is a bad factory! A more abstract types are required + private static void OverrideConfig(ConsulClientConfiguration to, ConsulRegistryConfiguration from) // TODO Why ConsulRegistryConfiguration? We use ServiceProviderConfiguration props only! :) { to.Address = new Uri($"{from.Scheme}://{from.Host}:{from.Port}"); diff --git a/src/Ocelot.Provider.Consul/ConsulProviderFactory.cs b/src/Ocelot.Provider.Consul/ConsulProviderFactory.cs index 00c2715ee..dbc4b9155 100644 --- a/src/Ocelot.Provider.Consul/ConsulProviderFactory.cs +++ b/src/Ocelot.Provider.Consul/ConsulProviderFactory.cs @@ -1,4 +1,5 @@ -using Microsoft.Extensions.DependencyInjection; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; using Ocelot.Configuration; using Ocelot.Logging; using Ocelot.Provider.Consul.Interfaces; @@ -6,36 +7,37 @@ namespace Ocelot.Provider.Consul; -public static class ConsulProviderFactory +/// +/// TODO It must be refactored converting to real factory-class and add to DI. +/// +/// +/// Must inherit from interface. +/// Also the must be removed from the design. +/// +public static class ConsulProviderFactory // TODO : IServiceDiscoveryProviderFactory { - /// - /// String constant used for provider type definition. - /// + /// String constant used for provider type definition. public const string PollConsul = nameof(Provider.Consul.PollConsul); - private static readonly List ServiceDiscoveryProviders = new(); - private static readonly object LockObject = new(); + private static readonly List ServiceDiscoveryProviders = new(); // TODO It must be scoped service in DI-container + private static readonly object SyncRoot = new(); public static ServiceDiscoveryFinderDelegate Get { get; } = CreateProvider; - - private static ConsulRegistryConfiguration configuration; - private static ConsulRegistryConfiguration ConfigurationGetter() => configuration; - public static Func GetConfiguration { get; } = ConfigurationGetter; - - private static IServiceDiscoveryProvider CreateProvider(IServiceProvider provider, - ServiceProviderConfiguration config, DownstreamRoute route) + private static IServiceDiscoveryProvider CreateProvider(IServiceProvider provider, ServiceProviderConfiguration config, DownstreamRoute route) { var factory = provider.GetService(); var consulFactory = provider.GetService(); + var contextAccessor = provider.GetService(); - configuration = new ConsulRegistryConfiguration(config.Scheme, config.Host, config.Port, route.ServiceName, config.Token); - var serviceBuilder = provider.GetService(); + var configuration = new ConsulRegistryConfiguration(config.Scheme, config.Host, config.Port, route.ServiceName, config.Token); // TODO Why not to pass 2 args only: config, route? LoL + contextAccessor.HttpContext.Items[nameof(ConsulRegistryConfiguration)] = configuration; // initialize data + var serviceBuilder = provider.GetService(); // consume data in default/custom builder - var consulProvider = new Consul(configuration, factory, consulFactory, serviceBuilder); + var consulProvider = new Consul(configuration, factory, consulFactory, serviceBuilder); // TODO It must be added to DI-container! if (PollConsul.Equals(config.Type, StringComparison.OrdinalIgnoreCase)) { - lock (LockObject) + lock (SyncRoot) { var discoveryProvider = ServiceDiscoveryProviders.FirstOrDefault(x => x.ServiceName == route.ServiceName); if (discoveryProvider != null) diff --git a/src/Ocelot.Provider.Consul/ConsulRegistryConfiguration.cs b/src/Ocelot.Provider.Consul/ConsulRegistryConfiguration.cs index 255c7b686..2109f9eaf 100644 --- a/src/Ocelot.Provider.Consul/ConsulRegistryConfiguration.cs +++ b/src/Ocelot.Provider.Consul/ConsulRegistryConfiguration.cs @@ -1,6 +1,6 @@ namespace Ocelot.Provider.Consul; -public class ConsulRegistryConfiguration +public class ConsulRegistryConfiguration // TODO Inherit from ServiceProviderConfiguration ? { /// /// Consul HTTP client default port. @@ -12,6 +12,7 @@ public class ConsulRegistryConfiguration public ConsulRegistryConfiguration(string scheme, string host, int port, string keyOfServiceInConsul, string token) { + // TODO Why not to encapsulate this biz logic right in ConsulProviderFactory? LoL Host = string.IsNullOrEmpty(host) ? "localhost" : host; Port = port > 0 ? port : DefaultHttpPort; Scheme = string.IsNullOrEmpty(scheme) ? Uri.UriSchemeHttp : scheme; diff --git a/src/Ocelot.Provider.Consul/DefaultConsulServiceBuilder.cs b/src/Ocelot.Provider.Consul/DefaultConsulServiceBuilder.cs index 7526bea65..4d9abe7a7 100644 --- a/src/Ocelot.Provider.Consul/DefaultConsulServiceBuilder.cs +++ b/src/Ocelot.Provider.Consul/DefaultConsulServiceBuilder.cs @@ -1,4 +1,5 @@ -using Ocelot.Infrastructure.Extensions; +using Microsoft.AspNetCore.Http; +using Ocelot.Infrastructure.Extensions; using Ocelot.Logging; using Ocelot.Provider.Consul.Interfaces; using Ocelot.Values; @@ -7,23 +8,31 @@ namespace Ocelot.Provider.Consul; public class DefaultConsulServiceBuilder : IConsulServiceBuilder { - private readonly ConsulRegistryConfiguration _configuration; - private readonly IConsulClient _client; - private readonly IOcelotLogger _logger; + private readonly HttpContext _context; + private readonly IConsulClientFactory _clientFactory; + private readonly IOcelotLoggerFactory _loggerFactory; + + private ConsulRegistryConfiguration _configuration; + private IConsulClient _client; + private IOcelotLogger _logger; public DefaultConsulServiceBuilder( - Func configurationFactory, + IHttpContextAccessor contextAccessor, IConsulClientFactory clientFactory, IOcelotLoggerFactory loggerFactory) { - _configuration = configurationFactory.Invoke(); - _client = clientFactory.Get(_configuration); - _logger = loggerFactory.CreateLogger(); + _context = contextAccessor.HttpContext; + _clientFactory = clientFactory; + _loggerFactory = loggerFactory; } - public ConsulRegistryConfiguration Configuration => _configuration; - protected IConsulClient Client => _client; - protected IOcelotLogger Logger => _logger; + // TODO See comment in the interface about the privacy. The goal is to eliminate IBC! + // So, we need more abstract type, and ServiceProviderConfiguration is a good choice. The rest of props can be obtained from HttpContext + protected /*public*/ ConsulRegistryConfiguration Configuration => _configuration + ??= _context.Items.TryGetValue(nameof(ConsulRegistryConfiguration), out var value) + ? value as ConsulRegistryConfiguration : default; + protected IConsulClient Client => _client ??= _clientFactory.Get(Configuration); + protected IOcelotLogger Logger => _logger ??= _loggerFactory.CreateLogger(); public virtual bool IsValid(ServiceEntry entry) { @@ -36,7 +45,7 @@ public virtual bool IsValid(ServiceEntry entry) if (!valid) { - _logger.LogWarning( + Logger.LogWarning( () => $"Unable to use service address: '{service.Address}' and port: {service.Port} as it is invalid for the service: '{service.Service}'. Address must contain host only e.g. 'localhost', and port must be greater than 0."); } diff --git a/src/Ocelot.Provider.Consul/Interfaces/IConsulServiceBuilder.cs b/src/Ocelot.Provider.Consul/Interfaces/IConsulServiceBuilder.cs index 0555b0144..fab45dfe4 100644 --- a/src/Ocelot.Provider.Consul/Interfaces/IConsulServiceBuilder.cs +++ b/src/Ocelot.Provider.Consul/Interfaces/IConsulServiceBuilder.cs @@ -4,7 +4,8 @@ namespace Ocelot.Provider.Consul.Interfaces; public interface IConsulServiceBuilder { - ConsulRegistryConfiguration Configuration { get; } + // Keep config private (deep encapsulation) until an architectural decision is made. + // ConsulRegistryConfiguration Configuration { get; } bool IsValid(ServiceEntry entry); IEnumerable BuildServices(ServiceEntry[] entries, Node[] nodes); Service CreateService(ServiceEntry serviceEntry, Node serviceNode); diff --git a/src/Ocelot.Provider.Consul/OcelotBuilderExtensions.cs b/src/Ocelot.Provider.Consul/OcelotBuilderExtensions.cs index 0c064f780..aed4a528d 100644 --- a/src/Ocelot.Provider.Consul/OcelotBuilderExtensions.cs +++ b/src/Ocelot.Provider.Consul/OcelotBuilderExtensions.cs @@ -24,9 +24,8 @@ public static IOcelotBuilder AddConsul(this IOcelotBuilder builder) { builder.Services .AddSingleton(ConsulProviderFactory.Get) - .AddSingleton(ConsulProviderFactory.GetConfiguration) .AddSingleton() - .AddSingleton() + .AddScoped() .RemoveAll(typeof(IFileConfigurationPollerOptions)) .AddSingleton(); return builder; @@ -49,7 +48,7 @@ public static IOcelotBuilder AddConsul(this IOcelotBuilder buil { AddConsul(builder).Services .RemoveAll() - .AddSingleton(typeof(IConsulServiceBuilder), typeof(TServiceBuilder)); + .AddScoped(typeof(IConsulServiceBuilder), typeof(TServiceBuilder)); return builder; } diff --git a/src/Ocelot/DependencyInjection/OcelotBuilder.cs b/src/Ocelot/DependencyInjection/OcelotBuilder.cs index 217280e37..146ba3c65 100644 --- a/src/Ocelot/DependencyInjection/OcelotBuilder.cs +++ b/src/Ocelot/DependencyInjection/OcelotBuilder.cs @@ -4,7 +4,6 @@ using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; using Ocelot.Authorization; -using Ocelot.Cache; using Ocelot.Claims; using Ocelot.Configuration; using Ocelot.Configuration.ChangeTracking; @@ -27,7 +26,6 @@ using Ocelot.Multiplexer; using Ocelot.PathManipulation; using Ocelot.QueryStrings; -using Ocelot.RateLimiting; using Ocelot.Request.Creator; using Ocelot.Request.Mapper; using Ocelot.Requester; @@ -119,10 +117,8 @@ public OcelotBuilder(IServiceCollection services, IConfiguration configurationRo Services.AddOcelotMetadata(); Services.AddOcelotMessageInvokerPool(); - // See this for why we register this as singleton: - // http://stackoverflow.com/questions/37371264/invalidoperationexception-unable-to-resolve-service-for-type-microsoft-aspnetc - // Could maybe use a scoped data repository - Services.TryAddSingleton(); + // Chinese developers should read StackOverflow ignoring Microsoft Learn docs -> http://stackoverflow.com/questions/37371264/invalidoperationexception-unable-to-resolve-service-for-type-microsoft-aspnetc + Services.AddHttpContextAccessor(); Services.TryAddSingleton(); Services.AddMemoryCache(); Services.TryAddSingleton(); @@ -206,44 +202,50 @@ public IOcelotBuilder AddTransientDefinedAggregator() return this; } - public IOcelotBuilder AddCustomLoadBalancer() - where T : ILoadBalancer, new() + public IOcelotBuilder AddCustomLoadBalancer() + where TLoadBalancer : ILoadBalancer, new() { - AddCustomLoadBalancer((provider, route, serviceDiscoveryProvider) => new T()); - return this; + TLoadBalancer Create(IServiceProvider provider, DownstreamRoute route, IServiceDiscoveryProvider discoveryProvider) + => new(); + return AddCustomLoadBalancer(Create); } - public IOcelotBuilder AddCustomLoadBalancer(Func loadBalancerFactoryFunc) - where T : ILoadBalancer + public IOcelotBuilder AddCustomLoadBalancer(Func loadBalancerFactoryFunc) + where TLoadBalancer : ILoadBalancer { - AddCustomLoadBalancer((provider, route, serviceDiscoveryProvider) => - loadBalancerFactoryFunc()); - return this; + TLoadBalancer Create(IServiceProvider provider, DownstreamRoute route, IServiceDiscoveryProvider discoveryProvider) + => loadBalancerFactoryFunc(); + return AddCustomLoadBalancer(Create); } - public IOcelotBuilder AddCustomLoadBalancer(Func loadBalancerFactoryFunc) - where T : ILoadBalancer + public IOcelotBuilder AddCustomLoadBalancer(Func loadBalancerFactoryFunc) + where TLoadBalancer : ILoadBalancer { - AddCustomLoadBalancer((provider, route, serviceDiscoveryProvider) => - loadBalancerFactoryFunc(provider)); - return this; + TLoadBalancer Create(IServiceProvider provider, DownstreamRoute route, IServiceDiscoveryProvider discoveryProvider) + => loadBalancerFactoryFunc(provider); + return AddCustomLoadBalancer(Create); } - public IOcelotBuilder AddCustomLoadBalancer(Func loadBalancerFactoryFunc) - where T : ILoadBalancer + public IOcelotBuilder AddCustomLoadBalancer(Func loadBalancerFactoryFunc) + where TLoadBalancer : ILoadBalancer { - AddCustomLoadBalancer((provider, route, serviceDiscoveryProvider) => - loadBalancerFactoryFunc(route, serviceDiscoveryProvider)); - return this; + TLoadBalancer Create(IServiceProvider provider, DownstreamRoute route, IServiceDiscoveryProvider discoveryProvider) + => loadBalancerFactoryFunc(route, discoveryProvider); + return AddCustomLoadBalancer(Create); } - public IOcelotBuilder AddCustomLoadBalancer(Func loadBalancerFactoryFunc) - where T : ILoadBalancer + public IOcelotBuilder AddCustomLoadBalancer(Func loadBalancerFactoryFunc) + where TLoadBalancer : ILoadBalancer { - Services.AddSingleton(provider => - new DelegateInvokingLoadBalancerCreator( - (route, serviceDiscoveryProvider) => - loadBalancerFactoryFunc(provider, route, serviceDiscoveryProvider))); + ILoadBalancer Create(DownstreamRoute route, IServiceDiscoveryProvider discoveryProvider) + => loadBalancerFactoryFunc(_serviceProvider, route, discoveryProvider); + ILoadBalancerCreator implementationFactory(IServiceProvider provider) + { + _serviceProvider = provider; + return new DelegateInvokingLoadBalancerCreator(Create); + } + + Services.AddSingleton(implementationFactory); return this; } @@ -257,9 +259,9 @@ public IOcelotBuilder AddDelegatingHandler(Type delegateType, bool global = fals if (global) { Services.AddTransient(delegateType); - Services.AddTransient(s => + Services.AddTransient(provider => { - var service = s.GetService(delegateType) as DelegatingHandler; + var service = provider.GetService(delegateType) as DelegatingHandler; return new GlobalDelegatingHandler(service); }); } @@ -277,9 +279,9 @@ public IOcelotBuilder AddDelegatingHandler(bool global = false) if (global) { Services.AddTransient(); - Services.AddTransient(s => + Services.AddTransient(provider => { - var service = s.GetService(); + var service = provider.GetService(); return new GlobalDelegatingHandler(service); }); } @@ -302,15 +304,20 @@ public IOcelotBuilder AddConfigPlaceholders() Services.Replace(ServiceDescriptor.Describe( typeof(IPlaceholders), - s => (IPlaceholders)objectFactory(s, - new[] { CreateInstance(s, wrappedDescriptor) }), + provider => (IPlaceholders)objectFactory( + provider, + new[] { CreateInstance(provider, wrappedDescriptor) }), wrappedDescriptor.Lifetime )); return this; } - private static object CreateInstance(IServiceProvider services, ServiceDescriptor descriptor) + + /// For local implementation purposes, so it MUST NOT be public!.. + private IServiceProvider _serviceProvider; // TODO Reuse ActivatorUtilities factories? + + private static object CreateInstance(IServiceProvider provider, ServiceDescriptor descriptor) { if (descriptor.ImplementationInstance != null) { @@ -319,10 +326,10 @@ private static object CreateInstance(IServiceProvider services, ServiceDescripto if (descriptor.ImplementationFactory != null) { - return descriptor.ImplementationFactory(services); + return descriptor.ImplementationFactory(provider); } - return ActivatorUtilities.GetServiceOrCreateInstance(services, descriptor.ImplementationType); + return ActivatorUtilities.GetServiceOrCreateInstance(provider, descriptor.ImplementationType); } } } diff --git a/src/Ocelot/Infrastructure/Placeholders.cs b/src/Ocelot/Infrastructure/Placeholders.cs index c0ca6a151..e69fbe578 100644 --- a/src/Ocelot/Infrastructure/Placeholders.cs +++ b/src/Ocelot/Infrastructure/Placeholders.cs @@ -12,24 +12,24 @@ public class Placeholders : IPlaceholders private readonly Dictionary> _requestPlaceholders; private readonly IBaseUrlFinder _finder; private readonly IRequestScopedDataRepository _repo; - private readonly IHttpContextAccessor _httpContextAccessor; + private readonly IHttpContextAccessor _contextAccessor; - public Placeholders(IBaseUrlFinder finder, IRequestScopedDataRepository repo, IHttpContextAccessor httpContextAccessor) + public Placeholders(IBaseUrlFinder finder, IRequestScopedDataRepository repo, IHttpContextAccessor contextAccessor) { _repo = repo; - _httpContextAccessor = httpContextAccessor; + _contextAccessor = contextAccessor; _finder = finder; _placeholders = new Dictionary>> { - { "{BaseUrl}", GetBaseUrl() }, - { "{TraceId}", GetTraceId() }, - { "{RemoteIpAddress}", GetRemoteIpAddress() }, - { "{UpstreamHost}", GetUpstreamHost() }, + { "{BaseUrl}", GetBaseUrl }, + { "{TraceId}", GetTraceId }, + { "{RemoteIpAddress}", GetRemoteIpAddress }, + { "{UpstreamHost}", GetUpstreamHost }, }; _requestPlaceholders = new Dictionary> { - { "{DownstreamBaseUrl}", GetDownstreamBaseUrl() }, + { "{DownstreamBaseUrl}", GetDownstreamBaseUrl }, }; } @@ -49,23 +49,16 @@ public Response Get(string key) public Response Get(string key, DownstreamRequest request) { - if (_requestPlaceholders.ContainsKey(key)) - { - return new OkResponse(_requestPlaceholders[key].Invoke(request)); - } - - return new ErrorResponse(new CouldNotFindPlaceholderError(key)); + return _requestPlaceholders.TryGetValue(key, out var func) + ? new OkResponse(func.Invoke(request)) + : new ErrorResponse(new CouldNotFindPlaceholderError(key)); } public Response Add(string key, Func> func) { - if (_placeholders.ContainsKey(key)) - { - return new ErrorResponse(new CannotAddPlaceholderError($"Unable to add placeholder: {key}, placeholder already exists")); - } - - _placeholders.Add(key, func); - return new OkResponse(); + return _placeholders.TryAdd(key, func) + ? new OkResponse() + : new ErrorResponse(new CannotAddPlaceholderError($"Unable to add placeholder: {key}, placeholder already exists")); } public Response Remove(string key) @@ -79,75 +72,53 @@ public Response Remove(string key) return new OkResponse(); } - private Func> GetRemoteIpAddress() + private Response GetRemoteIpAddress() { - return () => + // this can blow up so adding try catch and return error + try { - // this can blow up so adding try catch and return error - try - { - var remoteIdAddress = _httpContextAccessor.HttpContext.Connection.RemoteIpAddress.ToString(); - return new OkResponse(remoteIdAddress); - } - catch - { - return new ErrorResponse(new CouldNotFindPlaceholderError("{RemoteIpAddress}")); - } - }; - } - - private static Func GetDownstreamBaseUrl() - { - return x => + var remoteIdAddress = _contextAccessor.HttpContext.Connection.RemoteIpAddress.ToString(); + return new OkResponse(remoteIdAddress); + } + catch { - var downstreamUrl = $"{x.Scheme}://{x.Host}"; - - if (x.Port != 80 && x.Port != 443) - { - downstreamUrl = $"{downstreamUrl}:{x.Port}"; - } - - return $"{downstreamUrl}/"; - }; + return new ErrorResponse(new CouldNotFindPlaceholderError("{RemoteIpAddress}")); + } } - private Func> GetTraceId() + private static string GetDownstreamBaseUrl(DownstreamRequest x) { - return () => + var downstreamUrl = $"{x.Scheme}://{x.Host}"; + if (x.Port != 80 && x.Port != 443) { - var traceId = _repo.Get("TraceId"); - if (traceId.IsError) - { - return new ErrorResponse(traceId.Errors); - } + downstreamUrl = $"{downstreamUrl}:{x.Port}"; + } - return new OkResponse(traceId.Data); - }; + return $"{downstreamUrl}/"; } - private Func> GetBaseUrl() + private Response GetTraceId() { - return () => new OkResponse(_finder.Find()); + var traceId = _repo.Get("TraceId"); + return traceId.IsError + ? new ErrorResponse(traceId.Errors) + : new OkResponse(traceId.Data); } - private Func> GetUpstreamHost() + private Response GetBaseUrl() => new OkResponse(_finder.Find()); + + private Response GetUpstreamHost() { - return () => + try { - try - { - if (_httpContextAccessor.HttpContext.Request.Headers.TryGetValue("Host", out var upstreamHost)) - { - return new OkResponse(upstreamHost.First()); - } - - return new ErrorResponse(new CouldNotFindPlaceholderError("{UpstreamHost}")); - } - catch - { - return new ErrorResponse(new CouldNotFindPlaceholderError("{UpstreamHost}")); - } - }; + return _contextAccessor.HttpContext.Request.Headers.TryGetValue("Host", out var upstreamHost) + ? new OkResponse(upstreamHost.First()) + : new ErrorResponse(new CouldNotFindPlaceholderError("{UpstreamHost}")); + } + catch + { + return new ErrorResponse(new CouldNotFindPlaceholderError("{UpstreamHost}")); + } } } } diff --git a/src/Ocelot/Infrastructure/RequestData/HttpDataRepository.cs b/src/Ocelot/Infrastructure/RequestData/HttpDataRepository.cs index 24e0607f1..f1170d895 100644 --- a/src/Ocelot/Infrastructure/RequestData/HttpDataRepository.cs +++ b/src/Ocelot/Infrastructure/RequestData/HttpDataRepository.cs @@ -5,18 +5,18 @@ namespace Ocelot.Infrastructure.RequestData { public class HttpDataRepository : IRequestScopedDataRepository { - private readonly IHttpContextAccessor _httpContextAccessor; + private readonly IHttpContextAccessor _contextAccessor; - public HttpDataRepository(IHttpContextAccessor httpContextAccessor) + public HttpDataRepository(IHttpContextAccessor contextAccessor) { - _httpContextAccessor = httpContextAccessor; + _contextAccessor = contextAccessor; } public Response Add(string key, T value) { try { - _httpContextAccessor.HttpContext.Items.Add(key, value); + _contextAccessor.HttpContext.Items.Add(key, value); return new OkResponse(); } catch (Exception exception) @@ -29,7 +29,7 @@ public Response Update(string key, T value) { try { - _httpContextAccessor.HttpContext.Items[key] = value; + _contextAccessor.HttpContext.Items[key] = value; return new OkResponse(); } catch (Exception exception) @@ -40,18 +40,14 @@ public Response Update(string key, T value) public Response Get(string key) { - if (_httpContextAccessor.HttpContext == null || _httpContextAccessor.HttpContext.Items == null) + if (_contextAccessor?.HttpContext?.Items == null) { return new ErrorResponse(new CannotFindDataError($"Unable to find data for key: {key} because HttpContext or HttpContext.Items is null")); } - if (_httpContextAccessor.HttpContext.Items.TryGetValue(key, out var obj)) - { - var data = (T)obj; - return new OkResponse(data); - } - - return new ErrorResponse(new CannotFindDataError($"Unable to find data for key: {key}")); + return _contextAccessor.HttpContext.Items.TryGetValue(key, out var item) + ? new OkResponse((T)item) + : new ErrorResponse(new CannotFindDataError($"Unable to find data for key: {key}")); } } } diff --git a/src/Ocelot/LoadBalancer/LoadBalancers/Lease.cs b/src/Ocelot/LoadBalancer/Lease.cs similarity index 92% rename from src/Ocelot/LoadBalancer/LoadBalancers/Lease.cs rename to src/Ocelot/LoadBalancer/Lease.cs index 46d120e4c..f0ba048c3 100644 --- a/src/Ocelot/LoadBalancer/LoadBalancers/Lease.cs +++ b/src/Ocelot/LoadBalancer/Lease.cs @@ -1,63 +1,63 @@ -using Ocelot.Values; - -namespace Ocelot.LoadBalancer.LoadBalancers; - -public struct Lease : IEquatable -{ - public Lease() - { - HostAndPort = null; - Connections = 0; - } - - public Lease(Lease from) - { - HostAndPort = from.HostAndPort; - Connections = from.Connections; - } - - public Lease(ServiceHostAndPort hostAndPort) - { - HostAndPort = hostAndPort; - Connections = 0; - } - - public Lease(ServiceHostAndPort hostAndPort, int connections) - { - HostAndPort = hostAndPort; - Connections = connections; - } - - public ServiceHostAndPort HostAndPort { get; } - public int Connections { get; set; } - - public static Lease Null => new(); - - public override readonly string ToString() => $"({HostAndPort}+{Connections})"; - public override readonly int GetHashCode() => HostAndPort.GetHashCode(); - public override readonly bool Equals(object obj) => obj is Lease l && this == l; - public readonly bool Equals(Lease other) => this == other; - - /// Checks equality of two leases. - /// - /// Override default implementation of because we want to ignore the property. - /// Microsoft Learn | .NET | C# Docs: - /// - /// Equality operators - /// System.Object.Equals method - /// IEquatable<T>.Equals(T) Method - /// ValueType.Equals(Object) Method - /// - /// - /// First operand. - /// Second operand. - /// if both operands are equal; otherwise, . - public static bool operator ==(Lease x, Lease y) => x.HostAndPort == y.HostAndPort; // ignore -> x.Connections == y.Connections; - public static bool operator !=(Lease x, Lease y) => !(x == y); - - public static bool operator ==(ServiceHostAndPort h, Lease l) => h == l.HostAndPort; - public static bool operator !=(ServiceHostAndPort h, Lease l) => !(h == l); - - public static bool operator ==(Lease l, ServiceHostAndPort h) => l.HostAndPort == h; - public static bool operator !=(Lease l, ServiceHostAndPort h) => !(l == h); -} +using Ocelot.Values; + +namespace Ocelot.LoadBalancer; + +public struct Lease : IEquatable +{ + public Lease() + { + HostAndPort = null; + Connections = 0; + } + + public Lease(Lease from) + { + HostAndPort = from.HostAndPort; + Connections = from.Connections; + } + + public Lease(ServiceHostAndPort hostAndPort) + { + HostAndPort = hostAndPort; + Connections = 0; + } + + public Lease(ServiceHostAndPort hostAndPort, int connections) + { + HostAndPort = hostAndPort; + Connections = connections; + } + + public ServiceHostAndPort HostAndPort { get; } + public int Connections { get; set; } + + public static Lease Null => new(); + + public override readonly string ToString() => $"({HostAndPort}+{Connections})"; + public override readonly int GetHashCode() => HostAndPort.GetHashCode(); + public override readonly bool Equals(object obj) => obj is Lease l && this == l; + public readonly bool Equals(Lease other) => this == other; + + /// Checks equality of two leases. + /// + /// Override default implementation of because we want to ignore the property. + /// Microsoft Learn | .NET | C# Docs: + /// + /// Equality operators + /// System.Object.Equals method + /// IEquatable<T>.Equals(T) Method + /// ValueType.Equals(Object) Method + /// + /// + /// First operand. + /// Second operand. + /// if both operands are equal; otherwise, . + public static bool operator ==(Lease x, Lease y) => x.HostAndPort == y.HostAndPort; // ignore -> x.Connections == y.Connections; + public static bool operator !=(Lease x, Lease y) => !(x == y); + + public static bool operator ==(ServiceHostAndPort h, Lease l) => h == l.HostAndPort; + public static bool operator !=(ServiceHostAndPort h, Lease l) => !(h == l); + + public static bool operator ==(Lease l, ServiceHostAndPort h) => l.HostAndPort == h; + public static bool operator !=(Lease l, ServiceHostAndPort h) => !(l == h); +} diff --git a/src/Ocelot/LoadBalancer/LeaseEventArgs.cs b/src/Ocelot/LoadBalancer/LeaseEventArgs.cs new file mode 100644 index 000000000..f15e37e85 --- /dev/null +++ b/src/Ocelot/LoadBalancer/LeaseEventArgs.cs @@ -0,0 +1,17 @@ +using Ocelot.Values; + +namespace Ocelot.LoadBalancer; + +public class LeaseEventArgs : EventArgs +{ + public LeaseEventArgs(Lease lease, Service service, int serviceIndex) + { + Lease = lease; + Service = service; + ServiceIndex = serviceIndex; + } + + public Lease Lease { get; } + public Service Service { get; } + public int ServiceIndex { get; } +} diff --git a/src/Ocelot/LoadBalancer/LoadBalancers/CookieStickySessions.cs b/src/Ocelot/LoadBalancer/LoadBalancers/CookieStickySessions.cs index a135855a4..5499c9b19 100644 --- a/src/Ocelot/LoadBalancer/LoadBalancers/CookieStickySessions.cs +++ b/src/Ocelot/LoadBalancer/LoadBalancers/CookieStickySessions.cs @@ -16,6 +16,8 @@ public class CookieStickySessions : ILoadBalancer private static readonly object Locker = new(); private static readonly Dictionary Stored = new(); // TODO Inject instead of static sharing + public string Type => nameof(CookieStickySessions); + public CookieStickySessions(ILoadBalancer loadBalancer, string cookieName, int keyExpiryInMs, IBus bus) { _bus = bus; @@ -40,7 +42,7 @@ private void CheckExpiry(StickySession sticky) } } - public Task> Lease(HttpContext httpContext) + public Task> LeaseAsync(HttpContext httpContext) { var route = httpContext.Items.DownstreamRoute(); var serviceName = route.LoadBalancerKey; @@ -56,7 +58,7 @@ public Task> Lease(HttpContext httpContext) } // There is no value in the store, so lease it now! - var next = _loadBalancer.Lease(httpContext).GetAwaiter().GetResult(); // unfortunately the operation must be synchronous + var next = _loadBalancer.LeaseAsync(httpContext).GetAwaiter().GetResult(); // unfortunately the operation must be synchronous if (next.IsError) { return Task.FromResult>(new ErrorResponse(next.Errors)); diff --git a/src/Ocelot/LoadBalancer/LoadBalancers/ILoadBalancer.cs b/src/Ocelot/LoadBalancer/LoadBalancers/ILoadBalancer.cs index ccf997c55..4070bc01f 100644 --- a/src/Ocelot/LoadBalancer/LoadBalancers/ILoadBalancer.cs +++ b/src/Ocelot/LoadBalancer/LoadBalancers/ILoadBalancer.cs @@ -1,13 +1,20 @@ using Microsoft.AspNetCore.Http; using Ocelot.Responses; using Ocelot.Values; +using System.Reflection; namespace Ocelot.LoadBalancer.LoadBalancers { + // TODO Add sync & async pairs public interface ILoadBalancer { - Task> Lease(HttpContext httpContext); + Task> LeaseAsync(HttpContext httpContext); void Release(ServiceHostAndPort hostAndPort); + + /// Static name of the load balancer instance. + /// To avoid reflection calls of the property of the objects. + /// A object with type name value. + string Type { get; } } } diff --git a/src/Ocelot/LoadBalancer/LoadBalancers/LeastConnection.cs b/src/Ocelot/LoadBalancer/LoadBalancers/LeastConnection.cs index bede3ba8e..21d19b11b 100644 --- a/src/Ocelot/LoadBalancer/LoadBalancers/LeastConnection.cs +++ b/src/Ocelot/LoadBalancer/LoadBalancers/LeastConnection.cs @@ -9,7 +9,9 @@ public class LeastConnection : ILoadBalancer private readonly Func>> _services; private readonly List _leases; private readonly string _serviceName; - private static readonly object SyncLock = new(); + private static readonly object SyncRoot = new(); + + public string Type => nameof(LeastConnection); public LeastConnection(Func>> services, string serviceName) { @@ -18,31 +20,38 @@ public LeastConnection(Func>> services, string serviceName) _leases = new List(); } - public async Task> Lease(HttpContext httpContext) + public event EventHandler Leased; + protected virtual void OnLeased(LeaseEventArgs e) => Leased?.Invoke(this, e); + + public async Task> LeaseAsync(HttpContext httpContext) { var services = await _services.Invoke(); if ((services?.Count ?? 0) == 0) { - return new ErrorResponse(new ServicesAreNullError($"Services were null/empty in {nameof(LeastConnection)} for '{_serviceName}' during {nameof(Lease)} operation!")); + return new ErrorResponse(new ServicesAreNullError($"Services were null/empty in {Type} for '{_serviceName}' during {nameof(LeaseAsync)} operation!")); } - lock (SyncLock) + lock (SyncRoot) { //todo - maybe this should be moved somewhere else...? Maybe on a repeater on seperate thread? loop every second and update or something? UpdateLeasing(services); Lease wanted = GetLeaseWithLeastConnections(); _ = Update(ref wanted, true); + + var index = services.FindIndex(s => s.HostAndPort == wanted); + OnLeased(new(wanted, services[index], index)); + return new OkResponse(new(wanted.HostAndPort)); } } public void Release(ServiceHostAndPort hostAndPort) { - lock (SyncLock) + lock (SyncRoot) { var matchingLease = _leases.Find(l => l == hostAndPort); - if (matchingLease != LoadBalancers.Lease.Null) + if (matchingLease != Lease.Null) { _ = Update(ref matchingLease, false); } diff --git a/src/Ocelot/LoadBalancer/LoadBalancers/LeastConnectionCreator.cs b/src/Ocelot/LoadBalancer/LoadBalancers/LeastConnectionCreator.cs index e5d15fa2d..faa071e9d 100644 --- a/src/Ocelot/LoadBalancer/LoadBalancers/LeastConnectionCreator.cs +++ b/src/Ocelot/LoadBalancer/LoadBalancers/LeastConnectionCreator.cs @@ -8,7 +8,12 @@ public class LeastConnectionCreator : ILoadBalancerCreator { public Response Create(DownstreamRoute route, IServiceDiscoveryProvider serviceProvider) { - return new OkResponse(new LeastConnection(async () => await serviceProvider.GetAsync(), route.ServiceName)); + var loadBalancer = new LeastConnection( + serviceProvider.GetAsync, + !string.IsNullOrEmpty(route.ServiceName) + ? route.ServiceName + : route.LoadBalancerKey); // if service discovery mode then use service name; otherwise use balancer key + return new OkResponse(loadBalancer); } public string Type => nameof(LeastConnection); diff --git a/src/Ocelot/LoadBalancer/LoadBalancers/LoadBalancerHouse.cs b/src/Ocelot/LoadBalancer/LoadBalancers/LoadBalancerHouse.cs index dfa6279e6..79cb72ad1 100644 --- a/src/Ocelot/LoadBalancer/LoadBalancers/LoadBalancerHouse.cs +++ b/src/Ocelot/LoadBalancer/LoadBalancers/LoadBalancerHouse.cs @@ -1,5 +1,4 @@ using Ocelot.Configuration; -using Ocelot.Errors; using Ocelot.Responses; namespace Ocelot.LoadBalancer.LoadBalancers @@ -7,57 +6,45 @@ namespace Ocelot.LoadBalancer.LoadBalancers public class LoadBalancerHouse : ILoadBalancerHouse { private readonly ILoadBalancerFactory _factory; - private readonly ConcurrentDictionary _loadBalancers; + private readonly Dictionary _loadBalancers; + private static readonly object SyncRoot = new(); public LoadBalancerHouse(ILoadBalancerFactory factory) { _factory = factory; - _loadBalancers = new ConcurrentDictionary(); + _loadBalancers = new(); } public Response Get(DownstreamRoute route, ServiceProviderConfiguration config) { try { - if (_loadBalancers.TryGetValue(route.LoadBalancerKey, out var loadBalancer)) - { - // TODO Fix ugly reflection issue of dymanic detection in favor of static type property - if (route.LoadBalancerOptions.Type != loadBalancer.GetType().Name) - { - return GetResponse(route, config); - } - - return new OkResponse(loadBalancer); + lock (SyncRoot) + { + return (_loadBalancers.TryGetValue(route.LoadBalancerKey, out var loadBalancer) && + route.LoadBalancerOptions.Type == loadBalancer.Type) // TODO Case insensitive? + ? new OkResponse(loadBalancer) + : GetResponse(route, config); } - - return GetResponse(route, config); } catch (Exception ex) { - return new ErrorResponse(new List() - { - new UnableToFindLoadBalancerError($"Unable to find load balancer for '{route.LoadBalancerKey}'. Exception: {ex};"), - }); + return new ErrorResponse( + new UnableToFindLoadBalancerError($"Unable to find load balancer for '{route.LoadBalancerKey}'. Exception: {ex};")); } } private Response GetResponse(DownstreamRoute route, ServiceProviderConfiguration config) { var result = _factory.Get(route, config); - if (result.IsError) { return new ErrorResponse(result.Errors); } - var loadBalancer = result.Data; - AddLoadBalancer(route.LoadBalancerKey, loadBalancer); - return new OkResponse(loadBalancer); - } - - private void AddLoadBalancer(string key, ILoadBalancer loadBalancer) - { - _loadBalancers.AddOrUpdate(key, loadBalancer, (x, y) => loadBalancer); + var balancer = result.Data; + _loadBalancers[route.LoadBalancerKey] = balancer; // TODO TryAdd ? + return new OkResponse(balancer); } } } diff --git a/src/Ocelot/LoadBalancer/LoadBalancers/NoLoadBalancer.cs b/src/Ocelot/LoadBalancer/LoadBalancers/NoLoadBalancer.cs index 725b0d33d..6d3c0a94c 100644 --- a/src/Ocelot/LoadBalancer/LoadBalancers/NoLoadBalancer.cs +++ b/src/Ocelot/LoadBalancer/LoadBalancers/NoLoadBalancer.cs @@ -13,13 +13,15 @@ public NoLoadBalancer(Func>> services) _services = services; } - public async Task> Lease(HttpContext httpContext) + public string Type => nameof(NoLoadBalancer); + + public async Task> LeaseAsync(HttpContext httpContext) { var services = await _services(); if (services == null || services.Count == 0) { - return new ErrorResponse(new ServicesAreEmptyError("There were no services in NoLoadBalancer")); + return new ErrorResponse(new ServicesAreEmptyError($"There were no services in {Type}!")); } var service = await Task.FromResult(services.FirstOrDefault()); diff --git a/src/Ocelot/LoadBalancer/LoadBalancers/RoundRobin.cs b/src/Ocelot/LoadBalancer/LoadBalancers/RoundRobin.cs index 9c087f77a..8febb5b29 100644 --- a/src/Ocelot/LoadBalancer/LoadBalancers/RoundRobin.cs +++ b/src/Ocelot/LoadBalancer/LoadBalancers/RoundRobin.cs @@ -10,6 +10,8 @@ public class RoundRobin : ILoadBalancer private readonly string _serviceName; private readonly List _leasing; + public string Type => nameof(RoundRobin); + public RoundRobin(Func>> services, string serviceName) { _servicesDelegate = services; @@ -23,12 +25,12 @@ public RoundRobin(Func>> services, string serviceName) public event EventHandler Leased; protected virtual void OnLeased(LeaseEventArgs e) => Leased?.Invoke(this, e); - public virtual async Task> Lease(HttpContext httpContext) + public virtual async Task> LeaseAsync(HttpContext httpContext) { var services = await _servicesDelegate?.Invoke() ?? new List(); if (services.Count == 0) { - return new ErrorResponse(new ServicesAreEmptyError($"There were no services in {nameof(RoundRobin)} for '{_serviceName}' during {nameof(Lease)} operation!")); + return new ErrorResponse(new ServicesAreEmptyError($"There were no services in {Type} for '{_serviceName}' during {nameof(LeaseAsync)} operation!")); } lock (SyncRoot) @@ -36,7 +38,7 @@ public virtual async Task> Lease(HttpContext httpCo var readMe = CaptureState(services, out int count); if (!TryScanNext(readMe, out Service next, out int index)) { - return new ErrorResponse(new ServicesAreNullError($"The service at index {index} was null in {nameof(RoundRobin)} for {_serviceName} during the {nameof(Lease)} operation. Total services count: {count}.")); + return new ErrorResponse(new ServicesAreNullError($"The service at index {index} was null in {Type} for {_serviceName} during the {nameof(LeaseAsync)} operation. Total services count: {count}.")); } ProcessLeasing(readMe, next, index); // Happy path: Lease now @@ -117,17 +119,3 @@ private void UpdateLeasing(IList services) _leasing.AddRange(newLeases); } } - -public class LeaseEventArgs : EventArgs -{ - public LeaseEventArgs(Lease lease, Service service, int serviceIndex) - { - Lease = lease; - Service = service; - ServiceIndex = serviceIndex; - } - - public Lease Lease { get; } - public Service Service { get; } - public int ServiceIndex { get; } -} diff --git a/src/Ocelot/LoadBalancer/LoadBalancers/RoundRobinCreator.cs b/src/Ocelot/LoadBalancer/LoadBalancers/RoundRobinCreator.cs index 057fa95e6..c45720e28 100644 --- a/src/Ocelot/LoadBalancer/LoadBalancers/RoundRobinCreator.cs +++ b/src/Ocelot/LoadBalancer/LoadBalancers/RoundRobinCreator.cs @@ -8,7 +8,11 @@ public class RoundRobinCreator : ILoadBalancerCreator { public Response Create(DownstreamRoute route, IServiceDiscoveryProvider serviceProvider) { - var loadBalancer = new RoundRobin(serviceProvider.GetAsync, route.ServiceName); + var loadBalancer = new RoundRobin( + serviceProvider.GetAsync, + !string.IsNullOrEmpty(route.ServiceName) + ? route.ServiceName + : route.LoadBalancerKey); // if service discovery mode then use service name; otherwise use balancer key return new OkResponse(loadBalancer); } diff --git a/src/Ocelot/LoadBalancer/Middleware/LoadBalancingMiddleware.cs b/src/Ocelot/LoadBalancer/Middleware/LoadBalancingMiddleware.cs index bc894fc55..fa455198b 100644 --- a/src/Ocelot/LoadBalancer/Middleware/LoadBalancingMiddleware.cs +++ b/src/Ocelot/LoadBalancer/Middleware/LoadBalancingMiddleware.cs @@ -34,7 +34,7 @@ public async Task Invoke(HttpContext httpContext) return; } - var hostAndPort = await loadBalancer.Data.Lease(httpContext); + var hostAndPort = await loadBalancer.Data.LeaseAsync(httpContext); if (hostAndPort.IsError) { Logger.LogDebug("there was an error leasing the loadbalancer, setting pipeline error"); diff --git a/src/Ocelot/ServiceDiscovery/Providers/ConfigurationServiceProvider.cs b/src/Ocelot/ServiceDiscovery/Providers/ConfigurationServiceProvider.cs index b417c4c7c..dc09c94b9 100644 --- a/src/Ocelot/ServiceDiscovery/Providers/ConfigurationServiceProvider.cs +++ b/src/Ocelot/ServiceDiscovery/Providers/ConfigurationServiceProvider.cs @@ -1,19 +1,12 @@ using Ocelot.Values; -namespace Ocelot.ServiceDiscovery.Providers +namespace Ocelot.ServiceDiscovery.Providers; + +public class ConfigurationServiceProvider : IServiceDiscoveryProvider { - public class ConfigurationServiceProvider : IServiceDiscoveryProvider - { - private readonly List _services; + private readonly List _services; - public ConfigurationServiceProvider(List services) - { - _services = services; - } + public ConfigurationServiceProvider(List services) => _services = services; - public async Task> GetAsync() - { - return await Task.FromResult(_services); - } - } + public Task> GetAsync() => ValueTask.FromResult(_services).AsTask(); } diff --git a/test/Ocelot.AcceptanceTests/ConcurrentSteps.cs b/test/Ocelot.AcceptanceTests/ConcurrentSteps.cs new file mode 100644 index 000000000..6d323452f --- /dev/null +++ b/test/Ocelot.AcceptanceTests/ConcurrentSteps.cs @@ -0,0 +1,271 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Primitives; +using Ocelot.AcceptanceTests.LoadBalancer; +using Ocelot.LoadBalancer; +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Text; + +namespace Ocelot.AcceptanceTests; + +public class ConcurrentSteps : Steps, IDisposable +{ + protected Task[] _tasks; + protected ServiceHandler[] _handlers; + protected ConcurrentDictionary _responses; + protected volatile int[] _counters; + + public ConcurrentSteps() + { + _tasks = Array.Empty(); + _handlers = Array.Empty(); + _responses = new(); + _counters = Array.Empty(); + } + + public override void Dispose() + { + foreach (var handler in _handlers) + { + handler?.Dispose(); + } + + foreach (var response in _responses.Values) + { + response?.Dispose(); + } + + foreach (var task in _tasks) + { + task?.Dispose(); + } + + base.Dispose(); + GC.SuppressFinalize(this); + } + + protected void GivenServiceInstanceIsRunning(string url, string response) + => GivenServiceInstanceIsRunning(url, response, HttpStatusCode.OK); + + protected void GivenServiceInstanceIsRunning(string url, string response, HttpStatusCode statusCode) + { + _handlers = new ServiceHandler[1]; // allocate single instance + _counters = new int[1]; // single counter + GivenServiceIsRunning(url, response, 0, statusCode); + _counters[0] = 0; + } + + protected void GivenThereIsAServiceRunningOn(string url, string basePath, string responseBody) + { + var handler = new ServiceHandler(); + _handlers = new ServiceHandler[] { handler }; + handler.GivenThereIsAServiceRunningOn(url, basePath, MapGet(basePath, responseBody)); + } + + protected void GivenMultipleServiceInstancesAreRunning(string[] urls, [CallerMemberName] string serviceName = null) + { + serviceName ??= new Uri(urls[0]).Host; + string[] responses = urls.Select(u => $"{serviceName}|url({u})").ToArray(); + GivenMultipleServiceInstancesAreRunning(urls, responses, HttpStatusCode.OK); + } + + protected void GivenMultipleServiceInstancesAreRunning(string[] urls, string[] responses) + => GivenMultipleServiceInstancesAreRunning(urls, responses, HttpStatusCode.OK); + + protected void GivenMultipleServiceInstancesAreRunning(string[] urls, string[] responses, HttpStatusCode statusCode) + { + Debug.Assert(urls.Length == responses.Length, "Length mismatch!"); + _handlers = new ServiceHandler[urls.Length]; // allocate multiple instances + _counters = new int[urls.Length]; // multiple counters + for (int i = 0; i < urls.Length; i++) + { + GivenServiceIsRunning(urls[i], responses[i], i, statusCode); + _counters[i] = 0; + } + } + + private void GivenServiceIsRunning(string url, string response) + => GivenServiceIsRunning(url, response, 0, HttpStatusCode.OK); + private void GivenServiceIsRunning(string url, string response, int index) + => GivenServiceIsRunning(url, response, index, HttpStatusCode.OK); + + private void GivenServiceIsRunning(string url, string response, int index, HttpStatusCode successCode) + { + response ??= successCode.ToString(); + _handlers[index] ??= new(); + var serviceHandler = _handlers[index]; + serviceHandler.GivenThereIsAServiceRunningOn(url, MapGet(index, response, successCode)); + } + + protected static RequestDelegate MapGet(string path, string responseBody) => MapGet(path, responseBody, HttpStatusCode.OK); + protected static RequestDelegate MapGet(string path, string responseBody, HttpStatusCode statusCode) => async context => + { + var downstreamPath = !string.IsNullOrEmpty(context.Request.PathBase.Value) + ? context.Request.PathBase.Value + : context.Request.Path.Value; + bool isMatch = downstreamPath == path; + context.Response.StatusCode = (int)(isMatch ? statusCode : HttpStatusCode.NotFound); + await context.Response.WriteAsync(isMatch ? responseBody : "Not Found"); + }; + + public static class HeaderNames + { + public const string ServiceIndex = nameof(LeaseEventArgs.ServiceIndex); + public const string Host = nameof(Uri.Host); + public const string Port = nameof(Uri.Port); + public const string Counter = nameof(Counter); + } + + protected RequestDelegate MapGet(int index, string body) => MapGet(index, body, HttpStatusCode.OK); + protected RequestDelegate MapGet(int index, string body, HttpStatusCode successCode) => async context => + { + // Don't delay during the first service call + if (Volatile.Read(ref _counters[index]) > 0) + { + await Task.Delay(Random.Shared.Next(5, 15)); // emulate integration delay up to 15 milliseconds + } + + string responseBody; + var request = context.Request; + var response = context.Response; + try + { + int count = Interlocked.Increment(ref _counters[index]); + responseBody = string.Concat(count, ':', body); + + response.StatusCode = (int)successCode; + response.Headers.Append(HeaderNames.ServiceIndex, new StringValues(index.ToString())); + response.Headers.Append(HeaderNames.Host, new StringValues(request.Host.Host)); + response.Headers.Append(HeaderNames.Port, new StringValues(request.Host.Port.ToString())); + response.Headers.Append(HeaderNames.Counter, new StringValues(count.ToString())); + await response.WriteAsync(responseBody); + } + catch (Exception exception) + { + responseBody = string.Concat(1, ':', exception.StackTrace); + response.StatusCode = (int)HttpStatusCode.InternalServerError; + await response.WriteAsync(responseBody); + } + }; + + public Task[] WhenIGetUrlOnTheApiGatewayConcurrently(string url, int times) + => RunParallelRequests(times, (i) => url); + + public Task[] WhenIGetUrlOnTheApiGatewayConcurrently(int times, params string[] urls) + => RunParallelRequests(times, (i) => urls[i % urls.Length]); + + protected Task[] RunParallelRequests(int times, Func urlFunc) + { + _tasks = new Task[times]; + _responses = new(times, times); + for (var i = 0; i < times; i++) + { + var url = urlFunc(i); + _tasks[i] = GetParallelResponse(url, i); + _responses[i] = null; + } + + Task.WaitAll(_tasks); + return _tasks; + } + + private async Task GetParallelResponse(string url, int threadIndex) + { + var response = await _ocelotClient.GetAsync(url); + var content = await response.Content.ReadAsStringAsync(); + var counterString = content.Contains(':') + ? content.Split(':')[0] // let the first fragment is counter value + : "0"; + int count = int.Parse(counterString); + count.ShouldBeGreaterThan(0); + _responses[threadIndex] = response; + } + + public void ThenAllStatusCodesShouldBe(HttpStatusCode expected) + => _responses.ShouldAllBe(response => response.Value.StatusCode == expected); + public void ThenAllResponseBodiesShouldBe(string expectedBody) + => _responses.ShouldAllBe(response => response.Value.Content.ReadAsStringAsync().Result == expectedBody); + + protected string CalledTimesMessage() + => $"All values are [{string.Join(',', _counters)}]"; + + public void ThenAllServicesShouldHaveBeenCalledTimes(int expected) + => _counters.Sum().ShouldBe(expected, CalledTimesMessage()); + + public void ThenServiceShouldHaveBeenCalledTimes(int index, int expected) + => _counters[index].ShouldBe(expected, CalledTimesMessage()); + + public void ThenServicesShouldHaveBeenCalledTimes(params int[] expected) + { + for (int i = 0; i < expected.Length; i++) + { + _counters[i].ShouldBe(expected[i], CalledTimesMessage()); + } + } + + public static int Bottom(int totalRequests, int totalServices) + => totalRequests / totalServices; + public static int Top(int totalRequests, int totalServices) + { + int bottom = Bottom(totalRequests, totalServices); + return totalRequests - (bottom * totalServices) + bottom; + } + + public void ThenAllServicesCalledRealisticAmountOfTimes(int bottom, int top) + { + var customMessage = new StringBuilder() + .AppendLine($"{nameof(bottom)}: {bottom}") + .AppendLine($" {nameof(top)}: {top}") + .AppendLine($" All values are [{string.Join(',', _counters)}]") + .ToString(); + int sum = 0, totalSum = _counters.Sum(); + + // Last offline services cannot be called at all, thus don't assert zero counters + for (int i = 0; i < _counters.Length && sum < totalSum; i++) + { + int actual = _counters[i]; + actual.ShouldBeInRange(bottom, top, customMessage); + sum += actual; + } + } + + public void ThenAllServicesCalledOptimisticAmountOfTimes(ILoadBalancerAnalyzer analyzer) + { + if (analyzer == null) return; + int bottom = analyzer.BottomOfConnections(), + top = analyzer.TopOfConnections(); + ThenAllServicesCalledRealisticAmountOfTimes(bottom, top); // with unstable checkings + } + + public void ThenServiceCountersShouldMatchLeasingCounters(ILoadBalancerAnalyzer analyzer, int[] ports, int totalRequests) + { + if (analyzer == null || ports == null) + return; + + analyzer.ShouldNotBeNull().Analyze(); + analyzer.Events.Count.ShouldBe(totalRequests, $"{nameof(ILoadBalancerAnalyzer.ServiceName)}: {analyzer.ServiceName}"); + + var leasingCounters = analyzer?.GetHostCounters() ?? new(); + var sortedLeasingCountersByPort = ports.Select(port => leasingCounters.FirstOrDefault(kv => kv.Key.DownstreamPort == port).Value).ToArray(); + for (int i = 0; i < ports.Length; i++) + { + var host = leasingCounters.Keys.FirstOrDefault(k => k.DownstreamPort == ports[i]); + + // Leasing info/counters can be absent because of offline service instance with exact port in unstable scenario + if (host != null) + { + var customMessage = new StringBuilder() + .AppendLine($"{nameof(ILoadBalancerAnalyzer.ServiceName)}: {analyzer.ServiceName}") + .AppendLine($" Port: {ports[i]}") + .AppendLine($" Host: {host}") + .AppendLine($" Service counters: [{string.Join(',', _counters)}]") + .AppendLine($" Leasing counters: [{string.Join(',', sortedLeasingCountersByPort)}]") // should have order of _counters + .ToString(); + int counter1 = _counters[i]; + int counter2 = leasingCounters[host]; + counter1.ShouldBe(counter2, customMessage); + } + } + } +} diff --git a/test/Ocelot.AcceptanceTests/LoadBalancer/ILoadBalancerAnalyzer.cs b/test/Ocelot.AcceptanceTests/LoadBalancer/ILoadBalancerAnalyzer.cs new file mode 100644 index 000000000..5b2acc6ca --- /dev/null +++ b/test/Ocelot.AcceptanceTests/LoadBalancer/ILoadBalancerAnalyzer.cs @@ -0,0 +1,18 @@ +using Ocelot.LoadBalancer; +using Ocelot.Values; +using System.Collections.Concurrent; + +namespace Ocelot.AcceptanceTests.LoadBalancer; + +public interface ILoadBalancerAnalyzer +{ + string ServiceName { get; } + string GenerationPrefix { get; } + ConcurrentBag Events { get; } + object Analyze(); + Dictionary GetHostCounters(); + Dictionary ToHostCountersDictionary(IEnumerable> grouping); + bool HasManyServiceGenerations(int maxGeneration); + int BottomOfConnections(); + int TopOfConnections(); +} diff --git a/test/Ocelot.AcceptanceTests/LoadBalancer/LeastConnectionAnalyzer.cs b/test/Ocelot.AcceptanceTests/LoadBalancer/LeastConnectionAnalyzer.cs new file mode 100644 index 000000000..79ee9cf72 --- /dev/null +++ b/test/Ocelot.AcceptanceTests/LoadBalancer/LeastConnectionAnalyzer.cs @@ -0,0 +1,28 @@ +using Microsoft.AspNetCore.Http; +using Ocelot.LoadBalancer; +using Ocelot.LoadBalancer.LoadBalancers; +using Ocelot.Responses; +using Ocelot.Values; + +namespace Ocelot.AcceptanceTests.LoadBalancer; + +internal sealed class LeastConnectionAnalyzer : LoadBalancerAnalyzer, ILoadBalancer +{ + private readonly LeastConnection loadBalancer; + + public LeastConnectionAnalyzer(Func>> services, string serviceName) + : base(serviceName) + { + loadBalancer = new(services, serviceName); + loadBalancer.Leased += Me_Leased; + } + + private void Me_Leased(object sender, LeaseEventArgs args) => Events.Add(args); + + public override string Type => nameof(LeastConnectionAnalyzer); + public override Task> LeaseAsync(HttpContext httpContext) => loadBalancer.LeaseAsync(httpContext); + public override void Release(ServiceHostAndPort hostAndPort) => loadBalancer.Release(hostAndPort); + + public override Dictionary ToHostCountersDictionary(IEnumerable> grouping) + => grouping.ToDictionary(g => g.Key, g => g.Count(e => e.Lease == g.Key)); +} diff --git a/test/Ocelot.AcceptanceTests/LoadBalancer/LeastConnectionAnalyzerCreator.cs b/test/Ocelot.AcceptanceTests/LoadBalancer/LeastConnectionAnalyzerCreator.cs new file mode 100644 index 000000000..785189ce2 --- /dev/null +++ b/test/Ocelot.AcceptanceTests/LoadBalancer/LeastConnectionAnalyzerCreator.cs @@ -0,0 +1,22 @@ +using Ocelot.Configuration; +using Ocelot.LoadBalancer.LoadBalancers; +using Ocelot.Responses; +using Ocelot.ServiceDiscovery.Providers; + +namespace Ocelot.AcceptanceTests.LoadBalancer; + +internal sealed class LeastConnectionAnalyzerCreator : ILoadBalancerCreator +{ + // We need to adhere to the same implementations of RoundRobinCreator, which results in a significant design overhead, (until redesigned) + public Response Create(DownstreamRoute route, IServiceDiscoveryProvider serviceProvider) + { + var loadBalancer = new LeastConnectionAnalyzer( + serviceProvider.GetAsync, + !string.IsNullOrEmpty(route.ServiceName) // if service discovery mode then use service name; otherwise use balancer key + ? route.ServiceName + : route.LoadBalancerKey); + return new OkResponse(loadBalancer); + } + + public string Type => nameof(LeastConnectionAnalyzer); +} diff --git a/test/Ocelot.AcceptanceTests/LoadBalancer/LoadBalancerAnalyzer.cs b/test/Ocelot.AcceptanceTests/LoadBalancer/LoadBalancerAnalyzer.cs new file mode 100644 index 000000000..dc19a51e4 --- /dev/null +++ b/test/Ocelot.AcceptanceTests/LoadBalancer/LoadBalancerAnalyzer.cs @@ -0,0 +1,118 @@ +using Microsoft.AspNetCore.Http; +using Ocelot.LoadBalancer; +using Ocelot.LoadBalancer.LoadBalancers; +using Ocelot.Responses; +using Ocelot.Values; +using System.Collections.Concurrent; + +namespace Ocelot.AcceptanceTests.LoadBalancer; + +internal class LoadBalancerAnalyzer : ILoadBalancerAnalyzer, ILoadBalancer +{ + protected readonly string _serviceName; + protected LoadBalancerAnalyzer(string serviceName) => _serviceName = serviceName; + + public string ServiceName => _serviceName; + public virtual string GenerationPrefix => "Gen:"; + public ConcurrentBag Events { get; } = new(); + + public virtual object Analyze() + { + var allGenerations = Events + .Select(e => e.Service.Tags.FirstOrDefault(t => t.StartsWith(GenerationPrefix))) + .Where(generation => !string.IsNullOrEmpty(generation)) + .Distinct().ToArray(); + var allIndices = Events.Select(e => e.ServiceIndex) + .Distinct().OrderBy(index => index).ToArray(); + + Dictionary> eventsPerGeneration = new(); + foreach (var generation in allGenerations) + { + var l = Events.Where(e => e.Service.Tags.Contains(generation)).ToList(); + eventsPerGeneration.Add(generation, l); + } + + Dictionary> generationIndices = new(); + foreach (var generation in allGenerations) + { + var l = eventsPerGeneration[generation].Select(e => e.ServiceIndex).Distinct().ToList(); + generationIndices.Add(generation, l); + } + + Dictionary> generationLeases = new(); + foreach (var generation in allGenerations) + { + var l = eventsPerGeneration[generation].Select(e => e.Lease).ToList(); + generationLeases.Add(generation, l); + } + + Dictionary> generationHosts = new(); + foreach (var generation in allGenerations) + { + var l = eventsPerGeneration[generation].Select(e => e.Lease.HostAndPort).Distinct().ToList(); + generationHosts.Add(generation, l); + } + + Dictionary> generationLeasesWithMaxConnections = new(); + foreach (var generation in allGenerations) + { + List leases = new(); + var uniqueHosts = generationHosts[generation]; + foreach (var host in uniqueHosts) + { + int max = generationLeases[generation].Where(l => l == host).Max(l => l.Connections); + Lease wanted = generationLeases[generation].Find(l => l == host && l.Connections == max); + leases.Add(wanted); + } + + leases = leases.OrderBy(l => l.HostAndPort.DownstreamPort).ToList(); + generationLeasesWithMaxConnections.Add(generation, leases); + } + + return generationLeasesWithMaxConnections; + } + + public virtual bool HasManyServiceGenerations(int maxGeneration) + { + int[] generations = new int[maxGeneration + 1]; + string[] tags = new string[maxGeneration + 1]; + for (int i = 0; i < generations.Length; i++) + { + generations[i] = i; + tags[i] = GenerationPrefix + i; + } + + var all = Events + .Select(e => e.Service.Tags.FirstOrDefault(t => t.StartsWith(GenerationPrefix))) + .Distinct().ToArray(); + return all.All(tags.Contains); + } + + public virtual Dictionary GetHostCounters() + { + var hosts = Events.Select(e => e.Lease.HostAndPort).Distinct().ToList(); + var grouping = Events + .GroupBy(e => e.Lease.HostAndPort) + .OrderBy(g => g.Key.DownstreamPort); + return ToHostCountersDictionary(grouping); + } + + public virtual Dictionary ToHostCountersDictionary(IEnumerable> grouping) + => grouping.ToDictionary(g => g.Key, g => g.Count(e => e.Lease == g.Key)); + + public virtual int BottomOfConnections() + { + var hostCounters = GetHostCounters(); + return hostCounters.Min(_ => _.Value); + } + + public virtual int TopOfConnections() + { + var hostCounters = GetHostCounters(); + return hostCounters.Max(_ => _.Value); + } + + public virtual string Type => nameof(LoadBalancerAnalyzer); + public virtual Task> LeaseAsync(HttpContext httpContext) => Task.FromResult>(new ErrorResponse(new UnableToFindLoadBalancerError(GetType().Name))); + public virtual void Release(ServiceHostAndPort hostAndPort) { } +} diff --git a/test/Ocelot.AcceptanceTests/LoadBalancer/LoadBalancerTests.cs b/test/Ocelot.AcceptanceTests/LoadBalancer/LoadBalancerTests.cs new file mode 100644 index 000000000..b92a3d392 --- /dev/null +++ b/test/Ocelot.AcceptanceTests/LoadBalancer/LoadBalancerTests.cs @@ -0,0 +1,129 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Ocelot.Configuration; +using Ocelot.Configuration.File; +using Ocelot.DependencyInjection; +using Ocelot.LoadBalancer.LoadBalancers; +using Ocelot.Responses; +using Ocelot.ServiceDiscovery.Providers; +using Ocelot.Values; + +namespace Ocelot.AcceptanceTests.LoadBalancer; + +public sealed class LoadBalancerTests : ConcurrentSteps, IDisposable +{ + [Theory] + [Trait("Feat", "211")] + [InlineData(false)] // original scenario, clean config + [InlineData(true)] // extended scenario using analyzer + public void ShouldLoadBalanceRequestWithLeastConnection(bool withAnalyzer) + { + var ports = PortFinder.GetPorts(2); + var route = GivenRoute(withAnalyzer ? nameof(LeastConnectionAnalyzer) : nameof(LeastConnection), ports); + var configuration = GivenConfiguration(route); + var downstreamServiceUrls = ports.Select(DownstreamUrl).ToArray(); + LeastConnectionAnalyzer lbAnalyzer = null; + LeastConnectionAnalyzer getAnalyzer(DownstreamRoute route, IServiceDiscoveryProvider provider) + { + //lock (LoadBalancerHouse.SyncRoot) // Note, synch locking is implemented in LoadBalancerHouse + return lbAnalyzer ??= new LeastConnectionAnalyzerCreator().Create(route, provider)?.Data as LeastConnectionAnalyzer; + } + Action withLeastConnectionAnalyzer = (s) + => s.AddOcelot().AddCustomLoadBalancer(getAnalyzer); + GivenMultipleServiceInstancesAreRunning(downstreamServiceUrls); + this.Given(x => GivenThereIsAConfiguration(configuration)) + .And(x => GivenOcelotIsRunningWithServices(withAnalyzer ? withLeastConnectionAnalyzer : WithAddOcelot)) + .When(x => WhenIGetUrlOnTheApiGatewayConcurrently("/", 99)) + .Then(x => ThenAllServicesShouldHaveBeenCalledTimes(99)) + .And(x => ThenAllServicesCalledOptimisticAmountOfTimes(lbAnalyzer)) + .And(x => ThenServiceCountersShouldMatchLeasingCounters(lbAnalyzer, ports, 99)) + .And(x => ThenAllServicesCalledRealisticAmountOfTimes(Bottom(99, ports.Length), Top(99, ports.Length))) + .And(x => ThenServicesShouldHaveBeenCalledTimes(50, 49)) // strict assertion + .BDDfy(); + } + + [Theory] + [Trait("Bug", "365")] + [InlineData(false)] // original scenario, clean config + [InlineData(true)] // extended scenario using analyzer + public void ShouldLoadBalanceRequestWithRoundRobin(bool withAnalyzer) + { + var ports = PortFinder.GetPorts(2); + var route = GivenRoute(withAnalyzer ? nameof(RoundRobinAnalyzer) : nameof(RoundRobin), ports); + var configuration = GivenConfiguration(route); + var downstreamServiceUrls = ports.Select(DownstreamUrl).ToArray(); + RoundRobinAnalyzer lbAnalyzer = null; + RoundRobinAnalyzer getAnalyzer(DownstreamRoute route, IServiceDiscoveryProvider provider) + { + //lock (LoadBalancerHouse.SyncRoot) // Note, synch locking is implemented in LoadBalancerHouse + return lbAnalyzer ??= new RoundRobinAnalyzerCreator().Create(route, provider)?.Data as RoundRobinAnalyzer; + } + Action withRoundRobinAnalyzer = (s) + => s.AddOcelot().AddCustomLoadBalancer(getAnalyzer); + GivenMultipleServiceInstancesAreRunning(downstreamServiceUrls); + this.Given(x => GivenThereIsAConfiguration(configuration)) + .And(x => GivenOcelotIsRunningWithServices(withAnalyzer ? withRoundRobinAnalyzer : WithAddOcelot)) + .When(x => WhenIGetUrlOnTheApiGatewayConcurrently("/", 99)) + .Then(x => ThenAllServicesShouldHaveBeenCalledTimes(99)) + .And(x => ThenAllServicesCalledOptimisticAmountOfTimes(lbAnalyzer)) + .And(x => ThenServiceCountersShouldMatchLeasingCounters(lbAnalyzer, ports, 99)) + .And(x => ThenAllServicesCalledRealisticAmountOfTimes(Bottom(99, ports.Length), Top(99, ports.Length))) + .And(x => ThenServicesShouldHaveBeenCalledTimes(50, 49)) // strict assertion + .BDDfy(); + } + + [Fact] + [Trait("Feat", "961")] + public void ShouldLoadBalanceRequestWithCustomLoadBalancer() + { + Func loadBalancerFactoryFunc = + (serviceProvider, route, discoveryProvider) => new CustomLoadBalancer(discoveryProvider.GetAsync); + var ports = PortFinder.GetPorts(2); + var route = GivenRoute(nameof(CustomLoadBalancer), ports); + var configuration = GivenConfiguration(route); + var downstreamServiceUrls = ports.Select(DownstreamUrl).ToArray(); + Action withCustomLoadBalancer = (s) + => s.AddOcelot().AddCustomLoadBalancer(loadBalancerFactoryFunc); + GivenMultipleServiceInstancesAreRunning(downstreamServiceUrls); + this.Given(x => GivenThereIsAConfiguration(configuration)) + .And(x => GivenOcelotIsRunningWithServices(withCustomLoadBalancer)) + .When(x => WhenIGetUrlOnTheApiGatewayConcurrently("/", 50)) + .Then(x => ThenAllServicesShouldHaveBeenCalledTimes(50)) + .And(x => ThenAllServicesCalledRealisticAmountOfTimes(Bottom(50, ports.Length), Top(50, ports.Length))) + .And(x => ThenServicesShouldHaveBeenCalledTimes(25, 25)) // strict assertion + .BDDfy(); + } + + private sealed class CustomLoadBalancer : ILoadBalancer + { + private readonly Func>> _services; + private static readonly object _lock = new(); + private int _last; + + public string Type => nameof(CustomLoadBalancer); + public CustomLoadBalancer(Func>> services) => _services = services; + + public async Task> LeaseAsync(HttpContext httpContext) + { + var services = await _services(); + lock (_lock) + { + if (_last >= services.Count) _last = 0; + var next = services[_last++]; + return new OkResponse(next.HostAndPort); + } + } + + public void Release(ServiceHostAndPort hostAndPort) { } + } + + private FileRoute GivenRoute(string loadBalancer, params int[] ports) => new() + { + DownstreamPathTemplate = "/", + DownstreamScheme = Uri.UriSchemeHttp, + UpstreamPathTemplate = "/", + UpstreamHttpMethod = new() { HttpMethods.Get }, + LoadBalancerOptions = new() { Type = loadBalancer ?? nameof(LeastConnection) }, + DownstreamHostAndPorts = ports.Select(Localhost).ToList(), + }; +} diff --git a/test/Ocelot.AcceptanceTests/LoadBalancer/RoundRobinAnalyzer.cs b/test/Ocelot.AcceptanceTests/LoadBalancer/RoundRobinAnalyzer.cs new file mode 100644 index 000000000..8f5f479ee --- /dev/null +++ b/test/Ocelot.AcceptanceTests/LoadBalancer/RoundRobinAnalyzer.cs @@ -0,0 +1,31 @@ +using KubeClient.Models; +using Microsoft.AspNetCore.Http; +using Ocelot.LoadBalancer; +using Ocelot.LoadBalancer.LoadBalancers; +using Ocelot.Responses; +using Ocelot.Values; + +namespace Ocelot.AcceptanceTests.LoadBalancer; + +internal sealed class RoundRobinAnalyzer : LoadBalancerAnalyzer, ILoadBalancer +{ + private readonly RoundRobin loadBalancer; + + public RoundRobinAnalyzer(Func>> services, string serviceName) + : base(serviceName) + { + loadBalancer = new(services, serviceName); + loadBalancer.Leased += Me_Leased; + } + + private void Me_Leased(object sender, LeaseEventArgs args) => Events.Add(args); + + public override string Type => nameof(RoundRobinAnalyzer); + public override Task> LeaseAsync(HttpContext httpContext) => loadBalancer.LeaseAsync(httpContext); + public override void Release(ServiceHostAndPort hostAndPort) => loadBalancer.Release(hostAndPort); + + public override string GenerationPrefix => nameof(EndpointsV1.Metadata.Generation) + ":"; + + public override Dictionary ToHostCountersDictionary(IEnumerable> grouping) + => grouping.ToDictionary(g => g.Key, g => g.Max(e => e.Lease.Connections)); +} diff --git a/test/Ocelot.AcceptanceTests/LoadBalancer/RoundRobinAnalyzerCreator.cs b/test/Ocelot.AcceptanceTests/LoadBalancer/RoundRobinAnalyzerCreator.cs new file mode 100644 index 000000000..a8f7a2c44 --- /dev/null +++ b/test/Ocelot.AcceptanceTests/LoadBalancer/RoundRobinAnalyzerCreator.cs @@ -0,0 +1,22 @@ +using Ocelot.Configuration; +using Ocelot.LoadBalancer.LoadBalancers; +using Ocelot.Responses; +using Ocelot.ServiceDiscovery.Providers; + +namespace Ocelot.AcceptanceTests.LoadBalancer; + +internal sealed class RoundRobinAnalyzerCreator : ILoadBalancerCreator +{ + // We need to adhere to the same implementations of RoundRobinCreator, which results in a significant design overhead, (until redesigned) + public Response Create(DownstreamRoute route, IServiceDiscoveryProvider serviceProvider) + { + var loadBalancer = new RoundRobinAnalyzer( + serviceProvider.GetAsync, + !string.IsNullOrEmpty(route.ServiceName) // if service discovery mode then use service name; otherwise use balancer key + ? route.ServiceName + : route.LoadBalancerKey); + return new OkResponse(loadBalancer); + } + + public string Type => nameof(RoundRobinAnalyzer); +} diff --git a/test/Ocelot.AcceptanceTests/LoadBalancerTests.cs b/test/Ocelot.AcceptanceTests/LoadBalancerTests.cs deleted file mode 100644 index 104cdaaea..000000000 --- a/test/Ocelot.AcceptanceTests/LoadBalancerTests.cs +++ /dev/null @@ -1,274 +0,0 @@ -using Microsoft.AspNetCore.Http; -using Ocelot.Configuration; -using Ocelot.Configuration.File; -using Ocelot.LoadBalancer.LoadBalancers; -using Ocelot.Responses; -using Ocelot.ServiceDiscovery.Providers; -using Ocelot.Values; - -namespace Ocelot.AcceptanceTests; - -public sealed class LoadBalancerTests : IDisposable -{ - private readonly Steps _steps; - private int _counterOne; - private int _counterTwo; - private static readonly object SyncLock = new(); - private readonly ServiceHandler _serviceHandler; - - public LoadBalancerTests() - { - _serviceHandler = new ServiceHandler(); - _steps = new Steps(); - } - - [Fact] - public void Should_load_balance_request_with_least_connection() - { - var portOne = PortFinder.GetRandomPort(); - var portTwo = PortFinder.GetRandomPort(); - - var downstreamServiceOneUrl = $"http://localhost:{portOne}"; - var downstreamServiceTwoUrl = $"http://localhost:{portTwo}"; - - var configuration = new FileConfiguration - { - Routes = new List - { - new() - { - DownstreamPathTemplate = "/", - DownstreamScheme = "http", - UpstreamPathTemplate = "/", - UpstreamHttpMethod = new List { "Get" }, - LoadBalancerOptions = new FileLoadBalancerOptions { Type = nameof(LeastConnection) }, - DownstreamHostAndPorts = new List - { - new() - { - Host = "localhost", - Port = portOne, - }, - new() - { - Host = "localhost", - Port = portTwo, - }, - }, - }, - }, - GlobalConfiguration = new FileGlobalConfiguration(), - }; - - this.Given(x => x.GivenProductServiceOneIsRunning(downstreamServiceOneUrl, 200)) - .And(x => x.GivenProductServiceTwoIsRunning(downstreamServiceTwoUrl, 200)) - .And(x => _steps.GivenThereIsAConfiguration(configuration)) - .And(x => _steps.GivenOcelotIsRunning()) - .When(x => _steps.WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 50)) - .Then(x => x.ThenTheTwoServicesShouldHaveBeenCalledTimes(50)) - - // Quite risky assertion because the actual values based on health checks and threading - //.And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(24, 26)) - .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(1, 49)) - .BDDfy(); - } - - [Fact] - public void Should_load_balance_request_with_round_robin() - { - var downstreamPortOne = PortFinder.GetRandomPort(); - var downstreamPortTwo = PortFinder.GetRandomPort(); - var downstreamServiceOneUrl = $"http://localhost:{downstreamPortOne}"; - var downstreamServiceTwoUrl = $"http://localhost:{downstreamPortTwo}"; - - var configuration = new FileConfiguration - { - Routes = new List - { - new() - { - DownstreamPathTemplate = "/", - DownstreamScheme = "http", - UpstreamPathTemplate = "/", - UpstreamHttpMethod = new List { "Get" }, - LoadBalancerOptions = new FileLoadBalancerOptions { Type = nameof(RoundRobin) }, - DownstreamHostAndPorts = new List - { - new() - { - Host = "localhost", - Port = downstreamPortOne, - }, - new() - { - Host = "localhost", - Port = downstreamPortTwo, - }, - }, - }, - }, - GlobalConfiguration = new FileGlobalConfiguration(), - }; - - this.Given(x => x.GivenProductServiceOneIsRunning(downstreamServiceOneUrl, 200)) - .And(x => x.GivenProductServiceTwoIsRunning(downstreamServiceTwoUrl, 200)) - .And(x => _steps.GivenThereIsAConfiguration(configuration)) - .And(x => _steps.GivenOcelotIsRunning()) - .When(x => _steps.WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 50)) - .Then(x => x.ThenTheTwoServicesShouldHaveBeenCalledTimes(50)) - - // Quite risky assertion because the actual values based on health checks and threading - //.And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(24, 26)) - .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(1, 49)) - .BDDfy(); - } - - [Fact] - public void Should_load_balance_request_with_custom_load_balancer() - { - var downstreamPortOne = PortFinder.GetRandomPort(); - var downstreamPortTwo = PortFinder.GetRandomPort(); - var downstreamServiceOneUrl = $"http://localhost:{downstreamPortOne}"; - var downstreamServiceTwoUrl = $"http://localhost:{downstreamPortTwo}"; - - var configuration = new FileConfiguration - { - Routes = new List - { - new() - { - DownstreamPathTemplate = "/", - DownstreamScheme = "http", - UpstreamPathTemplate = "/", - UpstreamHttpMethod = new List { "Get" }, - LoadBalancerOptions = new FileLoadBalancerOptions { Type = nameof(CustomLoadBalancer) }, - DownstreamHostAndPorts = new List - { - new() - { - Host = "localhost", - Port = downstreamPortOne, - }, - new() - { - Host = "localhost", - Port = downstreamPortTwo, - }, - }, - }, - }, - GlobalConfiguration = new FileGlobalConfiguration(), - }; - - Func loadBalancerFactoryFunc = (serviceProvider, route, serviceDiscoveryProvider) => new CustomLoadBalancer(serviceDiscoveryProvider.GetAsync); - - this.Given(x => x.GivenProductServiceOneIsRunning(downstreamServiceOneUrl, 200)) - .And(x => x.GivenProductServiceTwoIsRunning(downstreamServiceTwoUrl, 200)) - .And(x => _steps.GivenThereIsAConfiguration(configuration)) - .And(x => _steps.GivenOcelotIsRunningWithCustomLoadBalancer(loadBalancerFactoryFunc)) - .When(x => _steps.WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 50)) - .Then(x => x.ThenTheTwoServicesShouldHaveBeenCalledTimes(50)) - - // Quite risky assertion because the actual values based on health checks and threading - //.And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(24, 26)) - .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(1, 49)) - .BDDfy(); - } - - private class CustomLoadBalancer : ILoadBalancer - { - private readonly Func>> _services; - private readonly object _lock = new(); - - private int _last; - - public CustomLoadBalancer(Func>> services) - { - _services = services; - } - - public async Task> Lease(HttpContext httpContext) - { - var services = await _services(); - lock (_lock) - { - if (_last >= services.Count) - { - _last = 0; - } - - var next = services[_last]; - _last++; - return new OkResponse(next.HostAndPort); - } - } - - public void Release(ServiceHostAndPort hostAndPort) - { - } - } - - private void ThenBothServicesCalledRealisticAmountOfTimes(int bottom, int top) - { - _counterOne.ShouldBeInRange(bottom, top); - _counterTwo.ShouldBeInRange(bottom, top); - } - - private void ThenTheTwoServicesShouldHaveBeenCalledTimes(int expected) - { - var total = _counterOne + _counterTwo; - total.ShouldBe(expected); - } - - private void GivenProductServiceOneIsRunning(string url, int statusCode) - { - _serviceHandler.GivenThereIsAServiceRunningOn(url, async context => - { - try - { - string response; - lock (SyncLock) - { - _counterOne++; - response = _counterOne.ToString(); - } - - context.Response.StatusCode = statusCode; - await context.Response.WriteAsync(response); - } - catch (Exception exception) - { - await context.Response.WriteAsync(exception.StackTrace); - } - }); - } - - private void GivenProductServiceTwoIsRunning(string url, int statusCode) - { - _serviceHandler.GivenThereIsAServiceRunningOn(url, async context => - { - try - { - string response; - lock (SyncLock) - { - _counterTwo++; - response = _counterTwo.ToString(); - } - - context.Response.StatusCode = statusCode; - await context.Response.WriteAsync(response); - } - catch (Exception exception) - { - await context.Response.WriteAsync(exception.StackTrace); - } - }); - } - - public void Dispose() - { - _serviceHandler?.Dispose(); - _steps.Dispose(); - } -} diff --git a/test/Ocelot.AcceptanceTests/Properties/GlobalSuppressions.cs b/test/Ocelot.AcceptanceTests/Properties/GlobalSuppressions.cs index 903242484..c7da8e7ec 100644 --- a/test/Ocelot.AcceptanceTests/Properties/GlobalSuppressions.cs +++ b/test/Ocelot.AcceptanceTests/Properties/GlobalSuppressions.cs @@ -7,3 +7,4 @@ [assembly: SuppressMessage("StyleCop.CSharp.ReadabilityRules", "SA1132:Do not combine fields", Justification = "Has no much sense in test projects", Scope = "namespaceanddescendants", Target = "~N:Ocelot.AcceptanceTests")] [assembly: SuppressMessage("StyleCop.CSharp.LayoutRules", "SA1513:Closing brace should be followed by blank line", Justification = "Has no much sense in test projects", Scope = "namespaceanddescendants", Target = "~N:Ocelot.AcceptanceTests")] +[assembly: SuppressMessage("StyleCop.CSharp.LayoutRules", "SA1503:Braces should not be omitted", Justification = "For if-shortcuts")] diff --git a/test/Ocelot.AcceptanceTests/ServiceDiscovery/ConsulServiceDiscoveryTests.cs b/test/Ocelot.AcceptanceTests/ServiceDiscovery/ConsulServiceDiscoveryTests.cs index dce57ba0f..5e9fae3c2 100644 --- a/test/Ocelot.AcceptanceTests/ServiceDiscovery/ConsulServiceDiscoveryTests.cs +++ b/test/Ocelot.AcceptanceTests/ServiceDiscovery/ConsulServiceDiscoveryTests.cs @@ -3,12 +3,15 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Net.Http.Headers; using Newtonsoft.Json; +using Ocelot.AcceptanceTests.LoadBalancer; +using Ocelot.Configuration; using Ocelot.Configuration.File; using Ocelot.DependencyInjection; using Ocelot.LoadBalancer.LoadBalancers; using Ocelot.Logging; using Ocelot.Provider.Consul; using Ocelot.Provider.Consul.Interfaces; +using Ocelot.ServiceDiscovery.Providers; using System.Runtime.CompilerServices; using System.Text.RegularExpressions; @@ -17,59 +20,49 @@ namespace Ocelot.AcceptanceTests.ServiceDiscovery; /// /// Tests for the provider. /// -public sealed partial class ConsulServiceDiscoveryTests : Steps, IDisposable +public sealed partial class ConsulServiceDiscoveryTests : ConcurrentSteps, IDisposable { + private readonly ServiceHandler _consulHandler; private readonly List _consulServices; private readonly List _consulNodes; - private int _counterOne; - private int _counterTwo; - private int _counterConsul; - private int _counterNodes; - private static readonly object SyncLock = new(); - private string _downstreamPath; + private string _receivedToken; - private readonly ServiceHandler _serviceHandler; - private readonly ServiceHandler _serviceHandler2; - private readonly ServiceHandler _consulHandler; + + private volatile int _counterConsul; + private volatile int _counterNodes; public ConsulServiceDiscoveryTests() { - _serviceHandler = new ServiceHandler(); - _serviceHandler2 = new ServiceHandler(); _consulHandler = new ServiceHandler(); - _consulServices = new(); - _consulNodes = new(); + _consulServices = new List(); + _consulNodes = new List(); } public override void Dispose() { - _serviceHandler?.Dispose(); - _serviceHandler2?.Dispose(); _consulHandler?.Dispose(); + base.Dispose(); } [Fact] - public void Should_use_consul_service_discovery_and_load_balance_request() + [Trait("Feat", "28")] + public void ShouldDiscoverServicesInConsulAndLoadBalanceByLeastConnectionWhenConfigInRoute() { const string serviceName = "product"; var consulPort = PortFinder.GetRandomPort(); - var port1 = PortFinder.GetRandomPort(); - var port2 = PortFinder.GetRandomPort(); - var serviceEntryOne = GivenServiceEntry(port1, serviceName: serviceName); - var serviceEntryTwo = GivenServiceEntry(port2, serviceName: serviceName); - var route = GivenRoute(serviceName: serviceName); + var ports = PortFinder.GetPorts(2); + var serviceEntries = ports.Select(port => GivenServiceEntry(port, serviceName: serviceName)).ToArray(); + var route = GivenRoute(serviceName: serviceName, loadBalancerType: nameof(LeastConnection)); var configuration = GivenServiceDiscovery(consulPort, route); - this.Given(x => x.GivenProductServiceOneIsRunning(DownstreamUrl(port1), 200)) - .And(x => x.GivenProductServiceTwoIsRunning(DownstreamUrl(port2), 200)) + var urls = ports.Select(DownstreamUrl).ToArray(); + this.Given(x => GivenMultipleServiceInstancesAreRunning(urls, serviceName)) .And(x => x.GivenThereIsAFakeConsulServiceDiscoveryProvider(DownstreamUrl(consulPort))) - .And(x => x.GivenTheServicesAreRegisteredWithConsul(serviceEntryOne, serviceEntryTwo)) + .And(x => x.GivenTheServicesAreRegisteredWithConsul(serviceEntries)) .And(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenOcelotIsRunningWithConsul()) - .When(x => WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 50)) - .Then(x => x.ThenTheTwoServicesShouldHaveBeenCalledTimes(50)) - - // Quite risky assertion because the actual values based on health checks and threading - .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(1, 49)) //(24, 26)) + .And(x => GivenOcelotIsRunningWithServices(WithConsul)) + .When(x => WhenIGetUrlOnTheApiGatewayConcurrently("/", 50)) + .Then(x => ThenAllServicesShouldHaveBeenCalledTimes(50)) + .And(x => ThenAllServicesCalledRealisticAmountOfTimes(/*25*/24, /*25*/26)) // TODO Check strict assertion .BDDfy(); } @@ -77,7 +70,9 @@ public void Should_use_consul_service_discovery_and_load_balance_request() private static readonly string[] GetVsOptionsMethods = new[] { "Get", "Options" }; [Fact] - public void Should_handle_request_to_consul_for_downstream_service_and_make_request() + [Trait("Feat", "201")] + [Trait("Bug", "213")] + public void ShouldHandleRequestToConsulForDownstreamServiceAndMakeRequest() { const string serviceName = "web"; var consulPort = PortFinder.GetRandomPort(); @@ -85,11 +80,11 @@ public void Should_handle_request_to_consul_for_downstream_service_and_make_requ var serviceEntryOne = GivenServiceEntry(servicePort, "localhost", "web_90_0_2_224_8080", VersionV1Tags, serviceName); var route = GivenRoute("/api/home", "/home", serviceName, httpMethods: GetVsOptionsMethods); var configuration = GivenServiceDiscovery(consulPort, route); - this.Given(x => x.GivenThereIsAServiceRunningOn(DownstreamUrl(servicePort), "/api/home", HttpStatusCode.OK, "Hello from Laura")) + this.Given(x => GivenThereIsAServiceRunningOn(DownstreamUrl(servicePort), "/api/home", "Hello from Laura")) .And(x => x.GivenThereIsAFakeConsulServiceDiscoveryProvider(DownstreamUrl(consulPort))) .And(x => x.GivenTheServicesAreRegisteredWithConsul(serviceEntryOne)) .And(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenOcelotIsRunningWithConsul()) + .And(x => GivenOcelotIsRunningWithServices(WithConsul)) .When(x => WhenIGetUrlOnTheApiGateway("/home")) .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) .And(x => ThenTheResponseBodyShouldBe("Hello from Laura")) @@ -97,7 +92,9 @@ public void Should_handle_request_to_consul_for_downstream_service_and_make_requ } [Fact] - public void Should_handle_request_to_consul_for_downstream_service_and_make_request_no_re_routes() + [Trait("Bug", "213")] + [Trait("Feat", "201 340")] + public void ShouldHandleRequestToConsulForDownstreamServiceAndMakeRequestWhenDynamicRoutingWithNoRoutes() { const string serviceName = "web"; var consulPort = PortFinder.GetRandomPort(); @@ -113,11 +110,11 @@ public void Should_handle_request_to_consul_for_downstream_service_and_make_requ UseTracing = false, }; - this.Given(x => x.GivenThereIsAServiceRunningOn(DownstreamUrl(servicePort), "/something", HttpStatusCode.OK, "Hello from Laura")) + this.Given(x => GivenThereIsAServiceRunningOn(DownstreamUrl(servicePort), "/something", "Hello from Laura")) .And(x => x.GivenThereIsAFakeConsulServiceDiscoveryProvider(DownstreamUrl(consulPort))) .And(x => x.GivenTheServicesAreRegisteredWithConsul(serviceEntry)) .And(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenOcelotIsRunningWithConsul()) + .And(x => GivenOcelotIsRunningWithServices(WithConsul)) .When(x => WhenIGetUrlOnTheApiGateway("/web/something")) .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) .And(x => ThenTheResponseBodyShouldBe("Hello from Laura")) @@ -125,35 +122,33 @@ public void Should_handle_request_to_consul_for_downstream_service_and_make_requ } [Fact] - public void Should_use_consul_service_discovery_and_load_balance_request_no_re_routes() + [Trait("Feat", "340")] + public void ShouldUseConsulServiceDiscoveryAndLoadBalanceRequestWhenDynamicRoutingWithNoRoutes() { const string serviceName = "product"; var consulPort = PortFinder.GetRandomPort(); - var port1 = PortFinder.GetRandomPort(); - var port2 = PortFinder.GetRandomPort(); - var serviceEntry1 = GivenServiceEntry(port1, serviceName: serviceName); - var serviceEntry2 = GivenServiceEntry(port2, serviceName: serviceName); + var ports = PortFinder.GetPorts(2); + var serviceEntries = ports.Select(port => GivenServiceEntry(port, serviceName: serviceName)).ToArray(); var configuration = GivenServiceDiscovery(consulPort); configuration.GlobalConfiguration.LoadBalancerOptions = new() { Type = nameof(LeastConnection) }; configuration.GlobalConfiguration.DownstreamScheme = "http"; - this.Given(x => x.GivenProductServiceOneIsRunning(DownstreamUrl(port1), 200)) - .And(x => x.GivenProductServiceTwoIsRunning(DownstreamUrl(port2), 200)) + var urls = ports.Select(DownstreamUrl).ToArray(); + this.Given(x => GivenMultipleServiceInstancesAreRunning(urls, serviceName)) .And(x => x.GivenThereIsAFakeConsulServiceDiscoveryProvider(DownstreamUrl(consulPort))) - .And(x => x.GivenTheServicesAreRegisteredWithConsul(serviceEntry1, serviceEntry2)) + .And(x => x.GivenTheServicesAreRegisteredWithConsul(serviceEntries)) .And(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenOcelotIsRunningWithConsul()) - .When(x => WhenIGetUrlOnTheApiGatewayMultipleTimes($"/{serviceName}/", 50)) - .Then(x => x.ThenTheTwoServicesShouldHaveBeenCalledTimes(50)) - - // Quite risky assertion because the actual values based on health checks and threading - .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(1, 49)) //(24, 26)) + .And(x => GivenOcelotIsRunningWithServices(WithConsul)) + .When(x => WhenIGetUrlOnTheApiGatewayConcurrently($"/{serviceName}/", 50)) + .Then(x => ThenAllServicesShouldHaveBeenCalledTimes(50)) + .And(x => ThenAllServicesCalledRealisticAmountOfTimes(/*25*/24, /*25*/26)) // TODO Check strict assertion .BDDfy(); } [Fact] - public void Should_use_token_to_make_request_to_consul() + [Trait("Feat", "295")] + public void ShouldUseAclTokenToMakeRequestToConsul() { const string serviceName = "web"; const string token = "abctoken"; @@ -165,54 +160,52 @@ public void Should_use_token_to_make_request_to_consul() var configuration = GivenServiceDiscovery(consulPort, route); configuration.GlobalConfiguration.ServiceDiscoveryProvider.Token = token; - this.Given(_ => GivenThereIsAServiceRunningOn(DownstreamUrl(servicePort), "/api/home", HttpStatusCode.OK, "Hello from Laura")) - .And(_ => GivenThereIsAFakeConsulServiceDiscoveryProvider(DownstreamUrl(consulPort))) - .And(_ => GivenTheServicesAreRegisteredWithConsul(serviceEntry)) - .And(_ => GivenThereIsAConfiguration(configuration)) - .And(_ => GivenOcelotIsRunningWithConsul()) - .When(_ => WhenIGetUrlOnTheApiGateway("/home")) - .Then(_ => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) - .And(_ => ThenTheResponseBodyShouldBe("Hello from Laura")) - .And(_ => ThenTheTokenIs(token)) + this.Given(x => GivenThereIsAServiceRunningOn(DownstreamUrl(servicePort), "/api/home", "Hello from Laura")) + .And(x => x.GivenThereIsAFakeConsulServiceDiscoveryProvider(DownstreamUrl(consulPort))) + .And(x => x.GivenTheServicesAreRegisteredWithConsul(serviceEntry)) + .And(x => GivenThereIsAConfiguration(configuration)) + .And(x => GivenOcelotIsRunningWithServices(WithConsul)) + .When(x => WhenIGetUrlOnTheApiGateway("/home")) + .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) + .And(x => ThenTheResponseBodyShouldBe("Hello from Laura")) + .And(x => x.ThenTheTokenIs(token)) .BDDfy(); } [Fact] - public void Should_send_request_to_service_after_it_becomes_available_in_consul() + [Trait("Bug", "181")] + public void ShouldSendRequestToServiceAfterItBecomesAvailableInConsul() { const string serviceName = "product"; var consulPort = PortFinder.GetRandomPort(); - var port1 = PortFinder.GetRandomPort(); - var port2 = PortFinder.GetRandomPort(); - var serviceEntry1 = GivenServiceEntry(port1, serviceName: serviceName); - var serviceEntry2 = GivenServiceEntry(port2, serviceName: serviceName); + var ports = PortFinder.GetPorts(2); + var serviceEntries = ports.Select(port => GivenServiceEntry(port, serviceName: serviceName)).ToArray(); var route = GivenRoute(serviceName: serviceName); var configuration = GivenServiceDiscovery(consulPort, route); - this.Given(x => x.GivenProductServiceOneIsRunning(DownstreamUrl(port1), 200)) - .And(x => x.GivenProductServiceTwoIsRunning(DownstreamUrl(port2), 200)) + var urls = ports.Select(DownstreamUrl).ToArray(); + this.Given(_ => GivenMultipleServiceInstancesAreRunning(urls, serviceName)) .And(x => x.GivenThereIsAFakeConsulServiceDiscoveryProvider(DownstreamUrl(consulPort))) - .And(x => x.GivenTheServicesAreRegisteredWithConsul(serviceEntry1, serviceEntry2)) + .And(x => x.GivenTheServicesAreRegisteredWithConsul(serviceEntries)) .And(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenOcelotIsRunningWithConsul()) - .And(x => WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 10)) - .And(x => x.ThenTheTwoServicesShouldHaveBeenCalledTimes(10)) - .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(1, 9)) //(4, 6)) - .And(x => WhenIRemoveAService(serviceEntry2)) - .And(x => GivenIResetCounters()) - .And(x => WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 10)) - .And(x => ThenOnlyOneServiceHasBeenCalled()) - .And(x => WhenIAddAServiceBackIn(serviceEntry2)) - .And(x => GivenIResetCounters()) - .When(x => WhenIGetUrlOnTheApiGatewayMultipleTimes("/", 10)) - .Then(x => x.ThenTheTwoServicesShouldHaveBeenCalledTimes(10)) - - // Quite risky assertion because the actual values based on health checks and threading - .And(x => x.ThenBothServicesCalledRealisticAmountOfTimes(1, 9)) //(4, 6)) + .And(x => GivenOcelotIsRunningWithServices(WithConsul)) + .And(x => WhenIGetUrlOnTheApiGatewayConcurrently("/", 10)) + .And(x => ThenAllServicesShouldHaveBeenCalledTimes(10)) + .And(x => ThenAllServicesCalledRealisticAmountOfTimes(/*5*/4, /*5*/6)) // TODO Check strict assertion + .And(x => x.WhenIRemoveAService(serviceEntries[1])) // 2nd entry + .And(x => x.GivenIResetCounters()) + .And(x => WhenIGetUrlOnTheApiGatewayConcurrently("/", 10)) + .And(x => ThenServicesShouldHaveBeenCalledTimes(10, 0)) // 2nd is offline + .And(x => x.WhenIAddAServiceBackIn(serviceEntries[1])) // 2nd entry + .And(x => x.GivenIResetCounters()) + .When(x => WhenIGetUrlOnTheApiGatewayConcurrently("/", 10)) + .Then(x => ThenAllServicesShouldHaveBeenCalledTimes(10)) + .And(x => ThenAllServicesCalledRealisticAmountOfTimes(/*5*/4, /*5*/6)) // TODO Check strict assertion .BDDfy(); } [Fact] - public void Should_handle_request_to_poll_consul_for_downstream_service_and_make_request() + [Trait("Feat", "374")] + public void ShouldPollConsulForDownstreamServiceAndMakeRequest() { const string serviceName = "web"; var consulPort = PortFinder.GetRandomPort(); @@ -222,15 +215,15 @@ public void Should_handle_request_to_poll_consul_for_downstream_service_and_make var configuration = GivenServiceDiscovery(consulPort, route); var sd = configuration.GlobalConfiguration.ServiceDiscoveryProvider; - sd.Type = nameof(PollConsul); + sd.Type = nameof(PollConsul); // !!! sd.PollingInterval = 0; sd.Namespace = string.Empty; - this.Given(x => x.GivenThereIsAServiceRunningOn(DownstreamUrl(servicePort), "/api/home", HttpStatusCode.OK, "Hello from Laura")) + this.Given(x => GivenThereIsAServiceRunningOn(DownstreamUrl(servicePort), "/api/home", "Hello from Laura")) .And(x => x.GivenThereIsAFakeConsulServiceDiscoveryProvider(DownstreamUrl(consulPort))) .And(x => x.GivenTheServicesAreRegisteredWithConsul(serviceEntry)) .And(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenOcelotIsRunningWithConsul()) + .And(x => GivenOcelotIsRunningWithServices(WithConsul)) .When(x => WhenIGetUrlOnTheApiGatewayWaitingForTheResponseToBeOk("/home")) .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) .And(x => ThenTheResponseBodyShouldBe("Hello from Laura")) @@ -240,11 +233,11 @@ public void Should_handle_request_to_poll_consul_for_downstream_service_and_make [Theory] [Trait("PR", "1944")] [Trait("Bugs", "849 1496")] - [InlineData(nameof(LeastConnection))] - [InlineData(nameof(RoundRobin))] [InlineData(nameof(NoLoadBalancer))] + [InlineData(nameof(RoundRobin))] + [InlineData(nameof(LeastConnection))] [InlineData(nameof(CookieStickySessions))] - public void Should_use_consul_service_discovery_based_on_upstream_host(string loadBalancerType) + public void ShouldUseConsulServiceDiscoveryWhenThereAreTwoUpstreamHosts(string loadBalancerType) { // Simulate two DIFFERENT downstream services (e.g. product services for US and EU markets) // with different ServiceNames (e.g. product-us and product-eu), @@ -272,12 +265,13 @@ public void Should_use_consul_service_discovery_based_on_upstream_host(string lo // Ocelot request for http://us-shop/ should find 'product-us' in Consul, call /products and return "Phone chargers with US plug" // Ocelot request for http://eu-shop/ should find 'product-eu' in Consul, call /products and return "Phone chargers with EU plug" - this.Given(x => x._serviceHandler.GivenThereIsAServiceRunningOn(DownstreamUrl(servicePortUS), "/products", MapGet("/products", responseBodyUS))) - .Given(x => x._serviceHandler2.GivenThereIsAServiceRunningOn(DownstreamUrl(servicePortEU), "/products", MapGet("/products", responseBodyEU))) + _handlers = new ServiceHandler[2] { new(), new() }; + this.Given(x => _handlers[0].GivenThereIsAServiceRunningOn(DownstreamUrl(servicePortUS), "/products", MapGet("/products", responseBodyUS))) + .Given(x => _handlers[1].GivenThereIsAServiceRunningOn(DownstreamUrl(servicePortEU), "/products", MapGet("/products", responseBodyEU))) .And(x => x.GivenThereIsAFakeConsulServiceDiscoveryProvider(DownstreamUrl(consulPort))) .And(x => x.GivenTheServicesAreRegisteredWithConsul(serviceEntryUS, serviceEntryEU)) .And(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenOcelotIsRunningWithConsul(publicUrlUS, publicUrlEU)) + .And(x => GivenOcelotIsRunningWithServices(WithConsul)) .When(x => x.WhenIGetUrl(publicUrlUS, sessionCookieUS), "When I get US shop for the first time") .Then(x => x.ThenConsulShouldHaveBeenCalledTimes(1)) .And(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) @@ -299,7 +293,7 @@ public void Should_use_consul_service_discovery_based_on_upstream_host(string lo [Fact] [Trait("Bug", "954")] - public void Should_return_service_address_by_overridden_service_builder_when_there_is_a_node() + public void ShouldReturnServiceAddressByOverriddenServiceBuilderWhenThereIsANode() { const string serviceName = "OpenTestService"; string[] methods = new[] { HttpMethods.Post, HttpMethods.Get }; @@ -314,12 +308,12 @@ public void Should_return_service_address_by_overridden_service_builder_when_the var route = GivenRoute("/api/{url}", "/open/{url}", serviceName, httpMethods: methods); var configuration = GivenServiceDiscovery(consulPort, route); - this.Given(x => x.GivenThereIsAServiceRunningOn(DownstreamUrl(servicePort), "/api/home", HttpStatusCode.OK, "Hello from Raman")) + this.Given(x => GivenThereIsAServiceRunningOn(DownstreamUrl(servicePort), "/api/home", "Hello from Raman")) .And(x => x.GivenThereIsAFakeConsulServiceDiscoveryProvider(DownstreamUrl(consulPort))) .And(x => x.GivenTheServicesAreRegisteredWithConsul(serviceEntry)) .And(x => x.GivenTheServiceNodesAreRegisteredWithConsul(serviceNode)) .And(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenOcelotIsRunningWithConsul()) // default services registration results with the bug: "n1" host issue + .And(x => GivenOcelotIsRunningWithServices(WithConsul)) // default services registration results with the bug: "n1" host issue .When(x => WhenIGetUrlOnTheApiGateway("/open/home")) .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.BadGateway)) .And(x => ThenTheResponseBodyShouldBe("")) @@ -336,13 +330,192 @@ public void Should_return_service_address_by_overridden_service_builder_when_the .BDDfy(); } - private static void WithOverriddenConsulServiceBuilder(IServiceCollection services) - => services.AddOcelot().AddConsul(); + private static readonly string[] Bug2119ServiceNames = new string[] { "ProjectsService", "CustomersService" }; + private readonly ILoadBalancer[] _lbAnalyzers = new ILoadBalancer[Bug2119ServiceNames.Length]; // emulate LoadBalancerHouse's collection + + private TLoadBalancer GetAnalyzer(DownstreamRoute route, IServiceDiscoveryProvider provider) + where TLoadBalancer : class, ILoadBalancer + where TLoadBalancerCreator : class, ILoadBalancerCreator, new() + { + //lock (LoadBalancerHouse.SyncRoot) // Note, synch locking is implemented in LoadBalancerHouse + int index = Array.IndexOf(Bug2119ServiceNames, route.ServiceName); // LoadBalancerHouse should return different balancers for different service names + _lbAnalyzers[index] ??= new TLoadBalancerCreator().Create(route, provider)?.Data; + return (TLoadBalancer)_lbAnalyzers[index]; + } + + private void WithLbAnalyzer(IServiceCollection services) + where TLoadBalancer : class, ILoadBalancer + where TLoadBalancerCreator : class, ILoadBalancerCreator, new() + => services.AddOcelot().AddConsul().AddCustomLoadBalancer(GetAnalyzer); + + [Theory] + [Trait("Bug", "2119")] + [InlineData(nameof(NoLoadBalancer))] + [InlineData(nameof(RoundRobin))] + [InlineData(nameof(LeastConnection))] // original scenario + public void ShouldReturnDifferentServicesWhenThereAre2SequentialRequestsToDifferentServices(string loadBalancer) + { + var consulPort = PortFinder.GetRandomPort(); + var ports = PortFinder.GetPorts(Bug2119ServiceNames.Length); + var service1 = GivenServiceEntry(ports[0], serviceName: Bug2119ServiceNames[0]); + var service2 = GivenServiceEntry(ports[1], serviceName: Bug2119ServiceNames[1]); + var route1 = GivenRoute("/{all}", "/projects/{all}", serviceName: Bug2119ServiceNames[0], loadBalancerType: loadBalancer); + var route2 = GivenRoute("/{all}", "/customers/{all}", serviceName: Bug2119ServiceNames[1], loadBalancerType: loadBalancer); + route1.UpstreamHttpMethod = route2.UpstreamHttpMethod = new() { HttpMethods.Get, HttpMethods.Post, HttpMethods.Put, HttpMethods.Delete }; + var configuration = GivenServiceDiscovery(consulPort, route1, route2); + var urls = ports.Select(DownstreamUrl).ToArray(); + this.Given(x => GivenMultipleServiceInstancesAreRunning(urls, Bug2119ServiceNames)) + .And(x => x.GivenThereIsAFakeConsulServiceDiscoveryProvider(DownstreamUrl(consulPort))) + .And(x => x.GivenTheServicesAreRegisteredWithConsul(service1, service2)) + .And(x => GivenThereIsAConfiguration(configuration)) + .And(x => GivenOcelotIsRunningWithServices(WithConsul)) + + // Step 1 + .When(x => WhenIGetUrlOnTheApiGateway("/projects/api/projects")) + .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) + .And(x => ThenServiceShouldHaveBeenCalledTimes(0, 1)) + .And(x => x.ThenTheResponseBodyShouldBe($"1:{Bug2119ServiceNames[0]}")) // ! + + // Step 2 + .When(x => WhenIGetUrlOnTheApiGateway("/customers/api/customers")) + .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) + .And(x => ThenServiceShouldHaveBeenCalledTimes(1, 1)) + .And(x => x.ThenTheResponseBodyShouldBe($"1:{Bug2119ServiceNames[1]}")) // !! + + // Finally + .Then(x => ThenAllStatusCodesShouldBe(HttpStatusCode.OK)) + .And(x => ThenAllServicesShouldHaveBeenCalledTimes(2)) + .And(x => ThenServicesShouldHaveBeenCalledTimes(1, 1)) + .BDDfy(); + } + + [Theory] + [Trait("Bug", "2119")] + [InlineData(false, nameof(NoLoadBalancer))] + [InlineData(false, nameof(LeastConnection))] // original scenario, clean config + [InlineData(true, nameof(LeastConnectionAnalyzer))] // extended scenario using analyzer + [InlineData(false, nameof(RoundRobin))] + [InlineData(true, nameof(RoundRobinAnalyzer))] + public void ShouldReturnDifferentServicesWhenSequentiallylyRequestingToDifferentServices(bool withAnalyzer, string loadBalancer) + { + var consulPort = PortFinder.GetRandomPort(); + var ports = PortFinder.GetPorts(Bug2119ServiceNames.Length); + var service1 = GivenServiceEntry(ports[0], serviceName: Bug2119ServiceNames[0]); + var service2 = GivenServiceEntry(ports[1], serviceName: Bug2119ServiceNames[1]); + var route1 = GivenRoute("/{all}", "/projects/{all}", serviceName: Bug2119ServiceNames[0], loadBalancerType: loadBalancer); + var route2 = GivenRoute("/{all}", "/customers/{all}", serviceName: Bug2119ServiceNames[1], loadBalancerType: loadBalancer); + route1.UpstreamHttpMethod = route2.UpstreamHttpMethod = new() { HttpMethods.Get, HttpMethods.Post, HttpMethods.Put, HttpMethods.Delete }; + var configuration = GivenServiceDiscovery(consulPort, route1, route2); + var urls = ports.Select(DownstreamUrl).ToArray(); + Action requestToProjectsAndThenRequestToCustomersAndAssert = (i) => + { + // Step 1 + int count = i + 1; + WhenIGetUrlOnTheApiGateway("/projects/api/projects"); + ThenTheStatusCodeShouldBe(HttpStatusCode.OK); + ThenServiceShouldHaveBeenCalledTimes(0, count); + ThenTheResponseBodyShouldBe($"{count}:{Bug2119ServiceNames[0]}", $"i is {i}"); + _responses[2 * i] = _response; + + // Step 2 + WhenIGetUrlOnTheApiGateway("/customers/api/customers"); + ThenTheStatusCodeShouldBe(HttpStatusCode.OK); + ThenServiceShouldHaveBeenCalledTimes(1, count); + ThenTheResponseBodyShouldBe($"{count}:{Bug2119ServiceNames[1]}", $"i is {i}"); + _responses[(2 * i) + 1] = _response; + }; + this.Given(x => GivenMultipleServiceInstancesAreRunning(urls, Bug2119ServiceNames)) // service names as responses + .And(x => x.GivenThereIsAFakeConsulServiceDiscoveryProvider(DownstreamUrl(consulPort))) + .And(x => x.GivenTheServicesAreRegisteredWithConsul(service1, service2)) + .And(x => GivenThereIsAConfiguration(configuration)) + .And(x => GivenOcelotIsRunningWithServices(withAnalyzer ? WithLbAnalyzer(loadBalancer) : WithConsul)) + .When(x => WhenIDoActionMultipleTimes(50, requestToProjectsAndThenRequestToCustomersAndAssert)) + .Then(x => ThenAllStatusCodesShouldBe(HttpStatusCode.OK)) + .And(x => x.ThenResponsesShouldHaveBodyFromDifferentServices(ports, Bug2119ServiceNames)) // !!! + .And(x => ThenAllServicesShouldHaveBeenCalledTimes(100)) + .And(x => ThenAllServicesCalledRealisticAmountOfTimes(50, 50)) + .And(x => ThenServicesShouldHaveBeenCalledTimes(50, 50)) // strict assertion + .BDDfy(); + } + + [Theory] + [Trait("Bug", "2119")] + [InlineData(false, nameof(NoLoadBalancer))] + [InlineData(false, nameof(LeastConnection))] // original scenario, clean config + [InlineData(true, nameof(LeastConnectionAnalyzer))] // extended scenario using analyzer + [InlineData(false, nameof(RoundRobin))] + [InlineData(true, nameof(RoundRobinAnalyzer))] + public void ShouldReturnDifferentServicesWhenConcurrentlyRequestingToDifferentServices(bool withAnalyzer, string loadBalancer) + { + const int total = 100; // concurrent requests + var consulPort = PortFinder.GetRandomPort(); + var ports = PortFinder.GetPorts(Bug2119ServiceNames.Length); + var service1 = GivenServiceEntry(ports[0], serviceName: Bug2119ServiceNames[0]); + var service2 = GivenServiceEntry(ports[1], serviceName: Bug2119ServiceNames[1]); + var route1 = GivenRoute("/{all}", "/projects/{all}", serviceName: Bug2119ServiceNames[0], loadBalancerType: loadBalancer); + var route2 = GivenRoute("/{all}", "/customers/{all}", serviceName: Bug2119ServiceNames[1], loadBalancerType: loadBalancer); + route1.UpstreamHttpMethod = route2.UpstreamHttpMethod = new() { HttpMethods.Get, HttpMethods.Post, HttpMethods.Put, HttpMethods.Delete }; + var configuration = GivenServiceDiscovery(consulPort, route1, route2); + var urls = ports.Select(DownstreamUrl).ToArray(); + this.Given(x => GivenMultipleServiceInstancesAreRunning(urls, Bug2119ServiceNames)) // service names as responses + .And(x => x.GivenThereIsAFakeConsulServiceDiscoveryProvider(DownstreamUrl(consulPort))) + .And(x => x.GivenTheServicesAreRegisteredWithConsul(service1, service2)) + .And(x => GivenThereIsAConfiguration(configuration)) + .And(x => GivenOcelotIsRunningWithServices(withAnalyzer ? WithLbAnalyzer(loadBalancer) : WithConsul)) + .When(x => WhenIGetUrlOnTheApiGatewayConcurrently(total, "/projects/api/projects", "/customers/api/customers")) + .Then(x => ThenAllStatusCodesShouldBe(HttpStatusCode.OK)) + .And(x => x.ThenResponsesShouldHaveBodyFromDifferentServices(ports, Bug2119ServiceNames)) // !!! + .And(x => ThenAllServicesShouldHaveBeenCalledTimes(total)) + .And(x => ThenServiceCountersShouldMatchLeasingCounters((ILoadBalancerAnalyzer)_lbAnalyzers[0], ports, 50)) // ProjectsService + .And(x => ThenServiceCountersShouldMatchLeasingCounters((ILoadBalancerAnalyzer)_lbAnalyzers[1], ports, 50)) // CustomersService + .And(x => ThenAllServicesCalledRealisticAmountOfTimes(Bottom(total, ports.Length), Top(total, ports.Length))) + .And(x => ThenServicesShouldHaveBeenCalledTimes(50, 50)) // strict assertion + .BDDfy(); + } + + private Action WithLbAnalyzer(string loadBalancer) => loadBalancer switch + { + nameof(LeastConnection) => WithLbAnalyzer, + nameof(LeastConnectionAnalyzer) => WithLbAnalyzer, + nameof(RoundRobin) => WithLbAnalyzer, + nameof(RoundRobinAnalyzer) => WithLbAnalyzer, + _ => WithLbAnalyzer, + }; + + private void ThenResponsesShouldHaveBodyFromDifferentServices(int[] ports, string[] serviceNames) + { + foreach (var response in _responses) + { + var headers = response.Value.Headers; + headers.TryGetValues(HeaderNames.ServiceIndex, out var indexValues).ShouldBeTrue(); + int serviceIndex = int.Parse(indexValues.FirstOrDefault() ?? "-1"); + serviceIndex.ShouldBeGreaterThanOrEqualTo(0); + + headers.TryGetValues(HeaderNames.Host, out var hostValues).ShouldBeTrue(); + hostValues.FirstOrDefault().ShouldBe("localhost"); + headers.TryGetValues(HeaderNames.Port, out var portValues).ShouldBeTrue(); + portValues.FirstOrDefault().ShouldBe(ports[serviceIndex].ToString()); + + var body = response.Value.Content.ReadAsStringAsync().Result; + var serviceName = serviceNames[serviceIndex]; + body.ShouldNotBeNull().ShouldEndWith(serviceName); + + headers.TryGetValues(HeaderNames.Counter, out var counterValues).ShouldBeTrue(); + var counter = counterValues.ShouldNotBeNull().FirstOrDefault().ShouldNotBeNull(); + body.ShouldBe($"{counter}:{serviceName}"); + } + } + + private static void WithConsul(IServiceCollection services) => services + .AddOcelot().AddConsul(); + + private static void WithOverriddenConsulServiceBuilder(IServiceCollection services) => services + .AddOcelot().AddConsul(); public class MyConsulServiceBuilder : DefaultConsulServiceBuilder { - public MyConsulServiceBuilder(Func configurationFactory, IConsulClientFactory clientFactory, IOcelotLoggerFactory loggerFactory) - : base(configurationFactory, clientFactory, loggerFactory) { } + public MyConsulServiceBuilder(IHttpContextAccessor contextAccessor, IConsulClientFactory clientFactory, IOcelotLoggerFactory loggerFactory) + : base(contextAccessor, clientFactory, loggerFactory) { } protected override string GetDownstreamHost(ServiceEntry entry, Node node) => entry.Service.Address; } @@ -406,12 +579,6 @@ private void WhenIAddAServiceBackIn(ServiceEntry serviceEntry) _consulServices.Add(serviceEntry); } - private void ThenOnlyOneServiceHasBeenCalled() - { - _counterOne.ShouldBe(10); - _counterTwo.ShouldBe(0); - } - private void WhenIRemoveAService(ServiceEntry serviceEntry) { _consulServices.Remove(serviceEntry); @@ -419,23 +586,10 @@ private void WhenIRemoveAService(ServiceEntry serviceEntry) private void GivenIResetCounters() { - _counterOne = 0; - _counterTwo = 0; + _counters[0] = _counters[1] = 0; _counterConsul = 0; } - private void ThenBothServicesCalledRealisticAmountOfTimes(int bottom, int top) - { - _counterOne.ShouldBeInRange(bottom, top); - _counterTwo.ShouldBeInRange(bottom, top); - } - - private void ThenTheTwoServicesShouldHaveBeenCalledTimes(int expected) - { - var total = _counterOne + _counterTwo; - total.ShouldBe(expected); - } - private void GivenTheServicesAreRegisteredWithConsul(params ServiceEntry[] serviceEntries) => _consulServices.AddRange(serviceEntries); private void GivenTheServiceNodesAreRegisteredWithConsul(params Node[] nodes) => _consulNodes.AddRange(nodes); @@ -459,12 +613,18 @@ private void GivenThereIsAFakeConsulServiceDiscoveryProvider(string url) var pathMatch = ServiceNameRegex().Match(context.Request.Path.Value); if (pathMatch.Success) { - _counterConsul++; + //string json; + //lock (ConsulCounterLocker) + //{ + //_counterConsul++; + int count = Interlocked.Increment(ref _counterConsul); // Use the parsed service name to filter the registered Consul services var serviceName = pathMatch.Groups["serviceName"].Value; var services = _consulServices.Where(x => x.Service.Service == serviceName).ToList(); var json = JsonConvert.SerializeObject(services); + + //} context.Response.Headers.Append("Content-Type", "application/json"); await context.Response.WriteAsync(json); return; @@ -472,7 +632,8 @@ private void GivenThereIsAFakeConsulServiceDiscoveryProvider(string url) if (context.Request.Path.Value == "/v1/catalog/nodes") { - _counterNodes++; + //_counterNodes++; + int count = Interlocked.Increment(ref _counterNodes); var json = JsonConvert.SerializeObject(_consulNodes); context.Response.Headers.Append("Content-Type", "application/json"); await context.Response.WriteAsync(json); @@ -482,84 +643,4 @@ private void GivenThereIsAFakeConsulServiceDiscoveryProvider(string url) private void ThenConsulShouldHaveBeenCalledTimes(int expected) => _counterConsul.ShouldBe(expected); private void ThenConsulNodesShouldHaveBeenCalledTimes(int expected) => _counterNodes.ShouldBe(expected); - - private void GivenProductServiceOneIsRunning(string url, int statusCode) - { - _serviceHandler.GivenThereIsAServiceRunningOn(url, async context => - { - try - { - string response; - lock (SyncLock) - { - _counterOne++; - response = _counterOne.ToString(); - } - - context.Response.StatusCode = statusCode; - await context.Response.WriteAsync(response); - } - catch (Exception exception) - { - await context.Response.WriteAsync(exception.StackTrace); - } - }); - } - - private void GivenProductServiceTwoIsRunning(string url, int statusCode) - { - _serviceHandler2.GivenThereIsAServiceRunningOn(url, async context => - { - try - { - string response; - lock (SyncLock) - { - _counterTwo++; - response = _counterTwo.ToString(); - } - - context.Response.StatusCode = statusCode; - await context.Response.WriteAsync(response); - } - catch (Exception exception) - { - await context.Response.WriteAsync(exception.StackTrace); - } - }); - } - - private void GivenThereIsAServiceRunningOn(string baseUrl, string basePath, HttpStatusCode statusCode, string responseBody) - { - _serviceHandler.GivenThereIsAServiceRunningOn(baseUrl, basePath, async context => - { - _downstreamPath = !string.IsNullOrEmpty(context.Request.PathBase.Value) ? context.Request.PathBase.Value : context.Request.Path.Value; - - if (_downstreamPath != basePath) - { - context.Response.StatusCode = (int)HttpStatusCode.NotFound; - await context.Response.WriteAsync("Downstream path doesn't match base path"); - } - else - { - context.Response.StatusCode = (int)statusCode; - await context.Response.WriteAsync(responseBody); - } - }); - } - - private static RequestDelegate MapGet(string path, string responseBody) => async context => - { - var downstreamPath = !string.IsNullOrEmpty(context.Request.PathBase.Value) ? context.Request.PathBase.Value : context.Request.Path.Value; - if (downstreamPath == path) - { - context.Response.StatusCode = (int)HttpStatusCode.OK; - await context.Response.WriteAsync(responseBody); - } - else - { - context.Response.StatusCode = (int)HttpStatusCode.NotFound; - await context.Response.WriteAsync("Not Found"); - } - }; } diff --git a/test/Ocelot.AcceptanceTests/ServiceDiscovery/EurekaServiceDiscoveryTests.cs b/test/Ocelot.AcceptanceTests/ServiceDiscovery/EurekaServiceDiscoveryTests.cs index 5ec8f194b..0a5967e11 100644 --- a/test/Ocelot.AcceptanceTests/ServiceDiscovery/EurekaServiceDiscoveryTests.cs +++ b/test/Ocelot.AcceptanceTests/ServiceDiscovery/EurekaServiceDiscoveryTests.cs @@ -20,9 +20,10 @@ public EurekaServiceDiscoveryTests() } [Theory] + [Trait("Feat", "262")] [InlineData(true)] [InlineData(false)] - public void should_use_eureka_service_discovery_and_make_request(bool dotnetRunningInContainer) + public void Should_use_eureka_service_discovery_and_make_request(bool dotnetRunningInContainer) { Environment.SetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER", dotnetRunningInContainer.ToString()); var eurekaPort = 8761; diff --git a/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs b/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs index 99fe18f14..daaa40911 100644 --- a/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs +++ b/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Newtonsoft.Json; +using Ocelot.AcceptanceTests.LoadBalancer; using Ocelot.Configuration; using Ocelot.Configuration.File; using Ocelot.DependencyInjection; @@ -13,17 +14,15 @@ using Ocelot.Provider.Kubernetes.Interfaces; using Ocelot.ServiceDiscovery.Providers; using Ocelot.Values; -using System.Collections.Concurrent; using System.Runtime.CompilerServices; using System.Text; namespace Ocelot.AcceptanceTests.ServiceDiscovery; -public sealed class KubernetesServiceDiscoveryTests : Steps, IDisposable +public sealed class KubernetesServiceDiscoveryTests : ConcurrentSteps, IDisposable { private readonly string _kubernetesUrl; private readonly IKubeApiClient _clientFactory; - private readonly List _serviceHandlers; private readonly ServiceHandler _kubernetesHandler; private string _receivedToken; @@ -38,13 +37,11 @@ public KubernetesServiceDiscoveryTests() AllowInsecure = true, }; _clientFactory = KubeApiClient.Create(option); - _serviceHandlers = new(); _kubernetesHandler = new(); } public override void Dispose() { - _serviceHandlers.ForEach(handler => handler?.Dispose()); _kubernetesHandler.Dispose(); base.Dispose(); } @@ -62,13 +59,14 @@ public void ShouldReturnServicesFromK8s() var route = GivenRouteWithServiceName(namespaces); var configuration = GivenKubeConfiguration(namespaces, route); var downstreamResponse = serviceName; - this.Given(x => x.GivenK8sProductServiceIsRunning(downstreamUrl, downstreamResponse)) + this.Given(x => GivenServiceInstanceIsRunning(downstreamUrl, downstreamResponse)) .And(x => x.GivenThereIsAFakeKubernetesProvider(endpoints, serviceName, namespaces)) .And(_ => GivenThereIsAConfiguration(configuration)) .And(_ => GivenOcelotIsRunningWithServices(WithKubernetes)) .When(_ => WhenIGetUrlOnTheApiGateway("/")) .Then(_ => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) .And(_ => ThenTheResponseBodyShouldBe($"1:{downstreamResponse}")) + .And(x => ThenAllServicesShouldHaveBeenCalledTimes(1)) .And(x => x.ThenTheTokenIs("Bearer txpc696iUhbVoudg164r93CxDTrKRVWG")) .BDDfy(); } @@ -101,7 +99,7 @@ public void ShouldReturnServicesByPortNameAsDownstreamScheme(string downstreamSc route.ServiceName = serviceName; // "example-web" var configuration = GivenKubeConfiguration(namespaces, route); - this.Given(x => x.GivenK8sProductServiceIsRunning(downstreamUrl, nameof(ShouldReturnServicesByPortNameAsDownstreamScheme))) + this.Given(x => GivenServiceInstanceIsRunning(downstreamUrl, nameof(ShouldReturnServicesByPortNameAsDownstreamScheme))) .And(x => x.GivenThereIsAFakeKubernetesProvider(endpoints, serviceName, namespaces)) .And(_ => GivenThereIsAConfiguration(configuration)) .And(_ => GivenOcelotIsRunningWithServices(WithKubernetes)) @@ -110,6 +108,7 @@ public void ShouldReturnServicesByPortNameAsDownstreamScheme(string downstreamSc .And(_ => ThenTheResponseBodyShouldBe(downstreamScheme == "http" ? "1:" + nameof(ShouldReturnServicesByPortNameAsDownstreamScheme) : string.Empty)) + .And(x => ThenAllServicesShouldHaveBeenCalledTimes(downstreamScheme == "http" ? 1 : 0)) .And(x => x.ThenTheTokenIs("Bearer txpc696iUhbVoudg164r93CxDTrKRVWG")) .BDDfy(); } @@ -137,7 +136,7 @@ public void ShouldHighlyLoadOnStableKubeProvider_WithRoundRobinLoadBalancing(int int bottom = totalRequests / totalServices, top = totalRequests - (bottom * totalServices) + bottom; ThenAllServicesCalledRealisticAmountOfTimes(bottom, top); - ThenServiceCountersShouldMatchLeasingCounters(servicePorts); + ThenServiceCountersShouldMatchLeasingCounters(_roundRobinAnalyzer, servicePorts, totalRequests); } [Theory] @@ -154,10 +153,8 @@ public void ShouldHighlyLoadOnUnstableKubeProvider_WithRoundRobinLoadBalancing(i HighlyLoadOnKubeProviderAndRoundRobinBalancer(totalRequests, k8sGeneration); - int bottom = _roundRobinAnalyzer.BottomOfConnections(), - top = _roundRobinAnalyzer.TopOfConnections(); - ThenAllServicesCalledRealisticAmountOfTimes(bottom, top); // with unstable checkings - ThenServiceCountersShouldMatchLeasingCounters(servicePorts); + ThenAllServicesCalledOptimisticAmountOfTimes(_roundRobinAnalyzer); // with unstable checkings + ThenServiceCountersShouldMatchLeasingCounters(_roundRobinAnalyzer, servicePorts, totalRequests); } private (EndpointsV1 Endpoints, int[] ServicePorts) ArrangeHighLoadOnKubeProviderAndRoundRobinBalancer( @@ -165,23 +162,21 @@ public void ShouldHighlyLoadOnUnstableKubeProvider_WithRoundRobinLoadBalancing(i [CallerMemberName] string serviceName = nameof(ArrangeHighLoadOnKubeProviderAndRoundRobinBalancer)) { const string namespaces = nameof(KubernetesServiceDiscoveryTests); - var servicePorts = Enumerable.Repeat(0, totalServices) - .Select(_ => PortFinder.GetRandomPort()) - .ToArray(); + var servicePorts = PortFinder.GetPorts(totalServices); var downstreamUrls = servicePorts .Select(port => LoopbackLocalhostUrl(port, Array.IndexOf(servicePorts, port))) - .ToList(); // based on localhost aka loopback network interface + .ToArray(); // based on localhost aka loopback network interface var downstreams = downstreamUrls.Select(url => new Uri(url)) .ToList(); var downstreamResponses = downstreams .Select(ds => $"{serviceName}:{ds.Host}:{ds.Port}") - .ToList(); + .ToArray(); var subset = new EndpointSubsetV1(); downstreams.ForEach(ds => GivenSubsetAddress(ds, subset)); var endpoints = GivenEndpoints(subset, serviceName); // totalServices service instances with different ports var route = GivenRouteWithServiceName(namespaces, serviceName, nameof(RoundRobinAnalyzer)); // !!! var configuration = GivenKubeConfiguration(namespaces, route); - GivenMultipleK8sProductServicesAreRunning(downstreamUrls, downstreamResponses); + GivenMultipleServiceInstancesAreRunning(downstreamUrls, downstreamResponses); GivenThereIsAConfiguration(configuration); GivenOcelotIsRunningWithServices(WithKubernetesAndRoundRobin); return (endpoints, servicePorts); @@ -190,7 +185,7 @@ public void ShouldHighlyLoadOnUnstableKubeProvider_WithRoundRobinLoadBalancing(i private void HighlyLoadOnKubeProviderAndRoundRobinBalancer(int totalRequests, int k8sGenerationNo) { // Act - WhenIGetUrlOnTheApiGatewayMultipleTimes("/", totalRequests); // load by X parallel requests + WhenIGetUrlOnTheApiGatewayConcurrently("/", totalRequests); // load by X parallel requests // Assert _k8sCounter.ShouldBeGreaterThanOrEqualTo(totalRequests); // integration endpoint called times @@ -198,6 +193,7 @@ private void HighlyLoadOnKubeProviderAndRoundRobinBalancer(int totalRequests, in ThenAllStatusCodesShouldBe(HttpStatusCode.OK); ThenAllServicesShouldHaveBeenCalledTimes(totalRequests); _roundRobinAnalyzer.ShouldNotBeNull().Analyze(); + _roundRobinAnalyzer.Events.Count.ShouldBe(totalRequests); _roundRobinAnalyzer.HasManyServiceGenerations(k8sGenerationNo).ShouldBeTrue(); } @@ -240,8 +236,7 @@ private static EndpointSubsetV1 GivenSubsetAddress(Uri downstream, EndpointSubse private FileRoute GivenRouteWithServiceName(string serviceNamespace, [CallerMemberName] string serviceName = null, - string loadBalancerType = nameof(LeastConnection)) - => new() + string loadBalancerType = nameof(LeastConnection)) => new() { DownstreamPathTemplate = "/", DownstreamScheme = null, // the scheme should not be defined in service discovery scenarios by default, only ServiceName @@ -281,11 +276,13 @@ private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, bool isS await Task.Delay(Random.Shared.Next(1, 10)); // emulate integration delay up to 10 milliseconds if (context.Request.Path.Value == $"/api/v1/namespaces/{namespaces}/endpoints/{serviceName}") { - // Each offlinePerThreads-th request to integrated K8s endpoint should fail + string json; lock (K8sCounterLocker) { _k8sCounter++; var subset = endpoints.Subsets[0]; + + // Each offlinePerThreads-th request to integrated K8s endpoint should fail if (!isStable && _k8sCounter % offlinePerThreads == 0 && _k8sCounter >= offlinePerThreads) { while (offlineServicesNo-- > 0) @@ -299,6 +296,7 @@ private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, bool isS } endpoints.Metadata.Generation = _k8sServiceGeneration; + json = JsonConvert.SerializeObject(endpoints); } if (context.Request.Headers.TryGetValue("Authorization", out var values)) @@ -306,7 +304,6 @@ private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, bool isS _receivedToken = values.First(); } - var json = JsonConvert.SerializeObject(endpoints); context.Response.Headers.Append("Content-Type", "application/json"); await context.Response.WriteAsync(json); } @@ -324,93 +321,14 @@ private void WithKubernetesAndRoundRobin(IServiceCollection services) => service .RemoveAll().AddSingleton(_clientFactory) .RemoveAll().AddSingleton(); + private int _k8sCounter, _k8sServiceGeneration; + private static readonly object K8sCounterLocker = new(); private RoundRobinAnalyzer _roundRobinAnalyzer; private RoundRobinAnalyzer GetRoundRobinAnalyzer(DownstreamRoute route, IServiceDiscoveryProvider provider) { lock (K8sCounterLocker) { - return _roundRobinAnalyzer ??= new RoundRobinAnalyzer(provider.GetAsync, route.ServiceName); - } - } - - private static readonly object ServiceCountersLocker = new(); - private Dictionary _serviceCounters; - - private static readonly object K8sCounterLocker = new(); - private int _k8sCounter, _k8sServiceGeneration; - - private void GivenK8sProductServiceIsRunning(string url, string response) - { - _serviceHandlers.Add(new()); // allocate single instance - _serviceCounters = new(); // single counter - GivenK8sProductServiceIsRunning(url, response, 0); - _serviceCounters[0] = 0; - } - - private void GivenMultipleK8sProductServicesAreRunning(List urls, List responses) - { - urls.ForEach(_ => _serviceHandlers.Add(new())); // allocate multiple instances - _serviceCounters = new(urls.Count); // multiple counters - for (int i = 0; i < urls.Count; i++) - { - GivenK8sProductServiceIsRunning(urls[i], responses[i], i); - _serviceCounters[i] = 0; - } - } - - private void GivenK8sProductServiceIsRunning(string url, string response, int handlerIndex) - { - var serviceHandler = _serviceHandlers[handlerIndex]; - serviceHandler.GivenThereIsAServiceRunningOn(url, async context => - { - await Task.Delay(Random.Shared.Next(5, 15)); // emulate integration delay up to 15 milliseconds - int count = 0; - lock (ServiceCountersLocker) - { - count = ++_serviceCounters[handlerIndex]; - } - - context.Response.StatusCode = (int)HttpStatusCode.OK; - var threadResponse = string.Concat(count, ':', response); - await context.Response.WriteAsync(threadResponse ?? ((int)HttpStatusCode.OK).ToString()); - }); - } - - private void ThenAllServicesShouldHaveBeenCalledTimes(int expected) - { - var sortedByIndex = _serviceCounters.OrderBy(_ => _.Key).Select(_ => _.Value).ToArray(); - var customMessage = $"All values are [{string.Join(',', sortedByIndex)}]"; - _serviceCounters.Sum(_ => _.Value).ShouldBe(expected, customMessage); - _roundRobinAnalyzer.Events.Count.ShouldBe(expected); - } - - private void ThenAllServicesCalledRealisticAmountOfTimes(int bottom, int top) - { - var sortedByIndex = _serviceCounters.OrderBy(_ => _.Key).Select(_ => _.Value).ToArray(); - var customMessage = $"{nameof(bottom)}: {bottom}\n {nameof(top)}: {top}\n All values are [{string.Join(',', sortedByIndex)}]"; - int sum = 0, totalSum = _serviceCounters.Sum(_ => _.Value); - - // Last services cannot be called at all, zero counters - for (int i = 0; i < _serviceCounters.Count && sum < totalSum; i++) - { - int actual = _serviceCounters[i]; - actual.ShouldBeInRange(bottom, top, customMessage); - sum += actual; - } - } - - private void ThenServiceCountersShouldMatchLeasingCounters(int[] ports) - { - var leasingCounters = _roundRobinAnalyzer.GetHostCounters(); - for (int i = 0; i < ports.Length; i++) - { - var host = leasingCounters.Keys.FirstOrDefault(k => k.DownstreamPort == ports[i]); - if (host != null) // leasing info/counters can be absent because of offline service instance with exact port in unstable scenario - { - int counter1 = _serviceCounters[i]; - int counter2 = leasingCounters[host]; - counter1.ShouldBe(counter2, $"Port: {ports[i]}\n Host: {host}"); - } + return _roundRobinAnalyzer ??= new RoundRobinAnalyzerCreator().Create(route, provider)?.Data as RoundRobinAnalyzer; //??= new RoundRobinAnalyzer(provider.GetAsync, route.ServiceName); } } } @@ -421,7 +339,6 @@ public FakeKubeServiceCreator(IOcelotLoggerFactory factory) : base(factory) { } protected override ServiceHostAndPort GetServiceHostAndPort(KubeRegistryConfiguration configuration, EndpointsV1 endpoint, EndpointSubsetV1 subset, EndpointAddressV1 address) { - //return base.GetServiceHostAndPort(configuration, endpoint, subset, address); var ports = subset.Ports; var index = subset.Addresses.IndexOf(address); var portV1 = ports[index]; @@ -438,109 +355,3 @@ protected override IEnumerable GetServiceTags(KubeRegistryConfiguration return tags; } } - -internal class RoundRobinAnalyzer : RoundRobin, ILoadBalancer -{ - public readonly ConcurrentBag Events = new(); - - public RoundRobinAnalyzer(Func>> services, string serviceName) - : base(services, serviceName) - { - this.Leased += Me_Leased; - } - - private void Me_Leased(object sender, LeaseEventArgs e) => Events.Add(e); - - public const string GenerationPrefix = nameof(EndpointsV1.Metadata.Generation) + ":"; - - public object Analyze() - { - var allGenerations = Events - .Select(e => e.Service.Tags.FirstOrDefault(t => t.StartsWith(GenerationPrefix))) - .Distinct().ToArray(); - var allIndices = Events.Select(e => e.ServiceIndex) - .Distinct().ToArray(); - - Dictionary> eventsPerGeneration = new(); - foreach (var generation in allGenerations) - { - var l = Events.Where(e => e.Service.Tags.Contains(generation)).ToList(); - eventsPerGeneration.Add(generation, l); - } - - Dictionary> generationIndices = new(); - foreach (var generation in allGenerations) - { - var l = eventsPerGeneration[generation].Select(e => e.ServiceIndex).Distinct().ToList(); - generationIndices.Add(generation, l); - } - - Dictionary> generationLeases = new(); - foreach (var generation in allGenerations) - { - var l = eventsPerGeneration[generation].Select(e => e.Lease).ToList(); - generationLeases.Add(generation, l); - } - - Dictionary> generationHosts = new(); - foreach (var generation in allGenerations) - { - var l = eventsPerGeneration[generation].Select(e => e.Lease.HostAndPort).Distinct().ToList(); - generationHosts.Add(generation, l); - } - - Dictionary> generationLeasesWithMaxConnections = new(); - foreach (var generation in allGenerations) - { - List leases = new(); - var uniqueHosts = generationHosts[generation]; - foreach (var host in uniqueHosts) - { - int max = generationLeases[generation].Where(l => l == host).Max(l => l.Connections); - Lease wanted = generationLeases[generation].Find(l => l == host && l.Connections == max); - leases.Add(wanted); - } - - leases = leases.OrderBy(l => l.HostAndPort.DownstreamPort).ToList(); - generationLeasesWithMaxConnections.Add(generation, leases); - } - - return generationLeasesWithMaxConnections; - } - - public bool HasManyServiceGenerations(int maxGeneration) - { - int[] generations = new int[maxGeneration + 1]; - string[] tags = new string[maxGeneration + 1]; - for (int i = 0; i < generations.Length; i++) - { - generations[i] = i; - tags[i] = GenerationPrefix + i; - } - - var all = Events - .Select(e => e.Service.Tags.FirstOrDefault(t => t.StartsWith(GenerationPrefix))) - .Distinct().ToArray(); - return all.All(tags.Contains); - } - - public Dictionary GetHostCounters() - { - var hosts = Events.Select(e => e.Lease.HostAndPort).Distinct().ToList(); - return Events - .GroupBy(e => e.Lease.HostAndPort) - .ToDictionary(g => g.Key, g => g.Max(e => e.Lease.Connections)); - } - - public int BottomOfConnections() - { - var hostCounters = GetHostCounters(); - return hostCounters.Min(_ => _.Value); - } - - public int TopOfConnections() - { - var hostCounters = GetHostCounters(); - return hostCounters.Max(_ => _.Value); - } -} diff --git a/test/Ocelot.AcceptanceTests/ServiceHandler.cs b/test/Ocelot.AcceptanceTests/ServiceHandler.cs index c2e10a819..5996387a3 100644 --- a/test/Ocelot.AcceptanceTests/ServiceHandler.cs +++ b/test/Ocelot.AcceptanceTests/ServiceHandler.cs @@ -12,7 +12,7 @@ public class ServiceHandler : IDisposable { private IWebHost _builder; - public void GivenThereIsAServiceRunningOn(string baseUrl, RequestDelegate del) + public void GivenThereIsAServiceRunningOn(string baseUrl, RequestDelegate handler) { _builder = new WebHostBuilder() .UseUrls(baseUrl) @@ -21,14 +21,14 @@ public void GivenThereIsAServiceRunningOn(string baseUrl, RequestDelegate del) .UseIISIntegration() .Configure(app => { - app.Run(del); + app.Run(handler); }) .Build(); _builder.Start(); } - public void GivenThereIsAServiceRunningOn(string baseUrl, string basePath, RequestDelegate del) + public void GivenThereIsAServiceRunningOn(string baseUrl, string basePath, RequestDelegate handler) { _builder = new WebHostBuilder() .UseUrls(baseUrl) @@ -38,14 +38,14 @@ public void GivenThereIsAServiceRunningOn(string baseUrl, string basePath, Reque .Configure(app => { app.UsePathBase(basePath); - app.Run(del); + app.Run(handler); }) .Build(); _builder.Start(); } - public void GivenThereIsAServiceRunningOnWithKestrelOptions(string baseUrl, string basePath, Action options, RequestDelegate del) + public void GivenThereIsAServiceRunningOnWithKestrelOptions(string baseUrl, string basePath, Action options, RequestDelegate handler) { _builder = new WebHostBuilder() .UseUrls(baseUrl) @@ -56,7 +56,7 @@ public void GivenThereIsAServiceRunningOnWithKestrelOptions(string baseUrl, stri .Configure(app => { app.UsePathBase(basePath); - app.Run(del); + app.Run(handler); }) .Build(); @@ -67,7 +67,7 @@ internal void WithDefaultKestrelServerOptions(KestrelServerOptions options) { } - public void GivenThereIsAServiceRunningOn(string baseUrl, string basePath, string fileName, string password, int port, RequestDelegate del) + public void GivenThereIsAServiceRunningOn(string baseUrl, string basePath, string fileName, string password, int port, RequestDelegate handler) { _builder = new WebHostBuilder() .UseUrls(baseUrl) @@ -82,7 +82,7 @@ public void GivenThereIsAServiceRunningOn(string baseUrl, string basePath, strin .Configure(app => { app.UsePathBase(basePath); - app.Run(del); + app.Run(handler); }) .Build(); diff --git a/test/Ocelot.AcceptanceTests/Steps.cs b/test/Ocelot.AcceptanceTests/Steps.cs index 501e741d4..d35651046 100644 --- a/test/Ocelot.AcceptanceTests/Steps.cs +++ b/test/Ocelot.AcceptanceTests/Steps.cs @@ -11,24 +11,20 @@ using Newtonsoft.Json; using Ocelot.AcceptanceTests.Caching; using Ocelot.Cache.CacheManager; -using Ocelot.Configuration; using Ocelot.Configuration.ChangeTracking; using Ocelot.Configuration.Creator; using Ocelot.Configuration.File; using Ocelot.Configuration.Repository; using Ocelot.DependencyInjection; -using Ocelot.LoadBalancer.LoadBalancers; using Ocelot.Logging; using Ocelot.Middleware; using Ocelot.Provider.Consul; using Ocelot.Provider.Eureka; using Ocelot.Provider.Polly; -using Ocelot.ServiceDiscovery.Providers; using Ocelot.Tracing.Butterfly; using Ocelot.Tracing.OpenTracing; using Serilog; using Serilog.Core; -using System.Collections.Concurrent; using System.IO.Compression; using System.Net.Http.Headers; using System.Text; @@ -44,7 +40,6 @@ public class Steps : IDisposable protected TestServer _ocelotServer; protected HttpClient _ocelotClient; protected HttpResponseMessage _response; - protected ConcurrentDictionary _parallelResponses; private HttpContent _postContent; private BearerToken _token; public string RequestIdKey = "OcRequestId"; @@ -63,7 +58,6 @@ public Steps() _ocelotConfigFileName = $"{_testId:N}-{ConfigurationBuilderExtensions.PrimaryConfigFile}"; Files = new() { _ocelotConfigFileName }; Folders = new(); - _parallelResponses = new(); } protected List Files { get; } @@ -281,62 +275,6 @@ protected void StartOcelot(Action _ocelotClient = _ocelotServer.CreateClient(); } - /// - /// This is annoying cos it should be in the constructor but we need to set up the file before calling startup so its a step. - /// - /// The type. - /// The delegate object to load balancer factory. - public void GivenOcelotIsRunningWithCustomLoadBalancer(Func loadBalancerFactoryFunc) - where T : ILoadBalancer - { - _webHostBuilder = new WebHostBuilder() - .ConfigureAppConfiguration((hostingContext, config) => - { - config.SetBasePath(hostingContext.HostingEnvironment.ContentRootPath); - var env = hostingContext.HostingEnvironment; - config.AddJsonFile("appsettings.json", true, false) - .AddJsonFile($"appsettings.{env.EnvironmentName}.json", true, false); - config.AddJsonFile(_ocelotConfigFileName, false, false); - config.AddEnvironmentVariables(); - }) - .ConfigureServices(s => - { - s.AddOcelot() - .AddCustomLoadBalancer(loadBalancerFactoryFunc); - }) - .Configure(app => { app.UseOcelot().Wait(); }); - - _ocelotServer = new TestServer(_webHostBuilder); - _ocelotClient = _ocelotServer.CreateClient(); - } - - public void GivenOcelotIsRunningWithConsul(params string[] urlsToListenOn) - { - _webHostBuilder = new WebHostBuilder(); - - if (urlsToListenOn?.Length > 0) - { - _webHostBuilder.UseUrls(urlsToListenOn); - } - - _webHostBuilder - .ConfigureAppConfiguration((hostingContext, config) => - { - config.SetBasePath(hostingContext.HostingEnvironment.ContentRootPath); - var env = hostingContext.HostingEnvironment; - config.AddJsonFile("appsettings.json", true, false) - .AddJsonFile($"appsettings.{env.EnvironmentName}.json", true, false); - config.AddJsonFile(_ocelotConfigFileName, false, false); - config.AddEnvironmentVariables(); - }) - .ConfigureServices(s => { s.AddOcelot().AddConsul(); }) - .Configure(app => { app.UseOcelot().Wait(); }); - - _ocelotServer = new TestServer(_webHostBuilder); - - _ocelotClient = _ocelotServer.CreateClient(); - } - public void ThenTheTraceHeaderIsSet(string key) { var header = _response.Headers.GetValues(key); @@ -891,34 +829,17 @@ public void GivenIAddAHeader(string key, string value) { _ocelotClient.DefaultRequestHeaders.TryAddWithoutValidation(key, value); } - - public Task[] WhenIGetUrlOnTheApiGatewayMultipleTimes(string url, int times) - { - var tasks = new Task[times]; - _parallelResponses = new(times, times); - for (var i = 0; i < times; i++) - { - tasks[i] = GetParallelResponse(url, i); - _parallelResponses[i] = null; - } - - Task.WaitAll(tasks); - return tasks; - } - - private async Task GetParallelResponse(string url, int threadIndex) - { - var response = await _ocelotClient.GetAsync(url); - //Thread.Sleep(_random.Next(40, 60)); - //var content = await response.Content.ReadAsStringAsync(); - //var counterValue = content.Contains(':') - // ? content.Split(':')[0] // let the first fragment is counter value - // : content; - //int count = int.Parse(counterValue); - //count.ShouldBeGreaterThan(0); - _parallelResponses[threadIndex] = response; - } + public static void WhenIDoActionMultipleTimes(int times, Action action) + { + for (int i = 0; i < times; i++) + action?.Invoke(); + } + public static void WhenIDoActionMultipleTimes(int times, Action action) + { + for (int i = 0; i < times; i++) + action?.Invoke(i); + } public void WhenIGetUrlOnTheApiGatewayMultipleTimesForRateLimit(string url, int times) { @@ -970,10 +891,10 @@ public void GivenThePostHasGzipContent(object input) _postContent = content; } - public void ThenTheResponseBodyShouldBe(string expectedBody) - { - _response.Content.ReadAsStringAsync().Result.ShouldBe(expectedBody); - } + public void ThenTheResponseBodyShouldBe(string expectedBody) + => _response.Content.ReadAsStringAsync().Result.ShouldBe(expectedBody); + public void ThenTheResponseBodyShouldBe(string expectedBody, string customMessage) + => _response.Content.ReadAsStringAsync().Result.ShouldBe(expectedBody, customMessage); public void ThenTheContentLengthIs(int expected) { @@ -982,9 +903,6 @@ public void ThenTheContentLengthIs(int expected) public void ThenTheStatusCodeShouldBe(HttpStatusCode expected) => _response.StatusCode.ShouldBe(expected); - - public void ThenAllStatusCodesShouldBe(HttpStatusCode expected) - => _parallelResponses.ShouldAllBe(response => response.Value.StatusCode == expected); public void ThenTheStatusCodeShouldBe(int expectedHttpStatusCode) { @@ -1195,11 +1113,6 @@ protected virtual void Dispose(bool disposing) _ocelotServer?.Dispose(); _ocelotHost?.Dispose(); _response?.Dispose(); - foreach (var response in _parallelResponses) - { - response.Value?.Dispose(); - } - DeleteFiles(); DeleteFolders(); } diff --git a/test/Ocelot.AcceptanceTests/StickySessionsTests.cs b/test/Ocelot.AcceptanceTests/StickySessionsTests.cs index cc7435e19..9127a02d3 100644 --- a/test/Ocelot.AcceptanceTests/StickySessionsTests.cs +++ b/test/Ocelot.AcceptanceTests/StickySessionsTests.cs @@ -28,6 +28,7 @@ public override void Dispose() } [Fact] + [Trait("Feat", "336")] public void ShouldUseSameDownstreamHost_ForSingleRouteWithHighLoad() { var port1 = PortFinder.GetRandomPort(); @@ -48,6 +49,7 @@ public void ShouldUseSameDownstreamHost_ForSingleRouteWithHighLoad() } [Fact] + [Trait("Feat", "336")] public void ShouldUseDifferentDownstreamHost_ForDoubleRoutesWithDifferentCookies() { var port1 = PortFinder.GetRandomPort(); @@ -71,6 +73,7 @@ public void ShouldUseDifferentDownstreamHost_ForDoubleRoutesWithDifferentCookies } [Fact] + [Trait("Feat", "336")] public void ShouldUseSameDownstreamHost_ForDifferentRoutesWithSameCookie() { var port1 = PortFinder.GetRandomPort(); diff --git a/test/Ocelot.Testing/PortFinder.cs b/test/Ocelot.Testing/PortFinder.cs index 6eb6b64d4..42b904991 100644 --- a/test/Ocelot.Testing/PortFinder.cs +++ b/test/Ocelot.Testing/PortFinder.cs @@ -12,26 +12,42 @@ public static class PortFinder private static readonly ConcurrentBag UsedPorts = new(); /// - /// Gets a pseudo-random port from the range [, ]. + /// Gets a pseudo-random port from the range [, ] for one testing scenario. /// - /// New allocated port for testing scenario. + /// New allocated port. /// Critical situation where available ports range has been exceeded. public static int GetRandomPort() { lock (LockObj) { - if (CurrentPort > EndPortRange) + ExceedingPortRangeException.ThrowIf(CurrentPort > EndPortRange); + return UsePort(CurrentPort++); + } + } + + /// + /// Gets the exact number of ports from the range [, ] for one testing scenario. + /// + /// The number of wanted ports. + /// Array of allocated ports. + /// Critical situation where available ports range has been exceeded. + public static int[] GetPorts(int count) + { + var ports = new int[count]; + lock (LockObj) + { + for (int i = 0; i < count; i++, CurrentPort++) { - throw new ExceedingPortRangeException(); + ExceedingPortRangeException.ThrowIf(CurrentPort > EndPortRange); + ports[i] = UsePort(CurrentPort); } - - return UsePort(CurrentPort++); } + return ports; } private static int UsePort(int port) { - UsedPorts.Add(port); + UsedPorts.Add(port); // TODO Review or remove, now useless var ipe = new IPEndPoint(IPAddress.Loopback, port); @@ -46,4 +62,7 @@ public class ExceedingPortRangeException : Exception { public ExceedingPortRangeException() : base("Cannot find available port to bind to!") { } + + public static void ThrowIf(bool condition) + => _ = condition ? throw new ExceedingPortRangeException() : 0; } diff --git a/test/Ocelot.UnitTests/Consul/ConsulTests.cs b/test/Ocelot.UnitTests/Consul/ConsulTests.cs index b9009d488..fff9eaf91 100644 --- a/test/Ocelot.UnitTests/Consul/ConsulTests.cs +++ b/test/Ocelot.UnitTests/Consul/ConsulTests.cs @@ -20,6 +20,7 @@ public sealed class ConsulTests : UnitTest, IDisposable private readonly List _consulServiceEntries; private readonly Mock _factory; private readonly Mock _logger; + private readonly Mock _contextAccessor; private IConsulClientFactory _clientFactory; private IConsulServiceBuilder _serviceBuilder; private ConsulRegistryConfiguration _config; @@ -36,6 +37,7 @@ public ConsulTests() _consulServiceEntries = new List(); _factory = new Mock(); _logger = new Mock(); + _contextAccessor = new Mock(); _factory.Setup(x => x.CreateLogger()).Returns(_logger.Object); _factory.Setup(x => x.CreateLogger()).Returns(_logger.Object); _factory.Setup(x => x.CreateLogger()).Returns(_logger.Object); @@ -49,8 +51,11 @@ public void Dispose() private void Arrange([CallerMemberName] string serviceName = null) { _config = new ConsulRegistryConfiguration(_consulScheme, _consulHost, _port, serviceName, null); + var context = new DefaultHttpContext(); + context.Items.Add(nameof(ConsulRegistryConfiguration), _config); + _contextAccessor.SetupGet(x => x.HttpContext).Returns(context); _clientFactory = new ConsulClientFactory(); - _serviceBuilder = new DefaultConsulServiceBuilder(() => _config, _clientFactory, _factory.Object); + _serviceBuilder = new DefaultConsulServiceBuilder(_contextAccessor.Object, _clientFactory, _factory.Object); _provider = new ConsulProvider(_config, _factory.Object, _clientFactory, _serviceBuilder); } diff --git a/test/Ocelot.UnitTests/Consul/DefaultConsulServiceBuilderTests.cs b/test/Ocelot.UnitTests/Consul/DefaultConsulServiceBuilderTests.cs index 25dc8d950..5858efb60 100644 --- a/test/Ocelot.UnitTests/Consul/DefaultConsulServiceBuilderTests.cs +++ b/test/Ocelot.UnitTests/Consul/DefaultConsulServiceBuilderTests.cs @@ -1,28 +1,25 @@ -using Castle.Components.DictionaryAdapter.Xml; -using Consul; +using Consul; +using Microsoft.AspNetCore.Http; using Ocelot.Logging; using Ocelot.Provider.Consul; using Ocelot.Provider.Consul.Interfaces; using System.Reflection; using System.Runtime.CompilerServices; -using System.Xml.Linq; namespace Ocelot.UnitTests.Consul; public sealed class DefaultConsulServiceBuilderTests { private DefaultConsulServiceBuilder sut; - private readonly Func configurationFactory; + private readonly Mock contextAccessor; private readonly Mock clientFactory; private readonly Mock loggerFactory; private readonly Mock logger; private ConsulRegistryConfiguration _configuration; - private ConsulRegistryConfiguration GetConfiguration() => _configuration; - public DefaultConsulServiceBuilderTests() { - configurationFactory = GetConfiguration; + contextAccessor = new(); clientFactory = new(); clientFactory.Setup(x => x.Get(It.IsAny())) .Returns(new ConsulClient()); @@ -35,20 +32,25 @@ public DefaultConsulServiceBuilderTests() private void Arrange([CallerMemberName] string testName = null) { _configuration = new(null, null, 0, testName, null); - sut = new DefaultConsulServiceBuilder(configurationFactory, clientFactory.Object, loggerFactory.Object); + var context = new DefaultHttpContext(); + context.Items.Add(nameof(ConsulRegistryConfiguration), _configuration); + contextAccessor.SetupGet(x => x.HttpContext).Returns(context); + sut = new DefaultConsulServiceBuilder(contextAccessor.Object, clientFactory.Object, loggerFactory.Object); } [Fact] public void Ctor_PrivateMembers_PropertiesAreInitialized() { Arrange(); - var methodClient = sut.GetType().GetProperty("Client", BindingFlags.NonPublic | BindingFlags.Instance); - var methodLogger = sut.GetType().GetProperty("Logger", BindingFlags.NonPublic | BindingFlags.Instance); + var propClient = sut.GetType().GetProperty("Client", BindingFlags.NonPublic | BindingFlags.Instance); + var propLogger = sut.GetType().GetProperty("Logger", BindingFlags.NonPublic | BindingFlags.Instance); + var propConfiguration = sut.GetType().GetProperty("Configuration", BindingFlags.NonPublic | BindingFlags.Instance); // Act - var actualConfiguration = sut.Configuration; - var actualClient = methodClient.GetValue(sut); - var actualLogger = methodLogger.GetValue(sut); + //var actualConfiguration = sut.Configuration; + var actualConfiguration = propConfiguration.GetValue(sut); + var actualClient = propClient.GetValue(sut); + var actualLogger = propLogger.GetValue(sut); // Assert actualConfiguration.ShouldNotBeNull().ShouldBe(_configuration); diff --git a/test/Ocelot.UnitTests/Consul/ProviderFactoryTests.cs b/test/Ocelot.UnitTests/Consul/ProviderFactoryTests.cs index d7b676a23..247af37e9 100644 --- a/test/Ocelot.UnitTests/Consul/ProviderFactoryTests.cs +++ b/test/Ocelot.UnitTests/Consul/ProviderFactoryTests.cs @@ -1,4 +1,5 @@ -using Microsoft.Extensions.DependencyInjection; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; using Ocelot.Configuration; using Ocelot.Configuration.Builder; using Ocelot.Logging; @@ -14,12 +15,20 @@ public class ProviderFactoryTests public ProviderFactoryTests() { - var services = new ServiceCollection(); - var loggerFactory = new Mock(); + var contextAccessor = new Mock(); + var context = new DefaultHttpContext(); + context.Items.Add(nameof(ConsulRegistryConfiguration), new ConsulRegistryConfiguration(null, null, 0, null, null)); + contextAccessor.SetupGet(x => x.HttpContext).Returns(context); + + var loggerFactory = new Mock(); var logger = new Mock(); loggerFactory.Setup(x => x.CreateLogger()).Returns(logger.Object); loggerFactory.Setup(x => x.CreateLogger()).Returns(logger.Object); + var consulFactory = new Mock(); + + var services = new ServiceCollection(); + services.AddSingleton(contextAccessor.Object); services.AddSingleton(consulFactory.Object); services.AddSingleton(loggerFactory.Object); _provider = services.BuildServiceProvider(); diff --git a/test/Ocelot.UnitTests/DependencyInjection/OcelotBuilderTests.cs b/test/Ocelot.UnitTests/DependencyInjection/OcelotBuilderTests.cs index 262014927..933578ccb 100644 --- a/test/Ocelot.UnitTests/DependencyInjection/OcelotBuilderTests.cs +++ b/test/Ocelot.UnitTests/DependencyInjection/OcelotBuilderTests.cs @@ -534,17 +534,13 @@ private void ThenAnExceptionIsntThrown() private class FakeCustomLoadBalancer : ILoadBalancer { - public Task> Lease(HttpContext httpContext) - { - // Not relevant for these tests - throw new NotImplementedException(); - } + public string Type => nameof(FakeCustomLoadBalancer); - public void Release(ServiceHostAndPort hostAndPort) - { - // Not relevant for these tests - throw new NotImplementedException(); - } + // Not relevant for these tests + public Task> LeaseAsync(HttpContext httpContext) => throw new NotImplementedException(); + + // Not relevant for these tests + public void Release(ServiceHostAndPort hostAndPort) => throw new NotImplementedException(); } } } diff --git a/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs b/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs index 1b3a3f416..c71444045 100644 --- a/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs +++ b/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs @@ -26,6 +26,7 @@ public KubeTests() } [Fact] + [Trait("Feat", "345")] public async Task Should_return_service_from_k8s() { // Arrange diff --git a/test/Ocelot.UnitTests/Kubernetes/OcelotBuilderExtensionsTests.cs b/test/Ocelot.UnitTests/Kubernetes/OcelotBuilderExtensionsTests.cs index d3c0b9967..7e60dd7e2 100644 --- a/test/Ocelot.UnitTests/Kubernetes/OcelotBuilderExtensionsTests.cs +++ b/test/Ocelot.UnitTests/Kubernetes/OcelotBuilderExtensionsTests.cs @@ -33,7 +33,8 @@ private static IWebHostEnvironment GetHostingEnvironment() } [Fact] - public void should_set_up_kubernetes() + [Trait("Feat", "345")] + public void Should_set_up_kubernetes() { this.Given(x => WhenISetUpOcelotServices()) .When(x => WhenISetUpKubernetes()) diff --git a/test/Ocelot.UnitTests/Kubernetes/PollKubeTests.cs b/test/Ocelot.UnitTests/Kubernetes/PollKubeTests.cs index 794589bc3..4d6c27483 100644 --- a/test/Ocelot.UnitTests/Kubernetes/PollKubeTests.cs +++ b/test/Ocelot.UnitTests/Kubernetes/PollKubeTests.cs @@ -27,7 +27,8 @@ public PollKubeTests() } [Fact] - public void should_return_service_from_kube() + [Trait("Feat", "345")] + public void Should_return_service_from_kube() { var service = new Service(string.Empty, new ServiceHostAndPort(string.Empty, 0), string.Empty, string.Empty, new List()); diff --git a/test/Ocelot.UnitTests/LoadBalancer/CookieStickySessionsTests.cs b/test/Ocelot.UnitTests/LoadBalancer/CookieStickySessionsTests.cs index c13155e46..c2c95f921 100644 --- a/test/Ocelot.UnitTests/LoadBalancer/CookieStickySessionsTests.cs +++ b/test/Ocelot.UnitTests/LoadBalancer/CookieStickySessionsTests.cs @@ -120,7 +120,7 @@ private void ThenAnErrorIsReturned() private void GivenTheLoadBalancerReturnsError() { _loadBalancer - .Setup(x => x.Lease(It.IsAny())) + .Setup(x => x.LeaseAsync(It.IsAny())) .ReturnsAsync(new ErrorResponse(new AnyError())); } @@ -149,14 +149,14 @@ private async Task WhenIMakeTwoRequetsWithDifferentSessionValues([CallerMemberNa contextTwo.Request.Cookies = cookiesTwo; contextTwo.Items.UpsertDownstreamRoute(route); - _firstHostAndPort = await _stickySessions.Lease(contextOne); - _secondHostAndPort = await _stickySessions.Lease(contextTwo); + _firstHostAndPort = await _stickySessions.LeaseAsync(contextOne); + _secondHostAndPort = await _stickySessions.LeaseAsync(contextTwo); } private void GivenTheLoadBalancerReturnsSequence() { _loadBalancer - .SetupSequence(x => x.Lease(It.IsAny())) + .SetupSequence(x => x.LeaseAsync(It.IsAny())) .ReturnsAsync(new OkResponse(new ServiceHostAndPort("one", 80))) .ReturnsAsync(new OkResponse(new ServiceHostAndPort("two", 80))); } @@ -169,8 +169,8 @@ private void ThenTheFirstAndSecondResponseAreTheSame() private async Task WhenILeaseTwiceInARow() { - _firstHostAndPort = await _stickySessions.Lease(_httpContext); - _secondHostAndPort = await _stickySessions.Lease(_httpContext); + _firstHostAndPort = await _stickySessions.LeaseAsync(_httpContext); + _secondHostAndPort = await _stickySessions.LeaseAsync(_httpContext); } private void GivenTheDownstreamRequestHasSessionId(string value) @@ -183,13 +183,13 @@ private void GivenTheDownstreamRequestHasSessionId(string value) private void GivenTheLoadBalancerReturns() { _loadBalancer - .Setup(x => x.Lease(It.IsAny())) + .Setup(x => x.LeaseAsync(It.IsAny())) .ReturnsAsync(new OkResponse(new ServiceHostAndPort(string.Empty, 80))); } private async Task WhenILease() { - _result = await _stickySessions.Lease(_httpContext); + _result = await _stickySessions.LeaseAsync(_httpContext); } private void ThenTheHostAndPortIsNotNull() diff --git a/test/Ocelot.UnitTests/LoadBalancer/DelegateInvokingLoadBalancerCreatorTests.cs b/test/Ocelot.UnitTests/LoadBalancer/DelegateInvokingLoadBalancerCreatorTests.cs index deb25a12e..188993807 100644 --- a/test/Ocelot.UnitTests/LoadBalancer/DelegateInvokingLoadBalancerCreatorTests.cs +++ b/test/Ocelot.UnitTests/LoadBalancer/DelegateInvokingLoadBalancerCreatorTests.cs @@ -107,15 +107,9 @@ public FakeLoadBalancer(DownstreamRoute downstreamRoute, IServiceDiscoveryProvid public DownstreamRoute DownstreamRoute { get; } public IServiceDiscoveryProvider ServiceDiscoveryProvider { get; } - public Task> Lease(HttpContext httpContext) - { - throw new NotImplementedException(); - } - - public void Release(ServiceHostAndPort hostAndPort) - { - throw new NotImplementedException(); - } + public string Type => nameof(FakeLoadBalancer); + public Task> LeaseAsync(HttpContext httpContext) => throw new NotImplementedException(); + public void Release(ServiceHostAndPort hostAndPort) => throw new NotImplementedException(); } } } diff --git a/test/Ocelot.UnitTests/LoadBalancer/LeastConnectionTests.cs b/test/Ocelot.UnitTests/LoadBalancer/LeastConnectionTests.cs index c0ee460ca..6ca090c97 100644 --- a/test/Ocelot.UnitTests/LoadBalancer/LeastConnectionTests.cs +++ b/test/Ocelot.UnitTests/LoadBalancer/LeastConnectionTests.cs @@ -58,9 +58,9 @@ public async Task Should_handle_service_returning_to_available() _leastConnection = new LeastConnection(() => Task.FromResult(availableServices), serviceName); - var hostAndPortOne = await _leastConnection.Lease(_httpContext); + var hostAndPortOne = await _leastConnection.LeaseAsync(_httpContext); hostAndPortOne.Data.DownstreamHost.ShouldBe("127.0.0.1"); - var hostAndPortTwo = await _leastConnection.Lease(_httpContext); + var hostAndPortTwo = await _leastConnection.LeaseAsync(_httpContext); hostAndPortTwo.Data.DownstreamHost.ShouldBe("127.0.0.2"); _leastConnection.Release(hostAndPortOne.Data); _leastConnection.Release(hostAndPortTwo.Data); @@ -70,9 +70,9 @@ public async Task Should_handle_service_returning_to_available() new(serviceName, new ServiceHostAndPort("127.0.0.1", 80), string.Empty, string.Empty, Array.Empty()), }; - hostAndPortOne = await _leastConnection.Lease(_httpContext); + hostAndPortOne = await _leastConnection.LeaseAsync(_httpContext); hostAndPortOne.Data.DownstreamHost.ShouldBe("127.0.0.1"); - hostAndPortTwo = await _leastConnection.Lease(_httpContext); + hostAndPortTwo = await _leastConnection.LeaseAsync(_httpContext); hostAndPortTwo.Data.DownstreamHost.ShouldBe("127.0.0.1"); _leastConnection.Release(hostAndPortOne.Data); _leastConnection.Release(hostAndPortTwo.Data); @@ -83,9 +83,9 @@ public async Task Should_handle_service_returning_to_available() new(serviceName, new ServiceHostAndPort("127.0.0.2", 80), string.Empty, string.Empty, Array.Empty()), }; - hostAndPortOne = await _leastConnection.Lease(_httpContext); + hostAndPortOne = await _leastConnection.LeaseAsync(_httpContext); hostAndPortOne.Data.DownstreamHost.ShouldBe("127.0.0.1"); - hostAndPortTwo = await _leastConnection.Lease(_httpContext); + hostAndPortTwo = await _leastConnection.LeaseAsync(_httpContext); hostAndPortTwo.Data.DownstreamHost.ShouldBe("127.0.0.2"); _leastConnection.Release(hostAndPortOne.Data); _leastConnection.Release(hostAndPortTwo.Data); @@ -93,7 +93,7 @@ public async Task Should_handle_service_returning_to_available() private async Task LeaseDelayAndRelease() { - var hostAndPort = await _leastConnection.Lease(_httpContext); + var hostAndPort = await _leastConnection.LeaseAsync(_httpContext); await Task.Delay(_random.Next(1, 100)); _leastConnection.Release(hostAndPort.Data); } @@ -132,15 +132,15 @@ public async Task Should_serve_from_service_with_least_connections() _services = availableServices; _leastConnection = new LeastConnection(() => Task.FromResult(_services), serviceName); - var response = await _leastConnection.Lease(_httpContext); + var response = await _leastConnection.LeaseAsync(_httpContext); response.Data.DownstreamHost.ShouldBe(availableServices[0].HostAndPort.DownstreamHost); - response = await _leastConnection.Lease(_httpContext); + response = await _leastConnection.LeaseAsync(_httpContext); response.Data.DownstreamHost.ShouldBe(availableServices[1].HostAndPort.DownstreamHost); - response = await _leastConnection.Lease(_httpContext); + response = await _leastConnection.LeaseAsync(_httpContext); response.Data.DownstreamHost.ShouldBe(availableServices[2].HostAndPort.DownstreamHost); } @@ -159,19 +159,19 @@ public async Task Should_build_connections_per_service() _services = availableServices; _leastConnection = new LeastConnection(() => Task.FromResult(_services), serviceName); - var response = await _leastConnection.Lease(_httpContext); + var response = await _leastConnection.LeaseAsync(_httpContext); response.Data.DownstreamHost.ShouldBe(availableServices[0].HostAndPort.DownstreamHost); - response = await _leastConnection.Lease(_httpContext); + response = await _leastConnection.LeaseAsync(_httpContext); response.Data.DownstreamHost.ShouldBe(availableServices[1].HostAndPort.DownstreamHost); - response = await _leastConnection.Lease(_httpContext); + response = await _leastConnection.LeaseAsync(_httpContext); response.Data.DownstreamHost.ShouldBe(availableServices[0].HostAndPort.DownstreamHost); - response = await _leastConnection.Lease(_httpContext); + response = await _leastConnection.LeaseAsync(_httpContext); response.Data.DownstreamHost.ShouldBe(availableServices[1].HostAndPort.DownstreamHost); } @@ -190,26 +190,26 @@ public async Task Should_release_connection() _services = availableServices; _leastConnection = new LeastConnection(() => Task.FromResult(_services), serviceName); - var response = await _leastConnection.Lease(_httpContext); + var response = await _leastConnection.LeaseAsync(_httpContext); response.Data.DownstreamHost.ShouldBe(availableServices[0].HostAndPort.DownstreamHost); - response = await _leastConnection.Lease(_httpContext); + response = await _leastConnection.LeaseAsync(_httpContext); response.Data.DownstreamHost.ShouldBe(availableServices[1].HostAndPort.DownstreamHost); - response = await _leastConnection.Lease(_httpContext); + response = await _leastConnection.LeaseAsync(_httpContext); response.Data.DownstreamHost.ShouldBe(availableServices[0].HostAndPort.DownstreamHost); - response = await _leastConnection.Lease(_httpContext); + response = await _leastConnection.LeaseAsync(_httpContext); response.Data.DownstreamHost.ShouldBe(availableServices[1].HostAndPort.DownstreamHost); //release this so 2 should have 1 connection and we should get 2 back as our next host and port _leastConnection.Release(availableServices[1].HostAndPort); - response = await _leastConnection.Lease(_httpContext); + response = await _leastConnection.LeaseAsync(_httpContext); response.Data.DownstreamHost.ShouldBe(availableServices[1].HostAndPort.DownstreamHost); } @@ -260,7 +260,7 @@ private void GivenAHostAndPort(ServiceHostAndPort hostAndPort) private void WhenIGetTheNextHostAndPort() { - _result = _leastConnection.Lease(_httpContext).Result; + _result = _leastConnection.LeaseAsync(_httpContext).Result; } private void ThenTheNextHostAndPortIsReturned() diff --git a/test/Ocelot.UnitTests/LoadBalancer/LoadBalancerFactoryTests.cs b/test/Ocelot.UnitTests/LoadBalancer/LoadBalancerFactoryTests.cs index 616168548..be83ec78c 100644 --- a/test/Ocelot.UnitTests/LoadBalancer/LoadBalancerFactoryTests.cs +++ b/test/Ocelot.UnitTests/LoadBalancer/LoadBalancerFactoryTests.cs @@ -219,54 +219,30 @@ public Response Create(DownstreamRoute route, IServiceDiscoveryPr private class FakeLoadBalancerOne : ILoadBalancer { - public Task> Lease(HttpContext httpContext) - { - throw new NotImplementedException(); - } - - public void Release(ServiceHostAndPort hostAndPort) - { - throw new NotImplementedException(); - } + public string Type => nameof(FakeLoadBalancerOne); + public Task> LeaseAsync(HttpContext httpContext) => throw new NotImplementedException(); + public void Release(ServiceHostAndPort hostAndPort) => throw new NotImplementedException(); } private class FakeLoadBalancerTwo : ILoadBalancer { - public Task> Lease(HttpContext httpContext) - { - throw new NotImplementedException(); - } - - public void Release(ServiceHostAndPort hostAndPort) - { - throw new NotImplementedException(); - } + public string Type => nameof(FakeLoadBalancerTwo); + public Task> LeaseAsync(HttpContext httpContext) => throw new NotImplementedException(); + public void Release(ServiceHostAndPort hostAndPort) => throw new NotImplementedException(); } private class FakeNoLoadBalancer : ILoadBalancer { - public Task> Lease(HttpContext httpContext) - { - throw new NotImplementedException(); - } - - public void Release(ServiceHostAndPort hostAndPort) - { - throw new NotImplementedException(); - } + public string Type => nameof(FakeNoLoadBalancer); + public Task> LeaseAsync(HttpContext httpContext) => throw new NotImplementedException(); + public void Release(ServiceHostAndPort hostAndPort) => throw new NotImplementedException(); } private class BrokenLoadBalancer : ILoadBalancer { - public Task> Lease(HttpContext httpContext) - { - throw new NotImplementedException(); - } - - public void Release(ServiceHostAndPort hostAndPort) - { - throw new NotImplementedException(); - } + public string Type => nameof(BrokenLoadBalancer); + public Task> LeaseAsync(HttpContext httpContext) => throw new NotImplementedException(); + public void Release(ServiceHostAndPort hostAndPort) => throw new NotImplementedException(); } } } diff --git a/test/Ocelot.UnitTests/LoadBalancer/LoadBalancerHouseTests.cs b/test/Ocelot.UnitTests/LoadBalancer/LoadBalancerHouseTests.cs index fa0b835ff..9a426fe77 100644 --- a/test/Ocelot.UnitTests/LoadBalancer/LoadBalancerHouseTests.cs +++ b/test/Ocelot.UnitTests/LoadBalancer/LoadBalancerHouseTests.cs @@ -149,28 +149,16 @@ private void ThenItIsReturned() private class FakeLoadBalancer : ILoadBalancer { - public Task> Lease(HttpContext httpContext) - { - throw new NotImplementedException(); - } - - public void Release(ServiceHostAndPort hostAndPort) - { - throw new NotImplementedException(); - } + public string Type => nameof(FakeLoadBalancer); + public Task> LeaseAsync(HttpContext httpContext) => throw new NotImplementedException(); + public void Release(ServiceHostAndPort hostAndPort) => throw new NotImplementedException(); } private class FakeRoundRobinLoadBalancer : ILoadBalancer { - public Task> Lease(HttpContext httpContext) - { - throw new NotImplementedException(); - } - - public void Release(ServiceHostAndPort hostAndPort) - { - throw new NotImplementedException(); - } + public string Type => nameof(FakeRoundRobinLoadBalancer); + public Task> LeaseAsync(HttpContext httpContext) => throw new NotImplementedException(); + public void Release(ServiceHostAndPort hostAndPort) => throw new NotImplementedException(); } } } diff --git a/test/Ocelot.UnitTests/LoadBalancer/LoadBalancerMiddlewareTests.cs b/test/Ocelot.UnitTests/LoadBalancer/LoadBalancerMiddlewareTests.cs index fd46e9a2a..1a1ab3b02 100644 --- a/test/Ocelot.UnitTests/LoadBalancer/LoadBalancerMiddlewareTests.cs +++ b/test/Ocelot.UnitTests/LoadBalancer/LoadBalancerMiddlewareTests.cs @@ -145,14 +145,14 @@ private void GivenTheLoadBalancerReturnsAnError() { _getHostAndPortError = new ErrorResponse(new List { new ServicesAreNullError("services were null for bah") }); _loadBalancer - .Setup(x => x.Lease(It.IsAny())) + .Setup(x => x.LeaseAsync(It.IsAny())) .ReturnsAsync(_getHostAndPortError); } private void GivenTheLoadBalancerReturnsOk() { _loadBalancer - .Setup(x => x.Lease(It.IsAny())) + .Setup(x => x.LeaseAsync(It.IsAny())) .ReturnsAsync(new OkResponse(new ServiceHostAndPort("abc", 123, "https"))); } @@ -160,7 +160,7 @@ private void GivenTheLoadBalancerReturns() { _hostAndPort = new ServiceHostAndPort("127.0.0.1", 80); _loadBalancer - .Setup(x => x.Lease(It.IsAny())) + .Setup(x => x.LeaseAsync(It.IsAny())) .ReturnsAsync(new OkResponse(_hostAndPort)); } diff --git a/test/Ocelot.UnitTests/LoadBalancer/NoLoadBalancerTests.cs b/test/Ocelot.UnitTests/LoadBalancer/NoLoadBalancerTests.cs index e1490e898..96494bd0b 100644 --- a/test/Ocelot.UnitTests/LoadBalancer/NoLoadBalancerTests.cs +++ b/test/Ocelot.UnitTests/LoadBalancer/NoLoadBalancerTests.cs @@ -85,7 +85,7 @@ private void GivenServices(List services) private void WhenIGetTheNextHostAndPort() { - _result = _loadBalancer.Lease(new DefaultHttpContext()).Result; + _result = _loadBalancer.LeaseAsync(new DefaultHttpContext()).Result; } private void ThenTheHostAndPortIs(ServiceHostAndPort expected) diff --git a/test/Ocelot.UnitTests/LoadBalancer/RoundRobinTests.cs b/test/Ocelot.UnitTests/LoadBalancer/RoundRobinTests.cs index 1eb8e9957..545f04403 100644 --- a/test/Ocelot.UnitTests/LoadBalancer/RoundRobinTests.cs +++ b/test/Ocelot.UnitTests/LoadBalancer/RoundRobinTests.cs @@ -142,9 +142,9 @@ private static RoundRobin GivenLoadBalancer(List services, bool immedia } private Response WhenIGetTheNextAddress(RoundRobin roundRobin) - => roundRobin.Lease(_httpContext).Result; + => roundRobin.LeaseAsync(_httpContext).Result; private Task> WhenIGetTheNextAddressAsync(RoundRobin roundRobin) - => roundRobin.Lease(_httpContext); + => roundRobin.LeaseAsync(_httpContext); private static void ThenServicesAreNullErrorIsReturned(Response response) { From acda395dfe18fb40f9af2aa55488d71f09b0c383 Mon Sep 17 00:00:00 2001 From: Raman Maksimchuk Date: Thu, 3 Oct 2024 23:05:42 +0300 Subject: [PATCH 8/8] Release 23.3.4 artifacts | +semver: patch (#2161) * Release notes * Update packages. Bump all packs except Serilog.* * Remove TestStack.BDDfy from IntegrationTests. The lib is not supported since September 2016 * Fix warnings * CA1866 The char overload is a better performing overload than a string with a single char. * Review docs, build script --- ReleaseNotes.md | 33 ++- build.cake | 2 +- docs/index.rst | 42 +++- .../Ocelot.Samples.OpenTracing.csproj | 3 +- .../Ocelot.Administration.csproj | 10 +- .../Ocelot.Cache.CacheManager.csproj | 10 +- .../Ocelot.Provider.Consul.csproj | 6 +- .../Ocelot.Provider.Eureka.csproj | 4 +- .../Ocelot.Provider.Kubernetes.csproj | 8 +- .../Ocelot.Provider.Polly.csproj | 6 +- .../OcelotResiliencePipelineKey.cs | 3 - .../Ocelot.Tracing.Butterfly.csproj | 4 +- .../Ocelot.Tracing.OpenTracing.csproj | 2 +- .../Validator/RouteFluentValidator.cs | 4 +- .../DependencyInjection/OcelotBuilder.cs | 1 - src/Ocelot/Ocelot.csproj | 18 +- src/Ocelot/RateLimiting/RateLimiting.cs | 15 +- .../Ocelot.AcceptanceTests.csproj | 38 ++-- .../Ocelot.Benchmarks.csproj | 30 +-- .../AdministrationTests.cs | 195 ++++++++---------- .../CacheManagerTests.cs | 13 +- test/Ocelot.IntegrationTests/HeaderTests.cs | 15 +- .../Ocelot.IntegrationTests.csproj | 24 +-- .../ThreadSafeHeadersTests.cs | 11 +- test/Ocelot.IntegrationTests/Usings.cs | 1 - .../Ocelot.ManualTest.csproj | 6 +- test/Ocelot.UnitTests/Ocelot.UnitTests.csproj | 33 +-- ...esiliencePipelineDelegatingHandlerTests.cs | 2 +- 28 files changed, 284 insertions(+), 255 deletions(-) diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 91cbe6cad..4a6e9ebdc 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -1,8 +1,31 @@ -Technical release, version {0} +## 🔥 Hot fixing v[23.3](https://github.com/ThreeMammals/Ocelot/releases/tag/23.3.0) (version {0}) aka [Blue Olympic Balumbes](https://www.youtube.com/live/j-Ou-ggS718?si=fPPwmOwjYEZq70H9&t=9518) release +> Codenamed: **[Blue Olympic Fiend](https://www.youtube.com/live/j-Ou-ggS718?si=fPPwmOwjYEZq70H9&t=9518)** +> Read the Docs: [Ocelot 23.3](https://ocelot.readthedocs.io/en/{0}/) +> Hot fixed versions: [23.3.0](https://github.com/ThreeMammals/Ocelot/releases/tag/23.3.0), [23.3.3](https://github.com/ThreeMammals/Ocelot/releases/tag/23.3.3) +> Milestone: [v23.3 Hotfixes](https://github.com/ThreeMammals/Ocelot/milestone/8) -### Breaking changes +❤️ A heartfelt "Thank You" to [Roman Shevchik](https://github.com/antikorol) and [Massimiliano Innocenti](https://github.com/minnocenti901) for their contributions in testing and reporting the [Service Discovery](https://github.com/ThreeMammals/Ocelot/labels/Service%20Discovery) issues, #2110 and #2119, respectively! -- The `ILoadBalancer` interface: The `Lease` method was renamed to `LeaseAsync`. +### ℹ️ About +This release delivers a number of bug fixes for the predecessor's [23.3.0](https://github.com/ThreeMammals/Ocelot/releases/tag/23.3.0) release, which is full of new features but was not tested well. All bugs were combined into the [v23.3 Hotfixes](https://github.com/ThreeMammals/Ocelot/milestone/8) milestone. + +Following the substantial refactoring of [Service Discovery](https://github.com/ThreeMammals/Ocelot/blob/main/docs/features/servicediscovery.rst) providers in the [23.3.0](https://github.com/ThreeMammals/Ocelot/releases/tag/23.3.0) release, the community identified and we have acknowledged several [critical service discovery defects](https://github.com/ThreeMammals/Ocelot/issues?q=is%3Aissue+milestone%3A%22v23.3+Hotfixes%22+label%3A%22Service+Discovery%22) with providers such as [Kube](https://github.com/ThreeMammals/Ocelot/blob/main/docs/features/kubernetes.rst) and [Consul](https://github.com/ThreeMammals/Ocelot/blob/main/docs/features/servicediscovery.rst#consul). The `Kube` provider, while somewhat unstable, remained operational; however, the `Consul` provider was entirely non-functional. + +📓 If your projects rely on the [Service Discovery](https://ocelot.readthedocs.io/en/latest/features/servicediscovery.html) feature and cannot function without it, please upgrade to this version to utilize the full list of features of version [23.3.0](https://github.com/ThreeMammals/Ocelot/releases/tag/23.3.0). + +### 🧑‍💻 Technical Information +A comprehensive explanation of the technical details would span several pages; therefore, it is advisable for fans of Ocelot to review all pertinent technical information within the issue descriptions associated with [the milestone](https://github.com/ThreeMammals/Ocelot/milestone/8). +Our team has implemented some **Breaking Changes** which we urge you to review carefully (details follow). + +### ⚠️ Breaking Changes +Listed by priority: +- `ILoadBalancer` interface alteration: Method `Lease` is now `LeaseAsync`. Interface FQN: `Ocelot.LoadBalancer.LoadBalancers.ILoadBalancer` - Method FQN: `Ocelot.LoadBalancer.LoadBalancers.ILoadBalancer.LeaseAsync` -- TO BE Written \ No newline at end of file + Method FQN: `Ocelot.LoadBalancer.LoadBalancers.ILoadBalancer.LeaseAsync` +- `DefaultConsulServiceBuilder` constructor modification: The first parameter's type has been changed from `Func` to `IHttpContextAccessor`. + Class FQN: `Ocelot.Provider.Consul.DefaultConsulServiceBuilder` + Constructor signature: `public DefaultConsulServiceBuilder(IHttpContextAccessor contextAccessor, IConsulClientFactory clientFactory, IOcelotLoggerFactory loggerFactory)` +- Adjustments to `Lease` type: The `Lease` has been restructured from a class to a structure and elevated in the namespace hierarchy. + Struct FQN: `Ocelot.LoadBalancer.Lease` + +📓 Should your [custom solutions](https://ocelot.readthedocs.io/en/latest/search.html?q=custom) involve overriding default Ocelot classes and their behavior, redevelopment or at least recompilation of the solution, followed by deployment, will be necessary. diff --git a/build.cake b/build.cake index f37ee080f..e72e7425d 100644 --- a/build.cake +++ b/build.cake @@ -3,7 +3,7 @@ #tool nuget:?package=ReportGenerator&version=5.2.4 #addin nuget:?package=Newtonsoft.Json&version=13.0.3 #addin nuget:?package=System.Text.Encodings.Web&version=8.0.0 -#addin nuget:?package=Cake.Coveralls&version=1.1.0 +#addin nuget:?package=Cake.Coveralls&version=4.0.0 #r "Spectre.Console" using Spectre.Console diff --git a/docs/index.rst b/docs/index.rst index d208ae6d3..562646151 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,9 +12,11 @@ .. _@thiagoloureiro: https://github.com/thiagoloureiro .. _@bbenameur: https://github.com/bbenameur -.. _23.3: https://github.com/ThreeMammals/Ocelot/releases/tag/23.3.0 -.. _23.3.0: https://github.com/ThreeMammals/Ocelot/releases/tag/23.3.0 .. _23.2.0: https://github.com/ThreeMammals/Ocelot/releases/tag/23.2.0 +.. _23.3.0: https://github.com/ThreeMammals/Ocelot/releases/tag/23.3.0 +.. _23.3.3: https://github.com/ThreeMammals/Ocelot/releases/tag/23.3.3 +.. _23.3.4: https://github.com/ThreeMammals/Ocelot/releases/tag/23.3.4 +.. _23.3: https://github.com/ThreeMammals/Ocelot/releases/tag/23.3.4 .. _954: https://github.com/ThreeMammals/Ocelot/issues/954 .. _957: https://github.com/ThreeMammals/Ocelot/issues/957 @@ -53,14 +55,34 @@ The main features are :doc:`../features/configuration` and :doc:`../features/rou We **do** follow development process which is described in :doc:`../building/releaseprocess`. +Patches +------- + +- `23.3.3`_, on Jun 11, 2024. Technical release with DevOps patch. +- `23.3.4`_, on Oct 3, 2024. Hot fixing version `23.3.0`_, codenamed `Blue Olympic Balumbes `_ release. + + :htm:`
Codename decoding links` + + - **for men** :htm:`→` naked `Blue Olympic Fiend `_ + - **for women** :htm:`→` not a well-dressed woman sings at the opening ceremony, so "Not `Celine Dion `_" + - **for black men** :htm:`→` don't care about White movements, so enjoy `Black Men's Basketball Final `_ in `Paris 2024 `_: + be proud of Stephen Curry, "just give me a ball" boy, as an absolute rockstar, made `shot 1 `_, `shot 2 `_, `shot 3 `_ and final `shot 4 `_. + + :htm:`
` + Release Notes ------------- | Release Tag: `23.3.0`_ -| Release Codename: **Twilight Texas** - :htm:`→` `for men `_ - :htm:`→` `for women `_ - :htm:`→` `for black men `_ +| Release Codename: `Twilight Texas `_ + + :htm:`
Codename decoding links` + + - `for men `_ + - `for women `_ + - `for black men `_ + + :htm:`
` What's new? ^^^^^^^^^^^ @@ -133,8 +155,8 @@ Ocelot extra packages If both `Circuit Breaker`_ and `Timeout`_ have :ref:`qos-configuration` with their respective properties in the ``QoSOptions`` of the route JSON, then the :ref:`qos-circuit-breaker-strategy` will take precedence in the constructed resilience pipeline. For more details, refer to PR `2086`_. -Stabilization aka bug fixing -"""""""""""""""""""""""""""" +Stabilization (bug fixing) +^^^^^^^^^^^^^^^^^^^^^^^^^^ - Fixed `2034`_ in PR `2045`_ by `@raman-m`_ - Fixed `2039`_ in PR `2050`_ by `@PaulARoy`_ @@ -146,8 +168,8 @@ Stabilization aka bug fixing See `all bugs `_ of the `Spring'24 `_ milestone -Documentation for version `23.3`_ -""""""""""""""""""""""""""""""""" +Documentation Summary +^^^^^^^^^^^^^^^^^^^^^ - :doc:`../features/caching`: New :ref:`cch-enablecontenthashing-option` and :ref:`cch-global-configuration` sections - :doc:`../features/configuration`: New :ref:`config-version-policy` and :ref:`config-route-metadata` sections diff --git a/samples/OpenTracing/Ocelot.Samples.OpenTracing.csproj b/samples/OpenTracing/Ocelot.Samples.OpenTracing.csproj index a05368868..a6733a358 100644 --- a/samples/OpenTracing/Ocelot.Samples.OpenTracing.csproj +++ b/samples/OpenTracing/Ocelot.Samples.OpenTracing.csproj @@ -7,7 +7,6 @@ - @@ -23,7 +22,7 @@ - + diff --git a/src/Ocelot.Administration/Ocelot.Administration.csproj b/src/Ocelot.Administration/Ocelot.Administration.csproj index 4bb397ab3..e2b8ed463 100644 --- a/src/Ocelot.Administration/Ocelot.Administration.csproj +++ b/src/Ocelot.Administration/Ocelot.Administration.csproj @@ -31,28 +31,28 @@ - + all - + - + - + - + diff --git a/src/Ocelot.Cache.CacheManager/Ocelot.Cache.CacheManager.csproj b/src/Ocelot.Cache.CacheManager/Ocelot.Cache.CacheManager.csproj index 4636540bc..0e8d704ee 100644 --- a/src/Ocelot.Cache.CacheManager/Ocelot.Cache.CacheManager.csproj +++ b/src/Ocelot.Cache.CacheManager/Ocelot.Cache.CacheManager.csproj @@ -31,7 +31,7 @@ - + all @@ -39,7 +39,7 @@ - + @@ -47,12 +47,12 @@ - + - + @@ -60,7 +60,7 @@ - + diff --git a/src/Ocelot.Provider.Consul/Ocelot.Provider.Consul.csproj b/src/Ocelot.Provider.Consul/Ocelot.Provider.Consul.csproj index f8149b859..ce4f544bf 100644 --- a/src/Ocelot.Provider.Consul/Ocelot.Provider.Consul.csproj +++ b/src/Ocelot.Provider.Consul/Ocelot.Provider.Consul.csproj @@ -30,12 +30,12 @@ - - + + all - + diff --git a/src/Ocelot.Provider.Eureka/Ocelot.Provider.Eureka.csproj b/src/Ocelot.Provider.Eureka/Ocelot.Provider.Eureka.csproj index 826126199..cf9955a40 100644 --- a/src/Ocelot.Provider.Eureka/Ocelot.Provider.Eureka.csproj +++ b/src/Ocelot.Provider.Eureka/Ocelot.Provider.Eureka.csproj @@ -33,11 +33,11 @@ - + all - + diff --git a/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj b/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj index f45dd07dd..4941194e9 100644 --- a/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj +++ b/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj @@ -29,9 +29,9 @@ - - - + + + all @@ -39,6 +39,6 @@ - + diff --git a/src/Ocelot.Provider.Polly/Ocelot.Provider.Polly.csproj b/src/Ocelot.Provider.Polly/Ocelot.Provider.Polly.csproj index 0d9c84e67..f39ebd8bd 100644 --- a/src/Ocelot.Provider.Polly/Ocelot.Provider.Polly.csproj +++ b/src/Ocelot.Provider.Polly/Ocelot.Provider.Polly.csproj @@ -31,12 +31,12 @@ - + all - + - + diff --git a/src/Ocelot.Provider.Polly/OcelotResiliencePipelineKey.cs b/src/Ocelot.Provider.Polly/OcelotResiliencePipelineKey.cs index 528df61ee..0d4a02c71 100644 --- a/src/Ocelot.Provider.Polly/OcelotResiliencePipelineKey.cs +++ b/src/Ocelot.Provider.Polly/OcelotResiliencePipelineKey.cs @@ -5,8 +5,5 @@ namespace Ocelot.Provider.Polly; /// /// Object used to identify a resilience pipeline in . /// -/// -/// Object used to identify a resilience pipeline in -/// /// The key for the resilience pipeline. public record OcelotResiliencePipelineKey(string Key); diff --git a/src/Ocelot.Tracing.Butterfly/Ocelot.Tracing.Butterfly.csproj b/src/Ocelot.Tracing.Butterfly/Ocelot.Tracing.Butterfly.csproj index 68b90652f..03a57aec8 100644 --- a/src/Ocelot.Tracing.Butterfly/Ocelot.Tracing.Butterfly.csproj +++ b/src/Ocelot.Tracing.Butterfly/Ocelot.Tracing.Butterfly.csproj @@ -33,11 +33,11 @@ - + all - + diff --git a/src/Ocelot.Tracing.OpenTracing/Ocelot.Tracing.OpenTracing.csproj b/src/Ocelot.Tracing.OpenTracing/Ocelot.Tracing.OpenTracing.csproj index 635e3c77d..14ee22ae9 100644 --- a/src/Ocelot.Tracing.OpenTracing/Ocelot.Tracing.OpenTracing.csproj +++ b/src/Ocelot.Tracing.OpenTracing/Ocelot.Tracing.OpenTracing.csproj @@ -20,7 +20,7 @@ - + all diff --git a/src/Ocelot/Configuration/Validator/RouteFluentValidator.cs b/src/Ocelot/Configuration/Validator/RouteFluentValidator.cs index 900c53189..fbcbd57d2 100644 --- a/src/Ocelot/Configuration/Validator/RouteFluentValidator.cs +++ b/src/Ocelot/Configuration/Validator/RouteFluentValidator.cs @@ -27,7 +27,7 @@ public RouteFluentValidator(IAuthenticationSchemeProvider authenticationSchemePr When(route => !string.IsNullOrEmpty(route.DownstreamPathTemplate), () => { RuleFor(route => route.DownstreamPathTemplate) - .Must(path => path.StartsWith("/")) + .Must(path => path.StartsWith('/')) .WithMessage("{PropertyName} {PropertyValue} doesnt start with forward slash"); RuleFor(route => route.DownstreamPathTemplate) @@ -46,7 +46,7 @@ public RouteFluentValidator(IAuthenticationSchemeProvider authenticationSchemePr .WithMessage("{PropertyName} {PropertyValue} contains double forward slash, Ocelot does not support this at the moment. Please raise an issue in GitHib if you need this feature."); RuleFor(route => route.UpstreamPathTemplate) - .Must(path => path.StartsWith("/")) + .Must(path => path.StartsWith('/')) .WithMessage("{PropertyName} {PropertyValue} doesnt start with forward slash"); RuleFor(route => route.UpstreamPathTemplate) diff --git a/src/Ocelot/DependencyInjection/OcelotBuilder.cs b/src/Ocelot/DependencyInjection/OcelotBuilder.cs index 146ba3c65..1e21081ad 100644 --- a/src/Ocelot/DependencyInjection/OcelotBuilder.cs +++ b/src/Ocelot/DependencyInjection/OcelotBuilder.cs @@ -313,7 +313,6 @@ public IOcelotBuilder AddConfigPlaceholders() return this; } - /// For local implementation purposes, so it MUST NOT be public!.. private IServiceProvider _serviceProvider; // TODO Reuse ActivatorUtilities factories? diff --git a/src/Ocelot/Ocelot.csproj b/src/Ocelot/Ocelot.csproj index b876ca4b7..2171a5f62 100644 --- a/src/Ocelot/Ocelot.csproj +++ b/src/Ocelot/Ocelot.csproj @@ -29,32 +29,32 @@ - + NU1701 - + all - + - - + + - - + + - - + + diff --git a/src/Ocelot/RateLimiting/RateLimiting.cs b/src/Ocelot/RateLimiting/RateLimiting.cs index 9edf4a310..c62e95382 100644 --- a/src/Ocelot/RateLimiting/RateLimiting.cs +++ b/src/Ocelot/RateLimiting/RateLimiting.cs @@ -65,15 +65,18 @@ public virtual RateLimitCounter ProcessRequest(ClientRequestIdentity identity, R public virtual RateLimitCounter Count(RateLimitCounter? entry, RateLimitRule rule) { var now = DateTime.UtcNow; - if (!entry.HasValue) // no entry, start counting + if (!entry.HasValue) { + // no entry, start counting return new RateLimitCounter(now, null, 1); // current request is the 1st one } var counter = entry.Value; var total = counter.TotalRequests + 1; // increment request count var startedAt = counter.StartedAt; - if (startedAt + ToTimespan(rule.Period) >= now) // counting Period is active + + // Counting Period is active + if (startedAt + ToTimespan(rule.Period) >= now) { var exceededAt = total >= rule.Limit && !counter.ExceededAt.HasValue // current request number equals to the limit ? now // the exceeding moment is now, the next request will fail but the current one doesn't @@ -144,7 +147,9 @@ public virtual double RetryAfter(RateLimitCounter counter, RateLimitRule rule) ? defaultSeconds // allow values which are greater or equal to 1 second : rule.PeriodTimespan; // good value var now = DateTime.UtcNow; - if (counter.StartedAt + ToTimespan(rule.Period) >= now) // counting Period is active + + // Counting Period is active + if (counter.StartedAt + ToTimespan(rule.Period) >= now) { return counter.TotalRequests < rule.Limit ? 0.0D // happy path, no need to retry, current request is valid @@ -153,8 +158,8 @@ public virtual double RetryAfter(RateLimitCounter counter, RateLimitRule rule) : periodTimespan; // exceeding not yet detected -> let's ban for whole period } - if (counter.ExceededAt.HasValue && // limit exceeding was happen - counter.ExceededAt + TimeSpan.FromSeconds(periodTimespan) >= now) // ban PeriodTimespan is active + // Limit exceeding was happen && ban PeriodTimespan is active + if (counter.ExceededAt.HasValue && counter.ExceededAt + TimeSpan.FromSeconds(periodTimespan) >= now) { var startedAt = counter.ExceededAt.Value; // ban period was started at double secondsPast = (now - startedAt).TotalSeconds; diff --git a/test/Ocelot.AcceptanceTests/Ocelot.AcceptanceTests.csproj b/test/Ocelot.AcceptanceTests/Ocelot.AcceptanceTests.csproj index c2b3e01bf..968fd5e79 100644 --- a/test/Ocelot.AcceptanceTests/Ocelot.AcceptanceTests.csproj +++ b/test/Ocelot.AcceptanceTests/Ocelot.AcceptanceTests.csproj @@ -45,36 +45,36 @@ - - - - + + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - - - - all - + - + - + + - + + + all + - + @@ -87,7 +87,7 @@ - + @@ -100,18 +100,18 @@ - + - + - + - + diff --git a/test/Ocelot.Benchmarks/Ocelot.Benchmarks.csproj b/test/Ocelot.Benchmarks/Ocelot.Benchmarks.csproj index 288d79019..eb906e816 100644 --- a/test/Ocelot.Benchmarks/Ocelot.Benchmarks.csproj +++ b/test/Ocelot.Benchmarks/Ocelot.Benchmarks.csproj @@ -21,24 +21,24 @@ - - + + all - - - - - - - - - - - - + + + + + + + + + + + + - + diff --git a/test/Ocelot.IntegrationTests/AdministrationTests.cs b/test/Ocelot.IntegrationTests/AdministrationTests.cs index 5ae8d0819..d9222c539 100644 --- a/test/Ocelot.IntegrationTests/AdministrationTests.cs +++ b/test/Ocelot.IntegrationTests/AdministrationTests.cs @@ -48,12 +48,10 @@ public AdministrationTests() public void Should_return_response_401_with_call_re_routes_controller() { var configuration = new FileConfiguration(); - - this.Given(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenOcelotIsRunning()) - .When(x => WhenIGetUrlOnTheApiGateway("/administration/configuration")) - .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.Unauthorized)) - .BDDfy(); + GivenThereIsAConfiguration(configuration); + GivenOcelotIsRunning(); + WhenIGetUrlOnTheApiGateway("/administration/configuration"); + ThenTheStatusCodeShouldBe(HttpStatusCode.Unauthorized); } //this seems to be be answer https://github.com/IdentityServer/IdentityServer4/issues/4914 @@ -61,14 +59,12 @@ public void Should_return_response_401_with_call_re_routes_controller() public void Should_return_response_200_with_call_re_routes_controller() { var configuration = new FileConfiguration(); - - this.Given(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenOcelotIsRunning()) - .And(x => GivenIHaveAnOcelotToken("/administration")) - .And(x => GivenIHaveAddedATokenToMyRequest()) - .When(x => WhenIGetUrlOnTheApiGateway("/administration/configuration")) - .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) - .BDDfy(); + GivenThereIsAConfiguration(configuration); + GivenOcelotIsRunning(); + GivenIHaveAnOcelotToken("/administration"); + GivenIHaveAddedATokenToMyRequest(); + WhenIGetUrlOnTheApiGateway("/administration/configuration"); + ThenTheStatusCodeShouldBe(HttpStatusCode.OK); } [Fact] @@ -87,13 +83,12 @@ public void Should_return_response_200_with_call_re_routes_controller_using_base }, }; - this.Given(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenOcelotIsRunningWithNoWebHostBuilder(_ocelotBaseUrl)) - .And(x => GivenIHaveAnOcelotToken("/administration")) - .And(x => GivenIHaveAddedATokenToMyRequest()) - .When(x => WhenIGetUrlOnTheApiGateway("/administration/configuration")) - .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) - .BDDfy(); + GivenThereIsAConfiguration(configuration); + GivenOcelotIsRunningWithNoWebHostBuilder(_ocelotBaseUrl); + GivenIHaveAnOcelotToken("/administration"); + GivenIHaveAddedATokenToMyRequest(); + WhenIGetUrlOnTheApiGateway("/administration/configuration"); + ThenTheStatusCodeShouldBe(HttpStatusCode.OK); } [Fact] @@ -109,14 +104,13 @@ public void Should_return_OK_status_and_multiline_indented_json_response_with_js .AddJsonOptions(options => { options.JsonSerializerOptions.WriteIndented = true; }); }; - this.Given(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenOcelotUsingBuilderIsRunning(customBuilder)) - .And(x => GivenIHaveAnOcelotToken("/administration")) - .And(x => GivenIHaveAddedATokenToMyRequest()) - .When(x => WhenIGetUrlOnTheApiGateway("/administration/configuration")) - .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) - .Then(x => ThenTheResultHaveMultiLineIndentedJson()) - .BDDfy(); + GivenThereIsAConfiguration(configuration); + GivenOcelotUsingBuilderIsRunning(customBuilder); + GivenIHaveAnOcelotToken("/administration"); + GivenIHaveAddedATokenToMyRequest(); + WhenIGetUrlOnTheApiGateway("/administration/configuration"); + ThenTheStatusCodeShouldBe(HttpStatusCode.OK); + ThenTheResultHaveMultiLineIndentedJson(); } [Fact] @@ -124,15 +118,13 @@ public void Should_be_able_to_use_token_from_ocelot_a_on_ocelot_b() { var configuration = new FileConfiguration(); var port = PortFinder.GetRandomPort(); - - this.Given(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenIdentityServerSigningEnvironmentalVariablesAreSet()) - .And(x => GivenOcelotIsRunning()) - .And(x => GivenIHaveAnOcelotToken("/administration")) - .And(x => GivenAnotherOcelotIsRunning($"http://localhost:{port}")) - .When(x => WhenIGetUrlOnTheSecondOcelot("/administration/configuration")) - .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) - .BDDfy(); + GivenThereIsAConfiguration(configuration); + GivenIdentityServerSigningEnvironmentalVariablesAreSet(); + GivenOcelotIsRunning(); + GivenIHaveAnOcelotToken("/administration"); + GivenAnotherOcelotIsRunning($"http://localhost:{port}"); + WhenIGetUrlOnTheSecondOcelot("/administration/configuration"); + ThenTheStatusCodeShouldBe(HttpStatusCode.OK); } [Fact] @@ -194,14 +186,13 @@ public void Should_return_file_configuration() }, }; - this.Given(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenOcelotIsRunning()) - .And(x => GivenIHaveAnOcelotToken("/administration")) - .And(x => GivenIHaveAddedATokenToMyRequest()) - .When(x => WhenIGetUrlOnTheApiGateway("/administration/configuration")) - .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) - .And(x => ThenTheResponseShouldBe(configuration)) - .BDDfy(); + GivenThereIsAConfiguration(configuration); + GivenOcelotIsRunning(); + GivenIHaveAnOcelotToken("/administration"); + GivenIHaveAddedATokenToMyRequest(); + WhenIGetUrlOnTheApiGateway("/administration/configuration"); + ThenTheStatusCodeShouldBe(HttpStatusCode.OK); + ThenTheResponseShouldBe(configuration); } [Fact] @@ -283,18 +274,17 @@ public void Should_get_file_configuration_edit_and_post_updated_version() }, }; - this.Given(x => GivenThereIsAConfiguration(initialConfiguration)) - .And(x => GivenOcelotIsRunning()) - .And(x => GivenIHaveAnOcelotToken("/administration")) - .And(x => GivenIHaveAddedATokenToMyRequest()) - .When(x => WhenIGetUrlOnTheApiGateway("/administration/configuration")) - .When(x => WhenIPostOnTheApiGateway("/administration/configuration", updatedConfiguration)) - .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) - .And(x => ThenTheResponseShouldBe(updatedConfiguration)) - .When(x => WhenIGetUrlOnTheApiGateway("/administration/configuration")) - .And(x => ThenTheResponseShouldBe(updatedConfiguration)) - .And(_ => ThenTheConfigurationIsSavedCorrectly(updatedConfiguration)) - .BDDfy(); + GivenThereIsAConfiguration(initialConfiguration); + GivenOcelotIsRunning(); + GivenIHaveAnOcelotToken("/administration"); + GivenIHaveAddedATokenToMyRequest(); + WhenIGetUrlOnTheApiGateway("/administration/configuration"); + WhenIPostOnTheApiGateway("/administration/configuration", updatedConfiguration); + ThenTheStatusCodeShouldBe(HttpStatusCode.OK); + ThenTheResponseShouldBe(updatedConfiguration); + WhenIGetUrlOnTheApiGateway("/administration/configuration"); + ThenTheResponseShouldBe(updatedConfiguration); + ThenTheConfigurationIsSavedCorrectly(updatedConfiguration); } [Fact] @@ -323,18 +313,17 @@ public void Should_activate_change_token_when_configuration_is_updated() }, }; - this.Given(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenOcelotIsRunning()) - .And(x => GivenIHaveAnOcelotToken("/administration")) - .And(x => GivenIHaveAddedATokenToMyRequest()) - .When(x => WhenIPostOnTheApiGateway("/administration/configuration", configuration)) - .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) - .And(x => TheChangeTokenShouldBeActive()) - .And(x => ThenTheResponseShouldBe(configuration)) - .When(x => WhenIGetUrlOnTheApiGateway("/administration/configuration")) - .And(x => ThenTheResponseShouldBe(configuration)) - .And(_ => ThenTheConfigurationIsSavedCorrectly(configuration)) - .BDDfy(); + GivenThereIsAConfiguration(configuration); + GivenOcelotIsRunning(); + GivenIHaveAnOcelotToken("/administration"); + GivenIHaveAddedATokenToMyRequest(); + WhenIPostOnTheApiGateway("/administration/configuration", configuration); + ThenTheStatusCodeShouldBe(HttpStatusCode.OK); + TheChangeTokenShouldBeActive(); + ThenTheResponseShouldBe(configuration); + WhenIGetUrlOnTheApiGateway("/administration/configuration"); + ThenTheResponseShouldBe(configuration); + ThenTheConfigurationIsSavedCorrectly(configuration); } private void TheChangeTokenShouldBeActive() @@ -406,25 +395,24 @@ public void Should_get_file_configuration_edit_and_post_updated_version_redirect }, }; - this.Given(x => GivenThereIsAConfiguration(initialConfiguration)) - .And(x => GivenThereIsAFooServiceRunningOn($"http://localhost:{fooPort}")) - .And(x => GivenThereIsABarServiceRunningOn($"http://localhost:{barPort}")) - .And(x => GivenOcelotIsRunning()) - .And(x => WhenIGetUrlOnTheApiGateway("/foo")) - .Then(x => ThenTheResponseBodyShouldBe("foo")) - .And(x => GivenIHaveAnOcelotToken("/administration")) - .And(x => GivenIHaveAddedATokenToMyRequest()) - .When(x => WhenIPostOnTheApiGateway("/administration/configuration", updatedConfiguration)) - .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) - .And(x => ThenTheResponseShouldBe(updatedConfiguration)) - .And(x => WhenIGetUrlOnTheApiGateway("/foo")) - .Then(x => ThenTheResponseBodyShouldBe("bar")) - .When(x => WhenIPostOnTheApiGateway("/administration/configuration", initialConfiguration)) - .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) - .And(x => ThenTheResponseShouldBe(initialConfiguration)) - .And(x => WhenIGetUrlOnTheApiGateway("/foo")) - .Then(x => ThenTheResponseBodyShouldBe("foo")) - .BDDfy(); + GivenThereIsAConfiguration(initialConfiguration); + GivenThereIsAFooServiceRunningOn($"http://localhost:{fooPort}"); + GivenThereIsABarServiceRunningOn($"http://localhost:{barPort}"); + GivenOcelotIsRunning(); + WhenIGetUrlOnTheApiGateway("/foo"); + ThenTheResponseBodyShouldBe("foo"); + GivenIHaveAnOcelotToken("/administration"); + GivenIHaveAddedATokenToMyRequest(); + WhenIPostOnTheApiGateway("/administration/configuration", updatedConfiguration); + ThenTheStatusCodeShouldBe(HttpStatusCode.OK); + ThenTheResponseShouldBe(updatedConfiguration); + WhenIGetUrlOnTheApiGateway("/foo"); + ThenTheResponseBodyShouldBe("bar"); + WhenIPostOnTheApiGateway("/administration/configuration", initialConfiguration); + ThenTheStatusCodeShouldBe(HttpStatusCode.OK); + ThenTheResponseShouldBe(initialConfiguration); + WhenIGetUrlOnTheApiGateway("/foo"); + ThenTheResponseBodyShouldBe("foo"); } [Fact] @@ -477,14 +465,12 @@ public void Should_clear_region() }; var regionToClear = "gettest"; - - this.Given(x => GivenThereIsAConfiguration(initialConfiguration)) - .And(x => GivenOcelotIsRunning()) - .And(x => GivenIHaveAnOcelotToken("/administration")) - .And(x => GivenIHaveAddedATokenToMyRequest()) - .When(x => WhenIDeleteOnTheApiGateway($"/administration/outputcache/{regionToClear}")) - .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.NoContent)) - .BDDfy(); + GivenThereIsAConfiguration(initialConfiguration); + GivenOcelotIsRunning(); + GivenIHaveAnOcelotToken("/administration"); + GivenIHaveAddedATokenToMyRequest(); + WhenIDeleteOnTheApiGateway($"/administration/outputcache/{regionToClear}"); + ThenTheStatusCodeShouldBe(HttpStatusCode.NoContent); } [Fact] @@ -505,14 +491,13 @@ public void Should_return_response_200_with_call_re_routes_controller_when_using }; }; - this.Given(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenThereIsAnIdentityServerOn(identityServerRootUrl, "api")) - .And(x => GivenOcelotIsRunningWithIdentityServerSettings(options)) - .And(x => GivenIHaveAToken(identityServerRootUrl)) - .And(x => GivenIHaveAddedATokenToMyRequest()) - .When(x => WhenIGetUrlOnTheApiGateway("/administration/configuration")) - .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) - .BDDfy(); + GivenThereIsAConfiguration(configuration); + GivenThereIsAnIdentityServerOn(identityServerRootUrl, "api"); + GivenOcelotIsRunningWithIdentityServerSettings(options); + GivenIHaveAToken(identityServerRootUrl); + GivenIHaveAddedATokenToMyRequest(); + WhenIGetUrlOnTheApiGateway("/administration/configuration"); + ThenTheStatusCodeShouldBe(HttpStatusCode.OK); } private void GivenIHaveAToken(string url) diff --git a/test/Ocelot.IntegrationTests/CacheManagerTests.cs b/test/Ocelot.IntegrationTests/CacheManagerTests.cs index 6af698854..e0a71672b 100644 --- a/test/Ocelot.IntegrationTests/CacheManagerTests.cs +++ b/test/Ocelot.IntegrationTests/CacheManagerTests.cs @@ -83,13 +83,12 @@ public void should_clear_region() var regionToClear = "gettest"; - this.Given(x => GivenThereIsAConfiguration(initialConfiguration)) - .And(x => GivenOcelotIsRunning()) - .And(x => GivenIHaveAnOcelotToken("/administration")) - .And(x => GivenIHaveAddedATokenToMyRequest()) - .When(x => WhenIDeleteOnTheApiGateway($"/administration/outputcache/{regionToClear}")) - .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.NoContent)) - .BDDfy(); + GivenThereIsAConfiguration(initialConfiguration); + GivenOcelotIsRunning(); + GivenIHaveAnOcelotToken("/administration"); + GivenIHaveAddedATokenToMyRequest(); + WhenIDeleteOnTheApiGateway($"/administration/outputcache/{regionToClear}"); + ThenTheStatusCodeShouldBe(HttpStatusCode.NoContent); } private void GivenIHaveAddedATokenToMyRequest() diff --git a/test/Ocelot.IntegrationTests/HeaderTests.cs b/test/Ocelot.IntegrationTests/HeaderTests.cs index 9072f3189..acef644b4 100644 --- a/test/Ocelot.IntegrationTests/HeaderTests.cs +++ b/test/Ocelot.IntegrationTests/HeaderTests.cs @@ -30,7 +30,7 @@ public HeaderTests() } [Fact] - public void Should_pass_remote_ip_address_if_as_x_forwarded_for_header() + public async Task Should_pass_remote_ip_address_if_as_x_forwarded_for_header() { var port = PortFinder.GetRandomPort(); var configuration = new FileConfiguration @@ -63,13 +63,12 @@ public void Should_pass_remote_ip_address_if_as_x_forwarded_for_header() }, }; - this.Given(x => GivenThereIsAServiceRunningOn($"http://localhost:{port}", 200, "X-Forwarded-For")) - .And(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenOcelotIsRunning()) - .When(x => WhenIGetUrlOnTheApiGateway("/")) - .Then(x => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) - .And(x => ThenXForwardedForIsSet()) - .BDDfy(); + GivenThereIsAServiceRunningOn($"http://localhost:{port}", 200, "X-Forwarded-For"); + GivenThereIsAConfiguration(configuration); + GivenOcelotIsRunning(); + await WhenIGetUrlOnTheApiGateway("/"); + ThenTheStatusCodeShouldBe(HttpStatusCode.OK); + ThenXForwardedForIsSet(); } private void GivenThereIsAServiceRunningOn(string url, int statusCode, string headerKey) diff --git a/test/Ocelot.IntegrationTests/Ocelot.IntegrationTests.csproj b/test/Ocelot.IntegrationTests/Ocelot.IntegrationTests.csproj index 857f2c0b5..0b5f20d0e 100644 --- a/test/Ocelot.IntegrationTests/Ocelot.IntegrationTests.csproj +++ b/test/Ocelot.IntegrationTests/Ocelot.IntegrationTests.csproj @@ -36,27 +36,27 @@ - - - + + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + all - - + - + @@ -67,7 +67,7 @@ - + @@ -78,9 +78,9 @@ - + - + @@ -88,6 +88,6 @@ - + diff --git a/test/Ocelot.IntegrationTests/ThreadSafeHeadersTests.cs b/test/Ocelot.IntegrationTests/ThreadSafeHeadersTests.cs index 48eac3686..8e2fc8a51 100644 --- a/test/Ocelot.IntegrationTests/ThreadSafeHeadersTests.cs +++ b/test/Ocelot.IntegrationTests/ThreadSafeHeadersTests.cs @@ -55,12 +55,11 @@ public void Should_return_same_response_for_each_different_header_under_load_to_ }, }; - this.Given(x => GivenThereIsAConfiguration(configuration)) - .And(x => GivenThereIsAServiceRunningOn($"http://localhost:{port}")) - .And(x => GivenOcelotIsRunning()) - .When(x => WhenIGetUrlOnTheApiGatewayMultipleTimesWithDifferentHeaderValues("/", 300)) - .Then(x => ThenTheSameHeaderValuesAreReturnedByTheDownstreamService()) - .BDDfy(); + GivenThereIsAConfiguration(configuration); + GivenThereIsAServiceRunningOn($"http://localhost:{port}"); + GivenOcelotIsRunning(); + WhenIGetUrlOnTheApiGatewayMultipleTimesWithDifferentHeaderValues("/", 300); + ThenTheSameHeaderValuesAreReturnedByTheDownstreamService(); } private void GivenThereIsAServiceRunningOn(string url) diff --git a/test/Ocelot.IntegrationTests/Usings.cs b/test/Ocelot.IntegrationTests/Usings.cs index 504bb7314..5f3d6a446 100644 --- a/test/Ocelot.IntegrationTests/Usings.cs +++ b/test/Ocelot.IntegrationTests/Usings.cs @@ -11,5 +11,4 @@ global using Ocelot; global using Ocelot.Testing; global using Shouldly; -global using TestStack.BDDfy; global using Xunit; diff --git a/test/Ocelot.ManualTest/Ocelot.ManualTest.csproj b/test/Ocelot.ManualTest/Ocelot.ManualTest.csproj index 919f6b913..45b21c6a5 100644 --- a/test/Ocelot.ManualTest/Ocelot.ManualTest.csproj +++ b/test/Ocelot.ManualTest/Ocelot.ManualTest.csproj @@ -32,7 +32,7 @@ - + all @@ -59,7 +59,7 @@ - + @@ -67,6 +67,6 @@ - + diff --git a/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj b/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj index bb3a32301..2ba5a4959 100644 --- a/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj +++ b/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj @@ -49,38 +49,38 @@ - + - - - + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + all - + - + - + - - + + - + @@ -92,7 +92,7 @@ - + @@ -104,14 +104,17 @@ - + - + + + + diff --git a/test/Ocelot.UnitTests/Polly/PollyResiliencePipelineDelegatingHandlerTests.cs b/test/Ocelot.UnitTests/Polly/PollyResiliencePipelineDelegatingHandlerTests.cs index f144fe305..ec0da4286 100644 --- a/test/Ocelot.UnitTests/Polly/PollyResiliencePipelineDelegatingHandlerTests.cs +++ b/test/Ocelot.UnitTests/Polly/PollyResiliencePipelineDelegatingHandlerTests.cs @@ -33,7 +33,7 @@ public PollyResiliencePipelineDelegatingHandlerTests() } [Fact] - public async void SendAsync_OnePolicy() + public async Task SendAsync_OnePolicy() { // Arrange var fakeResponse = new HttpResponseMessage(HttpStatusCode.NoContent);