[tools] Stress test improvements (#5381)

open-telemetry · Feb 23, 2024 · 73b6e30 · 73b6e30
1 parent 7e0213d
commit 73b6e30
Show file tree

Hide file tree

Showing 16 changed files with 508 additions and 300 deletions.
diff --git a/src/OpenTelemetry/AssemblyInfo.cs b/src/OpenTelemetry/AssemblyInfo.cs
@@ -16,6 +16,7 @@
 [assembly: InternalsVisibleTo("OpenTelemetry.Exporter.Console" + AssemblyInfo.PublicKey)]
 [assembly: InternalsVisibleTo("OpenTelemetry.Exporter.OpenTelemetryProtocol" + AssemblyInfo.PublicKey)]
 [assembly: InternalsVisibleTo("OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests" + AssemblyInfo.PublicKey)]
+[assembly: InternalsVisibleTo("OpenTelemetry.Tests.Stress.Metrics" + AssemblyInfo.PublicKey)]
 #endif
 
 #if SIGNED

diff --git a/test/OpenTelemetry.Tests.Stress.Logs/OpenTelemetry.Tests.Stress.Logs.csproj b/test/OpenTelemetry.Tests.Stress.Logs/OpenTelemetry.Tests.Stress.Logs.csproj
@@ -3,21 +3,13 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFrameworks>$(TargetFrameworksForTests)</TargetFrameworks>
-    <!-- this is temporary. will remove in future PR. -->
-    <Nullable>disable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
-    <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry.Exporter.Prometheus.HttpListener\OpenTelemetry.Exporter.Prometheus.HttpListener.csproj" />
+    <ProjectReference Include="$(RepoRoot)\test\OpenTelemetry.Tests.Stress\OpenTelemetry.Tests.Stress.csproj" />
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="OpenTelemetry.Instrumentation.Runtime" />
-    <PackageReference Include="System.Runtime.InteropServices.RuntimeInformation" Condition="'$(TargetFramework)' == '$(NetFrameworkMinimumSupportedVersion)'" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <Compile Include="$(RepoRoot)\test\OpenTelemetry.Tests.Stress\Skeleton.cs" Link="Includes\Skeleton.cs" />
     <Compile Include="$(RepoRoot)\test\OpenTelemetry.Tests\Shared\Utils.cs" Link="Includes\Utils.cs" />
   </ItemGroup>
 

diff --git a/test/OpenTelemetry.Tests.Stress.Logs/Program.cs b/test/OpenTelemetry.Tests.Stress.Logs/Program.cs
@@ -1,39 +1,55 @@
 // Copyright The OpenTelemetry Authors
 // SPDX-License-Identifier: Apache-2.0
 
-using System.Runtime.CompilerServices;
 using Microsoft.Extensions.Logging;
 
 namespace OpenTelemetry.Tests.Stress;
 
-public partial class Program
+public static class Program
 {
-    private static ILogger logger;
-    private static Payload payload = new Payload();
+    public static int Main(string[] args)
+    {
+        return StressTestFactory.RunSynchronously<LogsStressTest>(args);
+    }
 
-    public static void Main()
+    private sealed class LogsStressTest : StressTest<StressTestOptions>
     {
-        using var loggerFactory = LoggerFactory.Create(builder =>
+        private static readonly Payload Payload = new();
+        private readonly ILoggerFactory loggerFactory;
+        private readonly ILogger logger;
+
+        public LogsStressTest(StressTestOptions options)
+            : base(options)
         {
-            builder.AddOpenTelemetry(options =>
+            this.loggerFactory = LoggerFactory.Create(builder =>
             {
-                options.AddProcessor(new DummyProcessor());
+                builder.AddOpenTelemetry(options =>
+                {
+                    options.AddProcessor(new DummyProcessor());
+                });
             });
-        });
 
-        logger = loggerFactory.CreateLogger<Program>();
+            this.logger = this.loggerFactory.CreateLogger<LogsStressTest>();
+        }
 
-        Stress(prometheusPort: 9464);
-    }
+        protected override void RunWorkItemInParallel()
+        {
+            this.logger.Log(
+                logLevel: LogLevel.Information,
+                eventId: 2,
+                state: Payload,
+                exception: null,
+                formatter: (state, ex) => string.Empty);
+        }
 
-    [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    protected static void Run()
-    {
-        logger.Log(
-            logLevel: LogLevel.Information,
-            eventId: 2,
-            state: payload,
-            exception: null,
-            formatter: (state, ex) => string.Empty);
+        protected override void Dispose(bool isDisposing)
+        {
+            if (isDisposing)
+            {
+                this.loggerFactory.Dispose();
+            }
+
+            base.Dispose(isDisposing);
+        }
     }
 }
diff --git a/test/OpenTelemetry.Tests.Stress.Metrics/OpenTelemetry.Tests.Stress.Metrics.csproj b/test/OpenTelemetry.Tests.Stress.Metrics/OpenTelemetry.Tests.Stress.Metrics.csproj
@@ -3,23 +3,15 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFrameworks>$(TargetFrameworksForTests)</TargetFrameworks>
-    <!-- this is temporary. will remove in future PR. -->
-    <Nullable>disable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="OpenTelemetry.Instrumentation.Runtime" />
-    <PackageReference Include="System.Runtime.InteropServices.RuntimeInformation" Condition="'$(TargetFramework)' == '$(NetFrameworkMinimumSupportedVersion)'" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry\OpenTelemetry.csproj" />
-    <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry.Exporter.Prometheus.HttpListener\OpenTelemetry.Exporter.Prometheus.HttpListener.csproj" />
+    <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry.Exporter.OpenTelemetryProtocol\OpenTelemetry.Exporter.OpenTelemetryProtocol.csproj" />
+    <ProjectReference Include="$(RepoRoot)\test\OpenTelemetry.Tests.Stress\OpenTelemetry.Tests.Stress.csproj" />
   </ItemGroup>
 
   <ItemGroup>
     <Compile Include="$(RepoRoot)\test\OpenTelemetry.Tests\Shared\Utils.cs" Link="Includes\Utils.cs" />
-    <Compile Include="$(RepoRoot)\test\OpenTelemetry.Tests.Stress\Skeleton.cs" Link="Includes\Skeleton.cs" />
   </ItemGroup>
 
 </Project>
diff --git a/test/OpenTelemetry.Tests.Stress.Metrics/Program.cs b/test/OpenTelemetry.Tests.Stress.Metrics/Program.cs
@@ -2,65 +2,140 @@
 // SPDX-License-Identifier: Apache-2.0
 
 using System.Diagnostics.Metrics;
-using System.Runtime.CompilerServices;
+using System.Text.Json.Serialization;
+using CommandLine;
 using OpenTelemetry.Metrics;
 
 namespace OpenTelemetry.Tests.Stress;
 
-public partial class Program
+public static class Program
 {
-    private const int ArraySize = 10;
-
-    // Note: Uncomment the below line if you want to run Histogram stress test
-    private const int MaxHistogramMeasurement = 1000;
+    private enum MetricsStressTestType
+    {
+        /// <summary>Histogram.</summary>
+        Histogram,
 
-    private static readonly Meter TestMeter = new(Utils.GetCurrentMethodName());
-    private static readonly Counter<long> TestCounter = TestMeter.CreateCounter<long>("TestCounter");
-    private static readonly string[] DimensionValues = new string[ArraySize];
-    private static readonly ThreadLocal<Random> ThreadLocalRandom = new(() => new Random());
+        /// <summary>Counter.</summary>
+        Counter,
+    }
 
-    // Note: Uncomment the below line if you want to run Histogram stress test
-    private static readonly Histogram<long> TestHistogram = TestMeter.CreateHistogram<long>("TestHistogram");
+    public static int Main(string[] args)
+    {
+        return StressTestFactory.RunSynchronously<MetricsStressTest, MetricsStressTestOptions>(args);
+    }
 
-    public static void Main()
+    private sealed class MetricsStressTest : StressTest<MetricsStressTestOptions>
     {
-        for (int i = 0; i < ArraySize; i++)
+        private const int ArraySize = 10;
+        private const int MaxHistogramMeasurement = 1000;
+
+        private static readonly Meter TestMeter = new(Utils.GetCurrentMethodName());
+        private static readonly Histogram<long> TestHistogram = TestMeter.CreateHistogram<long>("TestHistogram");
+        private static readonly Counter<long> TestCounter = TestMeter.CreateCounter<long>("TestCounter");
+        private static readonly string[] DimensionValues = new string[ArraySize];
+        private static readonly ThreadLocal<Random> ThreadLocalRandom = new(() => new Random());
+        private readonly MeterProvider meterProvider;
+
+        static MetricsStressTest()
         {
-            DimensionValues[i] = $"DimValue{i}";
+            for (int i = 0; i < ArraySize; i++)
+            {
+                DimensionValues[i] = $"DimValue{i}";
+            }
         }
 
-        using var meterProvider = Sdk.CreateMeterProviderBuilder()
-            .AddMeter(TestMeter.Name)
+        public MetricsStressTest(MetricsStressTestOptions options)
+            : base(options)
+        {
+            var builder = Sdk.CreateMeterProviderBuilder().AddMeter(TestMeter.Name);
+
+            if (options.PrometheusTestMetricsPort != 0)
+            {
+                builder.AddPrometheusHttpListener(o => o.UriPrefixes = new string[] { $"http://localhost:{options.PrometheusTestMetricsPort}/" });
+            }
+
+            if (options.EnableExemplars)
+            {
+                builder.SetExemplarFilter(new AlwaysOnExemplarFilter());
+            }
+
+            if (options.AddViewToFilterTags)
+            {
+                builder
+                    .AddView("TestCounter", new MetricStreamConfiguration { TagKeys = new string[] { "DimName1" } })
+                    .AddView("TestHistogram", new MetricStreamConfiguration { TagKeys = new string[] { "DimName1" } });
+            }
 
-            // .SetExemplarFilter(new AlwaysOnExemplarFilter())
-            .AddPrometheusHttpListener(
-                options => options.UriPrefixes = new string[] { $"http://localhost:9185/" })
-            .Build();
+            if (options.AddOtlpExporter)
+            {
+                builder.AddOtlpExporter((exporterOptions, readerOptions) =>
+                {
+                    readerOptions.PeriodicExportingMetricReaderOptions.ExportIntervalMilliseconds = options.OtlpExporterExportIntervalMilliseconds;
+                });
+            }
 
-        Stress(prometheusPort: 9464);
+            this.meterProvider = builder.Build();
+        }
+
+        protected override void WriteRunInformationToConsole()
+        {
+            if (this.Options.PrometheusTestMetricsPort != 0)
+            {
+                Console.Write($", testPrometheusEndpoint = http://localhost:{this.Options.PrometheusTestMetricsPort}/metrics/");
+            }
+        }
+
+        protected override void RunWorkItemInParallel()
+        {
+            var random = ThreadLocalRandom.Value!;
+            if (this.Options.TestType == MetricsStressTestType.Histogram)
+            {
+                TestHistogram.Record(
+                    random.Next(MaxHistogramMeasurement),
+                    new("DimName1", DimensionValues[random.Next(0, ArraySize)]),
+                    new("DimName2", DimensionValues[random.Next(0, ArraySize)]),
+                    new("DimName3", DimensionValues[random.Next(0, ArraySize)]));
+            }
+            else if (this.Options.TestType == MetricsStressTestType.Counter)
+            {
+                TestCounter.Add(
+                   100,
+                   new("DimName1", DimensionValues[random.Next(0, ArraySize)]),
+                   new("DimName2", DimensionValues[random.Next(0, ArraySize)]),
+                   new("DimName3", DimensionValues[random.Next(0, ArraySize)]));
+            }
+        }
+
+        protected override void Dispose(bool isDisposing)
+        {
+            if (isDisposing)
+            {
+                this.meterProvider.Dispose();
+            }
+
+            base.Dispose(isDisposing);
+        }
     }
 
-    // Note: Uncomment the below lines if you want to run Counter stress test
-    // [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    // protected static void Run()
-    // {
-    //    var random = ThreadLocalRandom.Value;
-    //    TestCounter.Add(
-    //        100,
-    //        new("DimName1", DimensionValues[random.Next(0, ArraySize)]),
-    //        new("DimName2", DimensionValues[random.Next(0, ArraySize)]),
-    //        new("DimName3", DimensionValues[random.Next(0, ArraySize)]));
-    // }
-
-    // Note: Uncomment the below lines if you want to run Histogram stress test
-    [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    protected static void Run()
+    private sealed class MetricsStressTestOptions : StressTestOptions
     {
-        var random = ThreadLocalRandom.Value;
-        TestHistogram.Record(
-            random.Next(MaxHistogramMeasurement),
-            new("DimName1", DimensionValues[random.Next(0, ArraySize)]),
-            new("DimName2", DimensionValues[random.Next(0, ArraySize)]),
-            new("DimName3", DimensionValues[random.Next(0, ArraySize)]));
+        [JsonConverter(typeof(JsonStringEnumConverter))]
+        [Option('t', "type", HelpText = "The metrics stress test type to run. Valid values: [Histogram, Counter]. Default value: Histogram.", Required = false)]
+        public MetricsStressTestType TestType { get; set; } = MetricsStressTestType.Histogram;
+
+        [Option('m', "metrics_port", HelpText = "The Prometheus http listener port where Prometheus will be exposed for retrieving test metrics while the stress test is running. Set to '0' to disable. Default value: 9185.", Required = false)]
+        public int PrometheusTestMetricsPort { get; set; } = 9185;
+
+        [Option('v', "view", HelpText = "Whether or not a view should be configured to filter tags for the stress test. Default value: False.", Required = false)]
+        public bool AddViewToFilterTags { get; set; }
+
+        [Option('o', "otlp", HelpText = "Whether or not an OTLP exporter should be added for the stress test. Default value: False.", Required = false)]
+        public bool AddOtlpExporter { get; set; }
+
+        [Option('i', "interval", HelpText = "The OTLP exporter export interval in milliseconds. Default value: 5000.", Required = false)]
+        public int OtlpExporterExportIntervalMilliseconds { get; set; } = 5000;
+
+        [Option('e', "exemplars", HelpText = "Whether or not to enable exemplars for the stress test. Default value: False.", Required = false)]
+        public bool EnableExemplars { get; set; }
     }
 }
diff --git a/test/OpenTelemetry.Tests.Stress.Traces/OpenTelemetry.Tests.Stress.Traces.csproj b/test/OpenTelemetry.Tests.Stress.Traces/OpenTelemetry.Tests.Stress.Traces.csproj
@@ -6,17 +6,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="OpenTelemetry.Instrumentation.Runtime" />
-    <PackageReference Include="System.Runtime.InteropServices.RuntimeInformation" Condition="'$(TargetFramework)' == '$(NetFrameworkMinimumSupportedVersion)'" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry\OpenTelemetry.csproj" />
-    <ProjectReference Include="$(RepoRoot)\src\OpenTelemetry.Exporter.Prometheus.HttpListener\OpenTelemetry.Exporter.Prometheus.HttpListener.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <Compile Include="$(RepoRoot)\test\OpenTelemetry.Tests.Stress\Skeleton.cs" Link="Includes\Skeleton.cs" />
+    <ProjectReference Include="$(RepoRoot)\test\OpenTelemetry.Tests.Stress\OpenTelemetry.Tests.Stress.csproj" />
   </ItemGroup>
 
 </Project>
diff --git a/test/OpenTelemetry.Tests.Stress.Traces/Program.cs b/test/OpenTelemetry.Tests.Stress.Traces/Program.cs
@@ -2,31 +2,45 @@
 // SPDX-License-Identifier: Apache-2.0
 
 using System.Diagnostics;
-using System.Runtime.CompilerServices;
-using OpenTelemetry.Metrics;
 using OpenTelemetry.Trace;
 
 namespace OpenTelemetry.Tests.Stress;
 
-public partial class Program
+public static class Program
 {
-    private static readonly ActivitySource ActivitySource = new ActivitySource("OpenTelemetry.Tests.Stress");
-
-    public static void Main()
+    public static int Main(string[] args)
     {
-        using var tracerProvider = Sdk.CreateTracerProviderBuilder()
-            .AddSource(ActivitySource.Name)
-            .Build();
-
-        Stress(prometheusPort: 9464);
+        return StressTestFactory.RunSynchronously<TracesStressTest>(args);
     }
 
-    [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    protected static void Run()
+    private sealed class TracesStressTest : StressTest<StressTestOptions>
     {
-        using (var activity = ActivitySource.StartActivity("test"))
+        private static readonly ActivitySource ActivitySource = new("OpenTelemetry.Tests.Stress");
+        private readonly TracerProvider tracerProvider;
+
+        public TracesStressTest(StressTestOptions options)
+            : base(options)
+        {
+            this.tracerProvider = Sdk.CreateTracerProviderBuilder()
+                .AddSource(ActivitySource.Name)
+                .Build();
+        }
+
+        protected override void RunWorkItemInParallel()
         {
+            using var activity = ActivitySource.StartActivity("test");
+
             activity?.SetTag("foo", "value");
         }
+
+        protected override void Dispose(bool isDisposing)
+        {
+            if (isDisposing)
+            {
+                this.tracerProvider.Dispose();
+            }
+
+            base.Dispose(isDisposing);
+        }
     }
 }