Add a VSTest Adapter

src/BenchmarkDotNet.TestAdapter/BenchmarkCaseExtensions.cs

@@ -4,10 +4,8 @@
 using BenchmarkDotNet.Extensions;
 using BenchmarkDotNet.Running;
 using Microsoft.TestPlatform.AdapterUtilities;
-using Microsoft.TestPlatform.AdapterUtilities.ManagedNameUtilities;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 using System;
-using System.Linq;
 
 namespace BenchmarkDotNet.TestAdapter
 {
@@ -20,55 +18,41 @@ internal static class BenchmarkCaseExtensions
         /// Converts a BDN BenchmarkCase to a VSTest TestCase.
         /// </summary>
         /// <param name="benchmarkCase">The BenchmarkCase to convert.</param>
-        /// <param name="source">The dll or exe of the benchmark project.</param>
+        /// <param name="assemblyPath">The dll or exe of the benchmark project.</param>
         /// <param name="includeJobInName">Whether or not the display name should include the job name.</param>
         /// <returns>The VSTest TestCase.</returns>
-        internal static TestCase ToVSTestCase(this BenchmarkCase benchmarkCase, string source, bool includeJobInName=false)
+        internal static TestCase ToVSTestCase(this BenchmarkCase benchmarkCase, string assemblyPath, bool includeJobInName=false)
         {
             var benchmarkMethod = benchmarkCase.Descriptor.WorkloadMethod;
-            var fullClassName = benchmarkCase.Descriptor.Type.FullName;
-            var benchmarkMethodName = benchmarkCase.Descriptor.WorkloadMethodDisplayInfo;
+            var fullClassName = benchmarkCase.Descriptor.Type.GetCorrectCSharpTypeName();
+            var benchmarkMethodName = benchmarkCase.Descriptor.WorkloadMethod.Name;
             var benchmarkFullName = $"{fullClassName}.{benchmarkMethodName}";
 
-            ManagedNameHelper.GetManagedName(benchmarkMethod, out var managedType, out var managedMethod, out var hierarchyValues);
-            hierarchyValues[HierarchyConstants.Levels.ContainerIndex] = null; // Gets set by the test explorer window to the test project name
+            // Display name has arguments as well.
+            var displayMethodName = FullNameProvider.GetMethodName(benchmarkCase);
             if (includeJobInName)
-            {
-                hierarchyValues[HierarchyConstants.Levels.TestGroupIndex] += $" [{benchmarkCase.GetUnrandomizedJobDisplayInfo()}]";
-            }
+                displayMethodName += $" [{benchmarkCase.GetUnrandomizedJobDisplayInfo()}]";
 
-            var hasManagedMethodAndTypeProperties = !string.IsNullOrWhiteSpace(managedType) && !string.IsNullOrWhiteSpace(managedMethod);
+            var displayName = $"{fullClassName}.{displayMethodName}";
 
-            var vsTestCase = new TestCase(benchmarkFullName, VSTestAdapter.ExecutorUri, source)
+            var vsTestCase = new TestCase(benchmarkFullName, VSTestAdapter.ExecutorUri, assemblyPath)
             {
-                DisplayName = FullNameProvider.GetBenchmarkName(benchmarkCase),
-                Id = GetTestCaseId(benchmarkCase),
+                DisplayName = displayName,
+                Id = GetTestCaseId(benchmarkCase)
             };
 
-            if (includeJobInName)
-            {
-                vsTestCase.DisplayName += $" [{benchmarkCase.GetUnrandomizedJobDisplayInfo()}]";
-            }
-
             var benchmarkAttribute = benchmarkMethod.ResolveAttribute<BenchmarkAttribute>();
             if (benchmarkAttribute != null)
             {
                 vsTestCase.CodeFilePath = benchmarkAttribute.SourceCodeFile;
                 vsTestCase.LineNumber = benchmarkAttribute.SourceCodeLineNumber;
             }
 
-            vsTestCase.SetPropertyValue(VSTestProperties.HierarchyProperty, hierarchyValues.ToArray());
-            vsTestCase.SetPropertyValue(VSTestProperties.TestCategoryProperty, benchmarkCase.Descriptor.Categories);
-            if (hasManagedMethodAndTypeProperties)
-            {
-                vsTestCase.SetPropertyValue(VSTestProperties.ManagedTypeProperty, managedType);
-                vsTestCase.SetPropertyValue(VSTestProperties.ManagedMethodProperty, managedMethod);
-                vsTestCase.SetPropertyValue(VSTestProperties.TestClassNameProperty, managedType);
-            }
-            else
-            {
-                vsTestCase.SetPropertyValue(VSTestProperties.TestClassNameProperty, fullClassName);
-            }
+            var categories = DefaultCategoryDiscoverer.Instance.GetCategories(benchmarkMethod);
+            foreach (var category in categories)
+                vsTestCase.Traits.Add("Category", category);
+
+            vsTestCase.Traits.Add("", "BenchmarkDotNet");
 
             return vsTestCase;
         }

src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs

@@ -1,26 +1,29 @@
 using BenchmarkDotNet.Extensions;
 using BenchmarkDotNet.Helpers;
 using BenchmarkDotNet.Running;
+using System;
 using System.Linq;
 using System.Reflection;
 
 namespace BenchmarkDotNet.TestAdapter
 {
     /// <summary>
-    /// A class used for enumerating all the benchmarks in a source.
+    /// A class used for enumerating all the benchmarks in an assembly.
     /// </summary>
     internal static class BenchmarkEnumerator
     {
         /// <summary>
-        /// Returns all the BenchmarkRunInfo objects from a given source.
+        /// Returns all the BenchmarkRunInfo objects from a given assembly.
         /// </summary>
-        /// <param name="source">The dll or exe of the benchmark project.</param>
-        /// <returns>The benchmarks inside the source.</returns>
-        public static BenchmarkRunInfo[] GetBenchmarksFromSource(string source)
+        /// <param name="assemblyPath">The dll or exe of the benchmark project.</param>
+        /// <returns>The benchmarks inside the assembly.</returns>
+        public static BenchmarkRunInfo[] GetBenchmarksFromAssemblyPath(string assemblyPath)
         {
-            var assembly = Assembly.LoadFrom(source);
+            var assembly = Assembly.LoadFrom(assemblyPath);
+
+            if (assembly.IsDebug() ?? false)
+                return Array.Empty<BenchmarkRunInfo>();
 
-            // TODO: Allow for defining a base config inside the BDN project that is used by the VSTest Adapter.
             return GenericBenchmarksBuilder.GetRunnableBenchmarks(assembly.GetRunnableBenchmarks())
                 .Select(type => BenchmarkConverter.TypeToBenchmarks(type))
                 .Where(runInfo => runInfo.BenchmarksCases.Length > 0)

src/BenchmarkDotNet.TestAdapter/BenchmarkExecutor.cs

@@ -17,17 +17,17 @@ internal class BenchmarkExecutor
         private readonly CancellationTokenSource cts = new ();
 
         /// <summary>
-        /// Runs all the benchmarks in the given source, updating the TestExecutionRecorder as they get run.
+        /// Runs all the benchmarks in the given assembly, updating the TestExecutionRecorder as they get run.
         /// </summary>
-        /// <param name="source">The dll or exe of the benchmark project.</param>
+        /// <param name="assemblyPath">The dll or exe of the benchmark project.</param>
         /// <param name="recorder">The interface used to record the current test execution progress.</param>
         /// <param name="benchmarkIds">
         /// An optional list of benchmark IDs specifying which benchmarks to run.
         /// These IDs are the same as the ones generated for the VSTest TestCase.
         /// </param>
-        public void RunBenchmarks(string source, TestExecutionRecorderWrapper recorder, HashSet<Guid>? benchmarkIds = null)
+        public void RunBenchmarks(string assemblyPath, TestExecutionRecorderWrapper recorder, HashSet<Guid>? benchmarkIds = null)
         {
-            var benchmarks = BenchmarkEnumerator.GetBenchmarksFromSource(source);
+            var benchmarks = BenchmarkEnumerator.GetBenchmarksFromAssemblyPath(assemblyPath);
             var testCases = new List<TestCase>();
 
             var filteredBenchmarks = new List<BenchmarkRunInfo>();
@@ -41,7 +41,7 @@ public void RunBenchmarks(string source, TestExecutionRecorderWrapper recorder,
                     if (benchmarkIds != null && benchmarkIds.Contains(testId))
                     {
                         filteredCases.Add(benchmarkCase);
-                        testCases.Add(benchmarkCase.ToVSTestCase(source, needsJobInfo));
+                        testCases.Add(benchmarkCase.ToVSTestCase(assemblyPath, needsJobInfo));
                     }
                 }
 

src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkEnumeratorWrapper.cs

@@ -10,21 +10,21 @@ namespace BenchmarkDotNet.TestAdapter.Remoting
     internal class BenchmarkEnumeratorWrapper : MarshalByRefObject
     {
         /// <summary>
-        /// Gets a list of VSTest TestCases from the given source.
-        /// Each test case is serialize into a string so that it can be used across AppDomain boundaries.
+        /// Gets a list of VSTest TestCases from the given assembly.
+        /// Each test case is serialized into a string so that it can be used across AppDomain boundaries.
         /// </summary>
-        /// <param name="source">The dll or exe of the benchmark project.</param>
+        /// <param name="assemblyPath">The dll or exe of the benchmark project.</param>
         /// <returns>The serialized test cases.</returns>
-        public List<string> GetTestCasesFromSourceSerialized(string source)
+        public List<string> GetTestCasesFromAssemblyPathSerialized(string assemblyPath)
         {
             var serializedTestCases = new List<string>();
-            foreach (var runInfo in BenchmarkEnumerator.GetBenchmarksFromSource(source))
+            foreach (var runInfo in BenchmarkEnumerator.GetBenchmarksFromAssemblyPath(assemblyPath))
             {
                 // If all the benchmarks have the same job, then no need to include job info.
                 var needsJobInfo = runInfo.BenchmarksCases.Select(c => c.Job.DisplayInfo).Distinct().Count() > 1;
                 foreach (var benchmarkCase in runInfo.BenchmarksCases)
                 {
-                    var testCase = benchmarkCase.ToVSTestCase(source, needsJobInfo);
+                    var testCase = benchmarkCase.ToVSTestCase(assemblyPath, needsJobInfo);
                     serializedTestCases.Add(SerializationHelpers.Serialize(testCase));
                 }
             }

src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkExecutorWrapper.cs

@@ -10,9 +10,9 @@ internal class BenchmarkExecutorWrapper : MarshalByRefObject
     {
         private readonly BenchmarkExecutor benchmarkExecutor = new ();
 
-        public void RunBenchmarks(string source, TestExecutionRecorderWrapper recorder, HashSet<Guid>? benchmarkIds = null)
+        public void RunBenchmarks(string assemblyPath, TestExecutionRecorderWrapper recorder, HashSet<Guid>? benchmarkIds = null)
         {
-            benchmarkExecutor.RunBenchmarks(source, recorder, benchmarkIds);
+            benchmarkExecutor.RunBenchmarks(assemblyPath, recorder, benchmarkIds);
         }
 
         public void Cancel()

src/BenchmarkDotNet.TestAdapter/Remoting/SerializationHelpers.cs

@@ -7,14 +7,18 @@ namespace BenchmarkDotNet.TestAdapter.Remoting
     /// </summary>
     internal static class SerializationHelpers
     {
+        // Version number of the VSTest protocol that the adapter supports. Only needs to be updated when
+        // the VSTest protocol has a change and this test adapter wishes to take a dependency on it.
+        private const int VSTestProtocolVersion = 7;
+
         public static string Serialize<T>(T data)
         {
-            return JsonDataSerializer.Instance.Serialize(data, version: 7);
+            return JsonDataSerializer.Instance.Serialize(data, version: VSTestProtocolVersion);
         }
 
         public static T Deserialize<T>(string data)
         {
-            return JsonDataSerializer.Instance.Deserialize<T>(data, version: 7)!;
+            return JsonDataSerializer.Instance.Deserialize<T>(data, version: VSTestProtocolVersion)!;
         }
     }
 }

src/BenchmarkDotNet.TestAdapter/VSTestAdapter.cs

@@ -44,8 +44,8 @@ public void DiscoverTests(
         {
             foreach (var source in sources)
             {
-                ValidateSourceOrThrow(source);
-                foreach (var testCase in GetVSTestCasesFromSource(source, logger))
+                ValidateSourceIsAssemblyOrThrow(source);
+                foreach (var testCase in GetVSTestCasesFromAssembly(source, logger))
                 {
                     discoverySink.SendTestCase(testCase);
                 }
@@ -67,8 +67,8 @@ public void RunTests(IEnumerable<TestCase>? tests, IRunContext? runContext, IFra
 
             cts ??= new CancellationTokenSource();
 
-            foreach (var testsPerSource in tests.GroupBy(t => t.Source))
-                RunBenchmarks(testsPerSource.Key, frameworkHandle, testsPerSource);
+            foreach (var testsPerAssembly in tests.GroupBy(t => t.Source))
+                RunBenchmarks(testsPerAssembly.Key, frameworkHandle, testsPerAssembly);
 
             cts = null;
         }
@@ -103,21 +103,19 @@ public void Cancel()
         }
 
         /// <summary>
-        /// Gets the VSTest test cases in the given source.
+        /// Gets the VSTest test cases in the given assembly.
         /// </summary>
-        /// <param name="source">The dll or exe of the benchmark project.</param>
+        /// <param name="assemblyPath">The dll or exe of the benchmark project.</param>
         /// <param name="logger">A logger that sends logs to VSTest.</param>
-        /// <returns>The VSTest test cases inside the given source.</returns>
-        private static List<TestCase> GetVSTestCasesFromSource(string source, IMessageLogger logger)
+        /// <returns>The VSTest test cases inside the given assembly.</returns>
+        private static List<TestCase> GetVSTestCasesFromAssembly(string assemblyPath, IMessageLogger logger)
         {
-            ValidateSourceOrThrow(source);
-
             try
             {
                 // Ensure that the test enumeration is done inside the context of the source directory.
-                var enumerator = (BenchmarkEnumeratorWrapper)CreateIsolatedType(typeof(BenchmarkEnumeratorWrapper), source);
+                var enumerator = (BenchmarkEnumeratorWrapper)CreateIsolatedType(typeof(BenchmarkEnumeratorWrapper), assemblyPath);
                 var testCases = enumerator
-                    .GetTestCasesFromSourceSerialized(source)
+                    .GetTestCasesFromAssemblyPathSerialized(assemblyPath)
                     .Select(SerializationHelpers.Deserialize<TestCase>)
                     .ToList();
 
@@ -135,7 +133,7 @@ private static List<TestCase> GetVSTestCasesFromSource(string source, IMessageLo
             }
             catch (Exception ex)
             {
-                logger.SendMessage(TestMessageLevel.Error, $"Failed to load benchmarks from source\n{ex}");
+                logger.SendMessage(TestMessageLevel.Error, $"Failed to load benchmarks from assembly\n{ex}");
                 throw;
             }
         }
@@ -151,7 +149,7 @@ private static List<TestCase> GetVSTestCasesFromSource(string source, IMessageLo
         /// </param>
         private void RunBenchmarks(string source, IFrameworkHandle frameworkHandle, IEnumerable<TestCase>? testCases = null)
         {
-            ValidateSourceOrThrow(source);
+            ValidateSourceIsAssemblyOrThrow(source);
 
             // Create a HashSet of all the TestCase IDs to be run if specified.
             var caseIds = testCases == null ? null : new HashSet<Guid>(testCases.Select(c => c.Id));
@@ -166,7 +164,7 @@ private void RunBenchmarks(string source, IFrameworkHandle frameworkHandle, IEnu
             }
             catch (Exception ex)
             {
-                frameworkHandle.SendMessage(TestMessageLevel.Error, $"Failed to run benchmarks in source\n{ex}");
+                frameworkHandle.SendMessage(TestMessageLevel.Error, $"Failed to run benchmarks in assembly\n{ex}");
                 throw;
             }
         }
@@ -176,12 +174,12 @@ private void RunBenchmarks(string source, IFrameworkHandle frameworkHandle, IEnu
         /// If not in the .NET Framework, it will use the current
         /// </summary>
         /// <param name="type">The type to create.</param>
-        /// <param name="source">The dll or exe of the benchmark project.</param>
+        /// <param name="assemblyPath">The dll or exe of the benchmark project.</param>
         /// <returns>The created object.</returns>
-        private static object CreateIsolatedType(Type type, string source)
+        private static object CreateIsolatedType(Type type, string assemblyPath)
         {
 #if NETFRAMEWORK
-            var appBase = Path.GetDirectoryName(source);
+            var appBase = Path.GetDirectoryName(assemblyPath);
             var setup = new AppDomainSetup { ApplicationBase = appBase };
             var domainName = $"Isolated Domain for {type.Name}";
             var appDomain = AppDomain.CreateDomain(domainName, null, setup);
@@ -192,7 +190,7 @@ private static object CreateIsolatedType(Type type, string source)
 #endif
         }
 
-        private static void ValidateSourceOrThrow(string source)
+        private static void ValidateSourceIsAssemblyOrThrow(string source)
         {
             if (string.IsNullOrEmpty(source))
                 throw new ArgumentException($"'{nameof(source)}' cannot be null or whitespace.", nameof(source));

src/BenchmarkDotNet.TestAdapter/VSTestLogger.cs

@@ -7,7 +7,7 @@ namespace BenchmarkDotNet.TestAdapter
     /// <summary>
     /// A class to send logs from BDN to the VSTest output log.
     /// </summary>
-    internal class VSTestLogger : ILogger
+    internal sealed class VSTestLogger : ILogger
     {
         private readonly IMessageLogger messageLogger;
         private readonly StringBuilder currentLine = new StringBuilder();
@@ -31,7 +31,7 @@ public void Write(LogKind logKind, string text)
         {
             currentLine.Append(text);
 
-            // Assume that if the log kind if an error, that the whole line is treated as an error
+            // Assume that if the log kind is an error, that the whole line is treated as an error
             // The level will be reset to Informational when WriteLine() is called.
             if (logKind == LogKind.Error)
                 currentLevel = TestMessageLevel.Error;