Supporting workspaces on giant binlogs (#172)

The old method of converting a binlog to a Workspace was by first converting to a compiler log. That process is fine for small / medium logs but for gigantic logs that can be quite expensive. Was attempting this on a binlog that had 1,700+ compilations inside it and the process quickly OOM'd.

This PR addresses the problem by loading Workspace directly from a binlog. This moves more methods into ICompilerCallReader and makes that the basis for SolutionReader. This was a pretty substantial refactoring of the code base as it required sharing a lot more concepts between the two readers.

.github/actions/dotnet-test/action.yml

@@ -22,8 +22,9 @@ runs:
       run: >
         dotnet test --no-build --framework ${{ inputs.framework }}
         --blame-hang --blame-hang-dump-type full --blame-hang-timeout 10m
+        --results-directory ${{ inputs.test-results-dir }}
         --logger "console;verbosity=detailed"
-        --logger "trx;LogFileName=${{ inputs.test-results-dir }}/TestResults-${{ inputs.name }}.trx"
+        --logger "trx;LogFileName=TestResults-${{ inputs.name }}.trx"
         -p:CollectCoverage=true -p:CoverletOutputFormat=cobertura -p:CoverletOutput=${{ inputs.test-coverage-dir }}/coverage.${{ inputs.name }}.xml
       shell: pwsh
       env:

src/Basic.CompilerLog.UnitTests/BinaryLogReaderTests.cs

@@ -42,6 +42,21 @@ public void ReadCommandLineArgumentsOwnership()
         Assert.Throws<ArgumentException>(() => reader.ReadCommandLineArguments(compilerCall));
     }
 
+    /// <summary>
+    /// Creating a <see cref="CommandLineArguments"/> instance requires a non-trivial amount of 
+    /// work as it's parsed from a raw string. Several parts of the code base expect to be able
+    /// to get them cheaply with an amortized cost of a single parse. Verify that happens here.
+    /// </summary>
+    [Fact]
+    public void ReadCommandLineArgumentsIdentity()
+    {
+        using var reader = BinaryLogReader.Create(Fixture.Console.Value.BinaryLogPath!);
+        var compilerCall = reader.ReadAllCompilerCalls().First();
+        var arg1 = reader.ReadCommandLineArguments(compilerCall);
+        var arg2 = reader.ReadCommandLineArguments(compilerCall);
+        Assert.Same(arg1, arg2);
+    }
+
     [Fact]
     public void DisposeDouble()
     {

src/Basic.CompilerLog.UnitTests/CompilationDataTests.cs

@@ -139,8 +139,8 @@ public void GetCompilationAfterGeneratorsDiagnostics()
             using var reader = CompilerLogReader.Create(
                 logFilePath,
                 BasicAnalyzerHost.DefaultKind);
-            var rawData = reader.ReadRawCompilationData(0).Item2;
-            var analyzers = rawData.Analyzers
+            var rawData = reader.ReadAllAnalyzerData(0);
+            var analyzers = rawData
                 .Where(x => x.FileName != "Microsoft.CodeAnalysis.NetAnalyzers.dll")
                 .ToList();
             BasicAnalyzerHost host = IsNetCore

src/Basic.CompilerLog.UnitTests/CompilerLogFixture.cs

@@ -7,6 +7,7 @@
 using System.Linq;
 using System.Runtime.InteropServices;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using System.Web;
 using Xunit;
@@ -62,6 +63,11 @@ public sealed class CompilerLogFixture : FixtureBase, IDisposable
     /// </summary>
     internal Lazy<LogData> ConsoleWithReference { get; }
 
+    /// <summary>
+    /// A console project that has a reference to a library with an alias
+    /// </summary>
+    internal Lazy<LogData> ConsoleWithAliasReference { get; }
+
     /// <summary>
     /// This is a console project that has every nasty feature that can be thought of
     /// like resources, line directives, embeds, etc ... Rather than running a 
@@ -341,7 +347,7 @@ partial class Util {
             RunDotnetCommand("build -bl -nr:false", scratchPath);
         });
 
-        ConsoleWithReference = WithBuild("console-with-project-ref..complog", void (string scratchPath) =>
+        ConsoleWithReference = WithBuild("console-with-project-ref.complog", void (string scratchPath) =>
         {
             RunDotnetCommand("new sln -n ConsoleWithProjectRef", scratchPath);
 
@@ -380,6 +386,71 @@ public static class NameInfo
             RunDotnetCommand("build -bl -nr:false", scratchPath);
         });
 
+        ConsoleWithAliasReference = WithBuild("console-with-alias-project-ref.complog", void (string scratchPath) =>
+        {
+            RunDotnetCommand("new sln -n ConsoleWithProjectAliasRef", scratchPath);
+
+            // Create a class library for referencing
+            var classLibPath = Path.Combine(scratchPath, "classlib");
+            _ = Directory.CreateDirectory(classLibPath);
+            RunDotnetCommand("new classlib -o . -n util", classLibPath);
+            File.WriteAllText(
+                Path.Combine(classLibPath, "Class1.cs"),
+                """
+                using System;
+                namespace Util;
+                public static class NameInfo
+                {
+                    public static string GetName() => "Hello World";
+                }
+                """,
+                TestBase.DefaultEncoding);
+            RunDotnetCommand($@"sln add ""{classLibPath}""", scratchPath);
+
+            // Create a console project that references the class library
+            var consolePath = Path.Combine(scratchPath, "console");
+            _ = Directory.CreateDirectory(consolePath);
+            RunDotnetCommand("new console -o . -n console-with-alias-reference", consolePath);
+            File.WriteAllText(
+                Path.Combine(consolePath, "Program.cs"),
+                """
+                extern alias Util;
+                using System;
+                using Util;
+                Console.WriteLine(Util::Util.NameInfo.GetName());
+                """,
+                TestBase.DefaultEncoding);
+            RunDotnetCommand($@"add . reference ""{classLibPath}""", consolePath);
+
+            var consoleProjectPath = Path.Combine(consolePath, "console-with-alias-reference.csproj");
+            AddExternAlias(consoleProjectPath, "Util");
+            RunDotnetCommand($@"sln add ""{consolePath}""", scratchPath);
+
+            RunDotnetCommand("build -bl -nr:false", scratchPath);
+
+            static void AddExternAlias(string projectFilePath, string aliasName)
+            {
+                var oldLines = File.ReadAllLines(projectFilePath);
+                var newlines = new List<string>(capacity: oldLines.Length + 2);
+
+                for (int i = 0; i < oldLines.Length; i++)
+                {
+                    var line = oldLines[i];
+                    if (line.Contains("<ProjectReference", StringComparison.Ordinal))
+                    {
+                        newlines.Add(line.Replace("/>", ">"));
+                        newlines.Add($"        <Aliases>{aliasName}</Aliases>");
+                        newlines.Add($"    </ProjectReference>");
+                    }
+                    else
+                    {
+                        newlines.Add(line); 
+                    }
+                }
+                File.WriteAllLines(projectFilePath, newlines);
+            }
+        });
+
         if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
         {
             WpfApp = WithBuild("wpfapp.complog", void (string scratchPath) =>

src/Basic.CompilerLog.UnitTests/CompilerLogReaderTests.cs

@@ -49,8 +49,7 @@ public void ContentExtraSourceFile(string fileName)
         RunDotNet("build -bl -nr:false");
 
         using var reader = CompilerLogReader.Create(Path.Combine(RootDirectory, "msbuild.binlog"));
-        var rawData = reader.ReadRawCompilationData(0).Item2;
-        var extraData = rawData.Contents.Single(x => Path.GetFileName(x.FilePath) == fileName);
+        var extraData = reader.ReadAllRawContent(0).Single(x => Path.GetFileName(x.OriginalFilePath) == fileName);
         Assert.Equal("84C9FAFCF8C92F347B96D26B149295128B08B07A3C4385789FE4758A2B520FDE", extraData.ContentHash);
         var contentBytes = reader.GetContentBytes(extraData.ContentHash);
         Assert.Equal(content, DefaultEncoding.GetString(contentBytes));
@@ -121,8 +120,7 @@ public void MetadataVersion()
     public void ResourceSimpleEmbedded()
     {
         using var reader = CompilerLogReader.Create(Fixture.ConsoleComplex.Value.CompilerLogPath);
-        var rawData = reader.ReadRawCompilationData(0).Item2;
-        var d = rawData.Resources.Single();
+        var d = reader.ReadAllResourceData(0).Single();
         Assert.Equal("console-complex.resource.txt", reader.ReadResourceDescription(d).GetResourceName());
     }
 
@@ -214,10 +212,8 @@ public void AnalyzerLoadCaching(BasicAnalyzerKind kind)
         }
 
         using var reader = CompilerLogReader.Create(Fixture.Console.Value.CompilerLogPath, kind);
-        var (compilerCall, data) = reader.ReadRawCompilationData(0);
-
-        var host1 = reader.ReadAnalyzers(data);
-        var host2 = reader.ReadAnalyzers(data);
+        var host1 = reader.CreateBasicAnalyzerHost(0);
+        var host2 = reader.CreateBasicAnalyzerHost(0);
         Assert.Same(host1, host2);
         host1.Dispose();
         Assert.True(host1.IsDisposed);
@@ -250,8 +246,8 @@ public void AnalyzerLoadDispose(BasicAnalyzerKind kind)
     public void AnalyzerDiagnostics()
     {
         using var reader = CompilerLogReader.Create(Fixture.Console.Value.CompilerLogPath, BasicAnalyzerKind.InMemory);
-        var data = reader.ReadRawCompilationData(0).Item2;
-        var analyzers = data.Analyzers
+        var data = reader.ReadAllAnalyzerData(0);
+        var analyzers = data
             .Where(x => x.FileName != "Microsoft.CodeAnalysis.NetAnalyzers.dll")
             .ToList();
         var host = new BasicAnalyzerHostInMemory(reader, analyzers);
@@ -287,8 +283,7 @@ public void ProjectMultiTarget()
     public void NoneHostGeneratedFilesInRaw()
     {
         using var reader = CompilerLogReader.Create(Fixture.Console.Value.CompilerLogPath, BasicAnalyzerKind.None);
-        var (_, data) = reader.ReadRawCompilationData(0);
-        Assert.Equal(1, data.Contents.Count(x => x.Kind == RawContentKind.GeneratedText));
+        Assert.Single(reader.ReadAllRawContent(0, RawContentKind.GeneratedText));
     }
 
     [Fact]
@@ -349,8 +344,8 @@ public void NoneHostNativePdb()
         RunDotNet("build -bl -nr:false");
 
         using var reader = CompilerLogReader.Create(Path.Combine(RootDirectory, "msbuild.binlog"), BasicAnalyzerKind.None);
-        var rawData = reader.ReadRawCompilationData(0).Item2;
-        Assert.False(rawData.HasAllGeneratedFileContent);
+        var compilerCall = reader.ReadCompilerCall(0);
+        Assert.False(reader.HasAllGeneratedFileContent(compilerCall));
         var data = reader.ReadCompilationData(0);
         var compilation = data.GetCompilationAfterGenerators(out var diagnostics);
         Assert.Single(diagnostics);
@@ -512,6 +507,19 @@ public void ProjectReferences_ReadReference()
         Assert.Equal(1, count);
     }
 
+    [Fact]
+    public void ProjectReferences_Alias()
+    {
+        using var reader = CompilerLogReader.Create(Fixture.ConsoleWithAliasReference.Value.CompilerLogPath);
+        var consoleCompilerCall = reader
+            .ReadAllCompilerCalls(cc => cc.ProjectFileName == "console-with-alias-reference.csproj")
+            .Single();
+        var referenceData = reader
+            .ReadAllReferenceData(consoleCompilerCall)
+            .Single(x => x.Aliases.Length == 1);
+        Assert.Equal("Util", referenceData.Aliases.Single());
+    }
+
     [Fact]
     public void ProjectReferences_Corrupted()
     {

src/Basic.CompilerLog.UnitTests/RoslynUtilTests.cs

@@ -211,4 +211,19 @@ public void ReadAssemblyNameSimple()
         var name = RoslynUtil.ReadAssemblyName(path);
         Assert.Equal("Basic.CompilerLog.Util", name);
     }
+
+    [Fact]
+    public void TryReadMvid_FileMissing()
+    {
+        using var temp = new TempDir();
+        Assert.Null(RoslynUtil.TryReadMvid(Path.Combine(temp.DirectoryPath, "test.dll")));
+    }
+
+    [Fact]
+    public void TryReadMvid_FileNotPE()
+    {
+        using var temp = new TempDir();
+        var filePath = temp.NewFile("test.dll", "hello world");
+        Assert.Null(RoslynUtil.TryReadMvid(filePath));
+    }
 }

src/Basic.CompilerLog.UnitTests/SolutionReaderTests.cs

@@ -46,14 +46,20 @@ private Solution GetSolution(string compilerLogFilePath, BasicAnalyzerKind basic
     [MemberData(nameof(GetSimpleBasicAnalyzerKinds))]
     public async Task DocumentsGeneratedDefaultHost(BasicAnalyzerKind basicAnalyzerKind)
     {
-        var solution = GetSolution(Fixture.Console.Value.CompilerLogPath, basicAnalyzerKind);
-        var project = solution.Projects.Single();
-        Assert.NotEmpty(project.AnalyzerReferences);
-        var docs = project.Documents.ToList();
-        var generatedDocs = (await project.GetSourceGeneratedDocumentsAsync()).ToList();
-        Assert.Null(docs.FirstOrDefault(x => x.Name == "RegexGenerator.g.cs"));
-        Assert.Single(generatedDocs);
-        Assert.NotNull(generatedDocs.First(x => x.Name == "RegexGenerator.g.cs"));
+        await Run(Fixture.Console.Value.BinaryLogPath!);
+        await Run(Fixture.Console.Value.CompilerLogPath);
+
+        async Task Run(string filePath)
+        {
+            var solution = GetSolution(filePath, basicAnalyzerKind);
+            var project = solution.Projects.Single();
+            Assert.NotEmpty(project.AnalyzerReferences);
+            var docs = project.Documents.ToList();
+            var generatedDocs = (await project.GetSourceGeneratedDocumentsAsync()).ToList();
+            Assert.Null(docs.FirstOrDefault(x => x.Name == "RegexGenerator.g.cs"));
+            Assert.Single(generatedDocs);
+            Assert.NotNull(generatedDocs.First(x => x.Name == "RegexGenerator.g.cs"));
+        }
     }
 
     [Fact]
@@ -70,17 +76,23 @@ public void CreateRespectLeaveOpen()
     [Fact]
     public async Task ProjectReference_Simple()
     {
-        var solution = GetSolution(Fixture.ConsoleWithReference.Value.CompilerLogPath, BasicAnalyzerKind.None);
-        var consoleProject = solution.Projects
-            .Where(x => x.Name == "console-with-reference.csproj")
-            .Single();
-        var projectReference = consoleProject.ProjectReferences.Single();
-        var utilProject = solution.GetProject(projectReference.ProjectId);
-        Assert.NotNull(utilProject);
-        Assert.Equal("util.csproj", utilProject.Name);
-        var compilation = await consoleProject.GetCompilationAsync();
-        Assert.NotNull(compilation);
-        var result = compilation.EmitToMemory();
-        Assert.True(result.Success);
+        await Run(Fixture.ConsoleWithReference.Value.BinaryLogPath!);
+        await Run(Fixture.ConsoleWithReference.Value.CompilerLogPath);
+
+        async Task Run(string filePath)
+        {
+            var solution = GetSolution(filePath, BasicAnalyzerKind.None);
+            var consoleProject = solution.Projects
+                .Where(x => x.Name == "console-with-reference.csproj")
+                .Single();
+            var projectReference = consoleProject.ProjectReferences.Single();
+            var utilProject = solution.GetProject(projectReference.ProjectId);
+            Assert.NotNull(utilProject);
+            Assert.Equal("util.csproj", utilProject.Name);
+            var compilation = await consoleProject.GetCompilationAsync();
+            Assert.NotNull(compilation);
+            var result = compilation.EmitToMemory();
+            Assert.True(result.Success);
+        }
     }
 }

src/Basic.CompilerLog.Util/AssemblyIdentityData.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Basic.CompilerLog.Util;
+
+public sealed class AssemblyIdentityData(Guid mvid, string? assemblyName, string? assemblyInformationalVersion)
+{
+    public Guid Mvid { get; } = mvid;
+    public string? AssemblyName { get; } = assemblyName;
+    public string? AssemblyInformationalVersion { get; } = assemblyInformationalVersion;
+}