C#: Add binlog support to buildless with source generator support

csharp/autobuilder/Semmle.Autobuild.CSharp/BUILD.bazel

@@ -13,6 +13,7 @@ codeql_csharp_binary(
         "//csharp/autobuilder/Semmle.Autobuild.Shared",
         "//csharp/extractor/Semmle.Extraction.CSharp",
         "//csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching",
+        "//csharp/extractor/Semmle.Extraction.CSharp.Driver:bin/Semmle.Extraction.CSharp.Driver",
         "//csharp/extractor/Semmle.Extraction.CSharp.Standalone:bin/Semmle.Extraction.CSharp.Standalone",
         "//csharp/extractor/Semmle.Util",
         "@paket.main//microsoft.build",

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs

@@ -13,8 +13,10 @@ public class CSharpAutobuildOptions : AutobuildOptionsShared
     {
         private const string buildModeEnvironmentVariable = "CODEQL_EXTRACTOR_CSHARP_BUILD_MODE";
         internal const string ExtractorOptionBuildless = "CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS";
+        internal const string ExtractorOptionBinlog = "CODEQL_EXTRACTOR_CSHARP_OPTION_BINLOG";
 
         public bool Buildless { get; }
+        public string? Binlog { get; }
 
         public override Language Language => Language.CSharp;
 
@@ -29,7 +31,7 @@ public CSharpAutobuildOptions(IBuildActions actions) : base(actions)
                 actions.GetEnvironmentVariable(ExtractorOptionBuildless).AsBool("buildless", false) ||
                 actions.GetEnvironmentVariable(buildModeEnvironmentVariable)?.ToLower() == "none";
 
-
+            Binlog = actions.GetEnvironmentVariable(ExtractorOptionBinlog);
         }
     }
 
@@ -114,6 +116,20 @@ private BuildScript AddBuildlessStartedDiagnostic()
                     markdownMessage: "C# was extracted with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.",
                     severity: DiagnosticMessage.TspSeverity.Note
                 ));
+
+                // For the time being we are adding an additional message regarding the binlog usage. In the future, we might want to remove the buildless messages altogether when the binlog option is specified.
+                if (actions.GetEnvironmentVariable(CSharpAutobuildOptions.ExtractorOptionBinlog) is not null)
+                {
+                    AddDiagnostic(new DiagnosticMessage(
+                        Options.Language,
+                        "buildless/binlog",
+                        "C# was extracted with the experimental 'binlog' option",
+                        visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
+                        markdownMessage: "C# was extracted with the experimental 'binlog' option.",
+                        severity: DiagnosticMessage.TspSeverity.Note
+                    ));
+                }
+
                 return 0;
             });
         }

csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj

@@ -6,6 +6,7 @@
     <ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
     <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
     <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj" />
+    <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.Driver\Semmle.Extraction.CSharp.Driver.csproj" />
     <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
     <ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
   </ItemGroup>

csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs

@@ -10,7 +10,9 @@ internal class StandaloneBuildRule : IBuildRule<CSharpAutobuildOptions>
     {
         public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
         {
-            return BuildScript.Create(_ => Semmle.Extraction.CSharp.Standalone.Program.Main([]));
+            return builder.Options.Binlog is string binlog
+                ? BuildScript.Create(_ => Semmle.Extraction.CSharp.Driver.Main(["--binlog", binlog]))
+                : BuildScript.Create(_ => Semmle.Extraction.CSharp.Standalone.Program.Main([]));
         }
     }
 }

csharp/codeql-extractor.yml

@@ -65,3 +65,9 @@ options:
             - progress+++
         type: string
         pattern: "^(off|errors|warnings|(info|progress)|(debug|progress\\+)|(trace|progress\\+\\+)|progress\\+\\+\\+)$"
+  binlog:
+    title: Binlog
+    description: >
+      [EXPERIMENTAL] The value is a path to the MsBuild binary log file that should be extracted.
+      This option only works when `--build-mode none` is also specified.
+    type: string

csharp/extractor/Semmle.Extraction.CSharp.Driver/BUILD.bazel

@@ -8,7 +8,7 @@ codeql_csharp_binary(
     srcs = glob([
         "*.cs",
     ]),
-    visibility = ["//csharp:__pkg__"],
+    visibility = ["//csharp:__subpackages__"],
     deps = [
         "//csharp/extractor/Semmle.Extraction.CSharp",
     ],

csharp/extractor/Semmle.Extraction.CSharp/BUILD.bazel

@@ -19,6 +19,7 @@ codeql_csharp_library(
         "//csharp/extractor/Semmle.Extraction",
         "//csharp/extractor/Semmle.Extraction.CSharp.Util",
         "//csharp/extractor/Semmle.Util",
+        "@paket.main//basic.compilerlog.util",
         "@paket.main//microsoft.build",
         "@paket.main//microsoft.codeanalysis.csharp",
     ],

csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs

@@ -34,7 +34,14 @@ public override void Populate(TextWriter trapFile)
                         lineCounts.Total++;
 
                     trapFile.numlines(this, lineCounts);
-                    Context.TrapWriter.Archive(originalPath, TransformedPath, text.Encoding ?? System.Text.Encoding.Default);
+                    if (BinaryLogExtractionContext.GetAdjustedPath(Context.ExtractionContext, originalPath) is not null)
+                    {
+                        Context.TrapWriter.ArchiveContent(rawText, TransformedPath);
+                    }
+                    else
+                    {
+                        Context.TrapWriter.Archive(originalPath, TransformedPath, text.Encoding ?? System.Text.Encoding.Default);
+                    }
                 }
             }
             else if (IsPossiblyTextFile())

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -185,7 +185,8 @@ private void DoExtractTree(SyntaxTree tree)
             {
                 var stopwatch = new Stopwatch();
                 stopwatch.Start();
-                var sourcePath = tree.FilePath;
+                var sourcePath = BinaryLogExtractionContext.GetAdjustedPath(ExtractionContext, tree.FilePath) ?? tree.FilePath;
+
                 var transformedSourcePath = PathTransformer.Transform(sourcePath);
 
                 var trapPath = transformedSourcePath.GetTrapPath(Logger, options.TrapCompression);

csharp/extractor/Semmle.Extraction.CSharp/Extractor/BinaryLogAnalyser.cs

@@ -0,0 +1,29 @@
+using System.Collections.Generic;
+using Microsoft.CodeAnalysis.CSharp;
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp
+{
+    public class BinaryLogAnalyser : Analyser
+    {
+        public BinaryLogAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix)
+            : base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix)
+        {
+        }
+
+        public void Initialize(
+            string cwd, string[] args, string outputPath, CSharpCompilation compilation,
+            IEnumerable<Microsoft.CodeAnalysis.SyntaxTree> generatedSyntaxTrees,
+            string compilationIdentifier, CommonOptions options)
+        {
+            base.compilation = compilation;
+            ExtractionContext = new BinaryLogExtractionContext(
+                cwd, args, outputPath, generatedSyntaxTrees, compilationIdentifier,
+                Logger, PathTransformer, options.QlTest);
+            this.options = options;
+            LogExtractorInfo();
+            SetReferencePaths();
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

@@ -8,6 +8,7 @@
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using Basic.CompilerLog.Util;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.CodeAnalysis.Text;
@@ -102,55 +103,154 @@
 
             try
             {
-                if (options.ProjectsToLoad.Any())
+                var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
+                var pathTransformer = new PathTransformer(canonicalPathCache);
+
+                if (options.BinaryLogPath is string binlogPath)
                 {
-                    AddSourceFilesFromProjects(options.ProjectsToLoad, options.CompilerArguments, logger);
+                    logger.LogInfo(" Running binary log analysis.");
+                    return RunBinaryLogAnalysis(analyzerStopwatch, options, binlogPath, logger, canonicalPathCache, pathTransformer);
                 }
-
-                var compilerVersion = new CompilerVersion(options);
-                if (compilerVersion.SkipExtraction)
+                else
                 {
-                    logger.LogWarning($"  Unrecognized compiler '{compilerVersion.SpecifiedCompiler}' because {compilerVersion.SkipReason}");
-                    return ExitCode.Ok;
+                    logger.LogInfo(" Running tracing analysis.");
+                    return RunTracingAnalysis(analyzerStopwatch, options, logger, canonicalPathCache, pathTransformer);
                 }
+            }
+            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
+            {
+                logger.LogError($"  Unhandled exception: {ex}");
+                return ExitCode.Errors;
+            }
+        }
 
-                var workingDirectory = Directory.GetCurrentDirectory();
-                var compilerArgs = options.CompilerArguments.ToArray();
+        private static ExitCode RunBinaryLogAnalysis(Stopwatch stopwatch, Options options, string binlogPath, ILogger logger, CanonicalPathCache canonicalPathCache, PathTransformer pathTransformer)
+        {
+            logger.LogInfo($"Reading compiler calls from binary log {binlogPath}");
+            try
+            {
+                using var fileStream = new FileStream(binlogPath, FileMode.Open, FileAccess.Read, FileShare.Read);
+                using var reader = BinaryLogReader.Create(fileStream);
 
-                var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
-                var pathTransformer = new PathTransformer(canonicalPathCache);
+                // Filter out compiler calls that aren't interesting for examination
+                static bool filter(CompilerCall compilerCall)
+                {
+                    return compilerCall.IsCSharp &&
+                        compilerCall.Kind == CompilerCallKind.Regular;
+                }
 
-                using var analyser = new TracingAnalyser(new LogProgressMonitor(logger), logger, pathTransformer, canonicalPathCache, options.AssemblySensitiveTrap);
+                var allCompilationData = reader.ReadAllCompilationData(filter);
+                var allFailed = true;
 
-                var compilerArguments = CSharpCommandLineParser.Default.Parse(
-                    compilerVersion.ArgsWithResponse,
-                    workingDirectory,
-                    compilerVersion.FrameworkPath,
-                    compilerVersion.AdditionalReferenceDirectories
-                    );
+                logger.LogInfo($"  Found {allCompilationData.Count} compilations in binary log");
 
-                if (compilerArguments is null)
+                foreach (var compilationData in allCompilationData)
                 {
-                    var sb = new StringBuilder();
-                    sb.Append("  Failed to parse command line: ").AppendList(" ", compilerArgs);
-                    logger.LogError(sb.ToString());
-                    ++analyser.CompilationErrors;
-                    return ExitCode.Failed;
-                }
+                    if (compilationData.GetCompilationAfterGenerators() is not CSharpCompilation compilation)
+                    {
+                        logger.LogError("  Compilation data is not C#");
+                        continue;
+                    }
 
-                if (!analyser.BeginInitialize(compilerVersion.ArgsWithResponse))
-                {
-                    logger.LogInfo("Skipping extraction since files have already been extracted");
-                    return ExitCode.Ok;
+                    var compilerCall = compilationData.CompilerCall;
+                    var diagnosticName = compilerCall.GetDiagnosticName();
+                    logger.LogInfo($"  Processing compilation {diagnosticName} at {compilerCall.ProjectDirectory}");
+                    var compilerArgs = compilerCall.GetArguments();
+
+                    var compilationIdentifierPath = string.Empty;
+                    try
+                    {
+                        compilationIdentifierPath = FileUtils.ConvertPathToSafeRelativePath(
+                            Path.GetRelativePath(Directory.GetCurrentDirectory(), compilerCall.ProjectDirectory));
+                    }
+                    catch (ArgumentException exc)
+                    {
+                        logger.LogWarning($"  Failed to get relative path for {compilerCall.ProjectDirectory} from current working directory {Directory.GetCurrentDirectory()}: {exc.Message}");
+                    }
+
+                    var args = reader.ReadCommandLineArguments(compilerCall);
+                    var generatedSyntaxTrees = compilationData.GetGeneratedSyntaxTrees();
+
+                    using var analyser = new BinaryLogAnalyser(new LogProgressMonitor(logger), logger, pathTransformer, canonicalPathCache, options.AssemblySensitiveTrap);
+
+                    var exit = Analyse(stopwatch, analyser, options,
+                        references => [() => compilation.References.ForEach(r => references.Add(r))],
+                        (analyser, syntaxTrees) => [() => syntaxTrees.AddRange(compilation.SyntaxTrees)],
+                        (syntaxTrees, references) => compilation,
+                        (compilation, options) => analyser.Initialize(
+                            compilerCall.ProjectDirectory,
+                            compilerArgs?.ToArray() ?? [],
+                            TracingAnalyser.GetOutputName(compilation, args),
+                            compilation,
+                            generatedSyntaxTrees,
+                            Path.Combine(compilationIdentifierPath, diagnosticName),
+                            options),
+                        () => { });
+
+                    switch (exit)
+                    {
+                        case ExitCode.Ok:
+                            allFailed &= false;
+                            logger.LogInfo($"  Compilation {diagnosticName} succeeded");
+                            break;
+                        case ExitCode.Errors:
+                            allFailed &= false;
+                            logger.LogWarning($"  Compilation {diagnosticName} had errors");
+                            break;
+                        case ExitCode.Failed:
+                            logger.LogWarning($"  Compilation {diagnosticName} failed");
+                            break;
+                    }
                 }
+                return allFailed ? ExitCode.Failed : ExitCode.Ok;
+            }
+            catch (IOException ex)
+            {
+                logger.LogError($"Failed to open binary log: {ex.Message}");
+                return ExitCode.Failed;
+            }
+        }
 
-                return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, analyzerStopwatch);
+        private static ExitCode RunTracingAnalysis(Stopwatch analyzerStopwatch, Options options, ILogger logger, CanonicalPathCache canonicalPathCache, PathTransformer pathTransformer)
+        {
+            if (options.ProjectsToLoad.Any())
+            {
+                AddSourceFilesFromProjects(options.ProjectsToLoad, options.CompilerArguments, logger);
             }
-            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
+
+            var compilerVersion = new CompilerVersion(options);
+            if (compilerVersion.SkipExtraction)
             {
-                logger.LogError($"  Unhandled exception: {ex}");
-                return ExitCode.Errors;
+                logger.LogWarning($"  Unrecognized compiler '{compilerVersion.SpecifiedCompiler}' because {compilerVersion.SkipReason}");
+                return ExitCode.Ok;
+            }
+
+            var workingDirectory = Directory.GetCurrentDirectory();
+            var compilerArgs = options.CompilerArguments.ToArray();
+            using var analyser = new TracingAnalyser(new LogProgressMonitor(logger), logger, pathTransformer, canonicalPathCache, options.AssemblySensitiveTrap);
+            var compilerArguments = CSharpCommandLineParser.Default.Parse(
+                compilerVersion.ArgsWithResponse,
+                workingDirectory,
+                compilerVersion.FrameworkPath,
+                compilerVersion.AdditionalReferenceDirectories
+                );
+
+            if (compilerArguments is null)
+            {
+                var sb = new StringBuilder();
+                sb.Append("  Failed to parse command line: ").AppendList(" ", compilerArgs);
+                logger.LogError(sb.ToString());
+                ++analyser.CompilationErrors;
+                return ExitCode.Failed;
+            }
+
+            if (!analyser.BeginInitialize(compilerVersion.ArgsWithResponse))
+            {
+                logger.LogInfo("Skipping extraction since files have already been extracted");
+                return ExitCode.Ok;
             }
+
+            return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, analyzerStopwatch);
         }
 
         private static void AddSourceFilesFromProjects(IEnumerable<string> projectsToLoad, IList<string> compilerArguments, ILogger logger)

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Options.cs

@@ -32,6 +32,11 @@ public sealed class Options : CommonOptions
         /// </summary>
         public bool AssemblySensitiveTrap { get; private set; } = false;
 
+        /// <summary>
+        /// The path to the binary log file, or null if unspecified.
+        /// </summary>
+        public string? BinaryLogPath { get; set; }
+
         public static Options CreateWithEnvironment(string[] arguments)
         {
             var options = new Options();
@@ -65,6 +70,9 @@ public override bool HandleOption(string key, string value)
                 case "load-sources-from-project":
                     ProjectsToLoad.Add(value);
                     return true;
+                case "binlog":
+                    BinaryLogPath = value;
+                    return true;
                 default:
                     return base.HandleOption(key, value);
             }

csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs

@@ -107,11 +107,8 @@ private bool LogRoslynArgs(IEnumerable<string> roslynArgs)
         /// <summary>
         /// Determine the path of the output dll/exe.
         /// </summary>
-        /// <param name="compilation">Information about the compilation.</param>
-        /// <param name="cancel">Cancellation token required.</param>
-        /// <returns>The filename.</returns>
-        private static string GetOutputName(CSharpCompilation compilation,
-            CSharpCommandLineArguments commandLineArguments)
+        internal static string GetOutputName(CSharpCompilation compilation,
+            CommandLineArguments commandLineArguments)
         {
             // There's no apparent way to access the output filename from the compilation,
             // so we need to re-parse the command line arguments.

csharp/extractor/Semmle.Extraction.CSharp/paket.references

@@ -1,3 +1,3 @@
 Microsoft.Build
 Microsoft.CodeAnalysis.CSharp
-
+Basic.CompilerLog.Util

csharp/extractor/Semmle.Extraction/Entities/File.cs

@@ -8,7 +8,8 @@ protected File(Context cx, string path)
             : base(cx, path)
         {
             originalPath = path;
-            transformedPathLazy = new Lazy<PathTransformer.ITransformedPath>(() => Context.ExtractionContext.PathTransformer.Transform(originalPath));
+            var adjustedPath = BinaryLogExtractionContext.GetAdjustedPath(Context.ExtractionContext, originalPath) ?? path;
+            transformedPathLazy = new Lazy<PathTransformer.ITransformedPath>(() => Context.ExtractionContext.PathTransformer.Transform(adjustedPath));
         }
 
         protected readonly string originalPath;