Merge pull request #13999 from github/mbg/csharp/standalone/dotnet-ve…

…rsion C# Standalone: Install .NET SDK specified in `global.json`
github · Sep 7, 2023 · 38892bb · 38892bb
2 parents ec0529d + ccbc6f4
commit 38892bb
Show file tree

Hide file tree

Showing 10 changed files with 136 additions and 30 deletions.
diff --git a/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs b/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs
@@ -557,7 +557,7 @@ public void TestVcVarsAllBatFiles()
         [Fact]
         public void TestLinuxBuildlessExtractionSuccess()
         {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone --references:."] = 0;
+            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
             actions.FileExists["csharp.log"] = true;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
@@ -571,7 +571,7 @@ public void TestLinuxBuildlessExtractionSuccess()
         [Fact]
         public void TestLinuxBuildlessExtractionFailed()
         {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone --references:."] = 10;
+            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 10;
             actions.FileExists["csharp.log"] = true;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
@@ -585,7 +585,7 @@ public void TestLinuxBuildlessExtractionFailed()
         [Fact]
         public void TestLinuxBuildlessExtractionSolution()
         {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln --references:."] = 0;
+            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln"] = 0;
             actions.FileExists["csharp.log"] = true;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
@@ -873,7 +873,7 @@ public void TestSkipNugetMsBuild()
         [Fact]
         public void TestSkipNugetBuildless()
         {
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln --references:. --skip-nuget"] = 0;
+            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln --skip-nuget"] = 0;
             actions.FileExists["csharp.log"] = true;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";

diff --git a/csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs b/csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs
@@ -48,8 +48,11 @@ public override BuildScript GetBuildScript()
                     attempt = new BuildCommandRule(DotNetRule.WithDotNet).Analyse(this, false) & CheckExtractorRun(true);
                     break;
                 case CSharpBuildStrategy.Buildless:
-                    // No need to check that the extractor has been executed in buildless mode
-                    attempt = new StandaloneBuildRule().Analyse(this, false);
+                    attempt = DotNetRule.WithDotNet(this, (dotNetPath, env) =>
+                        {
+                            // No need to check that the extractor has been executed in buildless mode
+                            return new StandaloneBuildRule(dotNetPath).Analyse(this, false);
+                        });
                     break;
                 case CSharpBuildStrategy.MSBuild:
                     attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true);

diff --git a/csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs b/csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs
@@ -70,7 +70,16 @@ public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool au
                 });
         }
 
-        private static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, Func<string?, IDictionary<string, string>?, BuildScript> f)
+        /// <summary>
+        /// Returns a script that attempts to download relevant version(s) of the
+        /// .NET Core SDK, followed by running the script generated by <paramref name="f"/>.
+        ///
+        /// The arguments to <paramref name="f"/> are the path to the directory in which the
+        /// .NET Core SDK(s) were installed and any additional required environment
+        /// variables needed by the installed .NET Core (<code>null</code> when no variables
+        /// are needed).
+        /// </summary>
+        public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, Func<string?, IDictionary<string, string>?, BuildScript> f)
         {
             var installDir = builder.Actions.PathCombine(builder.Options.RootDirectory, ".dotnet");
             var installScript = DownloadDotNet(builder, installDir);

diff --git a/csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs b/csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs
@@ -8,6 +8,13 @@ namespace Semmle.Autobuild.CSharp
     /// </summary>
     internal class StandaloneBuildRule : IBuildRule<CSharpAutobuildOptions>
     {
+        private readonly string? dotNetPath;
+
+        internal StandaloneBuildRule(string? dotNetPath)
+        {
+            this.dotNetPath = dotNetPath;
+        }
+
         public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
         {
             BuildScript GetCommand(string? solution)
@@ -28,13 +35,17 @@ BuildScript GetCommand(string? solution)
                 if (solution is not null)
                     cmd.QuoteArgument(solution);
 
-                cmd.Argument("--references:.");
-
                 if (!builder.Options.NugetRestore)
                 {
                     cmd.Argument("--skip-nuget");
                 }
 
+                if (!string.IsNullOrEmpty(this.dotNetPath))
+                {
+                    cmd.Argument("--dotnet");
+                    cmd.QuoteArgument(this.dotNetPath);
+                }
+
                 return cmd.Script;
             }
 

diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs
@@ -15,14 +15,24 @@ internal class AssemblyCache
         /// <summary>
         /// Locate all reference files and index them.
         /// </summary>
-        /// <param name="dirs">Directories to search.</param>
+        /// <param name="paths">
+        /// Paths to search. Directories are searched recursively. Files are added directly to the
+        /// assembly cache.
+        /// </param>
         /// <param name="progressMonitor">Callback for progress.</param>
-        public AssemblyCache(IEnumerable<string> dirs, ProgressMonitor progressMonitor)
+        public AssemblyCache(IEnumerable<string> paths, ProgressMonitor progressMonitor)
         {
-            foreach (var dir in dirs)
+            foreach (var path in paths)
             {
-                progressMonitor.FindingFiles(dir);
-                AddReferenceDirectory(dir);
+                if (File.Exists(path))
+                {
+                    pendingDllsToIndex.Enqueue(path);
+                }
+                else
+                {
+                    progressMonitor.FindingFiles(path);
+                    AddReferenceDirectory(path);
+                }
             }
             IndexReferences();
         }

diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
@@ -31,6 +31,7 @@ public sealed class DependencyManager : IDisposable
         private readonly FileContent fileContent;
         private readonly TemporaryDirectory packageDirectory;
         private TemporaryDirectory? razorWorkingDirectory;
+        private readonly Git git;
 
 
         /// <summary>
@@ -48,7 +49,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
 
             try
             {
-                this.dotnet = new DotNet(progressMonitor);
+                this.dotnet = new DotNet(options, progressMonitor);
             }
             catch
             {
@@ -68,7 +69,11 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
                 ? new[] { options.SolutionFile }
                 : allFiles.SelectFileNamesByExtension(".sln");
 
-            var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList();
+            // If DLL reference paths are specified on the command-line, use those to discover
+            // assemblies. Otherwise (the default), query the git CLI to determine which DLL files
+            // are tracked as part of the repository.
+            this.git = new Git(this.progressMonitor);
+            var dllDirNames = options.DllDirs.Count == 0 ? this.git.ListFiles("*.dll") : options.DllDirs.Select(Path.GetFullPath).ToList();
 
             // Find DLLs in the .Net / Asp.Net Framework
             if (options.ScanNetFrameworkDlls)
@@ -98,13 +103,9 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
                     progressMonitor.MissingNuGet();
                 }
 
-                // TODO: remove the below when the required SDK is installed
-                using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories)))
-                {
-                    Restore(solutions);
-                    Restore(allProjects);
-                    DownloadMissingPackages(allFiles);
-                }
+                Restore(solutions);
+                Restore(allProjects);
+                DownloadMissingPackages(allFiles);
             }
 
             assemblyCache = new AssemblyCache(dllDirNames, progressMonitor);
@@ -157,7 +158,6 @@ private void GenerateSourceFilesFromWebViews(List<FileInfo> allFiles)
             {
                 progressMonitor.LogInfo($"Found {views.Length} cshtml and razor files.");
 
-                // TODO: use SDK specified in global.json
                 var sdk = new Sdk(dotnet).GetNewestSdk();
                 if (sdk != null)
                 {

diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyOptions.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyOptions.cs
@@ -51,6 +51,11 @@ public interface IDependencyOptions
         /// The number of threads to use.
         /// </summary>
         int Threads { get; }
+
+        /// <summary>
+        /// The path to the local ".dotnet" directory, if any.
+        /// </summary>
+        string? DotNetPath { get; }
     }
 
     public class DependencyOptions : IDependencyOptions
@@ -73,5 +78,7 @@ public bool ExcludesFile(string path) =>
             Excludes.Any(path.Contains);
 
         public int Threads { get; set; } = EnvironmentVariables.GetDefaultNumberOfThreads();
+
+        public string? DotNetPath { get; set; } = null;
     }
 }
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.IO;
 using Semmle.Util;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
@@ -10,12 +11,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     /// </summary>
     internal class DotNet : IDotNet
     {
-        private const string dotnet = "dotnet";
         private readonly ProgressMonitor progressMonitor;
+        private readonly string dotnet;
 
-        public DotNet(ProgressMonitor progressMonitor)
+        public DotNet(IDependencyOptions options, ProgressMonitor progressMonitor)
         {
             this.progressMonitor = progressMonitor;
+            this.dotnet = Path.Combine(options.DotNetPath ?? string.Empty, "dotnet");
             Info();
         }
 
@@ -29,7 +31,7 @@ private void Info()
             }
         }
 
-        private static ProcessStartInfo MakeDotnetStartInfo(string args, bool redirectStandardOutput) =>
+        private ProcessStartInfo MakeDotnetStartInfo(string args, bool redirectStandardOutput) =>
             new ProcessStartInfo(dotnet, args)
             {
                 UseShellExecute = false,
@@ -39,7 +41,7 @@ private static ProcessStartInfo MakeDotnetStartInfo(string args, bool redirectSt
         private bool RunCommand(string args)
         {
             progressMonitor.RunningProcess($"{dotnet} {args}");
-            using var proc = Process.Start(MakeDotnetStartInfo(args, redirectStandardOutput: false));
+            using var proc = Process.Start(this.MakeDotnetStartInfo(args, redirectStandardOutput: false));
             proc?.WaitForExit();
             var exitCode = proc?.ExitCode ?? -1;
             if (exitCode != 0)
@@ -77,7 +79,7 @@ public bool AddPackage(string folder, string package)
         private IList<string> GetListed(string args, string artifact)
         {
             progressMonitor.RunningProcess($"{dotnet} {args}");
-            var pi = MakeDotnetStartInfo(args, redirectStandardOutput: true);
+            var pi = this.MakeDotnetStartInfo(args, redirectStandardOutput: true);
             var exitCode = pi.ReadOutput(out var artifacts);
             if (exitCode != 0)
             {

diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Git.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Git.cs
@@ -0,0 +1,61 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    /// <summary>
+    /// Utilities for querying information from the git CLI.
+    /// </summary>
+    internal class Git
+    {
+        private readonly ProgressMonitor progressMonitor;
+        private const string git = "git";
+
+        public Git(ProgressMonitor progressMonitor)
+        {
+            this.progressMonitor = progressMonitor;
+        }
+
+        /// <summary>
+        /// Lists all files matching <paramref name="pattern"/> which are tracked in the
+        /// current git repository.
+        /// </summary>
+        /// <param name="pattern">The file pattern.</param>
+        /// <returns>A list of all tracked files which match <paramref name="pattern"/>.</returns>
+        /// <exception cref="Exception"></exception>
+        public List<string> ListFiles(string pattern)
+        {
+            var results = new List<string>();
+            var args = string.Join(' ', "ls-files", $"\"{pattern}\"");
+
+            progressMonitor.RunningProcess($"{git} {args}");
+            var pi = new ProcessStartInfo(git, args)
+            {
+                UseShellExecute = false,
+                RedirectStandardOutput = true
+            };
+
+            using var p = new Process() { StartInfo = pi };
+            p.OutputDataReceived += new DataReceivedEventHandler((sender, e) =>
+            {
+                if (!string.IsNullOrWhiteSpace(e.Data))
+                {
+                    results.Add(e.Data);
+                }
+            });
+            p.Start();
+            p.BeginOutputReadLine();
+            p.WaitForExit();
+
+            if (p.ExitCode != 0)
+            {
+                progressMonitor.CommandFailed(git, args, p.ExitCode);
+                throw new Exception($"{git} {args} failed");
+            }
+
+            return results;
+        }
+
+    }
+}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs
@@ -50,6 +50,9 @@ public override bool HandleOption(string key, string value)
                 case "references":
                     dependencies.DllDirs.Add(value);
                     return true;
+                case "dotnet":
+                    dependencies.DotNetPath = value;
+                    return true;
                 default:
                     return base.HandleOption(key, value);
             }