Merge commit 'ed659d7d8d6dfaf079df9544c21df441570d8559'

dotnet · Nov 14, 2024 · 3361f99 · 3361f99
2 parents d3a3e0f + ed659d7
commit 3361f99
Show file tree

Hide file tree

Showing 16 changed files with 260 additions and 186 deletions.
diff --git a/.azure/pipelines/ci-public.yml b/.azure/pipelines/ci-public.yml
@@ -116,6 +116,7 @@ stages:
       # The sign settings have been configured to
       - script: ./eng/build.cmd
                 -ci
+                -nativeToolsOnMachine
                 -arch x64
                 -pack
                 -all
@@ -588,11 +589,11 @@ stages:
         timeoutInMinutes: 240
         steps:
         # Build the shared framework
-        - script: ./eng/build.cmd -ci -nobl -all -pack -arch x64
+        - script: ./eng/build.cmd -ci -nativeToolsOnMachine -nobl -all -pack -arch x64
                 /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs)
           displayName: Build shared fx
         # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step.
-        - script: ./eng/build.cmd -ci -nobl -all -noBuildRepoTasks -noBuildNative -noBuild -test
+        - script: ./eng/build.cmd -ci -nativeToolsOnMachine -nobl -all -noBuildRepoTasks -noBuildNative -noBuild -test
                 -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true
                 /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs)
           displayName: Run build.cmd helix target
@@ -611,7 +612,7 @@ stages:
     parameters:
       platform:
         name: 'Managed'
-        container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9'
+        container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8'
         buildScript: './eng/build.sh $(_PublishArgs) --no-build-nodejs --no-build-repo-tasks $(_InternalRuntimeDownloadArgs)'
         skipPublishValidation: true
         jobProperties:

diff --git a/.azure/pipelines/ci.yml b/.azure/pipelines/ci.yml
@@ -187,6 +187,7 @@ extends:
           # The sign settings have been configured to
           - script: ./eng/build.cmd
                     -ci
+                    -nativeToolsOnMachine
                     -arch x64
                     -pack
                     -all
@@ -657,11 +658,11 @@ extends:
             timeoutInMinutes: 240
             steps:
             # Build the shared framework
-            - script: ./eng/build.cmd -ci -nobl -all -pack -arch x64
+            - script: ./eng/build.cmd -ci -nativeToolsOnMachine -nobl -all -pack -arch x64
                       /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs)
               displayName: Build shared fx
             # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step.
-            - script: ./eng/build.cmd -ci -nobl -all -noBuildRepoTasks -noBuildNative -noBuild -test
+            - script: ./eng/build.cmd -ci -nativeToolsOnMachine -nobl -all -noBuildRepoTasks -noBuildNative -noBuild -test
                       -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true
                       /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs)
               displayName: Run build.cmd helix target
@@ -681,7 +682,7 @@ extends:
           enableInternalSources: true
           platform:
             name: 'Managed'
-            container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9'
+            container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8'
             buildScript: './eng/build.sh $(_PublishArgs) --no-build-nodejs --no-build-repo-tasks $(_InternalRuntimeDownloadArgs)'
             skipPublishValidation: true
             jobProperties:

diff --git a/.azure/pipelines/jobs/codesign-xplat.yml b/.azure/pipelines/jobs/codesign-xplat.yml
@@ -13,7 +13,6 @@ jobs:
     jobDisplayName: "Code-sign ${{ parameters.inputName }} packages"
     agentOs: Windows
     installNodeJs: false
-    installJdk: false
     steps:
     - task: DownloadPipelineArtifact@2
       displayName: Download ${{ parameters.inputName }} artifacts

diff --git a/.azure/pipelines/jobs/default-build.yml b/.azure/pipelines/jobs/default-build.yml
@@ -68,7 +68,6 @@ parameters:
   container: ''
   enableRichCodeNavigation: ''
   installNodeJs: true
-  installJdk: true # Ignored unless agentOs == Windows.
   timeoutInMinutes: 180
   testRunTitle: $(AgentOsName)-$(BuildConfiguration)
   useHostedUbuntu: true
@@ -107,7 +106,7 @@ jobs:
       # See https://github.com/dotnet/arcade/blob/master/Documentation/ChoosingAMachinePool.md
       pool:
         ${{ if eq(parameters.agentOs, 'macOS') }}:
-          vmImage: macOS-12
+          vmImage: macOS-13
         ${{ if eq(parameters.agentOs, 'Linux') }}:
           ${{ if eq(parameters.useHostedUbuntu, true) }}:
             vmImage: ubuntu-20.04
@@ -116,7 +115,7 @@ jobs:
             demands: ImageOverride -equals Build.Ubuntu.2004.Amd64.Open
         ${{ if eq(parameters.agentOs, 'Windows') }}:
           name: $(DncEngPublicBuildPool)
-          demands: ImageOverride -equals 1es-windows-2022-open
+          demands: ImageOverride -equals windows.vs2022preview.amd64.open
       ${{ if ne(parameters.container, '') }}:
         container: ${{ parameters.container }}
       ${{ if ne(parameters.disableComponentGovernance, '') }}:
@@ -131,8 +130,6 @@ jobs:
       - DOTNET_CLI_HOME: $(System.DefaultWorkingDirectory)
       - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
       - TeamName: AspNetCore
-      - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}:
-        - JAVA_HOME: $(Agent.BuildDirectory)\.tools\jdk\win-x64
       - ${{ if ne(parameters.codeSign, true) }}:
         - _SignType: ''
       - LC_ALL: 'en_US.UTF-8'
@@ -167,8 +164,8 @@ jobs:
         - script: df -h
           displayName: Disk size
       - ${{ if eq(parameters.agentOs, 'macOS') }}:
-        - script: sudo xcode-select -s /Applications/Xcode_14.2.0.app/Contents/Developer
-          displayName: Use XCode 14.2.0
+        - script: sudo xcode-select -s /Applications/Xcode_15.2.0.app/Contents/Developer
+          displayName: Use XCode 15.2.0
       - checkout: self
         clean: true
       - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}:
@@ -181,14 +178,11 @@ jobs:
           displayName: Install Node 18.x
           inputs:
             versionSpec: 18.x
-      - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}:
-        - powershell: ./eng/scripts/InstallJdk.ps1
-          displayName: Install JDK 11
-        - ${{ if eq(parameters.isAzDOTestingJob, true) }}:
-          - powershell: |
-              Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\"
-              ./eng/scripts/InstallGoogleChrome.ps1
-            displayName: Install Chrome
+      - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}:
+        - powershell: |
+            Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\"
+            ./eng/scripts/InstallGoogleChrome.ps1
+          displayName: Install Chrome
       - ${{ if eq(parameters.agentOs, 'Windows') }}:
         - powershell: Write-Host "##vso[task.prependpath]$(DOTNET_CLI_HOME)\.dotnet\tools"
           displayName: Add dotnet tools to path
@@ -215,7 +209,7 @@ jobs:
                   ${{ step.env }}
       - ${{ if eq(parameters.steps, '')}}:
         - ${{ if eq(parameters.agentOs, 'Windows') }}:
-          - script: $(BuildDirectory)\build.cmd -ci -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs)
+          - script: $(BuildDirectory)\build.cmd -ci -nativeToolsOnMachine -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs)
               /p:DotNetSignType=$(_SignType)
             displayName: Run build.cmd
             env:
@@ -326,7 +320,7 @@ jobs:
       pool:
         ${{ if eq(parameters.agentOs, 'macOS') }}:
           name: Azure Pipelines
-          image: macOS-12
+          image: macOS-13
           os: macOS
         ${{ if eq(parameters.agentOs, 'Linux') }}:
           name: $(DncEngInternalBuildPool)
@@ -335,7 +329,7 @@ jobs:
         ${{ if eq(parameters.agentOs, 'Windows') }}:
           name: $(DncEngInternalBuildPool)
           # Visual Studio Enterprise - contains some stuff, like SQL Server and IIS Express, that we use for testing
-          image: 1es-windows-2022
+          image: windows.vs2022preview.amd64
           os: windows
       ${{ if ne(parameters.container, '') }}:
         container: ${{ parameters.container }}
@@ -351,8 +345,6 @@ jobs:
       - DOTNET_CLI_HOME: $(System.DefaultWorkingDirectory)
       - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
       - TeamName: AspNetCore
-      - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}:
-        - JAVA_HOME: $(Agent.BuildDirectory)\.tools\jdk\win-x64
       - ${{ if eq(parameters.codeSign, true) }}:
         - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}:
           - _SignType: real
@@ -390,8 +382,8 @@ jobs:
         - script: df -h
           displayName: Disk size
       - ${{ if eq(parameters.agentOs, 'macOS') }}:
-        - script: sudo xcode-select -s /Applications/Xcode_14.2.0.app/Contents/Developer
-          displayName: Use XCode 14.2.0
+        - script: sudo xcode-select -s /Applications/Xcode_15.2.0.app/Contents/Developer
+          displayName: Use XCode 15.2.0
       - checkout: self
         clean: true
       - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}:
@@ -404,14 +396,11 @@ jobs:
           displayName: Install Node 18.x
           inputs:
             versionSpec: 18.x
-      - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}:
-        - powershell: ./eng/scripts/InstallJdk.ps1
-          displayName: Install JDK 11
-        - ${{ if eq(parameters.isAzDOTestingJob, true) }}:
-          - powershell: |
-              Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\"
-              ./eng/scripts/InstallGoogleChrome.ps1
-            displayName: Install Chrome
+      - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}:
+        - powershell: |
+            Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\"
+            ./eng/scripts/InstallGoogleChrome.ps1
+          displayName: Install Chrome
       - ${{ if eq(parameters.agentOs, 'Windows') }}:
         - powershell: Write-Host "##vso[task.prependpath]$(DOTNET_CLI_HOME)\.dotnet\tools"
           displayName: Add dotnet tools to path
@@ -472,7 +461,7 @@ jobs:
                   ${{ step.env }}
       - ${{ if eq(parameters.steps, '')}}:
         - ${{ if eq(parameters.agentOs, 'Windows') }}:
-          - script: $(BuildDirectory)\build.cmd -ci -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs)
+          - script: $(BuildDirectory)\build.cmd -ci -nativeToolsOnMachine -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs)
               /p:DotNetSignType=$(_SignType)
             displayName: Run build.cmd
             env:

diff --git a/eng/build.ps1 b/eng/build.ps1
@@ -12,6 +12,9 @@ build projects, run tests, and generate code.
 .PARAMETER CI
 Sets up CI specific settings and variables.
 
+.PARAMETER NativeToolsOnMachine
+Turns on native tooling handling. On CI machines, promotes native tools listed in global.json to the path.
+
 .PARAMETER Restore
 Run restore.
 
@@ -121,6 +124,7 @@ Online version: https://github.com/dotnet/aspnetcore/blob/main/docs/BuildFromSou
 [CmdletBinding(PositionalBinding = $false, DefaultParameterSetName='Groups')]
 param(
     [switch]$CI,
+    [switch]$NativeToolsOnMachine,
 
     # Build lifecycle options
     [switch]$Restore,
@@ -304,58 +308,6 @@ $performDesktopBuild = ($BuildInstallers -and $Architecture -ne "arm") -or `
 $performDotnetBuild = $BuildJava -or $BuildManaged -or $BuildNodeJS -or `
     ($All -and -not ($NoBuildJava -and $NoBuildManaged -and $NoBuildNodeJS)) -or `
     ($Projects -and -not ($BuildInstallers -or $specifiedBuildNative))
-$foundJdk = $false
-$javac = Get-Command javac -ErrorAction Ignore -CommandType Application
-$localJdkPath = "$PSScriptRoot\..\.tools\jdk\win-x64\"
-if (Test-Path "$localJdkPath\bin\javac.exe") {
-    $foundJdk = $true
-    Write-Host -f Magenta "Detected JDK in $localJdkPath (via local repo convention)"
-    $env:JAVA_HOME = $localJdkPath
-}
-elseif ($env:JAVA_HOME) {
-    if (-not (Test-Path "${env:JAVA_HOME}\bin\javac.exe")) {
-        Write-Error "The environment variable JAVA_HOME was set, but ${env:JAVA_HOME}\bin\javac.exe does not exist. Remove JAVA_HOME or update it to the correct location for the JDK. See https://www.bing.com/search?q=java_home for details."
-    }
-    else {
-        Write-Host -f Magenta "Detected JDK in ${env:JAVA_HOME} (via JAVA_HOME)"
-        $foundJdk = $true
-    }
-}
-elseif ($javac) {
-    $foundJdk = $true
-    $javaHome = Split-Path -Parent (Split-Path -Parent $javac.Path)
-    $env:JAVA_HOME = $javaHome
-    Write-Host -f Magenta "Detected JDK in $javaHome (via PATH)"
-}
-else {
-    try {
-        $jdkRegistryKeys = @(
-            "HKLM:\SOFTWARE\JavaSoft\JDK",  # for JDK 10+
-            "HKLM:\SOFTWARE\JavaSoft\Java Development Kit"  # fallback for JDK 8
-        )
-        $jdkRegistryKey = $jdkRegistryKeys | Where-Object { Test-Path $_ } | Select-Object -First 1
-        if ($jdkRegistryKey) {
-            $jdkVersion = (Get-Item $jdkRegistryKey | Get-ItemProperty -name CurrentVersion).CurrentVersion
-            $javaHome = (Get-Item $jdkRegistryKey\$jdkVersion | Get-ItemProperty -Name JavaHome).JavaHome
-            if (Test-Path "${javaHome}\bin\javac.exe") {
-                $env:JAVA_HOME = $javaHome
-                Write-Host -f Magenta "Detected JDK $jdkVersion in $env:JAVA_HOME (via registry)"
-                $foundJdk = $true
-            }
-        }
-    }
-    catch {
-        Write-Verbose "Failed to detect Java: $_"
-    }
-}
-
-if ($env:PATH -notlike "*${env:JAVA_HOME}*") {
-    $env:PATH = "$(Join-Path $env:JAVA_HOME bin);${env:PATH}"
-}
-
-if (-not $foundJdk -and $RunBuild -and ($All -or $BuildJava) -and -not $NoBuildJava) {
-    Write-Error "Could not find the JDK. Either run $PSScriptRoot\scripts\InstallJdk.ps1 to install for this repo, or install the JDK globally on your machine (see $PSScriptRoot\..\docs\BuildFromSource.md for details)."
-}
 
 # Initialize global variables need to be set before the import of Arcade is imported
 $restore = $RunRestore
@@ -384,6 +336,61 @@ Remove-Item variable:global:_MSBuildExe -ea Ignore
 # Import Arcade
 . "$PSScriptRoot/common/tools.ps1"
 
+function LocateJava {
+    $foundJdk = $false
+    $javac = Get-Command javac -ErrorAction Ignore -CommandType Application
+    $localJdkPath = "$PSScriptRoot\..\.tools\jdk\win-x64\"
+    if (Test-Path "$localJdkPath\bin\javac.exe") {
+        $foundJdk = $true
+        Write-Host -f Magenta "Detected JDK in $localJdkPath (via local repo convention)"
+        $env:JAVA_HOME = $localJdkPath
+    }
+    elseif ($env:JAVA_HOME) {
+        if (-not (Test-Path "${env:JAVA_HOME}\bin\javac.exe")) {
+            Write-Error "The environment variable JAVA_HOME was set, but ${env:JAVA_HOME}\bin\javac.exe does not exist. Remove JAVA_HOME or update it to the correct location for the JDK. See https://www.bing.com/search?q=java_home for details."
+        }
+        else {
+            Write-Host -f Magenta "Detected JDK in ${env:JAVA_HOME} (via JAVA_HOME)"
+            $foundJdk = $true
+        }
+    }
+    elseif ($javac) {
+        $foundJdk = $true
+        $javaHome = Split-Path -Parent (Split-Path -Parent $javac.Path)
+        $env:JAVA_HOME = $javaHome
+        Write-Host -f Magenta "Detected JDK in $javaHome (via PATH)"
+    }
+    else {
+        try {
+            $jdkRegistryKeys = @(
+                "HKLM:\SOFTWARE\JavaSoft\JDK",  # for JDK 10+
+                "HKLM:\SOFTWARE\JavaSoft\Java Development Kit"  # fallback for JDK 8
+            )
+            $jdkRegistryKey = $jdkRegistryKeys | Where-Object { Test-Path $_ } | Select-Object -First 1
+            if ($jdkRegistryKey) {
+                $jdkVersion = (Get-Item $jdkRegistryKey | Get-ItemProperty -name CurrentVersion).CurrentVersion
+                $javaHome = (Get-Item $jdkRegistryKey\$jdkVersion | Get-ItemProperty -Name JavaHome).JavaHome
+                if (Test-Path "${javaHome}\bin\javac.exe") {
+                    $env:JAVA_HOME = $javaHome
+                    Write-Host -f Magenta "Detected JDK $jdkVersion in $env:JAVA_HOME (via registry)"
+                    $foundJdk = $true
+                }
+            }
+        }
+        catch {
+            Write-Verbose "Failed to detect Java: $_"
+        }
+    }
+
+    if ($env:PATH -notlike "*${env:JAVA_HOME}*") {
+        $env:PATH = "$(Join-Path $env:JAVA_HOME bin);${env:PATH}"
+    }
+
+    if (-not $foundJdk -and $RunBuild -and ($All -or $BuildJava) -and -not $NoBuildJava) {
+        Write-Error "Could not find the JDK. Either run $PSScriptRoot\scripts\InstallJdk.ps1 to install for this repo, or install the JDK globally on your machine (see $PSScriptRoot\..\docs\BuildFromSource.md for details)."
+    }
+}
+
 # Add default .binlog location if not already on the command line. tools.ps1 does not handle this; it just checks
 # $BinaryLog, $CI and $ExcludeCIBinarylog values for an error case. But tools.ps1 provides a nice function to help.
 if ($BinaryLog) {
@@ -416,6 +423,17 @@ try {
     $tmpRestore = $restore
     $restore = $true
 
+    # Initialize the native tools before locating java.
+    if ($NativeToolsOnMachine) {
+        $env:NativeToolsOnMachine=$true
+        # Do not promote native tools except in cases where -NativeToolsOnMachine is passed.
+        # Currently the JDK is laid out in an incorrect pattern: https://github.com/dotnet/dnceng/issues/2185
+        InitializeNativeTools
+    }
+
+    # Locate java, now that we may have java available after initializing native tools.
+    LocateJava
+
     $toolsetBuildProj = InitializeToolset
 
     $restore = $tmpRestore

diff --git a/eng/configure-toolset.sh b/eng/configure-toolset.sh
@@ -5,3 +5,6 @@
 if [ "${DotNetBuildFromSource:-false}" = false ]; then
     use_installed_dotnet_cli="false"
 fi
+
+# Working around issue https://github.com/dotnet/arcade/issues/2673
+DisableNativeToolsetInstalls=true