Merge pull request #1858 from aspnet/darc-release/3.0-preview9-cdafba…

…a3-61eb-46e8-85f6-0ebded42b1bc

[release/3.0-preview9] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -3,9 +3,9 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19416.16">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19421.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>0e36c2410b72166a1b9a67142e652225e22feada</Sha>
+      <Sha>7aa107d818fe87e627154c0331d6de5d47f39a45</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/post-build/sourcelink-validation.ps1

@@ -1,8 +1,8 @@
 param(
   [Parameter(Mandatory=$true)][string] $InputPath,              # Full path to directory where Symbols.NuGet packages to be checked are stored
   [Parameter(Mandatory=$true)][string] $ExtractPath,            # Full path to directory where the packages will be extracted during validation
-  [Parameter(Mandatory=$true)][string] $GHRepoName,             # GitHub name of the repo including the Org. E.g., dotnet/arcade
-  [Parameter(Mandatory=$true)][string] $GHCommit,               # GitHub commit SHA used to build the packages
+  [Parameter(Mandatory=$false)][string] $GHRepoName,            # GitHub name of the repo including the Org. E.g., dotnet/arcade
+  [Parameter(Mandatory=$false)][string] $GHCommit,              # GitHub commit SHA used to build the packages
   [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion    # Version of SourceLink CLI to use
 )
 
@@ -13,6 +13,12 @@ param(
 # all files present in the repo at a specific commit point.
 $global:RepoFiles = @{}
 
+# Maximum number of jobs to run in parallel
+$MaxParallelJobs = 6
+
+# Wait time between check for system load
+$SecondsBetweenLoadChecks = 10
+
 $ValidatePackage = {
   param( 
     [string] $PackagePath                                 # Full path to a Symbols.NuGet package
@@ -22,8 +28,8 @@ $ValidatePackage = {
 
   # Ensure input file exist
   if (!(Test-Path $PackagePath)) {
-    Write-PipelineTaskError "Input file does not exist: $PackagePath"
-    ExitWithExitCode 1
+    Write-Host "Input file does not exist: $PackagePath"
+    return 1
   }
 
   # Extensions for which we'll look for SourceLink information
@@ -38,7 +44,7 @@ $ValidatePackage = {
 
   Add-Type -AssemblyName System.IO.Compression.FileSystem
 
-  [System.IO.Directory]::CreateDirectory($ExtractPath);
+  [System.IO.Directory]::CreateDirectory($ExtractPath)  | Out-Null
 
   try {
     $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
@@ -138,62 +144,86 @@ $ValidatePackage = {
 
   if ($FailedFiles -eq 0) {
     Write-Host "Passed."
+    return 0
   }
   else {
-    Write-PipelineTaskError "$PackagePath has broken SourceLink links."
+    Write-Host "$PackagePath has broken SourceLink links."
+    return 1
   }
 }
 
 function ValidateSourceLinkLinks {
-  if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) {
+  if ($GHRepoName -ne "" -and !($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) {
     if (!($GHRepoName -Match "^[^\s-]+-[^\s]+$")) {
-      Write-PipelineTaskError "GHRepoName should be in the format <org>/<repo> or <org>-<repo>"
+      Write-PipelineTaskError "GHRepoName should be in the format <org>/<repo> or <org>-<repo>. '$GHRepoName'"
       ExitWithExitCode 1
     }
     else {
       $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2';
     }
   }
 
-  if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) {
-    Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string"
+  if ($GHCommit -ne "" -and !($GHCommit -Match "^[0-9a-fA-F]{40}$")) {
+    Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'"
     ExitWithExitCode 1
   }
 
-  $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1")
-  $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript")
+  if ($GHRepoName -ne "" -and $GHCommit -ne "") {
+    $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1")
+    $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript")
 
-  try {
-    # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
-    $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree
+    try {
+      # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
+      $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree
 
-    foreach ($file in $Data) {
-      $Extension = [System.IO.Path]::GetExtension($file.path)
+      foreach ($file in $Data) {
+        $Extension = [System.IO.Path]::GetExtension($file.path)
 
-      if ($CodeExtensions.Contains($Extension)) {
-        $RepoFiles[$file.path] = 1
+        if ($CodeExtensions.Contains($Extension)) {
+          $RepoFiles[$file.path] = 1
+        }
       }
     }
+    catch {
+      Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching."
+    }
   }
-  catch {
-    Write-PipelineTaskError "Problems downloading the list of files from the repo. Url used: $RepoTreeURL"
-    Write-Host $_
-    ExitWithExitCode 1
+  elseif ($GHRepoName -ne "" -or $GHCommit -ne "") {
+    Write-Host "For using the http caching mechanism both GHRepoName and GHCommit should be informed."
   }
 
   if (Test-Path $ExtractPath) {
     Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue
   }
 
   # Process each NuGet package in parallel
-  $Jobs = @()
   Get-ChildItem "$InputPath\*.symbols.nupkg" |
     ForEach-Object {
-      $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName
+      Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null
+      $NumJobs = @(Get-Job -State 'Running').Count
+
+      while ($NumJobs -ge $MaxParallelJobs) {
+        Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."
+        sleep $SecondsBetweenLoadChecks
+        $NumJobs = @(Get-Job -State 'Running').Count
+      }
+
+      foreach ($Job in @(Get-Job -State 'Completed')) {
+        Receive-Job -Id $Job.Id
+        Remove-Job -Id $Job.Id
+      }
     }
 
+  $ValidationFailures = 0
   foreach ($Job in $Jobs) {
-    Wait-Job -Id $Job.Id | Receive-Job
+    $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+    if ($jobResult -ne "0") {
+      $ValidationFailures++
+    }
+  }
+  if ($ValidationFailures -gt 0) {
+    Write-PipelineTaskError " $ValidationFailures package(s) failed validation."
+    ExitWithExitCode 1
   }
 }
 

eng/common/templates/post-build/channels/netcore-dev-5.yml

@@ -2,6 +2,7 @@ parameters:
   enableSymbolValidation: true
   symbolPublishingAdditionalParameters: ''
   artifactsPublishingAdditionalParameters: ''
+  publishInstallersAndChecksums: false
 
 stages:
 - stage: NetCore_Dev5_Publish
@@ -101,7 +102,12 @@ stages:
             /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' 
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' 
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' 
-            /p:Configuration=Release 
+            /p:Configuration=Release
+            /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+            /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+            /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+            /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+            /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
             ${{ parameters.artifactsPublishingAdditionalParameters }}
 
       - task: NuGetCommand@2

eng/common/templates/post-build/channels/netcore-tools-latest.yml

@@ -2,6 +2,7 @@ parameters:
   enableSymbolValidation: true
   symbolPublishingAdditionalParameters: ''
   artifactsPublishingAdditionalParameters: ''
+  publishInstallersAndChecksums: false
 
 stages:
 - stage: NetCore_Tools_Latest_Publish
@@ -101,7 +102,12 @@ stages:
             /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' 
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' 
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' 
-            /p:Configuration=Release 
+            /p:Configuration=Release
+            /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+            /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+            /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+            /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+            /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
             ${{ parameters.artifactsPublishingAdditionalParameters }}
 
       - task: NuGetCommand@2

eng/common/templates/post-build/channels/public-dev-release.yml

@@ -2,6 +2,7 @@ parameters:
   enableSymbolValidation: true
   symbolPublishingAdditionalParameters: ''
   artifactsPublishingAdditionalParameters: ''
+  publishInstallersAndChecksums: false
 
 stages:
 - stage: Publish
@@ -102,6 +103,11 @@ stages:
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' 
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' 
             /p:Configuration=Release 
+            /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+            /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+            /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+            /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+            /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
             ${{ parameters.artifactsPublishingAdditionalParameters }}
 
       - task: NuGetCommand@2

eng/common/templates/post-build/channels/public-release.yml

@@ -90,6 +90,7 @@ stages:
             /p:IsInternalBuild=$(IsInternalBuild)
             /p:RepositoryName=$(Build.Repository.Name)
             /p:CommitSha=$(Build.SourceVersion)
+            /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' 
             /p:AzureStorageAccountName=$(ProxyBackedFeedsAccountName)
             /p:AzureStorageAccountKey=$(dotnetfeed-storage-access-key-1)
             /p:AzureDevOpsFeedsBaseUrl=$(dotnetfeed-internal-private-feed-url)

eng/common/templates/post-build/channels/public-validation-release.yml

@@ -1,5 +1,6 @@
 parameters:
   artifactsPublishingAdditionalParameters: ''
+  publishInstallersAndChecksums: false
 
 stages:
 - stage: PVR_Publish
@@ -65,7 +66,12 @@ stages:
             /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' 
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' 
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' 
-            /p:Configuration=Release 
+            /p:Configuration=Release
+            /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+            /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+            /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+            /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+            /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
             ${{ parameters.artifactsPublishingAdditionalParameters }}
 
       - task: NuGetCommand@2

eng/common/templates/post-build/common-variables.yml

@@ -1,5 +1,6 @@
 variables:
   - group: Publish-Build-Assets
+  - group: DotNet-DotNetCli-Storage
 
   # .NET Core 3 Dev
   - name: PublicDevRelease_30_Channel_Id
@@ -45,3 +46,9 @@ variables:
     value: 3.0.0
   - name: SymbolToolVersion
     value: 1.0.1
+
+  # Default locations for Installers and checksums
+  - name: ChecksumsBlobFeedUrl
+    value: https://dotnetcli.blob.core.windows.net/dotnet/index.json
+  - name: InstallersBlobFeedUrl
+    value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json

eng/common/templates/post-build/post-build.yml

@@ -3,6 +3,7 @@ parameters:
   enableSigningValidation: true
   enableSymbolValidation: true
   enableNugetValidation: true
+  publishInstallersAndChecksums: false
   SDLValidationParameters:
     enable: false
     params: ''
@@ -85,6 +86,7 @@ stages:
               -GHRepoName $(Build.Repository.Name) 
               -GHCommit $(Build.SourceVersion)
               -SourcelinkCliVersion $(SourceLinkCLIVersion)
+          continueOnError: true
 
   - ${{ if eq(parameters.SDLValidationParameters.enable, 'true') }}:
     - template: /eng/common/templates/job/execute-sdl.yml
@@ -96,22 +98,26 @@ stages:
     enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
     symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
     artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
 
 - template: \eng\common\templates\post-build\channels\public-dev-release.yml
   parameters:
     enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
     symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
     artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
 
 - template: \eng\common\templates\post-build\channels\netcore-tools-latest.yml
   parameters:
     enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
     symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
     artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
 
 - template: \eng\common\templates\post-build\channels\public-validation-release.yml
   parameters:
     artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
 
 - template: \eng\common\templates\post-build\channels\public-release.yml
   parameters:

global.json

@@ -11,6 +11,6 @@
     }
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19416.16"
+    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19421.1"
   }
 }