Isolate package flow between repos (#18557)

As a prerequisite to enabling parallel builds of the repos in the VMR, it's necessary to isolate the package flow between repos. This ensures that repo dependencies are defined correctly. Prior to these changes, the packages outputted by repos went to the same directory (Shipping or NonShipping) and those two directories were used as the package feed inputs to dependent repos. This would be dangerous in the context of running builds in parallel across repos because any ill-defined dependencies could lead to race conditions. For example, consider the msbuild repo's dependency on System.Text.Json from the runtime repo. If runtime was not defined as a dependency of msbuild, then there's no guarantee that System.Text.Json will exist when it restores it. Based on timing, it may exist in one build and then not exist in another build.

To solve this problem, the packages output from a repos build are placed in a repo-specific package location as a sub-directory of the package location (Shipping or NonShipping). Previously, all repos would output their packages to `artifacts/packages/Release/[NonShipping|Shipping]`. With these changes, they output to `artifacts/packages/Release/[NonShipping|Shipping]/<repo-name>`. This isolates all packages on a per-repo basis. The next step is to provide access to these packages based on dependencies. Going back to msbuild's dependency on runtime, this is accomplished by modifying msbuild's nuget.config file to include feeds specific to runtime:

```xml
<add key="source-built-runtime" value="/vmr/artifacts/packages/Release/Shipping/runtime/" />
<add key="source-built-transport-runtime" value="/vmr/artifacts/packages/Release/NonShipping/runtime/" />
```

src/SourceBuild/content/repo-projects/Directory.Build.props

@@ -33,6 +33,11 @@
     <NuGetConfigFile Condition="'$(OriginalNuGetConfigFile)' != ''">$(BaseIntermediateOutputPath)$([System.IO.Path]::GetFileName('$(OriginalNuGetConfigFile)'))</NuGetConfigFile>
 
     <SourceBuiltSdksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'source-built-sdks'))</SourceBuiltSdksDir>
+    <RepoArtifactsShippingPackagesDir Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$([MSBuild]::NormalizeDirectory('$(ArtifactsShippingPackagesDir)', '$(RepositoryName)'))</RepoArtifactsShippingPackagesDir>
+    <RepoArtifactsShippingPackagesDir Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</RepoArtifactsShippingPackagesDir>
+    <RepoArtifactsNonShippingPackagesDir Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$([MSBuild]::NormalizeDirectory('$(ArtifactsNonShippingPackagesDir)', '$(RepositoryName)'))</RepoArtifactsNonShippingPackagesDir>
+    <RepoArtifactsNonShippingPackagesDir Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</RepoArtifactsNonShippingPackagesDir>
+
 
     <!-- Set the bootstrap version to the VMR's version if empty. (no bootstrap set). -->
     <ArcadeBootstrapVersion>$([MSBuild]::ValueOrDefault('$(ARCADE_BOOTSTRAP_VERSION)', '$(ArcadeSdkVersion)'))</ArcadeBootstrapVersion>

src/SourceBuild/content/repo-projects/aspnetcore.proj

@@ -22,11 +22,16 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
-    <RepositoryReference Include="runtime" />
-    <RepositoryReference Include="msbuild" />
     <RepositoryReference Include="roslyn" />
-    <RepositoryReference Include="roslyn-analyzers" />
+    <RepositoryReference Include="runtime" />
+    <RepositoryReference Include="xdt" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="nuget-client" />
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
+    <RepositoryReference Include="symreader" />
   </ItemGroup>
 
   <ItemGroup>

src/SourceBuild/content/repo-projects/cecil.proj

@@ -4,4 +4,8 @@
     <RepositoryReference Include="arcade" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-reference-packages" />
+  </ItemGroup>
+
 </Project>

src/SourceBuild/content/repo-projects/command-line-api.proj

@@ -10,4 +10,8 @@
     <RepositoryReference Include="arcade" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-reference-packages" />
+  </ItemGroup>
+
 </Project>

src/SourceBuild/content/repo-projects/deployment-tools.proj

@@ -2,7 +2,11 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="runtime" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
   <ItemGroup>

src/SourceBuild/content/repo-projects/diagnostics.proj

@@ -4,4 +4,8 @@
     <RepositoryReference Include="arcade" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-reference-packages" />
+  </ItemGroup>
+
 </Project>

src/SourceBuild/content/repo-projects/dotnet.proj

@@ -11,42 +11,12 @@
        - If we have a repo that is not in sdk's dependency tree, we can still build it by including it here.  -->
 
   <ItemGroup>
-    <!-- Toolsets -->
-    <RepositoryReference Include="source-build-reference-packages" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
-    <RepositoryReference Include="arcade" />
-
-    <!-- Product Repos -->
-    <RepositoryReference Include="command-line-api" />
-    <RepositoryReference Include="sourcelink" />
-    <RepositoryReference Include="diagnostics" />
-    <RepositoryReference Include="emsdk" />
-    <RepositoryReference Include="cecil" />
-    <RepositoryReference Include="symreader" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
-    <RepositoryReference Include="runtime" />
-    <RepositoryReference Include="roslyn" />
-    <RepositoryReference Include="windowsdesktop" Condition="'$(TargetOS)' == 'windows'" />
-    <RepositoryReference Include="xdt" />
-    <RepositoryReference Include="msbuild" />
-    <RepositoryReference Include="roslyn-analyzers" />
-    <RepositoryReference Include="aspnetcore" />
-    <RepositoryReference Include="razor" />
-    <RepositoryReference Include="deployment-tools" />
-    <RepositoryReference Include="format" />
-    <RepositoryReference Include="nuget-client" />
-    <RepositoryReference Include="templating" />
-    <RepositoryReference Include="test-templates" />
-    <RepositoryReference Include="fsharp" />
-    <RepositoryReference Include="vstest" />
-    <RepositoryReference Include="sdk" />
-    <RepositoryReference Include="aspire" />
     <RepositoryReference Include="installer" />
-
-    <!-- Package source-build artifacts -->
-    <RepositoryReference Include="package-source-build" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
-
-    <!-- Testing. -->
     <RepositoryReference Include="scenario-tests" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="package-source-build" />
+  </ItemGroup>
+
 </Project>

src/SourceBuild/content/repo-projects/emsdk.proj

@@ -14,4 +14,8 @@
     <RepositoryReference Include="arcade" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-reference-packages" />
+  </ItemGroup>
+
 </Project>

src/SourceBuild/content/repo-projects/format.proj

@@ -2,9 +2,17 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
+    <RepositoryReference Include="command-line-api" />
     <RepositoryReference Include="roslyn" />
+    <RepositoryReference Include="runtime" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="msbuild" />
     <RepositoryReference Include="roslyn-analyzers" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
+    <RepositoryReference Include="symreader" />
   </ItemGroup>
 
 </Project>

src/SourceBuild/content/repo-projects/fsharp.proj

@@ -20,8 +20,12 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="msbuild" /> 
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="msbuild" />
+    <RepositoryReference Include="runtime" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
 </Project>

src/SourceBuild/content/repo-projects/installer.proj

@@ -37,17 +37,26 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
+    <RepositoryReference Include="aspire" />
     <RepositoryReference Include="aspnetcore" />
+    <RepositoryReference Include="command-line-api" />
+    <RepositoryReference Include="deployment-tools" />
     <RepositoryReference Include="emsdk" />
     <RepositoryReference Include="fsharp" />
     <RepositoryReference Include="msbuild" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
     <RepositoryReference Include="nuget-client" />
     <RepositoryReference Include="roslyn" />
     <RepositoryReference Include="runtime" />
     <RepositoryReference Include="sdk" />
+    <RepositoryReference Include="symreader" />
     <RepositoryReference Include="test-templates" />
     <RepositoryReference Include="vstest" />
+    <RepositoryReference Include="windowsdesktop" Condition="'$(TargetOS)' == 'windows'" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
   <!--

src/SourceBuild/content/repo-projects/msbuild.proj

@@ -9,8 +9,12 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="runtime" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
     <RepositoryReference Include="roslyn" />
+    <RepositoryReference Include="runtime" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
 </Project>

src/SourceBuild/content/repo-projects/nuget-client.proj

@@ -8,9 +8,9 @@
     <BuildScript>$([MSBuild]::NormalizePath('$(ProjectDirectory)', 'eng', 'source-build', 'build$(ShellExtension)'))</BuildScript>
   </PropertyGroup>
 
-  <ItemGroup>
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
     <RepositoryReference Include="msbuild" />
+    <RepositoryReference Include="source-build-externals" />
     <RepositoryReference Include="xdt" />
   </ItemGroup>
 

src/SourceBuild/content/repo-projects/package-source-build.proj

@@ -14,56 +14,51 @@
   <Target Name="CustomRepoBuild"
           AfterTargets="RepoBuild"
           DependsOnTargets="CreateBuildInputProps;DetermineSourceBuiltSdkVersion">
-    <PropertyGroup>
-      <SourceBuildTarballStagingDir>$([MSBuild]::NormalizeDirectory('$(BaseIntermediateOutputPath)', '$(SourceBuiltArtifactsTarballName)'))</SourceBuildTarballStagingDir>
-      <SourceBuildReferencePackagesDestination>$([MSBuild]::NormalizeDirectory('$(SourceBuildTarballStagingDir)', 'SourceBuildReferencePackages'))</SourceBuildReferencePackagesDestination>
-    </PropertyGroup>
-
-    <!-- Copy PVP to staging dir in order to package them together. -->
-    <Copy SourceFiles="$(CurrentSourceBuiltPackageVersionPropsPath)"
-          DestinationFiles="$(SourceBuildTarballStagingDir)PackageVersions.props" />
-
-    <ItemGroup>
-      <SourceBuildReferencePackagesNupkgFiles Include="$(ReferencePackagesDir)**/*.nupkg" />
-    </ItemGroup>
-
-    <!-- Copy reference packages from ReferencePackagesDir to staging dir. -->
-    <Copy
-      SourceFiles="@(SourceBuildReferencePackagesNupkgFiles)"
-      DestinationFiles="@(SourceBuildReferencePackagesNupkgFiles -> '$(SourceBuildReferencePackagesDestination)%(Filename)%(Extension)')"
-      Condition="'@(SourceBuildReferencePackagesNupkgFiles)' != ''" />
-
-    <ItemGroup>
-      <ArtifactsPackage Include="$(ArtifactsPackagesDir)**\*.nupkg" />
-    </ItemGroup>
-
-    <!-- Copy packages to staging dir. -->
-    <Copy SourceFiles="@(ArtifactsPackage)"
-          DestinationFolder="$(SourceBuildTarballStagingDir)"
-          Condition="'@(ArtifactsPackage)' != ''" />
-
     <PropertyGroup>
       <SourceBuiltTarballName>$(ArtifactsAssetsDir)$(SourceBuiltArtifactsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</SourceBuiltTarballName>
       <SourceBuiltVersionFileName>.version</SourceBuiltVersionFileName>
     </PropertyGroup>
 
-    <!-- Content of the .version file to include in the tarball. Write to staging dir. -->
+    <!-- Content of the .version file to include in the tarball -->
     <ItemGroup>
       <VersionFileContent Include="$(RepositoryCommit);$(SourceBuiltSdkVersion)" />
     </ItemGroup>
 
     <WriteLinesToFile
-      File="$(SourceBuildTarballStagingDir)$(SourceBuiltVersionFileName)"
+      File="$(BaseIntermediateOutputPath)$(SourceBuiltVersionFileName)"
       Lines="@(VersionFileContent)"
       Overwrite="true" />
 
-    <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(SourceBuiltTarballName)'))" />
-    <Exec Command="tar --numeric-owner -czf $(SourceBuiltTarballName) $(SourceBuiltVersionFileName) *.nupkg *.props SourceBuildReferencePackages/"
-          WorkingDirectory="$(SourceBuildTarballStagingDir)" />
+    <ItemGroup>
+      <_AllRepoFiles Include="$(ArtifactsShippingPackagesDir)/**" />
+      <_AllRepoFiles Include="$(ArtifactsNonShippingPackagesDir)/**" />
+      <_ReferencePackageFiles Include="$(ReferencePackagesDir)**" />
+    </ItemGroup>
 
-    <Message Importance="High" Text="Packaged source-built artifacts to $(SourceBuiltTarballName)" />
+    <!-- Create a layout directory for the files that are to be included in the artifacts tarball. Since there are a large number of files, this will use symlinks
+         instead of copying files to make this execute quickly. -->
+    <PropertyGroup>
+      <_SourceBuiltLayoutDir>$([MSBuild]::NormalizeDirectory('$(BaseIntermediateOutputPath)', 'artifacts-layout'))</_SourceBuiltLayoutDir>
+      <SourceBuildReferencePackagesDestinationDirName>SourceBuildReferencePackages</SourceBuildReferencePackagesDestinationDirName>
+    </PropertyGroup>
+
+    <Copy SourceFiles="@(_AllRepoFiles)"
+          DestinationFolder="$(_SourceBuiltLayoutDir)"
+          UseSymbolicLinksIfPossible="true" />
+    <Copy SourceFiles="$(BaseIntermediateOutputPath)$(SourceBuiltVersionFileName)"
+          DestinationFolder="$(_SourceBuiltLayoutDir)"
+          UseSymbolicLinksIfPossible="true" />
+    <Copy SourceFiles="$(CurrentSourceBuiltPackageVersionPropsPath)"
+          DestinationFiles="$(_SourceBuiltLayoutDir)PackageVersions.props"
+          UseSymbolicLinksIfPossible="true" />
+    <Copy SourceFiles="@(_ReferencePackageFiles)"
+          DestinationFolder="$(_SourceBuiltLayoutDir)$(SourceBuildReferencePackagesDestinationDirName)"
+          UseSymbolicLinksIfPossible="true" />
 
-    <RemoveDir Directories="$(SourceBuildTarballStagingDir)" />
+    <Exec Command="tar --numeric-owner -czhf $(SourceBuiltTarballName) $(SourceBuiltVersionFileName) *"
+          WorkingDirectory="$(_SourceBuiltLayoutDir)" />
+
+    <Message Importance="High" Text="Packaged source-built artifacts to $(SourceBuiltTarballName)" />
   </Target>
 
 </Project>

src/SourceBuild/content/repo-projects/razor.proj

@@ -2,8 +2,11 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="runtime" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
     <RepositoryReference Include="aspnetcore" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
 </Project>

src/SourceBuild/content/repo-projects/roslyn-analyzers.proj

@@ -8,8 +8,12 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
     <RepositoryReference Include="runtime" />
-    <RepositoryReference Include="roslyn" />
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
 </Project>

src/SourceBuild/content/repo-projects/roslyn.proj

@@ -28,9 +28,12 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="command-line-api" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
     <RepositoryReference Include="runtime" />
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
   <ItemGroup>