dotnet · mthalman · Feb 22, 2024 · Feb 6, 2024 · Feb 12, 2024 · Feb 12, 2024
diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props
@@ -158,7 +158,7 @@
     <SharedIntermediateOutputPath>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', '$(TargetArchitecture)', '$(Configuration)'))</SharedIntermediateOutputPath>
 
     <SourceBuiltBlobFeedDir>$([MSBuild]::NormalizeDirectory('$(SharedIntermediateOutputPath)', 'blob-feed'))</SourceBuiltBlobFeedDir>
-    <SourceBuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(SourceBuiltBlobFeedDir)', 'packages'))</SourceBuiltPackagesPath>
+    <SourceBuiltBasePackagesPath>$([MSBuild]::NormalizeDirectory('$(SourceBuiltBlobFeedDir)', 'packages'))</SourceBuiltBasePackagesPath>
     <SourceBuiltAssetsDir>$([MSBuild]::NormalizeDirectory('$(SourceBuiltBlobFeedDir)', 'assets'))</SourceBuiltAssetsDir>
 
     <PrebuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'prebuilt'))</PrebuiltPackagesPath>

diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.props b/src/SourceBuild/content/repo-projects/Directory.Build.props
@@ -33,6 +33,7 @@
     <NuGetConfigFile Condition="'$(OriginalNuGetConfigFile)' != ''">$(BaseIntermediateOutputPath)$([System.IO.Path]::GetFileName('$(OriginalNuGetConfigFile)'))</NuGetConfigFile>
 
     <SourceBuiltSdksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'source-built-sdks'))</SourceBuiltSdksDir>
+    <SourceBuiltPackagesPath>$(SourceBuiltBasePackagesPath)$(RepositoryName)/</SourceBuiltPackagesPath>
 
     <!-- Set the bootstrap version to the VMR's version if empty. (no bootstrap set). -->
     <ArcadeBootstrapVersion>$([MSBuild]::ValueOrDefault('$(ARCADE_BOOTSTRAP_VERSION)', '$(ArcadeSdkVersion)'))</ArcadeBootstrapVersion>

diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.targets b/src/SourceBuild/content/repo-projects/Directory.Build.targets
@@ -54,6 +54,17 @@
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="WritePackageVersionsProps" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="WritePackageUsageData" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="WriteUsageReports" />
+
+  <Target Name="GetRepositoryReferenceInfo">
+    <ItemGroup>
+      <RepositoryReferenceInfo Include="@(RepositoryReference)">
+        <SourceName>source-built-%(Identity)</SourceName>
+        <PackagesPath>$(SourceBuiltBasePackagesPath)%(Identity)/</PackagesPath>
+      </RepositoryReferenceInfo>
+      <DependentRepoSourceName Include="@(RepositoryReferenceInfo->'%(SourceName)')" />
+      <DependentRepoPackageFile Include="%(RepositoryReferenceInfo.PackagesPath)**" />
+    </ItemGroup>
+  </Target>
 
   <Target Name="CopyNuGetConfig"
           Condition="'$(NuGetConfigFile)' != ''"
@@ -70,24 +81,32 @@
   </Target>
 
   <Target Name="UpdateNuGetConfig"
-          DependsOnTargets="CopyNuGetConfig"
+          DependsOnTargets="CopyNuGetConfig;GetRepositoryReferenceInfo"
           Condition="'$(NuGetConfigFile)' != ''"
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)UpdateNuGetConfig.complete">
     <PropertyGroup>
-      <SourceBuiltNuGetSourceName>source-built</SourceBuiltNuGetSourceName>
       <ExtraSourcesNuGetSourceName>ExtraSources</ExtraSourcesNuGetSourceName>
-      <SourceBuildSources>$(SourceBuiltNuGetSourceName)</SourceBuildSources>
-      <SourceBuildSources Condition="'$(ExtraRestoreSourcePath)' != ''">$(SourceBuildSources);$(ExtraSourcesNuGetSourceName)</SourceBuildSources>
     </PropertyGroup>
 
     <PropertyGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
       <PrebuiltNuGetSourceName>prebuilt</PrebuiltNuGetSourceName>
       <PreviouslySourceBuiltNuGetSourceName>previously-source-built</PreviouslySourceBuiltNuGetSourceName>
       <ReferencePackagesNuGetSourceName>reference-packages</ReferencePackagesNuGetSourceName>
-      <SourceBuildSources>$(SourceBuildSources);$(PrebuiltNuGetSourceName);$(PreviouslySourceBuiltNuGetSourceName);$(ReferencePackagesNuGetSourceName)</SourceBuildSources>
     </PropertyGroup>
 
+    <ItemGroup>
+      <_CommonSourceBuildSources Include="@(DependentRepoSourceName)" />
+      <_CommonSourceBuildSources Include="$(ExtraSourcesNuGetSourceName)" Condition="'$(ExtraRestoreSourcePath)' != ''" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <SourceBuildSources Condition="'$(DotNetBuildSourceOnly)' == 'true'"
+                          Include="$(PrebuiltNuGetSourceName);$(PreviouslySourceBuiltNuGetSourceName);$(ReferencePackagesNuGetSourceName);@(_CommonSourceBuildSources)" />
+      <SourceBuildSources Condition="'$(DotNetBuildSourceOnly)' != 'true'"
+                          Include="@(_CommonSourceBuildSources)" />
+    </ItemGroup>
+
     <PropertyGroup Condition="'$(DotNetBuildSourceOnly)' != 'true'">
       <!-- When not building source-only, repositories lagging behind on the .NET SDK (e.g. tooling repos)
           that need to utilize a VS-aligned version for development purposes may not have the latest product
@@ -126,8 +145,9 @@
                             Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
 
     <AddSourceToNuGetConfig NuGetConfigFile="$(NuGetConfigFile)"
-                            SourceName="$(SourceBuiltNuGetSourceName)"
-                            SourcePath="$(SourceBuiltPackagesPath)" />
+                            SourceName="%(RepositoryReferenceInfo.SourceName)"
+                            SourcePath="%(RepositoryReferenceInfo.PackagesPath)"
+                            Condition=" '@(RepositoryReferenceInfo)' != '' " />
 
     <AddSourceToNuGetConfig NuGetConfigFile="$(NuGetConfigFile)"
                             SourceName="$(ExtraSourcesNuGetSourceName)"
@@ -143,12 +163,16 @@
     <UpdateNuGetConfigPackageSourcesMappings
       NuGetConfigFile="$(NuGetConfigFile)"
       BuildWithOnlineFeeds="$(DotNetBuildWithOnlineFeeds)"
-      SourceBuildSources="$(SourceBuildSources)" />
+      SourceBuildSources="@(SourceBuildSources)" />
 
     <MakeDir Directories="$(BaseIntermediateOutputPath)" />
     <Touch Files="$(BaseIntermediateOutputPath)UpdateNuGetConfig.complete" AlwaysCreate="true">
       <Output TaskParameter="TouchedFiles" ItemName="FileWrites" />
     </Touch>
+
+    <ItemGroup>
+      <EmbedInBinlog Include="$(NuGetConfigFile)" />
+  </ItemGroup>
   </Target>
 
   <!-- Update the SDK version in the repo's global.json file.
@@ -193,11 +217,13 @@
   <!-- Before a repository builds, set up the version property files that override the repo's defaults.
        There are 3 files generated -->
   <Target Name="CreateBuildInputProps"
+          DependsOnTargets="GetRepositoryReferenceInfo"
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)CreateBuildInputProps.complete">
     <ItemGroup>
-      <_CurrentSourceBuiltPackages Include="$(SourceBuiltPackagesPath)*.nupkg"
-                                   Exclude="$(SourceBuiltPackagesPath)*.symbols.nupkg" />
+      <!-- Collect all the NuGet packages from dependent repos except for the symbols -->
+      <_CurrentSourceBuiltPackages Include="@(DependentRepoPackageFile)"
+                                   Condition="'%(Extension)' == '.nupkg' and !$([System.String]::new('%(Filename)').EndsWith('.symbols')) " />
       <_PreviouslyBuiltSourceBuiltPackages Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" />
     </ItemGroup>
 

diff --git a/src/SourceBuild/content/repo-projects/aspnetcore.proj b/src/SourceBuild/content/repo-projects/aspnetcore.proj
@@ -23,11 +23,12 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
-    <RepositoryReference Include="runtime" />
-    <RepositoryReference Include="msbuild" />
     <RepositoryReference Include="roslyn" />
-    <RepositoryReference Include="roslyn-analyzers" />
+    <RepositoryReference Include="runtime" />
+    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+    <RepositoryReference Include="source-build-reference-packages" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+    <RepositoryReference Include="symreader" />
+    <RepositoryReference Include="xdt" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/SourceBuild/content/repo-projects/cecil.proj b/src/SourceBuild/content/repo-projects/cecil.proj
@@ -2,6 +2,7 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
+    <RepositoryReference Include="source-build-reference-packages" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
   </ItemGroup>
 
 </Project>
diff --git a/src/SourceBuild/content/repo-projects/command-line-api.proj b/src/SourceBuild/content/repo-projects/command-line-api.proj
@@ -8,6 +8,7 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
+    <RepositoryReference Include="source-build-reference-packages" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
   </ItemGroup>
 
 </Project>
diff --git a/src/SourceBuild/content/repo-projects/deployment-tools.proj b/src/SourceBuild/content/repo-projects/deployment-tools.proj
@@ -2,7 +2,8 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="runtime" />
+    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+    <RepositoryReference Include="source-build-reference-packages" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/SourceBuild/content/repo-projects/diagnostics.proj b/src/SourceBuild/content/repo-projects/diagnostics.proj
@@ -2,6 +2,7 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
+    <RepositoryReference Include="source-build-reference-packages" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
   </ItemGroup>
 
 </Project>
diff --git a/src/SourceBuild/content/repo-projects/dotnet.proj b/src/SourceBuild/content/repo-projects/dotnet.proj
@@ -10,43 +10,12 @@
        - We need an "empty" build after the last real repo for prebuilt detection to work - this is that file.
        - If we have a repo that is not in sdk's dependency tree, we can still build it by including it here.  -->
 
-  <ItemGroup>
-    <!-- Toolsets -->
-    <RepositoryReference Include="source-build-reference-packages" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
-    <RepositoryReference Include="arcade" />
+  <!-- This import can be removed when https://github.com/dotnet/source-build/issues/4017 is resolved -->
+  <Import Project="$(MSBuildThisFileDirectory)installer-deps.props" />
 
-    <!-- Product Repos -->
-    <RepositoryReference Include="command-line-api" />
-    <RepositoryReference Include="sourcelink" />
-    <RepositoryReference Include="diagnostics" />
-    <RepositoryReference Include="emsdk" />
-    <RepositoryReference Include="cecil" />
-    <RepositoryReference Include="symreader" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
-    <RepositoryReference Include="runtime" />
-    <RepositoryReference Include="roslyn" />
-    <RepositoryReference Include="windowsdesktop" Condition="'$(TargetOS)' == 'windows'" />
-    <RepositoryReference Include="xdt" />
-    <RepositoryReference Include="msbuild" />
-    <RepositoryReference Include="roslyn-analyzers" />
-    <RepositoryReference Include="aspnetcore" />
-    <RepositoryReference Include="razor" />
-    <RepositoryReference Include="deployment-tools" />
-    <RepositoryReference Include="format" />
-    <RepositoryReference Include="nuget-client" />
-    <RepositoryReference Include="templating" />
-    <RepositoryReference Include="test-templates" />
-    <RepositoryReference Include="fsharp" />
-    <RepositoryReference Include="vstest" />
-    <RepositoryReference Include="sdk" />
-    <RepositoryReference Include="aspire" />
+  <ItemGroup>
     <RepositoryReference Include="installer" />
-
-    <!-- Package source-build artifacts -->
     <RepositoryReference Include="package-source-build" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
-
-    <!-- Testing. -->
-    <RepositoryReference Include="scenario-tests" />
   </ItemGroup>
 
 </Project>
diff --git a/src/SourceBuild/content/repo-projects/emsdk.proj b/src/SourceBuild/content/repo-projects/emsdk.proj
@@ -12,6 +12,7 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
+    <RepositoryReference Include="source-build-reference-packages" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
   </ItemGroup>
 
 </Project>
diff --git a/src/SourceBuild/content/repo-projects/format.proj b/src/SourceBuild/content/repo-projects/format.proj
@@ -2,9 +2,15 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
+    <RepositoryReference Include="command-line-api" />
+    <RepositoryReference Include="msbuild" />
     <RepositoryReference Include="roslyn" />
     <RepositoryReference Include="roslyn-analyzers" />
+    <RepositoryReference Include="runtime" />
     <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+    <RepositoryReference Include="source-build-reference-packages" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+    <RepositoryReference Include="sourcelink" />
+    <RepositoryReference Include="symreader" />
   </ItemGroup>
 
 </Project>
diff --git a/src/SourceBuild/content/repo-projects/fsharp.proj b/src/SourceBuild/content/repo-projects/fsharp.proj
@@ -20,8 +20,10 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="msbuild" /> 
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+    <RepositoryReference Include="msbuild" />
+    <RepositoryReference Include="runtime" />
+    <RepositoryReference Include="source-build-reference-packages" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+    <RepositoryReference Include="sourcelink" />
   </ItemGroup>
 
 </Project>
diff --git a/src/SourceBuild/content/repo-projects/installer-deps.props b/src/SourceBuild/content/repo-projects/installer-deps.props
@@ -0,0 +1,24 @@
+<Project>
+
+  <ItemGroup>
+    <RepositoryReference Include="arcade" />
+    <RepositoryReference Include="aspire" />
+    <RepositoryReference Include="aspnetcore" />
+    <RepositoryReference Include="command-line-api" />
+    <RepositoryReference Include="deployment-tools" />
+    <RepositoryReference Include="emsdk" />
+    <RepositoryReference Include="fsharp" />
+    <RepositoryReference Include="msbuild" />
+    <RepositoryReference Include="nuget-client" />
+    <RepositoryReference Include="roslyn" />
+    <RepositoryReference Include="runtime" />
+    <RepositoryReference Include="scenario-tests" />
+    <RepositoryReference Include="sdk" />
+    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+    <RepositoryReference Include="source-build-reference-packages" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+    <RepositoryReference Include="symreader" />
+    <RepositoryReference Include="test-templates" />
+    <RepositoryReference Include="vstest" />
+  </ItemGroup>
+
+</Project>
diff --git a/src/SourceBuild/content/repo-projects/installer.proj b/src/SourceBuild/content/repo-projects/installer.proj
@@ -10,6 +10,8 @@
 
     <!-- Restore and Build actions are already passed in by the root script. -->
     <BuildActions>$(FlagParameterPrefix)pack $(FlagParameterPrefix)publish</BuildActions>
+
+    <SkipEnsurePackagesCreated>true</SkipEnsurePackagesCreated>
 
     <BuildArgs>$(BuildArgs) $(FlagParameterPrefix)runtime-id $(TargetRid)</BuildArgs>
 
@@ -34,20 +36,8 @@
     <BuildArgs>$(BuildArgs) /p:UsePortableLinuxSharedFramework=false</BuildArgs>
   </PropertyGroup>
 
-  <ItemGroup>
-    <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="aspnetcore" />
-    <RepositoryReference Include="emsdk" />
-    <RepositoryReference Include="fsharp" />
-    <RepositoryReference Include="msbuild" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
-    <RepositoryReference Include="nuget-client" />
-    <RepositoryReference Include="roslyn" />
-    <RepositoryReference Include="runtime" />
-    <RepositoryReference Include="sdk" />
-    <RepositoryReference Include="test-templates" />
-    <RepositoryReference Include="vstest" />
-  </ItemGroup>
+  <!-- Import RepositoryReferences of installer from a shared file -->
+  <Import Project="$(MSBuildThisFileDirectory)installer-deps.props" />
 
   <!--
     If we have authentication, keep the templating internal feed (if one exists) to acquire the

diff --git a/src/SourceBuild/content/repo-projects/msbuild.proj b/src/SourceBuild/content/repo-projects/msbuild.proj
@@ -12,8 +12,9 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="runtime" />
     <RepositoryReference Include="roslyn" />
+    <RepositoryReference Include="runtime" />
+    <RepositoryReference Include="source-build-reference-packages" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
   </ItemGroup>
 
 </Project>
diff --git a/src/SourceBuild/content/repo-projects/nuget-client.proj b/src/SourceBuild/content/repo-projects/nuget-client.proj
@@ -9,8 +9,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
     <RepositoryReference Include="msbuild" />
+    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
     <RepositoryReference Include="xdt" />
   </ItemGroup>
 

diff --git a/src/SourceBuild/content/repo-projects/package-source-build.proj b/src/SourceBuild/content/repo-projects/package-source-build.proj
@@ -7,18 +7,18 @@
     <EnableSourceControlManagerQueries>false</EnableSourceControlManagerQueries>
   </PropertyGroup>
 
+  <Import Project="$(MSBuildThisFileDirectory)installer-deps.props" />
+
   <ItemGroup>
     <RepositoryReference Include="installer" />
   </ItemGroup>
 
   <Target Name="CustomRepoBuild"
           AfterTargets="RepoBuild"
           DependsOnTargets="CreateBuildInputProps;DetermineSourceBuiltSdkVersion">
-    <!-- Copy PVP to packages dir in order to package them together -->
-    <Copy SourceFiles="$(CurrentSourceBuiltPackageVersionPropsPath)" DestinationFiles="$(SourceBuiltPackagesPath)PackageVersions.props" />
-
     <PropertyGroup>
-      <SourceBuildReferencePackagesDestination>$(SourceBuiltPackagesPath)SourceBuildReferencePackages/</SourceBuildReferencePackagesDestination>
+      <SourceBuildReferencePackagesDestinationDirName>SourceBuildReferencePackages</SourceBuildReferencePackagesDestinationDirName>
+      <SourceBuildReferencePackagesDestination>$(SourceBuiltPackagesPath)$(SourceBuildReferencePackagesDestinationDirName)/</SourceBuildReferencePackagesDestination>
     </PropertyGroup>
 
     <ItemGroup>
@@ -47,10 +47,32 @@
       Overwrite="true" />
 
     <MakeDir Directories="$(SharedOutputPath)" />
-    <Exec Command="tar --numeric-owner --exclude='Microsoft.SourceBuild.Intermediate.*.nupkg' -czf $(SourceBuiltTarballName) $(SourceBuiltVersionFileName) *.nupkg *.props SourceBuildReferencePackages/"
-          WorkingDirectory="$(SourceBuiltPackagesPath)" />
+
+    <ItemGroup>
+      <_AllRepoDirs Include="$([System.IO.Directory]::GetDirectories($(SourceBuiltBasePackagesPath)))"
+                    Exclude="$(SourceBuiltBasePackagesPath)package-source-build" />
+    </ItemGroup>
+
+    <!-- Create a layout directory for the files that are to be included in the artifacts tarball. Since there are a large number of files, this will use symlinks
+         instead of copying files to make this execute quickly. -->
+    <PropertyGroup>
+      <_SourceBuiltLayoutDir>$(BaseIntermediateOutputPath)artifacts-layout/</_SourceBuiltLayoutDir>
+    </PropertyGroup>
+    <MakeDir Directories="$(_SourceBuiltLayoutDir)$(SourceBuildReferencePackagesDestinationDirName)" />
+    <Exec Command="ln -sf %(_AllRepoDirs.Identity)/* $(_SourceBuiltLayoutDir)"
+          Condition="$([System.IO.Directory]::GetFiles('%(_AllRepoDirs.Identity)')) != ''" />
+    <Exec Command="ln -sf $(SourceBuiltPackagesPath)$(SourceBuiltVersionFileName) $(_SourceBuiltLayoutDir)" />
+    <Exec Command="ln -sf $(CurrentSourceBuiltPackageVersionPropsPath) $(_SourceBuiltLayoutDir)PackageVersions.props" />
+    <Exec Command="ln -sf $(SourceBuildReferencePackagesDestination)* $(_SourceBuiltLayoutDir)$(SourceBuildReferencePackagesDestinationDirName)" />
+
+    <Exec Command="tar --numeric-owner -czhf $(SourceBuiltTarballName) $(SourceBuiltVersionFileName) *"
+          WorkingDirectory="$(_SourceBuiltLayoutDir)" />
 
     <Message Importance="High" Text="Packaged source-built artifacts to $(SourceBuiltTarballName)" />
   </Target>
 
+  <ItemGroup>
+    <RepositoryReference Include="installer" />
+  </ItemGroup>
+
 </Project>