Isolate package flow between repos

src/SourceBuild/content/repo-projects/Directory.Build.props

@@ -33,6 +33,11 @@
     <NuGetConfigFile Condition="'$(OriginalNuGetConfigFile)' != ''">$(BaseIntermediateOutputPath)$([System.IO.Path]::GetFileName('$(OriginalNuGetConfigFile)'))</NuGetConfigFile>
 
     <SourceBuiltSdksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'source-built-sdks'))</SourceBuiltSdksDir>
+    <RepoArtifactsShippingPackagesDir Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$([MSBuild]::NormalizeDirectory('$(ArtifactsShippingPackagesDir)', '$(RepositoryName)'))</RepoArtifactsShippingPackagesDir>
+    <RepoArtifactsShippingPackagesDir Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</RepoArtifactsShippingPackagesDir>
+    <RepoArtifactsNonShippingPackagesDir Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$([MSBuild]::NormalizeDirectory('$(ArtifactsNonShippingPackagesDir)', '$(RepositoryName)'))</RepoArtifactsNonShippingPackagesDir>
+    <RepoArtifactsNonShippingPackagesDir Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</RepoArtifactsNonShippingPackagesDir>
+
 
     <!-- Set the bootstrap version to the VMR's version if empty. (no bootstrap set). -->
     <ArcadeBootstrapVersion>$([MSBuild]::ValueOrDefault('$(ARCADE_BOOTSTRAP_VERSION)', '$(ArcadeSdkVersion)'))</ArcadeBootstrapVersion>

src/SourceBuild/content/repo-projects/Directory.Build.targets

@@ -32,19 +32,6 @@
     <IntermediateSymbolsRepoDir>$(IntermediateSymbolsRootDir)$(RepositoryName)</IntermediateSymbolsRepoDir>
   </PropertyGroup>
 
-  <!-- Exclude repositories that currently don't build when not building source-only. -->
-  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' != 'true'">
-    <RepositoryReference Remove="roslyn" />
-    <RepositoryReference Remove="msbuild" />
-    <RepositoryReference Remove="aspnetcore" />
-    <RepositoryReference Remove="razor" />
-    <RepositoryReference Remove="format" />
-    <RepositoryReference Remove="nuget-client" />
-    <RepositoryReference Remove="fsharp" />
-    <RepositoryReference Remove="vstest" />
-    <RepositoryReference Remove="installer" />
-  </ItemGroup>
-
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="AddSourceToNuGetConfig" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="RemoveInternetSourcesFromNuGetConfig" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="UpdateJson" />
@@ -53,6 +40,28 @@
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="WritePackageVersionsProps" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="WritePackageUsageData" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="WriteUsageReports" />
+
+  <!-- Wraps the transitive repo references with additional metadata -->
+  <Target Name="GetRepositoryReferenceInfo"
+          DependsOnTargets="GetTransitiveRepositoryReferences">
+    <!-- SBRP is explicitly excluded since it is output to its own special directory, $(ReferencePackagesDir). -->
+    <ItemGroup Condition="'@(TransitiveRepositoryReference)' != ''">
+      <RepositoryReferenceInfo Include="@(TransitiveRepositoryReference)"
+                               Exclude="source-build-reference-packages">
+        <ShippingSourceName>source-built-%(Identity)</ShippingSourceName>
+        <NonShippingSourceName>source-built-transport-%(Identity)</NonShippingSourceName>
+        <ShippingPackagesPath>$(ArtifactsShippingPackagesDir)/%(Identity)/</ShippingPackagesPath>
+        <NonShippingPackagesPath>$(ArtifactsNonShippingPackagesDir)/%(Identity)/</NonShippingPackagesPath>
+      </RepositoryReferenceInfo>
+    </ItemGroup>
+
+    <ItemGroup Condition="'@(RepositoryReferenceInfo)' != ''">
+      <DependentRepoSourceName Include="@(RepositoryReferenceInfo->'%(ShippingSourceName)')" />
+      <DependentRepoSourceName Include="@(RepositoryReferenceInfo->'%(NonShippingSourceName)')" />
+      <DependentRepoPackageFile Include="%(RepositoryReferenceInfo.ShippingPackagesPath)**" />
+      <DependentRepoPackageFile Include="%(RepositoryReferenceInfo.NonShippingPackagesPath)**" />
+    </ItemGroup>
+  </Target>
 
   <Target Name="CopyNuGetConfig"
           Condition="'$(NuGetConfigFile)' != ''"
@@ -69,29 +78,32 @@
   </Target>
 
   <Target Name="UpdateNuGetConfig"
-          DependsOnTargets="CopyNuGetConfig"
+          DependsOnTargets="CopyNuGetConfig;GetRepositoryReferenceInfo"
           Condition="'$(NuGetConfigFile)' != ''"
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)UpdateNuGetConfig.complete">
     <PropertyGroup>
-      <!-- Feed for the shipping live built packages -->
-      <SourceBuiltShippingNuGetSourceName>source-built</SourceBuiltShippingNuGetSourceName>
-      <!-- Feed for the non-shipping live built packages -->
-      <SourceBuiltNonShippingNuGetSourceName>source-built-transport</SourceBuiltNonShippingNuGetSourceName>
-      <SourceBuildSources>$(SourceBuiltShippingNuGetSourceName);$(SourceBuiltNonShippingNuGetSourceName)</SourceBuildSources>
-
       <!-- Dev innerloop opt-in feed: /p:ExtraRestoreSourcePath=... -->
       <ExtraSourcesNuGetSourceName>ExtraSources</ExtraSourcesNuGetSourceName>
-      <SourceBuildSources Condition="'$(ExtraRestoreSourcePath)' != ''">$(SourceBuildSources);$(ExtraSourcesNuGetSourceName)</SourceBuildSources>
     </PropertyGroup>
 
     <PropertyGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
       <PrebuiltNuGetSourceName>prebuilt</PrebuiltNuGetSourceName>
       <PreviouslySourceBuiltNuGetSourceName>previously-source-built</PreviouslySourceBuiltNuGetSourceName>
       <ReferencePackagesNuGetSourceName>reference-packages</ReferencePackagesNuGetSourceName>
-      <SourceBuildSources>$(SourceBuildSources);$(PrebuiltNuGetSourceName);$(PreviouslySourceBuiltNuGetSourceName);$(ReferencePackagesNuGetSourceName)</SourceBuildSources>
     </PropertyGroup>
 
+    <ItemGroup>
+      <_CommonBuildSources Include="@(DependentRepoSourceName)" />
+      <_CommonBuildSources Include="$(ExtraSourcesNuGetSourceName)" Condition="'$(ExtraRestoreSourcePath)' != ''" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <_BuildSources Condition="'$(DotNetBuildSourceOnly)' == 'true'"
+                     Include="$(PrebuiltNuGetSourceName);$(PreviouslySourceBuiltNuGetSourceName);$(ReferencePackagesNuGetSourceName)" />
+      <_BuildSources Include="@(_CommonBuildSources)" />
+    </ItemGroup>
+
     <PropertyGroup Condition="'$(DotNetBuildSourceOnly)' != 'true'">
       <!-- When not building source-only, repositories lagging behind on the .NET SDK (e.g. tooling repos)
           that need to utilize a VS-aligned version for development purposes may not have the latest product
@@ -130,12 +142,14 @@
                             Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
 
     <AddSourceToNuGetConfig NuGetConfigFile="$(NuGetConfigFile)"
-                            SourceName="$(SourceBuiltShippingNuGetSourceName)"
-                            SourcePath="$(ArtifactsShippingPackagesDir)" />
+                            SourceName="%(RepositoryReferenceInfo.ShippingSourceName)"
+                            SourcePath="%(RepositoryReferenceInfo.ShippingPackagesPath)"
+                            Condition="Exists('%(RepositoryReferenceInfo.ShippingPackagesPath)')" />
 
     <AddSourceToNuGetConfig NuGetConfigFile="$(NuGetConfigFile)"
-                            SourceName="$(SourceBuiltNonShippingNuGetSourceName)"
-                            SourcePath="$(ArtifactsNonShippingPackagesDir)" />
+                            SourceName="%(RepositoryReferenceInfo.NonShippingSourceName)"
+                            SourcePath="%(RepositoryReferenceInfo.NonShippingPackagesPath)"
+                            Condition="Exists('%(RepositoryReferenceInfo.NonShippingPackagesPath)')" />
 
     <AddSourceToNuGetConfig NuGetConfigFile="$(NuGetConfigFile)"
                             SourceName="$(ExtraSourcesNuGetSourceName)"
@@ -151,12 +165,18 @@
     <UpdateNuGetConfigPackageSourcesMappings
       NuGetConfigFile="$(NuGetConfigFile)"
       BuildWithOnlineFeeds="$(DotNetBuildWithOnlineFeeds)"
-      SourceBuildSources="$(SourceBuildSources)" />
+      SourceBuildSources="@(_BuildSources)" />
 
     <MakeDir Directories="$(BaseIntermediateOutputPath)" />
     <Touch Files="$(BaseIntermediateOutputPath)UpdateNuGetConfig.complete" AlwaysCreate="true">
       <Output TaskParameter="TouchedFiles" ItemName="FileWrites" />
     </Touch>
+
+    <ItemGroup>
+      <!-- This is defined in this target to ensure the NuGet.config file is updated before it gets embedded. If this item
+           was define globally, it would immediately get embedded with the original, unmodified version of the file. -->
+      <EmbedInBinlog Include="$(NuGetConfigFile)" />
+    </ItemGroup>
   </Target>
 
   <!-- Update the SDK version in the repo's global.json file.
@@ -201,10 +221,12 @@
   <!-- Before a repository builds, set up the version property files that override the repo's defaults.
        There are 3 files generated -->
   <Target Name="CreateBuildInputProps"
+          DependsOnTargets="GetRepositoryReferenceInfo"
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)CreateBuildInputProps.complete">
     <ItemGroup>
-      <_CurrentSourceBuiltPackages Include="$(ArtifactsPackagesDir)**\*.nupkg"
+      <!-- Collect all the NuGet packages from dependent repos except for the symbols -->
+      <_CurrentSourceBuiltPackages Include="@(DependentRepoPackageFile)"
                                    Condition="!$([System.String]::Copy('%(Identity)').EndsWith('.symbols.nupkg'))" />
       <_PreviouslyBuiltSourceBuiltPackages Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" />
     </ItemGroup>
@@ -268,11 +290,12 @@
 
   <!-- SkipRepoReferences is a developer innerloop switch to skip building dependencies. -->
   <Target Name="BuildRepoReferences"
+          DependsOnTargets="GetTransitiveRepositoryReferences"
           Condition="'@(RepositoryReference)' != '' and '$(SkipRepoReferences)' != 'true'">
-    <Message Importance="High" Text="Building dependencies [@(RepositoryReference)] needed by '$(RepositoryName)'." />
+    <Message Importance="High" Text="Building dependencies [@(TransitiveRepositoryReference)] needed by '$(RepositoryName)'." />
 
     <ItemGroup>
-      <_DependentProject Include="@(RepositoryReference -> '%(Identity).proj')" />
+      <_DependentProject Include="@(TransitiveRepositoryReference -> '%(Identity).proj')" />
     </ItemGroup>
 
     <MSBuild Projects="@(_DependentProject)"
@@ -324,8 +347,6 @@
     </PropertyGroup>
 
     <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(RepoConsoleLogFile)'));
-                          $(ArtifactsShippingPackagesDir);
-                          $(ArtifactsNonShippingPackagesDir);
                           $(ArtifactsAssetsDir)" />
 
     <!-- Create directories for extra debugging. -->
@@ -397,14 +418,12 @@
     <ItemGroup>
       <RepoManifestNonShippingPackage Include="@(RepoManifestPackage)"
                                       Condition="$([System.String]::Copy('%(Identity)').Contains('$([System.IO.Path]::DirectorySeparatorChar)NonShipping$([System.IO.Path]::DirectorySeparatorChar)'))">
-        <DestinationFolder Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$(ArtifactsNonShippingPackagesDir)</DestinationFolder>
-        <DestinationFolder Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</DestinationFolder>
+        <DestinationFolder>$(RepoArtifactsNonShippingPackagesDir)</DestinationFolder>
       </RepoManifestNonShippingPackage>
 
       <RepoManifestShippingPackage Include="@(RepoManifestPackage)"
                                    Exclude="@(RepoManifestNonShippingPackage)">
-        <DestinationFolder Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$(ArtifactsShippingPackagesDir)</DestinationFolder>
-        <DestinationFolder Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</DestinationFolder>
+        <DestinationFolder>$(RepoArtifactsShippingPackagesDir)</DestinationFolder>
       </RepoManifestShippingPackage>
     </ItemGroup>
 
@@ -505,8 +524,7 @@
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)ExtractToolPackage.complete">
     <PropertyGroup>
-      <_ToolPackagesRoot Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$(ArtifactsNonShippingPackagesDir)</_ToolPackagesRoot>
-      <_ToolPackagesRoot Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</_ToolPackagesRoot>
+      <_ToolPackagesRoot>$(RepoArtifactsNonShippingPackagesDir)</_ToolPackagesRoot>
     </PropertyGroup>
 
     <ItemGroup>
@@ -693,4 +711,77 @@
   <Target Name="GetProjectDirectory" Outputs="$(ProjectDirectory)" />
   <Target Name="GetRepositoryReferences" Outputs="@(RepositoryReference)" />
 
+  <!-- Returns the repository references of this project and all the projects this project references, recursively -->
+  <Target Name="GetTransitiveRepositoryReferences" Outputs="@(TransitiveRepositoryReference)">
+    <ItemGroup>
+      <_TransitiveRepositoryReference Include="@(RepositoryReference)" />
+    </ItemGroup>
+
+    <MSBuild Projects="@(RepositoryReference->'%(Identity).proj')"
+             Targets="GetTransitiveRepositoryReferences"
+             BuildInParallel="true">
+      <Output TaskParameter="TargetOutputs" ItemName="_DependencyTransitiveRepositoryReference" />
+    </MSBuild>
+
+    <!-- When items are transferred between projects, they get metadata indicating their source (e.g. MSBuildSourceProjectFile).
+         This is problematic because it causes introduces distinctness on the items. For example, an arcade RepositoryReference
+         that comes from both msbuild and source-build-externals will be treated as two separate items. But we only want one arcade
+         reference. To prevent this from happening, use the Distinct method which operates on just the Identity to get a unique set
+         of values. -->
+    <ItemGroup>
+      <TransitiveRepositoryReference Include="@(_DependencyTransitiveRepositoryReference->Distinct())" />
+    </ItemGroup>
+
+    <!-- Exclude repositories that currently don't build when not building source-only. -->
+    <ItemGroup Condition="'$(DotNetBuildSourceOnly)' != 'true'">
+      <TransitiveRepositoryReference Remove="roslyn" />
+      <TransitiveRepositoryReference Remove="msbuild" />
+      <TransitiveRepositoryReference Remove="aspnetcore" />
+      <TransitiveRepositoryReference Remove="razor" />
+      <TransitiveRepositoryReference Remove="format" />
+      <TransitiveRepositoryReference Remove="nuget-client" />
+      <TransitiveRepositoryReference Remove="fsharp" />
+      <TransitiveRepositoryReference Remove="vstest" />
+      <TransitiveRepositoryReference Remove="installer" />
+    </ItemGroup>
+  </Target>
+
+  <!-- Outputs a dependency graph of the repos, in YAML form, to the console. -->
+  <Target Name="ShowDependencyGraph">
+    <MSBuild Projects="$(MSBuildProjectFullPath)"
+             Targets="GetDependencyGraphString"
+             Properties="DependencyGraphIndent=">
+      <Output TaskParameter="TargetOutputs" PropertyName="DependencyGraphString" />
+    </MSBuild>
+
+    <!-- Replace the '_' placeholder with spaces -->
+    <Message Importance="High" Text="$(DependencyGraphString.Replace('_', ' ').Trim())" />
+  </Target>
+
+  <!-- Recursively walks the repo dependency graph gathering each repo's dependencies which are returned as a YAML string representation -->
+  <Target Name="GetDependencyGraphString"
+          DependsOnTargets="GetTransitiveRepositoryReferences"
+          Outputs="$(DependencyGraphString)">
+    <PropertyGroup>
+      <!-- Each step deeper in the graph builds up the indentation used for the YAML representation.
+           Use '_' as a placeholder for the space ' ' character since trailing spaces aren't handled well as property values. -->
+      <_NextIndent>$(DependencyGraphIndent)__</_NextIndent>
+    </PropertyGroup>
+
+    <!-- For each of the current project's direct repo dependencies, recursively call the target to get each of there dependency graphs -->
+    <MSBuild Projects="@(RepositoryReference->'%(Identity).proj')"
+             Targets="GetDependencyGraphString"
+             Properties="DependencyGraphIndent=$(_NextIndent)">
+      <Output TaskParameter="TargetOutputs" ItemName="_DependencyGraphString" />
+    </MSBuild>
+
+    <!-- Append all the separate dependency graph snippets together -->
+    <PropertyGroup>
+      <!-- Ensure the item `;` separators are removed from the the target output -->
+      <_DependencyGraphString>@(_DependencyGraphString, '')</_DependencyGraphString>
+      <_LineBreak>%0a</_LineBreak>
+      <DependencyGraphString>$(DependencyGraphIndent)-_$(RepositoryName)$(_LineBreak)$(_DependencyGraphString)</DependencyGraphString>
+    </PropertyGroup>
+  </Target>
+
 </Project>

src/SourceBuild/content/repo-projects/aspnetcore.proj

@@ -22,11 +22,16 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
-    <RepositoryReference Include="runtime" />
-    <RepositoryReference Include="msbuild" />
     <RepositoryReference Include="roslyn" />
-    <RepositoryReference Include="roslyn-analyzers" />
+    <RepositoryReference Include="runtime" />
+    <RepositoryReference Include="xdt" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="nuget-client" />
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
+    <RepositoryReference Include="symreader" />
   </ItemGroup>
 
   <ItemGroup>

src/SourceBuild/content/repo-projects/cecil.proj

@@ -4,4 +4,8 @@
     <RepositoryReference Include="arcade" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-reference-packages" />
+  </ItemGroup>
+
 </Project>

src/SourceBuild/content/repo-projects/command-line-api.proj

@@ -10,4 +10,8 @@
     <RepositoryReference Include="arcade" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-reference-packages" />
+  </ItemGroup>
+
 </Project>