From ed324e09c77034992ce5af746f37d99dd69580f6 Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Thu, 22 Feb 2024 10:25:55 -0600 Subject: [PATCH] Isolate package flow between repos (#18557) As a prerequisite to enabling parallel builds of the repos in the VMR, it's necessary to isolate the package flow between repos. This ensures that repo dependencies are defined correctly. Prior to these changes, the packages outputted by repos went to the same directory (Shipping or NonShipping) and those two directories were used as the package feed inputs to dependent repos. This would be dangerous in the context of running builds in parallel across repos because any ill-defined dependencies could lead to race conditions. For example, consider the msbuild repo's dependency on System.Text.Json from the runtime repo. If runtime was not defined as a dependency of msbuild, then there's no guarantee that System.Text.Json will exist when it restores it. Based on timing, it may exist in one build and then not exist in another build. To solve this problem, the packages output from a repos build are placed in a repo-specific package location as a sub-directory of the package location (Shipping or NonShipping). Previously, all repos would output their packages to `artifacts/packages/Release/[NonShipping|Shipping]`. With these changes, they output to `artifacts/packages/Release/[NonShipping|Shipping]/`. This isolates all packages on a per-repo basis. The next step is to provide access to these packages based on dependencies. Going back to msbuild's dependency on runtime, this is accomplished by modifying msbuild's nuget.config file to include feeds specific to runtime: ```xml ``` --- .../generate-graphviz/generate-graphviz.proj | 49 ----- .../repo-projects/Directory.Build.props | 5 + .../repo-projects/Directory.Build.targets | 168 ++++++++++++++---- .../content/repo-projects/aspnetcore.proj | 13 +- .../content/repo-projects/cecil.proj | 4 + .../repo-projects/command-line-api.proj | 4 + .../repo-projects/deployment-tools.proj | 6 +- .../content/repo-projects/diagnostics.proj | 4 + .../content/repo-projects/dotnet.proj | 38 +--- .../content/repo-projects/emsdk.proj | 4 + .../content/repo-projects/format.proj | 10 +- .../content/repo-projects/fsharp.proj | 8 +- .../content/repo-projects/installer.proj | 11 +- .../content/repo-projects/msbuild.proj | 6 +- .../content/repo-projects/nuget-client.proj | 4 +- .../repo-projects/package-source-build.proj | 65 ++++--- .../content/repo-projects/razor.proj | 5 +- .../repo-projects/roslyn-analyzers.proj | 6 +- .../content/repo-projects/roslyn.proj | 7 +- .../content/repo-projects/runtime.proj | 9 +- .../content/repo-projects/scenario-tests.proj | 8 +- .../content/repo-projects/sdk.proj | 25 ++- .../repo-projects/source-build-externals.proj | 5 + .../content/repo-projects/sourcelink.proj | 4 + .../content/repo-projects/symreader.proj | 4 + .../content/repo-projects/templating.proj | 8 +- .../content/repo-projects/test-templates.proj | 5 +- .../content/repo-projects/vstest.proj | 6 +- 28 files changed, 304 insertions(+), 187 deletions(-) delete mode 100644 src/SourceBuild/content/eng/tools/generate-graphviz/generate-graphviz.proj diff --git a/src/SourceBuild/content/eng/tools/generate-graphviz/generate-graphviz.proj b/src/SourceBuild/content/eng/tools/generate-graphviz/generate-graphviz.proj deleted file mode 100644 index b35a7aa50637..000000000000 --- a/src/SourceBuild/content/eng/tools/generate-graphviz/generate-graphviz.proj +++ /dev/null @@ -1,49 +0,0 @@ - - - - $(NetCurrent) - - - - - - - - - - - - - - - - - - - $(BaseIntermediateOutputPath)graphviz.dot - $(BaseIntermediateOutputPath)graphviz.png - digraph { -graph [ dpi = 150 ] -@(RepoLink -> '%(Text)') -} - - - - - - - - - - - - diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.props b/src/SourceBuild/content/repo-projects/Directory.Build.props index 77f9a7ca8373..ec56c01a858d 100644 --- a/src/SourceBuild/content/repo-projects/Directory.Build.props +++ b/src/SourceBuild/content/repo-projects/Directory.Build.props @@ -33,6 +33,11 @@ $(BaseIntermediateOutputPath)$([System.IO.Path]::GetFileName('$(OriginalNuGetConfigFile)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'source-built-sdks')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsShippingPackagesDir)', '$(RepositoryName)')) + $(ReferencePackagesDir) + $([MSBuild]::NormalizeDirectory('$(ArtifactsNonShippingPackagesDir)', '$(RepositoryName)')) + $(ReferencePackagesDir) + $([MSBuild]::ValueOrDefault('$(ARCADE_BOOTSTRAP_VERSION)', '$(ArcadeSdkVersion)')) diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.targets b/src/SourceBuild/content/repo-projects/Directory.Build.targets index 7830ecbde0c5..bfa4fc5945dc 100644 --- a/src/SourceBuild/content/repo-projects/Directory.Build.targets +++ b/src/SourceBuild/content/repo-projects/Directory.Build.targets @@ -32,19 +32,6 @@ $(IntermediateSymbolsRootDir)$(RepositoryName) - - - - - - - - - - - - - @@ -53,6 +40,28 @@ + + + + + + + source-built-%(Identity) + source-built-transport-%(Identity) + $(ArtifactsShippingPackagesDir)/%(Identity)/ + $(ArtifactsNonShippingPackagesDir)/%(Identity)/ + + + + + + + + + + - - source-built - - source-built-transport - $(SourceBuiltShippingNuGetSourceName);$(SourceBuiltNonShippingNuGetSourceName) - ExtraSources - $(SourceBuildSources);$(ExtraSourcesNuGetSourceName) prebuilt previously-source-built reference-packages - $(SourceBuildSources);$(PrebuiltNuGetSourceName);$(PreviouslySourceBuiltNuGetSourceName);$(ReferencePackagesNuGetSourceName) + + <_CommonBuildSources Include="@(DependentRepoSourceName)" /> + <_CommonBuildSources Include="$(ExtraSourcesNuGetSourceName)" Condition="'$(ExtraRestoreSourcePath)' != ''" /> + + + + <_BuildSources Condition="'$(DotNetBuildSourceOnly)' == 'true'" + Include="$(PrebuiltNuGetSourceName);$(PreviouslySourceBuiltNuGetSourceName);$(ReferencePackagesNuGetSourceName)" /> + <_BuildSources Include="@(_CommonBuildSources)" /> + + + + - <_CurrentSourceBuiltPackages Include="$(ArtifactsPackagesDir)**\*.nupkg" + + <_CurrentSourceBuiltPackages Include="@(DependentRepoPackageFile)" Condition="!$([System.String]::Copy('%(Identity)').EndsWith('.symbols.nupkg'))" /> <_PreviouslyBuiltSourceBuiltPackages Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" /> @@ -268,11 +290,12 @@ - + - <_DependentProject Include="@(RepositoryReference -> '%(Identity).proj')" /> + <_DependentProject Include="@(TransitiveRepositoryReference -> '%(Identity).proj')" /> @@ -397,14 +418,12 @@ - $(ArtifactsNonShippingPackagesDir) - $(ReferencePackagesDir) + $(RepoArtifactsNonShippingPackagesDir) - $(ArtifactsShippingPackagesDir) - $(ReferencePackagesDir) + $(RepoArtifactsShippingPackagesDir) @@ -505,8 +524,7 @@ Inputs="$(MSBuildProjectFullPath)" Outputs="$(BaseIntermediateOutputPath)ExtractToolPackage.complete"> - <_ToolPackagesRoot Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$(ArtifactsNonShippingPackagesDir) - <_ToolPackagesRoot Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir) + <_ToolPackagesRoot>$(RepoArtifactsNonShippingPackagesDir) @@ -693,4 +711,78 @@ + + + + <_TransitiveRepositoryReference Include="@(RepositoryReference)" /> + + + + + + + + + <_TransitiveRepositoryReference Include="@(_DependencyTransitiveRepositoryReference)" + RemoveMetadata="MSBuildSourceProjectFile;MSBuildSourceTargetName;OriginalItemSpec" /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_NextIndent>$(DependencyGraphIndent)__ + + + + + + + + + + + <_DependencyGraphString>@(_DependencyGraphString, '') + <_LineBreak>%0a + $(DependencyGraphIndent)-_$(RepositoryName)$(_LineBreak)$(_DependencyGraphString) + + + diff --git a/src/SourceBuild/content/repo-projects/aspnetcore.proj b/src/SourceBuild/content/repo-projects/aspnetcore.proj index 59bd8e296c61..24dddd442f40 100644 --- a/src/SourceBuild/content/repo-projects/aspnetcore.proj +++ b/src/SourceBuild/content/repo-projects/aspnetcore.proj @@ -22,11 +22,16 @@ - - - - + + + + + + + + + diff --git a/src/SourceBuild/content/repo-projects/cecil.proj b/src/SourceBuild/content/repo-projects/cecil.proj index c4068fe7b99c..9cb33b7c4fa7 100644 --- a/src/SourceBuild/content/repo-projects/cecil.proj +++ b/src/SourceBuild/content/repo-projects/cecil.proj @@ -4,4 +4,8 @@ + + + + diff --git a/src/SourceBuild/content/repo-projects/command-line-api.proj b/src/SourceBuild/content/repo-projects/command-line-api.proj index a16b2412a19b..28ba676c0e24 100644 --- a/src/SourceBuild/content/repo-projects/command-line-api.proj +++ b/src/SourceBuild/content/repo-projects/command-line-api.proj @@ -10,4 +10,8 @@ + + + + diff --git a/src/SourceBuild/content/repo-projects/deployment-tools.proj b/src/SourceBuild/content/repo-projects/deployment-tools.proj index 85976198a006..7fd4eb25fb1f 100644 --- a/src/SourceBuild/content/repo-projects/deployment-tools.proj +++ b/src/SourceBuild/content/repo-projects/deployment-tools.proj @@ -2,7 +2,11 @@ - + + + + + diff --git a/src/SourceBuild/content/repo-projects/diagnostics.proj b/src/SourceBuild/content/repo-projects/diagnostics.proj index c4068fe7b99c..9cb33b7c4fa7 100644 --- a/src/SourceBuild/content/repo-projects/diagnostics.proj +++ b/src/SourceBuild/content/repo-projects/diagnostics.proj @@ -4,4 +4,8 @@ + + + + diff --git a/src/SourceBuild/content/repo-projects/dotnet.proj b/src/SourceBuild/content/repo-projects/dotnet.proj index 0de74d67944c..c5f305b1bedc 100644 --- a/src/SourceBuild/content/repo-projects/dotnet.proj +++ b/src/SourceBuild/content/repo-projects/dotnet.proj @@ -11,42 +11,12 @@ - If we have a repo that is not in sdk's dependency tree, we can still build it by including it here. --> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + diff --git a/src/SourceBuild/content/repo-projects/emsdk.proj b/src/SourceBuild/content/repo-projects/emsdk.proj index df9c063263e6..a210d9c4bd7f 100644 --- a/src/SourceBuild/content/repo-projects/emsdk.proj +++ b/src/SourceBuild/content/repo-projects/emsdk.proj @@ -14,4 +14,8 @@ + + + + diff --git a/src/SourceBuild/content/repo-projects/format.proj b/src/SourceBuild/content/repo-projects/format.proj index 9418953b7e8d..91744a768953 100644 --- a/src/SourceBuild/content/repo-projects/format.proj +++ b/src/SourceBuild/content/repo-projects/format.proj @@ -2,9 +2,17 @@ + + + + + + - + + + diff --git a/src/SourceBuild/content/repo-projects/fsharp.proj b/src/SourceBuild/content/repo-projects/fsharp.proj index bac7243b5321..2749c6f02bb1 100644 --- a/src/SourceBuild/content/repo-projects/fsharp.proj +++ b/src/SourceBuild/content/repo-projects/fsharp.proj @@ -20,8 +20,12 @@ - - + + + + + + diff --git a/src/SourceBuild/content/repo-projects/installer.proj b/src/SourceBuild/content/repo-projects/installer.proj index 541b3205dc68..cb5fcc0512aa 100644 --- a/src/SourceBuild/content/repo-projects/installer.proj +++ b/src/SourceBuild/content/repo-projects/installer.proj @@ -37,17 +37,26 @@ + + + - + + + + + + + - - - - - - - - - - - - - - - - $(ArtifactsAssetsDir)$(SourceBuiltArtifactsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension) .version - + - - + + <_AllRepoFiles Include="$(ArtifactsShippingPackagesDir)/**" /> + <_AllRepoFiles Include="$(ArtifactsNonShippingPackagesDir)/**" /> + <_ReferencePackageFiles Include="$(ReferencePackagesDir)**" /> + - + + + <_SourceBuiltLayoutDir>$([MSBuild]::NormalizeDirectory('$(BaseIntermediateOutputPath)', 'artifacts-layout')) + SourceBuildReferencePackages + + + + + + - + + + diff --git a/src/SourceBuild/content/repo-projects/razor.proj b/src/SourceBuild/content/repo-projects/razor.proj index 800961adeb73..99eca651e3bc 100644 --- a/src/SourceBuild/content/repo-projects/razor.proj +++ b/src/SourceBuild/content/repo-projects/razor.proj @@ -2,8 +2,11 @@ - + + + + diff --git a/src/SourceBuild/content/repo-projects/roslyn-analyzers.proj b/src/SourceBuild/content/repo-projects/roslyn-analyzers.proj index 23819f4994d4..eee275a054be 100644 --- a/src/SourceBuild/content/repo-projects/roslyn-analyzers.proj +++ b/src/SourceBuild/content/repo-projects/roslyn-analyzers.proj @@ -8,8 +8,12 @@ + + + - + + diff --git a/src/SourceBuild/content/repo-projects/roslyn.proj b/src/SourceBuild/content/repo-projects/roslyn.proj index 5f1c34aa5e98..dfbeb7102904 100644 --- a/src/SourceBuild/content/repo-projects/roslyn.proj +++ b/src/SourceBuild/content/repo-projects/roslyn.proj @@ -28,9 +28,12 @@ - - + + + + + diff --git a/src/SourceBuild/content/repo-projects/runtime.proj b/src/SourceBuild/content/repo-projects/runtime.proj index aa5b3296298f..9ad2c103588c 100644 --- a/src/SourceBuild/content/repo-projects/runtime.proj +++ b/src/SourceBuild/content/repo-projects/runtime.proj @@ -33,8 +33,13 @@ - - + + + + + + + diff --git a/src/SourceBuild/content/repo-projects/scenario-tests.proj b/src/SourceBuild/content/repo-projects/scenario-tests.proj index 25c6f1b9cd57..6d9422b61f06 100644 --- a/src/SourceBuild/content/repo-projects/scenario-tests.proj +++ b/src/SourceBuild/content/repo-projects/scenario-tests.proj @@ -1,9 +1,13 @@ - - + + + + + + diff --git a/src/SourceBuild/content/repo-projects/sdk.proj b/src/SourceBuild/content/repo-projects/sdk.proj index 69c6a2826dc4..d307692ac8c1 100644 --- a/src/SourceBuild/content/repo-projects/sdk.proj +++ b/src/SourceBuild/content/repo-projects/sdk.proj @@ -14,17 +14,28 @@ - + + + + + + - + + + + + + - - - - - + + + + + + diff --git a/src/SourceBuild/content/repo-projects/source-build-externals.proj b/src/SourceBuild/content/repo-projects/source-build-externals.proj index a391c4cd6689..c86415ade2f4 100644 --- a/src/SourceBuild/content/repo-projects/source-build-externals.proj +++ b/src/SourceBuild/content/repo-projects/source-build-externals.proj @@ -10,4 +10,9 @@ true + + + + + diff --git a/src/SourceBuild/content/repo-projects/sourcelink.proj b/src/SourceBuild/content/repo-projects/sourcelink.proj index 87026a8e9930..3f0451a3867a 100644 --- a/src/SourceBuild/content/repo-projects/sourcelink.proj +++ b/src/SourceBuild/content/repo-projects/sourcelink.proj @@ -12,4 +12,8 @@ + + + + diff --git a/src/SourceBuild/content/repo-projects/symreader.proj b/src/SourceBuild/content/repo-projects/symreader.proj index c4068fe7b99c..9cb33b7c4fa7 100644 --- a/src/SourceBuild/content/repo-projects/symreader.proj +++ b/src/SourceBuild/content/repo-projects/symreader.proj @@ -4,4 +4,8 @@ + + + + diff --git a/src/SourceBuild/content/repo-projects/templating.proj b/src/SourceBuild/content/repo-projects/templating.proj index 999d5a167488..9677f1100ce1 100644 --- a/src/SourceBuild/content/repo-projects/templating.proj +++ b/src/SourceBuild/content/repo-projects/templating.proj @@ -9,8 +9,12 @@ - - + + + + + + diff --git a/src/SourceBuild/content/repo-projects/test-templates.proj b/src/SourceBuild/content/repo-projects/test-templates.proj index a272afd4f160..3013d77fdb4a 100644 --- a/src/SourceBuild/content/repo-projects/test-templates.proj +++ b/src/SourceBuild/content/repo-projects/test-templates.proj @@ -6,7 +6,10 @@ - + + + + diff --git a/src/SourceBuild/content/repo-projects/vstest.proj b/src/SourceBuild/content/repo-projects/vstest.proj index c228f58a635b..327d8aea1bdf 100644 --- a/src/SourceBuild/content/repo-projects/vstest.proj +++ b/src/SourceBuild/content/repo-projects/vstest.proj @@ -9,8 +9,12 @@ + + + - + +