From 926fd925af01b3fd9a91e20924e932273ba5ff49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=A4tzel?= Date: Thu, 10 Oct 2024 22:03:09 +0000 Subject: [PATCH] Speed up runtime startup by moving more of setup before dotnet build Since the building of the collection of reused Pine value instances took several seconds, move this process to an earlier phase and integrate a form that is fast to load as an embedded resource. --- .github/workflows/publish-to-release.yml | 7 +- .github/workflows/test-and-publish.yml | 5 + implement/Pine.Core/.gitignore | 1 + implement/Pine.Core/Pine.Core.csproj | 4 + implement/Pine.Core/PineValue.cs | 16 +- implement/Pine.Core/ReusedInstances.cs | 284 +++++++++++++++++- .../Pine.UnitTests/ReusedInstancesTests.cs | 40 ++- implement/prebuild/Program.cs | 38 +++ implement/prebuild/README.md | 3 + implement/prebuild/prebuild.csproj | 13 + 10 files changed, 396 insertions(+), 15 deletions(-) create mode 100644 implement/Pine.Core/.gitignore create mode 100644 implement/prebuild/Program.cs create mode 100644 implement/prebuild/README.md create mode 100644 implement/prebuild/prebuild.csproj diff --git a/.github/workflows/publish-to-release.yml b/.github/workflows/publish-to-release.yml index 8b74066cb..a32cdc448 100644 --- a/.github/workflows/publish-to-release.yml +++ b/.github/workflows/publish-to-release.yml @@ -41,8 +41,13 @@ jobs: - name: Clean package cache as a temporary workaround for https://github.com/actions/setup-dotnet/issues/155 run: dotnet clean ./implement/test-elm-time/test-elm-time.csproj && dotnet nuget locals all --clear + - name: Prebuild + working-directory: ./implement/ + run: | + dotnet run --project ./prebuild/prebuild.csproj + - name: dotnet publish - run: dotnet publish -c Debug -r ${{ matrix.publish-runtime-id }} --self-contained true -p:PublishSingleFile=true -p:PublishReadyToRun=true -p:PublishReadyToRunShowWarnings=true --output ./dotnet-build ./implement/pine + run: dotnet publish -c Release -r ${{ matrix.publish-runtime-id }} --self-contained true -p:PublishSingleFile=true -p:PublishReadyToRun=true -p:PublishReadyToRunShowWarnings=true --output ./dotnet-build ./implement/pine - name: Copy artifacts to publish shell: pwsh diff --git a/.github/workflows/test-and-publish.yml b/.github/workflows/test-and-publish.yml index 3c8f088fd..52773e44b 100644 --- a/.github/workflows/test-and-publish.yml +++ b/.github/workflows/test-and-publish.yml @@ -37,6 +37,11 @@ jobs: dotnet clean ./implement/PineTest/Pine.UnitTests/Pine.UnitTests.csproj && dotnet nuget locals all --clear dotnet clean ./implement/test-elm-time/test-elm-time.csproj && dotnet nuget locals all --clear + - name: Prebuild + working-directory: ./implement/ + run: | + dotnet run --project ./prebuild/prebuild.csproj + - name: Run Pine tests with dotnet test run: | dotnet test ./implement/PineTest/Pine.UnitTests/Pine.UnitTests.csproj --logger "trx" --diag "./implement/PineTest/Pine.UnitTests/TestResults/diag-log.txt" --verbosity diagnostic diff --git a/implement/Pine.Core/.gitignore b/implement/Pine.Core/.gitignore new file mode 100644 index 000000000..9388e4835 --- /dev/null +++ b/implement/Pine.Core/.gitignore @@ -0,0 +1 @@ +prebuilt-artifact/ diff --git a/implement/Pine.Core/Pine.Core.csproj b/implement/Pine.Core/Pine.Core.csproj index ce0779a19..eee8a373f 100644 --- a/implement/Pine.Core/Pine.Core.csproj +++ b/implement/Pine.Core/Pine.Core.csproj @@ -36,6 +36,10 @@ + + + + diff --git a/implement/Pine.Core/PineValue.cs b/implement/Pine.Core/PineValue.cs index 26e0a32ac..b640db6d3 100644 --- a/implement/Pine.Core/PineValue.cs +++ b/implement/Pine.Core/PineValue.cs @@ -199,10 +199,24 @@ public readonly record struct ListValueStruct /// Construct a list value from a sequence of other values. /// public ListValueStruct(IReadOnlyList elements) + : + this(elements, ComputeSlimHashCode(elements)) + { + } + + public ListValueStruct(ListValue instance) + : + this(instance.Elements, instance.slimHashCode) + { + } + + private ListValueStruct( + IReadOnlyList elements, + int slimHashCode) { Elements = elements; - slimHashCode = ComputeSlimHashCode(elements); + this.slimHashCode = slimHashCode; } public bool Equals(ListValueStruct other) diff --git a/implement/Pine.Core/ReusedInstances.cs b/implement/Pine.Core/ReusedInstances.cs index fbfe192ae..228fea051 100644 --- a/implement/Pine.Core/ReusedInstances.cs +++ b/implement/Pine.Core/ReusedInstances.cs @@ -7,10 +7,10 @@ namespace Pine.Core; public record ReusedInstances( - IEnumerable expressionRootsSource) + System.Func> LoadExpressionRootsSource) { public static readonly ReusedInstances Instance = - new(expressionRootsSource: ExpressionsSource()); + new(LoadExpressionRootsSource: ExpressionsSource); public FrozenDictionary? ListValues { private set; get; } @@ -44,7 +44,7 @@ static ReusedInstances() Instance.Build(); } - static IEnumerable ExpressionsSource() + public static IEnumerable ExpressionsSource() { foreach (var namedExpression in PopularExpression.BuildPopularExpressionDictionary()) { @@ -52,7 +52,250 @@ static IEnumerable ExpressionsSource() } } - public void Build() + + public record PrebuiltListEntry( + string Key, + PrebuiltListEntryValue Value); + + public record PrebuiltListEntryValue( + string? BlobBytesBase64, + IReadOnlyList? ListItemsKeys); + + const string expectedInCompilerKey = "expected-in-compiler-container"; + + public static System.ReadOnlyMemory BuildPrecompiledDictFile( + PineListValueReusedInstances source) + { + var entriesList = PrebuildListEntries(source); + + return System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(entriesList); + } + + public static IReadOnlyList PrebuildListEntries( + PineListValueReusedInstances source) + { + var (_, allBlobs) = + CollectAllComponentsFromRoots( + [.. source.PineValueLists.Values + ,..source.ValuesExpectedInCompilerLists + ,..source.ValuesExpectedInCompilerBlobs]); + + var mutatedBlobsDict = new Dictionary(); + + var blobEntriesList = + allBlobs + .OrderBy(blob => blob.Bytes.Length) + .Select((blobValue, blobIndex) => + { + var entryKey = "blob-" + blobIndex.ToString(); + + mutatedBlobsDict[blobValue] = entryKey; + + return + new PrebuiltListEntry( + Key: entryKey, + new PrebuiltListEntryValue( + BlobBytesBase64: System.Convert.ToBase64String(blobValue.Bytes.Span), + ListItemsKeys: null)); + }) + .ToList(); + + var listsOrdered = + source.PineValueLists.Values + .Concat(source.ValuesExpectedInCompilerLists) + .Distinct() + .OrderBy(l => l.NodesCount) + .ToList(); + + var mutatedListDict = new Dictionary(); + + string itemId(PineValue itemValue) + { + if (itemValue is PineValue.BlobValue itemBlob) + return mutatedBlobsDict[itemBlob]; + + if (itemValue is PineValue.ListValue itemList) + return mutatedListDict[itemList]; + + throw new System.NotImplementedException( + "Unexpected item value type: " + itemValue.GetType()); + } + + PrebuiltListEntryValue entryListFromItems( + IReadOnlyList itemValues) + { + var itemsIds = itemValues.Select(itemId).ToArray(); + + return new PrebuiltListEntryValue( + BlobBytesBase64: null, + ListItemsKeys: itemsIds); + } + + var listEntriesList = + listsOrdered + .Select((listInstance, index) => + { + var entryKey = "list-" + index.ToString(); + + mutatedListDict[listInstance] = entryKey; + + return + new PrebuiltListEntry( + Key: entryKey, + Value: entryListFromItems(listInstance.Elements)); + }) + .ToList(); + + var ventry = + new PrebuiltListEntry( + Key: expectedInCompilerKey, + Value: entryListFromItems( + [.. source.ValuesExpectedInCompilerBlobs.Cast().Concat(source.ValuesExpectedInCompilerLists)])); + + return + [..blobEntriesList + ,..listEntriesList + ,ventry]; + } + + public const string EmbeddedResourceFilePath = "prebuilt-artifact/reused-pine-values.json"; + + public static PineListValueReusedInstances LoadPrecompiledFromEmbeddedOrDefault( + System.Reflection.Assembly assembly, + System.Func> loadExpressionRootsSource) => + LoadEmbeddedPrebuilt(assembly) + .Extract(err => + { + System.Console.WriteLine("Failed loading from embedded resource: " + err); + + return BuildPineListValueReusedInstances(loadExpressionRootsSource()); + }); + + public static Result LoadEmbeddedPrebuilt( + System.Reflection.Assembly assembly, + string embeddedResourceFilePath = EmbeddedResourceFilePath) + { + /* + var inspect = + DotNetAssembly.LoadDirectoryFilesFromManifestEmbeddedFileProviderAsDictionary( + directoryPath: ["prebuilt-artifact"], + assembly: assembly); + */ + + var manifestEmbeddedProvider = + new Microsoft.Extensions.FileProviders.ManifestEmbeddedFileProvider(assembly); + + var embeddedFileInfo = manifestEmbeddedProvider.GetFileInfo(embeddedResourceFilePath); + + if (!embeddedFileInfo.Exists) + { + return "Did not find file " + embeddedResourceFilePath + " in assembly " + assembly.FullName; + } + + if (embeddedFileInfo.Length is 0) + { + return "File " + embeddedResourceFilePath + " in assembly " + assembly.FullName + " is empty"; + } + + using var readStream = embeddedFileInfo.CreateReadStream(); + + using var memoryStream = new System.IO.MemoryStream(); + + readStream.CopyTo(memoryStream); + + return LoadFromPrebuiltJson(memoryStream.ToArray()); + } + + + public static PineListValueReusedInstances LoadFromPrebuiltJson(System.ReadOnlyMemory json) + { + var parsed = + System.Text.Json.JsonSerializer.Deserialize>(json.Span); + + return LoadFromPrebuilt(parsed); + } + + public static PineListValueReusedInstances LoadFromPrebuilt( + IReadOnlyList entries) + { + var mutatedDict = new Dictionary(); + + PineValue contructValue( + PrebuiltListEntryValue entryValue) + { + if (entryValue.BlobBytesBase64 is { } bytesBase64) + { + var bytes = System.Convert.FromBase64String(bytesBase64); + + return PineValue.Blob(bytes); + } + + if (entryValue.ListItemsKeys is { } listItemsKeys) + { + var items = new PineValue[listItemsKeys.Count]; + + for (int i = 0; i < items.Length; ++i) + { + items[i] = mutatedDict[listItemsKeys[i]]; + } + + return PineValue.List(items); + } + + throw new System.NotImplementedException( + "Unexpected entry type: " + entryValue.GetType()); + } + + for (var i = 0; i < entries.Count; ++i) + { + var entry = entries[i]; + + mutatedDict[entry.Key] = contructValue(entry.Value); + } + + var valuesExpectedInCompilerContainer = mutatedDict[expectedInCompilerKey]; + + if (valuesExpectedInCompilerContainer is not PineValue.ListValue valuesExpectedInCompilerList) + { + throw new System.Exception( + "Did not find container with key " + expectedInCompilerKey); + } + + var listValuesExpectedInCompiler = + valuesExpectedInCompilerList.Elements.OfType() + .ToHashSet(); + + var blobValuesExpectedInCompiler = + valuesExpectedInCompilerList.Elements.OfType() + .ToHashSet(); + + var valueListsDict = new Dictionary(); + + foreach (var item in mutatedDict) + { + if (item.Key is expectedInCompilerKey) + continue; + + if (item.Value is not PineValue.ListValue listValue) + continue; + + valueListsDict[new PineValue.ListValue.ListValueStruct(listValue)] = listValue; + } + + return new PineListValueReusedInstances( + listValuesExpectedInCompiler, + blobValuesExpectedInCompiler, + valueListsDict); + } + + + public record PineListValueReusedInstances( + IReadOnlySet ValuesExpectedInCompilerLists, + IReadOnlySet ValuesExpectedInCompilerBlobs, + IReadOnlyDictionary PineValueLists); + + public static PineListValueReusedInstances BuildPineListValueReusedInstances( + IEnumerable expressionRootsSource) { var valueRootsFromProgramsSorted = expressionRootsSource @@ -64,12 +307,7 @@ public void Build() var (valuesExpectedInCompilerLists, valuesExpectedInCompilerBlobs) = CollectAllComponentsFromRoots(valueRootsFromProgramsSorted); - var valuesExpectedInCompilerSorted = - PineValue.ReusedBlobs - .Cast() - .Concat(valuesExpectedInCompilerBlobs) - .Concat(valuesExpectedInCompilerLists.OrderBy(listValue => listValue.NodesCount)) - .ToList(); + IReadOnlyDictionary PineValueLists; { var tempEncodingDict = new Dictionary(); @@ -139,13 +377,35 @@ public void Build() reusedListsDictInConstruction[rebuilt] = rebuilt; } - ListValues = + PineValueLists = reusedListsDictInConstruction .ToFrozenDictionary( keySelector: asRef => new PineValue.ListValue.ListValueStruct(asRef.Key.Elements), elementSelector: asRef => asRef.Value); } + return new PineListValueReusedInstances( + valuesExpectedInCompilerLists, + valuesExpectedInCompilerBlobs, + PineValueLists); + } + + public void Build() + { + var loadedPineListValues = + LoadPrecompiledFromEmbeddedOrDefault( + typeof(ReusedInstances).Assembly, + ExpressionsSource); + + ListValues = loadedPineListValues.PineValueLists.ToFrozenDictionary(); + + var valuesExpectedInCompilerSorted = + PineValue.ReusedBlobs + .Cast() + .Concat(loadedPineListValues.ValuesExpectedInCompilerBlobs) + .Concat(loadedPineListValues.ValuesExpectedInCompilerLists.OrderBy(listValue => listValue.NodesCount)) + .ToList(); + { /* * For the expression instances, we rebuild the collection again, because the literal expressions should @@ -153,7 +413,7 @@ public void Build() * */ var expressionsRoots = - expressionRootsSource.ToList(); + LoadExpressionRootsSource().ToList(); var allExpressionDescendants = Expression.CollectAllComponentsFromRoots(expressionsRoots); diff --git a/implement/PineTest/Pine.UnitTests/ReusedInstancesTests.cs b/implement/PineTest/Pine.UnitTests/ReusedInstancesTests.cs index 7a0d18892..9fbea362a 100644 --- a/implement/PineTest/Pine.UnitTests/ReusedInstancesTests.cs +++ b/implement/PineTest/Pine.UnitTests/ReusedInstancesTests.cs @@ -1,5 +1,6 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Pine.Core; +using System.Collections.Generic; namespace Pine.UnitTests; @@ -11,4 +12,41 @@ public void Ensure_reference_equality_between_mappings_between_reused_instances( { ReusedInstances.Instance.AssertReferenceEquality(); } -} + + [TestMethod] + public void Embedded_precompiled_pine_value_lists() + { + var fromFreshBuild = + ReusedInstances.BuildPineListValueReusedInstances( + ReusedInstances.ExpressionsSource()); + + var file = + ReusedInstances.BuildPrecompiledDictFile(fromFreshBuild); + + var parsedFile = + ReusedInstances.LoadFromPrebuiltJson(file); + + AssertPineValueListDictsAreEquivalent( + parsedFile.PineValueLists, + fromFreshBuild.PineValueLists); + + AssertPineValueListDictsAreEquivalent( + ReusedInstances.Instance.ListValues, + fromFreshBuild.PineValueLists); + } + + public static void AssertPineValueListDictsAreEquivalent( + IReadOnlyDictionary a, + IReadOnlyDictionary b) + { + if (a.Count != b.Count) + { + Assert.Fail("Counts are not equal: " + a.Count + " vs " + b.Count); + } + + foreach (var kv in a) + { + Assert.IsTrue(b.ContainsKey(kv.Key), "contains key"); + } + } +} \ No newline at end of file diff --git a/implement/prebuild/Program.cs b/implement/prebuild/Program.cs new file mode 100644 index 000000000..201040efc --- /dev/null +++ b/implement/prebuild/Program.cs @@ -0,0 +1,38 @@ +using Pine.Core; + +namespace prebuild; + +public class Program +{ + public const string DestinationFilePath = "./Pine.Core/" + ReusedInstances.EmbeddedResourceFilePath; + + public static void Main() + { + System.Console.WriteLine( + "Current working directory: " + System.Environment.CurrentDirectory); + + var fromFreshBuild = + ReusedInstances.BuildPineListValueReusedInstances( + ReusedInstances.ExpressionsSource()); + + var file = + ReusedInstances.BuildPrecompiledDictFile(fromFreshBuild); + + var absolutePath = System.IO.Path.GetFullPath(DestinationFilePath); + + System.Console.WriteLine( + "Resolved the destination path of " + DestinationFilePath + + " to " + absolutePath); + + System.IO.Directory.CreateDirectory( + System.IO.Path.GetDirectoryName(absolutePath)); + + System.IO.File.WriteAllBytes( + absolutePath, + file.ToArray()); + + System.Console.WriteLine( + "Saved the prebuilt dictionary with " + + fromFreshBuild.PineValueLists.Count + " list values to " + absolutePath); + } +} \ No newline at end of file diff --git a/implement/prebuild/README.md b/implement/prebuild/README.md new file mode 100644 index 000000000..1716c7e82 --- /dev/null +++ b/implement/prebuild/README.md @@ -0,0 +1,3 @@ +# prebuild + +The `prebuild.csproj` program builds artifacts to be embedded into the Pine assembly on build, and therefore, need to be built before building `Pine.Core.csproj` diff --git a/implement/prebuild/prebuild.csproj b/implement/prebuild/prebuild.csproj new file mode 100644 index 000000000..b7642c5ba --- /dev/null +++ b/implement/prebuild/prebuild.csproj @@ -0,0 +1,13 @@ + + + + Exe + net8.0 + enable + + + + + + +