diff --git a/.github/workflows/publish-to-release.yml b/.github/workflows/publish-to-release.yml index 8b74066cb..a32cdc448 100644 --- a/.github/workflows/publish-to-release.yml +++ b/.github/workflows/publish-to-release.yml @@ -41,8 +41,13 @@ jobs: - name: Clean package cache as a temporary workaround for https://github.com/actions/setup-dotnet/issues/155 run: dotnet clean ./implement/test-elm-time/test-elm-time.csproj && dotnet nuget locals all --clear + - name: Prebuild + working-directory: ./implement/ + run: | + dotnet run --project ./prebuild/prebuild.csproj + - name: dotnet publish - run: dotnet publish -c Debug -r ${{ matrix.publish-runtime-id }} --self-contained true -p:PublishSingleFile=true -p:PublishReadyToRun=true -p:PublishReadyToRunShowWarnings=true --output ./dotnet-build ./implement/pine + run: dotnet publish -c Release -r ${{ matrix.publish-runtime-id }} --self-contained true -p:PublishSingleFile=true -p:PublishReadyToRun=true -p:PublishReadyToRunShowWarnings=true --output ./dotnet-build ./implement/pine - name: Copy artifacts to publish shell: pwsh diff --git a/.github/workflows/test-and-publish.yml b/.github/workflows/test-and-publish.yml index 3c8f088fd..52773e44b 100644 --- a/.github/workflows/test-and-publish.yml +++ b/.github/workflows/test-and-publish.yml @@ -37,6 +37,11 @@ jobs: dotnet clean ./implement/PineTest/Pine.UnitTests/Pine.UnitTests.csproj && dotnet nuget locals all --clear dotnet clean ./implement/test-elm-time/test-elm-time.csproj && dotnet nuget locals all --clear + - name: Prebuild + working-directory: ./implement/ + run: | + dotnet run --project ./prebuild/prebuild.csproj + - name: Run Pine tests with dotnet test run: | dotnet test ./implement/PineTest/Pine.UnitTests/Pine.UnitTests.csproj --logger "trx" --diag "./implement/PineTest/Pine.UnitTests/TestResults/diag-log.txt" --verbosity diagnostic diff --git a/implement/Pine.Core/.gitignore b/implement/Pine.Core/.gitignore new file mode 100644 index 000000000..9388e4835 --- /dev/null +++ b/implement/Pine.Core/.gitignore @@ -0,0 +1 @@ +prebuilt-artifact/ diff --git a/implement/Pine.Core/Pine.Core.csproj b/implement/Pine.Core/Pine.Core.csproj index ce0779a19..eee8a373f 100644 --- a/implement/Pine.Core/Pine.Core.csproj +++ b/implement/Pine.Core/Pine.Core.csproj @@ -36,6 +36,10 @@ + + + + diff --git a/implement/Pine.Core/PineValue.cs b/implement/Pine.Core/PineValue.cs index 26e0a32ac..b640db6d3 100644 --- a/implement/Pine.Core/PineValue.cs +++ b/implement/Pine.Core/PineValue.cs @@ -199,10 +199,24 @@ public readonly record struct ListValueStruct /// Construct a list value from a sequence of other values. /// public ListValueStruct(IReadOnlyList elements) + : + this(elements, ComputeSlimHashCode(elements)) + { + } + + public ListValueStruct(ListValue instance) + : + this(instance.Elements, instance.slimHashCode) + { + } + + private ListValueStruct( + IReadOnlyList elements, + int slimHashCode) { Elements = elements; - slimHashCode = ComputeSlimHashCode(elements); + this.slimHashCode = slimHashCode; } public bool Equals(ListValueStruct other) diff --git a/implement/Pine.Core/ReusedInstances.cs b/implement/Pine.Core/ReusedInstances.cs index fbfe192ae..228fea051 100644 --- a/implement/Pine.Core/ReusedInstances.cs +++ b/implement/Pine.Core/ReusedInstances.cs @@ -7,10 +7,10 @@ namespace Pine.Core; public record ReusedInstances( - IEnumerable expressionRootsSource) + System.Func> LoadExpressionRootsSource) { public static readonly ReusedInstances Instance = - new(expressionRootsSource: ExpressionsSource()); + new(LoadExpressionRootsSource: ExpressionsSource); public FrozenDictionary? ListValues { private set; get; } @@ -44,7 +44,7 @@ static ReusedInstances() Instance.Build(); } - static IEnumerable ExpressionsSource() + public static IEnumerable ExpressionsSource() { foreach (var namedExpression in PopularExpression.BuildPopularExpressionDictionary()) { @@ -52,7 +52,250 @@ static IEnumerable ExpressionsSource() } } - public void Build() + + public record PrebuiltListEntry( + string Key, + PrebuiltListEntryValue Value); + + public record PrebuiltListEntryValue( + string? BlobBytesBase64, + IReadOnlyList? ListItemsKeys); + + const string expectedInCompilerKey = "expected-in-compiler-container"; + + public static System.ReadOnlyMemory BuildPrecompiledDictFile( + PineListValueReusedInstances source) + { + var entriesList = PrebuildListEntries(source); + + return System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(entriesList); + } + + public static IReadOnlyList PrebuildListEntries( + PineListValueReusedInstances source) + { + var (_, allBlobs) = + CollectAllComponentsFromRoots( + [.. source.PineValueLists.Values + ,..source.ValuesExpectedInCompilerLists + ,..source.ValuesExpectedInCompilerBlobs]); + + var mutatedBlobsDict = new Dictionary(); + + var blobEntriesList = + allBlobs + .OrderBy(blob => blob.Bytes.Length) + .Select((blobValue, blobIndex) => + { + var entryKey = "blob-" + blobIndex.ToString(); + + mutatedBlobsDict[blobValue] = entryKey; + + return + new PrebuiltListEntry( + Key: entryKey, + new PrebuiltListEntryValue( + BlobBytesBase64: System.Convert.ToBase64String(blobValue.Bytes.Span), + ListItemsKeys: null)); + }) + .ToList(); + + var listsOrdered = + source.PineValueLists.Values + .Concat(source.ValuesExpectedInCompilerLists) + .Distinct() + .OrderBy(l => l.NodesCount) + .ToList(); + + var mutatedListDict = new Dictionary(); + + string itemId(PineValue itemValue) + { + if (itemValue is PineValue.BlobValue itemBlob) + return mutatedBlobsDict[itemBlob]; + + if (itemValue is PineValue.ListValue itemList) + return mutatedListDict[itemList]; + + throw new System.NotImplementedException( + "Unexpected item value type: " + itemValue.GetType()); + } + + PrebuiltListEntryValue entryListFromItems( + IReadOnlyList itemValues) + { + var itemsIds = itemValues.Select(itemId).ToArray(); + + return new PrebuiltListEntryValue( + BlobBytesBase64: null, + ListItemsKeys: itemsIds); + } + + var listEntriesList = + listsOrdered + .Select((listInstance, index) => + { + var entryKey = "list-" + index.ToString(); + + mutatedListDict[listInstance] = entryKey; + + return + new PrebuiltListEntry( + Key: entryKey, + Value: entryListFromItems(listInstance.Elements)); + }) + .ToList(); + + var ventry = + new PrebuiltListEntry( + Key: expectedInCompilerKey, + Value: entryListFromItems( + [.. source.ValuesExpectedInCompilerBlobs.Cast().Concat(source.ValuesExpectedInCompilerLists)])); + + return + [..blobEntriesList + ,..listEntriesList + ,ventry]; + } + + public const string EmbeddedResourceFilePath = "prebuilt-artifact/reused-pine-values.json"; + + public static PineListValueReusedInstances LoadPrecompiledFromEmbeddedOrDefault( + System.Reflection.Assembly assembly, + System.Func> loadExpressionRootsSource) => + LoadEmbeddedPrebuilt(assembly) + .Extract(err => + { + System.Console.WriteLine("Failed loading from embedded resource: " + err); + + return BuildPineListValueReusedInstances(loadExpressionRootsSource()); + }); + + public static Result LoadEmbeddedPrebuilt( + System.Reflection.Assembly assembly, + string embeddedResourceFilePath = EmbeddedResourceFilePath) + { + /* + var inspect = + DotNetAssembly.LoadDirectoryFilesFromManifestEmbeddedFileProviderAsDictionary( + directoryPath: ["prebuilt-artifact"], + assembly: assembly); + */ + + var manifestEmbeddedProvider = + new Microsoft.Extensions.FileProviders.ManifestEmbeddedFileProvider(assembly); + + var embeddedFileInfo = manifestEmbeddedProvider.GetFileInfo(embeddedResourceFilePath); + + if (!embeddedFileInfo.Exists) + { + return "Did not find file " + embeddedResourceFilePath + " in assembly " + assembly.FullName; + } + + if (embeddedFileInfo.Length is 0) + { + return "File " + embeddedResourceFilePath + " in assembly " + assembly.FullName + " is empty"; + } + + using var readStream = embeddedFileInfo.CreateReadStream(); + + using var memoryStream = new System.IO.MemoryStream(); + + readStream.CopyTo(memoryStream); + + return LoadFromPrebuiltJson(memoryStream.ToArray()); + } + + + public static PineListValueReusedInstances LoadFromPrebuiltJson(System.ReadOnlyMemory json) + { + var parsed = + System.Text.Json.JsonSerializer.Deserialize>(json.Span); + + return LoadFromPrebuilt(parsed); + } + + public static PineListValueReusedInstances LoadFromPrebuilt( + IReadOnlyList entries) + { + var mutatedDict = new Dictionary(); + + PineValue contructValue( + PrebuiltListEntryValue entryValue) + { + if (entryValue.BlobBytesBase64 is { } bytesBase64) + { + var bytes = System.Convert.FromBase64String(bytesBase64); + + return PineValue.Blob(bytes); + } + + if (entryValue.ListItemsKeys is { } listItemsKeys) + { + var items = new PineValue[listItemsKeys.Count]; + + for (int i = 0; i < items.Length; ++i) + { + items[i] = mutatedDict[listItemsKeys[i]]; + } + + return PineValue.List(items); + } + + throw new System.NotImplementedException( + "Unexpected entry type: " + entryValue.GetType()); + } + + for (var i = 0; i < entries.Count; ++i) + { + var entry = entries[i]; + + mutatedDict[entry.Key] = contructValue(entry.Value); + } + + var valuesExpectedInCompilerContainer = mutatedDict[expectedInCompilerKey]; + + if (valuesExpectedInCompilerContainer is not PineValue.ListValue valuesExpectedInCompilerList) + { + throw new System.Exception( + "Did not find container with key " + expectedInCompilerKey); + } + + var listValuesExpectedInCompiler = + valuesExpectedInCompilerList.Elements.OfType() + .ToHashSet(); + + var blobValuesExpectedInCompiler = + valuesExpectedInCompilerList.Elements.OfType() + .ToHashSet(); + + var valueListsDict = new Dictionary(); + + foreach (var item in mutatedDict) + { + if (item.Key is expectedInCompilerKey) + continue; + + if (item.Value is not PineValue.ListValue listValue) + continue; + + valueListsDict[new PineValue.ListValue.ListValueStruct(listValue)] = listValue; + } + + return new PineListValueReusedInstances( + listValuesExpectedInCompiler, + blobValuesExpectedInCompiler, + valueListsDict); + } + + + public record PineListValueReusedInstances( + IReadOnlySet ValuesExpectedInCompilerLists, + IReadOnlySet ValuesExpectedInCompilerBlobs, + IReadOnlyDictionary PineValueLists); + + public static PineListValueReusedInstances BuildPineListValueReusedInstances( + IEnumerable expressionRootsSource) { var valueRootsFromProgramsSorted = expressionRootsSource @@ -64,12 +307,7 @@ public void Build() var (valuesExpectedInCompilerLists, valuesExpectedInCompilerBlobs) = CollectAllComponentsFromRoots(valueRootsFromProgramsSorted); - var valuesExpectedInCompilerSorted = - PineValue.ReusedBlobs - .Cast() - .Concat(valuesExpectedInCompilerBlobs) - .Concat(valuesExpectedInCompilerLists.OrderBy(listValue => listValue.NodesCount)) - .ToList(); + IReadOnlyDictionary PineValueLists; { var tempEncodingDict = new Dictionary(); @@ -139,13 +377,35 @@ public void Build() reusedListsDictInConstruction[rebuilt] = rebuilt; } - ListValues = + PineValueLists = reusedListsDictInConstruction .ToFrozenDictionary( keySelector: asRef => new PineValue.ListValue.ListValueStruct(asRef.Key.Elements), elementSelector: asRef => asRef.Value); } + return new PineListValueReusedInstances( + valuesExpectedInCompilerLists, + valuesExpectedInCompilerBlobs, + PineValueLists); + } + + public void Build() + { + var loadedPineListValues = + LoadPrecompiledFromEmbeddedOrDefault( + typeof(ReusedInstances).Assembly, + ExpressionsSource); + + ListValues = loadedPineListValues.PineValueLists.ToFrozenDictionary(); + + var valuesExpectedInCompilerSorted = + PineValue.ReusedBlobs + .Cast() + .Concat(loadedPineListValues.ValuesExpectedInCompilerBlobs) + .Concat(loadedPineListValues.ValuesExpectedInCompilerLists.OrderBy(listValue => listValue.NodesCount)) + .ToList(); + { /* * For the expression instances, we rebuild the collection again, because the literal expressions should @@ -153,7 +413,7 @@ public void Build() * */ var expressionsRoots = - expressionRootsSource.ToList(); + LoadExpressionRootsSource().ToList(); var allExpressionDescendants = Expression.CollectAllComponentsFromRoots(expressionsRoots); diff --git a/implement/PineTest/Pine.UnitTests/ReusedInstancesTests.cs b/implement/PineTest/Pine.UnitTests/ReusedInstancesTests.cs index 7a0d18892..9fbea362a 100644 --- a/implement/PineTest/Pine.UnitTests/ReusedInstancesTests.cs +++ b/implement/PineTest/Pine.UnitTests/ReusedInstancesTests.cs @@ -1,5 +1,6 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Pine.Core; +using System.Collections.Generic; namespace Pine.UnitTests; @@ -11,4 +12,41 @@ public void Ensure_reference_equality_between_mappings_between_reused_instances( { ReusedInstances.Instance.AssertReferenceEquality(); } -} + + [TestMethod] + public void Embedded_precompiled_pine_value_lists() + { + var fromFreshBuild = + ReusedInstances.BuildPineListValueReusedInstances( + ReusedInstances.ExpressionsSource()); + + var file = + ReusedInstances.BuildPrecompiledDictFile(fromFreshBuild); + + var parsedFile = + ReusedInstances.LoadFromPrebuiltJson(file); + + AssertPineValueListDictsAreEquivalent( + parsedFile.PineValueLists, + fromFreshBuild.PineValueLists); + + AssertPineValueListDictsAreEquivalent( + ReusedInstances.Instance.ListValues, + fromFreshBuild.PineValueLists); + } + + public static void AssertPineValueListDictsAreEquivalent( + IReadOnlyDictionary a, + IReadOnlyDictionary b) + { + if (a.Count != b.Count) + { + Assert.Fail("Counts are not equal: " + a.Count + " vs " + b.Count); + } + + foreach (var kv in a) + { + Assert.IsTrue(b.ContainsKey(kv.Key), "contains key"); + } + } +} \ No newline at end of file diff --git a/implement/prebuild/Program.cs b/implement/prebuild/Program.cs new file mode 100644 index 000000000..201040efc --- /dev/null +++ b/implement/prebuild/Program.cs @@ -0,0 +1,38 @@ +using Pine.Core; + +namespace prebuild; + +public class Program +{ + public const string DestinationFilePath = "./Pine.Core/" + ReusedInstances.EmbeddedResourceFilePath; + + public static void Main() + { + System.Console.WriteLine( + "Current working directory: " + System.Environment.CurrentDirectory); + + var fromFreshBuild = + ReusedInstances.BuildPineListValueReusedInstances( + ReusedInstances.ExpressionsSource()); + + var file = + ReusedInstances.BuildPrecompiledDictFile(fromFreshBuild); + + var absolutePath = System.IO.Path.GetFullPath(DestinationFilePath); + + System.Console.WriteLine( + "Resolved the destination path of " + DestinationFilePath + + " to " + absolutePath); + + System.IO.Directory.CreateDirectory( + System.IO.Path.GetDirectoryName(absolutePath)); + + System.IO.File.WriteAllBytes( + absolutePath, + file.ToArray()); + + System.Console.WriteLine( + "Saved the prebuilt dictionary with " + + fromFreshBuild.PineValueLists.Count + " list values to " + absolutePath); + } +} \ No newline at end of file diff --git a/implement/prebuild/README.md b/implement/prebuild/README.md new file mode 100644 index 000000000..1716c7e82 --- /dev/null +++ b/implement/prebuild/README.md @@ -0,0 +1,3 @@ +# prebuild + +The `prebuild.csproj` program builds artifacts to be embedded into the Pine assembly on build, and therefore, need to be built before building `Pine.Core.csproj` diff --git a/implement/prebuild/prebuild.csproj b/implement/prebuild/prebuild.csproj new file mode 100644 index 000000000..b7642c5ba --- /dev/null +++ b/implement/prebuild/prebuild.csproj @@ -0,0 +1,13 @@ + + + + Exe + net8.0 + enable + + + + + + +