diff --git a/.github/workflows/publish-to-release.yml b/.github/workflows/publish-to-release.yml
index 8b74066cb..a32cdc448 100644
--- a/.github/workflows/publish-to-release.yml
+++ b/.github/workflows/publish-to-release.yml
@@ -41,8 +41,13 @@ jobs:
- name: Clean package cache as a temporary workaround for https://github.com/actions/setup-dotnet/issues/155
run: dotnet clean ./implement/test-elm-time/test-elm-time.csproj && dotnet nuget locals all --clear
+ - name: Prebuild
+ working-directory: ./implement/
+ run: |
+ dotnet run --project ./prebuild/prebuild.csproj
+
- name: dotnet publish
- run: dotnet publish -c Debug -r ${{ matrix.publish-runtime-id }} --self-contained true -p:PublishSingleFile=true -p:PublishReadyToRun=true -p:PublishReadyToRunShowWarnings=true --output ./dotnet-build ./implement/pine
+ run: dotnet publish -c Release -r ${{ matrix.publish-runtime-id }} --self-contained true -p:PublishSingleFile=true -p:PublishReadyToRun=true -p:PublishReadyToRunShowWarnings=true --output ./dotnet-build ./implement/pine
- name: Copy artifacts to publish
shell: pwsh
diff --git a/.github/workflows/test-and-publish.yml b/.github/workflows/test-and-publish.yml
index 3c8f088fd..52773e44b 100644
--- a/.github/workflows/test-and-publish.yml
+++ b/.github/workflows/test-and-publish.yml
@@ -37,6 +37,11 @@ jobs:
dotnet clean ./implement/PineTest/Pine.UnitTests/Pine.UnitTests.csproj && dotnet nuget locals all --clear
dotnet clean ./implement/test-elm-time/test-elm-time.csproj && dotnet nuget locals all --clear
+ - name: Prebuild
+ working-directory: ./implement/
+ run: |
+ dotnet run --project ./prebuild/prebuild.csproj
+
- name: Run Pine tests with dotnet test
run: |
dotnet test ./implement/PineTest/Pine.UnitTests/Pine.UnitTests.csproj --logger "trx" --diag "./implement/PineTest/Pine.UnitTests/TestResults/diag-log.txt" --verbosity diagnostic
diff --git a/implement/Pine.Core/.gitignore b/implement/Pine.Core/.gitignore
new file mode 100644
index 000000000..9388e4835
--- /dev/null
+++ b/implement/Pine.Core/.gitignore
@@ -0,0 +1 @@
+prebuilt-artifact/
diff --git a/implement/Pine.Core/Pine.Core.csproj b/implement/Pine.Core/Pine.Core.csproj
index ce0779a19..eee8a373f 100644
--- a/implement/Pine.Core/Pine.Core.csproj
+++ b/implement/Pine.Core/Pine.Core.csproj
@@ -36,6 +36,10 @@
+
+
+
+
diff --git a/implement/Pine.Core/PineValue.cs b/implement/Pine.Core/PineValue.cs
index 26e0a32ac..b640db6d3 100644
--- a/implement/Pine.Core/PineValue.cs
+++ b/implement/Pine.Core/PineValue.cs
@@ -199,10 +199,24 @@ public readonly record struct ListValueStruct
/// Construct a list value from a sequence of other values.
///
public ListValueStruct(IReadOnlyList elements)
+ :
+ this(elements, ComputeSlimHashCode(elements))
+ {
+ }
+
+ public ListValueStruct(ListValue instance)
+ :
+ this(instance.Elements, instance.slimHashCode)
+ {
+ }
+
+ private ListValueStruct(
+ IReadOnlyList elements,
+ int slimHashCode)
{
Elements = elements;
- slimHashCode = ComputeSlimHashCode(elements);
+ this.slimHashCode = slimHashCode;
}
public bool Equals(ListValueStruct other)
diff --git a/implement/Pine.Core/ReusedInstances.cs b/implement/Pine.Core/ReusedInstances.cs
index fbfe192ae..228fea051 100644
--- a/implement/Pine.Core/ReusedInstances.cs
+++ b/implement/Pine.Core/ReusedInstances.cs
@@ -7,10 +7,10 @@
namespace Pine.Core;
public record ReusedInstances(
- IEnumerable expressionRootsSource)
+ System.Func> LoadExpressionRootsSource)
{
public static readonly ReusedInstances Instance =
- new(expressionRootsSource: ExpressionsSource());
+ new(LoadExpressionRootsSource: ExpressionsSource);
public FrozenDictionary? ListValues { private set; get; }
@@ -44,7 +44,7 @@ static ReusedInstances()
Instance.Build();
}
- static IEnumerable ExpressionsSource()
+ public static IEnumerable ExpressionsSource()
{
foreach (var namedExpression in PopularExpression.BuildPopularExpressionDictionary())
{
@@ -52,7 +52,250 @@ static IEnumerable ExpressionsSource()
}
}
- public void Build()
+
+ public record PrebuiltListEntry(
+ string Key,
+ PrebuiltListEntryValue Value);
+
+ public record PrebuiltListEntryValue(
+ string? BlobBytesBase64,
+ IReadOnlyList? ListItemsKeys);
+
+ const string expectedInCompilerKey = "expected-in-compiler-container";
+
+ public static System.ReadOnlyMemory BuildPrecompiledDictFile(
+ PineListValueReusedInstances source)
+ {
+ var entriesList = PrebuildListEntries(source);
+
+ return System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(entriesList);
+ }
+
+ public static IReadOnlyList PrebuildListEntries(
+ PineListValueReusedInstances source)
+ {
+ var (_, allBlobs) =
+ CollectAllComponentsFromRoots(
+ [.. source.PineValueLists.Values
+ ,..source.ValuesExpectedInCompilerLists
+ ,..source.ValuesExpectedInCompilerBlobs]);
+
+ var mutatedBlobsDict = new Dictionary();
+
+ var blobEntriesList =
+ allBlobs
+ .OrderBy(blob => blob.Bytes.Length)
+ .Select((blobValue, blobIndex) =>
+ {
+ var entryKey = "blob-" + blobIndex.ToString();
+
+ mutatedBlobsDict[blobValue] = entryKey;
+
+ return
+ new PrebuiltListEntry(
+ Key: entryKey,
+ new PrebuiltListEntryValue(
+ BlobBytesBase64: System.Convert.ToBase64String(blobValue.Bytes.Span),
+ ListItemsKeys: null));
+ })
+ .ToList();
+
+ var listsOrdered =
+ source.PineValueLists.Values
+ .Concat(source.ValuesExpectedInCompilerLists)
+ .Distinct()
+ .OrderBy(l => l.NodesCount)
+ .ToList();
+
+ var mutatedListDict = new Dictionary();
+
+ string itemId(PineValue itemValue)
+ {
+ if (itemValue is PineValue.BlobValue itemBlob)
+ return mutatedBlobsDict[itemBlob];
+
+ if (itemValue is PineValue.ListValue itemList)
+ return mutatedListDict[itemList];
+
+ throw new System.NotImplementedException(
+ "Unexpected item value type: " + itemValue.GetType());
+ }
+
+ PrebuiltListEntryValue entryListFromItems(
+ IReadOnlyList itemValues)
+ {
+ var itemsIds = itemValues.Select(itemId).ToArray();
+
+ return new PrebuiltListEntryValue(
+ BlobBytesBase64: null,
+ ListItemsKeys: itemsIds);
+ }
+
+ var listEntriesList =
+ listsOrdered
+ .Select((listInstance, index) =>
+ {
+ var entryKey = "list-" + index.ToString();
+
+ mutatedListDict[listInstance] = entryKey;
+
+ return
+ new PrebuiltListEntry(
+ Key: entryKey,
+ Value: entryListFromItems(listInstance.Elements));
+ })
+ .ToList();
+
+ var ventry =
+ new PrebuiltListEntry(
+ Key: expectedInCompilerKey,
+ Value: entryListFromItems(
+ [.. source.ValuesExpectedInCompilerBlobs.Cast().Concat(source.ValuesExpectedInCompilerLists)]));
+
+ return
+ [..blobEntriesList
+ ,..listEntriesList
+ ,ventry];
+ }
+
+ public const string EmbeddedResourceFilePath = "prebuilt-artifact/reused-pine-values.json";
+
+ public static PineListValueReusedInstances LoadPrecompiledFromEmbeddedOrDefault(
+ System.Reflection.Assembly assembly,
+ System.Func> loadExpressionRootsSource) =>
+ LoadEmbeddedPrebuilt(assembly)
+ .Extract(err =>
+ {
+ System.Console.WriteLine("Failed loading from embedded resource: " + err);
+
+ return BuildPineListValueReusedInstances(loadExpressionRootsSource());
+ });
+
+ public static Result LoadEmbeddedPrebuilt(
+ System.Reflection.Assembly assembly,
+ string embeddedResourceFilePath = EmbeddedResourceFilePath)
+ {
+ /*
+ var inspect =
+ DotNetAssembly.LoadDirectoryFilesFromManifestEmbeddedFileProviderAsDictionary(
+ directoryPath: ["prebuilt-artifact"],
+ assembly: assembly);
+ */
+
+ var manifestEmbeddedProvider =
+ new Microsoft.Extensions.FileProviders.ManifestEmbeddedFileProvider(assembly);
+
+ var embeddedFileInfo = manifestEmbeddedProvider.GetFileInfo(embeddedResourceFilePath);
+
+ if (!embeddedFileInfo.Exists)
+ {
+ return "Did not find file " + embeddedResourceFilePath + " in assembly " + assembly.FullName;
+ }
+
+ if (embeddedFileInfo.Length is 0)
+ {
+ return "File " + embeddedResourceFilePath + " in assembly " + assembly.FullName + " is empty";
+ }
+
+ using var readStream = embeddedFileInfo.CreateReadStream();
+
+ using var memoryStream = new System.IO.MemoryStream();
+
+ readStream.CopyTo(memoryStream);
+
+ return LoadFromPrebuiltJson(memoryStream.ToArray());
+ }
+
+
+ public static PineListValueReusedInstances LoadFromPrebuiltJson(System.ReadOnlyMemory json)
+ {
+ var parsed =
+ System.Text.Json.JsonSerializer.Deserialize>(json.Span);
+
+ return LoadFromPrebuilt(parsed);
+ }
+
+ public static PineListValueReusedInstances LoadFromPrebuilt(
+ IReadOnlyList entries)
+ {
+ var mutatedDict = new Dictionary();
+
+ PineValue contructValue(
+ PrebuiltListEntryValue entryValue)
+ {
+ if (entryValue.BlobBytesBase64 is { } bytesBase64)
+ {
+ var bytes = System.Convert.FromBase64String(bytesBase64);
+
+ return PineValue.Blob(bytes);
+ }
+
+ if (entryValue.ListItemsKeys is { } listItemsKeys)
+ {
+ var items = new PineValue[listItemsKeys.Count];
+
+ for (int i = 0; i < items.Length; ++i)
+ {
+ items[i] = mutatedDict[listItemsKeys[i]];
+ }
+
+ return PineValue.List(items);
+ }
+
+ throw new System.NotImplementedException(
+ "Unexpected entry type: " + entryValue.GetType());
+ }
+
+ for (var i = 0; i < entries.Count; ++i)
+ {
+ var entry = entries[i];
+
+ mutatedDict[entry.Key] = contructValue(entry.Value);
+ }
+
+ var valuesExpectedInCompilerContainer = mutatedDict[expectedInCompilerKey];
+
+ if (valuesExpectedInCompilerContainer is not PineValue.ListValue valuesExpectedInCompilerList)
+ {
+ throw new System.Exception(
+ "Did not find container with key " + expectedInCompilerKey);
+ }
+
+ var listValuesExpectedInCompiler =
+ valuesExpectedInCompilerList.Elements.OfType()
+ .ToHashSet();
+
+ var blobValuesExpectedInCompiler =
+ valuesExpectedInCompilerList.Elements.OfType()
+ .ToHashSet();
+
+ var valueListsDict = new Dictionary();
+
+ foreach (var item in mutatedDict)
+ {
+ if (item.Key is expectedInCompilerKey)
+ continue;
+
+ if (item.Value is not PineValue.ListValue listValue)
+ continue;
+
+ valueListsDict[new PineValue.ListValue.ListValueStruct(listValue)] = listValue;
+ }
+
+ return new PineListValueReusedInstances(
+ listValuesExpectedInCompiler,
+ blobValuesExpectedInCompiler,
+ valueListsDict);
+ }
+
+
+ public record PineListValueReusedInstances(
+ IReadOnlySet ValuesExpectedInCompilerLists,
+ IReadOnlySet ValuesExpectedInCompilerBlobs,
+ IReadOnlyDictionary PineValueLists);
+
+ public static PineListValueReusedInstances BuildPineListValueReusedInstances(
+ IEnumerable expressionRootsSource)
{
var valueRootsFromProgramsSorted =
expressionRootsSource
@@ -64,12 +307,7 @@ public void Build()
var (valuesExpectedInCompilerLists, valuesExpectedInCompilerBlobs) =
CollectAllComponentsFromRoots(valueRootsFromProgramsSorted);
- var valuesExpectedInCompilerSorted =
- PineValue.ReusedBlobs
- .Cast()
- .Concat(valuesExpectedInCompilerBlobs)
- .Concat(valuesExpectedInCompilerLists.OrderBy(listValue => listValue.NodesCount))
- .ToList();
+ IReadOnlyDictionary PineValueLists;
{
var tempEncodingDict = new Dictionary();
@@ -139,13 +377,35 @@ public void Build()
reusedListsDictInConstruction[rebuilt] = rebuilt;
}
- ListValues =
+ PineValueLists =
reusedListsDictInConstruction
.ToFrozenDictionary(
keySelector: asRef => new PineValue.ListValue.ListValueStruct(asRef.Key.Elements),
elementSelector: asRef => asRef.Value);
}
+ return new PineListValueReusedInstances(
+ valuesExpectedInCompilerLists,
+ valuesExpectedInCompilerBlobs,
+ PineValueLists);
+ }
+
+ public void Build()
+ {
+ var loadedPineListValues =
+ LoadPrecompiledFromEmbeddedOrDefault(
+ typeof(ReusedInstances).Assembly,
+ ExpressionsSource);
+
+ ListValues = loadedPineListValues.PineValueLists.ToFrozenDictionary();
+
+ var valuesExpectedInCompilerSorted =
+ PineValue.ReusedBlobs
+ .Cast()
+ .Concat(loadedPineListValues.ValuesExpectedInCompilerBlobs)
+ .Concat(loadedPineListValues.ValuesExpectedInCompilerLists.OrderBy(listValue => listValue.NodesCount))
+ .ToList();
+
{
/*
* For the expression instances, we rebuild the collection again, because the literal expressions should
@@ -153,7 +413,7 @@ public void Build()
* */
var expressionsRoots =
- expressionRootsSource.ToList();
+ LoadExpressionRootsSource().ToList();
var allExpressionDescendants =
Expression.CollectAllComponentsFromRoots(expressionsRoots);
diff --git a/implement/PineTest/Pine.UnitTests/ReusedInstancesTests.cs b/implement/PineTest/Pine.UnitTests/ReusedInstancesTests.cs
index 7a0d18892..9fbea362a 100644
--- a/implement/PineTest/Pine.UnitTests/ReusedInstancesTests.cs
+++ b/implement/PineTest/Pine.UnitTests/ReusedInstancesTests.cs
@@ -1,5 +1,6 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Pine.Core;
+using System.Collections.Generic;
namespace Pine.UnitTests;
@@ -11,4 +12,41 @@ public void Ensure_reference_equality_between_mappings_between_reused_instances(
{
ReusedInstances.Instance.AssertReferenceEquality();
}
-}
+
+ [TestMethod]
+ public void Embedded_precompiled_pine_value_lists()
+ {
+ var fromFreshBuild =
+ ReusedInstances.BuildPineListValueReusedInstances(
+ ReusedInstances.ExpressionsSource());
+
+ var file =
+ ReusedInstances.BuildPrecompiledDictFile(fromFreshBuild);
+
+ var parsedFile =
+ ReusedInstances.LoadFromPrebuiltJson(file);
+
+ AssertPineValueListDictsAreEquivalent(
+ parsedFile.PineValueLists,
+ fromFreshBuild.PineValueLists);
+
+ AssertPineValueListDictsAreEquivalent(
+ ReusedInstances.Instance.ListValues,
+ fromFreshBuild.PineValueLists);
+ }
+
+ public static void AssertPineValueListDictsAreEquivalent(
+ IReadOnlyDictionary a,
+ IReadOnlyDictionary b)
+ {
+ if (a.Count != b.Count)
+ {
+ Assert.Fail("Counts are not equal: " + a.Count + " vs " + b.Count);
+ }
+
+ foreach (var kv in a)
+ {
+ Assert.IsTrue(b.ContainsKey(kv.Key), "contains key");
+ }
+ }
+}
\ No newline at end of file
diff --git a/implement/prebuild/Program.cs b/implement/prebuild/Program.cs
new file mode 100644
index 000000000..201040efc
--- /dev/null
+++ b/implement/prebuild/Program.cs
@@ -0,0 +1,38 @@
+using Pine.Core;
+
+namespace prebuild;
+
+public class Program
+{
+ public const string DestinationFilePath = "./Pine.Core/" + ReusedInstances.EmbeddedResourceFilePath;
+
+ public static void Main()
+ {
+ System.Console.WriteLine(
+ "Current working directory: " + System.Environment.CurrentDirectory);
+
+ var fromFreshBuild =
+ ReusedInstances.BuildPineListValueReusedInstances(
+ ReusedInstances.ExpressionsSource());
+
+ var file =
+ ReusedInstances.BuildPrecompiledDictFile(fromFreshBuild);
+
+ var absolutePath = System.IO.Path.GetFullPath(DestinationFilePath);
+
+ System.Console.WriteLine(
+ "Resolved the destination path of " + DestinationFilePath +
+ " to " + absolutePath);
+
+ System.IO.Directory.CreateDirectory(
+ System.IO.Path.GetDirectoryName(absolutePath));
+
+ System.IO.File.WriteAllBytes(
+ absolutePath,
+ file.ToArray());
+
+ System.Console.WriteLine(
+ "Saved the prebuilt dictionary with " +
+ fromFreshBuild.PineValueLists.Count + " list values to " + absolutePath);
+ }
+}
\ No newline at end of file
diff --git a/implement/prebuild/README.md b/implement/prebuild/README.md
new file mode 100644
index 000000000..1716c7e82
--- /dev/null
+++ b/implement/prebuild/README.md
@@ -0,0 +1,3 @@
+# prebuild
+
+The `prebuild.csproj` program builds artifacts to be embedded into the Pine assembly on build, and therefore, need to be built before building `Pine.Core.csproj`
diff --git a/implement/prebuild/prebuild.csproj b/implement/prebuild/prebuild.csproj
new file mode 100644
index 000000000..b7642c5ba
--- /dev/null
+++ b/implement/prebuild/prebuild.csproj
@@ -0,0 +1,13 @@
+
+
+
+ Exe
+ net8.0
+ enable
+
+
+
+
+
+
+