Feature/Era1 import/export (#6547)

Co-authored-by: Ruben Buniatyan <[email protected]> Co-authored-by: Amirul Ashraf <[email protected]> Co-authored-by: Lukasz Rozmej <[email protected]> Co-authored-by: Alexey Osipov <[email protected]>
NethermindEth · Jan 20, 2025 · f9960a8 · f9960a8
1 parent 50a6218
commit f9960a8
Show file tree

Hide file tree

Showing 84 changed files with 3,424 additions and 58 deletions.
diff --git a/src/Nethermind/Directory.Packages.props b/src/Nethermind/Directory.Packages.props
@@ -13,6 +13,7 @@
     <PackageVersion Include="BouncyCastle.Cryptography" Version="2.5.0" />
     <PackageVersion Include="Ckzg.Bindings" Version="2.0.1.1258" />
     <PackageVersion Include="Colorful.Console" Version="1.2.15" />
+    <PackageVersion Include="CommunityToolkit.HighPerformance" Version="8.3.2" />
     <PackageVersion Include="ConcurrentHashSet" Version="1.3.0" />
     <PackageVersion Include="coverlet.collector" Version="6.0.3" />
     <PackageVersion Include="Crc32.NET" Version="1.2.0" />

diff --git a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs
@@ -17,6 +17,7 @@
 using Nethermind.Consensus.Scheduler;
 using Nethermind.Consensus.Validators;
 using Nethermind.Core;
+using Nethermind.Era1;
 using Nethermind.Evm.TransactionProcessing;
 using Nethermind.Facade;
 using Nethermind.Facade.Eth;
@@ -97,6 +98,8 @@ public interface IApiWithBlockchain : IApiWithStores, IBlockchainBridgeFactory
         BackgroundTaskScheduler BackgroundTaskScheduler { get; set; }
         CensorshipDetector CensorshipDetector { get; set; }
 
+        IAdminEraService AdminEraService { get; set; }
+
         public ContainerBuilder ConfigureContainerBuilderFromApiWithBlockchain(ContainerBuilder builder)
         {
             return ConfigureContainerBuilderFromApiWithStores(builder)

diff --git a/src/Nethermind/Nethermind.Api/IBasicApi.cs b/src/Nethermind/Nethermind.Api/IBasicApi.cs
@@ -15,6 +15,7 @@
 using Nethermind.Core.Timers;
 using Nethermind.Crypto;
 using Nethermind.Db;
+using Nethermind.Era1;
 using Nethermind.KeyStore;
 using Nethermind.Logging;
 using Nethermind.Serialization.Json;

diff --git a/src/Nethermind/Nethermind.Api/INethermindApi.cs b/src/Nethermind/Nethermind.Api/INethermindApi.cs
@@ -4,6 +4,7 @@
 #nullable enable
 using System;
 using Nethermind.Config;
+using Nethermind.Era1;
 using Nethermind.Facade.Eth;
 using Nethermind.Facade.Eth.RpcTransaction;
 using Nethermind.Serialization.Rlp;

diff --git a/src/Nethermind/Nethermind.Api/NethermindApi.cs b/src/Nethermind/Nethermind.Api/NethermindApi.cs
@@ -57,6 +57,7 @@
 using Nethermind.Sockets;
 using Nethermind.Trie;
 using Nethermind.Consensus.Processing.CensorshipDetector;
+using Nethermind.Era1;
 using Nethermind.Facade.Find;
 using Nethermind.Synchronization.FastSync;
 
@@ -225,6 +226,7 @@ public ISealEngine SealEngine
         public INodeStorageFactory NodeStorageFactory { get; set; } = null!;
         public BackgroundTaskScheduler BackgroundTaskScheduler { get; set; } = null!;
         public CensorshipDetector CensorshipDetector { get; set; } = null!;
+        public IAdminEraService AdminEraService { get; set; } = null!;
         public IWallet? Wallet { get; set; }
         public IBadBlockStore? BadBlocksStore { get; set; }
         public ITransactionComparerProvider? TransactionComparerProvider { get; set; }

diff --git a/src/Nethermind/Nethermind.Blockchain/BlockTreeSuggestPacer.cs b/src/Nethermind/Nethermind.Blockchain/BlockTreeSuggestPacer.cs
@@ -19,7 +19,7 @@ public class BlockTreeSuggestPacer : IDisposable
     private readonly long _resumeBatchSize;
     private readonly IBlockTree _blockTree;
 
-    public BlockTreeSuggestPacer(IBlockTree blockTree, long stopBatchSize, long resumeBatchSize)
+    public BlockTreeSuggestPacer(IBlockTree blockTree, long stopBatchSize = 4096, long resumeBatchSize = 2048)
     {
         blockTree.NewHeadBlock += BlockTreeOnNewHeadBlock;
         _blockTree = blockTree;

diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs
@@ -10,7 +10,7 @@ namespace Nethermind.Blockchain.Receipts
     public interface IReceiptStorage : IReceiptFinder
     {
         void Insert(Block block, params TxReceipt[]? txReceipts) => Insert(block, txReceipts, true);
-        void Insert(Block block, TxReceipt[]? txReceipts, bool ensureCanonical, WriteFlags writeFlags = WriteFlags.None);
+        void Insert(Block block, TxReceipt[]? txReceipts, bool ensureCanonical, WriteFlags writeFlags = WriteFlags.None, long? lastBlockNumber = null);
         long MigratedBlockNumber { get; set; }
         bool HasBlock(long blockNumber, Hash256 hash);
         void EnsureCanonical(Block block);

diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs
@@ -62,7 +62,7 @@ public bool TryGetReceiptsIterator(long blockNumber, Hash256 blockHash, out Rece
             }
         }
 
-        public void Insert(Block block, TxReceipt[] txReceipts, bool ensureCanonical = true, WriteFlags writeFlags = WriteFlags.None)
+        public void Insert(Block block, TxReceipt[] txReceipts, bool ensureCanonical = true, WriteFlags writeFlags = WriteFlags.None, long? lastBlockNumber = null)
         {
             _receipts[block.Hash] = txReceipts;
             if (ensureCanonical)

diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs
@@ -21,7 +21,7 @@ private NullReceiptStorage()
         {
         }
 
-        public void Insert(Block block, TxReceipt[] txReceipts, bool ensureCanonical, WriteFlags writeFlags) { }
+        public void Insert(Block block, TxReceipt[] txReceipts, bool ensureCanonical, WriteFlags writeFlags, long? lastBlockNumber = null) { }
 
         public TxReceipt[] Get(Block block, bool recover = true, bool recoverSender = false) => [];
         public TxReceipt[] Get(Hash256 blockHash, bool recover = true) => [];

diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs
@@ -256,7 +256,7 @@ public bool TryGetReceiptsIterator(long blockNumber, Hash256 blockHash, out Rece
         }
 
         [SkipLocalsInit]
-        public void Insert(Block block, TxReceipt[]? txReceipts, bool ensureCanonical = true, WriteFlags writeFlags = WriteFlags.None)
+        public void Insert(Block block, TxReceipt[]? txReceipts, bool ensureCanonical = true, WriteFlags writeFlags = WriteFlags.None, long? lastBlockNumber = null)
         {
             txReceipts ??= [];
             int txReceiptsLength = txReceipts.Length;
@@ -291,7 +291,7 @@ public void Insert(Block block, TxReceipt[]? txReceipts, bool ensureCanonical =
 
             if (ensureCanonical)
             {
-                EnsureCanonical(block);
+                EnsureCanonical(block, lastBlockNumber);
             }
         }
 
@@ -331,13 +331,18 @@ public bool HasBlock(long blockNumber, Hash256 blockHash)
         }
 
         public void EnsureCanonical(Block block)
+        {
+            EnsureCanonical(block, null);
+        }
+
+        private void EnsureCanonical(Block block, long? lastBlockNumber)
         {
             using IWriteBatch writeBatch = _transactionDb.StartWriteBatch();
 
-            long headNumber = _blockTree.FindBestSuggestedHeader()?.Number ?? 0;
+            lastBlockNumber ??= _blockTree.FindBestSuggestedHeader()?.Number ?? 0;
 
             if (_receiptConfig.TxLookupLimit == -1) return;
-            if (_receiptConfig.TxLookupLimit != 0 && block.Number <= headNumber - _receiptConfig.TxLookupLimit) return;
+            if (_receiptConfig.TxLookupLimit != 0 && block.Number <= lastBlockNumber - _receiptConfig.TxLookupLimit) return;
             if (_receiptConfig.CompactTxIndex)
             {
                 byte[] blockNumber = Rlp.Encode(block.Number).Bytes;

diff --git a/src/Nethermind/Nethermind.Core.Test/Builders/BlockBuilder.cs b/src/Nethermind/Nethermind.Core.Test/Builders/BlockBuilder.cs
@@ -3,6 +3,7 @@
 
 using System;
 using System.Linq;
+using Nethermind.Blockchain;
 using Nethermind.Core.Crypto;
 using Nethermind.Core.ExecutionRequest;
 using Nethermind.Core.Specs;
@@ -193,6 +194,7 @@ public BlockBuilder WithUncles(params BlockHeader[] uncles)
         {
             TestObjectInternal = TestObjectInternal.WithReplacedBody(
                 TestObjectInternal.Body.WithChangedUncles(uncles));
+            TestObjectInternal.Header.UnclesHash = UnclesHash.Calculate(uncles);
             return this;
         }
 

diff --git a/src/Nethermind/Nethermind.Core.Test/Builders/TestItem.cs b/src/Nethermind/Nethermind.Core.Test/Builders/TestItem.cs
@@ -187,5 +187,12 @@ public static byte[] GenerateIndexedAccountRlp(int index, AccountDecoder? accoun
             byte[] value = accountDecoder.Encode(account).Bytes;
             return value;
         }
+
+        public static UInt256 GetRandomAmount(Random? random = null)
+        {
+            Span<byte> buffer = stackalloc byte[32];
+            (random ?? Random).NextBytes(buffer);
+            return new UInt256(buffer);
+        }
     }
 }
diff --git a/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs b/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs
@@ -129,10 +129,10 @@ public static ContainerBuilder AddSingleton<T, TImpl>(this ContainerBuilder buil
         return builder;
     }
 
-    public static ContainerBuilder AddKeyedSingleton<T>(this ContainerBuilder builder, string key, T instance) where T : class
+    public static ContainerBuilder AddKeyedSingleton<T>(this ContainerBuilder builder, object key, T instance) where T : class
     {
         builder.RegisterInstance(instance)
-            .Named<T>(key)
+            .Keyed<T>(key)
             .ExternallyOwned()
             .SingleInstance();
 
@@ -334,6 +334,13 @@ public static ContainerBuilder Map<TTo, TFrom>(this ContainerBuilder builder, Fu
         return builder;
     }
 
+    public static ContainerBuilder OnBuild(this ContainerBuilder builder, Action<ILifetimeScope> action)
+    {
+        builder.RegisterBuildCallback(action);
+
+        return builder;
+    }
+
     public static ContainerBuilder Bind<TTo, TFrom>(this ContainerBuilder builder) where TFrom : TTo where TTo : notnull
     {
         builder.Register(static (it) => it.Resolve<TFrom>())

diff --git a/src/Nethermind/Nethermind.Crypto/EthereumEcdsa.cs b/src/Nethermind/Nethermind.Crypto/EthereumEcdsa.cs
@@ -6,6 +6,7 @@
 using System.Numerics;
 using Nethermind.Core;
 using Nethermind.Core.Crypto;
+using Nethermind.Core.Specs;
 using Nethermind.Serialization.Rlp;
 
 namespace Nethermind.Crypto
@@ -33,6 +34,10 @@ public EthereumEcdsa(ulong chainId)
             _chainIdValue = chainId;
         }
 
+        public EthereumEcdsa(ISpecProvider specProvider) : this(specProvider.ChainId)
+        {
+        }
+
         public Address? RecoverAddress(Signature signature, Hash256 message)
         {
             return RecoverAddress(signature.BytesWithRecovery, message);

diff --git a/src/Nethermind/Nethermind.Era.Test/AccumulatorCalculatorTests.cs b/src/Nethermind/Nethermind.Era.Test/AccumulatorCalculatorTests.cs
@@ -0,0 +1,46 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Runtime.Intrinsics.Arm;
+using System.Text;
+using System.Threading.Tasks;
+using Nethermind.Core.Crypto;
+
+namespace Nethermind.Era1.Test;
+public class AccumulatorCalculatorTests
+{
+    [Test]
+    public void Add_AddOneHashAndUInt256_DoesNotThrow()
+    {
+        using var sut = new AccumulatorCalculator();
+
+        Assert.That(() => sut.Add(Keccak.Zero, 0), Throws.Nothing);
+    }
+    [Test]
+    public void ComputeRoot_AddValues_ReturnsExpectedResult()
+    {
+        using var sut = new AccumulatorCalculator();
+        sut.Add(Keccak.Zero, 1);
+        sut.Add(Keccak.MaxValue, 2);
+
+        var result = sut.ComputeRoot().ToByteArray();
+
+        Assert.That(result, Is.EquivalentTo(new[] { 0x3E, 0xD6, 0x26, 0x52, 0xDF, 0xB7, 0xE1, 0x07, 0x2D, 0x0F, 0x04, 0x0F, 0xEB, 0x6D, 0x00, 0x2A, 0x9F, 0x7C, 0xE3, 0x7C, 0xF8, 0xDD, 0xB1, 0x65, 0x49, 0xA7, 0xAC, 0x5C, 0xF8, 0xE3, 0xB7, 0x91 }));
+    }
+
+    [Test]
+    public void ComputeRoot_AddOneHashAndUInt256_DoesNotThrow()
+    {
+        using var sut = new AccumulatorCalculator();
+        sut.Add(Keccak.Zero, 1);
+        sut.Add(Keccak.MaxValue, 2);
+
+        var result = sut.ComputeRoot().ToByteArray();
+
+        Assert.That(result, Is.EquivalentTo(new[] { 0x3E, 0xD6, 0x26, 0x52, 0xDF, 0xB7, 0xE1, 0x07, 0x2D, 0x0F, 0x04, 0x0F, 0xEB, 0x6D, 0x00, 0x2A, 0x9F, 0x7C, 0xE3, 0x7C, 0xF8, 0xDD, 0xB1, 0x65, 0x49, 0xA7, 0xAC, 0x5C, 0xF8, 0xE3, 0xB7, 0x91 }));
+    }
+}
diff --git a/src/Nethermind/Nethermind.Era.Test/AdminEraServiceTests.cs b/src/Nethermind/Nethermind.Era.Test/AdminEraServiceTests.cs
@@ -0,0 +1,83 @@
+// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using Nethermind.Config;
+using Nethermind.Logging;
+using NSubstitute;
+
+namespace Nethermind.Era1.Test;
+
+public class AdminEraServiceTests
+{
+    [Test]
+    public void CanCallcExport()
+    {
+        IEraImporter importer = Substitute.For<IEraImporter>();
+        AdminEraService adminEraService = new AdminEraService(
+            importer,
+            Substitute.For<IEraExporter>(),
+            Substitute.For<IProcessExitSource>(),
+            LimboLogs.Instance);
+
+        adminEraService.ImportHistory("somewhere", 99, 999, null);
+        importer.Received().Import("somewhere", 99, 999, null);
+    }
+
+    [Test]
+    public void ThrowsWhenExistingImportIsRunning()
+    {
+        IEraImporter importer = Substitute.For<IEraImporter>();
+        TaskCompletionSource importTcs = new TaskCompletionSource();
+        importer.Import("somewhere", 99, 999, null).Returns(importTcs.Task);
+        AdminEraService adminEraService = new AdminEraService(
+            importer,
+            Substitute.For<IEraExporter>(),
+            Substitute.For<IProcessExitSource>(),
+            LimboLogs.Instance);
+
+        adminEraService.ImportHistory("somewhere", 99, 999, null);
+
+        Assert.That(() => adminEraService.ImportHistory("somewhere", 99, 999, null), Throws.Exception);
+
+        importTcs.TrySetResult();
+
+        // Not throw
+        adminEraService.ImportHistory("somewhere", 99, 999, null);
+    }
+
+    [Test]
+    public void CanCallExport()
+    {
+        IEraExporter exporter = Substitute.For<IEraExporter>();
+        AdminEraService adminEraService = new AdminEraService(
+            Substitute.For<IEraImporter>(),
+            exporter,
+            Substitute.For<IProcessExitSource>(),
+            LimboLogs.Instance);
+
+        adminEraService.ExportHistory("somewhere", 99, 999);
+        exporter.Received().Export("somewhere", 99, 999);
+    }
+
+    [Test]
+    public void ThrowsWhenExistingExportIsRunning()
+    {
+        IEraExporter exporter = Substitute.For<IEraExporter>();
+        TaskCompletionSource importTcs = new TaskCompletionSource();
+        exporter.Export("somewhere", 99, 999).Returns(importTcs.Task);
+        AdminEraService adminEraService = new AdminEraService(
+            Substitute.For<IEraImporter>(),
+            exporter,
+            Substitute.For<IProcessExitSource>(),
+            LimboLogs.Instance);
+
+        adminEraService.ExportHistory("somewhere", 99, 999);
+
+        Assert.That(() => adminEraService.ExportHistory("somewhere", 99, 999), Throws.Exception);
+
+        importTcs.TrySetResult();
+
+        // Not throw
+        adminEraService.ExportHistory("somewhere", 99, 999);
+    }
+}