From fc912d11dad60a1784c1d381d862461ff411bd4d Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 30 Jan 2025 17:06:29 +0800 Subject: [PATCH 01/10] So close --- .../Nethermind.Config/ConfigProvider.cs | 25 +++ .../BlockDownloaderTests.Merge.cs | 157 ++++++++++------ .../BlockDownloaderTests.cs | 170 ++++++++++++------ .../Blocks/SyncBatchSize.cs | 6 +- 4 files changed, 248 insertions(+), 110 deletions(-) diff --git a/src/Nethermind/Nethermind.Config/ConfigProvider.cs b/src/Nethermind/Nethermind.Config/ConfigProvider.cs index 7faa007bfba..e65abe00c84 100644 --- a/src/Nethermind/Nethermind.Config/ConfigProvider.cs +++ b/src/Nethermind/Nethermind.Config/ConfigProvider.cs @@ -20,6 +20,30 @@ public class ConfigProvider : IConfigProvider private readonly Dictionary _implementations = []; + public ConfigProvider() + { + } + + public ConfigProvider(params IConfig[] existingConfigs) + { + foreach (IConfig existingConfig in existingConfigs) + { + Type type = existingConfig.GetType(); + if (!type.IsInterface) + { + // Try to get the interface type of the config + foreach (Type @interface in type.GetInterfaces()) + { + if (@interface.Name == $"I{type.Name}") + { + type = @interface; + } + } + } + _instances[type] = existingConfig; + } + } + public T GetConfig() where T : IConfig { return (T)GetConfig(typeof(T)); @@ -70,6 +94,7 @@ public void Initialize() foreach (Type @interface in interfaces) { Type directImplementation = @interface.GetDirectInterfaceImplementation(); + if (_instances.ContainsKey(@interface)) continue; if (directImplementation is not null) { diff --git a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs index 886cf38112c..bcfbae85593 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs @@ -5,29 +5,27 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Autofac; using FluentAssertions; using Nethermind.Blockchain; using Nethermind.Blockchain.Synchronization; +using Nethermind.Config; using Nethermind.Consensus; using Nethermind.Core; using Nethermind.Core.Crypto; -using Nethermind.Core.Specs; using Nethermind.Core.Test.Builders; +using Nethermind.Core.Test.Modules; using Nethermind.Db; using Nethermind.Int256; -using Nethermind.Logging; using Nethermind.Merge.Plugin; using Nethermind.Merge.Plugin.Synchronization; using Nethermind.Merge.Plugin.Test; -using Nethermind.Specs; -using Nethermind.Specs.ChainSpecStyle; using Nethermind.Stats; using Nethermind.Stats.Model; using Nethermind.Synchronization.Blocks; -using Nethermind.Synchronization.ParallelSync; using Nethermind.Synchronization.Peers; using Nethermind.Synchronization.Peers.AllocationStrategies; -using Nethermind.Synchronization.Reporting; +using Nethermind.TxPool; using NSubstitute; using NSubstitute.ClearExtensions; using NUnit.Framework; @@ -51,14 +49,15 @@ public async Task Merge_Happy_path(long pivot, long headNumber, int options, int .InsertBeaconHeaders(4, pivot - 1) .InsertBeaconBlocks(pivot + 1, insertedBeaconBlocks, BlockTreeTests.BlockTreeTestScenario.ScenarioBuilder.TotalDifficultyMode.Null); BlockTree syncedTree = blockTrees.SyncedTree; - PostMergeContext ctx = new() + + await using IContainer container = CreateMergeNode(blockTrees, new MergeConfig() { - BlockTreeScenario = blockTrees, - }; + TerminalTotalDifficulty = "0" + }); + PostMergeContext ctx = container.Resolve(); DownloaderOptions downloaderOptions = (DownloaderOptions)options; bool withReceipts = downloaderOptions == DownloaderOptions.WithReceipts; - ctx.MergeConfig = new MergeConfig { TerminalTotalDifficulty = "0" }; ctx.BeaconPivot.EnsurePivot(blockTrees.SyncedTree.FindHeader(pivot, BlockTreeLookupOptions.None)); ctx.BeaconPivot.ProcessDestination = blockTrees.SyncedTree.FindHeader(pivot, BlockTreeLookupOptions.None); @@ -101,13 +100,13 @@ public async Task Can_reach_terminal_block(long headNumber, int options, int thr .InsertBeaconHeaders(4, 15) .InsertBeaconBlocks(17, headNumber, BlockTreeTests.BlockTreeTestScenario.ScenarioBuilder.TotalDifficultyMode.Null); BlockTree syncedTree = blockTrees.SyncedTree; - PostMergeContext ctx = new() + await using IContainer container = CreateMergeNode(blockTrees, new MergeConfig() { - BlockTreeScenario = blockTrees, - }; + TerminalTotalDifficulty = $"{ttd}" + }); + PostMergeContext ctx = container.Resolve(); DownloaderOptions downloaderOptions = (DownloaderOptions)options; - ctx.MergeConfig = new MergeConfig { TerminalTotalDifficulty = $"{ttd}" }; if (withBeaconPivot) ctx.BeaconPivot.EnsurePivot(blockTrees.SyncedTree.FindHeader(16, BlockTreeLookupOptions.None)); @@ -138,13 +137,13 @@ public async Task IfNoBeaconPivot_thenStopAtPoS(long headNumber, int options, in BlockTree notSyncedTree = blockTrees.NotSyncedTree; BlockTree syncedTree = blockTrees.SyncedTree; - PostMergeContext ctx = new() + await using IContainer container = CreateMergeNode(blockTrees, new MergeConfig() { - BlockTreeScenario = blockTrees, - }; + TerminalTotalDifficulty = $"{ttd}" + }); + PostMergeContext ctx = container.Resolve(); DownloaderOptions downloaderOptions = (DownloaderOptions)options; - ctx.MergeConfig = new MergeConfig { TerminalTotalDifficulty = $"{ttd}" }; if (withBeaconPivot) ctx.BeaconPivot.EnsurePivot(blockTrees.SyncedTree.FindHeader(16, BlockTreeLookupOptions.None)); @@ -167,10 +166,8 @@ public async Task WillSkipBlocksToIgnore(long pivot, long headNumber, int blocks .InsertBeaconHeaders(4, pivot - 1); BlockTree syncedTree = blockTrees.SyncedTree; - PostMergeContext ctx = new() - { - BlockTreeScenario = blockTrees, - }; + await using IContainer container = CreateMergeNode(blockTrees); + PostMergeContext ctx = container.Resolve(); ctx.BeaconPivot.EnsurePivot(blockTrees.SyncedTree.FindHeader(pivot, BlockTreeLookupOptions.None)); ctx.BeaconPivot.ProcessDestination = blockTrees.SyncedTree.FindHeader(pivot, BlockTreeLookupOptions.None); @@ -202,18 +199,6 @@ public async Task Recalculate_header_total_difficulty() .InsertOtherChainToMain(notSyncedTree, 1, 3) // Need to have the header inserted to LRU which mean we need to move the head forward .InsertBeaconHeaders(1, 3, tdMode: BlockTreeTests.BlockTreeTestScenario.ScenarioBuilder.TotalDifficultyMode.Null); - PostMergeContext ctx = new() - { - BlockTreeScenario = blockTrees, - MergeConfig = new MergeConfig { TerminalTotalDifficulty = $"{ttd}" }, - }; - - BlockHeader lastHeader = syncedTree.FindHeader(3, BlockTreeLookupOptions.None)!; - // Because the FindHeader recalculated the TD. - lastHeader.TotalDifficulty = 0; - - ctx.BeaconPivot.EnsurePivot(lastHeader); - ISealValidator sealValidator = Substitute.For(); sealValidator.ValidateSeal(Arg.Any(), Arg.Any()).Returns((info => { @@ -222,7 +207,24 @@ public async Task Recalculate_header_total_difficulty() notSyncedTree.FindHeader(header.Hash, BlockTreeLookupOptions.DoNotCreateLevelIfMissing); return true; })); - ctx.SealValidator = sealValidator; + + await using IContainer container = CreateMergeNode((builder) => + { + builder + .AddSingleton(notSyncedTree) + .AddKeyedSingleton(DbNames.Metadata, blockTrees.NotSyncedTreeBuilder.MetadataDb) + .AddSingleton(sealValidator); + }, new MergeConfig() + { + TerminalTotalDifficulty = $"{ttd}" + }); + PostMergeContext ctx = container.Resolve(); + + BlockHeader lastHeader = syncedTree.FindHeader(3, BlockTreeLookupOptions.None)!; + // Because the FindHeader recalculated the TD. + lastHeader.TotalDifficulty = 0; + + ctx.BeaconPivot.EnsurePivot(lastHeader); BlockDownloader downloader = ctx.BlockDownloader; @@ -249,12 +251,6 @@ public async Task Does_not_deadlock_on_replace_peer() .GoesLikeThis() .WithBlockTrees(0, 4) .InsertBeaconPivot(3); - PostMergeContext ctx = new() - { - MergeConfig = new MergeConfig { TerminalTotalDifficulty = "0" }, - BlockTreeScenario = blockTrees, - }; - ctx.BeaconPivot.EnsurePivot(blockTrees.SyncedTree.FindHeader(3, BlockTreeLookupOptions.None)); ManualResetEventSlim chainLevelHelperBlocker = new ManualResetEventSlim(false); IChainLevelHelper chainLevelHelper = Substitute.For(); @@ -264,7 +260,21 @@ public async Task Does_not_deadlock_on_replace_peer() { chainLevelHelperBlocker.Wait(); }); - ctx.ChainLevelHelper = chainLevelHelper; + + await using IContainer container = CreateMergeNode((builder) => + { + builder + .AddSingleton(chainLevelHelper) + + .AddSingleton(blockTrees.NotSyncedTree) + .AddKeyedSingleton(DbNames.Metadata, blockTrees.NotSyncedTreeBuilder.MetadataDb); + }, new MergeConfig() + { + TerminalTotalDifficulty = $"{0}" + }); + PostMergeContext ctx = container.Resolve(); + + ctx.BeaconPivot.EnsurePivot(blockTrees.SyncedTree.FindHeader(3, BlockTreeLookupOptions.None)); IPeerAllocationStrategy peerAllocationStrategy = Substitute.For(); @@ -327,17 +337,20 @@ public void No_old_bodies_and_receipts() .InsertBeaconPivot(64) .InsertBeaconHeaders(4, 128); BlockTree syncedTree = blockTrees.SyncedTree; - PostMergeContext ctx = new() - { - BlockTreeScenario = blockTrees, - }; - ctx.Feed = new FastSyncFeed(new TestSyncConfig + using IContainer container = CreateMergeNode((builder) => + { + builder + .AddSingleton(blockTrees.NotSyncedTree) + .AddKeyedSingleton(DbNames.Metadata, blockTrees.NotSyncedTreeBuilder.MetadataDb); + }, new SyncConfig() { NonValidatorNode = true, DownloadBodiesInFastSync = false, DownloadReceiptsInFastSync = false - })!; + }); + + PostMergeContext ctx = container.Resolve(); ctx.BeaconPivot.EnsurePivot(blockTrees.SyncedTree.FindHeader(64, BlockTreeLookupOptions.None)); @@ -374,7 +387,9 @@ public void No_old_bodies_and_receipts() [TestCase(DownloaderOptions.Process)] public async Task BlockDownloader_works_correctly_with_withdrawals(int options) { - PostMergeContext ctx = new(); + await using IContainer container = CreateMergeNode(); + PostMergeContext ctx = container.Resolve(); + DownloaderOptions downloaderOptions = (DownloaderOptions)options; bool withReceipts = downloaderOptions == DownloaderOptions.WithReceipts; BlockDownloader downloader = ctx.BlockDownloader; @@ -433,11 +448,12 @@ public void BlockDownloader_does_not_stop_processing_when_main_chain_is_unknown( .InsertBeaconHeaders(1, pivot) .InsertBeaconBlocks(pivot, pivot, BlockTreeTests.BlockTreeTestScenario.ScenarioBuilder.TotalDifficultyMode.Null); - PostMergeContext ctx = new() + using IContainer container = CreateMergeNode(blockTrees, new MergeConfig() { - BlockTreeScenario = blockTrees, - MergeConfig = new MergeConfig { TerminalTotalDifficulty = "0" } - }; + TerminalTotalDifficulty = $"0" + }); + + PostMergeContext ctx = container.Resolve(); ctx.BeaconPivot.EnsurePivot(blockTrees.SyncedTree.FindHeader(pivot, BlockTreeLookupOptions.None)); ctx.BeaconPivot.ProcessDestination = blockTrees.SyncedTree.FindHeader(pivot, BlockTreeLookupOptions.None); @@ -445,6 +461,40 @@ public void BlockDownloader_does_not_stop_processing_when_main_chain_is_unknown( Assert.DoesNotThrowAsync(() => ctx.BlockDownloader.DownloadBlocks(new(syncPeer), new BlocksRequest(downloaderOptions), CancellationToken.None)); } + private IContainer CreateMergeNode(Action? configurer = null, params IConfig[] configs) + { + IConfigProvider configProvider = new ConfigProvider(configs); + return CreateNode((builder) => + { + builder + .AddModule(new MergeModule( + configProvider.GetConfig(), + configProvider.GetConfig(), + configProvider.GetConfig() + )) + .AddSingleton(); + configurer?.Invoke(builder); + }, configProvider); + } + + private IContainer CreateMergeNode(BlockTreeTests.BlockTreeTestScenario.ScenarioBuilder treeBuilder, params IConfig[] configs) + { + return CreateMergeNode((builder) => + { + builder + .AddSingleton(treeBuilder.NotSyncedTree) + .AddKeyedSingleton(DbNames.Metadata, treeBuilder.NotSyncedTreeBuilder.MetadataDb); + }, configs); + } + + private class PostMergeContext(IComponentContext scope) : Context(scope) + { + private readonly IComponentContext _scope = scope; + public IBeaconPivot BeaconPivot => _scope.Resolve(); + public IPoSSwitcher PosSwitcher => _scope.Resolve(); + } + + /* class PostMergeContext : Context { protected override ISpecProvider SpecProvider => _specProvider ??= new MainnetSpecProvider(); // PoSSwitcher changes TTD, so can't use MainnetSpecProvider.Instance @@ -532,4 +582,5 @@ public override BlockDownloader BlockDownloader _peerAllocationStrategy ??= new MergeBlocksSyncPeerAllocationStrategyFactory(PosSwitcher, BeaconPivot, LimboLogs.Instance); } + */ } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs index 4e7a2241e76..cc1707e3d4f 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs @@ -19,7 +19,6 @@ using Nethermind.Core.Specs; using Nethermind.Core.Test.Builders; using Nethermind.Crypto; -using Nethermind.Db; using Nethermind.Int256; using Nethermind.Evm; using Nethermind.Logging; @@ -31,14 +30,16 @@ using Nethermind.Network.P2P.Subprotocols.Eth.V63.Messages; using Nethermind.Serialization.Rlp; using Nethermind.Synchronization.Blocks; -using Nethermind.Synchronization.ParallelSync; using Nethermind.Synchronization.Peers; -using Nethermind.Synchronization.Reporting; using NSubstitute; using NUnit.Framework; using BlockTree = Nethermind.Blockchain.BlockTree; using System.Diagnostics.CodeAnalysis; +using Autofac; +using Nethermind.Config; using Nethermind.Core.Test; +using Nethermind.Core.Test.Modules; +using Nethermind.Synchronization.ParallelSync; namespace Nethermind.Synchronization.Test; @@ -65,7 +66,8 @@ public partial class BlockDownloaderTests [TestCase(SyncBatchSize.Max * 8, DownloaderOptions.Process, 32)] public async Task Happy_path(long headNumber, int options, int threshold) { - Context ctx = new(); + await using IContainer node = CreateNode(); + Context ctx = node.Resolve(); DownloaderOptions downloaderOptions = (DownloaderOptions)options; bool withReceipts = downloaderOptions == DownloaderOptions.WithReceipts; BlockDownloader downloader = ctx.BlockDownloader; @@ -107,10 +109,13 @@ public async Task Happy_path(long headNumber, int options, int threshold) [Test] public async Task Ancestor_lookup_simple() { - Context ctx = new() + IBlockTree instance = CachedBlockTreeBuilder.OfLength(1024); + await using IContainer node = CreateNode(builder => { - BlockTree = CachedBlockTreeBuilder.OfLength(1024), - }; + builder.AddSingleton(instance); + }); + Context ctx = node.Resolve(); + BlockDownloader downloader = ctx.BlockDownloader; Response blockResponseOptions = Response.AllCorrect; @@ -138,10 +143,12 @@ public async Task Ancestor_lookup_simple() [Test] public async Task Ancestor_lookup_headers() { - Context ctx = new() + await using IContainer node = CreateNode(builder => { - BlockTree = CachedBlockTreeBuilder.OfLength(1024), - }; + builder.AddSingleton(CachedBlockTreeBuilder.OfLength(1024)); + }); + Context ctx = node.Resolve(); + BlockDownloader downloader = ctx.BlockDownloader; Response responseOptions = Response.AllCorrect; @@ -167,10 +174,12 @@ public async Task Ancestor_lookup_headers() [Test] public void Ancestor_failure() { - Context ctx = new() + using IContainer node = CreateNode(builder => { - BlockTree = CachedBlockTreeBuilder.OfLength(2048 + 1), - }; + builder.AddSingleton(CachedBlockTreeBuilder.OfLength(2048 + 1)); + }); + Context ctx = node.Resolve(); + BlockDownloader downloader = ctx.BlockDownloader; Response blockResponseOptions = Response.AllCorrect; @@ -185,10 +194,11 @@ public void Ancestor_failure() [Test] public void Ancestor_failure_blocks() { - Context ctx = new() + using IContainer node = CreateNode(builder => { - BlockTree = CachedBlockTreeBuilder.OfLength(2048 + 1), - }; + builder.AddSingleton(CachedBlockTreeBuilder.OfLength(2048+1)); + }); + Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; Response responseOptions = Response.AllCorrect; @@ -208,10 +218,15 @@ public void Ancestor_failure_blocks() [TestCase(0, false)] public async Task Can_sync_with_peer_when_it_times_out_on_full_batch(int ignoredBlocks, bool mergeDownloader) { - Context ctx = mergeDownloader ? new PostMergeContext() : new Context(); - SyncBatchSize syncBatchSize = new SyncBatchSize(LimboLogs.Instance); - syncBatchSize.ExpandUntilMax(); - ctx.SyncBatchSize = syncBatchSize; + Action configurer = builder => + { + SyncBatchSize syncBatchSize = new SyncBatchSize(LimboLogs.Instance); + syncBatchSize.ExpandUntilMax(); + builder.AddSingleton(syncBatchSize); + }; + + await using IContainer node = mergeDownloader ? CreateMergeNode(configurer) : CreateNode(configurer); + Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; ISyncPeer syncPeer = Substitute.For(); @@ -246,7 +261,8 @@ public async Task Can_sync_with_peer_when_it_times_out_on_full_batch(int ignored [TestCase(32, 16, 100, false)] public async Task Can_sync_partially_when_only_some_bodies_is_available(int blockCount, int availableBlock, int minResponseLength, bool mergeDownloader) { - Context ctx = mergeDownloader ? new PostMergeContext() : new Context(); + await using IContainer node = mergeDownloader ? CreateMergeNode() : CreateNode(); + Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; ISyncPeer syncPeer = Substitute.For(); @@ -298,7 +314,8 @@ public async Task Can_sync_partially_when_only_some_bodies_is_available(int bloc [Test] public async Task Headers_already_known() { - Context ctx = new(); + await using IContainer node = CreateNode(); + Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; ISyncPeer syncPeer = Substitute.For(); @@ -322,7 +339,8 @@ public async Task Headers_already_known() [Test] public async Task Peer_only_advertise_one_header() { - Context ctx = new(); + await using IContainer node = CreateNode(); + Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; ISyncPeer syncPeer = Substitute.For(); @@ -342,7 +360,8 @@ public async Task Peer_only_advertise_one_header() [TestCase(65L)] public async Task Peer_sends_just_one_item_when_advertising_more_blocks_but_no_bodies(long headNumber) { - Context ctx = new(); + await using IContainer node = CreateNode(); + Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; ISyncPeer syncPeer = Substitute.For(); @@ -365,7 +384,8 @@ public async Task Peer_sends_just_one_item_when_advertising_more_blocks_but_no_b [Test] public async Task Throws_on_null_best_peer() { - Context ctx = new(); + await using IContainer node = CreateNode(); + Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; Task task1 = downloader.DownloadHeaders(null, new BlocksRequest(DownloaderOptions.WithBodies, 0), CancellationToken.None); await task1.ContinueWith(static t => Assert.That(t.IsFaulted, Is.True)); @@ -377,7 +397,8 @@ public async Task Throws_on_null_best_peer() [Test] public async Task Throws_on_inconsistent_batch() { - Context ctx = new(); + await using IContainer node = CreateNode(); + Context ctx = node.Resolve(); ISyncPeer syncPeer = Substitute.For(); syncPeer.GetBlockHeaders(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(ci => ctx.ResponseBuilder.BuildHeaderResponse(ci.ArgAt(0), ci.ArgAt(1), Response.AllCorrect ^ Response.Consistent)); @@ -394,10 +415,8 @@ public async Task Throws_on_inconsistent_batch() [Test] public async Task Throws_on_invalid_seal() { - Context ctx = new() - { - SealValidator = Always.Invalid, - }; + await using IContainer node = CreateNode(builder => builder.AddSingleton(Always.Invalid)); + Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; ISyncPeer syncPeer = Substitute.For(); @@ -415,10 +434,8 @@ public async Task Throws_on_invalid_seal() [Test] public async Task Throws_on_invalid_header() { - Context ctx = new() - { - BlockValidator = Always.Invalid, - }; + await using IContainer node = CreateNode(builder => builder.AddSingleton(Always.Invalid)); + Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; ISyncPeer syncPeer = Substitute.For(); @@ -522,10 +539,8 @@ public bool Validate(BlockHeader header, bool isUncle, [NotNullWhen(false)] out [Ignore("Fails OneLoggerLogManager Travis only")] public async Task Can_cancel_seal_validation() { - Context ctx = new() - { - SealValidator = new SlowSealValidator(), - }; + await using IContainer node = CreateNode(builder => builder.AddSingleton(new SlowSealValidator())); + Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; ISyncPeer syncPeer = Substitute.For(); @@ -554,10 +569,8 @@ public async Task Can_cancel_seal_validation() [Test, MaxTime(15000)] public async Task Can_cancel_adding_headers() { - Context ctx = new() - { - BlockValidator = new SlowHeaderValidator(), - }; + await using IContainer node = CreateNode(builder => builder.AddSingleton(new SlowHeaderValidator())); + Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; ISyncPeer syncPeer = Substitute.For(); @@ -590,10 +603,8 @@ public async Task Validate_always_the_last_seal_and_random_seal_in_the_package() { ISealValidator sealValidator = Substitute.For(); sealValidator.ValidateSeal(Arg.Any(), Arg.Any()).Returns(true); - Context ctx = new() - { - SealValidator = sealValidator, - }; + await using IContainer node = CreateNode(builder => builder.AddSingleton(sealValidator)); + Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; using IOwnedReadOnlyList? blockHeaders = await ctx.ResponseBuilder.BuildHeaderResponse(0, 512, Response.AllCorrect); @@ -695,7 +706,8 @@ public bool TryGetSatelliteProtocol(string protocol, out T protocolHandler) w [Test] public async Task Faults_on_get_headers_faulting() { - Context ctx = new(); + await using IContainer node = CreateNode(); + Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; ISyncPeer syncPeer = new ThrowingPeer(1000, UInt256.MaxValue); @@ -708,7 +720,8 @@ public async Task Faults_on_get_headers_faulting() [Test] public async Task Throws_on_block_task_exception() { - Context ctx = new(); + await using IContainer node = CreateNode(); + Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; ISyncPeer syncPeer = Substitute.For(); @@ -737,7 +750,8 @@ public async Task Throws_on_block_task_exception() [TestCase(DownloaderOptions.Process, false)] public async Task Throws_on_receipt_task_exception_when_downloading_receipts(int options, bool shouldThrow) { - Context ctx = new(); + await using IContainer node = CreateNode(); + Context ctx = node.Resolve(); DownloaderOptions downloaderOptions = (DownloaderOptions)options; BlockDownloader downloader = ctx.BlockDownloader; @@ -775,7 +789,8 @@ public async Task Throws_on_receipt_task_exception_when_downloading_receipts(int [TestCase(DownloaderOptions.Process, false)] public async Task Throws_on_null_receipt_downloaded(int options, bool shouldThrow) { - Context ctx = new(); + await using IContainer node = CreateNode(); + Context ctx = node.Resolve(); DownloaderOptions downloaderOptions = (DownloaderOptions)options; bool withReceipts = downloaderOptions == DownloaderOptions.WithReceipts; BlockDownloader downloader = ctx.BlockDownloader; @@ -837,7 +852,8 @@ public async Task Throws_on_null_receipt_downloaded(int options, bool shouldThro [TestCase(0)] public async Task Throws_on_block_bodies_count_higher_than_receipts_list_count(int threshold) { - Context ctx = new(); + await using IContainer node = CreateNode(); + Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; ISyncPeer syncPeer = Substitute.For(); @@ -866,7 +882,8 @@ public async Task Throws_on_block_bodies_count_higher_than_receipts_list_count(i [TestCase(1)] public async Task Does_throw_on_transaction_count_different_than_receipts_count_in_block(int threshold) { - Context ctx = new(); + await using IContainer node = CreateNode(); + Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; ISyncPeer syncPeer = Substitute.For(); @@ -895,7 +912,8 @@ public async Task Does_throw_on_transaction_count_different_than_receipts_count_ [TestCase(1)] public async Task Throws_on_incorrect_receipts_root(int threshold) { - Context ctx = new(); + await using IContainer node = CreateNode(); + Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; ISyncPeer syncPeer = Substitute.For(); @@ -933,8 +951,51 @@ private enum Response IncorrectReceiptRoot = 256 } - private class Context + private IContainer CreateNode(Action? configurer = null, IConfigProvider? configProvider = null) + { + configProvider ??= new ConfigProvider(); + + Block genesis = Build.A.Block.Genesis.TestObject; + ContainerBuilder b = new ContainerBuilder() + .AddModule(new TestNethermindModule(configProvider)) + .AddSingleton() + .AddSingleton(Always.Valid) + .AddSingleton(new MainnetSpecProvider()) + .AddSingleton(Always.Valid) + .AddSingleton(Substitute.For()) + .AddSingleton() + .AddDecorator((ctx, tree) => + { + if (tree.Genesis is null) tree.SuggestBlock(genesis); + return tree; + }) + + .AddSingleton, IBlockTree>((blockTree) => new Dictionary() + { + { + 0, blockTree.Genesis!.Hash! + }, + }) + .AddSingleton(); + + configurer?.Invoke(b); + return b + .Build(); + } + + private class Context(IComponentContext scope) { + public ResponseBuilder ResponseBuilder => scope.Resolve(); + private SyncFeedComponent FullSyncFeedComponent => + scope.ResolveNamed>(nameof(FullSyncFeed)); + public BlockDownloader BlockDownloader => FullSyncFeedComponent.BlockDownloader; + public IBlockTree BlockTree => scope.Resolve(); + public InMemoryReceiptStorage ReceiptStorage => scope.Resolve(); + public ISyncPeerPool PeerPool => scope.Resolve(); + public ActivatedSyncFeed Feed => (ActivatedSyncFeed)FullSyncFeedComponent.Feed; + public SyncDispatcher Dispatcher => FullSyncFeedComponent.Dispatcher; + + /* private readonly Block _genesis = Build.A.Block.Genesis.TestObject; private readonly MemDb _blockInfoDb = new(); private IBlockTree? _blockTree { get; set; } @@ -1050,6 +1111,7 @@ public Context() }, }; } + */ } private class SyncPeerMock : ISyncPeer diff --git a/src/Nethermind/Nethermind.Synchronization/Blocks/SyncBatchSize.cs b/src/Nethermind/Nethermind.Synchronization/Blocks/SyncBatchSize.cs index 795424cc5d3..9bfa98a9459 100644 --- a/src/Nethermind/Nethermind.Synchronization/Blocks/SyncBatchSize.cs +++ b/src/Nethermind/Nethermind.Synchronization/Blocks/SyncBatchSize.cs @@ -8,7 +8,7 @@ namespace Nethermind.Synchronization.Blocks { [DebuggerDisplay("{Current}")] - public struct SyncBatchSize + public class SyncBatchSize { private readonly ILogger _logger; @@ -22,9 +22,9 @@ public struct SyncBatchSize public int Current { get; private set; } - public readonly bool IsMin => Current == Min; + public bool IsMin => Current == Min; - public readonly bool IsMax => Current == Max; + public bool IsMax => Current == Max; public SyncBatchSize(ILogManager logManager) { From 64faa2ab8fcce2308ca35f9c82201ec803f99898 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 30 Jan 2025 22:56:44 +0800 Subject: [PATCH 02/10] Fixed --- .../ContainerBuilderExtensions.cs | 3 +- .../BlockDownloaderTests.Merge.cs | 96 +------------- .../BlockDownloaderTests.cs | 120 +----------------- 3 files changed, 8 insertions(+), 211 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs b/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs index efc206bffea..45b34e62b90 100644 --- a/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs +++ b/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs @@ -313,7 +313,8 @@ public static ContainerBuilder AddDecorator(this ContainerBuilder builder, Fu public static ContainerBuilder RegisterNamedComponentInItsOwnLifetime(this ContainerBuilder builder, string name, Action configurator) where T : notnull { builder.Register(ctx => ctx.BeginLifetimeScope(configurator).Resolve()) - .Named(name); + .Named(name) + .SingleInstance(); return builder; } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs index bcfbae85593..316fbe5feae 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs @@ -345,6 +345,7 @@ public void No_old_bodies_and_receipts() .AddKeyedSingleton(DbNames.Metadata, blockTrees.NotSyncedTreeBuilder.MetadataDb); }, new SyncConfig() { + FastSync = true, NonValidatorNode = true, DownloadBodiesInFastSync = false, DownloadReceiptsInFastSync = false @@ -369,10 +370,11 @@ public void No_old_bodies_and_receipts() .Allocate(Arg.Any(), Arg.Any(), Arg.Any()) .Returns(Task.FromResult(peerAllocation)); - ctx.Feed.Activate(); + SyncFeedComponent fastSyncFeedComponent = ctx.FastSyncFeedComponent; + fastSyncFeedComponent.Feed.Activate(); CancellationTokenSource cts = new(); - Task _ = ctx.Dispatcher.Start(cts.Token); + Task _ = fastSyncFeedComponent.Dispatcher.Start(cts.Token); Assert.That( () => ctx.BlockTree.BestKnownNumber, @@ -493,94 +495,4 @@ private class PostMergeContext(IComponentContext scope) : Context(scope) public IBeaconPivot BeaconPivot => _scope.Resolve(); public IPoSSwitcher PosSwitcher => _scope.Resolve(); } - - /* - class PostMergeContext : Context - { - protected override ISpecProvider SpecProvider => _specProvider ??= new MainnetSpecProvider(); // PoSSwitcher changes TTD, so can't use MainnetSpecProvider.Instance - - private BlockTreeTests.BlockTreeTestScenario.ScenarioBuilder? _blockTreeScenario; - - public BlockTreeTests.BlockTreeTestScenario.ScenarioBuilder BlockTreeScenario - { - get => - _blockTreeScenario ?? - new BlockTreeTests.BlockTreeTestScenario.ScenarioBuilder(); - set => _blockTreeScenario = value; - } - - public override IBlockTree BlockTree => _blockTreeScenario?.NotSyncedTree ?? base.BlockTree; - - private IDb? _metadataDb; - private IDb MetadataDb => (_metadataDb ?? _blockTreeScenario?.NotSyncedTreeBuilder.MetadataDb) ?? (_metadataDb ??= new MemDb()); - - private MergeConfig? _mergeConfig; - - public MergeConfig MergeConfig - { - get => _mergeConfig ??= new MergeConfig - { TerminalTotalDifficulty = "58750000000000000000000" }; // Main block downloader test assume pre-merge - set => _mergeConfig = value; - } - - private BeaconPivot? _beaconPivot; - - private PoSSwitcher? _posSwitcher; - - public PoSSwitcher PosSwitcher => _posSwitcher ??= new( - MergeConfig, - new TestSyncConfig(), - MetadataDb, - BlockTree, - SpecProvider, - new ChainSpec(), - LimboLogs.Instance); - public BeaconPivot BeaconPivot => _beaconPivot ??= new(new TestSyncConfig(), MetadataDb, BlockTree, _posSwitcher!, LimboLogs.Instance); - - protected override IBetterPeerStrategy BetterPeerStrategy => _betterPeerStrategy ??= - new MergeBetterPeerStrategy(new TotalDifficultyBetterPeerStrategy(LimboLogs.Instance), PosSwitcher, BeaconPivot, LimboLogs.Instance); - - private IChainLevelHelper? _chainLevelHelper; - - public IChainLevelHelper ChainLevelHelper - { - get => - _chainLevelHelper ??= new ChainLevelHelper( - BlockTree, - BeaconPivot, - new TestSyncConfig(), - LimboLogs.Instance); - set => _chainLevelHelper = value; - } - - private MergeBlockDownloader? _mergeBlockDownloader; - - public override BlockDownloader BlockDownloader - { - get - { - return _mergeBlockDownloader ??= new( - PosSwitcher, - BeaconPivot, - Feed, - PeerPool, - BlockTree, - BlockValidator, - SealValidator, - NullSyncReport.Instance, - ReceiptStorage, - SpecProvider, - BetterPeerStrategy, - ChainLevelHelper, - Substitute.For(), - LimboLogs.Instance); - } - } - - private IPeerAllocationStrategyFactory? _peerAllocationStrategy; - protected override IPeerAllocationStrategyFactory PeerAllocationStrategy => - _peerAllocationStrategy ??= new MergeBlocksSyncPeerAllocationStrategyFactory(PosSwitcher, BeaconPivot, LimboLogs.Instance); - - } - */ } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs index cc1707e3d4f..5ae802a17c1 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs @@ -986,6 +986,8 @@ private IContainer CreateNode(Action? configurer = null, IConf private class Context(IComponentContext scope) { public ResponseBuilder ResponseBuilder => scope.Resolve(); + public SyncFeedComponent FastSyncFeedComponent => + scope.ResolveNamed>(nameof(FastSyncFeed)); private SyncFeedComponent FullSyncFeedComponent => scope.ResolveNamed>(nameof(FullSyncFeed)); public BlockDownloader BlockDownloader => FullSyncFeedComponent.BlockDownloader; @@ -994,124 +996,6 @@ private class Context(IComponentContext scope) public ISyncPeerPool PeerPool => scope.Resolve(); public ActivatedSyncFeed Feed => (ActivatedSyncFeed)FullSyncFeedComponent.Feed; public SyncDispatcher Dispatcher => FullSyncFeedComponent.Dispatcher; - - /* - private readonly Block _genesis = Build.A.Block.Genesis.TestObject; - private readonly MemDb _blockInfoDb = new(); - private IBlockTree? _blockTree { get; set; } - private Dictionary TestHeaderMapping { get; } - public InMemoryReceiptStorage ReceiptStorage { get; } = new(); - - private SyncBatchSize? _syncBatchSize; - - public SyncBatchSize? SyncBatchSize - { - get => _syncBatchSize ??= new SyncBatchSize(LimboLogs.Instance); - set => _syncBatchSize = value; - } - - protected ISpecProvider? _specProvider; - protected virtual ISpecProvider SpecProvider => _specProvider ??= MainnetSpecProvider.Instance; - - public virtual IBlockTree BlockTree - { - get - { - if (_blockTree is null) - { - _blockTree = Build.A.BlockTree() - .WithoutSettingHead - .WithSpecProvider(SpecProvider) - .WithBlockInfoDb(_blockInfoDb) - .TestObject; - _blockTree.SuggestBlock(_genesis); - } - - return _blockTree; - } - set - { - _blockTree = value; - } - } - - private ISyncPeerPool? _peerPool; - public ISyncPeerPool PeerPool => _peerPool ??= Substitute.For(); - - private ResponseBuilder? _responseBuilder; - public ResponseBuilder ResponseBuilder => - _responseBuilder ??= new ResponseBuilder(BlockTree, TestHeaderMapping); - - protected IBetterPeerStrategy? _betterPeerStrategy; - - protected virtual IBetterPeerStrategy BetterPeerStrategy => - _betterPeerStrategy ??= new TotalDifficultyBetterPeerStrategy(LimboLogs.Instance); - - private ActivatedSyncFeed? _feed; - - public ActivatedSyncFeed Feed - { - get => _feed ??= new FullSyncFeed(); - set => _feed = value; - } - - private ISealValidator? _sealValidator; - public ISealValidator SealValidator - { - get => _sealValidator ??= Always.Valid; - set => _sealValidator = value; - } - - private IBlockValidator? _blockValidator; - public IBlockValidator BlockValidator - { - get => _blockValidator ??= Always.Valid; - set => _blockValidator = value; - } - - private BlockDownloader? _blockDownloader; - public virtual BlockDownloader BlockDownloader => _blockDownloader ??= new BlockDownloader( - Feed, - PeerPool, - BlockTree, - BlockValidator, - SealValidator, - NullSyncReport.Instance, - ReceiptStorage, - SpecProvider, - BetterPeerStrategy, - LimboLogs.Instance, - SyncBatchSize - ); - - private SyncDispatcher? _dispatcher; - public SyncDispatcher Dispatcher => _dispatcher ??= new SyncDispatcher( - new TestSyncConfig() - { - MaxProcessingThreads = 0, - }, - Feed!, - BlockDownloader, - PeerPool, - PeerAllocationStrategy, - LimboLogs.Instance - ); - - private IPeerAllocationStrategyFactory? _peerAllocationStrategy; - - protected virtual IPeerAllocationStrategyFactory PeerAllocationStrategy => - _peerAllocationStrategy ??= new BlocksSyncPeerAllocationStrategyFactory(); - - public Context() - { - TestHeaderMapping = new Dictionary - { - { - 0, _genesis.Hash! - }, - }; - } - */ } private class SyncPeerMock : ISyncPeer From 82bbc36dc9adc287c73a00d2faa4089475b3af05 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 31 Jan 2025 11:41:12 +0800 Subject: [PATCH 03/10] Move more sync tests --- .../ConfigProviderTests.cs | 13 ++++ .../Builders/BlockTreeExtensions.cs | 6 +- .../Modules/MergeModule.cs | 8 ++- .../BlockDownloaderTests.Merge.cs | 6 +- .../SynchronizerTests.cs | 69 ++----------------- 5 files changed, 30 insertions(+), 72 deletions(-) diff --git a/src/Nethermind/Nethermind.Config.Test/ConfigProviderTests.cs b/src/Nethermind/Nethermind.Config.Test/ConfigProviderTests.cs index 97958ce62a6..37d805ddea8 100644 --- a/src/Nethermind/Nethermind.Config.Test/ConfigProviderTests.cs +++ b/src/Nethermind/Nethermind.Config.Test/ConfigProviderTests.cs @@ -4,6 +4,7 @@ using System; using System.Collections; using System.Collections.Generic; +using FluentAssertions; using Nethermind.Core.Extensions; using Nethermind.JsonRpc; using Nethermind.Network.Config; @@ -80,5 +81,17 @@ public void Can_read_overwrites() Assert.That(config.Enabled, Is.EqualTo(expectedResult), bitArray.ToBitString()); } } + + [Test] + public void Can_useExistingConfig() + { + BlocksConfig blocksConfig = new() + { + MinGasPrice = 12345, + }; + IConfigProvider configProvider = new ConfigProvider(blocksConfig); + + configProvider.GetConfig().MinGasPrice.Should().Be(12345); + } } } diff --git a/src/Nethermind/Nethermind.Core.Test/Builders/BlockTreeExtensions.cs b/src/Nethermind/Nethermind.Core.Test/Builders/BlockTreeExtensions.cs index ca015aabb06..91eedda9183 100644 --- a/src/Nethermind/Nethermind.Core.Test/Builders/BlockTreeExtensions.cs +++ b/src/Nethermind/Nethermind.Core.Test/Builders/BlockTreeExtensions.cs @@ -9,7 +9,7 @@ namespace Nethermind.Core.Test.Builders { public static class BlockTreeExtensions { - public static void AddBranch(this BlockTree blockTree, int branchLength, int splitBlockNumber) + public static void AddBranch(this IBlockTree blockTree, int branchLength, int splitBlockNumber) { int splitVariant = 0; BlockTree alternative = Build.A.BlockTree(blockTree.FindBlock(0, BlockTreeLookupOptions.RequireCanonical)!).OfChainLength(branchLength, splitVariant).TestObject; @@ -35,7 +35,7 @@ public static void AddBranch(this BlockTree blockTree, int branchLength, int spl } } - public static void AddBranch(this BlockTree blockTree, int branchLength, int splitBlockNumber, int splitVariant) + public static void AddBranch(this IBlockTree blockTree, int branchLength, int splitBlockNumber, int splitVariant) { BlockTree alternative = Build.A.BlockTree(blockTree.FindBlock(0, BlockTreeLookupOptions.RequireCanonical)!).OfChainLength(branchLength, splitVariant).TestObject; List blocks = new(); @@ -52,7 +52,7 @@ public static void AddBranch(this BlockTree blockTree, int branchLength, int spl } } - public static void UpdateMainChain(this BlockTree blockTree, Block block) + public static void UpdateMainChain(this IBlockTree blockTree, Block block) { blockTree.UpdateMainChain(new[] { block }, true); } diff --git a/src/Nethermind/Nethermind.Core.Test/Modules/MergeModule.cs b/src/Nethermind/Nethermind.Core.Test/Modules/MergeModule.cs index 96277eb4e21..9b6cb9c20c2 100644 --- a/src/Nethermind/Nethermind.Core.Test/Modules/MergeModule.cs +++ b/src/Nethermind/Nethermind.Core.Test/Modules/MergeModule.cs @@ -26,13 +26,19 @@ using Nethermind.State; using Nethermind.Synchronization; using Nethermind.Synchronization.ParallelSync; -using Nethermind.Synchronization.Peers; using Nethermind.TxPool; namespace Nethermind.Core.Test.Modules; public class MergeModule(ITxPoolConfig txPoolConfig, IMergeConfig mergeConfig, IBlocksConfig blocksConfig) : Module { + public MergeModule(IConfigProvider configProvider) : this( + configProvider.GetConfig(), + configProvider.GetConfig(), + configProvider.GetConfig() + ) { + } + protected override void Load(ContainerBuilder builder) { base.Load(builder); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs index 316fbe5feae..6da6957eee6 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs @@ -469,11 +469,7 @@ private IContainer CreateMergeNode(Action? configurer = null, return CreateNode((builder) => { builder - .AddModule(new MergeModule( - configProvider.GetConfig(), - configProvider.GetConfig(), - configProvider.GetConfig() - )) + .AddModule(new MergeModule(configProvider)) .AddSingleton(); configurer?.Invoke(builder); }, configProvider); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs index 5864994a780..e2774e55b14 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs @@ -10,7 +10,6 @@ using System.Threading.Tasks; using Autofac; using Nethermind.Blockchain; -using Nethermind.Blockchain.Blocks; using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; using Nethermind.Config; @@ -21,26 +20,16 @@ using Nethermind.Core.Crypto; using Nethermind.Core.Specs; using Nethermind.Core.Test.Builders; -using Nethermind.Core.Timers; -using Nethermind.Db; +using Nethermind.Core.Test.Modules; using Nethermind.Int256; using Nethermind.Logging; using Nethermind.Specs; -using Nethermind.Specs.Forks; -using Nethermind.Stats; using Nethermind.Stats.Model; using Nethermind.Merge.Plugin; -using Nethermind.Merge.Plugin.InvalidChainTracker; -using Nethermind.Merge.Plugin.Synchronization; -using Nethermind.Merge.Plugin.Test; -using Nethermind.Specs.ChainSpecStyle; -using Nethermind.State; using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.Peers; -using Nethermind.Trie.Pruning; using NSubstitute; using NUnit.Framework; -using Nethermind.Trie; namespace Nethermind.Synchronization.Test; @@ -280,7 +269,7 @@ public class SyncingContext : IAsyncDisposable public static ConcurrentQueue AllInstances { get; } = new(); private readonly Dictionary _peers = new(); - private BlockTree BlockTree { get; } + private IBlockTree BlockTree => Container.Resolve(); private ISyncServer SyncServer => Container.Resolve(); @@ -312,71 +301,25 @@ ISyncConfig GetSyncConfig() => syncConfig.SyncDispatcherEmptyRequestDelayMs = 1; syncConfig.SyncDispatcherAllocateTimeoutMs = 1; - IDbProvider dbProvider = TestMemDbProvider.Init(); - IDb stateDb = new MemDb(); - IDb codeDb = dbProvider.CodeDb; - IBlockStore blockStore = new BlockStore(dbProvider.BlocksDb); - BlockTree = Build.A.BlockTree() - .WithSpecProvider(new TestSingleReleaseSpecProvider(Constantinople.Instance)) - .WithBlockStore(blockStore) - .WithoutSettingHead - .TestObject; - - ITimerFactory timerFactory = Substitute.For(); - NodeStatsManager stats = new(timerFactory, _logManager); - MergeConfig mergeConfig = new(); if (WithTTD(synchronizerType)) { + mergeConfig.Enabled = true; mergeConfig.TerminalTotalDifficulty = UInt256.MaxValue.ToString(CultureInfo.InvariantCulture); } - PoSSwitcher poSSwitcher = new(mergeConfig, syncConfig, dbProvider.MetadataDb, BlockTree, new TestSingleReleaseSpecProvider(Constantinople.Instance), new ChainSpec(), _logManager); - IBeaconPivot beaconPivot = new BeaconPivot(syncConfig, dbProvider.MetadataDb, BlockTree, poSSwitcher, _logManager); - - IWorldStateManager worldStateManager = WorldStateManager.CreateForTest(dbProvider, LimboLogs.Instance); - TotalDifficultyBetterPeerStrategy totalDifficultyBetterPeerStrategy = new(LimboLogs.Instance); - IBetterPeerStrategy bestPeerStrategy = IsMerge(synchronizerType) - ? new MergeBetterPeerStrategy(totalDifficultyBetterPeerStrategy, poSSwitcher, beaconPivot, LimboLogs.Instance) - : totalDifficultyBetterPeerStrategy; - - IStateReader reader = worldStateManager.GlobalStateReader; - INodeStorage nodeStorage = new NodeStorage(dbProvider.StateDb); - - Pivot pivot = new(syncConfig); - - IInvalidChainTracker invalidChainTracker = new NoopInvalidChainTracker(); - + IConfigProvider configProvider = new ConfigProvider(syncConfig, mergeConfig); ContainerBuilder builder = new ContainerBuilder() - .AddModule(new DbModule()) - .AddModule(new SynchronizerModule(syncConfig)) - .AddSingleton(new ReceiptConfig()) - .AddSingleton(dbProvider) - .AddSingleton(blockStore) - .AddSingleton(worldStateManager) - .AddSingleton(nodeStorage) + .AddModule(new TestNethermindModule(configProvider)) .AddSingleton(MainnetSpecProvider.Instance) - .AddSingleton(BlockTree) .AddSingleton(NullReceiptStorage.Instance) - .AddSingleton(stats) - .AddSingleton(syncConfig) - .AddSingleton(pivot) - .AddSingleton(poSSwitcher) - .AddSingleton(mergeConfig) - .AddSingleton(invalidChainTracker) .AddSingleton(Substitute.For()) - .AddSingleton(bestPeerStrategy) - .AddSingleton(new ChainSpec()) - .AddSingleton(No.BeaconSync) - .AddSingleton(reader) .AddSingleton(Always.Valid) .AddSingleton(Always.Valid) - .AddSingleton(beaconPivot) - .AddSingleton(Policy.FullGossip) .AddSingleton(_logManager); if (IsMerge(synchronizerType)) { - builder.RegisterModule(new MergeSynchronizerModule()); + builder.RegisterModule(new MergeModule(configProvider)); } Container = builder.Build(); From 5323b86dfa227909ffa85aa3293106e03f4649ad Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 31 Jan 2025 21:26:47 +0800 Subject: [PATCH 04/10] Whitespace --- src/Nethermind/Nethermind.Core.Test/Modules/MergeModule.cs | 3 ++- .../Nethermind.Synchronization.Test/BlockDownloaderTests.cs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Core.Test/Modules/MergeModule.cs b/src/Nethermind/Nethermind.Core.Test/Modules/MergeModule.cs index 9b6cb9c20c2..e9c0b96c0b8 100644 --- a/src/Nethermind/Nethermind.Core.Test/Modules/MergeModule.cs +++ b/src/Nethermind/Nethermind.Core.Test/Modules/MergeModule.cs @@ -36,7 +36,8 @@ public MergeModule(IConfigProvider configProvider) : this( configProvider.GetConfig(), configProvider.GetConfig(), configProvider.GetConfig() - ) { + ) + { } protected override void Load(ContainerBuilder builder) diff --git a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs index 5ae802a17c1..ea0793a4b27 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs @@ -196,7 +196,7 @@ public void Ancestor_failure_blocks() { using IContainer node = CreateNode(builder => { - builder.AddSingleton(CachedBlockTreeBuilder.OfLength(2048+1)); + builder.AddSingleton(CachedBlockTreeBuilder.OfLength(2048 + 1)); }); Context ctx = node.Resolve(); BlockDownloader downloader = ctx.BlockDownloader; From 8eb467fa3b27c5f1ebb9267c666eb5389c8a2d7b Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 31 Jan 2025 23:36:24 +0800 Subject: [PATCH 05/10] Dedup common code --- .../Synchronization/MergeBlockDownloader.cs | 339 +++++------------ .../Synchronization/MergeSynchronizer.cs | 8 +- .../BlockDownloaderTests.Merge.cs | 1 - .../Blocks/BlockDownloadContext.cs | 2 + .../Blocks/BlockDownloader.cs | 345 ++++++++++++------ .../Synchronizer.cs | 8 +- 6 files changed, 330 insertions(+), 373 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs index baf3cf32ecf..a5507d19b33 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs @@ -16,6 +16,7 @@ using Nethermind.Core.Specs; using Nethermind.Crypto; using Nethermind.Logging; +using Nethermind.Network; using Nethermind.Synchronization; using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.ParallelSync; @@ -29,14 +30,9 @@ public class MergeBlockDownloader : BlockDownloader, ISyncDownloader HandleSyncRequestResult(t, bestPeer), cancellation); - } - finally - { - _allocationWithCancellation.Dispose(); - } + await base.Dispatch(bestPeer, blocksRequest, cancellation); } public override async Task DownloadBlocks(PeerInfo? bestPeer, BlocksRequest blocksRequest, CancellationToken cancellation) { + // Note: Redundant with Dispatch, but test uses it. if (_beaconPivot.BeaconPivotExists() == false && _poSSwitcher.HasEverReachedTerminalBlock() == false) { if (_logger.IsDebug) _logger.Debug("Using pre merge block downloader"); - return await base.DownloadBlocks(bestPeer, blocksRequest, cancellation); + return await _preMergeBlockDownloader.DownloadBlocks(bestPeer, blocksRequest, cancellation); } - if (bestPeer is null) - { - string message = $"Not expecting best peer to be null inside the {nameof(BlockDownloader)}"; - if (_logger.IsError) _logger.Error(message); - throw new ArgumentNullException(message); - } - - DownloaderOptions options = blocksRequest.Options; - bool downloadReceipts = (options & DownloaderOptions.WithReceipts) == DownloaderOptions.WithReceipts; - bool shouldProcess = (options & DownloaderOptions.Process) == DownloaderOptions.Process; - bool shouldMoveToMain = (options & DownloaderOptions.MoveToMain) == DownloaderOptions.MoveToMain; - - int blocksSynced = 0; - long currentNumber = _blockTree.BestKnownNumber; - if (_logger.IsDebug) - _logger.Debug( - $"MergeBlockDownloader GetCurrentNumber: currentNumber {currentNumber}, beaconPivotExists: {_beaconPivot.BeaconPivotExists()}, BestSuggestedBody: {_blockTree.BestSuggestedBody?.Number}, BestKnownNumber: {_blockTree.BestKnownNumber}, BestPeer: {bestPeer}, BestKnownBeaconNumber {_blockTree.BestKnownBeaconNumber}"); + return await base.DownloadBlocks(bestPeer, blocksRequest, cancellation); + } - bool HasMoreToSync(out BlockHeader[]? headers, out int headersToRequest) + public override async Task DownloadHeaders(PeerInfo? bestPeer, BlocksRequest blocksRequest, + CancellationToken cancellation) + { + // Note: Redundant with Dispatch, but test uses it. + if (_beaconPivot.BeaconPivotExists() == false && _poSSwitcher.HasEverReachedTerminalBlock() == false) { if (_logger.IsDebug) - _logger.Debug($"Continue full sync with {bestPeer} (our best {_blockTree.BestKnownNumber})"); - - headersToRequest = Math.Min(_syncBatchSize.Current, bestPeer.MaxHeadersPerRequest()); - if (_logger.IsTrace) - _logger.Trace( - $"Full sync request {currentNumber}+{headersToRequest} to peer {bestPeer} with {bestPeer.HeadNumber} blocks. Got {currentNumber} and asking for {headersToRequest} more."); + _logger.Debug("Using pre merge block downloader"); + return await _preMergeBlockDownloader.DownloadHeaders(bestPeer, blocksRequest, cancellation); + } - headers = _chainLevelHelper.GetNextHeaders(headersToRequest, bestPeer.HeadNumber, blocksRequest.NumberOfLatestBlocksToBeIgnored ?? 0); - if (headers is null || headers.Length <= 1) - { - if (_logger.IsTrace) - _logger.Trace("Chain level helper got no headers suggestion"); - return false; - } + return await base.DownloadHeaders(bestPeer, blocksRequest, cancellation); + } - return true; - } + protected override Task?> HasMoreToSync(PeerInfo bestPeer, long currentNumber, BlocksRequest blocksRequest, CancellationToken cancellation) + { + if (_logger.IsDebug) + _logger.Debug($"Continue full sync with {bestPeer} (our best {_blockTree.BestKnownNumber})"); - long bestProcessedBlock = 0; + int headersToRequest = Math.Min(_syncBatchSize.Current, bestPeer.MaxHeadersPerRequest()); + if (_logger.IsTrace) + _logger.Trace( + $"Full sync request {currentNumber}+{headersToRequest} to peer {bestPeer} with {bestPeer.HeadNumber} blocks. Got {currentNumber} and asking for {headersToRequest} more."); - while (HasMoreToSync(out BlockHeader[]? headers, out int headersToRequest)) + BlockHeader?[]? headers = _chainLevelHelper.GetNextHeaders(headersToRequest, bestPeer.HeadNumber, blocksRequest.NumberOfLatestBlocksToBeIgnored ?? 0); + if (headers is null || headers.Length <= 1) { - if (HasBetterPeer) - { - if (_logger.IsDebug) _logger.Debug("Has better peer, stopping"); - break; - } - - if (cancellation.IsCancellationRequested) return blocksSynced; // check before every heavy operation - Block[]? blocks = null; - TxReceipt[]?[]? receipts = null; if (_logger.IsTrace) - _logger.Trace( - $"Downloading blocks from peer. CurrentNumber: {currentNumber}, BeaconPivot: {_beaconPivot.PivotNumber}, BestPeer: {bestPeer}, HeaderToRequest: {headersToRequest}"); - - // Alternatively we can do this in BeaconHeadersSyncFeed, but this seems easier. - ValidateSeals(headers!, cancellation); - - BlockDownloadContext context = new(_specProvider, bestPeer, headers!, downloadReceipts, _receiptsRecovery); - - if (cancellation.IsCancellationRequested) return blocksSynced; // check before every heavy operation - - long startTime = Stopwatch.GetTimestamp(); - await RequestBodies(bestPeer, cancellation, context); - - if (downloadReceipts) - { - if (cancellation.IsCancellationRequested) - return blocksSynced; // check before every heavy operation - await RequestReceipts(bestPeer, cancellation, context); - } - - AdjustSyncBatchSize(Stopwatch.GetElapsedTime(startTime)); - - blocks = context.Blocks; - receipts = context.ReceiptsForBlocks; - - if (!(blocks?.Length > 0)) - { - if (_logger.IsTrace) - _logger.Trace("Break early due to no blocks."); - break; - } - - for (int blockIndex = 0; blockIndex < blocks.Length; blockIndex++) - { - if (cancellation.IsCancellationRequested) - { - if (_logger.IsTrace) _logger.Trace("Peer sync cancelled"); - break; - } - - Block currentBlock = blocks[blockIndex]; - if (_logger.IsTrace) _logger.Trace($"Received {currentBlock} from {bestPeer}"); - - if (currentBlock.IsBodyMissing) - { - throw new EthSyncException($"{bestPeer} didn't send body for block {currentBlock.ToString(Block.Format.Short)}."); - } - - // can move this to block tree now? - if (!_blockValidator.ValidateSuggestedBlock(currentBlock, out string? errorMessage)) - { - string message = InvalidBlockHelper.GetMessage(currentBlock, $"invalid block sent by peer. {errorMessage}") + - $" PeerInfo {bestPeer}"; - if (_logger.IsWarn) _logger.Warn(message); - throw new EthSyncException(message); - } - - if (shouldProcess) - { - // An edge case where we already have the state but are still downloading preceding blocks. - // We cannot process such blocks, but we are still requested to process them via blocksRequest.Options. - // Therefore, we detect this situation and switch from processing to receipts downloading. - bool headIsGenesis = _blockTree.Head?.IsGenesis ?? false; - bool toBeProcessedHasNoProcessedParent = currentBlock.Number > (bestProcessedBlock + 1); - bool isFastSyncTransition = headIsGenesis && toBeProcessedHasNoProcessedParent; - if (isFastSyncTransition) - { - long bestFullState = _fullStateFinder.FindBestFullState(); - shouldProcess = currentBlock.Number > bestFullState && bestFullState != 0; - if (!shouldProcess && !downloadReceipts) - { - if (_logger.IsInfo) _logger.Info($"Skipping processing during fastSyncTransition, currentBlock: {currentBlock}, bestFullState: {bestFullState}, trying to load receipts"); - downloadReceipts = true; - context.SetDownloadReceipts(); - await RequestReceipts(bestPeer, cancellation, context); - receipts = context.ReceiptsForBlocks; - } - } - } - - if (downloadReceipts) - { - TxReceipt[]? contextReceiptsForBlock = receipts![blockIndex]; - if (currentBlock.Header.HasTransactions && contextReceiptsForBlock is null) - { - throw new EthSyncException($"{bestPeer} didn't send receipts for block {currentBlock.ToString(Block.Format.Short)}."); - } - } + _logger.Trace("Chain level helper got no headers suggestion"); + return Task.FromResult?>(null); + } - bool isKnownBeaconBlock = _blockTree.IsKnownBeaconBlock(currentBlock.Number, currentBlock.GetOrCalculateHash()); - BlockTreeSuggestOptions suggestOptions = - shouldProcess ? BlockTreeSuggestOptions.ShouldProcess : BlockTreeSuggestOptions.None; - if (_logger.IsTrace) - _logger.Trace( - $"Current block {currentBlock}, BeaconPivot: {_beaconPivot.PivotNumber}, IsKnownBeaconBlock: {isKnownBeaconBlock}"); + // Alternatively we can do this in BeaconHeadersSyncFeed, but this seems easier. + ValidateSeals(headers!, cancellation); - if (isKnownBeaconBlock) - { - suggestOptions |= BlockTreeSuggestOptions.FillBeaconBlock; - } + return Task.FromResult?>(headers.ToPooledList(0)); + } - if (_logger.IsTrace) - _logger.Trace( - $"MergeBlockDownloader - SuggestBlock {currentBlock}, IsKnownBeaconBlock {isKnownBeaconBlock} ShouldProcess: {shouldProcess}"); + protected override bool CheckAncestorJump(PeerInfo? bestPeer, Block[]? blocks, BlockDownloadContext context, ref long currentNumber) + { + // No ancestor jump check post merge. + return true; + } - AddBlockResult addResult = _blockTree.SuggestBlock(currentBlock, suggestOptions); - if (HandleAddResult(bestPeer, currentBlock.Header, blockIndex == 0, addResult)) - { - if (shouldProcess == false) - { - _blockTree.UpdateMainChain(new[] { currentBlock }, false); - } - else - { - bestProcessedBlock = currentBlock.Number; - } + protected override BlockTreeSuggestOptions DetermineSuggestOptions(bool shouldProcess, Block currentBlock) + { + BlockTreeSuggestOptions suggestOptions = + shouldProcess ? BlockTreeSuggestOptions.ShouldProcess : BlockTreeSuggestOptions.None; - TryUpdateTerminalBlock(currentBlock.Header, shouldProcess); + bool isKnownBeaconBlock = _blockTree.IsKnownBeaconBlock(currentBlock.Number, currentBlock.GetOrCalculateHash()); + if (_logger.IsTrace) + _logger.Trace( + $"Current block {currentBlock}, BeaconPivot: {_beaconPivot.PivotNumber}, IsKnownBeaconBlock: {isKnownBeaconBlock}"); - if (downloadReceipts) - { - TxReceipt[]? contextReceiptsForBlock = receipts![blockIndex]; - if (contextReceiptsForBlock is not null) - { - _receiptStorage.Insert(currentBlock, contextReceiptsForBlock); - } - else - { - // this shouldn't now happen with new validation above, still lets keep this check - if (currentBlock.Header.HasTransactions) - { - if (_logger.IsError) _logger.Error($"{currentBlock} is missing receipts"); - } - } - } + if (isKnownBeaconBlock) + { + suggestOptions |= BlockTreeSuggestOptions.FillBeaconBlock; + } - if ((_beaconPivot.ProcessDestination?.Number ?? long.MaxValue) < currentBlock.Number) - { - // Move the process destination in front, otherwise `ChainLevelHelper` would continue returning - // already processed header starting from `ProcessDestination`. - _beaconPivot.ProcessDestination = currentBlock.Header; - } - blocksSynced++; - } + if (_logger.IsTrace) + _logger.Trace( + $"MergeBlockDownloader - SuggestBlock {currentBlock}, IsKnownBeaconBlock {isKnownBeaconBlock} ShouldProcess: {shouldProcess}"); + return suggestOptions; + } - if (shouldMoveToMain) - { - _blockTree.UpdateMainChain(new[] { currentBlock }, false); - } + protected override void OnBlockAdded(Block currentBlock) + { + if ((_beaconPivot.ProcessDestination?.Number ?? long.MaxValue) < currentBlock.Number) + { + // Move the process destination in front, otherwise `ChainLevelHelper` would continue returning + // already processed header starting from `ProcessDestination`. + _beaconPivot.ProcessDestination = currentBlock.Header; + } + } + } - currentNumber += 1; - } + public class PosTransitionHook(IBlockTree blockTree, IPoSSwitcher poSSwitcher, ILogManager logManager) : IPosTransitionHook + { + private readonly ILogger _logger = logManager.GetClassLogger(); - if (blocksSynced > 0) - { - _syncReport.FullSyncBlocksDownloaded.TargetValue = bestPeer.HeadNumber; - _syncReport.FullSyncBlocksDownloaded.Update(_blockTree.BestSuggestedHeader?.Number ?? 0); - } - else - { - break; - } - } + public void TryUpdateTerminalBlock(BlockHeader currentHeader, bool shouldProcess) + { + if (shouldProcess == false) // if we're processing the block we will find TerminalBlock after processing + poSSwitcher.TryUpdateTerminalBlock(currentHeader); + } - return blocksSynced; + public bool ImprovementRequirementSatisfied(PeerInfo? bestPeer) + { + return bestPeer!.TotalDifficulty > (blockTree.BestSuggestedHeader?.TotalDifficulty ?? 0) && + poSSwitcher.HasEverReachedTerminalBlock() == false; } - protected override async Task> RequestHeaders(PeerInfo peer, CancellationToken cancellation, long currentNumber, int headersToRequest) + public IOwnedReadOnlyList FilterPosHeader(IOwnedReadOnlyList response) { // Override PoW's RequestHeaders so that it won't request beyond PoW. // This fixes `Incremental Sync` hive test. - IOwnedReadOnlyList response = await base.RequestHeaders(peer, cancellation, currentNumber, headersToRequest); if (response.Count > 0) { BlockHeader lastBlockHeader = response[^1]; - bool lastBlockIsPostMerge = _poSSwitcher.GetBlockConsensusInfo(response[^1]).IsPostMerge; + bool lastBlockIsPostMerge = poSSwitcher.GetBlockConsensusInfo(response[^1]).IsPostMerge; if (lastBlockIsPostMerge) // Initial check to prevent creating new array every time { response = response - .TakeWhile(header => !_poSSwitcher.GetBlockConsensusInfo(header).IsPostMerge) + .TakeWhile(header => !poSSwitcher.GetBlockConsensusInfo(header).IsPostMerge) .ToPooledList(response.Count); if (_logger.IsInfo) _logger.Info($"Last block is post merge. {lastBlockHeader.Hash}. Trimming to {response.Count} sized batch."); } } return response; } - - protected override void TryUpdateTerminalBlock(BlockHeader header, bool shouldProcess) - { - if (shouldProcess == false) // if we're processing the block we will find TerminalBlock after processing - _poSSwitcher.TryUpdateTerminalBlock(header); - } - - protected override bool ImprovementRequirementSatisfied(PeerInfo? bestPeer) - { - return bestPeer!.TotalDifficulty > (_blockTree.BestSuggestedHeader?.TotalDifficulty ?? 0) && - _poSSwitcher.HasEverReachedTerminalBlock() == false; - } } } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs index 957787f5768..b902f62e9d1 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs @@ -78,12 +78,8 @@ public class MergeSynchronizerModule : Module protected override void Load(ContainerBuilder builder) { builder - .RegisterType() - .As() - .As>() - .InstancePerLifetimeScope(); - - builder + .Add() + .AddDecorator() .AddSingleton() .AddSingleton() .AddScoped, MergeBlocksSyncPeerAllocationStrategyFactory>() diff --git a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs index 6da6957eee6..61756d0a734 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs @@ -25,7 +25,6 @@ using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.Peers; using Nethermind.Synchronization.Peers.AllocationStrategies; -using Nethermind.TxPool; using NSubstitute; using NSubstitute.ClearExtensions; using NUnit.Framework; diff --git a/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloadContext.cs b/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloadContext.cs index c3a87a72d5d..9bae7b456ba 100644 --- a/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloadContext.cs +++ b/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloadContext.cs @@ -71,6 +71,8 @@ public BlockDownloadContext(ISpecProvider specProvider, PeerInfo syncPeer, IRead public List NonEmptyBlockHashes { get; } + public bool DownloadReceipts => _downloadReceipts; + public IReadOnlyList GetHashesByOffset(int offset, int maxLength) { var hashesToRequest = diff --git a/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloader.cs b/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloader.cs index 93c50c1fe2f..b1ad76aab81 100644 --- a/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloader.cs +++ b/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloader.cs @@ -42,6 +42,8 @@ public class BlockDownloader : ISyncDownloader private readonly IReceiptsRecovery _receiptsRecovery; private readonly ISpecProvider _specProvider; private readonly IBetterPeerStrategy _betterPeerStrategy; + private readonly IFullStateFinder _fullStateFinder; + private readonly IPosTransitionHook _posTransitionHook; private readonly ILogger _logger; private readonly ISyncPeerPool _syncPeerPool; private readonly Guid _sealValidatorUserGuid = Guid.NewGuid(); @@ -54,6 +56,7 @@ public class BlockDownloader : ISyncDownloader protected SyncBatchSize _syncBatchSize; protected int _sinceLastTimeout; private readonly int[] _ancestorJumps = { 1, 2, 3, 8, 16, 32, 64, 128, 256, 384, 512, 640, 768, 896, 1024 }; + private int _ancestorLookupLevel; public BlockDownloader( ISyncFeed? feed, @@ -65,6 +68,8 @@ public BlockDownloader( IReceiptStorage? receiptStorage, ISpecProvider? specProvider, IBetterPeerStrategy betterPeerStrategy, + IFullStateFinder fullStateFinder, + IPosTransitionHook posTransitionHook, ILogManager? logManager, SyncBatchSize? syncBatchSize = null) { @@ -77,6 +82,8 @@ public BlockDownloader( _receiptStorage = receiptStorage ?? throw new ArgumentNullException(nameof(receiptStorage)); _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); _betterPeerStrategy = betterPeerStrategy ?? throw new ArgumentNullException(nameof(betterPeerStrategy)); + _fullStateFinder = fullStateFinder ?? throw new ArgumentNullException(nameof(fullStateFinder)); + _posTransitionHook = posTransitionHook; _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); _receiptsRecovery = new ReceiptsRecovery(new EthereumEcdsa(_specProvider.ChainId), _specProvider); @@ -138,7 +145,7 @@ await DownloadHeaders(bestPeer, blocksRequest, cancellation) } } - public async Task DownloadHeaders(PeerInfo? bestPeer, BlocksRequest blocksRequest, CancellationToken cancellation) + public virtual async Task DownloadHeaders(PeerInfo? bestPeer, BlocksRequest blocksRequest, CancellationToken cancellation) { if (bestPeer is null) { @@ -151,25 +158,13 @@ public async Task DownloadHeaders(PeerInfo? bestPeer, BlocksRequest blocks int ancestorLookupLevel = 0; long currentNumber = Math.Max(0, Math.Min(_blockTree.BestKnownNumber, bestPeer.HeadNumber - 1)); - bool HasMoreToSync() - => currentNumber <= bestPeer!.HeadNumber; - while (ImprovementRequirementSatisfied(bestPeer) && HasMoreToSync()) + IOwnedReadOnlyList? headers = null; + while ((headers = await HasMoreToSync(bestPeer, currentNumber, blocksRequest, cancellation)) is not null) { if (HasBetterPeer) break; int headersSyncedInPreviousRequests = headersSynced; if (_logger.IsTrace) _logger.Trace($"Continue headers sync with {bestPeer} (our best {_blockTree.BestKnownNumber})"); - long blocksLeft = bestPeer.HeadNumber - currentNumber - (blocksRequest.NumberOfLatestBlocksToBeIgnored ?? 0); - int headersToRequest = (int)Math.Min(blocksLeft + 1, _syncBatchSize.Current); - if (headersToRequest <= 1) - { - break; - } - - if (_logger.IsDebug) _logger.Debug($"Headers request {currentNumber}+{headersToRequest} to peer {bestPeer} with {bestPeer.HeadNumber} blocks. Got {currentNumber} and asking for {headersToRequest} more."); - long startTime = Stopwatch.GetTimestamp(); - using IOwnedReadOnlyList headers = await RequestHeaders(bestPeer, cancellation, currentNumber, headersToRequest); - Hash256? startHeaderHash = headers[0]?.Hash; BlockHeader? startHeader = (startHeaderHash is null) ? null : _blockTree.FindHeader(startHeaderHash, BlockTreeLookupOptions.TotalDifficultyNotNeeded); @@ -188,7 +183,6 @@ bool HasMoreToSync() } ancestorLookupLevel = 0; - AdjustSyncBatchSize(Stopwatch.GetElapsedTime(startTime)); for (int i = 1; i < headers.Count; i++) { @@ -220,7 +214,7 @@ bool HasMoreToSync() // loop iterator to start with o if (HandleAddResult(bestPeer, currentHeader, i == 0, _blockTree.Insert(currentHeader))) { - TryUpdateTerminalBlock(currentHeader, false); + _posTransitionHook.TryUpdateTerminalBlock(currentHeader, false); headersSynced++; } @@ -256,43 +250,21 @@ public virtual async Task DownloadBlocks(PeerInfo? bestPeer, BlocksRequest bool shouldMoveToMain = (options & DownloaderOptions.MoveToMain) == DownloaderOptions.MoveToMain; int blocksSynced = 0; - int ancestorLookupLevel = 0; + _ancestorLookupLevel = 0; long currentNumber = Math.Max(0, Math.Min(_blockTree.BestKnownNumber, bestPeer.HeadNumber - 1)); // pivot number - 6 for uncle validation // long currentNumber = Math.Max(Math.Max(0, pivotNumber - 6), Math.Min(_blockTree.BestKnownNumber, bestPeer.HeadNumber - 1)); + long bestProcessedBlock = 0; - bool HasMoreToSync() - => currentNumber <= bestPeer!.HeadNumber; - while (ImprovementRequirementSatisfied(bestPeer!) && HasMoreToSync()) + IOwnedReadOnlyList? headers = null; + while ((headers = await HasMoreToSync(bestPeer, currentNumber, blocksRequest, cancellation)) is not null) { if (HasBetterPeer) break; - if (_logger.IsDebug) _logger.Debug($"Continue full sync with {bestPeer} (our best {_blockTree.BestKnownNumber})"); - - long upperDownloadBoundary = bestPeer.HeadNumber - (blocksRequest.NumberOfLatestBlocksToBeIgnored ?? 0); - long blocksLeft = upperDownloadBoundary - currentNumber; - int headersToRequest = (int)Math.Min(blocksLeft + 1, _syncBatchSize.Current); - if (headersToRequest <= 1) - { - break; - } - - headersToRequest = Math.Min(headersToRequest, bestPeer.MaxHeadersPerRequest()); - if (_logger.IsTrace) _logger.Trace($"Full sync request {currentNumber}+{headersToRequest} to peer {bestPeer} with {bestPeer.HeadNumber} blocks. Got {currentNumber} and asking for {headersToRequest} more."); - if (cancellation.IsCancellationRequested) return blocksSynced; // check before every heavy operation - using IOwnedReadOnlyList headers = await RequestHeaders(bestPeer, cancellation, currentNumber, headersToRequest); - if (headers.Count < 2) - { - // Peer dont have new header - break; - } BlockDownloadContext context = new(_specProvider, bestPeer, headers, downloadReceipts, _receiptsRecovery); - - if (cancellation.IsCancellationRequested) return blocksSynced; // check before every heavy operation - - Stopwatch sw = Stopwatch.StartNew(); + long startTime = Stopwatch.GetTimestamp(); await RequestBodies(bestPeer, cancellation, context); if (downloadReceipts) @@ -301,29 +273,20 @@ bool HasMoreToSync() await RequestReceipts(bestPeer, cancellation, context); } - AdjustSyncBatchSize(sw.Elapsed); + AdjustSyncBatchSize(Stopwatch.GetElapsedTime(startTime)); - Block[] blocks = context.Blocks; - Block blockZero = blocks[0]; - if (context.FullBlocksCount > 0) - { - bool parentIsKnown = _blockTree.IsKnownBlock(blockZero.Number - 1, blockZero.ParentHash); - if (!parentIsKnown) - { - ancestorLookupLevel++; - if (ancestorLookupLevel >= _ancestorJumps.Length) - { - if (_logger.IsWarn) _logger.Warn($"Could not find common ancestor with {bestPeer}"); - throw new EthSyncException("Peer with inconsistent chain in sync"); - } + Block[]? blocks = context.Blocks; + TxReceipt[]?[]? receipts = context.ReceiptsForBlocks; - int ancestorJump = _ancestorJumps[ancestorLookupLevel] - _ancestorJumps[ancestorLookupLevel - 1]; - currentNumber = currentNumber >= ancestorJump ? (currentNumber - ancestorJump) : 0L; - continue; - } + if (!(blocks?.Length > 0)) + { + if (_logger.IsTrace) + _logger.Trace("Break early due to no blocks."); + break; } - ancestorLookupLevel = 0; + if (!CheckAncestorJump(bestPeer, blocks, context, ref currentNumber)) continue; + for (int blockIndex = 0; blockIndex < context.FullBlocksCount; blockIndex++) { if (cancellation.IsCancellationRequested) @@ -333,57 +296,19 @@ bool HasMoreToSync() } Block currentBlock = blocks[blockIndex]; - if (_logger.IsTrace) _logger.Trace($"Received {currentBlock} from {bestPeer}"); - - if (currentBlock.IsBodyMissing) - { - throw new EthSyncException($"{bestPeer} didn't send body for block {currentBlock.ToString(Block.Format.Short)}."); - } - - // can move this to block tree now? - if (!_blockValidator.ValidateSuggestedBlock(currentBlock, out _)) - { - throw new EthSyncException($"{bestPeer} sent an invalid block {currentBlock.ToString(Block.Format.Short)}."); - } - - if (downloadReceipts) + long blockNumber = currentBlock.Number; + (shouldProcess, receipts) = await ReceiptEdgeCase(bestPeer, cancellation, blockNumber, bestProcessedBlock, context, shouldProcess, receipts); + PreValidate(bestPeer, context, blockIndex); + if (SuggestBlock(bestPeer, currentBlock, blockIndex, shouldProcess, context.DownloadReceipts, receipts, shouldMoveToMain)) { - TxReceipt[]? contextReceiptsForBlock = context.ReceiptsForBlocks![blockIndex]; - if (currentBlock.Header.HasTransactions && contextReceiptsForBlock is null) + if (shouldProcess) { - throw new EthSyncException($"{bestPeer} didn't send receipts for block {currentBlock.ToString(Block.Format.Short)}."); - } - } - - if (_logger.IsTrace) _logger.Trace($"BlockDownloader - SuggestBlock {currentBlock}, ShouldProcess: {true}"); - if (HandleAddResult(bestPeer, currentBlock.Header, blockIndex == 0, _blockTree.SuggestBlock(currentBlock, shouldProcess ? BlockTreeSuggestOptions.ShouldProcess : BlockTreeSuggestOptions.None))) - { - TryUpdateTerminalBlock(currentBlock.Header, shouldProcess); - if (downloadReceipts) - { - TxReceipt[]? contextReceiptsForBlock = context.ReceiptsForBlocks![blockIndex]; - if (contextReceiptsForBlock is not null) - { - _receiptStorage.Insert(currentBlock, contextReceiptsForBlock); - } - else - { - // this shouldn't now happen with new validation above, still lets keep this check - if (currentBlock.Header.HasTransactions) - { - if (_logger.IsError) _logger.Error($"{currentBlock} is missing receipts"); - } - } + bestProcessedBlock = currentBlock.Number; } blocksSynced++; } - if (shouldMoveToMain) - { - _blockTree.UpdateMainChain(new[] { currentBlock }, false); - } - currentNumber += 1; } @@ -400,9 +325,181 @@ bool HasMoreToSync() return blocksSynced; } - protected virtual bool ImprovementRequirementSatisfied(PeerInfo? bestPeer) + + protected virtual async Task?> HasMoreToSync(PeerInfo bestPeer, long currentNumber, BlocksRequest blocksRequest, CancellationToken cancellation) + { + if (!_posTransitionHook.ImprovementRequirementSatisfied(bestPeer)) return null; + if (currentNumber > bestPeer!.HeadNumber) return null; + + if (_logger.IsDebug) _logger.Debug($"Continue full sync with {bestPeer} (our best {_blockTree.BestKnownNumber})"); + + long upperDownloadBoundary = bestPeer.HeadNumber - (blocksRequest.NumberOfLatestBlocksToBeIgnored ?? 0); + long blocksLeft = upperDownloadBoundary - currentNumber; + int headersToRequest = (int)Math.Min(blocksLeft + 1, _syncBatchSize.Current); + if (headersToRequest <= 1) + { + return null; + } + + headersToRequest = Math.Min(headersToRequest, bestPeer.MaxHeadersPerRequest()); + if (_logger.IsTrace) _logger.Trace($"Full sync request {currentNumber}+{headersToRequest} to peer {bestPeer} with {bestPeer.HeadNumber} blocks. Got {currentNumber} and asking for {headersToRequest} more."); + + cancellation.ThrowIfCancellationRequested(); + IOwnedReadOnlyList? headers = await RequestHeaders(bestPeer, cancellation, currentNumber, headersToRequest); + if (headers.Count < 2) + { + // Peer dont have new header + headers.Dispose(); + return null; + } + + return headers; + } + + protected virtual bool CheckAncestorJump(PeerInfo? bestPeer, Block[]? blocks, BlockDownloadContext context, ref long currentNumber) { - return bestPeer!.TotalDifficulty > (_blockTree.BestSuggestedHeader?.TotalDifficulty ?? 0); + Block blockZero = blocks[0]; + if (context.FullBlocksCount > 0) + { + bool parentIsKnown = _blockTree.IsKnownBlock(blockZero.Number - 1, blockZero.ParentHash); + if (!parentIsKnown) + { + _ancestorLookupLevel++; + if (_ancestorLookupLevel >= _ancestorJumps.Length) + { + if (_logger.IsWarn) _logger.Warn($"Could not find common ancestor with {bestPeer}"); + throw new EthSyncException("Peer with inconsistent chain in sync"); + } + + int ancestorJump = _ancestorJumps[_ancestorLookupLevel] - _ancestorJumps[_ancestorLookupLevel - 1]; + currentNumber = currentNumber >= ancestorJump ? (currentNumber - ancestorJump) : 0L; + return false; + } + } + _ancestorLookupLevel = 0; + return true; + } + + protected virtual BlockTreeSuggestOptions DetermineSuggestOptions(bool shouldProcess, Block currentBlock) + { + if (_logger.IsTrace) _logger.Trace($"BlockDownloader - SuggestBlock {currentBlock}, ShouldProcess: {true}"); + return shouldProcess ? BlockTreeSuggestOptions.ShouldProcess : BlockTreeSuggestOptions.None; + } + + protected bool SuggestBlock( + PeerInfo bestPeer, + Block currentBlock, + int blockIndex, + bool shouldProcess, + bool downloadReceipts, + TxReceipt[]?[]? receipts, bool shouldMoveToMain) + { + BlockTreeSuggestOptions suggestOptions = DetermineSuggestOptions(shouldProcess, currentBlock); + AddBlockResult addResult = _blockTree.SuggestBlock(currentBlock, suggestOptions); + bool handled = false; + if (HandleAddResult(bestPeer, currentBlock.Header, blockIndex == 0, addResult)) + { + if (shouldProcess == false) + { + _blockTree.UpdateMainChain(new[] { currentBlock }, false); + } + + _posTransitionHook.TryUpdateTerminalBlock(currentBlock.Header, shouldProcess); + if (downloadReceipts) + { + TxReceipt[]? contextReceiptsForBlock = receipts![blockIndex]; + if (contextReceiptsForBlock is not null) + { + _receiptStorage.Insert(currentBlock, contextReceiptsForBlock); + } + else + { + // this shouldn't now happen with new validation above, still lets keep this check + if (currentBlock.Header.HasTransactions) + { + if (_logger.IsError) _logger.Error($"{currentBlock} is missing receipts"); + } + } + } + + OnBlockAdded(currentBlock); + handled = true; + } + + if (shouldMoveToMain) + { + _blockTree.UpdateMainChain(new[] { currentBlock }, false); + } + + return handled; + } + + + protected virtual void OnBlockAdded(Block currentBlock) + { + } + + protected async Task<(bool shouldProcess, TxReceipt[]?[]? receipts)> ReceiptEdgeCase( + PeerInfo bestPeer, + CancellationToken cancellation, + long currentBlockNumber, + long bestProcessedBlock, + BlockDownloadContext context, + bool shouldProcess, + TxReceipt[]?[]? receipts) + { + if (shouldProcess) + { + // An edge case where we already have the state but are still downloading preceding blocks. + // We cannot process such blocks, but we are still requested to process them via blocksRequest.Options. + // Therefore, we detect this situation and switch from processing to receipts downloading. + bool headIsGenesis = _blockTree.Head?.IsGenesis ?? false; + bool toBeProcessedHasNoProcessedParent = currentBlockNumber > (bestProcessedBlock + 1); + bool isFastSyncTransition = headIsGenesis && toBeProcessedHasNoProcessedParent; + if (isFastSyncTransition) + { + long bestFullState = _fullStateFinder.FindBestFullState(); + shouldProcess = currentBlockNumber > bestFullState && bestFullState != 0; + if (!shouldProcess && !context.DownloadReceipts) + { + if (_logger.IsInfo) _logger.Info($"Skipping processing during fastSyncTransition, currentBlock: {currentBlockNumber}, bestFullState: {bestFullState}, trying to load receipts"); + context.SetDownloadReceipts(); + await RequestReceipts(bestPeer, cancellation, context); + receipts = context.ReceiptsForBlocks; + } + } + } + + return (shouldProcess, receipts); + } + + protected void PreValidate(PeerInfo bestPeer, BlockDownloadContext blockDownloadContext, int blockIndex) + { + Block currentBlock = blockDownloadContext.Blocks[blockIndex]; + if (_logger.IsTrace) _logger.Trace($"Received {currentBlock} from {bestPeer}"); + + if (currentBlock.IsBodyMissing) + { + throw new EthSyncException($"{bestPeer} didn't send body for block {currentBlock.ToString(Block.Format.Short)}."); + } + + // can move this to block tree now? + if (!_blockValidator.ValidateSuggestedBlock(currentBlock, out string? errorMessage)) + { + string message = InvalidBlockHelper.GetMessage(currentBlock, $"invalid block sent by peer. {errorMessage}") + + $" PeerInfo {bestPeer}"; + if (_logger.IsWarn) _logger.Warn(message); + throw new EthSyncException(message); + } + + if (blockDownloadContext.DownloadReceipts) + { + TxReceipt[]? contextReceiptsForBlock = blockDownloadContext.ReceiptsForBlocks![blockIndex]; + if (currentBlock.Header.HasTransactions && contextReceiptsForBlock is null) + { + throw new EthSyncException($"{bestPeer} didn't send receipts for block {currentBlock.ToString(Block.Format.Short)}."); + } + } } private ValueTask DownloadFailHandler(Task downloadTask, string entities) @@ -424,7 +521,7 @@ private ValueTask DownloadFailHandler(Task downloadTask, string entities) return default; } - protected virtual async Task> RequestHeaders(PeerInfo peer, CancellationToken cancellation, long currentNumber, int headersToRequest) + private async Task> RequestHeaders(PeerInfo peer, CancellationToken cancellation, long currentNumber, int headersToRequest) { _sealValidator.HintValidationRange(_sealValidatorUserGuid, currentNumber - 1028, currentNumber + 30000); Task> headersRequest = peer.SyncPeer.GetBlockHeaders(currentNumber, headersToRequest, 0, cancellation); @@ -433,6 +530,8 @@ protected virtual async Task> RequestHeaders(Pee cancellation.ThrowIfCancellationRequested(); IOwnedReadOnlyList headers = headersRequest.Result; + headers = _posTransitionHook.FilterPosHeader(headers); + ValidateSeals(headers, cancellation); ValidateBatchConsistencyAndSetParents(peer, headers); return headers; @@ -630,8 +729,6 @@ void UpdatePeerInfo(PeerInfo peer, BlockHeader header) } } - protected virtual void TryUpdateTerminalBlock(BlockHeader header, bool shouldProcess) { } - public event EventHandler? SyncEvent; protected void InvokeEvent(SyncEventArgs args) @@ -808,4 +905,28 @@ public void Dispose() } } } + + public interface IPosTransitionHook + { + void TryUpdateTerminalBlock(BlockHeader currentHeader, bool shouldProcess); + bool ImprovementRequirementSatisfied(PeerInfo? peerInfo); + IOwnedReadOnlyList FilterPosHeader(IOwnedReadOnlyList headers); + } + + public class NoPosTransition(IBlockTree blockTree) : IPosTransitionHook + { + public void TryUpdateTerminalBlock(BlockHeader currentHeader, bool shouldProcess) + { + } + + public bool ImprovementRequirementSatisfied(PeerInfo? bestPeer) + { + return bestPeer!.TotalDifficulty > (blockTree.BestSuggestedHeader?.TotalDifficulty ?? 0); + } + + public IOwnedReadOnlyList FilterPosHeader(IOwnedReadOnlyList headers) + { + return headers; + } + } } diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 44582f46c8b..53435f55f28 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -302,7 +302,13 @@ protected override void Load(ContainerBuilder builder) // For blocks. There are two block scope, Fast and Full .AddScoped>() - .AddScoped, BlockDownloader>() + + // The direct implementation is decorated by merge plugin (not the interface) + // so its declared on its own and other use is binded. + .AddScoped() + .Add() + .Bind, BlockDownloader>() + .AddScoped, BlocksSyncPeerAllocationStrategyFactory>() .AddScoped>() From 0d37819ed5378c7907e23c35515f8ddaa5d4b2ef Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 31 Jan 2025 23:56:50 +0800 Subject: [PATCH 06/10] Slight cleanup --- .../Synchronization/MergeBlockDownloader.cs | 13 ++++- .../Blocks/BlockDownloader.cs | 56 ++++++++++--------- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs index a5507d19b33..1cf9e7bac71 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs @@ -12,6 +12,7 @@ using Nethermind.Consensus.Validators; using Nethermind.Core; using Nethermind.Core.Collections; +using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Core.Specs; using Nethermind.Crypto; @@ -104,7 +105,7 @@ public override async Task DownloadHeaders(PeerInfo? bestPeer, BlocksReque return await base.DownloadHeaders(bestPeer, blocksRequest, cancellation); } - protected override Task?> HasMoreToSync(PeerInfo bestPeer, long currentNumber, BlocksRequest blocksRequest, CancellationToken cancellation) + protected override Task?> GetBlockHeaders(PeerInfo bestPeer, long currentNumber, BlocksRequest blocksRequest, CancellationToken cancellation) { if (_logger.IsDebug) _logger.Debug($"Continue full sync with {bestPeer} (our best {_blockTree.BestKnownNumber})"); @@ -128,13 +129,19 @@ public override async Task DownloadHeaders(PeerInfo? bestPeer, BlocksReque return Task.FromResult?>(headers.ToPooledList(0)); } - protected override bool CheckAncestorJump(PeerInfo? bestPeer, Block[]? blocks, BlockDownloadContext context, ref long currentNumber) + protected override bool CheckAncestorJump(PeerInfo? bestPeer, Hash256? startHeaderHash, ref long currentNumber) { // No ancestor jump check post merge. return true; } - protected override BlockTreeSuggestOptions DetermineSuggestOptions(bool shouldProcess, Block currentBlock) + protected override bool CheckAncestorJump(PeerInfo? bestPeer, BlockDownloadContext context, ref long currentNumber) + { + // No ancestor jump check post merge. + return true; + } + + protected override BlockTreeSuggestOptions GetSuggestOption(bool shouldProcess, Block currentBlock) { BlockTreeSuggestOptions suggestOptions = shouldProcess ? BlockTreeSuggestOptions.ShouldProcess : BlockTreeSuggestOptions.None; diff --git a/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloader.cs b/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloader.cs index b1ad76aab81..90f0a30f167 100644 --- a/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloader.cs +++ b/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloader.cs @@ -155,34 +155,17 @@ public virtual async Task DownloadHeaders(PeerInfo? bestPeer, BlocksReques } int headersSynced = 0; - int ancestorLookupLevel = 0; long currentNumber = Math.Max(0, Math.Min(_blockTree.BestKnownNumber, bestPeer.HeadNumber - 1)); IOwnedReadOnlyList? headers = null; - while ((headers = await HasMoreToSync(bestPeer, currentNumber, blocksRequest, cancellation)) is not null) + while ((headers = await GetBlockHeaders(bestPeer, currentNumber, blocksRequest, cancellation)) is not null) { if (HasBetterPeer) break; int headersSyncedInPreviousRequests = headersSynced; if (_logger.IsTrace) _logger.Trace($"Continue headers sync with {bestPeer} (our best {_blockTree.BestKnownNumber})"); Hash256? startHeaderHash = headers[0]?.Hash; - BlockHeader? startHeader = (startHeaderHash is null) - ? null : _blockTree.FindHeader(startHeaderHash, BlockTreeLookupOptions.TotalDifficultyNotNeeded); - if (startHeader is null) - { - ancestorLookupLevel++; - if (ancestorLookupLevel >= _ancestorJumps.Length) - { - if (_logger.IsWarn) _logger.Warn($"Could not find common ancestor with {bestPeer}"); - throw new EthSyncException("Peer with inconsistent chain in sync"); - } - - int ancestorJump = _ancestorJumps[ancestorLookupLevel] - _ancestorJumps[ancestorLookupLevel - 1]; - currentNumber = currentNumber >= ancestorJump ? (currentNumber - ancestorJump) : 0L; - continue; - } - - ancestorLookupLevel = 0; + if (!CheckAncestorJump(bestPeer, startHeaderHash, ref currentNumber)) continue; for (int i = 1; i < headers.Count; i++) { @@ -258,7 +241,7 @@ public virtual async Task DownloadBlocks(PeerInfo? bestPeer, BlocksRequest long bestProcessedBlock = 0; IOwnedReadOnlyList? headers = null; - while ((headers = await HasMoreToSync(bestPeer, currentNumber, blocksRequest, cancellation)) is not null) + while ((headers = await GetBlockHeaders(bestPeer, currentNumber, blocksRequest, cancellation)) is not null) { if (HasBetterPeer) break; if (cancellation.IsCancellationRequested) return blocksSynced; // check before every heavy operation @@ -285,7 +268,7 @@ public virtual async Task DownloadBlocks(PeerInfo? bestPeer, BlocksRequest break; } - if (!CheckAncestorJump(bestPeer, blocks, context, ref currentNumber)) continue; + if (!CheckAncestorJump(bestPeer, context, ref currentNumber)) continue; for (int blockIndex = 0; blockIndex < context.FullBlocksCount; blockIndex++) { @@ -326,7 +309,7 @@ public virtual async Task DownloadBlocks(PeerInfo? bestPeer, BlocksRequest return blocksSynced; } - protected virtual async Task?> HasMoreToSync(PeerInfo bestPeer, long currentNumber, BlocksRequest blocksRequest, CancellationToken cancellation) + protected virtual async Task?> GetBlockHeaders(PeerInfo bestPeer, long currentNumber, BlocksRequest blocksRequest, CancellationToken cancellation) { if (!_posTransitionHook.ImprovementRequirementSatisfied(bestPeer)) return null; if (currentNumber > bestPeer!.HeadNumber) return null; @@ -356,11 +339,32 @@ public virtual async Task DownloadBlocks(PeerInfo? bestPeer, BlocksRequest return headers; } - protected virtual bool CheckAncestorJump(PeerInfo? bestPeer, Block[]? blocks, BlockDownloadContext context, ref long currentNumber) + protected virtual bool CheckAncestorJump(PeerInfo? bestPeer, Hash256? startHeaderHash, ref long currentNumber) + { + BlockHeader? startHeader = (startHeaderHash is null) + ? null : _blockTree.FindHeader(startHeaderHash, BlockTreeLookupOptions.TotalDifficultyNotNeeded); + if (startHeader is null) + { + _ancestorLookupLevel++; + if (_ancestorLookupLevel >= _ancestorJumps.Length) + { + if (_logger.IsWarn) _logger.Warn($"Could not find common ancestor with {bestPeer}"); + throw new EthSyncException("Peer with inconsistent chain in sync"); + } + + int ancestorJump = _ancestorJumps[_ancestorLookupLevel] - _ancestorJumps[_ancestorLookupLevel - 1]; + currentNumber = currentNumber >= ancestorJump ? (currentNumber - ancestorJump) : 0L; + return false; + } + _ancestorLookupLevel = 0; + return true; + } + + protected virtual bool CheckAncestorJump(PeerInfo? bestPeer, BlockDownloadContext context, ref long currentNumber) { - Block blockZero = blocks[0]; if (context.FullBlocksCount > 0) { + Block blockZero = context.Blocks[0]; bool parentIsKnown = _blockTree.IsKnownBlock(blockZero.Number - 1, blockZero.ParentHash); if (!parentIsKnown) { @@ -380,7 +384,7 @@ protected virtual bool CheckAncestorJump(PeerInfo? bestPeer, Block[]? blocks, Bl return true; } - protected virtual BlockTreeSuggestOptions DetermineSuggestOptions(bool shouldProcess, Block currentBlock) + protected virtual BlockTreeSuggestOptions GetSuggestOption(bool shouldProcess, Block currentBlock) { if (_logger.IsTrace) _logger.Trace($"BlockDownloader - SuggestBlock {currentBlock}, ShouldProcess: {true}"); return shouldProcess ? BlockTreeSuggestOptions.ShouldProcess : BlockTreeSuggestOptions.None; @@ -394,7 +398,7 @@ protected bool SuggestBlock( bool downloadReceipts, TxReceipt[]?[]? receipts, bool shouldMoveToMain) { - BlockTreeSuggestOptions suggestOptions = DetermineSuggestOptions(shouldProcess, currentBlock); + BlockTreeSuggestOptions suggestOptions = GetSuggestOption(shouldProcess, currentBlock); AddBlockResult addResult = _blockTree.SuggestBlock(currentBlock, suggestOptions); bool handled = false; if (HandleAddResult(bestPeer, currentBlock.Header, blockIndex == 0, addResult)) From ed41195b549f451da376683d11db417b301e1dc8 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Sat, 1 Feb 2025 00:06:18 +0800 Subject: [PATCH 07/10] Separate hooks to its own file --- .../Synchronization/MergeBlockDownloader.cs | 39 --------------- .../Synchronization/PosTransitionHook.cs | 50 +++++++++++++++++++ .../Blocks/BlockDownloader.cs | 42 ++++------------ .../Blocks/IPosTransitionHook.cs | 15 ++++++ .../Blocks/NoPosTransition.cs | 26 ++++++++++ 5 files changed, 100 insertions(+), 72 deletions(-) create mode 100644 src/Nethermind/Nethermind.Merge.Plugin/Synchronization/PosTransitionHook.cs create mode 100644 src/Nethermind/Nethermind.Synchronization/Blocks/IPosTransitionHook.cs create mode 100644 src/Nethermind/Nethermind.Synchronization/Blocks/NoPosTransition.cs diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs index 1cf9e7bac71..809844f40da 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs @@ -2,8 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Diagnostics; -using System.Linq; using System.Threading; using System.Threading.Tasks; using Nethermind.Blockchain; @@ -17,7 +15,6 @@ using Nethermind.Core.Specs; using Nethermind.Crypto; using Nethermind.Logging; -using Nethermind.Network; using Nethermind.Synchronization; using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.ParallelSync; @@ -172,40 +169,4 @@ protected override void OnBlockAdded(Block currentBlock) } } } - - public class PosTransitionHook(IBlockTree blockTree, IPoSSwitcher poSSwitcher, ILogManager logManager) : IPosTransitionHook - { - private readonly ILogger _logger = logManager.GetClassLogger(); - - public void TryUpdateTerminalBlock(BlockHeader currentHeader, bool shouldProcess) - { - if (shouldProcess == false) // if we're processing the block we will find TerminalBlock after processing - poSSwitcher.TryUpdateTerminalBlock(currentHeader); - } - - public bool ImprovementRequirementSatisfied(PeerInfo? bestPeer) - { - return bestPeer!.TotalDifficulty > (blockTree.BestSuggestedHeader?.TotalDifficulty ?? 0) && - poSSwitcher.HasEverReachedTerminalBlock() == false; - } - - public IOwnedReadOnlyList FilterPosHeader(IOwnedReadOnlyList response) - { - // Override PoW's RequestHeaders so that it won't request beyond PoW. - // This fixes `Incremental Sync` hive test. - if (response.Count > 0) - { - BlockHeader lastBlockHeader = response[^1]; - bool lastBlockIsPostMerge = poSSwitcher.GetBlockConsensusInfo(response[^1]).IsPostMerge; - if (lastBlockIsPostMerge) // Initial check to prevent creating new array every time - { - response = response - .TakeWhile(header => !poSSwitcher.GetBlockConsensusInfo(header).IsPostMerge) - .ToPooledList(response.Count); - if (_logger.IsInfo) _logger.Info($"Last block is post merge. {lastBlockHeader.Hash}. Trimming to {response.Count} sized batch."); - } - } - return response; - } - } } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/PosTransitionHook.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/PosTransitionHook.cs new file mode 100644 index 00000000000..142d544ef20 --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/PosTransitionHook.cs @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Linq; +using Nethermind.Blockchain; +using Nethermind.Consensus; +using Nethermind.Core; +using Nethermind.Core.Collections; +using Nethermind.Core.Extensions; +using Nethermind.Logging; +using Nethermind.Synchronization.Blocks; +using Nethermind.Synchronization.Peers; + +namespace Nethermind.Merge.Plugin.Synchronization; + +public class PosTransitionHook(IBlockTree blockTree, IPoSSwitcher poSSwitcher, ILogManager logManager) : IPosTransitionHook +{ + private readonly ILogger _logger = logManager.GetClassLogger(); + + public void TryUpdateTerminalBlock(BlockHeader currentHeader, bool shouldProcess) + { + if (shouldProcess == false) // if we're processing the block we will find TerminalBlock after processing + poSSwitcher.TryUpdateTerminalBlock(currentHeader); + } + + public bool ImprovementRequirementSatisfied(PeerInfo? bestPeer) + { + return bestPeer!.TotalDifficulty > (blockTree.BestSuggestedHeader?.TotalDifficulty ?? 0) && + poSSwitcher.HasEverReachedTerminalBlock() == false; + } + + public IOwnedReadOnlyList FilterPosHeader(IOwnedReadOnlyList response) + { + // Override PoW's RequestHeaders so that it won't request beyond PoW. + // This fixes `Incremental Sync` hive test. + if (response.Count > 0) + { + BlockHeader lastBlockHeader = response[^1]; + bool lastBlockIsPostMerge = poSSwitcher.GetBlockConsensusInfo(response[^1]).IsPostMerge; + if (lastBlockIsPostMerge) // Initial check to prevent creating new array every time + { + response = response + .TakeWhile(header => !poSSwitcher.GetBlockConsensusInfo(header).IsPostMerge) + .ToPooledList(response.Count); + if (_logger.IsInfo) _logger.Info($"Last block is post merge. {lastBlockHeader.Hash}. Trimming to {response.Count} sized batch."); + } + } + return response; + } +} diff --git a/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloader.cs b/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloader.cs index 90f0a30f167..d2bd3e330da 100644 --- a/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloader.cs +++ b/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloader.cs @@ -103,7 +103,7 @@ private void BlockTreeOnNewHeadBlock(object? sender, BlockEventArgs e) _syncReport.FullSyncBlocksDownloaded.Update(_blockTree.BestSuggestedHeader?.Number ?? 0); } - protected PeerInfo? _previousBestPeer = null; + private PeerInfo? _previousBestPeer = null; public virtual async Task Dispatch(PeerInfo bestPeer, BlocksRequest? blocksRequest, CancellationToken cancellation) { @@ -390,7 +390,7 @@ protected virtual BlockTreeSuggestOptions GetSuggestOption(bool shouldProcess, B return shouldProcess ? BlockTreeSuggestOptions.ShouldProcess : BlockTreeSuggestOptions.None; } - protected bool SuggestBlock( + private bool SuggestBlock( PeerInfo bestPeer, Block currentBlock, int blockIndex, @@ -477,7 +477,7 @@ protected virtual void OnBlockAdded(Block currentBlock) return (shouldProcess, receipts); } - protected void PreValidate(PeerInfo bestPeer, BlockDownloadContext blockDownloadContext, int blockIndex) + private void PreValidate(PeerInfo bestPeer, BlockDownloadContext blockDownloadContext, int blockIndex) { Block currentBlock = blockDownloadContext.Blocks[blockIndex]; if (_logger.IsTrace) _logger.Trace($"Received {currentBlock} from {bestPeer}"); @@ -541,7 +541,7 @@ private async Task> RequestHeaders(PeerInfo peer return headers; } - protected async Task RequestBodies(PeerInfo peer, CancellationToken cancellation, BlockDownloadContext context) + private async Task RequestBodies(PeerInfo peer, CancellationToken cancellation, BlockDownloadContext context) { int offset = 0; while (offset != context.NonEmptyBlockHashes.Count) @@ -575,7 +575,7 @@ protected async Task RequestBodies(PeerInfo peer, CancellationToken cancellation } } - protected async Task RequestReceipts(PeerInfo peer, CancellationToken cancellation, BlockDownloadContext context) + private async Task RequestReceipts(PeerInfo peer, CancellationToken cancellation, BlockDownloadContext context) { int offset = 0; while (offset != context.NonEmptyBlockHashes.Count) @@ -686,7 +686,7 @@ protected void ValidateSeals(IReadOnlyList headers, CancellationTo } } - protected bool HandleAddResult(PeerInfo peerInfo, BlockHeader block, bool isFirstInBatch, AddBlockResult addResult) + private bool HandleAddResult(PeerInfo peerInfo, BlockHeader block, bool isFirstInBatch, AddBlockResult addResult) { void UpdatePeerInfo(PeerInfo peer, BlockHeader header) { @@ -735,12 +735,12 @@ void UpdatePeerInfo(PeerInfo peer, BlockHeader header) public event EventHandler? SyncEvent; - protected void InvokeEvent(SyncEventArgs args) + private void InvokeEvent(SyncEventArgs args) { SyncEvent?.Invoke(this, args); } - protected void HandleSyncRequestResult(Task task, PeerInfo? peerInfo) + private void HandleSyncRequestResult(Task task, PeerInfo? peerInfo) { switch (task) { @@ -804,7 +804,7 @@ protected void HandleSyncRequestResult(Task task, PeerInfo? peerInfo) /// Adjust the sync batch size according to how much time it take to download the batch. /// /// - protected void AdjustSyncBatchSize(TimeSpan downloadTime) + private void AdjustSyncBatchSize(TimeSpan downloadTime) { // We shrink the batch size to prevent timeout. Timeout are wasted bandwidth. if (downloadTime > SyncBatchDownloadTimeUpperBound) @@ -909,28 +909,4 @@ public void Dispose() } } } - - public interface IPosTransitionHook - { - void TryUpdateTerminalBlock(BlockHeader currentHeader, bool shouldProcess); - bool ImprovementRequirementSatisfied(PeerInfo? peerInfo); - IOwnedReadOnlyList FilterPosHeader(IOwnedReadOnlyList headers); - } - - public class NoPosTransition(IBlockTree blockTree) : IPosTransitionHook - { - public void TryUpdateTerminalBlock(BlockHeader currentHeader, bool shouldProcess) - { - } - - public bool ImprovementRequirementSatisfied(PeerInfo? bestPeer) - { - return bestPeer!.TotalDifficulty > (blockTree.BestSuggestedHeader?.TotalDifficulty ?? 0); - } - - public IOwnedReadOnlyList FilterPosHeader(IOwnedReadOnlyList headers) - { - return headers; - } - } } diff --git a/src/Nethermind/Nethermind.Synchronization/Blocks/IPosTransitionHook.cs b/src/Nethermind/Nethermind.Synchronization/Blocks/IPosTransitionHook.cs new file mode 100644 index 00000000000..6a7e9603868 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/Blocks/IPosTransitionHook.cs @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Core.Collections; +using Nethermind.Synchronization.Peers; + +namespace Nethermind.Synchronization.Blocks; + +public interface IPosTransitionHook +{ + void TryUpdateTerminalBlock(BlockHeader currentHeader, bool shouldProcess); + bool ImprovementRequirementSatisfied(PeerInfo? peerInfo); + IOwnedReadOnlyList FilterPosHeader(IOwnedReadOnlyList headers); +} diff --git a/src/Nethermind/Nethermind.Synchronization/Blocks/NoPosTransition.cs b/src/Nethermind/Nethermind.Synchronization/Blocks/NoPosTransition.cs new file mode 100644 index 00000000000..1228fbdc083 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/Blocks/NoPosTransition.cs @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Blockchain; +using Nethermind.Core; +using Nethermind.Core.Collections; +using Nethermind.Synchronization.Peers; + +namespace Nethermind.Synchronization.Blocks; + +public class NoPosTransition(IBlockTree blockTree) : IPosTransitionHook +{ + public void TryUpdateTerminalBlock(BlockHeader currentHeader, bool shouldProcess) + { + } + + public bool ImprovementRequirementSatisfied(PeerInfo? bestPeer) + { + return bestPeer!.TotalDifficulty > (blockTree.BestSuggestedHeader?.TotalDifficulty ?? 0); + } + + public IOwnedReadOnlyList FilterPosHeader(IOwnedReadOnlyList headers) + { + return headers; + } +} From 29c1e92771a585e4b714e516568c76b0c695271a Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 12 Feb 2025 12:31:38 +0800 Subject: [PATCH 08/10] Update src/Nethermind/Nethermind.Merge.Plugin/Synchronization/PosTransitionHook.cs Co-authored-by: Lukasz Rozmej --- .../Synchronization/PosTransitionHook.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/PosTransitionHook.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/PosTransitionHook.cs index 142d544ef20..6ccf438e42b 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/PosTransitionHook.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/PosTransitionHook.cs @@ -19,7 +19,7 @@ public class PosTransitionHook(IBlockTree blockTree, IPoSSwitcher poSSwitcher, I public void TryUpdateTerminalBlock(BlockHeader currentHeader, bool shouldProcess) { - if (shouldProcess == false) // if we're processing the block we will find TerminalBlock after processing + if (!shouldProcess) // if we're processing the block we will find TerminalBlock after processing poSSwitcher.TryUpdateTerminalBlock(currentHeader); } From 042f11cb54421ae0441cb8adfa7834ec70bc94ef Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 12 Feb 2025 15:37:12 +0800 Subject: [PATCH 09/10] Addressing comment --- .../Synchronization/MergeBlockDownloader.cs | 13 +++++++++---- .../Synchronization/PosTransitionHook.cs | 9 ++++++--- .../Blocks/BlockDownloader.cs | 13 +++++++++++-- .../Blocks/IPosTransitionHook.cs | 5 ++++- .../Blocks/NoPosTransition.cs | 2 +- 5 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs index 809844f40da..2a6e0927ac0 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs @@ -61,9 +61,14 @@ public MergeBlockDownloader( _logger = logManager.GetClassLogger(); } + private bool ShouldUsePreMerge() + { + return _beaconPivot.BeaconPivotExists() == false && _poSSwitcher.HasEverReachedTerminalBlock() == false; + } + public override async Task Dispatch(PeerInfo bestPeer, BlocksRequest? blocksRequest, CancellationToken cancellation) { - if (_beaconPivot.BeaconPivotExists() == false && _poSSwitcher.HasEverReachedTerminalBlock() == false) + if (ShouldUsePreMerge()) { if (_logger.IsDebug) _logger.Debug("Using pre merge dispatcher"); @@ -78,7 +83,7 @@ public override async Task DownloadBlocks(PeerInfo? bestPeer, BlocksReques CancellationToken cancellation) { // Note: Redundant with Dispatch, but test uses it. - if (_beaconPivot.BeaconPivotExists() == false && _poSSwitcher.HasEverReachedTerminalBlock() == false) + if (ShouldUsePreMerge()) { if (_logger.IsDebug) _logger.Debug("Using pre merge block downloader"); @@ -92,7 +97,7 @@ public override async Task DownloadHeaders(PeerInfo? bestPeer, BlocksReque CancellationToken cancellation) { // Note: Redundant with Dispatch, but test uses it. - if (_beaconPivot.BeaconPivotExists() == false && _poSSwitcher.HasEverReachedTerminalBlock() == false) + if (ShouldUsePreMerge()) { if (_logger.IsDebug) _logger.Debug("Using pre merge block downloader"); @@ -123,7 +128,7 @@ public override async Task DownloadHeaders(PeerInfo? bestPeer, BlocksReque // Alternatively we can do this in BeaconHeadersSyncFeed, but this seems easier. ValidateSeals(headers!, cancellation); - return Task.FromResult?>(headers.ToPooledList(0)); + return Task.FromResult?>(headers.ToPooledList(headers.Length)); } protected override bool CheckAncestorJump(PeerInfo? bestPeer, Hash256? startHeaderHash, ref long currentNumber) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/PosTransitionHook.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/PosTransitionHook.cs index 6ccf438e42b..d51b9d4599e 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/PosTransitionHook.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/PosTransitionHook.cs @@ -17,12 +17,13 @@ public class PosTransitionHook(IBlockTree blockTree, IPoSSwitcher poSSwitcher, I { private readonly ILogger _logger = logManager.GetClassLogger(); - public void TryUpdateTerminalBlock(BlockHeader currentHeader, bool shouldProcess) + public void TryUpdateTerminalBlock(BlockHeader currentHeader) { - if (!shouldProcess) // if we're processing the block we will find TerminalBlock after processing - poSSwitcher.TryUpdateTerminalBlock(currentHeader); + poSSwitcher.TryUpdateTerminalBlock(currentHeader); } + // Used only in get block header in pre merge block downloader, this hook stops pre merge block downloader + // from downloader header once poSSwitcher.HasEverReachedTerminalBlock(). public bool ImprovementRequirementSatisfied(PeerInfo? bestPeer) { return bestPeer!.TotalDifficulty > (blockTree.BestSuggestedHeader?.TotalDifficulty ?? 0) && @@ -39,9 +40,11 @@ public IOwnedReadOnlyList FilterPosHeader(IOwnedReadOnlyList oldResponse = response; response = response .TakeWhile(header => !poSSwitcher.GetBlockConsensusInfo(header).IsPostMerge) .ToPooledList(response.Count); + oldResponse.Dispose(); if (_logger.IsInfo) _logger.Info($"Last block is post merge. {lastBlockHeader.Hash}. Trimming to {response.Count} sized batch."); } } diff --git a/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloader.cs b/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloader.cs index d2bd3e330da..ca5bdc28ffa 100644 --- a/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloader.cs +++ b/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloader.cs @@ -183,6 +183,7 @@ public virtual async Task DownloadHeaders(PeerInfo? bestPeer, BlocksReques } _syncPeerPool.ReportNoSyncProgress(bestPeer, AllocationContexts.Blocks); + headers.Dispose(); return 0; } @@ -197,7 +198,7 @@ public virtual async Task DownloadHeaders(PeerInfo? bestPeer, BlocksReques // loop iterator to start with o if (HandleAddResult(bestPeer, currentHeader, i == 0, _blockTree.Insert(currentHeader))) { - _posTransitionHook.TryUpdateTerminalBlock(currentHeader, false); + _posTransitionHook.TryUpdateTerminalBlock(currentHeader); headersSynced++; } @@ -213,8 +214,11 @@ public virtual async Task DownloadHeaders(PeerInfo? bestPeer, BlocksReques { break; } + + headers.Dispose(); } + headers?.Dispose(); return headersSynced; } @@ -304,8 +308,11 @@ public virtual async Task DownloadBlocks(PeerInfo? bestPeer, BlocksRequest { break; } + + headers?.Dispose(); } + headers?.Dispose(); return blocksSynced; } @@ -406,9 +413,11 @@ private bool SuggestBlock( if (shouldProcess == false) { _blockTree.UpdateMainChain(new[] { currentBlock }, false); + // Needed to know if a block is the terminal block. + // Not needed if not processing for some reason. + _posTransitionHook.TryUpdateTerminalBlock(currentBlock.Header); } - _posTransitionHook.TryUpdateTerminalBlock(currentBlock.Header, shouldProcess); if (downloadReceipts) { TxReceipt[]? contextReceiptsForBlock = receipts![blockIndex]; diff --git a/src/Nethermind/Nethermind.Synchronization/Blocks/IPosTransitionHook.cs b/src/Nethermind/Nethermind.Synchronization/Blocks/IPosTransitionHook.cs index 6a7e9603868..4d5828f153d 100644 --- a/src/Nethermind/Nethermind.Synchronization/Blocks/IPosTransitionHook.cs +++ b/src/Nethermind/Nethermind.Synchronization/Blocks/IPosTransitionHook.cs @@ -9,7 +9,10 @@ namespace Nethermind.Synchronization.Blocks; public interface IPosTransitionHook { - void TryUpdateTerminalBlock(BlockHeader currentHeader, bool shouldProcess); + + // Needed to know what is the terminal block so in fast sync, for each + // header, it calls this. + void TryUpdateTerminalBlock(BlockHeader currentHeader); bool ImprovementRequirementSatisfied(PeerInfo? peerInfo); IOwnedReadOnlyList FilterPosHeader(IOwnedReadOnlyList headers); } diff --git a/src/Nethermind/Nethermind.Synchronization/Blocks/NoPosTransition.cs b/src/Nethermind/Nethermind.Synchronization/Blocks/NoPosTransition.cs index 1228fbdc083..f91eabb8c16 100644 --- a/src/Nethermind/Nethermind.Synchronization/Blocks/NoPosTransition.cs +++ b/src/Nethermind/Nethermind.Synchronization/Blocks/NoPosTransition.cs @@ -10,7 +10,7 @@ namespace Nethermind.Synchronization.Blocks; public class NoPosTransition(IBlockTree blockTree) : IPosTransitionHook { - public void TryUpdateTerminalBlock(BlockHeader currentHeader, bool shouldProcess) + public void TryUpdateTerminalBlock(BlockHeader currentHeader) { } From 8b2b30d20fe25ec94817a71316c9620716da77e1 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 12 Feb 2025 22:07:16 +0800 Subject: [PATCH 10/10] Using using --- .../Synchronization/PosTransitionHook.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/PosTransitionHook.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/PosTransitionHook.cs index d51b9d4599e..4b67dd16a38 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/PosTransitionHook.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/PosTransitionHook.cs @@ -40,11 +40,10 @@ public IOwnedReadOnlyList FilterPosHeader(IOwnedReadOnlyList oldResponse = response; + using IOwnedReadOnlyList oldResponse = response; response = response .TakeWhile(header => !poSSwitcher.GetBlockConsensusInfo(header).IsPostMerge) .ToPooledList(response.Count); - oldResponse.Dispose(); if (_logger.IsInfo) _logger.Info($"Last block is post merge. {lastBlockHeader.Hash}. Trimming to {response.Count} sized batch."); } }