Merge branch 'master' into deep/refactor-pivotupdator

.github/workflows/hive-tests.yml

@@ -40,6 +40,15 @@ on:
         required: false
         default: master
         type: string
+      buildarg:
+        description: Simulator build argument (e.g. fixtures URL)
+        required: false
+        type: string
+      enable_buildarg:
+        description: Enable buildarg flag
+        required: true
+        default: false
+        type: boolean
 
 jobs:
   test:
@@ -53,6 +62,8 @@ jobs:
           echo "LOG_LEVEL=${{ github.event.inputs.log-level || '3' }}" >> $GITHUB_ENV
           echo "HIVE_REPO=${{ github.event.inputs.hive-repo || 'ethereum/hive' }}" >> $GITHUB_ENV
           echo "HIVE_BRANCH=${{ github.event.inputs.hive-branch || 'master' }}" >> $GITHUB_ENV
+          echo "BUILDARG=${{ github.event.inputs.buildarg || '' }}" >> $GITHUB_ENV
+          echo "ENABLE_BUILDARG=${{ github.event.inputs.enable_buildarg || false }}" >> $GITHUB_ENV
       - name: Check out Nethermind repository
         uses: actions/checkout@v4
         with:
@@ -80,15 +91,22 @@ jobs:
       - name: Load Docker image
         run: docker load --input /tmp/image.tar
       - name: Run Hive
-        if: env.LIMIT == ''
         continue-on-error: true
         working-directory: hive
-        run: ./hive --client nethermind --sim $TEST_SUITE --sim.loglevel $LOG_LEVEL
-      - name: Run Hive with limit
-        if: env.LIMIT != ''
-        continue-on-error: true
-        working-directory: hive
-        run: ./hive --client nethermind --sim $TEST_SUITE --sim.loglevel $LOG_LEVEL --sim.limit $LIMIT
+        run: |
+          HIVE_ARGS="--client nethermind --sim $TEST_SUITE --sim.loglevel $LOG_LEVEL"
+          
+          # Add optional arguments based on conditions
+          if [ -n "$LIMIT" ]; then
+            HIVE_ARGS="$HIVE_ARGS --sim.limit $LIMIT"
+          fi
+          
+          if [ "$ENABLE_BUILDARG" = "true" ] && [ -n "$BUILDARG" ]; then
+            HIVE_ARGS="$HIVE_ARGS --sim.buildarg \"$BUILDARG\""
+          fi
+          
+          # Execute hive with constructed arguments
+          eval "./hive $HIVE_ARGS"
       - name: Upload results
         uses: actions/upload-artifact@v4
         with:

.github/workflows/sync-supported-chains.yml

@@ -35,6 +35,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           path: nethermind
+          ref: ${{ github.event.workflow_run.head_branch || github.ref_name || 'master' }}
       - name: Set Matrix
         id: set-matrix
         run: |
@@ -100,6 +101,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           clean: true
+          ref: ${{ github.event.workflow_run.head_branch || github.ref_name || 'master' }}
 
       - name: Checkout tests repository
         uses: actions/checkout@v4
@@ -142,9 +144,10 @@ jobs:
           network="${{ matrix.config.network }}"
           
           if [ -z "${{ inputs.nethermind_image }}" ]; then
-            REF_NAME=${{ github.ref }}
-            if [[ $REF_NAME == refs/heads/release/* ]]; then
-              CLEAN_REF=$(echo "${REF_NAME/refs\/heads\//}" | sed 's/[^a-zA-Z0-9._-]/-/g')
+            REF_NAME=${{ github.event.workflow_run.head_branch || github.ref_name || 'master' }}
+            CLEAN_REF=$(echo "${REF_NAME/refs\/heads\//}")
+            if [[ $CLEAN_REF == release/* ]]; then
+              CLEAN_REF=$(echo "$CLEAN_REF" | sed 's/[^a-zA-Z0-9._-]/-/g')
               docker_image="nethermindeth/nethermind:$CLEAN_REF"
             else
               docker_image="nethermindeth/nethermind:master"
@@ -183,7 +186,7 @@ jobs:
             -p $GITHUB_WORKSPACE/sedge \
             op-full-node \
             --op-execution opnethermind:$docker_image \
-            --op-image op-node:us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:latest \
+            --op-image op-node:us-docker.pkg.dev/oplabs-tools-artifacts/images/op-node:v1.10.3 \
             --map-all \
             --network $stripped_network \
             --consensus-url $CONSENSUS_URL \
@@ -365,5 +368,5 @@ jobs:
           ref: "main"
           token: "${{ steps.gh-app.outputs.token }}"
           inputs: '{
-            "nethermind_branch": "${{ github.ref}}"
+            "nethermind_branch": "${{ github.event.workflow_run.head_branch }}"
           }'

src/Nethermind/Directory.Build.props

@@ -15,7 +15,7 @@
     <Copyright>Demerzel Solutions Limited</Copyright>
     <Product>Nethermind</Product>
     <SourceRevisionId Condition="'$(Commit)' != ''">$(Commit)</SourceRevisionId>
-    <VersionPrefix>1.31.0</VersionPrefix>
+    <VersionPrefix>1.32.0</VersionPrefix>
     <VersionSuffix>unstable</VersionSuffix>
   </PropertyGroup>
 

src/Nethermind/Ethereum.Test.Base/BlockchainTest.cs

@@ -28,6 +28,7 @@ public class BlockchainTest : IEthereumTest
         public Hash256? PostStateRoot { get; set; }
         public bool SealEngineUsed { get; set; }
         public string? LoadFailure { get; set; }
+        public ulong ChainId { get; set; }
 
         public override string? ToString()
         {

src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs

@@ -27,6 +27,7 @@
 using Nethermind.Int256;
 using Nethermind.Logging;
 using Nethermind.Serialization.Rlp;
+using Nethermind.Specs;
 using Nethermind.Specs.Forks;
 using Nethermind.Specs.Test;
 using Nethermind.State;
@@ -77,22 +78,18 @@ protected async Task<EthereumTestResult> RunTest(BlockchainTest test, Stopwatch?
             TestContext.Out.WriteLine($"Network after transition: [{test.NetworkAfterTransition.Name}] at {test.TransitionForkActivation}");
         Assert.That(test.LoadFailure, Is.Null, "test data loading failure");
 
-        ISpecProvider specProvider;
+        List<(ForkActivation Activation, IReleaseSpec Spec)> transitions =
+            [((ForkActivation)0, Frontier.Instance), ((ForkActivation)1, test.Network)]; // TODO: this thing took a lot of time to find after it was removed!, genesis block is always initialized with Frontier
         if (test.NetworkAfterTransition is not null)
         {
-            specProvider = new CustomSpecProvider(
-                ((ForkActivation)0, Frontier.Instance),
-                ((ForkActivation)1, test.Network),
-                (test.TransitionForkActivation!.Value, test.NetworkAfterTransition));
-        }
-        else
-        {
-            specProvider = new CustomSpecProvider(
-                ((ForkActivation)0, Frontier.Instance), // TODO: this thing took a lot of time to find after it was removed!, genesis block is always initialized with Frontier
-                ((ForkActivation)1, test.Network));
+            transitions.Add((test.TransitionForkActivation!.Value, test.NetworkAfterTransition));
         }
 
-        if (specProvider.GenesisSpec != Frontier.Instance)
+        ISpecProvider specProvider = test.ChainId == GnosisSpecProvider.Instance.ChainId
+            ? GnosisSpecProvider.Instance
+            : new CustomSpecProvider(transitions.ToArray());
+
+        if (test.ChainId != GnosisSpecProvider.Instance.ChainId && specProvider.GenesisSpec != Frontier.Instance)
         {
             Assert.Fail("Expected genesis spec to be Frontier for blockchain tests");
         }

src/Nethermind/Ethereum.Test.Base/GeneralStateTest.cs

@@ -36,6 +36,7 @@ public class GeneralStateTest : IEthereumTest
         public ulong? CurrentExcessBlobGas { get; set; }
         public UInt256? ParentBlobGasUsed { get; set; }
         public UInt256? ParentExcessBlobGas { get; set; }
+        public ulong ChainId { get; set; }
 
         public Hash256? RequestsHash { get; set; }
 

src/Nethermind/Ethereum.Test.Base/GeneralTestBase.cs

@@ -62,11 +62,13 @@ protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer)
             TestContext.Out.Write($"Running {test.Name} at {DateTime.UtcNow:HH:mm:ss.ffffff}");
             Assert.That(test.LoadFailure, Is.Null, "test data loading failure");
 
-            ISpecProvider specProvider = new CustomSpecProvider(
-                ((ForkActivation)0, Frontier.Instance), // TODO: this thing took a lot of time to find after it was removed!, genesis block is always initialized with Frontier
-                ((ForkActivation)1, test.Fork));
+            ISpecProvider specProvider = test.ChainId == GnosisSpecProvider.Instance.ChainId
+                ? GnosisSpecProvider.Instance
+                : new CustomSpecProvider(
+                    ((ForkActivation)0, Frontier.Instance), // TODO: this thing took a lot of time to find after it was removed!, genesis block is always initialized with Frontier
+                    ((ForkActivation)1, test.Fork));
 
-            if (specProvider.GenesisSpec != Frontier.Instance)
+            if (test.ChainId != GnosisSpecProvider.Instance.ChainId && specProvider.GenesisSpec != Frontier.Instance)
             {
                 Assert.Fail("Expected genesis spec to be Frontier for blockchain tests");
             }

src/Nethermind/Nethermind.Api/IInitConfig.cs

@@ -89,6 +89,9 @@ public interface IInitConfig : IConfig
 
     [ConfigItem(Description = "[TECHNICAL] Specify concurrency limit for background task.", DefaultValue = "1", HiddenFromDocs = true)]
     int BackgroundTaskConcurrency { get; set; }
+
+    [ConfigItem(Description = "[TECHNICAL] Specify max number of background task.", DefaultValue = "65536", HiddenFromDocs = true)]
+    int BackgroundTaskMaxNumber { get; set; }
 }
 
 public enum DiagnosticMode

src/Nethermind/Nethermind.Api/InitConfig.cs

@@ -36,6 +36,7 @@ public class InitConfig : IInitConfig
         public INodeStorage.KeyScheme StateDbKeyScheme { get; set; } = INodeStorage.KeyScheme.Current;
         public long? ExitOnBlockNumber { get; set; } = null;
         public int BackgroundTaskConcurrency { get; set; } = 1;
+        public int BackgroundTaskMaxNumber { get; set; } = 65536;
 
         [Obsolete("Use DiagnosticMode with MemDb instead")]
         public bool UseMemDb

src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptsMigration.cs

@@ -7,6 +7,6 @@ namespace Nethermind.Blockchain.Receipts
 {
     public interface IReceiptsMigration
     {
-        Task<bool> Run(long blockNumber);
+        Task<bool> Run(long from, long to);
     }
 }

src/Nethermind/Nethermind.Blockchain/Utils/LastNStateRootTracker.cs

@@ -8,6 +8,7 @@
 using Nethermind.Core;
 using Nethermind.Core.Collections;
 using Nethermind.Core.Crypto;
+using Nethermind.State;
 
 namespace Nethermind.Blockchain.Utils;
 

src/Nethermind/Nethermind.Consensus.Test/Scheduler/BackgroundTaskSchedulerTests.cs

@@ -29,7 +29,7 @@ public void Setup()
     public async Task Test_task_will_execute()
     {
         TaskCompletionSource tcs = new TaskCompletionSource();
-        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, 1, LimboLogs.Instance);
+        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, 1, 65536, LimboLogs.Instance);
 
         scheduler.ScheduleTask(1, (_, token) =>
         {
@@ -43,7 +43,7 @@ public async Task Test_task_will_execute()
     [Test]
     public async Task Test_task_will_execute_concurrently_when_configured_so()
     {
-        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, 2, LimboLogs.Instance);
+        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, 2, 65536, LimboLogs.Instance);
 
         int counter = 0;
 
@@ -68,7 +68,7 @@ public async Task Test_task_will_execute_concurrently_when_configured_so()
     [Test]
     public async Task Test_task_will_cancel_on_block_processing()
     {
-        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, 2, LimboLogs.Instance);
+        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, 2, 65536, LimboLogs.Instance);
 
         bool wasCancelled = false;
 
@@ -94,7 +94,7 @@ public async Task Test_task_will_cancel_on_block_processing()
     [Test]
     public async Task Test_task_that_is_scheduled_during_block_processing_will_continue_after()
     {
-        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, 2, LimboLogs.Instance);
+        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, 2, 65536, LimboLogs.Instance);
         _blockProcessor.BlocksProcessing += Raise.EventWith(new BlocksProcessingEventArgs(null));
 
         int executionCount = 0;
@@ -117,7 +117,7 @@ public async Task Test_task_that_is_scheduled_during_block_processing_will_conti
     [Test]
     public async Task Test_task_that_is_scheduled_during_block_processing_but_deadlined_will_get_called_and_cancelled()
     {
-        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, 2, LimboLogs.Instance);
+        await using BackgroundTaskScheduler scheduler = new BackgroundTaskScheduler(_blockProcessor, 2, 65536, LimboLogs.Instance);
         _blockProcessor.BlocksProcessing += Raise.EventWith(new BlocksProcessingEventArgs(null));
 
         bool wasCancelled = false;

src/Nethermind/Nethermind.Consensus/Processing/BlockCachePreWarmer.cs

@@ -18,6 +18,7 @@
 using Nethermind.State;
 using Nethermind.Core.Eip2930;
 using Nethermind.Core.Collections;
+using Nethermind.Trie;
 
 namespace Nethermind.Consensus.Processing;
 
@@ -106,6 +107,9 @@ private void WarmupWithdrawals(ParallelOptions parallelOptions, IReleaseSpec spe
                             using IReadOnlyTxProcessingScope scope = env.Build(state.stateRoot);
                             scope.WorldState.WarmUp(state.block.Withdrawals[i].Address);
                         }
+                        catch (MissingTrieNodeException)
+                        {
+                        }
                         finally
                         {
                             state.envPool.Return(env);
@@ -276,14 +280,21 @@ private void WarmupAddresses(ParallelOptions parallelOptions, Block block)
                     Transaction tx = state.Block.Transactions[i];
                     Address? sender = tx.SenderAddress;
 
-                    if (sender is not null)
+                    try
                     {
-                        state.Scope.WorldState.WarmUp(sender);
+                        if (sender is not null)
+                        {
+                            state.Scope.WorldState.WarmUp(sender);
+                        }
+
+                        Address to = tx.To;
+                        if (to is not null)
+                        {
+                            state.Scope.WorldState.WarmUp(to);
+                        }
                     }
-                    Address to = tx.To;
-                    if (to is not null)
+                    catch (MissingTrieNodeException)
                     {
-                        state.Scope.WorldState.WarmUp(to);
                     }
 
                     return state;

src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs

@@ -505,7 +505,7 @@ void DeleteInvalidBlocks(in ProcessingBranch processingBranch, Hash256 invalidBl
                 TraceFailingBranch(
                     processingBranch,
                     options,
-                    new GethLikeBlockMemoryTracer(GethTraceOptions.Default),
+                    new GethLikeBlockMemoryTracer(new GethTraceOptions { EnableMemory = true }),
                     DumpOptions.Geth);
             }
 

src/Nethermind/Nethermind.Consensus/Scheduler/BackgroundTaskScheduler.cs

@@ -40,13 +40,14 @@ public class BackgroundTaskScheduler : IBackgroundTaskScheduler, IAsyncDisposabl
     private readonly ManualResetEvent _restartQueueSignal;
     private readonly Task<Task>[] _tasksExecutors;
 
-    public BackgroundTaskScheduler(IBlockProcessor blockProcessor, int concurrency, ILogManager logManager)
+    public BackgroundTaskScheduler(IBlockProcessor blockProcessor, int concurrency, int capacity, ILogManager logManager)
     {
         if (concurrency < 1) throw new ArgumentException("concurrency must be at least 1");
+        if (capacity < 1) throw new ArgumentException("capacity must be at least 1");
 
         _mainCancellationTokenSource = new CancellationTokenSource();
         _blockProcessorCancellationTokenSource = new CancellationTokenSource();
-        _taskQueue = Channel.CreateUnbounded<IActivity>();
+        _taskQueue = Channel.CreateBounded<IActivity>(capacity);
         _logger = logManager.GetClassLogger();
         _blockProcessor = blockProcessor;
         _restartQueueSignal = new ManualResetEvent(true);
@@ -130,6 +131,8 @@ public void ScheduleTask<TReq>(TReq request, Func<TReq, CancellationToken, Task>
             // This should never happen unless something goes very wrong.
             throw new InvalidOperationException("Unable to write to background task queue.");
         }
+
+        Evm.Metrics.NumberOfBackgroundTasksScheduled = _taskQueue.Reader.Count;
     }
 
     public async ValueTask DisposeAsync()

src/Nethermind/Nethermind.Core.Test/AutoCancelTokenSourceTests.cs

@@ -0,0 +1,39 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System.Threading;
+using FluentAssertions;
+using Nethermind.Core.Extensions;
+using Nethermind.Core.Utils;
+using NUnit.Framework;
+
+namespace Nethermind.Core.Test;
+
+public class AutoCancelTokenSourceTests
+{
+    [Test]
+    public void AutoCancelOnExitClosure()
+    {
+        CancellationToken TaskWithInnerCancellation(CancellationToken token)
+        {
+            using AutoCancelTokenSource cts = token.CreateChildTokenSource();
+            return cts.Token;
+        }
+
+        TaskWithInnerCancellation(default).IsCancellationRequested.Should().BeTrue();
+    }
+
+    [Test]
+    public void AutoCancelPropagateParentCancellation()
+    {
+        using CancellationTokenSource cts = new CancellationTokenSource();
+
+        using AutoCancelTokenSource acts = cts.Token.CreateChildTokenSource();
+
+        acts.Token.IsCancellationRequested.Should().BeFalse();
+
+        cts.Cancel();
+
+        acts.Token.IsCancellationRequested.Should().BeTrue();
+    }
+}