Merge remote-tracking branch 'origin/unstable' into verify-randao

.github/workflows/ci.yml

@@ -209,22 +209,13 @@ jobs:
         run: |
           scripts/setup_scenarios.sh fixturesCache
 
-      - name: Smoke test the Beacon Node and Validator Client with all tracing enabled
+      - name: Build binaries (with trace logging enabled)
         run: |
           if [[ "${{ matrix.branch }}" == "version-1-6" ]]; then
             # change to "|| true" to hide the CI failures in GitHub's UI (escape hatch if a blocker is detected in 1.6)
-            ${make_cmd} -j ${ncpu} V=1 NIM_COMMIT=${{ matrix.branch }} LOG_LEVEL=TRACE nimbus_beacon_node nimbus_validator_client || false
+            ${make_cmd} -j ${ncpu} V=1 NIM_COMMIT=${{ matrix.branch }} LOG_LEVEL=TRACE || false
           else
-            ${make_cmd} -j ${ncpu} V=1 NIM_COMMIT=${{ matrix.branch }} LOG_LEVEL=TRACE nimbus_beacon_node nimbus_validator_client
-          fi
-
-      - name: Build all tools
-        run: |
-          if [[ "${{ matrix.branch }}" == "version-1-6" ]]; then
-            # change to "|| true" to hide the CI failures in GitHub's UI (escape hatch if a blocker is detected in 1.6)
-            ${make_cmd} -j ${ncpu} V=1 NIM_COMMIT=${{ matrix.branch }} || false
-          else
-            ${make_cmd} -j ${ncpu} V=1 NIM_COMMIT=${{ matrix.branch }}
+            ${make_cmd} -j ${ncpu} V=1 NIM_COMMIT=${{ matrix.branch }} LOG_LEVEL=TRACE
           fi
           # The Windows image runs out of disk space, so make some room
           rm -rf nimcache

CHANGELOG.md

@@ -1,3 +1,46 @@
+2022-09-07 v22.9.0
+==================
+
+Nimbus `v22.9.0` is a `high-urgency` upgrade that fixes a critical pre-TTD block production issue affecting users that restarted their node after Bellatrix. It also improves compatibility with Besu, Prysm and slow block builders and provides a speed boost in block processing important for those running on Raspberry Pi and similar hardware.
+
+With the merge drawing near, the focus of this release has been to include low risk changes that improve stability and compatibility - if you are unsure whether to upgrade, do reach out to us in discord to discuss your particular deployment.
+
+A shout out to our great community for reporting and helping diagnose the issues that led up to this release - in particular Michael Sproul (Lighthouse) and Joe Clapis (Rocket Pool).
+
+### Improvements
+
+* Allow more time for block builder to deliver block
+  [#4088](https://github.com/status-im/nimbus-eth2/pull/4088)
+
+* Improve Bellatrix block processing performance
+  [#4085](https://github.com/status-im/nimbus-eth2/pull/4085) and [#4082](https://github.com/status-im/nimbus-eth2/pull/4082)
+
+* Optimize execution layer calls when not producing blocks, improving Besu performance and compatiblity
+  [#4055](https://github.com/status-im/nimbus-eth2/pull/4055)
+
+* Revise timing of execution layer configuration call, resolving warnings that no consensus client is present on Geth and Besu
+  [#4077](https://github.com/status-im/nimbus-eth2/pull/4077)
+
+* Log `Exchanged engine configuration` when first connected to correctly configured execution engine
+  [#4096](https://github.com/status-im/nimbus-eth2/pull/4096)
+
+* Switch to `nim-websock` for websocket connections, resolving delays when payloads exceed 1mb
+  [#4061](https://github.com/status-im/nimbus-eth2/pull/4061)
+
+### Fixes
+
+* Fix pre-TTD block proposals on nodes that (re-)started after Bellatrix
+  [#4094](https://github.com/status-im/nimbus-eth2/issues/4094)
+
+* Fix gossip message id, improving connectivity health with Prysm
+  [#4076](https://github.com/status-im/nimbus-eth2/pull/4076)
+
+* Improve handling of blocks deemed invalid by the execution layer
+  [#4081](https://github.com/status-im/nimbus-eth2/pull/4081)
+
+* Fix a rare crash that could happen when execution layer disconnected
+  [#4095](https://github.com/status-im/nimbus-eth2/pull/4095)
+
 2022-08-31 v22.8.2
 ==================
 

Makefile

@@ -129,7 +129,7 @@ DEPOSITS_DELAY := 0
 #- "--define:release" cannot be added to "config.nims"
 #- disable Nim's default parallelisation because it starts too many processes for too little gain
 #- https://github.com/status-im/nim-libp2p#use-identify-metrics
-NIM_PARAMS += -d:release --parallelBuild:1 -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,prysm,teku
+NIM_PARAMS += -d:release --parallelBuild:1 -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,lodestar,prysm,teku
 
 ifeq ($(USE_LIBBACKTRACE), 0)
 NIM_PARAMS += -d:disable_libbacktrace

README.md

@@ -1,6 +1,6 @@
 # Nimbus Eth2 (Beacon Chain)
 
-[![Github Actions CI](https://github.com/status-im/nimbus-eth2/actions/workflows/ci.yml/badge.svg)](https://github.com/status-im/nimbus-eth2/actions/workflows/ci.yml)
+[![Github Actions CI](https://github.com/status-im/nimbus-eth2/actions/workflows/ci.yml/badge.svg?branch=stable)](https://github.com/status-im/nimbus-eth2/actions/workflows/ci.yml?query=branch%3Astable)
 [![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
 ![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg)

beacon_chain/beacon_chain_db_immutable.nim

@@ -123,7 +123,7 @@ type
     current_sync_committee*: SyncCommittee     # [New in Altair]
     next_sync_committee*: SyncCommittee        # [New in Altair]
 
-  # https://github.com/ethereum/consensus-specs/blob/v1.2.0-rc.1/specs/bellatrix/beacon-chain.md#beaconstate
+  # https://github.com/ethereum/consensus-specs/blob/v1.2.0-rc.3/specs/bellatrix/beacon-chain.md#beaconstate
   # Memory-representation-equivalent to a Bellatrix BeaconState for in-place SSZ
   # reading and writing
   BellatrixBeaconStateNoImmutableValidators* = object

beacon_chain/beacon_node.nim

@@ -77,7 +77,6 @@ type
     syncManager*: SyncManager[Peer, PeerId]
     backfiller*: SyncManager[Peer, PeerId]
     genesisSnapshotContent*: string
-    actionTracker*: ActionTracker
     processor*: ref Eth2Processor
     blockProcessor*: ref BlockProcessor
     consensusManager*: ref ConsensusManager
@@ -91,6 +90,8 @@ type
     nextExchangeTransitionConfTime*: Moment
     router*: ref MessageRouter
     dynamicFeeRecipientsStore*: ref DynamicFeeRecipientsStore
+    externalBuilderRegistrations*:
+      Table[ValidatorPubKey, SignedValidatorRegistrationV1]
 
 const
   MaxEmptySlotCount* = uint64(10*60) div SECONDS_PER_SLOT

beacon_chain/consensus_object_pools/attestation_pool.nim

@@ -145,10 +145,10 @@ proc init*(T: type AttestationPool, dag: ChainDAGRef,
           if enableTestFeatures in dag.updateFlags and blckRef == dag.head:
             unrealized = withState(dag.headState):
               when stateFork >= BeaconStateFork.Altair:
-                state.data.compute_unrealized_finality()
+                forkyState.data.compute_unrealized_finality()
               else:
                 var cache: StateCache
-                state.data.compute_unrealized_finality(cache)
+                forkyState.data.compute_unrealized_finality(cache)
           withBlck(blck):
             forkChoice.process_block(
               dag, epochRef, blckRef, unrealized, blck.message,
@@ -536,7 +536,7 @@ proc getAttestationsForBlock*(pool: var AttestationPool,
                               cache: var StateCache): seq[Attestation] =
   ## Retrieve attestations that may be added to a new block at the slot of the
   ## given state
-  ## https://github.com/ethereum/consensus-specs/blob/v1.2.0-rc.1/specs/phase0/validator.md#attestations
+  ## https://github.com/ethereum/consensus-specs/blob/v1.2.0-rc.3/specs/phase0/validator.md#attestations
   let newBlockSlot = state.data.slot.uint64
 
   if newBlockSlot < MIN_ATTESTATION_INCLUSION_DELAY:

beacon_chain/consensus_object_pools/block_clearance.nim

@@ -105,9 +105,9 @@ proc addResolvedHeadBlock(
     if enableTestFeatures in dag.updateFlags:
       unrealized = withState(state):
         when stateFork >= BeaconStateFork.Altair:
-          state.data.compute_unrealized_finality()
+          forkyState.data.compute_unrealized_finality()
         else:
-          state.data.compute_unrealized_finality(cache)
+          forkyState.data.compute_unrealized_finality(cache)
     onBlockAdded(blockRef, trustedBlock, epochRef, unrealized)
   if not(isNil(dag.onBlockAdded)):
     dag.onBlockAdded(ForkedTrustedSignedBeaconBlock.init(trustedBlock))

beacon_chain/consensus_object_pools/block_pools_types.nim

@@ -233,7 +233,7 @@ type
       ## committee messages will be rejected
 
     optimisticRoots*: HashSet[Eth2Digest]
-      ## https://github.com/ethereum/consensus-specs/blob/v1.2.0-rc.1/sync/optimistic.md#helpers
+      ## https://github.com/ethereum/consensus-specs/blob/v1.2.0-rc.3/sync/optimistic.md#helpers
 
   EpochKey* = object
     ## The epoch key fully determines the shuffling for proposers and

beacon_chain/consensus_object_pools/blockchain_dag.nim

@@ -965,7 +965,7 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB,
     cfg, getStateField(dag.headState, genesis_validators_root))
 
   withState(dag.headState):
-    dag.validatorMonitor[].registerState(state.data)
+    dag.validatorMonitor[].registerState(forkyState.data)
 
   updateBeaconMetrics(dag.headState, dag.head.bid, cache)
 
@@ -1024,7 +1024,7 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB,
 
   withState(dag.headState):
     when stateFork >= BeaconStateFork.Altair:
-      dag.headSyncCommittees = state.data.get_sync_committee_cache(cache)
+      dag.headSyncCommittees = forkyState.data.get_sync_committee_cache(cache)
 
   info "Block DAG initialized",
     head = shortLog(dag.head),
@@ -1044,7 +1044,7 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB,
   withState(dag.headState):
     when stateFork >= BeaconStateFork.Bellatrix:
       template executionPayloadHeader(): auto =
-        state().data.latest_execution_payload_header
+        forkyState().data.latest_execution_payload_header
       const emptyExecutionPayloadHeader =
         default(type(executionPayloadHeader))
       if executionPayloadHeader != emptyExecutionPayloadHeader: