Fix consensus, SSZ, tree hash & run merge EF tests

.dockerignore

@@ -1,4 +1,4 @@
-testing/ef_tests/eth2.0-spec-tests
+testing/ef_tests/consensus-spec-tests
 target/
 *.data
 *.tar.gz

.github/workflows/test-suite.yml

@@ -98,7 +98,7 @@ jobs:
     - uses: actions/checkout@v1
     - name: Get latest version of stable Rust
       run: rustup update stable
-    - name: Run eth2.0-spec-tests with blst, milagro and fake_crypto
+    - name: Run consensus-spec-tests with blst, milagro and fake_crypto
       run: make test-ef
   dockerfile-ubuntu:
     name: dockerfile-ubuntu

Makefile

@@ -105,7 +105,7 @@ run-ef-tests:
 	cargo test --release --manifest-path=$(EF_TESTS)/Cargo.toml --features "ef_tests"
 	cargo test --release --manifest-path=$(EF_TESTS)/Cargo.toml --features "ef_tests,fake_crypto"
 	cargo test --release --manifest-path=$(EF_TESTS)/Cargo.toml --features "ef_tests,milagro"
-	./$(EF_TESTS)/check_all_files_accessed.py $(EF_TESTS)/.accessed_file_log.txt $(EF_TESTS)/eth2.0-spec-tests
+	./$(EF_TESTS)/check_all_files_accessed.py $(EF_TESTS)/.accessed_file_log.txt $(EF_TESTS)/consensus-spec-tests
 
 # Run the tests in the `beacon_chain` crate.
 test-beacon-chain: test-beacon-chain-base test-beacon-chain-altair

beacon_node/store/src/partial_beacon_state.rs

@@ -197,12 +197,17 @@ impl<T: EthSpec> PartialBeaconState<T> {
         let epoch = slot.epoch(T::slots_per_epoch());
 
         if spec
+            .merge_fork_epoch
+            .map_or(false, |merge_epoch| epoch >= merge_epoch)
+        {
+            PartialBeaconStateMerge::from_ssz_bytes(bytes).map(Self::Merge)
+        } else if spec
             .altair_fork_epoch
-            .map_or(true, |altair_epoch| epoch < altair_epoch)
+            .map_or(false, |altair_epoch| epoch >= altair_epoch)
         {
-            PartialBeaconStateBase::from_ssz_bytes(bytes).map(Self::Base)
-        } else {
             PartialBeaconStateAltair::from_ssz_bytes(bytes).map(Self::Altair)
+        } else {
+            PartialBeaconStateBase::from_ssz_bytes(bytes).map(Self::Base)
         }
     }
 

book/src/setup.md

@@ -36,15 +36,15 @@ you can run them locally and avoid CI failures:
 
 _The lighthouse test suite is quite extensive, running the whole suite may take 30+ minutes._
 
-### Ethereum 2.0 Spec Tests
+### Consensus Spec Tests
 
 The
-[ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests/)
+[ethereum/consensus-spec-tests](https://github.com/ethereum/consensus-spec-tests/)
 repository contains a large set of tests that verify Lighthouse behaviour
 against the Ethereum Foundation specifications.
 
 These tests are quite large (100's of MB) so they're only downloaded if you run
-`$ make test-ef` (or anything that run it). You may want to avoid
+`$ make test-ef` (or anything that runs it). You may want to avoid
 downloading these tests if you're on a slow or metered Internet connection. CI
 will require them to pass, though.
 

consensus/ssz_types/src/serde_utils/hex_fixed_vec.rs

@@ -0,0 +1,25 @@
+use crate::FixedVector;
+use eth2_serde_utils::hex::{self, PrefixedHexVisitor};
+use serde::{Deserializer, Serializer};
+use typenum::Unsigned;
+
+pub fn serialize<S, U>(bytes: &FixedVector<u8, U>, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+    U: Unsigned,
+{
+    let mut hex_string: String = "0x".to_string();
+    hex_string.push_str(&hex::encode(&bytes[..]));
+
+    serializer.serialize_str(&hex_string)
+}
+
+pub fn deserialize<'de, D, U>(deserializer: D) -> Result<FixedVector<u8, U>, D::Error>
+where
+    D: Deserializer<'de>,
+    U: Unsigned,
+{
+    let vec = deserializer.deserialize_string(PrefixedHexVisitor)?;
+    FixedVector::new(vec)
+        .map_err(|e| serde::de::Error::custom(format!("invalid fixed vector: {:?}", e)))
+}

consensus/ssz_types/src/serde_utils/hex_var_list.rs

@@ -0,0 +1,26 @@
+//! Serialize `VariableList<u8, N>` as 0x-prefixed hex string.
+use crate::VariableList;
+use eth2_serde_utils::hex::{self, PrefixedHexVisitor};
+use serde::{Deserializer, Serializer};
+use typenum::Unsigned;
+
+pub fn serialize<S, N>(bytes: &VariableList<u8, N>, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+    N: Unsigned,
+{
+    let mut hex_string: String = "0x".to_string();
+    hex_string.push_str(&hex::encode(&**bytes));
+
+    serializer.serialize_str(&hex_string)
+}
+
+pub fn deserialize<'de, D, N>(deserializer: D) -> Result<VariableList<u8, N>, D::Error>
+where
+    D: Deserializer<'de>,
+    N: Unsigned,
+{
+    let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?;
+    VariableList::new(bytes)
+        .map_err(|e| serde::de::Error::custom(format!("invalid variable list: {:?}", e)))
+}

consensus/ssz_types/src/serde_utils/mod.rs

@@ -1,2 +1,4 @@
+pub mod hex_fixed_vec;
+pub mod hex_var_list;
 pub mod quoted_u64_fixed_vec;
 pub mod quoted_u64_var_list;

consensus/state_processing/src/per_block_processing.rs

@@ -131,18 +131,22 @@ pub fn per_block_processing<T: EthSpec>(
     process_eth1_data(state, block.body().eth1_data())?;
     process_operations(state, block.body(), proposer_index, verify_signatures, spec)?;
 
-    if let BeaconBlockRef::Altair(inner) = block {
+    if let Some(sync_aggregate) = block.body().sync_aggregate() {
         process_sync_aggregate(
             state,
-            &inner.body.sync_aggregate,
+            sync_aggregate,
             proposer_index,
             verify_signatures,
             spec,
         )?;
     }
 
     if is_execution_enabled(state, block.body()) {
-        process_execution_payload(state, block.body().execution_payload().unwrap(), spec)?
+        let payload = block
+            .body()
+            .execution_payload()
+            .ok_or(BlockProcessingError::IncorrectStateType)?;
+        process_execution_payload(state, payload, spec)?;
     }
 
     Ok(())

consensus/state_processing/src/per_block_processing/errors.rs

@@ -77,6 +77,7 @@ pub enum BlockProcessingError {
         expected: u64,
         found: u64,
     },
+    ExecutionInvalid,
 }
 
 impl From<BeaconStateError> for BlockProcessingError {

consensus/state_processing/src/per_block_processing/process_operations.rs

@@ -353,15 +353,15 @@ pub fn process_deposit<T: EthSpec>(
         state.validators_mut().push(validator)?;
         state.balances_mut().push(deposit.data.amount)?;
 
-        // Altair-specific initializations.
-        if let BeaconState::Altair(altair_state) = state {
-            altair_state
-                .previous_epoch_participation
-                .push(ParticipationFlags::default())?;
-            altair_state
-                .current_epoch_participation
-                .push(ParticipationFlags::default())?;
-            altair_state.inactivity_scores.push(0)?;
+        // Altair or later initializations.
+        if let Ok(previous_epoch_participation) = state.previous_epoch_participation_mut() {
+            previous_epoch_participation.push(ParticipationFlags::default())?;
+        }
+        if let Ok(current_epoch_participation) = state.current_epoch_participation_mut() {
+            current_epoch_participation.push(ParticipationFlags::default())?;
+        }
+        if let Ok(inactivity_scores) = state.inactivity_scores_mut() {
+            inactivity_scores.push(0)?;
         }
     }
 

consensus/types/src/beacon_block.rs

@@ -89,12 +89,17 @@ impl<T: EthSpec> BeaconBlock<T> {
         let epoch = slot.epoch(T::slots_per_epoch());
 
         if spec
+            .merge_fork_epoch
+            .map_or(false, |merge_epoch| epoch >= merge_epoch)
+        {
+            BeaconBlockMerge::from_ssz_bytes(bytes).map(Self::Merge)
+        } else if spec
             .altair_fork_epoch
-            .map_or(true, |altair_epoch| epoch < altair_epoch)
+            .map_or(false, |altair_epoch| epoch >= altair_epoch)
         {
-            BeaconBlockBase::from_ssz_bytes(bytes).map(Self::Base)
-        } else {
             BeaconBlockAltair::from_ssz_bytes(bytes).map(Self::Altair)
+        } else {
+            BeaconBlockBase::from_ssz_bytes(bytes).map(Self::Base)
         }
     }
 
@@ -104,9 +109,13 @@ impl<T: EthSpec> BeaconBlock<T> {
     /// Usually it's better to prefer `from_ssz_bytes` which will decode the correct variant based
     /// on the fork slot.
     pub fn any_from_ssz_bytes(bytes: &[u8]) -> Result<Self, ssz::DecodeError> {
-        BeaconBlockAltair::from_ssz_bytes(bytes)
-            .map(BeaconBlock::Altair)
-            .or_else(|_| BeaconBlockBase::from_ssz_bytes(bytes).map(BeaconBlock::Base))
+        BeaconBlockMerge::from_ssz_bytes(bytes)
+            .map(BeaconBlock::Merge)
+            .or_else(|_| {
+                BeaconBlockAltair::from_ssz_bytes(bytes)
+                    .map(BeaconBlock::Altair)
+                    .or_else(|_| BeaconBlockBase::from_ssz_bytes(bytes).map(BeaconBlock::Base))
+            })
     }
 
     /// Convenience accessor for the `body` as a `BeaconBlockBodyRef`.

consensus/types/src/beacon_state.rs

@@ -418,12 +418,17 @@ impl<T: EthSpec> BeaconState<T> {
         let epoch = slot.epoch(T::slots_per_epoch());
 
         if spec
+            .merge_fork_epoch
+            .map_or(false, |merge_epoch| epoch >= merge_epoch)
+        {
+            BeaconStateMerge::from_ssz_bytes(bytes).map(Self::Merge)
+        } else if spec
             .altair_fork_epoch
-            .map_or(true, |altair_epoch| epoch < altair_epoch)
+            .map_or(false, |altair_epoch| epoch >= altair_epoch)
         {
-            BeaconStateBase::from_ssz_bytes(bytes).map(Self::Base)
-        } else {
             BeaconStateAltair::from_ssz_bytes(bytes).map(Self::Altair)
+        } else {
+            BeaconStateBase::from_ssz_bytes(bytes).map(Self::Base)
         }
     }
 
@@ -1686,7 +1691,8 @@ impl<T: EthSpec> CompareFields for BeaconState<T> {
         match (self, other) {
             (BeaconState::Base(x), BeaconState::Base(y)) => x.compare_fields(y),
             (BeaconState::Altair(x), BeaconState::Altair(y)) => x.compare_fields(y),
-            _ => panic!("compare_fields: mismatched state variants"),
+            (BeaconState::Merge(x), BeaconState::Merge(y)) => x.compare_fields(y),
+            _ => panic!("compare_fields: mismatched state variants",),
         }
     }
 }

consensus/types/src/beacon_state/tree_hash_cache.rs

@@ -328,16 +328,26 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
         )?;
         hasher.write(state.finalized_checkpoint().tree_hash_root().as_bytes())?;
 
-        // Inactivity & light-client sync committees
-        if let BeaconState::Altair(ref state) = state {
+        // Inactivity & light-client sync committees (Altair and later).
+        if let Ok(inactivity_scores) = state.inactivity_scores() {
             hasher.write(
                 self.inactivity_scores
-                    .recalculate_tree_hash_root(&state.inactivity_scores)?
+                    .recalculate_tree_hash_root(inactivity_scores)?
                     .as_bytes(),
             )?;
+        }
+
+        if let Ok(current_sync_committee) = state.current_sync_committee() {
+            hasher.write(current_sync_committee.tree_hash_root().as_bytes())?;
+        }
+
+        if let Ok(next_sync_committee) = state.next_sync_committee() {
+            hasher.write(next_sync_committee.tree_hash_root().as_bytes())?;
+        }
 
-            hasher.write(state.current_sync_committee.tree_hash_root().as_bytes())?;
-            hasher.write(state.next_sync_committee.tree_hash_root().as_bytes())?;
+        // Execution payload (merge and later).
+        if let Ok(payload_header) = state.latest_execution_payload_header() {
+            hasher.write(payload_header.tree_hash_root().as_bytes())?;
         }
 
         let root = hasher.finish()?;

consensus/types/src/chain_spec.rs

@@ -506,6 +506,7 @@ impl ChainSpec {
         Self {
             max_committees_per_slot: 4,
             target_committee_size: 4,
+            churn_limit_quotient: 32,
             shuffle_round_count: 10,
             min_genesis_active_validator_count: 64,
             min_genesis_time: 1578009600,