Fix consensus, SSZ, tree hash & run merge EF tests (#2622)

* Update to v1.1.0-beta.4 (squash of #2548)

* SSZ, cached tree hash, EF tests

beacon_node/store/src/partial_beacon_state.rs

@@ -197,12 +197,17 @@ impl<T: EthSpec> PartialBeaconState<T> {
         let epoch = slot.epoch(T::slots_per_epoch());
 
         if spec
+            .merge_fork_epoch
+            .map_or(false, |merge_epoch| epoch >= merge_epoch)
+        {
+            PartialBeaconStateMerge::from_ssz_bytes(bytes).map(Self::Merge)
+        } else if spec
             .altair_fork_epoch
-            .map_or(true, |altair_epoch| epoch < altair_epoch)
+            .map_or(false, |altair_epoch| epoch >= altair_epoch)
         {
-            PartialBeaconStateBase::from_ssz_bytes(bytes).map(Self::Base)
-        } else {
             PartialBeaconStateAltair::from_ssz_bytes(bytes).map(Self::Altair)
+        } else {
+            PartialBeaconStateBase::from_ssz_bytes(bytes).map(Self::Base)
         }
     }
 

consensus/ssz_types/src/serde_utils/hex_fixed_vec.rs

@@ -0,0 +1,25 @@
+use crate::FixedVector;
+use eth2_serde_utils::hex::{self, PrefixedHexVisitor};
+use serde::{Deserializer, Serializer};
+use typenum::Unsigned;
+
+pub fn serialize<S, U>(bytes: &FixedVector<u8, U>, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+    U: Unsigned,
+{
+    let mut hex_string: String = "0x".to_string();
+    hex_string.push_str(&hex::encode(&bytes[..]));
+
+    serializer.serialize_str(&hex_string)
+}
+
+pub fn deserialize<'de, D, U>(deserializer: D) -> Result<FixedVector<u8, U>, D::Error>
+where
+    D: Deserializer<'de>,
+    U: Unsigned,
+{
+    let vec = deserializer.deserialize_string(PrefixedHexVisitor)?;
+    FixedVector::new(vec)
+        .map_err(|e| serde::de::Error::custom(format!("invalid fixed vector: {:?}", e)))
+}

consensus/ssz_types/src/serde_utils/hex_var_list.rs

@@ -0,0 +1,26 @@
+//! Serialize `VariableList<u8, N>` as 0x-prefixed hex string.
+use crate::VariableList;
+use eth2_serde_utils::hex::{self, PrefixedHexVisitor};
+use serde::{Deserializer, Serializer};
+use typenum::Unsigned;
+
+pub fn serialize<S, N>(bytes: &VariableList<u8, N>, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+    N: Unsigned,
+{
+    let mut hex_string: String = "0x".to_string();
+    hex_string.push_str(&hex::encode(&**bytes));
+
+    serializer.serialize_str(&hex_string)
+}
+
+pub fn deserialize<'de, D, N>(deserializer: D) -> Result<VariableList<u8, N>, D::Error>
+where
+    D: Deserializer<'de>,
+    N: Unsigned,
+{
+    let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?;
+    VariableList::new(bytes)
+        .map_err(|e| serde::de::Error::custom(format!("invalid variable list: {:?}", e)))
+}

consensus/ssz_types/src/serde_utils/mod.rs

@@ -1,2 +1,4 @@
+pub mod hex_fixed_vec;
+pub mod hex_var_list;
 pub mod quoted_u64_fixed_vec;
 pub mod quoted_u64_var_list;

consensus/state_processing/src/per_block_processing.rs

@@ -131,18 +131,22 @@ pub fn per_block_processing<T: EthSpec>(
     process_eth1_data(state, block.body().eth1_data())?;
     process_operations(state, block.body(), proposer_index, verify_signatures, spec)?;
 
-    if let BeaconBlockRef::Altair(inner) = block {
+    if let Some(sync_aggregate) = block.body().sync_aggregate() {
         process_sync_aggregate(
             state,
-            &inner.body.sync_aggregate,
+            sync_aggregate,
             proposer_index,
             verify_signatures,
             spec,
         )?;
     }
 
     if is_execution_enabled(state, block.body()) {
-        process_execution_payload(state, block.body().execution_payload().unwrap(), spec)?
+        let payload = block
+            .body()
+            .execution_payload()
+            .ok_or(BlockProcessingError::IncorrectStateType)?;
+        process_execution_payload(state, payload, spec)?;
     }
 
     Ok(())

consensus/state_processing/src/per_block_processing/errors.rs

@@ -77,6 +77,7 @@ pub enum BlockProcessingError {
         expected: u64,
         found: u64,
     },
+    ExecutionInvalid,
 }
 
 impl From<BeaconStateError> for BlockProcessingError {

consensus/state_processing/src/per_block_processing/process_operations.rs

@@ -353,15 +353,15 @@ pub fn process_deposit<T: EthSpec>(
         state.validators_mut().push(validator)?;
         state.balances_mut().push(deposit.data.amount)?;
 
-        // Altair-specific initializations.
-        if let BeaconState::Altair(altair_state) = state {
-            altair_state
-                .previous_epoch_participation
-                .push(ParticipationFlags::default())?;
-            altair_state
-                .current_epoch_participation
-                .push(ParticipationFlags::default())?;
-            altair_state.inactivity_scores.push(0)?;
+        // Altair or later initializations.
+        if let Ok(previous_epoch_participation) = state.previous_epoch_participation_mut() {
+            previous_epoch_participation.push(ParticipationFlags::default())?;
+        }
+        if let Ok(current_epoch_participation) = state.current_epoch_participation_mut() {
+            current_epoch_participation.push(ParticipationFlags::default())?;
+        }
+        if let Ok(inactivity_scores) = state.inactivity_scores_mut() {
+            inactivity_scores.push(0)?;
         }
     }
 

consensus/types/src/beacon_block.rs

@@ -89,12 +89,17 @@ impl<T: EthSpec> BeaconBlock<T> {
         let epoch = slot.epoch(T::slots_per_epoch());
 
         if spec
+            .merge_fork_epoch
+            .map_or(false, |merge_epoch| epoch >= merge_epoch)
+        {
+            BeaconBlockMerge::from_ssz_bytes(bytes).map(Self::Merge)
+        } else if spec
             .altair_fork_epoch
-            .map_or(true, |altair_epoch| epoch < altair_epoch)
+            .map_or(false, |altair_epoch| epoch >= altair_epoch)
         {
-            BeaconBlockBase::from_ssz_bytes(bytes).map(Self::Base)
-        } else {
             BeaconBlockAltair::from_ssz_bytes(bytes).map(Self::Altair)
+        } else {
+            BeaconBlockBase::from_ssz_bytes(bytes).map(Self::Base)
         }
     }
 
@@ -104,9 +109,13 @@ impl<T: EthSpec> BeaconBlock<T> {
     /// Usually it's better to prefer `from_ssz_bytes` which will decode the correct variant based
     /// on the fork slot.
     pub fn any_from_ssz_bytes(bytes: &[u8]) -> Result<Self, ssz::DecodeError> {
-        BeaconBlockAltair::from_ssz_bytes(bytes)
-            .map(BeaconBlock::Altair)
-            .or_else(|_| BeaconBlockBase::from_ssz_bytes(bytes).map(BeaconBlock::Base))
+        BeaconBlockMerge::from_ssz_bytes(bytes)
+            .map(BeaconBlock::Merge)
+            .or_else(|_| {
+                BeaconBlockAltair::from_ssz_bytes(bytes)
+                    .map(BeaconBlock::Altair)
+                    .or_else(|_| BeaconBlockBase::from_ssz_bytes(bytes).map(BeaconBlock::Base))
+            })
     }
 
     /// Convenience accessor for the `body` as a `BeaconBlockBodyRef`.

consensus/types/src/beacon_state.rs

@@ -418,12 +418,17 @@ impl<T: EthSpec> BeaconState<T> {
         let epoch = slot.epoch(T::slots_per_epoch());
 
         if spec
+            .merge_fork_epoch
+            .map_or(false, |merge_epoch| epoch >= merge_epoch)
+        {
+            BeaconStateMerge::from_ssz_bytes(bytes).map(Self::Merge)
+        } else if spec
             .altair_fork_epoch
-            .map_or(true, |altair_epoch| epoch < altair_epoch)
+            .map_or(false, |altair_epoch| epoch >= altair_epoch)
         {
-            BeaconStateBase::from_ssz_bytes(bytes).map(Self::Base)
-        } else {
             BeaconStateAltair::from_ssz_bytes(bytes).map(Self::Altair)
+        } else {
+            BeaconStateBase::from_ssz_bytes(bytes).map(Self::Base)
         }
     }
 
@@ -1686,7 +1691,8 @@ impl<T: EthSpec> CompareFields for BeaconState<T> {
         match (self, other) {
             (BeaconState::Base(x), BeaconState::Base(y)) => x.compare_fields(y),
             (BeaconState::Altair(x), BeaconState::Altair(y)) => x.compare_fields(y),
-            _ => panic!("compare_fields: mismatched state variants"),
+            (BeaconState::Merge(x), BeaconState::Merge(y)) => x.compare_fields(y),
+            _ => panic!("compare_fields: mismatched state variants",),
         }
     }
 }

consensus/types/src/beacon_state/tree_hash_cache.rs

@@ -341,16 +341,26 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
         )?;
         hasher.write(state.finalized_checkpoint().tree_hash_root().as_bytes())?;
 
-        // Inactivity & light-client sync committees
-        if let BeaconState::Altair(ref state) = state {
+        // Inactivity & light-client sync committees (Altair and later).
+        if let Ok(inactivity_scores) = state.inactivity_scores() {
             hasher.write(
                 self.inactivity_scores
-                    .recalculate_tree_hash_root(&state.inactivity_scores)?
+                    .recalculate_tree_hash_root(inactivity_scores)?
                     .as_bytes(),
             )?;
+        }
+
+        if let Ok(current_sync_committee) = state.current_sync_committee() {
+            hasher.write(current_sync_committee.tree_hash_root().as_bytes())?;
+        }
+
+        if let Ok(next_sync_committee) = state.next_sync_committee() {
+            hasher.write(next_sync_committee.tree_hash_root().as_bytes())?;
+        }
 
-            hasher.write(state.current_sync_committee.tree_hash_root().as_bytes())?;
-            hasher.write(state.next_sync_committee.tree_hash_root().as_bytes())?;
+        // Execution payload (merge and later).
+        if let Ok(payload_header) = state.latest_execution_payload_header() {
+            hasher.write(payload_header.tree_hash_root().as_bytes())?;
         }
 
         let root = hasher.finish()?;

consensus/types/src/execution_payload.rs

@@ -8,8 +8,15 @@ use tree_hash_derive::TreeHash;
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)]
 #[ssz(enum_behaviour = "union")]
 #[tree_hash(enum_behaviour = "union")]
+#[serde(tag = "selector", content = "value")]
+#[serde(bound = "T: EthSpec")]
 pub enum Transaction<T: EthSpec> {
-    OpaqueTransaction(VariableList<u8, T::MaxBytesPerOpaqueTransaction>),
+    // FIXME(merge): renaming this enum variant to 0 is a bit of a hack...
+    #[serde(rename = "0")]
+    OpaqueTransaction(
+        #[serde(with = "ssz_types::serde_utils::hex_var_list")]
+        VariableList<u8, T::MaxBytesPerOpaqueTransaction>,
+    ),
 }
 
 impl<T: EthSpec, I: SliceIndex<[u8]>> Index<I> for Transaction<T> {
@@ -33,12 +40,13 @@ impl<T: EthSpec> From<VariableList<u8, T::MaxBytesPerOpaqueTransaction>> for Tra
 #[derive(
     Default, Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
 )]
+#[serde(bound = "T: EthSpec")]
 pub struct ExecutionPayload<T: EthSpec> {
     pub parent_hash: Hash256,
     pub coinbase: Address,
     pub state_root: Hash256,
     pub receipt_root: Hash256,
-    #[serde(with = "serde_logs_bloom")]
+    #[serde(with = "ssz_types::serde_utils::hex_fixed_vec")]
     pub logs_bloom: FixedVector<u8, T::BytesPerLogsBloom>,
     pub random: Hash256,
     #[serde(with = "eth2_serde_utils::quoted_u64")]
@@ -51,7 +59,6 @@ pub struct ExecutionPayload<T: EthSpec> {
     pub timestamp: u64,
     pub base_fee_per_gas: Hash256,
     pub block_hash: Hash256,
-    #[serde(with = "serde_transactions")]
     #[test_random(default)]
     pub transactions: VariableList<Transaction<T>, T::MaxTransactionsPerPayload>,
 }
@@ -76,99 +83,3 @@ impl<T: EthSpec> ExecutionPayload<T> {
         }
     }
 }
-
-/// Serializes the `logs_bloom` field.
-pub mod serde_logs_bloom {
-    use super::*;
-    use eth2_serde_utils::hex::PrefixedHexVisitor;
-    use serde::{Deserializer, Serializer};
-
-    pub fn serialize<S, U>(bytes: &FixedVector<u8, U>, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-        U: Unsigned,
-    {
-        let mut hex_string: String = "0x".to_string();
-        hex_string.push_str(&hex::encode(&bytes[..]));
-
-        serializer.serialize_str(&hex_string)
-    }
-
-    pub fn deserialize<'de, D, U>(deserializer: D) -> Result<FixedVector<u8, U>, D::Error>
-    where
-        D: Deserializer<'de>,
-        U: Unsigned,
-    {
-        let vec = deserializer.deserialize_string(PrefixedHexVisitor)?;
-
-        FixedVector::new(vec)
-            .map_err(|e| serde::de::Error::custom(format!("invalid logs bloom: {:?}", e)))
-    }
-}
-
-/// Serializes the `transactions` field.
-pub mod serde_transactions {
-    use super::*;
-    use eth2_serde_utils::hex;
-    use serde::ser::SerializeSeq;
-    use serde::{de, Deserializer, Serializer};
-    use std::marker::PhantomData;
-
-    pub struct ListOfBytesListVisitor<T: EthSpec> {
-        _t: PhantomData<T>,
-    }
-    impl<'a, T> serde::de::Visitor<'a> for ListOfBytesListVisitor<T>
-    where
-        T: EthSpec,
-    {
-        type Value = VariableList<Transaction<T>, T::MaxTransactionsPerPayload>;
-
-        fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
-            write!(formatter, "a list of 0x-prefixed byte lists")
-        }
-
-        fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
-        where
-            A: serde::de::SeqAccess<'a>,
-        {
-            let mut outer = VariableList::default();
-
-            while let Some(val) = seq.next_element::<String>()? {
-                let inner_vec = hex::decode(&val).map_err(de::Error::custom)?;
-                let inner = VariableList::new(inner_vec).map_err(|e| {
-                    serde::de::Error::custom(format!("invalid transaction: {:?}", e))
-                })?;
-                outer.push(inner.into()).map_err(|e| {
-                    serde::de::Error::custom(format!("too many transactions: {:?}", e))
-                })?;
-            }
-
-            Ok(outer)
-        }
-    }
-
-    pub fn serialize<S, T>(
-        value: &VariableList<Transaction<T>, T::MaxTransactionsPerPayload>,
-        serializer: S,
-    ) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-        T: EthSpec,
-    {
-        let mut seq = serializer.serialize_seq(Some(value.len()))?;
-        for val in value {
-            seq.serialize_element(&hex::encode(&val[..]))?;
-        }
-        seq.end()
-    }
-
-    pub fn deserialize<'de, D, T>(
-        deserializer: D,
-    ) -> Result<VariableList<Transaction<T>, T::MaxTransactionsPerPayload>, D::Error>
-    where
-        D: Deserializer<'de>,
-        T: EthSpec,
-    {
-        deserializer.deserialize_any(ListOfBytesListVisitor { _t: PhantomData })
-    }
-}

consensus/types/src/execution_payload_header.rs

@@ -1,4 +1,4 @@
-use crate::{execution_payload::serde_logs_bloom, test_utils::TestRandom, *};
+use crate::{test_utils::TestRandom, *};
 use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
 use test_random_derive::TestRandom;
@@ -13,7 +13,7 @@ pub struct ExecutionPayloadHeader<T: EthSpec> {
     pub coinbase: Address,
     pub state_root: Hash256,
     pub receipt_root: Hash256,
-    #[serde(with = "serde_logs_bloom")]
+    #[serde(with = "ssz_types::serde_utils::hex_fixed_vec")]
     pub logs_bloom: FixedVector<u8, T::BytesPerLogsBloom>,
     pub random: Hash256,
     #[serde(with = "eth2_serde_utils::quoted_u64")]