Merge branch 'unstable' of https://github.com/sigp/lighthouse into bl…

…ock-v3-vc

Makefile

@@ -215,7 +215,8 @@ lint:
 		-A clippy::upper-case-acronyms \
 		-A clippy::vec-init-then-push \
 		-A clippy::question-mark \
-		-A clippy::uninlined-format-args
+		-A clippy::uninlined-format-args \
+		-A clippy::enum_variant_names
 
 # Lints the code using Clippy and automatically fix some simple compiler warnings.
 lint-fix:

account_manager/src/validator/slashing_protection.rs

@@ -16,7 +16,6 @@ pub const EXPORT_CMD: &str = "export";
 pub const IMPORT_FILE_ARG: &str = "IMPORT-FILE";
 pub const EXPORT_FILE_ARG: &str = "EXPORT-FILE";
 
-pub const MINIFY_FLAG: &str = "minify";
 pub const PUBKEYS_FLAG: &str = "pubkeys";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
@@ -31,16 +30,6 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                         .value_name("FILE")
                         .help("The slashing protection interchange file to import (.json)"),
                 )
-                .arg(
-                    Arg::with_name(MINIFY_FLAG)
-                        .long(MINIFY_FLAG)
-                        .takes_value(true)
-                        .possible_values(&["false", "true"])
-                        .help(
-                            "Deprecated: Lighthouse no longer requires minification on import \
-                             because it always minifies",
-                        ),
-                ),
         )
         .subcommand(
             App::new(EXPORT_CMD)
@@ -61,17 +50,6 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                              comma-separated. All known keys will be exported if omitted",
                         ),
                 )
-                .arg(
-                    Arg::with_name(MINIFY_FLAG)
-                        .long(MINIFY_FLAG)
-                        .takes_value(true)
-                        .default_value("false")
-                        .possible_values(&["false", "true"])
-                        .help(
-                            "Minify the output file. This will make it smaller and faster to \
-                             import, but not faster to generate.",
-                        ),
-                ),
         )
 }
 
@@ -92,7 +70,6 @@ pub fn cli_run<T: EthSpec>(
     match matches.subcommand() {
         (IMPORT_CMD, Some(matches)) => {
             let import_filename: PathBuf = clap_utils::parse_required(matches, IMPORT_FILE_ARG)?;
-            let minify: Option<bool> = clap_utils::parse_optional(matches, MINIFY_FLAG)?;
             let import_file = File::open(&import_filename).map_err(|e| {
                 format!(
                     "Unable to open import file at {}: {:?}",
@@ -102,23 +79,10 @@ pub fn cli_run<T: EthSpec>(
             })?;
 
             eprint!("Loading JSON file into memory & deserializing");
-            let mut interchange = Interchange::from_json_reader(&import_file)
+            let interchange = Interchange::from_json_reader(&import_file)
                 .map_err(|e| format!("Error parsing file for import: {:?}", e))?;
             eprintln!(" [done].");
 
-            if let Some(minify) = minify {
-                eprintln!(
-                    "WARNING: --minify flag is deprecated and will be removed in a future release"
-                );
-                if minify {
-                    eprint!("Minifying input file for faster loading");
-                    interchange = interchange
-                        .minify()
-                        .map_err(|e| format!("Minification failed: {:?}", e))?;
-                    eprintln!(" [done].");
-                }
-            }
-
             let slashing_protection_database =
                 SlashingDatabase::open_or_create(&slashing_protection_db_path).map_err(|e| {
                     format!(
@@ -206,7 +170,6 @@ pub fn cli_run<T: EthSpec>(
         }
         (EXPORT_CMD, Some(matches)) => {
             let export_filename: PathBuf = clap_utils::parse_required(matches, EXPORT_FILE_ARG)?;
-            let minify: bool = clap_utils::parse_required(matches, MINIFY_FLAG)?;
 
             let selected_pubkeys = if let Some(pubkeys) =
                 clap_utils::parse_optional::<String>(matches, PUBKEYS_FLAG)?
@@ -237,17 +200,10 @@ pub fn cli_run<T: EthSpec>(
                     )
                 })?;
 
-            let mut interchange = slashing_protection_database
+            let interchange = slashing_protection_database
                 .export_interchange_info(genesis_validators_root, selected_pubkeys.as_deref())
                 .map_err(|e| format!("Error during export: {:?}", e))?;
 
-            if minify {
-                eprintln!("Minifying output file");
-                interchange = interchange
-                    .minify()
-                    .map_err(|e| format!("Unable to minify output: {:?}", e))?;
-            }
-
             let output_file = File::create(export_filename)
                 .map_err(|e| format!("Error creating output file: {:?}", e))?;
 

beacon_node/beacon_chain/src/attestation_rewards.rs

@@ -5,7 +5,9 @@ use participation_cache::ParticipationCache;
 use safe_arith::SafeArith;
 use serde_utils::quoted_u64::Quoted;
 use slog::debug;
-use state_processing::per_epoch_processing::altair::process_inactivity_updates;
+use state_processing::per_epoch_processing::altair::{
+    process_inactivity_updates, process_justification_and_finalization,
+};
 use state_processing::{
     common::altair::BaseRewardPerIncrement,
     per_epoch_processing::altair::{participation_cache, rewards_and_penalties::get_flag_weight},
@@ -27,6 +29,7 @@ use state_processing::per_epoch_processing::base::rewards_and_penalties::{
 };
 use state_processing::per_epoch_processing::base::validator_statuses::InclusionInfo;
 use state_processing::per_epoch_processing::base::{
+    process_justification_and_finalization as process_justification_and_finalization_base,
     TotalBalances, ValidatorStatus, ValidatorStatuses,
 };
 
@@ -67,6 +70,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         let mut validator_statuses = ValidatorStatuses::new(&state, spec)?;
         validator_statuses.process_attestations(&state)?;
 
+        process_justification_and_finalization_base(
+            &state,
+            &validator_statuses.total_balances,
+            spec,
+        )?
+        .apply_changes_to_state(&mut state);
+
         let ideal_rewards =
             self.compute_ideal_rewards_base(&state, &validator_statuses.total_balances)?;
 
@@ -125,6 +135,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 
         // Calculate ideal_rewards
         let participation_cache = ParticipationCache::new(&state, spec)?;
+        process_justification_and_finalization(&state, &participation_cache)?
+            .apply_changes_to_state(&mut state);
         process_inactivity_updates(&mut state, &participation_cache, spec)?;
 
         let previous_epoch = state.previous_epoch();

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -18,7 +18,7 @@ use crate::block_verification::{
 use crate::block_verification_types::{
     AsBlock, AvailableExecutedBlock, BlockImportData, ExecutedBlock, RpcBlock,
 };
-pub use crate::canonical_head::{CanonicalHead, CanonicalHeadRwLock};
+pub use crate::canonical_head::CanonicalHead;
 use crate::chain_config::ChainConfig;
 use crate::data_availability_checker::{
     Availability, AvailabilityCheckError, AvailableBlock, DataAvailabilityChecker,
@@ -453,7 +453,7 @@ pub struct BeaconChain<T: BeaconChainTypes> {
     /// A cache of eth1 deposit data at epoch boundaries for deposit finalization
     pub eth1_finalization_cache: TimeoutRwLock<Eth1FinalizationCache>,
     /// Caches the beacon block proposer shuffling for a given epoch and shuffling key root.
-    pub beacon_proposer_cache: Mutex<BeaconProposerCache>,
+    pub beacon_proposer_cache: Arc<Mutex<BeaconProposerCache>>,
     /// Caches a map of `validator_index -> validator_pubkey`.
     pub(crate) validator_pubkey_cache: TimeoutRwLock<ValidatorPubkeyCache<T>>,
     /// A cache used when producing attestations.

beacon_node/beacon_chain/src/beacon_proposer_cache.rs

@@ -25,7 +25,7 @@ const CACHE_SIZE: usize = 16;
 /// This value is fairly unimportant, it's used to avoid heap allocations. The result of it being
 /// incorrect is non-substantial from a consensus perspective (and probably also from a
 /// performance perspective).
-const TYPICAL_SLOTS_PER_EPOCH: usize = 32;
+pub const TYPICAL_SLOTS_PER_EPOCH: usize = 32;
 
 /// For some given slot, this contains the proposer index (`index`) and the `fork` that should be
 /// used to verify their signature.

beacon_node/beacon_chain/src/block_verification.rs

@@ -583,29 +583,33 @@ pub fn signature_verify_chain_segment<T: BeaconChainTypes>(
         &chain.spec,
     )?;
 
+    // unzip chain segment and verify kzg in bulk
+    let (roots, blocks): (Vec<_>, Vec<_>) = chain_segment.into_iter().unzip();
+    let maybe_available_blocks = chain
+        .data_availability_checker
+        .verify_kzg_for_rpc_blocks(blocks)?;
+    // zip it back up
+    let mut signature_verified_blocks = roots
+        .into_iter()
+        .zip(maybe_available_blocks)
+        .map(|(block_root, maybe_available_block)| {
+            let consensus_context = ConsensusContext::new(maybe_available_block.slot())
+                .set_current_block_root(block_root);
+            SignatureVerifiedBlock {
+                block: maybe_available_block,
+                block_root,
+                parent: None,
+                consensus_context,
+            }
+        })
+        .collect::<Vec<_>>();
+
+    // verify signatures
     let pubkey_cache = get_validator_pubkey_cache(chain)?;
     let mut signature_verifier = get_signature_verifier(&state, &pubkey_cache, &chain.spec);
-
-    let mut signature_verified_blocks = Vec::with_capacity(chain_segment.len());
-
-    for (block_root, block) in &chain_segment {
-        let mut consensus_context =
-            ConsensusContext::new(block.slot()).set_current_block_root(*block_root);
-
-        signature_verifier.include_all_signatures(block.as_block(), &mut consensus_context)?;
-
-        let maybe_available_block = chain
-            .data_availability_checker
-            .check_rpc_block_availability(block.clone())?;
-
-        // Save the block and its consensus context. The context will have had its proposer index
-        // and attesting indices filled in, which can be used to accelerate later block processing.
-        signature_verified_blocks.push(SignatureVerifiedBlock {
-            block: maybe_available_block,
-            block_root: *block_root,
-            parent: None,
-            consensus_context,
-        });
+    for svb in &mut signature_verified_blocks {
+        signature_verifier
+            .include_all_signatures(svb.block.as_block(), &mut svb.consensus_context)?;
     }
 
     if signature_verifier.verify().is_err() {
@@ -1159,10 +1163,7 @@ impl<T: BeaconChainTypes> IntoExecutionPendingBlock<T> for Arc<SignedBeaconBlock
             .map_err(|e| BlockSlashInfo::SignatureNotChecked(self.signed_block_header(), e))?;
         let maybe_available = chain
             .data_availability_checker
-            .check_rpc_block_availability(RpcBlock::new_without_blobs(
-                Some(block_root),
-                self.clone(),
-            ))
+            .verify_kzg_for_rpc_block(RpcBlock::new_without_blobs(Some(block_root), self.clone()))
             .map_err(|e| {
                 BlockSlashInfo::SignatureNotChecked(
                     self.signed_block_header(),
@@ -1192,7 +1193,7 @@ impl<T: BeaconChainTypes> IntoExecutionPendingBlock<T> for RpcBlock<T::EthSpec>
             .map_err(|e| BlockSlashInfo::SignatureNotChecked(self.signed_block_header(), e))?;
         let maybe_available = chain
             .data_availability_checker
-            .check_rpc_block_availability(self.clone())
+            .verify_kzg_for_rpc_block(self.clone())
             .map_err(|e| {
                 BlockSlashInfo::SignatureNotChecked(
                     self.signed_block_header(),

beacon_node/beacon_chain/src/block_verification_types.rs

@@ -45,6 +45,13 @@ impl<E: EthSpec> RpcBlock<E> {
             RpcBlockInner::BlockAndBlobs(block, _) => block,
         }
     }
+
+    pub fn blobs(&self) -> Option<&BlobSidecarList<E>> {
+        match &self.block {
+            RpcBlockInner::Block(_) => None,
+            RpcBlockInner::BlockAndBlobs(_, blobs) => Some(blobs),
+        }
+    }
 }
 
 /// Note: This variant is intentionally private because we want to safely construct the