Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(sidecar): fallback block builder refactor #512

Merged
merged 10 commits into from
Dec 10, 2024
1,120 changes: 572 additions & 548 deletions bolt-sidecar/Cargo.lock

Large diffs are not rendered by default.

9 changes: 5 additions & 4 deletions bolt-sidecar/Cargo.toml
Original file line number Diff line number Diff line change
@@ -15,6 +15,8 @@ tokio = { version = "1", features = ["full"] }
axum = { version = "0.7", features = ["macros"] }
tower-http = { version = "0.5.2", features = ["timeout"] }
axum-extra = "0.9.6"
tower = "0.5.1"
http-body-util = "0.1.2"
futures = "0.3"
tokio-retry = "0.3.0"

@@ -31,11 +33,10 @@ alloy = { version = "0.7.2", features = [
"full",
"provider-trace-api",
"rpc-types-beacon",
"rpc-types-engine",
] }
alloy-rpc-types-engine = { version = "0.7.2", default-features = false, features = [
"jwt",
] }
alloy-rpc-types-engine = { version = "0.7.2", default-features = false, features = ["jwt"] }
alloy-transport-http = { version = "0.7.2", default-features = false, features = ["jwt-auth"] }
alloy-provider = { version = "0.7.2", default-features = false, features = ["engine-api"] }

# reth
reth-primitives = { git = "https://github.com/paradigmxyz/reth", version = "1.1.2" }
25 changes: 5 additions & 20 deletions bolt-sidecar/src/builder/compat.rs
Original file line number Diff line number Diff line change
@@ -2,14 +2,9 @@ use alloy::{
consensus::BlockHeader,
eips::{eip2718::Encodable2718, eip4895::Withdrawal},
primitives::{Address, Bloom, B256, U256},
rpc::types::{
engine::{
ExecutionPayload as AlloyExecutionPayload, ExecutionPayloadV1, ExecutionPayloadV2,
ExecutionPayloadV3,
},
Withdrawals,
},
rpc::types::Withdrawals,
};
use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3};
use ethereum_consensus::{
bellatrix::mainnet::Transaction,
capella::spec,
@@ -81,7 +76,7 @@ pub(crate) fn to_execution_payload_header(
pub(crate) fn to_alloy_execution_payload(
block: &SealedBlock,
block_hash: B256,
) -> AlloyExecutionPayload {
) -> ExecutionPayloadV3 {
let alloy_withdrawals = block
.body
.withdrawals
@@ -99,7 +94,7 @@ pub(crate) fn to_alloy_execution_payload(
})
.unwrap_or_default();

AlloyExecutionPayload::V3(ExecutionPayloadV3 {
ExecutionPayloadV3 {
blob_gas_used: block.blob_gas_used().unwrap_or_default(),
excess_blob_gas: block.excess_blob_gas.unwrap_or_default(),
payload_inner: ExecutionPayloadV2 {
@@ -121,7 +116,7 @@ pub(crate) fn to_alloy_execution_payload(
},
withdrawals: alloy_withdrawals,
},
})
}
}

/// Compatibility: convert a sealed block into an ethereum-consensus execution payload
@@ -168,16 +163,6 @@ pub(crate) fn to_consensus_execution_payload(value: &SealedBlock) -> ConsensusEx
ConsensusExecutionPayload::Deneb(payload)
}

/// Compatibility: convert a Withdrawal from ethereum-consensus to alloy::primitives
pub(crate) fn to_alloy_withdrawal(value: ethereum_consensus::capella::Withdrawal) -> Withdrawal {
Withdrawal {
index: value.index as u64,
validator_index: value.validator_index as u64,
address: Address::from_slice(value.address.as_ref()),
amount: value.amount,
}
}

/// Compatibility: convert a withdrawal from alloy::primitives to ethereum-consensus
pub(crate) fn to_consensus_withdrawal(
value: &Withdrawal,
259 changes: 259 additions & 0 deletions bolt-sidecar/src/builder/fallback/engine_hinter.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,259 @@
use std::ops::Deref;

use alloy::{
consensus::EMPTY_OMMER_ROOT_HASH,
primitives::{Address, Bloom, Bytes, B256, B64, U256},
rpc::types::{Block, Withdrawal, Withdrawals},
};
use alloy_provider::ext::EngineApi;
use alloy_rpc_types_engine::{ClientCode, ExecutionPayloadV3, JwtSecret, PayloadStatusEnum};
use reqwest::Url;
use reth_primitives::{
BlockBody, Header as RethHeader, SealedBlock, SealedHeader, TransactionSigned,
};
use tracing::{debug, error};

use crate::{
builder::{compat::to_alloy_execution_payload, BuilderError},
client::EngineClient,
};

use super::engine_hints::parse_hint_from_engine_response;

/// The [EngineHinter] is responsible for gathering "hints" from the
/// engine API error responses to complete the sealed block.
///
/// Since error messages are not overly standardized across execution clients,
/// we need to know which execution client is being used to properly parse the hints.
///
/// This can be done by querying the EL `engine_getClientVersionV1` method.
#[derive(Debug)]
pub struct EngineHinter {
engine_client: EngineClient,
}

impl Deref for EngineHinter {
type Target = EngineClient;

fn deref(&self) -> &Self::Target {
&self.engine_client
}
}

impl EngineHinter {
/// Create a new [EngineHinter] instance with the given JWT and engine RPC URL.
pub fn new(jwt_secret: JwtSecret, engine_rpc_url: Url) -> Self {
Self { engine_client: EngineClient::new_http(engine_rpc_url, jwt_secret) }
}

/// Collect hints from the engine API to complete the sealed block.
/// This method will keep fetching hints until the payload is valid.
pub async fn fetch_payload_from_hints(
&self,
mut ctx: EngineHinterContext,
) -> Result<SealedBlock, BuilderError> {
// The block body can be the same for all iterations, since it only contains
// the transactions and withdrawals from the context.
let body = ctx.build_block_body();

// Loop until we get a valid payload from the engine API. On each iteration,
// we build a new block header with the hints from the context and fetch the next hint.
let max_iterations = 20;
let mut iteration = 0;
loop {
debug!(%iteration, "Fetching hint from engine API");

// Build a new block header using the hints from the context
let header = ctx.build_block_header_with_hints();

let sealed_hash = header.hash_slow();
let sealed_header = SealedHeader::new(header, sealed_hash);
let sealed_block = SealedBlock::new(sealed_header, body.clone());
let block_hash = ctx.hints.block_hash.unwrap_or(sealed_block.hash());

// build the new execution payload from the block header
let exec_payload = to_alloy_execution_payload(&sealed_block, block_hash);

// attempt to fetch the next hint from the engine API payload response
let hint = self.next_hint(exec_payload, &ctx).await?;

if matches!(hint, EngineApiHint::ValidPayload) {
return Ok(sealed_block);
}

// Populate the new hint in the context and continue the loop
ctx.hints.populate_new(hint);

iteration += 1;
if iteration >= max_iterations {
return Err(BuilderError::ExceededMaxHintIterations(max_iterations));
}
}
}

/// Yield the next hint from the engine API by calling `engine_newPayloadV3`
/// and parsing the response to extract the hint.
///
/// Returns Ok([EngineApiHint::ValidPayload]) if the payload is valid.
async fn next_hint(
&self,
exec_payload: ExecutionPayloadV3,
ctx: &EngineHinterContext,
) -> Result<EngineApiHint, BuilderError> {
let payload_status = self
.engine_client
.new_payload_v3(
exec_payload,
ctx.blob_versioned_hashes.clone(),
ctx.parent_beacon_block_root,
)
.await?;

let validation_error = match payload_status.status {
PayloadStatusEnum::Valid => return Ok(EngineApiHint::ValidPayload),
PayloadStatusEnum::Invalid { validation_error } => validation_error,
PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted => {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is Accepted an error?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because we don't want the EL to accept this block. According to the engine specs: https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md#engine_newpayloadv1 the ACCEPTED code MUST be returned only if the following conditions are met:

  • all transactions have non-zero length
  • the blockHash of the payload is valid
  • the payload doesn't extend the canonical chain
  • the payload hasn't been fully validated
  • ancestors of a payload are known and comprise a well-formed chain

in our case our block extends the canonical chain so it will be marked as VALID.

error!(status = ?payload_status.status, "Unexpected payload status from engine API");
return Err(BuilderError::UnexpectedPayloadStatus(payload_status.status))
}
};

// Parse the hint from the engine API response, based on the EL client code
let Some(hint) = parse_hint_from_engine_response(ctx.el_client_code, &validation_error)?
else {
return Err(BuilderError::FailedToParseHintsFromEngine);
};

Ok(hint)
}
}

/// Engine API hint values that can be fetched from the engine API
/// to complete the sealed block. These hints are used to fill in
/// missing values in the block header.
#[derive(Debug, Copy, Clone)]
#[allow(clippy::large_enum_variant)]
pub enum EngineApiHint {
BlockHash(B256),
GasUsed(u64),
StateRoot(B256),
ReceiptsRoot(B256),
LogsBloom(Bloom),
ValidPayload,
}

/// The collection of hints that can be fetched from the engine API
/// via the [EngineHinter] to complete the sealed block.
///
/// When a field is `None`, we set it to its default value in the [ExecutionPayload]
/// and try to get the hint from the engine API response to fill its value.
#[derive(Debug, Default)]
pub struct Hints {
pub gas_used: Option<u64>,
pub receipts_root: Option<B256>,
pub logs_bloom: Option<Bloom>,
pub state_root: Option<B256>,
pub block_hash: Option<B256>,
}

impl Hints {
/// Populate the new hint value in the context.
pub fn populate_new(&mut self, hint: EngineApiHint) {
match hint {
EngineApiHint::ValidPayload => { /* No-op */ }

// If we receive a block hash hint, set it and keep it for the next one.
// This should not happen, but in case it does, it doesn't break the flow.
EngineApiHint::BlockHash(hash) => self.block_hash = Some(hash),

// For regular hint types, set the value and reset the block hash
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we reset the block hash hint here every time?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It really depends on how the EL validation is structured, but me and @thedevbirb found that sometimes Geth would return a mismatching hash error after populating some extra field (e.g. state root) which means we can't get more information out of it (as it kept returning mismatching block hash and no other hints). So when this happens we set the block hash to the one hinted by Geth and then proceed with the next hints.

As the comment mentions, this shouldn't happen and it's more of an edge case

EngineApiHint::GasUsed(gas) => {
self.gas_used = Some(gas);
self.block_hash = None;
}
EngineApiHint::StateRoot(hash) => {
self.state_root = Some(hash);
self.block_hash = None;
}
EngineApiHint::ReceiptsRoot(hash) => {
self.receipts_root = Some(hash);
self.block_hash = None;
}
EngineApiHint::LogsBloom(bloom) => {
self.logs_bloom = Some(bloom);
self.block_hash = None;
}
}
}
}

/// Context holding the necessary values for
/// building a sealed block. Some of this data is fetched from the
/// beacon chain, while others are calculated locally or from the
/// transactions themselves.
#[derive(Debug)]
pub struct EngineHinterContext {
pub extra_data: Bytes,
pub base_fee: u64,
pub blob_gas_used: u64,
pub excess_blob_gas: u64,
pub prev_randao: B256,
pub fee_recipient: Address,
pub transactions_root: B256,
pub withdrawals_root: B256,
pub parent_beacon_block_root: B256,
pub blob_versioned_hashes: Vec<B256>,
pub block_timestamp: u64,
pub transactions: Vec<TransactionSigned>,
pub withdrawals: Vec<Withdrawal>,
pub head_block: Block,
pub hints: Hints,
pub el_client_code: ClientCode,
}

impl EngineHinterContext {
/// Build a block body using the transactions and withdrawals from the context.
pub fn build_block_body(&self) -> BlockBody {
BlockBody {
ommers: Vec::new(),
transactions: self.transactions.clone(),
withdrawals: Some(Withdrawals::new(self.withdrawals.clone())),
}
}

/// Build a header using the info from the context.
/// Use any hints available, and default to an empty value if not present.
pub fn build_block_header_with_hints(&self) -> RethHeader {
// Use the available hints, or default to an empty value if not present.
let gas_used = self.hints.gas_used.unwrap_or_default();
let receipts_root = self.hints.receipts_root.unwrap_or_default();
let logs_bloom = self.hints.logs_bloom.unwrap_or_default();
let state_root = self.hints.state_root.unwrap_or_default();

RethHeader {
parent_hash: self.head_block.header.hash,
ommers_hash: EMPTY_OMMER_ROOT_HASH,
beneficiary: self.fee_recipient,
state_root,
transactions_root: self.transactions_root,
receipts_root,
withdrawals_root: Some(self.withdrawals_root),
logs_bloom,
difficulty: U256::ZERO,
number: self.head_block.header.number + 1,
gas_limit: self.head_block.header.gas_limit,
gas_used,
timestamp: self.block_timestamp,
mix_hash: self.prev_randao,
nonce: B64::ZERO,
base_fee_per_gas: Some(self.base_fee),
blob_gas_used: Some(self.blob_gas_used),
excess_blob_gas: Some(self.excess_blob_gas),
parent_beacon_block_root: Some(self.parent_beacon_block_root),
extra_data: self.extra_data.clone(),
// TODO: handle the Pectra-related fields
requests_hash: None,
target_blobs_per_block: None,
}
}
}
52 changes: 52 additions & 0 deletions bolt-sidecar/src/builder/fallback/engine_hints/geth.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
use alloy::primitives::{Bloom, B256};
use hex::FromHex;
use regex::Regex;

use crate::builder::{fallback::engine_hinter::EngineApiHint, BuilderError};

/// Parse a hinted value from the engine response.
/// An example error message from the engine API looks like this:
///
/// ```json
/// {
/// "jsonrpc": "2.0",
/// "id": 1,
/// "error": {
/// "code":-32000,
/// "message": "local: blockhash mismatch: got 0x... expected 0x..."
/// }
/// }
/// ```
///
/// Geth Reference:
/// - [ValidateState](<https://github.com/ethereum/go-ethereum/blob/9298d2db884c4e3f9474880e3dcfd080ef9eacfa/core/block_validator.go#L122-L151>)
/// - [Blockhash Mismatch](<https://github.com/ethereum/go-ethereum/blob/9298d2db884c4e3f9474880e3dcfd080ef9eacfa/beacon/engine/types.go#L253-L256>)
pub fn parse_geth_engine_error_hint(error: &str) -> Result<Option<EngineApiHint>, BuilderError> {
// Capture either the "local" or "got" value from the error message
let re = Regex::new(r"(?:local:|got) ([0-9a-zA-Z]+)").expect("valid regex");

let raw_hint_value = match re.captures(error).and_then(|cap| cap.get(1)) {
Some(matched) => matched.as_str().to_string(),
None => return Ok(None),
};

// Match the hint value to the corresponding hint type based on other parts of the error message
if error.contains("blockhash mismatch") {
return Ok(Some(EngineApiHint::BlockHash(B256::from_hex(raw_hint_value)?)));
} else if error.contains("invalid gas used") {
return Ok(Some(EngineApiHint::GasUsed(raw_hint_value.parse()?)));
} else if error.contains("invalid merkle root") {
return Ok(Some(EngineApiHint::StateRoot(B256::from_hex(raw_hint_value)?)));
} else if error.contains("invalid receipt root hash") {
return Ok(Some(EngineApiHint::ReceiptsRoot(B256::from_hex(raw_hint_value)?)));
} else if error.contains("invalid bloom") {
return Ok(Some(EngineApiHint::LogsBloom(Bloom::from_hex(&raw_hint_value)?)));
};

// Match some error message that we don't know how to handle
if error.contains("could not apply tx") {
return Err(BuilderError::InvalidTransactions(error.to_string()));
}

Err(BuilderError::UnsupportedEngineHint(error.to_string()))
}
Loading