Merge pull request #84 from chainbound/feat/multiproofs-mev-boost

feat(mev-boost): merkle multiproofs

bolt-web-demo/frontend/src/app/page.tsx

@@ -90,7 +90,7 @@ export default function Home() {
       // If the event is a preconfirmation, extract the tx hash and slot number
       // and display a message with the explorer URL
       if (
-        event.message.toLowerCase().includes("verified merkle proof for tx")
+        event.message.toLowerCase().includes("verified merkle proof for slot")
       ) {
         setPreconfIncluded(true);
         setInclusionTimerActive(false);

builder/builder/utils_test.go

@@ -20,7 +20,7 @@ func TestGenerateMerkleMultiProofs(t *testing.T) {
 	// https://etherscan.io/tx/0x9d48b4a021898a605b7ae49bf93ad88fa6bd7050e9448f12dde064c10f22fe9c
 	// 0x02f87601836384348477359400850517683ba883019a28943678fce4028b6745eb04fa010d9c8e4b36d6288c872b0f1366ad800080c080a0b6b7aba1954160d081b2c8612e039518b9c46cd7df838b405a03f927ad196158a071d2fb6813e5b5184def6bd90fb5f29e0c52671dea433a7decb289560a58416e
 
-	raw := `["0x02f873011a8405f5e10085037fcc60e182520894f7eaaf75cb6ec4d0e2b53964ce6733f54f7d3ffc880b6139a7cbd2000080c080a095a7a3cbb7383fc3e7d217054f861b890a935adc1adf4f05e3a2f23688cf2416a00875cdc45f4395257e44d709d04990349b105c22c11034a60d7af749ffea2765","0x02f90176833018242585012a05f2008512a05f2000830249f0946c6340ba1dc72c59197825cd94eccc1f9c67416e80b901040cc7326300000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000008ffb6787e8ad80000000000000000000000000000b77d61ea79c7ea8bfa03d3604ce5eabfb95c2ab20000000000000000000000002c57d1cfc6d5f8e4182a56b4cf75421472ebaea4000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000001cd4af4a9bf33474c802d31790a195335f7a9ab8000000000000000000000000d676af79742bcaeb4a71cf62b85d5ba2d1deaf86c001a08d03bdca0c1647263ef73d916e949ccc53284c6fa208c3fa4f9ddfe67d9c45dfa055be5793b42f1818716276033eb36420fa4fb4e3efabd0bbb01c489f7d9cd43c","0xf8708305dc6885029332e35883019a2894500b0107e172e420561565c8177c28ac0f62017f8810ffb80e6cc327008025a0e9c0b380c68f040ae7affefd11979f5ed18ae82c00e46aa3238857c372a358eca06b26e179dd2f7a7f1601755249f4cff56690c4033553658f0d73e26c36fe7815", "0xf86c0785028fa6ae0082520894098d880c4753d0332ca737aa592332ed2522cd22880d2f09f6558750008026a0963e58027576b3a8930d7d9b4a49253b6e1a2060e259b2102e34a451d375ce87a063f802538d3efed17962c96fcea431388483bbe3860ea9bb3ef01d4781450fbf", "0x02f87601836384348477359400850517683ba883019a28943678fce4028b6745eb04fa010d9c8e4b36d6288c872b0f1366ad800080c080a0b6b7aba1954160d081b2c8612e039518b9c46cd7df838b405a03f927ad196158a071d2fb6813e5b5184def6bd90fb5f29e0c52671dea433a7decb289560a58416e"]`
+	raw := `["0x02f873011a8405f5e10085037fcc60e182520894f7eaaf75cb6ec4d0e2b53964ce6733f54f7d3ffc880b6139a7cbd2000080c080a095a7a3cbb7383fc3e7d217054f861b890a935adc1adf4f05e3a2f23688cf2416a00875cdc45f4395257e44d709d04990349b105c22c11034a60d7af749ffea2765","0xf8708305dc6885029332e35883019a2894500b0107e172e420561565c8177c28ac0f62017f8810ffb80e6cc327008025a0e9c0b380c68f040ae7affefd11979f5ed18ae82c00e46aa3238857c372a358eca06b26e179dd2f7a7f1601755249f4cff56690c4033553658f0d73e26c36fe7815", "0xf86c0785028fa6ae0082520894098d880c4753d0332ca737aa592332ed2522cd22880d2f09f6558750008026a0963e58027576b3a8930d7d9b4a49253b6e1a2060e259b2102e34a451d375ce87a063f802538d3efed17962c96fcea431388483bbe3860ea9bb3ef01d4781450fbf", "0x02f87601836384348477359400850517683ba883019a28943678fce4028b6745eb04fa010d9c8e4b36d6288c872b0f1366ad800080c080a0b6b7aba1954160d081b2c8612e039518b9c46cd7df838b405a03f927ad196158a071d2fb6813e5b5184def6bd90fb5f29e0c52671dea433a7decb289560a58416e"]`
 
 	byteTxs := make([]*common.HexBytes, 0, 2)
 	err := json.Unmarshal([]byte(raw), &byteTxs)

mev-boost-relay/services/api/service.go

@@ -70,7 +70,7 @@ var (
 	pathGetHeaderWithProofs = "/eth/v1/builder/header_with_proofs/{slot:[0-9]+}/{parent_hash:0x[a-fA-F0-9]+}/{pubkey:0x[a-fA-F0-9]+}"
 	pathGetPayload          = "/eth/v1/builder/blinded_blocks"
 	// BOLT: allow relay to receive constraints from the proposer
-	pathSubmitContraints = "/eth/v1/builder/constraints"
+	pathSubmitConstraints = "/eth/v1/builder/constraints"
 
 	// Block builder API
 	pathBuilderGetValidators = "/relay/v1/builder/validators"
@@ -358,7 +358,7 @@ func (api *RelayAPI) getRouter() http.Handler {
 		r.HandleFunc(pathGetHeader, api.handleGetHeader).Methods(http.MethodGet)
 		r.HandleFunc(pathGetHeaderWithProofs, api.handleGetHeaderWithProofs).Methods(http.MethodGet)
 		r.HandleFunc(pathGetPayload, api.handleGetPayload).Methods(http.MethodPost)
-		r.HandleFunc(pathSubmitContraints, api.handleSubmitConstraints).Methods(http.MethodPost)
+		r.HandleFunc(pathSubmitConstraints, api.handleSubmitConstraints).Methods(http.MethodPost)
 	}
 
 	// Builder API

mev-boost/server/mock_relay.go

@@ -3,7 +3,6 @@ package server
 import (
 	"encoding/json"
 	"fmt"
-	"math"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -27,6 +26,7 @@ import (
 	"github.com/flashbots/go-boost-utils/ssz"
 	"github.com/gorilla/mux"
 	"github.com/holiman/uint256"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 )
 
@@ -196,8 +196,8 @@ func (m *mockRelay) defaultHandleSubmitConstraint(w http.ResponseWriter, req *ht
 func (m *mockRelay) MakeGetHeaderWithConstraintsResponse(value uint64, blockHash, parentHash, publicKey string, version spec.DataVersion, constraints []struct {
 	tx   Transaction
 	hash phase0.Hash32
-}) *BidWithInclusionProofs {
-
+},
+) *BidWithInclusionProofs {
 	transactions := new(utilbellatrix.ExecutionPayloadTransactions)
 
 	for _, con := range constraints {
@@ -215,25 +215,16 @@ func (m *mockRelay) MakeGetHeaderWithConstraintsResponse(value uint64, blockHash
 	txsRoot := rootNode.Hash()
 
 	bidWithProofs := m.MakeGetHeaderWithProofsResponseWithTxsRoot(value, blockHash, parentHash, publicKey, version, phase0.Root(txsRoot))
-	bidWithProofs.Proofs = make([]*InclusionProof, len(constraints))
-
-	for i, con := range constraints {
-		generalizedIndex := int(math.Pow(float64(2), float64(21))) + i
-
-		proof, err := rootNode.Prove(generalizedIndex)
-		if err != nil {
-			panic(err)
-		}
-
-		merkleProof := new(SerializedMerkleProof)
-		merkleProof.FromFastSszProof(proof)
 
-		bidWithProofs.Proofs[i] = &InclusionProof{
-			TxHash:      con.hash,
-			MerkleProof: merkleProof,
-		}
+	// Calculate the inclusion proof
+	inclusionProof, err := CalculateMerkleMultiProofs(rootNode, constraints)
+	if err != nil {
+		logrus.WithError(err).Error("failed to calculate inclusion proof")
+		return nil
 	}
 
+	bidWithProofs.Proofs = inclusionProof
+
 	return bidWithProofs
 }
 

mev-boost/server/proofs.go

@@ -18,7 +18,7 @@ type BidWithInclusionProofs struct {
 	// The block bid
 	Bid *builderSpec.VersionedSignedBuilderBid `json:"bid"`
 	// The inclusion proofs
-	Proofs []*InclusionProof `json:"proofs"`
+	Proofs *InclusionProof `json:"proofs"`
 }
 
 func (b *BidWithInclusionProofs) String() string {
@@ -72,42 +72,33 @@ func (h *HexBytes) UnmarshalJSON(input []byte) error {
 	return nil
 }
 
-// SerializedMerkleProof contains a serialized Merkle proof of transaction inclusion.
-//   - `Index“ is the generalized index of the included transaction from the SSZ tree
-//     created from the list of transactions.
-//   - `Hashes` are the other branch hashes needed to reconstruct the Merkle proof.
-//
-// For reference, see https://github.com/ethereum/consensus-specs/blob/dev/ssz/simple-serialize.md
-type SerializedMerkleProof struct {
-	Index  int        `json:"index"`
-	Hashes []HexBytes `ssz-size:"dynamic" json:"hashes"`
+// InclusionProof is a Merkle Multiproof of inclusion of a set of TransactionHashes
+type InclusionProof struct {
+	TransactionHashes  []phase0.Hash32 `json:"transaction_hashes"`
+	GeneralizedIndexes []uint64        `json:"generalized_indexes"`
+	MerkleHashes       []*HexBytes     `json:"merkle_hashes"`
 }
 
-func (s *SerializedMerkleProof) FromFastSszProof(p *fastSsz.Proof) {
-	s.Index = p.Index
-	s.Hashes = make([]HexBytes, len(p.Hashes))
-	for i, h := range p.Hashes {
-		s.Hashes[i] = h
+// InclusionProofFromMultiProof converts a fastssz.Multiproof into an InclusionProof, without
+// filling the TransactionHashes
+func InclusionProofFromMultiProof(mp *fastSsz.Multiproof) *InclusionProof {
+	merkleHashes := make([]*HexBytes, len(mp.Hashes))
+	for i, h := range mp.Hashes {
+		merkleHashes[i] = new(HexBytes)
+		*(merkleHashes[i]) = h
 	}
-}
 
-// ToFastSszProof converts a SerializedMerkleProof to a fastssz.Proof.
-func (s *SerializedMerkleProof) ToFastSszProof(leaf []byte) *fastSsz.Proof {
-	p := &fastSsz.Proof{
-		Index:  s.Index,
-		Leaf:   leaf,
-		Hashes: make([][]byte, len(s.Hashes)),
+	leaves := make([]*HexBytes, len(mp.Leaves))
+	for i, h := range mp.Leaves {
+		leaves[i] = new(HexBytes)
+		*(leaves[i]) = h
 	}
-	for i, h := range s.Hashes {
-		p.Hashes[i] = h
+	generalIndexes := make([]uint64, len(mp.Indices))
+	for i, idx := range mp.Indices {
+		generalIndexes[i] = uint64(idx)
+	}
+	return &InclusionProof{
+		MerkleHashes:       merkleHashes,
+		GeneralizedIndexes: generalIndexes,
 	}
-	return p
-}
-
-// InclusionProof is a Merkle inclusion proof for a transaction hash.
-type InclusionProof struct {
-	// The transaction hash of the preconfirmation
-	TxHash phase0.Hash32 `ssz-size:"32" json:"txHash"`
-	// The Merkle proof of the preconfirmation
-	MerkleProof *SerializedMerkleProof `json:"merkleProof"`
 }

mev-boost/server/service.go

@@ -22,7 +22,6 @@ import (
 	eth2ApiV1Capella "github.com/attestantio/go-eth2-client/api/v1/capella"
 	eth2ApiV1Deneb "github.com/attestantio/go-eth2-client/api/v1/deneb"
 	"github.com/attestantio/go-eth2-client/spec/phase0"
-	"github.com/ethereum/go-ethereum/common"
 	fastSsz "github.com/ferranbt/fastssz"
 	"github.com/flashbots/go-boost-utils/ssz"
 	"github.com/flashbots/go-boost-utils/types"
@@ -337,8 +336,8 @@ func (m *BoostService) handleRegisterValidator(w http.ResponseWriter, req *http.
 	m.respondError(w, http.StatusBadGateway, errNoSuccessfulRelayResponse.Error())
 }
 
-// verifyConstraintProofs verifies the proofs against the constraints, and returns an error if the proofs are invalid.
-func (m *BoostService) verifyConstraintProofs(responsePayload *BidWithInclusionProofs, slot uint64) error {
+// verifyInclusionProof verifies the proofs against the constraints, and returns an error if the proofs are invalid.
+func (m *BoostService) verifyInclusionProof(responsePayload *BidWithInclusionProofs, slot uint64) error {
 	log := m.log.WithFields(logrus.Fields{})
 
 	// BOLT: get constraints for the slot
@@ -348,90 +347,74 @@ func (m *BoostService) verifyConstraintProofs(responsePayload *BidWithInclusionP
 		return errMissingConstraint
 	}
 
-	if len(responsePayload.Proofs) != len(inclusionConstraints) {
-		log.Warnf("[BOLT]: Proof verification failed - number of preconfirmations mismatch: proofs %d != constraints %d",
-			len(responsePayload.Proofs), len(inclusionConstraints))
+	if responsePayload.Proofs == nil {
+		return errNilProof
+	}
+
+	if len(responsePayload.Proofs.TransactionHashes) != len(inclusionConstraints) {
 		return errMismatchProofSize
 	}
 
-	// BOLT: verify preconfirmation inclusion proofs. If they don't match, we don't consider the bid to be valid.
-	if responsePayload.Proofs != nil {
-		// BOLT: remove unnecessary fields while logging
-		log.WithFields(logrus.Fields{})
+	log.Infof("[BOLT]: Verifying merkle multiproofs for %d transactions", len(responsePayload.Proofs.TransactionHashes))
+
+	transactionsRoot, err := responsePayload.Bid.TransactionsRoot()
+	if err != nil {
+		return errInvalidRoot
+	}
+
+	leaves := make([][]byte, len(inclusionConstraints))
+	i := 0
 
-		log.WithField("len", len(responsePayload.Proofs)).Info("[BOLT]: Verifying constraint proofs")
+	for hash, constraint := range inclusionConstraints {
+		if len(constraint.Tx) == 0 {
+			log.Warnf("[BOLT]: Raw tx is empty for constraint tx hash %s", hash)
+			continue
+		}
 
-		transactionsRoot, err := responsePayload.Bid.TransactionsRoot()
+		// Compute the hash tree root for the raw preconfirmed transaction
+		// and use it as "Leaf" in the proof to be verified against
+		tx := Transaction(constraint.Tx)
+		txHashTreeRoot, err := tx.HashTreeRoot()
 		if err != nil {
-			log.WithError(err).Error("[BOLT]: error getting tx root from bid")
 			return errInvalidRoot
 		}
 
-		for _, proof := range responsePayload.Proofs {
-			if proof == nil {
-				log.Warn("[BOLT]: Nil proof!")
-				return errNilProof
-			}
-
-			// Find the constraint associated with this transaction in the cache
-			constraint, ok := m.constraints.FindTransactionByHash(common.HexToHash(proof.TxHash.String()))
-			if !ok {
-				log.Warnf("[BOLT]: Tx hash %s not found in constraints", proof.TxHash.String())
-				// We don't actually have to return an error here, the relay just provided a proof that was unnecessary
-				continue
-			}
-
-			rawTx := constraint.Tx
-
-			log.Infof("[BOLT]: Raw tx: %x", rawTx)
-
-			if len(rawTx) == 0 {
-				log.Warnf("[BOLT]: Raw tx is empty for tx hash %s", proof.TxHash.String())
-				continue
-			}
-
-			// Compute the hash tree root for the raw preconfirmed transaction
-			// and use it as "Leaf" in the proof to be verified against
-			txHashTreeRoot, err := rawTx.HashTreeRoot()
-			if err != nil {
-				log.WithError(err).Error("[BOLT]: error getting tx hash tree root")
-				return errInvalidRoot
-			}
-
-			log.Infof("[BOLT]: Tx hash tree root: %x", txHashTreeRoot)
-
-			// Verify the proof
-			sszProof := proof.MerkleProof.ToFastSszProof(txHashTreeRoot[:])
-
-			log.Infof("[BOLT]: Fast sszProof index: %d", sszProof.Index)
-			log.Infof("[BOLT]: Fast sszProof hashes: %x", sszProof.Hashes)
-			log.Infof("[BOLT]: Fast sszProof leaf: %x. Raw tx: %x", sszProof.Leaf, rawTx)
+		leaves[i] = txHashTreeRoot[:]
+		i++
+	}
 
-			currentTime := time.Now()
-			ok, err = fastSsz.VerifyProof(transactionsRoot[:], sszProof)
-			elapsed := time.Since(currentTime)
+	hashes := make([][]byte, len(responsePayload.Proofs.MerkleHashes))
+	for i, hash := range responsePayload.Proofs.MerkleHashes {
+		hashes[i] = []byte(*hash)
+	}
+	indexes := make([]int, len(responsePayload.Proofs.GeneralizedIndexes))
+	for i, index := range responsePayload.Proofs.GeneralizedIndexes {
+		indexes[i] = int(index)
+	}
 
-			if err != nil {
-				log.WithError(err).Error("error verifying merkle proof")
-				return err
-			}
+	currentTime := time.Now()
+	ok, err := fastSsz.VerifyMultiproof(transactionsRoot[:], hashes, leaves, indexes)
+	elapsed := time.Since(currentTime)
+	if err != nil {
+		log.WithError(err).Error("error verifying merkle proof")
+		return err
+	}
 
-			if !ok {
-				log.Error("[BOLT]: proof verification failed: 'not ok' for tx hash: ", proof.TxHash.String())
+	if !ok {
+		log.Error("[BOLT]: proof verification failed")
 
-				// BOLT: send event to web demo
-				message := fmt.Sprintf("failed to verify merkle proof for tx_hash %s", proof.TxHash.String())
-				EmitBoltDemoEvent(message)
+		// BOLT: send event to web demo
+		message := fmt.Sprintf("failed to verify merkle proof for slot %d", slot)
+		EmitBoltDemoEvent(message)
 
-				return errInvalidProofs
-			} else {
-				log.Info(fmt.Sprintf("[BOLT]: Merkle proof verified for tx hash %s in %s", proof.TxHash.String(), elapsed))
+		return errInvalidProofs
+	} else {
+		log.Info(fmt.Sprintf("[BOLT]: merkle proof verified in %s", elapsed))
 
-				// BOLT: send event to web demo
-				message := fmt.Sprintf("verified merkle proof for tx: %s in %v", proof.TxHash.String(), elapsed)
-				EmitBoltDemoEvent(message)
-			}
-		}
+		// BOLT: send event to web demo
+		// verified merkle proof for tx: %s in %v", proof.TxHash.String(), elapsed)
+		message := fmt.Sprintf("verified merkle proof for slot %d in %v", slot, elapsed)
+		EmitBoltDemoEvent(message)
 	}
 
 	return nil
@@ -875,7 +858,7 @@ func (m *BoostService) handleGetHeaderWithProofs(w http.ResponseWriter, req *htt
 			// BOLT: verify preconfirmation inclusion proofs. If they don't match, we don't consider the bid to be valid.
 			if responsePayload.Proofs != nil {
 				// BOLT: verify the proofs against the constraints. If they don't match, we don't consider the bid to be valid.
-				if err := m.verifyConstraintProofs(responsePayload, slotUint); err != nil {
+				if err := m.verifyInclusionProof(responsePayload, slotUint); err != nil {
 					log.Warnf("[BOLT]: Proof verification failed for relay %s: %s", relay.URL, err)
 					return
 				}

mev-boost/server/service_test.go

@@ -314,7 +314,7 @@ func TestRegisterValidator(t *testing.T) {
 func TestParseConstraints(t *testing.T) {
 	jsonStr := `[{
 		"message": {
-			"validatorIndex": 12345,
+			"validator_index": 12345,
 			"slot": 8978583,
 			"constraints": [{
 				"tx": "0x02f871018304a5758085025ff11caf82565f94388c818ca8b9251b393131c08a736a67ccb1929787a41bb7ee22b41380c001a0c8630f734aba7acb4275a8f3b0ce831cf0c7c487fd49ee7bcca26ac622a28939a04c3745096fa0130a188fa249289fd9e60f9d6360854820dba22ae779ea6f573f",