Merge ethereum/hive/master

clients/besu/besu.sh

@@ -60,12 +60,12 @@ case "$HIVE_LOGLEVEL" in
     4)   LOG=DEBUG ;;
     5)   LOG=TRACE ;;
 esac
-FLAGS="--logging=$LOG"
+FLAGS="--logging=$LOG --data-storage-format=BONSAI"
 
 # Configure the chain.
 jq -f /mapper.jq /genesis.json > /besugenesis.json
 echo -n "Genesis: "; cat /besugenesis.json
-FLAGS="$FLAGS --genesis-file=/besugenesis.json"
+FLAGS="$FLAGS --genesis-file=/besugenesis.json "
 
 # Enable experimental 'berlin' hard-fork features if configured.
 #if [ -n "$HIVE_FORK_BERLIN" ]; then

clients/besu/mapper.jq

@@ -44,7 +44,8 @@ def to_int:
     "berlinBlock": env.HIVE_FORK_BERLIN|to_int,
     "londonBlock": env.HIVE_FORK_LONDON|to_int,
     "parisBlock": env.HIVE_MERGE_BLOCK_ID|to_int,
-    "shanghaiTime": env.HIVE_SHANGHAI_TIMESTAMP|to_int,
     "terminalTotalDifficulty": env.HIVE_TERMINAL_TOTAL_DIFFICULTY|to_int,
+    "shanghaiTime": env.HIVE_SHANGHAI_TIMESTAMP|to_int,
+    "cancunTime": env.HIVE_CANCUN_TIMESTAMP|to_int,
   }|remove_empty
 }

clients/erigon/erigon.sh

@@ -54,6 +54,7 @@ fi
 # Create the data directory.
 mkdir /erigon-hive-datadir
 FLAGS="$FLAGS --datadir /erigon-hive-datadir"
+FLAGS="$FLAGS --db.size.limit 2GB"
 
 # If a specific network ID is requested, use that
 if [ "$HIVE_NETWORK_ID" != "" ]; then
@@ -135,6 +136,6 @@ if [ "$HIVE_TERMINAL_TOTAL_DIFFICULTY" != "" ]; then
 fi
 
 # Launch the main client.
-FLAGS="$FLAGS --nat=none --externalcl"
+FLAGS="$FLAGS --nat=none"
 echo "Running erigon with flags $FLAGS"
 $erigon $FLAGS

clients/ethereumjs/Dockerfile

@@ -1,6 +1,6 @@
 FROM node:16-alpine as build
 RUN apk update && apk add --no-cache bash git jq curl python3 gcc make g++ && rm -rf /var/cache/apk/*
-RUN git clone --depth 1 https://github.com/ethereumjs/ethereumjs-monorepo.git && \
+RUN git clone --depth 1 -b develop-v7 https://github.com/ethereumjs/ethereumjs-monorepo.git && \
     (cd ethereumjs-monorepo && npm i)
 
 RUN cd ethereumjs-monorepo/packages/client && npm ethereumjs --version > /version.txt

clients/ethereumjs/ethereumjs.sh

@@ -64,7 +64,7 @@ cat /genesis.json
 if [ "$HIVE_CLIQUE_PRIVATEKEY" != "" ]; then
     # Create password file.
     echo "Importing clique key..."
-    echo "$HIVE_CLIQUE_PRIVATEKEY" > ./private_key.txt
+    echo -n "$HIVE_CLIQUE_PRIVATEKEY" > ./private_key.txt
     # Ensure password file is used when running ethereumjs in mining mode.
     if [ "$HIVE_MINER" != "" ]; then
         FLAGS="$FLAGS --mine --unlock ./private_key.txt --minerCoinbase $HIVE_MINER"
@@ -84,6 +84,8 @@ if [ -f /chain.rlp ]; then
   echo "Warning: chain.rlp not found."
 fi
 
-FLAGS="$FLAGS --bootnodes=$HIVE_BOOTNODE"
+if [ "$HIVE_BOOTNODE" != "" ]; then
+    FLAGS="$FLAGS --bootnodes=$HIVE_BOOTNODE"
+fi
 echo "Running ethereumjs with flags $FLAGS"
 $ethereumjs $FLAGS

clients/ethereumjs/mapper.jq

@@ -56,5 +56,6 @@ def to_bool:
     "mergeForkBlock": env.HIVE_MERGE_BLOCK_ID|to_int,
     "terminalTotalDifficulty": env.HIVE_TERMINAL_TOTAL_DIFFICULTY|to_int,
     "shanghaiTime": env.HIVE_SHANGHAI_TIMESTAMP|to_int,
+    "cancunTime": env.HIVE_CANCUN_TIMESTAMP|to_int,
   }|remove_empty
 }

clients/go-ethereum/Dockerfile

@@ -40,9 +40,4 @@ ADD genesis.json /genesis.json
 # Export the usual networking ports to allow outside access to the node
 EXPOSE 8545 8546 8547 8551 30303 30303/udp
 
-# Generate the ethash verification caches
-RUN \
- /usr/local/bin/geth makecache     1 ~/.ethereum/geth/ethash && \
- /usr/local/bin/geth makecache 30001 ~/.ethereum/geth/ethash
-
 ENTRYPOINT ["/geth.sh"]

clients/go-ethereum/geth.sh

@@ -58,10 +58,6 @@ fi
 # It doesn't make sense to dial out, use only a pre-set bootnode.
 FLAGS="$FLAGS --bootnodes=$HIVE_BOOTNODE"
 
-if [ "$HIVE_SKIP_POW" != "" ]; then
-    FLAGS="$FLAGS --fakepow"
-fi
-
 # If a specific network ID is requested, use that
 if [ "$HIVE_NETWORK_ID" != "" ]; then
     FLAGS="$FLAGS --networkid $HIVE_NETWORK_ID"
@@ -131,7 +127,7 @@ if [ "$HIVE_CLIQUE_PRIVATEKEY" != "" ]; then
     # Create password file.
     echo "Importing clique key..."
     echo "secret" > /geth-password-file.txt
-    $geth --nousb account import --password /geth-password-file.txt <(echo "$HIVE_CLIQUE_PRIVATEKEY")
+    $geth account import --password /geth-password-file.txt <(echo "$HIVE_CLIQUE_PRIVATEKEY")
 
     # Ensure password file is used when running geth in mining mode.
     if [ "$HIVE_MINER" != "" ]; then
@@ -141,7 +137,7 @@ fi
 
 # Configure any mining operation
 if [ "$HIVE_MINER" != "" ] && [ "$HIVE_NODETYPE" != "light" ]; then
-    FLAGS="$FLAGS --mine --miner.threads 1 --miner.etherbase $HIVE_MINER"
+    FLAGS="$FLAGS --mine --miner.etherbase $HIVE_MINER"
 fi
 if [ "$HIVE_MINER_EXTRA" != "" ]; then
     FLAGS="$FLAGS --miner.extradata $HIVE_MINER_EXTRA"

clients/go-ethereum/mapper.jq

@@ -55,5 +55,6 @@ def to_bool:
     "mergeForkBlock": env.HIVE_MERGE_BLOCK_ID|to_int,
     "terminalTotalDifficulty": env.HIVE_TERMINAL_TOTAL_DIFFICULTY|to_int,
     "shanghaiTime": env.HIVE_SHANGHAI_TIMESTAMP|to_int,
+    "terminalTotalDifficultyPassed": true,
   }|remove_empty
 }

clients/nethermind/Dockerfile

@@ -1,5 +1,7 @@
+ARG user=nethermindeth
+ARG repo=hive
 ARG branch=latest
-FROM nethermindeth/hive:$branch
+FROM $user/$repo:$branch
 
 RUN apt-get update && apt-get install -y wget
 RUN wget https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -O /usr/local/bin/jq && \

clients/nethermind/mapper.jq

@@ -142,6 +142,9 @@ def clique_engine:
     "eip3860TransitionTimestamp": env.HIVE_SHANGHAI_TIMESTAMP|to_hex,
     "eip4895TransitionTimestamp": env.HIVE_SHANGHAI_TIMESTAMP|to_hex,
 
+    # Cancun
+    "eip4844TransitionTimestamp": env.HIVE_CANCUN_TIMESTAMP|to_hex,
+
     # Other chain parameters
     "networkID": env.HIVE_NETWORK_ID|to_hex,
     "chainID": env.HIVE_CHAIN_ID|to_hex,
@@ -160,6 +163,7 @@ def clique_engine:
     "extraData": .extraData,
     "gasLimit": .gasLimit,
     "baseFeePerGas": .baseFeePerGas,
+    "excessDataGas": .excessDataGas,
   },
   "accounts": ((.alloc|with_entries(.key|="0x"+.)) * {
     "0x0000000000000000000000000000000000000001": {

clients/nethermind/nethermind.sh

@@ -62,6 +62,7 @@ jq . /chainspec/test.json
 
 # Generate the config file.
 mkdir /configs
+echo "Supplied genesis state:" 
 jq -n -f /mkconfig.jq > /configs/test.cfg
 
 echo "test.cfg"

simulators/devp2p/main.go

@@ -17,10 +17,18 @@ func main() {
 		Description: "This suite runs Discovery v4 protocol tests.",
 		Tests: []hivesim.AnyTest{
 			hivesim.ClientTestSpec{
-				Role:       "eth1",
-				Parameters: hivesim.Params{"HIVE_LOGLEVEL": "5"},
-				AlwaysRun:  true,
-				Run:        runDiscoveryTest,
+				Role: "eth1",
+				Parameters: hivesim.Params{
+					"HIVE_NETWORK_ID":     "19763",
+					"HIVE_CHAIN_ID":       "19763",
+					"HIVE_FORK_HOMESTEAD": "0",
+					"HIVE_FORK_TANGERINE": "0",
+					"HIVE_FORK_SPURIOUS":  "0",
+					"HIVE_FORK_BYZANTIUM": "0",
+					"HIVE_LOGLEVEL":       "5",
+				},
+				AlwaysRun: true,
+				Run:       runDiscoveryTest,
 			},
 		},
 	}

simulators/eth2/common/builder/builder.go

@@ -2,6 +2,7 @@ package builder
 
 import (
 	api "github.com/ethereum/go-ethereum/beacon/engine"
+	"github.com/ethereum/hive/simulators/eth2/common/builder/types/common"
 	beacon "github.com/protolambda/zrnt/eth2/beacon/common"
 )
 
@@ -10,6 +11,7 @@ type Builder interface {
 	Cancel() error
 	GetBuiltPayloadsCount() int
 	GetSignedBeaconBlockCount() int
+	GetSignedBeaconBlocks() map[beacon.Slot]common.SignedBeaconBlock
 	GetModifiedPayloads() map[beacon.Slot]*api.ExecutableData
 	GetBuiltPayloads() map[beacon.Slot]*api.ExecutableData
 }

simulators/eth2/common/builder/mock/README.md

@@ -2,6 +2,17 @@
 
 Instantiates a server that listens for Builder API (https://github.com/ethereum/builder-specs/) directives and responds with payloads built using an execution client.
 
+Currently, the builder will produce payloads with the following correct fields:
+- PrevRandao
+- Timestamp
+- SuggestedFeeRecipient
+- Withdrawals
+
+For the builder to function properly, the following parameters are necessary:
+- Execution client: Required to build the payloads
+- Beacon client: Required to fetch the state of the previous slot, and calculate, e.g., the prevrandao value
+
+## Mock Payload Building
 The builder can inject modifications into the built payloads at predefined slots by using configurable callbacks:
 - Before sending the ForkchoiceUpdated directive to the execution client, by modifying the payload attributes, using `WithPayloadAttributesModifier` option
 - Before responding with the build payload to the consensus client by modifying the any field in the payload, using `WithPayloadModifier` option
@@ -10,6 +21,23 @@ Both callbacks are supplied with either the `PayloadAttributesV1`/`PayloadAttrib
 
 The callbacks must respond with a boolean indicating whether any modification was performed, and an error, if any.
 
+Predefined invalidation can also be configured by using `WithPayloadInvalidatorAtEpoch`, `WithPayloadInvalidatorAtSlot`, `WithPayloadAttributesInvalidatorAtEpoch` or `WithPayloadAttributesInvalidatorAtSlot`.
+
+## Payload Invalidation Types
+- `state_root`: Inserts a random state root value in the built payload. Payload can only be deemed invalid after the payload has been unblinded
+- `parent_hash`: Inserts a random parent hash value in the built payload. Payload can be deemed invalid without needing to unblind
+- `coinbase`: Inserts a random address as coinbase in the built payload. Payload is not invalid
+- `base_fee`: Increases the base fee value by 1 in the built payload. Payload can only be deemed invalid after the payload has been unblinded
+- `uncle_hash`: Inserts a random uncle hash value in the built payload. Payload can be deemed invalid without needing to unblind
+- `receipt_hash`: Inserts a random receipt hash value in the built payload. Payload can only be deemed invalid after the payload has been unblinded
+
+## Payload Attributes Invalidation Types
+- `remove_withdrawal`: Removes a withdrawal from the correct list of expected withdrawals
+- `extra_withdrawal`: Inserts an extra withdrawal to the correct list of expected withdrawals
+- `withdrawal_address`, `withdrawal_amount`, `withdrawal_validator_index`, `withdrawal_index`: Invalidates a single withdrawal from the correct list of expected withdrawals
+- `timestamp`: Modifies the expected timestamp value of the block (-2 epochs)
+- `prevrandao`/`random`: Modifies the expected prevrandao
+
 The builder can also be configured to insert an error on:
 - `/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey}` using `WithErrorOnHeaderRequest` option
 - `/eth/v1/builder/blinded_blocks` using `WithErrorOnPayloadReveal` option
@@ -18,13 +46,20 @@ Both callbacks are supplied with the beacon slot number of the payload/blinded b
 
 The callback can then use the slot number to determine whether to throw an error or not.
 
-Currently, the builder will produce payloads with the following correct fields:
-- PrevRandao
-- Timestamp
-- SuggestedFeeRecipient
-- Withdrawals
+## Mock Builder REST API
+### Mock Error
+- `DELETE` `/mock/errors/payload_request`: Disables errors on `/eth/v1/builder/header/...`
+- `POST` `/mock/errors/payload_request`: Enables errors on `/eth/v1/builder/header/...`
+- `POST` `/mock/errors/payload_request/<slot|epoch>/{slot/epoch number}`: Enables errors on `/eth/v1/builder/header/...` starting at the slot or epoch specified
+- `DELETE` `/mock/errors/payload_reveal`: Disables errors on `/eth/v1/builder/blinded_blocks`
+- `POST` `/mock/errors/payload_reveal`: Enables errors on `/eth/v1/builder/blinded_blocks`
+- `POST` `/mock/errors/payload_reveal/<slot|epoch>/{slot/epoch number}`: Enables errors on `/eth/v1/builder/blinded_blocks` starting at the slot or epoch specified
 
-For the builder to function properly, the following parameters are necessary:
-- Execution client: Required to build the payloads
-- Beacon client: Required to fetch the state of the previous slot, and calculate, e.g., the prevrandao value
-- Beacon Spec: Required so the builder is aware of fork specific changes in the built payloads, as well as the beacon blocks
+### Mock Built Payloads
+- `DELETE` `/mock/invalid/payload_attributes`: Disables any payload attributes modification
+- `POST` `/mock/invalid/payload_attributes/{type}`: Enables specified [type](#payload-attributes-invalidation-types) payload attributes modification
+- `POST` `/mock/invalid/payload_attributes/{type}/<slot|epoch>/{slot/epoch number}`: Enables specified [type](#payload-attributes-invalidation-types) payload attributes modification starting at the slot or epoch specified
+
+- `DELETE` `/mock/invalid/payload`: Disables any modification to payload built
+- `POST` `/mock/invalid/payload/{type}`: Enables specified [type](#payload-invalidation-types) of modification to payload built
+- `POST` `/mock/invalid/payload/{type}/<slot|epoch>/{slot/epoch number}`: Enables specified [type](#payload-invalidation-types) of modification to payload built starting at the slot or epoch specified