refactor!: participant_network & rename participant fields.

.github/tests/holesky-shadowfork-verkle.yaml_norun

@@ -10,7 +10,6 @@ participants:
 network_params:
   electra_fork_epoch: 1
   network: holesky-shadowfork-verkle
-  genesis_delay: 300
 additional_services:
   - dora
 snooper_enabled: true

.github/tests/sepolia-mix.yaml

@@ -4,6 +4,7 @@ participants:
   - el_client_type: nethermind
     cl_client_type: prysm
   - el_client_type: erigon
+    el_client_image: ethpandaops/erigon:devel-d754b29 # this is a temp fix, till upstream is fixed
     cl_client_type: lighthouse
   - el_client_type: besu
     cl_client_type: lighthouse

README.md

@@ -422,6 +422,13 @@ network_params:
   # Defaults to 256 epoch ~27 hours
   shard_committee_period: 256
 
+  # Network sync base url for syncing public networks from a custom snapshot (mostly useful for shadowforks)
+  # Defaults to "https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/
+  # If you have a local snapshot, you can set this to the local url:
+  # network_snapshot_url_base = "http://10.10.101.21:10000/snapshots/"
+  # The snapshots are taken with https://github.com/ethpandaops/snapshotter
+  network_sync_base_url: https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/
+
 # Configuration place for transaction spammer - https:#github.com/MariusVanDerWijden/tx-fuzz
 tx_spammer_params:
   # A list of optional extra params that will be passed to the TX Spammer container for modifying its behaviour

network_params.yaml

@@ -58,6 +58,7 @@ network_params:
   network: kurtosis
   min_validator_withdrawability_delay: 256
   shard_committee_period: 256
+  network_sync_base_url: https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/
 
 additional_services:
   - tx_spammer

src/network_launcher/devnet.star

@@ -0,0 +1,34 @@
+shared_utils = import_module("../shared_utils/shared_utils.star")
+el_cl_genesis_data = import_module(
+    "../prelaunch_data_generator/el_cl_genesis/el_cl_genesis_data.star"
+)
+
+
+def launch(plan, network, cancun_time, prague_time):
+    # We are running a devnet
+    url = shared_utils.calculate_devnet_url(network)
+    el_cl_genesis_uuid = plan.upload_files(
+        src=url,
+        name="el_cl_genesis",
+    )
+    el_cl_genesis_data_uuid = plan.run_sh(
+        run="mkdir -p /network-configs/ && mv /opt/* /network-configs/",
+        store=[StoreSpec(src="/network-configs/", name="el_cl_genesis_data")],
+        files={"/opt": el_cl_genesis_uuid},
+    )
+    genesis_validators_root = read_file(url + "/genesis_validators_root.txt")
+
+    el_cl_data = el_cl_genesis_data.new_el_cl_genesis_data(
+        el_cl_genesis_data_uuid.files_artifacts[0],
+        genesis_validators_root,
+        cancun_time,
+        prague_time,
+    )
+    final_genesis_timestamp = shared_utils.read_genesis_timestamp_from_config(
+        plan, el_cl_genesis_data_uuid.files_artifacts[0]
+    )
+    network_id = shared_utils.read_genesis_network_id_from_config(
+        plan, el_cl_genesis_data_uuid.files_artifacts[0]
+    )
+    validator_data = None
+    return el_cl_data, final_genesis_timestamp, network_id, validator_data

src/network_launcher/ephemery.star

@@ -0,0 +1,30 @@
+shared_utils = import_module("../shared_utils/shared_utils.star")
+el_cl_genesis_data = import_module(
+    "../prelaunch_data_generator/el_cl_genesis/el_cl_genesis_data.star"
+)
+
+
+def launch(plan, cancun_time, prague_time):
+    el_cl_genesis_data_uuid = plan.run_sh(
+        run="mkdir -p /network-configs/ && \
+            curl -o latest.tar.gz https://ephemery.dev/latest.tar.gz && \
+            tar xvzf latest.tar.gz -C /network-configs && \
+            cat /network-configs/genesis_validators_root.txt",
+        image="badouralix/curl-jq",
+        store=[StoreSpec(src="/network-configs/", name="el_cl_genesis_data")],
+    )
+    genesis_validators_root = el_cl_genesis_data_uuid.output
+    el_cl_data = el_cl_genesis_data.new_el_cl_genesis_data(
+        el_cl_genesis_data_uuid.files_artifacts[0],
+        genesis_validators_root,
+        cancun_time,
+        prague_time,
+    )
+    final_genesis_timestamp = shared_utils.read_genesis_timestamp_from_config(
+        plan, el_cl_genesis_data_uuid.files_artifacts[0]
+    )
+    network_id = shared_utils.read_genesis_network_id_from_config(
+        plan, el_cl_genesis_data_uuid.files_artifacts[0]
+    )
+    validator_data = None
+    return el_cl_data, final_genesis_timestamp, network_id, validator_data

src/network_launcher/kurtosis.star

@@ -0,0 +1,95 @@
+shared_utils = import_module("../shared_utils/shared_utils.star")
+validator_keystores = import_module(
+    "../prelaunch_data_generator/validator_keystores/validator_keystore_generator.star"
+)
+
+constants = import_module("../package_io/constants.star")
+
+# The time that the CL genesis generation step takes to complete, based off what we've seen
+# This is in seconds
+CL_GENESIS_DATA_GENERATION_TIME = 5
+
+# Each CL node takes about this time to start up and start processing blocks, so when we create the CL
+#  genesis data we need to set the genesis timestamp in the future so that nodes don't miss important slots
+# (e.g. Altair fork)
+# TODO(old) Make this client-specific (currently this is Nimbus)
+# This is in seconds
+CL_NODE_STARTUP_TIME = 5
+
+
+def launch(plan, network_params, participants, parallel_keystore_generation):
+    num_participants = len(participants)
+    plan.print("Generating cl validator key stores")
+    validator_data = None
+    if not parallel_keystore_generation:
+        validator_data = validator_keystores.generate_validator_keystores(
+            plan, network_params.preregistered_validator_keys_mnemonic, participants
+        )
+    else:
+        validator_data = validator_keystores.generate_valdiator_keystores_in_parallel(
+            plan,
+            network_params.preregistered_validator_keys_mnemonic,
+            participants,
+        )
+
+    plan.print(json.indent(json.encode(validator_data)))
+
+    # We need to send the same genesis time to both the EL and the CL to ensure that timestamp based forking works as expected
+    final_genesis_timestamp = shared_utils.get_final_genesis_timestamp(
+        plan,
+        network_params.genesis_delay
+        + CL_GENESIS_DATA_GENERATION_TIME
+        + num_participants * CL_NODE_STARTUP_TIME,
+    )
+
+    # if preregistered validator count is 0 (default) then calculate the total number of validators from the participants
+    total_number_of_validator_keys = network_params.preregistered_validator_count
+
+    if network_params.preregistered_validator_count == 0:
+        for participant in participants:
+            total_number_of_validator_keys += participant.validator_count
+
+    plan.print("Generating EL CL data")
+
+    # we are running bellatrix genesis (deprecated) - will be removed in the future
+    if (
+        network_params.capella_fork_epoch > 0
+        and network_params.electra_fork_epoch == None
+    ):
+        ethereum_genesis_generator_image = (
+            constants.ETHEREUM_GENESIS_GENERATOR.bellatrix_genesis
+        )
+    # we are running capella genesis - default behavior
+    elif (
+        network_params.capella_fork_epoch == 0
+        and network_params.electra_fork_epoch == None
+        and network_params.deneb_fork_epoch > 0
+    ):
+        ethereum_genesis_generator_image = (
+            constants.ETHEREUM_GENESIS_GENERATOR.capella_genesis
+        )
+    # we are running deneb genesis -  experimental, soon to become default
+    elif network_params.deneb_fork_epoch == 0:
+        ethereum_genesis_generator_image = (
+            constants.ETHEREUM_GENESIS_GENERATOR.deneb_genesis
+        )
+    # we are running electra - experimental
+    elif network_params.electra_fork_epoch != None:
+        if network_params.electra_fork_epoch == 0:
+            ethereum_genesis_generator_image = (
+                constants.ETHEREUM_GENESIS_GENERATOR.verkle_genesis
+            )
+        else:
+            ethereum_genesis_generator_image = (
+                constants.ETHEREUM_GENESIS_GENERATOR.verkle_support_genesis
+            )
+    else:
+        fail(
+            "Unsupported fork epoch configuration, need to define either capella_fork_epoch, deneb_fork_epoch or electra_fork_epoch"
+        )
+    return (
+        total_number_of_validator_keys,
+        ethereum_genesis_generator_image,
+        final_genesis_timestamp,
+        validator_data,
+    )

src/network_launcher/public_network.star

@@ -0,0 +1,23 @@
+shared_utils = import_module("../shared_utils/shared_utils.star")
+el_cl_genesis_data = import_module(
+    "../prelaunch_data_generator/el_cl_genesis/el_cl_genesis_data.star"
+)
+constants = import_module("../package_io/constants.star")
+
+
+def launch(plan, network, cancun_time, prague_time):
+    # We are running a public network
+    dummy = plan.run_sh(
+        run="mkdir /network-configs",
+        store=[StoreSpec(src="/network-configs/", name="el_cl_genesis_data")],
+    )
+    el_cl_data = el_cl_genesis_data.new_el_cl_genesis_data(
+        dummy.files_artifacts[0],
+        constants.GENESIS_VALIDATORS_ROOT[network],
+        cancun_time,
+        prague_time,
+    )
+    final_genesis_timestamp = constants.GENESIS_TIME[network]
+    network_id = constants.NETWORK_ID[network]
+    validator_data = None
+    return el_cl_data, final_genesis_timestamp, network_id, validator_data

src/network_launcher/shadowfork.star

@@ -0,0 +1,110 @@
+shared_utils = import_module("../shared_utils/shared_utils.star")
+constants = import_module("../package_io/constants.star")
+input_parser = import_module("../package_io/input_parser.star")
+
+
+def shadowfork_prep(
+    plan,
+    network_params,
+    shadowfork_block,
+    participants,
+    global_tolerations,
+    global_node_selectors,
+):
+    base_network = shared_utils.get_network_name(network_params.network)
+    # overload the network name to remove the shadowfork suffix
+    if constants.NETWORK_NAME.ephemery in base_network:
+        chain_id = plan.run_sh(
+            run="curl -s https://ephemery.dev/latest/config.yaml | yq .DEPOSIT_CHAIN_ID | tr -d '\n'",
+            image="linuxserver/yq",
+        )
+        network_id = chain_id.output
+    else:
+        network_id = constants.NETWORK_ID[
+            base_network
+        ]  # overload the network id to match the network name
+    latest_block = plan.run_sh(  # fetch the latest block
+        run="mkdir -p /shadowfork && \
+            curl -o /shadowfork/latest_block.json "
+        + network_params.network_sync_base_url
+        + base_network
+        + "/geth/"
+        + shadowfork_block
+        + "/_snapshot_eth_getBlockByNumber.json",
+        image="badouralix/curl-jq",
+        store=[StoreSpec(src="/shadowfork", name="latest_blocks")],
+    )
+
+    for index, participant in enumerate(participants):
+        tolerations = input_parser.get_client_tolerations(
+            participant.el_tolerations,
+            participant.tolerations,
+            global_tolerations,
+        )
+        node_selectors = input_parser.get_client_node_selectors(
+            participant.node_selectors,
+            global_node_selectors,
+        )
+
+        cl_client_type = participant.cl_client_type
+        el_client_type = participant.el_client_type
+
+        # Zero-pad the index using the calculated zfill value
+        index_str = shared_utils.zfill_custom(index + 1, len(str(len(participants))))
+
+        el_service_name = "el-{0}-{1}-{2}".format(
+            index_str, el_client_type, cl_client_type
+        )
+        shadowfork_data = plan.add_service(
+            name="shadowfork-{0}".format(el_service_name),
+            config=ServiceConfig(
+                image="alpine:3.19.1",
+                cmd=[
+                    "apk add --no-cache curl tar zstd && curl -s -L "
+                    + network_params.network_sync_base_url
+                    + base_network
+                    + "/"
+                    + el_client_type
+                    + "/"
+                    + shadowfork_block
+                    + "/snapshot.tar.zst"
+                    + " | tar -I zstd -xvf - -C /data/"
+                    + el_client_type
+                    + "/execution-data"
+                    + " && touch /tmp/finished"
+                    + " && tail -f /dev/null"
+                ],
+                entrypoint=["/bin/sh", "-c"],
+                files={
+                    "/data/"
+                    + el_client_type
+                    + "/execution-data": Directory(
+                        persistent_key="data-{0}".format(el_service_name),
+                        size=constants.VOLUME_SIZE[base_network][
+                            el_client_type + "_volume_size"
+                        ],
+                    ),
+                },
+                tolerations=tolerations,
+                node_selectors=node_selectors,
+            ),
+        )
+    for index, participant in enumerate(participants):
+        cl_client_type = participant.cl_client_type
+        el_client_type = participant.el_client_type
+
+        # Zero-pad the index using the calculated zfill value
+        index_str = shared_utils.zfill_custom(index + 1, len(str(len(participants))))
+
+        el_service_name = "el-{0}-{1}-{2}".format(
+            index_str, el_client_type, cl_client_type
+        )
+        plan.wait(
+            service_name="shadowfork-{0}".format(el_service_name),
+            recipe=ExecRecipe(command=["cat", "/tmp/finished"]),
+            field="code",
+            assertion="==",
+            target_value=0,
+            interval="1s",
+            timeout="6h",  # 6 hours should be enough for the biggest network
+        )

src/package_io/input_parser.star

@@ -245,6 +245,7 @@ def input_parser(plan, input_args):
                 "min_validator_withdrawability_delay"
             ],
             shard_committee_period=result["network_params"]["shard_committee_period"],
+            network_sync_base_url=result["network_params"]["network_sync_base_url"],
         ),
         mev_params=struct(
             mev_relay_image=result["mev_params"]["mev_relay_image"],
@@ -614,6 +615,7 @@ def default_network_params():
         "network": "kurtosis",
         "min_validator_withdrawability_delay": 256,
         "shard_committee_period": 256,
+        "network_sync_base_url": "https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/",
     }