From e87e3b553904b1f59125dd8e8a13ac3a868a3171 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Wed, 14 Feb 2024 13:48:45 +0100 Subject: [PATCH 01/13] adding local snapshot point --- src/participant_network.star | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/participant_network.star b/src/participant_network.star index 13bb1e97a..d73e12c7a 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -141,12 +141,7 @@ def launch_participant_network( config=ServiceConfig( image="alpine:3.19.1", cmd=[ - "apk add --no-cache curl tar zstd && curl -s -L https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/" - + base_network - + "/" - + el_client_type - + "/" - + shadowfork_block + "apk add --no-cache curl tar zstd && http://10.10.101.5:9000/geth-snapshots/holesky.tar.zst" + "/snapshot.tar.zst" + " | tar -I zstd -xvf - -C /data/" + el_client_type From c129a65c4aaf5ec711e7c60b3a921996c71c3b15 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Wed, 14 Feb 2024 13:54:02 +0100 Subject: [PATCH 02/13] fix path --- src/participant_network.star | 1 - 1 file changed, 1 deletion(-) diff --git a/src/participant_network.star b/src/participant_network.star index d73e12c7a..ab0a655ea 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -142,7 +142,6 @@ def launch_participant_network( image="alpine:3.19.1", cmd=[ "apk add --no-cache curl tar zstd && http://10.10.101.5:9000/geth-snapshots/holesky.tar.zst" - + "/snapshot.tar.zst" + " | tar -I zstd -xvf - -C /data/" + el_client_type + "/execution-data" From 151a509fca5979a3bb6ee6362f5e96b79a1d7c32 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Wed, 14 Feb 2024 13:55:10 +0100 Subject: [PATCH 03/13] fix path --- src/participant_network.star | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/participant_network.star b/src/participant_network.star index ab0a655ea..c5bdd3ff8 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -141,7 +141,7 @@ def launch_participant_network( config=ServiceConfig( image="alpine:3.19.1", cmd=[ - "apk add --no-cache curl tar zstd && http://10.10.101.5:9000/geth-snapshots/holesky.tar.zst" + "apk add --no-cache curl tar zstd && curl -s -L http://10.10.101.5:9000/geth-snapshots/holesky.tar.zst" + " | tar -I zstd -xvf - -C /data/" + el_client_type + "/execution-data" From fa2b96af3a0111f09d32cb8e266df8771f4dde9f Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Thu, 29 Feb 2024 21:32:58 +0100 Subject: [PATCH 04/13] update the url --- src/participant_network.star | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/participant_network.star b/src/participant_network.star index bd8e73635..25a7a35bf 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -80,12 +80,14 @@ def launch_participant_network( cancun_time = 0 prague_time = 0 shadowfork_block = "latest" + snapshot_url_base = "https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/" if ( constants.NETWORK_NAME.shadowfork in network_params.network and ("verkle" in network_params.network) and ("holesky" in network_params.network) ): shadowfork_block = "793312" # Hardcodes verkle shadowfork block for holesky + snapshot_url_base = "http://10.10.101.7:9000/snapshots/" # for local testing if ( network_params.network == constants.NETWORK_NAME.kurtosis @@ -108,7 +110,8 @@ def launch_participant_network( ] # overload the network id to match the network name latest_block = plan.run_sh( # fetch the latest block run="mkdir -p /shadowfork && \ - curl -o /shadowfork/latest_block.json https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/" + curl -o /shadowfork/latest_block.json " + + snapshot_url_base + base_network + "/geth/" + shadowfork_block @@ -144,7 +147,14 @@ def launch_participant_network( config=ServiceConfig( image="alpine:3.19.1", cmd=[ - "apk add --no-cache curl tar zstd && curl -s -L http://10.10.101.5:9000/geth-snapshots/holesky.tar.zst" + "apk add --no-cache curl tar zstd && curl -s -L " + + snapshot_url_base + + base_network + + "/" + + el_client_type + + "/" + + shadowfork_block + + "/snapshot.tar.zst" + " | tar -I zstd -xvf - -C /data/" + el_client_type + "/execution-data" From 9f76b0f0fe8fd7a07d9597ce8cfbf663fd48bdd6 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Thu, 29 Feb 2024 21:35:04 +0100 Subject: [PATCH 05/13] fix lint --- src/participant_network.star | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/participant_network.star b/src/participant_network.star index 25a7a35bf..e617f45e4 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -80,14 +80,16 @@ def launch_participant_network( cancun_time = 0 prague_time = 0 shadowfork_block = "latest" - snapshot_url_base = "https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/" + snapshot_url_base = ( + "https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/" + ) if ( constants.NETWORK_NAME.shadowfork in network_params.network and ("verkle" in network_params.network) and ("holesky" in network_params.network) ): shadowfork_block = "793312" # Hardcodes verkle shadowfork block for holesky - snapshot_url_base = "http://10.10.101.7:9000/snapshots/" # for local testing + snapshot_url_base = "http://10.10.101.7:9000/snapshots/" # for local testing if ( network_params.network == constants.NETWORK_NAME.kurtosis From 090a76987b412bfd4abee6e6e4bfdea7f347981c Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Fri, 1 Mar 2024 11:25:38 +0100 Subject: [PATCH 06/13] add localhost as the source of snapshot --- src/participant_network.star | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/participant_network.star b/src/participant_network.star index e617f45e4..02e5349a3 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -89,7 +89,7 @@ def launch_participant_network( and ("holesky" in network_params.network) ): shadowfork_block = "793312" # Hardcodes verkle shadowfork block for holesky - snapshot_url_base = "http://10.10.101.7:9000/snapshots/" # for local testing + snapshot_url_base = "http://localhost:10000/snapshots/" # for local testing if ( network_params.network == constants.NETWORK_NAME.kurtosis From dc178ef6b234e80d312e024ee2bcd9be39dcd9de Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Fri, 1 Mar 2024 11:31:28 +0100 Subject: [PATCH 07/13] add localhost as the source of snapshot --- src/participant_network.star | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/participant_network.star b/src/participant_network.star index 02e5349a3..0ac69327f 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -89,7 +89,7 @@ def launch_participant_network( and ("holesky" in network_params.network) ): shadowfork_block = "793312" # Hardcodes verkle shadowfork block for holesky - snapshot_url_base = "http://localhost:10000/snapshots/" # for local testing + snapshot_url_base = "http://127.0.0.1:10000/snapshots/" # for local testing if ( network_params.network == constants.NETWORK_NAME.kurtosis From d008a11b41cc3e5ad0b83d1a09948cadbd149b03 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Fri, 1 Mar 2024 11:32:20 +0100 Subject: [PATCH 08/13] add localhost as the source of snapshot --- src/participant_network.star | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/participant_network.star b/src/participant_network.star index 0ac69327f..9d2494a6f 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -89,7 +89,7 @@ def launch_participant_network( and ("holesky" in network_params.network) ): shadowfork_block = "793312" # Hardcodes verkle shadowfork block for holesky - snapshot_url_base = "http://127.0.0.1:10000/snapshots/" # for local testing + snapshot_url_base = "http://10.10.101.21:10000/snapshots/" # for local testing if ( network_params.network == constants.NETWORK_NAME.kurtosis From 1acb16da644cf2b1cc4896507dc47059fec22f5c Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Fri, 1 Mar 2024 11:37:16 +0100 Subject: [PATCH 09/13] make genesis fast again --- .github/tests/holesky-shadowfork-verkle.yaml_norun | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/tests/holesky-shadowfork-verkle.yaml_norun b/.github/tests/holesky-shadowfork-verkle.yaml_norun index 84c79cad6..61f50a784 100644 --- a/.github/tests/holesky-shadowfork-verkle.yaml_norun +++ b/.github/tests/holesky-shadowfork-verkle.yaml_norun @@ -9,8 +9,8 @@ participants: cl_client_image: ethpandaops/lodestar:g11tech-verge-815364b network_params: electra_fork_epoch: 1 + seconds_per_slot: 3 network: holesky-shadowfork-verkle - genesis_delay: 300 additional_services: - dora snooper_enabled: true From f9b51e6988e3df86742807560ae1d17b74815dea Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Tue, 5 Mar 2024 10:33:05 +0100 Subject: [PATCH 10/13] make url configurable --- .github/tests/holesky-shadowfork-verkle.yaml_norun | 1 + README.md | 7 +++++++ src/package_io/input_parser.star | 2 ++ src/participant_network.star | 8 ++------ 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/tests/holesky-shadowfork-verkle.yaml_norun b/.github/tests/holesky-shadowfork-verkle.yaml_norun index 61f50a784..cbd9b361b 100644 --- a/.github/tests/holesky-shadowfork-verkle.yaml_norun +++ b/.github/tests/holesky-shadowfork-verkle.yaml_norun @@ -11,6 +11,7 @@ network_params: electra_fork_epoch: 1 seconds_per_slot: 3 network: holesky-shadowfork-verkle + #network_sync_base_url: localhost:10000/snapshots/ additional_services: - dora snooper_enabled: true diff --git a/README.md b/README.md index b4eff39c2..23d116a8e 100644 --- a/README.md +++ b/README.md @@ -422,6 +422,13 @@ network_params: # Defaults to 256 epoch ~27 hours shard_committee_period: 256 + # Network sync base url for syncing public networks from a custom snapshot (mostly useful for shadowforks) + # Defaults to "https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/ + # If you have a local snapshot, you can set this to the local url: + # network_snapshot_url_base = "http://10.10.101.21:10000/snapshots/" + # The snapshots are taken with https://github.com/ethpandaops/snapshotter + network_sync_base_url: "https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/" + # Configuration place for transaction spammer - https:#github.com/MariusVanDerWijden/tx-fuzz tx_spammer_params: # A list of optional extra params that will be passed to the TX Spammer container for modifying its behaviour diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index 7b4e050dd..585fc5db9 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -245,6 +245,7 @@ def input_parser(plan, input_args): "min_validator_withdrawability_delay" ], shard_committee_period=result["network_params"]["shard_committee_period"], + network_sync_base_url=result["network_params"]["network_sync_base_url"], ), mev_params=struct( mev_relay_image=result["mev_params"]["mev_relay_image"], @@ -614,6 +615,7 @@ def default_network_params(): "network": "kurtosis", "min_validator_withdrawability_delay": 256, "shard_committee_period": 256, + "network_sync_base_url": "https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/", } diff --git a/src/participant_network.star b/src/participant_network.star index 9d2494a6f..b5b91cc5c 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -80,16 +80,12 @@ def launch_participant_network( cancun_time = 0 prague_time = 0 shadowfork_block = "latest" - snapshot_url_base = ( - "https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/" - ) if ( constants.NETWORK_NAME.shadowfork in network_params.network and ("verkle" in network_params.network) and ("holesky" in network_params.network) ): shadowfork_block = "793312" # Hardcodes verkle shadowfork block for holesky - snapshot_url_base = "http://10.10.101.21:10000/snapshots/" # for local testing if ( network_params.network == constants.NETWORK_NAME.kurtosis @@ -113,7 +109,7 @@ def launch_participant_network( latest_block = plan.run_sh( # fetch the latest block run="mkdir -p /shadowfork && \ curl -o /shadowfork/latest_block.json " - + snapshot_url_base + + network_params.network_sync_base_url + base_network + "/geth/" + shadowfork_block @@ -150,7 +146,7 @@ def launch_participant_network( image="alpine:3.19.1", cmd=[ "apk add --no-cache curl tar zstd && curl -s -L " - + snapshot_url_base + + network_params.network_sync_base_url + base_network + "/" + el_client_type From 19ed47c958a1b555045a5d4c06580468a8418fcc Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Tue, 5 Mar 2024 10:34:06 +0100 Subject: [PATCH 11/13] remove shorter slot tiem --- .github/tests/holesky-shadowfork-verkle.yaml_norun | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/tests/holesky-shadowfork-verkle.yaml_norun b/.github/tests/holesky-shadowfork-verkle.yaml_norun index cbd9b361b..0332da6ae 100644 --- a/.github/tests/holesky-shadowfork-verkle.yaml_norun +++ b/.github/tests/holesky-shadowfork-verkle.yaml_norun @@ -9,7 +9,6 @@ participants: cl_client_image: ethpandaops/lodestar:g11tech-verge-815364b network_params: electra_fork_epoch: 1 - seconds_per_slot: 3 network: holesky-shadowfork-verkle #network_sync_base_url: localhost:10000/snapshots/ additional_services: From 0a5df2d2db63b38da161f084c95c062b20b8e85b Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Tue, 5 Mar 2024 13:13:14 +0100 Subject: [PATCH 12/13] fix example --- README.md | 2 +- network_params.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 23d116a8e..f6b837bc1 100644 --- a/README.md +++ b/README.md @@ -427,7 +427,7 @@ network_params: # If you have a local snapshot, you can set this to the local url: # network_snapshot_url_base = "http://10.10.101.21:10000/snapshots/" # The snapshots are taken with https://github.com/ethpandaops/snapshotter - network_sync_base_url: "https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/" + network_sync_base_url: https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/ # Configuration place for transaction spammer - https:#github.com/MariusVanDerWijden/tx-fuzz tx_spammer_params: diff --git a/network_params.yaml b/network_params.yaml index 35d2078d1..bbe181b0b 100644 --- a/network_params.yaml +++ b/network_params.yaml @@ -58,6 +58,7 @@ network_params: network: kurtosis min_validator_withdrawability_delay: 256 shard_committee_period: 256 + network_sync_base_url: https://ethpandaops-ethereum-node-snapshots.ams3.digitaloceanspaces.com/ additional_services: - tx_spammer From 0e4320b7a9571c3536ea8dacfd834e5bade21a90 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Tue, 5 Mar 2024 13:58:39 +0100 Subject: [PATCH 13/13] Update holesky-shadowfork-verkle.yaml_norun --- .github/tests/holesky-shadowfork-verkle.yaml_norun | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/tests/holesky-shadowfork-verkle.yaml_norun b/.github/tests/holesky-shadowfork-verkle.yaml_norun index 0332da6ae..52b3741d5 100644 --- a/.github/tests/holesky-shadowfork-verkle.yaml_norun +++ b/.github/tests/holesky-shadowfork-verkle.yaml_norun @@ -10,7 +10,6 @@ participants: network_params: electra_fork_epoch: 1 network: holesky-shadowfork-verkle - #network_sync_base_url: localhost:10000/snapshots/ additional_services: - dora snooper_enabled: true