Merge branch 'main' of https://github.com/kurtosis-tech/ethereum-package

into add-adminer
ethpandaops · Oct 30, 2023 · 6cde893 · 6cde893
2 parents 8f3f76d + a455213
commit 6cde893
Show file tree

Hide file tree

Showing 19 changed files with 3,901 additions and 10 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -103,6 +103,7 @@ jobs:
       - run: kurtosis run ${PWD} "$(cat ./.github/tests/mev.yaml)"
 
   mix_with_tools_k8s:
+    resource_class: xlarge
     executor: ubuntu_vm
     steps:
       - checkout

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -3,22 +3,24 @@ name: Nightly test Workflow
 on:
   schedule:
     - cron: "0 2 * * *" # This schedules the workflow to run at 02:00 UTC every day
+  workflow_dispatch:
 
 jobs:
   list-yamls:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-      - uses: actions/checkout@4
+      - uses: actions/checkout@v4
       - id: set-matrix
-        run: echo "::set-output name=matrix::$(ls ./.github/tests/*.yaml | jq -R -s -c 'split("\n")[:-1]')"
+        run: echo "matrix=$(ls ./.github/tests/*.yaml | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
   run_with_args:
     needs: list-yamls
     strategy:
       matrix:
           file_name: ${{ fromJson(needs.list-yamls.outputs.matrix) }}
     runs-on: ubuntu-latest
+    continue-on-error: true
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -31,4 +33,19 @@ jobs:
           kurtosis analytics disable
 
       - name: Run Starlark
-        run: kurtosis run ${{ github.workspace }} --args-file ${{ matrix.file_name }}
+        run: |
+          if [ "${{ matrix.file_name }}" != "./.github/tests/mix-with-tools-mev.yaml" ]; then
+            kurtosis run ${{ github.workspace }} --args-file ${{ matrix.file_name }}
+          else
+            echo "Skipping ./.github/tests/mix-with-tools-mev.yaml"
+          fi
+
+      - name: Notify
+        if: cancelled() || failure()
+        uses: Ilshidur/action-discord@master
+        env:
+          DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
+        with:
+          args: "The nightly test for ${{matrix.file_name}} on ethereum-package has failed find it here ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+
diff --git a/.github/workflows/per-pr.yml b/.github/workflows/per-pr.yml
@@ -52,7 +52,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Setup Kurtosis
         run: |

diff --git a/README.md b/README.md
@@ -427,6 +427,38 @@ snooper_enabled: true
 
 </details>
 
+## Custom labels for Docker and Kubernetes
+There are 4 custom labels that can be used to identify the nodes in the network. These labels are used to identify the nodes in the network and can be used to run chaos tests on specific nodes. An example for these labels are as follows:
+
+Execution Layer (EL) nodes:
+```sh
+  "com.kurtosistech.custom.ethereum-package-client": "geth",
+  "com.kurtosistech.custom.ethereum-package-client-image": "ethereum-client-go-latest",
+  "com.kurtosistech.custom.ethereum-package-client-type": "execution",
+  "com.kurtosistech.custom.ethereum-package-connected-client": "lighthouse",
+```
+
+Consensus Layer (CL) nodes - Beacon:
+```sh
+  "com.kurtosistech.custom.ethereum-package-client": "lighthouse",
+  "com.kurtosistech.custom.ethereum-package-client-image": "sigp-lighthouse-latest",
+  "com.kurtosistech.custom.ethereum-package-client-type": "beacon",
+  "com.kurtosistech.custom.ethereum-package-connected-client": "geth",
+```
+
+Consensus Layer (CL) nodes - Validator:
+```sh
+  "com.kurtosistech.custom.ethereum-package-client": "lighthouse",
+  "com.kurtosistech.custom.ethereum-package-client-image": "sigp-lighthouse-latest",
+  "com.kurtosistech.custom.ethereum-package-client-type": "validator",
+  "com.kurtosistech.custom.ethereum-package-connected-client": "geth",
+```
+
+`ethereum-package-client` describes which client is running on the node.
+`ethereum-package-client-image` describes the image that is used for the client.
+`ethereum-package-client-type` describes the type of client that is running on the node (`execution`,`beacon` or `validator`).
+`ethereum-package-connected-client` describes the CL/EL client that is connected to the EL/CL client.
+
 ## Proposer Builder Separation (PBS) emulation
 
 To spin up the network of Ethereum nodes with an external block building network (using Flashbot's `mev-boost` protocol), simply use:
@@ -516,6 +548,8 @@ When you're happy with your changes:
 1. Add one of the maintainers of the repo as a "Review Request":
    * `parithosh` (Ethereum Foundation)
    * `barnabasbusa` (Ethereum Foundation)
+   * `pk910` (Ethereum Foundation)
+   * `samcm` (Ethereum Foundation)
    * `h4ck3rk3y` (Kurtosis)
    * `mieubrisse` (Kurtosis)
    * `leederek` (Kurtosis)

diff --git a/src/cl/lighthouse/lighthouse_launcher.star b/src/cl/lighthouse/lighthouse_launcher.star
@@ -165,6 +165,7 @@ def launch(
             image,
             log_level,
             beacon_http_url,
+            el_client_context,
             node_keystore_files,
             v_min_cpu,
             v_max_cpu,
@@ -341,6 +342,12 @@ def get_beacon_config(
         max_cpu=bn_max_cpu,
         min_memory=bn_min_mem,
         max_memory=bn_max_mem,
+        labels=shared_utils.label_maker(
+            constants.CL_CLIENT_TYPE.lighthouse,
+            constants.CLIENT_TYPES.cl,
+            image,
+            el_client_context.client_name,
+        ),
     )
 
 
@@ -349,6 +356,7 @@ def get_validator_config(
     image,
     log_level,
     beacon_client_http_url,
+    el_client_context,
     node_keystore_files,
     v_min_cpu,
     v_max_cpu,
@@ -407,6 +415,12 @@ def get_validator_config(
         max_cpu=v_max_cpu,
         min_memory=v_min_mem,
         max_memory=v_max_mem,
+        labels=shared_utils.label_maker(
+            constants.CL_CLIENT_TYPE.lighthouse,
+            constants.CLIENT_TYPES.validator,
+            image,
+            el_client_context.client_name,
+        ),
     )
 
 

diff --git a/src/cl/lodestar/lodestar_launcher.star b/src/cl/lodestar/lodestar_launcher.star
@@ -141,6 +141,7 @@ def launch(
             image,
             log_level,
             beacon_http_url,
+            el_client_context,
             node_keystore_files,
             v_min_cpu,
             v_max_cpu,
@@ -286,6 +287,12 @@ def get_beacon_config(
         max_cpu=bn_max_cpu,
         min_memory=bn_min_mem,
         max_memory=bn_max_mem,
+        labels=shared_utils.label_maker(
+            constants.CL_CLIENT_TYPE.lodestar,
+            constants.CLIENT_TYPES.cl,
+            image,
+            el_client_context.client_name,
+        ),
     )
 
 
@@ -295,6 +302,7 @@ def get_validator_config(
     image,
     log_level,
     beacon_client_http_url,
+    el_client_context,
     node_keystore_files,
     v_min_cpu,
     v_max_cpu,
@@ -351,6 +359,12 @@ def get_validator_config(
         max_cpu=v_max_cpu,
         min_memory=v_min_mem,
         max_memory=v_max_mem,
+        labels=shared_utils.label_maker(
+            constants.CL_CLIENT_TYPE.lodestar,
+            constants.CLIENT_TYPES.validator,
+            image,
+            el_client_context.client_name,
+        ),
     )
 
 

diff --git a/src/cl/nimbus/nimbus_launcher.star b/src/cl/nimbus/nimbus_launcher.star
@@ -315,6 +315,12 @@ def get_config(
         max_cpu=bn_max_cpu,
         min_memory=bn_min_mem,
         max_memory=bn_max_mem,
+        labels=shared_utils.label_maker(
+            constants.CL_CLIENT_TYPE.nimbus,
+            constants.CLIENT_TYPES.cl,
+            image,
+            el_client_context.client_name,
+        ),
     )
 
 

diff --git a/src/cl/prysm/prysm_launcher.star b/src/cl/prysm/prysm_launcher.star
@@ -169,6 +169,7 @@ def launch(
             log_level,
             beacon_rpc_endpoint,
             beacon_http_endpoint,
+            el_client_context,
             node_keystore_files,
             v_min_cpu,
             v_max_cpu,
@@ -315,6 +316,12 @@ def get_beacon_config(
         max_cpu=bn_max_cpu,
         min_memory=bn_min_mem,
         max_memory=bn_max_mem,
+        labels=shared_utils.label_maker(
+            constants.CL_CLIENT_TYPE.prysm,
+            constants.CLIENT_TYPES.cl,
+            beacon_image,
+            el_client_context.client_name,
+        ),
     )
 
 
@@ -325,6 +332,7 @@ def get_validator_config(
     log_level,
     beacon_rpc_endpoint,
     beacon_http_endpoint,
+    el_client_context,
     node_keystore_files,
     v_min_cpu,
     v_max_cpu,
@@ -382,6 +390,12 @@ def get_validator_config(
         max_cpu=v_max_cpu,
         min_memory=v_min_mem,
         max_memory=v_max_mem,
+        labels=shared_utils.label_maker(
+            constants.CL_CLIENT_TYPE.prysm,
+            constants.CLIENT_TYPES.validator,
+            validator_image,
+            el_client_context.client_name,
+        ),
     )
 
 

diff --git a/src/cl/teku/teku_launcher.star b/src/cl/teku/teku_launcher.star
@@ -320,6 +320,12 @@ def get_config(
         max_cpu=bn_max_cpu,
         min_memory=bn_min_mem,
         max_memory=bn_max_mem,
+        labels=shared_utils.label_maker(
+            constants.CL_CLIENT_TYPE.teku,
+            constants.CLIENT_TYPES.cl,
+            image,
+            el_client_context.client_name,
+        ),
     )
 
 

diff --git a/src/el/besu/besu_launcher.star b/src/el/besu/besu_launcher.star
@@ -29,7 +29,7 @@ TCP_DISCOVERY_PORT_ID = "tcp-discovery"
 UDP_DISCOVERY_PORT_ID = "udp-discovery"
 ENGINE_HTTP_RPC_PORT_ID = "engine-rpc"
 METRICS_PORT_ID = "metrics"
-
+JAVA_OPTS = {"JAVA_OPTS": "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n"}
 PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
 
 USED_PORTS = {
@@ -84,11 +84,14 @@ def launch(
     el_min_mem = int(el_min_mem) if int(el_min_mem) > 0 else EXECUTION_MIN_MEMORY
     el_max_mem = int(el_max_mem) if int(el_max_mem) > 0 else EXECUTION_MAX_MEMORY
 
+    cl_client_name = service_name.split("-")[3]
+
     config = get_config(
         launcher.network_id,
         launcher.el_cl_genesis_data,
         image,
         existing_el_clients,
+        cl_client_name,
         log_level,
         el_min_cpu,
         el_max_cpu,
@@ -125,6 +128,7 @@ def get_config(
     el_cl_genesis_data,
     image,
     existing_el_clients,
+    cl_client_name,
     log_level,
     el_min_cpu,
     el_max_cpu,
@@ -183,20 +187,27 @@ def get_config(
 
     cmd_str = " ".join(cmd)
 
+    extra_env_vars = extra_env_vars | JAVA_OPTS
     return ServiceConfig(
         image=image,
         ports=USED_PORTS,
         cmd=[cmd_str],
         files={
             constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
         },
+        env_vars=extra_env_vars,
         entrypoint=ENTRYPOINT_ARGS,
         private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
         min_cpu=el_min_cpu,
         max_cpu=el_max_cpu,
         min_memory=el_min_mem,
         max_memory=el_max_mem,
-        env_vars=extra_env_vars,
+        labels=shared_utils.label_maker(
+            constants.EL_CLIENT_TYPE.besu,
+            constants.CLIENT_TYPES.el,
+            image,
+            cl_client_name,
+        ),
     )
 
 

diff --git a/src/el/erigon/erigon_launcher.star b/src/el/erigon/erigon_launcher.star
@@ -85,11 +85,14 @@ def launch(
     el_min_mem = el_min_mem if int(el_min_mem) > 0 else EXECUTION_MIN_MEMORY
     el_max_mem = el_max_mem if int(el_max_mem) > 0 else EXECUTION_MAX_MEMORY
 
+    cl_client_name = service_name.split("-")[3]
+
     config = get_config(
         launcher.network_id,
         launcher.el_cl_genesis_data,
         image,
         existing_el_clients,
+        cl_client_name,
         log_level,
         el_min_cpu,
         el_max_cpu,
@@ -128,6 +131,7 @@ def get_config(
     el_cl_genesis_data,
     image,
     existing_el_clients,
+    cl_client_name,
     verbosity_level,
     el_min_cpu,
     el_max_cpu,
@@ -209,6 +213,12 @@ def get_config(
         min_memory=el_min_mem,
         max_memory=el_max_mem,
         env_vars=extra_env_vars,
+        labels=shared_utils.label_maker(
+            constants.EL_CLIENT_TYPE.erigon,
+            constants.CLIENT_TYPES.el,
+            image,
+            cl_client_name,
+        ),
     )
 
 

diff --git a/src/el/ethereumjs/ethereumjs_launcher.star b/src/el/ethereumjs/ethereumjs_launcher.star
@@ -89,10 +89,13 @@ def launch(
     el_min_mem = el_min_mem if int(el_min_mem) > 0 else EXECUTION_MIN_MEMORY
     el_max_mem = el_max_mem if int(el_max_mem) > 0 else EXECUTION_MAX_MEMORY
 
+    cl_client_name = service_name.split("-")[3]
+
     config = get_config(
         launcher.el_cl_genesis_data,
         image,
         existing_el_clients,
+        cl_client_name,
         log_level,
         el_min_cpu,
         el_max_cpu,
@@ -127,6 +130,7 @@ def get_config(
     el_cl_genesis_data,
     image,
     existing_el_clients,
+    cl_client_name,
     verbosity_level,
     el_min_cpu,
     el_max_cpu,
@@ -189,6 +193,12 @@ def get_config(
         min_memory=el_min_mem,
         max_memory=el_max_mem,
         env_vars=extra_env_vars,
+        labels=shared_utils.label_maker(
+            constants.EL_CLIENT_TYPE.ethereumjs,
+            constants.CLIENT_TYPES.el,
+            image,
+            cl_client_name,
+        ),
     )