Merge branch 'main' into add-adminer

ethpandaops · Oct 30, 2023 · ed4c8f3 · ed4c8f3
2 parents 1b64b1d + 8aa88e3
commit ed4c8f3
Show file tree

Hide file tree

Showing 18 changed files with 3,891 additions and 8 deletions.
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -10,15 +10,16 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-      - uses: actions/checkout@4
+      - uses: actions/checkout@v4
       - id: set-matrix
-        run: echo "::set-output name=matrix::$(ls ./.github/tests/*.yaml | jq -R -s -c 'split("\n")[:-1]')"
+        run: echo "matrix=$(ls ./.github/tests/*.yaml | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
   run_with_args:
     needs: list-yamls
     strategy:
       matrix:
           file_name: ${{ fromJson(needs.list-yamls.outputs.matrix) }}
     runs-on: ubuntu-latest
+    continue-on-error: true
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -32,3 +33,12 @@ jobs:
 
       - name: Run Starlark
         run: kurtosis run ${{ github.workspace }} --args-file ${{ matrix.file_name }}
+
+      - name: Notify
+        if: cancelled() || failure()
+        uses: Ilshidur/action-discord@master
+        env:
+          DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
+        with:
+          args: "The nightly test for ${{matrix.file_name}} on ethereum-package has failed find it here ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
diff --git a/.github/workflows/per-pr.yml b/.github/workflows/per-pr.yml
@@ -52,7 +52,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Setup Kurtosis
         run: |

diff --git a/README.md b/README.md
@@ -427,6 +427,38 @@ snooper_enabled: true
 
 </details>
 
+## Custom labels for Docker and Kubernetes
+There are 4 custom labels that can be used to identify the nodes in the network. These labels are used to identify the nodes in the network and can be used to run chaos tests on specific nodes. An example for these labels are as follows:
+
+Execution Layer (EL) nodes:
+```sh
+  "com.kurtosistech.custom.ethereum-package-client": "geth",
+  "com.kurtosistech.custom.ethereum-package-client-image": "ethereum-client-go-latest",
+  "com.kurtosistech.custom.ethereum-package-client-type": "execution",
+  "com.kurtosistech.custom.ethereum-package-connected-client": "lighthouse",
+```
+
+Consensus Layer (CL) nodes - Beacon:
+```sh
+  "com.kurtosistech.custom.ethereum-package-client": "lighthouse",
+  "com.kurtosistech.custom.ethereum-package-client-image": "sigp-lighthouse-latest",
+  "com.kurtosistech.custom.ethereum-package-client-type": "beacon",
+  "com.kurtosistech.custom.ethereum-package-connected-client": "geth",
+```
+
+Consensus Layer (CL) nodes - Validator:
+```sh
+  "com.kurtosistech.custom.ethereum-package-client": "lighthouse",
+  "com.kurtosistech.custom.ethereum-package-client-image": "sigp-lighthouse-latest",
+  "com.kurtosistech.custom.ethereum-package-client-type": "validator",
+  "com.kurtosistech.custom.ethereum-package-connected-client": "geth",
+```
+
+`ethereum-package-client` describes which client is running on the node.
+`ethereum-package-client-image` describes the image that is used for the client.
+`ethereum-package-client-type` describes the type of client that is running on the node (`execution`,`beacon` or `validator`).
+`ethereum-package-connected-client` describes the CL/EL client that is connected to the EL/CL client.
+
 ## Proposer Builder Separation (PBS) emulation
 
 To spin up the network of Ethereum nodes with an external block building network (using Flashbot's `mev-boost` protocol), simply use:
@@ -516,6 +548,8 @@ When you're happy with your changes:
 1. Add one of the maintainers of the repo as a "Review Request":
    * `parithosh` (Ethereum Foundation)
    * `barnabasbusa` (Ethereum Foundation)
+   * `pk910` (Ethereum Foundation)
+   * `samcm` (Ethereum Foundation)
    * `h4ck3rk3y` (Kurtosis)
    * `mieubrisse` (Kurtosis)
    * `leederek` (Kurtosis)

diff --git a/src/cl/lighthouse/lighthouse_launcher.star b/src/cl/lighthouse/lighthouse_launcher.star
@@ -165,6 +165,7 @@ def launch(
             image,
             log_level,
             beacon_http_url,
+            el_client_context,
             node_keystore_files,
             v_min_cpu,
             v_max_cpu,
@@ -341,6 +342,12 @@ def get_beacon_config(
         max_cpu=bn_max_cpu,
         min_memory=bn_min_mem,
         max_memory=bn_max_mem,
+        labels=shared_utils.label_maker(
+            constants.CL_CLIENT_TYPE.lighthouse,
+            constants.CLIENT_TYPES.cl,
+            image,
+            el_client_context.client_name,
+        ),
     )
 
 
@@ -349,6 +356,7 @@ def get_validator_config(
     image,
     log_level,
     beacon_client_http_url,
+    el_client_context,
     node_keystore_files,
     v_min_cpu,
     v_max_cpu,
@@ -407,6 +415,12 @@ def get_validator_config(
         max_cpu=v_max_cpu,
         min_memory=v_min_mem,
         max_memory=v_max_mem,
+        labels=shared_utils.label_maker(
+            constants.CL_CLIENT_TYPE.lighthouse,
+            constants.CLIENT_TYPES.validator,
+            image,
+            el_client_context.client_name,
+        ),
     )
 
 

diff --git a/src/cl/lodestar/lodestar_launcher.star b/src/cl/lodestar/lodestar_launcher.star
@@ -141,6 +141,7 @@ def launch(
             image,
             log_level,
             beacon_http_url,
+            el_client_context,
             node_keystore_files,
             v_min_cpu,
             v_max_cpu,
@@ -286,6 +287,12 @@ def get_beacon_config(
         max_cpu=bn_max_cpu,
         min_memory=bn_min_mem,
         max_memory=bn_max_mem,
+        labels=shared_utils.label_maker(
+            constants.CL_CLIENT_TYPE.lodestar,
+            constants.CLIENT_TYPES.cl,
+            image,
+            el_client_context.client_name,
+        ),
     )
 
 
@@ -295,6 +302,7 @@ def get_validator_config(
     image,
     log_level,
     beacon_client_http_url,
+    el_client_context,
     node_keystore_files,
     v_min_cpu,
     v_max_cpu,
@@ -351,6 +359,12 @@ def get_validator_config(
         max_cpu=v_max_cpu,
         min_memory=v_min_mem,
         max_memory=v_max_mem,
+        labels=shared_utils.label_maker(
+            constants.CL_CLIENT_TYPE.lodestar,
+            constants.CLIENT_TYPES.validator,
+            image,
+            el_client_context.client_name,
+        ),
     )
 
 

diff --git a/src/cl/nimbus/nimbus_launcher.star b/src/cl/nimbus/nimbus_launcher.star
@@ -315,6 +315,12 @@ def get_config(
         max_cpu=bn_max_cpu,
         min_memory=bn_min_mem,
         max_memory=bn_max_mem,
+        labels=shared_utils.label_maker(
+            constants.CL_CLIENT_TYPE.nimbus,
+            constants.CLIENT_TYPES.cl,
+            image,
+            el_client_context.client_name,
+        ),
     )
 
 

diff --git a/src/cl/prysm/prysm_launcher.star b/src/cl/prysm/prysm_launcher.star
@@ -169,6 +169,7 @@ def launch(
             log_level,
             beacon_rpc_endpoint,
             beacon_http_endpoint,
+            el_client_context,
             node_keystore_files,
             v_min_cpu,
             v_max_cpu,
@@ -315,6 +316,12 @@ def get_beacon_config(
         max_cpu=bn_max_cpu,
         min_memory=bn_min_mem,
         max_memory=bn_max_mem,
+        labels=shared_utils.label_maker(
+            constants.CL_CLIENT_TYPE.prysm,
+            constants.CLIENT_TYPES.cl,
+            beacon_image,
+            el_client_context.client_name,
+        ),
     )
 
 
@@ -325,6 +332,7 @@ def get_validator_config(
     log_level,
     beacon_rpc_endpoint,
     beacon_http_endpoint,
+    el_client_context,
     node_keystore_files,
     v_min_cpu,
     v_max_cpu,
@@ -382,6 +390,12 @@ def get_validator_config(
         max_cpu=v_max_cpu,
         min_memory=v_min_mem,
         max_memory=v_max_mem,
+        labels=shared_utils.label_maker(
+            constants.CL_CLIENT_TYPE.prysm,
+            constants.CLIENT_TYPES.validator,
+            validator_image,
+            el_client_context.client_name,
+        ),
     )
 
 

diff --git a/src/cl/teku/teku_launcher.star b/src/cl/teku/teku_launcher.star
@@ -320,6 +320,12 @@ def get_config(
         max_cpu=bn_max_cpu,
         min_memory=bn_min_mem,
         max_memory=bn_max_mem,
+        labels=shared_utils.label_maker(
+            constants.CL_CLIENT_TYPE.teku,
+            constants.CLIENT_TYPES.cl,
+            image,
+            el_client_context.client_name,
+        ),
     )
 
 

diff --git a/src/el/besu/besu_launcher.star b/src/el/besu/besu_launcher.star
@@ -29,7 +29,7 @@ TCP_DISCOVERY_PORT_ID = "tcp-discovery"
 UDP_DISCOVERY_PORT_ID = "udp-discovery"
 ENGINE_HTTP_RPC_PORT_ID = "engine-rpc"
 METRICS_PORT_ID = "metrics"
-
+JAVA_OPTS = {"JAVA_OPTS": "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n"}
 PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER"
 
 USED_PORTS = {
@@ -84,11 +84,14 @@ def launch(
     el_min_mem = int(el_min_mem) if int(el_min_mem) > 0 else EXECUTION_MIN_MEMORY
     el_max_mem = int(el_max_mem) if int(el_max_mem) > 0 else EXECUTION_MAX_MEMORY
 
+    cl_client_name = service_name.split("-")[3]
+
     config = get_config(
         launcher.network_id,
         launcher.el_cl_genesis_data,
         image,
         existing_el_clients,
+        cl_client_name,
         log_level,
         el_min_cpu,
         el_max_cpu,
@@ -125,6 +128,7 @@ def get_config(
     el_cl_genesis_data,
     image,
     existing_el_clients,
+    cl_client_name,
     log_level,
     el_min_cpu,
     el_max_cpu,
@@ -183,20 +187,27 @@ def get_config(
 
     cmd_str = " ".join(cmd)
 
+    extra_env_vars = extra_env_vars | JAVA_OPTS
     return ServiceConfig(
         image=image,
         ports=USED_PORTS,
         cmd=[cmd_str],
         files={
             constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
         },
+        env_vars=extra_env_vars,
         entrypoint=ENTRYPOINT_ARGS,
         private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
         min_cpu=el_min_cpu,
         max_cpu=el_max_cpu,
         min_memory=el_min_mem,
         max_memory=el_max_mem,
-        env_vars=extra_env_vars,
+        labels=shared_utils.label_maker(
+            constants.EL_CLIENT_TYPE.besu,
+            constants.CLIENT_TYPES.el,
+            image,
+            cl_client_name,
+        ),
     )
 
 

diff --git a/src/el/erigon/erigon_launcher.star b/src/el/erigon/erigon_launcher.star
@@ -85,11 +85,14 @@ def launch(
     el_min_mem = el_min_mem if int(el_min_mem) > 0 else EXECUTION_MIN_MEMORY
     el_max_mem = el_max_mem if int(el_max_mem) > 0 else EXECUTION_MAX_MEMORY
 
+    cl_client_name = service_name.split("-")[3]
+
     config = get_config(
         launcher.network_id,
         launcher.el_cl_genesis_data,
         image,
         existing_el_clients,
+        cl_client_name,
         log_level,
         el_min_cpu,
         el_max_cpu,
@@ -128,6 +131,7 @@ def get_config(
     el_cl_genesis_data,
     image,
     existing_el_clients,
+    cl_client_name,
     verbosity_level,
     el_min_cpu,
     el_max_cpu,
@@ -209,6 +213,12 @@ def get_config(
         min_memory=el_min_mem,
         max_memory=el_max_mem,
         env_vars=extra_env_vars,
+        labels=shared_utils.label_maker(
+            constants.EL_CLIENT_TYPE.erigon,
+            constants.CLIENT_TYPES.el,
+            image,
+            cl_client_name,
+        ),
     )
 
 

diff --git a/src/el/ethereumjs/ethereumjs_launcher.star b/src/el/ethereumjs/ethereumjs_launcher.star
@@ -89,10 +89,13 @@ def launch(
     el_min_mem = el_min_mem if int(el_min_mem) > 0 else EXECUTION_MIN_MEMORY
     el_max_mem = el_max_mem if int(el_max_mem) > 0 else EXECUTION_MAX_MEMORY
 
+    cl_client_name = service_name.split("-")[3]
+
     config = get_config(
         launcher.el_cl_genesis_data,
         image,
         existing_el_clients,
+        cl_client_name,
         log_level,
         el_min_cpu,
         el_max_cpu,
@@ -127,6 +130,7 @@ def get_config(
     el_cl_genesis_data,
     image,
     existing_el_clients,
+    cl_client_name,
     verbosity_level,
     el_min_cpu,
     el_max_cpu,
@@ -189,6 +193,12 @@ def get_config(
         min_memory=el_min_mem,
         max_memory=el_max_mem,
         env_vars=extra_env_vars,
+        labels=shared_utils.label_maker(
+            constants.EL_CLIENT_TYPE.ethereumjs,
+            constants.CLIENT_TYPES.el,
+            image,
+            cl_client_name,
+        ),
     )