feat: separate validator clients from CL clients (#497)

Separates the validator clients more cleanly from the CL clients. This then allows the use of different combinations of CL/VC clients, e.g. Teku VC with Lodestar CL. The only VC that doesn't work with different beacon nodes is Prysm at the moment. The `use_separate_validator_client` flag defaults to false for CL clients that can run validators in the same process as the CL (preserving the way the `cl_split_mode_enabled` worked before this PR). I believe this can be quite useful to test different VC<->CL combinations for compatibility. --------- Co-authored-by: Barnabas Busa <[email protected]>
ethpandaops · Feb 27, 2024 · 90da2c3 · 90da2c3
1 parent fe2de7e
commit 90da2c3
Show file tree

Hide file tree

Showing 30 changed files with 910 additions and 945 deletions.
diff --git a/.github/tests/dencun-devnet-12.yaml b/.github/tests/dencun-devnet-12.yaml
@@ -6,7 +6,7 @@ participants:
   - el_client_type: nethermind
     el_client_image: ethpandaops/nethermind:master
     cl_client_type: prysm
-    cl_client_image: gcr.io/prysmaticlabs/prysm/beacon-chain:latest,gcr.io/prysmaticlabs/prysm/validator:latest
+    cl_client_image: gcr.io/prysmaticlabs/prysm/beacon-chain:latest
   - el_client_type: erigon
     el_client_image: ethpandaops/erigon:devel
     cl_client_type: nimbus

diff --git a/.github/tests/ephemery.yaml b/.github/tests/ephemery.yaml
@@ -6,7 +6,7 @@ participants:
   - el_client_type: nethermind
     el_client_image: ethpandaops/nethermind:master
     cl_client_type: prysm
-    cl_client_image: gcr.io/prysmaticlabs/prysm/beacon-chain:latest,gcr.io/prysmaticlabs/prysm/validator:latest
+    cl_client_image: gcr.io/prysmaticlabs/prysm/beacon-chain:latest
   - el_client_type: erigon
     el_client_image: ethpandaops/erigon:devel
     cl_client_type: nimbus

diff --git a/.github/tests/mix-persistence-k8s.yaml b/.github/tests/mix-persistence-k8s.yaml
@@ -1,13 +1,13 @@
 participants:
   - el_client_type: geth
     cl_client_type: teku
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
   - el_client_type: nethermind
     cl_client_type: prysm
   - el_client_type: erigon
     cl_client_type: nimbus
     cl_client_image: ethpandaops/nimbus:unstable
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
   - el_client_type: besu
     cl_client_type: lighthouse
   - el_client_type: reth

diff --git a/.github/tests/mix-persistence.yaml b/.github/tests/mix-persistence.yaml
@@ -1,13 +1,13 @@
 participants:
   - el_client_type: geth
     cl_client_type: teku
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
   - el_client_type: nethermind
     cl_client_type: prysm
   - el_client_type: erigon
     cl_client_type: nimbus
     cl_client_image: ethpandaops/nimbus:unstable
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
   - el_client_type: besu
     cl_client_type: lighthouse
   - el_client_type: reth

diff --git a/.github/tests/mixed-cl-vc.yml b/.github/tests/mixed-cl-vc.yml
@@ -0,0 +1,10 @@
+participants:
+  - el_client_type: geth
+    cl_client_type: teku
+    use_separate_validator_client: true
+    validator_client_type: lodestar
+  - el_client_type: besu
+    cl_client_type: nimbus
+    use_separate_validator_client: true
+    validator_client_type: lighthouse
+additional_services: []
diff --git a/.github/tests/node-selectors.yaml b/.github/tests/node-selectors.yaml
@@ -1,13 +1,13 @@
 participants:
   - el_client_type: reth
     cl_client_type: teku
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
     node_selectors: {
       "kubernetes.io/hostname": testing-1,
     }
   - el_client_type: reth
     cl_client_type: teku
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
 global_node_selectors: {
   "kubernetes.io/hostname": testing-2,
 }
diff --git a/.github/tests/split-nimbus.yaml b/.github/tests/split-nimbus.yaml
@@ -2,26 +2,26 @@ participants:
   - el_client_type: geth
     cl_client_type: nimbus
     cl_client_image: ethpandaops/nimbus:unstable
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
     validator_count: 0
   - el_client_type: nethermind
     cl_client_type: nimbus
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
     cl_client_image: ethpandaops/nimbus:unstable
   - el_client_type: erigon
     cl_client_type: nimbus
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
     cl_client_image: ethpandaops/nimbus:unstable
   - el_client_type: besu
     cl_client_type: nimbus
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
     cl_client_image: ethpandaops/nimbus:unstable
   - el_client_type: reth
     cl_client_type: nimbus
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
     cl_client_image: ethpandaops/nimbus:unstable
   - el_client_type: ethereumjs
     cl_client_type: nimbus
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
     cl_client_image: ethpandaops/nimbus:unstable
 additional_services: []
diff --git a/.github/tests/split-teku.yaml b/.github/tests/split-teku.yaml
@@ -1,21 +1,21 @@
 participants:
   - el_client_type: geth
     cl_client_type: teku
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
     validator_count: 0
   - el_client_type: nethermind
     cl_client_type: teku
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
   - el_client_type: erigon
     cl_client_type: teku
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
   - el_client_type: besu
     cl_client_type: teku
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
   - el_client_type: reth
     cl_client_type: teku
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
   - el_client_type: ethereumjs
     cl_client_type: teku
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
 additional_services: []
diff --git a/.github/tests/tolerations.yaml b/.github/tests/tolerations.yaml
@@ -1,7 +1,7 @@
 participants:
   - el_client_type: reth
     cl_client_type: teku
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
     cl_tolerations:
       - key: "node-role.kubernetes.io/master1"
         operator: "Exists"
@@ -19,14 +19,14 @@ participants:
         effect: "NoSchedule"
   - el_client_type: reth
     cl_client_type: teku
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
     tolerations:
       - key: "node-role.kubernetes.io/master5"
         operator: "Exists"
         effect: "NoSchedule"
   - el_client_type: reth
     cl_client_type: teku
-    cl_split_mode_enabled: true
+    use_separate_validator_client: true
 additional_services:
   - dora
 global_tolerations:

diff --git a/README.md b/README.md
@@ -56,7 +56,7 @@ When running on a public testnet using a cloud provider's Kubernetes cluster, th
 
 1. State Growth: The growth of the state might be faster than anticipated. This could potentially lead to issues if the default parameters become insufficient over time. It's important to monitor state growth and adjust parameters as necessary.
 
-2. Persistent Storage Speed: Most cloud providers provision their Kubernetes clusters with relatively slow persistent storage by default. This can cause performance issues, particularly with Ethereum Light (EL) clients.
+2. Persistent Storage Speed: Most cloud providers provision their Kubernetes clusters with relatively slow persistent storage by default. This can cause performance issues, particularly with Execution Layer (EL) clients.
 
 3. Network Syncing: The disk speed provided by cloud providers may not be sufficient to sync with networks that have high demands, such as the mainnet. This could lead to syncing issues and delays.
 
@@ -203,27 +203,40 @@ participants:
   # Valid values are nimbus, lighthouse, lodestar, teku, and prysm
   cl_client_type: lighthouse
 
-  # The Docker image that should be used for the EL client; leave blank to use the default for the client type
-  # Defaults by client (note that Prysm is different in that it requires two images - a Beacon and a validator - separated by a comma):
+  # The Docker image that should be used for the CL client; leave blank to use the default for the client type
+  # Defaults by client:
   # - lighthouse: sigp/lighthouse:latest
   # - teku: consensys/teku:latest
   # - nimbus: statusim/nimbus-eth2:multiarch-latest
-  # - prysm: gcr.io/prysmaticlabs/prysm/beacon-chain:latest,gcr.io/prysmaticlabs/prysm/validator:latest
+  # - prysm: gcr.io/prysmaticlabs/prysm/beacon-chain:latest
   # - lodestar: chainsafe/lodestar:next
   cl_client_image: ""
 
-  # The log level string that this participant's EL client should log at
+  # The log level string that this participant's CL client should log at
   # If this is emptystring then the global `logLevel` parameter's value will be translated into a string appropriate for the client (e.g. if
   # global `logLevel` = `info` then Teku would receive `INFO`, Prysm would receive `info`, etc.)
   # If this is not emptystring, then this value will override the global `logLevel` setting to allow for fine-grained control
   # over a specific participant's logging
   cl_client_log_level: ""
 
-  # A list of optional extra params that will be passed to the CL to run separate Beacon and validator nodes
-  # Only possible for nimbus or teku
-  # Please note that in order to get it to work with Nimbus, you have to use `ethpandaops/nimbus:unstable` as the image (default upstream image does not yet support this out of the box)
-  # Defaults to false
-  cl_split_mode_enabled: false
+  # Whether to use a separate validator client attached to the CL client.
+  # Defaults to false for clients that can run both in one process (Teku, Nimbus)
+  use_separate_validator_client: true/false
+
+  # The type of validator client that should be used
+  # Valid values are nimbus, lighthouse, lodestar, teku, and prysm
+  # ( The prysm validator only works with a prysm CL client )
+  # Defaults to matching the chosen CL client (cl_client_type)
+  validator_client_type: ""
+
+  # The Docker image that should be used for the separate validator client
+  # Defaults by client:
+  # - lighthouse: sigp/lighthouse:latest
+  # - lodestar: chainsafe/lodestar:latest
+  # - nimbus: statusim/nimbus-validator-client:multiarch-latest
+  # - prysm: gcr.io/prysmaticlabs/prysm/validator:latest
+  # - teku: consensys/teku:latest
+  validator_client_image: ""
 
   # Persistent storage size for the CL client container (in MB)
   # Defaults to 0, which means that the default size for the client will be used

diff --git a/main.star b/main.star
@@ -109,11 +109,13 @@ def run(plan, args={}):
 
     all_el_client_contexts = []
     all_cl_client_contexts = []
+    all_validator_client_contexts = []
     all_ethereum_metrics_exporter_contexts = []
     all_xatu_sentry_contexts = []
     for participant in all_participants:
         all_el_client_contexts.append(participant.el_client_context)
         all_cl_client_contexts.append(participant.cl_client_context)
+        all_validator_client_contexts.append(participant.validator_client_context)
         all_ethereum_metrics_exporter_contexts.append(
             participant.ethereum_metrics_exporter_context
         )
@@ -426,6 +428,7 @@ def run(plan, args={}):
             plan,
             all_el_client_contexts,
             all_cl_client_contexts,
+            all_validator_client_contexts,
             prometheus_additional_metrics_jobs,
             all_ethereum_metrics_exporter_contexts,
             all_xatu_sentry_contexts,

diff --git a/src/cl/cl_client_context.star b/src/cl/cl_client_context.star
@@ -5,7 +5,6 @@ def new_cl_client_context(
     http_port_num,
     cl_nodes_metrics_info,
     beacon_service_name,
-    validator_service_name="",
     multiaddr="",
     peer_id="",
     snooper_enabled=False,
@@ -19,7 +18,6 @@ def new_cl_client_context(
         http_port_num=http_port_num,
         cl_nodes_metrics_info=cl_nodes_metrics_info,
         beacon_service_name=beacon_service_name,
-        validator_service_name=validator_service_name,
         multiaddr=multiaddr,
         peer_id=peer_id,
         snooper_enabled=snooper_enabled,