feat: enable persistence (#422)

.circleci/config.yml

@@ -109,6 +109,13 @@ jobs:
       - checkout
       - run: kurtosis run ${PWD} "$(cat ./.github/tests/mix-with-tools-mev.yaml)"
 
+  mix_persistence_k8s:
+    resource_class: xlarge
+    executor: ubuntu_vm
+    steps:
+      - checkout
+      - run: kurtosis run ${PWD} "$(cat ./.github/tests/mix-persistence-k8s.yaml)"
+
 workflows:
   nightly_tests:
     when: << pipeline.parameters.should-enable-check-latest-version-workflow >>
@@ -155,3 +162,10 @@ workflows:
             branches:
               ignore:
                 - main
+
+      - mix_persistence_k8s:
+          <<: *setup_kurtosis_k3s
+          filters:
+            branches:
+              ignore:
+                - main

.github/tests/besu-persistence.yaml

@@ -0,0 +1,5 @@
+participants:
+  - el_client_type: besu
+    cl_client_type: lighthouse
+additional_services: []
+persistent: true

.github/tests/erigon-persistence.yaml

@@ -0,0 +1,5 @@
+participants:
+  - el_client_type: erigon
+    cl_client_type: lighthouse
+additional_services: []
+persistent: true

.github/tests/mix-persistence-k8s.yaml

@@ -0,0 +1,19 @@
+participants:
+  - el_client_type: geth
+    cl_client_type: prysm
+  - el_client_type: nethermind
+    cl_client_type: nimbus
+    cl_split_mode_enabled: true
+  - el_client_type: besu
+    cl_client_type: lighthouse
+  - el_client_type: reth
+    cl_client_type: lodestar
+  - el_client_type: ethereumjs
+    cl_client_type: teku
+    cl_split_mode_enabled: true
+  - el_client_type: erigon
+    cl_client_type: teku
+  - el_client_type: besu
+    cl_client_type: nimbus
+additional_services: []
+persistent: true

.github/tests/mix-persistence.yaml

@@ -0,0 +1,15 @@
+participants:
+  - el_client_type: geth
+    cl_client_type: prysm
+  - el_client_type: geth
+    cl_client_type: nimbus
+    cl_split_mode_enabled: true
+  - el_client_type: nethermind
+    cl_client_type: lighthouse
+  - el_client_type: reth
+    cl_client_type: lodestar
+  - el_client_type: ethereumjs
+    cl_client_type: lighthouse
+additional_services:
+  - dora
+persistent: true

.github/tests/teku-persistence.yaml

@@ -0,0 +1,5 @@
+participants:
+  - el_client_type: geth
+    cl_client_type: teku
+additional_services: []
+persistent: true

.github/workflows/per-pr.yml

@@ -32,6 +32,7 @@ jobs:
             "./.github/tests/mev.yaml",
             "./.github/tests/mev-mock.yaml",
             "./.github/tests/mix-with-tools.yaml",
+            "./.github/tests/mix-persistence.yaml",
             "./network_params.yaml"
           ]
     runs-on: ubuntu-latest

README.md

@@ -331,6 +331,7 @@ grafana_additional_dashboards: []
 
 # Whether the environment should be persistent; this is WIP and is slowly being rolled out accross services
 # Note this requires Kurtosis greater than 0.85.49 to work
+# Note Erigon, Besu, Teku persistence is not currently supported with docker.
 # Defaults to False
 persistent: False
 

main.star

@@ -86,6 +86,7 @@ def run(plan, args={}):
         args_with_right_defaults.participants,
         network_params,
         args_with_right_defaults.global_client_log_level,
+        persistent,
         parallel_keystore_generation,
     )
 

src/cl/lighthouse/lighthouse_launcher.star

@@ -10,13 +10,11 @@ blobber_launcher = import_module("../../blobber/blobber_launcher.star")
 
 LIGHTHOUSE_BINARY_COMMAND = "lighthouse"
 
-VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/validator-keys"
-
 RUST_BACKTRACE_ENVVAR_NAME = "RUST_BACKTRACE"
 RUST_FULL_BACKTRACE_KEYWORD = "full"
 
 #  ---------------------------------- Beacon client -------------------------------------
-CONSENSUS_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER = "/consensus-data"
+BEACON_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER = "/data/lighthouse/beacon-data"
 
 # Port IDs
 BEACON_TCP_DISCOVERY_PORT_ID = "tcp-discovery"
@@ -36,6 +34,7 @@ BEACON_MIN_MEMORY = 256
 BEACON_MAX_MEMORY = 1024
 
 #  ---------------------------------- Validator client -------------------------------------
+VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/data/lighthouse/validator-keys"
 VALIDATOR_HTTP_PORT_ID = "http"
 VALIDATOR_METRICS_PORT_ID = "metrics"
 VALIDATOR_HTTP_PORT_NUM = 5042
@@ -121,10 +120,11 @@ def launch(
     extra_validator_params,
     extra_beacon_labels,
     extra_validator_labels,
+    persistent,
     split_mode_enabled=False,
 ):
-    beacon_node_service_name = "{0}".format(service_name)
-    validator_node_service_name = "{0}-{1}".format(
+    beacon_service_name = "{0}".format(service_name)
+    validator_service_name = "{0}-{1}".format(
         service_name, VALIDATOR_SUFFIX_SERVICE_NAME
     )
 
@@ -141,6 +141,7 @@ def launch(
     beacon_config = get_beacon_config(
         launcher.el_cl_genesis_data,
         image,
+        beacon_service_name,
         bootnode_contexts,
         el_client_context,
         log_level,
@@ -152,17 +153,18 @@ def launch(
         snooper_engine_context,
         extra_beacon_params,
         extra_beacon_labels,
+        persistent,
     )
 
-    beacon_service = plan.add_service(beacon_node_service_name, beacon_config)
+    beacon_service = plan.add_service(beacon_service_name, beacon_config)
     beacon_http_port = beacon_service.ports[BEACON_HTTP_PORT_ID]
     beacon_http_url = "http://{0}:{1}".format(
         beacon_service.ip_address, beacon_http_port.number
     )
 
     # Blobber config
     if blobber_enabled:
-        blobber_service_name = "{0}-{1}".format("blobber", beacon_node_service_name)
+        blobber_service_name = "{0}-{1}".format("blobber", beacon_service_name)
         blobber_config = blobber_launcher.get_config(
             blobber_service_name,
             node_keystore_files,
@@ -190,6 +192,7 @@ def launch(
         validator_config = get_validator_config(
             launcher.el_cl_genesis_data,
             image,
+            validator_service_name,
             log_level,
             beacon_http_url,
             el_client_context,
@@ -198,14 +201,12 @@ def launch(
             v_max_cpu,
             v_min_mem,
             v_max_mem,
-            validator_node_service_name,
             extra_validator_params,
             extra_validator_labels,
+            persistent,
         )
 
-        validator_service = plan.add_service(
-            validator_node_service_name, validator_config
-        )
+        validator_service = plan.add_service(validator_service_name, validator_config)
 
     # TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module
     beacon_node_identity_recipe = GetHttpRequestRecipe(
@@ -218,7 +219,7 @@ def launch(
         },
     )
     response = plan.request(
-        recipe=beacon_node_identity_recipe, service_name=beacon_node_service_name
+        recipe=beacon_node_identity_recipe, service_name=beacon_service_name
     )
     beacon_node_enr = response["extract.enr"]
     beacon_multiaddr = response["extract.multiaddr"]
@@ -229,7 +230,7 @@ def launch(
         beacon_service.ip_address, beacon_metrics_port.number
     )
     beacon_node_metrics_info = node_metrics.new_node_metrics_info(
-        beacon_node_service_name, METRICS_PATH, beacon_metrics_url
+        beacon_service_name, METRICS_PATH, beacon_metrics_url
     )
     nodes_metrics_info = [beacon_node_metrics_info]
 
@@ -239,7 +240,7 @@ def launch(
             validator_service.ip_address, validator_metrics_port.number
         )
         validator_node_metrics_info = node_metrics.new_node_metrics_info(
-            validator_node_service_name, METRICS_PATH, validator_metrics_url
+            validator_service_name, METRICS_PATH, validator_metrics_url
         )
         nodes_metrics_info.append(validator_node_metrics_info)
 
@@ -249,8 +250,8 @@ def launch(
         beacon_service.ip_address,
         BEACON_HTTP_PORT_NUM,
         nodes_metrics_info,
-        beacon_node_service_name,
-        validator_node_service_name,
+        beacon_service_name,
+        validator_service_name,
         beacon_multiaddr,
         beacon_peer_id,
         snooper_enabled,
@@ -264,6 +265,7 @@ def launch(
 def get_beacon_config(
     el_cl_genesis_data,
     image,
+    service_name,
     boot_cl_client_ctxs,
     el_client_context,
     log_level,
@@ -275,6 +277,7 @@ def get_beacon_config(
     snooper_engine_context,
     extra_params,
     extra_labels,
+    persistent,
 ):
     # If snooper is enabled use the snooper engine context, otherwise use the execution client context
     if snooper_enabled:
@@ -300,7 +303,7 @@ def get_beacon_config(
         LIGHTHOUSE_BINARY_COMMAND,
         "beacon_node",
         "--debug-level=" + log_level,
-        "--datadir=" + CONSENSUS_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER,
+        "--datadir=" + BEACON_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER,
         "--testnet-dir=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER,
         # vvvvvvvvvvvvvvvvvvv REMOVE THESE WHEN CONNECTING TO EXTERNAL NET vvvvvvvvvvvvvvvvvvvvv
         "--disable-enr-auto-update",
@@ -358,14 +361,19 @@ def get_beacon_config(
     recipe = GetHttpRequestRecipe(
         endpoint="/eth/v1/node/identity", port_id=BEACON_HTTP_PORT_ID
     )
+    files = {
+        constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid
+    }
 
+    if persistent:
+        files[BEACON_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER] = Directory(
+            persistent_key="data-{0}".format(service_name),
+        )
     return ServiceConfig(
         image=image,
         ports=BEACON_USED_PORTS,
         cmd=cmd,
-        files={
-            constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid
-        },
+        files=files,
         env_vars={RUST_BACKTRACE_ENVVAR_NAME: RUST_FULL_BACKTRACE_KEYWORD},
         private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
         ready_conditions=cl_node_ready_conditions.get_ready_conditions(
@@ -388,6 +396,7 @@ def get_beacon_config(
 def get_validator_config(
     el_cl_genesis_data,
     image,
+    service_name,
     log_level,
     beacon_client_http_url,
     el_client_context,
@@ -396,9 +405,9 @@ def get_validator_config(
     v_max_cpu,
     v_min_mem,
     v_max_mem,
-    validator_node_service_name,
     extra_params,
     extra_labels,
+    persistent,
 ):
     validator_keys_dirpath = shared_utils.path_join(
         VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS,
@@ -442,14 +451,16 @@ def get_validator_config(
     if len(extra_params):
         cmd.extend([param for param in extra_params])
 
+    files = {
+        constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
+        VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid,
+    }
+
     return ServiceConfig(
         image=image,
         ports=VALIDATOR_USED_PORTS,
         cmd=cmd,
-        files={
-            constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
-            VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid,
-        },
+        files=files,
         env_vars={RUST_BACKTRACE_ENVVAR_NAME: RUST_FULL_BACKTRACE_KEYWORD},
         min_cpu=v_min_cpu,
         max_cpu=v_max_cpu,