Add support for multiple fee recipients

data/serializer/src/main/java/tech/pegasys/teku/provider/Bytes48KeyDeserializer.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2022 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.provider;
+
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.KeyDeserializer;
+import org.apache.tuweni.bytes.Bytes48;
+
+public class Bytes48KeyDeserializer extends KeyDeserializer {
+
+  @Override
+  public Object deserializeKey(String key, DeserializationContext ctxt) {
+    return Bytes48.fromHexStringStrict(key);
+  }
+}

data/serializer/src/main/java/tech/pegasys/teku/provider/JsonProvider.java

@@ -19,6 +19,7 @@
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
+import org.apache.tuweni.bytes.Bytes48;
 import org.apache.tuweni.units.bigints.UInt256;
 import tech.pegasys.teku.api.response.v1.validator.GetNewBlockResponse;
 import tech.pegasys.teku.api.response.v2.debug.GetStateResponseV2;
@@ -45,6 +46,8 @@ private void addTekuMappers() {
     module.addDeserializer(BLSSignature.class, new BLSSignatureDeserializer());
     module.addSerializer(BLSSignature.class, new BLSSignatureSerializer());
 
+    module.addKeyDeserializer(Bytes48.class, new Bytes48KeyDeserializer());
+
     module.addDeserializer(Bytes32.class, new Bytes32Deserializer());
     module.addDeserializer(Bytes4.class, new Bytes4Deserializer());
     module.addSerializer(Bytes4.class, new Bytes4Serializer());

teku/src/main/java/tech/pegasys/teku/cli/options/ValidatorProposerOptions.java

@@ -15,6 +15,7 @@
 
 import picocli.CommandLine.Option;
 import tech.pegasys.teku.config.TekuConfiguration;
+import tech.pegasys.teku.validator.api.ValidatorConfig;
 
 public class ValidatorProposerOptions {
   @Option(
@@ -26,7 +27,31 @@ public class ValidatorProposerOptions {
       hidden = true)
   private String proposerDefaultFeeRecipient = null;
 
+  @Option(
+      names = {"--Xvalidators-proposer-config"},
+      paramLabel = "<STRING>",
+      description = "remote URL or local file path to load proposer configuration from",
+      arity = "0..1",
+      hidden = true)
+  private String proposerConfig = null;
+
+  @Option(
+      names = {"--Xvalidators-proposer-config-refresh-enabled"},
+      paramLabel = "<BOOLEAN>",
+      description =
+          "Enable the proposer configuration reload on every proposer preparation (once per epoch)",
+      arity = "0..1",
+      fallbackValue = "true",
+      hidden = true)
+  private boolean proposerConfigRefreshEnabled =
+      ValidatorConfig.DEFAULT_VALIDATOR_PROPOSER_CONFIG_REFRESH_ENABLED;
+
   public void configure(TekuConfiguration.Builder builder) {
-    builder.validator(config -> config.proposerDefaultFeeRecipient(proposerDefaultFeeRecipient));
+    builder.validator(
+        config ->
+            config
+                .proposerDefaultFeeRecipient(proposerDefaultFeeRecipient)
+                .proposerConfigSource(proposerConfig)
+                .refreshProposerConfigFromSource(proposerConfigRefreshEnabled));
   }
 }

validator/api/src/main/java/tech/pegasys/teku/validator/api/ValidatorConfig.java

@@ -40,6 +40,7 @@ public class ValidatorConfig {
   public static final boolean DEFAULT_GENERATE_EARLY_ATTESTATIONS = true;
   public static final boolean DEFAULT_SEND_ATTESTATIONS_AS_BATCH = true;
   public static final Optional<Bytes32> DEFAULT_GRAFFITI = Optional.empty();
+  public static final boolean DEFAULT_VALIDATOR_PROPOSER_CONFIG_REFRESH_ENABLED = false;
 
   private final List<String> validatorKeys;
   private final List<String> validatorExternalSignerPublicKeySources;
@@ -58,6 +59,8 @@ public class ValidatorConfig {
   private final boolean useDependentRoots;
   private final boolean generateEarlyAttestations;
   private final Optional<Eth1Address> proposerDefaultFeeRecipient;
+  private final Optional<String> proposerConfigSource;
+  private final boolean refreshProposerConfigFromSource;
 
   private ValidatorConfig(
       final List<String> validatorKeys,
@@ -76,7 +79,9 @@ private ValidatorConfig(
       final int validatorExternalSignerConcurrentRequestLimit,
       final boolean useDependentRoots,
       final boolean generateEarlyAttestations,
-      final Optional<Eth1Address> proposerDefaultFeeRecipient) {
+      final Optional<Eth1Address> proposerDefaultFeeRecipient,
+      final Optional<String> proposerConfigSource,
+      final boolean refreshProposerConfigFromSource) {
     this.validatorKeys = validatorKeys;
     this.validatorExternalSignerPublicKeySources = validatorExternalSignerPublicKeySources;
     this.validatorExternalSignerUrl = validatorExternalSignerUrl;
@@ -97,6 +102,8 @@ private ValidatorConfig(
     this.useDependentRoots = useDependentRoots;
     this.generateEarlyAttestations = generateEarlyAttestations;
     this.proposerDefaultFeeRecipient = proposerDefaultFeeRecipient;
+    this.proposerConfigSource = proposerConfigSource;
+    this.refreshProposerConfigFromSource = refreshProposerConfigFromSource;
   }
 
   public static Builder builder() {
@@ -161,15 +168,25 @@ public boolean useDependentRoots() {
   }
 
   public Optional<Eth1Address> getProposerDefaultFeeRecipient() {
-    validateProposerDefaultFeeRecipient();
+    validateProposerDefaultFeeRecipientOrProposerConfigSource();
     return proposerDefaultFeeRecipient;
   }
 
-  private void validateProposerDefaultFeeRecipient() {
+  public Optional<String> getProposerConfigSource() {
+    validateProposerDefaultFeeRecipientOrProposerConfigSource();
+    return proposerConfigSource;
+  }
+
+  public boolean getRefreshProposerConfigFromSource() {
+    return refreshProposerConfigFromSource;
+  }
+
+  private void validateProposerDefaultFeeRecipientOrProposerConfigSource() {
     if (proposerDefaultFeeRecipient.isEmpty()
+        && proposerConfigSource.isEmpty()
         && !(validatorKeys.isEmpty() && validatorExternalSignerPublicKeySources.isEmpty())) {
       throw new InvalidConfigurationException(
-          "Invalid configuration. --Xvalidators-proposer-default-fee-recipient must be specified when Bellatrix milestone is active");
+          "Invalid configuration. --Xvalidators-proposer-default-fee-recipient or --Xvalidators-proposer-config must be specified when Bellatrix milestone is active");
     }
   }
 
@@ -195,6 +212,9 @@ public static final class Builder {
     private boolean useDependentRoots = DEFAULT_USE_DEPENDENT_ROOTS;
     private boolean generateEarlyAttestations = DEFAULT_GENERATE_EARLY_ATTESTATIONS;
     private Optional<Eth1Address> proposerDefaultFeeRecipient = Optional.empty();
+    private Optional<String> proposerConfigSource = Optional.empty();
+    private boolean refreshProposerConfigFromSource =
+        DEFAULT_VALIDATOR_PROPOSER_CONFIG_REFRESH_ENABLED;
 
     private Builder() {}
 
@@ -304,6 +324,16 @@ public Builder proposerDefaultFeeRecipient(final String proposerDefaultFeeRecipi
       return this;
     }
 
+    public Builder proposerConfigSource(final String proposerConfigSource) {
+      this.proposerConfigSource = Optional.ofNullable(proposerConfigSource);
+      return this;
+    }
+
+    public Builder refreshProposerConfigFromSource(final boolean refreshProposerConfigFromSource) {
+      this.refreshProposerConfigFromSource = refreshProposerConfigFromSource;
+      return this;
+    }
+
     public ValidatorConfig build() {
       validateExternalSignerUrlAndPublicKeys();
       validateExternalSignerKeystoreAndPasswordFileConfig();
@@ -326,7 +356,9 @@ public ValidatorConfig build() {
           validatorExternalSignerConcurrentRequestLimit,
           useDependentRoots,
           generateEarlyAttestations,
-          proposerDefaultFeeRecipient);
+          proposerDefaultFeeRecipient,
+          proposerConfigSource,
+          refreshProposerConfigFromSource);
     }
 
     private void validateExternalSignerUrlAndPublicKeys() {

validator/api/src/test/java/tech/pegasys/teku/validator/api/ValidatorConfigTest.java

@@ -21,14 +21,13 @@
 import org.junit.jupiter.api.Test;
 import tech.pegasys.teku.bls.BLSTestUtil;
 import tech.pegasys.teku.infrastructure.exceptions.InvalidConfigurationException;
-import tech.pegasys.teku.spec.datastructures.eth1.Eth1Address;
 
 class ValidatorConfigTest {
 
   private final ValidatorConfig.Builder configBuilder = ValidatorConfig.builder();
 
   @Test
-  public void shouldThrowExceptionIfExternalPublicKeysAreSpecifiedWithoutExternalSignerUrl() {
+  public void shouldThrowIfExternalPublicKeysAreSpecifiedWithoutExternalSignerUrl() {
     final ValidatorConfig.Builder builder =
         configBuilder.validatorExternalSignerPublicKeySources(
             List.of(BLSTestUtil.randomKeyPair(0).getPublicKey().toString()));
@@ -39,15 +38,15 @@ public void shouldThrowExceptionIfExternalPublicKeysAreSpecifiedWithoutExternalS
   }
 
   @Test
-  public void noExceptionThrownIfExternalSignerUrlIsSpecifiedWithoutExternalPublicKeys()
+  public void shouldNotThrowIfExternalSignerUrlIsSpecifiedWithoutExternalPublicKeys()
       throws MalformedURLException {
     final ValidatorConfig.Builder builder =
         configBuilder.validatorExternalSignerUrl(URI.create("http://localhost:9000").toURL());
     Assertions.assertThatCode(builder::build).doesNotThrowAnyException();
   }
 
   @Test
-  public void noExceptionThrownIfBothExternalSignerUrlAndPublicKeysAreSpecified()
+  public void shouldNotThrowIfBothExternalSignerUrlAndPublicKeysAreSpecified()
       throws MalformedURLException {
     final ValidatorConfig.Builder builder =
         configBuilder
@@ -59,7 +58,7 @@ public void noExceptionThrownIfBothExternalSignerUrlAndPublicKeysAreSpecified()
   }
 
   @Test
-  public void shouldThrowExceptionIfExternalSignerKeystoreSpecifiedWithoutPasswordFile() {
+  public void shouldThrowIfExternalSignerKeystoreSpecifiedWithoutPasswordFile() {
     final ValidatorConfig.Builder builder =
         configBuilder.validatorExternalSignerKeystore(Path.of("somepath"));
     Assertions.assertThatExceptionOfType(InvalidConfigurationException.class)
@@ -69,7 +68,7 @@ public void shouldThrowExceptionIfExternalSignerKeystoreSpecifiedWithoutPassword
   }
 
   @Test
-  public void shouldThrowExceptionIfExternalSignerKeystorePasswordFileIsSpecifiedWithoutKeystore() {
+  public void shouldThrowIfExternalSignerKeystorePasswordFileIsSpecifiedWithoutKeystore() {
     final ValidatorConfig.Builder builder =
         configBuilder.validatorExternalSignerKeystorePasswordFile(Path.of("somepath"));
     Assertions.assertThatExceptionOfType(InvalidConfigurationException.class)
@@ -79,7 +78,7 @@ public void shouldThrowExceptionIfExternalSignerKeystorePasswordFileIsSpecifiedW
   }
 
   @Test
-  public void noExceptionThrownIfBothExternalSignerKeystoreAndPasswordFileAreSpecified() {
+  public void shouldNotThrowIfBothExternalSignerKeystoreAndPasswordFileAreSpecified() {
     final ValidatorConfig.Builder builder =
         configBuilder
             .validatorExternalSignerKeystore(Path.of("somepath"))
@@ -89,7 +88,7 @@ public void noExceptionThrownIfBothExternalSignerKeystoreAndPasswordFileAreSpeci
   }
 
   @Test
-  public void shouldThrowExceptionIfExternalSignerTruststoreSpecifiedWithoutPasswordFile() {
+  public void shouldThrowIfExternalSignerTruststoreSpecifiedWithoutPasswordFile() {
     final ValidatorConfig.Builder builder =
         configBuilder.validatorExternalSignerTruststore(Path.of("somepath"));
     Assertions.assertThatExceptionOfType(InvalidConfigurationException.class)
@@ -99,8 +98,7 @@ public void shouldThrowExceptionIfExternalSignerTruststoreSpecifiedWithoutPasswo
   }
 
   @Test
-  public void
-      shouldThrowExceptionIfExternalSignerTruststorePasswordFileIsSpecifiedWithoutTruststore() {
+  public void shouldThrowIfExternalSignerTruststorePasswordFileIsSpecifiedWithoutTruststore() {
     final ValidatorConfig.Builder builder =
         configBuilder.validatorExternalSignerTruststorePasswordFile(Path.of("somepath"));
     Assertions.assertThatExceptionOfType(InvalidConfigurationException.class)
@@ -110,7 +108,7 @@ public void shouldThrowExceptionIfExternalSignerTruststoreSpecifiedWithoutPasswo
   }
 
   @Test
-  public void noExceptionThrownIfBothExternalSignerTruststoreAndPasswordFileAreSpecified() {
+  public void shouldNotThrowIfBothExternalSignerTruststoreAndPasswordFileAreSpecified() {
     final ValidatorConfig.Builder builder =
         configBuilder
             .validatorExternalSignerTruststore(Path.of("somepath"))
@@ -120,7 +118,7 @@ public void noExceptionThrownIfBothExternalSignerTruststoreAndPasswordFileAreSpe
   }
 
   @Test
-  public void bellatrix_shouldThrowExceptionIfExternalSignerPublicKeySourcesIsSpecified()
+  public void bellatrix_shouldThrowIfExternalSignerPublicKeySourcesIsSpecified()
       throws MalformedURLException {
     final ValidatorConfig config =
         configBuilder
@@ -129,24 +127,18 @@ public void bellatrix_shouldThrowExceptionIfExternalSignerPublicKeySourcesIsSpec
             .validatorExternalSignerUrl(URI.create("http://localhost:9000").toURL())
             .build();
 
-    Assertions.assertThatExceptionOfType(InvalidConfigurationException.class)
-        .isThrownBy(config::getProposerDefaultFeeRecipient)
-        .withMessageContaining(
-            "Invalid configuration. --Xvalidators-proposer-default-fee-recipient must be specified when Bellatrix milestone is active");
+    verifyProposerConfigOrProposerDefaultFeeRecipientThrow(config);
   }
 
   @Test
-  public void bellatrix_shouldThrowExceptionIfValidatorKeysAreSpecified() {
+  public void bellatrix_shouldThrowIfValidatorKeysAreSpecified() {
     final ValidatorConfig config = configBuilder.validatorKeys(List.of("some string")).build();
 
-    Assertions.assertThatExceptionOfType(InvalidConfigurationException.class)
-        .isThrownBy(config::getProposerDefaultFeeRecipient)
-        .withMessageContaining(
-            "Invalid configuration. --Xvalidators-proposer-default-fee-recipient must be specified when Bellatrix milestone is active");
+    verifyProposerConfigOrProposerDefaultFeeRecipientThrow(config);
   }
 
   @Test
-  public void bellatrix_noExceptionThrownIfIfExternalSignerPublicKeySourcesIsSpecified()
+  public void bellatrix_shouldNotThrowIfValidationIsActiveAndDefaultFeeRecipientIsSpecified()
       throws MalformedURLException {
     final ValidatorConfig config =
         configBuilder
@@ -156,18 +148,37 @@ public void bellatrix_noExceptionThrownIfIfExternalSignerPublicKeySourcesIsSpeci
             .proposerDefaultFeeRecipient("0x0000000000000000000000000000000000000000")
             .build();
 
-    Assertions.assertThatCode(config::getProposerDefaultFeeRecipient).doesNotThrowAnyException();
+    verifyProposerConfigOrProposerDefaultFeeRecipientNotThrow(config);
   }
 
   @Test
-  public void bellatrix_noExceptionThrownIfIfValidatorKeysAreSpecified() {
+  public void bellatrix_shouldNotThrowIfValidationIsActiveAndProposerConfigSourceIsSpecified()
+      throws MalformedURLException {
     final ValidatorConfig config =
         configBuilder
-            .validatorKeys(List.of("some string"))
-            .proposerDefaultFeeRecipient(
-                Eth1Address.fromHexString("0x0000000000000000000000000000000000000000"))
+            .validatorExternalSignerPublicKeySources(
+                List.of(BLSTestUtil.randomKeyPair(0).getPublicKey().toString()))
+            .validatorExternalSignerUrl(URI.create("http://localhost:9000").toURL())
+            .proposerConfigSource("some path")
             .build();
 
+    verifyProposerConfigOrProposerDefaultFeeRecipientNotThrow(config);
+  }
+
+  void verifyProposerConfigOrProposerDefaultFeeRecipientNotThrow(final ValidatorConfig config) {
     Assertions.assertThatCode(config::getProposerDefaultFeeRecipient).doesNotThrowAnyException();
+    Assertions.assertThatCode(config::getProposerConfigSource).doesNotThrowAnyException();
+  }
+
+  void verifyProposerConfigOrProposerDefaultFeeRecipientThrow(final ValidatorConfig config) {
+    verifyProposerConfigOrProposerDefaultFeeRecipientThrow(config::getProposerDefaultFeeRecipient);
+    verifyProposerConfigOrProposerDefaultFeeRecipientThrow(config::getProposerConfigSource);
+  }
+
+  void verifyProposerConfigOrProposerDefaultFeeRecipientThrow(final Runnable task) {
+    Assertions.assertThatExceptionOfType(InvalidConfigurationException.class)
+        .isThrownBy(task::run)
+        .withMessageContaining(
+            "Invalid configuration. --Xvalidators-proposer-default-fee-recipient or --Xvalidators-proposer-config must be specified when Bellatrix milestone is active");
   }
 }

validator/client/src/main/java/tech/pegasys/teku/validator/client/AbstractDutyLoader.java

@@ -38,7 +38,7 @@ protected AbstractDutyLoader(
   public SafeFuture<Optional<S>> loadDutiesForEpoch(final UInt64 epoch) {
     LOG.trace("Requesting duties for epoch {}", epoch);
     return validatorIndexProvider
-        .getValidatorIndices(validators.getPublicKeys())
+        .getValidatorIndices()
         .thenCompose(
             validatorIndices -> {
               if (validatorIndices.isEmpty()) {