Use go to keep config

docker-compose.yml

@@ -14,7 +14,7 @@ services:
       context: shutter
       args:
         ASSETS_VERSION: shutter-gnosis-1000-set-1v2 # $NETWORK-10*$CHAIN_ID-set-$VERSION
-        UPSTREAM_VERSION: gnosis-v1.2.1
+        UPSTREAM_VERSION: gnosis-v1.2.4b1
         KEYPER_CONFIG_DIR: /keyper/config
         SHUTTER_CHAIN_DIR: /chain
         STAKER_SCRIPTS_VERSION: v0.1.0

metrics/entrypoint.sh

@@ -19,7 +19,7 @@ update_user_setting() {
 
     if [ -z "$value" ]; then
         echo "[INFO | metrics] Skipped updating $key in user settings file (empty value)"
-        return 1
+        return 0
     fi
 
     if grep -q "^$key=" "$USER_SETTINGS_FILE"; then
@@ -33,7 +33,7 @@ update_user_setting() {
     fi
 }
 
-source_envs() {
+source_assets_envs() {
     set -a # Export all variables
 
     # shellcheck disable=SC1091
@@ -49,12 +49,19 @@ source_envs() {
         _ASSETS_VERSION="$(cat /assets/version)"
     fi
 
-    # shellcheck disable=SC1090
-    . "$USER_SETTINGS_FILE"
-
     set +a
 }
 
+source_user_settings() {
+    # Ensure user settings file exists
+    if [ -f "$USER_SETTINGS_FILE" ]; then
+        set -a
+        # shellcheck disable=SC1090
+        . "$USER_SETTINGS_FILE"
+        set +a
+    fi
+}
+
 replace_envs_in_yaml() {
     echo "[INFO | metrics] Replacing environment variables in the configuration file"
     sed "s|%{KEYPER_NAME}|$KEYPER_NAME|g; s|%{_ASSETS_VERSION}|$_ASSETS_VERSION|g" "$TEMPLATE_CONFIG_FILE" >"$CONFIG_FILE"
@@ -67,7 +74,9 @@ if [ "${SHUTTER_PUSH_METRICS_ENABLED}" = "false" ]; then
     exit 0
 fi
 
-source_envs
+source_assets_envs
+
+source_user_settings
 
 replace_envs_in_yaml
 

package_variants/gnosis/docker-compose.yml

@@ -4,7 +4,6 @@ services:
       args:
         NETWORK: gnosis
         CHAIN_PORT: 26656
-        PROMETHEUS_LISTEN_PORT: 26660
         KEYPER_PORT: 23003
         KEYPER_METRICS_PORT: 9100
 
@@ -17,4 +16,4 @@ services:
       - "23003:23003/tcp"
       - "26656:26656"
       - "26660:26660"
-      - "9100:9100"
+      - "9100:9100"

shutter/Dockerfile

@@ -13,7 +13,6 @@ ARG KEYPER_CONFIG_DIR
 ARG SHUTTER_CHAIN_DIR
 ARG KEYPER_PORT
 ARG CHAIN_PORT
-ARG PROMETHEUS_LISTEN_PORT
 ARG STAKER_SCRIPTS_VERSION
 
 RUN apt-get update && \
@@ -24,22 +23,27 @@ RUN apt-get update && \
 ENV SHUTTER_GNOSIS_SM_BLOCKTIME=10 \
     SHUTTER_GNOSIS_GENESIS_KEYPER=0x440Dc6F164e9241F04d282215ceF2780cd0B755e \
     SHUTTER_GNOSIS_MAXTXPOINTERAGE=5 \
-    SHUTTER_DATABASEURL=postgres://[email protected]${NETWORK}.dappnode:5432/keyper \
+    SHUTTER_DATABASE_URL=postgres://[email protected]${NETWORK}.dappnode:5432/keyper \
     SHUTTER_SHUTTERMINT_SHUTTERMINTURL=http://localhost:26657 \
     CHAIN_LISTEN_PORT=26657 \
     SHUTTER_BIN=/rolling-shutter \
     SHUTTER_ENV_FILE=/shutter.env \
     KEYPER_GENERATED_CONFIG_FILE=${KEYPER_CONFIG_DIR}/generated.toml \
     KEYPER_CONFIG_FILE=${KEYPER_CONFIG_DIR}/keyper.toml \
     SHUTTER_CHAIN_DIR=${SHUTTER_CHAIN_DIR} \
+    KEYPER_PORT=${KEYPER_PORT} \
     SHUTTER_CHAIN_CONFIG_FILE=${SHUTTER_CHAIN_DIR}/config/config.toml \
     ASSETS_DIR=/assets \
+    SHUTTER_SETTINGS_SRC_DIR=/usr/src/go-shutter-settings \
     STAKER_SCRIPTS_URL=https://github.com/dappnode/staker-package-scripts/releases/download/${STAKER_SCRIPTS_VERSION}
 
 ADD ${STAKER_SCRIPTS_URL}/dvt_lsd_tools.sh /etc/profile.d/
 
+COPY go-shutter-settings ${SHUTTER_SETTINGS_SRC_DIR}
 COPY supervisord.conf /etc/supervisord.conf
 
+RUN go build -C ${SHUTTER_SETTINGS_SRC_DIR} -o /usr/local/bin/go_shutter_settings
+
 RUN mkdir -p ${KEYPER_CONFIG_DIR} ${SHUTTER_CHAIN_DIR} ${ASSETS_DIR} /opt/supervisor && \
     chmod +rx /etc/profile.d/dvt_lsd_tools.sh
 
@@ -48,8 +52,7 @@ COPY --from=assets ${ASSETS_DIR}/ ${ASSETS_DIR}/
 
 # Placed here to rebuild less layers
 ENV CHAIN_PORT=${CHAIN_PORT} \
-    SHUTTER_P2P_LISTENADDRESSES=/ip4/0.0.0.0/tcp/${KEYPER_PORT} \
-    PROMETHEUS_LISTEN_PORT=${PROMETHEUS_LISTEN_PORT} \
+    SHUTTER_P2P_LISTENADDRESSES="/ip4/0.0.0.0/tcp/${KEYPER_PORT}" \
     NETWORK=${NETWORK}
 
 ENTRYPOINT ["supervisord", "-c", "/etc/supervisord.conf"]

shutter/go-shutter-settings/go.mod

@@ -0,0 +1,7 @@
+module go-shutter-settings
+
+go 1.21.0
+
+require github.com/joho/godotenv v1.5.1
+
+require github.com/pelletier/go-toml/v2 v2.2.3

shutter/go-shutter-settings/go.sum

@@ -0,0 +1,12 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
+github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

shutter/go-shutter-settings/main.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"go-shutter-settings/settings"
+	"log"
+	"os"
+
+	"github.com/joho/godotenv"
+)
+
+func main() {
+	var generatedFilePath, configFilePath, outputFilePath string
+
+	// Define flags for the generated, config, and output paths
+	flag.StringVar(&generatedFilePath, "generated", "", "Path to the generated file where settings will be included")
+	flag.StringVar(&configFilePath, "config", "", "Path to the config file where the settings will be read")
+	flag.StringVar(&outputFilePath, "output", "", "Path where the modified settings will be saved")
+
+	// Parse the flags
+	flag.Parse()
+
+	// Load environment variables from the .env file
+	err := godotenv.Load(os.Getenv("ASSETS_DIR") + "/variables.env")
+	if err != nil {
+		log.Fatalf("Error loading .env file: %v", err)
+	}
+
+	// Check for additional arguments, e.g., keyper or chain
+	if len(flag.Args()) < 1 {
+		fmt.Println("Error: missing argument. Use 'include-keyper-settings' or 'include-chain-settings'.")
+		os.Exit(1)
+	}
+
+	// Read the argument passed to the program
+	argument := flag.Arg(0)
+
+	// Call appropriate function based on the command
+	switch argument {
+	case "include-keyper-settings":
+		// Ensure generated, config, and output paths are provided
+		if generatedFilePath == "" || configFilePath == "" || outputFilePath == "" {
+			fmt.Println("Error: --generated, --config, and --output flags must be provided for keyper settings.")
+			flag.Usage()
+			os.Exit(1)
+		}
+
+		// Call the function to configure keyper
+		err := settings.AddSettingsToKeyper(generatedFilePath, configFilePath, outputFilePath)
+		if err != nil {
+			log.Fatalf("Failed to configure keyper: %v", err)
+		}
+
+	case "include-chain-settings":
+		// Ensure config and output paths are provided
+		if generatedFilePath == "" || outputFilePath == "" {
+			fmt.Println("Error: --config and --output flags must be provided for chain settings.")
+			flag.Usage()
+			os.Exit(1)
+		}
+
+		// Call the function to configure chain
+		err := settings.AddSettingsToChain(generatedFilePath, outputFilePath)
+		if err != nil {
+			log.Fatalf("Failed to configure chain: %v", err)
+		}
+
+	default:
+		fmt.Println("Invalid argument. Use 'include-keyper-settings' or 'include-chain-settings'.")
+		os.Exit(1)
+	}
+
+	fmt.Println("Configuration completed successfully!")
+}

shutter/go-shutter-settings/settings/chain.go

@@ -0,0 +1,56 @@
+package settings
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"strings"
+)
+
+type ChainConfig struct {
+	Moniker    string `env:"KEYPER_NAME"`
+    Genesis_file string `env:"ASSETS_GENESIS_FILE"`
+    P2P struct {
+        Seeds string `env:"_ASSETS_SHUTTERMINT_SEED_NODES"`
+        External_address string `env:"SHUTTER_EXTERNAL_ADDRESS"`
+        Addr_book_strict bool `env:"SHUTTER_ADDR_BOOK_STRICT"`
+        Pex bool `env:"SHUTTER_P2P_PEX"`
+    }
+    Instrumentation struct {
+        Prometheus bool `env:"SHUTTER_PUSH_METRICS_ENABLED"`
+        Prometheus_listen_addr string `env:"SHUTTER_PROMETHEUS_LISTEN_ADDR"`
+    }
+}
+
+func AddSettingsToChain(generatedFilePath, outputFilePath string) error {
+    var generatedConfig map[string]interface{}
+
+    fmt.Println("Adding user settings to chain...")
+
+    if _, err := os.Stat(generatedFilePath); os.IsNotExist(err) {
+		return fmt.Errorf("generated file does not exist: %s", generatedFilePath)
+	}
+
+	// Read and unmarshal the keyper config file
+	if err := UnmarshallFromFile(generatedFilePath, &generatedConfig); err != nil {
+		return err
+	}
+
+    chainConfig := getChainConfigFromEnvs()
+
+    // ToLower is used because chain cofig file fields are lower case, but the struct
+    // fields are upper case to be exported
+    ApplyConfigToGenerated(reflect.ValueOf(chainConfig), &generatedConfig, strings.ToLower)
+
+    MarshalToFile(outputFilePath, generatedConfig)
+
+    fmt.Println("Chain TOML file modified successfully and saved to", outputFilePath)
+
+    return nil
+}
+
+func getChainConfigFromEnvs() ChainConfig {
+	chainConfig := ChainConfig{}
+	PopulateFromEnv(&chainConfig)
+	return chainConfig
+}

shutter/go-shutter-settings/settings/keyper.go

@@ -0,0 +1,85 @@
+package settings
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+)
+
+type KeyperConfig struct {
+	InstanceID           int    `env:"_ASSETS_INSTANCE_ID"`
+	DatabaseURL          string `env:"SHUTTER_DATABASE_URL"`
+	BeaconAPIURL         string `env:"SHUTTER_BEACONAPIURL"`
+	MaxNumKeysPerMessage int    `env:"_ASSETS_MAX_NUM_KEYS_PER_MESSAGE"`
+	Gnosis               struct {
+		EncryptedGasLimit    int `env:"_ASSETS_ENCRYPTED_GAS_LIMIT"`
+		MaxTxPointerAge      int `env:"_ASSETS_MAX_TX_POINTER_AGE"`
+		GenesisSlotTimestamp int `env:"_ASSETS_GENESIS_SLOT_TIMESTAMP"`
+		SyncStartBlockNumber int `env:"_ASSETS_SYNC_START_BLOCK_NUMBER"`
+		Node                 struct {
+			PrivateKey    string `env:"SHUTTER_GNOSIS_NODE_PRIVATEKEY"`
+			ContractsURL  string `env:"SHUTTER_GNOSIS_NODE_CONTRACTSURL"`
+			DeploymentDir string `env:"SHUTTER_DEPLOYMENT_DIR"` // Unused
+			EthereumURL   string `env:"SHUTTER_GNOSIS_NODE_ETHEREUMURL"`
+		}
+		Contracts struct {
+			KeyperSetManager     string `env:"_ASSETS_KEYPER_SET_MANAGER"`
+			KeyBroadcastContract string `env:"_ASSETS_KEY_BROADCAST_CONTRACT"`
+			Sequencer            string `env:"_ASSETS_SEQUENCER"`
+			ValidatorRegistry    string `env:"_ASSETS_VALIDATOR_REGISTRY"`
+		}
+	}
+	P2P struct {
+		P2PKey                   string   `env:"SHUTTER_P2P_KEY"`
+		ListenAddresses          string   `env:"SHUTTER_P2P_LISTENADDRESSES"`
+		AdvertiseAddresses       string   `env:"SHUTTER_P2P_ADVERTISEADDRESSES"`
+		CustomBootstrapAddresses []string `env:"_ASSETS_CUSTOM_BOOTSTRAP_ADDRESSES"`
+		DiscoveryNamespace       string   `env:"_ASSETS_DISCOVERY_NAME_PREFIX"`
+	}
+	Shuttermint struct {
+		ShuttermintURL     string `env:"SHUTTER_SHUTTERMINT_SHUTTERMINTURL"`
+		ValidatorPublicKey string `env:"VALIDATOR_PUBLIC_KEY"`
+		EncryptionKey      string `env:"SHUTTER_SHUTTERMINT_ENCRYPTION_PUBLIC_KEY"`
+		DKGPhaseLength     int    `env:"_ASSETS_DKG_PHASE_LENGTH"`
+		DKGStartBlockDelta int    `env:"_ASSETS_DKG_START_BLOCK_DELTA"`
+	}
+	Metrics struct {
+		Enabled bool `env:"SHUTTER_ENABLED"`
+	}
+}
+
+// AddSettingsToKeyper modifies the keyper settings by combining the generated, config, and environment variables.
+func AddSettingsToKeyper(generatedFilePath, configFilePath, outputFilePath string) error {
+	var keyperConfig KeyperConfig
+	var generatedConfig map[string]interface{}
+
+	fmt.Println("Adding user settings to keyper...")
+
+	if _, err := os.Stat(generatedFilePath); os.IsNotExist(err) {
+		return fmt.Errorf("generated file does not exist: %s", generatedFilePath)
+	}
+
+	if _, err := os.Stat(configFilePath); os.IsNotExist(err) {
+		return fmt.Errorf("config file does not exist: %s", configFilePath)
+	}
+
+	// Read and unmarshal the keyper config file
+	if err := UnmarshallFromFile(configFilePath, &keyperConfig); err != nil {
+		return err
+	}
+
+	PopulateFromEnv(&keyperConfig)
+
+	// Read and unmarshal the generated file
+	if err := UnmarshallFromFile(generatedFilePath, &generatedConfig); err != nil {
+		return err
+	}
+
+	ApplyConfigToGenerated(reflect.ValueOf(keyperConfig), &generatedConfig, nil)
+
+	MarshalToFile(outputFilePath, generatedConfig)
+
+	fmt.Println("Keyper TOML file modified successfully and saved to", outputFilePath)
+
+	return nil
+}