From 34a1ebfb7ce7eb1be8f387792d49d304fed14ea5 Mon Sep 17 00:00:00 2001 From: dappnodedev Date: Fri, 13 Sep 2024 18:14:41 +0200 Subject: [PATCH 1/4] First approach --- shutter/Dockerfile | 2 +- shutter/go-dappnode-shutter/go.mod | 7 + shutter/go-dappnode-shutter/go.sum | 6 + shutter/go-dappnode-shutter/main.go | 73 +++++++++ shutter/go-dappnode-shutter/settings/chain.go | 105 ++++++++++++ .../go-dappnode-shutter/settings/keyper.go | 152 ++++++++++++++++++ shutter/scripts/configure.sh | 4 +- shutter/scripts/configure_keyper.sh | 2 +- shutter/scripts/configure_shuttermint.sh | 11 +- 9 files changed, 356 insertions(+), 6 deletions(-) create mode 100644 shutter/go-dappnode-shutter/go.mod create mode 100644 shutter/go-dappnode-shutter/go.sum create mode 100644 shutter/go-dappnode-shutter/main.go create mode 100644 shutter/go-dappnode-shutter/settings/chain.go create mode 100644 shutter/go-dappnode-shutter/settings/keyper.go diff --git a/shutter/Dockerfile b/shutter/Dockerfile index 33abb02..34b56ac 100644 --- a/shutter/Dockerfile +++ b/shutter/Dockerfile @@ -24,7 +24,7 @@ RUN apt-get update && \ ENV SHUTTER_GNOSIS_SM_BLOCKTIME=10 \ SHUTTER_GNOSIS_GENESIS_KEYPER=0x440Dc6F164e9241F04d282215ceF2780cd0B755e \ SHUTTER_GNOSIS_MAXTXPOINTERAGE=5 \ - SHUTTER_DATABASEURL=postgres://postgres@db.shutter-${NETWORK}.dappnode:5432/keyper \ + SHUTTER_DATABASE_URL=postgres://postgres@db.shutter-${NETWORK}.dappnode:5432/keyper \ SHUTTER_SHUTTERMINT_SHUTTERMINTURL=http://localhost:26657 \ CHAIN_LISTEN_PORT=26657 \ SHUTTER_BIN=/rolling-shutter \ diff --git a/shutter/go-dappnode-shutter/go.mod b/shutter/go-dappnode-shutter/go.mod new file mode 100644 index 0000000..7094bae --- /dev/null +++ b/shutter/go-dappnode-shutter/go.mod @@ -0,0 +1,7 @@ +module go-dappnode-shutter + +go 1.22.2 + +require github.com/joho/godotenv v1.5.1 + +require github.com/pelletier/go-toml/v2 v2.2.3 // indirect diff --git a/shutter/go-dappnode-shutter/go.sum b/shutter/go-dappnode-shutter/go.sum new file mode 100644 index 0000000..b994414 --- /dev/null +++ b/shutter/go-dappnode-shutter/go.sum @@ -0,0 +1,6 @@ +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= diff --git a/shutter/go-dappnode-shutter/main.go b/shutter/go-dappnode-shutter/main.go new file mode 100644 index 0000000..a26d3d3 --- /dev/null +++ b/shutter/go-dappnode-shutter/main.go @@ -0,0 +1,73 @@ +package main + +import ( + "flag" + "fmt" + "go-dappnode-shutter/settings" + "log" + "os" + + "github.com/joho/godotenv" +) + +func main() { + // Define flags for the template, config, and output paths + templateFilePath := flag.String("template", "", "Path to the template file where settings will be included") + configFilePath := flag.String("config", "", "Path to the config file where the settings will be read") + outputFilePath := flag.String("output", "", "Path where the modified settings will be saved") + + // Parse the flags + flag.Parse() + + // Load environment variables from the .env file + err := godotenv.Load(os.Getenv("ASSETS_DIR") + "/variables.env") + if err != nil { + log.Fatalf("Error loading .env file: %v", err) + } + + // Check for additional arguments, e.g., keyper or chain + if len(flag.Args()) < 1 { + fmt.Println("Error: missing argument. Use 'include-keyper-settings' or 'include-chain-settings'.") + os.Exit(1) + } + + // Read the argument passed to the program + argument := flag.Arg(0) + + // Call appropriate function based on the command + switch argument { + case "include-keyper-settings": + // Ensure template, config, and output paths are provided + if *templateFilePath == "" || *configFilePath == "" || *outputFilePath == "" { + fmt.Println("Error: --template, --config, and --output flags must be provided for keyper settings.") + flag.Usage() + os.Exit(1) + } + + // Call the function to configure keyper + err := settings.AddSettingsToKeyper(*templateFilePath, *configFilePath, *outputFilePath) + if err != nil { + log.Fatalf("Failed to configure keyper: %v", err) + } + + case "include-chain-settings": + // Ensure config and output paths are provided + if *configFilePath == "" || *outputFilePath == "" { + fmt.Println("Error: --config and --output flags must be provided for chain settings.") + flag.Usage() + os.Exit(1) + } + + // Call the function to configure chain + err := settings.AddSettingsToChain(*configFilePath, *outputFilePath) + if err != nil { + log.Fatalf("Failed to configure chain: %v", err) + } + + default: + fmt.Println("Invalid argument. Use 'include-keyper-settings' or 'include-chain-settings'.") + os.Exit(1) + } + + fmt.Println("Configuration completed successfully!") +} diff --git a/shutter/go-dappnode-shutter/settings/chain.go b/shutter/go-dappnode-shutter/settings/chain.go new file mode 100644 index 0000000..1ab4dd8 --- /dev/null +++ b/shutter/go-dappnode-shutter/settings/chain.go @@ -0,0 +1,105 @@ +package settings + +import ( + "fmt" + "os" + + "github.com/pelletier/go-toml/v2" +) + +type ChainConfig struct { + PrivateKey string + InstanceID string +} + +/** +* This function should: +* - Read all the values from the template configuration file +* - Read the values defined in ChainConfig from the chain configuration file +* - Read the environment variables +* - Copy all the values from the template configuration file to the output configuration file +* - Modify the values using the chain configuration file and environment variables (these have more priority) +* - Save the modified configuration file +* - Return an error if any +*/ +func AddSettingsToChain(templateFilePath, outputFilePath string) error { + + var chainConfig ChainConfig + + chainConfigPath := os.Getenv("SHUTTER_CHAIN_CONFIG_FILE") + + // Check that the chain configuration file exists + if _, err := os.Stat(chainConfigPath); os.IsNotExist(err) { + fmt.Println("Chain configuration file does not exist:", chainConfigPath) + os.Exit(1) + } + + + chainConfigFile, err := os.ReadFile(chainConfigPath) + if err != nil { + fmt.Println("Error reading TOML file:", err) + os.Exit(1) + } + + err = toml.Unmarshal(chainConfigFile, &chainConfig) + if err != nil { + fmt.Println("Error unmarshalling TOML file:", err) + os.Exit(1) + } + + // Read the template configuration file + templateFile, err := os.ReadFile(templateFilePath) + if err != nil { + return fmt.Errorf("error reading template TOML file: %v", err) + } + + // Create a map to hold the template configuration + var templateConfig map[string]interface{} + err = toml.Unmarshal(templateFile, &templateConfig) + if err != nil { + return fmt.Errorf("error unmarshalling template TOML file: %v", err) + } + + // Modify the template configuration based on ChainConfig and environment variables + // ChainConfig values take priority over the template values + applyChainConfig(&templateConfig, chainConfig) + + // Apply environment variables, which have even higher priority than the ChainConfig + applyEnvOverrides(&templateConfig) + + // Marshal the modified configuration to TOML format + modifiedConfig, err := toml.Marshal(templateConfig) + if err != nil { + return fmt.Errorf("error marshalling modified config to TOML: %v", err) + } + + // Write the modified configuration to the output file + err = os.WriteFile(outputFilePath, modifiedConfig, 0644) + if err != nil { + return fmt.Errorf("error writing modified TOML file: %v", err) + } + + fmt.Println("TOML file modified successfully and saved to", outputFilePath) + return nil +} + +// applyChainConfig modifies the template configuration based on the values from ChainConfig +func applyChainConfig(templateConfig *map[string]interface{}, chainConfig ChainConfig) { + if chainConfig.PrivateKey != "" { + (*templateConfig)["PrivateKey"] = chainConfig.PrivateKey + } + if chainConfig.InstanceID != "" { + (*templateConfig)["InstanceID"] = chainConfig.InstanceID + } +} + +// applyEnvOverrides applies environment variables to the template configuration, giving them the highest priority +func applyEnvOverrides(templateConfig *map[string]interface{}) { + // Example: Check for environment variables and override values if they exist + if privateKey := os.Getenv("PRIVATE_KEY"); privateKey != "" { + (*templateConfig)["PrivateKey"] = privateKey + } + if instanceID := os.Getenv("INSTANCE_ID"); instanceID != "" { + (*templateConfig)["InstanceID"] = instanceID + } +} \ No newline at end of file diff --git a/shutter/go-dappnode-shutter/settings/keyper.go b/shutter/go-dappnode-shutter/settings/keyper.go new file mode 100644 index 0000000..b09239c --- /dev/null +++ b/shutter/go-dappnode-shutter/settings/keyper.go @@ -0,0 +1,152 @@ +package settings + +import ( + "fmt" + "os" + "reflect" + "strconv" + "strings" + + "github.com/pelletier/go-toml/v2" +) + +type KeyperConfig struct { + InstanceID int `env:"_ASSETS_INSTANCE_ID"` + MaxNumKeysPerMessage int `env:"_ASSETS_MAX_NUM_KEYS_PER_MESSAGE"` + EncryptedGasLimit int `env:"_ASSETS_ENCRYPTED_GAS_LIMIT"` + GenesisSlotTimestamp int `env:"_ASSETS_GENESIS_SLOT_TIMESTAMP"` + SyncStartBlockNumber int `env:"_ASSETS_SYNC_START_BLOCK_NUMBER"` + KeyperSetManager string `env:"_ASSETS_KEYPER_SET_MANAGER"` + KeyBroadcastContract string `env:"_ASSETS_KEY_BROADCAST_CONTRACT"` + Sequencer string `env:"_ASSETS_SEQUENCER"` + ValidatorRegistry string `env:"_ASSETS_VALIDATOR_REGISTRY"` + DiscoveryNamespace string `env:"_ASSETS_DISCOVERY_NAME_PREFIX"` + CustomBootstrapAddresses []string `env:"_ASSETS_CUSTOM_BOOTSTRAP_ADDRESSES"` + DKGPhaseLength int `env:"_ASSETS_DKG_PHASE_LENGTH"` + DKGStartBlockDelta int `env:"_ASSETS_DKG_START_BLOCK_DELTA"` + + DatabaseURL string `env:"SHUTTER_DATABASE_URL"` + BeaconAPIURL string `env:"SHUTTER_BEACONAPIURL"` + ContractsURL string `env:"SHUTTER_GNOSIS_NODE_CONTRACTSURL"` + MaxTxPointerAge int `env:"SHUTTER_GNOSIS_MAXTXPOINTERAGE"` + DeploymentDir string `env:"SHUTTER_DEPLOYMENT_DIR"` // Unused, but you can still add an env if needed + EthereumURL string `env:"SHUTTER_GNOSIS_NODE_ETHEREUMURL"` + ShuttermintURL string `env:"SHUTTER_SHUTTERMINT_SHUTTERMINTURL"` + ListenAddresses string `env:"SHUTTER_P2P_LISTENADDRESSES"` + AdvertiseAddresses string `env:"SHUTTER_P2P_ADVERTISEADDRESSES"` + ValidatorPublicKey string `env:"VALIDATOR_PUBLIC_KEY"` + Enabled bool `env:"SHUTTER_ENABLED"` +} + +// AddSettingsToKeyper modifies the keyper settings by combining the template, config, and environment variables. +func AddSettingsToKeyper(templateFilePath, configFilePath, outputFilePath string) error { + var keyperConfig KeyperConfig + + fmt.Println("Adding user settings to keyper...") + + // Read the keyper config file + configFile, err := os.ReadFile(configFilePath) + if err != nil { + return fmt.Errorf("error reading chain config TOML file: %v", err) + } + + // Unmarshal the chain config TOML into the chainConfig struct + err = toml.Unmarshal(configFile, &keyperConfig) + if err != nil { + return fmt.Errorf("error unmarshalling chain config TOML file: %v", err) + } + + // Read the template file + templateFile, err := os.ReadFile(templateFilePath) + if err != nil { + return fmt.Errorf("error reading template TOML file: %v", err) + } + + // Create a map to hold the template configuration + var templateConfig map[string]interface{} + err = toml.Unmarshal(templateFile, &templateConfig) + if err != nil { + return fmt.Errorf("error unmarshalling template TOML file: %v", err) + } + + // Modify the template configuration based on ChainConfig and environment variables + applyKeyperConfig(&templateConfig, keyperConfig) + applyEnvOverrides(&templateConfig) + + // Marshal the modified configuration to TOML format + modifiedConfig, err := toml.Marshal(templateConfig) + if err != nil { + return fmt.Errorf("error marshalling modified config to TOML: %v", err) + } + + // Write the modified configuration to the output file + err = os.WriteFile(outputFilePath, modifiedConfig, 0644) + if err != nil { + return fmt.Errorf("error writing modified TOML file: %v", err) + } + + fmt.Println("Keyper TOML file modified successfully and saved to", outputFilePath) + return nil +} + +// applyKeyperConfig modifies the template based on environment variables, config, and default template values +func applyKeyperConfig(templateConfig *map[string]interface{}, keyperConfig KeyperConfig) { + v := reflect.ValueOf(keyperConfig) + t := reflect.TypeOf(keyperConfig) + + for i := 0; i < v.NumField(); i++ { + fieldValue := v.Field(i) + fieldType := t.Field(i) + + // Get the environment variable tag + envVar := fieldType.Tag.Get("env") + + // Get the corresponding field value based on the priority (env > config > template) + fieldName := fieldType.Name + + var finalValue interface{} + + // 1. Check environment variable + envValue := os.Getenv(envVar) + if envValue != "" { + finalValue = parseEnvValue(envValue, fieldType.Type) + } else if !isZeroValue(fieldValue.Interface()) { + // 2. Use config value if it exists and is non-zero + finalValue = fieldValue.Interface() + } else { + // 3. Fallback to template value (do nothing as it's already in template) + finalValue = (*templateConfig)[fieldName] + } + + setTemplateField(templateConfig, fieldName, finalValue) + } +} + +// Helper function to update template configuration +func setTemplateField(templateConfig *map[string]interface{}, key string, value interface{}) { + if !isZeroValue(value) { + (*templateConfig)[key] = value + } +} + +// Parse environment variable value into the correct type +func parseEnvValue(value string, valueType reflect.Type) interface{} { + switch valueType.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + intValue, _ := strconv.ParseInt(value, 10, 64) + return intValue + case reflect.Bool: + boolValue, _ := strconv.ParseBool(value) + return boolValue + case reflect.Slice: + // Assume comma-separated values for slice + return strings.Split(value, ",") + default: + return value + } +} + +// Check if a value is the zero value of its type +func isZeroValue(value interface{}) bool { + return reflect.DeepEqual(value, reflect.Zero(reflect.TypeOf(value)).Interface()) +} diff --git a/shutter/scripts/configure.sh b/shutter/scripts/configure.sh index a056901..d62aa04 100755 --- a/shutter/scripts/configure.sh +++ b/shutter/scripts/configure.sh @@ -12,8 +12,8 @@ generate_keyper_config() { # Check if the configuration file already exists if [ -f "$KEYPER_GENERATED_CONFIG_FILE" ]; then - echo "[INFO | configure] Configuration file already exists. Skipping generation..." - return + echo "[INFO | configure] Configuration file already exists. Removing it..." + rm "$KEYPER_GENERATED_CONFIG_FILE" fi echo "[INFO | configure] Generating configuration files..." diff --git a/shutter/scripts/configure_keyper.sh b/shutter/scripts/configure_keyper.sh index 346a32f..31da1dd 100755 --- a/shutter/scripts/configure_keyper.sh +++ b/shutter/scripts/configure_keyper.sh @@ -50,7 +50,7 @@ sed -i "/^DKGPhaseLength/c\DKGPhaseLength = ${_ASSETS_DKG_PHASE_LENGTH}" "$KEYPE sed -i "/^DKGStartBlockDelta/c\DKGStartBlockDelta = ${_ASSETS_DKG_START_BLOCK_DELTA}" "$KEYPER_CONFIG_FILE" # Dynamic values (regenerated on each start) -sed -i "/^DatabaseURL/c\DatabaseURL = \"${SHUTTER_DATABASEURL}\"" "$KEYPER_CONFIG_FILE" +sed -i "/^DatabaseURL/c\DatabaseURL = \"${SHUTTER_DATABASE_URL}\"" "$KEYPER_CONFIG_FILE" sed -i "/^BeaconAPIURL/c\BeaconAPIURL = \"${SHUTTER_BEACONAPIURL}\"" "$KEYPER_CONFIG_FILE" sed -i "/^ContractsURL/c\ContractsURL = \"${SHUTTER_GNOSIS_NODE_CONTRACTSURL}\"" "$KEYPER_CONFIG_FILE" sed -i "/^MaxTxPointerAge/c\MaxTxPointerAge = ${SHUTTER_GNOSIS_MAXTXPOINTERAGE}" "$KEYPER_CONFIG_FILE" diff --git a/shutter/scripts/configure_shuttermint.sh b/shutter/scripts/configure_shuttermint.sh index 92391ff..b4200d9 100755 --- a/shutter/scripts/configure_shuttermint.sh +++ b/shutter/scripts/configure_shuttermint.sh @@ -3,16 +3,23 @@ # shellcheck disable=SC1091 . "${ASSETS_DIR}/variables.env" +if [ ! -f "$SHUTTER_CHAIN_CONFIG_FILE" ]; then + echo "[ERROR | configure] Missing chain configuration file (${SHUTTER_CHAIN_CONFIG_FILE})" + exit 1 +fi + ASSETS_GENESIS_FILE="/assets/genesis.json" CHAIN_GENESIS_FILE="${SHUTTER_CHAIN_DIR}/config/genesis.json" rm "$CHAIN_GENESIS_FILE" ln -s "$ASSETS_GENESIS_FILE" "$CHAIN_GENESIS_FILE" -SHUTTERMINT_MONIKER=${KEYPER_NAME:-$(openssl rand -hex 8)} +# KEYPER_NAME=${KEYPER_NAME:-$(openssl rand -hex 8)} + +# TODO: Call the go binary sed -i "/^seeds =/c\seeds = \"${_ASSETS_SHUTTERMINT_SEED_NODES}\"" "$SHUTTER_CHAIN_CONFIG_FILE" -sed -i "/^moniker =/c\moniker = \"${SHUTTERMINT_MONIKER}\"" "$SHUTTER_CHAIN_CONFIG_FILE" +sed -i "/^moniker =/c\moniker = \"${KEYPER_NAME}\"" "$SHUTTER_CHAIN_CONFIG_FILE" sed -i "/^genesis_file =/c\genesis_file = \"${ASSETS_GENESIS_FILE}\"" "$SHUTTER_CHAIN_CONFIG_FILE" sed -i "/^external_address =/c\external_address = \"${_DAPPNODE_GLOBAL_PUBLIC_IP}:${CHAIN_PORT}\"" "$SHUTTER_CHAIN_CONFIG_FILE" sed -i "/^addr_book_strict =/c\addr_book_strict = true" "$SHUTTER_CHAIN_CONFIG_FILE" From f5fcb5f44aad6bd51e068afe771b7c6accbb1abd Mon Sep 17 00:00:00 2001 From: dappnodedev Date: Tue, 17 Sep 2024 12:39:08 +0200 Subject: [PATCH 2/4] Update go settings --- docker-compose.yml | 2 +- package_variants/gnosis/docker-compose.yml | 3 +- shutter/Dockerfile | 9 +- shutter/go-dappnode-shutter/go.mod | 7 - shutter/go-dappnode-shutter/go.sum | 6 - shutter/go-dappnode-shutter/settings/chain.go | 105 ------------ .../go-dappnode-shutter/settings/keyper.go | 152 ------------------ shutter/go-shutter-settings/go.mod | 7 + shutter/go-shutter-settings/go.sum | 12 ++ .../main.go | 24 +-- shutter/go-shutter-settings/settings/chain.go | 62 +++++++ .../go-shutter-settings/settings/keyper.go | 90 +++++++++++ shutter/go-shutter-settings/settings/utils.go | 142 ++++++++++++++++ shutter/scripts/configure_keyper.sh | 44 +---- shutter/scripts/configure_shuttermint.sh | 27 ++-- 15 files changed, 359 insertions(+), 333 deletions(-) delete mode 100644 shutter/go-dappnode-shutter/go.mod delete mode 100644 shutter/go-dappnode-shutter/go.sum delete mode 100644 shutter/go-dappnode-shutter/settings/chain.go delete mode 100644 shutter/go-dappnode-shutter/settings/keyper.go create mode 100644 shutter/go-shutter-settings/go.mod create mode 100644 shutter/go-shutter-settings/go.sum rename shutter/{go-dappnode-shutter => go-shutter-settings}/main.go (59%) create mode 100644 shutter/go-shutter-settings/settings/chain.go create mode 100644 shutter/go-shutter-settings/settings/keyper.go create mode 100644 shutter/go-shutter-settings/settings/utils.go diff --git a/docker-compose.yml b/docker-compose.yml index 1e5a8a2..494d9ec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: context: shutter args: ASSETS_VERSION: shutter-gnosis-1000-set-1v2 # $NETWORK-10*$CHAIN_ID-set-$VERSION - UPSTREAM_VERSION: gnosis-v1.2.1 + UPSTREAM_VERSION: gnosis-v1.2.4b1 KEYPER_CONFIG_DIR: /keyper/config SHUTTER_CHAIN_DIR: /chain STAKER_SCRIPTS_VERSION: v0.1.0 diff --git a/package_variants/gnosis/docker-compose.yml b/package_variants/gnosis/docker-compose.yml index 3894d65..e1dbd69 100644 --- a/package_variants/gnosis/docker-compose.yml +++ b/package_variants/gnosis/docker-compose.yml @@ -4,7 +4,6 @@ services: args: NETWORK: gnosis CHAIN_PORT: 26656 - PROMETHEUS_LISTEN_PORT: 26660 KEYPER_PORT: 23003 KEYPER_METRICS_PORT: 9100 @@ -17,4 +16,4 @@ services: - "23003:23003/tcp" - "26656:26656" - "26660:26660" - - "9100:9100" \ No newline at end of file + - "9100:9100" diff --git a/shutter/Dockerfile b/shutter/Dockerfile index 34b56ac..5d86603 100644 --- a/shutter/Dockerfile +++ b/shutter/Dockerfile @@ -13,7 +13,6 @@ ARG KEYPER_CONFIG_DIR ARG SHUTTER_CHAIN_DIR ARG KEYPER_PORT ARG CHAIN_PORT -ARG PROMETHEUS_LISTEN_PORT ARG STAKER_SCRIPTS_VERSION RUN apt-get update && \ @@ -32,14 +31,19 @@ ENV SHUTTER_GNOSIS_SM_BLOCKTIME=10 \ KEYPER_GENERATED_CONFIG_FILE=${KEYPER_CONFIG_DIR}/generated.toml \ KEYPER_CONFIG_FILE=${KEYPER_CONFIG_DIR}/keyper.toml \ SHUTTER_CHAIN_DIR=${SHUTTER_CHAIN_DIR} \ + KEYPER_PORT=${KEYPER_PORT} \ SHUTTER_CHAIN_CONFIG_FILE=${SHUTTER_CHAIN_DIR}/config/config.toml \ ASSETS_DIR=/assets \ + SHUTTER_SETTINGS_SRC_DIR=/usr/src/go-shutter-settings \ STAKER_SCRIPTS_URL=https://github.com/dappnode/staker-package-scripts/releases/download/${STAKER_SCRIPTS_VERSION} ADD ${STAKER_SCRIPTS_URL}/dvt_lsd_tools.sh /etc/profile.d/ +COPY go-shutter-settings ${SHUTTER_SETTINGS_SRC_DIR} COPY supervisord.conf /etc/supervisord.conf +RUN go build -C ${SHUTTER_SETTINGS_SRC_DIR} -o /usr/local/bin/go_shutter_settings + RUN mkdir -p ${KEYPER_CONFIG_DIR} ${SHUTTER_CHAIN_DIR} ${ASSETS_DIR} /opt/supervisor && \ chmod +rx /etc/profile.d/dvt_lsd_tools.sh @@ -48,8 +52,7 @@ COPY --from=assets ${ASSETS_DIR}/ ${ASSETS_DIR}/ # Placed here to rebuild less layers ENV CHAIN_PORT=${CHAIN_PORT} \ - SHUTTER_P2P_LISTENADDRESSES=/ip4/0.0.0.0/tcp/${KEYPER_PORT} \ - PROMETHEUS_LISTEN_PORT=${PROMETHEUS_LISTEN_PORT} \ + SHUTTER_P2P_LISTENADDRESSES="/ip4/0.0.0.0/tcp/${KEYPER_PORT}" \ NETWORK=${NETWORK} ENTRYPOINT ["supervisord", "-c", "/etc/supervisord.conf"] \ No newline at end of file diff --git a/shutter/go-dappnode-shutter/go.mod b/shutter/go-dappnode-shutter/go.mod deleted file mode 100644 index 7094bae..0000000 --- a/shutter/go-dappnode-shutter/go.mod +++ /dev/null @@ -1,7 +0,0 @@ -module go-dappnode-shutter - -go 1.22.2 - -require github.com/joho/godotenv v1.5.1 - -require github.com/pelletier/go-toml/v2 v2.2.3 // indirect diff --git a/shutter/go-dappnode-shutter/go.sum b/shutter/go-dappnode-shutter/go.sum deleted file mode 100644 index b994414..0000000 --- a/shutter/go-dappnode-shutter/go.sum +++ /dev/null @@ -1,6 +0,0 @@ -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= diff --git a/shutter/go-dappnode-shutter/settings/chain.go b/shutter/go-dappnode-shutter/settings/chain.go deleted file mode 100644 index 1ab4dd8..0000000 --- a/shutter/go-dappnode-shutter/settings/chain.go +++ /dev/null @@ -1,105 +0,0 @@ -package settings - -import ( - "fmt" - "os" - - "github.com/pelletier/go-toml/v2" -) - -type ChainConfig struct { - PrivateKey string - InstanceID string -} - -/** -* This function should: -* - Read all the values from the template configuration file -* - Read the values defined in ChainConfig from the chain configuration file -* - Read the environment variables -* - Copy all the values from the template configuration file to the output configuration file -* - Modify the values using the chain configuration file and environment variables (these have more priority) -* - Save the modified configuration file -* - Return an error if any -*/ -func AddSettingsToChain(templateFilePath, outputFilePath string) error { - - var chainConfig ChainConfig - - chainConfigPath := os.Getenv("SHUTTER_CHAIN_CONFIG_FILE") - - // Check that the chain configuration file exists - if _, err := os.Stat(chainConfigPath); os.IsNotExist(err) { - fmt.Println("Chain configuration file does not exist:", chainConfigPath) - os.Exit(1) - } - - - chainConfigFile, err := os.ReadFile(chainConfigPath) - if err != nil { - fmt.Println("Error reading TOML file:", err) - os.Exit(1) - } - - err = toml.Unmarshal(chainConfigFile, &chainConfig) - if err != nil { - fmt.Println("Error unmarshalling TOML file:", err) - os.Exit(1) - } - - // Read the template configuration file - templateFile, err := os.ReadFile(templateFilePath) - if err != nil { - return fmt.Errorf("error reading template TOML file: %v", err) - } - - // Create a map to hold the template configuration - var templateConfig map[string]interface{} - err = toml.Unmarshal(templateFile, &templateConfig) - if err != nil { - return fmt.Errorf("error unmarshalling template TOML file: %v", err) - } - - // Modify the template configuration based on ChainConfig and environment variables - // ChainConfig values take priority over the template values - applyChainConfig(&templateConfig, chainConfig) - - // Apply environment variables, which have even higher priority than the ChainConfig - applyEnvOverrides(&templateConfig) - - // Marshal the modified configuration to TOML format - modifiedConfig, err := toml.Marshal(templateConfig) - if err != nil { - return fmt.Errorf("error marshalling modified config to TOML: %v", err) - } - - // Write the modified configuration to the output file - err = os.WriteFile(outputFilePath, modifiedConfig, 0644) - if err != nil { - return fmt.Errorf("error writing modified TOML file: %v", err) - } - - fmt.Println("TOML file modified successfully and saved to", outputFilePath) - return nil -} - -// applyChainConfig modifies the template configuration based on the values from ChainConfig -func applyChainConfig(templateConfig *map[string]interface{}, chainConfig ChainConfig) { - if chainConfig.PrivateKey != "" { - (*templateConfig)["PrivateKey"] = chainConfig.PrivateKey - } - if chainConfig.InstanceID != "" { - (*templateConfig)["InstanceID"] = chainConfig.InstanceID - } -} - -// applyEnvOverrides applies environment variables to the template configuration, giving them the highest priority -func applyEnvOverrides(templateConfig *map[string]interface{}) { - // Example: Check for environment variables and override values if they exist - if privateKey := os.Getenv("PRIVATE_KEY"); privateKey != "" { - (*templateConfig)["PrivateKey"] = privateKey - } - if instanceID := os.Getenv("INSTANCE_ID"); instanceID != "" { - (*templateConfig)["InstanceID"] = instanceID - } -} \ No newline at end of file diff --git a/shutter/go-dappnode-shutter/settings/keyper.go b/shutter/go-dappnode-shutter/settings/keyper.go deleted file mode 100644 index b09239c..0000000 --- a/shutter/go-dappnode-shutter/settings/keyper.go +++ /dev/null @@ -1,152 +0,0 @@ -package settings - -import ( - "fmt" - "os" - "reflect" - "strconv" - "strings" - - "github.com/pelletier/go-toml/v2" -) - -type KeyperConfig struct { - InstanceID int `env:"_ASSETS_INSTANCE_ID"` - MaxNumKeysPerMessage int `env:"_ASSETS_MAX_NUM_KEYS_PER_MESSAGE"` - EncryptedGasLimit int `env:"_ASSETS_ENCRYPTED_GAS_LIMIT"` - GenesisSlotTimestamp int `env:"_ASSETS_GENESIS_SLOT_TIMESTAMP"` - SyncStartBlockNumber int `env:"_ASSETS_SYNC_START_BLOCK_NUMBER"` - KeyperSetManager string `env:"_ASSETS_KEYPER_SET_MANAGER"` - KeyBroadcastContract string `env:"_ASSETS_KEY_BROADCAST_CONTRACT"` - Sequencer string `env:"_ASSETS_SEQUENCER"` - ValidatorRegistry string `env:"_ASSETS_VALIDATOR_REGISTRY"` - DiscoveryNamespace string `env:"_ASSETS_DISCOVERY_NAME_PREFIX"` - CustomBootstrapAddresses []string `env:"_ASSETS_CUSTOM_BOOTSTRAP_ADDRESSES"` - DKGPhaseLength int `env:"_ASSETS_DKG_PHASE_LENGTH"` - DKGStartBlockDelta int `env:"_ASSETS_DKG_START_BLOCK_DELTA"` - - DatabaseURL string `env:"SHUTTER_DATABASE_URL"` - BeaconAPIURL string `env:"SHUTTER_BEACONAPIURL"` - ContractsURL string `env:"SHUTTER_GNOSIS_NODE_CONTRACTSURL"` - MaxTxPointerAge int `env:"SHUTTER_GNOSIS_MAXTXPOINTERAGE"` - DeploymentDir string `env:"SHUTTER_DEPLOYMENT_DIR"` // Unused, but you can still add an env if needed - EthereumURL string `env:"SHUTTER_GNOSIS_NODE_ETHEREUMURL"` - ShuttermintURL string `env:"SHUTTER_SHUTTERMINT_SHUTTERMINTURL"` - ListenAddresses string `env:"SHUTTER_P2P_LISTENADDRESSES"` - AdvertiseAddresses string `env:"SHUTTER_P2P_ADVERTISEADDRESSES"` - ValidatorPublicKey string `env:"VALIDATOR_PUBLIC_KEY"` - Enabled bool `env:"SHUTTER_ENABLED"` -} - -// AddSettingsToKeyper modifies the keyper settings by combining the template, config, and environment variables. -func AddSettingsToKeyper(templateFilePath, configFilePath, outputFilePath string) error { - var keyperConfig KeyperConfig - - fmt.Println("Adding user settings to keyper...") - - // Read the keyper config file - configFile, err := os.ReadFile(configFilePath) - if err != nil { - return fmt.Errorf("error reading chain config TOML file: %v", err) - } - - // Unmarshal the chain config TOML into the chainConfig struct - err = toml.Unmarshal(configFile, &keyperConfig) - if err != nil { - return fmt.Errorf("error unmarshalling chain config TOML file: %v", err) - } - - // Read the template file - templateFile, err := os.ReadFile(templateFilePath) - if err != nil { - return fmt.Errorf("error reading template TOML file: %v", err) - } - - // Create a map to hold the template configuration - var templateConfig map[string]interface{} - err = toml.Unmarshal(templateFile, &templateConfig) - if err != nil { - return fmt.Errorf("error unmarshalling template TOML file: %v", err) - } - - // Modify the template configuration based on ChainConfig and environment variables - applyKeyperConfig(&templateConfig, keyperConfig) - applyEnvOverrides(&templateConfig) - - // Marshal the modified configuration to TOML format - modifiedConfig, err := toml.Marshal(templateConfig) - if err != nil { - return fmt.Errorf("error marshalling modified config to TOML: %v", err) - } - - // Write the modified configuration to the output file - err = os.WriteFile(outputFilePath, modifiedConfig, 0644) - if err != nil { - return fmt.Errorf("error writing modified TOML file: %v", err) - } - - fmt.Println("Keyper TOML file modified successfully and saved to", outputFilePath) - return nil -} - -// applyKeyperConfig modifies the template based on environment variables, config, and default template values -func applyKeyperConfig(templateConfig *map[string]interface{}, keyperConfig KeyperConfig) { - v := reflect.ValueOf(keyperConfig) - t := reflect.TypeOf(keyperConfig) - - for i := 0; i < v.NumField(); i++ { - fieldValue := v.Field(i) - fieldType := t.Field(i) - - // Get the environment variable tag - envVar := fieldType.Tag.Get("env") - - // Get the corresponding field value based on the priority (env > config > template) - fieldName := fieldType.Name - - var finalValue interface{} - - // 1. Check environment variable - envValue := os.Getenv(envVar) - if envValue != "" { - finalValue = parseEnvValue(envValue, fieldType.Type) - } else if !isZeroValue(fieldValue.Interface()) { - // 2. Use config value if it exists and is non-zero - finalValue = fieldValue.Interface() - } else { - // 3. Fallback to template value (do nothing as it's already in template) - finalValue = (*templateConfig)[fieldName] - } - - setTemplateField(templateConfig, fieldName, finalValue) - } -} - -// Helper function to update template configuration -func setTemplateField(templateConfig *map[string]interface{}, key string, value interface{}) { - if !isZeroValue(value) { - (*templateConfig)[key] = value - } -} - -// Parse environment variable value into the correct type -func parseEnvValue(value string, valueType reflect.Type) interface{} { - switch valueType.Kind() { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - intValue, _ := strconv.ParseInt(value, 10, 64) - return intValue - case reflect.Bool: - boolValue, _ := strconv.ParseBool(value) - return boolValue - case reflect.Slice: - // Assume comma-separated values for slice - return strings.Split(value, ",") - default: - return value - } -} - -// Check if a value is the zero value of its type -func isZeroValue(value interface{}) bool { - return reflect.DeepEqual(value, reflect.Zero(reflect.TypeOf(value)).Interface()) -} diff --git a/shutter/go-shutter-settings/go.mod b/shutter/go-shutter-settings/go.mod new file mode 100644 index 0000000..5c9764e --- /dev/null +++ b/shutter/go-shutter-settings/go.mod @@ -0,0 +1,7 @@ +module go-shutter-settings + +go 1.21.0 + +require github.com/joho/godotenv v1.5.1 + +require github.com/pelletier/go-toml/v2 v2.2.3 diff --git a/shutter/go-shutter-settings/go.sum b/shutter/go-shutter-settings/go.sum new file mode 100644 index 0000000..8e1e558 --- /dev/null +++ b/shutter/go-shutter-settings/go.sum @@ -0,0 +1,12 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/shutter/go-dappnode-shutter/main.go b/shutter/go-shutter-settings/main.go similarity index 59% rename from shutter/go-dappnode-shutter/main.go rename to shutter/go-shutter-settings/main.go index a26d3d3..e18b279 100644 --- a/shutter/go-dappnode-shutter/main.go +++ b/shutter/go-shutter-settings/main.go @@ -3,7 +3,7 @@ package main import ( "flag" "fmt" - "go-dappnode-shutter/settings" + "go-shutter-settings/settings" "log" "os" @@ -11,10 +11,12 @@ import ( ) func main() { - // Define flags for the template, config, and output paths - templateFilePath := flag.String("template", "", "Path to the template file where settings will be included") - configFilePath := flag.String("config", "", "Path to the config file where the settings will be read") - outputFilePath := flag.String("output", "", "Path where the modified settings will be saved") + var generatedFilePath, configFilePath, outputFilePath string + + // Define flags for the generated, config, and output paths + flag.StringVar(&generatedFilePath, "generated", "", "Path to the generated file where settings will be included") + flag.StringVar(&configFilePath, "config", "", "Path to the config file where the settings will be read") + flag.StringVar(&outputFilePath, "output", "", "Path where the modified settings will be saved") // Parse the flags flag.Parse() @@ -37,29 +39,29 @@ func main() { // Call appropriate function based on the command switch argument { case "include-keyper-settings": - // Ensure template, config, and output paths are provided - if *templateFilePath == "" || *configFilePath == "" || *outputFilePath == "" { - fmt.Println("Error: --template, --config, and --output flags must be provided for keyper settings.") + // Ensure generated, config, and output paths are provided + if generatedFilePath == "" || configFilePath == "" || outputFilePath == "" { + fmt.Println("Error: --generated, --config, and --output flags must be provided for keyper settings.") flag.Usage() os.Exit(1) } // Call the function to configure keyper - err := settings.AddSettingsToKeyper(*templateFilePath, *configFilePath, *outputFilePath) + err := settings.AddSettingsToKeyper(generatedFilePath, configFilePath, outputFilePath) if err != nil { log.Fatalf("Failed to configure keyper: %v", err) } case "include-chain-settings": // Ensure config and output paths are provided - if *configFilePath == "" || *outputFilePath == "" { + if generatedFilePath == "" || outputFilePath == "" { fmt.Println("Error: --config and --output flags must be provided for chain settings.") flag.Usage() os.Exit(1) } // Call the function to configure chain - err := settings.AddSettingsToChain(*configFilePath, *outputFilePath) + err := settings.AddSettingsToChain(generatedFilePath, outputFilePath) if err != nil { log.Fatalf("Failed to configure chain: %v", err) } diff --git a/shutter/go-shutter-settings/settings/chain.go b/shutter/go-shutter-settings/settings/chain.go new file mode 100644 index 0000000..fdf5f48 --- /dev/null +++ b/shutter/go-shutter-settings/settings/chain.go @@ -0,0 +1,62 @@ +package settings + +import ( + "fmt" + "os" + "reflect" + "strings" +) + +type ChainConfig struct { + Moniker string `env:"KEYPER_NAME"` + Genesis_file string `env:"ASSETS_GENESIS_FILE"` + P2P struct { + Seeds string `env:"_ASSETS_SHUTTERMINT_SEED_NODES"` + External_address string `env:"SHUTTER_EXTERNAL_ADDRESS"` + Addr_book_strict bool `env:"SHUTTER_ADDR_BOOK_STRICT"` + Pex bool `env:"SHUTTER_P2P_PEX"` + } + Instrumentation struct { + Prometheus bool `env:"SHUTTER_PUSH_METRICS_ENABLED"` + Prometheus_listen_addr string `env:"SHUTTER_PROMETHEUS_LISTEN_ADDR"` + } +} + +func AddSettingsToChain(generatedFilePath, outputFilePath string) error { + var generatedConfig map[string]interface{} + + fmt.Println("Adding user settings to chain...") + + if _, err := os.Stat(generatedFilePath); os.IsNotExist(err) { + return fmt.Errorf("generated file does not exist: %s", generatedFilePath) + } + + // Read and unmarshal the keyper config file + if err := UnmarshallFromFile(generatedFilePath, &generatedConfig); err != nil { + return err + } + + // TODO: Remove this line + fmt.Printf("Generated chain config: %+v\n", generatedConfig) + + chainConfig := getChainConfigFromEnvs() + + // TODO: Remove this line + fmt.Printf("Chain config: %+v\n", chainConfig) + + // ToLower is used because chain cofig file fields are lower case, but the struct + // fields are upper case to be exported + ApplyConfigToGenerated(reflect.ValueOf(chainConfig), &generatedConfig, strings.ToLower) + + MarshalToFile(outputFilePath, generatedConfig) + + fmt.Println("Chain TOML file modified successfully and saved to", outputFilePath) + + return nil +} + +func getChainConfigFromEnvs() ChainConfig { + chainConfig := ChainConfig{} + PopulateFromEnv(&chainConfig) + return chainConfig +} \ No newline at end of file diff --git a/shutter/go-shutter-settings/settings/keyper.go b/shutter/go-shutter-settings/settings/keyper.go new file mode 100644 index 0000000..75da8bf --- /dev/null +++ b/shutter/go-shutter-settings/settings/keyper.go @@ -0,0 +1,90 @@ +package settings + +import ( + "fmt" + "os" + "reflect" +) + +type KeyperConfig struct { + InstanceID int `env:"_ASSETS_INSTANCE_ID"` + DatabaseURL string `env:"SHUTTER_DATABASE_URL"` + BeaconAPIURL string `env:"SHUTTER_BEACONAPIURL"` + MaxNumKeysPerMessage int `env:"_ASSETS_MAX_NUM_KEYS_PER_MESSAGE"` + Gnosis struct { + EncryptedGasLimit int `env:"_ASSETS_ENCRYPTED_GAS_LIMIT"` + MaxTxPointerAge int `env:"_ASSETS_MAX_TX_POINTER_AGE"` + GenesisSlotTimestamp int `env:"_ASSETS_GENESIS_SLOT_TIMESTAMP"` + SyncStartBlockNumber int `env:"_ASSETS_SYNC_START_BLOCK_NUMBER"` + Node struct { + PrivateKey string `env:"SHUTTER_GNOSIS_NODE_PRIVATEKEY"` + ContractsURL string `env:"SHUTTER_GNOSIS_NODE_CONTRACTSURL"` + DeploymentDir string `env:"SHUTTER_DEPLOYMENT_DIR"` // Unused + EthereumURL string `env:"SHUTTER_GNOSIS_NODE_ETHEREUMURL"` + } + Contracts struct { + KeyperSetManager string `env:"_ASSETS_KEYPER_SET_MANAGER"` + KeyBroadcastContract string `env:"_ASSETS_KEY_BROADCAST_CONTRACT"` + Sequencer string `env:"_ASSETS_SEQUENCER"` + ValidatorRegistry string `env:"_ASSETS_VALIDATOR_REGISTRY"` + } + } + P2P struct { + P2PKey string `env:"SHUTTER_P2P_KEY"` + ListenAddresses string `env:"SHUTTER_P2P_LISTENADDRESSES"` + AdvertiseAddresses string `env:"SHUTTER_P2P_ADVERTISEADDRESSES"` + CustomBootstrapAddresses []string `env:"_ASSETS_CUSTOM_BOOTSTRAP_ADDRESSES"` + DiscoveryNamespace string `env:"_ASSETS_DISCOVERY_NAME_PREFIX"` + } + Shuttermint struct { + ShuttermintURL string `env:"SHUTTER_SHUTTERMINT_SHUTTERMINTURL"` + ValidatorPublicKey string `env:"VALIDATOR_PUBLIC_KEY"` + EncryptionKey string `env:"SHUTTER_SHUTTERMINT_ENCRYPTION_PUBLIC_KEY"` + DKGPhaseLength int `env:"_ASSETS_DKG_PHASE_LENGTH"` + DKGStartBlockDelta int `env:"_ASSETS_DKG_START_BLOCK_DELTA"` + } + Metrics struct { + Enabled bool `env:"SHUTTER_ENABLED"` + } +} + +// AddSettingsToKeyper modifies the keyper settings by combining the generated, config, and environment variables. +func AddSettingsToKeyper(generatedFilePath, configFilePath, outputFilePath string) error { + var keyperConfig KeyperConfig + var generatedConfig map[string]interface{} + + fmt.Println("Adding user settings to keyper...") + + if _, err := os.Stat(generatedFilePath); os.IsNotExist(err) { + return fmt.Errorf("generated file does not exist: %s", generatedFilePath) + } + + if _, err := os.Stat(configFilePath); os.IsNotExist(err) { + return fmt.Errorf("config file does not exist: %s", configFilePath) + } + + // Read and unmarshal the keyper config file + if err := UnmarshallFromFile(configFilePath, &keyperConfig); err != nil { + return err + } + + // Print the config struct for debugging + fmt.Printf("Keyper config: %+v\n", keyperConfig) + + PopulateFromEnv(&keyperConfig) + + // Read and unmarshal the generated file + if err := UnmarshallFromFile(generatedFilePath, &generatedConfig); err != nil { + return err + } + + fmt.Printf("Generated config: %+v\n", generatedConfig) + + ApplyConfigToGenerated(reflect.ValueOf(keyperConfig), &generatedConfig, nil) + + MarshalToFile(outputFilePath, generatedConfig) + + fmt.Println("Keyper TOML file modified successfully and saved to", outputFilePath) + + return nil +} \ No newline at end of file diff --git a/shutter/go-shutter-settings/settings/utils.go b/shutter/go-shutter-settings/settings/utils.go new file mode 100644 index 0000000..47d35fb --- /dev/null +++ b/shutter/go-shutter-settings/settings/utils.go @@ -0,0 +1,142 @@ +package settings + +import ( + "fmt" + "os" + "reflect" + "strconv" + "strings" + + "github.com/pelletier/go-toml/v2" +) + +func UnmarshallFromFile(filePath string, destination interface{}) error { + // Read the file + fileContent, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("error reading file %s: %v", filePath, err) + } + + // Unmarshal the file content into the destination + err = toml.Unmarshal(fileContent, destination) + if err != nil { + return fmt.Errorf("error unmarshalling file %s: %v", filePath, err) + } + + return nil +} + +func MarshalToFile(filePath string, data interface{}) error { + // Marshal the modified configuration to TOML format + modifiedConfig, err := toml.Marshal(data) + if err != nil { + return fmt.Errorf("error marshalling modified config to TOML: %v", err) + } + + // Write the modified configuration to the output file + err = os.WriteFile(filePath, modifiedConfig, 0644) + if err != nil { + return fmt.Errorf("error writing modified TOML file: %v", err) + } + + return nil +} + +// populateFromEnv uses reflection to populate struct fields based on the `env` tag +func PopulateFromEnv(cfg interface{}) { + v := reflect.ValueOf(cfg).Elem() + t := v.Type() + + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + structField := t.Field(i) + tag := structField.Tag.Get("env") + + if tag != "" { + // Handle basic types (string, bool, int, etc.) + envValue := os.Getenv(tag) + if envValue != "" { + SetFieldValue(field, envValue) + } + } + + // If the field is another struct, recurse into it + if field.Kind() == reflect.Struct { + PopulateFromEnv(field.Addr().Interface()) + } + } +} + +func SetFieldValue(field reflect.Value, envValue string) { + switch field.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + // Convert string to integer + intValue, _ := strconv.ParseInt(envValue, 10, 64) + field.SetInt(intValue) + case reflect.Bool: + // Convert string to boolean + boolValue, _ := strconv.ParseBool(envValue) + field.SetBool(boolValue) + case reflect.Slice: + // Remove brackets from the string + envValue = strings.TrimSpace(envValue[1 : len(envValue)-1]) + + // Remove quotes + envValue = strings.ReplaceAll(envValue, `"`, "") + + // Split by commas + slice := strings.Split(envValue, ",") + + // Trim spaces from each element + for i := range slice { + slice[i] = strings.TrimSpace(slice[i]) + } + field.Set(reflect.ValueOf(slice)) + default: + field.SetString(envValue) + } +} + +func ApplyConfigToGenerated(config reflect.Value, generatedConfig *map[string]interface{}, fieldNameFormatter func(string) string) { + // Get the type of the config + configType := config.Type() + + // Iterate through all the fields in the keyper config + for i := 0; i < config.NumField(); i++ { + fieldValue := config.Field(i) + fieldType := configType.Field(i) + + fieldName := fieldType.Name + if fieldNameFormatter != nil { + // To avoid issues with private fields, use the field name formatter + fieldName = fieldNameFormatter(fieldType.Name) + } + + + // Convert the field name to its TOML equivalent + if fieldValue.Kind() == reflect.Struct { + // If the field is a struct, recursively apply its fields + if nestedGenerated, ok := (*generatedConfig)[fieldName].(map[string]interface{}); ok { + ApplyConfigToGenerated(fieldValue, &nestedGenerated, fieldNameFormatter) + (*generatedConfig)[fieldName] = nestedGenerated + } + } else { + // If the field is not a struct, apply its value to the generated if the key exists + if _, ok := (*generatedConfig)[fieldName]; ok && !isZeroValue(fieldValue) { + (*generatedConfig)[fieldName] = fieldValue.Interface() + } + } + } +} + +// Helper function to check if a value is the zero value of its type +func isZeroValue(value reflect.Value) bool { + switch value.Kind() { + case reflect.Slice, reflect.Array: + // For slices and arrays, a zero value is when the length is zero + return value.Len() == 0 + default: + // Use reflect.Zero comparison for other types + return reflect.DeepEqual(value.Interface(), reflect.Zero(value.Type()).Interface()) + } +} \ No newline at end of file diff --git a/shutter/scripts/configure_keyper.sh b/shutter/scripts/configure_keyper.sh index 31da1dd..fecaf01 100755 --- a/shutter/scripts/configure_keyper.sh +++ b/shutter/scripts/configure_keyper.sh @@ -10,11 +10,13 @@ echo "[INFO | configure] Calculating keyper configuration values..." SUPPORTED_NETWORKS="gnosis" -SHUTTER_P2P_ADVERTISEADDRESSES="/ip4/${_DAPPNODE_GLOBAL_PUBLIC_IP}/tcp/23003" -SHUTTER_BEACONAPIURL=$(get_beacon_api_url_from_global_env "$NETWORK" "$SUPPORTED_NETWORKS") -SHUTTER_GNOSIS_NODE_CONTRACTSURL=http://execution.gnosis.dncore.dappnode:8545 -SHUTTER_GNOSIS_NODE_ETHEREUMURL=$(get_execution_ws_url_from_global_env "$NETWORK" "$SUPPORTED_NETWORKS") -VALIDATOR_PUBLIC_KEY=$(cat "${SHUTTER_CHAIN_DIR}/config/priv_validator_pubkey.hex") +export SHUTTER_P2P_ADVERTISEADDRESSES="/ip4/${_DAPPNODE_GLOBAL_PUBLIC_IP}/tcp/${KEYPER_PORT}" +export SHUTTER_BEACONAPIURL=$(get_beacon_api_url_from_global_env "$NETWORK" "$SUPPORTED_NETWORKS") +export SHUTTER_GNOSIS_NODE_CONTRACTSURL=http://execution.gnosis.dncore.dappnode:8545 +export SHUTTER_GNOSIS_NODE_ETHEREUMURL=$(get_execution_ws_url_from_global_env "$NETWORK" "$SUPPORTED_NETWORKS") +export VALIDATOR_PUBLIC_KEY=$(cat "${SHUTTER_CHAIN_DIR}/config/priv_validator_pubkey.hex") + +echo "[INFO | configure] LISTEN: $SHUTTER_P2P_LISTENADDRESSES" # Check if the keyper configuration file already exists if [ -f "$KEYPER_CONFIG_FILE" ]; then @@ -28,36 +30,6 @@ else fi cp "$KEYPER_GENERATED_CONFIG_FILE" "$KEYPER_CONFIG_FILE" - - echo "[INFO | configure] Keyper configuration file created" - echo "[INFO | configure] Setting private key in the configuration file..." - sed -i "/^PrivateKey/c\PrivateKey = \"${SHUTTER_GNOSIS_NODE_PRIVATEKEY}\"" "$KEYPER_CONFIG_FILE" fi -# Assets values -sed -i "/^InstanceID/c\InstanceID = ${_ASSETS_INSTANCE_ID}" "$KEYPER_CONFIG_FILE" -sed -i "/^MaxNumKeysPerMessage/c\MaxNumKeysPerMessage = ${_ASSETS_MAX_NUM_KEYS_PER_MESSAGE}" "$KEYPER_CONFIG_FILE" -sed -i "/^EncryptedGasLimit/c\EncryptedGasLimit = ${_ASSETS_ENCRYPTED_GAS_LIMIT}" "$KEYPER_CONFIG_FILE" -sed -i "/^GenesisSlotTimestamp/c\GenesisSlotTimestamp = ${_ASSETS_GENESIS_SLOT_TIMESTAMP}" "$KEYPER_CONFIG_FILE" -sed -i "/^SyncStartBlockNumber/c\SyncStartBlockNumber = ${_ASSETS_SYNC_START_BLOCK_NUMBER}" "$KEYPER_CONFIG_FILE" -sed -i "/^KeyperSetManager/c\KeyperSetManager = \"${_ASSETS_KEYPER_SET_MANAGER}\"" "$KEYPER_CONFIG_FILE" -sed -i "/^KeyBroadcastContract/c\KeyBroadcastContract = \"${_ASSETS_KEY_BROADCAST_CONTRACT}\"" "$KEYPER_CONFIG_FILE" -sed -i "/^Sequencer/c\Sequencer = \"${_ASSETS_SEQUENCER}\"" "$KEYPER_CONFIG_FILE" -sed -i "/^ValidatorRegistry/c\ValidatorRegistry = \"${_ASSETS_VALIDATOR_REGISTRY}\"" "$KEYPER_CONFIG_FILE" -sed -i "/^DiscoveryNamespace/c\DiscoveryNamespace = \"${_ASSETS_DISCOVERY_NAME_PREFIX}-${_ASSETS_INSTANCE_ID}\"" "$KEYPER_CONFIG_FILE" -sed -i "/^CustomBootstrapAddresses/c\CustomBootstrapAddresses = ${_ASSETS_CUSTOM_BOOTSTRAP_ADDRESSES}" "$KEYPER_CONFIG_FILE" -sed -i "/^DKGPhaseLength/c\DKGPhaseLength = ${_ASSETS_DKG_PHASE_LENGTH}" "$KEYPER_CONFIG_FILE" -sed -i "/^DKGStartBlockDelta/c\DKGStartBlockDelta = ${_ASSETS_DKG_START_BLOCK_DELTA}" "$KEYPER_CONFIG_FILE" - -# Dynamic values (regenerated on each start) -sed -i "/^DatabaseURL/c\DatabaseURL = \"${SHUTTER_DATABASE_URL}\"" "$KEYPER_CONFIG_FILE" -sed -i "/^BeaconAPIURL/c\BeaconAPIURL = \"${SHUTTER_BEACONAPIURL}\"" "$KEYPER_CONFIG_FILE" -sed -i "/^ContractsURL/c\ContractsURL = \"${SHUTTER_GNOSIS_NODE_CONTRACTSURL}\"" "$KEYPER_CONFIG_FILE" -sed -i "/^MaxTxPointerAge/c\MaxTxPointerAge = ${SHUTTER_GNOSIS_MAXTXPOINTERAGE}" "$KEYPER_CONFIG_FILE" -sed -i "/^DeploymentDir/c\DeploymentDir = \"\" # unused" "$KEYPER_CONFIG_FILE" -sed -i "/^EthereumURL/c\EthereumURL = \"${SHUTTER_GNOSIS_NODE_ETHEREUMURL}\"" "$KEYPER_CONFIG_FILE" -sed -i "/^ShuttermintURL/c\ShuttermintURL = \"${SHUTTER_SHUTTERMINT_SHUTTERMINTURL}\"" "$KEYPER_CONFIG_FILE" -sed -i "/^ListenAddresses/c\ListenAddresses = \"${SHUTTER_P2P_LISTENADDRESSES}\"" "$KEYPER_CONFIG_FILE" -sed -i "/^AdvertiseAddresses/c\AdvertiseAddresses = \"${SHUTTER_P2P_ADVERTISEADDRESSES}\"" "$KEYPER_CONFIG_FILE" -sed -i "/^ValidatorPublicKey/c\ValidatorPublicKey = \"${VALIDATOR_PUBLIC_KEY}\"" "$KEYPER_CONFIG_FILE" -sed -i "/^Enabled/c\Enabled = true" "$KEYPER_CONFIG_FILE" +go_shutter_settings --generated "$KEYPER_GENERATED_CONFIG_FILE" --config "$KEYPER_CONFIG_FILE" --output "$KEYPER_CONFIG_FILE" include-keyper-settings diff --git a/shutter/scripts/configure_shuttermint.sh b/shutter/scripts/configure_shuttermint.sh index b4200d9..8446d95 100755 --- a/shutter/scripts/configure_shuttermint.sh +++ b/shutter/scripts/configure_shuttermint.sh @@ -14,17 +14,24 @@ CHAIN_GENESIS_FILE="${SHUTTER_CHAIN_DIR}/config/genesis.json" rm "$CHAIN_GENESIS_FILE" ln -s "$ASSETS_GENESIS_FILE" "$CHAIN_GENESIS_FILE" +export SHUTTER_ADDR_BOOK_STRICT=true +export SHUTTER_P2P_PEX=true +export SHUTTER_PROMETHEUS_LISTEN_ADDR="0.0.0.0:26660" +export SHUTTER_EXTERNAL_ADDRESS="${_DAPPNODE_GLOBAL_PUBLIC_IP}:${CHAIN_PORT}" + # KEYPER_NAME=${KEYPER_NAME:-$(openssl rand -hex 8)} +go_shutter_settings --generated "$SHUTTER_CHAIN_CONFIG_FILE" --output "$SHUTTER_CHAIN_CONFIG_FILE" include-chain-settings + # TODO: Call the go binary -sed -i "/^seeds =/c\seeds = \"${_ASSETS_SHUTTERMINT_SEED_NODES}\"" "$SHUTTER_CHAIN_CONFIG_FILE" -sed -i "/^moniker =/c\moniker = \"${KEYPER_NAME}\"" "$SHUTTER_CHAIN_CONFIG_FILE" -sed -i "/^genesis_file =/c\genesis_file = \"${ASSETS_GENESIS_FILE}\"" "$SHUTTER_CHAIN_CONFIG_FILE" -sed -i "/^external_address =/c\external_address = \"${_DAPPNODE_GLOBAL_PUBLIC_IP}:${CHAIN_PORT}\"" "$SHUTTER_CHAIN_CONFIG_FILE" -sed -i "/^addr_book_strict =/c\addr_book_strict = true" "$SHUTTER_CHAIN_CONFIG_FILE" -sed -i "/^pex =/c\pex = true" "$SHUTTER_CHAIN_CONFIG_FILE" -if [ "$SHUTTER_PUSH_METRICS_ENABLED" = "true" ]; then - sed -i "/^prometheus =/c\prometheus = true" "$SHUTTER_CHAIN_CONFIG_FILE" - sed -i "/^prometheus_listen_addr =/c\prometheus_listen_addr = \"0.0.0.0:${PROMETHEUS_LISTEN_PORT}\"" "$SHUTTER_CHAIN_CONFIG_FILE" -fi +# sed -i "/^seeds =/c\seeds = \"${_ASSETS_SHUTTERMINT_SEED_NODES}\"" "$SHUTTER_CHAIN_CONFIG_FILE" +# sed -i "/^moniker =/c\moniker = \"${KEYPER_NAME}\"" "$SHUTTER_CHAIN_CONFIG_FILE" +# sed -i "/^genesis_file =/c\genesis_file = \"${ASSETS_GENESIS_FILE}\"" "$SHUTTER_CHAIN_CONFIG_FILE" +# sed -i "/^external_address =/c\external_address = \"${_DAPPNODE_GLOBAL_PUBLIC_IP}:${CHAIN_PORT}\"" "$SHUTTER_CHAIN_CONFIG_FILE" +# sed -i "/^addr_book_strict =/c\addr_book_strict = true" "$SHUTTER_CHAIN_CONFIG_FILE" +# sed -i "/^pex =/c\pex = true" "$SHUTTER_CHAIN_CONFIG_FILE" +# if [ "$SHUTTER_PUSH_METRICS_ENABLED" = "true" ]; then +# sed -i "/^prometheus =/c\prometheus = true" "$SHUTTER_CHAIN_CONFIG_FILE" +# sed -i "/^prometheus_listen_addr =/c\prometheus_listen_addr = \"0.0.0.0:26660\"" "$SHUTTER_CHAIN_CONFIG_FILE" +# fi From af378a7d31320314dcfebe7fb28322a0dfa694ef Mon Sep 17 00:00:00 2001 From: dappnodedev Date: Tue, 17 Sep 2024 13:38:23 +0200 Subject: [PATCH 3/4] Fix metrics envs --- metrics/entrypoint.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/metrics/entrypoint.sh b/metrics/entrypoint.sh index 55ab8fb..d9284e1 100755 --- a/metrics/entrypoint.sh +++ b/metrics/entrypoint.sh @@ -19,7 +19,7 @@ update_user_setting() { if [ -z "$value" ]; then echo "[INFO | metrics] Skipped updating $key in user settings file (empty value)" - return 1 + return 0 fi if grep -q "^$key=" "$USER_SETTINGS_FILE"; then @@ -33,7 +33,7 @@ update_user_setting() { fi } -source_envs() { +source_assets_envs() { set -a # Export all variables # shellcheck disable=SC1091 @@ -49,12 +49,19 @@ source_envs() { _ASSETS_VERSION="$(cat /assets/version)" fi - # shellcheck disable=SC1090 - . "$USER_SETTINGS_FILE" - set +a } +source_user_settings() { + # Ensure user settings file exists + if [ -f "$USER_SETTINGS_FILE" ]; then + set -a + # shellcheck disable=SC1090 + . "$USER_SETTINGS_FILE" + set +a + fi +} + replace_envs_in_yaml() { echo "[INFO | metrics] Replacing environment variables in the configuration file" sed "s|%{KEYPER_NAME}|$KEYPER_NAME|g; s|%{_ASSETS_VERSION}|$_ASSETS_VERSION|g" "$TEMPLATE_CONFIG_FILE" >"$CONFIG_FILE" @@ -67,7 +74,9 @@ if [ "${SHUTTER_PUSH_METRICS_ENABLED}" = "false" ]; then exit 0 fi -source_envs +source_assets_envs + +source_user_settings replace_envs_in_yaml From 8f6e4b8c98aa047850a1df60013157455d393dbd Mon Sep 17 00:00:00 2001 From: dappnodedev Date: Tue, 17 Sep 2024 14:09:48 +0200 Subject: [PATCH 4/4] Remove noisy logging --- shutter/go-shutter-settings/settings/chain.go | 6 ------ shutter/go-shutter-settings/settings/keyper.go | 5 ----- 2 files changed, 11 deletions(-) diff --git a/shutter/go-shutter-settings/settings/chain.go b/shutter/go-shutter-settings/settings/chain.go index fdf5f48..f23d5bf 100644 --- a/shutter/go-shutter-settings/settings/chain.go +++ b/shutter/go-shutter-settings/settings/chain.go @@ -36,14 +36,8 @@ func AddSettingsToChain(generatedFilePath, outputFilePath string) error { return err } - // TODO: Remove this line - fmt.Printf("Generated chain config: %+v\n", generatedConfig) - chainConfig := getChainConfigFromEnvs() - // TODO: Remove this line - fmt.Printf("Chain config: %+v\n", chainConfig) - // ToLower is used because chain cofig file fields are lower case, but the struct // fields are upper case to be exported ApplyConfigToGenerated(reflect.ValueOf(chainConfig), &generatedConfig, strings.ToLower) diff --git a/shutter/go-shutter-settings/settings/keyper.go b/shutter/go-shutter-settings/settings/keyper.go index 75da8bf..a0310be 100644 --- a/shutter/go-shutter-settings/settings/keyper.go +++ b/shutter/go-shutter-settings/settings/keyper.go @@ -68,9 +68,6 @@ func AddSettingsToKeyper(generatedFilePath, configFilePath, outputFilePath strin return err } - // Print the config struct for debugging - fmt.Printf("Keyper config: %+v\n", keyperConfig) - PopulateFromEnv(&keyperConfig) // Read and unmarshal the generated file @@ -78,8 +75,6 @@ func AddSettingsToKeyper(generatedFilePath, configFilePath, outputFilePath strin return err } - fmt.Printf("Generated config: %+v\n", generatedConfig) - ApplyConfigToGenerated(reflect.ValueOf(keyperConfig), &generatedConfig, nil) MarshalToFile(outputFilePath, generatedConfig)