Add support for new flags in new config.toml, which were present in o…

…ld config.toml (ethereum#612)

* added HTTPTimeouts, and TrieTimeout flag in new tol, from old toml

* added RAW fields for these time.Duration flags

* updated the conversion script to support these extra 4 flags

* removed hcl and json config tests as we are only supporting toml config files

* updated toml files with cache.timeout field

* updated toml files with jsonrpc.timeouts field

* tests/bor: expect a call for latest checkpoint

* tests/bor: expect a call for latest checkpoint

* packaging/templates: update cache values for archive nodes

Co-authored-by: Manav Darji <[email protected]>

builder/files/config.toml

@@ -61,32 +61,36 @@ syncmode = "full"
 
 
 # [jsonrpc]
-  # ipcdisable = false
-  # ipcpath = ""
-  # gascap = 50000000
-  # txfeecap = 5.0
-  # [jsonrpc.http]
-    # enabled = false
-    # port = 8545
-    # prefix = ""
-    # host = "localhost"
-    # api = ["eth", "net", "web3", "txpool", "bor"]
-    # vhosts = ["*"]
-    # corsdomain = ["*"]
-  # [jsonrpc.ws]
-    # enabled = false
-    # port = 8546
-    # prefix = ""
-    # host = "localhost"
-    # api = ["web3", "net"]
-    # origins = ["*"]
-  # [jsonrpc.graphql]
-    # enabled = false
-    # port = 0
-    # prefix = ""
-    # host = ""
-    # vhosts = ["*"]
-    # corsdomain = ["*"]
+#   ipcdisable = false
+#   ipcpath = ""
+#   gascap = 50000000
+#   txfeecap = 5.0
+#   [jsonrpc.http]
+#     enabled = false
+#     port = 8545
+#     prefix = ""
+#     host = "localhost"
+#     api = ["eth", "net", "web3", "txpool", "bor"]
+#     vhosts = ["*"]
+#     corsdomain = ["*"]
+#   [jsonrpc.ws]
+#     enabled = false
+#     port = 8546
+#     prefix = ""
+#     host = "localhost"
+#     api = ["web3", "net"]
+#     origins = ["*"]
+#   [jsonrpc.graphql]
+#     enabled = false
+#     port = 0
+#     prefix = ""
+#     host = ""
+#     vhosts = ["*"]
+#     corsdomain = ["*"]
+#   [jsonrpc.timeouts]
+#     read = "30s"
+#     write = "30s"
+#     idle = "2m0s"
 
 [gpo]
   # blocks = 20
@@ -122,6 +126,7 @@ syncmode = "full"
   # noprefetch = false
   # preimages = false
   # txlookuplimit = 2350000
+  # timeout = "1h0m0s"
 
 [accounts]
   # allow-insecure-unlock = true

docs/cli/example_config.toml

@@ -86,6 +86,10 @@ ethstats = ""                # Reporting URL of a ethstats service (nodename:sec
     host = ""                   # 
     vhosts = ["localhost"]      # Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.
     corsdomain = ["localhost"]  # Comma separated list of domains from which to accept cross origin requests (browser enforced)
+  [jsonrpc.timeouts]
+    read = "30s"
+    write = "30s"
+    idle = "2m0s"
 
 [gpo]
   blocks = 20                 # Number of recent blocks to check for gas prices
@@ -126,6 +130,7 @@ ethstats = ""                # Reporting URL of a ethstats service (nodename:sec
   preimages = false        # Enable recording the SHA3/keccak preimages of trie keys
   txlookuplimit = 2350000  # Number of recent blocks to maintain transactions index for (default = about 56 days, 0 = entire chain)
   triesinmemory = 128      # Number of block states (tries) to keep in memory
+  timeout = "1h0m0s"       # Time after which the Merkle Patricia Trie is stored to disc from memory
 
 [accounts]
   unlock = []                    # Comma separated list of accounts to unlock

internal/cli/dumpconfig.go

@@ -52,12 +52,16 @@ func (c *DumpconfigCommand) Run(args []string) int {
 	userConfig := command.GetConfig()
 
 	// convert the big.Int and time.Duration fields to their corresponding Raw fields
+	userConfig.JsonRPC.HttpTimeout.ReadTimeoutRaw = userConfig.JsonRPC.HttpTimeout.ReadTimeout.String()
+	userConfig.JsonRPC.HttpTimeout.WriteTimeoutRaw = userConfig.JsonRPC.HttpTimeout.WriteTimeout.String()
+	userConfig.JsonRPC.HttpTimeout.IdleTimeoutRaw = userConfig.JsonRPC.HttpTimeout.IdleTimeout.String()
 	userConfig.TxPool.RejournalRaw = userConfig.TxPool.Rejournal.String()
 	userConfig.TxPool.LifeTimeRaw = userConfig.TxPool.LifeTime.String()
 	userConfig.Sealer.GasPriceRaw = userConfig.Sealer.GasPrice.String()
 	userConfig.Gpo.MaxPriceRaw = userConfig.Gpo.MaxPrice.String()
 	userConfig.Gpo.IgnorePriceRaw = userConfig.Gpo.IgnorePrice.String()
 	userConfig.Cache.RejournalRaw = userConfig.Cache.Rejournal.String()
+	userConfig.Cache.TrieTimeoutRaw = userConfig.Cache.TrieTimeout.String()
 
 	if err := toml.NewEncoder(os.Stdout).Encode(userConfig); err != nil {
 		c.UI.Error(err.Error())

internal/cli/server/config.go

@@ -33,6 +33,7 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
 )
 
 type Config struct {
@@ -238,6 +239,8 @@ type JsonRPCConfig struct {
 
 	// Graphql has the json-rpc graphql related settings
 	Graphql *APIConfig `hcl:"graphql,block" toml:"graphql,block"`
+
+	HttpTimeout *HttpTimeouts `hcl:"timeouts,block" toml:"timeouts,block"`
 }
 
 type GRPCConfig struct {
@@ -271,6 +274,33 @@ type APIConfig struct {
 	Origins []string `hcl:"origins,optional" toml:"origins,optional"`
 }
 
+// Used from rpc.HTTPTimeouts
+type HttpTimeouts struct {
+	// ReadTimeout is the maximum duration for reading the entire
+	// request, including the body.
+	//
+	// Because ReadTimeout does not let Handlers make per-request
+	// decisions on each request body's acceptable deadline or
+	// upload rate, most users will prefer to use
+	// ReadHeaderTimeout. It is valid to use them both.
+	ReadTimeout    time.Duration `hcl:"-,optional" toml:"-"`
+	ReadTimeoutRaw string        `hcl:"read,optional" toml:"read,optional"`
+
+	// WriteTimeout is the maximum duration before timing out
+	// writes of the response. It is reset whenever a new
+	// request's header is read. Like ReadTimeout, it does not
+	// let Handlers make decisions on a per-request basis.
+	WriteTimeout    time.Duration `hcl:"-,optional" toml:"-"`
+	WriteTimeoutRaw string        `hcl:"write,optional" toml:"write,optional"`
+
+	// IdleTimeout is the maximum amount of time to wait for the
+	// next request when keep-alives are enabled. If IdleTimeout
+	// is zero, the value of ReadTimeout is used. If both are
+	// zero, ReadHeaderTimeout is used.
+	IdleTimeout    time.Duration `hcl:"-,optional" toml:"-"`
+	IdleTimeoutRaw string        `hcl:"idle,optional" toml:"idle,optional"`
+}
+
 type GpoConfig struct {
 	// Blocks is the number of blocks to track to compute the price oracle
 	Blocks uint64 `hcl:"blocks,optional" toml:"blocks,optional"`
@@ -367,6 +397,10 @@ type CacheConfig struct {
 
 	// TxLookupLimit sets the maximum number of blocks from head whose tx indices are reserved.
 	TxLookupLimit uint64 `hcl:"txlookuplimit,optional" toml:"txlookuplimit,optional"`
+
+	// Time after which the Merkle Patricia Trie is stored to disc from memory
+	TrieTimeout    time.Duration `hcl:"-,optional" toml:"-"`
+	TrieTimeoutRaw string        `hcl:"timeout,optional" toml:"timeout,optional"`
 }
 
 type AccountsConfig struct {
@@ -480,6 +514,11 @@ func DefaultConfig() *Config {
 				Cors:    []string{"localhost"},
 				VHost:   []string{"localhost"},
 			},
+			HttpTimeout: &HttpTimeouts{
+				ReadTimeout:  30 * time.Second,
+				WriteTimeout: 30 * time.Second,
+				IdleTimeout:  120 * time.Second,
+			},
 		},
 		Ethstats: "",
 		Telemetry: &TelemetryConfig{
@@ -511,6 +550,7 @@ func DefaultConfig() *Config {
 			NoPrefetch:    false,
 			Preimages:     false,
 			TxLookupLimit: 2350000,
+			TrieTimeout:   60 * time.Minute,
 		},
 		Accounts: &AccountsConfig{
 			Unlock:              []string{},
@@ -570,9 +610,13 @@ func (c *Config) fillTimeDurations() error {
 		td   *time.Duration
 		str  *string
 	}{
+		{"jsonrpc.timeouts.read", &c.JsonRPC.HttpTimeout.ReadTimeout, &c.JsonRPC.HttpTimeout.ReadTimeoutRaw},
+		{"jsonrpc.timeouts.write", &c.JsonRPC.HttpTimeout.WriteTimeout, &c.JsonRPC.HttpTimeout.WriteTimeoutRaw},
+		{"jsonrpc.timeouts.idle", &c.JsonRPC.HttpTimeout.IdleTimeout, &c.JsonRPC.HttpTimeout.IdleTimeoutRaw},
 		{"txpool.lifetime", &c.TxPool.LifeTime, &c.TxPool.LifeTimeRaw},
 		{"txpool.rejournal", &c.TxPool.Rejournal, &c.TxPool.RejournalRaw},
 		{"cache.rejournal", &c.Cache.Rejournal, &c.Cache.RejournalRaw},
+		{"cache.timeout", &c.Cache.TrieTimeout, &c.Cache.TrieTimeoutRaw},
 	}
 
 	for _, x := range tds {
@@ -830,6 +874,7 @@ func (c *Config) buildEth(stack *node.Node, accountManager *accounts.Manager) (*
 		n.NoPrefetch = c.Cache.NoPrefetch
 		n.Preimages = c.Cache.Preimages
 		n.TxLookupLimit = c.Cache.TxLookupLimit
+		n.TrieTimeout = c.Cache.TrieTimeout
 	}
 
 	n.RPCGasCap = c.JsonRPC.GasCap
@@ -928,6 +973,11 @@ func (c *Config) buildNode() (*node.Config, error) {
 		WSPathPrefix:        c.JsonRPC.Ws.Prefix,
 		GraphQLCors:         c.JsonRPC.Graphql.Cors,
 		GraphQLVirtualHosts: c.JsonRPC.Graphql.VHost,
+		HTTPTimeouts: rpc.HTTPTimeouts{
+			ReadTimeout:  c.JsonRPC.HttpTimeout.ReadTimeout,
+			WriteTimeout: c.JsonRPC.HttpTimeout.WriteTimeout,
+			IdleTimeout:  c.JsonRPC.HttpTimeout.IdleTimeout,
+		},
 	}
 
 	// dev mode

internal/cli/server/config_test.go

@@ -1,7 +1,6 @@
 package server
 
 import (
-	"math/big"
 	"testing"
 	"time"
 
@@ -101,38 +100,6 @@ func TestDefaultDatatypeOverride(t *testing.T) {
 	assert.Equal(t, c0, expected)
 }
 
-func TestConfigLoadFile(t *testing.T) {
-	readFile := func(path string) {
-		config, err := readConfigFile(path)
-		assert.NoError(t, err)
-
-		assert.Equal(t, config, &Config{
-			DataDir: "./data",
-			P2P: &P2PConfig{
-				MaxPeers: 30,
-			},
-			TxPool: &TxPoolConfig{
-				LifeTime: 1 * time.Second,
-			},
-			Gpo: &GpoConfig{
-				MaxPrice: big.NewInt(100),
-			},
-			Sealer: &SealerConfig{},
-			Cache:  &CacheConfig{},
-		})
-	}
-
-	// read file in hcl format
-	t.Run("hcl", func(t *testing.T) {
-		readFile("./testdata/test.hcl")
-	})
-
-	// read file in json format
-	t.Run("json", func(t *testing.T) {
-		readFile("./testdata/test.json")
-	})
-}
-
 var dummyEnodeAddr = "enode://0cb82b395094ee4a2915e9714894627de9ed8498fb881cec6db7c65e8b9a5bd7f2f25cc84e71e89d0947e51c76e85d0847de848c7782b13c0255247a6758178c@44.232.55.71:30303"
 
 func TestConfigBootnodesDefault(t *testing.T) {

packaging/templates/mainnet-v1/archive/config.toml

@@ -79,6 +79,10 @@ gcmode = "archive"
         # host = ""
         # vhosts = ["*"]
         # corsdomain = ["*"]
+    # [jsonrpc.timeouts]
+        # read = "30s"
+        # write = "30s"
+        # idle = "2m0s"
 
 [gpo]
     # blocks = 20
@@ -105,15 +109,16 @@ gcmode = "archive"
 
 [cache]
     cache = 4096
-    # gc = 25
-    # snapshot = 10
+    gc = 0
+    snapshot = 20
     # database = 50
-    # trie = 15
+    trie = 30
     # journal = "triecache"
     # rejournal = "1h0m0s"
     # noprefetch = false
     # preimages = false
     # txlookuplimit = 2350000
+    # timeout = "1h0m0s"
 
 # [accounts]
     # unlock = []

packaging/templates/mainnet-v1/sentry/sentry/bor/config.toml

@@ -79,6 +79,10 @@ syncmode = "full"
         # host = ""
         # vhosts = ["*"]
         # corsdomain = ["*"]
+    # [jsonrpc.timeouts]
+        # read = "30s"
+        # write = "30s"
+        # idle = "2m0s"
 
 [gpo]
     # blocks = 20
@@ -114,6 +118,7 @@ syncmode = "full"
     # noprefetch = false
     # preimages = false
     # txlookuplimit = 2350000
+    # timeout = "1h0m0s"
 
 # [accounts]
     # unlock = []

packaging/templates/mainnet-v1/sentry/validator/bor/config.toml

@@ -81,6 +81,10 @@ syncmode = "full"
         # host = ""
         # vhosts = ["*"]
         # corsdomain = ["*"]
+    # [jsonrpc.timeouts]
+        # read = "30s"
+        # write = "30s"
+        # idle = "2m0s"
 
 [gpo]
     # blocks = 20
@@ -116,6 +120,7 @@ syncmode = "full"
     # noprefetch = false
     # preimages = false
     # txlookuplimit = 2350000
+    # timeout = "1h0m0s"
 
 [accounts]
     allow-insecure-unlock = true

packaging/templates/mainnet-v1/without-sentry/bor/config.toml

@@ -81,6 +81,10 @@ syncmode = "full"
         # host = ""
         # vhosts = ["*"]
         # corsdomain = ["*"]
+    # [jsonrpc.timeouts]
+        # read = "30s"
+        # write = "30s"
+        # idle = "2m0s"
 
 [gpo]
 #     blocks = 20
@@ -116,6 +120,7 @@ syncmode = "full"
 #     noprefetch = false
 #     preimages = false
 #     txlookuplimit = 2350000
+#     timeout = "1h0m0s"
 
 [accounts]
     allow-insecure-unlock = true