From 7e7a2cd2a266157fe05cd8169047a5a65f698ce2 Mon Sep 17 00:00:00 2001 From: Rohan Prasad Date: Thu, 25 Mar 2021 22:11:44 +0530 Subject: [PATCH 1/5] Move cache options to superflag for alpha --- dgraph/cmd/alpha/run.go | 27 ++++++++++++++------------- dgraph/cmd/zero/run.go | 3 +++ worker/server_state.go | 1 + x/flags.go | 3 --- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/dgraph/cmd/alpha/run.go b/dgraph/cmd/alpha/run.go index 17bb7fb8616..c1a9d9ccbde 100644 --- a/dgraph/cmd/alpha/run.go +++ b/dgraph/cmd/alpha/run.go @@ -108,10 +108,6 @@ they form a Raft group and provide synchronous replication. // --encryption_key_file enc.RegisterFlags(flag) - flag.String("cache_percentage", "0,65,35,0", - `Cache percentages summing up to 100 for various caches (FORMAT: PostingListCache,`+ - `PstoreBlockCache,PstoreIndexCache,WAL).`) - flag.StringP("postings", "p", "p", "Directory to store posting lists.") flag.String("tmp", "t", "Directory to store temporary buffers.") @@ -149,6 +145,16 @@ they form a Raft group and provide synchronous replication. "worker in a failed state. Use -1 to retry infinitely."). String()) + // Cache flags. + flag.String("cache", worker.CacheDefaults, z.NewSuperFlagHelp(worker.CacheDefaults). + Head("Cache options"). + Flag("size-mb", + "Total size of cache (in MB) to be used in Dgraph."). + Flag("percentage", + "Cache percentages summing up to 100 for various caches (FORMAT: PostingListCache,"+ + "PstoreBlockCache,PstoreIndexCache,WAL)"). + String()) + flag.String("raft", worker.RaftDefaults, z.NewSuperFlagHelp(worker.RaftDefaults). Head("Raft options"). Flag("idx", @@ -614,17 +620,12 @@ func run() { } bindall = Alpha.Conf.GetBool("bindall") - - totalCache := int64(Alpha.Conf.GetInt("cache_mb")) + cache := z.NewSuperFlag(Alpha.Conf.GetString("cache")).MergeAndCheckDefault( + worker.CacheDefaults) + totalCache := cache.GetInt64("size-mb") x.AssertTruef(totalCache >= 0, "ERROR: Cache size must be non-negative") - if Alpha.Conf.IsSet("lru_mb") { - glog.Warningln("--lru_mb is deprecated, use --cache_mb instead") - if !Alpha.Conf.IsSet("cache_mb") { - totalCache = int64(Alpha.Conf.GetFloat64("lru_mb")) - } - } - cachePercentage := Alpha.Conf.GetString("cache_percentage") + cachePercentage := cache.GetString("percentage") cachePercent, err := x.GetCachePercentages(cachePercentage, 4) x.Check(err) postingListCacheSize := (cachePercent[0] * (totalCache << 20)) / 100 diff --git a/dgraph/cmd/zero/run.go b/dgraph/cmd/zero/run.go index cb37e15489d..69009107cc9 100644 --- a/dgraph/cmd/zero/run.go +++ b/dgraph/cmd/zero/run.go @@ -90,6 +90,9 @@ instances to achieve high-availability. // --tls SuperFlag x.RegisterServerTLSFlags(flag) + // Cache flags. + flag.Int64("cache_mb", 1024, "Total size of cache (in MB) to be used in Dgraph.") + flag.IntP("port_offset", "o", 0, "Value added to all listening port numbers. [Grpc=5080, HTTP=6080]") flag.Int("replicas", 1, "How many Dgraph Alpha replicas to run per data shard group."+ diff --git a/worker/server_state.go b/worker/server_state.go index f87aabf29bb..367efb41c0a 100644 --- a/worker/server_state.go +++ b/worker/server_state.go @@ -50,6 +50,7 @@ const ( `mutations-nquad=1000000; disallow-drop=false; query-timeout=0ms;` GraphQLDefaults = `introspection=true; debug=false; extensions=true; poll-interval=1s; ` + `lambda-url=;` + CacheDefaults = `size-mb=1024; percentage=0,65,35,0;` ) // ServerState holds the state of the Dgraph server. diff --git a/x/flags.go b/x/flags.go index 13ea3b71cfb..198327dad02 100644 --- a/x/flags.go +++ b/x/flags.go @@ -54,9 +54,6 @@ func FillCommonFlags(flag *pflag.FlagSet) { `guaranteeing no data loss in case of hard reboot.`+"\n "+ `Most users should be OK with choosing "process".`) - // Cache flags. - flag.Int64("cache_mb", 1024, "Total size of cache (in MB) to be used in Dgraph.") - flag.String("telemetry", TelemetryDefaults, z.NewSuperFlagHelp(TelemetryDefaults). Head("Telemetry (diagnostic) options"). Flag("reports", From a0ae11d951421aedd84703b7f64c1c49234ac80e Mon Sep 17 00:00:00 2001 From: Rohan Prasad Date: Fri, 26 Mar 2021 16:31:38 +0530 Subject: [PATCH 2/5] Remove cache_mb flag from zero config --- dgraph/cmd/zero/run.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/dgraph/cmd/zero/run.go b/dgraph/cmd/zero/run.go index 69009107cc9..cb37e15489d 100644 --- a/dgraph/cmd/zero/run.go +++ b/dgraph/cmd/zero/run.go @@ -90,9 +90,6 @@ instances to achieve high-availability. // --tls SuperFlag x.RegisterServerTLSFlags(flag) - // Cache flags. - flag.Int64("cache_mb", 1024, "Total size of cache (in MB) to be used in Dgraph.") - flag.IntP("port_offset", "o", 0, "Value added to all listening port numbers. [Grpc=5080, HTTP=6080]") flag.Int("replicas", 1, "How many Dgraph Alpha replicas to run per data shard group."+ From 9e42afb9682e4e92d79ad58db1d78a297d8413d7 Mon Sep 17 00:00:00 2001 From: Rohan Prasad Date: Wed, 31 Mar 2021 13:53:22 +0530 Subject: [PATCH 3/5] Fix cache_mb endpoint for admin/config --- dgraph/cmd/alpha/admin.go | 2 +- dgraph/cmd/alpha/run.go | 1 + worker/worker.go | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dgraph/cmd/alpha/admin.go b/dgraph/cmd/alpha/admin.go index d43fb285b3d..32f498853d4 100644 --- a/dgraph/cmd/alpha/admin.go +++ b/dgraph/cmd/alpha/admin.go @@ -222,7 +222,7 @@ func memoryLimitPutHandler(w http.ResponseWriter, r *http.Request) { } gqlReq := &schema.Request{ Query: ` - mutation config($cacheMb: Int) { + mutation config($cacheMb: Float) { config(input: {cacheMb: $cacheMb}) { response { code diff --git a/dgraph/cmd/alpha/run.go b/dgraph/cmd/alpha/run.go index c1a9d9ccbde..ee2e5ac066e 100644 --- a/dgraph/cmd/alpha/run.go +++ b/dgraph/cmd/alpha/run.go @@ -645,6 +645,7 @@ func run() { WALDir: Alpha.Conf.GetString("wal"), PostingDirCompression: ctype, PostingDirCompressionLevel: clevel, + CacheMb: totalCache, CachePercentage: cachePercentage, PBlockCacheSize: pstoreBlockCacheSize, PIndexCacheSize: pstoreIndexCacheSize, diff --git a/worker/worker.go b/worker/worker.go index 87a2c31f3b8..ae87f431243 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -164,6 +164,8 @@ func UpdateCacheMb(memoryMB int64) error { if _, err := pstore.CacheMaxCost(badger.IndexCache, indexCacheSize); err != nil { return errors.Wrapf(err, "cannot update index cache size") } + + Config.CacheMb = memoryMB return nil } From 79166966d0165330f38a13cc00f99fca4c09e03e Mon Sep 17 00:00:00 2001 From: Rohan Prasad Date: Thu, 1 Apr 2021 13:08:59 +0530 Subject: [PATCH 4/5] Remove WAL cache --- dgraph/cmd/alpha/run.go | 6 ++---- worker/config.go | 2 -- worker/worker.go | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/dgraph/cmd/alpha/run.go b/dgraph/cmd/alpha/run.go index ee2e5ac066e..a1c8b6cb50d 100644 --- a/dgraph/cmd/alpha/run.go +++ b/dgraph/cmd/alpha/run.go @@ -152,7 +152,7 @@ they form a Raft group and provide synchronous replication. "Total size of cache (in MB) to be used in Dgraph."). Flag("percentage", "Cache percentages summing up to 100 for various caches (FORMAT: PostingListCache,"+ - "PstoreBlockCache,PstoreIndexCache,WAL)"). + "PstoreBlockCache,PstoreIndexCache)"). String()) flag.String("raft", worker.RaftDefaults, z.NewSuperFlagHelp(worker.RaftDefaults). @@ -626,12 +626,11 @@ func run() { x.AssertTruef(totalCache >= 0, "ERROR: Cache size must be non-negative") cachePercentage := cache.GetString("percentage") - cachePercent, err := x.GetCachePercentages(cachePercentage, 4) + cachePercent, err := x.GetCachePercentages(cachePercentage, 3) x.Check(err) postingListCacheSize := (cachePercent[0] * (totalCache << 20)) / 100 pstoreBlockCacheSize := (cachePercent[1] * (totalCache << 20)) / 100 pstoreIndexCacheSize := (cachePercent[2] * (totalCache << 20)) / 100 - walCache := (cachePercent[3] * (totalCache << 20)) / 100 badger := z.NewSuperFlag(Alpha.Conf.GetString("badger")).MergeAndCheckDefault( worker.BadgerDefaults) @@ -649,7 +648,6 @@ func run() { CachePercentage: cachePercentage, PBlockCacheSize: pstoreBlockCacheSize, PIndexCacheSize: pstoreIndexCacheSize, - WalCache: walCache, MutationsMode: worker.AllowMutations, AuthToken: security.GetString("token"), diff --git a/worker/config.go b/worker/config.go index 7d520a6bd08..0daae39e376 100644 --- a/worker/config.go +++ b/worker/config.go @@ -54,8 +54,6 @@ type Options struct { PBlockCacheSize int64 // PIndexCacheSize is the size of index cache for pstore PIndexCacheSize int64 - // WalCache is the size of block cache for wstore - WalCache int64 // HmacSecret stores the secret used to sign JSON Web Tokens (JWT). HmacSecret x.SensitiveByteSlice diff --git a/worker/worker.go b/worker/worker.go index ae87f431243..2be3103d95d 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -149,7 +149,7 @@ func UpdateCacheMb(memoryMB int64) error { return errors.Errorf("cache_mb must be non-negative") } - cachePercent, err := x.GetCachePercentages(Config.CachePercentage, 4) + cachePercent, err := x.GetCachePercentages(Config.CachePercentage, 3) if err != nil { return err } From a5c0cfe82e8ab959f44abe04dd0d01a5aaaec379 Mon Sep 17 00:00:00 2001 From: Rohan Prasad Date: Thu, 1 Apr 2021 13:12:13 +0530 Subject: [PATCH 5/5] Remove WAL cache default --- worker/server_state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/server_state.go b/worker/server_state.go index 367efb41c0a..eceaa020ce7 100644 --- a/worker/server_state.go +++ b/worker/server_state.go @@ -50,7 +50,7 @@ const ( `mutations-nquad=1000000; disallow-drop=false; query-timeout=0ms;` GraphQLDefaults = `introspection=true; debug=false; extensions=true; poll-interval=1s; ` + `lambda-url=;` - CacheDefaults = `size-mb=1024; percentage=0,65,35,0;` + CacheDefaults = `size-mb=1024; percentage=0,65,35;` ) // ServerState holds the state of the Dgraph server.