use badger superflag

dgraph/cmd/alpha/run.go

@@ -620,6 +620,7 @@ func run() {
 	}
 
 	bindall = Alpha.Conf.GetBool("bindall")
+
 	cache := z.NewSuperFlag(Alpha.Conf.GetString("cache")).MergeAndCheckDefault(
 		worker.CacheDefaults)
 	totalCache := cache.GetInt64("size-mb")
@@ -631,23 +632,20 @@ func run() {
 	postingListCacheSize := (cachePercent[0] * (totalCache << 20)) / 100
 	pstoreBlockCacheSize := (cachePercent[1] * (totalCache << 20)) / 100
 	pstoreIndexCacheSize := (cachePercent[2] * (totalCache << 20)) / 100
-
 	badger := z.NewSuperFlag(Alpha.Conf.GetString("badger")).MergeAndCheckDefault(
 		worker.BadgerDefaults)
-	ctype, clevel := x.ParseCompression(badger.GetString("compression"))
 
 	security := z.NewSuperFlag(Alpha.Conf.GetString("security")).MergeAndCheckDefault(
 		worker.SecurityDefaults)
 	conf := audit.GetAuditConf(Alpha.Conf.GetString("audit"))
 	opts := worker.Options{
-		PostingDir:                 Alpha.Conf.GetString("postings"),
-		WALDir:                     Alpha.Conf.GetString("wal"),
-		PostingDirCompression:      ctype,
-		PostingDirCompressionLevel: clevel,
-		CacheMb:                    totalCache,
-		CachePercentage:            cachePercentage,
-		PBlockCacheSize:            pstoreBlockCacheSize,
-		PIndexCacheSize:            pstoreIndexCacheSize,
+		PostingDir: Alpha.Conf.GetString("postings"),
+		WALDir:     Alpha.Conf.GetString("wal"),
+
+		CacheMb:         totalCache,
+		CachePercentage: cachePercentage,
+		PBlockCacheSize: pstoreBlockCacheSize,
+		PIndexCacheSize: pstoreIndexCacheSize,
 
 		MutationsMode:  worker.AllowMutations,
 		AuthToken:      security.GetString("token"),

dgraph/cmd/bulk/loader.go

@@ -36,8 +36,8 @@ import (
 	"google.golang.org/grpc/credentials"
 
 	"github.com/dgraph-io/badger/v3"
-	bo "github.com/dgraph-io/badger/v3/options"
 	"github.com/dgraph-io/badger/v3/y"
+	"github.com/dgraph-io/ristretto/z"
 
 	"github.com/dgraph-io/dgraph/chunker"
 	"github.com/dgraph-io/dgraph/ee/enc"
@@ -85,12 +85,8 @@ type options struct {
 	// ........... Badger options ..........
 	// EncryptionKey is the key used for encryption. Enterprise only feature.
 	EncryptionKey x.SensitiveByteSlice
-	// BadgerCompression is the compression algorithm to use while writing to badger.
-	BadgerCompression bo.CompressionType
-	// BadgerCompressionlevel is the compression level to use while writing to badger.
-	BadgerCompressionLevel int
-	BlockCacheSize         int64
-	IndexCacheSize         int64
+	// Super flag for various the badger options.
+	Badger *z.SuperFlag
 }
 
 type state struct {

dgraph/cmd/bulk/reduce.go

@@ -133,15 +133,16 @@ func (r *reducer) createBadgerInternal(dir string, compression bool) *badger.DB
 	opt := badger.DefaultOptions(dir).
 		WithSyncWrites(false).
 		WithEncryptionKey(key).
-		WithBlockCacheSize(r.opt.BlockCacheSize).
-		WithIndexCacheSize(r.opt.IndexCacheSize)
+		FromSuperFlag(r.state.opt.Badger.String())
 
 	opt.Compression = bo.None
 	opt.ZSTDCompressionLevel = 0
 	// Overwrite badger options based on the options provided by the user.
 	if compression {
-		opt.Compression = r.state.opt.BadgerCompression
-		opt.ZSTDCompressionLevel = r.state.opt.BadgerCompressionLevel
+		// TODO: Parse it once.
+		ctype, clevel := x.ParseCompression(r.state.opt.Badger.GetString("compression"))
+		opt.Compression = ctype
+		opt.ZSTDCompressionLevel = clevel
 	}
 
 	db, err := badger.OpenManaged(opt)

dgraph/cmd/bulk/run.go

@@ -129,9 +129,9 @@ func init() {
 				"zstd compression at level 1.").
 		Flag("goroutines",
 			"The number of goroutines to use in badger.Stream.").
-		Flag("cache-mb",
+		Flag("cache_mb",
 			"Total size of cache (in MB) per shard in the reducer.").
-		Flag("cache-percentage",
+		Flag("cache_percentage",
 			"Cache percentages summing up to 100 for various caches. (FORMAT: BlockCacheSize,"+
 				"IndexCacheSize)").
 		String())
@@ -144,7 +144,6 @@ func init() {
 func run() {
 	badger := z.NewSuperFlag(Bulk.Conf.GetString("badger")).MergeAndCheckDefault(
 		BulkBadgerDefaults)
-	ctype, clevel := x.ParseCompression(badger.GetString("compression"))
 	opt := options{
 		DataFiles:        Bulk.Conf.GetString("files"),
 		DataFormat:       Bulk.Conf.GetString("format"),
@@ -172,25 +171,14 @@ func run() {
 		NewUids:          Bulk.Conf.GetBool("new_uids"),
 		ClientDir:        Bulk.Conf.GetString("xidmap"),
 		Namespace:        Bulk.Conf.GetUint64("force-namespace"),
-
-		// Badger options
-		BadgerCompression:      ctype,
-		BadgerCompressionLevel: clevel,
+		Badger:           badger,
 	}
 
 	x.PrintVersion()
 	if opt.Version {
 		os.Exit(0)
 	}
 
-	totalCache := int64(badger.GetUint64("cache-mb"))
-	x.AssertTruef(totalCache >= 0, "ERROR: Cache size must be non-negative")
-	cachePercent, err := x.GetCachePercentages(badger.GetString("cache-percentage"), 2)
-	x.Check(err)
-	totalCache <<= 20 // Convert to MB.
-	opt.BlockCacheSize = (cachePercent[0] * totalCache) / 100
-	opt.IndexCacheSize = (cachePercent[1] * totalCache) / 100
-
 	_, opt.EncryptionKey = ee.GetKeys(Bulk.Conf)
 	if len(opt.EncryptionKey) == 0 {
 		if opt.Encrypted || opt.EncryptedOut {

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/Shopify/sarama v1.27.2
 	github.com/blevesearch/bleve v1.0.13
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd
-	github.com/dgraph-io/badger/v3 v3.0.0-20210309075542-2245c18dfd1f
+	github.com/dgraph-io/badger/v3 v3.0.0-20210401135229-49e68bb1ea60
 	github.com/dgraph-io/dgo/v200 v200.0.0-20210212152539-e0a5bde40ba2
 	github.com/dgraph-io/gqlgen v0.13.2
 	github.com/dgraph-io/gqlparser/v2 v2.2.0

go.sum

@@ -119,6 +119,8 @@ github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Ev
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
 github.com/dgraph-io/badger/v3 v3.0.0-20210309075542-2245c18dfd1f h1:dZpGNLp9YUpq4h2DRcWAjW5dWj47SM3W3NK71z6FRa0=
 github.com/dgraph-io/badger/v3 v3.0.0-20210309075542-2245c18dfd1f/go.mod h1:GHMCYxuDWyzbHkh4k3yyg4PM61tJPFfEGSMbE3Vd5QE=
+github.com/dgraph-io/badger/v3 v3.0.0-20210401125755-247641b95e30 h1:mfjubR/ExHQfzKXaQf1hZWB9odAgPNxGqJuMnR6A7dM=
+github.com/dgraph-io/badger/v3 v3.0.0-20210401125755-247641b95e30/go.mod h1:GHMCYxuDWyzbHkh4k3yyg4PM61tJPFfEGSMbE3Vd5QE=
 github.com/dgraph-io/dgo/v200 v200.0.0-20210212152539-e0a5bde40ba2 h1:3STgJCaLdsBynA0FDLqocwk/OdlnrQ5MGbR74C4NvUs=
 github.com/dgraph-io/dgo/v200 v200.0.0-20210212152539-e0a5bde40ba2/go.mod h1:zCfS4R3E/UC/PhETXJYq/Blia0eCH1EQqKrWDvvimxE=
 github.com/dgraph-io/gqlgen v0.13.2 h1:TNhndk+eHKj5qE7BenKKSYdSIdOGhLqxR1rCiMso9KM=

worker/config.go

@@ -20,7 +20,6 @@ import (
 	"path/filepath"
 	"time"
 
-	bo "github.com/dgraph-io/badger/v3/options"
 	"github.com/dgraph-io/dgraph/x"
 )
 
@@ -37,12 +36,6 @@ const (
 type Options struct {
 	// PostingDir is the path to the directory storing the postings..
 	PostingDir string
-	// PostingDirCompression is the compression algorithem used to compression Postings directory.
-	PostingDirCompression bo.CompressionType
-	// PostingDirCompressionLevel is the ZSTD compression level used by Postings directory. A
-	// higher value means more CPU intensive compression and better compression
-	// ratio.
-	PostingDirCompressionLevel int
 	// WALDir is the path to the directory storing the write-ahead log.
 	WALDir string
 	// MutationsMode is the mode used to handle mutation requests.

worker/server_state.go

@@ -96,10 +96,6 @@ func setBadgerOptions(opt badger.Options) badger.Options {
 	// saved by disabling it.
 	opt.DetectConflicts = false
 
-	glog.Infof("Setting Posting Dir Compression Level: %d", Config.PostingDirCompressionLevel)
-	opt.Compression = Config.PostingDirCompression
-	opt.ZSTDCompressionLevel = Config.PostingDirCompressionLevel
-
 	// Settings for the data directory.
 	return opt
 }
@@ -131,10 +127,10 @@ func (s *ServerState) initStorage() {
 		x.Check(os.MkdirAll(Config.PostingDir, 0700))
 		opt := badger.DefaultOptions(Config.PostingDir).
 			WithNumVersionsToKeep(math.MaxInt32).
-			WithNumGoroutines(int(x.WorkerConfig.Badger.GetUint64("goroutines"))).
 			WithBlockCacheSize(Config.PBlockCacheSize).
 			WithIndexCacheSize(Config.PIndexCacheSize).
-			WithNamespaceOffset(x.NamespaceOffset)
+			WithNamespaceOffset(x.NamespaceOffset).
+			FromSuperFlag(x.WorkerConfig.Badger.String())
 		opt = setBadgerOptions(opt)
 
 		// Print the options w/o exposing key.