fix: Prevent proposal from being dropped accidentally (#7741) (#7813)

Each proposal in alpha has a unique key but this unique key is not guaranteed to be unique. We could end up creating duplicate keys and these duplicate keys would get ignored in the raft loop. This PR proposes a different way of creating the proposal keys so that they are always unique. Fixes - DGRAPH-3012 Co-authored-by: Ibrahim Jarif <[email protected]>
hypermodeinc · May 13, 2021 · 594b39c · 594b39c
1 parent af5030a
commit 594b39c
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 1 deletion.
diff --git a/worker/draft.go b/worker/draft.go
@@ -1669,6 +1669,7 @@ func (n *node) retryUntilSuccess(fn func() error, pause time.Duration) {
 
 // InitAndStartNode gets called after having at least one membership sync with the cluster.
 func (n *node) InitAndStartNode() {
+	initProposalKey(n.Id)
 	_, restart, err := n.PastLife()
 	x.Check(err)
 

diff --git a/worker/proposal.go b/worker/proposal.go
@@ -27,6 +27,7 @@ import (
 	"github.com/dgraph-io/dgraph/protos/pb"
 	"github.com/dgraph-io/dgraph/schema"
 	"github.com/dgraph-io/dgraph/x"
+	"github.com/dgraph-io/ristretto/z"
 
 	ostats "go.opencensus.io/stats"
 	tag "go.opencensus.io/tag"
@@ -108,9 +109,18 @@ func (rl *rateLimiter) decr(retry int) {
 	rl.c.Broadcast()
 }
 
+var proposalKey uint64
+
+// {2 bytes Node ID} {4 bytes for random} {2 bytes zero}
+func initProposalKey(id uint64) {
+	x.AssertTrue(id != 0)
+	proposalKey = uint64(groups().Node.Id)<<48 | uint64(z.FastRand())<<16
+}
+
 // uniqueKey is meant to be unique across all the replicas.
+// initProposalKey should be called before calling uniqueKey.
 func uniqueKey() uint64 {
-	return uint64(groups().Node.Id)<<32 | uint64(groups().Node.Rand.Uint32())
+	return atomic.AddUint64(&proposalKey, 1)
 }
 
 var errInternalRetry = errors.New("Retry Raft proposal internally")