Added option to disable extended writes (#677)

* Added option to disable extended writes Signed-off-by: Joe Elliott <[email protected]> * Added changelog/docs Signed-off-by: Joe Elliott <[email protected]> * docs Signed-off-by: Joe Elliott <[email protected]> * Added more comments Signed-off-by: Joe Elliott <[email protected]>
grafana · Apr 27, 2021 · 341c01b · 341c01b
1 parent f048eb3
commit 341c01b
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,7 @@
 ## main / unreleased
 
 * [ENHANCEMENT] Improve WAL Replay by not rebuilding the WAL. [#668](https://github.com/grafana/tempo/pull/668)
+* [ENHANCEMENT] Add config option to disable write extension to the ingesters. [#677](https://github.com/grafana/tempo/pull/677)
 
 ## v0.7.0
 

diff --git a/docs/tempo/website/configuration/manifest.md b/docs/tempo/website/configuration/manifest.md
@@ -95,6 +95,7 @@ distributor:
         instance_addr: ""
     receivers: {}
     override_ring_key: distributor
+    extend_writes: true
 ingester_client:
     pool_config:
         checkinterval: 15s

diff --git a/modules/distributor/config.go b/modules/distributor/config.go
@@ -34,6 +34,10 @@ type Config struct {
 	Receivers       map[string]interface{} `yaml:"receivers"`
 	OverrideRingKey string                 `yaml:"override_ring_key"`
 
+	// disables write extension with inactive ingesters. Use this along with ingester.lifecycler.unregister_on_shutdown = true
+	//  note that setting these two config values reduces tolerance to failures on rollout b/c there is always one guaranteed to be failing replica
+	ExtendWrites bool `yaml:"extend_writes"`
+
 	// For testing.
 	factory func(addr string) (ring_client.PoolClient, error) `yaml:"-"`
 }
@@ -45,4 +49,5 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet)
 	cfg.DistributorRing.HeartbeatTimeout = 5 * time.Minute
 
 	cfg.OverrideRingKey = ring.DistributorRingKey
+	cfg.ExtendWrites = true
 }
diff --git a/modules/distributor/distributor.go b/modules/distributor/distributor.go
@@ -261,7 +261,12 @@ func (d *Distributor) sendToIngestersViaBytes(ctx context.Context, userID string
 		rawRequests[i] = b
 	}
 
-	err := ring.DoBatch(ctx, ring.Write, d.ingestersRing, keys, func(ingester ring.InstanceDesc, indexes []int) error {
+	op := ring.WriteNoExtend
+	if d.cfg.ExtendWrites {
+		op = ring.Write
+	}
+
+	err := ring.DoBatch(ctx, op, d.ingestersRing, keys, func(ingester ring.InstanceDesc, indexes []int) error {
 
 		localCtx, cancel := context.WithTimeout(context.Background(), d.clientCfg.RemoteTimeout)
 		defer cancel()