Added option to disable extended writes

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## main / unreleased
 
 * [ENHANCEMENT] Improve WAL Replay by not rebuilding the WAL. [#668](https://github.com/grafana/tempo/pull/668)
+* [ENHANCEMENT] Add config option to disable write extension to the ingesters. [#677](https://github.com/grafana/tempo/pull/677)
 
 ## v0.7.0
 

docs/tempo/website/configuration/manifest.md

@@ -95,6 +95,7 @@ distributor:
         instance_addr: ""
     receivers: {}
     override_ring_key: distributor
+    extend_writes: true
 ingester_client:
     pool_config:
         checkinterval: 15s

modules/distributor/config.go

@@ -34,6 +34,10 @@ type Config struct {
 	Receivers       map[string]interface{} `yaml:"receivers"`
 	OverrideRingKey string                 `yaml:"override_ring_key"`
 
+	// disables write extension with inactive ingesters. Use this along with ingester.lifecycler.unregister_on_shutdown = true
+	//  note that setting these two config values reduces tolerance to failures on rollout b/c there is always one guaranteed to be failing replica
+	ExtendWrites bool `yaml:"extend_writes"`
+
 	// For testing.
 	factory func(addr string) (ring_client.PoolClient, error) `yaml:"-"`
 }
@@ -45,4 +49,5 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet)
 	cfg.DistributorRing.HeartbeatTimeout = 5 * time.Minute
 
 	cfg.OverrideRingKey = ring.DistributorRingKey
+	cfg.ExtendWrites = true
 }

modules/distributor/distributor.go

@@ -261,7 +261,12 @@ func (d *Distributor) sendToIngestersViaBytes(ctx context.Context, userID string
 		rawRequests[i] = b
 	}
 
-	err := ring.DoBatch(ctx, ring.Write, d.ingestersRing, keys, func(ingester ring.InstanceDesc, indexes []int) error {
+	op := ring.WriteNoExtend
+	if d.cfg.ExtendWrites {
+		op = ring.Write
+	}
+
+	err := ring.DoBatch(ctx, op, d.ingestersRing, keys, func(ingester ring.InstanceDesc, indexes []int) error {
 
 		localCtx, cancel := context.WithTimeout(context.Background(), d.clientCfg.RemoteTimeout)
 		defer cancel()