From 7a00f22ad14d2ea973637037ef11bca8e20a8812 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 20 Dec 2024 08:48:47 -0500 Subject: [PATCH 1/4] cleanup Signed-off-by: Joe Elliott --- cmd/tempo/app/modules.go | 7 +++++++ modules/ingester/flush.go | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/cmd/tempo/app/modules.go b/cmd/tempo/app/modules.go index 6652706d972..9b784e790d3 100644 --- a/cmd/tempo/app/modules.go +++ b/cmd/tempo/app/modules.go @@ -462,6 +462,13 @@ func (t *App) initOptionalStore() (services.Service, error) { } func (t *App) initStore() (services.Service, error) { + // the only component that needs a functioning tempodb pool are the queriers. all other components will just spin up + // hundreds of never used pool goroutines. set pool size to 0 here to avoid that. + if t.cfg.Target != Querier && t.cfg.Target != SingleBinary && t.cfg.Target != ScalableSingleBinary { + t.cfg.StorageConfig.Trace.Pool.MaxWorkers = 0 + t.cfg.StorageConfig.Trace.Pool.QueueDepth = 0 + } + store, err := tempo_storage.NewStore(t.cfg.StorageConfig, t.cacheProvider, log.Logger) if err != nil { return nil, fmt.Errorf("failed to create store: %w", err) diff --git a/modules/ingester/flush.go b/modules/ingester/flush.go index 9bb3afd2291..44ad218470c 100644 --- a/modules/ingester/flush.go +++ b/modules/ingester/flush.go @@ -204,7 +204,13 @@ func (i *Ingester) flushLoop(j int) { if o == nil { return } + op := o.(*flushOp) + // if we're shutting down then abandon all work for the ingester to complete on startup + if i.flushQueues.IsStopped() { + handleAbandonedOp(op) + continue + } op.attempts++ var retry bool From 0910f0db738e83a32a8794cc206f5e7dc9000e6b Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 20 Dec 2024 09:40:14 -0500 Subject: [PATCH 2/4] changelog Signed-off-by: Joe Elliott --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d85ef145bde..0bab4b3e662 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,7 @@ querier: * [ENHANCEMENT] Update Azurite image. [#4298](https://github.com/grafana/tempo/pull/4464) (@javiermolinar) * [ENHANCEMENT] Update golang.org/x/crypto [#4474](https://github.com/grafana/tempo/pull/4474) (@javiermolinar) * [ENHANCEMENT] Distributor shim: add test verifying receiver works (including metrics) [#4477](https://github.com/grafana/tempo/pull/4477) (@yvrhdn) +* [ENHANCEMENT] Reduce goroutines in all non-querier components and reduce ingester shutdown time. [#4484](https://github.com/grafana/tempo/pull/4484) (@joe-elliott) * [BUGFIX] Handle invalid TraceQL query filter in tag values v2 disk cache [#4392](https://github.com/grafana/tempo/pull/4392) (@electron0zero) * [BUGFIX] Replace hedged requests roundtrips total with a counter. [#4063](https://github.com/grafana/tempo/pull/4063) [#4078](https://github.com/grafana/tempo/pull/4078) (@galalen) * [BUGFIX] Metrics generators: Correctly drop from the ring before stopping ingestion to reduce drops during a rollout. [#4101](https://github.com/grafana/tempo/pull/4101) (@joe-elliott) From 6f62d760c7eaad0812e58c6af142739e8a2a154e Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 20 Dec 2024 11:25:32 -0500 Subject: [PATCH 3/4] remove shutdown change Signed-off-by: Joe Elliott --- CHANGELOG.md | 2 +- modules/ingester/flush.go | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bab4b3e662..fed0dc79c2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,7 +92,7 @@ querier: * [ENHANCEMENT] Update Azurite image. [#4298](https://github.com/grafana/tempo/pull/4464) (@javiermolinar) * [ENHANCEMENT] Update golang.org/x/crypto [#4474](https://github.com/grafana/tempo/pull/4474) (@javiermolinar) * [ENHANCEMENT] Distributor shim: add test verifying receiver works (including metrics) [#4477](https://github.com/grafana/tempo/pull/4477) (@yvrhdn) -* [ENHANCEMENT] Reduce goroutines in all non-querier components and reduce ingester shutdown time. [#4484](https://github.com/grafana/tempo/pull/4484) (@joe-elliott) +* [ENHANCEMENT] Reduce goroutines in all non-querier components. [#4484](https://github.com/grafana/tempo/pull/4484) (@joe-elliott) * [BUGFIX] Handle invalid TraceQL query filter in tag values v2 disk cache [#4392](https://github.com/grafana/tempo/pull/4392) (@electron0zero) * [BUGFIX] Replace hedged requests roundtrips total with a counter. [#4063](https://github.com/grafana/tempo/pull/4063) [#4078](https://github.com/grafana/tempo/pull/4078) (@galalen) * [BUGFIX] Metrics generators: Correctly drop from the ring before stopping ingestion to reduce drops during a rollout. [#4101](https://github.com/grafana/tempo/pull/4101) (@joe-elliott) diff --git a/modules/ingester/flush.go b/modules/ingester/flush.go index 44ad218470c..1f811e59909 100644 --- a/modules/ingester/flush.go +++ b/modules/ingester/flush.go @@ -206,11 +206,6 @@ func (i *Ingester) flushLoop(j int) { } op := o.(*flushOp) - // if we're shutting down then abandon all work for the ingester to complete on startup - if i.flushQueues.IsStopped() { - handleAbandonedOp(op) - continue - } op.attempts++ var retry bool From 549112c319b591a5d8eb3abb1daabb20811b1382 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 20 Dec 2024 11:27:23 -0500 Subject: [PATCH 4/4] remove space Signed-off-by: Joe Elliott --- modules/ingester/flush.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/ingester/flush.go b/modules/ingester/flush.go index 1f811e59909..9bb3afd2291 100644 --- a/modules/ingester/flush.go +++ b/modules/ingester/flush.go @@ -204,7 +204,6 @@ func (i *Ingester) flushLoop(j int) { if o == nil { return } - op := o.(*flushOp) op.attempts++