From 56fbbcd107f7853e3e8f64df1e22f37a24cd367f Mon Sep 17 00:00:00 2001 From: Zach Leslie Date: Mon, 11 Dec 2023 21:12:12 +0000 Subject: [PATCH] Fix poller waitgroup handling (#3224) * Fix poller waitgroup handling * Update changelog --- CHANGELOG.md | 1 + tempodb/blocklist/poller.go | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e39c2cb158f..beef20e859c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ * [BUGFIX] Change exit code if config is successfully verified [#3174](https://github.com/grafana/tempo/pull/3174) (@am3o @agrib-01) * [BUGFIX] The tempo-cli analyse blocks command no longer fails on compacted blocks [#3183](https://github.com/grafana/tempo/pull/3183) (@stoewer) +* [BUGFIX] Move waitgroup handling for poller error condition [#3224](https://github.com/grafana/tempo/pull/3224) (@zalegrala) * [ENHANCEMENT] Introduced `AttributePolicyMatch` & `IntrinsicPolicyMatch` structures to match span attributes based on strongly typed values & precompiled regexp [#3025](https://github.com/grafana/tempo/pull/3025) (@andriusluk) * [CHANGE] TraceQL/Structural operators performance improvement. [#3088](https://github.com/grafana/tempo/pull/3088) (@joe-elliott) * [CHANGE] Merge the processors overrides set through runtime overrides and user-configurable overrides [#3125](https://github.com/grafana/tempo/pull/3125) (@kvrhdn) diff --git a/tempodb/blocklist/poller.go b/tempodb/blocklist/poller.go index b20df0f38f4..6afa995440b 100644 --- a/tempodb/blocklist/poller.go +++ b/tempodb/blocklist/poller.go @@ -14,6 +14,7 @@ import ( "github.com/go-kit/log/level" "github.com/google/uuid" opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go/ext" spanlog "github.com/opentracing/opentracing-go/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -335,6 +336,7 @@ func (p *Poller) pollUnknown( defer span.Finish() var ( + err error errs []error mtx sync.Mutex bg = boundedwaitgroup.New(p.cfg.PollConcurrency) @@ -343,8 +345,6 @@ func (p *Poller) pollUnknown( ) for blockID, compacted := range unknownBlocks { - bg.Add(1) - // Avoid polling if we've already encountered an error mtx.Lock() if len(errs) > 0 { @@ -353,6 +353,7 @@ func (p *Poller) pollUnknown( } mtx.Unlock() + bg.Add(1) go func(id uuid.UUID, compacted bool) { defer bg.Done() @@ -383,8 +384,11 @@ func (p *Poller) pollUnknown( if len(errs) > 0 { metricTenantIndexErrors.WithLabelValues(tenantID).Inc() - // TODO: add span status on error - return nil, nil, errors.Join(errs...) + err = errors.Join(errs...) + ext.Error.Set(span, true) + span.SetTag("err", err) + + return nil, nil, err } return newBlockList, newCompactedBlocklist, nil