Fix poller waitgroup handling (#3224)

* Fix poller waitgroup handling

* Update changelog

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 * [BUGFIX] Change exit code if config is successfully verified [#3174](https://github.com/grafana/tempo/pull/3174) (@am3o @agrib-01)
 * [BUGFIX] The tempo-cli analyse blocks command no longer fails on compacted blocks [#3183](https://github.com/grafana/tempo/pull/3183) (@stoewer)
+* [BUGFIX] Move waitgroup handling for poller error condition [#3224](https://github.com/grafana/tempo/pull/3224) (@zalegrala)
 * [ENHANCEMENT] Introduced `AttributePolicyMatch` & `IntrinsicPolicyMatch` structures to match span attributes based on strongly typed values & precompiled regexp [#3025](https://github.com/grafana/tempo/pull/3025) (@andriusluk)
 * [CHANGE] TraceQL/Structural operators performance improvement. [#3088](https://github.com/grafana/tempo/pull/3088) (@joe-elliott)
 * [CHANGE] Merge the processors overrides set through runtime overrides and user-configurable overrides [#3125](https://github.com/grafana/tempo/pull/3125) (@kvrhdn)

tempodb/blocklist/poller.go

@@ -14,6 +14,7 @@ import (
 	"github.com/go-kit/log/level"
 	"github.com/google/uuid"
 	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/ext"
 	spanlog "github.com/opentracing/opentracing-go/log"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
@@ -335,6 +336,7 @@ func (p *Poller) pollUnknown(
 	defer span.Finish()
 
 	var (
+		err                   error
 		errs                  []error
 		mtx                   sync.Mutex
 		bg                    = boundedwaitgroup.New(p.cfg.PollConcurrency)
@@ -343,8 +345,6 @@ func (p *Poller) pollUnknown(
 	)
 
 	for blockID, compacted := range unknownBlocks {
-		bg.Add(1)
-
 		// Avoid polling if we've already encountered an error
 		mtx.Lock()
 		if len(errs) > 0 {
@@ -353,6 +353,7 @@ func (p *Poller) pollUnknown(
 		}
 		mtx.Unlock()
 
+		bg.Add(1)
 		go func(id uuid.UUID, compacted bool) {
 			defer bg.Done()
 
@@ -383,8 +384,11 @@ func (p *Poller) pollUnknown(
 
 	if len(errs) > 0 {
 		metricTenantIndexErrors.WithLabelValues(tenantID).Inc()
-		// TODO: add span status on error
-		return nil, nil, errors.Join(errs...)
+		err = errors.Join(errs...)
+		ext.Error.Set(span, true)
+		span.SetTag("err", err)
+
+		return nil, nil, err
 	}
 
 	return newBlockList, newCompactedBlocklist, nil