diff --git a/CHANGELOG.md b/CHANGELOG.md index d98622c8604..775216d2571 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,7 @@ * [BUGFIX] Correct block end time when the ingested traces are outside the ingestion slack [#3954](https://github.com/grafana/tempo/pull/3954) (@javiermolinar) * [BUGFIX] Fix race condition where a streaming response could be marshalled while being modified in the combiner resulting in a panic. [#3961](https://github.com/grafana/tempo/pull/3961) (@joe-elliott) * [BUGFIX] Pass search options to the backend for SearchTagValuesBlocksV2 requests [#3971](https://github.com/grafana/tempo/pull/3971) (@javiermolinar) +* [BUGFIX] Replace hedged requests roundtrips total with a counter. [#4063](https://github.com/grafana/tempo/pull/4063) (@galalen) ## v2.5.0 diff --git a/modules/querier/external/client.go b/modules/querier/external/client.go index 1be7dfa1f61..5d601ca01cc 100644 --- a/modules/querier/external/client.go +++ b/modules/querier/external/client.go @@ -30,11 +30,11 @@ var ( NativeHistogramMaxBucketNumber: 100, NativeHistogramMinResetDuration: 1 * time.Hour, }, []string{"endpoint"}) - metricExternalHedgedRequests = promauto.NewGauge( - prometheus.GaugeOpts{ + metricExternalHedgedRequests = promauto.NewCounter( + prometheus.CounterOpts{ Namespace: "tempo", Name: "querier_external_endpoint_hedged_roundtrips_total", - Help: "Total number of hedged external requests. Registered as a gauge for code sanity. This is a counter.", + Help: "Total number of hedged external requests.", }, ) ) diff --git a/pkg/hedgedmetrics/metrics.go b/pkg/hedgedmetrics/metrics.go index 10a29b8a2a1..7b08af935f7 100644 --- a/pkg/hedgedmetrics/metrics.go +++ b/pkg/hedgedmetrics/metrics.go @@ -12,7 +12,7 @@ const ( ) // PublishHedgedMetrics flushes metrics from hedged requests every 10 seconds -func Publish(s *hedgedhttp.Stats, gauge prometheus.Gauge) { +func Publish(s *hedgedhttp.Stats, counter prometheus.Counter) { ticker := time.NewTicker(hedgedMetricsPublishDuration) go func() { for range ticker.C { @@ -21,7 +21,7 @@ func Publish(s *hedgedhttp.Stats, gauge prometheus.Gauge) { if hedgedRequests < 0 { hedgedRequests = 0 } - gauge.Set(float64(hedgedRequests)) + counter.Add(float64(hedgedRequests)) } }() } diff --git a/tempodb/backend/instrumentation/hedged_requests.go b/tempodb/backend/instrumentation/hedged_requests.go index f44b6035d15..e85cb5c9809 100644 --- a/tempodb/backend/instrumentation/hedged_requests.go +++ b/tempodb/backend/instrumentation/hedged_requests.go @@ -7,11 +7,11 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" ) -var hedgedRequestsMetrics = promauto.NewGauge( - prometheus.GaugeOpts{ +var hedgedRequestsMetrics = promauto.NewCounter( + prometheus.CounterOpts{ Namespace: "tempodb", Name: "backend_hedged_roundtrips_total", - Help: "Total number of hedged backend requests. Registered as a gauge for code sanity. This is a counter.", + Help: "Total number of hedged backend requests.", }, )