From e64d0ff173313f681e022ff357d8bac57a28b6df Mon Sep 17 00:00:00 2001 From: Mohammed Galalen Date: Sun, 8 Sep 2024 15:04:53 +0400 Subject: [PATCH 1/3] replace hedged requests roundtrips total with counter --- modules/querier/external/client.go | 6 +++--- pkg/hedgedmetrics/metrics.go | 4 ++-- tempodb/backend/instrumentation/hedged_requests.go | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/querier/external/client.go b/modules/querier/external/client.go index 1be7dfa1f61..5d601ca01cc 100644 --- a/modules/querier/external/client.go +++ b/modules/querier/external/client.go @@ -30,11 +30,11 @@ var ( NativeHistogramMaxBucketNumber: 100, NativeHistogramMinResetDuration: 1 * time.Hour, }, []string{"endpoint"}) - metricExternalHedgedRequests = promauto.NewGauge( - prometheus.GaugeOpts{ + metricExternalHedgedRequests = promauto.NewCounter( + prometheus.CounterOpts{ Namespace: "tempo", Name: "querier_external_endpoint_hedged_roundtrips_total", - Help: "Total number of hedged external requests. Registered as a gauge for code sanity. This is a counter.", + Help: "Total number of hedged external requests.", }, ) ) diff --git a/pkg/hedgedmetrics/metrics.go b/pkg/hedgedmetrics/metrics.go index 10a29b8a2a1..7b08af935f7 100644 --- a/pkg/hedgedmetrics/metrics.go +++ b/pkg/hedgedmetrics/metrics.go @@ -12,7 +12,7 @@ const ( ) // PublishHedgedMetrics flushes metrics from hedged requests every 10 seconds -func Publish(s *hedgedhttp.Stats, gauge prometheus.Gauge) { +func Publish(s *hedgedhttp.Stats, counter prometheus.Counter) { ticker := time.NewTicker(hedgedMetricsPublishDuration) go func() { for range ticker.C { @@ -21,7 +21,7 @@ func Publish(s *hedgedhttp.Stats, gauge prometheus.Gauge) { if hedgedRequests < 0 { hedgedRequests = 0 } - gauge.Set(float64(hedgedRequests)) + counter.Add(float64(hedgedRequests)) } }() } diff --git a/tempodb/backend/instrumentation/hedged_requests.go b/tempodb/backend/instrumentation/hedged_requests.go index f44b6035d15..e85cb5c9809 100644 --- a/tempodb/backend/instrumentation/hedged_requests.go +++ b/tempodb/backend/instrumentation/hedged_requests.go @@ -7,11 +7,11 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" ) -var hedgedRequestsMetrics = promauto.NewGauge( - prometheus.GaugeOpts{ +var hedgedRequestsMetrics = promauto.NewCounter( + prometheus.CounterOpts{ Namespace: "tempodb", Name: "backend_hedged_roundtrips_total", - Help: "Total number of hedged backend requests. Registered as a gauge for code sanity. This is a counter.", + Help: "Total number of hedged backend requests.", }, ) From 189d0b466f979c3592dfe6ad00d019c4d740fd07 Mon Sep 17 00:00:00 2001 From: Mohammed Galalen Date: Sun, 8 Sep 2024 16:50:16 +0400 Subject: [PATCH 2/3] update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d98622c8604..efe6c40cacd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,7 @@ * [BUGFIX] Correct block end time when the ingested traces are outside the ingestion slack [#3954](https://github.com/grafana/tempo/pull/3954) (@javiermolinar) * [BUGFIX] Fix race condition where a streaming response could be marshalled while being modified in the combiner resulting in a panic. [#3961](https://github.com/grafana/tempo/pull/3961) (@joe-elliott) * [BUGFIX] Pass search options to the backend for SearchTagValuesBlocksV2 requests [#3971](https://github.com/grafana/tempo/pull/3971) (@javiermolinar) +* [BUGFIX] Replace hedged requests roundtrips total with a counter. [#4060](https://github.com/grafana/tempo/issues/4060) (@galalen) ## v2.5.0 From f465f3926988c25663aaa71349931cf678d0caaf Mon Sep 17 00:00:00 2001 From: Mohammed Galalen Date: Mon, 9 Sep 2024 22:10:11 +0400 Subject: [PATCH 3/3] Update CHANGELOG.md Co-authored-by: Mario --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efe6c40cacd..775216d2571 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,7 +93,7 @@ * [BUGFIX] Correct block end time when the ingested traces are outside the ingestion slack [#3954](https://github.com/grafana/tempo/pull/3954) (@javiermolinar) * [BUGFIX] Fix race condition where a streaming response could be marshalled while being modified in the combiner resulting in a panic. [#3961](https://github.com/grafana/tempo/pull/3961) (@joe-elliott) * [BUGFIX] Pass search options to the backend for SearchTagValuesBlocksV2 requests [#3971](https://github.com/grafana/tempo/pull/3971) (@javiermolinar) -* [BUGFIX] Replace hedged requests roundtrips total with a counter. [#4060](https://github.com/grafana/tempo/issues/4060) (@galalen) +* [BUGFIX] Replace hedged requests roundtrips total with a counter. [#4063](https://github.com/grafana/tempo/pull/4063) (@galalen) ## v2.5.0