From 91c7d344fe244471ba4eda2c33c6027663445371 Mon Sep 17 00:00:00 2001 From: Christian Haudum Date: Tue, 8 Oct 2024 09:48:36 +0200 Subject: [PATCH] chore: Make metric for dequeued tasks in bloom-gateway a Histogram (#14413) This change allows to observe the distribution of how many tasks are dequeued at once over time. Signed-off-by: Christian Haudum --- pkg/bloomgateway/metrics.go | 9 +++++---- pkg/bloomgateway/worker.go | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/bloomgateway/metrics.go b/pkg/bloomgateway/metrics.go index 9fe096eec2ac4..4eeffbf8ad682 100644 --- a/pkg/bloomgateway/metrics.go +++ b/pkg/bloomgateway/metrics.go @@ -119,7 +119,7 @@ type workerMetrics struct { dequeueDuration *prometheus.HistogramVec queueDuration *prometheus.HistogramVec processDuration *prometheus.HistogramVec - tasksDequeued *prometheus.CounterVec + tasksDequeued *prometheus.HistogramVec tasksProcessed *prometheus.CounterVec blocksNotAvailable *prometheus.CounterVec blockQueryLatency *prometheus.HistogramVec @@ -147,11 +147,12 @@ func newWorkerMetrics(registerer prometheus.Registerer, namespace, subsystem str Name: "process_duration_seconds", Help: "Time spent processing tasks in seconds", }, append(labels, "status")), - tasksDequeued: r.NewCounterVec(prometheus.CounterOpts{ + tasksDequeued: r.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: subsystem, - Name: "tasks_dequeued_total", - Help: "Total amount of tasks that the worker dequeued from the queue", + Name: "tasks_dequeued", + Help: "Total amount of tasks that the worker dequeued from the queue at once", + Buckets: prometheus.ExponentialBuckets(1, 2, 8), // [1, 2, ..., 128] }, append(labels, "status")), tasksProcessed: r.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, diff --git a/pkg/bloomgateway/worker.go b/pkg/bloomgateway/worker.go index 6aa1082b89334..81092448ab52f 100644 --- a/pkg/bloomgateway/worker.go +++ b/pkg/bloomgateway/worker.go @@ -76,7 +76,7 @@ func (w *worker) running(_ context.Context) error { if err == queue.ErrStopped && len(items) == 0 { return err } - w.metrics.tasksDequeued.WithLabelValues(w.id, labelFailure).Inc() + w.metrics.tasksDequeued.WithLabelValues(w.id, labelFailure).Observe(1) level.Error(w.logger).Log("msg", "failed to dequeue tasks", "err", err, "items", len(items)) } idx = newIdx @@ -86,7 +86,7 @@ func (w *worker) running(_ context.Context) error { continue } - w.metrics.tasksDequeued.WithLabelValues(w.id, labelSuccess).Add(float64(len(items))) + w.metrics.tasksDequeued.WithLabelValues(w.id, labelSuccess).Observe(float64(len(items))) tasks := make([]Task, 0, len(items)) for _, item := range items {