From dc06bd0ac8ffd08ca9b3c0ca67bc489d58ee73b0 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 23 Jun 2021 17:51:46 +0100 Subject: [PATCH 1/3] Consolidate backend metrics, add heged metric --- CHANGELOG.md | 1 + go.mod | 2 +- go.sum | 4 +- tempodb/backend/azure/azure_helpers.go | 8 +- tempodb/backend/azure/instrumentation.go | 41 ----- tempodb/backend/gcs/gcs.go | 8 +- tempodb/backend/gcs/instrumentation.go | 43 ------ .../instrumentation/backend_transports.go | 80 ++++++++++ .../instrumentation/hedged_requests.go | 33 ++++ tempodb/backend/s3/instrumentation.go | 41 ----- tempodb/backend/s3/s3.go | 8 +- .../github.com/cristalhq/hedgedhttp/hedged.go | 145 ++++++++++++++++-- vendor/modules.txt | 2 +- 13 files changed, 272 insertions(+), 144 deletions(-) delete mode 100644 tempodb/backend/azure/instrumentation.go delete mode 100644 tempodb/backend/gcs/instrumentation.go create mode 100644 tempodb/backend/instrumentation/backend_transports.go create mode 100644 tempodb/backend/instrumentation/hedged_requests.go delete mode 100644 tempodb/backend/s3/instrumentation.go diff --git a/CHANGELOG.md b/CHANGELOG.md index ef690c19498..09aa95ba0c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## main / unreleased +* [ENHANCEMENT] Added hedged request metric `backend_hedged_roundtrips_total` and general purpose `backend_request_duration_seconds` metric that will supersede soon to be deprecated storage specific metrics (eg `request_duration_seconds`). [#790](https://github.com/grafana/tempo/pull/790) (@JosephWoodward) * [CHANGE] Jsonnet: use dedicated configmaps for distributors and ingesters [#775](https://github.com/grafana/tempo/pull/775) (@kvrhdn) * [FEATURE] Added the ability to hedge requests with all backends [#750](https://github.com/grafana/tempo/pull/750) (@joe-elliott) * [ENHANCEMENT] Performance: improve compaction speed with concurrent reads and writes [#754](https://github.com/grafana/tempo/pull/754) (@mdisibio) diff --git a/go.mod b/go.mod index a73f683cd71..96f6051ed4f 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/alecthomas/kong v0.2.11 github.com/cespare/xxhash v1.1.0 github.com/cortexproject/cortex v1.8.1-0.20210422151339-cf1c444e0905 - github.com/cristalhq/hedgedhttp v0.4.0 + github.com/cristalhq/hedgedhttp v0.6.0 github.com/drone/envsubst v1.0.3 github.com/dustin/go-humanize v1.0.0 github.com/go-kit/kit v0.10.0 diff --git a/go.sum b/go.sum index a509f048720..8cecbbe577b 100644 --- a/go.sum +++ b/go.sum @@ -313,8 +313,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cristalhq/hedgedhttp v0.4.0 h1:J1z1zKJ1bEFpMLjZWgtX0inUWlWecNyouWRIQknGzgM= -github.com/cristalhq/hedgedhttp v0.4.0/go.mod h1:XkqWU6qVMutbhW68NnzjWrGtH8NUx1UfYqGYtHVKIsI= +github.com/cristalhq/hedgedhttp v0.6.0 h1:32REZ0SZ1q0xoRNpHP5ab+Qd3VseyURXjN3HFpBqTiE= +github.com/cristalhq/hedgedhttp v0.6.0/go.mod h1:XkqWU6qVMutbhW68NnzjWrGtH8NUx1UfYqGYtHVKIsI= github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b/go.mod h1:v9FBN7gdVTpiD/+LZ7Po0UKvROyT87uLVxTHVky/dlQ= github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f/go.mod h1:8S58EK26zhXSxzv7NQFpnliaOQsmDUxvoQO3rt154Vg= diff --git a/tempodb/backend/azure/azure_helpers.go b/tempodb/backend/azure/azure_helpers.go index a2c6d48fe7d..39b989395dd 100644 --- a/tempodb/backend/azure/azure_helpers.go +++ b/tempodb/backend/azure/azure_helpers.go @@ -8,6 +8,8 @@ import ( "strings" "time" + "github.com/grafana/tempo/tempodb/backend/instrumentation" + "github.com/Azure/azure-pipeline-go/pipeline" blob "github.com/Azure/azure-storage-blob-go/azblob" "github.com/cristalhq/hedgedhttp" @@ -35,11 +37,13 @@ func GetContainerURL(ctx context.Context, cfg *Config, hedge bool) (blob.Contain customTransport := http.DefaultTransport.(*http.Transport).Clone() // add instrumentation - transport := newInstrumentedTransport(customTransport) + transport := instrumentation.NewAzureTransport(customTransport) + var stats *hedgedhttp.Stats // hedge if desired (0 means disabled) if hedge && cfg.HedgeRequestsAt != 0 { - transport = hedgedhttp.NewRoundTripper(cfg.HedgeRequestsAt, uptoHedgedRequests, transport) + transport, stats = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, uptoHedgedRequests, transport) + instrumentation.PublishHedgedMetrics(stats) } client := http.Client{Transport: transport} diff --git a/tempodb/backend/azure/instrumentation.go b/tempodb/backend/azure/instrumentation.go deleted file mode 100644 index f57a2ff47af..00000000000 --- a/tempodb/backend/azure/instrumentation.go +++ /dev/null @@ -1,41 +0,0 @@ -package azure - -import ( - "net/http" - "strconv" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" -) - -var ( - azureRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "tempodb", - Name: "azure_request_duration_seconds", - Help: "Time spent doing Azure requests.", - - Buckets: prometheus.ExponentialBuckets(0.005, 4, 6), - }, []string{"operation", "status_code"}) -) - -type instrumentedTransport struct { - observer prometheus.ObserverVec - next http.RoundTripper -} - -func newInstrumentedTransport(next http.RoundTripper) http.RoundTripper { - return instrumentedTransport{ - observer: azureRequestDuration, - next: next, - } -} - -func (i instrumentedTransport) RoundTrip(req *http.Request) (*http.Response, error) { - start := time.Now() - resp, err := i.next.RoundTrip(req) - if err == nil { - i.observer.WithLabelValues(req.Method, strconv.Itoa(resp.StatusCode)).Observe(time.Since(start).Seconds()) - } - return resp, err -} diff --git a/tempodb/backend/gcs/gcs.go b/tempodb/backend/gcs/gcs.go index bd6099e85fd..0d1ed13a47d 100644 --- a/tempodb/backend/gcs/gcs.go +++ b/tempodb/backend/gcs/gcs.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "github.com/grafana/tempo/tempodb/backend/instrumentation" + "cloud.google.com/go/storage" "github.com/cristalhq/hedgedhttp" "github.com/google/uuid" @@ -319,11 +321,13 @@ func createBucket(ctx context.Context, cfg *Config, hedge bool) (*storage.Bucket } // add instrumentation - transport = newInstrumentedTransport(transport) + transport = instrumentation.NewGCSTransport(transport) + var stats *hedgedhttp.Stats // hedge if desired (0 means disabled) if hedge && cfg.HedgeRequestsAt != 0 { - transport = hedgedhttp.NewRoundTripper(cfg.HedgeRequestsAt, uptoHedgedRequests, transport) + transport, stats = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, uptoHedgedRequests, transport) + instrumentation.PublishHedgedMetrics(stats) } // build client diff --git a/tempodb/backend/gcs/instrumentation.go b/tempodb/backend/gcs/instrumentation.go deleted file mode 100644 index e147cbc318d..00000000000 --- a/tempodb/backend/gcs/instrumentation.go +++ /dev/null @@ -1,43 +0,0 @@ -package gcs - -import ( - "net/http" - "strconv" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" -) - -var ( - gcsRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "tempodb", - Name: "gcs_request_duration_seconds", - Help: "Time spent doing GCS requests.", - - // GCS latency seems to range from a few ms to a few secs and is - // important. So use 6 buckets from 5ms to 5s. - Buckets: prometheus.ExponentialBuckets(0.005, 4, 6), - }, []string{"operation", "status_code"}) -) - -type instrumentedTransport struct { - observer prometheus.ObserverVec - next http.RoundTripper -} - -func newInstrumentedTransport(next http.RoundTripper) http.RoundTripper { - return instrumentedTransport{ - observer: gcsRequestDuration, - next: next, - } -} - -func (i instrumentedTransport) RoundTrip(req *http.Request) (*http.Response, error) { - start := time.Now() - resp, err := i.next.RoundTrip(req) - if err == nil { - i.observer.WithLabelValues(req.Method, strconv.Itoa(resp.StatusCode)).Observe(time.Since(start).Seconds()) - } - return resp, err -} diff --git a/tempodb/backend/instrumentation/backend_transports.go b/tempodb/backend/instrumentation/backend_transports.go new file mode 100644 index 00000000000..9a93ab480b2 --- /dev/null +++ b/tempodb/backend/instrumentation/backend_transports.go @@ -0,0 +1,80 @@ +package instrumentation + +import ( + "net/http" + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + gcsRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "tempodb", + Name: "gcs_request_duration_seconds", + Help: "Time spent doing GCS requests.", + Buckets: prometheus.ExponentialBuckets(0.005, 4, 6), + }, []string{"operation", "status_code"}) + + s3RequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "tempodb", + Name: "s3_request_duration_seconds", + Help: "Time spent doing AWS S3 requests.", + Buckets: prometheus.ExponentialBuckets(0.005, 4, 6), + }, []string{"operation", "status_code"}) + + azureRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "tempodb", + Name: "azure_request_duration_seconds", + Help: "Time spent doing Azure requests.", + Buckets: prometheus.ExponentialBuckets(0.005, 4, 6), + }, []string{"operation", "status_code"}) + + requestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "tempodb", + Name: "backend_request_duration_seconds", + Help: "Time spent doing requests.", + Buckets: prometheus.ExponentialBuckets(0.005, 4, 6), + }, []string{"operation", "status_code"}) +) + +type instrumentedTransport struct { + legacyObserver prometheus.ObserverVec + observer prometheus.ObserverVec + next http.RoundTripper +} + +func NewGCSTransport(next http.RoundTripper) http.RoundTripper { + return instrumentedTransport{ + next: next, + observer: requestDuration, + legacyObserver: gcsRequestDuration, + } +} + +func NewS3Transport(next http.RoundTripper) http.RoundTripper { + return instrumentedTransport{ + next: next, + observer: requestDuration, + legacyObserver: s3RequestDuration, + } +} + +func NewAzureTransport(next http.RoundTripper) http.RoundTripper { + return instrumentedTransport{ + next: next, + observer: requestDuration, + legacyObserver: azureRequestDuration, + } +} + +func (i instrumentedTransport) RoundTrip(req *http.Request) (*http.Response, error) { + start := time.Now() + resp, err := i.next.RoundTrip(req) + if err == nil { + i.legacyObserver.WithLabelValues(req.Method, strconv.Itoa(resp.StatusCode)).Observe(time.Since(start).Seconds()) + i.observer.WithLabelValues(req.Method, strconv.Itoa(resp.StatusCode)).Observe(time.Since(start).Seconds()) + } + return resp, err +} diff --git a/tempodb/backend/instrumentation/hedged_requests.go b/tempodb/backend/instrumentation/hedged_requests.go new file mode 100644 index 00000000000..2c6ff396f4c --- /dev/null +++ b/tempodb/backend/instrumentation/hedged_requests.go @@ -0,0 +1,33 @@ +package instrumentation + +import ( + "time" + + "github.com/cristalhq/hedgedhttp" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +const ( + hedgedMetricsPublishDuration = 10 * time.Second +) + +var ( + hedgedRequestsMetrics = promauto.NewCounter( + prometheus.CounterOpts{ + Namespace: "tempodb", + Name: "backend_hedged_roundtrips_total", + Help: "Total number of hedged backend requests", + }, + ) +) + +// PublishHedgedMetrics flushes metrics from hedged requests every 10 seconds +func PublishHedgedMetrics(s *hedgedhttp.Stats) { + ticker := time.NewTicker(hedgedMetricsPublishDuration) + go func() { + for range ticker.C { + hedgedRequestsMetrics.Add(float64(s.Snapshot().RequestedRoundTrips)) + } + }() +} diff --git a/tempodb/backend/s3/instrumentation.go b/tempodb/backend/s3/instrumentation.go deleted file mode 100644 index 221c64313e6..00000000000 --- a/tempodb/backend/s3/instrumentation.go +++ /dev/null @@ -1,41 +0,0 @@ -package s3 - -import ( - "net/http" - "strconv" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" -) - -var ( - s3RequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "tempodb", - Name: "s3_request_duration_seconds", - Help: "Time spent doing AWS S3 requests.", - - Buckets: prometheus.ExponentialBuckets(0.005, 4, 6), - }, []string{"operation", "status_code"}) -) - -type instrumentedTransport struct { - observer prometheus.ObserverVec - next http.RoundTripper -} - -func newInstrumentedTransport(next http.RoundTripper) http.RoundTripper { - return instrumentedTransport{ - observer: s3RequestDuration, - next: next, - } -} - -func (i instrumentedTransport) RoundTrip(req *http.Request) (*http.Response, error) { - start := time.Now() - resp, err := i.next.RoundTrip(req) - if err == nil { - i.observer.WithLabelValues(req.Method, strconv.Itoa(resp.StatusCode)).Observe(time.Since(start).Seconds()) - } - return resp, err -} diff --git a/tempodb/backend/s3/s3.go b/tempodb/backend/s3/s3.go index 256830367e1..247d8b73292 100644 --- a/tempodb/backend/s3/s3.go +++ b/tempodb/backend/s3/s3.go @@ -9,6 +9,8 @@ import ( "net/http" "strings" + "github.com/grafana/tempo/tempodb/backend/instrumentation" + log_util "github.com/cortexproject/cortex/pkg/util/log" "github.com/cristalhq/hedgedhttp" "github.com/go-kit/kit/log" @@ -400,10 +402,12 @@ func createCore(cfg *Config, hedge bool) (*minio.Core, error) { } // add instrumentation - transport := newInstrumentedTransport(customTransport) + transport := instrumentation.NewS3Transport(customTransport) + var stats *hedgedhttp.Stats if hedge && cfg.HedgeRequestsAt != 0 { - transport = hedgedhttp.NewRoundTripper(cfg.HedgeRequestsAt, uptoHedgedRequests, transport) + transport, stats = hedgedhttp.NewRoundTripperAndStats(cfg.HedgeRequestsAt, uptoHedgedRequests, transport) + instrumentation.PublishHedgedMetrics(stats) } opts := &minio.Options{ diff --git a/vendor/github.com/cristalhq/hedgedhttp/hedged.go b/vendor/github.com/cristalhq/hedgedhttp/hedged.go index a90a6de3c2d..9361f992757 100644 --- a/vendor/github.com/cristalhq/hedgedhttp/hedged.go +++ b/vendor/github.com/cristalhq/hedgedhttp/hedged.go @@ -5,6 +5,8 @@ import ( "fmt" "net/http" "strings" + "sync" + "sync/atomic" "time" ) @@ -14,56 +16,89 @@ const infiniteTimeout = 30 * 24 * time.Hour // domain specific infinite // Given Client starts a new request after a timeout from previous request. // Starts no more than upto requests. func NewClient(timeout time.Duration, upto int, client *http.Client) *http.Client { + newClient, _ := NewClientAndStats(timeout, upto, client) + return newClient +} + +// NewClientAndStats returns a new http.Client which implements hedged requests pattern +// And Stats object that can be queried to obtain client's metrics. +// Given Client starts a new request after a timeout from previous request. +// Starts no more than upto requests. +func NewClientAndStats(timeout time.Duration, upto int, client *http.Client) (*http.Client, *Stats) { if client == nil { client = &http.Client{ Timeout: 5 * time.Second, } } - client.Transport = NewRoundTripper(timeout, upto, client.Transport) + newTransport, metrics := NewRoundTripperAndStats(timeout, upto, client.Transport) - return client + client.Transport = newTransport + + return client, metrics } // NewRoundTripper returns a new http.RoundTripper which implements hedged requests pattern. // Given RoundTripper starts a new request after a timeout from previous request. // Starts no more than upto requests. func NewRoundTripper(timeout time.Duration, upto int, rt http.RoundTripper) http.RoundTripper { + newRT, _ := NewRoundTripperAndStats(timeout, upto, rt) + return newRT +} + +// NewRoundTripperAndStats returns a new http.RoundTripper which implements hedged requests pattern +// And Stats object that can be queried to obtain client's metrics. +// Given RoundTripper starts a new request after a timeout from previous request. +// Starts no more than upto requests. +func NewRoundTripperAndStats(timeout time.Duration, upto int, rt http.RoundTripper) (http.RoundTripper, *Stats) { + switch { + case timeout < 0: + panic("hedgedhttp: timeout cannot be negative") + case upto < 1: + panic("hedgedhttp: upto must be greater than 0") + } + if rt == nil { rt = http.DefaultTransport } + + if timeout == 0 { + timeout = time.Nanosecond // smallest possible timeout if not set + } + hedged := &hedgedTransport{ rt: rt, timeout: timeout, upto: upto, + metrics: &Stats{}, } - return hedged + return hedged, hedged.metrics } type hedgedTransport struct { rt http.RoundTripper timeout time.Duration upto int + metrics *Stats } func (ht *hedgedTransport) RoundTrip(req *http.Request) (*http.Response, error) { mainCtx := req.Context() timeout := ht.timeout - if timeout == 0 { - timeout = time.Nanosecond // smallest possible timeout if not set - } - errOverall := &MultiError{} resultCh := make(chan indexedResp, ht.upto) errorCh := make(chan error, ht.upto) + ht.metrics.requestedRoundTripsInc() + resultIdx := -1 cancels := make([]func(), ht.upto) defer runInPool(func() { for i, cancel := range cancels { if i != resultIdx && cancel != nil { + ht.metrics.canceledSubRequestsInc() cancel() } } @@ -76,8 +111,10 @@ func (ht *hedgedTransport) RoundTrip(req *http.Request) (*http.Response, error) cancels[idx] = cancel runInPool(func() { + ht.metrics.actualRoundTripsInc() resp, err := ht.rt.RoundTrip(subReq) if err != nil { + ht.metrics.failedRoundTripsInc() errorCh <- err } else { resultCh <- indexedResp{idx, resp} @@ -96,6 +133,7 @@ func (ht *hedgedTransport) RoundTrip(req *http.Request) (*http.Response, error) resultIdx = resp.Index return resp.Resp, nil case mainCtx.Err() != nil: + ht.metrics.canceledByUserRoundTripsInc() return nil, mainCtx.Err() case err != nil: errOverall.Errors = append(errOverall.Errors, err) @@ -112,8 +150,8 @@ func waitResult(ctx context.Context, resultCh <-chan indexedResp, errorCh <-chan case res := <-resultCh: return res, nil default: - timer := time.NewTimer(timeout) - defer timer.Stop() + timer := getTimer(timeout) + defer returnTimer(timer) select { case res := <-resultCh: @@ -142,6 +180,76 @@ func reqWithCtx(r *http.Request, ctx context.Context) (*http.Request, func()) { return req, cancel } +// atomicCounter is a false sharing safe counter. +type atomicCounter struct { + count uint64 + _ [7]uint64 +} + +type cacheLine [64]byte + +// Stats object that can be queried to obtain certain metrics and get better observability. +type Stats struct { + _ cacheLine + requestedRoundTrips atomicCounter + actualRoundTrips atomicCounter + failedRoundTrips atomicCounter + canceledByUserRoundTrips atomicCounter + canceledSubRequests atomicCounter + _ cacheLine +} + +func (s *Stats) requestedRoundTripsInc() { atomic.AddUint64(&s.requestedRoundTrips.count, 1) } +func (s *Stats) actualRoundTripsInc() { atomic.AddUint64(&s.actualRoundTrips.count, 1) } +func (s *Stats) failedRoundTripsInc() { atomic.AddUint64(&s.failedRoundTrips.count, 1) } +func (s *Stats) canceledByUserRoundTripsInc() { atomic.AddUint64(&s.canceledByUserRoundTrips.count, 1) } +func (s *Stats) canceledSubRequestsInc() { atomic.AddUint64(&s.canceledSubRequests.count, 1) } + +// RequestedRoundTrips returns count of requests that were requested by client. +func (m *Stats) RequestedRoundTrips() uint64 { + return atomic.LoadUint64(&m.requestedRoundTrips.count) +} + +// ActualRoundTrips returns count of requests that were actually sent. +func (m *Stats) ActualRoundTrips() uint64 { + return atomic.LoadUint64(&m.actualRoundTrips.count) +} + +// FailedRoundTrips returns count of requests that failed. +func (m *Stats) FailedRoundTrips() uint64 { + return atomic.LoadUint64(&m.failedRoundTrips.count) +} + +// CanceledByUserRoundTrips returns count of requests that were canceled by user, using request context. +func (m *Stats) CanceledByUserRoundTrips() uint64 { + return atomic.LoadUint64(&m.canceledByUserRoundTrips.count) +} + +// CanceledSubRequests returns count of hedged sub-requests that were canceled by transport. +func (m *Stats) CanceledSubRequests() uint64 { + return atomic.LoadUint64(&m.canceledSubRequests.count) +} + +// StatsSnapshot is a snapshot of Stats. +type StatsSnapshot struct { + RequestedRoundTrips uint64 // count of requests that were requested by client + ActualRoundTrips uint64 // count of requests that were actually sent + FailedRoundTrips uint64 // count of requests that failed + CanceledByUserRoundTrips uint64 // count of requests that were canceled by user, using request context + CanceledSubRequests uint64 // count of hedged sub-requests that were canceled by transport +} + +// Snapshot of the stats. +func (m *Stats) Snapshot() StatsSnapshot { + return StatsSnapshot{ + RequestedRoundTrips: m.RequestedRoundTrips(), + ActualRoundTrips: m.ActualRoundTrips(), + FailedRoundTrips: m.FailedRoundTrips(), + CanceledByUserRoundTrips: m.CanceledByUserRoundTrips(), + CanceledSubRequests: m.CanceledSubRequests(), + } +} + var taskQueue = make(chan func()) func runInPool(task func()) { @@ -216,3 +324,22 @@ func listFormatFunc(es []error) string { return fmt.Sprintf("%d errors occurred:\n\t%s\n\n", len(es), strings.Join(points, "\n\t")) } + +var timerPool = sync.Pool{New: func() interface{} { + return time.NewTimer(time.Second) +}} + +func getTimer(duration time.Duration) *time.Timer { + timer := timerPool.Get().(*time.Timer) + timer.Reset(duration) + return timer +} + +func returnTimer(timer *time.Timer) { + timer.Stop() + select { + case _ = <-timer.C: + default: + } + timerPool.Put(timer) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 6e2603462b7..04064448355 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -247,7 +247,7 @@ github.com/cortexproject/cortex/pkg/util/services github.com/cortexproject/cortex/pkg/util/spanlogger github.com/cortexproject/cortex/pkg/util/tls github.com/cortexproject/cortex/pkg/util/validation -# github.com/cristalhq/hedgedhttp v0.4.0 +# github.com/cristalhq/hedgedhttp v0.6.0 ## explicit github.com/cristalhq/hedgedhttp # github.com/davecgh/go-spew v1.1.1 From 4a1572fe0f037e72e21a90ffa6dcfb8a0bda247b Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 30 Jun 2021 13:59:42 +0100 Subject: [PATCH 2/3] Update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09aa95ba0c8..85be54631b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## main / unreleased -* [ENHANCEMENT] Added hedged request metric `backend_hedged_roundtrips_total` and general purpose `backend_request_duration_seconds` metric that will supersede soon to be deprecated storage specific metrics (eg `request_duration_seconds`). [#790](https://github.com/grafana/tempo/pull/790) (@JosephWoodward) +* [ENHANCEMENT] Added hedged request metric `tempodb_backend_hedged_roundtrips_total` and a new storage agnostic `tempodb_backend_request_duration_seconds` metric that + supersedes the soon-to-be deprecated storage specific metrics (`tempodb_azure_request_duration_seconds`, `tempodb_s3_request_duration_seconds` and `tempodb_gcs_request_duration_seconds`). [#790](https://github.com/grafana/tempo/pull/790) (@JosephWoodward) * [CHANGE] Jsonnet: use dedicated configmaps for distributors and ingesters [#775](https://github.com/grafana/tempo/pull/775) (@kvrhdn) * [FEATURE] Added the ability to hedge requests with all backends [#750](https://github.com/grafana/tempo/pull/750) (@joe-elliott) * [ENHANCEMENT] Performance: improve compaction speed with concurrent reads and writes [#754](https://github.com/grafana/tempo/pull/754) (@mdisibio) From 65adc869352bea0e40f0a760484e2eecc2a62e03 Mon Sep 17 00:00:00 2001 From: Joseph Woodward Date: Wed, 30 Jun 2021 14:21:14 +0100 Subject: [PATCH 3/3] Add deprecated message to older metrics --- tempodb/backend/instrumentation/backend_transports.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tempodb/backend/instrumentation/backend_transports.go b/tempodb/backend/instrumentation/backend_transports.go index 9a93ab480b2..376ca3eae60 100644 --- a/tempodb/backend/instrumentation/backend_transports.go +++ b/tempodb/backend/instrumentation/backend_transports.go @@ -13,28 +13,28 @@ var ( gcsRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "tempodb", Name: "gcs_request_duration_seconds", - Help: "Time spent doing GCS requests.", + Help: "Time spent doing GCS requests. (DEPRECATED: See tempodb_backend_request_duration_seconds)", Buckets: prometheus.ExponentialBuckets(0.005, 4, 6), }, []string{"operation", "status_code"}) s3RequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "tempodb", Name: "s3_request_duration_seconds", - Help: "Time spent doing AWS S3 requests.", + Help: "Time spent doing AWS S3 requests. (DEPRECATED: See tempodb_backend_request_duration_seconds)", Buckets: prometheus.ExponentialBuckets(0.005, 4, 6), }, []string{"operation", "status_code"}) azureRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "tempodb", Name: "azure_request_duration_seconds", - Help: "Time spent doing Azure requests.", + Help: "Time spent doing Azure requests. (DEPRECATED: See tempodb_backend_request_duration_seconds)", Buckets: prometheus.ExponentialBuckets(0.005, 4, 6), }, []string{"operation", "status_code"}) requestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "tempodb", Name: "backend_request_duration_seconds", - Help: "Time spent doing requests.", + Help: "Time spent doing backend storage requests.", Buckets: prometheus.ExponentialBuckets(0.005, 4, 6), }, []string{"operation", "status_code"}) )