Instrument AWS S3 requests with standard RED metrics (#776)

* Instrument s3 requests with RED metrics

* Update changelog

CHANGELOG.md

@@ -4,6 +4,7 @@
 * [ENHANCEMENT] Performance: improve compaction speed with concurrent reads and writes [#754](https://github.com/grafana/tempo/pull/754) (@mdisibio)
 * [ENHANCEMENT] Improve readability of cpu and memory metrics on operational dashboard [#764](https://github.com/grafana/tempo/pull/764) (@bboreham)
 * [ENHANCEMENT] Add `azure_request_duration_seconds` metric. [#767](https://github.com/grafana/tempo/pull/767) (@JosephWoodward)
+* [ENHANCEMENT] Add `s3_request_duration_seconds` metric. [#776](https://github.com/grafana/tempo/pull/776) (@JosephWoodward)
 
 ## v1.0.1
 

tempodb/backend/s3/instrumentation.go

@@ -0,0 +1,41 @@
+package s3
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	s3RequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "tempodb",
+		Name:      "s3_request_duration_seconds",
+		Help:      "Time spent doing AWS S3 requests.",
+
+		Buckets: prometheus.ExponentialBuckets(0.005, 4, 6),
+	}, []string{"operation", "status_code"})
+)
+
+type instrumentedTransport struct {
+	observer prometheus.ObserverVec
+	next     http.RoundTripper
+}
+
+func newInstrumentedTransport(next http.RoundTripper) http.RoundTripper {
+	return instrumentedTransport{
+		observer: s3RequestDuration,
+		next:     next,
+	}
+}
+
+func (i instrumentedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	start := time.Now()
+	resp, err := i.next.RoundTrip(req)
+	if err == nil {
+		i.observer.WithLabelValues(req.Method, strconv.Itoa(resp.StatusCode)).Observe(time.Since(start).Seconds())
+	}
+	return resp, err
+}

tempodb/backend/s3/s3.go

@@ -394,22 +394,23 @@ func createCore(cfg *Config, hedge bool) (*minio.Core, error) {
 		}),
 	})
 
-	transport, err := minio.DefaultTransport(!cfg.Insecure)
+	customTransport, err := minio.DefaultTransport(!cfg.Insecure)
 	if err != nil {
 		return nil, errors.Wrap(err, "create minio.DefaultTransport")
 	}
 
-	var roundtripper http.RoundTripper
-	roundtripper = transport
+	// add instrumentation
+	transport := newInstrumentedTransport(customTransport)
+
 	if hedge && cfg.HedgeRequestsAt != 0 {
-		roundtripper = hedgedhttp.NewRoundTripper(cfg.HedgeRequestsAt, uptoHedgedRequests, roundtripper)
+		transport = hedgedhttp.NewRoundTripper(cfg.HedgeRequestsAt, uptoHedgedRequests, transport)
 	}
 
 	opts := &minio.Options{
 		Region:    cfg.Region,
 		Secure:    !cfg.Insecure,
 		Creds:     creds,
-		Transport: roundtripper,
+		Transport: transport,
 	}
 
 	if cfg.ForcePathStyle {