Skip to content

Commit

Permalink
feat: add metrics to http endpoints (#3201)
Browse files Browse the repository at this point in the history
* feat: add metrics to http endpoints

* fix nil pointer error

* fix tests
  • Loading branch information
mathnogueira authored Oct 3, 2023
1 parent cf0ba12 commit 3ae60b0
Show file tree
Hide file tree
Showing 10 changed files with 221 additions and 11 deletions.
23 changes: 19 additions & 4 deletions server/app/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,15 @@ import (
"github.com/kubeshop/tracetest/server/provisioning"
"github.com/kubeshop/tracetest/server/resourcemanager"
"github.com/kubeshop/tracetest/server/subscription"
"github.com/kubeshop/tracetest/server/telemetry"
"github.com/kubeshop/tracetest/server/test"
"github.com/kubeshop/tracetest/server/testconnection"
"github.com/kubeshop/tracetest/server/testdb"
"github.com/kubeshop/tracetest/server/testsuite"
"github.com/kubeshop/tracetest/server/tracedb"
"github.com/kubeshop/tracetest/server/traces"
"github.com/kubeshop/tracetest/server/tracing"
"github.com/kubeshop/tracetest/server/variableset"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/trace"
)

Expand Down Expand Up @@ -180,13 +181,19 @@ func (app *App) Start(opts ...appOption) error {
configRepo := config.NewRepository(db, config.WithPublisher(subscriptionManager))
configFromDB := configRepo.Current(ctx)

tracer, err := tracing.NewTracer(ctx, app.cfg)
tracer, err := telemetry.NewTracer(ctx, app.cfg)
if err != nil {
log.Fatal(err)
}

meter, err := telemetry.NewMeter(ctx, app.cfg)
if err != nil {
log.Fatal(err)
}

app.registerStopFn(func() {
fmt.Println("stopping tracer")
tracing.ShutdownTracer(ctx)
telemetry.ShutdownTracer(ctx)
})

serverID, isNewInstall, err := testDB.ServerID()
Expand All @@ -208,7 +215,7 @@ func (app *App) Start(opts ...appOption) error {
}
}

applicationTracer, err := tracing.GetApplicationTracer(ctx, app.cfg)
applicationTracer, err := telemetry.GetApplicationTracer(ctx, app.cfg)
if err != nil {
return fmt.Errorf("could not create trigger span tracer: %w", err)
}
Expand Down Expand Up @@ -290,6 +297,7 @@ func (app *App) Start(opts ...appOption) error {

router, mappers := controller(app.cfg,
tracer,
meter,

testPipeline,
testSuitePipeline,
Expand All @@ -305,6 +313,10 @@ func (app *App) Start(opts ...appOption) error {
)
registerWSHandler(router, mappers, subscriptionManager)

// report metrics about endpoints, this is the first middleware to be run so
// it also accounts for the duration of all other middlewares
router.Use(middleware.NewMetricMiddleware(meter))

// use the analytics middleware on complete router
router.Use(analyticsMW)

Expand Down Expand Up @@ -551,6 +563,7 @@ func controller(
cfg httpServerConfig,

tracer trace.Tracer,
meter metric.Meter,

testRunner *executor.TestPipeline,
testSuitesRunner *executor.TestSuitesPipeline,
Expand All @@ -571,6 +584,7 @@ func controller(
cfg,

tracer,
meter,

testRunner,
testSuitesRunner,
Expand All @@ -594,6 +608,7 @@ func httpRouter(
cfg httpServerConfig,

tracer trace.Tracer,
meter metric.Meter,

testRunner *executor.TestPipeline,
testSuitesRunner *executor.TestSuitesPipeline,
Expand Down
8 changes: 5 additions & 3 deletions server/config/exporters.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package config

import (
"fmt"
"time"
)

type (
Expand All @@ -20,9 +21,10 @@ type (
}

TelemetryExporterOption struct {
ServiceName string `yaml:",omitempty" mapstructure:"serviceName"`
Sampling float64 `yaml:",omitempty" mapstructure:"sampling"`
Exporter ExporterConfig `yaml:",omitempty" mapstructure:"exporter"`
ServiceName string `yaml:",omitempty" mapstructure:"serviceName"`
Sampling float64 `yaml:",omitempty" mapstructure:"sampling"`
MetricsReaderInterval time.Duration `yaml:",omitempty" mapstructure:"metricsReaderInterval"`
Exporter ExporterConfig `yaml:",omitempty" mapstructure:"exporter"`
}

ExporterConfig struct {
Expand Down
4 changes: 2 additions & 2 deletions server/executor/poller_executor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,13 @@ import (
"github.com/kubeshop/tracetest/server/model"
"github.com/kubeshop/tracetest/server/pkg/id"
"github.com/kubeshop/tracetest/server/subscription"
"github.com/kubeshop/tracetest/server/telemetry"
"github.com/kubeshop/tracetest/server/test"
"github.com/kubeshop/tracetest/server/test/trigger"
"github.com/kubeshop/tracetest/server/testdb"
"github.com/kubeshop/tracetest/server/tracedb"
"github.com/kubeshop/tracetest/server/tracedb/connection"
"github.com/kubeshop/tracetest/server/traces"
"github.com/kubeshop/tracetest/server/tracing"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"

Expand Down Expand Up @@ -577,7 +577,7 @@ func getEventEmitterMock(t *testing.T, db *testdb.MockRepository) executor.Event
func getTracerMock(t *testing.T) trace.Tracer {
t.Helper()

tracer, err := tracing.NewTracer(context.TODO(), config.Must(config.New()))
tracer, err := telemetry.NewTracer(context.TODO(), config.Must(config.New()))
require.NoError(t, err)

return tracer
Expand Down
3 changes: 3 additions & 0 deletions server/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -148,8 +148,11 @@ require (
go.opentelemetry.io/collector/extension v0.80.0 // indirect
go.opentelemetry.io/collector/extension/auth v0.80.0 // indirect
go.opentelemetry.io/collector/featuregate v1.0.0-rcv0015 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect
go.opentelemetry.io/otel/metric v1.19.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.19.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.26.0 // indirect
Expand Down
7 changes: 7 additions & 0 deletions server/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1874,8 +1874,13 @@ go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydA
go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs=
go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY=
go.opentelemetry.io/otel/exporters/jaeger v1.0.0/go.mod h1:q10N1AolE1JjqKrFJK2tYw0iZpmX+HBaXBtuCzRnBGQ=
go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg=
go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM=
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0 h1:NmnYCiR0qNufkldjVvyQfZTHSdzeHoZ41zggMsdMcLM=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0/go.mod h1:UVAO61+umUsHLtYb8KXXRoHtxUkdOPkYidzW3gipRLQ=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE=
Expand Down Expand Up @@ -1905,6 +1910,8 @@ go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4
go.opentelemetry.io/otel/sdk/metric v0.23.0/go.mod h1:wa0sKK13eeIFW+0OFjcC3S1i7FTRRiLAXe1kjBVbhwg=
go.opentelemetry.io/otel/sdk/metric v0.39.0 h1:Kun8i1eYf48kHH83RucG93ffz0zGV1sh46FAScOTuDI=
go.opentelemetry.io/otel/sdk/metric v0.39.0/go.mod h1:piDIRgjcK7u0HCL5pCA4e74qpK/jk3NiUoAHATVAmiI=
go.opentelemetry.io/otel/sdk/metric v1.19.0 h1:EJoTO5qysMsYCa+w4UghwFV/ptQgqSL/8Ni+hx+8i1k=
go.opentelemetry.io/otel/sdk/metric v1.19.0/go.mod h1:XjG0jQyFJrv2PbMvwND7LwCEhsJzCzV5210euduKcKY=
go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
go.opentelemetry.io/otel/trace v1.0.0-RC3/go.mod h1:VUt2TUYd8S2/ZRX09ZDFZQwn2RqfMB5MzO17jBojGxo=
go.opentelemetry.io/otel/trace v1.0.0/go.mod h1:PXTWqayeFUlJV1YDNhsJYB184+IvAH814St6o6ajzIs=
Expand Down
69 changes: 69 additions & 0 deletions server/http/middleware/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package middleware

import (
"net/http"
"time"

"github.com/gorilla/mux"
semconv "go.opentelemetry.io/collector/semconv/v1.5.0"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
)

type httpMetricMiddleware struct {
next http.Handler
requestDurationHistogram metric.Int64Histogram
requestCounter metric.Int64Counter
}

func (m *httpMetricMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request) {
rw := NewStatusCodeCapturerWriter(w)
initialTime := time.Now()
m.next.ServeHTTP(rw, r)
duration := time.Since(initialTime)

route := mux.CurrentRoute(r)
pathTemplate, _ := route.GetPathTemplate()

metricAttributes := []attribute.KeyValue{
attribute.String(semconv.AttributeHTTPRoute, pathTemplate),
attribute.String(semconv.AttributeHTTPMethod, r.Method),
attribute.Int(semconv.AttributeHTTPStatusCode, rw.statusCode),
}

if tenantID := TenantIDFromContext(r.Context()); tenantID != "" {
metricAttributes = append(metricAttributes, attribute.String("tracetest.tenant_id", tenantID))
}

m.requestDurationHistogram.Record(r.Context(), duration.Milliseconds(), metric.WithAttributes(metricAttributes...))
m.requestCounter.Add(r.Context(), 1, metric.WithAttributes(metricAttributes...))
}

var _ http.Handler = &httpMetricMiddleware{}

type responseWriter struct {
http.ResponseWriter
statusCode int
}

func NewStatusCodeCapturerWriter(w http.ResponseWriter) *responseWriter {
return &responseWriter{w, http.StatusOK}
}

func (lrw *responseWriter) WriteHeader(code int) {
lrw.statusCode = code
lrw.ResponseWriter.WriteHeader(code)
}

func NewMetricMiddleware(meter metric.Meter) mux.MiddlewareFunc {
return func(next http.Handler) http.Handler {
durationHistogram, _ := meter.Int64Histogram("http.server.duration", metric.WithUnit("ms"))
requestCounter, _ := meter.Int64Counter("http.server.request.count")

return &httpMetricMiddleware{
next: next,
requestDurationHistogram: durationHistogram,
requestCounter: requestCounter,
}
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package tracing
package telemetry

import (
"context"
Expand Down
77 changes: 77 additions & 0 deletions server/telemetry/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
package telemetry

import (
"context"
"fmt"
"log"
"time"

"github.com/kubeshop/tracetest/server/config"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/metric/noop"
metricsdk "go.opentelemetry.io/otel/sdk/metric"
)

func NewMeter(ctx context.Context, cfg exporterConfig) (metric.Meter, error) {
provider, err := newMeterProvider(ctx, cfg)
if err != nil {
return nil, fmt.Errorf("could not create meter provider: %w", err)
}

return provider.Meter("tracetest"), nil
}

func newMeterProvider(ctx context.Context, cfg exporterConfig) (metric.MeterProvider, error) {
exporterConfig, err := cfg.Exporter()
if err != nil {
return nil, fmt.Errorf("could not get exporter config: %w", err)
}

if exporterConfig == nil {
log.Println("empty exporter config: falling back to noop meter provider")
return noop.NewMeterProvider(), nil
}

resource, err := getResource(exporterConfig)
if err != nil {
return nil, fmt.Errorf("could not get resource: %w", err)
}

collectorExporter, err := getOtelMetricsCollectorExporter(ctx, exporterConfig)
if err != nil {
return nil, fmt.Errorf("could not get collector exporter: %w", err)
}

interval := 60 * time.Second
if exporterConfig.MetricsReaderInterval != 0 {
interval = exporterConfig.MetricsReaderInterval
}

periodicReader := metricsdk.NewPeriodicReader(collectorExporter,
metricsdk.WithInterval(interval),
)

provider := metricsdk.NewMeterProvider(
metricsdk.WithResource(resource),
metricsdk.WithReader(periodicReader),
)

return provider, nil
}

func getOtelMetricsCollectorExporter(ctx context.Context, exporterConfig *config.TelemetryExporterOption) (metricsdk.Exporter, error) {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()

exporter, err := otlpmetricgrpc.New(ctx,
otlpmetricgrpc.WithEndpoint(exporterConfig.Exporter.CollectorConfiguration.Endpoint),
otlpmetricgrpc.WithCompressor("gzip"),
otlpmetricgrpc.WithInsecure(),
)
if err != nil {
return nil, fmt.Errorf("could not create trace exporter: %w", err)
}

return exporter, nil
}
37 changes: 37 additions & 0 deletions server/telemetry/resource.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package telemetry

import (
"fmt"
"os"

"github.com/kubeshop/tracetest/server/config"
"go.opentelemetry.io/otel/sdk/resource"
semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
)

func getResource(cfg *config.TelemetryExporterOption) (*resource.Resource, error) {
hostname, err := os.Hostname()
if err != nil {
return nil, fmt.Errorf("could not get OS hostname: %w", err)
}

serviceName := "tracetest"
if cfg != nil && cfg.ServiceName != "" {
serviceName = cfg.ServiceName
}

resource, err := resource.Merge(
resource.Default(),
resource.NewWithAttributes(
semconv.SchemaURL,
semconv.ServiceNameKey.String(serviceName),
semconv.HostName(hostname),
),
)

if err != nil {
return nil, fmt.Errorf("could not merge resources: %w", err)
}

return resource, nil
}
2 changes: 1 addition & 1 deletion server/tracing/tracer.go → server/telemetry/tracer.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package tracing
package telemetry

import (
"context"
Expand Down

0 comments on commit 3ae60b0

Please sign in to comment.