-
Notifications
You must be signed in to change notification settings - Fork 76
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add metrics to http endpoints (#3201)
* feat: add metrics to http endpoints * fix nil pointer error * fix tests
- Loading branch information
1 parent
cf0ba12
commit 3ae60b0
Showing
10 changed files
with
221 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,69 @@ | ||
package middleware | ||
|
||
import ( | ||
"net/http" | ||
"time" | ||
|
||
"github.com/gorilla/mux" | ||
semconv "go.opentelemetry.io/collector/semconv/v1.5.0" | ||
"go.opentelemetry.io/otel/attribute" | ||
"go.opentelemetry.io/otel/metric" | ||
) | ||
|
||
type httpMetricMiddleware struct { | ||
next http.Handler | ||
requestDurationHistogram metric.Int64Histogram | ||
requestCounter metric.Int64Counter | ||
} | ||
|
||
func (m *httpMetricMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request) { | ||
rw := NewStatusCodeCapturerWriter(w) | ||
initialTime := time.Now() | ||
m.next.ServeHTTP(rw, r) | ||
duration := time.Since(initialTime) | ||
|
||
route := mux.CurrentRoute(r) | ||
pathTemplate, _ := route.GetPathTemplate() | ||
|
||
metricAttributes := []attribute.KeyValue{ | ||
attribute.String(semconv.AttributeHTTPRoute, pathTemplate), | ||
attribute.String(semconv.AttributeHTTPMethod, r.Method), | ||
attribute.Int(semconv.AttributeHTTPStatusCode, rw.statusCode), | ||
} | ||
|
||
if tenantID := TenantIDFromContext(r.Context()); tenantID != "" { | ||
metricAttributes = append(metricAttributes, attribute.String("tracetest.tenant_id", tenantID)) | ||
} | ||
|
||
m.requestDurationHistogram.Record(r.Context(), duration.Milliseconds(), metric.WithAttributes(metricAttributes...)) | ||
m.requestCounter.Add(r.Context(), 1, metric.WithAttributes(metricAttributes...)) | ||
} | ||
|
||
var _ http.Handler = &httpMetricMiddleware{} | ||
|
||
type responseWriter struct { | ||
http.ResponseWriter | ||
statusCode int | ||
} | ||
|
||
func NewStatusCodeCapturerWriter(w http.ResponseWriter) *responseWriter { | ||
return &responseWriter{w, http.StatusOK} | ||
} | ||
|
||
func (lrw *responseWriter) WriteHeader(code int) { | ||
lrw.statusCode = code | ||
lrw.ResponseWriter.WriteHeader(code) | ||
} | ||
|
||
func NewMetricMiddleware(meter metric.Meter) mux.MiddlewareFunc { | ||
return func(next http.Handler) http.Handler { | ||
durationHistogram, _ := meter.Int64Histogram("http.server.duration", metric.WithUnit("ms")) | ||
requestCounter, _ := meter.Int64Counter("http.server.request.count") | ||
|
||
return &httpMetricMiddleware{ | ||
next: next, | ||
requestDurationHistogram: durationHistogram, | ||
requestCounter: requestCounter, | ||
} | ||
} | ||
} |
2 changes: 1 addition & 1 deletion
2
server/tracing/application_tracer.go → server/telemetry/application_tracer.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
package tracing | ||
package telemetry | ||
|
||
import ( | ||
"context" | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,77 @@ | ||
package telemetry | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"log" | ||
"time" | ||
|
||
"github.com/kubeshop/tracetest/server/config" | ||
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" | ||
"go.opentelemetry.io/otel/metric" | ||
"go.opentelemetry.io/otel/metric/noop" | ||
metricsdk "go.opentelemetry.io/otel/sdk/metric" | ||
) | ||
|
||
func NewMeter(ctx context.Context, cfg exporterConfig) (metric.Meter, error) { | ||
provider, err := newMeterProvider(ctx, cfg) | ||
if err != nil { | ||
return nil, fmt.Errorf("could not create meter provider: %w", err) | ||
} | ||
|
||
return provider.Meter("tracetest"), nil | ||
} | ||
|
||
func newMeterProvider(ctx context.Context, cfg exporterConfig) (metric.MeterProvider, error) { | ||
exporterConfig, err := cfg.Exporter() | ||
if err != nil { | ||
return nil, fmt.Errorf("could not get exporter config: %w", err) | ||
} | ||
|
||
if exporterConfig == nil { | ||
log.Println("empty exporter config: falling back to noop meter provider") | ||
return noop.NewMeterProvider(), nil | ||
} | ||
|
||
resource, err := getResource(exporterConfig) | ||
if err != nil { | ||
return nil, fmt.Errorf("could not get resource: %w", err) | ||
} | ||
|
||
collectorExporter, err := getOtelMetricsCollectorExporter(ctx, exporterConfig) | ||
if err != nil { | ||
return nil, fmt.Errorf("could not get collector exporter: %w", err) | ||
} | ||
|
||
interval := 60 * time.Second | ||
if exporterConfig.MetricsReaderInterval != 0 { | ||
interval = exporterConfig.MetricsReaderInterval | ||
} | ||
|
||
periodicReader := metricsdk.NewPeriodicReader(collectorExporter, | ||
metricsdk.WithInterval(interval), | ||
) | ||
|
||
provider := metricsdk.NewMeterProvider( | ||
metricsdk.WithResource(resource), | ||
metricsdk.WithReader(periodicReader), | ||
) | ||
|
||
return provider, nil | ||
} | ||
|
||
func getOtelMetricsCollectorExporter(ctx context.Context, exporterConfig *config.TelemetryExporterOption) (metricsdk.Exporter, error) { | ||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second) | ||
defer cancel() | ||
|
||
exporter, err := otlpmetricgrpc.New(ctx, | ||
otlpmetricgrpc.WithEndpoint(exporterConfig.Exporter.CollectorConfiguration.Endpoint), | ||
otlpmetricgrpc.WithCompressor("gzip"), | ||
otlpmetricgrpc.WithInsecure(), | ||
) | ||
if err != nil { | ||
return nil, fmt.Errorf("could not create trace exporter: %w", err) | ||
} | ||
|
||
return exporter, nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
package telemetry | ||
|
||
import ( | ||
"fmt" | ||
"os" | ||
|
||
"github.com/kubeshop/tracetest/server/config" | ||
"go.opentelemetry.io/otel/sdk/resource" | ||
semconv "go.opentelemetry.io/otel/semconv/v1.21.0" | ||
) | ||
|
||
func getResource(cfg *config.TelemetryExporterOption) (*resource.Resource, error) { | ||
hostname, err := os.Hostname() | ||
if err != nil { | ||
return nil, fmt.Errorf("could not get OS hostname: %w", err) | ||
} | ||
|
||
serviceName := "tracetest" | ||
if cfg != nil && cfg.ServiceName != "" { | ||
serviceName = cfg.ServiceName | ||
} | ||
|
||
resource, err := resource.Merge( | ||
resource.Default(), | ||
resource.NewWithAttributes( | ||
semconv.SchemaURL, | ||
semconv.ServiceNameKey.String(serviceName), | ||
semconv.HostName(hostname), | ||
), | ||
) | ||
|
||
if err != nil { | ||
return nil, fmt.Errorf("could not merge resources: %w", err) | ||
} | ||
|
||
return resource, nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
package tracing | ||
package telemetry | ||
|
||
import ( | ||
"context" | ||
|