diff --git a/CHANGELOG.md b/CHANGELOG.md index 73b4a04231b..e56e7f3bddc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,7 @@ Here is an overview of all new **experimental** features: ### Improvements - **General**: Add OPENTELEMETRY flag in e2e test YAML ([#5375](https://github.com/kedacore/keda/issues/5375)) +- **General**: Renamed Prometheus metrics to include units and `total` where approriate ([#4854](https://github.com/kedacore/keda/issues/4854)) ### Fixes @@ -74,7 +75,7 @@ You can find all deprecations in [this overview](https://github.com/kedacore/ked New deprecation(s): -- TODO ([#XXX](https://github.com/kedacore/keda/issues/XXX)) +- Various Prometheus metrics have been renamed to follow the preferred naming conventions. The old ones are still available, but will be removed in the future ([#4854](https://github.com/kedacore/keda/issues/4854)). ### Breaking Changes diff --git a/config/grafana/keda-dashboard.json b/config/grafana/keda-dashboard.json index 07abaec730c..e6bea125949 100644 --- a/config/grafana/keda-dashboard.json +++ b/config/grafana/keda-dashboard.json @@ -173,7 +173,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(job) (rate(keda_scaler_errors{}[5m]))", + "expr": "sum by(job) (rate(keda_scaler_errors_total{}[5m]))", "legendFormat": "{{ job }}", "range": true, "refId": "A" @@ -313,7 +313,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(scaler) (rate(keda_scaler_errors{exported_namespace=~\"$namespace\", scaledObject=~\"$scaledObject\", scaler=~\"$scaler\"}[5m]))", + "expr": "sum by(scaler) (rate(keda_scaler_errors_total{exported_namespace=~\"$namespace\", scaledObject=~\"$scaledObject\", scaler=~\"$scaler\"}[5m]))", "legendFormat": "{{ scaler }}", "range": true, "refId": "A" @@ -423,7 +423,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(scaledObject) (rate(keda_scaled_object_errors{exported_namespace=~\"$namespace\", scaledObject=~\"$scaledObject\"}[5m]))", + "expr": "sum by(scaledObject) (rate(keda_scaled_object_errors_total{exported_namespace=~\"$namespace\", scaledObject=~\"$scaledObject\"}[5m]))", "legendFormat": "{{ scaledObject }}", "range": true, "refId": "A" diff --git a/pkg/metricscollector/metricscollectors.go b/pkg/metricscollector/metricscollectors.go index 296bc6b86b6..849615d2f45 100644 --- a/pkg/metricscollector/metricscollectors.go +++ b/pkg/metricscollector/metricscollectors.go @@ -16,6 +16,8 @@ limitations under the License. package metricscollector +import "time" + const ( ClusterTriggerAuthenticationResource = "cluster_trigger_authentication" TriggerAuthenticationResource = "trigger_authentication" @@ -34,10 +36,10 @@ type MetricsCollector interface { RecordScalerMetric(namespace string, scaledResource string, scaler string, triggerIndex int, metric string, isScaledObject bool, value float64) // RecordScalerLatency create a measurement of the latency to external metric - RecordScalerLatency(namespace string, scaledResource string, scaler string, triggerIndex int, metric string, isScaledObject bool, value float64) + RecordScalerLatency(namespace string, scaledResource string, scaler string, triggerIndex int, metric string, isScaledObject bool, value time.Duration) // RecordScalableObjectLatency create a measurement of the latency executing scalable object loop - RecordScalableObjectLatency(namespace string, name string, isScaledObject bool, value float64) + RecordScalableObjectLatency(namespace string, name string, isScaledObject bool, value time.Duration) // RecordScalerActive create a measurement of the activity of the scaler RecordScalerActive(namespace string, scaledResource string, scaler string, triggerIndex int, metric string, isScaledObject bool, active bool) @@ -92,14 +94,14 @@ func RecordScalerMetric(namespace string, scaledObject string, scaler string, tr } // RecordScalerLatency create a measurement of the latency to external metric -func RecordScalerLatency(namespace string, scaledObject string, scaler string, triggerIndex int, metric string, isScaledObject bool, value float64) { +func RecordScalerLatency(namespace string, scaledObject string, scaler string, triggerIndex int, metric string, isScaledObject bool, value time.Duration) { for _, element := range collectors { element.RecordScalerLatency(namespace, scaledObject, scaler, triggerIndex, metric, isScaledObject, value) } } // RecordScalableObjectLatency create a measurement of the latency executing scalable object loop -func RecordScalableObjectLatency(namespace string, name string, isScaledObject bool, value float64) { +func RecordScalableObjectLatency(namespace string, name string, isScaledObject bool, value time.Duration) { for _, element := range collectors { element.RecordScalableObjectLatency(namespace, name, isScaledObject, value) } diff --git a/pkg/metricscollector/opentelemetry.go b/pkg/metricscollector/opentelemetry.go index ae3d00f1cb0..fe807260c04 100644 --- a/pkg/metricscollector/opentelemetry.go +++ b/pkg/metricscollector/opentelemetry.go @@ -5,6 +5,7 @@ import ( "fmt" "runtime" "strconv" + "time" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -22,13 +23,15 @@ const meterName = "keda-open-telemetry-metrics" const defaultNamespace = "default" var ( - meterProvider *metric.MeterProvider - meter api.Meter - otScalerErrorsCounter api.Int64Counter - otScaledObjectErrorsCounter api.Int64Counter - otScaledJobErrorsCounter api.Int64Counter - otTriggerTotalsCounter api.Int64UpDownCounter - otCrdTotalsCounter api.Int64UpDownCounter + meterProvider *metric.MeterProvider + meter api.Meter + otScalerErrorsCounter api.Int64Counter + otScaledObjectErrorsCounter api.Int64Counter + otScaledJobErrorsCounter api.Int64Counter + otTriggerTotalsCounterDeprecated api.Int64UpDownCounter + otCrdTotalsCounterDeprecated api.Int64UpDownCounter + otTriggerRegisteredTotalsCounter api.Int64UpDownCounter + otCrdRegisteredTotalsCounter api.Int64UpDownCounter otelScalerMetricVal OtelMetricFloat64Val otelScalerMetricsLatencyVal OtelMetricFloat64Val @@ -95,19 +98,29 @@ func initMeters() { otLog.Error(err, msg) } - otTriggerTotalsCounter, err = meter.Int64UpDownCounter("keda.trigger.totals", api.WithDescription("Total triggers")) + otTriggerTotalsCounterDeprecated, err = meter.Int64UpDownCounter("keda.trigger.totals", api.WithDescription("DEPRECATED - will be removed in 2.16 - use 'keda.trigger.registered.count' instead")) if err != nil { otLog.Error(err, msg) } - otCrdTotalsCounter, err = meter.Int64UpDownCounter("keda.resource.totals", api.WithDescription("Total resources")) + otTriggerRegisteredTotalsCounter, err = meter.Int64UpDownCounter("keda.trigger.registered.count", api.WithDescription("Total number of triggers per trigger type registered")) + if err != nil { + otLog.Error(err, msg) + } + + otCrdTotalsCounterDeprecated, err = meter.Int64UpDownCounter("keda.resource.totals", api.WithDescription("DEPRECATED - will be removed in 2.16 - use 'keda.resource.registered.count' instead")) + if err != nil { + otLog.Error(err, msg) + } + + otCrdRegisteredTotalsCounter, err = meter.Int64UpDownCounter("keda.resource.registered.count", api.WithDescription("Total number of KEDA custom resources per namespace for each custom resource type (CRD) registered")) if err != nil { otLog.Error(err, msg) } _, err = meter.Float64ObservableGauge( "keda.scaler.metrics.value", - api.WithDescription("Metric Value used for HPA"), + api.WithDescription("The current value for each scaler's metric that would be used by the HPA in computing the target average"), api.WithFloat64Callback(ScalerMetricValueCallback), ) if err != nil { @@ -116,7 +129,8 @@ func initMeters() { _, err = meter.Float64ObservableGauge( "keda.scaler.metrics.latency", - api.WithDescription("Scaler Metrics Latency"), + api.WithDescription("The latency of retrieving current metric from each scaler"), + api.WithUnit("s"), api.WithFloat64Callback(ScalerMetricsLatencyCallback), ) if err != nil { @@ -126,6 +140,7 @@ func initMeters() { _, err = meter.Float64ObservableGauge( "keda.internal.scale.loop.latency", api.WithDescription("Internal latency of ScaledObject/ScaledJob loop execution"), + api.WithUnit("s"), api.WithFloat64Callback(ScalableObjectLatencyCallback), ) if err != nil { @@ -134,7 +149,7 @@ func initMeters() { _, err = meter.Float64ObservableGauge( "keda.scaler.active", - api.WithDescription("Activity of a Scaler Metric"), + api.WithDescription("Indicates whether a scaler is active (1), or not (0)"), api.WithFloat64Callback(ScalerActiveCallback), ) if err != nil { @@ -208,8 +223,8 @@ func ScalerMetricsLatencyCallback(_ context.Context, obsrv api.Float64Observer) } // RecordScalerLatency create a measurement of the latency to external metric -func (o *OtelMetrics) RecordScalerLatency(namespace string, scaledResource string, scaler string, triggerIndex int, metric string, isScaledObject bool, value float64) { - otelScalerMetricsLatencyVal.val = value +func (o *OtelMetrics) RecordScalerLatency(namespace string, scaledResource string, scaler string, triggerIndex int, metric string, isScaledObject bool, value time.Duration) { + otelScalerMetricsLatencyVal.val = value.Seconds() otelScalerMetricsLatencyVal.measurementOption = getScalerMeasurementOption(namespace, scaledResource, scaler, triggerIndex, metric, isScaledObject) } @@ -222,7 +237,7 @@ func ScalableObjectLatencyCallback(_ context.Context, obsrv api.Float64Observer) } // RecordScalableObjectLatency create a measurement of the latency executing scalable object loop -func (o *OtelMetrics) RecordScalableObjectLatency(namespace string, name string, isScaledObject bool, value float64) { +func (o *OtelMetrics) RecordScalableObjectLatency(namespace string, name string, isScaledObject bool, value time.Duration) { resourceType := "scaledjob" if isScaledObject { resourceType = "scaledobject" @@ -233,7 +248,7 @@ func (o *OtelMetrics) RecordScalableObjectLatency(namespace string, name string, attribute.Key("type").String(resourceType), attribute.Key("name").String(name)) - otelInternalLoopLatencyVal.val = value + otelInternalLoopLatencyVal.val = value.Seconds() otelInternalLoopLatencyVal.measurementOption = opt } @@ -315,13 +330,15 @@ func (o *OtelMetrics) RecordScaledJobError(namespace string, scaledJob string, e func (o *OtelMetrics) IncrementTriggerTotal(triggerType string) { if triggerType != "" { - otTriggerTotalsCounter.Add(context.Background(), 1, api.WithAttributes(attribute.Key("type").String(triggerType))) + otTriggerTotalsCounterDeprecated.Add(context.Background(), 1, api.WithAttributes(attribute.Key("type").String(triggerType))) + otTriggerRegisteredTotalsCounter.Add(context.Background(), 1, api.WithAttributes(attribute.Key("type").String(triggerType))) } } func (o *OtelMetrics) DecrementTriggerTotal(triggerType string) { if triggerType != "" { - otTriggerTotalsCounter.Add(context.Background(), -1, api.WithAttributes(attribute.Key("type").String(triggerType))) + otTriggerTotalsCounterDeprecated.Add(context.Background(), -1, api.WithAttributes(attribute.Key("type").String(triggerType))) + otTriggerRegisteredTotalsCounter.Add(context.Background(), -1, api.WithAttributes(attribute.Key("type").String(triggerType))) } } @@ -334,7 +351,8 @@ func (o *OtelMetrics) IncrementCRDTotal(crdType, namespace string) { attribute.Key("type").String(crdType), ) - otCrdTotalsCounter.Add(context.Background(), 1, opt) + otCrdTotalsCounterDeprecated.Add(context.Background(), 1, opt) + otCrdRegisteredTotalsCounter.Add(context.Background(), 1, opt) } func (o *OtelMetrics) DecrementCRDTotal(crdType, namespace string) { @@ -346,7 +364,8 @@ func (o *OtelMetrics) DecrementCRDTotal(crdType, namespace string) { attribute.Key("namespace").String(namespace), attribute.Key("type").String(crdType), ) - otCrdTotalsCounter.Add(context.Background(), -1, opt) + otCrdTotalsCounterDeprecated.Add(context.Background(), -1, opt) + otCrdRegisteredTotalsCounter.Add(context.Background(), -1, opt) } func getScalerMeasurementOption(namespace string, scaledResource string, scaler string, triggerIndex int, metric string, isScaledObject bool) api.MeasurementOption { diff --git a/pkg/metricscollector/opentelemetry_test.go b/pkg/metricscollector/opentelemetry_test.go index 3b763d2017e..58096e636a8 100644 --- a/pkg/metricscollector/opentelemetry_test.go +++ b/pkg/metricscollector/opentelemetry_test.go @@ -3,6 +3,7 @@ package metricscollector import ( "context" "testing" + "time" "github.com/stretchr/testify/assert" "go.opentelemetry.io/otel/sdk/metric" @@ -59,11 +60,11 @@ func TestIncrementTriggerTotal(t *testing.T) { assert.Nil(t, err) scopeMetrics := got.ScopeMetrics[0] assert.NotEqual(t, len(scopeMetrics.Metrics), 0) - buildInfo := retrieveMetric(scopeMetrics.Metrics, "keda.trigger.totals") + triggercount := retrieveMetric(scopeMetrics.Metrics, "keda.trigger.registered.count") - assert.NotNil(t, buildInfo) + assert.NotNil(t, triggercount) - data := buildInfo.Data.(metricdata.Sum[int64]).DataPoints[0] + data := triggercount.Data.(metricdata.Sum[int64]).DataPoints[0] assert.Equal(t, data.Value, int64(1)) testOtel.DecrementTriggerTotal("testtrigger") @@ -72,10 +73,27 @@ func TestIncrementTriggerTotal(t *testing.T) { assert.Nil(t, err) scopeMetrics = got.ScopeMetrics[0] assert.NotEqual(t, len(scopeMetrics.Metrics), 0) - buildInfo = retrieveMetric(scopeMetrics.Metrics, "keda.trigger.totals") + triggercount = retrieveMetric(scopeMetrics.Metrics, "keda.trigger.registered.count") - assert.NotNil(t, buildInfo) + assert.NotNil(t, triggercount) - data = buildInfo.Data.(metricdata.Sum[int64]).DataPoints[0] + data = triggercount.Data.(metricdata.Sum[int64]).DataPoints[0] assert.Equal(t, data.Value, int64(0)) } + +func TestLoopLatency(t *testing.T) { + testOtel.RecordScalableObjectLatency("namespace", "name", true, 500*time.Millisecond) + got := metricdata.ResourceMetrics{} + err := testReader.Collect(context.Background(), &got) + + assert.Nil(t, err) + scopeMetrics := got.ScopeMetrics[0] + assert.NotEqual(t, len(scopeMetrics.Metrics), 0) + latency := retrieveMetric(scopeMetrics.Metrics, "keda.internal.scale.loop.latency") + + assert.NotNil(t, latency) + assert.Equal(t, latency.Unit, "s") + + data := latency.Data.(metricdata.Gauge[float64]).DataPoints[0] + assert.Equal(t, data.Value, float64(0.5)) +} diff --git a/pkg/metricscollector/prommetrics.go b/pkg/metricscollector/prommetrics.go index 6bbac8239be..1b44852869d 100644 --- a/pkg/metricscollector/prommetrics.go +++ b/pkg/metricscollector/prommetrics.go @@ -19,6 +19,7 @@ package metricscollector import ( "runtime" "strconv" + "time" "github.com/prometheus/client_golang/prometheus" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -35,16 +36,16 @@ var ( prometheus.GaugeOpts{ Namespace: DefaultPromMetricsNamespace, Name: "build_info", - Help: "A metric with a constant '1' value labeled by version, git_commit and goversion from which KEDA was built.", + Help: "Info metric, with static information about KEDA build like: version, git commit and Golang runtime info.", }, []string{"version", "git_commit", "goversion", "goos", "goarch"}, ) - scalerErrorsTotal = prometheus.NewCounterVec( + scalerErrorsTotalDeprecated = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: DefaultPromMetricsNamespace, Subsystem: "scaler", Name: "errors_total", - Help: "Total number of errors for all scalers", + Help: "DEPRECATED - will be removed in 2.16 - use a `sum(scaler_errors_total{scaler!=\"\"})` over all scalers", }, []string{}, ) @@ -53,16 +54,25 @@ var ( Namespace: DefaultPromMetricsNamespace, Subsystem: "scaler", Name: "metrics_value", - Help: "Metric Value used for HPA", + Help: "The current value for each scaler's metric that would be used by the HPA in computing the target average.", }, metricLabels, ) - scalerMetricsLatency = prometheus.NewGaugeVec( + scalerMetricsLatencyDeprecated = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: DefaultPromMetricsNamespace, Subsystem: "scaler", Name: "metrics_latency", - Help: "Scaler Metrics Latency", + Help: "DEPRECATED - will be removed in 2.16 use 'scaler_metrics_latency_seconds' instead.", + }, + metricLabels, + ) + scalerMetricsLatency = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: DefaultPromMetricsNamespace, + Subsystem: "scaler", + Name: "metrics_latency_seconds", + Help: "The latency of retrieving current metric from each scaler, in seconds.", }, metricLabels, ) @@ -71,7 +81,7 @@ var ( Namespace: DefaultPromMetricsNamespace, Subsystem: "scaler", Name: "active", - Help: "Activity of a Scaler Metric", + Help: "Indicates whether a scaler is active (1), or not (0).", }, metricLabels, ) @@ -80,62 +90,108 @@ var ( Namespace: DefaultPromMetricsNamespace, Subsystem: "scaled_object", Name: "paused", - Help: "Indicates whether a ScaledObject is paused", + Help: "Indicates whether a ScaledObject is paused (1), or not (0).", }, []string{"namespace", "scaledObject"}, ) - scalerErrors = prometheus.NewCounterVec( + scalerErrorsDeprecated = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: DefaultPromMetricsNamespace, Subsystem: "scaler", Name: "errors", - Help: "Number of scaler errors", + Help: "DEPRECATED - will be removed in 2.16 - use 'scaler_errors_total' instead.", }, metricLabels, ) - scaledObjectErrors = prometheus.NewCounterVec( + scalerErrors = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: DefaultPromMetricsNamespace, + Subsystem: "scaler", + Name: "errors_total", + Help: "The total number of errors encountered for each scaler.", + }, + metricLabels, + ) + scaledObjectErrorsDeprecated = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: DefaultPromMetricsNamespace, Subsystem: "scaled_object", Name: "errors", - Help: "Number of scaled object errors", + Help: "DEPRECATED - will be removed in 2.16 - use 'scaled_object_errors_total' instead.", }, []string{"namespace", "scaledObject"}, ) + scaledObjectErrors = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: DefaultPromMetricsNamespace, + Subsystem: "scaled_object", + Name: "errors_total", + Help: "The number of errors that have occurred for each ScaledObject.", + }, + []string{"namespace", "scaledObject"}, + ) + scaledJobErrors = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: DefaultPromMetricsNamespace, Subsystem: "scaled_job", - Name: "errors", + Name: "errors_total", Help: "Number of scaled job errors", }, []string{"namespace", "scaledJob"}, ) - triggerTotalsGaugeVec = prometheus.NewGaugeVec( + triggerTotalsGaugeVecDeprecated = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: DefaultPromMetricsNamespace, Subsystem: "trigger", Name: "totals", + Help: "DEPRECATED - will be removed in 2.16 - use 'trigger_registered_total' instead.", }, []string{"type"}, ) - - crdTotalsGaugeVec = prometheus.NewGaugeVec( + triggerRegistered = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: DefaultPromMetricsNamespace, + Subsystem: "trigger", + Name: "registered_total", + Help: "Total number of triggers per trigger type registered.", + }, + []string{"type"}, + ) + crdTotalsGaugeVecDeprecated = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: DefaultPromMetricsNamespace, Subsystem: "resource", Name: "totals", + Help: "DEPRECATED - will be removed in 2.16 - use 'resource_handled_total' instead.", }, []string{"type", "namespace"}, ) - - internalLoopLatency = prometheus.NewGaugeVec( + crdRegistered = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: DefaultPromMetricsNamespace, + Subsystem: "resource", + Name: "registered_total", + Help: "Total number of KEDA custom resources per namespace for each custom resource type (CRD) registered.", + }, + []string{"type", "namespace"}, + ) + internalLoopLatencyDeprecated = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: DefaultPromMetricsNamespace, Subsystem: "internal_scale_loop", Name: "latency", - Help: "Internal latency of ScaledObject/ScaledJob loop execution", + Help: "DEPRECATED - will be removed in 2.16 - use 'internal_scale_loop_latency_seconds' instead.", + }, + []string{"namespace", "type", "resource"}, + ) + internalLoopLatency = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: DefaultPromMetricsNamespace, + Subsystem: "internal_scale_loop", + Name: "latency_seconds", + Help: "Total deviation (in seconds) between the expected execution time and the actual execution time for the scaling loop.", }, []string{"namespace", "type", "resource"}, ) @@ -166,18 +222,24 @@ type PromMetrics struct { } func NewPromMetrics() *PromMetrics { - metrics.Registry.MustRegister(scalerErrorsTotal) + metrics.Registry.MustRegister(scalerErrorsTotalDeprecated) metrics.Registry.MustRegister(scalerMetricsValue) + metrics.Registry.MustRegister(scalerMetricsLatencyDeprecated) metrics.Registry.MustRegister(scalerMetricsLatency) + metrics.Registry.MustRegister(internalLoopLatencyDeprecated) metrics.Registry.MustRegister(internalLoopLatency) metrics.Registry.MustRegister(scalerActive) + metrics.Registry.MustRegister(scalerErrorsDeprecated) metrics.Registry.MustRegister(scalerErrors) + metrics.Registry.MustRegister(scaledObjectErrorsDeprecated) metrics.Registry.MustRegister(scaledObjectErrors) metrics.Registry.MustRegister(scaledObjectPaused) + metrics.Registry.MustRegister(triggerTotalsGaugeVecDeprecated) + metrics.Registry.MustRegister(triggerRegistered) + metrics.Registry.MustRegister(crdTotalsGaugeVecDeprecated) + metrics.Registry.MustRegister(crdRegistered) metrics.Registry.MustRegister(scaledJobErrors) - metrics.Registry.MustRegister(triggerTotalsGaugeVec) - metrics.Registry.MustRegister(crdTotalsGaugeVec) metrics.Registry.MustRegister(buildInfo) metrics.Registry.MustRegister(cloudeventEmitted) @@ -198,13 +260,15 @@ func (p *PromMetrics) RecordScalerMetric(namespace string, scaledResource string } // RecordScalerLatency create a measurement of the latency to external metric -func (p *PromMetrics) RecordScalerLatency(namespace string, scaledResource string, scaler string, triggerIndex int, metric string, isScaledObject bool, value float64) { - scalerMetricsLatency.With(getLabels(namespace, scaledResource, scaler, triggerIndex, metric, isScaledObject)).Set(value) +func (p *PromMetrics) RecordScalerLatency(namespace string, scaledResource string, scaler string, triggerIndex int, metric string, isScaledObject bool, value time.Duration) { + scalerMetricsLatency.With(getLabels(namespace, scaledResource, scaler, triggerIndex, metric, isScaledObject)).Set(value.Seconds()) + scalerMetricsLatencyDeprecated.With(getLabels(namespace, scaledResource, scaler, triggerIndex, metric, isScaledObject)).Set(float64(value.Milliseconds())) } // RecordScalableObjectLatency create a measurement of the latency executing scalable object loop -func (p *PromMetrics) RecordScalableObjectLatency(namespace string, name string, isScaledObject bool, value float64) { - internalLoopLatency.WithLabelValues(namespace, getResourceType(isScaledObject), name).Set(value) +func (p *PromMetrics) RecordScalableObjectLatency(namespace string, name string, isScaledObject bool, value time.Duration) { + internalLoopLatency.WithLabelValues(namespace, getResourceType(isScaledObject), name).Set(value.Seconds()) + internalLoopLatencyDeprecated.WithLabelValues(namespace, getResourceType(isScaledObject), name).Set(float64(value.Milliseconds())) } // RecordScalerActive create a measurement of the activity of the scaler @@ -233,14 +297,19 @@ func (p *PromMetrics) RecordScaledObjectPaused(namespace string, scaledObject st func (p *PromMetrics) RecordScalerError(namespace string, scaledResource string, scaler string, triggerIndex int, metric string, isScaledObject bool, err error) { if err != nil { scalerErrors.With(getLabels(namespace, scaledResource, scaler, triggerIndex, metric, isScaledObject)).Inc() + scalerErrorsDeprecated.With(getLabels(namespace, scaledResource, scaler, triggerIndex, metric, isScaledObject)).Inc() p.RecordScaledObjectError(namespace, scaledResource, err) - scalerErrorsTotal.With(prometheus.Labels{}).Inc() + scalerErrorsTotalDeprecated.With(prometheus.Labels{}).Inc() return } // initialize metric with 0 if not already set _, errscaler := scalerErrors.GetMetricWith(getLabels(namespace, scaledResource, scaler, triggerIndex, metric, isScaledObject)) if errscaler != nil { - log.Error(errscaler, "Unable to write to metrics to Prometheus Server: %v") + log.Error(errscaler, "Unable to record metrics: %v") + } + _, errscalerdep := scalerErrorsDeprecated.GetMetricWith(getLabels(namespace, scaledResource, scaler, triggerIndex, metric, isScaledObject)) + if errscalerdep != nil { + log.Error(errscaler, "Unable to record (deprecated) metrics: %v") } } @@ -249,12 +318,18 @@ func (p *PromMetrics) RecordScaledObjectError(namespace string, scaledObject str labels := prometheus.Labels{"namespace": namespace, "scaledObject": scaledObject} if err != nil { scaledObjectErrors.With(labels).Inc() + scaledObjectErrorsDeprecated.With(labels).Inc() return } // initialize metric with 0 if not already set _, errscaledobject := scaledObjectErrors.GetMetricWith(labels) if errscaledobject != nil { - log.Error(errscaledobject, "Unable to write to metrics to Prometheus Server: %v") + log.Error(errscaledobject, "Unable to record metrics: %v") + return + } + _, errscaledobjectdep := scaledObjectErrorsDeprecated.GetMetricWith(labels) + if errscaledobjectdep != nil { + log.Error(errscaledobject, "Unable to record metrics: %v") return } } @@ -287,13 +362,15 @@ func getResourceType(isScaledObject bool) string { func (p *PromMetrics) IncrementTriggerTotal(triggerType string) { if triggerType != "" { - triggerTotalsGaugeVec.WithLabelValues(triggerType).Inc() + triggerRegistered.WithLabelValues(triggerType).Inc() + triggerTotalsGaugeVecDeprecated.WithLabelValues(triggerType).Inc() } } func (p *PromMetrics) DecrementTriggerTotal(triggerType string) { if triggerType != "" { - triggerTotalsGaugeVec.WithLabelValues(triggerType).Dec() + triggerRegistered.WithLabelValues(triggerType).Dec() + triggerTotalsGaugeVecDeprecated.WithLabelValues(triggerType).Dec() } } @@ -302,7 +379,8 @@ func (p *PromMetrics) IncrementCRDTotal(crdType, namespace string) { namespace = defaultNamespace } - crdTotalsGaugeVec.WithLabelValues(crdType, namespace).Inc() + crdRegistered.WithLabelValues(crdType, namespace).Inc() + crdTotalsGaugeVecDeprecated.WithLabelValues(crdType, namespace).Inc() } func (p *PromMetrics) DecrementCRDTotal(crdType, namespace string) { @@ -310,7 +388,8 @@ func (p *PromMetrics) DecrementCRDTotal(crdType, namespace string) { namespace = defaultNamespace } - crdTotalsGaugeVec.WithLabelValues(crdType, namespace).Dec() + crdRegistered.WithLabelValues(crdType, namespace).Dec() + crdTotalsGaugeVecDeprecated.WithLabelValues(crdType, namespace).Dec() } // RecordCloudEventEmitted counts the number of cloudevent that emitted to user's sink diff --git a/pkg/metricscollector/webhook/webhook_prommetrics.go b/pkg/metricscollector/webhook/webhook_prommetrics.go index c7ba4df49c0..329d3e4b4e3 100644 --- a/pkg/metricscollector/webhook/webhook_prommetrics.go +++ b/pkg/metricscollector/webhook/webhook_prommetrics.go @@ -31,6 +31,15 @@ var ( Namespace: DefaultPromMetricsNamespace, Subsystem: "webhook", Name: "scaled_object_validation_total", + Help: "DEPRECATED - will be removed in 2.16 - Use `scaled_object_validations_total` instead.", + }, + []string{"namespace", "action"}, + ) + scaledObjectValidationsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: DefaultPromMetricsNamespace, + Subsystem: "webhook", + Name: "scaled_object_validations_total", Help: "Total number of scaled object validations", }, []string{"namespace", "action"}, @@ -40,6 +49,15 @@ var ( Namespace: DefaultPromMetricsNamespace, Subsystem: "webhook", Name: "scaled_object_validation_errors", + Help: "DEPRECATED - will be removed in 2.16 - Use `scaled_object_validation_errors_total` instead.", + }, + []string{"namespace", "action", "reason"}, + ) + scaledObjectValidationErrorsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: DefaultPromMetricsNamespace, + Subsystem: "webhook", + Name: "scaled_object_validation_errors_total", Help: "Total number of scaled object validating errors", }, []string{"namespace", "action", "reason"}, @@ -48,17 +66,21 @@ var ( func init() { metrics.Registry.MustRegister(scaledObjectValidatingTotal) + metrics.Registry.MustRegister(scaledObjectValidationsTotal) metrics.Registry.MustRegister(scaledObjectValidatingErrors) + metrics.Registry.MustRegister(scaledObjectValidationErrorsTotal) } // RecordScaledObjectValidatingTotal counts the number of ScaledObject validations func RecordScaledObjectValidatingTotal(namespace, action string) { labels := prometheus.Labels{"namespace": namespace, "action": action} scaledObjectValidatingTotal.With(labels).Inc() + scaledObjectValidationsTotal.With(labels).Inc() } // RecordScaledObjectValidatingErrors counts the number of ScaledObject validating errors func RecordScaledObjectValidatingErrors(namespace, action, reason string) { labels := prometheus.Labels{"namespace": namespace, "action": action, "reason": reason} scaledObjectValidatingErrors.With(labels).Inc() + scaledObjectValidationErrorsTotal.With(labels).Inc() } diff --git a/pkg/scaling/cache/scalers_cache.go b/pkg/scaling/cache/scalers_cache.go index 49042f86f0d..acd253804c4 100644 --- a/pkg/scaling/cache/scalers_cache.go +++ b/pkg/scaling/cache/scalers_cache.go @@ -122,14 +122,14 @@ func (c *ScalersCache) GetMetricSpecForScalingForScaler(ctx context.Context, ind // GetMetricsAndActivityForScaler returns metric value, activity and latency for a scaler identified by the metric name // and by the input index (from the list of scalers in this ScaledObject) -func (c *ScalersCache) GetMetricsAndActivityForScaler(ctx context.Context, index int, metricName string) ([]external_metrics.ExternalMetricValue, bool, int64, error) { +func (c *ScalersCache) GetMetricsAndActivityForScaler(ctx context.Context, index int, metricName string) ([]external_metrics.ExternalMetricValue, bool, time.Duration, error) { if index < 0 || index >= len(c.Scalers) { return nil, false, -1, fmt.Errorf("scaler with id %d not found. Len = %d", index, len(c.Scalers)) } startTime := time.Now() metric, activity, err := c.Scalers[index].Scaler.GetMetricsAndActivity(ctx, metricName) if err == nil { - return metric, activity, time.Since(startTime).Milliseconds(), nil + return metric, activity, time.Since(startTime), nil } ns, err := c.refreshScaler(ctx, index) @@ -138,7 +138,7 @@ func (c *ScalersCache) GetMetricsAndActivityForScaler(ctx context.Context, index } startTime = time.Now() metric, activity, err = ns.GetMetricsAndActivity(ctx, metricName) - return metric, activity, time.Since(startTime).Milliseconds(), err + return metric, activity, time.Since(startTime), err } func (c *ScalersCache) refreshScaler(ctx context.Context, id int) (scalers.Scaler, error) { diff --git a/pkg/scaling/scale_handler.go b/pkg/scaling/scale_handler.go index fcc060dc97c..e4dffae9d07 100644 --- a/pkg/scaling/scale_handler.go +++ b/pkg/scaling/scale_handler.go @@ -174,7 +174,7 @@ func (h *scaleHandler) startScaleLoop(ctx context.Context, withTriggers *kedav1a // we calculate the next execution time based on the pollingInterval and record the difference // between the expected execution time and the real execution time delay := time.Since(next) - metricscollector.RecordScalableObjectLatency(withTriggers.Namespace, withTriggers.Name, isScaledObject, float64(delay.Milliseconds())) + metricscollector.RecordScalableObjectLatency(withTriggers.Namespace, withTriggers.Name, isScaledObject, delay) tmr := time.NewTimer(pollingInterval) next = time.Now().Add(pollingInterval) @@ -522,10 +522,10 @@ func (h *scaleHandler) GetScaledObjectMetrics(ctx context.Context, scaledObjectN } if !metricsFoundInCache { - var latency int64 + var latency time.Duration metrics, _, latency, err = cache.GetMetricsAndActivityForScaler(ctx, triggerIndex, metricName) if latency != -1 { - metricscollector.RecordScalerLatency(scaledObjectNamespace, scaledObject.Name, triggerName, triggerIndex, metricName, true, float64(latency)) + metricscollector.RecordScalerLatency(scaledObjectNamespace, scaledObject.Name, triggerName, triggerIndex, metricName, true, latency) } logger.V(1).Info("Getting metrics from trigger", "trigger", triggerName, "metricName", metricName, "metrics", metrics, "scalerError", err) } @@ -739,10 +739,10 @@ func (*scaleHandler) getScalerState(ctx context.Context, scaler scalers.Scaler, metricName := spec.External.Metric.Name - var latency int64 + var latency time.Duration metrics, isMetricActive, latency, err := cache.GetMetricsAndActivityForScaler(ctx, triggerIndex, metricName) if latency != -1 { - metricscollector.RecordScalerLatency(scaledObject.Namespace, scaledObject.Name, triggerName, triggerIndex, metricName, true, float64(latency)) + metricscollector.RecordScalerLatency(scaledObject.Namespace, scaledObject.Name, triggerName, triggerIndex, metricName, true, latency) } result.Metrics = append(result.Metrics, metrics...) logger.V(1).Info("Getting metrics and activity from scaler", "scaler", triggerName, "metricName", metricName, "metrics", metrics, "activity", isMetricActive, "scalerError", err) @@ -830,7 +830,7 @@ func (h *scaleHandler) getScaledJobMetrics(ctx context.Context, scaledJob *kedav metrics, isTriggerActive, latency, err := cache.GetMetricsAndActivityForScaler(ctx, scalerIndex, metricName) metricscollector.RecordScaledJobError(scaledJob.Namespace, scaledJob.Name, err) if latency != -1 { - metricscollector.RecordScalerLatency(scaledJob.Namespace, scaledJob.Name, scalerName, scalerIndex, metricName, false, float64(latency)) + metricscollector.RecordScalerLatency(scaledJob.Namespace, scaledJob.Name, scalerName, scalerIndex, metricName, false, latency) } if err != nil { scalerLogger.V(1).Info("Error getting scaler metrics and activity, but continue", "error", err) diff --git a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go index 5ce2c93f7ba..b9193db52ba 100644 --- a/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go +++ b/tests/sequential/opentelemetry_metrics/opentelemetry_metrics_test.go @@ -910,8 +910,8 @@ func getLatestCommit(t *testing.T) string { func checkTriggerTotalValues(t *testing.T, families map[string]*prommodel.MetricFamily, expected map[string]int) { t.Log("--- testing trigger total metrics ---") - family, ok := families["keda_trigger_totals"] - assert.True(t, ok, "keda_trigger_totals not available") + family, ok := families["keda_triggers_count"] + assert.True(t, ok, "keda_triggers_count not available") if !ok { return } @@ -939,8 +939,8 @@ func checkTriggerTotalValues(t *testing.T, families map[string]*prommodel.Metric func checkCRTotalValues(t *testing.T, families map[string]*prommodel.MetricFamily, expected map[string]map[string]int) { t.Log("--- testing resource total metrics ---") - family, ok := families["keda_resource_totals"] - assert.True(t, ok, "keda_resource_totals not available") + family, ok := families["keda_resources_count"] + assert.True(t, ok, "keda_resources_count not available") if !ok { return } diff --git a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go index e516f6b8391..93ac598369c 100644 --- a/tests/sequential/prometheus_metrics/prometheus_metrics_test.go +++ b/tests/sequential/prometheus_metrics/prometheus_metrics_test.go @@ -443,7 +443,6 @@ func TestPrometheusMetrics(t *testing.T) { testScaledObjectErrors(t, data) testScaledJobErrors(t, data) testScalerErrors(t, data) - testScalerErrorsTotal(t, data) testOperatorMetrics(t, kc, data) testMetricServerMetrics(t) testWebhookMetrics(t, data) @@ -519,6 +518,8 @@ func testScalerMetricValue(t *testing.T) { } } assert.Equal(t, true, found) + } else { + t.Errorf("metric keda_scaler_metrics_value not available") } } @@ -533,8 +534,8 @@ func testScaledObjectErrors(t *testing.T, data templateData) { time.Sleep(20 * time.Second) family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL)) - val, ok := family["keda_scaled_object_errors"] - assert.True(t, ok, "keda_scaled_object_errors not available") + val, ok := family["keda_scaled_object_errors_total"] + assert.True(t, ok, "keda_scaled_object_errors_total not available") if ok { errCounterVal1 := getErrorMetricsValue(val) @@ -542,8 +543,8 @@ func testScaledObjectErrors(t *testing.T, data templateData) { time.Sleep(2 * time.Second) family = fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL)) - val, ok := family["keda_scaled_object_errors"] - assert.True(t, ok, "keda_scaled_object_errors not available") + val, ok := family["keda_scaled_object_errors_total"] + assert.True(t, ok, "keda_scaled_object_errors_total not available") if ok { errCounterVal2 := getErrorMetricsValue(val) assert.NotEqual(t, errCounterVal2, float64(0)) @@ -579,10 +580,10 @@ func testScaledJobErrors(t *testing.T, data templateData) { assert.NotEqual(t, errCounterVal2, float64(0)) assert.GreaterOrEqual(t, errCounterVal2, errCounterVal1) } else { - t.Errorf("metric not available") + t.Errorf("metric keda_scaled_object_errors_total not available") } } else { - t.Errorf("metric not available") + t.Errorf("metric keda_scaled_object_errors_total not available") } KubectlDeleteWithTemplate(t, data, "wrongScaledJobTemplate", wrongScaledJobTemplate) @@ -602,46 +603,14 @@ func testScalerErrors(t *testing.T, data templateData) { KubectlApplyWithTemplate(t, data, "wrongScaledJobTemplate", wrongScaledJobTemplate) family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL)) - val, ok := family["keda_scaler_errors"] - assert.True(t, ok, "keda_scaler_errors not available") - if ok { - errCounterVal1 := getErrorMetricsValue(val) - - // wait for 20 seconds to correctly fetch metrics. - time.Sleep(20 * time.Second) - - family = fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL)) - val, ok := family["keda_scaler_errors"] - assert.True(t, ok, "keda_scaler_errors not available") - if ok { - errCounterVal2 := getErrorMetricsValue(val) - assert.NotEqual(t, errCounterVal2, float64(0)) - assert.GreaterOrEqual(t, errCounterVal2, errCounterVal1) - } - } - KubectlDeleteWithTemplate(t, data, "wrongScaledJobTemplate", wrongScaledJobTemplate) - time.Sleep(2 * time.Second) - KubectlApplyWithTemplate(t, data, "scaledJobTemplate", scaledJobTemplate) - - KubectlDeleteWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate) - time.Sleep(2 * time.Second) - KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) -} - -func testScalerErrorsTotal(t *testing.T, data templateData) { - t.Log("--- testing scaler errors total ---") - - KubectlDeleteWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) - KubectlApplyWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate) - family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL)) val, ok := family["keda_scaler_errors_total"] assert.True(t, ok, "keda_scaler_errors_total not available") if ok { errCounterVal1 := getErrorMetricsValue(val) - // wait for 2 seconds as pollinginterval is 2 - time.Sleep(2 * time.Second) + // wait for 20 seconds to correctly fetch metrics. + time.Sleep(20 * time.Second) family = fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL)) val, ok := family["keda_scaler_errors_total"] @@ -652,6 +621,9 @@ func testScalerErrorsTotal(t *testing.T, data templateData) { assert.GreaterOrEqual(t, errCounterVal2, errCounterVal1) } } + KubectlDeleteWithTemplate(t, data, "wrongScaledJobTemplate", wrongScaledJobTemplate) + time.Sleep(2 * time.Second) + KubectlApplyWithTemplate(t, data, "scaledJobTemplate", scaledJobTemplate) KubectlDeleteWithTemplate(t, data, "wrongScaledObjectTemplate", wrongScaledObjectTemplate) time.Sleep(2 * time.Second) @@ -660,12 +632,7 @@ func testScalerErrorsTotal(t *testing.T, data templateData) { func getErrorMetricsValue(val *prommodel.MetricFamily) float64 { switch val.GetName() { - case "keda_scaler_errors_total": - metrics := val.GetMetric() - for _, metric := range metrics { - return metric.GetCounter().GetValue() - } - case "keda_scaled_object_errors": + case "keda_scaled_object_errors_total": metrics := val.GetMetric() for _, metric := range metrics { labels := metric.GetLabel() @@ -675,7 +642,7 @@ func getErrorMetricsValue(val *prommodel.MetricFamily) float64 { } } } - case "keda_scaled_job_errors": + case "keda_scaled_job_errors_total": metrics := val.GetMetric() for _, metric := range metrics { labels := metric.GetLabel() @@ -685,7 +652,7 @@ func getErrorMetricsValue(val *prommodel.MetricFamily) float64 { } } } - case "keda_scaler_errors": + case "keda_scaler_errors_total": metrics := val.GetMetric() for _, metric := range metrics { labels := metric.GetLabel() @@ -729,8 +696,8 @@ func testScalerMetricLatency(t *testing.T) { family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL)) - val, ok := family["keda_scaler_metrics_latency"] - assert.True(t, ok, "keda_scaler_metrics_latency not available") + val, ok := family["keda_scaler_metrics_latency_seconds"] + assert.True(t, ok, "keda_scaler_metrics_latency_seconds not available") if ok { var found bool metrics := val.GetMetric() @@ -745,6 +712,8 @@ func testScalerMetricLatency(t *testing.T) { } } assert.Equal(t, true, found) + } else { + t.Errorf("metric keda_scaler_metrics_latency_seconds not available") } } @@ -753,7 +722,7 @@ func testScalableObjectMetrics(t *testing.T) { family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL)) - if val, ok := family["keda_internal_scale_loop_latency"]; ok { + if val, ok := family["keda_internal_scale_loop_latency_seconds"]; ok { var found bool metrics := val.GetMetric() @@ -781,7 +750,7 @@ func testScalableObjectMetrics(t *testing.T) { } assert.Equal(t, true, found) } else { - t.Errorf("scaledobject metric not available") + t.Errorf("keda_internal_scale_loop_latency_seconds metric not available") } } @@ -806,6 +775,8 @@ func testScalerActiveMetric(t *testing.T) { } } assert.Equal(t, true, found) + } else { + t.Errorf("metric keda_scaler_active not available") } } @@ -935,6 +906,7 @@ func checkBuildInfo(t *testing.T, families map[string]*prommodel.MetricFamily) { family, ok := families["keda_build_info"] assert.True(t, ok, "keda_build_info not available") if !ok { + t.Errorf("metric keda_build_info not available") return } @@ -969,8 +941,8 @@ func getLatestCommit(t *testing.T) string { func checkTriggerTotalValues(t *testing.T, families map[string]*prommodel.MetricFamily, expected map[string]int) { t.Log("--- testing trigger total metrics ---") - family, ok := families["keda_trigger_totals"] - assert.True(t, ok, "keda_trigger_totals not available") + family, ok := families["keda_trigger_registered_total"] + assert.True(t, ok, "keda_trigger_registered_total not available") if !ok { return } @@ -998,8 +970,8 @@ func checkTriggerTotalValues(t *testing.T, families map[string]*prommodel.Metric func checkCRTotalValues(t *testing.T, families map[string]*prommodel.MetricFamily, expected map[string]map[string]int) { t.Log("--- testing resource total metrics ---") - family, ok := families["keda_resource_totals"] - assert.True(t, ok, "keda_resource_totals not available") + family, ok := families["keda_resource_registered_total"] + assert.True(t, ok, "keda_resource_registered_total not available") if !ok { return } @@ -1027,9 +999,9 @@ func checkCRTotalValues(t *testing.T, families map[string]*prommodel.MetricFamil func checkWebhookValues(t *testing.T, families map[string]*prommodel.MetricFamily) { t.Log("--- testing webhook metrics ---") - family, ok := families["keda_webhook_scaled_object_validation_errors"] + family, ok := families["keda_webhook_scaled_object_validation_errors_total"] if !ok { - t.Errorf("metric keda_webhook_scaled_object_validation_errors not available") + t.Errorf("metric keda_webhook_scaled_object_validation_errors_total not available") return } @@ -1046,9 +1018,9 @@ func checkWebhookValues(t *testing.T, families map[string]*prommodel.MetricFamil } assert.GreaterOrEqual(t, metricValue, 1.0, "keda_webhook_scaled_object_validation_errors has to be greater than 0") - family, ok = families["keda_webhook_scaled_object_validation_total"] + family, ok = families["keda_webhook_scaled_object_validations_total"] if !ok { - t.Errorf("metric keda_webhook_scaled_object_validation_total not available") + t.Errorf("metric keda_webhook_scaled_object_validations_total not available") return }