[ENHANCEMENT] Use resource attributes for multiplying span metrics

CHANGELOG.md

@@ -1,5 +1,5 @@
 ## main / unreleased
-
+* [ENHANCEMENT] The span multiplier now also sources its value from the resource attributes. [#4210](https://github.com/grafana/tempo/pull/4210)
 * [ENHANCEMENT] Changed log level from INFO to DEBUG for the TempoDB Find operation using traceId to reduce excessive/unwanted logs in log search. [#4179](https://github.com/grafana/tempo/pull/4179) (@Aki0x137)
 * [ENHANCEMENT] Pushdown collection of results from generators in the querier [#4119](https://github.com/grafana/tempo/pull/4119) (@electron0zero)
 * [CHANGE] Add throughput and SLO metrics in the tags and tag values endpoints [#4148](https://github.com/grafana/tempo/pull/4148) (@electron0zero)

docs/sources/tempo/configuration/_index.md

@@ -370,6 +370,8 @@ metrics_generator:
             [peer_attributes: <list of string> | default = ["peer.service", "db.name", "db.system"] ]
 
             # Attribute Key to multiply span metrics
+            # Note that the attribute name is searched for in both
+            # resouce and span level attributes
             [span_multiplier_key: <string> | default = ""]
 
             # Enables additional labels for services and virtual nodes.
@@ -413,6 +415,8 @@ metrics_generator:
             [enable_target_info: <bool> | default = false]
 
             # Attribute Key to multiply span metrics
+            # Note that the attribute name is searched for in both
+            # resouce and span level attributes
             [span_multiplier_key: <string> | default = ""]
 
             # List of policies that will be applied to spans for inclusion or exclusion.

modules/generator/processor/servicegraphs/servicegraphs.go

@@ -184,7 +184,7 @@ func (p *Processor) consume(resourceSpans []*v1_trace.ResourceSpans) (err error)
 		for _, ils := range rs.ScopeSpans {
 			for _, span := range ils.Spans {
 				connectionType := store.Unknown
-				spanMultiplier := processor_util.GetSpanMultiplier(p.Cfg.SpanMultiplierKey, span)
+				spanMultiplier := processor_util.GetSpanMultiplier(p.Cfg.SpanMultiplierKey, span, rs.Resource)
 				switch span.Kind {
 				case v1_trace.Span_SPAN_KIND_PRODUCER:
 					// override connection type and continue processing as span kind client

modules/generator/processor/spanmetrics/spanmetrics.go

@@ -201,7 +201,7 @@ func (p *Processor) aggregateMetricsForSpan(svcName string, jobName string, inst
 		labelValues = append(labelValues, instanceID)
 	}
 
-	spanMultiplier := processor_util.GetSpanMultiplier(p.Cfg.SpanMultiplierKey, span)
+	spanMultiplier := processor_util.GetSpanMultiplier(p.Cfg.SpanMultiplierKey, span, rs)
 
 	registryLabelValues := p.registry.NewLabelValueCombo(labels, labelValues)
 

modules/generator/processor/util/util.go

@@ -4,6 +4,7 @@ import (
 	semconv "go.opentelemetry.io/otel/semconv/v1.25.0"
 
 	v1_common "github.com/grafana/tempo/pkg/tempopb/common/v1"
+	v1_resource "github.com/grafana/tempo/pkg/tempopb/resource/v1"
 	v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1"
 	tempo_util "github.com/grafana/tempo/pkg/util"
 )
@@ -31,19 +32,26 @@ func FindAttributeValue(key string, attributes ...[]*v1_common.KeyValue) (string
 	return "", false
 }
 
-func GetSpanMultiplier(ratioKey string, span *v1.Span) float64 {
-	spanMultiplier := 1.0
+func GetSpanMultiplier(ratioKey string, span *v1.Span, rs *v1_resource.Resource) float64 {
 	if ratioKey != "" {
 		for _, kv := range span.Attributes {
 			if kv.Key == ratioKey {
 				v := kv.Value.GetDoubleValue()
 				if v > 0 {
-					spanMultiplier = 1.0 / v
+					return 1.0 / v
+				}
+			}
+		}
+		for _, kv := range rs.Attributes {
+			if kv.Key == ratioKey {
+				v := kv.Value.GetDoubleValue()
+				if v > 0 {
+					return 1.0 / v
 				}
 			}
 		}
 	}
-	return spanMultiplier
+	return 1.0
 }
 
 func GetJobValue(attributes []*v1_common.KeyValue) string {