From 84993ebfb7e3f28baf04c967f1240b1a505a5f78 Mon Sep 17 00:00:00 2001 From: Jan Michael Auer Date: Thu, 24 Oct 2024 14:31:34 +0200 Subject: [PATCH] feat(metrics): Add target_project_id tag to root counter metrics (#4170) Adds a new tag named `target_project_id` to the `count_per_root_project` metrics in the spans and transactions namespace. The tag contains the numeric identifier of the project where the transaction or spans are sent, in contrast to the root project where the trace originated. This tag is added so that we can track and display stats about the individual data volume that flows from a root project into a specific target project. The tag key will not be added to static strings in the metrics indexer. Therefore, it will not be possible run cross-org queries that filter or group by this tag. --- CHANGELOG.md | 3 ++- relay-server/src/metrics_extraction/event.rs | 23 ++++++++++++++++++- .../metrics_extraction/transactions/mod.rs | 17 ++++++++++++++ .../metrics_extraction/transactions/types.rs | 6 +++++ relay-server/src/services/processor.rs | 2 ++ .../src/services/processor/span/processing.rs | 9 ++++++-- tests/integration/test_metrics.py | 5 ++-- 7 files changed, 59 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c4a916d42..b3c3ad599b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ **Internal:** -- Add a metric that counts span volume in the root project for dynamic sampling (`c:spans/count_per_root_project@none`) ([#4134](https://github.com/getsentry/relay/pull/4134)) +- Add a metric that counts span volume in the root project for dynamic sampling (`c:spans/count_per_root_project@none`). ([#4134](https://github.com/getsentry/relay/pull/4134)) +- Add a tag `target_project_id` to both root project metrics for dynamic sampling (`c:transactions/count_per_root_project@none` and `c:spans/count_per_root_project@none`) which shows the flow trace traffic from root to target projects. ([#4170](https://github.com/getsentry/relay/pull/4170)) - Emit `indexed` tag for the usage metrics. ([#4158](https://github.com/getsentry/relay/pull/4158)) ## 24.10.0 diff --git a/relay-server/src/metrics_extraction/event.rs b/relay-server/src/metrics_extraction/event.rs index 96faccdf05..9de6668d33 100644 --- a/relay-server/src/metrics_extraction/event.rs +++ b/relay-server/src/metrics_extraction/event.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; +use relay_base_schema::project::ProjectId; use relay_common::time::UnixTimestamp; use relay_dynamic_config::CombinedMetricExtractionConfig; use relay_event_normalization::span::description::ScrubMongoDescription; @@ -54,6 +55,7 @@ pub fn extract_metrics( event: &mut Event, config: CombinedMetricExtractionConfig<'_>, sampling_decision: SamplingDecision, + target_project_id: ProjectId, max_tag_value_size: usize, extract_spans: bool, ) -> ExtractedMetrics { @@ -67,6 +69,7 @@ pub fn extract_metrics( event, config, sampling_decision, + target_project_id, max_tag_value_size, &mut metrics, ); @@ -79,6 +82,7 @@ fn extract_span_metrics_for_event( event: &mut Event, config: CombinedMetricExtractionConfig<'_>, sampling_decision: SamplingDecision, + target_project_id: ProjectId, max_tag_value_size: usize, output: &mut ExtractedMetrics, ) { @@ -113,7 +117,13 @@ fn extract_span_metrics_for_event( // This function assumes it is only called when span metrics should be extracted, hence we // extract the span root counter unconditionally. let transaction = transactions::get_transaction_name(event); - let bucket = create_span_root_counter(event, transaction, span_count, sampling_decision); + let bucket = create_span_root_counter( + event, + transaction, + span_count, + sampling_decision, + target_project_id, + ); output.sampling_metrics.extend(bucket); }); } @@ -127,6 +137,7 @@ pub fn create_span_root_counter( transaction: Option, span_count: u32, sampling_decision: SamplingDecision, + target_project_id: ProjectId, ) -> Option { if span_count == 0 { return None; @@ -144,6 +155,10 @@ pub fn create_span_root_counter( let mut tags = BTreeMap::new(); tags.insert("decision".to_owned(), sampling_decision.to_string()); + tags.insert( + "target_project_id".to_owned(), + target_project_id.to_string(), + ); if let Some(transaction) = transaction { tags.insert("transaction".to_owned(), transaction); } @@ -1267,6 +1282,7 @@ mod tests { event.value_mut().as_mut().unwrap(), combined_config(features, None).combined(), SamplingDecision::Keep, + ProjectId::new(4711), 200, true, ) @@ -1484,6 +1500,7 @@ mod tests { event.value_mut().as_mut().unwrap(), combined_config([Feature::ExtractCommonSpanMetricsFromEvent], None).combined(), SamplingDecision::Keep, + ProjectId::new(4711), 200, true, ); @@ -1541,6 +1558,7 @@ mod tests { event.value_mut().as_mut().unwrap(), combined_config([Feature::ExtractCommonSpanMetricsFromEvent], None).combined(), SamplingDecision::Keep, + ProjectId::new(4711), 200, true, ); @@ -1575,6 +1593,7 @@ mod tests { event.value_mut().as_mut().unwrap(), combined_config([Feature::ExtractCommonSpanMetricsFromEvent], None).combined(), SamplingDecision::Keep, + ProjectId::new(4711), 200, true, ); @@ -1839,6 +1858,7 @@ mod tests { event.value_mut().as_mut().unwrap(), combined_config([Feature::ExtractCommonSpanMetricsFromEvent], None).combined(), SamplingDecision::Keep, + ProjectId::new(4711), 200, true, ) @@ -1983,6 +2003,7 @@ mod tests { event.value_mut().as_mut().unwrap(), config, SamplingDecision::Keep, + ProjectId::new(4711), 200, true, ); diff --git a/relay-server/src/metrics_extraction/transactions/mod.rs b/relay-server/src/metrics_extraction/transactions/mod.rs index e12cc9794f..9828b5cc38 100644 --- a/relay-server/src/metrics_extraction/transactions/mod.rs +++ b/relay-server/src/metrics_extraction/transactions/mod.rs @@ -1,6 +1,7 @@ use std::collections::{BTreeMap, BTreeSet}; use relay_base_schema::events::EventType; +use relay_base_schema::project::ProjectId; use relay_common::time::UnixTimestamp; use relay_dynamic_config::{CombinedMetricExtractionConfig, TransactionMetricsConfig}; use relay_event_normalization::span::country_subregion::Subregion; @@ -255,6 +256,7 @@ pub struct TransactionExtractor<'a> { pub generic_config: Option>, pub transaction_from_dsc: Option<&'a str>, pub sampling_decision: SamplingDecision, + pub target_project_id: ProjectId, pub has_profile: bool, } @@ -434,6 +436,7 @@ impl TransactionExtractor<'_> { TransactionCPRTags { decision: self.sampling_decision.to_string(), + target_project_id: self.target_project_id, universal_tags, } }; @@ -615,6 +618,7 @@ mod tests { generic_config: None, transaction_from_dsc: Some("test_transaction"), sampling_decision: SamplingDecision::Keep, + target_project_id: ProjectId::new(4711), has_profile: false, }; @@ -994,6 +998,7 @@ mod tests { generic_config: None, transaction_from_dsc: Some("test_transaction"), sampling_decision: SamplingDecision::Keep, + target_project_id: ProjectId::new(4711), has_profile: false, }; @@ -1169,6 +1174,7 @@ mod tests { generic_config: None, transaction_from_dsc: Some("test_transaction"), sampling_decision: SamplingDecision::Keep, + target_project_id: ProjectId::new(4711), has_profile: false, }; @@ -1320,6 +1326,7 @@ mod tests { generic_config: None, transaction_from_dsc: Some("test_transaction"), sampling_decision: SamplingDecision::Keep, + target_project_id: ProjectId::new(4711), has_profile: false, }; @@ -1395,6 +1402,7 @@ mod tests { generic_config: None, transaction_from_dsc: Some("test_transaction"), sampling_decision: SamplingDecision::Keep, + target_project_id: ProjectId::new(4711), has_profile: false, }; @@ -1560,6 +1568,7 @@ mod tests { generic_config: None, transaction_from_dsc: Some("test_transaction"), sampling_decision: SamplingDecision::Keep, + target_project_id: ProjectId::new(4711), has_profile: false, }; @@ -1599,6 +1608,7 @@ mod tests { generic_config: None, transaction_from_dsc: Some("test_transaction"), sampling_decision: SamplingDecision::Keep, + target_project_id: ProjectId::new(4711), has_profile: false, }; @@ -1667,6 +1677,7 @@ mod tests { generic_config: None, transaction_from_dsc: Some("test_transaction"), sampling_decision: SamplingDecision::Keep, + target_project_id: ProjectId::new(4711), has_profile: false, }; @@ -1773,6 +1784,7 @@ mod tests { generic_config: None, transaction_from_dsc: Some("test_transaction"), sampling_decision: SamplingDecision::Keep, + target_project_id: ProjectId::new(4711), has_profile: false, }; @@ -1806,6 +1818,7 @@ mod tests { generic_config: None, transaction_from_dsc: Some("test_transaction"), sampling_decision: SamplingDecision::Keep, + target_project_id: ProjectId::new(4711), has_profile: false, }; @@ -1843,6 +1856,7 @@ mod tests { generic_config: None, transaction_from_dsc: Some("root_transaction"), sampling_decision: SamplingDecision::Keep, + target_project_id: ProjectId::new(4711), has_profile: false, }; @@ -1860,6 +1874,7 @@ mod tests { ), tags: { "decision": "keep", + "target_project_id": "4711", "transaction": "root_transaction", }, metadata: BucketMetadata { @@ -2112,6 +2127,7 @@ mod tests { generic_config: None, transaction_from_dsc: Some("test_transaction"), sampling_decision: SamplingDecision::Keep, + target_project_id: ProjectId::new(4711), has_profile: false, }; @@ -2213,6 +2229,7 @@ mod tests { generic_config: Some(combined_config), transaction_from_dsc: Some("test_transaction"), sampling_decision: SamplingDecision::Keep, + target_project_id: ProjectId::new(4711), has_profile: false, }; diff --git a/relay-server/src/metrics_extraction/transactions/types.rs b/relay-server/src/metrics_extraction/transactions/types.rs index 0f9923c4f1..6a8ce105d3 100644 --- a/relay-server/src/metrics_extraction/transactions/types.rs +++ b/relay-server/src/metrics_extraction/transactions/types.rs @@ -2,6 +2,7 @@ use std::borrow::Cow; use std::collections::BTreeMap; use std::fmt::Display; +use relay_base_schema::project::ProjectId; use relay_common::time::UnixTimestamp; use relay_metrics::{ Bucket, BucketMetadata, BucketValue, DistributionType, DurationUnit, MetricNamespace, @@ -193,6 +194,7 @@ impl From for BTreeMap { #[derive(Clone, Debug, PartialEq, Eq)] pub struct TransactionCPRTags { pub decision: String, + pub target_project_id: ProjectId, pub universal_tags: CommonTags, } @@ -200,6 +202,10 @@ impl From for BTreeMap { fn from(value: TransactionCPRTags) -> Self { let mut map: BTreeMap = value.universal_tags.into(); map.insert("decision".to_string(), value.decision); + map.insert( + "target_project_id".to_string(), + value.target_project_id.to_string(), + ); map } } diff --git a/relay-server/src/services/processor.rs b/relay-server/src/services/processor.rs index da90ce8d3d..5e5616a69f 100644 --- a/relay-server/src/services/processor.rs +++ b/relay-server/src/services/processor.rs @@ -1495,6 +1495,7 @@ impl EnvelopeProcessorService { event, combined_config, sampling_decision, + state.project_id, self.inner .config .aggregator_config_for(MetricNamespace::Spans) @@ -1519,6 +1520,7 @@ impl EnvelopeProcessorService { generic_config: Some(combined_config), transaction_from_dsc, sampling_decision, + target_project_id: state.project_id, has_profile: profile_id.is_some(), }; diff --git a/relay-server/src/services/processor/span/processing.rs b/relay-server/src/services/processor/span/processing.rs index 38b27f19de..87906b91b4 100644 --- a/relay-server/src/services/processor/span/processing.rs +++ b/relay-server/src/services/processor/span/processing.rs @@ -147,8 +147,13 @@ pub fn process( .value() .and_then(|d| d.segment_name.value()) .cloned(); - let bucket = - event::create_span_root_counter(span, transaction, 1, sampling_decision); + let bucket = event::create_span_root_counter( + span, + transaction, + 1, + sampling_decision, + state.project_id, + ); state .extracted_metrics .extend_sampling_metrics(bucket, Some(sampling_decision)); diff --git a/tests/integration/test_metrics.py b/tests/integration/test_metrics.py index 97a2adfa31..320e5fe0fe 100644 --- a/tests/integration/test_metrics.py +++ b/tests/integration/test_metrics.py @@ -919,6 +919,7 @@ def assert_transaction(): "retention_days": 90, "tags": { "decision": "drop" if discard_data else "keep", + "target_project_id": "42", "transaction": "transaction_which_starts_trace", }, "name": "c:transactions/count_per_root_project@none", @@ -989,7 +990,7 @@ def test_transaction_metrics_count_per_root_project( "org_id": 1, "project_id": 41, "retention_days": 90, - "tags": {"decision": "keep", "transaction": "test"}, + "tags": {"decision": "keep", "target_project_id": "42", "transaction": "test"}, "name": "c:transactions/count_per_root_project@none", "type": "c", "value": 1.0, @@ -1000,7 +1001,7 @@ def test_transaction_metrics_count_per_root_project( "org_id": 1, "project_id": 42, "retention_days": 90, - "tags": {"decision": "keep"}, + "tags": {"decision": "keep", "target_project_id": "42"}, "name": "c:transactions/count_per_root_project@none", "type": "c", "value": 2.0,