[exporter/datadog] Add host_metadata section (#9100)

* [exporter/datadog] Add `host_metadata` section * Add CHANGELOG entry * Apply suggestions from code review Co-authored-by: Albert Vaca Cintora <[email protected]> * Apply suggestions from code review Co-authored-by: Kylian Serrania <[email protected]> Co-authored-by: Albert Vaca Cintora <[email protected]> Co-authored-by: Kylian Serrania <[email protected]>
open-telemetry · Apr 11, 2022 · c03519a · c03519a
1 parent ee0c9ce
commit c03519a
Show file tree

Hide file tree

Showing 16 changed files with 309 additions and 67 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,6 +15,7 @@
 - `podmanreceiver`: Add API timeout configuration option (#9014)
 - `cmd/mdatagen`: Add `sem_conv_version` field to metadata.yaml that is used to set metrics SchemaURL (#9010)
 - `splunkheceporter`: Add an option to disable log or profiling data (#9065)
+- `datadogexporter`: Add `host_metadata` configuration section to configure host metadata export (#9100)
 
 ### 🛑 Breaking changes 🛑
 
@@ -41,9 +42,13 @@
 - `datadogexporter`: Deprecate `version` setting in favor of `service.version` semantic convention (#8784)
 - `datadogexporter`: Deprecate `env` setting in favor of `deployment.environment` semantic convention (#9017)
 - `datadogexporter`: Deprecate `GetHostTags` method from `TagsConfig` struct (#8975)
+- `datadogexporter`: Deprecate `tags` setting in favor of `host_metadata::tags` (#9100)
+- `datadogexporter`: Deprecate `send_metadata` setting in favor of `host_metadata::enabled` (#9100)
+- `datadogexporter`: Deprecate `use_resource_metadata` setting in favor of `host_metadata::hostname_source` (#9100)
 - `prometheusexecreceiver`: Deprecate prom_exec receiver (#9058)
 - `fluentbitextension`: Deprecate Fluentbit extension (#9062)
 
+
 ### 🚀 New components 🚀
 
 ### 🧰 Bug fixes 🧰

diff --git a/exporter/datadogexporter/README.md b/exporter/datadogexporter/README.md
@@ -69,8 +69,8 @@ exporters:
   datadog/api:
     hostname: customhostname
 
-    tags:
-      - example:tag
+    host_metadata:
+      tags: [example:tag]
 
     api:
       key: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa

diff --git a/exporter/datadogexporter/config/config.go b/exporter/datadogexporter/config/config.go
@@ -31,7 +31,7 @@ import (
 
 var (
 	errUnsetAPIKey = errors.New("api.key is not set")
-	errNoMetadata  = errors.New("only_metadata can't be enabled when send_metadata or use_resource_metadata is disabled")
+	errNoMetadata  = errors.New("only_metadata can't be enabled when host_metadata::enabled = false or host_metadata::hostname_source != first_resource")
 )
 
 // TODO: Import these from translator when we eliminate cyclic dependency.
@@ -228,10 +228,11 @@ type TagsConfig struct {
 	// Superseded by Tags if the latter is set.
 	// Should not be set in the user-provided config.
 	//
-	// Deprecated: [v0.47.0] Use Tags instead.
+	// Deprecated: [v0.47.0] Use `host_metadata::tags` HostMetadataConfig.Tags instead.
 	EnvVarTags string `mapstructure:"envvartags"`
 
 	// Tags is the list of default tags to add to every metric or trace.
+	// Deprecated: [v0.49.0] Use `host_metadata::tags` (HostMetadataConfig.Tags)
 	Tags []string `mapstructure:"tags"`
 }
 
@@ -250,6 +251,64 @@ func (t *TagsConfig) GetHostTags() []string {
 	return tags
 }
 
+// HostnameSource is the source for the hostname of host metadata.
+type HostnameSource string
+
+const (
+	// HostnameSourceFirstResource picks the host metadata hostname from the resource
+	// attributes on the first OTLP payload that gets to the exporter. If it is lacking any
+	// hostname-like attributes, it will fallback to 'config_or_system' behavior (see below).
+	//
+	// Do not use this hostname source if receiving data from multiple hosts.
+	HostnameSourceFirstResource HostnameSource = "first_resource"
+
+	// HostnameSourceConfigOrSystem picks the host metadata hostname from the 'hostname' setting,
+	// and if this is empty, from available system APIs and cloud provider endpoints.
+	HostnameSourceConfigOrSystem HostnameSource = "config_or_system"
+)
+
+var _ encoding.TextUnmarshaler = (*HostnameSource)(nil)
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface.
+func (sm *HostnameSource) UnmarshalText(in []byte) error {
+	switch mode := HostnameSource(in); mode {
+	case HostnameSourceFirstResource,
+		HostnameSourceConfigOrSystem:
+		*sm = mode
+		return nil
+	default:
+		return fmt.Errorf("invalid host metadata hostname source %q", mode)
+	}
+}
+
+// HostMetadataConfig defines the host metadata related configuration.
+// Host metadata is the information used for populating the infrastructure list,
+// the host map and providing host tags functionality.
+//
+// The exporter will send host metadata for a single host, whose name is chosen
+// according to `host_metadata::hostname_source`.
+type HostMetadataConfig struct {
+	// Enabled enables the host metadata functionality.
+	Enabled bool `mapstructure:"enabled"`
+
+	// HostnameSource is the source for the hostname of host metadata.
+	// Valid values are 'first_resource' and 'config_or_system':
+	// - 'first_resource' picks the host metadata hostname from the resource
+	//    attributes on the first OTLP payload that gets to the exporter.
+	//    If the first payload lacks hostname-like attributes, it will fallback to 'config_or_system'.
+	//    Do not use this hostname source if receiving data from multiple hosts.
+	// - 'config_or_system' picks the host metadata hostname from the 'hostname' setting,
+	//    If this is empty it will use available system APIs and cloud provider endpoints.
+	//
+	// The current default if 'first_resource'.
+	HostnameSource HostnameSource `mapstructure:"hostname_source"`
+
+	// Tags is a list of host tags.
+	// These tags will be attached to telemetry signals that have the host metadata hostname.
+	// To attach tags to telemetry signals regardless of the host, use a processor instead.
+	Tags []string `mapstructure:"tags"`
+}
+
 // LimitedTLSClientSetting is a subset of TLSClientSetting, see LimitedHTTPClientSettings for more details
 type LimitedTLSClientSettings struct {
 	// InsecureSkipVerify controls whether a client verifies the server's
@@ -281,19 +340,23 @@ type Config struct {
 	// Traces defines the Traces exporter specific configuration
 	Traces TracesConfig `mapstructure:"traces"`
 
+	// HostMetadata defines the host metadata specific configuration
+	HostMetadata HostMetadataConfig `mapstructure:"host_metadata"`
+
 	// SendMetadata defines whether to send host metadata
 	// This is undocumented and only used for unit testing.
 	//
 	// This can't be disabled if `only_metadata` is true.
+	// Deprecated: [v0.49.0] Use `host_metadata::enabled` (HostMetadata.Enabled) instead.
 	SendMetadata bool `mapstructure:"send_metadata"`
 
 	// OnlyMetadata defines whether to only send metadata
 	// This is useful for agent-collector setups, so that
 	// metadata about a host is sent to the backend even
 	// when telemetry data is reported via a different host.
 	//
-	// This flag is incompatible with disabling `send_metadata`
-	// or `use_resource_metadata`.
+	// This flag is incompatible with disabling host metadata,
+	// `use_resource_metadata`, or `host_metadata::hostname_source != first_resource`
 	OnlyMetadata bool `mapstructure:"only_metadata"`
 
 	// UseResourceMetadata defines whether to use resource attributes
@@ -302,6 +365,7 @@ type Config struct {
 	// By default this is true: the first resource attribute getting to
 	// the exporter will be used for host metadata.
 	// Disable this in the Collector if you are using an agent-collector setup.
+	// Deprecated: [v0.49.0] Use `host_metadata::hostname_source` (HostMetadata.HostnameSource) instead.
 	UseResourceMetadata bool `mapstructure:"use_resource_metadata"`
 
 	// warnings stores non-fatal configuration errors.
@@ -314,7 +378,7 @@ func (c *Config) Sanitize(logger *zap.Logger) error {
 		c.TagsConfig.Env = "none"
 	}
 
-	if c.OnlyMetadata && (!c.SendMetadata || !c.UseResourceMetadata) {
+	if c.OnlyMetadata && (!c.HostMetadata.Enabled || c.HostMetadata.HostnameSource != HostnameSourceFirstResource) {
 		return errNoMetadata
 	}
 

diff --git a/exporter/datadogexporter/config/config_test.go b/exporter/datadogexporter/config/config_test.go
@@ -180,16 +180,44 @@ func TestSpanNameRemappingsValidation(t *testing.T) {
 	require.Error(t, err)
 }
 
-func TestInvalidSumMode(t *testing.T) {
-	cfgMap := config.NewMapFromStringMap(map[string]interface{}{
-		"metrics": map[string]interface{}{
-			"sums": map[string]interface{}{
-				"cumulative_monotonic_mode": "invalid_mode",
-			},
+func TestUnmarshal(t *testing.T) {
+	tests := []struct {
+		name      string
+		configMap *config.Map
+		cfg       Config
+		err       string
+	}{
+		{
+			name: "invalid cumulative monotonic mode",
+			configMap: config.NewMapFromStringMap(map[string]interface{}{
+				"metrics": map[string]interface{}{
+					"sums": map[string]interface{}{
+						"cumulative_monotonic_mode": "invalid_mode",
+					},
+				},
+			}),
+			err: "1 error(s) decoding:\n\n* error decoding 'metrics.sums.cumulative_monotonic_mode': invalid cumulative monotonic sum mode \"invalid_mode\"",
+		},
+		{
+			name: "invalid host metadata hostname source",
+			configMap: config.NewMapFromStringMap(map[string]interface{}{
+				"host_metadata": map[string]interface{}{
+					"hostname_source": "invalid_source",
+				},
+			}),
+			err: "1 error(s) decoding:\n\n* error decoding 'host_metadata.hostname_source': invalid host metadata hostname source \"invalid_source\"",
 		},
-	})
+	}
 
-	cfg := futureDefaultConfig()
-	err := cfg.Unmarshal(cfgMap)
-	assert.EqualError(t, err, "1 error(s) decoding:\n\n* error decoding 'metrics.sums.cumulative_monotonic_mode': invalid cumulative monotonic sum mode \"invalid_mode\"")
+	for _, testInstance := range tests {
+		t.Run(testInstance.name, func(t *testing.T) {
+			cfg := futureDefaultConfig()
+			err := cfg.Unmarshal(testInstance.configMap)
+			if err != nil || testInstance.err != "" {
+				assert.EqualError(t, err, testInstance.err)
+			} else {
+				assert.Equal(t, testInstance.cfg, cfg)
+			}
+		})
+	}
 }
diff --git a/exporter/datadogexporter/config/warn_envvars.go b/exporter/datadogexporter/config/warn_envvars.go
@@ -51,6 +51,10 @@ func futureDefaultConfig() *Config {
 			SampleRate:      1,
 			IgnoreResources: []string{},
 		},
+		HostMetadata: HostMetadataConfig{
+			Enabled:        true,
+			HostnameSource: HostnameSourceFirstResource,
+		},
 		SendMetadata:        true,
 		UseResourceMetadata: true,
 	}

diff --git a/exporter/datadogexporter/config/warning_deprecated.go b/exporter/datadogexporter/config/warning_deprecated.go
@@ -53,6 +53,37 @@ var renamedSettings = []renameError{
 			}
 		},
 	},
+	{
+		oldName:      "tags",
+		newName:      "host_metadata::tags",
+		oldRemovedIn: "v0.52.0",
+		issueNumber:  9099,
+		updateFn: func(c *Config) {
+			c.HostMetadata.Tags = c.Tags
+		},
+	},
+	{
+		oldName:      "send_metadata",
+		newName:      "host_metadata::enabled",
+		oldRemovedIn: "v0.52.0",
+		issueNumber:  9099,
+		updateFn: func(c *Config) {
+			c.HostMetadata.Enabled = c.SendMetadata
+		},
+	},
+	{
+		oldName:      "use_resource_metadata",
+		newName:      "host_metadata::hostname_source",
+		oldRemovedIn: "v0.52.0",
+		issueNumber:  9099,
+		updateFn: func(c *Config) {
+			if c.UseResourceMetadata {
+				c.HostMetadata.HostnameSource = HostnameSourceFirstResource
+			} else {
+				c.HostMetadata.HostnameSource = HostnameSourceConfigOrSystem
+			}
+		},
+	},
 }
 
 // Error implements the error interface.

diff --git a/exporter/datadogexporter/example/config.yaml b/exporter/datadogexporter/example/config.yaml
@@ -38,16 +38,32 @@ exporters:
     # version: myversion
 
     ## @param tags - list of strings - optional - default: []
-    ## The list of default tags to add to every metric or trace.
+    ## The list of tags to send as host tags.
+    ## Deprecated: [v0.49.0] Use `host_metadata::tags` instead.
+    ## This option will be removed in v0.52.0.
     ## If unset it will be determined from the `DD_TAGS` environment variable, specified
     ## as a list of space-separated strings (Deprecated: [v0.47.0] use 'env' config source instead).
     #
     # tags: []
 
+    ## @params send_metadata - boolean - optional - default: true
+    ## Deprecated: [v0.49.0] Use `host_metadata::enabled` instead.
+    ## This option will be removed in v0.52.0.
+    #
+    # send_metadata: true
+
+    ## @params use_resource_metadata - boolean - optional - default: true
+    ## Deprecated: [v0.49.0] Use `host_metadata::hostname_source` instead.
+    ## This option will be removed in v0.52.0.
+    # 
+    # use_resource_metadata: true
+
     ## @param only_metadata - boolean - optional - default: false
     ## Whether to send only metadata. This is useful for agent-collector
     ## setups, so that metadata about a host is sent to the backend even
     ## when telemetry data is reported via a different host.
+    #
+    # only_metadata: false
 
     ## @param api - custom object - required.
     ## Specific API configuration.
@@ -177,6 +193,40 @@ exporters:
       ## https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/1909
       #
       # span_name_as_resource_name: true
+
+    ## @param host_metadata - custom object - optional
+    ## Host metadata specific configuration.
+    ## Host metadata is the information used for populating the infrastructure list, the host map and providing host tags functionality within the Datadog app.
+    ##
+    ## The exporter will only send host metadata for a single host, whose name is chosen
+    ## according to `host_metadata::hostname_source`.
+    #
+    # host_metadata:
+      ## @param enabled - boolean - optional - default: true
+      ## Enable the host metadata functionality
+      #
+      # enabled: true
+
+      ## @param hostname_source - enum - optional - default: first_resource
+      ## Source for the hostname of host metadata.
+      ## Valid values are 'first_resource' and 'config_or_system':
+      ## - 'first_resource' picks the host metadata hostname from the resource attributes on the first OTLP payload that gets to the exporter.
+      ##    If the first payload lacks hostname-like attributes, it will fallback to 'config_or_system' behavior.
+      ##    Do not use this hostname source if receiving data from multiple hosts.
+      ##
+      ## - 'config_or_system' picks the host metadata hostname from the 'hostname' setting, falling back to system and cloud provider APIs.
+      ##
+      ##  The current default is 'first_resource'.
+      #
+      # hostname_source: first_resource
+
+      ## @param tags - list of strings - optional - default: empty list
+      ## List of host tags to be sent as part of the host metadata.
+      ## These tags will be attached to telemetry signals that have the host metadata hostname.
+      ## 
+      ## To attach tags to telemetry signals regardless of the host, use a processor instead.
+      #
+      # tags: []
 
 
 service:

diff --git a/exporter/datadogexporter/factory.go b/exporter/datadogexporter/factory.go
@@ -108,6 +108,11 @@ func (*factory) createDefaultConfig() config.Exporter {
 			IgnoreResources: []string{},
 		},
 
+		HostMetadata: ddconfig.HostMetadataConfig{
+			Enabled:        true,
+			HostnameSource: ddconfig.HostnameSourceFirstResource,
+		},
+
 		SendMetadata:        true,
 		UseResourceMetadata: true,
 	}