feat(sumologicschemaprocessor): add config for nesting processor

CHANGELOG.md

@@ -7,7 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- feat(sumologicschemaprocessor): add nesting processor [#877]
+
 [Unreleased]: https://github.com/SumoLogic/sumologic-otel-collector/compare/v0.67.0-sumo-0...main
+[#877]: https://github.com/SumoLogic/sumologic-otel-collector/pull/877
 
 ## [v0.67.0-sumo-0]
 

pkg/processor/sumologicschemaprocessor/README.md

@@ -32,6 +32,17 @@ processors:
     # See `translate_telegraf_metrics_processor.go` for full list of translations.
     # default = true
     translate_telegraf_attributes: {true, false}
+
+    # Specifies if attributes should be nested, basing on their keys.
+    # See "Nesting attributes" documentation chapter from this document.
+    nest_attributes:
+      # Defines whether attributes should be nested.
+      # default = false
+      enabled: {true, false}
+
+      # Defines the string used to separate key names in attributes that are to be nested.
+      # default = "."
+      separator: <separator>
 ```
 
 ## Features
@@ -98,3 +109,72 @@ Below is a list of all attribute keys that are being translated.
 | `k8s.statefulset.name`    | `statefulset`       |
 | `service.name`            | `service`           |
 | `log.file.path_resolved`  | `_sourceName`       |
+
+### Nesting attributes
+
+Nesting attributes allows to change the structure of attributes (both resource level and record level attributes)
+basing on their keys by nesting attributes with common paths into maps.
+Common path is defined as a common prefix that consists of strings separated by a given separator string (by default a dot - `.`)
+and end with that separator, for example: if the separator is `.`,
+then `xyz` is a common prefix path for `xyz.abc.qwe` and `xyz.foo`,
+but `sumo` **is not** a prefix path for neither `sumologic.foo.baz` nor `sumologic.bar.baz`.
+
+#### Example
+
+The following set of attributes:
+
+```json
+{
+  "kubernetes.container_name": "xyz",
+  "kubernetes.host.name": "the host",
+  "kubernetes.host.address": "127.0.0.1",
+  "kubernetes.namespace_name": "sumologic",
+  "another_attr": "42"
+}
+```
+
+Should be translated into such set:
+
+```json
+{
+  "kubernetes": {
+    "container_name": "xyz",
+    "namespace_name": "sumologic",
+    "host": {
+      "name": "the host",
+      "address": "127.0.0.1"
+    }
+  },
+ "another_attr": "42"
+}
+```
+
+#### Known issues
+
+This feature has undefined behavior when in input there are various keys that will be mapped to the same nested structure. 
+For example, given the following attributes:
+
+```json
+{
+  "a.b.c.": "d",
+  "a": {
+    "b": {
+      "c": "e"
+    }
+  }
+}
+```
+
+It is not possible to predict, what will be the result. It will have the following structure:
+
+```json
+{
+  "a": {
+    "b": {
+      "c": ...
+    }
+  }
+}
+```
+
+However, it is not possible to know a priori if the value under key `c` will be equal to `d` or `e`.

pkg/processor/sumologicschemaprocessor/config.go

@@ -22,15 +22,20 @@ import (
 type Config struct {
 	config.ProcessorSettings `mapstructure:",squash"`
 
-	AddCloudNamespace           bool `mapstructure:"add_cloud_namespace"`
-	TranslateAttributes         bool `mapstructure:"translate_attributes"`
-	TranslateTelegrafAttributes bool `mapstructure:"translate_telegraf_attributes"`
+	AddCloudNamespace           bool                    `mapstructure:"add_cloud_namespace"`
+	TranslateAttributes         bool                    `mapstructure:"translate_attributes"`
+	TranslateTelegrafAttributes bool                    `mapstructure:"translate_telegraf_attributes"`
+	NestAttributes              *NestingProcessorConfig `mapstructure:"nest_attributes"`
 }
 
 const (
 	defaultAddCloudNamespace           = true
 	defaultTranslateAttributes         = true
 	defaultTranslateTelegrafAttributes = true
+
+	// Nesting processor default config
+	defaultNestingEnabled   = false
+	defaultNestingSeparator = "."
 )
 
 // Ensure the Config struct satisfies the config.Processor interface.
@@ -42,6 +47,10 @@ func createDefaultConfig() component.Config {
 		AddCloudNamespace:           defaultAddCloudNamespace,
 		TranslateAttributes:         defaultTranslateAttributes,
 		TranslateTelegrafAttributes: defaultTranslateTelegrafAttributes,
+		NestAttributes: &NestingProcessorConfig{
+			Separator: defaultNestingSeparator,
+			Enabled:   defaultNestingEnabled,
+		},
 	}
 }
 

pkg/processor/sumologicschemaprocessor/config_test.go

@@ -51,6 +51,10 @@ func TestLoadConfig(t *testing.T) {
 			AddCloudNamespace:           false,
 			TranslateAttributes:         true,
 			TranslateTelegrafAttributes: true,
+			NestAttributes: &NestingProcessorConfig{
+				Enabled:   false,
+				Separator: ".",
+			},
 		})
 
 	p2 := cfg.Processors[component.NewIDWithName(typeStr, "disabled-attribute-translation")]
@@ -64,6 +68,10 @@ func TestLoadConfig(t *testing.T) {
 			AddCloudNamespace:           true,
 			TranslateAttributes:         false,
 			TranslateTelegrafAttributes: true,
+			NestAttributes: &NestingProcessorConfig{
+				Enabled:   false,
+				Separator: ".",
+			},
 		})
 
 	p3 := cfg.Processors[component.NewIDWithName(typeStr, "disabled-telegraf-attribute-translation")]
@@ -77,5 +85,26 @@ func TestLoadConfig(t *testing.T) {
 			AddCloudNamespace:           true,
 			TranslateAttributes:         true,
 			TranslateTelegrafAttributes: false,
+			NestAttributes: &NestingProcessorConfig{
+				Enabled:   false,
+				Separator: ".",
+			},
+		})
+
+	p4 := cfg.Processors[component.NewIDWithName(typeStr, "enabled-nesting")]
+
+	assert.Equal(t, p4,
+		&Config{
+			ProcessorSettings: config.NewProcessorSettings(component.NewIDWithName(
+				typeStr,
+				"enabled-nesting",
+			)),
+			AddCloudNamespace:           true,
+			TranslateAttributes:         true,
+			TranslateTelegrafAttributes: true,
+			NestAttributes: &NestingProcessorConfig{
+				Enabled:   true,
+				Separator: "!",
+			},
 		})
 }

pkg/processor/sumologicschemaprocessor/nesting_processor.go

@@ -23,15 +23,20 @@ import (
 	"go.opentelemetry.io/collector/pdata/ptrace"
 )
 
+type NestingProcessorConfig struct {
+	Separator string `mapstructure:"separator"`
+	Enabled   bool   `mapstructure:"enabled"`
+}
+
 type NestingProcessor struct {
 	separator string
 	enabled   bool
 }
 
-func newNestingProcessor(separator string, enabled bool) (*NestingProcessor, error) {
+func newNestingProcessor(config *NestingProcessorConfig) (*NestingProcessor, error) {
 	proc := &NestingProcessor{
-		separator: separator,
-		enabled:   enabled,
+		separator: config.Separator,
+		enabled:   config.Enabled,
 	}
 
 	return proc, nil

pkg/processor/sumologicschemaprocessor/nesting_processor_test.go

@@ -95,7 +95,7 @@ func TestNestingAttributes(t *testing.T) {
 
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
-			proc, err := newNestingProcessor(".", true)
+			proc, err := newNestingProcessor(&NestingProcessorConfig{Separator: ".", Enabled: true})
 			require.NoError(t, err)
 
 			attrs := mapToPcommonMap(testCase.input)

pkg/processor/sumologicschemaprocessor/processor.go

@@ -54,7 +54,7 @@ func newSumologicSchemaProcessor(set component.ProcessorCreateSettings, config *
 		return nil, err
 	}
 
-	nestingProcessor, err := newNestingProcessor(".", false)
+	nestingProcessor, err := newNestingProcessor(config.NestAttributes)
 	if err != nil {
 		return nil, err
 	}

pkg/processor/sumologicschemaprocessor/testdata/config.yaml

@@ -9,6 +9,10 @@ processors:
     translate_attributes: false
   sumologic_schema/disabled-telegraf-attribute-translation:
     translate_telegraf_attributes: false
+  sumologic_schema/enabled-nesting:
+    nest_attributes:
+      enabled: true
+      separator: "!"
 
 exporters:
   nop:
@@ -39,6 +43,14 @@ service:
       exporters:
       - nop
 
+    metrics/3:
+      receivers:
+      - nop
+      processors:
+      - sumologic_schema/enabled-nesting
+      exporters:
+      - nop
+
     traces:
       receivers:
       - nop