feat(sumologicschemaprocessor): add allowlist and denylist for nestin…

…g processor
SumoLogic · Dec 23, 2022 · 96a8107 · 96a8107
1 parent ba4be7c
commit 96a8107
Show file tree

Hide file tree

Showing 4 changed files with 168 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,9 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - feat(sumologicschemaprocessor): add nesting processor [#877]
+- feat(sumologicschemaprocessor): add allowlist and denylist to nesting processor [#880]
 
 [Unreleased]: https://github.com/SumoLogic/sumologic-otel-collector/compare/v0.67.0-sumo-0...main
 [#877]: https://github.com/SumoLogic/sumologic-otel-collector/pull/877
+[#880]: https://github.com/SumoLogic/sumologic-otel-collector/pull/880
 
 ## [v0.67.0-sumo-0]
 

diff --git a/pkg/processor/sumologicschemaprocessor/README.md b/pkg/processor/sumologicschemaprocessor/README.md
@@ -43,6 +43,16 @@ processors:
       # Defines the string used to separate key names in attributes that are to be nested.
       # default = "."
       separator: <separator>
+
+      # Defines a list of allowed prefixes to be nested.
+      # For example, if "kubernetes." is in this list, then all "kubernetes.*" attributes will be nested.
+      # default = []
+      include: [<prefix>]
+
+      # Defined a list of prefixes not allowed to be nested.
+      # For example, if "k8s." is in this list, then all "k8s.*" attributes will not be nested.
+      # default = []
+      exclude: [<prefix>]
 ```
 
 ## Features
@@ -149,6 +159,47 @@ Should be translated into such set:
 }
 ```
 
+#### Allowlist and denylist
+
+Properties `include` and `exclude` in the config allow to define list of prefixes that are allowed or not allowed to be nested.
+
+For example, with the following config:
+
+```yaml
+nest_attributes:
+  enabled: true
+  include: ["kubernetes.host."]
+  exclude: ["kubernetes.host.naming"]
+```
+
+and following input:
+
+```json
+{
+"kubernetes.container_name": "xyz",
+"kubernetes.host.name": "the host",
+"kubernetes.host.naming_convention": "random",
+"kubernetes.host.address": "127.0.0.1",
+"kubernetes.namespace_name": "sumologic",
+}
+```
+
+The output is:
+
+```json
+{
+"kubernetes.container_name": "xyz",
+"kubernetes": {
+  "host": {
+    "name": "the host",
+    "address": "127.0.0.1"
+    }
+  },
+"kubernetes.host.naming_convention": "random",
+"kubernetes.namespace_name": "sumologic",
+}
+```
+
 #### Known issues
 
 This feature has undefined behavior when in input there are various keys that will be mapped to the same nested structure.

diff --git a/pkg/processor/sumologicschemaprocessor/nesting_processor.go b/pkg/processor/sumologicschemaprocessor/nesting_processor.go
@@ -130,6 +130,12 @@ func (proc *NestingProcessor) processAttributes(attributes pcommon.Map) error {
 	newMap := pcommon.NewMap()
 
 	attributes.Range(func(k string, v pcommon.Value) bool {
+		// If key is not on allow list or is on deny list, skip translating it.
+		if !proc.shouldTranslateKey(k) {
+			v.CopyTo(newMap.PutEmpty(k))
+			return true
+		}
+
 		keys := strings.Split(k, proc.separator)
 		if len(keys) == 0 {
 			// Split returns empty slice only if both string and separator are empty
@@ -189,6 +195,33 @@ func (proc *NestingProcessor) processAttributes(attributes pcommon.Map) error {
 	return nil
 }
 
+// Checks if given key fulfills the following condition:
+// - has a prefix that exists in
+func (proc *NestingProcessor) shouldTranslateKey(k string) bool {
+	if len(proc.allowlist) > 0 {
+		isOk := false
+		for i := 0; i < len(proc.allowlist); i++ {
+			if strings.HasPrefix(k, proc.allowlist[i]) {
+				isOk = true
+				break
+			}
+		}
+		if !isOk {
+			return false
+		}
+	}
+
+	if len(proc.denylist) > 0 {
+		for i := 0; i < len(proc.denylist); i++ {
+			if strings.HasPrefix(k, proc.denylist[i]) {
+				return false
+			}
+		}
+	}
+
+	return true
+}
+
 func (proc *NestingProcessor) isEnabled() bool {
 	return proc.enabled
 }

diff --git a/pkg/processor/sumologicschemaprocessor/nesting_processor_test.go b/pkg/processor/sumologicschemaprocessor/nesting_processor_test.go
@@ -26,6 +26,8 @@ func TestNestingAttributes(t *testing.T) {
 		name     string
 		input    map[string]pcommon.Value
 		expected map[string]pcommon.Value
+		include  []string
+		exclude  []string
 	}{
 		{
 			name: "sample nesting",
@@ -47,6 +49,8 @@ func TestNestingAttributes(t *testing.T) {
 				}),
 				"another_attr": pcommon.NewValueStr("42"),
 			},
+			include: []string{},
+			exclude: []string{},
 		},
 		{
 			name: "single values",
@@ -77,6 +81,8 @@ func TestNestingAttributes(t *testing.T) {
 					}),
 				}),
 			},
+			include: []string{},
+			exclude: []string{},
 		},
 		{
 			name: "overwrite map with simple value",
@@ -90,12 +96,87 @@ func TestNestingAttributes(t *testing.T) {
 					"logic": pcommon.NewValueBool(true),
 				}),
 			},
+			include: []string{},
+			exclude: []string{},
+		},
+		{
+			name: "allowlist",
+			input: map[string]pcommon.Value{
+				"kubernetes.container_name": pcommon.NewValueStr("xyz"),
+				"kubernetes.host.name":      pcommon.NewValueStr("the host"),
+				"kubernetes.host.address":   pcommon.NewValueStr("127.0.0.1"),
+				"kubernetes.namespace_name": pcommon.NewValueStr("sumologic"),
+				"another_attr":              pcommon.NewValueStr("42"),
+			},
+			expected: map[string]pcommon.Value{
+				"kubernetes": mapToPcommonValue(map[string]pcommon.Value{
+					"container_name": pcommon.NewValueStr("xyz"),
+					"host": mapToPcommonValue(map[string]pcommon.Value{
+						"name": pcommon.NewValueStr("the host"),
+					}),
+				}),
+				"kubernetes.host.address":   pcommon.NewValueStr("127.0.0.1"),
+				"kubernetes.namespace_name": pcommon.NewValueStr("sumologic"),
+				"another_attr":              pcommon.NewValueStr("42"),
+			},
+			include: []string{"kubernetes.container", "kubernetes.host.name"},
+			exclude: []string{},
+		},
+		{
+			name: "denylist",
+			input: map[string]pcommon.Value{
+				"kubernetes.container_name": pcommon.NewValueStr("xyz"),
+				"kubernetes.host.name":      pcommon.NewValueStr("the host"),
+				"kubernetes.host.address":   pcommon.NewValueStr("127.0.0.1"),
+				"kubernetes.namespace_name": pcommon.NewValueStr("sumologic"),
+				"another_attr":              pcommon.NewValueStr("42"),
+			},
+			expected: map[string]pcommon.Value{
+				"kubernetes.container_name": pcommon.NewValueStr("xyz"),
+				"kubernetes.host.name":      pcommon.NewValueStr("the host"),
+				"kubernetes.host.address":   pcommon.NewValueStr("127.0.0.1"),
+				"kubernetes": mapToPcommonValue(map[string]pcommon.Value{
+					"namespace_name": pcommon.NewValueStr("sumologic"),
+				}),
+				"another_attr": pcommon.NewValueStr("42"),
+			},
+			include: []string{},
+			exclude: []string{"kubernetes.container", "kubernetes.host"},
+		},
+		{
+			name: "denylist and allowlist",
+			input: map[string]pcommon.Value{
+				"kubernetes.container_name":         pcommon.NewValueStr("xyz"),
+				"kubernetes.host.name":              pcommon.NewValueStr("the host"),
+				"kubernetes.host.naming_convention": pcommon.NewValueStr("random"),
+				"kubernetes.host.address":           pcommon.NewValueStr("127.0.0.1"),
+				"kubernetes.namespace_name":         pcommon.NewValueStr("sumologic"),
+				"another_attr":                      pcommon.NewValueStr("42"),
+				"and_end":                           pcommon.NewValueStr("fin"),
+			},
+			expected: map[string]pcommon.Value{
+				"kubernetes.container_name":         pcommon.NewValueStr("xyz"),
+				"kubernetes.host.naming_convention": pcommon.NewValueStr("random"),
+				"kubernetes.namespace_name":         pcommon.NewValueStr("sumologic"),
+				"kubernetes": mapToPcommonValue(map[string]pcommon.Value{
+					"host": mapToPcommonValue(map[string]pcommon.Value{
+						"name":    pcommon.NewValueStr("the host"),
+						"address": pcommon.NewValueStr("127.0.0.1"),
+					}),
+				}),
+				"another_attr": pcommon.NewValueStr("42"),
+				"and_end":      pcommon.NewValueStr("fin"),
+			},
+			include: []string{"kubernetes.host."},
+			exclude: []string{"kubernetes.host.naming"},
 		},
 	}
 
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
-			proc, err := newNestingProcessor(&NestingProcessorConfig{Separator: ".", Enabled: true})
+			proc, err := newNestingProcessor(&NestingProcessorConfig{
+				Separator: ".", Enabled: true, Include: testCase.include, Exclude: testCase.exclude,
+			})
 			require.NoError(t, err)
 
 			attrs := mapToPcommonMap(testCase.input)