SumoLogic · jagan2221 · Sep 13, 2024 · Sep 13, 2024 · Sep 16, 2024 · Sep 16, 2024
@@ -0,0 +1 @@
+feat(otelcolLogs): Split kubelet from systemd logs with a separate feature flag
@@ -57,6 +57,7 @@ The following table lists the configurable parameters of the Sumo Logic chart an
 | `sumologic.logs.multiline.additional`                                                 | List of additional conditions and expressions to match first line of multiline logs. See [Multiline](https://help.sumologic.com/docs/send-data/kubernetes/collecting-logs/#conditional-multiline-log-parsing) for more information.                                                                        | `[]`                                                                                                                                                                                 |
 | `sumologic.logs.systemd.enabled`                                                      | Enable collecting systemd logs from Kubernets nodes.                                                                                                                                                                                                                                                       | `true`                                                                                                                                                                               |
 | `sumologic.logs.systemd.units`                                                        | List of systemd units to collect logs from.                                                                                                                                                                                                                                                                | See [values.yaml]                                                                                                                                                                    |
+| `sumologic.logs.kubelet.enabled`                                                      | Enable collecting kubelet logs from Kubernets nodes.                                                                                                                                                                                                                                                       | `true`                                                                                                                                                                               |
 | `sumologic.logs.container.keep_time_attribute`                                        | When set to `true`, preserves the `time` attribute, which is a string representation of the `timestamp` attribute.                                                                                                                                                                                         | `false`                                                                                                                                                                              |
 | `sumologic.logs.container.sourceHost`                                                 | Set the \_sourceHost metadata field in Sumo Logic.                                                                                                                                                                                                                                                         | `""`                                                                                                                                                                                 |
 | `sumologic.logs.container.sourceName`                                                 | Set the \_sourceName metadata field in Sumo Logic.                                                                                                                                                                                                                                                         | `"%{namespace}.%{pod}.%{container}"`                                                                                                                                                 |

@@ -27,7 +27,7 @@ processors:
     send_batch_size: 1000
     timeout: 1s
 
-{{- if .Values.sumologic.logs.systemd.enabled }}
+{{- if or .Values.sumologic.logs.systemd.enabled .Values.sumologic.logs.kubelet.enabled }}
   ## copy _SYSTEMD_UNIT, SYSLOG_FACILITY, _HOSTNAME and PRIORITY from body to attributes
   ## so they can be used by metadata processors same way like for fluentd
   ## build fluent.tag attribute as `host.{_SYSTEMD_UNIT}`
@@ -59,12 +59,13 @@ receivers:
 {{ tpl (.Files.Get "conf/logs/collector/common/filelog_receiver.yaml") . | nindent 2 }}
 {{- end }}
 
-{{- if .Values.sumologic.logs.systemd.enabled }}
+{{- if or .Values.sumologic.logs.systemd.enabled .Values.sumologic.logs.kubelet.enabled }}
   journald:
     directory: /var/log/journal
     ## This is not a full equivalent of fluent-bit filtering as fluent-bit filters by `_SYSTEMD_UNIT`
     ## Here is filtering by `UNIT`
     units:
+{{- if .Values.sumologic.logs.systemd.enabled }}
 {{- if .Values.sumologic.logs.systemd.units }}
 {{ toYaml .Values.sumologic.logs.systemd.units | nindent 6 }}
 {{- else }}
@@ -91,7 +92,6 @@ receivers:
       - flanneld.service
       - format-etcd2-volume.service
       - kube-node-taint-and-uncordon.service
-      - kubelet.service
       - ldconfig.service
       - locksmithd.service
       - logrotate.service
@@ -118,6 +118,10 @@ receivers:
       - var-lib-etcd2.service
 {{- end }}
 {{- end }}
+{{- if or .Values.sumologic.logs.kubelet.enabled }}
+      - kubelet.service
+{{- end }}
+{{- end }}
 
 service:
   extensions:
@@ -137,7 +141,7 @@ service:
       receivers:
         - filelog/containers
 {{- end }}
-{{- if .Values.sumologic.logs.systemd.enabled }}
+{{- if or .Values.sumologic.logs.systemd.enabled .Values.sumologic.logs.kubelet.enabled }}
     logs/systemd:
       exporters:
         - otlphttp

@@ -32,7 +32,7 @@ exporters:
 
 {{- end }}
 
-{{ if .Values.sumologic.logs.systemd.enabled }}
+{{ if or .Values.sumologic.logs.systemd.enabled .Values.sumologic.logs.kubelet.enabled }}
   sumologic/systemd:
     endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
     log_format: json
@@ -109,7 +109,7 @@ extensions:
   pprof: {}
 
 processors:
-{{ if .Values.sumologic.logs.systemd.enabled }}
+{{ if or .Values.sumologic.logs.systemd.enabled .Values.sumologic.logs.kubelet.enabled }}
   attributes/extract_systemd_source_fields:
     actions:
       - action: extract
@@ -144,7 +144,7 @@ processors:
             value: kubelet.service
 {{ end }}
 
-{{ if .Values.sumologic.logs.systemd.enabled }}
+{{ if .Values.sumologic.logs.kubelet.enabled }}
   filter/exclude_kubelet_hostname:
     logs:
       exclude:
@@ -154,7 +154,7 @@ processors:
             value: {{ .Values.sumologic.logs.kubelet.excludeHostRegex | default "$^" | quote }}
 {{ end }}
 
-{{ if .Values.sumologic.logs.systemd.enabled }}
+{{ if .Values.sumologic.logs.kubelet.enabled }}
   filter/exclude_kubelet_priority:
     logs:
       exclude:
@@ -164,7 +164,7 @@ processors:
             value: {{ .Values.sumologic.logs.kubelet.excludePriorityRegex | default "$^" | quote }}
 {{ end }}
 
-{{ if .Values.sumologic.logs.systemd.enabled }}
+{{ if .Values.sumologic.logs.kubelet.enabled }}
   filter/exclude_kubelet_syslog:
     logs:
       exclude:
@@ -174,7 +174,7 @@ processors:
             value: {{ .Values.sumologic.logs.kubelet.excludeFacilityRegex | default "$^" | quote }}
 {{ end }}
 
-{{ if .Values.sumologic.logs.systemd.enabled }}
+{{ if .Values.sumologic.logs.kubelet.enabled }}
   filter/exclude_kubelet_unit:
     logs:
       exclude:
@@ -234,7 +234,7 @@ processors:
             value: .+
 {{ end }}
 
-{{ if .Values.sumologic.logs.systemd.enabled }}
+{{ if or .Values.sumologic.logs.systemd.enabled .Values.sumologic.logs.kubelet.enabled }}
   filter/include_fluent_tag_host:
     logs:
       include:
@@ -244,7 +244,7 @@ processors:
             value: host\..+
 {{ end }}
 
-{{ if .Values.sumologic.logs.systemd.enabled }}
+{{ if .Values.sumologic.logs.kubelet.enabled }}
   filter/include_kubelet:
     logs:
       include:
@@ -274,7 +274,7 @@ processors:
       - _collector
 {{ end }}
 
-{{ if .Values.sumologic.logs.systemd.enabled }}
+{{ if or .Values.sumologic.logs.systemd.enabled .Values.sumologic.logs.kubelet.enabled }}
   groupbyattrs/systemd:
     keys:
       - _sourceName
@@ -436,7 +436,7 @@ processors:
     source_name: {{ .Values.sumologic.logs.container.sourceName | quote }}
 {{ end }}
 
-{{ if .Values.sumologic.logs.systemd.enabled }}
+{{ if .Values.sumologic.logs.kubelet.enabled }}
   source/kubelet:
     collector: {{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }}
     source_host: "%{_sourceHost}"
@@ -558,7 +558,7 @@ service:
         - otlp
 {{ end }}
 
-{{ if .Values.sumologic.logs.systemd.enabled }}
+{{ if .Values.sumologic.logs.kubelet.enabled }}
     logs/otlp/kubelet:
       exporters:
 {{ include "logs.otelcol.exporters" (dict "Values" .Values "Type" "systemd") | nindent 8}}
@@ -589,7 +589,9 @@ service:
 {{- end }}
       receivers:
         - otlp
+{{- end }}
 
+{{ if .Values.sumologic.logs.systemd.enabled }}
     logs/otlp/systemd:
       exporters:
 {{ include "logs.otelcol.exporters" (dict "Values" .Values "Type" "systemd") | nindent 8}}

@@ -416,6 +416,7 @@ sumologic:
       excludeUnitRegex: ""
 
     kubelet:
+      enabled: true
       otelcol:
         ## Extra processors for kubelet logs. See https://help.sumologic.com/docs/send-data/kubernetes/collecting-logs/ for details.
         extraProcessors: []

@@ -57,14 +57,16 @@ metadata:
 	require.Equal(t, expected, otelConfig)
 }
 
-func TestMetadataOtelConfigSystemdDisabled(t *testing.T) {
+func TestMetadataOtelConfigSystemdAndKubeletDisabled(t *testing.T) {
 	t.Parallel()
 	templatePath := "templates/logs/otelcol/configmap.yaml"
 	valuesYaml := `
 sumologic:
   logs:
     systemd:
       enabled: false
+    kubelet:
+      enabled: false
 `
 	otelConfigYaml := GetOtelConfigYaml(t, valuesYaml, templatePath)
 
@@ -334,6 +336,8 @@ sumologic:
   logs:
     systemd:
       enabled: false
+    kubelet:
+      enabled: false
 `
 	otelConfigYaml := GetOtelConfigYaml(t, valuesYaml, templatePath)
 
@@ -430,6 +434,8 @@ sumologic:
     systemd:
       units:
         - test
+    kubelet:
+      enabled: false
 `
 	otelConfigYaml := GetOtelConfigYaml(t, valuesYaml, templatePath)
 

@@ -3,3 +3,7 @@ sumologic:
     collector:
       otelcol:
         enabled: true
+    kubelet:
+      enabled: true
+    systemd:
+      enabled: true
@@ -188,7 +188,6 @@ data:
         - flanneld.service
         - format-etcd2-volume.service
         - kube-node-taint-and-uncordon.service
-        - kubelet.service
         - ldconfig.service
         - locksmithd.service
         - logrotate.service
@@ -213,6 +212,7 @@ data:
         - update-ca-certificates.service
         - user-cloudinit.service
         - var-lib-etcd2.service
+        - kubelet.service
     service:
       extensions:
       - health_check

@@ -3,6 +3,10 @@ sumologic:
     collector:
       otelcol:
         enabled: true
+    kubelet:
+      enabled: true
+    systemd:
+      enabled: true
 debug:
   logs:
     collector:

@@ -198,7 +198,6 @@ data:
         - flanneld.service
         - format-etcd2-volume.service
         - kube-node-taint-and-uncordon.service
-        - kubelet.service
         - ldconfig.service
         - locksmithd.service
         - logrotate.service
@@ -223,6 +222,7 @@ data:
         - update-ca-certificates.service
         - user-cloudinit.service
         - var-lib-etcd2.service
+        - kubelet.service
     service:
       extensions:
       - health_check

@@ -7,6 +7,8 @@ sumologic:
       enabled: false
     systemd:
       enabled: true
+    kubelet:
+      enabled: false
     container:
       enabled: false
 

@@ -81,7 +81,6 @@ data:
         - flanneld.service
         - format-etcd2-volume.service
         - kube-node-taint-and-uncordon.service
-        - kubelet.service
         - ldconfig.service
         - locksmithd.service
         - logrotate.service

@@ -1,5 +1,7 @@
 sumologic:
   logs:
+    kubelet:
+      enabled: true
     multiline:
       enabled: true
       first_line_regex: "^\\[?\\d{4}-\\d{1,2}-\\d{1,2}.\\d{2}:\\d{2}:\\d{2}"

@@ -221,7 +221,6 @@ data:
         - flanneld.service
         - format-etcd2-volume.service
         - kube-node-taint-and-uncordon.service
-        - kubelet.service
         - ldconfig.service
         - locksmithd.service
         - logrotate.service
@@ -246,6 +245,7 @@ data:
         - update-ca-certificates.service
         - user-cloudinit.service
         - var-lib-etcd2.service
+        - kubelet.service
     service:
       extensions:
       - health_check
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		feat(otelcolLogs): Split kubelet from systemd logs with a separate feature flag