added fluentd-record-modifier plugin

charts/logging-operator/crds/logging.banzaicloud.io_clusterflows.yaml

@@ -420,6 +420,56 @@ spec:
                           type: object
                         type: array
                     type: object
+                  record_modifier:
+                    properties:
+                      char_encoding:
+                        description: Fluentd including some plugins treats logs as
+                          a BINARY by default to forward. To overide that, use a target
+                          encoding or a from:to encoding here.
+                        type: string
+                      prepare_value:
+                        description: Prepare values for filtering in configure phase.
+                          Prepared values can be used in <record>. You can write any
+                          ruby code.
+                        type: string
+                      records:
+                        description: 'Add records docs at: https://github.com/repeatedly/fluent-plugin-record-modifier
+                          Records are represented as maps: `key: value`'
+                        items:
+                          additionalProperties:
+                            type: string
+                          description: Parameters inside record directives are considered
+                            to be new key-value pairs
+                          type: object
+                        type: array
+                      remove_keys:
+                        description: A comma-delimited list of keys to delete
+                        type: string
+                      replaces:
+                        description: Replace specific value for keys
+                        items:
+                          description: Specify replace rule. This directive contains
+                            three parameters.
+                          properties:
+                            expression:
+                              description: Regular expression
+                              type: string
+                            key:
+                              description: Key to search for
+                              type: string
+                            replace:
+                              description: Value to replace with
+                              type: string
+                          required:
+                          - expression
+                          - key
+                          - replace
+                          type: object
+                        type: array
+                      whitelist_keys:
+                        description: This is exclusive with remove_keys
+                        type: string
+                    type: object
                   record_transformer:
                     properties:
                       auto_typecast:

charts/logging-operator/crds/logging.banzaicloud.io_flows.yaml

@@ -425,6 +425,56 @@ spec:
                           type: object
                         type: array
                     type: object
+                  record_modifier:
+                    properties:
+                      char_encoding:
+                        description: Fluentd including some plugins treats logs as
+                          a BINARY by default to forward. To overide that, use a target
+                          encoding or a from:to encoding here.
+                        type: string
+                      prepare_value:
+                        description: Prepare values for filtering in configure phase.
+                          Prepared values can be used in <record>. You can write any
+                          ruby code.
+                        type: string
+                      records:
+                        description: 'Add records docs at: https://github.com/repeatedly/fluent-plugin-record-modifier
+                          Records are represented as maps: `key: value`'
+                        items:
+                          additionalProperties:
+                            type: string
+                          description: Parameters inside record directives are considered
+                            to be new key-value pairs
+                          type: object
+                        type: array
+                      remove_keys:
+                        description: A comma-delimited list of keys to delete
+                        type: string
+                      replaces:
+                        description: Replace specific value for keys
+                        items:
+                          description: Specify replace rule. This directive contains
+                            three parameters.
+                          properties:
+                            expression:
+                              description: Regular expression
+                              type: string
+                            key:
+                              description: Key to search for
+                              type: string
+                            replace:
+                              description: Value to replace with
+                              type: string
+                          required:
+                          - expression
+                          - key
+                          - replace
+                          type: object
+                        type: array
+                      whitelist_keys:
+                        description: This is exclusive with remove_keys
+                        type: string
+                    type: object
                   record_transformer:
                     properties:
                       auto_typecast:

config/crd/bases/logging.banzaicloud.io_clusterflows.yaml

@@ -420,6 +420,56 @@ spec:
                           type: object
                         type: array
                     type: object
+                  record_modifier:
+                    properties:
+                      char_encoding:
+                        description: Fluentd including some plugins treats logs as
+                          a BINARY by default to forward. To overide that, use a target
+                          encoding or a from:to encoding here.
+                        type: string
+                      prepare_value:
+                        description: Prepare values for filtering in configure phase.
+                          Prepared values can be used in <record>. You can write any
+                          ruby code.
+                        type: string
+                      records:
+                        description: 'Add records docs at: https://github.com/repeatedly/fluent-plugin-record-modifier
+                          Records are represented as maps: `key: value`'
+                        items:
+                          additionalProperties:
+                            type: string
+                          description: Parameters inside record directives are considered
+                            to be new key-value pairs
+                          type: object
+                        type: array
+                      remove_keys:
+                        description: A comma-delimited list of keys to delete
+                        type: string
+                      replaces:
+                        description: Replace specific value for keys
+                        items:
+                          description: Specify replace rule. This directive contains
+                            three parameters.
+                          properties:
+                            expression:
+                              description: Regular expression
+                              type: string
+                            key:
+                              description: Key to search for
+                              type: string
+                            replace:
+                              description: Value to replace with
+                              type: string
+                          required:
+                          - expression
+                          - key
+                          - replace
+                          type: object
+                        type: array
+                      whitelist_keys:
+                        description: This is exclusive with remove_keys
+                        type: string
+                    type: object
                   record_transformer:
                     properties:
                       auto_typecast:

config/crd/bases/logging.banzaicloud.io_flows.yaml

@@ -425,6 +425,56 @@ spec:
                           type: object
                         type: array
                     type: object
+                  record_modifier:
+                    properties:
+                      char_encoding:
+                        description: Fluentd including some plugins treats logs as
+                          a BINARY by default to forward. To overide that, use a target
+                          encoding or a from:to encoding here.
+                        type: string
+                      prepare_value:
+                        description: Prepare values for filtering in configure phase.
+                          Prepared values can be used in <record>. You can write any
+                          ruby code.
+                        type: string
+                      records:
+                        description: 'Add records docs at: https://github.com/repeatedly/fluent-plugin-record-modifier
+                          Records are represented as maps: `key: value`'
+                        items:
+                          additionalProperties:
+                            type: string
+                          description: Parameters inside record directives are considered
+                            to be new key-value pairs
+                          type: object
+                        type: array
+                      remove_keys:
+                        description: A comma-delimited list of keys to delete
+                        type: string
+                      replaces:
+                        description: Replace specific value for keys
+                        items:
+                          description: Specify replace rule. This directive contains
+                            three parameters.
+                          properties:
+                            expression:
+                              description: Regular expression
+                              type: string
+                            key:
+                              description: Key to search for
+                              type: string
+                            replace:
+                              description: Value to replace with
+                              type: string
+                          required:
+                          - expression
+                          - key
+                          - replace
+                          type: object
+                        type: array
+                      whitelist_keys:
+                        description: This is exclusive with remove_keys
+                        type: string
+                    type: object
                   record_transformer:
                     properties:
                       auto_typecast:

docs/crds/v1beta1/flow_types.md

@@ -17,6 +17,7 @@
 | tag_normaliser | *filter.TagNormaliser | No | - |  |
 | dedot | *filter.DedotFilterConfig | No | - |  |
 | record_transformer | *filter.RecordTransformer | No | - |  |
+| record_modifier | *filter.RecordModifier | No | - |  |
 | geoip | *filter.GeoIP | No | - |  |
 | concat | *filter.Concat | No | - |  |
 | detectExceptions | *filter.DetectExceptions | No | - |  |

docs/plugins/Readme.md

@@ -14,6 +14,7 @@ For more information please click on the plugin name
 | **[Grep](filters/grep.md)** | filters | Grep events by the values | GA | [more info](https://docs.fluentd.org/filter/grep) |
 | **[Parser](filters/parser.md)** | filters | Parses" string field in event records and mutates its | GA | [more info](https://docs.fluentd.org/filter/parser) |
 | **[Prometheus](filters/prometheus.md)** | filters | Prometheus Filter Plugin to count Incoming Records | GA | [more info](https://github.com/fluent/fluent-plugin-prometheus#prometheus-outputfilter-plugin) |
+| **[Record Modifier](filters/record_modifier.md)** | filters | Modify each event record. | GA | [more info](https://github.com/repeatedly/fluent-plugin-record-modifier) |
 | **[Record Transformer](filters/record_transformer.md)** | filters | Mutates/transforms incoming event streams. | GA | [more info](https://docs.fluentd.org/filter/record_transformer) |
 | **[Stdout](filters/stdout.md)** | filters | Prints events to stdout | GA | [more info](https://docs.fluentd.org/filter/stdout) |
 | **[Tag Normaliser](filters/tagnormaliser.md)** | filters | Re-tag based on log metadata | GA | [more info](https://github.com/banzaicloud/fluent-plugin-tag-normaliser) |

docs/plugins/filters/record_modifier.md

@@ -0,0 +1,50 @@
+# [Record Modifier](https://github.com/repeatedly/fluent-plugin-record-modifier)
+## Overview
+ Modify each event record.
+
+## Configuration
+### RecordModifier
+| Variable Name | Type | Required | Default | Description |
+|---|---|---|---|---|
+| prepare_value | string | No | - | Prepare values for filtering in configure phase. Prepared values can be used in <record>. You can write any ruby code.<br> |
+| char_encoding | string | No | - | Fluentd including some plugins treats logs as a BINARY by default to forward. To overide that, use a target encoding or a from:to encoding here.<br> |
+| remove_keys | string | No | - | A comma-delimited list of keys to delete<br> |
+| whitelist_keys | string | No | - | This is exclusive with remove_keys<br> |
+| replaces | []Replace | No | - | Replace specific value for keys<br> |
+| records | []Record | No | - | Add records docs at: https://github.com/repeatedly/fluent-plugin-record-modifier<br>Records are represented as maps: `key: value`<br> |
+ #### Example `Record Modifier` filter configurations
+ ```yaml
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Flow
+metadata:
+  name: demo-flow
+spec:
+  filters:
+    - record_modifier:
+        records:
+        - foo: "bar"
+  selectors: {}
+  outputRefs:
+    - demo-output
+ ```
+
+ #### Fluentd Config Result
+ ```yaml
+<filter **>
+  @type record_modifier
+  @id test_record_modifier
+  <record>
+    foo bar
+  </record>
+</filter>
+ ```
+
+---
+### [Replace Directive](https://github.com/repeatedly/fluent-plugin-record-modifier#replace_keys_value)
+#### Specify replace rule. This directive contains three parameters.
+
+| Variable Name | Type | Required | Default | Description |
+|---|---|---|---|---|
+| key | string | Yes | - | Key to search for<br> |
+| expression | string | Yes | - | Regular expression<br> |
+| replace | string | Yes | - | Value to replace with<br> |

fluentd-image/v1.7/Dockerfile

@@ -49,6 +49,7 @@ RUN apk update \
          fluent-plugin-parser-logfmt \
          fluent-plugin-detect-exceptions \
          fluent-plugin-multi-format-parser \
+         fluent-plugin-record-modifier \
  && gem specific_install -l https://github.com/banzaicloud/fluent-plugin-cloudwatch-logs/releases/download/v0.7.6/fluent-plugin-cloudwatch-logs-0.7.6.gem \
  && gem specific_install -l https://github.com/banzaicloud/fluent-plugin-gcs.git \
  && apk del .build-deps $buildDeps \

pkg/sdk/api/v1beta1/flow_types.go

@@ -34,6 +34,7 @@ type Filter struct {
 	TagNormaliser     *filter.TagNormaliser      `json:"tag_normaliser,omitempty"`
 	Dedot             *filter.DedotFilterConfig  `json:"dedot,omitempty"`
 	RecordTransformer *filter.RecordTransformer  `json:"record_transformer,omitempty"`
+	RecordModifier    *filter.RecordModifier     `json:"record_modifier,omitempty"`
 	GeoIP             *filter.GeoIP              `json:"geoip,omitempty"`
 	Concat            *filter.Concat             `json:"concat,omitempty"`
 	DetectExceptions  *filter.DetectExceptions   `json:"detectExceptions,omitempty"`