Add processors setting for k8s data_streams

packages/kubernetes/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.26.0"
+  changes:
+    - description: Add `processors` configuration option for Kubernetes data_streams
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/4363
 - version: "1.25.0"
   changes:
     - description: Add `condition` configuration option to container logs data stream

packages/kubernetes/data_stream/apiserver/agent/stream/stream.yml.hbs

@@ -16,3 +16,8 @@ ssl.certificate_authorities:
   - {{this}}
 {{/each}}
 {{/if}}
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/kubernetes/data_stream/apiserver/manifest.yml

@@ -40,5 +40,13 @@ streams:
         show_user: true
         default:
           - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
     title: Kubernetes API Server metrics
     description: Collect Kubernetes API Server metrics

packages/kubernetes/data_stream/audit_logs/agent/stream/stream.yml.hbs

@@ -38,3 +38,6 @@ processors:
             throw "expected kubernetes.audit.annotations.authorization_k8s_io/decision === allow";
         }
       }
+{{#if processors}}
+{{processors}}
+{{/if}}

packages/kubernetes/data_stream/audit_logs/manifest.yml

@@ -18,3 +18,11 @@ streams:
         multi: true
         default:
           - /var/log/kubernetes/kube-apiserver-audit.log
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.

packages/kubernetes/data_stream/container/agent/stream/stream.yml.hbs

@@ -18,3 +18,8 @@ ssl.certificate_authorities:
   - {{this}}
 {{/each}}
 {{/if}}
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/kubernetes/data_stream/container/manifest.yml

@@ -58,5 +58,13 @@ streams:
         multi: true
         required: false
         show_user: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
     title: Kubernetes Container metrics
     description: Collect Kubernetes Container metrics

packages/kubernetes/data_stream/container_logs/agent/stream/stream.yml.hbs

@@ -4,11 +4,13 @@ paths:
   - {{this}}
 {{/each}}
 prospector.scanner.symlinks: {{ symlinks }}
-{{#if condition}}
-condition: {{ condition }}
-{{/if}}
 parsers:
 - container:
     stream: {{ containerParserStream }}
     format: {{ containerParserFormat }}
 {{ additionalParsersConfig }}
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/kubernetes/data_stream/container_logs/manifest.yml

@@ -52,3 +52,11 @@ streams:
           #     pattern: '^\['
           #     negate: true
           #     match: after
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.

packages/kubernetes/data_stream/controllermanager/agent/stream/stream.yml.hbs

@@ -11,4 +11,9 @@ bearer_token_file: {{bearer_token_file}}
 ssl.verification_mode: {{ssl.verification_mode}}
 {{/if}}
 
-condition: ${kubernetes.labels.{{~controller_manager_label_key~}} } == '{{controller_manager_label_value}}'
+condition: ${kubernetes.labels.{{~controller_manager_label_key~}} } == '{{controller_manager_label_value}}'
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/kubernetes/data_stream/controllermanager/manifest.yml

@@ -47,5 +47,13 @@ streams:
         required: true
         show_user: false
         default: kube-controller-manager
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
     title: Kubernetes Controller Manager metrics
     description: Collect Kubernetes Controller Manager metrics

packages/kubernetes/data_stream/event/agent/stream/stream.yml.hbs

@@ -4,4 +4,9 @@ add_metadata: {{add_metadata}}
 skip_older: {{skip_older}}
 {{#if leaderelection}}
 condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}
+
+{{#if processors}}
+processors:
+{{processors}}
 {{/if}}

packages/kubernetes/data_stream/event/manifest.yml

@@ -31,5 +31,14 @@ streams:
         required: true
         show_user: true
         default: true
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
     title: Kubernetes Event metrics
     description: Collect Kubernetes Event metrics

packages/kubernetes/data_stream/node/agent/stream/stream.yml.hbs

@@ -15,3 +15,8 @@ ssl.certificate_authorities:
   - {{this}}
 {{/each}}
 {{/if}}
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/kubernetes/data_stream/node/manifest.yml

@@ -45,5 +45,14 @@ streams:
         multi: true
         required: false
         show_user: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
     title: Kubernetes Node metrics
     description: Collect Kubernetes Node metrics

packages/kubernetes/data_stream/pod/agent/stream/stream.yml.hbs

@@ -18,3 +18,8 @@ ssl.certificate_authorities:
   - {{this}}
 {{/each}}
 {{/if}}
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/kubernetes/data_stream/pod/manifest.yml

@@ -58,5 +58,14 @@ streams:
           #     include_labels: ["nodelabel2"]
           #     include_annotations: ["nodeannotation1"]
           #   deployment: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
     title: Kubernetes Pod metrics
     description: Collect Kubernetes Pod metrics

packages/kubernetes/data_stream/proxy/agent/stream/stream.yml.hbs

@@ -4,3 +4,8 @@ hosts:
   - {{this}}
 {{/each}}
 period: {{period}}
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/kubernetes/data_stream/proxy/manifest.yml

@@ -18,5 +18,14 @@ streams:
         required: true
         show_user: true
         default: 10s
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
     title: Kubernetes Proxy metrics
     description: Collect Kubernetes Proxy metrics

packages/kubernetes/data_stream/scheduler/agent/stream/stream.yml.hbs

@@ -9,4 +9,9 @@ period: {{period}}
 bearer_token_file: {{bearer_token_file}}
 ssl.verification_mode: {{ssl.verification_mode}}
 {{/if}}
-condition: ${kubernetes.labels.{{~scheduler_label_key~}} } == '{{scheduler_label_value}}'
+condition: ${kubernetes.labels.{{~scheduler_label_key~}} } == '{{scheduler_label_value}}'
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/kubernetes/data_stream/scheduler/manifest.yml

@@ -47,5 +47,14 @@ streams:
         required: true
         show_user: false
         default: kube-scheduler
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
     title: Kubernetes Scheduler metrics
     description: Collect Kubernetes Scheduler metrics

packages/kubernetes/data_stream/state_container/agent/stream/stream.yml.hbs

@@ -26,3 +26,8 @@ ssl.certificate_authorities:
   - {{this}}
 {{/each}}
 {{/if}}
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/kubernetes/data_stream/state_container/manifest.yml

@@ -71,5 +71,14 @@ streams:
           #     include_labels: ["nodelabel2"]
           #     include_annotations: ["nodeannotation1"]
           #   deployment: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
     title: Kubernetes Container metrics
     description: Collect Kubernetes Container metrics from kube_state_metrics

packages/kubernetes/data_stream/state_cronjob/agent/stream/stream.yml.hbs

@@ -23,3 +23,8 @@ ssl.certificate_authorities:
   - {{this}}
 {{/each}}
 {{/if}}
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/kubernetes/data_stream/state_cronjob/manifest.yml

@@ -58,5 +58,14 @@ streams:
         multi: true
         required: false
         show_user: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
     title: Kubernetes Cronjob metrics
     description: Collect Kubernetes Cronjob metrics from kube_state_metrics

packages/kubernetes/data_stream/state_daemonset/agent/stream/stream.yml.hbs

@@ -23,3 +23,8 @@ ssl.certificate_authorities:
   - {{this}}
 {{/each}}
 {{/if}}
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/kubernetes/data_stream/state_daemonset/manifest.yml

@@ -59,5 +59,14 @@ streams:
         required: false
         show_user: false
         default: # /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
     title: Kubernetes Deamonset metrics
     description: Collect Kubernetes Deamonset metrics from kube_state_metrics

packages/kubernetes/data_stream/state_deployment/agent/stream/stream.yml.hbs

@@ -23,3 +23,8 @@ ssl.certificate_authorities:
   - {{this}}
 {{/each}}
 {{/if}}
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/kubernetes/data_stream/state_deployment/manifest.yml

@@ -58,5 +58,14 @@ streams:
         multi: true
         required: false
         show_user: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
     title: Kubernetes Deployment metrics
     description: Collect Kubernetes Deployment metrics from kube_state_metrics

packages/kubernetes/data_stream/state_job/agent/stream/stream.yml.hbs

@@ -23,3 +23,8 @@ ssl.certificate_authorities:
   - {{this}}
 {{/each}}
 {{/if}}
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/kubernetes/data_stream/state_job/manifest.yml

@@ -58,5 +58,14 @@ streams:
         multi: true
         required: false
         show_user: false
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
     title: Kubernetes Job metrics
     description: Collect Kubernetes Job metrics from kube_state_metrics

packages/kubernetes/data_stream/state_node/agent/stream/stream.yml.hbs

@@ -23,3 +23,8 @@ ssl.certificate_authorities:
   - {{this}}
 {{/each}}
 {{/if}}
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}