Properly drop log.file.path from the log context

charts/k8s-monitoring/charts/feature-pod-logs/templates/_filelog.alloy.tpl

@@ -120,7 +120,6 @@ otelcol.processor.transform "pod_logs" {
     context = "resource"
     statements = [
       `delete_key(attributes, "k8s.container.restart_count")`,
-      `delete_key(attributes, "log.file.path")`,
 
       `set(attributes["service.name"], attributes["app.kubernetes.io/name"]) where attributes["service.name"] == nil`,
       `set(attributes["service.name"], attributes["k8s.deployment.name"]) where attributes["service.name"] == nil`,
@@ -143,6 +142,13 @@ otelcol.processor.transform "pod_logs" {
     ]
   }
 
+  log_statements {
+    context = "log"
+    statements = [
+      `delete_key(attributes, "log.file.path")`,
+    ]
+  }
+
   output {
     logs = [otelcol.exporter.loki.pod_logs.input]
   }

charts/k8s-monitoring/charts/feature-pod-logs/tests/__snapshot__/filelog_test.yaml.snap

@@ -0,0 +1,172 @@
+should render using the filelog receiver:
+  1: |
+    |-
+      declare "pod_logs" {
+        argument "logs_destinations" {
+          comment = "Must be a list of log destinations where collected logs should be forwarded to"
+        }
+
+        otelcol.receiver.filelog "pod_logs" {
+          include = ["/var/log/pods/*/*/*.log"]
+          start_at = "beginning"
+          include_file_name = false
+          include_file_path = true
+
+          operators = [
+            // Container operator will set k8s.pod.name, k8s.pod.uid, k8s.container.name, k8s.container.restart_count, and k8s.namespace.name
+            {
+              type                       = "container",
+              add_metadata_from_filepath = true,
+            },
+          ]
+
+          output {
+            logs = [otelcol.processor.k8sattributes.pod_logs.input]
+          }
+        }
+
+        otelcol.processor.k8sattributes "pod_logs" {
+          pod_association {
+            source {
+              from = "resource_attribute"
+              name = "k8s.pod.uid"
+            }
+          }
+
+          extract {
+            metadata = [
+              "k8s.deployment.name",
+              "k8s.statefulset.name",
+              "k8s.daemonset.name",
+              "k8s.cronjob.name",
+              "k8s.job.name",
+              "k8s.node.name",
+            ]
+            annotation {
+              key_regex = "(.*)"
+              tag_name  = "$1"
+              from      = "pod"
+            }
+            annotation {
+              key_regex = "resource.opentelemetry.io/(.*)"
+              tag_name  = "$1"
+              from      = "pod"
+            }
+            annotation {
+              tag_name = "job"
+              key      = "k8s.grafana.com/logs.job"
+              from     = "pod"
+            }
+            label {
+              key_regex = "(.*)"
+              tag_name  = "$1"
+              from      = "pod"
+            }
+            label {
+              tag_name = "app_kubernetes_io_name"
+              key      = "app.kubernetes.io/name"
+              from     = "pod"
+            }
+          }
+
+          output {
+            logs = [otelcol.processor.transform.pod_logs.input]
+          }
+        }
+
+        otelcol.processor.transform "pod_logs" {
+          error_mode = "ignore"
+          log_statements {
+            context = "resource"
+            statements = [
+              `delete_key(attributes, "k8s.container.restart_count")`,
+
+              `set(attributes["service.name"], attributes["app.kubernetes.io/name"]) where attributes["service.name"] == nil`,
+              `set(attributes["service.name"], attributes["k8s.deployment.name"]) where attributes["service.name"] == nil`,
+              `set(attributes["service.name"], attributes["k8s.replicaset.name"]) where attributes["service.name"] == nil`,
+              `set(attributes["service.name"], attributes["k8s.statefulset.name"]) where attributes["service.name"] == nil`,
+              `set(attributes["service.name"], attributes["k8s.daemonset.name"]) where attributes["service.name"] == nil`,
+              `set(attributes["service.name"], attributes["k8s.cronjob.name"]) where attributes["service.name"] == nil`,
+              `set(attributes["service.name"], attributes["k8s.job.name"]) where attributes["service.name"] == nil`,
+              `set(attributes["service.name"], attributes["k8s.pod.name"]) where attributes["service.name"] == nil`,
+              `set(attributes["service.name"], attributes["k8s.container.name"]) where attributes["service.name"] == nil`,
+
+              `set(attributes["service.namespace"], attributes["k8s.namespace.name"]) where attributes["service.namespace"] == nil`,
+
+              `set(attributes["service.version"], attributes["app.kubernetes.io/version"]) where attributes["service.version"] == nil`,
+
+              `set(attributes["service.instance.id"], Concat([attributes["k8s.namespace.name"], attributes["k8s.pod.name"], attributes["k8s.container.name"]], ".")) where attributes["service.instance.id"] == nil`,
+
+              `set(attributes["loki.resource.labels"], "app.kubernetes.io/name,container,instance,job,level,namespace,service.name,service.namespace,deployment.environment,deployment.environment.name,k8s.namespace.name,k8s.deployment.name,k8s.statefulset.name,k8s.daemonset.name,k8s.cronjob.name,k8s.job.name,k8s.node.name")`,   // Used to preserve the labels when converting to Loki
+              `keep_matching_keys(attributes, "loki.resource.labels|app.kubernetes.io/name|container|instance|job|level|namespace|service.name|service.namespace|deployment.environment|deployment.environment.name|k8s.namespace.name|k8s.deployment.name|k8s.statefulset.name|k8s.daemonset.name|k8s.cronjob.name|k8s.job.name|k8s.node.name")`,
+            ]
+          }
+
+          log_statements {
+            context = "log"
+            statements = [
+              `delete_key(attributes, "log.file.path")`,
+            ]
+          }
+
+          output {
+            logs = [otelcol.exporter.loki.pod_logs.input]
+          }
+        }
+
+        otelcol.exporter.loki "pod_logs" {
+          forward_to = [loki.process.pod_logs.receiver]
+        }
+
+        loki.process "pod_logs" {
+          stage.match {
+            selector = "{tmp_container_runtime=~\"containerd|cri-o\"}"
+            // the cri processing stage extracts the following k/v pairs: log, stream, time, flags
+            stage.cri {}
+
+            // Set the extract flags and stream values as labels
+            stage.labels {
+              values = {
+                flags  = "",
+                stream  = "",
+              }
+            }
+          }
+
+          stage.match {
+            selector = "{tmp_container_runtime=\"docker\"}"
+            // the docker processing stage extracts the following k/v pairs: log, stream, time
+            stage.docker {}
+
+            // Set the extract stream value as a label
+            stage.labels {
+              values = {
+                stream  = "",
+              }
+            }
+          }
+
+          // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster,
+          // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary
+          // container runtime label as it is no longer needed.
+          stage.label_drop {
+            values = [
+              "filename",
+              "tmp_container_runtime",
+            ]
+          }
+          stage.structured_metadata {
+            values = {
+              "k8s_pod_name" = "k8s_pod_name",
+              "pod" = "pod",
+            }
+          }
+
+          // Only keep the labels that are defined in the `keepLabels` list.
+          stage.label_keep {
+            values = ["app_kubernetes_io_name","container","instance","job","level","namespace","service_name","service_namespace","deployment_environment","deployment_environment_name","k8s_namespace_name","k8s_deployment_name","k8s_statefulset_name","k8s_daemonset_name","k8s_cronjob_name","k8s_job_name","k8s_node_name"]
+          }
+
+          forward_to = argument.logs_destinations.value
+        }
+      }

charts/k8s-monitoring/charts/feature-pod-logs/tests/__snapshot__/kubernetesApi_test.yaml.snap

@@ -0,0 +1,158 @@
+should render using the kubernetesApi receiver:
+  1: |
+    |-
+      declare "pod_logs" {
+        argument "logs_destinations" {
+          comment = "Must be a list of log destinations where collected logs should be forwarded to"
+        }
+
+        discovery.relabel "filtered_pods" {
+          targets = discovery.kubernetes.pods.targets
+          rule {
+            source_labels = ["__meta_kubernetes_namespace"]
+            action = "replace"
+            target_label = "namespace"
+          }
+          rule {
+            source_labels = ["__meta_kubernetes_pod_name"]
+            action = "replace"
+            target_label = "pod"
+          }
+          rule {
+            source_labels = ["__meta_kubernetes_pod_container_name"]
+            action = "replace"
+            target_label = "container"
+          }
+          rule {
+            source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"]
+            separator = "/"
+            action = "replace"
+            replacement = "$1"
+            target_label = "job"
+          }
+
+          // set the container runtime as a label
+          rule {
+            action = "replace"
+            source_labels = ["__meta_kubernetes_pod_container_id"]
+            regex = "^(\\S+):\\/\\/.+$"
+            replacement = "$1"
+            target_label = "tmp_container_runtime"
+          }
+
+          // make all labels on the pod available to the pipeline as labels,
+          // they are omitted before write to loki via stage.label_keep unless explicitly set
+          rule {
+            action = "labelmap"
+            regex = "__meta_kubernetes_pod_label_(.+)"
+          }
+
+          // make all annotations on the pod available to the pipeline as labels,
+          // they are omitted before write to loki via stage.label_keep unless explicitly set
+          rule {
+            action = "labelmap"
+            regex = "__meta_kubernetes_pod_annotation_(.+)"
+          }
+
+          // explicitly set service_name. if not set, loki will automatically try to populate a default.
+          // see https://grafana.com/docs/loki/latest/get-started/labels/#default-labels-for-all-users
+          //
+          // choose the first value found from the following ordered list:
+          // - pod.annotation[resource.opentelemetry.io/service.name]
+          // - pod.label[app.kubernetes.io/name]
+          // - k8s.pod.name
+          // - k8s.container.name
+          rule {
+            action = "replace"
+            source_labels = [
+              "__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_name",
+              "__meta_kubernetes_pod_label_app_kubernetes_io_name",
+              "__meta_kubernetes_pod_container_name",
+            ]
+            separator = ";"
+            regex = "^(?:;*)?([^;]+).*$"
+            replacement = "$1"
+            target_label = "service_name"
+          }
+
+          // set resource attributes
+          rule {
+            action = "labelmap"
+            regex = "__meta_kubernetes_pod_annotation_resource_opentelemetry_io_(.+)"
+          }
+          rule {
+            source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_logs_job"]
+            regex = "(.+)"
+            target_label = "job"
+          }
+          rule {
+            source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
+            regex = "(.+)"
+            target_label = "app_kubernetes_io_name"
+          }
+        }
+
+        discovery.kubernetes "pods" {
+          role = "pod"
+        }
+
+        loki.source.kubernetes "pod_logs" {
+          targets = discovery.relabel.filtered_pods.output
+          clustering {
+            enabled = true
+          }
+          forward_to = [loki.process.pod_logs.receiver]
+        }
+
+        loki.process "pod_logs" {
+          stage.match {
+            selector = "{tmp_container_runtime=~\"containerd|cri-o\"}"
+            // the cri processing stage extracts the following k/v pairs: log, stream, time, flags
+            stage.cri {}
+
+            // Set the extract flags and stream values as labels
+            stage.labels {
+              values = {
+                flags  = "",
+                stream  = "",
+              }
+            }
+          }
+
+          stage.match {
+            selector = "{tmp_container_runtime=\"docker\"}"
+            // the docker processing stage extracts the following k/v pairs: log, stream, time
+            stage.docker {}
+
+            // Set the extract stream value as a label
+            stage.labels {
+              values = {
+                stream  = "",
+              }
+            }
+          }
+
+          // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster,
+          // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary
+          // container runtime label as it is no longer needed.
+          stage.label_drop {
+            values = [
+              "filename",
+              "tmp_container_runtime",
+            ]
+          }
+          stage.structured_metadata {
+            values = {
+              "k8s_pod_name" = "k8s_pod_name",
+              "pod" = "pod",
+            }
+          }
+
+          // Only keep the labels that are defined in the `keepLabels` list.
+          stage.label_keep {
+            values = ["app_kubernetes_io_name","container","instance","job","level","namespace","service_name","service_namespace","deployment_environment","deployment_environment_name","k8s_namespace_name","k8s_deployment_name","k8s_statefulset_name","k8s_daemonset_name","k8s_cronjob_name","k8s_job_name","k8s_node_name"]
+          }
+
+          forward_to = argument.logs_destinations.value
+        }
+      }

charts/k8s-monitoring/charts/feature-pod-logs/tests/filelog_test.yaml

@@ -0,0 +1,14 @@
+# yamllint disable rule:document-start rule:line-length rule:trailing-spaces rule:empty-lines
+suite: Test - Pod Logs - Gather Method - OpenTelemetry Filelog Receiver
+templates:
+  - configmap.yaml
+tests:
+  - it: should render using the filelog receiver
+    set:
+      deployAsConfigMap: true
+      gatherMethod: filelog
+    asserts:
+      - isKind:
+          of: ConfigMap
+      - matchSnapshot:
+          path: data["module.alloy"]

charts/k8s-monitoring/charts/feature-pod-logs/tests/kubernetesApi_test.yaml

@@ -0,0 +1,14 @@
+# yamllint disable rule:document-start rule:line-length rule:trailing-spaces rule:empty-lines
+suite: Test - Pod Logs - Gather Method - Kubernetes API
+templates:
+  - configmap.yaml
+tests:
+  - it: should render using the kubernetesApi receiver
+    set:
+      deployAsConfigMap: true
+      gatherMethod: kubernetesApi
+    asserts:
+      - isKind:
+          of: ConfigMap
+      - matchSnapshot:
+          path: data["module.alloy"]