feat(operator)!: Add configuration option for dropping OTLP attributes (#15857)

Author: xperimental

operator/api/loki/v1/lokistack_types.go

@@ -308,18 +308,16 @@ type OpenshiftTenantSpec struct {
 
 // OpenshiftOTLPConfig defines configuration specific to users using OTLP together with an OpenShift tenancy mode.
 type OpenshiftOTLPConfig struct {
-	// DisableRecommendedAttributes can be used to reduce the number of attributes used for stream labels and structured
-	// metadata.
+	// DisableRecommendedAttributes can be used to reduce the number of attributes used as stream labels.
 	//
 	// Enabling this setting removes the "recommended attributes" from the generated Loki configuration. This will cause
-	// meta information to not be available as stream labels or structured metadata, potentially making queries more
-	// expensive and less performant.
+	// some stream labels to disappear from the index, potentially making queries more expensive and less performant.
 	//
 	// Note that there is a set of "required attributes", needed for OpenShift Logging to work properly. Those will be
 	// added to the configuration, even if this field is set to true.
 	//
-	// This option is supposed to be combined with a custom label configuration customizing the labels for the specific
-	// usecase.
+	// This option is supposed to be combined with a custom attribute configuration listing the stream labels that
+	// should continue to exist.
 	//
 	// +optional
 	// +kubebuilder:validation:Optional
@@ -827,8 +825,13 @@ type IngestionLimitSpec struct {
 	PerStreamRateLimitBurst int32 `json:"perStreamRateLimitBurst,omitempty"`
 }
 
-// OTLPSpec defines which resource, scope and log attributes should be used as stream labels or
-// stored as structured metadata.
+// OTLPSpec defines which resource, scope and log attributes should be used as stream labels or dropped before storing.
+//
+// Attributes need to be listed by their "OpenTelemetry name" and not the representation used by Loki. Please consult
+// the documentation of the collector or application emitting the OpenTelemetry data to find out which attributes
+// are emitted.
+//
+// Attributes not listed as stream labels or to be dropped are stored as structured metadata in Loki.
 type OTLPSpec struct {
 	// StreamLabels configures which resource attributes are converted to Loki stream labels.
 	//
@@ -837,12 +840,12 @@ type OTLPSpec struct {
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Stream Labels"
 	StreamLabels *OTLPStreamLabelSpec `json:"streamLabels,omitempty"`
 
-	// StructuredMetadata configures which attributes are saved in structured metadata.
+	// Drop configures which attributes are dropped from the log entry.
 	//
 	// +optional
 	// +kubebuilder:validation:Optional
-	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Structured Metadata"
-	StructuredMetadata *OTLPMetadataSpec `json:"structuredMetadata,omitempty"`
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Dropped Attributes"
+	Drop *OTLPMetadataSpec `json:"drop,omitempty"`
 }
 
 type OTLPStreamLabelSpec struct {

operator/api/loki/v1/v1.go

@@ -90,6 +90,8 @@ var (
 	ErrOTLPTenantMissing = errors.New("if no global OTLP configuration is present which defines at least one stream label, every tenant must have an OTLP configuration")
 	// ErrOTLPTenantNoStreamLabel when a tenant is defined but has no stream labels and there also no global stream labels.
 	ErrOTLPTenantNoStreamLabel = errors.New("if no global OTLP configuration is present which defines at least one stream label, every tenant must define at least one stream label")
+	// ErrOTLPInvalidDrop when a OTLP configuration tries to drop an attribute also listed as a stream label
+	ErrOTLPInvalidDrop = errors.New("can not drop OTLP attribute also listed as a stream label")
 
 	// ErrRuleMustMatchNamespace indicates that an expression used in an alerting or recording rule is missing
 	// matchers for a namespace.

operator/api/loki/v1/zz_generated.deepcopy.go

@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: docker.io/grafana/loki-operator:0.7.1
-    createdAt: "2025-01-16T18:30:58Z"
+    createdAt: "2025-01-28T12:36:38Z"
     description: The Community Loki Operator provides Kubernetes native deployment
       and management of Loki and related logging components.
     features.operators.openshift.io/disconnected: "true"
@@ -373,62 +373,61 @@ spec:
         path: limits.global.ingestion.perStreamRateLimitBurst
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
-      - description: StreamLabels configures which resource attributes are converted
-          to Loki stream labels.
-        displayName: Stream Labels
-        path: limits.global.otlp.streamLabels
-      - description: ResourceAttributes lists the names of the resource attributes
-          that should be converted into Loki stream labels.
-        displayName: Resource Attributes
-        path: limits.global.otlp.streamLabels.resourceAttributes
-      - description: Name contains either a verbatim name of an attribute or a regular
-          expression matching many attributes.
-        displayName: Name
-        path: limits.global.otlp.streamLabels.resourceAttributes[0].name
-      - description: If Regex is true, then Name is treated as a regular expression
-          instead of as a verbatim attribute name.
-        displayName: Treat name as regular expression
-        path: limits.global.otlp.streamLabels.resourceAttributes[0].regex
-      - description: StructuredMetadata configures which attributes are saved in structured
-          metadata.
-        displayName: Structured Metadata
-        path: limits.global.otlp.structuredMetadata
+      - description: Drop configures which attributes are dropped from the log entry.
+        displayName: Dropped Attributes
+        path: limits.global.otlp.drop
       - description: LogAttributes lists the names of log attributes that should be
           included in structured metadata.
         displayName: Log Attributes
-        path: limits.global.otlp.structuredMetadata.logAttributes
+        path: limits.global.otlp.drop.logAttributes
       - description: Name contains either a verbatim name of an attribute or a regular
           expression matching many attributes.
         displayName: Name
-        path: limits.global.otlp.structuredMetadata.logAttributes[0].name
+        path: limits.global.otlp.drop.logAttributes[0].name
       - description: If Regex is true, then Name is treated as a regular expression
           instead of as a verbatim attribute name.
         displayName: Treat name as regular expression
-        path: limits.global.otlp.structuredMetadata.logAttributes[0].regex
+        path: limits.global.otlp.drop.logAttributes[0].regex
       - description: ResourceAttributes lists the names of resource attributes that
           should be included in structured metadata.
         displayName: Resource Attributes
-        path: limits.global.otlp.structuredMetadata.resourceAttributes
+        path: limits.global.otlp.drop.resourceAttributes
       - description: Name contains either a verbatim name of an attribute or a regular
           expression matching many attributes.
         displayName: Name
-        path: limits.global.otlp.structuredMetadata.resourceAttributes[0].name
+        path: limits.global.otlp.drop.resourceAttributes[0].name
       - description: If Regex is true, then Name is treated as a regular expression
           instead of as a verbatim attribute name.
         displayName: Treat name as regular expression
-        path: limits.global.otlp.structuredMetadata.resourceAttributes[0].regex
+        path: limits.global.otlp.drop.resourceAttributes[0].regex
       - description: ScopeAttributes lists the names of scope attributes that should
           be included in structured metadata.
         displayName: Scope Attributes
-        path: limits.global.otlp.structuredMetadata.scopeAttributes
+        path: limits.global.otlp.drop.scopeAttributes
       - description: Name contains either a verbatim name of an attribute or a regular
           expression matching many attributes.
         displayName: Name
-        path: limits.global.otlp.structuredMetadata.scopeAttributes[0].name
+        path: limits.global.otlp.drop.scopeAttributes[0].name
       - description: If Regex is true, then Name is treated as a regular expression
           instead of as a verbatim attribute name.
         displayName: Treat name as regular expression
-        path: limits.global.otlp.structuredMetadata.scopeAttributes[0].regex
+        path: limits.global.otlp.drop.scopeAttributes[0].regex
+      - description: StreamLabels configures which resource attributes are converted
+          to Loki stream labels.
+        displayName: Stream Labels
+        path: limits.global.otlp.streamLabels
+      - description: ResourceAttributes lists the names of the resource attributes
+          that should be converted into Loki stream labels.
+        displayName: Resource Attributes
+        path: limits.global.otlp.streamLabels.resourceAttributes
+      - description: Name contains either a verbatim name of an attribute or a regular
+          expression matching many attributes.
+        displayName: Name
+        path: limits.global.otlp.streamLabels.resourceAttributes[0].name
+      - description: If Regex is true, then Name is treated as a regular expression
+          instead of as a verbatim attribute name.
+        displayName: Treat name as regular expression
+        path: limits.global.otlp.streamLabels.resourceAttributes[0].regex
       - description: CardinalityLimit defines the cardinality limit for index queries.
         displayName: Cardinality Limit
         path: limits.global.queries.cardinalityLimit
@@ -534,62 +533,61 @@ spec:
         path: limits.tenants.ingestion.perStreamRateLimitBurst
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
-      - description: StreamLabels configures which resource attributes are converted
-          to Loki stream labels.
-        displayName: Stream Labels
-        path: limits.tenants.otlp.streamLabels
-      - description: ResourceAttributes lists the names of the resource attributes
-          that should be converted into Loki stream labels.
-        displayName: Resource Attributes
-        path: limits.tenants.otlp.streamLabels.resourceAttributes
-      - description: Name contains either a verbatim name of an attribute or a regular
-          expression matching many attributes.
-        displayName: Name
-        path: limits.tenants.otlp.streamLabels.resourceAttributes[0].name
-      - description: If Regex is true, then Name is treated as a regular expression
-          instead of as a verbatim attribute name.
-        displayName: Treat name as regular expression
-        path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex
-      - description: StructuredMetadata configures which attributes are saved in structured
-          metadata.
-        displayName: Structured Metadata
-        path: limits.tenants.otlp.structuredMetadata
+      - description: Drop configures which attributes are dropped from the log entry.
+        displayName: Dropped Attributes
+        path: limits.tenants.otlp.drop
       - description: LogAttributes lists the names of log attributes that should be
           included in structured metadata.
         displayName: Log Attributes
-        path: limits.tenants.otlp.structuredMetadata.logAttributes
+        path: limits.tenants.otlp.drop.logAttributes
       - description: Name contains either a verbatim name of an attribute or a regular
           expression matching many attributes.
         displayName: Name
-        path: limits.tenants.otlp.structuredMetadata.logAttributes[0].name
+        path: limits.tenants.otlp.drop.logAttributes[0].name
       - description: If Regex is true, then Name is treated as a regular expression
           instead of as a verbatim attribute name.
         displayName: Treat name as regular expression
-        path: limits.tenants.otlp.structuredMetadata.logAttributes[0].regex
+        path: limits.tenants.otlp.drop.logAttributes[0].regex
       - description: ResourceAttributes lists the names of resource attributes that
           should be included in structured metadata.
         displayName: Resource Attributes
-        path: limits.tenants.otlp.structuredMetadata.resourceAttributes
+        path: limits.tenants.otlp.drop.resourceAttributes
       - description: Name contains either a verbatim name of an attribute or a regular
           expression matching many attributes.
         displayName: Name
-        path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].name
+        path: limits.tenants.otlp.drop.resourceAttributes[0].name
       - description: If Regex is true, then Name is treated as a regular expression
           instead of as a verbatim attribute name.
         displayName: Treat name as regular expression
-        path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].regex
+        path: limits.tenants.otlp.drop.resourceAttributes[0].regex
       - description: ScopeAttributes lists the names of scope attributes that should
           be included in structured metadata.
         displayName: Scope Attributes
-        path: limits.tenants.otlp.structuredMetadata.scopeAttributes
+        path: limits.tenants.otlp.drop.scopeAttributes
       - description: Name contains either a verbatim name of an attribute or a regular
           expression matching many attributes.
         displayName: Name
-        path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].name
+        path: limits.tenants.otlp.drop.scopeAttributes[0].name
       - description: If Regex is true, then Name is treated as a regular expression
           instead of as a verbatim attribute name.
         displayName: Treat name as regular expression
-        path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].regex
+        path: limits.tenants.otlp.drop.scopeAttributes[0].regex
+      - description: StreamLabels configures which resource attributes are converted
+          to Loki stream labels.
+        displayName: Stream Labels
+        path: limits.tenants.otlp.streamLabels
+      - description: ResourceAttributes lists the names of the resource attributes
+          that should be converted into Loki stream labels.
+        displayName: Resource Attributes
+        path: limits.tenants.otlp.streamLabels.resourceAttributes
+      - description: Name contains either a verbatim name of an attribute or a regular
+          expression matching many attributes.
+        displayName: Name
+        path: limits.tenants.otlp.streamLabels.resourceAttributes[0].name
+      - description: If Regex is true, then Name is treated as a regular expression
+          instead of as a verbatim attribute name.
+        displayName: Treat name as regular expression
+        path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex
       - description: Blocked defines the list of rules to block matching queries.
         displayName: Blocked
         path: limits.tenants.queries.blocked
@@ -1013,21 +1011,19 @@ spec:
         displayName: OpenTelemetry Protocol
         path: tenants.openshift.otlp
       - description: |-
-          DisableRecommendedAttributes can be used to reduce the number of attributes used for stream labels and structured
-          metadata.
+          DisableRecommendedAttributes can be used to reduce the number of attributes used as stream labels.
 
 
           Enabling this setting removes the "recommended attributes" from the generated Loki configuration. This will cause
-          meta information to not be available as stream labels or structured metadata, potentially making queries more
-          expensive and less performant.
+          some stream labels to disappear from the index, potentially making queries more expensive and less performant.
 
 
           Note that there is a set of "required attributes", needed for OpenShift Logging to work properly. Those will be
           added to the configuration, even if this field is set to true.
 
 
-          This option is supposed to be combined with a custom label configuration customizing the labels for the specific
-          usecase.
+          This option is supposed to be combined with a custom attribute configuration listing the stream labels that
+          should continue to exist.
         displayName: Disable recommended OTLP attributes
         path: tenants.openshift.otlp.disableRecommendedAttributes
       statusDescriptors: