Bug fix: the filter of a data stream alias is not always properly removed.

docs/changelog/139679.yaml

@@ -0,0 +1,5 @@
+pr: 139679
+summary: "Bug fix: the filter of a data stream alias is not always properly removed"
+area: Data streams
+type: bug
+issues: []

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/140_data_stream_aliases.yml

@@ -392,3 +392,78 @@
   - match: { action_results.1.status: 404 }
   - match: { action_results.1.action: { 'type': 'remove', 'indices': ['log-test-1', 'log-test-2'], 'aliases': ['test_non_existing'] } }
   - match: { action_results.1.error.type: aliases_not_found_exception }
+
+---
+"Ensure filtered data stream alias is properly removed":
+  - requires:
+      reason: "Bug was fixed in 9.3"
+      test_runner_features: [ "allowed_warnings", "capabilities" ]
+      capabilities:
+        - method: POST
+          path: /_aliases
+          capabilities: [ fix_filtered_data_stream_alias_removal ]
+  - do:
+      allowed_warnings:
+        - "index template [my-template] has index patterns [log-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-template
+        body:
+          index_patterns: [ log-* ]
+          template:
+            settings:
+              index.number_of_replicas: 0
+          data_stream: { }
+
+  - do:
+      indices.create_data_stream:
+        name: log-test-1
+  - is_true: acknowledged
+  - do:
+      indices.create_data_stream:
+        name: log-test-2
+  - is_true: acknowledged
+
+  - do:
+      indices.update_aliases:
+        body:
+          actions:
+            - add:
+                index: log-test-1
+                aliases: filtered-alias
+                filter:
+                  term:
+                    env: production
+            - add:
+                index: log-test-2
+                aliases: filtered-alias
+                filter:
+                  term:
+                    env: production
+  - is_true: acknowledged
+  - is_false: errors
+
+  - do:
+      indices.update_aliases:
+        body:
+          actions:
+            - remove:
+                index: log-test-1
+                aliases: filtered-alias
+  - is_true: acknowledged
+  - is_false: errors
+
+  - do:
+      indices.update_aliases:
+        body:
+          actions:
+            - add:
+                index: log-test-1
+                aliases: filtered-alias
+  - is_true: acknowledged
+  - is_false: errors
+
+  - do:
+      indices.get_alias: { }
+
+  - is_false: log-test-1.aliases.filtered-alias.filter
+  - is_true: log-test-2.aliases.filtered-alias.filter

server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAlias.java

@@ -240,7 +240,8 @@ public DataStreamAlias update(String dataStream, Boolean isWriteDataStream, Map<
                 filterUpdated = filterAsMap.equals(decompress(previousFilter)) == false;
             }
         } else {
-            filterUpdated = false;
+            // If the data stream alias contains an orphaned filter, we want to reset it. Otherwise, there the filter is preserved
+            filterUpdated = hasOrphanedFilter(dataStream);
         }
 
         Set<String> dataStreams = new HashSet<>(this.dataStreams);
@@ -249,13 +250,20 @@ public DataStreamAlias update(String dataStream, Boolean isWriteDataStream, Map<
             Map<String, CompressedXContent> newDataStreamToFilterMap = new HashMap<>(dataStreamToFilterMap);
             if (filterAsMap != null) {
                 newDataStreamToFilterMap.put(dataStream, compress(filterAsMap));
+            } else if (filterUpdated) {
+                // This is removing orphaned alias filters
+                newDataStreamToFilterMap.remove(dataStream);
             }
             return new DataStreamAlias(name, List.copyOf(dataStreams), newDataStreamToFilterMap, writeDataStream);
         } else {
             return this;
         }
     }
 
+    private boolean hasOrphanedFilter(String dataStream) {
+        return dataStreamToFilterMap.containsKey(dataStream) && dataStreams.contains(dataStream) == false;
+    }
+
     /**
      * Returns a {@link DataStreamAlias} instance based on this instance but with the specified data stream no longer referenced.
      * Returns <code>null</code> if because of the removal of the provided data stream name a new instance wouldn't reference to
@@ -264,7 +272,8 @@ public DataStreamAlias update(String dataStream, Boolean isWriteDataStream, Map<
     public DataStreamAlias removeDataStream(String dataStream) {
         Set<String> dataStreams = new HashSet<>(this.dataStreams);
         boolean removed = dataStreams.remove(dataStream);
-        if (removed == false) {
+        // This is removing orphaned alias filters
+        if (removed == false && dataStreamToFilterMap.containsKey(dataStream) == false) {
             return this;
         }
 
@@ -275,7 +284,12 @@ public DataStreamAlias removeDataStream(String dataStream) {
             if (dataStream.equals(writeDataStream)) {
                 writeDataStream = null;
             }
-            return new DataStreamAlias(name, List.copyOf(dataStreams), dataStreamToFilterMap, writeDataStream);
+            Map<String, CompressedXContent> updatedDataStreamMap = dataStreamToFilterMap;
+            if (dataStreamToFilterMap.containsKey(dataStream)) {
+                updatedDataStreamMap = new HashMap<>(dataStreamToFilterMap);
+                updatedDataStreamMap.remove(dataStream);
+            }
+            return new DataStreamAlias(name, List.copyOf(dataStreams), updatedDataStreamMap, writeDataStream);
         }
     }
 

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesAliasesAction.java

@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestUtils.getAckTimeout;
@@ -28,6 +29,8 @@
 @ServerlessScope(Scope.PUBLIC)
 public class RestIndicesAliasesAction extends BaseRestHandler {
 
+    private static final Set<String> CAPABILITIES = Set.of("fix_filtered_data_stream_alias_removal");
+
     @Override
     public String getName() {
         return "indices_aliases_action";
@@ -49,4 +52,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         }
         return channel -> client.admin().indices().aliases(indicesAliasesRequest, new RestToXContentListener<>(channel));
     }
+
+    @Override
+    public Set<String> supportedCapabilities() {
+        return CAPABILITIES;
+    }
 }

server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamAliasTests.java

@@ -174,6 +174,29 @@ public void testRemoveDataStream() {
         alias = new DataStreamAlias("my-alias", List.of("ds-1"), null, null);
         result = alias.removeDataStream("ds-2");
         assertThat(result, sameInstance(alias));
+        // Remove a filtered data stream
+        alias = new DataStreamAlias("my-alias", List.of("ds-1", "ds-2"), null, Map.of("ds-2", Map.of("term", Map.of("field", "value"))));
+        result = alias.removeDataStream("ds-2");
+        assertThat(result, not(sameInstance(alias)));
+        assertThat(result.getDataStreams(), containsInAnyOrder("ds-1"));
+        assertThat(result.getFilter("ds-2"), nullValue());
+    }
+
+    public void testRemovalOfOrphanedFilters() {
+        DataStreamAlias alias = new DataStreamAlias(
+            "my-alias",
+            List.of("ds-1", "ds-2"),
+            null,
+            Map.of("unknown", Map.of("term", Map.of("field", "value")))
+        );
+        DataStreamAlias result = alias.removeDataStream("unknown");
+        assertThat(result, not(sameInstance(alias)));
+        assertThat(result.getDataStreams(), containsInAnyOrder("ds-1", "ds-2"));
+        assertThat(result.getFilter("unknown"), nullValue());
+        result = alias.update("unknown", false, null);
+        assertThat(result, not(sameInstance(alias)));
+        assertThat(result.getDataStreams(), containsInAnyOrder("ds-1", "ds-2", "unknown"));
+        assertThat(result.getFilter("unknown"), nullValue());
     }
 
     public void testIntersect() {