[upgrade assistant] Account for max headroom setting when showing low watermark warning

x-pack/platform/plugins/private/upgrade_assistant/__jest__/client_integration/es_deprecations/data_streams_deprecation.test.ts

@@ -121,7 +121,6 @@ describe('Data streams deprecation', () => {
           nodeId: '9OFkjpAKS_aPzJAuEOSg7w',
           nodeName: 'MacBook-Pro.local',
           available: '25%',
-          lowDiskWatermarkSetting: '50%',
         },
       ]);
 
@@ -449,7 +448,6 @@ describe('Data streams deprecation', () => {
           nodeId: '9OFkjpAKS_aPzJAuEOSg7w',
           nodeName: 'MacBook-Pro.local',
           available: '25%',
-          lowDiskWatermarkSetting: '50%',
         },
       ]);
 

x-pack/platform/plugins/private/upgrade_assistant/__jest__/client_integration/es_deprecations/readonly_index_modal.test.ts

@@ -80,7 +80,6 @@ describe('Readonly index modal', () => {
           nodeId: '9OFkjpAKS_aPzJAuEOSg7w',
           nodeName: 'MacBook-Pro.local',
           available: '25%',
-          lowDiskWatermarkSetting: '50%',
         },
       ]);
 

x-pack/platform/plugins/private/upgrade_assistant/__jest__/client_integration/es_deprecations/reindex_deprecation_flyout.test.ts

@@ -248,7 +248,6 @@ describe('Reindex deprecation flyout', () => {
           nodeId: '9OFkjpAKS_aPzJAuEOSg7w',
           nodeName: 'MacBook-Pro.local',
           available: '25%',
-          lowDiskWatermarkSetting: '50%',
         },
       ]);
 

x-pack/platform/plugins/private/upgrade_assistant/public/application/components/es_deprecations/common/nodes_low_disk_space.tsx

@@ -15,7 +15,6 @@ interface Props {
     nodeId: string;
     nodeName: string;
     available: string;
-    lowDiskWatermarkSetting: string;
   }>;
 }
 export const NodesLowSpaceCallOut: React.FunctionComponent<Props> = ({ nodes }) => {

x-pack/platform/plugins/private/upgrade_assistant/public/application/lib/api.ts

@@ -315,7 +315,6 @@ export class ApiService {
         nodeId: string;
         nodeName: string;
         available: string;
-        lowDiskWatermarkSetting: string;
       }>
     >({
       path: `${API_BASE_PATH}/node_disk_space`,

x-pack/platform/plugins/private/upgrade_assistant/server/routes/node_disk_space.test.ts

@@ -49,6 +49,12 @@ describe('Disk space API', () => {
                 total_in_bytes: 100,
                 available_in_bytes: 20,
               },
+              data: [
+                {
+                  available_in_bytes: 20,
+                  low_watermark_free_space_in_bytes: 100,
+                },
+              ],
             },
           },
         },
@@ -77,8 +83,7 @@ describe('Disk space API', () => {
         {
           nodeName: 'node_name',
           nodeId: '1YOaoS9lTNOiTxR1uzSgRA',
-          available: '20%',
-          lowDiskWatermarkSetting: '75%',
+          available: '20b',
         },
       ]);
     });
@@ -103,8 +108,7 @@ describe('Disk space API', () => {
         {
           nodeName: 'node_name',
           nodeId: '1YOaoS9lTNOiTxR1uzSgRA',
-          available: '20%',
-          lowDiskWatermarkSetting: '75%',
+          available: '20b',
         },
       ]);
     });
@@ -129,8 +133,7 @@ describe('Disk space API', () => {
         {
           nodeName: 'node_name',
           nodeId: '1YOaoS9lTNOiTxR1uzSgRA',
-          available: '20%',
-          lowDiskWatermarkSetting: '79%',
+          available: '20b',
         },
       ]);
     });
@@ -157,41 +160,44 @@ describe('Disk space API', () => {
         {
           nodeName: 'node_name',
           nodeId: '1YOaoS9lTNOiTxR1uzSgRA',
-          available: '20%',
-          lowDiskWatermarkSetting: '80b',
+          available: '20b',
         },
       ]);
     });
 
-    it('returns empty array if low watermark disk usage setting is undefined', async () => {
+    it('returns empty array if nodes have not reached low disk usage', async () => {
       (
         routeHandlerContextMock.core.elasticsearch.client.asCurrentUser.cluster
           .getSettings as jest.Mock
       ).mockResolvedValue({
-        defaults: {},
+        defaults: {
+          'cluster.routing.allocation.disk.watermark.low': '85%',
+        },
         transient: {},
         persistent: {},
       });
 
-      const resp = await routeDependencies.router.getHandler({
-        method: 'get',
-        pathPattern: '/api/upgrade_assistant/node_disk_space',
-      })(routeHandlerContextMock, createRequestMock(), kibanaResponseFactory);
-
-      expect(resp.status).toEqual(200);
-      expect(resp.payload).toEqual([]);
-    });
-
-    it('returns empty array if nodes have not reached low disk usage', async () => {
       (
-        routeHandlerContextMock.core.elasticsearch.client.asCurrentUser.cluster
-          .getSettings as jest.Mock
+        routeHandlerContextMock.core.elasticsearch.client.asCurrentUser.nodes.stats as jest.Mock
       ).mockResolvedValue({
-        defaults: {
-          'cluster.routing.allocation.disk.watermark.low': '85%',
+        nodes: {
+          '1YOaoS9lTNOiTxR1uzSgRA': {
+            name: 'node_name',
+            fs: {
+              total: {
+                // Keeping these numbers (inaccurately) small so it's easier to reason the math when scanning through :)
+                total_in_bytes: 100,
+                available_in_bytes: 120,
+              },
+              data: [
+                {
+                  available_in_bytes: 120,
+                  low_watermark_free_space_in_bytes: 100,
+                },
+              ],
+            },
+          },
         },
-        transient: {},
-        persistent: {},
       });
 
       const resp = await routeDependencies.router.getHandler({
@@ -204,19 +210,6 @@ describe('Disk space API', () => {
     });
 
     describe('Error handling', () => {
-      it('returns an error if cluster.getSettings throws', async () => {
-        (
-          routeHandlerContextMock.core.elasticsearch.client.asCurrentUser.cluster
-            .getSettings as jest.Mock
-        ).mockRejectedValue(new Error('scary error!'));
-        await expect(
-          routeDependencies.router.getHandler({
-            method: 'get',
-            pathPattern: '/api/upgrade_assistant/node_disk_space',
-          })(routeHandlerContextMock, createRequestMock(), kibanaResponseFactory)
-        ).rejects.toThrow('scary error!');
-      });
-
       it('returns an error if node.stats throws', async () => {
         (
           routeHandlerContextMock.core.elasticsearch.client.asCurrentUser.cluster

x-pack/platform/plugins/private/upgrade_assistant/server/routes/node_disk_space.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { ClusterGetSettingsResponse } from '@elastic/elasticsearch/lib/api/types';
+import type { NodesStatsResponseBase } from '@elastic/elasticsearch/lib/api/types';
 import { ByteSizeValue } from '@kbn/config-schema';
 import { versionCheckHandlerWrapper } from '@kbn/upgrade-assistant-pkg-server';
 import { API_BASE_PATH } from '../../common/constants';
@@ -15,33 +15,54 @@ interface NodeWithLowDiskSpace {
   nodeId: string;
   nodeName: string;
   available: string;
-  lowDiskWatermarkSetting: string;
 }
 
-const getLowDiskWatermarkSetting = (
-  clusterSettings: ClusterGetSettingsResponse
-): string | undefined => {
-  const { defaults, persistent, transient } = clusterSettings;
+interface NodeWithLowDiskSpace {
+  nodeId: string;
+  nodeName: string;
+  available: string;
+}
+
+export function getNodesWithLowDiskSpace(
+  nodeStats: NodesStatsResponseBase
+): NodeWithLowDiskSpace[] {
+  const nodeIds = Object.keys(nodeStats.nodes);
+
+  const nodesWithLowDiskSpace: NodeWithLowDiskSpace[] = [];
 
-  const defaultLowDiskWatermarkSetting =
-    defaults && defaults['cluster.routing.allocation.disk.watermark.low'];
-  const transientLowDiskWatermarkSetting =
-    transient && transient['cluster.routing.allocation.disk.watermark.low'];
-  const persistentLowDiskWatermarkSetting =
-    persistent && persistent['cluster.routing.allocation.disk.watermark.low'];
+  nodeIds.forEach((nodeId) => {
+    const node = nodeStats.nodes[nodeId];
+
+    node?.fs?.data?.forEach((dataPath) => {
+      // @ts-expect-error low_watermark_free_space_in_bytes is missing from the types
+      const lowWatermark = dataPath.low_watermark_free_space_in_bytes;
+      const bytesAvailable = dataPath.available_in_bytes;
+      const fsWithLowDiskSpace = [];
+
+      if (lowWatermark && bytesAvailable && bytesAvailable < lowWatermark) {
+        fsWithLowDiskSpace.push({
+          nodeId,
+          nodeName: node.name || nodeId,
+          available: new ByteSizeValue(bytesAvailable).toString(),
+        });
+      }
+
+      if (fsWithLowDiskSpace.length > 0) {
+        // Having multiple data paths on a single node is deprecated in ES and considered rare
+        // If multiple data paths are above the low watermark, pick the one with the lowest available space
+        fsWithLowDiskSpace.sort((a, b) => {
+          const aBytes = ByteSizeValue.parse(a.available).getValueInBytes();
+          const bBytes = ByteSizeValue.parse(b.available).getValueInBytes();
+          return aBytes - bBytes;
+        });
 
-  // ES applies cluster settings in the following order of precendence: transient, persistent, default
-  if (transientLowDiskWatermarkSetting) {
-    return transientLowDiskWatermarkSetting;
-  } else if (persistentLowDiskWatermarkSetting) {
-    return persistentLowDiskWatermarkSetting;
-  } else if (defaultLowDiskWatermarkSetting) {
-    return defaultLowDiskWatermarkSetting;
-  }
+        nodesWithLowDiskSpace.push(fsWithLowDiskSpace[0]);
+      }
+    });
+  });
 
-  // May be undefined if defined in elasticsearch.yml
-  return undefined;
-};
+  return nodesWithLowDiskSpace;
+}
 
 export function registerNodeDiskSpaceRoute({
   router,
@@ -64,77 +85,14 @@ export function registerNodeDiskSpaceRoute({
         const {
           elasticsearch: { client },
         } = await core;
-        const clusterSettings = await client.asCurrentUser.cluster.getSettings({
-          flat_settings: true,
-          include_defaults: true,
-        });
-
-        const lowDiskWatermarkSetting = getLowDiskWatermarkSetting(clusterSettings);
-
-        if (lowDiskWatermarkSetting) {
-          const nodeStats = await client.asCurrentUser.nodes.stats({
-            metric: 'fs',
-          });
-
-          const nodeIds = Object.keys(nodeStats.nodes);
-
-          const nodesWithLowDiskSpace: NodeWithLowDiskSpace[] = [];
 
-          nodeIds.forEach((nodeId) => {
-            const node = nodeStats.nodes[nodeId];
-            const byteStats = node?.fs?.total;
-            // @ts-expect-error @elastic/elasticsearch not supported
-            const { total_in_bytes: totalInBytes, available_in_bytes: availableInBytes } =
-              byteStats;
-
-            // Regex to determine if the low disk watermark setting is configured as a percentage value
-            // Elasticsearch accepts a percentage or bytes value
-            const isLowDiskWatermarkPercentage = /^(\d+|(\.\d+))(\.\d+)?%$/.test(
-              lowDiskWatermarkSetting!
-            );
-
-            if (isLowDiskWatermarkPercentage) {
-              const percentageAvailable = (availableInBytes / totalInBytes) * 100;
-              const rawLowDiskWatermarkPercentageValue = Number(
-                lowDiskWatermarkSetting!.replace('%', '')
-              );
-              // ES interprets this setting as a threshold for used disk space; we want free disk space
-              const freeDiskSpaceAllocated = 100 - rawLowDiskWatermarkPercentageValue;
-
-              if (percentageAvailable < freeDiskSpaceAllocated) {
-                nodesWithLowDiskSpace.push({
-                  nodeId,
-                  nodeName: node.name || nodeId,
-                  available: `${Math.round(percentageAvailable)}%`,
-                  lowDiskWatermarkSetting: lowDiskWatermarkSetting!,
-                });
-              }
-            } else {
-              // If not a percentage value, assume user configured low disk watermark setting in bytes
-              const rawLowDiskWatermarkBytesValue = ByteSizeValue.parse(
-                lowDiskWatermarkSetting!
-              ).getValueInBytes();
-
-              const percentageAvailable = (availableInBytes / totalInBytes) * 100;
-
-              // If bytes available < the low disk watermarket setting, mark node as having low disk space
-              if (availableInBytes < rawLowDiskWatermarkBytesValue) {
-                nodesWithLowDiskSpace.push({
-                  nodeId,
-                  nodeName: node.name || nodeId,
-                  available: `${Math.round(percentageAvailable)}%`,
-                  lowDiskWatermarkSetting: lowDiskWatermarkSetting!,
-                });
-              }
-            }
-          });
+        const nodeStats = await client.asCurrentUser.nodes.stats({
+          metric: 'fs',
+        });
 
-          return response.ok({ body: nodesWithLowDiskSpace });
-        }
+        const nodesWithLowDiskSpace = getNodesWithLowDiskSpace(nodeStats);
 
-        // If the low disk watermark setting is undefined, send empty array
-        // This could occur if the setting is configured in elasticsearch.yml
-        return response.ok({ body: [] });
+        return response.ok({ body: nodesWithLowDiskSpace });
       } catch (error) {
         return handleEsError({ error, response });
       }