m365_defender,microsoft_defender_endpoint: switch vulnerability data stream to delta API

packages/m365_defender/_dev/benchmark/rally/vulnerability-benchmark/config.yml

@@ -52,8 +52,13 @@ fields:
   cardinality: 100000
 - name: RegistryPaths
   cardinality: 100000
-- name: SecurityUpdateAvailable
-  cardinality: 100000
+- name: EventTimestamp
+  period: -24h
+- name: Status
+  enum:
+    - New
+    - Fixed
+    - Updated
 - name: SoftwareName
   cardinality: 100000
 - name: SoftwareVendor

packages/m365_defender/_dev/benchmark/rally/vulnerability-benchmark/fields.yml

@@ -40,8 +40,10 @@
   type: keyword
 - name: RegistryPaths
   type: keyword
-- name: SecurityUpdateAvailable
-  type: boolean
+- name: EventTimestamp
+  type: date
+- name: Status
+  type: keyword
 - name: SoftwareName
   type: keyword
 - name: SoftwareVendor

packages/m365_defender/_dev/benchmark/rally/vulnerability-benchmark/template.ndjson

@@ -19,7 +19,8 @@
 {{- $RecommendedSecurityUpdateId := generate "RecommendedSecurityUpdateId" }}
 {{- $RecommendedSecurityUpdateUrl := generate "RecommendedSecurityUpdateUrl" }}
 {{- $RegistryPaths := generate "RegistryPaths" }}
-{{- $SecurityUpdateAvailable := generate "SecurityUpdateAvailable" }}
+{{- $EventTimestamp := generate "EventTimestamp" | date "2006-01-02T15:04:05.000000Z" }}
+{{- $Status := generate "Status" }}
 {{- $SoftwareName := generate "SoftwareName" }}
 {{- $SoftwareVendor := generate "SoftwareVendor" }}
 {{- $SoftwareVersion := generate "SoftwareVersion" }}
@@ -46,7 +47,8 @@
   "RecommendedSecurityUpdateId": "{{$RecommendedSecurityUpdateId}}",
   "RecommendedSecurityUpdateUrl": "{{$RecommendedSecurityUpdateUrl}}",
   "RegistryPaths": "{{$RegistryPaths}}",
-  "SecurityUpdateAvailable": "{{$SecurityUpdateAvailable}}",
+  "EventTimestamp": "{{$EventTimestamp}}",
+  "Status": "{{$Status}}",
   "SoftwareName": "{{$SoftwareName}}",
   "SoftwareVendor": "{{$SoftwareVendor}}",
   "SoftwareVersion": "{{$SoftwareVersion}}",
@@ -72,7 +74,7 @@
         "snapshot": false,
         "version": "8.18.0"
     },
-    "message": "{\"CveBatchTitle\":\"{{$CveBatchTitle}}\",\"CveBatchUrl\":\"{{$CveBatchUrl}}\",\"CveId\":\"{{$CveId}}\",\"CvssScore\":\"{{$CvssScore}}\",\"DeviceId\":\"{{$DeviceId}}\",\"DeviceName\":\"{{$DeviceName}}\",\"ExploitabilityLevel\":\"{{$ExploitabilityLevel}}\",\"Id\":\"{{$Id}}\",\"IsOnboarded\":\"{{$IsOnboarded}}\",\"OSArchitecture\":\"{{$OSArchitecture}}\",\"OSPlatform\":\"{{$OSPlatform}}\",\"OSVersion\":\"{{$OSVersion}}\",\"RbacGroupId\":\"{{$RbacGroupId}}\",\"RbacGroupName\":\"{{$RbacGroupName}}\",\"RecommendationReference\":\"{{$RecommendationReference}}\",\"RecommendedSecurityUpdate\":\"{{$RecommendedSecurityUpdate}}\",\"RecommendedSecurityUpdateId\":\"{{$RecommendedSecurityUpdateId}}\",\"RecommendedSecurityUpdateUrl\":\"{{$RecommendedSecurityUpdateUrl}}\",\"RegistryPaths\":\"{{$RegistryPaths}}\",\"SecurityUpdateAvailable\":\"{{$SecurityUpdateAvailable}}\",\"SoftwareName\":\"{{$SoftwareName}}\",\"SoftwareVendor\":\"{{$SoftwareVendor}}\",\"SoftwareVersion\":\"{{$SoftwareVersion}}\",\"VulnerabilitySeverityLevel\":\"{{$VulnerabilitySeverityLevel}}\",\"DiskPaths\":\"{{$DiskPaths}}\"}",
+    "message": "{\"CveBatchTitle\":\"{{$CveBatchTitle}}\",\"CveBatchUrl\":\"{{$CveBatchUrl}}\",\"CveId\":\"{{$CveId}}\",\"CvssScore\":\"{{$CvssScore}}\",\"DeviceId\":\"{{$DeviceId}}\",\"DeviceName\":\"{{$DeviceName}}\",\"ExploitabilityLevel\":\"{{$ExploitabilityLevel}}\",\"Id\":\"{{$Id}}\",\"IsOnboarded\":\"{{$IsOnboarded}}\",\"OSArchitecture\":\"{{$OSArchitecture}}\",\"OSPlatform\":\"{{$OSPlatform}}\",\"OSVersion\":\"{{$OSVersion}}\",\"RbacGroupId\":\"{{$RbacGroupId}}\",\"RbacGroupName\":\"{{$RbacGroupName}}\",\"RecommendationReference\":\"{{$RecommendationReference}}\",\"RecommendedSecurityUpdate\":\"{{$RecommendedSecurityUpdate}}\",\"RecommendedSecurityUpdateId\":\"{{$RecommendedSecurityUpdateId}}\",\"RecommendedSecurityUpdateUrl\":\"{{$RecommendedSecurityUpdateUrl}}\",\"RegistryPaths\":\"{{$RegistryPaths}}\",\"EventTimestamp\":\"{{$EventTimestamp}}\",\"Status\":\"{{$Status}}\",\"SoftwareName\":\"{{$SoftwareName}}\",\"SoftwareVendor\":\"{{$SoftwareVendor}}\",\"SoftwareVersion\":\"{{$SoftwareVersion}}\",\"VulnerabilitySeverityLevel\":\"{{$VulnerabilitySeverityLevel}}\",\"DiskPaths\":\"{{$DiskPaths}}\"}",
     "event": {
         "dataset": "m365_defender.vulnerability"
     },

packages/m365_defender/_dev/build/docs/README.md

@@ -19,9 +19,9 @@ This integration supports below API versions to collect data.
     - [Alerts](https://learn.microsoft.com/en-us/graph/api/security-list-alerts_v2?view=graph-rest-1.0)
     - [Incidents](https://learn.microsoft.com/en-us/graph/api/security-list-incidents?view=graph-rest-1.0)
   - [Microsoft Defender for Endpoint API v1.0](https://learn.microsoft.com/en-us/defender-endpoint/api/exposed-apis-list)
-    - [Vulnerabilities](https://learn.microsoft.com/en-us/defender-endpoint/api/get-assessment-software-vulnerabilities#2-export-software-vulnerabilities-assessment-via-files)
+    - [Vulnerabilities](https://learn.microsoft.com/en-us/defender-endpoint/api/get-assessment-software-vulnerabilities)
   - [Microsoft Defender XDR Streaming API](https://learn.microsoft.com/en-us/defender-xdr/streaming-api?view=o365-worldwide)
-    - Supported Microsoft Defender XDR Streaming event types are listed below. For more details on all available event types, refer to [documentation](https://learn.microsoft.com/en-us/defender-xdr/supported-event-types).
+    - Supported Microsoft Defender XDR Streaming event types are in the following table. For more details on all available event types, refer to [documentation](https://learn.microsoft.com/en-us/defender-xdr/supported-event-types).
 
 | Resource types | Description |
 | --- | --- |
@@ -67,7 +67,7 @@ The Microsoft Defender XDR integration collects logs for four types of events: A
 
 **Events:** This data stream uses the [Microsoft Defender XDR Streaming API](https://learn.microsoft.com/en-us/defender-xdr/streaming-api?view=o365-worldwide) to collect Alert, Device, Email, App and Identity Events. Events are streamed to an Azure Event Hub. For a list of supported events exposed by the Streaming API and supported by Elastic's integration, please refer to Microsoft's documentation [here](https://learn.microsoft.com/en-us/defender-xdr/supported-event-types?view=o365-worldwide).
 
-**Vulnerabilities:** This data stream uses the [Microsoft Defender for Endpoint API](https://learn.microsoft.com/en-us/defender-endpoint/api/exposed-apis-list)'s [`/api/machines/SoftwareVulnerabilitiesExport`](https://learn.microsoft.com/en-us/defender-endpoint/api/get-assessment-software-vulnerabilities#2-export-software-vulnerabilities-assessment-via-files) endpoint to collect vulnerability assessments.
+**Vulnerabilities:** This data stream uses the [Microsoft Defender for Endpoint API](https://learn.microsoft.com/en-us/defender-endpoint/api/exposed-apis-list)'s [`/api/machines/SoftwareVulnerabilityChangesByMachine`](https://learn.microsoft.com/en-us/defender-endpoint/api/get-assessment-software-vulnerabilities) delta endpoint to collect vulnerability change events (new, updated, and fixed vulnerabilities).
 
 **Note:** The **Alerts** data stream ingests individual detection events surfaced by Microsoft and partner security providers, while **Incidents** data stream ingests correlated collections of alerts that represent a broader attack.
 
@@ -177,7 +177,17 @@ The values used in `event.severity` are consistent with Elastic Detection Rules.
 
 ## Troubleshooting
 
-- Expiring SAS URLs: The option `SAS Valid Hours` in `vulnerability` data stream controls the duration that the `Shared Access Signature (SAS)` download URLs are valid for. The default value of this option is `1h` i.e., 1 hour, and the maximum allowed value is `6h` i.e., 6 hours. Increase the value of the option `SAS Valid Hours` when you see `error.message` indicates signatures are invalid, or when you notice invalid signature errors inside CEL trace logs.
+### Vulnerability data stream: Fixed vulnerabilities appearing on the Findings page
+
+The vulnerability data stream uses Microsoft Defender for Endpoint's delta API, which returns change events including remediated (`Fixed`) vulnerabilities. Each record carries a `vulnerability.status` field with one of the following values: `open`, `fixed`, or `unknown`.
+
+Until the Kibana Vulnerability Findings page adds a default filter on this field, remediated vulnerabilities will appear alongside open findings. To exclude them, add the following filter to the Findings page or any saved search:
+
+```
+NOT vulnerability.status: fixed
+```
+
+This is equivalent to showing only currently active vulnerabilities.
 
 ## Scaling
 
@@ -240,4 +250,4 @@ This integration dataset uses the following APIs:
 - `Alerts`: [List alerts_v2](https://learn.microsoft.com/en-us/graph/api/security-list-alerts_v2?view=graph-rest-1.0&tabs=http) endpoint from [Microsoft Graph Security REST API v1.0](https://learn.microsoft.com/en-us/graph/api/resources/security-api-overview?view=graph-rest-1.0)
 - `Events`: [Microsoft Defender XDR Streaming API](https://learn.microsoft.com/en-us/defender-xdr/streaming-api?view=o365-worldwide)
 - `Incidents`: [List incidents](https://learn.microsoft.com/en-us/graph/api/security-list-incidents?view=graph-rest-1.0&tabs=http) endpoint from [Microsoft Graph Security REST API v1.0](https://learn.microsoft.com/en-us/graph/api/resources/security-api-overview?view=graph-rest-1.0)
-- `Vulnerabilities`: [Get software vulnerabilities](https://learn.microsoft.com/en-us/defender-endpoint/api/get-assessment-software-vulnerabilities#2-export-software-vulnerabilities-assessment-via-files) endpoint from [Microsoft Defender for Endpoint API v1.0](https://learn.microsoft.com/en-us/defender-endpoint/api/exposed-apis-list)
+- `Vulnerabilities`: [SoftwareVulnerabilityChangesByMachine](https://learn.microsoft.com/en-us/defender-endpoint/api/get-assessment-software-vulnerabilities) delta endpoint from [Microsoft Defender for Endpoint API v1.0](https://learn.microsoft.com/en-us/defender-endpoint/api/exposed-apis-list)

packages/m365_defender/_dev/deploy/docker/docker-compose.yml

@@ -45,8 +45,6 @@ services:
       - 8080
     volumes:
       - ./vulnerability-http-mock-config.yml:/config.yml
-      - ./download-vulnerability.log.gz:/download-vulnerability.log.gz
-      - ./empty.log.gz:/empty.log.gz
     environment:
       PORT: 8080
     command:

packages/m365_defender/_dev/deploy/docker/vulnerability-http-mock-config.yml

@@ -37,10 +37,12 @@ rules:
             "id_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJub25lIn0.eyJhdWQiOiIyZDRkMTFhMi1mODE0LTQ2YTctOD..."
           }
           `}}
-  - path: /api/machines/SoftwareVulnerabilitiesExport
+  # Page 1: initial request with sinceTime — returns 3 events + nextLink for pagination
+  - path: /api/machines/SoftwareVulnerabilityChangesByMachine
     methods: ["GET"]
     query_params:
-      sasValidHours: "2"
+      sinceTime: "{sinceTime:.*}"
+      pageSize: "10000"
     request_headers:
       authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"]
     responses:
@@ -49,29 +51,19 @@ rules:
           Content-Type:
             - "application/json"
         body: |-
-          {{ minify_json `
-          {
-            "@odata.context": "https://api.security.microsoft.com/api/$metadata#microsoft.windowsDefenderATP.api.ExportFilesResponse",
-            "exportFiles": [
-                "http://svc-m365-defender-vulnerability-cel:8080/path/to/vuln",
-                "http://svc-m365-defender-vulnerability-cel:8080/path/to/empty_vuln"
-            ],
-            "generatedTime": "2025-10-09T00:00:00Z"
-          }
-          `}}
-  - path: /path/to/vuln
-    methods: ["GET"]
-    responses:
-      - status_code: 200
-        headers:
-          Content-Type:
-            - "application/octet-stream"
-        body: '{{file "/download-vulnerability.log.gz"}}'
-  - path: /path/to/empty_vuln
+          {"@odata.context":"https://api.security.microsoft.com/api/$metadata#Collection(microsoft.windowsDefenderATP.api.DeltaAssetVulnerability)","@odata.nextLink":"http://{{ hostname }}:{{ env "PORT" }}/api/machines/SoftwareVulnerabilityChangesByMachine?$skipToken=page2token&pageSize=10000","value":[{"id":"1212121212121212121212_red_hat_kernel_0:5.14.0-427.42.1.el9_4_CVE-2022-49226","deviceId":"1212121212121212121212","rbacGroupName":"Unassigned","deviceName":"sample-host-1","osPlatform":"Linux","osVersion":"enterprise_linux_9.4","osArchitecture":"x64","softwareVendor":"red_hat","softwareName":"kernel","softwareVersion":"0:5.14.0-427.42.1.el9_4","cveId":"CVE-2022-49226","vulnerabilitySeverityLevel":"Medium","recommendedSecurityUpdate":"CVE-2022-49226_oval:com.redhat.rhsa:def:20249315","recommendedSecurityUpdateId":"RHSA-2024:9315","recommendedSecurityUpdateUrl":null,"diskPaths":[],"registryPaths":[],"lastSeenTimestamp":"2026-05-20 22:45:00","firstSeenTimestamp":"2026-05-20 10:43:58","exploitabilityLevel":"NoExploit","recommendationReference":"va-_-red_hat-_-kernel","status":"New","eventTimestamp":"2026-05-20 12:00:00"},{"id":"11111111111111111_ubuntu_thunderbird-gnome-support_for_linux_1:115.18.0+build1-0ubuntu0.22.04.1_CVE-2024-43097","deviceId":"11111111111111111","rbacGroupName":"Unassigned","deviceName":"sample-host-2","osPlatform":"Linux","osVersion":"ubuntu_linux_22.04","osArchitecture":"x64","softwareVendor":"ubuntu","softwareName":"thunderbird-gnome-support_for_linux","softwareVersion":"1:115.18.0+build1-0ubuntu0.22.04.1","cveId":"CVE-2024-43097","vulnerabilitySeverityLevel":"High","recommendedSecurityUpdate":"CVE-2024-43097_oval:com.ubuntu.jammy:def:76631000000","recommendedSecurityUpdateId":"USN-7663-1","recommendedSecurityUpdateUrl":null,"diskPaths":[],"registryPaths":[],"lastSeenTimestamp":"2026-05-20 22:41:42","firstSeenTimestamp":"2026-05-20 10:41:29","exploitabilityLevel":"NoExploit","recommendationReference":"va-_-ubuntu-_-thunderbird-gnome-support_for_linux","status":"Fixed","eventTimestamp":"2026-05-20 12:00:00"},{"id":"aaasasasasasa_microsoft_windows_10_10.0.19045.6093_CVE-2025-49734","deviceId":"aaasasasasasa","rbacGroupName":"Unassigned","deviceName":"host-3","osPlatform":"Windows10","osVersion":"10.0.19045.6093","osArchitecture":"x64","softwareVendor":"microsoft","softwareName":"windows_10","softwareVersion":"10.0.19045.6093","cveId":"CVE-2025-49734","vulnerabilitySeverityLevel":"High","recommendedSecurityUpdate":"September 2025 Security Updates","recommendedSecurityUpdateId":"5065429","recommendedSecurityUpdateUrl":null,"diskPaths":[],"registryPaths":[],"lastSeenTimestamp":"2026-05-19 00:08:23","firstSeenTimestamp":"2026-05-15 17:30:58","exploitabilityLevel":"NoExploit","recommendationReference":"va-_-microsoft-_-windows_10","status":"Updated","eventTimestamp":"2026-05-15 18:00:00"}]}
+  # Page 2: pagination follow-up via @odata.nextLink — returns 1 event, no nextLink (terminates)
+  - path: /api/machines/SoftwareVulnerabilityChangesByMachine
     methods: ["GET"]
+    query_params:
+      $skipToken: "page2token"
+      pageSize: "10000"
+    request_headers:
+      authorization: ["Bearer topsecretaccesstokenthatshouldnotbeleakedforabit"]
     responses:
       - status_code: 200
         headers:
           Content-Type:
-            - "application/octet-stream"
-        body: '{{file "/empty.log.gz"}}'
+            - "application/json"
+        body: |-
+          {"@odata.context":"https://api.security.microsoft.com/api/$metadata#Collection(microsoft.windowsDefenderATP.api.DeltaAssetVulnerability)","value":[{"id":"bbbbbbbbbbbbbb_vmware_tools_12.0.6.0_CVE-2022-31676","deviceId":"bbbbbbbbbbbbbb","rbacGroupName":"Unassigned","deviceName":"host-4","osPlatform":"Windows10","osVersion":"10.0.19045.6332","osArchitecture":"x64","softwareVendor":"vmware","softwareName":"tools","softwareVersion":"12.0.6.0","cveId":"CVE-2022-31676","vulnerabilitySeverityLevel":"High","recommendedSecurityUpdate":"VMware Tools update addresses a local privilege escalation vulnerability (CVE-2022-31676)","recommendedSecurityUpdateId":null,"recommendedSecurityUpdateUrl":null,"diskPaths":["C:\\Program Files\\VMware\\VMware Tools\\VMware VGAuth\\VMwareAliasImport.exe"],"registryPaths":[],"lastSeenTimestamp":"2026-05-18 19:49:51","firstSeenTimestamp":"2026-05-10 15:09:51","exploitabilityLevel":"NoExploit","recommendationReference":"va-_-vmware-_-tools","status":"New","eventTimestamp":"2026-05-10 18:00:00"}]}

packages/m365_defender/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "5.14.0"
+  changes:
+    - description: Switch vulnerability data stream from gzip export API (SoftwareVulnerabilitiesExport) to paginated delta API (SoftwareVulnerabilityChangesByMachine) to eliminate in-memory gzip decompression and reduce RSS usage. Add initial_interval variable, new EventTimestamp and Status fields. Remove SAS Valid Hours and Max Retries variables. Widen transform retention from 4h to 90d for incremental semantics. Bump transform version to trigger reinstall.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/19034
 - version: "5.13.1"
   changes:
     - description: Unique `entity_id` values to avoid irrelevant documents in the Event Analyzer.

packages/m365_defender/data_stream/alert/_dev/test/policy/test-all.expected

@@ -8,10 +8,10 @@ inputs:
       streams:
         - auth.oauth2.client.id: xxxx
           auth.oauth2.client.secret: ${SECRET_0}
-          auth.oauth2.scopes: https://graph.microsoft.com/.default
-          auth.oauth2.token_url: http://host.tld/efgh/oauth2/v2.0/token
           auth.oauth2.endpoint_params:
             grant_type: client_credentials
+          auth.oauth2.scopes: https://graph.microsoft.com/.default
+          auth.oauth2.token_url: http://host.tld/efgh/oauth2/v2.0/token
           config_version: 2
           cursor:
             last_update_time: