microsoft_defender_endpoint: fix handling of empty array fields

packages/microsoft_defender_endpoint/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.24.1"
+  changes:
+    - description: Fix handling of empty arrays.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/9338
 - version: "2.24.0"
   changes:
     - description: Set sensitive values as secret.

packages/microsoft_defender_endpoint/data_stream/log/_dev/test/pipeline/test-defenderatp.log

@@ -2,4 +2,5 @@
 {"id":"da637291048912199236_1126926584","incidentId":11,"investigationId":7,"assignedTo":null,"severity":"Medium","status":"New","classification":null,"determination":null,"investigationState":"TerminatedByUser","detectionSource":"WindowsDefenderAtp","category":"DefenseEvasion","threatFamilyName":null,"title":"Suspicious process injection observed","description":"A process abnormally injected code into another process, As a result, unexpected code may be running in the target process memory. Injection is often used to hide malicious code execution within a trusted process. \nAs a result, the target process may exhibit abnormal behaviors such as opening a listening port or connecting to a command and control server.","alertCreationTime":"2020-06-30T09:08:11.1084877Z","firstEventTime":"2020-06-30T09:04:56.8490679Z","lastEventTime":"2020-06-30T09:45:39.5484377Z","lastUpdateTime":"2020-06-30T15:29:44.7733333Z","resolvedTime":null,"machineId":"543bc5a964f417c11f6277d5bf9489f0d","computerDnsName":"testserver4","rbacGroupName":null,"aadTenantId":"123543-d66c-4c7e-9e30-40034eb7c6f3","relatedUser":{"userName":"administrator1","domainName":"TestServer4"},"comments":[],"evidence":{"entityType":"Process","sha1":"b6d237154f2e528f0b503b58b025862d66b02b73","sha256":"a92056d772260b39a876d01552496b2f8b4610a0b1e084952fe1176784e2ce77","fileName":"notepad.exe","filePath":"C:\\Windows\\System32","processId":4104,"processCommandLine":"\"notepad.exe\"","processCreationTime":"2020-06-30T09:45:38.9784654Z","parentProcessId":6012,"parentProcessCreationTime":"2020-06-30T09:04:51.487396Z","ipAddress":null,"url":null,"accountName":null,"domainName":null,"userSid":null,"aadUserId":null,"userPrincipalName":null}}
 {"id":"da637291048912199236_1126926584","incidentId":11,"investigationId":7,"assignedTo":null,"severity":"Medium","status":"New","classification":null,"determination":null,"investigationState":"TerminatedByUser","detectionSource":"WindowsDefenderAtp","category":"DefenseEvasion","threatFamilyName":null,"title":"Suspicious process injection observed","description":"A process abnormally injected code into another process, As a result, unexpected code may be running in the target process memory. Injection is often used to hide malicious code execution within a trusted process. \nAs a result, the target process may exhibit abnormal behaviors such as opening a listening port or connecting to a command and control server.","alertCreationTime":"2020-06-30T09:08:11.1084877Z","firstEventTime":"2020-06-30T09:04:56.8490679Z","lastEventTime":"2020-06-30T09:45:39.5484377Z","lastUpdateTime":"2020-06-30T15:29:44.7733333Z","resolvedTime":null,"machineId":"53425a964f417c11f6277d5bf9489f0d","computerDnsName":"testserver4","rbacGroupName":null,"aadTenantId":"43521344-d66c-4c7e-9e30-40034eb7c6f3","relatedUser":{"userName":"administrator1","domainName":"TestServer4"},"comments":[],"evidence":{"entityType":"User","sha1":null,"sha256":null,"fileName":null,"filePath":null,"processId":null,"processCommandLine":null,"processCreationTime":null,"parentProcessId":null,"parentProcessCreationTime":null,"ipAddress":null,"url":null,"accountName":"administrator1","domainName":"TestServer4","userSid":"S-1-5-21-46152456-1367606905-4031241297-500","aadUserId":null,"userPrincipalName":null}}
 {"id":"da637291063515066999_-2102938302","incidentId":12,"investigationId":9,"assignedTo":"Automation","severity":"Informational","status":"Resolved","classification":null,"determination":null,"investigationState":"Benign","detectionSource":"WindowsDefenderAv","category":"Malware","threatFamilyName":null,"title":"'Mountsi' malware was detected","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.","alertCreationTime":"2020-06-30T09:32:31.4579225Z","firstEventTime":"2020-06-30T09:31:22.5729558Z","lastEventTime":"2020-06-30T09:46:15.0876676Z","lastUpdateTime":"2020-06-30T11:13:12.9Z","resolvedTime":"2020-06-30T11:13:12.2680434Z","machineId":"t4563234bc5a964f417c11f6277d5bf9489f0d","computerDnsName":"TESTSERVER4","rbacGroupName":null,"aadTenantId":"1234543-d66c-4c7e-9e30-40034eb7c6f3","relatedUser":null,"comments":[],"evidence":{"entityType":"File","sha1":"ffb1670c6c6a9c5b4c5cea8b6b8e68d62e7ff281","sha256":"fd46705c4f67a8ef16e76259ca6d6253241e51a1f8952223145f92aa1907d356","fileName":"amsistream-1D89ECED25A52AB98B76FF619B7BA07A","filePath":null,"processId":null,"processCommandLine":null,"processCreationTime":null,"parentProcessId":null,"parentProcessCreationTime":null,"ipAddress":null,"url":null,"accountName":null,"domainName":null,"userSid":null,"aadUserId":null,"userPrincipalName":null}}
+{"id":"da637291063515066999_-2102938302","incidentId":12,"investigationId":9,"assignedTo":"Automation","severity":"Informational","status":"Resolved","classification":null,"determination":null,"investigationState":"Benign","detectionSource":"WindowsDefenderAv","category":"Malware","threatFamilyName":null,"title":"'Mountsi' malware was detected","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.","alertCreationTime":"2020-06-30T09:32:31.4579225Z","firstEventTime":"2020-06-30T09:31:22.5729558Z","lastEventTime":"2020-06-30T09:46:15.0876676Z","lastUpdateTime":"2020-06-30T11:13:12.9Z","resolvedTime":"2020-06-30T11:13:12.2680434Z","machineId":"t4563234bc5a964f417c11f6277d5bf9489f0d","computerDnsName":"TESTSERVER4","rbacGroupName":null,"aadTenantId":"1234543-d66c-4c7e-9e30-40034eb7c6f3","relatedUser":null,"comments":[],"evidence":[]}
 {"value":[]}

packages/microsoft_defender_endpoint/data_stream/log/_dev/test/pipeline/test-defenderatp.log-expected.json

@@ -357,6 +357,80 @@
                 }
             }
         },
+        {
+            "cloud": {
+                "account": {
+                    "id": "1234543-d66c-4c7e-9e30-40034eb7c6f3"
+                },
+                "instance": {
+                    "id": "t4563234bc5a964f417c11f6277d5bf9489f0d"
+                },
+                "provider": "azure"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "Malware",
+                "category": [
+                    "host",
+                    "malware"
+                ],
+                "created": "2020-06-30T09:32:31.4579225Z",
+                "duration": 892514711800,
+                "end": "2020-06-30T09:46:15.0876676Z",
+                "id": "da637291063515066999_-2102938302",
+                "kind": "alert",
+                "original": "{\"id\":\"da637291063515066999_-2102938302\",\"incidentId\":12,\"investigationId\":9,\"assignedTo\":\"Automation\",\"severity\":\"Informational\",\"status\":\"Resolved\",\"classification\":null,\"determination\":null,\"investigationState\":\"Benign\",\"detectionSource\":\"WindowsDefenderAv\",\"category\":\"Malware\",\"threatFamilyName\":null,\"title\":\"'Mountsi' malware was detected\",\"description\":\"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\\n\\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.\",\"alertCreationTime\":\"2020-06-30T09:32:31.4579225Z\",\"firstEventTime\":\"2020-06-30T09:31:22.5729558Z\",\"lastEventTime\":\"2020-06-30T09:46:15.0876676Z\",\"lastUpdateTime\":\"2020-06-30T11:13:12.9Z\",\"resolvedTime\":\"2020-06-30T11:13:12.2680434Z\",\"machineId\":\"t4563234bc5a964f417c11f6277d5bf9489f0d\",\"computerDnsName\":\"TESTSERVER4\",\"rbacGroupName\":null,\"aadTenantId\":\"1234543-d66c-4c7e-9e30-40034eb7c6f3\",\"relatedUser\":null,\"comments\":[],\"evidence\":[]}",
+                "provider": "defender_endpoint",
+                "severity": 1,
+                "start": "2020-06-30T09:31:22.5729558Z",
+                "timezone": "UTC",
+                "type": [
+                    "end"
+                ]
+            },
+            "host": {
+                "hostname": "TESTSERVER4",
+                "name": "testserver4"
+            },
+            "message": "'Mountsi' malware was detected",
+            "microsoft": {
+                "defender_endpoint": {
+                    "assignedTo": "Automation",
+                    "incidentId": "12",
+                    "investigationId": "9",
+                    "investigationState": "Benign",
+                    "lastUpdateTime": "2020-06-30T11:13:12.9Z",
+                    "resolvedTime": "2020-06-30T11:13:12.2680434Z",
+                    "status": "Resolved"
+                }
+            },
+            "observer": {
+                "name": "WindowsDefenderAv",
+                "product": "Defender for Endpoint",
+                "vendor": "Microsoft"
+            },
+            "related": {
+                "hosts": [
+                    "testserver4"
+                ]
+            },
+            "rule": {
+                "description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection."
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "threat": {
+                "framework": "MITRE ATT&CK",
+                "technique": {
+                    "name": [
+                        "Malware"
+                    ]
+                }
+            }
+        },
         null
     ]
 }

packages/microsoft_defender_endpoint/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -34,7 +34,7 @@ processors:
           - "-"
           - "N/A"
       source: |
-        if (!ctx['json'].empty) {
+        if (!ctx.json.empty) {
           ctx.json.entrySet().removeIf(entry -> params.values.contains(entry.getValue()));
         }
   - script:
@@ -47,9 +47,22 @@ processors:
           - "-"
           - "N/A"
       source: |
-        if (!ctx.json['evidence'].empty) {
+        boolean drop(Object o) {
+          if (o == null || o == "") {
+            return true;
+          } else if (o instanceof Map) {
+            ((Map) o).values().removeIf(v -> drop(v));
+            return (((Map) o).size() == 0);
+          } else if (o instanceof List) {
+            ((List) o).removeIf(v -> drop(v));
+            return (((List) o).length == 0);
+          }
+          return false;
+        }
+        if (!ctx.json.evidence.empty) {
           ctx.json.evidence.entrySet().removeIf(entry -> params.values.contains(entry.getValue()));
         }
+        drop(ctx);
   - set:
       field: cloud.provider
       value: azure

packages/microsoft_defender_endpoint/data_stream/log/sample_event.json

@@ -108,4 +108,4 @@
         "id": "S-1-5-21-11111607-1111760036-109187956-75141",
         "name": "temp123"
     }
-}
+}

packages/microsoft_defender_endpoint/docs/README.md

@@ -157,7 +157,6 @@ An example event for `log` looks as following:
         "name": "temp123"
     }
 }
-
 ```
 
 **Exported fields**

packages/microsoft_defender_endpoint/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: microsoft_defender_endpoint
 title: Microsoft Defender for Endpoint
-version: "2.24.0"
+version: "2.24.1"
 description: Collect logs from Microsoft Defender for Endpoint with Elastic Agent.
 categories:
   - "security"