atlassian_{bitbucket,confluence,jira}: improve pipeline robustness to empty strings

packages/atlassian_bitbucket/changelog.yml

@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "2.5.0"
+  changes:
+    - description: Improve error reporting.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12977
+    - description: Handle empty string values more robustly.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/12977
 - version: "2.4.0"
   changes:
     - description: Update Kibana constraint to support 9.0.0.

packages/atlassian_bitbucket/data_stream/audit/_dev/test/pipeline/test-audit-api-empty.log-expected.json

@@ -2,4 +2,4 @@
     "expected": [
         null
     ]
-}
+}

packages/atlassian_bitbucket/data_stream/audit/_dev/test/pipeline/test-audit-api.log

@@ -174,4 +174,5 @@
 {"timestamp":"2021-11-27T17:29:00.687Z","author":{"name":"System","type":"system","id":"-1","avatarUri":""},"type":{"categoryI18nKey":"bitbucket.service.audit.category.apps","category":"Apps","actionI18nKey":"bitbucket.service.plugin.audit.action.pluginenabled","action":"Plugin enabled"},"affectedObjects":[{"name":"com.atlassian.templaterenderer.api","type":"MISC"}],"changedValues":[],"node":"8767044c-1b98-4d64-82db-ef29af8c3792","method":"System","extraAttributes":[{"nameI18nKey":"bitbucket.service.plugin.audit.attribute.version","name":"Version","value":"4.1.4"},{"nameI18nKey":"bitbucket.audit.attribute.legacy.target","name":"target","value":"com.atlassian.templaterenderer.api"}]}
 {"timestamp":"2021-11-27T17:26:26.205Z","author":{"name":"System","type":"system","id":"-1","avatarUri":""},"type":{"categoryI18nKey":"bitbucket.service.audit.category.globaladministration","category":"Global administration","actionI18nKey":"bitbucket.service.applicationconfiguration.audit.action.displaynamechanged","action":"Server name changed"},"affectedObjects":[],"changedValues":[{"key":"Name","i18nKey":"bitbucket.service.applicationconfiguration.audit.changedvalue.displaynamechanged.name","to":"Bitbucket"}],"node":"8767044c-1b98-4d64-82db-ef29af8c3792","method":"System","extraAttributes":[{"nameI18nKey":"bitbucket.audit.attribute.legacy.details","name":"details","value":"{\"new\":\"Bitbucket\",\"old\":null}"},{"nameI18nKey":"bitbucket.audit.attribute.legacy.target","name":"target","value":"DISPLAY_NAME"}]}
 {"timestamp":"2021-11-27T17:26:25.141Z","author":{"name":"System","type":"system","id":"-1","avatarUri":""},"type":{"categoryI18nKey":"bitbucket.service.audit.category.usersandgroups","category":"Users and groups","actionI18nKey":"bitbucket.service.user.audit.action.groupcreated","action":"User group created"},"affectedObjects":[{"name":"stash-users","type":"GROUP","uri":"http://bitbucket.internal:7990/admin/groups/view?name=stash-users","id":"stash-users"}],"changedValues":[],"node":"8767044c-1b98-4d64-82db-ef29af8c3792","method":"System","extraAttributes":[{"nameI18nKey":"bitbucket.audit.attribute.legacy.target","name":"target","value":"stash-users"}]}
-{"timestamp":"2021-11-27T17:26:25.045Z","author":{"name":"System","type":"system","id":"-1","avatarUri":""},"type":{"categoryI18nKey":"bitbucket.service.audit.category.usersandgroups","category":"Users and groups","actionI18nKey":"bitbucket.service.user.audit.action.directorycreated","action":"User directory created"},"affectedObjects":[],"changedValues":[],"node":"8767044c-1b98-4d64-82db-ef29af8c3792","method":"System","extraAttributes":[{"nameI18nKey":"bitbucket.service.user.audit.attribute.directory.name","name":"Directory name","value":"Bitbucket Internal Directory"},{"nameI18nKey":"bitbucket.audit.attribute.legacy.target","name":"target","value":"Bitbucket Internal Directory"}]}
+{"timestamp":"2021-11-27T17:26:25.045Z","author":{"name":"System","type":"system","id":"-1","avatarUri":""},"type":{"categoryI18nKey":"bitbucket.service.audit.category.usersandgroups","category":"Users and groups","actionI18nKey":"bitbucket.service.user.audit.action.directorycreated","action":"User directory created"},"affectedObjects":[],"changedValues":[],"node":"8767044c-1b98-4d64-82db-ef29af8c3792","method":"System","extraAttributes":[{"nameI18nKey":"bitbucket.service.user.audit.attribute.directory.name","name":"Directory name","value":"Bitbucket Internal Directory"},{"nameI18nKey":"bitbucket.audit.attribute.legacy.target","name":"target","value":"Bitbucket Internal Directory"}]}
+{"timestamp":"2021-11-27T17:38:53.360Z","author":{"name":"admin","type":"NORMAL","id":"2","uri":"http://bitbucket.internal:7990/users/admin","avatarUri":""},"type":{"categoryI18nKey":"bitbucket.service.audit.category.usersandgroups","category":"Users and groups","actionI18nKey":"bitbucket.service.user.audit.action.userrenamed","action":"Username changed"},"affectedObjects":[{"name":"test.user","type":"USER","id":"3"}],"changedValues":[{"key":"Username","i18nKey":"bitbucket.service.user.audit.attribute.user.name","from":"test","to":"test.user"}],"source":"","system":"http://bitbucket.internal:7990","node":"8767044c-1b98-4d64-82db-ef29af8c3792","method":"Browser","extraAttributes":[{"nameI18nKey":"bitbucket.audit.attribute.legacy.details","name":"details","value":"{\"oldUsername\":\"test\"}"},{"nameI18nKey":"bitbucket.audit.attribute.legacy.target","name":"target","value":"test.user"}]}

packages/atlassian_bitbucket/data_stream/audit/_dev/test/pipeline/test-audit-api.log-expected.json

@@ -10243,6 +10243,89 @@
                 "id": "-1",
                 "name": "System"
             }
+        },
+        {
+            "@timestamp": "2021-11-27T17:38:53.360Z",
+            "bitbucket": {
+                "audit": {
+                    "affected_objects": [
+                        {
+                            "id": "3",
+                            "name": "test.user",
+                            "type": "USER"
+                        }
+                    ],
+                    "changed_values": [
+                        {
+                            "from": "test",
+                            "i18nKey": "bitbucket.service.user.audit.attribute.user.name",
+                            "key": "Username",
+                            "to": "test.user"
+                        }
+                    ],
+                    "extra_attributes": [
+                        {
+                            "name": "details",
+                            "nameI18nKey": "bitbucket.audit.attribute.legacy.details",
+                            "value": "{\"oldUsername\":\"test\"}"
+                        },
+                        {
+                            "name": "target",
+                            "nameI18nKey": "bitbucket.audit.attribute.legacy.target",
+                            "value": "test.user"
+                        }
+                    ],
+                    "method": "Browser",
+                    "type": {
+                        "action": "Username changed",
+                        "actionI18nKey": "bitbucket.service.user.audit.action.userrenamed",
+                        "category": "Users and groups",
+                        "categoryI18nKey": "bitbucket.service.audit.category.usersandgroups"
+                    }
+                }
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "bitbucket.service.user.audit.action.userrenamed",
+                "category": [
+                    "iam"
+                ],
+                "kind": "event",
+                "original": "{\"timestamp\":\"2021-11-27T17:38:53.360Z\",\"author\":{\"name\":\"admin\",\"type\":\"NORMAL\",\"id\":\"2\",\"uri\":\"http://bitbucket.internal:7990/users/admin\",\"avatarUri\":\"\"},\"type\":{\"categoryI18nKey\":\"bitbucket.service.audit.category.usersandgroups\",\"category\":\"Users and groups\",\"actionI18nKey\":\"bitbucket.service.user.audit.action.userrenamed\",\"action\":\"Username changed\"},\"affectedObjects\":[{\"name\":\"test.user\",\"type\":\"USER\",\"id\":\"3\"}],\"changedValues\":[{\"key\":\"Username\",\"i18nKey\":\"bitbucket.service.user.audit.attribute.user.name\",\"from\":\"test\",\"to\":\"test.user\"}],\"source\":\"\",\"system\":\"http://bitbucket.internal:7990\",\"node\":\"8767044c-1b98-4d64-82db-ef29af8c3792\",\"method\":\"Browser\",\"extraAttributes\":[{\"nameI18nKey\":\"bitbucket.audit.attribute.legacy.details\",\"name\":\"details\",\"value\":\"{\\\"oldUsername\\\":\\\"test\\\"}\"},{\"nameI18nKey\":\"bitbucket.audit.attribute.legacy.target\",\"name\":\"target\",\"value\":\"test.user\"}]}",
+                "type": [
+                    "user",
+                    "change"
+                ]
+            },
+            "related": {
+                "hosts": [
+                    "bitbucket.internal"
+                ],
+                "user": [
+                    "admin",
+                    "test",
+                    "test.user"
+                ]
+            },
+            "service": {
+                "address": "http://bitbucket.internal:7990"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "changes": {
+                    "name": "test.user"
+                },
+                "id": "2",
+                "name": "admin",
+                "target": {
+                    "id": "3",
+                    "name": "test"
+                }
+            }
         }
     ]
-}
+}

packages/atlassian_bitbucket/data_stream/audit/_dev/test/pipeline/test-audit-files.log-expected.json

@@ -7865,4 +7865,4 @@
             }
         }
     ]
-}
+}

packages/atlassian_bitbucket/data_stream/audit/elasticsearch/ingest_pipeline/default.yml

@@ -14,6 +14,32 @@ processors:
       target_field: json
   - drop:
       if: ctx.json?.entities instanceof List && ctx.json.entities.length == 0
+  - script:
+      lang: painless
+      if: ctx.json instanceof Map
+      description: This script processor iterates over the whole document to remove fields with null values.
+      source: |
+        void handleMap(Map map) {
+          for (def x : map.values()) {
+            if (x instanceof Map) {
+                handleMap(x);
+            } else if (x instanceof List) {
+                handleList(x);
+            }
+          }
+          map.values().removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0));
+        }
+        void handleList(List list) {
+          for (def x : list) {
+              if (x instanceof Map) {
+                  handleMap(x);
+              } else if (x instanceof List) {
+                  handleList(x);
+              }
+          }
+          list.removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0));
+        }
+        handleMap(ctx.json);
   - set:
       field: _tmp.timestamp
       copy_from: json.timestamp
@@ -449,4 +475,8 @@ on_failure:
       allow_duplicates: false
   - append:
       field: error.message
-      value: '{{{ _ingest.on_failure_message }}}'
+      value: >
+        Processor "{{{ _ingest.on_failure_processor_type }}}"
+        with tag "{{{ _ingest.on_failure_processor_tag }}}"
+        in pipeline "{{{ _ingest.on_failure_pipeline }}}"
+        failed with message "{{{ _ingest.on_failure_message }}}"

packages/atlassian_bitbucket/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: atlassian_bitbucket
 title: Atlassian Bitbucket
-version: "2.4.0"
+version: "2.5.0"
 description: Collect logs from Atlassian Bitbucket with Elastic Agent.
 type: integration
 categories:

packages/atlassian_confluence/changelog.yml

@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "1.29.0"
+  changes:
+    - description: Improve error reporting.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12977
+    - description: Handle empty string values more robustly.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/12977
 - version: "1.28.0"
   changes:
     - description: Update Kibana constraint to support 9.0.0.

packages/atlassian_confluence/data_stream/audit/_dev/test/pipeline/test-audit-api-empty.log-expected.json

@@ -2,4 +2,4 @@
     "expected": [
         null
     ]
-}
+}

packages/atlassian_confluence/data_stream/audit/_dev/test/pipeline/test-audit-api.log

@@ -179,4 +179,5 @@
 {"timestamp":"2021-11-23T00:34:44.466Z","author":{"name":"Anonymous","type":"user","id":"-2","avatarUri":""},"type":{"categoryI18nKey":"audit.logging.category.plugins","category":"Apps","actionI18nKey":"audit.logging.summary.plugin.enabled","action":"App enabled"},"affectedObjects":[{"name":"Confluence Collaborative Editor Plugin","type":"App"}],"changedValues":[],"source":"81.2.69.143","system":"http://confluence.internal:8090","method":"Browser","extraAttributes":[]}
 {"timestamp":"2021-11-28T17:05:37.142Z","author":{"name":"Joe Bob","type":"user","id":"2c9680837d4a3682017d4a375a280000","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=admin123","avatarUri":""},"type":{"categoryI18nKey":"audit.logging.category.user.management","category":"Users and groups","actionI18nKey":"audit.logging.summary.user.renamed","action":"User renamed"},"affectedObjects":[{"name":"asdf","type":"User","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=asdf123","id":"2c9680837d4a3682017d67821e520003"}],"changedValues":[{"key":"Username","i18nKey":"audit.logging.changed.value.username","from":"asdf","to":"asdf123"}],"source":"10.100.100.2","system":"http://confluence.internal:8090","method":"Browser","extraAttributes":[]}
 {"timestamp":"2021-11-28T17:06:11.805Z","author":{"name":"Joe Bob","type":"user","id":"2c9680837d4a3682017d4a375a280000","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=admin123","avatarUri":""},"type":{"categoryI18nKey":"audit.logging.category.user.management","category":"Users and groups","actionI18nKey":"audit.logging.summary.user.updated","action":"User details updated"},"affectedObjects":[{"name":"asdf asdfasdf","type":"User","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=asdf123","id":"2c9680837d4a3682017d67821e520003"}],"changedValues":[],"source":"10.100.100.2","system":"http://confluence.internal:8090","method":"Browser","extraAttributes":[]}
-{"timestamp":"2021-11-28T17:05:37.158Z","author":{"name":"Joe Bob","type":"user","id":"2c9680837d4a3682017d4a375a280000","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=admin123","avatarUri":""},"type":{"categoryI18nKey":"audit.logging.category.user.management","category":"Users and groups","actionI18nKey":"audit.logging.summary.user.updated","action":"User details updated"},"affectedObjects":[{"name":"asdf asdfasdf","type":"User","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=asdf123","id":"2c9680837d4a3682017d67821e520003"}],"changedValues":[{"key":"Display name","i18nKey":"Display name","from":"asdf","to":"asdf asdfasdf"}],"source":"10.100.100.2","system":"http://confluence.internal:8090","method":"Browser","extraAttributes":[]}
+{"timestamp":"2021-11-28T17:05:37.158Z","author":{"name":"Joe Bob","type":"user","id":"2c9680837d4a3682017d4a375a280000","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=admin123","avatarUri":""},"type":{"categoryI18nKey":"audit.logging.category.user.management","category":"Users and groups","actionI18nKey":"audit.logging.summary.user.updated","action":"User details updated"},"affectedObjects":[{"name":"asdf asdfasdf","type":"User","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=asdf123","id":"2c9680837d4a3682017d67821e520003"}],"changedValues":[{"key":"Display name","i18nKey":"Display name","from":"asdf","to":"asdf asdfasdf"}],"source":"10.100.100.2","system":"http://confluence.internal:8090","method":"Browser","extraAttributes":[]}
+{"timestamp":"2021-11-28T17:05:37.158Z","author":{"name":"Joe Bob","type":"user","id":"2c9680837d4a3682017d4a375a280000","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=admin123","avatarUri":""},"type":{"categoryI18nKey":"audit.logging.category.user.management","category":"Users and groups","actionI18nKey":"audit.logging.summary.user.updated","action":"User details updated"},"affectedObjects":[{"name":"asdf asdfasdf","type":"User","uri":"http://confluence.internal:8090/admin/users/viewuser.action?username=asdf123","id":"2c9680837d4a3682017d67821e520003"}],"changedValues":[{"key":"Display name","i18nKey":"Display name","from":"asdf","to":"asdf asdfasdf"}],"source":"","system":"http://confluence.internal:8090","method":"Browser","extraAttributes":[]}

packages/atlassian_confluence/data_stream/audit/_dev/test/pipeline/test-audit-api.log-expected.json

@@ -16410,6 +16410,79 @@
                     "name": "asdf123"
                 }
             }
+        },
+        {
+            "@timestamp": "2021-11-28T17:05:37.158Z",
+            "confluence": {
+                "audit": {
+                    "affected_objects": [
+                        {
+                            "id": "2c9680837d4a3682017d67821e520003",
+                            "name": "asdf asdfasdf",
+                            "type": "User",
+                            "uri": "http://confluence.internal:8090/admin/users/viewuser.action?username=asdf123"
+                        }
+                    ],
+                    "changed_values": [
+                        {
+                            "from": "asdf",
+                            "i18nKey": "Display name",
+                            "key": "Display name",
+                            "to": "asdf asdfasdf"
+                        }
+                    ],
+                    "method": "Browser",
+                    "type": {
+                        "action": "User details updated",
+                        "actionI18nKey": "audit.logging.summary.user.updated",
+                        "category": "Users and groups",
+                        "categoryI18nKey": "audit.logging.category.user.management"
+                    }
+                }
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "audit.logging.summary.user.updated",
+                "category": [
+                    "iam"
+                ],
+                "kind": "event",
+                "original": "{\"timestamp\":\"2021-11-28T17:05:37.158Z\",\"author\":{\"name\":\"Joe Bob\",\"type\":\"user\",\"id\":\"2c9680837d4a3682017d4a375a280000\",\"uri\":\"http://confluence.internal:8090/admin/users/viewuser.action?username=admin123\",\"avatarUri\":\"\"},\"type\":{\"categoryI18nKey\":\"audit.logging.category.user.management\",\"category\":\"Users and groups\",\"actionI18nKey\":\"audit.logging.summary.user.updated\",\"action\":\"User details updated\"},\"affectedObjects\":[{\"name\":\"asdf asdfasdf\",\"type\":\"User\",\"uri\":\"http://confluence.internal:8090/admin/users/viewuser.action?username=asdf123\",\"id\":\"2c9680837d4a3682017d67821e520003\"}],\"changedValues\":[{\"key\":\"Display name\",\"i18nKey\":\"Display name\",\"from\":\"asdf\",\"to\":\"asdf asdfasdf\"}],\"source\":\"\",\"system\":\"http://confluence.internal:8090\",\"method\":\"Browser\",\"extraAttributes\":[]}",
+                "type": [
+                    "user",
+                    "change"
+                ]
+            },
+            "related": {
+                "hosts": [
+                    "confluence.internal"
+                ],
+                "user": [
+                    "admin123",
+                    "asdf123"
+                ]
+            },
+            "service": {
+                "address": "http://confluence.internal:8090"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "changes": {
+                    "full_name": "asdf asdfasdf"
+                },
+                "full_name": "Joe Bob",
+                "id": "2c9680837d4a3682017d4a375a280000",
+                "name": "admin123",
+                "target": {
+                    "full_name": "asdf",
+                    "id": "2c9680837d4a3682017d67821e520003",
+                    "name": "asdf123"
+                }
+            }
         }
     ]
-}
+}

packages/atlassian_confluence/data_stream/audit/_dev/test/pipeline/test-audit-cloud-empty.log-expected.json

@@ -2,4 +2,4 @@
     "expected": [
         null
     ]
-}
+}