[ti_threatq] Update ECS to 8.4.0

packages/ti_threatq/_dev/build/build.yml

@@ -1,3 +1,3 @@
 dependencies:
   ecs:
-    reference: git@v8.3.0
+    reference: git@v8.4.0-rc1

packages/ti_threatq/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.6.0"
+  changes:
+    - description: Update package to ECS 8.4.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3921
 - version: "1.5.1"
   changes:
     - description: Fix proxy URL documentation rendering.

packages/ti_threatq/data_stream/threat/_dev/test/pipeline/test-threatq-sample-ndjson.log-expected.json

@@ -3,7 +3,7 @@
         {
             "@timestamp": "2020-11-15T00:00:02.000Z",
             "ecs": {
-                "version": "8.3.0"
+                "version": "8.4.0"
             },
             "event": {
                 "category": "threat",
@@ -52,7 +52,7 @@
         {
             "@timestamp": "2020-11-15T00:00:02.000Z",
             "ecs": {
-                "version": "8.3.0"
+                "version": "8.4.0"
             },
             "event": {
                 "category": "threat",
@@ -101,7 +101,7 @@
         {
             "@timestamp": "2020-11-15T00:00:02.000Z",
             "ecs": {
-                "version": "8.3.0"
+                "version": "8.4.0"
             },
             "event": {
                 "category": "threat",
@@ -150,7 +150,7 @@
         {
             "@timestamp": "2020-11-15T00:00:02.000Z",
             "ecs": {
-                "version": "8.3.0"
+                "version": "8.4.0"
             },
             "event": {
                 "category": "threat",
@@ -193,7 +193,7 @@
         {
             "@timestamp": "2020-11-15T00:00:02.000Z",
             "ecs": {
-                "version": "8.3.0"
+                "version": "8.4.0"
             },
             "event": {
                 "category": "threat",
@@ -242,7 +242,7 @@
         {
             "@timestamp": "2020-11-15T00:00:02.000Z",
             "ecs": {
-                "version": "8.3.0"
+                "version": "8.4.0"
             },
             "event": {
                 "category": "threat",
@@ -257,11 +257,6 @@
                 "indicator": {
                     "confidence": "Low",
                     "ip": "89.160.20.156",
-                    "marking": {
-                        "tlp": [
-                            "testtlp"
-                        ]
-                    },
                     "provider": [
                         "testprovider"
                     ],
@@ -299,7 +294,7 @@
         {
             "@timestamp": "2020-11-15T00:00:02.000Z",
             "ecs": {
-                "version": "8.3.0"
+                "version": "8.4.0"
             },
             "event": {
                 "category": "threat",
@@ -348,7 +343,7 @@
         {
             "@timestamp": "2020-11-15T00:00:02.000Z",
             "ecs": {
-                "version": "8.3.0"
+                "version": "8.4.0"
             },
             "event": {
                 "category": "threat",
@@ -397,7 +392,7 @@
         {
             "@timestamp": "2020-11-15T00:00:02.000Z",
             "ecs": {
-                "version": "8.3.0"
+                "version": "8.4.0"
             },
             "event": {
                 "category": "threat",
@@ -446,7 +441,7 @@
         {
             "@timestamp": "2020-11-15T00:00:02.000Z",
             "ecs": {
-                "version": "8.3.0"
+                "version": "8.4.0"
             },
             "event": {
                 "category": "threat",

packages/ti_threatq/data_stream/threat/elasticsearch/ingest_pipeline/default.yml

@@ -6,7 +6,7 @@ processors:
   ####################
   - set:
       field: ecs.version
-      value: "8.3.0"
+      value: '8.4.0'
   - set:
       field: event.kind
       value: enrichment
@@ -231,6 +231,7 @@ processors:
       lang: painless
       description: "Extract TLP and providers from source"
       source: |-
+        def ecsTlps = ['WHITE', 'GREEN', 'AMBER', 'RED'];
         def providers = new ArrayList();
         def tlps = new ArrayList();
         for (source in ctx.json.sources) {
@@ -240,7 +241,7 @@ processors:
           if (source.containsKey("provider") && source["provider"] != null) {
             providers.add(source["provider"]);
           }
-          if (source.containsKey("tlp_name") && source["tlp_name"] != null) {
+          if (source.containsKey("tlp_name") && source["tlp_name"] != null && ecsTlps.contains(source["tlp_name"])) {
             tlps.add(source["tlp_name"]);
           }
         }

packages/ti_threatq/data_stream/threat/sample_event.json

@@ -1,31 +1,31 @@
 {
     "@timestamp": "2021-10-01T18:36:03.000Z",
     "agent": {
-        "ephemeral_id": "12c946b4-2bf4-4d07-8aec-d28310ed16c8",
-        "id": "394964aa-5974-455c-bea7-5c0b89b470bd",
+        "ephemeral_id": "9d50b50e-cf01-4905-b2ed-3557c6d540db",
+        "id": "a7be703c-0d78-40ea-8ad7-a02245cca635",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.0.0"
+        "version": "8.3.2"
     },
     "data_stream": {
         "dataset": "ti_threatq.threat",
         "namespace": "ep",
         "type": "logs"
     },
     "ecs": {
-        "version": "8.3.0"
+        "version": "8.4.0"
     },
     "elastic_agent": {
-        "id": "394964aa-5974-455c-bea7-5c0b89b470bd",
+        "id": "a7be703c-0d78-40ea-8ad7-a02245cca635",
         "snapshot": false,
-        "version": "8.0.0"
+        "version": "8.3.2"
     },
     "event": {
         "agent_id_status": "verified",
         "category": "threat",
-        "created": "2022-04-11T09:27:35.244Z",
+        "created": "2022-08-01T16:06:47.804Z",
         "dataset": "ti_threatq.threat",
-        "ingested": "2022-04-11T09:27:36Z",
+        "ingested": "2022-08-01T16:06:48Z",
         "kind": "enrichment",
         "original": "{\"adversaries\":[],\"attributes\":[{\"attribute_id\":5,\"created_at\":\"2021-10-01 18:36:06\",\"id\":4893068,\"indicator_id\":106767,\"name\":\"Contact\",\"touched_at\":\"2021-10-24 18:36:10\",\"updated_at\":\"2021-10-24 18:36:10\",\"value\":\"email:Quetzalcoatl_relays[]protonmail.com url:https://quetzalcoatl-relays.org proof:uri-rsa hoster:frantech.ca\"},{\"attribute_id\":9,\"created_at\":\"2021-10-01 18:36:06\",\"id\":4893069,\"indicator_id\":106767,\"name\":\"Router Port\",\"touched_at\":\"2021-10-24 18:36:10\",\"updated_at\":\"2021-10-24 18:36:10\",\"value\":\"9000\"},{\"attribute_id\":6,\"created_at\":\"2021-10-01 18:36:06\",\"id\":4893070,\"indicator_id\":106767,\"name\":\"Flags\",\"touched_at\":\"2021-10-02 18:36:08\",\"updated_at\":\"2021-10-02 18:36:08\",\"value\":\"ERDV\"}],\"class\":\"network\",\"created_at\":\"2021-10-01 18:36:03\",\"expires_calculated_at\":\"2021-10-23 18:40:17\",\"hash\":\"69beef49fdbd1f54eef3cab324c7b6cf\",\"id\":106767,\"published_at\":\"2021-10-01 18:36:03\",\"score\":0,\"sources\":[{\"created_at\":\"2021-10-01 18:36:06\",\"creator_source_id\":12,\"id\":3699669,\"indicator_id\":106767,\"indicator_status_id\":1,\"indicator_type_id\":15,\"name\":\"www.dan.me.uk Tor Node List\",\"published_at\":\"2021-10-01 18:36:06\",\"reference_id\":37,\"source_id\":12,\"source_type\":\"connectors\",\"updated_at\":\"2021-10-24 18:36:10\"}],\"status\":{\"description\":\"Poses a threat and is being exported to detection tools.\",\"id\":1,\"name\":\"Active\"},\"status_id\":1,\"touched_at\":\"2021-10-24 18:36:10\",\"type\":{\"class\":\"network\",\"id\":15,\"name\":\"IP Address\"},\"type_id\":15,\"updated_at\":\"2021-10-01 18:36:03\",\"value\":\"107.189.1.90\"}",
         "type": "indicator"

packages/ti_threatq/docs/README.md

@@ -66,7 +66,7 @@ By default the indicators will be collected every 1 minute, and deduplication is
 | tags | List of keywords used to tag each event. | keyword |
 | threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword |
 | threat.feed.name |  | keyword |
-| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. Expected values are:   \* Not Specified   \* None   \* Low   \* Medium   \* High | keyword |
+| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword |
 | threat.indicator.description | Describes the type of action conducted by the threat. | keyword |
 | threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword |
 | threat.indicator.file.hash.md5 | MD5 hash. | keyword |
@@ -76,9 +76,9 @@ By default the indicators will be collected every 1 minute, and deduplication is
 | threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date |
 | threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip |
 | threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date |
-| threat.indicator.marking.tlp | Traffic Light Protocol sharing markings. Recommended values are:   \* WHITE   \* GREEN   \* AMBER   \* RED | keyword |
+| threat.indicator.marking.tlp | Traffic Light Protocol sharing markings. | keyword |
 | threat.indicator.provider | The name of the indicator's provider. | keyword |
-| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. Recommended values:   \* autonomous-system   \* artifact   \* directory   \* domain-name   \* email-addr   \* file   \* ipv4-addr   \* ipv6-addr   \* mac-addr   \* mutex   \* port   \* process   \* software   \* url   \* user-account   \* windows-registry-key   \* x509-certificate | keyword |
+| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword |
 | threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword |
 | threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword |
 | threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard |
@@ -106,31 +106,31 @@ An example event for `threat` looks as following:
 {
     "@timestamp": "2021-10-01T18:36:03.000Z",
     "agent": {
-        "ephemeral_id": "12c946b4-2bf4-4d07-8aec-d28310ed16c8",
-        "id": "394964aa-5974-455c-bea7-5c0b89b470bd",
+        "ephemeral_id": "9d50b50e-cf01-4905-b2ed-3557c6d540db",
+        "id": "a7be703c-0d78-40ea-8ad7-a02245cca635",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.0.0"
+        "version": "8.3.2"
     },
     "data_stream": {
         "dataset": "ti_threatq.threat",
         "namespace": "ep",
         "type": "logs"
     },
     "ecs": {
-        "version": "8.3.0"
+        "version": "8.4.0"
     },
     "elastic_agent": {
-        "id": "394964aa-5974-455c-bea7-5c0b89b470bd",
+        "id": "a7be703c-0d78-40ea-8ad7-a02245cca635",
         "snapshot": false,
-        "version": "8.0.0"
+        "version": "8.3.2"
     },
     "event": {
         "agent_id_status": "verified",
         "category": "threat",
-        "created": "2022-04-11T09:27:35.244Z",
+        "created": "2022-08-01T16:06:47.804Z",
         "dataset": "ti_threatq.threat",
-        "ingested": "2022-04-11T09:27:36Z",
+        "ingested": "2022-08-01T16:06:48Z",
         "kind": "enrichment",
         "original": "{\"adversaries\":[],\"attributes\":[{\"attribute_id\":5,\"created_at\":\"2021-10-01 18:36:06\",\"id\":4893068,\"indicator_id\":106767,\"name\":\"Contact\",\"touched_at\":\"2021-10-24 18:36:10\",\"updated_at\":\"2021-10-24 18:36:10\",\"value\":\"email:Quetzalcoatl_relays[]protonmail.com url:https://quetzalcoatl-relays.org proof:uri-rsa hoster:frantech.ca\"},{\"attribute_id\":9,\"created_at\":\"2021-10-01 18:36:06\",\"id\":4893069,\"indicator_id\":106767,\"name\":\"Router Port\",\"touched_at\":\"2021-10-24 18:36:10\",\"updated_at\":\"2021-10-24 18:36:10\",\"value\":\"9000\"},{\"attribute_id\":6,\"created_at\":\"2021-10-01 18:36:06\",\"id\":4893070,\"indicator_id\":106767,\"name\":\"Flags\",\"touched_at\":\"2021-10-02 18:36:08\",\"updated_at\":\"2021-10-02 18:36:08\",\"value\":\"ERDV\"}],\"class\":\"network\",\"created_at\":\"2021-10-01 18:36:03\",\"expires_calculated_at\":\"2021-10-23 18:40:17\",\"hash\":\"69beef49fdbd1f54eef3cab324c7b6cf\",\"id\":106767,\"published_at\":\"2021-10-01 18:36:03\",\"score\":0,\"sources\":[{\"created_at\":\"2021-10-01 18:36:06\",\"creator_source_id\":12,\"id\":3699669,\"indicator_id\":106767,\"indicator_status_id\":1,\"indicator_type_id\":15,\"name\":\"www.dan.me.uk Tor Node List\",\"published_at\":\"2021-10-01 18:36:06\",\"reference_id\":37,\"source_id\":12,\"source_type\":\"connectors\",\"updated_at\":\"2021-10-24 18:36:10\"}],\"status\":{\"description\":\"Poses a threat and is being exported to detection tools.\",\"id\":1,\"name\":\"Active\"},\"status_id\":1,\"touched_at\":\"2021-10-24 18:36:10\",\"type\":{\"class\":\"network\",\"id\":15,\"name\":\"IP Address\"},\"type_id\":15,\"updated_at\":\"2021-10-01 18:36:03\",\"value\":\"107.189.1.90\"}",
         "type": "indicator"

packages/ti_threatq/manifest.yml

@@ -1,6 +1,6 @@
 name: ti_threatq
 title: ThreatQuotient
-version: "1.5.1"
+version: "1.6.0"
 release: ga
 description: Ingest threat intelligence indicators from ThreatQuotient with Elastic Agent.
 type: integration