Update AWS ELB ingest pipeline

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.6.2"
+  changes:
+    - description: Removed incorrect `http.request.referrer` field from elb logs
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/1212
 - version: "0.6.1"
   changes:
     - description: Add support for CloudTrail Digest & Insight logs

packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json

@@ -10,6 +10,13 @@
                 "port": "2817",
                 "ip": "192.168.131.39"
             },
+            "url": {
+                "path": "/",
+                "original": "http://www.example.com:80/",
+                "scheme": "http",
+                "port": 80,
+                "domain": "www.example.com"
+            },
             "tags": [
                 "preserve_original_event"
             ],
@@ -25,8 +32,7 @@
                     "method": "get",
                     "body": {
                         "bytes": 34
-                    },
-                    "referrer": "http://www.example.com:80/"
+                    }
                 },
                 "version": "1.1",
                 "response": {
@@ -37,7 +43,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-06-04T09:43:16.047944100Z",
+                "ingested": "2021-06-24T23:04:01.923552600Z",
                 "original": "http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.46.0\" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 \"Root=1-58337262-36d228ad5d99923122bbe354\" \"-\" \"-\" 0 2018-07-02T22:22:48.364000Z \"forward,redirect\" \"-\" \"-\" \"10.0.0.1:80\" \"200\" \"-\" \"-\"",
                 "kind": "event",
                 "start": "2018-07-02T22:22:48.364000Z",
@@ -86,7 +92,12 @@
                 }
             },
             "user_agent": {
-                "original": "curl/7.46.0"
+                "name": "curl",
+                "original": "curl/7.46.0",
+                "device": {
+                    "name": "Other"
+                },
+                "version": "7.46.0"
             }
         }
     ]

packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml

@@ -85,8 +85,8 @@ processors:
           (?:-|%{NUMBER:aws.elb.backend.http.response.status_code:long})
           %{NUMBER:http.request.body.bytes:long}
           %{NUMBER:http.response.body.bytes:long}
-          \"(?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:http.request.referrer}) (?:-|HTTP/%{NOTSPACE:http.version})\"
-          \"%{DATA:user_agent.original}\"
+          \"(?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:_tmp.uri_orig}) (?:-|HTTP/%{NOTSPACE:http.version})\"
+          \"%{DATA:_tmp.user_agent}\"
           %{ELBSSL}
         ELBTCPLOG: >-
           %{ELBCOMMON}
@@ -109,6 +109,17 @@ processors:
       if: ctx.http != null
       field: aws.elb.protocol
       value: http
+
+  - uri_parts:
+      if: 'ctx?._tmp?.uri_orig != null'
+      field: _tmp.uri_orig
+      ignore_failure: true
+
+  - user_agent:
+      if: 'ctx?._tmp?.user_agent != null'
+      field: _tmp.user_agent
+      ignore_missing: true
+
   - set:
       if: ctx.http != null
       field: event.category

packages/aws/data_stream/elb_logs/fields/ecs.yml

@@ -15,3 +15,41 @@
   example: '["production", "env2"]'
   ignore_above: 1024
   type: keyword
+- name: url
+  type: group
+  fields:
+    - name: domain
+      level: extended
+      type: keyword
+      description: Domain of the url, such as "www.elastic.co".
+    - name: original
+      level: extended
+      type: keyword
+      description: Unmodified original url as seen in the event source.
+    - name: path
+      level: extended
+      type: keyword
+      description: Path of the request, such as "/search".
+    - name: port
+      level: extended
+      type: long
+      description: Port of the request, such as 443.
+    - name: scheme
+      level: extended
+      type: keyword
+      description: Scheme of the request, such as "https".
+- name: user_agent
+  type: group
+  fields:
+    - name: device.name
+      level: extended
+      type: keyword
+      description: Name of the device.
+    - name: name
+      level: extended
+      type: keyword
+      description: Name of the user agent.
+    - name: version
+      level: extended
+      type: keyword
+      description: Version of the user agent.

packages/aws/docs/README.md

@@ -432,7 +432,15 @@ For network load balancer, please follow [enable access log for network load bal
 | source.port | Port of the source. | keyword |
 | tags | List of keywords used to tag each event. | keyword |
 | tracing.trace.id | Unique identifier of the trace. | keyword |
+| url.domain | Domain of the url, such as "www.elastic.co". | keyword |
+| url.original | Unmodified original url as seen in the event source. | keyword |
+| url.path | Path of the request, such as "/search". | keyword |
+| url.port | Port of the request, such as 443. | long |
+| url.scheme | Scheme of the request, such as "https". | keyword |
+| user_agent.device.name | Name of the device. | keyword |
+| user_agent.name | Name of the user agent. | keyword |
 | user_agent.original | Unparsed user_agent string. | keyword |
+| user_agent.version | Version of the user agent. | keyword |
 
 
 ### s3access

packages/aws/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: aws
 title: AWS
-version: 0.6.1
+version: 0.6.2
 license: basic
 description: AWS Integration
 type: integration