Remove event.original processors from several remaining integrations part 1

packages/gcp/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.42.0"
+  changes:
+    - description: Allow @custom pipeline access to event.original without setting preserve_original_event.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/13521
 - version: "2.41.1"
   changes:
     - description: Preserve original value of resource name.

packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml

@@ -630,11 +630,6 @@ processors:
         - _temp
         - json
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null and empty values and dotted keys recursively
       lang: painless

packages/gcp/data_stream/dns/elasticsearch/ingest_pipeline/default.yml

@@ -371,11 +371,6 @@ processors:
           list.removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0));
         }
         handleMap(ctx);
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

packages/gcp/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml

@@ -407,11 +407,6 @@ processors:
       field: destination.as.organization_name
       target_field: destination.as.organization.name
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

packages/gcp/data_stream/loadbalancing_logs/elasticsearch/ingest_pipeline/default.yml

@@ -207,11 +207,6 @@ processors:
       field: 
         - json
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

packages/gcp/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml

@@ -393,11 +393,6 @@ processors:
         - _conf
         - json
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

packages/gcp/manifest.yml

@@ -1,6 +1,6 @@
 name: gcp
 title: Google Cloud Platform
-version: "2.41.1"
+version: "2.42.0"
 description: Collect logs and metrics from Google Cloud Platform with Elastic Agent.
 type: integration
 icons:

packages/goflow2/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.5.0"
+  changes:
+    - description: Allow @custom pipeline access to event.original without setting preserve_original_event.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/13521
 - version: "0.4.0"
   changes:
     - description: Support stack version 9.0.

packages/goflow2/data_stream/sflow/elasticsearch/ingest_pipeline/default.yml

@@ -262,12 +262,6 @@ processors:
     - "{{{destination.ip}}}"
     if: ctx.destination?.ip != null && ctx.destination?.ip != ''
     tag: append_destination_ip_to_related_ip
-- remove:
-    field: event.original
-    if: ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))
-    ignore_failure: true
-    ignore_missing: true
-    tag: remove_event_orginial_if_not_tags_preserve_original_event
 on_failure:
 - set:
     field: error.message

packages/goflow2/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.2.1
 name: goflow2
 title: "GoFlow2 logs"
-version: "0.4.0"
+version: "0.5.0"
 description: "Collect logs from goflow2 with Elastic Agent."
 type: integration
 categories:

packages/golang/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.8.0"
+  changes:
+    - description: Allow @custom pipeline access to event.original without setting preserve_original_event.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/13521
 - version: "1.7.1"
   changes:
     - description: Added description to ssl nodes including links to documentation.

packages/golang/data_stream/expvar/elasticsearch/ingest_pipeline/default.yml

@@ -80,10 +80,6 @@ processors:
       field: json.memstats.StackInuse
       target_field: golang.expvar.stack.bytes
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_missing: true
   - remove:
       field:
         - json.memstats

packages/golang/data_stream/heap/elasticsearch/ingest_pipeline/default.yml

@@ -88,10 +88,6 @@ processors:
       field: json.memstats.HeapSys
       target_field: golang.heap.system.total.bytes
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_missing: true
   - remove:
       field:
         - json

packages/golang/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: golang
 title: Golang
-version: "1.7.1"
+version: "1.8.0"
 description: This Elastic integration collects metrics from Golang applications.
 type: integration
 categories:

packages/hadoop/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.9.0"
+  changes:
+    - description: Allow @custom pipeline access to event.original without setting preserve_original_event.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/13521
 - version: "1.8.1"
   changes:
     - description: Updated SSL description to be uniform and to include links to documentation.

packages/hadoop/data_stream/application/elasticsearch/ingest_pipeline/default.yml

@@ -98,11 +98,6 @@ processors:
           return false;
         }
         drop(ctx);
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message

packages/hadoop/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.0"
 name: hadoop
 title: Hadoop
-version: "1.8.1"
+version: "1.9.0"
 description: Collect metrics from Apache Hadoop with Elastic Agent.
 type: integration
 categories:

packages/hashicorp_vault/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.28.0"
+  changes:
+    - description: Allow @custom pipeline access to event.original without setting preserve_original_event.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/13521
 - version: "1.27.0"
   changes:
     - description: Support stack version 9.0.

packages/hashicorp_vault/data_stream/audit/elasticsearch/ingest_pipeline/default.yml

@@ -174,15 +174,6 @@ processors:
       value: '{{{source.ip}}}'
       allow_duplicates: false
       if: ctx?.source?.ip != null
-
-  #
-  # event.original - keep or drop
-  #
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

packages/hashicorp_vault/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -27,14 +27,6 @@ processors:
       field: message
       copy_from: event.original
       ignore_failure: true
-  #
-  # event.original - keep or drop
-  #
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

packages/hashicorp_vault/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: hashicorp_vault
 title: Hashicorp Vault
-version: "1.27.0"
+version: "1.28.0"
 description: Collect logs and metrics from Hashicorp Vault with Elastic Agent.
 type: integration
 categories:

packages/hid_bravura_monitor/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.21.0"
+  changes:
+    - description: Allow @custom pipeline access to event.original without setting preserve_original_event.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/13521
 - version: "1.20.0"
   changes:
     - description: ECS version updated to 8.17.0.

packages/hid_bravura_monitor/data_stream/log/_dev/test/pipeline/test-hid-bravura-monitor.log-expected.json

@@ -6,6 +6,7 @@
                 "version": "8.17.0"
             },
             "event": {
+                "original": "2021-01-16 00:38:18.515.1538 - [] idapi.exe [5368,54656] Debug: Thread: 54656, get the client id: 4272",
                 "timezone": "UTC"
             },
             "hid_bravura_monitor": {
@@ -35,6 +36,7 @@
                 "version": "8.17.0"
             },
             "event": {
+                "original": "\u00182021-01-16 00:35:25.258.7085 - [] pamlws.exe [44408,52004] Error: LWS [HID-TEST] foundcomputer record not found",
                 "timezone": "UTC"
             },
             "hid_bravura_monitor": {
@@ -64,6 +66,7 @@
                 "version": "8.17.0"
             },
             "event": {
+                "original": "\u00182021-01-27 00:31:24.499.5341 - [] idmlogsvc.exe [8620,7764] Info: Logging rotation started, rotate log number: 7",
                 "timezone": "UTC"
             },
             "hid_bravura_monitor": {
@@ -93,6 +96,7 @@
                 "version": "8.17.0"
             },
             "event": {
+                "original": "\u00182021-01-16 00:35:34.317.7766 - [09bc17eb-7b1d-4b1a-8c66-42c49384a552] agtssh.exe [50864,52168] Notice: parallel operations will be done in serial instead",
                 "timezone": "UTC"
             },
             "hid_bravura_monitor": {
@@ -122,6 +126,7 @@
                 "version": "8.17.0"
             },
             "event": {
+                "original": "\u00182021-02-05 08:41:11.845.2545 - [] ajaxsvc.exe [4248,10876] Perf: PerfAjax. Duration: {0} | Function: {session:expiry} | Operation: {POST} | TransID: {C_SELECT_ATTRDEF} | User: {joe.test}",
                 "timezone": "UTC"
             },
             "hid_bravura_monitor": {
@@ -158,6 +163,7 @@
                 "version": "8.17.0"
             },
             "event": {
+                "original": "\u00182021-01-16 11:54:34.234.2721 - [] iddiscover.exe [5356,10572] Perf: PerfBatchLoad. Duration: {0} | Event: {loaddb} | Records: {0} | Table: {NOSGROUP_ADD_STG}",
                 "timezone": "UTC"
             },
             "hid_bravura_monitor": {
@@ -194,6 +200,7 @@
                 "version": "8.17.0"
             },
             "event": {
+                "original": "\u00182021-10-21 19:13:31.679.5865 - [] mobworker.exe [3628,4320] Debug: #1 r522057: The HTTP client read returned: 408, [HTTP/1.1 408 Request Timeout\nConnection: Upgrade, close\nDate: Thu, 21 Oct 2021 19:12:40 GMT\nUpgrade: h2,h2c\nContent-Length: 49\nContent-Type: text/html; charset=utf-8\nServer: Apache/2.4.51 () OpenSSL/1.0.2k-fips\n\n] [Timed out for waiting a client( timeout = 51000 )]",
                 "timezone": "UTC"
             },
             "hid_bravura_monitor": {
@@ -223,6 +230,7 @@
                 "version": "8.17.0"
             },
             "event": {
+                "original": "\u00182021-01-16 00:35:32.941.8145 - [09bc17eb-7b1d-4b1a-8c66-42c49384a552] agtssh.exe [19360,54164] Perf: PerfConnector. Address: {[script=\"E:\\Program Files\\Bravura Security\\Bravura Security Fabric\\default\\agent\\64\\agtlinux.psl\"; server=1.128.3.4;port=22; compression=false; hostkeys=AllowAppend; privEscType=sudo; passwdAccessOnly=true; maxReadTimeout=6; maxWriteTimeout=20; maxReadSize=16384; maxReadLines=50000; pubkeyfiles=id_rsa.pub,id_dsa.pub; ]} | AdminID: {sys_padm} | Duration: {21030} | Event: {connector-operation} | Message: {Network error: Connection timed out.  Could not connect to address [[script=\"E:\\Program Files\\Bravura Security\\Bravura Security Fabric\\default\\agent\\64\\agtlinux.psl\"; server=1.128.3.4;port=22; compression=false; hostkeys=AllowAppend; privEscType=sudo; passwdAccessOnly=true; maxReadTimeout=6; maxWriteTimeout=20; maxReadSize=16384; maxReadLines=50000; pubkeyfiles=id_rsa.pub,id_dsa.pub; ]].} | Operation: {Connect} | Result: {1} | SysID: {} | TargetID: {D989E43017B94D78963CBB7113467696}",
                 "timezone": "UTC"
             },
             "hid_bravura_monitor": {
@@ -264,6 +272,7 @@
                 "version": "8.17.0"
             },
             "event": {
+                "original": "\u00182021-01-16 11:54:18.663.7558 - [joe.test] psf.exe [55252,51340] Perf: PerfExe. Duration: {625} | Kernel: {93} | SessionID: {S39ccf8f2-c7ab-4baa-bc7a-37c2b0a154b3} | TransID: {C_AUTHCHAIN_LOGIN} | User: {15}",
                 "timezone": "UTC"
             },
             "hid_bravura_monitor": {
@@ -302,6 +311,7 @@
                 "version": "8.17.0"
             },
             "event": {
+                "original": "\u00182021-02-05 08:41:11.845.2545 - [psupdate8112_10120] updinst.exe [4248,10876] Perf: PerfFileRep. Destination: {bravurasecurity.test.net} | Duration: {0} | Event: {filerep-sendreg} | File: {\\\\psupdate_finish}",
                 "timezone": "UTC"
             },
             "hid_bravura_monitor": {
@@ -338,6 +348,7 @@
                 "version": "8.17.0"
             },
             "event": {
+                "original": "\u00182021-02-05 08:41:11.845.2545 - [] idapi.exe [4248,10876] Perf: PerfIDAPI. Duration: {3} | Function: {AccountAttrsGet}",
                 "timezone": "UTC"
             },
             "hid_bravura_monitor": {
@@ -372,6 +383,7 @@
                 "version": "8.17.0"
             },
             "event": {
+                "original": "\u00182021-02-05 08:43:13.839.4409 - [] idwfm.exe [4512,7644] Perf: PerfIDWFM. Caller: {IDAQueue::PostInsert_Prot} | Duration: {0} | Type: {Queue}",
                 "timezone": "UTC"
             },
             "hid_bravura_monitor": {
@@ -407,6 +419,7 @@
                 "version": "8.17.0"
             },
             "event": {
+                "original": "\u00182021-01-16 11:54:25.839.2900 - [] iddiscover.exe [5356,10572] Perf: PerfPsupdate. DBCommand: {GroupMemberAny} | Duration: {6} | Event: {loaddb}",
                 "timezone": "UTC"
             },
             "hid_bravura_monitor": {
@@ -442,6 +455,7 @@
                 "version": "8.17.0"
             },
             "event": {
+                "original": "\u00182021-01-27 14:36:47.026.9701 - [] iddb.exe [4584,664] PerfReplication: PerfReplication. DBCommand: {Call} | Duration: {3951} | Function: {RequestActionUpdate} | ReceiveQueue: {node01}",
                 "timezone": "UTC"
             },
             "hid_bravura_monitor": {
@@ -478,6 +492,7 @@
                 "version": "8.17.0"
             },
             "event": {
+                "original": "\u00182021-02-04 18:03:38.605.2582 - [] kvgtodb.exe [8516,10044] PerfSproc: PerfSproc. Caller: {main} | Duration: {16087} | File: {ui\\tools\\kvgtodb.cpp} | Function: {LanguageStageMerge} | Line: {62}",
                 "timezone": "UTC"
             },
             "hid_bravura_monitor": {
@@ -515,6 +530,7 @@
                 "version": "8.17.0"
             },
             "event": {
+                "original": "\u00182021-01-16 00:35:32.958.5926 - [09bc17eb-7b1d-4b1a-8c66-42c49384a552] idarch.exe [4960,47448] Warning: Agent failed: return code [1] message [Failed: Network error: Connection timed out.  Could not connect to address [{script=\"E:\\Program Files\\Bravura Security\\Bravura Security Fabric\\default\\agent\\64\\agtlinux.psl\"; server=1.128.3.4;port=22; compression=false; hostkeys=AllowAppend; privEscType=sudo; passwdAccessOnly=true; maxReadTimeout=6; maxWriteTimeout=20; maxReadSize=16384; maxReadLines=50000; pubkeyfiles=id_rsa.pub,id_dsa.pub; }].]",
                 "timezone": "UTC"
             },
             "hid_bravura_monitor": {

packages/hid_bravura_monitor/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -183,11 +183,6 @@ processors:
           }
         }
         handleMap(ctx);
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind