[gcp] Update DNS Logs

packages/gcp/_dev/build/docs/README.md

@@ -1,4 +1,4 @@
-# Google Cloud Integration
+# Google Cloud Platform Integration
 
 The Google Cloud integration collects and parses Google Cloud [Audit Logs](https://cloud.google.com/logging/docs/audit), [VPC Flow Logs](https://cloud.google.com/vpc/docs/using-flow-logs), [Firewall Rules Logs](https://cloud.google.com/vpc/docs/firewall-rules-logging) and [Cloud DNS Logs](https://cloud.google.com/dns/docs/monitoring) that have been exported from Cloud Logging to a Google Pub/Sub topic sink.
 
@@ -110,19 +110,16 @@ More example filters for different log types:
 resource.type="gce_subnetwork" AND
 log_id("compute.googleapis.com/vpc_flows") AND
 resource.labels.subnetwork_name"=[SUBNET_NAME]"
-
 #
 # Audit: Google Compute Engine firewall rule deletion
 #
 resource.type="gce_firewall_rule" AND
 log_id("cloudaudit.googleapis.com/activity") AND
 protoPayload.methodName:"firewalls.delete"
-
 #
 # DNS: all DNS queries
 #
 resource.type="dns_query"
-
 #
 # Firewall: logs for a given country
 #

packages/gcp/_dev/build/docs/dns.md

@@ -6,4 +6,4 @@ The `dns` dataset collects queries that name servers resolve for your Virtual Pr
 
 {{fields "dns"}}
 
-{{event "dns"}}
+{{event "dns"}}

packages/gcp/_dev/deploy/docker/sample_logs/dns.log

@@ -3,3 +3,7 @@
 {"insertId":"3c0hhve794jt","jsonPayload":{"authAnswer":true,"protocol":"UDP","queryName":"elastic.co.","queryType":"MX","rdata":"elastic.co.\t300\tIN\tmx\t1 aspmx.l.google.com.\nelastic.co.\t300\tIN\tmx\t10 alt3.aspmx.l.google.com.\nelastic.co.\t300\tIN\tmx\t10 alt4.aspmx.l.google.com.\nelastic.co.\t300\tIN\tmx\t5 alt1.aspmx.l.google.com.\nelastic.co.\t300\tIN\tmx\t5 alt2.aspmx.l.google.com.","responseCode":"NOERROR","serverLatency":8,"sourceIP":"10.154.0.3","sourceNetwork":"default","vmInstanceId":8340998530665147,"vmInstanceIdString":"8340998530665147","vmInstanceName":"694119234537.instance","vmProjectId":"project","vmZoneName":"europe-west2-a"},"logName":"projects/project/logs/dns.googleapis.com%2Fdns_queries","receiveTimestamp":"2022-01-23T09:16:13.097205893Z","resource":{"labels":{"location":"europe-west2","project_id":"project","source_type":"gce-vm","target_name":"","target_type":"external"},"type":"dns_query"},"severity":"INFO","timestamp":"2022-01-23T09:16:11.834672077Z"}
 {"insertId":"1de2pp0e8q6k","jsonPayload":{"authAnswer":true,"protocol":"UDP","queryName":"elastic.co.","queryType":"NS","rdata":"elastic.co.\t21600\tIN\tns\tns-1168.awsdns-18.org.\nelastic.co.\t21600\tIN\tns\tns-1737.awsdns-25.co.uk.\nelastic.co.\t21600\tIN\tns\tns-339.awsdns-42.com.\nelastic.co.\t21600\tIN\tns\tns-785.awsdns-34.net.","responseCode":"NOERROR","serverLatency":9,"sourceIP":"10.154.0.3","sourceNetwork":"default","vmInstanceId":8340998530665147,"vmInstanceIdString":"8340998530665147","vmInstanceName":"694119234537.instance","vmProjectId":"project","vmZoneName":"europe-west2-a"},"logName":"projects/project/logs/dns.googleapis.com%2Fdns_queries","receiveTimestamp":"2022-01-23T09:16:14.900251293Z","resource":{"labels":{"location":"europe-west2","project_id":"project","source_type":"gce-vm","target_name":"","target_type":"external"},"type":"dns_query"},"severity":"INFO","timestamp":"2022-01-23T09:16:14.334350623Z"}
 {"insertId":"1oht95te9dnf","jsonPayload":{"authAnswer":true,"protocol":"UDP","queryName":"elastic.co.","queryType":"TXT","rdata":"elastic.co.\t300\tIN\ttxt\t\"adobe-idp-site-verification=f0ae487a2bd7cc4eb1fc64162e1a85c5b39002bc5fe0d7d7373ece0fdb3fd494\"\nelastic.co.\t300\tIN\ttxt\t\"atlassian-domain-verification=eFQHgCgWimOfFj3Ol7kfJg9RGDmaS8X6V2M6YZRgnEY6/iKi0SUYWRlOvBOmgV7H\"\nelastic.co.\t300\tIN\ttxt...","responseCode":"NOERROR","serverLatency":12,"sourceIP":"10.154.0.3","sourceNetwork":"default","vmInstanceId":8340998530665147,"vmInstanceIdString":"8340998530665147","vmInstanceName":"694119234537.instance","vmProjectId":"project","vmZoneName":"europe-west2-a"},"logName":"projects/project/logs/dns.googleapis.com%2Fdns_queries","receiveTimestamp":"2022-01-23T09:16:17.932779800Z","resource":{"labels":{"location":"europe-west2","project_id":"project","source_type":"gce-vm","target_name":"","target_type":"external"},"type":"dns_query"},"severity":"INFO","timestamp":"2022-01-23T09:16:17.088350121Z"}
+{"insertId":"zir4wud11tm","jsonPayload":{"sourceNetwork":"","destinationIP":"216.239.32.106","authAnswer":true,"queryName":"asdf.gcp.example.com.","protocol":"UDP","serverLatency":0,"responseCode":"NOERROR","structuredRdata":[{"domainName":"asdf.gcp.example.com.","rvalue":"67.43.156.13","type":"A","ttl":"300","class":"IN"}],"queryType":"A"},"resource":{"type":"dns_query","labels":{"location":"global","source_type":"internet","target_name":"test","project_id":"key-reference-123456","target_type":"public-zone"}},"timestamp":"2021-12-12T15:59:40.446715677Z","severity":"INFO","logName":"projects/key-reference-123456/logs/dns.googleapis.com%2Fdns_queries","receiveTimestamp":"2021-12-12T15:59:41.661374915Z"}
+{"insertId":"xkngkqcseu","jsonPayload":{"serverLatency":0,"responseCode":"NOERROR","structuredRdata":[],"protocol":"UDP","authAnswer":true,"destinationIP":"2001:4860:4802:34::6a","queryName":"asdf.gcp.example.com.","queryType":"NS","sourceNetwork":""},"resource":{"type":"dns_query","labels":{"target_type":"public-zone","source_type":"internet","project_id":"key-reference-123456","target_name":"test","location":"global"}},"timestamp":"2021-12-12T15:59:45.374096907Z","severity":"INFO","logName":"projects/key-reference-123456/logs/dns.googleapis.com%2Fdns_queries","receiveTimestamp":"2021-12-12T15:59:46.164778330Z"}
+{"insertId":"a65crc5oa","jsonPayload":{"authAnswer":true,"queryType":"A","destinationIP":"2001:4860:4802:34::6a","structuredRdata":[{"class":"IN","ttl":"300","type":"A","domainName":"asdf.gcp.example.com.","rvalue":"67.43.156.13"}],"serverLatency":1,"queryName":"asdf.gcp.example.com.","protocol":"UDP","sourceNetwork":"","responseCode":"NOERROR"},"resource":{"type":"dns_query","labels":{"target_name":"test","target_type":"public-zone","project_id":"key-reference-123456","source_type":"internet","location":"global"}},"timestamp":"2021-12-12T16:00:30.142344803Z","severity":"INFO","logName":"projects/key-reference-123456/logs/dns.googleapis.com%2Fdns_queries","receiveTimestamp":"2021-12-12T16:00:31.510661988Z"}
+{"insertId":"2s5jacc2sx","jsonPayload":{"structuredRdata":[],"protocol":"UDP","destinationIP":"216.239.38.106","responseCode":"NOERROR","sourceNetwork":"","serverLatency":0,"authAnswer":true,"queryType":"A","queryName":"gcp.example.com."},"resource":{"type":"dns_query","labels":{"target_type":"public-zone","location":"global","target_name":"test","project_id":"key-reference-123456","source_type":"internet"}},"timestamp":"2021-12-12T16:01:46.524989771Z","severity":"INFO","logName":"projects/key-reference-123456/logs/dns.googleapis.com%2Fdns_queries","receiveTimestamp":"2021-12-12T16:01:47.880199613Z"}

packages/gcp/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.3.0"
+  changes:
+    - description: Add additional parsing for DNS Public Zone Query Logs
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2340
 - version: "2.2.1"
   changes:
     - description: Fix Billing policy template title and default period for gcp.compute

packages/gcp/data_stream/audit/_dev/test/pipeline/test-common-config.yml

@@ -1,5 +1,3 @@
-dynamic_fields:
-  event.ingested: ".*"
 fields:
   tags:
     - preserve_original_event