[GCP] [CloudSQL] Fix metrics and fields

packages/gcp/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.29.0"
+  changes:
+    - description: Remove GCP CloudSQL deprecated, alpha or beta metrics and fix field types.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/7367
 - version: "2.28.5"
   changes:
     - description: Set metric type for GKE, Load Balancing, PubSub, Redis and Storage data streams.

packages/gcp/data_stream/cloudsql_mysql/agent/stream/stream.yml.hbs

@@ -15,15 +15,12 @@ metrics:
   - aligner: ALIGN_NONE
     service: cloudsql
     metric_types:
-      - "database/active_directory/domain_reachable"
-      - "database/active_directory/instance_available"
       - "database/auto_failover_request_count"
       - "database/available_for_failover"
       - "database/cpu/reserved_cores"
       - "database/cpu/usage_time"
       - "database/cpu/utilization"
       - "database/disk/bytes_used"
-      - "database/disk/bytes_used_by_data_type"
       - "database/disk/quota"
       - "database/disk/read_ops_count"
       - "database/disk/utilization"
@@ -54,11 +51,7 @@ metrics:
       - "database/network/connections"
       - "database/network/received_bytes_count"
       - "database/network/sent_bytes_count"
-      - "database/replication/log_archive_failure_count"
-      - "database/replication/log_archive_success_count"
       - "database/replication/network_lag"
       - "database/replication/replica_lag"
-      - "database/replication/state"
-      - "database/state"
       - "database/up"
       - "database/uptime"

packages/gcp/data_stream/cloudsql_mysql/elasticsearch/ingest_pipeline/default.yml

@@ -4,14 +4,6 @@ processors:
   - drop:
       description: Drop if database is not MySQL.
       if: "ctx?.gcp?.labels?.cloudsql?.name != 'mysql'"
-  - rename:
-      field: gcp.metrics.database.active_directory.domain_reachable.value
-      target_field: gcp.cloudsql_mysql.database.active_directory.domain_reachable
-      ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.active_directory.instance_available.value
-      target_field: gcp.cloudsql_mysql.database.active_directory.instance_available
-      ignore_missing: true
   - rename:
       field: gcp.metrics.database.auto_failover_request_count.value
       target_field: gcp.cloudsql_mysql.database.auto_failover_request.count
@@ -36,10 +28,6 @@ processors:
       field: gcp.metrics.database.disk.bytes_used.value
       target_field: gcp.cloudsql_mysql.database.disk.bytes_used.bytes
       ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.disk.bytes_used_by_data_type.value
-      target_field: gcp.cloudsql_mysql.database.disk.bytes_used_by_data_type.bytes
-      ignore_missing: true
   - rename:
       field: gcp.metrics.database.disk.quota.value
       target_field: gcp.cloudsql_mysql.database.disk.quota.bytes
@@ -160,14 +148,6 @@ processors:
       field: gcp.metrics.database.network.sent_bytes_count.value
       target_field: gcp.cloudsql_mysql.database.network.sent_bytes.count
       ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.replication.log_archive_failure_count.value
-      target_field: gcp.cloudsql_mysql.database.replication.log_archive_failure.count
-      ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.replication.log_archive_success_count.value
-      target_field: gcp.cloudsql_mysql.database.replication.log_archive_success.count
-      ignore_missing: true
   - rename:
       field: gcp.metrics.database.replication.network_lag.value
       target_field: gcp.cloudsql_mysql.database.replication.network_lag.sec
@@ -176,14 +156,6 @@ processors:
       field: gcp.metrics.database.replication.replica_lag.value
       target_field: gcp.cloudsql_mysql.database.replication.replica_lag.sec
       ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.replication.state.value
-      target_field: gcp.cloudsql_mysql.database.replication.state
-      ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.state.value
-      target_field: gcp.cloudsql_mysql.database.state
-      ignore_missing: true
   - rename:
       field: gcp.metrics.database.up.value
       target_field: gcp.cloudsql_mysql.database.up

packages/gcp/data_stream/cloudsql_mysql/fields/fields.yml

@@ -2,14 +2,6 @@
   description: Google Cloud CloudSQL MySQL metrics
   type: group
   fields:
-    - name: database.active_directory.domain_reachable
-      type: long
-      metric_type: gauge
-      description: Indicates whether the instance is able to ping a domain controller from the connected Managed Active Directory domain.
-    - name: database.active_directory.instance_available
-      type: long
-      metric_type: gauge
-      description: Indicates whether the instance is currently available using Windows Authentication.
     - name: database.auto_failover_request.count
       type: long
       metric_type: gauge
@@ -37,11 +29,6 @@
       unit: byte
       metric_type: gauge
       description: Data utilization in bytes.
-    - name: database.disk.bytes_used_by_data_type.bytes
-      type: long
-      unit: byte
-      metric_type: gauge
-      description: Data utilization in bytes.
     - name: database.disk.quota.bytes
       type: long
       unit: byte
@@ -61,8 +48,7 @@
       metric_type: gauge
       description: Delta count of data disk write IO operations.
     - name: database.instance_state
-      type: long
-      metric_type: gauge
+      type: boolean
       description: The current serving state of the Cloud SQL instance.
     - name: database.memory.quota.bytes
       type: long
@@ -141,15 +127,13 @@
       type: keyword
       description: Indicates whether the I/O thread for reading the primary's binary log is running. Possible values are Yes, No and Connecting.
     - name: database.replication.slave_io_running_state
-      type: long
-      metric_type: gauge
+      type: boolean
       description: Indicates whether the I/O thread for reading the primary's binary log is running. Possible values are Yes, No and Connecting, and the values are exposed through the 'state' field.
     - name: database.replication.slave_sql_running
       type: keyword
       description: Indicates whether the SQL thread for executing events in the relay log is running.
     - name: database.replication.slave_sql_running_state
-      type: long
-      metric_type: gauge
+      type: boolean
       description: Indicates whether the SQL thread for executing events in the relay log is running. Possible values are Yes / No, and the values are exposed through the 'state' field.
     - name: database.sent_bytes.count
       type: long
@@ -167,14 +151,6 @@
       type: long
       metric_type: gauge
       description: Delta count of bytes sent through the network.
-    - name: database.replication.log_archive_failure.count
-      type: long
-      metric_type: gauge
-      description: Delta number of failed attempts for archiving replication log files.
-    - name: database.replication.log_archive_success.count
-      type: long
-      metric_type: gauge
-      description: Delta number of successful attempts for archiving replication log files.
     - name: database.replication.network_lag.sec
       type: long
       unit: s
@@ -185,13 +161,6 @@
       unit: s
       metric_type: gauge
       description: Number of seconds the read replica is behind its primary (approximation).
-    - name: database.replication.state
-      type: long
-      metric_type: gauge
-      description: The current serving state of replication.
-    - name: database.state
-      type: keyword
-      description: The current serving state of the Cloud SQL instance.
     - name: database.up
       type: long
       metric_type: gauge

packages/gcp/data_stream/cloudsql_postgresql/agent/stream/stream.yml.hbs

@@ -15,15 +15,12 @@ metrics:
   - aligner: ALIGN_NONE
     service: cloudsql
     metric_types:
-      - "database/active_directory/domain_reachable"
-      - "database/active_directory/instance_available"
       - "database/auto_failover_request_count"
       - "database/available_for_failover"
       - "database/cpu/reserved_cores"
       - "database/cpu/usage_time"
       - "database/cpu/utilization"
       - "database/disk/bytes_used"
-      - "database/disk/bytes_used_by_data_type"
       - "database/disk/quota"
       - "database/disk/read_ops_count"
       - "database/disk/utilization"
@@ -36,10 +33,6 @@ metrics:
       - "database/network/connections"
       - "database/network/received_bytes_count"
       - "database/network/sent_bytes_count"
-      - "database/postgresql/blocks_read_count"
-      - "database/postgresql/deadlock_count"
-      - "database/postgresql/external_sync/initial_sync_complete"
-      - "database/postgresql/external_sync/max_replica_byte_lag"
       - "database/postgresql/insights/aggregate/execution_time"
       - "database/postgresql/insights/aggregate/io_time"
       - "database/postgresql/insights/aggregate/latencies"
@@ -59,19 +52,12 @@ metrics:
       - "database/postgresql/insights/pertag/row_count"
       - "database/postgresql/insights/pertag/shared_blk_access_count"
       - "database/postgresql/num_backends"
-      - "database/postgresql/num_backends_by_state"
       - "database/postgresql/replication/replica_byte_lag"
       - "database/postgresql/transaction_count"
       - "database/postgresql/transaction_id_count"
       - "database/postgresql/transaction_id_utilization"
-      - "database/postgresql/tuple_size"
-      - "database/postgresql/tuples_processed_count"
       - "database/postgresql/vacuum/oldest_transaction_age"
-      - "database/replication/log_archive_failure_count"
-      - "database/replication/log_archive_success_count"
       - "database/replication/network_lag"
       - "database/replication/replica_lag"
-      - "database/replication/state"
-      - "database/state"
       - "database/up"
       - "database/uptime"

packages/gcp/data_stream/cloudsql_postgresql/elasticsearch/ingest_pipeline/default.yml

@@ -4,14 +4,6 @@ processors:
   - drop:
       description: Drop if database is not PostgreSQL.
       if: "ctx?.gcp?.labels?.cloudsql?.name != 'postgres'"
-  - rename:
-      field: gcp.metrics.database.active_directory.domain_reachable.value
-      target_field: gcp.cloudsql_postgresql.database.active_directory.domain_reachable
-      ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.active_directory.instance_available.value
-      target_field: gcp.cloudsql_postgresql.database.active_directory.instance_available
-      ignore_missing: true
   - rename:
       field: gcp.metrics.database.auto_failover_request_count.value
       target_field: gcp.cloudsql_postgresql.database.auto_failover_request.count
@@ -36,10 +28,6 @@ processors:
       field: gcp.metrics.database.disk.bytes_used.value
       target_field: gcp.cloudsql_postgresql.database.disk.bytes_used.bytes
       ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.disk.bytes_used_by_data_type.value
-      target_field: gcp.cloudsql_postgresql.database.disk.bytes_used_by_data_type.bytes
-      ignore_missing: true
   - rename:
       field: gcp.metrics.database.disk.quota.value
       target_field: gcp.cloudsql_postgresql.database.disk.quota.bytes
@@ -88,22 +76,6 @@ processors:
       field: gcp.metrics.database.network.sent_bytes_count.value
       target_field: gcp.cloudsql_postgresql.database.network.sent_bytes.count
       ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.postgresql.blocks_read_count.value
-      target_field: gcp.cloudsql_postgresql.database.blocks_read.count
-      ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.postgresql.deadlock_count.value
-      target_field: gcp.cloudsql_postgresql.database.deadlock.count
-      ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.postgresql.external_sync.initial_sync_complete.value
-      target_field: gcp.cloudsql_postgresql.database.external_sync.initial_sync_complete
-      ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.postgresql.external_sync.max_replica_byte_lag.value
-      target_field: gcp.cloudsql_postgresql.database.external_sync.max_replica_byte_lag.bytes
-      ignore_missing: true
   - rename:
       field: gcp.metrics.database.postgresql.insights.aggregate.execution_time.value
       target_field: gcp.cloudsql_postgresql.database.insights.aggregate.execution_time
@@ -180,10 +152,6 @@ processors:
       field: gcp.metrics.database.postgresql.num_backends.value
       target_field: gcp.cloudsql_postgresql.database.num_backends.count
       ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.postgresql.num_backends_by_state.value
-      target_field: gcp.cloudsql_postgresql.database.num_backends_by_state.count
-      ignore_missing: true
   - rename:
       field: gcp.metrics.database.postgresql.replication.replica_byte_lag.value
       target_field: gcp.cloudsql_postgresql.database.replication.replica_byte_lag.bytes
@@ -200,26 +168,10 @@ processors:
       field: gcp.metrics.database.postgresql.transaction_id_utilization.value
       target_field: gcp.cloudsql_postgresql.database.transaction_id_utilization.pct
       ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.postgresql.tuple_size.value
-      target_field: gcp.cloudsql_postgresql.database.tuple_size.count
-      ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.postgresql.tuples_processed_count.value
-      target_field: gcp.cloudsql_postgresql.database.tuples_processed.count
-      ignore_missing: true
   - rename:
       field: gcp.metrics.database.postgresql.vacuum.oldest_transaction_age.value
       target_field: gcp.cloudsql_postgresql.database.vacuum.oldest_transaction_age
       ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.replication.log_archive_failure_count.value
-      target_field: gcp.cloudsql_postgresql.database.replication.log_archive_failure.count
-      ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.replication.log_archive_success_count.value
-      target_field: gcp.cloudsql_postgresql.database.replication.log_archive_success.count
-      ignore_missing: true
   - rename:
       field: gcp.metrics.database.replication.network_lag.value
       target_field: gcp.cloudsql_postgresql.database.replication.network_lag.sec
@@ -228,14 +180,6 @@ processors:
       field: gcp.metrics.database.replication.replica_lag.value
       target_field: gcp.cloudsql_postgresql.database.replication.replica_lag.sec
       ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.replication.state.value
-      target_field: gcp.cloudsql_postgresql.database.replication.state
-      ignore_missing: true
-  - rename:
-      field: gcp.metrics.database.state.value
-      target_field: gcp.cloudsql_postgresql.database.state
-      ignore_missing: true
   - rename:
       field: gcp.metrics.database.up.value
       target_field: gcp.cloudsql_postgresql.database.up