[microsoft_sqlserver] Add performance DataStream

packages/microsoft_sqlserver/_dev/build/build.yml

@@ -1,3 +1,3 @@
 dependencies:
   ecs:
-    reference: git@v8.2.0
+    reference: git@v8.2.1

packages/microsoft_sqlserver/_dev/deploy/docker/Dockerfile

@@ -0,0 +1,9 @@
+ARG IMAGE=${IMAGE:-mcr.microsoft.com/mssql/server:2019-latest}
+FROM ${IMAGE}
+
+ENV ACCEPT_EULA='Y'
+ENV SA_PASSWORD='1234_asdf'
+
+# Use the same healthcheck as the Windows version of the image.
+# https://github.com/Microsoft/mssql-docker/blob/a3020afeec9be1eb2d67645ac739438eb8f2c545/windows/mssql-server-windows/dockerfile#L31
+HEALTHCHECK --interval=1s --retries=90 CMD /opt/mssql-tools/bin/sqlcmd -U SA -P ${SA_PASSWORD} -Q "select 1"

packages/microsoft_sqlserver/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,10 @@
+version: '2.3'
+services:
+  microsoft_sqlserver:
+    image: mcr.microsoft.com/mssql/server:2019-latest
+    build:
+      context: .
+      args:
+        MSSQL_VERSION: ${MSSQL_VERSION:-2019-latest}
+    ports:
+      - 1433

packages/microsoft_sqlserver/_dev/deploy/variants.yml

@@ -0,0 +1,4 @@
+variants:
+  MSSQL_2019-latest:
+    IMAGE: 2019-latest
+default: 2019-latest

packages/microsoft_sqlserver/changelog.yml

@@ -1,4 +1,10 @@
 # newer versions go on top
+
+- version: "1.1.0"
+  changes:
+    - description: Added performance datastream
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3391
 - version: "1.0.0"
   changes:
     - description: Make GA

packages/microsoft_sqlserver/data_stream/performance/_dev/test/system/test-perf-config.yml

@@ -0,0 +1,5 @@
+vars:
+  hosts:
+    - "{{Hostname}}"
+  username: sa
+  password: 1234_asdf

packages/microsoft_sqlserver/data_stream/performance/agent/stream/stream.yml.hbs

@@ -0,0 +1,44 @@
+metricsets: ["query"]
+# Specify hosts in the below format. TODO: hosts need to be updated to support multiple entries.
+hosts: 
+  - sqlserver://{{username}}:{{password}}@{{hosts}}:{{port}}   
+period: {{period}}
+raw_data.enabled: true
+# Below dynamic_counter_name handles the dynamic counter name passing it to SQL query
+dynamic_counter_name: {{dynamic_counter_name}}
+driver: "mssql"
+sql_queries:
+  - query: "SELECT cntr_value As 'user_connections' FROM sys.dm_os_performance_counters WHERE counter_name= 'User Connections'"
+    response_format: table
+  - query: "SELECT cntr_value As 'active_temp_tables' FROM sys.dm_os_performance_counters WHERE counter_name = 'Active Temp Tables' AND object_name like '%General Statistics%'"
+    response_format: table
+  - query: "SELECT cntr_value As 'buffer_cache_hit_ratio' FROM sys.dm_os_performance_counters WHERE counter_name = 'Buffer cache hit ratio' AND object_name like '%Buffer Manager%'"
+    response_format: table
+  - query: "SELECT cntr_value As 'page_splits_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Page splits/sec'"
+    response_format: table
+  - query: "SELECT cntr_value As 'lock_waits_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Lock Waits/sec' AND instance_name = '_Total'"
+    response_format: table
+  - query: "SELECT cntr_value As 'compilations_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'SQL Compilations/sec'"
+    response_format: table
+  - query: "SELECT cntr_value As 'batch_requests_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Batch Requests/sec'"
+    response_format: table
+  - query: "SELECT cntr_value As 'buffer_checkpoint_pages_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Checkpoint pages/sec' AND object_name like '%Buffer Manager%'"
+    response_format: table
+  - query: "SELECT cntr_value As 'buffer_database_pages' FROM sys.dm_os_performance_counters WHERE counter_name = 'Database pages' AND object_name like '%Buffer Manager%'"
+    response_format: table
+  - query: "SELECT cntr_value As 'buffer_page_life_expectancy' FROM sys.dm_os_performance_counters WHERE counter_name = 'Page life expectancy' AND  object_name like '%Buffer Manager%'"
+    response_format: table
+  - query: "SELECT cntr_value As 'buffer_target_pages' FROM sys.dm_os_performance_counters WHERE counter_name = 'Target pages' AND  object_name like '%Buffer Manager%'"
+    response_format: table
+  - query: "SELECT cntr_value As 'connection_reset_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Connection Reset/sec' AND object_name like '%Buffer Manager%'"
+    response_format: table
+  - query: "SELECT cntr_value As 'logins_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Logins/sec' AND object_name like '%General Statistics%'"
+    response_format: table
+  - query: "SELECT cntr_value As 'logouts_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Logouts/sec' AND object_name like '%General Statistics%'"
+    response_format: table
+  - query: "SELECT cntr_value As 'transactions' FROM sys.dm_os_performance_counters WHERE counter_name = 'Transactions' AND object_name like '%General Statistics%'"
+    response_format: table
+  - query: "SELECT cntr_value As 're_compilations_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'SQL Re-Compilations/sec'"
+    response_format: table
+  - query: "SELECT counter_name As 'dynamic_counter.name', cntr_value As 'dynamic_counter.value' FROM sys.dm_os_performance_counters WHERE counter_name= '{{dynamic_counter_name}}'"
+    response_format: table

packages/microsoft_sqlserver/data_stream/performance/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,20 @@
+---
+description: Pipeline for processing mssql performance
+processors:
+- remove:
+    field: sql.driver
+    ignore_missing: true
+    ignore_failure: true
+- remove:
+    field: sql.query
+    ignore_missing: true
+    ignore_failure: true
+- rename:
+    field: sql
+    target_field: mssql
+    ignore_missing: true
+    ignore_failure: true
+on_failure:
+- set:
+    field: error.message
+    value: "{{ _ingest.on_failure_message }}"

packages/microsoft_sqlserver/data_stream/performance/fields/base-fields.yml

@@ -0,0 +1,8 @@
+- name: data_stream.type
+  external: ecs
+- name: data_stream.dataset
+  external: ecs
+- name: data_stream.namespace
+  external: ecs
+- name: '@timestamp'
+  external: ecs

packages/microsoft_sqlserver/data_stream/performance/fields/ecs.yml

@@ -0,0 +1,6 @@
+- external: ecs
+  name: ecs.version
+- external: ecs
+  name: service.address
+- external: ecs
+  name: service.type

packages/microsoft_sqlserver/data_stream/performance/fields/fields.yml

@@ -0,0 +1,70 @@
+- name: mssql
+  type: group
+  release: beta
+  fields:
+    - name: metrics
+      type: group
+      fields:
+        - name: page_splits_per_sec
+          type: float
+          metric_type: gauge
+          description: Number of page splits per second that occur as the result of overflowing index pages.
+        - name: lock_waits_per_sec
+          type: float
+          metric_type: gauge
+          description: Number of lock requests per second that required the caller to wait.
+        - name: user_connections
+          type: long
+          description: Total number of user connections.
+        - name: active_temp_tables
+          type: long
+          description: Number of temporary tables/table variables in use.
+        - name: transactions
+          type: long
+          description: Total number of transactions
+        - name: connection_reset_per_sec
+          type: float
+          metric_type: gauge
+          description: Total number of logins started per second from the connection pool.
+        - name: logins_per_sec
+          type: float
+          metric_type: gauge
+          description: Total number of logins started per second. This does not include pooled connections.
+        - name: logouts_per_sec
+          type: float
+          metric_type: gauge
+          description: Total number of logout operations started per second.
+        - name: re_compilations_per_sec
+          type: float
+          metric_type: gauge
+          description: Number of statement recompiles per second. Counts the number of times statement recompiles are triggered. Generally, you want the recompiles to be low.
+        - name: compilations_per_sec
+          type: float
+          metric_type: gauge
+          description: Number of SQL compilations per second. Indicates the number of times the compile code path is entered. Includes compiles caused by statement-level recompilations in SQL Server. After SQL Server user activity is stable, this value reaches a steady state.
+        - name: batch_requests_per_sec
+          type: float
+          metric_type: gauge
+          description: Number of Transact-SQL command batches received per second. This statistic is affected by all constraints (such as I/O, number of users, cache size, complexity of requests, and so on). High batch requests mean good throughput.
+        - name: buffer_cache_hit_ratio
+          type: double
+          description: The ratio is the total number of cache hits divided by the total number of cache lookups over the last few thousand page accesses. After a long period of time, the ratio moves very little. Because reading from the cache is much less expensive than reading from disk, you want this ratio to be high.
+        - name: buffer_page_life_expectancy
+          type: long
+          description: Indicates the number of seconds a page will stay in the buffer pool without references (in seconds).
+        - name: buffer_checkpoint_pages_per_sec
+          type: float
+          metric_type: gauge
+          description: Indicates the number of pages flushed to disk per second by a checkpoint or other operation that require all dirty pages to be flushed.
+        - name: buffer_database_pages
+          type: long
+          description: Indicates the number of pages in the buffer pool with database content.
+        - name: buffer_target_pages
+          type: long
+          description: Ideal number of pages in the buffer pool.
+        - name: dynamic_counter.value
+          type: long
+          description: Dynamic counter value is fetched from performance table for the dynamic counter name which is provided by user.
+        - name: dynamic_counter.name
+          type: keyword
+          description: Dynamic counter name is given by user.

packages/microsoft_sqlserver/data_stream/performance/manifest.yml

@@ -0,0 +1,24 @@
+type: metrics
+title: Microsoft SQL Server performance metrics
+release: beta
+streams:
+  - input: sql/metrics
+    enabled: false
+    vars:
+      - name: period
+        type: text
+        title: Period
+        multi: false
+        required: true
+        show_user: true
+        default: 60s
+      - name: dynamic_counter_name
+        type: text
+        title: Dynamic Counter Name
+        multi: false
+        required: false
+        show_user: true
+        default: Memory Grants Pending
+        description: Collect the dynamic counter value for the provided counter name from the performance table.
+    title: Microsoft SQL Server performance metrics
+    description: Collect Microsoft SQL Server performance metrics

packages/microsoft_sqlserver/data_stream/performance/sample_event.json

@@ -0,0 +1,79 @@
+{
+    "@timestamp": "2022-06-08T13:35:05.558Z",
+    "agent": {
+        "ephemeral_id": "16ad2de8-8ba3-496f-98d1-cbe19441c168",
+        "id": "848cea0e-c052-49b3-983d-64e13d3b9a6f",
+        "name": "docker-fleet-agent",
+        "type": "metricbeat",
+        "version": "8.3.0"
+    },
+    "cloud": {
+        "account": {},
+        "instance": {
+            "id": "b30e45e6-7900-4900-8d67-e37cb13374bc",
+            "name": "obs-int-windows-dev"
+        },
+        "machine": {
+            "type": "Standard_D16ds_v5"
+        },
+        "provider": "azure",
+        "region": "CentralIndia",
+        "service": {
+            "name": "Virtual Machines"
+        }
+    },
+    "data_stream": {
+        "dataset": "microsoft_sqlserver.performance",
+        "namespace": "ep",
+        "type": "metrics"
+    },
+    "ecs": {
+        "version": "8.0.0"
+    },
+    "elastic_agent": {
+        "id": "848cea0e-c052-49b3-983d-64e13d3b9a6f",
+        "snapshot": true,
+        "version": "8.3.0"
+    },
+    "event": {
+        "agent_id_status": "verified",
+        "dataset": "microsoft_sqlserver.performance",
+        "duration": 7151724,
+        "ingested": "2022-06-08T13:35:06Z",
+        "module": "sql"
+    },
+    "host": {
+        "architecture": "x86_64",
+        "containerized": true,
+        "hostname": "docker-fleet-agent",
+        "ip": [
+            "172.18.0.4"
+        ],
+        "mac": [
+            "02:42:ac:12:00:04"
+        ],
+        "name": "docker-fleet-agent",
+        "os": {
+            "codename": "focal",
+            "family": "debian",
+            "kernel": "5.10.16.3-microsoft-standard-WSL2",
+            "name": "Ubuntu",
+            "platform": "ubuntu",
+            "type": "linux",
+            "version": "20.04.4 LTS (Focal Fossa)"
+        }
+    },
+    "metricset": {
+        "name": "query",
+        "period": 60000
+    },
+    "mssql": {
+        "metrics": {
+            "user_connections": 1
+        }
+    },
+    "service": {
+        "address": "elastic-package-service-microsoft_sqlserver-1:1433",
+        "type": "sql"
+    }
+}