[GitLab] Added application, auth, and audit datastreams

packages/gitlab/_dev/build/docs/README.md

@@ -4,6 +4,12 @@ This integration is for ingesting logs from [GitLab](https://about.gitlab.com/).
 
 - `api`: Collect logs for HTTP requests made to the GitLab API
 
+- `application`: Collect logs for events in GitLab like user creation or project deletion.
+
+- `audit`: Collect logs for changes to group or project settings and memberships.
+
+- `auth`: Collect logs for protected paths abusive requests or requests over the Rate Limit.
+
 - `production`: Collect logs for Rails controller requests received from GitLab.
 
 See the GitLab [Log system docs](https://docs.gitlab.com/ee/administration/logs/) for more information.
@@ -14,7 +20,7 @@ The GitLab module has been developed with and tested against the [community edit
 
 ## Setup
 
-Refer to the GitLab documentation for the specific filepath(s) for your instance type. Both are provided as default in the configuration setup, but only one will be needed for use. See [API](https://docs.gitlab.com/ee/administration/logs/#api_jsonlog) and [Production](https://docs.gitlab.com/ee/administration/logs/#production_jsonlog) for details. 
+Refer to the [GitLab documentation](https://docs.gitlab.com/ee/administration/logs/) for the specific filepath(s) for your instance type. Both are provided as default in the configuration setup, but only one will be needed for use.
 
 ## Logs
 
@@ -26,10 +32,34 @@ Collect logs for HTTP requests made to the GitLab API. Check out the [GitLab API
 
 {{event "api"}}
 
+### application
+
+Collect logs for events happening in GitLab like user creation or project deletion. Check out the [GitLab Application log docs](https://docs.gitlab.com/ee/administration/logs/#application_jsonlog) for more information.
+
+{{fields "application"}}
+
+{{event "application"}}
+
+### audit
+
+Collect logs for changes to group or project settings and memberships. Check out the [GitLab Audit log docs](https://docs.gitlab.com/ee/administration/logs/#audit_jsonlog) for more information.
+
+{{fields "audit"}}
+
+{{event "audit"}}
+
+### auth
+
+Collect logs for abusive protect paths requests or requests over the Rate Limit. Check out the [GitLab Auth log docs](https://docs.gitlab.com/ee/administration/logs/#auth_jsonlog) for more information.
+
+{{fields "auth"}}
+
+{{event "auth"}}
+
 ### production
 
 Collect logs for Rails controller requests received from GitLab. Check out the [GitLab production log docs](https://docs.gitlab.com/ee/administration/logs/#production_jsonlog) for more information.
 
 {{fields "production"}}
 
-{{event "production"}}
+{{event "production"}}

packages/gitlab/_dev/deploy/docker/sample_logs/test-gitlab-audit.log

@@ -0,0 +1,2 @@
+{"severity": "INFO","time": "2024-05-10T17:46:49.065Z","correlation_id": "01HXHSTQ5A02PJVVW2H8FZZ77A","meta.caller_id": "SessionsController#create","meta.remote_ip": "67.43.156.18","meta.feature_category": "system_access","meta.user": "root","meta.user_id": 1,"meta.client_id": "user/1","author_id": 1,"author_name": "Administrator","entity_id": 1,"entity_type": "User","created_at": "2024-05-10T17:46:49.058+00:00","with": "standard","target_id": 1,"target_type": "User","target_details": "Administrator"}
+{"severity": "INFO","time": "2018-10-17T17:38:22.523Z","author_id": 3,"entity_id": 2,"entity_type": "Project","change": "visibility","from": "Private","to": "Public","author_name": "John Doe4","target_id": 2,"target_type": "Project","target_details": "namespace2/project2"}

packages/gitlab/_dev/deploy/docker/sample_logs/test-gitlab-auth.log

@@ -0,0 +1,5 @@
+{"severity": "ERROR","time": "2023-04-19T22:14:25.893Z","correlation_id": "01GYDSAKAN2SPZPAMJNRWW5H8S","message": "Rack_Attack","env": "blocklist","remote_ip": "67.43.156.18","request_method": "GET","path": "/group/project.git/info/refs?service=git-upload-pack"}
+{"severity":"ERROR","time":"2024-05-31T12:24:57.330Z","correlation_id":"01HZ79RFQ9K41JQBXF49J73B49","message":"Rack_Attack","env":"throttle","remote_ip":"192.168.65.1","request_method":"GET","path":"/api/v4/projects","matched":"throttle_unauthenticated_api","status":429,"redis_calls":1,"redis_duration_s":0.000269,"redis_read_bytes":1,"redis_write_bytes":81,"redis_rate_limiting_calls":1,"redis_rate_limiting_duration_s":0.000269,"redis_rate_limiting_read_bytes":1,"redis_rate_limiting_write_bytes":81,"db_count":0,"db_write_count":0,"db_cached_count":0,"db_txn_count":0,"db_replica_txn_count":0,"db_primary_txn_count":0,"db_main_txn_count":0,"db_ci_txn_count":0,"db_main_replica_txn_count":0,"db_ci_replica_txn_count":0,"db_replica_count":0,"db_primary_count":0,"db_main_count":0,"db_ci_count":0,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_ci_cached_count":0,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_txn_duration_s":0.0,"db_primary_txn_duration_s":0.0,"db_main_txn_duration_s":0.0,"db_ci_txn_duration_s":0.0,"db_main_replica_txn_duration_s":0.0,"db_ci_replica_txn_duration_s":0.0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.0,"db_main_duration_s":0.0,"db_ci_duration_s":0.0,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.006771,"mem_objects":3752,"mem_bytes":368312,"mem_mallocs":1283,"mem_total_bytes":518392,"pid":1162,"worker_id":"puma_4","rate_limiting_gates":[]}
+{"severity":"ERROR","time":"2024-05-31T12:24:57.729Z","correlation_id":"01HZ79RG3N4NP8RY61SVCKDDND","message":"Rack_Attack","env":"throttle","remote_ip":"192.168.65.1","request_method":"GET","path":"/api/v4/projects","matched":"throttle_unauthenticated_api","status":429,"redis_calls":2,"redis_duration_s":0.000666,"redis_read_bytes":1,"redis_write_bytes":81,"redis_rate_limiting_calls":2,"redis_rate_limiting_duration_s":0.000666,"redis_rate_limiting_read_bytes":1,"redis_rate_limiting_write_bytes":81,"db_count":0,"db_write_count":0,"db_cached_count":0,"db_txn_count":0,"db_replica_txn_count":0,"db_primary_txn_count":0,"db_main_txn_count":0,"db_ci_txn_count":0,"db_main_replica_txn_count":0,"db_ci_replica_txn_count":0,"db_replica_count":0,"db_primary_count":0,"db_main_count":0,"db_ci_count":0,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_ci_cached_count":0,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_txn_duration_s":0.0,"db_primary_txn_duration_s":0.0,"db_main_txn_duration_s":0.0,"db_ci_txn_duration_s":0.0,"db_main_replica_txn_duration_s":0.0,"db_ci_replica_txn_duration_s":0.0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.0,"db_main_duration_s":0.0,"db_ci_duration_s":0.0,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.00834,"mem_objects":3823,"mem_bytes":418584,"mem_mallocs":1694,"mem_total_bytes":571504,"pid":1159,"worker_id":"puma_0","rate_limiting_gates":[]}
+{"severity":"ERROR","time":"2024-05-31T12:24:58.112Z","correlation_id":"01HZ79RGFQE862KRM3SVDZ3GSE","message":"Rack_Attack","env":"throttle","remote_ip":"192.168.65.1","request_method":"GET","path":"/api/v4/projects","matched":"throttle_unauthenticated_api","status":429,"redis_calls":1,"redis_duration_s":0.00033,"redis_read_bytes":1,"redis_write_bytes":81,"redis_rate_limiting_calls":1,"redis_rate_limiting_duration_s":0.00033,"redis_rate_limiting_read_bytes":1,"redis_rate_limiting_write_bytes":81,"db_count":0,"db_write_count":0,"db_cached_count":0,"db_txn_count":0,"db_replica_txn_count":0,"db_primary_txn_count":0,"db_main_txn_count":0,"db_ci_txn_count":0,"db_main_replica_txn_count":0,"db_ci_replica_txn_count":0,"db_replica_count":0,"db_primary_count":0,"db_main_count":0,"db_ci_count":0,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_ci_cached_count":0,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_txn_duration_s":0.0,"db_primary_txn_duration_s":0.0,"db_main_txn_duration_s":0.0,"db_ci_txn_duration_s":0.0,"db_main_replica_txn_duration_s":0.0,"db_ci_replica_txn_duration_s":0.0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.0,"db_main_duration_s":0.0,"db_ci_duration_s":0.0,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.007194,"mem_objects":3821,"mem_bytes":368312,"mem_mallocs":1281,"mem_total_bytes":521152,"pid":1142,"worker_id":"puma_2","rate_limiting_gates":[]}
+{"severity":"ERROR","time":"2024-05-31T14:39:59.837Z","correlation_id":"01HZ7HFRAEYXN6YYZ45FKBF9B8","message":"Rack_Attack","env":"throttle","remote_ip":"192.168.65.1","request_method":"POST","path":"/api/v4/users?private_token=glpat-xxxxxxxxxxxx&email=test@elastic.co&name=test&username=test","matched":"throttle_authenticated_api","status":429,"user_id":2,"meta.user":"test","redis_calls":1,"redis_duration_s":0.000288,"redis_read_bytes":1,"redis_write_bytes":73,"redis_rate_limiting_calls":1,"redis_rate_limiting_duration_s":0.000288,"redis_rate_limiting_read_bytes":1,"redis_rate_limiting_write_bytes":73,"db_count":3,"db_write_count":0,"db_cached_count":1,"db_txn_count":0,"db_replica_txn_count":0,"db_primary_txn_count":0,"db_main_txn_count":0,"db_ci_txn_count":0,"db_main_replica_txn_count":0,"db_ci_replica_txn_count":0,"db_replica_count":0,"db_primary_count":3,"db_main_count":3,"db_ci_count":0,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":1,"db_main_cached_count":1,"db_ci_cached_count":0,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_txn_duration_s":0.0,"db_primary_txn_duration_s":0.0,"db_main_txn_duration_s":0.0,"db_ci_txn_duration_s":0.0,"db_main_replica_txn_duration_s":0.0,"db_ci_replica_txn_duration_s":0.0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.001,"db_main_duration_s":0.001,"db_ci_duration_s":0.0,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.011389,"mem_objects":5166,"mem_bytes":448024,"mem_mallocs":1444,"mem_total_bytes":654664,"pid":1068,"worker_id":"puma_4","rate_limiting_gates":[]}

packages/gitlab/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: 0.3.0
+  changes:
+    - description: Add application, audit, and auth datastreams
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10644
 - version: "0.2.0"
   changes:
     - description: Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.