[Kafka] New custom input package

.github/CODEOWNERS

@@ -65,6 +65,7 @@
 /packages/juniper_netscreen @elastic/security-external-integrations
 /packages/juniper_srx @elastic/security-external-integrations
 /packages/kafka @elastic/integrations
+/packages/kafka_log @elastic/obs-service-integrations
 /packages/keycloak @elastic/security-external-integrations
 /packages/kibana @elastic/integrations
 /packages/kubernetes @elastic/obs-cloudnative-monitoring

packages/kafka_log/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: git@8.0

packages/kafka_log/_dev/build/docs/README.md

@@ -0,0 +1,13 @@
+# Custom Kafka Log integration
+
+The custom Kafka log integration is used to read from topics in a Kafka cluster.
+
+To configure this integration, specify a list of one or more hosts in the cluster to bootstrap the connection with, a list of topics to track, and a group_id for the connection.
+
+
+## Compatibility
+This Integration works with all Kafka versions in between 0.11 and 2.8.0. Older versions might work as well, but are not supported.
+
+
+## Ingest Pipelines
+Custom ingest pipelines may be added by adding the name to the pipeline configuration option, creating custom ingest pipelines can be done either through the API or the [Ingest Node Pipeline UI](/app/management/ingest/ingest_pipelines/).

packages/kafka_log/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,32 @@
+version: '2.3'
+services:
+  kafka-service:
+    image: bashj79/kafka-kraft
+    healthcheck:
+      test: nc -z kafka-service 9094 || exit -1
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+    hostname: kafka-service
+    environment:
+      KAFKA_LISTENERS: "INTERNAL://kafka-service:9092,EXTERNAL://:9094, CONTROLLER://:9093"
+      KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka-service:9092,EXTERNAL://kafka-service:9094"
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT, CONTROLLER:PLAINTEXT"
+      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
+    ports:
+      - 9094
+  kafka-generic:
+    image: docker.elastic.co/observability/stream:v0.7.0
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    command:
+      - log
+      - --retry=30
+      - --addr=kafka-service:9094
+      - -p=kafka
+      - --kafka-topic=testTopic
+      - /sample_logs/testdata.log
+    depends_on:
+      kafka-service:
+        condition: service_healthy

packages/kafka_log/_dev/deploy/docker/sample_logs/testdata.log

@@ -0,0 +1,2 @@
+{"insertId":"ut8lbrffooxyw","jsonPayload":{"bytes_sent":"1776","connection":{"dest_ip":"67.43.156.13","dest_port":33478,"protocol":6,"src_ip":"10.87.40.76","src_port":5601},"dest_location":{"asn":15169,"continent":"America","country":"usa"},"end_time":"2019-06-14T03:45:37.301953198Z","packets_sent":"7","reporter":"SRC","rtt_msec":"36","src_instance":{"project_id":"my-sample-project","region":"us-east1","vm_name":"kibana","zone":"us-east1-b"},"src_vpc":{"project_id":"my-sample-project","subnetwork_name":"default","vpc_name":"default"},"start_time":"2019-06-14T03:45:37.186193305Z"},"logName":"projects/my-sample-project/logs/compute.googleapis.com%2Fvpc_flows","receiveTimestamp":"2019-06-14T03:50:10.845445834Z","resource":{"labels":{"location":"us-east1-b","project_id":"my-sample-project","subnetwork_id":"758019854043528829","subnetwork_name":"default"},"type":"gce_subnetwork"},"timestamp":"2019-06-14T03:50:10.845445834Z"}
+{"insertId":"ut8lbrffooxzb","jsonPayload":{"bytes_sent":"173663","connection":{"dest_ip":"10.87.40.76","dest_port":33970,"protocol":6,"src_ip":"67.43.156.14","src_port":9200},"dest_instance":{"project_id":"my-sample-project","region":"us-east1","vm_name":"kibana","zone":"us-east1-b"},"dest_vpc":{"project_id":"my-sample-project","subnetwork_name":"default","vpc_name":"default"},"end_time":"2019-06-14T03:49:51.821302149Z","packets_sent":"68","reporter":"DEST","rtt_msec":"1","src_instance":{"project_id":"my-sample-project","region":"us-east1","vm_name":"elasticsearch","zone":"us-east1-b"},"src_location":{"asn":15169,"continent":"America","country":"usa"},"src_vpc":{"project_id":"my-sample-project","subnetwork_name":"default","vpc_name":"default"},"start_time":"2019-06-14T03:40:08.466657665Z"},"logName":"projects/my-sample-project/logs/compute.googleapis.com%2Fvpc_flows","receiveTimestamp":"2019-06-14T03:50:10.845445834Z","resource":{"labels":{"location":"us-east1-b","project_id":"my-sample-project","subnetwork_id":"758019854043528829","subnetwork_name":"default"},"type":"gce_subnetwork"},"timestamp":"2019-06-14T03:50:10.845445834Z"}

packages/kafka_log/changelog.yml

@@ -0,0 +1,5 @@
+- version: "1.0.0"
+  changes:
+    - description: Initial Release
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2741

packages/kafka_log/data_stream/generic/_dev/test/system/test-kafka-config.yml

@@ -0,0 +1,9 @@
+service: kafka-service
+input: kafka
+data_stream:
+  vars:
+    topics:
+      - testTopic
+    hosts:
+      - "{{Hostname}}:{{Port}}"
+    group_id: system_test

packages/kafka_log/data_stream/generic/agent/stream/kafka.yml.hbs

@@ -0,0 +1,118 @@
+data_stream:
+  dataset: {{data_stream.dataset}}
+{{#if pipeline}}
+pipeline: {{pipeline}}
+{{/if}}
+hosts:
+{{#each hosts as |host i|}}
+  - {{host}}
+{{/each}}
+topics:
+{{#each topics as |topic i|}}
+  - {{topic}}
+{{/each}}
+{{#if group_id}}
+group_id: {{group_id}}
+{{/if}}
+{{#if client_id}}
+client_id: {{client_id}}
+{{/if}}
+{{#if username}}
+username: {{username}}
+{{/if}}
+{{#if password}}
+password: {{password}}
+{{/if}}
+{{#if version}}
+version: {{version}}
+{{/if}}
+{{#if initial_offset}}
+initial_offset: {{initial_offset}}
+{{/if}}
+{{#if connect_backoff}}
+connect_backoff: {{connect_backoff}}
+{{/if}}
+{{#if consume_backoff}}
+consume_backoff: {{consume_backoff}}
+{{/if}}
+{{#if max_wait_time}}
+max_wait_time: {{max_wait_time}}
+{{/if}}
+{{#if wait_close}}
+wait_close: {{wait_close}}
+{{/if}}
+{{#if isolation_level}}
+isolation_level: {{isolation_level}}
+{{/if}}
+{{#if expand_event_list_from_field}}
+expand_event_list_from_field: {{expand_event_list_from_field}}
+{{/if}}
+{{#if fetch_min}}
+fetch.min: {{fetch_min}}
+{{/if}}
+{{#if fetch_default}}
+fetch.default: {{fetch_default}}
+{{/if}}
+{{#if fetch_max}}
+fetch.max: {{fetch_max}}
+{{/if}}
+{{#if rebalance_strategy}}
+rebalance.strategy: {{rebalance_strategy}}
+{{/if}}
+{{#if rebalance_timeout}}
+rebalance.timeout: {{rebalance_timeout}}
+{{/if}}
+{{#if rebalance_max_retries}}
+rebalance.max_retries: {{rebalance_max_retries}}
+{{/if}}
+{{#if rebalance_retry_backoff}}
+rebalance.retry_backoff: {{rebalance_retry_backoff}}
+{{/if}}
+{{#if parsers}}
+parsers: 
+{{parsers}}
+{{/if}}
+{{#if kerberos_enabled}}
+kerberos.enabled: {{kerberos_enabled}}
+{{/if}}
+{{#if kerberos_auth_type}}
+kerberos.auth_type: {{kerberos_auth_type}}
+{{/if}}
+{{#if kerberos_config_path}}
+kerberos.config_path: {{kerberos_config_path}}
+{{/if}}
+{{#if kerberos_username}}
+kerberos.username: {{kerberos_username}}
+{{/if}}
+{{#if kerberos_password}}
+kerberos.password: {{kerberos_password}}
+{{/if}}
+{{#if kerberos_keytab}}
+kerberos.keytab: {{kerberos_keytab}}
+{{/if}}
+{{#if kerberos_service_name}}
+kerberos.service_name: {{kerberos_service_name}}
+{{/if}}
+{{#if kerberos_realm}}
+kerberos.realm: {{kerberos_realm}}
+{{/if}}
+{{#if kerberos_enable_krb5_fast}}
+kerberos.enable_krb5_fast: {{kerberos_enable_krb5_fast}}
+{{/if}}
+{{#if ssl}}
+ssl: 
+  {{ssl}}
+{{/if}}
+{{#if tags}}
+tags:
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{/if}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/kafka_log/data_stream/generic/fields/base-fields.yml

@@ -0,0 +1,26 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: event.module
+  type: constant_keyword
+  description: Event module
+  value: kafka_log
+- name: event.dataset
+  type: constant_keyword
+  description: Event dataset
+  value: kafka_log.generic
+- name: "@timestamp"
+  type: date
+  description: Event timestamp.
+- name: input.type
+  description: Type of Filebeat input.
+  type: keyword
+- name: tags
+  type: keyword
+  description: User defined tags

packages/kafka_log/data_stream/generic/fields/ecs.yml

@@ -0,0 +1,8 @@
+- name: ecs.version
+  external: ecs
+- name: log.level
+  external: ecs
+- name: message
+  external: ecs
+- name: event.original
+  external: ecs

packages/kafka_log/data_stream/generic/fields/fields.yml

@@ -0,0 +1,15 @@
+- name: kafka.headers
+  description: Included Kafka headers
+  type: keyword
+- name: kafka.key
+  description: Related Kafka key
+  type: keyword
+- name: kafka.offset
+  description: Current Kafka offset
+  type: long
+- name: kafka.partition
+  description: Current Kafka partition
+  type: long
+- name: kafka.topic
+  description: Related Kafka topic
+  type: keyword