44 lines
1.3 KiB
YAML
44 lines
1.3 KiB
YAML
host-metrics:
|
|
workers: 2
|
|
delay: 1000
|
|
source:
|
|
kafka:
|
|
bootstrap_servers: ["kafka:29092"]
|
|
client_dns_lookup: use_all_dns_ips
|
|
encryption:
|
|
type: none
|
|
acknowledgments: true
|
|
topics:
|
|
- name: host-metrics-v1
|
|
group_id: opensearch-host-metrics-v1
|
|
workers: 2
|
|
# Decode the Kafka VALUE as JSON; plaintext uses the producer key as
|
|
# the field name (e.g. hostname), so parse_json(source=message) fails.
|
|
serde_format: json
|
|
key_mode: discard
|
|
auto_offset_reset: earliest
|
|
auto_commit: false
|
|
buffer:
|
|
bounded_blocking:
|
|
buffer_size: 16384
|
|
batch_size: 512
|
|
processor:
|
|
- date:
|
|
match:
|
|
- key: timestamp
|
|
patterns: [epoch_milli]
|
|
destination: "@timestamp"
|
|
to_origination_metadata: true
|
|
- date:
|
|
from_time_received: true
|
|
destination: ingested_at
|
|
sink:
|
|
- opensearch:
|
|
hosts: ["http://opensearch:9200"]
|
|
index_type: custom
|
|
index: "host-metrics-v1-%{yyyy.MM.dd}"
|
|
bulk_size: 4
|
|
flush_timeout: 5000
|
|
# Unlimited retry for transient indexing failures; permanent failures go to DLQ.
|
|
dlq_file: /usr/share/data-prepper/dlq/failed-events.json
|