host-metrics:
  workers: 2
  delay: 1000
  source:
    kafka:
      bootstrap_servers: ["kafka:29092"]
      client_dns_lookup: use_all_dns_ips
      encryption:
        type: none
      acknowledgments: true
      topics:
        - name: host-metrics-v1
          group_id: opensearch-host-metrics-v1
          workers: 2
          # Decode the Kafka VALUE as JSON; plaintext uses the producer key as
          # the field name (e.g. hostname), so parse_json(source=message) fails.
          serde_format: json
          key_mode: discard
          auto_offset_reset: earliest
          auto_commit: false
  buffer:
    bounded_blocking:
      buffer_size: 16384
      batch_size: 512
  processor:
    - date:
        match:
          - key: timestamp
            patterns: [epoch_milli]
        destination: "@timestamp"
        to_origination_metadata: true
    - date:
        from_time_received: true
        destination: ingested_at
  sink:
    - opensearch:
        hosts: ["http://opensearch:9200"]
        index_type: custom
        index: "host-metrics-v1-%{yyyy.MM.dd}"
        bulk_size: 4
        flush_timeout: 5000
        # Unlimited retry for transient indexing failures; permanent failures go to DLQ.
        dlq_file: /usr/share/data-prepper/dlq/failed-events.json