host-metrics/compose.yaml

name: host-metrics

x-logging: &logging
  driver: json-file
  options:
    max-size: "10m"
    max-file: "3"

services:
  volume-init:
    image: apache/kafka:${KAFKA_VERSION:-4.3.1}
    user: "0:0"
    entrypoint: ["/bin/sh", "-ec"]
    command: ["chown 1000:1000 /data/kafka /data/dlq"]
    volumes:
      - kafka-data:/data/kafka
      - prepper-dlq:/data/dlq
    logging: *logging

  kafka:
    image: apache/kafka:${KAFKA_VERSION:-4.3.1}
    user: "1000:1000"
    restart: unless-stopped
    environment:
      KAFKA_HEAP_OPTS: -Xms512m -Xmx1g
      LOG_DIR: /tmp/kafka-logs
      KAFKA_ADVERTISED_HOST: ${KAFKA_ADVERTISED_HOST:?Run scripts/configure-access.sh with the VM LAN IP first}
    entrypoint: ["/bin/bash", "/etc/kafka/start.sh"]
    volumes:
      - ./configs/kafka/server.properties:/etc/kafka/server.properties:ro
      - ./configs/kafka/start.sh:/etc/kafka/start.sh:ro
      - kafka-data:/var/lib/kafka/data
    ports:
      - "127.0.0.1:9092:9092"
      - "${BIND_ADDRESS:?Run scripts/configure-access.sh with VM and workstation addresses}:9092:9092"
    depends_on:
      volume-init:
        condition: service_completed_successfully
    healthcheck:
      test: ["CMD", "/opt/kafka/bin/kafka-topics.sh", "--bootstrap-server", "kafka:29092", "--list"]
      interval: 15s
      timeout: 10s
      retries: 12
      start_period: 30s
    logging: *logging

  kafka-init:
    image: apache/kafka:${KAFKA_VERSION:-4.3.1}
    entrypoint: ["/bin/bash", "/etc/kafka/create-topic.sh"]
    volumes:
      - ./configs/kafka/create-topic.sh:/etc/kafka/create-topic.sh:ro
    depends_on:
      kafka:
        condition: service_healthy
    logging: *logging

  opensearch:
    image: opensearchproject/opensearch:${OPENSEARCH_VERSION:-3.8.0}
    restart: unless-stopped
    environment:
      DISABLE_INSTALL_DEMO_CONFIG: "true"
      DISABLE_SECURITY_PLUGIN: "true"
      OPENSEARCH_JAVA_OPTS: ${OPENSEARCH_JAVA_OPTS:--Xms2g -Xmx2g}
      DATASOURCE_MASTER_KEY: ${DATASOURCE_MASTER_KEY:?Run scripts/start.sh to generate the datasource encryption key}
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536
        hard: 65536
    volumes:
      - ./configs/opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml:ro
      - opensearch-data:/usr/share/opensearch/data
    ports:
      - "127.0.0.1:9200:9200"
      - "${BIND_ADDRESS:?Run scripts/configure-access.sh with VM and workstation addresses}:9200:9200"
    healthcheck:
      test: ["CMD-SHELL", "curl -fsS 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s' | grep -q '\"timed_out\":false'"]
      interval: 15s
      timeout: 10s
      retries: 20
      start_period: 60s
    logging: *logging

  opensearch-init:
    image: opensearchproject/opensearch:${OPENSEARCH_VERSION:-3.8.0}
    entrypoint: ["/bin/bash", "/bootstrap/bootstrap.sh"]
    volumes:
      - ./configs/opensearch:/bootstrap:ro
    depends_on:
      opensearch:
        condition: service_healthy
    logging: *logging

  data-prepper:
    image: opensearchproject/data-prepper:${DATA_PREPPER_VERSION:-2.16.0}
    restart: unless-stopped
    environment:
      JAVA_TOOL_OPTIONS: -Xms256m -Xmx1g
    volumes:
      - ./configs/data-prepper/pipelines.yaml:/usr/share/data-prepper/pipelines/pipelines.yaml:ro
      - ./configs/data-prepper/data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml:ro
      - prepper-dlq:/usr/share/data-prepper/dlq
    ports:
      - "127.0.0.1:4900:4900"
      - "${BIND_ADDRESS:?Run scripts/configure-access.sh with VM and workstation addresses}:4900:4900"
    depends_on:
      kafka-init:
        condition: service_completed_successfully
      opensearch-init:
        condition: service_completed_successfully
    logging: *logging

  dashboards:
    image: opensearchproject/opensearch-dashboards:${OPENSEARCH_VERSION:-3.8.0}
    restart: unless-stopped
    environment:
      DISABLE_SECURITY_DASHBOARDS_PLUGIN: "true"
    volumes:
      - ./configs/dashboards/opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml:ro
    ports:
      - "127.0.0.1:5601:5601"
      - "${BIND_ADDRESS:?Run scripts/configure-access.sh with VM and workstation addresses}:5601:5601"
    depends_on:
      opensearch-init:
        condition: service_completed_successfully
    healthcheck:
      test: ["CMD-SHELL", "curl -fsS http://localhost:5601/api/status >/dev/null"]
      interval: 15s
      timeout: 10s
      retries: 20
      start_period: 60s
    logging: *logging

  # Optional independent Kafka consumer. No host collection happens here.
  metrics-bridge:
    image: telegraf:${TELEGRAF_VERSION:-1.40.0}
    profiles: [metrics-analytics]
    restart: unless-stopped
    command: ["telegraf", "--config", "/etc/telegraf/bridge.conf"]
    volumes:
      - ./configs/metrics-bridge/telegraf.conf:/etc/telegraf/bridge.conf:ro
    ports:
      - "127.0.0.1:9273:9273"
      - "${BIND_ADDRESS:?Run scripts/configure-access.sh with VM and workstation addresses}:9273:9273"
    depends_on:
      kafka-init:
        condition: service_completed_successfully
    logging: *logging

  prometheus:
    image: prom/prometheus:${PROMETHEUS_VERSION:-v3.14.0}
    profiles: [metrics-analytics]
    restart: unless-stopped
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.path=/prometheus
      - --storage.tsdb.retention.time=7d
      - --storage.tsdb.retention.size=5GB
    volumes:
      - ./configs/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - prometheus-data:/prometheus
    ports:
      - "127.0.0.1:9090:9090"
      - "${BIND_ADDRESS:?Run scripts/configure-access.sh with VM and workstation addresses}:9090:9090"
    depends_on:
      - metrics-bridge
    logging: *logging

volumes:
  kafka-data:
  opensearch-data:
  prepper-dlq:
  prometheus-data: