68 lines
3.4 KiB
Bash
Executable File
68 lines
3.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Read-only diagnostics, except --probe explicitly sends one synthetic Kafka record.
|
|
set -uo pipefail
|
|
cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
|
|
[[ $# == 0 || ($# == 1 && $1 == --probe) ]] || { echo "Usage: $0 [--probe]" >&2; exit 1; }
|
|
mkdir -p diagnostics
|
|
report="diagnostics/report-$(date -u +%Y%m%dT%H%M%SZ).txt"
|
|
exec > >(tee "$report") 2>&1
|
|
section() { printf '\n### %s\n' "$1"; }
|
|
section 'Host time and native service'
|
|
date -u
|
|
systemctl --no-pager --full status telegraf-host-metrics.service
|
|
journalctl -u telegraf-host-metrics -n 100 --no-pager
|
|
section 'Native agent binary'
|
|
telegraf --version
|
|
section 'Native collection test (does not export to Kafka)'
|
|
# Parse only the four non-secret settings created by the installer.
|
|
if [[ $EUID == 0 && -f /etc/telegraf/host-metrics/environment ]]; then
|
|
python3 - <<'PY'
|
|
import os, shlex, subprocess
|
|
from pathlib import Path
|
|
env = os.environ.copy()
|
|
allowed = {'METRICS_HOST','METRICS_ENVIRONMENT','METRICS_ROLE','KAFKA_BOOTSTRAP'}
|
|
for line in Path('/etc/telegraf/host-metrics/environment').read_text().splitlines():
|
|
key, sep, raw = line.partition('=')
|
|
if sep and key in allowed:
|
|
values = shlex.split(raw)
|
|
if len(values) == 1:
|
|
env[key] = values[0]
|
|
print('Agent host:', env.get('METRICS_HOST'), 'Kafka bootstrap:', env.get('KAFKA_BOOTSTRAP'), flush=True)
|
|
try:
|
|
result = subprocess.run(['runuser','-u','telegraf','--','/usr/bin/telegraf','--config',
|
|
'/etc/telegraf/host-metrics/telegraf.conf','--test','--test-wait','3'],
|
|
env=env, timeout=35)
|
|
print('Native collection test exit:', result.returncode)
|
|
except subprocess.TimeoutExpired:
|
|
print('Native collection test timed out.')
|
|
PY
|
|
else
|
|
echo 'Run with sudo to include the native collection test and complete journal.'
|
|
fi
|
|
section 'Compose container states'
|
|
docker compose ps -a
|
|
section 'Initializer and ingestion logs'
|
|
docker compose logs --no-color --tail=100 kafka-init opensearch-init data-prepper
|
|
section 'Kafka broker logs'
|
|
docker compose logs --no-color --tail=50 kafka
|
|
section 'Kafka topic and latest offsets'
|
|
docker compose exec -T kafka /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 --describe --topic host-metrics-v1
|
|
docker compose exec -T kafka /opt/kafka/bin/kafka-get-offsets.sh --bootstrap-server kafka:29092 --topic host-metrics-v1 --time -1
|
|
section 'Data Prepper consumer group'
|
|
docker compose exec -T kafka /opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server kafka:29092 --describe --group opensearch-host-metrics-v1
|
|
section 'OpenSearch health, indices and newest document'
|
|
curl --max-time 10 -fsS 'http://127.0.0.1:9200/_cluster/health?pretty'
|
|
curl --max-time 10 -fsS 'http://127.0.0.1:9200/_cat/indices/host-metrics-v1-*?v'
|
|
curl --max-time 10 -fsS 'http://127.0.0.1:9200/host-metrics-v1-*/_search?size=1&sort=@timestamp:desc&pretty'
|
|
section 'Data Prepper DLQ'
|
|
docker compose exec -T data-prepper sh -c 'ls -lah /usr/share/data-prepper/dlq; if [ -f /usr/share/data-prepper/dlq/failed-events.json ]; then tail -n 5 /usr/share/data-prepper/dlq/failed-events.json; fi'
|
|
if [[ ${1:-} == --probe ]]; then
|
|
section 'End-to-end Kafka probe'
|
|
python3 scripts/probe-pipeline.py
|
|
probe_status=$?
|
|
printf 'Probe exit status: %s\n' "$probe_status"
|
|
fi
|
|
section 'Report saved'
|
|
echo "$report"
|
|
echo 'Review the report before sharing. It includes host labels, metrics and service logs; it does not read .env or print the datasource encryption key.'
|