windyboy
d0d5e5a704
Replace the single healthcheck_profile with a healthcheck_profiles list so a host can run several checks (e.g. hk2: pdns, rustdesk, hk2aux). Profiles emit per-check JSON to latest-<check>.json; the dispatcher clears stale per-check files, runs every profile, and merges them into latest.json. Contract: a single complete check keeps the historical verbatim latest.json shape; several checks produce a worst-status aggregate retaining every check's detail. A profile that crashes before reporting is aggregated as unknown so latest.json can never go stale while the dispatcher fails. The dispatcher exits with the worst (max) profile exit code.
168 lines
6.4 KiB
Django/Jinja
168 lines
6.4 KiB
Django/Jinja
#!/usr/bin/env bash
|
|
# Shared contract: one JSON object per run in /var/lib/vps-health/latest.json.
|
|
# Status values: ok, warning, critical, unknown. Exit codes: 0 healthy/warning,
|
|
# 2 critical, 3 unknown. Check detail must never include secret values.
|
|
set -uo pipefail
|
|
|
|
RESULT_DIR='{{ healthcheck_state_dir }}'
|
|
LOG_DIR='{{ healthcheck_log_dir }}'
|
|
HOST_NAME="$(hostname -f 2>/dev/null || hostname)"
|
|
CHECK_NAME="$(basename "$0" .sh)"
|
|
STATUS=ok
|
|
EXIT_CODE=0
|
|
DETAILS=()
|
|
|
|
record() {
|
|
local severity="$1" message="$2"
|
|
DETAILS+=("${severity}:${message}")
|
|
case "$severity" in
|
|
critical) STATUS=critical; EXIT_CODE=2 ;;
|
|
unknown) [[ "$STATUS" != critical ]] && STATUS=unknown; [[ "$EXIT_CODE" -eq 0 ]] && EXIT_CODE=3 ;;
|
|
warning) [[ "$STATUS" == ok ]] && STATUS=warning ;;
|
|
esac
|
|
}
|
|
|
|
require_command() {
|
|
command -v "$1" >/dev/null 2>&1 || record unknown "missing_command:$1"
|
|
}
|
|
|
|
compose_ps() {
|
|
docker compose --project-directory '{{ compose_project_dir }}' ps --all 2>&1
|
|
}
|
|
|
|
check_compose() {
|
|
local output
|
|
output="$(compose_ps)" || { record critical 'compose_ps_failed'; return; }
|
|
if grep -qiE 'Exited|Restarting|[[:space:]]Dead[[:space:]]' <<<"$output"; then
|
|
record critical 'compose_unhealthy_container'
|
|
else
|
|
record ok 'compose_ok'
|
|
fi
|
|
}
|
|
|
|
check_backup_freshness() {
|
|
local pattern="$1" newest now age
|
|
newest="$(find {{ compose_project_dir | quote }} -type f -path "$pattern" -printf '%T@\n' 2>/dev/null | sort -nr | head -n1)"
|
|
if [[ -z "$newest" ]]; then
|
|
record warning 'backup_not_found'
|
|
return
|
|
fi
|
|
now="$(date +%s)"
|
|
age="$(( now - ${newest%.*} ))"
|
|
if (( age > {{ healthcheck_backup_max_age_hours }} * 3600 )); then
|
|
record critical 'backup_stale'
|
|
else
|
|
record ok 'backup_fresh'
|
|
fi
|
|
}
|
|
|
|
check_https() {
|
|
local url="$1" expected="$2" code
|
|
code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 20 "$url" 2>/dev/null)" || {
|
|
record critical 'https_request_failed'; return;
|
|
}
|
|
[[ "$code" =~ $expected ]] && record ok "https_${code}" || record critical "https_${code}"
|
|
}
|
|
|
|
check_tls_days() {
|
|
local host="$1" port="$2" expires epoch remaining
|
|
expires="$(timeout 20 openssl s_client -connect "${host}:${port}" -servername "$host" </dev/null 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2-)" || {
|
|
record unknown 'tls_read_failed'; return;
|
|
}
|
|
epoch="$(date -d "$expires" +%s 2>/dev/null)" || { record unknown 'tls_date_parse_failed'; return; }
|
|
remaining="$(( (epoch - $(date +%s)) / 86400 ))"
|
|
if (( remaining < 0 )); then record critical 'tls_expired'
|
|
elif (( remaining < {{ healthcheck_tls_warn_days }} )); then record warning 'tls_near_expiry'
|
|
else record ok 'tls_valid'; fi
|
|
}
|
|
|
|
emit_result() {
|
|
# Per-check JSON at latest-<check>.json. The dispatcher merges these into
|
|
# latest.json so multiple profiles on one host do not overwrite each other.
|
|
local tmp path detail_json
|
|
path="${RESULT_DIR}/latest-${CHECK_NAME}.json"
|
|
tmp="$(mktemp "${RESULT_DIR}/.latest-${CHECK_NAME}.XXXXXX")"
|
|
detail_json="$(printf '%s\n' "${DETAILS[@]:-unknown:no_details}" | python3 -c 'import json,sys; print(json.dumps([line.rstrip() for line in sys.stdin if line.strip()]))')"
|
|
python3 - "$tmp" "$HOST_NAME" "$CHECK_NAME" "$STATUS" "$EXIT_CODE" "$detail_json" <<'PY'
|
|
import json, sys
|
|
path, host, check, status, code, details = sys.argv[1:]
|
|
with open(path, 'w', encoding='utf-8') as f:
|
|
json.dump({'schema': 1, 'host': host, 'check': check, 'status': status,
|
|
'exit_code': int(code), 'details': json.loads(details)}, f,
|
|
sort_keys=True, separators=(',', ':'))
|
|
f.write('\n')
|
|
PY
|
|
chmod 0640 "$tmp"
|
|
mv "$tmp" "$path"
|
|
cat "$path"
|
|
}
|
|
|
|
aggregate_result() {
|
|
# Merge the just-run per-check files into latest.json. With a single complete
|
|
# check this is a verbatim copy, preserving the historical one-object shape.
|
|
# With several checks it emits one object whose status is the worst of all
|
|
# checks; each check's own status/details are retained under `checks`. An
|
|
# expected check with no fresh result file (profile crashed before writing)
|
|
# is aggregated as `unknown`, so latest.json can never go stale while the
|
|
# dispatcher reports a failure.
|
|
case "$#" in
|
|
0) return 0 ;;
|
|
1) if [[ -f "${RESULT_DIR}/latest-$1.json" ]]; then
|
|
cp -f "${RESULT_DIR}/latest-$1.json" "${RESULT_DIR}/latest.json"
|
|
else
|
|
python3 - "$RESULT_DIR" "$HOST_NAME" "$@" <<'PY'
|
|
import json, os, sys
|
|
rdir, host = sys.argv[1], sys.argv[2]
|
|
checks = sys.argv[3:]
|
|
levels = {'ok': 0, 'warning': 1, 'unknown': 2, 'critical': 3}
|
|
worst, worst_code = 'ok', 0
|
|
items = []
|
|
for c in checks:
|
|
p = os.path.join(rdir, 'latest-%s.json' % c)
|
|
if os.path.exists(p):
|
|
d = json.load(open(p))
|
|
st, code = d['status'], d['exit_code']
|
|
items.append({'check': d['check'], 'status': st,
|
|
'exit_code': code, 'details': d['details']})
|
|
else:
|
|
st, code = 'unknown', 3
|
|
items.append({'check': c, 'status': st, 'exit_code': code,
|
|
'details': ['unknown:check_did_not_complete']})
|
|
if levels[st] > levels[worst]:
|
|
worst, worst_code = st, code
|
|
out = {'schema': 1, 'host': host, 'check': 'aggregate', 'status': worst,
|
|
'exit_code': worst_code, 'checks': items}
|
|
open(os.path.join(rdir, 'latest.json'), 'w').write(
|
|
json.dumps(out, sort_keys=True, separators=(',', ':')) + '\n')
|
|
PY
|
|
fi ;;
|
|
*) python3 - "$RESULT_DIR" "$HOST_NAME" "$@" <<'PY'
|
|
import json, os, sys
|
|
rdir, host = sys.argv[1], sys.argv[2]
|
|
checks = sys.argv[3:]
|
|
levels = {'ok': 0, 'warning': 1, 'unknown': 2, 'critical': 3}
|
|
worst, worst_code = 'ok', 0
|
|
items = []
|
|
for c in checks:
|
|
p = os.path.join(rdir, 'latest-%s.json' % c)
|
|
if os.path.exists(p):
|
|
d = json.load(open(p))
|
|
st, code = d['status'], d['exit_code']
|
|
items.append({'check': d['check'], 'status': st,
|
|
'exit_code': code, 'details': d['details']})
|
|
else:
|
|
st, code = 'unknown', 3
|
|
items.append({'check': c, 'status': st, 'exit_code': code,
|
|
'details': ['unknown:check_did_not_complete']})
|
|
if levels[st] > levels[worst]:
|
|
worst, worst_code = st, code
|
|
out = {'schema': 1, 'host': host, 'check': 'aggregate', 'status': worst,
|
|
'exit_code': worst_code, 'checks': items}
|
|
open(os.path.join(rdir, 'latest.json'), 'w').write(
|
|
json.dumps(out, sort_keys=True, separators=(',', ':')) + '\n')
|
|
PY
|
|
esac
|
|
chmod 0640 "${RESULT_DIR}/latest.json"
|
|
cat "${RESULT_DIR}/latest.json"
|
|
}
|