refactor(healthcheck): support multiple profiles per host with aggregate result
Replace the single healthcheck_profile with a healthcheck_profiles list so a host can run several checks (e.g. hk2: pdns, rustdesk, hk2aux). Profiles emit per-check JSON to latest-<check>.json; the dispatcher clears stale per-check files, runs every profile, and merges them into latest.json. Contract: a single complete check keeps the historical verbatim latest.json shape; several checks produce a worst-status aggregate retaining every check's detail. A profile that crashes before reporting is aggregated as unknown so latest.json can never go stale while the dispatcher fails. The dispatcher exits with the worst (max) profile exit code.
This commit is contained in:
@@ -7,7 +7,7 @@ set -uo pipefail
|
||||
RESULT_DIR='{{ healthcheck_state_dir }}'
|
||||
LOG_DIR='{{ healthcheck_log_dir }}'
|
||||
HOST_NAME="$(hostname -f 2>/dev/null || hostname)"
|
||||
CHECK_NAME='{{ healthcheck_profile }}'
|
||||
CHECK_NAME="$(basename "$0" .sh)"
|
||||
STATUS=ok
|
||||
EXIT_CODE=0
|
||||
DETAILS=()
|
||||
@@ -77,8 +77,11 @@ check_tls_days() {
|
||||
}
|
||||
|
||||
emit_result() {
|
||||
local tmp detail_json
|
||||
tmp="$(mktemp "${RESULT_DIR}/latest.json.XXXXXX")"
|
||||
# Per-check JSON at latest-<check>.json. The dispatcher merges these into
|
||||
# latest.json so multiple profiles on one host do not overwrite each other.
|
||||
local tmp path detail_json
|
||||
path="${RESULT_DIR}/latest-${CHECK_NAME}.json"
|
||||
tmp="$(mktemp "${RESULT_DIR}/.latest-${CHECK_NAME}.XXXXXX")"
|
||||
detail_json="$(printf '%s\n' "${DETAILS[@]:-unknown:no_details}" | python3 -c 'import json,sys; print(json.dumps([line.rstrip() for line in sys.stdin if line.strip()]))')"
|
||||
python3 - "$tmp" "$HOST_NAME" "$CHECK_NAME" "$STATUS" "$EXIT_CODE" "$detail_json" <<'PY'
|
||||
import json, sys
|
||||
@@ -90,6 +93,75 @@ with open(path, 'w', encoding='utf-8') as f:
|
||||
f.write('\n')
|
||||
PY
|
||||
chmod 0640 "$tmp"
|
||||
mv "$tmp" "${RESULT_DIR}/latest.json"
|
||||
mv "$tmp" "$path"
|
||||
cat "$path"
|
||||
}
|
||||
|
||||
aggregate_result() {
|
||||
# Merge the just-run per-check files into latest.json. With a single complete
|
||||
# check this is a verbatim copy, preserving the historical one-object shape.
|
||||
# With several checks it emits one object whose status is the worst of all
|
||||
# checks; each check's own status/details are retained under `checks`. An
|
||||
# expected check with no fresh result file (profile crashed before writing)
|
||||
# is aggregated as `unknown`, so latest.json can never go stale while the
|
||||
# dispatcher reports a failure.
|
||||
case "$#" in
|
||||
0) return 0 ;;
|
||||
1) if [[ -f "${RESULT_DIR}/latest-$1.json" ]]; then
|
||||
cp -f "${RESULT_DIR}/latest-$1.json" "${RESULT_DIR}/latest.json"
|
||||
else
|
||||
python3 - "$RESULT_DIR" "$HOST_NAME" "$@" <<'PY'
|
||||
import json, os, sys
|
||||
rdir, host = sys.argv[1], sys.argv[2]
|
||||
checks = sys.argv[3:]
|
||||
levels = {'ok': 0, 'warning': 1, 'unknown': 2, 'critical': 3}
|
||||
worst, worst_code = 'ok', 0
|
||||
items = []
|
||||
for c in checks:
|
||||
p = os.path.join(rdir, 'latest-%s.json' % c)
|
||||
if os.path.exists(p):
|
||||
d = json.load(open(p))
|
||||
st, code = d['status'], d['exit_code']
|
||||
items.append({'check': d['check'], 'status': st,
|
||||
'exit_code': code, 'details': d['details']})
|
||||
else:
|
||||
st, code = 'unknown', 3
|
||||
items.append({'check': c, 'status': st, 'exit_code': code,
|
||||
'details': ['unknown:check_did_not_complete']})
|
||||
if levels[st] > levels[worst]:
|
||||
worst, worst_code = st, code
|
||||
out = {'schema': 1, 'host': host, 'check': 'aggregate', 'status': worst,
|
||||
'exit_code': worst_code, 'checks': items}
|
||||
open(os.path.join(rdir, 'latest.json'), 'w').write(
|
||||
json.dumps(out, sort_keys=True, separators=(',', ':')) + '\n')
|
||||
PY
|
||||
fi ;;
|
||||
*) python3 - "$RESULT_DIR" "$HOST_NAME" "$@" <<'PY'
|
||||
import json, os, sys
|
||||
rdir, host = sys.argv[1], sys.argv[2]
|
||||
checks = sys.argv[3:]
|
||||
levels = {'ok': 0, 'warning': 1, 'unknown': 2, 'critical': 3}
|
||||
worst, worst_code = 'ok', 0
|
||||
items = []
|
||||
for c in checks:
|
||||
p = os.path.join(rdir, 'latest-%s.json' % c)
|
||||
if os.path.exists(p):
|
||||
d = json.load(open(p))
|
||||
st, code = d['status'], d['exit_code']
|
||||
items.append({'check': d['check'], 'status': st,
|
||||
'exit_code': code, 'details': d['details']})
|
||||
else:
|
||||
st, code = 'unknown', 3
|
||||
items.append({'check': c, 'status': st, 'exit_code': code,
|
||||
'details': ['unknown:check_did_not_complete']})
|
||||
if levels[st] > levels[worst]:
|
||||
worst, worst_code = st, code
|
||||
out = {'schema': 1, 'host': host, 'check': 'aggregate', 'status': worst,
|
||||
'exit_code': worst_code, 'checks': items}
|
||||
open(os.path.join(rdir, 'latest.json'), 'w').write(
|
||||
json.dumps(out, sort_keys=True, separators=(',', ':')) + '\n')
|
||||
PY
|
||||
esac
|
||||
chmod 0640 "${RESULT_DIR}/latest.json"
|
||||
cat "${RESULT_DIR}/latest.json"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user