refactor(healthcheck): support multiple profiles per host with aggregate result

Replace the single healthcheck_profile with a healthcheck_profiles list so a
host can run several checks (e.g. hk2: pdns, rustdesk, hk2aux). Profiles emit
per-check JSON to latest-<check>.json; the dispatcher clears stale per-check
files, runs every profile, and merges them into latest.json.

Contract: a single complete check keeps the historical verbatim latest.json
shape; several checks produce a worst-status aggregate retaining every check's
detail. A profile that crashes before reporting is aggregated as unknown so
latest.json can never go stale while the dispatcher fails. The dispatcher exits
with the worst (max) profile exit code.

ansible/roles/healthcheck/defaults/main.yml

@@ -7,9 +7,13 @@ healthcheck_timer_on_calendar: '*-*-* 06:15:00'
 healthcheck_timer_randomized_delay_sec: 15m
 healthcheck_backup_max_age_hours: 30
 healthcheck_tls_warn_days: 21
-healthcheck_profiles:
+# Map of profile name -> installed script filename. A host selects which
+# profiles it runs via the `healthcheck_profiles` list (inventory).
+healthcheck_profile_scripts:
   mailcow: mailcow.sh
   vaultwarden: vaultwarden.sh
   pdns: pdns.sh
   wireguard: wireguard.sh
   adguardhome: adguardhome.sh
+  rustdesk: rustdesk.sh
+  hk2aux: hk2aux.sh

ansible/roles/healthcheck/tasks/main.yml

@@ -1,9 +1,10 @@
 ---
-- name: Validate known health-check profile
+- name: Validate known health-check profiles
   ansible.builtin.assert:
     that:
-      - healthcheck_profile in healthcheck_profiles
+      - item in healthcheck_profile_scripts
-    fail_msg: "Unsupported healthcheck_profile: {{ healthcheck_profile }}"
+    fail_msg: "Unsupported healthcheck_profile: {{ item }}"
+  loop: "{{ healthcheck_profiles }}"
 
 - name: Install health-check directories
   ansible.builtin.file:
@@ -28,13 +29,14 @@
     group: root
     mode: "0755"
 
-- name: Install service health-check script
+- name: Install service health-check scripts
   ansible.builtin.template:
-    src: "{{ healthcheck_profiles[healthcheck_profile] }}.j2"
+    src: "{{ healthcheck_profile_scripts[item] }}.j2"
-    dest: "{{ healthcheck_install_root }}/{{ healthcheck_profiles[healthcheck_profile] }}"
+    dest: "{{ healthcheck_install_root }}/{{ healthcheck_profile_scripts[item] }}"
     owner: root
     group: root
     mode: "0755"
+  loop: "{{ healthcheck_profiles }}"
 
 - name: Install health-check dispatcher
   ansible.builtin.template:

ansible/roles/healthcheck/templates/health-common.sh.j2

@@ -7,7 +7,7 @@ set -uo pipefail
 RESULT_DIR='{{ healthcheck_state_dir }}'
 LOG_DIR='{{ healthcheck_log_dir }}'
 HOST_NAME="$(hostname -f 2>/dev/null || hostname)"
-CHECK_NAME='{{ healthcheck_profile }}'
+CHECK_NAME="$(basename "$0" .sh)"
 STATUS=ok
 EXIT_CODE=0
 DETAILS=()
@@ -77,8 +77,11 @@ check_tls_days() {
 }
 
 emit_result() {
-  local tmp detail_json
+  # Per-check JSON at latest-<check>.json. The dispatcher merges these into
-  tmp="$(mktemp "${RESULT_DIR}/latest.json.XXXXXX")"
+  # latest.json so multiple profiles on one host do not overwrite each other.
+  local tmp path detail_json
+  path="${RESULT_DIR}/latest-${CHECK_NAME}.json"
+  tmp="$(mktemp "${RESULT_DIR}/.latest-${CHECK_NAME}.XXXXXX")"
   detail_json="$(printf '%s\n' "${DETAILS[@]:-unknown:no_details}" | python3 -c 'import json,sys; print(json.dumps([line.rstrip() for line in sys.stdin if line.strip()]))')"
   python3 - "$tmp" "$HOST_NAME" "$CHECK_NAME" "$STATUS" "$EXIT_CODE" "$detail_json" <<'PY'
 import json, sys
@@ -90,6 +93,75 @@ with open(path, 'w', encoding='utf-8') as f:
     f.write('\n')
 PY
   chmod 0640 "$tmp"
-  mv "$tmp" "${RESULT_DIR}/latest.json"
+  mv "$tmp" "$path"
+  cat "$path"
+}
+
+aggregate_result() {
+  # Merge the just-run per-check files into latest.json. With a single complete
+  # check this is a verbatim copy, preserving the historical one-object shape.
+  # With several checks it emits one object whose status is the worst of all
+  # checks; each check's own status/details are retained under `checks`. An
+  # expected check with no fresh result file (profile crashed before writing)
+  # is aggregated as `unknown`, so latest.json can never go stale while the
+  # dispatcher reports a failure.
+  case "$#" in
+    0) return 0 ;;
+    1) if [[ -f "${RESULT_DIR}/latest-$1.json" ]]; then
+         cp -f "${RESULT_DIR}/latest-$1.json" "${RESULT_DIR}/latest.json"
+       else
+         python3 - "$RESULT_DIR" "$HOST_NAME" "$@" <<'PY'
+import json, os, sys
+rdir, host = sys.argv[1], sys.argv[2]
+checks = sys.argv[3:]
+levels = {'ok': 0, 'warning': 1, 'unknown': 2, 'critical': 3}
+worst, worst_code = 'ok', 0
+items = []
+for c in checks:
+    p = os.path.join(rdir, 'latest-%s.json' % c)
+    if os.path.exists(p):
+        d = json.load(open(p))
+        st, code = d['status'], d['exit_code']
+        items.append({'check': d['check'], 'status': st,
+                      'exit_code': code, 'details': d['details']})
+    else:
+        st, code = 'unknown', 3
+        items.append({'check': c, 'status': st, 'exit_code': code,
+                      'details': ['unknown:check_did_not_complete']})
+    if levels[st] > levels[worst]:
+        worst, worst_code = st, code
+out = {'schema': 1, 'host': host, 'check': 'aggregate', 'status': worst,
+       'exit_code': worst_code, 'checks': items}
+open(os.path.join(rdir, 'latest.json'), 'w').write(
+    json.dumps(out, sort_keys=True, separators=(',', ':')) + '\n')
+PY
+       fi ;;
+    *) python3 - "$RESULT_DIR" "$HOST_NAME" "$@" <<'PY'
+import json, os, sys
+rdir, host = sys.argv[1], sys.argv[2]
+checks = sys.argv[3:]
+levels = {'ok': 0, 'warning': 1, 'unknown': 2, 'critical': 3}
+worst, worst_code = 'ok', 0
+items = []
+for c in checks:
+    p = os.path.join(rdir, 'latest-%s.json' % c)
+    if os.path.exists(p):
+        d = json.load(open(p))
+        st, code = d['status'], d['exit_code']
+        items.append({'check': d['check'], 'status': st,
+                      'exit_code': code, 'details': d['details']})
+    else:
+        st, code = 'unknown', 3
+        items.append({'check': c, 'status': st, 'exit_code': code,
+                      'details': ['unknown:check_did_not_complete']})
+    if levels[st] > levels[worst]:
+        worst, worst_code = st, code
+out = {'schema': 1, 'host': host, 'check': 'aggregate', 'status': worst,
+       'exit_code': worst_code, 'checks': items}
+open(os.path.join(rdir, 'latest.json'), 'w').write(
+    json.dumps(out, sort_keys=True, separators=(',', ':')) + '\n')
+PY
+  esac
+  chmod 0640 "${RESULT_DIR}/latest.json"
   cat "${RESULT_DIR}/latest.json"
 }

ansible/roles/healthcheck/templates/healthcheck-runner.sh.j2

@@ -1,4 +1,18 @@
 #!/usr/bin/env bash
 set -o pipefail
-'{{ healthcheck_install_root }}/{{ healthcheck_profiles[healthcheck_profile] }}' 2>&1 | tee -a '{{ healthcheck_log_dir }}/healthcheck.log'
+source '{{ healthcheck_install_root }}/health-common.sh'
-exit "${PIPESTATUS[0]}"
+# Run every enabled health-check profile, exit with the worst (max) code, and
+# merge the per-check results into /var/lib/vps-health/latest.json.
+rc=0
+# Drop per-check results from any prior run so a profile that crashes before
+# reporting cannot leak a stale healthy result into the aggregate.
+{% for profile in healthcheck_profiles %}
+rm -f '{{ healthcheck_state_dir }}/latest-{{ healthcheck_profile_scripts[profile] | replace('.sh', '') }}.json'
+{% endfor %}
+{% for profile in healthcheck_profiles %}
+'{{ healthcheck_install_root }}/{{ healthcheck_profile_scripts[profile] }}' 2>&1 | tee -a '{{ healthcheck_log_dir }}/healthcheck.log'
+this_rc="${PIPESTATUS[0]}"
+[ "$this_rc" -gt "$rc" ] && rc="$this_rc"
+{% endfor %}
+aggregate_result{% for profile in healthcheck_profiles %} {{ healthcheck_profile_scripts[profile] | replace('.sh', '') }}{% endfor %}
+exit "$rc"