#!/usr/bin/env bash set -euo pipefail namespace='{{ matrix_healthcheck_namespace }}' backup_root='{{ matrix_healthcheck_backup_path }}' max_backup_age_seconds=$(({{ matrix_healthcheck_backup_max_age_hours }} * 3600)) warn_percent='{{ matrix_healthcheck_warn_percent }}' critical_percent='{{ matrix_healthcheck_critical_percent }}' status=ok messages=() set_status() { local next="$1" case "$next" in critical) status=critical ;; warning) [[ "$status" != critical ]] && status=warning ;; esac } if ! k3s kubectl get namespace "$namespace" >/dev/null 2>&1; then set_status critical messages+=("namespace $namespace is unavailable") else unready=$(k3s kubectl -n "$namespace" get pods --no-headers 2>/dev/null | awk '$2 !~ /^[0-9]+\/[0-9]+$/ || $3 != "Running" {print $1}') if [[ -n "$unready" ]]; then set_status critical messages+=("unready pods: ${unready//$'\n'/, }") fi fi usage=$(df -P / | awk 'NR == 2 {gsub(/%/, "", $5); print $5}') if (( usage >= critical_percent )); then set_status critical messages+=("root filesystem usage ${usage}%") elif (( usage >= warn_percent )); then set_status warning messages+=("root filesystem usage ${usage}%") fi latest=$(find "$backup_root" -mindepth 1 -maxdepth 1 -type d -name '20*Z' -printf '%T@ %p\n' 2>/dev/null | sort -nr | awk 'NR == 1 {print $1, $2}') if [[ -z "$latest" ]]; then set_status critical messages+=("no Matrix backup exists") else latest_epoch=${latest%% *} latest_path=${latest#* } age=$(( $(date +%s) - ${latest_epoch%.*} )) if (( age > max_backup_age_seconds )); then set_status critical messages+=("latest Matrix backup is ${age}s old") elif [[ ! -f "$latest_path/SHA256SUMS" || ! -f "$latest_path/manifest.json" ]]; then set_status critical messages+=("latest Matrix backup is incomplete") fi fi printf '{"service":"matrix_k3s","status":"%s","messages":[' "$status" for i in "${!messages[@]}"; do (( i > 0 )) && printf ',' printf '"%s"' "${messages[$i]//\"/\\\"}" done printf '],"root_usage_percent":%s}\n' "$usage"