63 lines
2.0 KiB
Django/Jinja
63 lines
2.0 KiB
Django/Jinja
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
namespace='{{ matrix_healthcheck_namespace }}'
|
|
backup_root='{{ matrix_healthcheck_backup_path }}'
|
|
max_backup_age_seconds=$(({{ matrix_healthcheck_backup_max_age_hours }} * 3600))
|
|
warn_percent='{{ matrix_healthcheck_warn_percent }}'
|
|
critical_percent='{{ matrix_healthcheck_critical_percent }}'
|
|
status=ok
|
|
messages=()
|
|
|
|
set_status() {
|
|
local next="$1"
|
|
case "$next" in
|
|
critical) status=critical ;;
|
|
warning) [[ "$status" != critical ]] && status=warning ;;
|
|
esac
|
|
}
|
|
|
|
if ! k3s kubectl get namespace "$namespace" >/dev/null 2>&1; then
|
|
set_status critical
|
|
messages+=("namespace $namespace is unavailable")
|
|
else
|
|
unready=$(k3s kubectl -n "$namespace" get pods --no-headers 2>/dev/null | awk '$2 !~ /^[0-9]+\/[0-9]+$/ || $3 != "Running" {print $1}')
|
|
if [[ -n "$unready" ]]; then
|
|
set_status critical
|
|
messages+=("unready pods: ${unready//$'\n'/, }")
|
|
fi
|
|
fi
|
|
|
|
usage=$(df -P / | awk 'NR == 2 {gsub(/%/, "", $5); print $5}')
|
|
if (( usage >= critical_percent )); then
|
|
set_status critical
|
|
messages+=("root filesystem usage ${usage}%")
|
|
elif (( usage >= warn_percent )); then
|
|
set_status warning
|
|
messages+=("root filesystem usage ${usage}%")
|
|
fi
|
|
|
|
latest=$(find "$backup_root" -mindepth 1 -maxdepth 1 -type d -name '20*Z' -printf '%T@ %p\n' 2>/dev/null | sort -nr | awk 'NR == 1 {print $1, $2}')
|
|
if [[ -z "$latest" ]]; then
|
|
set_status critical
|
|
messages+=("no Matrix backup exists")
|
|
else
|
|
latest_epoch=${latest%% *}
|
|
latest_path=${latest#* }
|
|
age=$(( $(date +%s) - ${latest_epoch%.*} ))
|
|
if (( age > max_backup_age_seconds )); then
|
|
set_status critical
|
|
messages+=("latest Matrix backup is ${age}s old")
|
|
elif [[ ! -f "$latest_path/SHA256SUMS" || ! -f "$latest_path/manifest.json" ]]; then
|
|
set_status critical
|
|
messages+=("latest Matrix backup is incomplete")
|
|
fi
|
|
fi
|
|
|
|
printf '{"service":"matrix_k3s","status":"%s","messages":[' "$status"
|
|
for i in "${!messages[@]}"; do
|
|
(( i > 0 )) && printf ','
|
|
printf '"%s"' "${messages[$i]//\"/\\\"}"
|
|
done
|
|
printf '],"root_usage_percent":%s}\n' "$usage"
|