Initial VPS operations handbook

ansible/roles/matrix_healthcheck/templates/matrix-k3s.service.j2

@@ -0,0 +1,16 @@
+[Unit]
+Description=Read-only Matrix K3s health check
+Wants=network-online.target
+After=network-online.target k3s.service
+
+[Service]
+Type=oneshot
+User=root
+Group=root
+UMask=0027
+ExecStart={{ matrix_healthcheck_script_path }}
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectHome=true
+ProtectSystem=full
+ReadWritePaths={{ matrix_healthcheck_state_path | dirname }}

ansible/roles/matrix_healthcheck/templates/matrix-k3s.sh.j2

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+namespace='{{ matrix_healthcheck_namespace }}'
+backup_root='{{ matrix_healthcheck_backup_path }}'
+max_backup_age_seconds=$(({{ matrix_healthcheck_backup_max_age_hours }} * 3600))
+warn_percent='{{ matrix_healthcheck_warn_percent }}'
+critical_percent='{{ matrix_healthcheck_critical_percent }}'
+status=ok
+messages=()
+
+set_status() {
+  local next="$1"
+  case "$next" in
+    critical) status=critical ;;
+    warning) [[ "$status" != critical ]] && status=warning ;;
+  esac
+}
+
+if ! k3s kubectl get namespace "$namespace" >/dev/null 2>&1; then
+  set_status critical
+  messages+=("namespace $namespace is unavailable")
+else
+  unready=$(k3s kubectl -n "$namespace" get pods --no-headers 2>/dev/null | awk '$2 !~ /^[0-9]+\/[0-9]+$/ || $3 != "Running" {print $1}')
+  if [[ -n "$unready" ]]; then
+    set_status critical
+    messages+=("unready pods: ${unready//$'\n'/, }")
+  fi
+fi
+
+usage=$(df -P / | awk 'NR == 2 {gsub(/%/, "", $5); print $5}')
+if (( usage >= critical_percent )); then
+  set_status critical
+  messages+=("root filesystem usage ${usage}%")
+elif (( usage >= warn_percent )); then
+  set_status warning
+  messages+=("root filesystem usage ${usage}%")
+fi
+
+latest=$(find "$backup_root" -mindepth 1 -maxdepth 1 -type d -name '20*Z' -printf '%T@ %p\n' 2>/dev/null | sort -nr | awk 'NR == 1 {print $1, $2}')
+if [[ -z "$latest" ]]; then
+  set_status critical
+  messages+=("no Matrix backup exists")
+else
+  latest_epoch=${latest%% *}
+  latest_path=${latest#* }
+  age=$(( $(date +%s) - ${latest_epoch%.*} ))
+  if (( age > max_backup_age_seconds )); then
+    set_status critical
+    messages+=("latest Matrix backup is ${age}s old")
+  elif [[ ! -f "$latest_path/SHA256SUMS" || ! -f "$latest_path/manifest.json" ]]; then
+    set_status critical
+    messages+=("latest Matrix backup is incomplete")
+  fi
+fi
+
+printf '{"service":"matrix_k3s","status":"%s","messages":[' "$status"
+for i in "${!messages[@]}"; do
+  (( i > 0 )) && printf ','
+  printf '"%s"' "${messages[$i]//\"/\\\"}"
+done
+printf '],"root_usage_percent":%s}\n' "$usage"

ansible/roles/matrix_healthcheck/templates/matrix-k3s.timer.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=Run Matrix K3s health check daily
+
+[Timer]
+OnCalendar={{ matrix_healthcheck_timer_on_calendar }}
+Persistent=true
+RandomizedDelaySec={{ matrix_healthcheck_timer_randomized_delay_sec }}
+Unit={{ matrix_healthcheck_service_name }}
+
+[Install]
+WantedBy=timers.target