Initial VPS operations handbook

ansible/roles/matrix_healthcheck/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+matrix_healthcheck_enabled: false
+matrix_healthcheck_script_path: /usr/local/lib/vps-health/matrix-k3s
+matrix_healthcheck_state_path: /var/lib/vps-health/matrix-k3s.json
+matrix_healthcheck_service_name: matrix-k3s-healthcheck.service
+matrix_healthcheck_timer_name: matrix-k3s-healthcheck.timer
+matrix_healthcheck_timer_on_calendar: '*-*-* 06:00:00 UTC'
+matrix_healthcheck_timer_randomized_delay_sec: 15m
+matrix_healthcheck_namespace: matrix-system
+matrix_healthcheck_backup_path: /var/backups/matrix
+matrix_healthcheck_backup_max_age_hours: 30
+matrix_healthcheck_warn_percent: 80
+matrix_healthcheck_critical_percent: 90

ansible/roles/matrix_healthcheck/tasks/main.yml

@@ -0,0 +1,53 @@
+---
+- name: Require explicit approval before installing Matrix health checks
+  ansible.builtin.assert:
+    that:
+      - matrix_healthcheck_enabled | bool
+    fail_msg: Set matrix_healthcheck_enabled=true only after the Matrix namespace and backup job exist.
+
+- name: Create Matrix healthcheck script directory
+  ansible.builtin.file:
+    path: "{{ matrix_healthcheck_script_path | dirname }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Create Matrix healthcheck state directory
+  ansible.builtin.file:
+    path: "{{ matrix_healthcheck_state_path | dirname }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0750"
+
+- name: Install Matrix K3s healthcheck script
+  ansible.builtin.template:
+    src: matrix-k3s.sh.j2
+    dest: "{{ matrix_healthcheck_script_path }}"
+    owner: root
+    group: root
+    mode: "0750"
+
+- name: Install Matrix K3s healthcheck systemd service
+  ansible.builtin.template:
+    src: matrix-k3s.service.j2
+    dest: "/etc/systemd/system/{{ matrix_healthcheck_service_name }}"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Install Matrix K3s healthcheck systemd timer
+  ansible.builtin.template:
+    src: matrix-k3s.timer.j2
+    dest: "/etc/systemd/system/{{ matrix_healthcheck_timer_name }}"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Enable Matrix K3s healthcheck timer
+  ansible.builtin.systemd:
+    name: "{{ matrix_healthcheck_timer_name }}"
+    enabled: true
+    state: started
+    daemon_reload: true

ansible/roles/matrix_healthcheck/templates/matrix-k3s.service.j2

@@ -0,0 +1,16 @@
+[Unit]
+Description=Read-only Matrix K3s health check
+Wants=network-online.target
+After=network-online.target k3s.service
+
+[Service]
+Type=oneshot
+User=root
+Group=root
+UMask=0027
+ExecStart={{ matrix_healthcheck_script_path }}
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectHome=true
+ProtectSystem=full
+ReadWritePaths={{ matrix_healthcheck_state_path | dirname }}

ansible/roles/matrix_healthcheck/templates/matrix-k3s.sh.j2

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+namespace='{{ matrix_healthcheck_namespace }}'
+backup_root='{{ matrix_healthcheck_backup_path }}'
+max_backup_age_seconds=$(({{ matrix_healthcheck_backup_max_age_hours }} * 3600))
+warn_percent='{{ matrix_healthcheck_warn_percent }}'
+critical_percent='{{ matrix_healthcheck_critical_percent }}'
+status=ok
+messages=()
+
+set_status() {
+  local next="$1"
+  case "$next" in
+    critical) status=critical ;;
+    warning) [[ "$status" != critical ]] && status=warning ;;
+  esac
+}
+
+if ! k3s kubectl get namespace "$namespace" >/dev/null 2>&1; then
+  set_status critical
+  messages+=("namespace $namespace is unavailable")
+else
+  unready=$(k3s kubectl -n "$namespace" get pods --no-headers 2>/dev/null | awk '$2 !~ /^[0-9]+\/[0-9]+$/ || $3 != "Running" {print $1}')
+  if [[ -n "$unready" ]]; then
+    set_status critical
+    messages+=("unready pods: ${unready//$'\n'/, }")
+  fi
+fi
+
+usage=$(df -P / | awk 'NR == 2 {gsub(/%/, "", $5); print $5}')
+if (( usage >= critical_percent )); then
+  set_status critical
+  messages+=("root filesystem usage ${usage}%")
+elif (( usage >= warn_percent )); then
+  set_status warning
+  messages+=("root filesystem usage ${usage}%")
+fi
+
+latest=$(find "$backup_root" -mindepth 1 -maxdepth 1 -type d -name '20*Z' -printf '%T@ %p\n' 2>/dev/null | sort -nr | awk 'NR == 1 {print $1, $2}')
+if [[ -z "$latest" ]]; then
+  set_status critical
+  messages+=("no Matrix backup exists")
+else
+  latest_epoch=${latest%% *}
+  latest_path=${latest#* }
+  age=$(( $(date +%s) - ${latest_epoch%.*} ))
+  if (( age > max_backup_age_seconds )); then
+    set_status critical
+    messages+=("latest Matrix backup is ${age}s old")
+  elif [[ ! -f "$latest_path/SHA256SUMS" || ! -f "$latest_path/manifest.json" ]]; then
+    set_status critical
+    messages+=("latest Matrix backup is incomplete")
+  fi
+fi
+
+printf '{"service":"matrix_k3s","status":"%s","messages":[' "$status"
+for i in "${!messages[@]}"; do
+  (( i > 0 )) && printf ','
+  printf '"%s"' "${messages[$i]//\"/\\\"}"
+done
+printf '],"root_usage_percent":%s}\n' "$usage"

ansible/roles/matrix_healthcheck/templates/matrix-k3s.timer.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=Run Matrix K3s health check daily
+
+[Timer]
+OnCalendar={{ matrix_healthcheck_timer_on_calendar }}
+Persistent=true
+RandomizedDelaySec={{ matrix_healthcheck_timer_randomized_delay_sec }}
+Unit={{ matrix_healthcheck_service_name }}
+
+[Install]
+WantedBy=timers.target