Initial VPS operations handbook

ansible/roles/matrix_backup/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+matrix_backup_enabled: false
+matrix_backup_path: /var/backups/matrix
+matrix_backup_retention_days: 7
+matrix_backup_warn_percent: 80
+matrix_backup_stop_percent: 90
+matrix_backup_script_path: /usr/local/sbin/matrix-backup
+matrix_backup_service_name: matrix-backup.service
+matrix_backup_timer_name: matrix-backup.timer
+matrix_namespace: matrix-system
+matrix_backup_postgres_pod_selector: app=matrix-postgres
+matrix_backup_media_pod_selector: app=synapse,component=synapse-media-repository
+matrix_backup_bootstrap_dir: /etc/matrix-bootstrap

ansible/roles/matrix_backup/handlers/main.yml

@@ -0,0 +1,4 @@
+---
+- name: Reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true

ansible/roles/matrix_backup/tasks/main.yml

@@ -0,0 +1,49 @@
+---
+- name: Require explicit approval before installing backup automation
+  ansible.builtin.assert:
+    that:
+      - matrix_backup_enabled | bool
+    fail_msg: >-
+      Refusing to install Matrix backup automation until matrix_backup_enabled=true
+      is supplied deliberately.
+
+- name: Create root-only Matrix backup directory
+  ansible.builtin.file:
+    path: "{{ matrix_backup_path }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
+
+- name: Install Matrix backup script
+  ansible.builtin.template:
+    src: matrix-backup.sh.j2
+    dest: "{{ matrix_backup_script_path }}"
+    owner: root
+    group: root
+    mode: "0700"
+
+- name: Install Matrix backup systemd service
+  ansible.builtin.template:
+    src: matrix-backup.service.j2
+    dest: "/etc/systemd/system/{{ matrix_backup_service_name }}"
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Reload systemd
+
+- name: Install Matrix backup systemd timer
+  ansible.builtin.template:
+    src: matrix-backup.timer.j2
+    dest: "/etc/systemd/system/{{ matrix_backup_timer_name }}"
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Reload systemd
+
+- name: Enable Matrix backup timer
+  ansible.builtin.systemd:
+    name: "{{ matrix_backup_timer_name }}"
+    enabled: true
+    state: started
+    daemon_reload: true

ansible/roles/matrix_backup/templates/matrix-backup.service.j2

@@ -0,0 +1,16 @@
+[Unit]
+Description=Create local consistent Matrix backup
+Wants=network-online.target
+After=network-online.target k3s.service
+
+[Service]
+Type=oneshot
+User=root
+Group=root
+UMask=0077
+ExecStart={{ matrix_backup_script_path }}
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectHome=true
+ProtectSystem=full
+ReadWritePaths={{ matrix_backup_path }} /etc/matrix-bootstrap

ansible/roles/matrix_backup/templates/matrix-backup.sh.j2

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+backup_root='{{ matrix_backup_path }}'
+namespace='{{ matrix_namespace }}'
+warn_percent='{{ matrix_backup_warn_percent }}'
+stop_percent='{{ matrix_backup_stop_percent }}'
+retention_days='{{ matrix_backup_retention_days }}'
+postgres_selector='{{ matrix_backup_postgres_pod_selector }}'
+media_selector='{{ matrix_backup_media_pod_selector }}'
+bootstrap_dir='{{ matrix_backup_bootstrap_dir }}'
+
+usage=$(df -P / | awk 'NR == 2 {gsub(/%/, "", $5); print $5}')
+if (( usage >= stop_percent )); then
+  printf 'Refusing Matrix backup: root filesystem usage is %s%% (stop threshold %s%%).\n' "$usage" "$stop_percent" >&2
+  exit 2
+fi
+
+stamp=$(date -u +%Y%m%dT%H%M%SZ)
+stage="$backup_root/.staging-$stamp"
+final="$backup_root/$stamp"
+mkdir -p "$stage" "$backup_root"
+trap 'rm -rf "$stage"' EXIT
+
+if ! command -v k3s >/dev/null 2>&1; then
+  printf 'K3s is unavailable; refusing Matrix backup.\n' >&2
+  exit 3
+fi
+
+postgres_pod=$(k3s kubectl -n "$namespace" get pod -l "$postgres_selector" -o jsonpath='{.items[0].metadata.name}')
+media_pod=$(k3s kubectl -n "$namespace" get pod -l "$media_selector" -o jsonpath='{.items[0].metadata.name}')
+if [[ -z "$postgres_pod" || -z "$media_pod" ]]; then
+  printf 'Required Matrix PostgreSQL or media pod is unavailable.\n' >&2
+  exit 3
+fi
+
+for database in synapse mas; do
+  k3s kubectl -n "$namespace" exec "$postgres_pod" -- \
+    pg_dump --username=postgres --format=custom --file="/tmp/$database-$stamp.dump" "$database"
+  k3s kubectl -n "$namespace" cp \
+    "$namespace/$postgres_pod:/tmp/$database-$stamp.dump" "$stage/$database.dump"
+  k3s kubectl -n "$namespace" exec "$postgres_pod" -- rm -f "/tmp/$database-$stamp.dump"
+done
+
+k3s kubectl -n "$namespace" exec "$media_pod" -- \
+  tar --create --gzip --file="/tmp/media-$stamp.tar.gz" --directory=/data media_store
+k3s kubectl -n "$namespace" cp \
+  "$namespace/$media_pod:/tmp/media-$stamp.tar.gz" "$stage/media.tar.gz"
+k3s kubectl -n "$namespace" exec "$media_pod" -- rm -f "/tmp/media-$stamp.tar.gz"
+
+tar --create --gzip --file="$stage/bootstrap.tar.gz" --directory="$(dirname "$bootstrap_dir")" "$(basename "$bootstrap_dir")"
+sha256sum "$stage"/* > "$stage/SHA256SUMS"
+printf '{"created_at":"%s","root_usage_percent":%s,"warning_threshold_percent":%s}\n' \
+  "$stamp" "$usage" "$warn_percent" > "$stage/manifest.json"
+mv "$stage" "$final"
+trap - EXIT
+find "$backup_root" -mindepth 1 -maxdepth 1 -type d -name '20*Z' -mtime +"$retention_days" -exec rm -rf {} +
+printf 'Matrix backup created: %s\n' "$final"

ansible/roles/matrix_backup/templates/matrix-backup.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Run Matrix local backup daily
+
+[Timer]
+OnCalendar=*-*-* 03:15:00 UTC
+Persistent=true
+RandomizedDelaySec=15m
+
+[Install]
+WantedBy=timers.target