windy/vps
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial VPS operations handbook

Browse Source
This commit is contained in:
windyboy
2026-08-03 12:26:42 +08:00
commit b73125e5bc
97 changed files with 3641 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ansible/ansible.cfg
+13
View File
@@ -0,0 +1,13 @@
[defaults]
inventory = inventory/hosts.yml
roles_path = roles
interpreter_python = auto_silent
stdout_callback = default
bin_ansible_callbacks = True
retry_files_enabled = False
host_key_checking = True
[privilege_escalation]
become = True
become_method = sudo
become_ask_pass = False
ansible/inventory/hosts.yml
+74
View File
@@ -0,0 +1,74 @@
---
# Sanitized control-plane inventory. Canonical human-readable facts remain
# in ../../inventory/hosts.md and ../../hosts/*.md. No passwords, keys, tokens,
# repository URLs, or private material belong here.
all:
vars:
ansible_user: windy
ansible_ssh_common_args: >-
-o BatchMode=yes -o ConnectTimeout=10 -o AddressFamily=inet
ansible_become: true
ansible_become_method: sudo
children:
managed:
hosts:
mx2:
ansible_host: mx2.windy.me
ansible_host_ipv4: 194.163.160.244
service_role: mailcow
compose_project_dir: /opt/mail
healthcheck_profile: mailcow
us2:
ansible_host: us2.wsvc.info
ansible_host_ipv4: 193.9.44.165
service_role: vaultwarden
compose_project_dir: /opt/vaultwarden
healthcheck_profile: vaultwarden
hk2:
ansible_host: hk2.chans.xyz
ansible_host_ipv4: 154.36.174.161
service_role: powerdns
compose_project_dir: /opt/pdns
healthcheck_profile: pdns
mailcow:
hosts:
mx2:
vaultwarden:
hosts:
us2:
powerdns:
hosts:
hk2:
docker_hosts:
children:
mailcow:
vaultwarden:
powerdns:
# Matrix is a dedicated K3s node and intentionally remains outside the
# Docker-oriented managed group.
k3s_servers:
hosts:
matrix_vps:
ansible_host: 169.58.86.13
ansible_host_ipv4: 169.58.86.13
service_role: matrix_k3s
matrix_server_name: chans.xyz
matrix_synapse_host: synapse.chans.xyz
matrix_element_host: chat.chans.xyz
matrix_mas_host: account.chans.xyz
matrix_admin_host: admin.chans.xyz
matrix_rtc_host: mrtc.chans.xyz
matrix_backup_path: /var/backups/matrix
matrix_bootstrap_dir: /etc/matrix-bootstrap
matrix_stack_enabled: false
# ESS OCI chart configuration
matrix_stack_chart_ref: oci://ghcr.io/element-hq/ess-helm/matrix-stack
matrix_stack_chart_version: 26.7.2
matrix_stack_release_name: ess
matrix_namespace: ess
matrix:
children:
k3s_servers:
matrix_production:
children:
matrix:
ansible/playbooks/audit.yml
+73
View File
@@ -0,0 +1,73 @@
---
# Read-only control-plane audit. This play intentionally contains no package,
# file, service, container, or configuration mutation tasks.
- name: Audit managed VPS hosts without changes
hosts: managed
gather_facts: true
become: false
any_errors_fatal: false
tasks:
- name: Verify Docker Compose command is available
ansible.builtin.command:
argv:
- docker
- compose
- version
changed_when: false
- name: Inspect configured Compose project
ansible.builtin.command:
argv:
- docker
- compose
- --project-directory
- "{{ compose_project_dir }}"
- ps
- --all
register: audit_compose_ps
changed_when: false
failed_when: false
- name: Inspect failed systemd units
ansible.builtin.command:
argv:
- systemctl
- --failed
- --no-legend
- --no-pager
register: audit_failed_units
changed_when: false
failed_when: false
- name: Inspect filesystem capacity
ansible.builtin.command:
argv:
- df
- -P
- -x
- tmpfs
- -x
- devtmpfs
register: audit_filesystems
changed_when: false
- name: Inspect active listeners
ansible.builtin.command:
argv:
- ss
- -lntup
register: audit_listeners
changed_when: false
failed_when: false
- name: Report sanitized audit summary
ansible.builtin.debug:
msg:
host: "{{ inventory_hostname }}"
profile: "{{ healthcheck_profile }}"
os: "{{ ansible_distribution }} {{ ansible_distribution_version }}"
kernel: "{{ ansible_kernel }}"
compose_rc: "{{ audit_compose_ps.rc }}"
failed_units: "{{ audit_failed_units.stdout_lines | default([]) }}"
filesystem_lines: "{{ audit_filesystems.stdout_lines | default([]) }}"
listener_lines: "{{ audit_listeners.stdout_lines | default([]) }}"
ansible/playbooks/baseline.yml
+9
View File
@@ -0,0 +1,9 @@
---
# Baseline starts audit-only. Opt-in variables are deliberately false by default.
- name: Apply controlled common baseline
hosts: managed
become: true
gather_facts: false
roles:
- role: baseline
tags: [baseline, audit]
ansible/playbooks/email-alerts.yml
+9
View File
@@ -0,0 +1,9 @@
---
# Requires a target-local root-owned SMTP config; no credentials are passed here.
- name: Deploy health email alert integration
hosts: managed
become: true
gather_facts: false
roles:
- role: email_alert
tags: [healthcheck, email]
ansible/playbooks/healthchecks.yml
+8
View File
@@ -0,0 +1,8 @@
---
- name: Deploy daily local health checks
hosts: managed
become: true
gather_facts: false
roles:
- role: healthcheck
tags: [healthcheck, timers]
ansible/playbooks/k3s-server.yml
+9
View File
@@ -0,0 +1,9 @@
---
- name: Install or reconcile the single-node K3s server
hosts: k3s_servers
become: true
gather_facts: true
serial: 1
roles:
- role: k3s_server
tags: [k3s, matrix, mutating]
ansible/playbooks/maintenance-preview.yml
+30
View File
@@ -0,0 +1,30 @@
---
# Preview only. This playbook does not install updates, restart services, or
# change DNS/secrets. A separate, manually reviewed change is required to act.
- name: Preview pending maintenance without changes
hosts: managed
become: true
gather_facts: false
tasks:
- name: Check reboot requirement marker
ansible.builtin.stat:
path: /var/run/reboot-required
register: maintenance_reboot_marker
- name: Preview available package updates on Debian-family hosts
ansible.builtin.command:
argv:
- apt-get
- --just-print
- upgrade
register: maintenance_apt_preview
changed_when: false
failed_when: false
when: ansible_facts.os_family | default('Debian') == 'Debian'
- name: Report maintenance preview
ansible.builtin.debug:
msg:
host: "{{ inventory_hostname }}"
reboot_required: "{{ maintenance_reboot_marker.stat.exists }}"
package_preview: "{{ maintenance_apt_preview.stdout_lines | default([]) }}"
ansible/playbooks/matrix-backup.yml
+8
View File
@@ -0,0 +1,8 @@
---
- name: Install local Matrix consistency backup jobs
hosts: matrix
become: true
gather_facts: false
roles:
- role: matrix_backup
tags: [matrix, backup, mutating]
ansible/playbooks/matrix-cert-manager.yml
+11
View File
@@ -0,0 +1,11 @@
---
- name: Reconcile Helm, cert-manager, and the Let's Encrypt issuer
hosts: k3s_servers
become: true
gather_facts: false
serial: 1
roles:
- role: helm_client
tags: [helm, matrix, mutating]
- role: cert_manager
tags: [cert_manager, matrix, mutating]
ansible/playbooks/matrix-certificate-smoke.yml
+9
View File
@@ -0,0 +1,9 @@
---
- name: Run the temporary Matrix HTTP-01 certificate smoke test
hosts: matrix
become: true
gather_facts: false
serial: 1
roles:
- role: matrix_certificate_smoke
tags: [matrix, certificates, smoke_test, mutating]
ansible/playbooks/matrix-cluster-base.yml
+8
View File
@@ -0,0 +1,8 @@
---
- name: Create non-secret Matrix Kubernetes foundation resources
hosts: matrix
become: true
gather_facts: false
roles:
- role: matrix_cluster_base
tags: [matrix, cluster_base, mutating]
ansible/playbooks/matrix-healthchecks.yml
+8
View File
@@ -0,0 +1,8 @@
---
- name: Install Matrix K3s local health checks
hosts: matrix
become: true
gather_facts: false
roles:
- role: matrix_healthcheck
tags: [matrix, healthcheck, timers]
ansible/playbooks/matrix-preflight.yml
+9
View File
@@ -0,0 +1,9 @@
---
# Read-only gate before a K3s or Matrix change.
- name: Validate Matrix K3s host readiness without changes
hosts: matrix
become: true
gather_facts: true
roles:
- role: k3s_preflight
tags: [matrix, preflight, read_only]
ansible/playbooks/matrix-secret-contract.yml
+8
View File
@@ -0,0 +1,8 @@
---
- name: Validate the pre-provisioned Matrix secret contract
hosts: matrix
become: true
gather_facts: false
roles:
- role: matrix_secret_contract
tags: [matrix, secrets, validation]
ansible/playbooks/matrix-stack.yml
+99
View File
@@ -0,0 +1,99 @@
---
# Deploy the official ESS OCI chart on a K3s node.
# Pre-tasks create the non-secret values files on the target host.
# The matrix_stack role then validates and deploys the chart.
- name: Deploy the Matrix stack (ESS OCI chart)
hosts: matrix
become: true
gather_facts: false
serial: 1
pre_tasks:
- name: Ensure the ESS values directory exists
ansible.builtin.file:
path: /etc/ess
state: directory
owner: root
group: root
mode: "0700"
- name: Write hostnames values file
ansible.builtin.copy:
dest: /etc/ess/hostnames.yaml
owner: root
group: root
mode: "0600"
content: |
serverName: {{ matrix_server_name }}
elementWeb:
ingress:
host: {{ matrix_element_host }}
synapse:
ingress:
host: {{ matrix_synapse_host }}
matrixAuthenticationService:
ingress:
host: {{ matrix_mas_host }}
elementAdmin:
ingress:
host: {{ matrix_admin_host }}
matrixRTC:
ingress:
host: {{ matrix_rtc_host }}
- name: Write TLS values file
ansible.builtin.copy:
dest: /etc/ess/tls.yaml
owner: root
group: root
mode: "0600"
content: |
certManager:
clusterIssuer: letsencrypt-prod
ingress:
className: traefik
tlsEnabled: true
- name: Write single-node tuning values file
ansible.builtin.copy:
dest: /etc/ess/single-node.yaml
owner: root
group: root
mode: "0600"
content: |
# ESS single-node resource tuning for K3s
# Chart defaults are already single-node-friendly
postgres:
storage:
size: 20Gi
redis:
maxMemory: 128mb
matrixRTC:
enabled: false
- name: Verify values files are in place
ansible.builtin.stat:
path: "/etc/ess/{{ item }}"
loop:
- hostnames.yaml
- tls.yaml
- single-node.yaml
register: _values_check
- name: Assert all values files exist
ansible.builtin.assert:
that:
- item.stat.exists
- item.stat.isreg
- item.stat.pw_name == 'root'
loop: "{{ _values_check.results }}"
loop_control:
label: "{{ item.stat.path | default(item.item) }}"
roles:
- role: matrix_stack
tags: [matrix, stack, mutating]
ansible/playbooks/restic.yml
+13
View File
@@ -0,0 +1,13 @@
---
# Intentionally targets only services with approved local dump/data sources.
# Set restic_enabled=true only after selecting a backend and provisioning the
# root-only repository config directly on each target.
- name: Deploy controlled Restic timers
hosts:
- vaultwarden
- powerdns
become: true
gather_facts: false
roles:
- role: restic
tags: [restic, backup]
ansible/roles/baseline/defaults/main.yml
+3
View File
@@ -0,0 +1,3 @@
---
baseline_manage_logrotate: false
baseline_require_chrony: false
ansible/roles/baseline/tasks/main.yml
+44
View File
@@ -0,0 +1,44 @@
---
- name: Audit SSH daemon effective configuration
ansible.builtin.command:
argv:
- sshd
- -T
register: baseline_sshd_effective
changed_when: false
failed_when: false
- name: Report SSH hardening observations without changing SSH
ansible.builtin.debug:
msg:
permit_root_login: >-
{{ baseline_sshd_effective.stdout_lines | select('match', '^permitrootlogin ') | list }}
password_authentication: >-
{{ baseline_sshd_effective.stdout_lines | select('match', '^passwordauthentication ') | list }}
- name: Audit time synchronization service state
ansible.builtin.command:
argv:
- systemctl
- is-active
- systemd-timesyncd
register: baseline_timesync_state
changed_when: false
failed_when: false
- name: Install baseline logrotate policy only with explicit opt-in
ansible.builtin.copy:
dest: /etc/logrotate.d/vps-baseline
content: |
/var/log/vps-health/*.log /var/log/vps-restic/*.log {
weekly
rotate 8
missingok
notifempty
compress
create 0640 root root
}
owner: root
group: root
mode: "0644"
when: baseline_manage_logrotate | bool
ansible/roles/cert_manager/defaults/main.yml
+13
View File
@@ -0,0 +1,13 @@
---
cert_manager_enabled: false
cert_manager_namespace: cert-manager
cert_manager_release_name: cert-manager
cert_manager_chart_repository_name: jetstack
cert_manager_chart_repository_url: https://charts.jetstack.io
cert_manager_chart_ref: jetstack/cert-manager
cert_manager_chart_version: v1.19.3
cert_manager_kubeconfig: /etc/rancher/k3s/k3s.yaml
cert_manager_cluster_issuer_name: letsencrypt-prod
cert_manager_acme_server: https://acme-v02.api.letsencrypt.org/directory
cert_manager_acme_private_key_secret: letsencrypt-prod-private-key
cert_manager_ingress_class: traefik
ansible/roles/cert_manager/tasks/main.yml
+94
View File
@@ -0,0 +1,94 @@
---
- name: Require explicit approval before cert-manager reconciliation
ansible.builtin.assert:
that:
- cert_manager_enabled | bool
fail_msg: >-
Refusing cert-manager changes. Re-run with cert_manager_enabled=true after
confirming public TCP 80 and 443 reach Traefik.
- name: Require Helm and the K3s kubeconfig
ansible.builtin.stat:
path: "{{ item }}"
loop:
- /usr/local/bin/helm
- "{{ cert_manager_kubeconfig }}"
register: cert_manager_prerequisites
- name: Assert Helm and kubeconfig are available
ansible.builtin.assert:
that:
- item.stat.exists
fail_msg: "Missing cert-manager prerequisite: {{ item.item }}"
loop: "{{ cert_manager_prerequisites.results }}"
- name: Add or update the Jetstack Helm repository
ansible.builtin.command:
argv:
- helm
- repo
- add
- "{{ cert_manager_chart_repository_name }}"
- "{{ cert_manager_chart_repository_url }}"
- --force-update
environment:
KUBECONFIG: "{{ cert_manager_kubeconfig }}"
changed_when: false
- name: Update Helm repositories
ansible.builtin.command:
argv: [helm, repo, update]
environment:
KUBECONFIG: "{{ cert_manager_kubeconfig }}"
changed_when: false
- name: Install or reconcile cert-manager
ansible.builtin.command:
argv:
- helm
- upgrade
- --install
- "{{ cert_manager_release_name }}"
- "{{ cert_manager_chart_ref }}"
- --namespace
- "{{ cert_manager_namespace }}"
- --create-namespace
- --version
- "{{ cert_manager_chart_version }}"
- --set
- crds.enabled=true
- --wait
- --timeout
- 10m
environment:
KUBECONFIG: "{{ cert_manager_kubeconfig }}"
changed_when: true
- name: Install ClusterIssuer manifest without secret material
ansible.builtin.template:
src: clusterissuer.yaml.j2
dest: "/etc/rancher/k3s/{{ cert_manager_cluster_issuer_name }}-clusterissuer.yaml"
owner: root
group: root
mode: "0600"
- name: Apply ClusterIssuer manifest
ansible.builtin.command:
argv:
- k3s
- kubectl
- apply
- -f
- "/etc/rancher/k3s/{{ cert_manager_cluster_issuer_name }}-clusterissuer.yaml"
changed_when: false
- name: Wait for ClusterIssuer readiness
ansible.builtin.command:
argv:
- k3s
- kubectl
- wait
- --for=condition=Ready
- "clusterissuer/{{ cert_manager_cluster_issuer_name }}"
- --timeout=180s
changed_when: false
ansible/roles/cert_manager/templates/clusterissuer.yaml.j2
+13
View File
@@ -0,0 +1,13 @@
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
name: {{ cert_manager_cluster_issuer_name }}
spec:
acme:
server: {{ cert_manager_acme_server }}
privateKeySecretRef:
name: {{ cert_manager_acme_private_key_secret }}
solvers:
- http01:
ingress:
class: {{ cert_manager_ingress_class }}
ansible/roles/email_alert/defaults/main.yml
+6
View File
@@ -0,0 +1,6 @@
---
email_alert_config_path: /etc/vps-health/alert-smtp.conf
email_alert_state_path: /var/lib/vps-health/alert-state
email_alert_recipient: ''
email_alert_enabled: false
email_alert_repeat_hours: 24
ansible/roles/email_alert/handlers/main.yml
+4
View File
@@ -0,0 +1,4 @@
---
- name: Reload systemd
ansible.builtin.systemd_service:
daemon_reload: true
ansible/roles/email_alert/tasks/main.yml
+50
View File
@@ -0,0 +1,50 @@
---
# The health-check service owns the sole dispatcher hook. This role only
# installs/removes that dispatcher according to the explicit opt-in below.
- name: Require explicit non-secret alert recipient when email is enabled
ansible.builtin.assert:
that:
- email_alert_recipient | length > 0
fail_msg: Set email_alert_recipient outside version control before enabling alerts.
when: email_alert_enabled | bool
- name: Install alert integration when explicitly enabled
when: email_alert_enabled | bool
block:
- name: Install alert state directory
ansible.builtin.file:
path: "{{ email_alert_state_path | dirname }}"
state: directory
owner: root
group: root
mode: "0750"
- name: Install secret-free alert dispatcher
ansible.builtin.template:
src: alert-dispatch.sh.j2
dest: /usr/local/lib/vps-health/alert-dispatch
owner: root
group: root
mode: "0750"
- name: Remove alert integration when disabled
when: not (email_alert_enabled | bool)
block:
- name: Remove alert dispatcher
ansible.builtin.file:
path: /usr/local/lib/vps-health/alert-dispatch
state: absent
- name: Remove legacy alert service drop-in
ansible.builtin.file:
path: /etc/systemd/system/vps-healthcheck.service.d/alerting.conf
state: absent
notify: Reload systemd
- name: Report required server-side alert configuration
ansible.builtin.debug:
msg: >-
Email alerts are {{ 'enabled' if email_alert_enabled | bool else 'disabled' }}.
When enabled, the root-owned {{ email_alert_config_path }} must be provisioned
directly on the host and must contain SMTP settings and recipient; it is never
created or populated by this repository.
ansible/roles/email_alert/templates/alert-dispatch.sh.j2
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env bash
# Sends sanitized health results through the host's locally provisioned SMTP
# credentials. The config is intentionally excluded from Ansible/Git.
set -uo pipefail
config='{{ email_alert_config_path }}'
result='/var/lib/vps-health/latest.json'
state='{{ email_alert_state_path }}'
[[ -r "$config" && -r "$result" ]] || exit 0
# shellcheck source=/dev/null
source "$config"
: "${SMTP_URL:?missing SMTP_URL in server-side alert config}"
: "${ALERT_TO:?missing ALERT_TO in server-side alert config}"
status="$(python3 -c 'import json; print(json.load(open("'"$result"'"))["status"])')"
fingerprint="$(sha256sum "$result" | cut -d' ' -f1)"
previous="$(cat "$state" 2>/dev/null || true)"
now="$(date +%s)"
last_time="${previous%%:*}"; last_fp="${previous#*:}"
if [[ "$status" =~ ^(critical|unknown)$ ]] && [[ "$fingerprint" != "$last_fp" || $((now-${last_time:-0})) -ge {{ email_alert_repeat_hours }}*3600 ]]; then
subject="[${status}] VPS health $(hostname -f 2>/dev/null || hostname)"
curl --fail --silent --show-error --url "$SMTP_URL" --mail-rcpt "$ALERT_TO" --upload-file <(printf 'To: %s\nSubject: %s\nContent-Type: application/json\n\n%s\n' "$ALERT_TO" "$subject" "$(cat "$result")")
printf '%s:%s\n' "$now" "$fingerprint" > "$state"
elif [[ "$status" =~ ^(ok|warning)$ ]]; then
subject="[${status}] daily VPS health $(hostname -f 2>/dev/null || hostname)"
curl --fail --silent --show-error --url "$SMTP_URL" --mail-rcpt "$ALERT_TO" --upload-file <(printf 'To: %s\nSubject: %s\nContent-Type: application/json\n\n%s\n' "$ALERT_TO" "$subject" "$(cat "$result")")
fi
ansible/roles/email_alert/templates/alerting.conf.j2
+2
View File
@@ -0,0 +1,2 @@
[Service]
ExecStartPost=/usr/local/lib/vps-health/alert-dispatch
ansible/roles/healthcheck/defaults/main.yml
+13
View File
@@ -0,0 +1,13 @@
---
healthcheck_install_root: /usr/local/lib/vps-health
healthcheck_state_dir: /var/lib/vps-health
healthcheck_log_dir: /var/log/vps-health
healthcheck_service_name: vps-healthcheck
healthcheck_timer_on_calendar: '*-*-* 06:15:00'
healthcheck_timer_randomized_delay_sec: 15m
healthcheck_backup_max_age_hours: 30
healthcheck_tls_warn_days: 21
healthcheck_profiles:
mailcow: mailcow.sh
vaultwarden: vaultwarden.sh
pdns: pdns.sh
ansible/roles/healthcheck/handlers/main.yml
+4
View File
@@ -0,0 +1,4 @@
---
- name: Reload systemd
ansible.builtin.systemd_service:
daemon_reload: true
ansible/roles/healthcheck/tasks/main.yml
+81
View File
@@ -0,0 +1,81 @@
---
- name: Validate known health-check profile
ansible.builtin.assert:
that:
- healthcheck_profile in healthcheck_profiles
fail_msg: "Unsupported healthcheck_profile: {{ healthcheck_profile }}"
- name: Install health-check directories
ansible.builtin.file:
path: "{{ item.path }}"
state: directory
owner: root
group: root
mode: "{{ item.mode }}"
loop:
- path: "{{ healthcheck_install_root }}"
mode: "0755"
- path: "{{ healthcheck_state_dir }}"
mode: "0750"
- path: "{{ healthcheck_log_dir }}"
mode: "0750"
- name: Install common health-check library
ansible.builtin.template:
src: health-common.sh.j2
dest: "{{ healthcheck_install_root }}/health-common.sh"
owner: root
group: root
mode: "0755"
- name: Install service health-check script
ansible.builtin.template:
src: "{{ healthcheck_profiles[healthcheck_profile] }}.j2"
dest: "{{ healthcheck_install_root }}/{{ healthcheck_profiles[healthcheck_profile] }}"
owner: root
group: root
mode: "0755"
- name: Install health-check dispatcher
ansible.builtin.template:
src: healthcheck-runner.sh.j2
dest: "{{ healthcheck_install_root }}/run"
owner: root
group: root
mode: "0755"
- name: Install health-check systemd unit
ansible.builtin.template:
src: vps-healthcheck.service.j2
dest: "/etc/systemd/system/{{ healthcheck_service_name }}.service"
owner: root
group: root
mode: "0644"
notify: Reload systemd
- name: Install persistent health-check timer
ansible.builtin.template:
src: vps-healthcheck.timer.j2
dest: "/etc/systemd/system/{{ healthcheck_service_name }}.timer"
owner: root
group: root
mode: "0644"
notify: Reload systemd
- name: Install health-check log rotation
ansible.builtin.template:
src: logrotate-vps-healthcheck.j2
dest: /etc/logrotate.d/vps-healthcheck
owner: root
group: root
mode: "0644"
- name: Enable health-check timer
ansible.builtin.systemd_service:
name: "{{ healthcheck_service_name }}.timer"
enabled: true
state: started
daemon_reload: true
- name: Flush health-check systemd changes
ansible.builtin.meta: flush_handlers
ansible/roles/healthcheck/templates/health-common.sh.j2
+95
View File
@@ -0,0 +1,95 @@
#!/usr/bin/env bash
# Shared contract: one JSON object per run in /var/lib/vps-health/latest.json.
# Status values: ok, warning, critical, unknown. Exit codes: 0 healthy/warning,
# 2 critical, 3 unknown. Check detail must never include secret values.
set -uo pipefail
RESULT_DIR='{{ healthcheck_state_dir }}'
LOG_DIR='{{ healthcheck_log_dir }}'
HOST_NAME="$(hostname -f 2>/dev/null || hostname)"
CHECK_NAME='{{ healthcheck_profile }}'
STATUS=ok
EXIT_CODE=0
DETAILS=()
record() {
local severity="$1" message="$2"
DETAILS+=("${severity}:${message}")
case "$severity" in
critical) STATUS=critical; EXIT_CODE=2 ;;
unknown) [[ "$STATUS" != critical ]] && STATUS=unknown; [[ "$EXIT_CODE" -eq 0 ]] && EXIT_CODE=3 ;;
warning) [[ "$STATUS" == ok ]] && STATUS=warning ;;
esac
}
require_command() {
command -v "$1" >/dev/null 2>&1 || record unknown "missing_command:$1"
}
compose_ps() {
docker compose --project-directory '{{ compose_project_dir }}' ps --all 2>&1
}
check_compose() {
local output
output="$(compose_ps)" || { record critical 'compose_ps_failed'; return; }
if grep -qiE 'Exited|Restarting|[[:space:]]Dead[[:space:]]' <<<"$output"; then
record critical 'compose_unhealthy_container'
else
record ok 'compose_ok'
fi
}
check_backup_freshness() {
local pattern="$1" newest now age
newest="$(find {{ compose_project_dir | quote }} -type f -path "$pattern" -printf '%T@\n' 2>/dev/null | sort -nr | head -n1)"
if [[ -z "$newest" ]]; then
record warning 'backup_not_found'
return
fi
now="$(date +%s)"
age="$(( now - ${newest%.*} ))"
if (( age > {{ healthcheck_backup_max_age_hours }} * 3600 )); then
record critical 'backup_stale'
else
record ok 'backup_fresh'
fi
}
check_https() {
local url="$1" expected="$2" code
code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 20 "$url" 2>/dev/null)" || {
record critical 'https_request_failed'; return;
}
[[ "$code" =~ $expected ]] && record ok "https_${code}" || record critical "https_${code}"
}
check_tls_days() {
local host="$1" port="$2" expires epoch remaining
expires="$(timeout 20 openssl s_client -connect "${host}:${port}" -servername "$host" </dev/null 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2-)" || {
record unknown 'tls_read_failed'; return;
}
epoch="$(date -d "$expires" +%s 2>/dev/null)" || { record unknown 'tls_date_parse_failed'; return; }
remaining="$(( (epoch - $(date +%s)) / 86400 ))"
if (( remaining < 0 )); then record critical 'tls_expired'
elif (( remaining < {{ healthcheck_tls_warn_days }} )); then record warning 'tls_near_expiry'
else record ok 'tls_valid'; fi
}
emit_result() {
local tmp detail_json
tmp="$(mktemp "${RESULT_DIR}/latest.json.XXXXXX")"
detail_json="$(printf '%s\n' "${DETAILS[@]:-unknown:no_details}" | python3 -c 'import json,sys; print(json.dumps([line.rstrip() for line in sys.stdin if line.strip()]))')"
python3 - "$tmp" "$HOST_NAME" "$CHECK_NAME" "$STATUS" "$EXIT_CODE" "$detail_json" <<'PY'
import json, sys
path, host, check, status, code, details = sys.argv[1:]
with open(path, 'w', encoding='utf-8') as f:
json.dump({'schema': 1, 'host': host, 'check': check, 'status': status,
'exit_code': int(code), 'details': json.loads(details)}, f,
sort_keys=True, separators=(',', ':'))
f.write('\n')
PY
chmod 0640 "$tmp"
mv "$tmp" "${RESULT_DIR}/latest.json"
cat "${RESULT_DIR}/latest.json"
}
ansible/roles/healthcheck/templates/healthcheck-runner.sh.j2
+3
View File
@@ -0,0 +1,3 @@
#!/usr/bin/env bash
set -uo pipefail
exec '{{ healthcheck_install_root }}/{{ healthcheck_profiles[healthcheck_profile] }}' >> '{{ healthcheck_log_dir }}/healthcheck.log' 2>&1
ansible/roles/healthcheck/templates/logrotate-vps-healthcheck.j2
+9
View File
@@ -0,0 +1,9 @@
{{ healthcheck_log_dir }}/healthcheck.log {
daily
rotate 14
missingok
notifempty
compress
delaycompress
create 0640 root root
}
ansible/roles/healthcheck/templates/mailcow.sh.j2
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env bash
set -uo pipefail
source '{{ healthcheck_install_root }}/health-common.sh'
require_command docker
require_command curl
require_command openssl
require_command dig
require_command timeout
check_compose
watchdog="$(docker compose --project-directory '{{ compose_project_dir }}' logs --tail=40 watchdog-mailcow 2>&1)" || record critical 'watchdog_log_failed'
grep -qiE '100%|healthy' <<<"$watchdog" || record warning 'watchdog_health_not_confirmed'
queue="$(docker compose --project-directory '{{ compose_project_dir }}' exec -T postfix-mailcow postqueue -p 2>&1)" || record critical 'mail_queue_check_failed'
grep -Fqi 'Mail queue is empty' <<<"$queue" || record warning 'mail_queue_nonempty'
listeners="$(ss -lnt 2>/dev/null)"
for port in 25 465 587 993 443; do
grep -qE ":${port}[[:space:]]" <<<"$listeners" || record critical "listener_missing_${port}"
done
check_https 'https://mx2.windy.me/' '^200$'
check_tls_days mx2.windy.me 443
smtp="$(timeout 10 bash -c "exec 3<>/dev/tcp/mx2.windy.me/25; printf 'EHLO health.local\\r\\nQUIT\\r\\n' >&3; cat <&3" 2>/dev/null)" || record critical 'smtp_connect_failed'
grep -qiE 'Postcow|ESMTP' <<<"$smtp" || record critical 'smtp_banner_unexpected'
for resolver in 1.1.1.1 8.8.8.8; do
mx="$(dig +short +time=3 +tries=1 "@${resolver}" windy.me MX 2>/dev/null)"
grep -Fqi 'mx2.windy.me' <<<"$mx" && { record ok 'mx_record_ok'; break; } || record warning 'mx_record_not_confirmed'
done
spf="$(dig +short +time=3 +tries=1 @1.1.1.1 windy.me TXT 2>/dev/null)"
grep -Fqi 'v=spf1' <<<"$spf" || record warning 'spf_not_confirmed'
check_backup_freshness '*/backup/*'
emit_result
exit "$EXIT_CODE"
ansible/roles/healthcheck/templates/pdns.sh.j2
+37
View File
@@ -0,0 +1,37 @@
#!/usr/bin/env bash
set -uo pipefail
source '{{ healthcheck_install_root }}/health-common.sh'
require_command docker
require_command curl
require_command dig
check_compose
image="$(docker inspect pdns-auth --format '{{ '{{' }}.Config.Image{{ '}}' }}' 2>/dev/null)" || record critical 'pdns_container_missing'
grep -Eq ':5\.0\.[6-9]|:5\.[1-9]\.' <<<"$image" || record warning 'pdns_version_not_confirmed'
security="$(docker logs pdns-auth 2>&1 | grep -i 'Mandatory.*Security Update' || true)"
[[ -z "$security" ]] || record critical 'pdns_security_update_banner'
# The API key remains in the container environment; the request emits only the
# reported version and no authentication material.
api="$(docker compose --project-directory '{{ compose_project_dir }}' exec -T auth python3 - <<'PY' 2>&1
import json, os, urllib.request
request=urllib.request.Request('http://127.0.0.1:8081/api/v1/servers/localhost', headers={'X-API-Key': os.environ['PDNS_API_KEY']})
print(json.load(urllib.request.urlopen(request, timeout=10))['version'])
PY
)" || record critical 'pdns_api_failed'
grep -Eq '^5\.' <<<"$api" || record critical 'pdns_api_version_invalid'
for zone in windy.me wsvc.info chans.xyz; do
primary="$(dig +short @154.36.174.161 SOA "$zone" 2>/dev/null)"
secondary="$(dig +short @202.91.35.141 SOA "$zone" 2>/dev/null)"
[[ -n "$primary" ]] || record critical "soa_primary_missing_${zone}"
[[ -n "$secondary" ]] || record critical "soa_secondary_missing_${zone}"
[[ "$primary" == "$secondary" ]] || record warning "soa_secondary_lag_${zone}"
done
check_https 'https://pdns.wsvc.info/' '^30[12]$'
check_https 'https://pgweb.wsvc.info/' '^(200|401)$'
check_backup_freshness '*/backup/*.sql.gz'
emit_result
exit "$EXIT_CODE"
ansible/roles/healthcheck/templates/vaultwarden.sh.j2
+51
View File
@@ -0,0 +1,51 @@
#!/usr/bin/env bash
set -uo pipefail
source '{{ healthcheck_install_root }}/health-common.sh'
require_command docker
require_command curl
require_command openssl
require_command python3
check_compose
health="$(docker compose --project-directory '{{ compose_project_dir }}' ps --format json 2>&1)" || record critical 'compose_status_unavailable'
grep -Fq 'vaultwarden' <<<"$health" || record critical 'vaultwarden_missing'
grep -Fq 'vw-db' <<<"$health" || record critical 'postgres_missing'
check_https 'https://auth.wsvc.info/' '^200$'
check_tls_days auth.wsvc.info 443
# Read effective config only inside the service and report booleans/fingerprints,
# never its SMTP password or other secret fields.
smtp_result="$(docker compose --project-directory '{{ compose_project_dir }}' exec -T vaultwarden python3 - <<'PY' 2>&1
import json, pathlib, smtplib, ssl
cfg=json.loads(pathlib.Path('/data/config.json').read_text())
host=cfg.get('smtp_host'); port=int(cfg.get('smtp_port') or 0)
user=cfg.get('smtp_username')
smtp_secret=cfg.get('smtp_password')
if not all((host, port, user, smtp_secret)):
raise SystemExit('smtp_config_incomplete')
with smtplib.SMTP(host, port, timeout=15) as client:
client.ehlo(); client.starttls(context=ssl.create_default_context()); client.ehlo(); client.login(user, smtp_secret)
print('smtp_auth_ok')
PY
)"
grep -Fqx 'smtp_auth_ok' <<<"$smtp_result" || record critical 'smtp_auth_failed'
# Detect drift without exposing the values: matching SHA-256 digests are used only
# internally and the result is a boolean.
drift_result="$(python3 - <<'PY'
import hashlib, json, pathlib, re
root=pathlib.Path('{{ compose_project_dir }}')
env=root.joinpath('.env').read_text()
match=re.search(r'^SMTP_PASSWORD=(.*)$', env, re.M)
config=json.loads(root.joinpath('vw-data/config.json').read_text())
value=(match.group(1).strip().strip('"\'') if match else '')
print('smtp_password_match' if value and value == (config.get('smtp_password') or '') else 'smtp_password_drift')
PY
2>&1)" || record unknown 'smtp_drift_check_failed'
grep -Fqx 'smtp_password_match' <<<"$drift_result" || record critical 'smtp_password_drift'
check_backup_freshness '*/backups/*.sql.gz'
emit_result
exit "$EXIT_CODE"
ansible/roles/healthcheck/templates/vps-healthcheck.service.j2
+19
View File
@@ -0,0 +1,19 @@
[Unit]
Description=Read-only VPS health check (%i)
Wants=network-online.target
After=network-online.target docker.service
[Service]
Type=oneshot
User=root
Group=root
UMask=0027
ExecStart={{ healthcheck_install_root }}/run
# The optional dispatcher is installed only by email_alert when explicitly enabled.
# This service remains healthy if alerting is intentionally unavailable.
ExecStartPost=-/usr/local/lib/vps-health/alert-dispatch
NoNewPrivileges=true
PrivateTmp=true
ProtectHome=true
ProtectSystem=full
ReadWritePaths={{ healthcheck_state_dir }} {{ healthcheck_log_dir }}
ansible/roles/healthcheck/templates/vps-healthcheck.timer.j2
+11
View File
@@ -0,0 +1,11 @@
[Unit]
Description=Daily read-only VPS health check
[Timer]
OnCalendar={{ healthcheck_timer_on_calendar }}
Persistent=true
RandomizedDelaySec={{ healthcheck_timer_randomized_delay_sec }}
Unit={{ healthcheck_service_name }}.service
[Install]
WantedBy=timers.target
ansible/roles/helm_client/defaults/main.yml
+6
View File
@@ -0,0 +1,6 @@
---
# Helm is required by the patched matrix-stack release. Existing installs are
# reported and retained; this role only installs Helm when it is absent.
helm_client_install_enabled: false
helm_client_install_script_url: https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
helm_client_binary: /usr/local/bin/helm
ansible/roles/helm_client/tasks/main.yml
+28
View File
@@ -0,0 +1,28 @@
---
- name: Check whether Helm is installed
ansible.builtin.stat:
path: "{{ helm_client_binary }}"
register: helm_client_binary_state
- name: Require explicit approval to install Helm when absent
ansible.builtin.assert:
that:
- helm_client_install_enabled | bool
fail_msg: >-
Helm is not installed. Re-run with helm_client_install_enabled=true after
reviewing the upstream installer source and checksum policy.
when: not helm_client_binary_state.stat.exists
- name: Install Helm only when explicitly approved and absent
ansible.builtin.shell:
cmd: "curl -fsSL {{ helm_client_install_script_url }} | bash"
creates: "{{ helm_client_binary }}"
when:
- helm_client_install_enabled | bool
- not helm_client_binary_state.stat.exists
no_log: true
- name: Read Helm version
ansible.builtin.command:
argv: ["{{ helm_client_binary }}", version, --short]
changed_when: false
ansible/roles/k3s_preflight/defaults/main.yml
+14
View File
@@ -0,0 +1,14 @@
---
# Read-only checks for a dedicated, single-node Matrix K3s host.
k3s_preflight_required_memory_mib: 6144
k3s_preflight_required_root_free_gib: 50
k3s_preflight_required_hosts:
- "{{ matrix_server_name }}"
- "{{ matrix_synapse_host }}"
- "{{ matrix_element_host }}"
- "{{ matrix_mas_host }}"
- "{{ matrix_admin_host }}"
- "{{ matrix_rtc_host }}"
k3s_preflight_required_ports:
- 80
- 443
ansible/roles/k3s_preflight/tasks/main.yml
+69
View File
@@ -0,0 +1,69 @@
---
- name: Read host memory in MiB
ansible.builtin.set_fact:
k3s_preflight_memory_mib: "{{ (ansible_memtotal_mb | int) }}"
- name: Read root filesystem capacity facts
ansible.builtin.set_fact:
k3s_preflight_root_mount: >-
{{ (ansible_mounts | selectattr('mount', 'equalto', '/') | list | first) | default({}) }}
- name: Assert host has the baseline resources for Matrix
ansible.builtin.assert:
that:
- k3s_preflight_memory_mib | int >= k3s_preflight_required_memory_mib | int
- (k3s_preflight_root_mount.size_available | default(0) | int) >= (k3s_preflight_required_root_free_gib | int * 1024 * 1024 * 1024)
fail_msg: >-
Matrix requires at least {{ k3s_preflight_required_memory_mib }} MiB RAM and
{{ k3s_preflight_required_root_free_gib }} GiB available on /. No change was made.
- name: Check whether K3s is already installed
ansible.builtin.stat:
path: /usr/local/bin/k3s
register: k3s_preflight_binary
- name: Read listeners on required public ports
ansible.builtin.command:
argv: [ss, -lntH]
changed_when: false
register: k3s_preflight_listeners
- name: Assert public ports are unused before initial K3s installation
ansible.builtin.assert:
that:
- >-
(k3s_preflight_binary.stat.exists | bool) or
((k3s_preflight_listeners.stdout_lines | select('search', '(:|\\.)' ~ (item | string) ~ '$') | list | length) == 0)
fail_msg: "Port {{ item }} is already listening; resolve the ingress ownership conflict first."
loop: "{{ k3s_preflight_required_ports }}"
- name: Resolve Matrix hostnames over IPv4
ansible.builtin.command:
argv: [getent, ahostsv4, "{{ item }}"]
changed_when: false
register: k3s_preflight_dns
loop: "{{ k3s_preflight_required_hosts }}"
- name: Assert every Matrix hostname resolves to the selected VPS
ansible.builtin.assert:
that:
- item.stdout is search(ansible_host_ipv4 | regex_escape)
fail_msg: >-
{{ item.item }} does not resolve to {{ ansible_host_ipv4 }} over IPv4. DNS must
be correct before HTTP-01 certificates can be issued.
loop: "{{ k3s_preflight_dns.results }}"
- name: Check the local backup path parent filesystem
ansible.builtin.command:
argv: [df, -P, "{{ matrix_backup_path | dirname }}"]
changed_when: false
register: k3s_preflight_backup_filesystem
- name: Report read-only preflight state
ansible.builtin.debug:
msg:
k3s_installed: "{{ k3s_preflight_binary.stat.exists }}"
memory_mib: "{{ k3s_preflight_memory_mib }}"
root_available_bytes: "{{ k3s_preflight_root_mount.size_available | default(0) }}"
backup_path: "{{ matrix_backup_path }}"
backup_filesystem: "{{ k3s_preflight_backup_filesystem.stdout_lines[-1] }}"
ansible/roles/k3s_server/defaults/main.yml
+11
View File
@@ -0,0 +1,11 @@
---
# The installer downloads the current stable channel only for an uninstalled node.
# Normal reruns never auto-upgrade an existing K3s installation.
k3s_install_enabled: false
k3s_install_url: https://get.k3s.io
k3s_binary_path: /usr/local/bin/k3s
k3s_service_name: k3s
k3s_server_args:
- --disable=servicelb
- --disable=metrics-server
k3s_kubeconfig: /etc/rancher/k3s/k3s.yaml
ansible/roles/k3s_server/tasks/main.yml
+48
View File
@@ -0,0 +1,48 @@
---
- name: Check whether K3s is installed
ansible.builtin.stat:
path: "{{ k3s_binary_path }}"
register: k3s_server_binary
- name: Require explicit approval before first K3s installation
ansible.builtin.assert:
that:
- k3s_install_enabled | bool
fail_msg: >-
Refusing first-time K3s installation. Re-run only after preflight succeeds
with k3s_install_enabled=true.
when: not (k3s_server_binary.stat.exists | bool)
- name: Install current stable K3s only on a new node
ansible.builtin.shell:
cmd: >-
curl -fsSL {{ k3s_install_url }} |
INSTALL_K3S_EXEC='server {{ k3s_server_args | join(' ') }}' sh -
creates: "{{ k3s_binary_path }}"
when: not (k3s_server_binary.stat.exists | bool)
no_log: true
- name: Wait for K3s API readiness
ansible.builtin.command:
argv:
- "{{ k3s_binary_path }}"
- kubectl
- get
- node
- --output=jsonpath={.items[0].status.conditions[?(@.type=="Ready")].status}
changed_when: false
register: k3s_server_ready
retries: 30
delay: 5
until: k3s_server_ready.stdout == 'True'
- name: Assert Traefik is installed and K3s optional components remain disabled
ansible.builtin.command:
argv:
- "{{ k3s_binary_path }}"
- kubectl
- get
- deployment
- traefik
- --namespace=kube-system
changed_when: false
ansible/roles/matrix_backup/defaults/main.yml
+13
View File
@@ -0,0 +1,13 @@
---
matrix_backup_enabled: false
matrix_backup_path: /var/backups/matrix
matrix_backup_retention_days: 7
matrix_backup_warn_percent: 80
matrix_backup_stop_percent: 90
matrix_backup_script_path: /usr/local/sbin/matrix-backup
matrix_backup_service_name: matrix-backup.service
matrix_backup_timer_name: matrix-backup.timer
matrix_namespace: matrix-system
matrix_backup_postgres_pod_selector: app=matrix-postgres
matrix_backup_media_pod_selector: app=synapse,component=synapse-media-repository
matrix_backup_bootstrap_dir: /etc/matrix-bootstrap
ansible/roles/matrix_backup/handlers/main.yml
+4
View File
@@ -0,0 +1,4 @@
---
- name: Reload systemd
ansible.builtin.systemd:
daemon_reload: true
ansible/roles/matrix_backup/tasks/main.yml
+49
View File
@@ -0,0 +1,49 @@
---
- name: Require explicit approval before installing backup automation
ansible.builtin.assert:
that:
- matrix_backup_enabled | bool
fail_msg: >-
Refusing to install Matrix backup automation until matrix_backup_enabled=true
is supplied deliberately.
- name: Create root-only Matrix backup directory
ansible.builtin.file:
path: "{{ matrix_backup_path }}"
state: directory
owner: root
group: root
mode: "0700"
- name: Install Matrix backup script
ansible.builtin.template:
src: matrix-backup.sh.j2
dest: "{{ matrix_backup_script_path }}"
owner: root
group: root
mode: "0700"
- name: Install Matrix backup systemd service
ansible.builtin.template:
src: matrix-backup.service.j2
dest: "/etc/systemd/system/{{ matrix_backup_service_name }}"
owner: root
group: root
mode: "0644"
notify: Reload systemd
- name: Install Matrix backup systemd timer
ansible.builtin.template:
src: matrix-backup.timer.j2
dest: "/etc/systemd/system/{{ matrix_backup_timer_name }}"
owner: root
group: root
mode: "0644"
notify: Reload systemd
- name: Enable Matrix backup timer
ansible.builtin.systemd:
name: "{{ matrix_backup_timer_name }}"
enabled: true
state: started
daemon_reload: true
ansible/roles/matrix_backup/templates/matrix-backup.service.j2
+16
View File
@@ -0,0 +1,16 @@
[Unit]
Description=Create local consistent Matrix backup
Wants=network-online.target
After=network-online.target k3s.service
[Service]
Type=oneshot
User=root
Group=root
UMask=0077
ExecStart={{ matrix_backup_script_path }}
NoNewPrivileges=true
PrivateTmp=true
ProtectHome=true
ProtectSystem=full
ReadWritePaths={{ matrix_backup_path }} /etc/matrix-bootstrap
ansible/roles/matrix_backup/templates/matrix-backup.sh.j2
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env bash
set -euo pipefail
umask 077
backup_root='{{ matrix_backup_path }}'
namespace='{{ matrix_namespace }}'
warn_percent='{{ matrix_backup_warn_percent }}'
stop_percent='{{ matrix_backup_stop_percent }}'
retention_days='{{ matrix_backup_retention_days }}'
postgres_selector='{{ matrix_backup_postgres_pod_selector }}'
media_selector='{{ matrix_backup_media_pod_selector }}'
bootstrap_dir='{{ matrix_backup_bootstrap_dir }}'
usage=$(df -P / | awk 'NR == 2 {gsub(/%/, "", $5); print $5}')
if (( usage >= stop_percent )); then
printf 'Refusing Matrix backup: root filesystem usage is %s%% (stop threshold %s%%).\n' "$usage" "$stop_percent" >&2
exit 2
fi
stamp=$(date -u +%Y%m%dT%H%M%SZ)
stage="$backup_root/.staging-$stamp"
final="$backup_root/$stamp"
mkdir -p "$stage" "$backup_root"
trap 'rm -rf "$stage"' EXIT
if ! command -v k3s >/dev/null 2>&1; then
printf 'K3s is unavailable; refusing Matrix backup.\n' >&2
exit 3
fi
postgres_pod=$(k3s kubectl -n "$namespace" get pod -l "$postgres_selector" -o jsonpath='{.items[0].metadata.name}')
media_pod=$(k3s kubectl -n "$namespace" get pod -l "$media_selector" -o jsonpath='{.items[0].metadata.name}')
if [[ -z "$postgres_pod" || -z "$media_pod" ]]; then
printf 'Required Matrix PostgreSQL or media pod is unavailable.\n' >&2
exit 3
fi
for database in synapse mas; do
k3s kubectl -n "$namespace" exec "$postgres_pod" -- \
pg_dump --username=postgres --format=custom --file="/tmp/$database-$stamp.dump" "$database"
k3s kubectl -n "$namespace" cp \
"$namespace/$postgres_pod:/tmp/$database-$stamp.dump" "$stage/$database.dump"
k3s kubectl -n "$namespace" exec "$postgres_pod" -- rm -f "/tmp/$database-$stamp.dump"
done
k3s kubectl -n "$namespace" exec "$media_pod" -- \
tar --create --gzip --file="/tmp/media-$stamp.tar.gz" --directory=/data media_store
k3s kubectl -n "$namespace" cp \
"$namespace/$media_pod:/tmp/media-$stamp.tar.gz" "$stage/media.tar.gz"
k3s kubectl -n "$namespace" exec "$media_pod" -- rm -f "/tmp/media-$stamp.tar.gz"
tar --create --gzip --file="$stage/bootstrap.tar.gz" --directory="$(dirname "$bootstrap_dir")" "$(basename "$bootstrap_dir")"
sha256sum "$stage"/* > "$stage/SHA256SUMS"
printf '{"created_at":"%s","root_usage_percent":%s,"warning_threshold_percent":%s}\n' \
"$stamp" "$usage" "$warn_percent" > "$stage/manifest.json"
mv "$stage" "$final"
trap - EXIT
find "$backup_root" -mindepth 1 -maxdepth 1 -type d -name '20*Z' -mtime +"$retention_days" -exec rm -rf {} +
printf 'Matrix backup created: %s\n' "$final"
ansible/roles/matrix_backup/templates/matrix-backup.timer.j2
+10
View File
@@ -0,0 +1,10 @@
[Unit]
Description=Run Matrix local backup daily
[Timer]
OnCalendar=*-*-* 03:15:00 UTC
Persistent=true
RandomizedDelaySec=15m
[Install]
WantedBy=timers.target
ansible/roles/matrix_certificate_smoke/defaults/main.yml
+11
View File
@@ -0,0 +1,11 @@
---
# A minimal HTTPS placeholder for the explicitly reserved RTC hostname. It also
# validates the Traefik HTTP-01 certificate path without deploying RTC services.
matrix_certificate_smoke_enabled: false
matrix_certificate_smoke_namespace: matrix-system
matrix_certificate_smoke_name: mrtc-placeholder
matrix_certificate_smoke_host: mrtc.chans.xyz
matrix_certificate_smoke_ingress_class: traefik
matrix_certificate_smoke_cluster_issuer: letsencrypt-prod
matrix_certificate_smoke_kubeconfig: /etc/rancher/k3s/k3s.yaml
matrix_certificate_smoke_cleanup: false
ansible/roles/matrix_certificate_smoke/tasks/main.yml
+63
View File
@@ -0,0 +1,63 @@
---
- name: Require explicit approval before certificate smoke test
ansible.builtin.assert:
that:
- matrix_certificate_smoke_enabled | bool
fail_msg: >-
Refusing certificate smoke test changes. Re-run with
matrix_certificate_smoke_enabled=true after creating the test DNS record.
- name: Render non-sensitive certificate smoke-test resources
ansible.builtin.template:
src: smoke.yaml.j2
dest: "/etc/rancher/k3s/{{ matrix_certificate_smoke_name }}.yaml"
owner: root
group: root
mode: "0600"
- name: Apply certificate smoke-test resources
ansible.builtin.command:
argv:
- k3s
- kubectl
- apply
- -f
- "/etc/rancher/k3s/{{ matrix_certificate_smoke_name }}.yaml"
changed_when: false
- name: Wait for smoke-test deployment
ansible.builtin.command:
argv:
- k3s
- kubectl
- rollout
- status
- "deployment/{{ matrix_certificate_smoke_name }}"
- "--namespace={{ matrix_certificate_smoke_namespace }}"
- --timeout=180s
changed_when: false
- name: Wait for smoke-test certificate
ansible.builtin.command:
argv:
- k3s
- kubectl
- wait
- --for=condition=Ready
- "certificate/{{ matrix_certificate_smoke_name }}-tls"
- "--namespace={{ matrix_certificate_smoke_namespace }}"
- --timeout=10m
changed_when: false
when: not (matrix_certificate_smoke_cleanup | bool)
- name: Remove smoke-test resources after verification
ansible.builtin.command:
argv:
- k3s
- kubectl
- delete
- -f
- "/etc/rancher/k3s/{{ matrix_certificate_smoke_name }}.yaml"
- --ignore-not-found=true
changed_when: false
when: matrix_certificate_smoke_cleanup | bool
ansible/roles/matrix_certificate_smoke/templates/smoke.yaml.j2
+82
View File
@@ -0,0 +1,82 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ matrix_certificate_smoke_name }}-content
namespace: {{ matrix_certificate_smoke_namespace }}
data:
index.html: MatrixRTC is not available yet.
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ matrix_certificate_smoke_name }}
namespace: {{ matrix_certificate_smoke_namespace }}
spec:
replicas: 1
selector:
matchLabels:
app: {{ matrix_certificate_smoke_name }}
template:
metadata:
labels:
app: {{ matrix_certificate_smoke_name }}
spec:
containers:
- name: static
image: nginx:1.27.5-alpine
ports:
- containerPort: 80
volumeMounts:
- name: content
mountPath: /usr/share/nginx/html/index.html
subPath: index.html
readOnly: true
resources:
requests:
cpu: 10m
memory: 32Mi
limits:
cpu: 50m
memory: 64Mi
volumes:
- name: content
configMap:
name: {{ matrix_certificate_smoke_name }}-content
---
apiVersion: v1
kind: Service
metadata:
name: {{ matrix_certificate_smoke_name }}
namespace: {{ matrix_certificate_smoke_namespace }}
spec:
selector:
app: {{ matrix_certificate_smoke_name }}
ports:
- name: http
port: 80
targetPort: 80
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ matrix_certificate_smoke_name }}
namespace: {{ matrix_certificate_smoke_namespace }}
annotations:
cert-manager.io/cluster-issuer: {{ matrix_certificate_smoke_cluster_issuer }}
spec:
ingressClassName: {{ matrix_certificate_smoke_ingress_class }}
tls:
- hosts:
- {{ matrix_certificate_smoke_host }}
secretName: {{ matrix_certificate_smoke_name }}-tls
rules:
- host: {{ matrix_certificate_smoke_host }}
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: {{ matrix_certificate_smoke_name }}
port:
name: http
ansible/roles/matrix_cluster_base/defaults/main.yml
+15
View File
@@ -0,0 +1,15 @@
---
# These resources are intentionally chart-independent and secret-free.
matrix_namespace: matrix-system
matrix_resource_quota:
requests.cpu: "4"
requests.memory: 6Gi
limits.cpu: "6"
limits.memory: 7Gi
persistentvolumeclaims: "4"
matrix_limit_range:
defaultRequest:
cpu: 50m
memory: 128Mi
default:
memory: 1Gi
ansible/roles/matrix_cluster_base/tasks/main.yml
+24
View File
@@ -0,0 +1,24 @@
---
- name: Require a working K3s kubeconfig
ansible.builtin.stat:
path: /etc/rancher/k3s/k3s.yaml
register: matrix_cluster_kubeconfig
- name: Assert K3s is ready before creating cluster resources
ansible.builtin.assert:
that:
- matrix_cluster_kubeconfig.stat.exists
fail_msg: Run k3s-server.yml successfully before matrix-cluster-base.yml.
- name: Install Matrix cluster base manifest
ansible.builtin.template:
src: base-resources.yaml.j2
dest: /etc/rancher/k3s/matrix-cluster-base.yaml
owner: root
group: root
mode: "0600"
- name: Apply Matrix cluster base manifest
ansible.builtin.command:
argv: [k3s, kubectl, apply, -f, /etc/rancher/k3s/matrix-cluster-base.yaml]
changed_when: false
ansible/roles/matrix_cluster_base/templates/base-resources.yaml.j2
+26
View File
@@ -0,0 +1,26 @@
apiVersion: v1
kind: Namespace
metadata:
name: {{ matrix_namespace }}
---
apiVersion: v1
kind: ResourceQuota
metadata:
name: matrix-system-quota
namespace: {{ matrix_namespace }}
spec:
hard:
{{ matrix_resource_quota | to_nice_yaml(indent=4) | indent(4, true) }}
---
apiVersion: v1
kind: LimitRange
metadata:
name: matrix-system-defaults
namespace: {{ matrix_namespace }}
spec:
limits:
- type: Container
defaultRequest:
{{ matrix_limit_range.defaultRequest | to_nice_yaml(indent=8) | indent(8, true) }}
default:
{{ matrix_limit_range.default | to_nice_yaml(indent=8) | indent(8, true) }}
ansible/roles/matrix_cluster_base/templates/letsencrypt-prod-clusterissuer.yaml.j2
+13
View File
@@ -0,0 +1,13 @@
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
name: letsencrypt-prod
spec:
acme:
server: https://acme-v02.api.letsencrypt.org/directory
privateKeySecretRef:
name: letsencrypt-prod-private-key
solvers:
- http01:
ingress:
class: traefik
ansible/roles/matrix_healthcheck/defaults/main.yml
+13
View File
@@ -0,0 +1,13 @@
---
matrix_healthcheck_enabled: false
matrix_healthcheck_script_path: /usr/local/lib/vps-health/matrix-k3s
matrix_healthcheck_state_path: /var/lib/vps-health/matrix-k3s.json
matrix_healthcheck_service_name: matrix-k3s-healthcheck.service
matrix_healthcheck_timer_name: matrix-k3s-healthcheck.timer
matrix_healthcheck_timer_on_calendar: '*-*-* 06:00:00 UTC'
matrix_healthcheck_timer_randomized_delay_sec: 15m
matrix_healthcheck_namespace: matrix-system
matrix_healthcheck_backup_path: /var/backups/matrix
matrix_healthcheck_backup_max_age_hours: 30
matrix_healthcheck_warn_percent: 80
matrix_healthcheck_critical_percent: 90
ansible/roles/matrix_healthcheck/tasks/main.yml
+53
View File
@@ -0,0 +1,53 @@
---
- name: Require explicit approval before installing Matrix health checks
ansible.builtin.assert:
that:
- matrix_healthcheck_enabled | bool
fail_msg: Set matrix_healthcheck_enabled=true only after the Matrix namespace and backup job exist.
- name: Create Matrix healthcheck script directory
ansible.builtin.file:
path: "{{ matrix_healthcheck_script_path | dirname }}"
state: directory
owner: root
group: root
mode: "0755"
- name: Create Matrix healthcheck state directory
ansible.builtin.file:
path: "{{ matrix_healthcheck_state_path | dirname }}"
state: directory
owner: root
group: root
mode: "0750"
- name: Install Matrix K3s healthcheck script
ansible.builtin.template:
src: matrix-k3s.sh.j2
dest: "{{ matrix_healthcheck_script_path }}"
owner: root
group: root
mode: "0750"
- name: Install Matrix K3s healthcheck systemd service
ansible.builtin.template:
src: matrix-k3s.service.j2
dest: "/etc/systemd/system/{{ matrix_healthcheck_service_name }}"
owner: root
group: root
mode: "0644"
- name: Install Matrix K3s healthcheck systemd timer
ansible.builtin.template:
src: matrix-k3s.timer.j2
dest: "/etc/systemd/system/{{ matrix_healthcheck_timer_name }}"
owner: root
group: root
mode: "0644"
- name: Enable Matrix K3s healthcheck timer
ansible.builtin.systemd:
name: "{{ matrix_healthcheck_timer_name }}"
enabled: true
state: started
daemon_reload: true
ansible/roles/matrix_healthcheck/templates/matrix-k3s.service.j2
+16
View File
@@ -0,0 +1,16 @@
[Unit]
Description=Read-only Matrix K3s health check
Wants=network-online.target
After=network-online.target k3s.service
[Service]
Type=oneshot
User=root
Group=root
UMask=0027
ExecStart={{ matrix_healthcheck_script_path }}
NoNewPrivileges=true
PrivateTmp=true
ProtectHome=true
ProtectSystem=full
ReadWritePaths={{ matrix_healthcheck_state_path | dirname }}
ansible/roles/matrix_healthcheck/templates/matrix-k3s.sh.j2
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env bash
set -euo pipefail
namespace='{{ matrix_healthcheck_namespace }}'
backup_root='{{ matrix_healthcheck_backup_path }}'
max_backup_age_seconds=$(({{ matrix_healthcheck_backup_max_age_hours }} * 3600))
warn_percent='{{ matrix_healthcheck_warn_percent }}'
critical_percent='{{ matrix_healthcheck_critical_percent }}'
status=ok
messages=()
set_status() {
local next="$1"
case "$next" in
critical) status=critical ;;
warning) [[ "$status" != critical ]] && status=warning ;;
esac
}
if ! k3s kubectl get namespace "$namespace" >/dev/null 2>&1; then
set_status critical
messages+=("namespace $namespace is unavailable")
else
unready=$(k3s kubectl -n "$namespace" get pods --no-headers 2>/dev/null | awk '$2 !~ /^[0-9]+\/[0-9]+$/ || $3 != "Running" {print $1}')
if [[ -n "$unready" ]]; then
set_status critical
messages+=("unready pods: ${unready//$'\n'/, }")
fi
fi
usage=$(df -P / | awk 'NR == 2 {gsub(/%/, "", $5); print $5}')
if (( usage >= critical_percent )); then
set_status critical
messages+=("root filesystem usage ${usage}%")
elif (( usage >= warn_percent )); then
set_status warning
messages+=("root filesystem usage ${usage}%")
fi
latest=$(find "$backup_root" -mindepth 1 -maxdepth 1 -type d -name '20*Z' -printf '%T@ %p\n' 2>/dev/null | sort -nr | awk 'NR == 1 {print $1, $2}')
if [[ -z "$latest" ]]; then
set_status critical
messages+=("no Matrix backup exists")
else
latest_epoch=${latest%% *}
latest_path=${latest#* }
age=$(( $(date +%s) - ${latest_epoch%.*} ))
if (( age > max_backup_age_seconds )); then
set_status critical
messages+=("latest Matrix backup is ${age}s old")
elif [[ ! -f "$latest_path/SHA256SUMS" || ! -f "$latest_path/manifest.json" ]]; then
set_status critical
messages+=("latest Matrix backup is incomplete")
fi
fi
printf '{"service":"matrix_k3s","status":"%s","messages":[' "$status"
for i in "${!messages[@]}"; do
(( i > 0 )) && printf ','
printf '"%s"' "${messages[$i]//\"/\\\"}"
done
printf '],"root_usage_percent":%s}\n' "$usage"
ansible/roles/matrix_healthcheck/templates/matrix-k3s.timer.j2
+11
View File
@@ -0,0 +1,11 @@
[Unit]
Description=Run Matrix K3s health check daily
[Timer]
OnCalendar={{ matrix_healthcheck_timer_on_calendar }}
Persistent=true
RandomizedDelaySec={{ matrix_healthcheck_timer_randomized_delay_sec }}
Unit={{ matrix_healthcheck_service_name }}
[Install]
WantedBy=timers.target
ansible/roles/matrix_secret_contract/defaults/main.yml
+21
View File
@@ -0,0 +1,21 @@
---
matrix_namespace: matrix-system
matrix_bootstrap_dir: /etc/matrix-bootstrap
# This role validates only names, keys, types, ownership and modes. It never reads
# bootstrap values or emits Kubernetes Secret contents.
matrix_required_bootstrap_files:
- smtp-password
- synapse-db-password
- mas-db-password
- synapse-signing-key
- synapse-macaroon-secret
- synapse-form-secret
- mas-encryption-secret
- mas-signing-key
matrix_required_kubernetes_secrets:
- name: matrix-synapse-master-config
keys: [homeserver.yaml, log.config, signing.key]
- name: matrix-authentication-config
keys: [mas-config.yaml]
- name: matrix-pgbouncer-userlist
keys: [userlist.txt]
ansible/roles/matrix_secret_contract/tasks/main.yml
+61
View File
@@ -0,0 +1,61 @@
---
- name: Inspect Matrix bootstrap directory without reading secrets
ansible.builtin.stat:
path: "{{ matrix_bootstrap_dir }}"
register: matrix_secret_bootstrap_dir
- name: Assert Matrix bootstrap directory is root-only
ansible.builtin.assert:
that:
- matrix_secret_bootstrap_dir.stat.exists
- matrix_secret_bootstrap_dir.stat.isdir
- matrix_secret_bootstrap_dir.stat.pw_name == 'root'
- matrix_secret_bootstrap_dir.stat.gr_name == 'root'
- matrix_secret_bootstrap_dir.stat.mode == '0700'
fail_msg: "{{ matrix_bootstrap_dir }} must be a root:root 0700 directory."
- name: Inspect required individual bootstrap secret files without reading them
ansible.builtin.stat:
path: "{{ matrix_bootstrap_dir }}/{{ item }}"
loop: "{{ matrix_required_bootstrap_files }}"
register: matrix_secret_bootstrap_files
no_log: true
- name: Assert required bootstrap secret file permissions
ansible.builtin.assert:
that:
- item.stat.exists
- item.stat.isreg
- item.stat.pw_name == 'root'
- item.stat.gr_name == 'root'
- item.stat.mode == '0600'
fail_msg: A required root-only Matrix bootstrap file is missing or has unsafe permissions.
loop: "{{ matrix_secret_bootstrap_files.results }}"
no_log: true
- name: Inspect Kubernetes Secret metadata without retrieving values
ansible.builtin.command:
argv:
- k3s
- kubectl
- describe
- secret
- "{{ item.name }}"
- --namespace={{ matrix_namespace }}
changed_when: false
loop: "{{ matrix_required_kubernetes_secrets }}"
register: matrix_secret_kubernetes_metadata
no_log: true
- name: Assert required Kubernetes Secret keys exist
ansible.builtin.assert:
that:
- item.stdout is search('(?m)^' ~ key ~ ':')
fail_msg: Required Matrix Kubernetes Secret key is missing.
loop: "{{ matrix_secret_kubernetes_metadata.results | subelements('item.keys') }}"
loop_control:
loop_var: matrix_secret_key_check
vars:
item: "{{ matrix_secret_key_check.0 }}"
key: "{{ matrix_secret_key_check.1 }}"
no_log: true
ansible/roles/matrix_stack/defaults/main.yml
+17
View File
@@ -0,0 +1,17 @@
---
# Matrix Stack deployment using the official ESS OCI chart
# All sensitive values are handled by the chart's built-in initSecrets
# or provided via pre-created Kubernetes Secrets.
matrix_stack_enabled: false
matrix_namespace: ess
matrix_stack_chart_ref: oci://ghcr.io/element-hq/ess-helm/matrix-stack
matrix_stack_chart_version: 26.7.2
matrix_stack_release_name: ess
# Root-only, non-secret values directory on the target host
matrix_stack_values_dir: /etc/ess
# Individual values files (must not contain secrets)
matrix_stack_values_files:
- hostnames.yaml
- tls.yaml
- single-node.yaml
ansible/roles/matrix_stack/tasks/main.yml
+97
View File
@@ -0,0 +1,97 @@
---
- name: Require explicit approval before deploying the Matrix stack
ansible.builtin.assert:
that:
- matrix_stack_enabled | bool
fail_msg: >-
Refusing Matrix deployment until matrix_stack_enabled=true is set deliberately.
- name: Require Helm on the Matrix host
ansible.builtin.command:
argv: [helm, version, --short]
changed_when: false
- name: Require non-secret values directory exists on the host
ansible.builtin.stat:
path: "{{ matrix_stack_values_dir }}"
register: _values_dir
- name: Assert values directory exists and is root-owned
ansible.builtin.assert:
that:
- _values_dir.stat.exists
- _values_dir.stat.isdir
- _values_dir.stat.pw_name == 'root'
- _values_dir.stat.gr_name == 'root'
- _values_dir.stat.mode == '0700'
fail_msg: >-
Create the root-owned directory {{ matrix_stack_values_dir }}
with mode 0700 and place the non-secret values files in it.
- name: Require each non-secret values file exists on the host
ansible.builtin.stat:
path: "{{ matrix_stack_values_dir }}/{{ item }}"
loop: "{{ matrix_stack_values_files }}"
register: _values_files
- name: Assert all values files exist and are root-owned
ansible.builtin.assert:
that:
- item.stat.exists
- item.stat.isreg
- item.stat.pw_name == 'root'
- item.stat.gr_name == 'root'
- item.stat.mode in ['0600', '0640']
fail_msg: >-
Values file {{ item.stat.path }} must be root-owned with restricted
permissions (0600 or 0640) and must not contain secrets.
loop: "{{ _values_files.results }}"
loop_control:
label: "{{ item.stat.path | default(item.item) }}"
- name: Build helm value arguments
ansible.builtin.set_fact:
_helm_values_args: >-
{%- for f in matrix_stack_values_files -%}
--values {{ matrix_stack_values_dir }}/{{ f }} {% endfor -%}
- name: Render the ESS chart without applying it (dry-run validation)
ansible.builtin.command:
cmd: >-
helm template {{ matrix_stack_release_name }}
{{ matrix_stack_chart_ref }}
--version {{ matrix_stack_chart_version }}
--namespace {{ matrix_namespace }}
--create-namespace
{{ _helm_values_args }}
environment:
KUBECONFIG: "{{ k3s_kubeconfig_path | default('/etc/rancher/k3s/k3s.yaml') }}"
changed_when: false
register: _chart_render
# No secrets in values, but render output may contain initSecrets-generated placeholders
- name: Verify the chart renders without errors
ansible.builtin.assert:
that:
- _chart_render.rc == 0
- _chart_render.stdout | length > 0
fail_msg: >-
Helm template rendering failed. Check values files for syntax errors.
Output: {{ _chart_render.stderr | default('(none)') }}
- name: Deploy the ESS chart via Helm upgrade --install
ansible.builtin.command:
cmd: >-
helm upgrade --install {{ matrix_stack_release_name }}
{{ matrix_stack_chart_ref }}
--version {{ matrix_stack_chart_version }}
--namespace {{ matrix_namespace }}
--create-namespace
{{ _helm_values_args }}
--wait
--timeout 15m
environment:
KUBECONFIG: "{{ k3s_kubeconfig_path | default('/etc/rancher/k3s/k3s.yaml') }}"
changed_when: true
register: _helm_deploy
# initSecrets may generate passwords at deploy time; those stay in-cluster only
ansible/roles/restic/defaults/main.yml
+21
View File
@@ -0,0 +1,21 @@
---
restic_enabled: false
restic_binary: /usr/bin/restic
restic_config_path: /etc/vps-restic/repository.env
restic_state_dir: /var/lib/vps-restic
restic_log_dir: /var/log/vps-restic
restic_backup_on_calendar: '*-*-* 04:30:00'
restic_check_on_calendar: 'Sun *-*-* 05:30:00'
restic_forget_on_calendar: 'Sat *-*-* 05:30:00'
restic_keep_daily: 7
restic_keep_weekly: 4
restic_keep_monthly: 6
restic_sources:
vaultwarden:
- /opt/vaultwarden/backups
- /opt/vaultwarden/vw-data
pdns:
- /opt/pdns/backup
- /opt/pdns/auth/pdns.conf
# Mailcow is deliberately excluded pending its official consistency and restore
# design review. No repository/backend value is supplied by this project.
ansible/roles/restic/handlers/main.yml
+4
View File
@@ -0,0 +1,4 @@
---
- name: Reload systemd
ansible.builtin.systemd_service:
daemon_reload: true
ansible/roles/restic/tasks/main.yml
+97
View File
@@ -0,0 +1,97 @@
---
- name: Require explicit Restic opt-in
ansible.builtin.assert:
that:
- restic_enabled | bool
fail_msg: >-
Restic is disabled by default. Set restic_enabled=true and provision the
repository configuration only on the target host after backend approval.
- name: Validate supported Restic source profile
ansible.builtin.assert:
that:
- healthcheck_profile in restic_sources
fail_msg: "No approved Restic source profile for {{ healthcheck_profile }}."
- name: Verify Restic binary exists on target
ansible.builtin.stat:
path: "{{ restic_binary }}"
register: restic_binary_stat
- name: Require target-side Restic binary
ansible.builtin.assert:
that: restic_binary_stat.stat.exists
fail_msg: "Install Restic through an approved host maintenance change first."
- name: Verify target-side repository configuration exists
ansible.builtin.stat:
path: "{{ restic_config_path }}"
register: restic_config_stat
- name: Require root-only repository configuration
ansible.builtin.assert:
that:
- restic_config_stat.stat.exists
- restic_config_stat.stat.mode == '0600'
fail_msg: >-
Provision {{ restic_config_path }} directly on the host with mode 0600.
It must contain RESTIC_REPOSITORY, RESTIC_PASSWORD_FILE, and any backend
credentials; do not commit or pass them via Ansible.
- name: Install Restic state and log directories
ansible.builtin.file:
path: "{{ item }}"
state: directory
owner: root
group: root
mode: "0750"
loop:
- "{{ restic_state_dir }}"
- "{{ restic_log_dir }}"
- name: Install Restic script directory
ansible.builtin.file:
path: /usr/local/lib/vps-restic
state: directory
owner: root
group: root
mode: "0755"
- name: Install Restic scripts
ansible.builtin.template:
src: "{{ item }}.j2"
dest: "/usr/local/lib/vps-restic/{{ item }}"
owner: root
group: root
mode: "0750"
loop:
- backup
- check
- forget-prune
- name: Install Restic systemd units and timers
ansible.builtin.template:
src: "{{ item }}.j2"
dest: "/etc/systemd/system/{{ item }}"
owner: root
group: root
mode: "0644"
loop:
- vps-restic-backup.service
- vps-restic-backup.timer
- vps-restic-check.service
- vps-restic-check.timer
- vps-restic-forget-prune.service
- vps-restic-forget-prune.timer
notify: Reload systemd
- name: Enable Restic timers
ansible.builtin.systemd_service:
name: "{{ item }}"
enabled: true
state: started
daemon_reload: true
loop:
- vps-restic-backup.timer
- vps-restic-check.timer
- vps-restic-forget-prune.timer
ansible/roles/restic/templates/backup.j2
+6
View File
@@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -euo pipefail
# Repository and password credentials are host-local in {{ restic_config_path }}.
# shellcheck source=/dev/null
source '{{ restic_config_path }}'
exec '{{ restic_binary }}' backup --tag '{{ healthcheck_profile }}' --tag "$(hostname -s)" {% for source in restic_sources[healthcheck_profile] %}{{ source | quote }} {% endfor %}
ansible/roles/restic/templates/check.j2
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
# shellcheck source=/dev/null
source '{{ restic_config_path }}'
exec '{{ restic_binary }}' check --read-data-subset=5%
ansible/roles/restic/templates/forget-prune.j2
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
# shellcheck source=/dev/null
source '{{ restic_config_path }}'
exec '{{ restic_binary }}' forget --prune --keep-daily {{ restic_keep_daily }} --keep-weekly {{ restic_keep_weekly }} --keep-monthly {{ restic_keep_monthly }} --tag '{{ healthcheck_profile }}'
ansible/roles/restic/templates/vps-restic-backup.service.j2
+11
View File
@@ -0,0 +1,11 @@
[Unit]
Description=Restic backup for approved {{ healthcheck_profile }} sources
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
User=root
Group=root
UMask=0077
ExecStart=/usr/local/lib/vps-restic/backup
ansible/roles/restic/templates/vps-restic-backup.timer.j2
+11
View File
@@ -0,0 +1,11 @@
[Unit]
Description=Daily Restic backup timer
[Timer]
OnCalendar={{ restic_backup_on_calendar }}
Persistent=true
RandomizedDelaySec=20m
Unit=vps-restic-backup.service
[Install]
WantedBy=timers.target
ansible/roles/restic/templates/vps-restic-check.service.j2
+9
View File
@@ -0,0 +1,9 @@
[Unit]
Description=Restic repository integrity check
[Service]
Type=oneshot
User=root
Group=root
UMask=0077
ExecStart=/usr/local/lib/vps-restic/check
ansible/roles/restic/templates/vps-restic-check.timer.j2
+11
View File
@@ -0,0 +1,11 @@
[Unit]
Description=Weekly Restic integrity check timer
[Timer]
OnCalendar={{ restic_check_on_calendar }}
Persistent=true
RandomizedDelaySec=30m
Unit=vps-restic-check.service
[Install]
WantedBy=timers.target
ansible/roles/restic/templates/vps-restic-forget-prune.service.j2
+9
View File
@@ -0,0 +1,9 @@
[Unit]
Description=Restic retention and prune
[Service]
Type=oneshot
User=root
Group=root
UMask=0077
ExecStart=/usr/local/lib/vps-restic/forget-prune
ansible/roles/restic/templates/vps-restic-forget-prune.timer.j2
+11
View File
@@ -0,0 +1,11 @@
[Unit]
Description=Weekly Restic retention timer
[Timer]
OnCalendar={{ restic_forget_on_calendar }}
Persistent=true
RandomizedDelaySec=30m
Unit=vps-restic-forget-prune.service
[Install]
WantedBy=timers.target