Initial VPS operations handbook

2026-08-03 12:26:42 +08:00
commit b73125e5bc
97 changed files with 3641 additions and 0 deletions
@@ -0,0 +1,13 @@
+---
+healthcheck_install_root: /usr/local/lib/vps-health
+healthcheck_state_dir: /var/lib/vps-health
+healthcheck_log_dir: /var/log/vps-health
+healthcheck_service_name: vps-healthcheck
+healthcheck_timer_on_calendar: '*-*-* 06:15:00'
+healthcheck_timer_randomized_delay_sec: 15m
+healthcheck_backup_max_age_hours: 30
+healthcheck_tls_warn_days: 21
+healthcheck_profiles:
+  mailcow: mailcow.sh
+  vaultwarden: vaultwarden.sh
+  pdns: pdns.sh
@@ -0,0 +1,4 @@
+---
+- name: Reload systemd
+  ansible.builtin.systemd_service:
+    daemon_reload: true
@@ -0,0 +1,81 @@
+---
+- name: Validate known health-check profile
+  ansible.builtin.assert:
+    that:
+      - healthcheck_profile in healthcheck_profiles
+    fail_msg: "Unsupported healthcheck_profile: {{ healthcheck_profile }}"
+
+- name: Install health-check directories
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "{{ item.mode }}"
+  loop:
+    - path: "{{ healthcheck_install_root }}"
+      mode: "0755"
+    - path: "{{ healthcheck_state_dir }}"
+      mode: "0750"
+    - path: "{{ healthcheck_log_dir }}"
+      mode: "0750"
+
+- name: Install common health-check library
+  ansible.builtin.template:
+    src: health-common.sh.j2
+    dest: "{{ healthcheck_install_root }}/health-common.sh"
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Install service health-check script
+  ansible.builtin.template:
+    src: "{{ healthcheck_profiles[healthcheck_profile] }}.j2"
+    dest: "{{ healthcheck_install_root }}/{{ healthcheck_profiles[healthcheck_profile] }}"
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Install health-check dispatcher
+  ansible.builtin.template:
+    src: healthcheck-runner.sh.j2
+    dest: "{{ healthcheck_install_root }}/run"
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Install health-check systemd unit
+  ansible.builtin.template:
+    src: vps-healthcheck.service.j2
+    dest: "/etc/systemd/system/{{ healthcheck_service_name }}.service"
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Reload systemd
+
+- name: Install persistent health-check timer
+  ansible.builtin.template:
+    src: vps-healthcheck.timer.j2
+    dest: "/etc/systemd/system/{{ healthcheck_service_name }}.timer"
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Reload systemd
+
+- name: Install health-check log rotation
+  ansible.builtin.template:
+    src: logrotate-vps-healthcheck.j2
+    dest: /etc/logrotate.d/vps-healthcheck
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Enable health-check timer
+  ansible.builtin.systemd_service:
+    name: "{{ healthcheck_service_name }}.timer"
+    enabled: true
+    state: started
+    daemon_reload: true
+
+- name: Flush health-check systemd changes
+  ansible.builtin.meta: flush_handlers
@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+# Shared contract: one JSON object per run in /var/lib/vps-health/latest.json.
+# Status values: ok, warning, critical, unknown. Exit codes: 0 healthy/warning,
+# 2 critical, 3 unknown. Check detail must never include secret values.
+set -uo pipefail
+
+RESULT_DIR='{{ healthcheck_state_dir }}'
+LOG_DIR='{{ healthcheck_log_dir }}'
+HOST_NAME="$(hostname -f 2>/dev/null || hostname)"
+CHECK_NAME='{{ healthcheck_profile }}'
+STATUS=ok
+EXIT_CODE=0
+DETAILS=()
+
+record() {
+  local severity="$1" message="$2"
+  DETAILS+=("${severity}:${message}")
+  case "$severity" in
+    critical) STATUS=critical; EXIT_CODE=2 ;;
+    unknown) [[ "$STATUS" != critical ]] && STATUS=unknown; [[ "$EXIT_CODE" -eq 0 ]] && EXIT_CODE=3 ;;
+    warning) [[ "$STATUS" == ok ]] && STATUS=warning ;;
+  esac
+}
+
+require_command() {
+  command -v "$1" >/dev/null 2>&1 || record unknown "missing_command:$1"
+}
+
+compose_ps() {
+  docker compose --project-directory '{{ compose_project_dir }}' ps --all 2>&1
+}
+
+check_compose() {
+  local output
+  output="$(compose_ps)" || { record critical 'compose_ps_failed'; return; }
+  if grep -qiE 'Exited|Restarting|[[:space:]]Dead[[:space:]]' <<<"$output"; then
+    record critical 'compose_unhealthy_container'
+  else
+    record ok 'compose_ok'
+  fi
+}
+
+check_backup_freshness() {
+  local pattern="$1" newest now age
+  newest="$(find {{ compose_project_dir | quote }} -type f -path "$pattern" -printf '%T@\n' 2>/dev/null | sort -nr | head -n1)"
+  if [[ -z "$newest" ]]; then
+    record warning 'backup_not_found'
+    return
+  fi
+  now="$(date +%s)"
+  age="$(( now - ${newest%.*} ))"
+  if (( age > {{ healthcheck_backup_max_age_hours }} * 3600 )); then
+    record critical 'backup_stale'
+  else
+    record ok 'backup_fresh'
+  fi
+}
+
+check_https() {
+  local url="$1" expected="$2" code
+  code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 20 "$url" 2>/dev/null)" || {
+    record critical 'https_request_failed'; return;
+  }
+  [[ "$code" =~ $expected ]] && record ok "https_${code}" || record critical "https_${code}"
+}
+
+check_tls_days() {
+  local host="$1" port="$2" expires epoch remaining
+  expires="$(timeout 20 openssl s_client -connect "${host}:${port}" -servername "$host" </dev/null 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2-)" || {
+    record unknown 'tls_read_failed'; return;
+  }
+  epoch="$(date -d "$expires" +%s 2>/dev/null)" || { record unknown 'tls_date_parse_failed'; return; }
+  remaining="$(( (epoch - $(date +%s)) / 86400 ))"
+  if (( remaining < 0 )); then record critical 'tls_expired'
+  elif (( remaining < {{ healthcheck_tls_warn_days }} )); then record warning 'tls_near_expiry'
+  else record ok 'tls_valid'; fi
+}
+
+emit_result() {
+  local tmp detail_json
+  tmp="$(mktemp "${RESULT_DIR}/latest.json.XXXXXX")"
+  detail_json="$(printf '%s\n' "${DETAILS[@]:-unknown:no_details}" | python3 -c 'import json,sys; print(json.dumps([line.rstrip() for line in sys.stdin if line.strip()]))')"
+  python3 - "$tmp" "$HOST_NAME" "$CHECK_NAME" "$STATUS" "$EXIT_CODE" "$detail_json" <<'PY'
+import json, sys
+path, host, check, status, code, details = sys.argv[1:]
+with open(path, 'w', encoding='utf-8') as f:
+    json.dump({'schema': 1, 'host': host, 'check': check, 'status': status,
+               'exit_code': int(code), 'details': json.loads(details)}, f,
+              sort_keys=True, separators=(',', ':'))
+    f.write('\n')
+PY
+  chmod 0640 "$tmp"
+  mv "$tmp" "${RESULT_DIR}/latest.json"
+  cat "${RESULT_DIR}/latest.json"
+}
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+set -uo pipefail
+exec '{{ healthcheck_install_root }}/{{ healthcheck_profiles[healthcheck_profile] }}' >> '{{ healthcheck_log_dir }}/healthcheck.log' 2>&1
@@ -0,0 +1,9 @@
+{{ healthcheck_log_dir }}/healthcheck.log {
+    daily
+    rotate 14
+    missingok
+    notifempty
+    compress
+    delaycompress
+    create 0640 root root
+}
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -uo pipefail
+source '{{ healthcheck_install_root }}/health-common.sh'
+
+require_command docker
+require_command curl
+require_command openssl
+require_command dig
+require_command timeout
+check_compose
+
+watchdog="$(docker compose --project-directory '{{ compose_project_dir }}' logs --tail=40 watchdog-mailcow 2>&1)" || record critical 'watchdog_log_failed'
+grep -qiE '100%|healthy' <<<"$watchdog" || record warning 'watchdog_health_not_confirmed'
+
+queue="$(docker compose --project-directory '{{ compose_project_dir }}' exec -T postfix-mailcow postqueue -p 2>&1)" || record critical 'mail_queue_check_failed'
+grep -Fqi 'Mail queue is empty' <<<"$queue" || record warning 'mail_queue_nonempty'
+
+listeners="$(ss -lnt 2>/dev/null)"
+for port in 25 465 587 993 443; do
+  grep -qE ":${port}[[:space:]]" <<<"$listeners" || record critical "listener_missing_${port}"
+done
+
+check_https 'https://mx2.windy.me/' '^200$'
+check_tls_days mx2.windy.me 443
+smtp="$(timeout 10 bash -c "exec 3<>/dev/tcp/mx2.windy.me/25; printf 'EHLO health.local\\r\\nQUIT\\r\\n' >&3; cat <&3" 2>/dev/null)" || record critical 'smtp_connect_failed'
+grep -qiE 'Postcow|ESMTP' <<<"$smtp" || record critical 'smtp_banner_unexpected'
+
+for resolver in 1.1.1.1 8.8.8.8; do
+  mx="$(dig +short +time=3 +tries=1 "@${resolver}" windy.me MX 2>/dev/null)"
+  grep -Fqi 'mx2.windy.me' <<<"$mx" && { record ok 'mx_record_ok'; break; } || record warning 'mx_record_not_confirmed'
+done
+spf="$(dig +short +time=3 +tries=1 @1.1.1.1 windy.me TXT 2>/dev/null)"
+grep -Fqi 'v=spf1' <<<"$spf" || record warning 'spf_not_confirmed'
+check_backup_freshness '*/backup/*'
+emit_result
+exit "$EXIT_CODE"
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -uo pipefail
+source '{{ healthcheck_install_root }}/health-common.sh'
+
+require_command docker
+require_command curl
+require_command dig
+check_compose
+
+image="$(docker inspect pdns-auth --format '{{ '{{' }}.Config.Image{{ '}}' }}' 2>/dev/null)" || record critical 'pdns_container_missing'
+grep -Eq ':5\.0\.[6-9]|:5\.[1-9]\.' <<<"$image" || record warning 'pdns_version_not_confirmed'
+security="$(docker logs pdns-auth 2>&1 | grep -i 'Mandatory.*Security Update' || true)"
+[[ -z "$security" ]] || record critical 'pdns_security_update_banner'
+
+# The API key remains in the container environment; the request emits only the
+# reported version and no authentication material.
+api="$(docker compose --project-directory '{{ compose_project_dir }}' exec -T auth python3 - <<'PY' 2>&1
+import json, os, urllib.request
+request=urllib.request.Request('http://127.0.0.1:8081/api/v1/servers/localhost', headers={'X-API-Key': os.environ['PDNS_API_KEY']})
+print(json.load(urllib.request.urlopen(request, timeout=10))['version'])
+PY
+)" || record critical 'pdns_api_failed'
+grep -Eq '^5\.' <<<"$api" || record critical 'pdns_api_version_invalid'
+
+for zone in windy.me wsvc.info chans.xyz; do
+  primary="$(dig +short @154.36.174.161 SOA "$zone" 2>/dev/null)"
+  secondary="$(dig +short @202.91.35.141 SOA "$zone" 2>/dev/null)"
+  [[ -n "$primary" ]] || record critical "soa_primary_missing_${zone}"
+  [[ -n "$secondary" ]] || record critical "soa_secondary_missing_${zone}"
+  [[ "$primary" == "$secondary" ]] || record warning "soa_secondary_lag_${zone}"
+done
+
+check_https 'https://pdns.wsvc.info/' '^30[12]$'
+check_https 'https://pgweb.wsvc.info/' '^(200|401)$'
+check_backup_freshness '*/backup/*.sql.gz'
+emit_result
+exit "$EXIT_CODE"
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -uo pipefail
+source '{{ healthcheck_install_root }}/health-common.sh'
+
+require_command docker
+require_command curl
+require_command openssl
+require_command python3
+check_compose
+
+health="$(docker compose --project-directory '{{ compose_project_dir }}' ps --format json 2>&1)" || record critical 'compose_status_unavailable'
+grep -Fq 'vaultwarden' <<<"$health" || record critical 'vaultwarden_missing'
+grep -Fq 'vw-db' <<<"$health" || record critical 'postgres_missing'
+
+check_https 'https://auth.wsvc.info/' '^200$'
+check_tls_days auth.wsvc.info 443
+
+# Read effective config only inside the service and report booleans/fingerprints,
+# never its SMTP password or other secret fields.
+smtp_result="$(docker compose --project-directory '{{ compose_project_dir }}' exec -T vaultwarden python3 - <<'PY' 2>&1
+import json, pathlib, smtplib, ssl
+cfg=json.loads(pathlib.Path('/data/config.json').read_text())
+host=cfg.get('smtp_host'); port=int(cfg.get('smtp_port') or 0)
+user=cfg.get('smtp_username')
+smtp_secret=cfg.get('smtp_password')
+if not all((host, port, user, smtp_secret)):
+    raise SystemExit('smtp_config_incomplete')
+with smtplib.SMTP(host, port, timeout=15) as client:
+    client.ehlo(); client.starttls(context=ssl.create_default_context()); client.ehlo(); client.login(user, smtp_secret)
+print('smtp_auth_ok')
+PY
+)"
+grep -Fqx 'smtp_auth_ok' <<<"$smtp_result" || record critical 'smtp_auth_failed'
+
+# Detect drift without exposing the values: matching SHA-256 digests are used only
+# internally and the result is a boolean.
+drift_result="$(python3 - <<'PY'
+import hashlib, json, pathlib, re
+root=pathlib.Path('{{ compose_project_dir }}')
+env=root.joinpath('.env').read_text()
+match=re.search(r'^SMTP_PASSWORD=(.*)$', env, re.M)
+config=json.loads(root.joinpath('vw-data/config.json').read_text())
+value=(match.group(1).strip().strip('"\'') if match else '')
+print('smtp_password_match' if value and value == (config.get('smtp_password') or '') else 'smtp_password_drift')
+PY
+ 2>&1)" || record unknown 'smtp_drift_check_failed'
+grep -Fqx 'smtp_password_match' <<<"$drift_result" || record critical 'smtp_password_drift'
+
+check_backup_freshness '*/backups/*.sql.gz'
+emit_result
+exit "$EXIT_CODE"
@@ -0,0 +1,19 @@
+[Unit]
+Description=Read-only VPS health check (%i)
+Wants=network-online.target
+After=network-online.target docker.service
+
+[Service]
+Type=oneshot
+User=root
+Group=root
+UMask=0027
+ExecStart={{ healthcheck_install_root }}/run
+# The optional dispatcher is installed only by email_alert when explicitly enabled.
+# This service remains healthy if alerting is intentionally unavailable.
+ExecStartPost=-/usr/local/lib/vps-health/alert-dispatch
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectHome=true
+ProtectSystem=full
+ReadWritePaths={{ healthcheck_state_dir }} {{ healthcheck_log_dir }}
@@ -0,0 +1,11 @@
+[Unit]
+Description=Daily read-only VPS health check
+
+[Timer]
+OnCalendar={{ healthcheck_timer_on_calendar }}
+Persistent=true
+RandomizedDelaySec={{ healthcheck_timer_randomized_delay_sec }}
+Unit={{ healthcheck_service_name }}.service
+
+[Install]
+WantedBy=timers.target