diff --git a/AGENTS.md b/AGENTS.md index 8e9f448..49b5d37 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,11 @@ Also readable as `agent.md` (symlink → this file). 2. Open the matching [`hosts/.md`](hosts/) for SSH, roles, paths, and quirks. 3. For common tasks, follow a runbook under [`runbooks/`](runbooks/). 4. Prefer read-only checks first; change only after confirming current state. -5. Default access: `ssh -4 windy@` (pubkey). Prefer **IPv4** from this WSL client (AAAA often exists but IPv6 route does not). +5. For routine checks and approved service reconciliation, run the matching + Ansible playbook from `ansible/`; see [routine Ansible operations](runbooks/ansible-operations.md). +6. Default SSH access (`ssh -4 windy@`) is for focused diagnostics, + imperative upstream procedures, and incident work. Prefer **IPv4** from this + WSL client (AAAA often exists but IPv6 route does not). ## Active hosts (quick map) @@ -59,6 +63,7 @@ Also readable as `agent.md` (symlink → this file). | PowerDNS upstream refs | [docs/pdns-upstream.md](docs/pdns-upstream.md) | | Matrix health | [runbooks/matrix-health.md](runbooks/matrix-health.md) | | Matrix upstream refs | [docs/matrix-upstream.md](docs/matrix-upstream.md) | +| Routine Ansible operations | [runbooks/ansible-operations.md](runbooks/ansible-operations.md) | ```bash ./scripts/check-mx2.sh @@ -71,6 +76,8 @@ Also readable as `agent.md` (symlink → this file). - Never commit secrets: passwords, API keys, private keys, `.env`, `mailcow.conf` DB passwords, Vaultwarden `ADMIN_TOKEN` / `.smtp-credentials`. - Secrets live **only on the servers** (e.g. `/opt/vaultwarden/.env`). Do not copy them from Obsidian into this repo. - Ask before destructive actions: `docker compose down -v`, DB drops, DNS changes, force-push, disk wipes. +- Ansible reconciliation is intentionally allowlisted and requires an explicit + confirmation variable; it is not a general remote-command facility. - Do not modify local `~/.ssh/config` or git config unless explicitly asked. ## SSH access safety — mandatory change procedure diff --git a/ansible/inventory/hosts.yml b/ansible/inventory/hosts.yml index 82f06ef..260aaf7 100644 --- a/ansible/inventory/hosts.yml +++ b/ansible/inventory/hosts.yml @@ -18,18 +18,33 @@ all: service_role: mailcow compose_project_dir: /opt/mail healthcheck_profile: mailcow + service_reconcile_services: + all: + compose_args: [--force-recreate] us2: ansible_host: us2.wsvc.info ansible_host_ipv4: 193.9.44.165 service_role: vaultwarden compose_project_dir: /opt/vaultwarden healthcheck_profile: vaultwarden + service_reconcile_services: + vaultwarden: + compose_args: [--force-recreate] + service_reconcile_traefik_restart_targets: [vaultwarden] hk2: ansible_host: hk2.chans.xyz ansible_host_ipv4: 154.36.174.161 service_role: powerdns compose_project_dir: /opt/pdns healthcheck_profile: pdns + service_reconcile_services: + auth: + compose_args: [--force-recreate] + poweradmin: + compose_args: [--no-deps, --force-recreate] + backup: + compose_args: [--no-deps, --force-recreate] + service_reconcile_traefik_restart_targets: [poweradmin] mailcow: hosts: mx2: diff --git a/ansible/playbooks/baseline.yml b/ansible/playbooks/baseline.yml index 95b3689..8f5d6ca 100644 --- a/ansible/playbooks/baseline.yml +++ b/ansible/playbooks/baseline.yml @@ -1,7 +1,9 @@ --- # Baseline starts audit-only. Opt-in variables are deliberately false by default. - name: Apply controlled common baseline - hosts: managed + # The baseline role is OS-level and does not assume Docker Compose, so it + # applies to the Matrix K3s node as well as the Compose hosts. + hosts: all become: true gather_facts: false roles: diff --git a/ansible/playbooks/compose-reconcile.yml b/ansible/playbooks/compose-reconcile.yml new file mode 100644 index 0000000..e19abae --- /dev/null +++ b/ansible/playbooks/compose-reconcile.yml @@ -0,0 +1,12 @@ +--- +# Intentional, narrowly scoped Compose reconciliation. This never changes +# compose files, images, secrets, DNS, or databases. It is gated because a +# container recreate can briefly interrupt a public service. +- name: Reconcile explicitly selected Docker Compose services + hosts: docker_hosts + become: true + gather_facts: false + serial: 1 + roles: + - role: compose_reconcile + tags: [compose, reconcile, mutating] diff --git a/ansible/playbooks/health-report.yml b/ansible/playbooks/health-report.yml new file mode 100644 index 0000000..963ba27 --- /dev/null +++ b/ansible/playbooks/health-report.yml @@ -0,0 +1,50 @@ +--- +# Execute the already-installed, sanitized local health checks on demand. +# This is read-only with respect to the managed applications; the scripts only +# inspect state and update their own health result files/logs. +- name: Report Docker Compose service health + hosts: managed + become: true + gather_facts: false + tasks: + - name: Run the deployed service health check + ansible.builtin.command: + argv: + - /usr/local/lib/vps-health/run + register: service_health_report + changed_when: false + failed_when: service_health_report.rc not in [0, 1, 2] + + - name: Display sanitized service health result + ansible.builtin.debug: + msg: "{{ service_health_report.stdout | default(service_health_report.stderr, true) }}" + + - name: Fail when service health is degraded + ansible.builtin.fail: + msg: "Health check reported a degraded state on {{ inventory_hostname }}." + when: service_health_report.rc != 0 + +- name: Report Matrix health + hosts: matrix + become: true + gather_facts: false + tasks: + - name: Run the deployed Matrix health check + ansible.builtin.command: + argv: + - "{{ matrix_healthcheck_script_path | default('/usr/local/lib/vps-health/matrix-k3s') }}" + register: matrix_health_report + changed_when: false + + - name: Parse Matrix health result + ansible.builtin.set_fact: + matrix_health_result: "{{ matrix_health_report.stdout | from_json }}" + + - name: Display Matrix health result + ansible.builtin.debug: + var: matrix_health_result + + - name: Fail when Matrix health is degraded + ansible.builtin.fail: + msg: "Matrix health is {{ matrix_health_result.status }} on {{ inventory_hostname }}." + when: matrix_health_result.status != 'ok' diff --git a/ansible/playbooks/maintenance-preview.yml b/ansible/playbooks/maintenance-preview.yml index b2729ef..918624f 100644 --- a/ansible/playbooks/maintenance-preview.yml +++ b/ansible/playbooks/maintenance-preview.yml @@ -2,7 +2,9 @@ # Preview only. This playbook does not install updates, restart services, or # change DNS/secrets. A separate, manually reviewed change is required to act. - name: Preview pending maintenance without changes - hosts: managed + # Package/reboot inspection is host-level and is safe for both Compose and + # K3s nodes. + hosts: all become: true gather_facts: false tasks: diff --git a/ansible/roles/compose_reconcile/defaults/main.yml b/ansible/roles/compose_reconcile/defaults/main.yml new file mode 100644 index 0000000..d87c7d7 --- /dev/null +++ b/ansible/roles/compose_reconcile/defaults/main.yml @@ -0,0 +1,6 @@ +--- +# Both variables must be supplied explicitly by the operator. Allowed services +# and their safe Compose flags belong in inventory, not on the command line. +service_reconcile_confirm: false +service_reconcile_targets: [] +service_reconcile_restart_traefik: false diff --git a/ansible/roles/compose_reconcile/tasks/main.yml b/ansible/roles/compose_reconcile/tasks/main.yml new file mode 100644 index 0000000..1c1f5e7 --- /dev/null +++ b/ansible/roles/compose_reconcile/tasks/main.yml @@ -0,0 +1,69 @@ +--- +- name: Require explicit confirmation and selected services + ansible.builtin.assert: + that: + - service_reconcile_confirm | bool + - service_reconcile_targets | length > 0 + fail_msg: >- + This operation can recreate containers. Supply + service_reconcile_confirm=true and a non-empty + service_reconcile_targets list after reviewing the change. + +- name: Require a host-specific reconciliation policy + ansible.builtin.assert: + that: + - service_reconcile_services is defined + fail_msg: "No reconciliation policy is defined for {{ inventory_hostname }}." + +- name: Validate selected services are allowed on this host + ansible.builtin.assert: + that: + - item in service_reconcile_services + fail_msg: "{{ item }} is not an allowed reconciliation target on {{ inventory_hostname }}." + loop: "{{ service_reconcile_targets }}" + +- name: Validate optional Traefik restart scope + ansible.builtin.assert: + that: + - >- + not (service_reconcile_restart_traefik | bool) or + (service_reconcile_targets + | select('in', service_reconcile_traefik_restart_targets | default([])) + | list | length > 0) + fail_msg: >- + Traefik may only be restarted as the explicit post-reconcile action for + an inventory-approved service on this host. + +- name: Reconcile approved Compose services + ansible.builtin.command: + argv: >- + {{ ['docker', 'compose', '--project-directory', compose_project_dir, 'up', '-d'] + + service_reconcile_services[item].compose_args + + ([] if item == 'all' else [item]) }} + loop: "{{ service_reconcile_targets }}" + register: service_reconcile_result + changed_when: true + loop_control: + label: "{{ item }}" + +- name: Restart Traefik after approved reconciliation when requested + ansible.builtin.command: + argv: [docker, restart, traefik] + when: service_reconcile_restart_traefik | bool + changed_when: true + +- name: Report reconciled Compose state + ansible.builtin.command: + argv: + - docker + - compose + - --project-directory + - "{{ compose_project_dir }}" + - ps + - --all + changed_when: false + register: service_reconcile_ps + +- name: Display reconciled Compose state + ansible.builtin.debug: + var: service_reconcile_ps.stdout_lines diff --git a/ansible/roles/matrix_healthcheck/defaults/main.yml b/ansible/roles/matrix_healthcheck/defaults/main.yml index 1c1bdeb..82d5873 100644 --- a/ansible/roles/matrix_healthcheck/defaults/main.yml +++ b/ansible/roles/matrix_healthcheck/defaults/main.yml @@ -6,7 +6,9 @@ matrix_healthcheck_service_name: matrix-k3s-healthcheck.service matrix_healthcheck_timer_name: matrix-k3s-healthcheck.timer matrix_healthcheck_timer_on_calendar: '*-*-* 06:00:00 UTC' matrix_healthcheck_timer_randomized_delay_sec: 15m -matrix_healthcheck_namespace: matrix-system +# ESS workloads live in this namespace; matrix-system only has supporting +# chart-independent resources. +matrix_healthcheck_namespace: ess matrix_healthcheck_backup_path: /var/backups/matrix matrix_healthcheck_backup_max_age_hours: 30 matrix_healthcheck_warn_percent: 80 diff --git a/docs/matrix-upstream.md b/docs/matrix-upstream.md index c779429..4782e8e 100644 --- a/docs/matrix-upstream.md +++ b/docs/matrix-upstream.md @@ -64,19 +64,20 @@ The ESS chart is deployed via Ansible (`ansible/playbooks/matrix-stack.yml`) wit ## Upgrades +Update the pinned `matrix_stack_chart_version` in +`ansible/inventory/hosts.yml` through normal review, then apply the chart via +the gated Ansible playbook: + ```bash -# Pull latest chart version -helm pull oci://ghcr.io/element-hq/ess-helm/matrix-stack --version - -# Review changes -helm diff upgrade ess oci://ghcr.io/element-hq/ess-helm/matrix-stack \ - -n ess -f /etc/ess/hostnames.yaml -f /etc/ess/tls.yaml - -# Apply -helm upgrade --install ess oci://ghcr.io/element-hq/ess-helm/matrix-stack \ - -n ess -f /etc/ess/hostnames.yaml -f /etc/ess/tls.yaml --wait +cd ansible +ansible-playbook playbooks/matrix-preflight.yml +ansible-playbook playbooks/matrix-stack.yml --limit matrix \ + -e 'matrix_stack_enabled=true' ``` +Use `helm diff` on the target only as a read-only supplement when the plugin is +installed; do not apply a release with a hand-written `helm upgrade` command. + Before upgrading: - Matrix backup automation is currently paused; establish and test a reviewed backup design before treating the deployment as recoverable. - Review chart changelog and database migration notes diff --git a/docs/pdns-upstream.md b/docs/pdns-upstream.md index a2e67b1..c6f2b90 100644 --- a/docs/pdns-upstream.md +++ b/docs/pdns-upstream.md @@ -79,3 +79,6 @@ Drop-in (from `.env`, never commit): `gpgsql-dbname`, `gpgsql-user`, `gpgsql-pas - Host facts: [hosts/hk2.chans.xyz.md](../hosts/hk2.chans.xyz.md) - Health runbook: [runbooks/pdns-health.md](../runbooks/pdns-health.md) +- Routine status: `ansible/playbooks/health-report.yml --limit powerdns` +- Reviewed Compose reconciliations: `ansible/playbooks/compose-reconcile.yml` + (`auth`, `poweradmin`, and `backup` are the only allowlisted targets) diff --git a/docs/vaultwarden-upstream.md b/docs/vaultwarden-upstream.md index 59b9fc1..a7fa5aa 100644 --- a/docs/vaultwarden-upstream.md +++ b/docs/vaultwarden-upstream.md @@ -50,4 +50,13 @@ Official stance: prefer environment variables; `config.json` is **not** the reco 5. Do not commit secrets; do not copy passwords from Obsidian into this repo. 6. Verify SMTP with AUTH (not TCP-only) — [vaultwarden-health](../runbooks/vaultwarden-health.md) §5. +## Operations path + +Use `ansible/playbooks/health-report.yml --limit vaultwarden` for routine +status. After a reviewed effective-configuration change that needs a container +recreate, use the allowlisted `compose-reconcile.yml` target `vaultwarden`. +Add `service_reconcile_restart_traefik=true` only when the documented +post-recreate 404 occurs. Direct Compose commands remain diagnostic tools, not +the routine control path. + Replay migration steps: [runbooks/vaultwarden-sqlite-to-postgres.md](../runbooks/vaultwarden-sqlite-to-postgres.md). diff --git a/hosts/hk2.chans.xyz.md b/hosts/hk2.chans.xyz.md index c1753eb..8dd7cf7 100644 --- a/hosts/hk2.chans.xyz.md +++ b/hosts/hk2.chans.xyz.md @@ -110,12 +110,18 @@ See full shape in [docs/pdns-upstream.md](../docs/pdns-upstream.md). Live secret - [pdns-upstream docs](../docs/pdns-upstream.md) ```bash -ssh -4 windy@hk2.chans.xyz -cd /opt/pdns && docker compose ps -a -docker compose logs -f auth poweradmin backup -./scripts/manual-backup.sh -# after pdns.conf / secrets.j2 / auth env changes: -docker compose up -d --force-recreate auth +cd ansible +ansible-playbook playbooks/health-report.yml --limit powerdns +# after reviewed auth config / secret-template / environment changes: +ansible-playbook playbooks/compose-reconcile.yml --limit powerdns \ + -e '{"service_reconcile_confirm": true, "service_reconcile_targets": ["auth"]}' +``` + +Direct SSH remains appropriate for streaming incident logs or running the +on-server, intentionally imperative manual backup command: + +```bash +ssh -4 windy@hk2.chans.xyz 'cd /opt/pdns && ./scripts/manual-backup.sh' ``` ```bash diff --git a/hosts/mx2.windy.me.md b/hosts/mx2.windy.me.md index 102dca1..cb8f3b2 100644 --- a/hosts/mx2.windy.me.md +++ b/hosts/mx2.windy.me.md @@ -29,7 +29,10 @@ ## Ops notes -- Manage with: `cd /opt/mail && docker compose …` +- Use the Ansible health report for routine checks. Use the gated Ansible + Compose reconcile playbook for a reviewed full-stack recreate; retain direct + Compose access only for focused incident diagnosis and Mailcow's interactive + upstream updater. - `SKIP_CLAMD=y` may still leave `clamd` running after updates; recreate if you need it truly skipped - Host has ~8G RAM, no swap; watch memory under load - Unrelated stopped containers may exist (`zc-*` monitoring stack) — not part of mailcow diff --git a/hosts/synapse.chans.xyz.md b/hosts/synapse.chans.xyz.md index 1322b8f..4f4afca 100644 --- a/hosts/synapse.chans.xyz.md +++ b/hosts/synapse.chans.xyz.md @@ -90,11 +90,16 @@ Path: `/etc/matrix-bootstrap/` (root:root, 0700) — individual files per secret Inventory group: `matrix_production` (but NOT in `managed` which expects Docker Compose). -Playbooks in `ansible/playbooks/matrix-*.yml`. +Routine status: `cd ansible && ansible-playbook playbooks/health-report.yml --limit matrix`. +Use `matrix-preflight.yml` before a Matrix/K3s change, and use the gated +`matrix-stack.yml` to apply a reviewed ESS chart version. Playbooks are in +`ansible/playbooks/matrix-*.yml`; direct SSH is reserved for focused K3s +diagnosis and imperative recovery work. ## Known limitations - MatrixRTC / Element Call / LiveKit / Coturn not deployed (`mrtc.chans.xyz` reserved only) -- SMTP email not yet configured (requires manual bootstrap + Helm upgrade) +- SMTP email not yet configured (requires manual secret bootstrap followed by a + reviewed Ansible stack deployment) - No off-site Restic backup - Single-node K3s (no HA for control plane) diff --git a/hosts/us2.wsvc.info.md b/hosts/us2.wsvc.info.md index 519cfbb..088a848 100644 --- a/hosts/us2.wsvc.info.md +++ b/hosts/us2.wsvc.info.md @@ -69,10 +69,11 @@ Effective `config.json`: domain `https://auth.wsvc.info`, SMTP 587/starttls to ` - [vaultwarden-upstream docs](../docs/vaultwarden-upstream.md) ```bash -ssh -4 windy@us2.wsvc.info -cd /opt/vaultwarden && docker compose ps -a -# after recreating vaultwarden, if auth 404s: -docker restart traefik +cd ansible +ansible-playbook playbooks/health-report.yml --limit vaultwarden +# after a reviewed Vaultwarden configuration change (and only if auth 404s): +ansible-playbook playbooks/compose-reconcile.yml --limit vaultwarden \ + -e '{"service_reconcile_confirm": true, "service_reconcile_targets": ["vaultwarden"], "service_reconcile_restart_traefik": true}' ``` ## Other software on this host (stubs) diff --git a/inventory/hosts.md b/inventory/hosts.md index 75cd4ec..71594b1 100644 --- a/inventory/hosts.md +++ b/inventory/hosts.md @@ -1,5 +1,10 @@ # Host inventory +Routine health, maintenance previews, and reviewed Compose reconciliations are +run through Ansible. SSH endpoints below remain the access path for focused +diagnosis and procedures that are deliberately interactive or destructive; see +[the Ansible operations runbook](../runbooks/ansible-operations.md). + | Host | Role | SSH | IPv4 | Status | Facts | |------|------|-----|------|--------|-------| | mx2.windy.me | mailcow (primary MX prio 20) | `ssh -4 windy@mx2.windy.me` | 194.163.160.244 | active | [hosts/mx2.windy.me.md](../hosts/mx2.windy.me.md) | @@ -26,4 +31,4 @@ | https://synapse.chans.xyz | Synapse API | Client-Server + Federation API | | https://account.chans.xyz | MAS | Matrix Authentication Service (local passwords) | | https://admin.chans.xyz | Element Admin | Admin console (MAS admin auth) | -| `mrtc.chans.xyz` | MatrixRTC | **Reserved** – not deployed | \ No newline at end of file +| `mrtc.chans.xyz` | MatrixRTC | **Reserved** – not deployed | diff --git a/plans/2026-08-02-vps-maintenance-ansible-restic-refactor-v1.md b/plans/2026-08-02-vps-maintenance-ansible-restic-refactor-v1.md index 3388f66..ba14396 100644 --- a/plans/2026-08-02-vps-maintenance-ansible-restic-refactor-v1.md +++ b/plans/2026-08-02-vps-maintenance-ansible-restic-refactor-v1.md @@ -2,12 +2,12 @@ ## Objective -將現有 VPS 維運知識庫漸進重構為一個以 Ansible 管理一致性、以 systemd timers 執行每日唯讀檢查、以 Restic 建立加密異機備份的個人 SRE 維運專案。第一期僅納管 mx2、us2、hk2;自動化只能觀測與告警,任何更新、重啟、修復、DNS 或機密變更都必須人工確認。 +將現有 VPS 維運知識庫漸進重構為一個以 Ansible 管理一致性、以 systemd timers 執行每日唯讀檢查、以 Restic 建立加密異機備份的個人 SRE 維運專案。現納管 mx2、us2、hk2 與 Matrix K3s 主機。自動化預設只能觀測與告警;受 allowlist 和顯式確認保護的 Compose reconciliation 可處理已審查的服務重建,DNS、機密與破壞性資料操作仍需人工確認。 ## Confirmed Decisions - [x] 定位為個人 SRE/維運手冊庫,而非全自動修復平台。 -- [x] 第一階段完整納管 mx2、us2、hk2;mx、us1、us4 待完成盤點與緊急存取驗證後才納管。 +- [x] 已納管 mx2、us2、hk2 與 Matrix K3s;mx、us1、us4 待完成盤點與緊急存取驗證後才納管。 - [x] 每日自動做唯讀檢查與 Email 通知;每週人工審查、每月人工維護、每季復原演練與權限盤點。 - [x] 以 Email 作為唯一告警與摘要通道。 - [x] 採用本機快速復原加密異機副本的 3-2-1 最小可行策略。 @@ -17,7 +17,7 @@ ## Implementation Plan -- [x] **Status: Done — 建立去敏 Ansible inventory。** 為 mx2、us2、hk2 建立主機與服務群組,將非機密主機資料與現有人類可讀的 `inventory/hosts.md` 對應;理由是提供可審查的機器可讀編排層,且不取代既有事實來源。 +- [x] **Status: Done — 建立去敏 Ansible inventory。** 為 mx2、us2、hk2 與 Matrix 建立主機與服務群組,將非機密主機資料與現有人類可讀的 `inventory/hosts.md` 對應;理由是提供可審查的機器可讀編排層,且不取代既有事實來源。 - [x] **Status: Done — 實作 audit-only Ansible playbook。** 只收集連線、OS、磁碟、systemd、Docker Compose 和依賴命令狀態,不做設定或服務改動;理由是先確認控制面與真實現況。 - [x] **Status: Done — 定義跨服務健康檢查結果合約。** 統一去敏結構化結果、日誌位置、嚴重度及 exit code,讓人工執行、systemd 和 Ansible audit 可共同消費;理由是避免各服務告警語意漂移。 - [x] **Status: Done — 保留並模組化 Mailcow 健康檢查。** 持續檢查 Compose、watchdog、queue、listeners、HTTP/HTTPS、TLS、SMTP、DNS/PTR/MX/SPF;理由是現有腳本已覆蓋關鍵郵件服務面向。 @@ -32,13 +32,14 @@ - [!] **Status: Blocked — 補齊服務級復原 runbook。** 阻塞:需在選定 Restic repository、建立實際 snapshot 並確認各服務的實際備份輸出後,才能編寫可驗證的 restore 前置條件、順序與 rollback;禁止臆造 backend/credentials 或未驗證還原命令。 - [!] **Status: Blocked — 執行隔離式復原演練。** 阻塞:尚未選定/配置 Restic 異機 repository,亦尚無可供還原的異機 snapshot;演練不得對現有生產資料執行。 - [x] **Status: Done — 實作受控 common baseline 與 maintenance playbook。** 僅在 audit 穩定後納入 SSH 稽核、時間同步、logrotate、更新預覽與人工確認的維護操作;理由是避免工具導入期間同時改變服務狀態。 +- [x] **Status: Done (2026-08-03) — 將例行操作收斂為 Ansible 入口。** 新增 on-demand health report 與 allowlisted Compose reconciliation;health、maintenance preview、baseline 現可涵蓋 Matrix。互動式 Mailcow 更新、資料遷移、DNS 與機密操作保留人工程序。 - [x] **Status: Done (separate change defined) — 將 PowerDNS API key、DB password 與 TSIG 輪替列為分離變更。** 已明確保持 API key、DB password 與 TSIG 為三項獨立、需人工核准的變更,並要求逐步驗證 Auth、Poweradmin、AXFR/NOTIFY、DNSSEC 與 ns2 同步;未輪替任何機密。 - [!] **Status: Blocked — 盤點 mx、us1、us4。** 阻塞:inventory 僅記錄 `TBD` SSH/角色或缺少完整事實,尚未提供可驗證的存取方式與 provider/recovery console 資訊;不得猜測或嘗試未授權存取。 ## Verification Criteria -- [ ] 三台 active 主機能被 Ansible 正確解析及執行 audit,且 audit 不產生主機變更。 -- [ ] 三台主機每日產生去敏健康結果;服務、容量、TLS、公開端點和備份新鮮度異常均能被偵測。 +- [ ] 四台 active 主機能被 Ansible 正確解析;Compose 主機可執行 audit,且 audit 不產生主機變更。 +- [ ] 四台主機每日產生去敏健康結果;服務、容量、TLS、公開端點和備份新鮮度異常均能被偵測。 - [ ] 正常狀態寄送 Email 摘要;受控測試異常能觸發一次可讀且不含機密的 Email 告警。 - [ ] 每個核心服務至少有一份加密、異機、可列出且通過完整性檢查的 Restic snapshot。 - [ ] 至少完成一次隔離式實際還原,並確認其結果符合或明確量化偏離服務 RPO/RTO。 diff --git a/runbooks/ansible-operations.md b/runbooks/ansible-operations.md new file mode 100644 index 0000000..b0f2318 --- /dev/null +++ b/runbooks/ansible-operations.md @@ -0,0 +1,61 @@ +# Runbook: routine operations through Ansible + +Run commands from `ansible/`. The inventory forces IPv4 and uses the `windy` +account with sudo. Do a read-only health pass before any reconciliation. + +## Health report (read-only) + +```bash +cd ansible +ansible-playbook playbooks/health-report.yml +ansible-playbook playbooks/health-report.yml --limit mailcow +ansible-playbook playbooks/health-report.yml --limit matrix +``` + +This invokes the existing server-local checks and prints sanitized output; it +does not expose secrets or alter application configuration. A warning or +critical health result makes the play fail deliberately. + +If the local health check has not been installed yet, deploy it first: + +```bash +ansible-playbook playbooks/healthchecks.yml +ansible-playbook playbooks/matrix-healthchecks.yml +``` + +## Reconcile a known Compose service (mutating) + +Only use this after a reviewed configuration change that requires a recreate. +The allowlist and flags are in inventory; arbitrary commands and service names +are rejected. Run one host at a time and re-run the health report afterwards. + +```bash +# PowerDNS auth after its configuration changed +ansible-playbook playbooks/compose-reconcile.yml --limit powerdns \ + -e '{"service_reconcile_confirm": true, "service_reconcile_targets": ["auth"]}' + +# Vaultwarden, including the documented Traefik refresh for a post-recreate 404 +ansible-playbook playbooks/compose-reconcile.yml --limit vaultwarden \ + -e '{"service_reconcile_confirm": true, "service_reconcile_targets": ["vaultwarden"], "service_reconcile_restart_traefik": true}' + +# Poweradmin after changing its environment or Traefik labels; restart its +# Traefik container only when the UI is returning 404 +ansible-playbook playbooks/compose-reconcile.yml --limit powerdns \ + -e '{"service_reconcile_confirm": true, "service_reconcile_targets": ["poweradmin"], "service_reconcile_restart_traefik": true}' +``` + +Do not use this playbook for a Mailcow update, database migration, DNS record +change, or secret rotation. Those operations require their dedicated reviewed +and, where appropriate, interactive procedures. + +## Host-level maintenance + +These playbooks cover every inventory host, including the Matrix K3s node: + +```bash +# Read-only pending package and reboot report +ansible-playbook playbooks/maintenance-preview.yml + +# Baseline observations; logrotate management remains opt-in +ansible-playbook playbooks/baseline.yml +``` diff --git a/runbooks/mailcow-health.md b/runbooks/mailcow-health.md index eca5956..e9c471a 100644 --- a/runbooks/mailcow-health.md +++ b/runbooks/mailcow-health.md @@ -2,9 +2,15 @@ Target: [mx2.windy.me](../hosts/mx2.windy.me.md) Path: `/opt/mail` -Prefer: `./scripts/check-mx2.sh` from this repo, or the steps below over SSH. +Prefer: the Ansible health report (`ansible/playbooks/health-report.yml`), or +`./scripts/check-mx2.sh` from this repo. The SSH commands below remain useful +for focused, interactive diagnosis after the report identifies a problem. -## 1. Containers +## Focused SSH diagnostics + +Use these only after the Ansible health report needs investigation. + +### 1. Containers ```bash ssh windy@mx2.windy.me 'cd /opt/mail && docker compose ps -a' @@ -12,7 +18,7 @@ ssh windy@mx2.windy.me 'cd /opt/mail && docker compose ps -a' Expect mailcow (`cow-*`) services **Up**. Note any Exit/Restarting. -## 2. Watchdog +### 2. Watchdog ```bash ssh windy@mx2.windy.me 'cd /opt/mail && docker compose logs --tail=40 watchdog-mailcow' @@ -20,7 +26,7 @@ ssh windy@mx2.windy.me 'cd /opt/mail && docker compose logs --tail=40 watchdog-m Expect recent lines at **100%** for Nginx, Postfix, Dovecot, MySQL, Redis, SOGo, Rspamd, ACME, etc. -## 3. Queue +### 3. Queue ```bash ssh windy@mx2.windy.me 'cd /opt/mail && docker compose exec -T postfix-mailcow postqueue -p' @@ -28,7 +34,7 @@ ssh windy@mx2.windy.me 'cd /opt/mail && docker compose exec -T postfix-mailcow p Healthy: empty / `Mail queue is empty`. -## 4. Local listeners +### 4. Local listeners ```bash ssh windy@mx2.windy.me 'ss -tlnp | grep -E ":(25|465|587|143|993|995|80|443|4190)\\s"' diff --git a/runbooks/mailcow-update.md b/runbooks/mailcow-update.md index af4382a..57e8950 100644 --- a/runbooks/mailcow-update.md +++ b/runbooks/mailcow-update.md @@ -31,7 +31,9 @@ If update asks to stop/recreate containers, allow it unless the user said otherw 4. If `SKIP_CLAMD=y` but `cow-clamd-mailcow-1` is still Up and that is unwanted: ```bash - ssh windy@mx2.windy.me 'cd /opt/mail && docker compose up -d --force-recreate' + cd ansible + ansible-playbook playbooks/compose-reconcile.yml --limit mailcow \ + -e '{"service_reconcile_confirm": true, "service_reconcile_targets": ["all"]}' ``` Only after user OK; recreate briefly interrupts mail. diff --git a/runbooks/matrix-health.md b/runbooks/matrix-health.md index a128ffa..b0d6ca1 100644 --- a/runbooks/matrix-health.md +++ b/runbooks/matrix-health.md @@ -2,7 +2,14 @@ Monitor the Matrix homeserver running on `synapse.chans.xyz` (ESS chart `26.7.2`, K3s node). -## Basic checks +Prefer `cd ansible && ansible-playbook playbooks/health-report.yml --limit matrix` +for the routine report. The SSH commands below remain diagnostic detail. + +## Focused SSH diagnostics + +Use these only after the Ansible health report needs investigation. + +### Basic checks ```bash # Overall status @@ -15,7 +22,7 @@ ssh -4 windy@synapse.chans.xyz 'sudo k3s kubectl -n ess get pods -o wide | grep ssh -4 windy@synapse.chans.xyz 'df -hP /' ``` -## Certificate verification +### Certificate verification ```bash ssh -4 windy@synapse.chans.xyz 'sudo k3s kubectl -n ess get certificate' @@ -29,7 +36,7 @@ All certificates should show `READY=True`. Domains: - `admin.chans.xyz` - `mrtc.chans.xyz` (placeholder only) -## External endpoint verification +### External endpoint verification ```bash # Client API diff --git a/runbooks/pdns-health.md b/runbooks/pdns-health.md index c74e1fa..1313b49 100644 --- a/runbooks/pdns-health.md +++ b/runbooks/pdns-health.md @@ -4,7 +4,15 @@ Read-only checks for the `/opt/pdns` stack on **hk2.chans.xyz** (`ns1.wsvc.info` Facts: [hosts/hk2.chans.xyz.md](../hosts/hk2.chans.xyz.md) · Upstream: [docs/pdns-upstream.md](../docs/pdns-upstream.md) -## Quick remote +Prefer the Ansible health report (`ansible/playbooks/health-report.yml` with +`--limit powerdns`) for routine checks. Keep the SSH commands below for focused +diagnosis, especially direct authoritative DNS queries. + +## Focused SSH diagnostics + +Use these only after the Ansible health report needs investigation. + +### Quick remote ```bash ssh -4 windy@hk2.chans.xyz 'cd /opt/pdns && docker compose ps -a' @@ -12,7 +20,7 @@ ssh -4 windy@hk2.chans.xyz 'cd /opt/pdns && docker compose ps -a' Expect `auth`, `db`, `poweradmin` healthy; `backup` Up; `pgweb` Up. Ignore stopped orphan `powerdns-admin` unless cleaning orphans. -## Version / security poll +### Version / security poll Expect Authoritative **5.0.6** (or newer). Startup logs should **not** show a Mandatory `Security Update` banner. If they do, check https://doc.powerdns.com/authoritative/security-advisories/ and bump the `powerdns/pdns-auth-50` image pin in `/opt/pdns/compose.yml`. @@ -21,7 +29,7 @@ ssh -4 windy@hk2.chans.xyz 'docker logs pdns-auth 2>&1 | grep -i "Security Updat ssh -4 windy@hk2.chans.xyz 'docker inspect pdns-auth --format "{{.Config.Image}}"' ``` -## DNS (from laptop / WSL) +### DNS (from laptop / WSL) Prefer querying auth/secondary IPs directly (local stub / public resolvers may be flaky or cached): @@ -40,7 +48,7 @@ dig @$NS1 +short A ns2.wsvc.info # expect 202.91.35.141 Served SOA serial can differ from the backend/API serial when `SOA-EDIT=INCEPTION-INCREMENT` is active — compare **ns1 vs ns2 served SOA**, not dig vs `pdnsutil`/API. -## API (on host) +### API (on host) ```bash ssh -4 windy@hk2.chans.xyz 'cd /opt/pdns && docker compose exec -T auth \ @@ -49,7 +57,7 @@ ssh -4 windy@hk2.chans.xyz 'cd /opt/pdns && docker compose exec -T auth \ Or use host loopback with the key from `.env` (do not paste the key into chat/logs). -## Web UIs +### Web UIs ```bash # If local stub DNS is flaky, pin SNI to the auth IP: @@ -57,7 +65,7 @@ curl -4 -sS -o /dev/null -w "%{http_code}\n" --resolve pdns.wsvc.info:443:154.36 curl -4 -sS -o /dev/null -w "%{http_code}\n" --resolve pgweb.wsvc.info:443:154.36.174.161 https://pgweb.wsvc.info/ # 401 without basic auth ``` -## Backup +### Backup ```bash ssh -4 windy@hk2.chans.xyz 'cd /opt/pdns && ./scripts/manual-backup.sh && ls -lt backup/*.sql.gz | head' @@ -66,7 +74,7 @@ ssh -4 windy@hk2.chans.xyz 'tail -30 /opt/pdns/backup/backup.log' Scheduler should log `scheduler started … (daily 03:00)`. If you see `can't execute 'bash'` or Alpine `apk` errors, the backup image/entrypoint was reverted — see [hosts/hk2.chans.xyz.md](../hosts/hk2.chans.xyz.md). -## Config sanity (docs-aligned) +### Config sanity (docs-aligned) ```bash ssh -4 windy@hk2.chans.xyz 'grep -E "^(primary|also-notify|only-notify|allow-axfr-ips|gpgsql-dnssec|default-soa-edit)" /opt/pdns/auth/pdns.conf' @@ -81,6 +89,6 @@ The sanitized Ansible health profile is `pdns` (`ansible/playbooks/healthchecks. ## After config changes -- `auth/pdns.conf`, `auth/templates.d/secrets.j2`, or auth-related `.env` → `docker compose up -d --force-recreate auth` -- Poweradmin env / Traefik labels → `docker compose up -d --no-deps --force-recreate poweradmin` (restart Traefik if UI 404s) -- backup schedule / retention env → `docker compose up -d --force-recreate --no-deps backup` +- `auth/pdns.conf`, `auth/templates.d/secrets.j2`, or auth-related `.env` → use the Ansible Compose reconcile playbook with target `auth` +- Poweradmin env / Traefik labels → use the Ansible Compose reconcile playbook with target `poweradmin`; add `service_reconcile_restart_traefik=true` only if its UI returns 404 +- backup schedule / retention env → use the Ansible Compose reconcile playbook with target `backup` diff --git a/runbooks/vaultwarden-health.md b/runbooks/vaultwarden-health.md index 1139d5d..421dffe 100644 --- a/runbooks/vaultwarden-health.md +++ b/runbooks/vaultwarden-health.md @@ -5,7 +5,15 @@ Path: `/opt/vaultwarden` URL: https://auth.wsvc.info/ Upstream: [docs/vaultwarden-upstream.md](../docs/vaultwarden-upstream.md) -## 1. Containers +Prefer the Ansible health report (`ansible/playbooks/health-report.yml` with +`--limit vaultwarden`) for the routine check. The SSH commands below are +focused diagnostics for an already identified problem. + +## Focused SSH diagnostics + +Use these only after the Ansible health report needs investigation. + +### 1. Containers ```bash ssh -4 windy@us2.wsvc.info 'cd /opt/vaultwarden && docker compose ps -a' @@ -13,14 +21,14 @@ ssh -4 windy@us2.wsvc.info 'cd /opt/vaultwarden && docker compose ps -a' Expect `vaultwarden` + `vw-db` **Up (healthy)**. `pgweb` should be stopped unless started with `--profile debug`. -## 2. Live DB is Postgres +### 2. Live DB is Postgres ```bash ssh -4 windy@us2.wsvc.info 'cd /opt/vaultwarden && docker compose exec -T vaultwarden sh -c "printenv DATABASE_URL" | sed -E "s#://[^@]+@#://REDACTED@#"' ssh -4 windy@us2.wsvc.info 'cd /opt/vaultwarden && docker compose exec -T pg psql -U vaultwarden -d vaultwarden -c "SELECT COUNT(*) AS users FROM users; SELECT COUNT(*) AS ciphers FROM ciphers;"' ``` -## 3. Effective config (config.json wins) +### 3. Effective config (config.json wins) ```bash ssh -4 windy@us2.wsvc.info 'cd /opt/vaultwarden && docker compose exec -T vaultwarden cat /data/config.json' \ @@ -48,15 +56,16 @@ EOF' If they diverge, **auth uses `config.json`** — sync password there (or via `/admin`) and update `.smtp-credentials`. -## 4. External HTTPS +### 4. External HTTPS ```bash curl -4 -sS -I --max-time 15 https://auth.wsvc.info/ | head -15 ``` -If **404** right after recreating VW: `ssh -4 windy@us2.wsvc.info 'docker restart traefik'` then retry. +If **404** right after recreating VW: use the Ansible Compose reconcile playbook +with `service_reconcile_restart_traefik=true`, then retry. -## 5. SMTP reachability + AUTH +### 5. SMTP reachability + AUTH TCP (expect `587:0`; `465` often times out from us2): diff --git a/runbooks/vaultwarden-sqlite-to-postgres.md b/runbooks/vaultwarden-sqlite-to-postgres.md index 3d7074a..84ae377 100644 --- a/runbooks/vaultwarden-sqlite-to-postgres.md +++ b/runbooks/vaultwarden-sqlite-to-postgres.md @@ -3,7 +3,9 @@ Target: [us2.wsvc.info](../hosts/us2.wsvc.info.md) Upstream: [Using the PostgreSQL Backend](https://github.com/dani-garcia/vaultwarden/wiki/Using-the-PostgreSQL-Backend) · [docs/vaultwarden-upstream.md](../docs/vaultwarden-upstream.md) -**Done on 2026-08-01** for this host. Keep this as the replay checklist. +**Done on 2026-08-01** for this host. Keep this as the replay checklist. This +is an exceptional, destructive migration procedure: it is intentionally not +automated by the routine Ansible reconciliation playbook. ## Summary of what we did