Manage routine operations through Ansible

ansible/inventory/hosts.yml

@@ -18,18 +18,33 @@ all:
           service_role: mailcow
           compose_project_dir: /opt/mail
           healthcheck_profile: mailcow
+          service_reconcile_services:
+            all:
+              compose_args: [--force-recreate]
         us2:
           ansible_host: us2.wsvc.info
           ansible_host_ipv4: 193.9.44.165
           service_role: vaultwarden
           compose_project_dir: /opt/vaultwarden
           healthcheck_profile: vaultwarden
+          service_reconcile_services:
+            vaultwarden:
+              compose_args: [--force-recreate]
+          service_reconcile_traefik_restart_targets: [vaultwarden]
         hk2:
           ansible_host: hk2.chans.xyz
           ansible_host_ipv4: 154.36.174.161
           service_role: powerdns
           compose_project_dir: /opt/pdns
           healthcheck_profile: pdns
+          service_reconcile_services:
+            auth:
+              compose_args: [--force-recreate]
+            poweradmin:
+              compose_args: [--no-deps, --force-recreate]
+            backup:
+              compose_args: [--no-deps, --force-recreate]
+          service_reconcile_traefik_restart_targets: [poweradmin]
     mailcow:
       hosts:
         mx2:

ansible/playbooks/baseline.yml

@@ -1,7 +1,9 @@
 ---
 # Baseline starts audit-only. Opt-in variables are deliberately false by default.
 - name: Apply controlled common baseline
-  hosts: managed
+  # The baseline role is OS-level and does not assume Docker Compose, so it
+  # applies to the Matrix K3s node as well as the Compose hosts.
+  hosts: all
   become: true
   gather_facts: false
   roles:

ansible/playbooks/compose-reconcile.yml

@@ -0,0 +1,12 @@
+---
+# Intentional, narrowly scoped Compose reconciliation. This never changes
+# compose files, images, secrets, DNS, or databases. It is gated because a
+# container recreate can briefly interrupt a public service.
+- name: Reconcile explicitly selected Docker Compose services
+  hosts: docker_hosts
+  become: true
+  gather_facts: false
+  serial: 1
+  roles:
+    - role: compose_reconcile
+      tags: [compose, reconcile, mutating]

ansible/playbooks/health-report.yml

@@ -0,0 +1,50 @@
+---
+# Execute the already-installed, sanitized local health checks on demand.
+# This is read-only with respect to the managed applications; the scripts only
+# inspect state and update their own health result files/logs.
+- name: Report Docker Compose service health
+  hosts: managed
+  become: true
+  gather_facts: false
+  tasks:
+    - name: Run the deployed service health check
+      ansible.builtin.command:
+        argv:
+          - /usr/local/lib/vps-health/run
+      register: service_health_report
+      changed_when: false
+      failed_when: service_health_report.rc not in [0, 1, 2]
+
+    - name: Display sanitized service health result
+      ansible.builtin.debug:
+        msg: "{{ service_health_report.stdout | default(service_health_report.stderr, true) }}"
+
+    - name: Fail when service health is degraded
+      ansible.builtin.fail:
+        msg: "Health check reported a degraded state on {{ inventory_hostname }}."
+      when: service_health_report.rc != 0
+
+- name: Report Matrix health
+  hosts: matrix
+  become: true
+  gather_facts: false
+  tasks:
+    - name: Run the deployed Matrix health check
+      ansible.builtin.command:
+        argv:
+          - "{{ matrix_healthcheck_script_path | default('/usr/local/lib/vps-health/matrix-k3s') }}"
+      register: matrix_health_report
+      changed_when: false
+
+    - name: Parse Matrix health result
+      ansible.builtin.set_fact:
+        matrix_health_result: "{{ matrix_health_report.stdout | from_json }}"
+
+    - name: Display Matrix health result
+      ansible.builtin.debug:
+        var: matrix_health_result
+
+    - name: Fail when Matrix health is degraded
+      ansible.builtin.fail:
+        msg: "Matrix health is {{ matrix_health_result.status }} on {{ inventory_hostname }}."
+      when: matrix_health_result.status != 'ok'

ansible/playbooks/maintenance-preview.yml

@@ -2,7 +2,9 @@
 # Preview only. This playbook does not install updates, restart services, or
 # change DNS/secrets. A separate, manually reviewed change is required to act.
 - name: Preview pending maintenance without changes
-  hosts: managed
+  # Package/reboot inspection is host-level and is safe for both Compose and
+  # K3s nodes.
+  hosts: all
   become: true
   gather_facts: false
   tasks:

ansible/roles/compose_reconcile/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+# Both variables must be supplied explicitly by the operator. Allowed services
+# and their safe Compose flags belong in inventory, not on the command line.
+service_reconcile_confirm: false
+service_reconcile_targets: []
+service_reconcile_restart_traefik: false

ansible/roles/compose_reconcile/tasks/main.yml

@@ -0,0 +1,69 @@
+---
+- name: Require explicit confirmation and selected services
+  ansible.builtin.assert:
+    that:
+      - service_reconcile_confirm | bool
+      - service_reconcile_targets | length > 0
+    fail_msg: >-
+      This operation can recreate containers. Supply
+      service_reconcile_confirm=true and a non-empty
+      service_reconcile_targets list after reviewing the change.
+
+- name: Require a host-specific reconciliation policy
+  ansible.builtin.assert:
+    that:
+      - service_reconcile_services is defined
+    fail_msg: "No reconciliation policy is defined for {{ inventory_hostname }}."
+
+- name: Validate selected services are allowed on this host
+  ansible.builtin.assert:
+    that:
+      - item in service_reconcile_services
+    fail_msg: "{{ item }} is not an allowed reconciliation target on {{ inventory_hostname }}."
+  loop: "{{ service_reconcile_targets }}"
+
+- name: Validate optional Traefik restart scope
+  ansible.builtin.assert:
+    that:
+      - >-
+        not (service_reconcile_restart_traefik | bool) or
+        (service_reconcile_targets
+         | select('in', service_reconcile_traefik_restart_targets | default([]))
+         | list | length > 0)
+    fail_msg: >-
+      Traefik may only be restarted as the explicit post-reconcile action for
+      an inventory-approved service on this host.
+
+- name: Reconcile approved Compose services
+  ansible.builtin.command:
+    argv: >-
+      {{ ['docker', 'compose', '--project-directory', compose_project_dir, 'up', '-d']
+         + service_reconcile_services[item].compose_args
+         + ([] if item == 'all' else [item]) }}
+  loop: "{{ service_reconcile_targets }}"
+  register: service_reconcile_result
+  changed_when: true
+  loop_control:
+    label: "{{ item }}"
+
+- name: Restart Traefik after approved reconciliation when requested
+  ansible.builtin.command:
+    argv: [docker, restart, traefik]
+  when: service_reconcile_restart_traefik | bool
+  changed_when: true
+
+- name: Report reconciled Compose state
+  ansible.builtin.command:
+    argv:
+      - docker
+      - compose
+      - --project-directory
+      - "{{ compose_project_dir }}"
+      - ps
+      - --all
+  changed_when: false
+  register: service_reconcile_ps
+
+- name: Display reconciled Compose state
+  ansible.builtin.debug:
+    var: service_reconcile_ps.stdout_lines

ansible/roles/matrix_healthcheck/defaults/main.yml

@@ -6,7 +6,9 @@ matrix_healthcheck_service_name: matrix-k3s-healthcheck.service
 matrix_healthcheck_timer_name: matrix-k3s-healthcheck.timer
 matrix_healthcheck_timer_on_calendar: '*-*-* 06:00:00 UTC'
 matrix_healthcheck_timer_randomized_delay_sec: 15m
-matrix_healthcheck_namespace: matrix-system
+# ESS workloads live in this namespace; matrix-system only has supporting
+# chart-independent resources.
+matrix_healthcheck_namespace: ess
 matrix_healthcheck_backup_path: /var/backups/matrix
 matrix_healthcheck_backup_max_age_hours: 30
 matrix_healthcheck_warn_percent: 80