diff --git a/ansible/playbooks/us4-firewalld.yml b/ansible/playbooks/us4-firewalld.yml new file mode 100644 index 0000000..2667637 --- /dev/null +++ b/ansible/playbooks/us4-firewalld.yml @@ -0,0 +1,456 @@ +--- +# Narrow reconciliation for the audited us4 public zone. This playbook never +# reloads or restarts firewalld and deliberately does not manage Docker rules. +- name: Safely remove audited stale firewalld allowances from us4 + hosts: wireguard + become: true + gather_facts: false + serial: 1 + any_errors_fatal: true + vars: + us4_firewalld_confirm: false + us4_console_confirm: false + us4_firewalld_zone: public + us4_firewalld_keep_services: + - dhcpv6-client + - http + - https + - smtp + - ssh + us4_firewalld_stale_services: + - imap + - imaps + - smtp-submission + - smtps + us4_firewalld_stale_ports: + - 24/tcp + - 6443/tcp + - 8443/tcp + us4_firewalld_expected_containers: + - nghttpx-proxy + - semaphoreui-postgres-1 + - semaphoreui-semaphore-1 + - squid-backend + - traefik + - trlm-server-trilium-1 + - wireguard + us4_firewalld_backup_root: /var/backups/us4-firewall + + tasks: + - name: Require the audited host and explicit apply confirmations + ansible.builtin.assert: + that: + - inventory_hostname == 'us4' + - ansible_host == 'us4.wsvc.info' + - ansible_host_ipv4 == '185.201.226.122' + - ansible_check_mode or (us4_firewalld_confirm | bool) + - ansible_check_mode or (us4_console_confirm | bool) + fail_msg: >- + Apply is allowed only for audited host us4 after the provider console + has been tested. Set both us4_firewalld_confirm=true and + us4_console_confirm=true. Check mode does not require confirmation. + + - name: Verify the remote host identity + ansible.builtin.command: + argv: [hostname, -f] + check_mode: false + changed_when: false + register: us4_firewalld_hostname + + - name: Reject an unexpected remote host + ansible.builtin.assert: + that: + - us4_firewalld_hostname.stdout == 'us4.wsvc.info' + + - name: Verify required services are active + ansible.builtin.command: + argv: [systemctl, is-active, --quiet, "{{ item }}"] + check_mode: false + changed_when: false + loop: + - atd + - firewalld + + - name: Verify firewalld Python bindings used by ansible.posix + ansible.builtin.command: + argv: [python3, -c, "import dbus, firewall, firewall.client"] + check_mode: false + changed_when: false + + - name: Verify the default firewalld zone + ansible.builtin.command: + argv: [firewall-cmd, --get-default-zone] + check_mode: false + changed_when: false + register: us4_firewalld_default_zone + + - name: Read runtime public-zone services + ansible.builtin.command: + argv: [firewall-cmd, --zone=public, --list-services] + check_mode: false + changed_when: false + register: us4_firewalld_runtime_services + + - name: Read permanent public-zone services + ansible.builtin.command: + argv: [firewall-cmd, --permanent, --zone=public, --list-services] + check_mode: false + changed_when: false + register: us4_firewalld_permanent_services + + - name: Read runtime public-zone ports + ansible.builtin.command: + argv: [firewall-cmd, --zone=public, --list-ports] + check_mode: false + changed_when: false + register: us4_firewalld_runtime_ports + + - name: Read permanent public-zone ports + ansible.builtin.command: + argv: [firewall-cmd, --permanent, --zone=public, --list-ports] + check_mode: false + changed_when: false + register: us4_firewalld_permanent_ports + + - name: Normalize the audited public-zone state + ansible.builtin.set_fact: + us4_firewalld_pre_services: "{{ us4_firewalld_runtime_services.stdout.split() | sort }}" + us4_firewalld_pre_permanent_services: "{{ us4_firewalld_permanent_services.stdout.split() | sort }}" + us4_firewalld_pre_ports: "{{ us4_firewalld_runtime_ports.stdout.split() | sort }}" + us4_firewalld_pre_permanent_ports: "{{ us4_firewalld_permanent_ports.stdout.split() | sort }}" + + - name: Fail closed on public-zone drift or unknown allowances + ansible.builtin.assert: + that: + - us4_firewalld_default_zone.stdout == us4_firewalld_zone + - us4_firewalld_pre_services == us4_firewalld_pre_permanent_services + - us4_firewalld_pre_ports == us4_firewalld_pre_permanent_ports + - us4_firewalld_keep_services | difference(us4_firewalld_pre_services) | length == 0 + - us4_firewalld_pre_services | difference(us4_firewalld_keep_services + us4_firewalld_stale_services) | length == 0 + - us4_firewalld_pre_ports | difference(us4_firewalld_stale_ports) | length == 0 + fail_msg: >- + The public zone differs from the audited baseline. Stop and review it; + this playbook will not infer whether an unknown allowance is required. + + - name: Select only audited stale entries that currently exist + ansible.builtin.set_fact: + us4_firewalld_cleanup_services: >- + {{ us4_firewalld_stale_services | intersect(us4_firewalld_pre_services) | sort }} + us4_firewalld_cleanup_ports: >- + {{ us4_firewalld_stale_ports | intersect(us4_firewalld_pre_ports) | sort }} + + - name: Report the proposed reconciliation + ansible.builtin.debug: + msg: + keep_services: "{{ us4_firewalld_keep_services }}" + remove_services: "{{ us4_firewalld_cleanup_services }}" + remove_ports: "{{ us4_firewalld_cleanup_ports }}" + reload_or_restart: false + + - name: Create rollback material when cleanup is required + when: + - not ansible_check_mode + - us4_firewalld_cleanup_services | length > 0 or us4_firewalld_cleanup_ports | length > 0 + block: + - name: Create the protected firewall backup root + ansible.builtin.file: + path: "{{ us4_firewalld_backup_root }}" + state: directory + owner: root + group: root + mode: "0700" + + - name: Create a backup timestamp + ansible.builtin.command: + argv: [date, +%Y%m%dT%H%M%S%z] + changed_when: false + register: us4_firewalld_backup_timestamp + + - name: Set the protected backup directory + ansible.builtin.set_fact: + us4_firewalld_backup_dir: >- + {{ us4_firewalld_backup_root }}/{{ us4_firewalld_backup_timestamp.stdout }} + us4_firewalld_rollback_command: >- + {{ us4_firewalld_backup_root }}/{{ us4_firewalld_backup_timestamp.stdout }}/rollback-phase1.sh + + - name: Create the protected backup directory + ansible.builtin.file: + path: "{{ us4_firewalld_backup_dir }}" + state: directory + owner: root + group: root + mode: "0700" + + - name: Back up the complete firewalld configuration + ansible.builtin.command: + argv: + - tar + - --create + - --gzip + - "--file={{ us4_firewalld_backup_dir }}/firewalld.tgz" + - --directory=/etc + - firewalld + changed_when: true + + - name: Capture the pre-change runtime ruleset + ansible.builtin.shell: + cmd: >- + umask 077 && nft list ruleset > + {{ us4_firewalld_backup_dir | quote }}/nft-ruleset.txt + executable: /bin/bash + changed_when: true + + - name: Install the exact pre-change rollback script + ansible.builtin.copy: + dest: "{{ us4_firewalld_rollback_command }}" + owner: root + group: root + mode: "0700" + content: | + #!/bin/sh + set -eu + exec >>/var/log/us4-firewalld-phase1-rollback.log 2>&1 + printf '%s rollback start\n' "$(date -Is)" + add_service() { + service=$1 + /usr/bin/firewall-cmd --permanent --zone=public \ + --query-service="$service" >/dev/null 2>&1 || + /usr/bin/firewall-cmd --permanent --zone=public \ + --add-service="$service" + /usr/bin/firewall-cmd --zone=public \ + --query-service="$service" >/dev/null 2>&1 || + /usr/bin/firewall-cmd --zone=public --add-service="$service" + } + add_port() { + port=$1 + /usr/bin/firewall-cmd --permanent --zone=public \ + --query-port="$port" >/dev/null 2>&1 || + /usr/bin/firewall-cmd --permanent --zone=public \ + --add-port="$port" + /usr/bin/firewall-cmd --zone=public \ + --query-port="$port" >/dev/null 2>&1 || + /usr/bin/firewall-cmd --zone=public --add-port="$port" + } + {% for service in us4_firewalld_cleanup_services %} + add_service {{ service }} + {% endfor %} + {% for port in us4_firewalld_cleanup_ports %} + add_port {{ port }} + {% endfor %} + /usr/bin/firewall-cmd --check-config + printf '%s rollback complete\n' "$(date -Is)" + + - name: Schedule the 15-minute automatic rollback + ansible.builtin.shell: + cmd: | + set -euo pipefail + output=$(printf '%s\n' {{ us4_firewalld_rollback_command | quote }} | at now + 15 minutes 2>&1) + job_id=$(printf '%s\n' "$output" | sed -n 's/^job \([0-9][0-9]*\).*/\1/p') + test -n "$job_id" + printf '%s\n' "$job_id" + executable: /bin/bash + changed_when: true + register: us4_firewalld_rollback_job + + - name: Record the automatic rollback job + ansible.builtin.set_fact: + us4_firewalld_rollback_job_id: "{{ us4_firewalld_rollback_job.stdout }}" + us4_firewalld_rollback_cancelled: false + + - name: Persist the rollback job ID beside the backup + ansible.builtin.copy: + dest: "{{ us4_firewalld_backup_dir }}/phase1-at-job-id" + owner: root + group: root + mode: "0600" + content: "{{ us4_firewalld_rollback_job_id }}\n" + + - name: Reconcile and verify the audited public zone + block: + - name: Remove audited stale firewalld services + ansible.posix.firewalld: + zone: "{{ us4_firewalld_zone }}" + service: "{{ item }}" + state: disabled + permanent: true + immediate: true + loop: "{{ us4_firewalld_stale_services }}" + + - name: Remove audited stale firewalld ports + ansible.posix.firewalld: + zone: "{{ us4_firewalld_zone }}" + port: "{{ item }}" + state: disabled + permanent: true + immediate: true + loop: "{{ us4_firewalld_stale_ports }}" + + - name: Verify the permanent firewalld configuration + ansible.builtin.command: + argv: [firewall-cmd, --check-config] + when: not ansible_check_mode + changed_when: false + + - name: Read reconciled runtime services + ansible.builtin.command: + argv: [firewall-cmd, --zone=public, --list-services] + changed_when: false + when: not ansible_check_mode + register: us4_firewalld_after_runtime_services + + - name: Read reconciled permanent services + ansible.builtin.command: + argv: [firewall-cmd, --permanent, --zone=public, --list-services] + changed_when: false + when: not ansible_check_mode + register: us4_firewalld_after_permanent_services + + - name: Read reconciled runtime ports + ansible.builtin.command: + argv: [firewall-cmd, --zone=public, --list-ports] + changed_when: false + when: not ansible_check_mode + register: us4_firewalld_after_runtime_ports + + - name: Read reconciled permanent ports + ansible.builtin.command: + argv: [firewall-cmd, --permanent, --zone=public, --list-ports] + changed_when: false + when: not ansible_check_mode + register: us4_firewalld_after_permanent_ports + + - name: Require the exact audited post-change public zone + ansible.builtin.assert: + that: + - us4_firewalld_after_runtime_services.stdout.split() | sort == us4_firewalld_keep_services | sort + - us4_firewalld_after_permanent_services.stdout.split() | sort == us4_firewalld_keep_services | sort + - us4_firewalld_after_runtime_ports.stdout.split() | length == 0 + - us4_firewalld_after_permanent_ports.stdout.split() | length == 0 + when: not ansible_check_mode + + - name: Verify a fresh independent SSH and sudo path + ansible.builtin.command: + argv: + - ssh + - -4 + - -o + - BatchMode=yes + - -o + - ConnectTimeout=10 + - -o + - ControlMaster=no + - -o + - ControlPath=none + - windy@us4.wsvc.info + - sudo -n true + delegate_to: localhost + become: false + changed_when: false + when: not ansible_check_mode + vars: + ansible_become: false + + - name: Verify public HTTPS routes + ansible.builtin.uri: + url: "{{ item.url }}" + follow_redirects: all + status_code: "{{ item.status }}" + validate_certs: true + use_proxy: false + loop: + - {url: https://update.wsvc.info/, status: 200} + - {url: https://us4-gate.wsvc.info/, status: 401} + - {url: https://trlm.wsvc.info/, status: 200} + delegate_to: localhost + become: false + when: not ansible_check_mode + vars: + ansible_become: false + + - name: Verify the secondary MX TCP listener externally + ansible.builtin.wait_for: + host: "{{ ansible_host_ipv4 }}" + port: 25 + state: started + connect_timeout: 5 + timeout: 10 + delegate_to: localhost + become: false + when: not ansible_check_mode + vars: + ansible_become: false + + - name: Verify all expected containers are running + ansible.builtin.command: + argv: [docker, ps, --format, "{{ '{{.Names}}' }}"] + changed_when: false + when: not ansible_check_mode + register: us4_firewalld_running_containers + + - name: Reject missing application containers + ansible.builtin.assert: + that: + - us4_firewalld_expected_containers | difference(us4_firewalld_running_containers.stdout_lines) | length == 0 + when: not ansible_check_mode + + - name: Verify Fail2ban remains active + ansible.builtin.command: + argv: [fail2ban-client, status] + changed_when: false + when: not ansible_check_mode + register: us4_firewalld_fail2ban + + - name: Require all audited Fail2ban jails + ansible.builtin.assert: + that: + - item in us4_firewalld_fail2ban.stdout + loop: + - postfix-postscreen + - postfix-sasl + - recidive + - sshd + when: not ansible_check_mode + + - name: Verify the deployed WireGuard health check + ansible.builtin.command: + argv: [/usr/local/lib/vps-health/run] + changed_when: false + when: not ansible_check_mode + register: us4_firewalld_wireguard_health + + - name: Cancel automatic rollback only after all checks pass + ansible.builtin.command: + argv: [at, -r, "{{ us4_firewalld_rollback_job_id }}"] + changed_when: true + when: + - not ansible_check_mode + - us4_firewalld_cleanup_services | length > 0 or us4_firewalld_cleanup_ports | length > 0 + + - name: Mark the automatic rollback as cancelled + ansible.builtin.set_fact: + us4_firewalld_rollback_cancelled: true + when: + - not ansible_check_mode + - us4_firewalld_cleanup_services | length > 0 or us4_firewalld_cleanup_ports | length > 0 + + rescue: + - name: Preserve the automatic rollback and stop + ansible.builtin.fail: + msg: >- + A reconciliation or verification task failed. No reload was + attempted. If cleanup was required, its automatic rollback remains + scheduled; do not remove it manually. + + always: + - name: Report backup and rollback disposition + ansible.builtin.debug: + msg: + backup: >- + {{ us4_firewalld_backup_dir | + default('not-created-in-check-mode' if ansible_check_mode else 'not-required') }} + automatic_rollback: >- + {{ 'not-created-in-check-mode' if ansible_check_mode else + ('cancelled-after-success' if (us4_firewalld_rollback_cancelled | default(false)) else + 'scheduled-or-executed') if + (us4_firewalld_cleanup_services | length > 0 or us4_firewalld_cleanup_ports | length > 0) + else 'not-required' }} diff --git a/ansible/requirements.yml b/ansible/requirements.yml new file mode 100644 index 0000000..9a095aa --- /dev/null +++ b/ansible/requirements.yml @@ -0,0 +1,5 @@ +--- +collections: + # us4-firewalld.yml was source-reviewed and exercised with this version. + - name: ansible.posix + version: 2.2.2 diff --git a/hosts/us4.wsvc.info.md b/hosts/us4.wsvc.info.md index 23cde3b..6511e49 100644 --- a/hosts/us4.wsvc.info.md +++ b/hosts/us4.wsvc.info.md @@ -9,10 +9,74 @@ | Compose file | `/opt/wireguard/compose.yml` | | Container | `wireguard` | | Image policy | Immutable digest, updated only in an approved maintenance window | -| Public port | UDP `51820` on IPv4 and IPv6 | +| Public endpoint | `us4.wsvc.info:51820/udp`; DNS publishes only A `185.201.226.122` (no native AAAA) | | Tunnel subnet | `10.13.13.0/24` | | Routing policy | IPv4-only full tunnel (`ALLOWEDIPS=0.0.0.0/0`); IPv6 traffic is not guaranteed to use the VPN | +Upstream image documentation: +[LinuxServer.io WireGuard](https://docs.linuxserver.io/images/docker-wireguard/). + +## Deployment configuration + +The repository-owned, non-secret Compose declaration is rendered from +`ansible/templates/wireguard-compose.yml.j2`. The live declaration was verified +on 2026-08-12 with these core settings: + +| Setting | Live value / intent | +|---------|---------------------| +| Image | `lscr.io/linuxserver/wireguard@sha256:ac43e1226878d2611315172d6ea357a95cb326ee73124b91108118efc8666889` | +| Image version | `1.0.20260223-r0-ls119` (build 2026-07-30) | +| Required capability | `NET_ADMIN` only; host kernel already supplies WireGuard/iptables, so `SYS_MODULE` and `/lib/modules` are not granted | +| Filesystem | Read-only container root; executable tmpfs at `/run`; writable bind mount `/opt/wireguard/config:/config` | +| Restart | `unless-stopped` | +| Server mode | Named peers `ha`, `phone`, `mbp`; runtime and configured peer counts both `3` | +| Client DNS | `1.1.1.1` | +| Tunnel routing | IPv4 full tunnel, `0.0.0.0/0`; no client IPv6 tunnel | +| Runtime interface | `wg0`, server address `10.13.13.1/32`, listen port `51820` | +| Forwarding/NAT | IPv4 forwarding enabled in the container namespace; `wg0` forwarding allowed and egress masqueraded on `eth+`; IPv6 forwarding disabled | + +Docker binds UDP `51820` on both host socket families, but the public hostname +has no AAAA record. Clients using `us4.wsvc.info` therefore reach the server over +IPv4. + +## Other host services and firewall (2026-08-12) + +This host also carries the `windy.me` secondary MX and several web applications; +do not build its firewall allowlist from the WireGuard role alone. + +| Port | Owner / purpose | Effective public state | +|------|-----------------|------------------------| +| TCP `22` | SSH management | Open | +| TCP `25` | Postfix, `mx.windy.me` (MX priority 30) | Open; retain until the secondary-MX role is explicitly retired | +| TCP `80`, `443` | Traefik for `update.wsvc.info`, `us4-gate.wsvc.info`, and `trlm.wsvc.info` | Open | +| TCP `3000` | Semaphore UI direct Docker publish | Open; redundant with the Traefik route and should be removed or bound to loopback | +| TCP `8080` | Traefik direct Docker publish | Open; redundant with the authenticated dashboard route and should be removed or bound to loopback | +| UDP `51820` | WireGuard | Required public endpoint | +| TCP `9443` | Host nghttpx-to-Squid proxy | Listening but blocked by the current firewall | +| UDP `123` | ntpsec | Listening but blocked by the current firewall | + +PostgreSQL (`5433`/`5434`/`5435`), MariaDB (`3306`), and the host Squid TCP +listener (`3128`) are loopback-only. Squid also owns wildcard UDP sockets, which +are not allowed by the current public zone. + +UFW is not installed. Firewalld `2.3.1` is active with nftables. On 2026-08-12, +the reviewed `ansible/playbooks/us4-firewalld.yml` reconciliation removed the +stale `imap`, `imaps`, `smtp-submission`, and `smtps` services plus TCP `24`, +`6443`, and `8443` without reloading or restarting firewalld. Runtime and +permanent public-zone state now match exactly: services `dhcpv6-client`, `http`, +`https`, `smtp`, and `ssh`, with no explicit ports. + +Docker-published ports are accepted through Docker's DNAT/FORWARD chains, so +the public-zone cleanup does not close `3000` or `8080`. Their Compose bindings +remain a separate, staged follow-up after the required observation window. +Firewalld logged Docker chain/policy conflicts during the 2026-08-10 boots; +treat any firewall reload or service restart as a maintenance-window operation +and reverify Docker routing. Tracking: Linear `W1N-60`. + +`mx.windy.me` also publishes AAAA `2602:f9f3:0:2::878`, while the host currently +has no global IPv6 address or IPv6 default route. Treat that as a separate +secondary-MX reachability issue. + ## Safety - Private keys, preshared keys, peer configuration files, and QR codes remain @@ -21,6 +85,17 @@ - Local rollback archives are stored in `/opt/wireguard/backups` (directory mode `0700`, archives mode `0600`). They contain private keys, are not an off-host disaster-recovery backup, and must never leave the server. +- Live private keys, preshared keys, generated peer configs, QR images, and + `wg0.conf` are mode `0600`. Template-only `peer.conf` and `server.conf` files + are mode `0644` and do not contain generated key material. +- `/opt/wireguard/config` is mode `0755`, but its sensitive files are `0600`. + The current files are owned by the image's numeric UID/GID rather than the + declared `PUID=1000` / `PGID=1000`; the root-run WireGuard processes can use + them, but reconcile ownership only after a protected backup and maintenance + review. +- `LOG_CONFS` is currently unset and the inspected container log contained no + QR-code/config banners. Do not enable config logging; generated QR images are + credentials. - Do not delete, move, or regenerate `/opt/wireguard/config` during maintenance. - Before a container recreation, validate `docker compose config` and retain a @@ -35,6 +110,23 @@ cd ansible ansible-playbook playbooks/health-report.yml --limit wireguard ``` +Preview the narrow, fail-closed public-zone reconciliation: + +```bash +ansible-galaxy collection install -r requirements.yml +ansible-playbook playbooks/us4-firewalld.yml --limit us4 --check --diff +``` + +Apply it only after testing the provider console and keeping an independent SSH +rollback session open. The playbook creates a protected server-local backup and +a 15-minute automatic rollback before changing rules; it cancels that rollback +only after SSH, HTTPS, SMTP, Docker, Fail2ban, and WireGuard checks pass: + +```bash +ansible-playbook playbooks/us4-firewalld.yml --limit us4 \ + -e '{"us4_firewalld_confirm": true, "us4_console_confirm": true}' +``` + The image update and recreate procedure is deliberately separate and requires an immutable image digest in the server-side Compose file plus an explicit maintenance-window confirmation: @@ -59,3 +151,43 @@ ansible-playbook playbooks/wireguard-harden.yml --limit wireguard \ - Validate a known client can handshake and sends IPv4 traffic through the VPN. - Do not treat inactive mobile peers as a failure solely because their latest handshake is old. + +## Live audit snapshot (2026-08-12) + +The WireGuard service itself is healthy and its installation is broadly +reasonable: + +- The sanitized Ansible health report returned `status=ok`; Compose is valid, + the container is running with zero restarts, `wg0` exists, and UDP `51820` is + listening. +- One of three peers had a current handshake during the audit. Two peers had + not handshaken since the current container/interface start; confirm those + clients only if they are expected to be active. +- The image is immutable-digest pinned, key-bearing files are protected, the + container root is read-only, and the container has `NET_ADMIN` without the + broader `SYS_MODULE` capability. +- Debian `13.6`, kernel `6.12.101+deb13-amd64`, Docker Engine `29.7.2`, and + Docker Compose `v5.4.0` were observed. No Debian package updates or reboot + requirement were pending. + +Open host-level follow-up (do not conflate these with a WireGuard outage): + +1. **Disk capacity:** `/` was 90% used with about 3.4 GiB free. Docker reported + about 2.48 GB of reclaimable images and the system journal used about 1.9 + GB, but do not prune or vacuum without reviewing retention and rollback + needs first. +2. **Docker exposure:** the firewalld public-zone cleanup is complete, but + Docker still publishes `3000` and `8080` outside the ordinary host INPUT + path. Remove those redundant Compose bindings in separate maintenance units + after the observation window, and confirm provider firewall rules first. +3. **Image maintenance:** the upstream `latest` amd64 image had advanced to + `1.0.20260223-r0-ls120` (build 2026-08-06). Review and pin its immutable + digest in a maintenance window rather than updating unattended. +4. **Host hygiene:** `apache2.service`, `certbot.service`, and + `postgresql@9.6-main.service` were in a failed state while unrelated Docker + workloads remained active. Establish ownership and remove or repair stale + units separately. +5. **Resource/log limits:** the WireGuard container has no memory, CPU, or PID + limit and uses Docker's `json-file` log driver without a per-container + rotation setting. Current log size was small, but limits/rotation should be + considered during a reviewed Compose update.