From b03d7019c44cac59b36d2e07cfd6aba04b330cea Mon Sep 17 00:00:00 2001 From: windyboy Date: Thu, 6 Aug 2026 15:48:48 +0800 Subject: [PATCH] docs: onboard AdGuard Home to Ansible; retire check-mx2; migrate plan to Linear - Add dns.windy.lan to Ansible inventory (adguardhome group/profile) with a read-only healthcheck template, systemd timer, and on-demand report parity with the other active hosts; verified green on the host. - Add synapse.chans.xyz to the AGENTS.md active-host quick map. - Remove plans/; migrate decisions/verification to a Linear vps project doc and file the remaining gaps as W1N-46..49 with blocking edges. - Retire scripts/check-mx2.sh; point mailcow health/update runbooks and the mx2 host file at the Ansible health report. - Trim us2 stub software to verified running services; note dormant dirs. - Normalize repo docs to English; drop the non-existent CONTEXT.md/ADR claim and remove docs/agents/domain.md. --- AGENTS.md | 20 +- ansible/inventory/hosts.yml | 13 ++ ansible/roles/healthcheck/defaults/main.yml | 1 + .../healthcheck/templates/adguardhome.sh.j2 | 75 +++++++ docs/adguard-home-official-review-2026-08.md | 164 +++++++++++++++ docs/agents/issue-tracker.md | 47 +++++ docs/agents/triage-labels.md | 15 ++ hosts/dns.windy.lan.md | 75 +++++++ hosts/mx2.windy.me.md | 5 +- hosts/us2.wsvc.info.md | 10 +- inventory/hosts.md | 1 + ...-maintenance-ansible-restic-refactor-v1.md | 70 ------- runbooks/adguard-home-health.md | 60 ++++++ runbooks/mailcow-health.md | 11 +- runbooks/mailcow-update.md | 2 +- scripts/check-mx2.sh | 195 ------------------ 16 files changed, 484 insertions(+), 280 deletions(-) create mode 100644 ansible/roles/healthcheck/templates/adguardhome.sh.j2 create mode 100644 docs/adguard-home-official-review-2026-08.md create mode 100644 docs/agents/issue-tracker.md create mode 100644 docs/agents/triage-labels.md create mode 100644 hosts/dns.windy.lan.md delete mode 100644 plans/2026-08-02-vps-maintenance-ansible-restic-refactor-v1.md create mode 100644 runbooks/adguard-home-health.md delete mode 100755 scripts/check-mx2.sh diff --git a/AGENTS.md b/AGENTS.md index 8e1b992..d3aa6ad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,6 +27,8 @@ Also readable as `agent.md` (symlink → this file). | **mx2.windy.me** | mailcow (`/opt/mail`, project `cow`) | `ssh -4 windy@mx2.windy.me` | [hosts/mx2.windy.me.md](hosts/mx2.windy.me.md) | | **us2.wsvc.info** | Vaultwarden + Traefik (+ Soft Serve, …) | `ssh -4 windy@us2.wsvc.info` | [hosts/us2.wsvc.info.md](hosts/us2.wsvc.info.md) | | **hk2.chans.xyz** | PowerDNS auth ns1 (`/opt/pdns`) | `ssh -4 windy@hk2.chans.xyz` | [hosts/hk2.chans.xyz.md](hosts/hk2.chans.xyz.md) | +| **synapse.chans.xyz** | Matrix ESS (Synapse + MAS + Element) on K3s | `ssh -4 windy@synapse.chans.xyz` | [hosts/synapse.chans.xyz.md](hosts/synapse.chans.xyz.md) | +| **dns.windy.lan** | AdGuard Home LAN DNS + Mihomo explicit proxy | `ssh -4 windy@192.168.66.36` | [hosts/dns.windy.lan.md](hosts/dns.windy.lan.md) | | **gfw.windy.lan** | OpenWrt LAN gateway / OpenClash | `ssh -4 root@192.168.66.1` | [hosts/gfw.windy.lan.md](hosts/gfw.windy.lan.md) | | **gw** | EdgeRouter X primary LAN gateway | `ssh -4 zhiqiang@192.168.66.254` | [hosts/gw.md](hosts/gw.md) | | **ubnt** | UniFi Network Controller | `ssh -4 windy@192.168.66.46` | [hosts/ubnt.md](hosts/ubnt.md) | @@ -60,7 +62,7 @@ Also readable as `agent.md` (symlink → this file). | Task | Path | |------|------| -| mailcow health | [runbooks/mailcow-health.md](runbooks/mailcow-health.md) · `./scripts/check-mx2.sh` | +| mailcow health | [runbooks/mailcow-health.md](runbooks/mailcow-health.md) | | mailcow update | [runbooks/mailcow-update.md](runbooks/mailcow-update.md) | | mailcow SMTP/IMAP client | [runbooks/mailcow-smtp-client.md](runbooks/mailcow-smtp-client.md) | | Vaultwarden health | [runbooks/vaultwarden-health.md](runbooks/vaultwarden-health.md) | @@ -69,16 +71,23 @@ Also readable as `agent.md` (symlink → this file). | PowerDNS health (hk2) | [runbooks/pdns-health.md](runbooks/pdns-health.md) | | PowerDNS upstream refs | [docs/pdns-upstream.md](docs/pdns-upstream.md) | | Matrix health | [runbooks/matrix-health.md](runbooks/matrix-health.md) | +| AdGuard Home health | [runbooks/adguard-home-health.md](runbooks/adguard-home-health.md) | | Matrix upstream refs | [docs/matrix-upstream.md](docs/matrix-upstream.md) | | Hermes Agent Matrix channel | [docs/hermes-matrix.md](docs/hermes-matrix.md) | | UniFi local-service proxy bypass | [docs/unifi-openclash-localhost.md](docs/unifi-openclash-localhost.md) | | Routine Ansible operations | [runbooks/ansible-operations.md](runbooks/ansible-operations.md) | -```bash -./scripts/check-mx2.sh -``` +Routine mailcow health: `cd ansible && ansible-playbook playbooks/health-report.yml --limit mailcow`. The local stub resolver is flaky; DNS probes use `1.1.1.1` / `8.8.8.8`. -`check-mx2.sh` defaults to IPv4; set `MX2_ALLOW_IPV6=1` to allow IPv6. DNS checks use `1.1.1.1` / `8.8.8.8` (local stub is flaky). +## Agent skills + +### Issue tracker + +Issues are tracked in Linear and created/updated via the Linear MCP (`vps` project). See `docs/agents/issue-tracker.md`. + +### Triage labels + +Default triage labels: needs-triage, needs-info, ready-for-agent, ready-for-human, wontfix. See `docs/agents/triage-labels.md`. ## Safety @@ -116,5 +125,4 @@ inventory/hosts.md # machine index hosts/ # per-host facts runbooks/ # step-by-step ops docs/ # upstream doc indexes / design notes -scripts/ # read-only helper scripts ``` diff --git a/ansible/inventory/hosts.yml b/ansible/inventory/hosts.yml index 5f02007..c063c03 100644 --- a/ansible/inventory/hosts.yml +++ b/ansible/inventory/hosts.yml @@ -56,6 +56,15 @@ all: service_reconcile_services: wireguard: compose_args: [--no-deps, --force-recreate] + dns_windy_lan: + ansible_host: 192.168.66.36 + ansible_host_ipv4: 192.168.66.36 + service_role: adguardhome + compose_project_dir: /opt/adguardhome + healthcheck_profile: adguardhome + service_reconcile_services: + adguardhome: + compose_args: [--no-deps, --force-recreate] mailcow: hosts: mx2: @@ -68,12 +77,16 @@ all: wireguard: hosts: us4: + adguardhome: + hosts: + dns_windy_lan: docker_hosts: children: mailcow: vaultwarden: powerdns: wireguard: + adguardhome: # Matrix is a dedicated K3s node and intentionally remains outside the # Docker-oriented managed group. k3s_servers: diff --git a/ansible/roles/healthcheck/defaults/main.yml b/ansible/roles/healthcheck/defaults/main.yml index b5b4c3e..a0d2ddd 100644 --- a/ansible/roles/healthcheck/defaults/main.yml +++ b/ansible/roles/healthcheck/defaults/main.yml @@ -12,3 +12,4 @@ healthcheck_profiles: vaultwarden: vaultwarden.sh pdns: pdns.sh wireguard: wireguard.sh + adguardhome: adguardhome.sh diff --git a/ansible/roles/healthcheck/templates/adguardhome.sh.j2 b/ansible/roles/healthcheck/templates/adguardhome.sh.j2 new file mode 100644 index 0000000..76348c0 --- /dev/null +++ b/ansible/roles/healthcheck/templates/adguardhome.sh.j2 @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -uo pipefail +source '{{ healthcheck_install_root }}/health-common.sh' + +require_command docker +require_command curl +require_command dig +require_command ss +require_command df +check_compose + +container='adguardhome' +if ! docker inspect "$container" >/dev/null 2>&1; then + record critical 'adguardhome_container_missing' +else + running="$(docker inspect "$container" --format '{{ '{{' }}.State.Running{{ '}}' }}' 2>/dev/null)" + [[ "$running" == true ]] || record critical 'adguardhome_container_not_running' +fi + +config_check="$(docker exec "$container" /opt/adguardhome/AdGuardHome --check-config \ + -c /opt/adguardhome/conf/AdGuardHome.yaml 2>&1 || true)" +if grep -qiE 'error|invalid|fatal' <<<"$config_check"; then + record critical 'adguardhome_config_invalid' +else + record ok 'adguardhome_config_valid' +fi + +if ss -H -ltn 2>/dev/null | awk '{print $4}' | grep -Eq '(^|:)53$'; then + record ok 'dns_tcp_53_listening' +else + record critical 'dns_tcp_53_missing' +fi +if ss -H -uln 2>/dev/null | awk '{print $4}' | grep -Eq '(^|:)53$'; then + record ok 'dns_udp_53_listening' +else + record critical 'dns_udp_53_missing' +fi +if ss -H -ltn 2>/dev/null | awk '{print $4}' | grep -Eq '(^|:)80$'; then + record ok 'ui_http_80_listening' +else + record critical 'ui_http_80_missing' +fi + +dns_a="$(dig +time=5 +tries=1 @192.168.66.36 example.com A +noall +answer 2>/dev/null)" +[[ -n "$dns_a" ]] && record ok 'dns_example_a' || record critical 'dns_example_a_failed' +dns_aaaa="$(dig +time=5 +tries=1 @192.168.66.36 example.com AAAA +noall +answer 2>/dev/null)" +[[ -n "$dns_aaaa" ]] && record ok 'dns_example_aaaa' || record warning 'dns_example_aaaa_none' +dns_local="$(dig +time=5 +tries=1 @192.168.66.36 hass.windy.lan A +noall +answer 2>/dev/null)" +[[ -n "$dns_local" ]] && record ok 'dns_local_name' || record warning 'dns_local_name_unresolved' + +ui_code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 10 http://192.168.66.36/ 2>/dev/null || true)" +case "$ui_code" in + 200|301|302) record ok "ui_http_${ui_code}" ;; + '') record critical 'ui_unreachable' ;; + *) record critical "ui_http_${ui_code}" ;; +esac + +usage="$(df -P /opt/adguardhome/work 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%')" +if [[ "$usage" =~ ^[0-9]+$ ]]; then + if (( usage >= 90 )); then record critical 'disk_usage_high' + elif (( usage >= 80 )); then record warning 'disk_usage_elevated' + else record ok 'disk_usage_ok'; fi +else + record unknown 'disk_usage_unknown' +fi + +logs="$(docker logs --since 30m "$container" 2>&1 || true)" +if grep -qiE '\[(error|warn|fatal)\]|level=(error|warn|fatal)' <<<"$logs"; then + record warning 'adguardhome_log_errors' +else + record ok 'adguardhome_logs_clean' +fi + +emit_result +exit "$EXIT_CODE" diff --git a/docs/adguard-home-official-review-2026-08.md b/docs/adguard-home-official-review-2026-08.md new file mode 100644 index 0000000..4098916 --- /dev/null +++ b/docs/adguard-home-official-review-2026-08.md @@ -0,0 +1,164 @@ +# AdGuard Home official configuration review basis (2026-08) + +This page is the **official source basis and recommendation baseline** used +during the on-site review of AdGuard Home (AGH) on `192.168.66.36`. It does not +claim to have read that host's configuration and contains no credentials. +On-site conclusions must be based on the live `AdGuardHome.yaml`, OpenWrt +`dnsmasq`/DHCP, firewall, routing, and actual client leases. + +Official sources (AdGuard only): + +- [Configuration](https://adguard-dns.io/kb/adguard-home/configuration/) +- [Setting up AdGuard Home securely](https://adguard-dns.io/kb/adguard-home/running-securely/) +- [Getting started](https://adguard-dns.io/kb/adguard-home/getting-started/) +- [DHCP](https://adguard-dns.io/kb/adguard-home/dhcp/) +- [DNS encryption](https://adguard-dns.io/kb/adguard-home/encryption/) + +## Recommended role split (for the existing OpenWrt gateway) + +Preferred: keep **OpenWrt/dnsmasq as the single DHCP and local-domain +authority, with AGH as the LAN DNS filter/forwarder**: DHCP option 6 hands +clients `.36`; AGH sets the router's local DNS (usually the gateway LAN IP) as +`local_ptr_upstreams` and, where needed, uses domain-specific upstreams to route +the LAN search domain / local names back to dnsmasq. Clients are filtered while +still keeping DHCP lease names, static hosts, local reverse resolution, and +gateway-internal service discovery. + +Rationale: AGH supports dnsmasq-style domain-specific upstreams for internal +private name servers; private PTR should be configured in **Private reverse DNS +servers** / `local_ptr_upstreams` rather than regular upstreams; when not +explicitly configured, it falls back to the OS default resolver, which shifts as +host networking changes. [Official config: domain-specific upstreams](https://adguard-dns.io/kb/adguard-home/configuration/#configuring-upstreams); +[private rDNS](https://adguard-dns.io/kb/adguard-home/configuration/#private-addresses). + +Do not have OpenWrt dnsmasq and AGH both listen on `:53` at the same LAN IP; +port conflicts, or pointing dnsmasq's upstream at AGH while AGH points back at +dnsmasq, cause failures/forwarding loops. When AGH runs on a separate `.36` +host, the simplest topology is clients → AGH `.36:53` → public/encrypted +upstreams, forwarding only local resolution (PTR, LAN domain) → gateway dnsmasq. +This is an operational inference based on the official capabilities above and +must be validated against the live `ss -lntup`, dnsmasq config, and query path. + +AGH also has built-in DHCP, but it should only be enabled after **disabling the +existing gateway DHCP, explicitly migrating leases/reservations and verifying +options 3/6**. The official docs describe it as an alternative when the router +cannot hand out custom DNS, not as a dual-DHCP setup; the DHCP docs also state +leases/static leases are managed by AGH. [Getting started: router DHCP/DNS +handout and alternative DHCP](https://adguard-dns.io/kb/adguard-home/getting-started/#router); +[DHCP configuration](https://adguard-dns.io/kb/adguard-home/dhcp/). + +## Exposure and access control + +1. `dns.bind_hosts` should bind only loopback and the actual LAN address (here + prefer `.36`'s LAN address); do not use `0.0.0.0`/public interfaces for a + home recursive resolver. The official docs note full-interface binding + exposes DDoS risk; a small isolated network should pick local service + interfaces and may add loopback alongside. [Secure deployment: server + addresses](https://adguard-dns.io/kb/adguard-home/running-securely/#choosing-server-addresses); + [`bind_hosts` definition](https://adguard-dns.io/kb/adguard-home/configuration/#dns). +2. In the firewall, allow only managed LAN/VLAN → `.36` TCP/UDP 53; bind the web + admin UI to the management network/loopback and avoid WAN forwarding. At the + app layer, further restrict with `allowed_clients` to the local subnets; when + non-empty, only the CIDR/IP/ClientIDs in it are accepted, and it overrides + the effect of `disallowed_clients`. [Official access + control](https://adguard-dns.io/kb/adguard-home/configuration/#dns). +3. Keep `refuse_any: true` and a non-zero `ratelimit` (default 20 qps), unless + on-site evidence shows the service can never be reached from the internet and + LAN high-concurrency needs are data-backed; the official docs list both as + protection against reflection/DDoS. [Configuration + notes](https://adguard-dns.io/kb/adguard-home/configuration/#dns). +4. The web UI must have a user with login-failure lockout enabled; + `users: []` disables authentication, and `auth_attempts` or `block_auth_min` + of 0 disables the corresponding lockout logic. [Web/auth + parameters](https://adguard-dns.io/kb/adguard-home/configuration/#users). + +## Upstreams, encryption, and DNSSEC + +- Use two trusted, differently-networked DoH/DoT/DoQ upstreams for normal public + resolution; AGH supports these protocols and DNS Stamps. If an upstream + URL/hostname needs resolving, `bootstrap_dns` is its initial resolver; do not + point bootstrap at a name that requires AGH itself to resolve, and use + directly reachable, stable IPs. A `fallback_dns` can be set separately; it is + used only when primary upstreams do not respond. [Upstream protocols and + bootstrap/fallback](https://adguard-dns.io/kb/adguard-home/configuration/#dns). +- If local IPv6's actual egress path is unstable, do not enable + `bootstrap_prefer_ipv6`; the official docs note it makes DoH/DoQ/DoT bootstrap + prefer IPv6. Decide on-site based on each upstream's IPv4/IPv6 reachability, + latency, and failure rate. [Same reference](https://adguard-dns.io/kb/adguard-home/configuration/#dns). +- Consider enabling `enable_dnssec`, choosing recursive upstreams that actually + handle DNSSEC; but understand precisely: officially AGH sets the DO flag on + requests to upstreams; it does not perform recursive validation itself. On + site, query a known-bad-signature domain and observe AD/failure responses to + confirm upstream validation behavior. [DNSSEC parameter + definition](https://adguard-dns.io/kb/adguard-home/configuration/#dns). +- Prefer `upstream_mode: load_balance` unless measurement shows `parallel` first + answer latency is clearly worth the privacy/traffic cost of querying all + upstreams. The former picks by failure rate and average latency weighting; the + latter queries all upstreams concurrently; this is an operational judgment + based on documented behavior. [Upstream + modes](https://adguard-dns.io/kb/adguard-home/configuration/#dns). + +## Cache, ECS, and privacy + +- Keep DNS cache enabled; use `cache_ttl_min`/`cache_ttl_max` overrides + cautiously to avoid stale or unnecessarily short TTLs. `cache_optimistic` + answers with a 10-second TTL after expiry and attempts refresh — good for + low-latency scenarios that tolerate briefly stale results, not to be enabled + blindly on internal domains sensitive to immediate changes. [Cache + parameters](https://adguard-dns.io/kb/adguard-home/configuration/#dns). +- ECS is off by default: when enabled, AGH adds EDNS Client Subnet to upstream + requests; if CDN geo-scheduling is truly needed, use `use_custom: true` with a + coarse, non-real-client-subnet `custom_ip`, then compare hit rate vs. privacy + impact. [ECS parameters](https://adguard-dns.io/kb/adguard-home/configuration/#dns). +- Review log retention and who can read it. If device identity is not needed, + consider `anonymize_client_ip: true`; this anonymizes client addresses in logs + and stats. [Anonymization + parameter](https://adguard-dns.io/kb/adguard-home/configuration/#dns). + +## Proxy / OpenClash coexistence + +If OpenClash/transparent proxy redirects DNS, validate hop by hop to ensure: +client DNS requests reach AGH uniquely; AGH's outbound traffic to public +encrypted upstreams is not hijacked back to AGH; and the router's local dnsmasq +PTR/local-domain requests return to dnsmasq. Create explicit proxy/hijack +exceptions for `.36` and its upstreams, or explicitly design the proxy to take +over AGH's outbound resolution; do not rely on "it seems to resolve" to conclude +there is no loop. + +This is not an AGH-official OpenClash instruction (the official docs do not +maintain that integration), but an on-site verification requirement derived from +AGH's independent listening, upstream forwarding, and local-PTR-split +capabilities. The officially supported mechanisms are referenced at +[`bind_hosts`/upstreams/PTR](https://adguard-dns.io/kb/adguard-home/configuration/#dns). + +If serving DoH via a reverse proxy, `trusted_proxies` must list only the actual +proxy IP/CIDR; AGH will only trust `X-Real-IP`, `X-Forwarded-For`, etc. from +those sources. Do not write the whole LAN/WAN for convenience; +`http.doh.insecure_enabled` should only be enabled when a reverse proxy really +terminates TLS in front. [Proxy and +DoH](https://adguard-dns.io/kb/adguard-home/configuration/#dns); [encrypted DNS +docs](https://adguard-dns.io/kb/adguard-home/encryption/). For a pure-LAN +plaintext-53 service, publishing DoH/DoT/DoQ outward is usually unnecessary. + +## On-site review checklist (read-only) + +1. Record the version, service method, and the full effective configuration + (secrets redacted); run the AGH config check. Officially provided via + `--check-config`. [Command-line + arguments](https://adguard-dns.io/kb/adguard-home/configuration/#command-line-arguments). +2. On `.36`, the gateway, and a wired/wireless client, record `ss` listeners, + DHCP option 6, default route, `/etc/config/dhcp`, and OpenClash + DNS/firewall-redirect rules. +3. From a client test: public A/AAAA, ad-rule hits, DHCP hostnames, forward LAN + domain, private PTR, DNSSEC bad-signature domain, and the expected failure + mode when AGH stops. Then check whether client IPs in AGH query logs are + still genuinely identifiable. +4. From the AGH host, test each upstream and bootstrap over the actual protocol + for IPv4 and IPv6 reachability; compare with transparent proxy on/off to rule + out loops and leaks. +5. Verify the web UI/DNS ports have no WAN exposure; confirm allowed clients and + firewall match actual LAN/VLAN, guest network, and VPN subnets. + +Before changing anything, export the current configuration and DHCP/firewall +state; adjust one layer at a time (DHCP handout, AGH listening, upstreams, or +DNS hijack) and keep a known-good management and DNS fallback path. \ No newline at end of file diff --git a/docs/agents/issue-tracker.md b/docs/agents/issue-tracker.md new file mode 100644 index 0000000..7f64289 --- /dev/null +++ b/docs/agents/issue-tracker.md @@ -0,0 +1,47 @@ +# Issue tracker: Linear + +Issues and PRDs for this repo are tracked in Linear, driven entirely through the +Linear MCP configured in the current runtime. VPS operation records use the +Linear **`vps`** project (team `W1ndy`). + +## Conventions + +- **Create**: `linear_save_issue` (requires `title` + `team`); description as + real-newline Markdown. +- **Read**: `linear_get_issue ` (e.g. LIN-123), optionally with + `includeRelations`. +- **List**: `linear_list_issues`, filtered by `query` / `state` / `assignee` / + `team` / `project`. +- **Comment**: `linear_save_comment` (pass `issueId` + `body`). +- **Labels**: `linear_save_issue` `labels` field (replaces the whole label set). +- **Status**: `linear_save_issue` `state` field (type / name / ID). + +Resolve team, project, and cycle names through the Linear MCP rather than +hard-coding them. + +## Cross-issue constraint baseline + +The `plans/` directory is retired (removed 2026-08-06). Plans and baselines now +live in the vps project document "VPS 运维重构:已确认决策与验收标准". Open work +items are tracked as individual issues with dependencies expressed via +`blockedBy` / `blocks`. Read that document before adding new tasks. + +## When a skill says "publish to the issue tracker" + +Create a Linear issue with `linear_save_issue` (`project: vps`). + +## When a skill says "fetch the relevant ticket" + +Use `linear_get_issue `. + +## Wayfinding operations + +- **Map**: one issue as the map; child tickets link via `parentId`. +- **Child ticket**: `linear_save_issue` with `parentId` set to the map; labels + `wayfinder:` (research / prototype / grilling / task). +- **Blocking**: `linear_save_issue` `blockedBy` / `blocks` (append-only). +- **Frontier query**: list the map's open children, excluding those blocked or + already assigned; take the first in map order. +- **Claim**: `linear_save_issue` `assignee: me`. +- **Resolve**: `linear_save_comment` the answer → update `state` → append a + pointer under the map's decisions-so-far. \ No newline at end of file diff --git a/docs/agents/triage-labels.md b/docs/agents/triage-labels.md new file mode 100644 index 0000000..b716855 --- /dev/null +++ b/docs/agents/triage-labels.md @@ -0,0 +1,15 @@ +# Triage Labels + +The skills speak in terms of five canonical triage roles. This file maps those roles to the actual label strings used in this repo's issue tracker. + +| Label in mattpocock/skills | Label in our tracker | Meaning | +| -------------------------- | -------------------- | ---------------------------------------- | +| `needs-triage` | `needs-triage` | Maintainer needs to evaluate this issue | +| `needs-info` | `needs-info` | Waiting on reporter for more information | +| `ready-for-agent` | `ready-for-agent` | Fully specified, ready for an AFK agent | +| `ready-for-human` | `ready-for-human` | Requires human implementation | +| `wontfix` | `wontfix` | Will not be actioned | + +When a skill mentions a role (e.g. "apply the AFK-ready triage label"), use the corresponding label string from this table. + +Edit the right-hand column to match whatever vocabulary you actually use. diff --git a/hosts/dns.windy.lan.md b/hosts/dns.windy.lan.md new file mode 100644 index 0000000..f5b2fee --- /dev/null +++ b/hosts/dns.windy.lan.md @@ -0,0 +1,75 @@ +# dns.windy.lan — AdGuard Home LAN DNS + +## Role and access + +| Item | Value | +|---|---| +| IPv4 | `192.168.66.36` | +| SSH | `ssh -4 windy@192.168.66.36` | +| OS | Debian 12 | +| Roles | AdGuard Home LAN DNS; Mihomo explicit-proxy service | + +Use `windy` for routine inspection. Do not store credentials, AdGuard Home +user hashes, proxy subscriptions, or query-log contents in this repository. + +## AdGuard Home + +AdGuard Home runs as the root-owned `adguardhome` Docker container in host +network mode. The persistent deployment is `/opt/adguardhome`: + +| Item | Path / setting | +|---|---| +| Compose file | `/opt/adguardhome/docker-compose.yml` | +| Configuration | `/opt/adguardhome/conf/AdGuardHome.yaml` | +| Mutable data and query logs | `/opt/adguardhome/work` | +| DNS | TCP/UDP `53`; intentionally no source ACL so both LAN55 and LAN66 work | +| HTTP UI | `192.168.66.36:80`; LAN66 only | +| Image policy | Pin the reviewed image digest; do not use `latest` for a rebuild | + +The EdgeRouter DHCP service advertises this host as DNS for LAN55 and LAN66. +Do not enable AdGuard Home DHCP unless the existing EdgeRouter DHCP service is +explicitly migrated and disabled first. + +`agh-ui-access.service` loads `/etc/nftables-agh-ui-access.nft`. It permits +only `192.168.66.0/24` to TCP/80 and drops other TCP/80 input. It deliberately +does **not** restrict DNS, SSH, Docker, or Mihomo ports. Keep it isolated from +Docker-managed nftables tables. + +Current query-log policy is 14 days with anonymized client IPs. Check free +space before increasing retention. DNSSEC is disabled because the selected +upstream path did not pass the known-bad-signature validation check; do not +enable it without re-testing validated upstreams. + +The compatible names `hass.windy.lan` and legacy `hass.local` currently point +to the same Home Assistant address. Migrate clients to `hass.windy.lan`; keep +the legacy rewrite until its planned retirement. + +## Mihomo and routing boundary + +Mihomo listens for explicit proxy traffic on this host. Read-only checks found +no host TUN policy routing or local nftables/iptables transparent-redirection +rules. A Mihomo listener alone does not mean it intercepts AdGuard Home +traffic. The default gateway (`gfw.windy.lan`) can still affect outbound +traffic, so inspect its OpenClash rules separately before changing upstream +routing. + +## Safety + +- Back up `/opt/adguardhome/conf/AdGuardHome.yaml` and `docker-compose.yml` + before changing either, then run the official `--check-config` command. +- Recreate only the AdGuard Home container and verify DNS A/AAAA, local names, + the UI from LAN66, and the UI denial from LAN55 afterwards. +- Private PTR currently has no local authoritative source: direct EdgeRouter + PTR queries return NXDOMAIN. Do not point `local_ptr_upstreams` at the + router until that source exists. + +See [the health runbook](../runbooks/adguard-home-health.md) and +[the official configuration review](../docs/adguard-home-official-review-2026-08.md). + +Routine health through Ansible (`--limit adguardhome`), with a daily timer + +email alerting like the other active hosts: + +```bash +cd ansible +ansible-playbook playbooks/health-report.yml --limit adguardhome +``` diff --git a/hosts/mx2.windy.me.md b/hosts/mx2.windy.me.md index cb8f3b2..f185ba5 100644 --- a/hosts/mx2.windy.me.md +++ b/hosts/mx2.windy.me.md @@ -50,8 +50,9 @@ See [mailcow-smtp-client](../runbooks/mailcow-smtp-client.md). - [mailcow-health](../runbooks/mailcow-health.md) - [mailcow-update](../runbooks/mailcow-update.md) - [mailcow-smtp-client](../runbooks/mailcow-smtp-client.md) -- Script: [`../scripts/check-mx2.sh`](../scripts/check-mx2.sh) + +Routine health: `cd ansible && ansible-playbook playbooks/health-report.yml --limit mailcow`. ## Verified -Last checked: 2026-08-01 — `./scripts/check-mx2.sh` exit 0 (compose up, watchdog 100%, empty queue, TLS LE OK, SMTP Postcow, A/AAAA/PTR/MX/SPF OK). +Last checked: 2026-08-01 — Ansible mailcow health profile OK (compose up, watchdog 100%, empty queue, TLS LE OK, SMTP Postcow, A/AAAA/PTR/MX/SPF OK). diff --git a/hosts/us2.wsvc.info.md b/hosts/us2.wsvc.info.md index 088a848..af7614e 100644 --- a/hosts/us2.wsvc.info.md +++ b/hosts/us2.wsvc.info.md @@ -76,9 +76,15 @@ ansible-playbook playbooks/compose-reconcile.yml --limit vaultwarden \ -e '{"service_reconcile_confirm": true, "service_reconcile_targets": ["vaultwarden"], "service_reconcile_restart_traefik": true}' ``` -## Other software on this host (stubs) +## Other running services on this host -`/opt/traefik`, `soft-serve`, `authelia`, `dendrite`/`conduit`, `mastodon`, `rustdesk`, `zitadel`, etc. +| Container | Status | Image / notes | +|-----------|--------|---------------| +| `soft-serve` | Up | `ghcr.io/charmbracelet/soft-serve:latest` (`repo.windy.me:2222`) | +| `traefik` | Up | `traefik:v3.6.2` (`/opt/traefik`, public `:80`/`:443`) | +| `nghttpx-proxy` + `squid-backend` | Up | HTTP forward-proxy stack (`/opt/nghttpx`), network `nghttpx_internal-net`; details TBD | + +Directories for `authelia`, `conduit`, `dendrite`, `mastodon`, `rustdesk`, `zitadel`, etc. exist under `/opt` but have no running containers; treat them as dormant, not documented services. ## Verified diff --git a/inventory/hosts.md b/inventory/hosts.md index c7885df..85c3d69 100644 --- a/inventory/hosts.md +++ b/inventory/hosts.md @@ -21,6 +21,7 @@ diagnosis and procedures that are deliberately interactive or destructive; see | pgweb.wsvc.info | PowerDNS Postgres UI | — (HTTPS) | → hk2 | active | see hk2 | | **synapse.chans.xyz** | Matrix homeserver (ESS: Synapse + MAS + Element) | `ssh -4 windy@synapse.chans.xyz` | `169.58.86.13` | **active** | [hosts/synapse.chans.xyz.md](../hosts/synapse.chans.xyz.md) | | **gfw.windy.lan** | OpenWrt LAN gateway / OpenClash | `ssh -4 root@192.168.66.1` | `192.168.66.1` | **active** | [hosts/gfw.windy.lan.md](../hosts/gfw.windy.lan.md) | +| **dns.windy.lan** | AdGuard Home LAN DNS + Mihomo explicit proxy | `ssh -4 windy@192.168.66.36` | `192.168.66.36` | **active** | [hosts/dns.windy.lan.md](../hosts/dns.windy.lan.md) | | **gw** | EdgeRouter X primary LAN gateway | `ssh -4 zhiqiang@192.168.66.254` | `192.168.66.254` | **active** | [hosts/gw.md](../hosts/gw.md) | | **ubnt** | UniFi Network Controller | `ssh -4 windy@192.168.66.46` | `192.168.66.46` | **active** | [hosts/ubnt.md](../hosts/ubnt.md) | diff --git a/plans/2026-08-02-vps-maintenance-ansible-restic-refactor-v1.md b/plans/2026-08-02-vps-maintenance-ansible-restic-refactor-v1.md deleted file mode 100644 index ba14396..0000000 --- a/plans/2026-08-02-vps-maintenance-ansible-restic-refactor-v1.md +++ /dev/null @@ -1,70 +0,0 @@ -# VPS 維護專案重構:Ansible + systemd timers + Restic - -## Objective - -將現有 VPS 維運知識庫漸進重構為一個以 Ansible 管理一致性、以 systemd timers 執行每日唯讀檢查、以 Restic 建立加密異機備份的個人 SRE 維運專案。現納管 mx2、us2、hk2 與 Matrix K3s 主機。自動化預設只能觀測與告警;受 allowlist 和顯式確認保護的 Compose reconciliation 可處理已審查的服務重建,DNS、機密與破壞性資料操作仍需人工確認。 - -## Confirmed Decisions - -- [x] 定位為個人 SRE/維運手冊庫,而非全自動修復平台。 -- [x] 已納管 mx2、us2、hk2 與 Matrix K3s;mx、us1、us4 待完成盤點與緊急存取驗證後才納管。 -- [x] 每日自動做唯讀檢查與 Email 通知;每週人工審查、每月人工維護、每季復原演練與權限盤點。 -- [x] 以 Email 作為唯一告警與摘要通道。 -- [x] 採用本機快速復原加密異機副本的 3-2-1 最小可行策略。 -- [x] 真實機密不進 Git;repo 只保存去敏設定結構與操作/驗證紀錄。 -- [x] 日常存取使用非 root SSH key;禁止 root/password SSH,維護 provider/recovery console 緊急存取。 -- [x] 採用 Ansible + systemd timers + Restic 作為技術基線。 - -## Implementation Plan - -- [x] **Status: Done — 建立去敏 Ansible inventory。** 為 mx2、us2、hk2 與 Matrix 建立主機與服務群組,將非機密主機資料與現有人類可讀的 `inventory/hosts.md` 對應;理由是提供可審查的機器可讀編排層,且不取代既有事實來源。 -- [x] **Status: Done — 實作 audit-only Ansible playbook。** 只收集連線、OS、磁碟、systemd、Docker Compose 和依賴命令狀態,不做設定或服務改動;理由是先確認控制面與真實現況。 -- [x] **Status: Done — 定義跨服務健康檢查結果合約。** 統一去敏結構化結果、日誌位置、嚴重度及 exit code,讓人工執行、systemd 和 Ansible audit 可共同消費;理由是避免各服務告警語意漂移。 -- [x] **Status: Done — 保留並模組化 Mailcow 健康檢查。** 持續檢查 Compose、watchdog、queue、listeners、HTTP/HTTPS、TLS、SMTP、DNS/PTR/MX/SPF;理由是現有腳本已覆蓋關鍵郵件服務面向。 -- [x] **Status: Done — 實作 Vaultwarden 唯讀健康檢查。** 驗證 Compose/Postgres、HTTPS、有效設定、SMTP AUTH、備份新鮮度及機密指紋一致性,禁止輸出機密;理由是 `config.json` 優先於 `.env`,且 SMTP 漂移已有已知風險。 -- [x] **Status: Done — 實作 PowerDNS 唯讀健康檢查。** 驗證容器、版本與安全公告、ns1/ns2 served SOA、API、Web UI、備份新鮮度與必要設定;理由是同時覆蓋公開 DNS 與 secondary 同步。 -- [x] **Status: Done — 使用 Ansible 部署 systemd healthcheck service/timer。** 每台主機本機執行每日檢查,具 persistent 排程、權限、logrotate 和一致錯誤處理;理由是控制端離線不應阻止巡檢。 -- [x] **Status: Done — 部署 Email 告警與每日摘要。** Critical/unknown 立即通知、健康狀態每日摘要、重複失敗抑制;SMTP 真實認證僅在各主機受限路徑保存;理由是已選定 Email 為唯一通知通道。 -- [!] **Status: Blocked — 決定 Restic 異機 repository 與存取隔離。** 選擇具加密傳輸、權限隔離與可承受保留需求的 S3/B2/SFTP 或等價目的地;理由是無異機 repository 即無法滿足已確認的 3-2-1 政策。阻塞:本地實作依要求未虛構 backend、repository 或 credentials。 -- [x] **Status: Done (templates gated) — 部署 Vaultwarden 與 PowerDNS 的 Restic 備份。** 將既有一致性資料庫 dump、必要資料目錄及復原元資料加密同步,並加入 snapshot 年齡和 `restic check` 驗證;理由是兩者已有本機備份但缺少異機保護。啟用被 repository 決策和主機端受限設定檔阻擋。 -- [!] **Status: Blocked — 完成 Mailcow 備份設計審查。** 官方流程已確認:以 `/opt/mail/helper-scripts/backup_and_restore.sh backup all`(或經明確核准的元件集)先產生一致性備份,再由 Restic 同步該輸出;不得直接複製 Docker volumes。阻塞:仍需決定本機備份位置、保留期、排程及異機 Restic repository。 -- [x] **Status: Done (templates gated) — 部署 Restic backup、retention、forget/prune 與 check timers。** 與現有資料庫 dump 時段錯開,所有 repository 認證與密碼留在伺服器端受限檔案;理由是保持備份可用與成本可控。啟用被 repository 決策和主機端受限設定檔阻擋。 -- [!] **Status: Blocked — 補齊服務級復原 runbook。** 阻塞:需在選定 Restic repository、建立實際 snapshot 並確認各服務的實際備份輸出後,才能編寫可驗證的 restore 前置條件、順序與 rollback;禁止臆造 backend/credentials 或未驗證還原命令。 -- [!] **Status: Blocked — 執行隔離式復原演練。** 阻塞:尚未選定/配置 Restic 異機 repository,亦尚無可供還原的異機 snapshot;演練不得對現有生產資料執行。 -- [x] **Status: Done — 實作受控 common baseline 與 maintenance playbook。** 僅在 audit 穩定後納入 SSH 稽核、時間同步、logrotate、更新預覽與人工確認的維護操作;理由是避免工具導入期間同時改變服務狀態。 -- [x] **Status: Done (2026-08-03) — 將例行操作收斂為 Ansible 入口。** 新增 on-demand health report 與 allowlisted Compose reconciliation;health、maintenance preview、baseline 現可涵蓋 Matrix。互動式 Mailcow 更新、資料遷移、DNS 與機密操作保留人工程序。 -- [x] **Status: Done (separate change defined) — 將 PowerDNS API key、DB password 與 TSIG 輪替列為分離變更。** 已明確保持 API key、DB password 與 TSIG 為三項獨立、需人工核准的變更,並要求逐步驗證 Auth、Poweradmin、AXFR/NOTIFY、DNSSEC 與 ns2 同步;未輪替任何機密。 -- [!] **Status: Blocked — 盤點 mx、us1、us4。** 阻塞:inventory 僅記錄 `TBD` SSH/角色或缺少完整事實,尚未提供可驗證的存取方式與 provider/recovery console 資訊;不得猜測或嘗試未授權存取。 - -## Verification Criteria - -- [ ] 四台 active 主機能被 Ansible 正確解析;Compose 主機可執行 audit,且 audit 不產生主機變更。 -- [ ] 四台主機每日產生去敏健康結果;服務、容量、TLS、公開端點和備份新鮮度異常均能被偵測。 -- [ ] 正常狀態寄送 Email 摘要;受控測試異常能觸發一次可讀且不含機密的 Email 告警。 -- [ ] 每個核心服務至少有一份加密、異機、可列出且通過完整性檢查的 Restic snapshot。 -- [ ] 至少完成一次隔離式實際還原,並確認其結果符合或明確量化偏離服務 RPO/RTO。 -- [ ] 自動化排程不包含更新、重啟、修復、秘密輪替或 DNS 變更。 -- [ ] Repo、Ansible vars、產出日誌及 Email 均不包含任何真實機密。 - -## Potential Risks and Mitigations - -1. **Mailcow 備份未保持資料一致性。** - Mitigation: 先依官方方式完成備份範圍與還原設計,再上線排程;以隔離還原驗證作為完成門檻。 - -2. **Ansible 設定錯誤改動生產系統。** - Mitigation: 先導入 audit-only;變更 playbook 必須使用 dry-run/diff、明確 tag 與人工確認,且先限制單一 host。 - -3. **機密出現在 Git、Ansible output 或 Email。** - Mitigation: 真實值只存主機受限檔案;去敏輸出、no-log 機制和測試用機密掃描為強制要求。 - -4. **Timer 成功但公開服務實際不可用。** - Mitigation: 同時執行本機 Compose/DB 檢查與外部 HTTP、SMTP、TLS、DNS 檢查。 - -5. **PowerDNS 密鑰輪替中斷 DNS 管理或 zone transfer。** - Mitigation: API key、DB password、TSIG 分批處理;每步都驗證 primary/secondary 同步、DNSSEC 和管理 UI。 - -## Alternative Approaches - -1. **純 shell + systemd,不使用 Ansible**:初期較快,但設定與排程容易跨主機漂移,且難以審查或擴充;不建議作為長期方案。 -2. **Ansible + 集中式監控平台**:可增加儀表板與趨勢,但平台本身需額外維護;等 Email 摘要無法滿足需求時再評估。 -3. **立即導入 SOPS + age**:可使加密設定進 Git,但需先完成密鑰生命週期與緊急存取設計;建議列為後續階段。 diff --git a/runbooks/adguard-home-health.md b/runbooks/adguard-home-health.md new file mode 100644 index 0000000..cd31234 --- /dev/null +++ b/runbooks/adguard-home-health.md @@ -0,0 +1,60 @@ +# AdGuard Home health — dns.windy.lan + +This runbook is read-only. It does not expose query-log contents or secrets. + +Routine checks run through Ansible on demand: + +```bash +cd ansible +ansible-playbook playbooks/health-report.yml --limit adguardhome +``` + +The same sanitized profile is installed as a daily server-local timer +(`vps-healthcheck.timer`, output at `/var/lib/vps-health/latest.json`) via +`playbooks/healthchecks.yml`; the manual checks below are for focused diagnosis. + +## Verify service and configuration + +```bash +ssh -4 -o BatchMode=yes windy@192.168.66.36 +sudo docker ps --filter name=^/adguardhome$ +sudo docker exec adguardhome /opt/adguardhome/AdGuardHome --check-config \ + -c /opt/adguardhome/conf/AdGuardHome.yaml +sudo ss -lntup | grep -E '(:53|:80)' +``` + +Expected: DNS listens on TCP/UDP 53, and the HTTP UI listens only on +`192.168.66.36:80`. + +## Verify DNS and storage + +```bash +dig +time=5 +tries=1 @192.168.66.36 example.com A +noall +comments +answer +dig +time=5 +tries=1 @192.168.66.36 example.com AAAA +noall +comments +answer +dig +time=5 +tries=1 @192.168.66.36 hass.windy.lan A +noall +comments +answer +sudo docker logs --since 30m adguardhome 2>&1 | grep -Ei '\[(error|warn|fatal)\]|level=(error|warn|fatal)' || true +df -h /opt/adguardhome/work +``` + +The host intentionally accepts DNS from LAN55 and LAN66. Do not infer a +broken DNS policy merely because `allowed_clients` is empty. + +## Verify management UI boundary + +From LAN66, the UI should return a login redirect: + +```bash +curl -sS -o /dev/null -w '%{http_code}\n' http://192.168.66.36/ +``` + +From a LAN55 source, TCP/80 should time out or be refused by the dedicated +`agh-ui-access.service` nftables rule. Do not alter this rule to restrict +port 53 without an explicit network-access review. + +## DNSSEC and PTR caveats + +DNSSEC is currently disabled. Before enabling it, select upstreams that pass +a known-bad-signature test; an enabled DO bit alone is not validation. + +Private PTR forwarding is intentionally absent because the EdgeRouter does +not currently answer private PTR requests. diff --git a/runbooks/mailcow-health.md b/runbooks/mailcow-health.md index e9c471a..8ccf692 100644 --- a/runbooks/mailcow-health.md +++ b/runbooks/mailcow-health.md @@ -2,9 +2,10 @@ Target: [mx2.windy.me](../hosts/mx2.windy.me.md) Path: `/opt/mail` -Prefer: the Ansible health report (`ansible/playbooks/health-report.yml`), or -`./scripts/check-mx2.sh` from this repo. The SSH commands below remain useful -for focused, interactive diagnosis after the report identifies a problem. +Prefer: the Ansible health report (`ansible/playbooks/health-report.yml --limit mailcow`), +or the server-local daily timer result at `/var/lib/vps-health/latest.json`. The +SSH commands below remain useful for focused, interactive diagnosis after the +report identifies a problem. ## Focused SSH diagnostics @@ -66,7 +67,9 @@ dig @1.1.1.1 +short windy.me TXT | grep -i spf ## Scheduled local check -The sanitized Ansible health profile is `mailcow` (`ansible/playbooks/healthchecks.yml`). It preserves the existing standalone `scripts/check-mx2.sh` as the richer operator check, while the server-local timer emits a sanitized result at `/var/lib/vps-health/latest.json`. Neither check changes Mailcow configuration or service state. +The sanitized Ansible health profile is `mailcow` (`ansible/playbooks/healthchecks.yml`). +The server-local timer emits a sanitized result at `/var/lib/vps-health/latest.json`. +It does not change Mailcow configuration or service state. ## Pass criteria diff --git a/runbooks/mailcow-update.md b/runbooks/mailcow-update.md index 57e8950..9b395e3 100644 --- a/runbooks/mailcow-update.md +++ b/runbooks/mailcow-update.md @@ -6,7 +6,7 @@ Path: `/opt/mail` ## Before -1. Run [mailcow-health](mailcow-health.md) (or `./scripts/check-mx2.sh`). Record baseline. +1. Run [mailcow-health](mailcow-health.md) (Ansible health report). Record baseline. 2. Note config quirks in `mailcow.conf`, especially: - `SKIP_CLAMD=y` — after update, `clamd` may still be running until services are recreated per mailcow docs. - `SKIP_LETS_ENCRYPT`, ports, hostname — should stay `mx2.windy.me`. diff --git a/scripts/check-mx2.sh b/scripts/check-mx2.sh deleted file mode 100755 index 6252f4c..0000000 --- a/scripts/check-mx2.sh +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env bash -# Read-only health check for mx2.windy.me mailcow. -# Usage: ./scripts/check-mx2.sh -# Env: -# MX2_HOST default mx2.windy.me -# MX2_SSH_USER default windy -# MX2_ALLOW_IPV6 set to 1 to allow IPv6 (default forces IPv4; WSL often lacks v6 route) -set -euo pipefail - -HOST="${MX2_HOST:-mx2.windy.me}" -SSH_USER="${MX2_SSH_USER:-windy}" - -# Default -4: host has AAAA but many clients (WSL) have no IPv6 route → "Network is unreachable" -SSH_AF_OPTS=(-4) -CURL_AF_OPTS=(-4) -OPENSSL_AF_OPTS=(-4) -if [[ "${MX2_ALLOW_IPV6:-0}" == "1" ]]; then - SSH_AF_OPTS=() - CURL_AF_OPTS=() - OPENSSL_AF_OPTS=() -fi - -SSH=(ssh "${SSH_AF_OPTS[@]}" -o BatchMode=yes -o ConnectTimeout=10 "${SSH_USER}@${HOST}") -# Prefer public resolvers: local stub (127.0.0.53) is flaky from some WSL setups -DIG_OPTS=(+time=3 +tries=2 +short) -DIG_SERVERS=(1.1.1.1 8.8.8.8) -FAILURES=0 - -section() { printf '\n===== %s =====\n' "$1"; } - -fail() { - printf 'FAIL: %s\n' "$1" >&2 - FAILURES=$((FAILURES + 1)) -} - -# Strip dig commentary (timeouts are printed on stdout as ";; ...") -dig_answers_only() { - grep -v '^;;' | grep -v '^$' || true -} - -# dig_short [type] — tries each public resolver; skips timeout noise -dig_short() { - local name="$1" - local typ="${2:-A}" - local server out - for server in "${DIG_SERVERS[@]}"; do - out="$(dig @"${server}" "${DIG_OPTS[@]}" "$name" "$typ" 2>/dev/null | dig_answers_only)" || out="" - if [[ -n "$out" ]]; then - printf '%s\n' "$out" - return 0 - fi - done - return 1 -} - -# dig_ptr -dig_ptr() { - local ip="$1" - local server out - for server in "${DIG_SERVERS[@]}"; do - out="$(dig @"${server}" "${DIG_OPTS[@]}" -x "$ip" 2>/dev/null | dig_answers_only)" || out="" - if [[ -n "$out" ]]; then - printf '%s\n' "$out" - return 0 - fi - done - return 1 -} - -check_ptrs() { - local ip ptr - while read -r ip; do - [[ -z "$ip" ]] && continue - ptr="$(dig_ptr "$ip" || true)" - printf 'PTR %s -> %s\n' "$ip" "${ptr:-}" - grep -Fiq "${HOST}" <<<"$ptr" || fail "PTR for ${ip} does not mention ${HOST}" - done -} - -need_cmd() { - command -v "$1" >/dev/null 2>&1 || { - echo "missing required command: $1" >&2 - exit 127 - } -} - -need_cmd ssh -need_cmd curl -need_cmd openssl -need_cmd dig -need_cmd timeout - -section "SSH + compose ps" -PS_OUT="$("${SSH[@]}" 'cd /opt/mail && docker compose ps -a')" || { - fail "ssh or docker compose ps" - echo "Cannot continue without SSH/compose." >&2 - exit 1 -} -printf '%s\n' "$PS_OUT" -if grep -qiE 'Exited|Restarting|[[:space:]]Dead[[:space:]]' <<<"$PS_OUT"; then - fail "compose has Exited/Restarting/Dead containers" -fi -for svc in nginx-mailcow postfix-mailcow dovecot-mailcow mysql-mailcow; do - grep -Fq "$svc" <<<"$PS_OUT" || fail "missing service ${svc}" -done - -section "Watchdog (tail)" -if ! "${SSH[@]}" 'cd /opt/mail && docker compose logs --tail=40 watchdog-mailcow'; then - fail "watchdog logs" -fi - -section "Mail queue" -QUEUE_OUT="$("${SSH[@]}" 'cd /opt/mail && docker compose exec -T postfix-mailcow postqueue -p' 2>&1)" || { - fail "postqueue" - QUEUE_OUT="" -} -printf '%s\n' "$QUEUE_OUT" -if [[ -n "$QUEUE_OUT" ]] && ! grep -Fiq 'Mail queue is empty' <<<"$QUEUE_OUT"; then - fail "mail queue not empty" -fi - -section "Listeners" -LISTEN_OUT="$("${SSH[@]}" 'ss -tlnp 2>/dev/null | grep -E ":(25|465|587|110|143|993|995|80|443|4190)[[:space:]]" || true')" -printf '%s\n' "$LISTEN_OUT" -for p in 25 465 587 110 143 993 995 80 443 4190; do - if ! grep -qE ":${p}[[:space:]]" <<<"$LISTEN_OUT"; then - fail "port ${p} not listening on host" - fi -done - -section "HTTP / HTTPS" -HTTP_OUT="$(curl "${CURL_AF_OPTS[@]}" -sS -I --max-time 10 "http://${HOST}/" 2>&1 | head -8)" || true -printf '%s\n' "$HTTP_OUT" -echo "---" -# No -k: surface TLS trust problems -HTTPS_OUT="$(curl "${CURL_AF_OPTS[@]}" -sS -I --max-time 10 "https://${HOST}/" 2>&1 | head -15)" || true -printf '%s\n' "$HTTPS_OUT" -grep -qE '^HTTP/[0-9.]+ 301' <<<"$HTTP_OUT" || fail "HTTP did not redirect (expect 301)" -grep -qE '^HTTP/[0-9.]+ 200' <<<"$HTTPS_OUT" || fail "HTTPS did not return 200" - -section "TLS cert (443)" -CERT_OUT="$( - timeout 15 openssl s_client "${OPENSSL_AF_OPTS[@]}" -connect "${HOST}:443" -servername "${HOST}" /dev/null \ - | openssl x509 -noout -subject -issuer -dates 2>/dev/null -)" || true -printf '%s\n' "$CERT_OUT" -if ! grep -Fq "CN=${HOST}" <<<"$CERT_OUT" && ! grep -Fq "CN = ${HOST}" <<<"$CERT_OUT"; then - fail "TLS subject missing ${HOST}" -fi -grep -Fiq "Let's Encrypt" <<<"$CERT_OUT" || fail "TLS issuer not Let's Encrypt" - -section "SMTP banner" -# Connect via A record so /dev/tcp does not pick unreachable AAAA first -SMTP_IP="$(dig_short "${HOST}" A 2>/dev/null | head -n1 || true)" -if [[ -z "$SMTP_IP" ]]; then - fail "cannot resolve A for SMTP check" - SMTP_OUT="" -else - SMTP_OUT="$( - timeout 8 bash -c "exec 3<>/dev/tcp/${SMTP_IP}/25; printf 'EHLO test.local\r\nQUIT\r\n' >&3; cat <&3" 2>/dev/null \ - | head -20 - )" || true -fi -printf 'connect %s:25\n' "${SMTP_IP:-?}" -printf '%s\n' "$SMTP_OUT" -grep -qiE 'Postcow|ESMTP' <<<"$SMTP_OUT" || fail "SMTP banner unexpected" - -section "DNS" -A_REC="$(dig_short "${HOST}" A || true)" -AAAA_REC="$(dig_short "${HOST}" AAAA || true)" -printf 'A: %s\n' "${A_REC:-}" -printf 'AAAA: %s\n' "${AAAA_REC:-}" -[[ -n "$A_REC" ]] || fail "missing A record" -[[ -n "$AAAA_REC" ]] || fail "missing AAAA record" - -check_ptrs <<<"$A_REC" -check_ptrs <<<"$AAAA_REC" - -MX_REC="$(dig_short windy.me MX || true)" -printf 'MX:\n%s\n' "${MX_REC:-}" -grep -Fiq "${HOST}" <<<"$MX_REC" || fail "windy.me MX missing ${HOST}" - -SPF="$(dig_short windy.me TXT || true)" -printf 'SPF:\n' -if ! grep -Fi 'v=spf1' <<<"$SPF"; then - fail "no SPF TXT on windy.me" -fi - -section "Done" -if [[ "$FAILURES" -gt 0 ]]; then - echo "Health check finished with ${FAILURES} failure(s)." - exit 1 -fi -echo "Health check OK. Update hosts/mx2.windy.me.md Verified line if desired." -exit 0