vps/scripts/check-mx2.sh

#!/usr/bin/env bash
# Read-only health check for mx2.windy.me mailcow.
# Usage: ./scripts/check-mx2.sh
# Env:
#   MX2_HOST       default mx2.windy.me
#   MX2_SSH_USER   default windy
#   MX2_ALLOW_IPV6 set to 1 to allow IPv6 (default forces IPv4; WSL often lacks v6 route)
set -euo pipefail

HOST="${MX2_HOST:-mx2.windy.me}"
SSH_USER="${MX2_SSH_USER:-windy}"

# Default -4: host has AAAA but many clients (WSL) have no IPv6 route → "Network is unreachable"
SSH_AF_OPTS=(-4)
CURL_AF_OPTS=(-4)
OPENSSL_AF_OPTS=(-4)
if [[ "${MX2_ALLOW_IPV6:-0}" == "1" ]]; then
  SSH_AF_OPTS=()
  CURL_AF_OPTS=()
  OPENSSL_AF_OPTS=()
fi

SSH=(ssh "${SSH_AF_OPTS[@]}" -o BatchMode=yes -o ConnectTimeout=10 "${SSH_USER}@${HOST}")
# Prefer public resolvers: local stub (127.0.0.53) is flaky from some WSL setups
DIG_OPTS=(+time=3 +tries=2 +short)
DIG_SERVERS=(1.1.1.1 8.8.8.8)
FAILURES=0

section() { printf '\n===== %s =====\n' "$1"; }

fail() {
  printf 'FAIL: %s\n' "$1" >&2
  FAILURES=$((FAILURES + 1))
}

# Strip dig commentary (timeouts are printed on stdout as ";; ...")
dig_answers_only() {
  grep -v '^;;' | grep -v '^$' || true
}

# dig_short <name> [type] — tries each public resolver; skips timeout noise
dig_short() {
  local name="$1"
  local typ="${2:-A}"
  local server out
  for server in "${DIG_SERVERS[@]}"; do
    out="$(dig @"${server}" "${DIG_OPTS[@]}" "$name" "$typ" 2>/dev/null | dig_answers_only)" || out=""
    if [[ -n "$out" ]]; then
      printf '%s\n' "$out"
      return 0
    fi
  done
  return 1
}

# dig_ptr <ip>
dig_ptr() {
  local ip="$1"
  local server out
  for server in "${DIG_SERVERS[@]}"; do
    out="$(dig @"${server}" "${DIG_OPTS[@]}" -x "$ip" 2>/dev/null | dig_answers_only)" || out=""
    if [[ -n "$out" ]]; then
      printf '%s\n' "$out"
      return 0
    fi
  done
  return 1
}

check_ptrs() {
  local ip ptr
  while read -r ip; do
    [[ -z "$ip" ]] && continue
    ptr="$(dig_ptr "$ip" || true)"
    printf 'PTR %s -> %s\n' "$ip" "${ptr:-<none>}"
    grep -Fiq "${HOST}" <<<"$ptr" || fail "PTR for ${ip} does not mention ${HOST}"
  done
}

need_cmd() {
  command -v "$1" >/dev/null 2>&1 || {
    echo "missing required command: $1" >&2
    exit 127
  }
}

need_cmd ssh
need_cmd curl
need_cmd openssl
need_cmd dig
need_cmd timeout

section "SSH + compose ps"
PS_OUT="$("${SSH[@]}" 'cd /opt/mail && docker compose ps -a')" || {
  fail "ssh or docker compose ps"
  echo "Cannot continue without SSH/compose." >&2
  exit 1
}
printf '%s\n' "$PS_OUT"
if grep -qiE 'Exited|Restarting|[[:space:]]Dead[[:space:]]' <<<"$PS_OUT"; then
  fail "compose has Exited/Restarting/Dead containers"
fi
for svc in nginx-mailcow postfix-mailcow dovecot-mailcow mysql-mailcow; do
  grep -Fq "$svc" <<<"$PS_OUT" || fail "missing service ${svc}"
done

section "Watchdog (tail)"
if ! "${SSH[@]}" 'cd /opt/mail && docker compose logs --tail=40 watchdog-mailcow'; then
  fail "watchdog logs"
fi

section "Mail queue"
QUEUE_OUT="$("${SSH[@]}" 'cd /opt/mail && docker compose exec -T postfix-mailcow postqueue -p' 2>&1)" || {
  fail "postqueue"
  QUEUE_OUT=""
}
printf '%s\n' "$QUEUE_OUT"
if [[ -n "$QUEUE_OUT" ]] && ! grep -Fiq 'Mail queue is empty' <<<"$QUEUE_OUT"; then
  fail "mail queue not empty"
fi

section "Listeners"
LISTEN_OUT="$("${SSH[@]}" 'ss -tlnp 2>/dev/null | grep -E ":(25|465|587|110|143|993|995|80|443|4190)[[:space:]]" || true')"
printf '%s\n' "$LISTEN_OUT"
for p in 25 465 587 110 143 993 995 80 443 4190; do
  if ! grep -qE ":${p}[[:space:]]" <<<"$LISTEN_OUT"; then
    fail "port ${p} not listening on host"
  fi
done

section "HTTP / HTTPS"
HTTP_OUT="$(curl "${CURL_AF_OPTS[@]}" -sS -I --max-time 10 "http://${HOST}/" 2>&1 | head -8)" || true
printf '%s\n' "$HTTP_OUT"
echo "---"
# No -k: surface TLS trust problems
HTTPS_OUT="$(curl "${CURL_AF_OPTS[@]}" -sS -I --max-time 10 "https://${HOST}/" 2>&1 | head -15)" || true
printf '%s\n' "$HTTPS_OUT"
grep -qE '^HTTP/[0-9.]+ 301' <<<"$HTTP_OUT" || fail "HTTP did not redirect (expect 301)"
grep -qE '^HTTP/[0-9.]+ 200' <<<"$HTTPS_OUT" || fail "HTTPS did not return 200"

section "TLS cert (443)"
CERT_OUT="$(
  timeout 15 openssl s_client "${OPENSSL_AF_OPTS[@]}" -connect "${HOST}:443" -servername "${HOST}" </dev/null 2>/dev/null \
    | openssl x509 -noout -subject -issuer -dates 2>/dev/null
)" || true
printf '%s\n' "$CERT_OUT"
if ! grep -Fq "CN=${HOST}" <<<"$CERT_OUT" && ! grep -Fq "CN = ${HOST}" <<<"$CERT_OUT"; then
  fail "TLS subject missing ${HOST}"
fi
grep -Fiq "Let's Encrypt" <<<"$CERT_OUT" || fail "TLS issuer not Let's Encrypt"

section "SMTP banner"
# Connect via A record so /dev/tcp does not pick unreachable AAAA first
SMTP_IP="$(dig_short "${HOST}" A 2>/dev/null | head -n1 || true)"
if [[ -z "$SMTP_IP" ]]; then
  fail "cannot resolve A for SMTP check"
  SMTP_OUT=""
else
  SMTP_OUT="$(
    timeout 8 bash -c "exec 3<>/dev/tcp/${SMTP_IP}/25; printf 'EHLO test.local\r\nQUIT\r\n' >&3; cat <&3" 2>/dev/null \
      | head -20
  )" || true
fi
printf 'connect %s:25\n' "${SMTP_IP:-?}"
printf '%s\n' "$SMTP_OUT"
grep -qiE 'Postcow|ESMTP' <<<"$SMTP_OUT" || fail "SMTP banner unexpected"

section "DNS"
A_REC="$(dig_short "${HOST}" A || true)"
AAAA_REC="$(dig_short "${HOST}" AAAA || true)"
printf 'A:    %s\n' "${A_REC:-<none>}"
printf 'AAAA: %s\n' "${AAAA_REC:-<none>}"
[[ -n "$A_REC" ]] || fail "missing A record"
[[ -n "$AAAA_REC" ]] || fail "missing AAAA record"

check_ptrs <<<"$A_REC"
check_ptrs <<<"$AAAA_REC"

MX_REC="$(dig_short windy.me MX || true)"
printf 'MX:\n%s\n' "${MX_REC:-<none>}"
grep -Fiq "${HOST}" <<<"$MX_REC" || fail "windy.me MX missing ${HOST}"

SPF="$(dig_short windy.me TXT || true)"
printf 'SPF:\n'
if ! grep -Fi 'v=spf1' <<<"$SPF"; then
  fail "no SPF TXT on windy.me"
fi

section "Done"
if [[ "$FAILURES" -gt 0 ]]; then
  echo "Health check finished with ${FAILURES} failure(s)."
  exit 1
fi
echo "Health check OK. Update hosts/mx2.windy.me.md Verified line if desired."
exit 0