Merge pgdb ops runbooks — health/restore/update (W1N-228)

2026-08-29 14:51:31 +08:00
parent c7dc4fd25c 2d95f87897
commit d2063e7496
12 changed files with 598 additions and 3 deletions
@@ -10,6 +10,7 @@ files: every secret is a `${VAR}` reference resolved from the **server-local
 |---------|------|----------------|-----------|
 | `vaultwarden` | us2 (`/opt/vaultwarden`) | `compose/vaultwarden/compose.yml` | static file + `compose-deploy.yml` |
 | `pdns` | hk2 (`/opt/pdns`) | `compose/pdns/compose.yml` | static file + `compose-deploy.yml` |
+| `pgdb` | pgdb (`/opt/database`, 无 ansible) | `compose/pgdb/compose.yml` | static file（手动部署：scp → `docker compose config -q` → `up -d`；服务器文件名 `docker-compose.yml`） |
 | `adguardhome` | dns.windy.lan (`/opt/adguardhome`) | — (待从 LAN 提取) | static file (pending) |
 | `unifi` | ubnt (`/home/windy/unifi-9`) | — (待从 LAN 提取) | static file (pending) |
 | `wireguard` | us4 (`/opt/wireguard`) | `ansible/templates/wireguard-compose.yml.j2` | role-rendered (inventory vars) |
@@ -0,0 +1,5 @@
+# pgdb compose secrets — copy to /opt/database/.env on the host, chmod 600.
+# NEVER commit the real values. Generate: openssl rand -hex 24
+POSTGRES_PASSWORD=change-me-strong-hex
+PGWEB_AUTH_USER=pgweb
+PGWEB_AUTH_PASS=change-me-strong-hex
@@ -0,0 +1,65 @@
+# pgdb (192.168.55.15) — TimescaleDB + pgweb GUI + nightly backup
+#
+# Deploy: copy this file to /opt/database/docker-compose.yml on pgdb,
+# create /opt/database/.env (chmod 600) from .env.example, plus
+# /opt/database/pgweb-bookmarks/{hass,scribe}.toml (chmod 600, contains DB password).
+# Then: docker compose config --quiet && docker compose up -d
+#
+# Rollback: previous launch command is kept at /opt/database/run
+# (container is stateless; data lives on /srv/pgdata).
+services:
+  timescaledb:
+    image: timescale/timescaledb:latest-pg18
+    container_name: timescaledb
+    restart: unless-stopped
+    ports:
+      - "192.168.55.15:5432:5432"   # bind VM IP only (no IPv6 wildcard)
+    environment:
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+    volumes:
+      - /srv/pgdata:/var/lib/postgresql   # data disk (ext4 /dev/sdb1)
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 30s
+      timeout: 5s
+      retries: 5
+      start_period: 10s
+
+  pgweb:
+    image: sosedoff/pgweb:latest
+    container_name: pgweb
+    restart: unless-stopped
+    # bind/listen/readonly/sessions/bookmarks-only/bookmarks-dir are CLI flags (no env equivalent in v0.17.0)
+    command: ["pgweb", "--bind", "0.0.0.0", "--listen", "8081", "--readonly", "--sessions", "--bookmarks-only", "--bookmarks-dir", "/bookmarks"]
+    ports:
+      - "192.168.55.15:8081:8081"   # LAN only + basic auth (see .env)
+    environment:
+      PGWEB_AUTH_USER: ${PGWEB_AUTH_USER}
+      PGWEB_AUTH_PASS: ${PGWEB_AUTH_PASS}
+      PGWEB_BOOKMARKS_DIR: /bookmarks
+    volumes:
+      - ./pgweb-bookmarks:/bookmarks:ro   # bookmark .toml files (contain DB password, keep 0600)
+    depends_on:
+      timescaledb:
+        condition: service_healthy
+
+  pg-backup:
+    image: prodrigestivill/postgres-backup-local:latest   # latest = postgres 18 base (pg_dump 18.x)
+    container_name: pg-backup
+    restart: unless-stopped
+    environment:
+      POSTGRES_HOST: timescaledb
+      POSTGRES_DB: "hass scribe postgres"
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_EXTRA_OPTS: "-Fc"          # custom-format dumps (pg_restore)
+      SCHEDULE: "0 2 * * *"               # nightly 02:00 (TZ=Asia/Shanghai -> local 02:00)
+      BACKUP_ON_START: "TRUE"             # immediate backup on first start
+      BACKUP_SUFFIX: ".dump"
+      HEALTHCHECK_PORT: "80"              # go-cron health endpoint for the image healthcheck
+      TZ: "Asia/Shanghai"                 # match original host-cron 02:00 local (container default is UTC)
+    volumes:
+      - /opt/database/backups:/backups    # POSIX fs required; root disk, separate from data disk
+    depends_on:
+      timescaledb:
+        condition: service_healthy
@@ -61,11 +61,12 @@ from each section below.
 | **dns.windy.lan** | AdGuard Home LAN DNS + Mihomo explicit proxy — **PVE VM 120** (`pihole`) | `ssh -4 windy@192.168.66.36` | `192.168.66.36` | [hosts/dns.windy.lan.md](../hosts/dns.windy.lan.md) |
 | **ubnt** | UniFi Network Controller — **PVE VM 160** | `ssh -4 windy@192.168.66.46` | `192.168.66.46` | [hosts/ubnt.md](../hosts/ubnt.md) |
 | **hass.windy.lan** | Home Assistant (HAOS) — **x88 Pro physical box** (LAN55) | `ssh hassio@hass.windy.lan` | `192.168.55.11` | [hosts/hass.windy.lan.md](../hosts/hass.windy.lan.md) |
+| **pgdb** | TimescaleDB PG18 (Docker) — HA recorder 后端 — **PVE VM** (LAN55) | `ssh -4 windy@192.168.55.15` | `192.168.55.15` | [hosts/pgdb.md](../hosts/pgdb.md) |
 | **NAS/FreeNAS** | NAS; `transmission` jail runs here (`.51`) | — | — | — |
 | **U6 Lite** | UniFi AP (LAN66) | `ssh -4 zhiqiangf@192.168.66.6` | `192.168.66.6` | [docs/unifi-network.md](../docs/unifi-network.md) |
 | **UAP-AC-Lite** | UniFi AP (LAN55) | `ssh -4 zhiqiangf@192.168.55.5` | `192.168.55.5` | [docs/unifi-network.md](../docs/unifi-network.md) |

-> **Positioning facts:** `dns`/`ubnt`/`gfw` are VMs on PVE; `haos` is a **physical x88 Pro
+> **Positioning facts:** `dns`/`ubnt`/`gfw`/`pgdb` are VMs on PVE; `haos` is a **physical x88 Pro
 > box** (HAOS bare-metal, `machine: green`), not a PVE VM (corrected 2026-08-15).
 > `transmission` is a FreeNAS/NAS jail. Physical SE5420 ports: gw, PVE, haos, NAS,
 > U6, UAP-AC-Lite, and wired PCs. See [lan-se5420-deployment-guide.md](lan-se5420-deployment-guide.md) §1.
@@ -319,6 +319,35 @@ deployed 2026-08-18 from `216cc99` (backup
  reworked in v0.3.1/v0.3.2 to wait for a peer-initiated inbound SAS from
  Element with emoji comparison), see
  [docs/home-assistant-matrix.md § Device verification](../docs/home-assistant-matrix.md).
+### Scribe long-term history (verified 2026-08-29)
+
+- **Scribe 3.8.0** (`/homeassistant/custom_components/scribe/`), configured from
+  `configuration.yaml` (`scribe:` block; config entry
+  `01KC2VFJWEQ3XDHY6TQKHPDVRB`, `source: import` — UI "Configure → Advanced"
+  edits are overridden by the YAML on restart; treat YAML as authoritative).
+- TimescaleDB at `192.168.55.15:5432/scribe` (DB user `hass`; host in inventory,
+  see [hosts/pgdb.md](../hosts/pgdb.md)). Database re-initialized 2026-08-29 14:06 CST
+  (user-handled; earlier `relation "entities" does not exist` errors resolved).
+  Health: `binary_sensor.scribe_database_connection`.
+- 2026-08-29 config applied (backup `/homeassistant/configuration.yaml.bak-20260829-scribe`):
+  - `record_events: true` with `include_events` whitelist: `automation_triggered`,
+    `matrix_e2ee_command`, `matrix_e2ee_message_received`,
+    `matrix_e2ee_verification_done`, `script_started`, `tag_scanned`,
+    `mobile_app_notification_action`, `homeassistant_start`, `homeassistant_stop`.
+  - State noise trimmed: `exclude_domains` update/button; glob
+    `sensor.zigbee2mqtt_bridge_*`; 4 hassio cpu/mem-percent entities.
+  - Global `exclude_attributes` drops tianqi `hourly_*` arrays (~19 KB/state —
+    the recorder-side `_unrecorded_attributes` patch does not apply to Scribe).
+  - `enable_stats_io` + `enable_stats_size` on → 14 `sensor.scribe_*` stats
+    entities (`scribe_states_written`, `scribe_events_written`, rates, sizes).
+  - Verified post-restart 14:23 CST: writer started, `scribe_events_written=1`
+    (homeassistant_start), states ~110/min, buffer 3, no scribe log errors.
+- **Scribe 3.8.0 has no retention option** (upstream feature, newer versions
+  only). Retention requires upgrading Scribe first; the DB is a fresh 3.x-schema
+  so an upgrade is safe. Do not expect retention YAML keys to validate on 3.8.0.
+- Recorder stays external-Postgres with `purge_keep_days: 30` for daily UI
+  history; Scribe is the permanent archive.
+
 ## Known issues

 **Bluetooth hci0 instability — RTL8821CS (verified 2026-08-13, W1N-74):**
@@ -0,0 +1,51 @@
+# pgdb — TimescaleDB (PG18, Docker)
+
+## Role and access
+
+| Item | Value |
+|---|---|
+| Role | TimescaleDB PostgreSQL 18 (Docker) — Home Assistant recorder 后端（`hass`/`scribe` 库） |
+| IPv4 | `192.168.55.15` (LAN55) |
+| DNS | (none) |
+| SSH | `ssh -4 windy@192.168.55.15`（key auth 已验证可用 2026-08-29；agent 沙箱用 `ssh -F /dev/null -o BatchMode=yes`；password auth 亦可） |
+| Host | PVE 管理的 QEMU VM（i440FX），Debian 13 (trixie)，内核 6.12.105 |
+| Resources | 2 GB RAM / 30 GB disk（26 G 空闲） |
+| Docker | 29.7.2；容器 `timescaledb` = `timescale/timescaledb:latest-pg18`（PG **18.6** + TimescaleDB **2.29.2**，Apache-2.0 版） |
+| Ports | `192.168.55.15:5432`（PG，IPv4 only）；`192.168.55.15:8081`（pgweb GUI，basic auth） |
+
+## Databases
+
+| DB | Owner | Size | 用途 |
+|---|---|---|---|
+| `hass` | hass | ~14 MB | HA recorder（states/events/statistics），客户端 HAOS `192.168.55.11` |
+| `scribe` | postgres | ~11 MB | HA scribe 集成（entities/areas/devices 注册表同步 + `states_raw` hypertable） |
+| `postgres` | postgres | ~9 MB | 默认库 |
+
+## Ops notes
+
+- **Docker compose 管理**（2026-08-29 改造）：`/opt/database/docker-compose.yml`（源码在仓库 `compose/pgdb/`）+ `/opt/database/.env`（0600，密钥）+ `/opt/database/pgweb-bookmarks/`（0600，bookmark 含 DB 密码）。三个服务：
+  | 服务 | 镜像 | 端口 | 说明 |
+  |---|---|---|---|
+  | `timescaledb` | `timescale/timescaledb:latest-pg18` | `192.168.55.15:5432`（IPv4 only） | PG 18.6 + TS 2.29.2；healthcheck pg_isready；`restart: unless-stopped` |
+  | `pgweb` | `sosedoff/pgweb:latest`（v0.17.0） | `192.168.55.15:8081` | Web GUI：http://192.168.55.15:8081；basic auth（用户名/密码见 .env `PGWEB_AUTH_USER/PASS`）；`--readonly --sessions --bookmarks-only --bookmarks-dir /bookmarks`（v0.17.0 不读 PGWEB_BOOKMARKS_DIR env，必须用 flag）；bookmarks = hass/scribe |
+  | `pg-backup` | `prodrigestivill/postgres-backup-local:latest`（=PG18 客户端） | — | 每日 02:00（`TZ=Asia/Shanghai`，本地时区）`pg_dump -Fc` 三库 → `/opt/database/backups/{daily,weekly,monthly}`；保留 7 天/4 周/6 月；`BACKUP_ON_START` |
+- **数据盘**：`/dev/sdb1`（32G ext4，label `pgdata`）挂载 `/srv/pgdata`，fstab 按 `UUID=c9e12e79-1f66-404c-ab7f-b8809be81d86`（defaults,noatime）持久化（2026-08-29 迁移）。容器 bind mount `/srv/pgdata:/var/lib/postgresql`。
+- 容器内 postgres 用户 uid/gid = **70**（Debian 系，非 999）；迁移数据后需 `chown -R 70:70`。
+- **密码**：postgres 超级用户已换强密码（hex，存 `/opt/database/.env` 0600，2026-08-29）。HA 用 `hass` 角色不受影响。
+- **备份**：由 `pg-backup` 容器接管（2026-08-29），宿主机 cron 与 `/opt/database/pg-backup.sh` 已退役。恢复用 `pg_restore`（custom format）——2026-08-29 已实测还原 hass 库 dump（states 10014 行）成功。
+- **认证**：外部连接 scram-sha-256（密码必填，改密码有效）；容器内 loopback 为 trust（官方镜像默认）。
+- **回滚**：旧启动命令保留在 `/opt/database/run`（容器无状态，数据在 /srv/pgdata）；旧匿名卷 `9375195843b950f4e04c34872409ca095e1136520dd019a8e86e2794be06c236`（根盘 ~82M）保留作兜底，确认稳定后可 `docker volume rm`。
+- 本机无防火墙（ufw/nft/iptables 均未装）——待办：如要彻底隔离可加 ufw 白名单 192.168.55.11。
+- `/opt/database/backups/` 根下残留 `*-2026-08-29_1359.dump`（compose 化之前旧备份机制产物）与 `backup.log`——健康检查只看 `daily/`，残留可清理。
+- **Runbooks**：[pgdb-health](../runbooks/pgdb-health.md)（只读健康检查）、[pgdb-restore](../runbooks/pgdb-restore.md)（pg_restore 还原）、[pgdb-update](../runbooks/pgdb-update.md)（镜像/compose 升级）。
+
+## Known issues
+
+- 2026-08-29：HA 侧 HACS 集成 `custom_components.scribe`（YAML `scribe: db_url:`，连 `scribe` 库）建表被拒（`permission denied for schema public`，hass 无 CREATE 权限），之后持续报 `relation "entities" does not exist`。**已解决**：① `GRANT CREATE ON SCHEMA public TO hass;`（scribe 库）② 重启 HA Core 触发重跑建表。重启后自动创建 `entities`（1591 行）/`users`/`areas`/`devices`/`integrations`/`states_raw` 表并启用 TimescaleDB 时间序列能力。报错已停止（最后一条 06:06 UTC），`states_raw` 持续写入。2026-08-29 复查：scribe 现有**两个** hypertable——`states_raw`（segmentby `metadata_id`、orderby `time`）与 `events`（segmentby `event_type`、orderby `time`），均 1 维 `time`；压缩已配置（`timescaledb_information.compression_settings` 可见对应行；2.29.x 该视图无 `compression_enabled` 列）。
+- `hass` 库的 recorder 表仍为普通表（无 hypertable）；`scribe` 集成负责时间序列历史（`states_raw` + `events` hypertable）。
+
+## Verification history
+
+- 2026-08-29：首次检查（只读）+ 修复 scribe 权限 + 安装夜间备份。见 Linear vps 项目登记。
+- 2026-08-29：**compose 改造完成**（W1N-227，用户已验收）：裸 `docker run` → `/opt/database/docker-compose.yml` 三服务（timescaledb + pgweb + pg-backup）；superuser 换强密码；端口收紧 IPv4；备份容器化（TZ=Asia/Shanghai，cron 02:00 本地）；`pg_restore` 还原实测通过；pgweb UI 用户确认可查 hass/scribe 数据。源码在仓库 `compose/pgdb/`。
+- 2026-08-29：**运维 runbook 落地**（W1N-228，已验收）：新增 `runbooks/pgdb-health.md`（只读，8 项诊断全绿）、`pgdb-restore.md`（流程式，temp-DB 安全还原 + 审批门）、`pgdb-update.md`（门控命令式，回滚=/opt/database/run + 旧卷）；README 索引与 validate-repo.sh 分类同步更新；runbook 命令已对活主机逐条实测（含 `pg_restore -l` 校验当日 dump）。同日修正：SSH key auth 可用（facts 原记"密钥未安装"已过时）；scribe 新增 `events` hypertable。
@@ -32,6 +32,7 @@ the software deployed there, see [the LAN overview](../docs/lan-overview.md).
 | **gw** | EdgeRouter X primary LAN gateway | `ssh -4 zhiqiang@192.168.66.254` | `192.168.66.254` | — (EdgeOS, no ansible) | **active** | [hosts/gw.md](../hosts/gw.md) |
 | **ubnt** | UniFi Network Controller (PVE VM 160) | `ssh -4 windy@192.168.66.46` | `192.168.66.46` | ✓ (ubnt) | **active** | [hosts/ubnt.md](../hosts/ubnt.md) |
 | **hass.windy.lan** | Home Assistant (HAOS, x88 Pro physical box, LAN55) | `ssh hassio@hass.windy.lan` | `192.168.55.11` | — (HAOS, no ansible) | **active** | [hosts/hass.windy.lan.md](../hosts/hass.windy.lan.md) |
+| **pgdb** | TimescaleDB PG18 (Docker) — HA recorder backend (PVE VM, LAN55) | `ssh -4 windy@192.168.55.15` | `192.168.55.15` | — (no ansible) | **active** | [hosts/pgdb.md](../hosts/pgdb.md) |

 `status: stub` = known to exist; fill `hosts/<name>.md` when next touched.

@@ -16,6 +16,9 @@ runbooks start from [`_template.md`](_template.md).
 | PowerDNS health check | [pdns-health.md](pdns-health.md) | read-only |
 | RustDesk health check | [rustdesk-health.md](rustdesk-health.md) | read-only |
 | Matrix health check | [matrix-health.md](matrix-health.md) | read-only |
+| pgdb health check | [pgdb-health.md](pgdb-health.md) | read-only |
+| pgdb DB restore (pg_restore) | [pgdb-restore.md](pgdb-restore.md) | change (procedure) |
+| pgdb image/compose update | [pgdb-update.md](pgdb-update.md) | change (gated) |
 | AdGuard Home health check | [adguard-home-health.md](adguard-home-health.md) | read-only |
 | Matter packet capture | [matter-packet-capture.md](matter-packet-capture.md) | read-only |
 | Home Assistant maintenance | [home-assistant-maintenance.md](home-assistant-maintenance.md) | change (gated) |
@@ -0,0 +1,181 @@
+# Runbook: pgdb health (TimescaleDB + pgweb + pg-backup)
+
+## Purpose
+
+Read-only health check of the pgdb TimescaleDB compose stack (PG18 + pgweb GUI + nightly custom-format backups). Confirms the stack is serving Home Assistant (hass/scribe) and that backups are current and restorable.
+
+## Scope
+
+- Applicable: [pgdb](../hosts/pgdb.md) (`192.168.55.15`), `/opt/database` compose stack.
+- Read-only: never mutates containers, databases, backups, or secrets.
+- Not applicable: restoring data (use [pgdb-restore](pgdb-restore.md)), upgrading images (use [pgdb-update](pgdb-update.md)), HA-side changes (see `hosts/hass.windy.lan.md`).
+
+## Ownership
+
+- Owner: personal ops (Windy)
+- Last reviewed: 2026-08-29
+- Related systems: pgdb (`/opt/database`, TimescaleDB 18.6 / TS 2.29.2), HA `192.168.55.11` (hass/scribe clients)
+
+## Access
+
+SSH to pgdb (key auth works from the WSL agent shell as of 2026-08-29):
+
+```bash
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15
+```
+
+From the agent sandbox, always use `-F /dev/null` (system ssh config is unreadable there) and prefer IPv4. The compose project lives at `/opt/database` — prefix every `docker compose` call with `cd /opt/database`. Never print `.env` values or bookmarks (they contain DB passwords); compare or use them only inside commands that output non-secret signals (status codes, counts, names).
+
+## Safety
+
+### Non-negotiable rules
+
+- Read-only diagnosis only; never "fix while checking".
+- Never print passwords or secrets — redact/consume them inside commands.
+- If live state conflicts with an expected value, `STOP` and record evidence; do not invent parameters or bypass a failed check.
+- Restore/update work belongs to the change runbooks, not this one.
+
+### Stop conditions
+
+- Any container `Exited`, `Restarting`, or not `healthy` where expected.
+- Expected database/table/hypertable missing or a key query errors.
+- Write-activity sample does not increase (scribe `states_raw` static).
+- Latest daily backup older than today, not custom format, or `pg_restore -l` fails.
+- Disk usage near full on `/srv/pgdata` or `/`.
+- New `ERROR`/`FATAL` lines in the timescaledb log or backup failures in the pg-backup log.
+
+## Pass criteria
+
+- `docker compose ps -a`: `timescaledb` + `pg-backup` **Up (healthy)**, `pgweb` **Up**; ports bound to `192.168.55.15:5432` and `:8081`.
+- PG 18.x; databases `hass`, `scribe`, `postgres` present; HA (`192.168.55.11`) connected as `hass` to both `hass` and `scribe`.
+- `hass.states` and `scribe.states_raw` row counts grow between two samples (scribe writes continuously).
+- TimescaleDB extension 2.29.x; scribe hypertables `states_raw` + `events` (1-dim, `time`); compression configured (segmentby/orderby rows in `timescaledb_information.compression_settings`); `entities` table exists.
+- pgweb: no credentials → HTTP 401; with credentials → HTTP 200; `/api/bookmarks` → `["hass","scribe"]`.
+- `daily/*-latest.dump` symlinks point to today's dumps; `file -L` reports `PostgreSQL custom database dump`.
+- `/srv/pgdata` (`/dev/sdb1`, 32G) and `/` not near full; fstab mounts `/srv/pgdata` by `UUID=c9e12e79-1f66-404c-ab7f-b8809be81d86` with `defaults,noatime`.
+- timescaledb log: no new `ERROR`/`FATAL`; pg-backup log: recent successful backup.
+
+## Checks
+
+### 1. Containers
+
+```bash
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose ps -a --format "table {{.Name}}\t{{.Status}}\t{{.Ports}}"'
+```
+
+**Expected**
+
+- `timescaledb` Up (healthy), `pg-backup` Up (healthy), `pgweb` Up.
+- Ports: `192.168.55.15:5432->5432/tcp` (timescaledb), `192.168.55.15:8081->8081/tcp` (pgweb).
+
+**Stop** if any container is `Exited`/`Restarting`/`unhealthy`, or a port binding changed.
+
+### 2. PG core and HA clients
+
+```bash
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose exec -T timescaledb psql -U postgres -Atc "select version();" | head -1'
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose exec -T timescaledb psql -U postgres -Atc "select datname from pg_database where datistemplate=false order by 1;"'
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose exec -T timescaledb psql -U postgres -Atc "select datname, usename, client_addr from pg_stat_activity where client_addr is not null group by 1,2,3 order by 1;"'
+```
+
+**Expected**
+
+- `PostgreSQL 18.x` (verified: 18.6).
+- Databases: `hass`, `postgres`, `scribe`.
+- HA sessions: `hass|hass|192.168.55.11` and `scribe|hass|192.168.55.11` (the HAOS recorder/scribe clients from `192.168.55.11`).
+
+**Stop** if a database is missing, the version is not 18.x, or HA has no live sessions (scribe connectivity is part of the HA pipeline).
+
+### 3. Write activity
+
+```bash
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database
+A=$(docker compose exec -T timescaledb psql -U postgres -d scribe -Atc "select count(*) from states_raw;")
+sleep 30
+B=$(docker compose exec -T timescaledb psql -U postgres -d scribe -Atc "select count(*) from states_raw;")
+echo "states_raw $A -> $B"'
+```
+
+Also sample `hass.states` once (recorder table, bulk-writes on HA restart): `docker compose exec -T timescaledb psql -U postgres -d hass -Atc "select count(*) from states;"`.
+
+**Expected** — `states_raw` increases between samples (verified: 2665 → 2693 in 30 s). `states` count is sane (thousands).
+
+**Stop** if `states_raw` is static across samples while HA is up — writes have stalled.
+
+### 4. TimescaleDB (extension, hypertables, compression)
+
+```bash
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose exec -T timescaledb psql -U postgres -Atc "select extversion from pg_extension where extname='"'"'timescaledb'"'"';"'
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose exec -T timescaledb psql -U postgres -d scribe -Atc "select hypertable_name, num_dimensions from timescaledb_information.hypertables order by 1;"'
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose exec -T timescaledb psql -U postgres -d scribe -Atc "select hypertable_name, attname, segmentby_column_index, orderby_column_index from timescaledb_information.compression_settings order by 1,3,4;"'
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose exec -T timescaledb psql -U postgres -d scribe -Atc "select to_regclass('"'"'public.entities'"'"');"'
+```
+
+**Expected**
+
+- Extension version `2.29.x` (verified: 2.29.2).
+- Scribe hypertables: `states_raw` and `events`, both `1` dimension.
+- Compression configured for `states_raw` (segmentby `metadata_id` idx 1, orderby `time` idx 1) and `events` (segmentby `event_type`, orderby `time`). Note: TimescaleDB 2.29.x has **no** `compression_enabled` column in this view — row presence is the enabled signal.
+- `entities` resolves (scribe registry table).
+
+**Stop** if the extension version differs from the pinned 2.29.x line, a hypertable is missing, compression rows vanish, or `entities` is absent (scribe schema broke — see Known issues in [hosts/pgdb.md](../hosts/pgdb.md)).
+
+### 5. pgweb GUI
+
+```bash
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database
+curl -s -o /dev/null -w "no-auth:%{http_code}\n" --max-time 8 http://192.168.55.15:8081/
+U=$(grep -E "^PGWEB_AUTH_USER=" .env | cut -d= -f2-); P=$(grep -E "^PGWEB_AUTH_PASS=" .env | cut -d= -f2-)
+curl -s -o /dev/null -w "auth:%{http_code}\n" --max-time 8 -u "$U:$P" http://192.168.55.15:8081/
+echo -n "bookmarks:"; curl -s --max-time 8 -u "$U:$P" http://192.168.55.15:8081/api/bookmarks; echo'
+```
+
+Use `192.168.55.15:8081` (pgweb binds the VM IP only — loopback is not bound). Credentials are read from `.env` on the host and never printed.
+
+**Expected** — `no-auth:401`, `auth:200`, `bookmarks:["hass","scribe"]`.
+
+**Stop** if pgweb is unreachable, unauthenticated access is not 401, or bookmarks diverge from `["hass","scribe"]`.
+
+### 6. Backups
+
+```bash
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'ls -l --time-style=long-iso /opt/database/backups/daily/'
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'file -L /opt/database/backups/daily/hass-latest.dump'
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose exec -T pg-backup pg_restore -l /backups/daily/hass-latest.dump | head -4'
+```
+
+**Expected**
+
+- `daily/*-latest.dump` symlinks point to **today's** `*-YYYYMMDD.dump` (nightly 02:00 local `Asia/Shanghai`; a fresh container also fires `BACKUP_ON_START`).
+- `file -L` reports `PostgreSQL custom database dump` (pg_restore format; verified v1.16-0).
+- `pg_restore -l` from the **pg-backup** container lists the archive TOC without error (timescaledb does not mount `/backups`).
+
+**Stop** if the latest dump is not from today, is not custom format, or `pg_restore -l` fails. Stray non-`daily/` dumps at the `/opt/database/backups/` root are pre-compose leftovers — ignore for health, flag for cleanup.
+
+### 7. Disk and mount
+
+```bash
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'df -h /srv/pgdata /'
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'grep -E "srv/pgdata" /etc/fstab'
+```
+
+**Expected**
+
+- `/srv/pgdata` = `/dev/sdb1` 32G (verified: 88M used / 30G avail) and `/` with comfortable headroom.
+- fstab: `UUID=c9e12e79-1f66-404c-ab7f-b8809be81d86 /srv/pgdata ext4 defaults,noatime 0 2`.
+
+**Stop** if either filesystem is near full (define threshold before acting) or the fstab entry is missing/changed.
+
+### 8. Logs
+
+```bash
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose logs --since 24h timescaledb 2>&1 | grep -E "ERROR|FATAL" | tail -10'
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose logs --since 24h pg-backup 2>&1 | tail -5'
+```
+
+**Expected**
+
+- timescaledb: no new `ERROR`/`FATAL`. Known benign history: an old `relation "hass.states" does not exist` from a wrong-schema probe and `compression_enabled` column errors from an outdated query — neither recurs with the commands above.
+- pg-backup: recent successful run (`SQL backup created successfully` for each database, no restore/cleanup errors).
+
+**Stop** if repeated `ERROR`/`FATAL` appear or a backup run failed.
@@ -0,0 +1,181 @@
+# Runbook: pgdb database restore (pg_restore, custom format)
+
+## Purpose
+
+Restore one pgdb database (`hass`, `scribe`, or `postgres`) from a pg-backup custom-format dump via `pg_restore`. The dump → restore path was exercised on 2026-08-29 (hass restore test, `states` 10014 rows); this runbook makes it repeatable and safe.
+
+## Scope
+
+- Applicable: single-database restore on [pgdb](../hosts/pgdb.md) (`192.168.55.15`) from `/opt/database/backups/{daily,weekly,monthly}/*.dump`.
+- Not applicable: full-stack/VM recovery, HAOS-side changes, secret rotation, or restoring to a different host (adjust `-h` and auth first).
+
+## Ownership
+
+- Owner: personal ops (Windy)
+- Last reviewed: 2026-08-29
+- Related systems: pgdb (`/opt/database`), HA `192.168.55.11` (hass/scribe clients), [pgdb-health](pgdb-health.md)
+
+## Preconditions
+
+- Health baseline recorded ([pgdb-health](pgdb-health.md)): stack healthy, current backup is from today and passes `pg_restore -l`.
+- A valid dump exists for the target database (see Step 1).
+- Explicit user approval obtained for the chosen restore mode (see `## Approval gates`) and recorded in the Linear `vps` project.
+- Disk headroom on `/srv/pgdata` for a second copy of the database (temp-DB mode).
+
+## Inputs
+
+| Input | Source | Required | Validation |
+|---|---:|---|---|
+| Target database (`hass`/`scribe`/`postgres`) | [hosts/pgdb.md](../hosts/pgdb.md) | yes | exists in `pg_database` |
+| Dump path (inside pg-backup, `/backups/...`) | `ls /opt/database/backups/daily/` | yes | `pg_restore -l` lists TOC |
+| Restore mode (temp-DB swap recommended / in-place `--clean`) | operator + user | yes | approval record |
+| Downtime window | user | yes (for in-place) | user-confirmed |
+
+## Safety
+
+### Non-negotiable rules
+
+- Never delete the existing configuration or the last known-good dump as the first recovery action.
+- Prefer read-only diagnosis before any mutation; the restore itself is the only mutation.
+- Never print passwords — use `PGPASSWORD="$POSTGRES_PASSWORD"` inside the pg-backup container.
+- If actual state conflicts with this runbook (e.g. dump corrupt, DB missing), `STOP`.
+- Do not invent missing parameters; do not bypass failed checks or approval.
+
+### Stop conditions
+
+- Dump fails `pg_restore -l` (corrupt/truncated) — do not restore from it.
+- Target DB contains data newer than the dump and the operator did not accept the data-loss window.
+- Disk space insufficient for the temp copy.
+- Approval missing or stale (recorded in Linear).
+- Post-restore verification fails — do not declare success.
+
+### Approval gates
+
+| Action | Risk | Explicit approval | Recorded at |
+|---|---|---|---|
+| Create temp DB + `pg_restore` into it (no data loss) | Low | No (executor can run after plan review) | Linear vps issue |
+| Drop/rename the live target DB (temp-DB swap) | High | Yes — user confirmation | Linear vps issue |
+| In-place restore with `--clean --if-exists` | High | Yes — user confirmation | Linear vps issue |
+
+## Procedure
+
+### Step 1 — Diagnose
+
+**Action**
+
+```bash
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'ls -l --time-style=long-iso /opt/database/backups/daily/'
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose exec -T pg-backup sh -c "PGPASSWORD=\"\$POSTGRES_PASSWORD\" pg_restore -h timescaledb -U postgres -l /backups/daily/<db>-latest.dump" | head -6'
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose exec -T timescaledb psql -U postgres -Atc "select datname, pg_size_pretty(pg_database_size(datname)) from pg_database where datistemplate=false order by 1;"'
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'df -h /srv/pgdata'
+```
+
+**Expected**
+
+- Dump exists for today and is custom format (`file -L` → `PostgreSQL custom database dump`).
+- `pg_restore -l` lists the archive TOC (dbname matches the target) without error.
+- Target DB exists; disk has headroom for a second copy.
+
+**Decision**
+
+- Dump valid + DB present → Step 2.
+- Dump invalid or target missing → `STOP`; record evidence, do not restore.
+
+### Step 2 — Approve
+
+**Action** — Present the plan: target DB, dump (path + creation time), restore mode (temp-DB swap recommended), estimated write downtime (none for temp mode; brief for in-place), and rollback plan. Obtain explicit user approval and record it in the Linear `vps` project issue for this work.
+
+**Expected** — User approval recorded with the chosen mode.
+
+**Decision**
+
+- Approved → Step 3.
+- Not approved / mode changed → `STOP` and report; re-plan if needed.
+
+### Step 3 — Restore into a temp database (recommended, no data loss)
+
+**Action**
+
+```bash
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database
+docker compose exec -T timescaledb psql -U postgres -c "CREATE DATABASE <db>_restore OWNER <owner>;"
+docker compose exec -T pg-backup sh -c "PGPASSWORD=\"\$POSTGRES_PASSWORD\" pg_restore -h timescaledb -U postgres -d <db>_restore /backups/daily/<db>-latest.dump"
+docker compose exec -T timescaledb psql -U postgres -Atc "select count(*) from pg_database where datname='"'"'<db>_restore'"'"';"'
+```
+
+`<owner>` is the database owner from [hosts/pgdb.md](../hosts/pgdb.md) (`hass` → `hass`, `scribe`/`postgres` → `postgres`). Run inside the pg-backup container so the password stays server-side; `-h timescaledb` routes over the compose network.
+
+**Expected** — `CREATE DATABASE` succeeds; `pg_restore` exits 0 with no error lines; the temp DB exists.
+
+**Decision**
+
+- Restore OK → Step 4.
+- Restore errors → diagnose (dump integrity, permissions), fix, or `STOP`; the live DB is untouched in this mode.
+
+**Verification**
+
+```bash
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose exec -T timescaledb psql -U postgres -d <db>_restore -Atc "select count(*) from <key_table>;"'
+```
+
+Compare against the pre-restore baseline (e.g. `states` ≈ dump's row count). Row counts match → proceed to swap; mismatch → `STOP` and inspect.
+
+### Step 4 — Swap temp DB into place (approval-gated)
+
+**Action** — After user approval: drop the live DB and rename the temp (or `pg_dump` the live DB as a safety copy first).
+
+```bash
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database
+docker compose exec -T timescaledb psql -U postgres -c "DROP DATABASE <db>;"
+docker compose exec -T timescaledb psql -U postgres -c "ALTER DATABASE <db>_restore RENAME TO <db>;"'
+```
+
+**Expected** — Rename succeeds; `pg_database` shows `<db>` with the restored size.
+
+**Decision**
+
+- Swap OK → Step 5.
+- Swap fails → `STOP`; see Rollback.
+
+**Verification**
+
+- `select count(*)` on key tables matches the dump.
+- HA sessions re-establish: `pg_stat_activity` shows `hass|hass|192.168.55.11` on `<db>` (scribe writes resume).
+- pgweb bookmarks show restored data (read-only browse).
+
+**Rollback**
+
+- Trigger: verification fails or swap is wrong.
+- Action: restore again from the dump taken before the operation; if the live DB was dropped, recreate `<db>` and restore the pre-change dump (`--clean` not needed on an empty DB).
+- Verify: row counts + HA sessions match pre-change baseline; record in Linear.
+
+### In-place variant (only with explicit approval)
+
+```bash
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose exec -T pg-backup sh -c "PGPASSWORD=\"\$POSTGRES_PASSWORD\" pg_restore -h timescaledb -U postgres --clean --if-exists -d <db> /backups/daily/<db>-latest.dump"'
+```
+
+Destructive to current data — requires the High-risk approval gate and a fresh dump of the current state taken immediately before. Follow Steps 4–5 verification afterwards.
+
+## Final Verification
+
+Success requires all of:
+
+- Target DB row counts match the dump (`states`, `states_raw`, `events` etc. sane).
+- HA (`192.168.55.11`) reconnects and writes resume (write-activity sample grows).
+- [pgdb-health](pgdb-health.md) passes (containers, backups still current, logs clean).
+- Outcome recorded in [hosts/pgdb.md](../hosts/pgdb.md) and the Linear `vps` project.
+
+## Failure Handling
+
+1. Stop further changes; do not guess.
+2. Collect: dump path + `pg_restore`/`psql` output, DB sizes, log tail.
+3. Record completed steps, actual vs expected, and whether rollback ran.
+4. Escalate to the user with evidence; continue only with explicit direction.
+
+## References
+
+- [hosts/pgdb.md](../hosts/pgdb.md) — roles, owners, backup layout, verification history
+- [pgdb-health](pgdb-health.md) — pre/post health baseline
+- [pgdb-update](pgdb-update.md) — image/compose upgrades
+- Linear `vps`: W1N-226 (first check), W1N-227 (compose-化 + restore test), W1N-228 (this runbook)
@@ -0,0 +1,77 @@
+# Runbook: pgdb update (image / compose upgrade)
+
+## Purpose
+
+Upgrade the pgdb compose stack (`timescaledb` + `pgweb` + `pg-backup`) to newer images with gated commands and verification. Data lives on the `/srv/pgdata` volume and survives container recreation.
+
+## Scope
+
+- Applicable: [pgdb](../hosts/pgdb.md) (`192.168.55.15`), `/opt/database/docker-compose.yml` (source: `compose/pgdb/compose.yml` in this repo).
+- Not applicable: config/secret changes, `.env` rotation, DB migration, or pgweb bookmark changes.
+
+## Ownership
+
+- Owner: personal ops (Windy)
+- Last reviewed: 2026-08-29
+- Related systems: pgdb (`/opt/database`), HA `192.168.55.11`, [pgdb-health](pgdb-health.md)
+
+## Approval gates
+
+| Action | Risk | Explicit approval |
+|---|---|---|
+| `docker compose pull` + `up -d` (recreates changed containers; timescaledb recreate = brief write interruption for HA) | Medium | Yes — user confirmation required |
+
+## Safety Rules
+
+- Never delete an existing configuration as the first recovery action.
+- Prefer read-only diagnosis before mutation.
+- After every mutation, verify the expected state.
+- If actual state conflicts with this runbook, `STOP`.
+- Do not invent missing parameters (e.g. new env vars required by a major image bump).
+- Do not bypass failed tests.
+- Destructive actions require explicit approval.
+
+## Before
+
+1. Record a health baseline: run [pgdb-health](pgdb-health.md). Note current versions (`select version()` → PG 18.x, TimescaleDB 2.29.x) and image digests:
+   ```bash
+   ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker inspect --format "{{.Name}} {{.Config.Image}} {{.Image}}" timescaledb pgweb pg-backup'
+   ```
+2. Confirm parity between the repo source and the host compose file:
+   ```bash
+   diff compose/pgdb/compose.yml <(ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cat /opt/database/docker-compose.yml')
+   ```
+3. Check disk: `df -h /srv/pgdata /` (new images need pull space on `/`).
+4. Confirm the intended change with the user (approval gate above).
+
+## Update
+
+```bash
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose pull'
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose config -q && echo CONFIG_OK'
+ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose up -d'
+```
+
+- `compose config -q` fails fast on invalid compose syntax — `STOP` there, do not `up`.
+- `up -d` recreates only containers whose image/config changed. Recreating `pg-backup` triggers an immediate `BACKUP_ON_START` dump (expected, keep it).
+- HA (`192.168.55.11`) recorder reconnects automatically after a brief write interruption; do not restart HA unless it fails to recover.
+
+## After
+
+1. Re-run [pgdb-health](pgdb-health.md). Confirm:
+   - Containers `Up (healthy)`; ports still `192.168.55.15:5432` / `:8081`.
+   - `select version()` still PG 18.x; TimescaleDB extension version still the pinned 2.29.x line (image `latest-pg18` should not jump major/minor unexpectedly — if it does, `STOP` and review before accepting).
+   - Write activity resumes (`states_raw` grows); HA sessions from `192.168.55.11` present.
+   - A fresh backup exists (today) and passes `pg_restore -l`.
+2. Update **Verified** on [hosts/pgdb.md](../hosts/pgdb.md) with date, new versions, and outcome.
+
+## Rollback
+
+- Trigger: health check fails after the update, or the new image misbehaves.
+- Action (data-safe — `/srv/pgdata` volume is untouched):
+  1. Pin the previous image tags in `/opt/database/docker-compose.yml` (from the `docker inspect` baseline above) and sync the repo source if changed:
+     ```bash
+     ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose up -d --force-recreate'
+     ```
+  2. Last resort: the pre-compose launcher `/opt/database/run` (legacy `docker run` command) with the same `/srv/pgdata` mount and the old anonymous volume as fallback — see [hosts/pgdb.md](../hosts/pgdb.md).
+- Verify: [pgdb-health](pgdb-health.md) passes and versions match the pre-update baseline.
@@ -138,8 +138,8 @@ fi

 # ---------------------------------------------------------------------------
 say "== 4. Runbook spec compliance (RUNBOOKS.md) =="
-PROCEDURE_RUNBOOKS="fix-ci.md issue-to-merge.md network-change.md network-recovery.md release.md rollback.md"
-GATED_REF_RUNBOOKS="mailcow-update.md ansible-operations.md home-assistant-maintenance.md matrix-e2ee-update.md vaultwarden-sqlite-to-postgres.md"
+PROCEDURE_RUNBOOKS="fix-ci.md issue-to-merge.md network-change.md network-recovery.md pgdb-restore.md release.md rollback.md"
+GATED_REF_RUNBOOKS="mailcow-update.md ansible-operations.md home-assistant-maintenance.md matrix-e2ee-update.md pgdb-update.md vaultwarden-sqlite-to-postgres.md"
 RUNBOOK_INDEX="runbooks/README.md"
 for f in runbooks/*.md; do
  base="$(basename "$f")"