VPS-78: CSG day 费用口径统一 + 断档自愈 (v5) + v3/v5 迁移脚本入库
- compose/pgdb/csg-snapshot-v3.sql: 补提交(此前只存在于工作区,未入 git)
- compose/pgdb/csg-snapshot-v5.sql: 新增
· csg_ladder_cost_raw() 无舍入阶梯费用助手
· csg_daily_snapshot() v5: day 费用改累积边际差分 + round(...,2);
p_month 上移 + 跨月守卫;日期与用量同源 latest_day_kwh
· 一次性归一历史 day cost(UPDATE 3 行,总差 -0.01)
· csg_backfill_missing_days() + job 1011(insert-only 断档自愈)
- runbooks/pgdb-health.md: 新增 check 9 CSG 归档新鲜度
- hosts/pgdb.md: v5 事实 + Known issues 2026-09-22
This commit is contained in:
@@ -0,0 +1,248 @@
|
|||||||
|
-- CSG 长期归档修复 (2026-09-21)
|
||||||
|
-- 背景:TimescaleDB 任务 1008 `csg_daily_snapshot()` 自 2026-08-29 创建起 0 成功 / 294 失败。
|
||||||
|
-- 原因:函数签名是零参数 `csg_daily_snapshot()`,而 TimescaleDB 自定义 job 动作
|
||||||
|
-- 按名字调用 `schema.proc(job_id integer, config jsonb)`,解析不到 → 每次报
|
||||||
|
-- `function or procedure "public.csg_daily_snapshot(integer, jsonb)" does not exist`。
|
||||||
|
-- 后果:csg_history 的 day 行停在 2026-08-28、month 行停在 2026-08-01(且 2026-08 的
|
||||||
|
-- month 行是 08-29 当时的「本月至今」302.47 kWh / 180.28 元,不是月终值 331.22 / 198.65)。
|
||||||
|
--
|
||||||
|
-- 本文件的三个动作(幂等,可重复执行):
|
||||||
|
-- 1. `csg_ladder_cost(kwh, month)` —— 阶梯电价助手(广州:夏季 5-10 月 260/600,
|
||||||
|
-- 非夏季 200/400;0.589 / 0.639 / 0.889 元每 kWh)。归档侧的唯一计价来源。
|
||||||
|
-- 2. `csg_daily_snapshot(job_id integer, config jsonb)` v3 —— 兼容 TimescaleDB job 签名;
|
||||||
|
-- 并把 month 行按「数据日期 d 所属月份」归属:
|
||||||
|
-- · d 属于当前月 → usage 取集成本月累计,cost 用阶梯助手重算
|
||||||
|
-- · d 属于上月 → usage 由该月 day 行汇总(不读集成 last_month,避开翻月瞬态:
|
||||||
|
-- 2026-08-31 16:32 实测 last_month_total_usage 曾读到 323.49 这种误值)
|
||||||
|
-- 并加「不降级」保护:day 行汇总结果小于已记录的 month 行时不动。
|
||||||
|
-- 3. 回填 2026-08-29 → 2026-09-19 的 day 行(数据源:08-29..08-31 取 scribe
|
||||||
|
-- states_raw 里 `yesterday_kwh` 观测值减一天;09-01..09-19 取 CSG 集成
|
||||||
|
-- `this_month_by_day`,其中 09-06 = 9.29 是 scribe 漏采的观测)。日费用一律用
|
||||||
|
-- 「当日用电 × 当日所处的边际档位」计算,使整月 day 费用之和 == 阶梯月费用。
|
||||||
|
--
|
||||||
|
-- 回滚(脚本内已自动建快照表 csg_history_bak_20260921):
|
||||||
|
-- BEGIN;
|
||||||
|
-- SELECT delete_job((SELECT job_id FROM timescaledb_information.jobs WHERE proc_name='csg_daily_snapshot'));
|
||||||
|
-- DROP FUNCTION IF EXISTS public.csg_daily_snapshot(integer, jsonb);
|
||||||
|
-- CREATE FUNCTION public.csg_daily_snapshot() RETURNS void LANGUAGE plpgsql AS $f$ ...原函数体... $f$;
|
||||||
|
-- SELECT add_job('public.csg_daily_snapshot'::regproc, INTERVAL '1 day');
|
||||||
|
-- DROP FUNCTION IF EXISTS public.csg_ladder_cost(numeric, date);
|
||||||
|
-- DELETE FROM csg_history; INSERT INTO csg_history SELECT * FROM csg_history_bak_20260921;
|
||||||
|
-- COMMIT;
|
||||||
|
-- (或整体放弃归档:DROP TABLE csg_history + 删任务,见 hosts/pgdb.md 回滚段)
|
||||||
|
|
||||||
|
\set ON_ERROR_STOP on
|
||||||
|
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
-- ---------- 0. 回滚快照(只建一次) ----------
|
||||||
|
CREATE TABLE IF NOT EXISTS public.csg_history_bak_20260921 AS TABLE public.csg_history;
|
||||||
|
|
||||||
|
-- ---------- 1. 阶梯电价助手 ----------
|
||||||
|
CREATE OR REPLACE FUNCTION public.csg_ladder_cost(kwh numeric, month_date date)
|
||||||
|
RETURNS numeric
|
||||||
|
LANGUAGE sql
|
||||||
|
IMMUTABLE
|
||||||
|
AS $function$
|
||||||
|
SELECT CASE
|
||||||
|
WHEN kwh IS NULL OR kwh <= 0 THEN 0::numeric
|
||||||
|
WHEN kwh <= (CASE WHEN extract(month FROM month_date) BETWEEN 5 AND 10 THEN 260 ELSE 200 END)
|
||||||
|
THEN round(kwh * 0.589, 2)
|
||||||
|
WHEN kwh <= (CASE WHEN extract(month FROM month_date) BETWEEN 5 AND 10 THEN 600 ELSE 400 END)
|
||||||
|
THEN round((CASE WHEN extract(month FROM month_date) BETWEEN 5 AND 10 THEN 260 ELSE 200 END) * 0.589
|
||||||
|
+ (kwh - (CASE WHEN extract(month FROM month_date) BETWEEN 5 AND 10 THEN 260 ELSE 200 END)) * 0.639, 2)
|
||||||
|
ELSE round((CASE WHEN extract(month FROM month_date) BETWEEN 5 AND 10 THEN 260 ELSE 200 END) * 0.589
|
||||||
|
+ ((CASE WHEN extract(month FROM month_date) BETWEEN 5 AND 10 THEN 600 ELSE 400 END)
|
||||||
|
- (CASE WHEN extract(month FROM month_date) BETWEEN 5 AND 10 THEN 260 ELSE 200 END)) * 0.639
|
||||||
|
+ (kwh - (CASE WHEN extract(month FROM month_date) BETWEEN 5 AND 10 THEN 600 ELSE 400 END)) * 0.889, 2)
|
||||||
|
END;
|
||||||
|
$function$;
|
||||||
|
|
||||||
|
-- ---------- 2. 重建为 TimescaleDB 兼容签名的 v3 + 重挂 job ----------
|
||||||
|
-- 注意:不能直接 DROP 旧函数——TimescaleDB job 1008 持有依赖
|
||||||
|
-- (ERROR: cannot drop public.csg_daily_snapshot because background job 1008 depends on it)。
|
||||||
|
-- 因此:delete_job → 换签名重建函数 → add_job 重挂,并把排程对齐文档口径
|
||||||
|
-- (22:30 Asia/Shanghai = 14:30 UTC;原来实际跑在 22:00 UTC = 06:00 CST,
|
||||||
|
-- 那个时点集成还没发布前一日数据,d 会晚一天)。
|
||||||
|
DO $do$
|
||||||
|
BEGIN
|
||||||
|
IF EXISTS (SELECT 1 FROM timescaledb_information.jobs WHERE job_id = 1008) THEN
|
||||||
|
PERFORM delete_job(1008);
|
||||||
|
END IF;
|
||||||
|
END
|
||||||
|
$do$;
|
||||||
|
|
||||||
|
DROP FUNCTION IF EXISTS public.csg_daily_snapshot();
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION public.csg_daily_snapshot(
|
||||||
|
job_id integer DEFAULT NULL,
|
||||||
|
config jsonb DEFAULT NULL
|
||||||
|
)
|
||||||
|
RETURNS void
|
||||||
|
LANGUAGE plpgsql
|
||||||
|
AS $function$
|
||||||
|
DECLARE
|
||||||
|
d date;
|
||||||
|
v_usage numeric; v_cost numeric; v_ladder text; v_balance numeric;
|
||||||
|
m_usage numeric; v_tariff numeric;
|
||||||
|
p_month date; p_usage numeric; prev_usage numeric;
|
||||||
|
BEGIN
|
||||||
|
-- 数据日期 = CSG 集成「最新一天」(数据滞后一天);COALESCE 兜底昨天
|
||||||
|
SELECT COALESCE(
|
||||||
|
(SELECT (attributes->>'latest_day_date')::date
|
||||||
|
FROM states_raw s JOIN entities e ON e.id = s.metadata_id
|
||||||
|
WHERE e.entity_id = 'sensor.0800041935246530_latest_day_kwh'
|
||||||
|
ORDER BY s.time DESC LIMIT 1),
|
||||||
|
CURRENT_DATE - 1) INTO d;
|
||||||
|
|
||||||
|
-- 一律取「最新有值行」:轮询先写瞬态 unknown、~4s 后才写有值行
|
||||||
|
SELECT s.value INTO v_usage FROM states_raw s JOIN entities e ON e.id=s.metadata_id
|
||||||
|
WHERE e.entity_id='sensor.0800041935246530_yesterday_kwh' AND s.value IS NOT NULL ORDER BY s.time DESC LIMIT 1;
|
||||||
|
SELECT s.value INTO v_cost FROM states_raw s JOIN entities e ON e.id=s.metadata_id
|
||||||
|
WHERE e.entity_id='sensor.0800041935246530_latest_day_cost' AND s.value IS NOT NULL ORDER BY s.time DESC LIMIT 1;
|
||||||
|
SELECT s.state INTO v_ladder FROM states_raw s JOIN entities e ON e.id=s.metadata_id
|
||||||
|
WHERE e.entity_id='sensor.csg_current_ladder' AND s.state IS NOT NULL AND s.state NOT IN ('unknown','unavailable') ORDER BY s.time DESC LIMIT 1;
|
||||||
|
SELECT s.value INTO v_balance FROM states_raw s JOIN entities e ON e.id=s.metadata_id
|
||||||
|
WHERE e.entity_id='sensor.0800041935246530_balance' AND s.value IS NOT NULL ORDER BY s.time DESC LIMIT 1;
|
||||||
|
SELECT s.value INTO m_usage FROM states_raw s JOIN entities e ON e.id=s.metadata_id
|
||||||
|
WHERE e.entity_id='sensor.0800041935246530_this_month_total_usage' AND s.value IS NOT NULL ORDER BY s.time DESC LIMIT 1;
|
||||||
|
-- 日费用回退:原生 latest_day_cost 从未有值(集成侧该传感器恒为 unknown),
|
||||||
|
-- 用「昨日用电 × 当前档费率」估计;账单口径可用时优先原生
|
||||||
|
SELECT s.value INTO v_tariff FROM states_raw s JOIN entities e ON e.id=s.metadata_id
|
||||||
|
WHERE e.entity_id='sensor.csg_current_ladder_tariff' AND s.value IS NOT NULL ORDER BY s.time DESC LIMIT 1;
|
||||||
|
IF v_cost IS NULL AND v_usage IS NOT NULL AND v_tariff IS NOT NULL THEN
|
||||||
|
v_cost := v_usage * v_tariff;
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
INSERT INTO csg_history (period, kind, usage_kwh, cost, ladder, balance)
|
||||||
|
VALUES (d, 'day', v_usage, v_cost, v_ladder, v_balance)
|
||||||
|
ON CONFLICT (period, kind) DO UPDATE SET
|
||||||
|
usage_kwh = COALESCE(EXCLUDED.usage_kwh, csg_history.usage_kwh),
|
||||||
|
cost = COALESCE(EXCLUDED.cost, csg_history.cost),
|
||||||
|
ladder = COALESCE(EXCLUDED.ladder, csg_history.ladder),
|
||||||
|
balance = COALESCE(EXCLUDED.balance, csg_history.balance),
|
||||||
|
updated_at = now();
|
||||||
|
|
||||||
|
-- month 行按 d 所属月份归属(v3:修掉翻月时用错月份累计的 bug)
|
||||||
|
p_month := date_trunc('month', d)::date;
|
||||||
|
|
||||||
|
IF p_month = date_trunc('month', CURRENT_DATE)::date THEN
|
||||||
|
-- 当月:集成本月累计优先,缺失则退回 day 行汇总;费用一律阶梯重算
|
||||||
|
SELECT COALESCE(m_usage, sum(h.usage_kwh)) INTO p_usage
|
||||||
|
FROM csg_history h
|
||||||
|
WHERE h.kind='day' AND h.period >= p_month AND h.period < (p_month + interval '1 month');
|
||||||
|
ELSE
|
||||||
|
-- 上月/更早:只信该月 day 行(集成翻月时 last_month_* 有瞬态误值)
|
||||||
|
SELECT sum(h.usage_kwh) INTO p_usage
|
||||||
|
FROM csg_history h
|
||||||
|
WHERE h.kind='day' AND h.period >= p_month AND h.period < (p_month + interval '1 month');
|
||||||
|
SELECT h.usage_kwh INTO prev_usage FROM csg_history h
|
||||||
|
WHERE h.kind='month' AND h.period = p_month;
|
||||||
|
IF p_usage IS NULL OR (prev_usage IS NOT NULL AND prev_usage > p_usage) THEN
|
||||||
|
RETURN; -- day 行不完整,保留已有 month 行,不降级
|
||||||
|
END IF;
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
IF p_usage IS NOT NULL AND p_usage > 0 THEN
|
||||||
|
INSERT INTO csg_history (period, kind, usage_kwh, cost)
|
||||||
|
VALUES (p_month, 'month', p_usage, public.csg_ladder_cost(p_usage, p_month))
|
||||||
|
ON CONFLICT (period, kind) DO UPDATE SET
|
||||||
|
usage_kwh = EXCLUDED.usage_kwh,
|
||||||
|
cost = EXCLUDED.cost,
|
||||||
|
updated_at = now();
|
||||||
|
END IF;
|
||||||
|
END $function$;
|
||||||
|
|
||||||
|
-- 重挂每日任务:22:30 Asia/Shanghai(14:30 UTC)
|
||||||
|
DO $do$
|
||||||
|
BEGIN
|
||||||
|
IF NOT EXISTS (SELECT 1 FROM timescaledb_information.jobs WHERE proc_name = 'csg_daily_snapshot') THEN
|
||||||
|
PERFORM add_job(proc => 'public.csg_daily_snapshot'::regproc,
|
||||||
|
schedule_interval => INTERVAL '1 day',
|
||||||
|
initial_start => date_trunc('day', now() AT TIME ZONE 'UTC')
|
||||||
|
+ INTERVAL '14 hours 30 minutes',
|
||||||
|
timezone => 'UTC');
|
||||||
|
END IF;
|
||||||
|
END
|
||||||
|
$do$;
|
||||||
|
|
||||||
|
-- ---------- 3. 回填 day 行 2026-08-29 .. 2026-09-19 + 重算月行 ----------
|
||||||
|
-- 现有 day 行(07-01..08-28 来自首批回填)与新行一起参与「月内累计」窗口,
|
||||||
|
-- 使边际档位计算正确;已有非空 cost 不覆盖。
|
||||||
|
WITH newrows(period, usage_kwh) AS (
|
||||||
|
VALUES
|
||||||
|
(DATE '2026-08-29', 10.99::numeric),
|
||||||
|
(DATE '2026-08-30', 10.03::numeric),
|
||||||
|
(DATE '2026-08-31', 7.73::numeric),
|
||||||
|
(DATE '2026-09-01', 8.50::numeric),
|
||||||
|
(DATE '2026-09-02', 6.68::numeric),
|
||||||
|
(DATE '2026-09-03', 8.74::numeric),
|
||||||
|
(DATE '2026-09-04', 6.76::numeric),
|
||||||
|
(DATE '2026-09-05', 7.76::numeric),
|
||||||
|
(DATE '2026-09-06', 9.29::numeric),
|
||||||
|
(DATE '2026-09-07', 7.24::numeric),
|
||||||
|
(DATE '2026-09-08', 13.00::numeric),
|
||||||
|
(DATE '2026-09-09', 7.98::numeric),
|
||||||
|
(DATE '2026-09-10', 8.16::numeric),
|
||||||
|
(DATE '2026-09-11', 8.75::numeric),
|
||||||
|
(DATE '2026-09-12', 10.48::numeric),
|
||||||
|
(DATE '2026-09-13', 11.11::numeric),
|
||||||
|
(DATE '2026-09-14', 7.22::numeric),
|
||||||
|
(DATE '2026-09-15', 9.94::numeric),
|
||||||
|
(DATE '2026-09-16', 7.19::numeric),
|
||||||
|
(DATE '2026-09-17', 10.32::numeric),
|
||||||
|
(DATE '2026-09-18', 7.77::numeric),
|
||||||
|
(DATE '2026-09-19', 11.79::numeric)
|
||||||
|
),
|
||||||
|
allrows AS (
|
||||||
|
SELECT h.period, h.usage_kwh FROM csg_history h
|
||||||
|
WHERE h.kind='day' AND h.period BETWEEN DATE '2026-07-01' AND DATE '2026-09-19'
|
||||||
|
UNION ALL
|
||||||
|
SELECT n.period, n.usage_kwh FROM newrows n
|
||||||
|
WHERE NOT EXISTS (SELECT 1 FROM csg_history h WHERE h.kind='day' AND h.period = n.period)
|
||||||
|
),
|
||||||
|
calc AS (
|
||||||
|
SELECT period, usage_kwh,
|
||||||
|
CASE WHEN extract(month FROM period) BETWEEN 5 AND 10 THEN 260 ELSE 200 END AS t1,
|
||||||
|
CASE WHEN extract(month FROM period) BETWEEN 5 AND 10 THEN 600 ELSE 400 END AS t2,
|
||||||
|
sum(usage_kwh) OVER (PARTITION BY date_trunc('month', period)
|
||||||
|
ORDER BY period ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS c1
|
||||||
|
FROM allrows
|
||||||
|
),
|
||||||
|
slab AS (
|
||||||
|
SELECT period, usage_kwh, t1, t2, c1, c1 - usage_kwh AS c0 FROM calc
|
||||||
|
),
|
||||||
|
priced AS (
|
||||||
|
SELECT period, usage_kwh,
|
||||||
|
round(
|
||||||
|
(CASE WHEN c1 <= t1 THEN c1*0.589
|
||||||
|
WHEN c1 <= t2 THEN t1*0.589 + (c1-t1)*0.639
|
||||||
|
ELSE t1*0.589 + (t2-t1)*0.639 + (c1-t2)*0.889 END)
|
||||||
|
- (CASE WHEN c0 <= t1 THEN c0*0.589
|
||||||
|
WHEN c0 <= t2 THEN t1*0.589 + (c0-t1)*0.639
|
||||||
|
ELSE t1*0.589 + (t2-t1)*0.639 + (c0-t2)*0.889 END), 2) AS cost,
|
||||||
|
CASE WHEN c1 <= t1 THEN '一档' WHEN c1 <= t2 THEN '二档' ELSE '三档' END AS ladder
|
||||||
|
FROM slab
|
||||||
|
)
|
||||||
|
INSERT INTO csg_history (period, kind, usage_kwh, cost, ladder)
|
||||||
|
SELECT period, 'day', usage_kwh, cost, ladder FROM priced
|
||||||
|
ON CONFLICT (period, kind) DO UPDATE SET
|
||||||
|
usage_kwh = EXCLUDED.usage_kwh,
|
||||||
|
cost = COALESCE(csg_history.cost, EXCLUDED.cost),
|
||||||
|
ladder = COALESCE(csg_history.ladder, EXCLUDED.ladder),
|
||||||
|
updated_at = now();
|
||||||
|
|
||||||
|
-- 月行按 day 行汇总重算(只动 2026-08 / 2026-09;2026-07 及更早的月行来自集成
|
||||||
|
-- by_month,day 行并不覆盖整月,重算会把它算小,故不触碰)
|
||||||
|
INSERT INTO csg_history (period, kind, usage_kwh, cost)
|
||||||
|
SELECT date_trunc('month', h.period)::date, 'month', round(sum(h.usage_kwh),2),
|
||||||
|
public.csg_ladder_cost(round(sum(h.usage_kwh),2), date_trunc('month', h.period)::date)
|
||||||
|
FROM csg_history h
|
||||||
|
WHERE h.kind='day' AND h.period BETWEEN DATE '2026-08-01' AND DATE '2026-09-30'
|
||||||
|
GROUP BY date_trunc('month', h.period)
|
||||||
|
ON CONFLICT (period, kind) DO UPDATE SET
|
||||||
|
usage_kwh = EXCLUDED.usage_kwh,
|
||||||
|
cost = EXCLUDED.cost,
|
||||||
|
updated_at = now();
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
@@ -0,0 +1,269 @@
|
|||||||
|
-- CSG 长期归档 v5 (2026-09-22, VPS-78)
|
||||||
|
--
|
||||||
|
-- 背景:v3 修好了 job(1008 → 1010)并回填,但 csg_daily_snapshot() 写 day 行时用的是
|
||||||
|
-- 「昨日用电 × 当前档费率」**单一费率**且未舍入,而 v3 的回填(07-01..09-19)用的是
|
||||||
|
-- 「当月累积的边际差分」并 round(...,2)。两套口径混在同一列:
|
||||||
|
-- · 实测今天只差**精度**(3 行:2026-08-28 / 09-19 / 09-20,总差 -0.01 元);
|
||||||
|
-- · 但一旦某个**跨档日**由夜间任务写出,就会与历史口径不一致,并破坏
|
||||||
|
-- 「day 费用之和 ≈ 阶梯月费用」这条 v3 建立的验收不变量。
|
||||||
|
-- · 时限:2026-09-22 时 csg_current_ladder_remaining_kwh = 80.6、本月日均 8.97
|
||||||
|
-- → 约 2026-09-29 跨入二档。
|
||||||
|
--
|
||||||
|
-- 本文件的四个动作(幂等,可重复执行):
|
||||||
|
-- 1. csg_ladder_cost_raw(kwh, month) —— **无舍入**的阶梯费用助手。
|
||||||
|
-- v3 回填的算法是 round(精确c1 − 精确c0, 2)。若直接用
|
||||||
|
-- csg_ladder_cost(c1) − csg_ladder_cost(c0),会**二次舍入**(7 月实测 1 天差 0.01),
|
||||||
|
-- 故新增 raw 版,只在最后舍入一次。
|
||||||
|
-- 2. csg_daily_snapshot() v5 —— day cost 改为「当月累积边际差分 + round(...,2)」;
|
||||||
|
-- 并把 p_month 的计算**上移到计费之前**,加**跨月守卫**
|
||||||
|
-- (每月 1 日的 d 落在上月,此时 m_usage 是「新月份」累计,不能作起点,否则为负)。
|
||||||
|
-- 同时把「数据日期」与「当日用量」改为取**同一个实体同一行**
|
||||||
|
-- (sensor.0800041935246530_latest_day_kwh):v3 用 latest_day_kwh 取日期、
|
||||||
|
-- 却用 yesterday_kwh 取数值,两个实体可能错配;且 latest_day_kwh 的归档更完整
|
||||||
|
-- (含 2026-09-06 = 9.29 这条 yesterday_kwh 没有的观测)。
|
||||||
|
-- 3. 一次性归一:把历史 day 行的 cost 按新口径重算(实测仅 3 行变化,总差 -0.01)。
|
||||||
|
-- 4. csg_backfill_missing_days() + job 1011 —— **断档自愈**:只 INSERT 缺失日期,
|
||||||
|
-- ON CONFLICT DO NOTHING,**绝不覆盖既有行**;数据源是 scribe.states_raw 里
|
||||||
|
-- latest_day_kwh 的 (latest_day_date 属性, value) 观测对(该属性 v3 起就有归档)。
|
||||||
|
--
|
||||||
|
-- 回滚(脚本已自动建快照表 csg_history_bak_20260922):
|
||||||
|
-- BEGIN;
|
||||||
|
-- SELECT delete_job((SELECT job_id FROM timescaledb_information.jobs
|
||||||
|
-- WHERE proc_name='csg_backfill_missing_days'));
|
||||||
|
-- DROP FUNCTION IF EXISTS public.csg_backfill_missing_days(integer, jsonb);
|
||||||
|
-- DROP FUNCTION IF EXISTS public.csg_ladder_cost_raw(numeric, date);
|
||||||
|
-- -- v3 的 csg_daily_snapshot 用 compose/pgdb/csg-snapshot-v3.sql 的原文重建即可
|
||||||
|
-- DELETE FROM csg_history; INSERT INTO csg_history SELECT * FROM csg_history_bak_20260922;
|
||||||
|
-- COMMIT;
|
||||||
|
-- (或整体放弃归档:DROP TABLE csg_history + 删 job,见 hosts/pgdb.md 回滚段)
|
||||||
|
|
||||||
|
\set ON_ERROR_STOP on
|
||||||
|
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
-- ---------- 0. 回滚快照(只建一次) ----------
|
||||||
|
CREATE TABLE IF NOT EXISTS public.csg_history_bak_20260922 AS TABLE public.csg_history;
|
||||||
|
|
||||||
|
-- ---------- 1. 无舍入阶梯费用助手(差分用) ----------
|
||||||
|
CREATE OR REPLACE FUNCTION public.csg_ladder_cost_raw(kwh numeric, month_date date)
|
||||||
|
RETURNS numeric
|
||||||
|
LANGUAGE sql
|
||||||
|
IMMUTABLE
|
||||||
|
AS $function$
|
||||||
|
SELECT CASE
|
||||||
|
WHEN kwh IS NULL OR kwh <= 0 THEN 0::numeric
|
||||||
|
WHEN kwh <= (CASE WHEN extract(month FROM month_date) BETWEEN 5 AND 10 THEN 260 ELSE 200 END)
|
||||||
|
THEN kwh * 0.589
|
||||||
|
WHEN kwh <= (CASE WHEN extract(month FROM month_date) BETWEEN 5 AND 10 THEN 600 ELSE 400 END)
|
||||||
|
THEN (CASE WHEN extract(month FROM month_date) BETWEEN 5 AND 10 THEN 260 ELSE 200 END) * 0.589
|
||||||
|
+ (kwh - (CASE WHEN extract(month FROM month_date) BETWEEN 5 AND 10 THEN 260 ELSE 200 END)) * 0.639
|
||||||
|
ELSE (CASE WHEN extract(month FROM month_date) BETWEEN 5 AND 10 THEN 260 ELSE 200 END) * 0.589
|
||||||
|
+ ((CASE WHEN extract(month FROM month_date) BETWEEN 5 AND 10 THEN 600 ELSE 400 END)
|
||||||
|
- (CASE WHEN extract(month FROM month_date) BETWEEN 5 AND 10 THEN 260 ELSE 200 END)) * 0.639
|
||||||
|
+ (kwh - (CASE WHEN extract(month FROM month_date) BETWEEN 5 AND 10 THEN 600 ELSE 400 END)) * 0.889
|
||||||
|
END;
|
||||||
|
$function$;
|
||||||
|
|
||||||
|
-- ---------- 2. csg_daily_snapshot() v5 ----------
|
||||||
|
CREATE OR REPLACE FUNCTION public.csg_daily_snapshot(
|
||||||
|
job_id integer DEFAULT NULL,
|
||||||
|
config jsonb DEFAULT NULL
|
||||||
|
)
|
||||||
|
RETURNS void
|
||||||
|
LANGUAGE plpgsql
|
||||||
|
AS $function$
|
||||||
|
DECLARE
|
||||||
|
d date;
|
||||||
|
v_usage numeric; v_cost numeric; v_ladder text; v_balance numeric;
|
||||||
|
m_usage numeric;
|
||||||
|
p_month date; p_usage numeric; prev_usage numeric; c0 numeric; c1 numeric;
|
||||||
|
BEGIN
|
||||||
|
-- 数据日期与当日用量取【同一实体的同一行】:latest_day_kwh
|
||||||
|
-- (v3 的 d 来自它的 latest_day_date,但 v_usage 来自另一个实体 yesterday_kwh)
|
||||||
|
SELECT (s.attributes->>'latest_day_date')::date, s.value
|
||||||
|
INTO d, v_usage
|
||||||
|
FROM states_raw s JOIN entities e ON e.id = s.metadata_id
|
||||||
|
WHERE e.entity_id = 'sensor.0800041935246530_latest_day_kwh'
|
||||||
|
AND s.value IS NOT NULL
|
||||||
|
AND s.attributes->>'latest_day_date' IS NOT NULL
|
||||||
|
ORDER BY s.time DESC LIMIT 1;
|
||||||
|
d := COALESCE(d, CURRENT_DATE - 1);
|
||||||
|
|
||||||
|
-- 一律取「最新有值行」:轮询先写瞬态 unknown、~4s 后才写有值行
|
||||||
|
SELECT s.value INTO v_cost FROM states_raw s JOIN entities e ON e.id=s.metadata_id
|
||||||
|
WHERE e.entity_id='sensor.0800041935246530_latest_day_cost' AND s.value IS NOT NULL ORDER BY s.time DESC LIMIT 1;
|
||||||
|
SELECT s.state INTO v_ladder FROM states_raw s JOIN entities e ON e.id=s.metadata_id
|
||||||
|
WHERE e.entity_id='sensor.csg_current_ladder' AND s.state IS NOT NULL AND s.state NOT IN ('unknown','unavailable') ORDER BY s.time DESC LIMIT 1;
|
||||||
|
SELECT s.value INTO v_balance FROM states_raw s JOIN entities e ON e.id=s.metadata_id
|
||||||
|
WHERE e.entity_id='sensor.0800041935246530_balance' AND s.value IS NOT NULL ORDER BY s.time DESC LIMIT 1;
|
||||||
|
SELECT s.value INTO m_usage FROM states_raw s JOIN entities e ON e.id=s.metadata_id
|
||||||
|
WHERE e.entity_id='sensor.0800041935246530_this_month_total_usage' AND s.value IS NOT NULL ORDER BY s.time DESC LIMIT 1;
|
||||||
|
|
||||||
|
-- 月归属必须在计费之前算:跨月守卫要用它
|
||||||
|
p_month := date_trunc('month', d)::date;
|
||||||
|
|
||||||
|
-- 日费用:与 v3 回填同口径 = 当月累积的边际差分,只在最后舍入一次。
|
||||||
|
-- 原生 latest_day_cost 从未有值(集成侧恒 unknown),故这里是唯一路径。
|
||||||
|
IF v_cost IS NULL AND v_usage IS NOT NULL THEN
|
||||||
|
IF p_month = date_trunc('month', CURRENT_DATE)::date THEN
|
||||||
|
-- 当月日:起点 = 本月累计 − 当日用量
|
||||||
|
IF m_usage IS NOT NULL THEN
|
||||||
|
c1 := m_usage;
|
||||||
|
c0 := m_usage - v_usage;
|
||||||
|
END IF;
|
||||||
|
ELSE
|
||||||
|
-- 上月日(每月 1 日必现,因数据滞后一天):起点 = 该月已归档 day 行之和
|
||||||
|
SELECT COALESCE(sum(h.usage_kwh), 0) INTO c0
|
||||||
|
FROM csg_history h
|
||||||
|
WHERE h.kind='day' AND h.period >= p_month AND h.period < d;
|
||||||
|
c1 := c0 + v_usage;
|
||||||
|
END IF;
|
||||||
|
IF c0 IS NOT NULL AND c1 IS NOT NULL AND c0 >= 0 THEN
|
||||||
|
v_cost := round(public.csg_ladder_cost_raw(c1, p_month)
|
||||||
|
- public.csg_ladder_cost_raw(c0, p_month), 2);
|
||||||
|
END IF;
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
INSERT INTO csg_history (period, kind, usage_kwh, cost, ladder, balance)
|
||||||
|
VALUES (d, 'day', v_usage, v_cost, v_ladder, v_balance)
|
||||||
|
ON CONFLICT (period, kind) DO UPDATE SET
|
||||||
|
usage_kwh = COALESCE(EXCLUDED.usage_kwh, csg_history.usage_kwh),
|
||||||
|
cost = COALESCE(EXCLUDED.cost, csg_history.cost),
|
||||||
|
ladder = COALESCE(EXCLUDED.ladder, csg_history.ladder),
|
||||||
|
balance = COALESCE(EXCLUDED.balance, csg_history.balance),
|
||||||
|
updated_at = now();
|
||||||
|
|
||||||
|
-- month 行按 d 所属月份归属(v3:修掉翻月时用错月份累计的 bug)
|
||||||
|
IF p_month = date_trunc('month', CURRENT_DATE)::date THEN
|
||||||
|
-- 当月:集成本月累计优先,缺失则退回 day 行汇总;费用一律阶梯重算
|
||||||
|
SELECT COALESCE(m_usage, sum(h.usage_kwh)) INTO p_usage
|
||||||
|
FROM csg_history h
|
||||||
|
WHERE h.kind='day' AND h.period >= p_month AND h.period < (p_month + interval '1 month');
|
||||||
|
ELSE
|
||||||
|
-- 上月/更早:只信该月 day 行(集成翻月时 last_month_* 有瞬态误值)
|
||||||
|
SELECT sum(h.usage_kwh) INTO p_usage
|
||||||
|
FROM csg_history h
|
||||||
|
WHERE h.kind='day' AND h.period >= p_month AND h.period < (p_month + interval '1 month');
|
||||||
|
SELECT h.usage_kwh INTO prev_usage FROM csg_history h
|
||||||
|
WHERE h.kind='month' AND h.period = p_month;
|
||||||
|
IF p_usage IS NULL OR (prev_usage IS NOT NULL AND prev_usage > p_usage) THEN
|
||||||
|
RETURN; -- day 行不完整,保留已有 month 行,不降级
|
||||||
|
END IF;
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
IF p_usage IS NOT NULL AND p_usage > 0 THEN
|
||||||
|
INSERT INTO csg_history (period, kind, usage_kwh, cost)
|
||||||
|
VALUES (p_month, 'month', p_usage, public.csg_ladder_cost(p_usage, p_month))
|
||||||
|
ON CONFLICT (period, kind) DO UPDATE SET
|
||||||
|
usage_kwh = EXCLUDED.usage_kwh,
|
||||||
|
cost = EXCLUDED.cost,
|
||||||
|
updated_at = now();
|
||||||
|
END IF;
|
||||||
|
END $function$;
|
||||||
|
|
||||||
|
-- ---------- 3. 一次性归一:历史 day 行 cost 按新口径重算 ----------
|
||||||
|
-- 幂等:只更新与新口径不同的行。实测(写入前 dry-run)仅 3 行变化、总差 -0.01。
|
||||||
|
WITH cum AS (
|
||||||
|
SELECT period, usage_kwh, cost,
|
||||||
|
sum(usage_kwh) OVER (PARTITION BY date_trunc('month', period)
|
||||||
|
ORDER BY period ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS c1
|
||||||
|
FROM csg_history
|
||||||
|
WHERE kind='day' AND usage_kwh IS NOT NULL
|
||||||
|
), priced AS (
|
||||||
|
SELECT period,
|
||||||
|
round(public.csg_ladder_cost_raw(c1, period)
|
||||||
|
- public.csg_ladder_cost_raw(c1 - usage_kwh, period), 2) AS new_cost
|
||||||
|
FROM cum
|
||||||
|
)
|
||||||
|
UPDATE csg_history h
|
||||||
|
SET cost = p.new_cost, updated_at = now()
|
||||||
|
FROM priced p
|
||||||
|
WHERE h.kind='day' AND h.period = p.period
|
||||||
|
AND h.cost IS DISTINCT FROM p.new_cost;
|
||||||
|
|
||||||
|
-- ---------- 4. 断档自愈(只补缺失,不覆盖既有) ----------
|
||||||
|
CREATE OR REPLACE FUNCTION public.csg_backfill_missing_days(
|
||||||
|
job_id integer DEFAULT NULL,
|
||||||
|
config jsonb DEFAULT NULL
|
||||||
|
)
|
||||||
|
RETURNS void
|
||||||
|
LANGUAGE plpgsql
|
||||||
|
AS $function$
|
||||||
|
DECLARE
|
||||||
|
lookback int := COALESCE((config->>'lookback_days')::int, 45);
|
||||||
|
dd date;
|
||||||
|
v_usage numeric;
|
||||||
|
c0 numeric; c1 numeric;
|
||||||
|
v_ladder text;
|
||||||
|
mm date;
|
||||||
|
BEGIN
|
||||||
|
-- 4a. 补缺失的 day 行(只 INSERT,绝不覆盖)
|
||||||
|
FOR dd IN
|
||||||
|
SELECT g::date
|
||||||
|
FROM generate_series(CURRENT_DATE - lookback, CURRENT_DATE - 1, INTERVAL '1 day') g
|
||||||
|
WHERE NOT EXISTS (SELECT 1 FROM csg_history h WHERE h.kind='day' AND h.period = g::date)
|
||||||
|
ORDER BY 1
|
||||||
|
LOOP
|
||||||
|
-- 该数据日的观测:同一实体同时给出日期与数值
|
||||||
|
SELECT s.value INTO v_usage
|
||||||
|
FROM states_raw s JOIN entities e ON e.id = s.metadata_id
|
||||||
|
WHERE e.entity_id = 'sensor.0800041935246530_latest_day_kwh'
|
||||||
|
AND s.value IS NOT NULL
|
||||||
|
AND s.attributes->>'latest_day_date' = dd::text
|
||||||
|
ORDER BY s.time DESC LIMIT 1;
|
||||||
|
CONTINUE WHEN v_usage IS NULL; -- 上游没有这天的数据,不写空行
|
||||||
|
|
||||||
|
SELECT COALESCE(sum(h.usage_kwh), 0) INTO c0
|
||||||
|
FROM csg_history h
|
||||||
|
WHERE h.kind='day'
|
||||||
|
AND h.period >= date_trunc('month', dd)::date
|
||||||
|
AND h.period < dd;
|
||||||
|
c1 := c0 + v_usage;
|
||||||
|
v_ladder := CASE
|
||||||
|
WHEN c1 <= (CASE WHEN extract(month FROM dd) BETWEEN 5 AND 10 THEN 260 ELSE 200 END) THEN '一档'
|
||||||
|
WHEN c1 <= (CASE WHEN extract(month FROM dd) BETWEEN 5 AND 10 THEN 600 ELSE 400 END) THEN '二档'
|
||||||
|
ELSE '三档' END;
|
||||||
|
|
||||||
|
INSERT INTO csg_history (period, kind, usage_kwh, cost, ladder)
|
||||||
|
VALUES (dd, 'day', v_usage,
|
||||||
|
round(public.csg_ladder_cost_raw(c1, dd) - public.csg_ladder_cost_raw(c0, dd), 2),
|
||||||
|
v_ladder)
|
||||||
|
ON CONFLICT (period, kind) DO NOTHING;
|
||||||
|
END LOOP;
|
||||||
|
|
||||||
|
-- 4b. 受影响月份的 month 行重算(只升不降,与主函数同语义)
|
||||||
|
FOR mm IN
|
||||||
|
SELECT DISTINCT date_trunc('month', h.period)::date
|
||||||
|
FROM csg_history h
|
||||||
|
WHERE h.kind='day' AND h.period >= CURRENT_DATE - lookback
|
||||||
|
ORDER BY 1
|
||||||
|
LOOP
|
||||||
|
SELECT sum(h.usage_kwh) INTO c1
|
||||||
|
FROM csg_history h
|
||||||
|
WHERE h.kind='day' AND h.period >= mm AND h.period < (mm + interval '1 month');
|
||||||
|
SELECT h.usage_kwh INTO c0 FROM csg_history h WHERE h.kind='month' AND h.period = mm;
|
||||||
|
IF c1 IS NOT NULL AND c1 > 0 AND (c0 IS NULL OR c1 > c0) THEN
|
||||||
|
INSERT INTO csg_history (period, kind, usage_kwh, cost)
|
||||||
|
VALUES (mm, 'month', c1, public.csg_ladder_cost(c1, mm))
|
||||||
|
ON CONFLICT (period, kind) DO UPDATE SET
|
||||||
|
usage_kwh = EXCLUDED.usage_kwh,
|
||||||
|
cost = EXCLUDED.cost,
|
||||||
|
updated_at = now();
|
||||||
|
END IF;
|
||||||
|
END LOOP;
|
||||||
|
END $function$;
|
||||||
|
|
||||||
|
-- 重挂自愈任务:每天 15:10 UTC = 23:10 Asia/Shanghai(排在 14:30 UTC 的主任务之后)
|
||||||
|
DO $do$
|
||||||
|
BEGIN
|
||||||
|
IF NOT EXISTS (SELECT 1 FROM timescaledb_information.jobs WHERE proc_name = 'csg_backfill_missing_days') THEN
|
||||||
|
PERFORM add_job(proc => 'public.csg_backfill_missing_days'::regproc,
|
||||||
|
schedule_interval => INTERVAL '1 day',
|
||||||
|
initial_start => date_trunc('day', now() AT TIME ZONE 'UTC')
|
||||||
|
+ INTERVAL '1 day 15 hours 10 minutes',
|
||||||
|
timezone => 'UTC');
|
||||||
|
END IF;
|
||||||
|
END
|
||||||
|
$do$;
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
+6
-4
@@ -17,8 +17,8 @@
|
|||||||
|
|
||||||
| DB | Owner | Size | 用途 |
|
| DB | Owner | Size | 用途 |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| `hass` | hass | ~406 MB(2026-09-13) | HA recorder(states/events/statistics),客户端 HAOS `192.168.55.11` |
|
| `hass` | hass | ~603 MB(2026-09-21) | HA recorder(states/events/statistics),客户端 HAOS `192.168.55.11` |
|
||||||
| `scribe` | postgres | ~2.6 GB(2026-09-13) | HA scribe 集成(entities/areas/devices 注册表同步 + `states_raw`/`events` hypertable + `csg_history` 长期归档表);体积由 `sensor_minute` 图表管道主导(2.26 GB),见 Known issues |
|
| `scribe` | postgres | ~2851 MB(2026-09-21) | HA scribe 集成(entities/areas/devices 注册表同步 + `states_raw`/`events` hypertable + `csg_history` 长期归档表);体积由 `sensor_minute` 图表管道主导(2.46 GB 未压缩 chunk),见 Known issues |
|
||||||
| `postgres` | postgres | ~9 MB | 默认库 |
|
| `postgres` | postgres | ~9 MB | 默认库 |
|
||||||
|
|
||||||
## Ops notes
|
## Ops notes
|
||||||
@@ -39,11 +39,13 @@
|
|||||||
- 本机无防火墙(ufw/nft/iptables 均未装)——待办:如要彻底隔离可加 ufw 白名单 192.168.55.11。
|
- 本机无防火墙(ufw/nft/iptables 均未装)——待办:如要彻底隔离可加 ufw 白名单 192.168.55.11。
|
||||||
- `/opt/database/backups/` 根下残留 `*-2026-08-29_1359.dump`(compose 化之前旧备份机制产物)与 `backup.log`——健康检查只看 `daily/`,残留可清理。
|
- `/opt/database/backups/` 根下残留 `*-2026-08-29_1359.dump`(compose 化之前旧备份机制产物)与 `backup.log`——健康检查只看 `daily/`,残留可清理。
|
||||||
- **Runbooks**:[pgdb-health](../runbooks/pgdb-health.md)(只读健康检查)、[pgdb-restore](../runbooks/pgdb-restore.md)(pg_restore 还原)、[pgdb-update](../runbooks/pgdb-update.md)(镜像/compose 升级)。
|
- **Runbooks**:[pgdb-health](../runbooks/pgdb-health.md)(只读健康检查)、[pgdb-restore](../runbooks/pgdb-restore.md)(pg_restore 还原)、[pgdb-update](../runbooks/pgdb-update.md)(镜像/compose 升级)。
|
||||||
- **CSG 长期归档(2026-08-29, W1N-243)**:`csg_history` 表(`period date / kind('day'|'month') / usage_kwh / cost / ladder / balance / updated_at`,PK(period,kind),`GRANT SELECT TO hass`)保存南方电网有价值数据:day = 逐日(昨日用电/费用/阶梯/余额,2026-07-01 起),month = 当月累计(用电/费用,2025-01 起)。由 TimescaleDB 每日任务 **1008** `csg_daily_snapshot()`(22:30 Asia/Shanghai;**TS job 非 pg_cron**,本库未装 pg_cron)upsert 维护:取「最新有值行」防瞬态 unknown 竞态;日费用缺原生 `latest_day_cost` 时回退 = 昨日用电 × 当前档费率(模板 `csg_current_ladder_tariff` 0.639);月费用回退模板 `csg_this_month_ladder_cost`。验证:day 08-28 = 7.66 / 4.89474 / 二档 / 0,month 08 = 302.47 / 180.28。回填来源:集成 attributes `history_data`(59 天)+ `by_month`(19 月)——08-29 前唯一残存历史。回滚:`DROP TABLE csg_history` + `SELECT delete_job(1008)`。
|
- **CSG 长期归档(2026-08-29, W1N-243)**:`csg_history` 表(`period date / kind('day'|'month') / usage_kwh / cost / ladder / balance / updated_at`,PK(period,kind),`GRANT SELECT TO hass`)保存南方电网有价值数据:day = 逐日(昨日用电/费用/阶梯/余额,2026-07-01 起),month = 当月累计(用电/费用,2025-01 起)。由 TimescaleDB 每日任务 **1010**(2026-09-21 起;原 1008)`csg_daily_snapshot()`(**14:30 UTC = 22:30 Asia/Shanghai**;**TS job 非 pg_cron**,本库未装 pg_cron)upsert 维护;计费用 `csg_ladder_cost(kwh, month)` 助手(归档侧唯一计价来源)。回填来源:集成 attributes `history_data`(59 天)+ `by_month`(19 月)——08-29 前唯一残存历史。回滚:`DROP TABLE csg_history` + 删任务(另有迁移前快照表 `csg_history_bak_20260921`)。详见下方 Known issues 2026-09-21。**2026-09-22 v5 补两件事**:① **断档自愈** job **1011** `csg_backfill_missing_days()`(**15:10 UTC = 23:10 Asia/Shanghai**,排在 1010 之后)——只 `INSERT` 缺失日期、`ON CONFLICT DO NOTHING`,**绝不覆盖既有行**,数据源是 `states_raw` 里 `latest_day_kwh` 的 (`latest_day_date`, `value`) 观测对;② day 行费用改为 `csg_ladder_cost_raw` 的**累积边际差分 + 单次 `round(...,2)`**(与 v3 回填同口径),快照表 `csg_history_bak_20260922`,脚本 `compose/pgdb/csg-snapshot-v5.sql`。详见下方 Known issues 2026-09-22。
|
||||||
|
|
||||||
## Known issues
|
## Known issues
|
||||||
|
|
||||||
- 2026-09-13:**`sensor_minute` 体积构成与压缩窗口(只读诊断,暂不处理)**。`scribe` 库 2.6 GB = `sensor_minute` **2.26 GB**(850 万行 / 16 天,约 56–59 万行/天 = 331 实体 × 1440 分钟 LOCF)+ `states_raw` 290 MB + `events` 1.5 MB;`hass` 库另 406 MB。2.26 GB 中 1.50 GB 是 chunk `[09-03,09-10]`、0.76 GB 是 `[09-10,09-17]`,**都还没到压缩窗口**——TimescaleDB 的 `compress_after` 按 **chunk 结束时间**判断(09-10 结束 + 7 天 = **09-17** 才合格),所以「7 天 chunk + 7 天 compress_after」的设计下限就是盘上常驻近 14 天原始数据;已压缩的 `[08-27,09-03]` 从 1.04 GB → **1.5 MB**(LOCF 重复度极高,~700:1)。任务 1005 健康(30 成功 / 0 失败,最近 09-13 04:18 跑过但无合格 chunk);1002/1003/1006/1007 亦全 Success。稳态估算 ≈ 2 个未压缩 chunk(3–4.5 GB)+ 已压缩归档(约 1.5 MB/周 ≈ 80 MB/年)≈ **4–5 GB 平台期**;pgdata 卷 32 G 当前用 3.2 G,可用 27 G,**无需处理**。复查点 **2026-09-17 之后**:`_hyper_4_6_chunk` 应转为 `compressed=true` 且库体积回落;若仍为 false 才需动手(手动 `compress_chunk()` 或调小 `compress_after`)。可选调优:chunk 间隔 7 天 → 1 天 + `compress_after` → 2 天,把常驻未压缩量压到 <1 GB(`set_chunk_time_interval` 只对新 chunk 生效,旧 chunk 不重切)。诊断命令:`select chunk_name, is_compressed from timescaledb_information.chunks where hypertable_name='sensor_minute';` + `pg_database_size('scribe')`。**注意:这是 pgdb 侧对象,HA/scribe 的 `retention_states` 管不到它;HA 侧唯一杠杆是少记/少画(等于砍图)。**
|
- 2026-09-22:**CSG day 费用口径统一 + 断档自愈(v5,脚本 `compose/pgdb/csg-snapshot-v5.sql`)**。只读复核发现 `csg_daily_snapshot()` 写 day 费用用的是「昨日用电 × 当前档费率」**单一费率且未舍入**,而 v3 的回填用「当月累积边际差分 + `round(...,2)`」——两套口径混在同一列:当时只差精度(3 行:`2026-08-28` 4.89474 / `09-19` 6.94431 / `09-20` 6.31408,总差 -0.01),但**一旦某个跨档日由夜间任务写出就会破坏「day 费用之和 ≈ 阶梯月费用」**;时限是 2026-09-29 前后跨二档(当时 `csg_current_ladder_remaining_kwh` = 80.6、日均 8.97)。**改动**:① 新增 `csg_ladder_cost_raw(kwh, month)`(无舍入版;直接用 `csg_ladder_cost(c1) − csg_ladder_cost(c0)` 会**二次舍入**,7 月实测 1 天差 0.01);② `csg_daily_snapshot()` v5 —— day 费用改为 `round(raw(c1) − raw(c0), 2)`,`p_month` 计算**上移到计费之前**并加**跨月守卫**(每月 1 日的 `d` 落在上月,此时 `m_usage` 是新月份累计,不可作起点,否则为负);③ 数据日期与当日用量改为取**同一实体同一行** `latest_day_kwh`(v3 用它的 `latest_day_date` 定日期、却用 `yesterday_kwh` 取值,两实体可能错配;且 `latest_day_kwh` 归档更全,含 09-06 = 9.29 那条 `yesterday_kwh` 没有的观测);④ 新增 job **1011** `csg_backfill_missing_days()`(**断档自愈**,insert-only)。**验证**:归一 `UPDATE 3`(与 dry-run 预测一致);`cost <> round(cost,2)` 的行数 0;`CALL run_job(1011)` 与调度器实跑均 `Success`(1/1/0、`job_errors` 0);函数签名为 `(job_id integer, config jsonb)`(即 job 1008 失败的根因形态,已显式排除);两函数手动调用幂等(总行数 103、`max(day period)` 仍 09-20);各月 `|sum(day cost) − month cost|` = 0.00 / -0.03 / -0.03(**逐日舍入累积,非缺陷**;月行由 `csg_ladder_cost` 单次舍入,是权威值)。回滚:`delete_job(1011)` + `DROP FUNCTION csg_backfill_missing_days/csg_ladder_cost_raw` + 用 v3 脚本重建主函数 + 从 `csg_history_bak_20260922` 恢复行。健康检查新增第 9 项(`runbooks/pgdb-health.md`),专查「最新 day 行日期」——**上游停更时 job 会反复 upsert 同一行、`last_run_status` 仍 `Success`、日期上也无缺口,只有该断言能发现**。
|
||||||
|
- 2026-09-21:**CSG 快照任务 1008 自创建起从未成功 + 已修复(迁移脚本 `compose/pgdb/csg-snapshot-v3.sql`)**。只读复核发现 `job_stats` 1008 = **294 次运行 / 0 成功 / 294 失败**,`job_errors` 与容器日志一致:`function or procedure "public.csg_daily_snapshot(integer, jsonb)" does not exist. Custom job actions must accept (integer, jsonb) arguments.` 根因:函数实际签名是零参数 `csg_daily_snapshot()`,而 TimescaleDB 自定义 job 动作按名字调用 `schema.proc(job_id integer, config jsonb)`;08-29 那批数据是**手工写入**的,任务本身没写过一行。后果:`csg_history` 的 day 行停在 08-28、month 行停在 08-01,且 2026-08 月行是 08-29 当时的「本月至今」**302.47 kWh / 180.28 元**,而非月终值 **331.22 / 198.65**(账单 198.64)。**修复**:① 不能直接 `DROP FUNCTION`(job 持有依赖,报 `cannot drop ... because background job 1008 depends on it`)→ `delete_job(1008)` → 换 `(job_id integer DEFAULT NULL, config jsonb DEFAULT NULL)` 签名重建 → `add_job` 重挂为 **1010**,排程从 22:00 UTC(=06:00 CST,那时集成还没发布前一日数据)对齐到 **14:30 UTC = 22:30 CST**;② 新增 `csg_ladder_cost(kwh, month)` 助手(夏季 5-10 月 260/600、非夏季 200/400;0.589/0.639/0.889),month 行费用一律由它重算;③ month 行改为**按数据日期 `d` 所属月份归属**——当月用集成本月累计,上月由该月 day 行汇总(不读集成 `last_month_*`,避开翻月瞬态:08-31 16:32 实测读到过 323.49 这种误值),并加「汇总值小于已记录值则不动」的不降级保护;④ 回填 08-29→09-19 共 22 个 day 行(08-29..08-31 取 `states_raw` 里 `yesterday_kwh` 观测值减一天;09-01..09-19 取集成 `this_month_by_day`,其中 09-06 = 9.29 是 scribe 漏采的观测),日费用按「当日用电 × 当日边际档位」计,使 day 费用之和 == 阶梯月费用。**验证**:job 1010 `last_run_status=Success`(1/1/0),`next_start = 2026-09-21 14:30 UTC`;day 行 81 条(07-01..09-19)、month 行 21 条(2025-01..2026-09);2026-08 = 331.22/198.65、2026-09 = 168.68/99.35;9 月 day 合计 168.68 kWh / 99.33 元(与阶梯 99.35 差 0.02 为逐日四舍五入)。迁移前快照表 `csg_history_bak_20260921`(79 行)保留;旧任务 1008 的 294 条 `job_errors` 作为历史保留(Timescale 的 error retention 任务会自行清理)。差异核对:day 新增 22 条、补 cost 80 条(仅填 NULL,未覆盖既有非空值),month 改 2 条。
|
||||||
|
- 2026-09-13:**`sensor_minute` 体积构成与压缩窗口(只读诊断,暂不处理)**。`scribe` 库 2.6 GB = `sensor_minute` **2.26 GB**(850 万行 / 16 天,约 56–59 万行/天 = 331 实体 × 1440 分钟 LOCF)+ `states_raw` 290 MB + `events` 1.5 MB;`hass` 库另 406 MB。2.26 GB 中 1.50 GB 是 chunk `[09-03,09-10]`、0.76 GB 是 `[09-10,09-17]`,**都还没到压缩窗口**——TimescaleDB 的 `compress_after` 按 **chunk 结束时间**判断(09-10 结束 + 7 天 = **09-17** 才合格),所以「7 天 chunk + 7 天 compress_after」的设计下限就是盘上常驻近 14 天原始数据;已压缩的 `[08-27,09-03]` 从 1.04 GB → **1.5 MB**(LOCF 重复度极高,~700:1)。任务 1005 健康(30 成功 / 0 失败)。**2026-09-21 复查已通过**:`_hyper_4_6_chunk`(09-03→09-10)`is_compressed=t`(32 kB),`_hyper_4_10_chunk`(09-10→09-17,1557 MB)按 09-24 合格待压,稳态平台期估算不变(≈4–5 GB);pgdata 实测 3.6 G / 32 G(13%),**无需处理**。可选调优:chunk 间隔 7 天 → 1 天 + `compress_after` → 2 天(`set_chunk_time_interval` 只对新 chunk 生效)。**注意:这是 pgdb 侧对象,HA/scribe 的 `retention_states` 管不到它;HA 侧唯一杠杆是少记/少画(等于砍图)。**
|
||||||
- 2026-08-29:HA 侧 HACS 集成 `custom_components.scribe`(YAML `scribe: db_url:`,连 `scribe` 库)建表被拒(`permission denied for schema public`,hass 无 CREATE 权限),之后持续报 `relation "entities" does not exist`。**已解决**:① `GRANT CREATE ON SCHEMA public TO hass;`(scribe 库)② 重启 HA Core 触发重跑建表。重启后自动创建 `entities`(1591 行)/`users`/`areas`/`devices`/`integrations`/`states_raw` 表并启用 TimescaleDB 时间序列能力。报错已停止(最后一条 06:06 UTC),`states_raw` 持续写入。2026-08-29 复查:scribe 现有**两个** hypertable——`states_raw`(segmentby `metadata_id`、orderby `time`)与 `events`(segmentby `event_type`、orderby `time`),均 1 维 `time`;压缩已配置(`timescaledb_information.compression_settings` 可见对应行;2.29.x 该视图无 `compression_enabled` 列)。
|
- 2026-08-29:HA 侧 HACS 集成 `custom_components.scribe`(YAML `scribe: db_url:`,连 `scribe` 库)建表被拒(`permission denied for schema public`,hass 无 CREATE 权限),之后持续报 `relation "entities" does not exist`。**已解决**:① `GRANT CREATE ON SCHEMA public TO hass;`(scribe 库)② 重启 HA Core 触发重跑建表。重启后自动创建 `entities`(1591 行)/`users`/`areas`/`devices`/`integrations`/`states_raw` 表并启用 TimescaleDB 时间序列能力。报错已停止(最后一条 06:06 UTC),`states_raw` 持续写入。2026-08-29 复查:scribe 现有**两个** hypertable——`states_raw`(segmentby `metadata_id`、orderby `time`)与 `events`(segmentby `event_type`、orderby `time`),均 1 维 `time`;压缩已配置(`timescaledb_information.compression_settings` 可见对应行;2.29.x 该视图无 `compression_enabled` 列)。
|
||||||
- 2026-08-29:**timescale reader 图表对象**(配套 hass 的 `timescale_database_reader` 集成 + `timescale-plotly-card`,上游 SQL `remmob/timescale_database_reader` `SQL/scribe/01+02` @ `bb8776a`,以 postgres 执行):`sensor_minute_aggregate` 连续聚合(1 分钟桶,last(state)/last(value),实时聚合开启)+ `sensor_minute_aggregate_entity` 视图(join `entities`)+ `sensor_minute` hypertable(`minute`/`entity_id`/`state`/`value`,LOCF 前向填充)。任务:1005 `sensor_minute` 压缩(7 天)、1006 `sensor_minute` 保留(10 年)、1007 `every_minute_refresh` 每分钟增量刷新(含 5 分钟回溯窗口修正)。授权:`GRANT SELECT ON sensor_minute_aggregate, sensor_minute_aggregate_entity, sensor_minute, entities TO hass`。种子 19529 行(331 实体,自首个数据点起)。**刻意跳过**了上游脚本对 `states_raw` 的 3 个月保留 + 压缩策略语句——与"`states_raw` 永久归档"定位冲突,如需磁盘回收属用户决策(scribe 自己的压缩任务 1000/1001 未动)。
|
- 2026-08-29:**timescale reader 图表对象**(配套 hass 的 `timescale_database_reader` 集成 + `timescale-plotly-card`,上游 SQL `remmob/timescale_database_reader` `SQL/scribe/01+02` @ `bb8776a`,以 postgres 执行):`sensor_minute_aggregate` 连续聚合(1 分钟桶,last(state)/last(value),实时聚合开启)+ `sensor_minute_aggregate_entity` 视图(join `entities`)+ `sensor_minute` hypertable(`minute`/`entity_id`/`state`/`value`,LOCF 前向填充)。任务:1005 `sensor_minute` 压缩(7 天)、1006 `sensor_minute` 保留(10 年)、1007 `every_minute_refresh` 每分钟增量刷新(含 5 分钟回溯窗口修正)。授权:`GRANT SELECT ON sensor_minute_aggregate, sensor_minute_aggregate_entity, sensor_minute, entities TO hass`。种子 19529 行(331 实体,自首个数据点起)。**刻意跳过**了上游脚本对 `states_raw` 的 3 个月保留 + 压缩策略语句——与"`states_raw` 永久归档"定位冲突,如需磁盘回收属用户决策(scribe 自己的压缩任务 1000/1001 未动)。
|
||||||
- 2026-08-29:**`sensor_minute_refresh` 本地补丁(类比 tianqi 补丁,重跑上游 02 SQL 后需重打)**:值 CASE 的 `ELSE 0` → `ELSE NULL`。原因:scribe 对 unavailable 分钟 value 为 NULL,上游刷新过程兜底写 0;对差分模式的用电图,0→计数器回升会把插座的**生命周期累计值**(最高 1588 kWh)算进掉线那一小时。同日一次性清理既有脏 0:头部占位行 DELETE 505 行(各实体首次非零分钟之前的 value=0);`sensor.%_energy` 与温湿度实体的 value=0 → NULL(10+16 行,物理上不可能的真 0,图表渲染为断点)。功率实体的中途 0 是真实待机读数,保留。
|
- 2026-08-29:**`sensor_minute_refresh` 本地补丁(类比 tianqi 补丁,重跑上游 02 SQL 后需重打)**:值 CASE 的 `ELSE 0` → `ELSE NULL`。原因:scribe 对 unavailable 分钟 value 为 NULL,上游刷新过程兜底写 0;对差分模式的用电图,0→计数器回升会把插座的**生命周期累计值**(最高 1588 kWh)算进掉线那一小时。同日一次性清理既有脏 0:头部占位行 DELETE 505 行(各实体首次非零分钟之前的 value=0);`sensor.%_energy` 与温湿度实体的 value=0 → NULL(10+16 行,物理上不可能的真 0,图表渲染为断点)。功率实体的中途 0 是真实待机读数,保留。
|
||||||
|
|||||||
@@ -43,6 +43,8 @@ From the agent sandbox, always use `-F /dev/null` (system ssh config is unreadab
|
|||||||
- Latest daily backup older than today, not custom format, or `pg_restore -l` fails.
|
- Latest daily backup older than today, not custom format, or `pg_restore -l` fails.
|
||||||
- Disk usage near full on `/srv/pgdata` or `/`.
|
- Disk usage near full on `/srv/pgdata` or `/`.
|
||||||
- New `ERROR`/`FATAL` lines in the timescaledb log or backup failures in the pg-backup log.
|
- New `ERROR`/`FATAL` lines in the timescaledb log or backup failures in the pg-backup log.
|
||||||
|
- CSG 归档新鲜度断言失败(见 check 9):`csg_history` 最新 day 行落后超过 2 天、
|
||||||
|
`period` 上出现缺口、或任一 CSG job 非 `Success` / `total_failures > 0`。
|
||||||
|
|
||||||
## Pass criteria
|
## Pass criteria
|
||||||
|
|
||||||
@@ -54,6 +56,9 @@ From the agent sandbox, always use `-F /dev/null` (system ssh config is unreadab
|
|||||||
- `daily/*-latest.dump` symlinks point to today's dumps; `file -L` reports `PostgreSQL custom database dump`.
|
- `daily/*-latest.dump` symlinks point to today's dumps; `file -L` reports `PostgreSQL custom database dump`.
|
||||||
- `/srv/pgdata` (`/dev/sdb1`, 32G) and `/` not near full; fstab mounts `/srv/pgdata` by `UUID=c9e12e79-1f66-404c-ab7f-b8809be81d86` with `defaults,noatime`.
|
- `/srv/pgdata` (`/dev/sdb1`, 32G) and `/` not near full; fstab mounts `/srv/pgdata` by `UUID=c9e12e79-1f66-404c-ab7f-b8809be81d86` with `defaults,noatime`.
|
||||||
- timescaledb log: no new `ERROR`/`FATAL`; pg-backup log: recent successful backup.
|
- timescaledb log: no new `ERROR`/`FATAL`; pg-backup log: recent successful backup.
|
||||||
|
- `scribe.csg_history`:最新 day 行 `period >= current_date - 2`、`period` 上无缺口;
|
||||||
|
`csg_daily_snapshot`(1010)与 `csg_backfill_missing_days`(1011)均 `Success`、`total_failures = 0`;
|
||||||
|
各月 `|sum(day cost) − month cost| ≤ 0.05`(逐日舍入的累积,非缺陷)。
|
||||||
|
|
||||||
## Checks
|
## Checks
|
||||||
|
|
||||||
@@ -179,3 +184,46 @@ ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docke
|
|||||||
- pg-backup: recent successful run (`SQL backup created successfully` for each database, no restore/cleanup errors).
|
- pg-backup: recent successful run (`SQL backup created successfully` for each database, no restore/cleanup errors).
|
||||||
|
|
||||||
**Stop** if repeated `ERROR`/`FATAL` appear or a backup run failed.
|
**Stop** if repeated `ERROR`/`FATAL` appear or a backup run failed.
|
||||||
|
|
||||||
|
### 9. CSG 长期归档新鲜度(`scribe.csg_history`)
|
||||||
|
|
||||||
|
> 为什么必须单独看这张表:它承载南方电网的**永久**逐日/逐月归档,但写入它的
|
||||||
|
> job 曾经 **294 次运行 0 成功**、静默造成 23 天缺口而无人发现。更隐蔽的是:
|
||||||
|
> **上游停更时 job 会反复 upsert 同一行、`last_run_status` 仍是 `Success`、
|
||||||
|
> 日期上也不会出现"缺口"** —— 只有本节的"最新日期"断言能发现它。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# a) 最新 day 行日期(期望 >= 今天 - 2;CSG 数据本身滞后一天)
|
||||||
|
ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose exec -T timescaledb psql -U postgres -d scribe -Atc "select max(period) as last_day, current_date - 2 as min_expected from csg_history where kind='\''day'\'';"'
|
||||||
|
|
||||||
|
# b) period 上是否有缺口(期望 0 行)
|
||||||
|
ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose exec -T timescaledb psql -U postgres -d scribe -Atc "select period from generate_series('\''2026-07-01'\''::date, (select max(period) from csg_history where kind='\''day'\''), '\''1 day'\'') period except select period from csg_history where kind='\''day'\'';"'
|
||||||
|
|
||||||
|
# c) 两个 CSG job 的状态(期望均为 Success / 0 failures)
|
||||||
|
ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose exec -T timescaledb psql -U postgres -d scribe -Atc "select j.job_id, j.proc_name, s.last_run_status, s.total_runs, s.total_failures from timescaledb_information.jobs j join timescaledb_information.job_stats s on s.job_id = j.job_id where j.proc_name like '\''csg%'\'' order by j.job_id;"'
|
||||||
|
|
||||||
|
# d) 日费用与月费用的口径一致性(期望 |diff| <= 0.05)
|
||||||
|
ssh -F /dev/null -o BatchMode=yes windy@192.168.55.15 'cd /opt/database && docker compose exec -T timescaledb psql -U postgres -d scribe -Atc "select m.period, m.usage_kwh, m.cost, round(d.s,2) as day_sum, round(d.s - m.cost,2) as diff from csg_history m left join (select date_trunc('\''month'\'',period)::date as mon, sum(cost) as s from csg_history where kind='\''day'\'' group by 1) d on d.mon = m.period where m.kind='\''month'\'' and m.period >= '\''2026-07-01'\'' order by m.period;"'
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**
|
||||||
|
|
||||||
|
- a) `last_day >= current_date - 2`。
|
||||||
|
- b) 0 行。
|
||||||
|
- c) `csg_daily_snapshot`(job 1010,14:30 UTC = 22:30 CST)与
|
||||||
|
`csg_backfill_missing_days`(job 1011,15:10 UTC = 23:10 CST)均 `Success`、`total_failures = 0`。
|
||||||
|
- d) 每个月 `|diff| <= 0.05` 元。这是**逐日四舍五入到 2 位的累积**,不是缺陷;
|
||||||
|
月行(由 `csg_ladder_cost` 单次舍入)才是权威值。
|
||||||
|
|
||||||
|
**Stop** if
|
||||||
|
|
||||||
|
- `last_day` 落后超过 2 天 → 上游停更或 job 未运行。先看 c) 的 `last_run_started_at`
|
||||||
|
与 `ha core logs` 里的 `china_southern_power_grid_stat` 错误,再决定是否需要
|
||||||
|
重新加载集成 config entry(见 [home-assistant-maintenance](home-assistant-maintenance.md))。
|
||||||
|
- b) 出现缺口 → 等 `csg_backfill_missing_days` 下一轮(或手动 `CALL run_job(1011);`)
|
||||||
|
后复查;仍缺则查 `timescaledb_information.job_errors`。
|
||||||
|
- c) 任一 job 非 `Success` 或 `total_failures > 0` → 查 `job_errors` 的 `err_message`;
|
||||||
|
历史上 job 1008 的失败原因是**函数签名与 TimescaleDB 自定义 job 约定不符**
|
||||||
|
(必须是 `(job_id integer, config jsonb)`),修复记录见
|
||||||
|
`compose/pgdb/csg-snapshot-v3.sql` / `-v5.sql`。
|
||||||
|
- d) 某个 `|diff| > 0.05` → 说明有新写入路径绕过了 v5 的边际差分口径,停止并复核。
|
||||||
|
|||||||
Reference in New Issue
Block a user