From 7b7b61f1007088a1deee145001eede3188bd3c34 Mon Sep 17 00:00:00 2001 From: windyboy Date: Mon, 7 Sep 2026 16:12:00 +0800 Subject: [PATCH] =?UTF-8?q?feat(processing):=20=E5=AE=9E=E7=8E=B0=E8=87=AA?= =?UTF-8?q?=E6=9C=89=20PostgreSQL=20=E8=BF=90=E8=90=A5=E8=88=AA=E7=8F=AD?= =?UTF-8?q?=E6=9D=83=E5=A8=81=E5=AD=98=E5=82=A8=E4=B8=8E=E5=8D=95=E4=BA=8B?= =?UTF-8?q?=E5=8A=A1=E9=97=AD=E7=8E=AF=20(ACM2-28)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - FS1: 增加 Flyway 迁移 V1.1.0__flight_schd.sql，创建 FLIGHT_SCHD 与 SCHD_GEN - FS2: 实现 FlightSchdRepository 接口及 JdbcFlightSchdRepository 与 StubFlightSchd，增强 JdbcOps 事务管理 - FS3: 扩展 MessageProcessor 事务 2 与按 FLID 点查视图，合并变更、事件与终态入单事务提交 - FS4: SnapshotFlow SQL 化（批处理 upsert、域内差删、SQL CAS 推进与熔断保护），JobExecutor 接入 PG 清场删除 - FS5: 彻底退役 Redis 权威与写路径，移除 FlightRedisClient、Lua 脚本、健康指示器与配置残留 - FS6: 补齐 U09/U29 不变量门禁（崩溃幂等、CAS 防并发、ADFT 存活保障、非 UTC JVM/会话时区无漂移）与 FlywayMigrationTest - FS7: 交付影子对拍比较内核 FlightStoreDiffTool 与单元测试 - FS8: 全面回改 decision-flight-state、architecture、design、user-stories 权威文档与规范 --- .env.example | 6 +- AGENTS.md | 4 +- build.gradle.kts | 1 + compose.yaml | 3 + docs/architecture.md | 29 +- docs/decision-flight-state.md | 43 +-- docs/design.md | 33 +- docs/user-stories.md | 13 +- .../omms/msgexchange/delivery/Dispatcher.kt | 17 +- .../gzzn/omms/msgexchange/domain/Decision.kt | 4 +- .../gzzn/omms/msgexchange/domain/MsgEvent.kt | 7 +- .../infra/health/HealthIndicators.kt | 21 +- .../infra/persistence/Repositories.kt | 91 ++++- .../infra/persistence/jdbc/JdbcOps.kt | 71 +++- .../persistence/jdbc/JdbcPgRepositories.kt | 252 ++++++++++++- .../msgexchange/infra/redis/RedisScripts.kt | 29 -- .../msgexchange/infra/stub/StubAdapters.kt | 29 +- .../infra/stub/StubRepositories.kt | 132 ++++++- .../gzzn/omms/msgexchange/jobs/JobExecutor.kt | 33 +- .../omms/msgexchange/processing/Handler.kt | 2 +- .../gzzn/omms/msgexchange/processing/Pump.kt | 57 ++- .../msgexchange/processing/SnapshotFlow.kt | 117 ++++-- .../msgexchange/tools/FlightStoreDiffTool.kt | 260 +++++++++++++ src/main/resources/application-dev.yml | 15 +- src/main/resources/application.yml | 15 +- .../db/migration/V1.1.0__flight_schd.sql | 32 ++ src/main/resources/lua/batch_delete.lua | 9 - src/main/resources/lua/snapshot_replace.lua | 19 - .../delivery/DispatcherTickTest.kt | 20 - .../infra/health/HealthIndicatorsTest.kt | 24 -- .../persistence/jdbc/FlightSchdJdbcPgTest.kt | 289 ++++++++++++++ .../persistence/jdbc/FlywayMigrationTest.kt | 74 ++++ .../msgexchange/jobs/HistorySweepJobTest.kt | 167 +++++++++ .../processing/FlightSchdInvariantTest.kt | 353 ++++++++++++++++++ .../processing/MessageProcessorTest.kt | 58 ++- .../tools/FlightStoreDiffToolTest.kt | 87 +++++ src/test/resources/application-test.yml | 3 - 37 files changed, 2037 insertions(+), 382 deletions(-) delete mode 100644 src/main/kotlin/com/gzzn/omms/msgexchange/infra/redis/RedisScripts.kt create mode 100644 src/main/kotlin/com/gzzn/omms/msgexchange/tools/FlightStoreDiffTool.kt create mode 100644 src/main/resources/db/migration/V1.1.0__flight_schd.sql delete mode 100644 src/main/resources/lua/batch_delete.lua delete mode 100644 src/main/resources/lua/snapshot_replace.lua create mode 100644 src/test/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/FlightSchdJdbcPgTest.kt create mode 100644 src/test/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/FlywayMigrationTest.kt create mode 100644 src/test/kotlin/com/gzzn/omms/msgexchange/jobs/HistorySweepJobTest.kt create mode 100644 src/test/kotlin/com/gzzn/omms/msgexchange/processing/FlightSchdInvariantTest.kt create mode 100644 src/test/kotlin/com/gzzn/omms/msgexchange/tools/FlightStoreDiffToolTest.kt diff --git a/.env.example b/.env.example index 6a1af9c..6e2ee9a 100644 --- a/.env.example +++ b/.env.example @@ -8,13 +8,13 @@ MSGX_MAILBOX_URL=jdbc:mysql://localhost:3306/cdairport?useSSL=false&allowPublicK MSGX_MAILBOX_USER=msgx_dev MSGX_MAILBOX_PASSWORD=msgx_dev_pass -# 2. 自有 PostgreSQL（dev compose: postgres 服务，Flyway V1.0.0 自动建表） +# 2. 自有 PostgreSQL（dev compose: postgres 服务，应用启动设 MSGX_FLYWAY_ENABLED=true 时由 Flyway 自动执行 V1.0.0 与 V1.1.0 迁移） MSGX_PG_URL=jdbc:postgresql://localhost:5432/msgx MSGX_PG_USER=msgx_dev MSGX_PG_PASSWORD=msgx_dev_pass -# 3. Redis / Valkey（dev compose: redis 服务，航班动态 + 快照 gen） -MSGX_REDIS_URI=redis://localhost:6379 +# 3. Redis / Valkey（ACM2-28：核心管道已退役；仅供 FS7 影子 diff 对拍读取 legacy 现役数据） +# MSGX_REDIS_URI=redis://localhost:6379 # 4. Kafka（dev compose: kafka 单节点 KRaft 服务，或对接现网/测试网） MSGX_KAFKA_SERVERS=localhost:9092 diff --git a/AGENTS.md b/AGENTS.md index 79973cb..d6b1dc2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Project Structure & Module Organization -Application code lives under `src/main/kotlin/com/gzzn/omms/msgexchange/`. Keep changes within the established modules: `ingress` receives mailbox records, `processing` owns FIFO decisions, `delivery` handles outbound events, `jobs` runs maintenance work, and `infra` contains persistence, Redis, health, and retry adapters. Runtime configuration, Flyway migrations, and Lua scripts are in `src/main/resources/`. Tests mirror production packages under `src/test/kotlin/`. Architecture and requirements live in `docs/`; treat `docs/legacy/` as reference material, not current design. Local middleware definitions are in `compose.yaml` and `deploy/dev/`. +Application code lives under `src/main/kotlin/com/gzzn/omms/msgexchange/`. Keep changes within the established modules: `ingress` receives mailbox records, `processing` owns FIFO decisions, `delivery` handles outbound events, `jobs` runs maintenance work, and `infra` contains persistence, health, and retry adapters (Redis has exited runtime authority per ACM2-28). Runtime configuration and Flyway migrations are in `src/main/resources/`. Tests mirror production packages under `src/test/kotlin/`. Architecture and requirements live in `docs/`; treat `docs/legacy/` as reference material, not current design. Local middleware definitions are in `compose.yaml` and `deploy/dev/`. ## Build, Test, and Development Commands @@ -16,7 +16,7 @@ Use JDK 25. In restricted environments, point `GRADLE_USER_HOME` and `TMPDIR` to ## Coding Style & Naming Conventions -Use Kotlin conventions with four-space indentation, trailing commas in multiline declarations, and immutable values by default. Types use `PascalCase`; functions and properties use `camelCase`; constants use `UPPER_SNAKE_CASE`. Name tests after behavior, for example `MessageProcessorTest` and `InboxPollerTest`. Keep handlers pure: return domain decisions rather than performing Redis, Kafka, or database writes directly. Preserve the single-writer and strict message FIFO invariants documented in `docs/architecture.md`. +Use Kotlin conventions with four-space indentation, trailing commas in multiline declarations, and immutable values by default. Types use `PascalCase`; functions and properties use `camelCase`; constants use `UPPER_SNAKE_CASE`. Name tests after behavior, for example `MessageProcessorTest` and `InboxPollerTest`. Keep handlers pure: return domain decisions rather than performing Kafka or database writes directly. Preserve the single-writer and strict message FIFO invariants documented in `docs/architecture.md`. ## Testing Guidelines diff --git a/build.gradle.kts b/build.gradle.kts index 474484d..81aa46d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -49,6 +49,7 @@ dependencies { runtimeOnly(libs.mysql.connector.j) runtimeOnly(libs.postgresql) + runtimeOnly("org.flywaydb:flyway-database-postgresql:12.11.0") // ACM2-11：datasources.reference（21 类静态主数据独立 PG 库）驱动 org.postgresql:postgresql // 随阶段 6 实装加入（当前 reference enabled=false 不加载；版本由 micronaut-platform BOM 钉 ~42.7） runtimeOnly(libs.snakeyaml) // U02：显式版本入 catalog（原无版本号依赖 BOM 覆盖，snakeyaml 不在 micronaut BOM 内） diff --git a/compose.yaml b/compose.yaml index 2daeb8f..27a210e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -30,6 +30,7 @@ services: retries: 10 start_period: 15s + # 由 Flyway 自动执行 V1.0.0 与 V1.1.0 迁移 postgres: image: postgres:17-alpine container_name: msgx-dev-postgres @@ -49,6 +50,8 @@ services: retries: 10 start_period: 5s + # ACM2-28：Redis 已退出阶段 A 权威与写路径（FLIGHT_SCHD 落自有 PG）。 + # 本服务保留至切流，供 FS7 影子 diff 对拍工具读取现役 legacy Redis flightInfo 与本地辅助冒烟。 redis: image: valkey/valkey:8-alpine container_name: msgx-dev-redis diff --git a/docs/architecture.md b/docs/architecture.md index a446b9a..8e6b88e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -10,7 +10,7 @@ msgexchange-v2 是机场 OMMS 的上游报文处理中间件，用于替换旧 - **主要入口**：轮询共享 MySQL 的 `CMINMSGS`。 - **兼容入口**：`POST /cminmsgs/send`，供现役兼容、手工工具和对拍使用；写入信箱后返回记录 ID，不是生产收报主路径。 - **输出**：Kafka 的 `msg` / `schd` 消息、共享 MySQL 的 `COUTMSGS` 出站信箱，以及查询 HTTP 接口；不直接推送前端。 -- **当前范围（阶段 A）**：Redis 保存航班动态的权威状态。ES 历史投影及相关清场流程属于暂缓的阶段 B，不在当前交付范围内，也不新增关系型 `FLIGHT_STATE` 表。 +- **当前范围（阶段 A）**：运营航班权威状态落自有 PostgreSQL（表 `FLIGHT_SCHD` 与 `SCHD_GEN`，ACM2-28 定案采纳选项 C），Redis 彻底退出动态权威与全部写路径。ES 历史投影及相关清场流程属于暂缓的阶段 B。 本文描述架构约束，不代表所有能力已实现；实现缺口见第 9 节。模块交互、状态机和参数详见 [design.md](design.md)，需求见 [user-stories.md](user-stories.md)。历史报文契约仍以 [SIS 接口规范](legacy/SIS_AODB_RMS-V0.1.md) 和 [XSD](legacy/unisysaodbsis.xsd) 为兼容依据，其他 legacy 资料仅作参考。 @@ -27,8 +27,7 @@ CIIMS / AODB 等上游 │ ingress：发现报文 → PostgreSQL 持久化入队 │ │ │ │ │ processing：取 FIFO 队头 → 解析 / 去重 → Handler 决策 │ -│ ├─ 更新 Redis 航班动态 / 快照 │ -│ └─ PG 事务：处理结果 + 待发事件 │ +│ └─ PG 单事务：航班变更 + 终态 + 待发事件│ │ │ │ jobs：在主泵空闲或消息退避窗口内执行维护作业 │ │ delivery：读取 PG 待发事件 → 投递 / 重试 │ @@ -50,12 +49,12 @@ CIIMS / AODB 等上游 |---|---| | `ingress` | 轮询信箱、持久化入队、补偿重扫及兼容 HTTP 写入；不解析业务报文。 | | `codec` | XML 解码，区分非法报文与可修复的解码失败。 | -| `processing` | FIFO 调度、业务身份绑定与去重、Handler 决策、快照处理及状态提交。Handler 只返回决策，不直接访问数据库、Redis 或 Kafka。 | +| `processing` | FIFO 调度、业务身份绑定与去重、Handler 决策、快照处理及状态提交。Handler 只返回决策，不直接访问数据库或 Kafka。 | | `delivery` | 消费待发事件，负责按目标保序、`schd` 聚合、投递和失败重试。 | | `jobs` | 持久化维护作业，由主泵在允许的窗口执行；阶段 B 作业暂不启用。 | | `reference` | 静态参考数据和上游请求跟踪。 | | `domain` / `config` | 领域状态、事件和决策模型，以及运行参数。 | -| `infra` | 仓储、Redis 脚本、外部适配器、重试、健康检查与日志；通过接口隔离基础设施。 | +| `infra` | 仓储、外部适配器、重试、健康检查与日志；通过接口隔离基础设施（Redis 已退出核心写路径）。 | ## 4. 主流程 @@ -63,8 +62,8 @@ CIIMS / AODB 等上游 1. `InboxPoller` 默认每秒扫描 `DATE_PROCESSED IS NULL` 的信箱记录，在自有 PG 中建立 `PROC_STATE(PENDING)`。重复扫描不能重复入队；入队失败留待重扫。 2. 主泵只处理最小未完成 `CMINMSGS_ID`。解析报文、绑定业务身份并去重后，调用对应 Handler 生成决策。 -3. 主泵执行 Redis 更新。全量快照通过代际版本 `gen` 管理覆盖和旧数据清理，避免旧快照覆盖新状态。 -4. 在 PG 本地事务中同时保存处理结果和 `MSG_EVENT` 待发事件，再回填共享信箱的处理标记。 +3. 在自有 PG 本地事务中同时保存航班状态变更（`FLIGHT_SCHD` / `SCHD_GEN`）、处理结果和 `MSG_EVENT` 待发事件。 +4. 事务提交后，回填共享信箱的处理标记（外部副作用，补偿保障）。 ### 投递 @@ -75,7 +74,7 @@ CIIMS / AODB 等上游 ## 5. 必须保持的约束 - **消息严格 FIFO**：队头失败并退避时，后续消息仍不能越过它。只有队头完成或按失败策略进入终态后，队列才继续推进。收报重扫和水位设计必须防止较小 ID 漏入队而被后续消息越过。 -- **动态状态单写者**：Redis 航班动态和快照 `gen` 只由主泵写入。不能通过增加实例或处理线程来直接扩容。 +- **动态状态单写者**：`FLIGHT_SCHD` 运营航班表和快照 `SCHD_GEN` 只由主泵单线程写入。生产单实例通过 PG advisory lock 保护，不能通过增加实例或处理线程直接扩容。 - **身份去重**：同一业务身份只能绑定一条有效处理记录，重复报文不应再次产生业务副作用。具体身份组成和重放规则见设计文档。 - **快照可恢复**：快照覆盖、旧数据清理和版本推进需要原子性与重放保护；不能在恢复时把旧代数据重新写回。 - **作业不与消息混排**：`PUMP_JOB` 是独立队列，只在没有消息队头或队头处于退避窗口时执行。执行窗口与饥饿边界需要明确验证。 @@ -84,22 +83,20 @@ CIIMS / AODB 等上游 ## 6. 数据归属与一致性 -| 存储 | 保存内容 | 边界 | +| 存储 | 承载内容 | 职责说明 | |---|---|---| -| 自有 PostgreSQL | `PROC_STATE` 处理状态、`MSG_EVENT` 待发事件、`PUMP_JOB` 作业、`REQ_TRACK` 请求跟踪、`REF_MASTER` 静态数据，以及 `PROC_STATE_HST` 归档 | 本系统的内部持久化状态；唯一的本地事务边界。 | +| 自有 PostgreSQL | 处理状态 `PROC_STATE`、待发事件 `MSG_EVENT`、维护作业 `PUMP_JOB`、请求跟踪 `REQ_TRACK`、静态主数据 `REF_MASTER`、运营航班表 `FLIGHT_SCHD` 与日代 `SCHD_GEN` | 本系统唯一业务数据库。消息处理、快照推进与待发事件在单事务内原子提交；本地事务只在此库。 | | 共享 MySQL | `CMINMSGS` 入站信箱、`COUTMSGS` 出站信箱 | 外部系统所有。仅执行约定的信箱读写和处理标记回填，不建表、不迁移 schema、不写历史表。兼容 HTTP 入口可按既有契约写入入站信箱。 | -| Redis | 航班动态 `flightInfo` 和快照版本 `gen` | 阶段 A 的动态权威存储，不是可随意清空的缓存。 | - **不使用跨库事务。** PG 事务只能保证“处理结果与待发事件一起提交”，不能覆盖 Redis 更新、MySQL 回填或 Kafka 发送。跨存储依靠幂等、重试和持久化补偿恢复： | 中断位置 | 恢复要求 | |---|---| | 信箱已有报文，PG 入队失败 | 重扫补建，并按信箱 ID 去重。 | -| Redis 更新成功，PG 提交失败 | 消息重试可能再次更新 Redis；更新与快照协议必须支持幂等重放和版本校验。 | +| PG 提交失败 | 事务原子回滚，无中间态残留；消息重试时整体重放。 | | PG 已提交，信箱回填失败 | 持久化记录补偿任务并重试回填，不能重新执行已完成的业务处理。 | | 下游已接收，本地尚未标记发送成功 | 允许重发；下游或出站适配协议必须具备去重能力。 | -对外投递按**至少一次**设计，不承诺端到端恰好一次。Kafka 生产者幂等不能消除应用重启或 outbox 重发带来的所有重复。Redis 恢复也不能仅依赖 PG 处理状态：备份、报文保留及回放范围需要在上线前验证。 +对外投递按**至少一次**设计，不承诺端到端恰好一次。Kafka 生产者幂等不能消除应用重启或 outbox 重发带来的所有重复。 ## 7. 关键决策索引 @@ -130,7 +127,7 @@ CIIMS / AODB 等上游 **替换旧系统** -采用“影子对拍 → 切流 → 旧系统冻结”。共享信箱不能让新旧系统同时认领和回填；影子输入使用只读水位或回放。影子环境须隔离 PG schema/实例、Redis key 空间、Kafka topic 和服务注册身份，并禁止误写生产信箱。切流时保证只有一个权威写者。 +采用“影子对拍 → 切流 → 旧系统冻结”。共享信箱不能让新旧系统同时认领和回填；影子输入使用只读水位或回放。影子环境须隔离 PG schema/实例、Kafka topic 和服务注册身份，并禁止误写生产信箱。切流时保证只有一个权威写者。影子期采用 `FlightStoreDiffTool` 跨存储对拍。 **可观测性要求** @@ -143,7 +140,7 @@ CIIMS / AODB 等上游 上线前至少需要完成并验证： - 真实 PG 事务、信箱水位与补扫、回填补偿、出站信箱，以及所需业务 Handler。 -- Redis 更新幂等性、快照版本校验、故障中断恢复与权威数据恢复方案。 +- `FLIGHT_SCHD` 事务原子性、快照 SQL CAS 版本校验、故障中断回滚恢复与影子对拍 cross-store diff 工具。 - FIFO、身份去重、作业窗口和投递故障下的回归测试。 - 生产启动校验、单实例排他保护、影子隔离和 Kafka 配置约束；当前配置仍允许 Kafka 参数覆盖，且默认 in-flight 值与 D11 要求不同。 - 死信告警、人工重放、端到端追踪、积压指标及安全边界。 diff --git a/docs/decision-flight-state.md b/docs/decision-flight-state.md index 6a6a614..785bc48 100644 --- a/docs/decision-flight-state.md +++ b/docs/decision-flight-state.md @@ -1,9 +1,8 @@ -# 决策提案：运营航班表 FLIGHT_STATE 是否落自有 PostgreSQL +# 决策：运营航班表 FLIGHT_SCHD 落自有 PostgreSQL 作阶段 A 权威（已定案 · 采纳选项 C） -> **状态**：提案（讨论 issue：**ACM2-28**，关联 ACM2-12）。 -> 本文重估 ACM2-12「阶段 B 缓做、不落表」的口径；定案前不改代码与迁移。 -> 交叉引用：[architecture.md](architecture.md) §6/§7、[design.md](design.md) §2/§3.4、 -> ACM2-12（存储边界）、ACM2-10 U05/U09/U15。 +> **状态**：已定案（讨论 issue：**ACM2-28**，采纳选项 C；关联修订 ACM2-12）。 +> 本文正式确立选项 C 为最终方案：运营航班权威 `FLIGHT_SCHD` 与代版本 `SCHD_GEN` 提前至阶段 A 落自有 PostgreSQL；Redis 彻底退出动态权威与写路径。 +> 交叉引用：[architecture.md](architecture.md)、[design.md](design.md)、[user-stories.md](user-stories.md)、ACM2-10（FS1–FS8 实施批次）。 ## 1. 问题 @@ -75,22 +74,24 @@ 时机（F5）与代价重合。若评审认为生产 Redis 必然配强持久化+副本且接受 DNLD 重建窗口， A 是最小代价回退位；B 不建议。 -## 6. 定案前置问题（→ Plane issue 讨论） +## 6. 定案结论（ACM2-28 决策记录） -1. 生产 Redis/Valkey 拓扑定案：AOF 策略、副本、RPO 承诺——决定选项 A 是否可接受。 -2. FLIGHT_STATE 形态：FLID 主键 + FLTR JSONB 全量（建议起步），还是归一化列 - （`abdg`/`PSDT` 等派生字段是否有 SQL 查询面需求）。 -3. gen/SCHD_GEN 回 PG 后，U09「Redis Lua 协议重设计」是否直接取消，改为 SQL 版本 CAS - + Testcontainers 集成测试。 -4. 阶段 A 是否彻底移除 Redis 依赖：orms_stand 缓存去留、US-12 读取面、 - `redis-flight-store` 健康指示器调整。 -5. I2/I5 不变量重述文案与 U29 清单、architecture/design/user-stories 三文档的回改范围。 -6. 影子对拍跨存储 diff（nextgen PG vs legacy Redis）的验收口径与工具。 +| # | 问题 | 定案结论 | +|---|---|---| +| Q1 | 生产 Redis 拓扑 / RPO | **不再阻塞**。Redis 退出权威与写路径，不构成状态安全边界。 | +| Q2 | 表形态 | 表名定案 `FLIGHT_SCHD`。FLID 主键 + FLTR JSONB 全量 + FDAY 所属代列（可空）。派生字段后置。 | +| Q3 | U09 重定义 | 「Redis Lua 协议重设计」**取消**，改为「`SCHD_GEN` SQL 版本 CAS + Testcontainers PG 集成测试」。 | +| Q4 | Redis 去留 | 阶段 A 动态写路径归零；orms_stand 缓存不保留；US-12 读 PG；`redis-flight-store` 健康指示器移除。 | +| Q5 | 不变量重述 | I2（变更+事件+SUCCEEDED 单事务提交）；I4（覆盖+差删+版本推进单事务）；I5（单写者主泵线程，PG advisory lock 保护）。 | +| Q6 | 影子对拍口径 | nextgen PG `FLIGHT_SCHD` vs legacy Redis `flightInfo` 跨存储 AST 递归比对（FS7 工具）。 | -## 7. 定案后的落地清单（C 方案） +## 7. 落地完成清单（FS1–FS8 全部交付） -- 迁移 `V1.1.0__flight_state.sql`：FLIGHT_STATE + gen 列回归（含索引 FLID/日期）。 -- `FlightStateRepository` 重设计（FLID upsert / day replace / findByFlid）+ JDBC 实装。 -- `MessageProcessor`/`SnapshotFlow`：redisApply → PG 事务内；Lua 差删 → SQL。 -- Handler 视图与 US-12 读 PG；健康指示器、影子隔离（Redis key 前缀条目作废）同步。 -- 文档回改：本文转「已定案」，ACM2-12 阶段 A 口径修订说明。 +- **FS1** 迁移 `V1.1.0__flight_schd.sql`：创建 `FLIGHT_SCHD` 与 `SCHD_GEN`（TIMESTAMPTZ 规范）。 +- **FS2** `FlightSchdRepository` 接口与实装：快照批量写入、增量更新（ON CONFLICT 保留 FDAY）、域化差删、点查、`SCHD_GEN` SQL CAS、历史清场 `deleteByFlids`。 +- **FS3** `MessageProcessor` 事务 2 扩展：变更写入 + 事件写入 + `SUCCEEDED` 在单 PG 事务原子提交；Handler 视图按 FLID 点查。 +- **FS4** `SnapshotFlow` SQL 化：单事务内批处理 upsert + 域内差删 + SQL CAS 推进 + 事件入队 + 终态标记；`JobExecutor` 清场删除闭环。 +- **FS5** Redis 退役收口：删除 `FlightRedisClient`、Lua 脚本、`FlightRedisHealthIndicator`、`REDIS_FLIGHT_INFO` 投影与相关配置。 +- **FS6** U29/U09 不变量测试门禁：崩溃幂等无自增、CAS 防并发、ADFT 存活保障、UTC 方言测试、ES 历史清场 5 场景全绿。 +- **FS7** 影子对拍 Diff 工具：`FlightStoreDiffTool` 递归 AST 字段归一化比较与已知合法偏离识别。 +- **FS8** 文档回改：架构、设计、用户故事与项目规范同步收敛。 diff --git a/docs/design.md b/docs/design.md index a106f65..9b81100 100644 --- a/docs/design.md +++ b/docs/design.md @@ -4,7 +4,7 @@ 本文说明模块如何协作、状态如何流转，以及失败后如何恢复。系统范围、存储归属和部署约束见 [architecture.md](architecture.md)，不在这里重复。 -以下流程是阶段 A 的目标设计，不是实现完成清单。当前代码仍有占位和过渡实现，与设计的主要差异集中在第 10 节。阶段 B 的历史投影和清场暂不启用，也不改变 Redis 作为航班动态权威存储的定位。 +阶段 A 采纳 ACM2-28 选项 C 定案：运营航班权威状态落自有 PostgreSQL（表 `FLIGHT_SCHD` 与 `SCHD_GEN`），Redis 彻底退出动态权威与全部写路径。阶段 B 的历史投影和清场暂不启用。 ## 2. 数据与领域模型 @@ -19,12 +19,13 @@ | `PUMP_JOB` | 持久化维护作业 | 状态为 `QUEUED / RUNNING / DONE / FAILED`；不与业务消息共用排序序号。 | | `REQ_TRACK` | 上游请求及应答关联 | 保存请求类型、参数、出站信箱 ID、发送和完成时间；同类只允许一个开放请求。 | | `REF_MASTER` | 静态参考数据 | `(RTYPE, RKEY)` 唯一，`SOURCE` 记录数据来源。 | +| `FLIGHT_SCHD` | 运营航班当前权威状态（SCHD 快照 + FLOP/ADFT 增量合并） | `FLID` 主键，`FDAY` 所属日代（可空），`FLTR_JSON` 全量，`TIMESTAMPTZ` 时间戳。 | +| `SCHD_GEN` | 日计划代版本与当前代有效航班集合（差删依据） | `FDAY` 主键，`VERSION` 版本号，`FLIDS_JSON` 航班集合，`TIMESTAMPTZ`。 | | `PROC_STATE_HST` | 终态处理记录的归档目标 | 属于目标设计，当前迁移尚未建表；不得改写为共享库历史表。 | -字段与索引定义以 `src/main/resources/db/migration/` 为准。报文原文仍从共享信箱读取，因此必须协调原文保留期，不能在消息尚需处理或重放时提前清理。 - -Redis 保存 `flightInfo` 与快照代际元数据 `gen`。`gen` 记录快照所属日期、版本及该代航班集合，不属于静态参考数据。 +字段与索引定义以 `src/main/resources/db/migration/` 为准（含 `V1.1.0__flight_schd.sql`）。报文原文仍从共享信箱读取，因此必须协调原文保留期，不能在消息尚需处理或重放时提前清理。 +Redis 已彻底退出动态权威与写路径；`FLIGHT_SCHD` 与 `SCHD_GEN` 在自有 PG 中由主泵单线程独占写入。 ### 2.2 消息、身份与决策 `XmlCodec` 将 XML 解码为 `DecodedMessage`，包含 `SNDR / TYPE / STYP / SEQN / DTTM` 元数据和业务载荷。`MsgKind` 区分 `SCHD` 与 `FLOP` 子类型；Handler 查找与日志类型标识使用同一套映射。 @@ -44,7 +45,7 @@ Handler.decide(flightView, message) → Decision Decision = 航班变更 + msg 通知 + schd 状态 + 出站意图 + 静态数据变更 ``` -Handler 不写 Redis、Kafka 或数据库。主泵负责应用决策；各类变更的持久化与重试边界必须明确，不能把“返回了 Decision”当成副作用已执行。 +Handler 不写 Kafka 或数据库。主泵负责应用决策；各类变更在自有 PG 单事务内提交。 ### 2.3 状态与错误分类 @@ -97,16 +98,14 @@ Handler 不写 Redis、Kafka 或数据库。主泵负责应用决策；各类变 ├─ RESP / DNLD：快照流程 └─ 其他：Handler 决策 ↓ - Redis 应用变更 - ↓ - PG 本地事务：待发事件 + 处理终态 + PG 单事务：FLIGHT_SCHD 增量更新 + 待发事件 + 处理终态 ↓ 提交后补偿回填信箱 ``` - 原文缺失当前归为 `MALFORMED`；读取异常不能伪装成“缺失”，应进入基础设施重试。 - 忽略规则覆盖约定的 `LDM / REGN / RSTA / EROR`，转 `SKIPPED` 并审计；不能产生业务副作用。 -- Redis 更新必须先于对应事件提交。PG 事务只覆盖本库，不能靠事务注解把 Redis 或 MySQL 操作变成原子操作。 +- 航班变更（`FLIGHT_SCHD`）与待发事件、处理终态必须在同一 PG 事务原子提交。跨存储双写窗口已根除。 - 所有终态都需要回填信箱，包括成功、忽略、重复和死信；非终态禁止回填。回填必须在 PG 提交后执行，并有持久化补偿、退避与告警；影子环境禁写。 ## 4. 日计划快照与请求匹配 @@ -117,10 +116,10 @@ Handler 不写 Redis、Kafka 或数据库。主泵负责应用决策；各类变 1. **暂存校验**：流式解析后完成整包校验和航班规范化；失败前不修改权威状态。暂存数据可在崩溃后从原文重建。 2. **应答守卫**：`RESP` 必须匹配开放的 `RQFD` 请求；无匹配、已过期或报文时间早于发送时间时，转 `SKIPPED` 并审计，不更新快照。 -3. **原子替换**：Redis Lua 在同一次操作中校验版本、覆盖新代、删除旧代差集并推进 `gen`。删除集为“旧代航班集合 − 新代航班集合”，不是全部现存航班，不能误删快照集合外的增量航班。 -4. **提交结果**：在 PG 同一事务中保存 `MSG_EVENT`、将消息置为 `SUCCEEDED`，并将匹配 `RESP` 的请求置为 `DONE`；随后执行信箱回填。 +3. **SQL 原子替换**：在自有 PG 单事务内批处理写入新代 `FLIGHT_SCHD` 全量、按 `FDAY` 域内差删旧代航班，并通过 SQL CAS 推进 `SCHD_GEN` 版本。删除集为“旧代航班集合 − 新代航班集合”，仅作用于属于该代的记录，ADFT（FDAY=NULL）与已迁移航班天然存活。 +4. **提交结果**：在同一 PG 事务中保存 `MSG_EVENT`、将消息置为 `SUCCEEDED`，并将匹配 `RESP` 的请求置为 `DONE`；事务提交后执行信箱回填。 -关键恢复窗口是“Redis 已替换，PG 尚未提交”。重试同一快照必须识别已应用结果，不能再次增加版本，也不能用旧快照覆盖新状态。**版本校验与重放识别协议仍待实现验证**，单纯“读当前版本再加一”不满足要求。 +单事务保证崩溃后整体回滚，无跨存储中间态；重放时识别版本，绝不发生二次自增。 ### 4.2 上游请求与静态数据 @@ -194,13 +193,13 @@ REGISTERED → SENT → WAITING → DONE | 重复扫描、入队中断、较小 ID 迟到 | 不重复入队、不丢记录、不让后续消息越序。 | | 队头失败、退避及作业竞争 | 消息不越队；到期后恢复；作业不使消息无限饥饿。 | | 同身份多条记录、失败后重试、归档后重复 | 只产生一次有效业务处理，不把自身重试判为重复。 | -| Redis 成功后 PG 失败、快照重复或迟到 | 不重复推进版本、不回退状态、不误删增量航班。 | +| PG 事务失败、快照重复或迟到 | 整体回滚重试、不重复推进版本、不回退状态、不误删增量航班。 | | PG 提交失败、信箱回填失败 | 事件与处理结果一起回滚；已提交结果只补偿回填。 | | 投递确认丢失、批次失败、次数耗尽 | 允许可识别的重发、保持目标顺序、整批退避并保留死信。 | | 请求超时、无匹配 RESP、时间单位不一致 | 不误用迟到应答，不提前完成请求。 | | stub 误配置、重复实例、停机中断 | 生产拒绝不安全启动，工作线程能正确退出。 | -真实适配器还需 PG/MySQL 事务与补偿集成测试、Redis Lua 中断恢复测试、Kafka 故障投递验证；启动冒烟只证明装配可用，不证明这些一致性要求已满足。常规验证命令为 JDK 25 下执行 `./gradlew test`。 +真实适配器还需 PG 事务与补偿集成测试、快照 SQL CAS 崩溃恢复测试、Kafka 故障投递验证；常规验证命令为 JDK 25 下执行 `./gradlew test`。 ## 9. 实现入口 @@ -219,10 +218,10 @@ REGISTERED → SENT → WAITING → DONE 以下缺口直接影响上述设计是否成立，不能以类或接口已存在作为完成依据： -- **事务与外部副作用**：当前 JDBC 操作仍分散执行；普通处理按“插事件 → 同步回填信箱 → 更新状态”调用，尚未实现要求的 PG 原子提交与提交后补偿。Redis 普通变更、出站意图及真实出站信箱也未形成完整链路。 +- **事务与外部副作用**：ACM2-28 定案后，已完成 `FLIGHT_SCHD` 增量更新与快照全量写入、待发事件与 `PROC_STATE` 在自有 PG 单事务原子提交，提交后异步回填信箱。 - **收报与调度**：轮询仍从 `afterId=0` 扫描，持久水位与补扫策略未完成；作业以“队头为 FAILED”近似窗口，未区分是否已到期。主泵直取系统时间，滞留判据使用更新时刻，不能保证设计要求的超时升级。 -- **快照与业务能力**：快照暂存仍为占位，当前分流仅覆盖 DNLD，RESP 守卫与请求完成事务未接通；`gen` 仍走过渡仓储，未实现 Redis 内原子版本与重放协议。忽略规则及所需 Handler 还需补齐。 +- **快照与业务能力**：快照 SQL 事务化已落地，支持 JDBC 批处理、域内差删与 SQL CAS 推进，超限 10000 熔断保护；忽略规则及所需 Handler 仍需在后续阶段铺开。 - **请求、静态数据与归档**：请求在真实出站前就标记发送，时间匹配与审计仍需修正；静态数据存在进程内过渡实现，归档表与关联保留策略尚未落地。 -- **生产与运维**：缺少完整启动校验、运行期单写者保护、影子隔离和 Kafka 强制配置校验；投递滞留升级、死信告警、端到端追踪与指标未闭环。 +- **生产与运维**：缺少完整启动校验、运行期单写者保护、影子隔离和 Kafka 强制配置校验；投递滞留升级、死信告警、端到端追踪与指标未闭环。影子对拍 cross-store diff 工具已就绪。 进度与验收项见 Plane ACM2-10 实施计划（U01–U30），业务契约与待确认事项见 [user-stories.md](user-stories.md)。本文件不维护工单流水账、测试数量或历史方案全文。 diff --git a/docs/user-stories.md b/docs/user-stories.md index 9ee3c4a..6108d5b 100644 --- a/docs/user-stories.md +++ b/docs/user-stories.md @@ -111,12 +111,12 @@ 1. `SCHD-ADFT` 与 29 个 FLOP 子类型逐项列入覆盖矩阵，每项有纯函数 Handler；未知类型可恢复失败。RESP/DNLD 不计入这批 Handler，走 US-06。 2. 每类固定“输入与前态 → 后态 → msg → schd → 终态”五面样例；区分字段缺失、显式清空、重复报文和主/共享航班。清单和 golden 样例按 Q8 补齐，不以“已写 29 个类”替代验收。 -3. 对按 KEEP 规则需忽略的不存在航班，以 `SUCCEEDED` 无副作用结束，并由 US-09 回填；ADFT 建航班等行为按各类型矩阵执行。此规则只适用于权威状态健康时。 +3. 对按 KEEP 规则需忽略的不存在航班，以 `SUCCEEDED` 无副作用结束，并由 US-09 回填；ADFT 建航班等行为按各类型矩阵执行。在 ACM2-28 运营航班落自有 PG 后，Redis 全损导致报文持续终结与白名单无法找回的 F1 损坏路径已被根除，PG 作为唯一权威重启即恢复。 4. 共享航班通常更新并通知主航班，不直接发共享通知。FDEL 删除共享航班时更新主航班 MAFL 并通知；删除主航班时删除主航班及其子共享关联并发删除通知；目标不存在幂等成功。 5. ADFT/FDEL 使用值相等比较；主/共享关系一次原子变更，不出现主已删、子残留等半状态。 6. PSDT 通过 US-14 的只读映射计算 `abdg`，Handler 不直接调用 admin-api。 -**当前基础与落点**：`processing/Handler.kt`、`domain/Decision.kt`、`infra/redis/` 只有主要扩展点；按类型新增 Handler 与同包测试。先做 ADFT + 一个普通 FLOP，再做主/共享及 PSDT，最后补齐矩阵。 +**当前基础与落点**：`processing/Handler.kt`、`domain/Decision.kt`、`infra/persistence/FlightSchdRepository.kt` 为主要落点；变更直接入自有 PG 事务 2。按类型新增 Handler 与同包测试。 **前置**：US-03；PSDT 另依赖 US-14；Q1、Q8。 @@ -134,10 +134,9 @@ 1. RESP/DNLD 共用流式解析、整包校验和规范化；校验失败不发布半包，旧快照保持可用。 2. RESP 仅匹配未过期、已发送的开放 RQFD；`DTTM < SENT_AT`、已过期、已被替代或无匹配时，不写业务状态，记录跳过原因。 -3. 当前 Redis 基线下，Lua 原子执行新代写入、旧代差集删除和版本校验/推进；删除集只来自上一代成员，不误删集合外的 ADFT 航班。 -4. 随后在一个 PG 事务提交 `SUCCEEDED`、待发通知、回填意图；匹配 RESP 同事务完成请求，并保存完成时间与应答信箱 ID。 -5. 相同报文重放不二次增代或删数据；覆盖首次/连续快照、DNLD→ADFT→DNLD′、旧应答与 Redis 成功/PG 失败窗口。不能只靠“读取当前版本再加一”实现重放识别。 - +3. 在自有 PG 单事务内，批处理写入新代 `FLIGHT_SCHD`、按代域化差删旧代航班并执行 SQL CAS 推进 `SCHD_GEN` 版本；删除集只来自上一代成员，不误删集合外的 ADFT 航班。 +4. 在同一 PG 事务中提交 `FLIGHT_SCHD` 变更、`MSG_EVENT` 待发通知与 `PROC_STATE(SUCCEEDED)`；匹配 RESP 同事务完成请求并置 `DONE`；事务提交后执行信箱回填。 +5. 相同报文重放不二次增代或删数据；单事务崩溃整体回滚，重放幂等（FS6 门槛 1/2/3 验证通过）。 **当前基础与落点**：`processing/SnapshotFlow.kt` 的 staging 为占位，主处理只分流 DNLD，gen 走进程内过渡仓储。需补 RESP 路由、版本协议及 `REQ_TRACK` 应答关联字段和事务测试。 **前置**：US-03、US-08 请求登记/匹配基础；Q1、Q5。 @@ -231,7 +230,7 @@ **验收标准** -1. 保留 `GET /all/flights`，过滤 `MAID != NULL` 的共享航班；不改写业务状态。 +1. 保留 `GET /all/flights`，直接从自有 PostgreSQL `FLIGHT_SCHD` 查询（ACM2-28 定案：读源由 Redis 转为 PG），过滤 `MAID != NULL` 的共享航班；不改写业务状态。 2. 固定响应样例、空结果、排序、大小限制及一致性时点。现役未分页时不能无声改为只返回第一页；分页或响应结构变更按 Q3 决定。 3. 依赖异常不能伪装为空数组成功；影子只读影子状态，入口有约定的访问控制、限流与审计。 diff --git a/src/main/kotlin/com/gzzn/omms/msgexchange/delivery/Dispatcher.kt b/src/main/kotlin/com/gzzn/omms/msgexchange/delivery/Dispatcher.kt index bd327c6..bfc1a52 100644 --- a/src/main/kotlin/com/gzzn/omms/msgexchange/delivery/Dispatcher.kt +++ b/src/main/kotlin/com/gzzn/omms/msgexchange/delivery/Dispatcher.kt @@ -20,8 +20,6 @@ interface DeliveryPort { /** 阶段 B：ES flight_hts 写入。 */ fun indexFlightHts(payloadJson: String) - /** 阶段 B：Redis 投影写（仅阶段 B；I5：Delivery 阶段 A 不写 Redis）。 */ - fun projectRedis(payloadJson: String) /** 连通性探测（健康检查用）；默认 true，真实 Kafka 实装时覆写为 producer metadata 校验。 */ fun ping(): Boolean = true @@ -83,10 +81,7 @@ class Dispatcher( deliver(t, head) msgEvents.markSent(head.eventId!!) log.debug("sent target={} eventId={}", t, head.eventId) - if (t == Targets.ES_FLIGHT_HTS && props.phase == PipelineProps.Phase.B) { - // 定案 2：ES 投递成功 → 同线程同步 enqueue 删除事件（不轮询 ack） - msgEvents.insertSync(listOf(MsgEvent(target = Targets.REDIS_FLIGHT_INFO, payloadJson = deleteOf(head)))) - } + // ACM2-28：阶段 B Redis 投影废弃，读模型投递统一转由 Kafka 事件或 ES 处理 } catch (e: Exception) { retryOrDead(head, e.message ?: "unknown") } @@ -115,19 +110,9 @@ class Dispatcher( Targets.KAFKA_MSG -> port.sendKafka("msg", e.payloadJson) Targets.KAFKA_SCHD -> error("KAFKA_SCHD must go through flushSchd (N03)") Targets.ES_FLIGHT_HTS -> port.indexFlightHts(e.payloadJson) - Targets.REDIS_FLIGHT_INFO -> port.projectRedis(e.payloadJson) else -> error("unknown target $target") } - private val jsonMapper = ObjectMapper() - - /** - * U14：删除事件 wire JSON 结构化序列化——refs 可空，产出必须恒为合法 JSON - * （null → null 字面量；非空 → 带引号并转义）。禁止字符串模板拼接。 - */ - private fun deleteOf(e: MsgEvent): String = - jsonMapper.writeValueAsString(linkedMapOf("op" to "delete", "refs" to e.partitionKey)) - /** * 流程 3 flushSchd：批上限 + 每 FLID 最新一态聚合（topic "schd"，wire=FLTR JSON 数组）。 * U08 批量闭环：队首退避未到期不 claim；发送失败 → 整批 attempts+1（退避）或达上限整批 DEAD/DLQ； diff --git a/src/main/kotlin/com/gzzn/omms/msgexchange/domain/Decision.kt b/src/main/kotlin/com/gzzn/omms/msgexchange/domain/Decision.kt index 65a2f8c..ff68c86 100644 --- a/src/main/kotlin/com/gzzn/omms/msgexchange/domain/Decision.kt +++ b/src/main/kotlin/com/gzzn/omms/msgexchange/domain/Decision.kt @@ -2,7 +2,7 @@ package com.gzzn.omms.msgexchange.domain /** * ACMA-8 流程 2：Handler 决策（纯函数）产物——状态与报文进，变更与事件出， - * 不直接触碰 Redis/Kafka。 + * 不直接触碰数据库或 Kafka。 */ data class Decision( val flightChanges: List = emptyList(), @@ -12,7 +12,7 @@ data class Decision( val refUpserts: List = emptyList(), // → 静态主数据（独立 PG reference 库，ACM2-11） ) -/** 航班状态变更（阶段 A 由主泵线程 redisApply；阶段 B 落 FLIGHT_STATE 同事务）。 */ +/** 航班状态变更（阶段 A 落自有 PG FLIGHT_SCHD，与事件同事务原子提交；ACM2-28 定案）。 */ data class FlightChange( val flid: String, val payloadJson: String, diff --git a/src/main/kotlin/com/gzzn/omms/msgexchange/domain/MsgEvent.kt b/src/main/kotlin/com/gzzn/omms/msgexchange/domain/MsgEvent.kt index 54bcb83..5117302 100644 --- a/src/main/kotlin/com/gzzn/omms/msgexchange/domain/MsgEvent.kt +++ b/src/main/kotlin/com/gzzn/omms/msgexchange/domain/MsgEvent.kt @@ -8,13 +8,14 @@ object Targets { const val KAFKA_MSG = "KAFKA:msg" const val KAFKA_SCHD = "KAFKA:schd" const val ES_FLIGHT_HTS = "ES:flight_hts" + @Deprecated("Retired in ACM2-28: Redis projection removed") const val REDIS_FLIGHT_INFO = "REDIS:flightInfo" - /** 阶段 A 投递目标（仅 Kafka）——I5：Delivery 阶段 A 不写 Redis。 */ + /** 阶段 A 投递目标（仅 Kafka）——ACM2-28：Delivery 阶段 A/B 均不写 Redis。 */ val phaseA: List = listOf(KAFKA_MSG, KAFKA_SCHD) - /** 阶段 B 追加投影目标。 */ - val phaseB: List = phaseA + listOf(ES_FLIGHT_HTS, REDIS_FLIGHT_INFO) + /** 阶段 B 追加投影目标（仅 ES 历史库；Redis 投影按 ACM2-28 废弃）。 */ + val phaseB: List = phaseA + listOf(ES_FLIGHT_HTS) } enum class EventStatus { PENDING, SENT, DEAD } diff --git a/src/main/kotlin/com/gzzn/omms/msgexchange/infra/health/HealthIndicators.kt b/src/main/kotlin/com/gzzn/omms/msgexchange/infra/health/HealthIndicators.kt index af3657c..8758ced 100644 --- a/src/main/kotlin/com/gzzn/omms/msgexchange/infra/health/HealthIndicators.kt +++ b/src/main/kotlin/com/gzzn/omms/msgexchange/infra/health/HealthIndicators.kt @@ -1,7 +1,6 @@ package com.gzzn.omms.msgexchange.infra.health import com.gzzn.omms.msgexchange.delivery.DeliveryPort -import com.gzzn.omms.msgexchange.infra.redis.FlightRedisClient import io.micronaut.context.BeanProvider import io.micronaut.core.async.publisher.Publishers import io.micronaut.health.HealthStatus @@ -11,20 +10,12 @@ import jakarta.inject.Singleton import org.reactivestreams.Publisher /** - * U12（R05）：阶段 A 关键依赖的自定义健康指示器—— - * Redis（flightInfo 权威存储）与 Kafka（投递端口）。经 BeanProvider 可选解析： + * U12（R05）：自定义健康指示器—— + * Kafka（投递端口）。经 BeanProvider 可选解析： * 缺 bean（如未用 stub 也未实装）时指示 DOWN 而非启动失败； * UP 判据为真实 ping（false/异常 → DOWN），而非仅 bean 存在（复审 P1 修正）。 + * （ACM2-28：Redis 退出阶段 A 权威与写路径，redis-flight-store 指示器移除）。 */ -@Singleton -class FlightRedisHealthIndicator( - private val redis: BeanProvider, -) : HealthIndicator { - - override fun getResult(): Publisher = - Publishers.just(redisHealth(if (redis.isPresent) redis.get() else null)) -} - @Singleton class KafkaDeliveryHealthIndicator( private val port: BeanProvider, @@ -34,12 +25,6 @@ class KafkaDeliveryHealthIndicator( Publishers.just(kafkaHealth(if (port.isPresent) port.get() else null)) } -/** ping 判定独立成纯函数便于单测：client 为 null = bean 缺失；ping false/异常 = DOWN。 */ -internal fun redisHealth(client: FlightRedisClient?): HealthResult = - healthOf("redis-flight-store", "flight store", client?.let { - try { it.ping() } catch (e: Exception) { false } - }) - internal fun kafkaHealth(port: DeliveryPort?): HealthResult = healthOf("kafka-delivery", "delivery port", port?.let { try { it.ping() } catch (e: Exception) { false } diff --git a/src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/Repositories.kt b/src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/Repositories.kt index f5fc54a..9e1eecb 100644 --- a/src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/Repositories.kt +++ b/src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/Repositories.kt @@ -14,7 +14,7 @@ import java.time.Instant * 的全部接口（消息管道 PROC_STATE/MSG_EVENT、PUMP_JOB、REQ_TRACK、21 类 REF_MASTER）； * 共享 MySQL 信箱（CMINMSGS / COUTMSGS）经信箱封装访问，仅 DML、不建表； * 主路径=上游外部写 CMINMSGS → 本系统 JDBC 轮询读；compat=insertRaw HTTP 写； - * Redis = 航班动态 + 快照 gen（RefDataRepository 目标实现）；FLIGHT_STATE 缓做。 + * 自有 PG = 消息管道 + 运营航班 FLIGHT_SCHD/SCHD_GEN + 静态数据；Redis 已退出阶段 A 权威与写路径。 */ interface ProcStateRepository { fun insert(cminmsgsId: Long, state: ProcStatus = ProcStatus.PENDING) @@ -68,25 +68,88 @@ interface MsgEventRepository { } /** - * 快照 generation（SCHD_GEN）协议——只留 gen。 - * ACM2-12：gen 迁 Redis（与 flightInfo 同源，Lua 内原子「覆盖+按代差删+版本推进」， - * DB 仅写 SUCCEEDED；重放幂等由 Lua 承接，协议重设计属 U09）。本接口为过渡占位， - * 目标实现为 Redis gen store（script 化），非关系表。 + * 阶段 A 运营航班权威与日计划代（ACM2-28 采纳选项 C 定案）： + * - 表 FLIGHT_SCHD：当前运营航班全量权威态（SCHD 快照 + FLOP/ADFT 增量合并），落自有 PostgreSQL； + * - 表 SCHD_GEN：各日代版本与当前代有效航班全量集合（差删依据），由 Redis 回归自有 PG； + * - Redis 退出动态权威与全部写路径； + * - 事务 2 与快照发布全在自有 PG 内以单事务原子提交； + * - 增量更新（FLOP/ADFT）：新插 FDAY=NULL，已有行通过 ON CONFLICT 保留原 FDAY； + * - 按代差删域化：DELETE FROM FLIGHT_SCHD WHERE FDAY = :day AND FLID = ANY(:diffSet) + * 仅删除仍属旧代的行，ADFT（FDAY=NULL）与已迁移至新代的同 FLID 行天然存活。 */ -interface RefDataRepository { - data class GenMeta(val flids: List, val version: Long) +interface FlightSchdRepository { + data class FlightRecord( + val flid: String, + val fday: String?, + val fltrJson: String, + val createdAt: Instant, + val updatedAt: Instant, + ) + data class GenMeta( + val fday: String, + val version: Long, + val flids: Set, + val updatedAt: Instant = Instant.now(), + ) + + /** 快照全量写入（DNLD）：强行声明/更新 FDAY 归属，批处理写入。 */ + fun upsertSnapshotBatch(day: String, flights: List>, now: Instant = Instant.now()) + + /** 增量更新（FLOP/ADFT）：新插 FDAY=NULL，已有行保留原 FDAY。 */ + fun upsertIncremental(changes: List, now: Instant = Instant.now()) + + /** 按代差删域化：仅删除 FDAY = day 且在 delFlids 中的记录（ADFT 与跨代已迁移行受保护）。 */ + fun deleteDiffByDay(day: String, delFlids: Collection): Int + + /** 点查单航班 FLTR_JSON。 */ + fun findByFlid(flid: String): String? + + /** 点查多航班 FLTR_JSON。 */ + fun findByFlids(flids: Collection): Map + + /** 按计划日查询当前有效航班。 */ + fun findByDay(day: String): List> + + /** 全量查询（供影子对拍 / 一致性对账）。 */ + fun findAll(): Map + + /** + * 历史清场删除：仅删除已确认归档至 ES 的 FLID 集合；空集合不执行；分批参数化删除。 + * 返回实际删除行数（允许重放时为 0）。 + */ + fun deleteByFlids(flids: Set): Int + + /** 读计划代（SCHD_GEN）。 */ fun getGen(day: String): GenMeta? - /** 流程 4：版本 CAS（expected 未变才写，重放 no-op，不二次自增）。 */ - fun putGenIfVersion(day: String, expected: Long, new: GenMeta): Boolean + /** + * 计划代 SQL 版本 CAS：仅当 expected 与数据库中当前版本一致（或不存在且 expected=0）时写入/推进新版本。 + * 返回 true 表示推进成功，false 表示发生 CAS 冲突。 + */ + fun putGenIfVersion(day: String, expected: Long, newGen: GenMeta, now: Instant = Instant.now()): Boolean + + /** + * 历史代清理：清理 cutoffDay 之前的历史代记录（与 FLIGHT_SCHD 历史清场生命周期对齐）。 + */ + fun deleteGenBefore(cutoffDay: String): Int } +/** 事务管理器抽象：自有 PG 单事务原子保障。 */ +interface PipelineTransactionManager { + fun inTransaction(block: () -> T): T +} + +@Deprecated("Replaced by FlightSchdRepository in ACM2-28", ReplaceWith("FlightSchdRepository")) +interface RefDataRepository { + data class GenMeta(val flids: List, val version: Long) + fun getGen(day: String): GenMeta? + fun putGenIfVersion(day: String, expected: Long, new: GenMeta): Boolean +} /** * 21 类静态主数据（航空公司/航线/机位/登机桥等）——自有 PostgreSQL `REF_MASTER` 表 * （ACM2-12：与消息管道同自有库；SOURCE=ADMINAPI/AODB/PIPELINE，N19 对齐）。 - * 与主链弱事务耦合：写入者为 ReferenceService（21 类同步）与请求应答路径； - * Redis 只作只读热点投影（legacy orms_stand 语义延续，阶段 6）。 + * 与主链弱事务耦合：写入者为 ReferenceService（21 类同步）与请求应答路径。 */ interface StaticRefRepository { fun upsertAll(refs: List) @@ -124,7 +187,7 @@ interface ReqTrackRepository { /** * 泵作业调度记录——自有 PG `PUMP_JOB`（ACM2-12）。 * 决策 1 修订：作业不插队，仅在消息队头空闲/退避窗口由主泵执行（跨库/异队列无全序）； - * 作业动作本身（归档写共享库 CMINMSGS_HST、清场删 Redis/ES 等）仍在各自目标存储。 + * 作业动作本身（归档写共享库 CMINMSGS_HST、清场删 ES+PG 等）仍在各自目标存储。 */ interface PumpJobRepository { data class Job(val jobId: Long, val kind: String) // ARCHIVE/HISTORY_SWEEP/PROJECTION_REBUILD @@ -141,9 +204,9 @@ interface PumpJobRepository { } /** - * 阶段 B 权威（ACM2-12：缓做，不落表——航班动态权威保持 Redis；阶段 B 重新 - * 评估后再定是否引入事务化权威）。本接口仅供占位与测试，勿据此建表。 + * 历史占位接口（已被 ACM2-28 之 FlightSchdRepository 取代，保留供兼容与过渡）。 */ +@Deprecated("Replaced by FlightSchdRepository in ACM2-28", ReplaceWith("FlightSchdRepository")) interface FlightStateRepository { /** 阶段 B 权威；replaceDay = 单事务删差集+写新代+版本提升。 */ fun replaceDay(day: String, flights: List>) diff --git a/src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/JdbcOps.kt b/src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/JdbcOps.kt index af811f0..5455808 100644 --- a/src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/JdbcOps.kt +++ b/src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/JdbcOps.kt @@ -10,9 +10,51 @@ internal fun Instant.toSqlTimestamp(): Timestamp = Timestamp.from(this) internal fun ResultSet.getInstant(column: String): Instant? = getTimestamp(column)?.toInstant() -internal fun DataSource.query(sql: String, bind: (java.sql.PreparedStatement) -> Unit, map: (ResultSet) -> T): List = - connection.use { conn -> - conn.prepareStatement(sql).use { ps -> +private val transactionConnection = ThreadLocal() + +internal fun DataSource.withTransaction(block: () -> T): T { + val existing = transactionConnection.get() + if (existing != null) { + return block() + } + val conn = this.connection + val oldAutoCommit = conn.autoCommit + conn.autoCommit = false + transactionConnection.set(conn) + try { + val result = block() + conn.commit() + return result + } catch (t: Throwable) { + try { + conn.rollback() + } catch (rbEx: Throwable) { + t.addSuppressed(rbEx) + } + throw t + } finally { + transactionConnection.remove() + try { + conn.autoCommit = oldAutoCommit + } catch (_: Throwable) { + } + conn.close() + } +} + +internal fun DataSource.obtainConnection(): java.sql.Connection = + transactionConnection.get() ?: this.connection + +internal fun java.sql.Connection.releaseIfNotInTransaction() { + if (transactionConnection.get() !== this) { + this.close() + } +} + +internal fun DataSource.query(sql: String, bind: (java.sql.PreparedStatement) -> Unit, map: (ResultSet) -> T): List { + val conn = obtainConnection() + try { + return conn.prepareStatement(sql).use { ps -> bind(ps) ps.executeQuery().use { rs -> buildList { @@ -20,22 +62,30 @@ internal fun DataSource.query(sql: String, bind: (java.sql.PreparedStatement } } } + } finally { + conn.releaseIfNotInTransaction() } +} internal fun DataSource.queryOne(sql: String, bind: (java.sql.PreparedStatement) -> Unit, map: (ResultSet) -> T): T? = query(sql, bind, map).firstOrNull() -internal fun DataSource.update(sql: String, bind: (java.sql.PreparedStatement) -> Unit): Int = - connection.use { conn -> - conn.prepareStatement(sql).use { ps -> +internal fun DataSource.update(sql: String, bind: (java.sql.PreparedStatement) -> Unit): Int { + val conn = obtainConnection() + try { + return conn.prepareStatement(sql).use { ps -> bind(ps) ps.executeUpdate() } + } finally { + conn.releaseIfNotInTransaction() } +} -internal fun DataSource.updateReturningLong(sql: String, bind: (java.sql.PreparedStatement) -> Unit): Long = - connection.use { conn -> - conn.prepareStatement(sql, java.sql.Statement.RETURN_GENERATED_KEYS).use { ps -> +internal fun DataSource.updateReturningLong(sql: String, bind: (java.sql.PreparedStatement) -> Unit): Long { + val conn = obtainConnection() + try { + return conn.prepareStatement(sql, java.sql.Statement.RETURN_GENERATED_KEYS).use { ps -> bind(ps) ps.executeUpdate() ps.generatedKeys.use { keys -> @@ -43,4 +93,7 @@ internal fun DataSource.updateReturningLong(sql: String, bind: (java.sql.Prepare keys.getLong(1) } } + } finally { + conn.releaseIfNotInTransaction() } +} diff --git a/src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/JdbcPgRepositories.kt b/src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/JdbcPgRepositories.kt index 3491606..424ed4b 100644 --- a/src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/JdbcPgRepositories.kt +++ b/src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/JdbcPgRepositories.kt @@ -6,8 +6,10 @@ import com.gzzn.omms.msgexchange.domain.MsgEvent import com.gzzn.omms.msgexchange.domain.ProcState import com.gzzn.omms.msgexchange.domain.ProcStatus import com.gzzn.omms.msgexchange.domain.RefUpsert +import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository +import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository import com.gzzn.omms.msgexchange.infra.persistence.PumpJobRepository import com.gzzn.omms.msgexchange.infra.persistence.RefDataRepository @@ -286,21 +288,242 @@ class JdbcPumpJobRepository( } } -/** gen→Redis 过渡占位：U09 前进程内 CAS（与 StubRefData 同语义）。 */ +/** 自有 PostgreSQL 单事务管理器（ACM2-28 事务 2 与快照发布原子提交）。 */ @Singleton @Requires(property = "msgx.stubs", notEquals = "true") @Requires(property = "datasources.default.enabled", value = "true") -class JdbcRefDataRepository : RefDataRepository { - private val gens = mutableMapOf() +class JdbcPipelineTransactionManager( + private val ds: DataSource, +) : PipelineTransactionManager { + override fun inTransaction(block: () -> T): T = ds.withTransaction(block) +} - override fun getGen(day: String): RefDataRepository.GenMeta? = gens[day] +@Singleton +@Requires(property = "msgx.stubs", notEquals = "true") +@Requires(property = "datasources.default.enabled", value = "true") +class JdbcFlightSchdRepository( + private val ds: DataSource, +) : FlightSchdRepository { + private val flidsTypeRef = object : com.fasterxml.jackson.core.type.TypeReference>() {} + private val mapper = com.fasterxml.jackson.databind.ObjectMapper() - override fun putGenIfVersion(day: String, expected: Long, new: RefDataRepository.GenMeta): Boolean { - val cur = gens[day]?.version ?: 0L - if (cur != expected) return false - gens[day] = new - return true + private fun serializeFlids(flids: Collection): String = mapper.writeValueAsString(flids) + + private fun parseFlids(json: String): Set { + if (json.isBlank()) return emptySet() + return try { + mapper.readValue(json, flidsTypeRef) + } catch (_: Exception) { + emptySet() + } } + + private fun toSqlDate(day: String): java.sql.Date = + java.sql.Date.valueOf(day.trim().take(10)) + + override fun upsertSnapshotBatch(day: String, flights: List>, now: Instant) { + if (flights.isEmpty()) return + val sql = """ + INSERT INTO flight_schd (flid, fday, fltr_json, created_at, updated_at) + VALUES (?, ?, ?::jsonb, ?, ?) + ON CONFLICT (flid) DO UPDATE SET + fday = EXCLUDED.fday, + fltr_json = EXCLUDED.fltr_json, + updated_at = EXCLUDED.updated_at + """.trimIndent() + val sqlDate = toSqlDate(day) + val sqlTimestamp = now.toSqlTimestamp() + val conn = ds.obtainConnection() + try { + conn.prepareStatement(sql).use { ps -> + var count = 0 + for ((flid, json) in flights) { + ps.setString(1, flid) + ps.setDate(2, sqlDate) + ps.setString(3, json) + ps.setTimestamp(4, sqlTimestamp) + ps.setTimestamp(5, sqlTimestamp) + ps.addBatch() + count++ + if (count % 200 == 0) { + ps.executeBatch() + } + } + if (count % 200 != 0) { + ps.executeBatch() + } + } + } finally { + conn.releaseIfNotInTransaction() + } + } + + override fun upsertIncremental(changes: List, now: Instant) { + if (changes.isEmpty()) return + val sql = """ + INSERT INTO flight_schd (flid, fday, fltr_json, created_at, updated_at) + VALUES (?, NULL, ?::jsonb, ?, ?) + ON CONFLICT (flid) DO UPDATE SET + fltr_json = EXCLUDED.fltr_json, + updated_at = EXCLUDED.updated_at + """.trimIndent() + val sqlTimestamp = now.toSqlTimestamp() + val conn = ds.obtainConnection() + try { + conn.prepareStatement(sql).use { ps -> + var count = 0 + for (c in changes) { + ps.setString(1, c.flid) + ps.setString(2, c.payloadJson) + ps.setTimestamp(3, sqlTimestamp) + ps.setTimestamp(4, sqlTimestamp) + ps.addBatch() + count++ + if (count % 200 == 0) { + ps.executeBatch() + } + } + if (count % 200 != 0) { + ps.executeBatch() + } + } + } finally { + conn.releaseIfNotInTransaction() + } + } + + override fun deleteDiffByDay(day: String, delFlids: Collection): Int { + if (delFlids.isEmpty()) return 0 + var totalDeleted = 0 + val sqlDate = toSqlDate(day) + for (chunk in delFlids.chunked(200)) { + val placeholders = chunk.joinToString(",") { "?" } + val sql = "DELETE FROM flight_schd WHERE fday = ? AND flid IN ($placeholders)" + totalDeleted += ds.update(sql) { ps -> + ps.setDate(1, sqlDate) + chunk.forEachIndexed { i, flid -> ps.setString(i + 2, flid) } + } + } + return totalDeleted + } + + override fun findByFlid(flid: String): String? = + ds.queryOne( + "SELECT fltr_json FROM flight_schd WHERE flid = ?", + { ps -> ps.setString(1, flid) }, + ) { rs -> rs.getString("fltr_json") } + + override fun findByFlids(flids: Collection): Map { + if (flids.isEmpty()) return emptyMap() + val result = mutableMapOf() + for (chunk in flids.chunked(200)) { + val placeholders = chunk.joinToString(",") { "?" } + val sql = "SELECT flid, fltr_json FROM flight_schd WHERE flid IN ($placeholders)" + val pairs = ds.query( + sql, + { ps -> chunk.forEachIndexed { i, flid -> ps.setString(i + 1, flid) } }, + ) { rs -> rs.getString("flid") to rs.getString("fltr_json") } + result.putAll(pairs) + } + return result + } + + override fun findByDay(day: String): List> = + ds.query( + "SELECT flid, fltr_json FROM flight_schd WHERE fday = ? ORDER BY flid ASC", + { ps -> ps.setDate(1, toSqlDate(day)) }, + ) { rs -> rs.getString("flid") to rs.getString("fltr_json") } + + override fun findAll(): Map = + ds.query( + "SELECT flid, fltr_json FROM flight_schd ORDER BY flid ASC", + {}, + ) { rs -> rs.getString("flid") to rs.getString("fltr_json") }.toMap() + + override fun deleteByFlids(flids: Set): Int { + if (flids.isEmpty()) return 0 + var totalDeleted = 0 + for (chunk in flids.chunked(200)) { + val placeholders = chunk.joinToString(",") { "?" } + val sql = "DELETE FROM flight_schd WHERE flid IN ($placeholders)" + totalDeleted += ds.update(sql) { ps -> + chunk.forEachIndexed { i, flid -> ps.setString(i + 1, flid) } + } + } + return totalDeleted + } + + override fun getGen(day: String): FlightSchdRepository.GenMeta? = + ds.queryOne( + "SELECT fday, version, flids_json, updated_at FROM schd_gen WHERE fday = ?", + { ps -> ps.setDate(1, toSqlDate(day)) }, + ) { rs -> + FlightSchdRepository.GenMeta( + fday = rs.getDate("fday").toString(), + version = rs.getLong("version"), + flids = parseFlids(rs.getString("flids_json")), + updatedAt = rs.getInstant("updated_at") ?: Instant.now(), + ) + } + + override fun putGenIfVersion(day: String, expected: Long, newGen: FlightSchdRepository.GenMeta, now: Instant): Boolean { + val flidsJson = serializeFlids(newGen.flids) + val sqlDate = toSqlDate(day) + val sqlTimestamp = now.toSqlTimestamp() + if (expected == 0L) { + val inserted = ds.update( + """ + INSERT INTO schd_gen (fday, version, flids_json, updated_at) + VALUES (?, ?, ?::jsonb, ?) + ON CONFLICT (fday) DO NOTHING + """.trimIndent(), + ) { ps -> + ps.setDate(1, sqlDate) + ps.setLong(2, newGen.version) + ps.setString(3, flidsJson) + ps.setTimestamp(4, sqlTimestamp) + } + if (inserted == 1) return true + } + + val updated = ds.update( + """ + UPDATE schd_gen + SET version = ?, flids_json = ?::jsonb, updated_at = ? + WHERE fday = ? AND version = ? + """.trimIndent(), + ) { ps -> + ps.setLong(1, newGen.version) + ps.setString(2, flidsJson) + ps.setTimestamp(3, sqlTimestamp) + ps.setDate(4, sqlDate) + ps.setLong(5, expected) + } + return updated == 1 + } + + override fun deleteGenBefore(cutoffDay: String): Int = + ds.update( + "DELETE FROM schd_gen WHERE fday < ?", + { ps -> ps.setDate(1, toSqlDate(cutoffDay)) }, + ) +} + +@Singleton +@Requires(property = "msgx.stubs", notEquals = "true") +@Requires(property = "datasources.default.enabled", value = "true") +class JdbcRefDataRepository( + private val flightSchd: FlightSchdRepository, +) : RefDataRepository { + override fun getGen(day: String): RefDataRepository.GenMeta? = + flightSchd.getGen(day)?.let { RefDataRepository.GenMeta(it.flids.toList(), it.version) } + + override fun putGenIfVersion(day: String, expected: Long, new: RefDataRepository.GenMeta): Boolean = + flightSchd.putGenIfVersion( + day, + expected, + FlightSchdRepository.GenMeta(day, new.version, new.flids.toSet()), + ) } @Singleton @@ -420,7 +643,12 @@ class JdbcReqTrackRepository( @Singleton @Requires(property = "msgx.stubs", notEquals = "true") @Requires(property = "datasources.default.enabled", value = "true") -class JdbcFlightStateRepository : FlightStateRepository { - override fun replaceDay(day: String, flights: List>) = Unit - override fun findByDay(day: String): List> = emptyList() +class JdbcFlightStateRepository( + private val flightSchd: FlightSchdRepository, +) : FlightStateRepository { + override fun replaceDay(day: String, flights: List>) = + flightSchd.upsertSnapshotBatch(day, flights) + + override fun findByDay(day: String): List> = + flightSchd.findByDay(day) } diff --git a/src/main/kotlin/com/gzzn/omms/msgexchange/infra/redis/RedisScripts.kt b/src/main/kotlin/com/gzzn/omms/msgexchange/infra/redis/RedisScripts.kt deleted file mode 100644 index 9fa1f9f..0000000 --- a/src/main/kotlin/com/gzzn/omms/msgexchange/infra/redis/RedisScripts.kt +++ /dev/null @@ -1,29 +0,0 @@ -package com.gzzn.omms.msgexchange.infra.redis - -/** - * ACMA-8 流程 4 / I4 / I5：Redis Lua 脚本装载与执行入口。 - * 阶段 A 全部 flightInfo 写均经此处，且仅由主泵线程调用（I5）。 - */ -enum class RedisScript(val classpathLocation: String) { - /** 同一 hash 原子“覆盖新代 + 按代差删”（setArg 为 N 对 field/value，delArg 为差集）。 */ - SNAPSHOT_REPLACE("lua/snapshot_replace.lua"), - - /** 3:30 清场批量删除（仅 ES 归档成功集）。 */ - BATCH_DELETE("lua/batch_delete.lua"), -} - -interface FlightRedisClient { - /** - * 执行脚本。SNAPSHOT_REPLACE：setPairs 为新代全量 field/value，delFields 为按代差集； - * BATCH_DELETE：delFields 为待删 FLID 集。 - */ - fun eval(script: RedisScript, setPairs: List> = emptyList(), delFields: List = emptyList()) - - /** 阶段 A 权威读（处理决策 loadState、3:30 清场 findAll）。 */ - fun hgetAllFlightInfo(): Map - - /** 连通性探测（健康检查用；实现必须为快速调用，失败返回 false 而非抛出穿出）。 */ - fun ping(): Boolean -} - -// TODO(阶段1后续): 基于 micronaut-redis-lettuce 的实装（脚本自 classpath 装载并缓存 SHA）。 diff --git a/src/main/kotlin/com/gzzn/omms/msgexchange/infra/stub/StubAdapters.kt b/src/main/kotlin/com/gzzn/omms/msgexchange/infra/stub/StubAdapters.kt index 1814190..7f38981 100644 --- a/src/main/kotlin/com/gzzn/omms/msgexchange/infra/stub/StubAdapters.kt +++ b/src/main/kotlin/com/gzzn/omms/msgexchange/infra/stub/StubAdapters.kt @@ -5,8 +5,6 @@ import com.gzzn.omms.msgexchange.codec.DecodeResult import com.gzzn.omms.msgexchange.codec.XmlCodec import com.gzzn.omms.msgexchange.delivery.DeliveryPort import com.gzzn.omms.msgexchange.domain.ErrorClass -import com.gzzn.omms.msgexchange.infra.redis.FlightRedisClient -import com.gzzn.omms.msgexchange.infra.redis.RedisScript import com.gzzn.omms.msgexchange.processing.CodecHolder import com.gzzn.omms.msgexchange.processing.HandlerHolder import com.gzzn.omms.msgexchange.processing.HandlerRegistry @@ -16,7 +14,7 @@ import io.micronaut.context.annotation.Requires import jakarta.inject.Singleton /** - * U07：stub 适配层——XmlCodec / Redis 客户端 / DeliveryPort / Handler 装配，仅在 msgx.stubs=true 时生效。 + * U07：stub 适配层——XmlCodec / DeliveryPort / Handler 装配，仅在 msgx.stubs=true 时生效（Redis 客户端已按 ACM2-28 移除）。 * stub codec 未实装 → 报文 decode 返回 CODEC_ERROR（走 FAILED 可重放路径，链路上可观测）， * 语义见 ACM2-10 U11：不把“未实装”写成报文非法/终态。 */ @@ -29,30 +27,6 @@ class StubXmlCodec : XmlCodec { override fun encodeRqrd(kind: String, rangeJson: String): String = "" } -@Requires(property = "msgx.stubs", value = "true") -@Singleton -class StubRedis : FlightRedisClient { - val hash = mutableMapOf() - val evalCalls = mutableListOf() - - fun clear() { hash.clear(); evalCalls.clear() } - - override fun eval(script: RedisScript, setPairs: List>, delFields: List) { - evalCalls += script - when (script) { - RedisScript.SNAPSHOT_REPLACE -> { - hash.putAll(setPairs) - delFields.forEach { hash.remove(it) } - } - RedisScript.BATCH_DELETE -> delFields.forEach { hash.remove(it) } - } - } - - override fun hgetAllFlightInfo(): Map = hash.toMap() - - override fun ping(): Boolean = true -} - @Requires(property = "msgx.stubs", value = "true") @Singleton class StubDeliveryPort : DeliveryPort { @@ -62,7 +36,6 @@ class StubDeliveryPort : DeliveryPort { override fun sendKafka(topic: String, payloadJson: String) { sent += topic to payloadJson } override fun indexFlightHts(payloadJson: String) = Unit - override fun projectRedis(payloadJson: String) = Unit } /** Holder 工厂：CodecHolder/HandlerHolder 由 Micronaut Bean 提供（取代直连构造占位）。 */ diff --git a/src/main/kotlin/com/gzzn/omms/msgexchange/infra/stub/StubRepositories.kt b/src/main/kotlin/com/gzzn/omms/msgexchange/infra/stub/StubRepositories.kt index 33d59c8..023ee03 100644 --- a/src/main/kotlin/com/gzzn/omms/msgexchange/infra/stub/StubRepositories.kt +++ b/src/main/kotlin/com/gzzn/omms/msgexchange/infra/stub/StubRepositories.kt @@ -7,8 +7,10 @@ import com.gzzn.omms.msgexchange.domain.ProcState import com.gzzn.omms.msgexchange.domain.ProcStatus import com.gzzn.omms.msgexchange.domain.RefUpsert import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository +import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository +import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository import com.gzzn.omms.msgexchange.infra.persistence.PumpJobRepository import com.gzzn.omms.msgexchange.infra.persistence.RefDataRepository @@ -187,22 +189,118 @@ class StubPumpJobs : PumpJobRepository { override fun markFailed(jobId: Long, lastError: String) { rows[jobId]?.state = "FAILED"; rows[jobId]?.lastError = lastError } } -/** 快照 gen 协议 stub（业务库 REF_DATA 的 SCHD_GEN 行；ACM2-11 拆分后 21 类不在本实现）。 */ @Requires(property = "msgx.stubs", value = "true") @Singleton -class StubRefData : RefDataRepository { - private val gens = mutableMapOf() +class StubPipelineTransactionManager : PipelineTransactionManager { + override fun inTransaction(block: () -> T): T = block() +} - fun clear() { gens.clear() } +@Requires(property = "msgx.stubs", value = "true") +@Singleton +class StubFlightSchd : FlightSchdRepository { + data class Record( + val flid: String, + val fday: String?, + val fltrJson: String, + val createdAt: Instant, + val updatedAt: Instant, + ) - override fun getGen(day: String): RefDataRepository.GenMeta? = gens[day] + private val records = mutableMapOf() + private val gens = mutableMapOf() - override fun putGenIfVersion(day: String, expected: Long, new: RefDataRepository.GenMeta): Boolean { - val cur = gens[day]?.version ?: 0L - if (cur != expected) return false - gens[day] = new + fun clear() { + records.clear() + gens.clear() + } + + override fun upsertSnapshotBatch(day: String, flights: List>, now: Instant) { + for ((flid, json) in flights) { + val existing = records[flid] + records[flid] = Record( + flid = flid, + fday = day, + fltrJson = json, + createdAt = existing?.createdAt ?: now, + updatedAt = now, + ) + } + } + + override fun upsertIncremental(changes: List, now: Instant) { + for (c in changes) { + val existing = records[c.flid] + records[c.flid] = Record( + flid = c.flid, + fday = existing?.fday, // preserve existing fday; null if new + fltrJson = c.payloadJson, + createdAt = existing?.createdAt ?: now, + updatedAt = now, + ) + } + } + + override fun deleteDiffByDay(day: String, delFlids: Collection): Int { + if (delFlids.isEmpty()) return 0 + val delSet = delFlids.toSet() + val toRemove = records.filter { (flid, rec) -> rec.fday == day && flid in delSet }.keys + toRemove.forEach { records.remove(it) } + return toRemove.size + } + + override fun findByFlid(flid: String): String? = records[flid]?.fltrJson + + override fun findByFlids(flids: Collection): Map = + flids.mapNotNull { flid -> records[flid]?.let { flid to it.fltrJson } }.toMap() + + override fun findByDay(day: String): List> = + records.values.filter { it.fday == day } + .sortedBy { it.flid } + .map { it.flid to it.fltrJson } + + override fun findAll(): Map = + records.mapValues { it.value.fltrJson } + + override fun deleteByFlids(flids: Set): Int { + if (flids.isEmpty()) return 0 + var count = 0 + for (flid in flids) { + if (records.remove(flid) != null) count++ + } + return count + } + + override fun getGen(day: String): FlightSchdRepository.GenMeta? = gens[day] + + override fun putGenIfVersion(day: String, expected: Long, newGen: FlightSchdRepository.GenMeta, now: Instant): Boolean { + val current = gens[day]?.version ?: 0L + if (current != expected) return false + gens[day] = newGen.copy(updatedAt = now) return true } + + override fun deleteGenBefore(cutoffDay: String): Int { + val toRemove = gens.keys.filter { it < cutoffDay } + toRemove.forEach { gens.remove(it) } + return toRemove.size + } +} + +/** 快照 gen 协议 stub：委托至 StubFlightSchd。 */ +@Requires(property = "msgx.stubs", value = "true") +@Singleton +class StubRefData( + private val flightSchd: FlightSchdRepository, +) : RefDataRepository { + override fun getGen(day: String): RefDataRepository.GenMeta? = + flightSchd.getGen(day)?.let { RefDataRepository.GenMeta(it.flids.toList(), it.version) } + + override fun putGenIfVersion(day: String, expected: Long, new: RefDataRepository.GenMeta): Boolean = + flightSchd.putGenIfVersion( + day, + expected, + FlightSchdRepository.GenMeta(day, new.version, new.flids.toSet()), + ) } /** 21 类静态主数据 stub（独立 PG reference 库；内存实现，source 保留供审计断言）。 */ @@ -254,14 +352,12 @@ class StubReqTrack : ReqTrackRepository { @Requires(property = "msgx.stubs", value = "true") @Singleton -class StubFlightState : FlightStateRepository { - private val byDay = mutableMapOf>>() +class StubFlightState( + private val flightSchd: FlightSchdRepository, +) : FlightStateRepository { + override fun replaceDay(day: String, flights: List>) = + flightSchd.upsertSnapshotBatch(day, flights) - fun clear() { byDay.clear() } - - override fun replaceDay(day: String, flights: List>) { - byDay[day] = flights.toMutableList() - } - - override fun findByDay(day: String): List> = byDay[day] ?: emptyList() + override fun findByDay(day: String): List> = + flightSchd.findByDay(day) } diff --git a/src/main/kotlin/com/gzzn/omms/msgexchange/jobs/JobExecutor.kt b/src/main/kotlin/com/gzzn/omms/msgexchange/jobs/JobExecutor.kt index b224ed7..84e1fa6 100644 --- a/src/main/kotlin/com/gzzn/omms/msgexchange/jobs/JobExecutor.kt +++ b/src/main/kotlin/com/gzzn/omms/msgexchange/jobs/JobExecutor.kt @@ -1,8 +1,6 @@ package com.gzzn.omms.msgexchange.jobs -import com.gzzn.omms.msgexchange.infra.redis.FlightRedisClient -import com.gzzn.omms.msgexchange.infra.redis.RedisScript -import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository +import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository import com.gzzn.omms.msgexchange.infra.persistence.PumpJobRepository import com.gzzn.omms.msgexchange.domain.MsgEvent @@ -33,21 +31,31 @@ class JobExecutor( */ @Singleton class HistorySweepJob( - private val redis: FlightRedisClient, - // TODO(阶段2): esFlightHts.saveSync(history) 返回成功集（同步写，不经事件 ack 回查） + private val flightSchd: FlightSchdRepository, ) { + companion object { + var historyPicker: ((Map) -> Map)? = null + var esArchiver: ((Map) -> Set)? = null + var cutoffProvider: (() -> String?)? = null + } + fun run() { - val all = redis.hgetAllFlightInfo() + val all = flightSchd.findAll() val history = pickHistory(all) // U10/T07：saveSync 接线前判史为空集——禁止“全量可删”默认 - val success = emptyList() // TODO(阶段2): esFlightHts.saveSync(history)（返回成功集后再接线删除） + val success = esArchiver?.invoke(history) ?: emptySet() // 仅收集已确认写入 ES 的成功 FLID 集合 if (success.isNotEmpty()) { - redis.eval(RedisScript.BATCH_DELETE, delFields = success) + flightSchd.deleteByFlids(success) // FS2/FS4：仅删除已确认写入 ES 的成功集合，分批参数化删除 + } + val cutoff = cutoffProvider?.invoke() + if (cutoff != null) { + flightSchd.deleteGenBefore(cutoff) } } /** U10/T07（修订）：接入 ES success 集之前的门禁——占位默认返回空集； * 现役五条判史规则（SODT 3 天 / CNCL 1 小时 / 备降 / 离港 / 到港）golden 通过后才允许接线。 */ - private fun pickHistory(all: Map): Map = emptyMap() // TODO(阶段2) + private fun pickHistory(all: Map): Map = + historyPicker?.invoke(all) ?: emptyMap() } /** 流程 5 runJob(ARCHIVE)：3:00 归档——1 天前且仅终态可迁（矩阵 #12）。 */ @@ -63,19 +71,18 @@ class ArchiveJob( /** 流程 7：阶段 B 投影重建——切入阶段 B 时全量重建一次，此后增量走事件。 */ @Singleton class ProjectionRebuildJob( - private val flightState: FlightStateRepository, + private val flightSchd: FlightSchdRepository, private val msgEvents: MsgEventRepository, ) { fun run() { for (day in activeDays()) { - val flights = flightState.findByDay(day) + val flights = flightSchd.findByDay(day) flights.forEach { (flid, payload) -> msgEvents.insertSync( - listOf(MsgEvent(target = Targets.REDIS_FLIGHT_INFO, partitionKey = flid, payloadJson = payload)), + listOf(MsgEvent(target = Targets.KAFKA_SCHD, partitionKey = flid, payloadJson = payload)), ) } } - // 重建完成前旧 Redis 视图继续服务（数据同源，仅短暂滞后） } private fun activeDays(): List = emptyList() // TODO(阶段B) diff --git a/src/main/kotlin/com/gzzn/omms/msgexchange/processing/Handler.kt b/src/main/kotlin/com/gzzn/omms/msgexchange/processing/Handler.kt index e13b272..81e7f55 100644 --- a/src/main/kotlin/com/gzzn/omms/msgexchange/processing/Handler.kt +++ b/src/main/kotlin/com/gzzn/omms/msgexchange/processing/Handler.kt @@ -11,7 +11,7 @@ import com.gzzn.omms.msgexchange.domain.MsgKind interface Handler { val kind: MsgKind - /** 在线状态以只读快照传入（阶段 A 读 Redis 权威、阶段 B 读 FLIGHT_STATE，由调用方装配）。 */ + /** 在线状态以只读视图传入（阶段 A 点查自有 PG FLIGHT_SCHD，由调用方装配，ACM2-28 定案）。 */ fun decide(flightView: Map, msg: DecodedMessage): Decision } diff --git a/src/main/kotlin/com/gzzn/omms/msgexchange/processing/Pump.kt b/src/main/kotlin/com/gzzn/omms/msgexchange/processing/Pump.kt index 2fca032..5183b28 100644 --- a/src/main/kotlin/com/gzzn/omms/msgexchange/processing/Pump.kt +++ b/src/main/kotlin/com/gzzn/omms/msgexchange/processing/Pump.kt @@ -11,7 +11,8 @@ import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository import com.gzzn.omms.msgexchange.infra.persistence.PumpJobRepository -import com.gzzn.omms.msgexchange.infra.redis.FlightRedisClient +import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository +import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager import com.gzzn.omms.msgexchange.infra.retry.ProcFailure import com.gzzn.omms.msgexchange.jobs.JobExecutor import jakarta.inject.Singleton @@ -20,7 +21,7 @@ import java.time.Instant /** * ACMA-8 流程 2：处理主泵——严格 FIFO + HOL + 毒丸 DEAD 升级（I1）； - * 阶段 A 全部 Redis 写在本线程（I5），且先于事件创建（I2 happens-before）。 + * 阶段 A 运营航班 FLIGHT_SCHD 与待发事件、终态同事务原子提交（I2/I5，ACM2-28 定案）。 */ @Singleton class Pump( @@ -114,12 +115,25 @@ class MessageProcessor( private val msgEvents: MsgEventRepository, private val codecHolder: CodecHolder, private val handlers: HandlerHolder, - private val redis: FlightRedisClient, + private val flightSchd: FlightSchdRepository, private val snapshotFlow: SnapshotFlow, private val procFailure: ProcFailure, private val props: PipelineProps, + private val txManager: PipelineTransactionManager, ) { private val log = org.slf4j.LoggerFactory.getLogger(MessageProcessor::class.java) + private val flidRegex = Regex("(.*?)", RegexOption.IGNORE_CASE) + + private fun extractCandidateFlids(decoded: DecodedMessage): Set { + val fromXml = flidRegex.findAll(decoded.rawXml).map { it.groupValues[1].trim() }.filter { it.isNotEmpty() }.toSet() + if (fromXml.isNotEmpty()) return fromXml + val b = decoded.body + if (b is Map<*, *>) { + val flid = b["FLID"] ?: b["flid"] + if (flid != null) return setOf(flid.toString()) + } + return emptySet() + } fun processOne(head: ProcState) { com.gzzn.omms.msgexchange.infra.log.TraceLog.withTrace(head.cminmsgsId) { @@ -176,8 +190,7 @@ class MessageProcessor( return } } - - // 快照消息走专属流程（流程 4：staging→Lua→putGenIfVersion CAS 同事务） + // 快照消息走专属流程（流程 4：staging → SQL 批处理/域内差删/CAS 同事务） val handler = handlers.registry.dispatcherFor(decoded) if (handler == null) { // U10（N21）：未注册 ≠ 报文非法——写 FAILED（可重放），绝不写终态 @@ -193,23 +206,35 @@ class MessageProcessor( return } - val decision = handler.decide(redis.hgetAllFlightInfo(), decoded) // 纯函数 - - if (props.phase == PipelineProps.Phase.A) { - // 阶段 A：主泵线程先写 Redis（幂等），先于事件创建（I2）；TODO: redisApply(flightChanges) + val candidateFlids = extractCandidateFlids(decoded) + val flightView = if (candidateFlids.isNotEmpty()) { + flightSchd.findByFlids(candidateFlids) + } else { + emptyMap() } - // 事务 2（自有 PG 内原子）：事件 + SUCCEEDED（实装后 @Transactional）； - // CMINMSGS 回填 = 共享信箱外部副作用（最终一致，ACM2-12）； - // 静态主数据（refUpserts）同自有 PG 但弱事务独立提交（21 类 REF_MASTER）。 + + val decision = handler.decide(flightView, decoded) // 纯函数 + + // 事务 2（自有 PG 单事务原子）： + // upsert FLIGHT_SCHD(decision.flightChanges) + insert MSG_EVENT + PROC_STATE → SUCCEEDED val events = buildList { decision.msgNotifies.forEach { add(MsgEvent(target = Targets.KAFKA_MSG, payloadJson = it.payloadJson)) } decision.schdPush.forEach { add(MsgEvent(target = Targets.KAFKA_SCHD, partitionKey = it.flid, payloadJson = it.fltrJson)) } } - msgEvents.insertAll(events) + + txManager.inTransaction { + if (decision.flightChanges.isNotEmpty()) { + flightSchd.upsertIncremental(decision.flightChanges) + } + if (events.isNotEmpty()) { + msgEvents.insertAll(events) + } + procState.update(head.cminmsgsId, ProcStatus.SUCCEEDED) + } + + // 提交后：backfill CMINMSGS（共享信箱外部副作用，补偿链路） inbox.backfillOnSuccess(head.cminmsgsId, decoded.meta.sndr, decoded.meta.type, decoded.meta.styp, decoded.meta.seqn) - procState.update(head.cminmsgsId, ProcStatus.SUCCEEDED) - log.info("SUCCEEDED id={} events={}", head.cminmsgsId, events.size) - // 阶段 B：flightState.apply(decision.flightChanges) 进入同一事务；投影事件（ES/REDIS）追加。 + log.info("SUCCEEDED id={} events={} flightChanges={}", head.cminmsgsId, events.size, decision.flightChanges.size) } } diff --git a/src/main/kotlin/com/gzzn/omms/msgexchange/processing/SnapshotFlow.kt b/src/main/kotlin/com/gzzn/omms/msgexchange/processing/SnapshotFlow.kt index 51811e0..2f8ce6a 100644 --- a/src/main/kotlin/com/gzzn/omms/msgexchange/processing/SnapshotFlow.kt +++ b/src/main/kotlin/com/gzzn/omms/msgexchange/processing/SnapshotFlow.kt @@ -4,59 +4,114 @@ import com.gzzn.omms.msgexchange.domain.DecodedMessage import com.gzzn.omms.msgexchange.domain.ErrorClass import com.gzzn.omms.msgexchange.domain.ProcState import com.gzzn.omms.msgexchange.domain.ProcStatus +import com.gzzn.omms.msgexchange.domain.MsgEvent +import com.gzzn.omms.msgexchange.domain.MsgKind +import com.gzzn.omms.msgexchange.domain.Targets +import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository +import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository +import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository +import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository -import com.gzzn.omms.msgexchange.infra.persistence.RefDataRepository -import com.gzzn.omms.msgexchange.infra.redis.FlightRedisClient -import com.gzzn.omms.msgexchange.infra.redis.RedisScript +import com.gzzn.omms.msgexchange.infra.persistence.ReqTrackRepository import com.gzzn.omms.msgexchange.infra.retry.ProcFailure import jakarta.inject.Singleton /** - * ACMA-8 流程 4：日计划快照（generation，主泵内执行）。 - * staging（内存瞬态，崩溃从 raw 整包重放）→ Lua 原子“覆盖+按代差删”（I4/I5） - * → putGenIfVersion CAS + SUCCEEDED（重放幂等，版本不二次自增——恢复协议细节见 U09）。 + * staging（内存瞬态，崩溃从 raw 整包重放）→ PG 单事务原子“覆盖+按代差删+SQL CAS 推进+事件入队+SUCCEEDED”（I4/I5，ACM2-28 定案）。 * U10/T07 占位安全化 + U08 统一失败迁移（ProcFailure）：staging 未实装 → FAILED(UNSUPPORTED)+退避 * （可重放，绝不写终态）；CAS 冲突 → FAILED(INFRA)+退避；达上限统一 DEAD(EXHAUSTED)。 */ @Singleton class SnapshotFlow( private val procState: ProcStateRepository, - private val refData: RefDataRepository, - private val redis: FlightRedisClient, + private val flightSchd: FlightSchdRepository, + private val msgEvents: MsgEventRepository, + private val reqTrack: ReqTrackRepository, private val procFailure: ProcFailure, + private val txManager: PipelineTransactionManager, + private val inbox: CminmsgInboxRepository, ) { private val log = org.slf4j.LoggerFactory.getLogger(SnapshotFlow::class.java) fun publishSnapshot(head: ProcState, msg: DecodedMessage) { - // 1) staging：流式解析 + 整包校验（TODO(阶段2): 流式 codec；千级 FLTR 为 MB 级，内存瞬态） - val staged = StageResult.stagingOf(msg) // 骨架：TODO 解析 FLTR 集与重组（KEEP 现役 MAFL/登机桥规则） + // 1) staging：流式解析 + 整包校验（内存瞬态，崩溃从 raw 整包重放） + val staged = StageResult.stagingOf(msg) if (staged is StageResult.Invalid) { log.warn("staging not implemented -> FAILED(UNSUPPORTED) id={} reason={}", head.cminmsgsId, staged.reason) procFailure.fail(head, ErrorClass.UNSUPPORTED, staged.reason) // U10：未实装 → 可重放，非终态 return } - val normalized = (staged as StageResult.Ok).flights // (flid, payloadJson) + val ok = staged as StageResult.Ok + val normalized = ok.flights // (flid, payloadJson) + val day = ok.day - // 2) 发布：Lua 原子覆盖 + 按代差删（删除集 = gen.flids − 新代，ADFT 自动存活） - val day = staged.day - val gen = refData.getGen(day) - val newFlids = normalized.map { it.first } - val delFields = gen?.flids?.minus(newFlids.toSet()) ?: emptyList() - redis.eval(RedisScript.SNAPSHOT_REPLACE, setPairs = normalized, delFields = delFields) - - // 3) 事务：putGen CAS + SUCCEEDED（实装后同 @Transactional；CAS 失败=并发，串行泵下不应发生→告警） + // 内存与超大包熔断防御（ACM2-28 评论 4 项 3） + if (normalized.size > MAX_FLIGHTS_PER_SNAPSHOT) { + log.error("snapshot flights exceed limit -> DEAD(MALFORMED) id={} size={}", head.cminmsgsId, normalized.size) + procState.update(head.cminmsgsId, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, + lastError = "flights-exceed-limit:${normalized.size}") + return + } + + // 2) 准备代元数据与差集（删除集 = gen.flids − 新代，ADFT 自动存活） + val gen = flightSchd.getGen(day) val expected = gen?.version ?: 0L - if (!refData.putGenIfVersion(day, expected, RefDataRepository.GenMeta(newFlids, expected + 1))) { - // 重放路径：version 已是目标值 → no-op 视为成功 - val again = refData.getGen(day) - if (again == null || again.version != expected + 1) { - log.warn("gen CAS conflict -> FAILED(INFRA) id={}", head.cminmsgsId) - procFailure.fail(head, ErrorClass.INFRA, "gen-cas-conflict") // N06/N28：带退避，禁止紧循环 - return + val newFlids = normalized.map { it.first }.toSet() + val delFields = gen?.flids?.minus(newFlids) ?: emptySet() + val newVersion = expected + 1L + + // 3) 自有 PG 单事务原子提交（ACM2-28 定案：覆盖新代 + 域内差删 + SQL CAS + 事件 + SUCCEEDED） + try { + txManager.inTransaction { + // 覆盖新代全量：强行声明 FDAY 归属（JDBC batch 批处理） + flightSchd.upsertSnapshotBatch(day, normalized) + + // 按代差删域化：仅删除 FDAY = day 且在 delFields 中的记录（ADFT 与跨代已迁移行存活） + if (delFields.isNotEmpty()) { + flightSchd.deleteDiffByDay(day, delFields) + } + + // SQL CAS 版本推进（防双写断言） + val casSuccess = flightSchd.putGenIfVersion( + day = day, + expected = expected, + newGen = FlightSchdRepository.GenMeta(fday = day, version = newVersion, flids = newFlids), + ) + if (!casSuccess) { + // 重放路径：version 已是目标值 → no-op 视为成功，否则为 CAS 冲突 + val again = flightSchd.getGen(day) + if (again == null || again.version != newVersion) { + throw CasConflictException("gen-cas-conflict: expected=$expected current=${again?.version}") + } + } + + // RESP 匹配 REQ_TRACK -> DONE + val schdKind = msg.kind as? MsgKind.Schd + if (schdKind?.subtype == MsgKind.SchdSubtype.RESP) { + reqTrack.findOpenByKind("SCHD")?.let { req -> + reqTrack.markDone(req.reqId) + } + } + + // 构造并批量写入 schd 投递事件 + val events = normalized.map { (flid, json) -> + MsgEvent(target = Targets.KAFKA_SCHD, partitionKey = flid, payloadJson = json) + } + if (events.isNotEmpty()) { + msgEvents.insertAll(events) + } + + // 终态置 SUCCEEDED + procState.update(head.cminmsgsId, ProcStatus.SUCCEEDED) } + + // 提交后：backfill CMINMSGS（外部副作用，补偿保障） + inbox.backfillOnSuccess(head.cminmsgsId, msg.meta.sndr, msg.meta.type, msg.meta.styp, msg.meta.seqn) + log.info("snapshot SUCCEEDED id={} day={} flights={}", head.cminmsgsId, day, normalized.size) + } catch (e: CasConflictException) { + log.warn("gen CAS conflict -> FAILED(INFRA) id={} msg={}", head.cminmsgsId, e.message) + procFailure.fail(head, ErrorClass.INFRA, e.message ?: "gen-cas-conflict") } - procState.update(head.cminmsgsId, ProcStatus.SUCCEEDED) - log.info("snapshot SUCCEEDED id={} day={} flights={}", head.cminmsgsId, day, normalized.size) } /** staging 结果（骨架）。 */ @@ -65,8 +120,12 @@ class SnapshotFlow( data class Invalid(val reason: String) : StageResult companion object { + var parser: ((DecodedMessage) -> StageResult)? = null + fun stagingOf(msg: DecodedMessage): StageResult = - Invalid("staging-not-implemented(${msg.typeTag})") // TODO(阶段2) + parser?.invoke(msg) ?: Invalid("staging-not-implemented(${msg.typeTag})") // TODO(阶段2) } } } +class CasConflictException(message: String) : RuntimeException(message) +const val MAX_FLIGHTS_PER_SNAPSHOT = 10000 diff --git a/src/main/kotlin/com/gzzn/omms/msgexchange/tools/FlightStoreDiffTool.kt b/src/main/kotlin/com/gzzn/omms/msgexchange/tools/FlightStoreDiffTool.kt new file mode 100644 index 0000000..13bc8ef --- /dev/null +++ b/src/main/kotlin/com/gzzn/omms/msgexchange/tools/FlightStoreDiffTool.kt @@ -0,0 +1,260 @@ +package com.gzzn.omms.msgexchange.tools + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.databind.node.ValueNode +import java.math.BigDecimal + +/** + * ACM2-28 FS7：影子对拍跨存储 Diff 比较内核。 + * + * 核心定位： + * 本工具提供跨存储航班 AST 结构对比、字段规范化与已知合法偏离识别内核； + * 外部装配侧（从 PostgreSQL/Redis 读取数据并按 CMINMSGS_ID 水位对齐取数）属于影子期装配调用方职责。 + * 字段归一化规则（ACM2-28 评论 4 项 4）： + * 1. 递归 AST/Map 结构对比，绝不做裸字符串比对（规避 PG jsonb 自动去重重排 key 的分叉）； + * 2. 数值精度归一化（1.0 vs 1 视为等价）； + * 3. 空值规范化（JSON null 与字段缺失视为等价，规避无语义偏离）； + * + * 已知合法偏离声明（ACM2-28 方案六）： + * ① FDAY 域化差删对跨代迁移行的保护（legacy 误删、nextgen 不删，属于修正性偏离）； + * ② JSONB 与 Redis string 的 key 排序差异。 + */ +class FlightStoreDiffTool( + private val mapper: ObjectMapper = ObjectMapper(), +) { + + enum class DeviationKind { + /** 已知合法偏离 ①：跨日迁移航班在 nextgen PG 中受 FDAY 保护未被误删，而 legacy 误删 */ + CROSS_DAY_PROTECTED, + /** 非预期偏离：字段值不匹配 */ + FIELD_MISMATCH, + /** 非预期偏离：PG 缺失该航班 */ + MISSING_IN_PG, + /** 非预期偏离：Redis 缺失且非已知迁移保护行 */ + UNEXPECTED_EXTRA_IN_PG, + } + + data class Deviation( + val flid: String, + val kind: DeviationKind, + val path: String? = null, + val pgValue: Any? = null, + val legacyValue: Any? = null, + val detail: String, + ) + + data class DiffReport( + val totalPg: Int, + val totalLegacy: Int, + val matchedCount: Int, + val knownDeviations: List, + val unexpectedDeviations: List, + ) { + val isGreen: Boolean + get() = unexpectedDeviations.isEmpty() + + fun formatSummary(): String { + val sb = StringBuilder() + sb.appendLine("========== 影子对拍跨存储 Diff 报告 ==========") + sb.appendLine("Nextgen PG 航班总数 : $totalPg") + sb.appendLine("Legacy Redis 航班总数 : $totalLegacy") + sb.appendLine("完全一致航班数 : $matchedCount") + sb.appendLine("已知合法偏离数 (偏差①) : ${knownDeviations.size}") + sb.appendLine("非预期异常数 : ${unexpectedDeviations.size}") + sb.appendLine("验收红绿灯状态 : ${if (isGreen) "GREEN (通过)" else "RED (未通过)"}") + sb.appendLine("----------------------------------------------") + if (knownDeviations.isNotEmpty()) { + sb.appendLine("[已知合法偏离]") + knownDeviations.forEach { d -> + sb.appendLine(" FLID=${d.flid}: ${d.detail}") + } + } + if (unexpectedDeviations.isNotEmpty()) { + sb.appendLine("[非预期异常偏离]") + unexpectedDeviations.forEach { d -> + sb.appendLine(" FLID=${d.flid} [${d.kind}] path=${d.path}: PG=${d.pgValue}, Legacy=${d.legacyValue} (${d.detail})") + } + } + sb.appendLine("==============================================") + return sb.toString() + } + } + + /** + * 执行全量比对。 + * @param pgFlights nextgen FLIGHT_SCHD 行（flid -> fltr_json） + * @param legacyFlights legacy Redis flightInfo 行（flid -> value string） + * @param crossDayMigratedFlids 已知在多日之间迁移的航班 FLID 集合（用于识别合法偏差 ①） + */ + fun diff( + pgFlights: Map, + legacyFlights: Map, + crossDayMigratedFlids: Set = emptySet(), + ): DiffReport { + val allFlids = (pgFlights.keys + legacyFlights.keys).toSortedSet() + var matched = 0 + val known = mutableListOf() + val unexpected = mutableListOf() + + for (flid in allFlids) { + val pgRaw = pgFlights[flid] + val legacyRaw = legacyFlights[flid] + + if (pgRaw == null) { + unexpected += Deviation( + flid = flid, + kind = DeviationKind.MISSING_IN_PG, + legacyValue = legacyRaw, + detail = "Flight present in Redis but missing in PG", + ) + continue + } + + if (legacyRaw == null) { + if (flid in crossDayMigratedFlids) { + known += Deviation( + flid = flid, + kind = DeviationKind.CROSS_DAY_PROTECTED, + pgValue = pgRaw, + detail = "Known Deviation 1: Cross-day migrated flight protected by FDAY domain-scoped delete in nextgen PG, erroneously deleted in legacy Redis", + ) + } else { + unexpected += Deviation( + flid = flid, + kind = DeviationKind.UNEXPECTED_EXTRA_IN_PG, + pgValue = pgRaw, + detail = "Flight present in PG but missing in Redis without cross-day migration justification", + ) + } + continue + } + + // 两侧均存在：进行 AST 递归规范化比较 + val mismatches = compareJsonTree(flid, pgRaw, legacyRaw) + if (mismatches.isEmpty()) { + matched++ + } else { + unexpected.addAll(mismatches) + } + } + + return DiffReport( + totalPg = pgFlights.size, + totalLegacy = legacyFlights.size, + matchedCount = matched, + knownDeviations = known, + unexpectedDeviations = unexpected, + ) + } + + /** + * 递归 AST 比较两段 JSON。 + */ + internal fun compareJsonTree(flid: String, pgJson: String, legacyJson: String): List { + val pgNode = try { + mapper.readTree(pgJson) + } catch (e: Exception) { + return listOf(Deviation(flid, DeviationKind.FIELD_MISMATCH, detail = "PG JSON parse failed: ${e.message}")) + } + + val legacyNode = try { + mapper.readTree(legacyJson) + } catch (e: Exception) { + return listOf(Deviation(flid, DeviationKind.FIELD_MISMATCH, detail = "Legacy JSON parse failed: ${e.message}")) + } + + val mismatches = mutableListOf() + compareNodes(flid, "", pgNode, legacyNode, mismatches) + return mismatches + } + + private fun compareNodes(flid: String, path: String, n1: JsonNode?, n2: JsonNode?, acc: MutableList) { + if (n1 == null && n2 == null) return + + // 规范化：null 节点与缺失节点视为等价 + val isN1Empty = n1 == null || n1.isNull + val isN2Empty = n2 == null || n2.isNull + if (isN1Empty && isN2Empty) return + + if (isN1Empty != isN2Empty) { + acc += Deviation( + flid = flid, + kind = DeviationKind.FIELD_MISMATCH, + path = path, + pgValue = n1?.asText(), + legacyValue = n2?.asText(), + detail = "One side is null/absent while other is present", + ) + return + } + + val nonNull1 = n1!! + val nonNull2 = n2!! + + // 数值归一化比较 + if (nonNull1.isNumber && nonNull2.isNumber) { + val d1 = BigDecimal(nonNull1.asText()).stripTrailingZeros() + val d2 = BigDecimal(nonNull2.asText()).stripTrailingZeros() + if (d1.compareTo(d2) != 0) { + acc += Deviation( + flid = flid, + kind = DeviationKind.FIELD_MISMATCH, + path = path, + pgValue = nonNull1.numberValue(), + legacyValue = nonNull2.numberValue(), + detail = "Numeric value mismatch: $d1 != $d2", + ) + } + return + } + + // 对象类型：递归属性比较（忽略 key 顺序） + if (nonNull1.isObject && nonNull2.isObject) { + val obj1 = nonNull1 as ObjectNode + val obj2 = nonNull2 as ObjectNode + val allKeys = (obj1.fieldNames().asSequence().toSet() + obj2.fieldNames().asSequence().toSet()).toSortedSet() + + for (key in allKeys) { + val nextPath = if (path.isEmpty()) key else "$path.$key" + compareNodes(flid, nextPath, obj1.get(key), obj2.get(key), acc) + } + return + } + + // 数组类型：逐项比较 + if (nonNull1.isArray && nonNull2.isArray) { + val arr1 = nonNull1 as ArrayNode + val arr2 = nonNull2 as ArrayNode + if (arr1.size() != arr2.size()) { + acc += Deviation( + flid = flid, + kind = DeviationKind.FIELD_MISMATCH, + path = path, + pgValue = "size=${arr1.size()}", + legacyValue = "size=${arr2.size()}", + detail = "Array size mismatch", + ) + return + } + for (i in 0 until arr1.size()) { + compareNodes(flid, "$path[$i]", arr1.get(i), arr2.get(i), acc) + } + return + } + + // 基本类型：文本值比较 + if (nonNull1.asText() != nonNull2.asText()) { + acc += Deviation( + flid = flid, + kind = DeviationKind.FIELD_MISMATCH, + path = path, + pgValue = nonNull1.asText(), + legacyValue = nonNull2.asText(), + detail = "Value mismatch: ${nonNull1.asText()} != ${nonNull2.asText()}", + ) + } + } +} diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index de639de..dd38756 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -36,19 +36,14 @@ mailbox: username: ${MSGX_MAILBOX_USER:msgx_dev} password: ${MSGX_MAILBOX_PASSWORD:msgx_dev_pass} driver-class-name: com.mysql.cj.jdbc.Driver -# stub 模式无 DataSource：Flyway 显式关闭（主配置默认 enabled:true 只作用于有 DataSource 的环境） +# dev 默认 profile 为纯内存 stub（stubs=true），无外部数据库，故默认关 Flyway 避免启动报错； +# 当本地开发启用真实 PostgreSQL（datasources.default.enabled=true）时，设 MSGX_FLYWAY_ENABLED=true 自动运行 Flyway 迁移。 flyway: datasources: default: - enabled: false - -# stub 模式无真实 Redis/Kafka：关掉 micronaut 自带的 RedisHealthIndicator / KafkaHealthIndicator -# （二者装配需构造真实 client/config；实测未设 MSGX_REDIS_URI / MSGX_KAFKA_SERVERS 时 /health 整体报 500）。 -# 健康由自定义 redis-flight-store / kafka-delivery indicator 承担（stub ping=true → UP）。 -redis: - health: - enabled: false - + enabled: ${MSGX_FLYWAY_ENABLED:false} +# stub 模式无真实 Kafka：关掉 micronaut 自带的 KafkaHealthIndicator +# （健康由自定义 kafka-delivery indicator 承担；ACM2-28：Redis 已退出阶段 A 权威与写路径）。 kafka: health: enabled: false diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 60c5c6a..2393a35 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -3,14 +3,14 @@ # 系统角色：机场 OMMS 上游报文处理中间件（JDBC 轮询共享 CMINMSGS 为主路径； # compat HTTP POST /cminmsgs/send 为辅助写路径）。 # 口令/端点全部环境变量外置——零入库。 -# 存储边界：自有 PostgreSQL（datasources.default，全部内部状态）+ -# 共享 MySQL 信箱（mailbox.cminmsgs / mailbox.coutmsgs，仅 DML，不建表）+ -# Redis（航班动态 + 快照 gen）+ Kafka + Eureka/logstash（见 ACM2-12 与 docs）。 +# 存储边界：自有 PostgreSQL（datasources.default，全部内部状态与运营航班权威 FLIGHT_SCHD/SCHD_GEN）+ +# 共享 MySQL 信箱（mailbox.shared-mysql，仅 DML，不建表）+ +# Kafka + Eureka/logstash（见 ACM2-28 与 docs；Redis 彻底退出阶段 A 权威与写路径）。 # 键位口径：Micronaut 5.1（U03）：datasources.* / flyway.datasources.* / # kafka.producers..* / eureka.client.*（Micronaut 原生键）。 # ===================================================================== msgx: - phase: A # 航班动态权威 Redis（主泵单线程写，I5）；阶段 B 缓做（ACM2-12） + phase: A # 航班动态权威落自有 PG（FLIGHT_SCHD，主泵单线程写，I5；ACM2-28 定案） service-name: msgexchangeapi # 契约冻结；影子实例用 msgexchangeapi-shadow register-eureka: true # 影子对拍期置 false 或独立服务名 pipeline: # —— ACMA-8 参数表初值（阶段 0 基线校准）—— @@ -38,7 +38,7 @@ micronaut: # 管理端点（U03/N32）：/env、/beans 默认 sensitive；仅开发/影子环境放开——见 application-dev.yml # 自有 PostgreSQL：全部内部状态（消息管道 PROC_STATE/MSG_EVENT、PUMP_JOB、REQ_TRACK、 -# 21 类 REF_MASTER；迁移 db/migration，ACM2-12）。数据层实装（U05/阶段 6）前 enabled=false。 +# 21 类 REF_MASTER、运营航班表 FLIGHT_SCHD 与 SCHD_GEN；迁移 db/migration，ACM2-28）。数据层实装前 enabled=false。 datasources: default: enabled: false @@ -50,7 +50,7 @@ datasources: flyway: datasources: default: - enabled: false + enabled: ${MSGX_FLYWAY_ENABLED:true} locations: classpath:db/migration # 共享 MySQL 信箱（他人系统库，仅 DML：上游外部写 CMINMSGS；本系统 JDBC 轮询读 + @@ -64,9 +64,6 @@ mailbox: password: ${MSGX_MAILBOX_PASSWORD} driver-class-name: com.mysql.cj.jdbc.Driver -redis: - uri: ${MSGX_REDIS_URI} # 航班动态权威 + 快照 gen；仅主泵线程写（I5） - kafka: bootstrap: servers: ${MSGX_KAFKA_SERVERS} diff --git a/src/main/resources/db/migration/V1.1.0__flight_schd.sql b/src/main/resources/db/migration/V1.1.0__flight_schd.sql new file mode 100644 index 0000000..04523e6 --- /dev/null +++ b/src/main/resources/db/migration/V1.1.0__flight_schd.sql @@ -0,0 +1,32 @@ +-- ===================================================================== +-- 自有 PostgreSQL 库 · 阶段 A 运营航班表与计划代迁移 · ACM2-28 定案（选项 C） +-- --------------------------------------------------------------------- +-- 存储边界（ACM2-28 定案，取代 ACM2-12「FLIGHT_STATE 缓做不落表、Redis 永续权威」口径）： +-- · FLIGHT_SCHD：SCHD 日计划快照 + FLOP/ADFT 增量合并后的当前运营航班权威态，落自有 PG。 +-- · SCHD_GEN：各日代版本与当前代航班全量集合（差删依据），由 Redis 回归自有 PG。 +-- · Redis：彻底退出动态权威与全部写路径；阶段 A 读写事务全在自有 PG 原子完成。 +-- · 增量 Upsert 语义： +-- - 快照全量写入（DNLD）：声明/更新 FDAY 归属，更新 FLTR_JSON； +-- - 增量写入（FLOP/ADFT）：新插置 NULL，已有行通过 ON CONFLICT 保留原 FDAY； +-- - 按代差删域化：DELETE FROM FLIGHT_SCHD WHERE FDAY = :day AND FLID = ANY(:diffSet) +-- 仅删除仍属旧代的行，ADFT（FDAY=NULL）与已迁移至新代的同 FLID 行天然存活。 +-- · 时间列规范：统一使用 TIMESTAMP(6) WITH TIME ZONE (TIMESTAMPTZ)。 +-- ===================================================================== + +-- ① 运营航班表（当前运营航班动态权威，FLID 与 legacy flightInfo 同键） +CREATE TABLE FLIGHT_SCHD ( + FLID VARCHAR(32) NOT NULL PRIMARY KEY, -- AODB 唯一航班标识 + FDAY DATE NULL, -- 所属日计划代；NULL = 非代所有（ADFT/增量增建） + FLTR_JSON JSONB NOT NULL, -- 航班全量记录（与 legacy flightInfo value 同构） + CREATED_AT TIMESTAMP(6) WITH TIME ZONE NOT NULL, + UPDATED_AT TIMESTAMP(6) WITH TIME ZONE NOT NULL +); +CREATE INDEX idx_flight_schd_fday ON FLIGHT_SCHD (FDAY); -- 日代扫描与影子对拍分片 + +-- ② 日计划代与差删元数据表（代版本推进与版本 CAS） +CREATE TABLE SCHD_GEN ( + FDAY DATE NOT NULL PRIMARY KEY, -- 计划日（yyyy-MM-dd） + VERSION BIGINT NOT NULL, -- 代版本号（单调递增） + FLIDS_JSON JSONB NOT NULL, -- 当前代有效航班 FLID 集合 + UPDATED_AT TIMESTAMP(6) WITH TIME ZONE NOT NULL +); diff --git a/src/main/resources/lua/batch_delete.lua b/src/main/resources/lua/batch_delete.lua deleted file mode 100644 index bdca89b..0000000 --- a/src/main/resources/lua/batch_delete.lua +++ /dev/null @@ -1,9 +0,0 @@ --- ACMA-8 流程 4（runJob HISTORY_SWEEP）/ I4：3:30 清场批量删除。 --- 仅删除“ES 归档成功集”（调用方以 saveSync 成功返回集为参），同一泵线程程序序保证依赖。 --- KEYS[1] = flightInfo；ARGV = 待删除 FLID 列表 -local n = 0 -for i = 1, #ARGV do - redis.call('HDEL', KEYS[1], ARGV[i]) - n = n + 1 -end -return n diff --git a/src/main/resources/lua/snapshot_replace.lua b/src/main/resources/lua/snapshot_replace.lua deleted file mode 100644 index 9aaa492..0000000 --- a/src/main/resources/lua/snapshot_replace.lua +++ /dev/null @@ -1,19 +0,0 @@ --- ACMA-8 流程 4 / I4 / I5：日计划快照“覆盖新代 + 按代差删”，同一 hash 原子执行。 --- KEYS[1] = flightInfo --- ARGV[1] = set 对数 N，其后为 N 对 field/value；再后为待删除 field（按代差集） --- 删除集边界：仅上一代 gen.flids − 新代 flids（按代，非全 hash）→ ADFT 自动存活。 -local key = KEYS[1] -local nSet = tonumber(ARGV[1]) -local i = 2 -while i <= 1 + nSet * 2 do - redis.call('HSET', key, ARGV[i], ARGV[i + 1]) - i = i + 2 -end -local j = 2 + nSet * 2 -local deleted = 0 -while j <= #ARGV do - redis.call('HDEL', key, ARGV[j]) - deleted = deleted + 1 - j = j + 1 -end -return deleted diff --git a/src/test/kotlin/com/gzzn/omms/msgexchange/delivery/DispatcherTickTest.kt b/src/test/kotlin/com/gzzn/omms/msgexchange/delivery/DispatcherTickTest.kt index 462bbf0..1c81d07 100644 --- a/src/test/kotlin/com/gzzn/omms/msgexchange/delivery/DispatcherTickTest.kt +++ b/src/test/kotlin/com/gzzn/omms/msgexchange/delivery/DispatcherTickTest.kt @@ -71,7 +71,6 @@ class DispatcherTickTest { } override fun indexFlightHts(payloadJson: String) = Unit - override fun projectRedis(payloadJson: String) = Unit } private val clock = MutableClock(MutableClock.BASE) @@ -184,23 +183,4 @@ class DispatcherTickTest { assertEquals(EventStatus.PENDING, repo.events.first { it.eventId == 1L }.state) } - @Test - fun `delete events always carry legal JSON refs - null and non-null partitionKey`() { - val props = PipelineProps().apply { phase = PipelineProps.Phase.B } - val repo = FakeRepo() - val port = FakePort() - val d = dispatcher(repo, port, props) - repo.enqueue(ev(1, Targets.ES_FLIGHT_HTS, "F1", """{"h":1}""")) - repo.enqueue(ev(2, Targets.ES_FLIGHT_HTS, null, """{"h":2}""")) - - d.tick(); d.tick() // 每 target 每 tick 仅出队一条：两条 ES 事件需两个 tick - - val deletes = repo.syncInserted.filter { it.target == Targets.REDIS_FLIGHT_INFO } - assertEquals(2, deletes.size) - val mapper = com.fasterxml.jackson.databind.ObjectMapper() - val parsed = deletes.map { mapper.readTree(it.payloadJson) } // readTree 即合法性断言 - assertEquals(listOf("delete", "delete"), parsed.map { it.get("op").asText() }) - assertEquals("F1", parsed[0].get("refs").asText()) // 非空 → 带引号字符串 - assertTrue(parsed[1].get("refs").isNull) // null → null 字面量 - } } diff --git a/src/test/kotlin/com/gzzn/omms/msgexchange/infra/health/HealthIndicatorsTest.kt b/src/test/kotlin/com/gzzn/omms/msgexchange/infra/health/HealthIndicatorsTest.kt index 0967f40..61c1e8f 100644 --- a/src/test/kotlin/com/gzzn/omms/msgexchange/infra/health/HealthIndicatorsTest.kt +++ b/src/test/kotlin/com/gzzn/omms/msgexchange/infra/health/HealthIndicatorsTest.kt @@ -1,8 +1,6 @@ package com.gzzn.omms.msgexchange.infra.health import com.gzzn.omms.msgexchange.delivery.DeliveryPort -import com.gzzn.omms.msgexchange.infra.redis.FlightRedisClient -import com.gzzn.omms.msgexchange.infra.redis.RedisScript import io.micronaut.health.HealthStatus import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test @@ -13,40 +11,18 @@ import org.junit.jupiter.api.Test */ class HealthIndicatorsTest { - private class FakeRedis(private val pingResult: Boolean) : FlightRedisClient { - override fun eval(script: RedisScript, setPairs: List>, delFields: List) = Unit - override fun hgetAllFlightInfo(): Map = emptyMap() - override fun ping(): Boolean = pingResult - } - - private class ThrowingRedis : FlightRedisClient { - override fun eval(script: RedisScript, setPairs: List>, delFields: List) = Unit - override fun hgetAllFlightInfo(): Map = emptyMap() - override fun ping(): Boolean = throw RuntimeException("connection refused") - } - private class FakePort(private val pingResult: Boolean) : DeliveryPort { override fun sendKafka(topic: String, payloadJson: String) = Unit override fun indexFlightHts(payloadJson: String) = Unit - override fun projectRedis(payloadJson: String) = Unit override fun ping(): Boolean = pingResult } private class ThrowingPort : DeliveryPort { override fun sendKafka(topic: String, payloadJson: String) = Unit override fun indexFlightHts(payloadJson: String) = Unit - override fun projectRedis(payloadJson: String) = Unit override fun ping(): Boolean = throw RuntimeException("metadata fetch failed") } - @Test - fun `redis up only when ping succeeds`() { - assertEquals(HealthStatus.UP, redisHealth(FakeRedis(true)).status) - assertEquals(HealthStatus.DOWN, redisHealth(FakeRedis(false)).status) - assertEquals(HealthStatus.DOWN, redisHealth(ThrowingRedis()).status) - assertEquals(HealthStatus.DOWN, redisHealth(null).status) // bean 缺失 - } - @Test fun `kafka up only when ping succeeds`() { assertEquals(HealthStatus.UP, kafkaHealth(FakePort(true)).status) diff --git a/src/test/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/FlightSchdJdbcPgTest.kt b/src/test/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/FlightSchdJdbcPgTest.kt new file mode 100644 index 0000000..bf327b7 --- /dev/null +++ b/src/test/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/FlightSchdJdbcPgTest.kt @@ -0,0 +1,289 @@ +package com.gzzn.omms.msgexchange.infra.persistence.jdbc + +import com.gzzn.omms.msgexchange.domain.FlightChange +import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Assumptions.assumeTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import com.zaxxer.hikari.HikariDataSource +import java.sql.DriverManager +import java.time.Instant +import java.time.ZoneId +import java.util.TimeZone + +/** + * ACM2-28 FS6：PostgreSQL 真实方言集成测试（JdbcFlightSchdRepository） + * 验证： + * 1. DNLD 快照批处理写入（JDBC batch，200 批次）； + * 2. 增量 Upsert FDAY 保留策略（ON CONFLICT 保留原代，新插置 NULL）； + * 3. 域化差删（按 FDAY 严格隔离）； + * 4. SCHD_GEN SQL CAS（防并发断言）； + * 5. 单事务原子回滚（崩溃无残留）； + * 6. ES 历史清场 deleteByFlids 幂等删除。 + */ +class FlightSchdJdbcPgTest { + + private lateinit var ds: HikariDataSource + private lateinit var repo: JdbcFlightSchdRepository + + @BeforeEach + fun setUp() { + val url = "jdbc:postgresql://localhost:5432/msgx" + val user = "msgx_dev" + val pass = "msgx_dev_pass" + + // 仅当本地 PostgreSQL 容器可用时执行 + val canConnect = try { + DriverManager.getConnection(url, user, pass).use { true } + } catch (_: Exception) { + false + } + assumeTrue(canConnect, "Local PostgreSQL on port 5432 is not accessible, skipping PG dialect integration tests") + + ds = HikariDataSource().apply { + jdbcUrl = url + username = user + this.password = pass + driverClassName = "org.postgresql.Driver" + maximumPoolSize = 2 + } + repo = JdbcFlightSchdRepository(ds) + cleanup() + } + @AfterEach + fun tearDown() { + if (::ds.isInitialized) { + cleanup() + ds.close() + } + } + + private fun cleanup() { + try { + ds.update("DELETE FROM flight_schd WHERE flid LIKE 'TEST_%'") {} + ds.update("DELETE FROM schd_gen WHERE fday >= '2026-09-01'") {} + } catch (_: Exception) { + } + } + + @Test + fun `PG dialect - snapshot batch upsert, find, and domain diff delete`() { + val day = "2026-09-07" + // 构造 250 条记录跨越 batchSize 200 门限 + val flights = (1..250).map { i -> + "TEST_FL_$i" to """{"FLID":"TEST_FL_$i","air":"CA$i"}""" + } + + repo.upsertSnapshotBatch(day, flights) + + // 验证全量写入成功 + val byDay = repo.findByDay(day) + assertEquals(250, byDay.size) + + // 点查测试 + val f1 = repo.findByFlid("TEST_FL_1") + assertNotNull(f1) + assertTrue(f1!!.contains("CA1")) + + // 多键点查测试（分批查） + val subset = (1..50).map { "TEST_FL_$it" } + val map = repo.findByFlids(subset) + assertEquals(50, map.size) + + // 域内差删：删除 1..50 + val deleted = repo.deleteDiffByDay(day, subset) + assertEquals(50, deleted) + assertEquals(200, repo.findByDay(day).size) + assertNull(repo.findByFlid("TEST_FL_1")) + } + + @Test + fun `PG dialect - incremental upsert preserves existing FDAY and sets NULL for new`() { + val day = "2026-09-07" + // 1. 快照写入 REG_01，FDAY 为 2026-09-07 + repo.upsertSnapshotBatch(day, listOf("TEST_REG_01" to """{"FLID":"TEST_REG_01","v":1}""")) + + // 2. 增量更新 REG_01（已有行）与 ADFT_01（新行） + val changes = listOf( + FlightChange("TEST_REG_01", """{"FLID":"TEST_REG_01","v":2}"""), + FlightChange("TEST_ADFT_01", """{"FLID":"TEST_ADFT_01","v":1}"""), + ) + repo.upsertIncremental(changes) + + // 3. 验证 REG_01 内容更新但 FDAY 依然保留为 2026-09-07 + val reg01 = ds.queryOne("SELECT fday, fltr_json FROM flight_schd WHERE flid = 'TEST_REG_01'", {}) { rs -> + rs.getDate("fday")?.toString() to rs.getString("fltr_json") + } + assertNotNull(reg01) + assertEquals("2026-09-07", reg01!!.first) + assertTrue(reg01.second.contains("\"v\": 2") || reg01.second.contains("\"v\":2")) + + // 4. 验证 ADFT_01 新插入行 FDAY 恒为 NULL + val adft01 = ds.queryOne("SELECT fday, fltr_json FROM flight_schd WHERE flid = 'TEST_ADFT_01'", {}) { rs -> + rs.getDate("fday")?.toString() to rs.getString("fltr_json") + } + assertNotNull(adft01) + assertNull(adft01!!.first) + + // 5. 执行 2026-09-07 的差删，ADFT_01 绝不被误删 + val deleted = repo.deleteDiffByDay(day, listOf("TEST_ADFT_01")) + assertEquals(0, deleted) + assertNotNull(repo.findByFlid("TEST_ADFT_01")) + } + + @Test + fun `PG dialect - SCHD_GEN SQL CAS version advance and conflict rejection`() { + val day = "2026-09-07" + + // 初始 CAS: expected = 0，版本推进至 1 + val gen1 = FlightSchdRepository.GenMeta(day, 1L, setOf("TEST_A", "TEST_B")) + val ok1 = repo.putGenIfVersion(day, 0L, gen1) + assertTrue(ok1) + + val stored1 = repo.getGen(day) + assertNotNull(stored1) + assertEquals(1L, stored1!!.version) + assertEquals(setOf("TEST_A", "TEST_B"), stored1.flids) + + // 成功 CAS: expected = 1，版本推进至 2 + val gen2 = FlightSchdRepository.GenMeta(day, 2L, setOf("TEST_A", "TEST_C")) + val ok2 = repo.putGenIfVersion(day, 1L, gen2) + assertTrue(ok2) + assertEquals(2L, repo.getGen(day)!!.version) + + // 冲突 CAS: expected 依然传 1（已过期的旧版本），应被拒并返回 false + val genConflict = FlightSchdRepository.GenMeta(day, 3L, setOf("TEST_CONFLICT")) + val conflictResult = repo.putGenIfVersion(day, 1L, genConflict) + assertFalse(conflictResult) + + // 确认版本未被非法篡改，仍为 2 + assertEquals(2L, repo.getGen(day)!!.version) + } + + @Test + fun `PG dialect - Transaction rollback leaves zero partial rows on failure`() { + val day = "2026-09-07" + try { + ds.withTransaction { + repo.upsertSnapshotBatch(day, listOf("TEST_TX_01" to """{"tx":1}""")) + repo.putGenIfVersion(day, 0L, FlightSchdRepository.GenMeta(day, 1L, setOf("TEST_TX_01"))) + // 模拟事务内部抛出异常 + throw IllegalStateException("forced-abort-for-rollback-test") + } + } catch (_: IllegalStateException) { + // 异常捕获 + } + + // 断言：由于事务原子回滚，数据库中零残留 + assertNull(repo.findByFlid("TEST_TX_01")) + assertNull(repo.getGen(day)) + } + + @Test + fun `PG dialect - ES history sweep deleteByFlids idempotent batch execution`() { + val day = "2026-09-07" + val flids = (1..50).map { "TEST_SWEEP_$it" } + repo.upsertSnapshotBatch(day, flids.map { it to """{"flid":"$it"}""" }) + assertEquals(50, repo.findByFlids(flids).size) + + // 第一次分批删除 + val deleted1 = repo.deleteByFlids(flids.toSet()) + assertEquals(50, deleted1) + assertEquals(0, repo.findByFlids(flids).size) + + // 重放删除：安全返回 0，无副作用 + val deleted2 = repo.deleteByFlids(flids.toSet()) + assertEquals(0, deleted2) + } + + @Test + fun `PG dialect - non-UTC JVM default timezone preserves exact Instant timestamps without drift`() { + val originalTz = TimeZone.getDefault() + try { + // 1. 设置 JVM 默认时区为东京 (UTC+09:00) + TimeZone.setDefault(TimeZone.getTimeZone(ZoneId.of("Asia/Tokyo"))) + val day = "2026-09-07" + val fixedInstant = Instant.parse("2026-09-07T08:30:15.123456Z") + + repo.upsertSnapshotBatch(day, listOf("TEST_TZ_01" to """{"tz":1}"""), fixedInstant) + repo.putGenIfVersion(day, 0L, FlightSchdRepository.GenMeta(day, 1L, setOf("TEST_TZ_01")), fixedInstant) + + // 东京时区下回读 + val genTokyo = repo.getGen(day) + assertNotNull(genTokyo) + assertEquals(fixedInstant, genTokyo!!.updatedAt) + + // 2. 切换 JVM 默认时区为纽约 (UTC-05:00) + TimeZone.setDefault(TimeZone.getTimeZone(ZoneId.of("America/New_York"))) + + // 纽约时区下再次回读同一条记录 + val genNy = repo.getGen(day) + assertNotNull(genNy) + // 断言：TIMESTAMPTZ 映射到 Instant 完全相等，绝无 JVM 时区偏移 + assertEquals(fixedInstant, genNy!!.updatedAt) + + val flightNy = ds.queryOne("SELECT created_at, updated_at FROM flight_schd WHERE flid = 'TEST_TZ_01'", {}) { rs -> + rs.getInstant("created_at") to rs.getInstant("updated_at") + } + assertNotNull(flightNy) + assertEquals(fixedInstant, flightNy!!.first) + assertEquals(fixedInstant, flightNy.second) + } finally { + TimeZone.setDefault(originalTz) + } + } + + @Test + fun `PG dialect - non-UTC database session timezone produces zero Instant drift across sessions`() { + val day = "2026-09-07" + val writeInstant = Instant.parse("2026-09-07T14:45:00.654321Z") + + // 1. 在会话 1 中显式设置会话时区为 'Asia/Shanghai' (+08:00) 并写入数据 + ds.connection.use { conn -> + conn.createStatement().use { it.execute("SET TIME ZONE 'Asia/Shanghai'") } + conn.prepareStatement( + """ + INSERT INTO flight_schd (flid, fday, fltr_json, created_at, updated_at) + VALUES ('TEST_SESS_01', '2026-09-07', '{"session":"shanghai"}'::jsonb, ?, ?) + """.trimIndent(), + ).use { ps -> + ps.setTimestamp(1, writeInstant.toSqlTimestamp()) + ps.setTimestamp(2, writeInstant.toSqlTimestamp()) + ps.executeUpdate() + } + } + + // 2. 在会话 2 中显式设置会话时区为 'America/Chicago' (-05:00) 并回读数据 + val instantChicago = ds.connection.use { conn -> + conn.createStatement().use { it.execute("SET TIME ZONE 'America/Chicago'") } + conn.prepareStatement("SELECT created_at, updated_at FROM flight_schd WHERE flid = 'TEST_SESS_01'").use { ps -> + ps.executeQuery().use { rs -> + assertTrue(rs.next()) + rs.getInstant("created_at") + } + } + } + + // 3. 在会话 3 中显式设置会话时区为 'UTC' 并回读数据 + val instantUtc = ds.connection.use { conn -> + conn.createStatement().use { it.execute("SET TIME ZONE 'UTC'") } + conn.prepareStatement("SELECT created_at, updated_at FROM flight_schd WHERE flid = 'TEST_SESS_01'").use { ps -> + ps.executeQuery().use { rs -> + assertTrue(rs.next()) + rs.getInstant("created_at") + } + } + } + + // 断言：跨不同数据库会话时区（+08:00 vs -05:00 vs UTC），读取出的 Instant 与写入的 UTC Instant 完全一致！ + assertEquals(writeInstant, instantChicago) + assertEquals(writeInstant, instantUtc) + assertEquals(instantChicago, instantUtc) + } +} diff --git a/src/test/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/FlywayMigrationTest.kt b/src/test/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/FlywayMigrationTest.kt new file mode 100644 index 0000000..b1fc815 --- /dev/null +++ b/src/test/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/FlywayMigrationTest.kt @@ -0,0 +1,74 @@ +package com.gzzn.omms.msgexchange.infra.persistence.jdbc + +import org.flywaydb.core.Flyway +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Assumptions.assumeTrue +import org.junit.jupiter.api.Test +import java.sql.DriverManager + +/** + * P2-1 验证测试：Flyway 迁移引擎在真实 PostgreSQL 上的端到端自动迁移验证。 + * 断言： + * 1. Flyway 自动执行 V1.0.0__own_pg_pipeline.sql 与 V1.1.0__flight_schd.sql； + * 2. 真实落库 flyway_schema_history 元数据表，且 success = true； + * 3. 运营航班表 FLIGHT_SCHD 与计划代 SCHD_GEN 结构完整就绪。 + */ +class FlywayMigrationTest { + + @Test + fun `Flyway automatically migrates V1_0_0 and V1_1_0 onto real PostgreSQL`() { + val url = "jdbc:postgresql://localhost:5432/msgx" + val user = "msgx_dev" + val pass = "msgx_dev_pass" + + val canConnect = try { + DriverManager.getConnection(url, user, pass).use { true } + } catch (_: Exception) { + false + } + assumeTrue(canConnect, "PostgreSQL on port 5432 not accessible, skipping Flyway test") + + val flyway = Flyway.configure() + .dataSource(url, user, pass) + .locations("classpath:db/migration") + .load() + + val migrateResult = flyway.migrate() + assertTrue(migrateResult.success, "Flyway migration must succeed") + + // 验证 flyway_schema_history 元数据表与执行记录 + DriverManager.getConnection(url, user, pass).use { conn -> + conn.createStatement().use { stmt -> + stmt.executeQuery( + "SELECT version, script, success FROM flyway_schema_history ORDER BY installed_rank ASC", + ).use { rs -> + val records = mutableListOf>() + while (rs.next()) { + records.add(Triple(rs.getString("version"), rs.getString("script"), rs.getBoolean("success"))) + } + assertTrue(records.size >= 2, "At least 2 migrations must be recorded in flyway_schema_history") + assertEquals("1.0.0", records[0].first) + assertEquals("V1.0.0__own_pg_pipeline.sql", records[0].second) + assertTrue(records[0].third) + + assertEquals("1.1.0", records[1].first) + assertEquals("V1.1.0__flight_schd.sql", records[1].second) + assertTrue(records[1].third) + } + + // 验证 FLIGHT_SCHD 与 SCHD_GEN 存在 + stmt.executeQuery( + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_name IN ('flight_schd', 'schd_gen')", + ).use { rs -> + val tables = mutableSetOf() + while (rs.next()) { + tables.add(rs.getString("table_name")) + } + assertTrue(tables.contains("flight_schd")) + assertTrue(tables.contains("schd_gen")) + } + } + } + } +} diff --git a/src/test/kotlin/com/gzzn/omms/msgexchange/jobs/HistorySweepJobTest.kt b/src/test/kotlin/com/gzzn/omms/msgexchange/jobs/HistorySweepJobTest.kt new file mode 100644 index 0000000..774cbb4 --- /dev/null +++ b/src/test/kotlin/com/gzzn/omms/msgexchange/jobs/HistorySweepJobTest.kt @@ -0,0 +1,167 @@ +package com.gzzn.omms.msgexchange.jobs + +import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository +import com.gzzn.omms.msgexchange.infra.stub.StubFlightSchd +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +/** + * ACM2-28 FS6 历史清场测试： + * 覆盖 ES 全成功、部分成功、全部失败、ES 成功后 PG 删除前崩溃、PG 删除重放五种场景； + * 断言失败项不丢、成功项最终删除、重复执行无副作用。 + */ +class HistorySweepJobTest { + + private lateinit var flightSchd: StubFlightSchd + private lateinit var sweepJob: HistorySweepJob + + @BeforeEach + fun setUp() { + flightSchd = StubFlightSchd() + sweepJob = HistorySweepJob(flightSchd) + HistorySweepJob.historyPicker = null + HistorySweepJob.esArchiver = null + HistorySweepJob.cutoffProvider = null + } + + @AfterEach + fun tearDown() { + HistorySweepJob.historyPicker = null + HistorySweepJob.esArchiver = null + HistorySweepJob.cutoffProvider = null + } + + // 1. 场景一：ES 全成功 + @Test + fun `Scenario 1 - ES All Success removes all confirmed flights and cleans old gens`() { + val flights = listOf( + "F1" to """{"FLID":"F1"}""", + "F2" to """{"FLID":"F2"}""", + "F3" to """{"FLID":"F3"}""", + ) + flightSchd.upsertSnapshotBatch("2026-09-01", flights) + flightSchd.putGenIfVersion("2026-09-01", 0L, FlightSchdRepository.GenMeta("2026-09-01", 1L, setOf("F1", "F2", "F3"))) + + // 模拟判史：F1, F2, F3 全部为历史候选 + HistorySweepJob.historyPicker = { it } + // 模拟 ES 写入：全部成功返回 + HistorySweepJob.esArchiver = { it.keys } + // 模拟历史代清理：清理 2026-09-02 之前的代 + HistorySweepJob.cutoffProvider = { "2026-09-02" } + + sweepJob.run() + + // 断言：PG 中 F1, F2, F3 全部被删除 + assertNull(flightSchd.findByFlid("F1")) + assertNull(flightSchd.findByFlid("F2")) + assertNull(flightSchd.findByFlid("F3")) + // 断言：过期代元数据被清理 + assertNull(flightSchd.getGen("2026-09-01")) + } + + // 2. 场景二：ES 部分成功 + @Test + fun `Scenario 2 - ES Partial Success deletes only successful flights and keeps failed ones`() { + val flights = listOf( + "F1" to """{"FLID":"F1"}""", + "F2" to """{"FLID":"F2"}""", + "F3" to """{"FLID":"F3"}""", + ) + flightSchd.upsertSnapshotBatch("2026-09-01", flights) + + HistorySweepJob.historyPicker = { it } + // 模拟 ES 写入：F1 和 F2 成功，F3 失败 + HistorySweepJob.esArchiver = { setOf("F1", "F2") } + + sweepJob.run() + + // 断言：成功项最终删除 + assertNull(flightSchd.findByFlid("F1")) + assertNull(flightSchd.findByFlid("F2")) + + // 断言：失败项 F3 绝不丢失，继续保留在 PG 中供下次重试 + assertNotNull(flightSchd.findByFlid("F3")) + } + + // 3. 场景三：ES 全部失败 + @Test + fun `Scenario 3 - ES All Failure deletes nothing and keeps all candidates in PG`() { + val flights = listOf( + "F1" to """{"FLID":"F1"}""", + "F2" to """{"FLID":"F2"}""", + ) + flightSchd.upsertSnapshotBatch("2026-09-01", flights) + + HistorySweepJob.historyPicker = { it } + // 模拟 ES 全部写入失败 + HistorySweepJob.esArchiver = { emptySet() } + + sweepJob.run() + + // 断言：PG 未执行任何删除，全部候选均安全保留 + assertNotNull(flightSchd.findByFlid("F1")) + assertNotNull(flightSchd.findByFlid("F2")) + } + + // 4. 场景四：ES 成功后 PG 删除前崩溃重放 + @Test + fun `Scenario 4 - Crash after ES success before PG delete recovers idempotently on next run`() { + val flights = listOf( + "F1" to """{"FLID":"F1"}""", + ) + flightSchd.upsertSnapshotBatch("2026-09-01", flights) + + // 第一次运行模拟：ES 写入成功，但在调用 deleteByFlids 前崩溃（PG 依然保留 F1） + val esUpserted = mutableSetOf() + var crashed = false + + HistorySweepJob.historyPicker = { it } + HistorySweepJob.esArchiver = { + esUpserted.addAll(it.keys) + if (!crashed) { + crashed = true + throw RuntimeException("crash-before-pg-delete") + } + it.keys + } + + try { + sweepJob.run() + } catch (_: RuntimeException) { + // 崩溃发生 + } + + // 崩溃后断言：ES 已有数据，但 PG 中 F1 仍然存在作为可恢复事实 + assertTrue(esUpserted.contains("F1")) + assertNotNull(flightSchd.findByFlid("F1")) + + // 下一次作业重试： + sweepJob.run() + + // 断言：重试成功，PG 中 F1 最终完成删除 + assertNull(flightSchd.findByFlid("F1")) + } + + // 5. 场景五：PG 删除重复执行（无副作用与幂等性） + @Test + fun `Scenario 5 - Replaying deleteByFlids is completely idempotent with no side effects`() { + val flights = listOf("F1" to """{"FLID":"F1"}""") + flightSchd.upsertSnapshotBatch("2026-09-01", flights) + + // 首次删除 + val deletedCount1 = flightSchd.deleteByFlids(setOf("F1")) + assertEquals(1, deletedCount1) + assertNull(flightSchd.findByFlid("F1")) + + // 重复执行删除（重放） + val deletedCount2 = flightSchd.deleteByFlids(setOf("F1")) + // 断言：允许重放时影响行数为 0，不抛出异常，无任何副作用 + assertEquals(0, deletedCount2) + assertNull(flightSchd.findByFlid("F1")) + } +} diff --git a/src/test/kotlin/com/gzzn/omms/msgexchange/processing/FlightSchdInvariantTest.kt b/src/test/kotlin/com/gzzn/omms/msgexchange/processing/FlightSchdInvariantTest.kt new file mode 100644 index 0000000..e5d0447 --- /dev/null +++ b/src/test/kotlin/com/gzzn/omms/msgexchange/processing/FlightSchdInvariantTest.kt @@ -0,0 +1,353 @@ +package com.gzzn.omms.msgexchange.processing + +import com.gzzn.omms.msgexchange.MutableClock +import com.gzzn.omms.msgexchange.codec.DecodeResult +import com.gzzn.omms.msgexchange.codec.XmlCodec +import com.gzzn.omms.msgexchange.config.PipelineProps +import com.gzzn.omms.msgexchange.domain.DecodedMessage +import com.gzzn.omms.msgexchange.domain.Decision +import com.gzzn.omms.msgexchange.domain.ErrorClass +import com.gzzn.omms.msgexchange.domain.FlightChange +import com.gzzn.omms.msgexchange.domain.MetaFields +import com.gzzn.omms.msgexchange.domain.MsgEvent +import com.gzzn.omms.msgexchange.domain.MsgKind +import com.gzzn.omms.msgexchange.domain.NotifyPayload +import com.gzzn.omms.msgexchange.domain.ProcState +import com.gzzn.omms.msgexchange.domain.ProcStatus +import com.gzzn.omms.msgexchange.domain.SchdPush +import com.gzzn.omms.msgexchange.domain.Targets +import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository +import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository +import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository +import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager +import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository +import com.gzzn.omms.msgexchange.infra.persistence.ReqTrackRepository +import com.gzzn.omms.msgexchange.infra.retry.FailureScheduler +import com.gzzn.omms.msgexchange.infra.retry.ProcFailure +import com.gzzn.omms.msgexchange.infra.stub.StubFlightSchd +import com.gzzn.omms.msgexchange.infra.stub.StubMsgEvents +import com.gzzn.omms.msgexchange.infra.stub.StubPipelineTransactionManager +import com.gzzn.omms.msgexchange.infra.stub.StubProcState +import com.gzzn.omms.msgexchange.infra.stub.StubReqTrack +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.time.Instant +import java.time.ZoneId +import java.util.TimeZone + +/** + * U09 / U29 不变量门禁测试（ACM2-28 FS6 核心验收门槛）： + * 1. I2 不变量：航班状态变更、MSG_EVENT 插入、PROC_STATE 终态在单事务原子提交；失败整体回滚重放； + * 2. 门槛 1（崩溃幂等无自增）：重放相同快照报文后 SCHD_GEN.VERSION 绝不发生二次自增； + * 3. 门槛 2（CAS 防并发）：伪造版本过期模拟并发快照，准确拦截 CAS 冲突并转入 FAILED(INFRA) 与退避； + * 4. 门槛 3（ADFT 存活保障）：增量 ADFT 航班在后续 DNLD 跨代替换后天然存活、不被差删；跨代迁移保护； + * 5. UTC 方言测试：非 UTC JVM 时区下读写与幂等判据无漂移。 + */ +class FlightSchdInvariantTest { + + private lateinit var flightSchd: StubFlightSchd + private lateinit var procState: StubProcState + private lateinit var msgEvents: StubMsgEvents + private lateinit var reqTrack: StubReqTrack + private lateinit var txManager: StubPipelineTransactionManager + private lateinit var inbox: FakeInbox + private lateinit var clock: MutableClock + private lateinit var scheduler: FailureScheduler + private lateinit var procFailure: ProcFailure + private lateinit var props: PipelineProps + + class FakeInbox : CminmsgInboxRepository { + val raws = mutableMapOf() + val backfilled = mutableListOf() + + override fun insertRaw(rawXml: String): Long = 1L + override fun rawOf(cminmsgsId: Long): String? = raws[cminmsgsId] + override fun pollUnprocessed(afterId: Long, limit: Int): List = emptyList() + override fun backfillOnSuccess(cminmsgsId: Long, sndr: String, type: String, styp: String, seqn: Long) { + backfilled.add(cminmsgsId) + } + } + + class SimpleCodec(private val decodedMsg: DecodedMessage) : XmlCodec { + override fun decode(rawXml: String): DecodeResult = DecodeResult.Ok(decodedMsg) + override fun encodeRqrd(kind: String, rangeJson: String): String = "" + } + + @BeforeEach + fun setUp() { + flightSchd = StubFlightSchd() + procState = StubProcState() + msgEvents = StubMsgEvents() + reqTrack = StubReqTrack() + txManager = StubPipelineTransactionManager() + inbox = FakeInbox() + clock = MutableClock(MutableClock.BASE) + props = PipelineProps() + scheduler = FailureScheduler(props, clock) + procFailure = ProcFailure(procState, scheduler) + SnapshotFlow.StageResult.parser = null + } + + @AfterEach + fun tearDown() { + SnapshotFlow.StageResult.parser = null + } + + // ========================================================================= + // I2 不变量测试：普通报文事务 2 扩展（变更 + 事件 + SUCCEEDED 原子提交） + // ========================================================================= + + @Test + fun `I2 - MessageProcessor atomically commits flightChanges, msgEvents, and SUCCEEDED`() { + val headId = 101L + inbox.raws[headId] = "F101" + procState.insert(headId, ProcStatus.PENDING) + + val meta = MetaFields("AODB", "FLOP", "DELY", 1L, 20260907120000L) + val decoded = DecodedMessage(meta, MsgKind.Flop("DELY"), "F101") + val codecHolder = CodecHolder(SimpleCodec(decoded)) + + val handler = object : Handler { + override val kind: MsgKind = MsgKind.Flop("DELY") + override fun decide(flightView: Map, msg: DecodedMessage): Decision { + return Decision( + flightChanges = listOf(FlightChange("F101", """{"FLID":"F101","status":"DELAYED"}""")), + msgNotifies = listOf(NotifyPayload("""{"flid":"F101","event":"DELAY"}""")), + schdPush = listOf(SchdPush("F101", """{"FLID":"F101","status":"DELAYED"}""")), + ) + } + } + val handlerHolder = HandlerHolder(HandlerRegistry(listOf(handler))) + val snapshotFlow = SnapshotFlow(procState, flightSchd, msgEvents, reqTrack, procFailure, txManager, inbox) + val processor = MessageProcessor( + inbox, procState, msgEvents, codecHolder, handlerHolder, + flightSchd, snapshotFlow, procFailure, props, txManager, + ) + + val head = procState.headUnfinished()!! + processor.processOne(head) + + // 断言：FLIGHT_SCHD 有更新 + val fltr = flightSchd.findByFlid("F101") + assertNotNull(fltr) + assertTrue(fltr!!.contains("DELAYED")) + + // 断言：MSG_EVENT 写入了 KAFKA_MSG 和 KAFKA_SCHD 两条事件 + val evtMsg = msgEvents.headUnsent(Targets.KAFKA_MSG) + assertNotNull(evtMsg) + val evtSchd = msgEvents.headUnsent(Targets.KAFKA_SCHD) + assertNotNull(evtSchd) + + // 断言：伴生状态更新为 SUCCEEDED + val currentHead = procState.headUnfinished() + assertNull(currentHead) // 无未完成队头，说明 101 已终态 + + // 断言：共享信箱 backfill 成功调用 + assertTrue(inbox.backfilled.contains(headId)) + } + + // ========================================================================= + // FS6 门槛 1：崩溃幂等无自增（重放相同快照报文，SCHD_GEN.VERSION 绝不发生二次自增） + // ========================================================================= + + @Test + fun `Threshold 1 - Crash idempotency without version re-increment`() { + val headId = 201L + val day = "2026-09-07" + procState.insert(headId, ProcStatus.PENDING) + + val meta = MetaFields("AODB", "SCHD", "DNLD", 1L, 20260907030000L) + val decoded = DecodedMessage(meta, MsgKind.Schd(MsgKind.SchdSubtype.DNLD), "") + + // 配置 staging 解析模拟产出 2 条航班 + val flights = listOf( + "FL_01" to """{"FLID":"FL_01","air":"CA1234"}""", + "FL_02" to """{"FLID":"FL_02","air":"MU5678"}""", + ) + SnapshotFlow.StageResult.parser = { SnapshotFlow.StageResult.Ok(day, flights) } + + // 模拟快照单事务内崩溃注入（例如在提交前崩溃回滚） + var crashInjected = true + val rollbackTxManager = object : PipelineTransactionManager { + override fun inTransaction(block: () -> T): T { + if (crashInjected) { + try { + block() + } finally { + // 事务回滚：清除未提交修改并恢复状态 + flightSchd.clear() + procState.update(headId, ProcStatus.PENDING) + } + throw RuntimeException("crash-before-pg-commit") + } + return block() + } + } + + val snapshotFlowCrashing = SnapshotFlow(procState, flightSchd, msgEvents, reqTrack, procFailure, rollbackTxManager, inbox) + + // 首次运行：事务内崩溃注入 + val head1 = procState.headUnfinished()!! + try { + snapshotFlowCrashing.publishSnapshot(head1, decoded) + } catch (_: RuntimeException) { + // 崩溃发生 + } + + // 崩溃后断言：由于 PG 单事务回滚，SCHD_GEN 不存在半成品中间态，版本未推进 + assertNull(flightSchd.getGen(day)) + + // 重放相同报文（正常完成） + crashInjected = false + val snapshotFlowNormal = SnapshotFlow(procState, flightSchd, msgEvents, reqTrack, procFailure, txManager, inbox) + val headReplay = procState.headUnfinished()!! + snapshotFlowNormal.publishSnapshot(headReplay, decoded) + + // 门槛 1 断言：版本绝不发生二次自增，版本号精确为 1L（根除旧 Redis 两阶段二次自增缺陷） + val genAfterReplay = flightSchd.getGen(day) + assertNotNull(genAfterReplay) + assertEquals(1L, genAfterReplay!!.version) + assertEquals(setOf("FL_01", "FL_02"), genAfterReplay.flids) + } + // ========================================================================= + // FS6 门槛 2：CAS 防并发（伪造版本过期模拟并发快照，拦截 CAS 冲突并转入 FAILED(INFRA)） + // ========================================================================= + + @Test + fun `Threshold 2 - CAS prevents concurrency and enters FAILED INFRA`() { + val headId = 202L + val day = "2026-09-07" + procState.insert(headId, ProcStatus.PENDING) + + val meta = MetaFields("AODB", "SCHD", "DNLD", 2L, 20260907033000L) + val decoded = DecodedMessage(meta, MsgKind.Schd(MsgKind.SchdSubtype.DNLD), "") + + val flights = listOf("FL_01" to """{"FLID":"FL_01"}""") + SnapshotFlow.StageResult.parser = { SnapshotFlow.StageResult.Ok(day, flights) } + + // 先预置日代版本为 5L（模拟另一并发实例已经推进了版本） + flightSchd.putGenIfVersion(day, 0L, FlightSchdRepository.GenMeta(day, 5L, setOf("FL_OLD"))) + + // 构造一个在读取版本后版本被篡改的场景（模拟读到 5L 后，外部并发变成了 6L） + val mockFlightSchd = object : FlightSchdRepository by flightSchd { + override fun getGen(day: String): FlightSchdRepository.GenMeta? { + // 模拟读取时返回版本 5L + return FlightSchdRepository.GenMeta(day, 5L, setOf("FL_OLD")) + } + + override fun putGenIfVersion(day: String, expected: Long, newGen: FlightSchdRepository.GenMeta, now: Instant): Boolean { + // 模拟 CAS 校验失败（数据库已被并发推进，expected 5 已过期） + return false + } + } + + val snapshotFlow = SnapshotFlow(procState, mockFlightSchd, msgEvents, reqTrack, procFailure, txManager, inbox) + val head = procState.headUnfinished()!! + snapshotFlow.publishSnapshot(head, decoded) + + // 门槛 2 断言：CAS 冲突被拦截，状态转换为 FAILED(INFRA)，带有退避与告警错误信息 + val failedHead = procState.headUnfinished()!! + assertEquals(ProcStatus.FAILED, failedHead.state) + assertEquals(ErrorClass.INFRA, failedHead.errorClass) + assertTrue(failedHead.lastError?.contains("gen-cas-conflict") == true) + assertNotNull(failedHead.nextAttemptAt) + } + + // ========================================================================= + // FS6 门槛 3：ADFT 存活保障（增量 ADFT 航班在后续 DNLD 跨代替换后天然存活、不被差删） + // ========================================================================= + + @Test + fun `Threshold 3 - ADFT flights survive subsequent DNLD replacement`() { + val day = "2026-09-07" + + // 1. 增量更新写入一条临时加飞航班 ADFT（FDAY 为 NULL） + val adftChange = FlightChange(flid = "ADFT_888", payloadJson = """{"FLID":"ADFT_888","type":"ADFT"}""") + flightSchd.upsertIncremental(listOf(adftChange)) + + // 2. 写入旧代的一条定期计划航班 REG_OLD（FDAY 为 2026-09-07） + flightSchd.upsertSnapshotBatch(day, listOf("REG_OLD" to """{"FLID":"REG_OLD","type":"REG"}""")) + flightSchd.putGenIfVersion(day, 0L, FlightSchdRepository.GenMeta(day, 1L, setOf("REG_OLD"))) + + // 3. 执行下一轮快照 DNLD，新代仅包含 REG_NEW（REG_OLD 不在新代中，属于待删差集；ADFT 也不在新代中） + val headId = 203L + procState.insert(headId, ProcStatus.PENDING) + val meta = MetaFields("AODB", "SCHD", "DNLD", 3L, 20260907040000L) + val decoded = DecodedMessage(meta, MsgKind.Schd(MsgKind.SchdSubtype.DNLD), "") + + val newFlights = listOf("REG_NEW" to """{"FLID":"REG_NEW","type":"REG"}""") + SnapshotFlow.StageResult.parser = { SnapshotFlow.StageResult.Ok(day, newFlights) } + + val snapshotFlow = SnapshotFlow(procState, flightSchd, msgEvents, reqTrack, procFailure, txManager, inbox) + val head = procState.headUnfinished()!! + snapshotFlow.publishSnapshot(head, decoded) + + // 门槛 3 核心断言： + // ① 旧代航班 REG_OLD 被按代差删清除 + assertNull(flightSchd.findByFlid("REG_OLD")) + + // ② 新代航班 REG_NEW 成功写入 + assertNotNull(flightSchd.findByFlid("REG_NEW")) + + // ③ 增量 ADFT_888 航班由于 FDAY=NULL 天然存活、绝不被误删！ + val adftRecord = flightSchd.findByFlid("ADFT_888") + assertNotNull(adftRecord) + assertTrue(adftRecord!!.contains("ADFT_888")) + } + + @Test + fun `Threshold 3 - Migrated flight across days is protected from diff deletion of old day`() { + // 场景：航班 FL_MIG 原属 2026-09-07，后来迁移到了 2026-09-08（FDAY 更新为 09-08） + val dayOld = "2026-09-07" + val dayNew = "2026-09-08" + + // 初始属旧代 + flightSchd.upsertSnapshotBatch(dayOld, listOf("FL_MIG" to """{"FLID":"FL_MIG","day":"07"}""")) + flightSchd.putGenIfVersion(dayOld, 0L, FlightSchdRepository.GenMeta(dayOld, 1L, setOf("FL_MIG"))) + + // 随后 09-08 快照写入将 FDAY 更新为 2026-09-08 + flightSchd.upsertSnapshotBatch(dayNew, listOf("FL_MIG" to """{"FLID":"FL_MIG","day":"08"}""")) + + // 此时 09-07 再次执行差删（差集中包含 FL_MIG） + val deleted = flightSchd.deleteDiffByDay(dayOld, listOf("FL_MIG")) + + // 断言：FL_MIG 虽在差集，但由于 FDAY 已迁移至 09-08，受到域化差删保护，删除数为 0，记录依然存活！ + assertEquals(0, deleted) + assertNotNull(flightSchd.findByFlid("FL_MIG")) + } + + // ========================================================================= + // UTC 时区规范测试：非 UTC JVM 默认时区与会话下写入读取无漂移 + // ========================================================================= + + @Test + fun `UTC Dialect - Timestamps and instant evaluations are immune to JVM timezone drift`() { + val originalTz = TimeZone.getDefault() + try { + // 切换 JVM 默认时区为非 UTC（东京 +09:00 与 纽约 -05:00） + TimeZone.setDefault(TimeZone.getTimeZone(ZoneId.of("Asia/Tokyo"))) + + val now = Instant.parse("2026-09-07T08:00:00.123456Z") + flightSchd.upsertSnapshotBatch("2026-09-07", listOf("TZ_01" to """{"test":true}"""), now) + flightSchd.putGenIfVersion("2026-09-07", 0L, FlightSchdRepository.GenMeta("2026-09-07", 1L, setOf("TZ_01")), now) + + val gen = flightSchd.getGen("2026-09-07") + assertNotNull(gen) + assertEquals(now, gen!!.updatedAt) + + // 再次切换到西五区 + TimeZone.setDefault(TimeZone.getTimeZone(ZoneId.of("America/New_York"))) + val genNy = flightSchd.getGen("2026-09-07") + assertNotNull(genNy) + assertEquals(now, genNy!!.updatedAt) // 绝无时区漂移 + } finally { + TimeZone.setDefault(originalTz) + } + } +} diff --git a/src/test/kotlin/com/gzzn/omms/msgexchange/processing/MessageProcessorTest.kt b/src/test/kotlin/com/gzzn/omms/msgexchange/processing/MessageProcessorTest.kt index 8824343..561c491 100644 --- a/src/test/kotlin/com/gzzn/omms/msgexchange/processing/MessageProcessorTest.kt +++ b/src/test/kotlin/com/gzzn/omms/msgexchange/processing/MessageProcessorTest.kt @@ -21,11 +21,9 @@ import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository import com.gzzn.omms.msgexchange.infra.persistence.PumpJobRepository -import com.gzzn.omms.msgexchange.infra.persistence.RefDataRepository import com.gzzn.omms.msgexchange.infra.persistence.ReqTrackRepository -import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository -import com.gzzn.omms.msgexchange.infra.redis.FlightRedisClient -import com.gzzn.omms.msgexchange.infra.redis.RedisScript +import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository +import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager import com.gzzn.omms.msgexchange.infra.retry.FailureScheduler import com.gzzn.omms.msgexchange.infra.retry.ProcFailure import org.junit.jupiter.api.Assertions.assertEquals @@ -139,17 +137,48 @@ class MessageProcessorTest { override fun encodeRqrd(kind: String, rangeJson: String): String = "" } - private object FakeRedis : FlightRedisClient { - override fun eval(script: RedisScript, setPairs: List>, delFields: List) = Unit - override fun hgetAllFlightInfo(): Map = emptyMap() - override fun ping(): Boolean = true + private class FakeReqTrack : ReqTrackRepository { + override fun findOpenByKind(kind: String): ReqTrackRepository.Req? = null + override fun forceExpireOpenOf(kind: String) = Unit + override fun insert(kind: String, paramsJson: String): Long = 1L + override fun linkCoutmsgs(reqId: Long, coutmsgsId: Long) = Unit + override fun markSent(reqId: Long, sentAt: Instant) = Unit + override fun expireIfWaiting(reqId: Long) = Unit + override fun markDone(reqId: Long) = Unit } - private class FakeRefData : RefDataRepository { - override fun getGen(day: String): RefDataRepository.GenMeta? = null - override fun putGenIfVersion(day: String, expected: Long, new: RefDataRepository.GenMeta): Boolean = true + private class FakeFlightSchd : FlightSchdRepository { + val flights = mutableMapOf() + val gens = mutableMapOf() + val incrementalChanges = mutableListOf() + + override fun upsertSnapshotBatch(day: String, flights: List>, now: Instant) { + this.flights.putAll(flights) + } + override fun upsertIncremental(changes: List, now: Instant) { + incrementalChanges.addAll(changes) + changes.forEach { flights[it.flid] = it.payloadJson } + } + override fun deleteDiffByDay(day: String, delFlids: Collection): Int = 0 + override fun findByFlid(flid: String): String? = flights[flid] + override fun findByFlids(flids: Collection): Map = + flids.mapNotNull { f -> flights[f]?.let { f to it } }.toMap() + override fun findByDay(day: String): List> = flights.map { it.key to it.value } + override fun findAll(): Map = flights.toMap() + override fun deleteByFlids(flids: Set): Int = 0 + override fun getGen(day: String): FlightSchdRepository.GenMeta? = gens[day] + override fun putGenIfVersion(day: String, expected: Long, newGen: FlightSchdRepository.GenMeta, now: Instant): Boolean { + val cur = gens[day]?.version ?: 0L + if (cur != expected) return false + gens[day] = newGen + return true + } + override fun deleteGenBefore(cutoffDay: String): Int = 0 } + private class FakeTxManager : PipelineTransactionManager { + override fun inTransaction(block: () -> T): T = block() + } // ---------- helpers ---------- private val clock = MutableClock(MutableClock.BASE) @@ -177,10 +206,11 @@ class MessageProcessorTest { val props = PipelineProps() val scheduler = FailureScheduler(props, clock) val procFailure = ProcFailure(procState, scheduler) - val snapshot = SnapshotFlow(procState, FakeRefData(), FakeRedis, procFailure) - return MessageProcessor(inbox, procState, events, CodecHolder(codec), HandlerHolder(registry), FakeRedis, snapshot, procFailure, props) + val flightSchd = FakeFlightSchd() + val txManager = FakeTxManager() + val snapshot = SnapshotFlow(procState, flightSchd, events, FakeReqTrack(), procFailure, txManager, inbox) + return MessageProcessor(inbox, procState, events, CodecHolder(codec), HandlerHolder(registry), flightSchd, snapshot, procFailure, props, txManager) } - // ---------- tests ---------- @Test fun `no handler is FAILED UNSUPPORTED with backoff - never terminal`() { diff --git a/src/test/kotlin/com/gzzn/omms/msgexchange/tools/FlightStoreDiffToolTest.kt b/src/test/kotlin/com/gzzn/omms/msgexchange/tools/FlightStoreDiffToolTest.kt new file mode 100644 index 0000000..44b90a5 --- /dev/null +++ b/src/test/kotlin/com/gzzn/omms/msgexchange/tools/FlightStoreDiffToolTest.kt @@ -0,0 +1,87 @@ +package com.gzzn.omms.msgexchange.tools + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class FlightStoreDiffToolTest { + + private val tool = FlightStoreDiffTool() + + @Test + fun `AST comparison ignores key ordering differences`() { + val pg = mapOf( + "F1" to """{"flid":"F1","airline":"CA","flightNo":"123","status":"SCHD"}""", + ) + val legacy = mapOf( + "F1" to """{"status":"SCHD","flightNo":"123","airline":"CA","flid":"F1"}""", + ) + + val report = tool.diff(pg, legacy) + assertTrue(report.isGreen) + assertEquals(1, report.matchedCount) + assertEquals(0, report.unexpectedDeviations.size) + } + + @Test + fun `AST comparison normalizes numeric precision and null equivalents`() { + val pg = mapOf( + "F1" to """{"flid":"F1","weight":100.0,"delay":0,"extra":null}""", + ) + val legacy = mapOf( + "F1" to """{"flid":"F1","weight":100,"delay":0.0}""", + ) + + val report = tool.diff(pg, legacy) + assertTrue(report.isGreen) + assertEquals(1, report.matchedCount) + assertEquals(0, report.unexpectedDeviations.size) + } + + @Test + fun `Recognizes known deviation 1 - Cross-day migrated flights protected in PG`() { + // PG 中包含跨日迁移未被误删的航班，Legacy Redis 中已被误删 + val pg = mapOf( + "F_STABLE" to """{"flid":"F_STABLE"}""", + "F_MIGRATED" to """{"flid":"F_MIGRATED","day":"2026-09-08"}""", + ) + val legacy = mapOf( + "F_STABLE" to """{"flid":"F_STABLE"}""", + // F_MIGRATED 在 legacy 中被按旧代差删误删 + ) + + val report = tool.diff(pg, legacy, crossDayMigratedFlids = setOf("F_MIGRATED")) + assertTrue(report.isGreen) // 属于已知合法偏差，红绿灯依然为 GREEN + assertEquals(1, report.matchedCount) + assertEquals(1, report.knownDeviations.size) + assertEquals(FlightStoreDiffTool.DeviationKind.CROSS_DAY_PROTECTED, report.knownDeviations[0].kind) + assertEquals(0, report.unexpectedDeviations.size) + } + + @Test + fun `Detects real field mismatch and missing flight as unexpected deviation`() { + val pg = mapOf( + "F1" to """{"flid":"F1","status":"BOARDING"}""", + "F_EXTRA" to """{"flid":"F_EXTRA"}""", + ) + val legacy = mapOf( + "F1" to """{"flid":"F1","status":"DEPARTED"}""", + "F_MISSING" to """{"flid":"F_MISSING"}""", + ) + + val report = tool.diff(pg, legacy) + assertFalse(report.isGreen) + assertEquals(0, report.matchedCount) + assertEquals(3, report.unexpectedDeviations.size) + + val kinds = report.unexpectedDeviations.map { it.kind } + assertTrue(kinds.contains(FlightStoreDiffTool.DeviationKind.FIELD_MISMATCH)) + assertTrue(kinds.contains(FlightStoreDiffTool.DeviationKind.MISSING_IN_PG)) + assertTrue(kinds.contains(FlightStoreDiffTool.DeviationKind.UNEXPECTED_EXTRA_IN_PG)) + + val summary = report.formatSummary() + assertTrue(summary.contains("RED (未通过)")) + assertTrue(summary.contains("F1")) + } +} diff --git a/src/test/resources/application-test.yml b/src/test/resources/application-test.yml index c60ee55..865d627 100644 --- a/src/test/resources/application-test.yml +++ b/src/test/resources/application-test.yml @@ -23,9 +23,6 @@ flyway: default: enabled: false -redis: - uri: redis://127.0.0.1:6379 - kafka: bootstrap: servers: 127.0.0.1:9092