windy/msgexchange-v2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(flight-schd): 按现场 11g 约束将运营航班存储改为宽表 (ACM2-28 修订)

Browse Source 
现场环境仅提供 Oracle 11g（无任何 JSON 能力），废除 FLTR_JSON JSONB 整文档存储：

- V1.1.0 迁移重写：FLIGHT_SCHD 改为一行一航班宽表（SCHD.FLTR 标量字段列 +
  1:N 明细集合序列化文本列），SCHD_GEN.FLIDS_JSON 展开为 SCHD_GEN_FLID(FDAY, FLID)；
  已应用过旧版迁移的环境需重建 schema 重放
- 契约层：FlightChange/Handler 在线视图改为字段集映射（与 legacy flightInfo
  hash 同构），仓储白名单拒绝未知字段；增量写=字段级合并（hmset 同语义），
  快照=整体替换
- 事件载荷：快照投递与投影重建经 FlightFieldsJson 确定性序列化（键序稳定）
- 影子对拍：FlightStoreDiffTool 改为 PG 宽表列值 vs legacy FLTR JSON 逐字段
  归一化比对，新增嵌套集合比对用例
- 测试：61 用例全绿（不变量门槛 1/2/3、UTC 方言、真实 PG 方言集成、清场五场景）
- 文档：decision-flight-state.md 追加 §8 修订记录（不改写定案历史），design.md 表说明同步

11g 方言移植（ON CONFLICT→MERGE、advisory lock 替代、Flyway/驱动矩阵）、
集合列升子表、类型化列提升等未尽事项另立 Plane issue 跟踪。
This commit is contained in:
windyboy
2026-09-07 17:04:57 +08:00
parent 7b7b61f100
commit ff6cec08c8
19 changed files with 556 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/FlightSchdJdbcPgTest.kt
+15 -15
View File
@@ -77,7 +77,7 @@ class FlightSchdJdbcPgTest {
val day = "2026-09-07"
// 构造 250 条记录跨越 batchSize 200 门限
val flights = (1..250).map { i ->
"TEST_FL_$i" to """{"FLID":"TEST_FL_$i","air":"CA$i"}"""
"TEST_FL_$i" to mapOf("FLID" to "TEST_FL_$i", "FLNO" to "CA$i")
}
repo.upsertSnapshotBatch(day, flights)
@@ -89,7 +89,7 @@ class FlightSchdJdbcPgTest {
// 点查测试
val f1 = repo.findByFlid("TEST_FL_1")
assertNotNull(f1)
assertTrue(f1!!.contains("CA1"))
assertEquals("CA1", f1!!["FLNO"])
// 多键点查测试（分批查）
val subset = (1..50).map { "TEST_FL_$it" }
@@ -107,26 +107,26 @@ class FlightSchdJdbcPgTest {
fun `PG dialect - incremental upsert preserves existing FDAY and sets NULL for new`() {
val day = "2026-09-07"
// 1. 快照写入 REG_01，FDAY 为 2026-09-07
repo.upsertSnapshotBatch(day, listOf("TEST_REG_01" to """{"FLID":"TEST_REG_01","v":1}"""))
repo.upsertSnapshotBatch(day, listOf("TEST_REG_01" to mapOf("FLID" to "TEST_REG_01", "REMC" to "1")))
// 2. 增量更新 REG_01（已有行）与 ADFT_01（新行）
val changes = listOf(
FlightChange("TEST_REG_01", """{"FLID":"TEST_REG_01","v":2}"""),
FlightChange("TEST_ADFT_01", """{"FLID":"TEST_ADFT_01","v":1}"""),
FlightChange("TEST_REG_01", mapOf("FLID" to "TEST_REG_01", "REMC" to "2")),
FlightChange("TEST_ADFT_01", mapOf("FLID" to "TEST_ADFT_01", "REMC" to "1")),
)
repo.upsertIncremental(changes)
// 3. 验证 REG_01 内容更新但 FDAY 依然保留为 2026-09-07
val reg01 = ds.queryOne("SELECT fday, fltr_json FROM flight_schd WHERE flid = 'TEST_REG_01'", {}) { rs ->
rs.getDate("fday")?.toString() to rs.getString("fltr_json")
val reg01 = ds.queryOne("SELECT fday, remc FROM flight_schd WHERE flid = 'TEST_REG_01'", {}) { rs ->
rs.getDate("fday")?.toString() to rs.getString("remc")
}
assertNotNull(reg01)
assertEquals("2026-09-07", reg01!!.first)
assertTrue(reg01.second.contains("\"v\": 2") || reg01.second.contains("\"v\":2"))
assertEquals("2", reg01.second)
// 4. 验证 ADFT_01 新插入行 FDAY 恒为 NULL
val adft01 = ds.queryOne("SELECT fday, fltr_json FROM flight_schd WHERE flid = 'TEST_ADFT_01'", {}) { rs ->
rs.getDate("fday")?.toString() to rs.getString("fltr_json")
val adft01 = ds.queryOne("SELECT fday, remc FROM flight_schd WHERE flid = 'TEST_ADFT_01'", {}) { rs ->
rs.getDate("fday")?.toString() to rs.getString("remc")
}
assertNotNull(adft01)
assertNull(adft01!!.first)
@@ -171,7 +171,7 @@ class FlightSchdJdbcPgTest {
val day = "2026-09-07"
try {
ds.withTransaction {
repo.upsertSnapshotBatch(day, listOf("TEST_TX_01" to """{"tx":1}"""))
repo.upsertSnapshotBatch(day, listOf("TEST_TX_01" to mapOf("REMC" to "tx")))
repo.putGenIfVersion(day, 0L, FlightSchdRepository.GenMeta(day, 1L, setOf("TEST_TX_01")))
// 模拟事务内部抛出异常
throw IllegalStateException("forced-abort-for-rollback-test")
@@ -189,7 +189,7 @@ class FlightSchdJdbcPgTest {
fun `PG dialect - ES history sweep deleteByFlids idempotent batch execution`() {
val day = "2026-09-07"
val flids = (1..50).map { "TEST_SWEEP_$it" }
repo.upsertSnapshotBatch(day, flids.map { it to """{"flid":"$it"}""" })
repo.upsertSnapshotBatch(day, flids.map { it to mapOf("FLID" to it) })
assertEquals(50, repo.findByFlids(flids).size)
// 第一次分批删除
@@ -211,7 +211,7 @@ class FlightSchdJdbcPgTest {
val day = "2026-09-07"
val fixedInstant = Instant.parse("2026-09-07T08:30:15.123456Z")
repo.upsertSnapshotBatch(day, listOf("TEST_TZ_01" to """{"tz":1}"""), fixedInstant)
repo.upsertSnapshotBatch(day, listOf("TEST_TZ_01" to mapOf("FLID" to "TEST_TZ_01")), fixedInstant)
repo.putGenIfVersion(day, 0L, FlightSchdRepository.GenMeta(day, 1L, setOf("TEST_TZ_01")), fixedInstant)
// 东京时区下回读
@@ -249,8 +249,8 @@ class FlightSchdJdbcPgTest {
conn.createStatement().use { it.execute("SET TIME ZONE 'Asia/Shanghai'") }
conn.prepareStatement(
"""
INSERT INTO flight_schd (flid, fday, fltr_json, created_at, updated_at)
VALUES ('TEST_SESS_01', '2026-09-07', '{"session":"shanghai"}'::jsonb, ?, ?)
INSERT INTO flight_schd (flid, fday, remc, created_at, updated_at)
VALUES ('TEST_SESS_01', '2026-09-07', 'shanghai', ?, ?)
""".trimIndent(),
).use { ps ->
ps.setTimestamp(1, writeInstant.toSqlTimestamp())
src/test/kotlin/com/gzzn/omms/msgexchange/jobs/HistorySweepJobTest.kt
+10 -10
View File
@@ -40,9 +40,9 @@ class HistorySweepJobTest {
@Test
fun `Scenario 1 - ES All Success removes all confirmed flights and cleans old gens`() {
val flights = listOf(
"F1" to """{"FLID":"F1"}""",
"F2" to """{"FLID":"F2"}""",
"F3" to """{"FLID":"F3"}""",
"F1" to mapOf("FLID" to "F1"),
"F2" to mapOf("FLID" to "F2"),
"F3" to mapOf("FLID" to "F3"),
)
flightSchd.upsertSnapshotBatch("2026-09-01", flights)
flightSchd.putGenIfVersion("2026-09-01", 0L, FlightSchdRepository.GenMeta("2026-09-01", 1L, setOf("F1", "F2", "F3")))
@@ -68,9 +68,9 @@ class HistorySweepJobTest {
@Test
fun `Scenario 2 - ES Partial Success deletes only successful flights and keeps failed ones`() {
val flights = listOf(
"F1" to """{"FLID":"F1"}""",
"F2" to """{"FLID":"F2"}""",
"F3" to """{"FLID":"F3"}""",
"F1" to mapOf("FLID" to "F1"),
"F2" to mapOf("FLID" to "F2"),
"F3" to mapOf("FLID" to "F3"),
)
flightSchd.upsertSnapshotBatch("2026-09-01", flights)
@@ -92,8 +92,8 @@ class HistorySweepJobTest {
@Test
fun `Scenario 3 - ES All Failure deletes nothing and keeps all candidates in PG`() {
val flights = listOf(
"F1" to """{"FLID":"F1"}""",
"F2" to """{"FLID":"F2"}""",
"F1" to mapOf("FLID" to "F1"),
"F2" to mapOf("FLID" to "F2"),
)
flightSchd.upsertSnapshotBatch("2026-09-01", flights)
@@ -112,7 +112,7 @@ class HistorySweepJobTest {
@Test
fun `Scenario 4 - Crash after ES success before PG delete recovers idempotently on next run`() {
val flights = listOf(
"F1" to """{"FLID":"F1"}""",
"F1" to mapOf("FLID" to "F1"),
)
flightSchd.upsertSnapshotBatch("2026-09-01", flights)
@@ -150,7 +150,7 @@ class HistorySweepJobTest {
// 5. 场景五：PG 删除重复执行（无副作用与幂等性）
@Test
fun `Scenario 5 - Replaying deleteByFlids is completely idempotent with no side effects`() {
val flights = listOf("F1" to """{"FLID":"F1"}""")
val flights = listOf("F1" to mapOf("FLID" to "F1"))
flightSchd.upsertSnapshotBatch("2026-09-01", flights)
// 首次删除
src/test/kotlin/com/gzzn/omms/msgexchange/processing/FlightSchdInvariantTest.kt
+17 -17
View File
@@ -115,11 +115,11 @@ class FlightSchdInvariantTest {
val handler = object : Handler {
override val kind: MsgKind = MsgKind.Flop("DELY")
override fun decide(flightView: Map<String, String>, msg: DecodedMessage): Decision {
override fun decide(flightView: Map<String, Map<String, String>>, msg: DecodedMessage): Decision {
return Decision(
flightChanges = listOf(FlightChange("F101", """{"FLID":"F101","status":"DELAYED"}""")),
flightChanges = listOf(FlightChange("F101", mapOf("FLID" to "F101", "STAT" to "DELAYED"))),
msgNotifies = listOf(NotifyPayload("""{"flid":"F101","event":"DELAY"}""")),
schdPush = listOf(SchdPush("F101", """{"FLID":"F101","status":"DELAYED"}""")),
schdPush = listOf(SchdPush("F101", """{"FLID":"F101","STAT":"DELAYED"}""")),
)
}
}
@@ -133,10 +133,10 @@ class FlightSchdInvariantTest {
val head = procState.headUnfinished()!!
processor.processOne(head)
// 断言：FLIGHT_SCHD 有更新
val fltr = flightSchd.findByFlid("F101")
assertNotNull(fltr)
assertTrue(fltr!!.contains("DELAYED"))
// 断言：FLIGHT_SCHD 有更新（字段列 DELAYED 已落库）
val fields = flightSchd.findByFlid("F101")
assertNotNull(fields)
assertEquals("DELAYED", fields!!["STAT"])
// 断言：MSG_EVENT 写入了 KAFKA_MSG 和 KAFKA_SCHD 两条事件
val evtMsg = msgEvents.headUnsent(Targets.KAFKA_MSG)
@@ -167,8 +167,8 @@ class FlightSchdInvariantTest {
// 配置 staging 解析模拟产出 2 条航班
val flights = listOf(
"FL_01" to """{"FLID":"FL_01","air":"CA1234"}""",
"FL_02" to """{"FLID":"FL_02","air":"MU5678"}""",
"FL_01" to mapOf("FLID" to "FL_01", "FLNO" to "CA1234"),
"FL_02" to mapOf("FLID" to "FL_02", "FLNO" to "MU5678"),
)
SnapshotFlow.StageResult.parser = { SnapshotFlow.StageResult.Ok(day, flights) }
@@ -228,7 +228,7 @@ class FlightSchdInvariantTest {
val meta = MetaFields("AODB", "SCHD", "DNLD", 2L, 20260907033000L)
val decoded = DecodedMessage(meta, MsgKind.Schd(MsgKind.SchdSubtype.DNLD), "<SCHD/>")
val flights = listOf("FL_01" to """{"FLID":"FL_01"}""")
val flights = listOf("FL_01" to mapOf("FLID" to "FL_01"))
SnapshotFlow.StageResult.parser = { SnapshotFlow.StageResult.Ok(day, flights) }
// 先预置日代版本为 5L（模拟另一并发实例已经推进了版本）
@@ -268,11 +268,11 @@ class FlightSchdInvariantTest {
val day = "2026-09-07"
// 1. 增量更新写入一条临时加飞航班 ADFT（FDAY 为 NULL）
val adftChange = FlightChange(flid = "ADFT_888", payloadJson = """{"FLID":"ADFT_888","type":"ADFT"}""")
val adftChange = FlightChange(flid = "ADFT_888", fields = mapOf("FLID" to "ADFT_888", "FLTY" to "ADFT"))
flightSchd.upsertIncremental(listOf(adftChange))
// 2. 写入旧代的一条定期计划航班 REG_OLD（FDAY 为 2026-09-07）
flightSchd.upsertSnapshotBatch(day, listOf("REG_OLD" to """{"FLID":"REG_OLD","type":"REG"}"""))
flightSchd.upsertSnapshotBatch(day, listOf("REG_OLD" to mapOf("FLID" to "REG_OLD", "FLTY" to "REG")))
flightSchd.putGenIfVersion(day, 0L, FlightSchdRepository.GenMeta(day, 1L, setOf("REG_OLD")))
// 3. 执行下一轮快照 DNLD，新代仅包含 REG_NEW（REG_OLD 不在新代中，属于待删差集；ADFT 也不在新代中）
@@ -281,7 +281,7 @@ class FlightSchdInvariantTest {
val meta = MetaFields("AODB", "SCHD", "DNLD", 3L, 20260907040000L)
val decoded = DecodedMessage(meta, MsgKind.Schd(MsgKind.SchdSubtype.DNLD), "<SCHD/>")
val newFlights = listOf("REG_NEW" to """{"FLID":"REG_NEW","type":"REG"}""")
val newFlights = listOf("REG_NEW" to mapOf("FLID" to "REG_NEW", "FLTY" to "REG"))
SnapshotFlow.StageResult.parser = { SnapshotFlow.StageResult.Ok(day, newFlights) }
val snapshotFlow = SnapshotFlow(procState, flightSchd, msgEvents, reqTrack, procFailure, txManager, inbox)
@@ -298,7 +298,7 @@ class FlightSchdInvariantTest {
// ③ 增量 ADFT_888 航班由于 FDAY=NULL 天然存活、绝不被误删！
val adftRecord = flightSchd.findByFlid("ADFT_888")
assertNotNull(adftRecord)
assertTrue(adftRecord!!.contains("ADFT_888"))
assertEquals("ADFT_888", adftRecord!!["FLID"])
}
@Test
@@ -308,11 +308,11 @@ class FlightSchdInvariantTest {
val dayNew = "2026-09-08"
// 初始属旧代
flightSchd.upsertSnapshotBatch(dayOld, listOf("FL_MIG" to """{"FLID":"FL_MIG","day":"07"}"""))
flightSchd.upsertSnapshotBatch(dayOld, listOf("FL_MIG" to mapOf("FLID" to "FL_MIG", "REMC" to "07")))
flightSchd.putGenIfVersion(dayOld, 0L, FlightSchdRepository.GenMeta(dayOld, 1L, setOf("FL_MIG")))
// 随后 09-08 快照写入将 FDAY 更新为 2026-09-08
flightSchd.upsertSnapshotBatch(dayNew, listOf("FL_MIG" to """{"FLID":"FL_MIG","day":"08"}"""))
flightSchd.upsertSnapshotBatch(dayNew, listOf("FL_MIG" to mapOf("FLID" to "FL_MIG", "REMC" to "08")))
// 此时 09-07 再次执行差删（差集中包含 FL_MIG）
val deleted = flightSchd.deleteDiffByDay(dayOld, listOf("FL_MIG"))
@@ -334,7 +334,7 @@ class FlightSchdInvariantTest {
TimeZone.setDefault(TimeZone.getTimeZone(ZoneId.of("Asia/Tokyo")))
val now = Instant.parse("2026-09-07T08:00:00.123456Z")
flightSchd.upsertSnapshotBatch("2026-09-07", listOf("TZ_01" to """{"test":true}"""), now)
flightSchd.upsertSnapshotBatch("2026-09-07", listOf("TZ_01" to mapOf("FLID" to "TZ_01")), now)
flightSchd.putGenIfVersion("2026-09-07", 0L, FlightSchdRepository.GenMeta("2026-09-07", 1L, setOf("TZ_01")), now)
val gen = flightSchd.getGen("2026-09-07")
src/test/kotlin/com/gzzn/omms/msgexchange/processing/MessageProcessorTest.kt
+10 -9
View File
@@ -18,6 +18,7 @@ import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.domain.SchdPush
import com.gzzn.omms.msgexchange.domain.Targets
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
import com.gzzn.omms.msgexchange.infra.persistence.FlightFields
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import com.gzzn.omms.msgexchange.infra.persistence.PumpJobRepository
@@ -148,23 +149,23 @@ class MessageProcessorTest {
}
private class FakeFlightSchd : FlightSchdRepository {
val flights = mutableMapOf<String, String>()
val flights = mutableMapOf<String, FlightFields>()
val gens = mutableMapOf<String, FlightSchdRepository.GenMeta>()
val incrementalChanges = mutableListOf<com.gzzn.omms.msgexchange.domain.FlightChange>()
override fun upsertSnapshotBatch(day: String, flights: List<Pair<String, String>>, now: Instant) {
override fun upsertSnapshotBatch(day: String, flights: List<Pair<String, FlightFields>>, now: Instant) {
this.flights.putAll(flights)
}
override fun upsertIncremental(changes: List<com.gzzn.omms.msgexchange.domain.FlightChange>, now: Instant) {
incrementalChanges.addAll(changes)
changes.forEach { flights[it.flid] = it.payloadJson }
changes.forEach { flights[it.flid] = it.fields }
}
override fun deleteDiffByDay(day: String, delFlids: Collection<String>): Int = 0
override fun findByFlid(flid: String): String? = flights[flid]
override fun findByFlids(flids: Collection<String>): Map<String, String> =
override fun findByFlid(flid: String): FlightFields? = flights[flid]
override fun findByFlids(flids: Collection<String>): Map<String, FlightFields> =
flids.mapNotNull { f -> flights[f]?.let { f to it } }.toMap()
override fun findByDay(day: String): List<Pair<String, String>> = flights.map { it.key to it.value }
override fun findAll(): Map<String, String> = flights.toMap()
override fun findByDay(day: String): List<Pair<String, FlightFields>> = flights.map { it.key to it.value }
override fun findAll(): Map<String, FlightFields> = flights.toMap()
override fun deleteByFlids(flids: Set<String>): Int = 0
override fun getGen(day: String): FlightSchdRepository.GenMeta? = gens[day]
override fun putGenIfVersion(day: String, expected: Long, newGen: FlightSchdRepository.GenMeta, now: Instant): Boolean {
@@ -193,9 +194,9 @@ class MessageProcessorTest {
private fun deliveHandler() = object : Handler {
override val kind: MsgKind = MsgKind.Flop("DELY")
override fun decide(flightView: Map<String, String>, msg: DecodedMessage): Decision = Decision(
override fun decide(flightView: Map<String, Map<String, String>>, msg: DecodedMessage): Decision = Decision(
msgNotifies = listOf(NotifyPayload("""{"n":1}""")),
schdPush = listOf(SchdPush(flid = "F1", fltrJson = """{"FLID":"F1","v":2}""")),
schdPush = listOf(SchdPush(flid = "F1", payloadJson = """{"FLID":"F1","v":2}""")),
)
}
src/test/kotlin/com/gzzn/omms/msgexchange/tools/FlightStoreDiffToolTest.kt
+37 -9
View File
@@ -10,9 +10,14 @@ class FlightStoreDiffToolTest {
private val tool = FlightStoreDiffTool()
@Test
fun `AST comparison ignores key ordering differences`() {
fun `Field map comparison ignores legacy JSON key ordering differences`() {
val pg = mapOf(
"F1" to """{"flid":"F1","airline":"CA","flightNo":"123","status":"SCHD"}""",
"F1" to mapOf(
"flid" to "F1",
"airline" to "CA",
"flightNo" to "123",
"status" to "SCHD",
),
)
val legacy = mapOf(
"F1" to """{"status":"SCHD","flightNo":"123","airline":"CA","flid":"F1"}""",
@@ -25,12 +30,16 @@ class FlightStoreDiffToolTest {
}
@Test
fun `AST comparison normalizes numeric precision and null equivalents`() {
fun `Field map comparison normalizes numeric precision and null equivalents`() {
val pg = mapOf(
"F1" to """{"flid":"F1","weight":100.0,"delay":0,"extra":null}""",
"F1" to mapOf(
"flid" to "F1",
"weight" to "100.0",
"delay" to "0",
),
)
val legacy = mapOf(
"F1" to """{"flid":"F1","weight":100,"delay":0.0}""",
"F1" to """{"flid":"F1","weight":100,"delay":0.0,"extra":null}""",
)
val report = tool.diff(pg, legacy)
@@ -43,8 +52,8 @@ class FlightStoreDiffToolTest {
fun `Recognizes known deviation 1 - Cross-day migrated flights protected in PG`() {
// PG 中包含跨日迁移未被误删的航班，Legacy Redis 中已被误删
val pg = mapOf(
"F_STABLE" to """{"flid":"F_STABLE"}""",
"F_MIGRATED" to """{"flid":"F_MIGRATED","day":"2026-09-08"}""",
"F_STABLE" to mapOf("flid" to "F_STABLE"),
"F_MIGRATED" to mapOf("flid" to "F_MIGRATED", "FDAY" to "2026-09-08"),
)
val legacy = mapOf(
"F_STABLE" to """{"flid":"F_STABLE"}""",
@@ -62,8 +71,8 @@ class FlightStoreDiffToolTest {
@Test
fun `Detects real field mismatch and missing flight as unexpected deviation`() {
val pg = mapOf(
"F1" to """{"flid":"F1","status":"BOARDING"}""",
"F_EXTRA" to """{"flid":"F_EXTRA"}""",
"F1" to mapOf("flid" to "F1", "status" to "BOARDING"),
"F_EXTRA" to mapOf("flid" to "F_EXTRA"),
)
val legacy = mapOf(
"F1" to """{"flid":"F1","status":"DEPARTED"}""",
@@ -84,4 +93,23 @@ class FlightStoreDiffToolTest {
assertTrue(summary.contains("RED (未通过)"))
assertTrue(summary.contains("F1"))
}
@Test
fun `Nested collection value is compared as parsed JSON tree`() {
val pg = mapOf(
"F1" to mapOf(
"flid" to "F1",
// 1:N 明细集合列：序列化文本（JSON 数组），按解析树与 legacy 递归比对
"GTDT" to """[{"GATE":"A1","PGOT":"07SEP260800"}]""",
),
)
val legacy = mapOf(
"F1" to """{"flid":"F1","GTDT":[{"PGOT":"07SEP260800","GATE":"A1"}]}""",
)
val report = tool.diff(pg, legacy)
assertTrue(report.isGreen)
assertEquals(1, report.matchedCount)
assertEquals(0, report.unexpectedDeviations.size)
}
}