cdia/msgexchange-v2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(processing): 实现自有 PostgreSQL 运营航班权威存储与单事务闭环 (ACM2-28)

Browse Source 
- FS1: 增加 Flyway 迁移 V1.1.0__flight_schd.sql，创建 FLIGHT_SCHD 与 SCHD_GEN
- FS2: 实现 FlightSchdRepository 接口及 JdbcFlightSchdRepository 与 StubFlightSchd，增强 JdbcOps 事务管理
- FS3: 扩展 MessageProcessor 事务 2 与按 FLID 点查视图，合并变更、事件与终态入单事务提交
- FS4: SnapshotFlow SQL 化（批处理 upsert、域内差删、SQL CAS 推进与熔断保护），JobExecutor 接入 PG 清场删除
- FS5: 彻底退役 Redis 权威与写路径，移除 FlightRedisClient、Lua 脚本、健康指示器与配置残留
- FS6: 补齐 U09/U29 不变量门禁（崩溃幂等、CAS 防并发、ADFT 存活保障、非 UTC JVM/会话时区无漂移）与 FlywayMigrationTest
- FS7: 交付影子对拍比较内核 FlightStoreDiffTool 与单元测试
- FS8: 全面回改 decision-flight-state、architecture、design、user-stories 权威文档与规范
This commit is contained in:
windyboy
2026-09-07 16:12:00 +08:00
parent dd4bbc86ca
commit 7b7b61f100
37 changed files with 2037 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/kotlin/com/gzzn/omms/msgexchange/processing/FlightSchdInvariantTest.kt
+353
View File
@@ -0,0 +1,353 @@
package com.gzzn.omms.msgexchange.processing
import com.gzzn.omms.msgexchange.MutableClock
import com.gzzn.omms.msgexchange.codec.DecodeResult
import com.gzzn.omms.msgexchange.codec.XmlCodec
import com.gzzn.omms.msgexchange.config.PipelineProps
import com.gzzn.omms.msgexchange.domain.DecodedMessage
import com.gzzn.omms.msgexchange.domain.Decision
import com.gzzn.omms.msgexchange.domain.ErrorClass
import com.gzzn.omms.msgexchange.domain.FlightChange
import com.gzzn.omms.msgexchange.domain.MetaFields
import com.gzzn.omms.msgexchange.domain.MsgEvent
import com.gzzn.omms.msgexchange.domain.MsgKind
import com.gzzn.omms.msgexchange.domain.NotifyPayload
import com.gzzn.omms.msgexchange.domain.ProcState
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.domain.SchdPush
import com.gzzn.omms.msgexchange.domain.Targets
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import com.gzzn.omms.msgexchange.infra.persistence.ReqTrackRepository
import com.gzzn.omms.msgexchange.infra.retry.FailureScheduler
import com.gzzn.omms.msgexchange.infra.retry.ProcFailure
import com.gzzn.omms.msgexchange.infra.stub.StubFlightSchd
import com.gzzn.omms.msgexchange.infra.stub.StubMsgEvents
import com.gzzn.omms.msgexchange.infra.stub.StubPipelineTransactionManager
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
import com.gzzn.omms.msgexchange.infra.stub.StubReqTrack
import org.junit.jupiter.api.AfterEach
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertNotNull
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import java.time.Instant
import java.time.ZoneId
import java.util.TimeZone
/**
* U09 / U29 不变量门禁测试（ACM2-28 FS6 核心验收门槛）：
* 1. I2 不变量：航班状态变更、MSG_EVENT 插入、PROC_STATE 终态在单事务原子提交；失败整体回滚重放；
* 2. 门槛 1（崩溃幂等无自增）：重放相同快照报文后 SCHD_GEN.VERSION 绝不发生二次自增；
* 3. 门槛 2（CAS 防并发）：伪造版本过期模拟并发快照，准确拦截 CAS 冲突并转入 FAILED(INFRA) 与退避；
* 4. 门槛 3（ADFT 存活保障）：增量 ADFT 航班在后续 DNLD 跨代替换后天然存活、不被差删；跨代迁移保护；
* 5. UTC 方言测试：非 UTC JVM 时区下读写与幂等判据无漂移。
*/
class FlightSchdInvariantTest {
private lateinit var flightSchd: StubFlightSchd
private lateinit var procState: StubProcState
private lateinit var msgEvents: StubMsgEvents
private lateinit var reqTrack: StubReqTrack
private lateinit var txManager: StubPipelineTransactionManager
private lateinit var inbox: FakeInbox
private lateinit var clock: MutableClock
private lateinit var scheduler: FailureScheduler
private lateinit var procFailure: ProcFailure
private lateinit var props: PipelineProps
class FakeInbox : CminmsgInboxRepository {
val raws = mutableMapOf<Long, String>()
val backfilled = mutableListOf<Long>()
override fun insertRaw(rawXml: String): Long = 1L
override fun rawOf(cminmsgsId: Long): String? = raws[cminmsgsId]
override fun pollUnprocessed(afterId: Long, limit: Int): List<Long> = emptyList()
override fun backfillOnSuccess(cminmsgsId: Long, sndr: String, type: String, styp: String, seqn: Long) {
backfilled.add(cminmsgsId)
}
}
class SimpleCodec(private val decodedMsg: DecodedMessage) : XmlCodec {
override fun decode(rawXml: String): DecodeResult = DecodeResult.Ok(decodedMsg)
override fun encodeRqrd(kind: String, rangeJson: String): String = ""
}
@BeforeEach
fun setUp() {
flightSchd = StubFlightSchd()
procState = StubProcState()
msgEvents = StubMsgEvents()
reqTrack = StubReqTrack()
txManager = StubPipelineTransactionManager()
inbox = FakeInbox()
clock = MutableClock(MutableClock.BASE)
props = PipelineProps()
scheduler = FailureScheduler(props, clock)
procFailure = ProcFailure(procState, scheduler)
SnapshotFlow.StageResult.parser = null
}
@AfterEach
fun tearDown() {
SnapshotFlow.StageResult.parser = null
}
// =========================================================================
// I2 不变量测试：普通报文事务 2 扩展（变更 + 事件 + SUCCEEDED 原子提交）
// =========================================================================
@Test
fun `I2 - MessageProcessor atomically commits flightChanges, msgEvents, and SUCCEEDED`() {
val headId = 101L
inbox.raws[headId] = "<MSG><FLID>F101</FLID></MSG>"
procState.insert(headId, ProcStatus.PENDING)
val meta = MetaFields("AODB", "FLOP", "DELY", 1L, 20260907120000L)
val decoded = DecodedMessage(meta, MsgKind.Flop("DELY"), "<MSG><FLID>F101</FLID></MSG>")
val codecHolder = CodecHolder(SimpleCodec(decoded))
val handler = object : Handler {
override val kind: MsgKind = MsgKind.Flop("DELY")
override fun decide(flightView: Map<String, String>, msg: DecodedMessage): Decision {
return Decision(
flightChanges = listOf(FlightChange("F101", """{"FLID":"F101","status":"DELAYED"}""")),
msgNotifies = listOf(NotifyPayload("""{"flid":"F101","event":"DELAY"}""")),
schdPush = listOf(SchdPush("F101", """{"FLID":"F101","status":"DELAYED"}""")),
)
}
}
val handlerHolder = HandlerHolder(HandlerRegistry(listOf(handler)))
val snapshotFlow = SnapshotFlow(procState, flightSchd, msgEvents, reqTrack, procFailure, txManager, inbox)
val processor = MessageProcessor(
inbox, procState, msgEvents, codecHolder, handlerHolder,
flightSchd, snapshotFlow, procFailure, props, txManager,
)
val head = procState.headUnfinished()!!
processor.processOne(head)
// 断言：FLIGHT_SCHD 有更新
val fltr = flightSchd.findByFlid("F101")
assertNotNull(fltr)
assertTrue(fltr!!.contains("DELAYED"))
// 断言：MSG_EVENT 写入了 KAFKA_MSG 和 KAFKA_SCHD 两条事件
val evtMsg = msgEvents.headUnsent(Targets.KAFKA_MSG)
assertNotNull(evtMsg)
val evtSchd = msgEvents.headUnsent(Targets.KAFKA_SCHD)
assertNotNull(evtSchd)
// 断言：伴生状态更新为 SUCCEEDED
val currentHead = procState.headUnfinished()
assertNull(currentHead) // 无未完成队头，说明 101 已终态
// 断言：共享信箱 backfill 成功调用
assertTrue(inbox.backfilled.contains(headId))
}
// =========================================================================
// FS6 门槛 1：崩溃幂等无自增（重放相同快照报文，SCHD_GEN.VERSION 绝不发生二次自增）
// =========================================================================
@Test
fun `Threshold 1 - Crash idempotency without version re-increment`() {
val headId = 201L
val day = "2026-09-07"
procState.insert(headId, ProcStatus.PENDING)
val meta = MetaFields("AODB", "SCHD", "DNLD", 1L, 20260907030000L)
val decoded = DecodedMessage(meta, MsgKind.Schd(MsgKind.SchdSubtype.DNLD), "<SCHD/>")
// 配置 staging 解析模拟产出 2 条航班
val flights = listOf(
"FL_01" to """{"FLID":"FL_01","air":"CA1234"}""",
"FL_02" to """{"FLID":"FL_02","air":"MU5678"}""",
)
SnapshotFlow.StageResult.parser = { SnapshotFlow.StageResult.Ok(day, flights) }
// 模拟快照单事务内崩溃注入（例如在提交前崩溃回滚）
var crashInjected = true
val rollbackTxManager = object : PipelineTransactionManager {
override fun <T> inTransaction(block: () -> T): T {
if (crashInjected) {
try {
block()
} finally {
// 事务回滚：清除未提交修改并恢复状态
flightSchd.clear()
procState.update(headId, ProcStatus.PENDING)
}
throw RuntimeException("crash-before-pg-commit")
}
return block()
}
}
val snapshotFlowCrashing = SnapshotFlow(procState, flightSchd, msgEvents, reqTrack, procFailure, rollbackTxManager, inbox)
// 首次运行：事务内崩溃注入
val head1 = procState.headUnfinished()!!
try {
snapshotFlowCrashing.publishSnapshot(head1, decoded)
} catch (_: RuntimeException) {
// 崩溃发生
}
// 崩溃后断言：由于 PG 单事务回滚，SCHD_GEN 不存在半成品中间态，版本未推进
assertNull(flightSchd.getGen(day))
// 重放相同报文（正常完成）
crashInjected = false
val snapshotFlowNormal = SnapshotFlow(procState, flightSchd, msgEvents, reqTrack, procFailure, txManager, inbox)
val headReplay = procState.headUnfinished()!!
snapshotFlowNormal.publishSnapshot(headReplay, decoded)
// 门槛 1 断言：版本绝不发生二次自增，版本号精确为 1L（根除旧 Redis 两阶段二次自增缺陷）
val genAfterReplay = flightSchd.getGen(day)
assertNotNull(genAfterReplay)
assertEquals(1L, genAfterReplay!!.version)
assertEquals(setOf("FL_01", "FL_02"), genAfterReplay.flids)
}
// =========================================================================
// FS6 门槛 2：CAS 防并发（伪造版本过期模拟并发快照，拦截 CAS 冲突并转入 FAILED(INFRA)）
// =========================================================================
@Test
fun `Threshold 2 - CAS prevents concurrency and enters FAILED INFRA`() {
val headId = 202L
val day = "2026-09-07"
procState.insert(headId, ProcStatus.PENDING)
val meta = MetaFields("AODB", "SCHD", "DNLD", 2L, 20260907033000L)
val decoded = DecodedMessage(meta, MsgKind.Schd(MsgKind.SchdSubtype.DNLD), "<SCHD/>")
val flights = listOf("FL_01" to """{"FLID":"FL_01"}""")
SnapshotFlow.StageResult.parser = { SnapshotFlow.StageResult.Ok(day, flights) }
// 先预置日代版本为 5L（模拟另一并发实例已经推进了版本）
flightSchd.putGenIfVersion(day, 0L, FlightSchdRepository.GenMeta(day, 5L, setOf("FL_OLD")))
// 构造一个在读取版本后版本被篡改的场景（模拟读到 5L 后，外部并发变成了 6L）
val mockFlightSchd = object : FlightSchdRepository by flightSchd {
override fun getGen(day: String): FlightSchdRepository.GenMeta? {
// 模拟读取时返回版本 5L
return FlightSchdRepository.GenMeta(day, 5L, setOf("FL_OLD"))
}
override fun putGenIfVersion(day: String, expected: Long, newGen: FlightSchdRepository.GenMeta, now: Instant): Boolean {
// 模拟 CAS 校验失败（数据库已被并发推进，expected 5 已过期）
return false
}
}
val snapshotFlow = SnapshotFlow(procState, mockFlightSchd, msgEvents, reqTrack, procFailure, txManager, inbox)
val head = procState.headUnfinished()!!
snapshotFlow.publishSnapshot(head, decoded)
// 门槛 2 断言：CAS 冲突被拦截，状态转换为 FAILED(INFRA)，带有退避与告警错误信息
val failedHead = procState.headUnfinished()!!
assertEquals(ProcStatus.FAILED, failedHead.state)
assertEquals(ErrorClass.INFRA, failedHead.errorClass)
assertTrue(failedHead.lastError?.contains("gen-cas-conflict") == true)
assertNotNull(failedHead.nextAttemptAt)
}
// =========================================================================
// FS6 门槛 3：ADFT 存活保障（增量 ADFT 航班在后续 DNLD 跨代替换后天然存活、不被差删）
// =========================================================================
@Test
fun `Threshold 3 - ADFT flights survive subsequent DNLD replacement`() {
val day = "2026-09-07"
// 1. 增量更新写入一条临时加飞航班 ADFT（FDAY 为 NULL）
val adftChange = FlightChange(flid = "ADFT_888", payloadJson = """{"FLID":"ADFT_888","type":"ADFT"}""")
flightSchd.upsertIncremental(listOf(adftChange))
// 2. 写入旧代的一条定期计划航班 REG_OLD（FDAY 为 2026-09-07）
flightSchd.upsertSnapshotBatch(day, listOf("REG_OLD" to """{"FLID":"REG_OLD","type":"REG"}"""))
flightSchd.putGenIfVersion(day, 0L, FlightSchdRepository.GenMeta(day, 1L, setOf("REG_OLD")))
// 3. 执行下一轮快照 DNLD，新代仅包含 REG_NEW（REG_OLD 不在新代中，属于待删差集；ADFT 也不在新代中）
val headId = 203L
procState.insert(headId, ProcStatus.PENDING)
val meta = MetaFields("AODB", "SCHD", "DNLD", 3L, 20260907040000L)
val decoded = DecodedMessage(meta, MsgKind.Schd(MsgKind.SchdSubtype.DNLD), "<SCHD/>")
val newFlights = listOf("REG_NEW" to """{"FLID":"REG_NEW","type":"REG"}""")
SnapshotFlow.StageResult.parser = { SnapshotFlow.StageResult.Ok(day, newFlights) }
val snapshotFlow = SnapshotFlow(procState, flightSchd, msgEvents, reqTrack, procFailure, txManager, inbox)
val head = procState.headUnfinished()!!
snapshotFlow.publishSnapshot(head, decoded)
// 门槛 3 核心断言：
// ① 旧代航班 REG_OLD 被按代差删清除
assertNull(flightSchd.findByFlid("REG_OLD"))
// ② 新代航班 REG_NEW 成功写入
assertNotNull(flightSchd.findByFlid("REG_NEW"))
// ③ 增量 ADFT_888 航班由于 FDAY=NULL 天然存活、绝不被误删！
val adftRecord = flightSchd.findByFlid("ADFT_888")
assertNotNull(adftRecord)
assertTrue(adftRecord!!.contains("ADFT_888"))
}
@Test
fun `Threshold 3 - Migrated flight across days is protected from diff deletion of old day`() {
// 场景：航班 FL_MIG 原属 2026-09-07，后来迁移到了 2026-09-08（FDAY 更新为 09-08）
val dayOld = "2026-09-07"
val dayNew = "2026-09-08"
// 初始属旧代
flightSchd.upsertSnapshotBatch(dayOld, listOf("FL_MIG" to """{"FLID":"FL_MIG","day":"07"}"""))
flightSchd.putGenIfVersion(dayOld, 0L, FlightSchdRepository.GenMeta(dayOld, 1L, setOf("FL_MIG")))
// 随后 09-08 快照写入将 FDAY 更新为 2026-09-08
flightSchd.upsertSnapshotBatch(dayNew, listOf("FL_MIG" to """{"FLID":"FL_MIG","day":"08"}"""))
// 此时 09-07 再次执行差删（差集中包含 FL_MIG）
val deleted = flightSchd.deleteDiffByDay(dayOld, listOf("FL_MIG"))
// 断言：FL_MIG 虽在差集，但由于 FDAY 已迁移至 09-08，受到域化差删保护，删除数为 0，记录依然存活！
assertEquals(0, deleted)
assertNotNull(flightSchd.findByFlid("FL_MIG"))
}
// =========================================================================
// UTC 时区规范测试：非 UTC JVM 默认时区与会话下写入读取无漂移
// =========================================================================
@Test
fun `UTC Dialect - Timestamps and instant evaluations are immune to JVM timezone drift`() {
val originalTz = TimeZone.getDefault()
try {
// 切换 JVM 默认时区为非 UTC（东京 +09:00 与 纽约 -05:00）
TimeZone.setDefault(TimeZone.getTimeZone(ZoneId.of("Asia/Tokyo")))
val now = Instant.parse("2026-09-07T08:00:00.123456Z")
flightSchd.upsertSnapshotBatch("2026-09-07", listOf("TZ_01" to """{"test":true}"""), now)
flightSchd.putGenIfVersion("2026-09-07", 0L, FlightSchdRepository.GenMeta("2026-09-07", 1L, setOf("TZ_01")), now)
val gen = flightSchd.getGen("2026-09-07")
assertNotNull(gen)
assertEquals(now, gen!!.updatedAt)
// 再次切换到西五区
TimeZone.setDefault(TimeZone.getTimeZone(ZoneId.of("America/New_York")))
val genNy = flightSchd.getGen("2026-09-07")
assertNotNull(genNy)
assertEquals(now, genNy!!.updatedAt) // 绝无时区漂移
} finally {
TimeZone.setDefault(originalTz)
}
}
}
src/test/kotlin/com/gzzn/omms/msgexchange/processing/MessageProcessorTest.kt
+44 -14
View File
@@ -21,11 +21,9 @@ import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import com.gzzn.omms.msgexchange.infra.persistence.PumpJobRepository
import com.gzzn.omms.msgexchange.infra.persistence.RefDataRepository
import com.gzzn.omms.msgexchange.infra.persistence.ReqTrackRepository
import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository
import com.gzzn.omms.msgexchange.infra.redis.FlightRedisClient
import com.gzzn.omms.msgexchange.infra.redis.RedisScript
import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository
import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
import com.gzzn.omms.msgexchange.infra.retry.FailureScheduler
import com.gzzn.omms.msgexchange.infra.retry.ProcFailure
import org.junit.jupiter.api.Assertions.assertEquals
@@ -139,17 +137,48 @@ class MessageProcessorTest {
override fun encodeRqrd(kind: String, rangeJson: String): String = ""
}
private object FakeRedis : FlightRedisClient {
override fun eval(script: RedisScript, setPairs: List<Pair<String, String>>, delFields: List<String>) = Unit
override fun hgetAllFlightInfo(): Map<String, String> = emptyMap()
override fun ping(): Boolean = true
private class FakeReqTrack : ReqTrackRepository {
override fun findOpenByKind(kind: String): ReqTrackRepository.Req? = null
override fun forceExpireOpenOf(kind: String) = Unit
override fun insert(kind: String, paramsJson: String): Long = 1L
override fun linkCoutmsgs(reqId: Long, coutmsgsId: Long) = Unit
override fun markSent(reqId: Long, sentAt: Instant) = Unit
override fun expireIfWaiting(reqId: Long) = Unit
override fun markDone(reqId: Long) = Unit
}
private class FakeRefData : RefDataRepository {
override fun getGen(day: String): RefDataRepository.GenMeta? = null
override fun putGenIfVersion(day: String, expected: Long, new: RefDataRepository.GenMeta): Boolean = true
private class FakeFlightSchd : FlightSchdRepository {
val flights = mutableMapOf<String, String>()
val gens = mutableMapOf<String, FlightSchdRepository.GenMeta>()
val incrementalChanges = mutableListOf<com.gzzn.omms.msgexchange.domain.FlightChange>()
override fun upsertSnapshotBatch(day: String, flights: List<Pair<String, String>>, now: Instant) {
this.flights.putAll(flights)
}
override fun upsertIncremental(changes: List<com.gzzn.omms.msgexchange.domain.FlightChange>, now: Instant) {
incrementalChanges.addAll(changes)
changes.forEach { flights[it.flid] = it.payloadJson }
}
override fun deleteDiffByDay(day: String, delFlids: Collection<String>): Int = 0
override fun findByFlid(flid: String): String? = flights[flid]
override fun findByFlids(flids: Collection<String>): Map<String, String> =
flids.mapNotNull { f -> flights[f]?.let { f to it } }.toMap()
override fun findByDay(day: String): List<Pair<String, String>> = flights.map { it.key to it.value }
override fun findAll(): Map<String, String> = flights.toMap()
override fun deleteByFlids(flids: Set<String>): Int = 0
override fun getGen(day: String): FlightSchdRepository.GenMeta? = gens[day]
override fun putGenIfVersion(day: String, expected: Long, newGen: FlightSchdRepository.GenMeta, now: Instant): Boolean {
val cur = gens[day]?.version ?: 0L
if (cur != expected) return false
gens[day] = newGen
return true
}
override fun deleteGenBefore(cutoffDay: String): Int = 0
}
private class FakeTxManager : PipelineTransactionManager {
override fun <T> inTransaction(block: () -> T): T = block()
}
// ---------- helpers ----------
private val clock = MutableClock(MutableClock.BASE)
@@ -177,10 +206,11 @@ class MessageProcessorTest {
val props = PipelineProps()
val scheduler = FailureScheduler(props, clock)
val procFailure = ProcFailure(procState, scheduler)
val snapshot = SnapshotFlow(procState, FakeRefData(), FakeRedis, procFailure)
return MessageProcessor(inbox, procState, events, CodecHolder(codec), HandlerHolder(registry), FakeRedis, snapshot, procFailure, props)
val flightSchd = FakeFlightSchd()
val txManager = FakeTxManager()
val snapshot = SnapshotFlow(procState, flightSchd, events, FakeReqTrack(), procFailure, txManager, inbox)
return MessageProcessor(inbox, procState, events, CodecHolder(codec), HandlerHolder(registry), flightSchd, snapshot, procFailure, props, txManager)
}
// ---------- tests ----------
@Test
fun `no handler is FAILED UNSUPPORTED with backoff - never terminal`() {