refactor(flight-state): 按 flight-state.md 审计定稿全量重构脚手架与 SQL (ACM2-31)

- SQL 基线 V1__flight_state_baseline.sql 整体取代 V1.0.0–V1.4.0:
  PIPELINE_LOCK/PROC_STATE/MSG_EVENT/REQ_TRACK/BACKFILL_TODO/FLIGHT_SCHD
  + 8 张资源明细表 + FLIGHT_ROUTE_POINT + SCHD_SNAP_LOG 留痕层
- 废除 FDAY 日代/SCHD_GEN/名单差删:OPERATION_DAY 不可变(应用层校验 +
  条件更新强化 §7.4),STATE 仅 ACTIVE/DELETED,物理清除只在历史归档后
- 处理器化:applyScheduleRecords(§5.1 七步同一事务,重放判定/整包
  DEAD(PROTOCOL)/归属冲突不落地)+ FLOP/FDEL/ADFT(tombstone 仅
  ACTIVE→DELETED,重复 FDEL 幂等不推进版本)
- 投递:KAFKA_SCHD 同 FLID 按最新 STATE_VERSION 合并,被压掉事件关闭,
  TOMBSTONE 发 null 值消息(键缺失=删除旧值 §7.3)
- 回填待办改为业务事务内预登记,消除提交后写待办的崩溃窗口(§7.2/§10)
- XML 解码改为 jackson-dataformat-xml 数据类直接映射(SIS 信封强类型,
  FLTR 开放标签泛型承载)
- 历史归档/物理清除顺序不可颠倒:归档确认成功集才物理删除,未接通删 0 条
- 移除 PUMP_JOB 队列/ReferenceService/FlightStoreDiffTool 等旧机制与测试,
  新增运营日/引擎/快照/FDEL/归档顺序不变性回归测试
This commit is contained in:
windyboy
2026-09-09 17:53:08 +08:00
parent 6af292d103
commit 879d658159
83 changed files with 3435 additions and 6355 deletions
@@ -1,37 +1,32 @@
package com.gzzn.omms.msgexchange.processing
import com.gzzn.omms.msgexchange.codec.FlopPayload
import com.gzzn.omms.msgexchange.codec.JacksonXmlCodec
import com.gzzn.omms.msgexchange.codec.ScheduleBody
import com.gzzn.omms.msgexchange.config.PipelineProps
import com.gzzn.omms.msgexchange.domain.DecodedMessage
import com.gzzn.omms.msgexchange.domain.flight.FlightStateEngine
import com.gzzn.omms.msgexchange.domain.ErrorClass
import com.gzzn.omms.msgexchange.domain.MsgEvent
import com.gzzn.omms.msgexchange.domain.MsgKind
import com.gzzn.omms.msgexchange.domain.ProcState
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.domain.Targets
import com.gzzn.omms.msgexchange.infra.log.TraceLog
import com.gzzn.omms.msgexchange.infra.persistence.BackfillTodoRepository
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import com.gzzn.omms.msgexchange.infra.persistence.PumpJobRepository
import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository
import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
import com.gzzn.omms.msgexchange.infra.retry.ProcFailure
import com.gzzn.omms.msgexchange.jobs.JobExecutor
import jakarta.inject.Singleton
import java.time.Duration
import java.time.Instant
/**
* ACMA-8 流程 2:处理主泵——严格 FIFO + HOL + 毒丸 DEAD 升级I1
* 阶段 A 运营航班 FLIGHT_SCHD 与待发事件、终态同事务原子提交(I2/I5ACM2-28 定案)。
* 处理主泵(docs/flight-state.md §1 目标 2):单活动主泵严格 FIFO + HOL + 毒丸 DEAD 升级;
* 航班状态、事件、处理终态在处理器事务原子提交(§1 目标 3)。
*/
@Singleton
class Pump(
private val procState: ProcStateRepository,
private val pumpJobs: PumpJobRepository,
private val inbox: CminmsgInboxRepository,
private val processor: MessageProcessor,
private val jobExecutor: JobExecutor,
private val props: PipelineProps,
) {
private val log = org.slf4j.LoggerFactory.getLogger(Pump::class.java)
@@ -39,7 +34,7 @@ class Pump(
@Volatile
private var running = true
/** 优雅停机:loop 在当前 tick 收尾后退出;线程中断由 Runner 负责。 */
/** 优雅停机:loop 在当前 tick 收尾后退出;线程中断由 PipelineLifecycle 负责。 */
fun stop() {
running = false
}
@@ -52,27 +47,23 @@ class Pump(
Thread.currentThread().interrupt()
return
} catch (e: Exception) {
// U08loop 只作最后防线——失败状态迁移已在 processOne/execute 的边界内完成;
// 致命 Error 不在此捕获(任其终止进程,保证“异常必可见”)。
// 最后防线失败状态迁移已在 processOne 边界内完成;致命 Error 不捕获
sleepQuietly(props.pipeline.pollInterval)
}
}
}
internal fun tick() {
val job = pumpJobs.headQueued()
val head = procState.headUnfinished()
// 统一 FIFO:JOB 与消息同队列语义(定时任务产物不绕过队头顺序,决策 1)。
val nextJob = job?.takeIf { jobBefore(job, head) }
when {
nextJob != null -> execute(nextJob)
head == null -> sleepQuietly(props.pipeline.pollInterval)
head.state == ProcStatus.FAILED && (head.nextAttemptAt ?: Instant.EPOCH) > Instant.now() ->
if (poisoned(head)) {
log.error("poison -> DEAD id={} attempts={} lastError={}", head.cminmsgsId, head.attempts, head.lastError)
procState.update(head.cminmsgsId, ProcStatus.DEAD,
errorClass = ErrorClass.EXHAUSTED, lastError = head.lastError ?: "head-deadline-exceeded")
log.error("poison -> DEAD msgId={} attempts={} lastError={}", head.msgId, head.attempts, head.lastError)
procState.update(
head.msgId, ProcStatus.DEAD,
errorClass = ErrorClass.EXHAUSTED, lastError = head.lastError ?: "head-deadline-exceeded",
)
} else {
sleepQuietly(Duration.between(Instant.now(), head.nextAttemptAt))
}
@@ -85,69 +76,41 @@ class Pump(
head.attempts >= props.pipeline.maxAttempts ||
Duration.between(head.updatedAt, Instant.now()) > props.pipeline.headDeadline
/** JOB 与队头消息的先后由入队时间近似;实装以统一序号列保证(阶段 1 后续)。 */
private fun jobBefore(job: PumpJobRepository.Job, head: ProcState?) =
head == null || head.state == ProcStatus.FAILED
private fun execute(job: PumpJobRepository.Job) {
pumpJobs.markRunning(job.jobId)
try {
jobExecutor.execute(job)
pumpJobs.markDone(job.jobId)
} catch (e: Exception) {
pumpJobs.markFailed(job.jobId, e.message ?: "unknown")
}
}
private fun sleepQuietly(d: Duration) {
if (!d.isNegative && !d.isZero) Thread.sleep(d.toMillis().coerceAtLeast(1))
}
}
/**
* ACMA-8 流程 2 processOne:解码→绑定→纯函数决策→提交
* U08/U10/U11:失败状态迁移全部在“持有具体 head”的边界内完成——
* 任何意外异常 → FAILED(INFRA)+退避(ProcFailure);attempts 达上限 → DEAD(EXHAUSTED)
* MALFORMED 直接 DEADInterruptedException 恢复中断位并上抛(不被普通恢复逻辑吞掉)。
* processOne:解码 → 绑定 → 处理器(事务内决策+落库)→ 终态迁移 → 回填
* 边界化失败迁移(ProcFailure):任何意外异常归于本条 headFAILED(INFRA)+退避,不穿出杀泵;
* MALFORMED / PROTOCOL 直接 DEAD 不重试(§9)。
*/
@Singleton
class MessageProcessor(
private val inbox: CminmsgInboxRepository,
private val procState: ProcStateRepository,
private val msgEvents: MsgEventRepository,
private val codecHolder: CodecHolder,
private val handlers: HandlerHolder,
private val flightSchd: FlightSchdRepository,
private val snapshotFlow: SnapshotFlow,
private val codec: JacksonXmlCodec,
private val scheduleProcessor: ScheduleProcessor,
private val flopProcessor: FlopProcessor,
private val fdelProcessor: FdelProcessor,
private val adftProcessor: AdftProcessor,
private val procFailure: ProcFailure,
private val props: PipelineProps,
private val txManager: PipelineTransactionManager,
private val backfillTodo: BackfillTodoRepository? = null,
) {
private val log = org.slf4j.LoggerFactory.getLogger(MessageProcessor::class.java)
private val flidRegex = Regex("<FLID>(.*?)</FLID>", RegexOption.IGNORE_CASE)
private fun extractCandidateFlids(decoded: DecodedMessage): Set<String> {
val fromXml = flidRegex.findAll(decoded.rawXml).map { it.groupValues[1].trim() }.filter { it.isNotEmpty() }.toSet()
if (fromXml.isNotEmpty()) return fromXml
val b = decoded.body
if (b is Map<*, *>) {
val flid = b["FLID"] ?: b["flid"]
if (flid != null) return setOf(flid.toString())
}
return emptySet()
}
fun processOne(head: ProcState) {
com.gzzn.omms.msgexchange.infra.log.TraceLog.withTrace(head.cminmsgsId) {
TraceLog.withTrace(head.msgId) {
try {
processInternal(head)
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
throw e
} catch (e: Exception) {
// U08边界化——异常归于本条 head,写 FAILED/DEAD,而不是穿出杀 pump
log.warn("processOne unexpected failure id={} ec=INFRA msg={}", head.cminmsgsId, e.message ?: e.javaClass.simpleName)
// 边界化异常归于本条 head,写 FAILED(INFRA)/DEAD,而不是穿出杀 pump
log.warn("processOne unexpected failure msgId={} ec=INFRA msg={}", head.msgId, e.message ?: e.javaClass.simpleName)
procFailure.fail(head, ErrorClass.INFRA, e.message ?: e.javaClass.simpleName)
}
}
@@ -156,126 +119,134 @@ class MessageProcessor(
private fun processInternal(head: ProcState) {
// 守卫:手工/遗留 FAILED 行若 attempts 已达上限,直接终态(防止退避到期后无限重试)
if (head.state == ProcStatus.FAILED && procFailure.scheduler.exhausted(head.attempts)) {
log.error("head exhausted at entry -> DEAD id={} attempts={}", head.cminmsgsId, head.attempts)
procState.update(head.cminmsgsId, ProcStatus.DEAD, errorClass = ErrorClass.EXHAUSTED,
lastError = head.lastError ?: "max-attempts")
log.error("head exhausted at entry -> DEAD msgId={} attempts={}", head.msgId, head.attempts)
procState.update(head.msgId, ProcStatus.DEAD, errorClass = ErrorClass.EXHAUSTED, lastError = head.lastError ?: "max-attempts")
return
}
val raw = inbox.rawOf(head.cminmsgsId) ?: run {
log.error("raw missing -> DEAD(MALFORMED) id={}", head.cminmsgsId)
procState.update(head.cminmsgsId, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, lastError = "raw-missing")
val raw = inbox.rawOf(head.msgId) ?: run {
log.error("raw missing -> DEAD(MALFORMED) msgId={}", head.msgId)
procState.update(head.msgId, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, lastError = "raw-missing")
return
}
val decoded = when (val r = codecHolder.codec.decode(raw)) {
val decoded = when (val r = codec.decode(raw)) {
is com.gzzn.omms.msgexchange.codec.DecodeResult.Ok -> r.message
is com.gzzn.omms.msgexchange.codec.DecodeResult.Err -> {
// T06U11):MALFORMED(报文非法)→ DEAD 不重试;CODEC_ERROR(可随 codec 修复重放)→ FAILED 退避
// MALFORMED(报文非法)→ DEAD 不重试;CODEC_ERROR(可随 codec 修复重放)→ FAILED 退避
if (r.failure.errorClass == ErrorClass.MALFORMED) {
log.error("decode MALFORMED -> DEAD id={} detail={}", head.cminmsgsId, r.failure.detail)
procState.update(head.cminmsgsId, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED,
lastError = r.failure.detail)
log.error("decode MALFORMED -> DEAD msgId={} detail={}", head.msgId, r.failure.detail)
procState.update(head.msgId, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, lastError = r.failure.detail)
} else {
log.warn("decode {} -> FAILED id={} detail={}", r.failure.errorClass, head.cminmsgsId, r.failure.detail)
log.warn("decode {} -> FAILED msgId={} detail={}", r.failure.errorClass, head.msgId, r.failure.detail)
procFailure.fail(head, r.failure.errorClass, r.failure.detail)
}
return
}
}
// I3identity 仅首次绑定(head.identityKey == null);FAILED 重试不重绑(N25:入参传递 head,不再二次查询)
// I3identity 仅首次绑定(head.identityKey == null);FAILED 重试不重绑
if (head.identityKey == null) {
val identity = Identity.of(decoded, props.identity)
if (!procState.tryBindIdentity(head.cminmsgsId, identity)) {
if (!procState.tryBindIdentity(head.msgId, identity)) {
val owner = procState.ownerOfIdentity(identity) ?: -1L
log.info("duplicate-of:{} -> SKIPPED id={}", owner, head.cminmsgsId)
procState.update(head.cminmsgsId, ProcStatus.SKIPPED, lastError = "duplicate-of:$owner")
// 重复报文自身也是一条信箱记录:同样回填共享信箱,防止被反复轮询(失败落补偿)
log.info("duplicate-of:{} -> SKIPPED msgId={}", owner, head.msgId)
procState.update(head.msgId, ProcStatus.SKIPPED, lastError = "duplicate-of:$owner")
compensateBackfill(head, decoded)
return
}
}
// 快照消息走专属流程(流程 4:staging → SQL 批处理/域内差删/CAS 同事务)
val handler = handlers.registry.dispatcherFor(decoded)
if (handler == null) {
// U10(N21):未注册 ≠ 报文非法——写 FAILED(可重放),绝不写终态
log.warn("no-handler:{} -> FAILED(UNSUPPORTED) id={}", decoded.typeTag, head.cminmsgsId)
procFailure.fail(head, ErrorClass.UNSUPPORTED, "no-handler:${decoded.typeTag}")
return
}
val schdKind = decoded.kind as? com.gzzn.omms.msgexchange.domain.MsgKind.Schd
if (schdKind != null &&
schdKind.subtype == com.gzzn.omms.msgexchange.domain.MsgKind.SchdSubtype.DNLD
) {
snapshotFlow.publishSnapshot(head, decoded)
return
}
val candidateFlids = extractCandidateFlids(decoded)
val flightView = if (candidateFlids.isNotEmpty()) {
flightSchd.findByFlids(candidateFlids)
} else {
emptyMap()
}
val decision = handler.decide(flightView, decoded) // 纯函数
// 事务 2(自有 PG 单事务原子):
// upsert FLIGHT_SCHD(decision.flightChanges) + insert MSG_EVENT + PROC_STATE → SUCCEEDED
val events = buildList {
decision.msgNotifies.forEach { add(MsgEvent(target = Targets.KAFKA_MSG, payloadJson = it.payloadJson)) }
decision.schdPush.forEach { add(MsgEvent(target = Targets.KAFKA_SCHD, partitionKey = it.flid, payloadJson = it.payloadJson)) }
}
val messageId = head.cminmsgsId.toString()
txManager.inTransaction {
if (decision.flightChanges.isNotEmpty()) {
val nextStates = decision.flightChanges.map { change ->
val current = flightSchd.findNextStateByFlid(change.flid)
val commands = FlightStateEngine.commandsFromFields(change.flid, change.fields)
FlightStateEngine.apply(current, commands, messageId, bumpVersion = true)
// 处理器分派:SCHD 快照主链路(§5.1/ FLOP / FDEL / ADFT;缺载荷按 MALFORMED 终态
val result: ApplyResult = when (val kind = decoded.kind) {
is MsgKind.Schd -> {
val body = decoded.body as? ScheduleBody
if (body == null) {
deadMalformed(head, "missing-schd-body")
return
}
when (kind.subtype) {
MsgKind.SchdSubtype.DNLD, MsgKind.SchdSubtype.RESP ->
scheduleProcessor.applyScheduleRecords(head, decoded)
MsgKind.SchdSubtype.ADFT -> {
val record = body.records.singleOrNull()
if (record == null) {
deadMalformed(head, "adft-needs-single-fltr")
return
}
adftProcessor.apply(head, decoded, record)
}
}
flightSchd.persistNextStates(null, nextStates, snapshotReplace = false)
}
if (events.isNotEmpty()) {
msgEvents.insertAll(events)
MsgKind.Fdel -> {
val payload = decoded.body as? FlopPayload
if (payload == null) {
deadMalformed(head, "missing-fdel-flid")
return
}
fdelProcessor.apply(head, decoded, payload)
}
is MsgKind.Flop -> {
val payload = decoded.body as? FlopPayload
if (payload == null) {
deadMalformed(head, "missing-flop-body")
return
}
flopProcessor.apply(head, decoded, payload)
}
is MsgKind.Unsupported -> {
// §9:未支持类型 → FAILED(UNSUPPORTED) 退避重试,达阈值转 DEAD;绝不写终态
log.warn("unsupported type -> FAILED(UNSUPPORTED) msgId={} tag={}", head.msgId, kind.tag)
procFailure.fail(head, ErrorClass.UNSUPPORTED, "no-handler:${kind.tag}")
return
}
procState.update(head.cminmsgsId, ProcStatus.SUCCEEDED)
}
try {
inbox.backfillOnSuccess(head.cminmsgsId, decoded.meta.sndr, decoded.meta.type, decoded.meta.styp, decoded.meta.seqn)
} catch (e: Exception) {
log.error("backfill failed after SUCCEEDED id={} (compensation required)", head.cminmsgsId, e)
recordBackfillTodo(head.cminmsgsId, decoded, e)
when (result) {
is ApplyResult.Succeeded, ApplyResult.ReplaySkipped ->
procState.update(head.msgId, ProcStatus.SUCCEEDED)
is ApplyResult.DeadProtocol -> {
// §9:整包拒绝 DEAD(PROTOCOL),立即释放队头,交人工确认
log.error("DEAD(PROTOCOL) msgId={} reason={} flags={}", head.msgId, result.reason, result.flags)
procState.update(head.msgId, ProcStatus.DEAD, errorClass = ErrorClass.PROTOCOL, lastError = result.reason.take(1000))
compensateBackfill(head, decoded) // 拒绝包同样要回填信箱,防止反复轮询
return
}
}
log.info("SUCCEEDED id={} events={} flightChanges={}", head.cminmsgsId, events.size, decision.flightChanges.size)
backfill(head, decoded)
log.info("SUCCEEDED msgId={} kind={}", head.msgId, decoded.typeTag)
}
/** v2 §5:回填失败只落补偿待办(成功终态不降级,业务不重放);由 BackfillSweepJob 到期重试。 */
/** §7.2:提交后回填共享信箱;失败不得把 SUCCEEDED 改回 FAILED,待办已事务内预登记。 */
private fun backfill(head: ProcState, decoded: DecodedMessage) {
try {
inbox.backfillOnSuccess(head.msgId, decoded.meta.sndr, decoded.meta.type, decoded.meta.styp, decoded.meta.seqn)
backfillTodo?.delete(head.msgId)
} catch (e: Exception) {
log.error("backfill failed after SUCCEEDED msgId={} (todo pre-registered, sweep will retry)", head.msgId, e)
}
}
/** SKIPPED/DEAD(PROTOCOL) 包的回填(无预登记待办):失败落补偿待办。 */
private fun compensateBackfill(head: ProcState, decoded: DecodedMessage) {
try {
inbox.backfillOnSuccess(head.cminmsgsId, decoded.meta.sndr, decoded.meta.type, decoded.meta.styp, decoded.meta.seqn)
inbox.backfillOnSuccess(head.msgId, decoded.meta.sndr, decoded.meta.type, decoded.meta.styp, decoded.meta.seqn)
} catch (e: Exception) {
log.error("backfill failed for SKIPPED duplicate id={} (compensation required)", head.cminmsgsId, e)
recordBackfillTodo(head.cminmsgsId, decoded, e)
log.error("backfill failed msgId={} (compensation required)", head.msgId, e)
backfillTodo?.record(
BackfillTodoRepository.BackfillTask(
msgId = head.msgId,
sndr = decoded.meta.sndr,
type = decoded.meta.type,
styp = decoded.meta.styp,
seqn = decoded.meta.seqn,
),
e.message ?: e.javaClass.simpleName,
) ?: log.warn("no backfill-todo repository bound; compensation NOT persisted msgId={}", head.msgId)
}
}
private fun recordBackfillTodo(cminmsgsId: Long, decoded: DecodedMessage, e: Exception) {
backfillTodo?.record(
BackfillTodoRepository.BackfillTask(
cminmsgsId = cminmsgsId,
sndr = decoded.meta.sndr,
type = decoded.meta.type,
styp = decoded.meta.styp,
seqn = decoded.meta.seqn,
),
e.message ?: e.javaClass.simpleName,
) ?: log.warn("no backfill-todo repository bound; compensation NOT persisted id={}", cminmsgsId)
private fun deadMalformed(head: ProcState, detail: String) {
procState.update(head.msgId, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, lastError = detail)
}
}
/** 延迟装配占位(阶段 1 后续以 Micronaut Bean 替换直连构造)。 */
class CodecHolder(val codec: com.gzzn.omms.msgexchange.codec.XmlCodec)
class HandlerHolder(val registry: HandlerRegistry)