feat(processing): 领域/投影/终态三步提交与 FlightProjectionPort(ACM2-79)

PIPELINE_LOCK 跨 Redis 写;MSG_EVENT HELD→PENDING;Redis 失败不终态不投递;V3 迁移。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
windyboy
2026-09-21 15:24:34 +08:00
co-authored by Cursor
parent 891088235c
commit 2c3d6af309
28 changed files with 928 additions and 167 deletions
@@ -4,11 +4,10 @@ import com.fasterxml.jackson.databind.ObjectMapper
import com.gzzn.omms.msgexchange.codec.FlopPayload
import com.gzzn.omms.msgexchange.config.OperationDayProps
import com.gzzn.omms.msgexchange.domain.DecodedMessage
import com.gzzn.omms.msgexchange.domain.EventType
import com.gzzn.omms.msgexchange.domain.EventStatus
import com.gzzn.omms.msgexchange.domain.MsgEvent
import com.gzzn.omms.msgexchange.domain.OperationDayCalculator
import com.gzzn.omms.msgexchange.domain.ProcState
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.domain.Targets
import com.gzzn.omms.msgexchange.domain.flight.FlightSnapshot
import com.gzzn.omms.msgexchange.domain.flight.FlightState
@@ -18,9 +17,6 @@ import com.gzzn.omms.msgexchange.domain.flight.ScheduleRecord
import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
import com.gzzn.omms.msgexchange.infra.persistence.PersistOutcome
import com.gzzn.omms.msgexchange.infra.persistence.PipelineLockRepository
import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import jakarta.inject.Singleton
import java.time.Clock
import java.time.Instant
@@ -35,29 +31,22 @@ import java.time.ZoneId
*/
@Singleton
class FlopProcessor(
private val txManager: PipelineTransactionManager,
private val lock: PipelineLockRepository,
private val commit: FlightCommit,
private val flightState: FlightStateRepository,
private val msgEvents: MsgEventRepository,
private val procState: ProcStateRepository,
private val mapper: ObjectMapper,
private val clock: Clock,
) {
fun apply(head: ProcState, msg: DecodedMessage, payload: FlopPayload): ApplyResult = txManager.inTransaction {
lock.lock()
fun apply(head: ProcState, msg: DecodedMessage, payload: FlopPayload): ApplyResult = commit.commit(head) {
// 迟到/未知航班:幂等成功,不创建(创建入口只有 SCHD/ADFT),也没有投影要刷
val current = flightState.loadFullSnapshot(payload.flid)
if (current == null) {
// 迟到/未知航班:幂等成功,不创建(创建入口只有 SCHD/ADFT);终态同事务落库
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
return@inTransaction ApplyResult.Succeeded
}
?: return@commit DomainOutcome(ApplyResult.Succeeded)
val change = MergeChange(flid = payload.flid, scalars = payload.scalars, collections = payload.collections)
val next = FlightStateEngine.mergedState(current, change)
flightState.persistFullState(next, msgId = head.msgId, now = clock.instant())
msgEvents.insertAll(eventsFor(next, mapper, clock.instant()))
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
ApplyResult.Succeeded
msgEvents.insertAll(eventsFor(next, head.msgId, mapper, clock.instant()))
DomainOutcome(ApplyResult.Succeeded, listOf(projectionOf(next, mapper)))
}
}
@@ -65,43 +54,47 @@ class FlopProcessor(
* 处理 FDEL 删除报文:把在用的航班标记为 DELETED,版本号加一,明细数据保留,
* 并且只发布一次删除事件。
*
* 已经删除过、或航班本来就不存在时,算处理成功但不再动版本、不重复发事件
* 已经删除过、或航班本来就不存在时,算处理成功但不再动版本、不重复发事件
* 但只要航班还在库里且是删除态,投影删除就照发一次——幂等命令换来"上一轮没删干净"能自愈(`INV-8`)。
*/
@Singleton
class FdelProcessor(
private val txManager: PipelineTransactionManager,
private val lock: PipelineLockRepository,
private val commit: FlightCommit,
private val flightState: FlightStateRepository,
private val msgEvents: MsgEventRepository,
private val procState: ProcStateRepository,
private val mapper: ObjectMapper,
private val clock: Clock,
) {
fun apply(head: ProcState, msg: DecodedMessage, payload: FlopPayload): ApplyResult = txManager.inTransaction {
lock.lock()
fun apply(head: ProcState, msg: DecodedMessage, payload: FlopPayload): ApplyResult = commit.commit(head) {
val deleted = flightState.markDeleted(payload.flid, msgId = head.msgId, now = clock.instant())
if (deleted) {
val current = flightState.loadFullSnapshot(payload.flid)
// 只有"在用 → 删除"这一步才发删除通知,而且和状态变更写在同一个事务里
val createdAt = clock.instant()
// 航班根本不存在(迟到或多余的删除报文):算成功,也没有投影要删
val main = flightState.findMainRow(payload.flid)
?: return@commit DomainOutcome(ApplyResult.Succeeded)
// 这一次删掉的,或者上一轮就是本消息删掉的(投影写失败后重处理),都要登记删除通知:
// 重处理开始时已把上一轮没放行的事件清掉,这里不补登就会把这条通知丢了。
if (deleted || (main.state == FlightState.DELETED && main.lastMsgId == head.msgId)) {
msgEvents.insertAll(
listOf(
// C-9:删除通知只走 KAFKA:msgschd 不再发 tombstone
// value 形态("deleted":true 的 JSON)沿用现状,待 Q5 定稿
MsgEvent(
msgId = head.msgId,
target = Targets.KAFKA_MSG,
partitionKey = payload.flid,
stateVersion = current?.stateVersion ?: 0L,
stateVersion = main.stateVersion,
payloadJson = mapper.writeValueAsString(
mapOf("flid" to payload.flid, "stateVersion" to (current?.stateVersion ?: 0L), "deleted" to true),
mapOf("flid" to payload.flid, "stateVersion" to main.stateVersion, "deleted" to true),
),
createdAt = createdAt,
state = EventStatus.HELD,
createdAt = clock.instant(),
),
),
)
}
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant()) // 没删到东西说明是迟到或重复报文,照样算成功
ApplyResult.Succeeded
// INV-8:只要航班是删除态就得从投影里抹掉。重复报文也重删一次——代价是一条幂等命令,
// 换来"上一轮删投影没成功"能自愈。
DomainOutcome(ApplyResult.Succeeded, listOf(FlightProjectionWrite.Delete(payload.flid, main.stateVersion)))
}
}
@@ -114,11 +107,9 @@ class FdelProcessor(
*/
@Singleton
class AdftProcessor(
private val txManager: PipelineTransactionManager,
private val lock: PipelineLockRepository,
private val commit: FlightCommit,
private val flightState: FlightStateRepository,
private val msgEvents: MsgEventRepository,
private val procState: ProcStateRepository,
operationDayProps: OperationDayProps,
private val mapper: ObjectMapper,
private val clock: Clock,
@@ -130,24 +121,27 @@ class AdftProcessor(
fun apply(head: ProcState, msg: DecodedMessage, record: ScheduleRecord): ApplyResult =
try {
txManager.inTransaction {
lock.lock()
commit.commit(head) {
val main = flightState.findMainRow(record.flid)
if (main != null && main.state == FlightState.DELETED) {
// 已删除的航班重新激活:状态改回 ACTIVE、版本号加一,并登记状态事件
var revived: FlightSnapshot? = null
if (flightState.revive(record.flid, msgId = head.msgId, now = clock.instant())) {
val current = flightState.loadFullSnapshot(record.flid)
if (current != null) {
val next = FlightStateEngine.mergedState(current, setOnly(record))
flightState.persistFullState(next, msgId = head.msgId, now = clock.instant())
msgEvents.insertAll(eventsFor(next, mapper, clock.instant()))
msgEvents.insertAll(eventsFor(next, head.msgId, mapper, clock.instant()))
revived = next
}
} else {
// 并发下可能已被别的消息处理掉:不改版本、不重复发事件,但留下痕迹便于对账。
log.warn("adft revive no-op (already active or vanished) msgId={} flid={}", head.msgId, record.flid)
}
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
return@inTransaction ApplyResult.Succeeded
return@commit DomainOutcome(
ApplyResult.Succeeded,
listOfNotNull(revived?.let { projectionOf(it, mapper) }),
)
}
val current = flightState.loadFullSnapshot(record.flid)
@@ -173,9 +167,8 @@ class AdftProcessor(
// 若按 INFRA 抛出去,会被当成暂时性故障白白重试到耗尽,并给出误导的错误类别。
throw ProtocolViolation("operation-day guard violated flid=${record.flid}")
}
msgEvents.insertAll(eventsFor(next, mapper, clock.instant()))
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
ApplyResult.Succeeded
msgEvents.insertAll(eventsFor(next, head.msgId, mapper, clock.instant()))
DomainOutcome(ApplyResult.Succeeded, listOf(projectionOf(next, mapper)))
}
} catch (e: ProtocolViolation) {
log.error("adft DEAD(PROTOCOL) msgId={} reason={}", head.msgId, e.message)
@@ -201,28 +194,37 @@ class AdftProcessor(
// 共享小工具(处理器层私有约定)
// =====================================================================
/** 航班状态变化后要发的两类事件:KAFKA_SCHD 发完整状态,KAFKA_MSG 发一条变更通知。 */
internal fun eventsFor(next: FlightSnapshot, mapper: ObjectMapper, createdAt: Instant): List<MsgEvent> {
val payload = linkedMapOf<String, Any>(
"flid" to next.flid,
"stateVersion" to next.stateVersion,
"scalars" to next.scalars,
"collections" to next.collections,
)
return listOf(
/** 航班整态载荷:`KAFKA:schd` 事件与 Redis 投影共用这一份形状。 */
internal fun flightPayload(next: FlightSnapshot): Map<String, Any> = linkedMapOf(
"flid" to next.flid,
"stateVersion" to next.stateVersion,
"scalars" to next.scalars,
"collections" to next.collections,
)
/**
* 航班状态变化后要发的两类事件:KAFKA_SCHD 发完整状态,KAFKA_MSG 发一条变更通知。
*
* 两条都以 [EventStatus.HELD] 登记,投影写成功后由终态事务放行(`INV-3`、`INV-10`)。
*/
internal fun eventsFor(next: FlightSnapshot, msgId: Long, mapper: ObjectMapper, createdAt: Instant): List<MsgEvent> =
listOf(
MsgEvent(
msgId = msgId,
target = Targets.KAFKA_SCHD,
partitionKey = next.flid,
stateVersion = next.stateVersion,
payloadJson = mapper.writeValueAsString(payload),
payloadJson = mapper.writeValueAsString(flightPayload(next)),
state = EventStatus.HELD,
createdAt = createdAt,
),
MsgEvent(
msgId = msgId,
target = Targets.KAFKA_MSG,
partitionKey = next.flid,
stateVersion = next.stateVersion,
payloadJson = mapper.writeValueAsString(mapOf("flid" to next.flid, "stateVersion" to next.stateVersion)),
state = EventStatus.HELD,
createdAt = createdAt,
),
)
}
@@ -0,0 +1,67 @@
package com.gzzn.omms.msgexchange.processing
import com.gzzn.omms.msgexchange.domain.ProcState
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
import com.gzzn.omms.msgexchange.infra.persistence.PipelineLockRepository
import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import jakarta.inject.Singleton
import java.time.Clock
/**
* 领域事务的产物:交给调用方的结果,加上这次变更要刷进查询投影的航班。
* 没有航班变更(迟到报文、空日计划)时投影为空,这一步直接跳过。
*/
data class DomainOutcome<T>(val value: T, val projections: List<FlightProjectionWrite> = emptyList())
/**
* 业务型成功的三步提交(`INV-3`、`INV-10`,见 docs/implementation.md「事务边界」):
*
* 1. 领域事务:航班变更与待发事件一起提交,事件落 `HELD`(投递领不到);
* 2. 写 Redis 投影:失败就抛出去,这条消息保持未完成,下轮重处理;
* 3. 终态事务:放行待发事件,写 `SUCCEEDED` 与回填意图。
*
* 三步整段持 `PIPELINE_LOCK`:行锁提交即释放,盖不住中间的投影写,所以由
* [PipelineLockRepository.holdAcrossTransactions] 在一条连接上从头持到尾。
*
* 投影失败时第一步已经提交,**不回滚**(`US-03` AC3):重处理按当前完整态重写投影,
* 上一轮的 `HELD` 事件先清后补,不会重复投递。
*/
@Singleton
class FlightCommit(
private val txManager: PipelineTransactionManager,
private val lock: PipelineLockRepository,
private val procState: ProcStateRepository,
private val msgEvents: MsgEventRepository,
private val projection: FlightProjectionPort,
private val clock: Clock,
) {
fun <T> commit(head: ProcState, domain: () -> DomainOutcome<T>): T = lock.holdAcrossTransactions {
val outcome = txManager.inTransaction {
lock.lock()
msgEvents.discardHeld(head.msgId) // 清掉上一轮没放行的事件,再登记本轮的
domain()
}
writeProjection(head, outcome.projections)
txManager.inTransaction {
lock.lock()
msgEvents.releaseHeld(head.msgId)
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
}
outcome.value
}
private fun writeProjection(head: ProcState, writes: List<FlightProjectionWrite>) {
if (writes.isEmpty()) return
try {
projection.write(writes)
} catch (e: InterruptedException) {
throw e // 停机信号不归到这条消息头上
} catch (e: Exception) {
throw FlightProjectionFailure(head.msgId, e)
}
}
}
@@ -0,0 +1,47 @@
package com.gzzn.omms.msgexchange.processing
import com.fasterxml.jackson.databind.ObjectMapper
import com.gzzn.omms.msgexchange.domain.flight.FlightSnapshot
/**
* 一次投影写:把某个航班的整态写进查询投影,或者把它从投影里删掉。
* 带上版本号,适配层据此做"只进不退"的覆盖判断。
*/
sealed interface FlightProjectionWrite {
val flid: String
val stateVersion: Long
data class Upsert(
override val flid: String,
override val stateVersion: Long,
val payloadJson: String,
) : FlightProjectionWrite
data class Delete(override val flid: String, override val stateVersion: Long) : FlightProjectionWrite
}
/**
* 航班查询投影(Redis)的出口。投影是 PG 当前态的副本,只读不权威(`INV-5`、`INV-11`)。
*
* 写成功这条消息才算处理完成(`INV-10`):**写不成功必须抛异常**,不能吞掉——
* 调用方据此保持未完成状态、下轮整条重处理(`US-05` AC4)。
*/
interface FlightProjectionPort {
/** 一批一起写:同一条消息产生的写要么整批成功,要么抛异常整批重来。 */
fun write(writes: List<FlightProjectionWrite>)
/** 给健康检查用的连通性探测。 */
fun ping(): Boolean = true
}
/** 投影写失败:包一层好让日志和错误分类看得出是投影这一步挂的,处理侧按 `INFRA` 重试。 */
class FlightProjectionFailure(msgId: Long, cause: Throwable) :
RuntimeException("flight projection write failed msgId=$msgId: ${cause.message ?: cause.javaClass.simpleName}", cause)
/** 投影载荷与 `KAFKA:schd` 整态同一份 JSON:投影和通知同源,对账时不用比两种形状(`C-11`)。 */
internal fun projectionOf(snapshot: FlightSnapshot, mapper: ObjectMapper): FlightProjectionWrite.Upsert =
FlightProjectionWrite.Upsert(
flid = snapshot.flid,
stateVersion = snapshot.stateVersion,
payloadJson = mapper.writeValueAsString(flightPayload(snapshot)),
)
@@ -106,7 +106,8 @@ class Pump(
/**
* 处理一条消息:读原文 → 解码 → 绑定业务身份 → 分派给对应处理器。
*
* 业务数据、终态与回填意图由各处理器在自己的事务里写入(终态与回填意图是同一条 UPDATE)。
* 业务数据、Redis 投影、终态与回填意图由各处理器按三步提交写入([FlightCommit]):
* 领域事务提交后才写投影,投影成功后才记终态与回填意图(同一条 UPDATE)。
* **这里不做信箱回填**:主泵是 FIFO 关键路径,跨库写会把它绑在共享 MySQL 的可用性上。
* 回填由 `JobRunner` 定时的 `BackfillService.sweep` 驱动(调度周期不等于完成时限)。
*
@@ -6,33 +6,26 @@ import com.gzzn.omms.msgexchange.config.OperationDayProps
import com.gzzn.omms.msgexchange.domain.DecodedMessage
import com.gzzn.omms.msgexchange.domain.MsgEvent
import com.gzzn.omms.msgexchange.domain.ProcState
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.domain.SnapshotFlag
import com.gzzn.omms.msgexchange.domain.SnapshotLogEntry
import com.gzzn.omms.msgexchange.domain.SnapshotResult
import com.gzzn.omms.msgexchange.domain.Targets
import com.gzzn.omms.msgexchange.domain.OperationDayCalculator
import com.gzzn.omms.msgexchange.domain.flight.FlightSnapshot
import com.gzzn.omms.msgexchange.domain.flight.FlightState
import com.gzzn.omms.msgexchange.domain.flight.FlightStateEngine
import com.gzzn.omms.msgexchange.domain.flight.ScheduleRecord
import com.gzzn.omms.msgexchange.domain.flight.SnapshotValidation
import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
import com.gzzn.omms.msgexchange.infra.persistence.PersistOutcome
import com.gzzn.omms.msgexchange.infra.persistence.PipelineLockRepository
import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import com.gzzn.omms.msgexchange.infra.persistence.SnapshotLogRepository
import jakarta.inject.Singleton
import java.time.Clock
import java.time.Instant
import java.time.LocalDate
import java.time.ZoneId
/**
* 处理器告诉调用方这一条消息处理成了什么。
* 无论哪种结果,终态和回填待办都由处理器在自己的事务里写好
* 业务型成功的终态和回填待办已由三步提交写好([FlightCommit]),调用方不必再补
*/
sealed interface ApplyResult {
/** 业务处理成功(包含"重复写入但结果一致"这种幂等成功)。 */
@@ -52,15 +45,14 @@ class ProtocolViolation(message: String) : RuntimeException(message)
* 处理 SCHD 日计划报文(DNLD 和 RESP 走同一条路):把报文里的航班记录合并进航班当前态。
*
* 顺序是:整包校验 → 加锁 → 核对每条航班的运营日 → 逐条合并写入 → 登记待发事件 →
* 写终态和回填待办。除了校验,后面所有步骤都在同一个事务里,任何一步失败整包回滚,
* 不会留下写了一半的数据。
* 刷投影 → 写终态和回填待办。领域这一段在同一个事务里,任何一步失败整包回滚,
* 不会留下写了一半的数据;投影与终态按三步提交各走一步([FlightCommit]
*
* 报文里没提到的航班不会被删除——日计划只负责写它带来的那部分。
*/
@Singleton
class ScheduleProcessor(
private val txManager: PipelineTransactionManager,
private val lock: PipelineLockRepository,
private val commit: FlightCommit,
private val procState: ProcStateRepository,
private val flightState: FlightStateRepository,
private val msgEvents: MsgEventRepository,
@@ -98,21 +90,17 @@ class ScheduleProcessor(
val ok = validation as SnapshotValidation.Ok
if (ok.perRecordDay.isEmpty()) {
// 报文合法但没有记录:不写航班,但终态与回填意图仍在锁事务内一起提交
txManager.inTransaction {
lock.lock()
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
}
// 报文合法但没有记录:不写航班、没有投影要刷,终态与回填意图仍走三步提交的最后一步
commit.commit(head) { DomainOutcome(Unit) }
logSnapshot(head, body, SnapshotResult.COMMITTED, upserted = 0, setOf(SnapshotFlag.EMPTY), started)
return ApplyResult.Succeeded
}
val flags = linkedSetOf<SnapshotFlag>()
return try {
// 以下都在同一个事务里:加锁 → 校验运营日 → 逐条合并写入 → 登记事件与回填待办
val upserted = txManager.inTransaction {
lock.lock()
// 领域这一段在同一个事务里:加锁 → 校验运营日 → 逐条合并写入 → 登记事件
// 投影与终态由 FlightCommit 接着走后两步
val upserted = commit.commit(head) {
// 一次批量查出这些航班现有的运营日,逐个比对(避免逐条查询)
val mains = flightState.findMainRows(ok.perRecordDay.keys)
ok.perRecordDay.forEach { (flid, day) ->
@@ -126,6 +114,7 @@ class ScheduleProcessor(
var written = 0
val events = mutableListOf<MsgEvent>()
val projections = mutableListOf<FlightProjectionWrite>()
// 一次建索引,避免在航班级循环里反复线性扫描(大日计划下是 O(N²))。
val recordsByFlid = body.records.associateBy { it.flid }
ok.perRecordDay.forEach { (flid, day) ->
@@ -145,12 +134,11 @@ class ScheduleProcessor(
throw ProtocolViolation("operation-day guard violated flid=$flid")
else -> written++
}
events += snapshotEvents(next)
events += eventsFor(next, head.msgId, mapper, clock.instant())
projections += projectionOf(next, mapper)
}
if (events.isNotEmpty()) msgEvents.insertAll(events)
// 终态与回填待办跟业务数据同事务提交:要么全成,要么全回滚
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
written
DomainOutcome(written, projections)
}
logSnapshot(head, body, SnapshotResult.COMMITTED, upserted, flags, started)
ApplyResult.Succeeded
@@ -160,34 +148,6 @@ class ScheduleProcessor(
}
}
/** 每次航班状态变化登记两个事件:KAFKA_SCHD 发完整状态,KAFKA_MSG 发一条变更通知。 */
private fun snapshotEvents(next: FlightSnapshot): List<MsgEvent> {
val createdAt = clock.instant()
val payload = linkedMapOf<String, Any>(
"flid" to next.flid,
"stateVersion" to next.stateVersion,
"scalars" to next.scalars,
"collections" to next.collections,
)
val notify = mapper.writeValueAsString(mapOf("flid" to next.flid, "stateVersion" to next.stateVersion))
return listOf(
MsgEvent(
target = Targets.KAFKA_SCHD,
partitionKey = next.flid,
stateVersion = next.stateVersion,
payloadJson = mapper.writeValueAsString(payload),
createdAt = createdAt,
),
MsgEvent(
target = Targets.KAFKA_MSG,
partitionKey = next.flid,
stateVersion = next.stateVersion,
payloadJson = notify,
createdAt = createdAt,
),
)
}
/**
* 写一条处理留痕(仅供排查和统计,不参与业务判断)。放在事务外做,
* 写失败也只记日志,不会连累处理结果。scope 是这份报文覆盖的运营日范围。