feat(processing): 领域/投影/终态三步提交与 FlightProjectionPort(ACM2-79)

PIPELINE_LOCK 跨 Redis 写;MSG_EVENT HELD→PENDING;Redis 失败不终态不投递;V3 迁移。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
windyboy
2026-09-21 15:24:34 +08:00
co-authored by Cursor
parent 891088235c
commit 2c3d6af309
28 changed files with 928 additions and 167 deletions
@@ -0,0 +1,13 @@
package com.gzzn.omms.msgexchange.config
import io.micronaut.context.annotation.ConfigurationProperties
/** 航班查询投影所在的 Redis,对应 `msgx.redis.*`;键位沿用旧系统(`C-11`、`Q6`)。 */
@ConfigurationProperties("msgx.redis")
class RedisProps {
/** 有没有 Redis 可用。关着的时候投影写是空操作(`G-REDIS-PROJECTION`)。 */
var enabled: Boolean = false
/** 航班快照所在的哈希键,字段名是 `FLID`。 */
var flightKey: String = "flightInfo"
}
@@ -6,8 +6,14 @@ package com.gzzn.omms.msgexchange.domain
*/
enum class EventType { UPSERT, TOMBSTONE }
/** 投递状态:PENDING 待发 → SENT 已发出;发送失败按退避重试,重试次数用尽转 DEAD,留在表里当死信队列。 */
enum class EventStatus { PENDING, SENT, DEAD }
/**
* 投递状态:HELD 暂存 → PENDING 待发 → SENT 已发出;发送失败按退避重试,重试次数用尽转 DEAD,
* 留在表里当死信队列。
*
* HELD 是领域事务登记、还没放行的事件:Redis 投影写成功后由终态事务转成 PENDING,
* 投递只领 PENDING,因此处理完成前不会发 Kafka`INV-3`、`INV-10`)。
*/
enum class EventStatus { HELD, PENDING, SENT, DEAD }
/**
* MSG_EVENT:一张待发事件表(outbox),记录航班状态变更要对外发什么。
@@ -21,6 +27,8 @@ enum class EventStatus { PENDING, SENT, DEAD }
*/
data class MsgEvent(
val eventId: Long? = null,
/** 登记这条事件的消息 ID;暂存事件按它清理与放行(`HELD`)。非管道写入方可留空。 */
val msgId: Long? = null,
val target: String,
val partitionKey: String, // 分区键,恒为 FLID(航班实例 ID)
val eventType: EventType = EventType.UPSERT,
@@ -31,6 +31,14 @@ interface PipelineTransactionManager {
/** 单行锁:进事务后先 `SELECT ... FOR UPDATE`,让并发的状态写事务排队执行。 */
interface PipelineLockRepository {
fun lock()
/**
* 跨事务持锁:块内的多次事务复用同一条连接,锁到块结束才释放。
*
* 「领域提交 → 写 Redis → 终态提交」整段都要互斥,而行锁在提交那一刻就没了,
* 盖不住中间的 Redis 写(docs/implementation.md「事务边界」)。
*/
fun <T> holdAcrossTransactions(block: () -> T): T
}
/**
@@ -172,6 +180,20 @@ data class Backlog(
interface MsgEventRepository {
fun insertAll(events: List<MsgEvent>): List<Long>
/**
* 清掉这条消息上一轮残留的暂存事件(`HELD`)。
*
* 领域事务重跑前调用:投影写失败或进程崩溃时,上一轮登记的事件还停在 `HELD`,
* 不清掉就会在重处理后变成永远没人放行的悬挂行。
*/
fun discardHeld(msgId: Long): Int
/**
* 放行这条消息的暂存事件:`HELD` → `PENDING`,投递这才领得到。
* 由终态事务调用,所以 Redis 投影没写成功之前不会发出去(`INV-3`、`INV-10`)。
*/
fun releaseHeld(msgId: Long): Int
/**
* 领取待发事件;[excludePartitionKeys] 为本轮已暂停的 FLID,排除后避免单航班占满整批
* 导致其他航班饿死(ACM2-95)。
@@ -14,13 +14,36 @@ private val transactionConnections = ThreadLocal.withInitial {
java.util.IdentityHashMap<DataSource, java.sql.Connection>()
}
/**
* 被钉住的连接:块内的事务与单语句都走它。会话级锁(`pg_advisory_lock`)只在自己的会话里成立,
* 所以要跨事务持锁,就得让这几次事务落在同一条连接上。
*/
private val pinnedConnections = ThreadLocal.withInitial {
java.util.IdentityHashMap<DataSource, java.sql.Connection>()
}
internal fun <T> DataSource.withPinnedSession(block: (java.sql.Connection) -> T): T {
val pinned = pinnedConnections.get()
check(pinned[this] == null) { "session already pinned for this data source" }
val conn = this.connection
pinned[this] = conn
try {
return block(conn)
} finally {
pinned.remove(this)
if (pinned.isEmpty()) pinnedConnections.remove()
conn.close()
}
}
internal fun <T> DataSource.withTransaction(block: () -> T): T {
val connections = transactionConnections.get()
val existing = connections[this]
if (existing != null) {
return block()
}
val conn = this.connection
val pinned = pinnedConnections.get()[this]
val conn = pinned ?: this.connection
val oldAutoCommit = conn.autoCommit
conn.autoCommit = false
connections[this] = conn
@@ -42,19 +65,20 @@ internal fun <T> DataSource.withTransaction(block: () -> T): T {
conn.autoCommit = oldAutoCommit
} catch (_: Throwable) {
}
conn.close()
// 钉住的连接由 withPinnedSession 关闭:提前还池会连同会话锁一起丢掉。
if (pinned == null) conn.close()
}
}
internal fun DataSource.obtainConnection(): java.sql.Connection =
transactionConnections.get()[this] ?: this.connection
transactionConnections.get()[this] ?: pinnedConnections.get()[this] ?: this.connection
internal fun java.sql.Connection.releaseIfNotInTransaction(dataSource: DataSource) {
val connections = transactionConnections.get()
if (connections[dataSource] !== this) {
this.close()
if (connections.isEmpty()) transactionConnections.remove()
}
if (connections[dataSource] === this) return
if (pinnedConnections.get()[dataSource] === this) return
this.close()
if (connections.isEmpty()) transactionConnections.remove()
}
internal fun <T> DataSource.query(sql: String, bind: (java.sql.PreparedStatement) -> Unit, map: (ResultSet) -> T): List<T> {
@@ -63,10 +63,40 @@ class JdbcPipelineTransactionManager(
class JdbcPipelineLockRepository(
private val ds: DataSource,
) : PipelineLockRepository {
/** 进事务后的第一步:对锁行 `FOR UPDATE`,让并发的状态写事务排队。 */
/**
* 进事务后的第一步:先在咨询锁上排队,再对锁行 `FOR UPDATE`。
*
* 行锁在提交时就释放,盖不住处理步骤中段的 Redis 写,所以两个写者都要过同一把咨询锁:
* 主泵整段持会话级([holdAcrossTransactions]),历史清理在自己的事务里持事务级。
* 同一会话重复取同一把咨询锁不自阻塞,因此持锁期间内层事务照常通过。
*/
override fun lock() {
ds.queryOne("SELECT pg_advisory_xact_lock(?)", { ps -> ps.setLong(1, PIPELINE_ADVISORY_KEY) }) { 1 }
ds.queryOne("SELECT lock_id FROM pipeline_lock WHERE lock_id = 1 FOR UPDATE", {}) { 1 } ?: error("PIPELINE_LOCK row missing")
}
/** 钉住一条连接并在其上持会话级咨询锁:块内的两次事务复用它,出块才解锁还池。 */
override fun <T> holdAcrossTransactions(block: () -> T): T = ds.withPinnedSession {
advisory("SELECT pg_advisory_lock(?)")
try {
block()
} finally {
// 解锁失败只能记账:连接随后归还连接池,会话锁若残留会挡住后续处理步骤。
runCatching { advisory("SELECT pg_advisory_unlock(?)") }
.onFailure { log.error("pipeline advisory unlock failed", it) }
}
}
private fun advisory(sql: String) {
ds.queryOne(sql, { ps -> ps.setLong(1, PIPELINE_ADVISORY_KEY) }) { 1 }
}
private companion object {
private val log = org.slf4j.LoggerFactory.getLogger(JdbcPipelineLockRepository::class.java)
/** `PIPELINE_LOCK` 的咨询锁键('msgx' 的 ASCII 加序号 1):同库其他应用撞键的概率足够低。 */
private const val PIPELINE_ADVISORY_KEY = 0x6D73_6778_01L
}
}
@Singleton
@@ -379,8 +409,8 @@ class JdbcMsgEventRepository(
private fun insertOne(e: MsgEvent): Long =
ds.updateReturningLong(
"""
INSERT INTO msg_event (target, partition_key, event_type, state_version, payload_json, state, attempts, next_attempt_at, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING event_id
INSERT INTO msg_event (target, partition_key, event_type, state_version, payload_json, state, attempts, next_attempt_at, created_at, msg_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING event_id
""".trimIndent(),
{ ps -> bindEvent(ps, e) },
)
@@ -393,15 +423,16 @@ class JdbcMsgEventRepository(
private fun upsertSchd(e: MsgEvent): Long? =
ds.queryOne(
"""
INSERT INTO msg_event (target, partition_key, event_type, state_version, payload_json, state, attempts, next_attempt_at, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
INSERT INTO msg_event (target, partition_key, event_type, state_version, payload_json, state, attempts, next_attempt_at, created_at, msg_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (target, partition_key) WHERE target = 'KAFKA:schd'
DO UPDATE SET
event_id = EXCLUDED.event_id,
event_type = EXCLUDED.event_type,
state_version = EXCLUDED.state_version,
payload_json = EXCLUDED.payload_json,
state = 'PENDING',
msg_id = EXCLUDED.msg_id,
state = EXCLUDED.state,
attempts = 0,
next_attempt_at = NULL,
error_class = NULL,
@@ -425,8 +456,23 @@ class JdbcMsgEventRepository(
ps.setInt(7, e.attempts)
ps.setTimestamp(8, e.nextAttemptAt?.toSqlTimestamp())
ps.setTimestamp(9, e.createdAt.toSqlTimestamp())
e.msgId?.let { ps.setLong(10, it) } ?: ps.setNull(10, java.sql.Types.BIGINT)
}
/** 重处理前清残:上一轮登记但没放行的事件行删掉,避免留下永远发不出去的悬挂行。 */
override fun discardHeld(msgId: Long): Int =
ds.update(
"DELETE FROM msg_event WHERE msg_id = ? AND state = 'HELD'",
{ ps -> ps.setLong(1, msgId) },
)
/** 放行:投影已写成功,事件转 PENDING 交给投递。 */
override fun releaseHeld(msgId: Long): Int =
ds.update(
"UPDATE msg_event SET state = 'PENDING' WHERE msg_id = ? AND state = 'HELD'",
{ ps -> ps.setLong(1, msgId) },
)
override fun claimBatch(target: String, limit: Int, excludePartitionKeys: Set<String>): List<MsgEvent> {
if (excludePartitionKeys.isEmpty()) {
return ds.query(
@@ -536,6 +582,7 @@ class JdbcMsgEventRepository(
private fun mapEvent(rs: ResultSet) = MsgEvent(
eventId = rs.getLong("event_id"),
msgId = rs.getLong("msg_id").takeIf { !rs.wasNull() },
target = rs.getString("target"),
partitionKey = rs.getString("partition_key"),
eventType = EventType.valueOf(rs.getString("event_type")),
@@ -0,0 +1,49 @@
package com.gzzn.omms.msgexchange.infra.projection
import com.gzzn.omms.msgexchange.config.RedisProps
import com.gzzn.omms.msgexchange.processing.FlightProjectionPort
import com.gzzn.omms.msgexchange.processing.FlightProjectionWrite
import io.micronaut.context.annotation.Requires
import jakarta.inject.Singleton
/**
* Redis 投影适配器骨架:键位与写法在这里定死,**但客户端还没接进来**`G-REDIS-PROJECTION`)。
*
* 写入形态沿用旧系统:一个哈希键(`PARAM:msgx.redis.flight-key`)装全部航班,
* field 是 `FLID`value 是整态 JSON,不设过期;删除航班就删掉这个 field(`INV-8`)。
*
* 开了 `msgx.redis.enabled` 却没有客户端时,写入直接抛异常而不是假装成功:
* 按 `INV-10`,这条消息就停在未完成、下轮重试,不会被当成已处理写回信箱。
*/
@Requires(property = "msgx.stubs", notEquals = "true")
@Requires(property = "msgx.redis.enabled", value = "true")
@Singleton
class RedisFlightProjectionPort(
private val props: RedisProps,
) : FlightProjectionPort {
override fun write(writes: List<FlightProjectionWrite>) {
throw UnsupportedOperationException(
"redis client not wired yet (G-REDIS-PROJECTION); pending writes=${writes.size} key=${props.flightKey}",
)
}
override fun ping(): Boolean = false
}
/**
* 没接 Redis 时的投影出口:什么都不做。
*
* 这不是"投影写成功"的承诺——`INV-10` 在真实客户端接通之前只能空转(`G-REDIS-PROJECTION`)。
* 摆这个 bean 是为了让三步提交的时序在没有 Redis 的环境里也照常跑,而不是让每条航班报文都失败。
*/
@Requires(property = "msgx.stubs", notEquals = "true")
@Requires(property = "msgx.redis.enabled", notEquals = "true")
@Singleton
class NoopFlightProjectionPort : FlightProjectionPort {
private val log = org.slf4j.LoggerFactory.getLogger(NoopFlightProjectionPort::class.java)
override fun write(writes: List<FlightProjectionWrite>) {
log.debug("redis projection disabled, skipping {} write(s) [G-REDIS-PROJECTION]", writes.size)
}
}
@@ -1,6 +1,8 @@
package com.gzzn.omms.msgexchange.infra.stub
import com.gzzn.omms.msgexchange.delivery.DeliveryPort
import com.gzzn.omms.msgexchange.processing.FlightProjectionPort
import com.gzzn.omms.msgexchange.processing.FlightProjectionWrite
import io.micronaut.context.annotation.Requires
import jakarta.inject.Singleton
@@ -32,3 +34,37 @@ class StubDeliveryPort : DeliveryPort {
sent += Sent(topic, key, null)
}
}
/**
* 航班查询投影的内存假实现,只有配置 msgx.stubs=true 时才装配。
*
* [snapshots] 是投影的当前内容(FLID → 整态 JSON),删除会把条目移走,行为对齐 Redis 哈希。
* [failWith] 用来做故障注入:置上以后每次写都抛这个异常,用于验证"投影写不成功就不算处理完成"
* `INV-10`);置回 null 即恢复。
*/
@Requires(property = "msgx.stubs", value = "true")
@Singleton
class StubFlightProjectionPort : FlightProjectionPort {
val snapshots = linkedMapOf<String, String>()
val writes = mutableListOf<FlightProjectionWrite>()
@Volatile
var failWith: RuntimeException? = null
fun clear() {
snapshots.clear(); writes.clear(); failWith = null
}
override fun write(writes: List<FlightProjectionWrite>) {
failWith?.let { throw it }
this.writes += writes
writes.forEach { w ->
when (w) {
is FlightProjectionWrite.Upsert -> snapshots[w.flid] = w.payloadJson
is FlightProjectionWrite.Delete -> snapshots.remove(w.flid)
}
}
}
override fun ping(): Boolean = failWith == null
}
@@ -51,6 +51,8 @@ class StubPipelineTx : PipelineTransactionManager {
@Requires(property = "msgx.stubs", value = "true")
class StubPipelineLock : PipelineLockRepository {
override fun lock() = Unit
override fun <T> holdAcrossTransactions(block: () -> T): T = block()
}
/** 内存版 PROC_STATE:入队幂等,写终态时一并写下回填待办。 */
@@ -261,7 +263,7 @@ class StubMsgEvents : MsgEventRepository {
/**
* 与 JDBC upsert 同口径的单行投影:只进不退;同版本只有 tombstone 能覆盖非 tombstone
* 接受的写代次换新 `EVENT_ID` 并重置 `PENDING`(清空 attempts/next/error)。
* 接受的写代次换新 `EVENT_ID` 并沿用本次写入的状态(清空 attempts/next/error)。
*/
private fun upsertSchd(e: MsgEvent): Long {
val existing = rows.values.firstOrNull { it.target == Targets.KAFKA_SCHD && it.partitionKey == e.partitionKey }
@@ -273,12 +275,26 @@ class StubMsgEvents : MsgEventRepository {
}
val id = ids.incrementAndGet()
rows[id] = e.copy(
eventId = id, state = EventStatus.PENDING, attempts = 0, nextAttemptAt = null,
eventId = id, attempts = 0, nextAttemptAt = null,
errorClass = null, lastError = null, sentAt = null,
)
return id
}
/** 清残:上一轮登记但没放行的事件行删掉(对齐 JDBC 的按 MSG_ID 删 HELD)。 */
override fun discardHeld(msgId: Long): Int {
val ids = rows.values.filter { it.msgId == msgId && it.state == EventStatus.HELD }.map { it.eventId!! }
ids.forEach { rows.remove(it) }
return ids.size
}
/** 放行:HELD → PENDING,投递这才领得到。 */
override fun releaseHeld(msgId: Long): Int {
val held = rows.values.filter { it.msgId == msgId && it.state == EventStatus.HELD }
held.forEach { rows[it.eventId!!] = it.copy(state = EventStatus.PENDING) }
return held.size
}
override fun claimBatch(target: String, limit: Int, excludePartitionKeys: Set<String>): List<MsgEvent> =
rows.values
.filter { it.target == target && it.state == EventStatus.PENDING }
@@ -4,11 +4,10 @@ import com.fasterxml.jackson.databind.ObjectMapper
import com.gzzn.omms.msgexchange.codec.FlopPayload
import com.gzzn.omms.msgexchange.config.OperationDayProps
import com.gzzn.omms.msgexchange.domain.DecodedMessage
import com.gzzn.omms.msgexchange.domain.EventType
import com.gzzn.omms.msgexchange.domain.EventStatus
import com.gzzn.omms.msgexchange.domain.MsgEvent
import com.gzzn.omms.msgexchange.domain.OperationDayCalculator
import com.gzzn.omms.msgexchange.domain.ProcState
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.domain.Targets
import com.gzzn.omms.msgexchange.domain.flight.FlightSnapshot
import com.gzzn.omms.msgexchange.domain.flight.FlightState
@@ -18,9 +17,6 @@ import com.gzzn.omms.msgexchange.domain.flight.ScheduleRecord
import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
import com.gzzn.omms.msgexchange.infra.persistence.PersistOutcome
import com.gzzn.omms.msgexchange.infra.persistence.PipelineLockRepository
import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import jakarta.inject.Singleton
import java.time.Clock
import java.time.Instant
@@ -35,29 +31,22 @@ import java.time.ZoneId
*/
@Singleton
class FlopProcessor(
private val txManager: PipelineTransactionManager,
private val lock: PipelineLockRepository,
private val commit: FlightCommit,
private val flightState: FlightStateRepository,
private val msgEvents: MsgEventRepository,
private val procState: ProcStateRepository,
private val mapper: ObjectMapper,
private val clock: Clock,
) {
fun apply(head: ProcState, msg: DecodedMessage, payload: FlopPayload): ApplyResult = txManager.inTransaction {
lock.lock()
fun apply(head: ProcState, msg: DecodedMessage, payload: FlopPayload): ApplyResult = commit.commit(head) {
// 迟到/未知航班:幂等成功,不创建(创建入口只有 SCHD/ADFT),也没有投影要刷
val current = flightState.loadFullSnapshot(payload.flid)
if (current == null) {
// 迟到/未知航班:幂等成功,不创建(创建入口只有 SCHD/ADFT);终态同事务落库
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
return@inTransaction ApplyResult.Succeeded
}
?: return@commit DomainOutcome(ApplyResult.Succeeded)
val change = MergeChange(flid = payload.flid, scalars = payload.scalars, collections = payload.collections)
val next = FlightStateEngine.mergedState(current, change)
flightState.persistFullState(next, msgId = head.msgId, now = clock.instant())
msgEvents.insertAll(eventsFor(next, mapper, clock.instant()))
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
ApplyResult.Succeeded
msgEvents.insertAll(eventsFor(next, head.msgId, mapper, clock.instant()))
DomainOutcome(ApplyResult.Succeeded, listOf(projectionOf(next, mapper)))
}
}
@@ -65,43 +54,47 @@ class FlopProcessor(
* 处理 FDEL 删除报文:把在用的航班标记为 DELETED,版本号加一,明细数据保留,
* 并且只发布一次删除事件。
*
* 已经删除过、或航班本来就不存在时,算处理成功但不再动版本、不重复发事件
* 已经删除过、或航班本来就不存在时,算处理成功但不再动版本、不重复发事件
* 但只要航班还在库里且是删除态,投影删除就照发一次——幂等命令换来"上一轮没删干净"能自愈(`INV-8`)。
*/
@Singleton
class FdelProcessor(
private val txManager: PipelineTransactionManager,
private val lock: PipelineLockRepository,
private val commit: FlightCommit,
private val flightState: FlightStateRepository,
private val msgEvents: MsgEventRepository,
private val procState: ProcStateRepository,
private val mapper: ObjectMapper,
private val clock: Clock,
) {
fun apply(head: ProcState, msg: DecodedMessage, payload: FlopPayload): ApplyResult = txManager.inTransaction {
lock.lock()
fun apply(head: ProcState, msg: DecodedMessage, payload: FlopPayload): ApplyResult = commit.commit(head) {
val deleted = flightState.markDeleted(payload.flid, msgId = head.msgId, now = clock.instant())
if (deleted) {
val current = flightState.loadFullSnapshot(payload.flid)
// 只有"在用 → 删除"这一步才发删除通知,而且和状态变更写在同一个事务里
val createdAt = clock.instant()
// 航班根本不存在(迟到或多余的删除报文):算成功,也没有投影要删
val main = flightState.findMainRow(payload.flid)
?: return@commit DomainOutcome(ApplyResult.Succeeded)
// 这一次删掉的,或者上一轮就是本消息删掉的(投影写失败后重处理),都要登记删除通知:
// 重处理开始时已把上一轮没放行的事件清掉,这里不补登就会把这条通知丢了。
if (deleted || (main.state == FlightState.DELETED && main.lastMsgId == head.msgId)) {
msgEvents.insertAll(
listOf(
// C-9:删除通知只走 KAFKA:msgschd 不再发 tombstone
// value 形态("deleted":true 的 JSON)沿用现状,待 Q5 定稿
MsgEvent(
msgId = head.msgId,
target = Targets.KAFKA_MSG,
partitionKey = payload.flid,
stateVersion = current?.stateVersion ?: 0L,
stateVersion = main.stateVersion,
payloadJson = mapper.writeValueAsString(
mapOf("flid" to payload.flid, "stateVersion" to (current?.stateVersion ?: 0L), "deleted" to true),
mapOf("flid" to payload.flid, "stateVersion" to main.stateVersion, "deleted" to true),
),
createdAt = createdAt,
state = EventStatus.HELD,
createdAt = clock.instant(),
),
),
)
}
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant()) // 没删到东西说明是迟到或重复报文,照样算成功
ApplyResult.Succeeded
// INV-8:只要航班是删除态就得从投影里抹掉。重复报文也重删一次——代价是一条幂等命令,
// 换来"上一轮删投影没成功"能自愈。
DomainOutcome(ApplyResult.Succeeded, listOf(FlightProjectionWrite.Delete(payload.flid, main.stateVersion)))
}
}
@@ -114,11 +107,9 @@ class FdelProcessor(
*/
@Singleton
class AdftProcessor(
private val txManager: PipelineTransactionManager,
private val lock: PipelineLockRepository,
private val commit: FlightCommit,
private val flightState: FlightStateRepository,
private val msgEvents: MsgEventRepository,
private val procState: ProcStateRepository,
operationDayProps: OperationDayProps,
private val mapper: ObjectMapper,
private val clock: Clock,
@@ -130,24 +121,27 @@ class AdftProcessor(
fun apply(head: ProcState, msg: DecodedMessage, record: ScheduleRecord): ApplyResult =
try {
txManager.inTransaction {
lock.lock()
commit.commit(head) {
val main = flightState.findMainRow(record.flid)
if (main != null && main.state == FlightState.DELETED) {
// 已删除的航班重新激活:状态改回 ACTIVE、版本号加一,并登记状态事件
var revived: FlightSnapshot? = null
if (flightState.revive(record.flid, msgId = head.msgId, now = clock.instant())) {
val current = flightState.loadFullSnapshot(record.flid)
if (current != null) {
val next = FlightStateEngine.mergedState(current, setOnly(record))
flightState.persistFullState(next, msgId = head.msgId, now = clock.instant())
msgEvents.insertAll(eventsFor(next, mapper, clock.instant()))
msgEvents.insertAll(eventsFor(next, head.msgId, mapper, clock.instant()))
revived = next
}
} else {
// 并发下可能已被别的消息处理掉:不改版本、不重复发事件,但留下痕迹便于对账。
log.warn("adft revive no-op (already active or vanished) msgId={} flid={}", head.msgId, record.flid)
}
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
return@inTransaction ApplyResult.Succeeded
return@commit DomainOutcome(
ApplyResult.Succeeded,
listOfNotNull(revived?.let { projectionOf(it, mapper) }),
)
}
val current = flightState.loadFullSnapshot(record.flid)
@@ -173,9 +167,8 @@ class AdftProcessor(
// 若按 INFRA 抛出去,会被当成暂时性故障白白重试到耗尽,并给出误导的错误类别。
throw ProtocolViolation("operation-day guard violated flid=${record.flid}")
}
msgEvents.insertAll(eventsFor(next, mapper, clock.instant()))
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
ApplyResult.Succeeded
msgEvents.insertAll(eventsFor(next, head.msgId, mapper, clock.instant()))
DomainOutcome(ApplyResult.Succeeded, listOf(projectionOf(next, mapper)))
}
} catch (e: ProtocolViolation) {
log.error("adft DEAD(PROTOCOL) msgId={} reason={}", head.msgId, e.message)
@@ -201,28 +194,37 @@ class AdftProcessor(
// 共享小工具(处理器层私有约定)
// =====================================================================
/** 航班状态变化后要发的两类事件:KAFKA_SCHD 发完整状态,KAFKA_MSG 发一条变更通知。 */
internal fun eventsFor(next: FlightSnapshot, mapper: ObjectMapper, createdAt: Instant): List<MsgEvent> {
val payload = linkedMapOf<String, Any>(
"flid" to next.flid,
"stateVersion" to next.stateVersion,
"scalars" to next.scalars,
"collections" to next.collections,
)
return listOf(
/** 航班整态载荷:`KAFKA:schd` 事件与 Redis 投影共用这一份形状。 */
internal fun flightPayload(next: FlightSnapshot): Map<String, Any> = linkedMapOf(
"flid" to next.flid,
"stateVersion" to next.stateVersion,
"scalars" to next.scalars,
"collections" to next.collections,
)
/**
* 航班状态变化后要发的两类事件:KAFKA_SCHD 发完整状态,KAFKA_MSG 发一条变更通知。
*
* 两条都以 [EventStatus.HELD] 登记,投影写成功后由终态事务放行(`INV-3`、`INV-10`)。
*/
internal fun eventsFor(next: FlightSnapshot, msgId: Long, mapper: ObjectMapper, createdAt: Instant): List<MsgEvent> =
listOf(
MsgEvent(
msgId = msgId,
target = Targets.KAFKA_SCHD,
partitionKey = next.flid,
stateVersion = next.stateVersion,
payloadJson = mapper.writeValueAsString(payload),
payloadJson = mapper.writeValueAsString(flightPayload(next)),
state = EventStatus.HELD,
createdAt = createdAt,
),
MsgEvent(
msgId = msgId,
target = Targets.KAFKA_MSG,
partitionKey = next.flid,
stateVersion = next.stateVersion,
payloadJson = mapper.writeValueAsString(mapOf("flid" to next.flid, "stateVersion" to next.stateVersion)),
state = EventStatus.HELD,
createdAt = createdAt,
),
)
}
@@ -0,0 +1,67 @@
package com.gzzn.omms.msgexchange.processing
import com.gzzn.omms.msgexchange.domain.ProcState
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
import com.gzzn.omms.msgexchange.infra.persistence.PipelineLockRepository
import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import jakarta.inject.Singleton
import java.time.Clock
/**
* 领域事务的产物:交给调用方的结果,加上这次变更要刷进查询投影的航班。
* 没有航班变更(迟到报文、空日计划)时投影为空,这一步直接跳过。
*/
data class DomainOutcome<T>(val value: T, val projections: List<FlightProjectionWrite> = emptyList())
/**
* 业务型成功的三步提交(`INV-3`、`INV-10`,见 docs/implementation.md「事务边界」):
*
* 1. 领域事务:航班变更与待发事件一起提交,事件落 `HELD`(投递领不到);
* 2. 写 Redis 投影:失败就抛出去,这条消息保持未完成,下轮重处理;
* 3. 终态事务:放行待发事件,写 `SUCCEEDED` 与回填意图。
*
* 三步整段持 `PIPELINE_LOCK`:行锁提交即释放,盖不住中间的投影写,所以由
* [PipelineLockRepository.holdAcrossTransactions] 在一条连接上从头持到尾。
*
* 投影失败时第一步已经提交,**不回滚**(`US-03` AC3):重处理按当前完整态重写投影,
* 上一轮的 `HELD` 事件先清后补,不会重复投递。
*/
@Singleton
class FlightCommit(
private val txManager: PipelineTransactionManager,
private val lock: PipelineLockRepository,
private val procState: ProcStateRepository,
private val msgEvents: MsgEventRepository,
private val projection: FlightProjectionPort,
private val clock: Clock,
) {
fun <T> commit(head: ProcState, domain: () -> DomainOutcome<T>): T = lock.holdAcrossTransactions {
val outcome = txManager.inTransaction {
lock.lock()
msgEvents.discardHeld(head.msgId) // 清掉上一轮没放行的事件,再登记本轮的
domain()
}
writeProjection(head, outcome.projections)
txManager.inTransaction {
lock.lock()
msgEvents.releaseHeld(head.msgId)
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
}
outcome.value
}
private fun writeProjection(head: ProcState, writes: List<FlightProjectionWrite>) {
if (writes.isEmpty()) return
try {
projection.write(writes)
} catch (e: InterruptedException) {
throw e // 停机信号不归到这条消息头上
} catch (e: Exception) {
throw FlightProjectionFailure(head.msgId, e)
}
}
}
@@ -0,0 +1,47 @@
package com.gzzn.omms.msgexchange.processing
import com.fasterxml.jackson.databind.ObjectMapper
import com.gzzn.omms.msgexchange.domain.flight.FlightSnapshot
/**
* 一次投影写:把某个航班的整态写进查询投影,或者把它从投影里删掉。
* 带上版本号,适配层据此做"只进不退"的覆盖判断。
*/
sealed interface FlightProjectionWrite {
val flid: String
val stateVersion: Long
data class Upsert(
override val flid: String,
override val stateVersion: Long,
val payloadJson: String,
) : FlightProjectionWrite
data class Delete(override val flid: String, override val stateVersion: Long) : FlightProjectionWrite
}
/**
* 航班查询投影(Redis)的出口。投影是 PG 当前态的副本,只读不权威(`INV-5`、`INV-11`)。
*
* 写成功这条消息才算处理完成(`INV-10`):**写不成功必须抛异常**,不能吞掉——
* 调用方据此保持未完成状态、下轮整条重处理(`US-05` AC4)。
*/
interface FlightProjectionPort {
/** 一批一起写:同一条消息产生的写要么整批成功,要么抛异常整批重来。 */
fun write(writes: List<FlightProjectionWrite>)
/** 给健康检查用的连通性探测。 */
fun ping(): Boolean = true
}
/** 投影写失败:包一层好让日志和错误分类看得出是投影这一步挂的,处理侧按 `INFRA` 重试。 */
class FlightProjectionFailure(msgId: Long, cause: Throwable) :
RuntimeException("flight projection write failed msgId=$msgId: ${cause.message ?: cause.javaClass.simpleName}", cause)
/** 投影载荷与 `KAFKA:schd` 整态同一份 JSON:投影和通知同源,对账时不用比两种形状(`C-11`)。 */
internal fun projectionOf(snapshot: FlightSnapshot, mapper: ObjectMapper): FlightProjectionWrite.Upsert =
FlightProjectionWrite.Upsert(
flid = snapshot.flid,
stateVersion = snapshot.stateVersion,
payloadJson = mapper.writeValueAsString(flightPayload(snapshot)),
)
@@ -106,7 +106,8 @@ class Pump(
/**
* 处理一条消息:读原文 → 解码 → 绑定业务身份 → 分派给对应处理器。
*
* 业务数据、终态与回填意图由各处理器在自己的事务里写入(终态与回填意图是同一条 UPDATE)。
* 业务数据、Redis 投影、终态与回填意图由各处理器按三步提交写入([FlightCommit]):
* 领域事务提交后才写投影,投影成功后才记终态与回填意图(同一条 UPDATE)。
* **这里不做信箱回填**:主泵是 FIFO 关键路径,跨库写会把它绑在共享 MySQL 的可用性上。
* 回填由 `JobRunner` 定时的 `BackfillService.sweep` 驱动(调度周期不等于完成时限)。
*
@@ -6,33 +6,26 @@ import com.gzzn.omms.msgexchange.config.OperationDayProps
import com.gzzn.omms.msgexchange.domain.DecodedMessage
import com.gzzn.omms.msgexchange.domain.MsgEvent
import com.gzzn.omms.msgexchange.domain.ProcState
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.domain.SnapshotFlag
import com.gzzn.omms.msgexchange.domain.SnapshotLogEntry
import com.gzzn.omms.msgexchange.domain.SnapshotResult
import com.gzzn.omms.msgexchange.domain.Targets
import com.gzzn.omms.msgexchange.domain.OperationDayCalculator
import com.gzzn.omms.msgexchange.domain.flight.FlightSnapshot
import com.gzzn.omms.msgexchange.domain.flight.FlightState
import com.gzzn.omms.msgexchange.domain.flight.FlightStateEngine
import com.gzzn.omms.msgexchange.domain.flight.ScheduleRecord
import com.gzzn.omms.msgexchange.domain.flight.SnapshotValidation
import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
import com.gzzn.omms.msgexchange.infra.persistence.PersistOutcome
import com.gzzn.omms.msgexchange.infra.persistence.PipelineLockRepository
import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import com.gzzn.omms.msgexchange.infra.persistence.SnapshotLogRepository
import jakarta.inject.Singleton
import java.time.Clock
import java.time.Instant
import java.time.LocalDate
import java.time.ZoneId
/**
* 处理器告诉调用方这一条消息处理成了什么。
* 无论哪种结果,终态和回填待办都由处理器在自己的事务里写好
* 业务型成功的终态和回填待办已由三步提交写好([FlightCommit]),调用方不必再补
*/
sealed interface ApplyResult {
/** 业务处理成功(包含"重复写入但结果一致"这种幂等成功)。 */
@@ -52,15 +45,14 @@ class ProtocolViolation(message: String) : RuntimeException(message)
* 处理 SCHD 日计划报文(DNLD 和 RESP 走同一条路):把报文里的航班记录合并进航班当前态。
*
* 顺序是:整包校验 → 加锁 → 核对每条航班的运营日 → 逐条合并写入 → 登记待发事件 →
* 写终态和回填待办。除了校验,后面所有步骤都在同一个事务里,任何一步失败整包回滚,
* 不会留下写了一半的数据。
* 刷投影 → 写终态和回填待办。领域这一段在同一个事务里,任何一步失败整包回滚,
* 不会留下写了一半的数据;投影与终态按三步提交各走一步([FlightCommit]
*
* 报文里没提到的航班不会被删除——日计划只负责写它带来的那部分。
*/
@Singleton
class ScheduleProcessor(
private val txManager: PipelineTransactionManager,
private val lock: PipelineLockRepository,
private val commit: FlightCommit,
private val procState: ProcStateRepository,
private val flightState: FlightStateRepository,
private val msgEvents: MsgEventRepository,
@@ -98,21 +90,17 @@ class ScheduleProcessor(
val ok = validation as SnapshotValidation.Ok
if (ok.perRecordDay.isEmpty()) {
// 报文合法但没有记录:不写航班,但终态与回填意图仍在锁事务内一起提交
txManager.inTransaction {
lock.lock()
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
}
// 报文合法但没有记录:不写航班、没有投影要刷,终态与回填意图仍走三步提交的最后一步
commit.commit(head) { DomainOutcome(Unit) }
logSnapshot(head, body, SnapshotResult.COMMITTED, upserted = 0, setOf(SnapshotFlag.EMPTY), started)
return ApplyResult.Succeeded
}
val flags = linkedSetOf<SnapshotFlag>()
return try {
// 以下都在同一个事务里:加锁 → 校验运营日 → 逐条合并写入 → 登记事件与回填待办
val upserted = txManager.inTransaction {
lock.lock()
// 领域这一段在同一个事务里:加锁 → 校验运营日 → 逐条合并写入 → 登记事件
// 投影与终态由 FlightCommit 接着走后两步
val upserted = commit.commit(head) {
// 一次批量查出这些航班现有的运营日,逐个比对(避免逐条查询)
val mains = flightState.findMainRows(ok.perRecordDay.keys)
ok.perRecordDay.forEach { (flid, day) ->
@@ -126,6 +114,7 @@ class ScheduleProcessor(
var written = 0
val events = mutableListOf<MsgEvent>()
val projections = mutableListOf<FlightProjectionWrite>()
// 一次建索引,避免在航班级循环里反复线性扫描(大日计划下是 O(N²))。
val recordsByFlid = body.records.associateBy { it.flid }
ok.perRecordDay.forEach { (flid, day) ->
@@ -145,12 +134,11 @@ class ScheduleProcessor(
throw ProtocolViolation("operation-day guard violated flid=$flid")
else -> written++
}
events += snapshotEvents(next)
events += eventsFor(next, head.msgId, mapper, clock.instant())
projections += projectionOf(next, mapper)
}
if (events.isNotEmpty()) msgEvents.insertAll(events)
// 终态与回填待办跟业务数据同事务提交:要么全成,要么全回滚
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
written
DomainOutcome(written, projections)
}
logSnapshot(head, body, SnapshotResult.COMMITTED, upserted, flags, started)
ApplyResult.Succeeded
@@ -160,34 +148,6 @@ class ScheduleProcessor(
}
}
/** 每次航班状态变化登记两个事件:KAFKA_SCHD 发完整状态,KAFKA_MSG 发一条变更通知。 */
private fun snapshotEvents(next: FlightSnapshot): List<MsgEvent> {
val createdAt = clock.instant()
val payload = linkedMapOf<String, Any>(
"flid" to next.flid,
"stateVersion" to next.stateVersion,
"scalars" to next.scalars,
"collections" to next.collections,
)
val notify = mapper.writeValueAsString(mapOf("flid" to next.flid, "stateVersion" to next.stateVersion))
return listOf(
MsgEvent(
target = Targets.KAFKA_SCHD,
partitionKey = next.flid,
stateVersion = next.stateVersion,
payloadJson = mapper.writeValueAsString(payload),
createdAt = createdAt,
),
MsgEvent(
target = Targets.KAFKA_MSG,
partitionKey = next.flid,
stateVersion = next.stateVersion,
payloadJson = notify,
createdAt = createdAt,
),
)
}
/**
* 写一条处理留痕(仅供排查和统计,不参与业务判断)。放在事务外做,
* 写失败也只记日志,不会连累处理结果。scope 是这份报文覆盖的运营日范围。