cdia/msgexchange-v2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(ingress): 信箱边界层契约重构——发现与处理标记解耦、回填事实并入 PROC_STATE

Browse Source 
收报扫描不再以 DATE_PROCESSED 为谓词：终态而未回填的行（解码失败死信等）会永久占据
有限批次，累积到 claim-batch 后收报整体停摆（US-01 条目 3 / message-lifecycle §5.3）。

ingress/InboxPoller.kt：按 ID 区间升序有界读取（ID > W），水位落 INBOX_CURSOR 并与入队
同一 PG 事务推进（中断后重扫补建）；遇空洞即停，空洞超过 pipeline.max-commit-delay 判定
为永久并放行——否则水位永久停摆于一次自增回滚留下的空位。删除 InboxEnqueue（改由
insertIfAbsent 幂等入队）与 poller 内的回填扫描（消除 ingress→jobs 反向依赖）。

infra/persistence：端口按事实重画为 readRange/maxId/markProcessedIfUnmarked，标记 UPDATE
带 DATE_PROCESSED IS NULL 守卫，只把空标写为已处理（§11 单调，重复执行无副作用）。
回填事实并入 PROC_STATE（RECEIVED_AT/BACKFILL_AT/NEXT_AT/ATTEMPTS/ERROR），BACKFILL_TODO
随 V2 迁移下线；终态与回填意图是同一条 UPDATE，由处理器在自己的业务事务内落库，
message-lifecycle §4 登记的两个崩溃窗口（提交后回填前崩溃、待办二次落账失败）不再是缺口。

processing/BackfillService.kt（取代 BackfillSweepJob）：终态提交后立即尝试一次，失败按
30s→15min 指数退避重试；扫描条件「终态 + 未确认标记 +（已到期 或 接收时间早于 NOW − R）」
使 §5.2 的超期期限 R 覆盖退避，中间态永不补写。死信同样可补写——回填只需消息 ID，
不再依赖 META。Pump 改用可注入 Clock。

infra/health：InboxLifecycleHealthIndicator 输出积压条数、最老未处理信龄、未回填终态数与
水位滞后（OPS-2 / §5.3 验收）。预计消化时长需吞吐采样，留待接入指标注册表时补。

配置：pipeline.max-commit-delay / overdue-backfill / backfill-batch、mailbox.processed-value
（Q2/Q6/Q7 未书面确认前取保守初值，不得为提速下调）。

不变量回归测试：死信不阻断后续发现、水位遇空洞即停与老化放行、终态+意图同事务、
超期 R 覆盖退避、中间态不补写、标记单调；InboxLifecycleJdbcSqlTest 以 H2 的 PostgreSQL
兼容模式直连验证上述 SQL 语义（不依赖 docker）。libs.h2 由 testRuntimeOnly 提为
testImplementation 以支持该用例。

验证：gradle clean test --offline → 78 tests / 0 failures / 1 skipped
（PG Testcontainers 集成用例在本机无 docker 时按既有约定 assumeTrue 跳过）。
This commit is contained in:
windyboy
2026-09-10 10:59:58 +08:00
parent ffd3abd655
commit 09b53c77bb
32 changed files with 1402 additions and 548 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/kotlin/com/gzzn/omms/msgexchange/processing/Pump.kt
+57 -86
View File
@@ -10,25 +10,28 @@ import com.gzzn.omms.msgexchange.domain.MsgKind
import com.gzzn.omms.msgexchange.domain.ProcState
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.infra.log.TraceLog
import com.gzzn.omms.msgexchange.infra.persistence.BackfillTodoRepository
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import com.gzzn.omms.msgexchange.infra.retry.ProcFailure
import jakarta.inject.Singleton
import java.time.Clock
import java.time.Duration
import java.time.Instant
/**
* 处理主泵（docs/flight-state.md §1 严格有序 + §4 处理事务与失败规则）：
* 单活动主泵严格 FIFO + HOL 阻塞 + 队头滞留转 DEAD；航班状态、事件、处理终态
* 与回填待办在处理器事务内原子提交。
* 单活动主泵严格 FIFO + HOL 阻塞 + 队头滞留转 DEAD。
*
* 终态与业务写入在处理器事务内原子提交；本泵只负责调度、边界化失败迁移与
* 提交后的最佳努力回填（message-lifecycle §3/§4）。
*/
@Singleton
class Pump(
private val procState: ProcStateRepository,
private val inbox: CminmsgInboxRepository,
private val processor: MessageProcessor,
private val backfill: BackfillService,
private val props: PipelineProps,
private val clock: Clock,
) {
private val log = org.slf4j.LoggerFactory.getLogger(Pump::class.java)
@@ -58,15 +61,18 @@ class Pump(
val head = procState.headUnfinished()
when {
head == null -> sleepQuietly(props.pipeline.pollInterval)
head.state == ProcStatus.FAILED && (head.nextAttemptAt ?: Instant.EPOCH) > Instant.now() ->
head.state == ProcStatus.FAILED && (head.nextAttemptAt ?: Instant.EPOCH) > clock.instant() ->
if (poisoned(head)) {
log.error("poison -> DEAD msgId={} attempts={} lastError={}", head.msgId, head.attempts, head.lastError)
procState.update(
procState.markTerminal(
head.msgId, ProcStatus.DEAD,
errorClass = ErrorClass.EXHAUSTED, lastError = head.lastError ?: "head-deadline-exceeded",
errorClass = ErrorClass.EXHAUSTED,
lastError = head.lastError ?: "head-deadline-exceeded",
attempts = head.attempts,
)
backfill.attempt(head.msgId)
} else {
sleepQuietly(Duration.between(Instant.now(), head.nextAttemptAt))
sleepQuietly(Duration.between(clock.instant(), head.nextAttemptAt))
}
// PENDING、或 FAILED 退避已到期：交处理入口（内部有边界化失败迁移与 attempts 守卫）
else -> processor.processOne(head)
@@ -75,7 +81,7 @@ class Pump(
private fun poisoned(head: ProcState): Boolean =
head.attempts >= props.pipeline.maxAttempts ||
Duration.between(head.updatedAt, Instant.now()) > props.pipeline.headDeadline
Duration.between(head.updatedAt, clock.instant()) > props.pipeline.headDeadline
private fun sleepQuietly(d: Duration) {
if (!d.isNegative && !d.isZero) Thread.sleep(d.toMillis().coerceAtLeast(1))
@@ -83,7 +89,7 @@ class Pump(
}
/**
* processOne：解码 → 绑定 → 处理器（事务内决策+落库）→ 终态迁移 → 回填。
* processOne：解码 → 绑定 → 处理器（事务内决策 + 落库 + 终态 + 回填意图）→ 提交后最佳努力回填。
* 边界化失败迁移（ProcFailure）：任何意外异常归于本条 head，FAILED(INFRA)+退避，不穿出杀泵；
* MALFORMED / PROTOCOL 直接 DEAD 不重试（docs/design.md §2.3 错误分类）。
*/
@@ -97,14 +103,14 @@ class MessageProcessor(
private val fdelProcessor: FdelProcessor,
private val adftProcessor: AdftProcessor,
private val procFailure: ProcFailure,
private val backfill: BackfillService,
private val props: PipelineProps,
private val backfillTodo: BackfillTodoRepository? = null,
) {
private val log = org.slf4j.LoggerFactory.getLogger(MessageProcessor::class.java)
fun processOne(head: ProcState) {
TraceLog.withTrace(head.msgId) {
try {
val terminal = try {
processInternal(head)
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
@@ -114,21 +120,30 @@ class MessageProcessor(
log.warn("processOne unexpected failure msgId={} ec=INFRA msg={}", head.msgId, e.message ?: e.javaClass.simpleName)
procFailure.fail(head, ErrorClass.INFRA, e.message ?: e.javaClass.simpleName)
}
// 终态已提交：立即尝试一次回填；失败留待回填扫描按退避重试（意图已在终态事务内登记）。
// 中间态（PENDING/FAILED）不适用（message-lifecycle §5.2）。
if (terminal) backfill.attempt(head.msgId)
}
}
private fun processInternal(head: ProcState) {
/** @return 是否已达终态（终态才允许回填信箱标记） */
private fun processInternal(head: ProcState): Boolean {
// 守卫：手工/遗留 FAILED 行若 attempts 已达上限，直接终态（防止退避到期后无限重试）
if (head.state == ProcStatus.FAILED && procFailure.scheduler.exhausted(head.attempts)) {
log.error("head exhausted at entry -> DEAD msgId={} attempts={}", head.msgId, head.attempts)
procState.update(head.msgId, ProcStatus.DEAD, errorClass = ErrorClass.EXHAUSTED, lastError = head.lastError ?: "max-attempts")
return
procState.markTerminal(
head.msgId, ProcStatus.DEAD,
errorClass = ErrorClass.EXHAUSTED,
lastError = head.lastError ?: "max-attempts",
attempts = head.attempts,
)
return true
}
val raw = inbox.rawOf(head.msgId) ?: run {
val raw = inbox.rawOf(head.msgId)
if (raw == null) {
log.error("raw missing -> DEAD(MALFORMED) msgId={}", head.msgId)
procState.update(head.msgId, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, lastError = "raw-missing")
return
return deadMalformed(head, "raw-missing")
}
val decoded = when (val r = codec.decode(raw)) {
is com.gzzn.omms.msgexchange.codec.DecodeResult.Ok -> r.message
@@ -136,12 +151,10 @@ class MessageProcessor(
// MALFORMED（报文非法）→ DEAD 不重试；CODEC_ERROR（可随 codec 修复重放）→ FAILED 退避
if (r.failure.errorClass == ErrorClass.MALFORMED) {
log.error("decode MALFORMED -> DEAD msgId={} detail={}", head.msgId, r.failure.detail)
procState.update(head.msgId, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, lastError = r.failure.detail)
} else {
log.warn("decode {} -> FAILED msgId={} detail={}", r.failure.errorClass, head.msgId, r.failure.detail)
procFailure.fail(head, r.failure.errorClass, r.failure.detail)
return deadMalformed(head, r.failure.detail)
}
return
log.warn("decode {} -> FAILED msgId={} detail={}", r.failure.errorClass, head.msgId, r.failure.detail)
return procFailure.fail(head, r.failure.errorClass, r.failure.detail)
}
}
@@ -151,9 +164,8 @@ class MessageProcessor(
if (!procState.tryBindIdentity(head.msgId, identity)) {
val owner = procState.ownerOfIdentity(identity) ?: -1L
log.info("duplicate-of:{} -> SKIPPED msgId={}", owner, head.msgId)
procState.update(head.msgId, ProcStatus.SKIPPED, lastError = "duplicate-of:$owner")
compensateBackfill(head, decoded)
return
procState.markTerminal(head.msgId, ProcStatus.SKIPPED, lastError = "duplicate-of:$owner")
return true
}
}
@@ -161,93 +173,52 @@ class MessageProcessor(
val result: ApplyResult = when (val kind = decoded.kind) {
is MsgKind.Schd -> {
val body = decoded.body as? ScheduleBody
if (body == null) {
deadMalformed(head, "missing-schd-body")
return
}
if (body == null) return deadMalformed(head, "missing-schd-body")
when (kind.subtype) {
MsgKind.SchdSubtype.DNLD, MsgKind.SchdSubtype.RESP ->
scheduleProcessor.applyScheduleRecords(head, decoded)
MsgKind.SchdSubtype.ADFT -> {
val record = body.records.singleOrNull()
if (record == null) {
deadMalformed(head, "adft-needs-single-fltr")
return
}
if (record == null) return deadMalformed(head, "adft-needs-single-fltr")
adftProcessor.apply(head, decoded, record)
}
}
}
MsgKind.Fdel -> {
val payload = decoded.body as? FlopPayload
if (payload == null) {
deadMalformed(head, "missing-fdel-flid")
return
}
if (payload == null) return deadMalformed(head, "missing-fdel-flid")
fdelProcessor.apply(head, decoded, payload)
}
is MsgKind.Flop -> {
val payload = decoded.body as? FlopPayload
if (payload == null) {
deadMalformed(head, "missing-flop-body")
return
}
if (payload == null) return deadMalformed(head, "missing-flop-body")
flopProcessor.apply(head, decoded, payload)
}
is MsgKind.Unsupported -> {
// design.md §2.3：未支持类型 → FAILED(UNSUPPORTED) 退避重试，达阈值转 DEAD；绝不写终态
log.warn("unsupported type -> FAILED(UNSUPPORTED) msgId={} tag={}", head.msgId, kind.tag)
procFailure.fail(head, ErrorClass.UNSUPPORTED, "no-handler:${kind.tag}")
return
return procFailure.fail(head, ErrorClass.UNSUPPORTED, "no-handler:${kind.tag}")
}
}
when (result) {
is ApplyResult.Succeeded, ApplyResult.ReplaySkipped ->
procState.update(head.msgId, ProcStatus.SUCCEEDED)
is ApplyResult.DeadProtocol -> {
// design.md §2.3：整包拒绝 DEAD(PROTOCOL)，立即释放队头，交人工确认
log.error("DEAD(PROTOCOL) msgId={} reason={} flags={}", head.msgId, result.reason, result.flags)
procState.update(head.msgId, ProcStatus.DEAD, errorClass = ErrorClass.PROTOCOL, lastError = result.reason.take(1000))
compensateBackfill(head, decoded) // 拒绝包同样要回填信箱，防止反复轮询
return
}
if (result is ApplyResult.DeadProtocol) {
// design.md §2.3：整包拒绝 DEAD(PROTOCOL)，立即释放队头，交人工确认
log.error("DEAD(PROTOCOL) msgId={} reason={} flags={}", head.msgId, result.reason, result.flags)
procState.markTerminal(
head.msgId, ProcStatus.DEAD,
errorClass = ErrorClass.PROTOCOL,
lastError = result.reason.take(1000),
)
return true
}
backfill(head, decoded)
// Succeeded / ReplaySkipped：SUCCEEDED 终态与回填意图已由处理器在自己的事务内落库
log.info("SUCCEEDED msgId={} kind={}", head.msgId, decoded.typeTag)
return true
}
/** design.md §3.3/§6.1：提交后回填共享信箱；失败不得把 SUCCEEDED 改回 FAILED，待办已事务内预登记。 */
private fun backfill(head: ProcState, decoded: DecodedMessage) {
try {
inbox.backfillOnSuccess(head.msgId, decoded.meta.sndr, decoded.meta.type, decoded.meta.styp, decoded.meta.seqn)
backfillTodo?.delete(head.msgId)
} catch (e: Exception) {
log.error("backfill failed after SUCCEEDED msgId={} (todo pre-registered, sweep will retry)", head.msgId, e)
}
}
/** SKIPPED/DEAD(PROTOCOL) 包的回填（无预登记待办）：失败落补偿待办。 */
private fun compensateBackfill(head: ProcState, decoded: DecodedMessage) {
try {
inbox.backfillOnSuccess(head.msgId, decoded.meta.sndr, decoded.meta.type, decoded.meta.styp, decoded.meta.seqn)
} catch (e: Exception) {
log.error("backfill failed msgId={} (compensation required)", head.msgId, e)
backfillTodo?.record(
BackfillTodoRepository.BackfillTask(
msgId = head.msgId,
sndr = decoded.meta.sndr,
type = decoded.meta.type,
styp = decoded.meta.styp,
seqn = decoded.meta.seqn,
),
e.message ?: e.javaClass.simpleName,
) ?: log.warn("no backfill-todo repository bound; compensation NOT persisted msgId={}", head.msgId)
}
}
private fun deadMalformed(head: ProcState, detail: String) {
procState.update(head.msgId, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, lastError = detail)
private fun deadMalformed(head: ProcState, detail: String): Boolean {
procState.markTerminal(head.msgId, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, lastError = detail)
return true
}
}