fix(processing): 收紧消息生命周期与投递约束

This commit is contained in:
windyboy
2026-09-10 20:39:22 +08:00
parent dd839e6abb
commit eea11203a2
27 changed files with 502 additions and 79 deletions
@@ -2,7 +2,9 @@ package com.gzzn.omms.msgexchange.processing
import com.gzzn.omms.msgexchange.config.MailboxProps
import com.gzzn.omms.msgexchange.config.PipelineProps
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
import com.gzzn.omms.msgexchange.infra.persistence.MailboxMarkResult
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import jakarta.inject.Singleton
import java.time.Clock
@@ -32,12 +34,14 @@ class BackfillService(
private val mailboxProps: MailboxProps,
private val props: PipelineProps,
private val clock: Clock,
private val lifecycleGate: MessageLifecycleGate = MessageLifecycleGate(),
) {
private val log = org.slf4j.LoggerFactory.getLogger(BackfillService::class.java)
companion object {
private val INITIAL_BACKOFF: Duration = Duration.ofSeconds(30)
private val MAX_BACKOFF: Duration = Duration.ofMinutes(15)
private val TERMINAL_STATES = setOf(ProcStatus.SUCCEEDED, ProcStatus.SKIPPED, ProcStatus.DEAD)
fun backoffDelayFor(attempts: Int): Duration {
val shift = (attempts - 1).coerceIn(0, 20)
@@ -50,8 +54,13 @@ class BackfillService(
* 调用方是主泵的处理路径,不能被回填问题拖住。
*/
fun attempt(msgId: Long, now: Instant = clock.instant()) {
record(msgId, attempts = 0, now = now)?.let {
log.warn("backfill failed msgId={} error={} (sweep will retry)", msgId, it)
lifecycleGate.exclusive {
val row = procState.find(msgId) ?: return@exclusive
if (row.state !in TERMINAL_STATES) return@exclusive
if (row.backfillAt != null) return@exclusive
record(msgId, attempts = row.backfillAttempts, now = now)?.let {
log.warn("backfill failed msgId={} error={} (sweep will retry)", msgId, it)
}
}
}
@@ -62,7 +71,7 @@ class BackfillService(
*/
fun sweep(now: Instant = clock.instant()): Int {
val due = procState.findBackfillDue(now, now.minus(props.pipeline.overdueBackfill), props.pipeline.backfillBatch)
due.forEach { record(it.msgId, it.attempts, now) }
due.forEach { attempt(it.msgId, now) }
return due.size
}
@@ -70,7 +79,8 @@ class BackfillService(
private fun record(msgId: Long, attempts: Int, now: Instant): String? =
try {
// 没有真正写进去说明库里已经有标记了,同样算成功(不覆盖已有值)
mailbox.markProcessedIfUnmarked(msgId, mailboxProps.processedValue)
val result = mailbox.markProcessedIfUnmarked(msgId, mailboxProps.processedValue)
check(result != MailboxMarkResult.MISSING) { "mailbox-row-missing" }
procState.markBackfilled(msgId, now)
null
} catch (e: Exception) {
@@ -80,4 +90,5 @@ class BackfillService(
}.onFailure { log.error("record backfill failure failed msgId={}", msgId, it) }
reason
}
}
@@ -0,0 +1,17 @@
package com.gzzn.omms.msgexchange.processing
import jakarta.inject.Singleton
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
/**
* 串行化“终态回填”与“人工重放”,避免旧回填任务在消息重新入队后写入信箱标记。
* 生产只允许单活动实例;该锁处理同一实例内的 job 线程和管理请求并发。
*/
@Singleton
class MessageLifecycleGate {
private val lock = ReentrantLock()
fun <T> exclusive(block: () -> T): T = lock.withLock(block)
}
@@ -66,33 +66,39 @@ class Pump(
val head = procState.headUnfinished()
when {
head == null -> sleepQuietly(props.pipeline.pollInterval)
head.state == ProcStatus.FAILED && poisoned(head) -> {
log.error("poison -> DEAD msgId={} attempts={} lastError={}", head.msgId, head.attempts, head.lastError)
procState.markTerminal(
head.msgId, ProcStatus.DEAD,
errorClass = ErrorClass.EXHAUSTED,
lastError = head.lastError ?: "head-deadline-exceeded",
attempts = head.attempts,
now = clock.instant(),
)
backfill.attempt(head.msgId)
}
head.state == ProcStatus.FAILED && (head.nextAttemptAt ?: Instant.EPOCH) > clock.instant() ->
if (poisoned(head)) {
log.error("poison -> DEAD msgId={} attempts={} lastError={}", head.msgId, head.attempts, head.lastError)
procState.markTerminal(
head.msgId, ProcStatus.DEAD,
errorClass = ErrorClass.EXHAUSTED,
lastError = head.lastError ?: "head-deadline-exceeded",
attempts = head.attempts,
)
backfill.attempt(head.msgId)
} else {
sleepQuietly(Duration.between(clock.instant(), head.nextAttemptAt))
}
sleepQuietly(Duration.between(clock.instant(), head.nextAttemptAt))
// 其余情况(新消息,或退避到期的重试)交给处理入口
else -> processor.processOne(head)
else -> {
procState.markProcessingStartedIfAbsent(head.msgId, clock.instant())
processor.processOne(head)
}
}
}
private fun poisoned(head: ProcState): Boolean =
head.attempts >= props.pipeline.maxAttempts ||
Duration.between(head.updatedAt, clock.instant()) > props.pipeline.headDeadline
isHeadPoisoned(head, clock.instant(), props)
private fun sleepQuietly(d: Duration) {
if (!d.isNegative && !d.isZero) Thread.sleep(d.toMillis().coerceAtLeast(1))
}
}
internal fun isHeadPoisoned(head: ProcState, now: Instant, props: PipelineProps): Boolean =
head.attempts >= props.pipeline.maxAttempts ||
Duration.between(head.processingStartedAt ?: head.updatedAt, now) >= props.pipeline.headDeadline
/**
* 处理一条消息:读原文 → 解码 → 绑定业务身份 → 分派给对应处理器 → 提交后回填标记。
*
@@ -95,7 +95,11 @@ class ScheduleProcessor(
val ok = validation as SnapshotValidation.Ok
if (ok.perRecordDay.isEmpty()) {
// 报文合法但没有记录:不需要写数据,照样算处理成功
// 报文合法但没有记录:不写航班,但终态与回填意图仍在锁事务内一起提交
txManager.inTransaction {
lock.lock()
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED)
}
logSnapshot(head, body, SnapshotResult.COMMITTED, upserted = 0, setOf(SnapshotFlag.EMPTY), started)
return ApplyResult.Succeeded
}