fix(processing): 收紧消息生命周期与投递约束
This commit is contained in:
@@ -2,7 +2,9 @@ package com.gzzn.omms.msgexchange.processing
|
||||
|
||||
import com.gzzn.omms.msgexchange.config.MailboxProps
|
||||
import com.gzzn.omms.msgexchange.config.PipelineProps
|
||||
import com.gzzn.omms.msgexchange.domain.ProcStatus
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.MailboxMarkResult
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
|
||||
import jakarta.inject.Singleton
|
||||
import java.time.Clock
|
||||
@@ -32,12 +34,14 @@ class BackfillService(
|
||||
private val mailboxProps: MailboxProps,
|
||||
private val props: PipelineProps,
|
||||
private val clock: Clock,
|
||||
private val lifecycleGate: MessageLifecycleGate = MessageLifecycleGate(),
|
||||
) {
|
||||
private val log = org.slf4j.LoggerFactory.getLogger(BackfillService::class.java)
|
||||
|
||||
companion object {
|
||||
private val INITIAL_BACKOFF: Duration = Duration.ofSeconds(30)
|
||||
private val MAX_BACKOFF: Duration = Duration.ofMinutes(15)
|
||||
private val TERMINAL_STATES = setOf(ProcStatus.SUCCEEDED, ProcStatus.SKIPPED, ProcStatus.DEAD)
|
||||
|
||||
fun backoffDelayFor(attempts: Int): Duration {
|
||||
val shift = (attempts - 1).coerceIn(0, 20)
|
||||
@@ -50,8 +54,13 @@ class BackfillService(
|
||||
* 调用方是主泵的处理路径,不能被回填问题拖住。
|
||||
*/
|
||||
fun attempt(msgId: Long, now: Instant = clock.instant()) {
|
||||
record(msgId, attempts = 0, now = now)?.let {
|
||||
log.warn("backfill failed msgId={} error={} (sweep will retry)", msgId, it)
|
||||
lifecycleGate.exclusive {
|
||||
val row = procState.find(msgId) ?: return@exclusive
|
||||
if (row.state !in TERMINAL_STATES) return@exclusive
|
||||
if (row.backfillAt != null) return@exclusive
|
||||
record(msgId, attempts = row.backfillAttempts, now = now)?.let {
|
||||
log.warn("backfill failed msgId={} error={} (sweep will retry)", msgId, it)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -62,7 +71,7 @@ class BackfillService(
|
||||
*/
|
||||
fun sweep(now: Instant = clock.instant()): Int {
|
||||
val due = procState.findBackfillDue(now, now.minus(props.pipeline.overdueBackfill), props.pipeline.backfillBatch)
|
||||
due.forEach { record(it.msgId, it.attempts, now) }
|
||||
due.forEach { attempt(it.msgId, now) }
|
||||
return due.size
|
||||
}
|
||||
|
||||
@@ -70,7 +79,8 @@ class BackfillService(
|
||||
private fun record(msgId: Long, attempts: Int, now: Instant): String? =
|
||||
try {
|
||||
// 没有真正写进去说明库里已经有标记了,同样算成功(不覆盖已有值)
|
||||
mailbox.markProcessedIfUnmarked(msgId, mailboxProps.processedValue)
|
||||
val result = mailbox.markProcessedIfUnmarked(msgId, mailboxProps.processedValue)
|
||||
check(result != MailboxMarkResult.MISSING) { "mailbox-row-missing" }
|
||||
procState.markBackfilled(msgId, now)
|
||||
null
|
||||
} catch (e: Exception) {
|
||||
@@ -80,4 +90,5 @@ class BackfillService(
|
||||
}.onFailure { log.error("record backfill failure failed msgId={}", msgId, it) }
|
||||
reason
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.gzzn.omms.msgexchange.processing
|
||||
|
||||
import jakarta.inject.Singleton
|
||||
import java.util.concurrent.locks.ReentrantLock
|
||||
import kotlin.concurrent.withLock
|
||||
|
||||
/**
|
||||
* 串行化“终态回填”与“人工重放”,避免旧回填任务在消息重新入队后写入信箱标记。
|
||||
* 生产只允许单活动实例;该锁处理同一实例内的 job 线程和管理请求并发。
|
||||
*/
|
||||
@Singleton
|
||||
class MessageLifecycleGate {
|
||||
private val lock = ReentrantLock()
|
||||
|
||||
fun <T> exclusive(block: () -> T): T = lock.withLock(block)
|
||||
}
|
||||
|
||||
@@ -66,33 +66,39 @@ class Pump(
|
||||
val head = procState.headUnfinished()
|
||||
when {
|
||||
head == null -> sleepQuietly(props.pipeline.pollInterval)
|
||||
head.state == ProcStatus.FAILED && poisoned(head) -> {
|
||||
log.error("poison -> DEAD msgId={} attempts={} lastError={}", head.msgId, head.attempts, head.lastError)
|
||||
procState.markTerminal(
|
||||
head.msgId, ProcStatus.DEAD,
|
||||
errorClass = ErrorClass.EXHAUSTED,
|
||||
lastError = head.lastError ?: "head-deadline-exceeded",
|
||||
attempts = head.attempts,
|
||||
now = clock.instant(),
|
||||
)
|
||||
backfill.attempt(head.msgId)
|
||||
}
|
||||
head.state == ProcStatus.FAILED && (head.nextAttemptAt ?: Instant.EPOCH) > clock.instant() ->
|
||||
if (poisoned(head)) {
|
||||
log.error("poison -> DEAD msgId={} attempts={} lastError={}", head.msgId, head.attempts, head.lastError)
|
||||
procState.markTerminal(
|
||||
head.msgId, ProcStatus.DEAD,
|
||||
errorClass = ErrorClass.EXHAUSTED,
|
||||
lastError = head.lastError ?: "head-deadline-exceeded",
|
||||
attempts = head.attempts,
|
||||
)
|
||||
backfill.attempt(head.msgId)
|
||||
} else {
|
||||
sleepQuietly(Duration.between(clock.instant(), head.nextAttemptAt))
|
||||
}
|
||||
sleepQuietly(Duration.between(clock.instant(), head.nextAttemptAt))
|
||||
// 其余情况(新消息,或退避到期的重试)交给处理入口
|
||||
else -> processor.processOne(head)
|
||||
else -> {
|
||||
procState.markProcessingStartedIfAbsent(head.msgId, clock.instant())
|
||||
processor.processOne(head)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun poisoned(head: ProcState): Boolean =
|
||||
head.attempts >= props.pipeline.maxAttempts ||
|
||||
Duration.between(head.updatedAt, clock.instant()) > props.pipeline.headDeadline
|
||||
isHeadPoisoned(head, clock.instant(), props)
|
||||
|
||||
private fun sleepQuietly(d: Duration) {
|
||||
if (!d.isNegative && !d.isZero) Thread.sleep(d.toMillis().coerceAtLeast(1))
|
||||
}
|
||||
}
|
||||
|
||||
internal fun isHeadPoisoned(head: ProcState, now: Instant, props: PipelineProps): Boolean =
|
||||
head.attempts >= props.pipeline.maxAttempts ||
|
||||
Duration.between(head.processingStartedAt ?: head.updatedAt, now) >= props.pipeline.headDeadline
|
||||
|
||||
/**
|
||||
* 处理一条消息:读原文 → 解码 → 绑定业务身份 → 分派给对应处理器 → 提交后回填标记。
|
||||
*
|
||||
|
||||
@@ -95,7 +95,11 @@ class ScheduleProcessor(
|
||||
val ok = validation as SnapshotValidation.Ok
|
||||
|
||||
if (ok.perRecordDay.isEmpty()) {
|
||||
// 报文合法但没有记录:不需要写数据,照样算处理成功
|
||||
// 报文合法但没有记录:不写航班,但终态与回填意图仍在锁事务内一起提交
|
||||
txManager.inTransaction {
|
||||
lock.lock()
|
||||
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED)
|
||||
}
|
||||
logSnapshot(head, body, SnapshotResult.COMMITTED, upserted = 0, setOf(SnapshotFlag.EMPTY), started)
|
||||
return ApplyResult.Succeeded
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user