cdia/msgexchange-v2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(processing): 收紧消息生命周期与投递约束

Browse Source
This commit is contained in:
windyboy
2026-09-10 20:39:22 +08:00
parent dd839e6abb
commit eea11203a2
27 changed files with 502 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/kotlin/com/gzzn/omms/msgexchange/delivery/Dispatcher.kt
+3 -3
View File
@@ -79,7 +79,7 @@ class Dispatcher(
}
private fun flushDue(): Boolean =
lastFlush?.let { Duration.between(it, scheduler.now()) >= props.schd.flushPeriod } ?: false
lastFlush?.let { Duration.between(it, scheduler.now()) >= props.schd.flushPeriod } ?: true
private fun deliver(target: String, e: MsgEvent) {
try {
@@ -95,7 +95,7 @@ class Dispatcher(
/** 批量发 KAFKA_SCHD：每个 FLID 只发版本号最新的那条未发事件，删除通知发空 value。 */
internal fun flushSchd() {
val batch = try {
msgEvents.mergePendingSchd(props.schd.flushLimit)
msgEvents.mergePendingSchd(scheduler.now(), props.schd.flushLimit)
} catch (e: Exception) {
log.warn("mergePendingSchd failed: {}", e.message)
return
@@ -121,7 +121,7 @@ class Dispatcher(
val sentVersions = batch.associate { it.partitionKey to it.stateVersion }
runCatching {
while (true) {
val superseded = msgEvents.mergePendingSchd(props.schd.flushLimit)
val superseded = msgEvents.mergePendingSchd(scheduler.now(), props.schd.flushLimit)
.filter { sentVersions[it.partitionKey]?.let { v -> it.stateVersion < v } == true }
if (superseded.isEmpty()) break
msgEvents.markAllSent(superseded.mapNotNull { it.eventId })
src/main/kotlin/com/gzzn/omms/msgexchange/domain/ProcState.kt
+2
View File
@@ -72,5 +72,7 @@ data class ProcState(
val backfillNextAt: Instant? = null,
val backfillAttempts: Int = 0,
val backfillError: String? = null,
/** 首次被主泵取得的时刻；重试不刷新，用作 HOL deadline 的稳定起点。 */
val processingStartedAt: Instant? = null,
val updatedAt: Instant = Instant.now(),
)
src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/Repositories.kt
+12 -3
View File
@@ -58,6 +58,9 @@ interface ProcStateRepository {
/** 当前队头：还没处理完的消息里 ID 最小的那条。 */
fun headUnfinished(): ProcState?
/** 首次开始处理时记下稳定起点；重试不覆盖。 */
fun markProcessingStartedIfAbsent(msgId: Long, now: Instant)
/** 给消息绑定业务身份；返回 false 表示这个身份已经被另一条消息占了（业务重复）。 */
fun tryBindIdentity(msgId: Long, identityKey: String): Boolean
@@ -135,7 +138,7 @@ interface MsgEventRepository {
fun claimBatch(target: String, limit: Int): List<MsgEvent>
/** 挑出待发的整态事件，同一条航班只取版本最高的那一条（中间的版本不用发）。 */
fun mergePendingSchd(limit: Int): List<MsgEvent>
fun mergePendingSchd(now: Instant, limit: Int): List<MsgEvent>
fun markSent(eventId: Long)
@@ -287,6 +290,9 @@ interface CminmsgInboxRepository {
/** 读某条消息的原文；返回 null 表示读不到（行已被清除，或原文本身为空）。 */
fun rawOf(msgId: Long): String?
/** 读取信箱库记录的实际接收时间。 */
fun receivedAtOf(msgId: Long): Instant?
/** 按 ID 升序读一批 `ID > fromExclusive` 的行，不带别的过滤条件。 */
fun readRange(fromExclusive: Long, limit: Int): List<MailboxRow>
@@ -295,10 +301,13 @@ interface CminmsgInboxRepository {
/**
* 把处理标记写回信箱，并且**只写还是空标记的行**：库里已有值时不覆盖、不回退，
* 重复调用没有副作用。返回 true 表示这次真的写进去了。
* 重复调用没有副作用。结果明确区分本次写入、已有标记和信箱行缺失。
*/
fun markProcessedIfUnmarked(msgId: Long, value: String): Boolean
fun markProcessedIfUnmarked(msgId: Long, value: String): MailboxMarkResult
}
/** 信箱回填的可区分结果；行缺失不能冒充“已有标记”。 */
enum class MailboxMarkResult { MARKED, ALREADY_MARKED, MISSING }
/** 从信箱读到的一行：ID 加接收时间。原文按需再取，扫描时不读大字段。 */
data class MailboxRow(val msgId: Long, val receivedAt: Instant?)
src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/JdbcCminmsgInboxRepository.kt
+19 -4
View File
@@ -2,10 +2,12 @@ package com.gzzn.omms.msgexchange.infra.persistence.jdbc
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
import com.gzzn.omms.msgexchange.infra.persistence.MailboxRow
import com.gzzn.omms.msgexchange.infra.persistence.MailboxMarkResult
import io.micronaut.context.annotation.Requires
import jakarta.inject.Named
import jakarta.inject.Singleton
import javax.sql.DataSource
import java.time.Instant
/**
* 共享 MySQL CMINMSGS 信箱的 JDBC 实现。列名沿用 legacy 的 `CMINMSGS_*`。
@@ -39,6 +41,12 @@ class JdbcCminmsgInboxRepository(
{ ps -> ps.setLong(1, msgId) },
) { rs -> rs.getString("CMINMSGS_CLOB_MSG") }
override fun receivedAtOf(msgId: Long): Instant? =
ds.queryOne(
"SELECT CMINMSGS_DATE_RECEIVED FROM cminmsgs WHERE CMINMSGS_ID = ?",
{ ps -> ps.setLong(1, msgId) },
) { rs -> rs.getTimestamp("CMINMSGS_DATE_RECEIVED")?.toInstant() }
override fun readRange(fromExclusive: Long, limit: Int): List<MailboxRow> =
ds.query(
"""
@@ -65,11 +73,11 @@ class JdbcCminmsgInboxRepository(
/**
* 只更新还是空标记的行，所以重复调用不会覆盖库里已有的值；
* 影响 0 行说明已经被标记过了，调用方按"成功"处理即可。
* 影响 0 行时再查一次主键，区分“已有标记”和“信箱行缺失”；后者不能记为回填成功。
* 具体写什么值、时间怎么解释，以与库方约定为准。
*/
override fun markProcessedIfUnmarked(msgId: Long, value: String): Boolean =
ds.update(
override fun markProcessedIfUnmarked(msgId: Long, value: String): MailboxMarkResult {
val updated = ds.update(
"""
UPDATE cminmsgs
SET CMINMSGS_DATE_PROCESSED = CURRENT_TIMESTAMP,
@@ -79,5 +87,12 @@ class JdbcCminmsgInboxRepository(
) { ps ->
ps.setString(1, value)
ps.setLong(2, msgId)
} == 1
}
if (updated == 1) return MailboxMarkResult.MARKED
val exists = ds.queryOne(
"SELECT 1 FROM cminmsgs WHERE CMINMSGS_ID = ?",
{ ps -> ps.setLong(1, msgId) },
) { 1 } != null
return if (exists) MailboxMarkResult.ALREADY_MARKED else MailboxMarkResult.MISSING
}
}
src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/JdbcOps.kt
+16 -10
View File
@@ -10,17 +10,20 @@ internal fun Instant.toSqlTimestamp(): Timestamp = Timestamp.from(this)
internal fun ResultSet.getInstant(column: String): Instant? =
getTimestamp(column)?.toInstant()
private val transactionConnection = ThreadLocal<java.sql.Connection?>()
private val transactionConnections = ThreadLocal.withInitial {
java.util.IdentityHashMap<DataSource, java.sql.Connection>()
}
internal fun <T> DataSource.withTransaction(block: () -> T): T {
val existing = transactionConnection.get()
val connections = transactionConnections.get()
val existing = connections[this]
if (existing != null) {
return block()
}
val conn = this.connection
val oldAutoCommit = conn.autoCommit
conn.autoCommit = false
transactionConnection.set(conn)
connections[this] = conn
try {
val result = block()
conn.commit()
@@ -33,7 +36,8 @@ internal fun <T> DataSource.withTransaction(block: () -> T): T {
}
throw t
} finally {
transactionConnection.remove()
connections.remove(this)
if (connections.isEmpty()) transactionConnections.remove()
try {
conn.autoCommit = oldAutoCommit
} catch (_: Throwable) {
@@ -43,11 +47,13 @@ internal fun <T> DataSource.withTransaction(block: () -> T): T {
}
internal fun DataSource.obtainConnection(): java.sql.Connection =
transactionConnection.get() ?: this.connection
transactionConnections.get()[this] ?: this.connection
internal fun java.sql.Connection.releaseIfNotInTransaction() {
if (transactionConnection.get() !== this) {
internal fun java.sql.Connection.releaseIfNotInTransaction(dataSource: DataSource) {
val connections = transactionConnections.get()
if (connections[dataSource] !== this) {
this.close()
if (connections.isEmpty()) transactionConnections.remove()
}
}
@@ -63,7 +69,7 @@ internal fun <T> DataSource.query(sql: String, bind: (java.sql.PreparedStatement
}
}
} finally {
conn.releaseIfNotInTransaction()
conn.releaseIfNotInTransaction(this)
}
}
@@ -78,7 +84,7 @@ internal fun DataSource.update(sql: String, bind: (java.sql.PreparedStatement) -
ps.executeUpdate()
}
} finally {
conn.releaseIfNotInTransaction()
conn.releaseIfNotInTransaction(this)
}
}
@@ -94,6 +100,6 @@ internal fun DataSource.updateReturningLong(sql: String, bind: (java.sql.Prepare
}
}
} finally {
conn.releaseIfNotInTransaction()
conn.releaseIfNotInTransaction(this)
}
}
src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/JdbcPgRepositories.kt
+21 -7
View File
@@ -99,6 +99,16 @@ class JdbcProcStateRepository(
::mapProcState,
)
override fun markProcessingStartedIfAbsent(msgId: Long, now: Instant) {
ds.update(
"UPDATE proc_state SET processing_started_at = ? WHERE msg_id = ? AND processing_started_at IS NULL",
{ ps ->
ps.setTimestamp(1, now.toSqlTimestamp())
ps.setLong(2, msgId)
},
)
}
override fun tryBindIdentity(msgId: Long, identityKey: String): Boolean {
ownerOfIdentity(identityKey)?.let { owner ->
return owner == msgId
@@ -223,7 +233,7 @@ class JdbcProcStateRepository(
if (errorClasses.isEmpty()) return 0
val placeholders = errorClasses.joinToString(",") { "?" }
return ds.update(
"UPDATE proc_state SET state = 'PENDING', attempts = 0, next_attempt_at = NULL, updated_at = ? " +
"UPDATE proc_state SET state = 'PENDING', attempts = 0, next_attempt_at = NULL, processing_started_at = NULL, updated_at = ? " +
"WHERE state IN ('FAILED', 'DEAD') AND error_class IN ($placeholders)",
{ ps ->
ps.setTimestamp(1, Instant.now().toSqlTimestamp())
@@ -264,13 +274,14 @@ class JdbcProcStateRepository(
backfillNextAt = rs.getInstant("backfill_next_at"),
backfillAttempts = rs.getInt("backfill_attempts"),
backfillError = rs.getString("backfill_error"),
processingStartedAt = rs.getInstant("processing_started_at"),
updatedAt = rs.getInstant("updated_at") ?: Instant.now(),
)
private companion object {
const val SELECT_PROC =
"SELECT msg_id, state, identity_key, attempts, next_attempt_at, error_class, last_error, " +
"received_at, backfill_at, backfill_next_at, backfill_attempts, backfill_error, updated_at FROM proc_state"
"received_at, backfill_at, backfill_next_at, backfill_attempts, backfill_error, processing_started_at, updated_at FROM proc_state"
}
}
@@ -342,16 +353,21 @@ class JdbcMsgEventRepository(
)
/** 同一条航班只取版本最高的待发整态事件；用了 PostgreSQL 的 DISTINCT ON 语法。 */
override fun mergePendingSchd(limit: Int): List<MsgEvent> =
override fun mergePendingSchd(now: Instant, limit: Int): List<MsgEvent> =
ds.query(
"""
SELECT * FROM (
SELECT DISTINCT ON (partition_key) *
FROM msg_event WHERE target = 'KAFKA:schd' AND state = 'PENDING'
ORDER BY partition_key, state_version DESC, event_id DESC
) latest ORDER BY event_id ASC LIMIT ?
) latest
WHERE next_attempt_at IS NULL OR next_attempt_at <= ?
ORDER BY event_id ASC LIMIT ?
""".trimIndent(),
{ ps -> ps.setInt(1, limit) },
{ ps ->
ps.setTimestamp(1, now.toSqlTimestamp())
ps.setInt(2, limit)
},
::mapEvent,
)
@@ -805,5 +821,3 @@ class JdbcReqTrackRepository(
sentAt = rs.getInstant("sent_at"),
)
}
src/main/kotlin/com/gzzn/omms/msgexchange/infra/retry/ReplayService.kt
+3 -1
View File
@@ -2,6 +2,7 @@ package com.gzzn.omms.msgexchange.infra.retry
import com.gzzn.omms.msgexchange.domain.ErrorClass
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import com.gzzn.omms.msgexchange.processing.MessageLifecycleGate
import jakarta.inject.Singleton
/**
@@ -13,6 +14,7 @@ import jakarta.inject.Singleton
@Singleton
class ReplayService(
private val procState: ProcStateRepository,
private val lifecycleGate: MessageLifecycleGate = MessageLifecycleGate(),
) {
private val log = org.slf4j.LoggerFactory.getLogger(ReplayService::class.java)
/** 可以重放的错误类：解码逻辑修好后能过、处理器补齐后能过、基础设施抖动已恢复、以及重试耗尽但人工复核认为还能再试的。 */
@@ -26,7 +28,7 @@ class ReplayService(
log.warn("replay requested only non-replayable classes: {}", requested)
return 0
}
val n = procState.requeueByErrorClasses(allowed)
val n = lifecycleGate.exclusive { procState.requeueByErrorClasses(allowed) }
log.info("replayed rows={} classes={}", n, allowed)
return n
}
src/main/kotlin/com/gzzn/omms/msgexchange/infra/stub/StubRepositories.kt
+23 -6
View File
@@ -16,6 +16,7 @@ import com.gzzn.omms.msgexchange.infra.persistence.Backlog
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
import com.gzzn.omms.msgexchange.infra.persistence.InboxCursorRepository
import com.gzzn.omms.msgexchange.infra.persistence.MailboxRow
import com.gzzn.omms.msgexchange.infra.persistence.MailboxMarkResult
import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
import com.gzzn.omms.msgexchange.infra.persistence.PersistOutcome
@@ -74,6 +75,12 @@ class StubProcState : ProcStateRepository {
override fun headUnfinished(): ProcState? =
rows.values.filter { it.state == ProcStatus.PENDING || it.state == ProcStatus.FAILED }.minByOrNull { it.msgId }
override fun markProcessingStartedIfAbsent(msgId: Long, now: Instant) {
rows[msgId]?.let { row ->
if (row.processingStartedAt == null) rows[msgId] = row.copy(processingStartedAt = now)
}
}
override fun tryBindIdentity(msgId: Long, identityKey: String): Boolean {
val owner = bound[identityKey]
if (owner != null && owner != msgId) return false
@@ -168,7 +175,7 @@ class StubProcState : ProcStateRepository {
var n = 0
rows.forEach { (id, s) ->
if (s.errorClass in errorClasses && (s.state == ProcStatus.FAILED || s.state == ProcStatus.DEAD)) {
rows[id] = s.copy(state = ProcStatus.PENDING, attempts = 0, nextAttemptAt = null)
rows[id] = s.copy(state = ProcStatus.PENDING, attempts = 0, nextAttemptAt = null, processingStartedAt = null)
n++
}
}
@@ -200,11 +207,12 @@ class StubMsgEvents : MsgEventRepository {
rows.values.filter { it.target == target && it.state == EventStatus.PENDING }.sortedBy { it.eventId!! }.take(limit)
/** 同一条航班只取版本最高的待发整态事件，按事件先后输出。 */
override fun mergePendingSchd(limit: Int): List<MsgEvent> =
override fun mergePendingSchd(now: Instant, limit: Int): List<MsgEvent> =
rows.values
.filter { it.target == "KAFKA:schd" && it.state == EventStatus.PENDING }
.groupBy { it.partitionKey }
.map { (_, group) -> group.maxBy { it.stateVersion } }
.filter { it.nextAttemptAt == null || it.nextAttemptAt <= now }
.sortedBy { it.eventId!! }
.take(limit)
@@ -389,16 +397,19 @@ class StubInbox : CminmsgInboxRepository {
override fun rawOf(msgId: Long): String? = raws[msgId]
override fun receivedAtOf(msgId: Long): Instant? = received[msgId]
override fun readRange(fromExclusive: Long, limit: Int): List<MailboxRow> =
raws.keys.filter { it > fromExclusive }.sorted().take(limit).map { MailboxRow(it, received[it]) }
override fun maxId(): Long? = raws.keys.maxOrNull()
/** 只写还没有标记的行；已经标记过就返回 false，不覆盖已有值。 */
override fun markProcessedIfUnmarked(msgId: Long, value: String): Boolean {
if (!raws.containsKey(msgId) || marks.containsKey(msgId)) return false
/** 只写还没有标记的行，并区分已有标记与行缺失。 */
override fun markProcessedIfUnmarked(msgId: Long, value: String): MailboxMarkResult {
if (!raws.containsKey(msgId)) return MailboxMarkResult.MISSING
if (marks.containsKey(msgId)) return MailboxMarkResult.ALREADY_MARKED
marks[msgId] = value
return true
return MailboxMarkResult.MARKED
}
fun markOf(msgId: Long): String? = marks[msgId]
@@ -408,6 +419,12 @@ class StubInbox : CminmsgInboxRepository {
/** 测试辅助：模拟上游直接往信箱写报文（不经过本系统）。 */
fun simulateExternalWrite(rawXml: String): Long = insertRaw(rawXml)
/** 测试辅助：模拟先分配 ID、稍后才可见的迟提交行。 */
fun restoreRow(msgId: Long, rawXml: String, receivedAt: Instant = Instant.now()) {
raws[msgId] = rawXml
received[msgId] = receivedAt
}
/** 测试辅助：模拟库方清除这条行，用来构造"原文读不到"和 ID 缺口。 */
fun removeRow(msgId: Long) {
raws.remove(msgId); received.remove(msgId); marks.remove(msgId)
src/main/kotlin/com/gzzn/omms/msgexchange/ingress/InboxPoller.kt
+2 -1
View File
@@ -54,7 +54,8 @@ class InboxPoller(
var committedTo = contiguous
var holeSince: Instant? = null
if (rows.last().msgId > contiguous) {
val since = watermark.holeSince ?: now
// 连续上界已前移说明旧空洞已补齐；后面是新空洞，不得继承旧等待时间。
val since = if (contiguous == watermark.committedUpTo) watermark.holeSince ?: now else now
if (Duration.between(since, now) < props.pipeline.maxCommitDelay) {
holeSince = since
} else {
src/main/kotlin/com/gzzn/omms/msgexchange/ingress/InboxService.kt
+4 -3
View File
@@ -21,10 +21,11 @@ class InboxService(
data class Receipt(val msgId: Long, val receivedAt: Instant)
fun accept(rawXml: String, now: Instant = Instant.now()): Receipt {
fun accept(rawXml: String): Receipt {
val id = inbox.insertRaw(rawXml)
procState.insertIfAbsent(id, now)
val receivedAt = checkNotNull(inbox.receivedAtOf(id)) { "mailbox receive time missing for msgId=$id" }
procState.insertIfAbsent(id, receivedAt)
log.info("compat-accepted msgId={}", id)
return Receipt(id, now)
return Receipt(id, receivedAt)
}
}
src/main/kotlin/com/gzzn/omms/msgexchange/processing/BackfillService.kt
+15 -4
View File
@@ -2,7 +2,9 @@ package com.gzzn.omms.msgexchange.processing
import com.gzzn.omms.msgexchange.config.MailboxProps
import com.gzzn.omms.msgexchange.config.PipelineProps
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
import com.gzzn.omms.msgexchange.infra.persistence.MailboxMarkResult
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import jakarta.inject.Singleton
import java.time.Clock
@@ -32,12 +34,14 @@ class BackfillService(
private val mailboxProps: MailboxProps,
private val props: PipelineProps,
private val clock: Clock,
private val lifecycleGate: MessageLifecycleGate = MessageLifecycleGate(),
) {
private val log = org.slf4j.LoggerFactory.getLogger(BackfillService::class.java)
companion object {
private val INITIAL_BACKOFF: Duration = Duration.ofSeconds(30)
private val MAX_BACKOFF: Duration = Duration.ofMinutes(15)
private val TERMINAL_STATES = setOf(ProcStatus.SUCCEEDED, ProcStatus.SKIPPED, ProcStatus.DEAD)
fun backoffDelayFor(attempts: Int): Duration {
val shift = (attempts - 1).coerceIn(0, 20)
@@ -50,8 +54,13 @@ class BackfillService(
* 调用方是主泵的处理路径，不能被回填问题拖住。
*/
fun attempt(msgId: Long, now: Instant = clock.instant()) {
record(msgId, attempts = 0, now = now)?.let {
log.warn("backfill failed msgId={} error={} (sweep will retry)", msgId, it)
lifecycleGate.exclusive {
val row = procState.find(msgId) ?: return@exclusive
if (row.state !in TERMINAL_STATES) return@exclusive
if (row.backfillAt != null) return@exclusive
record(msgId, attempts = row.backfillAttempts, now = now)?.let {
log.warn("backfill failed msgId={} error={} (sweep will retry)", msgId, it)
}
}
}
@@ -62,7 +71,7 @@ class BackfillService(
*/
fun sweep(now: Instant = clock.instant()): Int {
val due = procState.findBackfillDue(now, now.minus(props.pipeline.overdueBackfill), props.pipeline.backfillBatch)
due.forEach { record(it.msgId, it.attempts, now) }
due.forEach { attempt(it.msgId, now) }
return due.size
}
@@ -70,7 +79,8 @@ class BackfillService(
private fun record(msgId: Long, attempts: Int, now: Instant): String? =
try {
// 没有真正写进去说明库里已经有标记了，同样算成功（不覆盖已有值）
mailbox.markProcessedIfUnmarked(msgId, mailboxProps.processedValue)
val result = mailbox.markProcessedIfUnmarked(msgId, mailboxProps.processedValue)
check(result != MailboxMarkResult.MISSING) { "mailbox-row-missing" }
procState.markBackfilled(msgId, now)
null
} catch (e: Exception) {
@@ -80,4 +90,5 @@ class BackfillService(
}.onFailure { log.error("record backfill failure failed msgId={}", msgId, it) }
reason
}
}
src/main/kotlin/com/gzzn/omms/msgexchange/processing/MessageLifecycleGate.kt
+17
View File
@@ -0,0 +1,17 @@
package com.gzzn.omms.msgexchange.processing
import jakarta.inject.Singleton
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
/**
* 串行化“终态回填”与“人工重放”，避免旧回填任务在消息重新入队后写入信箱标记。
* 生产只允许单活动实例；该锁处理同一实例内的 job 线程和管理请求并发。
*/
@Singleton
class MessageLifecycleGate {
private val lock = ReentrantLock()
fun <T> exclusive(block: () -> T): T = lock.withLock(block)
}
src/main/kotlin/com/gzzn/omms/msgexchange/processing/Pump.kt
+21 -15
View File
@@ -66,33 +66,39 @@ class Pump(
val head = procState.headUnfinished()
when {
head == null -> sleepQuietly(props.pipeline.pollInterval)
head.state == ProcStatus.FAILED && poisoned(head) -> {
log.error("poison -> DEAD msgId={} attempts={} lastError={}", head.msgId, head.attempts, head.lastError)
procState.markTerminal(
head.msgId, ProcStatus.DEAD,
errorClass = ErrorClass.EXHAUSTED,
lastError = head.lastError ?: "head-deadline-exceeded",
attempts = head.attempts,
now = clock.instant(),
)
backfill.attempt(head.msgId)
}
head.state == ProcStatus.FAILED && (head.nextAttemptAt ?: Instant.EPOCH) > clock.instant() ->
if (poisoned(head)) {
log.error("poison -> DEAD msgId={} attempts={} lastError={}", head.msgId, head.attempts, head.lastError)
procState.markTerminal(
head.msgId, ProcStatus.DEAD,
errorClass = ErrorClass.EXHAUSTED,
lastError = head.lastError ?: "head-deadline-exceeded",
attempts = head.attempts,
)
backfill.attempt(head.msgId)
} else {
sleepQuietly(Duration.between(clock.instant(), head.nextAttemptAt))
}
sleepQuietly(Duration.between(clock.instant(), head.nextAttemptAt))
// 其余情况（新消息，或退避到期的重试）交给处理入口
else -> processor.processOne(head)
else -> {
procState.markProcessingStartedIfAbsent(head.msgId, clock.instant())
processor.processOne(head)
}
}
}
private fun poisoned(head: ProcState): Boolean =
head.attempts >= props.pipeline.maxAttempts ||
Duration.between(head.updatedAt, clock.instant()) > props.pipeline.headDeadline
isHeadPoisoned(head, clock.instant(), props)
private fun sleepQuietly(d: Duration) {
if (!d.isNegative && !d.isZero) Thread.sleep(d.toMillis().coerceAtLeast(1))
}
}
internal fun isHeadPoisoned(head: ProcState, now: Instant, props: PipelineProps): Boolean =
head.attempts >= props.pipeline.maxAttempts ||
Duration.between(head.processingStartedAt ?: head.updatedAt, now) >= props.pipeline.headDeadline
/**
* 处理一条消息：读原文 → 解码 → 绑定业务身份 → 分派给对应处理器 → 提交后回填标记。
*
src/main/kotlin/com/gzzn/omms/msgexchange/processing/ScheduleProcessor.kt
+5 -1
View File
@@ -95,7 +95,11 @@ class ScheduleProcessor(
val ok = validation as SnapshotValidation.Ok
if (ok.perRecordDay.isEmpty()) {
// 报文合法但没有记录：不需要写数据，照样算处理成功
// 报文合法但没有记录：不写航班，但终态与回填意图仍在锁事务内一起提交
txManager.inTransaction {
lock.lock()
procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED)
}
logSnapshot(head, body, SnapshotResult.COMMITTED, upserted = 0, setOf(SnapshotFlag.EMPTY), started)
return ApplyResult.Succeeded
}
src/main/resources/db/migration/V3__stable_processing_start.sql
+3
View File
@@ -0,0 +1,3 @@
-- HOL deadline 使用首次开始处理的稳定时刻，不再被每次重试更新的 UPDATED_AT 推后。
ALTER TABLE PROC_STATE ADD COLUMN PROCESSING_STARTED_AT TIMESTAMP(6) WITH TIME ZONE;