cdia/msgexchange-v2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(processing): 收紧消息生命周期与投递约束

Browse Source
This commit is contained in:
windyboy
2026-09-10 20:39:22 +08:00
parent dd839e6abb
commit eea11203a2
27 changed files with 502 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/Repositories.kt
+12 -3
View File
@@ -58,6 +58,9 @@ interface ProcStateRepository {
/** 当前队头：还没处理完的消息里 ID 最小的那条。 */
fun headUnfinished(): ProcState?
/** 首次开始处理时记下稳定起点；重试不覆盖。 */
fun markProcessingStartedIfAbsent(msgId: Long, now: Instant)
/** 给消息绑定业务身份；返回 false 表示这个身份已经被另一条消息占了（业务重复）。 */
fun tryBindIdentity(msgId: Long, identityKey: String): Boolean
@@ -135,7 +138,7 @@ interface MsgEventRepository {
fun claimBatch(target: String, limit: Int): List<MsgEvent>
/** 挑出待发的整态事件，同一条航班只取版本最高的那一条（中间的版本不用发）。 */
fun mergePendingSchd(limit: Int): List<MsgEvent>
fun mergePendingSchd(now: Instant, limit: Int): List<MsgEvent>
fun markSent(eventId: Long)
@@ -287,6 +290,9 @@ interface CminmsgInboxRepository {
/** 读某条消息的原文；返回 null 表示读不到（行已被清除，或原文本身为空）。 */
fun rawOf(msgId: Long): String?
/** 读取信箱库记录的实际接收时间。 */
fun receivedAtOf(msgId: Long): Instant?
/** 按 ID 升序读一批 `ID > fromExclusive` 的行，不带别的过滤条件。 */
fun readRange(fromExclusive: Long, limit: Int): List<MailboxRow>
@@ -295,10 +301,13 @@ interface CminmsgInboxRepository {
/**
* 把处理标记写回信箱，并且**只写还是空标记的行**：库里已有值时不覆盖、不回退，
* 重复调用没有副作用。返回 true 表示这次真的写进去了。
* 重复调用没有副作用。结果明确区分本次写入、已有标记和信箱行缺失。
*/
fun markProcessedIfUnmarked(msgId: Long, value: String): Boolean
fun markProcessedIfUnmarked(msgId: Long, value: String): MailboxMarkResult
}
/** 信箱回填的可区分结果；行缺失不能冒充“已有标记”。 */
enum class MailboxMarkResult { MARKED, ALREADY_MARKED, MISSING }
/** 从信箱读到的一行：ID 加接收时间。原文按需再取，扫描时不读大字段。 */
data class MailboxRow(val msgId: Long, val receivedAt: Instant?)
src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/JdbcCminmsgInboxRepository.kt
+19 -4
View File
@@ -2,10 +2,12 @@ package com.gzzn.omms.msgexchange.infra.persistence.jdbc
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
import com.gzzn.omms.msgexchange.infra.persistence.MailboxRow
import com.gzzn.omms.msgexchange.infra.persistence.MailboxMarkResult
import io.micronaut.context.annotation.Requires
import jakarta.inject.Named
import jakarta.inject.Singleton
import javax.sql.DataSource
import java.time.Instant
/**
* 共享 MySQL CMINMSGS 信箱的 JDBC 实现。列名沿用 legacy 的 `CMINMSGS_*`。
@@ -39,6 +41,12 @@ class JdbcCminmsgInboxRepository(
{ ps -> ps.setLong(1, msgId) },
) { rs -> rs.getString("CMINMSGS_CLOB_MSG") }
override fun receivedAtOf(msgId: Long): Instant? =
ds.queryOne(
"SELECT CMINMSGS_DATE_RECEIVED FROM cminmsgs WHERE CMINMSGS_ID = ?",
{ ps -> ps.setLong(1, msgId) },
) { rs -> rs.getTimestamp("CMINMSGS_DATE_RECEIVED")?.toInstant() }
override fun readRange(fromExclusive: Long, limit: Int): List<MailboxRow> =
ds.query(
"""
@@ -65,11 +73,11 @@ class JdbcCminmsgInboxRepository(
/**
* 只更新还是空标记的行，所以重复调用不会覆盖库里已有的值；
* 影响 0 行说明已经被标记过了，调用方按"成功"处理即可。
* 影响 0 行时再查一次主键，区分“已有标记”和“信箱行缺失”；后者不能记为回填成功。
* 具体写什么值、时间怎么解释，以与库方约定为准。
*/
override fun markProcessedIfUnmarked(msgId: Long, value: String): Boolean =
ds.update(
override fun markProcessedIfUnmarked(msgId: Long, value: String): MailboxMarkResult {
val updated = ds.update(
"""
UPDATE cminmsgs
SET CMINMSGS_DATE_PROCESSED = CURRENT_TIMESTAMP,
@@ -79,5 +87,12 @@ class JdbcCminmsgInboxRepository(
) { ps ->
ps.setString(1, value)
ps.setLong(2, msgId)
} == 1
}
if (updated == 1) return MailboxMarkResult.MARKED
val exists = ds.queryOne(
"SELECT 1 FROM cminmsgs WHERE CMINMSGS_ID = ?",
{ ps -> ps.setLong(1, msgId) },
) { 1 } != null
return if (exists) MailboxMarkResult.ALREADY_MARKED else MailboxMarkResult.MISSING
}
}
src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/JdbcOps.kt
+16 -10
View File
@@ -10,17 +10,20 @@ internal fun Instant.toSqlTimestamp(): Timestamp = Timestamp.from(this)
internal fun ResultSet.getInstant(column: String): Instant? =
getTimestamp(column)?.toInstant()
private val transactionConnection = ThreadLocal<java.sql.Connection?>()
private val transactionConnections = ThreadLocal.withInitial {
java.util.IdentityHashMap<DataSource, java.sql.Connection>()
}
internal fun <T> DataSource.withTransaction(block: () -> T): T {
val existing = transactionConnection.get()
val connections = transactionConnections.get()
val existing = connections[this]
if (existing != null) {
return block()
}
val conn = this.connection
val oldAutoCommit = conn.autoCommit
conn.autoCommit = false
transactionConnection.set(conn)
connections[this] = conn
try {
val result = block()
conn.commit()
@@ -33,7 +36,8 @@ internal fun <T> DataSource.withTransaction(block: () -> T): T {
}
throw t
} finally {
transactionConnection.remove()
connections.remove(this)
if (connections.isEmpty()) transactionConnections.remove()
try {
conn.autoCommit = oldAutoCommit
} catch (_: Throwable) {
@@ -43,11 +47,13 @@ internal fun <T> DataSource.withTransaction(block: () -> T): T {
}
internal fun DataSource.obtainConnection(): java.sql.Connection =
transactionConnection.get() ?: this.connection
transactionConnections.get()[this] ?: this.connection
internal fun java.sql.Connection.releaseIfNotInTransaction() {
if (transactionConnection.get() !== this) {
internal fun java.sql.Connection.releaseIfNotInTransaction(dataSource: DataSource) {
val connections = transactionConnections.get()
if (connections[dataSource] !== this) {
this.close()
if (connections.isEmpty()) transactionConnections.remove()
}
}
@@ -63,7 +69,7 @@ internal fun <T> DataSource.query(sql: String, bind: (java.sql.PreparedStatement
}
}
} finally {
conn.releaseIfNotInTransaction()
conn.releaseIfNotInTransaction(this)
}
}
@@ -78,7 +84,7 @@ internal fun DataSource.update(sql: String, bind: (java.sql.PreparedStatement) -
ps.executeUpdate()
}
} finally {
conn.releaseIfNotInTransaction()
conn.releaseIfNotInTransaction(this)
}
}
@@ -94,6 +100,6 @@ internal fun DataSource.updateReturningLong(sql: String, bind: (java.sql.Prepare
}
}
} finally {
conn.releaseIfNotInTransaction()
conn.releaseIfNotInTransaction(this)
}
}
src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/JdbcPgRepositories.kt
+21 -7
View File
@@ -99,6 +99,16 @@ class JdbcProcStateRepository(
::mapProcState,
)
override fun markProcessingStartedIfAbsent(msgId: Long, now: Instant) {
ds.update(
"UPDATE proc_state SET processing_started_at = ? WHERE msg_id = ? AND processing_started_at IS NULL",
{ ps ->
ps.setTimestamp(1, now.toSqlTimestamp())
ps.setLong(2, msgId)
},
)
}
override fun tryBindIdentity(msgId: Long, identityKey: String): Boolean {
ownerOfIdentity(identityKey)?.let { owner ->
return owner == msgId
@@ -223,7 +233,7 @@ class JdbcProcStateRepository(
if (errorClasses.isEmpty()) return 0
val placeholders = errorClasses.joinToString(",") { "?" }
return ds.update(
"UPDATE proc_state SET state = 'PENDING', attempts = 0, next_attempt_at = NULL, updated_at = ? " +
"UPDATE proc_state SET state = 'PENDING', attempts = 0, next_attempt_at = NULL, processing_started_at = NULL, updated_at = ? " +
"WHERE state IN ('FAILED', 'DEAD') AND error_class IN ($placeholders)",
{ ps ->
ps.setTimestamp(1, Instant.now().toSqlTimestamp())
@@ -264,13 +274,14 @@ class JdbcProcStateRepository(
backfillNextAt = rs.getInstant("backfill_next_at"),
backfillAttempts = rs.getInt("backfill_attempts"),
backfillError = rs.getString("backfill_error"),
processingStartedAt = rs.getInstant("processing_started_at"),
updatedAt = rs.getInstant("updated_at") ?: Instant.now(),
)
private companion object {
const val SELECT_PROC =
"SELECT msg_id, state, identity_key, attempts, next_attempt_at, error_class, last_error, " +
"received_at, backfill_at, backfill_next_at, backfill_attempts, backfill_error, updated_at FROM proc_state"
"received_at, backfill_at, backfill_next_at, backfill_attempts, backfill_error, processing_started_at, updated_at FROM proc_state"
}
}
@@ -342,16 +353,21 @@ class JdbcMsgEventRepository(
)
/** 同一条航班只取版本最高的待发整态事件；用了 PostgreSQL 的 DISTINCT ON 语法。 */
override fun mergePendingSchd(limit: Int): List<MsgEvent> =
override fun mergePendingSchd(now: Instant, limit: Int): List<MsgEvent> =
ds.query(
"""
SELECT * FROM (
SELECT DISTINCT ON (partition_key) *
FROM msg_event WHERE target = 'KAFKA:schd' AND state = 'PENDING'
ORDER BY partition_key, state_version DESC, event_id DESC
) latest ORDER BY event_id ASC LIMIT ?
) latest
WHERE next_attempt_at IS NULL OR next_attempt_at <= ?
ORDER BY event_id ASC LIMIT ?
""".trimIndent(),
{ ps -> ps.setInt(1, limit) },
{ ps ->
ps.setTimestamp(1, now.toSqlTimestamp())
ps.setInt(2, limit)
},
::mapEvent,
)
@@ -805,5 +821,3 @@ class JdbcReqTrackRepository(
sentAt = rs.getInstant("sent_at"),
)
}
src/main/kotlin/com/gzzn/omms/msgexchange/infra/retry/ReplayService.kt
+3 -1
View File
@@ -2,6 +2,7 @@ package com.gzzn.omms.msgexchange.infra.retry
import com.gzzn.omms.msgexchange.domain.ErrorClass
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import com.gzzn.omms.msgexchange.processing.MessageLifecycleGate
import jakarta.inject.Singleton
/**
@@ -13,6 +14,7 @@ import jakarta.inject.Singleton
@Singleton
class ReplayService(
private val procState: ProcStateRepository,
private val lifecycleGate: MessageLifecycleGate = MessageLifecycleGate(),
) {
private val log = org.slf4j.LoggerFactory.getLogger(ReplayService::class.java)
/** 可以重放的错误类：解码逻辑修好后能过、处理器补齐后能过、基础设施抖动已恢复、以及重试耗尽但人工复核认为还能再试的。 */
@@ -26,7 +28,7 @@ class ReplayService(
log.warn("replay requested only non-replayable classes: {}", requested)
return 0
}
val n = procState.requeueByErrorClasses(allowed)
val n = lifecycleGate.exclusive { procState.requeueByErrorClasses(allowed) }
log.info("replayed rows={} classes={}", n, allowed)
return n
}
src/main/kotlin/com/gzzn/omms/msgexchange/infra/stub/StubRepositories.kt
+23 -6
View File
@@ -16,6 +16,7 @@ import com.gzzn.omms.msgexchange.infra.persistence.Backlog
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
import com.gzzn.omms.msgexchange.infra.persistence.InboxCursorRepository
import com.gzzn.omms.msgexchange.infra.persistence.MailboxRow
import com.gzzn.omms.msgexchange.infra.persistence.MailboxMarkResult
import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
import com.gzzn.omms.msgexchange.infra.persistence.PersistOutcome
@@ -74,6 +75,12 @@ class StubProcState : ProcStateRepository {
override fun headUnfinished(): ProcState? =
rows.values.filter { it.state == ProcStatus.PENDING || it.state == ProcStatus.FAILED }.minByOrNull { it.msgId }
override fun markProcessingStartedIfAbsent(msgId: Long, now: Instant) {
rows[msgId]?.let { row ->
if (row.processingStartedAt == null) rows[msgId] = row.copy(processingStartedAt = now)
}
}
override fun tryBindIdentity(msgId: Long, identityKey: String): Boolean {
val owner = bound[identityKey]
if (owner != null && owner != msgId) return false
@@ -168,7 +175,7 @@ class StubProcState : ProcStateRepository {
var n = 0
rows.forEach { (id, s) ->
if (s.errorClass in errorClasses && (s.state == ProcStatus.FAILED || s.state == ProcStatus.DEAD)) {
rows[id] = s.copy(state = ProcStatus.PENDING, attempts = 0, nextAttemptAt = null)
rows[id] = s.copy(state = ProcStatus.PENDING, attempts = 0, nextAttemptAt = null, processingStartedAt = null)
n++
}
}
@@ -200,11 +207,12 @@ class StubMsgEvents : MsgEventRepository {
rows.values.filter { it.target == target && it.state == EventStatus.PENDING }.sortedBy { it.eventId!! }.take(limit)
/** 同一条航班只取版本最高的待发整态事件，按事件先后输出。 */
override fun mergePendingSchd(limit: Int): List<MsgEvent> =
override fun mergePendingSchd(now: Instant, limit: Int): List<MsgEvent> =
rows.values
.filter { it.target == "KAFKA:schd" && it.state == EventStatus.PENDING }
.groupBy { it.partitionKey }
.map { (_, group) -> group.maxBy { it.stateVersion } }
.filter { it.nextAttemptAt == null || it.nextAttemptAt <= now }
.sortedBy { it.eventId!! }
.take(limit)
@@ -389,16 +397,19 @@ class StubInbox : CminmsgInboxRepository {
override fun rawOf(msgId: Long): String? = raws[msgId]
override fun receivedAtOf(msgId: Long): Instant? = received[msgId]
override fun readRange(fromExclusive: Long, limit: Int): List<MailboxRow> =
raws.keys.filter { it > fromExclusive }.sorted().take(limit).map { MailboxRow(it, received[it]) }
override fun maxId(): Long? = raws.keys.maxOrNull()
/** 只写还没有标记的行；已经标记过就返回 false，不覆盖已有值。 */
override fun markProcessedIfUnmarked(msgId: Long, value: String): Boolean {
if (!raws.containsKey(msgId) || marks.containsKey(msgId)) return false
/** 只写还没有标记的行，并区分已有标记与行缺失。 */
override fun markProcessedIfUnmarked(msgId: Long, value: String): MailboxMarkResult {
if (!raws.containsKey(msgId)) return MailboxMarkResult.MISSING
if (marks.containsKey(msgId)) return MailboxMarkResult.ALREADY_MARKED
marks[msgId] = value
return true
return MailboxMarkResult.MARKED
}
fun markOf(msgId: Long): String? = marks[msgId]
@@ -408,6 +419,12 @@ class StubInbox : CminmsgInboxRepository {
/** 测试辅助：模拟上游直接往信箱写报文（不经过本系统）。 */
fun simulateExternalWrite(rawXml: String): Long = insertRaw(rawXml)
/** 测试辅助：模拟先分配 ID、稍后才可见的迟提交行。 */
fun restoreRow(msgId: Long, rawXml: String, receivedAt: Instant = Instant.now()) {
raws[msgId] = rawXml
received[msgId] = receivedAt
}
/** 测试辅助：模拟库方清除这条行，用来构造"原文读不到"和 ID 缺口。 */
fun removeRow(msgId: Long) {
raws.remove(msgId); received.remove(msgId); marks.remove(msgId)