fix(processing): 收紧消息生命周期与投递约束

This commit is contained in:
windyboy
2026-09-10 20:39:22 +08:00
parent dd839e6abb
commit eea11203a2
27 changed files with 502 additions and 79 deletions
@@ -33,7 +33,7 @@ class DispatcherTickTest {
MsgEvent(eventId = id, target = target, partitionKey = key, stateVersion = version, payloadJson = payload)
@Test
fun `per-target tick delivers KAFKA_MSG only and never consumes schd`() {
fun `first tick delivers msg and starts the independent schd flush`() {
val repo = StubMsgEvents()
val port = StubDeliveryPort()
val props = PipelineProps().apply { schd.flushPeriod = java.time.Duration.ofHours(1) }
@@ -47,7 +47,19 @@ class DispatcherTickTest {
d.tick()
assertEquals(1, port.sent.count { it.topic == "msg" })
assertEquals(0, port.sent.count { it.topic == "schd" }) // KAFKA_SCHD 只能由 flushSchd 发,tick 不许碰
assertEquals(1, port.sent.count { it.topic == "schd" })
}
@Test
fun `first tick starts schd delivery without a prior manual flush`() {
val repo = StubMsgEvents()
val port = StubDeliveryPort()
val props = PipelineProps().apply { pipeline.pollInterval = java.time.Duration.ofMillis(1) }
repo.insertAll(listOf(ev(1, Targets.KAFKA_SCHD, "F1", """{"v":1}""", 1)))
dispatcher(repo, port, props).tick()
assertEquals(1, port.sent.count { it.topic == "schd" })
}
@Test
@@ -70,6 +82,20 @@ class DispatcherTickTest {
assertEquals(0, repo.rows.values.count { it.state.name == "PENDING" })
}
@Test
fun `schd retry is not selected before its backoff expires`() {
val repo = StubMsgEvents()
val port = FailingSchdPort()
val d = dispatcher(repo, port)
repo.insertAll(listOf(ev(1, Targets.KAFKA_SCHD, "F1", """{"v":1}""", 1)))
d.flushSchd()
d.flushSchd()
assertEquals(1, port.calls)
assertEquals(1, repo.rows.values.single().attempts)
}
@Test
fun `tombstone is delivered as null value message`() {
val repo = StubMsgEvents()
@@ -10,7 +10,7 @@ import java.sql.DriverManager
/**
* 在真实 PostgreSQL 上跑一遍迁移,确认结果符合预期:
* V1 基线V2 信箱生命周期都能成功执行迁移记录显示成功该建的表和单行种子
* V1 基线V2 信箱生命周期和 V3 稳定处理起点都能成功执行迁移记录显示成功该建的表和单行种子
* PIPELINE_LOCK、INBOX_CURSOR)都在,回填相关字段进了 PROC_STATE、BACKFILL_TODO 已下线。
*
* 没有可用的 PostgreSQL 时跳过(不假装通过)。
@@ -40,7 +40,7 @@ class FlywayMigrationTest {
DriverManager.getConnection(url, user, pass).use { conn ->
conn.createStatement().use { stmt ->
// 迁移记录:V1 基线 + V2 信箱生命周期
// 迁移记录:V1 基线 + V2 信箱生命周期 + V3 稳定处理起点
stmt.executeQuery(
"SELECT version, script, success FROM flyway_schema_history ORDER BY installed_rank ASC",
).use { rs ->
@@ -48,11 +48,13 @@ class FlywayMigrationTest {
while (rs.next()) {
records.add(Triple(rs.getString("version"), rs.getString("script"), rs.getBoolean("success")))
}
assertTrue(records.size >= 2, "flyway_schema_history must record both migrations")
assertTrue(records.size >= 3, "flyway_schema_history must record all migrations")
assertEquals("1", records[0].first)
assertEquals("V1__flight_state_baseline.sql", records[0].second)
assertEquals("2", records[1].first)
assertEquals("V2__inbox_lifecycle.sql", records[1].second)
assertEquals("3", records[2].first)
assertEquals("V3__stable_processing_start.sql", records[2].second)
assertTrue(records.all { it.third })
}
@@ -81,12 +83,12 @@ class FlywayMigrationTest {
stmt.executeQuery(
"SELECT column_name FROM information_schema.columns WHERE table_name = 'proc_state' " +
"AND column_name IN ('received_at', 'backfill_at', 'backfill_next_at', " +
"'backfill_attempts', 'backfill_error')",
"'backfill_attempts', 'backfill_error', 'processing_started_at')",
).use { rs ->
val cols = mutableSetOf<String>()
while (rs.next()) cols.add(rs.getString("column_name"))
assertEquals(
setOf("received_at", "backfill_at", "backfill_next_at", "backfill_attempts", "backfill_error"),
setOf("received_at", "backfill_at", "backfill_next_at", "backfill_attempts", "backfill_error", "processing_started_at"),
cols,
)
}
@@ -3,6 +3,7 @@ package com.gzzn.omms.msgexchange.infra.persistence.jdbc
import com.gzzn.omms.msgexchange.domain.ErrorClass
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.infra.persistence.InboxCursorRepository
import com.gzzn.omms.msgexchange.infra.persistence.MailboxMarkResult
import org.h2.jdbcx.JdbcDataSource
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertFalse
@@ -50,6 +51,7 @@ class InboxLifecycleJdbcSqlTest {
st.execute(PROC_STATE_DDL)
st.execute(CURSOR_DDL)
st.execute(MAILBOX_DDL)
st.execute(EVENT_DDL)
st.execute(
"INSERT INTO inbox_cursor (cursor_id, committed_up_to, hole_since, updated_at) " +
"VALUES (1, 0, NULL, CURRENT_TIMESTAMP)",
@@ -137,6 +139,33 @@ class InboxLifecycleJdbcSqlTest {
assertEquals(InboxCursorRepository.Cursor(42L, t0), cursor.load())
}
@Test
fun `processing start is written once and is not refreshed`() {
seed(11L, t0)
proc.markProcessingStartedIfAbsent(11L, t0.plusSeconds(10))
proc.markProcessingStartedIfAbsent(11L, t0.plusSeconds(20))
assertEquals(t0.plusSeconds(10), proc.find(11L)!!.processingStartedAt)
}
@Test
fun `pipeline transaction rolls back terminal and outbox writes after a late failure`() {
seed(11L, t0)
val tx = JdbcPipelineTransactionManager(ds)
org.junit.jupiter.api.Assertions.assertThrows(IllegalStateException::class.java) {
tx.inTransaction {
ds.update("INSERT INTO msg_event_test (event_id) VALUES (1)", {})
proc.markTerminal(11L, ProcStatus.SUCCEEDED, now = t0)
error("late-failure")
}
}
assertEquals(ProcStatus.PENDING, proc.find(11L)!!.state)
assertNull(proc.find(11L)!!.backfillNextAt)
assertEquals(0, ds.queryOne("SELECT count(*) AS n FROM msg_event_test", {}) { it.getInt("n") })
}
@Test
fun `mailbox range read ignores processing marks and marking never overwrites`() {
val first = mailbox.insertRaw("<A/>")
@@ -149,12 +178,12 @@ class InboxLifecycleJdbcSqlTest {
assertEquals(listOf(second, third), mailbox.readRange(first, 50).map { it.msgId })
assertEquals(third, mailbox.maxId())
assertTrue(mailbox.markProcessedIfUnmarked(second, "PROCESSED"))
assertFalse(mailbox.markProcessedIfUnmarked(second, "OTHER")) // 已有标记,不覆盖
assertEquals(MailboxMarkResult.MARKED, mailbox.markProcessedIfUnmarked(second, "PROCESSED"))
assertEquals(MailboxMarkResult.ALREADY_MARKED, mailbox.markProcessedIfUnmarked(second, "OTHER"))
assertEquals("PROCESSED", statusOf(second))
// 已标记行仍出现在区间读结果中(发现与标记彻底解耦)
assertEquals(listOf(first, second, third), mailbox.readRange(0L, 50).map { it.msgId })
assertFalse(mailbox.markProcessedIfUnmarked(999L, "PROCESSED"))
assertEquals(MailboxMarkResult.MISSING, mailbox.markProcessedIfUnmarked(999L, "PROCESSED"))
}
private fun seed(msgId: Long, receivedAt: Instant?, state: ProcStatus = ProcStatus.PENDING) {
@@ -194,6 +223,7 @@ class InboxLifecycleJdbcSqlTest {
backfill_next_at TIMESTAMP WITH TIME ZONE,
backfill_attempts INT NOT NULL DEFAULT 0,
backfill_error VARCHAR(512),
processing_started_at TIMESTAMP WITH TIME ZONE,
updated_at TIMESTAMP WITH TIME ZONE NOT NULL,
CONSTRAINT uk_proc_identity UNIQUE (identity_key)
)
@@ -217,5 +247,11 @@ class InboxLifecycleJdbcSqlTest {
CMINMSGS_STATUS VARCHAR(32)
)
"""
const val EVENT_DDL = """
CREATE TABLE msg_event_test (
event_id BIGINT PRIMARY KEY
)
"""
}
}
@@ -0,0 +1,31 @@
package com.gzzn.omms.msgexchange.infra.persistence.jdbc
import org.h2.jdbcx.JdbcDataSource
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
import java.util.UUID
class JdbcOpsTest {
@Test
fun `transaction connection is scoped to its data source`() {
val first = dataSource("first")
val second = dataSource("second")
first.update("CREATE TABLE marker (marker_value INT)", {})
second.update("CREATE TABLE marker (marker_value INT)", {})
second.update("INSERT INTO marker (marker_value) VALUES (2)", {})
first.withTransaction {
first.update("INSERT INTO marker (marker_value) VALUES (1)", {})
val fromSecond = second.queryOne("SELECT marker_value FROM marker", {}) { it.getInt(1) }
assertEquals(2, fromSecond)
}
assertEquals(1, first.queryOne("SELECT marker_value FROM marker", {}) { it.getInt(1) })
}
private fun dataSource(label: String) = JdbcDataSource().apply {
setURL("jdbc:h2:mem:$label-${UUID.randomUUID()};DB_CLOSE_DELAY=-1;MODE=PostgreSQL")
user = "sa"
password = ""
}
}
@@ -20,13 +20,17 @@ class ReplayServiceTest {
val requeueCalls = mutableListOf<List<ErrorClass>>()
fun seed(id: Long, status: ProcStatus, ec: ErrorClass?) {
rows[id] = ProcState(id, status, identityKey = "k$id", attempts = 3, errorClass = ec, lastError = "x")
rows[id] = ProcState(
id, status, identityKey = "k$id", attempts = 3, errorClass = ec, lastError = "x",
processingStartedAt = Instant.parse("2026-09-08T03:00:00Z"),
)
}
override fun insertIfAbsent(msgId: Long, receivedAt: Instant?): Boolean = rows.putIfAbsent(msgId, ProcState(msgId, ProcStatus.PENDING, receivedAt = receivedAt)) == null
override fun find(msgId: Long): ProcState? = rows[msgId]
override fun findSuccessTerminal(msgId: Long): Boolean = rows[msgId]?.state == ProcStatus.SUCCEEDED
override fun headUnfinished(): ProcState? = null
override fun markProcessingStartedIfAbsent(msgId: Long, now: Instant) = Unit
override fun tryBindIdentity(msgId: Long, identityKey: String): Boolean = true
override fun ownerOfIdentity(identityKey: String): Long? = null
override fun update(
@@ -55,7 +59,7 @@ class ReplayServiceTest {
val s = rows[id]!!
if (s.errorClass != null && s.errorClass in errorClasses &&
(s.state == ProcStatus.FAILED || s.state == ProcStatus.DEAD)) {
rows[id] = s.copy(state = ProcStatus.PENDING, attempts = 0, nextAttemptAt = null)
rows[id] = s.copy(state = ProcStatus.PENDING, attempts = 0, nextAttemptAt = null, processingStartedAt = null)
n++
}
}
@@ -77,6 +81,7 @@ class ReplayServiceTest {
assertEquals(ProcStatus.PENDING, repo.rows[1]!!.state)
assertEquals(0, repo.rows[1]!!.attempts)
assertNull(repo.rows[1]!!.nextAttemptAt)
assertNull(repo.rows[1]!!.processingStartedAt)
assertEquals(ProcStatus.DEAD, repo.rows[2]!!.state) // MALFORMED 永不被重放
assertEquals(ProcStatus.PENDING, repo.rows[3]!!.state)
}
@@ -112,10 +112,36 @@ class InboxPollerTest {
@Test
fun `compat http path and poller do not double enqueue the same message`() {
val receipt = InboxService(inbox, proc).accept("<MSG/>", t0)
val receipt = InboxService(inbox, proc).accept("<MSG/>")
assertEquals(0, poller.pollOnce(t0))
assertEquals(receipt.msgId, cursor.cursor.committedUpTo) // 已在 PG:读取进度照常推进
assertNotNull(proc.find(receipt.msgId))
}
@Test
fun `a newly exposed hole does not inherit the age of the previous hole`() {
val first = inbox.simulateExternalWrite("<MSG/>")
val oldHole = inbox.simulateExternalWrite("<MSG/>")
val third = inbox.simulateExternalWrite("<MSG/>")
val newHole = inbox.simulateExternalWrite("<MSG/>")
val fifth = inbox.simulateExternalWrite("<MSG/>")
inbox.removeRow(oldHole)
inbox.removeRow(newHole)
assertEquals(1, poller.pollOnce(t0))
val almostAged = t0.plus(props.pipeline.maxCommitDelay).minusSeconds(1)
inbox.restoreRow(oldHole, "<MSG/>", almostAged)
assertEquals(2, poller.pollOnce(almostAged))
assertEquals(third, cursor.cursor.committedUpTo)
assertEquals(almostAged, cursor.cursor.holeSince)
assertNull(proc.find(fifth))
// 旧空洞的期限已到,但新空洞必须获得完整等待窗口。
assertEquals(0, poller.pollOnce(t0.plus(props.pipeline.maxCommitDelay)))
assertEquals(third, cursor.cursor.committedUpTo)
assertNull(proc.find(fifth))
assertEquals(first + 2, third)
}
}
@@ -6,18 +6,26 @@ import com.gzzn.omms.msgexchange.domain.ErrorClass
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
import com.gzzn.omms.msgexchange.infra.persistence.MailboxRow
import com.gzzn.omms.msgexchange.infra.persistence.MailboxMarkResult
import com.gzzn.omms.msgexchange.infra.stub.StubInbox
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
import com.gzzn.omms.msgexchange.infra.retry.ReplayService
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertNotNull
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Assertions.assertThrows
import org.junit.jupiter.api.Test
import java.time.Clock
import java.time.Duration
import java.time.Instant
import java.time.ZoneOffset
import java.util.concurrent.CountDownLatch
import java.util.concurrent.Callable
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.TimeoutException
/**
* 回填环节的规矩:
@@ -37,11 +45,12 @@ class BackfillServiceTest {
val marked = linkedSetOf<Long>()
override fun insertRaw(rawXml: String): Long = 1L
override fun rawOf(msgId: Long): String? = null
override fun receivedAtOf(msgId: Long): Instant? = null
override fun readRange(fromExclusive: Long, limit: Int): List<MailboxRow> = emptyList()
override fun maxId(): Long? = null
override fun markProcessedIfUnmarked(msgId: Long, value: String): Boolean {
override fun markProcessedIfUnmarked(msgId: Long, value: String): MailboxMarkResult {
if (fail) throw IllegalStateException("mysql-down")
return marked.add(msgId)
return if (marked.add(msgId)) MailboxMarkResult.MARKED else MailboxMarkResult.ALREADY_MARKED
}
}
@@ -75,8 +84,8 @@ class BackfillServiceTest {
val inbox = StubInbox().apply { clear() }
val id = inbox.insertRaw("<MSG/>")
assertTrue(inbox.markProcessedIfUnmarked(id, "PROCESSED"))
assertFalse(inbox.markProcessedIfUnmarked(id, "OTHER")) // 已经有标记了,不再写第二次
assertEquals(MailboxMarkResult.MARKED, inbox.markProcessedIfUnmarked(id, "PROCESSED"))
assertEquals(MailboxMarkResult.ALREADY_MARKED, inbox.markProcessedIfUnmarked(id, "OTHER"))
assertEquals("PROCESSED", inbox.markOf(id))
}
@@ -89,10 +98,12 @@ class BackfillServiceTest {
val backfill = service(proc, inbox)
backfill.attempt(id)
val firstCompletion = proc.find(id)!!.backfillAt
backfill.attempt(id) // 重复补写没有副作用
assertNull(proc.find(id)!!.backfillError)
assertEquals(0, proc.find(id)!!.backfillAttempts)
assertEquals(firstCompletion, proc.find(id)!!.backfillAt)
}
@Test
@@ -111,6 +122,23 @@ class BackfillServiceTest {
assertNull(row.backfillAt)
}
@Test
fun `missing mailbox row remains an unconfirmed backfill failure`() {
val proc = StubProcState()
val inbox = StubInbox().apply { clear() }
val id = inbox.insertRaw("<MSG/>")
succeeded(proc, id)
inbox.removeRow(id)
service(proc, inbox).attempt(id)
val row = proc.find(id)!!
assertNull(row.backfillAt)
assertEquals(1, row.backfillAttempts)
assertEquals("mailbox-row-missing", row.backfillError)
assertNotNull(row.backfillNextAt)
}
@Test
fun `sweep retries due rows and completes once the mailbox recovers`() {
val proc = StubProcState()
@@ -163,6 +191,25 @@ class BackfillServiceTest {
assertFalse(inbox.isMarked(failed))
}
@Test
fun `immediate attempt also refuses pending and failed messages`() {
val proc = StubProcState()
val inbox = StubInbox().apply { clear() }
val pending = inbox.insertRaw("<MSG/>")
val failed = inbox.insertRaw("<MSG/>")
proc.insertIfAbsent(pending, t0)
proc.insertIfAbsent(failed, t0)
proc.update(failed, ProcStatus.FAILED, errorClass = ErrorClass.INFRA)
service(proc, inbox).attempt(pending)
service(proc, inbox).attempt(failed)
assertFalse(inbox.isMarked(pending))
assertFalse(inbox.isMarked(failed))
assertNull(proc.find(pending)!!.backfillAt)
assertNull(proc.find(failed)!!.backfillAt)
}
@Test
fun `backoff delay doubles per attempt and caps at fifteen minutes`() {
assertEquals(Duration.ofSeconds(30), BackfillService.backoffDelayFor(1))
@@ -170,4 +217,49 @@ class BackfillServiceTest {
assertEquals(Duration.ofMinutes(15), BackfillService.backoffDelayFor(10))
assertEquals(Duration.ofMinutes(15), BackfillService.backoffDelayFor(50))
}
@Test
fun `replay waits for an in-flight backfill before reopening the message`() {
val entered = CountDownLatch(1)
val release = CountDownLatch(1)
val replayStarted = CountDownLatch(1)
val mailbox = object : CminmsgInboxRepository {
override fun insertRaw(rawXml: String) = 1L
override fun rawOf(msgId: Long): String? = "<MSG/>"
override fun receivedAtOf(msgId: Long) = t0
override fun readRange(fromExclusive: Long, limit: Int) = emptyList<MailboxRow>()
override fun maxId(): Long? = 1L
override fun markProcessedIfUnmarked(msgId: Long, value: String): MailboxMarkResult {
entered.countDown()
release.await()
return MailboxMarkResult.MARKED
}
}
val proc = StubProcState().apply {
insertIfAbsent(1L, t0)
markTerminal(1L, ProcStatus.DEAD, errorClass = ErrorClass.EXHAUSTED, now = t0)
}
val gate = MessageLifecycleGate()
val backfill = BackfillService(proc, mailbox, MailboxProps(), props, Clock.fixed(t0, ZoneOffset.UTC), gate)
val replay = ReplayService(proc, gate)
val pool = Executors.newFixedThreadPool(2)
try {
val backfillTask = pool.submit { backfill.attempt(1L) }
assertTrue(entered.await(1, TimeUnit.SECONDS))
val replayTask = pool.submit(Callable {
replayStarted.countDown()
replay.replay(listOf(ErrorClass.EXHAUSTED))
})
assertTrue(replayStarted.await(1, TimeUnit.SECONDS))
assertThrows(TimeoutException::class.java) { replayTask.get(100, TimeUnit.MILLISECONDS) }
release.countDown()
backfillTask.get(1, TimeUnit.SECONDS)
assertEquals(1, replayTask.get(1, TimeUnit.SECONDS))
assertEquals(ProcStatus.PENDING, proc.find(1L)!!.state)
} finally {
release.countDown()
pool.shutdownNow()
}
}
}
@@ -0,0 +1,44 @@
package com.gzzn.omms.msgexchange.processing
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import java.util.concurrent.CountDownLatch
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
class MessageLifecycleGateTest {
@Test
fun `replay cannot enter while a backfill operation owns the lifecycle gate`() {
val gate = MessageLifecycleGate()
val entered = CountDownLatch(1)
val release = CountDownLatch(1)
val replayStarted = CountDownLatch(1)
val replayEntered = CountDownLatch(1)
val pool = Executors.newFixedThreadPool(2)
try {
val backfill = pool.submit {
gate.exclusive {
entered.countDown()
release.await()
}
}
assertTrue(entered.await(1, TimeUnit.SECONDS))
val replay = pool.submit {
replayStarted.countDown()
gate.exclusive { replayEntered.countDown() }
}
assertTrue(replayStarted.await(1, TimeUnit.SECONDS))
assertFalse(replayEntered.await(100, TimeUnit.MILLISECONDS))
release.countDown()
backfill.get(1, TimeUnit.SECONDS)
replay.get(1, TimeUnit.SECONDS)
assertTrue(replayEntered.await(1, TimeUnit.SECONDS))
} finally {
release.countDown()
pool.shutdownNow()
}
}
}
@@ -0,0 +1,40 @@
package com.gzzn.omms.msgexchange.processing
import com.gzzn.omms.msgexchange.config.PipelineProps
import com.gzzn.omms.msgexchange.domain.ProcState
import com.gzzn.omms.msgexchange.domain.ProcStatus
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import java.time.Instant
class PumpDeadlineTest {
private val props = PipelineProps()
private val started = Instant.parse("2026-09-08T03:00:00Z")
@Test
fun `deadline uses stable processing start rather than refreshed update time`() {
val row = ProcState(
msgId = 1,
state = ProcStatus.FAILED,
attempts = 1,
processingStartedAt = started,
updatedAt = started.plusSeconds(590),
)
assertTrue(isHeadPoisoned(row, started.plusSeconds(600), props))
}
@Test
fun `head below attempt and time limits remains retryable`() {
val row = ProcState(
msgId = 1,
state = ProcStatus.FAILED,
attempts = 1,
processingStartedAt = started,
updatedAt = started.plusSeconds(590),
)
assertFalse(isHeadPoisoned(row, started.plusSeconds(599), props))
}
}
@@ -112,6 +112,19 @@ class ScheduleProcessorTest {
assertNotNull(row.backfillNextAt)
}
@Test
fun `valid empty schedule commits terminal state and backfill intent`() {
val proc = StubProcState()
proc.insertIfAbsent(msgId, null)
val body = ScheduleBody(recsDeclared = 0, records = emptyList())
val result = processor(proc = proc).applyScheduleRecords(head(), message(body))
assertEquals(ApplyResult.Succeeded, result)
assertEquals(ProcStatus.SUCCEEDED, proc.find(msgId)!!.state)
assertNotNull(proc.find(msgId)!!.backfillNextAt)
}
@Test
fun `replay of succeeded message records idempotent success without writes`() {
val proc = StubProcState()