refactor(ingress): 信箱边界层契约重构——发现与处理标记解耦、回填事实并入 PROC_STATE
收报扫描不再以 DATE_PROCESSED 为谓词：终态而未回填的行（解码失败死信等）会永久占据 有限批次，累积到 claim-batch 后收报整体停摆（US-01 条目 3 / message-lifecycle §5.3）。 ingress/InboxPoller.kt：按 ID 区间升序有界读取（ID > W），水位落 INBOX_CURSOR 并与入队 同一 PG 事务推进（中断后重扫补建）；遇空洞即停，空洞超过 pipeline.max-commit-delay 判定 为永久并放行——否则水位永久停摆于一次自增回滚留下的空位。删除 InboxEnqueue（改由 insertIfAbsent 幂等入队）与 poller 内的回填扫描（消除 ingress→jobs 反向依赖）。 infra/persistence：端口按事实重画为 readRange/maxId/markProcessedIfUnmarked，标记 UPDATE 带 DATE_PROCESSED IS NULL 守卫，只把空标写为已处理（§11 单调，重复执行无副作用）。 回填事实并入 PROC_STATE（RECEIVED_AT/BACKFILL_AT/NEXT_AT/ATTEMPTS/ERROR），BACKFILL_TODO 随 V2 迁移下线；终态与回填意图是同一条 UPDATE，由处理器在自己的业务事务内落库， message-lifecycle §4 登记的两个崩溃窗口（提交后回填前崩溃、待办二次落账失败）不再是缺口。 processing/BackfillService.kt（取代 BackfillSweepJob）：终态提交后立即尝试一次，失败按 30s→15min 指数退避重试；扫描条件「终态 + 未确认标记 +（已到期 或 接收时间早于 NOW − R）」 使 §5.2 的超期期限 R 覆盖退避，中间态永不补写。死信同样可补写——回填只需消息 ID， 不再依赖 META。Pump 改用可注入 Clock。 infra/health：InboxLifecycleHealthIndicator 输出积压条数、最老未处理信龄、未回填终态数与 水位滞后（OPS-2 / §5.3 验收）。预计消化时长需吞吐采样，留待接入指标注册表时补。 配置：pipeline.max-commit-delay / overdue-backfill / backfill-batch、mailbox.processed-value （Q2/Q6/Q7 未书面确认前取保守初值，不得为提速下调）。 不变量回归测试：死信不阻断后续发现、水位遇空洞即停与老化放行、终态+意图同事务、 超期 R 覆盖退避、中间态不补写、标记单调；InboxLifecycleJdbcSqlTest 以 H2 的 PostgreSQL 兼容模式直连验证上述 SQL 语义（不依赖 docker）。libs.h2 由 testRuntimeOnly 提为 testImplementation 以支持该用例。 验证：gradle clean test --offline → 78 tests / 0 failures / 1 skipped （PG Testcontainers 集成用例在本机无 docker 时按既有约定 assumeTrue 跳过）。
This commit is contained in:
windyboy
@@ -0,0 +1,170 @@
|
||||
package com.gzzn.omms.msgexchange.processing
|
||||
|
||||
import com.gzzn.omms.msgexchange.config.MailboxProps
|
||||
import com.gzzn.omms.msgexchange.config.PipelineProps
|
||||
import com.gzzn.omms.msgexchange.domain.ErrorClass
|
||||
import com.gzzn.omms.msgexchange.domain.ProcStatus
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.MailboxRow
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubInbox
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertFalse
|
||||
import org.junit.jupiter.api.Assertions.assertNotNull
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.time.Clock
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
import java.time.ZoneOffset
|
||||
|
||||
/**
|
||||
* 回填通道不变量（docs/message-lifecycle.md §3/§4/§5.2/§11）：
|
||||
* 终态即刻补写、失败退避重试、超期期限 R 覆盖退避、标记单调只写空标、
|
||||
* 中间态不适用、回填失败不回改终态、重启后扫描不依赖内存状态。
|
||||
*/
|
||||
class BackfillServiceTest {
|
||||
|
||||
private val t0: Instant = Instant.parse("2026-09-08T03:00:00Z")
|
||||
private val props = PipelineProps()
|
||||
|
||||
/** 可注入故障的信箱：验证回填失败路径（§3 失败处理）。 */
|
||||
private class FakeMailbox(var fail: Boolean = false) : CminmsgInboxRepository {
|
||||
val marked = linkedSetOf<Long>()
|
||||
override fun insertRaw(rawXml: String): Long = 1L
|
||||
override fun rawOf(msgId: Long): String? = null
|
||||
override fun readRange(fromExclusive: Long, limit: Int): List<MailboxRow> = emptyList()
|
||||
override fun maxId(): Long? = null
|
||||
override fun markProcessedIfUnmarked(msgId: Long, value: String): Boolean {
|
||||
if (fail) throw IllegalStateException("mysql-down")
|
||||
return marked.add(msgId)
|
||||
}
|
||||
}
|
||||
|
||||
private fun service(proc: StubProcState, mailbox: CminmsgInboxRepository, now: Instant = t0) =
|
||||
BackfillService(proc, mailbox, MailboxProps(), props, Clock.fixed(now, ZoneOffset.UTC))
|
||||
|
||||
/** 终态 + 回填意图（固定时刻，避免依赖真实时钟）。 */
|
||||
private fun succeeded(proc: StubProcState, id: Long) {
|
||||
proc.insertIfAbsent(id, t0)
|
||||
proc.markTerminal(id, ProcStatus.SUCCEEDED, now = t0)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `terminal message is marked immediately and the intent is cleared`() {
|
||||
val proc = StubProcState()
|
||||
val inbox = StubInbox().apply { clear() }
|
||||
val id = inbox.insertRaw("<MSG/>")
|
||||
succeeded(proc, id)
|
||||
|
||||
service(proc, inbox).attempt(id)
|
||||
|
||||
assertEquals("PROCESSED", inbox.markOf(id))
|
||||
val row = proc.find(id)!!
|
||||
assertNotNull(row.backfillAt)
|
||||
assertNull(row.backfillNextAt)
|
||||
assertNull(row.backfillError)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `marking is monotonic - an existing value is never overwritten`() {
|
||||
val inbox = StubInbox().apply { clear() }
|
||||
val id = inbox.insertRaw("<MSG/>")
|
||||
|
||||
assertTrue(inbox.markProcessedIfUnmarked(id, "PROCESSED"))
|
||||
assertFalse(inbox.markProcessedIfUnmarked(id, "OTHER")) // §11 只把空标写为已处理
|
||||
assertEquals("PROCESSED", inbox.markOf(id))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `already marked rows are idempotent and never recorded as failures`() {
|
||||
val proc = StubProcState()
|
||||
val inbox = StubInbox().apply { clear() }
|
||||
val id = inbox.insertRaw("<MSG/>")
|
||||
succeeded(proc, id)
|
||||
|
||||
val backfill = service(proc, inbox)
|
||||
backfill.attempt(id)
|
||||
backfill.attempt(id) // 重复执行无副作用（§5.2 补写只针对空标记）
|
||||
|
||||
assertNull(proc.find(id)!!.backfillError)
|
||||
assertEquals(0, proc.find(id)!!.backfillAttempts)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `failed attempt records backoff and never touches the terminal state`() {
|
||||
val proc = StubProcState()
|
||||
val mailbox = FakeMailbox(fail = true)
|
||||
succeeded(proc, 901L)
|
||||
|
||||
service(proc, mailbox).attempt(901L)
|
||||
|
||||
val row = proc.find(901L)!!
|
||||
assertEquals(ProcStatus.SUCCEEDED, row.state) // §11 终态不可逆
|
||||
assertEquals(1, row.backfillAttempts)
|
||||
assertEquals("mysql-down", row.backfillError)
|
||||
assertEquals(t0.plus(Duration.ofSeconds(30)), row.backfillNextAt)
|
||||
assertNull(row.backfillAt)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `sweep retries due rows and completes once the mailbox recovers`() {
|
||||
val proc = StubProcState()
|
||||
val mailbox = FakeMailbox(fail = true)
|
||||
succeeded(proc, 901L)
|
||||
val backfill = service(proc, mailbox)
|
||||
|
||||
assertEquals(1, backfill.sweep(t0)) // 到期 → 失败 → 退避
|
||||
assertEquals(0, backfill.sweep(t0.plusSeconds(29))) // 未到期
|
||||
assertEquals(1, backfill.sweep(t0.plusSeconds(30))) // 到期再试 → 仍失败（重启后同样收敛）
|
||||
|
||||
mailbox.fail = false
|
||||
assertEquals(1, backfill.sweep(t0.plusSeconds(90)))
|
||||
|
||||
assertTrue(901L in mailbox.marked)
|
||||
assertNotNull(proc.find(901L)!!.backfillAt)
|
||||
}
|
||||
|
||||
/** §5.2：超期期限 R 覆盖退避，保证库方清除前提「边界内无未标记行」在有限时间内成立。 */
|
||||
@Test
|
||||
fun `overdue rows bypass the retry backoff`() {
|
||||
val proc = StubProcState()
|
||||
val inbox = StubInbox().apply { clear() }
|
||||
val id = inbox.insertRaw("<MSG/>")
|
||||
val old = t0.minus(props.pipeline.overdueBackfill).minusSeconds(60)
|
||||
proc.insertIfAbsent(id, old) // 接收时间早于 R
|
||||
proc.markTerminal(id, ProcStatus.SUCCEEDED, now = t0)
|
||||
proc.recordBackfillFailure(id, "mysql-down", 5, t0.plus(Duration.ofMinutes(15)), t0) // 退避推到很远之后
|
||||
|
||||
assertEquals(1, service(proc, inbox).sweep(t0))
|
||||
|
||||
assertNotNull(proc.find(id)!!.backfillAt)
|
||||
}
|
||||
|
||||
/** §5.2：中间态（PENDING / FAILED）不适用超期补写——处理未完成时不打标。 */
|
||||
@Test
|
||||
fun `mid states are never marked even when far past the deadline`() {
|
||||
val proc = StubProcState()
|
||||
val inbox = StubInbox().apply { clear() }
|
||||
val pending = inbox.insertRaw("<MSG/>")
|
||||
val failed = inbox.insertRaw("<MSG/>")
|
||||
val old = t0.minus(props.pipeline.overdueBackfill).minusSeconds(60)
|
||||
proc.insertIfAbsent(pending, old)
|
||||
proc.insertIfAbsent(failed, old)
|
||||
proc.update(failed, ProcStatus.FAILED, errorClass = ErrorClass.INFRA)
|
||||
|
||||
assertEquals(0, service(proc, inbox).sweep(t0))
|
||||
|
||||
assertFalse(inbox.isMarked(pending))
|
||||
assertFalse(inbox.isMarked(failed))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `backoff delay doubles per attempt and caps at fifteen minutes`() {
|
||||
assertEquals(Duration.ofSeconds(30), BackfillService.backoffDelayFor(1))
|
||||
assertEquals(Duration.ofMinutes(2), BackfillService.backoffDelayFor(3))
|
||||
assertEquals(Duration.ofMinutes(15), BackfillService.backoffDelayFor(10))
|
||||
assertEquals(Duration.ofMinutes(15), BackfillService.backoffDelayFor(50))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user