2026-09-10 10:59:58 +08:00
|
|
|
package com.gzzn.omms.msgexchange.processing
|
|
|
|
|
|
|
|
|
|
import com.gzzn.omms.msgexchange.config.MailboxProps
|
|
|
|
|
import com.gzzn.omms.msgexchange.config.PipelineProps
|
|
|
|
|
import com.gzzn.omms.msgexchange.domain.ErrorClass
|
|
|
|
|
import com.gzzn.omms.msgexchange.domain.ProcStatus
|
|
|
|
|
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
|
|
|
|
|
import com.gzzn.omms.msgexchange.infra.persistence.MailboxRow
|
|
|
|
|
import com.gzzn.omms.msgexchange.infra.stub.StubInbox
|
|
|
|
|
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
|
|
|
|
|
import org.junit.jupiter.api.Assertions.assertEquals
|
|
|
|
|
import org.junit.jupiter.api.Assertions.assertFalse
|
|
|
|
|
import org.junit.jupiter.api.Assertions.assertNotNull
|
|
|
|
|
import org.junit.jupiter.api.Assertions.assertNull
|
|
|
|
|
import org.junit.jupiter.api.Assertions.assertTrue
|
|
|
|
|
import org.junit.jupiter.api.Test
|
|
|
|
|
import java.time.Clock
|
|
|
|
|
import java.time.Duration
|
|
|
|
|
import java.time.Instant
|
|
|
|
|
import java.time.ZoneOffset
|
|
|
|
|
|
|
|
|
|
/**
|
2026-09-10 11:05:55 +08:00
|
|
|
* 回填环节的规矩:
|
|
|
|
|
* - 处理完马上写标记,写不进去就按退避重试,重启后接着重试(状态都在数据库里);
|
|
|
|
|
* - 等得太久的消息无视退避强制补写,保证标记最终一定会写上;
|
|
|
|
|
* - 只写还没有标记的行,已有的值不覆盖;
|
|
|
|
|
* - 还没处理完的消息(PENDING / FAILED)不许写标记;
|
|
|
|
|
* - 回填失败只是回填的事,不会把处理结果改回去。
|
2026-09-10 10:59:58 +08:00
|
|
|
*/
|
|
|
|
|
class BackfillServiceTest {
|
|
|
|
|
|
|
|
|
|
private val t0: Instant = Instant.parse("2026-09-08T03:00:00Z")
|
|
|
|
|
private val props = PipelineProps()
|
|
|
|
|
|
|
|
|
|
/** 可注入故障的信箱:验证回填失败路径(§3 失败处理)。 */
|
|
|
|
|
private class FakeMailbox(var fail: Boolean = false) : CminmsgInboxRepository {
|
|
|
|
|
val marked = linkedSetOf<Long>()
|
|
|
|
|
override fun insertRaw(rawXml: String): Long = 1L
|
|
|
|
|
override fun rawOf(msgId: Long): String? = null
|
|
|
|
|
override fun readRange(fromExclusive: Long, limit: Int): List<MailboxRow> = emptyList()
|
|
|
|
|
override fun maxId(): Long? = null
|
|
|
|
|
override fun markProcessedIfUnmarked(msgId: Long, value: String): Boolean {
|
|
|
|
|
if (fail) throw IllegalStateException("mysql-down")
|
|
|
|
|
return marked.add(msgId)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private fun service(proc: StubProcState, mailbox: CminmsgInboxRepository, now: Instant = t0) =
|
|
|
|
|
BackfillService(proc, mailbox, MailboxProps(), props, Clock.fixed(now, ZoneOffset.UTC))
|
|
|
|
|
|
|
|
|
|
/** 终态 + 回填意图(固定时刻,避免依赖真实时钟)。 */
|
|
|
|
|
private fun succeeded(proc: StubProcState, id: Long) {
|
|
|
|
|
proc.insertIfAbsent(id, t0)
|
|
|
|
|
proc.markTerminal(id, ProcStatus.SUCCEEDED, now = t0)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
fun `terminal message is marked immediately and the intent is cleared`() {
|
|
|
|
|
val proc = StubProcState()
|
|
|
|
|
val inbox = StubInbox().apply { clear() }
|
|
|
|
|
val id = inbox.insertRaw("<MSG/>")
|
|
|
|
|
succeeded(proc, id)
|
|
|
|
|
|
|
|
|
|
service(proc, inbox).attempt(id)
|
|
|
|
|
|
|
|
|
|
assertEquals("PROCESSED", inbox.markOf(id))
|
|
|
|
|
val row = proc.find(id)!!
|
|
|
|
|
assertNotNull(row.backfillAt)
|
|
|
|
|
assertNull(row.backfillNextAt)
|
|
|
|
|
assertNull(row.backfillError)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
fun `marking is monotonic - an existing value is never overwritten`() {
|
|
|
|
|
val inbox = StubInbox().apply { clear() }
|
|
|
|
|
val id = inbox.insertRaw("<MSG/>")
|
|
|
|
|
|
|
|
|
|
assertTrue(inbox.markProcessedIfUnmarked(id, "PROCESSED"))
|
|
|
|
|
assertFalse(inbox.markProcessedIfUnmarked(id, "OTHER")) // §11 只把空标写为已处理
|
|
|
|
|
assertEquals("PROCESSED", inbox.markOf(id))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
fun `already marked rows are idempotent and never recorded as failures`() {
|
|
|
|
|
val proc = StubProcState()
|
|
|
|
|
val inbox = StubInbox().apply { clear() }
|
|
|
|
|
val id = inbox.insertRaw("<MSG/>")
|
|
|
|
|
succeeded(proc, id)
|
|
|
|
|
|
|
|
|
|
val backfill = service(proc, inbox)
|
|
|
|
|
backfill.attempt(id)
|
|
|
|
|
backfill.attempt(id) // 重复执行无副作用(§5.2 补写只针对空标记)
|
|
|
|
|
|
|
|
|
|
assertNull(proc.find(id)!!.backfillError)
|
|
|
|
|
assertEquals(0, proc.find(id)!!.backfillAttempts)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
fun `failed attempt records backoff and never touches the terminal state`() {
|
|
|
|
|
val proc = StubProcState()
|
|
|
|
|
val mailbox = FakeMailbox(fail = true)
|
|
|
|
|
succeeded(proc, 901L)
|
|
|
|
|
|
|
|
|
|
service(proc, mailbox).attempt(901L)
|
|
|
|
|
|
|
|
|
|
val row = proc.find(901L)!!
|
|
|
|
|
assertEquals(ProcStatus.SUCCEEDED, row.state) // §11 终态不可逆
|
|
|
|
|
assertEquals(1, row.backfillAttempts)
|
|
|
|
|
assertEquals("mysql-down", row.backfillError)
|
|
|
|
|
assertEquals(t0.plus(Duration.ofSeconds(30)), row.backfillNextAt)
|
|
|
|
|
assertNull(row.backfillAt)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
fun `sweep retries due rows and completes once the mailbox recovers`() {
|
|
|
|
|
val proc = StubProcState()
|
|
|
|
|
val mailbox = FakeMailbox(fail = true)
|
|
|
|
|
succeeded(proc, 901L)
|
|
|
|
|
val backfill = service(proc, mailbox)
|
|
|
|
|
|
|
|
|
|
assertEquals(1, backfill.sweep(t0)) // 到期 → 失败 → 退避
|
|
|
|
|
assertEquals(0, backfill.sweep(t0.plusSeconds(29))) // 未到期
|
|
|
|
|
assertEquals(1, backfill.sweep(t0.plusSeconds(30))) // 到期再试 → 仍失败(重启后同样收敛)
|
|
|
|
|
|
|
|
|
|
mailbox.fail = false
|
|
|
|
|
assertEquals(1, backfill.sweep(t0.plusSeconds(90)))
|
|
|
|
|
|
|
|
|
|
assertTrue(901L in mailbox.marked)
|
|
|
|
|
assertNotNull(proc.find(901L)!!.backfillAt)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** §5.2:超期期限 R 覆盖退避,保证库方清除前提「边界内无未标记行」在有限时间内成立。 */
|
|
|
|
|
@Test
|
|
|
|
|
fun `overdue rows bypass the retry backoff`() {
|
|
|
|
|
val proc = StubProcState()
|
|
|
|
|
val inbox = StubInbox().apply { clear() }
|
|
|
|
|
val id = inbox.insertRaw("<MSG/>")
|
|
|
|
|
val old = t0.minus(props.pipeline.overdueBackfill).minusSeconds(60)
|
|
|
|
|
proc.insertIfAbsent(id, old) // 接收时间早于 R
|
|
|
|
|
proc.markTerminal(id, ProcStatus.SUCCEEDED, now = t0)
|
|
|
|
|
proc.recordBackfillFailure(id, "mysql-down", 5, t0.plus(Duration.ofMinutes(15)), t0) // 退避推到很远之后
|
|
|
|
|
|
|
|
|
|
assertEquals(1, service(proc, inbox).sweep(t0))
|
|
|
|
|
|
|
|
|
|
assertNotNull(proc.find(id)!!.backfillAt)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** §5.2:中间态(PENDING / FAILED)不适用超期补写——处理未完成时不打标。 */
|
|
|
|
|
@Test
|
|
|
|
|
fun `mid states are never marked even when far past the deadline`() {
|
|
|
|
|
val proc = StubProcState()
|
|
|
|
|
val inbox = StubInbox().apply { clear() }
|
|
|
|
|
val pending = inbox.insertRaw("<MSG/>")
|
|
|
|
|
val failed = inbox.insertRaw("<MSG/>")
|
|
|
|
|
val old = t0.minus(props.pipeline.overdueBackfill).minusSeconds(60)
|
|
|
|
|
proc.insertIfAbsent(pending, old)
|
|
|
|
|
proc.insertIfAbsent(failed, old)
|
|
|
|
|
proc.update(failed, ProcStatus.FAILED, errorClass = ErrorClass.INFRA)
|
|
|
|
|
|
|
|
|
|
assertEquals(0, service(proc, inbox).sweep(t0))
|
|
|
|
|
|
|
|
|
|
assertFalse(inbox.isMarked(pending))
|
|
|
|
|
assertFalse(inbox.isMarked(failed))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
fun `backoff delay doubles per attempt and caps at fifteen minutes`() {
|
|
|
|
|
assertEquals(Duration.ofSeconds(30), BackfillService.backoffDelayFor(1))
|
|
|
|
|
assertEquals(Duration.ofMinutes(2), BackfillService.backoffDelayFor(3))
|
|
|
|
|
assertEquals(Duration.ofMinutes(15), BackfillService.backoffDelayFor(10))
|
|
|
|
|
assertEquals(Duration.ofMinutes(15), BackfillService.backoffDelayFor(50))
|
|
|
|
|
}
|
|
|
|
|
}
|