package com.gzzn.omms.msgexchange.processing import com.gzzn.omms.msgexchange.config.MailboxProps import com.gzzn.omms.msgexchange.config.PipelineProps import com.gzzn.omms.msgexchange.domain.ErrorClass import com.gzzn.omms.msgexchange.domain.ProcStatus import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository import com.gzzn.omms.msgexchange.infra.persistence.MailboxRow import com.gzzn.omms.msgexchange.infra.stub.StubInbox import com.gzzn.omms.msgexchange.infra.stub.StubProcState import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import java.time.Clock import java.time.Duration import java.time.Instant import java.time.ZoneOffset /** * 回填环节的规矩: * - 处理完马上写标记,写不进去就按退避重试,重启后接着重试(状态都在数据库里); * - 等得太久的消息无视退避强制补写,保证标记最终一定会写上; * - 只写还没有标记的行,已有的值不覆盖; * - 还没处理完的消息(PENDING / FAILED)不许写标记; * - 回填失败只是回填的事,不会把处理结果改回去。 */ class BackfillServiceTest { private val t0: Instant = Instant.parse("2026-09-08T03:00:00Z") private val props = PipelineProps() /** 可以人为制造故障的信箱,用来验证回填失败时怎么处理。 */ private class FakeMailbox(var fail: Boolean = false) : CminmsgInboxRepository { val marked = linkedSetOf() override fun insertRaw(rawXml: String): Long = 1L override fun rawOf(msgId: Long): String? = null override fun readRange(fromExclusive: Long, limit: Int): List = emptyList() override fun maxId(): Long? = null override fun markProcessedIfUnmarked(msgId: Long, value: String): Boolean { if (fail) throw IllegalStateException("mysql-down") return marked.add(msgId) } } private fun service(proc: StubProcState, mailbox: CminmsgInboxRepository, now: Instant = t0) = BackfillService(proc, mailbox, MailboxProps(), props, Clock.fixed(now, ZoneOffset.UTC)) /** 终态 + 回填意图(固定时刻,避免依赖真实时钟)。 */ private fun succeeded(proc: StubProcState, id: Long) { proc.insertIfAbsent(id, t0) proc.markTerminal(id, ProcStatus.SUCCEEDED, now = t0) } @Test fun `terminal message is marked immediately and the intent is cleared`() { val proc = StubProcState() val inbox = StubInbox().apply { clear() } val id = inbox.insertRaw("") succeeded(proc, id) service(proc, inbox).attempt(id) assertEquals("PROCESSED", inbox.markOf(id)) val row = proc.find(id)!! assertNotNull(row.backfillAt) assertNull(row.backfillNextAt) assertNull(row.backfillError) } @Test fun `marking is monotonic - an existing value is never overwritten`() { val inbox = StubInbox().apply { clear() } val id = inbox.insertRaw("") assertTrue(inbox.markProcessedIfUnmarked(id, "PROCESSED")) assertFalse(inbox.markProcessedIfUnmarked(id, "OTHER")) // 已经有标记了,不再写第二次 assertEquals("PROCESSED", inbox.markOf(id)) } @Test fun `already marked rows are idempotent and never recorded as failures`() { val proc = StubProcState() val inbox = StubInbox().apply { clear() } val id = inbox.insertRaw("") succeeded(proc, id) val backfill = service(proc, inbox) backfill.attempt(id) backfill.attempt(id) // 重复补写没有副作用 assertNull(proc.find(id)!!.backfillError) assertEquals(0, proc.find(id)!!.backfillAttempts) } @Test fun `failed attempt records backoff and never touches the terminal state`() { val proc = StubProcState() val mailbox = FakeMailbox(fail = true) succeeded(proc, 901L) service(proc, mailbox).attempt(901L) val row = proc.find(901L)!! assertEquals(ProcStatus.SUCCEEDED, row.state) // 回填失败不会把处理结果改回去 assertEquals(1, row.backfillAttempts) assertEquals("mysql-down", row.backfillError) assertEquals(t0.plus(Duration.ofSeconds(30)), row.backfillNextAt) assertNull(row.backfillAt) } @Test fun `sweep retries due rows and completes once the mailbox recovers`() { val proc = StubProcState() val mailbox = FakeMailbox(fail = true) succeeded(proc, 901L) val backfill = service(proc, mailbox) assertEquals(1, backfill.sweep(t0)) // 到期 → 失败 → 退避 assertEquals(0, backfill.sweep(t0.plusSeconds(29))) // 未到期 assertEquals(1, backfill.sweep(t0.plusSeconds(30))) // 到期再试 → 仍失败(重启后同样收敛) mailbox.fail = false assertEquals(1, backfill.sweep(t0.plusSeconds(90))) assertTrue(901L in mailbox.marked) assertNotNull(proc.find(901L)!!.backfillAt) } /** 等得太久的消息不再等退避、直接补写:保证标记最终一定会写上。 */ @Test fun `overdue rows bypass the retry backoff`() { val proc = StubProcState() val inbox = StubInbox().apply { clear() } val id = inbox.insertRaw("") val old = t0.minus(props.pipeline.overdueBackfill).minusSeconds(60) proc.insertIfAbsent(id, old) // 接收时间早于 R proc.markTerminal(id, ProcStatus.SUCCEEDED, now = t0) proc.recordBackfillFailure(id, "mysql-down", 5, t0.plus(Duration.ofMinutes(15)), t0) // 退避推到很远之后 assertEquals(1, service(proc, inbox).sweep(t0)) assertNotNull(proc.find(id)!!.backfillAt) } /** 还没处理完的消息(PENDING / FAILED)永远不打标,等再久也不行。 */ @Test fun `mid states are never marked even when far past the deadline`() { val proc = StubProcState() val inbox = StubInbox().apply { clear() } val pending = inbox.insertRaw("") val failed = inbox.insertRaw("") val old = t0.minus(props.pipeline.overdueBackfill).minusSeconds(60) proc.insertIfAbsent(pending, old) proc.insertIfAbsent(failed, old) proc.update(failed, ProcStatus.FAILED, errorClass = ErrorClass.INFRA) assertEquals(0, service(proc, inbox).sweep(t0)) assertFalse(inbox.isMarked(pending)) assertFalse(inbox.isMarked(failed)) } @Test fun `backoff delay doubles per attempt and caps at fifteen minutes`() { assertEquals(Duration.ofSeconds(30), BackfillService.backoffDelayFor(1)) assertEquals(Duration.ofMinutes(2), BackfillService.backoffDelayFor(3)) assertEquals(Duration.ofMinutes(15), BackfillService.backoffDelayFor(10)) assertEquals(Duration.ofMinutes(15), BackfillService.backoffDelayFor(50)) } }