package com.gzzn.omms.msgexchange.processing import com.gzzn.omms.msgexchange.config.MailboxProps import com.gzzn.omms.msgexchange.config.PipelineProps import com.gzzn.omms.msgexchange.domain.ErrorClass import com.gzzn.omms.msgexchange.domain.ProcStatus import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository import com.gzzn.omms.msgexchange.infra.persistence.MailboxRow import com.gzzn.omms.msgexchange.infra.persistence.MailboxMarkResult import com.gzzn.omms.msgexchange.infra.stub.StubInbox import com.gzzn.omms.msgexchange.infra.stub.StubProcState import com.gzzn.omms.msgexchange.infra.retry.ReplayService import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Assertions.assertThrows import org.junit.jupiter.api.Test import java.time.Clock import java.time.Duration import java.time.Instant import java.time.ZoneOffset import java.util.concurrent.CountDownLatch import java.util.concurrent.Callable import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import java.util.concurrent.TimeoutException /** * 回填环节的规矩: * - 处理完马上写标记,写不进去就按退避重试,重启后接着重试(状态都在数据库里); * - 等得太久的消息无视退避强制补写,保证标记最终一定会写上; * - 只写还没有标记的行,已有的值不覆盖; * - 还没处理完的消息(PENDING / FAILED)不许写标记; * - 回填失败只是回填的事,不会把处理结果改回去。 */ class BackfillServiceTest { private val t0: Instant = Instant.parse("2026-09-08T03:00:00Z") private val props = PipelineProps() /** 可以人为制造故障的信箱,用来验证回填失败时怎么处理。 */ private class FakeMailbox(var fail: Boolean = false) : CminmsgInboxRepository { val marked = linkedSetOf() override fun insertRaw(rawXml: String): Long = 1L override fun rawOf(msgId: Long): String? = null override fun receivedAtOf(msgId: Long): Instant? = null override fun readRange(fromExclusive: Long, limit: Int): List = emptyList() override fun maxId(): Long? = null override fun markProcessedIfUnmarked(msgId: Long, value: String): MailboxMarkResult { if (fail) throw IllegalStateException("mysql-down") return if (marked.add(msgId)) MailboxMarkResult.MARKED else MailboxMarkResult.ALREADY_MARKED } } private fun service(proc: StubProcState, mailbox: CminmsgInboxRepository, now: Instant = t0) = BackfillService(proc, mailbox, MailboxProps(), props, Clock.fixed(now, ZoneOffset.UTC)) /** 终态 + 回填意图(固定时刻,避免依赖真实时钟)。 */ private fun succeeded(proc: StubProcState, id: Long) { proc.insertIfAbsent(id, t0) proc.markTerminal(id, ProcStatus.SUCCEEDED, now = t0) } @Test fun `terminal message is marked immediately and the intent is cleared`() { val proc = StubProcState() val inbox = StubInbox().apply { clear() } val id = inbox.insertRaw("") succeeded(proc, id) service(proc, inbox).attempt(id) assertEquals("PROCESSED", inbox.markOf(id)) val row = proc.find(id)!! assertNotNull(row.backfillAt) assertNull(row.backfillNextAt) assertNull(row.backfillError) } @Test fun `marking is monotonic - an existing value is never overwritten`() { val inbox = StubInbox().apply { clear() } val id = inbox.insertRaw("") assertEquals(MailboxMarkResult.MARKED, inbox.markProcessedIfUnmarked(id, "PROCESSED")) assertEquals(MailboxMarkResult.ALREADY_MARKED, inbox.markProcessedIfUnmarked(id, "OTHER")) assertEquals("PROCESSED", inbox.markOf(id)) } @Test fun `already marked rows are idempotent and never recorded as failures`() { val proc = StubProcState() val inbox = StubInbox().apply { clear() } val id = inbox.insertRaw("") succeeded(proc, id) val backfill = service(proc, inbox) backfill.attempt(id) val firstCompletion = proc.find(id)!!.backfillAt backfill.attempt(id) // 重复补写没有副作用 assertNull(proc.find(id)!!.backfillError) assertEquals(0, proc.find(id)!!.backfillAttempts) assertEquals(firstCompletion, proc.find(id)!!.backfillAt) } @Test fun `failed attempt records backoff and never touches the terminal state`() { val proc = StubProcState() val mailbox = FakeMailbox(fail = true) succeeded(proc, 901L) service(proc, mailbox).attempt(901L) val row = proc.find(901L)!! assertEquals(ProcStatus.SUCCEEDED, row.state) // 回填失败不会把处理结果改回去 assertEquals(1, row.backfillAttempts) assertEquals("mysql-down", row.backfillError) assertEquals(t0.plus(Duration.ofSeconds(30)), row.backfillNextAt) assertNull(row.backfillAt) } @Test fun `missing mailbox row remains an unconfirmed backfill failure`() { val proc = StubProcState() val inbox = StubInbox().apply { clear() } val id = inbox.insertRaw("") succeeded(proc, id) inbox.removeRow(id) service(proc, inbox).attempt(id) val row = proc.find(id)!! assertNull(row.backfillAt) assertEquals(1, row.backfillAttempts) assertEquals("mailbox-row-missing", row.backfillError) assertNotNull(row.backfillNextAt) } @Test fun `sweep retries due rows and completes once the mailbox recovers`() { val proc = StubProcState() val mailbox = FakeMailbox(fail = true) succeeded(proc, 901L) val backfill = service(proc, mailbox) assertEquals(1, backfill.sweep(t0)) // 到期 → 失败 → 退避 assertEquals(0, backfill.sweep(t0.plusSeconds(29))) // 未到期 assertEquals(1, backfill.sweep(t0.plusSeconds(30))) // 到期再试 → 仍失败(重启后同样收敛) mailbox.fail = false assertEquals(1, backfill.sweep(t0.plusSeconds(90))) assertTrue(901L in mailbox.marked) assertNotNull(proc.find(901L)!!.backfillAt) } /** 等得太久的消息不再等退避、直接补写:保证标记最终一定会写上。 */ @Test fun `overdue rows bypass the retry backoff`() { val proc = StubProcState() val inbox = StubInbox().apply { clear() } val id = inbox.insertRaw("") val old = t0.minus(props.pipeline.overdueBackfill).minusSeconds(60) proc.insertIfAbsent(id, old) // 接收时间早于 R proc.markTerminal(id, ProcStatus.SUCCEEDED, now = t0) proc.recordBackfillFailure(id, "mysql-down", 5, t0.plus(Duration.ofMinutes(15)), t0) // 退避推到很远之后 assertEquals(1, service(proc, inbox).sweep(t0)) assertNotNull(proc.find(id)!!.backfillAt) } /** 还没处理完的消息(PENDING / FAILED)永远不打标,等再久也不行。 */ @Test fun `mid states are never marked even when far past the deadline`() { val proc = StubProcState() val inbox = StubInbox().apply { clear() } val pending = inbox.insertRaw("") val failed = inbox.insertRaw("") val old = t0.minus(props.pipeline.overdueBackfill).minusSeconds(60) proc.insertIfAbsent(pending, old) proc.insertIfAbsent(failed, old) proc.update(failed, ProcStatus.FAILED, errorClass = ErrorClass.INFRA) assertEquals(0, service(proc, inbox).sweep(t0)) assertFalse(inbox.isMarked(pending)) assertFalse(inbox.isMarked(failed)) } @Test fun `immediate attempt also refuses pending and failed messages`() { val proc = StubProcState() val inbox = StubInbox().apply { clear() } val pending = inbox.insertRaw("") val failed = inbox.insertRaw("") proc.insertIfAbsent(pending, t0) proc.insertIfAbsent(failed, t0) proc.update(failed, ProcStatus.FAILED, errorClass = ErrorClass.INFRA) service(proc, inbox).attempt(pending) service(proc, inbox).attempt(failed) assertFalse(inbox.isMarked(pending)) assertFalse(inbox.isMarked(failed)) assertNull(proc.find(pending)!!.backfillAt) assertNull(proc.find(failed)!!.backfillAt) } @Test fun `backoff delay doubles per attempt and caps at fifteen minutes`() { assertEquals(Duration.ofSeconds(30), BackfillService.backoffDelayFor(1)) assertEquals(Duration.ofMinutes(2), BackfillService.backoffDelayFor(3)) assertEquals(Duration.ofMinutes(15), BackfillService.backoffDelayFor(10)) assertEquals(Duration.ofMinutes(15), BackfillService.backoffDelayFor(50)) } @Test fun `replay waits for an in-flight backfill before reopening the message`() { val entered = CountDownLatch(1) val release = CountDownLatch(1) val replayStarted = CountDownLatch(1) val mailbox = object : CminmsgInboxRepository { override fun insertRaw(rawXml: String) = 1L override fun rawOf(msgId: Long): String? = "" override fun receivedAtOf(msgId: Long) = t0 override fun readRange(fromExclusive: Long, limit: Int) = emptyList() override fun maxId(): Long? = 1L override fun markProcessedIfUnmarked(msgId: Long, value: String): MailboxMarkResult { entered.countDown() release.await() return MailboxMarkResult.MARKED } } val proc = StubProcState().apply { insertIfAbsent(1L, t0) markTerminal(1L, ProcStatus.DEAD, errorClass = ErrorClass.EXHAUSTED, now = t0) } val gate = MessageLifecycleGate() val backfill = BackfillService(proc, mailbox, MailboxProps(), props, Clock.fixed(t0, ZoneOffset.UTC), gate) val replay = ReplayService(proc, gate) val pool = Executors.newFixedThreadPool(2) try { val backfillTask = pool.submit { backfill.attempt(1L) } assertTrue(entered.await(1, TimeUnit.SECONDS)) val replayTask = pool.submit(Callable { replayStarted.countDown() replay.replay(listOf(ErrorClass.EXHAUSTED)) }) assertTrue(replayStarted.await(1, TimeUnit.SECONDS)) assertThrows(TimeoutException::class.java) { replayTask.get(100, TimeUnit.MILLISECONDS) } release.countDown() backfillTask.get(1, TimeUnit.SECONDS) assertEquals(1, replayTask.get(1, TimeUnit.SECONDS)) assertEquals(ProcStatus.PENDING, proc.find(1L)!!.state) } finally { release.countDown() pool.shutdownNow() } } }