2026-09-07 15:11:33 +08:00
|
|
|
|
package com.gzzn.omms.msgexchange.delivery
|
2026-09-06 18:19:04 +08:00
|
|
|
|
|
2026-09-07 15:11:33 +08:00
|
|
|
|
import com.gzzn.omms.msgexchange.MutableClock
|
|
|
|
|
|
import com.gzzn.omms.msgexchange.config.PipelineProps
|
|
|
|
|
|
import com.gzzn.omms.msgexchange.domain.ErrorClass
|
|
|
|
|
|
import com.gzzn.omms.msgexchange.domain.EventStatus
|
|
|
|
|
|
import com.gzzn.omms.msgexchange.domain.MsgEvent
|
|
|
|
|
|
import com.gzzn.omms.msgexchange.domain.Targets
|
|
|
|
|
|
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
|
|
|
|
|
|
import com.gzzn.omms.msgexchange.infra.retry.FailureScheduler
|
2026-09-06 18:19:04 +08:00
|
|
|
|
import org.junit.jupiter.api.Assertions.assertEquals
|
2026-09-06 19:08:02 +08:00
|
|
|
|
import org.junit.jupiter.api.Assertions.assertNotNull
|
|
|
|
|
|
import org.junit.jupiter.api.Assertions.assertNull
|
2026-09-06 18:19:04 +08:00
|
|
|
|
import org.junit.jupiter.api.Assertions.assertTrue
|
|
|
|
|
|
import org.junit.jupiter.api.Test
|
|
|
|
|
|
import java.time.Instant
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
2026-09-06 19:08:02 +08:00
|
|
|
|
* U06(N03)+ U08 闭环:schd 出口只走 flushSchd;
|
|
|
|
|
|
* 批发送失败 → 整批 attempts 递增 + 指数退避(队首未到期不 claim),达上限整批 DEAD/DLQ;
|
|
|
|
|
|
* 时间经可注入 Clock(MutableClock),不依赖真实睡眠。
|
2026-09-06 18:19:04 +08:00
|
|
|
|
*/
|
|
|
|
|
|
class DispatcherTickTest {
|
|
|
|
|
|
|
|
|
|
|
|
private class FakeRepo : MsgEventRepository {
|
|
|
|
|
|
val events = mutableListOf<MsgEvent>()
|
|
|
|
|
|
|
|
|
|
|
|
fun enqueue(e: MsgEvent) { events += e }
|
|
|
|
|
|
|
|
|
|
|
|
override fun insertAll(events: List<MsgEvent>) = events.map { it.eventId ?: 0L }
|
|
|
|
|
|
|
|
|
|
|
|
override fun headUnsent(target: String): MsgEvent? =
|
|
|
|
|
|
events.filter { it.target == target && it.state != EventStatus.SENT && it.state != EventStatus.DEAD }
|
|
|
|
|
|
.minByOrNull { it.eventId ?: Long.MAX_VALUE }
|
|
|
|
|
|
|
|
|
|
|
|
override fun claimBatch(target: String, limit: Int): List<MsgEvent> =
|
|
|
|
|
|
events.filter { it.target == target && it.state == EventStatus.PENDING }
|
|
|
|
|
|
.sortedBy { it.eventId ?: Long.MAX_VALUE }
|
|
|
|
|
|
.take(limit)
|
|
|
|
|
|
|
|
|
|
|
|
override fun markSent(eventId: Long) {
|
|
|
|
|
|
val i = events.indexOfFirst { it.eventId == eventId }
|
|
|
|
|
|
if (i >= 0) events[i] = events[i].copy(state = EventStatus.SENT)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
override fun markAllSent(eventIds: List<Long>) = eventIds.forEach(::markSent)
|
|
|
|
|
|
|
|
|
|
|
|
override fun scheduleRetry(eventId: Long, nextAttemptAt: Instant, attempts: Int) {
|
|
|
|
|
|
val i = events.indexOfFirst { it.eventId == eventId }
|
|
|
|
|
|
if (i >= 0) events[i] = events[i].copy(state = EventStatus.PENDING, attempts = attempts, nextAttemptAt = nextAttemptAt)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-09-06 19:08:02 +08:00
|
|
|
|
override fun markDead(eventId: Long, errorClass: ErrorClass, lastError: String, attempts: Int?) {
|
2026-09-06 18:19:04 +08:00
|
|
|
|
val i = events.indexOfFirst { it.eventId == eventId }
|
2026-09-06 19:08:02 +08:00
|
|
|
|
if (i >= 0) events[i] = events[i].copy(state = EventStatus.DEAD, errorClass = errorClass, lastError = lastError,
|
|
|
|
|
|
attempts = attempts ?: events[i].attempts)
|
2026-09-06 18:19:04 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-09-06 22:12:23 +08:00
|
|
|
|
override fun insertSync(events: List<MsgEvent>) { syncInserted += events }
|
|
|
|
|
|
|
|
|
|
|
|
val syncInserted = mutableListOf<MsgEvent>()
|
2026-09-06 18:19:04 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private class FakePort : DeliveryPort {
|
|
|
|
|
|
val sent = mutableListOf<Pair<String, String>>()
|
|
|
|
|
|
var failTopic: String? = null
|
|
|
|
|
|
|
|
|
|
|
|
override fun sendKafka(topic: String, payloadJson: String) {
|
|
|
|
|
|
if (failTopic == topic) throw RuntimeException("send-fail:$topic")
|
|
|
|
|
|
sent += topic to payloadJson
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
override fun indexFlightHts(payloadJson: String) = Unit
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-09-06 19:08:02 +08:00
|
|
|
|
private val clock = MutableClock(MutableClock.BASE)
|
|
|
|
|
|
|
|
|
|
|
|
private fun dispatcher(repo: FakeRepo, port: FakePort, p: PipelineProps = PipelineProps()): Dispatcher =
|
|
|
|
|
|
Dispatcher(repo, port, p, FailureScheduler(p, clock))
|
2026-09-06 18:19:04 +08:00
|
|
|
|
|
|
|
|
|
|
private fun ev(id: Long, target: String, key: String?, payload: String) =
|
|
|
|
|
|
MsgEvent(eventId = id, target = target, partitionKey = key, payloadJson = payload, state = EventStatus.PENDING)
|
|
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
|
fun `schd flushed once via batch path, per-target tick never consumes it`() {
|
|
|
|
|
|
val repo = FakeRepo()
|
|
|
|
|
|
val port = FakePort()
|
2026-09-06 19:08:02 +08:00
|
|
|
|
val d = dispatcher(repo, port)
|
2026-09-06 18:19:04 +08:00
|
|
|
|
repo.enqueue(ev(1, Targets.KAFKA_MSG, null, """{"msg":1}"""))
|
|
|
|
|
|
repo.enqueue(ev(2, Targets.KAFKA_SCHD, "F1", """{"FLID":"F1","v":1}"""))
|
|
|
|
|
|
|
|
|
|
|
|
d.tick() // lastFlush=EPOCH → 首 tick 即到 flush 周期
|
|
|
|
|
|
|
|
|
|
|
|
assertEquals(1, port.sent.count { it.first == "msg" })
|
|
|
|
|
|
assertEquals(1, port.sent.count { it.first == "schd" })
|
|
|
|
|
|
assertEquals(EventStatus.SENT, repo.events.first { it.eventId == 2L }.state)
|
|
|
|
|
|
|
|
|
|
|
|
repo.enqueue(ev(4, Targets.KAFKA_SCHD, "F1", """{"FLID":"F1","v":4}"""))
|
2026-09-06 19:08:02 +08:00
|
|
|
|
d.tick() // 时钟未推进 → flush 周期未到;逐条循环不得消费 schd
|
2026-09-06 18:19:04 +08:00
|
|
|
|
assertEquals(EventStatus.PENDING, repo.events.first { it.eventId == 4L }.state)
|
2026-09-06 19:08:02 +08:00
|
|
|
|
assertEquals(1, port.sent.count { it.first == "schd" })
|
2026-09-06 18:19:04 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
|
fun `flushSchd aggregates latest per flight and marks all sent`() {
|
|
|
|
|
|
val repo = FakeRepo()
|
|
|
|
|
|
val port = FakePort()
|
2026-09-06 19:08:02 +08:00
|
|
|
|
val d = dispatcher(repo, port)
|
2026-09-06 18:19:04 +08:00
|
|
|
|
repo.enqueue(ev(1, Targets.KAFKA_SCHD, "F1", "old"))
|
|
|
|
|
|
repo.enqueue(ev(2, Targets.KAFKA_SCHD, "F2", "only"))
|
|
|
|
|
|
repo.enqueue(ev(3, Targets.KAFKA_SCHD, "F1", "new"))
|
|
|
|
|
|
|
|
|
|
|
|
d.flushSchd()
|
|
|
|
|
|
|
|
|
|
|
|
val payload = port.sent.first { it.first == "schd" }.second
|
|
|
|
|
|
assertTrue(payload.contains("new") && payload.contains("only") && !payload.contains("old"))
|
|
|
|
|
|
assertEquals(2, payload.removeSurrounding("[", "]").split(",").size) // 同 FLID 只发最新(矩阵 #7)
|
|
|
|
|
|
assertTrue(repo.events.all { it.state == EventStatus.SENT })
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@Test
|
2026-09-06 19:08:02 +08:00
|
|
|
|
fun `flush failure schedules per-event backoff and retries after due`() {
|
2026-09-06 18:19:04 +08:00
|
|
|
|
val repo = FakeRepo()
|
|
|
|
|
|
val port = FakePort().apply { failTopic = "schd" }
|
2026-09-06 19:08:02 +08:00
|
|
|
|
val d = dispatcher(repo, port)
|
2026-09-06 18:19:04 +08:00
|
|
|
|
repo.enqueue(ev(1, Targets.KAFKA_SCHD, "F1", """{"v":"1"}"""))
|
|
|
|
|
|
|
2026-09-06 19:08:02 +08:00
|
|
|
|
d.flushSchd()
|
|
|
|
|
|
val e1 = repo.events.single()
|
|
|
|
|
|
assertEquals(EventStatus.PENDING, e1.state)
|
|
|
|
|
|
assertEquals(1, e1.attempts)
|
|
|
|
|
|
assertNotNull(e1.nextAttemptAt) // 指数退避落库(attempts=1 → 首档)
|
2026-09-06 18:19:04 +08:00
|
|
|
|
assertEquals(0, port.sent.size)
|
|
|
|
|
|
|
2026-09-06 19:08:02 +08:00
|
|
|
|
clock.advance(1000) // 退避到期
|
2026-09-06 18:19:04 +08:00
|
|
|
|
port.failTopic = null
|
2026-09-06 19:08:02 +08:00
|
|
|
|
d.flushSchd()
|
2026-09-06 18:19:04 +08:00
|
|
|
|
assertEquals(EventStatus.SENT, repo.events.single().state)
|
|
|
|
|
|
assertEquals(1, port.sent.count { it.first == "schd" })
|
|
|
|
|
|
}
|
2026-09-06 19:08:02 +08:00
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
|
fun `repeated batch failures backoff grows and whole batch goes DEAD DLQ at limit`() {
|
|
|
|
|
|
val props = PipelineProps().apply { pipeline.maxAttempts = 2 }
|
|
|
|
|
|
val repo = FakeRepo()
|
|
|
|
|
|
val port = FakePort().apply { failTopic = "schd" }
|
|
|
|
|
|
val d = dispatcher(repo, port, props)
|
|
|
|
|
|
repo.enqueue(ev(1, Targets.KAFKA_SCHD, "F1", """{"v":"1"}"""))
|
|
|
|
|
|
repo.enqueue(ev(2, Targets.KAFKA_SCHD, "F2", """{"v":"2"}"""))
|
|
|
|
|
|
|
|
|
|
|
|
d.flushSchd() // 失败#1:attempts=1,退避 backoff[1]=1000ms
|
|
|
|
|
|
val after1 = repo.events.map { it.attempts to it.state }
|
|
|
|
|
|
assertEquals(listOf(1 to EventStatus.PENDING, 1 to EventStatus.PENDING), after1)
|
|
|
|
|
|
assertTrue(repo.events.all { it.nextAttemptAt == MutableClock.BASE.plusMillis(1000) })
|
|
|
|
|
|
|
|
|
|
|
|
d.flushSchd() // 时钟未推进 → 队首未到期,不 claim(不推进 lastFlush)
|
|
|
|
|
|
assertEquals(1, repo.events.first { it.eventId == 1L }.attempts)
|
|
|
|
|
|
|
|
|
|
|
|
clock.advance(1000)
|
|
|
|
|
|
d.flushSchd() // 失败#2:attempts=2 == maxAttempts → 整批 DEAD
|
|
|
|
|
|
assertTrue(repo.events.all { it.state == EventStatus.DEAD && it.errorClass == ErrorClass.EXHAUSTED })
|
|
|
|
|
|
assertTrue(repo.events.all { it.attempts == 2 })
|
|
|
|
|
|
assertEquals(0, port.sent.size)
|
|
|
|
|
|
|
|
|
|
|
|
// DEAD 行不再参与 claim:空批 → 无新发送、无异常
|
|
|
|
|
|
d.flushSchd()
|
|
|
|
|
|
assertEquals(0, port.sent.size)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
|
fun `KAFKA msg delivery unaffected while schd batch is retrying`() {
|
|
|
|
|
|
val props = PipelineProps().apply { pipeline.maxAttempts = 2 }
|
|
|
|
|
|
val repo = FakeRepo()
|
|
|
|
|
|
val port = FakePort().apply { failTopic = "schd" }
|
|
|
|
|
|
val d = dispatcher(repo, port, props)
|
|
|
|
|
|
repo.enqueue(ev(1, Targets.KAFKA_SCHD, "F1", """{"v":"1"}"""))
|
|
|
|
|
|
repo.enqueue(ev(2, Targets.KAFKA_MSG, null, """{"m":1}"""))
|
|
|
|
|
|
|
|
|
|
|
|
d.tick()
|
|
|
|
|
|
|
|
|
|
|
|
assertEquals(EventStatus.SENT, repo.events.first { it.eventId == 2L }.state) // msg 正常
|
|
|
|
|
|
assertEquals(1, repo.events.first { it.eventId == 1L }.attempts) // schd 进退避
|
|
|
|
|
|
assertEquals(EventStatus.PENDING, repo.events.first { it.eventId == 1L }.state)
|
|
|
|
|
|
}
|
2026-09-06 22:12:23 +08:00
|
|
|
|
|
2026-09-06 18:19:04 +08:00
|
|
|
|
}
|