fix(processing): U08 重试闭环 + FailureScheduler + U11 重放入口(ACM2-10)
- U08 闭环:失败状态迁移移到持有 head/事件的边界内——processOne/SnapshotFlow 统一经 ProcFailure(FAILED+attempts+退避,达上限 DEAD/EXHAUSTED 并落库 attempts);Dispatcher 逐条与 SCHD 批量共用 retryOrDead;flushSchd 失败整批 attempts+1/指数退避,队首未到期不 claim,达上限整批 DEAD(DLQ);loop 只作最后防线——仅 catch Exception,InterruptedException 恢复中断位并上抛,致命 Error 不吞(异常必可见) - 统一策略:新增 FailureScheduler(可注入 java.time.Clock + TimeFactory,backoff/exhausted 单一来源)与 ProcFailure,MessageProcessor/SnapshotFlow/Dispatcher 共用 - U11:ReplayService 显式重放入口 + ProcStateRepository.requeueByErrorClasses——仅白名单内 可恢复类(CODEC_ERROR/UNSUPPORTED/INFRA/EXHAUSTED)从 FAILED/DEAD 回 PENDING(attempts 清零), MALFORMED 永不重放 - 测试(30 个全绿):①Pump 内部异常→FAILED(INFRA)→重试达上限 DEAD;②SCHD 连续失败退避递增→ 整批 DEAD 且 KAFKA:msg 不受影响;③AssertionError/InterruptedException 不被普通恢复吞掉; 另有 ReplayService(3)、Dispatcher 批退避、InfraBindingStartupTest(DataSource 启动级绑定) - 收尾:.gitattributes(行尾/二进制);Gradle 10 弃用告警核查——仅来自 micronaut-application 插件(BOM 注入 + IDE 文件生成),升级 Gradle 10 前需先升插件版本(已记录)
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
package com.gzzn.omms.msgexchange.nextgen
|
||||
|
||||
import java.time.Clock
|
||||
import java.time.Instant
|
||||
import java.time.ZoneId
|
||||
import java.time.ZoneOffset
|
||||
|
||||
/** 可手动推进的可注入时钟(测试用),替代真实时间避免脆弱睡眠测试。 */
|
||||
internal class MutableClock(
|
||||
initial: Instant,
|
||||
private val zone: ZoneId = ZoneOffset.UTC,
|
||||
) : Clock() {
|
||||
|
||||
var instant: Instant = initial
|
||||
|
||||
override fun instant(): Instant = instant
|
||||
|
||||
override fun getZone(): ZoneId = zone
|
||||
|
||||
override fun withZone(zoneId: ZoneId): Clock = MutableClock(instant, zoneId)
|
||||
|
||||
fun advance(millis: Long) { instant = instant.plusMillis(millis) }
|
||||
|
||||
companion object {
|
||||
val BASE: Instant = Instant.parse("2026-09-06T02:00:00Z")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package com.gzzn.omms.msgexchange.nextgen.config
|
||||
|
||||
import io.micronaut.test.extensions.junit5.annotation.MicronautTest
|
||||
import jakarta.inject.Inject
|
||||
import org.junit.jupiter.api.Assertions.assertNotNull
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.sql.Connection
|
||||
import javax.sql.DataSource
|
||||
|
||||
/**
|
||||
* U03(N02/R09/R01a 验收的“启动级”版本):基础设施配置在真实上下文启动时绑定生效——
|
||||
* 注入 DataSource 并建立真实连接(H2 内存,application-test.yml),证明 datasources.default.*
|
||||
* 键位与驱动解析正确(而非“看似配置实则未生效”)。Pump 等业务 bean 懒加载,不依赖实仓储。
|
||||
*/
|
||||
@MicronautTest
|
||||
class InfraBindingStartupTest {
|
||||
|
||||
@Inject
|
||||
lateinit var dataSource: DataSource
|
||||
|
||||
@Inject
|
||||
lateinit var props: PipelineProps
|
||||
|
||||
@Test
|
||||
fun `datasource binds and yields a live connection`() {
|
||||
assertNotNull(dataSource)
|
||||
dataSource.connection.use { c: Connection ->
|
||||
assertNotNull(c.metaData)
|
||||
c.createStatement().use { it.execute("SELECT 1") }
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `registration disabled by test profile`() {
|
||||
assertNotNull(props)
|
||||
// eureka 注册在 application-test.yml 经 msgx.register-eureka=false 关闭——启动级验证绑定可达
|
||||
assertNotNull(dataSource)
|
||||
}
|
||||
}
|
||||
@@ -1,21 +1,24 @@
|
||||
package com.gzzn.omms.msgexchange.nextgen.delivery
|
||||
|
||||
import com.gzzn.omms.msgexchange.nextgen.MutableClock
|
||||
import com.gzzn.omms.msgexchange.nextgen.config.PipelineProps
|
||||
import com.gzzn.omms.msgexchange.nextgen.domain.ErrorClass
|
||||
import com.gzzn.omms.msgexchange.nextgen.domain.EventStatus
|
||||
import com.gzzn.omms.msgexchange.nextgen.domain.MsgEvent
|
||||
import com.gzzn.omms.msgexchange.nextgen.domain.Targets
|
||||
import com.gzzn.omms.msgexchange.nextgen.infra.persistence.MsgEventRepository
|
||||
import com.gzzn.omms.msgexchange.nextgen.infra.retry.FailureScheduler
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertNotNull
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
import kotlin.test.assertFails
|
||||
|
||||
/**
|
||||
* U06(N03):KAFKA_SCHD 不被逐条循环吞掉——唯一出口 flushSchd 批量聚合;
|
||||
* 聚合只发每 FLID 最新一态;发送失败整批保持 PENDING 且不推进 lastFlush(N24)。
|
||||
* U06(N03)+ U08 闭环:schd 出口只走 flushSchd;
|
||||
* 批发送失败 → 整批 attempts 递增 + 指数退避(队首未到期不 claim),达上限整批 DEAD/DLQ;
|
||||
* 时间经可注入 Clock(MutableClock),不依赖真实睡眠。
|
||||
*/
|
||||
class DispatcherTickTest {
|
||||
|
||||
@@ -47,9 +50,10 @@ class DispatcherTickTest {
|
||||
if (i >= 0) events[i] = events[i].copy(state = EventStatus.PENDING, attempts = attempts, nextAttemptAt = nextAttemptAt)
|
||||
}
|
||||
|
||||
override fun markDead(eventId: Long, errorClass: ErrorClass, lastError: String) {
|
||||
override fun markDead(eventId: Long, errorClass: ErrorClass, lastError: String, attempts: Int?) {
|
||||
val i = events.indexOfFirst { it.eventId == eventId }
|
||||
if (i >= 0) events[i] = events[i].copy(state = EventStatus.DEAD, errorClass = errorClass, lastError = lastError)
|
||||
if (i >= 0) events[i] = events[i].copy(state = EventStatus.DEAD, errorClass = errorClass, lastError = lastError,
|
||||
attempts = attempts ?: events[i].attempts)
|
||||
}
|
||||
|
||||
override fun insertSync(events: List<MsgEvent>) = Unit
|
||||
@@ -68,7 +72,10 @@ class DispatcherTickTest {
|
||||
override fun projectRedis(payloadJson: String) = Unit
|
||||
}
|
||||
|
||||
private fun props(): PipelineProps = PipelineProps() // schd.flush-period 默认 3s
|
||||
private val clock = MutableClock(MutableClock.BASE)
|
||||
|
||||
private fun dispatcher(repo: FakeRepo, port: FakePort, p: PipelineProps = PipelineProps()): Dispatcher =
|
||||
Dispatcher(repo, port, p, FailureScheduler(p, clock))
|
||||
|
||||
private fun ev(id: Long, target: String, key: String?, payload: String) =
|
||||
MsgEvent(eventId = id, target = target, partitionKey = key, payloadJson = payload, state = EventStatus.PENDING)
|
||||
@@ -77,57 +84,101 @@ class DispatcherTickTest {
|
||||
fun `schd flushed once via batch path, per-target tick never consumes it`() {
|
||||
val repo = FakeRepo()
|
||||
val port = FakePort()
|
||||
val d = Dispatcher(repo, port, props())
|
||||
val d = dispatcher(repo, port)
|
||||
repo.enqueue(ev(1, Targets.KAFKA_MSG, null, """{"msg":1}"""))
|
||||
repo.enqueue(ev(2, Targets.KAFKA_SCHD, "F1", """{"FLID":"F1","v":1}"""))
|
||||
|
||||
d.tick() // lastFlush=EPOCH → 首 tick 即到 flush 周期
|
||||
|
||||
// KAFKA:msg 逐条投递;schd 经 flushSchd 聚合发出一次(而非逐条 markSent 吞掉)
|
||||
assertEquals(1, port.sent.count { it.first == "msg" })
|
||||
assertEquals(1, port.sent.count { it.first == "schd" })
|
||||
assertEquals(EventStatus.SENT, repo.events.first { it.eventId == 2L }.state)
|
||||
|
||||
// 新的 schd 事件:flush 周期未到 → 逐条循环不得消费它(仍 PENDING、未发)
|
||||
repo.enqueue(ev(4, Targets.KAFKA_SCHD, "F1", """{"FLID":"F1","v":4}"""))
|
||||
d.tick()
|
||||
d.tick() // 时钟未推进 → flush 周期未到;逐条循环不得消费 schd
|
||||
assertEquals(EventStatus.PENDING, repo.events.first { it.eventId == 4L }.state)
|
||||
assertEquals(1, port.sent.count { it.first == "schd" }) // 第二次 tick 未额外发送
|
||||
assertEquals(1, port.sent.count { it.first == "schd" })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `flushSchd aggregates latest per flight and marks all sent`() {
|
||||
val repo = FakeRepo()
|
||||
val port = FakePort()
|
||||
val d = Dispatcher(repo, port, props())
|
||||
val d = dispatcher(repo, port)
|
||||
repo.enqueue(ev(1, Targets.KAFKA_SCHD, "F1", "old"))
|
||||
repo.enqueue(ev(2, Targets.KAFKA_SCHD, "F2", "only"))
|
||||
repo.enqueue(ev(3, Targets.KAFKA_SCHD, "F1", "new"))
|
||||
|
||||
d.flushSchd()
|
||||
|
||||
assertEquals(1, port.sent.count { it.first == "schd" })
|
||||
val payload = port.sent.first { it.first == "schd" }.second
|
||||
assertTrue(payload.startsWith("[") && payload.endsWith("]"))
|
||||
assertTrue(payload.contains("new") && payload.contains("only") && !payload.contains("old"))
|
||||
assertEquals(2, payload.removeSurrounding("[", "]").split(",").size) // 同 FLID 只发最新(矩阵 #7)
|
||||
assertTrue(repo.events.all { it.state == EventStatus.SENT })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `flush failure keeps batch PENDING and retries after next due`() {
|
||||
fun `flush failure schedules per-event backoff and retries after due`() {
|
||||
val repo = FakeRepo()
|
||||
val port = FakePort().apply { failTopic = "schd" }
|
||||
val d = Dispatcher(repo, port, props())
|
||||
val d = dispatcher(repo, port)
|
||||
repo.enqueue(ev(1, Targets.KAFKA_SCHD, "F1", """{"v":"1"}"""))
|
||||
|
||||
assertFails { d.flushSchd() } // 发送失败 → 异常上抛(tick 侧隔离),事件保持 PENDING
|
||||
assertEquals(EventStatus.PENDING, repo.events.single().state)
|
||||
d.flushSchd()
|
||||
val e1 = repo.events.single()
|
||||
assertEquals(EventStatus.PENDING, e1.state)
|
||||
assertEquals(1, e1.attempts)
|
||||
assertNotNull(e1.nextAttemptAt) // 指数退避落库(attempts=1 → 首档)
|
||||
assertEquals(0, port.sent.size)
|
||||
|
||||
clock.advance(1000) // 退避到期
|
||||
port.failTopic = null
|
||||
d.flushSchd() // lastFlush 未推进 → 下个周期整批重试成功
|
||||
d.flushSchd()
|
||||
assertEquals(EventStatus.SENT, repo.events.single().state)
|
||||
assertEquals(1, port.sent.count { it.first == "schd" })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `repeated batch failures backoff grows and whole batch goes DEAD DLQ at limit`() {
|
||||
val props = PipelineProps().apply { pipeline.maxAttempts = 2 }
|
||||
val repo = FakeRepo()
|
||||
val port = FakePort().apply { failTopic = "schd" }
|
||||
val d = dispatcher(repo, port, props)
|
||||
repo.enqueue(ev(1, Targets.KAFKA_SCHD, "F1", """{"v":"1"}"""))
|
||||
repo.enqueue(ev(2, Targets.KAFKA_SCHD, "F2", """{"v":"2"}"""))
|
||||
|
||||
d.flushSchd() // 失败#1:attempts=1,退避 backoff[1]=1000ms
|
||||
val after1 = repo.events.map { it.attempts to it.state }
|
||||
assertEquals(listOf(1 to EventStatus.PENDING, 1 to EventStatus.PENDING), after1)
|
||||
assertTrue(repo.events.all { it.nextAttemptAt == MutableClock.BASE.plusMillis(1000) })
|
||||
|
||||
d.flushSchd() // 时钟未推进 → 队首未到期,不 claim(不推进 lastFlush)
|
||||
assertEquals(1, repo.events.first { it.eventId == 1L }.attempts)
|
||||
|
||||
clock.advance(1000)
|
||||
d.flushSchd() // 失败#2:attempts=2 == maxAttempts → 整批 DEAD
|
||||
assertTrue(repo.events.all { it.state == EventStatus.DEAD && it.errorClass == ErrorClass.EXHAUSTED })
|
||||
assertTrue(repo.events.all { it.attempts == 2 })
|
||||
assertEquals(0, port.sent.size)
|
||||
|
||||
// DEAD 行不再参与 claim:空批 → 无新发送、无异常
|
||||
d.flushSchd()
|
||||
assertEquals(0, port.sent.size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `KAFKA msg delivery unaffected while schd batch is retrying`() {
|
||||
val props = PipelineProps().apply { pipeline.maxAttempts = 2 }
|
||||
val repo = FakeRepo()
|
||||
val port = FakePort().apply { failTopic = "schd" }
|
||||
val d = dispatcher(repo, port, props)
|
||||
repo.enqueue(ev(1, Targets.KAFKA_SCHD, "F1", """{"v":"1"}"""))
|
||||
repo.enqueue(ev(2, Targets.KAFKA_MSG, null, """{"m":1}"""))
|
||||
|
||||
d.tick()
|
||||
|
||||
assertEquals(EventStatus.SENT, repo.events.first { it.eventId == 2L }.state) // msg 正常
|
||||
assertEquals(1, repo.events.first { it.eventId == 1L }.attempts) // schd 进退避
|
||||
assertEquals(EventStatus.PENDING, repo.events.first { it.eventId == 1L }.state)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package com.gzzn.omms.msgexchange.nextgen.infra.retry
|
||||
|
||||
import com.gzzn.omms.msgexchange.nextgen.domain.ErrorClass
|
||||
import com.gzzn.omms.msgexchange.nextgen.domain.ProcState
|
||||
import com.gzzn.omms.msgexchange.nextgen.domain.ProcStatus
|
||||
import com.gzzn.omms.msgexchange.nextgen.infra.persistence.ProcStateRepository
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.time.Instant
|
||||
|
||||
/**
|
||||
* U11:显式重放入口只允许可恢复错误从 FAILED/DEAD 返回 PENDING(attempts 清零、立即重试);
|
||||
* MALFORMED(报文非法)永不被重放。
|
||||
*/
|
||||
class ReplayServiceTest {
|
||||
|
||||
private class FakeRepo : ProcStateRepository {
|
||||
val rows = linkedMapOf<Long, ProcState>()
|
||||
val requeueCalls = mutableListOf<List<ErrorClass>>()
|
||||
|
||||
fun seed(id: Long, status: ProcStatus, ec: ErrorClass?) {
|
||||
rows[id] = ProcState(id, status, identityKey = "k$id", attempts = 3, errorClass = ec, lastError = "x")
|
||||
}
|
||||
|
||||
override fun insert(cminmsgsId: Long, state: ProcStatus) = Unit
|
||||
override fun headUnfinished(): ProcState? = null
|
||||
override fun tryBindIdentity(cminmsgsId: Long, identityKey: String): Boolean = true
|
||||
override fun ownerOfIdentity(identityKey: String): Long? = null
|
||||
override fun update(
|
||||
cminmsgsId: Long, state: ProcStatus, nextAttemptAt: Instant?, attempts: Int?,
|
||||
errorClass: ErrorClass?, lastError: String?,
|
||||
) = Unit
|
||||
|
||||
override fun requeueByErrorClasses(errorClasses: List<ErrorClass>): Int {
|
||||
requeueCalls += errorClasses
|
||||
var n = 0
|
||||
rows.keys.toList().forEach { id ->
|
||||
val s = rows[id]!!
|
||||
if (s.errorClass != null && s.errorClass in errorClasses &&
|
||||
(s.state == ProcStatus.FAILED || s.state == ProcStatus.DEAD)) {
|
||||
rows[id] = s.copy(state = ProcStatus.PENDING, attempts = 0, nextAttemptAt = null)
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `only recoverable classes are requeued, MALFORMED stays terminal`() {
|
||||
val repo = FakeRepo()
|
||||
repo.seed(1, ProcStatus.DEAD, ErrorClass.CODEC_ERROR)
|
||||
repo.seed(2, ProcStatus.DEAD, ErrorClass.MALFORMED)
|
||||
repo.seed(3, ProcStatus.FAILED, ErrorClass.UNSUPPORTED)
|
||||
|
||||
val n = ReplayService(repo).replay(listOf(ErrorClass.CODEC_ERROR, ErrorClass.MALFORMED, ErrorClass.UNSUPPORTED))
|
||||
|
||||
assertEquals(2, n)
|
||||
assertEquals(listOf(listOf(ErrorClass.CODEC_ERROR, ErrorClass.UNSUPPORTED)), repo.requeueCalls)
|
||||
assertEquals(ProcStatus.PENDING, repo.rows[1]!!.state)
|
||||
assertEquals(0, repo.rows[1]!!.attempts)
|
||||
assertNull(repo.rows[1]!!.nextAttemptAt)
|
||||
assertEquals(ProcStatus.DEAD, repo.rows[2]!!.state) // MALFORMED 永不被重放
|
||||
assertEquals(ProcStatus.PENDING, repo.rows[3]!!.state)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `requesting only MALFORMED is a no-op`() {
|
||||
val repo = FakeRepo()
|
||||
repo.seed(2, ProcStatus.DEAD, ErrorClass.MALFORMED)
|
||||
|
||||
val n = ReplayService(repo).replay(listOf(ErrorClass.MALFORMED))
|
||||
|
||||
assertEquals(0, n)
|
||||
assertNull(repo.requeueCalls.lastOrNull()) // 白名单过滤后为空 → 不触达仓储
|
||||
assertEquals(ProcStatus.DEAD, repo.rows[2]!!.state)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `replayAll reopens every recoverable class including EXHAUSTED`() {
|
||||
val repo = FakeRepo()
|
||||
repo.seed(4, ProcStatus.FAILED, ErrorClass.INFRA)
|
||||
repo.seed(5, ProcStatus.DEAD, ErrorClass.EXHAUSTED)
|
||||
repo.seed(6, ProcStatus.DEAD, ErrorClass.MALFORMED)
|
||||
|
||||
val n = ReplayService(repo).replayAll()
|
||||
|
||||
assertEquals(2, n)
|
||||
assertEquals(ProcStatus.PENDING, repo.rows[4]!!.state)
|
||||
assertEquals(ProcStatus.PENDING, repo.rows[5]!!.state)
|
||||
assertEquals(ProcStatus.DEAD, repo.rows[6]!!.state)
|
||||
}
|
||||
}
|
||||
+102
-16
@@ -1,5 +1,6 @@
|
||||
package com.gzzn.omms.msgexchange.nextgen.processing
|
||||
|
||||
import com.gzzn.omms.msgexchange.nextgen.MutableClock
|
||||
import com.gzzn.omms.msgexchange.nextgen.codec.DecodeFailure
|
||||
import com.gzzn.omms.msgexchange.nextgen.codec.DecodeResult
|
||||
import com.gzzn.omms.msgexchange.nextgen.codec.XmlCodec
|
||||
@@ -25,24 +26,27 @@ import com.gzzn.omms.msgexchange.nextgen.infra.persistence.ReqTrackRepository
|
||||
import com.gzzn.omms.msgexchange.nextgen.infra.persistence.FlightStateRepository
|
||||
import com.gzzn.omms.msgexchange.nextgen.infra.redis.FlightRedisClient
|
||||
import com.gzzn.omms.msgexchange.nextgen.infra.redis.RedisScript
|
||||
import com.gzzn.omms.msgexchange.nextgen.infra.retry.FailureScheduler
|
||||
import com.gzzn.omms.msgexchange.nextgen.infra.retry.ProcFailure
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertNotNull
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.Assertions.assertThrows
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.time.Instant
|
||||
|
||||
/**
|
||||
* U10/U11/U08(N21/T06/N06/N28):processOne 失败语义——
|
||||
* 未注册 handler / CODEC_ERROR / staging 未实装 → FAILED(可重放,带退避,绝不写终态);
|
||||
* MALFORMED → DEAD(不重试);FAILED attempts≥maxAttempts 入口升级 DEAD;成功路径事件+回填+SUCCEEDED。
|
||||
* U08/U10/U11:processOne 失败闭环——
|
||||
* ① 意外异常在边界内写 FAILED(INFRA)+退避,重试达上限转 DEAD(EXHAUSTED)(Pump 内部异常不杀泵);
|
||||
* ② 未注册/CODEC_ERROR/staging 未实装 → FAILED(绝不直接终态);MALFORMED → DEAD;
|
||||
* ③ InterruptedException/致命 Error 不被普通恢复逻辑吞掉。
|
||||
*/
|
||||
class MessageProcessorTest {
|
||||
|
||||
// ---------- fakes ----------
|
||||
private class FakeProcState : ProcStateRepository {
|
||||
var record = mutableMapOf<Long, ProcState>()
|
||||
val bound = mutableMapOf<String, Long>() // identityKey -> owner
|
||||
val bound = mutableMapOf<String, Long>()
|
||||
|
||||
override fun insert(cminmsgsId: Long, state: ProcStatus) { record[cminmsgsId] = ProcState(cminmsgsId, state) }
|
||||
|
||||
@@ -72,16 +76,33 @@ class MessageProcessorTest {
|
||||
)
|
||||
}
|
||||
|
||||
override fun requeueByErrorClasses(errorClasses: List<ErrorClass>): Int {
|
||||
var n = 0
|
||||
record.keys.toList().forEach { id ->
|
||||
val s = record[id]!!
|
||||
if (s.errorClass != null && s.errorClass in errorClasses &&
|
||||
(s.state == ProcStatus.FAILED || s.state == ProcStatus.DEAD)) {
|
||||
record[id] = s.copy(state = ProcStatus.PENDING, attempts = 0, nextAttemptAt = null)
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
fun state(id: Long): ProcState = record.getValue(id)
|
||||
}
|
||||
|
||||
private class FakeInbox : CminmsgInboxRepository {
|
||||
var raws = mutableMapOf<Long, String>()
|
||||
var throwOnRawOf: Throwable? = null
|
||||
val backfilled = mutableListOf<List<Any?>>()
|
||||
|
||||
override fun insertRaw(rawXml: String): Long = 0
|
||||
|
||||
override fun rawOf(cminmsgsId: Long): String? = raws[cminmsgsId]
|
||||
override fun rawOf(cminmsgsId: Long): String? {
|
||||
throwOnRawOf?.let { throw it }
|
||||
return raws[cminmsgsId]
|
||||
}
|
||||
|
||||
override fun backfillOnSuccess(cminmsgsId: Long, sndr: String, type: String, styp: String, seqn: Long) {
|
||||
backfilled += listOf(cminmsgsId, sndr, type, styp, seqn)
|
||||
@@ -98,7 +119,7 @@ class MessageProcessorTest {
|
||||
override fun markSent(eventId: Long) = Unit
|
||||
override fun markAllSent(eventIds: List<Long>) = Unit
|
||||
override fun scheduleRetry(eventId: Long, nextAttemptAt: Instant, attempts: Int) = Unit
|
||||
override fun markDead(eventId: Long, errorClass: ErrorClass, lastError: String) = Unit
|
||||
override fun markDead(eventId: Long, errorClass: ErrorClass, lastError: String, attempts: Int?) = Unit
|
||||
override fun insertSync(events: List<MsgEvent>) = Unit
|
||||
}
|
||||
|
||||
@@ -126,6 +147,8 @@ class MessageProcessorTest {
|
||||
}
|
||||
|
||||
// ---------- helpers ----------
|
||||
private val clock = MutableClock(MutableClock.BASE)
|
||||
|
||||
private fun msg(seqn: String) = DecodedMessage(
|
||||
meta = MetaFields(sndr = "AODB", type = "FLOP", styp = "DELY", seqn = seqn.toLong(), dttm = 20260906120000L),
|
||||
kind = MsgKind.Flop("DELY"),
|
||||
@@ -148,8 +171,10 @@ class MessageProcessorTest {
|
||||
registry: HandlerRegistry = HandlerRegistry(emptyList()),
|
||||
): MessageProcessor {
|
||||
val props = PipelineProps()
|
||||
val snapshot = SnapshotFlow(procState, FakeRefData(), FakeRedis, props)
|
||||
return MessageProcessor(inbox, procState, events, CodecHolder(codec), HandlerHolder(registry), FakeRedis, snapshot, props)
|
||||
val scheduler = FailureScheduler(props, clock)
|
||||
val procFailure = ProcFailure(procState, scheduler)
|
||||
val snapshot = SnapshotFlow(procState, FakeRefData(), FakeRedis, procFailure)
|
||||
return MessageProcessor(inbox, procState, events, CodecHolder(codec), HandlerHolder(registry), FakeRedis, snapshot, procFailure, props)
|
||||
}
|
||||
|
||||
// ---------- tests ----------
|
||||
@@ -177,6 +202,7 @@ class MessageProcessorTest {
|
||||
val s = ps.state(1)
|
||||
assertEquals(ProcStatus.FAILED, s.state)
|
||||
assertEquals(ErrorClass.CODEC_ERROR, s.errorClass)
|
||||
assertEquals(1, s.attempts)
|
||||
assertNotNull(s.nextAttemptAt)
|
||||
}
|
||||
|
||||
@@ -193,15 +219,13 @@ class MessageProcessorTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `failed head at max attempts upgrades to DEAD at entry`() {
|
||||
fun `legacy failed head at max attempts upgrades to DEAD at entry`() {
|
||||
val ps = FakeProcState(); val inbox = FakeInbox(); val ev = FakeEvents()
|
||||
ps.record[1] = head(status = ProcStatus.FAILED, attempts = 5, identityKey = "k")
|
||||
processor(ps, inbox, ev, FakeCodec()).processOne(ps.state(1))
|
||||
|
||||
val s = ps.state(1)
|
||||
assertEquals(ProcStatus.DEAD, s.state)
|
||||
assertEquals(ErrorClass.EXHAUSTED, s.errorClass)
|
||||
assertTrue(inbox.raws.isEmpty() || inbox.backfilled.isEmpty()) // 入口即升级,未进入处理
|
||||
assertEquals(ProcStatus.DEAD, ps.state(1).state)
|
||||
assertEquals(ErrorClass.EXHAUSTED, ps.state(1).errorClass)
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -212,7 +236,6 @@ class MessageProcessorTest {
|
||||
processor(ps, inbox, ev, FakeCodec(), registry).processOne(ps.state(1))
|
||||
|
||||
assertEquals(ProcStatus.SUCCEEDED, ps.state(1).state)
|
||||
assertEquals(1, ev.inserted.size)
|
||||
val events = ev.inserted.single()
|
||||
assertEquals(listOf(Targets.KAFKA_MSG, Targets.KAFKA_SCHD), events.map { it.target })
|
||||
assertEquals("F1", events.first { it.target == Targets.KAFKA_SCHD }.partitionKey)
|
||||
@@ -222,7 +245,7 @@ class MessageProcessorTest {
|
||||
@Test
|
||||
fun `duplicate identity leads to SKIPPED`() {
|
||||
val ps = FakeProcState(); val inbox = FakeInbox(); val ev = FakeEvents()
|
||||
ps.record[1] = head(); ps.record[2] = head(id = 2)
|
||||
ps.record[1] = head()
|
||||
inbox.raws[1] = "<MSG/>"
|
||||
ps.bound["AODB|FLOP|DELY|1"] = 2L // 另一条消息已持有该键
|
||||
|
||||
@@ -233,4 +256,67 @@ class MessageProcessorTest {
|
||||
assertTrue(s.lastError == "duplicate-of:2")
|
||||
assertTrue(ev.inserted.isEmpty())
|
||||
}
|
||||
|
||||
// ---------- U08 边界化(评审要求①③) ----------
|
||||
@Test
|
||||
fun `unexpected exception maps to FAILED INFRA with backoff at boundary`() {
|
||||
val ps = FakeProcState(); val inbox = FakeInbox(); val ev = FakeEvents()
|
||||
ps.record[1] = head()
|
||||
inbox.throwOnRawOf = RuntimeException("db-down")
|
||||
|
||||
processor(ps, inbox, ev, FakeCodec()).processOne(ps.state(1))
|
||||
|
||||
val s = ps.state(1)
|
||||
assertEquals(ProcStatus.FAILED, s.state)
|
||||
assertEquals(ErrorClass.INFRA, s.errorClass)
|
||||
assertEquals(1, s.attempts)
|
||||
assertNotNull(s.nextAttemptAt)
|
||||
assertTrue(s.lastError == "db-down")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `repeated unexpected exceptions escalate to DEAD EXHAUSTED at max attempts`() {
|
||||
val ps = FakeProcState(); val inbox = FakeInbox(); val ev = FakeEvents()
|
||||
ps.record[1] = head()
|
||||
inbox.throwOnRawOf = RuntimeException("db-down")
|
||||
val p = processor(ps, inbox, ev, FakeCodec())
|
||||
|
||||
var attempts = 0
|
||||
while (ps.state(1).state != ProcStatus.DEAD && attempts < 10) {
|
||||
p.processOne(ps.state(1))
|
||||
clock.advance(60_000) // 推进时钟:让退避/毒丸时间语义确定
|
||||
attempts++
|
||||
}
|
||||
|
||||
val s = ps.state(1)
|
||||
assertEquals(ProcStatus.DEAD, s.state, "should exhaust within maxAttempts(=5)")
|
||||
assertEquals(ErrorClass.EXHAUSTED, s.errorClass)
|
||||
assertTrue(s.lastError?.contains("attempts=5") == true)
|
||||
assertEquals(5, s.attempts)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `fatal Error is not swallowed by exception recovery`() {
|
||||
val ps = FakeProcState(); val inbox = FakeInbox(); val ev = FakeEvents()
|
||||
ps.record[1] = head()
|
||||
inbox.throwOnRawOf = AssertionError("fatal")
|
||||
|
||||
assertThrows(AssertionError::class.java) {
|
||||
processor(ps, inbox, ev, FakeCodec()).processOne(ps.state(1))
|
||||
}
|
||||
// 未被普通恢复逻辑改写成 FAILED(Error 不落入 catch Exception)
|
||||
assertEquals(ProcStatus.PENDING, ps.state(1).state)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `InterruptedException restores flag and propagates`() {
|
||||
val ps = FakeProcState(); val inbox = FakeInbox(); val ev = FakeEvents()
|
||||
ps.record[1] = head()
|
||||
inbox.throwOnRawOf = InterruptedException("stop")
|
||||
|
||||
assertThrows(InterruptedException::class.java) {
|
||||
processor(ps, inbox, ev, FakeCodec()).processOne(ps.state(1))
|
||||
}
|
||||
assertTrue(Thread.interrupted(), "interrupt flag must be restored")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user