cdia/msgexchange-v2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(acm2): 闭合 G-KAFKA-D3 / G-IGNORE / G-EVENT-RETENTION 三个缺口

Browse Source 
- G-KAFKA-D3：max-in-flight 默认收敛到 1，KafkaD3Check 启动自检钉住三项联合满足 D3
- G-IGNORE：IgnoreRules 在身份绑定后精确匹配 TYPE 字段，命中写 SKIPPED + 回填意图
- G-EVENT-RETENTION：V10 迁移加 SENT_AT 列，投递原子写，EventCleanupJob 有界清理过期 SENT 行
This commit is contained in:
windyboy
2026-09-13 15:48:16 +08:00
parent d37a9eb38e
commit fc744d0cfd
23 changed files with 540 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/kotlin/com/gzzn/omms/msgexchange/config/KafkaD3CheckTest.kt
+39
View File
@@ -0,0 +1,39 @@
package com.gzzn.omms.msgexchange.config
import org.junit.jupiter.api.Assertions.assertDoesNotThrow
import org.junit.jupiter.api.Assertions.assertThrows
import org.junit.jupiter.api.Test
class KafkaD3CheckTest {
@Test
fun `default config satisfies D3`() {
assertDoesNotThrow { KafkaD3Check(acks = "all", idempotence = "true", maxInFlight = "1") }
}
@Test
fun `acks=-1 also satisfies D3`() {
assertDoesNotThrow { KafkaD3Check(acks = "-1", idempotence = "true", maxInFlight = "1") }
}
@Test
fun `max-in-flight not 1 rejects`() {
assertThrows(IllegalArgumentException::class.java) {
KafkaD3Check(acks = "all", idempotence = "true", maxInFlight = "5")
}
}
@Test
fun `idempotence false rejects`() {
assertThrows(IllegalArgumentException::class.java) {
KafkaD3Check(acks = "all", idempotence = "false", maxInFlight = "1")
}
}
@Test
fun `acks not all rejects`() {
assertThrows(IllegalArgumentException::class.java) {
KafkaD3Check(acks = "1", idempotence = "true", maxInFlight = "1")
}
}
}
src/test/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/JdbcMsgEventUpsertPgTest.kt
+3 -2
View File
@@ -45,8 +45,9 @@ class JdbcMsgEventUpsertPgTest {
assertEquals(EventType.TOMBSTONE, pendingSchd(repo, key).single().eventType)
// 条件确认：旧代次影响 0 行，当前代次标记成功
assertEquals(0, repo.markSentIfVersion(firstId, 1))
assertEquals(1, repo.markSentIfVersion(tombstoneId, 2))
val now = java.time.Instant.parse("2025-01-01T00:00:00Z")
assertEquals(0, repo.markSentIfVersion(firstId, 1, now))
assertEquals(1, repo.markSentIfVersion(tombstoneId, 2, now))
assertEquals(0, pendingSchd(repo, key).size)
}
src/test/kotlin/com/gzzn/omms/msgexchange/jobs/EventCleanupJobTest.kt
+78
View File
@@ -0,0 +1,78 @@
package com.gzzn.omms.msgexchange.jobs
import com.gzzn.omms.msgexchange.MutableClock
import com.gzzn.omms.msgexchange.config.PipelineProps
import com.gzzn.omms.msgexchange.domain.EventStatus
import com.gzzn.omms.msgexchange.domain.MsgEvent
import com.gzzn.omms.msgexchange.domain.Targets
import com.gzzn.omms.msgexchange.infra.stub.StubMsgEvents
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import java.time.Duration
import java.time.Instant
/**
* 清理作业只删 `STATE='SENT'` 且 `SENT_AT` 超过保留期的行；
* 未确认投递（`PENDING`）和未超期的 SENT 行不受影响。
*/
class EventCleanupJobTest {
private val clock = MutableClock(MutableClock.BASE)
private val props = PipelineProps().apply { pipeline.eventRetention = Duration.ofDays(7) }
private fun sentEvent(sentAt: Instant): MsgEvent = MsgEvent(
target = Targets.KAFKA_MSG,
partitionKey = "pk",
payloadJson = "{}",
createdAt = MutableClock.BASE,
sentAt = sentAt,
)
@Test
fun `deletes SENT rows older than retention`() {
val repo = StubMsgEvents()
val old = repo.insertAll(listOf(sentEvent(clock.instant().minus(Duration.ofDays(8))))).single()
repo.markSent(old, clock.instant().minus(Duration.ofDays(8)))
val recent = repo.insertAll(listOf(sentEvent(clock.instant().minus(Duration.ofDays(2))))).single()
repo.markSent(recent, clock.instant().minus(Duration.ofDays(2)))
val deleted = EventCleanupJob(repo, props).run(clock.instant()).deleted
assertEquals(1, deleted)
assertEquals(EventStatus.SENT, repo.rows[recent]!!.state)
assertTrue(old !in repo.rows)
}
@Test
fun `does not delete PENDING rows regardless of age`() {
val repo = StubMsgEvents()
val pending = MsgEvent(
target = Targets.KAFKA_MSG,
partitionKey = "pk",
payloadJson = "{}",
createdAt = MutableClock.BASE,
)
val id = repo.insertAll(listOf(pending)).single()
val deleted = EventCleanupJob(repo, props).run(clock.instant()).deleted
assertEquals(0, deleted)
assertEquals(EventStatus.PENDING, repo.rows[id]!!.state)
}
@Test
fun `cleans up multiple expired SENT rows in one run`() {
val repo = StubMsgEvents()
val oldSentAt = clock.instant().minus(Duration.ofDays(30))
repeat(5) {
val id = repo.insertAll(listOf(sentEvent(oldSentAt))).single()
repo.markSent(id, oldSentAt)
}
val smallBatchProps = PipelineProps().apply { pipeline.eventRetention = Duration.ofDays(7) }
val deleted = EventCleanupJob(repo, smallBatchProps).run(clock.instant()).deleted
assertEquals(5, deleted)
}
}
src/test/kotlin/com/gzzn/omms/msgexchange/jobs/JobRunnerTest.kt
+1 -1
View File
@@ -38,7 +38,7 @@ class JobRunnerTest {
private fun runner(proc: ProcStateRepository, activity: JobActivity) = JobRunner(
BackfillService(proc, StubInbox(), MailboxProps(), props, clock, MessageLifecycleGate()),
historySweep(), clock, activity, OperationDayProps(),
historySweep(), EventCleanupJob(StubMsgEvents(), props), clock, activity, props, OperationDayProps(),
)
@Test
src/test/kotlin/com/gzzn/omms/msgexchange/processing/IgnoreBranchTest.kt
+205
View File
@@ -0,0 +1,205 @@
package com.gzzn.omms.msgexchange.processing
import com.gzzn.omms.msgexchange.codec.DecodeResult
import com.gzzn.omms.msgexchange.codec.XmlCodec
import com.gzzn.omms.msgexchange.config.OperationDayProps
import com.gzzn.omms.msgexchange.config.PipelineProps
import com.gzzn.omms.msgexchange.domain.DecodedMessage
import com.gzzn.omms.msgexchange.domain.MetaFields
import com.gzzn.omms.msgexchange.domain.MsgKind
import com.gzzn.omms.msgexchange.domain.ProcState
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.infra.metrics.PipelineCounters
import com.gzzn.omms.msgexchange.infra.persistence.InboxCursorRepository
import com.gzzn.omms.msgexchange.infra.persistence.PipelineLockRepository
import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
import com.gzzn.omms.msgexchange.infra.retry.ProcFailure
import com.gzzn.omms.msgexchange.infra.retry.FailureScheduler
import com.gzzn.omms.msgexchange.infra.stub.StubFlightState
import com.gzzn.omms.msgexchange.infra.stub.StubInbox
import com.gzzn.omms.msgexchange.infra.stub.StubInboxCursor
import com.gzzn.omms.msgexchange.infra.stub.StubMsgEvents
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
import com.gzzn.omms.msgexchange.infra.stub.StubSnapshotLog
import com.fasterxml.jackson.databind.ObjectMapper
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertNotNull
import org.junit.jupiter.api.Test
import java.time.Clock
import java.time.Instant
/**
* 忽略类报文（US-04）：命中忽略清单的报文先绑定身份再写 SKIPPED，
* 不产生航班或 outbox 副作用；未命中的合法类型继续按 UNSUPPORTED 处理。
*/
class IgnoreBranchTest {
private val msgId = 99L
private fun head() = ProcState(msgId, ProcStatus.PENDING, updatedAt = Instant.EPOCH)
private fun decoded(type: String, styp: String) = DecodedMessage(
meta = MetaFields("AODB", type, styp, 200L, 1L),
kind = MsgKind.Unsupported("$type-$styp"),
rawXml = "<MSG/>",
)
private fun codecReturning(msg: DecodedMessage) = object : XmlCodec {
override fun decode(rawXml: String): DecodeResult = DecodeResult.Ok(msg)
override fun encodeRqrd(kind: String, rangeJson: String): String = ""
}
private val noopTx = object : PipelineTransactionManager {
override fun <T> inTransaction(block: () -> T): T = block()
}
private val noopLock = object : PipelineLockRepository { override fun lock() = Unit }
private fun processor(
proc: StubProcState = StubProcState(),
inbox: StubInbox = StubInbox(),
counters: PipelineCounters = PipelineCounters(),
codec: XmlCodec,
): MessageProcessor {
val cursor = StubInboxCursor()
cursor.cursor = InboxCursorRepository.Cursor(committedUpTo = Long.MAX_VALUE)
val clock = Clock.systemUTC()
val props = PipelineProps()
val opDay = OperationDayProps().apply { zone = "Asia/Shanghai"; cutoffHour = 0 }
val flights = StubFlightState()
val events = StubMsgEvents()
val log = StubSnapshotLog()
val procFailure = ProcFailure(proc, FailureScheduler(props, clock))
return MessageProcessor(
inbox = inbox,
procState = proc,
codec = codec,
scheduleProcessor = ScheduleProcessor(noopTx, noopLock, proc, flights, events, log, opDay, ObjectMapper(), clock),
flopProcessor = FlopProcessor(noopTx, noopLock, flights, events, proc, ObjectMapper(), clock),
fdelProcessor = FdelProcessor(noopTx, noopLock, flights, events, proc, ObjectMapper(), clock),
adftProcessor = AdftProcessor(noopTx, noopLock, flights, events, proc, opDay, ObjectMapper(), clock),
procFailure = procFailure,
props = props,
clock = clock,
operationDayProps = opDay,
counters = counters,
)
}
@Test
fun `LDM subtype hits ignore rule and writes SKIPPED with reason`() {
val proc = StubProcState()
proc.insertIfAbsent(msgId, null)
val inbox = StubInbox()
inbox.raws[msgId] = "<RAW/>"
val msg = decoded("LDM", "ADQ")
val p = processor(proc = proc, inbox = inbox, codec = codecReturning(msg))
p.processOne(head())
val row = proc.find(msgId)!!
assertEquals(ProcStatus.SKIPPED, row.state)
assertEquals("ignored:LDM-*", row.lastError)
assertNotNull(row.backfillNextAt)
}
@Test
fun `EROR subtype hits ignore rule`() {
val proc = StubProcState()
proc.insertIfAbsent(msgId, null)
val inbox = StubInbox()
inbox.raws[msgId] = "<RAW/>"
val msg = decoded("EROR", "GEN")
val p = processor(proc = proc, inbox = inbox, codec = codecReturning(msg))
p.processOne(head())
assertEquals(ProcStatus.SKIPPED, proc.find(msgId)!!.state)
assertEquals("ignored:EROR-*", proc.find(msgId)!!.lastError)
}
@Test
fun `REGN and RSTA both hit ignore rules`() {
for ((type, rule) in listOf("REGN" to "REGN-*", "RSTA" to "RSTA-*")) {
val proc = StubProcState()
val id = msgId + type.hashCode().toLong()
proc.insertIfAbsent(id, null)
val inbox = StubInbox()
inbox.raws[id] = "<RAW/>"
val msg = DecodedMessage(
meta = MetaFields("AODB", type, "X", 300L, 1L),
kind = MsgKind.Unsupported("$type-X"),
rawXml = "<MSG/>",
)
val p = processor(proc = proc, inbox = inbox, codec = codecReturning(msg))
p.processOne(ProcState(id, ProcStatus.PENDING, updatedAt = Instant.EPOCH))
assertEquals(ProcStatus.SKIPPED, proc.find(id)!!.state)
assertEquals("ignored:$rule", proc.find(id)!!.lastError)
}
}
@Test
fun `ignore binds identity before skipping`() {
val proc = StubProcState()
proc.insertIfAbsent(msgId, null)
val inbox = StubInbox()
inbox.raws[msgId] = "<RAW/>"
val msg = decoded("LDM", "ADQ")
val p = processor(proc = proc, inbox = inbox, codec = codecReturning(msg))
p.processOne(head())
assertNotNull(proc.find(msgId)!!.identityKey)
}
@Test
fun `duplicate identity takes precedence over ignore`() {
val ownerMsgId = msgId + 1
val proc = StubProcState()
proc.insertIfAbsent(msgId, null)
proc.insertIfAbsent(ownerMsgId, null)
proc.tryBindIdentity(ownerMsgId, "AODB|LDM|ADQ|200")
val inbox = StubInbox()
inbox.raws[msgId] = "<RAW/>"
val msg = decoded("LDM", "ADQ")
val p = processor(proc = proc, inbox = inbox, codec = codecReturning(msg))
p.processOne(head())
val row = proc.find(msgId)!!
assertEquals(ProcStatus.SKIPPED, row.state)
assertEquals("duplicate-of:$ownerMsgId", row.lastError)
}
@Test
fun `non-ignored unsupported type continues as UNSUPPORTED`() {
val proc = StubProcState()
proc.insertIfAbsent(msgId, null)
val inbox = StubInbox()
inbox.raws[msgId] = "<RAW/>"
val msg = decoded("XYZZ", "TEST")
val counters = PipelineCounters()
val p = processor(proc = proc, inbox = inbox, counters = counters, codec = codecReturning(msg))
p.processOne(head())
assertEquals(ProcStatus.FAILED, proc.find(msgId)!!.state)
assertEquals(0L, counters.ignoredCount())
}
@Test
fun `ignore increments counter`() {
val proc = StubProcState()
proc.insertIfAbsent(msgId, null)
val inbox = StubInbox()
inbox.raws[msgId] = "<RAW/>"
val msg = decoded("LDM", "ADQ")
val counters = PipelineCounters()
val p = processor(proc = proc, inbox = inbox, counters = counters, codec = codecReturning(msg))
p.processOne(head())
assertEquals(1L, counters.ignoredCount())
}
}