fix(pipeline): 修复取报-处理-回填链路的超时、吞吐、回填与 FIFO 问题
外部调用（ACM2-33） - 共享 MySQL 与自有 PG 全部补有界超时：驱动 connect/socket 超时 + Hikari 池超时 （注意 Micronaut 的 Hikari 项是毫秒数，不是 Duration 字面量） - 删除主泵/毒丸路径的 inline 回填：跨库写不再占用 FIFO 关键路径，回填统一由扫描驱动 - 游标自愈：save() 改为「先 UPDATE、缺行 INSERT」，缺行只记一次 ERROR - Pump/Dispatcher 外层 catch 补 error 日志与失败计数 投递吞吐（ACM2-34） - Dispatcher 改批量领取；队头退避未到期或首条发送失败即停止本轮（保持目标内保序） - 有活不再 sleep；删除不可达的 state==SENT 死条件；markSent 移入分支； superseded 清理加轮数上限与空 eventId 保护 回填闭环（ACM2-36，V4） - MISSING（运行时确认行不存在）立即放弃自动重试并告警；暂时性故障达上限后停止自动重试 - 新增 reopen 人工恢复入口；放弃 ≠ 标记已确认（BACKFILL_AT 仍为空，清除前提不成立） - 扫描改 (BACKFILL_ATTEMPTS, MSG_ID) 公平轮转并排除放弃行，消除全局回填饥饿 - V4 只加字段与必要索引，不按年龄做任何存量推断 切流播种（ACM2-35，V5） - cutover-watermark 四模式（min/zero/max/显式 ID），默认不播种、代码不做默认选择 - 升级实例拒绝重新播种（SEEDED_AT 为 NULL ≠ 从未消费）；播种与水位同语句落库 - 非法取值由启动自检挡下 错误分类与入口契约（ACM2-37） - 未知 SCHD 子类型改为 UNSUPPORTED，不再静默当全量日计划合并 - ADFT 运营日冲突改走 ProtocolViolation → DEAD(PROTOCOL) - 兼容入口 receivedAt 缺失回退到注入 Clock；MessageLifecycleGate 强制注入 + 装配断言 - FIFO：主泵只领取 msgId ≤ W，兼容入口登记的行在水位追平前不被领取 时间源与可观测（ACM2-38 / ACM2-41 阶段 0） - 仓储/处理器/作业全部经注入 Clock；移除 markTerminal/markBackfilled 的 Instant.now() 默认值 - 退避表档位与 max-attempts 对齐并加启动自检 - Micrometer 7 个 gauge（@Context 急切注册）+ /health 与 /metrics 共用积压快照缓存 - 只读迟到检测：监视被放行的空洞 ID 是否后来真的出现，只计数告警、不补入队 Plane: ACM2-33 ACM2-34 ACM2-35 ACM2-36 ACM2-37 ACM2-38 ACM2-41 Tests: 88 → 120（1 skipped 需真实 PG）
This commit is contained in:
windyboy
@@ -14,9 +14,15 @@ import io.micronaut.context.ApplicationContext
|
||||
import io.micronaut.test.extensions.junit5.annotation.MicronautTest
|
||||
import jakarta.inject.Inject
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertFalse
|
||||
import org.junit.jupiter.api.Assertions.assertNotNull
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.Assertions.assertSame
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.time.Instant
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
* 不接任何外部中间件，用内存实现把整条链跑通：
|
||||
@@ -59,6 +65,18 @@ class PipelineSmokeTest {
|
||||
assertNotNull(controller)
|
||||
}
|
||||
|
||||
/**
|
||||
* 装配不变量：回填与人工重放必须共用**同一个** [com.gzzn.omms.msgexchange.processing.MessageLifecycleGate]。
|
||||
* 若两者各持一把锁，互斥失效，"旧回填给已重新入队的消息写标记"的窗口会重新打开。
|
||||
* 这条断言取代了以前"靠 Kotlin 默认参数值兜底"的隐患。
|
||||
*/
|
||||
@Test
|
||||
fun `backfill and replay share the same lifecycle gate`() {
|
||||
val backfill = ctx.getBean(com.gzzn.omms.msgexchange.processing.BackfillService::class.java)
|
||||
val replay = ctx.getBean(com.gzzn.omms.msgexchange.infra.retry.ReplayService::class.java)
|
||||
assertSame(backfill.lifecycleGate, replay.lifecycleGate)
|
||||
}
|
||||
|
||||
companion object {
|
||||
/** 报文头合法，但类型没有对应处理器，会走"暂不支持、先重试"这条路 */
|
||||
val UNSUPPORTED_XML = """
|
||||
@@ -73,6 +91,8 @@ class PipelineSmokeTest {
|
||||
val receipt = controller.send(UNSUPPORTED_XML)
|
||||
assertNotNull(receipt.body()) // 受理 ID
|
||||
val id = receipt.body()!!.toLong()
|
||||
// 兼容入口只保证"已落信 + 已入队"，**不推进水位**；必须先被收报发现（W 追平）才可领取。
|
||||
ctx.getBean(com.gzzn.omms.msgexchange.ingress.InboxPoller::class.java).pollOnce()
|
||||
|
||||
pump.tick() // 解码成功但无 DELY Handler → FAILED(UNSUPPORTED)
|
||||
|
||||
@@ -89,6 +109,7 @@ class PipelineSmokeTest {
|
||||
fun `replay reopens failed UNSUPPORTED row to PENDING`() {
|
||||
val receipt = controller.send(UNSUPPORTED_XML)
|
||||
val id = receipt.body()!!.toLong()
|
||||
ctx.getBean(com.gzzn.omms.msgexchange.ingress.InboxPoller::class.java).pollOnce()
|
||||
pump.tick()
|
||||
val stub = ctx.getBean(StubProcState::class.java)
|
||||
assertEquals(ProcStatus.FAILED, stub.snapshotOf(id)!!.state)
|
||||
@@ -103,22 +124,31 @@ class PipelineSmokeTest {
|
||||
}
|
||||
|
||||
/**
|
||||
* 回归用例：死信在进入终态时同时记下回填待办，并马上把标记写回信箱。
|
||||
* 回归用例：死信在进入终态时同时记下回填意图（同一条 UPDATE），由回填扫描把标记写回信箱。
|
||||
* 少了这一步，这些行永远占着每批的名额，攒够一批就再也发现不了新消息了。
|
||||
*
|
||||
* 注意：主泵**不再**在终态后立即回填（跨库写不能占用 FIFO 关键路径），
|
||||
* 因此这里显式触发一次扫描来验证回填链路。
|
||||
*/
|
||||
@Test
|
||||
fun `dead letter reaches terminal state, gets marked and cannot block later discovery`() {
|
||||
fun `dead letter reaches terminal state, gets marked by sweep and cannot block later discovery`() {
|
||||
val inbox = ctx.getBean(com.gzzn.omms.msgexchange.infra.stub.StubInbox::class.java)
|
||||
val proc = ctx.getBean(StubProcState::class.java)
|
||||
val dead = inbox.simulateExternalWrite("<MSG/>")
|
||||
ctx.getBean(com.gzzn.omms.msgexchange.ingress.InboxPoller::class.java).pollOnce()
|
||||
|
||||
pump.tick() // 解码 MALFORMED → DEAD + 回填意图（同一条 UPDATE）→ 提交后立即回填
|
||||
pump.tick() // 解码 MALFORMED → DEAD + 回填意图（同一条 UPDATE）
|
||||
|
||||
val row = proc.find(dead)!!
|
||||
assertEquals(ProcStatus.DEAD, row.state)
|
||||
assertEquals(ErrorClass.MALFORMED, row.errorClass)
|
||||
assertNotNull(row.backfillAt)
|
||||
assertNotNull(row.backfillNextAt) // 终态已登记回填意图
|
||||
assertFalse(inbox.isMarked(dead)) // 主泵不再内联回填
|
||||
assertNull(row.backfillAt)
|
||||
|
||||
ctx.getBean(com.gzzn.omms.msgexchange.processing.BackfillService::class.java).sweep()
|
||||
|
||||
assertNotNull(proc.find(dead)!!.backfillAt)
|
||||
assertTrue(inbox.isMarked(dead))
|
||||
|
||||
val fresh = inbox.simulateExternalWrite("<MSG/>")
|
||||
@@ -126,6 +156,105 @@ class PipelineSmokeTest {
|
||||
assertNotNull(proc.find(fresh)) // 死信不阻断后续发现
|
||||
}
|
||||
|
||||
/**
|
||||
* 不变量：非业务型终态（MALFORMED / PROTOCOL / SKIPPED / EXHAUSTED）**不触碰航班表、也不登记待发事件**——
|
||||
* 它们只写 `PROC_STATE` 一条记录（终态与回填意图是同一条 UPDATE）。
|
||||
*/
|
||||
@Test
|
||||
fun `non-business terminal states touch neither flight tables nor outbox`() {
|
||||
val inbox = ctx.getBean(com.gzzn.omms.msgexchange.infra.stub.StubInbox::class.java)
|
||||
val proc = ctx.getBean(StubProcState::class.java)
|
||||
val flights = ctx.getBean(com.gzzn.omms.msgexchange.infra.stub.StubFlightState::class.java)
|
||||
val events = ctx.getBean(StubMsgEvents::class.java)
|
||||
val id = inbox.simulateExternalWrite("<MSG/>") // 非法报文（无 META）→ MALFORMED
|
||||
ctx.getBean(com.gzzn.omms.msgexchange.ingress.InboxPoller::class.java).pollOnce()
|
||||
|
||||
pump.tick()
|
||||
|
||||
assertEquals(ProcStatus.DEAD, proc.find(id)!!.state)
|
||||
assertEquals(ErrorClass.MALFORMED, proc.find(id)!!.errorClass)
|
||||
assertTrue(flights.mains.isEmpty()) // 没碰航班表
|
||||
assertTrue(events.rows.isEmpty()) // 没登记待发事件
|
||||
}
|
||||
|
||||
/**
|
||||
* 【缺口基线 · G5】毒丸升级路径**不在** `MessageLifecycleGate` 内：即使门被别的线程持有，
|
||||
* 它也会照常把队头置为 `DEAD`。这不是期望行为（与人工重放并发时存在窗口），
|
||||
* 而是当前实现的已知缺口；把门覆盖到毒丸路径后，本用例必须反转成"先等待门"。
|
||||
*/
|
||||
@Test
|
||||
fun `baseline - poison escalation writes DEAD without taking the lifecycle gate`() {
|
||||
val proc = ctx.getBean(StubProcState::class.java)
|
||||
val props = ctx.getBean(com.gzzn.omms.msgexchange.config.PipelineProps::class.java)
|
||||
val backfill = ctx.getBean(com.gzzn.omms.msgexchange.processing.BackfillService::class.java)
|
||||
// 队头必须在水位以内才可领取：直接播种 PROC_STATE 的用例要显式把水位推上去。
|
||||
ctx.getBean(com.gzzn.omms.msgexchange.infra.stub.StubInboxCursor::class.java)
|
||||
.save(com.gzzn.omms.msgexchange.infra.persistence.InboxCursorRepository.Cursor(committedUpTo = 9001L))
|
||||
proc.insertIfAbsent(9001L, Instant.now())
|
||||
proc.update(
|
||||
9001L, ProcStatus.FAILED,
|
||||
attempts = props.pipeline.maxAttempts,
|
||||
lastError = "boom",
|
||||
)
|
||||
|
||||
val held = CountDownLatch(1)
|
||||
val release = CountDownLatch(1)
|
||||
val holder = Thread.ofPlatform().daemon(true).start {
|
||||
backfill.lifecycleGate.exclusive {
|
||||
held.countDown()
|
||||
release.await()
|
||||
}
|
||||
}
|
||||
try {
|
||||
assertTrue(held.await(1, TimeUnit.SECONDS), "持有者必须先拿到 gate")
|
||||
val ticker = Thread.ofPlatform().start { pump.tick() }
|
||||
ticker.join(2_000)
|
||||
assertFalse(ticker.isAlive, "毒丸路径不应等待 lifecycle gate（已知 G5 缺口）")
|
||||
assertEquals(ProcStatus.DEAD, proc.find(9001L)!!.state)
|
||||
assertEquals(ErrorClass.EXHAUSTED, proc.find(9001L)!!.errorClass)
|
||||
} finally {
|
||||
release.countDown()
|
||||
holder.join(1_000)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* FIFO 修复（G2）端到端验收：兼容入口写入的高 ID **不会被提前领取**，
|
||||
* 必须等水位追平（较小 ID 补齐并入队）之后才按顺序处理。
|
||||
*/
|
||||
@Test
|
||||
fun `compat injected high id is not claimed until the watermark catches up`() {
|
||||
val inbox = ctx.getBean(com.gzzn.omms.msgexchange.infra.stub.StubInbox::class.java)
|
||||
val proc = ctx.getBean(StubProcState::class.java)
|
||||
val cursor = ctx.getBean(com.gzzn.omms.msgexchange.infra.stub.StubInboxCursor::class.java)
|
||||
val poller = ctx.getBean(com.gzzn.omms.msgexchange.ingress.InboxPoller::class.java)
|
||||
|
||||
// 信箱：1 存在、2 是空洞（被删）、3 存在 → 水位停在 1
|
||||
inbox.simulateExternalWrite("<MSG/>")
|
||||
val hole = inbox.simulateExternalWrite("<MSG/>")
|
||||
inbox.simulateExternalWrite("<MSG/>")
|
||||
inbox.removeRow(hole)
|
||||
poller.pollOnce(Instant.now())
|
||||
assertEquals(1L, cursor.cursor.committedUpTo)
|
||||
|
||||
// 兼容入口写入高 ID：直接进 PG，但水位没追平（这正是原来的越序路径）
|
||||
val high = controller.send(UNSUPPORTED_XML).body()!!.toLong()
|
||||
assertTrue(high > 3L)
|
||||
|
||||
pump.tick() // 先按 FIFO 处理 ID=1
|
||||
assertEquals(ProcStatus.DEAD, proc.find(1L)!!.state)
|
||||
pump.tick() // 队头变成高 ID，但它在水位之外 → 不领取
|
||||
assertEquals(ProcStatus.PENDING, proc.find(high)!!.state)
|
||||
|
||||
// 空洞补齐 → 水位追平 → 才允许继续按顺序处理
|
||||
inbox.restoreRow(hole, "<MSG/>", Instant.now())
|
||||
poller.pollOnce(Instant.now())
|
||||
assertTrue(cursor.cursor.committedUpTo >= high)
|
||||
repeat(3) { pump.tick() } // 依次处理 2、3、高 ID
|
||||
assertEquals(ProcStatus.FAILED, proc.find(high)!!.state)
|
||||
assertEquals(ErrorClass.UNSUPPORTED, proc.find(high)!!.errorClass)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `dispatcher flushes schd batch through stub port`() {
|
||||
val events = ctx.getBean(StubMsgEvents::class.java)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.gzzn.omms.msgexchange.config
|
||||
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertThrows
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
/**
|
||||
@@ -15,11 +16,36 @@ class PipelinePropsTest {
|
||||
fun `backoff follows table then caps`() {
|
||||
assertEquals(1000, pipeline.backoffFor(1))
|
||||
assertEquals(2000, pipeline.backoffFor(2))
|
||||
assertEquals(16000, pipeline.backoffFor(5))
|
||||
assertEquals(60_000, pipeline.backoffFor(6)) // 表外 → 封顶
|
||||
assertEquals(8000, pipeline.backoffFor(4)) // 最后一档：max-attempts - 1 = 4
|
||||
assertEquals(60_000, pipeline.backoffFor(5)) // 表外 → 封顶
|
||||
assertEquals(60_000, pipeline.backoffFor(99))
|
||||
}
|
||||
|
||||
/**
|
||||
* 配置自检：档位数必须等于 max-attempts − 1，否则有一段退避**永远走不到**
|
||||
* （这正是默认配置曾经的错误：5 次尝试配了 5 档，16 秒档不可达）。
|
||||
*/
|
||||
@Test
|
||||
fun `validate rejects a backoff table whose size does not match max-attempts`() {
|
||||
pipeline.validate() // 默认配置必须自洽
|
||||
|
||||
pipeline.backoffMs = listOf(1000, 2000, 4000, 8000, 16000)
|
||||
assertThrows(IllegalArgumentException::class.java) { pipeline.validate() }
|
||||
}
|
||||
|
||||
/** 切流播种只接受 min|zero|max|<id>：非法值必须在启动时挡掉。 */
|
||||
@Test
|
||||
fun `validate rejects an unknown cutover watermark mode`() {
|
||||
pipeline.cutoverWatermark = "bogus"
|
||||
assertThrows(IllegalArgumentException::class.java) { pipeline.validate() }
|
||||
|
||||
pipeline.cutoverWatermark = "max"
|
||||
pipeline.validate()
|
||||
|
||||
pipeline.cutoverWatermark = "12345"
|
||||
pipeline.validate()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `non-positive attempt never throws and falls back to first slot`() {
|
||||
assertEquals(1000, pipeline.backoffFor(0))
|
||||
|
||||
@@ -48,7 +48,7 @@ class HealthIndicatorsTest {
|
||||
val oldest = MutableClock.BASE.minusSeconds(3600)
|
||||
proc.insertIfAbsent(1L, oldest)
|
||||
proc.insertIfAbsent(2L, MutableClock.BASE)
|
||||
proc.markTerminal(2L, ProcStatus.SUCCEEDED)
|
||||
proc.markTerminal(2L, ProcStatus.SUCCEEDED, now = MutableClock.BASE)
|
||||
val newest = inbox.insertRaw("<MSG/>")
|
||||
cursor.save(InboxCursorRepository.Cursor(committedUpTo = newest - 2))
|
||||
|
||||
@@ -65,7 +65,7 @@ class HealthIndicatorsTest {
|
||||
|
||||
@Test
|
||||
fun `inbox lifecycle stays up when ports are not bound`() {
|
||||
val result = lifecycleHealth(procState = null, cursor = null, mailbox = null)
|
||||
val result = lifecycleHealth(procState = null, cursor = null, mailbox = null, now = MutableClock.BASE)
|
||||
|
||||
assertEquals(HealthStatus.UP, result.status) // 可用性由依赖自身指示器承担
|
||||
}
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
package com.gzzn.omms.msgexchange.infra.metrics
|
||||
|
||||
import com.gzzn.omms.msgexchange.domain.ProcStatus
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
|
||||
import io.micrometer.core.instrument.MeterRegistry
|
||||
import io.micronaut.context.ApplicationContext
|
||||
import io.micronaut.context.annotation.Property
|
||||
import io.micronaut.test.extensions.junit5.annotation.MicronautTest
|
||||
import jakarta.inject.Inject
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.BeforeEach
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.time.Instant
|
||||
|
||||
/**
|
||||
* 指标接入回归：管道关键量必须真的注册进 `MeterRegistry` 且取数正确。
|
||||
* 抓取端点由 Micronaut Micrometer 提供，这里只验证"注册 + 取值"。
|
||||
*
|
||||
* 缓存窗口在测试里置 0，保证断言看到的是当前 stub 状态而不是缓存快照。
|
||||
*/
|
||||
@MicronautTest
|
||||
@Property(name = "msgx.health.backlog-cache-ttl-ms", value = "0")
|
||||
class PipelineMetricsTest {
|
||||
|
||||
@Inject
|
||||
lateinit var ctx: ApplicationContext
|
||||
|
||||
@Inject
|
||||
lateinit var registry: MeterRegistry
|
||||
|
||||
@BeforeEach
|
||||
fun clean() {
|
||||
ctx.getBean(StubProcState::class.java).clear()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `backlog gauges are registered and reflect the current pipeline state`() {
|
||||
val proc = ctx.getBean(StubProcState::class.java)
|
||||
val t0 = Instant.parse("2026-09-08T03:00:00Z")
|
||||
proc.insertIfAbsent(1L, t0) // 未处理 → backlog
|
||||
proc.insertIfAbsent(2L, t0)
|
||||
proc.markTerminal(2L, ProcStatus.SUCCEEDED, now = t0) // 终态未打标 → unmarked
|
||||
|
||||
assertEquals(1.0, gauge("msgx.pipeline.backlog.unfinished"), 0.001)
|
||||
assertEquals(1.0, gauge("msgx.pipeline.backfill.unmarked_terminal"), 0.001)
|
||||
assertEquals(0.0, gauge("msgx.pipeline.backfill.abandoned"), 0.001)
|
||||
// 信箱为空 → 没有"最新 ID"可比，滞后用 -1（无值）而不是伪造 0
|
||||
assertEquals(-1.0, gauge("msgx.pipeline.watermark.lag"), 0.001)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `permanent hole releases are exposed as a monotonic total`() {
|
||||
val counters = ctx.getBean(PipelineCounters::class.java)
|
||||
val before = gauge("msgx.pipeline.hole.aged_out.total")
|
||||
counters.holeAgedOutIncrement()
|
||||
counters.holeAgedOutIncrement()
|
||||
|
||||
assertEquals(before + 2.0, gauge("msgx.pipeline.hole.aged_out.total"), 0.001)
|
||||
}
|
||||
|
||||
private fun gauge(name: String): Double =
|
||||
requireNotNull(registry.find(name).gauge()) { "gauge not registered: $name" }.value()
|
||||
}
|
||||
+13
-5
@@ -40,7 +40,7 @@ class FlywayMigrationTest {
|
||||
|
||||
DriverManager.getConnection(url, user, pass).use { conn ->
|
||||
conn.createStatement().use { stmt ->
|
||||
// 迁移记录：V1 基线 + V2 信箱生命周期 + V3 稳定处理起点
|
||||
// 迁移记录：V1 基线 + V2 信箱生命周期 + V3 稳定处理起点 + V4 回填闭环 + V5 切流播种
|
||||
stmt.executeQuery(
|
||||
"SELECT version, script, success FROM flyway_schema_history ORDER BY installed_rank ASC",
|
||||
).use { rs ->
|
||||
@@ -48,13 +48,17 @@ class FlywayMigrationTest {
|
||||
while (rs.next()) {
|
||||
records.add(Triple(rs.getString("version"), rs.getString("script"), rs.getBoolean("success")))
|
||||
}
|
||||
assertTrue(records.size >= 3, "flyway_schema_history must record all migrations")
|
||||
assertTrue(records.size >= 5, "flyway_schema_history must record all migrations")
|
||||
assertEquals("1", records[0].first)
|
||||
assertEquals("V1__flight_state_baseline.sql", records[0].second)
|
||||
assertEquals("2", records[1].first)
|
||||
assertEquals("V2__inbox_lifecycle.sql", records[1].second)
|
||||
assertEquals("3", records[2].first)
|
||||
assertEquals("V3__stable_processing_start.sql", records[2].second)
|
||||
assertEquals("4", records[3].first)
|
||||
assertEquals("V4__backfill_closure.sql", records[3].second)
|
||||
assertEquals("5", records[4].first)
|
||||
assertEquals("V5__cutover_seed.sql", records[4].second)
|
||||
assertTrue(records.all { it.third })
|
||||
}
|
||||
|
||||
@@ -79,16 +83,20 @@ class FlywayMigrationTest {
|
||||
assertEquals(setOf("flid", "operation_day", "state", "state_version", "last_msg_id"), cols)
|
||||
}
|
||||
|
||||
// 回填相关的列都落在 PROC_STATE 上（收信时间用来判断超期，其余记录回填进度）
|
||||
// 回填相关的列都落在 PROC_STATE 上（收信时间判断超期；abandoned 记录"停止自动重试"）
|
||||
stmt.executeQuery(
|
||||
"SELECT column_name FROM information_schema.columns WHERE table_name = 'proc_state' " +
|
||||
"AND column_name IN ('received_at', 'backfill_at', 'backfill_next_at', " +
|
||||
"'backfill_attempts', 'backfill_error', 'processing_started_at')",
|
||||
"'backfill_attempts', 'backfill_error', 'backfill_abandoned_at', " +
|
||||
"'backfill_abandoned_reason', 'processing_started_at')",
|
||||
).use { rs ->
|
||||
val cols = mutableSetOf<String>()
|
||||
while (rs.next()) cols.add(rs.getString("column_name"))
|
||||
assertEquals(
|
||||
setOf("received_at", "backfill_at", "backfill_next_at", "backfill_attempts", "backfill_error", "processing_started_at"),
|
||||
setOf(
|
||||
"received_at", "backfill_at", "backfill_next_at", "backfill_attempts", "backfill_error",
|
||||
"backfill_abandoned_at", "backfill_abandoned_reason", "processing_started_at",
|
||||
),
|
||||
cols,
|
||||
)
|
||||
}
|
||||
|
||||
+5
-2
@@ -58,9 +58,9 @@ class InboxLifecycleJdbcSqlTest {
|
||||
)
|
||||
}
|
||||
}
|
||||
proc = JdbcProcStateRepository(ds)
|
||||
proc = JdbcProcStateRepository(ds, java.time.Clock.systemUTC())
|
||||
mailbox = JdbcCminmsgInboxRepository(ds)
|
||||
cursor = JdbcInboxCursorRepository(ds)
|
||||
cursor = JdbcInboxCursorRepository(ds, java.time.Clock.systemUTC())
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -223,6 +223,8 @@ class InboxLifecycleJdbcSqlTest {
|
||||
backfill_next_at TIMESTAMP WITH TIME ZONE,
|
||||
backfill_attempts INT NOT NULL DEFAULT 0,
|
||||
backfill_error VARCHAR(512),
|
||||
backfill_abandoned_at TIMESTAMP WITH TIME ZONE,
|
||||
backfill_abandoned_reason VARCHAR(64),
|
||||
processing_started_at TIMESTAMP WITH TIME ZONE,
|
||||
updated_at TIMESTAMP WITH TIME ZONE NOT NULL,
|
||||
CONSTRAINT uk_proc_identity UNIQUE (identity_key)
|
||||
@@ -234,6 +236,7 @@ class InboxLifecycleJdbcSqlTest {
|
||||
cursor_id INT PRIMARY KEY,
|
||||
committed_up_to BIGINT NOT NULL,
|
||||
hole_since TIMESTAMP WITH TIME ZONE,
|
||||
seeded_at TIMESTAMP WITH TIME ZONE,
|
||||
updated_at TIMESTAMP WITH TIME ZONE NOT NULL
|
||||
)
|
||||
"""
|
||||
|
||||
@@ -47,11 +47,17 @@ class ReplayServiceTest {
|
||||
|
||||
override fun recordBackfillFailure(msgId: Long, error: String?, attempts: Int, nextAttemptAt: Instant, now: Instant) = Unit
|
||||
|
||||
override fun markBackfillAbandoned(msgId: Long, reason: String, now: Instant): Boolean = false
|
||||
|
||||
override fun reopenBackfill(msgId: Long, now: Instant): Boolean = false
|
||||
|
||||
override fun findBackfillDue(now: Instant, overdueBefore: Instant, limit: Int) =
|
||||
emptyList<com.gzzn.omms.msgexchange.infra.persistence.BackfillDue>()
|
||||
|
||||
override fun backlog() = com.gzzn.omms.msgexchange.infra.persistence.Backlog(0, null, 0)
|
||||
|
||||
override fun hasAny(): Boolean = rows.isNotEmpty()
|
||||
|
||||
override fun requeueByErrorClasses(errorClasses: List<ErrorClass>): Int {
|
||||
requeueCalls += errorClasses
|
||||
var n = 0
|
||||
@@ -74,7 +80,8 @@ class ReplayServiceTest {
|
||||
repo.seed(2, ProcStatus.DEAD, ErrorClass.MALFORMED)
|
||||
repo.seed(3, ProcStatus.FAILED, ErrorClass.UNSUPPORTED)
|
||||
|
||||
val n = ReplayService(repo).replay(listOf(ErrorClass.CODEC_ERROR, ErrorClass.MALFORMED, ErrorClass.UNSUPPORTED))
|
||||
val n = ReplayService(repo, com.gzzn.omms.msgexchange.processing.MessageLifecycleGate())
|
||||
.replay(listOf(ErrorClass.CODEC_ERROR, ErrorClass.MALFORMED, ErrorClass.UNSUPPORTED))
|
||||
|
||||
assertEquals(2, n)
|
||||
assertEquals(listOf(listOf(ErrorClass.CODEC_ERROR, ErrorClass.UNSUPPORTED)), repo.requeueCalls)
|
||||
@@ -91,7 +98,8 @@ class ReplayServiceTest {
|
||||
val repo = FakeRepo()
|
||||
repo.seed(2, ProcStatus.DEAD, ErrorClass.MALFORMED)
|
||||
|
||||
val n = ReplayService(repo).replay(listOf(ErrorClass.MALFORMED))
|
||||
val n = ReplayService(repo, com.gzzn.omms.msgexchange.processing.MessageLifecycleGate())
|
||||
.replay(listOf(ErrorClass.MALFORMED))
|
||||
|
||||
assertEquals(0, n)
|
||||
assertNull(repo.requeueCalls.lastOrNull()) // 白名单过滤后为空 → 不触达仓储
|
||||
@@ -105,7 +113,7 @@ class ReplayServiceTest {
|
||||
repo.seed(5, ProcStatus.DEAD, ErrorClass.EXHAUSTED)
|
||||
repo.seed(6, ProcStatus.DEAD, ErrorClass.MALFORMED)
|
||||
|
||||
val n = ReplayService(repo).replayAll()
|
||||
val n = ReplayService(repo, com.gzzn.omms.msgexchange.processing.MessageLifecycleGate()).replayAll()
|
||||
|
||||
assertEquals(2, n)
|
||||
assertEquals(ProcStatus.PENDING, repo.rows[4]!!.state)
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
package com.gzzn.omms.msgexchange.ingress
|
||||
|
||||
import com.gzzn.omms.msgexchange.config.PipelineProps
|
||||
import com.gzzn.omms.msgexchange.infra.metrics.PipelineCounters
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.InboxCursorRepository
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubInbox
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubInboxCursor
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubPipelineTx
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertNotEquals
|
||||
import org.junit.jupiter.api.Assertions.assertNotNull
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.BeforeEach
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.time.Clock
|
||||
import java.time.Instant
|
||||
import java.time.ZoneOffset
|
||||
|
||||
/**
|
||||
* 切流水位播种（ACM2-35）：**显式、一次性、升级安全**。
|
||||
*
|
||||
* 覆盖评审提出的四条硬要求：
|
||||
* 1. 默认（未配置）不动作，代码不做默认选择；
|
||||
* 2. 四种模式严格区分（`min` 读现存全部 / `zero` 从 0 按空洞规则 / `max` 跳过可见存量 / 显式 ID）；
|
||||
* 3. 升级实例（已有水位或已有处理记录）**拒绝重新播种**——`SEEDED_AT` 为 NULL 不等于"从未消费"；
|
||||
* 4. 播种至多一次，且不会被普通的水位推进抹掉。
|
||||
*/
|
||||
class CutoverSeedTest {
|
||||
|
||||
private val t0: Instant = Instant.parse("2026-09-08T03:00:00Z")
|
||||
private val props = PipelineProps()
|
||||
|
||||
private lateinit var inbox: StubInbox
|
||||
private lateinit var proc: StubProcState
|
||||
private lateinit var cursor: StubInboxCursor
|
||||
private lateinit var poller: InboxPoller
|
||||
|
||||
@BeforeEach
|
||||
fun setUp() {
|
||||
inbox = StubInbox().apply { clear() }
|
||||
proc = StubProcState().apply { clear() }
|
||||
cursor = StubInboxCursor().apply { clear() }
|
||||
props.pipeline.cutoverWatermark = null
|
||||
poller = InboxPoller(
|
||||
inbox, proc, cursor, StubPipelineTx(), props,
|
||||
Clock.fixed(t0, ZoneOffset.UTC), PipelineCounters(),
|
||||
)
|
||||
}
|
||||
|
||||
/** 造一个 MIN(ID)=5 的信箱：1..4 已被库方清除（典型"最老分区已 DROP"）。 */
|
||||
private fun mailboxWithMinId5(): List<Long> {
|
||||
val ids = (1..8).map { inbox.insertRaw("<MSG/>") }
|
||||
ids.take(4).forEach { inbox.removeRow(it) }
|
||||
return ids.drop(4)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `default does nothing - no seeding without explicit configuration`() {
|
||||
val kept = mailboxWithMinId5()
|
||||
|
||||
assertEquals(0, poller.pollOnce(t0)) // W=0 → ID=1 判为空洞，不推进
|
||||
|
||||
assertNull(cursor.cursor.seededAt)
|
||||
assertEquals(0L, cursor.cursor.committedUpTo)
|
||||
assertNotNull(cursor.cursor.holeSince)
|
||||
assertTrue(kept.all { proc.find(it) == null })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `min mode reads all currently existing rows`() {
|
||||
val kept = mailboxWithMinId5()
|
||||
props.pipeline.cutoverWatermark = "min"
|
||||
|
||||
assertEquals(4, poller.pollOnce(t0)) // 播种 W=4，同一轮把 5..8 全部读入
|
||||
|
||||
assertNotNull(cursor.cursor.seededAt)
|
||||
assertEquals(8L, cursor.cursor.committedUpTo)
|
||||
assertTrue(kept.all { proc.find(it) != null })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `max mode skips the currently visible backlog`() {
|
||||
mailboxWithMinId5()
|
||||
props.pipeline.cutoverWatermark = "max"
|
||||
|
||||
assertEquals(0, poller.pollOnce(t0))
|
||||
|
||||
assertNotNull(cursor.cursor.seededAt)
|
||||
assertEquals(8L, cursor.cursor.committedUpTo) // 直接跳到 MAX(ID)
|
||||
assertEquals(0, proc.rows.size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `zero mode scans from zero and stops at the first hole`() {
|
||||
mailboxWithMinId5()
|
||||
props.pipeline.cutoverWatermark = "zero"
|
||||
|
||||
assertEquals(0, poller.pollOnce(t0))
|
||||
|
||||
assertNotNull(cursor.cursor.seededAt)
|
||||
assertEquals(0L, cursor.cursor.committedUpTo) // 1..4 是空洞 → 按空洞规则停住
|
||||
assertNotNull(cursor.cursor.holeSince)
|
||||
assertEquals(0, proc.rows.size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `explicit boundary mode is accepted`() {
|
||||
mailboxWithMinId5()
|
||||
props.pipeline.cutoverWatermark = "6"
|
||||
|
||||
poller.pollOnce(t0)
|
||||
|
||||
assertNotNull(cursor.cursor.seededAt)
|
||||
assertEquals(8L, cursor.cursor.committedUpTo) // 从 6 起读 7..8（6 本身已在界内）
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `upgraded instance with existing watermark refuses to re-seed`() {
|
||||
mailboxWithMinId5()
|
||||
cursor.save(InboxCursorRepository.Cursor(committedUpTo = 3L, holeSince = t0))
|
||||
props.pipeline.cutoverWatermark = "max"
|
||||
|
||||
poller.pollOnce(t0)
|
||||
|
||||
assertNull(cursor.cursor.seededAt) // 没有重新播种
|
||||
assertNotEquals(8L, cursor.cursor.committedUpTo)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `instance that already consumed messages refuses to seed even at W zero`() {
|
||||
mailboxWithMinId5()
|
||||
proc.insertIfAbsent(1L, t0) // 已有处理记录（即使水位还是 0）
|
||||
props.pipeline.cutoverWatermark = "max"
|
||||
|
||||
poller.pollOnce(t0)
|
||||
|
||||
assertNull(cursor.cursor.seededAt)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `seeding happens at most once and survives normal watermark saves`() {
|
||||
mailboxWithMinId5()
|
||||
props.pipeline.cutoverWatermark = "max"
|
||||
poller.pollOnce(t0)
|
||||
val seededAt = cursor.cursor.seededAt
|
||||
assertNotNull(seededAt)
|
||||
|
||||
cursor.save(InboxCursorRepository.Cursor(committedUpTo = 2L, holeSince = null))
|
||||
poller.pollOnce(t0.plusSeconds(1))
|
||||
|
||||
assertEquals(seededAt, cursor.cursor.seededAt) // 标记仍在：普通 save 不抹播种事实
|
||||
assertEquals(2L, cursor.cursor.committedUpTo)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `empty mailbox is left unseeded`() {
|
||||
props.pipeline.cutoverWatermark = "max"
|
||||
|
||||
assertEquals(0, poller.pollOnce(t0))
|
||||
|
||||
assertNull(cursor.cursor.seededAt)
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertFalse
|
||||
import org.junit.jupiter.api.Assertions.assertNotNull
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.BeforeEach
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.time.Instant
|
||||
@@ -38,7 +39,11 @@ class InboxPollerTest {
|
||||
inbox = StubInbox().apply { clear() }
|
||||
proc = StubProcState().apply { clear() }
|
||||
cursor = StubInboxCursor().apply { clear() }
|
||||
poller = InboxPoller(inbox, proc, cursor, StubPipelineTx(), props)
|
||||
poller = InboxPoller(
|
||||
inbox, proc, cursor, StubPipelineTx(), props,
|
||||
java.time.Clock.fixed(t0, java.time.ZoneOffset.UTC),
|
||||
com.gzzn.omms.msgexchange.infra.metrics.PipelineCounters(),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -67,7 +72,7 @@ class InboxPollerTest {
|
||||
val dead = (1..3).map { inbox.simulateExternalWrite("<MSG/>") }
|
||||
assertEquals(3, poller.pollOnce(t0))
|
||||
dead.forEach {
|
||||
proc.markTerminal(it, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, lastError = "raw-missing")
|
||||
proc.markTerminal(it, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, lastError = "raw-missing", now = t0)
|
||||
}
|
||||
|
||||
val fresh = inbox.simulateExternalWrite("<MSG/>")
|
||||
@@ -112,7 +117,7 @@ class InboxPollerTest {
|
||||
|
||||
@Test
|
||||
fun `compat http path and poller do not double enqueue the same message`() {
|
||||
val receipt = InboxService(inbox, proc).accept("<MSG/>")
|
||||
val receipt = InboxService(inbox, proc, java.time.Clock.fixed(t0, java.time.ZoneOffset.UTC)).accept("<MSG/>")
|
||||
|
||||
assertEquals(0, poller.pollOnce(t0))
|
||||
assertEquals(receipt.msgId, cursor.cursor.committedUpTo) // 已在 PG：读取进度照常推进
|
||||
@@ -144,4 +149,60 @@ class InboxPollerTest {
|
||||
assertNull(proc.find(fifth))
|
||||
assertEquals(first + 2, third)
|
||||
}
|
||||
|
||||
/**
|
||||
* 【缺口基线 · G1】快路径只读 `ID > W`、从不回头：水位越过某个 ID 之后，该 ID 即使后来
|
||||
* 出现在信箱里也不会再被发现。这不是期望行为，而是"没有补偿扫描"的**已知缺口**；
|
||||
* 本用例把它固定成基线，补偿扫描（ACM2-41 / G1）落地后必须反转成"能被发现并安全处置"。
|
||||
*/
|
||||
@Test
|
||||
fun `baseline - an id that appears after the watermark passed it is never discovered`() {
|
||||
inbox.simulateExternalWrite("<MSG/>") // 1
|
||||
val hole = inbox.simulateExternalWrite("<MSG/>") // 2
|
||||
val third = inbox.simulateExternalWrite("<MSG/>") // 3
|
||||
inbox.removeRow(hole)
|
||||
|
||||
poller.pollOnce(t0) // W 停在 1，空洞在 2
|
||||
val agedOut = t0.plus(props.pipeline.maxCommitDelay)
|
||||
poller.pollOnce(agedOut) // 空洞判永久 → 放行
|
||||
poller.pollOnce(agedOut) // 发现 3
|
||||
assertEquals(third, cursor.cursor.committedUpTo)
|
||||
|
||||
// 迟到的 2 现在才出现：水位已经越过它，快路径再也不会读它。
|
||||
inbox.restoreRow(hole, "<MSG/>", agedOut)
|
||||
|
||||
assertEquals(0, poller.pollOnce(agedOut.plusSeconds(1)))
|
||||
assertNull(proc.find(hole))
|
||||
}
|
||||
|
||||
/**
|
||||
* 兼容入口会把消息登记在水位**之外**（`msgId > W`），因此它天然成为"最小未完成行"。
|
||||
* 领取侧的守卫在主泵：只领 `msgId <= W`（端到端验收见 `PipelineSmokeTest` 的
|
||||
* "compat injected high id is not claimed until the watermark catches up"）。
|
||||
* 本用例只固定收报侧的事实：**登记发生，但水位不动**。
|
||||
*/
|
||||
@Test
|
||||
fun `compat accept registers a row above the watermark without advancing it`() {
|
||||
props.pipeline.claimBatch = 10
|
||||
val one = inbox.simulateExternalWrite("<MSG/>") // 1
|
||||
val hole = inbox.simulateExternalWrite("<MSG/>") // 2（空洞）
|
||||
val third = inbox.simulateExternalWrite("<MSG/>") // 3
|
||||
inbox.removeRow(hole)
|
||||
|
||||
assertEquals(1, poller.pollOnce(t0)) // W=1，遇空洞即停
|
||||
assertEquals(one, cursor.cursor.committedUpTo)
|
||||
assertNull(proc.find(third)) // 3 还没入队
|
||||
|
||||
val receipt = InboxService(inbox, proc, java.time.Clock.fixed(t0, java.time.ZoneOffset.UTC))
|
||||
.accept("<MSG/>") // 兼容入口：高 ID 直接进 PG
|
||||
val high = receipt.msgId
|
||||
assertTrue(high > third)
|
||||
|
||||
assertEquals(one, cursor.cursor.committedUpTo) // 水位不动（不参与水位）
|
||||
// 主泵处理掉队头 1 之后，最小未完成行就是兼容入口写进来的高 ID……
|
||||
proc.markTerminal(one, ProcStatus.SUCCEEDED, now = t0)
|
||||
assertEquals(high, proc.headUnfinished()!!.msgId)
|
||||
assertTrue(high > cursor.cursor.committedUpTo) // ……但它超出水位，主泵不会领取
|
||||
assertNull(proc.find(third))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
package com.gzzn.omms.msgexchange.ingress
|
||||
|
||||
import com.gzzn.omms.msgexchange.config.PipelineProps
|
||||
import com.gzzn.omms.msgexchange.infra.metrics.PipelineCounters
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubInbox
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubInboxCursor
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubPipelineTx
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertNotNull
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.BeforeEach
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.time.Clock
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
import java.time.ZoneOffset
|
||||
|
||||
/**
|
||||
* 迟到到达检测（ACM2-41 阶段 0）：**只读观测**。
|
||||
*
|
||||
* 语义：被判定为永久空洞并放行的 ID，如果后来真的出现在信箱里，就是"上游提交晚于水位推进"。
|
||||
* 阶段 0 只计数与告警，**不入队、不改变处理语义**（补入队属阶段 1，需先与库方定案）。
|
||||
*/
|
||||
class LateArrivalDetectTest {
|
||||
|
||||
private val t0: Instant = Instant.parse("2026-09-08T03:00:00Z")
|
||||
private val props = PipelineProps()
|
||||
|
||||
private lateinit var inbox: StubInbox
|
||||
private lateinit var proc: StubProcState
|
||||
private lateinit var cursor: StubInboxCursor
|
||||
private lateinit var counters: PipelineCounters
|
||||
private lateinit var poller: InboxPoller
|
||||
|
||||
@BeforeEach
|
||||
fun setUp() {
|
||||
inbox = StubInbox().apply { clear() }
|
||||
proc = StubProcState().apply { clear() }
|
||||
cursor = StubInboxCursor().apply { clear() }
|
||||
counters = PipelineCounters()
|
||||
poller = InboxPoller(inbox, proc, cursor, StubPipelineTx(), props, Clock.fixed(t0, ZoneOffset.UTC), counters)
|
||||
}
|
||||
|
||||
/** 造出"1 存在、2 是空洞、3 存在"，并把空洞等到超期放行。返回迟到的那个 ID。 */
|
||||
private fun ageOutHoleAt2(): Long {
|
||||
inbox.simulateExternalWrite("<MSG/>") // 1
|
||||
val hole = inbox.simulateExternalWrite("<MSG/>") // 2
|
||||
inbox.simulateExternalWrite("<MSG/>") // 3
|
||||
inbox.removeRow(hole)
|
||||
poller.pollOnce(t0) // W=1，空洞在 2
|
||||
poller.pollOnce(t0.plus(props.pipeline.maxCommitDelay)) // 空洞判永久 → 放行
|
||||
return hole
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a hole that really appears later is detected and counted`() {
|
||||
val hole = ageOutHoleAt2()
|
||||
inbox.restoreRow(hole, "<MSG/>", t0.plus(props.pipeline.maxCommitDelay))
|
||||
|
||||
// 过了检测周期再轮询一次
|
||||
poller.pollOnce(t0.plus(props.pipeline.maxCommitDelay).plus(props.pipeline.lateDetectPeriod))
|
||||
|
||||
assertEquals(1L, counters.lateArrivalDetectedCount())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `detection does not enqueue the late message - phase 0 is observation only`() {
|
||||
val hole = ageOutHoleAt2()
|
||||
inbox.restoreRow(hole, "<MSG/>", t0.plus(props.pipeline.maxCommitDelay))
|
||||
|
||||
poller.pollOnce(t0.plus(props.pipeline.maxCommitDelay).plus(props.pipeline.lateDetectPeriod))
|
||||
|
||||
assertEquals(1L, counters.lateArrivalDetectedCount())
|
||||
assertNull(proc.find(hole)) // 仍然不会被补入队
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a hole that stays absent is not counted`() {
|
||||
ageOutHoleAt2()
|
||||
|
||||
poller.pollOnce(t0.plus(props.pipeline.maxCommitDelay).plus(props.pipeline.lateDetectPeriod))
|
||||
|
||||
assertEquals(0L, counters.lateArrivalDetectedCount())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `detection does not run before the configured period elapses`() {
|
||||
val hole = ageOutHoleAt2()
|
||||
inbox.restoreRow(hole, "<MSG/>", t0)
|
||||
|
||||
// 只过了一个 max-commit-delay，未到检测周期
|
||||
poller.pollOnce(t0.plus(props.pipeline.maxCommitDelay).plusSeconds(1))
|
||||
|
||||
assertEquals(0L, counters.lateArrivalDetectedCount())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `detection can be switched off`() {
|
||||
props.pipeline.lateDetectPeriod = Duration.ZERO
|
||||
val hole = ageOutHoleAt2()
|
||||
inbox.restoreRow(hole, "<MSG/>", t0)
|
||||
|
||||
poller.pollOnce(t0.plus(props.pipeline.maxCommitDelay).plus(Duration.ofHours(1)))
|
||||
|
||||
assertEquals(0L, counters.lateArrivalDetectedCount())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `the same late id is counted once even after the hole is aged out again`() {
|
||||
val hole = ageOutHoleAt2()
|
||||
inbox.restoreRow(hole, "<MSG/>", t0.plus(props.pipeline.maxCommitDelay))
|
||||
val later = t0.plus(props.pipeline.maxCommitDelay).plus(props.pipeline.lateDetectPeriod)
|
||||
poller.pollOnce(later)
|
||||
assertEquals(1L, counters.lateArrivalDetectedCount())
|
||||
|
||||
poller.pollOnce(later.plus(props.pipeline.lateDetectPeriod))
|
||||
|
||||
assertEquals(1L, counters.lateArrivalDetectedCount()) // 已命中的 ID 不再重复计数
|
||||
assertNotNull(cursor.cursor)
|
||||
}
|
||||
}
|
||||
@@ -41,21 +41,24 @@ class BackfillServiceTest {
|
||||
private val props = PipelineProps()
|
||||
|
||||
/** 可以人为制造故障的信箱，用来验证回填失败时怎么处理。 */
|
||||
private class FakeMailbox(var fail: Boolean = false) : CminmsgInboxRepository {
|
||||
private class FakeMailbox(var fail: Boolean = false, var missing: Boolean = false) : CminmsgInboxRepository {
|
||||
val marked = linkedSetOf<Long>()
|
||||
override fun insertRaw(rawXml: String): Long = 1L
|
||||
override fun rawOf(msgId: Long): String? = null
|
||||
override fun receivedAtOf(msgId: Long): Instant? = null
|
||||
override fun readRange(fromExclusive: Long, limit: Int): List<MailboxRow> = emptyList()
|
||||
override fun maxId(): Long? = null
|
||||
override fun minId(): Long? = null
|
||||
override fun existingIds(msgIds: Collection<Long>): Set<Long> = emptySet()
|
||||
override fun markProcessedIfUnmarked(msgId: Long, value: String): MailboxMarkResult {
|
||||
if (fail) throw IllegalStateException("mysql-down")
|
||||
if (missing) return MailboxMarkResult.MISSING
|
||||
return if (marked.add(msgId)) MailboxMarkResult.MARKED else MailboxMarkResult.ALREADY_MARKED
|
||||
}
|
||||
}
|
||||
|
||||
private fun service(proc: StubProcState, mailbox: CminmsgInboxRepository, now: Instant = t0) =
|
||||
BackfillService(proc, mailbox, MailboxProps(), props, Clock.fixed(now, ZoneOffset.UTC))
|
||||
BackfillService(proc, mailbox, MailboxProps(), props, Clock.fixed(now, ZoneOffset.UTC), MessageLifecycleGate())
|
||||
|
||||
/** 终态 + 回填意图（固定时刻，避免依赖真实时钟）。 */
|
||||
private fun succeeded(proc: StubProcState, id: Long) {
|
||||
@@ -122,8 +125,13 @@ class BackfillServiceTest {
|
||||
assertNull(row.backfillAt)
|
||||
}
|
||||
|
||||
/**
|
||||
* 评审修正 R3：信箱行不存在是**确定性结论**，不再当作"可重试失败"——
|
||||
* 重试不会改变结果，只会每 30 秒重试一次并永久占满扫描批次。
|
||||
* 但仍必须区分"停止自动重试"与"标记已确认"：backfillAt 保持为空。
|
||||
*/
|
||||
@Test
|
||||
fun `missing mailbox row remains an unconfirmed backfill failure`() {
|
||||
fun `missing mailbox row is abandoned instead of retried forever`() {
|
||||
val proc = StubProcState()
|
||||
val inbox = StubInbox().apply { clear() }
|
||||
val id = inbox.insertRaw("<MSG/>")
|
||||
@@ -133,10 +141,12 @@ class BackfillServiceTest {
|
||||
service(proc, inbox).attempt(id)
|
||||
|
||||
val row = proc.find(id)!!
|
||||
assertNull(row.backfillAt)
|
||||
assertEquals(1, row.backfillAttempts)
|
||||
assertEquals("mailbox-row-missing", row.backfillError)
|
||||
assertNotNull(row.backfillNextAt)
|
||||
assertNull(row.backfillAt) // 放弃 ≠ 已确认
|
||||
assertEquals(BackfillService.ABANDON_MISSING_ROW, row.backfillAbandonedReason)
|
||||
assertNotNull(row.backfillAbandonedAt)
|
||||
assertNull(row.backfillNextAt) // 不再排下一次重试
|
||||
assertNull(row.backfillError) // 不再是"失败重试"，而是放弃
|
||||
assertEquals(0, service(proc, inbox).sweep(t0)) // 也不再被扫描
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -229,6 +239,8 @@ class BackfillServiceTest {
|
||||
override fun receivedAtOf(msgId: Long) = t0
|
||||
override fun readRange(fromExclusive: Long, limit: Int) = emptyList<MailboxRow>()
|
||||
override fun maxId(): Long? = 1L
|
||||
override fun minId(): Long? = 1L
|
||||
override fun existingIds(msgIds: Collection<Long>): Set<Long> = emptySet()
|
||||
override fun markProcessedIfUnmarked(msgId: Long, value: String): MailboxMarkResult {
|
||||
entered.countDown()
|
||||
release.await()
|
||||
@@ -262,4 +274,90 @@ class BackfillServiceTest {
|
||||
pool.shutdownNow()
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// 回填闭环（评审修正 R3 + 饥饿回归）
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
@Test
|
||||
fun `missing mailbox row is abandoned immediately and is never treated as marked`() {
|
||||
val proc = StubProcState()
|
||||
val mailbox = FakeMailbox(missing = true)
|
||||
succeeded(proc, 911L)
|
||||
|
||||
service(proc, mailbox).attempt(911L)
|
||||
|
||||
val row = proc.find(911L)!!
|
||||
assertEquals(BackfillService.ABANDON_MISSING_ROW, row.backfillAbandonedReason)
|
||||
assertNotNull(row.backfillAbandonedAt)
|
||||
// 放弃 ≠ 标记已确认：清除前提（边界内全部行已打标）因此仍然不成立。
|
||||
assertNull(row.backfillAt)
|
||||
assertNull(row.backfillNextAt)
|
||||
// 已放弃的行不再进入扫描：不会每 30 秒无限重试。
|
||||
assertEquals(0, service(proc, mailbox).sweep(t0))
|
||||
// 但它**不是**永久失去补偿：人工恢复后可以重新排队。
|
||||
assertTrue(service(proc, mailbox).reopen(911L))
|
||||
assertNull(proc.find(911L)!!.backfillAbandonedAt)
|
||||
assertNotNull(proc.find(911L)!!.backfillNextAt)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `transient failures keep retrying until the attempt cap and stay recoverable`() {
|
||||
props.pipeline.backfillMaxAttempts = 3
|
||||
val proc = StubProcState()
|
||||
val mailbox = FakeMailbox(fail = true)
|
||||
succeeded(proc, 912L)
|
||||
val svc = service(proc, mailbox)
|
||||
|
||||
svc.attempt(912L) // 1 次：暂时性故障，只退避
|
||||
svc.attempt(912L) // 2 次
|
||||
assertNull(proc.find(912L)!!.backfillAbandonedAt)
|
||||
|
||||
svc.attempt(912L) // 3 次：达到上限 → 停止自动重试
|
||||
val row = proc.find(912L)!!
|
||||
assertEquals(BackfillService.ABANDON_MAX_ATTEMPTS, row.backfillAbandonedReason)
|
||||
assertNull(row.backfillAt)
|
||||
|
||||
assertTrue(svc.reopen(912L)) // 人工恢复入口存在且有效
|
||||
assertNull(proc.find(912L)!!.backfillAbandonedAt)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `permanently failing oldest rows do not starve later rows (fair rotation)`() {
|
||||
props.pipeline.backfillBatch = 3
|
||||
val proc = StubProcState()
|
||||
val overdueBefore = t0.plusSeconds(3600)
|
||||
// 1..3：模拟"最旧且已尝试多次"的永久失败行，保持 due（nextAt <= now）
|
||||
(1L..3L).forEach { id ->
|
||||
proc.insertIfAbsent(id, t0)
|
||||
proc.markTerminal(id, ProcStatus.SUCCEEDED, now = t0)
|
||||
proc.recordBackfillFailure(id, "still-failing", attempts = 50, nextAttemptAt = t0, now = t0)
|
||||
}
|
||||
// 200：新行，从未尝试
|
||||
proc.insertIfAbsent(200L, t0)
|
||||
proc.markTerminal(200L, ProcStatus.SUCCEEDED, now = t0)
|
||||
|
||||
val due = proc.findBackfillDue(t0, overdueBefore, limit = 3)
|
||||
|
||||
assertEquals(3, due.size)
|
||||
// 公平轮转：尝试次数少的先被扫描，因此新行不会被最旧的一批永久失败行饿死。
|
||||
assertEquals(200L, due.first().msgId)
|
||||
assertTrue(due.any { it.msgId == 200L })
|
||||
}
|
||||
|
||||
/**
|
||||
* 语义固定（G4）：`RECEIVED_AT` 为 NULL 时"超期"分支不成立，`R` 兜底**不生效**，
|
||||
* 该行只能靠退避重试。把这条钉住，避免以后误以为 `R` 一定能兜底。
|
||||
*/
|
||||
@Test
|
||||
fun `null received time disables the overdue shortcut so only backoff applies`() {
|
||||
val proc = StubProcState()
|
||||
proc.insertIfAbsent(921L, null) // 上游未提供接收时间
|
||||
proc.markTerminal(921L, ProcStatus.SUCCEEDED, now = t0)
|
||||
proc.recordBackfillFailure(921L, "mysql-down", attempts = 1, nextAttemptAt = t0.plusSeconds(3600), now = t0)
|
||||
|
||||
val due = proc.findBackfillDue(now = t0, overdueBefore = t0.plusSeconds(10_000), limit = 10)
|
||||
|
||||
assertTrue(due.isEmpty()) // 超期分支无效 + 退避未到期
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,7 +63,7 @@ class FdelAndAdftProcessorTest {
|
||||
val proc = StubProcState()
|
||||
proc.insertIfAbsent(msgId, null)
|
||||
|
||||
val result = FdelProcessor(StubPipelineTx(), StubPipelineLock(), f, events, proc, ObjectMapper())
|
||||
val result = FdelProcessor(StubPipelineTx(), StubPipelineLock(), f, events, proc, ObjectMapper(), java.time.Clock.systemUTC())
|
||||
.apply(head(), msg(), FlopPayload("121"))
|
||||
|
||||
assertEquals(ApplyResult.Succeeded, result)
|
||||
@@ -83,7 +83,7 @@ class FdelAndAdftProcessorTest {
|
||||
fun `repeated FDEL is idempotent without version bump or duplicate event`() {
|
||||
val f = flights()
|
||||
val events = StubMsgEvents()
|
||||
val proc = FdelProcessor(StubPipelineTx(), StubPipelineLock(), f, events, StubProcState(), ObjectMapper())
|
||||
val proc = FdelProcessor(StubPipelineTx(), StubPipelineLock(), f, events, StubProcState(), ObjectMapper(), java.time.Clock.systemUTC())
|
||||
|
||||
proc.apply(head(), msg(), FlopPayload("121"))
|
||||
val version = f.findMainRow("121")!!.stateVersion
|
||||
@@ -99,7 +99,7 @@ class FdelAndAdftProcessorTest {
|
||||
val f = StubFlightState()
|
||||
val events = StubMsgEvents()
|
||||
|
||||
val result = FdelProcessor(StubPipelineTx(), StubPipelineLock(), f, events, StubProcState(), ObjectMapper())
|
||||
val result = FdelProcessor(StubPipelineTx(), StubPipelineLock(), f, events, StubProcState(), ObjectMapper(), java.time.Clock.systemUTC())
|
||||
.apply(head(), msg(), FlopPayload("999"))
|
||||
|
||||
assertEquals(ApplyResult.Succeeded, result) // 航班不存在（迟到或多余）也算成功
|
||||
@@ -113,7 +113,7 @@ class FdelAndAdftProcessorTest {
|
||||
val events = StubMsgEvents()
|
||||
val adft = AdftProcessor(
|
||||
StubPipelineTx(), StubPipelineLock(), f, events, StubProcState(),
|
||||
OperationDayProps().apply { zone = "Asia/Shanghai" }, ObjectMapper(),
|
||||
OperationDayProps().apply { zone = "Asia/Shanghai" }, ObjectMapper(), java.time.Clock.systemUTC(),
|
||||
)
|
||||
val record = com.gzzn.omms.msgexchange.domain.flight.ScheduleRecord(
|
||||
"121", mapOf("SODT" to "15DEC261723", "FLNO" to "CA002"),
|
||||
@@ -134,7 +134,7 @@ class FdelAndAdftProcessorTest {
|
||||
val events = StubMsgEvents()
|
||||
val adft = AdftProcessor(
|
||||
StubPipelineTx(), StubPipelineLock(), f, events, StubProcState(),
|
||||
OperationDayProps().apply { zone = "Asia/Shanghai" }, ObjectMapper(),
|
||||
OperationDayProps().apply { zone = "Asia/Shanghai" }, ObjectMapper(), java.time.Clock.systemUTC(),
|
||||
)
|
||||
|
||||
val result = adft.apply(
|
||||
@@ -160,7 +160,7 @@ class FdelAndAdftProcessorTest {
|
||||
)
|
||||
val adft = AdftProcessor(
|
||||
StubPipelineTx(), StubPipelineLock(), f, StubMsgEvents(), StubProcState(),
|
||||
OperationDayProps().apply { zone = "Asia/Shanghai" }, ObjectMapper(),
|
||||
OperationDayProps().apply { zone = "Asia/Shanghai" }, ObjectMapper(), java.time.Clock.systemUTC(),
|
||||
)
|
||||
|
||||
adft.apply(head(), msg(), com.gzzn.omms.msgexchange.domain.flight.ScheduleRecord("555", mapOf("FLNO" to "XX200")))
|
||||
|
||||
@@ -83,6 +83,7 @@ class ScheduleProcessorTest {
|
||||
snapshotLog = log,
|
||||
operationDayProps = OperationDayProps().apply { zone = "Asia/Shanghai"; cutoffHour = 0 },
|
||||
mapper = ObjectMapper(),
|
||||
clock = java.time.Clock.systemUTC(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -129,7 +130,7 @@ class ScheduleProcessorTest {
|
||||
fun `replay of succeeded message records idempotent success without writes`() {
|
||||
val proc = StubProcState()
|
||||
proc.insertIfAbsent(msgId, null)
|
||||
proc.markTerminal(msgId, ProcStatus.SUCCEEDED)
|
||||
proc.markTerminal(msgId, ProcStatus.SUCCEEDED, now = java.time.Instant.now())
|
||||
val flights = StubFlightState()
|
||||
val log = StubSnapshotLog()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user