fix(pipeline): 修复取报-处理-回填链路的超时、吞吐、回填与 FIFO 问题
外部调用（ACM2-33） - 共享 MySQL 与自有 PG 全部补有界超时：驱动 connect/socket 超时 + Hikari 池超时 （注意 Micronaut 的 Hikari 项是毫秒数，不是 Duration 字面量） - 删除主泵/毒丸路径的 inline 回填：跨库写不再占用 FIFO 关键路径，回填统一由扫描驱动 - 游标自愈：save() 改为「先 UPDATE、缺行 INSERT」，缺行只记一次 ERROR - Pump/Dispatcher 外层 catch 补 error 日志与失败计数 投递吞吐（ACM2-34） - Dispatcher 改批量领取；队头退避未到期或首条发送失败即停止本轮（保持目标内保序） - 有活不再 sleep；删除不可达的 state==SENT 死条件；markSent 移入分支； superseded 清理加轮数上限与空 eventId 保护 回填闭环（ACM2-36，V4） - MISSING（运行时确认行不存在）立即放弃自动重试并告警；暂时性故障达上限后停止自动重试 - 新增 reopen 人工恢复入口；放弃 ≠ 标记已确认（BACKFILL_AT 仍为空，清除前提不成立） - 扫描改 (BACKFILL_ATTEMPTS, MSG_ID) 公平轮转并排除放弃行，消除全局回填饥饿 - V4 只加字段与必要索引，不按年龄做任何存量推断 切流播种（ACM2-35，V5） - cutover-watermark 四模式（min/zero/max/显式 ID），默认不播种、代码不做默认选择 - 升级实例拒绝重新播种（SEEDED_AT 为 NULL ≠ 从未消费）；播种与水位同语句落库 - 非法取值由启动自检挡下 错误分类与入口契约（ACM2-37） - 未知 SCHD 子类型改为 UNSUPPORTED，不再静默当全量日计划合并 - ADFT 运营日冲突改走 ProtocolViolation → DEAD(PROTOCOL) - 兼容入口 receivedAt 缺失回退到注入 Clock；MessageLifecycleGate 强制注入 + 装配断言 - FIFO：主泵只领取 msgId ≤ W，兼容入口登记的行在水位追平前不被领取 时间源与可观测（ACM2-38 / ACM2-41 阶段 0） - 仓储/处理器/作业全部经注入 Clock；移除 markTerminal/markBackfilled 的 Instant.now() 默认值 - 退避表档位与 max-attempts 对齐并加启动自检 - Micrometer 7 个 gauge（@Context 急切注册）+ /health 与 /metrics 共用积压快照缓存 - 只读迟到检测：监视被放行的空洞 ID 是否后来真的出现，只计数告警、不补入队 Plane: ACM2-33 ACM2-34 ACM2-35 ACM2-36 ACM2-37 ACM2-38 ACM2-41 Tests: 88 → 120（1 skipped 需真实 PG）
This commit is contained in:
windyboy
@@ -0,0 +1,165 @@
|
||||
package com.gzzn.omms.msgexchange.ingress
|
||||
|
||||
import com.gzzn.omms.msgexchange.config.PipelineProps
|
||||
import com.gzzn.omms.msgexchange.infra.metrics.PipelineCounters
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.InboxCursorRepository
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubInbox
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubInboxCursor
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubPipelineTx
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertNotEquals
|
||||
import org.junit.jupiter.api.Assertions.assertNotNull
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.BeforeEach
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.time.Clock
|
||||
import java.time.Instant
|
||||
import java.time.ZoneOffset
|
||||
|
||||
/**
|
||||
* 切流水位播种（ACM2-35）：**显式、一次性、升级安全**。
|
||||
*
|
||||
* 覆盖评审提出的四条硬要求：
|
||||
* 1. 默认（未配置）不动作，代码不做默认选择；
|
||||
* 2. 四种模式严格区分（`min` 读现存全部 / `zero` 从 0 按空洞规则 / `max` 跳过可见存量 / 显式 ID）；
|
||||
* 3. 升级实例（已有水位或已有处理记录）**拒绝重新播种**——`SEEDED_AT` 为 NULL 不等于"从未消费"；
|
||||
* 4. 播种至多一次，且不会被普通的水位推进抹掉。
|
||||
*/
|
||||
class CutoverSeedTest {
|
||||
|
||||
private val t0: Instant = Instant.parse("2026-09-08T03:00:00Z")
|
||||
private val props = PipelineProps()
|
||||
|
||||
private lateinit var inbox: StubInbox
|
||||
private lateinit var proc: StubProcState
|
||||
private lateinit var cursor: StubInboxCursor
|
||||
private lateinit var poller: InboxPoller
|
||||
|
||||
@BeforeEach
|
||||
fun setUp() {
|
||||
inbox = StubInbox().apply { clear() }
|
||||
proc = StubProcState().apply { clear() }
|
||||
cursor = StubInboxCursor().apply { clear() }
|
||||
props.pipeline.cutoverWatermark = null
|
||||
poller = InboxPoller(
|
||||
inbox, proc, cursor, StubPipelineTx(), props,
|
||||
Clock.fixed(t0, ZoneOffset.UTC), PipelineCounters(),
|
||||
)
|
||||
}
|
||||
|
||||
/** 造一个 MIN(ID)=5 的信箱：1..4 已被库方清除（典型"最老分区已 DROP"）。 */
|
||||
private fun mailboxWithMinId5(): List<Long> {
|
||||
val ids = (1..8).map { inbox.insertRaw("<MSG/>") }
|
||||
ids.take(4).forEach { inbox.removeRow(it) }
|
||||
return ids.drop(4)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `default does nothing - no seeding without explicit configuration`() {
|
||||
val kept = mailboxWithMinId5()
|
||||
|
||||
assertEquals(0, poller.pollOnce(t0)) // W=0 → ID=1 判为空洞，不推进
|
||||
|
||||
assertNull(cursor.cursor.seededAt)
|
||||
assertEquals(0L, cursor.cursor.committedUpTo)
|
||||
assertNotNull(cursor.cursor.holeSince)
|
||||
assertTrue(kept.all { proc.find(it) == null })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `min mode reads all currently existing rows`() {
|
||||
val kept = mailboxWithMinId5()
|
||||
props.pipeline.cutoverWatermark = "min"
|
||||
|
||||
assertEquals(4, poller.pollOnce(t0)) // 播种 W=4，同一轮把 5..8 全部读入
|
||||
|
||||
assertNotNull(cursor.cursor.seededAt)
|
||||
assertEquals(8L, cursor.cursor.committedUpTo)
|
||||
assertTrue(kept.all { proc.find(it) != null })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `max mode skips the currently visible backlog`() {
|
||||
mailboxWithMinId5()
|
||||
props.pipeline.cutoverWatermark = "max"
|
||||
|
||||
assertEquals(0, poller.pollOnce(t0))
|
||||
|
||||
assertNotNull(cursor.cursor.seededAt)
|
||||
assertEquals(8L, cursor.cursor.committedUpTo) // 直接跳到 MAX(ID)
|
||||
assertEquals(0, proc.rows.size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `zero mode scans from zero and stops at the first hole`() {
|
||||
mailboxWithMinId5()
|
||||
props.pipeline.cutoverWatermark = "zero"
|
||||
|
||||
assertEquals(0, poller.pollOnce(t0))
|
||||
|
||||
assertNotNull(cursor.cursor.seededAt)
|
||||
assertEquals(0L, cursor.cursor.committedUpTo) // 1..4 是空洞 → 按空洞规则停住
|
||||
assertNotNull(cursor.cursor.holeSince)
|
||||
assertEquals(0, proc.rows.size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `explicit boundary mode is accepted`() {
|
||||
mailboxWithMinId5()
|
||||
props.pipeline.cutoverWatermark = "6"
|
||||
|
||||
poller.pollOnce(t0)
|
||||
|
||||
assertNotNull(cursor.cursor.seededAt)
|
||||
assertEquals(8L, cursor.cursor.committedUpTo) // 从 6 起读 7..8（6 本身已在界内）
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `upgraded instance with existing watermark refuses to re-seed`() {
|
||||
mailboxWithMinId5()
|
||||
cursor.save(InboxCursorRepository.Cursor(committedUpTo = 3L, holeSince = t0))
|
||||
props.pipeline.cutoverWatermark = "max"
|
||||
|
||||
poller.pollOnce(t0)
|
||||
|
||||
assertNull(cursor.cursor.seededAt) // 没有重新播种
|
||||
assertNotEquals(8L, cursor.cursor.committedUpTo)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `instance that already consumed messages refuses to seed even at W zero`() {
|
||||
mailboxWithMinId5()
|
||||
proc.insertIfAbsent(1L, t0) // 已有处理记录（即使水位还是 0）
|
||||
props.pipeline.cutoverWatermark = "max"
|
||||
|
||||
poller.pollOnce(t0)
|
||||
|
||||
assertNull(cursor.cursor.seededAt)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `seeding happens at most once and survives normal watermark saves`() {
|
||||
mailboxWithMinId5()
|
||||
props.pipeline.cutoverWatermark = "max"
|
||||
poller.pollOnce(t0)
|
||||
val seededAt = cursor.cursor.seededAt
|
||||
assertNotNull(seededAt)
|
||||
|
||||
cursor.save(InboxCursorRepository.Cursor(committedUpTo = 2L, holeSince = null))
|
||||
poller.pollOnce(t0.plusSeconds(1))
|
||||
|
||||
assertEquals(seededAt, cursor.cursor.seededAt) // 标记仍在：普通 save 不抹播种事实
|
||||
assertEquals(2L, cursor.cursor.committedUpTo)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `empty mailbox is left unseeded`() {
|
||||
props.pipeline.cutoverWatermark = "max"
|
||||
|
||||
assertEquals(0, poller.pollOnce(t0))
|
||||
|
||||
assertNull(cursor.cursor.seededAt)
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertFalse
|
||||
import org.junit.jupiter.api.Assertions.assertNotNull
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.BeforeEach
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.time.Instant
|
||||
@@ -38,7 +39,11 @@ class InboxPollerTest {
|
||||
inbox = StubInbox().apply { clear() }
|
||||
proc = StubProcState().apply { clear() }
|
||||
cursor = StubInboxCursor().apply { clear() }
|
||||
poller = InboxPoller(inbox, proc, cursor, StubPipelineTx(), props)
|
||||
poller = InboxPoller(
|
||||
inbox, proc, cursor, StubPipelineTx(), props,
|
||||
java.time.Clock.fixed(t0, java.time.ZoneOffset.UTC),
|
||||
com.gzzn.omms.msgexchange.infra.metrics.PipelineCounters(),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -67,7 +72,7 @@ class InboxPollerTest {
|
||||
val dead = (1..3).map { inbox.simulateExternalWrite("<MSG/>") }
|
||||
assertEquals(3, poller.pollOnce(t0))
|
||||
dead.forEach {
|
||||
proc.markTerminal(it, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, lastError = "raw-missing")
|
||||
proc.markTerminal(it, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, lastError = "raw-missing", now = t0)
|
||||
}
|
||||
|
||||
val fresh = inbox.simulateExternalWrite("<MSG/>")
|
||||
@@ -112,7 +117,7 @@ class InboxPollerTest {
|
||||
|
||||
@Test
|
||||
fun `compat http path and poller do not double enqueue the same message`() {
|
||||
val receipt = InboxService(inbox, proc).accept("<MSG/>")
|
||||
val receipt = InboxService(inbox, proc, java.time.Clock.fixed(t0, java.time.ZoneOffset.UTC)).accept("<MSG/>")
|
||||
|
||||
assertEquals(0, poller.pollOnce(t0))
|
||||
assertEquals(receipt.msgId, cursor.cursor.committedUpTo) // 已在 PG：读取进度照常推进
|
||||
@@ -144,4 +149,60 @@ class InboxPollerTest {
|
||||
assertNull(proc.find(fifth))
|
||||
assertEquals(first + 2, third)
|
||||
}
|
||||
|
||||
/**
|
||||
* 【缺口基线 · G1】快路径只读 `ID > W`、从不回头：水位越过某个 ID 之后，该 ID 即使后来
|
||||
* 出现在信箱里也不会再被发现。这不是期望行为，而是"没有补偿扫描"的**已知缺口**；
|
||||
* 本用例把它固定成基线，补偿扫描（ACM2-41 / G1）落地后必须反转成"能被发现并安全处置"。
|
||||
*/
|
||||
@Test
|
||||
fun `baseline - an id that appears after the watermark passed it is never discovered`() {
|
||||
inbox.simulateExternalWrite("<MSG/>") // 1
|
||||
val hole = inbox.simulateExternalWrite("<MSG/>") // 2
|
||||
val third = inbox.simulateExternalWrite("<MSG/>") // 3
|
||||
inbox.removeRow(hole)
|
||||
|
||||
poller.pollOnce(t0) // W 停在 1，空洞在 2
|
||||
val agedOut = t0.plus(props.pipeline.maxCommitDelay)
|
||||
poller.pollOnce(agedOut) // 空洞判永久 → 放行
|
||||
poller.pollOnce(agedOut) // 发现 3
|
||||
assertEquals(third, cursor.cursor.committedUpTo)
|
||||
|
||||
// 迟到的 2 现在才出现：水位已经越过它，快路径再也不会读它。
|
||||
inbox.restoreRow(hole, "<MSG/>", agedOut)
|
||||
|
||||
assertEquals(0, poller.pollOnce(agedOut.plusSeconds(1)))
|
||||
assertNull(proc.find(hole))
|
||||
}
|
||||
|
||||
/**
|
||||
* 兼容入口会把消息登记在水位**之外**（`msgId > W`），因此它天然成为"最小未完成行"。
|
||||
* 领取侧的守卫在主泵：只领 `msgId <= W`（端到端验收见 `PipelineSmokeTest` 的
|
||||
* "compat injected high id is not claimed until the watermark catches up"）。
|
||||
* 本用例只固定收报侧的事实：**登记发生，但水位不动**。
|
||||
*/
|
||||
@Test
|
||||
fun `compat accept registers a row above the watermark without advancing it`() {
|
||||
props.pipeline.claimBatch = 10
|
||||
val one = inbox.simulateExternalWrite("<MSG/>") // 1
|
||||
val hole = inbox.simulateExternalWrite("<MSG/>") // 2（空洞）
|
||||
val third = inbox.simulateExternalWrite("<MSG/>") // 3
|
||||
inbox.removeRow(hole)
|
||||
|
||||
assertEquals(1, poller.pollOnce(t0)) // W=1，遇空洞即停
|
||||
assertEquals(one, cursor.cursor.committedUpTo)
|
||||
assertNull(proc.find(third)) // 3 还没入队
|
||||
|
||||
val receipt = InboxService(inbox, proc, java.time.Clock.fixed(t0, java.time.ZoneOffset.UTC))
|
||||
.accept("<MSG/>") // 兼容入口：高 ID 直接进 PG
|
||||
val high = receipt.msgId
|
||||
assertTrue(high > third)
|
||||
|
||||
assertEquals(one, cursor.cursor.committedUpTo) // 水位不动（不参与水位）
|
||||
// 主泵处理掉队头 1 之后，最小未完成行就是兼容入口写进来的高 ID……
|
||||
proc.markTerminal(one, ProcStatus.SUCCEEDED, now = t0)
|
||||
assertEquals(high, proc.headUnfinished()!!.msgId)
|
||||
assertTrue(high > cursor.cursor.committedUpTo) // ……但它超出水位，主泵不会领取
|
||||
assertNull(proc.find(third))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
package com.gzzn.omms.msgexchange.ingress
|
||||
|
||||
import com.gzzn.omms.msgexchange.config.PipelineProps
|
||||
import com.gzzn.omms.msgexchange.infra.metrics.PipelineCounters
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubInbox
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubInboxCursor
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubPipelineTx
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertNotNull
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.BeforeEach
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.time.Clock
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
import java.time.ZoneOffset
|
||||
|
||||
/**
|
||||
* 迟到到达检测（ACM2-41 阶段 0）：**只读观测**。
|
||||
*
|
||||
* 语义：被判定为永久空洞并放行的 ID，如果后来真的出现在信箱里，就是"上游提交晚于水位推进"。
|
||||
* 阶段 0 只计数与告警，**不入队、不改变处理语义**（补入队属阶段 1，需先与库方定案）。
|
||||
*/
|
||||
class LateArrivalDetectTest {
|
||||
|
||||
private val t0: Instant = Instant.parse("2026-09-08T03:00:00Z")
|
||||
private val props = PipelineProps()
|
||||
|
||||
private lateinit var inbox: StubInbox
|
||||
private lateinit var proc: StubProcState
|
||||
private lateinit var cursor: StubInboxCursor
|
||||
private lateinit var counters: PipelineCounters
|
||||
private lateinit var poller: InboxPoller
|
||||
|
||||
@BeforeEach
|
||||
fun setUp() {
|
||||
inbox = StubInbox().apply { clear() }
|
||||
proc = StubProcState().apply { clear() }
|
||||
cursor = StubInboxCursor().apply { clear() }
|
||||
counters = PipelineCounters()
|
||||
poller = InboxPoller(inbox, proc, cursor, StubPipelineTx(), props, Clock.fixed(t0, ZoneOffset.UTC), counters)
|
||||
}
|
||||
|
||||
/** 造出"1 存在、2 是空洞、3 存在"，并把空洞等到超期放行。返回迟到的那个 ID。 */
|
||||
private fun ageOutHoleAt2(): Long {
|
||||
inbox.simulateExternalWrite("<MSG/>") // 1
|
||||
val hole = inbox.simulateExternalWrite("<MSG/>") // 2
|
||||
inbox.simulateExternalWrite("<MSG/>") // 3
|
||||
inbox.removeRow(hole)
|
||||
poller.pollOnce(t0) // W=1，空洞在 2
|
||||
poller.pollOnce(t0.plus(props.pipeline.maxCommitDelay)) // 空洞判永久 → 放行
|
||||
return hole
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a hole that really appears later is detected and counted`() {
|
||||
val hole = ageOutHoleAt2()
|
||||
inbox.restoreRow(hole, "<MSG/>", t0.plus(props.pipeline.maxCommitDelay))
|
||||
|
||||
// 过了检测周期再轮询一次
|
||||
poller.pollOnce(t0.plus(props.pipeline.maxCommitDelay).plus(props.pipeline.lateDetectPeriod))
|
||||
|
||||
assertEquals(1L, counters.lateArrivalDetectedCount())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `detection does not enqueue the late message - phase 0 is observation only`() {
|
||||
val hole = ageOutHoleAt2()
|
||||
inbox.restoreRow(hole, "<MSG/>", t0.plus(props.pipeline.maxCommitDelay))
|
||||
|
||||
poller.pollOnce(t0.plus(props.pipeline.maxCommitDelay).plus(props.pipeline.lateDetectPeriod))
|
||||
|
||||
assertEquals(1L, counters.lateArrivalDetectedCount())
|
||||
assertNull(proc.find(hole)) // 仍然不会被补入队
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a hole that stays absent is not counted`() {
|
||||
ageOutHoleAt2()
|
||||
|
||||
poller.pollOnce(t0.plus(props.pipeline.maxCommitDelay).plus(props.pipeline.lateDetectPeriod))
|
||||
|
||||
assertEquals(0L, counters.lateArrivalDetectedCount())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `detection does not run before the configured period elapses`() {
|
||||
val hole = ageOutHoleAt2()
|
||||
inbox.restoreRow(hole, "<MSG/>", t0)
|
||||
|
||||
// 只过了一个 max-commit-delay，未到检测周期
|
||||
poller.pollOnce(t0.plus(props.pipeline.maxCommitDelay).plusSeconds(1))
|
||||
|
||||
assertEquals(0L, counters.lateArrivalDetectedCount())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `detection can be switched off`() {
|
||||
props.pipeline.lateDetectPeriod = Duration.ZERO
|
||||
val hole = ageOutHoleAt2()
|
||||
inbox.restoreRow(hole, "<MSG/>", t0)
|
||||
|
||||
poller.pollOnce(t0.plus(props.pipeline.maxCommitDelay).plus(Duration.ofHours(1)))
|
||||
|
||||
assertEquals(0L, counters.lateArrivalDetectedCount())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `the same late id is counted once even after the hole is aged out again`() {
|
||||
val hole = ageOutHoleAt2()
|
||||
inbox.restoreRow(hole, "<MSG/>", t0.plus(props.pipeline.maxCommitDelay))
|
||||
val later = t0.plus(props.pipeline.maxCommitDelay).plus(props.pipeline.lateDetectPeriod)
|
||||
poller.pollOnce(later)
|
||||
assertEquals(1L, counters.lateArrivalDetectedCount())
|
||||
|
||||
poller.pollOnce(later.plus(props.pipeline.lateDetectPeriod))
|
||||
|
||||
assertEquals(1L, counters.lateArrivalDetectedCount()) // 已命中的 ID 不再重复计数
|
||||
assertNotNull(cursor.cursor)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user