feat(ingress): 按处理时间为空扫描信箱,移除水位消费(ACM2-78)

readUnprocessed 替代 ID 水位;去掉 INBOX_CURSOR/cutover-watermark/max-commit-delay 及相关健康与指标。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
windyboy
2026-09-21 14:49:40 +08:00
co-authored by Cursor
parent 23b74ed554
commit 891088235c
29 changed files with 166 additions and 805 deletions
@@ -52,7 +52,6 @@ class PipelineSmokeTest {
fun cleanStubs() {
ctx.getBean(StubProcState::class.java).clear()
ctx.getBean(com.gzzn.omms.msgexchange.infra.stub.StubInbox::class.java).clear()
ctx.getBean(com.gzzn.omms.msgexchange.infra.stub.StubInboxCursor::class.java).clear()
ctx.getBean(StubMsgEvents::class.java).clear()
ctx.getBean(StubDeliveryPort::class.java).clear()
}
@@ -98,8 +97,6 @@ class PipelineSmokeTest {
val receipt = controller.send(UNSUPPORTED_XML)
assertNotNull(receipt.body()) // 受理 ID
val id = receipt.body()!!.toLong()
// 兼容入口只保证"已落信 + 已入队",**不推进水位**;必须先被收报发现(W 追平)才可领取。
ctx.getBean(com.gzzn.omms.msgexchange.ingress.InboxPoller::class.java).pollOnce(Instant.now())
pump.tick() // 解码成功但无 XYZQ Handler → SKIPPED(unsupported)
@@ -115,7 +112,6 @@ class PipelineSmokeTest {
fun `replay reopens failed UNSUPPORTED row to PENDING`() {
val receipt = controller.send(REFDATA_XML)
val id = receipt.body()!!.toLong()
ctx.getBean(com.gzzn.omms.msgexchange.ingress.InboxPoller::class.java).pollOnce(Instant.now())
pump.tick()
val stub = ctx.getBean(StubProcState::class.java)
assertEquals(ProcStatus.FAILED, stub.snapshotOf(id)!!.state)
@@ -193,9 +189,6 @@ class PipelineSmokeTest {
val proc = ctx.getBean(StubProcState::class.java)
val props = ctx.getBean(com.gzzn.omms.msgexchange.config.PipelineProps::class.java)
val backfill = ctx.getBean(com.gzzn.omms.msgexchange.processing.BackfillService::class.java)
// 队头必须在水位以内才可领取:直接播种 PROC_STATE 的用例要显式把水位推上去。
ctx.getBean(com.gzzn.omms.msgexchange.infra.stub.StubInboxCursor::class.java)
.save(com.gzzn.omms.msgexchange.infra.persistence.InboxCursorRepository.Cursor(committedUpTo = 9001L))
proc.insertIfAbsent(9001L, Instant.now())
proc.update(
9001L, ProcStatus.FAILED,
@@ -225,40 +218,32 @@ class PipelineSmokeTest {
}
/**
* FIFO 不变量"只领取已发现的行"端到端验收:兼容入口写入的高 ID **不会被提前领取**,
* 必须等水位追平(较小 ID 补齐并入队)之后才按顺序处理
* 扫描谓词端到端验收(`INV-1`):处理标记为空的行都会被发现并登记,处理完并回填之后
* 退出扫描谓词;兼容入口已登记的行不会被收报建成第二条记录
*/
@Test
fun `compat injected high id is not claimed until the watermark catches up`() {
fun `scan predicate discovers every unmarked row exactly once`() {
val inbox = ctx.getBean(com.gzzn.omms.msgexchange.infra.stub.StubInbox::class.java)
val proc = ctx.getBean(StubProcState::class.java)
val cursor = ctx.getBean(com.gzzn.omms.msgexchange.infra.stub.StubInboxCursor::class.java)
val poller = ctx.getBean(com.gzzn.omms.msgexchange.ingress.InboxPoller::class.java)
// 信箱:1 存在、2 是空洞(被删)、3 存在 → 水位停在 1
inbox.simulateExternalWrite("<MSG/>")
val hole = inbox.simulateExternalWrite("<MSG/>")
inbox.simulateExternalWrite("<MSG/>")
inbox.removeRow(hole)
val external = inbox.simulateExternalWrite("<MSG/>") // 上游直接写信箱
val compat = controller.send(UNSUPPORTED_XML).body()!!.toLong() // 兼容入口:已自行入队
poller.pollOnce(Instant.now())
assertEquals(1L, cursor.cursor.committedUpTo)
assertEquals(setOf(external, compat), proc.rows.keys.toSet())
// 兼容入口写入高 ID:直接进 PG,但水位没追平(这正是原来的越序路径)
val high = controller.send(UNSUPPORTED_XML).body()!!.toLong()
assertTrue(high > 3L)
poller.pollOnce(Instant.now()) // 重扫不建第二条
assertEquals(2, proc.rows.size)
pump.tick() // 先按 FIFO 处理 ID=1
assertEquals(ProcStatus.DEAD, proc.find(1L)!!.state)
pump.tick() // 队头变成高 ID,但它在水位之外 → 不领取
assertEquals(ProcStatus.PENDING, proc.find(high)!!.state)
repeat(2) { pump.tick() }
ctx.getBean(com.gzzn.omms.msgexchange.processing.BackfillService::class.java).sweep()
assertTrue(inbox.isMarked(external) && inbox.isMarked(compat))
// 空洞补齐 → 水位追平 → 才允许继续按顺序处理
inbox.restoreRow(hole, "<MSG/>", Instant.now())
// 已回填的行退出扫描谓词:清掉处理记录后也不会被重新登记
proc.clear()
poller.pollOnce(Instant.now())
assertTrue(cursor.cursor.committedUpTo >= high)
repeat(3) { pump.tick() } // 依次处理 2、3、高 ID
assertEquals(ProcStatus.SKIPPED, proc.find(high)!!.state)
assertEquals("unsupported:XYZQ-FOO", proc.find(high)!!.lastError)
assertTrue(proc.rows.isEmpty())
}
@Test
@@ -23,13 +23,4 @@ class PipelineConfigCheckTest {
assertThrows(IllegalArgumentException::class.java) { PipelineConfigCheck(props) }
}
@Test
fun `illegal cutover watermark rejects startup even without autostart`() {
val props = PipelineProps().apply {
pipeline.cutoverWatermark = "yesterday"
}
assertThrows(IllegalArgumentException::class.java) { PipelineConfigCheck(props) }
}
}
@@ -33,19 +33,6 @@ class PipelinePropsTest {
assertThrows(IllegalArgumentException::class.java) { pipeline.validate() }
}
/** 切流播种只接受 min|zero|max|<id>:非法值必须在启动时挡掉。 */
@Test
fun `validate rejects an unknown cutover watermark mode`() {
pipeline.cutoverWatermark = "bogus"
assertThrows(IllegalArgumentException::class.java) { pipeline.validate() }
pipeline.cutoverWatermark = "max"
pipeline.validate()
pipeline.cutoverWatermark = "12345"
pipeline.validate()
}
@Test
fun `non-positive attempt never throws and falls back to first slot`() {
assertEquals(1000, pipeline.backoffFor(0))
@@ -3,9 +3,6 @@ package com.gzzn.omms.msgexchange.infra.health
import com.gzzn.omms.msgexchange.MutableClock
import com.gzzn.omms.msgexchange.delivery.DeliveryPort
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.infra.persistence.InboxCursorRepository
import com.gzzn.omms.msgexchange.infra.stub.StubInbox
import com.gzzn.omms.msgexchange.infra.stub.StubInboxCursor
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
import io.micronaut.health.HealthStatus
import org.junit.jupiter.api.Assertions.assertEquals
@@ -41,18 +38,14 @@ class HealthIndicatorsTest {
/** 检查 /health 里那几个积压指标算得对不对。 */
@Test
fun `inbox lifecycle reports backlog, oldest age, unmarked terminals and watermark lag`() {
fun `inbox lifecycle reports backlog, oldest age and unmarked terminals`() {
val proc = StubProcState()
val inbox = StubInbox().apply { clear() }
val cursor = StubInboxCursor()
val oldest = MutableClock.BASE.minusSeconds(3600)
proc.insertIfAbsent(1L, oldest)
proc.insertIfAbsent(2L, MutableClock.BASE)
proc.markTerminal(2L, ProcStatus.SUCCEEDED, now = MutableClock.BASE)
val newest = inbox.insertRaw("<MSG/>")
cursor.save(InboxCursorRepository.Cursor(committedUpTo = newest - 2))
val result = lifecycleHealth(proc, cursor, inbox, now = MutableClock.BASE)
val result = lifecycleHealth(proc, now = MutableClock.BASE)
assertEquals(HealthStatus.UP, result.status)
@Suppress("UNCHECKED_CAST")
@@ -60,12 +53,11 @@ class HealthIndicatorsTest {
assertEquals(1, details["backlog"])
assertEquals(3600L, details["oldestUnprocessedSeconds"])
assertEquals(1, details["unmarkedTerminal"])
assertEquals(2L, details["watermarkLag"])
}
@Test
fun `inbox lifecycle stays up when ports are not bound`() {
val result = lifecycleHealth(procState = null, cursor = null, mailbox = null, now = MutableClock.BASE)
val result = lifecycleHealth(procState = null, now = MutableClock.BASE)
assertEquals(HealthStatus.UP, result.status) // 可用性由依赖自身指示器承担
}
@@ -48,8 +48,6 @@ class PipelineMetricsTest {
assertEquals(1.0, gauge("msgx.pipeline.backlog.unfinished"), 0.001)
assertEquals(1.0, gauge("msgx.pipeline.backfill.unmarked_terminal"), 0.001)
assertEquals(0.0, gauge("msgx.pipeline.backfill.abandoned"), 0.001)
// 信箱为空 → 没有"最新 ID"可比,滞后用 -1(无值)而不是伪造 0
assertEquals(-1.0, gauge("msgx.pipeline.watermark.lag"), 0.001)
}
@Test
@@ -11,8 +11,8 @@ import java.sql.DriverManager
/**
* 在真实 PostgreSQL 上跑一遍迁移链,确认结果符合预期:`V1__flight_state_baseline.sql`
* 基线加 `V2__flight_chute_class_type_rename.sql` 更名依序执行成功,该建的表和单行种子
* PIPELINE_LOCK、INBOX_CURSOR)都在,回填事实与收报水位都落在基线里,`BACKFILL_TODO`、
* 基线加 `V2__flight_chute_class_type_rename.sql` 更名依序执行成功,该建的表和 PIPELINE_LOCK
* 单行种子都在,回填事实落在基线里,`INBOX_CURSOR`、`BACKFILL_TODO`、
* `idx_evt_flid`、`PROC_STATE` 的处理开始时间列都不复存在;FLIGHT_CHUTE 的类字段列
* 已由 V2 更名为 CCLS/CTYPSIS 口径)。
*
@@ -24,7 +24,7 @@ class FlywayMigrationTest {
"pipeline_lock", "proc_state", "flight_schd",
"flight_gate", "flight_checkin", "flight_belt", "flight_stand_plan", "flight_chute",
"flight_delay", "flight_bridge_op", "flight_chock_op", "flight_route_point",
"msg_event", "req_track", "inbox_cursor", "schd_snap_log",
"msg_event", "req_track", "schd_snap_log",
)
@Test
@@ -78,6 +78,7 @@ class FlywayMigrationTest {
val missing = expectedTables - tables
assertTrue(missing.isEmpty(), "missing tables: $missing")
assertTrue("backfill_todo" !in tables, "BACKFILL_TODO 已由 PROC_STATE 回填列取代")
assertTrue("inbox_cursor" !in tables, "扫描谓词按处理时间取行,不再有消费水位表")
}
// 身份不变量基础结构:FLIGHT_SCHD 主键 + STATE 列
@@ -200,17 +201,11 @@ class FlywayMigrationTest {
assertEquals("NO", rs.getString("is_nullable"), "ENQUEUED_AT 必须非空")
}
// 两张单行表(管道锁、收报水位)的种子数据都要在
// 管道锁的单行种子要在
stmt.executeQuery("SELECT count(*) FROM pipeline_lock WHERE lock_id = 1").use { rs ->
assertTrue(rs.next())
assertEquals(1, rs.getInt(1))
}
stmt.executeQuery("SELECT committed_up_to, hole_since FROM inbox_cursor WHERE cursor_id = 1").use { rs ->
assertTrue(rs.next(), "INBOX_CURSOR 单行种子必须就位")
assertEquals(0L, rs.getLong("committed_up_to"))
rs.getTimestamp("hole_since")
assertTrue(rs.wasNull(), "初始无空洞观测")
}
}
}
}
@@ -2,7 +2,6 @@ package com.gzzn.omms.msgexchange.infra.persistence.jdbc
import com.gzzn.omms.msgexchange.domain.ErrorClass
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.infra.persistence.InboxCursorRepository
import com.gzzn.omms.msgexchange.infra.persistence.MailboxMarkResult
import org.h2.jdbcx.JdbcDataSource
import org.junit.jupiter.api.Assertions.assertEquals
@@ -24,7 +23,7 @@ import javax.sql.DataSource
*
* 库用 H2 的 PostgreSQL 兼容模式,表结构照抄 V1 + V2,因此不需要 docker 或外接数据库
* 就能跑。覆盖两边的真实语句:自有 PG 侧(写终态时一并写下回填待办、挑选待回填记录、
* 积压统计、水位游标读写)和共享信箱侧(按 ID 区间读、标记只写一次)。
* 积压统计)和共享信箱侧(按处理时间为空扫描、标记只写一次)。
*
* 有一处覆盖不到:H2 不支持 `INSERT ... ON CONFLICT DO NOTHING`,所以入队幂等没在这里验证,
* 由 `InboxPollerTest`(重复轮询不再登记)和 PostgreSQL 本身的语义来保证。
@@ -34,7 +33,6 @@ class InboxLifecycleJdbcSqlTest {
private lateinit var ds: DataSource
private lateinit var proc: JdbcProcStateRepository
private lateinit var mailbox: JdbcCminmsgInboxRepository
private lateinit var cursor: JdbcInboxCursorRepository
private val t0: Instant = Instant.parse("2026-09-08T03:00:00Z")
private val overdue: Instant = t0.minus(Duration.ofDays(31))
@@ -49,18 +47,12 @@ class InboxLifecycleJdbcSqlTest {
ds.connection.use { conn ->
conn.createStatement().use { st ->
st.execute(PROC_STATE_DDL)
st.execute(CURSOR_DDL)
st.execute(MAILBOX_DDL)
st.execute(EVENT_DDL)
st.execute(
"INSERT INTO inbox_cursor (cursor_id, committed_up_to, hole_since, updated_at) " +
"VALUES (1, 0, NULL, CURRENT_TIMESTAMP)",
)
}
}
proc = JdbcProcStateRepository(ds, java.time.Clock.systemUTC())
mailbox = JdbcCminmsgInboxRepository(ds)
cursor = JdbcInboxCursorRepository(ds, java.time.Clock.systemUTC())
}
@Test
@@ -133,15 +125,6 @@ class InboxLifecycleJdbcSqlTest {
assertEquals(1, backlog.unmarkedTerminal)
}
@Test
fun `watermark round trips with and without a pending hole`() {
assertEquals(InboxCursorRepository.Cursor(0L, null), cursor.load())
cursor.save(InboxCursorRepository.Cursor(42L, t0))
assertEquals(InboxCursorRepository.Cursor(42L, t0), cursor.load())
}
@Test
fun `replay resets attempts, backoff and the previous error reason`() {
seed(12L, t0)
@@ -185,22 +168,22 @@ class InboxLifecycleJdbcSqlTest {
}
@Test
fun `mailbox range read ignores processing marks and marking never overwrites`() {
fun `mailbox scan takes unprocessed rows in id order and marking never overwrites`() {
val first = mailbox.insertRaw("<A/>")
val second = mailbox.insertRaw("<B/>")
val third = mailbox.insertRaw("<C/>")
// 发现按 ID 区间;处理标记不参与扫描谓词
assertEquals(listOf(first, second, third), mailbox.readRange(0L, 50).map { it.msgId })
assertNotNull(mailbox.readRange(0L, 50).first().receivedAt)
assertEquals(listOf(second, third), mailbox.readRange(first, 50).map { it.msgId })
assertEquals(third, mailbox.maxId())
// 扫描谓词只看处理时间为空;afterId 是同一轮的续读起点
assertEquals(listOf(first, second, third), mailbox.readUnprocessed(0L, 50).map { it.msgId })
assertNotNull(mailbox.readUnprocessed(0L, 50).first().receivedAt)
assertEquals(listOf(second, third), mailbox.readUnprocessed(first, 50).map { it.msgId })
assertEquals(listOf(first, second), mailbox.readUnprocessed(0L, 2).map { it.msgId })
assertEquals(MailboxMarkResult.MARKED, mailbox.markProcessedIfUnmarked(second, "PROCESSED"))
assertEquals(MailboxMarkResult.ALREADY_MARKED, mailbox.markProcessedIfUnmarked(second, "OTHER"))
assertEquals("PROCESSED", statusOf(second))
// 已标记行仍出现在区间读结果中(发现与标记彻底解耦)
assertEquals(listOf(first, second, third), mailbox.readRange(0L, 50).map { it.msgId })
// 打过标记的行退出扫描谓词
assertEquals(listOf(first, third), mailbox.readUnprocessed(0L, 50).map { it.msgId })
assertEquals(MailboxMarkResult.MISSING, mailbox.markProcessedIfUnmarked(999L, "PROCESSED"))
}
@@ -252,16 +235,6 @@ class InboxLifecycleJdbcSqlTest {
)
"""
const val CURSOR_DDL = """
CREATE TABLE inbox_cursor (
cursor_id INT PRIMARY KEY,
committed_up_to BIGINT NOT NULL,
hole_since TIMESTAMP WITH TIME ZONE,
seeded_at TIMESTAMP WITH TIME ZONE,
updated_at TIMESTAMP WITH TIME ZONE NOT NULL
)
"""
const val MAILBOX_DDL = """
CREATE TABLE cminmsgs (
CMINMSGS_ID BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
@@ -1,164 +0,0 @@
package com.gzzn.omms.msgexchange.ingress
import com.gzzn.omms.msgexchange.config.PipelineProps
import com.gzzn.omms.msgexchange.infra.persistence.InboxCursorRepository
import com.gzzn.omms.msgexchange.infra.stub.StubInbox
import com.gzzn.omms.msgexchange.infra.stub.StubInboxCursor
import com.gzzn.omms.msgexchange.infra.stub.StubPipelineTx
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertNotEquals
import org.junit.jupiter.api.Assertions.assertNotNull
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import java.time.Clock
import java.time.Instant
import java.time.ZoneOffset
/**
* 切流水位播种(ACM2-35):**显式、一次性、升级安全**。
*
* 覆盖评审提出的四条硬要求:
* 1. 默认(未配置)不动作,代码不做默认选择;
* 2. 四种模式严格区分(`min` 读现存全部 / `zero` 从 0 按空洞规则 / `max` 跳过可见存量 / 显式 ID);
* 3. 升级实例(已有水位或已有处理记录)**拒绝重新播种**——`SEEDED_AT` 为 NULL 不等于"从未消费"
* 4. 播种至多一次,且不会被普通的水位推进抹掉。
*/
class CutoverSeedTest {
private val t0: Instant = Instant.parse("2026-09-08T03:00:00Z")
private val props = PipelineProps()
private lateinit var inbox: StubInbox
private lateinit var proc: StubProcState
private lateinit var cursor: StubInboxCursor
private lateinit var poller: InboxPoller
@BeforeEach
fun setUp() {
inbox = StubInbox().apply { clear() }
proc = StubProcState().apply { clear() }
cursor = StubInboxCursor().apply { clear() }
props.pipeline.cutoverWatermark = null
poller = InboxPoller(
inbox, proc, cursor, StubPipelineTx(), props,
Clock.fixed(t0, ZoneOffset.UTC),
)
}
/** 造一个 MIN(ID)=5 的信箱:1..4 已被库方清除(典型"最老分区已 DROP")。 */
private fun mailboxWithMinId5(): List<Long> {
val ids = (1..8).map { inbox.insertRaw("<MSG/>") }
ids.take(4).forEach { inbox.removeRow(it) }
return ids.drop(4)
}
@Test
fun `default does nothing - no seeding without explicit configuration`() {
val kept = mailboxWithMinId5()
assertEquals(0, poller.pollOnce(t0)) // W=0 → ID=1 判为空洞,不推进
assertNull(cursor.cursor.seededAt)
assertEquals(0L, cursor.cursor.committedUpTo)
assertNotNull(cursor.cursor.holeSince)
assertTrue(kept.all { proc.find(it) == null })
}
@Test
fun `min mode reads all currently existing rows`() {
val kept = mailboxWithMinId5()
props.pipeline.cutoverWatermark = "min"
assertEquals(4, poller.pollOnce(t0)) // 播种 W=4,同一轮把 5..8 全部读入
assertNotNull(cursor.cursor.seededAt)
assertEquals(8L, cursor.cursor.committedUpTo)
assertTrue(kept.all { proc.find(it) != null })
}
@Test
fun `max mode skips the currently visible backlog`() {
mailboxWithMinId5()
props.pipeline.cutoverWatermark = "max"
assertEquals(0, poller.pollOnce(t0))
assertNotNull(cursor.cursor.seededAt)
assertEquals(8L, cursor.cursor.committedUpTo) // 直接跳到 MAX(ID)
assertEquals(0, proc.rows.size)
}
@Test
fun `zero mode scans from zero and stops at the first hole`() {
mailboxWithMinId5()
props.pipeline.cutoverWatermark = "zero"
assertEquals(0, poller.pollOnce(t0))
assertNotNull(cursor.cursor.seededAt)
assertEquals(0L, cursor.cursor.committedUpTo) // 1..4 是空洞 → 按空洞规则停住
assertNotNull(cursor.cursor.holeSince)
assertEquals(0, proc.rows.size)
}
@Test
fun `explicit boundary mode is accepted`() {
mailboxWithMinId5()
props.pipeline.cutoverWatermark = "6"
poller.pollOnce(t0)
assertNotNull(cursor.cursor.seededAt)
assertEquals(8L, cursor.cursor.committedUpTo) // 从 6 起读 7..8(6 本身已在界内)
}
@Test
fun `upgraded instance with existing watermark refuses to re-seed`() {
mailboxWithMinId5()
cursor.save(InboxCursorRepository.Cursor(committedUpTo = 3L, holeSince = t0))
props.pipeline.cutoverWatermark = "max"
poller.pollOnce(t0)
assertNull(cursor.cursor.seededAt) // 没有重新播种
assertNotEquals(8L, cursor.cursor.committedUpTo)
}
@Test
fun `instance that already consumed messages refuses to seed even at W zero`() {
mailboxWithMinId5()
proc.insertIfAbsent(1L, t0) // 已有处理记录(即使水位还是 0)
props.pipeline.cutoverWatermark = "max"
poller.pollOnce(t0)
assertNull(cursor.cursor.seededAt)
}
@Test
fun `seeding happens at most once and survives normal watermark saves`() {
mailboxWithMinId5()
props.pipeline.cutoverWatermark = "max"
poller.pollOnce(t0)
val seededAt = cursor.cursor.seededAt
assertNotNull(seededAt)
cursor.save(InboxCursorRepository.Cursor(committedUpTo = 2L, holeSince = null))
poller.pollOnce(t0.plusSeconds(1))
assertEquals(seededAt, cursor.cursor.seededAt) // 标记仍在:普通 save 不抹播种事实
assertEquals(2L, cursor.cursor.committedUpTo)
}
@Test
fun `empty mailbox is left unseeded`() {
props.pipeline.cutoverWatermark = "max"
assertEquals(0, poller.pollOnce(t0))
assertNull(cursor.cursor.seededAt)
}
}
@@ -4,49 +4,46 @@ import com.gzzn.omms.msgexchange.config.PipelineProps
import com.gzzn.omms.msgexchange.domain.ErrorClass
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.infra.stub.StubInbox
import com.gzzn.omms.msgexchange.infra.stub.StubInboxCursor
import com.gzzn.omms.msgexchange.infra.stub.StubPipelineTx
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertNotNull
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import java.time.Clock
import java.time.Instant
import java.time.ZoneOffset
/**
* 收报环节最要紧的几条规矩:
* - 取新消息只看 ID,不看处理标记。处理完却没能回填的行(尤其是永远不回填的死信)
* 不允许占住批次,也不允许挡住后面的新消息——这是曾经的线上隐患
* - 水位只在成功登记后才推进,而且和登记写在同一个事务里,中断后重扫就能补齐
* - 遇到 ID 缺口先停下来(可能有更小的消息还没到),缺口等太久则跳过(否则水位永远卡住);
* 收报环节最要紧的几条规矩`INV-1`
* - 取新消息只看处理时间是否为空,按编号升序、每批 `claim-batch` 条;
* - 同一编号只建一条处理记录:重扫、重启、与兼容入口并发都不会多建
* - 已回填的行不再被扫到;终态但未回填的行会重复扫到,但不许占住批次挡住后面的新行
* - 这一层不碰信箱的处理标记,标记留给回填环节写。
*/
class InboxPollerTest {
private val t0: Instant = Instant.parse("2026-09-08T03:00:00Z")
private val props = PipelineProps()
private val clock: Clock = Clock.fixed(t0, ZoneOffset.UTC)
private lateinit var inbox: StubInbox
private lateinit var proc: StubProcState
private lateinit var cursor: StubInboxCursor
private lateinit var poller: InboxPoller
@BeforeEach
fun setUp() {
inbox = StubInbox().apply { clear() }
proc = StubProcState().apply { clear() }
cursor = StubInboxCursor().apply { clear() }
poller = InboxPoller(
inbox, proc, cursor, StubPipelineTx(), props,
java.time.Clock.fixed(t0, java.time.ZoneOffset.UTC),
)
poller = newPoller()
}
private fun newPoller() = InboxPoller(inbox, proc, StubPipelineTx(), props, clock)
@Test
fun `external rows are enqueued in id order and advance the watermark without marking the mailbox`() {
fun `unprocessed rows are enqueued in id order without marking the mailbox`() {
val first = inbox.simulateExternalWrite("<MSG/>")
val second = inbox.simulateExternalWrite("<MSG/>")
@@ -54,16 +51,29 @@ class InboxPollerTest {
assertEquals(ProcStatus.PENDING, proc.find(first)!!.state)
assertEquals(ProcStatus.PENDING, proc.find(second)!!.state)
assertEquals(second, cursor.cursor.committedUpTo)
assertNull(cursor.cursor.holeSince)
assertEquals(listOf(first, second), proc.rows.keys.sorted())
// 收报只写自有库,不碰信箱的处理标记
assertFalse(inbox.isMarked(first))
assertEquals(0, poller.pollOnce(t0)) // 重复扫描幂等
assertEquals(0, poller.pollOnce(t0)) // 重扫幂等:同编号不再建第二行
assertEquals(2, proc.rows.size)
}
/** 已写回处理时间的行退出扫描谓词,不再被读到。 */
@Test
fun `rows already backfilled leave the scan predicate`() {
val done = inbox.simulateExternalWrite("<MSG/>")
assertEquals(1, poller.pollOnce(t0))
proc.markTerminal(done, ProcStatus.SUCCEEDED, now = t0)
inbox.markProcessedIfUnmarked(done, "PROCESSED")
proc.rows.clear() // 模拟到期清理后处理记录已删除
assertEquals(0, poller.pollOnce(t0)) // 标记已写回 → 不再重建记录
assertNull(proc.find(done))
}
/**
* 回归用例:处理完却永远不会回填的行(典型是解码失败的死信)曾经占满每一批的名额
* 导致收报整体停摆。取新消息这件事必须和"有没有处理标记"彻底分开
* 回归用例:处理完却还没回填的行(典型是等待回填的死信)会被反复扫到
* 但不得占满首批窗口——否则攒够一批之后新消息就再也发现不了(曾经的线上隐患)
*/
@Test
fun `terminal rows without a mailbox mark do not block discovery of later messages`() {
@@ -78,105 +88,60 @@ class InboxPollerTest {
assertEquals(1, poller.pollOnce(t0))
assertEquals(ProcStatus.PENDING, proc.find(fresh)!!.state)
assertEquals(fresh, cursor.cursor.committedUpTo)
}
/** 未回填的存量远多于一批时,续读要一直翻到最后一批,才能发现末尾的新行。 */
@Test
fun `watermark stops at a hole so later ids cannot overtake a missing smaller id`() {
val first = inbox.simulateExternalWrite("<MSG/>")
val hole = inbox.simulateExternalWrite("<MSG/>")
val afterHole = inbox.simulateExternalWrite("<MSG/>")
inbox.removeRow(hole)
fun `multi batch scan reaches new rows behind several full batches`() {
props.pipeline.claimBatch = 2
val stale = (1..5).map { inbox.simulateExternalWrite("<MSG/>") }
assertEquals(5, poller.pollOnce(t0))
stale.forEach { proc.markTerminal(it, ProcStatus.SUCCEEDED, now = t0) }
val fresh = inbox.simulateExternalWrite("<MSG/>")
assertEquals(1, poller.pollOnce(t0))
assertEquals(first, cursor.cursor.committedUpTo)
assertNotNull(cursor.cursor.holeSince)
assertNull(proc.find(afterHole)) // 不得越过空洞入队(FIFO
assertEquals(0, poller.pollOnce(t0.plusSeconds(60))) // 宽限期内水位不推进
assertEquals(first, cursor.cursor.committedUpTo)
assertNotNull(proc.find(fresh))
}
/** 重启不带任何消费边界:从头重扫,已登记的行不会被建成第二行。 */
@Test
fun `an aged hole is released and later ids resume enqueuing`() {
val hole = inbox.simulateExternalWrite("<MSG/>")
val afterHole = inbox.simulateExternalWrite("<MSG/>")
inbox.removeRow(hole)
poller.pollOnce(t0) // 记录空洞观测时刻
fun `a restarted poller rescans from the start without duplicating rows`() {
val ids = (1..3).map { inbox.simulateExternalWrite("<MSG/>") }
assertEquals(3, poller.pollOnce(t0))
val agedOut = t0.plus(props.pipeline.maxCommitDelay)
assertEquals(0, poller.pollOnce(agedOut)) // 空洞判永久:推进水位但不越过入队
val restarted = newPoller()
assertNull(cursor.cursor.holeSince)
assertEquals(afterHole - 1, cursor.cursor.committedUpTo)
assertEquals(1, poller.pollOnce(agedOut)) // 下一轮恢复发现
assertEquals(afterHole, cursor.cursor.committedUpTo)
assertNotNull(proc.find(afterHole))
assertEquals(0, restarted.pollOnce(t0.plusSeconds(60)))
assertEquals(ids.size, proc.rows.size)
assertEquals(ids, proc.rows.keys.sorted())
}
/** 编号较小的行晚提交:本轮读不到不算丢,下一轮照常发现。 */
@Test
fun `a late committed smaller id is discovered on the next round`() {
val first = inbox.simulateExternalWrite("<MSG/>")
val late = inbox.simulateExternalWrite("<MSG/>")
val third = inbox.simulateExternalWrite("<MSG/>")
inbox.removeRow(late)
assertEquals(2, poller.pollOnce(t0))
assertNotNull(proc.find(first))
assertNotNull(proc.find(third))
assertNull(proc.find(late))
inbox.restoreRow(late, "<MSG/>", t0.plusSeconds(30))
assertEquals(1, poller.pollOnce(t0.plusSeconds(30)))
assertEquals(ProcStatus.PENDING, proc.find(late)!!.state)
}
@Test
fun `compat http path and poller do not double enqueue the same message`() {
val receipt = InboxService(inbox, proc, java.time.Clock.fixed(t0, java.time.ZoneOffset.UTC)).accept("<MSG/>")
val receipt = InboxService(inbox, proc, clock).accept("<MSG/>")
assertEquals(0, poller.pollOnce(t0))
assertEquals(receipt.msgId, cursor.cursor.committedUpTo) // 已在 PG:读取进度照常推进
assertEquals(1, proc.rows.size)
assertNotNull(proc.find(receipt.msgId))
}
@Test
fun `a newly exposed hole does not inherit the age of the previous hole`() {
val first = inbox.simulateExternalWrite("<MSG/>")
val oldHole = inbox.simulateExternalWrite("<MSG/>")
val third = inbox.simulateExternalWrite("<MSG/>")
val newHole = inbox.simulateExternalWrite("<MSG/>")
val fifth = inbox.simulateExternalWrite("<MSG/>")
inbox.removeRow(oldHole)
inbox.removeRow(newHole)
assertEquals(1, poller.pollOnce(t0))
val almostAged = t0.plus(props.pipeline.maxCommitDelay).minusSeconds(1)
inbox.restoreRow(oldHole, "<MSG/>", almostAged)
assertEquals(2, poller.pollOnce(almostAged))
assertEquals(third, cursor.cursor.committedUpTo)
assertEquals(almostAged, cursor.cursor.holeSince)
assertNull(proc.find(fifth))
// 旧空洞的期限已到,但新空洞必须获得完整等待窗口。
assertEquals(0, poller.pollOnce(t0.plus(props.pipeline.maxCommitDelay)))
assertEquals(third, cursor.cursor.committedUpTo)
assertNull(proc.find(fifth))
assertEquals(first + 2, third)
}
/**
* 兼容入口会把消息登记在水位**之外**(`msgId > W`),因此它天然成为"最小未完成行"。
* 领取侧的守卫在主泵:只领 `msgId <= W`(端到端验收见 `PipelineSmokeTest` 的
* "compat injected high id is not claimed until the watermark catches up")。
* 本用例只固定收报侧的事实:**登记发生,但水位不动**。
*/
@Test
fun `compat accept registers a row above the watermark without advancing it`() {
props.pipeline.claimBatch = 10
val one = inbox.simulateExternalWrite("<MSG/>") // 1
val hole = inbox.simulateExternalWrite("<MSG/>") // 2(空洞)
val third = inbox.simulateExternalWrite("<MSG/>") // 3
inbox.removeRow(hole)
assertEquals(1, poller.pollOnce(t0)) // W=1,遇空洞即停
assertEquals(one, cursor.cursor.committedUpTo)
assertNull(proc.find(third)) // 3 还没入队
val receipt = InboxService(inbox, proc, java.time.Clock.fixed(t0, java.time.ZoneOffset.UTC))
.accept("<MSG/>") // 兼容入口:高 ID 直接进 PG
val high = receipt.msgId
assertTrue(high > third)
assertEquals(one, cursor.cursor.committedUpTo) // 水位不动(不参与水位)
// 主泵处理掉队头 1 之后,最小未完成行就是兼容入口写进来的高 ID……
proc.markTerminal(one, ProcStatus.SUCCEEDED, now = t0)
assertEquals(high, proc.headUnfinished()!!.msgId)
assertTrue(high > cursor.cursor.committedUpTo) // ……但它超出水位,主泵不会领取
assertNull(proc.find(third))
}
}
@@ -7,7 +7,6 @@ import com.gzzn.omms.msgexchange.infra.persistence.MailboxMarkResult
import com.gzzn.omms.msgexchange.infra.persistence.MailboxRow
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import com.gzzn.omms.msgexchange.infra.stub.StubInbox
import com.gzzn.omms.msgexchange.infra.stub.StubInboxCursor
import com.gzzn.omms.msgexchange.infra.stub.StubPipelineTx
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
import org.junit.jupiter.api.Assertions.assertEquals
@@ -43,11 +42,7 @@ class InboxServiceTest {
else -> Instant.parse("2026-09-06T01:59:55Z")
}
override fun readRange(fromExclusive: Long, limit: Int): List<MailboxRow> = emptyList()
override fun maxId(): Long? = null
override fun minId(): Long? = null
override fun readUnprocessed(afterId: Long, limit: Int): List<MailboxRow> = emptyList()
override fun markProcessedIfUnmarked(msgId: Long, value: String): MailboxMarkResult = MailboxMarkResult.MISSING
}
@@ -100,10 +95,9 @@ class InboxServiceTest {
fun `the poller backfills a compat receipt whose pg enqueue failed, exactly once`() {
val inbox = StubInbox().apply { clear() }
val proc = StubProcState().apply { clear() }
val cursor = StubInboxCursor().apply { clear() }
val clock = MutableClock(t0)
val service = InboxService(inbox, FlakyProcState(proc), clock)
val poller = InboxPoller(inbox, proc, cursor, StubPipelineTx(), PipelineProps(), clock)
val poller = InboxPoller(inbox, proc, StubPipelineTx(), PipelineProps(), clock)
val receipt = service.accept("<MSG/>") // 落信成功;PG 入队失败但不抛
assertEquals(1L, service.pgEnqueueFailures.get())
@@ -112,7 +106,6 @@ class InboxServiceTest {
assertEquals(1, poller.pollOnce(t0)) // 轮询补建
assertNotNull(proc.find(receipt.msgId))
assertEquals(1, proc.rows.size)
assertEquals(receipt.msgId, cursor.cursor.committedUpTo)
assertEquals(0, poller.pollOnce(t0)) // 已存在:不重复建行,也不重复计数
assertEquals(1, proc.rows.size)
@@ -46,9 +46,7 @@ class BackfillServiceTest {
override fun insertRaw(rawXml: String): Long = 1L
override fun rawOf(msgId: Long): String? = null
override fun receivedAtOf(msgId: Long): Instant? = null
override fun readRange(fromExclusive: Long, limit: Int): List<MailboxRow> = emptyList()
override fun maxId(): Long? = null
override fun minId(): Long? = null
override fun readUnprocessed(afterId: Long, limit: Int): List<MailboxRow> = emptyList()
override fun markProcessedIfUnmarked(msgId: Long, value: String): MailboxMarkResult {
if (fail) throw IllegalStateException("mysql-down")
if (missing) return MailboxMarkResult.MISSING
@@ -237,9 +235,7 @@ class BackfillServiceTest {
override fun insertRaw(rawXml: String) = 1L
override fun rawOf(msgId: Long): String? = "<MSG/>"
override fun receivedAtOf(msgId: Long) = t0
override fun readRange(fromExclusive: Long, limit: Int) = emptyList<MailboxRow>()
override fun maxId(): Long? = 1L
override fun minId(): Long? = 1L
override fun readUnprocessed(afterId: Long, limit: Int) = emptyList<MailboxRow>()
override fun markProcessedIfUnmarked(msgId: Long, value: String): MailboxMarkResult {
entered.countDown()
release.await()
@@ -11,14 +11,12 @@ import com.gzzn.omms.msgexchange.domain.MsgKind
import com.gzzn.omms.msgexchange.domain.ProcState
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.infra.metrics.PipelineCounters
import com.gzzn.omms.msgexchange.infra.persistence.InboxCursorRepository
import com.gzzn.omms.msgexchange.infra.persistence.PipelineLockRepository
import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
import com.gzzn.omms.msgexchange.infra.retry.ProcFailure
import com.gzzn.omms.msgexchange.infra.retry.FailureScheduler
import com.gzzn.omms.msgexchange.infra.stub.StubFlightState
import com.gzzn.omms.msgexchange.infra.stub.StubInbox
import com.gzzn.omms.msgexchange.infra.stub.StubInboxCursor
import com.gzzn.omms.msgexchange.infra.stub.StubMsgEvents
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
import com.gzzn.omms.msgexchange.infra.stub.StubSnapshotLog
@@ -63,8 +61,6 @@ class IgnoreBranchTest {
counters: PipelineCounters = PipelineCounters(),
codec: XmlCodec,
): MessageProcessor {
val cursor = StubInboxCursor()
cursor.cursor = InboxCursorRepository.Cursor(committedUpTo = Long.MAX_VALUE)
val clock = Clock.systemUTC()
val props = PipelineProps()
val opDay = OperationDayProps().apply { zone = "Asia/Shanghai"; cutoffHour = 0 }