cdia/msgexchange-v2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(processing): 落地 D1/D5/D6 三项裁决，删除 head-deadline 参数

Browse Source 
D5（终态判据只保留尝试上限）：
- Pump.tick 内联 attempts 判定，删除 head-deadline 相关的毒丸分支与滞留告警代码
- 删除配置项 head-deadline（PipelineProps / application.yml）与 PumpDeadlineTest
- PROCESSING_STARTED_AT 变为只写，注释如实说明当前无判据消费它

D1（回填放弃判据改为时间）：
- 暂时性故障在 R 之前只退避重试，不再按尝试次数放弃；到 R 才放弃并记 TRANSIENT_DEADLINE
- backfill-max-attempts 降级为单行重试的告警阈值

D6（超期判据改用本地入队时间）：
- 新增 V6 迁移：PROC_STATE 加 ENQUEUED_AT（回填存量后置为非空 + 默认）
- findBackfillDue 的谓词与 overdue 标记改比较 enqueued_at，不再用库方时钟的 received_at
- BackfillDue 增加 overdue；收报与兼容入口显式写入本地入队时间

文档同步：
- 清理 4 处 message-lifecycle.md 章节号死链（Pump/InboxService/PipelineProps/application.yml）
- 关闭 G-HEAD-DEADLINE、G-BACKFILL-ABANDON-BYTIME、G-ENQUEUED-AT 三条缺口登记
- reference/user-stories/README 与实现对齐

验证：./gradlew test ⇒ 122 tests, 0 failures, 1 skipped

Refs: ACM2-45
This commit is contained in:
windyboy
2026-09-11 20:44:30 +08:00
parent 6eade95a97
commit 817236ca26
21 changed files with 217 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/FlywayMigrationTest.kt
+14 -4
View File
@@ -83,10 +83,10 @@ class FlywayMigrationTest {
assertEquals(setOf("flid", "operation_day", "state", "state_version", "last_msg_id"), cols)
}
// 回填相关的列都落在 PROC_STATE 上（收信时间判断超期；abandoned 记录"停止自动重试"）
// 回填相关的列都落在 PROC_STATE 上（本地入队时间判断超期；abandoned 记录"停止自动重试"）
stmt.executeQuery(
"SELECT column_name FROM information_schema.columns WHERE table_name = 'proc_state' " +
"AND column_name IN ('received_at', 'backfill_at', 'backfill_next_at', " +
"AND column_name IN ('received_at', 'enqueued_at', 'backfill_at', 'backfill_next_at', " +
"'backfill_attempts', 'backfill_error', 'backfill_abandoned_at', " +
"'backfill_abandoned_reason', 'processing_started_at')",
).use { rs ->
@@ -94,13 +94,23 @@ class FlywayMigrationTest {
while (rs.next()) cols.add(rs.getString("column_name"))
assertEquals(
setOf(
"received_at", "backfill_at", "backfill_next_at", "backfill_attempts", "backfill_error",
"backfill_abandoned_at", "backfill_abandoned_reason", "processing_started_at",
"received_at", "enqueued_at", "backfill_at", "backfill_next_at", "backfill_attempts",
"backfill_error", "backfill_abandoned_at", "backfill_abandoned_reason",
"processing_started_at",
),
cols,
)
}
// V6：入队时间是超期判据 R 的比较对象，必须非空（received_at 则允许为 NULL）
stmt.executeQuery(
"SELECT is_nullable FROM information_schema.columns " +
"WHERE table_name = 'proc_state' AND column_name = 'enqueued_at'",
).use { rs ->
assertTrue(rs.next(), "V6 必须已加上 ENQUEUED_AT 列")
assertEquals("NO", rs.getString("is_nullable"), "ENQUEUED_AT 必须非空")
}
// 两张单行表（管道锁、收报水位）的种子数据都要在
stmt.executeQuery("SELECT count(*) FROM pipeline_lock WHERE lock_id = 1").use { rs ->
assertTrue(rs.next())
src/test/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/InboxLifecycleJdbcSqlTest.kt
+12 -5
View File
@@ -100,7 +100,7 @@ class InboxLifecycleJdbcSqlTest {
seed(2L, t0)
proc.markTerminal(2L, ProcStatus.SUCCEEDED, now = t0)
proc.recordBackfillFailure(2L, "mysql-down", 1, t0.plus(Duration.ofMinutes(15)), t0)
// ③ 退避还没到，但收信时间已经很久了：应当无视退避直接补写
// ③ 退避还没到，但本地入队时间已经超过 R：应当无视退避直接补写
seed(3L, overdue)
proc.markTerminal(3L, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, now = t0)
proc.recordBackfillFailure(3L, "mysql-down", 1, t0.plus(Duration.ofMinutes(15)), t0)
@@ -111,10 +111,13 @@ class InboxLifecycleJdbcSqlTest {
proc.markTerminal(5L, ProcStatus.SUCCEEDED, now = t0)
proc.markBackfilled(5L, t0)
val due = proc.findBackfillDue(t0, t0.minus(Duration.ofDays(30)), limit = 100).map { it.msgId }
val due = proc.findBackfillDue(t0, t0.minus(Duration.ofDays(30)), limit = 100)
val ids = due.map { it.msgId }
assertEquals(listOf(1L, 3L), due)
assertEquals(listOf(1L, 3L), ids)
assertEquals(listOf(1L), proc.findBackfillDue(t0, t0.minus(Duration.ofDays(30)), limit = 1).map { it.msgId })
// overdue 标记只在入队时间早于 NOW − R 时为真；调用方据此决定暂时性故障是否到放弃期限。
assertEquals(listOf(1L to false, 3L to true), due.map { it.msgId to it.overdue })
}
@Test
@@ -186,15 +189,18 @@ class InboxLifecycleJdbcSqlTest {
assertEquals(MailboxMarkResult.MISSING, mailbox.markProcessedIfUnmarked(999L, "PROCESSED"))
}
/** 播种一行：`enqueued_at` 与 `received_at` 取同一时刻（超期判据只看前者）。 */
private fun seed(msgId: Long, receivedAt: Instant?, state: ProcStatus = ProcStatus.PENDING) {
ds.connection.use { conn: Connection ->
conn.prepareStatement(
"INSERT INTO proc_state (msg_id, state, received_at, backfill_attempts, updated_at) VALUES (?, ?, ?, 0, ?)",
"INSERT INTO proc_state (msg_id, state, received_at, enqueued_at, backfill_attempts, updated_at) " +
"VALUES (?, ?, ?, ?, 0, ?)",
).use { ps ->
ps.setLong(1, msgId)
ps.setString(2, state.name)
ps.setTimestamp(3, receivedAt?.let { Timestamp.from(it) })
ps.setTimestamp(4, Timestamp.from(t0))
ps.setTimestamp(4, receivedAt?.let { Timestamp.from(it) } ?: Timestamp.from(t0))
ps.setTimestamp(5, Timestamp.from(t0))
ps.executeUpdate()
}
}
@@ -219,6 +225,7 @@ class InboxLifecycleJdbcSqlTest {
error_class VARCHAR(20),
last_error VARCHAR(1000),
received_at TIMESTAMP WITH TIME ZONE,
enqueued_at TIMESTAMP WITH TIME ZONE NOT NULL,
backfill_at TIMESTAMP WITH TIME ZONE,
backfill_next_at TIMESTAMP WITH TIME ZONE,
backfill_attempts INT NOT NULL DEFAULT 0,
src/test/kotlin/com/gzzn/omms/msgexchange/infra/retry/ReplayServiceTest.kt
+5 -1
View File
@@ -26,7 +26,11 @@ class ReplayServiceTest {
)
}
override fun insertIfAbsent(msgId: Long, receivedAt: Instant?): Boolean = rows.putIfAbsent(msgId, ProcState(msgId, ProcStatus.PENDING, receivedAt = receivedAt)) == null
override fun insertIfAbsent(msgId: Long, receivedAt: Instant?, enqueuedAt: Instant?): Boolean =
rows.putIfAbsent(
msgId,
ProcState(msgId, ProcStatus.PENDING, receivedAt = receivedAt, enqueuedAt = enqueuedAt ?: receivedAt),
) == null
override fun find(msgId: Long): ProcState? = rows[msgId]
override fun findSuccessTerminal(msgId: Long): Boolean = rows[msgId]?.state == ProcStatus.SUCCEEDED
override fun headUnfinished(): ProcState? = null
src/test/kotlin/com/gzzn/omms/msgexchange/processing/BackfillServiceTest.kt
+35 -14
View File
@@ -301,25 +301,44 @@ class BackfillServiceTest {
assertNotNull(proc.find(911L)!!.backfillNextAt)
}
/**
* 门禁裁决 D1：暂时性故障**不按次数放弃**。即使超过 `backfill-max-attempts` 这个
* 告警阈值，也继续退避重试——一次小时级的共享库故障不该把待回填行成批判死。
*/
@Test
fun `transient failures keep retrying until the attempt cap and stay recoverable`() {
fun `transient failures keep retrying past the warning threshold`() {
props.pipeline.backfillMaxAttempts = 3
val proc = StubProcState()
val mailbox = FakeMailbox(fail = true)
succeeded(proc, 912L)
val svc = service(proc, mailbox)
svc.attempt(912L) // 1 次：暂时性故障，只退避
svc.attempt(912L) // 2 次
assertNull(proc.find(912L)!!.backfillAbandonedAt)
repeat(5) { svc.attempt(912L) }
svc.attempt(912L) // 3 次：达到上限 → 停止自动重试
val row = proc.find(912L)!!
assertEquals(BackfillService.ABANDON_MAX_ATTEMPTS, row.backfillAbandonedReason)
assertEquals(5, row.backfillAttempts)
assertNull(row.backfillAbandonedAt) // 次数不是放弃判据
assertNull(row.backfillAt)
assertNotNull(row.backfillNextAt) // 仍在退避重试
}
assertTrue(svc.reopen(912L)) // 人工恢复入口存在且有效
assertNull(proc.find(912L)!!.backfillAbandonedAt)
/** 到 `R` 仍未打标（overdue）时，暂时性故障才停止自动重试并进放弃清单。 */
@Test
fun `overdue transient failure is abandoned with the deadline reason and stays recoverable`() {
val proc = StubProcState()
val mailbox = FakeMailbox(fail = true)
succeeded(proc, 913L)
val svc = service(proc, mailbox)
svc.attempt(913L, overdue = true)
val row = proc.find(913L)!!
assertEquals(BackfillService.ABANDON_TRANSIENT_DEADLINE, row.backfillAbandonedReason)
assertNotNull(row.backfillAbandonedAt)
assertNull(row.backfillAt) // 放弃 ≠ 标记已确认
assertTrue(svc.reopen(913L)) // 人工恢复入口存在且有效
assertNull(proc.find(913L)!!.backfillAbandonedAt)
}
@Test
@@ -346,18 +365,20 @@ class BackfillServiceTest {
}
/**
* 语义固定（G4）：`RECEIVED_AT` 为 NULL 时"超期"分支不成立，`R` 兜底**不生效**，
* 该行只能靠退避重试。把这条钉住，避免以后误以为 `R` 一定能兜底。
* 语义固定（门禁裁决 D6）：超期判据是**本地入队时间** `ENQUEUED_AT`，与 `RECEIVED_AT` 无关。
* 上游没给接收时间（NULL）不再让 `R` 兜底失效——这正是引入 `ENQUEUED_AT` 要消除的窗口。
*/
@Test
fun `null received time disables the overdue shortcut so only backoff applies`() {
fun `null received time no longer disables the overdue shortcut`() {
val proc = StubProcState()
proc.insertIfAbsent(921L, null) // 上游未提供接收时间
val old = t0.minus(props.pipeline.overdueBackfill).minusSeconds(60)
proc.insertIfAbsent(921L, null, enqueuedAt = old) // 上游未提供接收时间
proc.markTerminal(921L, ProcStatus.SUCCEEDED, now = t0)
proc.recordBackfillFailure(921L, "mysql-down", attempts = 1, nextAttemptAt = t0.plusSeconds(3600), now = t0)
val due = proc.findBackfillDue(now = t0, overdueBefore = t0.plusSeconds(10_000), limit = 10)
val due = proc.findBackfillDue(now = t0, overdueBefore = t0.minus(props.pipeline.overdueBackfill), limit = 10)
assertTrue(due.isEmpty()) // 超期分支无效 + 退避未到期
assertEquals(listOf(921L), due.map { it.msgId }) // 退避未到期，但入队时间已超 R → 仍被扫描
assertTrue(due.single().overdue)
}
}
src/test/kotlin/com/gzzn/omms/msgexchange/processing/PumpDeadlineTest.kt
-40
View File
@@ -1,40 +0,0 @@
package com.gzzn.omms.msgexchange.processing
import com.gzzn.omms.msgexchange.config.PipelineProps
import com.gzzn.omms.msgexchange.domain.ProcState
import com.gzzn.omms.msgexchange.domain.ProcStatus
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import java.time.Instant
class PumpDeadlineTest {
private val props = PipelineProps()
private val started = Instant.parse("2026-09-08T03:00:00Z")
@Test
fun `deadline uses stable processing start rather than refreshed update time`() {
val row = ProcState(
msgId = 1,
state = ProcStatus.FAILED,
attempts = 1,
processingStartedAt = started,
updatedAt = started.plusSeconds(590),
)
assertTrue(isHeadPoisoned(row, started.plusSeconds(600), props))
}
@Test
fun `head below attempt and time limits remains retryable`() {
val row = ProcState(
msgId = 1,
state = ProcStatus.FAILED,
attempts = 1,
processingStartedAt = started,
updatedAt = started.plusSeconds(590),
)
assertFalse(isHeadPoisoned(row, started.plusSeconds(599), props))
}
}