fix(processing): 落地 D1/D5/D6 三项裁决,删除 head-deadline 参数

D5(终态判据只保留尝试上限):
- Pump.tick 内联 attempts 判定,删除 head-deadline 相关的毒丸分支与滞留告警代码
- 删除配置项 head-deadline(PipelineProps / application.yml)与 PumpDeadlineTest
- PROCESSING_STARTED_AT 变为只写,注释如实说明当前无判据消费它

D1(回填放弃判据改为时间):
- 暂时性故障在 R 之前只退避重试,不再按尝试次数放弃;到 R 才放弃并记 TRANSIENT_DEADLINE
- backfill-max-attempts 降级为单行重试的告警阈值

D6(超期判据改用本地入队时间):
- 新增 V6 迁移:PROC_STATE 加 ENQUEUED_AT(回填存量后置为非空 + 默认)
- findBackfillDue 的谓词与 overdue 标记改比较 enqueued_at,不再用库方时钟的 received_at
- BackfillDue 增加 overdue;收报与兼容入口显式写入本地入队时间

文档同步:
- 清理 4 处 message-lifecycle.md 章节号死链(Pump/InboxService/PipelineProps/application.yml)
- 关闭 G-HEAD-DEADLINE、G-BACKFILL-ABANDON-BYTIME、G-ENQUEUED-AT 三条缺口登记
- reference/user-stories/README 与实现对齐

验证:./gradlew test ⇒ 122 tests, 0 failures, 1 skipped

Refs: ACM2-45
This commit is contained in:
windyboy
2026-09-11 20:44:30 +08:00
parent 6eade95a97
commit 817236ca26
21 changed files with 217 additions and 152 deletions
@@ -19,9 +19,12 @@ import java.time.Instant
* 这个类负责第二件:
*
* - [attempt]:单行写标记,只由 [sweep] 逐行调用——主泵与处理器不直接调它(回填一律扫描驱动)。
* - [sweep]:定时把还欠回填的记录挑出来重试。失败按 30 秒起步、最长 15 分钟的
* 退避往后推;如果一条消息从收到现在已经超过超期期限,则无视退避强制补写——
* 否则退避可能一直失败下去,这些行永远打不上标记,库方就没法清理信箱。
* - [sweep]:定时把还欠回填的记录挑出来重试。失败按 30 秒起步、最长 15 分钟的退避往后推;
* 入队时间早于 `NOW − R` 的行无视退避、每轮都试。
*
* 放弃判据是**时间**(`R` 超期)而不是尝试次数:一次小时级的共享库故障不该把待回填行成批
* 判死、再要求人工成批恢复。`R` 之前只退避重试;到 `R` 仍未打标才停止自动重试并进放弃清单
* (保留 [reopen] 人工恢复)。`backfill-max-attempts` 因此降级为单行重试的告警阈值。
*
* 两条底线:回填失败不会把终态改回去,也不会重新执行业务逻辑;写标记只写还是空标记的
* 行,重复执行没有副作用。
@@ -46,8 +49,8 @@ class BackfillService(
/** 放弃原因:运行时查询确认信箱行不存在(确定性结论,重试不会改变结果)。 */
const val ABANDON_MISSING_ROW = "MISSING_ROW"
/** 放弃原因:暂时性故障达到尝试上限;停止自动重试,但保留人工恢复能力。 */
const val ABANDON_MAX_ATTEMPTS = "MAX_ATTEMPTS"
/** 放弃原因:暂时性故障持续到 `R` 仍未打标;停止自动重试,但保留人工恢复能力。 */
const val ABANDON_TRANSIENT_DEADLINE = "TRANSIENT_DEADLINE"
fun backoffDelayFor(attempts: Int): Duration {
val shift = (attempts - 1).coerceIn(0, 20)
@@ -59,13 +62,17 @@ class BackfillService(
* 处理完立刻试一次。失败只记一笔退避信息就返回,不抛异常——
* 调用方是主泵的处理路径,不能被回填问题拖住。
*/
fun attempt(msgId: Long, now: Instant = clock.instant()) {
/**
* @param overdue 该行入队时间早于 `NOW − R`(已进入强补写窗口)。通常来自 [sweep] 的扫描结果;
* 直接调用默认 false,即按普通退避处理。
*/
fun attempt(msgId: Long, overdue: Boolean = false, now: Instant = clock.instant()) {
lifecycleGate.exclusive {
val row = procState.find(msgId) ?: return@exclusive
if (row.state !in TERMINAL_STATES) return@exclusive
if (row.backfillAt != null) return@exclusive
if (row.backfillAbandonedAt != null) return@exclusive
record(msgId, attempts = row.backfillAttempts, now = now)?.let {
record(msgId, attempts = row.backfillAttempts, overdue = overdue, now = now)?.let {
log.warn("backfill failed msgId={} error={} (sweep will retry)", msgId, it)
}
}
@@ -89,12 +96,12 @@ class BackfillService(
*/
fun sweep(now: Instant = clock.instant()): Int {
val due = procState.findBackfillDue(now, now.minus(props.pipeline.overdueBackfill), props.pipeline.backfillBatch)
due.forEach { attempt(it.msgId, now) }
due.forEach { attempt(it.msgId, it.overdue, now) }
return due.size
}
/** 回填一条。@return 失败原因;返回 null 表示已处理完(写成功/早已标记/已放弃)。 */
private fun record(msgId: Long, attempts: Int, now: Instant): String? =
private fun record(msgId: Long, attempts: Int, overdue: Boolean, now: Instant): String? =
try {
when (mailbox.markProcessedIfUnmarked(msgId, mailboxProps.processedValue)) {
MailboxMarkResult.MARKED, MailboxMarkResult.ALREADY_MARKED -> {
@@ -111,18 +118,25 @@ class BackfillService(
}
}
} catch (e: Exception) {
// 超时/连接失败是暂时性的,**不能**当作缺行证据:按退避重试。
// 达到上限后停止自动重试(保留人工恢复能力),避免永久占满扫描批次造成饥饿
// 超时/连接失败是暂时性的,**不能**当作缺行证据,也不能按次数放弃:
// 按退避重试,直到入队时间超过 R 才停止自动重试(保留人工恢复能力)。
val reason = e.message ?: e.javaClass.simpleName
val nextAttempts = attempts + 1
if (nextAttempts >= props.pipeline.backfillMaxAttempts) {
runCatching { procState.markBackfillAbandoned(msgId, ABANDON_MAX_ATTEMPTS, now) }
if (overdue) {
runCatching { procState.markBackfillAbandoned(msgId, ABANDON_TRANSIENT_DEADLINE, now) }
.onFailure { log.error("abandon backfill failed msgId={}", msgId, it) }
log.error("backfill abandoned after {} attempts msgId={} error={}", nextAttempts, msgId, reason)
log.error("backfill abandoned at R msgId={} attempts={} error={}", msgId, nextAttempts, reason)
} else {
runCatching {
procState.recordBackfillFailure(msgId, reason, nextAttempts, now.plus(backoffDelayFor(nextAttempts)), now)
}.onFailure { log.error("record backfill failure failed msgId={}", msgId, it) }
if (nextAttempts >= props.pipeline.backfillMaxAttempts) {
log.warn(
"backfill retries for msgId={} reached the warning threshold ({}); " +
"still retrying until R elapses — 放弃判据是 R 超期,不是次数",
msgId, props.pipeline.backfillMaxAttempts,
)
}
}
reason
}
@@ -25,8 +25,7 @@ import java.util.concurrent.atomic.AtomicLong
*
* 每次 tick 只看当前最小的未完成消息("队头"):
* - 没有待处理消息就睡一个轮询间隔;
* - 队头失败了还在退避期,就等到能重试的时刻;如果重试次数用尽或滞留太久,
* 直接转死信,不放任它一直堵着;
* - 队头失败了还在退避期,就等到能重试的时刻;重试次数用尽才转死信,不放任它一直堵着;
* - 其余情况交给 [MessageProcessor] 处理。
*
* 一次只处理一条是刻意的。后面的消息不能越过卡住的队头,否则同一条航班的报文
@@ -81,38 +80,36 @@ class Pump(
//
// 水位以内的行都是收报按 ID 顺序发现并登记的;水位之外的行只可能来自兼容入口
// 直接写 PROC_STATE(它不参与水位)。若允许领取,它就会越过那些尚未入队的较小 ID,
// 破坏 FIFO(不变量"只领取已发现的行",message-lifecycle.md §11)。这种行在空洞补齐、`W` 追平之后自然可领取。
// 破坏 FIFO(不变量"只领取已发现的行",`invariants.md` INV-4)。这种行在空洞补齐、`W` 追平之后自然可领取。
val watermark = cursor.load().committedUpTo
if (head.msgId > watermark) {
warnBeyondWatermark(head.msgId, watermark)
sleepQuietly(props.pipeline.pollInterval)
return
}
val now = clock.instant()
when {
head.state == ProcStatus.FAILED && poisoned(head) -> {
head.state == ProcStatus.FAILED && head.attempts >= props.pipeline.maxAttempts -> {
log.error("poison -> DEAD msgId={} attempts={} lastError={}", head.msgId, head.attempts, head.lastError)
// markTerminal 在同一条 UPDATE 里登记回填意图;回填由扫描补写,不在这里做跨库写。
procState.markTerminal(
head.msgId, ProcStatus.DEAD,
errorClass = ErrorClass.EXHAUSTED,
lastError = head.lastError ?: "head-deadline-exceeded",
lastError = head.lastError ?: "attempts-exhausted",
attempts = head.attempts,
now = clock.instant(),
now = now,
)
}
head.state == ProcStatus.FAILED && (head.nextAttemptAt ?: Instant.EPOCH) > clock.instant() ->
sleepQuietly(Duration.between(clock.instant(), head.nextAttemptAt))
head.state == ProcStatus.FAILED && (head.nextAttemptAt ?: Instant.EPOCH) > now ->
sleepQuietly(Duration.between(now, head.nextAttemptAt))
// 其余情况(新消息,或退避到期的重试)交给处理入口
else -> {
procState.markProcessingStartedIfAbsent(head.msgId, clock.instant())
procState.markProcessingStartedIfAbsent(head.msgId, now)
processor.processOne(head)
}
}
}
private fun poisoned(head: ProcState): Boolean =
isHeadPoisoned(head, clock.instant(), props)
/** 上一次"队头在水位之外"告警时的水位值:只在它变化时告警,避免每秒刷屏。 */
private val warnedWatermark = AtomicLong(Long.MIN_VALUE)
@@ -131,10 +128,6 @@ class Pump(
}
}
internal fun isHeadPoisoned(head: ProcState, now: Instant, props: PipelineProps): Boolean =
head.attempts >= props.pipeline.maxAttempts ||
Duration.between(head.processingStartedAt ?: head.updatedAt, now) >= props.pipeline.headDeadline
/**
* 处理一条消息:读原文 → 解码 → 绑定业务身份 → 分派给对应处理器。
*