fix(processing): 落地 D1/D5/D6 三项裁决,删除 head-deadline 参数
D5(终态判据只保留尝试上限): - Pump.tick 内联 attempts 判定,删除 head-deadline 相关的毒丸分支与滞留告警代码 - 删除配置项 head-deadline(PipelineProps / application.yml)与 PumpDeadlineTest - PROCESSING_STARTED_AT 变为只写,注释如实说明当前无判据消费它 D1(回填放弃判据改为时间): - 暂时性故障在 R 之前只退避重试,不再按尝试次数放弃;到 R 才放弃并记 TRANSIENT_DEADLINE - backfill-max-attempts 降级为单行重试的告警阈值 D6(超期判据改用本地入队时间): - 新增 V6 迁移:PROC_STATE 加 ENQUEUED_AT(回填存量后置为非空 + 默认) - findBackfillDue 的谓词与 overdue 标记改比较 enqueued_at,不再用库方时钟的 received_at - BackfillDue 增加 overdue;收报与兼容入口显式写入本地入队时间 文档同步: - 清理 4 处 message-lifecycle.md 章节号死链(Pump/InboxService/PipelineProps/application.yml) - 关闭 G-HEAD-DEADLINE、G-BACKFILL-ABANDON-BYTIME、G-ENQUEUED-AT 三条缺口登记 - reference/user-stories/README 与实现对齐 验证:./gradlew test ⇒ 122 tests, 0 failures, 1 skipped Refs: ACM2-45
This commit is contained in:
@@ -19,9 +19,12 @@ import java.time.Instant
|
||||
* 这个类负责第二件:
|
||||
*
|
||||
* - [attempt]:单行写标记,只由 [sweep] 逐行调用——主泵与处理器不直接调它(回填一律扫描驱动)。
|
||||
* - [sweep]:定时把还欠回填的记录挑出来重试。失败就按 30 秒起步、最长 15 分钟的
|
||||
* 退避往后推;如果一条消息从收到现在已经超过超期期限,则无视退避强制补写——
|
||||
* 否则退避可能一直失败下去,这些行永远打不上标记,库方就没法清理信箱。
|
||||
* - [sweep]:定时把还欠回填的记录挑出来重试。失败按 30 秒起步、最长 15 分钟的退避往后推;
|
||||
* 入队时间早于 `NOW − R` 的行无视退避、每轮都试。
|
||||
*
|
||||
* 放弃判据是**时间**(`R` 超期)而不是尝试次数:一次小时级的共享库故障不该把待回填行成批
|
||||
* 判死、再要求人工成批恢复。`R` 之前只退避重试;到 `R` 仍未打标才停止自动重试并进放弃清单
|
||||
* (保留 [reopen] 人工恢复)。`backfill-max-attempts` 因此降级为单行重试的告警阈值。
|
||||
*
|
||||
* 两条底线:回填失败不会把终态改回去,也不会重新执行业务逻辑;写标记只写还是空标记的
|
||||
* 行,重复执行没有副作用。
|
||||
@@ -46,8 +49,8 @@ class BackfillService(
|
||||
/** 放弃原因:运行时查询确认信箱行不存在(确定性结论,重试不会改变结果)。 */
|
||||
const val ABANDON_MISSING_ROW = "MISSING_ROW"
|
||||
|
||||
/** 放弃原因:暂时性故障达到尝试上限;停止自动重试,但保留人工恢复能力。 */
|
||||
const val ABANDON_MAX_ATTEMPTS = "MAX_ATTEMPTS"
|
||||
/** 放弃原因:暂时性故障持续到 `R` 仍未打标;停止自动重试,但保留人工恢复能力。 */
|
||||
const val ABANDON_TRANSIENT_DEADLINE = "TRANSIENT_DEADLINE"
|
||||
|
||||
fun backoffDelayFor(attempts: Int): Duration {
|
||||
val shift = (attempts - 1).coerceIn(0, 20)
|
||||
@@ -59,13 +62,17 @@ class BackfillService(
|
||||
* 处理完立刻试一次。失败只记一笔退避信息就返回,不抛异常——
|
||||
* 调用方是主泵的处理路径,不能被回填问题拖住。
|
||||
*/
|
||||
fun attempt(msgId: Long, now: Instant = clock.instant()) {
|
||||
/**
|
||||
* @param overdue 该行入队时间早于 `NOW − R`(已进入强补写窗口)。通常来自 [sweep] 的扫描结果;
|
||||
* 直接调用默认 false,即按普通退避处理。
|
||||
*/
|
||||
fun attempt(msgId: Long, overdue: Boolean = false, now: Instant = clock.instant()) {
|
||||
lifecycleGate.exclusive {
|
||||
val row = procState.find(msgId) ?: return@exclusive
|
||||
if (row.state !in TERMINAL_STATES) return@exclusive
|
||||
if (row.backfillAt != null) return@exclusive
|
||||
if (row.backfillAbandonedAt != null) return@exclusive
|
||||
record(msgId, attempts = row.backfillAttempts, now = now)?.let {
|
||||
record(msgId, attempts = row.backfillAttempts, overdue = overdue, now = now)?.let {
|
||||
log.warn("backfill failed msgId={} error={} (sweep will retry)", msgId, it)
|
||||
}
|
||||
}
|
||||
@@ -89,12 +96,12 @@ class BackfillService(
|
||||
*/
|
||||
fun sweep(now: Instant = clock.instant()): Int {
|
||||
val due = procState.findBackfillDue(now, now.minus(props.pipeline.overdueBackfill), props.pipeline.backfillBatch)
|
||||
due.forEach { attempt(it.msgId, now) }
|
||||
due.forEach { attempt(it.msgId, it.overdue, now) }
|
||||
return due.size
|
||||
}
|
||||
|
||||
/** 回填一条。@return 失败原因;返回 null 表示已处理完(写成功/早已标记/已放弃)。 */
|
||||
private fun record(msgId: Long, attempts: Int, now: Instant): String? =
|
||||
private fun record(msgId: Long, attempts: Int, overdue: Boolean, now: Instant): String? =
|
||||
try {
|
||||
when (mailbox.markProcessedIfUnmarked(msgId, mailboxProps.processedValue)) {
|
||||
MailboxMarkResult.MARKED, MailboxMarkResult.ALREADY_MARKED -> {
|
||||
@@ -111,18 +118,25 @@ class BackfillService(
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
// 超时/连接失败是暂时性的,**不能**当作缺行证据:按退避重试。
|
||||
// 达到上限后停止自动重试(保留人工恢复能力),避免永久占满扫描批次造成饥饿。
|
||||
// 超时/连接失败是暂时性的,**不能**当作缺行证据,也不能按次数放弃:
|
||||
// 按退避重试,直到入队时间超过 R 才停止自动重试(保留人工恢复能力)。
|
||||
val reason = e.message ?: e.javaClass.simpleName
|
||||
val nextAttempts = attempts + 1
|
||||
if (nextAttempts >= props.pipeline.backfillMaxAttempts) {
|
||||
runCatching { procState.markBackfillAbandoned(msgId, ABANDON_MAX_ATTEMPTS, now) }
|
||||
if (overdue) {
|
||||
runCatching { procState.markBackfillAbandoned(msgId, ABANDON_TRANSIENT_DEADLINE, now) }
|
||||
.onFailure { log.error("abandon backfill failed msgId={}", msgId, it) }
|
||||
log.error("backfill abandoned after {} attempts msgId={} error={}", nextAttempts, msgId, reason)
|
||||
log.error("backfill abandoned at R msgId={} attempts={} error={}", msgId, nextAttempts, reason)
|
||||
} else {
|
||||
runCatching {
|
||||
procState.recordBackfillFailure(msgId, reason, nextAttempts, now.plus(backoffDelayFor(nextAttempts)), now)
|
||||
}.onFailure { log.error("record backfill failure failed msgId={}", msgId, it) }
|
||||
if (nextAttempts >= props.pipeline.backfillMaxAttempts) {
|
||||
log.warn(
|
||||
"backfill retries for msgId={} reached the warning threshold ({}); " +
|
||||
"still retrying until R elapses — 放弃判据是 R 超期,不是次数",
|
||||
msgId, props.pipeline.backfillMaxAttempts,
|
||||
)
|
||||
}
|
||||
}
|
||||
reason
|
||||
}
|
||||
|
||||
@@ -25,8 +25,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
*
|
||||
* 每次 tick 只看当前最小的未完成消息("队头"):
|
||||
* - 没有待处理消息就睡一个轮询间隔;
|
||||
* - 队头失败了还在退避期,就等到能重试的时刻;如果重试次数用尽或滞留太久,
|
||||
* 直接转死信,不放任它一直堵着;
|
||||
* - 队头失败了还在退避期,就等到能重试的时刻;重试次数用尽才转死信,不放任它一直堵着;
|
||||
* - 其余情况交给 [MessageProcessor] 处理。
|
||||
*
|
||||
* 一次只处理一条是刻意的。后面的消息不能越过卡住的队头,否则同一条航班的报文
|
||||
@@ -81,38 +80,36 @@ class Pump(
|
||||
//
|
||||
// 水位以内的行都是收报按 ID 顺序发现并登记的;水位之外的行只可能来自兼容入口
|
||||
// 直接写 PROC_STATE(它不参与水位)。若允许领取,它就会越过那些尚未入队的较小 ID,
|
||||
// 破坏 FIFO(不变量"只领取已发现的行",message-lifecycle.md §11)。这种行在空洞补齐、`W` 追平之后自然可领取。
|
||||
// 破坏 FIFO(不变量"只领取已发现的行",`invariants.md` INV-4)。这种行在空洞补齐、`W` 追平之后自然可领取。
|
||||
val watermark = cursor.load().committedUpTo
|
||||
if (head.msgId > watermark) {
|
||||
warnBeyondWatermark(head.msgId, watermark)
|
||||
sleepQuietly(props.pipeline.pollInterval)
|
||||
return
|
||||
}
|
||||
val now = clock.instant()
|
||||
when {
|
||||
head.state == ProcStatus.FAILED && poisoned(head) -> {
|
||||
head.state == ProcStatus.FAILED && head.attempts >= props.pipeline.maxAttempts -> {
|
||||
log.error("poison -> DEAD msgId={} attempts={} lastError={}", head.msgId, head.attempts, head.lastError)
|
||||
// markTerminal 在同一条 UPDATE 里登记回填意图;回填由扫描补写,不在这里做跨库写。
|
||||
procState.markTerminal(
|
||||
head.msgId, ProcStatus.DEAD,
|
||||
errorClass = ErrorClass.EXHAUSTED,
|
||||
lastError = head.lastError ?: "head-deadline-exceeded",
|
||||
lastError = head.lastError ?: "attempts-exhausted",
|
||||
attempts = head.attempts,
|
||||
now = clock.instant(),
|
||||
now = now,
|
||||
)
|
||||
}
|
||||
head.state == ProcStatus.FAILED && (head.nextAttemptAt ?: Instant.EPOCH) > clock.instant() ->
|
||||
sleepQuietly(Duration.between(clock.instant(), head.nextAttemptAt))
|
||||
head.state == ProcStatus.FAILED && (head.nextAttemptAt ?: Instant.EPOCH) > now ->
|
||||
sleepQuietly(Duration.between(now, head.nextAttemptAt))
|
||||
// 其余情况(新消息,或退避到期的重试)交给处理入口
|
||||
else -> {
|
||||
procState.markProcessingStartedIfAbsent(head.msgId, clock.instant())
|
||||
procState.markProcessingStartedIfAbsent(head.msgId, now)
|
||||
processor.processOne(head)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun poisoned(head: ProcState): Boolean =
|
||||
isHeadPoisoned(head, clock.instant(), props)
|
||||
|
||||
/** 上一次"队头在水位之外"告警时的水位值:只在它变化时告警,避免每秒刷屏。 */
|
||||
private val warnedWatermark = AtomicLong(Long.MIN_VALUE)
|
||||
|
||||
@@ -131,10 +128,6 @@ class Pump(
|
||||
}
|
||||
}
|
||||
|
||||
internal fun isHeadPoisoned(head: ProcState, now: Instant, props: PipelineProps): Boolean =
|
||||
head.attempts >= props.pipeline.maxAttempts ||
|
||||
Duration.between(head.processingStartedAt ?: head.updatedAt, now) >= props.pipeline.headDeadline
|
||||
|
||||
/**
|
||||
* 处理一条消息:读原文 → 解码 → 绑定业务身份 → 分派给对应处理器。
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user