fix(processing): 落地 D1/D5/D6 三项裁决,删除 head-deadline 参数
D5(终态判据只保留尝试上限): - Pump.tick 内联 attempts 判定,删除 head-deadline 相关的毒丸分支与滞留告警代码 - 删除配置项 head-deadline(PipelineProps / application.yml)与 PumpDeadlineTest - PROCESSING_STARTED_AT 变为只写,注释如实说明当前无判据消费它 D1(回填放弃判据改为时间): - 暂时性故障在 R 之前只退避重试,不再按尝试次数放弃;到 R 才放弃并记 TRANSIENT_DEADLINE - backfill-max-attempts 降级为单行重试的告警阈值 D6(超期判据改用本地入队时间): - 新增 V6 迁移:PROC_STATE 加 ENQUEUED_AT(回填存量后置为非空 + 默认) - findBackfillDue 的谓词与 overdue 标记改比较 enqueued_at,不再用库方时钟的 received_at - BackfillDue 增加 overdue;收报与兼容入口显式写入本地入队时间 文档同步: - 清理 4 处 message-lifecycle.md 章节号死链(Pump/InboxService/PipelineProps/application.yml) - 关闭 G-HEAD-DEADLINE、G-BACKFILL-ABANDON-BYTIME、G-ENQUEUED-AT 三条缺口登记 - reference/user-stories/README 与实现对齐 验证:./gradlew test ⇒ 122 tests, 0 failures, 1 skipped Refs: ACM2-45
This commit is contained in:
@@ -19,9 +19,12 @@ import java.time.Instant
|
||||
* 这个类负责第二件:
|
||||
*
|
||||
* - [attempt]:单行写标记,只由 [sweep] 逐行调用——主泵与处理器不直接调它(回填一律扫描驱动)。
|
||||
* - [sweep]:定时把还欠回填的记录挑出来重试。失败就按 30 秒起步、最长 15 分钟的
|
||||
* 退避往后推;如果一条消息从收到现在已经超过超期期限,则无视退避强制补写——
|
||||
* 否则退避可能一直失败下去,这些行永远打不上标记,库方就没法清理信箱。
|
||||
* - [sweep]:定时把还欠回填的记录挑出来重试。失败按 30 秒起步、最长 15 分钟的退避往后推;
|
||||
* 入队时间早于 `NOW − R` 的行无视退避、每轮都试。
|
||||
*
|
||||
* 放弃判据是**时间**(`R` 超期)而不是尝试次数:一次小时级的共享库故障不该把待回填行成批
|
||||
* 判死、再要求人工成批恢复。`R` 之前只退避重试;到 `R` 仍未打标才停止自动重试并进放弃清单
|
||||
* (保留 [reopen] 人工恢复)。`backfill-max-attempts` 因此降级为单行重试的告警阈值。
|
||||
*
|
||||
* 两条底线:回填失败不会把终态改回去,也不会重新执行业务逻辑;写标记只写还是空标记的
|
||||
* 行,重复执行没有副作用。
|
||||
@@ -46,8 +49,8 @@ class BackfillService(
|
||||
/** 放弃原因:运行时查询确认信箱行不存在(确定性结论,重试不会改变结果)。 */
|
||||
const val ABANDON_MISSING_ROW = "MISSING_ROW"
|
||||
|
||||
/** 放弃原因:暂时性故障达到尝试上限;停止自动重试,但保留人工恢复能力。 */
|
||||
const val ABANDON_MAX_ATTEMPTS = "MAX_ATTEMPTS"
|
||||
/** 放弃原因:暂时性故障持续到 `R` 仍未打标;停止自动重试,但保留人工恢复能力。 */
|
||||
const val ABANDON_TRANSIENT_DEADLINE = "TRANSIENT_DEADLINE"
|
||||
|
||||
fun backoffDelayFor(attempts: Int): Duration {
|
||||
val shift = (attempts - 1).coerceIn(0, 20)
|
||||
@@ -59,13 +62,17 @@ class BackfillService(
|
||||
* 处理完立刻试一次。失败只记一笔退避信息就返回,不抛异常——
|
||||
* 调用方是主泵的处理路径,不能被回填问题拖住。
|
||||
*/
|
||||
fun attempt(msgId: Long, now: Instant = clock.instant()) {
|
||||
/**
|
||||
* @param overdue 该行入队时间早于 `NOW − R`(已进入强补写窗口)。通常来自 [sweep] 的扫描结果;
|
||||
* 直接调用默认 false,即按普通退避处理。
|
||||
*/
|
||||
fun attempt(msgId: Long, overdue: Boolean = false, now: Instant = clock.instant()) {
|
||||
lifecycleGate.exclusive {
|
||||
val row = procState.find(msgId) ?: return@exclusive
|
||||
if (row.state !in TERMINAL_STATES) return@exclusive
|
||||
if (row.backfillAt != null) return@exclusive
|
||||
if (row.backfillAbandonedAt != null) return@exclusive
|
||||
record(msgId, attempts = row.backfillAttempts, now = now)?.let {
|
||||
record(msgId, attempts = row.backfillAttempts, overdue = overdue, now = now)?.let {
|
||||
log.warn("backfill failed msgId={} error={} (sweep will retry)", msgId, it)
|
||||
}
|
||||
}
|
||||
@@ -89,12 +96,12 @@ class BackfillService(
|
||||
*/
|
||||
fun sweep(now: Instant = clock.instant()): Int {
|
||||
val due = procState.findBackfillDue(now, now.minus(props.pipeline.overdueBackfill), props.pipeline.backfillBatch)
|
||||
due.forEach { attempt(it.msgId, now) }
|
||||
due.forEach { attempt(it.msgId, it.overdue, now) }
|
||||
return due.size
|
||||
}
|
||||
|
||||
/** 回填一条。@return 失败原因;返回 null 表示已处理完(写成功/早已标记/已放弃)。 */
|
||||
private fun record(msgId: Long, attempts: Int, now: Instant): String? =
|
||||
private fun record(msgId: Long, attempts: Int, overdue: Boolean, now: Instant): String? =
|
||||
try {
|
||||
when (mailbox.markProcessedIfUnmarked(msgId, mailboxProps.processedValue)) {
|
||||
MailboxMarkResult.MARKED, MailboxMarkResult.ALREADY_MARKED -> {
|
||||
@@ -111,18 +118,25 @@ class BackfillService(
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
// 超时/连接失败是暂时性的,**不能**当作缺行证据:按退避重试。
|
||||
// 达到上限后停止自动重试(保留人工恢复能力),避免永久占满扫描批次造成饥饿。
|
||||
// 超时/连接失败是暂时性的,**不能**当作缺行证据,也不能按次数放弃:
|
||||
// 按退避重试,直到入队时间超过 R 才停止自动重试(保留人工恢复能力)。
|
||||
val reason = e.message ?: e.javaClass.simpleName
|
||||
val nextAttempts = attempts + 1
|
||||
if (nextAttempts >= props.pipeline.backfillMaxAttempts) {
|
||||
runCatching { procState.markBackfillAbandoned(msgId, ABANDON_MAX_ATTEMPTS, now) }
|
||||
if (overdue) {
|
||||
runCatching { procState.markBackfillAbandoned(msgId, ABANDON_TRANSIENT_DEADLINE, now) }
|
||||
.onFailure { log.error("abandon backfill failed msgId={}", msgId, it) }
|
||||
log.error("backfill abandoned after {} attempts msgId={} error={}", nextAttempts, msgId, reason)
|
||||
log.error("backfill abandoned at R msgId={} attempts={} error={}", msgId, nextAttempts, reason)
|
||||
} else {
|
||||
runCatching {
|
||||
procState.recordBackfillFailure(msgId, reason, nextAttempts, now.plus(backoffDelayFor(nextAttempts)), now)
|
||||
}.onFailure { log.error("record backfill failure failed msgId={}", msgId, it) }
|
||||
if (nextAttempts >= props.pipeline.backfillMaxAttempts) {
|
||||
log.warn(
|
||||
"backfill retries for msgId={} reached the warning threshold ({}); " +
|
||||
"still retrying until R elapses — 放弃判据是 R 超期,不是次数",
|
||||
msgId, props.pipeline.backfillMaxAttempts,
|
||||
)
|
||||
}
|
||||
}
|
||||
reason
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user