fix(pipeline): 修复取报-处理-回填链路的超时、吞吐、回填与 FIFO 问题
外部调用(ACM2-33) - 共享 MySQL 与自有 PG 全部补有界超时:驱动 connect/socket 超时 + Hikari 池超时 (注意 Micronaut 的 Hikari 项是毫秒数,不是 Duration 字面量) - 删除主泵/毒丸路径的 inline 回填:跨库写不再占用 FIFO 关键路径,回填统一由扫描驱动 - 游标自愈:save() 改为「先 UPDATE、缺行 INSERT」,缺行只记一次 ERROR - Pump/Dispatcher 外层 catch 补 error 日志与失败计数 投递吞吐(ACM2-34) - Dispatcher 改批量领取;队头退避未到期或首条发送失败即停止本轮(保持目标内保序) - 有活不再 sleep;删除不可达的 state==SENT 死条件;markSent 移入分支; superseded 清理加轮数上限与空 eventId 保护 回填闭环(ACM2-36,V4) - MISSING(运行时确认行不存在)立即放弃自动重试并告警;暂时性故障达上限后停止自动重试 - 新增 reopen 人工恢复入口;放弃 ≠ 标记已确认(BACKFILL_AT 仍为空,清除前提不成立) - 扫描改 (BACKFILL_ATTEMPTS, MSG_ID) 公平轮转并排除放弃行,消除全局回填饥饿 - V4 只加字段与必要索引,不按年龄做任何存量推断 切流播种(ACM2-35,V5) - cutover-watermark 四模式(min/zero/max/显式 ID),默认不播种、代码不做默认选择 - 升级实例拒绝重新播种(SEEDED_AT 为 NULL ≠ 从未消费);播种与水位同语句落库 - 非法取值由启动自检挡下 错误分类与入口契约(ACM2-37) - 未知 SCHD 子类型改为 UNSUPPORTED,不再静默当全量日计划合并 - ADFT 运营日冲突改走 ProtocolViolation → DEAD(PROTOCOL) - 兼容入口 receivedAt 缺失回退到注入 Clock;MessageLifecycleGate 强制注入 + 装配断言 - FIFO:主泵只领取 msgId ≤ W,兼容入口登记的行在水位追平前不被领取 时间源与可观测(ACM2-38 / ACM2-41 阶段 0) - 仓储/处理器/作业全部经注入 Clock;移除 markTerminal/markBackfilled 的 Instant.now() 默认值 - 退避表档位与 max-attempts 对齐并加启动自检 - Micrometer 7 个 gauge(@Context 急切注册)+ /health 与 /metrics 共用积压快照缓存 - 只读迟到检测:监视被放行的空洞 ID 是否后来真的出现,只计数告警、不补入队 Plane: ACM2-33 ACM2-34 ACM2-35 ACM2-36 ACM2-37 ACM2-38 ACM2-41 Tests: 88 → 120(1 skipped 需真实 PG)
This commit is contained in:
@@ -34,7 +34,8 @@ class BackfillService(
|
||||
private val mailboxProps: MailboxProps,
|
||||
private val props: PipelineProps,
|
||||
private val clock: Clock,
|
||||
private val lifecycleGate: MessageLifecycleGate = MessageLifecycleGate(),
|
||||
/** 与人工重放共用的互斥门;`internal` 以便装配测试断言两者拿到同一实例。 */
|
||||
internal val lifecycleGate: MessageLifecycleGate,
|
||||
) {
|
||||
private val log = org.slf4j.LoggerFactory.getLogger(BackfillService::class.java)
|
||||
|
||||
@@ -43,6 +44,12 @@ class BackfillService(
|
||||
private val MAX_BACKOFF: Duration = Duration.ofMinutes(15)
|
||||
private val TERMINAL_STATES = setOf(ProcStatus.SUCCEEDED, ProcStatus.SKIPPED, ProcStatus.DEAD)
|
||||
|
||||
/** 放弃原因:运行时查询确认信箱行不存在(确定性结论,重试不会改变结果)。 */
|
||||
const val ABANDON_MISSING_ROW = "MISSING_ROW"
|
||||
|
||||
/** 放弃原因:暂时性故障达到尝试上限;停止自动重试,但保留人工恢复能力。 */
|
||||
const val ABANDON_MAX_ATTEMPTS = "MAX_ATTEMPTS"
|
||||
|
||||
fun backoffDelayFor(attempts: Int): Duration {
|
||||
val shift = (attempts - 1).coerceIn(0, 20)
|
||||
return INITIAL_BACKOFF.multipliedBy(1L shl shift).coerceAtMost(MAX_BACKOFF)
|
||||
@@ -58,15 +65,27 @@ class BackfillService(
|
||||
val row = procState.find(msgId) ?: return@exclusive
|
||||
if (row.state !in TERMINAL_STATES) return@exclusive
|
||||
if (row.backfillAt != null) return@exclusive
|
||||
if (row.backfillAbandonedAt != null) return@exclusive
|
||||
record(msgId, attempts = row.backfillAttempts, now = now)?.let {
|
||||
log.warn("backfill failed msgId={} error={} (sweep will retry)", msgId, it)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 人工恢复入口:清除放弃标记并重排一次回填。
|
||||
* 用于暂时性故障恢复之后,或人工对账确认该行仍应打标之后。
|
||||
*/
|
||||
fun reopen(msgId: Long, now: Instant = clock.instant()): Boolean =
|
||||
lifecycleGate.exclusive { procState.reopenBackfill(msgId, now) }
|
||||
|
||||
/**
|
||||
* 批量补写,由 JobRunner 每 30 秒调用一次。待办状态都在数据库里,
|
||||
* 进程重启后接着跑,不需要额外恢复步骤。
|
||||
*
|
||||
* 注意:**调度周期不是完成时限**——扫描与历史作业串行,批次积压、单行调用超时与
|
||||
* 历史作业耗时都会延长实际回填延迟。
|
||||
*
|
||||
* @return 本批处理的条数
|
||||
*/
|
||||
fun sweep(now: Instant = clock.instant()): Int {
|
||||
@@ -75,19 +94,37 @@ class BackfillService(
|
||||
return due.size
|
||||
}
|
||||
|
||||
/** 回填一条。@return 失败原因;返回 null 表示标记已确认(包括"别的路径已经标过了")。 */
|
||||
/** 回填一条。@return 失败原因;返回 null 表示已处理完(写成功/早已标记/已放弃)。 */
|
||||
private fun record(msgId: Long, attempts: Int, now: Instant): String? =
|
||||
try {
|
||||
// 没有真正写进去说明库里已经有标记了,同样算成功(不覆盖已有值)
|
||||
val result = mailbox.markProcessedIfUnmarked(msgId, mailboxProps.processedValue)
|
||||
check(result != MailboxMarkResult.MISSING) { "mailbox-row-missing" }
|
||||
procState.markBackfilled(msgId, now)
|
||||
null
|
||||
when (mailbox.markProcessedIfUnmarked(msgId, mailboxProps.processedValue)) {
|
||||
MailboxMarkResult.MARKED, MailboxMarkResult.ALREADY_MARKED -> {
|
||||
// 没有真正写进去说明库里已经有标记了,同样算成功(不覆盖已有值)
|
||||
procState.markBackfilled(msgId, now)
|
||||
null
|
||||
}
|
||||
MailboxMarkResult.MISSING -> {
|
||||
// 确定性结论:行不存在,重试不会改变结果。停止自动补偿并把事实留痕。
|
||||
// 这**不等于**标记已确认(BACKFILL_AT 仍为空),清除前提因此仍然不成立。
|
||||
procState.markBackfillAbandoned(msgId, ABANDON_MISSING_ROW, now)
|
||||
log.error("backfill abandoned: mailbox row missing msgId={}", msgId)
|
||||
null
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
// 超时/连接失败是暂时性的,**不能**当作缺行证据:按退避重试。
|
||||
// 达到上限后停止自动重试(保留人工恢复能力),避免永久占满扫描批次造成饥饿。
|
||||
val reason = e.message ?: e.javaClass.simpleName
|
||||
runCatching {
|
||||
procState.recordBackfillFailure(msgId, reason, attempts + 1, now.plus(backoffDelayFor(attempts + 1)), now)
|
||||
}.onFailure { log.error("record backfill failure failed msgId={}", msgId, it) }
|
||||
val nextAttempts = attempts + 1
|
||||
if (nextAttempts >= props.pipeline.backfillMaxAttempts) {
|
||||
runCatching { procState.markBackfillAbandoned(msgId, ABANDON_MAX_ATTEMPTS, now) }
|
||||
.onFailure { log.error("abandon backfill failed msgId={}", msgId, it) }
|
||||
log.error("backfill abandoned after {} attempts msgId={} error={}", nextAttempts, msgId, reason)
|
||||
} else {
|
||||
runCatching {
|
||||
procState.recordBackfillFailure(msgId, reason, nextAttempts, now.plus(backoffDelayFor(nextAttempts)), now)
|
||||
}.onFailure { log.error("record backfill failure failed msgId={}", msgId, it) }
|
||||
}
|
||||
reason
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user