feat(jobs): 提交后回填补偿落库 backfill_todo 并到期重试 (ACM2-29 P2-3)

- V1.3.2 迁移:backfill_todo 持久补偿表(attempts/next_attempt_at/last_error)
- BackfillTodoRepository 接口 + PG 实现与 stub;record 幂等 upsert
- BackfillSweepJob:到期重试,成功即删,失败指数退避(30s 起步封顶 15 分钟);
  触发点 = InboxPoller 每轮心跳(重启即对账)+ JobExecutor BACKFILL_SWEEP 类型
- MessageProcessor/SnapshotFlow 回填失败落待办(成功终态不降级,业务不重放);
  SKIPPED 重复报文同样补偿共享信箱回填,防止重复信件被反复轮询
- 新测试:扫描重试/退避/恢复对账 4 用例;快照回填失败落库断言;
  跳过路径补偿落库断言

验证:MSGX_PG_PORT=5433 真实 PG 全量 102 用例 0 失败 0 跳过 (ACM2-29)
This commit is contained in:
windyboy
2026-09-08 13:04:43 +08:00
parent 44fbac0d8b
commit 48a27b26dd
12 changed files with 402 additions and 3 deletions
@@ -8,6 +8,7 @@ import com.gzzn.omms.msgexchange.domain.MsgEvent
import com.gzzn.omms.msgexchange.domain.ProcState
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.domain.Targets
import com.gzzn.omms.msgexchange.infra.persistence.BackfillTodoRepository
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
@@ -121,6 +122,7 @@ class MessageProcessor(
private val procFailure: ProcFailure,
private val props: PipelineProps,
private val txManager: PipelineTransactionManager,
private val backfillTodo: BackfillTodoRepository? = null,
) {
private val log = org.slf4j.LoggerFactory.getLogger(MessageProcessor::class.java)
private val flidRegex = Regex("<FLID>(.*?)</FLID>", RegexOption.IGNORE_CASE)
@@ -188,6 +190,8 @@ class MessageProcessor(
val owner = procState.ownerOfIdentity(identity) ?: -1L
log.info("duplicate-of:{} -> SKIPPED id={}", owner, head.cminmsgsId)
procState.update(head.cminmsgsId, ProcStatus.SKIPPED, lastError = "duplicate-of:$owner")
// 重复报文自身也是一条信箱记录:同样回填共享信箱,防止被反复轮询(失败落补偿)
compensateBackfill(head, decoded)
return
}
}
@@ -243,9 +247,33 @@ class MessageProcessor(
inbox.backfillOnSuccess(head.cminmsgsId, decoded.meta.sndr, decoded.meta.type, decoded.meta.styp, decoded.meta.seqn)
} catch (e: Exception) {
log.error("backfill failed after SUCCEEDED id={} (compensation required)", head.cminmsgsId, e)
recordBackfillTodo(head.cminmsgsId, decoded, e)
}
log.info("SUCCEEDED id={} events={} flightChanges={}", head.cminmsgsId, events.size, decision.flightChanges.size)
}
/** v2 §5:回填失败只落补偿待办(成功终态不降级,业务不重放);由 BackfillSweepJob 到期重试。 */
private fun compensateBackfill(head: ProcState, decoded: DecodedMessage) {
try {
inbox.backfillOnSuccess(head.cminmsgsId, decoded.meta.sndr, decoded.meta.type, decoded.meta.styp, decoded.meta.seqn)
} catch (e: Exception) {
log.error("backfill failed for SKIPPED duplicate id={} (compensation required)", head.cminmsgsId, e)
recordBackfillTodo(head.cminmsgsId, decoded, e)
}
}
private fun recordBackfillTodo(cminmsgsId: Long, decoded: DecodedMessage, e: Exception) {
backfillTodo?.record(
BackfillTodoRepository.BackfillTask(
cminmsgsId = cminmsgsId,
sndr = decoded.meta.sndr,
type = decoded.meta.type,
styp = decoded.meta.styp,
seqn = decoded.meta.seqn,
),
e.message ?: e.javaClass.simpleName,
) ?: log.warn("no backfill-todo repository bound; compensation NOT persisted id={}", cminmsgsId)
}
}
/** 延迟装配占位(阶段 1 后续以 Micronaut Bean 替换直连构造)。 */
@@ -8,6 +8,7 @@ import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.domain.MsgEvent
import com.gzzn.omms.msgexchange.domain.MsgKind
import com.gzzn.omms.msgexchange.domain.Targets
import com.gzzn.omms.msgexchange.infra.persistence.BackfillTodoRepository
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
import com.gzzn.omms.msgexchange.infra.persistence.FlightFields
import com.gzzn.omms.msgexchange.infra.persistence.FlightFieldsJson
@@ -33,6 +34,7 @@ class SnapshotFlow(
private val procFailure: ProcFailure,
private val txManager: PipelineTransactionManager,
private val inbox: CminmsgInboxRepository,
private val backfillTodo: BackfillTodoRepository? = null,
) {
private val log = org.slf4j.LoggerFactory.getLogger(SnapshotFlow::class.java)
@@ -139,6 +141,16 @@ class SnapshotFlow(
inbox.backfillOnSuccess(head.cminmsgsId, msg.meta.sndr, msg.meta.type, msg.meta.styp, msg.meta.seqn)
} catch (e: Exception) {
log.error("backfill failed after SUCCEEDED id={} (compensation required)", head.cminmsgsId, e)
backfillTodo?.record(
BackfillTodoRepository.BackfillTask(
cminmsgsId = head.cminmsgsId,
sndr = msg.meta.sndr,
type = msg.meta.type,
styp = msg.meta.styp,
seqn = msg.meta.seqn,
),
e.message ?: e.javaClass.simpleName,
) ?: log.warn("no backfill-todo repository bound; compensation NOT persisted id={}", head.cminmsgsId)
}
}