feat(jobs): 提交后回填补偿落库 backfill_todo 并到期重试 (ACM2-29 P2-3)

- V1.3.2 迁移：backfill_todo 持久补偿表（attempts/next_attempt_at/last_error）
- BackfillTodoRepository 接口 + PG 实现与 stub；record 幂等 upsert
- BackfillSweepJob：到期重试，成功即删，失败指数退避（30s 起步封顶 15 分钟）；
  触发点 = InboxPoller 每轮心跳（重启即对账）+ JobExecutor BACKFILL_SWEEP 类型
- MessageProcessor/SnapshotFlow 回填失败落待办（成功终态不降级，业务不重放）；
  SKIPPED 重复报文同样补偿共享信箱回填，防止重复信件被反复轮询
- 新测试：扫描重试/退避/恢复对账 4 用例；快照回填失败落库断言；
  跳过路径补偿落库断言

验证：MSGX_PG_PORT=5433 真实 PG 全量 102 用例 0 失败 0 跳过 (ACM2-29)

src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/Repositories.kt

@@ -162,6 +162,37 @@ interface PipelineTransactionManager {
     fun <T> inTransaction(block: () -> T): T
 }
 
+/**
+ * v2 §5（ACM2-29 P2-3）：提交后共享信箱回填的持久补偿待办。
+ * 业务事务已提交（PROC_STATE=SUCCEEDED）后 backfillOnSuccess 失败 → 落本表重试；
+ * 回填失败不得把已成功的业务事务重新标记为失败，也不得重放业务变更。
+ */
+interface BackfillTodoRepository {
+    data class BackfillTask(
+        val cminmsgsId: Long,
+        val sndr: String,
+        val type: String,
+        val styp: String,
+        val seqn: Long,
+        val attempts: Int = 0,
+    )
+
+    /** 失败即落库（幂等 upsert，同 ID 重复失败只刷新错误与重试时间）。 */
+    fun record(task: BackfillTask, lastError: String?, now: Instant = Instant.now())
+
+    /** 到期待重试的补偿任务（next_attempt_at <= now，按到期时间升序）。 */
+    fun findDue(now: Instant = Instant.now(), limit: Int = 50): List<BackfillTask>
+
+    /** 重试失败：累加 attempts 并按退避推后 next_attempt_at。 */
+    fun markFailed(cminmsgsId: Long, lastError: String?, nextAttemptAt: Instant, now: Instant = Instant.now())
+
+    /** 重试成功：移除补偿待办。 */
+    fun delete(cminmsgsId: Long)
+
+    /** 当前待办数量（测试/对账用）。 */
+    fun count(): Int
+}
+
 @Deprecated("Replaced by FlightSchdRepository in ACM2-28", ReplaceWith("FlightSchdRepository"))
 interface RefDataRepository {
     data class GenMeta(val flids: List<String>, val version: Long)

src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/JdbcPgRepositories.kt

@@ -9,6 +9,7 @@ import com.gzzn.omms.msgexchange.domain.MsgEvent
 import com.gzzn.omms.msgexchange.domain.ProcState
 import com.gzzn.omms.msgexchange.domain.ProcStatus
 import com.gzzn.omms.msgexchange.domain.RefUpsert
+import com.gzzn.omms.msgexchange.infra.persistence.BackfillTodoRepository
 import com.gzzn.omms.msgexchange.infra.persistence.FlightFields
 import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository
 import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository
@@ -311,6 +312,78 @@ class JdbcPipelineTransactionManager(
     }
 }
 
+/** v2 §5（ACM2-29 P2-3）：提交后共享信箱回填补偿待办的 PG 实现。 */
+@Singleton
+@Requires(property = "msgx.stubs", notEquals = "true")
+@Requires(property = "datasources.default.enabled", value = "true")
+class JdbcBackfillTodoRepository(
+    private val ds: DataSource,
+) : BackfillTodoRepository {
+
+    override fun record(task: BackfillTodoRepository.BackfillTask, lastError: String?, now: Instant) {
+        val ts = now.toSqlTimestamp()
+        ds.update(
+            """
+            INSERT INTO backfill_todo (cminmsgs_id, sndr, type, styp, seqn, attempts, next_attempt_at, last_error, created_at, updated_at)
+            VALUES (?, ?, ?, ?, ?, 0, ?, ?, ?, ?)
+            ON CONFLICT (cminmsgs_id) DO UPDATE SET
+              last_error = EXCLUDED.last_error,
+              next_attempt_at = EXCLUDED.next_attempt_at,
+              updated_at = EXCLUDED.updated_at
+            """.trimIndent(),
+        ) { ps ->
+            ps.setLong(1, task.cminmsgsId)
+            ps.setString(2, task.sndr)
+            ps.setString(3, task.type)
+            ps.setString(4, task.styp)
+            ps.setLong(5, task.seqn)
+            ps.setTimestamp(6, ts)
+            ps.setString(7, lastError?.take(512))
+            ps.setTimestamp(8, ts)
+            ps.setTimestamp(9, ts)
+        }
+    }
+
+    override fun findDue(now: Instant, limit: Int): List<BackfillTodoRepository.BackfillTask> =
+        ds.query(
+            """
+            SELECT cminmsgs_id, sndr, type, styp, seqn, attempts FROM backfill_todo
+            WHERE next_attempt_at <= ? ORDER BY next_attempt_at ASC LIMIT ?
+            """.trimIndent(),
+            { ps ->
+                ps.setTimestamp(1, now.toSqlTimestamp())
+                ps.setInt(2, limit)
+            },
+        ) { rs ->
+            BackfillTodoRepository.BackfillTask(
+                cminmsgsId = rs.getLong("cminmsgs_id"),
+                sndr = rs.getString("sndr"),
+                type = rs.getString("type"),
+                styp = rs.getString("styp") ?: "",
+                seqn = rs.getLong("seqn"),
+                attempts = rs.getInt("attempts"),
+            )
+        }
+
+    override fun markFailed(cminmsgsId: Long, lastError: String?, nextAttemptAt: Instant, now: Instant) {
+        ds.update(
+            "UPDATE backfill_todo SET attempts = attempts + 1, next_attempt_at = ?, last_error = ?, updated_at = ? WHERE cminmsgs_id = ?",
+        ) { ps ->
+            ps.setTimestamp(1, nextAttemptAt.toSqlTimestamp())
+            ps.setString(2, lastError?.take(512))
+            ps.setTimestamp(3, now.toSqlTimestamp())
+            ps.setLong(4, cminmsgsId)
+        }
+    }
+
+    override fun delete(cminmsgsId: Long) {
+        ds.update("DELETE FROM backfill_todo WHERE cminmsgs_id = ?") { ps -> ps.setLong(1, cminmsgsId) }
+    }
+
+    override fun count(): Int =
+        ds.queryOne("SELECT COUNT(*) AS c FROM backfill_todo", {}) { rs -> rs.getInt("c") } ?: 0
+}
+
 @Singleton
 @Requires(property = "msgx.stubs", notEquals = "true")
 @Requires(property = "datasources.default.enabled", value = "true")

src/main/kotlin/com/gzzn/omms/msgexchange/infra/stub/StubRepositories.kt

@@ -6,6 +6,7 @@ import com.gzzn.omms.msgexchange.domain.MsgEvent
 import com.gzzn.omms.msgexchange.domain.ProcState
 import com.gzzn.omms.msgexchange.domain.ProcStatus
 import com.gzzn.omms.msgexchange.domain.RefUpsert
+import com.gzzn.omms.msgexchange.infra.persistence.BackfillTodoRepository
 import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
 import com.gzzn.omms.msgexchange.infra.persistence.FlightFields
 import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository
@@ -435,3 +436,49 @@ class StubFlightState(
     override fun findByDay(day: String): List<Pair<String, FlightFields>> =
         flightSchd.findByDay(day)
 }
+
+@Requires(property = "msgx.stubs", value = "true")
+@Singleton
+class StubBackfillTodo : BackfillTodoRepository {
+    private data class Row(val task: BackfillTodoRepository.BackfillTask, var nextAttemptAt: Instant)
+
+    private val rows = linkedMapOf<Long, Row>()
+    private val errors = linkedMapOf<Long, String?>()
+
+    @Synchronized
+    fun clear() {
+        rows.clear()
+        errors.clear()
+    }
+
+    @Synchronized
+    fun tasks(): List<BackfillTodoRepository.BackfillTask> = rows.values.map { it.task }
+
+    @Synchronized
+    fun lastErrorOf(cminmsgsId: Long): String? = errors[cminmsgsId]
+
+    @Synchronized
+    override fun record(task: BackfillTodoRepository.BackfillTask, lastError: String?, now: Instant) {
+        rows[task.cminmsgsId] = Row(task, now)
+        errors[task.cminmsgsId] = lastError
+    }
+
+    @Synchronized
+    override fun findDue(now: Instant, limit: Int): List<BackfillTodoRepository.BackfillTask> =
+        rows.values.filter { it.nextAttemptAt <= now }.map { it.task }.take(limit)
+
+    @Synchronized
+    override fun markFailed(cminmsgsId: Long, lastError: String?, nextAttemptAt: Instant, now: Instant) {
+        rows[cminmsgsId]?.let { rows[cminmsgsId] = it.copy(task = it.task.copy(attempts = it.task.attempts + 1), nextAttemptAt = nextAttemptAt) }
+        errors[cminmsgsId] = lastError
+    }
+
+    @Synchronized
+    override fun delete(cminmsgsId: Long) {
+        rows.remove(cminmsgsId)
+        errors.remove(cminmsgsId)
+    }
+
+    @Synchronized
+    override fun count(): Int = rows.size
+}

src/main/kotlin/com/gzzn/omms/msgexchange/ingress/InboxPoller.kt

@@ -2,6 +2,7 @@ package com.gzzn.omms.msgexchange.ingress
 
 import com.gzzn.omms.msgexchange.config.PipelineProps
 import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
+import com.gzzn.omms.msgexchange.jobs.BackfillSweepJob
 import jakarta.inject.Singleton
 
 /**
@@ -12,6 +13,7 @@ import jakarta.inject.Singleton
 class InboxPoller(
     private val inbox: CminmsgInboxRepository,
     private val enqueue: InboxEnqueue,
+    private val backfillSweep: BackfillSweepJob,
     private val props: PipelineProps,
 ) {
     private val log = org.slf4j.LoggerFactory.getLogger(InboxPoller::class.java)
@@ -30,9 +32,22 @@ class InboxPoller(
                 log.info("polled cminmsgsId={}", id)
             }
         }
+        sweepBackfillTodos()
         return enqueued
     }
 
+    /** v2 §5：每轮心跳顺带对账回填补偿待办（重启即恢复；空表只花一次索引 SELECT）。 */
+    private fun sweepBackfillTodos() {
+        try {
+            val outcome = backfillSweep.sweep()
+            if (outcome.inspected > 0) {
+                log.info("backfill sweep inspected={} succeeded={} failed={}", outcome.inspected, outcome.succeeded, outcome.failed)
+            }
+        } catch (e: Exception) {
+            log.error("backfill sweep tick failed", e)
+        }
+    }
+
     fun loop() {
         running = true
         log.info("inbox poller loop started")

src/main/kotlin/com/gzzn/omms/msgexchange/jobs/BackfillSweepJob.kt

@@ -0,0 +1,53 @@
+package com.gzzn.omms.msgexchange.jobs
+
+import com.gzzn.omms.msgexchange.config.PipelineProps
+import com.gzzn.omms.msgexchange.infra.persistence.BackfillTodoRepository
+import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
+import jakarta.inject.Singleton
+import java.time.Duration
+import java.time.Instant
+
+/**
+ * v2 §5（ACM2-29 P2-3）：共享信箱回填补偿扫描。
+ *
+ * 业务事务提交（SUCCEEDED）后 backfillOnSuccess 失败 → 落 backfill_todo；
+ * 本任务到期重试：成功即删除待办，失败按指数退避推后（30s 起步，封顶 15 分钟）。
+ * 回填失败绝不重放航班业务变更——待办里只有信箱回填所需的路由元数据。
+ *
+ * 触发点：InboxPoller 心跳（重启即对账）+ JobExecutor("BACKFILL_SWEEP")（FIFO/手工）。
+ */
+@Singleton
+class BackfillSweepJob(
+    private val todo: BackfillTodoRepository,
+    private val inbox: CminmsgInboxRepository,
+    @Suppress("unused") private val props: PipelineProps,
+) {
+    data class SweepOutcome(val inspected: Int, val succeeded: Int, val failed: Int)
+
+    companion object {
+        private val INITIAL_BACKOFF: Duration = Duration.ofSeconds(30)
+        private val MAX_BACKOFF: Duration = Duration.ofMinutes(15)
+
+        fun backoffDelayFor(attempts: Int): Duration {
+            val shift = (attempts - 1).coerceIn(0, 20)
+            return INITIAL_BACKOFF.multipliedBy(1L shl shift).coerceAtMost(MAX_BACKOFF)
+        }
+    }
+
+    fun sweep(now: Instant = Instant.now()): SweepOutcome {
+        val due = todo.findDue(now)
+        var succeeded = 0
+        var failed = 0
+        for (task in due) {
+            try {
+                inbox.backfillOnSuccess(task.cminmsgsId, task.sndr, task.type, task.styp, task.seqn)
+                todo.delete(task.cminmsgsId)
+                succeeded++
+            } catch (e: Exception) {
+                todo.markFailed(task.cminmsgsId, e.message, now.plus(backoffDelayFor(task.attempts + 1)), now)
+                failed++
+            }
+        }
+        return SweepOutcome(due.size, succeeded, failed)
+    }
+}

src/main/kotlin/com/gzzn/omms/msgexchange/jobs/JobExecutor.kt

@@ -18,11 +18,13 @@ class JobExecutor(
     private val historySweep: HistorySweepJob,
     private val archive: ArchiveJob,
     private val projectionRebuild: ProjectionRebuildJob,
+    private val backfillSweep: BackfillSweepJob,
 ) {
     fun execute(job: PumpJobRepository.Job) = when (job.kind) {
         "HISTORY_SWEEP" -> historySweep.run()
         "ARCHIVE" -> archive.run()
         "PROJECTION_REBUILD" -> projectionRebuild.run()
+        "BACKFILL_SWEEP" -> backfillSweep.sweep()
         else -> error("unknown job ${job.kind}")
     }
 }

src/main/kotlin/com/gzzn/omms/msgexchange/processing/Pump.kt

@@ -8,6 +8,7 @@ import com.gzzn.omms.msgexchange.domain.MsgEvent
 import com.gzzn.omms.msgexchange.domain.ProcState
 import com.gzzn.omms.msgexchange.domain.ProcStatus
 import com.gzzn.omms.msgexchange.domain.Targets
+import com.gzzn.omms.msgexchange.infra.persistence.BackfillTodoRepository
 import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
 import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
 import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
@@ -121,6 +122,7 @@ class MessageProcessor(
     private val procFailure: ProcFailure,
     private val props: PipelineProps,
     private val txManager: PipelineTransactionManager,
+    private val backfillTodo: BackfillTodoRepository? = null,
 ) {
     private val log = org.slf4j.LoggerFactory.getLogger(MessageProcessor::class.java)
     private val flidRegex = Regex("<FLID>(.*?)</FLID>", RegexOption.IGNORE_CASE)
@@ -188,6 +190,8 @@ class MessageProcessor(
                 val owner = procState.ownerOfIdentity(identity) ?: -1L
                 log.info("duplicate-of:{} -> SKIPPED id={}", owner, head.cminmsgsId)
                 procState.update(head.cminmsgsId, ProcStatus.SKIPPED, lastError = "duplicate-of:$owner")
+                // 重复报文自身也是一条信箱记录：同样回填共享信箱，防止被反复轮询（失败落补偿）
+                compensateBackfill(head, decoded)
                 return
             }
         }
@@ -243,9 +247,33 @@ class MessageProcessor(
             inbox.backfillOnSuccess(head.cminmsgsId, decoded.meta.sndr, decoded.meta.type, decoded.meta.styp, decoded.meta.seqn)
         } catch (e: Exception) {
             log.error("backfill failed after SUCCEEDED id={} (compensation required)", head.cminmsgsId, e)
+            recordBackfillTodo(head.cminmsgsId, decoded, e)
         }
         log.info("SUCCEEDED id={} events={} flightChanges={}", head.cminmsgsId, events.size, decision.flightChanges.size)
     }
+
+    /** v2 §5：回填失败只落补偿待办（成功终态不降级，业务不重放）；由 BackfillSweepJob 到期重试。 */
+    private fun compensateBackfill(head: ProcState, decoded: DecodedMessage) {
+        try {
+            inbox.backfillOnSuccess(head.cminmsgsId, decoded.meta.sndr, decoded.meta.type, decoded.meta.styp, decoded.meta.seqn)
+        } catch (e: Exception) {
+            log.error("backfill failed for SKIPPED duplicate id={} (compensation required)", head.cminmsgsId, e)
+            recordBackfillTodo(head.cminmsgsId, decoded, e)
+        }
+    }
+
+    private fun recordBackfillTodo(cminmsgsId: Long, decoded: DecodedMessage, e: Exception) {
+        backfillTodo?.record(
+            BackfillTodoRepository.BackfillTask(
+                cminmsgsId = cminmsgsId,
+                sndr = decoded.meta.sndr,
+                type = decoded.meta.type,
+                styp = decoded.meta.styp,
+                seqn = decoded.meta.seqn,
+            ),
+            e.message ?: e.javaClass.simpleName,
+        ) ?: log.warn("no backfill-todo repository bound; compensation NOT persisted id={}", cminmsgsId)
+    }
 }
 
 /** 延迟装配占位（阶段 1 后续以 Micronaut Bean 替换直连构造）。 */

src/main/kotlin/com/gzzn/omms/msgexchange/processing/SnapshotFlow.kt

@@ -8,6 +8,7 @@ import com.gzzn.omms.msgexchange.domain.ProcStatus
 import com.gzzn.omms.msgexchange.domain.MsgEvent
 import com.gzzn.omms.msgexchange.domain.MsgKind
 import com.gzzn.omms.msgexchange.domain.Targets
+import com.gzzn.omms.msgexchange.infra.persistence.BackfillTodoRepository
 import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
 import com.gzzn.omms.msgexchange.infra.persistence.FlightFields
 import com.gzzn.omms.msgexchange.infra.persistence.FlightFieldsJson
@@ -33,6 +34,7 @@ class SnapshotFlow(
     private val procFailure: ProcFailure,
     private val txManager: PipelineTransactionManager,
     private val inbox: CminmsgInboxRepository,
+    private val backfillTodo: BackfillTodoRepository? = null,
 ) {
     private val log = org.slf4j.LoggerFactory.getLogger(SnapshotFlow::class.java)
 
@@ -139,6 +141,16 @@ class SnapshotFlow(
             inbox.backfillOnSuccess(head.cminmsgsId, msg.meta.sndr, msg.meta.type, msg.meta.styp, msg.meta.seqn)
         } catch (e: Exception) {
             log.error("backfill failed after SUCCEEDED id={} (compensation required)", head.cminmsgsId, e)
+            backfillTodo?.record(
+                BackfillTodoRepository.BackfillTask(
+                    cminmsgsId = head.cminmsgsId,
+                    sndr = msg.meta.sndr,
+                    type = msg.meta.type,
+                    styp = msg.meta.styp,
+                    seqn = msg.meta.seqn,
+                ),
+                e.message ?: e.javaClass.simpleName,
+            ) ?: log.warn("no backfill-todo repository bound; compensation NOT persisted id={}", head.cminmsgsId)
         }
     }
 

src/main/resources/db/migration/V1.3.2__backfill_todo.sql

@@ -0,0 +1,18 @@
+-- flight-state-design-v2 §5 (ACM2-29 P2-3): 提交后共享信箱回填的持久补偿待办。
+-- 业务事务已提交（PROC_STATE=SUCCEEDED）后回填失败 → 落本表重试；
+-- 回填失败不得把已成功的业务事务重新标记为失败，也不得重放业务变更。
+
+CREATE TABLE IF NOT EXISTS backfill_todo (
+  cminmsgs_id     BIGINT       PRIMARY KEY,
+  sndr            VARCHAR(64)  NOT NULL,
+  type            VARCHAR(32)  NOT NULL,
+  styp            VARCHAR(32),
+  seqn            BIGINT,
+  attempts        INTEGER      NOT NULL DEFAULT 0,
+  next_attempt_at TIMESTAMP(6) WITH TIME ZONE NOT NULL,
+  last_error      VARCHAR(512),
+  created_at      TIMESTAMP(6) WITH TIME ZONE NOT NULL,
+  updated_at      TIMESTAMP(6) WITH TIME ZONE NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_backfill_todo_due ON backfill_todo (next_attempt_at);