feat(jobs): 提交后回填补偿落库 backfill_todo 并到期重试 (ACM2-29 P2-3)

- V1.3.2 迁移：backfill_todo 持久补偿表（attempts/next_attempt_at/last_error） - BackfillTodoRepository 接口 + PG 实现与 stub；record 幂等 upsert - BackfillSweepJob：到期重试，成功即删，失败指数退避（30s 起步封顶 15 分钟）；触发点 = InboxPoller 每轮心跳（重启即对账）+ JobExecutor BACKFILL_SWEEP 类型 - MessageProcessor/SnapshotFlow 回填失败落待办（成功终态不降级，业务不重放）； SKIPPED 重复报文同样补偿共享信箱回填，防止重复信件被反复轮询 - 新测试：扫描重试/退避/恢复对账 4 用例；快照回填失败落库断言；跳过路径补偿落库断言验证：MSGX_PG_PORT=5433 真实 PG 全量 102 用例 0 失败 0 跳过 (ACM2-29)
2026-09-08 13:04:43 +08:00
parent 44fbac0d8b
commit 48a27b26dd
12 changed files with 402 additions and 3 deletions
@@ -162,6 +162,37 @@ interface PipelineTransactionManager {
    fun <T> inTransaction(block: () -> T): T
 }

+/**
+ * v2 §5（ACM2-29 P2-3）：提交后共享信箱回填的持久补偿待办。
+ * 业务事务已提交（PROC_STATE=SUCCEEDED）后 backfillOnSuccess 失败 → 落本表重试；
+ * 回填失败不得把已成功的业务事务重新标记为失败，也不得重放业务变更。
+ */
+interface BackfillTodoRepository {
+    data class BackfillTask(
+        val cminmsgsId: Long,
+        val sndr: String,
+        val type: String,
+        val styp: String,
+        val seqn: Long,
+        val attempts: Int = 0,
+    )
+
+    /** 失败即落库（幂等 upsert，同 ID 重复失败只刷新错误与重试时间）。 */
+    fun record(task: BackfillTask, lastError: String?, now: Instant = Instant.now())
+
+    /** 到期待重试的补偿任务（next_attempt_at <= now，按到期时间升序）。 */
+    fun findDue(now: Instant = Instant.now(), limit: Int = 50): List<BackfillTask>
+
+    /** 重试失败：累加 attempts 并按退避推后 next_attempt_at。 */
+    fun markFailed(cminmsgsId: Long, lastError: String?, nextAttemptAt: Instant, now: Instant = Instant.now())
+
+    /** 重试成功：移除补偿待办。 */
+    fun delete(cminmsgsId: Long)
+
+    /** 当前待办数量（测试/对账用）。 */
+    fun count(): Int
+}
+
@Deprecated("Replaced by FlightSchdRepository in ACM2-28", ReplaceWith("FlightSchdRepository"))
 interface RefDataRepository {
    data class GenMeta(val flids: List<String>, val version: Long)
@@ -9,6 +9,7 @@ import com.gzzn.omms.msgexchange.domain.MsgEvent
 import com.gzzn.omms.msgexchange.domain.ProcState
 import com.gzzn.omms.msgexchange.domain.ProcStatus
 import com.gzzn.omms.msgexchange.domain.RefUpsert
+import com.gzzn.omms.msgexchange.infra.persistence.BackfillTodoRepository
 import com.gzzn.omms.msgexchange.infra.persistence.FlightFields
 import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository
 import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository
@@ -311,6 +312,78 @@ class JdbcPipelineTransactionManager(
    }
 }

+/** v2 §5（ACM2-29 P2-3）：提交后共享信箱回填补偿待办的 PG 实现。 */
+@Singleton
+@Requires(property = "msgx.stubs", notEquals = "true")
+@Requires(property = "datasources.default.enabled", value = "true")
+class JdbcBackfillTodoRepository(
+    private val ds: DataSource,
+) : BackfillTodoRepository {
+
+    override fun record(task: BackfillTodoRepository.BackfillTask, lastError: String?, now: Instant) {
+        val ts = now.toSqlTimestamp()
+        ds.update(
+            """
+            INSERT INTO backfill_todo (cminmsgs_id, sndr, type, styp, seqn, attempts, next_attempt_at, last_error, created_at, updated_at)
+            VALUES (?, ?, ?, ?, ?, 0, ?, ?, ?, ?)
+            ON CONFLICT (cminmsgs_id) DO UPDATE SET
+              last_error = EXCLUDED.last_error,
+              next_attempt_at = EXCLUDED.next_attempt_at,
+              updated_at = EXCLUDED.updated_at
+            """.trimIndent(),
+        ) { ps ->
+            ps.setLong(1, task.cminmsgsId)
+            ps.setString(2, task.sndr)
+            ps.setString(3, task.type)
+            ps.setString(4, task.styp)
+            ps.setLong(5, task.seqn)
+            ps.setTimestamp(6, ts)
+            ps.setString(7, lastError?.take(512))
+            ps.setTimestamp(8, ts)
+            ps.setTimestamp(9, ts)
+        }
+    }
+
+    override fun findDue(now: Instant, limit: Int): List<BackfillTodoRepository.BackfillTask> =
+        ds.query(
+            """
+            SELECT cminmsgs_id, sndr, type, styp, seqn, attempts FROM backfill_todo
+            WHERE next_attempt_at <= ? ORDER BY next_attempt_at ASC LIMIT ?
+            """.trimIndent(),
+            { ps ->
+                ps.setTimestamp(1, now.toSqlTimestamp())
+                ps.setInt(2, limit)
+            },
+        ) { rs ->
+            BackfillTodoRepository.BackfillTask(
+                cminmsgsId = rs.getLong("cminmsgs_id"),
+                sndr = rs.getString("sndr"),
+                type = rs.getString("type"),
+                styp = rs.getString("styp") ?: "",
+                seqn = rs.getLong("seqn"),
+                attempts = rs.getInt("attempts"),
+            )
+        }
+
+    override fun markFailed(cminmsgsId: Long, lastError: String?, nextAttemptAt: Instant, now: Instant) {
+        ds.update(
+            "UPDATE backfill_todo SET attempts = attempts + 1, next_attempt_at = ?, last_error = ?, updated_at = ? WHERE cminmsgs_id = ?",
+        ) { ps ->
+            ps.setTimestamp(1, nextAttemptAt.toSqlTimestamp())
+            ps.setString(2, lastError?.take(512))
+            ps.setTimestamp(3, now.toSqlTimestamp())
+            ps.setLong(4, cminmsgsId)
+        }
+    }
+
+    override fun delete(cminmsgsId: Long) {
+        ds.update("DELETE FROM backfill_todo WHERE cminmsgs_id = ?") { ps -> ps.setLong(1, cminmsgsId) }
+    }
+
+    override fun count(): Int =
+        ds.queryOne("SELECT COUNT(*) AS c FROM backfill_todo", {}) { rs -> rs.getInt("c") } ?: 0
+}
+
@Singleton
@Requires(property = "msgx.stubs", notEquals = "true")
@Requires(property = "datasources.default.enabled", value = "true")
@@ -6,6 +6,7 @@ import com.gzzn.omms.msgexchange.domain.MsgEvent
 import com.gzzn.omms.msgexchange.domain.ProcState
 import com.gzzn.omms.msgexchange.domain.ProcStatus
 import com.gzzn.omms.msgexchange.domain.RefUpsert
+import com.gzzn.omms.msgexchange.infra.persistence.BackfillTodoRepository
 import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
 import com.gzzn.omms.msgexchange.infra.persistence.FlightFields
 import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository
@@ -435,3 +436,49 @@ class StubFlightState(
    override fun findByDay(day: String): List<Pair<String, FlightFields>> =
        flightSchd.findByDay(day)
 }
+
+@Requires(property = "msgx.stubs", value = "true")
+@Singleton
+class StubBackfillTodo : BackfillTodoRepository {
+    private data class Row(val task: BackfillTodoRepository.BackfillTask, var nextAttemptAt: Instant)
+
+    private val rows = linkedMapOf<Long, Row>()
+    private val errors = linkedMapOf<Long, String?>()
+
+    @Synchronized
+    fun clear() {
+        rows.clear()
+        errors.clear()
+    }
+
+    @Synchronized
+    fun tasks(): List<BackfillTodoRepository.BackfillTask> = rows.values.map { it.task }
+
+    @Synchronized
+    fun lastErrorOf(cminmsgsId: Long): String? = errors[cminmsgsId]
+
+    @Synchronized
+    override fun record(task: BackfillTodoRepository.BackfillTask, lastError: String?, now: Instant) {
+        rows[task.cminmsgsId] = Row(task, now)
+        errors[task.cminmsgsId] = lastError
+    }
+
+    @Synchronized
+    override fun findDue(now: Instant, limit: Int): List<BackfillTodoRepository.BackfillTask> =
+        rows.values.filter { it.nextAttemptAt <= now }.map { it.task }.take(limit)
+
+    @Synchronized
+    override fun markFailed(cminmsgsId: Long, lastError: String?, nextAttemptAt: Instant, now: Instant) {
+        rows[cminmsgsId]?.let { rows[cminmsgsId] = it.copy(task = it.task.copy(attempts = it.task.attempts + 1), nextAttemptAt = nextAttemptAt) }
+        errors[cminmsgsId] = lastError
+    }
+
+    @Synchronized
+    override fun delete(cminmsgsId: Long) {
+        rows.remove(cminmsgsId)
+        errors.remove(cminmsgsId)
+    }
+
+    @Synchronized
+    override fun count(): Int = rows.size
+}