fix(delivery): upsert latest schedule event

KAFKA:schd 收敛为每 FLID 单行：V9 先按 (STATE_VERSION DESC, EVENT_ID DESC) 收敛存量再建部分唯一索引 uq_schd_event；写入走带谓词的单行 upsert（只进不退、同版本 tombstone 优先、接受新代次即换 EVENT_ID 并重置 PENDING/清旧错误）；读取不再 DISTINCT ON；成功确认改按 (EVENT_ID, STATE_VERSION, PENDING) 条件更新，删除 Dispatcher superseded 清理循环。msg 路径不变。验证：全量 ./gradlew test 129 tests / 0 fail（新增 3 个用例：单行只进不退、同版本 tombstone 不可复活、发送期间新代次保持 PENDING）。V9 的存量收敛/索引与 upsert SQL 需真实 PG，本机无 Docker 时 FlywayMigrationTest 跳过。
2026-09-12 20:38:18 +08:00
parent 21739a439d
commit d21555091f
7 changed files with 238 additions and 67 deletions
@@ -31,8 +31,8 @@ interface DeliveryPort {
 /**
 * 投递调度：把 outbox（待发事件表）里的事件发给下游。
 *
- * KAFKA_MSG 一条一条按登记顺序发，不插队。KAFKA_SCHD 走 flushSchd 批量发：同一个 FLID 攒了
- * 多条未发事件时只发版本号最新的那条，旧的自然作废；删除通知发 value 为空的 tombstone。
+ * KAFKA_MSG 一条一条按登记顺序发，不插队。KAFKA_SCHD 走 flushSchd：outbox 每个 FLID 只保留
+ * 一行（写入侧单行 upsert），发出后按读取时刻的代次做条件确认；删除通知发 value 为空的 tombstone。
 * 两个主题之间不保证先后顺序。
 *
 * 失败处理：队首的重试时间没到就不取；一批里有发送失败，整批重试次数加一并推后退避，
@@ -138,7 +138,7 @@ class Dispatcher(
        }
    }

-    /** 批量发 KAFKA_SCHD：每个 FLID 只发版本号最新的那条未发事件，删除通知发空 value。 */
+    /** 发 KAFKA_SCHD：每个 FLID 只有一行待发事件，删除通知发空 value，成功后按代次条件确认。 */
    internal fun flushSchd() {
        val batch = try {
            msgEvents.mergePendingSchd(scheduler.now(), props.schd.flushLimit)
@@ -157,34 +157,12 @@ class Dispatcher(
                    EventType.TOMBSTONE -> port.sendKafkaNull("schd", e.partitionKey)
                    EventType.UPSERT -> port.sendKafkaSchd("schd", e.partitionKey, e.payloadJson)
                }
+                // 条件确认：读取时刻的代次（EVENT_ID + STATE_VERSION）被新写入覆盖时不标记，留待下一轮重发。
+                e.eventId?.let { msgEvents.markSentIfVersion(it, e.stateVersion) }
            } catch (ex: Exception) {
                failures.add(e)
            }
        }
-        val sentIds = batch.mapNotNull { it.eventId }.toSet() - failures.mapNotNull { it.eventId }.toSet()
-        if (sentIds.isNotEmpty()) msgEvents.markAllSent(sentIds.toList())
-        val sentVersions = batch.associate { it.partitionKey to it.stateVersion }
-        // 被更新版本压掉的旧事件也要标成已发，否则它们会一直留在队里：同一个 FLID 只按最新版本输出一次。
-        // 这里必须**有界**：若 markAllSent 因任何原因没有生效（例如行缺 eventId），
-        // 旧的无界 while(true) 会原地空转。
-        runCatching {
-            var rounds = 0
-            while (rounds < SUPERSEDED_CLEANUP_MAX_ROUNDS) {
-                rounds++
-                val superseded = msgEvents.mergePendingSchd(scheduler.now(), props.schd.flushLimit)
-                    .filter { sentVersions[it.partitionKey]?.let { v -> it.stateVersion < v } == true }
-                if (superseded.isEmpty()) break
-                val ids = superseded.mapNotNull { it.eventId }
-                if (ids.isEmpty()) {
-                    log.warn("superseded cleanup: {} rows without event_id, stop to avoid a spin", superseded.size)
-                    break
-                }
-                msgEvents.markAllSent(ids)
-                if (rounds == SUPERSEDED_CLEANUP_MAX_ROUNDS) {
-                    log.warn("superseded cleanup hit round cap ({}); remaining rows will be handled next flush", rounds)
-                }
-            }
-        }.onFailure { log.warn("superseded cleanup failed: {}", it.message) }
        failures.forEach { retryOrDead(it, it.lastError ?: "send-failed") }
        lastFlush = scheduler.now()
    }
@@ -203,9 +181,4 @@ class Dispatcher(
    private fun sleepQuietly(d: Duration) {
        if (!d.isNegative && !d.isZero) Thread.sleep(d.toMillis().coerceAtLeast(1))
    }
-
-    private companion object {
-        /** superseded 清理的轮数上限：只用于防止"标不掉又不报错"时的原地空转。 */
-        const val SUPERSEDED_CLEANUP_MAX_ROUNDS = 100
-    }
 }
@@ -167,11 +167,19 @@ interface MsgEventRepository {

    fun claimBatch(target: String, limit: Int): List<MsgEvent>

-    /** 挑出待发的整态事件，同一条航班只取版本最高的那一条（中间的版本不用发）。 */
+    /** 领取待发的整态事件：写入侧已保证每个 `FLID` 只有一行，读端不再去重合并。 */
    fun mergePendingSchd(now: Instant, limit: Int): List<MsgEvent>

    fun markSent(eventId: Long)

+    /**
+     * 条件确认：只在该行仍是本批读取到的代次（`EVENT_ID` + `STATE_VERSION`）且尚未发出时标记 `SENT`。
+     * 发送期间被新代次覆盖的行影响 0 行，留待下一轮重发。
+     *
+     * @return 受影响行数（0 或 1）
+     */
+    fun markSentIfVersion(eventId: Long, stateVersion: Long): Int
+
    fun markAllSent(eventIds: List<Long>)

    fun scheduleRetry(eventId: Long, nextAttemptAt: Instant, attempts: Int)
@@ -7,6 +7,7 @@ import com.gzzn.omms.msgexchange.domain.EventType
 import com.gzzn.omms.msgexchange.domain.MsgEvent
 import com.gzzn.omms.msgexchange.domain.ProcState
 import com.gzzn.omms.msgexchange.domain.ProcStatus
+import com.gzzn.omms.msgexchange.domain.Targets
 import com.gzzn.omms.msgexchange.domain.flight.FlightMainRow
 import com.gzzn.omms.msgexchange.domain.flight.FlightSnapshot
 import com.gzzn.omms.msgexchange.domain.flight.FlightState
@@ -433,27 +434,65 @@ class JdbcMsgEventRepository(
    private val ds: DataSource,
    private val clock: Clock,
 ) : MsgEventRepository {
+    /**
+     * 写待发事件。`KAFKA:schd` 走按 `FLID` 的单行 upsert（同一 `FLID` 只保留最新代次），
+     * 其他目标仍是一行一条的普通插入；返回实际生效的事件 ID（被版本守卫拒绝的 schd 写不返回值）。
+     */
    override fun insertAll(events: List<MsgEvent>): List<Long> =
-        events.map { e ->
-            ds.updateReturningLong(
-                """
-                INSERT INTO msg_event (target, partition_key, event_type, state_version, payload_json, state, attempts, next_attempt_at, created_at)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING event_id
-                """.trimIndent(),
-                { ps ->
-                    ps.setString(1, e.target)
-                    ps.setString(2, e.partitionKey)
-                    ps.setString(3, e.eventType.name)
-                    ps.setLong(4, e.stateVersion)
-                    ps.setString(5, e.payloadJson)
-                    ps.setString(6, e.state.name)
-                    ps.setInt(7, e.attempts)
-                    ps.setTimestamp(8, e.nextAttemptAt?.toSqlTimestamp())
-                    ps.setTimestamp(9, e.createdAt.toSqlTimestamp())
-                },
-            )
+        events.mapNotNull { e ->
+            if (e.target == Targets.KAFKA_SCHD) upsertSchd(e) else insertOne(e)
        }

+    private fun insertOne(e: MsgEvent): Long =
+        ds.updateReturningLong(
+            """
+            INSERT INTO msg_event (target, partition_key, event_type, state_version, payload_json, state, attempts, next_attempt_at, created_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING event_id
+            """.trimIndent(),
+            { ps -> bindEvent(ps, e) },
+        )
+
+    /**
+     * `KAFKA:schd` 单行 upsert：只进不退（仅更高 `STATE_VERSION` 覆盖），同版本只有 tombstone
+     * 能覆盖非 tombstone；接受的写代次复用本次 `EXCLUDED.EVENT_ID` 作行主键，并重置为 `PENDING`
+     * 且清空旧错误，因此发送期间被覆盖的旧批次条件确认只会影响 0 行（`design` 事件投递）。
+     */
+    private fun upsertSchd(e: MsgEvent): Long? =
+        ds.queryOne(
+            """
+            INSERT INTO msg_event (target, partition_key, event_type, state_version, payload_json, state, attempts, next_attempt_at, created_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT (target, partition_key) WHERE target = 'KAFKA:schd'
+            DO UPDATE SET
+                event_id = EXCLUDED.event_id,
+                event_type = EXCLUDED.event_type,
+                state_version = EXCLUDED.state_version,
+                payload_json = EXCLUDED.payload_json,
+                state = 'PENDING',
+                attempts = 0,
+                next_attempt_at = NULL,
+                error_class = NULL,
+                last_error = NULL,
+                created_at = EXCLUDED.created_at
+            WHERE EXCLUDED.state_version > msg_event.state_version
+               OR (EXCLUDED.state_version = msg_event.state_version AND EXCLUDED.event_type = 'TOMBSTONE')
+            RETURNING event_id
+            """.trimIndent(),
+            { ps -> bindEvent(ps, e) },
+        ) { rs -> rs.getLong("event_id") }
+
+    private fun bindEvent(ps: java.sql.PreparedStatement, e: MsgEvent) {
+        ps.setString(1, e.target)
+        ps.setString(2, e.partitionKey)
+        ps.setString(3, e.eventType.name)
+        ps.setLong(4, e.stateVersion)
+        ps.setString(5, e.payloadJson)
+        ps.setString(6, e.state.name)
+        ps.setInt(7, e.attempts)
+        ps.setTimestamp(8, e.nextAttemptAt?.toSqlTimestamp())
+        ps.setTimestamp(9, e.createdAt.toSqlTimestamp())
+    }
+
    override fun claimBatch(target: String, limit: Int): List<MsgEvent> =
        ds.query(
            "SELECT * FROM msg_event WHERE target = ? AND state = 'PENDING' ORDER BY event_id ASC LIMIT ?",
@@ -461,16 +500,16 @@ class JdbcMsgEventRepository(
            ::mapEvent,
        )

-    /** 同一条航班只取版本最高的待发整态事件；用了 PostgreSQL 的 DISTINCT ON 语法。 */
+    /**
+     * 领取待发的整态事件。写入侧已保证每个 `FLID` 只有一行（部分唯一索引 `uq_schd_event`），
+     * 读端不再做去重合并，只按退避与 `EVENT_ID` 顺序取。
+     */
    override fun mergePendingSchd(now: Instant, limit: Int): List<MsgEvent> =
        ds.query(
            """
-            SELECT * FROM (
-                SELECT DISTINCT ON (partition_key) *
-                FROM msg_event WHERE target = 'KAFKA:schd' AND state = 'PENDING'
-                ORDER BY partition_key, state_version DESC, event_id DESC
-            ) latest
-            WHERE next_attempt_at IS NULL OR next_attempt_at <= ?
+            SELECT * FROM msg_event
+            WHERE target = 'KAFKA:schd' AND state = 'PENDING'
+              AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
            ORDER BY event_id ASC LIMIT ?
            """.trimIndent(),
            { ps ->
@@ -484,6 +523,13 @@ class JdbcMsgEventRepository(
        ds.update("UPDATE msg_event SET state = 'SENT' WHERE event_id = ?", { ps -> ps.setLong(1, eventId) })
    }

+    /** 条件确认：仅当行仍是本批读到的代次且尚未发出时才标记，被新代次覆盖则影响 0 行。 */
+    override fun markSentIfVersion(eventId: Long, stateVersion: Long): Int =
+        ds.update(
+            "UPDATE msg_event SET state = 'SENT' WHERE event_id = ? AND state_version = ? AND state = 'PENDING'",
+            { ps -> ps.setLong(1, eventId); ps.setLong(2, stateVersion) },
+        )
+
    override fun markAllSent(eventIds: List<Long>) {
        if (eventIds.isEmpty()) return
        val placeholders = eventIds.joinToString(",") { "?" }
@@ -6,6 +6,7 @@ import com.gzzn.omms.msgexchange.domain.EventType
 import com.gzzn.omms.msgexchange.domain.MsgEvent
 import com.gzzn.omms.msgexchange.domain.ProcState
 import com.gzzn.omms.msgexchange.domain.ProcStatus
+import com.gzzn.omms.msgexchange.domain.Targets
 import com.gzzn.omms.msgexchange.domain.flight.FlightMainRow
 import com.gzzn.omms.msgexchange.domain.flight.FlightSnapshot
 import com.gzzn.omms.msgexchange.domain.flight.FlightState
@@ -233,20 +234,41 @@ class StubMsgEvents : MsgEventRepository {

    override fun insertAll(events: List<MsgEvent>): List<Long> =
        events.map { e ->
-            val id = ids.incrementAndGet()
-            rows[id] = e.copy(eventId = id)
-            id
+            if (e.target == Targets.KAFKA_SCHD) upsertSchd(e) else insertOne(e)
        }

+    private fun insertOne(e: MsgEvent): Long {
+        val id = ids.incrementAndGet()
+        rows[id] = e.copy(eventId = id)
+        return id
+    }
+
+    /**
+     * 与 JDBC upsert 同口径的单行投影：只进不退；同版本只有 tombstone 能覆盖非 tombstone；
+     * 接受的写代次换新 `EVENT_ID` 并重置 `PENDING`（清空 attempts/next/error）。
+     */
+    private fun upsertSchd(e: MsgEvent): Long {
+        val existing = rows.values.firstOrNull { it.target == Targets.KAFKA_SCHD && it.partitionKey == e.partitionKey }
+        val accepted = existing == null ||
+            e.stateVersion > existing.stateVersion ||
+            (e.stateVersion == existing.stateVersion && e.eventType == EventType.TOMBSTONE && existing.eventType != EventType.TOMBSTONE)
+        if (!accepted) return existing!!.eventId!!
+        existing?.let { rows.remove(it.eventId!!) }
+        val id = ids.incrementAndGet()
+        rows[id] = e.copy(
+            eventId = id, state = EventStatus.PENDING, attempts = 0, nextAttemptAt = null,
+            errorClass = null, lastError = null,
+        )
+        return id
+    }
+
    override fun claimBatch(target: String, limit: Int): List<MsgEvent> =
        rows.values.filter { it.target == target && it.state == EventStatus.PENDING }.sortedBy { it.eventId!! }.take(limit)

-    /** 同一条航班只取版本最高的待发整态事件，按事件先后输出。 */
+    /** 写入侧已保证每个 FLID 单行，按退避与事件先后输出。 */
    override fun mergePendingSchd(now: Instant, limit: Int): List<MsgEvent> =
        rows.values
-            .filter { it.target == "KAFKA:schd" && it.state == EventStatus.PENDING }
-            .groupBy { it.partitionKey }
-            .map { (_, group) -> group.maxBy { it.stateVersion } }
+            .filter { it.target == Targets.KAFKA_SCHD && it.state == EventStatus.PENDING }
            .filter { it.nextAttemptAt == null || it.nextAttemptAt <= now }
            .sortedBy { it.eventId!! }
            .take(limit)
@@ -255,6 +277,13 @@ class StubMsgEvents : MsgEventRepository {
        rows[eventId] = (rows[eventId] ?: return).copy(state = EventStatus.SENT)
    }

+    override fun markSentIfVersion(eventId: Long, stateVersion: Long): Int {
+        val row = rows[eventId] ?: return 0
+        if (row.state != EventStatus.PENDING || row.stateVersion != stateVersion) return 0
+        rows[eventId] = row.copy(state = EventStatus.SENT)
+        return 1
+    }
+
    override fun markAllSent(eventIds: List<Long>) {
        eventIds.forEach { markSent(it) }
    }
@@ -0,0 +1,24 @@
+-- =====================================================================
+-- V9：KAFKA:schd outbox 单行化（按 FLID）
+-- ---------------------------------------------------------------------
+-- 只动自有 PostgreSQL；共享 MySQL 不建表、不改结构。
+--
+-- design「事件投递」：`KAFKA:schd` 只提供最新状态，outbox 按 `FLID` 单行 upsert。
+-- 存量是按版本追加的多行，必须先收敛再建部分唯一约束，否则升级直接失败：
+-- 每个 (target='KAFKA:schd', partition_key) 留下 `STATE_VERSION DESC, EVENT_ID DESC`
+-- 的第一行，与运行时的「只进不退」规则一致；`KAFKA:msg` 仍是多行 append-log，不受约束。
+-- =====================================================================
+
+DELETE FROM msg_event a
+ USING msg_event b
+ WHERE a.target = 'KAFKA:schd'
+   AND b.target = 'KAFKA:schd'
+   AND a.partition_key = b.partition_key
+   AND (
+        a.state_version < b.state_version
+        OR (a.state_version = b.state_version AND a.event_id < b.event_id)
+   );
+
+CREATE UNIQUE INDEX uq_schd_event
+    ON msg_event (target, partition_key)
+ WHERE target = 'KAFKA:schd';