fix(pipeline): 修复取报-处理-回填链路的超时、吞吐、回填与 FIFO 问题

外部调用（ACM2-33） - 共享 MySQL 与自有 PG 全部补有界超时：驱动 connect/socket 超时 + Hikari 池超时（注意 Micronaut 的 Hikari 项是毫秒数，不是 Duration 字面量） - 删除主泵/毒丸路径的 inline 回填：跨库写不再占用 FIFO 关键路径，回填统一由扫描驱动 - 游标自愈：save() 改为「先 UPDATE、缺行 INSERT」，缺行只记一次 ERROR - Pump/Dispatcher 外层 catch 补 error 日志与失败计数投递吞吐（ACM2-34） - Dispatcher 改批量领取；队头退避未到期或首条发送失败即停止本轮（保持目标内保序） - 有活不再 sleep；删除不可达的 state==SENT 死条件；markSent 移入分支； superseded 清理加轮数上限与空 eventId 保护回填闭环（ACM2-36，V4） - MISSING（运行时确认行不存在）立即放弃自动重试并告警；暂时性故障达上限后停止自动重试 - 新增 reopen 人工恢复入口；放弃 ≠ 标记已确认（BACKFILL_AT 仍为空，清除前提不成立） - 扫描改 (BACKFILL_ATTEMPTS, MSG_ID) 公平轮转并排除放弃行，消除全局回填饥饿 - V4 只加字段与必要索引，不按年龄做任何存量推断切流播种（ACM2-35，V5） - cutover-watermark 四模式（min/zero/max/显式 ID），默认不播种、代码不做默认选择 - 升级实例拒绝重新播种（SEEDED_AT 为 NULL ≠ 从未消费）；播种与水位同语句落库 - 非法取值由启动自检挡下错误分类与入口契约（ACM2-37） - 未知 SCHD 子类型改为 UNSUPPORTED，不再静默当全量日计划合并 - ADFT 运营日冲突改走 ProtocolViolation → DEAD(PROTOCOL) - 兼容入口 receivedAt 缺失回退到注入 Clock；MessageLifecycleGate 强制注入 + 装配断言 - FIFO：主泵只领取 msgId ≤ W，兼容入口登记的行在水位追平前不被领取时间源与可观测（ACM2-38 / ACM2-41 阶段 0） - 仓储/处理器/作业全部经注入 Clock；移除 markTerminal/markBackfilled 的 Instant.now() 默认值 - 退避表档位与 max-attempts 对齐并加启动自检 - Micrometer 7 个 gauge（@Context 急切注册）+ /health 与 /metrics 共用积压快照缓存 - 只读迟到检测：监视被放行的空洞 ID 是否后来真的出现，只计数告警、不补入队 Plane: ACM2-33 ACM2-34 ACM2-35 ACM2-36 ACM2-37 ACM2-38 ACM2-41 Tests: 88 → 120（1 skipped 需真实 PG）
2026-09-11 08:08:35 +08:00
parent b65728f312
commit 1e75a81107
42 changed files with 1745 additions and 168 deletions
@@ -34,7 +34,8 @@ class BackfillService(
    private val mailboxProps: MailboxProps,
    private val props: PipelineProps,
    private val clock: Clock,
-    private val lifecycleGate: MessageLifecycleGate = MessageLifecycleGate(),
+    /** 与人工重放共用的互斥门；`internal` 以便装配测试断言两者拿到同一实例。 */
+    internal val lifecycleGate: MessageLifecycleGate,
 ) {
    private val log = org.slf4j.LoggerFactory.getLogger(BackfillService::class.java)

@@ -43,6 +44,12 @@ class BackfillService(
        private val MAX_BACKOFF: Duration = Duration.ofMinutes(15)
        private val TERMINAL_STATES = setOf(ProcStatus.SUCCEEDED, ProcStatus.SKIPPED, ProcStatus.DEAD)

+        /** 放弃原因：运行时查询确认信箱行不存在（确定性结论，重试不会改变结果）。 */
+        const val ABANDON_MISSING_ROW = "MISSING_ROW"
+
+        /** 放弃原因：暂时性故障达到尝试上限；停止自动重试，但保留人工恢复能力。 */
+        const val ABANDON_MAX_ATTEMPTS = "MAX_ATTEMPTS"
+
        fun backoffDelayFor(attempts: Int): Duration {
            val shift = (attempts - 1).coerceIn(0, 20)
            return INITIAL_BACKOFF.multipliedBy(1L shl shift).coerceAtMost(MAX_BACKOFF)
@@ -58,15 +65,27 @@ class BackfillService(
            val row = procState.find(msgId) ?: return@exclusive
            if (row.state !in TERMINAL_STATES) return@exclusive
            if (row.backfillAt != null) return@exclusive
+            if (row.backfillAbandonedAt != null) return@exclusive
            record(msgId, attempts = row.backfillAttempts, now = now)?.let {
                log.warn("backfill failed msgId={} error={} (sweep will retry)", msgId, it)
            }
        }
    }

+    /**
+     * 人工恢复入口：清除放弃标记并重排一次回填。
+     * 用于暂时性故障恢复之后，或人工对账确认该行仍应打标之后。
+     */
+    fun reopen(msgId: Long, now: Instant = clock.instant()): Boolean =
+        lifecycleGate.exclusive { procState.reopenBackfill(msgId, now) }
+
    /**
     * 批量补写，由 JobRunner 每 30 秒调用一次。待办状态都在数据库里，
     * 进程重启后接着跑，不需要额外恢复步骤。
+     *
+     * 注意：**调度周期不是完成时限**——扫描与历史作业串行，批次积压、单行调用超时与
+     * 历史作业耗时都会延长实际回填延迟。
+     *
     * @return 本批处理的条数
     */
    fun sweep(now: Instant = clock.instant()): Int {
@@ -75,19 +94,37 @@ class BackfillService(
        return due.size
    }

-    /** 回填一条。@return 失败原因；返回 null 表示标记已确认（包括"别的路径已经标过了"）。 */
+    /** 回填一条。@return 失败原因；返回 null 表示已处理完（写成功／早已标记／已放弃）。 */
    private fun record(msgId: Long, attempts: Int, now: Instant): String? =
        try {
-            // 没有真正写进去说明库里已经有标记了，同样算成功（不覆盖已有值）
-            val result = mailbox.markProcessedIfUnmarked(msgId, mailboxProps.processedValue)
-            check(result != MailboxMarkResult.MISSING) { "mailbox-row-missing" }
-            procState.markBackfilled(msgId, now)
-            null
+            when (mailbox.markProcessedIfUnmarked(msgId, mailboxProps.processedValue)) {
+                MailboxMarkResult.MARKED, MailboxMarkResult.ALREADY_MARKED -> {
+                    // 没有真正写进去说明库里已经有标记了，同样算成功（不覆盖已有值）
+                    procState.markBackfilled(msgId, now)
+                    null
+                }
+                MailboxMarkResult.MISSING -> {
+                    // 确定性结论：行不存在，重试不会改变结果。停止自动补偿并把事实留痕。
+                    // 这**不等于**标记已确认（BACKFILL_AT 仍为空），清除前提因此仍然不成立。
+                    procState.markBackfillAbandoned(msgId, ABANDON_MISSING_ROW, now)
+                    log.error("backfill abandoned: mailbox row missing msgId={}", msgId)
+                    null
+                }
+            }
        } catch (e: Exception) {
+            // 超时/连接失败是暂时性的，**不能**当作缺行证据：按退避重试。
+            // 达到上限后停止自动重试（保留人工恢复能力），避免永久占满扫描批次造成饥饿。
            val reason = e.message ?: e.javaClass.simpleName
-            runCatching {
-                procState.recordBackfillFailure(msgId, reason, attempts + 1, now.plus(backoffDelayFor(attempts + 1)), now)
-            }.onFailure { log.error("record backfill failure failed msgId={}", msgId, it) }
+            val nextAttempts = attempts + 1
+            if (nextAttempts >= props.pipeline.backfillMaxAttempts) {
+                runCatching { procState.markBackfillAbandoned(msgId, ABANDON_MAX_ATTEMPTS, now) }
+                    .onFailure { log.error("abandon backfill failed msgId={}", msgId, it) }
+                log.error("backfill abandoned after {} attempts msgId={} error={}", nextAttempts, msgId, reason)
+            } else {
+                runCatching {
+                    procState.recordBackfillFailure(msgId, reason, nextAttempts, now.plus(backoffDelayFor(nextAttempts)), now)
+                }.onFailure { log.error("record backfill failure failed msgId={}", msgId, it) }
+            }
            reason
        }

@@ -17,10 +17,12 @@ import com.gzzn.omms.msgexchange.domain.flight.MergeChange
 import com.gzzn.omms.msgexchange.domain.flight.ScheduleRecord
 import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository
 import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
+import com.gzzn.omms.msgexchange.infra.persistence.PersistOutcome
 import com.gzzn.omms.msgexchange.infra.persistence.PipelineLockRepository
 import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
 import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
 import jakarta.inject.Singleton
+import java.time.Clock
 import java.time.Instant
 import java.time.ZoneId

@@ -39,21 +41,22 @@ class FlopProcessor(
    private val msgEvents: MsgEventRepository,
    private val procState: ProcStateRepository,
    private val mapper: ObjectMapper,
+    private val clock: Clock,
 ) {
    fun apply(head: ProcState, msg: DecodedMessage, payload: FlopPayload): ApplyResult = txManager.inTransaction {
        lock.lock()
        val current = flightState.loadFullSnapshot(payload.flid)
        if (current == null) {
            // 迟到/未知航班：幂等成功，不创建（创建入口只有 SCHD/ADFT）；终态同事务落库
-            procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED)
+            procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
            return@inTransaction ApplyResult.Succeeded
        }

        val change = MergeChange(flid = payload.flid, scalars = payload.scalars, collections = payload.collections)
        val next = FlightStateEngine.mergedState(current, change)
-        flightState.persistFullState(next, msgId = head.msgId, now = Instant.now())
+        flightState.persistFullState(next, msgId = head.msgId, now = clock.instant())
        msgEvents.insertAll(eventsFor(next, mapper))
-        procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED)
+        procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
        ApplyResult.Succeeded
    }
 }
@@ -72,10 +75,11 @@ class FdelProcessor(
    private val msgEvents: MsgEventRepository,
    private val procState: ProcStateRepository,
    private val mapper: ObjectMapper,
+    private val clock: Clock,
 ) {
    fun apply(head: ProcState, msg: DecodedMessage, payload: FlopPayload): ApplyResult = txManager.inTransaction {
        lock.lock()
-        val deleted = flightState.markDeleted(payload.flid, msgId = head.msgId, now = Instant.now())
+        val deleted = flightState.markDeleted(payload.flid, msgId = head.msgId, now = clock.instant())
        if (deleted) {
            val current = flightState.loadFullSnapshot(payload.flid)
            // 只有"在用 → 删除"这一步才发删除通知，而且和状态变更写在同一个事务里
@@ -105,7 +109,7 @@ class FdelProcessor(
                ),
            )
        }
-        procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED)   // 没删到东西说明是迟到或重复报文，照样算成功
+        procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())   // 没删到东西说明是迟到或重复报文，照样算成功
        ApplyResult.Succeeded
    }
 }
@@ -126,6 +130,7 @@ class AdftProcessor(
    private val procState: ProcStateRepository,
    operationDayProps: OperationDayProps,
    private val mapper: ObjectMapper,
+    private val clock: Clock,
 ) {
    private val opDay = OperationDayCalculator(
        zone = runCatching { ZoneId.of(operationDayProps.zone) }.getOrElse { ZoneId.of("Asia/Shanghai") },
@@ -133,47 +138,57 @@ class AdftProcessor(
    )

    fun apply(head: ProcState, msg: DecodedMessage, record: ScheduleRecord): ApplyResult =
-        txManager.inTransaction {
-            lock.lock()
-            val main = flightState.findMainRow(record.flid)
-            if (main != null && main.state == FlightState.DELETED) {
-                // 已删除的航班重新激活：状态改回 ACTIVE、版本号加一，并登记状态事件
-                if (flightState.revive(record.flid, msgId = head.msgId, now = Instant.now())) {
-                    val current = flightState.loadFullSnapshot(record.flid)
-                    if (current != null) {
-                        val next = FlightStateEngine.mergedState(current, setOnly(record))
-                        flightState.persistFullState(next, msgId = head.msgId, now = Instant.now())
-                        msgEvents.insertAll(eventsFor(next, mapper))
+        try {
+            txManager.inTransaction {
+                lock.lock()
+                val main = flightState.findMainRow(record.flid)
+                if (main != null && main.state == FlightState.DELETED) {
+                    // 已删除的航班重新激活：状态改回 ACTIVE、版本号加一，并登记状态事件
+                    if (flightState.revive(record.flid, msgId = head.msgId, now = clock.instant())) {
+                        val current = flightState.loadFullSnapshot(record.flid)
+                        if (current != null) {
+                            val next = FlightStateEngine.mergedState(current, setOnly(record))
+                            flightState.persistFullState(next, msgId = head.msgId, now = clock.instant())
+                            msgEvents.insertAll(eventsFor(next, mapper))
+                        }
+                    } else {
+                        // 并发下可能已被别的消息处理掉：不改版本、不重复发事件，但留下痕迹便于对账。
+                        log.warn("adft revive no-op (already active or vanished) msgId={} flid={}", head.msgId, record.flid)
                    }
+                    procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
+                    return@inTransaction ApplyResult.Succeeded
                }
-                procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED)
-                return@inTransaction ApplyResult.Succeeded
-            }

-            val current = flightState.loadFullSnapshot(record.flid)
-            val next: FlightSnapshot = if (current == null) {
-                // 新建航班：带了计划时间就算出运营日，算不出来先留空，等日计划报文来收录
-                val day = opDay.compute(record.scalars["SODT"])
-                FlightSnapshot(
-                    flid = record.flid,
-                    operationDay = day,   // 算不出来就留空，不能默认拿收报当天顶上
-                    state = FlightState.ACTIVE,
-                    stateVersion = 1L,
-                    scalars = record.scalars,
-                    collections = FlightStateEngine.COLLECTION_KEYS.associateWith { key ->
-                        record.collections[key] ?: emptyList()
-                    },
-                )
-            } else {
-                FlightStateEngine.mergedState(current, setOnly(record))
+                val current = flightState.loadFullSnapshot(record.flid)
+                val next: FlightSnapshot = if (current == null) {
+                    // 新建航班：带了计划时间就算出运营日，算不出来先留空，等日计划报文来收录
+                    val day = opDay.compute(record.scalars["SODT"])
+                    FlightSnapshot(
+                        flid = record.flid,
+                        operationDay = day,   // 算不出来就留空，不能默认拿收报当天顶上
+                        state = FlightState.ACTIVE,
+                        stateVersion = 1L,
+                        scalars = record.scalars,
+                        collections = FlightStateEngine.COLLECTION_KEYS.associateWith { key ->
+                            record.collections[key] ?: emptyList()
+                        },
+                    )
+                } else {
+                    FlightStateEngine.mergedState(current, setOnly(record))
+                }
+                val outcome = flightState.persistFullState(next, msgId = head.msgId, now = clock.instant())
+                if (outcome == PersistOutcome.DAY_GUARD_VIOLATION) {
+                    // 运营日冲突是**协议级**问题：与 SCHD 同分类 —— 整笔回滚、不重试、交人工。
+                    // 若按 INFRA 抛出去，会被当成暂时性故障白白重试到耗尽，并给出误导的错误类别。
+                    throw ProtocolViolation("operation-day guard violated flid=${record.flid}")
+                }
+                msgEvents.insertAll(eventsFor(next, mapper))
+                procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
+                ApplyResult.Succeeded
            }
-            val outcome = flightState.persistFullState(next, msgId = head.msgId, now = Instant.now())
-            check(outcome != com.gzzn.omms.msgexchange.infra.persistence.PersistOutcome.DAY_GUARD_VIOLATION) {
-                "operation-day guard violated flid=${record.flid}"
-            }
-            msgEvents.insertAll(eventsFor(next, mapper))
-            procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED)
-            ApplyResult.Succeeded
+        } catch (e: ProtocolViolation) {
+            log.error("adft DEAD(PROTOCOL) msgId={} reason={}", head.msgId, e.message)
+            ApplyResult.DeadProtocol(e.message ?: "protocol-violation")
        }

    /**
@@ -185,6 +200,10 @@ class AdftProcessor(
        scalars = record.scalars,
        collections = record.collections,
    )
+
+    private companion object {
+        private val log = org.slf4j.LoggerFactory.getLogger(AdftProcessor::class.java)
+    }
 }

 // =====================================================================
@@ -11,12 +11,14 @@ import com.gzzn.omms.msgexchange.domain.ProcState
 import com.gzzn.omms.msgexchange.domain.ProcStatus
 import com.gzzn.omms.msgexchange.infra.log.TraceLog
 import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
+import com.gzzn.omms.msgexchange.infra.persistence.InboxCursorRepository
 import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
 import com.gzzn.omms.msgexchange.infra.retry.ProcFailure
 import jakarta.inject.Singleton
 import java.time.Clock
 import java.time.Duration
 import java.time.Instant
+import java.util.concurrent.atomic.AtomicLong

 /**
 * 处理主泵：一个线程按消息 ID 从小到大一条条处理，保证先来的先处理。
@@ -33,13 +35,16 @@ import java.time.Instant
@Singleton
 class Pump(
    private val procState: ProcStateRepository,
+    private val cursor: InboxCursorRepository,
    private val processor: MessageProcessor,
-    private val backfill: BackfillService,
    private val props: PipelineProps,
    private val clock: Clock,
 ) {
    private val log = org.slf4j.LoggerFactory.getLogger(Pump::class.java)

+    /** 连续失败计数：仅用于日志/排障，不代表业务状态。 */
+    private val tickFailures = AtomicLong(0)
+
    @Volatile
    private var running = true

@@ -52,11 +57,15 @@ class Pump(
        while (running) {
            try {
                tick()
+                tickFailures.set(0)
            } catch (e: InterruptedException) {
                Thread.currentThread().interrupt()
                return
            } catch (e: Exception) {
-                // 兜底：单条消息的失败状态已在 processOne 内记录，这里只避免线程退出
+                // 兜底：单条消息的失败状态已在 processOne 内记录；但 tick 整体失败（DB 断连、
+                // 锁超时）在别处没有痕迹，必须留日志，否则排障无据可查。
+                val failures = tickFailures.incrementAndGet()
+                log.error("pump tick failed (failure #{})", failures, e)
                sleepQuietly(props.pipeline.pollInterval)
            }
        }
@@ -64,10 +73,25 @@ class Pump(

    internal fun tick() {
        val head = procState.headUnfinished()
+        if (head == null) {
+            sleepQuietly(props.pipeline.pollInterval)
+            return
+        }
+        // 只领取"已被水位覆盖"的队头（`msgId <= W`）。
+        //
+        // 水位以内的行都是收报按 ID 顺序发现并登记的；水位之外的行只可能来自兼容入口
+        // 直接写 PROC_STATE（它不参与水位）。若允许领取，它就会越过那些尚未入队的较小 ID，
+        // 破坏 FIFO（缺口 G2）。这种行在空洞补齐、`W` 追平之后自然可领取。
+        val watermark = cursor.load().committedUpTo
+        if (head.msgId > watermark) {
+            warnBeyondWatermark(head.msgId, watermark)
+            sleepQuietly(props.pipeline.pollInterval)
+            return
+        }
        when {
-            head == null -> sleepQuietly(props.pipeline.pollInterval)
            head.state == ProcStatus.FAILED && poisoned(head) -> {
                log.error("poison -> DEAD msgId={} attempts={} lastError={}", head.msgId, head.attempts, head.lastError)
+                // markTerminal 在同一条 UPDATE 里登记回填意图；回填由扫描补写，不在这里做跨库写。
                procState.markTerminal(
                    head.msgId, ProcStatus.DEAD,
                    errorClass = ErrorClass.EXHAUSTED,
@@ -75,7 +99,6 @@ class Pump(
                    attempts = head.attempts,
                    now = clock.instant(),
                )
-                backfill.attempt(head.msgId)
            }
            head.state == ProcStatus.FAILED && (head.nextAttemptAt ?: Instant.EPOCH) > clock.instant() ->
                sleepQuietly(Duration.between(clock.instant(), head.nextAttemptAt))
@@ -90,6 +113,19 @@ class Pump(
    private fun poisoned(head: ProcState): Boolean =
        isHeadPoisoned(head, clock.instant(), props)

+    /** 上一次"队头在水位之外"告警时的水位值：只在它变化时告警，避免每秒刷屏。 */
+    private val warnedWatermark = AtomicLong(Long.MIN_VALUE)
+
+    private fun warnBeyondWatermark(msgId: Long, watermark: Long) {
+        if (warnedWatermark.getAndSet(watermark) != watermark) {
+            log.warn(
+                "head msgId={} is beyond watermark W={}; waiting for discovery " +
+                    "(row injected by the compat entry point?)",
+                msgId, watermark,
+            )
+        }
+    }
+
    private fun sleepQuietly(d: Duration) {
        if (!d.isNegative && !d.isZero) Thread.sleep(d.toMillis().coerceAtLeast(1))
    }
@@ -100,10 +136,11 @@ internal fun isHeadPoisoned(head: ProcState, now: Instant, props: PipelineProps)
        Duration.between(head.processingStartedAt ?: head.updatedAt, now) >= props.pipeline.headDeadline

 /**
- * 处理一条消息：读原文 → 解码 → 绑定业务身份 → 分派给对应处理器 → 提交后回填标记。
+ * 处理一条消息：读原文 → 解码 → 绑定业务身份 → 分派给对应处理器。
 *
- * 业务数据和终态由各处理器在自己的事务里写入。终态一旦落下（成功、跳过或死信），
- * 这里马上试一次把处理标记写回信箱；写不进去也没关系，回填扫描会按退避继续重试。
+ * 业务数据、终态与回填意图都由各处理器在自己的事务里写入（终态与回填意图是同一条 UPDATE）。
+ * **这里不做信箱回填**：主泵是 FIFO 关键路径，跨库写会把它绑在共享 MySQL 的可用性上。
+ * 回填由 `JobRunner` 定时的 `BackfillService.sweep` 驱动（调度周期不等于完成时限）。
 *
 * 任何意外异常都算在当前这条消息头上（记 FAILED(INFRA) 后重试），不会把主泵线程带崩。
 * 报文非法和整包协议拒绝不重试，直接进死信等人工处置。
@@ -118,14 +155,14 @@ class MessageProcessor(
    private val fdelProcessor: FdelProcessor,
    private val adftProcessor: AdftProcessor,
    private val procFailure: ProcFailure,
-    private val backfill: BackfillService,
    private val props: PipelineProps,
+    private val clock: Clock,
 ) {
    private val log = org.slf4j.LoggerFactory.getLogger(MessageProcessor::class.java)

    fun processOne(head: ProcState) {
        TraceLog.withTrace(head.msgId) {
-            val terminal = try {
+            try {
                processInternal(head)
            } catch (e: InterruptedException) {
                Thread.currentThread().interrupt()
@@ -135,13 +172,10 @@ class MessageProcessor(
                log.warn("processOne unexpected failure msgId={} ec=INFRA msg={}", head.msgId, e.message ?: e.javaClass.simpleName)
                procFailure.fail(head, ErrorClass.INFRA, e.message ?: e.javaClass.simpleName)
            }
-            // 终态已经写好了，马上试一次回填；写不进去也没关系，回填扫描会按退避继续重试
-            // （待办在写终态时就一起登记了）。还没处理完的消息不打标记。
-            if (terminal) backfill.attempt(head.msgId)
        }
    }

-    /** @return 这条消息是否已经落到终态（只有终态才允许回填信箱标记） */
+    /** @return 这条消息是否落到终态；回填由扫描驱动，调用方不再据此立即回填。 */
    private fun processInternal(head: ProcState): Boolean {
        // 守卫：手工/遗留 FAILED 行若 attempts 已达上限，直接终态（防止退避到期后无限重试）
        if (head.state == ProcStatus.FAILED && procFailure.scheduler.exhausted(head.attempts)) {
@@ -151,6 +185,7 @@ class MessageProcessor(
                errorClass = ErrorClass.EXHAUSTED,
                lastError = head.lastError ?: "max-attempts",
                attempts = head.attempts,
+                now = clock.instant(),
            )
            return true
        }
@@ -179,7 +214,7 @@ class MessageProcessor(
            if (!procState.tryBindIdentity(head.msgId, identity)) {
                val owner = procState.ownerOfIdentity(identity) ?: -1L
                log.info("duplicate-of:{} -> SKIPPED msgId={}", owner, head.msgId)
-                procState.markTerminal(head.msgId, ProcStatus.SKIPPED, lastError = "duplicate-of:$owner")
+                procState.markTerminal(head.msgId, ProcStatus.SKIPPED, lastError = "duplicate-of:$owner", now = clock.instant())
                return true
            }
        }
@@ -223,6 +258,7 @@ class MessageProcessor(
                head.msgId, ProcStatus.DEAD,
                errorClass = ErrorClass.PROTOCOL,
                lastError = result.reason.take(1000),
+                now = clock.instant(),
            )
            return true
        }
@@ -233,7 +269,7 @@ class MessageProcessor(
    }

    private fun deadMalformed(head: ProcState, detail: String): Boolean {
-        procState.markTerminal(head.msgId, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, lastError = detail)
+        procState.markTerminal(head.msgId, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, lastError = detail, now = clock.instant())
        return true
    }
 }
@@ -25,6 +25,7 @@ import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
 import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
 import com.gzzn.omms.msgexchange.infra.persistence.SnapshotLogRepository
 import jakarta.inject.Singleton
+import java.time.Clock
 import java.time.Instant
 import java.time.LocalDate
 import java.time.ZoneId
@@ -66,9 +67,11 @@ class ScheduleProcessor(
    private val snapshotLog: SnapshotLogRepository,
    operationDayProps: OperationDayProps,
    private val mapper: ObjectMapper,
+    private val clock: Clock,
 ) {
+    private val zone: ZoneId = runCatching { ZoneId.of(operationDayProps.zone) }.getOrElse { ZoneId.of("Asia/Shanghai") }
    private val opDay = OperationDayCalculator(
-        zone = runCatching { ZoneId.of(operationDayProps.zone) }.getOrElse { ZoneId.of("Asia/Shanghai") },
+        zone = zone,
        cutoffHour = operationDayProps.cutoffHour,
    )

@@ -98,7 +101,7 @@ class ScheduleProcessor(
            // 报文合法但没有记录：不写航班，但终态与回填意图仍在锁事务内一起提交
            txManager.inTransaction {
                lock.lock()
-                procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED)
+                procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
            }
            logSnapshot(head, body, SnapshotResult.COMMITTED, upserted = 0, setOf(SnapshotFlag.EMPTY), started)
            return ApplyResult.Succeeded
@@ -123,8 +126,10 @@ class ScheduleProcessor(

                var written = 0
                val events = mutableListOf<MsgEvent>()
+                // 一次建索引，避免在航班级循环里反复线性扫描（大日计划下是 O(N²)）。
+                val recordsByFlid = body.records.associateBy { it.flid }
                ok.perRecordDay.forEach { (flid, day) ->
-                    val record = body.records.first { it.flid == flid }
+                    val record = recordsByFlid.getValue(flid)
                    val existingMain = mains[flid]
                    val keepDeleted = existingMain?.state == FlightState.DELETED
                    if (keepDeleted) flags.add(SnapshotFlag.SCHD_REVIVE_CONFLICT)   // 日计划不会让已删除的航班复活
@@ -135,7 +140,7 @@ class ScheduleProcessor(
                        operationDay = day,
                        keepDeleted = keepDeleted,
                    )
-                    when (flightState.persistFullState(next, msgId = head.msgId, now = Instant.now())) {
+                    when (flightState.persistFullState(next, msgId = head.msgId, now = clock.instant())) {
                        PersistOutcome.DAY_GUARD_VIOLATION ->
                            throw ProtocolViolation("operation-day guard violated flid=$flid")
                        else -> written++
@@ -144,7 +149,7 @@ class ScheduleProcessor(
                }
                if (events.isNotEmpty()) msgEvents.insertAll(events)
                // 终态与回填待办跟业务数据同事务提交：要么全成，要么全回滚
-                procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED)
+                procState.markTerminal(head.msgId, ProcStatus.SUCCEEDED, now = clock.instant())
                written
            }
            logSnapshot(head, body, SnapshotResult.COMMITTED, upserted, flags, started)
@@ -192,9 +197,9 @@ class ScheduleProcessor(
            snapshotLog.append(
                SnapshotLogEntry(
                    msgId = head.msgId,
-                    recvAt = Instant.now(),
-                    scopeStart = days.minOrNull() ?: LocalDate.now(),
-                    scopeEnd = days.maxOrNull() ?: LocalDate.now(),
+                    recvAt = clock.instant(),
+                    scopeStart = days.minOrNull() ?: LocalDate.now(zone),
+                    scopeEnd = days.maxOrNull() ?: LocalDate.now(zone),
                    recs = body.records.size,
                    upserted = upserted,
                    durationMs = (System.nanoTime() - startedNanos) / 1_000_000,