fix(jobs): 留痕清理独立接通历史存储开关;精简死逻辑并对齐陈旧注释

This commit is contained in:
windyboy
2026-09-11 14:26:44 +08:00
parent d909a88167
commit a84751bf03
14 changed files with 75 additions and 72 deletions
@@ -38,11 +38,11 @@ class PipelineProps {
var maxCommitDelay: Duration = Duration.ofMinutes(5)
/**
* 超期补写期限:一条消息处理完之后,如果过了这么久还是没能把处理标记写回信箱
* (比如回填一直失败),就直接强制补写一次,不再等退避
* 超期补写期限 R:一条消息到达终态后,过了这么久处理标记仍未写回信箱
* (比如回填一直失败),扫描谓词的超期分支成立,无视退避强制补写(只会提前、从不推迟打标)
*
* 这是保证库方能清理信箱的兜底期限,必须覆盖人工重放所需的保留期,
* 确认之前不要调小,否则还在重放窗口内的消息会先被库方清掉
* 唯一约束是 R ≤ R_keep,**不保护重放窗口**(打标时刻与 R 解耦,完整论证见
* message-lifecycle.md §5.2)。取值待 Q6 定案,确认之前不要为提速下调
*/
var overdueBackfill: Duration = Duration.ofDays(30)
@@ -104,7 +104,7 @@ class PipelineProps {
* 配置自检:退避表档位数必须等于 `max-attempts 1`。
*
* 因为 attempts 一达到 `max-attempts` 就转 `DEAD`、不再计算下次重试,能真正用到的档位
* 只有 `max-attempts 1` 个。表更长会有一段**永远走不到**(默认配置里的 16 秒档就是这样)
* 只有 `max-attempts 1` 个。表更长会有一段**永远走不到**,
* 表更短则会提前封顶。两种错位都应该在启动时暴露,而不是静默生效。
*/
fun validate() {
@@ -160,8 +160,6 @@ data class Backlog(
interface MsgEventRepository {
fun insertAll(events: List<MsgEvent>): List<Long>
fun headUnsent(target: String): MsgEvent?
fun claimBatch(target: String, limit: Int): List<MsgEvent>
/** 挑出待发的整态事件,同一条航班只取版本最高的那一条(中间的版本不用发)。 */
@@ -227,9 +225,6 @@ interface FlightStateRepository {
* 历史存储没接通时,调用方必须传空集合,也就是一条都不删。
*/
fun purgeArchived(flids: Collection<String>): Int
/** 运营日还没定下来的航班有多少条(观测用;这些航班暂时不参与清理)。 */
fun countOperationDayNull(): Int
}
/** 日计划处理留痕:只追加、不参与业务判断,一次处理(含重放)记一行,写失败不影响业务。 */
@@ -0,0 +1,12 @@
package com.gzzn.omms.msgexchange.infra.persistence
import java.time.Instant
/**
* 留痕清理端口:物理删除指定时刻之前的 `SCHD_SNAP_LOG` 行。
*
* 只依赖自有 PG,不依赖历史存储开关——留痕是本地可重建记录,清理不需要外部归档确认。
*/
fun interface SnapshotLogPurge {
fun purgeBefore(instant: Instant): Int
}
@@ -23,6 +23,7 @@ import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import com.gzzn.omms.msgexchange.infra.persistence.ReqTrackRepository
import com.gzzn.omms.msgexchange.domain.SnapshotLogEntry
import com.gzzn.omms.msgexchange.infra.persistence.SnapshotLogPurge
import com.gzzn.omms.msgexchange.infra.persistence.SnapshotLogRepository
import io.micronaut.context.annotation.Requires
import jakarta.inject.Singleton
@@ -450,13 +451,6 @@ class JdbcMsgEventRepository(
)
}
override fun headUnsent(target: String): MsgEvent? =
ds.queryOne(
"SELECT * FROM msg_event WHERE target = ? AND state = 'PENDING' ORDER BY event_id ASC LIMIT 1",
{ ps -> ps.setString(1, target) },
::mapEvent,
)
override fun claimBatch(target: String, limit: Int): List<MsgEvent> =
ds.query(
"SELECT * FROM msg_event WHERE target = ? AND state = 'PENDING' ORDER BY event_id ASC LIMIT ?",
@@ -682,9 +676,6 @@ class JdbcFlightStateRepository(
return purged
}
override fun countOperationDayNull(): Int =
ds.queryOne("SELECT count(*) AS n FROM flight_schd WHERE operation_day IS NULL", {}) { rs -> rs.getInt("n") } ?: 0
// ---- 内部实现 ----
private fun bindMain(
@@ -825,7 +816,7 @@ class JdbcFlightStateRepository(
class JdbcSnapshotLogRepository(
private val ds: DataSource,
private val clock: Clock,
) : SnapshotLogRepository {
) : SnapshotLogRepository, SnapshotLogPurge {
/** 只追加写;这里不抛异常,写失败由调用方捕获并记为指标。 */
override fun append(entry: SnapshotLogEntry) {
ds.update(
@@ -848,6 +839,17 @@ class JdbcSnapshotLogRepository(
},
)
}
/** 留痕清理:收信时间与覆盖窗尾都早于分界点的行物理删除(走 idx_snaplog_cleanup)。 */
override fun purgeBefore(instant: Instant): Int =
ds.update(
"DELETE FROM schd_snap_log WHERE scope_end < ? AND recv_at < ?",
{ ps ->
val cutoffDay = java.time.LocalDate.ofInstant(instant, java.time.ZoneId.of("Asia/Shanghai"))
ps.setDate(1, java.sql.Date.valueOf(cutoffDay))
ps.setTimestamp(2, instant.toSqlTimestamp())
},
)
}
@Singleton
@@ -20,8 +20,7 @@ class ProcFailure(
private val procState: ProcStateRepository,
val scheduler: FailureScheduler,
) {
/** @return 是否已经落到终态:DEAD 是终态,FAILED 还会再试 */
fun fail(head: ProcState, ec: ErrorClass, reason: String): Boolean {
fun fail(head: ProcState, ec: ErrorClass, reason: String) {
val attempts = head.attempts + 1
if (scheduler.exhausted(attempts)) {
procState.markTerminal(
@@ -31,7 +30,6 @@ class ProcFailure(
lastError = "$reason; attempts=$attempts",
now = scheduler.now(),
)
return true
}
procState.update(
head.msgId, ProcStatus.FAILED,
@@ -40,6 +38,5 @@ class ProcFailure(
errorClass = ec,
lastError = reason,
)
return false
}
}
@@ -19,6 +19,7 @@ import com.gzzn.omms.msgexchange.infra.persistence.MailboxRow
import com.gzzn.omms.msgexchange.infra.persistence.MailboxMarkResult
import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
import com.gzzn.omms.msgexchange.infra.persistence.SnapshotLogPurge
import com.gzzn.omms.msgexchange.infra.persistence.PersistOutcome
import com.gzzn.omms.msgexchange.infra.persistence.PipelineLockRepository
import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
@@ -233,9 +234,6 @@ class StubMsgEvents : MsgEventRepository {
id
}
override fun headUnsent(target: String): MsgEvent? =
rows.values.filter { it.target == target && it.state == EventStatus.PENDING }.minByOrNull { it.eventId!! }
override fun claimBatch(target: String, limit: Int): List<MsgEvent> =
rows.values.filter { it.target == target && it.state == EventStatus.PENDING }.sortedBy { it.eventId!! }.take(limit)
@@ -344,12 +342,11 @@ class StubFlightState : FlightStateRepository {
return n
}
override fun countOperationDayNull(): Int = mains.values.count { it.operationDay == null }
}
@Singleton
@Requires(property = "msgx.stubs", value = "true")
class StubSnapshotLog : SnapshotLogRepository {
class StubSnapshotLog : SnapshotLogRepository, SnapshotLogPurge {
val entries = mutableListOf<SnapshotLogEntry>()
fun clear() = entries.clear()
@@ -357,6 +354,12 @@ class StubSnapshotLog : SnapshotLogRepository {
override fun append(entry: SnapshotLogEntry) {
entries.add(entry)
}
override fun purgeBefore(instant: Instant): Int {
val before = entries.size
entries.removeAll { it.recvAt < instant }
return before - entries.size
}
}
@Singleton
@@ -13,8 +13,8 @@ import java.time.Instant
* 两步不在同一个事务里(跨库没有事务)。如果信箱写成功、PG 入队失败,原文仍然在信箱里,
* 收报轮询会按 ID 把它补进来,所以不会丢消息。
*
* 【缺口】这里**不参与水位**:它直接写 PROC_STATE因此当水位卡在低位空洞时
* 这条高 ID 会被主泵提前领取,破坏 FIFO。修复方向见 ACM2-37 / message-lifecycle.md §5.1。
* 该入口**不参与水位**:它直接写 PROC_STATE登记的行可能超出水位;主泵只领 `msgId ≤ W`
* 因此不破坏 FIFO——这类行等水位追平后按序自然领取(message-lifecycle.md §5.1
*/
@Singleton
class InboxService(
@@ -8,6 +8,7 @@ import com.gzzn.omms.msgexchange.domain.flight.HistoryCandidate
import com.gzzn.omms.msgexchange.domain.flight.HistoryRules
import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
import com.gzzn.omms.msgexchange.infra.persistence.SnapshotLogPurge
import jakarta.inject.Singleton
import java.time.Duration
import java.time.Instant
@@ -32,7 +33,7 @@ class HistorySweepJob(
private val props: HistoryProps,
/** 历史存储端口:返回确认归档成功的 FLID 集合;部署侧没接通时是 null。 */
private val historyStore: HistoryStore? = null,
/** 留痕清理端口:删掉超过保留期(默认 90 天)的日计划留痕行;没接通时 null。 */
/** 留痕清理端口:删掉超过保留期(默认 90 天)的日计划留痕行;不依赖历史存储开关,没接通时 null。 */
private val snapLogPurge: SnapshotLogPurge? = null,
) {
/** 归档到历史存储的接口,由部署侧适配具体存储;默认不接通。 */
@@ -41,25 +42,24 @@ class HistorySweepJob(
fun archive(candidates: List<HistoryCandidate>): Set<String>
}
fun interface SnapshotLogPurge {
fun purgeBefore(instant: Instant): Int
}
data class SweepOutcome(val selected: Int, val archived: Int, val purged: Int, val snapLogPurged: Int = 0)
fun run(now: Instant = Instant.now()): SweepOutcome {
// 留痕是本地可重建记录,清理不需要外部归档确认,不受历史存储开关牵制。
val snapLogPurged = snapLogPurge?.purgeBefore(now.minus(Duration.ofDays(props.snapLogRetentionDays))) ?: 0
if (!props.historyStoreEnabled || historyStore == null) {
// 红线:历史存储没接通就一条都不删。先删当前态、事后再补历史是不允许的。
return SweepOutcome(selected = 0, archived = 0, purged = 0)
// 红线:历史存储没接通就一条航班都不删。先删当前态、事后再补历史是不允许的。
return SweepOutcome(selected = 0, archived = 0, purged = 0, snapLogPurged = snapLogPurged)
}
val rules = HistoryRules(props.cancelledHours, props.terminalHours, props.deletedHours, props.idleHours)
val zone = ZoneId.of("Asia/Shanghai") // 保留期窗口按机场时区算,不用 UTC
val candidates = flightState.findHistoryCandidates(rules, zone, now)
if (candidates.isEmpty()) return SweepOutcome(0, 0, 0)
if (candidates.isEmpty()) return SweepOutcome(0, 0, 0, snapLogPurged = snapLogPurged)
val archivedFlids = historyStore.archive(candidates)
if (archivedFlids.isEmpty()) return SweepOutcome(candidates.size, archived = 0, purged = 0)
if (archivedFlids.isEmpty()) return SweepOutcome(candidates.size, archived = 0, purged = 0, snapLogPurged = snapLogPurged)
val toPurge = candidates.filter { it.flid in archivedFlids }
// 从没收到过 FDEL、被这里直接清掉的航班,删除前补发一次通知,否则下游一直以为它还在
@@ -68,8 +68,6 @@ class HistorySweepJob(
msgEvents.insertAll(preDelete.map { tombstone(it) })
}
val purged = flightState.purgeArchived(archivedFlids)
val snapLogPurged = snapLogPurge?.purgeBefore(now.minus(Duration.ofDays(props.snapLogRetentionDays))) ?: 0
return SweepOutcome(candidates.size, archivedFlids.size, purged, snapLogPurged)
}
@@ -1,7 +1,5 @@
package com.gzzn.omms.msgexchange.jobs
import com.gzzn.omms.msgexchange.config.HistoryProps
import com.gzzn.omms.msgexchange.config.PipelineProps
import com.gzzn.omms.msgexchange.processing.BackfillService
import jakarta.inject.Singleton
import java.time.Clock
@@ -23,8 +21,6 @@ import java.time.ZoneId
class JobRunner(
private val backfill: BackfillService,
private val historySweep: HistorySweepJob,
@Suppress("unused") private val pipelineProps: PipelineProps,
@Suppress("unused") private val historyProps: HistoryProps,
private val clock: Clock,
) {
private val log = org.slf4j.LoggerFactory.getLogger(JobRunner::class.java)
@@ -18,8 +18,7 @@ import java.time.Instant
* 同时留下"还欠一次回填"的意图(PROC_STATE 上的 BACKFILL_NEXT_AT);
* 这个类负责第二件:
*
* - [attempt]处理刚结束时马上试一次,让标记尽快落到信箱。失败也不影响处理结果,
* 留给扫描重试即可。
* - [attempt]单行写标记,只由 [sweep] 逐行调用——主泵与处理器不直接调它(回填一律扫描驱动)。
* - [sweep]:定时把还欠回填的记录挑出来重试。失败就按 30 秒起步、最长 15 分钟的
* 退避往后推;如果一条消息从收到现在已经超过超期期限,则无视退避强制补写——
* 否则退避可能一直失败下去,这些行永远打不上标记,库方就没法清理信箱。
@@ -81,7 +81,7 @@ class Pump(
//
// 水位以内的行都是收报按 ID 顺序发现并登记的;水位之外的行只可能来自兼容入口
// 直接写 PROC_STATE(它不参与水位)。若允许领取,它就会越过那些尚未入队的较小 ID,
// 破坏 FIFO缺口 G2)。这种行在空洞补齐、`W` 追平之后自然可领取。
// 破坏 FIFO不变量"只领取已发现的行"message-lifecycle.md §11)。这种行在空洞补齐、`W` 追平之后自然可领取。
val watermark = cursor.load().committedUpTo
if (head.msgId > watermark) {
warnBeyondWatermark(head.msgId, watermark)
@@ -175,21 +175,7 @@ class MessageProcessor(
}
}
/** @return 这条消息是否落到终态;回填由扫描驱动,调用方不再据此立即回填。 */
private fun processInternal(head: ProcState): Boolean {
// 守卫:手工/遗留 FAILED 行若 attempts 已达上限,直接终态(防止退避到期后无限重试)
if (head.state == ProcStatus.FAILED && procFailure.scheduler.exhausted(head.attempts)) {
log.error("head exhausted at entry -> DEAD msgId={} attempts={}", head.msgId, head.attempts)
procState.markTerminal(
head.msgId, ProcStatus.DEAD,
errorClass = ErrorClass.EXHAUSTED,
lastError = head.lastError ?: "max-attempts",
attempts = head.attempts,
now = clock.instant(),
)
return true
}
private fun processInternal(head: ProcState) {
val raw = inbox.rawOf(head.msgId)
if (raw == null) {
log.error("raw missing -> DEAD(MALFORMED) msgId={}", head.msgId)
@@ -215,7 +201,7 @@ class MessageProcessor(
val owner = procState.ownerOfIdentity(identity) ?: -1L
log.info("duplicate-of:{} -> SKIPPED msgId={}", owner, head.msgId)
procState.markTerminal(head.msgId, ProcStatus.SKIPPED, lastError = "duplicate-of:$owner", now = clock.instant())
return true
return
}
}
@@ -260,16 +246,14 @@ class MessageProcessor(
lastError = result.reason.take(1000),
now = clock.instant(),
)
return true
return
}
// Succeeded / ReplaySkippedSUCCEEDED 终态与回填意图已由处理器在自己的事务内落库
log.info("SUCCEEDED msgId={} kind={}", head.msgId, decoded.typeTag)
return true
}
private fun deadMalformed(head: ProcState, detail: String): Boolean {
private fun deadMalformed(head: ProcState, detail: String) {
procState.markTerminal(head.msgId, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, lastError = detail, now = clock.instant())
return true
}
}
+1 -1
View File
@@ -20,7 +20,7 @@ msgx:
backoff-cap-ms: 60000
head-deadline: 10m # 队头滞留上界 = 最坏 HOL 时长(毒丸升级)
max-commit-delay: 5m # §5.1 空洞老化:W+1 空洞超过该时延判定为永久(Q2 最大提交时延)
overdue-backfill: 30d # §5.2 超期补写期限 R:≥ 人工重放期限 + 人工处置期限(Q6
overdue-backfill: 30d # §5.2 超期补写期限 RQ6):仅须 R ≤ R_keep,不保护重放窗口(message-lifecycle §5.2
backfill-batch: 100 # 回填扫描单批条数
backfill-max-attempts: 100 # 回填自动重试上限;达上限停止自动重试(可人工恢复),避免占满批次
# 一次性切流播种:默认(注释掉)不播种。min=读现存全部 | zero=从 0 按空洞规则 | max=跳过可见存量 | <id>
@@ -219,7 +219,7 @@ class PipelineSmokeTest {
}
/**
* FIFO 修复(G2端到端验收:兼容入口写入的高 ID **不会被提前领取**,
* FIFO 不变量"只领取已发现的行"端到端验收:兼容入口写入的高 ID **不会被提前领取**,
* 必须等水位追平(较小 ID 补齐并入队)之后才按顺序处理。
*/
@Test
@@ -101,4 +101,21 @@ class HistorySweepJobTest {
assertTrue(tombstone.payloadJson.contains("\"deleted\":true"))
assertEquals(null, f.findMainRow("F3"))
}
@Test
fun `snap log purge runs independently of the history store switch`() {
val f = seededFlight("F1", deleted = true, idleDays = 30)
var cutoff: Instant? = null
val purge = com.gzzn.omms.msgexchange.infra.persistence.SnapshotLogPurge { instant -> cutoff = instant; 7 }
val job = HistorySweepJob(
f, StubMsgEvents(), HistoryProps().apply { historyStoreEnabled = false }, snapLogPurge = purge,
)
val outcome = job.run(now)
assertEquals(7, outcome.snapLogPurged) // 留痕清理不依赖历史存储开关
assertEquals(0, outcome.purged) // 航班当前态仍一条不许删
assertTrue(f.findMainRow("F1") != null)
assertTrue(cutoff != null)
}
}