fix(pipeline): 修复取报-处理-回填链路的超时、吞吐、回填与 FIFO 问题
外部调用（ACM2-33） - 共享 MySQL 与自有 PG 全部补有界超时：驱动 connect/socket 超时 + Hikari 池超时 （注意 Micronaut 的 Hikari 项是毫秒数，不是 Duration 字面量） - 删除主泵/毒丸路径的 inline 回填：跨库写不再占用 FIFO 关键路径，回填统一由扫描驱动 - 游标自愈：save() 改为「先 UPDATE、缺行 INSERT」，缺行只记一次 ERROR - Pump/Dispatcher 外层 catch 补 error 日志与失败计数 投递吞吐（ACM2-34） - Dispatcher 改批量领取；队头退避未到期或首条发送失败即停止本轮（保持目标内保序） - 有活不再 sleep；删除不可达的 state==SENT 死条件；markSent 移入分支； superseded 清理加轮数上限与空 eventId 保护 回填闭环（ACM2-36，V4） - MISSING（运行时确认行不存在）立即放弃自动重试并告警；暂时性故障达上限后停止自动重试 - 新增 reopen 人工恢复入口；放弃 ≠ 标记已确认（BACKFILL_AT 仍为空，清除前提不成立） - 扫描改 (BACKFILL_ATTEMPTS, MSG_ID) 公平轮转并排除放弃行，消除全局回填饥饿 - V4 只加字段与必要索引，不按年龄做任何存量推断 切流播种（ACM2-35，V5） - cutover-watermark 四模式（min/zero/max/显式 ID），默认不播种、代码不做默认选择 - 升级实例拒绝重新播种（SEEDED_AT 为 NULL ≠ 从未消费）；播种与水位同语句落库 - 非法取值由启动自检挡下 错误分类与入口契约（ACM2-37） - 未知 SCHD 子类型改为 UNSUPPORTED，不再静默当全量日计划合并 - ADFT 运营日冲突改走 ProtocolViolation → DEAD(PROTOCOL) - 兼容入口 receivedAt 缺失回退到注入 Clock；MessageLifecycleGate 强制注入 + 装配断言 - FIFO：主泵只领取 msgId ≤ W，兼容入口登记的行在水位追平前不被领取 时间源与可观测（ACM2-38 / ACM2-41 阶段 0） - 仓储/处理器/作业全部经注入 Clock；移除 markTerminal/markBackfilled 的 Instant.now() 默认值 - 退避表档位与 max-attempts 对齐并加启动自检 - Micrometer 7 个 gauge（@Context 急切注册）+ /health 与 /metrics 共用积压快照缓存 - 只读迟到检测：监视被放行的空洞 ID 是否后来真的出现，只计数告警、不补入队 Plane: ACM2-33 ACM2-34 ACM2-35 ACM2-36 ACM2-37 ACM2-38 ACM2-41 Tests: 88 → 120（1 skipped 需真实 PG）
This commit is contained in:
windyboy
@@ -0,0 +1,48 @@
|
||||
package com.gzzn.omms.msgexchange.infra.health
|
||||
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.Backlog
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
|
||||
import io.micronaut.context.BeanProvider
|
||||
import io.micronaut.context.annotation.Value
|
||||
import jakarta.inject.Singleton
|
||||
import java.time.Clock
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
|
||||
/**
|
||||
* 处理侧积压快照的唯一取数点，带 TTL 缓存。
|
||||
*
|
||||
* `backlog()` 是 `PROC_STATE` 的**全表聚合**；健康检查（`/health`）与指标抓取（`/metrics`）
|
||||
* 都可能被高频调用，因此两者共用同一份缓存，避免把数据库拖慢。代价是数字最多滞后一个
|
||||
* 缓存窗口——对积压/信龄这类告警信号足够。
|
||||
*
|
||||
* 缓存窗口由 `msgx.health.backlog-cache-ttl-ms` 配置（默认 30 秒；设为 0 表示不缓存）。
|
||||
* 需要实时精确计数时应改为维护计数器，而不是把窗口调到 0。
|
||||
*/
|
||||
@Singleton
|
||||
class BacklogSnapshotProvider(
|
||||
private val procState: BeanProvider<ProcStateRepository>,
|
||||
private val clock: Clock,
|
||||
@Value("\${msgx.health.backlog-cache-ttl-ms:30000}") private val ttlMs: Long = 30_000,
|
||||
) {
|
||||
@Volatile
|
||||
private var cached: Backlog? = null
|
||||
|
||||
@Volatile
|
||||
private var cachedAt: Instant? = null
|
||||
|
||||
/** @return 积压快照；仓储未绑定（例如 stub 关闭）时返回 null，由调用方按"未绑定"处理。 */
|
||||
fun snapshot(): Backlog? {
|
||||
if (!procState.isPresent) return null
|
||||
val now = clock.instant()
|
||||
val at = cachedAt
|
||||
val value = cached
|
||||
if (ttlMs > 0 && at != null && value != null && Duration.between(at, now).toMillis() < ttlMs) {
|
||||
return value
|
||||
}
|
||||
val fresh = procState.get().backlog()
|
||||
cached = fresh
|
||||
cachedAt = now
|
||||
return fresh
|
||||
}
|
||||
}
|
||||
+20
-5
@@ -1,5 +1,6 @@
|
||||
package com.gzzn.omms.msgexchange.infra.health
|
||||
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.Backlog
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.InboxCursorRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
|
||||
@@ -10,6 +11,7 @@ import io.micronaut.management.health.indicator.HealthIndicator
|
||||
import io.micronaut.management.health.indicator.HealthResult
|
||||
import jakarta.inject.Singleton
|
||||
import org.reactivestreams.Publisher
|
||||
import java.time.Clock
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
|
||||
@@ -28,6 +30,8 @@ class InboxLifecycleHealthIndicator(
|
||||
private val procState: BeanProvider<ProcStateRepository>,
|
||||
private val cursor: BeanProvider<InboxCursorRepository>,
|
||||
private val mailbox: BeanProvider<CminmsgInboxRepository>,
|
||||
private val clock: Clock,
|
||||
private val backlogs: BacklogSnapshotProvider,
|
||||
) : HealthIndicator {
|
||||
|
||||
override fun getResult(): Publisher<HealthResult> =
|
||||
@@ -37,6 +41,9 @@ class InboxLifecycleHealthIndicator(
|
||||
procState = if (procState.isPresent) procState.get() else null,
|
||||
cursor = if (cursor.isPresent) cursor.get() else null,
|
||||
mailbox = if (mailbox.isPresent) mailbox.get() else null,
|
||||
now = clock.instant(),
|
||||
// 与 /metrics 共用同一份 30 秒缓存：两者都不该把全表聚合打成高频查询。
|
||||
backlog = backlogs.snapshot(),
|
||||
)
|
||||
}.getOrElse { down(it) },
|
||||
)
|
||||
@@ -47,23 +54,31 @@ internal fun lifecycleHealth(
|
||||
procState: ProcStateRepository?,
|
||||
cursor: InboxCursorRepository?,
|
||||
mailbox: CminmsgInboxRepository?,
|
||||
now: Instant = Instant.now(),
|
||||
now: Instant,
|
||||
/** 允许调用方传入缓存/已算好的积压快照；为空时现查（`backlog()` 是全表聚合）。 */
|
||||
backlog: Backlog? = null,
|
||||
): HealthResult {
|
||||
if (procState == null) {
|
||||
return HealthResult.builder(NAME).status(HealthStatus.UP)
|
||||
.details(mapOf("message" to "proc_state repository not bound (stub off, impl pending)"))
|
||||
.build()
|
||||
}
|
||||
val backlog = procState.backlog()
|
||||
val snapshot = backlog ?: procState.backlog()
|
||||
val watermark = cursor?.load()?.committedUpTo
|
||||
val maxId = runCatching { mailbox?.maxId() }.getOrNull()
|
||||
return HealthResult.builder(NAME).status(HealthStatus.UP).details(
|
||||
linkedMapOf<String, Any>(
|
||||
"backlog" to backlog.unfinished,
|
||||
"backlog" to snapshot.unfinished,
|
||||
"oldestUnprocessedSeconds" to (
|
||||
backlog.oldestReceivedAt?.let { Duration.between(it, now).seconds } ?: -1L
|
||||
snapshot.oldestReceivedAt?.let { Duration.between(it, now).seconds } ?: -1L
|
||||
),
|
||||
"unmarkedTerminal" to snapshot.unmarkedTerminal,
|
||||
// 已放弃自动回填的条数：**不等于**标记已完成，需要人工对账；非 0 应告警。
|
||||
"backfillAbandoned" to snapshot.abandonedBackfill,
|
||||
// 最老一条"仍待自动回填"记录的年龄（秒）：回填延迟的真实观测值。
|
||||
"oldestUnmarkedBackfillSeconds" to (
|
||||
snapshot.oldestUnmarkedAt?.let { Duration.between(it, now).seconds } ?: -1L
|
||||
),
|
||||
"unmarkedTerminal" to backlog.unmarkedTerminal,
|
||||
"watermark" to (watermark ?: -1L),
|
||||
"watermarkLag" to if (watermark != null && maxId != null) maxId - watermark else -1L,
|
||||
),
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package com.gzzn.omms.msgexchange.infra.metrics
|
||||
|
||||
import jakarta.inject.Singleton
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 管道运行期的**进程内**计数。
|
||||
*
|
||||
* 存在的意义是让业务模块（如收报）能零依赖地上报事实：模块只依赖本类，
|
||||
* 不直接依赖 Micrometer；指标如何在 `/metrics` 上暴露由 [PipelineMetrics] 决定。
|
||||
*
|
||||
* 注意：计数在重启后归零。需要跨重启的累计值应由指标后端聚合，不在这里做持久化。
|
||||
*/
|
||||
@Singleton
|
||||
class PipelineCounters {
|
||||
private val holeAgedOut = AtomicLong(0)
|
||||
private val lateArrivalDetected = AtomicLong(0)
|
||||
|
||||
/** 水位因空洞超过老化阈值而放行（判定为永久空洞）的次数。 */
|
||||
fun holeAgedOutIncrement() {
|
||||
holeAgedOut.incrementAndGet()
|
||||
}
|
||||
|
||||
fun holeAgedOutCount(): Long = holeAgedOut.get()
|
||||
|
||||
/**
|
||||
* "迟到到达"检测命中的**不同**消息 ID 数（ACM2-41 阶段 0）。
|
||||
*
|
||||
* 含义：该 ID 曾被判定为永久空洞并放行，之后却真的出现在信箱里——即上游提交晚于水位推进。
|
||||
* 阶段 0 只计数与告警，**不会**补入队；因此这个值 > 0 表示"确有迟到发生，需要与库方对契约"。
|
||||
*/
|
||||
fun lateArrivalDetectedIncrement() {
|
||||
lateArrivalDetected.incrementAndGet()
|
||||
}
|
||||
|
||||
fun lateArrivalDetectedCount(): Long = lateArrivalDetected.get()
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package com.gzzn.omms.msgexchange.infra.metrics
|
||||
|
||||
import com.gzzn.omms.msgexchange.infra.health.BacklogSnapshotProvider
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.InboxCursorRepository
|
||||
import io.micronaut.context.BeanProvider
|
||||
import io.micronaut.context.annotation.Context
|
||||
import io.micronaut.context.annotation.Requires
|
||||
import io.micrometer.core.instrument.Gauge
|
||||
import io.micrometer.core.instrument.MeterRegistry
|
||||
import jakarta.annotation.PostConstruct
|
||||
import java.time.Clock
|
||||
import java.time.Duration
|
||||
|
||||
/**
|
||||
* 把管道的可观测事实暴露成 Micrometer 指标（默认经 `/metrics` 抓取）。
|
||||
*
|
||||
* 只暴露规范里要求观测的量：
|
||||
* - `msgx.pipeline.backlog.unfinished`：还没处理完的消息条数
|
||||
* - `msgx.pipeline.backlog.oldest_unprocessed_seconds`：最老未处理消息的信龄
|
||||
* - `msgx.pipeline.backfill.unmarked_terminal`：已终态但未打标的条数
|
||||
* - `msgx.pipeline.backfill.abandoned`：已放弃自动回填的条数（**非 0 需人工对账**）
|
||||
* - `msgx.pipeline.backfill.oldest_unmarked_seconds`：最老一条仍待自动回填的年龄
|
||||
* - `msgx.pipeline.watermark.lag`：水位落后信箱最新 ID 的距离
|
||||
* - `msgx.pipeline.hole.aged_out.total`：永久空洞放行次数
|
||||
*
|
||||
* 取数统一走 [BacklogSnapshotProvider]（30 秒 TTL），因此指标抓取不会打穿数据库。
|
||||
* 无法取数时以 `NaN` 上报（Micrometer 的惯例表示"本次无值"），而不是伪造 0。
|
||||
*
|
||||
* `Gauge` 默认对目标对象持**弱引用**，这里显式 `strongReference(true)`：
|
||||
* 单例被容器持有本不会被回收，但显式声明可避免将来重构踩坑。
|
||||
*
|
||||
* 用 `@Context` 而不是 `@Singleton`：没有任何 bean 依赖它，惰性单例永远不会被创建，
|
||||
* 指标也就注册不上。指标装配必须在启动时急切完成。
|
||||
*/
|
||||
@Context
|
||||
@Requires(beans = [MeterRegistry::class])
|
||||
class PipelineMetrics(
|
||||
private val registry: MeterRegistry,
|
||||
private val backlogs: BacklogSnapshotProvider,
|
||||
private val cursor: BeanProvider<InboxCursorRepository>,
|
||||
private val mailbox: BeanProvider<CminmsgInboxRepository>,
|
||||
private val counters: PipelineCounters,
|
||||
private val clock: Clock,
|
||||
) {
|
||||
|
||||
@PostConstruct
|
||||
fun bind() {
|
||||
backlogGauge("msgx.pipeline.backlog.unfinished") { it.unfinished.toDouble() }
|
||||
backlogGauge("msgx.pipeline.backlog.oldest_unprocessed_seconds") { snapshot ->
|
||||
snapshot.oldestReceivedAt
|
||||
?.let { Duration.between(it, clock.instant()).seconds.toDouble() }
|
||||
?: -1.0
|
||||
}
|
||||
backlogGauge("msgx.pipeline.backfill.unmarked_terminal") { it.unmarkedTerminal.toDouble() }
|
||||
backlogGauge("msgx.pipeline.backfill.abandoned") { it.abandonedBackfill.toDouble() }
|
||||
backlogGauge("msgx.pipeline.backfill.oldest_unmarked_seconds") { snapshot ->
|
||||
snapshot.oldestUnmarkedAt
|
||||
?.let { Duration.between(it, clock.instant()).seconds.toDouble() }
|
||||
?: -1.0
|
||||
}
|
||||
|
||||
Gauge.builder("msgx.pipeline.watermark.lag", backlogs) { _ ->
|
||||
val watermark = if (cursor.isPresent) cursor.get().load().committedUpTo else null
|
||||
val maxId = if (mailbox.isPresent) runCatching { mailbox.get().maxId() }.getOrNull() else null
|
||||
if (watermark != null && maxId != null) (maxId - watermark).toDouble() else -1.0
|
||||
}.strongReference(true).register(registry)
|
||||
|
||||
Gauge.builder("msgx.pipeline.hole.aged_out.total", counters) { it.holeAgedOutCount().toDouble() }
|
||||
.strongReference(true)
|
||||
.register(registry)
|
||||
|
||||
// 迟到到达检测命中数（阶段 0 只观测）：> 0 表示上游提交确实晚于水位推进，需要与库方对契约。
|
||||
Gauge.builder("msgx.pipeline.late_arrival.detected.total", counters) { it.lateArrivalDetectedCount().toDouble() }
|
||||
.strongReference(true)
|
||||
.register(registry)
|
||||
}
|
||||
|
||||
private fun backlogGauge(name: String, value: (com.gzzn.omms.msgexchange.infra.persistence.Backlog) -> Double) {
|
||||
Gauge.builder(name, backlogs) { provider ->
|
||||
provider.snapshot()?.let(value) ?: Double.NaN
|
||||
}.strongReference(true).register(registry)
|
||||
}
|
||||
}
|
||||
@@ -89,15 +89,31 @@ interface ProcStateRepository {
|
||||
errorClass: ErrorClass? = null,
|
||||
lastError: String? = null,
|
||||
attempts: Int? = null,
|
||||
now: Instant = Instant.now(),
|
||||
now: Instant,
|
||||
)
|
||||
|
||||
/** 回填成功：记下完成时间，清掉待办。 */
|
||||
fun markBackfilled(msgId: Long, now: Instant = Instant.now())
|
||||
fun markBackfilled(msgId: Long, now: Instant)
|
||||
|
||||
/** 回填失败：次数 +1、按退避推后、记下原因。处理终态不受影响，不会被改回去。 */
|
||||
fun recordBackfillFailure(msgId: Long, error: String?, attempts: Int, nextAttemptAt: Instant, now: Instant)
|
||||
|
||||
/**
|
||||
* 放弃回填：判定该行不必再自动尝试（信箱行不存在，或达到尝试上限）。
|
||||
*
|
||||
* **abandoned ≠ 标记已确认**：`BACKFILL_AT` 仍为空，因此**不**满足"边界内全部行已打标"的
|
||||
* 清除前提；放弃只是停止自动重试并把事实留痕，供人工对账。
|
||||
*
|
||||
* @return false 表示该行不存在
|
||||
*/
|
||||
fun markBackfillAbandoned(msgId: Long, reason: String, now: Instant): Boolean
|
||||
|
||||
/**
|
||||
* 人工恢复入口：清除放弃标记并重新排队一次回填（暂时性故障恢复后、或人工对账确认后使用）。
|
||||
* @return false 表示该行不存在或本来就未被放弃
|
||||
*/
|
||||
fun reopenBackfill(msgId: Long, now: Instant): Boolean
|
||||
|
||||
/**
|
||||
* 找出现在该回填的记录：已经到终态、还没确认回填，并且退避时间已到。
|
||||
*
|
||||
@@ -111,6 +127,9 @@ interface ProcStateRepository {
|
||||
|
||||
/** 积压观测：还没处理完的条数、最老一条的接收时间、处理完但还没回填的条数。 */
|
||||
fun backlog(): Backlog
|
||||
|
||||
/** 是否已有任何处理记录（含终态）。切流播种用它判断"这个实例是否已经消费过"。 */
|
||||
fun hasAny(): Boolean
|
||||
}
|
||||
|
||||
/** 扫描到的待回填记录。 */
|
||||
@@ -120,9 +139,17 @@ data class BackfillDue(val msgId: Long, val attempts: Int)
|
||||
* 处理侧积压快照。
|
||||
* @param unfinished 还没处理完的消息条数
|
||||
* @param oldestReceivedAt 其中最早一条的接收时间（据此算信龄）
|
||||
* @param unmarkedTerminal 已经处理完、但还没把标记写回信箱的条数
|
||||
* @param unmarkedTerminal 已经处理完、但还没把标记写回信箱的条数（含已放弃的）
|
||||
* @param abandonedBackfill 其中已放弃自动回填的条数（需要人工对账的信号）
|
||||
* @param oldestUnmarkedAt 最老一条待回填记录的接收时间（回填延迟观测）
|
||||
*/
|
||||
data class Backlog(val unfinished: Int, val oldestReceivedAt: Instant?, val unmarkedTerminal: Int)
|
||||
data class Backlog(
|
||||
val unfinished: Int,
|
||||
val oldestReceivedAt: Instant?,
|
||||
val unmarkedTerminal: Int,
|
||||
val abandonedBackfill: Int = 0,
|
||||
val oldestUnmarkedAt: Instant? = null,
|
||||
)
|
||||
|
||||
/**
|
||||
* 待发事件（outbox）表：业务提交时把要发的事件一起写进来，投递线程再从这张表往外发，
|
||||
@@ -263,12 +290,25 @@ interface InboxCursorRepository {
|
||||
/**
|
||||
* @param committedUpTo 水位 W
|
||||
* @param holeSince W 后面那个缺口最早被发现的时刻；当前没有缺口时为 null
|
||||
* @param seededAt 非空 = 已按 `msgx.pipeline.cutover-watermark` 播种过。
|
||||
* **它为空不等于"从未消费"**：已有库新增该列后同样是 null，判断必须叠加"水位为 0 且无处理记录"。
|
||||
*/
|
||||
data class Cursor(val committedUpTo: Long = 0L, val holeSince: Instant? = null)
|
||||
data class Cursor(
|
||||
val committedUpTo: Long = 0L,
|
||||
val holeSince: Instant? = null,
|
||||
val seededAt: Instant? = null,
|
||||
)
|
||||
|
||||
fun load(): Cursor
|
||||
|
||||
/** 只推进水位与空洞计时，不改动 [Cursor.seededAt]。 */
|
||||
fun save(cursor: Cursor)
|
||||
|
||||
/**
|
||||
* 切流播种：一次性写入水位并把空洞计时清空，同时记录播种事实。
|
||||
* 与 [save] 分开，避免"普通轮次推进水位"把播种标记抹掉。
|
||||
*/
|
||||
fun markSeeded(committedUpTo: Long, now: Instant)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -299,6 +339,15 @@ interface CminmsgInboxRepository {
|
||||
/** 信箱当前最大 ID，空表返回 null；只用来观测收报落后了多少。 */
|
||||
fun maxId(): Long?
|
||||
|
||||
/** 信箱当前最小 ID，空表返回 null；切流播种用它推算 `W = MIN(ID) − 1`。 */
|
||||
fun minId(): Long?
|
||||
|
||||
/**
|
||||
* 批量查询这些 ID 里哪些**当前存在于信箱**。
|
||||
* 只用于"迟到到达"检测（被放行的空洞 ID 后来是否真的出现），不读大字段。
|
||||
*/
|
||||
fun existingIds(msgIds: Collection<Long>): Set<Long>
|
||||
|
||||
/**
|
||||
* 把处理标记写回信箱，并且**只写还是空标记的行**：库里已有值时不覆盖、不回退，
|
||||
* 重复调用没有副作用。结果明确区分本次写入、已有标记和信箱行缺失。
|
||||
|
||||
+19
@@ -71,6 +71,25 @@ class JdbcCminmsgInboxRepository(
|
||||
rs.getLong("max_id").takeIf { !rs.wasNull() }
|
||||
}
|
||||
|
||||
override fun minId(): Long? =
|
||||
ds.queryOne("SELECT MIN(CMINMSGS_ID) AS min_id FROM cminmsgs", {}) { rs ->
|
||||
rs.getLong("min_id").takeIf { !rs.wasNull() }
|
||||
}
|
||||
|
||||
override fun existingIds(msgIds: Collection<Long>): Set<Long> {
|
||||
if (msgIds.isEmpty()) return emptySet()
|
||||
val out = linkedSetOf<Long>()
|
||||
// 分块避免 IN 列表过长（迟到检测一次最多查 late-detect-batch 个）。
|
||||
msgIds.chunked(200).forEach { chunk ->
|
||||
val placeholders = chunk.joinToString(",") { "?" }
|
||||
ds.query(
|
||||
"SELECT CMINMSGS_ID FROM cminmsgs WHERE CMINMSGS_ID IN ($placeholders)",
|
||||
{ ps -> chunk.forEachIndexed { i, id -> ps.setLong(i + 1, id) } },
|
||||
) { rs -> rs.getLong("CMINMSGS_ID") }.forEach { out += it }
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* 只更新还是空标记的行，所以重复调用不会覆盖库里已有的值；
|
||||
* 影响 0 行时再查一次主键，区分“已有标记”和“信箱行缺失”；后者不能记为回填成功。
|
||||
|
||||
+133
-18
@@ -27,11 +27,13 @@ import com.gzzn.omms.msgexchange.infra.persistence.SnapshotLogRepository
|
||||
import io.micronaut.context.annotation.Requires
|
||||
import jakarta.inject.Singleton
|
||||
import java.sql.ResultSet
|
||||
import java.time.Clock
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
import java.time.LocalDate
|
||||
import java.time.LocalDateTime
|
||||
import java.time.ZoneId
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import com.gzzn.omms.msgexchange.domain.OperationDayCalculator
|
||||
import java.time.format.DateTimeFormatter
|
||||
import java.util.Locale
|
||||
@@ -70,6 +72,7 @@ class JdbcPipelineLockRepository(
|
||||
@Requires(missingProperty = "msgx.stubs")
|
||||
class JdbcProcStateRepository(
|
||||
private val ds: DataSource,
|
||||
private val clock: Clock,
|
||||
) : ProcStateRepository {
|
||||
/** 入队（幂等）：主键冲突时什么都不做，所以重复扫描和兼容入口并发调用都安全。 */
|
||||
override fun insertIfAbsent(msgId: Long, receivedAt: Instant?): Boolean =
|
||||
@@ -79,7 +82,7 @@ class JdbcProcStateRepository(
|
||||
{ ps ->
|
||||
ps.setLong(1, msgId)
|
||||
ps.setTimestamp(2, receivedAt?.toSqlTimestamp())
|
||||
ps.setTimestamp(3, Instant.now().toSqlTimestamp())
|
||||
ps.setTimestamp(3, clock.instant().toSqlTimestamp())
|
||||
},
|
||||
) == 1
|
||||
|
||||
@@ -117,7 +120,7 @@ class JdbcProcStateRepository(
|
||||
"UPDATE proc_state SET identity_key = ?, updated_at = ? WHERE msg_id = ? AND identity_key IS NULL",
|
||||
{ ps ->
|
||||
ps.setString(1, identityKey)
|
||||
ps.setTimestamp(2, Instant.now().toSqlTimestamp())
|
||||
ps.setTimestamp(2, clock.instant().toSqlTimestamp())
|
||||
ps.setLong(3, msgId)
|
||||
},
|
||||
)
|
||||
@@ -149,7 +152,7 @@ class JdbcProcStateRepository(
|
||||
ps.setInt(3, attempts ?: 0)
|
||||
ps.setString(4, errorClass?.name)
|
||||
ps.setString(5, lastError)
|
||||
ps.setTimestamp(6, Instant.now().toSqlTimestamp())
|
||||
ps.setTimestamp(6, clock.instant().toSqlTimestamp())
|
||||
ps.setLong(7, msgId)
|
||||
},
|
||||
)
|
||||
@@ -170,6 +173,7 @@ class JdbcProcStateRepository(
|
||||
SET state = ?, error_class = ?, last_error = ?, attempts = COALESCE(?, attempts),
|
||||
next_attempt_at = NULL,
|
||||
backfill_at = NULL, backfill_next_at = ?, backfill_attempts = 0, backfill_error = NULL,
|
||||
backfill_abandoned_at = NULL, backfill_abandoned_reason = NULL,
|
||||
updated_at = ?
|
||||
WHERE msg_id = ?
|
||||
""".trimIndent(),
|
||||
@@ -209,18 +213,55 @@ class JdbcProcStateRepository(
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* 放弃自动回填。注意 **不写 `BACKFILL_AT`**：放弃 ≠ 标记已确认，
|
||||
* 因此"边界内全部行已打标"的清除前提仍然不成立，库方不应据此清除。
|
||||
*/
|
||||
override fun markBackfillAbandoned(msgId: Long, reason: String, now: Instant): Boolean =
|
||||
ds.update(
|
||||
"UPDATE proc_state SET backfill_abandoned_at = ?, backfill_abandoned_reason = ?, " +
|
||||
"backfill_next_at = NULL, updated_at = ? " +
|
||||
"WHERE msg_id = ? AND backfill_at IS NULL AND backfill_abandoned_at IS NULL",
|
||||
{ ps ->
|
||||
ps.setTimestamp(1, now.toSqlTimestamp())
|
||||
ps.setString(2, reason.take(64))
|
||||
ps.setTimestamp(3, now.toSqlTimestamp())
|
||||
ps.setLong(4, msgId)
|
||||
},
|
||||
) == 1
|
||||
|
||||
/** 人工恢复：清掉放弃标记并立刻重排一次回填（暂时性故障恢复后或对账后使用）。 */
|
||||
override fun reopenBackfill(msgId: Long, now: Instant): Boolean =
|
||||
ds.update(
|
||||
"UPDATE proc_state SET backfill_abandoned_at = NULL, backfill_abandoned_reason = NULL, " +
|
||||
"backfill_next_at = ?, updated_at = ? " +
|
||||
"WHERE msg_id = ? AND backfill_at IS NULL AND backfill_abandoned_at IS NOT NULL",
|
||||
{ ps ->
|
||||
ps.setTimestamp(1, now.toSqlTimestamp())
|
||||
ps.setTimestamp(2, now.toSqlTimestamp())
|
||||
ps.setLong(3, msgId)
|
||||
},
|
||||
) == 1
|
||||
|
||||
/**
|
||||
* 到了该回填的时候：终态 + 还没有标记 + （退避到期 或 收信时间已经很久）。
|
||||
* 后面这个"很久"是兜底，保证标记最终一定会补上，库方才能按标记清理信箱。
|
||||
*/
|
||||
/**
|
||||
* 到了该回填的时候：终态 + 还没有标记 + **未放弃** + （退避到期 或 收信时间已经很久）。
|
||||
*
|
||||
* 排序用**公平轮转**：先按已尝试次数升序，再按 msg_id。若只按 msg_id 升序，
|
||||
* 最旧的一批永久失败行会持续占满批次，后面的记录永远轮不到（全局回填饥饿）。
|
||||
*/
|
||||
override fun findBackfillDue(now: Instant, overdueBefore: Instant, limit: Int): List<BackfillDue> =
|
||||
ds.query(
|
||||
"""
|
||||
SELECT msg_id, backfill_attempts FROM proc_state
|
||||
WHERE backfill_at IS NULL
|
||||
AND backfill_abandoned_at IS NULL
|
||||
AND state IN ('SUCCEEDED', 'SKIPPED', 'DEAD')
|
||||
AND (backfill_next_at IS NULL OR backfill_next_at <= ? OR (received_at IS NOT NULL AND received_at < ?))
|
||||
ORDER BY msg_id ASC LIMIT ?
|
||||
ORDER BY backfill_attempts ASC, msg_id ASC LIMIT ?
|
||||
""".trimIndent(),
|
||||
{ ps ->
|
||||
ps.setTimestamp(1, now.toSqlTimestamp())
|
||||
@@ -236,20 +277,27 @@ class JdbcProcStateRepository(
|
||||
"UPDATE proc_state SET state = 'PENDING', attempts = 0, next_attempt_at = NULL, processing_started_at = NULL, updated_at = ? " +
|
||||
"WHERE state IN ('FAILED', 'DEAD') AND error_class IN ($placeholders)",
|
||||
{ ps ->
|
||||
ps.setTimestamp(1, Instant.now().toSqlTimestamp())
|
||||
ps.setTimestamp(1, clock.instant().toSqlTimestamp())
|
||||
errorClasses.forEachIndexed { i, ec -> ps.setString(i + 2, ec.name) }
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/** 积压观测：未处理条数、最老一条的接收时间、处理完但未回填的条数。 */
|
||||
override fun hasAny(): Boolean =
|
||||
ds.queryOne("SELECT 1 FROM proc_state LIMIT 1", {}) { 1 } != null
|
||||
|
||||
override fun backlog(): Backlog =
|
||||
ds.queryOne(
|
||||
"""
|
||||
SELECT
|
||||
count(*) FILTER (WHERE state IN ('PENDING', 'FAILED')) AS unfinished,
|
||||
min(received_at) FILTER (WHERE state IN ('PENDING', 'FAILED')) AS oldest_received_at,
|
||||
count(*) FILTER (WHERE state IN ('SUCCEEDED', 'SKIPPED', 'DEAD') AND backfill_at IS NULL) AS unmarked_terminal
|
||||
count(*) FILTER (WHERE state IN ('SUCCEEDED', 'SKIPPED', 'DEAD') AND backfill_at IS NULL) AS unmarked_terminal,
|
||||
count(*) FILTER (WHERE state IN ('SUCCEEDED', 'SKIPPED', 'DEAD') AND backfill_at IS NULL
|
||||
AND backfill_abandoned_at IS NOT NULL) AS abandoned_backfill,
|
||||
min(received_at) FILTER (WHERE state IN ('SUCCEEDED', 'SKIPPED', 'DEAD') AND backfill_at IS NULL
|
||||
AND backfill_abandoned_at IS NULL) AS oldest_unmarked_at
|
||||
FROM proc_state
|
||||
""".trimIndent(),
|
||||
{},
|
||||
@@ -258,6 +306,8 @@ class JdbcProcStateRepository(
|
||||
unfinished = rs.getInt("unfinished"),
|
||||
oldestReceivedAt = rs.getInstant("oldest_received_at"),
|
||||
unmarkedTerminal = rs.getInt("unmarked_terminal"),
|
||||
abandonedBackfill = rs.getInt("abandoned_backfill"),
|
||||
oldestUnmarkedAt = rs.getInstant("oldest_unmarked_at"),
|
||||
)
|
||||
} ?: Backlog(0, null, 0)
|
||||
|
||||
@@ -274,14 +324,17 @@ class JdbcProcStateRepository(
|
||||
backfillNextAt = rs.getInstant("backfill_next_at"),
|
||||
backfillAttempts = rs.getInt("backfill_attempts"),
|
||||
backfillError = rs.getString("backfill_error"),
|
||||
backfillAbandonedAt = rs.getInstant("backfill_abandoned_at"),
|
||||
backfillAbandonedReason = rs.getString("backfill_abandoned_reason"),
|
||||
processingStartedAt = rs.getInstant("processing_started_at"),
|
||||
updatedAt = rs.getInstant("updated_at") ?: Instant.now(),
|
||||
updatedAt = rs.getInstant("updated_at") ?: clock.instant(),
|
||||
)
|
||||
|
||||
private companion object {
|
||||
const val SELECT_PROC =
|
||||
"SELECT msg_id, state, identity_key, attempts, next_attempt_at, error_class, last_error, " +
|
||||
"received_at, backfill_at, backfill_next_at, backfill_attempts, backfill_error, processing_started_at, updated_at FROM proc_state"
|
||||
"received_at, backfill_at, backfill_next_at, backfill_attempts, backfill_error, " +
|
||||
"backfill_abandoned_at, backfill_abandoned_reason, processing_started_at, updated_at FROM proc_state"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -291,24 +344,82 @@ class JdbcProcStateRepository(
|
||||
@Requires(missingProperty = "msgx.stubs")
|
||||
class JdbcInboxCursorRepository(
|
||||
private val ds: DataSource,
|
||||
private val clock: Clock,
|
||||
) : InboxCursorRepository {
|
||||
override fun load(): InboxCursorRepository.Cursor =
|
||||
ds.queryOne(
|
||||
"SELECT committed_up_to, hole_since FROM inbox_cursor WHERE cursor_id = 1",
|
||||
"SELECT committed_up_to, hole_since, seeded_at FROM inbox_cursor WHERE cursor_id = 1",
|
||||
{},
|
||||
) { rs -> InboxCursorRepository.Cursor(rs.getLong("committed_up_to"), rs.getInstant("hole_since")) }
|
||||
?: InboxCursorRepository.Cursor()
|
||||
) { rs ->
|
||||
InboxCursorRepository.Cursor(
|
||||
rs.getLong("committed_up_to"),
|
||||
rs.getInstant("hole_since"),
|
||||
rs.getInstant("seeded_at"),
|
||||
)
|
||||
}
|
||||
?: InboxCursorRepository.Cursor().also {
|
||||
// 缺行不是"水位为 0"的同义词：这里只报一次，随后 save() 会 upsert 自愈。
|
||||
if (missingCursorWarned.compareAndSet(false, true)) {
|
||||
log.error("INBOX_CURSOR row (cursor_id=1) is missing; watermark will be re-created on the next save()")
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 水位推进必须是 upsert，不能是裸 UPDATE：
|
||||
* 裸 UPDATE 在游标行缺失时影响 0 行且不报错，会让水位永远停在初值、每轮重扫同一批，
|
||||
* 收报静默死锁在第一批。upsert 让缺行自愈，且与入队同事务提交。
|
||||
*/
|
||||
override fun save(cursor: InboxCursorRepository.Cursor) {
|
||||
ds.update(
|
||||
// 先 UPDATE、缺行再 INSERT：比 ON CONFLICT 更可移植（H2 的 PostgreSQL 兼容模式不支持 ON CONFLICT），
|
||||
// 且 InboxPoller 始终把它放在同一个 PG 事务里提交，因此两条语句对外仍是原子的。
|
||||
// 关键点是**缺行必须能自愈**：裸 UPDATE 影响 0 行却不报错，会让水位永远停在初值、每轮重扫同一批。
|
||||
val updated = ds.update(
|
||||
"UPDATE inbox_cursor SET committed_up_to = ?, hole_since = ?, updated_at = ? WHERE cursor_id = 1",
|
||||
{ ps ->
|
||||
ps.setLong(1, cursor.committedUpTo)
|
||||
ps.setTimestamp(2, cursor.holeSince?.toSqlTimestamp())
|
||||
ps.setTimestamp(3, Instant.now().toSqlTimestamp())
|
||||
ps.setTimestamp(3, clock.instant().toSqlTimestamp())
|
||||
},
|
||||
)
|
||||
if (updated > 0) return
|
||||
ds.update(
|
||||
"INSERT INTO inbox_cursor (cursor_id, committed_up_to, hole_since, updated_at) VALUES (1, ?, ?, ?)",
|
||||
{ ps ->
|
||||
ps.setLong(1, cursor.committedUpTo)
|
||||
ps.setTimestamp(2, cursor.holeSince?.toSqlTimestamp())
|
||||
ps.setTimestamp(3, clock.instant().toSqlTimestamp())
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* 切流播种：一次性写入水位 + 清空空洞计时 + 记录播种事实（同一条语句）。
|
||||
* 与 [save] 分开，普通轮次推进水位不会把播种标记抹掉。
|
||||
*/
|
||||
override fun markSeeded(committedUpTo: Long, now: Instant) {
|
||||
val updated = ds.update(
|
||||
"UPDATE inbox_cursor SET committed_up_to = ?, hole_since = NULL, seeded_at = ?, updated_at = ? " +
|
||||
"WHERE cursor_id = 1",
|
||||
{ ps ->
|
||||
ps.setLong(1, committedUpTo)
|
||||
ps.setTimestamp(2, now.toSqlTimestamp())
|
||||
ps.setTimestamp(3, now.toSqlTimestamp())
|
||||
},
|
||||
)
|
||||
if (updated > 0) return
|
||||
ds.update(
|
||||
"INSERT INTO inbox_cursor (cursor_id, committed_up_to, hole_since, seeded_at, updated_at) " +
|
||||
"VALUES (1, ?, NULL, ?, ?)",
|
||||
{ ps ->
|
||||
ps.setLong(1, committedUpTo)
|
||||
ps.setTimestamp(2, now.toSqlTimestamp())
|
||||
ps.setTimestamp(3, now.toSqlTimestamp())
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
private val missingCursorWarned = AtomicBoolean(false)
|
||||
private val log = org.slf4j.LoggerFactory.getLogger(JdbcInboxCursorRepository::class.java)
|
||||
}
|
||||
|
||||
@Singleton
|
||||
@@ -316,6 +427,7 @@ class JdbcInboxCursorRepository(
|
||||
@Requires(missingProperty = "msgx.stubs")
|
||||
class JdbcMsgEventRepository(
|
||||
private val ds: DataSource,
|
||||
private val clock: Clock,
|
||||
) : MsgEventRepository {
|
||||
override fun insertAll(events: List<MsgEvent>): List<Long> =
|
||||
events.map { e ->
|
||||
@@ -414,7 +526,7 @@ class JdbcMsgEventRepository(
|
||||
nextAttemptAt = rs.getInstant("next_attempt_at"),
|
||||
errorClass = rs.getString("error_class")?.let(ErrorClass::valueOf),
|
||||
lastError = rs.getString("last_error"),
|
||||
createdAt = rs.getInstant("created_at") ?: Instant.now(),
|
||||
createdAt = rs.getInstant("created_at") ?: clock.instant(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -423,6 +535,7 @@ class JdbcMsgEventRepository(
|
||||
@Requires(missingProperty = "msgx.stubs")
|
||||
class JdbcFlightStateRepository(
|
||||
private val ds: DataSource,
|
||||
private val clock: Clock,
|
||||
) : FlightStateRepository {
|
||||
|
||||
private fun mapMainRow(rs: ResultSet) = FlightMainRow(
|
||||
@@ -431,7 +544,7 @@ class JdbcFlightStateRepository(
|
||||
state = FlightState.valueOf(rs.getString("state")),
|
||||
stateVersion = rs.getLong("state_version"),
|
||||
lastMsgId = rs.getLong("last_msg_id").takeIf { !rs.wasNull() },
|
||||
updatedAt = rs.getInstant("updated_at") ?: Instant.now(),
|
||||
updatedAt = rs.getInstant("updated_at") ?: clock.instant(),
|
||||
)
|
||||
|
||||
override fun findMainRow(flid: String): FlightMainRow? =
|
||||
@@ -711,6 +824,7 @@ class JdbcFlightStateRepository(
|
||||
@Requires(missingProperty = "msgx.stubs")
|
||||
class JdbcSnapshotLogRepository(
|
||||
private val ds: DataSource,
|
||||
private val clock: Clock,
|
||||
) : SnapshotLogRepository {
|
||||
/** 只追加写；这里不抛异常，写失败由调用方捕获并记为指标。 */
|
||||
override fun append(entry: SnapshotLogEntry) {
|
||||
@@ -730,7 +844,7 @@ class JdbcSnapshotLogRepository(
|
||||
ps.setString(8, entry.result.name)
|
||||
ps.setString(9, entry.flags.joinToString(",") { it.name })
|
||||
ps.setString(10, entry.archiveKey)
|
||||
ps.setTimestamp(11, Instant.now().toSqlTimestamp())
|
||||
ps.setTimestamp(11, clock.instant().toSqlTimestamp())
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -741,6 +855,7 @@ class JdbcSnapshotLogRepository(
|
||||
@Requires(missingProperty = "msgx.stubs")
|
||||
class JdbcReqTrackRepository(
|
||||
private val ds: DataSource,
|
||||
private val clock: Clock,
|
||||
) : ReqTrackRepository {
|
||||
override fun insert(reqType: String, operationDay: LocalDate, sender: String): Long =
|
||||
ds.updateReturningLong(
|
||||
@@ -749,7 +864,7 @@ class JdbcReqTrackRepository(
|
||||
ps.setString(1, reqType)
|
||||
ps.setDate(2, java.sql.Date.valueOf(operationDay))
|
||||
ps.setString(3, sender)
|
||||
ps.setTimestamp(4, Instant.now().toSqlTimestamp())
|
||||
ps.setTimestamp(4, clock.instant().toSqlTimestamp())
|
||||
},
|
||||
)
|
||||
|
||||
@@ -789,7 +904,7 @@ class JdbcReqTrackRepository(
|
||||
)
|
||||
""".trimIndent(),
|
||||
{ ps ->
|
||||
ps.setTimestamp(1, Instant.now().toSqlTimestamp())
|
||||
ps.setTimestamp(1, clock.instant().toSqlTimestamp())
|
||||
ps.setString(2, reqType)
|
||||
ps.setDate(3, java.sql.Date.valueOf(operationDay))
|
||||
ps.setString(4, sender)
|
||||
|
||||
+6
@@ -24,6 +24,12 @@ class MailboxDataSourceFactory {
|
||||
driverClassName = cfg.driverClassName
|
||||
maximumPoolSize = 5
|
||||
poolName = "mailbox"
|
||||
// 有界外部调用：连接、取连接、校验都必须有上限，否则一次网络黑洞会永久挂住调用线程。
|
||||
connectionTimeout = cfg.poolConnectionTimeoutMs
|
||||
validationTimeout = cfg.poolValidationTimeoutMs
|
||||
// 驱动级超时通过 dataSourceProperties 透传给 Connector/J（socketTimeout 默认 0 = 无限等待）。
|
||||
addDataSourceProperty("connectTimeout", cfg.connectTimeoutMs.toString())
|
||||
addDataSourceProperty("socketTimeout", cfg.socketTimeoutMs.toString())
|
||||
}
|
||||
return HikariDataSource(hikari)
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ class ProcFailure(
|
||||
attempts = attempts,
|
||||
errorClass = ErrorClass.EXHAUSTED,
|
||||
lastError = "$reason; attempts=$attempts",
|
||||
now = scheduler.now(),
|
||||
)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -10,11 +10,16 @@ import jakarta.inject.Singleton
|
||||
*
|
||||
* 只有"再试一次有可能成功"的错误类才放行。报文本身不合法的（MALFORMED）重放多少次都一样，
|
||||
* 所以不在白名单里；没列出的错误类也一律不放行。
|
||||
*
|
||||
* 与回填共用同一个 [MessageLifecycleGate]，**必须由容器注入同一个单例**：若两者各持一把锁，
|
||||
* 互斥失效，"旧回填给已重新入队的消息写标记"的窗口就会重新打开。
|
||||
* 装配正确性由 `PipelineSmokeTest` 的同例断言守（不依赖 Kotlin 默认参数值）。
|
||||
*/
|
||||
@Singleton
|
||||
class ReplayService(
|
||||
private val procState: ProcStateRepository,
|
||||
private val lifecycleGate: MessageLifecycleGate = MessageLifecycleGate(),
|
||||
/** 与回填共用的互斥门；`internal` 以便装配测试断言两者拿到同一实例。 */
|
||||
internal val lifecycleGate: MessageLifecycleGate,
|
||||
) {
|
||||
private val log = org.slf4j.LoggerFactory.getLogger(ReplayService::class.java)
|
||||
/** 可以重放的错误类：解码逻辑修好后能过、处理器补齐后能过、基础设施抖动已恢复、以及重试耗尽但人工复核认为还能再试的。 */
|
||||
|
||||
@@ -130,6 +130,8 @@ class StubProcState : ProcStateRepository {
|
||||
backfillNextAt = now,
|
||||
backfillAttempts = 0,
|
||||
backfillError = null,
|
||||
backfillAbandonedAt = null,
|
||||
backfillAbandonedReason = null,
|
||||
updatedAt = now,
|
||||
)
|
||||
}
|
||||
@@ -151,23 +153,54 @@ class StubProcState : ProcStateRepository {
|
||||
}
|
||||
}
|
||||
|
||||
override fun markBackfillAbandoned(msgId: Long, reason: String, now: Instant): Boolean {
|
||||
val old = rows[msgId] ?: return false
|
||||
if (old.backfillAt != null || old.backfillAbandonedAt != null) return false
|
||||
// 放弃 ≠ 标记已确认：这里**不**设 backfillAt，清除前提因此仍不成立。
|
||||
rows[msgId] = old.copy(
|
||||
backfillAbandonedAt = now,
|
||||
backfillAbandonedReason = reason,
|
||||
backfillNextAt = null,
|
||||
updatedAt = now,
|
||||
)
|
||||
return true
|
||||
}
|
||||
|
||||
override fun reopenBackfill(msgId: Long, now: Instant): Boolean {
|
||||
val old = rows[msgId] ?: return false
|
||||
if (old.backfillAt != null || old.backfillAbandonedAt == null) return false
|
||||
rows[msgId] = old.copy(
|
||||
backfillAbandonedAt = null,
|
||||
backfillAbandonedReason = null,
|
||||
backfillNextAt = now,
|
||||
updatedAt = now,
|
||||
)
|
||||
return true
|
||||
}
|
||||
|
||||
override fun findBackfillDue(now: Instant, overdueBefore: Instant, limit: Int): List<BackfillDue> =
|
||||
rows.values
|
||||
.filter { it.state.isTerminal() && it.backfillAt == null }
|
||||
.filter { it.state.isTerminal() && it.backfillAt == null && it.backfillAbandonedAt == null }
|
||||
.filter {
|
||||
it.backfillNextAt == null || it.backfillNextAt <= now ||
|
||||
(it.receivedAt != null && it.receivedAt < overdueBefore)
|
||||
}
|
||||
.sortedBy { it.msgId }
|
||||
// 公平轮转：先按已尝试次数，再按 msg_id。只按 msg_id 会让最旧的一批永久失败行占满批次。
|
||||
.sortedWith(compareBy({ it.backfillAttempts }, { it.msgId }))
|
||||
.take(limit)
|
||||
.map { BackfillDue(it.msgId, it.backfillAttempts) }
|
||||
|
||||
override fun hasAny(): Boolean = rows.isNotEmpty()
|
||||
|
||||
override fun backlog(): Backlog {
|
||||
val unfinished = rows.values.filter { it.state == ProcStatus.PENDING || it.state == ProcStatus.FAILED }
|
||||
val unmarked = rows.values.filter { it.state.isTerminal() && it.backfillAt == null }
|
||||
return Backlog(
|
||||
unfinished = unfinished.size,
|
||||
oldestReceivedAt = unfinished.mapNotNull { it.receivedAt }.minOrNull(),
|
||||
unmarkedTerminal = rows.values.count { it.state.isTerminal() && it.backfillAt == null },
|
||||
unmarkedTerminal = unmarked.size,
|
||||
abandonedBackfill = unmarked.count { it.backfillAbandonedAt != null },
|
||||
oldestUnmarkedAt = unmarked.filter { it.backfillAbandonedAt == null }.mapNotNull { it.receivedAt }.minOrNull(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -404,6 +437,10 @@ class StubInbox : CminmsgInboxRepository {
|
||||
|
||||
override fun maxId(): Long? = raws.keys.maxOrNull()
|
||||
|
||||
override fun minId(): Long? = raws.keys.minOrNull()
|
||||
|
||||
override fun existingIds(msgIds: Collection<Long>): Set<Long> = msgIds.filter { raws.containsKey(it) }.toSet()
|
||||
|
||||
/** 只写还没有标记的行，并区分已有标记与行缺失。 */
|
||||
override fun markProcessedIfUnmarked(msgId: Long, value: String): MailboxMarkResult {
|
||||
if (!raws.containsKey(msgId)) return MailboxMarkResult.MISSING
|
||||
@@ -444,7 +481,12 @@ class StubInboxCursor : InboxCursorRepository {
|
||||
override fun load(): InboxCursorRepository.Cursor = cursor
|
||||
|
||||
override fun save(cursor: InboxCursorRepository.Cursor) {
|
||||
this.cursor = cursor
|
||||
// 普通轮次只推进水位与空洞计时；播种标记由 markSeeded 负责，不能被这里抹掉。
|
||||
this.cursor = this.cursor.copy(committedUpTo = cursor.committedUpTo, holeSince = cursor.holeSince)
|
||||
}
|
||||
|
||||
override fun markSeeded(committedUpTo: Long, now: Instant) {
|
||||
cursor = InboxCursorRepository.Cursor(committedUpTo = committedUpTo, holeSince = null, seededAt = now)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user