fix(pipeline): 修复取报-处理-回填链路的超时、吞吐、回填与 FIFO 问题
外部调用(ACM2-33) - 共享 MySQL 与自有 PG 全部补有界超时:驱动 connect/socket 超时 + Hikari 池超时 (注意 Micronaut 的 Hikari 项是毫秒数,不是 Duration 字面量) - 删除主泵/毒丸路径的 inline 回填:跨库写不再占用 FIFO 关键路径,回填统一由扫描驱动 - 游标自愈:save() 改为「先 UPDATE、缺行 INSERT」,缺行只记一次 ERROR - Pump/Dispatcher 外层 catch 补 error 日志与失败计数 投递吞吐(ACM2-34) - Dispatcher 改批量领取;队头退避未到期或首条发送失败即停止本轮(保持目标内保序) - 有活不再 sleep;删除不可达的 state==SENT 死条件;markSent 移入分支; superseded 清理加轮数上限与空 eventId 保护 回填闭环(ACM2-36,V4) - MISSING(运行时确认行不存在)立即放弃自动重试并告警;暂时性故障达上限后停止自动重试 - 新增 reopen 人工恢复入口;放弃 ≠ 标记已确认(BACKFILL_AT 仍为空,清除前提不成立) - 扫描改 (BACKFILL_ATTEMPTS, MSG_ID) 公平轮转并排除放弃行,消除全局回填饥饿 - V4 只加字段与必要索引,不按年龄做任何存量推断 切流播种(ACM2-35,V5) - cutover-watermark 四模式(min/zero/max/显式 ID),默认不播种、代码不做默认选择 - 升级实例拒绝重新播种(SEEDED_AT 为 NULL ≠ 从未消费);播种与水位同语句落库 - 非法取值由启动自检挡下 错误分类与入口契约(ACM2-37) - 未知 SCHD 子类型改为 UNSUPPORTED,不再静默当全量日计划合并 - ADFT 运营日冲突改走 ProtocolViolation → DEAD(PROTOCOL) - 兼容入口 receivedAt 缺失回退到注入 Clock;MessageLifecycleGate 强制注入 + 装配断言 - FIFO:主泵只领取 msgId ≤ W,兼容入口登记的行在水位追平前不被领取 时间源与可观测(ACM2-38 / ACM2-41 阶段 0) - 仓储/处理器/作业全部经注入 Clock;移除 markTerminal/markBackfilled 的 Instant.now() 默认值 - 退避表档位与 max-attempts 对齐并加启动自检 - Micrometer 7 个 gauge(@Context 急切注册)+ /health 与 /metrics 共用积压快照缓存 - 只读迟到检测:监视被放行的空洞 ID 是否后来真的出现,只计数告警、不补入队 Plane: ACM2-33 ACM2-34 ACM2-35 ACM2-36 ACM2-37 ACM2-38 ACM2-41 Tests: 88 → 120(1 skipped 需真实 PG)
This commit is contained in:
@@ -0,0 +1,48 @@
|
||||
package com.gzzn.omms.msgexchange.infra.health
|
||||
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.Backlog
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
|
||||
import io.micronaut.context.BeanProvider
|
||||
import io.micronaut.context.annotation.Value
|
||||
import jakarta.inject.Singleton
|
||||
import java.time.Clock
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
|
||||
/**
|
||||
* 处理侧积压快照的唯一取数点,带 TTL 缓存。
|
||||
*
|
||||
* `backlog()` 是 `PROC_STATE` 的**全表聚合**;健康检查(`/health`)与指标抓取(`/metrics`)
|
||||
* 都可能被高频调用,因此两者共用同一份缓存,避免把数据库拖慢。代价是数字最多滞后一个
|
||||
* 缓存窗口——对积压/信龄这类告警信号足够。
|
||||
*
|
||||
* 缓存窗口由 `msgx.health.backlog-cache-ttl-ms` 配置(默认 30 秒;设为 0 表示不缓存)。
|
||||
* 需要实时精确计数时应改为维护计数器,而不是把窗口调到 0。
|
||||
*/
|
||||
@Singleton
|
||||
class BacklogSnapshotProvider(
|
||||
private val procState: BeanProvider<ProcStateRepository>,
|
||||
private val clock: Clock,
|
||||
@Value("\${msgx.health.backlog-cache-ttl-ms:30000}") private val ttlMs: Long = 30_000,
|
||||
) {
|
||||
@Volatile
|
||||
private var cached: Backlog? = null
|
||||
|
||||
@Volatile
|
||||
private var cachedAt: Instant? = null
|
||||
|
||||
/** @return 积压快照;仓储未绑定(例如 stub 关闭)时返回 null,由调用方按"未绑定"处理。 */
|
||||
fun snapshot(): Backlog? {
|
||||
if (!procState.isPresent) return null
|
||||
val now = clock.instant()
|
||||
val at = cachedAt
|
||||
val value = cached
|
||||
if (ttlMs > 0 && at != null && value != null && Duration.between(at, now).toMillis() < ttlMs) {
|
||||
return value
|
||||
}
|
||||
val fresh = procState.get().backlog()
|
||||
cached = fresh
|
||||
cachedAt = now
|
||||
return fresh
|
||||
}
|
||||
}
|
||||
+20
-5
@@ -1,5 +1,6 @@
|
||||
package com.gzzn.omms.msgexchange.infra.health
|
||||
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.Backlog
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.InboxCursorRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
|
||||
@@ -10,6 +11,7 @@ import io.micronaut.management.health.indicator.HealthIndicator
|
||||
import io.micronaut.management.health.indicator.HealthResult
|
||||
import jakarta.inject.Singleton
|
||||
import org.reactivestreams.Publisher
|
||||
import java.time.Clock
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
|
||||
@@ -28,6 +30,8 @@ class InboxLifecycleHealthIndicator(
|
||||
private val procState: BeanProvider<ProcStateRepository>,
|
||||
private val cursor: BeanProvider<InboxCursorRepository>,
|
||||
private val mailbox: BeanProvider<CminmsgInboxRepository>,
|
||||
private val clock: Clock,
|
||||
private val backlogs: BacklogSnapshotProvider,
|
||||
) : HealthIndicator {
|
||||
|
||||
override fun getResult(): Publisher<HealthResult> =
|
||||
@@ -37,6 +41,9 @@ class InboxLifecycleHealthIndicator(
|
||||
procState = if (procState.isPresent) procState.get() else null,
|
||||
cursor = if (cursor.isPresent) cursor.get() else null,
|
||||
mailbox = if (mailbox.isPresent) mailbox.get() else null,
|
||||
now = clock.instant(),
|
||||
// 与 /metrics 共用同一份 30 秒缓存:两者都不该把全表聚合打成高频查询。
|
||||
backlog = backlogs.snapshot(),
|
||||
)
|
||||
}.getOrElse { down(it) },
|
||||
)
|
||||
@@ -47,23 +54,31 @@ internal fun lifecycleHealth(
|
||||
procState: ProcStateRepository?,
|
||||
cursor: InboxCursorRepository?,
|
||||
mailbox: CminmsgInboxRepository?,
|
||||
now: Instant = Instant.now(),
|
||||
now: Instant,
|
||||
/** 允许调用方传入缓存/已算好的积压快照;为空时现查(`backlog()` 是全表聚合)。 */
|
||||
backlog: Backlog? = null,
|
||||
): HealthResult {
|
||||
if (procState == null) {
|
||||
return HealthResult.builder(NAME).status(HealthStatus.UP)
|
||||
.details(mapOf("message" to "proc_state repository not bound (stub off, impl pending)"))
|
||||
.build()
|
||||
}
|
||||
val backlog = procState.backlog()
|
||||
val snapshot = backlog ?: procState.backlog()
|
||||
val watermark = cursor?.load()?.committedUpTo
|
||||
val maxId = runCatching { mailbox?.maxId() }.getOrNull()
|
||||
return HealthResult.builder(NAME).status(HealthStatus.UP).details(
|
||||
linkedMapOf<String, Any>(
|
||||
"backlog" to backlog.unfinished,
|
||||
"backlog" to snapshot.unfinished,
|
||||
"oldestUnprocessedSeconds" to (
|
||||
backlog.oldestReceivedAt?.let { Duration.between(it, now).seconds } ?: -1L
|
||||
snapshot.oldestReceivedAt?.let { Duration.between(it, now).seconds } ?: -1L
|
||||
),
|
||||
"unmarkedTerminal" to snapshot.unmarkedTerminal,
|
||||
// 已放弃自动回填的条数:**不等于**标记已完成,需要人工对账;非 0 应告警。
|
||||
"backfillAbandoned" to snapshot.abandonedBackfill,
|
||||
// 最老一条"仍待自动回填"记录的年龄(秒):回填延迟的真实观测值。
|
||||
"oldestUnmarkedBackfillSeconds" to (
|
||||
snapshot.oldestUnmarkedAt?.let { Duration.between(it, now).seconds } ?: -1L
|
||||
),
|
||||
"unmarkedTerminal" to backlog.unmarkedTerminal,
|
||||
"watermark" to (watermark ?: -1L),
|
||||
"watermarkLag" to if (watermark != null && maxId != null) maxId - watermark else -1L,
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user