feat(metrics): 投递失败与死信可观测指标,修 flushSchd 首次失败原因丢失(ACM2-98)

- PipelineCounters 按 target 累计发送失败;Dispatcher.retryOrDead 记账(OPS-2)
- 新增 deadByTarget 查询与 DeliveryDeadSnapshotProvider(TTL 缓存),死信存量按 target 暴露
- /metrics 注册 send_failures.total{target} 与 dead{target},登记进 reference.md「指标与健康」
- flushSchd 失败改记真实异常原因,不再退回固定 send-failed
This commit is contained in:
windyboy
2026-09-21 12:40:53 +08:00
parent a1c56f546f
commit 9bf879ff76
10 changed files with 142 additions and 6 deletions
@@ -45,6 +45,7 @@ class Dispatcher(
private val port: DeliveryPort,
private val props: PipelineProps,
private val scheduler: FailureScheduler,
private val counters: com.gzzn.omms.msgexchange.infra.metrics.PipelineCounters,
) {
private val log = org.slf4j.LoggerFactory.getLogger(Dispatcher::class.java)
@@ -162,7 +163,7 @@ class Dispatcher(
lastFlush = scheduler.now()
return
}
val failures = mutableListOf<MsgEvent>()
val failures = mutableListOf<Pair<MsgEvent, String>>()
for (e in batch) {
try {
when (e.eventType) {
@@ -172,15 +173,17 @@ class Dispatcher(
// 条件确认:读取时刻的代次(EVENT_ID + STATE_VERSION)被新写入覆盖时不标记,留待下一轮重发。
e.eventId?.let { msgEvents.markSentIfVersion(it, e.stateVersion, scheduler.now()) }
} catch (ex: Exception) {
failures.add(e)
// 首次失败就带上真实原因;不能退回字面量把根因抹掉
failures += e to (ex.message ?: ex.javaClass.simpleName)
}
}
failures.forEach { retryOrDead(it, it.lastError ?: "send-failed") }
failures.forEach { (e, error) -> retryOrDead(e, error) }
lastFlush = scheduler.now()
}
/** 一条事件发失败之后怎么走:重试次数加一,到上限就标成 DEAD(EXHAUSTED) 留作死信(次数落库便于追查),否则按退避推到下次再发。 */
private fun retryOrDead(e: MsgEvent, lastError: String) {
counters.sendFailedAdd(e.target) // OPS-2:发送失败按 target 计数,供 /metrics 告警
val eventId = e.eventId ?: return
val attempts = e.attempts + 1
if (scheduler.exhausted(attempts)) {