feat(metrics): 投递失败与死信可观测指标,修 flushSchd 首次失败原因丢失(ACM2-98)
- PipelineCounters 按 target 累计发送失败;Dispatcher.retryOrDead 记账(OPS-2)
- 新增 deadByTarget 查询与 DeliveryDeadSnapshotProvider(TTL 缓存),死信存量按 target 暴露
- /metrics 注册 send_failures.total{target} 与 dead{target},登记进 reference.md「指标与健康」
- flushSchd 失败改记真实异常原因,不再退回固定 send-failed
This commit is contained in:
@@ -45,6 +45,7 @@ class Dispatcher(
|
||||
private val port: DeliveryPort,
|
||||
private val props: PipelineProps,
|
||||
private val scheduler: FailureScheduler,
|
||||
private val counters: com.gzzn.omms.msgexchange.infra.metrics.PipelineCounters,
|
||||
) {
|
||||
private val log = org.slf4j.LoggerFactory.getLogger(Dispatcher::class.java)
|
||||
|
||||
@@ -162,7 +163,7 @@ class Dispatcher(
|
||||
lastFlush = scheduler.now()
|
||||
return
|
||||
}
|
||||
val failures = mutableListOf<MsgEvent>()
|
||||
val failures = mutableListOf<Pair<MsgEvent, String>>()
|
||||
for (e in batch) {
|
||||
try {
|
||||
when (e.eventType) {
|
||||
@@ -172,15 +173,17 @@ class Dispatcher(
|
||||
// 条件确认:读取时刻的代次(EVENT_ID + STATE_VERSION)被新写入覆盖时不标记,留待下一轮重发。
|
||||
e.eventId?.let { msgEvents.markSentIfVersion(it, e.stateVersion, scheduler.now()) }
|
||||
} catch (ex: Exception) {
|
||||
failures.add(e)
|
||||
// 首次失败就带上真实原因;不能退回字面量把根因抹掉
|
||||
failures += e to (ex.message ?: ex.javaClass.simpleName)
|
||||
}
|
||||
}
|
||||
failures.forEach { retryOrDead(it, it.lastError ?: "send-failed") }
|
||||
failures.forEach { (e, error) -> retryOrDead(e, error) }
|
||||
lastFlush = scheduler.now()
|
||||
}
|
||||
|
||||
/** 一条事件发失败之后怎么走:重试次数加一,到上限就标成 DEAD(EXHAUSTED) 留作死信(次数落库便于追查),否则按退避推到下次再发。 */
|
||||
private fun retryOrDead(e: MsgEvent, lastError: String) {
|
||||
counters.sendFailedAdd(e.target) // OPS-2:发送失败按 target 计数,供 /metrics 告警
|
||||
val eventId = e.eventId ?: return
|
||||
val attempts = e.attempts + 1
|
||||
if (scheduler.exhausted(attempts)) {
|
||||
|
||||
Reference in New Issue
Block a user