feat(metrics): 投递失败与死信可观测指标,修 flushSchd 首次失败原因丢失(ACM2-98)

- PipelineCounters 按 target 累计发送失败;Dispatcher.retryOrDead 记账(OPS-2)
- 新增 deadByTarget 查询与 DeliveryDeadSnapshotProvider(TTL 缓存),死信存量按 target 暴露
- /metrics 注册 send_failures.total{target} 与 dead{target},登记进 reference.md「指标与健康」
- flushSchd 失败改记真实异常原因,不再退回固定 send-failed
This commit is contained in:
windyboy
2026-09-21 12:40:53 +08:00
parent a1c56f546f
commit 9bf879ff76
10 changed files with 142 additions and 6 deletions
@@ -8,6 +8,7 @@ import com.gzzn.omms.msgexchange.domain.EventType
import com.gzzn.omms.msgexchange.domain.MsgEvent
import com.gzzn.omms.msgexchange.domain.Targets
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
import com.gzzn.omms.msgexchange.infra.metrics.PipelineCounters
import com.gzzn.omms.msgexchange.infra.retry.FailureScheduler
import com.gzzn.omms.msgexchange.infra.stub.StubDeliveryPort
import com.gzzn.omms.msgexchange.infra.stub.StubMsgEvents
@@ -28,8 +29,12 @@ class DispatcherTickTest {
private val clock = MutableClock(MutableClock.BASE)
private fun dispatcher(repo: MsgEventRepository, port: DeliveryPort, p: PipelineProps = PipelineProps()): Dispatcher =
Dispatcher(repo, port, p, FailureScheduler(p, clock))
private fun dispatcher(
repo: MsgEventRepository,
port: DeliveryPort,
p: PipelineProps = PipelineProps(),
counters: PipelineCounters = PipelineCounters(),
): Dispatcher = Dispatcher(repo, port, p, FailureScheduler(p, clock), counters)
private fun ev(id: Long, target: String, key: String, payload: String, version: Long = 0) =
MsgEvent(eventId = id, target = target, partitionKey = key, stateVersion = version, payloadJson = payload, createdAt = MutableClock.BASE)
@@ -123,11 +128,14 @@ class DispatcherTickTest {
fun `schd send failure schedules backoff and goes DEAD at limit`() {
val repo = StubMsgEvents()
val p = PipelineProps()
val d = dispatcher(repo, FailingSchdPort(), p)
val counters = PipelineCounters()
val d = dispatcher(repo, FailingSchdPort(), p, counters)
repo.insertAll(listOf(ev(1, Targets.KAFKA_SCHD, "F1", """{"v":"1"}""", 1)))
d.flushSchd()
assertEquals(1, repo.rows[1]!!.attempts)
// OPS-2:发送失败按 target 计数,且失败原因不是固定字面量
assertEquals(1L, counters.sendFailedCount(Targets.KAFKA_SCHD))
repeat(p.pipeline.maxAttempts - 1) {
clock.advance(p.pipeline.backoffFor(repo.rows[1]!!.attempts) + 1)
@@ -135,6 +143,7 @@ class DispatcherTickTest {
}
val final = repo.rows[1]!!
assertTrue(final.attempts >= p.pipeline.maxAttempts || final.state.name == "DEAD")
assertEquals(0L, counters.sendFailedCount(Targets.KAFKA_MSG)) // msg 目标不受 schd 失败影响
}
@Test
@@ -1,6 +1,10 @@
package com.gzzn.omms.msgexchange.infra.metrics
import com.gzzn.omms.msgexchange.domain.ErrorClass
import com.gzzn.omms.msgexchange.domain.MsgEvent
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.domain.Targets
import com.gzzn.omms.msgexchange.infra.stub.StubMsgEvents
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
import io.micrometer.core.instrument.MeterRegistry
import io.micronaut.context.ApplicationContext
@@ -74,6 +78,34 @@ class PipelineMetricsTest {
assertEquals(vipfBefore + 1.0, gauge("msgx.pipeline.codec.vipf_seen.total"), 0.001)
}
@Test
fun `delivery send-failure and dead-letter gauges are registered per target`() {
val counters = ctx.getBean(PipelineCounters::class.java)
val events = ctx.getBean(StubMsgEvents::class.java)
val failuresBefore = taggedGauge("msgx.pipeline.delivery.send_failures.total", Targets.KAFKA_SCHD)
counters.sendFailedAdd(Targets.KAFKA_SCHD)
val ids = events.insertAll(
listOf(
MsgEvent(target = Targets.KAFKA_MSG, partitionKey = "F1", stateVersion = 1, payloadJson = "{}", createdAt = Instant.now()),
),
)
events.markDead(ids.single(), ErrorClass.EXHAUSTED, "boom", attempts = 5)
assertEquals(
failuresBefore + 1.0,
taggedGauge("msgx.pipeline.delivery.send_failures.total", Targets.KAFKA_SCHD),
0.001,
)
assertEquals(1.0, taggedGauge("msgx.pipeline.delivery.dead", Targets.KAFKA_MSG), 0.001)
assertEquals(0.0, taggedGauge("msgx.pipeline.delivery.dead", Targets.KAFKA_SCHD), 0.001)
}
private fun gauge(name: String): Double =
requireNotNull(registry.find(name).gauge()) { "gauge not registered: $name" }.value()
private fun taggedGauge(name: String, target: String): Double =
requireNotNull(registry.find(name).tag("target", target).gauge()) {
"gauge not registered: $name(target=$target)"
}.value()
}