feat(metrics): 投递失败与死信可观测指标,修 flushSchd 首次失败原因丢失(ACM2-98)
- PipelineCounters 按 target 累计发送失败;Dispatcher.retryOrDead 记账(OPS-2)
- 新增 deadByTarget 查询与 DeliveryDeadSnapshotProvider(TTL 缓存),死信存量按 target 暴露
- /metrics 注册 send_failures.total{target} 与 dead{target},登记进 reference.md「指标与健康」
- flushSchd 失败改记真实异常原因,不再退回固定 send-failed
This commit is contained in:
@@ -8,6 +8,7 @@ import com.gzzn.omms.msgexchange.domain.EventType
|
||||
import com.gzzn.omms.msgexchange.domain.MsgEvent
|
||||
import com.gzzn.omms.msgexchange.domain.Targets
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
|
||||
import com.gzzn.omms.msgexchange.infra.metrics.PipelineCounters
|
||||
import com.gzzn.omms.msgexchange.infra.retry.FailureScheduler
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubDeliveryPort
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubMsgEvents
|
||||
@@ -28,8 +29,12 @@ class DispatcherTickTest {
|
||||
|
||||
private val clock = MutableClock(MutableClock.BASE)
|
||||
|
||||
private fun dispatcher(repo: MsgEventRepository, port: DeliveryPort, p: PipelineProps = PipelineProps()): Dispatcher =
|
||||
Dispatcher(repo, port, p, FailureScheduler(p, clock))
|
||||
private fun dispatcher(
|
||||
repo: MsgEventRepository,
|
||||
port: DeliveryPort,
|
||||
p: PipelineProps = PipelineProps(),
|
||||
counters: PipelineCounters = PipelineCounters(),
|
||||
): Dispatcher = Dispatcher(repo, port, p, FailureScheduler(p, clock), counters)
|
||||
|
||||
private fun ev(id: Long, target: String, key: String, payload: String, version: Long = 0) =
|
||||
MsgEvent(eventId = id, target = target, partitionKey = key, stateVersion = version, payloadJson = payload, createdAt = MutableClock.BASE)
|
||||
@@ -123,11 +128,14 @@ class DispatcherTickTest {
|
||||
fun `schd send failure schedules backoff and goes DEAD at limit`() {
|
||||
val repo = StubMsgEvents()
|
||||
val p = PipelineProps()
|
||||
val d = dispatcher(repo, FailingSchdPort(), p)
|
||||
val counters = PipelineCounters()
|
||||
val d = dispatcher(repo, FailingSchdPort(), p, counters)
|
||||
repo.insertAll(listOf(ev(1, Targets.KAFKA_SCHD, "F1", """{"v":"1"}""", 1)))
|
||||
|
||||
d.flushSchd()
|
||||
assertEquals(1, repo.rows[1]!!.attempts)
|
||||
// OPS-2:发送失败按 target 计数,且失败原因不是固定字面量
|
||||
assertEquals(1L, counters.sendFailedCount(Targets.KAFKA_SCHD))
|
||||
|
||||
repeat(p.pipeline.maxAttempts - 1) {
|
||||
clock.advance(p.pipeline.backoffFor(repo.rows[1]!!.attempts) + 1)
|
||||
@@ -135,6 +143,7 @@ class DispatcherTickTest {
|
||||
}
|
||||
val final = repo.rows[1]!!
|
||||
assertTrue(final.attempts >= p.pipeline.maxAttempts || final.state.name == "DEAD")
|
||||
assertEquals(0L, counters.sendFailedCount(Targets.KAFKA_MSG)) // msg 目标不受 schd 失败影响
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
package com.gzzn.omms.msgexchange.infra.metrics
|
||||
|
||||
import com.gzzn.omms.msgexchange.domain.ErrorClass
|
||||
import com.gzzn.omms.msgexchange.domain.MsgEvent
|
||||
import com.gzzn.omms.msgexchange.domain.ProcStatus
|
||||
import com.gzzn.omms.msgexchange.domain.Targets
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubMsgEvents
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
|
||||
import io.micrometer.core.instrument.MeterRegistry
|
||||
import io.micronaut.context.ApplicationContext
|
||||
@@ -74,6 +78,34 @@ class PipelineMetricsTest {
|
||||
assertEquals(vipfBefore + 1.0, gauge("msgx.pipeline.codec.vipf_seen.total"), 0.001)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `delivery send-failure and dead-letter gauges are registered per target`() {
|
||||
val counters = ctx.getBean(PipelineCounters::class.java)
|
||||
val events = ctx.getBean(StubMsgEvents::class.java)
|
||||
val failuresBefore = taggedGauge("msgx.pipeline.delivery.send_failures.total", Targets.KAFKA_SCHD)
|
||||
|
||||
counters.sendFailedAdd(Targets.KAFKA_SCHD)
|
||||
val ids = events.insertAll(
|
||||
listOf(
|
||||
MsgEvent(target = Targets.KAFKA_MSG, partitionKey = "F1", stateVersion = 1, payloadJson = "{}", createdAt = Instant.now()),
|
||||
),
|
||||
)
|
||||
events.markDead(ids.single(), ErrorClass.EXHAUSTED, "boom", attempts = 5)
|
||||
|
||||
assertEquals(
|
||||
failuresBefore + 1.0,
|
||||
taggedGauge("msgx.pipeline.delivery.send_failures.total", Targets.KAFKA_SCHD),
|
||||
0.001,
|
||||
)
|
||||
assertEquals(1.0, taggedGauge("msgx.pipeline.delivery.dead", Targets.KAFKA_MSG), 0.001)
|
||||
assertEquals(0.0, taggedGauge("msgx.pipeline.delivery.dead", Targets.KAFKA_SCHD), 0.001)
|
||||
}
|
||||
|
||||
private fun gauge(name: String): Double =
|
||||
requireNotNull(registry.find(name).gauge()) { "gauge not registered: $name" }.value()
|
||||
|
||||
private fun taggedGauge(name: String, target: String): Double =
|
||||
requireNotNull(registry.find(name).tag("target", target).gauge()) {
|
||||
"gauge not registered: $name(target=$target)"
|
||||
}.value()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user