feat(jobs): expose job heartbeat and scan backlog
新增 JobActivity 持有作业心跳与扫描积压,JobRunner 每轮 tick 上报(失败只计数、不刷新心跳),PipelineMetrics 注册 msgx.pipeline.job.heartbeat_age_seconds / ticks.total / failures.total / last_sweep_selected,新增 JobRunnerHealthIndicator(未启动 UP 并标 started=false;已启动但心跳缺失或超过 3×扫描周期 DOWN)。对应 reference 作业健康段与 [G-JOB-HEARTBEAT]。 验证:./gradlew test 145 tests / 0 fail / 0 skipped(新增 JobRunnerTest 2、JobRunnerHealthIndicatorTest 4、PipelineMetrics 指标注册 1)。
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
package com.gzzn.omms.msgexchange.infra.metrics
|
||||
|
||||
import jakarta.inject.Singleton
|
||||
import java.time.Instant
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 维护作业的进程内活动事实(作业心跳、回填扫描积压),供 `/metrics` 与 `/health` 取用。
|
||||
*
|
||||
* 放在 `infra` 是为了让 `jobs`(写)与指标/健康(读)都零依赖:作业线程只更新这个持有者,
|
||||
* 指标名与暴露方式由 [PipelineMetrics] 决定,存活判定由 `JobRunnerHealthIndicator` 决定。
|
||||
* 计数在重启后归零;跨重启的累计值由指标后端聚合,这里不做持久化。
|
||||
*/
|
||||
@Singleton
|
||||
class JobActivity {
|
||||
@Volatile
|
||||
private var started = false
|
||||
|
||||
@Volatile
|
||||
private var lastTickAt: Instant? = null
|
||||
|
||||
@Volatile
|
||||
private var lastTickMillis = 0L
|
||||
|
||||
/** 上一轮回填扫描选中的待办条数(扫描积压的即时值)。 */
|
||||
@Volatile
|
||||
var lastSweepSelected = 0
|
||||
private set
|
||||
|
||||
private val ticks = AtomicLong(0)
|
||||
private val failures = AtomicLong(0)
|
||||
|
||||
fun started() {
|
||||
started = true
|
||||
}
|
||||
|
||||
fun stopped() {
|
||||
started = false
|
||||
}
|
||||
|
||||
/** 一次作业 tick 正常跑完:记录心跳时刻、耗时与本次扫描积压。 */
|
||||
fun tickFinished(now: Instant, durationMillis: Long, sweepSelected: Int) {
|
||||
lastTickAt = now
|
||||
lastTickMillis = durationMillis
|
||||
lastSweepSelected = sweepSelected
|
||||
ticks.incrementAndGet()
|
||||
}
|
||||
|
||||
/** 一次作业 tick 抛错:只累计失败,**不更新心跳时刻**——停摆由心跳年龄暴露。 */
|
||||
fun tickFailed() {
|
||||
failures.incrementAndGet()
|
||||
}
|
||||
|
||||
fun snapshot(): Snapshot =
|
||||
Snapshot(started, lastTickAt, lastTickMillis, lastSweepSelected, ticks.get(), failures.get())
|
||||
|
||||
data class Snapshot(
|
||||
val started: Boolean,
|
||||
val lastTickAt: Instant?,
|
||||
val lastTickMillis: Long,
|
||||
val lastSweepSelected: Int,
|
||||
val ticks: Long,
|
||||
val failures: Long,
|
||||
)
|
||||
}
|
||||
@@ -23,6 +23,9 @@ import java.time.Duration
|
||||
* - `msgx.pipeline.backfill.oldest_unmarked_seconds`:最老一条仍待自动回填的年龄
|
||||
* - `msgx.pipeline.watermark.lag`:水位落后信箱最新 ID 的距离
|
||||
* - `msgx.pipeline.hole.aged_out.total`:永久空洞放行次数
|
||||
* - `msgx.pipeline.job.heartbeat_age_seconds`:距上一次作业 tick 完成的秒数(未跑过为 -1)
|
||||
* - `msgx.pipeline.job.ticks.total` / `msgx.pipeline.job.failures.total`:作业 tick 完成/抛错次数
|
||||
* - `msgx.pipeline.job.last_sweep_selected`:上一轮回填扫描选中的待办条数(扫描积压)
|
||||
*
|
||||
* 取数统一走 [BacklogSnapshotProvider](30 秒 TTL),因此指标抓取不会打穿数据库。
|
||||
* 无法取数时以 `NaN` 上报(Micrometer 的惯例表示"本次无值"),而不是伪造 0。
|
||||
@@ -41,6 +44,7 @@ class PipelineMetrics(
|
||||
private val cursor: BeanProvider<InboxCursorRepository>,
|
||||
private val mailbox: BeanProvider<CminmsgInboxRepository>,
|
||||
private val counters: PipelineCounters,
|
||||
private val activity: JobActivity,
|
||||
private val clock: Clock,
|
||||
) {
|
||||
|
||||
@@ -74,6 +78,23 @@ class PipelineMetrics(
|
||||
Gauge.builder("msgx.pipeline.late_arrival.detected.total", counters) { it.lateArrivalDetectedCount().toDouble() }
|
||||
.strongReference(true)
|
||||
.register(registry)
|
||||
|
||||
// 作业心跳与扫描积压:作业线程是回填的唯一驱动,停摆必须能被 /metrics 与 /health 看见。
|
||||
Gauge.builder("msgx.pipeline.job.heartbeat_age_seconds", activity) { a ->
|
||||
a.snapshot().lastTickAt?.let { Duration.between(it, clock.instant()).seconds.toDouble() } ?: -1.0
|
||||
}.strongReference(true).register(registry)
|
||||
|
||||
Gauge.builder("msgx.pipeline.job.ticks.total", activity) { it.snapshot().ticks.toDouble() }
|
||||
.strongReference(true)
|
||||
.register(registry)
|
||||
|
||||
Gauge.builder("msgx.pipeline.job.failures.total", activity) { it.snapshot().failures.toDouble() }
|
||||
.strongReference(true)
|
||||
.register(registry)
|
||||
|
||||
Gauge.builder("msgx.pipeline.job.last_sweep_selected", activity) { it.snapshot().lastSweepSelected.toDouble() }
|
||||
.strongReference(true)
|
||||
.register(registry)
|
||||
}
|
||||
|
||||
private fun backlogGauge(name: String, value: (com.gzzn.omms.msgexchange.infra.persistence.Backlog) -> Double) {
|
||||
|
||||
Reference in New Issue
Block a user