feat(jobs): expose job heartbeat and scan backlog

新增 JobActivity 持有作业心跳与扫描积压,JobRunner 每轮 tick 上报(失败只计数、不刷新心跳),PipelineMetrics 注册 msgx.pipeline.job.heartbeat_age_seconds / ticks.total / failures.total / last_sweep_selected,新增 JobRunnerHealthIndicator(未启动 UP 并标 started=false;已启动但心跳缺失或超过 3×扫描周期 DOWN)。对应 reference 作业健康段与 [G-JOB-HEARTBEAT]。

验证:./gradlew test 145 tests / 0 fail / 0 skipped(新增 JobRunnerTest 2、JobRunnerHealthIndicatorTest 4、PipelineMetrics 指标注册 1)。
This commit is contained in:
windyboy
2026-09-12 20:52:24 +08:00
parent ab18e4ec3f
commit 4b71b5bb09
7 changed files with 333 additions and 6 deletions
@@ -0,0 +1,70 @@
package com.gzzn.omms.msgexchange.infra.health
import com.gzzn.omms.msgexchange.MutableClock
import com.gzzn.omms.msgexchange.infra.metrics.JobActivity
import io.micronaut.health.HealthStatus
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
/**
* 作业存活判定:没启动不误报;启动了但心跳缺失或超过 3 个扫描周期必须 DOWN;
* 正常心跳保持 UP 并把扫描积压等事实带进详情。
*/
class JobRunnerHealthIndicatorTest {
private val now = MutableClock.BASE
private fun details(result: io.micronaut.management.health.indicator.HealthResult): Map<String, Any> {
@Suppress("UNCHECKED_CAST")
return result.details as Map<String, Any>
}
@Test
fun `a job runner that never started is up but flagged`() {
val result = jobRunnerHealth(JobActivity().snapshot(), now)
assertEquals(HealthStatus.UP, result.status)
assertEquals(false, details(result)["started"])
assertEquals(-1L, details(result)["lastTickAgeSeconds"])
}
@Test
fun `a started job with a fresh heartbeat is up`() {
val activity = JobActivity().apply {
started()
tickFinished(now.minusSeconds(30), durationMillis = 12, sweepSelected = 3)
}
val result = jobRunnerHealth(activity.snapshot(), now)
assertEquals(HealthStatus.UP, result.status)
assertEquals(1L, details(result)["ticks"])
assertEquals(3, details(result)["lastSweepSelected"])
assertEquals(12L, details(result)["lastTickDurationMs"])
}
@Test
fun `a started job that never ticked or went stale is down`() {
val neverTicked = JobActivity().apply { started() }
assertEquals(HealthStatus.DOWN, jobRunnerHealth(neverTicked.snapshot(), now).status)
val stale = JobActivity().apply {
started()
tickFinished(now.minusSeconds(STALE_AFTER_SECONDS + 1), durationMillis = 5, sweepSelected = 0)
}
assertEquals(HealthStatus.DOWN, jobRunnerHealth(stale.snapshot(), now).status)
}
@Test
fun `a failed tick counts up without refreshing the heartbeat`() {
val activity = JobActivity().apply { started() }
activity.tickFailed()
activity.tickFailed()
val snapshot = activity.snapshot()
assertEquals(2L, snapshot.failures)
assertEquals(0L, snapshot.ticks)
assertEquals(null, snapshot.lastTickAt)
}
}
@@ -58,6 +58,20 @@ class PipelineMetricsTest {
assertEquals(before + 2.0, gauge("msgx.pipeline.hole.aged_out.total"), 0.001)
}
@Test
fun `job heartbeat gauges are registered and reflect job activity`() {
val activity = ctx.getBean(JobActivity::class.java)
val ticksBefore = gauge("msgx.pipeline.job.ticks.total")
val failuresBefore = gauge("msgx.pipeline.job.failures.total")
activity.tickFinished(Instant.now(), durationMillis = 12, sweepSelected = 7)
assertEquals(ticksBefore + 1.0, gauge("msgx.pipeline.job.ticks.total"), 0.001)
assertEquals(failuresBefore, gauge("msgx.pipeline.job.failures.total"), 0.001)
assertEquals(7.0, gauge("msgx.pipeline.job.last_sweep_selected"), 0.001)
org.junit.jupiter.api.Assertions.assertTrue(gauge("msgx.pipeline.job.heartbeat_age_seconds") >= 0.0)
}
private fun gauge(name: String): Double =
requireNotNull(registry.find(name).gauge()) { "gauge not registered: $name" }.value()
}
@@ -0,0 +1,75 @@
package com.gzzn.omms.msgexchange.jobs
import com.gzzn.omms.msgexchange.MutableClock
import com.gzzn.omms.msgexchange.config.HistoryProps
import com.gzzn.omms.msgexchange.config.MailboxProps
import com.gzzn.omms.msgexchange.config.OperationDayProps
import com.gzzn.omms.msgexchange.config.PipelineProps
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.infra.metrics.JobActivity
import com.gzzn.omms.msgexchange.infra.persistence.BackfillDue
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import com.gzzn.omms.msgexchange.infra.stub.StubFlightState
import com.gzzn.omms.msgexchange.infra.stub.StubInbox
import com.gzzn.omms.msgexchange.infra.stub.StubMsgEvents
import com.gzzn.omms.msgexchange.infra.stub.StubPipelineLock
import com.gzzn.omms.msgexchange.infra.stub.StubPipelineTx
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
import com.gzzn.omms.msgexchange.processing.BackfillService
import com.gzzn.omms.msgexchange.processing.MessageLifecycleGate
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.Test
import java.time.Instant
/**
* 作业心跳的接线:一轮 tick 跑完必须上报心跳与扫描积压;tick 抛错只累计失败、
* **不刷新心跳时刻**(停摆由心跳年龄暴露)。用 `TestClocks` 与内存适配器,不 sleep。
*/
class JobRunnerTest {
private val clock = MutableClock(MutableClock.BASE)
private val props = PipelineProps()
private fun historySweep() = HistorySweepJob(
StubFlightState(), StubMsgEvents(), HistoryProps().apply { historyStoreEnabled = false },
OperationDayProps(), StubPipelineTx(), StubPipelineLock(),
)
private fun runner(proc: ProcStateRepository, activity: JobActivity) = JobRunner(
BackfillService(proc, StubInbox(), MailboxProps(), props, clock, MessageLifecycleGate()),
historySweep(), clock, activity, OperationDayProps(),
)
@Test
fun `a completed tick reports the heartbeat and the backfill scan backlog`() {
val proc = StubProcState()
proc.insertIfAbsent(1L, MutableClock.BASE.minusSeconds(30L * 86400))
proc.markTerminal(1L, ProcStatus.SUCCEEDED, now = MutableClock.BASE) // 终态未打标 → 本轮到期待办
val activity = JobActivity()
runner(proc, activity).tickOnce()
val snapshot = activity.snapshot()
assertEquals(1, snapshot.lastSweepSelected)
assertEquals(1L, snapshot.ticks)
assertEquals(0L, snapshot.failures)
assertEquals(MutableClock.BASE, snapshot.lastTickAt)
}
@Test
fun `a failing tick counts the failure and leaves the heartbeat untouched`() {
val broken = object : ProcStateRepository by StubProcState() {
override fun findBackfillDue(now: Instant, overdueBefore: Instant, limit: Int): List<BackfillDue> =
throw IllegalStateException("db-down")
}
val activity = JobActivity()
runner(broken, activity).tickOnce()
val snapshot = activity.snapshot()
assertNull(snapshot.lastTickAt)
assertEquals(0L, snapshot.ticks)
assertEquals(1L, snapshot.failures)
}
}