feat(jobs): expose job heartbeat and scan backlog
新增 JobActivity 持有作业心跳与扫描积压,JobRunner 每轮 tick 上报(失败只计数、不刷新心跳),PipelineMetrics 注册 msgx.pipeline.job.heartbeat_age_seconds / ticks.total / failures.total / last_sweep_selected,新增 JobRunnerHealthIndicator(未启动 UP 并标 started=false;已启动但心跳缺失或超过 3×扫描周期 DOWN)。对应 reference 作业健康段与 [G-JOB-HEARTBEAT]。 验证:./gradlew test 145 tests / 0 fail / 0 skipped(新增 JobRunnerTest 2、JobRunnerHealthIndicatorTest 4、PipelineMetrics 指标注册 1)。
This commit is contained in:
+70
@@ -0,0 +1,70 @@
|
||||
package com.gzzn.omms.msgexchange.infra.health
|
||||
|
||||
import com.gzzn.omms.msgexchange.MutableClock
|
||||
import com.gzzn.omms.msgexchange.infra.metrics.JobActivity
|
||||
import io.micronaut.health.HealthStatus
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
/**
|
||||
* 作业存活判定:没启动不误报;启动了但心跳缺失或超过 3 个扫描周期必须 DOWN;
|
||||
* 正常心跳保持 UP 并把扫描积压等事实带进详情。
|
||||
*/
|
||||
class JobRunnerHealthIndicatorTest {
|
||||
|
||||
private val now = MutableClock.BASE
|
||||
|
||||
private fun details(result: io.micronaut.management.health.indicator.HealthResult): Map<String, Any> {
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
return result.details as Map<String, Any>
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a job runner that never started is up but flagged`() {
|
||||
val result = jobRunnerHealth(JobActivity().snapshot(), now)
|
||||
|
||||
assertEquals(HealthStatus.UP, result.status)
|
||||
assertEquals(false, details(result)["started"])
|
||||
assertEquals(-1L, details(result)["lastTickAgeSeconds"])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a started job with a fresh heartbeat is up`() {
|
||||
val activity = JobActivity().apply {
|
||||
started()
|
||||
tickFinished(now.minusSeconds(30), durationMillis = 12, sweepSelected = 3)
|
||||
}
|
||||
|
||||
val result = jobRunnerHealth(activity.snapshot(), now)
|
||||
|
||||
assertEquals(HealthStatus.UP, result.status)
|
||||
assertEquals(1L, details(result)["ticks"])
|
||||
assertEquals(3, details(result)["lastSweepSelected"])
|
||||
assertEquals(12L, details(result)["lastTickDurationMs"])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a started job that never ticked or went stale is down`() {
|
||||
val neverTicked = JobActivity().apply { started() }
|
||||
assertEquals(HealthStatus.DOWN, jobRunnerHealth(neverTicked.snapshot(), now).status)
|
||||
|
||||
val stale = JobActivity().apply {
|
||||
started()
|
||||
tickFinished(now.minusSeconds(STALE_AFTER_SECONDS + 1), durationMillis = 5, sweepSelected = 0)
|
||||
}
|
||||
assertEquals(HealthStatus.DOWN, jobRunnerHealth(stale.snapshot(), now).status)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a failed tick counts up without refreshing the heartbeat`() {
|
||||
val activity = JobActivity().apply { started() }
|
||||
|
||||
activity.tickFailed()
|
||||
activity.tickFailed()
|
||||
|
||||
val snapshot = activity.snapshot()
|
||||
assertEquals(2L, snapshot.failures)
|
||||
assertEquals(0L, snapshot.ticks)
|
||||
assertEquals(null, snapshot.lastTickAt)
|
||||
}
|
||||
}
|
||||
@@ -58,6 +58,20 @@ class PipelineMetricsTest {
|
||||
assertEquals(before + 2.0, gauge("msgx.pipeline.hole.aged_out.total"), 0.001)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `job heartbeat gauges are registered and reflect job activity`() {
|
||||
val activity = ctx.getBean(JobActivity::class.java)
|
||||
val ticksBefore = gauge("msgx.pipeline.job.ticks.total")
|
||||
val failuresBefore = gauge("msgx.pipeline.job.failures.total")
|
||||
|
||||
activity.tickFinished(Instant.now(), durationMillis = 12, sweepSelected = 7)
|
||||
|
||||
assertEquals(ticksBefore + 1.0, gauge("msgx.pipeline.job.ticks.total"), 0.001)
|
||||
assertEquals(failuresBefore, gauge("msgx.pipeline.job.failures.total"), 0.001)
|
||||
assertEquals(7.0, gauge("msgx.pipeline.job.last_sweep_selected"), 0.001)
|
||||
org.junit.jupiter.api.Assertions.assertTrue(gauge("msgx.pipeline.job.heartbeat_age_seconds") >= 0.0)
|
||||
}
|
||||
|
||||
private fun gauge(name: String): Double =
|
||||
requireNotNull(registry.find(name).gauge()) { "gauge not registered: $name" }.value()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
package com.gzzn.omms.msgexchange.jobs
|
||||
|
||||
import com.gzzn.omms.msgexchange.MutableClock
|
||||
import com.gzzn.omms.msgexchange.config.HistoryProps
|
||||
import com.gzzn.omms.msgexchange.config.MailboxProps
|
||||
import com.gzzn.omms.msgexchange.config.OperationDayProps
|
||||
import com.gzzn.omms.msgexchange.config.PipelineProps
|
||||
import com.gzzn.omms.msgexchange.domain.ProcStatus
|
||||
import com.gzzn.omms.msgexchange.infra.metrics.JobActivity
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.BackfillDue
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubFlightState
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubInbox
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubMsgEvents
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubPipelineLock
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubPipelineTx
|
||||
import com.gzzn.omms.msgexchange.infra.stub.StubProcState
|
||||
import com.gzzn.omms.msgexchange.processing.BackfillService
|
||||
import com.gzzn.omms.msgexchange.processing.MessageLifecycleGate
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.time.Instant
|
||||
|
||||
/**
|
||||
* 作业心跳的接线:一轮 tick 跑完必须上报心跳与扫描积压;tick 抛错只累计失败、
|
||||
* **不刷新心跳时刻**(停摆由心跳年龄暴露)。用 `TestClocks` 与内存适配器,不 sleep。
|
||||
*/
|
||||
class JobRunnerTest {
|
||||
|
||||
private val clock = MutableClock(MutableClock.BASE)
|
||||
private val props = PipelineProps()
|
||||
|
||||
private fun historySweep() = HistorySweepJob(
|
||||
StubFlightState(), StubMsgEvents(), HistoryProps().apply { historyStoreEnabled = false },
|
||||
OperationDayProps(), StubPipelineTx(), StubPipelineLock(),
|
||||
)
|
||||
|
||||
private fun runner(proc: ProcStateRepository, activity: JobActivity) = JobRunner(
|
||||
BackfillService(proc, StubInbox(), MailboxProps(), props, clock, MessageLifecycleGate()),
|
||||
historySweep(), clock, activity, OperationDayProps(),
|
||||
)
|
||||
|
||||
@Test
|
||||
fun `a completed tick reports the heartbeat and the backfill scan backlog`() {
|
||||
val proc = StubProcState()
|
||||
proc.insertIfAbsent(1L, MutableClock.BASE.minusSeconds(30L * 86400))
|
||||
proc.markTerminal(1L, ProcStatus.SUCCEEDED, now = MutableClock.BASE) // 终态未打标 → 本轮到期待办
|
||||
val activity = JobActivity()
|
||||
|
||||
runner(proc, activity).tickOnce()
|
||||
|
||||
val snapshot = activity.snapshot()
|
||||
assertEquals(1, snapshot.lastSweepSelected)
|
||||
assertEquals(1L, snapshot.ticks)
|
||||
assertEquals(0L, snapshot.failures)
|
||||
assertEquals(MutableClock.BASE, snapshot.lastTickAt)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a failing tick counts the failure and leaves the heartbeat untouched`() {
|
||||
val broken = object : ProcStateRepository by StubProcState() {
|
||||
override fun findBackfillDue(now: Instant, overdueBefore: Instant, limit: Int): List<BackfillDue> =
|
||||
throw IllegalStateException("db-down")
|
||||
}
|
||||
val activity = JobActivity()
|
||||
|
||||
runner(broken, activity).tickOnce()
|
||||
|
||||
val snapshot = activity.snapshot()
|
||||
assertNull(snapshot.lastTickAt)
|
||||
assertEquals(0L, snapshot.ticks)
|
||||
assertEquals(1L, snapshot.failures)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user