feat(jobs): expose job heartbeat and scan backlog

新增 JobActivity 持有作业心跳与扫描积压，JobRunner 每轮 tick 上报（失败只计数、不刷新心跳），PipelineMetrics 注册 msgx.pipeline.job.heartbeat_age_seconds / ticks.total / failures.total / last_sweep_selected，新增 JobRunnerHealthIndicator（未启动 UP 并标 started=false；已启动但心跳缺失或超过 3×扫描周期 DOWN）。对应 reference 作业健康段与 [G-JOB-HEARTBEAT]。验证：./gradlew test 145 tests / 0 fail / 0 skipped（新增 JobRunnerTest 2、JobRunnerHealthIndicatorTest 4、PipelineMetrics 指标注册 1）。
2026-09-12 20:52:24 +08:00
parent ab18e4ec3f
commit 4b71b5bb09
7 changed files with 333 additions and 6 deletions
@@ -0,0 +1,57 @@
+package com.gzzn.omms.msgexchange.infra.health
+
+import com.gzzn.omms.msgexchange.infra.metrics.JobActivity
+import io.micronaut.core.async.publisher.Publishers
+import io.micronaut.health.HealthStatus
+import io.micronaut.management.health.indicator.HealthIndicator
+import io.micronaut.management.health.indicator.HealthResult
+import jakarta.inject.Singleton
+import org.reactivestreams.Publisher
+import java.time.Clock
+import java.time.Duration
+import java.time.Instant
+
+/**
+ * 维护作业的存活判定：作业线程是回填的唯一驱动，停摆必须可见，而不是只留一条日志。
+ *
+ * - 从未启动（例如 `msgx.pipeline.autostart=false`）→ `UP`，详情标 `started=false`，避免误报；
+ * - 已启动但心跳缺失或超过 `3 × 扫描周期`（周期是 `JobRunner` 的代码常量，见 reference）→ `DOWN`；
+ * - 其余 → `UP`，并给出心跳年龄、上一轮耗时、tick/失败数与扫描积压。
+ */
+@Singleton
+class JobRunnerHealthIndicator(
+    private val activity: JobActivity,
+    private val clock: Clock,
+) : HealthIndicator {
+
+    override fun getResult(): Publisher<HealthResult> =
+        Publishers.just(jobRunnerHealth(activity.snapshot(), clock.instant()))
+}
+
+/** 纯判定，便于单测；`now` 由调用方传入注入时钟的时刻。 */
+internal fun jobRunnerHealth(snapshot: JobActivity.Snapshot, now: Instant): HealthResult {
+    val ageSeconds = snapshot.lastTickAt?.let { Duration.between(it, now).seconds }
+    val stale = snapshot.started && (ageSeconds == null || ageSeconds > STALE_AFTER_SECONDS)
+    val message = when {
+        !snapshot.started -> "job runner not started (started=false)"
+        stale -> "job heartbeat stale: age=${ageSeconds ?: "never"}s > ${STALE_AFTER_SECONDS}s"
+        else -> "job heartbeat ok: age=${ageSeconds}s ticks=${snapshot.ticks}"
+    }
+    return HealthResult.builder("job-runner")
+        .status(if (stale) HealthStatus.DOWN else HealthStatus.UP)
+        .details(
+            mapOf(
+                "message" to message,
+                "started" to snapshot.started,
+                "lastTickAgeSeconds" to (ageSeconds ?: -1L),
+                "lastTickDurationMs" to snapshot.lastTickMillis,
+                "ticks" to snapshot.ticks,
+                "failures" to snapshot.failures,
+                "lastSweepSelected" to snapshot.lastSweepSelected,
+            ),
+        )
+        .build()
+}
+
+/** 3 × 30 秒扫描周期：连续两轮都没跑完才算停摆。 */
+internal const val STALE_AFTER_SECONDS = 90L