feat(jobs): expose job heartbeat and scan backlog

新增 JobActivity 持有作业心跳与扫描积压,JobRunner 每轮 tick 上报(失败只计数、不刷新心跳),PipelineMetrics 注册 msgx.pipeline.job.heartbeat_age_seconds / ticks.total / failures.total / last_sweep_selected,新增 JobRunnerHealthIndicator(未启动 UP 并标 started=false;已启动但心跳缺失或超过 3×扫描周期 DOWN)。对应 reference 作业健康段与 [G-JOB-HEARTBEAT]。

验证:./gradlew test 145 tests / 0 fail / 0 skipped(新增 JobRunnerTest 2、JobRunnerHealthIndicatorTest 4、PipelineMetrics 指标注册 1)。
This commit is contained in:
windyboy
2026-09-12 20:52:24 +08:00
parent ab18e4ec3f
commit 4b71b5bb09
7 changed files with 333 additions and 6 deletions
@@ -1,6 +1,7 @@
package com.gzzn.omms.msgexchange.jobs
import com.gzzn.omms.msgexchange.config.OperationDayProps
import com.gzzn.omms.msgexchange.infra.metrics.JobActivity
import com.gzzn.omms.msgexchange.processing.BackfillService
import jakarta.inject.Singleton
import java.time.Clock
@@ -23,6 +24,7 @@ class JobRunner(
private val backfill: BackfillService,
private val historySweep: HistorySweepJob,
private val clock: Clock,
private val activity: JobActivity,
operationDayProps: OperationDayProps,
) {
private val log = org.slf4j.LoggerFactory.getLogger(JobRunner::class.java)
@@ -39,28 +41,46 @@ class JobRunner(
fun start() {
if (thread != null) return
running = true
activity.started()
thread = Thread.ofPlatform().name("msgx-jobs").daemon(true).start { loop() }
log.info("job runner started (backfill 30s, history daily 03:30 {})", zone)
log.info("job runner started (backfill {}s, history daily 03:30 {})", TICK_PERIOD.seconds, zone)
}
fun stop() {
running = false
thread?.interrupt()
thread = null
activity.stopped()
}
internal fun loop() {
while (running) {
try {
backfill.sweep()
maybeHistorySweep()
tickOnce()
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
return
} catch (e: Exception) {
log.warn("job tick failed: {}", e.message ?: e.javaClass.simpleName)
}
if (running) runCatching { Thread.sleep(Duration.ofSeconds(30).toMillis()) }
if (running) runCatching { Thread.sleep(TICK_PERIOD.toMillis()) }
}
}
/**
* 跑一轮作业并上报心跳(拉出来单独可测,避免测试里 sleep)。
* 回填扫描的选中条数即"扫描积压"的即时值;失败只累计计数,不刷新心跳时刻。
*/
internal fun tickOnce() {
val startedAt = clock.instant()
val startedNanos = System.nanoTime()
try {
val selected = backfill.sweep(startedAt)
maybeHistorySweep()
activity.tickFinished(startedAt, (System.nanoTime() - startedNanos) / 1_000_000, selected)
} catch (e: InterruptedException) {
throw e
} catch (e: Exception) {
activity.tickFailed()
log.warn("job tick failed: {}", e.message ?: e.javaClass.simpleName)
}
}
@@ -78,4 +98,9 @@ class JobRunner(
}
}
}
private companion object {
/** 作业循环周期:回填扫描与历史作业共用的代码常量(reference 记为 `backfill-scan-period`)。 */
val TICK_PERIOD: Duration = Duration.ofSeconds(30)
}
}