新增 JobActivity 持有作业心跳与扫描积压,JobRunner 每轮 tick 上报(失败只计数、不刷新心跳),PipelineMetrics 注册 msgx.pipeline.job.heartbeat_age_seconds / ticks.total / failures.total / last_sweep_selected,新增 JobRunnerHealthIndicator(未启动 UP 并标 started=false;已启动但心跳缺失或超过 3×扫描周期 DOWN)。对应 reference 作业健康段与 [G-JOB-HEARTBEAT]。 验证:./gradlew test 145 tests / 0 fail / 0 skipped(新增 JobRunnerTest 2、JobRunnerHealthIndicatorTest 4、PipelineMetrics 指标注册 1)。
107 lines
3.7 KiB
Kotlin
107 lines
3.7 KiB
Kotlin
package com.gzzn.omms.msgexchange.jobs
|
|
|
|
import com.gzzn.omms.msgexchange.config.OperationDayProps
|
|
import com.gzzn.omms.msgexchange.infra.metrics.JobActivity
|
|
import com.gzzn.omms.msgexchange.processing.BackfillService
|
|
import jakarta.inject.Singleton
|
|
import java.time.Clock
|
|
import java.time.Duration
|
|
import java.time.Instant
|
|
import java.time.LocalDate
|
|
import java.time.ZoneId
|
|
|
|
/**
|
|
* 维护作业线程:定时做那些不需要跟消息一起排队的事。
|
|
*
|
|
* - 每 30 秒扫一次还欠回填的记录,把处理标记补写回共享信箱;
|
|
* - 每天机场时间 3:30 之后跑一次航班历史归档与留痕清理。
|
|
*
|
|
* 作业跑在自己的线程上,不占用消息处理循环,也不会让到期的消息饿死在这里。
|
|
* 回填意图在处理完成时就已经写进数据库了,本线程只负责到期重试。
|
|
*/
|
|
@Singleton
|
|
class JobRunner(
|
|
private val backfill: BackfillService,
|
|
private val historySweep: HistorySweepJob,
|
|
private val clock: Clock,
|
|
private val activity: JobActivity,
|
|
operationDayProps: OperationDayProps,
|
|
) {
|
|
private val log = org.slf4j.LoggerFactory.getLogger(JobRunner::class.java)
|
|
private val zone: ZoneId = operationDayProps.zoneId()
|
|
|
|
@Volatile
|
|
private var running = true
|
|
|
|
@Volatile
|
|
private var lastHistoryDay: LocalDate? = null
|
|
|
|
private var thread: Thread? = null
|
|
|
|
fun start() {
|
|
if (thread != null) return
|
|
running = true
|
|
activity.started()
|
|
thread = Thread.ofPlatform().name("msgx-jobs").daemon(true).start { loop() }
|
|
log.info("job runner started (backfill {}s, history daily 03:30 {})", TICK_PERIOD.seconds, zone)
|
|
}
|
|
|
|
fun stop() {
|
|
running = false
|
|
thread?.interrupt()
|
|
thread = null
|
|
activity.stopped()
|
|
}
|
|
|
|
internal fun loop() {
|
|
while (running) {
|
|
try {
|
|
tickOnce()
|
|
} catch (e: InterruptedException) {
|
|
Thread.currentThread().interrupt()
|
|
return
|
|
}
|
|
if (running) runCatching { Thread.sleep(TICK_PERIOD.toMillis()) }
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 跑一轮作业并上报心跳(拉出来单独可测,避免测试里 sleep)。
|
|
* 回填扫描的选中条数即"扫描积压"的即时值;失败只累计计数,不刷新心跳时刻。
|
|
*/
|
|
internal fun tickOnce() {
|
|
val startedAt = clock.instant()
|
|
val startedNanos = System.nanoTime()
|
|
try {
|
|
val selected = backfill.sweep(startedAt)
|
|
maybeHistorySweep()
|
|
activity.tickFinished(startedAt, (System.nanoTime() - startedNanos) / 1_000_000, selected)
|
|
} catch (e: InterruptedException) {
|
|
throw e
|
|
} catch (e: Exception) {
|
|
activity.tickFailed()
|
|
log.warn("job tick failed: {}", e.message ?: e.javaClass.simpleName)
|
|
}
|
|
}
|
|
|
|
private fun maybeHistorySweep() {
|
|
// 时区换算后的机场时钟:切日与 03:30 门槛都以机场时区为准,且走注入 Clock(测试可确定)。
|
|
val airportClock = clock.withZone(zone)
|
|
val today = LocalDate.now(airportClock)
|
|
if (lastHistoryDay == today) return
|
|
val now = java.time.LocalTime.now(airportClock)
|
|
if ((now.hour == 3 && now.minute >= 30) || now.hour > 3) {
|
|
val outcome = historySweep.run(clock.instant())
|
|
lastHistoryDay = today
|
|
if (outcome.selected > 0 || outcome.snapLogPurged > 0) {
|
|
log.info("history sweep: {}", outcome)
|
|
}
|
|
}
|
|
}
|
|
|
|
private companion object {
|
|
/** 作业循环周期:回填扫描与历史作业共用的代码常量(reference 记为 `backfill-scan-period`)。 */
|
|
val TICK_PERIOD: Duration = Duration.ofSeconds(30)
|
|
}
|
|
}
|