Files
msgexchange-v2/src/main/kotlin/com/gzzn/omms/msgexchange/processing/Pump.kt
T

240 lines
11 KiB
Kotlin
Raw Normal View History

package com.gzzn.omms.msgexchange.processing
import com.gzzn.omms.msgexchange.codec.FlopPayload
import com.gzzn.omms.msgexchange.codec.ScheduleBody
import com.gzzn.omms.msgexchange.codec.XmlCodec
import com.gzzn.omms.msgexchange.config.PipelineProps
import com.gzzn.omms.msgexchange.domain.DecodedMessage
import com.gzzn.omms.msgexchange.domain.ErrorClass
import com.gzzn.omms.msgexchange.domain.MsgKind
import com.gzzn.omms.msgexchange.domain.ProcState
import com.gzzn.omms.msgexchange.domain.ProcStatus
import com.gzzn.omms.msgexchange.infra.log.TraceLog
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
import com.gzzn.omms.msgexchange.infra.retry.ProcFailure
import jakarta.inject.Singleton
import java.time.Clock
import java.time.Duration
import java.time.Instant
/**
* 处理主泵:一个线程按消息 ID 从小到大一条条处理,保证先来的先处理。
*
* 每次 tick 只看当前最小的未完成消息("队头"):
* - 没有待处理消息就睡一个轮询间隔;
* - 队头失败了还在退避期,就等到能重试的时刻;如果重试次数用尽或滞留太久,
* 直接转死信,不放任它一直堵着;
* - 其余情况交给 [MessageProcessor] 处理。
*
* 一次只处理一条是刻意的。后面的消息不能越过卡住的队头,否则同一条航班的报文
* 可能被乱序应用,几十秒后才到的旧报文会把新状态覆盖回去。
*/
@Singleton
class Pump(
private val procState: ProcStateRepository,
private val processor: MessageProcessor,
private val backfill: BackfillService,
private val props: PipelineProps,
private val clock: Clock,
) {
private val log = org.slf4j.LoggerFactory.getLogger(Pump::class.java)
@Volatile
private var running = true
/** 请求停机:当前 tick 跑完就退出。线程中断由 PipelineLifecycle 负责。 */
fun stop() {
running = false
}
fun loop() {
while (running) {
try {
tick()
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
return
} catch (e: Exception) {
// 兜底:单条消息的失败状态已在 processOne 内记录,这里只避免线程退出
sleepQuietly(props.pipeline.pollInterval)
}
}
}
internal fun tick() {
val head = procState.headUnfinished()
when {
head == null -> sleepQuietly(props.pipeline.pollInterval)
head.state == ProcStatus.FAILED && poisoned(head) -> {
log.error("poison -> DEAD msgId={} attempts={} lastError={}", head.msgId, head.attempts, head.lastError)
procState.markTerminal(
head.msgId, ProcStatus.DEAD,
errorClass = ErrorClass.EXHAUSTED,
lastError = head.lastError ?: "head-deadline-exceeded",
attempts = head.attempts,
now = clock.instant(),
)
backfill.attempt(head.msgId)
}
head.state == ProcStatus.FAILED && (head.nextAttemptAt ?: Instant.EPOCH) > clock.instant() ->
sleepQuietly(Duration.between(clock.instant(), head.nextAttemptAt))
// 其余情况(新消息,或退避到期的重试)交给处理入口
else -> {
procState.markProcessingStartedIfAbsent(head.msgId, clock.instant())
processor.processOne(head)
}
}
}
private fun poisoned(head: ProcState): Boolean =
isHeadPoisoned(head, clock.instant(), props)
private fun sleepQuietly(d: Duration) {
if (!d.isNegative && !d.isZero) Thread.sleep(d.toMillis().coerceAtLeast(1))
}
}
internal fun isHeadPoisoned(head: ProcState, now: Instant, props: PipelineProps): Boolean =
head.attempts >= props.pipeline.maxAttempts ||
Duration.between(head.processingStartedAt ?: head.updatedAt, now) >= props.pipeline.headDeadline
/**
* 处理一条消息:读原文 → 解码 → 绑定业务身份 → 分派给对应处理器 → 提交后回填标记。
*
* 业务数据和终态由各处理器在自己的事务里写入。终态一旦落下(成功、跳过或死信),
* 这里马上试一次把处理标记写回信箱;写不进去也没关系,回填扫描会按退避继续重试。
*
* 任何意外异常都算在当前这条消息头上(记 FAILED(INFRA) 后重试),不会把主泵线程带崩。
* 报文非法和整包协议拒绝不重试,直接进死信等人工处置。
*/
@Singleton
class MessageProcessor(
private val inbox: CminmsgInboxRepository,
private val procState: ProcStateRepository,
private val codec: XmlCodec,
private val scheduleProcessor: ScheduleProcessor,
private val flopProcessor: FlopProcessor,
private val fdelProcessor: FdelProcessor,
private val adftProcessor: AdftProcessor,
private val procFailure: ProcFailure,
private val backfill: BackfillService,
private val props: PipelineProps,
) {
private val log = org.slf4j.LoggerFactory.getLogger(MessageProcessor::class.java)
fun processOne(head: ProcState) {
TraceLog.withTrace(head.msgId) {
val terminal = try {
processInternal(head)
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
throw e
} catch (e: Exception) {
// 边界化:异常归于本条 head,写 FAILED(INFRA)/DEAD,而不是穿出杀 pump
log.warn("processOne unexpected failure msgId={} ec=INFRA msg={}", head.msgId, e.message ?: e.javaClass.simpleName)
procFailure.fail(head, ErrorClass.INFRA, e.message ?: e.javaClass.simpleName)
}
// 终态已经写好了,马上试一次回填;写不进去也没关系,回填扫描会按退避继续重试
// (待办在写终态时就一起登记了)。还没处理完的消息不打标记。
if (terminal) backfill.attempt(head.msgId)
}
}
/** @return 这条消息是否已经落到终态(只有终态才允许回填信箱标记) */
private fun processInternal(head: ProcState): Boolean {
// 守卫:手工/遗留 FAILED 行若 attempts 已达上限,直接终态(防止退避到期后无限重试)
if (head.state == ProcStatus.FAILED && procFailure.scheduler.exhausted(head.attempts)) {
log.error("head exhausted at entry -> DEAD msgId={} attempts={}", head.msgId, head.attempts)
procState.markTerminal(
head.msgId, ProcStatus.DEAD,
errorClass = ErrorClass.EXHAUSTED,
lastError = head.lastError ?: "max-attempts",
attempts = head.attempts,
)
return true
}
val raw = inbox.rawOf(head.msgId)
if (raw == null) {
log.error("raw missing -> DEAD(MALFORMED) msgId={}", head.msgId)
return deadMalformed(head, "raw-missing")
}
val decoded = when (val r = codec.decode(raw)) {
is com.gzzn.omms.msgexchange.codec.DecodeResult.Ok -> r.message
is com.gzzn.omms.msgexchange.codec.DecodeResult.Err -> {
// MALFORMED(报文非法)→ DEAD 不重试;CODEC_ERROR(可随 codec 修复重放)→ FAILED 退避
if (r.failure.errorClass == ErrorClass.MALFORMED) {
log.error("decode MALFORMED -> DEAD msgId={} detail={}", head.msgId, r.failure.detail)
return deadMalformed(head, r.failure.detail)
}
log.warn("decode {} -> FAILED msgId={} detail={}", r.failure.errorClass, head.msgId, r.failure.detail)
return procFailure.fail(head, r.failure.errorClass, r.failure.detail)
}
}
// I3identity 仅首次绑定(head.identityKey == null);FAILED 重试不重绑
if (head.identityKey == null) {
val identity = Identity.of(decoded, props.identity)
if (!procState.tryBindIdentity(head.msgId, identity)) {
val owner = procState.ownerOfIdentity(identity) ?: -1L
log.info("duplicate-of:{} -> SKIPPED msgId={}", owner, head.msgId)
procState.markTerminal(head.msgId, ProcStatus.SKIPPED, lastError = "duplicate-of:$owner")
return true
}
}
// 按报文类型分派:日计划走 SCHD,其余走 FLOP / FDEL / ADFT;报文缺载荷直接判为非法报文的死信
val result: ApplyResult = when (val kind = decoded.kind) {
is MsgKind.Schd -> {
val body = decoded.body as? ScheduleBody
if (body == null) return deadMalformed(head, "missing-schd-body")
when (kind.subtype) {
MsgKind.SchdSubtype.DNLD, MsgKind.SchdSubtype.RESP ->
scheduleProcessor.applyScheduleRecords(head, decoded)
MsgKind.SchdSubtype.ADFT -> {
val record = body.records.singleOrNull()
if (record == null) return deadMalformed(head, "adft-needs-single-fltr")
adftProcessor.apply(head, decoded, record)
}
}
}
MsgKind.Fdel -> {
val payload = decoded.body as? FlopPayload
if (payload == null) return deadMalformed(head, "missing-fdel-flid")
fdelProcessor.apply(head, decoded, payload)
}
is MsgKind.Flop -> {
val payload = decoded.body as? FlopPayload
if (payload == null) return deadMalformed(head, "missing-flop-body")
flopProcessor.apply(head, decoded, payload)
}
is MsgKind.Unsupported -> {
// 还没有对应处理器的报文类型:先按可重试的失败处理,等能力补齐,不直接判死
log.warn("unsupported type -> FAILED(UNSUPPORTED) msgId={} tag={}", head.msgId, kind.tag)
return procFailure.fail(head, ErrorClass.UNSUPPORTED, "no-handler:${kind.tag}")
}
}
if (result is ApplyResult.DeadProtocol) {
// 整包被拒绝:不重试、立刻放掉队头,等人工确认
log.error("DEAD(PROTOCOL) msgId={} reason={} flags={}", head.msgId, result.reason, result.flags)
procState.markTerminal(
head.msgId, ProcStatus.DEAD,
errorClass = ErrorClass.PROTOCOL,
lastError = result.reason.take(1000),
)
return true
}
// Succeeded / ReplaySkippedSUCCEEDED 终态与回填意图已由处理器在自己的事务内落库
log.info("SUCCEEDED msgId={} kind={}", head.msgId, decoded.typeTag)
return true
}
private fun deadMalformed(head: ProcState, detail: String): Boolean {
procState.markTerminal(head.msgId, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, lastError = detail)
return true
}
}