2026-09-07 15:11:33 +08:00
|
|
|
|
package com.gzzn.omms.msgexchange.processing
|
2026-09-06 16:13:50 +08:00
|
|
|
|
|
2026-09-09 17:53:08 +08:00
|
|
|
|
import com.gzzn.omms.msgexchange.codec.FlopPayload
|
|
|
|
|
|
import com.gzzn.omms.msgexchange.codec.ScheduleBody
|
2026-09-09 19:48:45 +08:00
|
|
|
|
import com.gzzn.omms.msgexchange.codec.XmlCodec
|
2026-09-07 15:11:33 +08:00
|
|
|
|
import com.gzzn.omms.msgexchange.config.PipelineProps
|
|
|
|
|
|
import com.gzzn.omms.msgexchange.domain.DecodedMessage
|
|
|
|
|
|
import com.gzzn.omms.msgexchange.domain.ErrorClass
|
2026-09-09 17:53:08 +08:00
|
|
|
|
import com.gzzn.omms.msgexchange.domain.MsgKind
|
2026-09-07 15:11:33 +08:00
|
|
|
|
import com.gzzn.omms.msgexchange.domain.ProcState
|
|
|
|
|
|
import com.gzzn.omms.msgexchange.domain.ProcStatus
|
2026-09-09 17:53:08 +08:00
|
|
|
|
import com.gzzn.omms.msgexchange.infra.log.TraceLog
|
2026-09-07 15:11:33 +08:00
|
|
|
|
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
|
|
|
|
|
|
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
|
|
|
|
|
|
import com.gzzn.omms.msgexchange.infra.retry.ProcFailure
|
2026-09-06 16:13:50 +08:00
|
|
|
|
import jakarta.inject.Singleton
|
2026-09-10 10:59:58 +08:00
|
|
|
|
import java.time.Clock
|
2026-09-06 16:13:50 +08:00
|
|
|
|
import java.time.Duration
|
|
|
|
|
|
import java.time.Instant
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
2026-09-10 11:05:55 +08:00
|
|
|
|
* 处理主泵:一个线程按消息 ID 从小到大一条条处理,保证先来的先处理。
|
2026-09-10 10:59:58 +08:00
|
|
|
|
*
|
2026-09-10 11:05:55 +08:00
|
|
|
|
* 每次 tick 只看当前最小的未完成消息("队头"):
|
|
|
|
|
|
* - 没有待处理消息就睡一个轮询间隔;
|
|
|
|
|
|
* - 队头失败了还在退避期,就等到能重试的时刻;如果重试次数用尽或滞留太久,
|
|
|
|
|
|
* 直接转死信,不放任它一直堵着;
|
|
|
|
|
|
* - 其余情况交给 [MessageProcessor] 处理。
|
|
|
|
|
|
*
|
|
|
|
|
|
* 一次只处理一条是刻意的。后面的消息不能越过卡住的队头,否则同一条航班的报文
|
|
|
|
|
|
* 可能被乱序应用,几十秒后才到的旧报文会把新状态覆盖回去。
|
2026-09-06 16:13:50 +08:00
|
|
|
|
*/
|
|
|
|
|
|
@Singleton
|
|
|
|
|
|
class Pump(
|
|
|
|
|
|
private val procState: ProcStateRepository,
|
|
|
|
|
|
private val processor: MessageProcessor,
|
2026-09-10 10:59:58 +08:00
|
|
|
|
private val backfill: BackfillService,
|
2026-09-06 16:13:50 +08:00
|
|
|
|
private val props: PipelineProps,
|
2026-09-10 10:59:58 +08:00
|
|
|
|
private val clock: Clock,
|
2026-09-06 16:13:50 +08:00
|
|
|
|
) {
|
2026-09-06 21:02:27 +08:00
|
|
|
|
private val log = org.slf4j.LoggerFactory.getLogger(Pump::class.java)
|
|
|
|
|
|
|
2026-09-06 16:13:50 +08:00
|
|
|
|
@Volatile
|
|
|
|
|
|
private var running = true
|
|
|
|
|
|
|
2026-09-10 11:05:55 +08:00
|
|
|
|
/** 请求停机:当前 tick 跑完就退出。线程中断由 PipelineLifecycle 负责。 */
|
2026-09-06 20:53:14 +08:00
|
|
|
|
fun stop() {
|
|
|
|
|
|
running = false
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-09-06 16:13:50 +08:00
|
|
|
|
fun loop() {
|
|
|
|
|
|
while (running) {
|
2026-09-06 18:19:04 +08:00
|
|
|
|
try {
|
|
|
|
|
|
tick()
|
2026-09-06 19:08:02 +08:00
|
|
|
|
} catch (e: InterruptedException) {
|
|
|
|
|
|
Thread.currentThread().interrupt()
|
|
|
|
|
|
return
|
|
|
|
|
|
} catch (e: Exception) {
|
2026-09-10 11:05:55 +08:00
|
|
|
|
// 兜底:单条消息的失败状态已在 processOne 内记录,这里只避免线程退出
|
2026-09-06 18:19:04 +08:00
|
|
|
|
sleepQuietly(props.pipeline.pollInterval)
|
|
|
|
|
|
}
|
2026-09-06 16:13:50 +08:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
internal fun tick() {
|
|
|
|
|
|
val head = procState.headUnfinished()
|
|
|
|
|
|
when {
|
|
|
|
|
|
head == null -> sleepQuietly(props.pipeline.pollInterval)
|
2026-09-10 20:39:22 +08:00
|
|
|
|
head.state == ProcStatus.FAILED && poisoned(head) -> {
|
|
|
|
|
|
log.error("poison -> DEAD msgId={} attempts={} lastError={}", head.msgId, head.attempts, head.lastError)
|
|
|
|
|
|
procState.markTerminal(
|
|
|
|
|
|
head.msgId, ProcStatus.DEAD,
|
|
|
|
|
|
errorClass = ErrorClass.EXHAUSTED,
|
|
|
|
|
|
lastError = head.lastError ?: "head-deadline-exceeded",
|
|
|
|
|
|
attempts = head.attempts,
|
|
|
|
|
|
now = clock.instant(),
|
|
|
|
|
|
)
|
|
|
|
|
|
backfill.attempt(head.msgId)
|
|
|
|
|
|
}
|
2026-09-10 10:59:58 +08:00
|
|
|
|
head.state == ProcStatus.FAILED && (head.nextAttemptAt ?: Instant.EPOCH) > clock.instant() ->
|
2026-09-10 20:39:22 +08:00
|
|
|
|
sleepQuietly(Duration.between(clock.instant(), head.nextAttemptAt))
|
2026-09-10 11:05:55 +08:00
|
|
|
|
// 其余情况(新消息,或退避到期的重试)交给处理入口
|
2026-09-10 20:39:22 +08:00
|
|
|
|
else -> {
|
|
|
|
|
|
procState.markProcessingStartedIfAbsent(head.msgId, clock.instant())
|
|
|
|
|
|
processor.processOne(head)
|
|
|
|
|
|
}
|
2026-09-06 16:13:50 +08:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-09-06 18:19:04 +08:00
|
|
|
|
private fun poisoned(head: ProcState): Boolean =
|
2026-09-10 20:39:22 +08:00
|
|
|
|
isHeadPoisoned(head, clock.instant(), props)
|
2026-09-06 16:13:50 +08:00
|
|
|
|
|
|
|
|
|
|
private fun sleepQuietly(d: Duration) {
|
|
|
|
|
|
if (!d.isNegative && !d.isZero) Thread.sleep(d.toMillis().coerceAtLeast(1))
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-09-10 20:39:22 +08:00
|
|
|
|
internal fun isHeadPoisoned(head: ProcState, now: Instant, props: PipelineProps): Boolean =
|
|
|
|
|
|
head.attempts >= props.pipeline.maxAttempts ||
|
|
|
|
|
|
Duration.between(head.processingStartedAt ?: head.updatedAt, now) >= props.pipeline.headDeadline
|
|
|
|
|
|
|
2026-09-06 18:19:04 +08:00
|
|
|
|
/**
|
2026-09-10 11:05:55 +08:00
|
|
|
|
* 处理一条消息:读原文 → 解码 → 绑定业务身份 → 分派给对应处理器 → 提交后回填标记。
|
|
|
|
|
|
*
|
|
|
|
|
|
* 业务数据和终态由各处理器在自己的事务里写入。终态一旦落下(成功、跳过或死信),
|
|
|
|
|
|
* 这里马上试一次把处理标记写回信箱;写不进去也没关系,回填扫描会按退避继续重试。
|
|
|
|
|
|
*
|
|
|
|
|
|
* 任何意外异常都算在当前这条消息头上(记 FAILED(INFRA) 后重试),不会把主泵线程带崩。
|
|
|
|
|
|
* 报文非法和整包协议拒绝不重试,直接进死信等人工处置。
|
2026-09-06 18:19:04 +08:00
|
|
|
|
*/
|
2026-09-06 16:13:50 +08:00
|
|
|
|
@Singleton
|
|
|
|
|
|
class MessageProcessor(
|
|
|
|
|
|
private val inbox: CminmsgInboxRepository,
|
|
|
|
|
|
private val procState: ProcStateRepository,
|
2026-09-09 19:48:45 +08:00
|
|
|
|
private val codec: XmlCodec,
|
2026-09-09 17:53:08 +08:00
|
|
|
|
private val scheduleProcessor: ScheduleProcessor,
|
|
|
|
|
|
private val flopProcessor: FlopProcessor,
|
|
|
|
|
|
private val fdelProcessor: FdelProcessor,
|
|
|
|
|
|
private val adftProcessor: AdftProcessor,
|
2026-09-06 19:08:02 +08:00
|
|
|
|
private val procFailure: ProcFailure,
|
2026-09-10 10:59:58 +08:00
|
|
|
|
private val backfill: BackfillService,
|
2026-09-06 16:13:50 +08:00
|
|
|
|
private val props: PipelineProps,
|
|
|
|
|
|
) {
|
2026-09-06 21:02:27 +08:00
|
|
|
|
private val log = org.slf4j.LoggerFactory.getLogger(MessageProcessor::class.java)
|
|
|
|
|
|
|
2026-09-06 18:19:04 +08:00
|
|
|
|
fun processOne(head: ProcState) {
|
2026-09-09 17:53:08 +08:00
|
|
|
|
TraceLog.withTrace(head.msgId) {
|
2026-09-10 10:59:58 +08:00
|
|
|
|
val terminal = try {
|
2026-09-06 21:02:27 +08:00
|
|
|
|
processInternal(head)
|
|
|
|
|
|
} catch (e: InterruptedException) {
|
|
|
|
|
|
Thread.currentThread().interrupt()
|
|
|
|
|
|
throw e
|
|
|
|
|
|
} catch (e: Exception) {
|
2026-09-09 17:53:08 +08:00
|
|
|
|
// 边界化:异常归于本条 head,写 FAILED(INFRA)/DEAD,而不是穿出杀 pump
|
|
|
|
|
|
log.warn("processOne unexpected failure msgId={} ec=INFRA msg={}", head.msgId, e.message ?: e.javaClass.simpleName)
|
2026-09-06 21:02:27 +08:00
|
|
|
|
procFailure.fail(head, ErrorClass.INFRA, e.message ?: e.javaClass.simpleName)
|
|
|
|
|
|
}
|
2026-09-10 11:09:19 +08:00
|
|
|
|
// 终态已经写好了,马上试一次回填;写不进去也没关系,回填扫描会按退避继续重试
|
|
|
|
|
|
// (待办在写终态时就一起登记了)。还没处理完的消息不打标记。
|
2026-09-10 10:59:58 +08:00
|
|
|
|
if (terminal) backfill.attempt(head.msgId)
|
2026-09-06 19:08:02 +08:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-09-10 11:05:55 +08:00
|
|
|
|
/** @return 这条消息是否已经落到终态(只有终态才允许回填信箱标记) */
|
2026-09-10 10:59:58 +08:00
|
|
|
|
private fun processInternal(head: ProcState): Boolean {
|
2026-09-06 19:08:02 +08:00
|
|
|
|
// 守卫:手工/遗留 FAILED 行若 attempts 已达上限,直接终态(防止退避到期后无限重试)
|
|
|
|
|
|
if (head.state == ProcStatus.FAILED && procFailure.scheduler.exhausted(head.attempts)) {
|
2026-09-09 17:53:08 +08:00
|
|
|
|
log.error("head exhausted at entry -> DEAD msgId={} attempts={}", head.msgId, head.attempts)
|
2026-09-10 10:59:58 +08:00
|
|
|
|
procState.markTerminal(
|
|
|
|
|
|
head.msgId, ProcStatus.DEAD,
|
|
|
|
|
|
errorClass = ErrorClass.EXHAUSTED,
|
|
|
|
|
|
lastError = head.lastError ?: "max-attempts",
|
|
|
|
|
|
attempts = head.attempts,
|
|
|
|
|
|
)
|
|
|
|
|
|
return true
|
2026-09-06 18:19:04 +08:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-09-10 10:59:58 +08:00
|
|
|
|
val raw = inbox.rawOf(head.msgId)
|
|
|
|
|
|
if (raw == null) {
|
2026-09-09 17:53:08 +08:00
|
|
|
|
log.error("raw missing -> DEAD(MALFORMED) msgId={}", head.msgId)
|
2026-09-10 10:59:58 +08:00
|
|
|
|
return deadMalformed(head, "raw-missing")
|
2026-09-06 16:13:50 +08:00
|
|
|
|
}
|
2026-09-09 17:53:08 +08:00
|
|
|
|
val decoded = when (val r = codec.decode(raw)) {
|
2026-09-07 15:11:33 +08:00
|
|
|
|
is com.gzzn.omms.msgexchange.codec.DecodeResult.Ok -> r.message
|
|
|
|
|
|
is com.gzzn.omms.msgexchange.codec.DecodeResult.Err -> {
|
2026-09-09 17:53:08 +08:00
|
|
|
|
// MALFORMED(报文非法)→ DEAD 不重试;CODEC_ERROR(可随 codec 修复重放)→ FAILED 退避
|
2026-09-06 18:19:04 +08:00
|
|
|
|
if (r.failure.errorClass == ErrorClass.MALFORMED) {
|
2026-09-09 17:53:08 +08:00
|
|
|
|
log.error("decode MALFORMED -> DEAD msgId={} detail={}", head.msgId, r.failure.detail)
|
2026-09-10 10:59:58 +08:00
|
|
|
|
return deadMalformed(head, r.failure.detail)
|
2026-09-06 18:19:04 +08:00
|
|
|
|
}
|
2026-09-10 10:59:58 +08:00
|
|
|
|
log.warn("decode {} -> FAILED msgId={} detail={}", r.failure.errorClass, head.msgId, r.failure.detail)
|
|
|
|
|
|
return procFailure.fail(head, r.failure.errorClass, r.failure.detail)
|
2026-09-06 16:13:50 +08:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-09-09 17:53:08 +08:00
|
|
|
|
// I3:identity 仅首次绑定(head.identityKey == null);FAILED 重试不重绑
|
2026-09-06 18:19:04 +08:00
|
|
|
|
if (head.identityKey == null) {
|
2026-09-06 16:13:50 +08:00
|
|
|
|
val identity = Identity.of(decoded, props.identity)
|
2026-09-09 17:53:08 +08:00
|
|
|
|
if (!procState.tryBindIdentity(head.msgId, identity)) {
|
2026-09-06 16:13:50 +08:00
|
|
|
|
val owner = procState.ownerOfIdentity(identity) ?: -1L
|
2026-09-09 17:53:08 +08:00
|
|
|
|
log.info("duplicate-of:{} -> SKIPPED msgId={}", owner, head.msgId)
|
2026-09-10 10:59:58 +08:00
|
|
|
|
procState.markTerminal(head.msgId, ProcStatus.SKIPPED, lastError = "duplicate-of:$owner")
|
|
|
|
|
|
return true
|
2026-09-06 16:13:50 +08:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-09-10 11:09:19 +08:00
|
|
|
|
// 按报文类型分派:日计划走 SCHD,其余走 FLOP / FDEL / ADFT;报文缺载荷直接判为非法报文的死信
|
2026-09-09 17:53:08 +08:00
|
|
|
|
val result: ApplyResult = when (val kind = decoded.kind) {
|
|
|
|
|
|
is MsgKind.Schd -> {
|
|
|
|
|
|
val body = decoded.body as? ScheduleBody
|
2026-09-10 10:59:58 +08:00
|
|
|
|
if (body == null) return deadMalformed(head, "missing-schd-body")
|
2026-09-09 17:53:08 +08:00
|
|
|
|
when (kind.subtype) {
|
|
|
|
|
|
MsgKind.SchdSubtype.DNLD, MsgKind.SchdSubtype.RESP ->
|
|
|
|
|
|
scheduleProcessor.applyScheduleRecords(head, decoded)
|
|
|
|
|
|
MsgKind.SchdSubtype.ADFT -> {
|
|
|
|
|
|
val record = body.records.singleOrNull()
|
2026-09-10 10:59:58 +08:00
|
|
|
|
if (record == null) return deadMalformed(head, "adft-needs-single-fltr")
|
2026-09-09 17:53:08 +08:00
|
|
|
|
adftProcessor.apply(head, decoded, record)
|
|
|
|
|
|
}
|
2026-09-08 11:26:20 +08:00
|
|
|
|
}
|
2026-09-07 16:12:00 +08:00
|
|
|
|
}
|
2026-09-09 17:53:08 +08:00
|
|
|
|
MsgKind.Fdel -> {
|
|
|
|
|
|
val payload = decoded.body as? FlopPayload
|
2026-09-10 10:59:58 +08:00
|
|
|
|
if (payload == null) return deadMalformed(head, "missing-fdel-flid")
|
2026-09-09 17:53:08 +08:00
|
|
|
|
fdelProcessor.apply(head, decoded, payload)
|
|
|
|
|
|
}
|
|
|
|
|
|
is MsgKind.Flop -> {
|
|
|
|
|
|
val payload = decoded.body as? FlopPayload
|
2026-09-10 10:59:58 +08:00
|
|
|
|
if (payload == null) return deadMalformed(head, "missing-flop-body")
|
2026-09-09 17:53:08 +08:00
|
|
|
|
flopProcessor.apply(head, decoded, payload)
|
|
|
|
|
|
}
|
|
|
|
|
|
is MsgKind.Unsupported -> {
|
2026-09-10 11:09:19 +08:00
|
|
|
|
// 还没有对应处理器的报文类型:先按可重试的失败处理,等能力补齐,不直接判死
|
2026-09-09 17:53:08 +08:00
|
|
|
|
log.warn("unsupported type -> FAILED(UNSUPPORTED) msgId={} tag={}", head.msgId, kind.tag)
|
2026-09-10 10:59:58 +08:00
|
|
|
|
return procFailure.fail(head, ErrorClass.UNSUPPORTED, "no-handler:${kind.tag}")
|
2026-09-07 16:12:00 +08:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-09-10 10:59:58 +08:00
|
|
|
|
if (result is ApplyResult.DeadProtocol) {
|
2026-09-10 11:09:19 +08:00
|
|
|
|
// 整包被拒绝:不重试、立刻放掉队头,等人工确认
|
2026-09-10 10:59:58 +08:00
|
|
|
|
log.error("DEAD(PROTOCOL) msgId={} reason={} flags={}", head.msgId, result.reason, result.flags)
|
|
|
|
|
|
procState.markTerminal(
|
|
|
|
|
|
head.msgId, ProcStatus.DEAD,
|
|
|
|
|
|
errorClass = ErrorClass.PROTOCOL,
|
|
|
|
|
|
lastError = result.reason.take(1000),
|
|
|
|
|
|
)
|
|
|
|
|
|
return true
|
2026-09-08 11:26:20 +08:00
|
|
|
|
}
|
2026-09-09 17:53:08 +08:00
|
|
|
|
|
2026-09-10 10:59:58 +08:00
|
|
|
|
// Succeeded / ReplaySkipped:SUCCEEDED 终态与回填意图已由处理器在自己的事务内落库
|
2026-09-09 17:53:08 +08:00
|
|
|
|
log.info("SUCCEEDED msgId={} kind={}", head.msgId, decoded.typeTag)
|
2026-09-10 10:59:58 +08:00
|
|
|
|
return true
|
2026-09-06 16:13:50 +08:00
|
|
|
|
}
|
2026-09-08 13:04:43 +08:00
|
|
|
|
|
2026-09-10 10:59:58 +08:00
|
|
|
|
private fun deadMalformed(head: ProcState, detail: String): Boolean {
|
|
|
|
|
|
procState.markTerminal(head.msgId, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED, lastError = detail)
|
|
|
|
|
|
return true
|
2026-09-08 13:04:43 +08:00
|
|
|
|
}
|
2026-09-06 16:13:50 +08:00
|
|
|
|
}
|