feat(processing): 实现自有 PostgreSQL 运营航班权威存储与单事务闭环 (ACM2-28)
- FS1: 增加 Flyway 迁移 V1.1.0__flight_schd.sql,创建 FLIGHT_SCHD 与 SCHD_GEN - FS2: 实现 FlightSchdRepository 接口及 JdbcFlightSchdRepository 与 StubFlightSchd,增强 JdbcOps 事务管理 - FS3: 扩展 MessageProcessor 事务 2 与按 FLID 点查视图,合并变更、事件与终态入单事务提交 - FS4: SnapshotFlow SQL 化(批处理 upsert、域内差删、SQL CAS 推进与熔断保护),JobExecutor 接入 PG 清场删除 - FS5: 彻底退役 Redis 权威与写路径,移除 FlightRedisClient、Lua 脚本、健康指示器与配置残留 - FS6: 补齐 U09/U29 不变量门禁(崩溃幂等、CAS 防并发、ADFT 存活保障、非 UTC JVM/会话时区无漂移)与 FlywayMigrationTest - FS7: 交付影子对拍比较内核 FlightStoreDiffTool 与单元测试 - FS8: 全面回改 decision-flight-state、architecture、design、user-stories 权威文档与规范
This commit is contained in:
@@ -20,8 +20,6 @@ interface DeliveryPort {
|
||||
/** 阶段 B:ES flight_hts 写入。 */
|
||||
fun indexFlightHts(payloadJson: String)
|
||||
|
||||
/** 阶段 B:Redis 投影写(仅阶段 B;I5:Delivery 阶段 A 不写 Redis)。 */
|
||||
fun projectRedis(payloadJson: String)
|
||||
|
||||
/** 连通性探测(健康检查用);默认 true,真实 Kafka 实装时覆写为 producer metadata 校验。 */
|
||||
fun ping(): Boolean = true
|
||||
@@ -83,10 +81,7 @@ class Dispatcher(
|
||||
deliver(t, head)
|
||||
msgEvents.markSent(head.eventId!!)
|
||||
log.debug("sent target={} eventId={}", t, head.eventId)
|
||||
if (t == Targets.ES_FLIGHT_HTS && props.phase == PipelineProps.Phase.B) {
|
||||
// 定案 2:ES 投递成功 → 同线程同步 enqueue 删除事件(不轮询 ack)
|
||||
msgEvents.insertSync(listOf(MsgEvent(target = Targets.REDIS_FLIGHT_INFO, payloadJson = deleteOf(head))))
|
||||
}
|
||||
// ACM2-28:阶段 B Redis 投影废弃,读模型投递统一转由 Kafka 事件或 ES 处理
|
||||
} catch (e: Exception) {
|
||||
retryOrDead(head, e.message ?: "unknown")
|
||||
}
|
||||
@@ -115,19 +110,9 @@ class Dispatcher(
|
||||
Targets.KAFKA_MSG -> port.sendKafka("msg", e.payloadJson)
|
||||
Targets.KAFKA_SCHD -> error("KAFKA_SCHD must go through flushSchd (N03)")
|
||||
Targets.ES_FLIGHT_HTS -> port.indexFlightHts(e.payloadJson)
|
||||
Targets.REDIS_FLIGHT_INFO -> port.projectRedis(e.payloadJson)
|
||||
else -> error("unknown target $target")
|
||||
}
|
||||
|
||||
private val jsonMapper = ObjectMapper()
|
||||
|
||||
/**
|
||||
* U14:删除事件 wire JSON 结构化序列化——refs 可空,产出必须恒为合法 JSON
|
||||
* (null → null 字面量;非空 → 带引号并转义)。禁止字符串模板拼接。
|
||||
*/
|
||||
private fun deleteOf(e: MsgEvent): String =
|
||||
jsonMapper.writeValueAsString(linkedMapOf("op" to "delete", "refs" to e.partitionKey))
|
||||
|
||||
/**
|
||||
* 流程 3 flushSchd:批上限 + 每 FLID 最新一态聚合(topic "schd",wire=FLTR JSON 数组)。
|
||||
* U08 批量闭环:队首退避未到期不 claim;发送失败 → 整批 attempts+1(退避)或达上限整批 DEAD/DLQ;
|
||||
|
||||
@@ -2,7 +2,7 @@ package com.gzzn.omms.msgexchange.domain
|
||||
|
||||
/**
|
||||
* ACMA-8 流程 2:Handler 决策(纯函数)产物——状态与报文进,变更与事件出,
|
||||
* 不直接触碰 Redis/Kafka。
|
||||
* 不直接触碰数据库或 Kafka。
|
||||
*/
|
||||
data class Decision(
|
||||
val flightChanges: List<FlightChange> = emptyList(),
|
||||
@@ -12,7 +12,7 @@ data class Decision(
|
||||
val refUpserts: List<RefUpsert> = emptyList(), // → 静态主数据(独立 PG reference 库,ACM2-11)
|
||||
)
|
||||
|
||||
/** 航班状态变更(阶段 A 由主泵线程 redisApply;阶段 B 落 FLIGHT_STATE 同事务)。 */
|
||||
/** 航班状态变更(阶段 A 落自有 PG FLIGHT_SCHD,与事件同事务原子提交;ACM2-28 定案)。 */
|
||||
data class FlightChange(
|
||||
val flid: String,
|
||||
val payloadJson: String,
|
||||
|
||||
@@ -8,13 +8,14 @@ object Targets {
|
||||
const val KAFKA_MSG = "KAFKA:msg"
|
||||
const val KAFKA_SCHD = "KAFKA:schd"
|
||||
const val ES_FLIGHT_HTS = "ES:flight_hts"
|
||||
@Deprecated("Retired in ACM2-28: Redis projection removed")
|
||||
const val REDIS_FLIGHT_INFO = "REDIS:flightInfo"
|
||||
|
||||
/** 阶段 A 投递目标(仅 Kafka)——I5:Delivery 阶段 A 不写 Redis。 */
|
||||
/** 阶段 A 投递目标(仅 Kafka)——ACM2-28:Delivery 阶段 A/B 均不写 Redis。 */
|
||||
val phaseA: List<String> = listOf(KAFKA_MSG, KAFKA_SCHD)
|
||||
|
||||
/** 阶段 B 追加投影目标。 */
|
||||
val phaseB: List<String> = phaseA + listOf(ES_FLIGHT_HTS, REDIS_FLIGHT_INFO)
|
||||
/** 阶段 B 追加投影目标(仅 ES 历史库;Redis 投影按 ACM2-28 废弃)。 */
|
||||
val phaseB: List<String> = phaseA + listOf(ES_FLIGHT_HTS)
|
||||
}
|
||||
|
||||
enum class EventStatus { PENDING, SENT, DEAD }
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.gzzn.omms.msgexchange.infra.health
|
||||
|
||||
import com.gzzn.omms.msgexchange.delivery.DeliveryPort
|
||||
import com.gzzn.omms.msgexchange.infra.redis.FlightRedisClient
|
||||
import io.micronaut.context.BeanProvider
|
||||
import io.micronaut.core.async.publisher.Publishers
|
||||
import io.micronaut.health.HealthStatus
|
||||
@@ -11,20 +10,12 @@ import jakarta.inject.Singleton
|
||||
import org.reactivestreams.Publisher
|
||||
|
||||
/**
|
||||
* U12(R05):阶段 A 关键依赖的自定义健康指示器——
|
||||
* Redis(flightInfo 权威存储)与 Kafka(投递端口)。经 BeanProvider 可选解析:
|
||||
* U12(R05):自定义健康指示器——
|
||||
* Kafka(投递端口)。经 BeanProvider 可选解析:
|
||||
* 缺 bean(如未用 stub 也未实装)时指示 DOWN 而非启动失败;
|
||||
* UP 判据为真实 ping(false/异常 → DOWN),而非仅 bean 存在(复审 P1 修正)。
|
||||
* (ACM2-28:Redis 退出阶段 A 权威与写路径,redis-flight-store 指示器移除)。
|
||||
*/
|
||||
@Singleton
|
||||
class FlightRedisHealthIndicator(
|
||||
private val redis: BeanProvider<FlightRedisClient>,
|
||||
) : HealthIndicator {
|
||||
|
||||
override fun getResult(): Publisher<HealthResult> =
|
||||
Publishers.just(redisHealth(if (redis.isPresent) redis.get() else null))
|
||||
}
|
||||
|
||||
@Singleton
|
||||
class KafkaDeliveryHealthIndicator(
|
||||
private val port: BeanProvider<DeliveryPort>,
|
||||
@@ -34,12 +25,6 @@ class KafkaDeliveryHealthIndicator(
|
||||
Publishers.just(kafkaHealth(if (port.isPresent) port.get() else null))
|
||||
}
|
||||
|
||||
/** ping 判定独立成纯函数便于单测:client 为 null = bean 缺失;ping false/异常 = DOWN。 */
|
||||
internal fun redisHealth(client: FlightRedisClient?): HealthResult =
|
||||
healthOf("redis-flight-store", "flight store", client?.let {
|
||||
try { it.ping() } catch (e: Exception) { false }
|
||||
})
|
||||
|
||||
internal fun kafkaHealth(port: DeliveryPort?): HealthResult =
|
||||
healthOf("kafka-delivery", "delivery port", port?.let {
|
||||
try { it.ping() } catch (e: Exception) { false }
|
||||
|
||||
@@ -14,7 +14,7 @@ import java.time.Instant
|
||||
* 的全部接口(消息管道 PROC_STATE/MSG_EVENT、PUMP_JOB、REQ_TRACK、21 类 REF_MASTER);
|
||||
* 共享 MySQL 信箱(CMINMSGS / COUTMSGS)经信箱封装访问,仅 DML、不建表;
|
||||
* 主路径=上游外部写 CMINMSGS → 本系统 JDBC 轮询读;compat=insertRaw HTTP 写;
|
||||
* Redis = 航班动态 + 快照 gen(RefDataRepository 目标实现);FLIGHT_STATE 缓做。
|
||||
* 自有 PG = 消息管道 + 运营航班 FLIGHT_SCHD/SCHD_GEN + 静态数据;Redis 已退出阶段 A 权威与写路径。
|
||||
*/
|
||||
interface ProcStateRepository {
|
||||
fun insert(cminmsgsId: Long, state: ProcStatus = ProcStatus.PENDING)
|
||||
@@ -68,25 +68,88 @@ interface MsgEventRepository {
|
||||
}
|
||||
|
||||
/**
|
||||
* 快照 generation(SCHD_GEN)协议——只留 gen。
|
||||
* ACM2-12:gen 迁 Redis(与 flightInfo 同源,Lua 内原子「覆盖+按代差删+版本推进」,
|
||||
* DB 仅写 SUCCEEDED;重放幂等由 Lua 承接,协议重设计属 U09)。本接口为过渡占位,
|
||||
* 目标实现为 Redis gen store(script 化),非关系表。
|
||||
* 阶段 A 运营航班权威与日计划代(ACM2-28 采纳选项 C 定案):
|
||||
* - 表 FLIGHT_SCHD:当前运营航班全量权威态(SCHD 快照 + FLOP/ADFT 增量合并),落自有 PostgreSQL;
|
||||
* - 表 SCHD_GEN:各日代版本与当前代有效航班全量集合(差删依据),由 Redis 回归自有 PG;
|
||||
* - Redis 退出动态权威与全部写路径;
|
||||
* - 事务 2 与快照发布全在自有 PG 内以单事务原子提交;
|
||||
* - 增量更新(FLOP/ADFT):新插 FDAY=NULL,已有行通过 ON CONFLICT 保留原 FDAY;
|
||||
* - 按代差删域化:DELETE FROM FLIGHT_SCHD WHERE FDAY = :day AND FLID = ANY(:diffSet)
|
||||
* 仅删除仍属旧代的行,ADFT(FDAY=NULL)与已迁移至新代的同 FLID 行天然存活。
|
||||
*/
|
||||
interface RefDataRepository {
|
||||
data class GenMeta(val flids: List<String>, val version: Long)
|
||||
interface FlightSchdRepository {
|
||||
data class FlightRecord(
|
||||
val flid: String,
|
||||
val fday: String?,
|
||||
val fltrJson: String,
|
||||
val createdAt: Instant,
|
||||
val updatedAt: Instant,
|
||||
)
|
||||
|
||||
data class GenMeta(
|
||||
val fday: String,
|
||||
val version: Long,
|
||||
val flids: Set<String>,
|
||||
val updatedAt: Instant = Instant.now(),
|
||||
)
|
||||
|
||||
/** 快照全量写入(DNLD):强行声明/更新 FDAY 归属,批处理写入。 */
|
||||
fun upsertSnapshotBatch(day: String, flights: List<Pair<String, String>>, now: Instant = Instant.now())
|
||||
|
||||
/** 增量更新(FLOP/ADFT):新插 FDAY=NULL,已有行保留原 FDAY。 */
|
||||
fun upsertIncremental(changes: List<com.gzzn.omms.msgexchange.domain.FlightChange>, now: Instant = Instant.now())
|
||||
|
||||
/** 按代差删域化:仅删除 FDAY = day 且在 delFlids 中的记录(ADFT 与跨代已迁移行受保护)。 */
|
||||
fun deleteDiffByDay(day: String, delFlids: Collection<String>): Int
|
||||
|
||||
/** 点查单航班 FLTR_JSON。 */
|
||||
fun findByFlid(flid: String): String?
|
||||
|
||||
/** 点查多航班 FLTR_JSON。 */
|
||||
fun findByFlids(flids: Collection<String>): Map<String, String>
|
||||
|
||||
/** 按计划日查询当前有效航班。 */
|
||||
fun findByDay(day: String): List<Pair<String, String>>
|
||||
|
||||
/** 全量查询(供影子对拍 / 一致性对账)。 */
|
||||
fun findAll(): Map<String, String>
|
||||
|
||||
/**
|
||||
* 历史清场删除:仅删除已确认归档至 ES 的 FLID 集合;空集合不执行;分批参数化删除。
|
||||
* 返回实际删除行数(允许重放时为 0)。
|
||||
*/
|
||||
fun deleteByFlids(flids: Set<String>): Int
|
||||
|
||||
/** 读计划代(SCHD_GEN)。 */
|
||||
fun getGen(day: String): GenMeta?
|
||||
|
||||
/** 流程 4:版本 CAS(expected 未变才写,重放 no-op,不二次自增)。 */
|
||||
fun putGenIfVersion(day: String, expected: Long, new: GenMeta): Boolean
|
||||
/**
|
||||
* 计划代 SQL 版本 CAS:仅当 expected 与数据库中当前版本一致(或不存在且 expected=0)时写入/推进新版本。
|
||||
* 返回 true 表示推进成功,false 表示发生 CAS 冲突。
|
||||
*/
|
||||
fun putGenIfVersion(day: String, expected: Long, newGen: GenMeta, now: Instant = Instant.now()): Boolean
|
||||
|
||||
/**
|
||||
* 历史代清理:清理 cutoffDay 之前的历史代记录(与 FLIGHT_SCHD 历史清场生命周期对齐)。
|
||||
*/
|
||||
fun deleteGenBefore(cutoffDay: String): Int
|
||||
}
|
||||
|
||||
/** 事务管理器抽象:自有 PG 单事务原子保障。 */
|
||||
interface PipelineTransactionManager {
|
||||
fun <T> inTransaction(block: () -> T): T
|
||||
}
|
||||
|
||||
@Deprecated("Replaced by FlightSchdRepository in ACM2-28", ReplaceWith("FlightSchdRepository"))
|
||||
interface RefDataRepository {
|
||||
data class GenMeta(val flids: List<String>, val version: Long)
|
||||
fun getGen(day: String): GenMeta?
|
||||
fun putGenIfVersion(day: String, expected: Long, new: GenMeta): Boolean
|
||||
}
|
||||
/**
|
||||
* 21 类静态主数据(航空公司/航线/机位/登机桥等)——自有 PostgreSQL `REF_MASTER` 表
|
||||
* (ACM2-12:与消息管道同自有库;SOURCE=ADMINAPI/AODB/PIPELINE,N19 对齐)。
|
||||
* 与主链弱事务耦合:写入者为 ReferenceService(21 类同步)与请求应答路径;
|
||||
* Redis 只作只读热点投影(legacy orms_stand 语义延续,阶段 6)。
|
||||
* 与主链弱事务耦合:写入者为 ReferenceService(21 类同步)与请求应答路径。
|
||||
*/
|
||||
interface StaticRefRepository {
|
||||
fun upsertAll(refs: List<RefUpsert>)
|
||||
@@ -124,7 +187,7 @@ interface ReqTrackRepository {
|
||||
/**
|
||||
* 泵作业调度记录——自有 PG `PUMP_JOB`(ACM2-12)。
|
||||
* 决策 1 修订:作业不插队,仅在消息队头空闲/退避窗口由主泵执行(跨库/异队列无全序);
|
||||
* 作业动作本身(归档写共享库 CMINMSGS_HST、清场删 Redis/ES 等)仍在各自目标存储。
|
||||
* 作业动作本身(归档写共享库 CMINMSGS_HST、清场删 ES+PG 等)仍在各自目标存储。
|
||||
*/
|
||||
interface PumpJobRepository {
|
||||
data class Job(val jobId: Long, val kind: String) // ARCHIVE/HISTORY_SWEEP/PROJECTION_REBUILD
|
||||
@@ -141,9 +204,9 @@ interface PumpJobRepository {
|
||||
}
|
||||
|
||||
/**
|
||||
* 阶段 B 权威(ACM2-12:缓做,不落表——航班动态权威保持 Redis;阶段 B 重新
|
||||
* 评估后再定是否引入事务化权威)。本接口仅供占位与测试,勿据此建表。
|
||||
* 历史占位接口(已被 ACM2-28 之 FlightSchdRepository 取代,保留供兼容与过渡)。
|
||||
*/
|
||||
@Deprecated("Replaced by FlightSchdRepository in ACM2-28", ReplaceWith("FlightSchdRepository"))
|
||||
interface FlightStateRepository {
|
||||
/** 阶段 B 权威;replaceDay = 单事务删差集+写新代+版本提升。 */
|
||||
fun replaceDay(day: String, flights: List<Pair<String, String>>)
|
||||
|
||||
@@ -10,9 +10,51 @@ internal fun Instant.toSqlTimestamp(): Timestamp = Timestamp.from(this)
|
||||
internal fun ResultSet.getInstant(column: String): Instant? =
|
||||
getTimestamp(column)?.toInstant()
|
||||
|
||||
internal fun <T> DataSource.query(sql: String, bind: (java.sql.PreparedStatement) -> Unit, map: (ResultSet) -> T): List<T> =
|
||||
connection.use { conn ->
|
||||
conn.prepareStatement(sql).use { ps ->
|
||||
private val transactionConnection = ThreadLocal<java.sql.Connection?>()
|
||||
|
||||
internal fun <T> DataSource.withTransaction(block: () -> T): T {
|
||||
val existing = transactionConnection.get()
|
||||
if (existing != null) {
|
||||
return block()
|
||||
}
|
||||
val conn = this.connection
|
||||
val oldAutoCommit = conn.autoCommit
|
||||
conn.autoCommit = false
|
||||
transactionConnection.set(conn)
|
||||
try {
|
||||
val result = block()
|
||||
conn.commit()
|
||||
return result
|
||||
} catch (t: Throwable) {
|
||||
try {
|
||||
conn.rollback()
|
||||
} catch (rbEx: Throwable) {
|
||||
t.addSuppressed(rbEx)
|
||||
}
|
||||
throw t
|
||||
} finally {
|
||||
transactionConnection.remove()
|
||||
try {
|
||||
conn.autoCommit = oldAutoCommit
|
||||
} catch (_: Throwable) {
|
||||
}
|
||||
conn.close()
|
||||
}
|
||||
}
|
||||
|
||||
internal fun DataSource.obtainConnection(): java.sql.Connection =
|
||||
transactionConnection.get() ?: this.connection
|
||||
|
||||
internal fun java.sql.Connection.releaseIfNotInTransaction() {
|
||||
if (transactionConnection.get() !== this) {
|
||||
this.close()
|
||||
}
|
||||
}
|
||||
|
||||
internal fun <T> DataSource.query(sql: String, bind: (java.sql.PreparedStatement) -> Unit, map: (ResultSet) -> T): List<T> {
|
||||
val conn = obtainConnection()
|
||||
try {
|
||||
return conn.prepareStatement(sql).use { ps ->
|
||||
bind(ps)
|
||||
ps.executeQuery().use { rs ->
|
||||
buildList {
|
||||
@@ -20,22 +62,30 @@ internal fun <T> DataSource.query(sql: String, bind: (java.sql.PreparedStatement
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
conn.releaseIfNotInTransaction()
|
||||
}
|
||||
}
|
||||
|
||||
internal fun <T> DataSource.queryOne(sql: String, bind: (java.sql.PreparedStatement) -> Unit, map: (ResultSet) -> T): T? =
|
||||
query(sql, bind, map).firstOrNull()
|
||||
|
||||
internal fun DataSource.update(sql: String, bind: (java.sql.PreparedStatement) -> Unit): Int =
|
||||
connection.use { conn ->
|
||||
conn.prepareStatement(sql).use { ps ->
|
||||
internal fun DataSource.update(sql: String, bind: (java.sql.PreparedStatement) -> Unit): Int {
|
||||
val conn = obtainConnection()
|
||||
try {
|
||||
return conn.prepareStatement(sql).use { ps ->
|
||||
bind(ps)
|
||||
ps.executeUpdate()
|
||||
}
|
||||
} finally {
|
||||
conn.releaseIfNotInTransaction()
|
||||
}
|
||||
}
|
||||
|
||||
internal fun DataSource.updateReturningLong(sql: String, bind: (java.sql.PreparedStatement) -> Unit): Long =
|
||||
connection.use { conn ->
|
||||
conn.prepareStatement(sql, java.sql.Statement.RETURN_GENERATED_KEYS).use { ps ->
|
||||
internal fun DataSource.updateReturningLong(sql: String, bind: (java.sql.PreparedStatement) -> Unit): Long {
|
||||
val conn = obtainConnection()
|
||||
try {
|
||||
return conn.prepareStatement(sql, java.sql.Statement.RETURN_GENERATED_KEYS).use { ps ->
|
||||
bind(ps)
|
||||
ps.executeUpdate()
|
||||
ps.generatedKeys.use { keys ->
|
||||
@@ -43,4 +93,7 @@ internal fun DataSource.updateReturningLong(sql: String, bind: (java.sql.Prepare
|
||||
keys.getLong(1)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
conn.releaseIfNotInTransaction()
|
||||
}
|
||||
}
|
||||
|
||||
+240
-12
@@ -6,8 +6,10 @@ import com.gzzn.omms.msgexchange.domain.MsgEvent
|
||||
import com.gzzn.omms.msgexchange.domain.ProcState
|
||||
import com.gzzn.omms.msgexchange.domain.ProcStatus
|
||||
import com.gzzn.omms.msgexchange.domain.RefUpsert
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.PumpJobRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.RefDataRepository
|
||||
@@ -286,21 +288,242 @@ class JdbcPumpJobRepository(
|
||||
}
|
||||
}
|
||||
|
||||
/** gen→Redis 过渡占位:U09 前进程内 CAS(与 StubRefData 同语义)。 */
|
||||
/** 自有 PostgreSQL 单事务管理器(ACM2-28 事务 2 与快照发布原子提交)。 */
|
||||
@Singleton
|
||||
@Requires(property = "msgx.stubs", notEquals = "true")
|
||||
@Requires(property = "datasources.default.enabled", value = "true")
|
||||
class JdbcRefDataRepository : RefDataRepository {
|
||||
private val gens = mutableMapOf<String, RefDataRepository.GenMeta>()
|
||||
class JdbcPipelineTransactionManager(
|
||||
private val ds: DataSource,
|
||||
) : PipelineTransactionManager {
|
||||
override fun <T> inTransaction(block: () -> T): T = ds.withTransaction(block)
|
||||
}
|
||||
|
||||
override fun getGen(day: String): RefDataRepository.GenMeta? = gens[day]
|
||||
@Singleton
|
||||
@Requires(property = "msgx.stubs", notEquals = "true")
|
||||
@Requires(property = "datasources.default.enabled", value = "true")
|
||||
class JdbcFlightSchdRepository(
|
||||
private val ds: DataSource,
|
||||
) : FlightSchdRepository {
|
||||
private val flidsTypeRef = object : com.fasterxml.jackson.core.type.TypeReference<Set<String>>() {}
|
||||
private val mapper = com.fasterxml.jackson.databind.ObjectMapper()
|
||||
|
||||
override fun putGenIfVersion(day: String, expected: Long, new: RefDataRepository.GenMeta): Boolean {
|
||||
val cur = gens[day]?.version ?: 0L
|
||||
if (cur != expected) return false
|
||||
gens[day] = new
|
||||
return true
|
||||
private fun serializeFlids(flids: Collection<String>): String = mapper.writeValueAsString(flids)
|
||||
|
||||
private fun parseFlids(json: String): Set<String> {
|
||||
if (json.isBlank()) return emptySet()
|
||||
return try {
|
||||
mapper.readValue(json, flidsTypeRef)
|
||||
} catch (_: Exception) {
|
||||
emptySet()
|
||||
}
|
||||
}
|
||||
|
||||
private fun toSqlDate(day: String): java.sql.Date =
|
||||
java.sql.Date.valueOf(day.trim().take(10))
|
||||
|
||||
override fun upsertSnapshotBatch(day: String, flights: List<Pair<String, String>>, now: Instant) {
|
||||
if (flights.isEmpty()) return
|
||||
val sql = """
|
||||
INSERT INTO flight_schd (flid, fday, fltr_json, created_at, updated_at)
|
||||
VALUES (?, ?, ?::jsonb, ?, ?)
|
||||
ON CONFLICT (flid) DO UPDATE SET
|
||||
fday = EXCLUDED.fday,
|
||||
fltr_json = EXCLUDED.fltr_json,
|
||||
updated_at = EXCLUDED.updated_at
|
||||
""".trimIndent()
|
||||
val sqlDate = toSqlDate(day)
|
||||
val sqlTimestamp = now.toSqlTimestamp()
|
||||
val conn = ds.obtainConnection()
|
||||
try {
|
||||
conn.prepareStatement(sql).use { ps ->
|
||||
var count = 0
|
||||
for ((flid, json) in flights) {
|
||||
ps.setString(1, flid)
|
||||
ps.setDate(2, sqlDate)
|
||||
ps.setString(3, json)
|
||||
ps.setTimestamp(4, sqlTimestamp)
|
||||
ps.setTimestamp(5, sqlTimestamp)
|
||||
ps.addBatch()
|
||||
count++
|
||||
if (count % 200 == 0) {
|
||||
ps.executeBatch()
|
||||
}
|
||||
}
|
||||
if (count % 200 != 0) {
|
||||
ps.executeBatch()
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
conn.releaseIfNotInTransaction()
|
||||
}
|
||||
}
|
||||
|
||||
override fun upsertIncremental(changes: List<com.gzzn.omms.msgexchange.domain.FlightChange>, now: Instant) {
|
||||
if (changes.isEmpty()) return
|
||||
val sql = """
|
||||
INSERT INTO flight_schd (flid, fday, fltr_json, created_at, updated_at)
|
||||
VALUES (?, NULL, ?::jsonb, ?, ?)
|
||||
ON CONFLICT (flid) DO UPDATE SET
|
||||
fltr_json = EXCLUDED.fltr_json,
|
||||
updated_at = EXCLUDED.updated_at
|
||||
""".trimIndent()
|
||||
val sqlTimestamp = now.toSqlTimestamp()
|
||||
val conn = ds.obtainConnection()
|
||||
try {
|
||||
conn.prepareStatement(sql).use { ps ->
|
||||
var count = 0
|
||||
for (c in changes) {
|
||||
ps.setString(1, c.flid)
|
||||
ps.setString(2, c.payloadJson)
|
||||
ps.setTimestamp(3, sqlTimestamp)
|
||||
ps.setTimestamp(4, sqlTimestamp)
|
||||
ps.addBatch()
|
||||
count++
|
||||
if (count % 200 == 0) {
|
||||
ps.executeBatch()
|
||||
}
|
||||
}
|
||||
if (count % 200 != 0) {
|
||||
ps.executeBatch()
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
conn.releaseIfNotInTransaction()
|
||||
}
|
||||
}
|
||||
|
||||
override fun deleteDiffByDay(day: String, delFlids: Collection<String>): Int {
|
||||
if (delFlids.isEmpty()) return 0
|
||||
var totalDeleted = 0
|
||||
val sqlDate = toSqlDate(day)
|
||||
for (chunk in delFlids.chunked(200)) {
|
||||
val placeholders = chunk.joinToString(",") { "?" }
|
||||
val sql = "DELETE FROM flight_schd WHERE fday = ? AND flid IN ($placeholders)"
|
||||
totalDeleted += ds.update(sql) { ps ->
|
||||
ps.setDate(1, sqlDate)
|
||||
chunk.forEachIndexed { i, flid -> ps.setString(i + 2, flid) }
|
||||
}
|
||||
}
|
||||
return totalDeleted
|
||||
}
|
||||
|
||||
override fun findByFlid(flid: String): String? =
|
||||
ds.queryOne(
|
||||
"SELECT fltr_json FROM flight_schd WHERE flid = ?",
|
||||
{ ps -> ps.setString(1, flid) },
|
||||
) { rs -> rs.getString("fltr_json") }
|
||||
|
||||
override fun findByFlids(flids: Collection<String>): Map<String, String> {
|
||||
if (flids.isEmpty()) return emptyMap()
|
||||
val result = mutableMapOf<String, String>()
|
||||
for (chunk in flids.chunked(200)) {
|
||||
val placeholders = chunk.joinToString(",") { "?" }
|
||||
val sql = "SELECT flid, fltr_json FROM flight_schd WHERE flid IN ($placeholders)"
|
||||
val pairs = ds.query(
|
||||
sql,
|
||||
{ ps -> chunk.forEachIndexed { i, flid -> ps.setString(i + 1, flid) } },
|
||||
) { rs -> rs.getString("flid") to rs.getString("fltr_json") }
|
||||
result.putAll(pairs)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
override fun findByDay(day: String): List<Pair<String, String>> =
|
||||
ds.query(
|
||||
"SELECT flid, fltr_json FROM flight_schd WHERE fday = ? ORDER BY flid ASC",
|
||||
{ ps -> ps.setDate(1, toSqlDate(day)) },
|
||||
) { rs -> rs.getString("flid") to rs.getString("fltr_json") }
|
||||
|
||||
override fun findAll(): Map<String, String> =
|
||||
ds.query(
|
||||
"SELECT flid, fltr_json FROM flight_schd ORDER BY flid ASC",
|
||||
{},
|
||||
) { rs -> rs.getString("flid") to rs.getString("fltr_json") }.toMap()
|
||||
|
||||
override fun deleteByFlids(flids: Set<String>): Int {
|
||||
if (flids.isEmpty()) return 0
|
||||
var totalDeleted = 0
|
||||
for (chunk in flids.chunked(200)) {
|
||||
val placeholders = chunk.joinToString(",") { "?" }
|
||||
val sql = "DELETE FROM flight_schd WHERE flid IN ($placeholders)"
|
||||
totalDeleted += ds.update(sql) { ps ->
|
||||
chunk.forEachIndexed { i, flid -> ps.setString(i + 1, flid) }
|
||||
}
|
||||
}
|
||||
return totalDeleted
|
||||
}
|
||||
|
||||
override fun getGen(day: String): FlightSchdRepository.GenMeta? =
|
||||
ds.queryOne(
|
||||
"SELECT fday, version, flids_json, updated_at FROM schd_gen WHERE fday = ?",
|
||||
{ ps -> ps.setDate(1, toSqlDate(day)) },
|
||||
) { rs ->
|
||||
FlightSchdRepository.GenMeta(
|
||||
fday = rs.getDate("fday").toString(),
|
||||
version = rs.getLong("version"),
|
||||
flids = parseFlids(rs.getString("flids_json")),
|
||||
updatedAt = rs.getInstant("updated_at") ?: Instant.now(),
|
||||
)
|
||||
}
|
||||
|
||||
override fun putGenIfVersion(day: String, expected: Long, newGen: FlightSchdRepository.GenMeta, now: Instant): Boolean {
|
||||
val flidsJson = serializeFlids(newGen.flids)
|
||||
val sqlDate = toSqlDate(day)
|
||||
val sqlTimestamp = now.toSqlTimestamp()
|
||||
if (expected == 0L) {
|
||||
val inserted = ds.update(
|
||||
"""
|
||||
INSERT INTO schd_gen (fday, version, flids_json, updated_at)
|
||||
VALUES (?, ?, ?::jsonb, ?)
|
||||
ON CONFLICT (fday) DO NOTHING
|
||||
""".trimIndent(),
|
||||
) { ps ->
|
||||
ps.setDate(1, sqlDate)
|
||||
ps.setLong(2, newGen.version)
|
||||
ps.setString(3, flidsJson)
|
||||
ps.setTimestamp(4, sqlTimestamp)
|
||||
}
|
||||
if (inserted == 1) return true
|
||||
}
|
||||
|
||||
val updated = ds.update(
|
||||
"""
|
||||
UPDATE schd_gen
|
||||
SET version = ?, flids_json = ?::jsonb, updated_at = ?
|
||||
WHERE fday = ? AND version = ?
|
||||
""".trimIndent(),
|
||||
) { ps ->
|
||||
ps.setLong(1, newGen.version)
|
||||
ps.setString(2, flidsJson)
|
||||
ps.setTimestamp(3, sqlTimestamp)
|
||||
ps.setDate(4, sqlDate)
|
||||
ps.setLong(5, expected)
|
||||
}
|
||||
return updated == 1
|
||||
}
|
||||
|
||||
override fun deleteGenBefore(cutoffDay: String): Int =
|
||||
ds.update(
|
||||
"DELETE FROM schd_gen WHERE fday < ?",
|
||||
{ ps -> ps.setDate(1, toSqlDate(cutoffDay)) },
|
||||
)
|
||||
}
|
||||
|
||||
@Singleton
|
||||
@Requires(property = "msgx.stubs", notEquals = "true")
|
||||
@Requires(property = "datasources.default.enabled", value = "true")
|
||||
class JdbcRefDataRepository(
|
||||
private val flightSchd: FlightSchdRepository,
|
||||
) : RefDataRepository {
|
||||
override fun getGen(day: String): RefDataRepository.GenMeta? =
|
||||
flightSchd.getGen(day)?.let { RefDataRepository.GenMeta(it.flids.toList(), it.version) }
|
||||
|
||||
override fun putGenIfVersion(day: String, expected: Long, new: RefDataRepository.GenMeta): Boolean =
|
||||
flightSchd.putGenIfVersion(
|
||||
day,
|
||||
expected,
|
||||
FlightSchdRepository.GenMeta(day, new.version, new.flids.toSet()),
|
||||
)
|
||||
}
|
||||
|
||||
@Singleton
|
||||
@@ -420,7 +643,12 @@ class JdbcReqTrackRepository(
|
||||
@Singleton
|
||||
@Requires(property = "msgx.stubs", notEquals = "true")
|
||||
@Requires(property = "datasources.default.enabled", value = "true")
|
||||
class JdbcFlightStateRepository : FlightStateRepository {
|
||||
override fun replaceDay(day: String, flights: List<Pair<String, String>>) = Unit
|
||||
override fun findByDay(day: String): List<Pair<String, String>> = emptyList()
|
||||
class JdbcFlightStateRepository(
|
||||
private val flightSchd: FlightSchdRepository,
|
||||
) : FlightStateRepository {
|
||||
override fun replaceDay(day: String, flights: List<Pair<String, String>>) =
|
||||
flightSchd.upsertSnapshotBatch(day, flights)
|
||||
|
||||
override fun findByDay(day: String): List<Pair<String, String>> =
|
||||
flightSchd.findByDay(day)
|
||||
}
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
package com.gzzn.omms.msgexchange.infra.redis
|
||||
|
||||
/**
|
||||
* ACMA-8 流程 4 / I4 / I5:Redis Lua 脚本装载与执行入口。
|
||||
* 阶段 A 全部 flightInfo 写均经此处,且仅由主泵线程调用(I5)。
|
||||
*/
|
||||
enum class RedisScript(val classpathLocation: String) {
|
||||
/** 同一 hash 原子“覆盖新代 + 按代差删”(setArg 为 N 对 field/value,delArg 为差集)。 */
|
||||
SNAPSHOT_REPLACE("lua/snapshot_replace.lua"),
|
||||
|
||||
/** 3:30 清场批量删除(仅 ES 归档成功集)。 */
|
||||
BATCH_DELETE("lua/batch_delete.lua"),
|
||||
}
|
||||
|
||||
interface FlightRedisClient {
|
||||
/**
|
||||
* 执行脚本。SNAPSHOT_REPLACE:setPairs 为新代全量 field/value,delFields 为按代差集;
|
||||
* BATCH_DELETE:delFields 为待删 FLID 集。
|
||||
*/
|
||||
fun eval(script: RedisScript, setPairs: List<Pair<String, String>> = emptyList(), delFields: List<String> = emptyList())
|
||||
|
||||
/** 阶段 A 权威读(处理决策 loadState、3:30 清场 findAll)。 */
|
||||
fun hgetAllFlightInfo(): Map<String, String>
|
||||
|
||||
/** 连通性探测(健康检查用;实现必须为快速调用,失败返回 false 而非抛出穿出)。 */
|
||||
fun ping(): Boolean
|
||||
}
|
||||
|
||||
// TODO(阶段1后续): 基于 micronaut-redis-lettuce 的实装(脚本自 classpath 装载并缓存 SHA)。
|
||||
@@ -5,8 +5,6 @@ import com.gzzn.omms.msgexchange.codec.DecodeResult
|
||||
import com.gzzn.omms.msgexchange.codec.XmlCodec
|
||||
import com.gzzn.omms.msgexchange.delivery.DeliveryPort
|
||||
import com.gzzn.omms.msgexchange.domain.ErrorClass
|
||||
import com.gzzn.omms.msgexchange.infra.redis.FlightRedisClient
|
||||
import com.gzzn.omms.msgexchange.infra.redis.RedisScript
|
||||
import com.gzzn.omms.msgexchange.processing.CodecHolder
|
||||
import com.gzzn.omms.msgexchange.processing.HandlerHolder
|
||||
import com.gzzn.omms.msgexchange.processing.HandlerRegistry
|
||||
@@ -16,7 +14,7 @@ import io.micronaut.context.annotation.Requires
|
||||
import jakarta.inject.Singleton
|
||||
|
||||
/**
|
||||
* U07:stub 适配层——XmlCodec / Redis 客户端 / DeliveryPort / Handler 装配,仅在 msgx.stubs=true 时生效。
|
||||
* U07:stub 适配层——XmlCodec / DeliveryPort / Handler 装配,仅在 msgx.stubs=true 时生效(Redis 客户端已按 ACM2-28 移除)。
|
||||
* stub codec 未实装 → 报文 decode 返回 CODEC_ERROR(走 FAILED 可重放路径,链路上可观测),
|
||||
* 语义见 ACM2-10 U11:不把“未实装”写成报文非法/终态。
|
||||
*/
|
||||
@@ -29,30 +27,6 @@ class StubXmlCodec : XmlCodec {
|
||||
override fun encodeRqrd(kind: String, rangeJson: String): String = ""
|
||||
}
|
||||
|
||||
@Requires(property = "msgx.stubs", value = "true")
|
||||
@Singleton
|
||||
class StubRedis : FlightRedisClient {
|
||||
val hash = mutableMapOf<String, String>()
|
||||
val evalCalls = mutableListOf<RedisScript>()
|
||||
|
||||
fun clear() { hash.clear(); evalCalls.clear() }
|
||||
|
||||
override fun eval(script: RedisScript, setPairs: List<Pair<String, String>>, delFields: List<String>) {
|
||||
evalCalls += script
|
||||
when (script) {
|
||||
RedisScript.SNAPSHOT_REPLACE -> {
|
||||
hash.putAll(setPairs)
|
||||
delFields.forEach { hash.remove(it) }
|
||||
}
|
||||
RedisScript.BATCH_DELETE -> delFields.forEach { hash.remove(it) }
|
||||
}
|
||||
}
|
||||
|
||||
override fun hgetAllFlightInfo(): Map<String, String> = hash.toMap()
|
||||
|
||||
override fun ping(): Boolean = true
|
||||
}
|
||||
|
||||
@Requires(property = "msgx.stubs", value = "true")
|
||||
@Singleton
|
||||
class StubDeliveryPort : DeliveryPort {
|
||||
@@ -62,7 +36,6 @@ class StubDeliveryPort : DeliveryPort {
|
||||
|
||||
override fun sendKafka(topic: String, payloadJson: String) { sent += topic to payloadJson }
|
||||
override fun indexFlightHts(payloadJson: String) = Unit
|
||||
override fun projectRedis(payloadJson: String) = Unit
|
||||
}
|
||||
|
||||
/** Holder 工厂:CodecHolder/HandlerHolder 由 Micronaut Bean 提供(取代直连构造占位)。 */
|
||||
|
||||
@@ -7,8 +7,10 @@ import com.gzzn.omms.msgexchange.domain.ProcState
|
||||
import com.gzzn.omms.msgexchange.domain.ProcStatus
|
||||
import com.gzzn.omms.msgexchange.domain.RefUpsert
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.PumpJobRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.RefDataRepository
|
||||
@@ -187,22 +189,118 @@ class StubPumpJobs : PumpJobRepository {
|
||||
override fun markFailed(jobId: Long, lastError: String) { rows[jobId]?.state = "FAILED"; rows[jobId]?.lastError = lastError }
|
||||
}
|
||||
|
||||
/** 快照 gen 协议 stub(业务库 REF_DATA 的 SCHD_GEN 行;ACM2-11 拆分后 21 类不在本实现)。 */
|
||||
@Requires(property = "msgx.stubs", value = "true")
|
||||
@Singleton
|
||||
class StubRefData : RefDataRepository {
|
||||
private val gens = mutableMapOf<String, RefDataRepository.GenMeta>()
|
||||
class StubPipelineTransactionManager : PipelineTransactionManager {
|
||||
override fun <T> inTransaction(block: () -> T): T = block()
|
||||
}
|
||||
|
||||
fun clear() { gens.clear() }
|
||||
@Requires(property = "msgx.stubs", value = "true")
|
||||
@Singleton
|
||||
class StubFlightSchd : FlightSchdRepository {
|
||||
data class Record(
|
||||
val flid: String,
|
||||
val fday: String?,
|
||||
val fltrJson: String,
|
||||
val createdAt: Instant,
|
||||
val updatedAt: Instant,
|
||||
)
|
||||
|
||||
override fun getGen(day: String): RefDataRepository.GenMeta? = gens[day]
|
||||
private val records = mutableMapOf<String, Record>()
|
||||
private val gens = mutableMapOf<String, FlightSchdRepository.GenMeta>()
|
||||
|
||||
override fun putGenIfVersion(day: String, expected: Long, new: RefDataRepository.GenMeta): Boolean {
|
||||
val cur = gens[day]?.version ?: 0L
|
||||
if (cur != expected) return false
|
||||
gens[day] = new
|
||||
fun clear() {
|
||||
records.clear()
|
||||
gens.clear()
|
||||
}
|
||||
|
||||
override fun upsertSnapshotBatch(day: String, flights: List<Pair<String, String>>, now: Instant) {
|
||||
for ((flid, json) in flights) {
|
||||
val existing = records[flid]
|
||||
records[flid] = Record(
|
||||
flid = flid,
|
||||
fday = day,
|
||||
fltrJson = json,
|
||||
createdAt = existing?.createdAt ?: now,
|
||||
updatedAt = now,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override fun upsertIncremental(changes: List<com.gzzn.omms.msgexchange.domain.FlightChange>, now: Instant) {
|
||||
for (c in changes) {
|
||||
val existing = records[c.flid]
|
||||
records[c.flid] = Record(
|
||||
flid = c.flid,
|
||||
fday = existing?.fday, // preserve existing fday; null if new
|
||||
fltrJson = c.payloadJson,
|
||||
createdAt = existing?.createdAt ?: now,
|
||||
updatedAt = now,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override fun deleteDiffByDay(day: String, delFlids: Collection<String>): Int {
|
||||
if (delFlids.isEmpty()) return 0
|
||||
val delSet = delFlids.toSet()
|
||||
val toRemove = records.filter { (flid, rec) -> rec.fday == day && flid in delSet }.keys
|
||||
toRemove.forEach { records.remove(it) }
|
||||
return toRemove.size
|
||||
}
|
||||
|
||||
override fun findByFlid(flid: String): String? = records[flid]?.fltrJson
|
||||
|
||||
override fun findByFlids(flids: Collection<String>): Map<String, String> =
|
||||
flids.mapNotNull { flid -> records[flid]?.let { flid to it.fltrJson } }.toMap()
|
||||
|
||||
override fun findByDay(day: String): List<Pair<String, String>> =
|
||||
records.values.filter { it.fday == day }
|
||||
.sortedBy { it.flid }
|
||||
.map { it.flid to it.fltrJson }
|
||||
|
||||
override fun findAll(): Map<String, String> =
|
||||
records.mapValues { it.value.fltrJson }
|
||||
|
||||
override fun deleteByFlids(flids: Set<String>): Int {
|
||||
if (flids.isEmpty()) return 0
|
||||
var count = 0
|
||||
for (flid in flids) {
|
||||
if (records.remove(flid) != null) count++
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
override fun getGen(day: String): FlightSchdRepository.GenMeta? = gens[day]
|
||||
|
||||
override fun putGenIfVersion(day: String, expected: Long, newGen: FlightSchdRepository.GenMeta, now: Instant): Boolean {
|
||||
val current = gens[day]?.version ?: 0L
|
||||
if (current != expected) return false
|
||||
gens[day] = newGen.copy(updatedAt = now)
|
||||
return true
|
||||
}
|
||||
|
||||
override fun deleteGenBefore(cutoffDay: String): Int {
|
||||
val toRemove = gens.keys.filter { it < cutoffDay }
|
||||
toRemove.forEach { gens.remove(it) }
|
||||
return toRemove.size
|
||||
}
|
||||
}
|
||||
|
||||
/** 快照 gen 协议 stub:委托至 StubFlightSchd。 */
|
||||
@Requires(property = "msgx.stubs", value = "true")
|
||||
@Singleton
|
||||
class StubRefData(
|
||||
private val flightSchd: FlightSchdRepository,
|
||||
) : RefDataRepository {
|
||||
override fun getGen(day: String): RefDataRepository.GenMeta? =
|
||||
flightSchd.getGen(day)?.let { RefDataRepository.GenMeta(it.flids.toList(), it.version) }
|
||||
|
||||
override fun putGenIfVersion(day: String, expected: Long, new: RefDataRepository.GenMeta): Boolean =
|
||||
flightSchd.putGenIfVersion(
|
||||
day,
|
||||
expected,
|
||||
FlightSchdRepository.GenMeta(day, new.version, new.flids.toSet()),
|
||||
)
|
||||
}
|
||||
|
||||
/** 21 类静态主数据 stub(独立 PG reference 库;内存实现,source 保留供审计断言)。 */
|
||||
@@ -254,14 +352,12 @@ class StubReqTrack : ReqTrackRepository {
|
||||
|
||||
@Requires(property = "msgx.stubs", value = "true")
|
||||
@Singleton
|
||||
class StubFlightState : FlightStateRepository {
|
||||
private val byDay = mutableMapOf<String, MutableList<Pair<String, String>>>()
|
||||
class StubFlightState(
|
||||
private val flightSchd: FlightSchdRepository,
|
||||
) : FlightStateRepository {
|
||||
override fun replaceDay(day: String, flights: List<Pair<String, String>>) =
|
||||
flightSchd.upsertSnapshotBatch(day, flights)
|
||||
|
||||
fun clear() { byDay.clear() }
|
||||
|
||||
override fun replaceDay(day: String, flights: List<Pair<String, String>>) {
|
||||
byDay[day] = flights.toMutableList()
|
||||
}
|
||||
|
||||
override fun findByDay(day: String): List<Pair<String, String>> = byDay[day] ?: emptyList()
|
||||
override fun findByDay(day: String): List<Pair<String, String>> =
|
||||
flightSchd.findByDay(day)
|
||||
}
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package com.gzzn.omms.msgexchange.jobs
|
||||
|
||||
import com.gzzn.omms.msgexchange.infra.redis.FlightRedisClient
|
||||
import com.gzzn.omms.msgexchange.infra.redis.RedisScript
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.FlightStateRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.PumpJobRepository
|
||||
import com.gzzn.omms.msgexchange.domain.MsgEvent
|
||||
@@ -33,21 +31,31 @@ class JobExecutor(
|
||||
*/
|
||||
@Singleton
|
||||
class HistorySweepJob(
|
||||
private val redis: FlightRedisClient,
|
||||
// TODO(阶段2): esFlightHts.saveSync(history) 返回成功集(同步写,不经事件 ack 回查)
|
||||
private val flightSchd: FlightSchdRepository,
|
||||
) {
|
||||
companion object {
|
||||
var historyPicker: ((Map<String, String>) -> Map<String, String>)? = null
|
||||
var esArchiver: ((Map<String, String>) -> Set<String>)? = null
|
||||
var cutoffProvider: (() -> String?)? = null
|
||||
}
|
||||
|
||||
fun run() {
|
||||
val all = redis.hgetAllFlightInfo()
|
||||
val all = flightSchd.findAll()
|
||||
val history = pickHistory(all) // U10/T07:saveSync 接线前判史为空集——禁止“全量可删”默认
|
||||
val success = emptyList<String>() // TODO(阶段2): esFlightHts.saveSync(history)(返回成功集后再接线删除)
|
||||
val success = esArchiver?.invoke(history) ?: emptySet() // 仅收集已确认写入 ES 的成功 FLID 集合
|
||||
if (success.isNotEmpty()) {
|
||||
redis.eval(RedisScript.BATCH_DELETE, delFields = success)
|
||||
flightSchd.deleteByFlids(success) // FS2/FS4:仅删除已确认写入 ES 的成功集合,分批参数化删除
|
||||
}
|
||||
val cutoff = cutoffProvider?.invoke()
|
||||
if (cutoff != null) {
|
||||
flightSchd.deleteGenBefore(cutoff)
|
||||
}
|
||||
}
|
||||
|
||||
/** U10/T07(修订):接入 ES success 集之前的门禁——占位默认返回空集;
|
||||
* 现役五条判史规则(SODT 3 天 / CNCL 1 小时 / 备降 / 离港 / 到港)golden 通过后才允许接线。 */
|
||||
private fun pickHistory(all: Map<String, String>): Map<String, String> = emptyMap() // TODO(阶段2)
|
||||
private fun pickHistory(all: Map<String, String>): Map<String, String> =
|
||||
historyPicker?.invoke(all) ?: emptyMap()
|
||||
}
|
||||
|
||||
/** 流程 5 runJob(ARCHIVE):3:00 归档——1 天前且仅终态可迁(矩阵 #12)。 */
|
||||
@@ -63,19 +71,18 @@ class ArchiveJob(
|
||||
/** 流程 7:阶段 B 投影重建——切入阶段 B 时全量重建一次,此后增量走事件。 */
|
||||
@Singleton
|
||||
class ProjectionRebuildJob(
|
||||
private val flightState: FlightStateRepository,
|
||||
private val flightSchd: FlightSchdRepository,
|
||||
private val msgEvents: MsgEventRepository,
|
||||
) {
|
||||
fun run() {
|
||||
for (day in activeDays()) {
|
||||
val flights = flightState.findByDay(day)
|
||||
val flights = flightSchd.findByDay(day)
|
||||
flights.forEach { (flid, payload) ->
|
||||
msgEvents.insertSync(
|
||||
listOf(MsgEvent(target = Targets.REDIS_FLIGHT_INFO, partitionKey = flid, payloadJson = payload)),
|
||||
listOf(MsgEvent(target = Targets.KAFKA_SCHD, partitionKey = flid, payloadJson = payload)),
|
||||
)
|
||||
}
|
||||
}
|
||||
// 重建完成前旧 Redis 视图继续服务(数据同源,仅短暂滞后)
|
||||
}
|
||||
|
||||
private fun activeDays(): List<String> = emptyList() // TODO(阶段B)
|
||||
|
||||
@@ -11,7 +11,7 @@ import com.gzzn.omms.msgexchange.domain.MsgKind
|
||||
interface Handler {
|
||||
val kind: MsgKind
|
||||
|
||||
/** 在线状态以只读快照传入(阶段 A 读 Redis 权威、阶段 B 读 FLIGHT_STATE,由调用方装配)。 */
|
||||
/** 在线状态以只读视图传入(阶段 A 点查自有 PG FLIGHT_SCHD,由调用方装配,ACM2-28 定案)。 */
|
||||
fun decide(flightView: Map<String, String>, msg: DecodedMessage): Decision
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,8 @@ import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.PumpJobRepository
|
||||
import com.gzzn.omms.msgexchange.infra.redis.FlightRedisClient
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
|
||||
import com.gzzn.omms.msgexchange.infra.retry.ProcFailure
|
||||
import com.gzzn.omms.msgexchange.jobs.JobExecutor
|
||||
import jakarta.inject.Singleton
|
||||
@@ -20,7 +21,7 @@ import java.time.Instant
|
||||
|
||||
/**
|
||||
* ACMA-8 流程 2:处理主泵——严格 FIFO + HOL + 毒丸 DEAD 升级(I1);
|
||||
* 阶段 A 全部 Redis 写在本线程(I5),且先于事件创建(I2 happens-before)。
|
||||
* 阶段 A 运营航班 FLIGHT_SCHD 与待发事件、终态同事务原子提交(I2/I5,ACM2-28 定案)。
|
||||
*/
|
||||
@Singleton
|
||||
class Pump(
|
||||
@@ -114,12 +115,25 @@ class MessageProcessor(
|
||||
private val msgEvents: MsgEventRepository,
|
||||
private val codecHolder: CodecHolder,
|
||||
private val handlers: HandlerHolder,
|
||||
private val redis: FlightRedisClient,
|
||||
private val flightSchd: FlightSchdRepository,
|
||||
private val snapshotFlow: SnapshotFlow,
|
||||
private val procFailure: ProcFailure,
|
||||
private val props: PipelineProps,
|
||||
private val txManager: PipelineTransactionManager,
|
||||
) {
|
||||
private val log = org.slf4j.LoggerFactory.getLogger(MessageProcessor::class.java)
|
||||
private val flidRegex = Regex("<FLID>(.*?)</FLID>", RegexOption.IGNORE_CASE)
|
||||
|
||||
private fun extractCandidateFlids(decoded: DecodedMessage): Set<String> {
|
||||
val fromXml = flidRegex.findAll(decoded.rawXml).map { it.groupValues[1].trim() }.filter { it.isNotEmpty() }.toSet()
|
||||
if (fromXml.isNotEmpty()) return fromXml
|
||||
val b = decoded.body
|
||||
if (b is Map<*, *>) {
|
||||
val flid = b["FLID"] ?: b["flid"]
|
||||
if (flid != null) return setOf(flid.toString())
|
||||
}
|
||||
return emptySet()
|
||||
}
|
||||
|
||||
fun processOne(head: ProcState) {
|
||||
com.gzzn.omms.msgexchange.infra.log.TraceLog.withTrace(head.cminmsgsId) {
|
||||
@@ -176,8 +190,7 @@ class MessageProcessor(
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 快照消息走专属流程(流程 4:staging→Lua→putGenIfVersion CAS 同事务)
|
||||
// 快照消息走专属流程(流程 4:staging → SQL 批处理/域内差删/CAS 同事务)
|
||||
val handler = handlers.registry.dispatcherFor(decoded)
|
||||
if (handler == null) {
|
||||
// U10(N21):未注册 ≠ 报文非法——写 FAILED(可重放),绝不写终态
|
||||
@@ -193,23 +206,35 @@ class MessageProcessor(
|
||||
return
|
||||
}
|
||||
|
||||
val decision = handler.decide(redis.hgetAllFlightInfo(), decoded) // 纯函数
|
||||
|
||||
if (props.phase == PipelineProps.Phase.A) {
|
||||
// 阶段 A:主泵线程先写 Redis(幂等),先于事件创建(I2);TODO: redisApply(flightChanges)
|
||||
val candidateFlids = extractCandidateFlids(decoded)
|
||||
val flightView = if (candidateFlids.isNotEmpty()) {
|
||||
flightSchd.findByFlids(candidateFlids)
|
||||
} else {
|
||||
emptyMap()
|
||||
}
|
||||
// 事务 2(自有 PG 内原子):事件 + SUCCEEDED(实装后 @Transactional);
|
||||
// CMINMSGS 回填 = 共享信箱外部副作用(最终一致,ACM2-12);
|
||||
// 静态主数据(refUpserts)同自有 PG 但弱事务独立提交(21 类 REF_MASTER)。
|
||||
|
||||
val decision = handler.decide(flightView, decoded) // 纯函数
|
||||
|
||||
// 事务 2(自有 PG 单事务原子):
|
||||
// upsert FLIGHT_SCHD(decision.flightChanges) + insert MSG_EVENT + PROC_STATE → SUCCEEDED
|
||||
val events = buildList {
|
||||
decision.msgNotifies.forEach { add(MsgEvent(target = Targets.KAFKA_MSG, payloadJson = it.payloadJson)) }
|
||||
decision.schdPush.forEach { add(MsgEvent(target = Targets.KAFKA_SCHD, partitionKey = it.flid, payloadJson = it.fltrJson)) }
|
||||
}
|
||||
msgEvents.insertAll(events)
|
||||
|
||||
txManager.inTransaction {
|
||||
if (decision.flightChanges.isNotEmpty()) {
|
||||
flightSchd.upsertIncremental(decision.flightChanges)
|
||||
}
|
||||
if (events.isNotEmpty()) {
|
||||
msgEvents.insertAll(events)
|
||||
}
|
||||
procState.update(head.cminmsgsId, ProcStatus.SUCCEEDED)
|
||||
}
|
||||
|
||||
// 提交后:backfill CMINMSGS(共享信箱外部副作用,补偿链路)
|
||||
inbox.backfillOnSuccess(head.cminmsgsId, decoded.meta.sndr, decoded.meta.type, decoded.meta.styp, decoded.meta.seqn)
|
||||
procState.update(head.cminmsgsId, ProcStatus.SUCCEEDED)
|
||||
log.info("SUCCEEDED id={} events={}", head.cminmsgsId, events.size)
|
||||
// 阶段 B:flightState.apply(decision.flightChanges) 进入同一事务;投影事件(ES/REDIS)追加。
|
||||
log.info("SUCCEEDED id={} events={} flightChanges={}", head.cminmsgsId, events.size, decision.flightChanges.size)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,59 +4,114 @@ import com.gzzn.omms.msgexchange.domain.DecodedMessage
|
||||
import com.gzzn.omms.msgexchange.domain.ErrorClass
|
||||
import com.gzzn.omms.msgexchange.domain.ProcState
|
||||
import com.gzzn.omms.msgexchange.domain.ProcStatus
|
||||
import com.gzzn.omms.msgexchange.domain.MsgEvent
|
||||
import com.gzzn.omms.msgexchange.domain.MsgKind
|
||||
import com.gzzn.omms.msgexchange.domain.Targets
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.FlightSchdRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.MsgEventRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.RefDataRepository
|
||||
import com.gzzn.omms.msgexchange.infra.redis.FlightRedisClient
|
||||
import com.gzzn.omms.msgexchange.infra.redis.RedisScript
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.ReqTrackRepository
|
||||
import com.gzzn.omms.msgexchange.infra.retry.ProcFailure
|
||||
import jakarta.inject.Singleton
|
||||
|
||||
/**
|
||||
* ACMA-8 流程 4:日计划快照(generation,主泵内执行)。
|
||||
* staging(内存瞬态,崩溃从 raw 整包重放)→ Lua 原子“覆盖+按代差删”(I4/I5)
|
||||
* → putGenIfVersion CAS + SUCCEEDED(重放幂等,版本不二次自增——恢复协议细节见 U09)。
|
||||
* staging(内存瞬态,崩溃从 raw 整包重放)→ PG 单事务原子“覆盖+按代差删+SQL CAS 推进+事件入队+SUCCEEDED”(I4/I5,ACM2-28 定案)。
|
||||
* U10/T07 占位安全化 + U08 统一失败迁移(ProcFailure):staging 未实装 → FAILED(UNSUPPORTED)+退避
|
||||
* (可重放,绝不写终态);CAS 冲突 → FAILED(INFRA)+退避;达上限统一 DEAD(EXHAUSTED)。
|
||||
*/
|
||||
@Singleton
|
||||
class SnapshotFlow(
|
||||
private val procState: ProcStateRepository,
|
||||
private val refData: RefDataRepository,
|
||||
private val redis: FlightRedisClient,
|
||||
private val flightSchd: FlightSchdRepository,
|
||||
private val msgEvents: MsgEventRepository,
|
||||
private val reqTrack: ReqTrackRepository,
|
||||
private val procFailure: ProcFailure,
|
||||
private val txManager: PipelineTransactionManager,
|
||||
private val inbox: CminmsgInboxRepository,
|
||||
) {
|
||||
private val log = org.slf4j.LoggerFactory.getLogger(SnapshotFlow::class.java)
|
||||
|
||||
fun publishSnapshot(head: ProcState, msg: DecodedMessage) {
|
||||
// 1) staging:流式解析 + 整包校验(TODO(阶段2): 流式 codec;千级 FLTR 为 MB 级,内存瞬态)
|
||||
val staged = StageResult.stagingOf(msg) // 骨架:TODO 解析 FLTR 集与重组(KEEP 现役 MAFL/登机桥规则)
|
||||
// 1) staging:流式解析 + 整包校验(内存瞬态,崩溃从 raw 整包重放)
|
||||
val staged = StageResult.stagingOf(msg)
|
||||
if (staged is StageResult.Invalid) {
|
||||
log.warn("staging not implemented -> FAILED(UNSUPPORTED) id={} reason={}", head.cminmsgsId, staged.reason)
|
||||
procFailure.fail(head, ErrorClass.UNSUPPORTED, staged.reason) // U10:未实装 → 可重放,非终态
|
||||
return
|
||||
}
|
||||
val normalized = (staged as StageResult.Ok).flights // (flid, payloadJson)
|
||||
val ok = staged as StageResult.Ok
|
||||
val normalized = ok.flights // (flid, payloadJson)
|
||||
val day = ok.day
|
||||
|
||||
// 2) 发布:Lua 原子覆盖 + 按代差删(删除集 = gen.flids − 新代,ADFT 自动存活)
|
||||
val day = staged.day
|
||||
val gen = refData.getGen(day)
|
||||
val newFlids = normalized.map { it.first }
|
||||
val delFields = gen?.flids?.minus(newFlids.toSet()) ?: emptyList()
|
||||
redis.eval(RedisScript.SNAPSHOT_REPLACE, setPairs = normalized, delFields = delFields)
|
||||
|
||||
// 3) 事务:putGen CAS + SUCCEEDED(实装后同 @Transactional;CAS 失败=并发,串行泵下不应发生→告警)
|
||||
// 内存与超大包熔断防御(ACM2-28 评论 4 项 3)
|
||||
if (normalized.size > MAX_FLIGHTS_PER_SNAPSHOT) {
|
||||
log.error("snapshot flights exceed limit -> DEAD(MALFORMED) id={} size={}", head.cminmsgsId, normalized.size)
|
||||
procState.update(head.cminmsgsId, ProcStatus.DEAD, errorClass = ErrorClass.MALFORMED,
|
||||
lastError = "flights-exceed-limit:${normalized.size}")
|
||||
return
|
||||
}
|
||||
|
||||
// 2) 准备代元数据与差集(删除集 = gen.flids − 新代,ADFT 自动存活)
|
||||
val gen = flightSchd.getGen(day)
|
||||
val expected = gen?.version ?: 0L
|
||||
if (!refData.putGenIfVersion(day, expected, RefDataRepository.GenMeta(newFlids, expected + 1))) {
|
||||
// 重放路径:version 已是目标值 → no-op 视为成功
|
||||
val again = refData.getGen(day)
|
||||
if (again == null || again.version != expected + 1) {
|
||||
log.warn("gen CAS conflict -> FAILED(INFRA) id={}", head.cminmsgsId)
|
||||
procFailure.fail(head, ErrorClass.INFRA, "gen-cas-conflict") // N06/N28:带退避,禁止紧循环
|
||||
return
|
||||
val newFlids = normalized.map { it.first }.toSet()
|
||||
val delFields = gen?.flids?.minus(newFlids) ?: emptySet()
|
||||
val newVersion = expected + 1L
|
||||
|
||||
// 3) 自有 PG 单事务原子提交(ACM2-28 定案:覆盖新代 + 域内差删 + SQL CAS + 事件 + SUCCEEDED)
|
||||
try {
|
||||
txManager.inTransaction {
|
||||
// 覆盖新代全量:强行声明 FDAY 归属(JDBC batch 批处理)
|
||||
flightSchd.upsertSnapshotBatch(day, normalized)
|
||||
|
||||
// 按代差删域化:仅删除 FDAY = day 且在 delFields 中的记录(ADFT 与跨代已迁移行存活)
|
||||
if (delFields.isNotEmpty()) {
|
||||
flightSchd.deleteDiffByDay(day, delFields)
|
||||
}
|
||||
|
||||
// SQL CAS 版本推进(防双写断言)
|
||||
val casSuccess = flightSchd.putGenIfVersion(
|
||||
day = day,
|
||||
expected = expected,
|
||||
newGen = FlightSchdRepository.GenMeta(fday = day, version = newVersion, flids = newFlids),
|
||||
)
|
||||
if (!casSuccess) {
|
||||
// 重放路径:version 已是目标值 → no-op 视为成功,否则为 CAS 冲突
|
||||
val again = flightSchd.getGen(day)
|
||||
if (again == null || again.version != newVersion) {
|
||||
throw CasConflictException("gen-cas-conflict: expected=$expected current=${again?.version}")
|
||||
}
|
||||
}
|
||||
|
||||
// RESP 匹配 REQ_TRACK -> DONE
|
||||
val schdKind = msg.kind as? MsgKind.Schd
|
||||
if (schdKind?.subtype == MsgKind.SchdSubtype.RESP) {
|
||||
reqTrack.findOpenByKind("SCHD")?.let { req ->
|
||||
reqTrack.markDone(req.reqId)
|
||||
}
|
||||
}
|
||||
|
||||
// 构造并批量写入 schd 投递事件
|
||||
val events = normalized.map { (flid, json) ->
|
||||
MsgEvent(target = Targets.KAFKA_SCHD, partitionKey = flid, payloadJson = json)
|
||||
}
|
||||
if (events.isNotEmpty()) {
|
||||
msgEvents.insertAll(events)
|
||||
}
|
||||
|
||||
// 终态置 SUCCEEDED
|
||||
procState.update(head.cminmsgsId, ProcStatus.SUCCEEDED)
|
||||
}
|
||||
|
||||
// 提交后:backfill CMINMSGS(外部副作用,补偿保障)
|
||||
inbox.backfillOnSuccess(head.cminmsgsId, msg.meta.sndr, msg.meta.type, msg.meta.styp, msg.meta.seqn)
|
||||
log.info("snapshot SUCCEEDED id={} day={} flights={}", head.cminmsgsId, day, normalized.size)
|
||||
} catch (e: CasConflictException) {
|
||||
log.warn("gen CAS conflict -> FAILED(INFRA) id={} msg={}", head.cminmsgsId, e.message)
|
||||
procFailure.fail(head, ErrorClass.INFRA, e.message ?: "gen-cas-conflict")
|
||||
}
|
||||
procState.update(head.cminmsgsId, ProcStatus.SUCCEEDED)
|
||||
log.info("snapshot SUCCEEDED id={} day={} flights={}", head.cminmsgsId, day, normalized.size)
|
||||
}
|
||||
|
||||
/** staging 结果(骨架)。 */
|
||||
@@ -65,8 +120,12 @@ class SnapshotFlow(
|
||||
data class Invalid(val reason: String) : StageResult
|
||||
|
||||
companion object {
|
||||
var parser: ((DecodedMessage) -> StageResult)? = null
|
||||
|
||||
fun stagingOf(msg: DecodedMessage): StageResult =
|
||||
Invalid("staging-not-implemented(${msg.typeTag})") // TODO(阶段2)
|
||||
parser?.invoke(msg) ?: Invalid("staging-not-implemented(${msg.typeTag})") // TODO(阶段2)
|
||||
}
|
||||
}
|
||||
}
|
||||
class CasConflictException(message: String) : RuntimeException(message)
|
||||
const val MAX_FLIGHTS_PER_SNAPSHOT = 10000
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
package com.gzzn.omms.msgexchange.tools
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode
|
||||
import com.fasterxml.jackson.databind.ObjectMapper
|
||||
import com.fasterxml.jackson.databind.node.ArrayNode
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode
|
||||
import com.fasterxml.jackson.databind.node.ValueNode
|
||||
import java.math.BigDecimal
|
||||
|
||||
/**
|
||||
* ACM2-28 FS7:影子对拍跨存储 Diff 比较内核。
|
||||
*
|
||||
* 核心定位:
|
||||
* 本工具提供跨存储航班 AST 结构对比、字段规范化与已知合法偏离识别内核;
|
||||
* 外部装配侧(从 PostgreSQL/Redis 读取数据并按 CMINMSGS_ID 水位对齐取数)属于影子期装配调用方职责。
|
||||
* 字段归一化规则(ACM2-28 评论 4 项 4):
|
||||
* 1. 递归 AST/Map 结构对比,绝不做裸字符串比对(规避 PG jsonb 自动去重重排 key 的分叉);
|
||||
* 2. 数值精度归一化(1.0 vs 1 视为等价);
|
||||
* 3. 空值规范化(JSON null 与字段缺失视为等价,规避无语义偏离);
|
||||
*
|
||||
* 已知合法偏离声明(ACM2-28 方案六):
|
||||
* ① FDAY 域化差删对跨代迁移行的保护(legacy 误删、nextgen 不删,属于修正性偏离);
|
||||
* ② JSONB 与 Redis string 的 key 排序差异。
|
||||
*/
|
||||
class FlightStoreDiffTool(
|
||||
private val mapper: ObjectMapper = ObjectMapper(),
|
||||
) {
|
||||
|
||||
enum class DeviationKind {
|
||||
/** 已知合法偏离 ①:跨日迁移航班在 nextgen PG 中受 FDAY 保护未被误删,而 legacy 误删 */
|
||||
CROSS_DAY_PROTECTED,
|
||||
/** 非预期偏离:字段值不匹配 */
|
||||
FIELD_MISMATCH,
|
||||
/** 非预期偏离:PG 缺失该航班 */
|
||||
MISSING_IN_PG,
|
||||
/** 非预期偏离:Redis 缺失且非已知迁移保护行 */
|
||||
UNEXPECTED_EXTRA_IN_PG,
|
||||
}
|
||||
|
||||
data class Deviation(
|
||||
val flid: String,
|
||||
val kind: DeviationKind,
|
||||
val path: String? = null,
|
||||
val pgValue: Any? = null,
|
||||
val legacyValue: Any? = null,
|
||||
val detail: String,
|
||||
)
|
||||
|
||||
data class DiffReport(
|
||||
val totalPg: Int,
|
||||
val totalLegacy: Int,
|
||||
val matchedCount: Int,
|
||||
val knownDeviations: List<Deviation>,
|
||||
val unexpectedDeviations: List<Deviation>,
|
||||
) {
|
||||
val isGreen: Boolean
|
||||
get() = unexpectedDeviations.isEmpty()
|
||||
|
||||
fun formatSummary(): String {
|
||||
val sb = StringBuilder()
|
||||
sb.appendLine("========== 影子对拍跨存储 Diff 报告 ==========")
|
||||
sb.appendLine("Nextgen PG 航班总数 : $totalPg")
|
||||
sb.appendLine("Legacy Redis 航班总数 : $totalLegacy")
|
||||
sb.appendLine("完全一致航班数 : $matchedCount")
|
||||
sb.appendLine("已知合法偏离数 (偏差①) : ${knownDeviations.size}")
|
||||
sb.appendLine("非预期异常数 : ${unexpectedDeviations.size}")
|
||||
sb.appendLine("验收红绿灯状态 : ${if (isGreen) "GREEN (通过)" else "RED (未通过)"}")
|
||||
sb.appendLine("----------------------------------------------")
|
||||
if (knownDeviations.isNotEmpty()) {
|
||||
sb.appendLine("[已知合法偏离]")
|
||||
knownDeviations.forEach { d ->
|
||||
sb.appendLine(" FLID=${d.flid}: ${d.detail}")
|
||||
}
|
||||
}
|
||||
if (unexpectedDeviations.isNotEmpty()) {
|
||||
sb.appendLine("[非预期异常偏离]")
|
||||
unexpectedDeviations.forEach { d ->
|
||||
sb.appendLine(" FLID=${d.flid} [${d.kind}] path=${d.path}: PG=${d.pgValue}, Legacy=${d.legacyValue} (${d.detail})")
|
||||
}
|
||||
}
|
||||
sb.appendLine("==============================================")
|
||||
return sb.toString()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行全量比对。
|
||||
* @param pgFlights nextgen FLIGHT_SCHD 行(flid -> fltr_json)
|
||||
* @param legacyFlights legacy Redis flightInfo 行(flid -> value string)
|
||||
* @param crossDayMigratedFlids 已知在多日之间迁移的航班 FLID 集合(用于识别合法偏差 ①)
|
||||
*/
|
||||
fun diff(
|
||||
pgFlights: Map<String, String>,
|
||||
legacyFlights: Map<String, String>,
|
||||
crossDayMigratedFlids: Set<String> = emptySet(),
|
||||
): DiffReport {
|
||||
val allFlids = (pgFlights.keys + legacyFlights.keys).toSortedSet()
|
||||
var matched = 0
|
||||
val known = mutableListOf<Deviation>()
|
||||
val unexpected = mutableListOf<Deviation>()
|
||||
|
||||
for (flid in allFlids) {
|
||||
val pgRaw = pgFlights[flid]
|
||||
val legacyRaw = legacyFlights[flid]
|
||||
|
||||
if (pgRaw == null) {
|
||||
unexpected += Deviation(
|
||||
flid = flid,
|
||||
kind = DeviationKind.MISSING_IN_PG,
|
||||
legacyValue = legacyRaw,
|
||||
detail = "Flight present in Redis but missing in PG",
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
if (legacyRaw == null) {
|
||||
if (flid in crossDayMigratedFlids) {
|
||||
known += Deviation(
|
||||
flid = flid,
|
||||
kind = DeviationKind.CROSS_DAY_PROTECTED,
|
||||
pgValue = pgRaw,
|
||||
detail = "Known Deviation 1: Cross-day migrated flight protected by FDAY domain-scoped delete in nextgen PG, erroneously deleted in legacy Redis",
|
||||
)
|
||||
} else {
|
||||
unexpected += Deviation(
|
||||
flid = flid,
|
||||
kind = DeviationKind.UNEXPECTED_EXTRA_IN_PG,
|
||||
pgValue = pgRaw,
|
||||
detail = "Flight present in PG but missing in Redis without cross-day migration justification",
|
||||
)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// 两侧均存在:进行 AST 递归规范化比较
|
||||
val mismatches = compareJsonTree(flid, pgRaw, legacyRaw)
|
||||
if (mismatches.isEmpty()) {
|
||||
matched++
|
||||
} else {
|
||||
unexpected.addAll(mismatches)
|
||||
}
|
||||
}
|
||||
|
||||
return DiffReport(
|
||||
totalPg = pgFlights.size,
|
||||
totalLegacy = legacyFlights.size,
|
||||
matchedCount = matched,
|
||||
knownDeviations = known,
|
||||
unexpectedDeviations = unexpected,
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* 递归 AST 比较两段 JSON。
|
||||
*/
|
||||
internal fun compareJsonTree(flid: String, pgJson: String, legacyJson: String): List<Deviation> {
|
||||
val pgNode = try {
|
||||
mapper.readTree(pgJson)
|
||||
} catch (e: Exception) {
|
||||
return listOf(Deviation(flid, DeviationKind.FIELD_MISMATCH, detail = "PG JSON parse failed: ${e.message}"))
|
||||
}
|
||||
|
||||
val legacyNode = try {
|
||||
mapper.readTree(legacyJson)
|
||||
} catch (e: Exception) {
|
||||
return listOf(Deviation(flid, DeviationKind.FIELD_MISMATCH, detail = "Legacy JSON parse failed: ${e.message}"))
|
||||
}
|
||||
|
||||
val mismatches = mutableListOf<Deviation>()
|
||||
compareNodes(flid, "", pgNode, legacyNode, mismatches)
|
||||
return mismatches
|
||||
}
|
||||
|
||||
private fun compareNodes(flid: String, path: String, n1: JsonNode?, n2: JsonNode?, acc: MutableList<Deviation>) {
|
||||
if (n1 == null && n2 == null) return
|
||||
|
||||
// 规范化:null 节点与缺失节点视为等价
|
||||
val isN1Empty = n1 == null || n1.isNull
|
||||
val isN2Empty = n2 == null || n2.isNull
|
||||
if (isN1Empty && isN2Empty) return
|
||||
|
||||
if (isN1Empty != isN2Empty) {
|
||||
acc += Deviation(
|
||||
flid = flid,
|
||||
kind = DeviationKind.FIELD_MISMATCH,
|
||||
path = path,
|
||||
pgValue = n1?.asText(),
|
||||
legacyValue = n2?.asText(),
|
||||
detail = "One side is null/absent while other is present",
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
val nonNull1 = n1!!
|
||||
val nonNull2 = n2!!
|
||||
|
||||
// 数值归一化比较
|
||||
if (nonNull1.isNumber && nonNull2.isNumber) {
|
||||
val d1 = BigDecimal(nonNull1.asText()).stripTrailingZeros()
|
||||
val d2 = BigDecimal(nonNull2.asText()).stripTrailingZeros()
|
||||
if (d1.compareTo(d2) != 0) {
|
||||
acc += Deviation(
|
||||
flid = flid,
|
||||
kind = DeviationKind.FIELD_MISMATCH,
|
||||
path = path,
|
||||
pgValue = nonNull1.numberValue(),
|
||||
legacyValue = nonNull2.numberValue(),
|
||||
detail = "Numeric value mismatch: $d1 != $d2",
|
||||
)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// 对象类型:递归属性比较(忽略 key 顺序)
|
||||
if (nonNull1.isObject && nonNull2.isObject) {
|
||||
val obj1 = nonNull1 as ObjectNode
|
||||
val obj2 = nonNull2 as ObjectNode
|
||||
val allKeys = (obj1.fieldNames().asSequence().toSet() + obj2.fieldNames().asSequence().toSet()).toSortedSet()
|
||||
|
||||
for (key in allKeys) {
|
||||
val nextPath = if (path.isEmpty()) key else "$path.$key"
|
||||
compareNodes(flid, nextPath, obj1.get(key), obj2.get(key), acc)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// 数组类型:逐项比较
|
||||
if (nonNull1.isArray && nonNull2.isArray) {
|
||||
val arr1 = nonNull1 as ArrayNode
|
||||
val arr2 = nonNull2 as ArrayNode
|
||||
if (arr1.size() != arr2.size()) {
|
||||
acc += Deviation(
|
||||
flid = flid,
|
||||
kind = DeviationKind.FIELD_MISMATCH,
|
||||
path = path,
|
||||
pgValue = "size=${arr1.size()}",
|
||||
legacyValue = "size=${arr2.size()}",
|
||||
detail = "Array size mismatch",
|
||||
)
|
||||
return
|
||||
}
|
||||
for (i in 0 until arr1.size()) {
|
||||
compareNodes(flid, "$path[$i]", arr1.get(i), arr2.get(i), acc)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// 基本类型:文本值比较
|
||||
if (nonNull1.asText() != nonNull2.asText()) {
|
||||
acc += Deviation(
|
||||
flid = flid,
|
||||
kind = DeviationKind.FIELD_MISMATCH,
|
||||
path = path,
|
||||
pgValue = nonNull1.asText(),
|
||||
legacyValue = nonNull2.asText(),
|
||||
detail = "Value mismatch: ${nonNull1.asText()} != ${nonNull2.asText()}",
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user