refactor(ingress): 信箱边界层契约重构——发现与处理标记解耦、回填事实并入 PROC_STATE
收报扫描不再以 DATE_PROCESSED 为谓词:终态而未回填的行(解码失败死信等)会永久占据 有限批次,累积到 claim-batch 后收报整体停摆(US-01 条目 3 / message-lifecycle §5.3)。 ingress/InboxPoller.kt:按 ID 区间升序有界读取(ID > W),水位落 INBOX_CURSOR 并与入队 同一 PG 事务推进(中断后重扫补建);遇空洞即停,空洞超过 pipeline.max-commit-delay 判定 为永久并放行——否则水位永久停摆于一次自增回滚留下的空位。删除 InboxEnqueue(改由 insertIfAbsent 幂等入队)与 poller 内的回填扫描(消除 ingress→jobs 反向依赖)。 infra/persistence:端口按事实重画为 readRange/maxId/markProcessedIfUnmarked,标记 UPDATE 带 DATE_PROCESSED IS NULL 守卫,只把空标写为已处理(§11 单调,重复执行无副作用)。 回填事实并入 PROC_STATE(RECEIVED_AT/BACKFILL_AT/NEXT_AT/ATTEMPTS/ERROR),BACKFILL_TODO 随 V2 迁移下线;终态与回填意图是同一条 UPDATE,由处理器在自己的业务事务内落库, message-lifecycle §4 登记的两个崩溃窗口(提交后回填前崩溃、待办二次落账失败)不再是缺口。 processing/BackfillService.kt(取代 BackfillSweepJob):终态提交后立即尝试一次,失败按 30s→15min 指数退避重试;扫描条件「终态 + 未确认标记 +(已到期 或 接收时间早于 NOW − R)」 使 §5.2 的超期期限 R 覆盖退避,中间态永不补写。死信同样可补写——回填只需消息 ID, 不再依赖 META。Pump 改用可注入 Clock。 infra/health:InboxLifecycleHealthIndicator 输出积压条数、最老未处理信龄、未回填终态数与 水位滞后(OPS-2 / §5.3 验收)。预计消化时长需吞吐采样,留待接入指标注册表时补。 配置:pipeline.max-commit-delay / overdue-backfill / backfill-batch、mailbox.processed-value (Q2/Q6/Q7 未书面确认前取保守初值,不得为提速下调)。 不变量回归测试:死信不阻断后续发现、水位遇空洞即停与老化放行、终态+意图同事务、 超期 R 覆盖退避、中间态不补写、标记单调;InboxLifecycleJdbcSqlTest 以 H2 的 PostgreSQL 兼容模式直连验证上述 SQL 语义(不依赖 docker)。libs.h2 由 testRuntimeOnly 提为 testImplementation 以支持该用例。 验证:gradle clean test --offline → 78 tests / 0 failures / 1 skipped (PG Testcontainers 集成用例在本机无 docker 时按既有约定 assumeTrue 跳过)。
This commit is contained in:
@@ -2,18 +2,34 @@ package com.gzzn.omms.msgexchange.ingress
|
||||
|
||||
import com.gzzn.omms.msgexchange.config.PipelineProps
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.CminmsgInboxRepository
|
||||
import com.gzzn.omms.msgexchange.jobs.BackfillSweepJob
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.InboxCursorRepository
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.MailboxRow
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.PipelineTransactionManager
|
||||
import com.gzzn.omms.msgexchange.infra.persistence.ProcStateRepository
|
||||
import jakarta.inject.Singleton
|
||||
import java.time.Duration
|
||||
import java.time.Instant
|
||||
|
||||
/**
|
||||
* ACMA-8 流程 1 · 主路径:JDBC 轮询共享 MySQL CMINMSGS(`DATE_PROCESSED IS NULL`),
|
||||
* 发现新信后入队自有 PG。与 legacy `MsgExchangeRunner.getNewMsgsAfterId(0L)` 同语义。
|
||||
* 收报(docs/message-lifecycle.md §5.1):共享 MySQL 信箱按 **ID 区间**升序有界读取,
|
||||
* 在自有 PG 建 `PROC_STATE(PENDING)`,并把消费水位推进到连续上界;入队与水位推进在同一
|
||||
* PG 事务内提交,中断即重扫补建(§4 第一行)。收报层不解析业务载荷、不写处理标记。
|
||||
*
|
||||
* 三条红线:
|
||||
* - **扫描谓词不含处理标记**:标记只用于回填与库方清除,不参与消息发现;否则已入队而未
|
||||
* 回填的行会永久占据批次,死信累积到批大小时收报整体停摆;
|
||||
* - **水位遇空洞即停**:不越过空洞入队(越过后较小 ID 迟到即 FIFO 越序,architecture §5);
|
||||
* - **空洞老化**:超过最大提交时延(Q2 承诺)的空洞判定为永久并放行,否则水位会永久停摆于
|
||||
* 一次自增回滚留下的空位。
|
||||
*
|
||||
* 空转代价为每轮一次区间 SELECT。
|
||||
*/
|
||||
@Singleton
|
||||
class InboxPoller(
|
||||
private val inbox: CminmsgInboxRepository,
|
||||
private val enqueue: InboxEnqueue,
|
||||
private val backfillSweep: BackfillSweepJob,
|
||||
private val mailbox: CminmsgInboxRepository,
|
||||
private val procState: ProcStateRepository,
|
||||
private val cursor: InboxCursorRepository,
|
||||
private val txManager: PipelineTransactionManager,
|
||||
private val props: PipelineProps,
|
||||
) {
|
||||
private val log = org.slf4j.LoggerFactory.getLogger(InboxPoller::class.java)
|
||||
@@ -21,31 +37,40 @@ class InboxPoller(
|
||||
@Volatile
|
||||
private var running = false
|
||||
|
||||
/** legacy 现役:afterId=0,每轮扫全部未处理行;PROC_STATE 判重防重复入队。 */
|
||||
fun pollOnce(): Int {
|
||||
/** @return 本轮新建的入队条数 */
|
||||
fun pollOnce(now: Instant = Instant.now()): Int {
|
||||
val batch = props.pipeline.claimBatch.coerceAtLeast(1)
|
||||
val ids = inbox.pollUnprocessed(afterId = 0L, limit = batch)
|
||||
var enqueued = 0
|
||||
for (id in ids) {
|
||||
if (enqueue.enqueue(id)) {
|
||||
enqueued++
|
||||
log.info("polled msgId={}", id)
|
||||
}
|
||||
}
|
||||
sweepBackfillTodos()
|
||||
return enqueued
|
||||
}
|
||||
val watermark = cursor.load()
|
||||
val rows = mailbox.readRange(watermark.committedUpTo, batch)
|
||||
if (rows.isEmpty()) return 0
|
||||
|
||||
/** v2 §5:每轮心跳顺带对账回填补偿待办(重启即恢复;空表只花一次索引 SELECT)。 */
|
||||
private fun sweepBackfillTodos() {
|
||||
try {
|
||||
val outcome = backfillSweep.sweep()
|
||||
if (outcome.inspected > 0) {
|
||||
log.info("backfill sweep inspected={} succeeded={} failed={}", outcome.inspected, outcome.succeeded, outcome.failed)
|
||||
// 连续上界;读取区间内出现空洞时,只推进到连续部分,空洞之后的行暂不入队(防较小 ID 迟到被越过)
|
||||
val contiguous = contiguousUpTo(watermark.committedUpTo, rows) ?: watermark.committedUpTo
|
||||
var committedTo = contiguous
|
||||
var holeSince: Instant? = null
|
||||
if (rows.last().msgId > contiguous) {
|
||||
val since = watermark.holeSince ?: now
|
||||
if (Duration.between(since, now) < props.pipeline.maxCommitDelay) {
|
||||
holeSince = since
|
||||
} else {
|
||||
// 空洞老化:超过最大提交时延仍缺席即判永久(Q2),放行水位,否则永久停摆
|
||||
committedTo = rows.first { it.msgId > contiguous }.msgId - 1
|
||||
log.warn("hole after W={} aged out, watermark advanced to {}", contiguous, committedTo)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
log.error("backfill sweep tick failed", e)
|
||||
}
|
||||
|
||||
val enqueued = txManager.inTransaction {
|
||||
var n = 0
|
||||
rows.takeWhile { it.msgId <= committedTo }.forEach { row ->
|
||||
if (procState.insertIfAbsent(row.msgId, row.receivedAt)) n++
|
||||
}
|
||||
cursor.save(InboxCursorRepository.Cursor(committedTo, holeSince))
|
||||
n
|
||||
}
|
||||
if (enqueued > 0) {
|
||||
log.info("polled {} new messages, W {} -> {}", enqueued, watermark.committedUpTo, committedTo)
|
||||
}
|
||||
return enqueued
|
||||
}
|
||||
|
||||
fun loop() {
|
||||
@@ -70,7 +95,19 @@ class InboxPoller(
|
||||
running = false
|
||||
}
|
||||
|
||||
private fun sleepQuietly(d: java.time.Duration) {
|
||||
/** 连续上界:从 W+1 起 ID 逐 1 相邻的最后一个;首个空位即停(rows 为升序且覆盖该区间)。 */
|
||||
private fun contiguousUpTo(from: Long, rows: List<MailboxRow>): Long? {
|
||||
var expected = from + 1
|
||||
var last: Long? = null
|
||||
for (row in rows) {
|
||||
if (row.msgId != expected) break
|
||||
last = row.msgId
|
||||
expected++
|
||||
}
|
||||
return last
|
||||
}
|
||||
|
||||
private fun sleepQuietly(d: Duration) {
|
||||
try {
|
||||
Thread.sleep(d.toMillis().coerceAtLeast(1))
|
||||
} catch (_: InterruptedException) {
|
||||
|
||||
Reference in New Issue
Block a user