cdia/msgexchange-v2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(acm2): 闭合 G-KAFKA-D3 / G-IGNORE / G-EVENT-RETENTION 三个缺口

Browse Source 
- G-KAFKA-D3：max-in-flight 默认收敛到 1，KafkaD3Check 启动自检钉住三项联合满足 D3
- G-IGNORE：IgnoreRules 在身份绑定后精确匹配 TYPE 字段，命中写 SKIPPED + 回填意图
- G-EVENT-RETENTION：V10 迁移加 SENT_AT 列，投递原子写，EventCleanupJob 有界清理过期 SENT 行
This commit is contained in:
windyboy
2026-09-13 15:48:16 +08:00
parent d37a9eb38e
commit fc744d0cfd
23 changed files with 540 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/kotlin/com/gzzn/omms/msgexchange/infra/metrics/PipelineCounters.kt
+6
View File
@@ -15,6 +15,7 @@ import java.util.concurrent.atomic.AtomicLong
class PipelineCounters {
private val srvtSeen = AtomicLong(0)
private val vipfSeen = AtomicLong(0)
private val ignored = AtomicLong(0)
/**
* 入站记录里出现 `SRVT`/`VIPF` 段的条数（`[G-SRVT-VIPF]`）。
@@ -30,4 +31,9 @@ class PipelineCounters {
fun srvtSeenCount(): Long = srvtSeen.get()
fun vipfSeenCount(): Long = vipfSeen.get()
/** US-04 忽略清单命中计数。 */
fun ignoredAdd(count: Int = 1) { ignored.addAndGet(count.toLong()) }
fun ignoredCount(): Long = ignored.get()
}
src/main/kotlin/com/gzzn/omms/msgexchange/infra/metrics/PipelineMetrics.kt
+5
View File
@@ -28,6 +28,7 @@ import java.time.Duration
* - `msgx.pipeline.job.last_sweep_selected`：上一轮回填扫描选中的待办条数（扫描积压）
* - `msgx.pipeline.codec.srvt_seen.total` / `msgx.pipeline.codec.vipf_seen.total`：入站记录里出现
* `SRVT`/`VIPF` 段的条数（尚未落明细表，`[G-SRVT-VIPF]`；> 0 表示真实流量确有该段）
* - `msgx.pipeline.processing.ignored.total`：命中 US-04 忽略清单的报文条数
*
* 取数统一走 [BacklogSnapshotProvider]（30 秒 TTL），因此指标抓取不会打穿数据库。
* 无法取数时以 `NaN` 上报（Micrometer 的惯例表示"本次无值"），而不是伪造 0。
@@ -101,6 +102,10 @@ class PipelineMetrics(
Gauge.builder("msgx.pipeline.codec.vipf_seen.total", counters) { it.vipfSeenCount().toDouble() }
.strongReference(true)
.register(registry)
Gauge.builder("msgx.pipeline.processing.ignored.total", counters) { it.ignoredCount().toDouble() }
.strongReference(true)
.register(registry)
}
private fun backlogGauge(name: String, value: (com.gzzn.omms.msgexchange.infra.persistence.Backlog) -> Double) {
src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/Repositories.kt
+10 -3
View File
@@ -170,7 +170,7 @@ interface MsgEventRepository {
/** 领取待发的整态事件：写入侧已保证每个 `FLID` 只有一行，读端不再去重合并。 */
fun mergePendingSchd(now: Instant, limit: Int): List<MsgEvent>
fun markSent(eventId: Long)
fun markSent(eventId: Long, now: Instant)
/**
* 条件确认：只在该行仍是本批读取到的代次（`EVENT_ID` + `STATE_VERSION`）且尚未发出时标记 `SENT`。
@@ -178,13 +178,20 @@ interface MsgEventRepository {
*
* @return 受影响行数（0 或 1）
*/
fun markSentIfVersion(eventId: Long, stateVersion: Long): Int
fun markSentIfVersion(eventId: Long, stateVersion: Long, now: Instant): Int
fun markAllSent(eventIds: List<Long>)
fun markAllSent(eventIds: List<Long>, now: Instant)
fun scheduleRetry(eventId: Long, nextAttemptAt: Instant, attempts: Int)
fun markDead(eventId: Long, errorClass: ErrorClass, lastError: String, attempts: Int? = null)
/**
* 删除已确认投递且超过保留期的事件行：`STATE='SENT' AND SENT_AT < cutoff`。
* 带 `STATE='SENT'` 条件，避免与 upsertSchd 的并发写入冲突。
* @return 实际删除的行数
*/
fun deleteExpiredSent(cutoff: Instant, limit: Int): Int
}
/**
src/main/kotlin/com/gzzn/omms/msgexchange/infra/persistence/jdbc/JdbcPgRepositories.kt
+29 -8
View File
@@ -473,6 +473,7 @@ class JdbcMsgEventRepository(
next_attempt_at = NULL,
error_class = NULL,
last_error = NULL,
sent_at = NULL,
created_at = EXCLUDED.created_at
WHERE EXCLUDED.state_version > msg_event.state_version
OR (EXCLUDED.state_version = msg_event.state_version AND EXCLUDED.event_type = 'TOMBSTONE')
@@ -519,22 +520,26 @@ class JdbcMsgEventRepository(
::mapEvent,
)
override fun markSent(eventId: Long) {
ds.update("UPDATE msg_event SET state = 'SENT' WHERE event_id = ?", { ps -> ps.setLong(1, eventId) })
override fun markSent(eventId: Long, now: Instant) {
ds.update(
"UPDATE msg_event SET state = 'SENT', sent_at = ? WHERE event_id = ?",
{ ps -> ps.setTimestamp(1, now.toSqlTimestamp()); ps.setLong(2, eventId) },
)
}
/** 条件确认：仅当行仍是本批读到的代次且尚未发出时才标记，被新代次覆盖则影响 0 行。 */
override fun markSentIfVersion(eventId: Long, stateVersion: Long): Int =
override fun markSentIfVersion(eventId: Long, stateVersion: Long, now: Instant): Int =
ds.update(
"UPDATE msg_event SET state = 'SENT' WHERE event_id = ? AND state_version = ? AND state = 'PENDING'",
{ ps -> ps.setLong(1, eventId); ps.setLong(2, stateVersion) },
"UPDATE msg_event SET state = 'SENT', sent_at = ? WHERE event_id = ? AND state_version = ? AND state = 'PENDING'",
{ ps -> ps.setTimestamp(1, now.toSqlTimestamp()); ps.setLong(2, eventId); ps.setLong(3, stateVersion) },
)
override fun markAllSent(eventIds: List<Long>) {
override fun markAllSent(eventIds: List<Long>, now: Instant) {
if (eventIds.isEmpty()) return
val placeholders = eventIds.joinToString(",") { "?" }
ds.update("UPDATE msg_event SET state = 'SENT' WHERE event_id IN ($placeholders)") { ps ->
eventIds.forEachIndexed { i, id -> ps.setLong(i + 1, id) }
ds.update("UPDATE msg_event SET state = 'SENT', sent_at = ? WHERE event_id IN ($placeholders)") { ps ->
ps.setTimestamp(1, now.toSqlTimestamp())
eventIds.forEachIndexed { i, id -> ps.setLong(i + 2, id) }
}
}
@@ -557,6 +562,21 @@ class JdbcMsgEventRepository(
)
}
override fun deleteExpiredSent(cutoff: Instant, limit: Int): Int =
ds.update(
"""
DELETE FROM msg_event WHERE event_id IN (
SELECT event_id FROM msg_event
WHERE state = 'SENT' AND sent_at IS NOT NULL AND sent_at < ?
ORDER BY event_id LIMIT ?
)
""".trimIndent(),
{ ps ->
ps.setTimestamp(1, cutoff.toSqlTimestamp())
ps.setInt(2, limit)
},
)
private fun mapEvent(rs: ResultSet) = MsgEvent(
eventId = rs.getLong("event_id"),
target = rs.getString("target"),
@@ -570,6 +590,7 @@ class JdbcMsgEventRepository(
errorClass = rs.getString("error_class")?.let(ErrorClass::valueOf),
lastError = rs.getString("last_error"),
createdAt = rs.getInstant("created_at") ?: clock.instant(),
sentAt = rs.getInstant("sent_at"),
)
}
src/main/kotlin/com/gzzn/omms/msgexchange/infra/stub/StubRepositories.kt
+17 -7
View File
@@ -262,7 +262,7 @@ class StubMsgEvents : MsgEventRepository {
val id = ids.incrementAndGet()
rows[id] = e.copy(
eventId = id, state = EventStatus.PENDING, attempts = 0, nextAttemptAt = null,
errorClass = null, lastError = null,
errorClass = null, lastError = null, sentAt = null,
)
return id
}
@@ -278,19 +278,19 @@ class StubMsgEvents : MsgEventRepository {
.sortedBy { it.eventId!! }
.take(limit)
override fun markSent(eventId: Long) {
rows[eventId] = (rows[eventId] ?: return).copy(state = EventStatus.SENT)
override fun markSent(eventId: Long, now: Instant) {
rows[eventId] = (rows[eventId] ?: return).copy(state = EventStatus.SENT, sentAt = now)
}
override fun markSentIfVersion(eventId: Long, stateVersion: Long): Int {
override fun markSentIfVersion(eventId: Long, stateVersion: Long, now: Instant): Int {
val row = rows[eventId] ?: return 0
if (row.state != EventStatus.PENDING || row.stateVersion != stateVersion) return 0
rows[eventId] = row.copy(state = EventStatus.SENT)
rows[eventId] = row.copy(state = EventStatus.SENT, sentAt = now)
return 1
}
override fun markAllSent(eventIds: List<Long>) {
eventIds.forEach { markSent(it) }
override fun markAllSent(eventIds: List<Long>, now: Instant) {
eventIds.forEach { markSent(it, now) }
}
override fun scheduleRetry(eventId: Long, nextAttemptAt: Instant, attempts: Int) {
@@ -303,6 +303,16 @@ class StubMsgEvents : MsgEventRepository {
attempts = attempts ?: rows[eventId]?.attempts ?: 0,
)
}
override fun deleteExpiredSent(cutoff: Instant, limit: Int): Int {
val ids = rows.values
.filter { it.state == EventStatus.SENT && it.sentAt != null && it.sentAt < cutoff }
.sortedBy { it.eventId!! }
.take(limit)
.map { it.eventId!! }
ids.forEach { rows.remove(it) }
return ids.size
}
}
@Singleton