✨ Add AFTN protocol validation and serial reader health monitoring to enhance aviation telegram processing reliability and observability
Implement comprehensive AFTN/ICAO protocol compliance validation with configurable enforcement, enabling early detection of malformed telegrams and reducing downstream processing errors. Add real-time serial reader health monitoring to automatically detect message flow interruptions and sequence gaps, ensuring operational visibility into the telegram ingestion pipeline. Key enhancements: - AFTN validator validates priority indicators (FF/GG/QU/DD/SS/KK), ICAO addresses (4-char alphanumeric), and datetime formats (DDHHMM) with detailed error categorization - Invalid telegrams automatically routed to DLQ with full context for offline review and correction - Serial reader health monitoring tracks message gaps and sequence numbers to detect stalled readers or missing messages within configurable threshold (default: 2 minutes) - Four new Prometheus metrics expose validation errors by type, message gaps, sequence gaps, and health status for operational alerting - Pre-configured Prometheus alert rules for critical conditions (stalled reader, high error rates, consumer lag) - Grafana dashboard provides real-time visibility into AFTN compliance and serial reader health - Validation disabled by default for safe rollout with zero breaking changes to existing functionality Implementation maintains clean architecture with validator in adapter layer, extends processor and consumer with health tracking, and ensures thread-safe concurrent access to tracking state. All changes fully tested with 48 validator tests, 10 processor tests, and 21 consumer tests passing. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
windyboy
co-authored by
Claude Sonnet 4.5
Claude Sonnet 4.5
parent
6eff35b56f
commit
7b6f6383ad
@@ -118,3 +118,12 @@ health_timeout = "2s"
|
||||
enabled = true # Set to true when switching to JetStream mode
|
||||
# subject: NATS subject where failed messages will be published for manual inspection
|
||||
subject = "caatsm.dlq"
|
||||
|
||||
[aftn]
|
||||
# AFTN Protocol Validation and Monitoring
|
||||
# validation_enabled: Enable AFTN protocol validation for telegrams (disabled by default for safe rollout)
|
||||
validation_enabled = false
|
||||
# message_gap_threshold: Duration after which serial reader is considered stalled (no messages received)
|
||||
message_gap_threshold = "2m"
|
||||
# enable_sequence_gap_detection: Monitor for missing sequence numbers in telegram stream
|
||||
enable_sequence_gap_detection = true
|
||||
|
||||
@@ -0,0 +1,863 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"description": "Health status of the serial reader. 1 = healthy (messages flowing), 0 = stalled (no messages received for > 2 minutes)",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [
|
||||
{
|
||||
"options": {
|
||||
"0": {
|
||||
"color": "red",
|
||||
"index": 1,
|
||||
"text": "STALLED"
|
||||
},
|
||||
"1": {
|
||||
"color": "green",
|
||||
"index": 0,
|
||||
"text": "HEALTHY"
|
||||
}
|
||||
},
|
||||
"type": "value"
|
||||
}
|
||||
],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"text": {},
|
||||
"textMode": "value_and_name",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "10.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "caatsm_serial_reader_healthy{stream=\"$stream\", consumer=\"$consumer\"}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Serial Reader Health",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"description": "Time in seconds since the last message was received from the serial reader",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 60
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 120
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 0
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "center",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"text": {},
|
||||
"textMode": "value_and_name",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "10.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "caatsm_message_gap_seconds{stream=\"$stream\", consumer=\"$consumer\"}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Message Gap (seconds)",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"description": "Total number of AFTN validation errors in the last 5 minutes",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"tooltip": false,
|
||||
"viz": false,
|
||||
"legend": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 2,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"last"
|
||||
],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "sum(rate(caatsm_aftn_validation_errors_total[5m])) by (error_type)",
|
||||
"legendFormat": "{{error_type}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "AFTN Validation Errors by Type (5m rate)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"description": "Percentage of messages failing AFTN validation",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"max": 100,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 1
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 5
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 0
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"displayMode": "gradient",
|
||||
"maxVizHeight": 300,
|
||||
"minVizHeight": 10,
|
||||
"minVizWidth": 0,
|
||||
"namePlacement": "auto",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showUnfilled": true,
|
||||
"sizing": "auto",
|
||||
"text": {},
|
||||
"valueMode": "color"
|
||||
},
|
||||
"pluginVersion": "10.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "100 * (sum(rate(caatsm_aftn_validation_errors_total[5m])) / sum(rate(caatsm_processed_total[5m])))",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "AFTN Error Rate %",
|
||||
"type": "bargauge"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"description": "Time series showing message gap evolution over time",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "Seconds",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"tooltip": false,
|
||||
"viz": false,
|
||||
"legend": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "line"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 120
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 6
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "caatsm_message_gap_seconds{stream=\"$stream\", consumer=\"$consumer\"}",
|
||||
"legendFormat": "{{stream}}/{{consumer}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Message Gap Over Time",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"description": "Rate of sequence gaps detected (missing message sequence numbers)",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "Gaps/sec",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"tooltip": false,
|
||||
"viz": false,
|
||||
"legend": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "stepAfter",
|
||||
"lineWidth": 2,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 6
|
||||
},
|
||||
"id": 6,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"sum"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "rate(caatsm_message_sequence_gap_total{stream=\"$stream\", consumer=\"$consumer\"}[5m])",
|
||||
"legendFormat": "{{stream}}/{{consumer}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Sequence Gap Rate (5m)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"description": "Number of pending messages in the JetStream consumer queue",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "Messages",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"tooltip": false,
|
||||
"viz": false,
|
||||
"legend": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "line"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 1000
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 5000
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 14
|
||||
},
|
||||
"id": 7,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "caatsm_nats_consumer_pending_messages{stream=\"$stream\", consumer=\"$consumer\"}",
|
||||
"legendFormat": "{{stream}}/{{consumer}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Consumer Pending Messages",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"description": "Message processing throughput by status",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "Messages/sec",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"tooltip": false,
|
||||
"viz": false,
|
||||
"legend": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 2,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "normal"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*error.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "red",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*parsed.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "green",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 14
|
||||
},
|
||||
"id": 8,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"mean"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "sum(rate(caatsm_processed_total[5m])) by (status)",
|
||||
"legendFormat": "{{status}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Message Processing Rate by Status",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"schemaVersion": 39,
|
||||
"tags": [
|
||||
"caatsm",
|
||||
"aftn",
|
||||
"aviation",
|
||||
"telegram"
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": "Prometheus",
|
||||
"value": "Prometheus"
|
||||
},
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Datasource",
|
||||
"multi": false,
|
||||
"name": "DS_PROMETHEUS",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": "TELEGRAM",
|
||||
"value": "TELEGRAM"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"definition": "label_values(caatsm_nats_consumer_pending_messages, stream)",
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Stream",
|
||||
"multi": false,
|
||||
"name": "stream",
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values(caatsm_nats_consumer_pending_messages, stream)",
|
||||
"refId": "PrometheusVariableQueryEditor-VariableQuery"
|
||||
},
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 0,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": "telegram-consumer",
|
||||
"value": "telegram-consumer"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"definition": "label_values(caatsm_nats_consumer_pending_messages{stream=\"$stream\"}, consumer)",
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Consumer",
|
||||
"multi": false,
|
||||
"name": "consumer",
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values(caatsm_nats_consumer_pending_messages{stream=\"$stream\"}, consumer)",
|
||||
"refId": "PrometheusVariableQueryEditor-VariableQuery"
|
||||
},
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 0,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "CAATSM AFTN Health & Validation",
|
||||
"uid": "caatsm-aftn-health",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
# Prometheus Alert Rules for CAATSM AFTN Telegram Processor
|
||||
#
|
||||
# Installation:
|
||||
# 1. Copy this file to your Prometheus server's rules directory
|
||||
# 2. Add to prometheus.yml:
|
||||
# rule_files:
|
||||
# - "prometheus-alerts.yml"
|
||||
# 3. Reload Prometheus configuration
|
||||
#
|
||||
# Alert Severity Levels:
|
||||
# - critical: Immediate action required (pages on-call)
|
||||
# - warning: Investigation needed (notify team channel)
|
||||
|
||||
groups:
|
||||
- name: caatsm_aftn_health
|
||||
interval: 30s
|
||||
rules:
|
||||
# Critical: Serial reader has stopped publishing messages
|
||||
- alert: SerialReaderStalled
|
||||
expr: caatsm_serial_reader_healthy == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
component: serial_reader
|
||||
annotations:
|
||||
summary: "Serial reader stalled for {{ $labels.stream }}/{{ $labels.consumer }}"
|
||||
description: |
|
||||
No messages have been received from the serial reader for more than 2 minutes.
|
||||
This indicates the serial port reader may have crashed or the hardware connection is broken.
|
||||
|
||||
Current gap: {{ with query "caatsm_message_gap_seconds{stream=\"" }}{{ . | first | value | humanizeDuration }}{{ end }}
|
||||
Stream: {{ $labels.stream }}
|
||||
Consumer: {{ $labels.consumer }}
|
||||
|
||||
ACTION REQUIRED:
|
||||
1. Check serial reader process status
|
||||
2. Verify serial port connection
|
||||
3. Check hardware status
|
||||
4. Review serial reader logs
|
||||
|
||||
# Warning: Message gap is growing but not yet critical
|
||||
- alert: HighMessageGap
|
||||
expr: caatsm_message_gap_seconds > 60 and caatsm_serial_reader_healthy == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
component: serial_reader
|
||||
annotations:
|
||||
summary: "High message gap detected: {{ $labels.stream }}/{{ $labels.consumer }}"
|
||||
description: |
|
||||
Message gap is {{ $value }}s but still below critical threshold.
|
||||
This may indicate slow message processing or reduced incoming message rate.
|
||||
|
||||
Stream: {{ $labels.stream }}
|
||||
Consumer: {{ $labels.consumer }}
|
||||
|
||||
# Warning: Sequence gaps detected (missing messages)
|
||||
- alert: MessageSequenceGaps
|
||||
expr: rate(caatsm_message_sequence_gap_total[5m]) > 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
component: serial_reader
|
||||
annotations:
|
||||
summary: "Message sequence gaps detected: {{ $labels.stream }}/{{ $labels.consumer }}"
|
||||
description: |
|
||||
Missing message sequence numbers detected at {{ $value | humanize }} gaps/sec.
|
||||
This indicates messages are being lost or skipped in the stream.
|
||||
|
||||
Stream: {{ $labels.stream }}
|
||||
Consumer: {{ $labels.consumer }}
|
||||
Rate: {{ $value | humanize }} gaps/sec
|
||||
|
||||
Possible causes:
|
||||
- Serial reader buffer overflow
|
||||
- Network packet loss (if messages forwarded over network)
|
||||
- Stream retention limits exceeded
|
||||
- Consumer processing too slow
|
||||
|
||||
# Warning: High AFTN validation error rate
|
||||
- alert: HighAFTNValidationErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(caatsm_aftn_validation_errors_total[5m])) by (stream, consumer)
|
||||
/
|
||||
sum(rate(caatsm_processed_total[5m])) by (stream, consumer)
|
||||
) > 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
component: aftn_validator
|
||||
annotations:
|
||||
summary: "High AFTN validation error rate: {{ $value | humanizePercentage }}"
|
||||
description: |
|
||||
More than 5% of incoming telegrams are failing AFTN protocol validation.
|
||||
Current error rate: {{ $value | humanizePercentage }}
|
||||
|
||||
This may indicate:
|
||||
- Upstream system sending malformed telegrams
|
||||
- Serial port data corruption
|
||||
- Configuration mismatch
|
||||
|
||||
Check DLQ for error details and patterns.
|
||||
|
||||
# Warning: Consumer lag is growing
|
||||
- alert: ConsumerLagGrowing
|
||||
expr: |
|
||||
deriv(caatsm_nats_consumer_pending_messages[5m]) > 10
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
component: consumer
|
||||
annotations:
|
||||
summary: "Consumer lag growing: {{ $labels.stream }}/{{ $labels.consumer }}"
|
||||
description: |
|
||||
Consumer pending messages is growing at {{ $value | humanize }} msgs/sec.
|
||||
Current pending: {{ with query "caatsm_nats_consumer_pending_messages" }}{{ . | first | value }}{{ end }}
|
||||
|
||||
This indicates the consumer cannot keep up with incoming message rate.
|
||||
|
||||
Stream: {{ $labels.stream }}
|
||||
Consumer: {{ $labels.consumer }}
|
||||
|
||||
# Critical: Consumer critically behind
|
||||
- alert: ConsumerCriticallyBehind
|
||||
expr: caatsm_nats_consumer_pending_messages > 5000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
component: consumer
|
||||
annotations:
|
||||
summary: "Consumer critically behind: {{ $value }} pending messages"
|
||||
description: |
|
||||
Consumer has {{ $value }} pending messages - critically behind.
|
||||
This will cause message processing delays and may trigger stream retention limits.
|
||||
|
||||
Stream: {{ $labels.stream }}
|
||||
Consumer: {{ $labels.consumer }}
|
||||
Pending: {{ $value }}
|
||||
|
||||
ACTION REQUIRED:
|
||||
1. Check processor performance and errors
|
||||
2. Check database connection and performance
|
||||
3. Consider scaling consumers horizontally
|
||||
4. Review stream retention settings
|
||||
|
||||
# Warning: High processing failure rate
|
||||
- alert: HighProcessingFailureRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(caatsm_messages_total{result="fail"}[5m])) by (stream, consumer)
|
||||
/
|
||||
sum(rate(caatsm_messages_total[5m])) by (stream, consumer)
|
||||
) > 0.10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
component: processor
|
||||
annotations:
|
||||
summary: "High processing failure rate: {{ $value | humanizePercentage }}"
|
||||
description: |
|
||||
More than 10% of messages are failing to process.
|
||||
Current failure rate: {{ $value | humanizePercentage }}
|
||||
|
||||
Stream: {{ $labels.stream }}
|
||||
Consumer: {{ $labels.consumer }}
|
||||
|
||||
Check application logs for error details.
|
||||
|
||||
# Warning: High publish failure rate
|
||||
- alert: HighPublishFailureRate
|
||||
expr: |
|
||||
sum(rate(caatsm_publish_failures_total[5m])) by (category)
|
||||
/
|
||||
sum(rate(caatsm_processed_total[5m])) by (category) > 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
component: publisher
|
||||
annotations:
|
||||
summary: "High publish failure rate for {{ $labels.category }}: {{ $value | humanizePercentage }}"
|
||||
description: |
|
||||
More than 5% of {{ $labels.category }} messages failing to publish.
|
||||
Current failure rate: {{ $value | humanizePercentage }}
|
||||
|
||||
Category: {{ $labels.category }}
|
||||
|
||||
Check NATS JetStream connectivity and publisher logs.
|
||||
|
||||
# Warning: DLQ publish failures (messages lost)
|
||||
- alert: DLQPublishFailures
|
||||
expr: rate(caatsm_dlq_publish_failures_total[5m]) > 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
component: dlq
|
||||
annotations:
|
||||
summary: "DLQ publish failures detected"
|
||||
description: |
|
||||
Failed messages cannot be published to DLQ - messages may be lost!
|
||||
Failure rate: {{ $value | humanize }} msgs/sec
|
||||
|
||||
Stream: {{ $labels.stream }}
|
||||
Consumer: {{ $labels.consumer }}
|
||||
|
||||
Check DLQ subject configuration and NATS JetStream health.
|
||||
|
||||
- name: caatsm_aftn_validation_details
|
||||
interval: 1m
|
||||
rules:
|
||||
# Recording rule: AFTN error rate by type
|
||||
- record: caatsm:aftn_validation_error_rate:5m
|
||||
expr: |
|
||||
rate(caatsm_aftn_validation_errors_total[5m])
|
||||
|
||||
# Recording rule: Total processing rate
|
||||
- record: caatsm:processing_rate:5m
|
||||
expr: |
|
||||
sum(rate(caatsm_processed_total[5m])) by (stream, consumer, status)
|
||||
|
||||
# Recording rule: Average message gap
|
||||
- record: caatsm:message_gap_seconds:avg
|
||||
expr: |
|
||||
avg(caatsm_message_gap_seconds) by (stream, consumer)
|
||||
Reference in New Issue
Block a user