✨ Implement dead-letter queue (DLQ) functionality for handling permanent failures in message processing. Update configuration to enable DLQ and specify the subject for routing failed messages. Enhance observability by adding metrics for message handling, retries, and database operations. Introduce structured logging for better traceability of message processing events.
This commit is contained in:
windyboy
@@ -62,4 +62,8 @@ enable_metrics = true
|
||||
enable_health = true
|
||||
read_timeout = "5s"
|
||||
write_timeout = "5s"
|
||||
health_timeout = "2s"
|
||||
health_timeout = "2s"
|
||||
|
||||
[dlq]
|
||||
enabled = true
|
||||
subject = "caatsm.dlq"
|
||||
@@ -0,0 +1,265 @@
|
||||
{
|
||||
"id": null,
|
||||
"uid": "caatsm-overview",
|
||||
"title": "CAATSM – Receiver Overview",
|
||||
"tags": ["caatsm", "receiver", "dev"],
|
||||
"schemaVersion": 38,
|
||||
"version": 1,
|
||||
"refresh": "10s",
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "Messages by result (5m rate)",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus-dev"
|
||||
},
|
||||
"gridPos": { "h": 4, "w": 8, "x": 0, "y": 0 },
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "red", "value": 0 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (result) (rate(caatsm_messages_total[5m]))",
|
||||
"legendFormat": "{{result}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "Messages per stream/consumer",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus-dev"
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 16, "x": 8, "y": 0 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": [],
|
||||
"thresholds": { "mode": "absolute", "steps": [] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "bottom" },
|
||||
"tooltip": { "mode": "single" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (stream, consumer) (rate(caatsm_messages_total[5m]))",
|
||||
"legendFormat": "{{stream}} / {{consumer}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "End-to-end handle latency (P50/P95/P99)",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus-dev"
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 4 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": [],
|
||||
"thresholds": { "mode": "absolute", "steps": [] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "bottom" },
|
||||
"tooltip": { "mode": "single" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "P50",
|
||||
"expr": "histogram_quantile(0.50, sum by (le) (rate(caatsm_handle_latency_seconds_bucket[5m])))",
|
||||
"legendFormat": "P50"
|
||||
},
|
||||
{
|
||||
"refId": "P95",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(caatsm_handle_latency_seconds_bucket[5m])))",
|
||||
"legendFormat": "P95"
|
||||
},
|
||||
{
|
||||
"refId": "P99",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(caatsm_handle_latency_seconds_bucket[5m])))",
|
||||
"legendFormat": "P99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "DB query rate by operation/result",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus-dev"
|
||||
},
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 12 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": [],
|
||||
"thresholds": { "mode": "absolute", "steps": [] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "bottom" },
|
||||
"tooltip": { "mode": "single" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (operation, result) (rate(caatsm_db_queries_total[5m]))",
|
||||
"legendFormat": "{{operation}} / {{result}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "DB query latency P95 by operation",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus-dev"
|
||||
},
|
||||
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 12 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": [],
|
||||
"thresholds": { "mode": "absolute", "steps": [] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "bottom" },
|
||||
"tooltip": { "mode": "single" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le, operation) (rate(caatsm_db_query_latency_seconds_bucket[5m])))",
|
||||
"legendFormat": "{{operation}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Retries and permanent failures",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus-dev"
|
||||
},
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 19 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": [],
|
||||
"thresholds": { "mode": "absolute", "steps": [] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "bottom" },
|
||||
"tooltip": { "mode": "single" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "retries",
|
||||
"expr": "sum(rate(caatsm_retries_total[5m]))",
|
||||
"legendFormat": "retries"
|
||||
},
|
||||
{
|
||||
"refId": "permanent_fail",
|
||||
"expr": "sum(rate(caatsm_messages_total{result=\"permanent_fail\"}[5m]))",
|
||||
"legendFormat": "permanent_fail"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "Publish failures by category",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus-dev"
|
||||
},
|
||||
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 19 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": [],
|
||||
"thresholds": { "mode": "absolute", "steps": [] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "bottom" },
|
||||
"tooltip": { "mode": "single" }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (message_category) (rate(caatsm_publish_failures_total[5m]))",
|
||||
"legendFormat": "{{message_category}}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user