windy/go-caatsm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ Refactor telemetry and metrics handling to enhance observability. Remove TLS configuration from telemetry initialization, replacing it with an insecure option for HTTP. Update OpenTelemetry endpoint handling to normalize URLs. Introduce new metrics for publish failures categorized by message type. Enhance Grafana dashboards with detailed descriptions and additional metrics for better monitoring insights. Update documentation to reflect changes in metrics and observability features.

Browse Source
This commit is contained in:
windyboy
2025-11-19 14:53:58 +08:00
parent c687fdcde8
commit f01dd3bf1f
13 changed files with 224 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files
configs/grafana-dashboards.dev/caatsm-overview.json
+116 -12
View File
@@ -103,6 +103,7 @@
"id": 3,
"type": "timeseries",
"title": "End-to-end handle latency (P50/P95/P99)",
"description": "Latency breakdown by stream and consumer for detailed analysis.",
"datasource": {
"type": "prometheus",
"uid": "prometheus-dev"
@@ -123,18 +124,18 @@
"targets": [
{
"refId": "P50",
"expr": "histogram_quantile(0.50, sum by (le) (rate(caatsm_handle_latency_seconds_bucket[5m])))",
"legendFormat": "P50"
"expr": "histogram_quantile(0.50, sum by (le, stream, consumer) (rate(caatsm_handle_latency_seconds_bucket[5m])))",
"legendFormat": "P50 ({{stream}}/{{consumer}})"
},
{
"refId": "P95",
"expr": "histogram_quantile(0.95, sum by (le) (rate(caatsm_handle_latency_seconds_bucket[5m])))",
"legendFormat": "P95"
"expr": "histogram_quantile(0.95, sum by (le, stream, consumer) (rate(caatsm_handle_latency_seconds_bucket[5m])))",
"legendFormat": "P95 ({{stream}}/{{consumer}})"
},
{
"refId": "P99",
"expr": "histogram_quantile(0.99, sum by (le) (rate(caatsm_handle_latency_seconds_bucket[5m])))",
"legendFormat": "P99"
"expr": "histogram_quantile(0.99, sum by (le, stream, consumer) (rate(caatsm_handle_latency_seconds_bucket[5m])))",
"legendFormat": "P99 ({{stream}}/{{consumer}})"
}
]
},
@@ -200,6 +201,7 @@
"id": 6,
"type": "timeseries",
"title": "Retries and permanent failures",
"description": "Retries broken down by stream, consumer, and reason. Permanent failures by stream/consumer.",
"datasource": {
"type": "prometheus",
"uid": "prometheus-dev"
@@ -220,13 +222,13 @@
"targets": [
{
"refId": "retries",
"expr": "sum(rate(caatsm_retries_total[5m]))",
"legendFormat": "retries"
"expr": "sum by (stream, consumer, reason) (rate(caatsm_retries_total[5m]))",
"legendFormat": "retries ({{stream}}/{{consumer}}/{{reason}})"
},
{
"refId": "permanent_fail",
"expr": "sum(rate(caatsm_messages_total{result=\"permanent_fail\"}[5m]))",
"legendFormat": "permanent_fail"
"expr": "sum by (stream, consumer) (rate(caatsm_messages_total{result=\"permanent_fail\"}[5m]))",
"legendFormat": "permanent_fail ({{stream}}/{{consumer}})"
}
]
},
@@ -234,6 +236,7 @@
"id": 7,
"type": "timeseries",
"title": "Publish failures by category",
"description": "Prometheus-sourced publish failures. Panel 11 shows OTEL-sourced failures.",
"datasource": {
"type": "prometheus",
"uid": "prometheus-dev"
@@ -254,8 +257,8 @@
"targets": [
{
"refId": "A",
"expr": "sum by (message_category) (rate(caatsm_publish_failures_total[5m]))",
"legendFormat": "{{message_category}}"
"expr": "sum by (category) (rate(caatsm_publish_failures_total[5m]))",
"legendFormat": "{{category}}"
}
]
},
@@ -287,6 +290,107 @@
"legendFormat": "{{stream}} / {{consumer}}"
}
]
},
{
"id": 9,
"type": "timeseries",
"title": "Messages Processed (OTEL - by status/category)",
"description": "OTEL metrics exported via collector. Labels: message_status, message_category (OTEL attributes converted from message.status, message.category).",
"datasource": {
"type": "prometheus",
"uid": "prometheus-dev"
},
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 26 },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [] }
},
"overrides": []
},
"options": {
"legend": { "displayMode": "table", "placement": "bottom" },
"tooltip": { "mode": "single" }
},
"targets": [
{
"refId": "A",
"expr": "sum by (message_status, message_category) (rate(caatsm_messages_processed_total[5m]))",
"legendFormat": "{{message_status}} / {{message_category}}"
}
]
},
{
"id": 10,
"type": "timeseries",
"title": "Parse Duration (OTEL - P50/P95/P99)",
"description": "OTEL histogram metrics for parse latency, broken down by message status and category.",
"datasource": {
"type": "prometheus",
"uid": "prometheus-dev"
},
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 33 },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [] },
"unit": "s"
},
"overrides": []
},
"options": {
"legend": { "displayMode": "table", "placement": "bottom" },
"tooltip": { "mode": "single" }
},
"targets": [
{
"refId": "P50",
"expr": "histogram_quantile(0.50, sum by (le, message_status, message_category) (rate(caatsm_parse_duration_seconds_bucket[5m])))",
"legendFormat": "P50 ({{message_status}}/{{message_category}})"
},
{
"refId": "P95",
"expr": "histogram_quantile(0.95, sum by (le, message_status, message_category) (rate(caatsm_parse_duration_seconds_bucket[5m])))",
"legendFormat": "P95 ({{message_status}}/{{message_category}})"
},
{
"refId": "P99",
"expr": "histogram_quantile(0.99, sum by (le, message_status, message_category) (rate(caatsm_parse_duration_seconds_bucket[5m])))",
"legendFormat": "P99 ({{message_status}}/{{message_category}})"
}
]
},
{
"id": 11,
"type": "timeseries",
"title": "Publish Failures (OTEL - by category)",
"description": "OTEL-sourced publish failures. Panel 7 shows Prometheus-sourced failures.",
"datasource": {
"type": "prometheus",
"uid": "prometheus-dev"
},
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 33 },
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"mappings": [],
"thresholds": { "mode": "absolute", "steps": [] }
},
"overrides": []
},
"options": {
"legend": { "displayMode": "table", "placement": "bottom" },
"tooltip": { "mode": "single" }
},
"targets": [
{
"refId": "A",
"expr": "sum by (message_category) (rate(caatsm_publish_failures_total[5m]))",
"legendFormat": "{{message_category}}"
}
]
}
]
}
configs/grafana-dashboards.dev/otel-collector.json
+5 -175
View File
@@ -140,12 +140,12 @@
"targets": [
{
"refId": "A",
"expr": "sum(rate(otelcol_exporter_sent_spans{job=\"otel-collector\",exporter=\"otlp/jaeger\"}[1m]))",
"expr": "sum(rate(otelcol_exporter_sent_spans{job=\"otel-collector\",exporter=\"otlphttp/jaeger\"}[1m]))",
"legendFormat": "sent"
},
{
"refId": "B",
"expr": "sum(rate(otelcol_exporter_failed_spans{job=\"otel-collector\",exporter=\"otlp/jaeger\"}[1m]))",
"expr": "sum(rate(otelcol_exporter_failed_spans{job=\"otel-collector\",exporter=\"otlphttp/jaeger\"}[1m]))",
"legendFormat": "failed"
}
]
@@ -199,7 +199,7 @@
"targets": [
{
"refId": "A",
"expr": "avg(otelcol_exporter_queue_size{job=\"otel-collector\",exporter=\"otlp/jaeger\"})",
"expr": "avg(otelcol_exporter_queue_size{job=\"otel-collector\",exporter=\"otlphttp/jaeger\"})",
"legendFormat": "queue size"
}
]
@@ -318,116 +318,6 @@
}
]
},
{
"id": 6,
"type": "timeseries",
"title": "Messages Processed Rate",
"datasource": {
"type": "prometheus",
"uid": "prometheus-dev"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 0
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 12
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single"
}
},
"targets": [
{
"refId": "A",
"expr": "sum by (message_status) (rate(caatsm_messages_processed_total[1m]))",
"legendFormat": "{{message_status}}"
}
]
},
{
"id": 7,
"type": "timeseries",
"title": "Publish Failures Rate",
"datasource": {
"type": "prometheus",
"uid": "prometheus-dev"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "orange",
"value": 100
},
{
"color": "red",
"value": 500
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 12
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single"
}
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(caatsm_publish_failures_total[1m]))",
"legendFormat": "failures/s"
}
]
},
{
"id": 8,
"type": "timeseries",
@@ -476,75 +366,16 @@
"targets": [
{
"refId": "A",
"expr": "avg(otelcol_exporter_queue_size{job=\"otel-collector\",exporter=\"otlp/jaeger\"})",
"expr": "avg(otelcol_exporter_queue_size{job=\"otel-collector\",exporter=\"otlphttp/jaeger\"})",
"legendFormat": "queue size"
},
{
"refId": "B",
"expr": "avg(otelcol_exporter_queue_capacity{job=\"otel-collector\",exporter=\"otlp/jaeger\"})",
"expr": "avg(otelcol_exporter_queue_capacity{job=\"otel-collector\",exporter=\"otlphttp/jaeger\"})",
"legendFormat": "capacity"
}
]
},
{
"id": 9,
"type": "timeseries",
"title": "Parse Duration (P95 / P50)",
"datasource": {
"type": "prometheus",
"uid": "prometheus-dev"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 0
}
]
},
"unit": "ms"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 20
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single"
}
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95, sum(rate(caatsm_parse_duration_seconds_bucket[5m])) by (le))",
"legendFormat": "p95"
},
{
"refId": "B",
"expr": "histogram_quantile(0.50, sum(rate(caatsm_parse_duration_seconds_bucket[5m])) by (le))",
"legendFormat": "p50"
}
]
},
{
"id": 10,
"type": "timeseries",
@@ -711,4 +542,3 @@
}
]
}
configs/otel-collector.dev.yaml
-6
View File
@@ -4,10 +4,8 @@ receivers:
http:
endpoint: 0.0.0.0:4318
max_request_body_size: 20971520 # 20MB
max_concurrent_streams: 16
grpc:
endpoint: 0.0.0.0:4317
max_recv_msg_size: 4194304 # 4MB
max_concurrent_streams: 16
processors:
@@ -50,10 +48,6 @@ exporters:
endpoint: "0.0.0.0:8889"
const_labels:
source: "otel-collector"
sending_queue:
queue_size: 10000
retry_on_failure:
enabled: true
service:
pipelines:
configs/prometheus.dev.yml
+1
View File
@@ -6,6 +6,7 @@ scrape_configs:
- job_name: "otel-collector"
static_configs:
- targets:
- "otel-collector:8888"
- "otel-collector:8889"
- job_name: "nats-exporter"
static_configs:
configs/prometheus/targets/caatsm-receiver.json
+1 -1
View File
@@ -1 +1 @@
[{"labels":{"job":"caatsm-receiver","instance":"10.16.66.236:2112"},"targets":["10.16.66.236:2112"]}]
[{"labels":{"job":"caatsm-receiver","instance":"host.docker.internal:2112"},"targets":["host.docker.internal:2112"]}]