✨ Refactor telemetry and metrics handling to enhance observability. Remove TLS configuration from telemetry initialization, replacing it with an insecure option for HTTP. Update OpenTelemetry endpoint handling to normalize URLs. Introduce new metrics for publish failures categorized by message type. Enhance Grafana dashboards with detailed descriptions and additional metrics for better monitoring insights. Update documentation to reflect changes in metrics and observability features.
This commit is contained in:
windyboy
@@ -140,12 +140,12 @@
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(otelcol_exporter_sent_spans{job=\"otel-collector\",exporter=\"otlp/jaeger\"}[1m]))",
|
||||
"expr": "sum(rate(otelcol_exporter_sent_spans{job=\"otel-collector\",exporter=\"otlphttp/jaeger\"}[1m]))",
|
||||
"legendFormat": "sent"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum(rate(otelcol_exporter_failed_spans{job=\"otel-collector\",exporter=\"otlp/jaeger\"}[1m]))",
|
||||
"expr": "sum(rate(otelcol_exporter_failed_spans{job=\"otel-collector\",exporter=\"otlphttp/jaeger\"}[1m]))",
|
||||
"legendFormat": "failed"
|
||||
}
|
||||
]
|
||||
@@ -199,7 +199,7 @@
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "avg(otelcol_exporter_queue_size{job=\"otel-collector\",exporter=\"otlp/jaeger\"})",
|
||||
"expr": "avg(otelcol_exporter_queue_size{job=\"otel-collector\",exporter=\"otlphttp/jaeger\"})",
|
||||
"legendFormat": "queue size"
|
||||
}
|
||||
]
|
||||
@@ -318,116 +318,6 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Messages Processed Rate",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus-dev"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 12
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (message_status) (rate(caatsm_messages_processed_total[1m]))",
|
||||
"legendFormat": "{{message_status}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "Publish Failures Rate",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus-dev"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "orange",
|
||||
"value": 100
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 500
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 12
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(caatsm_publish_failures_total[1m]))",
|
||||
"legendFormat": "failures/s"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "timeseries",
|
||||
@@ -476,75 +366,16 @@
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "avg(otelcol_exporter_queue_size{job=\"otel-collector\",exporter=\"otlp/jaeger\"})",
|
||||
"expr": "avg(otelcol_exporter_queue_size{job=\"otel-collector\",exporter=\"otlphttp/jaeger\"})",
|
||||
"legendFormat": "queue size"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "avg(otelcol_exporter_queue_capacity{job=\"otel-collector\",exporter=\"otlp/jaeger\"})",
|
||||
"expr": "avg(otelcol_exporter_queue_capacity{job=\"otel-collector\",exporter=\"otlphttp/jaeger\"})",
|
||||
"legendFormat": "capacity"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "timeseries",
|
||||
"title": "Parse Duration (P95 / P50)",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus-dev"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "ms"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 20
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum(rate(caatsm_parse_duration_seconds_bucket[5m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.50, sum(rate(caatsm_parse_duration_seconds_bucket[5m])) by (le))",
|
||||
"legendFormat": "p50"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "timeseries",
|
||||
@@ -711,4 +542,3 @@
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user