diff --git a/configs/grafana-dashboards.dev.yml b/configs/grafana-dashboards.dev.yml new file mode 100644 index 0000000..85f3319 --- /dev/null +++ b/configs/grafana-dashboards.dev.yml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: Dev Observability + orgId: 1 + folder: Dev Observability + type: file + disableDeletion: false + allowUiUpdates: true + options: + path: /etc/grafana/provisioning/dashboards/dev + recursive: true + diff --git a/configs/grafana-dashboards.dev/nats-overview.json b/configs/grafana-dashboards.dev/nats-overview.json new file mode 100644 index 0000000..9eb03a3 --- /dev/null +++ b/configs/grafana-dashboards.dev/nats-overview.json @@ -0,0 +1,388 @@ +{ + "id": null, + "uid": "nats-dev", + "title": "NATS / JetStream (Dev)", + "tags": [ + "nats", + "jetstream", + "dev" + ], + "schemaVersion": 38, + "version": 1, + "refresh": "10s", + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "templating": { + "list": [] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Active Connections", + "datasource": { + "type": "prometheus", + "uid": "prometheus-dev" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 200 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(nats_varz_total_connections)" + } + ] + }, + { + "id": 2, + "type": "stat", + "title": "Slow Consumers", + "datasource": { + "type": "prometheus", + "uid": "prometheus-dev" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(nats_varz_slow_consumers)" + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "JetStream Enabled", + "datasource": { + "type": "prometheus", + "uid": "prometheus-dev" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": 1, + "options": { + "0": { + "text": "Disabled" + }, + "1": { + "text": "Enabled" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "refId": "A", + "expr": "max(nats_varz_jetstream)" + } + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "Ingress vs Egress Messages", + "datasource": { + "type": "prometheus", + "uid": "prometheus-dev" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(nats_varz_in_msgs[1m]))", + "legendFormat": "in" + }, + { + "refId": "B", + "expr": "sum(rate(nats_varz_out_msgs[1m]))", + "legendFormat": "out" + } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Ingress vs Egress Bytes", + "datasource": { + "type": "prometheus", + "uid": "prometheus-dev" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(nats_varz_in_bytes[1m]))", + "legendFormat": "in" + }, + { + "refId": "B", + "expr": "sum(rate(nats_varz_out_bytes[1m]))", + "legendFormat": "out" + } + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "JetStream Storage Used", + "datasource": { + "type": "prometheus", + "uid": "prometheus-dev" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1e9 + }, + { + "color": "red", + "value": 5e9 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 12 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "refId": "A", + "expr": "sum(nats_varz_jetstream_bytes)" + } + ] + } + ] +} + diff --git a/configs/grafana-dashboards.dev/otel-collector.json b/configs/grafana-dashboards.dev/otel-collector.json new file mode 100644 index 0000000..a7b5c66 --- /dev/null +++ b/configs/grafana-dashboards.dev/otel-collector.json @@ -0,0 +1,323 @@ +{ + "id": null, + "uid": "otel-collector-dev", + "title": "OpenTelemetry Collector (Dev)", + "tags": [ + "otel", + "dev" + ], + "timezone": "", + "schemaVersion": 38, + "version": 1, + "refresh": "10s", + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "templating": { + "list": [] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Accepted vs Refused Spans", + "datasource": { + "type": "prometheus", + "uid": "prometheus-dev" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(otelcol_receiver_accepted_spans{job=\"otel-collector\"}[1m]))", + "legendFormat": "accepted" + }, + { + "refId": "B", + "expr": "sum(rate(otelcol_receiver_refused_spans{job=\"otel-collector\"}[1m]))", + "legendFormat": "refused" + } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "Exporter Success vs Failures", + "datasource": { + "type": "prometheus", + "uid": "prometheus-dev" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(otelcol_exporter_sent_spans{job=\"otel-collector\",exporter=\"otlp/jaeger\"}[1m]))", + "legendFormat": "sent" + }, + { + "refId": "B", + "expr": "sum(rate(otelcol_exporter_failed_spans{job=\"otel-collector\",exporter=\"otlp/jaeger\"}[1m]))", + "legendFormat": "failed" + } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "Exporter Queue Size", + "datasource": { + "type": "prometheus", + "uid": "prometheus-dev" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1000 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "refId": "A", + "expr": "avg(otelcol_exporter_queue_size{job=\"otel-collector\",exporter=\"otlp/jaeger\"})", + "legendFormat": "queue size" + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Process Memory (MB)", + "datasource": { + "type": "prometheus", + "uid": "prometheus-dev" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 500 + }, + { + "color": "red", + "value": 700 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 8 + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "refId": "A", + "expr": "process_resident_memory_bytes{job=\"otel-collector\"} / 1e6" + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Process CPU (s)", + "datasource": { + "type": "prometheus", + "uid": "prometheus-dev" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 8 + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "delta" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "refId": "A", + "expr": "rate(process_cpu_seconds_total{job=\"otel-collector\"}[5m])" + } + ] + } + ] +} + diff --git a/configs/grafana-datasources.dev.yml b/configs/grafana-datasources.dev.yml index 9bdc40d..89eaf18 100644 --- a/configs/grafana-datasources.dev.yml +++ b/configs/grafana-datasources.dev.yml @@ -2,6 +2,7 @@ apiVersion: 1 datasources: - name: Prometheus + uid: prometheus-dev type: prometheus access: proxy url: http://prometheus:9090 @@ -9,6 +10,7 @@ datasources: editable: true - name: Jaeger + uid: jaeger-dev type: jaeger access: proxy url: http://jaeger:16686 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 5b148ad..00bf6d0 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -141,6 +141,8 @@ services: volumes: - grafana-data:/var/lib/grafana - ./configs/grafana-datasources.dev.yml:/etc/grafana/provisioning/datasources/datasources.yml:ro + - ./configs/grafana-dashboards.dev.yml:/etc/grafana/provisioning/dashboards/dashboards.yml:ro + - ./configs/grafana-dashboards.dev:/etc/grafana/provisioning/dashboards/dev:ro depends_on: - prometheus - jaeger diff --git a/docs/dev-guide.md b/docs/dev-guide.md index 57e89e0..e761414 100644 --- a/docs/dev-guide.md +++ b/docs/dev-guide.md @@ -117,6 +117,7 @@ Services: - Uses `configs/prometheus.dev.yml` to scrape the collector and NATS monitoring endpoint; UI available at - `grafana` - Persists data in `grafana-data`, provisions datasources via `configs/grafana-datasources.dev.yml`, and listens on (login `admin` / `admin`) + - Automatically loads dashboards from `configs/grafana-dashboards.dev/`, including OpenTelemetry Collector and NATS/JetStream overviews (find them under the **Dev Observability** folder) ### Customizing Collections & Dashboards