✨ Refactor telemetry and metrics handling to enhance observability. Remove TLS configuration from telemetry initialization, replacing it with an insecure option for HTTP. Update OpenTelemetry endpoint handling to normalize URLs. Introduce new metrics for publish failures categorized by message type. Enhance Grafana dashboards with detailed descriptions and additional metrics for better monitoring insights. Update documentation to reflect changes in metrics and observability features.

internal/infra/metrics/metrics.go

@@ -27,6 +27,7 @@ const (
 	MetricDBQueryLatencySeconds = "caatsm_db_query_latency_seconds"
 	MetricDLQMessagesTotal      = "caatsm_dlq_messages_total"
 	MetricDLQPublishFailures    = "caatsm_dlq_publish_failures_total"
+	MetricPublishFailuresTotal  = "caatsm_publish_failures_total"
 	MetricNATSConsumerPending   = "caatsm_nats_consumer_pending_messages"
 
 	// Common label keys.
@@ -69,6 +70,7 @@ var (
 	jsAPICallsTotal    *prometheus.CounterVec
 	dlqMessagesTotal   *prometheus.CounterVec
 	dlqPublishFailures *prometheus.CounterVec
+	publishFailuresTotal *prometheus.CounterVec
 
 	// Database metrics.
 	dbQueriesTotal *prometheus.CounterVec
@@ -125,6 +127,11 @@ func initCollectors() {
 		Help: "Total number of failures when publishing to the DLQ, labelled by stream and consumer.",
 	}, []string{LabelStream, LabelConsumer})
 
+	publishFailuresTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: MetricPublishFailuresTotal,
+		Help: "Total number of telegram publish failures, labelled by category.",
+	}, []string{LabelCategory})
+
 	jsAPICallsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Name: MetricJSAPICallsTotal,
 		Help: "Count of JetStream API calls made by the receiver.",
@@ -157,6 +164,7 @@ func initCollectors() {
 		jsAPICallsTotal,
 		dlqMessagesTotal,
 		dlqPublishFailures,
+		publishFailuresTotal,
 		dbQueriesTotal,
 		dbQueryLatency,
 		natsConsumerPending,
@@ -218,6 +226,13 @@ func RecordDLQPublishFailure(stream, consumer string) {
 	dlqPublishFailures.WithLabelValues(labelValue(stream), labelValue(consumer)).Inc()
 }
 
+// RecordPublishFailure increments the publish failure counter for the given category.
+// This tracks general publish failures (not DLQ-specific).
+func RecordPublishFailure(category string) {
+	ensureCollectors()
+	publishFailuresTotal.WithLabelValues(labelValue(category)).Inc()
+}
+
 // RecordDBQuery records metrics for a single database operation.
 // Operation examples: "insert_one", "insert_batch", "insert_raw".
 // Result is usually "ok" or "error".