✨ Refactor telemetry and metrics handling to enhance observability. Remove TLS configuration from telemetry initialization, replacing it with an insecure option for HTTP. Update OpenTelemetry endpoint handling to normalize URLs. Introduce new metrics for publish failures categorized by message type. Enhance Grafana dashboards with detailed descriptions and additional metrics for better monitoring insights. Update documentation to reflect changes in metrics and observability features.

internal/infra/metrics/metrics.go

@@ -27,6 +27,7 @@ const (
 	MetricDBQueryLatencySeconds = "caatsm_db_query_latency_seconds"
 	MetricDLQMessagesTotal      = "caatsm_dlq_messages_total"
 	MetricDLQPublishFailures    = "caatsm_dlq_publish_failures_total"
+	MetricPublishFailuresTotal  = "caatsm_publish_failures_total"
 	MetricNATSConsumerPending   = "caatsm_nats_consumer_pending_messages"
 
 	// Common label keys.
@@ -69,6 +70,7 @@ var (
 	jsAPICallsTotal    *prometheus.CounterVec
 	dlqMessagesTotal   *prometheus.CounterVec
 	dlqPublishFailures *prometheus.CounterVec
+	publishFailuresTotal *prometheus.CounterVec
 
 	// Database metrics.
 	dbQueriesTotal *prometheus.CounterVec
@@ -125,6 +127,11 @@ func initCollectors() {
 		Help: "Total number of failures when publishing to the DLQ, labelled by stream and consumer.",
 	}, []string{LabelStream, LabelConsumer})
 
+	publishFailuresTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: MetricPublishFailuresTotal,
+		Help: "Total number of telegram publish failures, labelled by category.",
+	}, []string{LabelCategory})
+
 	jsAPICallsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Name: MetricJSAPICallsTotal,
 		Help: "Count of JetStream API calls made by the receiver.",
@@ -157,6 +164,7 @@ func initCollectors() {
 		jsAPICallsTotal,
 		dlqMessagesTotal,
 		dlqPublishFailures,
+		publishFailuresTotal,
 		dbQueriesTotal,
 		dbQueryLatency,
 		natsConsumerPending,
@@ -218,6 +226,13 @@ func RecordDLQPublishFailure(stream, consumer string) {
 	dlqPublishFailures.WithLabelValues(labelValue(stream), labelValue(consumer)).Inc()
 }
 
+// RecordPublishFailure increments the publish failure counter for the given category.
+// This tracks general publish failures (not DLQ-specific).
+func RecordPublishFailure(category string) {
+	ensureCollectors()
+	publishFailuresTotal.WithLabelValues(labelValue(category)).Inc()
+}
+
 // RecordDBQuery records metrics for a single database operation.
 // Operation examples: "insert_one", "insert_batch", "insert_raw".
 // Result is usually "ok" or "error".

internal/infra/nats/jetstream.go

@@ -72,7 +72,14 @@ func ProvideNATSConn(cfg *config.Config, logger *zap.Logger) (*nats.Conn, error)
 }
 
 // ProvideJetStream creates a JetStream context from a NATS connection.
-func ProvideJetStream(nc *nats.Conn, logger *zap.Logger) (nats.JetStreamContext, error) {
+// Returns nil when mode is "core" to indicate JetStream should not be used.
+func ProvideJetStream(nc *nats.Conn, cfg *config.Config, logger *zap.Logger) (nats.JetStreamContext, error) {
+	// In core mode, return nil so that publishers/consumers use core NATS
+	if cfg.NATS.Mode == "core" {
+		logger.Debug("Skipping JetStream initialization (core mode)")
+		return nil, nil
+	}
+
 	js, err := nc.JetStream()
 	if err != nil {
 		logger.Error("failed to create JetStream context", zap.Error(err))

internal/infra/telemetry/otel.go

@@ -4,9 +4,9 @@ import (
 	"caatsm/internal/infra/buildinfo"
 	"caatsm/internal/infra/config"
 	"context"
-	"crypto/tls"
 	"fmt"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
@@ -39,8 +39,10 @@ func InitOTEL(ctx context.Context, cfg *config.Config, logger *zap.Logger) error
 
 	// Log sensitive telemetry configuration to internal debug logs only
 	if logger != nil {
-		logger.Debug("Initializing OpenTelemetry",
-			zap.String("telemetry.endpoint", cfg.Telemetry.Endpoint),
+		normalizedEndpoint := normalizeEndpoint(cfg.Telemetry.Endpoint)
+		logger.Info("Initializing OpenTelemetry",
+			zap.String("telemetry.endpoint.original", cfg.Telemetry.Endpoint),
+			zap.String("telemetry.endpoint.normalized", normalizedEndpoint),
 			zap.Bool("telemetry.insecure", cfg.Telemetry.Insecure),
 		)
 	}
@@ -135,17 +137,33 @@ func createResource(ctx context.Context, cfg *config.Config) (*resource.Resource
 	return resource.New(ctx, resource.WithAttributes(attrs...))
 }
 
+// normalizeEndpoint removes the scheme from the endpoint, returning just host:port
+// The OpenTelemetry SDK's WithEndpoint() expects host:port, and WithInsecure() controls the protocol
+func normalizeEndpoint(endpoint string) string {
+	if endpoint == "" {
+		return endpoint
+	}
+
+	endpoint = strings.TrimSpace(endpoint)
+	
+	// Remove http:// or https:// scheme if present
+	endpoint = strings.TrimPrefix(endpoint, "http://")
+	endpoint = strings.TrimPrefix(endpoint, "https://")
+	
+	return endpoint
+}
+
 // initTracing sets up the trace provider with appropriate sampling
 func initTracing(ctx context.Context, cfg *config.Config, res *resource.Resource) error {
 	var traceExporterOptions []otlptracehttp.Option
 
-	traceExporterOptions = append(traceExporterOptions, otlptracehttp.WithEndpoint(cfg.Telemetry.Endpoint))
+	// Normalize endpoint to remove scheme (WithEndpoint expects host:port)
+	endpoint := normalizeEndpoint(cfg.Telemetry.Endpoint)
+	traceExporterOptions = append(traceExporterOptions, otlptracehttp.WithEndpoint(endpoint))
 
 	if cfg.Telemetry.Insecure {
-		traceExporterOptions = append(traceExporterOptions, otlptracehttp.WithTLSClientConfig(&tls.Config{
-			InsecureSkipVerify: true,
-			MinVersion:         tls.VersionTLS13,
-		}))
+		// Use HTTP instead of HTTPS when insecure is true
+		traceExporterOptions = append(traceExporterOptions, otlptracehttp.WithInsecure())
 	}
 
 	traceExporter, err := otlptracehttp.New(ctx, traceExporterOptions...)
@@ -174,13 +192,13 @@ func initTracing(ctx context.Context, cfg *config.Config, res *resource.Resource
 func initMetrics(ctx context.Context, cfg *config.Config, res *resource.Resource) error {
 	var metricExporterOptions []otlpmetrichttp.Option
 
-	metricExporterOptions = append(metricExporterOptions, otlpmetrichttp.WithEndpoint(cfg.Telemetry.Endpoint))
+	// Normalize endpoint to remove scheme (WithEndpoint expects host:port)
+	endpoint := normalizeEndpoint(cfg.Telemetry.Endpoint)
+	metricExporterOptions = append(metricExporterOptions, otlpmetrichttp.WithEndpoint(endpoint))
 
 	if cfg.Telemetry.Insecure {
-		metricExporterOptions = append(metricExporterOptions, otlpmetrichttp.WithTLSClientConfig(&tls.Config{
-			InsecureSkipVerify: true,
-			MinVersion:         tls.VersionTLS12,
-		}))
+		// Use HTTP instead of HTTPS when insecure is true
+		metricExporterOptions = append(metricExporterOptions, otlpmetrichttp.WithInsecure())
 	}
 
 	metricExporter, err := otlpmetrichttp.New(ctx, metricExporterOptions...)

internal/infra/telemetry/telemetry.go

@@ -183,8 +183,9 @@ func (p *promRecorder) RecordProcessingResult(ctx context.Context, status, categ
 }
 
 func (p *promRecorder) RecordPublishFailure(ctx context.Context, category string) {
-	// Prometheus metrics currently only expose failures via caatsm_failures_total,
-	// so we record the publisher failure there.
+	// Record publish failure with category label for detailed tracking.
+	obsmetrics.RecordPublishFailure(category)
+	// Also record to legacy failure counter for backward compatibility.
 	obsmetrics.RecordFailure("publisher")
 }