✨ Enhance observability and error handling in NATS integration. Introduce comprehensive OpenTelemetry support with environment-based sampling and semantic attributes for tracing and metrics. Implement an advisory dead-letter queue (DLQ) handler for managing message delivery failures. Update NATS consumer to utilize structured logging and improve error handling strategies. Refactor configuration files for OpenTelemetry collector in both development and production environments, ensuring robust telemetry integration. Enhance documentation to reflect new features and best practices for observability.

internal/infra/nats/consumer_error.go

@@ -0,0 +1,74 @@
+package nats
+
+import (
+	"caatsm/internal/app"
+	obsmetrics "caatsm/internal/infra/metrics"
+	"context"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"go.uber.org/zap"
+)
+
+// handleMessageError handles errors that occur during message processing.
+func (c *Consumer) handleMessageError(ctx context.Context, msg *nats.Msg, err error, elapsed time.Duration) {
+	c.logger.Error("Failed to process message",
+		zap.String("subject", msg.Subject),
+		zap.Error(err),
+		zap.Bool("permanent", app.IsPermanent(err)),
+	)
+
+	result := obsmetrics.ResultFail
+	if app.IsPermanent(err) {
+		result = obsmetrics.ResultPermanentFail
+	}
+	c.telemetry.RecordMessageHandled(ctx, c.config.streamName, c.config.consumerName, result, elapsed)
+
+	processingResult := c.errorHandler.HandleProcessingError(c.consecutiveProcessErrors, err, c.logger, msg.Subject)
+
+	if processingResult.IsPermanent {
+		c.handlePermanentError(ctx, msg, err)
+		return
+	}
+
+	c.handleTransientError(ctx, msg, processingResult)
+}
+
+// handlePermanentError handles permanent/poison messages.
+func (c *Consumer) handlePermanentError(ctx context.Context, msg *nats.Msg, err error) {
+	c.consecutiveProcessErrors = 0
+	// Poison/permanent message: route to DLQ if configured, then ACK
+	if dlqErr := c.routeToDLQ(ctx, msg, err); dlqErr != nil {
+		c.logger.Error("Failed to route permanent-error message to DLQ", zap.Error(dlqErr))
+	}
+	if ackErr := msg.Ack(); ackErr != nil {
+		c.logger.Error("Failed to ACK permanent-error message", zap.Error(ackErr))
+	}
+}
+
+// handleTransientError handles transient errors with backpressure and redelivery.
+func (c *Consumer) handleTransientError(ctx context.Context, msg *nats.Msg, processingResult ProcessingErrorResult) {
+	// Increment error streak
+	if c.consecutiveProcessErrors < 0 {
+		c.consecutiveProcessErrors = 0
+	}
+	c.consecutiveProcessErrors++
+
+	if processingResult.ShouldApplyBackpressure {
+		c.logger.Warn("Applying backpressure due to consecutive processing errors",
+			zap.Int("consecutive_errors", c.consecutiveProcessErrors),
+			zap.Duration("sleep", processingResult.BackpressureDelay),
+		)
+		// Use context-aware sleep instead of blocking time.Sleep
+		if !sleepWithContext(ctx, processingResult.BackpressureDelay) {
+			// Context canceled, stop processing
+			return
+		}
+	}
+
+	// Transient error: request redelivery with optional delay
+	c.telemetry.RecordRetry(ctx, c.config.streamName, c.config.consumerName, obsmetrics.RetryReasonProcessorError)
+	if nakErr := c.nakWithStrategy(msg); nakErr != nil {
+		c.logger.Error("Failed to NAK message", zap.Error(nakErr))
+	}
+}