package nats import ( "caatsm/internal/app" "caatsm/internal/infra/config" "caatsm/internal/infra/telemetry" "context" "encoding/json" "errors" "fmt" "strings" "time" "github.com/nats-io/nats.go" "go.opentelemetry.io/otel/metric" "go.uber.org/zap" ) // MessageFetcher defines the interface for fetching messages from NATS type MessageFetcher interface { FetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error) HandleFetchError(ctx context.Context, err error, sub **nats.Subscription, fetchErrorStreak *int) (bool, error) } // MessageProcessor defines the interface for processing message batches type MessageProcessor interface { ProcessBatch(ctx context.Context, msgs []*nats.Msg) } // DLQHandler defines the interface for dead letter queue operations type DLQHandler interface { RouteToDLQ(ctx context.Context, msg *nats.Msg, cause error) error ValidateDLQ() error } // Consumer handles NATS JetStream message consumption with clean separation of concerns type Consumer struct { // Core dependencies conn *nats.Conn js nats.JetStreamContext processor *app.MessageProcessor cfg *config.Config logger *zap.Logger telemetry telemetry.Recorder // Configuration config consumerConfig // Collaborators (injected for testability) fetcher MessageFetcher batchProcessor MessageProcessor dlqHandler DLQHandler errorHandler *ErrorHandler // Resource managers consumerManager *ConsumerManager streamManager *StreamManager // Advisory DLQ handler for messages exhausting MaxDeliver advisoryDLQHandler *AdvisoryDLQHandler // Metrics meter metric.Meter ackPending metric.Int64Histogram redelivered metric.Int64Histogram pending metric.Int64Histogram delivered metric.Int64Histogram // State consecutiveProcessErrors int } // consumerConfig holds normalized consumer configuration values. type consumerConfig struct { subject string consumerName string mode string streamName string dlqSubject string ackWait time.Duration batchSize int batchTimeout time.Duration monitorInterval time.Duration } // defaultMessageFetcher implements MessageFetcher interface type defaultMessageFetcher struct { batchSize int batchTimeout time.Duration logger *zap.Logger conn *nats.Conn js nats.JetStreamContext consumerManager *ConsumerManager streamManager *StreamManager config *consumerConfig cfg *config.Config } func (f *defaultMessageFetcher) FetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error) { return f.fetchBatch(ctx, sub) } // fetchBatch fetches a batch of messages from the subscription with context awareness func (f *defaultMessageFetcher) fetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error) { // Check context before fetching select { case <-ctx.Done(): return nil, ctx.Err() default: } // Use a shorter timeout for better responsiveness to cancellation timeout := f.batchTimeout if timeout > 500*time.Millisecond { timeout = 500 * time.Millisecond } return sub.Fetch(f.batchSize, nats.MaxWait(timeout)) } func (f *defaultMessageFetcher) HandleFetchError(ctx context.Context, err error, sub **nats.Subscription, fetchErrorStreak *int) (bool, error) { // Check context cancellation first if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { f.logger.Info("Fetch error due to context cancellation", zap.Error(err)) return false, err } // Timeout errors are expected when no messages are available - not an error condition if errors.Is(err, nats.ErrTimeout) { return true, nil } // Check connection health before proceeding if f.conn != nil { status := f.conn.Status() if status != nats.CONNECTED { f.logger.Warn("NATS connection not in CONNECTED state", zap.String("status", status.String()), zap.Error(err), ) // Connection is down - this is a transient error, apply backoff *fetchErrorStreak++ backoff := f.calculateExponentialBackoff(*fetchErrorStreak) f.logger.Warn("Connection unhealthy, applying backoff before retry", zap.String("status", status.String()), zap.Int("error_streak", *fetchErrorStreak), zap.Duration("backoff", backoff), ) if !sleepWithContext(ctx, backoff) { return false, ctx.Err() } // Check if connection recovered after backoff if f.conn.Status() == nats.CONNECTED { *fetchErrorStreak = 0 return true, nil } // Still not connected - continue with error handling } } // Check for connection closed errors if errors.Is(err, nats.ErrConnectionClosed) { f.logger.Error("NATS connection closed", zap.Error(err), zap.String("stream", f.config.streamName), zap.String("consumer", f.config.consumerName), ) // Connection closed is fatal - cannot recover subscription if *sub != nil { (*sub).Unsubscribe() *sub = nil } return false, fmt.Errorf("connection closed: %w", err) } // JetStream API unavailable (e.g., NATS restarted or JetStream not ready) if errors.Is(err, nats.ErrNoResponders) { *fetchErrorStreak++ backoff := f.calculateExponentialBackoff(*fetchErrorStreak) backoff = min(backoff, 30*time.Second) f.logger.Warn("JetStream not available, will retry with backoff", zap.Error(err), zap.String("stream", f.config.streamName), zap.String("consumer", f.config.consumerName), zap.Int("error_streak", *fetchErrorStreak), zap.Duration("backoff", backoff), ) if !sleepWithContext(ctx, backoff) { return false, ctx.Err() } return true, nil } // Check for JetStream resource not found errors if isJetStreamResourceNotFound(err) { if isDevLikeEnv() && shouldBootstrapStream() { f.logger.Warn("JetStream consumer or stream missing; attempting to recreate", zap.Error(err), zap.String("stream", f.config.streamName), zap.String("consumer", f.config.consumerName), ) // Attempt to recover resources and recreate subscription if f.consumerManager == nil || f.streamManager == nil { return false, fmt.Errorf("cannot recover: consumer/stream manager not available: %w", err) } consumerConfig := f.buildConsumerConfig() if recErr := f.consumerManager.RecoverResources(f.streamManager, consumerConfig); recErr != nil { return false, fmt.Errorf("failed to recover JetStream resources: %w", recErr) } // Unsubscribe old subscription before creating new one if *sub != nil { (*sub).Unsubscribe() } // Create new subscription newSub, subErr := f.consumerManager.CreatePullSubscription() if subErr != nil { return false, fmt.Errorf("failed to create pull subscription after recovery: %w", subErr) } *sub = newSub *fetchErrorStreak = 0 f.logger.Info("Successfully recovered subscription after resource recreation") return true, nil } // Production: treat as configuration/operational error - fatal f.logger.Error("JetStream consumer or stream missing; not auto-recreating in this environment", zap.Error(err), zap.String("stream", f.config.streamName), zap.String("consumer", f.config.consumerName), ) if *sub != nil { (*sub).Unsubscribe() *sub = nil } return false, fmt.Errorf("JetStream resource not found: %w", err) } // Check for network/temporary errors if f.isTemporaryError(err) { *fetchErrorStreak++ backoff := f.calculateExponentialBackoff(*fetchErrorStreak) f.logger.Warn("Temporary network error, applying backoff", zap.Error(err), zap.Int("error_streak", *fetchErrorStreak), zap.Duration("backoff", backoff), ) if !sleepWithContext(ctx, backoff) { return false, ctx.Err() } // Verify subscription is still valid before returning success if *sub != nil && f.conn != nil && f.conn.Status() == nats.CONNECTED { return true, nil } // Subscription or connection invalid - attempt recovery return f.attemptSubscriptionRecovery(ctx, sub, fetchErrorStreak) } // Generic error path with exponential backoff *fetchErrorStreak++ backoff := f.calculateExponentialBackoff(*fetchErrorStreak) f.logger.Error("Failed to fetch messages; backing off", zap.Error(err), zap.Int("error_streak", *fetchErrorStreak), zap.Duration("backoff", backoff), ) if !sleepWithContext(ctx, backoff) { return false, ctx.Err() } // Verify subscription and connection health before returning success if *sub == nil || (f.conn != nil && f.conn.Status() != nats.CONNECTED) { return f.attemptSubscriptionRecovery(ctx, sub, fetchErrorStreak) } return true, nil } // calculateExponentialBackoff calculates exponential backoff duration with a cap func (f *defaultMessageFetcher) calculateExponentialBackoff(streak int) time.Duration { if streak <= 0 { return 0 } // Exponential backoff: 2^(streak-1) seconds, capped at 30 seconds backoff := time.Duration(1<