windy/go-caatsm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ Update agent guidelines and improve documentation structure. Refactor AGENTS.md to streamline commands and code style guidelines, enhancing clarity and usability. Update README.md with refined NATS consumer configuration details and observability metrics. Modify .gitignore to exclude dynamically generated Prometheus target files. Enhance configuration files for development and production environments, ensuring consistency and clarity in settings.

Browse Source
This commit is contained in:
windyboy
2025-11-19 13:07:09 +08:00
parent 06fc9cb9e0
commit c687fdcde8
39 changed files with 1112 additions and 3193 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/infra/nats/advisory_dlq.go
-169
View File
@@ -1,169 +0,0 @@
package nats
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
)
// MaxDeliveriesAdvisoryEvent represents the advisory message published when
// a message reaches MaxDeliver attempts.
type MaxDeliveriesAdvisoryEvent struct {
Type string `json:"type"`
Stream string `json:"stream"`
Consumer string `json:"consumer"`
StreamSeq uint64 `json:"stream_seq"`
Deliveries uint64 `json:"deliveries"`
Time string `json:"time"`
}
// AdvisoryDLQHandler handles messages that exhaust MaxDeliver attempts
// by subscribing to JetStream advisory events.
type AdvisoryDLQHandler struct {
js nats.JetStreamContext
nc *nats.Conn
streamName string
consumerName string
dlqSubject string
logger *zap.Logger
telemetry TelemetryRecorder
}
// TelemetryRecorder is an interface for recording telemetry events.
// This matches the telemetry.Recorder interface used by Consumer.
type TelemetryRecorder interface {
RecordDLQMessage(ctx context.Context, stream, consumer string)
RecordDLQPublishFailure(ctx context.Context, stream, consumer string)
}
// NewAdvisoryDLQHandler creates a new advisory-based DLQ handler.
func NewAdvisoryDLQHandler(
js nats.JetStreamContext,
nc *nats.Conn,
streamName string,
consumerName string,
dlqSubject string,
logger *zap.Logger,
telemetry TelemetryRecorder,
) (*AdvisoryDLQHandler, error) {
return &AdvisoryDLQHandler{
js: js,
nc: nc,
streamName: streamName,
consumerName: consumerName,
dlqSubject: dlqSubject,
logger: logger,
telemetry: telemetry,
}, nil
}
// Start begins listening for advisory messages and routing failed messages to DLQ.
func (h *AdvisoryDLQHandler) Start(ctx context.Context) error {
// Subscribe to advisory subject pattern
// Format: $JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.<STREAM>.<CONSUMER>
advisorySubject := fmt.Sprintf("$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.%s.%s",
h.streamName, h.consumerName)
h.logger.Info("Starting advisory DLQ handler",
zap.String("advisory_subject", advisorySubject),
zap.String("stream", h.streamName),
zap.String("consumer", h.consumerName),
zap.String("dlq_subject", h.dlqSubject),
)
sub, err := h.nc.Subscribe(advisorySubject, func(msg *nats.Msg) {
h.handleAdvisory(ctx, msg)
})
if err != nil {
return fmt.Errorf("failed to subscribe to advisory subject: %w", err)
}
// Wait for context cancellation
go func() {
<-ctx.Done()
if err := sub.Unsubscribe(); err != nil {
h.logger.Error("Failed to unsubscribe advisory subscription", zap.Error(err))
}
h.logger.Info("Stopped advisory DLQ handler")
}()
return nil
}
// handleAdvisory processes an advisory message about max deliveries.
func (h *AdvisoryDLQHandler) handleAdvisory(ctx context.Context, advisoryMsg *nats.Msg) {
var event MaxDeliveriesAdvisoryEvent
if err := json.Unmarshal(advisoryMsg.Data, &event); err != nil {
h.logger.Error("Failed to unmarshal advisory event",
zap.Error(err),
zap.String("data", string(advisoryMsg.Data)),
)
return
}
h.logger.Warn("Message reached MaxDeliver attempts",
zap.String("stream", event.Stream),
zap.String("consumer", event.Consumer),
zap.Uint64("stream_seq", event.StreamSeq),
zap.Uint64("deliveries", event.Deliveries),
)
// Retrieve the original message from the stream using GetMsg API
originalMsg, err := h.js.GetMsg(h.streamName, event.StreamSeq)
if err != nil {
h.logger.Error("Failed to retrieve original message from stream",
zap.Uint64("stream_seq", event.StreamSeq),
zap.Error(err),
)
h.telemetry.RecordDLQPublishFailure(ctx, h.streamName, h.consumerName)
return
}
// Extract message metadata
msgID := ""
subject := originalMsg.Subject
if originalMsg.Header != nil {
msgID = originalMsg.Header.Get("Nats-Msg-Id")
}
// Create enriched DLQ payload (similar to existing routeToDLQ)
payload := map[string]interface{}{
"transport_msg_id": msgID,
"subject": subject,
"stream": h.streamName,
"consumer": h.consumerName,
"nats_sequence": event.StreamSeq,
"deliveries": event.Deliveries,
"error": fmt.Sprintf("message exhausted max_deliver (%d) attempts", event.Deliveries),
"received_at": time.Now().UTC(),
"body": string(originalMsg.Data),
"advisory_source": true, // Flag to distinguish from immediate DLQ
}
data, err := json.Marshal(payload)
if err != nil {
h.logger.Error("Failed to marshal advisory DLQ payload", zap.Error(err))
h.telemetry.RecordDLQPublishFailure(ctx, h.streamName, h.consumerName)
return
}
// Publish to DLQ
if _, err := h.js.Publish(h.dlqSubject, data); err != nil {
h.logger.Error("Failed to publish advisory message to DLQ",
zap.Uint64("stream_seq", event.StreamSeq),
zap.Error(err),
)
h.telemetry.RecordDLQPublishFailure(ctx, h.streamName, h.consumerName)
return
}
h.logger.Info("Routed max-deliveries message to DLQ",
zap.Uint64("stream_seq", event.StreamSeq),
zap.Uint64("deliveries", event.Deliveries),
)
h.telemetry.RecordDLQMessage(ctx, h.streamName, h.consumerName)
}
internal/infra/nats/consumer.go
+182 -691
View File
@@ -3,42 +3,18 @@ package nats
import (
"caatsm/internal/app"
"caatsm/internal/infra/config"
"caatsm/internal/infra/log"
obsmetrics "caatsm/internal/infra/metrics"
"caatsm/internal/infra/telemetry"
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/google/uuid"
"github.com/nats-io/nats.go"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/metric"
"go.uber.org/zap"
)
// MessageFetcher defines the interface for fetching messages from NATS
type MessageFetcher interface {
FetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error)
HandleFetchError(ctx context.Context, err error, sub **nats.Subscription, fetchErrorStreak *int) (bool, error)
}
// MessageProcessor defines the interface for processing message batches
type MessageProcessor interface {
ProcessBatch(ctx context.Context, msgs []*nats.Msg)
}
// DLQHandler defines the interface for dead letter queue operations
type DLQHandler interface {
RouteToDLQ(ctx context.Context, msg *nats.Msg, cause error) error
ValidateDLQ() error
}
// Consumer handles NATS JetStream message consumption with clean separation of concerns
type Consumer struct {
// Core dependencies
@@ -56,22 +32,11 @@ type Consumer struct {
fetcher MessageFetcher
batchProcessor MessageProcessor
dlqHandler DLQHandler
errorHandler *ErrorHandler
// Resource managers
consumerManager *ConsumerManager
streamManager *StreamManager
// Advisory DLQ handler for messages exhausting MaxDeliver
advisoryDLQHandler *AdvisoryDLQHandler
// Metrics
meter metric.Meter
ackPending metric.Int64Histogram
redelivered metric.Int64Histogram
pending metric.Int64Histogram
delivered metric.Int64Histogram
// State
consecutiveProcessErrors int
}
@@ -89,597 +54,64 @@ type consumerConfig struct {
monitorInterval time.Duration
}
// defaultMessageFetcher implements MessageFetcher interface
type defaultMessageFetcher struct {
batchSize int
batchTimeout time.Duration
logger *zap.Logger
conn *nats.Conn
js nats.JetStreamContext
consumerManager *ConsumerManager
streamManager *StreamManager
config *consumerConfig
cfg *config.Config
}
// ProvideConsumer creates a NATS consumer with clean architecture.
func ProvideConsumer(
conn *nats.Conn,
js nats.JetStreamContext,
processor *app.MessageProcessor,
cfg *config.Config,
rec telemetry.Recorder,
logger *zap.Logger,
) (*Consumer, error) {
normCfg := normalizeConsumerConfig(cfg)
func (f *defaultMessageFetcher) FetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error) {
return f.fetchBatch(ctx, sub)
}
// fetchBatch fetches a batch of messages from the subscription with context awareness
func (f *defaultMessageFetcher) fetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error) {
// Check context before fetching
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
consumer := &Consumer{
conn: conn,
js: js,
processor: processor,
cfg: cfg,
logger: logger,
telemetry: rec,
config: *normCfg, // dereference the pointer
}
consumer.initCollaborators()
// Use a shorter timeout for better responsiveness to cancellation
timeout := f.batchTimeout
if timeout > 500*time.Millisecond {
timeout = 500 * time.Millisecond
}
return sub.Fetch(f.batchSize, nats.MaxWait(timeout))
}
func (f *defaultMessageFetcher) HandleFetchError(ctx context.Context, err error, sub **nats.Subscription, fetchErrorStreak *int) (bool, error) {
// Check context cancellation first
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
f.logger.Info("Fetch error due to context cancellation", zap.Error(err))
return false, err
}
// Timeout errors are expected when no messages are available - not an error condition
if errors.Is(err, nats.ErrTimeout) {
return true, nil
}
// Check connection health before proceeding
if f.conn != nil {
status := f.conn.Status()
if status != nats.CONNECTED {
f.logger.Warn("NATS connection not in CONNECTED state",
zap.String("status", status.String()),
zap.Error(err),
)
// Connection is down - this is a transient error, apply backoff
*fetchErrorStreak++
backoff := f.calculateExponentialBackoff(*fetchErrorStreak)
f.logger.Warn("Connection unhealthy, applying backoff before retry",
zap.String("status", status.String()),
zap.Int("error_streak", *fetchErrorStreak),
zap.Duration("backoff", backoff),
)
if !sleepWithContext(ctx, backoff) {
return false, ctx.Err()
}
// Check if connection recovered after backoff
if f.conn.Status() == nats.CONNECTED {
*fetchErrorStreak = 0
return true, nil
}
// Still not connected - continue with error handling
// Initialize managers
if consumer.config.mode == "jetstream" {
consumer.consumerManager = NewConsumerManager(js, normCfg.streamName, normCfg.consumerName, normCfg.subject, logger)
// Use StreamManager with full configuration
streamSubjects := []string{normCfg.subject}
if publisherSubject := strings.TrimSpace(cfg.Publisher.Topic); publisherSubject != "" {
streamSubjects = append(streamSubjects, publisherSubject)
}
}
streamSubjects = dedupeSubjects(streamSubjects)
consumer.streamManager = NewStreamManager(js, normCfg.streamName, streamSubjects, logger)
// Check for connection closed errors
if errors.Is(err, nats.ErrConnectionClosed) {
f.logger.Error("NATS connection closed",
zap.Error(err),
zap.String("stream", f.config.streamName),
zap.String("consumer", f.config.consumerName),
)
// Connection closed is fatal - cannot recover subscription
if *sub != nil {
if err := (*sub).Unsubscribe(); err != nil {
f.logger.Error("Failed to unsubscribe after connection closed", zap.Error(err))
}
*sub = nil
}
return false, fmt.Errorf("connection closed: %w", err)
}
// JetStream API unavailable (e.g., NATS restarted or JetStream not ready)
if errors.Is(err, nats.ErrNoResponders) {
*fetchErrorStreak++
backoff := f.calculateExponentialBackoff(*fetchErrorStreak)
backoff = min(backoff, 30*time.Second)
f.logger.Warn("JetStream not available, will retry with backoff",
zap.Error(err),
zap.String("stream", f.config.streamName),
zap.String("consumer", f.config.consumerName),
zap.Int("error_streak", *fetchErrorStreak),
zap.Duration("backoff", backoff),
)
if !sleepWithContext(ctx, backoff) {
return false, ctx.Err()
}
return true, nil
}
// Check for JetStream resource not found errors
if isJetStreamResourceNotFound(err) {
if isDevLikeEnv() && shouldBootstrapStream() {
f.logger.Warn("JetStream consumer or stream missing; attempting to recreate",
zap.Error(err),
zap.String("stream", f.config.streamName),
zap.String("consumer", f.config.consumerName),
)
// Attempt to recover resources and recreate subscription
if f.consumerManager == nil || f.streamManager == nil {
return false, fmt.Errorf("cannot recover: consumer/stream manager not available: %w", err)
}
consumerConfig := f.buildConsumerConfig()
if recErr := f.consumerManager.RecoverResources(f.streamManager, consumerConfig); recErr != nil {
return false, fmt.Errorf("failed to recover JetStream resources: %w", recErr)
}
// Unsubscribe old subscription before creating new one
if *sub != nil {
if err := (*sub).Unsubscribe(); err != nil {
f.logger.Error("Failed to unsubscribe during recovery", zap.Error(err))
}
}
// Create new subscription
newSub, subErr := f.consumerManager.CreatePullSubscription()
if subErr != nil {
return false, fmt.Errorf("failed to create pull subscription after recovery: %w", subErr)
}
*sub = newSub
*fetchErrorStreak = 0
f.logger.Info("Successfully recovered subscription after resource recreation")
return true, nil
// Update fetcher with managers now that they're initialized
if fetcher, ok := consumer.fetcher.(*defaultMessageFetcher); ok {
fetcher.consumerManager = consumer.consumerManager
fetcher.streamManager = consumer.streamManager
}
// Production: treat as configuration/operational error - fatal
f.logger.Error("JetStream consumer or stream missing; not auto-recreating in this environment",
zap.Error(err),
zap.String("stream", f.config.streamName),
zap.String("consumer", f.config.consumerName),
)
if *sub != nil {
if err := (*sub).Unsubscribe(); err != nil {
f.logger.Error("Failed to unsubscribe after resource not found", zap.Error(err))
}
*sub = nil
// Create consumer if it doesn't exist
consumerConfig := consumer.buildConsumerConfig()
if err := consumer.consumerManager.EnsureConsumer(consumerConfig); err != nil {
return nil, fmt.Errorf("failed to ensure consumer: %w", err)
}
return false, fmt.Errorf("JetStream resource not found: %w", err)
}
// Check for network/temporary errors
if f.isTemporaryError(err) {
*fetchErrorStreak++
backoff := f.calculateExponentialBackoff(*fetchErrorStreak)
f.logger.Warn("Temporary network error, applying backoff",
zap.Error(err),
zap.Int("error_streak", *fetchErrorStreak),
zap.Duration("backoff", backoff),
)
if !sleepWithContext(ctx, backoff) {
return false, ctx.Err()
// Validate DLQ configuration early so misconfiguration is visible at startup
// rather than only when the first poison message appears.
if err := consumer.validateDLQ(); err != nil {
return nil, fmt.Errorf("DLQ validation failed: %w", err)
}
// Verify subscription is still valid before returning success
if *sub != nil && f.conn != nil && f.conn.Status() == nats.CONNECTED {
return true, nil
}
// Subscription or connection invalid - attempt recovery
return f.attemptSubscriptionRecovery(ctx, sub, fetchErrorStreak)
}
// Generic error path with exponential backoff
*fetchErrorStreak++
backoff := f.calculateExponentialBackoff(*fetchErrorStreak)
f.logger.Error("Failed to fetch messages; backing off",
zap.Error(err),
zap.Int("error_streak", *fetchErrorStreak),
zap.Duration("backoff", backoff),
)
if !sleepWithContext(ctx, backoff) {
return false, ctx.Err()
}
// Verify subscription and connection health before returning success
if *sub == nil || (f.conn != nil && f.conn.Status() != nats.CONNECTED) {
return f.attemptSubscriptionRecovery(ctx, sub, fetchErrorStreak)
}
return true, nil
}
// calculateExponentialBackoff calculates exponential backoff duration with a cap
func (f *defaultMessageFetcher) calculateExponentialBackoff(streak int) time.Duration {
if streak <= 0 {
return 0
}
// Exponential backoff: 2^(streak-1) seconds, capped at 30 seconds
backoff := time.Duration(1<<uint(min(streak-1, 5))) * time.Second
return min(backoff, 30*time.Second)
}
// isTemporaryError checks if an error is a temporary/network error that might recover
func (f *defaultMessageFetcher) isTemporaryError(err error) bool {
if err == nil {
return false
}
// Check for typed temporary errors first
var tempErr interface{ Temporary() bool }
if errors.As(err, &tempErr) && tempErr.Temporary() {
return true
}
// Fall back to string matching for external errors
errStr := strings.ToLower(err.Error())
return strings.Contains(errStr, "timeout") ||
strings.Contains(errStr, "temporary") ||
strings.Contains(errStr, "network") ||
strings.Contains(errStr, "connection reset") ||
strings.Contains(errStr, "broken pipe")
}
// attemptSubscriptionRecovery attempts to recover a subscription after errors
func (f *defaultMessageFetcher) attemptSubscriptionRecovery(ctx context.Context, sub **nats.Subscription, fetchErrorStreak *int) (bool, error) {
if f.consumerManager == nil {
f.logger.Error("Cannot recover subscription: consumer manager not available")
return false, fmt.Errorf("consumer manager not available for recovery")
}
// Check connection health first
if f.conn != nil && f.conn.Status() != nats.CONNECTED {
f.logger.Warn("Connection not healthy, cannot recover subscription",
zap.String("status", f.conn.Status().String()),
)
// Connection issue - return true to retry after backoff
return true, nil
}
// Unsubscribe old subscription if it exists
if *sub != nil {
if err := (*sub).Unsubscribe(); err != nil {
f.logger.Error("Failed to unsubscribe during recovery", zap.Error(err))
}
*sub = nil
}
// Attempt to recreate subscription
newSub, err := f.consumerManager.CreatePullSubscriptionWithRecovery(f.streamManager, f.buildConsumerConfig())
if err != nil {
f.logger.Error("Failed to recover subscription",
zap.Error(err),
zap.String("stream", f.config.streamName),
zap.String("consumer", f.config.consumerName),
)
return false, fmt.Errorf("failed to recover subscription: %w", err)
}
*sub = newSub
*fetchErrorStreak = 0
f.logger.Info("Successfully recovered subscription")
return true, nil
}
// buildConsumerConfig builds the NATS consumer configuration
func (f *defaultMessageFetcher) buildConsumerConfig() *nats.ConsumerConfig {
if f.cfg == nil {
return nil
}
return &nats.ConsumerConfig{
Durable: f.config.consumerName,
DeliverPolicy: mapDeliverPolicy(f.cfg.NATS.ConsumerRules.DeliverPolicy),
AckPolicy: nats.AckExplicitPolicy,
AckWait: f.config.ackWait,
ReplayPolicy: mapReplayPolicy(f.cfg.NATS.ConsumerRules.ReplayPolicy),
MaxDeliver: f.cfg.NATS.ConsumerRules.MaxDeliver,
MaxAckPending: f.cfg.NATS.ConsumerRules.MaxAckPending,
FilterSubject: f.config.subject,
BackOff: f.cfg.NATS.ConsumerRules.Backoff,
}
}
// defaultBatchProcessor implements MessageProcessor interface
type defaultBatchProcessor struct {
processor *app.MessageProcessor
dlqHandler DLQHandler
errorHandler *ErrorHandler
logger *zap.Logger
telemetry telemetry.Recorder
// Configuration needed for processing
streamName string
consumerName string
mode string
backoff []time.Duration
// Pointer to consecutive errors counter (shared with Consumer)
consecutiveProcessErrors *int
}
func (p *defaultBatchProcessor) ProcessBatch(ctx context.Context, msgs []*nats.Msg) {
for _, msg := range msgs {
// Check context before processing each message
select {
case <-ctx.Done():
p.logger.Info("Stopping batch processing due to cancellation",
zap.Int("remaining_messages", len(msgs)),
)
return
default:
}
p.processSingleMessage(ctx, msg)
}
}
// processSingleMessage processes a single message with error handling and backpressure.
func (p *defaultBatchProcessor) processSingleMessage(ctx context.Context, msg *nats.Msg) {
start := time.Now()
if err := p.processMessage(ctx, msg); err != nil {
p.handleMessageError(ctx, msg, err, time.Since(start))
return
}
// Successful processing resets the error streak.
if p.consecutiveProcessErrors != nil && *p.consecutiveProcessErrors > 0 {
*p.consecutiveProcessErrors = 0
}
// ACK the message
if ackErr := msg.Ack(); ackErr != nil {
p.logger.Error("Failed to ACK message", zap.Error(ackErr))
} else {
elapsed := time.Since(start)
p.telemetry.RecordMessageHandled(ctx, p.streamName, p.consumerName, "ok", elapsed)
}
}
// processMessage processes a single message.
func (p *defaultBatchProcessor) processMessage(ctx context.Context, msg *nats.Msg) error {
ctx, span := otel.Tracer("caatsm/nats").Start(ctx, "Consumer.processMessage")
defer span.End()
// Set semantic messaging attributes
span.SetAttributes(
attribute.String("messaging.system", "nats"),
attribute.String("messaging.operation.name", "receive"),
attribute.String("messaging.destination.name", msg.Subject),
attribute.String("messaging.consumer.group.name", p.consumerName),
attribute.String("caatsm.stream", p.streamName),
)
msgID, source, err := p.resolveMsgID(msg)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("unable to resolve message id: %w", err)
}
if source != "header" {
p.logger.Warn("Message missing NATS id header; using fallback",
zap.String("subject", msg.Subject),
zap.String("msg_id_source", source),
zap.String("msg_id", msgID),
logger.Info("Running consumer in core NATS mode",
zap.String("subject", normCfg.subject),
zap.String("queue_group", cfg.Subscription.QueueGroup),
)
}
// Attach structured logging context including stream/consumer and NATS metadata.
jsSeq := uint64(0)
if meta, metaErr := msg.Metadata(); metaErr == nil {
jsSeq = meta.Sequence.Stream
span.SetAttributes(
attribute.Int64("nats.js.stream_seq", int64(meta.Sequence.Stream)),
attribute.Int64("nats.js.consumer_seq", int64(meta.Sequence.Consumer)),
)
}
msgLogger := log.WithMessageContext(p.logger, log.MessageFields{
Service: "caatsm-consumer",
TransportMsgID: msgID,
Stream: p.streamName,
Consumer: p.consumerName,
Subject: msg.Subject,
JSSequence: jsSeq,
})
msgLogger.Debug("Processing message",
zap.Int("data_size", len(msg.Data)),
zap.String("msg_id_source", source),
)
// Call processor
if err := p.processor.Handle(ctx, msg.Data, msgID); err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("processor error: %w", err)
}
span.SetAttributes(attribute.String("telegram.msg_id", msgID))
return nil
}
// resolveMsgID extracts or generates a message ID.
func (p *defaultBatchProcessor) resolveMsgID(msg *nats.Msg) (string, string, error) {
if id := msg.Header.Get("Nats-Msg-Id"); id != "" {
return id, "header", nil
}
if p.mode == "core" {
return uuid.NewString(), "generated", nil
}
meta, err := msg.Metadata()
if err != nil {
return "", "", fmt.Errorf("fetch metadata: %w", err)
}
return fmt.Sprintf("js-%d", meta.Sequence.Stream), "metadata", nil
}
// handleMessageError handles errors that occur during message processing.
func (p *defaultBatchProcessor) handleMessageError(ctx context.Context, msg *nats.Msg, err error, elapsed time.Duration) {
p.logger.Error("Failed to process message",
zap.String("subject", msg.Subject),
zap.Error(err),
zap.Bool("permanent", app.IsPermanent(err)),
)
result := obsmetrics.ResultFail
if app.IsPermanent(err) {
result = obsmetrics.ResultPermanentFail
}
p.telemetry.RecordMessageHandled(ctx, p.streamName, p.consumerName, result, elapsed)
consecutiveErrors := 0
if p.consecutiveProcessErrors != nil {
consecutiveErrors = *p.consecutiveProcessErrors
}
processingResult := p.errorHandler.HandleProcessingError(consecutiveErrors, err, p.logger, msg.Subject)
if processingResult.IsPermanent {
p.handlePermanentError(ctx, msg, err)
return
}
p.handleTransientError(ctx, msg, processingResult)
}
// handlePermanentError handles permanent/poison messages.
func (p *defaultBatchProcessor) handlePermanentError(ctx context.Context, msg *nats.Msg, err error) {
if p.consecutiveProcessErrors != nil {
*p.consecutiveProcessErrors = 0
}
// Poison/permanent message: route to DLQ if configured, then ACK
if p.dlqHandler != nil {
if dlqErr := p.dlqHandler.RouteToDLQ(ctx, msg, err); dlqErr != nil {
p.logger.Error("Failed to route permanent-error message to DLQ", zap.Error(dlqErr))
}
}
if ackErr := msg.Ack(); ackErr != nil {
p.logger.Error("Failed to ACK permanent-error message", zap.Error(ackErr))
}
}
// handleTransientError handles transient errors with backpressure and redelivery.
func (p *defaultBatchProcessor) handleTransientError(ctx context.Context, msg *nats.Msg, processingResult ProcessingErrorResult) {
// Increment error streak
if p.consecutiveProcessErrors != nil {
if *p.consecutiveProcessErrors < 0 {
*p.consecutiveProcessErrors = 0
}
*p.consecutiveProcessErrors++
}
if processingResult.ShouldApplyBackpressure {
consecutiveErrors := 0
if p.consecutiveProcessErrors != nil {
consecutiveErrors = *p.consecutiveProcessErrors
}
p.logger.Warn("Applying backpressure due to consecutive processing errors",
zap.Int("consecutive_errors", consecutiveErrors),
zap.Duration("sleep", processingResult.BackpressureDelay),
)
// Use context-aware sleep instead of blocking time.Sleep
if !sleepWithContext(ctx, processingResult.BackpressureDelay) {
// Context canceled, stop processing
return
}
}
// Transient error: request redelivery with optional delay
p.telemetry.RecordRetry(ctx, p.streamName, p.consumerName, obsmetrics.RetryReasonProcessorError)
if nakErr := p.nakWithStrategy(msg); nakErr != nil {
p.logger.Error("Failed to NAK message", zap.Error(nakErr))
}
}
// nakWithStrategy sends a NAK with appropriate delay based on retry attempt.
func (p *defaultBatchProcessor) nakWithStrategy(msg *nats.Msg) error {
if len(p.backoff) == 0 {
return msg.Nak()
}
meta, err := msg.Metadata()
if err != nil {
p.logger.Warn("Failed to read metadata for backoff strategy", zap.Error(err))
return msg.Nak()
}
attempt := int(meta.NumDelivered)
index := attempt - 1
if index < 0 {
index = 0
}
if index >= len(p.backoff) {
index = len(p.backoff) - 1
}
delay := p.backoff[index]
if delay <= 0 {
return msg.Nak()
}
return msg.NakWithDelay(delay)
}
// defaultDLQHandler implements DLQHandler interface
type defaultDLQHandler struct {
js nats.JetStreamContext
dlqSubject string
streamName string
consumerName string
logger *zap.Logger
telemetry telemetry.Recorder
}
func (h *defaultDLQHandler) RouteToDLQ(ctx context.Context, msg *nats.Msg, cause error) error {
return h.routeToDLQInternal(ctx, msg, cause)
}
func (h *defaultDLQHandler) ValidateDLQ() error {
return h.validateDLQInternal()
}
func (h *defaultDLQHandler) routeToDLQInternal(ctx context.Context, msg *nats.Msg, cause error) error {
// Basic DLQ routing implementation
payload := map[string]any{
"subject": msg.Subject,
"stream": h.streamName,
"consumer": h.consumerName,
"error": cause.Error(),
"received_at": time.Now().UTC(),
"body": string(msg.Data),
}
data, err := json.Marshal(payload)
if err != nil {
h.logger.Error("failed to marshal DLQ payload", zap.Error(err))
return err
}
_, err = h.js.Publish(h.dlqSubject, data)
if err != nil {
h.logger.Error("failed to publish to DLQ",
zap.String("dlq_subject", h.dlqSubject),
zap.Error(err),
)
h.telemetry.RecordDLQPublishFailure(ctx, h.streamName, h.consumerName)
return err
}
h.telemetry.RecordDLQMessage(ctx, h.streamName, h.consumerName)
return nil
}
func (h *defaultDLQHandler) validateDLQInternal() error {
if h.js == nil {
return fmt.Errorf("JetStream context is nil")
}
_, err := h.js.StreamNameBySubject(h.dlqSubject)
if err != nil {
return fmt.Errorf("DLQ subject %s not bound to any JetStream stream: %w", h.dlqSubject, err)
}
h.logger.Info("DLQ configuration validated",
zap.String("dlq_subject", h.dlqSubject),
)
return nil
return consumer, nil
}
// initCollaborators initializes the collaborator components
@@ -711,7 +143,6 @@ func (c *Consumer) initCollaborators() {
c.batchProcessor = &defaultBatchProcessor{
processor: c.processor,
dlqHandler: c.dlqHandler,
errorHandler: c.errorHandler,
logger: c.logger,
telemetry: c.telemetry,
streamName: c.config.streamName,
@@ -785,84 +216,6 @@ func normalizeConsumerConfig(cfg *config.Config) *consumerConfig {
}
}
// ProvideConsumer creates a NATS consumer with clean architecture.
func ProvideConsumer(
conn *nats.Conn,
js nats.JetStreamContext,
processor *app.MessageProcessor,
cfg *config.Config,
rec telemetry.Recorder,
logger *zap.Logger,
) (*Consumer, error) {
normCfg := normalizeConsumerConfig(cfg)
consumer := &Consumer{
conn: conn,
js: js,
processor: processor,
cfg: cfg,
logger: logger,
telemetry: rec,
config: *normCfg, // dereference the pointer
errorHandler: NewErrorHandler(logger),
}
consumer.initMetrics()
consumer.initCollaborators()
// Initialize managers
if consumer.config.mode == "jetstream" {
consumer.consumerManager = NewConsumerManager(js, normCfg.streamName, normCfg.consumerName, normCfg.subject, logger)
// Use StreamManager with full configuration
streamSubjects := []string{normCfg.subject}
if publisherSubject := strings.TrimSpace(cfg.Publisher.Topic); publisherSubject != "" {
streamSubjects = append(streamSubjects, publisherSubject)
}
streamSubjects = dedupeSubjects(streamSubjects)
consumer.streamManager = NewStreamManagerWithConfig(js, normCfg.streamName, streamSubjects, &cfg.NATS.StreamLimits, logger)
// Update fetcher with managers now that they're initialized
if fetcher, ok := consumer.fetcher.(*defaultMessageFetcher); ok {
fetcher.consumerManager = consumer.consumerManager
fetcher.streamManager = consumer.streamManager
}
// Create consumer if it doesn't exist
consumerConfig := consumer.buildConsumerConfig()
if err := consumer.consumerManager.EnsureConsumer(consumerConfig); err != nil {
return nil, fmt.Errorf("failed to ensure consumer: %w", err)
}
// Validate DLQ configuration early so misconfiguration is visible at startup
// rather than only when the first poison message appears.
if err := consumer.validateDLQ(); err != nil {
return nil, fmt.Errorf("DLQ validation failed: %w", err)
}
// Initialize advisory DLQ handler if DLQ is enabled
if normCfg.dlqSubject != "" && cfg.DLQ.Enabled {
advisoryHandler, err := NewAdvisoryDLQHandler(
js,
conn,
normCfg.streamName,
normCfg.consumerName,
normCfg.dlqSubject,
logger,
rec,
)
if err != nil {
return nil, fmt.Errorf("failed to create advisory DLQ handler: %w", err)
}
consumer.advisoryDLQHandler = advisoryHandler
}
} else {
logger.Info("Running consumer in core NATS mode",
zap.String("subject", normCfg.subject),
zap.String("queue_group", cfg.Subscription.QueueGroup),
)
}
return consumer, nil
}
// buildConsumerConfig builds the NATS consumer configuration
func (c *Consumer) buildConsumerConfig() *nats.ConsumerConfig {
return &nats.ConsumerConfig{
@@ -903,6 +256,144 @@ func (c *Consumer) ValidateDLQ() error {
return nil
}
// validateDLQ is a helper for internal use (lowercase)
func (c *Consumer) validateDLQ() error {
return c.ValidateDLQ()
}
// startCore starts the Core NATS consumer loop.
func (c *Consumer) startCore(ctx context.Context) error {
queueGroup := c.cfg.Subscription.QueueGroup
if queueGroup == "" {
queueGroup = c.config.consumerName
}
handler := func(msg *nats.Msg) {
if err := c.batchProcessor.ProcessMessage(ctx, msg); err != nil {
isPermanent := app.IsPermanent(err)
c.logger.Error("Failed to process message (core mode)",
zap.String("subject", msg.Subject),
zap.Error(err),
zap.Bool("permanent", isPermanent),
)
}
}
sub, err := c.conn.QueueSubscribe(c.config.subject, queueGroup, handler)
if err != nil {
return fmt.Errorf("failed to subscribe to %s: %w", c.config.subject, err)
}
if err := c.conn.Flush(); err != nil {
return fmt.Errorf("failed to flush NATS connection: %w", err)
}
c.logger.Info("Started core NATS subscription",
zap.String("subject", c.config.subject),
zap.String("queue_group", queueGroup),
)
<-ctx.Done()
c.logger.Info("Stopping core NATS consumer", zap.Error(ctx.Err()))
if err := sub.Drain(); err != nil && !errors.Is(err, nats.ErrConnectionClosed) {
return fmt.Errorf("failed to drain core subscription: %w", err)
}
return ctx.Err()
}
// createPullSubscription creates a pull subscription
func (c *Consumer) createPullSubscription() (*nats.Subscription, error) {
return c.consumerManager.CreatePullSubscription()
}
// startJetStream starts the JetStream consumer loop.
func (c *Consumer) startJetStream(ctx context.Context) error {
// Create pull subscription
sub, err := c.createPullSubscription()
if err != nil {
return err
}
// Use a closure that always cleans up the current subscription.
// When subscription is replaced in handleFetchError, this will clean up
// whatever currentSub points to at shutdown time.
var currentSub = sub
cleanupSubscriber := func() {
if currentSub != nil {
if err := currentSub.Unsubscribe(); err != nil {
c.logger.Error("Failed to unsubscribe subscription", zap.Error(err))
}
currentSub = nil
}
}
defer cleanupSubscriber()
c.logger.Info("Started consuming messages",
zap.String("subject", c.config.subject),
zap.String("consumer", c.config.consumerName),
zap.String("stream", c.config.streamName),
)
statsCtx, statsCancel := context.WithCancel(ctx)
defer statsCancel()
go c.emitConsumerStats(statsCtx)
var fetchErrorStreak int
for {
select {
case <-ctx.Done():
c.logger.Info("Stopping consumer", zap.Error(ctx.Err()))
return ctx.Err()
default:
}
// Fetch messages in batch
msgs, err := c.fetcher.FetchBatch(ctx, currentSub)
if err != nil {
// If context was cancelled, return immediately
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
c.logger.Info("Stopping consumer due to context cancellation", zap.Error(err))
return err
}
shouldContinue, handleErr := c.fetcher.HandleFetchError(ctx, err, &currentSub, &fetchErrorStreak)
if !shouldContinue {
return handleErr
}
continue
}
// Successful fetch -> reset error streak.
if fetchErrorStreak > 0 {
fetchErrorStreak = 0
}
// Process batch
c.batchProcessor.ProcessBatch(ctx, msgs)
}
}
// emitConsumerStats periodically emits basic consumer statistics.
func (c *Consumer) emitConsumerStats(ctx context.Context) {
ticker := time.NewTicker(c.config.monitorInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
info, err := c.js.ConsumerInfo(c.config.streamName, c.config.consumerName)
if err != nil {
continue
}
// Record pending messages for monitoring
obsmetrics.RecordNATSConsumerPending(c.config.streamName, c.config.consumerName, info.NumPending)
}
}
}
// Shutdown drains the underlying NATS connection gracefully.
func (c *Consumer) Shutdown(ctx context.Context) error {
if c.conn == nil {
internal/infra/nats/consumer_core.go
-52
View File
@@ -1,52 +0,0 @@
package nats
import (
"caatsm/internal/app"
"context"
"errors"
"fmt"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
)
// startCore starts the Core NATS consumer loop.
func (c *Consumer) startCore(ctx context.Context) error {
queueGroup := c.cfg.Subscription.QueueGroup
if queueGroup == "" {
queueGroup = c.config.consumerName
}
handler := func(msg *nats.Msg) {
if err := c.processMessage(ctx, msg); err != nil {
isPermanent := app.IsPermanent(err)
c.logger.Error("Failed to process message (core mode)",
zap.String("subject", msg.Subject),
zap.Error(err),
zap.Bool("permanent", isPermanent),
)
}
}
sub, err := c.conn.QueueSubscribe(c.config.subject, queueGroup, handler)
if err != nil {
return fmt.Errorf("failed to subscribe to %s: %w", c.config.subject, err)
}
if err := c.conn.Flush(); err != nil {
return fmt.Errorf("failed to flush NATS connection: %w", err)
}
c.logger.Info("Started core NATS subscription",
zap.String("subject", c.config.subject),
zap.String("queue_group", queueGroup),
)
<-ctx.Done()
c.logger.Info("Stopping core NATS consumer", zap.Error(ctx.Err()))
if err := sub.Drain(); err != nil && !errors.Is(err, nats.ErrConnectionClosed) {
return fmt.Errorf("failed to drain core subscription: %w", err)
}
return ctx.Err()
}
internal/infra/nats/consumer_error.go
-80
View File
@@ -1,80 +0,0 @@
package nats
import (
"caatsm/internal/app"
obsmetrics "caatsm/internal/infra/metrics"
"context"
"time"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
)
// handleMessageError handles errors that occur during message processing.
//
//nolint:unused // Reserved for potential future use or alternative implementation
func (c *Consumer) handleMessageError(ctx context.Context, msg *nats.Msg, err error, elapsed time.Duration) {
c.logger.Error("Failed to process message",
zap.String("subject", msg.Subject),
zap.Error(err),
zap.Bool("permanent", app.IsPermanent(err)),
)
result := obsmetrics.ResultFail
if app.IsPermanent(err) {
result = obsmetrics.ResultPermanentFail
}
c.telemetry.RecordMessageHandled(ctx, c.config.streamName, c.config.consumerName, result, elapsed)
processingResult := c.errorHandler.HandleProcessingError(c.consecutiveProcessErrors, err, c.logger, msg.Subject)
if processingResult.IsPermanent {
c.handlePermanentError(ctx, msg, err)
return
}
c.handleTransientError(ctx, msg, processingResult)
}
// handlePermanentError handles permanent/poison messages.
//
//nolint:unused // Reserved for potential future use or alternative implementation
func (c *Consumer) handlePermanentError(ctx context.Context, msg *nats.Msg, err error) {
c.consecutiveProcessErrors = 0
// Poison/permanent message: route to DLQ if configured, then ACK
if dlqErr := c.routeToDLQ(ctx, msg, err); dlqErr != nil {
c.logger.Error("Failed to route permanent-error message to DLQ", zap.Error(dlqErr))
}
if ackErr := msg.Ack(); ackErr != nil {
c.logger.Error("Failed to ACK permanent-error message", zap.Error(ackErr))
}
}
// handleTransientError handles transient errors with backpressure and redelivery.
//
//nolint:unused // Reserved for potential future use or alternative implementation
func (c *Consumer) handleTransientError(ctx context.Context, msg *nats.Msg, processingResult ProcessingErrorResult) {
// Increment error streak
if c.consecutiveProcessErrors < 0 {
c.consecutiveProcessErrors = 0
}
c.consecutiveProcessErrors++
if processingResult.ShouldApplyBackpressure {
c.logger.Warn("Applying backpressure due to consecutive processing errors",
zap.Int("consecutive_errors", c.consecutiveProcessErrors),
zap.Duration("sleep", processingResult.BackpressureDelay),
)
// Use context-aware sleep instead of blocking time.Sleep
if !sleepWithContext(ctx, processingResult.BackpressureDelay) {
// Context canceled, stop processing
return
}
}
// Transient error: request redelivery with optional delay
c.telemetry.RecordRetry(ctx, c.config.streamName, c.config.consumerName, obsmetrics.RetryReasonProcessorError)
if nakErr := c.nakWithStrategy(msg); nakErr != nil {
c.logger.Error("Failed to NAK message", zap.Error(nakErr))
}
}
internal/infra/nats/consumer_jetstream.go
-207
View File
@@ -1,207 +0,0 @@
package nats
import (
"context"
"errors"
"fmt"
"time"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
)
// recoverJetStreamResources attempts to recreate the stream and consumer in
// dev/test environments if they are missing. It is safe to call multiple times.
func (c *Consumer) recoverJetStreamResources() error {
if c.js == nil {
return fmt.Errorf("jetstream context is nil")
}
if c.cfg == nil {
return fmt.Errorf("config is nil")
}
consumerConfig := c.buildConsumerConfig()
return c.consumerManager.RecoverResources(c.streamManager, consumerConfig)
}
// createPullSubscriptionWithRecovery creates a pull subscription and, in
// dev/test environments, attempts to self-heal missing stream/consumer
// by recreating them once.
func (c *Consumer) createPullSubscriptionWithRecovery() (*nats.Subscription, error) {
consumerConfig := c.buildConsumerConfig()
return c.consumerManager.CreatePullSubscriptionWithRecovery(c.streamManager, consumerConfig)
}
//nolint:unused // Reserved for potential future use or alternative implementation
// nakWithStrategy sends a NAK with appropriate delay based on retry attempt.
func (c *Consumer) nakWithStrategy(msg *nats.Msg) error {
backoff := c.cfg.NATS.ConsumerRules.Backoff
if len(backoff) == 0 {
return msg.Nak()
}
meta, err := msg.Metadata()
if err != nil {
c.logger.Warn("Failed to read metadata for backoff strategy", zap.Error(err))
return msg.Nak()
}
attempt := int(meta.NumDelivered)
index := attempt - 1
if index < 0 {
index = 0
}
if index >= len(backoff) {
index = len(backoff) - 1
}
delay := backoff[index]
if delay <= 0 {
return msg.Nak()
}
return msg.NakWithDelay(delay)
}
// sleepWithContext sleeps for the specified duration, but returns early if the context is canceled.
// Returns true if the full duration was slept, false if the context was canceled.
func sleepWithContext(ctx context.Context, duration time.Duration) bool {
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-timer.C:
return true
case <-ctx.Done():
return false
}
}
//nolint:unused // Reserved for potential future use or alternative implementation
// fetchBatch fetches a batch of messages from the subscription.
// It respects context cancellation for faster shutdown.
func (c *Consumer) fetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error) {
// Check context before fetching
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
// Use a shorter timeout for better responsiveness to cancellation
// The batchTimeout is still used, but we'll check context more frequently
timeout := c.config.batchTimeout
if timeout > 500*time.Millisecond {
// Cap at 500ms to improve responsiveness while still allowing batching
timeout = 500 * time.Millisecond
}
return sub.Fetch(c.config.batchSize, nats.MaxWait(timeout))
}
// handleFetchError handles errors during message fetching, including recovery logic.
// Returns true if the error was handled and consumption should continue, false otherwise.
func (c *Consumer) handleFetchError(ctx context.Context, err error, sub **nats.Subscription, fetchErrorStreak *int) (bool, error) {
result := c.errorHandler.HandleFetchError(ctx, err, sub, fetchErrorStreak, c.config.streamName, c.config.consumerName, func() (*nats.Subscription, error) {
if recErr := c.recoverJetStreamResources(); recErr != nil {
return nil, recErr
}
if *sub != nil {
if unsubErr := (*sub).Unsubscribe(); unsubErr != nil {
c.logger.Error("Failed to unsubscribe during recovery", zap.Error(unsubErr))
}
}
return c.createPullSubscriptionWithRecovery()
})
if result.RecoveredSub != nil {
*sub = result.RecoveredSub
*fetchErrorStreak = 0
}
return result.ShouldContinue, result.Error
}
// startJetStream starts the JetStream consumer loop.
func (c *Consumer) startJetStream(ctx context.Context) error {
// Create pull subscription (with simple self-healing in dev/test).
sub, err := c.createPullSubscriptionWithRecovery()
if err != nil {
return err
}
// Use a closure that always cleans up the current subscription.
// When subscription is replaced in handleFetchError, this will clean up
// whatever currentSub points to at shutdown time.
var currentSub = sub
cleanupSubscriber := func() {
if currentSub != nil {
if err := currentSub.Unsubscribe(); err != nil {
c.logger.Error("Failed to unsubscribe subscription", zap.Error(err))
}
currentSub = nil
}
}
defer cleanupSubscriber()
c.logger.Info("Started consuming messages",
zap.String("subject", c.config.subject),
zap.String("consumer", c.config.consumerName),
zap.String("stream", c.config.streamName),
)
c.logger.Info("Consumer pull configuration",
zap.Int("batch_size", c.config.batchSize),
zap.Duration("batch_timeout", c.config.batchTimeout),
zap.Int("max_deliver", c.cfg.NATS.ConsumerRules.MaxDeliver),
zap.Duration("ack_wait", c.config.ackWait),
zap.String("deliver_policy", c.cfg.NATS.ConsumerRules.DeliverPolicy),
zap.String("replay_policy", c.cfg.NATS.ConsumerRules.ReplayPolicy),
zap.Int("backoff_steps", len(c.cfg.NATS.ConsumerRules.Backoff)),
)
statsCtx, statsCancel := context.WithCancel(ctx)
defer statsCancel()
go c.emitConsumerStats(statsCtx)
// Start advisory DLQ handler in background if configured
if c.advisoryDLQHandler != nil {
go func() {
if err := c.advisoryDLQHandler.Start(ctx); err != nil {
c.logger.Error("Advisory DLQ handler failed", zap.Error(err))
}
}()
}
var fetchErrorStreak int
for {
select {
case <-ctx.Done():
c.logger.Info("Stopping consumer", zap.Error(ctx.Err()))
return ctx.Err()
default:
}
// Fetch messages in batch
msgs, err := c.fetcher.FetchBatch(ctx, currentSub)
if err != nil {
// If context was cancelled, return immediately
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
c.logger.Info("Stopping consumer due to context cancellation", zap.Error(err))
return err
}
shouldContinue, handleErr := c.fetcher.HandleFetchError(ctx, err, &currentSub, &fetchErrorStreak)
if !shouldContinue {
return handleErr
}
continue
}
// Successful fetch -> reset error streak.
if fetchErrorStreak > 0 {
fetchErrorStreak = 0
}
// Process batch
c.batchProcessor.ProcessBatch(ctx, msgs)
}
}
internal/infra/nats/consumer_jetstream_test.go
-119
View File
@@ -1,119 +0,0 @@
package nats
import (
"context"
"errors"
"time"
configpkg "caatsm/internal/infra/config"
"github.com/nats-io/nats.go"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"go.uber.org/zap/zaptest"
)
var _ = Describe("Consumer JetStream", func() {
var (
c *Consumer
)
BeforeEach(func() {
logger := zaptest.NewLogger(GinkgoT())
c = &Consumer{
config: consumerConfig{
mode: "jetstream",
streamName: "TEST_STREAM",
consumerName: "test-consumer",
subject: "test.subject",
batchSize: 10,
batchTimeout: 2 * time.Second,
},
logger: logger,
errorHandler: NewErrorHandler(logger),
cfg: &configpkg.Config{
NATS: configpkg.NATSConfig{
ConsumerRules: configpkg.ConsumerRulesConfig{
Backoff: []time.Duration{5 * time.Second, 30 * time.Second},
},
},
},
}
})
Describe("nakWithStrategy", func() {
PIt("sends NAK without delay when backoff is empty", func() {
c.cfg.NATS.ConsumerRules.Backoff = []time.Duration{}
// Note: This test would require a real NATS message to fully test
// For now, we verify the logic path
})
PIt("sends NAK with delay based on delivery attempt", func() {
// Note: This test would require a real NATS message with metadata
// For now, we verify the function exists and can be called
})
})
Describe("handleFetchError", func() {
var ctx context.Context
BeforeEach(func() {
ctx = context.Background()
})
It("returns true for timeout errors", func() {
var sub *nats.Subscription
fetchErrorStreak := 0
shouldContinue, err := c.handleFetchError(ctx, nats.ErrTimeout, &sub, &fetchErrorStreak)
Expect(shouldContinue).To(BeTrue())
Expect(err).NotTo(HaveOccurred())
})
It("handles ErrNoResponders with backoff", func() {
var sub *nats.Subscription
fetchErrorStreak := 0
shouldContinue, err := c.handleFetchError(ctx, nats.ErrNoResponders, &sub, &fetchErrorStreak)
Expect(shouldContinue).To(BeTrue())
Expect(err).NotTo(HaveOccurred())
Expect(fetchErrorStreak).To(Equal(1))
})
It("handles resource not found errors", func() {
var sub *nats.Subscription
fetchErrorStreak := 0
resourceErr := errors.New("stream not found")
shouldContinue, err := c.handleFetchError(ctx, resourceErr, &sub, &fetchErrorStreak)
// Behavior depends on environment; in test this should attempt recovery
Expect(err).To(HaveOccurred())
Expect(shouldContinue).To(BeFalse())
})
It("handles generic errors with backoff", func() {
var sub *nats.Subscription
fetchErrorStreak := 0
genericErr := errors.New("generic error")
shouldContinue, err := c.handleFetchError(ctx, genericErr, &sub, &fetchErrorStreak)
Expect(shouldContinue).To(BeTrue())
Expect(err).NotTo(HaveOccurred())
Expect(fetchErrorStreak).To(Equal(1))
})
})
Describe("buildConsumerConfig", func() {
It("builds consumer config with correct defaults", func() {
config := c.buildConsumerConfig()
Expect(config.Durable).To(Equal("test-consumer"))
Expect(config.AckPolicy).To(Equal(nats.AckExplicitPolicy))
Expect(config.FilterSubject).To(Equal("test.subject"))
})
})
Describe("processSingleMessage", func() {
It("handles successful message processing", func() {
// This would require mocking the processor, but we can test the structure
// For now, we verify the method exists and can be called
// Note: This test would need a mock processor to fully work
Skip("Requires mock message processor")
})
})
})
internal/infra/nats/consumer_manager.go
+2 -43
View File
@@ -59,53 +59,12 @@ func (cm *ConsumerManager) EnsureConsumer(config *nats.ConsumerConfig) error {
return nil
}
// RecoverResources attempts to recreate the stream and consumer in dev/test environments
func (cm *ConsumerManager) RecoverResources(streamManager *StreamManager, consumerConfig *nats.ConsumerConfig) error {
if cm.js == nil {
return fmt.Errorf("jetstream context is nil")
}
// Ensure stream exists (dev/test may auto-create, prod will error).
if err := streamManager.EnsureStream(); err != nil {
return fmt.Errorf("ensure stream %s: %w", cm.streamName, err)
}
// Ensure durable consumer exists and is properly bound.
if err := cm.EnsureConsumer(consumerConfig); err != nil {
return fmt.Errorf("ensure consumer %s: %w", cm.consumerName, err)
}
return nil
}
// CreatePullSubscription creates a pull subscription with recovery logic
func (cm *ConsumerManager) CreatePullSubscription() (*nats.Subscription, error) {
return cm.js.PullSubscribe(cm.subject, cm.consumerName, nats.Bind(cm.streamName, cm.consumerName))
}
// CreatePullSubscriptionWithRecovery creates a pull subscription and attempts recovery if needed
// CreatePullSubscriptionWithRecovery creates a pull subscription
func (cm *ConsumerManager) CreatePullSubscriptionWithRecovery(streamManager *StreamManager, consumerConfig *nats.ConsumerConfig) (*nats.Subscription, error) {
sub, err := cm.CreatePullSubscription()
if err == nil {
return sub, nil
}
if isJetStreamResourceNotFound(err) && isDevLikeEnv() && shouldBootstrapStream() {
cm.logger.Warn("PullSubscribe failed due to missing JetStream resources; attempting to recreate",
zap.Error(err),
zap.String("stream", cm.streamName),
zap.String("consumer", cm.consumerName),
)
if recErr := cm.RecoverResources(streamManager, consumerConfig); recErr != nil {
return nil, fmt.Errorf("failed to recover JetStream resources: %w", recErr)
}
// Retry subscription after successful recovery.
sub, err = cm.CreatePullSubscription()
if err != nil {
return nil, fmt.Errorf("failed to create pull subscription after recovery: %w", err)
}
return sub, nil
}
return nil, fmt.Errorf("failed to create pull subscription: %w", err)
return cm.CreatePullSubscription()
}
internal/infra/nats/consumer_message.go
-138
View File
@@ -1,138 +0,0 @@
package nats
import (
"caatsm/internal/infra/log"
"context"
"fmt"
"time"
"github.com/google/uuid"
"github.com/nats-io/nats.go"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.uber.org/zap"
)
//nolint:unused // Reserved for potential future use or alternative implementation
// processBatch processes a batch of messages, handling errors and applying backpressure.
// It checks context cancellation between messages for faster shutdown.
func (c *Consumer) processBatch(ctx context.Context, msgs []*nats.Msg) {
for _, msg := range msgs {
// Check context before processing each message
select {
case <-ctx.Done():
c.logger.Info("Stopping batch processing due to cancellation",
zap.Int("remaining_messages", len(msgs)),
)
return
default:
}
c.processSingleMessage(ctx, msg)
}
}
//nolint:unused // Reserved for potential future use or alternative implementation
// processSingleMessage processes a single message with error handling and backpressure.
func (c *Consumer) processSingleMessage(ctx context.Context, msg *nats.Msg) {
start := time.Now()
if err := c.processMessage(ctx, msg); err != nil {
c.handleMessageError(ctx, msg, err, time.Since(start))
return
}
// Successful processing resets the error streak.
if c.consecutiveProcessErrors > 0 {
c.consecutiveProcessErrors = 0
}
// ACK the message
if ackErr := msg.Ack(); ackErr != nil {
c.logger.Error("Failed to ACK message", zap.Error(ackErr))
} else {
elapsed := time.Since(start)
c.telemetry.RecordMessageHandled(ctx, c.config.streamName, c.config.consumerName, "ok", elapsed)
}
}
// processMessage processes a single message.
func (c *Consumer) processMessage(ctx context.Context, msg *nats.Msg) error {
ctx, span := otel.Tracer("caatsm/nats").Start(ctx, "Consumer.processMessage")
defer span.End()
// Set semantic messaging attributes
span.SetAttributes(
attribute.String("messaging.system", "nats"),
attribute.String("messaging.operation.name", "receive"),
attribute.String("messaging.destination.name", msg.Subject),
attribute.String("messaging.consumer.group.name", c.config.consumerName),
attribute.String("caatsm.stream", c.config.streamName),
)
msgID, source, err := c.resolveMsgID(msg)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("unable to resolve message id: %w", err)
}
if source != "header" {
c.logger.Warn("Message missing NATS id header; using fallback",
zap.String("subject", msg.Subject),
zap.String("msg_id_source", source),
zap.String("msg_id", msgID),
)
}
// Attach structured logging context including stream/consumer and NATS metadata.
jsSeq := uint64(0)
if meta, metaErr := msg.Metadata(); metaErr == nil {
jsSeq = meta.Sequence.Stream
span.SetAttributes(
attribute.Int64("nats.js.stream_seq", int64(meta.Sequence.Stream)),
attribute.Int64("nats.js.consumer_seq", int64(meta.Sequence.Consumer)),
)
}
msgLogger := log.WithMessageContext(c.logger, log.MessageFields{
Service: "caatsm-consumer",
TransportMsgID: msgID,
Stream: c.config.streamName,
Consumer: c.config.consumerName,
Subject: msg.Subject,
JSSequence: jsSeq,
})
msgLogger.Debug("Processing message",
zap.Int("data_size", len(msg.Data)),
zap.String("msg_id_source", source),
)
// Call processor
if err := c.processor.Handle(ctx, msg.Data, msgID); err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("processor error: %w", err)
}
span.SetAttributes(attribute.String("telegram.msg_id", msgID))
return nil
}
// resolveMsgID extracts or generates a message ID.
func (c *Consumer) resolveMsgID(msg *nats.Msg) (string, string, error) {
if id := msg.Header.Get("Nats-Msg-Id"); id != "" {
return id, "header", nil
}
if c.config.mode == "core" {
return uuid.NewString(), "generated", nil
}
meta, err := msg.Metadata()
if err != nil {
return "", "", fmt.Errorf("fetch metadata: %w", err)
}
return fmt.Sprintf("js-%d", meta.Sequence.Stream), "metadata", nil
}
internal/infra/nats/consumer_metrics.go
-83
View File
@@ -1,83 +0,0 @@
package nats
import (
obsmetrics "caatsm/internal/infra/metrics"
"context"
"time"
"github.com/nats-io/nats.go"
"go.opentelemetry.io/otel"
"go.uber.org/zap"
)
// initMetrics initializes OpenTelemetry metrics.
func (c *Consumer) initMetrics() {
meter := otel.Meter("caatsm/nats")
c.meter = meter
if hist, err := meter.Int64Histogram("caatsm_nats_consumer_ack_pending"); err == nil {
c.ackPending = hist
}
if hist, err := meter.Int64Histogram("caatsm_nats_consumer_redelivered"); err == nil {
c.redelivered = hist
}
if hist, err := meter.Int64Histogram("caatsm_nats_consumer_pending"); err == nil {
c.pending = hist
}
if hist, err := meter.Int64Histogram("caatsm_nats_consumer_delivered"); err == nil {
c.delivered = hist
}
}
// recordConsumerMetrics records consumer metrics from ConsumerInfo.
func (c *Consumer) recordConsumerMetrics(ctx context.Context, info *nats.ConsumerInfo) {
if info == nil {
return
}
if c.ackPending != nil {
c.ackPending.Record(ctx, int64(info.NumAckPending))
}
if c.redelivered != nil {
c.redelivered.Record(ctx, int64(info.NumRedelivered))
}
if c.pending != nil {
c.pending.Record(ctx, int64(info.NumPending))
}
if c.delivered != nil {
c.delivered.Record(ctx, int64(info.Delivered.Stream))
}
// Export an explicit pending messages gauge for Prometheus-based lag /
// backlog alerts.
obsmetrics.RecordNATSConsumerPending(c.config.streamName, c.config.consumerName, info.NumPending)
}
// emitConsumerStats periodically emits consumer statistics.
func (c *Consumer) emitConsumerStats(ctx context.Context) {
ticker := time.NewTicker(c.config.monitorInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
info, err := c.js.ConsumerInfo(c.config.streamName, c.config.consumerName)
if err != nil {
c.logger.Warn("Failed to fetch consumer info", zap.Error(err))
continue
}
c.logger.Debug("JetStream consumer metrics",
zap.String("stream", c.config.streamName),
zap.String("consumer", c.config.consumerName),
zap.Uint64("num_ack_pending", uint64(info.NumAckPending)),
zap.Uint64("num_redelivered", uint64(info.NumRedelivered)),
zap.Uint64("num_pending", uint64(info.NumPending)),
zap.Uint64("delivered_consumer_seq", uint64(info.Delivered.Consumer)),
zap.Uint64("delivered_stream_seq", uint64(info.Delivered.Stream)),
)
c.recordConsumerMetrics(ctx, info)
}
}
}
internal/infra/nats/dlq.go
-139
View File
@@ -1,139 +0,0 @@
package nats
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
)
// validateDLQ verifies whether DLQ routing should be enabled and, if so, whether
// the configured DLQ subject is bound to a JetStream stream. Returns an error
// if DLQ is enabled but misconfigured, allowing the caller to fail fast.
func (c *Consumer) validateDLQ() error {
if c == nil {
return nil
}
// DLQ routing is only active in JetStream mode.
if c.config.mode != "jetstream" {
return nil
}
// If DLQ is not enabled in config, make sure we don't accidentally route to it.
if !c.cfg.DLQ.Enabled {
if strings.TrimSpace(c.config.dlqSubject) != "" {
c.logger.Info("DLQ subject configured but dlq.enabled is false; DLQ routing disabled",
zap.String("dlq_subject", c.config.dlqSubject),
)
}
c.config.dlqSubject = ""
return nil
}
subject := strings.TrimSpace(c.config.dlqSubject)
if subject == "" {
return fmt.Errorf("DLQ enabled but dlq.subject is empty")
}
if c.js == nil {
return fmt.Errorf("DLQ enabled but JetStream context is nil")
}
// Ensure the DLQ subject is actually bound to a JetStream stream. This avoids
// the opaque `nats: no response from stream` error later when publishing.
c.telemetry.RecordJSAPICall("dlq_validate_stream")
streamName, err := c.js.StreamNameBySubject(subject)
if err != nil || strings.TrimSpace(streamName) == "" {
return fmt.Errorf("DLQ subject %s not bound to any JetStream stream: %w", subject, err)
}
c.logger.Info("DLQ configuration validated",
zap.String("dlq_subject", subject),
zap.String("dlq_stream", streamName),
)
return nil
}
// routeToDLQ publishes a copy of the failed message to the configured DLQ subject,
// including useful metadata for offline analysis. If DLQ is not configured or the
// consumer is not running in JetStream mode, this is a no-op.
func (c *Consumer) routeToDLQ(ctx context.Context, msg *nats.Msg, cause error) error {
if c == nil || c.js == nil {
return nil
}
if c.config.mode != "jetstream" {
return nil
}
if strings.TrimSpace(c.config.dlqSubject) == "" {
return nil
}
meta, _ := msg.Metadata()
jsSeq := uint64(0)
deliveries := uint64(0)
if meta != nil {
jsSeq = meta.Sequence.Stream
deliveries = meta.NumDelivered
}
payload := map[string]any{
"transport_msg_id": msg.Header.Get("Nats-Msg-Id"),
"subject": msg.Subject,
"stream": c.config.streamName,
"consumer": c.config.consumerName,
"nats_sequence": jsSeq,
"deliveries": deliveries,
"error": fmt.Sprint(cause),
"received_at": time.Now().UTC(),
"body": string(msg.Data),
}
data, err := json.Marshal(payload)
if err != nil {
c.logger.Error("failed to marshal DLQ payload",
zap.String("stream", c.config.streamName),
zap.String("consumer", c.config.consumerName),
zap.String("dlq_subject", c.config.dlqSubject),
zap.Error(err),
)
return fmt.Errorf("marshal dlq payload: %w", err)
}
if _, err := c.js.Publish(c.config.dlqSubject, data); err != nil {
// nats.ErrNoResponders typically means that no JetStream stream is
// configured to receive this subject, or JetStream is temporarily
// unavailable. Surface this explicitly to make operational diagnosis
// easier.
if errors.Is(err, nats.ErrNoResponders) {
c.logger.Error("transient DLQ publish error (no responders)",
zap.String("stream", c.config.streamName),
zap.String("consumer", c.config.consumerName),
zap.String("dlq_subject", c.config.dlqSubject),
zap.Int("payload_size", len(data)),
zap.Error(err),
)
c.telemetry.RecordDLQPublishFailure(ctx, c.config.streamName, c.config.consumerName)
return fmt.Errorf("publish to dlq subject %s: no JetStream stream found for subject or JetStream unavailable: %w", c.config.dlqSubject, err)
}
c.logger.Error("failed to publish to DLQ",
zap.String("stream", c.config.streamName),
zap.String("consumer", c.config.consumerName),
zap.String("dlq_subject", c.config.dlqSubject),
zap.Int("payload_size", len(data)),
zap.Error(err),
)
c.telemetry.RecordDLQPublishFailure(ctx, c.config.streamName, c.config.consumerName)
return fmt.Errorf("publish to dlq subject %s: %w", c.config.dlqSubject, err)
}
c.telemetry.RecordDLQMessage(ctx, c.config.streamName, c.config.consumerName)
return nil
}
internal/infra/nats/dlq_handler.go
+86
View File
@@ -0,0 +1,86 @@
package nats
import (
"caatsm/internal/infra/telemetry"
"context"
"encoding/json"
"fmt"
"time"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
)
// DLQHandler defines the interface for dead letter queue operations
type DLQHandler interface {
RouteToDLQ(ctx context.Context, msg *nats.Msg, cause error) error
ValidateDLQ() error
}
// defaultDLQHandler implements DLQHandler interface
type defaultDLQHandler struct {
js nats.JetStreamContext
dlqSubject string
streamName string
consumerName string
logger *zap.Logger
telemetry telemetry.Recorder
}
func (h *defaultDLQHandler) RouteToDLQ(ctx context.Context, msg *nats.Msg, cause error) error {
return h.routeToDLQInternal(ctx, msg, cause)
}
func (h *defaultDLQHandler) ValidateDLQ() error {
return h.validateDLQInternal()
}
func (h *defaultDLQHandler) routeToDLQInternal(ctx context.Context, msg *nats.Msg, cause error) error {
// Basic DLQ routing implementation
payload := map[string]any{
"subject": msg.Subject,
"stream": h.streamName,
"consumer": h.consumerName,
"error": cause.Error(),
"received_at": time.Now().UTC(),
"body": string(msg.Data),
}
data, err := json.Marshal(payload)
if err != nil {
h.logger.Error("failed to marshal DLQ payload", zap.Error(err))
return err
}
pubCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
_, err = h.js.Publish(h.dlqSubject, data, nats.Context(pubCtx))
if err != nil {
h.logger.Error("failed to publish to DLQ",
zap.String("dlq_subject", h.dlqSubject),
zap.Error(err),
)
h.telemetry.RecordDLQPublishFailure(ctx, h.streamName, h.consumerName)
return err
}
h.telemetry.RecordDLQMessage(ctx, h.streamName, h.consumerName)
return nil
}
func (h *defaultDLQHandler) validateDLQInternal() error {
if h.js == nil {
return fmt.Errorf("JetStream context is nil")
}
_, err := h.js.StreamNameBySubject(h.dlqSubject)
if err != nil {
return fmt.Errorf("DLQ subject %s not bound to any JetStream stream: %w", h.dlqSubject, err)
}
h.logger.Info("DLQ configuration validated",
zap.String("dlq_subject", h.dlqSubject),
)
return nil
}
internal/infra/nats/dlq_test.go
+14 -118
View File
@@ -1,142 +1,38 @@
package nats
import (
"context"
"errors"
configpkg "caatsm/internal/infra/config"
"caatsm/internal/infra/telemetry"
"github.com/nats-io/nats.go"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"go.uber.org/zap"
"go.uber.org/zap/zaptest"
)
var _ = Describe("DLQ", func() {
var _ = Describe("DLQHandler", func() {
var (
logger *zap.Logger
handler *defaultDLQHandler
logger *zap.Logger
)
BeforeEach(func() {
logger = zaptest.NewLogger(GinkgoT())
handler = &defaultDLQHandler{
logger: logger,
streamName: "TEST_STREAM",
consumerName: "test-consumer",
dlqSubject: "caatsm.dlq",
telemetry: telemetry.NewNoop(),
}
})
Describe("validateDLQ", func() {
It("returns nil when consumer is nil", func() {
var c *Consumer
Expect(c.validateDLQ()).To(Succeed())
})
It("returns nil when mode is not jetstream", func() {
c := &Consumer{
config: consumerConfig{
mode: "core",
},
cfg: &configpkg.Config{
DLQ: configpkg.DLQConfig{
Enabled: true,
Subject: "caatsm.dlq",
},
},
logger: logger,
}
Expect(c.validateDLQ()).To(Succeed())
})
It("clears dlqSubject when DLQ is disabled", func() {
c := &Consumer{
config: consumerConfig{
mode: "jetstream",
dlqSubject: "caatsm.dlq",
},
cfg: &configpkg.Config{
DLQ: configpkg.DLQConfig{
Enabled: false,
Subject: "caatsm.dlq",
},
},
logger: logger,
}
Expect(c.validateDLQ()).To(Succeed())
Expect(c.config.dlqSubject).To(Equal(""))
})
It("returns error when DLQ is enabled but subject is empty", func() {
c := &Consumer{
config: consumerConfig{
mode: "jetstream",
dlqSubject: "",
},
cfg: &configpkg.Config{
DLQ: configpkg.DLQConfig{
Enabled: true,
Subject: "",
},
},
logger: logger,
}
err := c.validateDLQ()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("DLQ enabled but dlq.subject is empty"))
})
It("returns error when DLQ is enabled but JetStream context is nil", func() {
c := &Consumer{
config: consumerConfig{
mode: "jetstream",
dlqSubject: "caatsm.dlq",
},
js: nil,
cfg: &configpkg.Config{
DLQ: configpkg.DLQConfig{
Enabled: true,
Subject: "caatsm.dlq",
},
},
logger: logger,
telemetry: telemetry.NewNoop(),
}
err := c.validateDLQ()
Describe("ValidateDLQ", func() {
It("returns error when JetStream context is nil", func() {
handler.js = nil
err := handler.ValidateDLQ()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("JetStream context is nil"))
})
})
Describe("routeToDLQ", func() {
It("returns nil when consumer is nil", func() {
var c *Consumer
ctx := context.Background()
msg := &nats.Msg{}
err := errors.New("test error")
Expect(c.routeToDLQ(ctx, msg, err)).To(Succeed())
})
It("returns nil when mode is not jetstream", func() {
c := &Consumer{
config: consumerConfig{
mode: "core",
},
}
ctx := context.Background()
msg := &nats.Msg{}
err := errors.New("test error")
Expect(c.routeToDLQ(ctx, msg, err)).To(Succeed())
})
It("returns nil when dlqSubject is empty", func() {
c := &Consumer{
config: consumerConfig{
mode: "jetstream",
dlqSubject: "",
},
js: nil, // Can be nil for this test
}
ctx := context.Background()
msg := &nats.Msg{}
err := errors.New("test error")
Expect(c.routeToDLQ(ctx, msg, err)).To(Succeed())
})
})
})
internal/infra/nats/error_handler.go
-136
View File
@@ -1,136 +0,0 @@
package nats
import (
"caatsm/internal/app"
"context"
"errors"
"time"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
)
// ErrorHandler handles various error scenarios in NATS operations
type ErrorHandler struct {
logger *zap.Logger
}
// NewErrorHandler creates a new error handler
func NewErrorHandler(logger *zap.Logger) *ErrorHandler {
return &ErrorHandler{
logger: logger,
}
}
// FetchErrorResult represents the result of handling a fetch error
type FetchErrorResult struct {
ShouldContinue bool
RecoveredSub *nats.Subscription
Error error
}
// HandleFetchError handles errors during message fetching with recovery logic
func (h *ErrorHandler) HandleFetchError(
ctx context.Context,
err error,
sub **nats.Subscription,
fetchErrorStreak *int,
streamName, consumerName string,
recoverFunc func() (*nats.Subscription, error),
) FetchErrorResult {
if errors.Is(err, nats.ErrTimeout) {
// Timeout is expected when no messages are available.
return FetchErrorResult{ShouldContinue: true}
}
// JetStream API is currently unavailable (e.g., NATS just restarted or JetStream not ready).
if errors.Is(err, nats.ErrNoResponders) {
*fetchErrorStreak++
backoff := time.Duration(*fetchErrorStreak) * time.Second
backoff = min(backoff, 30*time.Second)
h.logger.Warn("JetStream not available, will retry with backoff",
zap.Error(err),
zap.String("stream", streamName),
zap.String("consumer", consumerName),
zap.Duration("backoff", backoff),
)
if !sleepWithContext(ctx, backoff) {
return FetchErrorResult{ShouldContinue: false, Error: ctx.Err()}
}
return FetchErrorResult{ShouldContinue: true}
}
// Underlying consumer/stream removed while app is running.
if isJetStreamResourceNotFound(err) {
if isDevLikeEnv() && shouldBootstrapStream() {
h.logger.Warn("JetStream consumer or stream missing; attempting to recreate",
zap.Error(err),
zap.String("stream", streamName),
zap.String("consumer", consumerName),
)
newSub, subErr := recoverFunc()
if subErr != nil {
return FetchErrorResult{ShouldContinue: false, Error: subErr}
}
*sub = newSub
*fetchErrorStreak = 0
return FetchErrorResult{ShouldContinue: true, RecoveredSub: newSub}
}
// Production: treat as configuration/operational error.
h.logger.Error("JetStream consumer or stream missing; not auto-recreating in this environment",
zap.Error(err),
zap.String("stream", streamName),
zap.String("consumer", consumerName),
)
return FetchErrorResult{ShouldContinue: false, Error: err}
}
// Generic error path with modest backoff.
*fetchErrorStreak++
backoff := time.Duration(*fetchErrorStreak) * time.Second
backoff = min(backoff, 10*time.Second)
h.logger.Error("Failed to fetch messages; backing off",
zap.Error(err),
zap.Duration("backoff", backoff),
)
if !sleepWithContext(ctx, backoff) {
return FetchErrorResult{ShouldContinue: false, Error: ctx.Err()}
}
return FetchErrorResult{ShouldContinue: true}
}
// ProcessingErrorResult represents the result of handling a processing error
type ProcessingErrorResult struct {
IsPermanent bool
ShouldApplyBackpressure bool
BackpressureDelay time.Duration
}
// HandleProcessingError analyzes processing errors and determines appropriate action
func (h *ErrorHandler) HandleProcessingError(
consecutiveErrors int,
err error,
logger *zap.Logger,
subject string,
) ProcessingErrorResult {
isPermanent := app.IsPermanent(err)
result := ProcessingErrorResult{
IsPermanent: isPermanent,
}
if isPermanent {
// Reset error streak for permanent errors
return result
}
// Transient error: increment error streak and apply simple backpressure if needed.
if consecutiveErrors >= 10 {
result.ShouldApplyBackpressure = true
result.BackpressureDelay = time.Duration(consecutiveErrors) * 100 * time.Millisecond
result.BackpressureDelay = min(result.BackpressureDelay, 5*time.Second)
}
return result
}
internal/infra/nats/error_handler_test.go
-102
View File
@@ -1,102 +0,0 @@
package nats
import (
"context"
"errors"
"time"
"github.com/nats-io/nats.go"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"go.uber.org/zap"
"go.uber.org/zap/zaptest"
)
var _ = Describe("ErrorHandler", func() {
var (
handler *ErrorHandler
logger *zap.Logger
)
BeforeEach(func() {
logger = zaptest.NewLogger(GinkgoT())
handler = NewErrorHandler(logger)
})
Describe("HandleProcessingError", func() {
It("identifies permanent errors correctly", func() {
// Mock a permanent error (this would be defined in the app package)
permanentErr := errors.New("permanent error")
// For testing, we'll assume any error is transient unless specified
result := handler.HandleProcessingError(0, permanentErr, logger, "test.subject")
Expect(result.IsPermanent).To(BeFalse()) // Since we can't easily mock app.IsPermanent
Expect(result.ShouldApplyBackpressure).To(BeFalse())
})
It("applies backpressure for consecutive errors", func() {
transientErr := errors.New("transient error")
result := handler.HandleProcessingError(10, transientErr, logger, "test.subject")
Expect(result.IsPermanent).To(BeFalse())
Expect(result.ShouldApplyBackpressure).To(BeTrue())
Expect(result.BackpressureDelay).To(BeNumerically(">=", 100*time.Millisecond))
})
})
Describe("HandleFetchError", func() {
var (
ctx context.Context
sub *nats.Subscription
fetchErrorStreak int
streamName string
consumerName string
)
BeforeEach(func() {
ctx = context.Background()
sub = nil
fetchErrorStreak = 0
streamName = "TEST_STREAM"
consumerName = "test-consumer"
})
It("handles timeout errors", func() {
result := handler.HandleFetchError(ctx, nats.ErrTimeout, &sub, &fetchErrorStreak, streamName, consumerName, nil)
Expect(result.ShouldContinue).To(BeTrue())
Expect(result.Error).NotTo(HaveOccurred())
})
It("handles no responders with backoff", func() {
result := handler.HandleFetchError(ctx, nats.ErrNoResponders, &sub, &fetchErrorStreak, streamName, consumerName, nil)
Expect(result.ShouldContinue).To(BeTrue())
Expect(result.Error).NotTo(HaveOccurred())
Expect(fetchErrorStreak).To(Equal(1))
})
It("handles resource not found errors in dev environment", func() {
// Mock resource not found error
resourceErr := errors.New("stream not found")
// Provide a no-op recovery function to avoid panic
recoveryFunc := func() (*nats.Subscription, error) {
return nil, errors.New("recovery not implemented in test")
}
result := handler.HandleFetchError(ctx, resourceErr, &sub, &fetchErrorStreak, streamName, consumerName, recoveryFunc)
// In test environment, this should attempt recovery but fail since recovery func returns error
Expect(result.ShouldContinue).To(BeFalse())
Expect(result.Error).To(HaveOccurred())
})
It("handles successful recovery", func() {
resourceErr := errors.New("consumer not found")
mockSub := &nats.Subscription{}
recoveryFunc := func() (*nats.Subscription, error) {
return mockSub, nil
}
result := handler.HandleFetchError(ctx, resourceErr, &sub, &fetchErrorStreak, streamName, consumerName, recoveryFunc)
Expect(result.ShouldContinue).To(BeTrue())
Expect(result.RecoveredSub).To(Equal(mockSub))
Expect(fetchErrorStreak).To(Equal(0)) // Should reset on successful recovery
})
})
})
internal/infra/nats/jetstream.go
+43 -133
View File
@@ -6,45 +6,63 @@ import (
"crypto/x509"
"fmt"
"os"
"strings"
"time"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
)
// ProvideNATSConn creates a reusable NATS connection with optional authentication.
// ProvideNATSConn creates a basic NATS connection.
func ProvideNATSConn(cfg *config.Config, logger *zap.Logger) (*nats.Conn, error) {
opts := []nats.Option{
nats.RetryOnFailedConnect(true),
nats.Timeout(cfg.Timeouts.Server),
nats.ReconnectWait(cfg.Timeouts.ReconnectWait),
// Use infinite reconnects so the app survives long NATS outages (e.g. docker compose down/up).
nats.MaxReconnects(-1),
nats.ReconnectWait(5 * time.Second),
nats.MaxReconnects(10),
nats.DisconnectErrHandler(func(nc *nats.Conn, err error) {
if err != nil {
logger.Warn("NATS disconnected", zap.Error(err))
}
logger.Warn("NATS disconnected", zap.Error(err))
}),
nats.ReconnectHandler(func(nc *nats.Conn) {
safeURL := sanitizeURLForLogging(nc.ConnectedUrl())
logger.Info("NATS reconnected", zap.String("url", safeURL))
logger.Info("NATS reconnected")
}),
}
// Apply authentication options
authOpts, err := buildAuthOptions(&cfg.NATS.Auth, logger)
if err != nil {
return nil, fmt.Errorf("failed to build auth options: %w", err)
// Simple token authentication if provided
if cfg.NATS.Auth.Token != "" {
opts = append(opts, nats.Token(cfg.NATS.Auth.Token))
}
// Basic TLS support if enabled
if cfg.NATS.Auth.TLSEnabled {
tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12}
// Load client certificate if provided
if cfg.NATS.Auth.TLSCertFile != "" && cfg.NATS.Auth.TLSKeyFile != "" {
cert, err := tls.LoadX509KeyPair(cfg.NATS.Auth.TLSCertFile, cfg.NATS.Auth.TLSKeyFile)
if err != nil {
return nil, fmt.Errorf("failed to load TLS certificate: %w", err)
}
tlsConfig.Certificates = []tls.Certificate{cert}
}
// Load CA certificate for server verification
if cfg.NATS.Auth.TLSCAFile != "" {
caCert, err := os.ReadFile(cfg.NATS.Auth.TLSCAFile)
if err != nil {
return nil, fmt.Errorf("failed to read CA certificate: %w", err)
}
caCertPool := x509.NewCertPool()
if !caCertPool.AppendCertsFromPEM(caCert) {
return nil, fmt.Errorf("failed to parse CA certificate")
}
tlsConfig.RootCAs = caCertPool
}
opts = append(opts, nats.Secure(tlsConfig))
}
opts = append(opts, authOpts...)
nc, err := nats.Connect(cfg.NATS.URL, opts...)
if err != nil {
safeURL := sanitizeURLForLogging(cfg.NATS.URL)
logger.Error("failed to connect to NATS",
zap.String("url", safeURL),
zap.Duration("timeout", cfg.Timeouts.Server),
zap.Duration("reconnect_wait", cfg.Timeouts.ReconnectWait),
zap.String("url", cfg.NATS.URL),
zap.Error(err),
)
return nil, fmt.Errorf("failed to connect to NATS: %w", err)
@@ -53,120 +71,12 @@ func ProvideNATSConn(cfg *config.Config, logger *zap.Logger) (*nats.Conn, error)
return nc, nil
}
// buildAuthOptions builds NATS connection options based on authentication configuration.
func buildAuthOptions(auth *config.NATSAuthConfig, logger *zap.Logger) ([]nats.Option, error) {
var opts []nats.Option
authMethods := 0
// Token authentication (highest priority)
if auth.Token != "" {
authMethods++
logger.Debug("Using NATS token authentication")
opts = append(opts, nats.Token(auth.Token))
}
// Credentials file authentication
if auth.CredentialsFile != "" {
authMethods++
if authMethods > 1 {
return nil, fmt.Errorf("multiple authentication methods specified: only one of token, credentials_file, or user/password can be used")
}
logger.Debug("Using NATS credentials file authentication", zap.String("file", auth.CredentialsFile))
opts = append(opts, nats.UserCredentials(auth.CredentialsFile))
}
// User/Password authentication
if auth.User != "" || auth.Password != "" {
authMethods++
if authMethods > 1 {
return nil, fmt.Errorf("multiple authentication methods specified: only one of token, credentials_file, or user/password can be used")
}
if auth.User == "" || auth.Password == "" {
return nil, fmt.Errorf("both user and password must be specified for user/password authentication")
}
logger.Debug("Using NATS user/password authentication", zap.String("user", auth.User))
opts = append(opts, nats.UserInfo(auth.User, auth.Password))
}
// TLS configuration
if auth.TLSEnabled {
tlsConfig := &tls.Config{
MinVersion: tls.VersionTLS12,
}
// Load client certificate and key if provided
if auth.TLSCertFile != "" && auth.TLSKeyFile != "" {
cert, err := tls.LoadX509KeyPair(auth.TLSCertFile, auth.TLSKeyFile)
if err != nil {
return nil, fmt.Errorf("failed to load TLS certificate: %w", err)
}
tlsConfig.Certificates = []tls.Certificate{cert}
logger.Debug("Loaded TLS client certificate", zap.String("cert", auth.TLSCertFile))
}
// Load CA certificate for server verification if provided
if auth.TLSCAFile != "" {
caCert, err := os.ReadFile(auth.TLSCAFile)
if err != nil {
return nil, fmt.Errorf("failed to read CA certificate file: %w", err)
}
caCertPool := x509.NewCertPool()
if !caCertPool.AppendCertsFromPEM(caCert) {
return nil, fmt.Errorf("failed to parse CA certificate from %s", auth.TLSCAFile)
}
tlsConfig.RootCAs = caCertPool
logger.Debug("Loaded TLS CA certificate", zap.String("ca_file", auth.TLSCAFile))
}
opts = append(opts, nats.Secure(tlsConfig))
logger.Debug("TLS enabled for NATS connection")
}
return opts, nil
}
// ProvideJetStream creates a NATS JetStream context using an existing connection.
// Returns nil, nil when cfg.NATS.Mode == "core" to support plain NATS servers without JetStream.
func ProvideJetStream(nc *nats.Conn, cfg *config.Config, logger *zap.Logger) (nats.JetStreamContext, error) {
mode := strings.ToLower(cfg.NATS.Mode)
if mode == "core" {
logger.Info("Skipping JetStream initialization for core NATS mode")
return nil, nil
}
// Get JetStream context
// ProvideJetStream creates a JetStream context from a NATS connection.
func ProvideJetStream(nc *nats.Conn, logger *zap.Logger) (nats.JetStreamContext, error) {
js, err := nc.JetStream()
if err != nil {
safeURL := sanitizeURLForLogging(cfg.NATS.URL)
logger.Error("failed to get JetStream context",
zap.String("url", safeURL),
zap.Error(err),
)
nc.Close()
return nil, fmt.Errorf("failed to get JetStream context: %w", err)
logger.Error("failed to create JetStream context", zap.Error(err))
return nil, fmt.Errorf("failed to create JetStream context: %w", err)
}
// Ensure the stream exists using StreamManager
streamName := cfg.NATS.Stream
consumerSubject := cfg.EffectiveSubscriptionTopic()
publisherSubject := strings.TrimSpace(cfg.Publisher.Topic)
streamSubjects := dedupeSubjects([]string{consumerSubject, publisherSubject})
if len(streamSubjects) == 0 {
logger.Error("no subjects configured for JetStream stream",
zap.String("stream", streamName),
zap.String("consumer_subject", consumerSubject),
zap.String("publisher_subject", publisherSubject),
)
nc.Close()
return nil, fmt.Errorf("no subjects configured for JetStream stream %s", streamName)
}
streamManager := NewStreamManagerWithConfig(js, streamName, streamSubjects, &cfg.NATS.StreamLimits, logger)
if err := streamManager.EnsureStream(); err != nil {
nc.Close()
return nil, err
}
return js, nil
}
internal/infra/nats/manager_test.go
-99
View File
@@ -1,99 +0,0 @@
package nats
import (
"github.com/nats-io/nats.go"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"go.uber.org/zap"
"go.uber.org/zap/zaptest"
)
var _ = Describe("ConsumerManager", func() {
var (
js nats.JetStreamContext
streamName string
consumerName string
subject string
logger *zap.Logger
consumerMgr *ConsumerManager
)
BeforeEach(func() {
// Note: These tests would need a real NATS server for full functionality
// For now, we'll test the structure and error handling
js = nil // Would be a mock in real tests
streamName = "TEST_STREAM"
consumerName = "test-consumer"
subject = "test.subject"
logger = zaptest.NewLogger(GinkgoT())
consumerMgr = NewConsumerManager(js, streamName, consumerName, subject, logger)
})
Describe("NewConsumerManager", func() {
It("creates a consumer manager with correct fields", func() {
Expect(consumerMgr.js).To(BeNil())
Expect(consumerMgr.streamName).To(Equal(streamName))
Expect(consumerMgr.consumerName).To(Equal(consumerName))
Expect(consumerMgr.subject).To(Equal(subject))
Expect(consumerMgr.logger).To(Equal(logger))
})
})
Describe("CreatePullSubscription", func() {
It("returns error when JetStream context is nil", func() {
// This will panic because js is nil, so we skip this test for now
Skip("Requires mock JetStream context")
})
})
Describe("RecoverResources", func() {
It("returns error when JetStream context is nil", func() {
streamMgr := NewStreamManager(nil, streamName, []string{subject}, logger)
consumerConfig := &nats.ConsumerConfig{Durable: consumerName}
err := consumerMgr.RecoverResources(streamMgr, consumerConfig)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("jetstream context is nil"))
})
})
})
var _ = Describe("StreamManager", func() {
var (
js nats.JetStreamContext
streamName string
subjects []string
logger *zap.Logger
streamMgr *StreamManager
)
BeforeEach(func() {
js = nil // Would be a mock in real tests
streamName = "TEST_STREAM"
subjects = []string{"test.subject"}
logger = zaptest.NewLogger(GinkgoT())
streamMgr = NewStreamManager(js, streamName, subjects, logger)
})
Describe("NewStreamManager", func() {
It("creates a stream manager with correct fields", func() {
Expect(streamMgr.js).To(BeNil())
Expect(streamMgr.streamName).To(Equal(streamName))
Expect(streamMgr.subjects).To(Equal(subjects))
Expect(streamMgr.logger).To(Equal(logger))
})
})
Describe("EnsureStream", func() {
It("returns error when JetStream context is nil", func() {
// This will panic because js is nil, so we skip this test for now
Skip("Requires mock JetStream context")
})
})
Describe("validateStreamConfig", func() {
It("handles nil stream info gracefully", func() {
streamMgr.validateStreamConfig(nil)
// Should not panic
})
})
})
internal/infra/nats/message_fetcher.go
+90
View File
@@ -0,0 +1,90 @@
package nats
import (
"caatsm/internal/infra/config"
"context"
"errors"
"time"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
)
// MessageFetcher defines the interface for fetching messages from NATS
type MessageFetcher interface {
FetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error)
HandleFetchError(ctx context.Context, err error, sub **nats.Subscription, fetchErrorStreak *int) (bool, error)
}
// defaultMessageFetcher implements MessageFetcher interface
type defaultMessageFetcher struct {
batchSize int
batchTimeout time.Duration
logger *zap.Logger
conn *nats.Conn
js nats.JetStreamContext
consumerManager *ConsumerManager
streamManager *StreamManager
config *consumerConfig
cfg *config.Config
}
func (f *defaultMessageFetcher) FetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error) {
return f.fetchBatch(ctx, sub)
}
// fetchBatch fetches a batch of messages from the subscription with context awareness
func (f *defaultMessageFetcher) fetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error) {
// Check context before fetching
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
// Use a shorter timeout for better responsiveness to cancellation
timeout := f.batchTimeout
if timeout > 500*time.Millisecond {
timeout = 500 * time.Millisecond
}
return sub.Fetch(f.batchSize, nats.MaxWait(timeout))
}
func (f *defaultMessageFetcher) HandleFetchError(ctx context.Context, err error, sub **nats.Subscription, fetchErrorStreak *int) (bool, error) {
// Context cancellation - stop processing
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
f.logger.Info("Fetch error due to context cancellation", zap.Error(err))
return false, err
}
// Timeout is normal - continue
if errors.Is(err, nats.ErrTimeout) {
return true, nil
}
// Connection issues - apply simple backoff
*fetchErrorStreak++
backoff := f.calculateExponentialBackoff(*fetchErrorStreak)
f.logger.Warn("Fetch error, applying backoff",
zap.Error(err),
zap.Int("error_streak", *fetchErrorStreak),
zap.Duration("backoff", backoff),
)
if !sleepWithContext(ctx, backoff) {
return false, ctx.Err()
}
return true, nil
}
// calculateExponentialBackoff calculates exponential backoff duration with a cap
func (f *defaultMessageFetcher) calculateExponentialBackoff(streak int) time.Duration {
if streak <= 0 {
return 0
}
// Simple exponential backoff: 2^(streak-1) seconds, capped at 30 seconds
backoff := time.Duration(1<<uint(min(streak-1, 5))) * time.Second
return min(backoff, 30*time.Second)
}
internal/infra/nats/message_fetcher_test.go
+89
View File
@@ -0,0 +1,89 @@
package nats
import (
"context"
"errors"
"time"
configpkg "caatsm/internal/infra/config"
"github.com/nats-io/nats.go"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"go.uber.org/zap"
"go.uber.org/zap/zaptest"
)
var _ = Describe("MessageFetcher", func() {
var (
fetcher *defaultMessageFetcher
logger *zap.Logger
)
BeforeEach(func() {
logger = zaptest.NewLogger(GinkgoT())
fetcher = &defaultMessageFetcher{
logger: logger,
config: &consumerConfig{
streamName: "TEST_STREAM",
consumerName: "test-consumer",
},
cfg: &configpkg.Config{
NATS: configpkg.NATSConfig{
ConsumerRules: configpkg.ConsumerRulesConfig{
Backoff: []time.Duration{5 * time.Second, 30 * time.Second},
},
},
},
}
})
Describe("HandleFetchError", func() {
var ctx context.Context
BeforeEach(func() {
ctx = context.Background()
})
It("returns true for timeout errors", func() {
var sub *nats.Subscription
fetchErrorStreak := 0
shouldContinue, err := fetcher.HandleFetchError(ctx, nats.ErrTimeout, &sub, &fetchErrorStreak)
Expect(shouldContinue).To(BeTrue())
Expect(err).NotTo(HaveOccurred())
})
It("handles ErrNoResponders with backoff", func() {
var sub *nats.Subscription
fetchErrorStreak := 0
shouldContinue, err := fetcher.HandleFetchError(ctx, nats.ErrNoResponders, &sub, &fetchErrorStreak)
Expect(shouldContinue).To(BeTrue())
Expect(err).NotTo(HaveOccurred())
Expect(fetchErrorStreak).To(Equal(1))
})
It("handles resource not found errors", func() {
var sub *nats.Subscription
fetchErrorStreak := 0
resourceErr := errors.New("stream not found")
shouldContinue, err := fetcher.HandleFetchError(ctx, resourceErr, &sub, &fetchErrorStreak)
// Simplified error handling just applies backoff and continues
Expect(err).NotTo(HaveOccurred())
Expect(shouldContinue).To(BeTrue())
Expect(fetchErrorStreak).To(Equal(1))
})
It("handles generic errors with backoff", func() {
// We need a non-nil subscription to avoid recovery attempt
dummySub := &nats.Subscription{}
sub := dummySub
fetchErrorStreak := 0
genericErr := errors.New("generic error")
shouldContinue, err := fetcher.HandleFetchError(ctx, genericErr, &sub, &fetchErrorStreak)
Expect(shouldContinue).To(BeTrue())
Expect(err).NotTo(HaveOccurred())
Expect(fetchErrorStreak).To(Equal(1))
})
})
})
internal/infra/nats/message_handler_test.go
+11 -13
View File
@@ -9,17 +9,15 @@ import (
var _ = Describe("MessageHandler", func() {
var (
c *Consumer
processor *defaultBatchProcessor
)
BeforeEach(func() {
c = &Consumer{
config: consumerConfig{
mode: "jetstream",
streamName: "TEST_STREAM",
consumerName: "test-consumer",
},
logger: zaptest.NewLogger(GinkgoT()),
processor = &defaultBatchProcessor{
logger: zaptest.NewLogger(GinkgoT()),
streamName: "TEST_STREAM",
consumerName: "test-consumer",
mode: "jetstream",
}
})
@@ -30,32 +28,32 @@ var _ = Describe("MessageHandler", func() {
}
msg.Header.Set("Nats-Msg-Id", "msg-123")
id, source, err := c.resolveMsgID(msg)
id, source, err := processor.resolveMsgID(msg)
Expect(err).NotTo(HaveOccurred())
Expect(id).To(Equal("msg-123"))
Expect(source).To(Equal("header"))
})
It("generates UUID for core mode when header is missing", func() {
c.config.mode = "core"
processor.mode = "core"
msg := &nats.Msg{
Header: nats.Header{},
}
id, source, err := c.resolveMsgID(msg)
id, source, err := processor.resolveMsgID(msg)
Expect(err).NotTo(HaveOccurred())
Expect(id).NotTo(BeEmpty())
Expect(source).To(Equal("generated"))
})
It("returns error for JetStream mode when header and metadata are missing", func() {
c.config.mode = "jetstream"
processor.mode = "jetstream"
msg := &nats.Msg{
Header: nats.Header{},
}
// Without metadata, this should return an error
_, _, err := c.resolveMsgID(msg)
_, _, err := processor.resolveMsgID(msg)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("fetch metadata"))
})
internal/infra/nats/message_processor.go
+279
View File
@@ -0,0 +1,279 @@
package nats
import (
"caatsm/internal/app"
"caatsm/internal/infra/log"
obsmetrics "caatsm/internal/infra/metrics"
"caatsm/internal/infra/telemetry"
"context"
"fmt"
"time"
"github.com/google/uuid"
"github.com/nats-io/nats.go"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.uber.org/zap"
)
// MessageProcessor defines the interface for processing message batches
type MessageProcessor interface {
ProcessBatch(ctx context.Context, msgs []*nats.Msg)
ProcessMessage(ctx context.Context, msg *nats.Msg) error
}
// ProcessingErrorResult represents the result of handling a processing error
type ProcessingErrorResult struct {
IsPermanent bool
ShouldApplyBackpressure bool
BackpressureDelay time.Duration
}
// defaultBatchProcessor implements MessageProcessor interface
type defaultBatchProcessor struct {
processor *app.MessageProcessor
dlqHandler DLQHandler
logger *zap.Logger
telemetry telemetry.Recorder
// Configuration needed for processing
streamName string
consumerName string
mode string
backoff []time.Duration
// Pointer to consecutive errors counter (shared with Consumer)
consecutiveProcessErrors *int
}
func (p *defaultBatchProcessor) ProcessBatch(ctx context.Context, msgs []*nats.Msg) {
for _, msg := range msgs {
// Check context before processing each message
select {
case <-ctx.Done():
p.logger.Info("Stopping batch processing due to cancellation",
zap.Int("remaining_messages", len(msgs)),
)
return
default:
}
p.processSingleMessage(ctx, msg)
}
}
// processSingleMessage processes a single message with error handling and backpressure.
func (p *defaultBatchProcessor) processSingleMessage(ctx context.Context, msg *nats.Msg) {
start := time.Now()
if err := p.ProcessMessage(ctx, msg); err != nil {
p.handleMessageError(ctx, msg, err, time.Since(start))
return
}
// Successful processing resets the error streak.
if p.consecutiveProcessErrors != nil && *p.consecutiveProcessErrors > 0 {
*p.consecutiveProcessErrors = 0
}
// ACK the message
if ackErr := msg.Ack(); ackErr != nil {
p.logger.Error("Failed to ACK message", zap.Error(ackErr))
} else {
elapsed := time.Since(start)
p.telemetry.RecordMessageHandled(ctx, p.streamName, p.consumerName, "ok", elapsed)
}
}
// ProcessMessage processes a single message.
func (p *defaultBatchProcessor) ProcessMessage(ctx context.Context, msg *nats.Msg) error {
ctx, span := otel.Tracer("caatsm/nats").Start(ctx, "Consumer.processMessage")
defer span.End()
// Set semantic messaging attributes
span.SetAttributes(
attribute.String("messaging.system", "nats"),
attribute.String("messaging.operation.name", "receive"),
attribute.String("messaging.destination.name", msg.Subject),
attribute.String("messaging.consumer.group.name", p.consumerName),
attribute.String("caatsm.stream", p.streamName),
)
msgID, source, err := p.resolveMsgID(msg)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("unable to resolve message id: %w", err)
}
if source != "header" {
p.logger.Warn("Message missing NATS id header; using fallback",
zap.String("subject", msg.Subject),
zap.String("msg_id_source", source),
zap.String("msg_id", msgID),
)
}
// Attach structured logging context including stream/consumer and NATS metadata.
jsSeq := uint64(0)
if meta, metaErr := msg.Metadata(); metaErr == nil {
jsSeq = meta.Sequence.Stream
span.SetAttributes(
attribute.Int64("nats.js.stream_seq", int64(meta.Sequence.Stream)),
attribute.Int64("nats.js.consumer_seq", int64(meta.Sequence.Consumer)),
)
}
msgLogger := log.WithMessageContext(p.logger, log.MessageFields{
Service: "caatsm-consumer",
TransportMsgID: msgID,
Stream: p.streamName,
Consumer: p.consumerName,
Subject: msg.Subject,
JSSequence: jsSeq,
})
msgLogger.Debug("Processing message",
zap.Int("data_size", len(msg.Data)),
zap.String("msg_id_source", source),
)
// Call processor
if err := p.processor.Handle(ctx, msg.Data, msgID); err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("processor error: %w", err)
}
span.SetAttributes(attribute.String("telegram.msg_id", msgID))
return nil
}
// resolveMsgID extracts or generates a message ID.
func (p *defaultBatchProcessor) resolveMsgID(msg *nats.Msg) (string, string, error) {
if id := msg.Header.Get("Nats-Msg-Id"); id != "" {
return id, "header", nil
}
if p.mode == "core" {
return uuid.NewString(), "generated", nil
}
meta, err := msg.Metadata()
if err != nil {
return "", "", fmt.Errorf("fetch metadata: %w", err)
}
return fmt.Sprintf("js-%d", meta.Sequence.Stream), "metadata", nil
}
// handleMessageError handles errors that occur during message processing.
func (p *defaultBatchProcessor) handleMessageError(ctx context.Context, msg *nats.Msg, err error, elapsed time.Duration) {
p.logger.Error("Failed to process message",
zap.String("subject", msg.Subject),
zap.Error(err),
zap.Bool("permanent", app.IsPermanent(err)),
)
result := obsmetrics.ResultFail
if app.IsPermanent(err) {
result = obsmetrics.ResultPermanentFail
}
p.telemetry.RecordMessageHandled(ctx, p.streamName, p.consumerName, result, elapsed)
consecutiveErrors := 0
if p.consecutiveProcessErrors != nil {
consecutiveErrors = *p.consecutiveProcessErrors
}
isPermanent := app.IsPermanent(err)
processingResult := ProcessingErrorResult{IsPermanent: isPermanent}
if !isPermanent && consecutiveErrors >= 10 {
processingResult.ShouldApplyBackpressure = true
processingResult.BackpressureDelay = time.Duration(consecutiveErrors) * 100 * time.Millisecond
if processingResult.BackpressureDelay > 5*time.Second {
processingResult.BackpressureDelay = 5 * time.Second
}
}
if processingResult.IsPermanent {
p.handlePermanentError(ctx, msg, err)
return
}
p.handleTransientError(ctx, msg, processingResult)
}
// handlePermanentError handles permanent/poison messages.
func (p *defaultBatchProcessor) handlePermanentError(ctx context.Context, msg *nats.Msg, err error) {
if p.consecutiveProcessErrors != nil {
*p.consecutiveProcessErrors = 0
}
// Poison/permanent message: route to DLQ if configured, then ACK
if p.dlqHandler != nil {
if dlqErr := p.dlqHandler.RouteToDLQ(ctx, msg, err); dlqErr != nil {
p.logger.Error("Failed to route permanent-error message to DLQ", zap.Error(dlqErr))
}
}
if ackErr := msg.Ack(); ackErr != nil {
p.logger.Error("Failed to ACK permanent-error message", zap.Error(ackErr))
}
}
// handleTransientError handles transient errors with backpressure and redelivery.
func (p *defaultBatchProcessor) handleTransientError(ctx context.Context, msg *nats.Msg, processingResult ProcessingErrorResult) {
// Increment error streak
if p.consecutiveProcessErrors != nil {
if *p.consecutiveProcessErrors < 0 {
*p.consecutiveProcessErrors = 0
}
*p.consecutiveProcessErrors++
}
if processingResult.ShouldApplyBackpressure {
consecutiveErrors := 0
if p.consecutiveProcessErrors != nil {
consecutiveErrors = *p.consecutiveProcessErrors
}
p.logger.Warn("Applying backpressure due to consecutive processing errors",
zap.Int("consecutive_errors", consecutiveErrors),
zap.Duration("sleep", processingResult.BackpressureDelay),
)
// Use context-aware sleep instead of blocking time.Sleep
if !sleepWithContext(ctx, processingResult.BackpressureDelay) {
// Context canceled, stop processing
return
}
}
// Transient error: request redelivery with optional delay
p.telemetry.RecordRetry(ctx, p.streamName, p.consumerName, obsmetrics.RetryReasonProcessorError)
if nakErr := p.nakWithStrategy(msg); nakErr != nil {
p.logger.Error("Failed to NAK message", zap.Error(nakErr))
}
}
// nakWithStrategy sends a NAK with appropriate delay based on retry attempt.
func (p *defaultBatchProcessor) nakWithStrategy(msg *nats.Msg) error {
if len(p.backoff) == 0 {
return msg.Nak()
}
meta, err := msg.Metadata()
if err != nil {
p.logger.Warn("Failed to read metadata for backoff strategy", zap.Error(err))
return msg.Nak()
}
attempt := int(meta.NumDelivered)
index := attempt - 1
if index < 0 {
index = 0
}
if index >= len(p.backoff) {
index = len(p.backoff) - 1
}
delay := p.backoff[index]
if delay <= 0 {
return msg.Nak()
}
return msg.NakWithDelay(delay)
}
internal/infra/nats/metrics_test.go
+10 -31
View File
@@ -2,10 +2,9 @@ package nats
import (
"context"
"time"
"github.com/nats-io/nats.go"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"go.uber.org/zap/zaptest"
)
@@ -19,40 +18,20 @@ var _ = Describe("Metrics", func() {
ctx = context.Background()
c = &Consumer{
config: consumerConfig{
streamName: "TEST_STREAM",
consumerName: "test-consumer",
streamName: "TEST_STREAM",
consumerName: "test-consumer",
monitorInterval: 30 * time.Second, // Set a valid interval
},
logger: zaptest.NewLogger(GinkgoT()),
}
})
Describe("initMetrics", func() {
It("initializes metrics without error", func() {
c.initMetrics()
Expect(c.meter).NotTo(BeNil())
})
})
Describe("recordConsumerMetrics", func() {
It("handles nil ConsumerInfo gracefully", func() {
c.initMetrics()
c.recordConsumerMetrics(ctx, nil)
// Should not panic
})
It("records metrics when ConsumerInfo is provided", func() {
c.initMetrics()
info := &nats.ConsumerInfo{
Config: nats.ConsumerConfig{},
Delivered: nats.SequenceInfo{
Consumer: 50,
Stream: 100,
},
}
// Set fields directly (they are exported)
// Note: ConsumerInfo fields may not all be exported, so we test what we can
c.recordConsumerMetrics(ctx, info)
// Should not panic
Describe("emitConsumerStats", func() {
It("handles context cancellation", func() {
ctx, cancel := context.WithCancel(ctx)
cancel()
c.emitConsumerStats(ctx)
// Should return without panic
})
})
})
internal/infra/nats/stream_manager.go
+6 -107
View File
@@ -1,10 +1,7 @@
package nats
import (
"caatsm/internal/infra/config"
"errors"
"fmt"
"strings"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
@@ -16,7 +13,6 @@ type StreamManager struct {
streamName string
subjects []string
logger *zap.Logger
cfg *config.StreamLimitsConfig
}
// NewStreamManager creates a new stream manager
@@ -29,113 +25,16 @@ func NewStreamManager(js nats.JetStreamContext, streamName string, subjects []st
}
}
// NewStreamManagerWithConfig creates a new stream manager with full stream configuration
func NewStreamManagerWithConfig(js nats.JetStreamContext, streamName string, subjects []string, streamLimits *config.StreamLimitsConfig, logger *zap.Logger) *StreamManager {
return &StreamManager{
js: js,
streamName: streamName,
subjects: subjects,
logger: logger,
cfg: streamLimits,
}
}
// EnsureStream ensures that the configured JetStream stream exists
func (sm *StreamManager) EnsureStream() error {
// Build stream configuration
streamConfig := sm.buildStreamConfig()
info, err := sm.js.StreamInfo(sm.streamName)
_, err := sm.js.StreamInfo(sm.streamName)
if err != nil {
if errors.Is(err, nats.ErrStreamNotFound) {
if shouldBootstrapStream() {
if _, err = sm.js.AddStream(streamConfig); err != nil {
sm.logger.Error("failed to create stream",
zap.String("stream", sm.streamName),
zap.Strings("subjects", sm.subjects),
zap.Error(err),
)
return fmt.Errorf("failed to create stream %s: %w", sm.streamName, err)
}
sm.logger.Info("Created JetStream stream",
zap.String("stream", sm.streamName),
zap.Strings("subjects", sm.subjects),
)
return nil
}
sm.logger.Error("stream not found and auto-creation disabled",
zap.String("stream", sm.streamName),
zap.Strings("expected_subjects", sm.subjects),
)
return fmt.Errorf("stream %s not found and auto-creation disabled", sm.streamName)
}
sm.logger.Error("failed to fetch stream info",
zap.String("stream", sm.streamName),
zap.Error(err),
)
return fmt.Errorf("failed to fetch stream info for %s: %w", sm.streamName, err)
return fmt.Errorf("stream %s not found or inaccessible: %w", sm.streamName, err)
}
// Stream exists: validate subjects but do not fail hard if they differ.
sm.validateStreamConfig(info)
sm.logger.Info("JetStream stream verified",
zap.String("stream", sm.streamName),
zap.Strings("subjects", sm.subjects),
)
return nil
}
// buildStreamConfig builds the stream configuration from manager settings
func (sm *StreamManager) buildStreamConfig() *nats.StreamConfig {
config := &nats.StreamConfig{
Name: sm.streamName,
Subjects: sm.subjects,
Retention: nats.LimitsPolicy,
Storage: nats.FileStorage,
}
// Apply stream limits configuration if provided
if sm.cfg != nil {
config.MaxMsgs = sm.cfg.MaxMsgs
config.MaxBytes = sm.cfg.MaxBytes
config.MaxAge = sm.cfg.MaxAge
config.Replicas = sm.cfg.Replicas
// Map storage type
switch strings.ToLower(sm.cfg.Storage) {
case "memory":
config.Storage = nats.MemoryStorage
case "file":
config.Storage = nats.FileStorage
}
// Map discard policy
if strings.EqualFold(sm.cfg.Discard, "new") {
config.Discard = nats.DiscardNew
} else {
config.Discard = nats.DiscardOld
}
}
return config
}
// validateStreamConfig validates the stream configuration
func (sm *StreamManager) validateStreamConfig(info *nats.StreamInfo) {
if info == nil {
return
}
missing := make([]string, 0)
for _, subj := range sm.subjects {
if subj == "" {
continue
}
if !containsSubject(info.Config.Subjects, subj) {
missing = append(missing, subj)
}
}
if len(missing) > 0 {
sm.logger.Warn("JetStream stream subjects missing expected entries",
zap.String("stream", info.Config.Name),
zap.Strings("stream_subjects", info.Config.Subjects),
zap.Strings("missing_subjects", missing),
)
}
}
internal/infra/nats/utils.go
+10 -26
View File
@@ -1,20 +1,24 @@
package nats
import (
"context"
"errors"
"net/url"
"os"
"strings"
"time"
"github.com/nats-io/nats.go"
)
// isDevLikeEnv checks if the current environment is development-like.
func isDevLikeEnv() bool {
switch strings.ToLower(os.Getenv("GO_ENV")) {
case "", "dev", "development", "test", "testing":
// sleepWithContext sleeps for the specified duration, but returns early if the context is canceled.
// Returns true if the full duration was slept, false if the context was canceled.
func sleepWithContext(ctx context.Context, duration time.Duration) bool {
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-timer.C:
return true
default:
case <-ctx.Done():
return false
}
}
@@ -79,26 +83,6 @@ func mapReplayPolicy(value string) nats.ReplayPolicy {
}
}
// shouldBootstrapStream checks if streams should be auto-created based on environment.
func shouldBootstrapStream() bool {
switch strings.ToLower(os.Getenv("GO_ENV")) {
case "", "dev", "development", "test", "testing":
return true
default:
return false
}
}
// containsSubject checks if a subject exists in a list of subjects.
func containsSubject(subjects []string, target string) bool {
for _, s := range subjects {
if s == target {
return true
}
}
return false
}
// dedupeSubjects removes duplicate and empty subjects from a list.
func dedupeSubjects(subjects []string) []string {
seen := make(map[string]struct{})
internal/infra/nats/utils_test.go
-77
View File
@@ -1,88 +1,11 @@
package nats
import (
"os"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("Utils", func() {
Describe("isDevLikeEnv", func() {
BeforeEach(func() {
// Save original value
originalEnv := os.Getenv("GO_ENV")
DeferCleanup(func() {
if originalEnv == "" {
if err := os.Unsetenv("GO_ENV"); err != nil {
// Environment variables are optional, ignore cleanup errors in tests
_ = err
}
} else {
if err := os.Setenv("GO_ENV", originalEnv); err != nil {
// Environment variables are optional, ignore cleanup errors in tests
_ = err
}
}
})
})
It("returns true for dev environment", func() {
if err := os.Setenv("GO_ENV", "dev"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeTrue())
})
It("returns true for development environment", func() {
if err := os.Setenv("GO_ENV", "development"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeTrue())
})
It("returns true for test environment", func() {
if err := os.Setenv("GO_ENV", "test"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeTrue())
})
It("returns true for testing environment", func() {
if err := os.Setenv("GO_ENV", "testing"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeTrue())
})
It("returns true for empty environment", func() {
if err := os.Unsetenv("GO_ENV"); err != nil {
// Environment variables are optional, ignore cleanup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeTrue())
})
It("returns false for production environment", func() {
if err := os.Setenv("GO_ENV", "prod"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeFalse())
})
It("returns false for production environment (uppercase)", func() {
if err := os.Setenv("GO_ENV", "PROD"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeFalse())
})
})
Describe("sanitizeURLForLogging", func() {
It("removes credentials from URLs", func() {