✨ Enhance observability and error handling in NATS integration. Introduce comprehensive OpenTelemetry support with environment-based sampling and semantic attributes for tracing and metrics. Implement an advisory dead-letter queue (DLQ) handler for managing message delivery failures. Update NATS consumer to utilize structured logging and improve error handling strategies. Refactor configuration files for OpenTelemetry collector in both development and production environments, ensuring robust telemetry integration. Enhance documentation to reflect new features and best practices for observability.
This commit is contained in:
@@ -1,8 +1,8 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"caatsm/internal/adapter/parser"
|
||||
"caatsm/internal/adapter/dto"
|
||||
"caatsm/internal/adapter/parser"
|
||||
"caatsm/internal/infra/log"
|
||||
"caatsm/internal/infra/telemetry"
|
||||
"caatsm/internal/port"
|
||||
@@ -64,7 +64,14 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
|
||||
tracer := otel.Tracer("caatsm/app")
|
||||
ctx, span := tracer.Start(ctx, "MessageProcessor.Handle")
|
||||
defer span.End()
|
||||
span.SetAttributes(attribute.String("nats.msg_id", msgID))
|
||||
|
||||
// Set semantic attributes following OpenTelemetry conventions
|
||||
span.SetAttributes(
|
||||
attribute.String("messaging.system", "nats"),
|
||||
attribute.String("messaging.operation", "receive"),
|
||||
attribute.String("messaging.message_id", msgID),
|
||||
attribute.String("caatsm.component", "processor"),
|
||||
)
|
||||
|
||||
receivedAt := time.Now()
|
||||
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
package nats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// MaxDeliveriesAdvisoryEvent represents the advisory message published when
|
||||
// a message reaches MaxDeliver attempts.
|
||||
type MaxDeliveriesAdvisoryEvent struct {
|
||||
Type string `json:"type"`
|
||||
Stream string `json:"stream"`
|
||||
Consumer string `json:"consumer"`
|
||||
StreamSeq uint64 `json:"stream_seq"`
|
||||
Deliveries uint64 `json:"deliveries"`
|
||||
Time string `json:"time"`
|
||||
}
|
||||
|
||||
// AdvisoryDLQHandler handles messages that exhaust MaxDeliver attempts
|
||||
// by subscribing to JetStream advisory events.
|
||||
type AdvisoryDLQHandler struct {
|
||||
js nats.JetStreamContext
|
||||
nc *nats.Conn
|
||||
streamName string
|
||||
consumerName string
|
||||
dlqSubject string
|
||||
logger *zap.Logger
|
||||
telemetry TelemetryRecorder
|
||||
}
|
||||
|
||||
// TelemetryRecorder is an interface for recording telemetry events.
|
||||
// This matches the telemetry.Recorder interface used by Consumer.
|
||||
type TelemetryRecorder interface {
|
||||
RecordDLQMessage(ctx context.Context, stream, consumer string)
|
||||
RecordDLQPublishFailure(ctx context.Context, stream, consumer string)
|
||||
}
|
||||
|
||||
// NewAdvisoryDLQHandler creates a new advisory-based DLQ handler.
|
||||
func NewAdvisoryDLQHandler(
|
||||
js nats.JetStreamContext,
|
||||
nc *nats.Conn,
|
||||
streamName string,
|
||||
consumerName string,
|
||||
dlqSubject string,
|
||||
logger *zap.Logger,
|
||||
telemetry TelemetryRecorder,
|
||||
) (*AdvisoryDLQHandler, error) {
|
||||
return &AdvisoryDLQHandler{
|
||||
js: js,
|
||||
nc: nc,
|
||||
streamName: streamName,
|
||||
consumerName: consumerName,
|
||||
dlqSubject: dlqSubject,
|
||||
logger: logger,
|
||||
telemetry: telemetry,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Start begins listening for advisory messages and routing failed messages to DLQ.
|
||||
func (h *AdvisoryDLQHandler) Start(ctx context.Context) error {
|
||||
// Subscribe to advisory subject pattern
|
||||
// Format: $JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.<STREAM>.<CONSUMER>
|
||||
advisorySubject := fmt.Sprintf("$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.%s.%s",
|
||||
h.streamName, h.consumerName)
|
||||
|
||||
h.logger.Info("Starting advisory DLQ handler",
|
||||
zap.String("advisory_subject", advisorySubject),
|
||||
zap.String("stream", h.streamName),
|
||||
zap.String("consumer", h.consumerName),
|
||||
zap.String("dlq_subject", h.dlqSubject),
|
||||
)
|
||||
|
||||
sub, err := h.nc.Subscribe(advisorySubject, func(msg *nats.Msg) {
|
||||
h.handleAdvisory(ctx, msg)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to subscribe to advisory subject: %w", err)
|
||||
}
|
||||
|
||||
// Wait for context cancellation
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
sub.Unsubscribe()
|
||||
h.logger.Info("Stopped advisory DLQ handler")
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleAdvisory processes an advisory message about max deliveries.
|
||||
func (h *AdvisoryDLQHandler) handleAdvisory(ctx context.Context, advisoryMsg *nats.Msg) {
|
||||
var event MaxDeliveriesAdvisoryEvent
|
||||
if err := json.Unmarshal(advisoryMsg.Data, &event); err != nil {
|
||||
h.logger.Error("Failed to unmarshal advisory event",
|
||||
zap.Error(err),
|
||||
zap.String("data", string(advisoryMsg.Data)),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
h.logger.Warn("Message reached MaxDeliver attempts",
|
||||
zap.String("stream", event.Stream),
|
||||
zap.String("consumer", event.Consumer),
|
||||
zap.Uint64("stream_seq", event.StreamSeq),
|
||||
zap.Uint64("deliveries", event.Deliveries),
|
||||
)
|
||||
|
||||
// Retrieve the original message from the stream using GetMsg API
|
||||
originalMsg, err := h.js.GetMsg(h.streamName, event.StreamSeq)
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to retrieve original message from stream",
|
||||
zap.Uint64("stream_seq", event.StreamSeq),
|
||||
zap.Error(err),
|
||||
)
|
||||
h.telemetry.RecordDLQPublishFailure(ctx, h.streamName, h.consumerName)
|
||||
return
|
||||
}
|
||||
|
||||
// Extract message metadata
|
||||
msgID := ""
|
||||
subject := originalMsg.Subject
|
||||
if originalMsg.Header != nil {
|
||||
msgID = originalMsg.Header.Get("Nats-Msg-Id")
|
||||
}
|
||||
|
||||
// Create enriched DLQ payload (similar to existing routeToDLQ)
|
||||
payload := map[string]interface{}{
|
||||
"transport_msg_id": msgID,
|
||||
"subject": subject,
|
||||
"stream": h.streamName,
|
||||
"consumer": h.consumerName,
|
||||
"nats_sequence": event.StreamSeq,
|
||||
"deliveries": event.Deliveries,
|
||||
"error": fmt.Sprintf("message exhausted max_deliver (%d) attempts", event.Deliveries),
|
||||
"received_at": time.Now().UTC(),
|
||||
"body": string(originalMsg.Data),
|
||||
"advisory_source": true, // Flag to distinguish from immediate DLQ
|
||||
}
|
||||
|
||||
data, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to marshal advisory DLQ payload", zap.Error(err))
|
||||
h.telemetry.RecordDLQPublishFailure(ctx, h.streamName, h.consumerName)
|
||||
return
|
||||
}
|
||||
|
||||
// Publish to DLQ
|
||||
if _, err := h.js.Publish(h.dlqSubject, data); err != nil {
|
||||
h.logger.Error("Failed to publish advisory message to DLQ",
|
||||
zap.Uint64("stream_seq", event.StreamSeq),
|
||||
zap.Error(err),
|
||||
)
|
||||
h.telemetry.RecordDLQPublishFailure(ctx, h.streamName, h.consumerName)
|
||||
return
|
||||
}
|
||||
|
||||
h.logger.Info("Routed max-deliveries message to DLQ",
|
||||
zap.Uint64("stream_seq", event.StreamSeq),
|
||||
zap.Uint64("deliveries", event.Deliveries),
|
||||
)
|
||||
h.telemetry.RecordDLQMessage(ctx, h.streamName, h.consumerName)
|
||||
}
|
||||
+502
-49
@@ -5,6 +5,8 @@ import (
|
||||
"caatsm/internal/infra/config"
|
||||
"caatsm/internal/infra/telemetry"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -14,35 +16,57 @@ import (
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// Consumer handles NATS JetStream message consumption
|
||||
type Consumer struct {
|
||||
conn *nats.Conn
|
||||
js nats.JetStreamContext
|
||||
processor *app.MessageProcessor
|
||||
cfg *config.Config
|
||||
logger *zap.Logger
|
||||
telemetry telemetry.Recorder
|
||||
subject string
|
||||
consumerName string
|
||||
mode string
|
||||
streamName string
|
||||
dlqSubject string
|
||||
ackWait time.Duration
|
||||
batchSize int
|
||||
batchTimeout time.Duration
|
||||
monitorInterval time.Duration
|
||||
meter metric.Meter
|
||||
ackPending metric.Int64Histogram
|
||||
redelivered metric.Int64Histogram
|
||||
pending metric.Int64Histogram
|
||||
delivered metric.Int64Histogram
|
||||
// MessageFetcher defines the interface for fetching messages from NATS
|
||||
type MessageFetcher interface {
|
||||
FetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error)
|
||||
HandleFetchError(ctx context.Context, err error, sub **nats.Subscription, fetchErrorStreak *int) (bool, error)
|
||||
}
|
||||
|
||||
// managers for resource lifecycle
|
||||
// MessageProcessor defines the interface for processing message batches
|
||||
type MessageProcessor interface {
|
||||
ProcessBatch(ctx context.Context, msgs []*nats.Msg)
|
||||
}
|
||||
|
||||
// DLQHandler defines the interface for dead letter queue operations
|
||||
type DLQHandler interface {
|
||||
RouteToDLQ(ctx context.Context, msg *nats.Msg, cause error) error
|
||||
ValidateDLQ() error
|
||||
}
|
||||
|
||||
// Consumer handles NATS JetStream message consumption with clean separation of concerns
|
||||
type Consumer struct {
|
||||
// Core dependencies
|
||||
conn *nats.Conn
|
||||
js nats.JetStreamContext
|
||||
processor *app.MessageProcessor
|
||||
cfg *config.Config
|
||||
logger *zap.Logger
|
||||
telemetry telemetry.Recorder
|
||||
|
||||
// Configuration
|
||||
config consumerConfig
|
||||
|
||||
// Collaborators (injected for testability)
|
||||
fetcher MessageFetcher
|
||||
batchProcessor MessageProcessor
|
||||
dlqHandler DLQHandler
|
||||
errorHandler *ErrorHandler
|
||||
|
||||
// Resource managers
|
||||
consumerManager *ConsumerManager
|
||||
streamManager *StreamManager
|
||||
errorHandler *ErrorHandler
|
||||
|
||||
// simple backpressure / degradation state
|
||||
// Advisory DLQ handler for messages exhausting MaxDeliver
|
||||
advisoryDLQHandler *AdvisoryDLQHandler
|
||||
|
||||
// Metrics
|
||||
meter metric.Meter
|
||||
ackPending metric.Int64Histogram
|
||||
redelivered metric.Int64Histogram
|
||||
pending metric.Int64Histogram
|
||||
delivered metric.Int64Histogram
|
||||
|
||||
// State
|
||||
consecutiveProcessErrors int
|
||||
}
|
||||
|
||||
@@ -59,6 +83,397 @@ type consumerConfig struct {
|
||||
monitorInterval time.Duration
|
||||
}
|
||||
|
||||
// defaultMessageFetcher implements MessageFetcher interface
|
||||
type defaultMessageFetcher struct {
|
||||
batchSize int
|
||||
batchTimeout time.Duration
|
||||
logger *zap.Logger
|
||||
conn *nats.Conn
|
||||
js nats.JetStreamContext
|
||||
consumerManager *ConsumerManager
|
||||
streamManager *StreamManager
|
||||
config *consumerConfig
|
||||
cfg *config.Config
|
||||
}
|
||||
|
||||
func (f *defaultMessageFetcher) FetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error) {
|
||||
return f.fetchBatch(ctx, sub)
|
||||
}
|
||||
|
||||
// fetchBatch fetches a batch of messages from the subscription with context awareness
|
||||
func (f *defaultMessageFetcher) fetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error) {
|
||||
// Check context before fetching
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
// Use a shorter timeout for better responsiveness to cancellation
|
||||
timeout := f.batchTimeout
|
||||
if timeout > 500*time.Millisecond {
|
||||
timeout = 500 * time.Millisecond
|
||||
}
|
||||
|
||||
return sub.Fetch(f.batchSize, nats.MaxWait(timeout))
|
||||
}
|
||||
|
||||
func (f *defaultMessageFetcher) HandleFetchError(ctx context.Context, err error, sub **nats.Subscription, fetchErrorStreak *int) (bool, error) {
|
||||
// Check context cancellation first
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
f.logger.Info("Fetch error due to context cancellation", zap.Error(err))
|
||||
return false, err
|
||||
}
|
||||
|
||||
// Timeout errors are expected when no messages are available - not an error condition
|
||||
if errors.Is(err, nats.ErrTimeout) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Check connection health before proceeding
|
||||
if f.conn != nil {
|
||||
status := f.conn.Status()
|
||||
if status != nats.CONNECTED {
|
||||
f.logger.Warn("NATS connection not in CONNECTED state",
|
||||
zap.String("status", status.String()),
|
||||
zap.Error(err),
|
||||
)
|
||||
// Connection is down - this is a transient error, apply backoff
|
||||
*fetchErrorStreak++
|
||||
backoff := f.calculateExponentialBackoff(*fetchErrorStreak)
|
||||
f.logger.Warn("Connection unhealthy, applying backoff before retry",
|
||||
zap.String("status", status.String()),
|
||||
zap.Int("error_streak", *fetchErrorStreak),
|
||||
zap.Duration("backoff", backoff),
|
||||
)
|
||||
if !sleepWithContext(ctx, backoff) {
|
||||
return false, ctx.Err()
|
||||
}
|
||||
// Check if connection recovered after backoff
|
||||
if f.conn.Status() == nats.CONNECTED {
|
||||
*fetchErrorStreak = 0
|
||||
return true, nil
|
||||
}
|
||||
// Still not connected - continue with error handling
|
||||
}
|
||||
}
|
||||
|
||||
// Check for connection closed errors
|
||||
if errors.Is(err, nats.ErrConnectionClosed) {
|
||||
f.logger.Error("NATS connection closed",
|
||||
zap.Error(err),
|
||||
zap.String("stream", f.config.streamName),
|
||||
zap.String("consumer", f.config.consumerName),
|
||||
)
|
||||
// Connection closed is fatal - cannot recover subscription
|
||||
if *sub != nil {
|
||||
(*sub).Unsubscribe()
|
||||
*sub = nil
|
||||
}
|
||||
return false, fmt.Errorf("connection closed: %w", err)
|
||||
}
|
||||
|
||||
// JetStream API unavailable (e.g., NATS restarted or JetStream not ready)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
*fetchErrorStreak++
|
||||
backoff := f.calculateExponentialBackoff(*fetchErrorStreak)
|
||||
backoff = min(backoff, 30*time.Second)
|
||||
f.logger.Warn("JetStream not available, will retry with backoff",
|
||||
zap.Error(err),
|
||||
zap.String("stream", f.config.streamName),
|
||||
zap.String("consumer", f.config.consumerName),
|
||||
zap.Int("error_streak", *fetchErrorStreak),
|
||||
zap.Duration("backoff", backoff),
|
||||
)
|
||||
if !sleepWithContext(ctx, backoff) {
|
||||
return false, ctx.Err()
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Check for JetStream resource not found errors
|
||||
if isJetStreamResourceNotFound(err) {
|
||||
if isDevLikeEnv() && shouldBootstrapStream() {
|
||||
f.logger.Warn("JetStream consumer or stream missing; attempting to recreate",
|
||||
zap.Error(err),
|
||||
zap.String("stream", f.config.streamName),
|
||||
zap.String("consumer", f.config.consumerName),
|
||||
)
|
||||
// Attempt to recover resources and recreate subscription
|
||||
if f.consumerManager == nil || f.streamManager == nil {
|
||||
return false, fmt.Errorf("cannot recover: consumer/stream manager not available: %w", err)
|
||||
}
|
||||
consumerConfig := f.buildConsumerConfig()
|
||||
if recErr := f.consumerManager.RecoverResources(f.streamManager, consumerConfig); recErr != nil {
|
||||
return false, fmt.Errorf("failed to recover JetStream resources: %w", recErr)
|
||||
}
|
||||
// Unsubscribe old subscription before creating new one
|
||||
if *sub != nil {
|
||||
(*sub).Unsubscribe()
|
||||
}
|
||||
// Create new subscription
|
||||
newSub, subErr := f.consumerManager.CreatePullSubscription()
|
||||
if subErr != nil {
|
||||
return false, fmt.Errorf("failed to create pull subscription after recovery: %w", subErr)
|
||||
}
|
||||
*sub = newSub
|
||||
*fetchErrorStreak = 0
|
||||
f.logger.Info("Successfully recovered subscription after resource recreation")
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Production: treat as configuration/operational error - fatal
|
||||
f.logger.Error("JetStream consumer or stream missing; not auto-recreating in this environment",
|
||||
zap.Error(err),
|
||||
zap.String("stream", f.config.streamName),
|
||||
zap.String("consumer", f.config.consumerName),
|
||||
)
|
||||
if *sub != nil {
|
||||
(*sub).Unsubscribe()
|
||||
*sub = nil
|
||||
}
|
||||
return false, fmt.Errorf("JetStream resource not found: %w", err)
|
||||
}
|
||||
|
||||
// Check for network/temporary errors
|
||||
if f.isTemporaryError(err) {
|
||||
*fetchErrorStreak++
|
||||
backoff := f.calculateExponentialBackoff(*fetchErrorStreak)
|
||||
f.logger.Warn("Temporary network error, applying backoff",
|
||||
zap.Error(err),
|
||||
zap.Int("error_streak", *fetchErrorStreak),
|
||||
zap.Duration("backoff", backoff),
|
||||
)
|
||||
if !sleepWithContext(ctx, backoff) {
|
||||
return false, ctx.Err()
|
||||
}
|
||||
// Verify subscription is still valid before returning success
|
||||
if *sub != nil && f.conn != nil && f.conn.Status() == nats.CONNECTED {
|
||||
return true, nil
|
||||
}
|
||||
// Subscription or connection invalid - attempt recovery
|
||||
return f.attemptSubscriptionRecovery(ctx, sub, fetchErrorStreak)
|
||||
}
|
||||
|
||||
// Generic error path with exponential backoff
|
||||
*fetchErrorStreak++
|
||||
backoff := f.calculateExponentialBackoff(*fetchErrorStreak)
|
||||
f.logger.Error("Failed to fetch messages; backing off",
|
||||
zap.Error(err),
|
||||
zap.Int("error_streak", *fetchErrorStreak),
|
||||
zap.Duration("backoff", backoff),
|
||||
)
|
||||
if !sleepWithContext(ctx, backoff) {
|
||||
return false, ctx.Err()
|
||||
}
|
||||
|
||||
// Verify subscription and connection health before returning success
|
||||
if *sub == nil || (f.conn != nil && f.conn.Status() != nats.CONNECTED) {
|
||||
return f.attemptSubscriptionRecovery(ctx, sub, fetchErrorStreak)
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// calculateExponentialBackoff calculates exponential backoff duration with a cap
|
||||
func (f *defaultMessageFetcher) calculateExponentialBackoff(streak int) time.Duration {
|
||||
if streak <= 0 {
|
||||
return 0
|
||||
}
|
||||
// Exponential backoff: 2^(streak-1) seconds, capped at 30 seconds
|
||||
backoff := time.Duration(1<<uint(min(streak-1, 5))) * time.Second
|
||||
return min(backoff, 30*time.Second)
|
||||
}
|
||||
|
||||
// isTemporaryError checks if an error is a temporary/network error that might recover
|
||||
func (f *defaultMessageFetcher) isTemporaryError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
// Check for typed temporary errors first
|
||||
var tempErr interface{ Temporary() bool }
|
||||
if errors.As(err, &tempErr) && tempErr.Temporary() {
|
||||
return true
|
||||
}
|
||||
// Fall back to string matching for external errors
|
||||
errStr := strings.ToLower(err.Error())
|
||||
return strings.Contains(errStr, "timeout") ||
|
||||
strings.Contains(errStr, "temporary") ||
|
||||
strings.Contains(errStr, "network") ||
|
||||
strings.Contains(errStr, "connection reset") ||
|
||||
strings.Contains(errStr, "broken pipe")
|
||||
}
|
||||
|
||||
// attemptSubscriptionRecovery attempts to recover a subscription after errors
|
||||
func (f *defaultMessageFetcher) attemptSubscriptionRecovery(ctx context.Context, sub **nats.Subscription, fetchErrorStreak *int) (bool, error) {
|
||||
if f.consumerManager == nil {
|
||||
f.logger.Error("Cannot recover subscription: consumer manager not available")
|
||||
return false, fmt.Errorf("consumer manager not available for recovery")
|
||||
}
|
||||
|
||||
// Check connection health first
|
||||
if f.conn != nil && f.conn.Status() != nats.CONNECTED {
|
||||
f.logger.Warn("Connection not healthy, cannot recover subscription",
|
||||
zap.String("status", f.conn.Status().String()),
|
||||
)
|
||||
// Connection issue - return true to retry after backoff
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Unsubscribe old subscription if it exists
|
||||
if *sub != nil {
|
||||
(*sub).Unsubscribe()
|
||||
*sub = nil
|
||||
}
|
||||
|
||||
// Attempt to recreate subscription
|
||||
newSub, err := f.consumerManager.CreatePullSubscriptionWithRecovery(f.streamManager, f.buildConsumerConfig())
|
||||
if err != nil {
|
||||
f.logger.Error("Failed to recover subscription",
|
||||
zap.Error(err),
|
||||
zap.String("stream", f.config.streamName),
|
||||
zap.String("consumer", f.config.consumerName),
|
||||
)
|
||||
return false, fmt.Errorf("failed to recover subscription: %w", err)
|
||||
}
|
||||
|
||||
*sub = newSub
|
||||
*fetchErrorStreak = 0
|
||||
f.logger.Info("Successfully recovered subscription")
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// buildConsumerConfig builds the NATS consumer configuration
|
||||
func (f *defaultMessageFetcher) buildConsumerConfig() *nats.ConsumerConfig {
|
||||
if f.cfg == nil {
|
||||
return nil
|
||||
}
|
||||
return &nats.ConsumerConfig{
|
||||
Durable: f.config.consumerName,
|
||||
DeliverPolicy: mapDeliverPolicy(f.cfg.NATS.ConsumerRules.DeliverPolicy),
|
||||
AckPolicy: nats.AckExplicitPolicy,
|
||||
AckWait: f.config.ackWait,
|
||||
ReplayPolicy: mapReplayPolicy(f.cfg.NATS.ConsumerRules.ReplayPolicy),
|
||||
MaxDeliver: f.cfg.NATS.ConsumerRules.MaxDeliver,
|
||||
MaxAckPending: f.cfg.NATS.ConsumerRules.MaxAckPending,
|
||||
FilterSubject: f.config.subject,
|
||||
BackOff: f.cfg.NATS.ConsumerRules.Backoff,
|
||||
}
|
||||
}
|
||||
|
||||
// defaultBatchProcessor implements MessageProcessor interface
|
||||
type defaultBatchProcessor struct {
|
||||
processor *app.MessageProcessor
|
||||
dlqHandler DLQHandler
|
||||
errorHandler *ErrorHandler
|
||||
logger *zap.Logger
|
||||
telemetry telemetry.Recorder
|
||||
}
|
||||
|
||||
func (p *defaultBatchProcessor) ProcessBatch(ctx context.Context, msgs []*nats.Msg) {
|
||||
// This will be implemented when we refactor the batch processing
|
||||
}
|
||||
|
||||
// defaultDLQHandler implements DLQHandler interface
|
||||
type defaultDLQHandler struct {
|
||||
js nats.JetStreamContext
|
||||
dlqSubject string
|
||||
streamName string
|
||||
consumerName string
|
||||
logger *zap.Logger
|
||||
telemetry telemetry.Recorder
|
||||
}
|
||||
|
||||
func (h *defaultDLQHandler) RouteToDLQ(ctx context.Context, msg *nats.Msg, cause error) error {
|
||||
return h.routeToDLQInternal(ctx, msg, cause)
|
||||
}
|
||||
|
||||
func (h *defaultDLQHandler) ValidateDLQ() error {
|
||||
return h.validateDLQInternal()
|
||||
}
|
||||
|
||||
func (h *defaultDLQHandler) routeToDLQInternal(ctx context.Context, msg *nats.Msg, cause error) error {
|
||||
// Basic DLQ routing implementation
|
||||
payload := map[string]any{
|
||||
"subject": msg.Subject,
|
||||
"stream": h.streamName,
|
||||
"consumer": h.consumerName,
|
||||
"error": cause.Error(),
|
||||
"received_at": time.Now().UTC(),
|
||||
"body": string(msg.Data),
|
||||
}
|
||||
|
||||
data, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
h.logger.Error("failed to marshal DLQ payload", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = h.js.Publish(h.dlqSubject, data)
|
||||
if err != nil {
|
||||
h.logger.Error("failed to publish to DLQ",
|
||||
zap.String("dlq_subject", h.dlqSubject),
|
||||
zap.Error(err),
|
||||
)
|
||||
h.telemetry.RecordDLQPublishFailure(ctx, h.streamName, h.consumerName)
|
||||
return err
|
||||
}
|
||||
|
||||
h.telemetry.RecordDLQMessage(ctx, h.streamName, h.consumerName)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *defaultDLQHandler) validateDLQInternal() error {
|
||||
if h.js == nil {
|
||||
return fmt.Errorf("JetStream context is nil")
|
||||
}
|
||||
|
||||
_, err := h.js.StreamNameBySubject(h.dlqSubject)
|
||||
if err != nil {
|
||||
return fmt.Errorf("DLQ subject %s not bound to any JetStream stream: %w", h.dlqSubject, err)
|
||||
}
|
||||
|
||||
h.logger.Info("DLQ configuration validated",
|
||||
zap.String("dlq_subject", h.dlqSubject),
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// initCollaborators initializes the collaborator components
|
||||
func (c *Consumer) initCollaborators() {
|
||||
c.fetcher = &defaultMessageFetcher{
|
||||
batchSize: c.config.batchSize,
|
||||
batchTimeout: c.config.batchTimeout,
|
||||
logger: c.logger,
|
||||
conn: c.conn,
|
||||
js: c.js,
|
||||
consumerManager: c.consumerManager,
|
||||
streamManager: c.streamManager,
|
||||
config: &c.config,
|
||||
cfg: c.cfg,
|
||||
}
|
||||
|
||||
c.batchProcessor = &defaultBatchProcessor{
|
||||
processor: c.processor,
|
||||
dlqHandler: c.dlqHandler,
|
||||
errorHandler: c.errorHandler,
|
||||
logger: c.logger,
|
||||
telemetry: c.telemetry,
|
||||
}
|
||||
|
||||
if c.config.dlqSubject != "" {
|
||||
c.dlqHandler = &defaultDLQHandler{
|
||||
js: c.js,
|
||||
dlqSubject: c.config.dlqSubject,
|
||||
streamName: c.config.streamName,
|
||||
consumerName: c.config.consumerName,
|
||||
logger: c.logger,
|
||||
telemetry: c.telemetry,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeConsumerConfig extracts and normalizes consumer configuration from the application config.
|
||||
// This function can be unit-tested without requiring a JetStream context.
|
||||
func normalizeConsumerConfig(cfg *config.Config) *consumerConfig {
|
||||
@@ -122,7 +537,7 @@ func normalizeConsumerConfig(cfg *config.Config) *consumerConfig {
|
||||
}
|
||||
}
|
||||
|
||||
// ProvideConsumer creates a NATS consumer.
|
||||
// ProvideConsumer creates a NATS consumer with clean architecture.
|
||||
func ProvideConsumer(
|
||||
conn *nats.Conn,
|
||||
js nats.JetStreamContext,
|
||||
@@ -134,29 +549,34 @@ func ProvideConsumer(
|
||||
normCfg := normalizeConsumerConfig(cfg)
|
||||
|
||||
consumer := &Consumer{
|
||||
conn: conn,
|
||||
js: js,
|
||||
processor: processor,
|
||||
cfg: cfg,
|
||||
logger: logger,
|
||||
telemetry: rec,
|
||||
subject: normCfg.subject,
|
||||
consumerName: normCfg.consumerName,
|
||||
mode: normCfg.mode,
|
||||
streamName: normCfg.streamName,
|
||||
dlqSubject: normCfg.dlqSubject,
|
||||
ackWait: normCfg.ackWait,
|
||||
batchSize: normCfg.batchSize,
|
||||
batchTimeout: normCfg.batchTimeout,
|
||||
monitorInterval: normCfg.monitorInterval,
|
||||
conn: conn,
|
||||
js: js,
|
||||
processor: processor,
|
||||
cfg: cfg,
|
||||
logger: logger,
|
||||
telemetry: rec,
|
||||
config: *normCfg, // dereference the pointer
|
||||
errorHandler: NewErrorHandler(logger),
|
||||
}
|
||||
consumer.initMetrics()
|
||||
consumer.initCollaborators()
|
||||
|
||||
// Initialize managers
|
||||
consumer.errorHandler = NewErrorHandler(logger)
|
||||
if consumer.mode == "jetstream" {
|
||||
if consumer.config.mode == "jetstream" {
|
||||
consumer.consumerManager = NewConsumerManager(js, normCfg.streamName, normCfg.consumerName, normCfg.subject, logger)
|
||||
consumer.streamManager = NewStreamManager(js, normCfg.streamName, []string{normCfg.subject}, logger)
|
||||
// Use StreamManager with full configuration
|
||||
streamSubjects := []string{normCfg.subject}
|
||||
if publisherSubject := strings.TrimSpace(cfg.Publisher.Topic); publisherSubject != "" {
|
||||
streamSubjects = append(streamSubjects, publisherSubject)
|
||||
}
|
||||
streamSubjects = dedupeSubjects(streamSubjects)
|
||||
consumer.streamManager = NewStreamManagerWithConfig(js, normCfg.streamName, streamSubjects, &cfg.NATS.StreamLimits, logger)
|
||||
|
||||
// Update fetcher with managers now that they're initialized
|
||||
if fetcher, ok := consumer.fetcher.(*defaultMessageFetcher); ok {
|
||||
fetcher.consumerManager = consumer.consumerManager
|
||||
fetcher.streamManager = consumer.streamManager
|
||||
}
|
||||
|
||||
// Create consumer if it doesn't exist
|
||||
consumerConfig := consumer.buildConsumerConfig()
|
||||
@@ -168,6 +588,23 @@ func ProvideConsumer(
|
||||
if err := consumer.validateDLQ(); err != nil {
|
||||
return nil, fmt.Errorf("DLQ validation failed: %w", err)
|
||||
}
|
||||
|
||||
// Initialize advisory DLQ handler if DLQ is enabled
|
||||
if normCfg.dlqSubject != "" && cfg.DLQ.Enabled {
|
||||
advisoryHandler, err := NewAdvisoryDLQHandler(
|
||||
js,
|
||||
conn,
|
||||
normCfg.streamName,
|
||||
normCfg.consumerName,
|
||||
normCfg.dlqSubject,
|
||||
logger,
|
||||
rec,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create advisory DLQ handler: %w", err)
|
||||
}
|
||||
consumer.advisoryDLQHandler = advisoryHandler
|
||||
}
|
||||
} else {
|
||||
logger.Info("Running consumer in core NATS mode",
|
||||
zap.String("subject", normCfg.subject),
|
||||
@@ -181,27 +618,43 @@ func ProvideConsumer(
|
||||
// buildConsumerConfig builds the NATS consumer configuration
|
||||
func (c *Consumer) buildConsumerConfig() *nats.ConsumerConfig {
|
||||
return &nats.ConsumerConfig{
|
||||
Durable: c.consumerName,
|
||||
Durable: c.config.consumerName,
|
||||
DeliverPolicy: mapDeliverPolicy(c.cfg.NATS.ConsumerRules.DeliverPolicy),
|
||||
AckPolicy: nats.AckExplicitPolicy,
|
||||
AckWait: c.ackWait,
|
||||
AckWait: c.config.ackWait,
|
||||
ReplayPolicy: mapReplayPolicy(c.cfg.NATS.ConsumerRules.ReplayPolicy),
|
||||
MaxDeliver: c.cfg.NATS.ConsumerRules.MaxDeliver,
|
||||
MaxAckPending: c.cfg.NATS.ConsumerRules.MaxAckPending,
|
||||
FilterSubject: c.subject,
|
||||
FilterSubject: c.config.subject,
|
||||
BackOff: c.cfg.NATS.ConsumerRules.Backoff,
|
||||
}
|
||||
}
|
||||
|
||||
// Start starts consuming messages.
|
||||
func (c *Consumer) Start(ctx context.Context) error {
|
||||
if c.mode == "core" {
|
||||
if c.config.mode == "core" {
|
||||
return c.startCore(ctx)
|
||||
}
|
||||
|
||||
return c.startJetStream(ctx)
|
||||
}
|
||||
|
||||
// RouteToDLQ implements DLQHandler interface
|
||||
func (c *Consumer) RouteToDLQ(ctx context.Context, msg *nats.Msg, cause error) error {
|
||||
if c.dlqHandler != nil {
|
||||
return c.dlqHandler.RouteToDLQ(ctx, msg, cause)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateDLQ implements DLQHandler interface
|
||||
func (c *Consumer) ValidateDLQ() error {
|
||||
if c.dlqHandler != nil {
|
||||
return c.dlqHandler.ValidateDLQ()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown drains the underlying NATS connection gracefully.
|
||||
func (c *Consumer) Shutdown(ctx context.Context) error {
|
||||
if c.conn == nil {
|
||||
@@ -210,7 +663,7 @@ func (c *Consumer) Shutdown(ctx context.Context) error {
|
||||
|
||||
timeout := c.cfg.Timeouts.Close
|
||||
if timeout <= 0 {
|
||||
timeout = 10 * time.Second
|
||||
timeout = 2 * time.Second // Reduced from 10s for faster shutdown
|
||||
}
|
||||
|
||||
closeCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
func (c *Consumer) startCore(ctx context.Context) error {
|
||||
queueGroup := c.cfg.Subscription.QueueGroup
|
||||
if queueGroup == "" {
|
||||
queueGroup = c.consumerName
|
||||
queueGroup = c.config.consumerName
|
||||
}
|
||||
|
||||
handler := func(msg *nats.Msg) {
|
||||
@@ -28,16 +28,16 @@ func (c *Consumer) startCore(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
sub, err := c.conn.QueueSubscribe(c.subject, queueGroup, handler)
|
||||
sub, err := c.conn.QueueSubscribe(c.config.subject, queueGroup, handler)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to subscribe to %s: %w", c.subject, err)
|
||||
return fmt.Errorf("failed to subscribe to %s: %w", c.config.subject, err)
|
||||
}
|
||||
if err := c.conn.Flush(); err != nil {
|
||||
return fmt.Errorf("failed to flush NATS connection: %w", err)
|
||||
}
|
||||
|
||||
c.logger.Info("Started core NATS subscription",
|
||||
zap.String("subject", c.subject),
|
||||
zap.String("subject", c.config.subject),
|
||||
zap.String("queue_group", queueGroup),
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
package nats
|
||||
|
||||
import (
|
||||
"caatsm/internal/app"
|
||||
obsmetrics "caatsm/internal/infra/metrics"
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// handleMessageError handles errors that occur during message processing.
|
||||
func (c *Consumer) handleMessageError(ctx context.Context, msg *nats.Msg, err error, elapsed time.Duration) {
|
||||
c.logger.Error("Failed to process message",
|
||||
zap.String("subject", msg.Subject),
|
||||
zap.Error(err),
|
||||
zap.Bool("permanent", app.IsPermanent(err)),
|
||||
)
|
||||
|
||||
result := obsmetrics.ResultFail
|
||||
if app.IsPermanent(err) {
|
||||
result = obsmetrics.ResultPermanentFail
|
||||
}
|
||||
c.telemetry.RecordMessageHandled(ctx, c.config.streamName, c.config.consumerName, result, elapsed)
|
||||
|
||||
processingResult := c.errorHandler.HandleProcessingError(c.consecutiveProcessErrors, err, c.logger, msg.Subject)
|
||||
|
||||
if processingResult.IsPermanent {
|
||||
c.handlePermanentError(ctx, msg, err)
|
||||
return
|
||||
}
|
||||
|
||||
c.handleTransientError(ctx, msg, processingResult)
|
||||
}
|
||||
|
||||
// handlePermanentError handles permanent/poison messages.
|
||||
func (c *Consumer) handlePermanentError(ctx context.Context, msg *nats.Msg, err error) {
|
||||
c.consecutiveProcessErrors = 0
|
||||
// Poison/permanent message: route to DLQ if configured, then ACK
|
||||
if dlqErr := c.routeToDLQ(ctx, msg, err); dlqErr != nil {
|
||||
c.logger.Error("Failed to route permanent-error message to DLQ", zap.Error(dlqErr))
|
||||
}
|
||||
if ackErr := msg.Ack(); ackErr != nil {
|
||||
c.logger.Error("Failed to ACK permanent-error message", zap.Error(ackErr))
|
||||
}
|
||||
}
|
||||
|
||||
// handleTransientError handles transient errors with backpressure and redelivery.
|
||||
func (c *Consumer) handleTransientError(ctx context.Context, msg *nats.Msg, processingResult ProcessingErrorResult) {
|
||||
// Increment error streak
|
||||
if c.consecutiveProcessErrors < 0 {
|
||||
c.consecutiveProcessErrors = 0
|
||||
}
|
||||
c.consecutiveProcessErrors++
|
||||
|
||||
if processingResult.ShouldApplyBackpressure {
|
||||
c.logger.Warn("Applying backpressure due to consecutive processing errors",
|
||||
zap.Int("consecutive_errors", c.consecutiveProcessErrors),
|
||||
zap.Duration("sleep", processingResult.BackpressureDelay),
|
||||
)
|
||||
// Use context-aware sleep instead of blocking time.Sleep
|
||||
if !sleepWithContext(ctx, processingResult.BackpressureDelay) {
|
||||
// Context canceled, stop processing
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Transient error: request redelivery with optional delay
|
||||
c.telemetry.RecordRetry(ctx, c.config.streamName, c.config.consumerName, obsmetrics.RetryReasonProcessorError)
|
||||
if nakErr := c.nakWithStrategy(msg); nakErr != nil {
|
||||
c.logger.Error("Failed to NAK message", zap.Error(nakErr))
|
||||
}
|
||||
}
|
||||
@@ -1,38 +1,15 @@
|
||||
package nats
|
||||
|
||||
import (
|
||||
"caatsm/internal/app"
|
||||
obsmetrics "caatsm/internal/infra/metrics"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// ensureConsumer creates the consumer if it doesn't exist; if it already exists, it is reused.
|
||||
func (c *Consumer) ensureConsumer() error {
|
||||
consumerConfig := c.buildConsumerConfig()
|
||||
if consumerConfig.DeliverPolicy == nats.DeliverByStartSequencePolicy && c.cfg.NATS.ConsumerRules.StartSequence > 0 {
|
||||
consumerConfig.OptStartSeq = c.cfg.NATS.ConsumerRules.StartSequence
|
||||
}
|
||||
if consumerConfig.DeliverPolicy == nats.DeliverByStartTimePolicy && strings.TrimSpace(c.cfg.NATS.ConsumerRules.StartTime) != "" {
|
||||
startTime, err := time.Parse(time.RFC3339, c.cfg.NATS.ConsumerRules.StartTime)
|
||||
if err != nil {
|
||||
c.logger.Warn("Invalid start time, falling back to deliver policy defaults",
|
||||
zap.String("start_time", c.cfg.NATS.ConsumerRules.StartTime),
|
||||
zap.Error(err),
|
||||
)
|
||||
} else {
|
||||
consumerConfig.OptStartTime = &startTime
|
||||
}
|
||||
}
|
||||
|
||||
return c.consumerManager.EnsureConsumer(consumerConfig)
|
||||
}
|
||||
|
||||
// recoverJetStreamResources attempts to recreate the stream and consumer in
|
||||
// dev/test environments if they are missing. It is safe to call multiple times.
|
||||
func (c *Consumer) recoverJetStreamResources() error {
|
||||
@@ -98,14 +75,30 @@ func sleepWithContext(ctx context.Context, duration time.Duration) bool {
|
||||
}
|
||||
|
||||
// fetchBatch fetches a batch of messages from the subscription.
|
||||
func (c *Consumer) fetchBatch(sub *nats.Subscription) ([]*nats.Msg, error) {
|
||||
return sub.Fetch(c.batchSize, nats.MaxWait(c.batchTimeout))
|
||||
// It respects context cancellation for faster shutdown.
|
||||
func (c *Consumer) fetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error) {
|
||||
// Check context before fetching
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
// Use a shorter timeout for better responsiveness to cancellation
|
||||
// The batchTimeout is still used, but we'll check context more frequently
|
||||
timeout := c.config.batchTimeout
|
||||
if timeout > 500*time.Millisecond {
|
||||
// Cap at 500ms to improve responsiveness while still allowing batching
|
||||
timeout = 500 * time.Millisecond
|
||||
}
|
||||
|
||||
return sub.Fetch(c.config.batchSize, nats.MaxWait(timeout))
|
||||
}
|
||||
|
||||
// handleFetchError handles errors during message fetching, including recovery logic.
|
||||
// Returns true if the error was handled and consumption should continue, false otherwise.
|
||||
func (c *Consumer) handleFetchError(ctx context.Context, err error, sub **nats.Subscription, fetchErrorStreak *int) (bool, error) {
|
||||
result := c.errorHandler.HandleFetchError(ctx, err, sub, fetchErrorStreak, c.streamName, c.consumerName, func() (*nats.Subscription, error) {
|
||||
result := c.errorHandler.HandleFetchError(ctx, err, sub, fetchErrorStreak, c.config.streamName, c.config.consumerName, func() (*nats.Subscription, error) {
|
||||
if recErr := c.recoverJetStreamResources(); recErr != nil {
|
||||
return nil, recErr
|
||||
}
|
||||
@@ -121,99 +114,6 @@ func (c *Consumer) handleFetchError(ctx context.Context, err error, sub **nats.S
|
||||
return result.ShouldContinue, result.Error
|
||||
}
|
||||
|
||||
// processBatch processes a batch of messages, handling errors and applying backpressure.
|
||||
func (c *Consumer) processBatch(ctx context.Context, msgs []*nats.Msg) {
|
||||
for _, msg := range msgs {
|
||||
c.processSingleMessage(ctx, msg)
|
||||
}
|
||||
}
|
||||
|
||||
// processSingleMessage processes a single message with error handling and backpressure.
|
||||
func (c *Consumer) processSingleMessage(ctx context.Context, msg *nats.Msg) {
|
||||
start := time.Now()
|
||||
|
||||
if err := c.processMessage(ctx, msg); err != nil {
|
||||
c.handleMessageError(ctx, msg, err, time.Since(start))
|
||||
return
|
||||
}
|
||||
|
||||
// Successful processing resets the error streak.
|
||||
if c.consecutiveProcessErrors > 0 {
|
||||
c.consecutiveProcessErrors = 0
|
||||
}
|
||||
|
||||
// ACK the message
|
||||
if ackErr := msg.Ack(); ackErr != nil {
|
||||
c.logger.Error("Failed to ACK message", zap.Error(ackErr))
|
||||
} else {
|
||||
elapsed := time.Since(start)
|
||||
c.telemetry.RecordMessageHandled(ctx, c.streamName, c.consumerName, "ok", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
// handleMessageError handles errors that occur during message processing.
|
||||
func (c *Consumer) handleMessageError(ctx context.Context, msg *nats.Msg, err error, elapsed time.Duration) {
|
||||
c.logger.Error("Failed to process message",
|
||||
zap.String("subject", msg.Subject),
|
||||
zap.Error(err),
|
||||
zap.Bool("permanent", app.IsPermanent(err)),
|
||||
)
|
||||
|
||||
result := obsmetrics.ResultFail
|
||||
if app.IsPermanent(err) {
|
||||
result = obsmetrics.ResultPermanentFail
|
||||
}
|
||||
c.telemetry.RecordMessageHandled(ctx, c.streamName, c.consumerName, result, elapsed)
|
||||
|
||||
processingResult := c.errorHandler.HandleProcessingError(c.consecutiveProcessErrors, err, c.logger, msg.Subject)
|
||||
|
||||
if processingResult.IsPermanent {
|
||||
c.handlePermanentError(ctx, msg, err)
|
||||
return
|
||||
}
|
||||
|
||||
c.handleTransientError(ctx, msg, processingResult)
|
||||
}
|
||||
|
||||
// handlePermanentError handles permanent/poison messages.
|
||||
func (c *Consumer) handlePermanentError(ctx context.Context, msg *nats.Msg, err error) {
|
||||
c.consecutiveProcessErrors = 0
|
||||
// Poison/permanent message: route to DLQ if configured, then ACK
|
||||
if dlqErr := c.routeToDLQ(ctx, msg, err); dlqErr != nil {
|
||||
c.logger.Error("Failed to route permanent-error message to DLQ", zap.Error(dlqErr))
|
||||
}
|
||||
if ackErr := msg.Ack(); ackErr != nil {
|
||||
c.logger.Error("Failed to ACK permanent-error message", zap.Error(ackErr))
|
||||
}
|
||||
}
|
||||
|
||||
// handleTransientError handles transient errors with backpressure and redelivery.
|
||||
func (c *Consumer) handleTransientError(ctx context.Context, msg *nats.Msg, processingResult ProcessingErrorResult) {
|
||||
// Increment error streak
|
||||
if c.consecutiveProcessErrors < 0 {
|
||||
c.consecutiveProcessErrors = 0
|
||||
}
|
||||
c.consecutiveProcessErrors++
|
||||
|
||||
if processingResult.ShouldApplyBackpressure {
|
||||
c.logger.Warn("Applying backpressure due to consecutive processing errors",
|
||||
zap.Int("consecutive_errors", c.consecutiveProcessErrors),
|
||||
zap.Duration("sleep", processingResult.BackpressureDelay),
|
||||
)
|
||||
// Use context-aware sleep instead of blocking time.Sleep
|
||||
if !sleepWithContext(ctx, processingResult.BackpressureDelay) {
|
||||
// Context canceled, stop processing
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Transient error: request redelivery with optional delay
|
||||
c.telemetry.RecordRetry(ctx, c.streamName, c.consumerName, obsmetrics.RetryReasonProcessorError)
|
||||
if nakErr := c.nakWithStrategy(msg); nakErr != nil {
|
||||
c.logger.Error("Failed to NAK message", zap.Error(nakErr))
|
||||
}
|
||||
}
|
||||
|
||||
// startJetStream starts the JetStream consumer loop.
|
||||
func (c *Consumer) startJetStream(ctx context.Context) error {
|
||||
// Create pull subscription (with simple self-healing in dev/test).
|
||||
@@ -235,16 +135,16 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
|
||||
defer cleanupSubscriber()
|
||||
|
||||
c.logger.Info("Started consuming messages",
|
||||
zap.String("subject", c.subject),
|
||||
zap.String("consumer", c.consumerName),
|
||||
zap.String("stream", c.streamName),
|
||||
zap.String("subject", c.config.subject),
|
||||
zap.String("consumer", c.config.consumerName),
|
||||
zap.String("stream", c.config.streamName),
|
||||
)
|
||||
|
||||
c.logger.Info("Consumer pull configuration",
|
||||
zap.Int("batch_size", c.batchSize),
|
||||
zap.Duration("batch_timeout", c.batchTimeout),
|
||||
zap.Int("batch_size", c.config.batchSize),
|
||||
zap.Duration("batch_timeout", c.config.batchTimeout),
|
||||
zap.Int("max_deliver", c.cfg.NATS.ConsumerRules.MaxDeliver),
|
||||
zap.Duration("ack_wait", c.ackWait),
|
||||
zap.Duration("ack_wait", c.config.ackWait),
|
||||
zap.String("deliver_policy", c.cfg.NATS.ConsumerRules.DeliverPolicy),
|
||||
zap.String("replay_policy", c.cfg.NATS.ConsumerRules.ReplayPolicy),
|
||||
zap.Int("backoff_steps", len(c.cfg.NATS.ConsumerRules.Backoff)),
|
||||
@@ -254,6 +154,15 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
|
||||
defer statsCancel()
|
||||
go c.emitConsumerStats(statsCtx)
|
||||
|
||||
// Start advisory DLQ handler in background if configured
|
||||
if c.advisoryDLQHandler != nil {
|
||||
go func() {
|
||||
if err := c.advisoryDLQHandler.Start(ctx); err != nil {
|
||||
c.logger.Error("Advisory DLQ handler failed", zap.Error(err))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
var fetchErrorStreak int
|
||||
|
||||
for {
|
||||
@@ -265,8 +174,13 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// Fetch messages in batch
|
||||
msgs, err := c.fetchBatch(currentSub)
|
||||
msgs, err := c.fetchBatch(ctx, currentSub)
|
||||
if err != nil {
|
||||
// If context was cancelled, return immediately
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
c.logger.Info("Stopping consumer due to context cancellation", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
shouldContinue, handleErr := c.handleFetchError(ctx, err, ¤tSub, &fetchErrorStreak)
|
||||
if !shouldContinue {
|
||||
return handleErr
|
||||
|
||||
@@ -21,12 +21,14 @@ var _ = Describe("Consumer JetStream", func() {
|
||||
BeforeEach(func() {
|
||||
logger := zaptest.NewLogger(GinkgoT())
|
||||
c = &Consumer{
|
||||
mode: "jetstream",
|
||||
streamName: "TEST_STREAM",
|
||||
consumerName: "test-consumer",
|
||||
subject: "test.subject",
|
||||
batchSize: 10,
|
||||
batchTimeout: 2 * time.Second,
|
||||
config: consumerConfig{
|
||||
mode: "jetstream",
|
||||
streamName: "TEST_STREAM",
|
||||
consumerName: "test-consumer",
|
||||
subject: "test.subject",
|
||||
batchSize: 10,
|
||||
batchTimeout: 2 * time.Second,
|
||||
},
|
||||
logger: logger,
|
||||
errorHandler: NewErrorHandler(logger),
|
||||
cfg: &configpkg.Config{
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"caatsm/internal/infra/log"
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -13,11 +14,59 @@ import (
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// processBatch processes a batch of messages, handling errors and applying backpressure.
|
||||
// It checks context cancellation between messages for faster shutdown.
|
||||
func (c *Consumer) processBatch(ctx context.Context, msgs []*nats.Msg) {
|
||||
for _, msg := range msgs {
|
||||
// Check context before processing each message
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
c.logger.Info("Stopping batch processing due to cancellation",
|
||||
zap.Int("remaining_messages", len(msgs)),
|
||||
)
|
||||
return
|
||||
default:
|
||||
}
|
||||
c.processSingleMessage(ctx, msg)
|
||||
}
|
||||
}
|
||||
|
||||
// processSingleMessage processes a single message with error handling and backpressure.
|
||||
func (c *Consumer) processSingleMessage(ctx context.Context, msg *nats.Msg) {
|
||||
start := time.Now()
|
||||
|
||||
if err := c.processMessage(ctx, msg); err != nil {
|
||||
c.handleMessageError(ctx, msg, err, time.Since(start))
|
||||
return
|
||||
}
|
||||
|
||||
// Successful processing resets the error streak.
|
||||
if c.consecutiveProcessErrors > 0 {
|
||||
c.consecutiveProcessErrors = 0
|
||||
}
|
||||
|
||||
// ACK the message
|
||||
if ackErr := msg.Ack(); ackErr != nil {
|
||||
c.logger.Error("Failed to ACK message", zap.Error(ackErr))
|
||||
} else {
|
||||
elapsed := time.Since(start)
|
||||
c.telemetry.RecordMessageHandled(ctx, c.config.streamName, c.config.consumerName, "ok", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
// processMessage processes a single message.
|
||||
func (c *Consumer) processMessage(ctx context.Context, msg *nats.Msg) error {
|
||||
ctx, span := otel.Tracer("caatsm/nats").Start(ctx, "Consumer.processMessage")
|
||||
defer span.End()
|
||||
span.SetAttributes(attribute.String("nats.subject", msg.Subject))
|
||||
|
||||
// Set semantic messaging attributes
|
||||
span.SetAttributes(
|
||||
attribute.String("messaging.system", "nats"),
|
||||
attribute.String("messaging.operation.name", "receive"),
|
||||
attribute.String("messaging.destination.name", msg.Subject),
|
||||
attribute.String("messaging.consumer.group.name", c.config.consumerName),
|
||||
attribute.String("caatsm.stream", c.config.streamName),
|
||||
)
|
||||
|
||||
msgID, source, err := c.resolveMsgID(msg)
|
||||
if err != nil {
|
||||
@@ -46,8 +95,8 @@ func (c *Consumer) processMessage(ctx context.Context, msg *nats.Msg) error {
|
||||
msgLogger := log.WithMessageContext(c.logger, log.MessageFields{
|
||||
Service: "caatsm-consumer",
|
||||
TransportMsgID: msgID,
|
||||
Stream: c.streamName,
|
||||
Consumer: c.consumerName,
|
||||
Stream: c.config.streamName,
|
||||
Consumer: c.config.consumerName,
|
||||
Subject: msg.Subject,
|
||||
JSSequence: jsSeq,
|
||||
})
|
||||
@@ -74,7 +123,7 @@ func (c *Consumer) resolveMsgID(msg *nats.Msg) (string, string, error) {
|
||||
return id, "header", nil
|
||||
}
|
||||
|
||||
if c.mode == "core" {
|
||||
if c.config.mode == "core" {
|
||||
return uuid.NewString(), "generated", nil
|
||||
}
|
||||
|
||||
@@ -85,4 +134,3 @@ func (c *Consumer) resolveMsgID(msg *nats.Msg) (string, string, error) {
|
||||
|
||||
return fmt.Sprintf("js-%d", meta.Sequence.Stream), "metadata", nil
|
||||
}
|
||||
|
||||
@@ -49,12 +49,12 @@ func (c *Consumer) recordConsumerMetrics(ctx context.Context, info *nats.Consume
|
||||
|
||||
// Export an explicit pending messages gauge for Prometheus-based lag /
|
||||
// backlog alerts.
|
||||
obsmetrics.RecordNATSConsumerPending(c.streamName, c.consumerName, info.NumPending)
|
||||
obsmetrics.RecordNATSConsumerPending(c.config.streamName, c.config.consumerName, info.NumPending)
|
||||
}
|
||||
|
||||
// emitConsumerStats periodically emits consumer statistics.
|
||||
func (c *Consumer) emitConsumerStats(ctx context.Context) {
|
||||
ticker := time.NewTicker(c.monitorInterval)
|
||||
ticker := time.NewTicker(c.config.monitorInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
@@ -62,15 +62,15 @@ func (c *Consumer) emitConsumerStats(ctx context.Context) {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
info, err := c.js.ConsumerInfo(c.streamName, c.consumerName)
|
||||
info, err := c.js.ConsumerInfo(c.config.streamName, c.config.consumerName)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to fetch consumer info", zap.Error(err))
|
||||
continue
|
||||
}
|
||||
|
||||
c.logger.Debug("JetStream consumer metrics",
|
||||
zap.String("stream", c.streamName),
|
||||
zap.String("consumer", c.consumerName),
|
||||
zap.String("stream", c.config.streamName),
|
||||
zap.String("consumer", c.config.consumerName),
|
||||
zap.Uint64("num_ack_pending", uint64(info.NumAckPending)),
|
||||
zap.Uint64("num_redelivered", uint64(info.NumRedelivered)),
|
||||
zap.Uint64("num_pending", uint64(info.NumPending)),
|
||||
+25
-25
@@ -21,22 +21,22 @@ func (c *Consumer) validateDLQ() error {
|
||||
}
|
||||
|
||||
// DLQ routing is only active in JetStream mode.
|
||||
if c.mode != "jetstream" {
|
||||
if c.config.mode != "jetstream" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// If DLQ is not enabled in config, make sure we don't accidentally route to it.
|
||||
if !c.cfg.DLQ.Enabled {
|
||||
if strings.TrimSpace(c.dlqSubject) != "" {
|
||||
if strings.TrimSpace(c.config.dlqSubject) != "" {
|
||||
c.logger.Info("DLQ subject configured but dlq.enabled is false; DLQ routing disabled",
|
||||
zap.String("dlq_subject", c.dlqSubject),
|
||||
zap.String("dlq_subject", c.config.dlqSubject),
|
||||
)
|
||||
}
|
||||
c.dlqSubject = ""
|
||||
c.config.dlqSubject = ""
|
||||
return nil
|
||||
}
|
||||
|
||||
subject := strings.TrimSpace(c.dlqSubject)
|
||||
subject := strings.TrimSpace(c.config.dlqSubject)
|
||||
if subject == "" {
|
||||
return fmt.Errorf("DLQ enabled but dlq.subject is empty")
|
||||
}
|
||||
@@ -68,10 +68,10 @@ func (c *Consumer) routeToDLQ(ctx context.Context, msg *nats.Msg, cause error) e
|
||||
if c == nil || c.js == nil {
|
||||
return nil
|
||||
}
|
||||
if c.mode != "jetstream" {
|
||||
if c.config.mode != "jetstream" {
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(c.dlqSubject) == "" {
|
||||
if strings.TrimSpace(c.config.dlqSubject) == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -83,11 +83,11 @@ func (c *Consumer) routeToDLQ(ctx context.Context, msg *nats.Msg, cause error) e
|
||||
deliveries = meta.NumDelivered
|
||||
}
|
||||
|
||||
payload := map[string]interface{}{
|
||||
payload := map[string]any{
|
||||
"transport_msg_id": msg.Header.Get("Nats-Msg-Id"),
|
||||
"subject": msg.Subject,
|
||||
"stream": c.streamName,
|
||||
"consumer": c.consumerName,
|
||||
"stream": c.config.streamName,
|
||||
"consumer": c.config.consumerName,
|
||||
"nats_sequence": jsSeq,
|
||||
"deliveries": deliveries,
|
||||
"error": fmt.Sprint(cause),
|
||||
@@ -98,42 +98,42 @@ func (c *Consumer) routeToDLQ(ctx context.Context, msg *nats.Msg, cause error) e
|
||||
data, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
c.logger.Error("failed to marshal DLQ payload",
|
||||
zap.String("stream", c.streamName),
|
||||
zap.String("consumer", c.consumerName),
|
||||
zap.String("dlq_subject", c.dlqSubject),
|
||||
zap.String("stream", c.config.streamName),
|
||||
zap.String("consumer", c.config.consumerName),
|
||||
zap.String("dlq_subject", c.config.dlqSubject),
|
||||
zap.Error(err),
|
||||
)
|
||||
return fmt.Errorf("marshal dlq payload: %w", err)
|
||||
}
|
||||
|
||||
if _, err := c.js.Publish(c.dlqSubject, data); err != nil {
|
||||
if _, err := c.js.Publish(c.config.dlqSubject, data); err != nil {
|
||||
// nats.ErrNoResponders typically means that no JetStream stream is
|
||||
// configured to receive this subject, or JetStream is temporarily
|
||||
// unavailable. Surface this explicitly to make operational diagnosis
|
||||
// easier.
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
c.logger.Error("transient DLQ publish error (no responders)",
|
||||
zap.String("stream", c.streamName),
|
||||
zap.String("consumer", c.consumerName),
|
||||
zap.String("dlq_subject", c.dlqSubject),
|
||||
zap.String("stream", c.config.streamName),
|
||||
zap.String("consumer", c.config.consumerName),
|
||||
zap.String("dlq_subject", c.config.dlqSubject),
|
||||
zap.Int("payload_size", len(data)),
|
||||
zap.Error(err),
|
||||
)
|
||||
c.telemetry.RecordDLQPublishFailure(ctx, c.streamName, c.consumerName)
|
||||
return fmt.Errorf("publish to dlq subject %s: no JetStream stream found for subject or JetStream unavailable: %w", c.dlqSubject, err)
|
||||
c.telemetry.RecordDLQPublishFailure(ctx, c.config.streamName, c.config.consumerName)
|
||||
return fmt.Errorf("publish to dlq subject %s: no JetStream stream found for subject or JetStream unavailable: %w", c.config.dlqSubject, err)
|
||||
}
|
||||
c.logger.Error("failed to publish to DLQ",
|
||||
zap.String("stream", c.streamName),
|
||||
zap.String("consumer", c.consumerName),
|
||||
zap.String("dlq_subject", c.dlqSubject),
|
||||
zap.String("stream", c.config.streamName),
|
||||
zap.String("consumer", c.config.consumerName),
|
||||
zap.String("dlq_subject", c.config.dlqSubject),
|
||||
zap.Int("payload_size", len(data)),
|
||||
zap.Error(err),
|
||||
)
|
||||
c.telemetry.RecordDLQPublishFailure(ctx, c.streamName, c.consumerName)
|
||||
return fmt.Errorf("publish to dlq subject %s: %w", c.dlqSubject, err)
|
||||
c.telemetry.RecordDLQPublishFailure(ctx, c.config.streamName, c.config.consumerName)
|
||||
return fmt.Errorf("publish to dlq subject %s: %w", c.config.dlqSubject, err)
|
||||
}
|
||||
|
||||
c.telemetry.RecordDLQMessage(ctx, c.streamName, c.consumerName)
|
||||
c.telemetry.RecordDLQMessage(ctx, c.config.streamName, c.config.consumerName)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -7,9 +7,9 @@ import (
|
||||
configpkg "caatsm/internal/infra/config"
|
||||
"caatsm/internal/infra/telemetry"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/nats-io/nats.go"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zaptest"
|
||||
)
|
||||
@@ -31,7 +31,9 @@ var _ = Describe("DLQ", func() {
|
||||
|
||||
It("returns nil when mode is not jetstream", func() {
|
||||
c := &Consumer{
|
||||
mode: "core",
|
||||
config: consumerConfig{
|
||||
mode: "core",
|
||||
},
|
||||
cfg: &configpkg.Config{
|
||||
DLQ: configpkg.DLQConfig{
|
||||
Enabled: true,
|
||||
@@ -45,8 +47,10 @@ var _ = Describe("DLQ", func() {
|
||||
|
||||
It("clears dlqSubject when DLQ is disabled", func() {
|
||||
c := &Consumer{
|
||||
mode: "jetstream",
|
||||
dlqSubject: "caatsm.dlq",
|
||||
config: consumerConfig{
|
||||
mode: "jetstream",
|
||||
dlqSubject: "caatsm.dlq",
|
||||
},
|
||||
cfg: &configpkg.Config{
|
||||
DLQ: configpkg.DLQConfig{
|
||||
Enabled: false,
|
||||
@@ -56,13 +60,15 @@ var _ = Describe("DLQ", func() {
|
||||
logger: logger,
|
||||
}
|
||||
Expect(c.validateDLQ()).To(Succeed())
|
||||
Expect(c.dlqSubject).To(Equal(""))
|
||||
Expect(c.config.dlqSubject).To(Equal(""))
|
||||
})
|
||||
|
||||
It("returns error when DLQ is enabled but subject is empty", func() {
|
||||
c := &Consumer{
|
||||
mode: "jetstream",
|
||||
dlqSubject: "",
|
||||
config: consumerConfig{
|
||||
mode: "jetstream",
|
||||
dlqSubject: "",
|
||||
},
|
||||
cfg: &configpkg.Config{
|
||||
DLQ: configpkg.DLQConfig{
|
||||
Enabled: true,
|
||||
@@ -78,9 +84,11 @@ var _ = Describe("DLQ", func() {
|
||||
|
||||
It("returns error when DLQ is enabled but JetStream context is nil", func() {
|
||||
c := &Consumer{
|
||||
mode: "jetstream",
|
||||
dlqSubject: "caatsm.dlq",
|
||||
js: nil,
|
||||
config: consumerConfig{
|
||||
mode: "jetstream",
|
||||
dlqSubject: "caatsm.dlq",
|
||||
},
|
||||
js: nil,
|
||||
cfg: &configpkg.Config{
|
||||
DLQ: configpkg.DLQConfig{
|
||||
Enabled: true,
|
||||
@@ -107,7 +115,9 @@ var _ = Describe("DLQ", func() {
|
||||
|
||||
It("returns nil when mode is not jetstream", func() {
|
||||
c := &Consumer{
|
||||
mode: "core",
|
||||
config: consumerConfig{
|
||||
mode: "core",
|
||||
},
|
||||
}
|
||||
ctx := context.Background()
|
||||
msg := &nats.Msg{}
|
||||
@@ -117,9 +127,11 @@ var _ = Describe("DLQ", func() {
|
||||
|
||||
It("returns nil when dlqSubject is empty", func() {
|
||||
c := &Consumer{
|
||||
mode: "jetstream",
|
||||
dlqSubject: "",
|
||||
js: nil, // Can be nil for this test
|
||||
config: consumerConfig{
|
||||
mode: "jetstream",
|
||||
dlqSubject: "",
|
||||
},
|
||||
js: nil, // Can be nil for this test
|
||||
}
|
||||
ctx := context.Background()
|
||||
msg := &nats.Msg{}
|
||||
@@ -128,4 +140,3 @@ var _ = Describe("DLQ", func() {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -47,9 +47,7 @@ func (h *ErrorHandler) HandleFetchError(
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
*fetchErrorStreak++
|
||||
backoff := time.Duration(*fetchErrorStreak) * time.Second
|
||||
if backoff > 30*time.Second {
|
||||
backoff = 30 * time.Second
|
||||
}
|
||||
backoff = min(backoff, 30*time.Second)
|
||||
h.logger.Warn("JetStream not available, will retry with backoff",
|
||||
zap.Error(err),
|
||||
zap.String("stream", streamName),
|
||||
@@ -91,9 +89,7 @@ func (h *ErrorHandler) HandleFetchError(
|
||||
// Generic error path with modest backoff.
|
||||
*fetchErrorStreak++
|
||||
backoff := time.Duration(*fetchErrorStreak) * time.Second
|
||||
if backoff > 10*time.Second {
|
||||
backoff = 10 * time.Second
|
||||
}
|
||||
backoff = min(backoff, 10*time.Second)
|
||||
h.logger.Error("Failed to fetch messages; backing off",
|
||||
zap.Error(err),
|
||||
zap.Duration("backoff", backoff),
|
||||
@@ -133,9 +129,7 @@ func (h *ErrorHandler) HandleProcessingError(
|
||||
if consecutiveErrors >= 10 {
|
||||
result.ShouldApplyBackpressure = true
|
||||
result.BackpressureDelay = time.Duration(consecutiveErrors) * 100 * time.Millisecond
|
||||
if result.BackpressureDelay > 5*time.Second {
|
||||
result.BackpressureDelay = 5 * time.Second
|
||||
}
|
||||
result.BackpressureDelay = min(result.BackpressureDelay, 5*time.Second)
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
@@ -2,9 +2,7 @@ package nats
|
||||
|
||||
import (
|
||||
"caatsm/internal/infra/config"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -65,22 +63,7 @@ func ProvideJetStream(nc *nats.Conn, cfg *config.Config, logger *zap.Logger) (na
|
||||
return nil, fmt.Errorf("failed to get JetStream context: %w", err)
|
||||
}
|
||||
|
||||
// Ensure the stream exists and is minimally aligned with configuration.
|
||||
if err := EnsureStream(js, cfg, logger); err != nil {
|
||||
nc.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return js, nil
|
||||
}
|
||||
|
||||
// EnsureStream ensures that the configured JetStream stream exists and has
|
||||
// at least the expected subjects bound. It is safe to call multiple times.
|
||||
//
|
||||
// In dev/test environments (see shouldBootstrapStream), the stream will be
|
||||
// auto-created if it does not exist. In production, a missing stream results
|
||||
// in an error so that operators can intervene.
|
||||
func EnsureStream(js nats.JetStreamContext, cfg *config.Config, logger *zap.Logger) error {
|
||||
// Ensure the stream exists using StreamManager
|
||||
streamName := cfg.NATS.Stream
|
||||
consumerSubject := cfg.EffectiveSubscriptionTopic()
|
||||
publisherSubject := strings.TrimSpace(cfg.Publisher.Topic)
|
||||
@@ -92,130 +75,15 @@ func EnsureStream(js nats.JetStreamContext, cfg *config.Config, logger *zap.Logg
|
||||
zap.String("consumer_subject", consumerSubject),
|
||||
zap.String("publisher_subject", publisherSubject),
|
||||
)
|
||||
return fmt.Errorf("no subjects configured for JetStream stream %s", streamName)
|
||||
nc.Close()
|
||||
return nil, fmt.Errorf("no subjects configured for JetStream stream %s", streamName)
|
||||
}
|
||||
|
||||
streamLimits := cfg.NATS.StreamLimits
|
||||
storage := nats.FileStorage
|
||||
switch strings.ToLower(streamLimits.Storage) {
|
||||
case "memory":
|
||||
storage = nats.MemoryStorage
|
||||
case "file":
|
||||
storage = nats.FileStorage
|
||||
streamManager := NewStreamManagerWithConfig(js, streamName, streamSubjects, &cfg.NATS.StreamLimits, logger)
|
||||
if err := streamManager.EnsureStream(); err != nil {
|
||||
nc.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
discard := nats.DiscardOld
|
||||
if strings.EqualFold(streamLimits.Discard, "new") {
|
||||
discard = nats.DiscardNew
|
||||
}
|
||||
|
||||
streamConfig := &nats.StreamConfig{
|
||||
Name: streamName,
|
||||
Subjects: streamSubjects,
|
||||
Retention: nats.LimitsPolicy,
|
||||
MaxMsgs: streamLimits.MaxMsgs,
|
||||
MaxBytes: streamLimits.MaxBytes,
|
||||
MaxAge: streamLimits.MaxAge,
|
||||
Discard: discard,
|
||||
Storage: storage,
|
||||
Replicas: streamLimits.Replicas,
|
||||
}
|
||||
|
||||
info, err := js.StreamInfo(streamName)
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrStreamNotFound) {
|
||||
if shouldBootstrapStream() {
|
||||
if _, err = js.AddStream(streamConfig); err != nil {
|
||||
logger.Error("failed to create stream",
|
||||
zap.String("stream", streamName),
|
||||
zap.Strings("subjects", streamSubjects),
|
||||
zap.Error(err),
|
||||
)
|
||||
return fmt.Errorf("failed to create stream %s: %w", streamName, err)
|
||||
}
|
||||
logger.Info("Created JetStream stream",
|
||||
zap.String("stream", streamName),
|
||||
zap.Strings("subjects", streamSubjects),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
logger.Error("stream not found and auto-creation disabled",
|
||||
zap.String("stream", streamName),
|
||||
zap.Strings("expected_subjects", streamSubjects),
|
||||
)
|
||||
return fmt.Errorf("stream %s not found and auto-creation disabled", streamName)
|
||||
}
|
||||
logger.Error("failed to fetch stream info",
|
||||
zap.String("stream", streamName),
|
||||
zap.Error(err),
|
||||
)
|
||||
return fmt.Errorf("failed to fetch stream info for %s: %w", streamName, err)
|
||||
}
|
||||
|
||||
// Stream exists: validate subjects but do not fail hard if they differ.
|
||||
validateStreamConfig(info, streamSubjects, logger)
|
||||
return nil
|
||||
}
|
||||
|
||||
func shouldBootstrapStream() bool {
|
||||
switch strings.ToLower(os.Getenv("GO_ENV")) {
|
||||
case "", "dev", "development", "test", "testing":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func validateStreamConfig(info *nats.StreamInfo, expectedSubjects []string, logger *zap.Logger) {
|
||||
if info == nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if len(expectedSubjects) == 0 {
|
||||
expectedSubjects = []string{"<none>"}
|
||||
}
|
||||
}()
|
||||
|
||||
missing := make([]string, 0)
|
||||
for _, subj := range expectedSubjects {
|
||||
if subj == "" {
|
||||
continue
|
||||
}
|
||||
if !containsSubject(info.Config.Subjects, subj) {
|
||||
missing = append(missing, subj)
|
||||
}
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
logger.Warn("JetStream stream subjects missing expected entries",
|
||||
zap.String("stream", info.Config.Name),
|
||||
zap.Strings("stream_subjects", info.Config.Subjects),
|
||||
zap.Strings("missing_subjects", missing),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func containsSubject(subjects []string, target string) bool {
|
||||
for _, s := range subjects {
|
||||
if s == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func dedupeSubjects(subjects []string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
result := make([]string, 0, len(subjects))
|
||||
for _, subj := range subjects {
|
||||
subj = strings.TrimSpace(subj)
|
||||
if subj == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[subj]; ok {
|
||||
continue
|
||||
}
|
||||
seen[subj] = struct{}{}
|
||||
result = append(result, subj)
|
||||
}
|
||||
return result
|
||||
return js, nil
|
||||
}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
package nats
|
||||
|
||||
import (
|
||||
"github.com/nats-io/nats.go"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/nats-io/nats.go"
|
||||
"go.uber.org/zap/zaptest"
|
||||
)
|
||||
|
||||
@@ -14,10 +14,12 @@ var _ = Describe("MessageHandler", func() {
|
||||
|
||||
BeforeEach(func() {
|
||||
c = &Consumer{
|
||||
mode: "jetstream",
|
||||
streamName: "TEST_STREAM",
|
||||
consumerName: "test-consumer",
|
||||
logger: zaptest.NewLogger(GinkgoT()),
|
||||
config: consumerConfig{
|
||||
mode: "jetstream",
|
||||
streamName: "TEST_STREAM",
|
||||
consumerName: "test-consumer",
|
||||
},
|
||||
logger: zaptest.NewLogger(GinkgoT()),
|
||||
}
|
||||
})
|
||||
|
||||
@@ -35,7 +37,7 @@ var _ = Describe("MessageHandler", func() {
|
||||
})
|
||||
|
||||
It("generates UUID for core mode when header is missing", func() {
|
||||
c.mode = "core"
|
||||
c.config.mode = "core"
|
||||
msg := &nats.Msg{
|
||||
Header: nats.Header{},
|
||||
}
|
||||
@@ -47,7 +49,7 @@ var _ = Describe("MessageHandler", func() {
|
||||
})
|
||||
|
||||
It("returns error for JetStream mode when header and metadata are missing", func() {
|
||||
c.mode = "jetstream"
|
||||
c.config.mode = "jetstream"
|
||||
msg := &nats.Msg{
|
||||
Header: nats.Header{},
|
||||
}
|
||||
@@ -59,4 +61,3 @@ var _ = Describe("MessageHandler", func() {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -3,9 +3,9 @@ package nats
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/nats-io/nats.go"
|
||||
"go.uber.org/zap/zaptest"
|
||||
)
|
||||
|
||||
@@ -18,9 +18,11 @@ var _ = Describe("Metrics", func() {
|
||||
BeforeEach(func() {
|
||||
ctx = context.Background()
|
||||
c = &Consumer{
|
||||
streamName: "TEST_STREAM",
|
||||
consumerName: "test-consumer",
|
||||
logger: zaptest.NewLogger(GinkgoT()),
|
||||
config: consumerConfig{
|
||||
streamName: "TEST_STREAM",
|
||||
consumerName: "test-consumer",
|
||||
},
|
||||
logger: zaptest.NewLogger(GinkgoT()),
|
||||
}
|
||||
})
|
||||
|
||||
@@ -54,4 +56,3 @@ var _ = Describe("Metrics", func() {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package nats
|
||||
|
||||
import (
|
||||
"caatsm/internal/infra/config"
|
||||
"caatsm/internal/adapter/dto"
|
||||
"caatsm/internal/infra/config"
|
||||
"caatsm/internal/port"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -59,7 +59,7 @@ func ProvideCorePublisher(
|
||||
}
|
||||
|
||||
// Publish publishes a message using plain NATS
|
||||
func (p *CorePublisher) Publish(message interface{}) error {
|
||||
func (p *CorePublisher) Publish(message any) error {
|
||||
topic := p.cfg.Publisher.Topic
|
||||
if topic == "" {
|
||||
p.logger.Error("publisher topic is not configured")
|
||||
@@ -96,7 +96,7 @@ func (p *CorePublisher) Publish(message interface{}) error {
|
||||
}
|
||||
|
||||
// Publish publishes a message
|
||||
func (p *Publisher) Publish(message interface{}) error {
|
||||
func (p *Publisher) Publish(message any) error {
|
||||
topic := p.cfg.Publisher.Topic
|
||||
if topic == "" {
|
||||
p.logger.Error("publisher topic is not configured")
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package nats
|
||||
|
||||
import (
|
||||
"caatsm/internal/infra/config"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"go.uber.org/zap"
|
||||
@@ -14,6 +16,7 @@ type StreamManager struct {
|
||||
streamName string
|
||||
subjects []string
|
||||
logger *zap.Logger
|
||||
cfg *config.StreamLimitsConfig
|
||||
}
|
||||
|
||||
// NewStreamManager creates a new stream manager
|
||||
@@ -26,14 +29,21 @@ func NewStreamManager(js nats.JetStreamContext, streamName string, subjects []st
|
||||
}
|
||||
}
|
||||
|
||||
// NewStreamManagerWithConfig creates a new stream manager with full stream configuration
|
||||
func NewStreamManagerWithConfig(js nats.JetStreamContext, streamName string, subjects []string, streamLimits *config.StreamLimitsConfig, logger *zap.Logger) *StreamManager {
|
||||
return &StreamManager{
|
||||
js: js,
|
||||
streamName: streamName,
|
||||
subjects: subjects,
|
||||
logger: logger,
|
||||
cfg: streamLimits,
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureStream ensures that the configured JetStream stream exists
|
||||
func (sm *StreamManager) EnsureStream() error {
|
||||
streamConfig := &nats.StreamConfig{
|
||||
Name: sm.streamName,
|
||||
Subjects: sm.subjects,
|
||||
Retention: nats.LimitsPolicy,
|
||||
Storage: nats.FileStorage,
|
||||
}
|
||||
// Build stream configuration
|
||||
streamConfig := sm.buildStreamConfig()
|
||||
|
||||
info, err := sm.js.StreamInfo(sm.streamName)
|
||||
if err != nil {
|
||||
@@ -71,6 +81,41 @@ func (sm *StreamManager) EnsureStream() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildStreamConfig builds the stream configuration from manager settings
|
||||
func (sm *StreamManager) buildStreamConfig() *nats.StreamConfig {
|
||||
config := &nats.StreamConfig{
|
||||
Name: sm.streamName,
|
||||
Subjects: sm.subjects,
|
||||
Retention: nats.LimitsPolicy,
|
||||
Storage: nats.FileStorage,
|
||||
}
|
||||
|
||||
// Apply stream limits configuration if provided
|
||||
if sm.cfg != nil {
|
||||
config.MaxMsgs = sm.cfg.MaxMsgs
|
||||
config.MaxBytes = sm.cfg.MaxBytes
|
||||
config.MaxAge = sm.cfg.MaxAge
|
||||
config.Replicas = sm.cfg.Replicas
|
||||
|
||||
// Map storage type
|
||||
switch strings.ToLower(sm.cfg.Storage) {
|
||||
case "memory":
|
||||
config.Storage = nats.MemoryStorage
|
||||
case "file":
|
||||
config.Storage = nats.FileStorage
|
||||
}
|
||||
|
||||
// Map discard policy
|
||||
if strings.EqualFold(sm.cfg.Discard, "new") {
|
||||
config.Discard = nats.DiscardNew
|
||||
} else {
|
||||
config.Discard = nats.DiscardOld
|
||||
}
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
// validateStreamConfig validates the stream configuration
|
||||
func (sm *StreamManager) validateStreamConfig(info *nats.StreamInfo) {
|
||||
if info == nil {
|
||||
|
||||
@@ -78,3 +78,41 @@ func mapReplayPolicy(value string) nats.ReplayPolicy {
|
||||
return nats.ReplayInstantPolicy
|
||||
}
|
||||
}
|
||||
|
||||
// shouldBootstrapStream checks if streams should be auto-created based on environment.
|
||||
func shouldBootstrapStream() bool {
|
||||
switch strings.ToLower(os.Getenv("GO_ENV")) {
|
||||
case "", "dev", "development", "test", "testing":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// containsSubject checks if a subject exists in a list of subjects.
|
||||
func containsSubject(subjects []string, target string) bool {
|
||||
for _, s := range subjects {
|
||||
if s == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// dedupeSubjects removes duplicate and empty subjects from a list.
|
||||
func dedupeSubjects(subjects []string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
result := make([]string, 0, len(subjects))
|
||||
for _, subj := range subjects {
|
||||
subj = strings.TrimSpace(subj)
|
||||
if subj == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[subj]; ok {
|
||||
continue
|
||||
}
|
||||
seen[subj] = struct{}{}
|
||||
result = append(result, subj)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -84,4 +84,3 @@ var _ = Describe("Utils", func() {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -39,7 +39,16 @@ func ProvideRepository(pool *pgxpool.Pool, logger *zap.Logger) (port.Repository,
|
||||
func (r *Repository) InsertOne(ctx context.Context, msg *dto.ParsedTelegram) error {
|
||||
ctx, span := otel.Tracer("caatsm/postgres").Start(ctx, "Repository.InsertOne")
|
||||
defer span.End()
|
||||
span.SetAttributes(attribute.String("db.table", "aviation.telegrams"))
|
||||
|
||||
// Set semantic database attributes
|
||||
span.SetAttributes(
|
||||
attribute.String("db.system", "postgresql"),
|
||||
attribute.String("db.operation", "insert"),
|
||||
attribute.String("db.name", "aviation"),
|
||||
attribute.String("db.table", "telegrams"),
|
||||
attribute.String("caatsm.message.id", msg.MessageID),
|
||||
attribute.String("caatsm.message.category", msg.Category),
|
||||
)
|
||||
|
||||
// Optional idempotency check based on business message identity. If we have a
|
||||
// non-empty message ID and date/time, we can cheaply skip duplicates here to
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"caatsm/internal/infra/buildinfo"
|
||||
"caatsm/internal/infra/config"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
|
||||
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
var (
|
||||
// instanceID stores a unique identifier for this running process instance.
|
||||
// It is initialized once at startup and remains stable for the lifetime of the process.
|
||||
instanceID string
|
||||
instanceIDOnce sync.Once
|
||||
)
|
||||
|
||||
// InitOTEL initializes the OpenTelemetry SDK with proper resource attributes,
|
||||
// sampling configuration, and exporters. This should be called once at application startup.
|
||||
// logger is optional; if provided, sensitive telemetry configuration will be logged at debug level.
|
||||
func InitOTEL(ctx context.Context, cfg *config.Config, logger *zap.Logger) error {
|
||||
if !cfg.Telemetry.Enabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Log sensitive telemetry configuration to internal debug logs only
|
||||
if logger != nil {
|
||||
logger.Debug("Initializing OpenTelemetry",
|
||||
zap.String("telemetry.endpoint", cfg.Telemetry.Endpoint),
|
||||
zap.Bool("telemetry.insecure", cfg.Telemetry.Insecure),
|
||||
)
|
||||
}
|
||||
|
||||
// Create resource with comprehensive service information
|
||||
res, err := createResource(ctx, cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create OTEL resource: %w", err)
|
||||
}
|
||||
|
||||
// Initialize tracing
|
||||
if err := initTracing(ctx, cfg, res); err != nil {
|
||||
return fmt.Errorf("failed to initialize tracing: %w", err)
|
||||
}
|
||||
|
||||
// Initialize metrics
|
||||
if err := initMetrics(ctx, cfg, res); err != nil {
|
||||
return fmt.Errorf("failed to initialize metrics: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getInstanceID returns a unique identifier for this running process instance.
|
||||
// It checks environment variables (POD_NAME, CONTAINER_ID, HOSTNAME) first, then generates
|
||||
// a UUID if no environment variable is available. The ID is initialized once and remains
|
||||
// stable for the lifetime of the process.
|
||||
func getInstanceID() string {
|
||||
instanceIDOnce.Do(func() {
|
||||
// Check for Kubernetes pod name first (most common in containerized deployments)
|
||||
if podName := os.Getenv("POD_NAME"); podName != "" {
|
||||
instanceID = podName
|
||||
return
|
||||
}
|
||||
|
||||
// Check for container ID (Docker, containerd, etc.)
|
||||
if containerID := os.Getenv("CONTAINER_ID"); containerID != "" {
|
||||
instanceID = containerID
|
||||
return
|
||||
}
|
||||
|
||||
// Check for HOSTNAME (often set in containers)
|
||||
if hostname := os.Getenv("HOSTNAME"); hostname != "" {
|
||||
// Use hostname if it's not a generic default
|
||||
if hostname != "localhost" && hostname != "localhost.localdomain" {
|
||||
instanceID = hostname
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Generate a UUID for this process instance
|
||||
instanceID = uuid.NewString()
|
||||
})
|
||||
|
||||
return instanceID
|
||||
}
|
||||
|
||||
// createResource creates a resource with standard and custom attributes.
|
||||
// Sensitive infrastructure details (endpoint, insecure flag) are excluded from resource
|
||||
// attributes to prevent leakage. These values are logged internally at debug level if
|
||||
// a logger is provided to InitOTEL.
|
||||
func createResource(ctx context.Context, cfg *config.Config) (*resource.Resource, error) {
|
||||
// Get the runtime instance ID (falls back to buildinfo.Commit if needed)
|
||||
runtimeInstanceID := getInstanceID()
|
||||
if runtimeInstanceID == "" {
|
||||
// Final fallback to build commit if somehow instance ID is empty
|
||||
runtimeInstanceID = buildinfo.Commit
|
||||
}
|
||||
|
||||
attrs := []attribute.KeyValue{
|
||||
// Standard semantic conventions
|
||||
semconv.ServiceName("caatsm"),
|
||||
semconv.ServiceVersion(buildinfo.Version),
|
||||
semconv.ServiceInstanceID(runtimeInstanceID),
|
||||
semconv.ServiceNamespace("airport"),
|
||||
|
||||
// Custom attributes
|
||||
attribute.String("service.component", "receiver"),
|
||||
attribute.String("service.environment", getEnvironment()),
|
||||
attribute.String("build.commit", buildinfo.Commit),
|
||||
attribute.String("build.built_at", buildinfo.BuiltAt),
|
||||
}
|
||||
|
||||
// Add non-sensitive indicator for telemetry endpoint configuration
|
||||
// (without exposing the actual endpoint value)
|
||||
if cfg.Telemetry.Endpoint != "" {
|
||||
attrs = append(attrs, attribute.Bool("telemetry.endpoint.configured", true))
|
||||
} else {
|
||||
attrs = append(attrs, attribute.Bool("telemetry.endpoint.configured", false))
|
||||
}
|
||||
|
||||
return resource.New(ctx, resource.WithAttributes(attrs...))
|
||||
}
|
||||
|
||||
// initTracing sets up the trace provider with appropriate sampling
|
||||
func initTracing(ctx context.Context, cfg *config.Config, res *resource.Resource) error {
|
||||
var traceExporterOptions []otlptracehttp.Option
|
||||
|
||||
traceExporterOptions = append(traceExporterOptions, otlptracehttp.WithEndpoint(cfg.Telemetry.Endpoint))
|
||||
|
||||
if cfg.Telemetry.Insecure {
|
||||
traceExporterOptions = append(traceExporterOptions, otlptracehttp.WithTLSClientConfig(&tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
MinVersion: tls.VersionTLS13,
|
||||
}))
|
||||
}
|
||||
|
||||
traceExporter, err := otlptracehttp.New(ctx, traceExporterOptions...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create trace exporter: %w", err)
|
||||
}
|
||||
|
||||
// Configure sampling based on environment
|
||||
sampler := getSamplerForEnvironment(getEnvironment())
|
||||
|
||||
tracerProvider := sdktrace.NewTracerProvider(
|
||||
sdktrace.WithBatcher(traceExporter,
|
||||
sdktrace.WithBatchTimeout(1*time.Second),
|
||||
sdktrace.WithMaxExportBatchSize(512),
|
||||
sdktrace.WithMaxQueueSize(2048),
|
||||
),
|
||||
sdktrace.WithResource(res),
|
||||
sdktrace.WithSampler(sdktrace.ParentBased(sampler)),
|
||||
)
|
||||
|
||||
otel.SetTracerProvider(tracerProvider)
|
||||
return nil
|
||||
}
|
||||
|
||||
// initMetrics sets up the meter provider
|
||||
func initMetrics(ctx context.Context, cfg *config.Config, res *resource.Resource) error {
|
||||
var metricExporterOptions []otlpmetrichttp.Option
|
||||
|
||||
metricExporterOptions = append(metricExporterOptions, otlpmetrichttp.WithEndpoint(cfg.Telemetry.Endpoint))
|
||||
|
||||
if cfg.Telemetry.Insecure {
|
||||
metricExporterOptions = append(metricExporterOptions, otlpmetrichttp.WithTLSClientConfig(&tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
MinVersion: tls.VersionTLS12,
|
||||
}))
|
||||
}
|
||||
|
||||
metricExporter, err := otlpmetrichttp.New(ctx, metricExporterOptions...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create metric exporter: %w", err)
|
||||
}
|
||||
|
||||
meterProvider := sdkmetric.NewMeterProvider(
|
||||
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExporter,
|
||||
sdkmetric.WithInterval(30*time.Second),
|
||||
)),
|
||||
sdkmetric.WithResource(res),
|
||||
)
|
||||
|
||||
otel.SetMeterProvider(meterProvider)
|
||||
return nil
|
||||
}
|
||||
|
||||
// getSamplerForEnvironment returns appropriate sampling strategy for each environment
|
||||
func getSamplerForEnvironment(env string) sdktrace.Sampler {
|
||||
switch env {
|
||||
case "prod", "production":
|
||||
// 1% sampling in production to control costs and performance
|
||||
return sdktrace.TraceIDRatioBased(0.01)
|
||||
case "staging":
|
||||
// 10% sampling in staging for better observability
|
||||
return sdktrace.TraceIDRatioBased(0.1)
|
||||
case "test", "testing":
|
||||
// Always sample in testing for complete coverage
|
||||
return sdktrace.AlwaysSample()
|
||||
default:
|
||||
// 100% sampling in development for debugging
|
||||
return sdktrace.AlwaysSample()
|
||||
}
|
||||
}
|
||||
|
||||
// ShutdownOTEL gracefully shuts down the OTEL providers
|
||||
func ShutdownOTEL(ctx context.Context) error {
|
||||
var errs []error
|
||||
|
||||
if tracerProvider, ok := otel.GetTracerProvider().(*sdktrace.TracerProvider); ok {
|
||||
if err := tracerProvider.Shutdown(ctx); err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to shutdown tracer provider: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
if meterProvider, ok := otel.GetMeterProvider().(*sdkmetric.MeterProvider); ok {
|
||||
if err := meterProvider.Shutdown(ctx); err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to shutdown meter provider: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
if len(errs) > 0 {
|
||||
return fmt.Errorf("OTEL shutdown errors: %v", errs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getEnvironment returns the current environment from GO_ENV
|
||||
func getEnvironment() string {
|
||||
if env := os.Getenv("GO_ENV"); env != "" {
|
||||
return env
|
||||
}
|
||||
return "dev"
|
||||
}
|
||||
Reference in New Issue
Block a user