Enhance observability and error handling in NATS integration. Introduce comprehensive OpenTelemetry support with environment-based sampling and semantic attributes for tracing and metrics. Implement an advisory dead-letter queue (DLQ) handler for managing message delivery failures. Update NATS consumer to utilize structured logging and improve error handling strategies. Refactor configuration files for OpenTelemetry collector in both development and production environments, ensuring robust telemetry integration. Enhance documentation to reflect new features and best practices for observability.

This commit is contained in:
windyboy
2025-11-18 11:47:35 +08:00
parent 5d05237283
commit 7f44b5389d
31 changed files with 2546 additions and 476 deletions
+9 -2
View File
@@ -1,8 +1,8 @@
package app
import (
"caatsm/internal/adapter/parser"
"caatsm/internal/adapter/dto"
"caatsm/internal/adapter/parser"
"caatsm/internal/infra/log"
"caatsm/internal/infra/telemetry"
"caatsm/internal/port"
@@ -64,7 +64,14 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
tracer := otel.Tracer("caatsm/app")
ctx, span := tracer.Start(ctx, "MessageProcessor.Handle")
defer span.End()
span.SetAttributes(attribute.String("nats.msg_id", msgID))
// Set semantic attributes following OpenTelemetry conventions
span.SetAttributes(
attribute.String("messaging.system", "nats"),
attribute.String("messaging.operation", "receive"),
attribute.String("messaging.message_id", msgID),
attribute.String("caatsm.component", "processor"),
)
receivedAt := time.Now()
+167
View File
@@ -0,0 +1,167 @@
package nats
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
)
// MaxDeliveriesAdvisoryEvent represents the advisory message published when
// a message reaches MaxDeliver attempts.
type MaxDeliveriesAdvisoryEvent struct {
Type string `json:"type"`
Stream string `json:"stream"`
Consumer string `json:"consumer"`
StreamSeq uint64 `json:"stream_seq"`
Deliveries uint64 `json:"deliveries"`
Time string `json:"time"`
}
// AdvisoryDLQHandler handles messages that exhaust MaxDeliver attempts
// by subscribing to JetStream advisory events.
type AdvisoryDLQHandler struct {
js nats.JetStreamContext
nc *nats.Conn
streamName string
consumerName string
dlqSubject string
logger *zap.Logger
telemetry TelemetryRecorder
}
// TelemetryRecorder is an interface for recording telemetry events.
// This matches the telemetry.Recorder interface used by Consumer.
type TelemetryRecorder interface {
RecordDLQMessage(ctx context.Context, stream, consumer string)
RecordDLQPublishFailure(ctx context.Context, stream, consumer string)
}
// NewAdvisoryDLQHandler creates a new advisory-based DLQ handler.
func NewAdvisoryDLQHandler(
js nats.JetStreamContext,
nc *nats.Conn,
streamName string,
consumerName string,
dlqSubject string,
logger *zap.Logger,
telemetry TelemetryRecorder,
) (*AdvisoryDLQHandler, error) {
return &AdvisoryDLQHandler{
js: js,
nc: nc,
streamName: streamName,
consumerName: consumerName,
dlqSubject: dlqSubject,
logger: logger,
telemetry: telemetry,
}, nil
}
// Start begins listening for advisory messages and routing failed messages to DLQ.
func (h *AdvisoryDLQHandler) Start(ctx context.Context) error {
// Subscribe to advisory subject pattern
// Format: $JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.<STREAM>.<CONSUMER>
advisorySubject := fmt.Sprintf("$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.%s.%s",
h.streamName, h.consumerName)
h.logger.Info("Starting advisory DLQ handler",
zap.String("advisory_subject", advisorySubject),
zap.String("stream", h.streamName),
zap.String("consumer", h.consumerName),
zap.String("dlq_subject", h.dlqSubject),
)
sub, err := h.nc.Subscribe(advisorySubject, func(msg *nats.Msg) {
h.handleAdvisory(ctx, msg)
})
if err != nil {
return fmt.Errorf("failed to subscribe to advisory subject: %w", err)
}
// Wait for context cancellation
go func() {
<-ctx.Done()
sub.Unsubscribe()
h.logger.Info("Stopped advisory DLQ handler")
}()
return nil
}
// handleAdvisory processes an advisory message about max deliveries.
func (h *AdvisoryDLQHandler) handleAdvisory(ctx context.Context, advisoryMsg *nats.Msg) {
var event MaxDeliveriesAdvisoryEvent
if err := json.Unmarshal(advisoryMsg.Data, &event); err != nil {
h.logger.Error("Failed to unmarshal advisory event",
zap.Error(err),
zap.String("data", string(advisoryMsg.Data)),
)
return
}
h.logger.Warn("Message reached MaxDeliver attempts",
zap.String("stream", event.Stream),
zap.String("consumer", event.Consumer),
zap.Uint64("stream_seq", event.StreamSeq),
zap.Uint64("deliveries", event.Deliveries),
)
// Retrieve the original message from the stream using GetMsg API
originalMsg, err := h.js.GetMsg(h.streamName, event.StreamSeq)
if err != nil {
h.logger.Error("Failed to retrieve original message from stream",
zap.Uint64("stream_seq", event.StreamSeq),
zap.Error(err),
)
h.telemetry.RecordDLQPublishFailure(ctx, h.streamName, h.consumerName)
return
}
// Extract message metadata
msgID := ""
subject := originalMsg.Subject
if originalMsg.Header != nil {
msgID = originalMsg.Header.Get("Nats-Msg-Id")
}
// Create enriched DLQ payload (similar to existing routeToDLQ)
payload := map[string]interface{}{
"transport_msg_id": msgID,
"subject": subject,
"stream": h.streamName,
"consumer": h.consumerName,
"nats_sequence": event.StreamSeq,
"deliveries": event.Deliveries,
"error": fmt.Sprintf("message exhausted max_deliver (%d) attempts", event.Deliveries),
"received_at": time.Now().UTC(),
"body": string(originalMsg.Data),
"advisory_source": true, // Flag to distinguish from immediate DLQ
}
data, err := json.Marshal(payload)
if err != nil {
h.logger.Error("Failed to marshal advisory DLQ payload", zap.Error(err))
h.telemetry.RecordDLQPublishFailure(ctx, h.streamName, h.consumerName)
return
}
// Publish to DLQ
if _, err := h.js.Publish(h.dlqSubject, data); err != nil {
h.logger.Error("Failed to publish advisory message to DLQ",
zap.Uint64("stream_seq", event.StreamSeq),
zap.Error(err),
)
h.telemetry.RecordDLQPublishFailure(ctx, h.streamName, h.consumerName)
return
}
h.logger.Info("Routed max-deliveries message to DLQ",
zap.Uint64("stream_seq", event.StreamSeq),
zap.Uint64("deliveries", event.Deliveries),
)
h.telemetry.RecordDLQMessage(ctx, h.streamName, h.consumerName)
}
+502 -49
View File
@@ -5,6 +5,8 @@ import (
"caatsm/internal/infra/config"
"caatsm/internal/infra/telemetry"
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
@@ -14,35 +16,57 @@ import (
"go.uber.org/zap"
)
// Consumer handles NATS JetStream message consumption
type Consumer struct {
conn *nats.Conn
js nats.JetStreamContext
processor *app.MessageProcessor
cfg *config.Config
logger *zap.Logger
telemetry telemetry.Recorder
subject string
consumerName string
mode string
streamName string
dlqSubject string
ackWait time.Duration
batchSize int
batchTimeout time.Duration
monitorInterval time.Duration
meter metric.Meter
ackPending metric.Int64Histogram
redelivered metric.Int64Histogram
pending metric.Int64Histogram
delivered metric.Int64Histogram
// MessageFetcher defines the interface for fetching messages from NATS
type MessageFetcher interface {
FetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error)
HandleFetchError(ctx context.Context, err error, sub **nats.Subscription, fetchErrorStreak *int) (bool, error)
}
// managers for resource lifecycle
// MessageProcessor defines the interface for processing message batches
type MessageProcessor interface {
ProcessBatch(ctx context.Context, msgs []*nats.Msg)
}
// DLQHandler defines the interface for dead letter queue operations
type DLQHandler interface {
RouteToDLQ(ctx context.Context, msg *nats.Msg, cause error) error
ValidateDLQ() error
}
// Consumer handles NATS JetStream message consumption with clean separation of concerns
type Consumer struct {
// Core dependencies
conn *nats.Conn
js nats.JetStreamContext
processor *app.MessageProcessor
cfg *config.Config
logger *zap.Logger
telemetry telemetry.Recorder
// Configuration
config consumerConfig
// Collaborators (injected for testability)
fetcher MessageFetcher
batchProcessor MessageProcessor
dlqHandler DLQHandler
errorHandler *ErrorHandler
// Resource managers
consumerManager *ConsumerManager
streamManager *StreamManager
errorHandler *ErrorHandler
// simple backpressure / degradation state
// Advisory DLQ handler for messages exhausting MaxDeliver
advisoryDLQHandler *AdvisoryDLQHandler
// Metrics
meter metric.Meter
ackPending metric.Int64Histogram
redelivered metric.Int64Histogram
pending metric.Int64Histogram
delivered metric.Int64Histogram
// State
consecutiveProcessErrors int
}
@@ -59,6 +83,397 @@ type consumerConfig struct {
monitorInterval time.Duration
}
// defaultMessageFetcher implements MessageFetcher interface
type defaultMessageFetcher struct {
batchSize int
batchTimeout time.Duration
logger *zap.Logger
conn *nats.Conn
js nats.JetStreamContext
consumerManager *ConsumerManager
streamManager *StreamManager
config *consumerConfig
cfg *config.Config
}
func (f *defaultMessageFetcher) FetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error) {
return f.fetchBatch(ctx, sub)
}
// fetchBatch fetches a batch of messages from the subscription with context awareness
func (f *defaultMessageFetcher) fetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error) {
// Check context before fetching
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
// Use a shorter timeout for better responsiveness to cancellation
timeout := f.batchTimeout
if timeout > 500*time.Millisecond {
timeout = 500 * time.Millisecond
}
return sub.Fetch(f.batchSize, nats.MaxWait(timeout))
}
func (f *defaultMessageFetcher) HandleFetchError(ctx context.Context, err error, sub **nats.Subscription, fetchErrorStreak *int) (bool, error) {
// Check context cancellation first
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
f.logger.Info("Fetch error due to context cancellation", zap.Error(err))
return false, err
}
// Timeout errors are expected when no messages are available - not an error condition
if errors.Is(err, nats.ErrTimeout) {
return true, nil
}
// Check connection health before proceeding
if f.conn != nil {
status := f.conn.Status()
if status != nats.CONNECTED {
f.logger.Warn("NATS connection not in CONNECTED state",
zap.String("status", status.String()),
zap.Error(err),
)
// Connection is down - this is a transient error, apply backoff
*fetchErrorStreak++
backoff := f.calculateExponentialBackoff(*fetchErrorStreak)
f.logger.Warn("Connection unhealthy, applying backoff before retry",
zap.String("status", status.String()),
zap.Int("error_streak", *fetchErrorStreak),
zap.Duration("backoff", backoff),
)
if !sleepWithContext(ctx, backoff) {
return false, ctx.Err()
}
// Check if connection recovered after backoff
if f.conn.Status() == nats.CONNECTED {
*fetchErrorStreak = 0
return true, nil
}
// Still not connected - continue with error handling
}
}
// Check for connection closed errors
if errors.Is(err, nats.ErrConnectionClosed) {
f.logger.Error("NATS connection closed",
zap.Error(err),
zap.String("stream", f.config.streamName),
zap.String("consumer", f.config.consumerName),
)
// Connection closed is fatal - cannot recover subscription
if *sub != nil {
(*sub).Unsubscribe()
*sub = nil
}
return false, fmt.Errorf("connection closed: %w", err)
}
// JetStream API unavailable (e.g., NATS restarted or JetStream not ready)
if errors.Is(err, nats.ErrNoResponders) {
*fetchErrorStreak++
backoff := f.calculateExponentialBackoff(*fetchErrorStreak)
backoff = min(backoff, 30*time.Second)
f.logger.Warn("JetStream not available, will retry with backoff",
zap.Error(err),
zap.String("stream", f.config.streamName),
zap.String("consumer", f.config.consumerName),
zap.Int("error_streak", *fetchErrorStreak),
zap.Duration("backoff", backoff),
)
if !sleepWithContext(ctx, backoff) {
return false, ctx.Err()
}
return true, nil
}
// Check for JetStream resource not found errors
if isJetStreamResourceNotFound(err) {
if isDevLikeEnv() && shouldBootstrapStream() {
f.logger.Warn("JetStream consumer or stream missing; attempting to recreate",
zap.Error(err),
zap.String("stream", f.config.streamName),
zap.String("consumer", f.config.consumerName),
)
// Attempt to recover resources and recreate subscription
if f.consumerManager == nil || f.streamManager == nil {
return false, fmt.Errorf("cannot recover: consumer/stream manager not available: %w", err)
}
consumerConfig := f.buildConsumerConfig()
if recErr := f.consumerManager.RecoverResources(f.streamManager, consumerConfig); recErr != nil {
return false, fmt.Errorf("failed to recover JetStream resources: %w", recErr)
}
// Unsubscribe old subscription before creating new one
if *sub != nil {
(*sub).Unsubscribe()
}
// Create new subscription
newSub, subErr := f.consumerManager.CreatePullSubscription()
if subErr != nil {
return false, fmt.Errorf("failed to create pull subscription after recovery: %w", subErr)
}
*sub = newSub
*fetchErrorStreak = 0
f.logger.Info("Successfully recovered subscription after resource recreation")
return true, nil
}
// Production: treat as configuration/operational error - fatal
f.logger.Error("JetStream consumer or stream missing; not auto-recreating in this environment",
zap.Error(err),
zap.String("stream", f.config.streamName),
zap.String("consumer", f.config.consumerName),
)
if *sub != nil {
(*sub).Unsubscribe()
*sub = nil
}
return false, fmt.Errorf("JetStream resource not found: %w", err)
}
// Check for network/temporary errors
if f.isTemporaryError(err) {
*fetchErrorStreak++
backoff := f.calculateExponentialBackoff(*fetchErrorStreak)
f.logger.Warn("Temporary network error, applying backoff",
zap.Error(err),
zap.Int("error_streak", *fetchErrorStreak),
zap.Duration("backoff", backoff),
)
if !sleepWithContext(ctx, backoff) {
return false, ctx.Err()
}
// Verify subscription is still valid before returning success
if *sub != nil && f.conn != nil && f.conn.Status() == nats.CONNECTED {
return true, nil
}
// Subscription or connection invalid - attempt recovery
return f.attemptSubscriptionRecovery(ctx, sub, fetchErrorStreak)
}
// Generic error path with exponential backoff
*fetchErrorStreak++
backoff := f.calculateExponentialBackoff(*fetchErrorStreak)
f.logger.Error("Failed to fetch messages; backing off",
zap.Error(err),
zap.Int("error_streak", *fetchErrorStreak),
zap.Duration("backoff", backoff),
)
if !sleepWithContext(ctx, backoff) {
return false, ctx.Err()
}
// Verify subscription and connection health before returning success
if *sub == nil || (f.conn != nil && f.conn.Status() != nats.CONNECTED) {
return f.attemptSubscriptionRecovery(ctx, sub, fetchErrorStreak)
}
return true, nil
}
// calculateExponentialBackoff calculates exponential backoff duration with a cap
func (f *defaultMessageFetcher) calculateExponentialBackoff(streak int) time.Duration {
if streak <= 0 {
return 0
}
// Exponential backoff: 2^(streak-1) seconds, capped at 30 seconds
backoff := time.Duration(1<<uint(min(streak-1, 5))) * time.Second
return min(backoff, 30*time.Second)
}
// isTemporaryError checks if an error is a temporary/network error that might recover
func (f *defaultMessageFetcher) isTemporaryError(err error) bool {
if err == nil {
return false
}
// Check for typed temporary errors first
var tempErr interface{ Temporary() bool }
if errors.As(err, &tempErr) && tempErr.Temporary() {
return true
}
// Fall back to string matching for external errors
errStr := strings.ToLower(err.Error())
return strings.Contains(errStr, "timeout") ||
strings.Contains(errStr, "temporary") ||
strings.Contains(errStr, "network") ||
strings.Contains(errStr, "connection reset") ||
strings.Contains(errStr, "broken pipe")
}
// attemptSubscriptionRecovery attempts to recover a subscription after errors
func (f *defaultMessageFetcher) attemptSubscriptionRecovery(ctx context.Context, sub **nats.Subscription, fetchErrorStreak *int) (bool, error) {
if f.consumerManager == nil {
f.logger.Error("Cannot recover subscription: consumer manager not available")
return false, fmt.Errorf("consumer manager not available for recovery")
}
// Check connection health first
if f.conn != nil && f.conn.Status() != nats.CONNECTED {
f.logger.Warn("Connection not healthy, cannot recover subscription",
zap.String("status", f.conn.Status().String()),
)
// Connection issue - return true to retry after backoff
return true, nil
}
// Unsubscribe old subscription if it exists
if *sub != nil {
(*sub).Unsubscribe()
*sub = nil
}
// Attempt to recreate subscription
newSub, err := f.consumerManager.CreatePullSubscriptionWithRecovery(f.streamManager, f.buildConsumerConfig())
if err != nil {
f.logger.Error("Failed to recover subscription",
zap.Error(err),
zap.String("stream", f.config.streamName),
zap.String("consumer", f.config.consumerName),
)
return false, fmt.Errorf("failed to recover subscription: %w", err)
}
*sub = newSub
*fetchErrorStreak = 0
f.logger.Info("Successfully recovered subscription")
return true, nil
}
// buildConsumerConfig builds the NATS consumer configuration
func (f *defaultMessageFetcher) buildConsumerConfig() *nats.ConsumerConfig {
if f.cfg == nil {
return nil
}
return &nats.ConsumerConfig{
Durable: f.config.consumerName,
DeliverPolicy: mapDeliverPolicy(f.cfg.NATS.ConsumerRules.DeliverPolicy),
AckPolicy: nats.AckExplicitPolicy,
AckWait: f.config.ackWait,
ReplayPolicy: mapReplayPolicy(f.cfg.NATS.ConsumerRules.ReplayPolicy),
MaxDeliver: f.cfg.NATS.ConsumerRules.MaxDeliver,
MaxAckPending: f.cfg.NATS.ConsumerRules.MaxAckPending,
FilterSubject: f.config.subject,
BackOff: f.cfg.NATS.ConsumerRules.Backoff,
}
}
// defaultBatchProcessor implements MessageProcessor interface
type defaultBatchProcessor struct {
processor *app.MessageProcessor
dlqHandler DLQHandler
errorHandler *ErrorHandler
logger *zap.Logger
telemetry telemetry.Recorder
}
func (p *defaultBatchProcessor) ProcessBatch(ctx context.Context, msgs []*nats.Msg) {
// This will be implemented when we refactor the batch processing
}
// defaultDLQHandler implements DLQHandler interface
type defaultDLQHandler struct {
js nats.JetStreamContext
dlqSubject string
streamName string
consumerName string
logger *zap.Logger
telemetry telemetry.Recorder
}
func (h *defaultDLQHandler) RouteToDLQ(ctx context.Context, msg *nats.Msg, cause error) error {
return h.routeToDLQInternal(ctx, msg, cause)
}
func (h *defaultDLQHandler) ValidateDLQ() error {
return h.validateDLQInternal()
}
func (h *defaultDLQHandler) routeToDLQInternal(ctx context.Context, msg *nats.Msg, cause error) error {
// Basic DLQ routing implementation
payload := map[string]any{
"subject": msg.Subject,
"stream": h.streamName,
"consumer": h.consumerName,
"error": cause.Error(),
"received_at": time.Now().UTC(),
"body": string(msg.Data),
}
data, err := json.Marshal(payload)
if err != nil {
h.logger.Error("failed to marshal DLQ payload", zap.Error(err))
return err
}
_, err = h.js.Publish(h.dlqSubject, data)
if err != nil {
h.logger.Error("failed to publish to DLQ",
zap.String("dlq_subject", h.dlqSubject),
zap.Error(err),
)
h.telemetry.RecordDLQPublishFailure(ctx, h.streamName, h.consumerName)
return err
}
h.telemetry.RecordDLQMessage(ctx, h.streamName, h.consumerName)
return nil
}
func (h *defaultDLQHandler) validateDLQInternal() error {
if h.js == nil {
return fmt.Errorf("JetStream context is nil")
}
_, err := h.js.StreamNameBySubject(h.dlqSubject)
if err != nil {
return fmt.Errorf("DLQ subject %s not bound to any JetStream stream: %w", h.dlqSubject, err)
}
h.logger.Info("DLQ configuration validated",
zap.String("dlq_subject", h.dlqSubject),
)
return nil
}
// initCollaborators initializes the collaborator components
func (c *Consumer) initCollaborators() {
c.fetcher = &defaultMessageFetcher{
batchSize: c.config.batchSize,
batchTimeout: c.config.batchTimeout,
logger: c.logger,
conn: c.conn,
js: c.js,
consumerManager: c.consumerManager,
streamManager: c.streamManager,
config: &c.config,
cfg: c.cfg,
}
c.batchProcessor = &defaultBatchProcessor{
processor: c.processor,
dlqHandler: c.dlqHandler,
errorHandler: c.errorHandler,
logger: c.logger,
telemetry: c.telemetry,
}
if c.config.dlqSubject != "" {
c.dlqHandler = &defaultDLQHandler{
js: c.js,
dlqSubject: c.config.dlqSubject,
streamName: c.config.streamName,
consumerName: c.config.consumerName,
logger: c.logger,
telemetry: c.telemetry,
}
}
}
// normalizeConsumerConfig extracts and normalizes consumer configuration from the application config.
// This function can be unit-tested without requiring a JetStream context.
func normalizeConsumerConfig(cfg *config.Config) *consumerConfig {
@@ -122,7 +537,7 @@ func normalizeConsumerConfig(cfg *config.Config) *consumerConfig {
}
}
// ProvideConsumer creates a NATS consumer.
// ProvideConsumer creates a NATS consumer with clean architecture.
func ProvideConsumer(
conn *nats.Conn,
js nats.JetStreamContext,
@@ -134,29 +549,34 @@ func ProvideConsumer(
normCfg := normalizeConsumerConfig(cfg)
consumer := &Consumer{
conn: conn,
js: js,
processor: processor,
cfg: cfg,
logger: logger,
telemetry: rec,
subject: normCfg.subject,
consumerName: normCfg.consumerName,
mode: normCfg.mode,
streamName: normCfg.streamName,
dlqSubject: normCfg.dlqSubject,
ackWait: normCfg.ackWait,
batchSize: normCfg.batchSize,
batchTimeout: normCfg.batchTimeout,
monitorInterval: normCfg.monitorInterval,
conn: conn,
js: js,
processor: processor,
cfg: cfg,
logger: logger,
telemetry: rec,
config: *normCfg, // dereference the pointer
errorHandler: NewErrorHandler(logger),
}
consumer.initMetrics()
consumer.initCollaborators()
// Initialize managers
consumer.errorHandler = NewErrorHandler(logger)
if consumer.mode == "jetstream" {
if consumer.config.mode == "jetstream" {
consumer.consumerManager = NewConsumerManager(js, normCfg.streamName, normCfg.consumerName, normCfg.subject, logger)
consumer.streamManager = NewStreamManager(js, normCfg.streamName, []string{normCfg.subject}, logger)
// Use StreamManager with full configuration
streamSubjects := []string{normCfg.subject}
if publisherSubject := strings.TrimSpace(cfg.Publisher.Topic); publisherSubject != "" {
streamSubjects = append(streamSubjects, publisherSubject)
}
streamSubjects = dedupeSubjects(streamSubjects)
consumer.streamManager = NewStreamManagerWithConfig(js, normCfg.streamName, streamSubjects, &cfg.NATS.StreamLimits, logger)
// Update fetcher with managers now that they're initialized
if fetcher, ok := consumer.fetcher.(*defaultMessageFetcher); ok {
fetcher.consumerManager = consumer.consumerManager
fetcher.streamManager = consumer.streamManager
}
// Create consumer if it doesn't exist
consumerConfig := consumer.buildConsumerConfig()
@@ -168,6 +588,23 @@ func ProvideConsumer(
if err := consumer.validateDLQ(); err != nil {
return nil, fmt.Errorf("DLQ validation failed: %w", err)
}
// Initialize advisory DLQ handler if DLQ is enabled
if normCfg.dlqSubject != "" && cfg.DLQ.Enabled {
advisoryHandler, err := NewAdvisoryDLQHandler(
js,
conn,
normCfg.streamName,
normCfg.consumerName,
normCfg.dlqSubject,
logger,
rec,
)
if err != nil {
return nil, fmt.Errorf("failed to create advisory DLQ handler: %w", err)
}
consumer.advisoryDLQHandler = advisoryHandler
}
} else {
logger.Info("Running consumer in core NATS mode",
zap.String("subject", normCfg.subject),
@@ -181,27 +618,43 @@ func ProvideConsumer(
// buildConsumerConfig builds the NATS consumer configuration
func (c *Consumer) buildConsumerConfig() *nats.ConsumerConfig {
return &nats.ConsumerConfig{
Durable: c.consumerName,
Durable: c.config.consumerName,
DeliverPolicy: mapDeliverPolicy(c.cfg.NATS.ConsumerRules.DeliverPolicy),
AckPolicy: nats.AckExplicitPolicy,
AckWait: c.ackWait,
AckWait: c.config.ackWait,
ReplayPolicy: mapReplayPolicy(c.cfg.NATS.ConsumerRules.ReplayPolicy),
MaxDeliver: c.cfg.NATS.ConsumerRules.MaxDeliver,
MaxAckPending: c.cfg.NATS.ConsumerRules.MaxAckPending,
FilterSubject: c.subject,
FilterSubject: c.config.subject,
BackOff: c.cfg.NATS.ConsumerRules.Backoff,
}
}
// Start starts consuming messages.
func (c *Consumer) Start(ctx context.Context) error {
if c.mode == "core" {
if c.config.mode == "core" {
return c.startCore(ctx)
}
return c.startJetStream(ctx)
}
// RouteToDLQ implements DLQHandler interface
func (c *Consumer) RouteToDLQ(ctx context.Context, msg *nats.Msg, cause error) error {
if c.dlqHandler != nil {
return c.dlqHandler.RouteToDLQ(ctx, msg, cause)
}
return nil
}
// ValidateDLQ implements DLQHandler interface
func (c *Consumer) ValidateDLQ() error {
if c.dlqHandler != nil {
return c.dlqHandler.ValidateDLQ()
}
return nil
}
// Shutdown drains the underlying NATS connection gracefully.
func (c *Consumer) Shutdown(ctx context.Context) error {
if c.conn == nil {
@@ -210,7 +663,7 @@ func (c *Consumer) Shutdown(ctx context.Context) error {
timeout := c.cfg.Timeouts.Close
if timeout <= 0 {
timeout = 10 * time.Second
timeout = 2 * time.Second // Reduced from 10s for faster shutdown
}
closeCtx, cancel := context.WithTimeout(ctx, timeout)
+4 -4
View File
@@ -14,7 +14,7 @@ import (
func (c *Consumer) startCore(ctx context.Context) error {
queueGroup := c.cfg.Subscription.QueueGroup
if queueGroup == "" {
queueGroup = c.consumerName
queueGroup = c.config.consumerName
}
handler := func(msg *nats.Msg) {
@@ -28,16 +28,16 @@ func (c *Consumer) startCore(ctx context.Context) error {
}
}
sub, err := c.conn.QueueSubscribe(c.subject, queueGroup, handler)
sub, err := c.conn.QueueSubscribe(c.config.subject, queueGroup, handler)
if err != nil {
return fmt.Errorf("failed to subscribe to %s: %w", c.subject, err)
return fmt.Errorf("failed to subscribe to %s: %w", c.config.subject, err)
}
if err := c.conn.Flush(); err != nil {
return fmt.Errorf("failed to flush NATS connection: %w", err)
}
c.logger.Info("Started core NATS subscription",
zap.String("subject", c.subject),
zap.String("subject", c.config.subject),
zap.String("queue_group", queueGroup),
)
+74
View File
@@ -0,0 +1,74 @@
package nats
import (
"caatsm/internal/app"
obsmetrics "caatsm/internal/infra/metrics"
"context"
"time"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
)
// handleMessageError handles errors that occur during message processing.
func (c *Consumer) handleMessageError(ctx context.Context, msg *nats.Msg, err error, elapsed time.Duration) {
c.logger.Error("Failed to process message",
zap.String("subject", msg.Subject),
zap.Error(err),
zap.Bool("permanent", app.IsPermanent(err)),
)
result := obsmetrics.ResultFail
if app.IsPermanent(err) {
result = obsmetrics.ResultPermanentFail
}
c.telemetry.RecordMessageHandled(ctx, c.config.streamName, c.config.consumerName, result, elapsed)
processingResult := c.errorHandler.HandleProcessingError(c.consecutiveProcessErrors, err, c.logger, msg.Subject)
if processingResult.IsPermanent {
c.handlePermanentError(ctx, msg, err)
return
}
c.handleTransientError(ctx, msg, processingResult)
}
// handlePermanentError handles permanent/poison messages.
func (c *Consumer) handlePermanentError(ctx context.Context, msg *nats.Msg, err error) {
c.consecutiveProcessErrors = 0
// Poison/permanent message: route to DLQ if configured, then ACK
if dlqErr := c.routeToDLQ(ctx, msg, err); dlqErr != nil {
c.logger.Error("Failed to route permanent-error message to DLQ", zap.Error(dlqErr))
}
if ackErr := msg.Ack(); ackErr != nil {
c.logger.Error("Failed to ACK permanent-error message", zap.Error(ackErr))
}
}
// handleTransientError handles transient errors with backpressure and redelivery.
func (c *Consumer) handleTransientError(ctx context.Context, msg *nats.Msg, processingResult ProcessingErrorResult) {
// Increment error streak
if c.consecutiveProcessErrors < 0 {
c.consecutiveProcessErrors = 0
}
c.consecutiveProcessErrors++
if processingResult.ShouldApplyBackpressure {
c.logger.Warn("Applying backpressure due to consecutive processing errors",
zap.Int("consecutive_errors", c.consecutiveProcessErrors),
zap.Duration("sleep", processingResult.BackpressureDelay),
)
// Use context-aware sleep instead of blocking time.Sleep
if !sleepWithContext(ctx, processingResult.BackpressureDelay) {
// Context canceled, stop processing
return
}
}
// Transient error: request redelivery with optional delay
c.telemetry.RecordRetry(ctx, c.config.streamName, c.config.consumerName, obsmetrics.RetryReasonProcessorError)
if nakErr := c.nakWithStrategy(msg); nakErr != nil {
c.logger.Error("Failed to NAK message", zap.Error(nakErr))
}
}
+41 -127
View File
@@ -1,38 +1,15 @@
package nats
import (
"caatsm/internal/app"
obsmetrics "caatsm/internal/infra/metrics"
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
)
// ensureConsumer creates the consumer if it doesn't exist; if it already exists, it is reused.
func (c *Consumer) ensureConsumer() error {
consumerConfig := c.buildConsumerConfig()
if consumerConfig.DeliverPolicy == nats.DeliverByStartSequencePolicy && c.cfg.NATS.ConsumerRules.StartSequence > 0 {
consumerConfig.OptStartSeq = c.cfg.NATS.ConsumerRules.StartSequence
}
if consumerConfig.DeliverPolicy == nats.DeliverByStartTimePolicy && strings.TrimSpace(c.cfg.NATS.ConsumerRules.StartTime) != "" {
startTime, err := time.Parse(time.RFC3339, c.cfg.NATS.ConsumerRules.StartTime)
if err != nil {
c.logger.Warn("Invalid start time, falling back to deliver policy defaults",
zap.String("start_time", c.cfg.NATS.ConsumerRules.StartTime),
zap.Error(err),
)
} else {
consumerConfig.OptStartTime = &startTime
}
}
return c.consumerManager.EnsureConsumer(consumerConfig)
}
// recoverJetStreamResources attempts to recreate the stream and consumer in
// dev/test environments if they are missing. It is safe to call multiple times.
func (c *Consumer) recoverJetStreamResources() error {
@@ -98,14 +75,30 @@ func sleepWithContext(ctx context.Context, duration time.Duration) bool {
}
// fetchBatch fetches a batch of messages from the subscription.
func (c *Consumer) fetchBatch(sub *nats.Subscription) ([]*nats.Msg, error) {
return sub.Fetch(c.batchSize, nats.MaxWait(c.batchTimeout))
// It respects context cancellation for faster shutdown.
func (c *Consumer) fetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error) {
// Check context before fetching
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
// Use a shorter timeout for better responsiveness to cancellation
// The batchTimeout is still used, but we'll check context more frequently
timeout := c.config.batchTimeout
if timeout > 500*time.Millisecond {
// Cap at 500ms to improve responsiveness while still allowing batching
timeout = 500 * time.Millisecond
}
return sub.Fetch(c.config.batchSize, nats.MaxWait(timeout))
}
// handleFetchError handles errors during message fetching, including recovery logic.
// Returns true if the error was handled and consumption should continue, false otherwise.
func (c *Consumer) handleFetchError(ctx context.Context, err error, sub **nats.Subscription, fetchErrorStreak *int) (bool, error) {
result := c.errorHandler.HandleFetchError(ctx, err, sub, fetchErrorStreak, c.streamName, c.consumerName, func() (*nats.Subscription, error) {
result := c.errorHandler.HandleFetchError(ctx, err, sub, fetchErrorStreak, c.config.streamName, c.config.consumerName, func() (*nats.Subscription, error) {
if recErr := c.recoverJetStreamResources(); recErr != nil {
return nil, recErr
}
@@ -121,99 +114,6 @@ func (c *Consumer) handleFetchError(ctx context.Context, err error, sub **nats.S
return result.ShouldContinue, result.Error
}
// processBatch processes a batch of messages, handling errors and applying backpressure.
func (c *Consumer) processBatch(ctx context.Context, msgs []*nats.Msg) {
for _, msg := range msgs {
c.processSingleMessage(ctx, msg)
}
}
// processSingleMessage processes a single message with error handling and backpressure.
func (c *Consumer) processSingleMessage(ctx context.Context, msg *nats.Msg) {
start := time.Now()
if err := c.processMessage(ctx, msg); err != nil {
c.handleMessageError(ctx, msg, err, time.Since(start))
return
}
// Successful processing resets the error streak.
if c.consecutiveProcessErrors > 0 {
c.consecutiveProcessErrors = 0
}
// ACK the message
if ackErr := msg.Ack(); ackErr != nil {
c.logger.Error("Failed to ACK message", zap.Error(ackErr))
} else {
elapsed := time.Since(start)
c.telemetry.RecordMessageHandled(ctx, c.streamName, c.consumerName, "ok", elapsed)
}
}
// handleMessageError handles errors that occur during message processing.
func (c *Consumer) handleMessageError(ctx context.Context, msg *nats.Msg, err error, elapsed time.Duration) {
c.logger.Error("Failed to process message",
zap.String("subject", msg.Subject),
zap.Error(err),
zap.Bool("permanent", app.IsPermanent(err)),
)
result := obsmetrics.ResultFail
if app.IsPermanent(err) {
result = obsmetrics.ResultPermanentFail
}
c.telemetry.RecordMessageHandled(ctx, c.streamName, c.consumerName, result, elapsed)
processingResult := c.errorHandler.HandleProcessingError(c.consecutiveProcessErrors, err, c.logger, msg.Subject)
if processingResult.IsPermanent {
c.handlePermanentError(ctx, msg, err)
return
}
c.handleTransientError(ctx, msg, processingResult)
}
// handlePermanentError handles permanent/poison messages.
func (c *Consumer) handlePermanentError(ctx context.Context, msg *nats.Msg, err error) {
c.consecutiveProcessErrors = 0
// Poison/permanent message: route to DLQ if configured, then ACK
if dlqErr := c.routeToDLQ(ctx, msg, err); dlqErr != nil {
c.logger.Error("Failed to route permanent-error message to DLQ", zap.Error(dlqErr))
}
if ackErr := msg.Ack(); ackErr != nil {
c.logger.Error("Failed to ACK permanent-error message", zap.Error(ackErr))
}
}
// handleTransientError handles transient errors with backpressure and redelivery.
func (c *Consumer) handleTransientError(ctx context.Context, msg *nats.Msg, processingResult ProcessingErrorResult) {
// Increment error streak
if c.consecutiveProcessErrors < 0 {
c.consecutiveProcessErrors = 0
}
c.consecutiveProcessErrors++
if processingResult.ShouldApplyBackpressure {
c.logger.Warn("Applying backpressure due to consecutive processing errors",
zap.Int("consecutive_errors", c.consecutiveProcessErrors),
zap.Duration("sleep", processingResult.BackpressureDelay),
)
// Use context-aware sleep instead of blocking time.Sleep
if !sleepWithContext(ctx, processingResult.BackpressureDelay) {
// Context canceled, stop processing
return
}
}
// Transient error: request redelivery with optional delay
c.telemetry.RecordRetry(ctx, c.streamName, c.consumerName, obsmetrics.RetryReasonProcessorError)
if nakErr := c.nakWithStrategy(msg); nakErr != nil {
c.logger.Error("Failed to NAK message", zap.Error(nakErr))
}
}
// startJetStream starts the JetStream consumer loop.
func (c *Consumer) startJetStream(ctx context.Context) error {
// Create pull subscription (with simple self-healing in dev/test).
@@ -235,16 +135,16 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
defer cleanupSubscriber()
c.logger.Info("Started consuming messages",
zap.String("subject", c.subject),
zap.String("consumer", c.consumerName),
zap.String("stream", c.streamName),
zap.String("subject", c.config.subject),
zap.String("consumer", c.config.consumerName),
zap.String("stream", c.config.streamName),
)
c.logger.Info("Consumer pull configuration",
zap.Int("batch_size", c.batchSize),
zap.Duration("batch_timeout", c.batchTimeout),
zap.Int("batch_size", c.config.batchSize),
zap.Duration("batch_timeout", c.config.batchTimeout),
zap.Int("max_deliver", c.cfg.NATS.ConsumerRules.MaxDeliver),
zap.Duration("ack_wait", c.ackWait),
zap.Duration("ack_wait", c.config.ackWait),
zap.String("deliver_policy", c.cfg.NATS.ConsumerRules.DeliverPolicy),
zap.String("replay_policy", c.cfg.NATS.ConsumerRules.ReplayPolicy),
zap.Int("backoff_steps", len(c.cfg.NATS.ConsumerRules.Backoff)),
@@ -254,6 +154,15 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
defer statsCancel()
go c.emitConsumerStats(statsCtx)
// Start advisory DLQ handler in background if configured
if c.advisoryDLQHandler != nil {
go func() {
if err := c.advisoryDLQHandler.Start(ctx); err != nil {
c.logger.Error("Advisory DLQ handler failed", zap.Error(err))
}
}()
}
var fetchErrorStreak int
for {
@@ -265,8 +174,13 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
}
// Fetch messages in batch
msgs, err := c.fetchBatch(currentSub)
msgs, err := c.fetchBatch(ctx, currentSub)
if err != nil {
// If context was cancelled, return immediately
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
c.logger.Info("Stopping consumer due to context cancellation", zap.Error(err))
return err
}
shouldContinue, handleErr := c.handleFetchError(ctx, err, &currentSub, &fetchErrorStreak)
if !shouldContinue {
return handleErr
@@ -21,12 +21,14 @@ var _ = Describe("Consumer JetStream", func() {
BeforeEach(func() {
logger := zaptest.NewLogger(GinkgoT())
c = &Consumer{
mode: "jetstream",
streamName: "TEST_STREAM",
consumerName: "test-consumer",
subject: "test.subject",
batchSize: 10,
batchTimeout: 2 * time.Second,
config: consumerConfig{
mode: "jetstream",
streamName: "TEST_STREAM",
consumerName: "test-consumer",
subject: "test.subject",
batchSize: 10,
batchTimeout: 2 * time.Second,
},
logger: logger,
errorHandler: NewErrorHandler(logger),
cfg: &configpkg.Config{
@@ -4,6 +4,7 @@ import (
"caatsm/internal/infra/log"
"context"
"fmt"
"time"
"github.com/google/uuid"
"github.com/nats-io/nats.go"
@@ -13,11 +14,59 @@ import (
"go.uber.org/zap"
)
// processBatch processes a batch of messages, handling errors and applying backpressure.
// It checks context cancellation between messages for faster shutdown.
func (c *Consumer) processBatch(ctx context.Context, msgs []*nats.Msg) {
for _, msg := range msgs {
// Check context before processing each message
select {
case <-ctx.Done():
c.logger.Info("Stopping batch processing due to cancellation",
zap.Int("remaining_messages", len(msgs)),
)
return
default:
}
c.processSingleMessage(ctx, msg)
}
}
// processSingleMessage processes a single message with error handling and backpressure.
func (c *Consumer) processSingleMessage(ctx context.Context, msg *nats.Msg) {
start := time.Now()
if err := c.processMessage(ctx, msg); err != nil {
c.handleMessageError(ctx, msg, err, time.Since(start))
return
}
// Successful processing resets the error streak.
if c.consecutiveProcessErrors > 0 {
c.consecutiveProcessErrors = 0
}
// ACK the message
if ackErr := msg.Ack(); ackErr != nil {
c.logger.Error("Failed to ACK message", zap.Error(ackErr))
} else {
elapsed := time.Since(start)
c.telemetry.RecordMessageHandled(ctx, c.config.streamName, c.config.consumerName, "ok", elapsed)
}
}
// processMessage processes a single message.
func (c *Consumer) processMessage(ctx context.Context, msg *nats.Msg) error {
ctx, span := otel.Tracer("caatsm/nats").Start(ctx, "Consumer.processMessage")
defer span.End()
span.SetAttributes(attribute.String("nats.subject", msg.Subject))
// Set semantic messaging attributes
span.SetAttributes(
attribute.String("messaging.system", "nats"),
attribute.String("messaging.operation.name", "receive"),
attribute.String("messaging.destination.name", msg.Subject),
attribute.String("messaging.consumer.group.name", c.config.consumerName),
attribute.String("caatsm.stream", c.config.streamName),
)
msgID, source, err := c.resolveMsgID(msg)
if err != nil {
@@ -46,8 +95,8 @@ func (c *Consumer) processMessage(ctx context.Context, msg *nats.Msg) error {
msgLogger := log.WithMessageContext(c.logger, log.MessageFields{
Service: "caatsm-consumer",
TransportMsgID: msgID,
Stream: c.streamName,
Consumer: c.consumerName,
Stream: c.config.streamName,
Consumer: c.config.consumerName,
Subject: msg.Subject,
JSSequence: jsSeq,
})
@@ -74,7 +123,7 @@ func (c *Consumer) resolveMsgID(msg *nats.Msg) (string, string, error) {
return id, "header", nil
}
if c.mode == "core" {
if c.config.mode == "core" {
return uuid.NewString(), "generated", nil
}
@@ -85,4 +134,3 @@ func (c *Consumer) resolveMsgID(msg *nats.Msg) (string, string, error) {
return fmt.Sprintf("js-%d", meta.Sequence.Stream), "metadata", nil
}
@@ -49,12 +49,12 @@ func (c *Consumer) recordConsumerMetrics(ctx context.Context, info *nats.Consume
// Export an explicit pending messages gauge for Prometheus-based lag /
// backlog alerts.
obsmetrics.RecordNATSConsumerPending(c.streamName, c.consumerName, info.NumPending)
obsmetrics.RecordNATSConsumerPending(c.config.streamName, c.config.consumerName, info.NumPending)
}
// emitConsumerStats periodically emits consumer statistics.
func (c *Consumer) emitConsumerStats(ctx context.Context) {
ticker := time.NewTicker(c.monitorInterval)
ticker := time.NewTicker(c.config.monitorInterval)
defer ticker.Stop()
for {
@@ -62,15 +62,15 @@ func (c *Consumer) emitConsumerStats(ctx context.Context) {
case <-ctx.Done():
return
case <-ticker.C:
info, err := c.js.ConsumerInfo(c.streamName, c.consumerName)
info, err := c.js.ConsumerInfo(c.config.streamName, c.config.consumerName)
if err != nil {
c.logger.Warn("Failed to fetch consumer info", zap.Error(err))
continue
}
c.logger.Debug("JetStream consumer metrics",
zap.String("stream", c.streamName),
zap.String("consumer", c.consumerName),
zap.String("stream", c.config.streamName),
zap.String("consumer", c.config.consumerName),
zap.Uint64("num_ack_pending", uint64(info.NumAckPending)),
zap.Uint64("num_redelivered", uint64(info.NumRedelivered)),
zap.Uint64("num_pending", uint64(info.NumPending)),
+25 -25
View File
@@ -21,22 +21,22 @@ func (c *Consumer) validateDLQ() error {
}
// DLQ routing is only active in JetStream mode.
if c.mode != "jetstream" {
if c.config.mode != "jetstream" {
return nil
}
// If DLQ is not enabled in config, make sure we don't accidentally route to it.
if !c.cfg.DLQ.Enabled {
if strings.TrimSpace(c.dlqSubject) != "" {
if strings.TrimSpace(c.config.dlqSubject) != "" {
c.logger.Info("DLQ subject configured but dlq.enabled is false; DLQ routing disabled",
zap.String("dlq_subject", c.dlqSubject),
zap.String("dlq_subject", c.config.dlqSubject),
)
}
c.dlqSubject = ""
c.config.dlqSubject = ""
return nil
}
subject := strings.TrimSpace(c.dlqSubject)
subject := strings.TrimSpace(c.config.dlqSubject)
if subject == "" {
return fmt.Errorf("DLQ enabled but dlq.subject is empty")
}
@@ -68,10 +68,10 @@ func (c *Consumer) routeToDLQ(ctx context.Context, msg *nats.Msg, cause error) e
if c == nil || c.js == nil {
return nil
}
if c.mode != "jetstream" {
if c.config.mode != "jetstream" {
return nil
}
if strings.TrimSpace(c.dlqSubject) == "" {
if strings.TrimSpace(c.config.dlqSubject) == "" {
return nil
}
@@ -83,11 +83,11 @@ func (c *Consumer) routeToDLQ(ctx context.Context, msg *nats.Msg, cause error) e
deliveries = meta.NumDelivered
}
payload := map[string]interface{}{
payload := map[string]any{
"transport_msg_id": msg.Header.Get("Nats-Msg-Id"),
"subject": msg.Subject,
"stream": c.streamName,
"consumer": c.consumerName,
"stream": c.config.streamName,
"consumer": c.config.consumerName,
"nats_sequence": jsSeq,
"deliveries": deliveries,
"error": fmt.Sprint(cause),
@@ -98,42 +98,42 @@ func (c *Consumer) routeToDLQ(ctx context.Context, msg *nats.Msg, cause error) e
data, err := json.Marshal(payload)
if err != nil {
c.logger.Error("failed to marshal DLQ payload",
zap.String("stream", c.streamName),
zap.String("consumer", c.consumerName),
zap.String("dlq_subject", c.dlqSubject),
zap.String("stream", c.config.streamName),
zap.String("consumer", c.config.consumerName),
zap.String("dlq_subject", c.config.dlqSubject),
zap.Error(err),
)
return fmt.Errorf("marshal dlq payload: %w", err)
}
if _, err := c.js.Publish(c.dlqSubject, data); err != nil {
if _, err := c.js.Publish(c.config.dlqSubject, data); err != nil {
// nats.ErrNoResponders typically means that no JetStream stream is
// configured to receive this subject, or JetStream is temporarily
// unavailable. Surface this explicitly to make operational diagnosis
// easier.
if errors.Is(err, nats.ErrNoResponders) {
c.logger.Error("transient DLQ publish error (no responders)",
zap.String("stream", c.streamName),
zap.String("consumer", c.consumerName),
zap.String("dlq_subject", c.dlqSubject),
zap.String("stream", c.config.streamName),
zap.String("consumer", c.config.consumerName),
zap.String("dlq_subject", c.config.dlqSubject),
zap.Int("payload_size", len(data)),
zap.Error(err),
)
c.telemetry.RecordDLQPublishFailure(ctx, c.streamName, c.consumerName)
return fmt.Errorf("publish to dlq subject %s: no JetStream stream found for subject or JetStream unavailable: %w", c.dlqSubject, err)
c.telemetry.RecordDLQPublishFailure(ctx, c.config.streamName, c.config.consumerName)
return fmt.Errorf("publish to dlq subject %s: no JetStream stream found for subject or JetStream unavailable: %w", c.config.dlqSubject, err)
}
c.logger.Error("failed to publish to DLQ",
zap.String("stream", c.streamName),
zap.String("consumer", c.consumerName),
zap.String("dlq_subject", c.dlqSubject),
zap.String("stream", c.config.streamName),
zap.String("consumer", c.config.consumerName),
zap.String("dlq_subject", c.config.dlqSubject),
zap.Int("payload_size", len(data)),
zap.Error(err),
)
c.telemetry.RecordDLQPublishFailure(ctx, c.streamName, c.consumerName)
return fmt.Errorf("publish to dlq subject %s: %w", c.dlqSubject, err)
c.telemetry.RecordDLQPublishFailure(ctx, c.config.streamName, c.config.consumerName)
return fmt.Errorf("publish to dlq subject %s: %w", c.config.dlqSubject, err)
}
c.telemetry.RecordDLQMessage(ctx, c.streamName, c.consumerName)
c.telemetry.RecordDLQMessage(ctx, c.config.streamName, c.config.consumerName)
return nil
}
+26 -15
View File
@@ -7,9 +7,9 @@ import (
configpkg "caatsm/internal/infra/config"
"caatsm/internal/infra/telemetry"
"github.com/nats-io/nats.go"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
"go.uber.org/zap/zaptest"
)
@@ -31,7 +31,9 @@ var _ = Describe("DLQ", func() {
It("returns nil when mode is not jetstream", func() {
c := &Consumer{
mode: "core",
config: consumerConfig{
mode: "core",
},
cfg: &configpkg.Config{
DLQ: configpkg.DLQConfig{
Enabled: true,
@@ -45,8 +47,10 @@ var _ = Describe("DLQ", func() {
It("clears dlqSubject when DLQ is disabled", func() {
c := &Consumer{
mode: "jetstream",
dlqSubject: "caatsm.dlq",
config: consumerConfig{
mode: "jetstream",
dlqSubject: "caatsm.dlq",
},
cfg: &configpkg.Config{
DLQ: configpkg.DLQConfig{
Enabled: false,
@@ -56,13 +60,15 @@ var _ = Describe("DLQ", func() {
logger: logger,
}
Expect(c.validateDLQ()).To(Succeed())
Expect(c.dlqSubject).To(Equal(""))
Expect(c.config.dlqSubject).To(Equal(""))
})
It("returns error when DLQ is enabled but subject is empty", func() {
c := &Consumer{
mode: "jetstream",
dlqSubject: "",
config: consumerConfig{
mode: "jetstream",
dlqSubject: "",
},
cfg: &configpkg.Config{
DLQ: configpkg.DLQConfig{
Enabled: true,
@@ -78,9 +84,11 @@ var _ = Describe("DLQ", func() {
It("returns error when DLQ is enabled but JetStream context is nil", func() {
c := &Consumer{
mode: "jetstream",
dlqSubject: "caatsm.dlq",
js: nil,
config: consumerConfig{
mode: "jetstream",
dlqSubject: "caatsm.dlq",
},
js: nil,
cfg: &configpkg.Config{
DLQ: configpkg.DLQConfig{
Enabled: true,
@@ -107,7 +115,9 @@ var _ = Describe("DLQ", func() {
It("returns nil when mode is not jetstream", func() {
c := &Consumer{
mode: "core",
config: consumerConfig{
mode: "core",
},
}
ctx := context.Background()
msg := &nats.Msg{}
@@ -117,9 +127,11 @@ var _ = Describe("DLQ", func() {
It("returns nil when dlqSubject is empty", func() {
c := &Consumer{
mode: "jetstream",
dlqSubject: "",
js: nil, // Can be nil for this test
config: consumerConfig{
mode: "jetstream",
dlqSubject: "",
},
js: nil, // Can be nil for this test
}
ctx := context.Background()
msg := &nats.Msg{}
@@ -128,4 +140,3 @@ var _ = Describe("DLQ", func() {
})
})
})
+3 -9
View File
@@ -47,9 +47,7 @@ func (h *ErrorHandler) HandleFetchError(
if errors.Is(err, nats.ErrNoResponders) {
*fetchErrorStreak++
backoff := time.Duration(*fetchErrorStreak) * time.Second
if backoff > 30*time.Second {
backoff = 30 * time.Second
}
backoff = min(backoff, 30*time.Second)
h.logger.Warn("JetStream not available, will retry with backoff",
zap.Error(err),
zap.String("stream", streamName),
@@ -91,9 +89,7 @@ func (h *ErrorHandler) HandleFetchError(
// Generic error path with modest backoff.
*fetchErrorStreak++
backoff := time.Duration(*fetchErrorStreak) * time.Second
if backoff > 10*time.Second {
backoff = 10 * time.Second
}
backoff = min(backoff, 10*time.Second)
h.logger.Error("Failed to fetch messages; backing off",
zap.Error(err),
zap.Duration("backoff", backoff),
@@ -133,9 +129,7 @@ func (h *ErrorHandler) HandleProcessingError(
if consecutiveErrors >= 10 {
result.ShouldApplyBackpressure = true
result.BackpressureDelay = time.Duration(consecutiveErrors) * 100 * time.Millisecond
if result.BackpressureDelay > 5*time.Second {
result.BackpressureDelay = 5 * time.Second
}
result.BackpressureDelay = min(result.BackpressureDelay, 5*time.Second)
}
return result
+8 -140
View File
@@ -2,9 +2,7 @@ package nats
import (
"caatsm/internal/infra/config"
"errors"
"fmt"
"os"
"strings"
"github.com/nats-io/nats.go"
@@ -65,22 +63,7 @@ func ProvideJetStream(nc *nats.Conn, cfg *config.Config, logger *zap.Logger) (na
return nil, fmt.Errorf("failed to get JetStream context: %w", err)
}
// Ensure the stream exists and is minimally aligned with configuration.
if err := EnsureStream(js, cfg, logger); err != nil {
nc.Close()
return nil, err
}
return js, nil
}
// EnsureStream ensures that the configured JetStream stream exists and has
// at least the expected subjects bound. It is safe to call multiple times.
//
// In dev/test environments (see shouldBootstrapStream), the stream will be
// auto-created if it does not exist. In production, a missing stream results
// in an error so that operators can intervene.
func EnsureStream(js nats.JetStreamContext, cfg *config.Config, logger *zap.Logger) error {
// Ensure the stream exists using StreamManager
streamName := cfg.NATS.Stream
consumerSubject := cfg.EffectiveSubscriptionTopic()
publisherSubject := strings.TrimSpace(cfg.Publisher.Topic)
@@ -92,130 +75,15 @@ func EnsureStream(js nats.JetStreamContext, cfg *config.Config, logger *zap.Logg
zap.String("consumer_subject", consumerSubject),
zap.String("publisher_subject", publisherSubject),
)
return fmt.Errorf("no subjects configured for JetStream stream %s", streamName)
nc.Close()
return nil, fmt.Errorf("no subjects configured for JetStream stream %s", streamName)
}
streamLimits := cfg.NATS.StreamLimits
storage := nats.FileStorage
switch strings.ToLower(streamLimits.Storage) {
case "memory":
storage = nats.MemoryStorage
case "file":
storage = nats.FileStorage
streamManager := NewStreamManagerWithConfig(js, streamName, streamSubjects, &cfg.NATS.StreamLimits, logger)
if err := streamManager.EnsureStream(); err != nil {
nc.Close()
return nil, err
}
discard := nats.DiscardOld
if strings.EqualFold(streamLimits.Discard, "new") {
discard = nats.DiscardNew
}
streamConfig := &nats.StreamConfig{
Name: streamName,
Subjects: streamSubjects,
Retention: nats.LimitsPolicy,
MaxMsgs: streamLimits.MaxMsgs,
MaxBytes: streamLimits.MaxBytes,
MaxAge: streamLimits.MaxAge,
Discard: discard,
Storage: storage,
Replicas: streamLimits.Replicas,
}
info, err := js.StreamInfo(streamName)
if err != nil {
if errors.Is(err, nats.ErrStreamNotFound) {
if shouldBootstrapStream() {
if _, err = js.AddStream(streamConfig); err != nil {
logger.Error("failed to create stream",
zap.String("stream", streamName),
zap.Strings("subjects", streamSubjects),
zap.Error(err),
)
return fmt.Errorf("failed to create stream %s: %w", streamName, err)
}
logger.Info("Created JetStream stream",
zap.String("stream", streamName),
zap.Strings("subjects", streamSubjects),
)
return nil
}
logger.Error("stream not found and auto-creation disabled",
zap.String("stream", streamName),
zap.Strings("expected_subjects", streamSubjects),
)
return fmt.Errorf("stream %s not found and auto-creation disabled", streamName)
}
logger.Error("failed to fetch stream info",
zap.String("stream", streamName),
zap.Error(err),
)
return fmt.Errorf("failed to fetch stream info for %s: %w", streamName, err)
}
// Stream exists: validate subjects but do not fail hard if they differ.
validateStreamConfig(info, streamSubjects, logger)
return nil
}
func shouldBootstrapStream() bool {
switch strings.ToLower(os.Getenv("GO_ENV")) {
case "", "dev", "development", "test", "testing":
return true
default:
return false
}
}
func validateStreamConfig(info *nats.StreamInfo, expectedSubjects []string, logger *zap.Logger) {
if info == nil {
return
}
defer func() {
if len(expectedSubjects) == 0 {
expectedSubjects = []string{"<none>"}
}
}()
missing := make([]string, 0)
for _, subj := range expectedSubjects {
if subj == "" {
continue
}
if !containsSubject(info.Config.Subjects, subj) {
missing = append(missing, subj)
}
}
if len(missing) > 0 {
logger.Warn("JetStream stream subjects missing expected entries",
zap.String("stream", info.Config.Name),
zap.Strings("stream_subjects", info.Config.Subjects),
zap.Strings("missing_subjects", missing),
)
}
}
func containsSubject(subjects []string, target string) bool {
for _, s := range subjects {
if s == target {
return true
}
}
return false
}
func dedupeSubjects(subjects []string) []string {
seen := make(map[string]struct{})
result := make([]string, 0, len(subjects))
for _, subj := range subjects {
subj = strings.TrimSpace(subj)
if subj == "" {
continue
}
if _, ok := seen[subj]; ok {
continue
}
seen[subj] = struct{}{}
result = append(result, subj)
}
return result
return js, nil
}
+9 -8
View File
@@ -1,9 +1,9 @@
package nats
import (
"github.com/nats-io/nats.go"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/nats-io/nats.go"
"go.uber.org/zap/zaptest"
)
@@ -14,10 +14,12 @@ var _ = Describe("MessageHandler", func() {
BeforeEach(func() {
c = &Consumer{
mode: "jetstream",
streamName: "TEST_STREAM",
consumerName: "test-consumer",
logger: zaptest.NewLogger(GinkgoT()),
config: consumerConfig{
mode: "jetstream",
streamName: "TEST_STREAM",
consumerName: "test-consumer",
},
logger: zaptest.NewLogger(GinkgoT()),
}
})
@@ -35,7 +37,7 @@ var _ = Describe("MessageHandler", func() {
})
It("generates UUID for core mode when header is missing", func() {
c.mode = "core"
c.config.mode = "core"
msg := &nats.Msg{
Header: nats.Header{},
}
@@ -47,7 +49,7 @@ var _ = Describe("MessageHandler", func() {
})
It("returns error for JetStream mode when header and metadata are missing", func() {
c.mode = "jetstream"
c.config.mode = "jetstream"
msg := &nats.Msg{
Header: nats.Header{},
}
@@ -59,4 +61,3 @@ var _ = Describe("MessageHandler", func() {
})
})
})
+6 -5
View File
@@ -3,9 +3,9 @@ package nats
import (
"context"
"github.com/nats-io/nats.go"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/nats-io/nats.go"
"go.uber.org/zap/zaptest"
)
@@ -18,9 +18,11 @@ var _ = Describe("Metrics", func() {
BeforeEach(func() {
ctx = context.Background()
c = &Consumer{
streamName: "TEST_STREAM",
consumerName: "test-consumer",
logger: zaptest.NewLogger(GinkgoT()),
config: consumerConfig{
streamName: "TEST_STREAM",
consumerName: "test-consumer",
},
logger: zaptest.NewLogger(GinkgoT()),
}
})
@@ -54,4 +56,3 @@ var _ = Describe("Metrics", func() {
})
})
})
+3 -3
View File
@@ -1,8 +1,8 @@
package nats
import (
"caatsm/internal/infra/config"
"caatsm/internal/adapter/dto"
"caatsm/internal/infra/config"
"caatsm/internal/port"
"encoding/json"
"errors"
@@ -59,7 +59,7 @@ func ProvideCorePublisher(
}
// Publish publishes a message using plain NATS
func (p *CorePublisher) Publish(message interface{}) error {
func (p *CorePublisher) Publish(message any) error {
topic := p.cfg.Publisher.Topic
if topic == "" {
p.logger.Error("publisher topic is not configured")
@@ -96,7 +96,7 @@ func (p *CorePublisher) Publish(message interface{}) error {
}
// Publish publishes a message
func (p *Publisher) Publish(message interface{}) error {
func (p *Publisher) Publish(message any) error {
topic := p.cfg.Publisher.Topic
if topic == "" {
p.logger.Error("publisher topic is not configured")
+51 -6
View File
@@ -1,8 +1,10 @@
package nats
import (
"caatsm/internal/infra/config"
"errors"
"fmt"
"strings"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
@@ -14,6 +16,7 @@ type StreamManager struct {
streamName string
subjects []string
logger *zap.Logger
cfg *config.StreamLimitsConfig
}
// NewStreamManager creates a new stream manager
@@ -26,14 +29,21 @@ func NewStreamManager(js nats.JetStreamContext, streamName string, subjects []st
}
}
// NewStreamManagerWithConfig creates a new stream manager with full stream configuration
func NewStreamManagerWithConfig(js nats.JetStreamContext, streamName string, subjects []string, streamLimits *config.StreamLimitsConfig, logger *zap.Logger) *StreamManager {
return &StreamManager{
js: js,
streamName: streamName,
subjects: subjects,
logger: logger,
cfg: streamLimits,
}
}
// EnsureStream ensures that the configured JetStream stream exists
func (sm *StreamManager) EnsureStream() error {
streamConfig := &nats.StreamConfig{
Name: sm.streamName,
Subjects: sm.subjects,
Retention: nats.LimitsPolicy,
Storage: nats.FileStorage,
}
// Build stream configuration
streamConfig := sm.buildStreamConfig()
info, err := sm.js.StreamInfo(sm.streamName)
if err != nil {
@@ -71,6 +81,41 @@ func (sm *StreamManager) EnsureStream() error {
return nil
}
// buildStreamConfig builds the stream configuration from manager settings
func (sm *StreamManager) buildStreamConfig() *nats.StreamConfig {
config := &nats.StreamConfig{
Name: sm.streamName,
Subjects: sm.subjects,
Retention: nats.LimitsPolicy,
Storage: nats.FileStorage,
}
// Apply stream limits configuration if provided
if sm.cfg != nil {
config.MaxMsgs = sm.cfg.MaxMsgs
config.MaxBytes = sm.cfg.MaxBytes
config.MaxAge = sm.cfg.MaxAge
config.Replicas = sm.cfg.Replicas
// Map storage type
switch strings.ToLower(sm.cfg.Storage) {
case "memory":
config.Storage = nats.MemoryStorage
case "file":
config.Storage = nats.FileStorage
}
// Map discard policy
if strings.EqualFold(sm.cfg.Discard, "new") {
config.Discard = nats.DiscardNew
} else {
config.Discard = nats.DiscardOld
}
}
return config
}
// validateStreamConfig validates the stream configuration
func (sm *StreamManager) validateStreamConfig(info *nats.StreamInfo) {
if info == nil {
+38
View File
@@ -78,3 +78,41 @@ func mapReplayPolicy(value string) nats.ReplayPolicy {
return nats.ReplayInstantPolicy
}
}
// shouldBootstrapStream checks if streams should be auto-created based on environment.
func shouldBootstrapStream() bool {
switch strings.ToLower(os.Getenv("GO_ENV")) {
case "", "dev", "development", "test", "testing":
return true
default:
return false
}
}
// containsSubject checks if a subject exists in a list of subjects.
func containsSubject(subjects []string, target string) bool {
for _, s := range subjects {
if s == target {
return true
}
}
return false
}
// dedupeSubjects removes duplicate and empty subjects from a list.
func dedupeSubjects(subjects []string) []string {
seen := make(map[string]struct{})
result := make([]string, 0, len(subjects))
for _, subj := range subjects {
subj = strings.TrimSpace(subj)
if subj == "" {
continue
}
if _, ok := seen[subj]; ok {
continue
}
seen[subj] = struct{}{}
result = append(result, subj)
}
return result
}
-1
View File
@@ -84,4 +84,3 @@ var _ = Describe("Utils", func() {
})
})
})
+10 -1
View File
@@ -39,7 +39,16 @@ func ProvideRepository(pool *pgxpool.Pool, logger *zap.Logger) (port.Repository,
func (r *Repository) InsertOne(ctx context.Context, msg *dto.ParsedTelegram) error {
ctx, span := otel.Tracer("caatsm/postgres").Start(ctx, "Repository.InsertOne")
defer span.End()
span.SetAttributes(attribute.String("db.table", "aviation.telegrams"))
// Set semantic database attributes
span.SetAttributes(
attribute.String("db.system", "postgresql"),
attribute.String("db.operation", "insert"),
attribute.String("db.name", "aviation"),
attribute.String("db.table", "telegrams"),
attribute.String("caatsm.message.id", msg.MessageID),
attribute.String("caatsm.message.category", msg.Category),
)
// Optional idempotency check based on business message identity. If we have a
// non-empty message ID and date/time, we can cheaply skip duplicates here to
+248
View File
@@ -0,0 +1,248 @@
package telemetry
import (
"caatsm/internal/infra/buildinfo"
"caatsm/internal/infra/config"
"context"
"crypto/tls"
"fmt"
"os"
"sync"
"time"
"github.com/google/uuid"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
"go.uber.org/zap"
)
var (
// instanceID stores a unique identifier for this running process instance.
// It is initialized once at startup and remains stable for the lifetime of the process.
instanceID string
instanceIDOnce sync.Once
)
// InitOTEL initializes the OpenTelemetry SDK with proper resource attributes,
// sampling configuration, and exporters. This should be called once at application startup.
// logger is optional; if provided, sensitive telemetry configuration will be logged at debug level.
func InitOTEL(ctx context.Context, cfg *config.Config, logger *zap.Logger) error {
if !cfg.Telemetry.Enabled {
return nil
}
// Log sensitive telemetry configuration to internal debug logs only
if logger != nil {
logger.Debug("Initializing OpenTelemetry",
zap.String("telemetry.endpoint", cfg.Telemetry.Endpoint),
zap.Bool("telemetry.insecure", cfg.Telemetry.Insecure),
)
}
// Create resource with comprehensive service information
res, err := createResource(ctx, cfg)
if err != nil {
return fmt.Errorf("failed to create OTEL resource: %w", err)
}
// Initialize tracing
if err := initTracing(ctx, cfg, res); err != nil {
return fmt.Errorf("failed to initialize tracing: %w", err)
}
// Initialize metrics
if err := initMetrics(ctx, cfg, res); err != nil {
return fmt.Errorf("failed to initialize metrics: %w", err)
}
return nil
}
// getInstanceID returns a unique identifier for this running process instance.
// It checks environment variables (POD_NAME, CONTAINER_ID, HOSTNAME) first, then generates
// a UUID if no environment variable is available. The ID is initialized once and remains
// stable for the lifetime of the process.
func getInstanceID() string {
instanceIDOnce.Do(func() {
// Check for Kubernetes pod name first (most common in containerized deployments)
if podName := os.Getenv("POD_NAME"); podName != "" {
instanceID = podName
return
}
// Check for container ID (Docker, containerd, etc.)
if containerID := os.Getenv("CONTAINER_ID"); containerID != "" {
instanceID = containerID
return
}
// Check for HOSTNAME (often set in containers)
if hostname := os.Getenv("HOSTNAME"); hostname != "" {
// Use hostname if it's not a generic default
if hostname != "localhost" && hostname != "localhost.localdomain" {
instanceID = hostname
return
}
}
// Generate a UUID for this process instance
instanceID = uuid.NewString()
})
return instanceID
}
// createResource creates a resource with standard and custom attributes.
// Sensitive infrastructure details (endpoint, insecure flag) are excluded from resource
// attributes to prevent leakage. These values are logged internally at debug level if
// a logger is provided to InitOTEL.
func createResource(ctx context.Context, cfg *config.Config) (*resource.Resource, error) {
// Get the runtime instance ID (falls back to buildinfo.Commit if needed)
runtimeInstanceID := getInstanceID()
if runtimeInstanceID == "" {
// Final fallback to build commit if somehow instance ID is empty
runtimeInstanceID = buildinfo.Commit
}
attrs := []attribute.KeyValue{
// Standard semantic conventions
semconv.ServiceName("caatsm"),
semconv.ServiceVersion(buildinfo.Version),
semconv.ServiceInstanceID(runtimeInstanceID),
semconv.ServiceNamespace("airport"),
// Custom attributes
attribute.String("service.component", "receiver"),
attribute.String("service.environment", getEnvironment()),
attribute.String("build.commit", buildinfo.Commit),
attribute.String("build.built_at", buildinfo.BuiltAt),
}
// Add non-sensitive indicator for telemetry endpoint configuration
// (without exposing the actual endpoint value)
if cfg.Telemetry.Endpoint != "" {
attrs = append(attrs, attribute.Bool("telemetry.endpoint.configured", true))
} else {
attrs = append(attrs, attribute.Bool("telemetry.endpoint.configured", false))
}
return resource.New(ctx, resource.WithAttributes(attrs...))
}
// initTracing sets up the trace provider with appropriate sampling
func initTracing(ctx context.Context, cfg *config.Config, res *resource.Resource) error {
var traceExporterOptions []otlptracehttp.Option
traceExporterOptions = append(traceExporterOptions, otlptracehttp.WithEndpoint(cfg.Telemetry.Endpoint))
if cfg.Telemetry.Insecure {
traceExporterOptions = append(traceExporterOptions, otlptracehttp.WithTLSClientConfig(&tls.Config{
InsecureSkipVerify: true,
MinVersion: tls.VersionTLS13,
}))
}
traceExporter, err := otlptracehttp.New(ctx, traceExporterOptions...)
if err != nil {
return fmt.Errorf("failed to create trace exporter: %w", err)
}
// Configure sampling based on environment
sampler := getSamplerForEnvironment(getEnvironment())
tracerProvider := sdktrace.NewTracerProvider(
sdktrace.WithBatcher(traceExporter,
sdktrace.WithBatchTimeout(1*time.Second),
sdktrace.WithMaxExportBatchSize(512),
sdktrace.WithMaxQueueSize(2048),
),
sdktrace.WithResource(res),
sdktrace.WithSampler(sdktrace.ParentBased(sampler)),
)
otel.SetTracerProvider(tracerProvider)
return nil
}
// initMetrics sets up the meter provider
func initMetrics(ctx context.Context, cfg *config.Config, res *resource.Resource) error {
var metricExporterOptions []otlpmetrichttp.Option
metricExporterOptions = append(metricExporterOptions, otlpmetrichttp.WithEndpoint(cfg.Telemetry.Endpoint))
if cfg.Telemetry.Insecure {
metricExporterOptions = append(metricExporterOptions, otlpmetrichttp.WithTLSClientConfig(&tls.Config{
InsecureSkipVerify: true,
MinVersion: tls.VersionTLS12,
}))
}
metricExporter, err := otlpmetrichttp.New(ctx, metricExporterOptions...)
if err != nil {
return fmt.Errorf("failed to create metric exporter: %w", err)
}
meterProvider := sdkmetric.NewMeterProvider(
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExporter,
sdkmetric.WithInterval(30*time.Second),
)),
sdkmetric.WithResource(res),
)
otel.SetMeterProvider(meterProvider)
return nil
}
// getSamplerForEnvironment returns appropriate sampling strategy for each environment
func getSamplerForEnvironment(env string) sdktrace.Sampler {
switch env {
case "prod", "production":
// 1% sampling in production to control costs and performance
return sdktrace.TraceIDRatioBased(0.01)
case "staging":
// 10% sampling in staging for better observability
return sdktrace.TraceIDRatioBased(0.1)
case "test", "testing":
// Always sample in testing for complete coverage
return sdktrace.AlwaysSample()
default:
// 100% sampling in development for debugging
return sdktrace.AlwaysSample()
}
}
// ShutdownOTEL gracefully shuts down the OTEL providers
func ShutdownOTEL(ctx context.Context) error {
var errs []error
if tracerProvider, ok := otel.GetTracerProvider().(*sdktrace.TracerProvider); ok {
if err := tracerProvider.Shutdown(ctx); err != nil {
errs = append(errs, fmt.Errorf("failed to shutdown tracer provider: %w", err))
}
}
if meterProvider, ok := otel.GetMeterProvider().(*sdkmetric.MeterProvider); ok {
if err := meterProvider.Shutdown(ctx); err != nil {
errs = append(errs, fmt.Errorf("failed to shutdown meter provider: %w", err))
}
}
if len(errs) > 0 {
return fmt.Errorf("OTEL shutdown errors: %v", errs)
}
return nil
}
// getEnvironment returns the current environment from GO_ENV
func getEnvironment() string {
if env := os.Getenv("GO_ENV"); env != "" {
return env
}
return "dev"
}