🔧 Update Go version in go.mod and enhance build process with versioning information. Modify Makefile and Taskfile to inject build metadata (version, commit, build time) into the binary. Improve README with instructions for custom version builds and document new build info features. Add benchmarks for message parsing and processing to improve performance testing capabilities.

This commit is contained in:
windyboy
2025-11-18 14:15:58 +08:00
parent 7f44b5389d
commit 06fc9cb9e0
27 changed files with 3009 additions and 55 deletions
+3 -1
View File
@@ -85,7 +85,9 @@ func (h *AdvisoryDLQHandler) Start(ctx context.Context) error {
// Wait for context cancellation
go func() {
<-ctx.Done()
sub.Unsubscribe()
if err := sub.Unsubscribe(); err != nil {
h.logger.Error("Failed to unsubscribe advisory subscription", zap.Error(err))
}
h.logger.Info("Stopped advisory DLQ handler")
}()
+261 -13
View File
@@ -3,6 +3,8 @@ package nats
import (
"caatsm/internal/app"
"caatsm/internal/infra/config"
"caatsm/internal/infra/log"
obsmetrics "caatsm/internal/infra/metrics"
"caatsm/internal/infra/telemetry"
"context"
"encoding/json"
@@ -11,7 +13,11 @@ import (
"strings"
"time"
"github.com/google/uuid"
"github.com/nats-io/nats.go"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/metric"
"go.uber.org/zap"
)
@@ -167,7 +173,9 @@ func (f *defaultMessageFetcher) HandleFetchError(ctx context.Context, err error,
)
// Connection closed is fatal - cannot recover subscription
if *sub != nil {
(*sub).Unsubscribe()
if err := (*sub).Unsubscribe(); err != nil {
f.logger.Error("Failed to unsubscribe after connection closed", zap.Error(err))
}
*sub = nil
}
return false, fmt.Errorf("connection closed: %w", err)
@@ -209,7 +217,9 @@ func (f *defaultMessageFetcher) HandleFetchError(ctx context.Context, err error,
}
// Unsubscribe old subscription before creating new one
if *sub != nil {
(*sub).Unsubscribe()
if err := (*sub).Unsubscribe(); err != nil {
f.logger.Error("Failed to unsubscribe during recovery", zap.Error(err))
}
}
// Create new subscription
newSub, subErr := f.consumerManager.CreatePullSubscription()
@@ -229,7 +239,9 @@ func (f *defaultMessageFetcher) HandleFetchError(ctx context.Context, err error,
zap.String("consumer", f.config.consumerName),
)
if *sub != nil {
(*sub).Unsubscribe()
if err := (*sub).Unsubscribe(); err != nil {
f.logger.Error("Failed to unsubscribe after resource not found", zap.Error(err))
}
*sub = nil
}
return false, fmt.Errorf("JetStream resource not found: %w", err)
@@ -322,7 +334,9 @@ func (f *defaultMessageFetcher) attemptSubscriptionRecovery(ctx context.Context,
// Unsubscribe old subscription if it exists
if *sub != nil {
(*sub).Unsubscribe()
if err := (*sub).Unsubscribe(); err != nil {
f.logger.Error("Failed to unsubscribe during recovery", zap.Error(err))
}
*sub = nil
}
@@ -368,10 +382,238 @@ type defaultBatchProcessor struct {
errorHandler *ErrorHandler
logger *zap.Logger
telemetry telemetry.Recorder
// Configuration needed for processing
streamName string
consumerName string
mode string
backoff []time.Duration
// Pointer to consecutive errors counter (shared with Consumer)
consecutiveProcessErrors *int
}
func (p *defaultBatchProcessor) ProcessBatch(ctx context.Context, msgs []*nats.Msg) {
// This will be implemented when we refactor the batch processing
for _, msg := range msgs {
// Check context before processing each message
select {
case <-ctx.Done():
p.logger.Info("Stopping batch processing due to cancellation",
zap.Int("remaining_messages", len(msgs)),
)
return
default:
}
p.processSingleMessage(ctx, msg)
}
}
// processSingleMessage processes a single message with error handling and backpressure.
func (p *defaultBatchProcessor) processSingleMessage(ctx context.Context, msg *nats.Msg) {
start := time.Now()
if err := p.processMessage(ctx, msg); err != nil {
p.handleMessageError(ctx, msg, err, time.Since(start))
return
}
// Successful processing resets the error streak.
if p.consecutiveProcessErrors != nil && *p.consecutiveProcessErrors > 0 {
*p.consecutiveProcessErrors = 0
}
// ACK the message
if ackErr := msg.Ack(); ackErr != nil {
p.logger.Error("Failed to ACK message", zap.Error(ackErr))
} else {
elapsed := time.Since(start)
p.telemetry.RecordMessageHandled(ctx, p.streamName, p.consumerName, "ok", elapsed)
}
}
// processMessage processes a single message.
func (p *defaultBatchProcessor) processMessage(ctx context.Context, msg *nats.Msg) error {
ctx, span := otel.Tracer("caatsm/nats").Start(ctx, "Consumer.processMessage")
defer span.End()
// Set semantic messaging attributes
span.SetAttributes(
attribute.String("messaging.system", "nats"),
attribute.String("messaging.operation.name", "receive"),
attribute.String("messaging.destination.name", msg.Subject),
attribute.String("messaging.consumer.group.name", p.consumerName),
attribute.String("caatsm.stream", p.streamName),
)
msgID, source, err := p.resolveMsgID(msg)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("unable to resolve message id: %w", err)
}
if source != "header" {
p.logger.Warn("Message missing NATS id header; using fallback",
zap.String("subject", msg.Subject),
zap.String("msg_id_source", source),
zap.String("msg_id", msgID),
)
}
// Attach structured logging context including stream/consumer and NATS metadata.
jsSeq := uint64(0)
if meta, metaErr := msg.Metadata(); metaErr == nil {
jsSeq = meta.Sequence.Stream
span.SetAttributes(
attribute.Int64("nats.js.stream_seq", int64(meta.Sequence.Stream)),
attribute.Int64("nats.js.consumer_seq", int64(meta.Sequence.Consumer)),
)
}
msgLogger := log.WithMessageContext(p.logger, log.MessageFields{
Service: "caatsm-consumer",
TransportMsgID: msgID,
Stream: p.streamName,
Consumer: p.consumerName,
Subject: msg.Subject,
JSSequence: jsSeq,
})
msgLogger.Debug("Processing message",
zap.Int("data_size", len(msg.Data)),
zap.String("msg_id_source", source),
)
// Call processor
if err := p.processor.Handle(ctx, msg.Data, msgID); err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("processor error: %w", err)
}
span.SetAttributes(attribute.String("telegram.msg_id", msgID))
return nil
}
// resolveMsgID extracts or generates a message ID.
func (p *defaultBatchProcessor) resolveMsgID(msg *nats.Msg) (string, string, error) {
if id := msg.Header.Get("Nats-Msg-Id"); id != "" {
return id, "header", nil
}
if p.mode == "core" {
return uuid.NewString(), "generated", nil
}
meta, err := msg.Metadata()
if err != nil {
return "", "", fmt.Errorf("fetch metadata: %w", err)
}
return fmt.Sprintf("js-%d", meta.Sequence.Stream), "metadata", nil
}
// handleMessageError handles errors that occur during message processing.
func (p *defaultBatchProcessor) handleMessageError(ctx context.Context, msg *nats.Msg, err error, elapsed time.Duration) {
p.logger.Error("Failed to process message",
zap.String("subject", msg.Subject),
zap.Error(err),
zap.Bool("permanent", app.IsPermanent(err)),
)
result := obsmetrics.ResultFail
if app.IsPermanent(err) {
result = obsmetrics.ResultPermanentFail
}
p.telemetry.RecordMessageHandled(ctx, p.streamName, p.consumerName, result, elapsed)
consecutiveErrors := 0
if p.consecutiveProcessErrors != nil {
consecutiveErrors = *p.consecutiveProcessErrors
}
processingResult := p.errorHandler.HandleProcessingError(consecutiveErrors, err, p.logger, msg.Subject)
if processingResult.IsPermanent {
p.handlePermanentError(ctx, msg, err)
return
}
p.handleTransientError(ctx, msg, processingResult)
}
// handlePermanentError handles permanent/poison messages.
func (p *defaultBatchProcessor) handlePermanentError(ctx context.Context, msg *nats.Msg, err error) {
if p.consecutiveProcessErrors != nil {
*p.consecutiveProcessErrors = 0
}
// Poison/permanent message: route to DLQ if configured, then ACK
if p.dlqHandler != nil {
if dlqErr := p.dlqHandler.RouteToDLQ(ctx, msg, err); dlqErr != nil {
p.logger.Error("Failed to route permanent-error message to DLQ", zap.Error(dlqErr))
}
}
if ackErr := msg.Ack(); ackErr != nil {
p.logger.Error("Failed to ACK permanent-error message", zap.Error(ackErr))
}
}
// handleTransientError handles transient errors with backpressure and redelivery.
func (p *defaultBatchProcessor) handleTransientError(ctx context.Context, msg *nats.Msg, processingResult ProcessingErrorResult) {
// Increment error streak
if p.consecutiveProcessErrors != nil {
if *p.consecutiveProcessErrors < 0 {
*p.consecutiveProcessErrors = 0
}
*p.consecutiveProcessErrors++
}
if processingResult.ShouldApplyBackpressure {
consecutiveErrors := 0
if p.consecutiveProcessErrors != nil {
consecutiveErrors = *p.consecutiveProcessErrors
}
p.logger.Warn("Applying backpressure due to consecutive processing errors",
zap.Int("consecutive_errors", consecutiveErrors),
zap.Duration("sleep", processingResult.BackpressureDelay),
)
// Use context-aware sleep instead of blocking time.Sleep
if !sleepWithContext(ctx, processingResult.BackpressureDelay) {
// Context canceled, stop processing
return
}
}
// Transient error: request redelivery with optional delay
p.telemetry.RecordRetry(ctx, p.streamName, p.consumerName, obsmetrics.RetryReasonProcessorError)
if nakErr := p.nakWithStrategy(msg); nakErr != nil {
p.logger.Error("Failed to NAK message", zap.Error(nakErr))
}
}
// nakWithStrategy sends a NAK with appropriate delay based on retry attempt.
func (p *defaultBatchProcessor) nakWithStrategy(msg *nats.Msg) error {
if len(p.backoff) == 0 {
return msg.Nak()
}
meta, err := msg.Metadata()
if err != nil {
p.logger.Warn("Failed to read metadata for backoff strategy", zap.Error(err))
return msg.Nak()
}
attempt := int(meta.NumDelivered)
index := attempt - 1
if index < 0 {
index = 0
}
if index >= len(p.backoff) {
index = len(p.backoff) - 1
}
delay := p.backoff[index]
if delay <= 0 {
return msg.Nak()
}
return msg.NakWithDelay(delay)
}
// defaultDLQHandler implements DLQHandler interface
@@ -454,14 +696,7 @@ func (c *Consumer) initCollaborators() {
cfg: c.cfg,
}
c.batchProcessor = &defaultBatchProcessor{
processor: c.processor,
dlqHandler: c.dlqHandler,
errorHandler: c.errorHandler,
logger: c.logger,
telemetry: c.telemetry,
}
// Initialize DLQ handler first if needed, so batch processor can reference it
if c.config.dlqSubject != "" {
c.dlqHandler = &defaultDLQHandler{
js: c.js,
@@ -472,6 +707,19 @@ func (c *Consumer) initCollaborators() {
telemetry: c.telemetry,
}
}
c.batchProcessor = &defaultBatchProcessor{
processor: c.processor,
dlqHandler: c.dlqHandler,
errorHandler: c.errorHandler,
logger: c.logger,
telemetry: c.telemetry,
streamName: c.config.streamName,
consumerName: c.config.consumerName,
mode: c.config.mode,
backoff: c.cfg.NATS.ConsumerRules.Backoff,
consecutiveProcessErrors: &c.consecutiveProcessErrors,
}
}
// normalizeConsumerConfig extracts and normalizes consumer configuration from the application config.
+6
View File
@@ -11,6 +11,8 @@ import (
)
// handleMessageError handles errors that occur during message processing.
//
//nolint:unused // Reserved for potential future use or alternative implementation
func (c *Consumer) handleMessageError(ctx context.Context, msg *nats.Msg, err error, elapsed time.Duration) {
c.logger.Error("Failed to process message",
zap.String("subject", msg.Subject),
@@ -35,6 +37,8 @@ func (c *Consumer) handleMessageError(ctx context.Context, msg *nats.Msg, err er
}
// handlePermanentError handles permanent/poison messages.
//
//nolint:unused // Reserved for potential future use or alternative implementation
func (c *Consumer) handlePermanentError(ctx context.Context, msg *nats.Msg, err error) {
c.consecutiveProcessErrors = 0
// Poison/permanent message: route to DLQ if configured, then ACK
@@ -47,6 +51,8 @@ func (c *Consumer) handlePermanentError(ctx context.Context, msg *nats.Msg, err
}
// handleTransientError handles transient errors with backpressure and redelivery.
//
//nolint:unused // Reserved for potential future use or alternative implementation
func (c *Consumer) handleTransientError(ctx context.Context, msg *nats.Msg, processingResult ProcessingErrorResult) {
// Increment error streak
if c.consecutiveProcessErrors < 0 {
+14 -6
View File
@@ -32,6 +32,7 @@ func (c *Consumer) createPullSubscriptionWithRecovery() (*nats.Subscription, err
return c.consumerManager.CreatePullSubscriptionWithRecovery(c.streamManager, consumerConfig)
}
//nolint:unused // Reserved for potential future use or alternative implementation
// nakWithStrategy sends a NAK with appropriate delay based on retry attempt.
func (c *Consumer) nakWithStrategy(msg *nats.Msg) error {
backoff := c.cfg.NATS.ConsumerRules.Backoff
@@ -74,6 +75,7 @@ func sleepWithContext(ctx context.Context, duration time.Duration) bool {
}
}
//nolint:unused // Reserved for potential future use or alternative implementation
// fetchBatch fetches a batch of messages from the subscription.
// It respects context cancellation for faster shutdown.
func (c *Consumer) fetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error) {
@@ -102,7 +104,11 @@ func (c *Consumer) handleFetchError(ctx context.Context, err error, sub **nats.S
if recErr := c.recoverJetStreamResources(); recErr != nil {
return nil, recErr
}
(*sub).Unsubscribe()
if *sub != nil {
if unsubErr := (*sub).Unsubscribe(); unsubErr != nil {
c.logger.Error("Failed to unsubscribe during recovery", zap.Error(unsubErr))
}
}
return c.createPullSubscriptionWithRecovery()
})
@@ -125,10 +131,12 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
// Use a closure that always cleans up the current subscription.
// When subscription is replaced in handleFetchError, this will clean up
// whatever currentSub points to at shutdown time.
var currentSub *nats.Subscription = sub
var currentSub = sub
cleanupSubscriber := func() {
if currentSub != nil {
currentSub.Unsubscribe()
if err := currentSub.Unsubscribe(); err != nil {
c.logger.Error("Failed to unsubscribe subscription", zap.Error(err))
}
currentSub = nil
}
}
@@ -174,14 +182,14 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
}
// Fetch messages in batch
msgs, err := c.fetchBatch(ctx, currentSub)
msgs, err := c.fetcher.FetchBatch(ctx, currentSub)
if err != nil {
// If context was cancelled, return immediately
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
c.logger.Info("Stopping consumer due to context cancellation", zap.Error(err))
return err
}
shouldContinue, handleErr := c.handleFetchError(ctx, err, &currentSub, &fetchErrorStreak)
shouldContinue, handleErr := c.fetcher.HandleFetchError(ctx, err, &currentSub, &fetchErrorStreak)
if !shouldContinue {
return handleErr
}
@@ -194,6 +202,6 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
}
// Process batch
c.processBatch(ctx, msgs)
c.batchProcessor.ProcessBatch(ctx, msgs)
}
}
+2
View File
@@ -14,6 +14,7 @@ import (
"go.uber.org/zap"
)
//nolint:unused // Reserved for potential future use or alternative implementation
// processBatch processes a batch of messages, handling errors and applying backpressure.
// It checks context cancellation between messages for faster shutdown.
func (c *Consumer) processBatch(ctx context.Context, msgs []*nats.Msg) {
@@ -31,6 +32,7 @@ func (c *Consumer) processBatch(ctx context.Context, msgs []*nats.Msg) {
}
}
//nolint:unused // Reserved for potential future use or alternative implementation
// processSingleMessage processes a single message with error handling and backpressure.
func (c *Consumer) processSingleMessage(ctx context.Context, msg *nats.Msg) {
start := time.Now()
+87 -4
View File
@@ -2,17 +2,19 @@ package nats
import (
"caatsm/internal/infra/config"
"crypto/tls"
"crypto/x509"
"fmt"
"os"
"strings"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
)
// ProvideNATSConn creates a reusable NATS connection.
// ProvideNATSConn creates a reusable NATS connection with optional authentication.
func ProvideNATSConn(cfg *config.Config, logger *zap.Logger) (*nats.Conn, error) {
nc, err := nats.Connect(
cfg.NATS.URL,
opts := []nats.Option{
nats.RetryOnFailedConnect(true),
nats.Timeout(cfg.Timeouts.Server),
nats.ReconnectWait(cfg.Timeouts.ReconnectWait),
@@ -27,7 +29,16 @@ func ProvideNATSConn(cfg *config.Config, logger *zap.Logger) (*nats.Conn, error)
safeURL := sanitizeURLForLogging(nc.ConnectedUrl())
logger.Info("NATS reconnected", zap.String("url", safeURL))
}),
)
}
// Apply authentication options
authOpts, err := buildAuthOptions(&cfg.NATS.Auth, logger)
if err != nil {
return nil, fmt.Errorf("failed to build auth options: %w", err)
}
opts = append(opts, authOpts...)
nc, err := nats.Connect(cfg.NATS.URL, opts...)
if err != nil {
safeURL := sanitizeURLForLogging(cfg.NATS.URL)
logger.Error("failed to connect to NATS",
@@ -42,6 +53,78 @@ func ProvideNATSConn(cfg *config.Config, logger *zap.Logger) (*nats.Conn, error)
return nc, nil
}
// buildAuthOptions builds NATS connection options based on authentication configuration.
func buildAuthOptions(auth *config.NATSAuthConfig, logger *zap.Logger) ([]nats.Option, error) {
var opts []nats.Option
authMethods := 0
// Token authentication (highest priority)
if auth.Token != "" {
authMethods++
logger.Debug("Using NATS token authentication")
opts = append(opts, nats.Token(auth.Token))
}
// Credentials file authentication
if auth.CredentialsFile != "" {
authMethods++
if authMethods > 1 {
return nil, fmt.Errorf("multiple authentication methods specified: only one of token, credentials_file, or user/password can be used")
}
logger.Debug("Using NATS credentials file authentication", zap.String("file", auth.CredentialsFile))
opts = append(opts, nats.UserCredentials(auth.CredentialsFile))
}
// User/Password authentication
if auth.User != "" || auth.Password != "" {
authMethods++
if authMethods > 1 {
return nil, fmt.Errorf("multiple authentication methods specified: only one of token, credentials_file, or user/password can be used")
}
if auth.User == "" || auth.Password == "" {
return nil, fmt.Errorf("both user and password must be specified for user/password authentication")
}
logger.Debug("Using NATS user/password authentication", zap.String("user", auth.User))
opts = append(opts, nats.UserInfo(auth.User, auth.Password))
}
// TLS configuration
if auth.TLSEnabled {
tlsConfig := &tls.Config{
MinVersion: tls.VersionTLS12,
}
// Load client certificate and key if provided
if auth.TLSCertFile != "" && auth.TLSKeyFile != "" {
cert, err := tls.LoadX509KeyPair(auth.TLSCertFile, auth.TLSKeyFile)
if err != nil {
return nil, fmt.Errorf("failed to load TLS certificate: %w", err)
}
tlsConfig.Certificates = []tls.Certificate{cert}
logger.Debug("Loaded TLS client certificate", zap.String("cert", auth.TLSCertFile))
}
// Load CA certificate for server verification if provided
if auth.TLSCAFile != "" {
caCert, err := os.ReadFile(auth.TLSCAFile)
if err != nil {
return nil, fmt.Errorf("failed to read CA certificate file: %w", err)
}
caCertPool := x509.NewCertPool()
if !caCertPool.AppendCertsFromPEM(caCert) {
return nil, fmt.Errorf("failed to parse CA certificate from %s", auth.TLSCAFile)
}
tlsConfig.RootCAs = caCertPool
logger.Debug("Loaded TLS CA certificate", zap.String("ca_file", auth.TLSCAFile))
}
opts = append(opts, nats.Secure(tlsConfig))
logger.Debug("TLS enabled for NATS connection")
}
return opts, nil
}
// ProvideJetStream creates a NATS JetStream context using an existing connection.
// Returns nil, nil when cfg.NATS.Mode == "core" to support plain NATS servers without JetStream.
func ProvideJetStream(nc *nats.Conn, cfg *config.Config, logger *zap.Logger) (nats.JetStreamContext, error) {
+36 -9
View File
@@ -14,45 +14,72 @@ var _ = Describe("Utils", func() {
originalEnv := os.Getenv("GO_ENV")
DeferCleanup(func() {
if originalEnv == "" {
os.Unsetenv("GO_ENV")
if err := os.Unsetenv("GO_ENV"); err != nil {
// Environment variables are optional, ignore cleanup errors in tests
_ = err
}
} else {
os.Setenv("GO_ENV", originalEnv)
if err := os.Setenv("GO_ENV", originalEnv); err != nil {
// Environment variables are optional, ignore cleanup errors in tests
_ = err
}
}
})
})
It("returns true for dev environment", func() {
os.Setenv("GO_ENV", "dev")
if err := os.Setenv("GO_ENV", "dev"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeTrue())
})
It("returns true for development environment", func() {
os.Setenv("GO_ENV", "development")
if err := os.Setenv("GO_ENV", "development"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeTrue())
})
It("returns true for test environment", func() {
os.Setenv("GO_ENV", "test")
if err := os.Setenv("GO_ENV", "test"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeTrue())
})
It("returns true for testing environment", func() {
os.Setenv("GO_ENV", "testing")
if err := os.Setenv("GO_ENV", "testing"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeTrue())
})
It("returns true for empty environment", func() {
os.Unsetenv("GO_ENV")
if err := os.Unsetenv("GO_ENV"); err != nil {
// Environment variables are optional, ignore cleanup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeTrue())
})
It("returns false for production environment", func() {
os.Setenv("GO_ENV", "prod")
if err := os.Setenv("GO_ENV", "prod"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeFalse())
})
It("returns false for production environment (uppercase)", func() {
os.Setenv("GO_ENV", "PROD")
if err := os.Setenv("GO_ENV", "PROD"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeFalse())
})
})