🔧 Update Go version in go.mod and enhance build process with versioning information. Modify Makefile and Taskfile to inject build metadata (version, commit, build time) into the binary. Improve README with instructions for custom version builds and document new build info features. Add benchmarks for message parsing and processing to improve performance testing capabilities.

This commit is contained in:
windyboy
2025-11-18 14:15:58 +08:00
parent 7f44b5389d
commit 06fc9cb9e0
27 changed files with 3009 additions and 55 deletions
@@ -0,0 +1,137 @@
package parser
import (
"testing"
)
// Sample messages for benchmarking
var (
benchARRMessage = `ZCZC TMQ2526 141605
FF ZBTJZPZX
141604 ZBACZQZX
(ARR-JAE7433/A0132-RKSI-ZBTJ1604)
NNNN`
benchDEPMessage = `ZCZC DEP5678 120915
DD KLAXZPZX
120914 KSFOZQZX
(DEP-ABC5678-A1234-ZBTJ1440-ZGGG)
NNNN`
benchCNLMessage = `ZCZC CNL9012 150631
FF ZBTJZPZX
(CNL-CCA9012-ZBTJ-ZGGG)
NNNN`
benchDLAMessage = `ZCZC DLA3456 150631
FF ZBTJZPZX
(DLA-CCA3456-A1234-ZBTJ1600-ZGGG0200)
NNNN`
benchFPLMessage = `ZCZC TMQ2617 142150
GG ZBTJZPZX
150551 ZBTJUOBK
(FPL-OKA2861-IS
-MA60/M-SHID/C
-ZBTJ0030
-K0420S0450 CG J1 FZ
-ZSYT0100 ZSQD ZYTL
-REG/B3710 SEL/ RMK/TCAS )
NNNN`
benchComplexFPLMessage = `ZCZC FPL7890 150631
FF ZBTJZPZX
(FPL-JAE7433-IS
-B744/H-SXIRPZJWY/S
-ZBTJ1755
-K0926S0920 CG A326 VYK W80 HUR B339 GM A575 MANSA/K0919S0980
-EDDF0948 EDDK
-EET/ZMUB0100 UNKL0236
REG/B2422 SEL/JLAD
NAV/RNAV1 RNAV5 RNP4
RMK/AGCS EQUIPPED)
NNNN`
)
// BenchmarkParseARR benchmarks parsing ARR messages
func BenchmarkParseARR(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = Parse(benchARRMessage)
}
}
// BenchmarkParseDEP benchmarks parsing DEP messages
func BenchmarkParseDEP(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = Parse(benchDEPMessage)
}
}
// BenchmarkParseCNL benchmarks parsing CNL messages
func BenchmarkParseCNL(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = Parse(benchCNLMessage)
}
}
// BenchmarkParseDLA benchmarks parsing DLA messages
func BenchmarkParseDLA(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = Parse(benchDLAMessage)
}
}
// BenchmarkParseFPL benchmarks parsing simple FPL messages
func BenchmarkParseFPL(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = Parse(benchFPLMessage)
}
}
// BenchmarkParseComplexFPL benchmarks parsing complex FPL messages with extensive route and metadata
func BenchmarkParseComplexFPL(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = Parse(benchComplexFPLMessage)
}
}
// BenchmarkParseHeader benchmarks header parsing only
func BenchmarkParseHeader(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = ParseHeader(benchARRMessage)
}
}
// BenchmarkParseBody benchmarks body parsing only (ARR)
func BenchmarkParseBody(b *testing.B) {
body := `(ARR-JAE7433/A0132-RKSI-ZBTJ1604)`
parser := NewBodyParser(body)
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _, _ = parser.Parse()
}
}
// BenchmarkParseMixed benchmarks parsing a mix of message types
func BenchmarkParseMixed(b *testing.B) {
messages := []string{
benchARRMessage,
benchDEPMessage,
benchCNLMessage,
benchDLAMessage,
benchFPLMessage,
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
msg := messages[i%len(messages)]
_, _ = Parse(msg)
}
}
+147
View File
@@ -0,0 +1,147 @@
package app
import (
"caatsm/internal/adapter/dto"
"caatsm/internal/adapter/parser"
"caatsm/internal/infra/telemetry"
"context"
"testing"
"go.uber.org/zap"
)
// Mock implementations for benchmarking
type mockRepository struct{}
func (m *mockRepository) InsertOne(ctx context.Context, msg *dto.ParsedTelegram) error {
return nil
}
func (m *mockRepository) InsertBatch(ctx context.Context, msgs []*dto.ParsedTelegram) error {
return nil
}
func (m *mockRepository) InsertRaw(ctx context.Context, msg *dto.ParsedTelegram) error {
return nil
}
type mockPublisher struct{}
func (m *mockPublisher) Publish(message interface{}) error {
return nil
}
// Benchmark data
var (
benchARRRaw = []byte(`ZCZC TMQ2526 141605
FF ZBTJZPZX
141604 ZBACZQZX
(ARR-JAE7433/A0132-RKSI-ZBTJ1604)
NNNN`)
benchDEPRaw = []byte(`ZCZC DEP5678 120915
DD KLAXZPZX
120914 KSFOZQZX
(DEP-ABC5678-A1234-ZBTJ1440-ZGGG)
NNNN`)
benchFPLRaw = []byte(`ZCZC TMQ2617 142150
GG ZBTJZPZX
150551 ZBTJUOBK
(FPL-OKA2861-IS
-MA60/M-SHID/C
-ZBTJ0030
-K0420S0450 CG J1 FZ
-ZSYT0100 ZSQD ZYTL
-REG/B3710 SEL/ RMK/TCAS )
NNNN`)
)
// createBenchmarkProcessor creates a processor with mocks for benchmarking
func createBenchmarkProcessor() *MessageProcessor {
aviationParser := parser.ProvideParser()
mockRepo := &mockRepository{}
mockPub := &mockPublisher{}
logger := zap.NewNop()
recorder := telemetry.NewNoop()
return NewMessageProcessor(aviationParser, mockRepo, mockPub, recorder, logger)
}
// BenchmarkHandleARR benchmarks processing ARR messages end-to-end
func BenchmarkHandleARR(b *testing.B) {
processor := createBenchmarkProcessor()
ctx := context.Background()
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_ = processor.Handle(ctx, benchARRRaw, "msg-arr-123")
}
}
// BenchmarkHandleDEP benchmarks processing DEP messages end-to-end
func BenchmarkHandleDEP(b *testing.B) {
processor := createBenchmarkProcessor()
ctx := context.Background()
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_ = processor.Handle(ctx, benchDEPRaw, "msg-dep-123")
}
}
// BenchmarkHandleFPL benchmarks processing FPL messages end-to-end
func BenchmarkHandleFPL(b *testing.B) {
processor := createBenchmarkProcessor()
ctx := context.Background()
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_ = processor.Handle(ctx, benchFPLRaw, "msg-fpl-123")
}
}
// BenchmarkHandleMixed benchmarks processing a mix of message types
func BenchmarkHandleMixed(b *testing.B) {
processor := createBenchmarkProcessor()
ctx := context.Background()
messages := [][]byte{benchARRRaw, benchDEPRaw, benchFPLRaw}
msgIDs := []string{"msg-arr", "msg-dep", "msg-fpl"}
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
idx := i % len(messages)
_ = processor.Handle(ctx, messages[idx], msgIDs[idx])
}
}
// BenchmarkHandleParseOnly benchmarks parsing without persistence/publishing
// This isolates parser performance
func BenchmarkHandleParseOnly(b *testing.B) {
aviationParser := parser.ProvideParser()
// Use a repository that does nothing
mockRepo := &mockRepository{}
// Use a publisher that does nothing
mockPub := &mockPublisher{}
logger := zap.NewNop()
recorder := telemetry.NewNoop()
processor := NewMessageProcessor(aviationParser, mockRepo, mockPub, recorder, logger)
ctx := context.Background()
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_ = processor.Handle(ctx, benchARRRaw, "msg-parse-only")
}
}
+24 -2
View File
@@ -35,11 +35,32 @@ type NATSConfig struct {
Consumer string `koanf:"consumer"`
StreamLimits StreamLimitsConfig `koanf:"stream_limits"`
ConsumerRules ConsumerRulesConfig `koanf:"consumer_rules"`
Auth NATSAuthConfig `koanf:"auth"`
// Legacy fields
Client string `koanf:"client"`
Cluster string `koanf:"cluster"`
}
// NATSAuthConfig holds NATS authentication configuration
type NATSAuthConfig struct {
// Token authentication (mutually exclusive with User/Password and CredentialsFile)
Token string `koanf:"token"`
// Credentials file authentication (mutually exclusive with Token and User/Password)
// Path to NATS credentials file (e.g., /path/to/user.creds)
CredentialsFile string `koanf:"credentials_file"`
// User/Password authentication (mutually exclusive with Token and CredentialsFile)
User string `koanf:"user"`
Password string `koanf:"password"`
// TLS configuration
TLSEnabled bool `koanf:"tls_enabled"`
TLSCertFile string `koanf:"tls_cert_file"` // Client certificate file
TLSKeyFile string `koanf:"tls_key_file"` // Client private key file
TLSCAFile string `koanf:"tls_ca_file"` // CA certificate file for server verification
}
// StreamLimitsConfig defines JetStream retention controls.
type StreamLimitsConfig struct {
MaxMsgs int64 `koanf:"max_msgs"`
@@ -173,8 +194,9 @@ func LoadConfig() (*Config, error) {
return strings.ToLower(strings.ReplaceAll(s, "_", "."))
})
if err := k.Load(envProvider, nil); err != nil {
// Environment variables are optional, so we don't fail if they're not present
// This allows the config to work with just the file
// Environment variables are optional, so we don't fail if they're not present.
// This allows the config to work with just the file.
_ = err // explicitly ignore
}
// Unmarshal into Config struct
+6 -1
View File
@@ -37,7 +37,12 @@ var _ = Describe("ProvideLogger", func() {
Context("when file output is configured", func() {
It("creates the directory before writing logs", func() {
tmpDir := filepath.Join(os.TempDir(), "caatsm-log-test")
defer os.RemoveAll(tmpDir)
defer func() {
if err := os.RemoveAll(tmpDir); err != nil {
// Cleanup errors in tests are not critical
_ = err
}
}()
logPath := filepath.Join(tmpDir, "child", "app.log")
cfg := &configpkg.Config{
Log: configpkg.LogConfig{
+3 -1
View File
@@ -85,7 +85,9 @@ func (h *AdvisoryDLQHandler) Start(ctx context.Context) error {
// Wait for context cancellation
go func() {
<-ctx.Done()
sub.Unsubscribe()
if err := sub.Unsubscribe(); err != nil {
h.logger.Error("Failed to unsubscribe advisory subscription", zap.Error(err))
}
h.logger.Info("Stopped advisory DLQ handler")
}()
+261 -13
View File
@@ -3,6 +3,8 @@ package nats
import (
"caatsm/internal/app"
"caatsm/internal/infra/config"
"caatsm/internal/infra/log"
obsmetrics "caatsm/internal/infra/metrics"
"caatsm/internal/infra/telemetry"
"context"
"encoding/json"
@@ -11,7 +13,11 @@ import (
"strings"
"time"
"github.com/google/uuid"
"github.com/nats-io/nats.go"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/metric"
"go.uber.org/zap"
)
@@ -167,7 +173,9 @@ func (f *defaultMessageFetcher) HandleFetchError(ctx context.Context, err error,
)
// Connection closed is fatal - cannot recover subscription
if *sub != nil {
(*sub).Unsubscribe()
if err := (*sub).Unsubscribe(); err != nil {
f.logger.Error("Failed to unsubscribe after connection closed", zap.Error(err))
}
*sub = nil
}
return false, fmt.Errorf("connection closed: %w", err)
@@ -209,7 +217,9 @@ func (f *defaultMessageFetcher) HandleFetchError(ctx context.Context, err error,
}
// Unsubscribe old subscription before creating new one
if *sub != nil {
(*sub).Unsubscribe()
if err := (*sub).Unsubscribe(); err != nil {
f.logger.Error("Failed to unsubscribe during recovery", zap.Error(err))
}
}
// Create new subscription
newSub, subErr := f.consumerManager.CreatePullSubscription()
@@ -229,7 +239,9 @@ func (f *defaultMessageFetcher) HandleFetchError(ctx context.Context, err error,
zap.String("consumer", f.config.consumerName),
)
if *sub != nil {
(*sub).Unsubscribe()
if err := (*sub).Unsubscribe(); err != nil {
f.logger.Error("Failed to unsubscribe after resource not found", zap.Error(err))
}
*sub = nil
}
return false, fmt.Errorf("JetStream resource not found: %w", err)
@@ -322,7 +334,9 @@ func (f *defaultMessageFetcher) attemptSubscriptionRecovery(ctx context.Context,
// Unsubscribe old subscription if it exists
if *sub != nil {
(*sub).Unsubscribe()
if err := (*sub).Unsubscribe(); err != nil {
f.logger.Error("Failed to unsubscribe during recovery", zap.Error(err))
}
*sub = nil
}
@@ -368,10 +382,238 @@ type defaultBatchProcessor struct {
errorHandler *ErrorHandler
logger *zap.Logger
telemetry telemetry.Recorder
// Configuration needed for processing
streamName string
consumerName string
mode string
backoff []time.Duration
// Pointer to consecutive errors counter (shared with Consumer)
consecutiveProcessErrors *int
}
func (p *defaultBatchProcessor) ProcessBatch(ctx context.Context, msgs []*nats.Msg) {
// This will be implemented when we refactor the batch processing
for _, msg := range msgs {
// Check context before processing each message
select {
case <-ctx.Done():
p.logger.Info("Stopping batch processing due to cancellation",
zap.Int("remaining_messages", len(msgs)),
)
return
default:
}
p.processSingleMessage(ctx, msg)
}
}
// processSingleMessage processes a single message with error handling and backpressure.
func (p *defaultBatchProcessor) processSingleMessage(ctx context.Context, msg *nats.Msg) {
start := time.Now()
if err := p.processMessage(ctx, msg); err != nil {
p.handleMessageError(ctx, msg, err, time.Since(start))
return
}
// Successful processing resets the error streak.
if p.consecutiveProcessErrors != nil && *p.consecutiveProcessErrors > 0 {
*p.consecutiveProcessErrors = 0
}
// ACK the message
if ackErr := msg.Ack(); ackErr != nil {
p.logger.Error("Failed to ACK message", zap.Error(ackErr))
} else {
elapsed := time.Since(start)
p.telemetry.RecordMessageHandled(ctx, p.streamName, p.consumerName, "ok", elapsed)
}
}
// processMessage processes a single message.
func (p *defaultBatchProcessor) processMessage(ctx context.Context, msg *nats.Msg) error {
ctx, span := otel.Tracer("caatsm/nats").Start(ctx, "Consumer.processMessage")
defer span.End()
// Set semantic messaging attributes
span.SetAttributes(
attribute.String("messaging.system", "nats"),
attribute.String("messaging.operation.name", "receive"),
attribute.String("messaging.destination.name", msg.Subject),
attribute.String("messaging.consumer.group.name", p.consumerName),
attribute.String("caatsm.stream", p.streamName),
)
msgID, source, err := p.resolveMsgID(msg)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("unable to resolve message id: %w", err)
}
if source != "header" {
p.logger.Warn("Message missing NATS id header; using fallback",
zap.String("subject", msg.Subject),
zap.String("msg_id_source", source),
zap.String("msg_id", msgID),
)
}
// Attach structured logging context including stream/consumer and NATS metadata.
jsSeq := uint64(0)
if meta, metaErr := msg.Metadata(); metaErr == nil {
jsSeq = meta.Sequence.Stream
span.SetAttributes(
attribute.Int64("nats.js.stream_seq", int64(meta.Sequence.Stream)),
attribute.Int64("nats.js.consumer_seq", int64(meta.Sequence.Consumer)),
)
}
msgLogger := log.WithMessageContext(p.logger, log.MessageFields{
Service: "caatsm-consumer",
TransportMsgID: msgID,
Stream: p.streamName,
Consumer: p.consumerName,
Subject: msg.Subject,
JSSequence: jsSeq,
})
msgLogger.Debug("Processing message",
zap.Int("data_size", len(msg.Data)),
zap.String("msg_id_source", source),
)
// Call processor
if err := p.processor.Handle(ctx, msg.Data, msgID); err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("processor error: %w", err)
}
span.SetAttributes(attribute.String("telegram.msg_id", msgID))
return nil
}
// resolveMsgID extracts or generates a message ID.
func (p *defaultBatchProcessor) resolveMsgID(msg *nats.Msg) (string, string, error) {
if id := msg.Header.Get("Nats-Msg-Id"); id != "" {
return id, "header", nil
}
if p.mode == "core" {
return uuid.NewString(), "generated", nil
}
meta, err := msg.Metadata()
if err != nil {
return "", "", fmt.Errorf("fetch metadata: %w", err)
}
return fmt.Sprintf("js-%d", meta.Sequence.Stream), "metadata", nil
}
// handleMessageError handles errors that occur during message processing.
func (p *defaultBatchProcessor) handleMessageError(ctx context.Context, msg *nats.Msg, err error, elapsed time.Duration) {
p.logger.Error("Failed to process message",
zap.String("subject", msg.Subject),
zap.Error(err),
zap.Bool("permanent", app.IsPermanent(err)),
)
result := obsmetrics.ResultFail
if app.IsPermanent(err) {
result = obsmetrics.ResultPermanentFail
}
p.telemetry.RecordMessageHandled(ctx, p.streamName, p.consumerName, result, elapsed)
consecutiveErrors := 0
if p.consecutiveProcessErrors != nil {
consecutiveErrors = *p.consecutiveProcessErrors
}
processingResult := p.errorHandler.HandleProcessingError(consecutiveErrors, err, p.logger, msg.Subject)
if processingResult.IsPermanent {
p.handlePermanentError(ctx, msg, err)
return
}
p.handleTransientError(ctx, msg, processingResult)
}
// handlePermanentError handles permanent/poison messages.
func (p *defaultBatchProcessor) handlePermanentError(ctx context.Context, msg *nats.Msg, err error) {
if p.consecutiveProcessErrors != nil {
*p.consecutiveProcessErrors = 0
}
// Poison/permanent message: route to DLQ if configured, then ACK
if p.dlqHandler != nil {
if dlqErr := p.dlqHandler.RouteToDLQ(ctx, msg, err); dlqErr != nil {
p.logger.Error("Failed to route permanent-error message to DLQ", zap.Error(dlqErr))
}
}
if ackErr := msg.Ack(); ackErr != nil {
p.logger.Error("Failed to ACK permanent-error message", zap.Error(ackErr))
}
}
// handleTransientError handles transient errors with backpressure and redelivery.
func (p *defaultBatchProcessor) handleTransientError(ctx context.Context, msg *nats.Msg, processingResult ProcessingErrorResult) {
// Increment error streak
if p.consecutiveProcessErrors != nil {
if *p.consecutiveProcessErrors < 0 {
*p.consecutiveProcessErrors = 0
}
*p.consecutiveProcessErrors++
}
if processingResult.ShouldApplyBackpressure {
consecutiveErrors := 0
if p.consecutiveProcessErrors != nil {
consecutiveErrors = *p.consecutiveProcessErrors
}
p.logger.Warn("Applying backpressure due to consecutive processing errors",
zap.Int("consecutive_errors", consecutiveErrors),
zap.Duration("sleep", processingResult.BackpressureDelay),
)
// Use context-aware sleep instead of blocking time.Sleep
if !sleepWithContext(ctx, processingResult.BackpressureDelay) {
// Context canceled, stop processing
return
}
}
// Transient error: request redelivery with optional delay
p.telemetry.RecordRetry(ctx, p.streamName, p.consumerName, obsmetrics.RetryReasonProcessorError)
if nakErr := p.nakWithStrategy(msg); nakErr != nil {
p.logger.Error("Failed to NAK message", zap.Error(nakErr))
}
}
// nakWithStrategy sends a NAK with appropriate delay based on retry attempt.
func (p *defaultBatchProcessor) nakWithStrategy(msg *nats.Msg) error {
if len(p.backoff) == 0 {
return msg.Nak()
}
meta, err := msg.Metadata()
if err != nil {
p.logger.Warn("Failed to read metadata for backoff strategy", zap.Error(err))
return msg.Nak()
}
attempt := int(meta.NumDelivered)
index := attempt - 1
if index < 0 {
index = 0
}
if index >= len(p.backoff) {
index = len(p.backoff) - 1
}
delay := p.backoff[index]
if delay <= 0 {
return msg.Nak()
}
return msg.NakWithDelay(delay)
}
// defaultDLQHandler implements DLQHandler interface
@@ -454,14 +696,7 @@ func (c *Consumer) initCollaborators() {
cfg: c.cfg,
}
c.batchProcessor = &defaultBatchProcessor{
processor: c.processor,
dlqHandler: c.dlqHandler,
errorHandler: c.errorHandler,
logger: c.logger,
telemetry: c.telemetry,
}
// Initialize DLQ handler first if needed, so batch processor can reference it
if c.config.dlqSubject != "" {
c.dlqHandler = &defaultDLQHandler{
js: c.js,
@@ -472,6 +707,19 @@ func (c *Consumer) initCollaborators() {
telemetry: c.telemetry,
}
}
c.batchProcessor = &defaultBatchProcessor{
processor: c.processor,
dlqHandler: c.dlqHandler,
errorHandler: c.errorHandler,
logger: c.logger,
telemetry: c.telemetry,
streamName: c.config.streamName,
consumerName: c.config.consumerName,
mode: c.config.mode,
backoff: c.cfg.NATS.ConsumerRules.Backoff,
consecutiveProcessErrors: &c.consecutiveProcessErrors,
}
}
// normalizeConsumerConfig extracts and normalizes consumer configuration from the application config.
+6
View File
@@ -11,6 +11,8 @@ import (
)
// handleMessageError handles errors that occur during message processing.
//
//nolint:unused // Reserved for potential future use or alternative implementation
func (c *Consumer) handleMessageError(ctx context.Context, msg *nats.Msg, err error, elapsed time.Duration) {
c.logger.Error("Failed to process message",
zap.String("subject", msg.Subject),
@@ -35,6 +37,8 @@ func (c *Consumer) handleMessageError(ctx context.Context, msg *nats.Msg, err er
}
// handlePermanentError handles permanent/poison messages.
//
//nolint:unused // Reserved for potential future use or alternative implementation
func (c *Consumer) handlePermanentError(ctx context.Context, msg *nats.Msg, err error) {
c.consecutiveProcessErrors = 0
// Poison/permanent message: route to DLQ if configured, then ACK
@@ -47,6 +51,8 @@ func (c *Consumer) handlePermanentError(ctx context.Context, msg *nats.Msg, err
}
// handleTransientError handles transient errors with backpressure and redelivery.
//
//nolint:unused // Reserved for potential future use or alternative implementation
func (c *Consumer) handleTransientError(ctx context.Context, msg *nats.Msg, processingResult ProcessingErrorResult) {
// Increment error streak
if c.consecutiveProcessErrors < 0 {
+14 -6
View File
@@ -32,6 +32,7 @@ func (c *Consumer) createPullSubscriptionWithRecovery() (*nats.Subscription, err
return c.consumerManager.CreatePullSubscriptionWithRecovery(c.streamManager, consumerConfig)
}
//nolint:unused // Reserved for potential future use or alternative implementation
// nakWithStrategy sends a NAK with appropriate delay based on retry attempt.
func (c *Consumer) nakWithStrategy(msg *nats.Msg) error {
backoff := c.cfg.NATS.ConsumerRules.Backoff
@@ -74,6 +75,7 @@ func sleepWithContext(ctx context.Context, duration time.Duration) bool {
}
}
//nolint:unused // Reserved for potential future use or alternative implementation
// fetchBatch fetches a batch of messages from the subscription.
// It respects context cancellation for faster shutdown.
func (c *Consumer) fetchBatch(ctx context.Context, sub *nats.Subscription) ([]*nats.Msg, error) {
@@ -102,7 +104,11 @@ func (c *Consumer) handleFetchError(ctx context.Context, err error, sub **nats.S
if recErr := c.recoverJetStreamResources(); recErr != nil {
return nil, recErr
}
(*sub).Unsubscribe()
if *sub != nil {
if unsubErr := (*sub).Unsubscribe(); unsubErr != nil {
c.logger.Error("Failed to unsubscribe during recovery", zap.Error(unsubErr))
}
}
return c.createPullSubscriptionWithRecovery()
})
@@ -125,10 +131,12 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
// Use a closure that always cleans up the current subscription.
// When subscription is replaced in handleFetchError, this will clean up
// whatever currentSub points to at shutdown time.
var currentSub *nats.Subscription = sub
var currentSub = sub
cleanupSubscriber := func() {
if currentSub != nil {
currentSub.Unsubscribe()
if err := currentSub.Unsubscribe(); err != nil {
c.logger.Error("Failed to unsubscribe subscription", zap.Error(err))
}
currentSub = nil
}
}
@@ -174,14 +182,14 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
}
// Fetch messages in batch
msgs, err := c.fetchBatch(ctx, currentSub)
msgs, err := c.fetcher.FetchBatch(ctx, currentSub)
if err != nil {
// If context was cancelled, return immediately
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
c.logger.Info("Stopping consumer due to context cancellation", zap.Error(err))
return err
}
shouldContinue, handleErr := c.handleFetchError(ctx, err, &currentSub, &fetchErrorStreak)
shouldContinue, handleErr := c.fetcher.HandleFetchError(ctx, err, &currentSub, &fetchErrorStreak)
if !shouldContinue {
return handleErr
}
@@ -194,6 +202,6 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
}
// Process batch
c.processBatch(ctx, msgs)
c.batchProcessor.ProcessBatch(ctx, msgs)
}
}
+2
View File
@@ -14,6 +14,7 @@ import (
"go.uber.org/zap"
)
//nolint:unused // Reserved for potential future use or alternative implementation
// processBatch processes a batch of messages, handling errors and applying backpressure.
// It checks context cancellation between messages for faster shutdown.
func (c *Consumer) processBatch(ctx context.Context, msgs []*nats.Msg) {
@@ -31,6 +32,7 @@ func (c *Consumer) processBatch(ctx context.Context, msgs []*nats.Msg) {
}
}
//nolint:unused // Reserved for potential future use or alternative implementation
// processSingleMessage processes a single message with error handling and backpressure.
func (c *Consumer) processSingleMessage(ctx context.Context, msg *nats.Msg) {
start := time.Now()
+87 -4
View File
@@ -2,17 +2,19 @@ package nats
import (
"caatsm/internal/infra/config"
"crypto/tls"
"crypto/x509"
"fmt"
"os"
"strings"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
)
// ProvideNATSConn creates a reusable NATS connection.
// ProvideNATSConn creates a reusable NATS connection with optional authentication.
func ProvideNATSConn(cfg *config.Config, logger *zap.Logger) (*nats.Conn, error) {
nc, err := nats.Connect(
cfg.NATS.URL,
opts := []nats.Option{
nats.RetryOnFailedConnect(true),
nats.Timeout(cfg.Timeouts.Server),
nats.ReconnectWait(cfg.Timeouts.ReconnectWait),
@@ -27,7 +29,16 @@ func ProvideNATSConn(cfg *config.Config, logger *zap.Logger) (*nats.Conn, error)
safeURL := sanitizeURLForLogging(nc.ConnectedUrl())
logger.Info("NATS reconnected", zap.String("url", safeURL))
}),
)
}
// Apply authentication options
authOpts, err := buildAuthOptions(&cfg.NATS.Auth, logger)
if err != nil {
return nil, fmt.Errorf("failed to build auth options: %w", err)
}
opts = append(opts, authOpts...)
nc, err := nats.Connect(cfg.NATS.URL, opts...)
if err != nil {
safeURL := sanitizeURLForLogging(cfg.NATS.URL)
logger.Error("failed to connect to NATS",
@@ -42,6 +53,78 @@ func ProvideNATSConn(cfg *config.Config, logger *zap.Logger) (*nats.Conn, error)
return nc, nil
}
// buildAuthOptions builds NATS connection options based on authentication configuration.
func buildAuthOptions(auth *config.NATSAuthConfig, logger *zap.Logger) ([]nats.Option, error) {
var opts []nats.Option
authMethods := 0
// Token authentication (highest priority)
if auth.Token != "" {
authMethods++
logger.Debug("Using NATS token authentication")
opts = append(opts, nats.Token(auth.Token))
}
// Credentials file authentication
if auth.CredentialsFile != "" {
authMethods++
if authMethods > 1 {
return nil, fmt.Errorf("multiple authentication methods specified: only one of token, credentials_file, or user/password can be used")
}
logger.Debug("Using NATS credentials file authentication", zap.String("file", auth.CredentialsFile))
opts = append(opts, nats.UserCredentials(auth.CredentialsFile))
}
// User/Password authentication
if auth.User != "" || auth.Password != "" {
authMethods++
if authMethods > 1 {
return nil, fmt.Errorf("multiple authentication methods specified: only one of token, credentials_file, or user/password can be used")
}
if auth.User == "" || auth.Password == "" {
return nil, fmt.Errorf("both user and password must be specified for user/password authentication")
}
logger.Debug("Using NATS user/password authentication", zap.String("user", auth.User))
opts = append(opts, nats.UserInfo(auth.User, auth.Password))
}
// TLS configuration
if auth.TLSEnabled {
tlsConfig := &tls.Config{
MinVersion: tls.VersionTLS12,
}
// Load client certificate and key if provided
if auth.TLSCertFile != "" && auth.TLSKeyFile != "" {
cert, err := tls.LoadX509KeyPair(auth.TLSCertFile, auth.TLSKeyFile)
if err != nil {
return nil, fmt.Errorf("failed to load TLS certificate: %w", err)
}
tlsConfig.Certificates = []tls.Certificate{cert}
logger.Debug("Loaded TLS client certificate", zap.String("cert", auth.TLSCertFile))
}
// Load CA certificate for server verification if provided
if auth.TLSCAFile != "" {
caCert, err := os.ReadFile(auth.TLSCAFile)
if err != nil {
return nil, fmt.Errorf("failed to read CA certificate file: %w", err)
}
caCertPool := x509.NewCertPool()
if !caCertPool.AppendCertsFromPEM(caCert) {
return nil, fmt.Errorf("failed to parse CA certificate from %s", auth.TLSCAFile)
}
tlsConfig.RootCAs = caCertPool
logger.Debug("Loaded TLS CA certificate", zap.String("ca_file", auth.TLSCAFile))
}
opts = append(opts, nats.Secure(tlsConfig))
logger.Debug("TLS enabled for NATS connection")
}
return opts, nil
}
// ProvideJetStream creates a NATS JetStream context using an existing connection.
// Returns nil, nil when cfg.NATS.Mode == "core" to support plain NATS servers without JetStream.
func ProvideJetStream(nc *nats.Conn, cfg *config.Config, logger *zap.Logger) (nats.JetStreamContext, error) {
+36 -9
View File
@@ -14,45 +14,72 @@ var _ = Describe("Utils", func() {
originalEnv := os.Getenv("GO_ENV")
DeferCleanup(func() {
if originalEnv == "" {
os.Unsetenv("GO_ENV")
if err := os.Unsetenv("GO_ENV"); err != nil {
// Environment variables are optional, ignore cleanup errors in tests
_ = err
}
} else {
os.Setenv("GO_ENV", originalEnv)
if err := os.Setenv("GO_ENV", originalEnv); err != nil {
// Environment variables are optional, ignore cleanup errors in tests
_ = err
}
}
})
})
It("returns true for dev environment", func() {
os.Setenv("GO_ENV", "dev")
if err := os.Setenv("GO_ENV", "dev"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeTrue())
})
It("returns true for development environment", func() {
os.Setenv("GO_ENV", "development")
if err := os.Setenv("GO_ENV", "development"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeTrue())
})
It("returns true for test environment", func() {
os.Setenv("GO_ENV", "test")
if err := os.Setenv("GO_ENV", "test"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeTrue())
})
It("returns true for testing environment", func() {
os.Setenv("GO_ENV", "testing")
if err := os.Setenv("GO_ENV", "testing"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeTrue())
})
It("returns true for empty environment", func() {
os.Unsetenv("GO_ENV")
if err := os.Unsetenv("GO_ENV"); err != nil {
// Environment variables are optional, ignore cleanup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeTrue())
})
It("returns false for production environment", func() {
os.Setenv("GO_ENV", "prod")
if err := os.Setenv("GO_ENV", "prod"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeFalse())
})
It("returns false for production environment (uppercase)", func() {
os.Setenv("GO_ENV", "PROD")
if err := os.Setenv("GO_ENV", "PROD"); err != nil {
// Environment variables are optional, ignore setup errors in tests
_ = err
}
Expect(isDevLikeEnv()).To(BeFalse())
})
})
+5 -1
View File
@@ -37,6 +37,10 @@ func ProvideRepository(pool *pgxpool.Pool, logger *zap.Logger) (port.Repository,
// InsertOne inserts a single telegram message
func (r *Repository) InsertOne(ctx context.Context, msg *dto.ParsedTelegram) error {
if msg == nil {
return fmt.Errorf("message cannot be nil")
}
ctx, span := otel.Tracer("caatsm/postgres").Start(ctx, "Repository.InsertOne")
defer span.End()
@@ -53,7 +57,7 @@ func (r *Repository) InsertOne(ctx context.Context, msg *dto.ParsedTelegram) err
// Optional idempotency check based on business message identity. If we have a
// non-empty message ID and date/time, we can cheaply skip duplicates here to
// avoid applying the same business event multiple times.
if msg != nil && msg.MessageID != "" && msg.DateTime != "" {
if msg.MessageID != "" && msg.DateTime != "" {
exists, err := r.messageExists(ctx, msg.MessageID, msg.DateTime)
if err != nil {
span.RecordError(err)
@@ -0,0 +1,169 @@
package postgres
import (
"caatsm/internal/adapter/dto"
"caatsm/internal/adapter/mapper"
"context"
"strconv"
"testing"
"time"
"github.com/google/uuid"
)
// Benchmark data setup
func createBenchmarkTelegram() *dto.ParsedTelegram {
return &dto.ParsedTelegram{
Uuid: uuid.New().String(),
MessageID: "TMQ1234",
DateTime: "150631",
PriorityIndicator: "FF",
PrimaryAddress: "ZBTJZPZX",
Category: "ARR",
Content: "ZCZC TMQ1234 150631\nFF ZBTJZPZX\n(ARR-ABC123-ZBTJ-ZGGG)\nNNNN",
Status: dto.MessageStatusParsed,
Parsed: true,
ReceivedAt: time.Now(),
ParsedAt: time.Now(),
}
}
func createBenchmarkTelegrams(count int) []*dto.ParsedTelegram {
telegrams := make([]*dto.ParsedTelegram, count)
for i := 0; i < count; i++ {
tg := createBenchmarkTelegram()
tg.Uuid = uuid.New().String()
tg.MessageID = "TMQ" + strconv.Itoa(1000+i)
telegrams[i] = tg
}
return telegrams
}
// BenchmarkInsertOne benchmarks single message insertion
// Note: This requires a database connection. Run with -tags=integration or provide test DB.
func BenchmarkInsertOne(b *testing.B) {
// Skip if no database available (integration tests only)
if testing.Short() {
b.Skip("Skipping benchmark in short mode")
}
// This benchmark requires a real database connection
// In practice, you would set up a test database connection here
// For now, we'll skip if not running integration tests
b.Skip("Requires database connection - run with integration tests")
}
// BenchmarkInsertOneWithDB benchmarks single message insertion with database
// This is a helper that can be used in integration test suites
//
//nolint:unused // Benchmark helper for future use
func benchmarkInsertOneWithDB(b *testing.B, repo *Repository) {
ctx := context.Background()
telegram := createBenchmarkTelegram()
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
// Update UUID for each iteration to avoid conflicts
telegram.Uuid = uuid.New().String()
telegram.MessageID = "TMQ" + strconv.Itoa(1000+i)
_ = repo.InsertOne(ctx, telegram)
}
}
// BenchmarkInsertBatch benchmarks batch message insertion
// Note: This requires a database connection. Run with -tags=integration or provide test DB.
func BenchmarkInsertBatch(b *testing.B) {
// Skip if no database available (integration tests only)
if testing.Short() {
b.Skip("Skipping benchmark in short mode")
}
// This benchmark requires a real database connection
// In practice, you would set up a test database connection here
// For now, we'll skip if not running integration tests
b.Skip("Requires database connection - run with integration tests")
}
// BenchmarkInsertBatchWithDB benchmarks batch insertion with database
// This is a helper that can be used in integration test suites
//
//nolint:unused // Benchmark helper for future use
func benchmarkInsertBatchWithDB(b *testing.B, repo *Repository, batchSize int) {
ctx := context.Background()
telegrams := createBenchmarkTelegrams(batchSize)
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
// Update UUIDs for each iteration to avoid conflicts
for j := range telegrams {
telegrams[j].Uuid = uuid.New().String()
telegrams[j].MessageID = "TMQ" + strconv.Itoa(1000+i*batchSize+j)
}
_ = repo.InsertBatch(ctx, telegrams)
}
}
// BenchmarkInsertBatch10 benchmarks batch insertion with 10 messages
func BenchmarkInsertBatch10(b *testing.B) {
if testing.Short() {
b.Skip("Skipping benchmark in short mode")
}
b.Skip("Requires database connection - run with integration tests")
}
// BenchmarkInsertBatch50 benchmarks batch insertion with 50 messages
func BenchmarkInsertBatch50(b *testing.B) {
if testing.Short() {
b.Skip("Skipping benchmark in short mode")
}
b.Skip("Requires database connection - run with integration tests")
}
// BenchmarkInsertBatch100 benchmarks batch insertion with 100 messages
func BenchmarkInsertBatch100(b *testing.B) {
if testing.Short() {
b.Skip("Skipping benchmark in short mode")
}
b.Skip("Requires database connection - run with integration tests")
}
// BenchmarkInsertRaw benchmarks raw message insertion
func BenchmarkInsertRaw(b *testing.B) {
if testing.Short() {
b.Skip("Skipping benchmark in short mode")
}
b.Skip("Requires database connection - run with integration tests")
}
// BenchmarkMapperToDBRow benchmarks the mapping from DTO to DB row
// This doesn't require a database connection
func BenchmarkMapperToDBRow(b *testing.B) {
m := mapper.NewTelegramMapper()
telegram := createBenchmarkTelegram()
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_, _ = m.ToDBRow(telegram)
}
}
// BenchmarkMapperToDBRowBatch benchmarks mapping multiple telegrams
func BenchmarkMapperToDBRowBatch(b *testing.B) {
m := mapper.NewTelegramMapper()
telegrams := createBenchmarkTelegrams(100)
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
for _, tg := range telegrams {
_, _ = m.ToDBRow(tg)
}
}
}