Enhance NATS configuration and error handling. Introduce stream limits and consumer rules in configuration files. Refactor message processing to handle permanent errors. Update README and development configuration to reflect changes. Add tests for new error handling mechanisms.

This commit is contained in:
windyboy
2025-11-14 21:42:04 +08:00
parent a574cfcf27
commit 9cce4610b6
13 changed files with 510 additions and 57 deletions
+17 -3
View File
@@ -78,11 +78,24 @@ url = "nats://localhost:4222"
stream = "TELEGRAM" stream = "TELEGRAM"
consumer = "telegram-consumer" consumer = "telegram-consumer"
[nats.stream_limits]
max_msgs = 100000
max_bytes = 67108864
max_age = "24h"
discard = "old"
storage = "file"
replicas = 1
[nats.consumer]
max_deliver = 5
ack_wait = "30s"
max_ack_pending = 1024
[subscription] [subscription]
topic = "Telegram.Serial" topic = "telegram.serial"
[publisher] [publisher]
topic = "Telegram.Json" topic = "telegram.json"
[postgres] [postgres]
url = "postgres://user:password@localhost:5432/aviation?sslmode=disable" url = "postgres://user:password@localhost:5432/aviation?sslmode=disable"
@@ -92,6 +105,7 @@ min_conns = 2
[app] [app]
batch_size = 50 batch_size = 50
batch_timeout = "2s" batch_timeout = "2s"
monitor_interval = "30s"
[log] [log]
level = "info" level = "info"
@@ -147,7 +161,7 @@ task run-dev
Flags: Flags:
-n, --nats string Nats server address (default: "nats://localhost:4222") -n, --nats string Nats server address (default: "nats://localhost:4222")
-t, --topic string Nats topic to listen to (default: "Telegram.Serial") -t, --topic string Nats topic to listen to (default: "telegram.serial")
``` ```
## Development ## Development
+26 -4
View File
@@ -3,10 +3,12 @@ package main
import ( import (
"caatsm/pkg/di" "caatsm/pkg/di"
"context" "context"
"errors"
"fmt" "fmt"
"os" "os"
"os/signal" "os/signal"
"syscall" "syscall"
"time"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
) )
@@ -39,7 +41,7 @@ func setupApp() *cli.App {
Name: "topic", Name: "topic",
Aliases: []string{"t"}, Aliases: []string{"t"},
Usage: "Nats topic to listen to", Usage: "Nats topic to listen to",
Value: "Telegram.Serial", Value: "telegram.serial",
EnvVars: []string{"NATS_SUBJECT"}, EnvVars: []string{"NATS_SUBJECT"},
}, },
}, },
@@ -72,18 +74,38 @@ func executeListen(c *cli.Context) error {
} }
}() }()
var runErr error
// Wait for signal or error // Wait for signal or error
select { select {
case sig := <-sigChan: case sig := <-sigChan:
fmt.Printf("Received signal: %v, shutting down...\n", sig) fmt.Printf("Received signal: %v, shutting down...\n", sig)
cancel() cancel()
case err := <-errChan: case err := <-errChan:
return err cancel()
if err != nil && !errors.Is(err, context.Canceled) {
runErr = err
}
}
waitTimeout := 5 * time.Second
select {
case err := <-errChan:
if err != nil && !errors.Is(err, context.Canceled) {
runErr = err
}
case <-time.After(waitTimeout):
fmt.Printf("Timed out waiting for consumer shutdown after %s\n", waitTimeout)
}
if err := consumer.Shutdown(context.Background()); err != nil {
if runErr == nil {
runErr = fmt.Errorf("failed to drain NATS connection: %w", err)
}
} }
// Note: processor is initialized but not directly used here // Note: processor is initialized but not directly used here
// It's used by the consumer internally
_ = processor _ = processor
return nil return runErr
} }
+16 -2
View File
@@ -5,12 +5,25 @@ cluster = "tele-cluster"
stream = "TELEGRAM" stream = "TELEGRAM"
consumer = "telegram-consumer" consumer = "telegram-consumer"
[nats.stream_limits]
max_msgs = 100000
max_bytes = 67108864
max_age = "24h"
discard = "old"
storage = "file"
replicas = 1
[nats.consumer]
max_deliver = 5
ack_wait = "30s"
max_ack_pending = 1024
[subscription] [subscription]
topic = "Telegram.Serial" topic = "telegram.serial"
queue = "tele-queue" queue = "tele-queue"
[publisher] [publisher]
topic = "Telegram.Json" topic = "telegram.json"
[timeouts] [timeouts]
server = "5s" server = "5s"
@@ -26,6 +39,7 @@ min_conns = 2
[app] [app]
batch_size = 50 batch_size = 50
batch_timeout = "2s" batch_timeout = "2s"
monitor_interval = "30s"
[log] [log]
level = "info" level = "info"
+39
View File
@@ -0,0 +1,39 @@
package app
import "errors"
// PermanentError indicates a failure that should not be retried.
type PermanentError struct {
err error
}
// Error implements the error interface.
func (e *PermanentError) Error() string {
if e == nil || e.err == nil {
return ""
}
return e.err.Error()
}
// Unwrap allows errors.Unwrap/Is/As to inspect the underlying error.
func (e *PermanentError) Unwrap() error {
if e == nil {
return nil
}
return e.err
}
// Permanent wraps err to mark it as non-retriable.
func Permanent(err error) error {
if err == nil {
return nil
}
return &PermanentError{err: err}
}
// IsPermanent reports whether the error or any wrapped error is permanent.
func IsPermanent(err error) bool {
var target *PermanentError
return errors.As(err, &target)
}
+33
View File
@@ -0,0 +1,33 @@
package app
import (
"errors"
"testing"
)
func TestPermanentWrapsError(t *testing.T) {
base := errors.New("boom")
perr := Permanent(base)
if perr == nil {
t.Fatalf("expected wrapped error, got nil")
}
if !IsPermanent(perr) {
t.Fatalf("expected IsPermanent to be true")
}
if !errors.Is(perr, base) {
t.Fatalf("expected wrapped error to unwrap to base")
}
if errors.Is(base, perr) {
t.Fatalf("expected base not to consider wrapper as same")
}
}
func TestPermanentNil(t *testing.T) {
if Permanent(nil) != nil {
t.Fatalf("Permanent(nil) should return nil")
}
if IsPermanent(nil) {
t.Fatalf("IsPermanent(nil) should be false")
}
}
+3 -3
View File
@@ -1,9 +1,9 @@
package app package app
import ( import (
"context"
"caatsm/internal/adapter" "caatsm/internal/adapter"
"caatsm/internal/adapter/parser" "caatsm/internal/adapter/parser"
"context"
"fmt" "fmt"
"go.uber.org/zap" "go.uber.org/zap"
) )
@@ -34,13 +34,13 @@ func NewMessageProcessor(
// Handle processes a message // Handle processes a message
func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string) error { func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string) error {
if raw == nil || len(raw) == 0 { if raw == nil || len(raw) == 0 {
return fmt.Errorf("empty message") return Permanent(fmt.Errorf("empty message"))
} }
// Parse the message // Parse the message
parsed := p.parser.Parse(string(raw)) parsed := p.parser.Parse(string(raw))
if parsed == nil { if parsed == nil {
return fmt.Errorf("parser returned nil") return Permanent(fmt.Errorf("parser returned nil"))
} }
// Set the message ID from NATS // Set the message ID from NATS
+115
View File
@@ -0,0 +1,115 @@
package app
import (
"caatsm/internal/adapter"
"caatsm/internal/adapter/parser"
"caatsm/internal/domain"
"context"
"errors"
"testing"
"go.uber.org/zap"
)
type stubParser struct {
value *domain.ParsedMessage
}
func (s *stubParser) Parse(rawText string) *domain.ParsedMessage {
return s.value
}
type stubRepository struct {
inserted []*domain.ParsedMessage
err error
}
func (s *stubRepository) InsertOne(ctx context.Context, msg *domain.ParsedMessage) error {
if s.err != nil {
return s.err
}
s.inserted = append(s.inserted, msg)
return nil
}
func (s *stubRepository) InsertBatch(ctx context.Context, msgs []*domain.ParsedMessage) error {
return errors.New("not implemented")
}
type stubPublisher struct {
last interface{}
err error
}
func (s *stubPublisher) Publish(message interface{}) error {
s.last = message
return s.err
}
func newTestProcessor(p parser.Parser, repo adapter.Repository, pub adapter.Publisher) *MessageProcessor {
return NewMessageProcessor(p, repo, pub, zap.NewNop())
}
func TestHandleEmptyMessageIsPermanent(t *testing.T) {
proc := newTestProcessor(&stubParser{}, &stubRepository{}, &stubPublisher{})
err := proc.Handle(context.Background(), nil, "id-1")
if err == nil || !IsPermanent(err) {
t.Fatalf("expected permanent error for empty message, got %v", err)
}
}
func TestHandleNilParserResultIsPermanent(t *testing.T) {
proc := newTestProcessor(&stubParser{value: nil}, &stubRepository{}, &stubPublisher{})
err := proc.Handle(context.Background(), []byte("payload"), "id-2")
if err == nil || !IsPermanent(err) {
t.Fatalf("expected permanent error for nil parser result, got %v", err)
}
}
func TestHandleSuccessSetsUuidAndPublishes(t *testing.T) {
parsed := domain.NewParsedMessage()
parsed.Parsed = true
repo := &stubRepository{}
pub := &stubPublisher{}
proc := newTestProcessor(&stubParser{value: parsed}, repo, pub)
const msgID = "uuid-123"
err := proc.Handle(context.Background(), []byte("payload"), msgID)
if err != nil {
t.Fatalf("expected success, got %v", err)
}
if len(repo.inserted) != 1 {
t.Fatalf("expected one inserted message, got %d", len(repo.inserted))
}
if repo.inserted[0].Uuid != msgID {
t.Fatalf("expected message uuid to be %s, got %s", msgID, repo.inserted[0].Uuid)
}
if pub.last == nil {
t.Fatalf("expected publisher to receive message")
}
if pub.last != repo.inserted[0] {
t.Fatalf("publisher received unexpected message pointer")
}
}
func TestHandlePublisherErrorIsRetriable(t *testing.T) {
parsed := domain.NewParsedMessage()
parsed.Parsed = true
repo := &stubRepository{}
pub := &stubPublisher{err: errors.New("publish failed")}
proc := newTestProcessor(&stubParser{value: parsed}, repo, pub)
err := proc.Handle(context.Background(), []byte("payload"), "id-3")
if err == nil {
t.Fatalf("expected error when publisher fails")
}
if IsPermanent(err) {
t.Fatalf("publisher failure should not be permanent")
}
if len(repo.inserted) != 1 {
t.Fatalf("expected message to insert before publish failure")
}
}
+89 -12
View File
@@ -6,18 +6,18 @@ import (
"strings" "strings"
"time" "time"
"github.com/knadh/koanf/v2"
"github.com/knadh/koanf/parsers/toml" "github.com/knadh/koanf/parsers/toml"
"github.com/knadh/koanf/providers/file"
envprovider "github.com/knadh/koanf/providers/env" envprovider "github.com/knadh/koanf/providers/env"
"github.com/knadh/koanf/providers/file"
"github.com/knadh/koanf/v2"
) )
// Config holds all application configuration // Config holds all application configuration
type Config struct { type Config struct {
NATS NATSConfig `koanf:"nats"` NATS NATSConfig `koanf:"nats"`
Postgres PostgresConfig `koanf:"postgres"` Postgres PostgresConfig `koanf:"postgres"`
App AppConfig `koanf:"app"` App AppConfig `koanf:"app"`
Log LogConfig `koanf:"log"` Log LogConfig `koanf:"log"`
Publisher PublisherConfig `koanf:"publisher"` Publisher PublisherConfig `koanf:"publisher"`
// Legacy fields for backward compatibility during migration // Legacy fields for backward compatibility during migration
Subscription SubscriptionConfig `koanf:"subscription"` Subscription SubscriptionConfig `koanf:"subscription"`
@@ -26,14 +26,33 @@ type Config struct {
// NATSConfig holds NATS/JetStream configuration // NATSConfig holds NATS/JetStream configuration
type NATSConfig struct { type NATSConfig struct {
URL string `koanf:"url"` URL string `koanf:"url"`
Stream string `koanf:"stream"` Stream string `koanf:"stream"`
Consumer string `koanf:"consumer"` Consumer string `koanf:"consumer"`
StreamLimits StreamLimitsConfig `koanf:"stream_limits"`
ConsumerRules ConsumerRulesConfig `koanf:"consumer"`
// Legacy fields // Legacy fields
Client string `koanf:"client"` Client string `koanf:"client"`
Cluster string `koanf:"cluster"` Cluster string `koanf:"cluster"`
} }
// StreamLimitsConfig defines JetStream retention controls.
type StreamLimitsConfig struct {
MaxMsgs int64 `koanf:"max_msgs"`
MaxBytes int64 `koanf:"max_bytes"`
MaxAge time.Duration `koanf:"max_age"`
Discard string `koanf:"discard"`
Storage string `koanf:"storage"`
Replicas int `koanf:"replicas"`
}
// ConsumerRulesConfig captures consumer-level options.
type ConsumerRulesConfig struct {
MaxDeliver int `koanf:"max_deliver"`
AckWait time.Duration `koanf:"ack_wait"`
MaxAckPending int `koanf:"max_ack_pending"`
}
// PostgresConfig holds PostgreSQL configuration // PostgresConfig holds PostgreSQL configuration
type PostgresConfig struct { type PostgresConfig struct {
URL string `koanf:"url"` URL string `koanf:"url"`
@@ -43,8 +62,9 @@ type PostgresConfig struct {
// AppConfig holds application-level configuration // AppConfig holds application-level configuration
type AppConfig struct { type AppConfig struct {
BatchSize int `koanf:"batch_size"` BatchSize int `koanf:"batch_size"`
BatchTimeout time.Duration `koanf:"batch_timeout"` BatchTimeout time.Duration `koanf:"batch_timeout"`
MonitorInterval time.Duration `koanf:"monitor_interval"`
} }
// LogConfig holds logging configuration // LogConfig holds logging configuration
@@ -112,6 +132,9 @@ func LoadConfig() (*Config, error) {
if cfg.App.BatchTimeout == 0 { if cfg.App.BatchTimeout == 0 {
cfg.App.BatchTimeout = 2 * time.Second cfg.App.BatchTimeout = 2 * time.Second
} }
if cfg.App.MonitorInterval == 0 {
cfg.App.MonitorInterval = 30 * time.Second
}
if cfg.Postgres.MaxConns == 0 { if cfg.Postgres.MaxConns == 0 {
cfg.Postgres.MaxConns = 10 cfg.Postgres.MaxConns = 10
} }
@@ -130,6 +153,37 @@ func LoadConfig() (*Config, error) {
if cfg.NATS.Consumer == "" { if cfg.NATS.Consumer == "" {
cfg.NATS.Consumer = "telegram-consumer" cfg.NATS.Consumer = "telegram-consumer"
} }
if cfg.NATS.StreamLimits.MaxMsgs == 0 {
cfg.NATS.StreamLimits.MaxMsgs = 100000
}
if cfg.NATS.StreamLimits.MaxBytes == 0 {
cfg.NATS.StreamLimits.MaxBytes = 64 * 1024 * 1024
}
if cfg.NATS.StreamLimits.MaxAge == 0 {
cfg.NATS.StreamLimits.MaxAge = 24 * time.Hour
}
if cfg.NATS.StreamLimits.Discard == "" {
cfg.NATS.StreamLimits.Discard = "old"
}
if cfg.NATS.StreamLimits.Storage == "" {
cfg.NATS.StreamLimits.Storage = "file"
}
if cfg.NATS.StreamLimits.Replicas == 0 {
cfg.NATS.StreamLimits.Replicas = 1
}
if cfg.NATS.ConsumerRules.MaxDeliver == 0 {
cfg.NATS.ConsumerRules.MaxDeliver = 5
}
if cfg.NATS.ConsumerRules.AckWait == 0 {
if cfg.Timeouts.AckWait != 0 {
cfg.NATS.ConsumerRules.AckWait = cfg.Timeouts.AckWait
} else {
cfg.NATS.ConsumerRules.AckWait = 30 * time.Second
}
}
if cfg.NATS.ConsumerRules.MaxAckPending == 0 {
cfg.NATS.ConsumerRules.MaxAckPending = 1024
}
// Validate configuration // Validate configuration
if err := cfg.Validate(); err != nil { if err := cfg.Validate(); err != nil {
@@ -159,6 +213,30 @@ func (c *Config) Validate() error {
if c.App.BatchTimeout <= 0 { if c.App.BatchTimeout <= 0 {
return fmt.Errorf("app.batch_timeout must be greater than 0") return fmt.Errorf("app.batch_timeout must be greater than 0")
} }
if c.App.MonitorInterval <= 0 {
return fmt.Errorf("app.monitor_interval must be greater than 0")
}
if c.NATS.StreamLimits.MaxMsgs < 0 {
return fmt.Errorf("nats.stream_limits.max_msgs must be >= 0")
}
if c.NATS.StreamLimits.MaxBytes < 0 {
return fmt.Errorf("nats.stream_limits.max_bytes must be >= 0")
}
if c.NATS.StreamLimits.MaxAge < 0 {
return fmt.Errorf("nats.stream_limits.max_age must be >= 0")
}
if c.NATS.StreamLimits.Replicas <= 0 {
return fmt.Errorf("nats.stream_limits.replicas must be greater than 0")
}
if c.NATS.ConsumerRules.MaxDeliver <= 0 {
return fmt.Errorf("nats.consumer.max_deliver must be greater than 0")
}
if c.NATS.ConsumerRules.AckWait <= 0 {
return fmt.Errorf("nats.consumer.ack_wait must be greater than 0")
}
if c.NATS.ConsumerRules.MaxAckPending < 0 {
return fmt.Errorf("nats.consumer.max_ack_pending must be >= 0")
}
return nil return nil
} }
@@ -166,4 +244,3 @@ func (c *Config) Validate() error {
func ProvideConfig() (*Config, error) { func ProvideConfig() (*Config, error) {
return LoadConfig() return LoadConfig()
} }
+110 -14
View File
@@ -6,23 +6,25 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"go.uber.org/zap"
"github.com/nats-io/nats.go" "github.com/nats-io/nats.go"
"go.uber.org/zap"
"time" "time"
) )
// Consumer handles NATS JetStream message consumption // Consumer handles NATS JetStream message consumption
type Consumer struct { type Consumer struct {
js nats.JetStreamContext conn *nats.Conn
processor *app.MessageProcessor js nats.JetStreamContext
cfg *config.Config processor *app.MessageProcessor
logger *zap.Logger cfg *config.Config
subject string logger *zap.Logger
subject string
consumerName string consumerName string
} }
// ProvideConsumer creates a NATS consumer // ProvideConsumer creates a NATS consumer
func ProvideConsumer( func ProvideConsumer(
conn *nats.Conn,
js nats.JetStreamContext, js nats.JetStreamContext,
processor *app.MessageProcessor, processor *app.MessageProcessor,
cfg *config.Config, cfg *config.Config,
@@ -39,6 +41,7 @@ func ProvideConsumer(
} }
consumer := &Consumer{ consumer := &Consumer{
conn: conn,
js: js, js: js,
processor: processor, processor: processor,
cfg: cfg, cfg: cfg,
@@ -62,12 +65,21 @@ func (c *Consumer) ensureConsumer() error {
streamName = "TELEGRAM" streamName = "TELEGRAM"
} }
ackWait := c.cfg.NATS.ConsumerRules.AckWait
if ackWait == 0 {
ackWait = c.cfg.Timeouts.AckWait
}
if ackWait == 0 {
ackWait = 30 * time.Second
}
consumerConfig := &nats.ConsumerConfig{ consumerConfig := &nats.ConsumerConfig{
Durable: c.consumerName, Durable: c.consumerName,
DeliverPolicy: nats.DeliverAllPolicy, DeliverPolicy: nats.DeliverAllPolicy,
AckPolicy: nats.AckExplicitPolicy, AckPolicy: nats.AckExplicitPolicy,
AckWait: c.cfg.Timeouts.AckWait, AckWait: ackWait,
MaxDeliver: 5, // Maximum number of delivery attempts MaxDeliver: c.cfg.NATS.ConsumerRules.MaxDeliver,
MaxAckPending: c.cfg.NATS.ConsumerRules.MaxAckPending,
FilterSubject: c.subject, FilterSubject: c.subject,
} }
@@ -116,6 +128,17 @@ func (c *Consumer) Start(ctx context.Context) error {
batchTimeout = 2 * time.Second batchTimeout = 2 * time.Second
} }
c.logger.Info("Consumer pull configuration",
zap.Int("batch_size", batchSize),
zap.Duration("batch_timeout", batchTimeout),
zap.Int("max_deliver", c.cfg.NATS.ConsumerRules.MaxDeliver),
zap.Duration("ack_wait", c.cfg.NATS.ConsumerRules.AckWait),
)
statsCtx, statsCancel := context.WithCancel(ctx)
defer statsCancel()
go c.emitConsumerStats(statsCtx, streamName)
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
@@ -139,24 +162,97 @@ func (c *Consumer) Start(ctx context.Context) error {
// Process each message // Process each message
for _, msg := range msgs { for _, msg := range msgs {
if err := c.processMessage(ctx, msg); err != nil { if err := c.processMessage(ctx, msg); err != nil {
isPermanent := app.IsPermanent(err)
c.logger.Error("Failed to process message", c.logger.Error("Failed to process message",
zap.String("subject", msg.Subject), zap.String("subject", msg.Subject),
zap.Error(err), zap.Error(err),
zap.Bool("permanent", isPermanent),
) )
// NAK the message to retry
if isPermanent {
if termErr := msg.Term(); termErr != nil {
c.logger.Error("Failed to TERM message", zap.Error(termErr))
}
continue
}
// Transient error: request redelivery
if nakErr := msg.Nak(); nakErr != nil { if nakErr := msg.Nak(); nakErr != nil {
c.logger.Error("Failed to NAK message", zap.Error(nakErr)) c.logger.Error("Failed to NAK message", zap.Error(nakErr))
} }
} else { continue
// ACK the message }
if ackErr := msg.Ack(); ackErr != nil {
c.logger.Error("Failed to ACK message", zap.Error(ackErr)) // ACK the message
} if ackErr := msg.Ack(); ackErr != nil {
c.logger.Error("Failed to ACK message", zap.Error(ackErr))
} }
} }
} }
} }
func (c *Consumer) emitConsumerStats(ctx context.Context, streamName string) {
interval := c.cfg.App.MonitorInterval
if interval <= 0 {
interval = 30 * time.Second
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
info, err := c.js.ConsumerInfo(streamName, c.consumerName)
if err != nil {
c.logger.Warn("Failed to fetch consumer info", zap.Error(err))
continue
}
c.logger.Info("JetStream consumer metrics",
zap.String("stream", streamName),
zap.String("consumer", c.consumerName),
zap.Uint64("num_ack_pending", uint64(info.NumAckPending)),
zap.Uint64("num_redelivered", uint64(info.NumRedelivered)),
zap.Uint64("num_pending", uint64(info.NumPending)),
zap.Uint64("delivered_consumer_seq", uint64(info.Delivered.Consumer)),
zap.Uint64("delivered_stream_seq", uint64(info.Delivered.Stream)),
)
}
}
}
// Shutdown drains the underlying NATS connection gracefully.
func (c *Consumer) Shutdown(ctx context.Context) error {
if c.conn == nil {
return nil
}
timeout := c.cfg.Timeouts.Close
if timeout <= 0 {
timeout = 10 * time.Second
}
closeCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
errCh := make(chan error, 1)
go func() {
errCh <- c.conn.Drain()
}()
select {
case err := <-errCh:
c.conn.Close()
return err
case <-closeCtx.Done():
c.conn.Close()
return fmt.Errorf("nats drain timeout: %w", closeCtx.Err())
}
}
// processMessage processes a single message // processMessage processes a single message
func (c *Consumer) processMessage(ctx context.Context, msg *nats.Msg) error { func (c *Consumer) processMessage(ctx context.Context, msg *nats.Msg) error {
msgID := msg.Header.Get("Nats-Msg-Id") msgID := msg.Header.Get("Nats-Msg-Id")
+30 -8
View File
@@ -3,14 +3,14 @@ package nats
import ( import (
"caatsm/internal/infra/config" "caatsm/internal/infra/config"
"fmt" "fmt"
"go.uber.org/zap" "strings"
"github.com/nats-io/nats.go" "github.com/nats-io/nats.go"
"time" "go.uber.org/zap"
) )
// ProvideJetStream creates a NATS JetStream connection // ProvideNATSConn creates a reusable NATS connection.
func ProvideJetStream(cfg *config.Config, logger *zap.Logger) (nats.JetStreamContext, error) { func ProvideNATSConn(cfg *config.Config, logger *zap.Logger) (*nats.Conn, error) {
// Connect to NATS
nc, err := nats.Connect( nc, err := nats.Connect(
cfg.NATS.URL, cfg.NATS.URL,
nats.RetryOnFailedConnect(true), nats.RetryOnFailedConnect(true),
@@ -29,6 +29,11 @@ func ProvideJetStream(cfg *config.Config, logger *zap.Logger) (nats.JetStreamCon
return nil, fmt.Errorf("failed to connect to NATS: %w", err) return nil, fmt.Errorf("failed to connect to NATS: %w", err)
} }
return nc, nil
}
// ProvideJetStream creates a NATS JetStream context using an existing connection.
func ProvideJetStream(nc *nats.Conn, cfg *config.Config, logger *zap.Logger) (nats.JetStreamContext, error) {
// Get JetStream context // Get JetStream context
js, err := nc.JetStream() js, err := nc.JetStream()
if err != nil { if err != nil {
@@ -43,13 +48,30 @@ func ProvideJetStream(cfg *config.Config, logger *zap.Logger) (nats.JetStreamCon
subject = "telegram.>" subject = "telegram.>"
} }
streamLimits := cfg.NATS.StreamLimits
storage := nats.FileStorage
switch strings.ToLower(streamLimits.Storage) {
case "memory":
storage = nats.MemoryStorage
case "file":
storage = nats.FileStorage
}
discard := nats.DiscardOld
if strings.EqualFold(streamLimits.Discard, "new") {
discard = nats.DiscardNew
}
streamConfig := &nats.StreamConfig{ streamConfig := &nats.StreamConfig{
Name: streamName, Name: streamName,
Subjects: []string{subject}, Subjects: []string{subject},
Retention: nats.LimitsPolicy, Retention: nats.LimitsPolicy,
MaxAge: 24 * time.Hour, MaxMsgs: streamLimits.MaxMsgs,
Storage: nats.FileStorage, MaxBytes: streamLimits.MaxBytes,
Replicas: 1, MaxAge: streamLimits.MaxAge,
Discard: discard,
Storage: storage,
Replicas: streamLimits.Replicas,
} }
_, err = js.AddStream(streamConfig) _, err = js.AddStream(streamConfig)
+19 -2
View File
@@ -2,11 +2,13 @@ package nats
import ( import (
"caatsm/internal/adapter" "caatsm/internal/adapter"
"caatsm/internal/domain"
"caatsm/internal/infra/config" "caatsm/internal/infra/config"
"encoding/json" "encoding/json"
"fmt" "fmt"
"go.uber.org/zap" "github.com/google/uuid"
"github.com/nats-io/nats.go" "github.com/nats-io/nats.go"
"go.uber.org/zap"
) )
// Publisher publishes messages to NATS JetStream // Publisher publishes messages to NATS JetStream
@@ -42,8 +44,23 @@ func (p *Publisher) Publish(message interface{}) error {
return fmt.Errorf("failed to marshal message: %w", err) return fmt.Errorf("failed to marshal message: %w", err)
} }
// Build JetStream message to attach dedup headers
jsMsg := nats.NewMsg(topic)
jsMsg.Data = messageBytes
switch typed := message.(type) {
case *domain.ParsedMessage:
if typed != nil && typed.Uuid != "" {
jsMsg.Header.Set("Nats-Msg-Id", typed.Uuid)
} else {
jsMsg.Header.Set("Nats-Msg-Id", uuid.NewString())
}
default:
jsMsg.Header.Set("Nats-Msg-Id", uuid.NewString())
}
// Publish to JetStream // Publish to JetStream
_, err = p.js.Publish(topic, messageBytes) _, err = p.js.PublishMsg(jsMsg)
if err != nil { if err != nil {
return fmt.Errorf("failed to publish message: %w", err) return fmt.Errorf("failed to publish message: %w", err)
} }
+1 -1
View File
@@ -28,6 +28,7 @@ func InitializeApp() (*app.MessageProcessor, *nats.Consumer, error) {
postgres.ProvideRepository, postgres.ProvideRepository,
// NATS // NATS
nats.ProvideNATSConn,
nats.ProvideJetStream, nats.ProvideJetStream,
nats.ProvidePublisher, nats.ProvidePublisher,
@@ -42,4 +43,3 @@ func InitializeApp() (*app.MessageProcessor, *nats.Consumer, error) {
) )
return nil, nil, nil return nil, nil, nil
} }
+6 -2
View File
@@ -32,7 +32,11 @@ func InitializeApp() (*app.MessageProcessor, *nats.Consumer, error) {
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
natsJetStreamContext, err := nats.ProvideJetStream(configConfig, zapLogger) natsConn, err := nats.ProvideNATSConn(configConfig, zapLogger)
if err != nil {
return nil, nil, err
}
natsJetStreamContext, err := nats.ProvideJetStream(natsConn, configConfig, zapLogger)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@@ -42,7 +46,7 @@ func InitializeApp() (*app.MessageProcessor, *nats.Consumer, error) {
} }
parserParser := parser.ProvideParser() parserParser := parser.ProvideParser()
appMessageProcessor := app.NewMessageProcessor(parserParser, adapterRepository, adapterPublisher, zapLogger) appMessageProcessor := app.NewMessageProcessor(parserParser, adapterRepository, adapterPublisher, zapLogger)
natsConsumer, err := nats.ProvideConsumer(natsJetStreamContext, appMessageProcessor, configConfig, zapLogger) natsConsumer, err := nats.ProvideConsumer(natsConn, natsJetStreamContext, appMessageProcessor, configConfig, zapLogger)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }