🔧 Update Go module dependencies and modify NATS configuration to exclusively support JetStream mode. Upgrade go.uber.org/zap to version 1.27.1 and golang.org/x/crypto to version 0.45.0. Revise Makefile and documentation to reflect changes in NATS mode, ensuring consistent messaging behavior across development and production environments. Enhance README and configuration files to clarify JetStream usage and remove references to Core NATS mode.
This commit is contained in:
windyboy
@@ -330,10 +330,10 @@ func (c *Config) Validate() error {
|
||||
if c.NATS.URL == "" {
|
||||
return fmt.Errorf("nats.url is required")
|
||||
}
|
||||
switch strings.ToLower(c.NATS.Mode) {
|
||||
case "", "jetstream", "core":
|
||||
default:
|
||||
return fmt.Errorf("nats.mode must be 'jetstream' or 'core'")
|
||||
// Validate NATS mode - only JetStream is supported
|
||||
mode := strings.ToLower(c.NATS.Mode)
|
||||
if mode != "" && mode != "jetstream" {
|
||||
return fmt.Errorf("nats.mode must be 'jetstream' or empty (defaults to 'jetstream'), got: %s", c.NATS.Mode)
|
||||
}
|
||||
if c.NATS.Stream == "" {
|
||||
return fmt.Errorf("nats.stream is required")
|
||||
|
||||
+46
-113
@@ -45,7 +45,6 @@ type Consumer struct {
|
||||
type consumerConfig struct {
|
||||
subject string
|
||||
consumerName string
|
||||
mode string
|
||||
streamName string
|
||||
dlqSubject string
|
||||
ackWait time.Duration
|
||||
@@ -78,75 +77,56 @@ func ProvideConsumer(
|
||||
|
||||
// Initialize the pending messages metric early (set to 0) so it appears in Prometheus
|
||||
// even before the consumer starts. This ensures the metric is always visible.
|
||||
// We do this first, before any operations that might fail, to ensure the metric exists.
|
||||
// Initialize the metric unconditionally when in JetStream mode, even if js is nil,
|
||||
// as it will be updated later when js becomes available.
|
||||
if normCfg.mode == "jetstream" {
|
||||
logger.Info("Initializing NATS consumer pending messages metric",
|
||||
zap.String("stream", normCfg.streamName),
|
||||
zap.String("consumer", normCfg.consumerName),
|
||||
zap.Uint64("pending", 0),
|
||||
zap.Bool("js_available", js != nil),
|
||||
)
|
||||
// Always initialize the metric in JetStream mode to ensure it appears in Prometheus
|
||||
// The metric will be updated with actual values when the consumer starts
|
||||
obsmetrics.RecordNATSConsumerPending(normCfg.streamName, normCfg.consumerName, 0)
|
||||
} else {
|
||||
logger.Debug("Skipping NATS consumer pending messages metric initialization (not JetStream mode)",
|
||||
zap.String("mode", normCfg.mode),
|
||||
)
|
||||
}
|
||||
logger.Info("Initializing NATS consumer pending messages metric",
|
||||
zap.String("stream", normCfg.streamName),
|
||||
zap.String("consumer", normCfg.consumerName),
|
||||
zap.Uint64("pending", 0),
|
||||
zap.Bool("js_available", js != nil),
|
||||
)
|
||||
obsmetrics.RecordNATSConsumerPending(normCfg.streamName, normCfg.consumerName, 0)
|
||||
|
||||
// Initialize managers
|
||||
if consumer.config.mode == "jetstream" {
|
||||
consumer.consumerManager = NewConsumerManager(js, normCfg.streamName, normCfg.consumerName, normCfg.subject, logger)
|
||||
// Use StreamManager with full configuration
|
||||
streamSubjects := []string{normCfg.subject}
|
||||
if publisherSubject := strings.TrimSpace(cfg.Publisher.Topic); publisherSubject != "" {
|
||||
streamSubjects = append(streamSubjects, publisherSubject)
|
||||
}
|
||||
// Add DLQ subject to stream if DLQ is enabled
|
||||
if normCfg.dlqSubject != "" {
|
||||
streamSubjects = append(streamSubjects, normCfg.dlqSubject)
|
||||
}
|
||||
streamSubjects = dedupeSubjects(streamSubjects)
|
||||
consumer.streamManager = NewStreamManager(js, normCfg.streamName, streamSubjects, logger)
|
||||
consumer.consumerManager = NewConsumerManager(js, normCfg.streamName, normCfg.consumerName, normCfg.subject, logger)
|
||||
// Use StreamManager with full configuration
|
||||
streamSubjects := []string{normCfg.subject}
|
||||
if publisherSubject := strings.TrimSpace(cfg.Publisher.Topic); publisherSubject != "" {
|
||||
streamSubjects = append(streamSubjects, publisherSubject)
|
||||
}
|
||||
// Add DLQ subject to stream if DLQ is enabled
|
||||
if normCfg.dlqSubject != "" {
|
||||
streamSubjects = append(streamSubjects, normCfg.dlqSubject)
|
||||
}
|
||||
streamSubjects = dedupeSubjects(streamSubjects)
|
||||
consumer.streamManager = NewStreamManager(js, normCfg.streamName, streamSubjects, logger)
|
||||
|
||||
// Update fetcher with managers now that they're initialized
|
||||
if fetcher, ok := consumer.fetcher.(*defaultMessageFetcher); ok {
|
||||
fetcher.consumerManager = consumer.consumerManager
|
||||
fetcher.streamManager = consumer.streamManager
|
||||
}
|
||||
// Update fetcher with managers now that they're initialized
|
||||
if fetcher, ok := consumer.fetcher.(*defaultMessageFetcher); ok {
|
||||
fetcher.consumerManager = consumer.consumerManager
|
||||
fetcher.streamManager = consumer.streamManager
|
||||
}
|
||||
|
||||
// Ensure stream exists before creating consumer
|
||||
streamCfg := &StreamConfig{
|
||||
MaxMsgs: cfg.NATS.StreamLimits.MaxMsgs,
|
||||
MaxBytes: cfg.NATS.StreamLimits.MaxBytes,
|
||||
MaxAge: cfg.NATS.StreamLimits.MaxAge,
|
||||
Discard: cfg.NATS.StreamLimits.Discard,
|
||||
Storage: cfg.NATS.StreamLimits.Storage,
|
||||
Replicas: cfg.NATS.StreamLimits.Replicas,
|
||||
}
|
||||
if err := consumer.streamManager.EnsureStream(streamCfg); err != nil {
|
||||
return nil, fmt.Errorf("failed to ensure stream: %w", err)
|
||||
}
|
||||
// Ensure stream exists before creating consumer
|
||||
streamCfg := &StreamConfig{
|
||||
MaxMsgs: cfg.NATS.StreamLimits.MaxMsgs,
|
||||
MaxBytes: cfg.NATS.StreamLimits.MaxBytes,
|
||||
MaxAge: cfg.NATS.StreamLimits.MaxAge,
|
||||
Discard: cfg.NATS.StreamLimits.Discard,
|
||||
Storage: cfg.NATS.StreamLimits.Storage,
|
||||
Replicas: cfg.NATS.StreamLimits.Replicas,
|
||||
}
|
||||
if err := consumer.streamManager.EnsureStream(streamCfg); err != nil {
|
||||
return nil, fmt.Errorf("failed to ensure stream: %w", err)
|
||||
}
|
||||
|
||||
// Create consumer if it doesn't exist
|
||||
consumerConfig := consumer.buildConsumerConfig()
|
||||
if err := consumer.consumerManager.EnsureConsumer(consumerConfig); err != nil {
|
||||
return nil, fmt.Errorf("failed to ensure consumer: %w", err)
|
||||
}
|
||||
// Validate DLQ configuration early so misconfiguration is visible at startup
|
||||
// rather than only when the first poison message appears.
|
||||
if err := consumer.validateDLQ(); err != nil {
|
||||
return nil, fmt.Errorf("DLQ validation failed: %w", err)
|
||||
}
|
||||
|
||||
} else {
|
||||
logger.Info("Running consumer in core NATS mode",
|
||||
zap.String("subject", normCfg.subject),
|
||||
zap.String("queue_group", cfg.Subscription.QueueGroup),
|
||||
)
|
||||
// Create consumer if it doesn't exist
|
||||
consumerConfig := consumer.buildConsumerConfig()
|
||||
if err := consumer.consumerManager.EnsureConsumer(consumerConfig); err != nil {
|
||||
return nil, fmt.Errorf("failed to ensure consumer: %w", err)
|
||||
}
|
||||
// Validate DLQ configuration early so misconfiguration is visible at startup
|
||||
// rather than only when the first poison message appears.
|
||||
if err := consumer.validateDLQ(); err != nil {
|
||||
return nil, fmt.Errorf("DLQ validation failed: %w", err)
|
||||
}
|
||||
|
||||
return consumer, nil
|
||||
@@ -185,7 +165,6 @@ func (c *Consumer) initCollaborators() {
|
||||
telemetry: c.telemetry,
|
||||
streamName: c.config.streamName,
|
||||
consumerName: c.config.consumerName,
|
||||
mode: c.config.mode,
|
||||
backoff: c.cfg.NATS.ConsumerRules.Backoff,
|
||||
consecutiveProcessErrors: &c.consecutiveProcessErrors,
|
||||
}
|
||||
@@ -201,11 +180,6 @@ func normalizeConsumerConfig(cfg *config.Config) *consumerConfig {
|
||||
consumerName = "telegram-consumer"
|
||||
}
|
||||
|
||||
mode := strings.ToLower(cfg.NATS.Mode)
|
||||
if mode == "" {
|
||||
mode = "jetstream"
|
||||
}
|
||||
|
||||
streamName := cfg.NATS.Stream
|
||||
if streamName == "" {
|
||||
streamName = "TELEGRAM"
|
||||
@@ -244,7 +218,6 @@ func normalizeConsumerConfig(cfg *config.Config) *consumerConfig {
|
||||
return &consumerConfig{
|
||||
subject: subject,
|
||||
consumerName: consumerName,
|
||||
mode: mode,
|
||||
streamName: streamName,
|
||||
dlqSubject: dlqSubject,
|
||||
ackWait: ackWait,
|
||||
@@ -269,12 +242,8 @@ func (c *Consumer) buildConsumerConfig() *nats.ConsumerConfig {
|
||||
}
|
||||
}
|
||||
|
||||
// Start starts consuming messages.
|
||||
// Start starts consuming messages from JetStream.
|
||||
func (c *Consumer) Start(ctx context.Context) error {
|
||||
if c.config.mode == "core" {
|
||||
return c.startCore(ctx)
|
||||
}
|
||||
|
||||
return c.startJetStream(ctx)
|
||||
}
|
||||
|
||||
@@ -299,42 +268,6 @@ func (c *Consumer) validateDLQ() error {
|
||||
return c.ValidateDLQ()
|
||||
}
|
||||
|
||||
// startCore starts the Core NATS consumer loop.
|
||||
func (c *Consumer) startCore(ctx context.Context) error {
|
||||
queueGroup := c.cfg.Subscription.QueueGroup
|
||||
if queueGroup == "" {
|
||||
queueGroup = c.config.consumerName
|
||||
}
|
||||
|
||||
handler := func(msg *nats.Msg) {
|
||||
// Use ProcessBatch to ensure metrics are recorded via processSingleMessage
|
||||
// ProcessBatch handles error recording and metrics for both success and failure cases
|
||||
c.batchProcessor.ProcessBatch(ctx, []*nats.Msg{msg})
|
||||
}
|
||||
|
||||
sub, err := c.conn.QueueSubscribe(c.config.subject, queueGroup, handler)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to subscribe to %s: %w", c.config.subject, err)
|
||||
}
|
||||
if err := c.conn.Flush(); err != nil {
|
||||
return fmt.Errorf("failed to flush NATS connection: %w", err)
|
||||
}
|
||||
|
||||
c.logger.Info("Started core NATS subscription",
|
||||
zap.String("subject", c.config.subject),
|
||||
zap.String("queue_group", queueGroup),
|
||||
)
|
||||
|
||||
<-ctx.Done()
|
||||
c.logger.Info("Stopping core NATS consumer", zap.Error(ctx.Err()))
|
||||
|
||||
if err := sub.Drain(); err != nil && !errors.Is(err, nats.ErrConnectionClosed) {
|
||||
return fmt.Errorf("failed to drain core subscription: %w", err)
|
||||
}
|
||||
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// createPullSubscription creates a pull subscription
|
||||
func (c *Consumer) createPullSubscription() (*nats.Subscription, error) {
|
||||
return c.consumerManager.CreatePullSubscription()
|
||||
|
||||
@@ -62,7 +62,6 @@ var _ = Describe("Consumer helpers", func() {
|
||||
normCfg := normalizeConsumerConfig(cfg)
|
||||
|
||||
Expect(normCfg.consumerName).To(Equal("telegram-consumer"))
|
||||
Expect(normCfg.mode).To(Equal("jetstream"))
|
||||
Expect(normCfg.streamName).To(Equal("TELEGRAM"))
|
||||
Expect(normCfg.batchSize).To(Equal(50))
|
||||
Expect(normCfg.batchTimeout).To(Equal(2 * time.Second))
|
||||
@@ -74,7 +73,6 @@ var _ = Describe("Consumer helpers", func() {
|
||||
cfg := &configpkg.Config{
|
||||
NATS: configpkg.NATSConfig{
|
||||
Consumer: "custom-consumer",
|
||||
Mode: "core",
|
||||
Stream: "CUSTOM_STREAM",
|
||||
ConsumerRules: configpkg.ConsumerRulesConfig{
|
||||
AckWait: 60 * time.Second,
|
||||
@@ -93,7 +91,6 @@ var _ = Describe("Consumer helpers", func() {
|
||||
normCfg := normalizeConsumerConfig(cfg)
|
||||
|
||||
Expect(normCfg.consumerName).To(Equal("custom-consumer"))
|
||||
Expect(normCfg.mode).To(Equal("core"))
|
||||
Expect(normCfg.streamName).To(Equal("CUSTOM_STREAM"))
|
||||
Expect(normCfg.batchSize).To(Equal(100))
|
||||
Expect(normCfg.batchTimeout).To(Equal(5 * time.Second))
|
||||
|
||||
@@ -72,14 +72,7 @@ func ProvideNATSConn(cfg *config.Config, logger *zap.Logger) (*nats.Conn, error)
|
||||
}
|
||||
|
||||
// ProvideJetStream creates a JetStream context from a NATS connection.
|
||||
// Returns nil when mode is "core" to indicate JetStream should not be used.
|
||||
func ProvideJetStream(nc *nats.Conn, cfg *config.Config, logger *zap.Logger) (nats.JetStreamContext, error) {
|
||||
// In core mode, return nil so that publishers/consumers use core NATS
|
||||
if cfg.NATS.Mode == "core" {
|
||||
logger.Debug("Skipping JetStream initialization (core mode)")
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
js, err := nc.JetStream()
|
||||
if err != nil {
|
||||
logger.Error("failed to create JetStream context", zap.Error(err))
|
||||
|
||||
@@ -17,7 +17,6 @@ var _ = Describe("MessageHandler", func() {
|
||||
logger: zaptest.NewLogger(GinkgoT()),
|
||||
streamName: "TEST_STREAM",
|
||||
consumerName: "test-consumer",
|
||||
mode: "jetstream",
|
||||
}
|
||||
})
|
||||
|
||||
@@ -34,20 +33,7 @@ var _ = Describe("MessageHandler", func() {
|
||||
Expect(source).To(Equal("header"))
|
||||
})
|
||||
|
||||
It("generates UUID for core mode when header is missing", func() {
|
||||
processor.mode = "core"
|
||||
msg := &nats.Msg{
|
||||
Header: nats.Header{},
|
||||
}
|
||||
|
||||
id, source, err := processor.resolveMsgID(msg)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(id).NotTo(BeEmpty())
|
||||
Expect(source).To(Equal("generated"))
|
||||
})
|
||||
|
||||
It("returns error for JetStream mode when header and metadata are missing", func() {
|
||||
processor.mode = "jetstream"
|
||||
It("returns error when header and metadata are missing", func() {
|
||||
msg := &nats.Msg{
|
||||
Header: nats.Header{},
|
||||
}
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/nats-io/nats.go"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
@@ -39,7 +38,6 @@ type defaultBatchProcessor struct {
|
||||
// Configuration needed for processing
|
||||
streamName string
|
||||
consumerName string
|
||||
mode string
|
||||
backoff []time.Duration
|
||||
// Pointer to consecutive errors counter (shared with Consumer)
|
||||
consecutiveProcessErrors *int
|
||||
@@ -76,19 +74,12 @@ func (p *defaultBatchProcessor) processSingleMessage(ctx context.Context, msg *n
|
||||
|
||||
elapsed := time.Since(start)
|
||||
|
||||
// ACK the message (only in JetStream mode; Core NATS doesn't support ACK)
|
||||
if p.mode == "jetstream" {
|
||||
if ackErr := msg.Ack(); ackErr != nil {
|
||||
p.logger.Error("Failed to ACK message", zap.Error(ackErr))
|
||||
// Still record metrics even if ACK fails
|
||||
p.telemetry.RecordMessageHandled(ctx, p.streamName, p.consumerName, "ok", elapsed)
|
||||
} else {
|
||||
p.telemetry.RecordMessageHandled(ctx, p.streamName, p.consumerName, "ok", elapsed)
|
||||
}
|
||||
} else {
|
||||
// Core NATS mode: record metrics without ACK (ACK not supported)
|
||||
p.telemetry.RecordMessageHandled(ctx, p.streamName, p.consumerName, "ok", elapsed)
|
||||
// ACK the message
|
||||
if ackErr := msg.Ack(); ackErr != nil {
|
||||
p.logger.Error("Failed to ACK message", zap.Error(ackErr))
|
||||
// Still record metrics even if ACK fails
|
||||
}
|
||||
p.telemetry.RecordMessageHandled(ctx, p.streamName, p.consumerName, "ok", elapsed)
|
||||
}
|
||||
|
||||
// ProcessMessage processes a single message.
|
||||
@@ -160,10 +151,6 @@ func (p *defaultBatchProcessor) resolveMsgID(msg *nats.Msg) (string, string, err
|
||||
return id, "header", nil
|
||||
}
|
||||
|
||||
if p.mode == "core" {
|
||||
return uuid.NewString(), "generated", nil
|
||||
}
|
||||
|
||||
meta, err := msg.Metadata()
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("fetch metadata: %w", err)
|
||||
@@ -232,13 +219,6 @@ func (p *defaultBatchProcessor) handlePermanentError(ctx context.Context, msg *n
|
||||
*p.consecutiveProcessErrors = 0
|
||||
}
|
||||
|
||||
if p.mode != "jetstream" {
|
||||
p.logger.Debug("Permanent-error message in core mode; skipping DLQ/ACK (not supported)",
|
||||
zap.String("subject", msg.Subject),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
// Extract message ID for better logging
|
||||
msgID, _, _ := p.resolveMsgID(msg)
|
||||
if msgID == "" {
|
||||
@@ -310,13 +290,6 @@ func (p *defaultBatchProcessor) handleTransientError(ctx context.Context, msg *n
|
||||
}
|
||||
}
|
||||
|
||||
if p.mode != "jetstream" {
|
||||
p.logger.Debug("Transient-error message in core mode; skipping retry (ACK/NAK unsupported)",
|
||||
zap.String("subject", msg.Subject),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
// Transient error: request redelivery with optional delay
|
||||
p.telemetry.RecordRetry(ctx, p.streamName, p.consumerName, obsmetrics.RetryReasonProcessorError)
|
||||
if nakErr := p.nakWithStrategy(msg); nakErr != nil {
|
||||
|
||||
@@ -20,8 +20,7 @@ type Publisher struct {
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// ProvidePublisher creates a NATS publisher.
|
||||
// When js is nil (core mode), returns a CorePublisher that uses plain NATS.
|
||||
// ProvidePublisher creates a NATS JetStream publisher.
|
||||
func ProvidePublisher(
|
||||
js nats.JetStreamContext,
|
||||
nc *nats.Conn,
|
||||
@@ -29,7 +28,7 @@ func ProvidePublisher(
|
||||
logger *zap.Logger,
|
||||
) (port.Publisher, error) {
|
||||
if js == nil {
|
||||
return ProvideCorePublisher(nc, cfg, logger)
|
||||
return nil, fmt.Errorf("JetStream context is required")
|
||||
}
|
||||
return &Publisher{
|
||||
js: js,
|
||||
@@ -38,63 +37,6 @@ func ProvidePublisher(
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CorePublisher publishes messages to plain NATS (non-JetStream)
|
||||
type CorePublisher struct {
|
||||
conn *nats.Conn
|
||||
cfg *config.Config
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// ProvideCorePublisher creates a NATS publisher for core mode
|
||||
func ProvideCorePublisher(
|
||||
conn *nats.Conn,
|
||||
cfg *config.Config,
|
||||
logger *zap.Logger,
|
||||
) (port.Publisher, error) {
|
||||
return &CorePublisher{
|
||||
conn: conn,
|
||||
cfg: cfg,
|
||||
logger: logger,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Publish publishes a message using plain NATS
|
||||
func (p *CorePublisher) Publish(message any) error {
|
||||
topic := p.cfg.Publisher.Topic
|
||||
if topic == "" {
|
||||
p.logger.Error("publisher topic is not configured")
|
||||
return fmt.Errorf("publisher topic is not configured")
|
||||
}
|
||||
|
||||
// Marshal message to JSON
|
||||
messageBytes, err := json.Marshal(message)
|
||||
if err != nil {
|
||||
p.logger.Error("failed to marshal message",
|
||||
zap.String("topic", topic),
|
||||
zap.Error(err),
|
||||
)
|
||||
return fmt.Errorf("failed to marshal message: %w", err)
|
||||
}
|
||||
|
||||
// Publish to plain NATS
|
||||
err = p.conn.Publish(topic, messageBytes)
|
||||
if err != nil {
|
||||
p.logger.Error("failed to publish message",
|
||||
zap.String("topic", topic),
|
||||
zap.Int("message_size", len(messageBytes)),
|
||||
zap.Error(err),
|
||||
)
|
||||
return fmt.Errorf("failed to publish message: %w", err)
|
||||
}
|
||||
|
||||
p.logger.Debug("Published message",
|
||||
zap.String("topic", topic),
|
||||
zap.Int("size", len(messageBytes)),
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Publish publishes a message
|
||||
func (p *Publisher) Publish(message any) error {
|
||||
topic := p.cfg.Publisher.Topic
|
||||
|
||||
Reference in New Issue
Block a user