✨ Introduce seed-telegrams tool for generating synthetic telegrams for testing and development. Enhance README with detailed usage instructions, command-line parameters, and examples. Add unit tests for telegram generation and status selection logic to ensure robustness. Update documentation to reflect new features and usage scenarios.

internal/app/processor.go

@@ -169,7 +169,17 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
 		p.telemetry.RecordFailure("publisher")
 		p.telemetry.RecordProcessingResult(ctx, string(parsed.Status), parsed.Category, latency)
 		p.persistRaw(ctx, parsed)
-		// Mark as permanent so the consumer will ack instead of retrying
+
+		// Treat clearly temporary JetStream issues (e.g. no responders) as transient so
+		// the consumer will NAK and retry according to backoff settings.
+		lowerErr := strings.ToLower(err.Error())
+		if strings.Contains(lowerErr, "no responders") {
+			pubSpan.End()
+			// Return a non-permanent error to trigger retry via nakWithStrategy in the consumer.
+			return fmt.Errorf("transient publish error: %w", err)
+		}
+
+		// Other publish errors are treated as permanent and will go to DLQ + ACK.
 		pubSpan.End()
 		return Permanent(fmt.Errorf("failed to publish message: %w", err))
 	}

internal/infra/nats/consumer.go

@@ -10,6 +10,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
 	"strings"
 	"time"
 
@@ -49,6 +50,32 @@ type Consumer struct {
 	consecutiveProcessErrors int
 }
 
+func isDevLikeEnv() bool {
+	switch strings.ToLower(os.Getenv("GO_ENV")) {
+	case "", "dev", "development", "test", "testing":
+		return true
+	default:
+		return false
+	}
+}
+
+func isJetStreamResourceNotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+	if errors.Is(err, nats.ErrStreamNotFound) || errors.Is(err, nats.ErrConsumerNotFound) {
+		return true
+	}
+
+	// Some JetStream API errors are only exposed via error strings.
+	msg := strings.ToLower(err.Error())
+	if strings.Contains(msg, "stream not found") || strings.Contains(msg, "consumer not found") {
+		return true
+	}
+
+	return false
+}
+
 // ProvideConsumer creates a NATS consumer
 func ProvideConsumer(
 	conn *nats.Conn,
@@ -201,6 +228,58 @@ func (c *Consumer) ensureConsumer() error {
 	return nil
 }
 
+// recoverJetStreamResources attempts to recreate the stream and consumer in
+// dev/test environments if they are missing. It is safe to call multiple times.
+func (c *Consumer) recoverJetStreamResources() error {
+	if c.js == nil {
+		return fmt.Errorf("jetstream context is nil")
+	}
+	if c.cfg == nil {
+		return fmt.Errorf("config is nil")
+	}
+
+	// Ensure stream exists (dev/test may auto-create, prod will error).
+	if err := EnsureStream(c.js, c.cfg, c.logger); err != nil {
+		return fmt.Errorf("ensure stream %s: %w", c.streamName, err)
+	}
+
+	// Ensure durable consumer exists and is properly bound.
+	if err := c.ensureConsumer(); err != nil {
+		return fmt.Errorf("ensure consumer %s: %w", c.consumerName, err)
+	}
+
+	return nil
+}
+
+// createPullSubscriptionWithRecovery creates a pull subscription and, in
+// dev/test environments, attempts to self-heal missing stream/consumer
+// by recreating them once.
+func (c *Consumer) createPullSubscriptionWithRecovery() (*nats.Subscription, error) {
+	sub, err := c.js.PullSubscribe(c.subject, c.consumerName, nats.Bind(c.streamName, c.consumerName))
+	if err == nil {
+		return sub, nil
+	}
+
+	if isJetStreamResourceNotFound(err) && isDevLikeEnv() && shouldBootstrapStream() {
+		c.logger.Warn("PullSubscribe failed due to missing JetStream resources; attempting to recreate",
+			zap.Error(err),
+			zap.String("stream", c.streamName),
+			zap.String("consumer", c.consumerName),
+		)
+		if recErr := c.recoverJetStreamResources(); recErr != nil {
+			return nil, fmt.Errorf("failed to recover JetStream resources: %w", recErr)
+		}
+		// Retry subscription after successful recovery.
+		sub, err = c.js.PullSubscribe(c.subject, c.consumerName, nats.Bind(c.streamName, c.consumerName))
+		if err != nil {
+			return nil, fmt.Errorf("failed to create pull subscription after recovery: %w", err)
+		}
+		return sub, nil
+	}
+
+	return nil, fmt.Errorf("failed to create pull subscription: %w", err)
+}
+
 // validateDLQ verifies whether DLQ routing should be enabled and, if so, whether
 // the configured DLQ subject is bound to a JetStream stream. If validation fails,
 // DLQ routing is disabled (by clearing c.dlqSubject) and a warning is logged,
@@ -269,10 +348,10 @@ func (c *Consumer) Start(ctx context.Context) error {
 }
 
 func (c *Consumer) startJetStream(ctx context.Context) error {
-	// Create pull subscription
-	sub, err := c.js.PullSubscribe(c.subject, c.consumerName, nats.Bind(c.streamName, c.consumerName))
+	// Create pull subscription (with simple self-healing in dev/test).
+	sub, err := c.createPullSubscriptionWithRecovery()
 	if err != nil {
-		return fmt.Errorf("failed to create pull subscription: %w", err)
+		return err
 	}
 	defer sub.Unsubscribe()
 
@@ -296,6 +375,8 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
 	defer statsCancel()
 	go c.emitConsumerStats(statsCtx)
 
+	var fetchErrorStreak int
+
 	for {
 		select {
 		case <-ctx.Done():
@@ -308,25 +389,80 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
 		msgs, err := sub.Fetch(c.batchSize, nats.MaxWait(c.batchTimeout))
 		if err != nil {
 			if errors.Is(err, nats.ErrTimeout) {
-				// Timeout is expected when no messages are available
+				// Timeout is expected when no messages are available.
 				continue
 			}
+
+			// JetStream API is currently unavailable (e.g., NATS just restarted or JetStream not ready).
 			if errors.Is(err, nats.ErrNoResponders) {
-				// JetStream API is currently unavailable (e.g., NATS just restarted or JetStream not ready).
-				// Back off a bit to avoid log spam while allowing the system to recover.
-				c.logger.Warn("JetStream not available, will retry",
+				fetchErrorStreak++
+				backoff := time.Duration(fetchErrorStreak) * time.Second
+				if backoff > 30*time.Second {
+					backoff = 30 * time.Second
+				}
+				c.logger.Warn("JetStream not available, will retry with backoff",
+					zap.Error(err),
+					zap.String("stream", c.streamName),
+					zap.String("consumer", c.consumerName),
+					zap.Duration("backoff", backoff),
+				)
+				time.Sleep(backoff)
+				continue
+			}
+
+			// Underlying consumer/stream removed while app is running.
+			if isJetStreamResourceNotFound(err) {
+				if isDevLikeEnv() && shouldBootstrapStream() {
+					c.logger.Warn("JetStream consumer or stream missing; attempting to recreate",
+						zap.Error(err),
+						zap.String("stream", c.streamName),
+						zap.String("consumer", c.consumerName),
+					)
+					if recErr := c.recoverJetStreamResources(); recErr != nil {
+						c.logger.Error("Failed to recover JetStream resources", zap.Error(recErr))
+						return recErr
+					}
+
+					// Recreate subscription after successful recovery.
+					sub.Unsubscribe()
+					sub, err = c.createPullSubscriptionWithRecovery()
+					if err != nil {
+						return err
+					}
+
+					// Reset error streak after successful recovery.
+					fetchErrorStreak = 0
+					continue
+				}
+
+				// Production: treat as configuration/operational error.
+				c.logger.Error("JetStream consumer or stream missing; not auto-recreating in this environment",
 					zap.Error(err),
 					zap.String("stream", c.streamName),
 					zap.String("consumer", c.consumerName),
 				)
-				time.Sleep(5 * time.Second)
-				continue
+				return err
 			}
-			c.logger.Error("Failed to fetch messages", zap.Error(err))
-			time.Sleep(time.Second)
+
+			// Generic error path with modest backoff.
+			fetchErrorStreak++
+			backoff := time.Duration(fetchErrorStreak) * time.Second
+			if backoff > 10*time.Second {
+				backoff = 10 * time.Second
+			}
+			c.logger.Error("Failed to fetch messages; backing off",
+				zap.Error(err),
+				zap.Duration("backoff", backoff),
+			)
+			time.Sleep(backoff)
 			continue
 		}
 
+		// Successful fetch -> reset error streak.
+		if fetchErrorStreak > 0 {
+			fetchErrorStreak = 0
+		}
+
 		// Process each message
 		// TODO: consider buffering messages to take advantage of Repository.InsertBatch for higher throughput.
 		for _, msg := range msgs {

internal/infra/nats/jetstream.go

@@ -45,15 +45,29 @@ func ProvideJetStream(nc *nats.Conn, cfg *config.Config, logger *zap.Logger) (na
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	// Create stream if it doesn't exist
+	// Ensure the stream exists and is minimally aligned with configuration.
+	if err := EnsureStream(js, cfg, logger); err != nil {
+		nc.Close()
+		return nil, err
+	}
+
+	return js, nil
+}
+
+// EnsureStream ensures that the configured JetStream stream exists and has
+// at least the expected subjects bound. It is safe to call multiple times.
+//
+// In dev/test environments (see shouldBootstrapStream), the stream will be
+// auto-created if it does not exist. In production, a missing stream results
+// in an error so that operators can intervene.
+func EnsureStream(js nats.JetStreamContext, cfg *config.Config, logger *zap.Logger) error {
 	streamName := cfg.NATS.Stream
 	consumerSubject := cfg.EffectiveSubscriptionTopic()
 	publisherSubject := strings.TrimSpace(cfg.Publisher.Topic)
 
 	streamSubjects := dedupeSubjects([]string{consumerSubject, publisherSubject})
 	if len(streamSubjects) == 0 {
-		nc.Close()
-		return nil, fmt.Errorf("no subjects configured for JetStream stream %s", streamName)
+		return fmt.Errorf("no subjects configured for JetStream stream %s", streamName)
 	}
 
 	streamLimits := cfg.NATS.StreamLimits
@@ -87,26 +101,22 @@ func ProvideJetStream(nc *nats.Conn, cfg *config.Config, logger *zap.Logger) (na
 		if errors.Is(err, nats.ErrStreamNotFound) {
 			if shouldBootstrapStream() {
 				if _, err = js.AddStream(streamConfig); err != nil {
-					nc.Close()
-					return nil, fmt.Errorf("failed to create stream: %w", err)
+					return fmt.Errorf("failed to create stream %s: %w", streamName, err)
 				}
-				logger.Info("Created JetStream",
+				logger.Info("Created JetStream stream",
 					zap.String("stream", streamName),
 					zap.Strings("subjects", streamSubjects),
 				)
-			} else {
-				nc.Close()
-				return nil, fmt.Errorf("stream %s not found and auto-creation disabled", streamName)
+				return nil
 			}
-		} else {
-			nc.Close()
-			return nil, fmt.Errorf("failed to fetch stream info: %w", err)
+			return fmt.Errorf("stream %s not found and auto-creation disabled", streamName)
 		}
-	} else {
-		validateStreamConfig(info, streamSubjects, logger)
+		return fmt.Errorf("failed to fetch stream info for %s: %w", streamName, err)
 	}
 
-	return js, nil
+	// Stream exists: validate subjects but do not fail hard if they differ.
+	validateStreamConfig(info, streamSubjects, logger)
+	return nil
 }
 
 func shouldBootstrapStream() bool {

internal/infra/nats/publisher.go

@@ -5,7 +5,9 @@ import (
 	"caatsm/internal/infra/config"
 	"caatsm/internal/model"
 	"encoding/json"
+	"errors"
 	"fmt"
+
 	"github.com/google/uuid"
 	"github.com/nats-io/nats.go"
 	"go.uber.org/zap"
@@ -62,6 +64,10 @@ func (p *Publisher) Publish(message interface{}) error {
 	// Publish to JetStream
 	_, err = p.js.PublishMsg(jsMsg)
 	if err != nil {
+		// Distinguish temporary JetStream unavailability from permanent config errors.
+		if errors.Is(err, nats.ErrNoResponders) {
+			return fmt.Errorf("transient publish error (no responders): %w", err)
+		}
 		return fmt.Errorf("failed to publish message: %w", err)
 	}