✨ Expand Docker Compose development stack to include observability tools: OpenTelemetry Collector, Jaeger, Prometheus, and Grafana. Update README with new usage instructions for the observability stack and enhance Taskfile for streamlined development commands. Introduce sample telegram generation utility and improve NATS configuration for core and JetStream modes.

2025-11-15 12:08:26 +08:00
parent 4f16bd6d90
commit 184c5453fb
13 changed files with 619 additions and 43 deletions
@@ -28,6 +28,7 @@ type Config struct {
 // NATSConfig holds NATS/JetStream configuration
 type NATSConfig struct {
 	URL           string              `koanf:"url"`
+	Mode          string              `koanf:"mode"`
 	Stream        string              `koanf:"stream"`
 	Consumer      string              `koanf:"consumer"`
 	StreamLimits  StreamLimitsConfig  `koanf:"stream_limits"`
@@ -49,14 +50,14 @@ type StreamLimitsConfig struct {

 // ConsumerRulesConfig captures consumer-level options.
 type ConsumerRulesConfig struct {
-	MaxDeliver    int           `koanf:"max_deliver"`
-	AckWait       time.Duration `koanf:"ack_wait"`
-	MaxAckPending int           `koanf:"max_ack_pending"`
-	DeliverPolicy string        `koanf:"deliver_policy"`
-	ReplayPolicy  string        `koanf:"replay_policy"`
+	MaxDeliver    int             `koanf:"max_deliver"`
+	AckWait       time.Duration   `koanf:"ack_wait"`
+	MaxAckPending int             `koanf:"max_ack_pending"`
+	DeliverPolicy string          `koanf:"deliver_policy"`
+	ReplayPolicy  string          `koanf:"replay_policy"`
 	Backoff       []time.Duration `koanf:"backoff"`
-	StartSequence uint64        `koanf:"start_sequence"`
-	StartTime     string        `koanf:"start_time"`
+	StartSequence uint64          `koanf:"start_sequence"`
+	StartTime     string          `koanf:"start_time"`
 }

 // PostgresConfig holds PostgreSQL configuration
@@ -160,6 +161,11 @@ func LoadConfig() (*Config, error) {
 	if cfg.Log.Format == "" {
 		cfg.Log.Format = "json"
 	}
+	if cfg.NATS.Mode == "" {
+		cfg.NATS.Mode = "jetstream"
+	} else {
+		cfg.NATS.Mode = strings.ToLower(cfg.NATS.Mode)
+	}
 	if cfg.NATS.Stream == "" {
 		cfg.NATS.Stream = "TELEGRAM"
 	}
@@ -219,6 +225,11 @@ func (c *Config) Validate() error {
 	if c.NATS.URL == "" {
 		return fmt.Errorf("nats.url is required")
 	}
+	switch strings.ToLower(c.NATS.Mode) {
+	case "", "jetstream", "core":
+	default:
+		return fmt.Errorf("nats.mode must be 'jetstream' or 'core'")
+	}
 	if c.NATS.Stream == "" {
 		return fmt.Errorf("nats.stream is required")
 	}
@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"

+	"github.com/google/uuid"
 	"github.com/nats-io/nats.go"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
@@ -26,6 +27,7 @@ type Consumer struct {
 	logger       *zap.Logger
 	subject      string
 	consumerName string
+	mode         string
 	meter        metric.Meter
 	ackPending   metric.Int64Histogram
 	redelivered  metric.Int64Histogram
@@ -48,6 +50,11 @@ func ProvideConsumer(
 		consumerName = "telegram-consumer"
 	}

+	mode := strings.ToLower(cfg.NATS.Mode)
+	if mode == "" {
+		mode = "jetstream"
+	}
+
 	consumer := &Consumer{
 		conn:         conn,
 		js:           js,
@@ -56,12 +63,20 @@ func ProvideConsumer(
 		logger:       logger,
 		subject:      subject,
 		consumerName: consumerName,
+		mode:         mode,
 	}
 	consumer.initMetrics()

-	// Create consumer if it doesn't exist
-	if err := consumer.ensureConsumer(); err != nil {
-		return nil, fmt.Errorf("failed to ensure consumer: %w", err)
+	if consumer.mode == "jetstream" {
+		// Create consumer if it doesn't exist
+		if err := consumer.ensureConsumer(); err != nil {
+			return nil, fmt.Errorf("failed to ensure consumer: %w", err)
+		}
+	} else {
+		logger.Info("Running consumer in core NATS mode",
+			zap.String("subject", subject),
+			zap.String("queue_group", cfg.Subscription.QueueGroup),
+		)
 	}

 	return consumer, nil
@@ -130,6 +145,14 @@ func (c *Consumer) ensureConsumer() error {

 // Start starts consuming messages
 func (c *Consumer) Start(ctx context.Context) error {
+	if c.mode == "core" {
+		return c.startCore(ctx)
+	}
+
+	return c.startJetStream(ctx)
+}
+
+func (c *Consumer) startJetStream(ctx context.Context) error {
 	streamName := c.cfg.NATS.Stream
 	if streamName == "" {
 		streamName = "TELEGRAM"
@@ -224,6 +247,46 @@ func (c *Consumer) Start(ctx context.Context) error {
 	}
 }

+func (c *Consumer) startCore(ctx context.Context) error {
+	queueGroup := c.cfg.Subscription.QueueGroup
+	if queueGroup == "" {
+		queueGroup = c.consumerName
+	}
+
+	handler := func(msg *nats.Msg) {
+		if err := c.processMessage(ctx, msg); err != nil {
+			isPermanent := app.IsPermanent(err)
+			c.logger.Error("Failed to process message (core mode)",
+				zap.String("subject", msg.Subject),
+				zap.Error(err),
+				zap.Bool("permanent", isPermanent),
+			)
+		}
+	}
+
+	sub, err := c.conn.QueueSubscribe(c.subject, queueGroup, handler)
+	if err != nil {
+		return fmt.Errorf("failed to subscribe to %s: %w", c.subject, err)
+	}
+	if err := c.conn.Flush(); err != nil {
+		return fmt.Errorf("failed to flush NATS connection: %w", err)
+	}
+
+	c.logger.Info("Started core NATS subscription",
+		zap.String("subject", c.subject),
+		zap.String("queue_group", queueGroup),
+	)
+
+	<-ctx.Done()
+	c.logger.Info("Stopping core NATS consumer", zap.Error(ctx.Err()))
+
+	if err := sub.Drain(); err != nil && !errors.Is(err, nats.ErrConnectionClosed) {
+		return fmt.Errorf("failed to drain core subscription: %w", err)
+	}
+
+	return ctx.Err()
+}
+
 func (c *Consumer) emitConsumerStats(ctx context.Context, streamName string) {
 	interval := c.cfg.App.MonitorInterval
 	if interval <= 0 {
@@ -393,6 +456,10 @@ func (c *Consumer) resolveMsgID(msg *nats.Msg) (string, string, error) {
 		return id, "header", nil
 	}

+	if c.mode == "core" {
+		return uuid.NewString(), "generated", nil
+	}
+
 	meta, err := msg.Metadata()
 	if err != nil {
 		return "", "", fmt.Errorf("fetch metadata: %w", err)
@@ -20,7 +20,8 @@ CREATE TABLE aviation.telegrams (
    parsed_at TIMESTAMPTZ,
    dispatched_at TIMESTAMPTZ,
    need_dispatch BOOLEAN,
-    PRIMARY KEY (uuid, received_at)
+    PRIMARY KEY (uuid, received_at),
+    CONSTRAINT telegrams_uuid_unique UNIQUE (uuid)
 );

 SELECT create_hypertable('aviation.telegrams', 'received_at', if_not_exists => TRUE);
@@ -40,5 +41,6 @@ CREATE TABLE IF NOT EXISTS aviation.telegrams_raw (
    content TEXT NOT NULL,
    received_at TIMESTAMPTZ NOT NULL,
    metadata JSONB,
-    PRIMARY KEY (uuid, received_at)
+    PRIMARY KEY (uuid, received_at),
+    CONSTRAINT telegrams_raw_uuid_unique UNIQUE (uuid)
 );