✨ Add repository guidelines and enhance documentation for project structure, build commands, coding standards, and testing practices. Introduce AGENTS.md for contributor guidance, update README.md to reference new guidelines, and improve configuration documentation for NATS modes. Update Makefile and Taskfile with clearer run commands and requirements for development and production modes. Add production deployment guide and improve logging configuration for better observability.

2025-11-17 16:25:23 +08:00
parent 67abd66fa3
commit 704c7b80f6
34 changed files with 2976 additions and 782 deletions
@@ -1,5 +1,20 @@
 [nats]
 url = "nats://localhost:4222"
+# mode: "core" (default for dev) or "jetstream" (recommended for production)
+#   - "core": Uses Core NATS for simple pub/sub messaging with:
+#     * No message persistence (messages lost if consumer offline)
+#     * No ACK mechanism (fire-and-forget delivery)
+#     * No automatic retry on processing failures
+#     * Queue groups for load balancing only
+#     * Suitable for development/testing or real-time scenarios where message loss is acceptable
+#     * Recommended for local development and testing
+#   - "jetstream": Uses NATS JetStream for persistent message streaming with:
+#     * Message persistence and replay capability
+#     * ACK/NAK mechanism for guaranteed delivery
+#     * Automatic retry with configurable backoff
+#     * Dead-letter queue (DLQ) support
+#     * Batch processing and consumer monitoring
+#     * Recommended for production environments
 mode = "core"
 client = "serial-client"
 cluster = "tele-cluster"
@@ -7,21 +22,45 @@ stream = "TELEGRAM"
 consumer = "telegram-consumer"

 [nats.stream_limits]
+# Stream retention and storage limits (only applies when mode = "jetstream")
+# max_msgs: Maximum number of messages to keep in the stream (0 = unlimited)
 max_msgs = 100000
+# max_bytes: Maximum total size of messages in bytes (0 = unlimited, 67108864 = 64MB)
 max_bytes = 67108864
+# max_age: Maximum age of messages before automatic deletion (e.g., "24h", "7d")
 max_age = "24h"
+# discard: What to do when limits are reached: "old" (delete oldest) or "new" (reject new)
 discard = "old"
+# storage: "file" (persistent to disk) or "memory" (ephemeral, faster but lost on restart)
 storage = "file"
+# replicas: Number of stream replicas for high availability (1 = single node, 3+ for production cluster)
 replicas = 1

 [nats.consumer_rules]
+# Consumer delivery and retry rules (only applies when mode = "jetstream")
+# max_deliver: Maximum number of delivery attempts before giving up (0 = unlimited)
 max_deliver = 5
+# ack_wait: Time to wait for ACK before redelivering message (e.g., "30s", "2m")
 ack_wait = "30s"
+# max_ack_pending: Maximum number of unacknowledged messages before pausing delivery
 max_ack_pending = 1024
+# deliver_policy: When to start delivering messages:
+#   - "all": Deliver all messages from the stream
+#   - "new": Only deliver new messages after consumer creation
+#   - "last": Deliver only the last message
+#   - "last_per_subject": Deliver last message per subject
+#   - "sequence": Start from a specific sequence (requires start_sequence)
+#   - "time": Start from a specific time (requires start_time in RFC3339 format)
 deliver_policy = "all"
+# replay_policy: How to replay messages: "instant" (as fast as possible) or "original" (preserve timing)
 replay_policy = "instant"
+# backoff: Array of delays between retry attempts (e.g., ["5s", "30s", "2m"])
+#   First retry waits 5s, second waits 30s, third and beyond wait 2m
 backoff = ["5s", "30s", "2m"]
+# start_sequence: Starting sequence number (only used when deliver_policy = "sequence")
 start_sequence = 0
+# start_time: Starting time in RFC3339 format (only used when deliver_policy = "time")
+#   Example: "2024-11-15T08:00:00Z"
 start_time = ""

 [subscription]
@@ -43,8 +82,12 @@ max_conns = 10
 min_conns = 2

 [app]
+# Batch processing configuration (applies to both core and jetstream modes)
+# batch_size: Number of messages to fetch in each batch (JetStream pull batch size)
 batch_size = 50
+# batch_timeout: Maximum time to wait when fetching a batch (e.g., "2s")
 batch_timeout = "2s"
+# monitor_interval: How often to emit consumer statistics and metrics
 monitor_interval = "30s"

 [log]
@@ -85,5 +128,8 @@ write_timeout = "5s"
 health_timeout = "2s"

 [dlq]
-enabled = true
-subject = "caatsm.dlq"
+# Dead-Letter Queue configuration (only applies when mode = "jetstream")
+# enabled: Enable DLQ routing for poison messages (messages that fail after max_deliver attempts)
+enabled = false  # Set to true when switching to JetStream mode
+# subject: NATS subject where failed messages will be published for manual inspection
+subject = "caatsm.dlq"
@@ -0,0 +1,143 @@
+# Production Configuration for CAATSM
+# 
+# This configuration is optimized for production environments.
+# Key differences from dev config:
+#   - Uses JetStream mode (required for production)
+#   - Higher resource limits and connection pools
+#   - JSON logging (for log aggregation)
+#   - SSL/TLS enabled for secure connections
+#   - Higher stream replicas for HA (3+)
+#   - Longer retention periods
+#
+# IMPORTANT: Stream and Consumer must be created manually in production.
+# The application does NOT auto-create them in production mode.
+
+[nats]
+url = "nats://nats.prod:4222"
+# Production MUST use JetStream mode for message reliability
+mode = "jetstream"
+client = "caatsm-prod-client"
+cluster = "prod-cluster"
+stream = "TELEGRAM"
+consumer = "telegram-consumer"
+
+[nats.stream_limits]
+# Production stream limits - adjust based on your requirements
+# max_msgs: Maximum number of messages to keep in the stream (0 = unlimited)
+max_msgs = 1000000
+# max_bytes: Maximum total size of messages in bytes (1GB = 1073741824)
+max_bytes = 1073741824
+# max_age: Maximum age of messages before automatic deletion (7 days)
+max_age = "168h"
+# discard: What to do when limits are reached: "old" (delete oldest) or "new" (reject new)
+discard = "old"
+# storage: "file" (persistent to disk) - REQUIRED for production
+storage = "file"
+# replicas: Number of stream replicas for high availability (3+ for production cluster)
+replicas = 3
+
+[nats.consumer_rules]
+# Consumer delivery and retry rules for production
+# max_deliver: Maximum number of delivery attempts before giving up
+max_deliver = 5
+# ack_wait: Time to wait for ACK before redelivering message
+ack_wait = "30s"
+# max_ack_pending: Maximum number of unacknowledged messages before pausing delivery
+max_ack_pending = 1024
+# deliver_policy: "new" - Start from new messages after consumer creation (recommended for production)
+#   Other options: "all", "last", "last_per_subject", "sequence", "time"
+deliver_policy = "new"
+# replay_policy: How to replay messages: "instant" (as fast as possible) or "original" (preserve timing)
+replay_policy = "instant"
+# backoff: Array of delays between retry attempts
+#   First retry waits 5s, second waits 30s, third waits 2m, fourth+ wait 5m
+backoff = ["5s", "30s", "2m", "5m"]
+# start_sequence: Starting sequence number (only used when deliver_policy = "sequence")
+start_sequence = 0
+# start_time: Starting time in RFC3339 format (only used when deliver_policy = "time")
+#   Example: "2024-11-15T08:00:00Z"
+start_time = ""
+
+[subscription]
+topic = "telegram.serial"
+queue_group = "tele-queue"
+
+[publisher]
+topic = "telegram.json"
+
+[timeouts]
+server = "10s"
+reconnect_wait = "5s"
+close = "30s"
+ack_wait = "30s"
+
+[postgres]
+# Production PostgreSQL connection - USE SSL/TLS
+# Replace with your production database URL
+url = "postgres://user:password@db.prod:5432/aviation?sslmode=require"
+# Higher connection pool for production workloads
+max_conns = 20
+min_conns = 5
+
+[app]
+# Production batch processing configuration
+# batch_size: Larger batch size for better throughput
+batch_size = 100
+# batch_timeout: Maximum time to wait when fetching a batch
+batch_timeout = "2s"
+# monitor_interval: How often to emit consumer statistics and metrics
+monitor_interval = "30s"
+
+[log]
+# Production logging configuration
+# level: Use "info" or "warn" in production (avoid "debug")
+level = "info"
+# format: "json" for log aggregation systems (ELK, Loki, etc.)
+format = "json"
+# output: Only stdout in production (let container/logging system handle file rotation)
+output = ["stdout"]
+# file: Not used in production (logging to stdout)
+# file = "logs/caatsm.log"
+
+# File rotation settings (not used when output = ["stdout"])
+# max_size = 100      # MB
+# max_backups = 7     # Keep 7 rotated files
+# max_age = 30        # Keep logs for 30 days
+# compress = true     # Compress old log files
+
+# Advanced options
+disable_caller = false
+disable_stacktrace = false
+development = false
+
+# Sampling configuration (optional, for high-volume scenarios)
+# [log.sampling]
+# initial = 100       # Log first 100 messages
+# thereafter = 100    # Then log every 100th message
+# tick = "1s"         # Per second
+
+[telemetry]
+# Production telemetry configuration
+enabled = true
+# Replace with your production OTLP collector endpoint
+endpoint = "otel-collector.prod:4318"
+# Use TLS in production (set to false)
+insecure = false
+
+[monitoring]
+# Production monitoring configuration
+disabled = false
+addr = ":2112"
+enable_metrics = true
+enable_health = true
+read_timeout = "5s"
+write_timeout = "5s"
+health_timeout = "2s"
+
+[dlq]
+# Dead-Letter Queue configuration (REQUIRED for production)
+# enabled: Enable DLQ routing for poison messages
+enabled = true
+# subject: NATS subject where failed messages will be published for manual inspection
+subject = "caatsm.dlq"
+
@@ -267,7 +267,7 @@
        "type": "prometheus",
        "uid": "prometheus-dev"
      },
-      "gridPos": { "h": 7, "w": 24, "x": 0, "y": 26 },
+      "gridPos": { "h": 7, "w": 12, "x": 0, "y": 26 },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
@@ -287,6 +287,93 @@
          "legendFormat": "{{stream}} / {{consumer}}"
        }
      ]
+    },
+    {
+      "id": 9,
+      "type": "timeseries",
+      "title": "NATS consumer ACK pending",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus-dev"
+      },
+      "gridPos": { "h": 7, "w": 12, "x": 12, "y": 26 },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "mappings": [],
+          "thresholds": { "mode": "absolute", "steps": [] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": { "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "single" }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "last_over_time(caatsm_nats_consumer_ack_pending_sum[1m])",
+          "legendFormat": "ACK pending"
+        }
+      ]
+    },
+    {
+      "id": 10,
+      "type": "timeseries",
+      "title": "NATS consumer redelivered messages",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus-dev"
+      },
+      "gridPos": { "h": 7, "w": 12, "x": 0, "y": 33 },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "mappings": [],
+          "thresholds": { "mode": "absolute", "steps": [] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": { "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "single" }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "last_over_time(caatsm_nats_consumer_redelivered_sum[1m])",
+          "legendFormat": "Redelivered"
+        }
+      ]
+    },
+    {
+      "id": 11,
+      "type": "timeseries",
+      "title": "NATS consumer delivered messages",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus-dev"
+      },
+      "gridPos": { "h": 7, "w": 12, "x": 12, "y": 33 },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "mappings": [],
+          "thresholds": { "mode": "absolute", "steps": [] }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": { "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "single" }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "last_over_time(caatsm_nats_consumer_delivered_sum[1m])",
+          "legendFormat": "Delivered"
+        }
+      ]
    }
  ]
 }
@@ -13,6 +13,10 @@ exporters:
    endpoint: jaeger:4317
    tls:
      insecure: true
+  prometheus:
+    endpoint: "0.0.0.0:8888"
+    const_labels:
+      source: "otel-collector"

 service:
  pipelines:
@@ -21,5 +25,5 @@ service:
      exporters: [logging, otlp/jaeger]
    metrics:
      receivers: [otlp]
-      exporters: [logging]
+      exporters: [logging, prometheus]