🔧 Update Go module dependencies and modify NATS configuration to exclusively support JetStream mode. Upgrade go.uber.org/zap to version 1.27.1 and golang.org/x/crypto to version 0.45.0. Revise Makefile and documentation to reflect changes in NATS mode, ensuring consistent messaging behavior across development and production environments. Enhance README and configuration files to clarify JetStream usage and remove references to Core NATS mode.
This commit is contained in:
@@ -29,8 +29,8 @@ build: ## Build the receiver binary
|
|||||||
run: run-dev ## Alias for run-dev
|
run: run-dev ## Alias for run-dev
|
||||||
|
|
||||||
.PHONY: run-dev
|
.PHONY: run-dev
|
||||||
run-dev: build ## Run the receiver in development mode (uses core NATS mode by default). Use MONITORING_ADDR=ip:port to set monitoring address
|
run-dev: build ## Run the receiver in development mode (uses JetStream mode). Use MONITORING_ADDR=ip:port to set monitoring address
|
||||||
@echo "Running receiver in development mode (NATS mode: core by default)..."
|
@echo "Running receiver in development mode (NATS mode: JetStream)..."
|
||||||
@if [ -n "$(MONITORING_ADDR)" ]; then \
|
@if [ -n "$(MONITORING_ADDR)" ]; then \
|
||||||
echo "Monitoring address: $(MONITORING_ADDR)"; \
|
echo "Monitoring address: $(MONITORING_ADDR)"; \
|
||||||
GO_ENV=dev $(BINARY) listen --monitoring-addr $(MONITORING_ADDR); \
|
GO_ENV=dev $(BINARY) listen --monitoring-addr $(MONITORING_ADDR); \
|
||||||
@@ -54,8 +54,8 @@ run-test: build ## Run the receiver in test mode
|
|||||||
@GO_ENV=test $(BINARY) listen
|
@GO_ENV=test $(BINARY) listen
|
||||||
|
|
||||||
.PHONY: run-local
|
.PHONY: run-local
|
||||||
run-local: ## Run receiver directly via go run (uses core NATS mode by default in dev). Use MONITORING_ADDR=ip:port to set monitoring address
|
run-local: ## Run receiver directly via go run (uses JetStream mode). Use MONITORING_ADDR=ip:port to set monitoring address
|
||||||
@echo "Running receiver via go run (GO_ENV=$(GO_ENV), NATS mode: core by default in dev)..."
|
@echo "Running receiver via go run (GO_ENV=$(GO_ENV), NATS mode: JetStream)..."
|
||||||
@if [ -n "$(MONITORING_ADDR)" ]; then \
|
@if [ -n "$(MONITORING_ADDR)" ]; then \
|
||||||
echo "Monitoring address: $(MONITORING_ADDR)"; \
|
echo "Monitoring address: $(MONITORING_ADDR)"; \
|
||||||
GO_ENV=$(GO_ENV) go run $(CMD) listen --monitoring-addr $(MONITORING_ADDR); \
|
GO_ENV=$(GO_ENV) go run $(CMD) listen --monitoring-addr $(MONITORING_ADDR); \
|
||||||
@@ -121,7 +121,7 @@ clean: ## Clean build artifacts and coverage files
|
|||||||
@rm -rf $(BUILD_DIR) coverage
|
@rm -rf $(BUILD_DIR) coverage
|
||||||
|
|
||||||
.PHONY: seed
|
.PHONY: seed
|
||||||
seed: ## Generate sample telegrams (publishes to Core NATS by default)
|
seed: ## Generate sample telegrams (publishes to JetStream)
|
||||||
@GO_ENV=dev \
|
@GO_ENV=dev \
|
||||||
NATS_URL=$${CAATSM_NATS_URL:-nats://localhost:4222} \
|
NATS_URL=$${CAATSM_NATS_URL:-nats://localhost:4222} \
|
||||||
SUBJECT=$${CAATSM_NATS_SUBJECT:-telegram.serial} \
|
SUBJECT=$${CAATSM_NATS_SUBJECT:-telegram.serial} \
|
||||||
|
|||||||
@@ -87,8 +87,7 @@ Configuration is loaded from TOML files and environment variables. The configura
|
|||||||
```toml
|
```toml
|
||||||
[nats]
|
[nats]
|
||||||
url = "nats://localhost:4222"
|
url = "nats://localhost:4222"
|
||||||
# mode: "jetstream" (default) or "core"
|
# mode: "jetstream" (required - only JetStream mode is supported)
|
||||||
# See "NATS Mode Selection" section below for detailed comparison
|
|
||||||
mode = "jetstream"
|
mode = "jetstream"
|
||||||
stream = "TELEGRAM"
|
stream = "TELEGRAM"
|
||||||
consumer = "telegram-consumer"
|
consumer = "telegram-consumer"
|
||||||
@@ -96,7 +95,7 @@ client = "serial-client"
|
|||||||
cluster = "tele-cluster"
|
cluster = "tele-cluster"
|
||||||
|
|
||||||
[nats.stream_limits]
|
[nats.stream_limits]
|
||||||
# These settings only apply when mode = "jetstream"
|
# Stream retention and storage limits
|
||||||
max_msgs = 100000
|
max_msgs = 100000
|
||||||
max_bytes = 67108864
|
max_bytes = 67108864
|
||||||
max_age = "24h"
|
max_age = "24h"
|
||||||
@@ -105,7 +104,7 @@ storage = "file"
|
|||||||
replicas = 1
|
replicas = 1
|
||||||
|
|
||||||
[nats.consumer_rules]
|
[nats.consumer_rules]
|
||||||
# Consumer delivery rules (only applies when mode = "jetstream")
|
# Consumer delivery rules
|
||||||
# max_deliver: Maximum number of delivery attempts before giving up
|
# max_deliver: Maximum number of delivery attempts before giving up
|
||||||
max_deliver = 3
|
max_deliver = 3
|
||||||
# ack_wait: Time to wait for ACK before redelivering message
|
# ack_wait: Time to wait for ACK before redelivering message
|
||||||
@@ -116,7 +115,7 @@ max_ack_pending = 1000
|
|||||||
[subscription]
|
[subscription]
|
||||||
# Optional. Defaults to "telegram.>" when omitted.
|
# Optional. Defaults to "telegram.>" when omitted.
|
||||||
topic = "telegram.serial"
|
topic = "telegram.serial"
|
||||||
# queue_group: Used in both core and jetstream modes for load balancing
|
# queue_group: Queue group for load balancing (JetStream uses durable consumers)
|
||||||
queue_group = "tele-queue"
|
queue_group = "tele-queue"
|
||||||
|
|
||||||
[publisher]
|
[publisher]
|
||||||
@@ -295,76 +294,6 @@ The application supports two NATS consumption modes, controlled by `nats.mode`:
|
|||||||
- **High pending count**: Increase `batch_size` or add more consumer instances
|
- **High pending count**: Increase `batch_size` or add more consumer instances
|
||||||
- **Messages being redelivered**: Check processing logs for errors; adjust `ack_wait` if processing takes longer
|
- **Messages being redelivered**: Check processing logs for errors; adjust `ack_wait` if processing takes longer
|
||||||
|
|
||||||
#### Core NATS Mode (Default for Development)
|
|
||||||
|
|
||||||
**Configuration:** `nats.mode = "core"` (default in `config.dev.toml`)
|
|
||||||
|
|
||||||
**Features:**
|
|
||||||
- ⚡ **Simple Pub/Sub**: Basic publish/subscribe messaging
|
|
||||||
- ⚡ **Queue Groups**: Load balancing across multiple consumers
|
|
||||||
- ⚡ **Low Latency**: No persistence overhead
|
|
||||||
- ⚡ **Fast Startup**: No stream/consumer setup required
|
|
||||||
- ❌ **No Persistence**: Messages are lost if no consumer is available
|
|
||||||
- ❌ **No ACK**: No delivery guarantees
|
|
||||||
- ❌ **No Retry**: Processing failures are logged but not retried
|
|
||||||
- ❌ **No DLQ**: Failed messages cannot be routed to a dead-letter queue
|
|
||||||
|
|
||||||
**Use Cases:**
|
|
||||||
- **Local development** (recommended default)
|
|
||||||
- Quick testing and iteration
|
|
||||||
- Real-time monitoring/logging where message loss is acceptable
|
|
||||||
- Simple pub/sub scenarios without reliability requirements
|
|
||||||
- Performance testing without persistence overhead
|
|
||||||
|
|
||||||
**Configuration Requirements:**
|
|
||||||
- Works with any NATS server (JetStream not required)
|
|
||||||
- Only `[subscription]` settings are used (queue_group for load balancing)
|
|
||||||
- `[nats.consumer_rules]` and `[nats.stream_limits]` are ignored
|
|
||||||
|
|
||||||
**How to Use Core Mode:**
|
|
||||||
|
|
||||||
1. **Start NATS Server** (JetStream not required, but can be enabled):
|
|
||||||
```bash
|
|
||||||
# Simple NATS server
|
|
||||||
nats-server
|
|
||||||
|
|
||||||
# Or with Docker Compose (JetStream enabled but not required for core mode)
|
|
||||||
docker compose -f docker-compose.dev.yml up -d nats
|
|
||||||
```
|
|
||||||
|
|
||||||
2. **Start the Application** (Core mode is default in dev config):
|
|
||||||
```bash
|
|
||||||
# Core mode is default, no need to specify
|
|
||||||
GO_ENV=dev \
|
|
||||||
CAATSM_POSTGRES_URL=postgres://user:pass@localhost:5432/aviation \
|
|
||||||
go run ./cmd/main listen
|
|
||||||
```
|
|
||||||
|
|
||||||
3. **Publish Messages** (use standard NATS publish):
|
|
||||||
```bash
|
|
||||||
# Using nats-box
|
|
||||||
docker compose exec nats-box nats pub telegram.serial "ZCZC TEST123 150631..."
|
|
||||||
|
|
||||||
# Or use seed-telegrams without --jetstream flag
|
|
||||||
go run ./cmd/seed-telegrams \
|
|
||||||
--nats-url nats://localhost:4222 \
|
|
||||||
--subject telegram.serial \
|
|
||||||
--count 10
|
|
||||||
```
|
|
||||||
|
|
||||||
**Switching Modes:**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Use Core NATS mode (default for development)
|
|
||||||
CAATSM_NATS_MODE=core go run ./cmd/main listen
|
|
||||||
# Or simply (core is default in config.dev.toml)
|
|
||||||
go run ./cmd/main listen
|
|
||||||
|
|
||||||
# Use JetStream mode (for production or integration testing)
|
|
||||||
CAATSM_NATS_MODE=jetstream go run ./cmd/main listen
|
|
||||||
```
|
|
||||||
|
|
||||||
**Note:** The publisher always uses JetStream for deduplicated fan-out, regardless of the consumer mode. If you need pure Core NATS, ensure publishers also use Core NATS subjects.
|
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
@@ -416,7 +345,7 @@ Build information is automatically populated from:
|
|||||||
Use Make targets (binary mode):
|
Use Make targets (binary mode):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make run-dev # GO_ENV=dev (uses core NATS mode by default)
|
make run-dev # GO_ENV=dev (uses JetStream mode)
|
||||||
make run-local # go run ./cmd/main listen (honors GO_ENV)
|
make run-local # go run ./cmd/main listen (honors GO_ENV)
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -425,7 +354,7 @@ Task equivalents:
|
|||||||
```bash
|
```bash
|
||||||
task run-dev
|
task run-dev
|
||||||
task run-local # go run ./cmd/main listen
|
task run-local # go run ./cmd/main listen
|
||||||
task dev-run # boots docker-compose dev stack + go run (core NATS mode)
|
task dev-run # boots docker-compose dev stack + go run (JetStream mode)
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Production Mode
|
#### Production Mode
|
||||||
@@ -567,7 +496,7 @@ docker compose -f docker-compose.dev.yml up -d otel-collector jaeger prometheus
|
|||||||
Run the processor locally while the infra runs in Docker:
|
Run the processor locally while the infra runs in Docker:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Core NATS mode (default for development, fast and lightweight)
|
# Development mode (JetStream enabled)
|
||||||
GO_ENV=dev \
|
GO_ENV=dev \
|
||||||
CAATSM_POSTGRES_URL=postgres://caatsm:caatsm@localhost:5432/aviation?sslmode=disable \
|
CAATSM_POSTGRES_URL=postgres://caatsm:caatsm@localhost:5432/aviation?sslmode=disable \
|
||||||
go run ./cmd/main listen
|
go run ./cmd/main listen
|
||||||
@@ -655,9 +584,8 @@ The project keeps tests close to the code that they exercise:
|
|||||||
|
|
||||||
## Message Flow
|
## Message Flow
|
||||||
|
|
||||||
1. **NATS Consumer** receives raw telegram messages from NATS:
|
1. **NATS Consumer** receives raw telegram messages from NATS JetStream:
|
||||||
- **JetStream mode** (default): Uses durable pull consumer with batch processing, ACK/NAK, and retry logic
|
- Uses durable pull consumer with batch processing, ACK/NAK, and retry logic
|
||||||
- **Core mode**: Uses `QueueSubscribe` for simple pub/sub with queue group load balancing (no persistence or retries)
|
|
||||||
2. **MessageProcessor** orchestrates the processing:
|
2. **MessageProcessor** orchestrates the processing:
|
||||||
- Parses the message using the Parser adapter
|
- Parses the message using the Parser adapter
|
||||||
- Stores the parsed message in PostgreSQL via Repository
|
- Stores the parsed message in PostgreSQL via Repository
|
||||||
|
|||||||
+2
-2
@@ -85,13 +85,13 @@ tasks:
|
|||||||
- GO_ENV=test {{.binary}} listen
|
- GO_ENV=test {{.binary}} listen
|
||||||
|
|
||||||
run-local:
|
run-local:
|
||||||
desc: 'Run receiver via go run (default GO_ENV=dev, uses core NATS mode by default). Use MONITORING_ADDR=ip:port to set monitoring address'
|
desc: 'Run receiver via go run (default GO_ENV=dev, uses JetStream mode). Use MONITORING_ADDR=ip:port to set monitoring address'
|
||||||
vars:
|
vars:
|
||||||
MONITORING_ADDR:
|
MONITORING_ADDR:
|
||||||
sh: echo "${MONITORING_ADDR:-}"
|
sh: echo "${MONITORING_ADDR:-}"
|
||||||
cmds:
|
cmds:
|
||||||
- |
|
- |
|
||||||
echo "Running receiver via go run (GO_ENV=${GO_ENV:-dev}, NATS mode: core by default in dev)..."
|
echo "Running receiver via go run (GO_ENV=${GO_ENV:-dev}, NATS mode: JetStream)..."
|
||||||
- |
|
- |
|
||||||
cmd="GO_ENV=${GO_ENV:-dev} go run {{.cmd}} listen"
|
cmd="GO_ENV=${GO_ENV:-dev} go run {{.cmd}} listen"
|
||||||
if [ -n "${MONITORING_ADDR:-}" ]; then
|
if [ -n "${MONITORING_ADDR:-}" ]; then
|
||||||
|
|||||||
+6
-14
@@ -1,20 +1,12 @@
|
|||||||
[nats]
|
[nats]
|
||||||
url = "nats://localhost:4222"
|
url = "nats://localhost:4222"
|
||||||
# mode: "core" (default for dev) or "jetstream" (recommended for production)
|
# mode: "jetstream" (required - only JetStream mode is supported)
|
||||||
# - "core": Uses Core NATS for simple pub/sub messaging with:
|
# JetStream provides:
|
||||||
# * No message persistence (messages lost if consumer offline)
|
|
||||||
# * No ACK mechanism (fire-and-forget delivery)
|
|
||||||
# * No automatic retry on processing failures
|
|
||||||
# * Queue groups for load balancing only
|
|
||||||
# * Suitable for development/testing or real-time scenarios where message loss is acceptable
|
|
||||||
# * Recommended for local development and testing
|
|
||||||
# - "jetstream": Uses NATS JetStream for persistent message streaming with:
|
|
||||||
# * Message persistence and replay capability
|
# * Message persistence and replay capability
|
||||||
# * ACK/NAK mechanism for guaranteed delivery
|
# * ACK/NAK mechanism for guaranteed delivery
|
||||||
# * Automatic retry with configurable backoff
|
# * Automatic retry with configurable backoff
|
||||||
# * Dead-letter queue (DLQ) support
|
# * Dead-letter queue (DLQ) support
|
||||||
# * Batch processing and consumer monitoring
|
# * Batch processing and consumer monitoring
|
||||||
# * Recommended for production environments
|
|
||||||
mode = "jetstream"
|
mode = "jetstream"
|
||||||
client = "serial-client"
|
client = "serial-client"
|
||||||
cluster = "tele-cluster"
|
cluster = "tele-cluster"
|
||||||
@@ -22,7 +14,7 @@ stream = "TELEGRAM"
|
|||||||
consumer = "telegram-consumer"
|
consumer = "telegram-consumer"
|
||||||
|
|
||||||
[nats.stream_limits]
|
[nats.stream_limits]
|
||||||
# Stream retention and storage limits (only applies when mode = "jetstream")
|
# Stream retention and storage limits
|
||||||
# max_msgs: Maximum number of messages to keep in the stream (0 = unlimited)
|
# max_msgs: Maximum number of messages to keep in the stream (0 = unlimited)
|
||||||
max_msgs = 100000
|
max_msgs = 100000
|
||||||
# max_bytes: Maximum total size of messages in bytes (0 = unlimited, 67108864 = 64MB)
|
# max_bytes: Maximum total size of messages in bytes (0 = unlimited, 67108864 = 64MB)
|
||||||
@@ -37,7 +29,7 @@ storage = "file"
|
|||||||
replicas = 1
|
replicas = 1
|
||||||
|
|
||||||
[nats.consumer_rules]
|
[nats.consumer_rules]
|
||||||
# Consumer delivery rules (only applies when mode = "jetstream")
|
# Consumer delivery rules
|
||||||
# max_deliver: Maximum number of delivery attempts before giving up (0 = unlimited)
|
# max_deliver: Maximum number of delivery attempts before giving up (0 = unlimited)
|
||||||
max_deliver = 3
|
max_deliver = 3
|
||||||
# ack_wait: Time to wait for ACK before redelivering message (e.g., "30s", "2m")
|
# ack_wait: Time to wait for ACK before redelivering message (e.g., "30s", "2m")
|
||||||
@@ -75,7 +67,7 @@ max_conns = 10
|
|||||||
min_conns = 2
|
min_conns = 2
|
||||||
|
|
||||||
[app]
|
[app]
|
||||||
# Batch processing configuration (applies to both core and jetstream modes)
|
# Batch processing configuration
|
||||||
# batch_size: Number of messages to fetch in each batch (JetStream pull batch size)
|
# batch_size: Number of messages to fetch in each batch (JetStream pull batch size)
|
||||||
batch_size = 50
|
batch_size = 50
|
||||||
# batch_timeout: Maximum time to wait when fetching a batch (e.g., "2s")
|
# batch_timeout: Maximum time to wait when fetching a batch (e.g., "2s")
|
||||||
@@ -121,7 +113,7 @@ write_timeout = "5s"
|
|||||||
health_timeout = "2s"
|
health_timeout = "2s"
|
||||||
|
|
||||||
[dlq]
|
[dlq]
|
||||||
# Dead-Letter Queue configuration (only applies when mode = "jetstream")
|
# Dead-Letter Queue configuration
|
||||||
# enabled: Enable DLQ routing for poison messages (messages that fail after max_deliver attempts)
|
# enabled: Enable DLQ routing for poison messages (messages that fail after max_deliver attempts)
|
||||||
enabled = true # Set to true when switching to JetStream mode
|
enabled = true # Set to true when switching to JetStream mode
|
||||||
# subject: NATS subject where failed messages will be published for manual inspection
|
# subject: NATS subject where failed messages will be published for manual inspection
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
[nats]
|
[nats]
|
||||||
url = "nats://localhost:4222"
|
url = "nats://localhost:4222"
|
||||||
|
mode = "jetstream"
|
||||||
stream = "TELEGRAM"
|
stream = "TELEGRAM"
|
||||||
|
|
||||||
[publisher]
|
[publisher]
|
||||||
|
|||||||
+5
-35
@@ -10,11 +10,7 @@ Spin up PostgreSQL/TimescaleDB and NATS JetStream in the background:
|
|||||||
docker compose -f docker-compose.dev.yml up -d postgres nats nats-box
|
docker compose -f docker-compose.dev.yml up -d postgres nats nats-box
|
||||||
```
|
```
|
||||||
|
|
||||||
> **NATS Mode Selection:** The application supports two consumption modes:
|
> **NATS Mode:** The application uses JetStream mode exclusively for message persistence, ACK/NAK, automatic retries, and DLQ support. This ensures consistent behavior across development, testing, and production environments.
|
||||||
> - **Core NATS mode** (default for development): Simple pub/sub without persistence or retry mechanisms. Recommended for local development and testing where message loss is acceptable. Fast and lightweight.
|
|
||||||
> - **JetStream mode**: Provides message persistence, ACK/NAK, automatic retries, and DLQ support. Recommended for production environments and integration testing.
|
|
||||||
>
|
|
||||||
> Development mode defaults to `nats.mode = "core"` in `config.dev.toml`. To use JetStream in development, set `CAATSM_NATS_MODE=jetstream` or change the config file. **Note:** The publisher always targets JetStream for deduplicated fan-out, so if you use Core mode for consumption, ensure your publishers align with your messaging strategy. See the README.md "NATS Mode Selection" section for a detailed comparison.
|
|
||||||
|
|
||||||
- `postgres` seeds the `aviation` schema using `internal/infra/postgres/telegrams.ddl` and exposes port `5432`.
|
- `postgres` seeds the `aviation` schema using `internal/infra/postgres/telegrams.ddl` and exposes port `5432`.
|
||||||
- `nats` enables JetStream with client port `4222` and monitoring/UI on `8222`.
|
- `nats` enables JetStream with client port `4222` and monitoring/UI on `8222`.
|
||||||
@@ -55,7 +51,7 @@ docker compose -f docker-compose.dev.yml up -d postgres nats nats-box
|
|||||||
The `Taskfile.yml` includes helper targets that wrap the commands above:
|
The `Taskfile.yml` includes helper targets that wrap the commands above:
|
||||||
|
|
||||||
- `task up` – starts PostgreSQL, NATS (JetStream, toolbox, and Prometheus exporter), and the observability stack (OpenTelemetry Collector, Jaeger, Prometheus, Grafana) using Docker Compose.
|
- `task up` – starts PostgreSQL, NATS (JetStream, toolbox, and Prometheus exporter), and the observability stack (OpenTelemetry Collector, Jaeger, Prometheus, Grafana) using Docker Compose.
|
||||||
- `task dev-run` – ensures `task up` has run, exports the necessary `CAATSM_*` environment variables (defaults to `CAATSM_NATS_MODE=core` for development), and executes `go run ./cmd/main listen` with telemetry enabled.
|
- `task dev-run` – ensures `task up` has run, exports the necessary `CAATSM_*` environment variables, and executes `go run ./cmd/main listen` with telemetry enabled.
|
||||||
- `task down` – stops the entire stack and removes containers/volumes.
|
- `task down` – stops the entire stack and removes containers/volumes.
|
||||||
|
|
||||||
Use these tasks if you prefer a one-command workflow instead of invoking `docker compose` and environment exports manually.
|
Use these tasks if you prefer a one-command workflow instead of invoking `docker compose` and environment exports manually.
|
||||||
@@ -80,31 +76,17 @@ GO_ENV=dev go run ./cmd/seed-telegrams \
|
|||||||
--status random
|
--status random
|
||||||
```
|
```
|
||||||
|
|
||||||
### Publishing to Core NATS
|
|
||||||
|
|
||||||
For Core NATS mode, use standard publish:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Publish to Core NATS (no persistence)
|
|
||||||
GO_ENV=dev go run ./cmd/seed-telegrams \
|
|
||||||
--nats-url nats://127.0.0.1:4222 \
|
|
||||||
--subject telegram.serial \
|
|
||||||
--count 20 \
|
|
||||||
--category mixed \
|
|
||||||
--status random
|
|
||||||
```
|
|
||||||
|
|
||||||
### Using Makefile/Taskfile Tasks
|
### Using Makefile/Taskfile Tasks
|
||||||
|
|
||||||
For convenience, you can use the provided tasks:
|
For convenience, you can use the provided tasks:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Quick seed (10 messages, Core NATS)
|
# Quick seed (10 messages, JetStream)
|
||||||
make seed
|
make seed
|
||||||
# or
|
# or
|
||||||
task seed
|
task seed
|
||||||
|
|
||||||
# Continuous slow seeding (until Ctrl-C, JetStream by default)
|
# Continuous slow seeding (until Ctrl-C, JetStream)
|
||||||
make seed-slow
|
make seed-slow
|
||||||
# or
|
# or
|
||||||
task seed-slow
|
task seed-slow
|
||||||
@@ -125,8 +107,6 @@ make seed-slow
|
|||||||
# Custom interval and category
|
# Custom interval and category
|
||||||
make seed-slow INTERVAL_MIN=5s INTERVAL_MAX=10s CATEGORY=DEP
|
make seed-slow INTERVAL_MIN=5s INTERVAL_MAX=10s CATEGORY=DEP
|
||||||
|
|
||||||
# Use Core NATS instead of JetStream
|
|
||||||
make seed-slow USE_JS=false INTERVAL_MIN=1s INTERVAL_MAX=3s
|
|
||||||
```
|
```
|
||||||
|
|
||||||
This is equivalent to running:
|
This is equivalent to running:
|
||||||
@@ -164,7 +144,7 @@ Setting `--count 0` makes it run indefinitely until interrupted.
|
|||||||
# View messages in JetStream stream
|
# View messages in JetStream stream
|
||||||
docker compose exec nats-box nats stream view TELEGRAM
|
docker compose exec nats-box nats stream view TELEGRAM
|
||||||
|
|
||||||
# Subscribe to messages (Core NATS or JetStream)
|
# Subscribe to messages (JetStream)
|
||||||
docker compose exec nats-box nats sub 'telegram.>'
|
docker compose exec nats-box nats sub 'telegram.>'
|
||||||
|
|
||||||
# View consumer status and pending messages
|
# View consumer status and pending messages
|
||||||
@@ -184,22 +164,12 @@ The main processor keeps consuming `subscription.topic` (defaults to `telegram.>
|
|||||||
|
|
||||||
2. **Run the processor with telemetry enabled**
|
2. **Run the processor with telemetry enabled**
|
||||||
```bash
|
```bash
|
||||||
# Using Core NATS mode (default for development)
|
|
||||||
CAATSM_TELEMETRY_ENABLED=true \
|
CAATSM_TELEMETRY_ENABLED=true \
|
||||||
CAATSM_TELEMETRY_ENDPOINT=localhost:4318 \
|
CAATSM_TELEMETRY_ENDPOINT=localhost:4318 \
|
||||||
CAATSM_TELEMETRY_INSECURE=true \
|
CAATSM_TELEMETRY_INSECURE=true \
|
||||||
GO_ENV=dev \
|
GO_ENV=dev \
|
||||||
CAATSM_POSTGRES_URL=postgres://caatsm:caatsm@localhost:5432/aviation?sslmode=disable \
|
CAATSM_POSTGRES_URL=postgres://caatsm:caatsm@localhost:5432/aviation?sslmode=disable \
|
||||||
go run ./cmd/main listen
|
go run ./cmd/main listen
|
||||||
|
|
||||||
# Or use JetStream mode for integration testing
|
|
||||||
CAATSM_TELEMETRY_ENABLED=true \
|
|
||||||
CAATSM_TELEMETRY_ENDPOINT=localhost:4318 \
|
|
||||||
CAATSM_TELEMETRY_INSECURE=true \
|
|
||||||
GO_ENV=dev \
|
|
||||||
CAATSM_NATS_MODE=jetstream \
|
|
||||||
CAATSM_POSTGRES_URL=postgres://caatsm:caatsm@localhost:5432/aviation?sslmode=disable \
|
|
||||||
go run ./cmd/main listen
|
|
||||||
```
|
```
|
||||||
- The service name reported to Jaeger is `caatsm`.
|
- The service name reported to Jaeger is `caatsm`.
|
||||||
|
|
||||||
|
|||||||
+2
-4
@@ -445,9 +445,7 @@ Success
|
|||||||
|
|
||||||
The consumer handles message consumption with the following features:
|
The consumer handles message consumption with the following features:
|
||||||
|
|
||||||
#### Modes
|
The consumer uses JetStream mode exclusively for persistent, durable message processing with acknowledgments.
|
||||||
- **JetStream Mode**: Persistent, durable message processing with acknowledgments
|
|
||||||
- **Core Mode**: Fire-and-forget message processing for simple use cases
|
|
||||||
|
|
||||||
#### Key Features
|
#### Key Features
|
||||||
- **Batch Processing**: Configurable batch sizes for efficient processing
|
- **Batch Processing**: Configurable batch sizes for efficient processing
|
||||||
@@ -508,7 +506,7 @@ The publisher handles message publishing with deduplication and observability.
|
|||||||
|
|
||||||
#### Features
|
#### Features
|
||||||
- **Message Deduplication**: Automatic UUID-based deduplication headers
|
- **Message Deduplication**: Automatic UUID-based deduplication headers
|
||||||
- **JetStream/Core Mode Support**: Adapts based on available JetStream context
|
- **JetStream Publishing**: Uses JetStream for reliable message delivery
|
||||||
- **Structured Logging**: Comprehensive logging of publish operations
|
- **Structured Logging**: Comprehensive logging of publish operations
|
||||||
- **Error Classification**: Distinguishes between transient and permanent errors
|
- **Error Classification**: Distinguishes between transient and permanent errors
|
||||||
|
|
||||||
|
|||||||
+1
-5
@@ -15,11 +15,7 @@ subject = "caatsm.dlq"
|
|||||||
- When `dlq.enabled` is `true` and `dlq.subject` is non-empty, **permanent** failures are routed to the DLQ subject.
|
- When `dlq.enabled` is `true` and `dlq.subject` is non-empty, **permanent** failures are routed to the DLQ subject.
|
||||||
- A permanent failure is indicated by wrapping an error with `app.Permanent` and is treated as a **poison message**.
|
- A permanent failure is indicated by wrapping an error with `app.Permanent` and is treated as a **poison message**.
|
||||||
|
|
||||||
> Note: DLQ routing only applies when the NATS mode is `jetstream`. In `core`
|
Behaviour:
|
||||||
> mode, the consumer does not attempt to publish to the DLQ even if it is
|
|
||||||
> configured.
|
|
||||||
|
|
||||||
Behaviour (JetStream mode):
|
|
||||||
|
|
||||||
1. The NATS consumer calls `processor.Handle`.
|
1. The NATS consumer calls `processor.Handle`.
|
||||||
2. If an error is returned and `app.IsPermanent(err)` is true:
|
2. If an error is returned and `app.IsPermanent(err)` is true:
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ require (
|
|||||||
go.opentelemetry.io/otel/sdk v1.38.0
|
go.opentelemetry.io/otel/sdk v1.38.0
|
||||||
go.opentelemetry.io/otel/sdk/metric v1.38.0
|
go.opentelemetry.io/otel/sdk/metric v1.38.0
|
||||||
go.opentelemetry.io/otel/trace v1.38.0
|
go.opentelemetry.io/otel/trace v1.38.0
|
||||||
go.uber.org/zap v1.27.0
|
go.uber.org/zap v1.27.1
|
||||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1
|
gopkg.in/natefinch/lumberjack.v2 v2.2.1
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -100,7 +100,7 @@ require (
|
|||||||
go.uber.org/multierr v1.11.0 // indirect
|
go.uber.org/multierr v1.11.0 // indirect
|
||||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||||
golang.org/x/crypto v0.44.0 // indirect
|
golang.org/x/crypto v0.45.0 // indirect
|
||||||
golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect
|
golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect
|
||||||
golang.org/x/mod v0.30.0 // indirect
|
golang.org/x/mod v0.30.0 // indirect
|
||||||
golang.org/x/net v0.47.0 // indirect
|
golang.org/x/net v0.47.0 // indirect
|
||||||
|
|||||||
@@ -330,10 +330,10 @@ func (c *Config) Validate() error {
|
|||||||
if c.NATS.URL == "" {
|
if c.NATS.URL == "" {
|
||||||
return fmt.Errorf("nats.url is required")
|
return fmt.Errorf("nats.url is required")
|
||||||
}
|
}
|
||||||
switch strings.ToLower(c.NATS.Mode) {
|
// Validate NATS mode - only JetStream is supported
|
||||||
case "", "jetstream", "core":
|
mode := strings.ToLower(c.NATS.Mode)
|
||||||
default:
|
if mode != "" && mode != "jetstream" {
|
||||||
return fmt.Errorf("nats.mode must be 'jetstream' or 'core'")
|
return fmt.Errorf("nats.mode must be 'jetstream' or empty (defaults to 'jetstream'), got: %s", c.NATS.Mode)
|
||||||
}
|
}
|
||||||
if c.NATS.Stream == "" {
|
if c.NATS.Stream == "" {
|
||||||
return fmt.Errorf("nats.stream is required")
|
return fmt.Errorf("nats.stream is required")
|
||||||
|
|||||||
+46
-113
@@ -45,7 +45,6 @@ type Consumer struct {
|
|||||||
type consumerConfig struct {
|
type consumerConfig struct {
|
||||||
subject string
|
subject string
|
||||||
consumerName string
|
consumerName string
|
||||||
mode string
|
|
||||||
streamName string
|
streamName string
|
||||||
dlqSubject string
|
dlqSubject string
|
||||||
ackWait time.Duration
|
ackWait time.Duration
|
||||||
@@ -78,75 +77,56 @@ func ProvideConsumer(
|
|||||||
|
|
||||||
// Initialize the pending messages metric early (set to 0) so it appears in Prometheus
|
// Initialize the pending messages metric early (set to 0) so it appears in Prometheus
|
||||||
// even before the consumer starts. This ensures the metric is always visible.
|
// even before the consumer starts. This ensures the metric is always visible.
|
||||||
// We do this first, before any operations that might fail, to ensure the metric exists.
|
logger.Info("Initializing NATS consumer pending messages metric",
|
||||||
// Initialize the metric unconditionally when in JetStream mode, even if js is nil,
|
zap.String("stream", normCfg.streamName),
|
||||||
// as it will be updated later when js becomes available.
|
zap.String("consumer", normCfg.consumerName),
|
||||||
if normCfg.mode == "jetstream" {
|
zap.Uint64("pending", 0),
|
||||||
logger.Info("Initializing NATS consumer pending messages metric",
|
zap.Bool("js_available", js != nil),
|
||||||
zap.String("stream", normCfg.streamName),
|
)
|
||||||
zap.String("consumer", normCfg.consumerName),
|
obsmetrics.RecordNATSConsumerPending(normCfg.streamName, normCfg.consumerName, 0)
|
||||||
zap.Uint64("pending", 0),
|
|
||||||
zap.Bool("js_available", js != nil),
|
|
||||||
)
|
|
||||||
// Always initialize the metric in JetStream mode to ensure it appears in Prometheus
|
|
||||||
// The metric will be updated with actual values when the consumer starts
|
|
||||||
obsmetrics.RecordNATSConsumerPending(normCfg.streamName, normCfg.consumerName, 0)
|
|
||||||
} else {
|
|
||||||
logger.Debug("Skipping NATS consumer pending messages metric initialization (not JetStream mode)",
|
|
||||||
zap.String("mode", normCfg.mode),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize managers
|
// Initialize managers
|
||||||
if consumer.config.mode == "jetstream" {
|
consumer.consumerManager = NewConsumerManager(js, normCfg.streamName, normCfg.consumerName, normCfg.subject, logger)
|
||||||
consumer.consumerManager = NewConsumerManager(js, normCfg.streamName, normCfg.consumerName, normCfg.subject, logger)
|
// Use StreamManager with full configuration
|
||||||
// Use StreamManager with full configuration
|
streamSubjects := []string{normCfg.subject}
|
||||||
streamSubjects := []string{normCfg.subject}
|
if publisherSubject := strings.TrimSpace(cfg.Publisher.Topic); publisherSubject != "" {
|
||||||
if publisherSubject := strings.TrimSpace(cfg.Publisher.Topic); publisherSubject != "" {
|
streamSubjects = append(streamSubjects, publisherSubject)
|
||||||
streamSubjects = append(streamSubjects, publisherSubject)
|
}
|
||||||
}
|
// Add DLQ subject to stream if DLQ is enabled
|
||||||
// Add DLQ subject to stream if DLQ is enabled
|
if normCfg.dlqSubject != "" {
|
||||||
if normCfg.dlqSubject != "" {
|
streamSubjects = append(streamSubjects, normCfg.dlqSubject)
|
||||||
streamSubjects = append(streamSubjects, normCfg.dlqSubject)
|
}
|
||||||
}
|
streamSubjects = dedupeSubjects(streamSubjects)
|
||||||
streamSubjects = dedupeSubjects(streamSubjects)
|
consumer.streamManager = NewStreamManager(js, normCfg.streamName, streamSubjects, logger)
|
||||||
consumer.streamManager = NewStreamManager(js, normCfg.streamName, streamSubjects, logger)
|
|
||||||
|
|
||||||
// Update fetcher with managers now that they're initialized
|
// Update fetcher with managers now that they're initialized
|
||||||
if fetcher, ok := consumer.fetcher.(*defaultMessageFetcher); ok {
|
if fetcher, ok := consumer.fetcher.(*defaultMessageFetcher); ok {
|
||||||
fetcher.consumerManager = consumer.consumerManager
|
fetcher.consumerManager = consumer.consumerManager
|
||||||
fetcher.streamManager = consumer.streamManager
|
fetcher.streamManager = consumer.streamManager
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure stream exists before creating consumer
|
// Ensure stream exists before creating consumer
|
||||||
streamCfg := &StreamConfig{
|
streamCfg := &StreamConfig{
|
||||||
MaxMsgs: cfg.NATS.StreamLimits.MaxMsgs,
|
MaxMsgs: cfg.NATS.StreamLimits.MaxMsgs,
|
||||||
MaxBytes: cfg.NATS.StreamLimits.MaxBytes,
|
MaxBytes: cfg.NATS.StreamLimits.MaxBytes,
|
||||||
MaxAge: cfg.NATS.StreamLimits.MaxAge,
|
MaxAge: cfg.NATS.StreamLimits.MaxAge,
|
||||||
Discard: cfg.NATS.StreamLimits.Discard,
|
Discard: cfg.NATS.StreamLimits.Discard,
|
||||||
Storage: cfg.NATS.StreamLimits.Storage,
|
Storage: cfg.NATS.StreamLimits.Storage,
|
||||||
Replicas: cfg.NATS.StreamLimits.Replicas,
|
Replicas: cfg.NATS.StreamLimits.Replicas,
|
||||||
}
|
}
|
||||||
if err := consumer.streamManager.EnsureStream(streamCfg); err != nil {
|
if err := consumer.streamManager.EnsureStream(streamCfg); err != nil {
|
||||||
return nil, fmt.Errorf("failed to ensure stream: %w", err)
|
return nil, fmt.Errorf("failed to ensure stream: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create consumer if it doesn't exist
|
// Create consumer if it doesn't exist
|
||||||
consumerConfig := consumer.buildConsumerConfig()
|
consumerConfig := consumer.buildConsumerConfig()
|
||||||
if err := consumer.consumerManager.EnsureConsumer(consumerConfig); err != nil {
|
if err := consumer.consumerManager.EnsureConsumer(consumerConfig); err != nil {
|
||||||
return nil, fmt.Errorf("failed to ensure consumer: %w", err)
|
return nil, fmt.Errorf("failed to ensure consumer: %w", err)
|
||||||
}
|
}
|
||||||
// Validate DLQ configuration early so misconfiguration is visible at startup
|
// Validate DLQ configuration early so misconfiguration is visible at startup
|
||||||
// rather than only when the first poison message appears.
|
// rather than only when the first poison message appears.
|
||||||
if err := consumer.validateDLQ(); err != nil {
|
if err := consumer.validateDLQ(); err != nil {
|
||||||
return nil, fmt.Errorf("DLQ validation failed: %w", err)
|
return nil, fmt.Errorf("DLQ validation failed: %w", err)
|
||||||
}
|
|
||||||
|
|
||||||
} else {
|
|
||||||
logger.Info("Running consumer in core NATS mode",
|
|
||||||
zap.String("subject", normCfg.subject),
|
|
||||||
zap.String("queue_group", cfg.Subscription.QueueGroup),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return consumer, nil
|
return consumer, nil
|
||||||
@@ -185,7 +165,6 @@ func (c *Consumer) initCollaborators() {
|
|||||||
telemetry: c.telemetry,
|
telemetry: c.telemetry,
|
||||||
streamName: c.config.streamName,
|
streamName: c.config.streamName,
|
||||||
consumerName: c.config.consumerName,
|
consumerName: c.config.consumerName,
|
||||||
mode: c.config.mode,
|
|
||||||
backoff: c.cfg.NATS.ConsumerRules.Backoff,
|
backoff: c.cfg.NATS.ConsumerRules.Backoff,
|
||||||
consecutiveProcessErrors: &c.consecutiveProcessErrors,
|
consecutiveProcessErrors: &c.consecutiveProcessErrors,
|
||||||
}
|
}
|
||||||
@@ -201,11 +180,6 @@ func normalizeConsumerConfig(cfg *config.Config) *consumerConfig {
|
|||||||
consumerName = "telegram-consumer"
|
consumerName = "telegram-consumer"
|
||||||
}
|
}
|
||||||
|
|
||||||
mode := strings.ToLower(cfg.NATS.Mode)
|
|
||||||
if mode == "" {
|
|
||||||
mode = "jetstream"
|
|
||||||
}
|
|
||||||
|
|
||||||
streamName := cfg.NATS.Stream
|
streamName := cfg.NATS.Stream
|
||||||
if streamName == "" {
|
if streamName == "" {
|
||||||
streamName = "TELEGRAM"
|
streamName = "TELEGRAM"
|
||||||
@@ -244,7 +218,6 @@ func normalizeConsumerConfig(cfg *config.Config) *consumerConfig {
|
|||||||
return &consumerConfig{
|
return &consumerConfig{
|
||||||
subject: subject,
|
subject: subject,
|
||||||
consumerName: consumerName,
|
consumerName: consumerName,
|
||||||
mode: mode,
|
|
||||||
streamName: streamName,
|
streamName: streamName,
|
||||||
dlqSubject: dlqSubject,
|
dlqSubject: dlqSubject,
|
||||||
ackWait: ackWait,
|
ackWait: ackWait,
|
||||||
@@ -269,12 +242,8 @@ func (c *Consumer) buildConsumerConfig() *nats.ConsumerConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start starts consuming messages.
|
// Start starts consuming messages from JetStream.
|
||||||
func (c *Consumer) Start(ctx context.Context) error {
|
func (c *Consumer) Start(ctx context.Context) error {
|
||||||
if c.config.mode == "core" {
|
|
||||||
return c.startCore(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
return c.startJetStream(ctx)
|
return c.startJetStream(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -299,42 +268,6 @@ func (c *Consumer) validateDLQ() error {
|
|||||||
return c.ValidateDLQ()
|
return c.ValidateDLQ()
|
||||||
}
|
}
|
||||||
|
|
||||||
// startCore starts the Core NATS consumer loop.
|
|
||||||
func (c *Consumer) startCore(ctx context.Context) error {
|
|
||||||
queueGroup := c.cfg.Subscription.QueueGroup
|
|
||||||
if queueGroup == "" {
|
|
||||||
queueGroup = c.config.consumerName
|
|
||||||
}
|
|
||||||
|
|
||||||
handler := func(msg *nats.Msg) {
|
|
||||||
// Use ProcessBatch to ensure metrics are recorded via processSingleMessage
|
|
||||||
// ProcessBatch handles error recording and metrics for both success and failure cases
|
|
||||||
c.batchProcessor.ProcessBatch(ctx, []*nats.Msg{msg})
|
|
||||||
}
|
|
||||||
|
|
||||||
sub, err := c.conn.QueueSubscribe(c.config.subject, queueGroup, handler)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to subscribe to %s: %w", c.config.subject, err)
|
|
||||||
}
|
|
||||||
if err := c.conn.Flush(); err != nil {
|
|
||||||
return fmt.Errorf("failed to flush NATS connection: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
c.logger.Info("Started core NATS subscription",
|
|
||||||
zap.String("subject", c.config.subject),
|
|
||||||
zap.String("queue_group", queueGroup),
|
|
||||||
)
|
|
||||||
|
|
||||||
<-ctx.Done()
|
|
||||||
c.logger.Info("Stopping core NATS consumer", zap.Error(ctx.Err()))
|
|
||||||
|
|
||||||
if err := sub.Drain(); err != nil && !errors.Is(err, nats.ErrConnectionClosed) {
|
|
||||||
return fmt.Errorf("failed to drain core subscription: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return ctx.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
// createPullSubscription creates a pull subscription
|
// createPullSubscription creates a pull subscription
|
||||||
func (c *Consumer) createPullSubscription() (*nats.Subscription, error) {
|
func (c *Consumer) createPullSubscription() (*nats.Subscription, error) {
|
||||||
return c.consumerManager.CreatePullSubscription()
|
return c.consumerManager.CreatePullSubscription()
|
||||||
|
|||||||
@@ -62,7 +62,6 @@ var _ = Describe("Consumer helpers", func() {
|
|||||||
normCfg := normalizeConsumerConfig(cfg)
|
normCfg := normalizeConsumerConfig(cfg)
|
||||||
|
|
||||||
Expect(normCfg.consumerName).To(Equal("telegram-consumer"))
|
Expect(normCfg.consumerName).To(Equal("telegram-consumer"))
|
||||||
Expect(normCfg.mode).To(Equal("jetstream"))
|
|
||||||
Expect(normCfg.streamName).To(Equal("TELEGRAM"))
|
Expect(normCfg.streamName).To(Equal("TELEGRAM"))
|
||||||
Expect(normCfg.batchSize).To(Equal(50))
|
Expect(normCfg.batchSize).To(Equal(50))
|
||||||
Expect(normCfg.batchTimeout).To(Equal(2 * time.Second))
|
Expect(normCfg.batchTimeout).To(Equal(2 * time.Second))
|
||||||
@@ -74,7 +73,6 @@ var _ = Describe("Consumer helpers", func() {
|
|||||||
cfg := &configpkg.Config{
|
cfg := &configpkg.Config{
|
||||||
NATS: configpkg.NATSConfig{
|
NATS: configpkg.NATSConfig{
|
||||||
Consumer: "custom-consumer",
|
Consumer: "custom-consumer",
|
||||||
Mode: "core",
|
|
||||||
Stream: "CUSTOM_STREAM",
|
Stream: "CUSTOM_STREAM",
|
||||||
ConsumerRules: configpkg.ConsumerRulesConfig{
|
ConsumerRules: configpkg.ConsumerRulesConfig{
|
||||||
AckWait: 60 * time.Second,
|
AckWait: 60 * time.Second,
|
||||||
@@ -93,7 +91,6 @@ var _ = Describe("Consumer helpers", func() {
|
|||||||
normCfg := normalizeConsumerConfig(cfg)
|
normCfg := normalizeConsumerConfig(cfg)
|
||||||
|
|
||||||
Expect(normCfg.consumerName).To(Equal("custom-consumer"))
|
Expect(normCfg.consumerName).To(Equal("custom-consumer"))
|
||||||
Expect(normCfg.mode).To(Equal("core"))
|
|
||||||
Expect(normCfg.streamName).To(Equal("CUSTOM_STREAM"))
|
Expect(normCfg.streamName).To(Equal("CUSTOM_STREAM"))
|
||||||
Expect(normCfg.batchSize).To(Equal(100))
|
Expect(normCfg.batchSize).To(Equal(100))
|
||||||
Expect(normCfg.batchTimeout).To(Equal(5 * time.Second))
|
Expect(normCfg.batchTimeout).To(Equal(5 * time.Second))
|
||||||
|
|||||||
@@ -72,14 +72,7 @@ func ProvideNATSConn(cfg *config.Config, logger *zap.Logger) (*nats.Conn, error)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ProvideJetStream creates a JetStream context from a NATS connection.
|
// ProvideJetStream creates a JetStream context from a NATS connection.
|
||||||
// Returns nil when mode is "core" to indicate JetStream should not be used.
|
|
||||||
func ProvideJetStream(nc *nats.Conn, cfg *config.Config, logger *zap.Logger) (nats.JetStreamContext, error) {
|
func ProvideJetStream(nc *nats.Conn, cfg *config.Config, logger *zap.Logger) (nats.JetStreamContext, error) {
|
||||||
// In core mode, return nil so that publishers/consumers use core NATS
|
|
||||||
if cfg.NATS.Mode == "core" {
|
|
||||||
logger.Debug("Skipping JetStream initialization (core mode)")
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
js, err := nc.JetStream()
|
js, err := nc.JetStream()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Error("failed to create JetStream context", zap.Error(err))
|
logger.Error("failed to create JetStream context", zap.Error(err))
|
||||||
|
|||||||
@@ -17,7 +17,6 @@ var _ = Describe("MessageHandler", func() {
|
|||||||
logger: zaptest.NewLogger(GinkgoT()),
|
logger: zaptest.NewLogger(GinkgoT()),
|
||||||
streamName: "TEST_STREAM",
|
streamName: "TEST_STREAM",
|
||||||
consumerName: "test-consumer",
|
consumerName: "test-consumer",
|
||||||
mode: "jetstream",
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -34,20 +33,7 @@ var _ = Describe("MessageHandler", func() {
|
|||||||
Expect(source).To(Equal("header"))
|
Expect(source).To(Equal("header"))
|
||||||
})
|
})
|
||||||
|
|
||||||
It("generates UUID for core mode when header is missing", func() {
|
It("returns error when header and metadata are missing", func() {
|
||||||
processor.mode = "core"
|
|
||||||
msg := &nats.Msg{
|
|
||||||
Header: nats.Header{},
|
|
||||||
}
|
|
||||||
|
|
||||||
id, source, err := processor.resolveMsgID(msg)
|
|
||||||
Expect(err).NotTo(HaveOccurred())
|
|
||||||
Expect(id).NotTo(BeEmpty())
|
|
||||||
Expect(source).To(Equal("generated"))
|
|
||||||
})
|
|
||||||
|
|
||||||
It("returns error for JetStream mode when header and metadata are missing", func() {
|
|
||||||
processor.mode = "jetstream"
|
|
||||||
msg := &nats.Msg{
|
msg := &nats.Msg{
|
||||||
Header: nats.Header{},
|
Header: nats.Header{},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"go.opentelemetry.io/otel"
|
"go.opentelemetry.io/otel"
|
||||||
"go.opentelemetry.io/otel/attribute"
|
"go.opentelemetry.io/otel/attribute"
|
||||||
@@ -39,7 +38,6 @@ type defaultBatchProcessor struct {
|
|||||||
// Configuration needed for processing
|
// Configuration needed for processing
|
||||||
streamName string
|
streamName string
|
||||||
consumerName string
|
consumerName string
|
||||||
mode string
|
|
||||||
backoff []time.Duration
|
backoff []time.Duration
|
||||||
// Pointer to consecutive errors counter (shared with Consumer)
|
// Pointer to consecutive errors counter (shared with Consumer)
|
||||||
consecutiveProcessErrors *int
|
consecutiveProcessErrors *int
|
||||||
@@ -76,19 +74,12 @@ func (p *defaultBatchProcessor) processSingleMessage(ctx context.Context, msg *n
|
|||||||
|
|
||||||
elapsed := time.Since(start)
|
elapsed := time.Since(start)
|
||||||
|
|
||||||
// ACK the message (only in JetStream mode; Core NATS doesn't support ACK)
|
// ACK the message
|
||||||
if p.mode == "jetstream" {
|
if ackErr := msg.Ack(); ackErr != nil {
|
||||||
if ackErr := msg.Ack(); ackErr != nil {
|
p.logger.Error("Failed to ACK message", zap.Error(ackErr))
|
||||||
p.logger.Error("Failed to ACK message", zap.Error(ackErr))
|
// Still record metrics even if ACK fails
|
||||||
// Still record metrics even if ACK fails
|
|
||||||
p.telemetry.RecordMessageHandled(ctx, p.streamName, p.consumerName, "ok", elapsed)
|
|
||||||
} else {
|
|
||||||
p.telemetry.RecordMessageHandled(ctx, p.streamName, p.consumerName, "ok", elapsed)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Core NATS mode: record metrics without ACK (ACK not supported)
|
|
||||||
p.telemetry.RecordMessageHandled(ctx, p.streamName, p.consumerName, "ok", elapsed)
|
|
||||||
}
|
}
|
||||||
|
p.telemetry.RecordMessageHandled(ctx, p.streamName, p.consumerName, "ok", elapsed)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProcessMessage processes a single message.
|
// ProcessMessage processes a single message.
|
||||||
@@ -160,10 +151,6 @@ func (p *defaultBatchProcessor) resolveMsgID(msg *nats.Msg) (string, string, err
|
|||||||
return id, "header", nil
|
return id, "header", nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if p.mode == "core" {
|
|
||||||
return uuid.NewString(), "generated", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
meta, err := msg.Metadata()
|
meta, err := msg.Metadata()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", fmt.Errorf("fetch metadata: %w", err)
|
return "", "", fmt.Errorf("fetch metadata: %w", err)
|
||||||
@@ -232,13 +219,6 @@ func (p *defaultBatchProcessor) handlePermanentError(ctx context.Context, msg *n
|
|||||||
*p.consecutiveProcessErrors = 0
|
*p.consecutiveProcessErrors = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
if p.mode != "jetstream" {
|
|
||||||
p.logger.Debug("Permanent-error message in core mode; skipping DLQ/ACK (not supported)",
|
|
||||||
zap.String("subject", msg.Subject),
|
|
||||||
)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extract message ID for better logging
|
// Extract message ID for better logging
|
||||||
msgID, _, _ := p.resolveMsgID(msg)
|
msgID, _, _ := p.resolveMsgID(msg)
|
||||||
if msgID == "" {
|
if msgID == "" {
|
||||||
@@ -310,13 +290,6 @@ func (p *defaultBatchProcessor) handleTransientError(ctx context.Context, msg *n
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if p.mode != "jetstream" {
|
|
||||||
p.logger.Debug("Transient-error message in core mode; skipping retry (ACK/NAK unsupported)",
|
|
||||||
zap.String("subject", msg.Subject),
|
|
||||||
)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Transient error: request redelivery with optional delay
|
// Transient error: request redelivery with optional delay
|
||||||
p.telemetry.RecordRetry(ctx, p.streamName, p.consumerName, obsmetrics.RetryReasonProcessorError)
|
p.telemetry.RecordRetry(ctx, p.streamName, p.consumerName, obsmetrics.RetryReasonProcessorError)
|
||||||
if nakErr := p.nakWithStrategy(msg); nakErr != nil {
|
if nakErr := p.nakWithStrategy(msg); nakErr != nil {
|
||||||
|
|||||||
@@ -20,8 +20,7 @@ type Publisher struct {
|
|||||||
logger *zap.Logger
|
logger *zap.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProvidePublisher creates a NATS publisher.
|
// ProvidePublisher creates a NATS JetStream publisher.
|
||||||
// When js is nil (core mode), returns a CorePublisher that uses plain NATS.
|
|
||||||
func ProvidePublisher(
|
func ProvidePublisher(
|
||||||
js nats.JetStreamContext,
|
js nats.JetStreamContext,
|
||||||
nc *nats.Conn,
|
nc *nats.Conn,
|
||||||
@@ -29,7 +28,7 @@ func ProvidePublisher(
|
|||||||
logger *zap.Logger,
|
logger *zap.Logger,
|
||||||
) (port.Publisher, error) {
|
) (port.Publisher, error) {
|
||||||
if js == nil {
|
if js == nil {
|
||||||
return ProvideCorePublisher(nc, cfg, logger)
|
return nil, fmt.Errorf("JetStream context is required")
|
||||||
}
|
}
|
||||||
return &Publisher{
|
return &Publisher{
|
||||||
js: js,
|
js: js,
|
||||||
@@ -38,63 +37,6 @@ func ProvidePublisher(
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// CorePublisher publishes messages to plain NATS (non-JetStream)
|
|
||||||
type CorePublisher struct {
|
|
||||||
conn *nats.Conn
|
|
||||||
cfg *config.Config
|
|
||||||
logger *zap.Logger
|
|
||||||
}
|
|
||||||
|
|
||||||
// ProvideCorePublisher creates a NATS publisher for core mode
|
|
||||||
func ProvideCorePublisher(
|
|
||||||
conn *nats.Conn,
|
|
||||||
cfg *config.Config,
|
|
||||||
logger *zap.Logger,
|
|
||||||
) (port.Publisher, error) {
|
|
||||||
return &CorePublisher{
|
|
||||||
conn: conn,
|
|
||||||
cfg: cfg,
|
|
||||||
logger: logger,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Publish publishes a message using plain NATS
|
|
||||||
func (p *CorePublisher) Publish(message any) error {
|
|
||||||
topic := p.cfg.Publisher.Topic
|
|
||||||
if topic == "" {
|
|
||||||
p.logger.Error("publisher topic is not configured")
|
|
||||||
return fmt.Errorf("publisher topic is not configured")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Marshal message to JSON
|
|
||||||
messageBytes, err := json.Marshal(message)
|
|
||||||
if err != nil {
|
|
||||||
p.logger.Error("failed to marshal message",
|
|
||||||
zap.String("topic", topic),
|
|
||||||
zap.Error(err),
|
|
||||||
)
|
|
||||||
return fmt.Errorf("failed to marshal message: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Publish to plain NATS
|
|
||||||
err = p.conn.Publish(topic, messageBytes)
|
|
||||||
if err != nil {
|
|
||||||
p.logger.Error("failed to publish message",
|
|
||||||
zap.String("topic", topic),
|
|
||||||
zap.Int("message_size", len(messageBytes)),
|
|
||||||
zap.Error(err),
|
|
||||||
)
|
|
||||||
return fmt.Errorf("failed to publish message: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
p.logger.Debug("Published message",
|
|
||||||
zap.String("topic", topic),
|
|
||||||
zap.Int("size", len(messageBytes)),
|
|
||||||
)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Publish publishes a message
|
// Publish publishes a message
|
||||||
func (p *Publisher) Publish(message any) error {
|
func (p *Publisher) Publish(message any) error {
|
||||||
topic := p.cfg.Publisher.Topic
|
topic := p.cfg.Publisher.Topic
|
||||||
|
|||||||
Reference in New Issue
Block a user