✨ Add new dev-run-compose task to Taskfile for running the CAATSM receiver in a Docker Compose development stack. Update monitoring configuration in config.dev.toml to use console format. Modify Prometheus scrape configuration to target a specific IP address. Enhance NATS consumer error handling with retry logic for JetStream availability and adjust logging level for consumer metrics.
This commit is contained in:
+13
-3
@@ -147,6 +147,13 @@ tasks:
|
|||||||
- echo "Stopping dev infrastructure..."
|
- echo "Stopping dev infrastructure..."
|
||||||
- docker compose -f docker-compose.dev.yml down -v
|
- docker compose -f docker-compose.dev.yml down -v
|
||||||
|
|
||||||
|
dev-run-compose:
|
||||||
|
desc: Run receiver inside docker-compose dev stack
|
||||||
|
cmds:
|
||||||
|
- echo "Starting dev stack including caatsm-receiver..."
|
||||||
|
- docker compose -f docker-compose.dev.yml up -d postgres nats nats-box nats-exporter
|
||||||
|
- docker compose -f docker-compose.dev.yml up -d otel-collector jaeger prometheus grafana caatsm-receiver
|
||||||
|
|
||||||
dev-run:
|
dev-run:
|
||||||
desc: Run receiver locally against dev stack
|
desc: Run receiver locally against dev stack
|
||||||
deps:
|
deps:
|
||||||
@@ -158,6 +165,9 @@ tasks:
|
|||||||
CAATSM_TELEMETRY_ENABLED: "true"
|
CAATSM_TELEMETRY_ENABLED: "true"
|
||||||
CAATSM_TELEMETRY_ENDPOINT: localhost:4318
|
CAATSM_TELEMETRY_ENDPOINT: localhost:4318
|
||||||
CAATSM_TELEMETRY_INSECURE: "true"
|
CAATSM_TELEMETRY_INSECURE: "true"
|
||||||
|
CAATSM_MONITORING_ADDR: 0.0.0.0:2112
|
||||||
|
CAATSM_MONITORING_ENABLE_METRICS: "true"
|
||||||
|
CAATSM_MONITORING_ENABLE_HEALTH: "true"
|
||||||
GO_ENV: dev
|
GO_ENV: dev
|
||||||
cmds:
|
cmds:
|
||||||
- |
|
- |
|
||||||
@@ -174,9 +184,9 @@ tasks:
|
|||||||
GO_ENV=dev \
|
GO_ENV=dev \
|
||||||
NATS_URL=${CAATSM_NATS_URL:-nats://localhost:4222} \
|
NATS_URL=${CAATSM_NATS_URL:-nats://localhost:4222} \
|
||||||
SUBJECT=${CAATSM_NATS_SUBJECT:-telegram.serial} \
|
SUBJECT=${CAATSM_NATS_SUBJECT:-telegram.serial} \
|
||||||
COUNT=${COUNT:-10} \
|
COUNT={{.COUNT | default "10"}} \
|
||||||
CATEGORY=${CATEGORY:-mixed} \
|
CATEGORY={{.CATEGORY | default "mixed"}} \
|
||||||
STATUS=${STATUS:-random} \
|
STATUS={{.STATUS | default "random"}} \
|
||||||
bash -c '
|
bash -c '
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
cmd=(go run ./cmd/seed-telegrams)
|
cmd=(go run ./cmd/seed-telegrams)
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ monitor_interval = "30s"
|
|||||||
|
|
||||||
[log]
|
[log]
|
||||||
level = "info"
|
level = "info"
|
||||||
format = "json"
|
format = "console"
|
||||||
|
|
||||||
[telemetry]
|
[telemetry]
|
||||||
enabled = true
|
enabled = true
|
||||||
|
|||||||
@@ -15,7 +15,5 @@ scrape_configs:
|
|||||||
- job_name: "caatsm-receiver"
|
- job_name: "caatsm-receiver"
|
||||||
static_configs:
|
static_configs:
|
||||||
- targets:
|
- targets:
|
||||||
# When go-caatsm runs on the host/WSL and Prometheus runs in Docker,
|
- 172.23.189.12:2112
|
||||||
# use host.docker.internal to reach the host monitoring server.
|
|
||||||
- "host.docker.internal:2112"
|
|
||||||
|
|
||||||
|
|||||||
@@ -241,6 +241,17 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
|
|||||||
// Timeout is expected when no messages are available
|
// Timeout is expected when no messages are available
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if errors.Is(err, nats.ErrNoResponders) {
|
||||||
|
// JetStream API is currently unavailable (e.g., NATS just restarted or JetStream not ready).
|
||||||
|
// Back off a bit to avoid log spam while allowing the system to recover.
|
||||||
|
c.logger.Warn("JetStream not available, will retry",
|
||||||
|
zap.Error(err),
|
||||||
|
zap.String("stream", c.streamName),
|
||||||
|
zap.String("consumer", c.consumerName),
|
||||||
|
)
|
||||||
|
time.Sleep(5 * time.Second)
|
||||||
|
continue
|
||||||
|
}
|
||||||
c.logger.Error("Failed to fetch messages", zap.Error(err))
|
c.logger.Error("Failed to fetch messages", zap.Error(err))
|
||||||
time.Sleep(time.Second)
|
time.Sleep(time.Second)
|
||||||
continue
|
continue
|
||||||
@@ -377,7 +388,7 @@ func (c *Consumer) emitConsumerStats(ctx context.Context) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
c.logger.Info("JetStream consumer metrics",
|
c.logger.Debug("JetStream consumer metrics",
|
||||||
zap.String("stream", c.streamName),
|
zap.String("stream", c.streamName),
|
||||||
zap.String("consumer", c.consumerName),
|
zap.String("consumer", c.consumerName),
|
||||||
zap.Uint64("num_ack_pending", uint64(info.NumAckPending)),
|
zap.Uint64("num_ack_pending", uint64(info.NumAckPending)),
|
||||||
|
|||||||
@@ -18,6 +18,8 @@ func ProvideNATSConn(cfg *config.Config, logger *zap.Logger) (*nats.Conn, error)
|
|||||||
nats.RetryOnFailedConnect(true),
|
nats.RetryOnFailedConnect(true),
|
||||||
nats.Timeout(cfg.Timeouts.Server),
|
nats.Timeout(cfg.Timeouts.Server),
|
||||||
nats.ReconnectWait(cfg.Timeouts.ReconnectWait),
|
nats.ReconnectWait(cfg.Timeouts.ReconnectWait),
|
||||||
|
// Use infinite reconnects so the app survives long NATS outages (e.g. docker compose down/up).
|
||||||
|
nats.MaxReconnects(-1),
|
||||||
nats.DisconnectErrHandler(func(nc *nats.Conn, err error) {
|
nats.DisconnectErrHandler(func(nc *nats.Conn, err error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Warn("NATS disconnected", zap.Error(err))
|
logger.Warn("NATS disconnected", zap.Error(err))
|
||||||
|
|||||||
Reference in New Issue
Block a user