Add new dev-run-compose task to Taskfile for running the CAATSM receiver in a Docker Compose development stack. Update monitoring configuration in config.dev.toml to use console format. Modify Prometheus scrape configuration to target a specific IP address. Enhance NATS consumer error handling with retry logic for JetStream availability and adjust logging level for consumer metrics.

This commit is contained in:
windyboy
2025-11-16 11:38:49 +08:00
parent 7e7b8ca412
commit 6616c7e10d
5 changed files with 29 additions and 8 deletions
+13 -3
View File
@@ -147,6 +147,13 @@ tasks:
- echo "Stopping dev infrastructure..." - echo "Stopping dev infrastructure..."
- docker compose -f docker-compose.dev.yml down -v - docker compose -f docker-compose.dev.yml down -v
dev-run-compose:
desc: Run receiver inside docker-compose dev stack
cmds:
- echo "Starting dev stack including caatsm-receiver..."
- docker compose -f docker-compose.dev.yml up -d postgres nats nats-box nats-exporter
- docker compose -f docker-compose.dev.yml up -d otel-collector jaeger prometheus grafana caatsm-receiver
dev-run: dev-run:
desc: Run receiver locally against dev stack desc: Run receiver locally against dev stack
deps: deps:
@@ -158,6 +165,9 @@ tasks:
CAATSM_TELEMETRY_ENABLED: "true" CAATSM_TELEMETRY_ENABLED: "true"
CAATSM_TELEMETRY_ENDPOINT: localhost:4318 CAATSM_TELEMETRY_ENDPOINT: localhost:4318
CAATSM_TELEMETRY_INSECURE: "true" CAATSM_TELEMETRY_INSECURE: "true"
CAATSM_MONITORING_ADDR: 0.0.0.0:2112
CAATSM_MONITORING_ENABLE_METRICS: "true"
CAATSM_MONITORING_ENABLE_HEALTH: "true"
GO_ENV: dev GO_ENV: dev
cmds: cmds:
- | - |
@@ -174,9 +184,9 @@ tasks:
GO_ENV=dev \ GO_ENV=dev \
NATS_URL=${CAATSM_NATS_URL:-nats://localhost:4222} \ NATS_URL=${CAATSM_NATS_URL:-nats://localhost:4222} \
SUBJECT=${CAATSM_NATS_SUBJECT:-telegram.serial} \ SUBJECT=${CAATSM_NATS_SUBJECT:-telegram.serial} \
COUNT=${COUNT:-10} \ COUNT={{.COUNT | default "10"}} \
CATEGORY=${CATEGORY:-mixed} \ CATEGORY={{.CATEGORY | default "mixed"}} \
STATUS=${STATUS:-random} \ STATUS={{.STATUS | default "random"}} \
bash -c ' bash -c '
set -euo pipefail set -euo pipefail
cmd=(go run ./cmd/seed-telegrams) cmd=(go run ./cmd/seed-telegrams)
+1 -1
View File
@@ -49,7 +49,7 @@ monitor_interval = "30s"
[log] [log]
level = "info" level = "info"
format = "json" format = "console"
[telemetry] [telemetry]
enabled = true enabled = true
+1 -3
View File
@@ -15,7 +15,5 @@ scrape_configs:
- job_name: "caatsm-receiver" - job_name: "caatsm-receiver"
static_configs: static_configs:
- targets: - targets:
# When go-caatsm runs on the host/WSL and Prometheus runs in Docker, - 172.23.189.12:2112
# use host.docker.internal to reach the host monitoring server.
- "host.docker.internal:2112"
+12 -1
View File
@@ -241,6 +241,17 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
// Timeout is expected when no messages are available // Timeout is expected when no messages are available
continue continue
} }
if errors.Is(err, nats.ErrNoResponders) {
// JetStream API is currently unavailable (e.g., NATS just restarted or JetStream not ready).
// Back off a bit to avoid log spam while allowing the system to recover.
c.logger.Warn("JetStream not available, will retry",
zap.Error(err),
zap.String("stream", c.streamName),
zap.String("consumer", c.consumerName),
)
time.Sleep(5 * time.Second)
continue
}
c.logger.Error("Failed to fetch messages", zap.Error(err)) c.logger.Error("Failed to fetch messages", zap.Error(err))
time.Sleep(time.Second) time.Sleep(time.Second)
continue continue
@@ -377,7 +388,7 @@ func (c *Consumer) emitConsumerStats(ctx context.Context) {
continue continue
} }
c.logger.Info("JetStream consumer metrics", c.logger.Debug("JetStream consumer metrics",
zap.String("stream", c.streamName), zap.String("stream", c.streamName),
zap.String("consumer", c.consumerName), zap.String("consumer", c.consumerName),
zap.Uint64("num_ack_pending", uint64(info.NumAckPending)), zap.Uint64("num_ack_pending", uint64(info.NumAckPending)),
+2
View File
@@ -18,6 +18,8 @@ func ProvideNATSConn(cfg *config.Config, logger *zap.Logger) (*nats.Conn, error)
nats.RetryOnFailedConnect(true), nats.RetryOnFailedConnect(true),
nats.Timeout(cfg.Timeouts.Server), nats.Timeout(cfg.Timeouts.Server),
nats.ReconnectWait(cfg.Timeouts.ReconnectWait), nats.ReconnectWait(cfg.Timeouts.ReconnectWait),
// Use infinite reconnects so the app survives long NATS outages (e.g. docker compose down/up).
nats.MaxReconnects(-1),
nats.DisconnectErrHandler(func(nc *nats.Conn, err error) { nats.DisconnectErrHandler(func(nc *nats.Conn, err error) {
if err != nil { if err != nil {
logger.Warn("NATS disconnected", zap.Error(err)) logger.Warn("NATS disconnected", zap.Error(err))