Implement dead-letter queue (DLQ) functionality for handling permanent failures in message processing. Update configuration to enable DLQ and specify the subject for routing failed messages. Enhance observability by adding metrics for message handling, retries, and database operations. Introduce structured logging for better traceability of message processing events.

This commit is contained in:
windyboy
2025-11-16 09:15:37 +08:00
parent c98baa0ff4
commit a96cb8a40b
12 changed files with 1053 additions and 31 deletions
+25 -14
View File
@@ -4,6 +4,7 @@ import (
"caatsm/internal/adapter"
"caatsm/internal/adapter/parser"
"caatsm/internal/domain"
obslogging "caatsm/internal/observability/logging"
obsmetrics "caatsm/internal/observability/metrics"
"context"
"fmt"
@@ -110,6 +111,20 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
}
}
// Enrich span with parsed telegram information as soon as we have it.
span.SetAttributes(
attribute.String("telegram.message_id", parsed.MessageID),
attribute.String("telegram.category", parsed.Category),
attribute.String("telegram.status", string(parsed.Status)),
)
msgLogger := obslogging.WithMessageContext(p.logger, obslogging.MessageFields{
Service: "caatsm-processor",
TransportMsgID: msgID,
BusinessMsgID: parsed.MessageID,
Category: parsed.Category,
})
if parseErr != nil || !parsed.Parsed {
if parsed.ErrorReason == "" && parseErr != nil {
parsed.ErrorReason = parseErr.Error()
@@ -117,12 +132,11 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
span.RecordError(parseErr)
span.SetStatus(codes.Error, parseErr.Error())
p.persistRaw(ctx, parsed)
p.logger.Warn("Message failed to parse",
zap.String("msg_id", msgID),
zap.String("status", string(parsed.Status)),
zap.String("content_preview", truncateContent(parsed.Content, 256)),
zap.Error(parseErr),
)
msgLogger.With(zap.String("status", string(parsed.Status))).
Warn("Message failed to parse",
zap.String("content_preview", truncateContent(parsed.Content, 256)),
zap.Error(parseErr),
)
latency := parsed.ParsedAt.Sub(receivedAt)
parseLatencyHistogram.Record(ctx, float64(latency.Milliseconds()),
metric.WithAttributes(
@@ -143,10 +157,7 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
attribute.String("telegram.category", parsed.Category),
)
p.logger.Info("Message parsed successfully",
zap.String("msg_id", msgID),
zap.String("message_id", parsed.MessageID),
zap.String("category", parsed.Category),
msgLogger.Info("Message parsed successfully",
zap.Time("received_at", parsed.ReceivedAt),
zap.Time("parsed_at", parsed.ParsedAt),
)
@@ -172,10 +183,10 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
_, pubSpan := tracer.Start(ctx, "Publisher.Publish")
if err := p.publisher.Publish(parsed); err != nil {
// Log error but don't fail the entire operation
p.logger.Error("Failed to publish message",
zap.String("msg_id", msgID),
zap.Error(err),
)
msgLogger.With(zap.String("status", string(parsed.Status))).
Error("Failed to publish message",
zap.Error(err),
)
pubSpan.RecordError(err)
pubSpan.SetStatus(codes.Error, err.Error())
parsed.Status = domain.MessageStatusPublishFail
+9
View File
@@ -21,6 +21,7 @@ type Config struct {
Publisher PublisherConfig `koanf:"publisher"`
Telemetry TelemetryConfig `koanf:"telemetry"`
Monitoring MonitoringConfig `koanf:"monitoring"`
DLQ DLQConfig `koanf:"dlq"`
// Legacy fields for backward compatibility during migration
Subscription SubscriptionConfig `koanf:"subscription"`
Timeouts TimeoutsConfig `koanf:"timeouts"`
@@ -93,6 +94,14 @@ type TelemetryConfig struct {
Insecure bool `koanf:"insecure"`
}
// DLQConfig defines the dead-letter queue routing for poison/permanent messages.
// If Enabled is true and Subject is non-empty, permanent failures will be published
// to the configured subject for offline processing.
type DLQConfig struct {
Enabled bool `koanf:"enabled"`
Subject string `koanf:"subject"`
}
// MonitoringConfig controls the lightweight HTTP server that exposes health and metrics endpoints.
type MonitoringConfig struct {
Disabled bool `koanf:"disabled"`
+5
View File
@@ -47,7 +47,12 @@ func ProvideServer(
routes := 0
if cfg.Monitoring.EnableHealth {
// Liveness: basic process check. For now this reuses the same implementation
// as readiness but can diverge in the future if we need a cheaper liveness probe.
mux.HandleFunc("/healthz", server.handleHealth)
// Readiness: alias to the same implementation so consumers can adopt /readyz
// without breaking existing /healthz users.
mux.HandleFunc("/readyz", server.handleHealth)
routes++
}
if cfg.Monitoring.EnableMetrics {
+124 -7
View File
@@ -3,7 +3,10 @@ package nats
import (
"caatsm/internal/app"
"caatsm/internal/infra/config"
obslogging "caatsm/internal/observability/logging"
obsmetrics "caatsm/internal/observability/metrics"
"context"
"encoding/json"
"errors"
"fmt"
"strings"
@@ -29,6 +32,7 @@ type Consumer struct {
consumerName string
mode string
streamName string
dlqSubject string
ackWait time.Duration
batchSize int
batchTimeout time.Duration
@@ -38,6 +42,9 @@ type Consumer struct {
redelivered metric.Int64Histogram
pending metric.Int64Histogram
delivered metric.Int64Histogram
// simple backpressure / degradation state
consecutiveProcessErrors int
}
// ProvideConsumer creates a NATS consumer
@@ -65,6 +72,8 @@ func ProvideConsumer(
streamName = "TELEGRAM"
}
dlqSubject := strings.TrimSpace(cfg.DLQ.Subject)
ackWait := cfg.NATS.ConsumerRules.AckWait
if ackWait == 0 {
ackWait = cfg.Timeouts.AckWait
@@ -98,6 +107,7 @@ func ProvideConsumer(
consumerName: consumerName,
mode: mode,
streamName: streamName,
dlqSubject: dlqSubject,
ackWait: ackWait,
batchSize: batchSize,
batchTimeout: batchTimeout,
@@ -227,31 +237,74 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
// Process each message
// TODO: consider buffering messages to take advantage of Repository.InsertBatch for higher throughput.
for _, msg := range msgs {
start := time.Now()
if err := c.processMessage(ctx, msg); err != nil {
isPermanent := app.IsPermanent(err)
elapsed := time.Since(start)
c.logger.Error("Failed to process message",
zap.String("subject", msg.Subject),
zap.Error(err),
zap.Bool("permanent", isPermanent),
)
result := obsmetrics.ResultFail
if isPermanent {
result = obsmetrics.ResultPermanentFail
}
obsmetrics.RecordMessageHandled(c.streamName, c.consumerName, result, elapsed)
if isPermanent {
c.consecutiveProcessErrors = 0
// Poison/permanent message: route to DLQ if configured, then ACK
if err := c.routeToDLQ(ctx, msg, err); err != nil {
c.logger.Error("Failed to route permanent-error message to DLQ", zap.Error(err))
}
if ackErr := msg.Ack(); ackErr != nil {
c.logger.Error("Failed to ACK permanent-error message", zap.Error(ackErr))
}
continue
}
// Transient error: increment error streak and apply simple backpressure if needed.
if c.consecutiveProcessErrors < 0 {
c.consecutiveProcessErrors = 0
}
c.consecutiveProcessErrors++
if c.consecutiveProcessErrors >= 10 {
// Apply a brief sleep to slow down consumption when the system
// is failing many messages in a row (e.g. DB unavailable).
backoff := time.Duration(c.consecutiveProcessErrors) * 100 * time.Millisecond
if backoff > 5*time.Second {
backoff = 5 * time.Second
}
c.logger.Warn("Applying backpressure due to consecutive processing errors",
zap.Int("consecutive_errors", c.consecutiveProcessErrors),
zap.Duration("sleep", backoff),
)
time.Sleep(backoff)
}
// Transient error: request redelivery with optional delay
obsmetrics.RecordRetry(c.streamName, c.consumerName, obsmetrics.RetryReasonProcessorError)
if nakErr := c.nakWithStrategy(msg); nakErr != nil {
c.logger.Error("Failed to NAK message", zap.Error(nakErr))
}
continue
}
// Successful processing resets the error streak.
if c.consecutiveProcessErrors > 0 {
c.consecutiveProcessErrors = 0
}
// ACK the message
if ackErr := msg.Ack(); ackErr != nil {
c.logger.Error("Failed to ACK message", zap.Error(ackErr))
} else {
elapsed := time.Since(start)
obsmetrics.RecordMessageHandled(c.streamName, c.consumerName, "ok", elapsed)
}
}
}
@@ -359,16 +412,16 @@ func (c *Consumer) initMetrics() {
meter := otel.Meter("caatsm/nats")
c.meter = meter
if hist, err := meter.Int64Histogram("nats.consumer.ack_pending"); err == nil {
if hist, err := meter.Int64Histogram("caatsm_nats_consumer_ack_pending"); err == nil {
c.ackPending = hist
}
if hist, err := meter.Int64Histogram("nats.consumer.redelivered"); err == nil {
if hist, err := meter.Int64Histogram("caatsm_nats_consumer_redelivered"); err == nil {
c.redelivered = hist
}
if hist, err := meter.Int64Histogram("nats.consumer.pending"); err == nil {
if hist, err := meter.Int64Histogram("caatsm_nats_consumer_pending"); err == nil {
c.pending = hist
}
if hist, err := meter.Int64Histogram("nats.consumer.delivered"); err == nil {
if hist, err := meter.Int64Histogram("caatsm_nats_consumer_delivered"); err == nil {
c.delivered = hist
}
}
@@ -391,6 +444,52 @@ func (c *Consumer) recordConsumerMetrics(ctx context.Context, info *nats.Consume
}
}
// routeToDLQ publishes a copy of the failed message to the configured DLQ subject,
// including useful metadata for offline analysis. If DLQ is not configured or the
// consumer is not running in JetStream mode, this is a no-op.
func (c *Consumer) routeToDLQ(ctx context.Context, msg *nats.Msg, cause error) error {
if c == nil || c.js == nil {
return nil
}
if c.mode != "jetstream" {
return nil
}
if strings.TrimSpace(c.dlqSubject) == "" {
return nil
}
meta, _ := msg.Metadata()
jsSeq := uint64(0)
deliveries := uint64(0)
if meta != nil {
jsSeq = meta.Sequence.Stream
deliveries = meta.NumDelivered
}
payload := map[string]interface{}{
"transport_msg_id": msg.Header.Get("Nats-Msg-Id"),
"subject": msg.Subject,
"stream": c.streamName,
"consumer": c.consumerName,
"nats_sequence": jsSeq,
"deliveries": deliveries,
"error": fmt.Sprint(cause),
"received_at": time.Now().UTC(),
"body": string(msg.Data),
}
data, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("marshal dlq payload: %w", err)
}
if _, err := c.js.Publish(c.dlqSubject, data); err != nil {
return fmt.Errorf("publish to dlq subject %s: %w", c.dlqSubject, err)
}
return nil
}
func (c *Consumer) nakWithStrategy(msg *nats.Msg) error {
backoff := c.cfg.NATS.ConsumerRules.Backoff
if len(backoff) == 0 {
@@ -439,10 +538,28 @@ func (c *Consumer) processMessage(ctx context.Context, msg *nats.Msg) error {
)
}
c.logger.Debug("Processing message",
zap.String("subject", msg.Subject),
zap.String("msg_id", msgID),
// Attach structured logging context including stream/consumer and NATS metadata.
jsSeq := uint64(0)
if meta, metaErr := msg.Metadata(); metaErr == nil {
jsSeq = meta.Sequence.Stream
span.SetAttributes(
attribute.Int64("nats.js.stream_seq", int64(meta.Sequence.Stream)),
attribute.Int64("nats.js.consumer_seq", int64(meta.Sequence.Consumer)),
)
}
msgLogger := obslogging.WithMessageContext(c.logger, obslogging.MessageFields{
Service: "caatsm-consumer",
TransportMsgID: msgID,
Stream: c.streamName,
Consumer: c.consumerName,
Subject: msg.Subject,
JSSequence: jsSeq,
})
msgLogger.Debug("Processing message",
zap.Int("data_size", len(msg.Data)),
zap.String("msg_id_source", source),
)
// Call processor
+70
View File
@@ -4,6 +4,7 @@ import (
"caatsm/internal/adapter"
"caatsm/internal/adapter/mapper"
"caatsm/internal/domain"
obsmetrics "caatsm/internal/observability/metrics"
"context"
"encoding/json"
"fmt"
@@ -40,6 +41,25 @@ func (r *Repository) InsertOne(ctx context.Context, msg *domain.ParsedMessage) e
defer span.End()
span.SetAttributes(attribute.String("db.table", "aviation.telegrams"))
// Optional idempotency check based on business message identity. If we have a
// non-empty message ID and date/time, we can cheaply skip duplicates here to
// avoid applying the same business event multiple times.
if msg != nil && msg.MessageID != "" && msg.DateTime != "" {
exists, err := r.messageExists(ctx, msg.MessageID, msg.DateTime)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("failed to check existing message: %w", err)
}
if exists {
r.logger.Info("Duplicate message detected by message_id/date_time; skipping insert",
zap.String("message_id", msg.MessageID),
zap.String("date_time", msg.DateTime),
)
return nil
}
}
row, err := r.mapper.ToDBRow(msg)
if err != nil {
span.RecordError(err)
@@ -59,16 +79,24 @@ func (r *Repository) InsertOne(ctx context.Context, msg *domain.ParsedMessage) e
ON CONFLICT (uuid, received_at) DO NOTHING
`
start := time.Now()
tag, err := r.pool.Exec(ctx, query,
row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8],
row[9], row[10], row[11], row[12], row[13], row[14], row[15], row[16],
)
elapsed := time.Since(start)
result := obsmetrics.DBResultOK
if err != nil {
result = obsmetrics.DBResultError
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
obsmetrics.RecordDBQuery("insert_one", result, elapsed)
return fmt.Errorf("failed to insert message: %w", err)
}
obsmetrics.RecordDBQuery("insert_one", result, elapsed)
if tag.RowsAffected() == 0 {
r.logger.Info("Duplicate message skipped",
zap.String("uuid", msg.Uuid),
@@ -106,6 +134,7 @@ func (r *Repository) InsertBatch(ctx context.Context, msgs []*domain.ParsedMessa
}
// Use CopyFrom for efficient batch insert
start := time.Now()
copyCount, err := r.pool.CopyFrom(
ctx,
pgx.Identifier{"aviation", "telegrams"},
@@ -117,12 +146,19 @@ func (r *Repository) InsertBatch(ctx context.Context, msgs []*domain.ParsedMessa
},
pgx.CopyFromRows(rows),
)
elapsed := time.Since(start)
result := obsmetrics.DBResultOK
if err != nil {
result = obsmetrics.DBResultError
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
obsmetrics.RecordDBQuery("insert_batch", result, elapsed)
return fmt.Errorf("failed to batch insert messages: %w", err)
}
obsmetrics.RecordDBQuery("insert_batch", result, elapsed)
span.SetAttributes(attribute.Int64("db.inserted", copyCount))
r.logger.Info("Batch inserted messages",
zap.Int("count", int(copyCount)),
@@ -174,6 +210,7 @@ func (r *Repository) InsertRaw(ctx context.Context, msg *domain.ParsedMessage) e
metadata = EXCLUDED.metadata
`
start := time.Now()
_, err = r.pool.Exec(ctx, query,
msg.Uuid,
string(msg.Status),
@@ -182,12 +219,19 @@ func (r *Repository) InsertRaw(ctx context.Context, msg *domain.ParsedMessage) e
msg.ReceivedAt,
metadataJSON,
)
elapsed := time.Since(start)
result := obsmetrics.DBResultOK
if err != nil {
result = obsmetrics.DBResultError
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
obsmetrics.RecordDBQuery("insert_raw", result, elapsed)
return fmt.Errorf("failed to insert raw telegram: %w", err)
}
obsmetrics.RecordDBQuery("insert_raw", result, elapsed)
span.SetAttributes(attribute.String("telegram.uuid", msg.Uuid), attribute.String("telegram.status", string(msg.Status)))
r.logger.Debug("Persisted raw telegram",
zap.String("uuid", msg.Uuid),
@@ -196,3 +240,29 @@ func (r *Repository) InsertRaw(ctx context.Context, msg *domain.ParsedMessage) e
return nil
}
// messageExists performs a lightweight existence check for a business message,
// using (message_id, date_time) as the idempotency key. This avoids requiring a
// unique constraint at the database level while still preventing duplicate effects.
func (r *Repository) messageExists(ctx context.Context, messageID, dateTime string) (bool, error) {
if messageID == "" || dateTime == "" {
return false, nil
}
const query = `
SELECT 1
FROM aviation.telegrams
WHERE message_id = $1 AND date_time = $2
LIMIT 1
`
var one int
if err := r.pool.QueryRow(ctx, query, messageID, dateTime).Scan(&one); err != nil {
if err == pgx.ErrNoRows {
return false, nil
}
return false, err
}
return true, nil
}
+102
View File
@@ -0,0 +1,102 @@
package logging
import "go.uber.org/zap"
// ErrorType represents a coarse-grained categorisation of errors for logging and alerting.
// Typical values:
// - business: validation failures, domain rule violations, payload issues.
// - transient: network / DB / NATS glitches that may succeed on retry.
// - fatal: programming bugs, schema mismatches, or conditions that require operator action.
type ErrorType string
const (
ErrorTypeBusiness ErrorType = "business"
ErrorTypeTransient ErrorType = "transient"
ErrorTypeFatal ErrorType = "fatal"
)
// Canonical logging field names for structured logs produced by the CAATSM
// receiver. Using constants avoids scattering magic strings and keeps log
// analysis queries stable over time.
const (
FieldService = "service"
FieldTransportMsgID = "transport_msg_id"
FieldTelegramMsgID = "telegram_message_id"
FieldCategory = "category"
FieldStream = "stream"
FieldConsumer = "consumer"
FieldSubject = "subject"
FieldNATSSequence = "nats_sequence"
FieldRequestID = "request_id"
FieldTraceID = "trace_id"
FieldErrorType = "error_type"
)
// MessageFields captures the common structured logging contract for message-processing logs.
// All fields are optional; empty values will simply be skipped.
type MessageFields struct {
// Identifiers
Service string
TransportMsgID string
BusinessMsgID string
Category string
// NATS / JetStream context
Stream string
Consumer string
Subject string
JSSequence uint64
// Correlation / tracing
RequestID string
TraceID string
// Error classification
ErrorType ErrorType
}
// WithMessageContext returns a logger pre-populated with the structured fields defined in MessageFields.
// This is the primary entry point for enforcing the logging contract in the codebase.
func WithMessageContext(logger *zap.Logger, mf MessageFields) *zap.Logger {
if logger == nil {
return zap.NewNop()
}
fields := make([]zap.Field, 0, 12)
if mf.Service != "" {
fields = append(fields, zap.String(FieldService, mf.Service))
}
if mf.TransportMsgID != "" {
fields = append(fields, zap.String(FieldTransportMsgID, mf.TransportMsgID))
}
if mf.BusinessMsgID != "" {
fields = append(fields, zap.String(FieldTelegramMsgID, mf.BusinessMsgID))
}
if mf.Category != "" {
fields = append(fields, zap.String(FieldCategory, mf.Category))
}
if mf.Stream != "" {
fields = append(fields, zap.String(FieldStream, mf.Stream))
}
if mf.Consumer != "" {
fields = append(fields, zap.String(FieldConsumer, mf.Consumer))
}
if mf.Subject != "" {
fields = append(fields, zap.String(FieldSubject, mf.Subject))
}
if mf.JSSequence > 0 {
fields = append(fields, zap.Uint64(FieldNATSSequence, mf.JSSequence))
}
if mf.RequestID != "" {
fields = append(fields, zap.String(FieldRequestID, mf.RequestID))
}
if mf.TraceID != "" {
fields = append(fields, zap.String(FieldTraceID, mf.TraceID))
}
if mf.ErrorType != "" {
fields = append(fields, zap.String(FieldErrorType, string(mf.ErrorType)))
}
return logger.With(fields...)
}
+144 -9
View File
@@ -11,31 +11,130 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
)
// Metric and label key/value contracts for the CAATSM receiver. Centralising these
// names avoids scattering magic strings across the codebase and keeps PromQL and
// documentation aligned with the implementation.
const (
// Metric names.
MetricProcessedTotal = "caatsm_processed_total"
MetricFailuresTotal = "caatsm_failures_total"
MetricParseLatencySeconds = "caatsm_parse_latency_seconds"
MetricMessagesTotal = "caatsm_messages_total"
MetricHandleLatencySeconds = "caatsm_handle_latency_seconds"
MetricRetriesTotal = "caatsm_retries_total"
MetricJSAPICallsTotal = "caatsm_js_api_calls_total"
MetricDBQueriesTotal = "caatsm_db_queries_total"
MetricDBQueryLatencySeconds = "caatsm_db_query_latency_seconds"
// Common label keys.
LabelStatus = "status"
LabelCategory = "category"
LabelStage = "stage"
LabelStream = "stream"
LabelConsumer = "consumer"
LabelResult = "result"
LabelReason = "reason"
LabelOperation = "operation"
// Standard result label values for caatsm_messages_total.
ResultOK = "ok"
ResultFail = "fail"
ResultPermanentFail = "permanent_fail"
// Standard result values for DB operations.
DBResultOK = "ok"
DBResultError = "error"
// Standard retry reasons.
RetryReasonProcessorError = "processor_error"
)
var (
once sync.Once
registry *prometheus.Registry
once sync.Once
registry *prometheus.Registry
// Legacy metrics (kept for backward compatibility).
processedCounter *prometheus.CounterVec
failureCounter *prometheus.CounterVec
parseLatency *prometheus.HistogramVec
// Message handling metrics (per stream / consumer).
messagesTotal *prometheus.CounterVec
handleLatency *prometheus.HistogramVec
retriesTotal *prometheus.CounterVec
jsAPICallsTotal *prometheus.CounterVec
// Database metrics.
dbQueriesTotal *prometheus.CounterVec
dbQueryLatency *prometheus.HistogramVec
)
func initCollectors() {
registry = prometheus.NewRegistry()
// Legacy metrics.
processedCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "caatsm_processed_total",
Name: MetricProcessedTotal,
Help: "Count of telegrams processed by status and category.",
}, []string{"status", "category"})
}, []string{LabelStatus, LabelCategory})
failureCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "caatsm_failures_total",
Name: MetricFailuresTotal,
Help: "Count of processor failures by stage (parser, repository, publisher).",
}, []string{"stage"})
}, []string{LabelStage})
parseLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "caatsm_parse_latency_seconds",
Name: MetricParseLatencySeconds,
Help: "Latency between reception and parse completion.",
Buckets: prometheus.DefBuckets,
}, []string{"status", "category"})
}, []string{LabelStatus, LabelCategory})
registry.MustRegister(processedCounter, failureCounter, parseLatency)
// New message handling metrics.
messagesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: MetricMessagesTotal,
Help: "Total number of messages handled by the receiver, labelled by stream, consumer and result.",
}, []string{LabelStream, LabelConsumer, LabelResult})
handleLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: MetricHandleLatencySeconds,
Help: "Latency of end-to-end message handling in seconds, from NATS receive to handler completion.",
Buckets: prometheus.DefBuckets,
}, []string{LabelStream, LabelConsumer})
retriesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: MetricRetriesTotal,
Help: "Total number of message retries (negative acknowledgements), labelled by stream, consumer and reason.",
}, []string{LabelStream, LabelConsumer, LabelReason})
jsAPICallsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: MetricJSAPICallsTotal,
Help: "Count of JetStream API calls made by the receiver.",
}, []string{LabelOperation})
// Database metrics.
dbQueriesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: MetricDBQueriesTotal,
Help: "Total number of database operations, labelled by operation and result.",
}, []string{LabelOperation, LabelResult})
dbQueryLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: MetricDBQueryLatencySeconds,
Help: "Latency of database operations in seconds, labelled by operation.",
Buckets: prometheus.DefBuckets,
}, []string{LabelOperation})
registry.MustRegister(
processedCounter,
failureCounter,
parseLatency,
messagesTotal,
handleLatency,
retriesTotal,
jsAPICallsTotal,
dbQueriesTotal,
dbQueryLatency,
)
}
func ensureCollectors() {
@@ -62,6 +161,42 @@ func RecordFailure(stage string) {
failureCounter.WithLabelValues(labelValue(stage)).Inc()
}
// RecordMessageHandled records end-to-end message handling metrics (per stream / consumer).
// Result is expected to be values such as "ok", "fail", or "retry".
func RecordMessageHandled(stream, consumer, result string, elapsed time.Duration) {
ensureCollectors()
if elapsed < 0 {
elapsed = 0
}
messagesTotal.WithLabelValues(labelValue(stream), labelValue(consumer), labelValue(result)).Inc()
handleLatency.WithLabelValues(labelValue(stream), labelValue(consumer)).Observe(elapsed.Seconds())
}
// RecordRetry increments the retry counter for a message that is being negatively acknowledged.
// Reason can capture the high-level cause, e.g. "processor_error" or "nats_timeout".
func RecordRetry(stream, consumer, reason string) {
ensureCollectors()
retriesTotal.WithLabelValues(labelValue(stream), labelValue(consumer), labelValue(reason)).Inc()
}
// RecordDBQuery records metrics for a single database operation.
// Operation examples: "insert_one", "insert_batch", "insert_raw".
// Result is usually "ok" or "error".
func RecordDBQuery(operation, result string, elapsed time.Duration) {
ensureCollectors()
if elapsed < 0 {
elapsed = 0
}
dbQueriesTotal.WithLabelValues(labelValue(operation), labelValue(result)).Inc()
dbQueryLatency.WithLabelValues(labelValue(operation)).Observe(elapsed.Seconds())
}
// RecordJSAPICall increments the JetStream API call counter for the given operation.
func RecordJSAPICall(operation string) {
ensureCollectors()
jsAPICallsTotal.WithLabelValues(labelValue(operation)).Inc()
}
func labelValue(value string) string {
value = strings.TrimSpace(value)
if value == "" {