✨ Implement dead-letter queue (DLQ) functionality for handling permanent failures in message processing. Update configuration to enable DLQ and specify the subject for routing failed messages. Enhance observability by adding metrics for message handling, retries, and database operations. Introduce structured logging for better traceability of message processing events.
This commit is contained in:
windyboy
@@ -0,0 +1,102 @@
|
||||
package logging
|
||||
|
||||
import "go.uber.org/zap"
|
||||
|
||||
// ErrorType represents a coarse-grained categorisation of errors for logging and alerting.
|
||||
// Typical values:
|
||||
// - business: validation failures, domain rule violations, payload issues.
|
||||
// - transient: network / DB / NATS glitches that may succeed on retry.
|
||||
// - fatal: programming bugs, schema mismatches, or conditions that require operator action.
|
||||
type ErrorType string
|
||||
|
||||
const (
|
||||
ErrorTypeBusiness ErrorType = "business"
|
||||
ErrorTypeTransient ErrorType = "transient"
|
||||
ErrorTypeFatal ErrorType = "fatal"
|
||||
)
|
||||
|
||||
// Canonical logging field names for structured logs produced by the CAATSM
|
||||
// receiver. Using constants avoids scattering magic strings and keeps log
|
||||
// analysis queries stable over time.
|
||||
const (
|
||||
FieldService = "service"
|
||||
FieldTransportMsgID = "transport_msg_id"
|
||||
FieldTelegramMsgID = "telegram_message_id"
|
||||
FieldCategory = "category"
|
||||
FieldStream = "stream"
|
||||
FieldConsumer = "consumer"
|
||||
FieldSubject = "subject"
|
||||
FieldNATSSequence = "nats_sequence"
|
||||
FieldRequestID = "request_id"
|
||||
FieldTraceID = "trace_id"
|
||||
FieldErrorType = "error_type"
|
||||
)
|
||||
|
||||
// MessageFields captures the common structured logging contract for message-processing logs.
|
||||
// All fields are optional; empty values will simply be skipped.
|
||||
type MessageFields struct {
|
||||
// Identifiers
|
||||
Service string
|
||||
TransportMsgID string
|
||||
BusinessMsgID string
|
||||
Category string
|
||||
|
||||
// NATS / JetStream context
|
||||
Stream string
|
||||
Consumer string
|
||||
Subject string
|
||||
JSSequence uint64
|
||||
|
||||
// Correlation / tracing
|
||||
RequestID string
|
||||
TraceID string
|
||||
|
||||
// Error classification
|
||||
ErrorType ErrorType
|
||||
}
|
||||
|
||||
// WithMessageContext returns a logger pre-populated with the structured fields defined in MessageFields.
|
||||
// This is the primary entry point for enforcing the logging contract in the codebase.
|
||||
func WithMessageContext(logger *zap.Logger, mf MessageFields) *zap.Logger {
|
||||
if logger == nil {
|
||||
return zap.NewNop()
|
||||
}
|
||||
|
||||
fields := make([]zap.Field, 0, 12)
|
||||
|
||||
if mf.Service != "" {
|
||||
fields = append(fields, zap.String(FieldService, mf.Service))
|
||||
}
|
||||
if mf.TransportMsgID != "" {
|
||||
fields = append(fields, zap.String(FieldTransportMsgID, mf.TransportMsgID))
|
||||
}
|
||||
if mf.BusinessMsgID != "" {
|
||||
fields = append(fields, zap.String(FieldTelegramMsgID, mf.BusinessMsgID))
|
||||
}
|
||||
if mf.Category != "" {
|
||||
fields = append(fields, zap.String(FieldCategory, mf.Category))
|
||||
}
|
||||
if mf.Stream != "" {
|
||||
fields = append(fields, zap.String(FieldStream, mf.Stream))
|
||||
}
|
||||
if mf.Consumer != "" {
|
||||
fields = append(fields, zap.String(FieldConsumer, mf.Consumer))
|
||||
}
|
||||
if mf.Subject != "" {
|
||||
fields = append(fields, zap.String(FieldSubject, mf.Subject))
|
||||
}
|
||||
if mf.JSSequence > 0 {
|
||||
fields = append(fields, zap.Uint64(FieldNATSSequence, mf.JSSequence))
|
||||
}
|
||||
if mf.RequestID != "" {
|
||||
fields = append(fields, zap.String(FieldRequestID, mf.RequestID))
|
||||
}
|
||||
if mf.TraceID != "" {
|
||||
fields = append(fields, zap.String(FieldTraceID, mf.TraceID))
|
||||
}
|
||||
if mf.ErrorType != "" {
|
||||
fields = append(fields, zap.String(FieldErrorType, string(mf.ErrorType)))
|
||||
}
|
||||
|
||||
return logger.With(fields...)
|
||||
}
|
||||
@@ -11,31 +11,130 @@ import (
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
// Metric and label key/value contracts for the CAATSM receiver. Centralising these
|
||||
// names avoids scattering magic strings across the codebase and keeps PromQL and
|
||||
// documentation aligned with the implementation.
|
||||
const (
|
||||
// Metric names.
|
||||
MetricProcessedTotal = "caatsm_processed_total"
|
||||
MetricFailuresTotal = "caatsm_failures_total"
|
||||
MetricParseLatencySeconds = "caatsm_parse_latency_seconds"
|
||||
MetricMessagesTotal = "caatsm_messages_total"
|
||||
MetricHandleLatencySeconds = "caatsm_handle_latency_seconds"
|
||||
MetricRetriesTotal = "caatsm_retries_total"
|
||||
MetricJSAPICallsTotal = "caatsm_js_api_calls_total"
|
||||
MetricDBQueriesTotal = "caatsm_db_queries_total"
|
||||
MetricDBQueryLatencySeconds = "caatsm_db_query_latency_seconds"
|
||||
|
||||
// Common label keys.
|
||||
LabelStatus = "status"
|
||||
LabelCategory = "category"
|
||||
LabelStage = "stage"
|
||||
LabelStream = "stream"
|
||||
LabelConsumer = "consumer"
|
||||
LabelResult = "result"
|
||||
LabelReason = "reason"
|
||||
LabelOperation = "operation"
|
||||
|
||||
// Standard result label values for caatsm_messages_total.
|
||||
ResultOK = "ok"
|
||||
ResultFail = "fail"
|
||||
ResultPermanentFail = "permanent_fail"
|
||||
|
||||
// Standard result values for DB operations.
|
||||
DBResultOK = "ok"
|
||||
DBResultError = "error"
|
||||
|
||||
// Standard retry reasons.
|
||||
RetryReasonProcessorError = "processor_error"
|
||||
)
|
||||
|
||||
var (
|
||||
once sync.Once
|
||||
registry *prometheus.Registry
|
||||
once sync.Once
|
||||
|
||||
registry *prometheus.Registry
|
||||
|
||||
// Legacy metrics (kept for backward compatibility).
|
||||
processedCounter *prometheus.CounterVec
|
||||
failureCounter *prometheus.CounterVec
|
||||
parseLatency *prometheus.HistogramVec
|
||||
|
||||
// Message handling metrics (per stream / consumer).
|
||||
messagesTotal *prometheus.CounterVec
|
||||
handleLatency *prometheus.HistogramVec
|
||||
retriesTotal *prometheus.CounterVec
|
||||
jsAPICallsTotal *prometheus.CounterVec
|
||||
|
||||
// Database metrics.
|
||||
dbQueriesTotal *prometheus.CounterVec
|
||||
dbQueryLatency *prometheus.HistogramVec
|
||||
)
|
||||
|
||||
func initCollectors() {
|
||||
registry = prometheus.NewRegistry()
|
||||
|
||||
// Legacy metrics.
|
||||
processedCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "caatsm_processed_total",
|
||||
Name: MetricProcessedTotal,
|
||||
Help: "Count of telegrams processed by status and category.",
|
||||
}, []string{"status", "category"})
|
||||
}, []string{LabelStatus, LabelCategory})
|
||||
|
||||
failureCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "caatsm_failures_total",
|
||||
Name: MetricFailuresTotal,
|
||||
Help: "Count of processor failures by stage (parser, repository, publisher).",
|
||||
}, []string{"stage"})
|
||||
}, []string{LabelStage})
|
||||
|
||||
parseLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "caatsm_parse_latency_seconds",
|
||||
Name: MetricParseLatencySeconds,
|
||||
Help: "Latency between reception and parse completion.",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
}, []string{"status", "category"})
|
||||
}, []string{LabelStatus, LabelCategory})
|
||||
|
||||
registry.MustRegister(processedCounter, failureCounter, parseLatency)
|
||||
// New message handling metrics.
|
||||
messagesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: MetricMessagesTotal,
|
||||
Help: "Total number of messages handled by the receiver, labelled by stream, consumer and result.",
|
||||
}, []string{LabelStream, LabelConsumer, LabelResult})
|
||||
|
||||
handleLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: MetricHandleLatencySeconds,
|
||||
Help: "Latency of end-to-end message handling in seconds, from NATS receive to handler completion.",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
}, []string{LabelStream, LabelConsumer})
|
||||
|
||||
retriesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: MetricRetriesTotal,
|
||||
Help: "Total number of message retries (negative acknowledgements), labelled by stream, consumer and reason.",
|
||||
}, []string{LabelStream, LabelConsumer, LabelReason})
|
||||
|
||||
jsAPICallsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: MetricJSAPICallsTotal,
|
||||
Help: "Count of JetStream API calls made by the receiver.",
|
||||
}, []string{LabelOperation})
|
||||
|
||||
// Database metrics.
|
||||
dbQueriesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: MetricDBQueriesTotal,
|
||||
Help: "Total number of database operations, labelled by operation and result.",
|
||||
}, []string{LabelOperation, LabelResult})
|
||||
|
||||
dbQueryLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: MetricDBQueryLatencySeconds,
|
||||
Help: "Latency of database operations in seconds, labelled by operation.",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
}, []string{LabelOperation})
|
||||
|
||||
registry.MustRegister(
|
||||
processedCounter,
|
||||
failureCounter,
|
||||
parseLatency,
|
||||
messagesTotal,
|
||||
handleLatency,
|
||||
retriesTotal,
|
||||
jsAPICallsTotal,
|
||||
dbQueriesTotal,
|
||||
dbQueryLatency,
|
||||
)
|
||||
}
|
||||
|
||||
func ensureCollectors() {
|
||||
@@ -62,6 +161,42 @@ func RecordFailure(stage string) {
|
||||
failureCounter.WithLabelValues(labelValue(stage)).Inc()
|
||||
}
|
||||
|
||||
// RecordMessageHandled records end-to-end message handling metrics (per stream / consumer).
|
||||
// Result is expected to be values such as "ok", "fail", or "retry".
|
||||
func RecordMessageHandled(stream, consumer, result string, elapsed time.Duration) {
|
||||
ensureCollectors()
|
||||
if elapsed < 0 {
|
||||
elapsed = 0
|
||||
}
|
||||
messagesTotal.WithLabelValues(labelValue(stream), labelValue(consumer), labelValue(result)).Inc()
|
||||
handleLatency.WithLabelValues(labelValue(stream), labelValue(consumer)).Observe(elapsed.Seconds())
|
||||
}
|
||||
|
||||
// RecordRetry increments the retry counter for a message that is being negatively acknowledged.
|
||||
// Reason can capture the high-level cause, e.g. "processor_error" or "nats_timeout".
|
||||
func RecordRetry(stream, consumer, reason string) {
|
||||
ensureCollectors()
|
||||
retriesTotal.WithLabelValues(labelValue(stream), labelValue(consumer), labelValue(reason)).Inc()
|
||||
}
|
||||
|
||||
// RecordDBQuery records metrics for a single database operation.
|
||||
// Operation examples: "insert_one", "insert_batch", "insert_raw".
|
||||
// Result is usually "ok" or "error".
|
||||
func RecordDBQuery(operation, result string, elapsed time.Duration) {
|
||||
ensureCollectors()
|
||||
if elapsed < 0 {
|
||||
elapsed = 0
|
||||
}
|
||||
dbQueriesTotal.WithLabelValues(labelValue(operation), labelValue(result)).Inc()
|
||||
dbQueryLatency.WithLabelValues(labelValue(operation)).Observe(elapsed.Seconds())
|
||||
}
|
||||
|
||||
// RecordJSAPICall increments the JetStream API call counter for the given operation.
|
||||
func RecordJSAPICall(operation string) {
|
||||
ensureCollectors()
|
||||
jsAPICallsTotal.WithLabelValues(labelValue(operation)).Inc()
|
||||
}
|
||||
|
||||
func labelValue(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
|
||||
Reference in New Issue
Block a user