✨ Revise observability features by introducing new health and liveness endpoints (/livez and /readyz), updating Prometheus metrics to track NATS consumer pending messages, and enhancing telemetry integration for better monitoring. Update documentation to reflect these changes and ensure consistency in metric naming conventions.
This commit is contained in:
+12
-80
@@ -5,7 +5,7 @@ import (
|
||||
"caatsm/internal/adapter/parser"
|
||||
"caatsm/internal/model"
|
||||
obslogging "caatsm/internal/observability/logging"
|
||||
obsmetrics "caatsm/internal/observability/metrics"
|
||||
"caatsm/internal/observability/telemetry"
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
@@ -25,6 +24,7 @@ type MessageProcessor struct {
|
||||
repository adapter.Repository
|
||||
publisher adapter.Publisher
|
||||
logger *zap.Logger
|
||||
telemetry telemetry.Recorder
|
||||
}
|
||||
|
||||
// ProcessingStatus represents the outcome of the processing pipeline
|
||||
@@ -38,37 +38,12 @@ const (
|
||||
ProcessingStatusPublishFailed ProcessingStatus = "publish_failed"
|
||||
)
|
||||
|
||||
var (
|
||||
appMeter = otel.Meter("caatsm/app")
|
||||
messageStatusAttrKey = attribute.Key("message.status")
|
||||
messageCategoryAttrKey = attribute.Key("message.category")
|
||||
|
||||
messageProcessedCounter = mustInt64Counter("caatsm_messages_processed_total", "Total number of telegrams processed by the CAATSM processor.")
|
||||
messagePublishFailCounter = mustInt64Counter("caatsm_publish_failures_total", "Total number of telegram publish failures.")
|
||||
parseLatencyHistogram = mustFloat64Histogram("caatsm_parse_duration_ms", "Latency of parsing a telegram, in milliseconds.", metric.WithUnit("ms"))
|
||||
)
|
||||
|
||||
func mustInt64Counter(name, description string, opts ...metric.Int64CounterOption) metric.Int64Counter {
|
||||
counter, err := appMeter.Int64Counter(name, append([]metric.Int64CounterOption{metric.WithDescription(description)}, opts...)...)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("failed to create counter %s: %v", name, err))
|
||||
}
|
||||
return counter
|
||||
}
|
||||
|
||||
func mustFloat64Histogram(name, description string, opts ...metric.Float64HistogramOption) metric.Float64Histogram {
|
||||
hist, err := appMeter.Float64Histogram(name, append([]metric.Float64HistogramOption{metric.WithDescription(description)}, opts...)...)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("failed to create histogram %s: %v", name, err))
|
||||
}
|
||||
return hist
|
||||
}
|
||||
|
||||
// NewMessageProcessor creates a new message processor
|
||||
func NewMessageProcessor(
|
||||
parser parser.Parser,
|
||||
repository adapter.Repository,
|
||||
publisher adapter.Publisher,
|
||||
rec telemetry.Recorder,
|
||||
logger *zap.Logger,
|
||||
) *MessageProcessor {
|
||||
return &MessageProcessor{
|
||||
@@ -76,6 +51,7 @@ func NewMessageProcessor(
|
||||
repository: repository,
|
||||
publisher: publisher,
|
||||
logger: logger,
|
||||
telemetry: rec,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,14 +125,8 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
|
||||
zap.Error(parseErr),
|
||||
)
|
||||
latency := parsed.ParsedAt.Sub(receivedAt)
|
||||
parseLatencyHistogram.Record(ctx, float64(latency.Milliseconds()),
|
||||
metric.WithAttributes(
|
||||
messageStatusAttrKey.String(string(parsed.Status)),
|
||||
messageCategoryAttrKey.String(parsed.Category),
|
||||
),
|
||||
)
|
||||
obsmetrics.RecordFailure("parser")
|
||||
recordProcessedMetric(ctx, parsed, latency)
|
||||
p.telemetry.RecordFailure("parser")
|
||||
p.telemetry.RecordProcessingResult(ctx, string(parsed.Status), parsed.Category, latency)
|
||||
return Permanent(fmt.Errorf("parser error: %w", parseErr))
|
||||
}
|
||||
parsed.ErrorReason = ""
|
||||
@@ -178,14 +148,8 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
latency := parsed.ParsedAt.Sub(receivedAt)
|
||||
parseLatencyHistogram.Record(ctx, float64(latency.Milliseconds()),
|
||||
metric.WithAttributes(
|
||||
messageStatusAttrKey.String(string(parsed.Status)),
|
||||
messageCategoryAttrKey.String(parsed.Category),
|
||||
),
|
||||
)
|
||||
obsmetrics.RecordFailure("repository")
|
||||
recordProcessedMetric(ctx, parsed, latency)
|
||||
p.telemetry.RecordFailure("repository")
|
||||
p.telemetry.RecordProcessingResult(ctx, string(parsed.Status), parsed.Category, latency)
|
||||
return fmt.Errorf("failed to insert message: %w", err)
|
||||
}
|
||||
|
||||
@@ -200,20 +164,10 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
|
||||
pubSpan.RecordError(err)
|
||||
pubSpan.SetStatus(codes.Error, err.Error())
|
||||
parsed.ErrorReason = err.Error()
|
||||
messagePublishFailCounter.Add(ctx, 1,
|
||||
metric.WithAttributes(
|
||||
messageCategoryAttrKey.String(parsed.Category),
|
||||
),
|
||||
)
|
||||
p.telemetry.RecordPublishFailure(ctx, parsed.Category)
|
||||
latency := parsed.ParsedAt.Sub(receivedAt)
|
||||
parseLatencyHistogram.Record(ctx, float64(latency.Milliseconds()),
|
||||
metric.WithAttributes(
|
||||
messageStatusAttrKey.String(string(parsed.Status)),
|
||||
messageCategoryAttrKey.String(parsed.Category),
|
||||
),
|
||||
)
|
||||
obsmetrics.RecordFailure("publisher")
|
||||
recordProcessedMetric(ctx, parsed, latency)
|
||||
p.telemetry.RecordFailure("publisher")
|
||||
p.telemetry.RecordProcessingResult(ctx, string(parsed.Status), parsed.Category, latency)
|
||||
p.persistRaw(ctx, parsed)
|
||||
// Mark as permanent so the consumer will ack instead of retrying
|
||||
pubSpan.End()
|
||||
@@ -222,13 +176,7 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
|
||||
pubSpan.End()
|
||||
|
||||
latency := parsed.ParsedAt.Sub(receivedAt)
|
||||
parseLatencyHistogram.Record(ctx, float64(latency.Milliseconds()),
|
||||
metric.WithAttributes(
|
||||
messageStatusAttrKey.String(string(parsed.Status)),
|
||||
messageCategoryAttrKey.String(parsed.Category),
|
||||
),
|
||||
)
|
||||
recordProcessedMetric(ctx, parsed, latency)
|
||||
p.telemetry.RecordProcessingResult(ctx, string(parsed.Status), parsed.Category, latency)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -269,19 +217,3 @@ func truncateContent(content string, limit int) string {
|
||||
}
|
||||
return content[:limit-3] + "..."
|
||||
}
|
||||
|
||||
func recordProcessedMetric(ctx context.Context, msg *model.ParsedTelegram, elapsed time.Duration) {
|
||||
if msg == nil {
|
||||
return
|
||||
}
|
||||
messageProcessedCounter.Add(ctx, 1,
|
||||
metric.WithAttributes(
|
||||
messageStatusAttrKey.String(string(msg.Status)),
|
||||
messageCategoryAttrKey.String(msg.Category),
|
||||
),
|
||||
)
|
||||
if elapsed < 0 {
|
||||
elapsed = 0
|
||||
}
|
||||
obsmetrics.RecordProcessed(string(msg.Status), msg.Category, elapsed)
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"caatsm/internal/adapter"
|
||||
"caatsm/internal/adapter/parser"
|
||||
"caatsm/internal/model"
|
||||
"caatsm/internal/observability/telemetry"
|
||||
|
||||
"github.com/google/uuid"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
@@ -121,7 +122,7 @@ var _ = Describe("MessageProcessor", func() {
|
||||
},
|
||||
err: errors.New("parse failure"),
|
||||
}
|
||||
proc = NewMessageProcessor(parserStub, repo, pub, logger)
|
||||
proc = NewMessageProcessor(parserStub, repo, pub, telemetry.NewNoop(), logger)
|
||||
|
||||
err := proc.Handle(ctx, []byte("raw"), "msg-6")
|
||||
Expect(err).To(HaveOccurred())
|
||||
@@ -145,7 +146,7 @@ var _ = Describe("MessageProcessor", func() {
|
||||
})
|
||||
|
||||
func newTestProcessor(p parser.Parser, repo adapter.Repository, pub adapter.Publisher) *MessageProcessor {
|
||||
return NewMessageProcessor(p, repo, pub, zap.NewNop())
|
||||
return NewMessageProcessor(p, repo, pub, telemetry.NewNoop(), zap.NewNop())
|
||||
}
|
||||
|
||||
type stubParser struct {
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package buildinfo
|
||||
|
||||
// Version, Commit, and BuiltAt are populated via -ldflags at build time. They
|
||||
// default to development-friendly values when not provided.
|
||||
//
|
||||
// Example:
|
||||
// go build -ldflags "\
|
||||
// -X 'caatsm/internal/infra/buildinfo.Version=v0.4.3' \
|
||||
// -X 'caatsm/internal/infra/buildinfo.Commit=abc1234' \
|
||||
// -X 'caatsm/internal/infra/buildinfo.BuiltAt=2025-11-16T08:35:00Z' \
|
||||
// "
|
||||
|
||||
var (
|
||||
Version = "dev"
|
||||
Commit = "unknown"
|
||||
BuiltAt = ""
|
||||
)
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"caatsm/internal/infra/buildinfo"
|
||||
"caatsm/internal/infra/config"
|
||||
obsmetrics "caatsm/internal/observability/metrics"
|
||||
"context"
|
||||
@@ -47,11 +48,13 @@ func ProvideServer(
|
||||
|
||||
routes := 0
|
||||
if cfg.Monitoring.EnableHealth {
|
||||
// Liveness: basic process check. For now this reuses the same implementation
|
||||
// as readiness but can diverge in the future if we need a cheaper liveness probe.
|
||||
// Liveness: cheap process check that does not hit external dependencies.
|
||||
mux.HandleFunc("/livez", server.handleLive)
|
||||
// Backward-compatible health endpoint. For now this keeps the same
|
||||
// semantics as readiness but will remain stable for existing users.
|
||||
mux.HandleFunc("/healthz", server.handleHealth)
|
||||
// Readiness: alias to the same implementation so consumers can adopt /readyz
|
||||
// without breaking existing /healthz users.
|
||||
// Readiness: dependency-aware check intended for load balancers and
|
||||
// orchestrators.
|
||||
mux.HandleFunc("/readyz", server.handleHealth)
|
||||
routes++
|
||||
}
|
||||
@@ -111,33 +114,52 @@ func (s *Server) Shutdown(ctx context.Context) error {
|
||||
|
||||
func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
|
||||
status := http.StatusOK
|
||||
result := map[string]interface{}{
|
||||
"postgres": "ok",
|
||||
"nats": "ok",
|
||||
deps := map[string]map[string]interface{}{
|
||||
"postgres": {
|
||||
"status": "ok",
|
||||
},
|
||||
"nats": {
|
||||
"status": "ok",
|
||||
},
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), s.healthTimeout())
|
||||
defer cancel()
|
||||
|
||||
if s.pool == nil {
|
||||
result["postgres"] = "unconfigured"
|
||||
status = http.StatusServiceUnavailable
|
||||
} else if err := s.pool.Ping(ctx); err != nil {
|
||||
result["postgres"] = err.Error()
|
||||
deps["postgres"]["status"] = "unconfigured"
|
||||
status = http.StatusServiceUnavailable
|
||||
} else {
|
||||
start := time.Now()
|
||||
if err := s.pool.Ping(ctx); err != nil {
|
||||
deps["postgres"]["status"] = err.Error()
|
||||
status = http.StatusServiceUnavailable
|
||||
} else {
|
||||
deps["postgres"]["latency_ms"] = time.Since(start).Milliseconds()
|
||||
}
|
||||
}
|
||||
|
||||
if s.conn == nil {
|
||||
result["nats"] = "unconfigured"
|
||||
deps["nats"]["status"] = "unconfigured"
|
||||
status = http.StatusServiceUnavailable
|
||||
} else if s.conn.Status() != nats.CONNECTED {
|
||||
result["nats"] = s.conn.Status().String()
|
||||
deps["nats"]["status"] = s.conn.Status().String()
|
||||
status = http.StatusServiceUnavailable
|
||||
}
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"status": httpStatusLabel(status),
|
||||
"build": map[string]interface{}{
|
||||
"version": buildinfo.Version,
|
||||
"rev": buildinfo.Commit,
|
||||
"built_at": buildinfo.BuiltAt,
|
||||
},
|
||||
"dependencies": deps,
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(result)
|
||||
_ = json.NewEncoder(w).Encode(payload)
|
||||
}
|
||||
|
||||
func (s *Server) healthTimeout() time.Duration {
|
||||
@@ -147,3 +169,28 @@ func (s *Server) healthTimeout() time.Duration {
|
||||
}
|
||||
return timeout
|
||||
}
|
||||
|
||||
// handleLive reports basic process liveness and build information without
|
||||
// consulting external dependencies. It is suitable for liveness probes.
|
||||
func (s *Server) handleLive(w http.ResponseWriter, r *http.Request) {
|
||||
payload := map[string]interface{}{
|
||||
"status": "ok",
|
||||
"build": map[string]interface{}{
|
||||
"version": buildinfo.Version,
|
||||
"rev": buildinfo.Commit,
|
||||
"built_at": buildinfo.BuiltAt,
|
||||
},
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_ = json.NewEncoder(w).Encode(payload)
|
||||
}
|
||||
|
||||
func httpStatusLabel(code int) string {
|
||||
if code >= 200 && code < 300 {
|
||||
return "ok"
|
||||
}
|
||||
return "error"
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"caatsm/internal/infra/config"
|
||||
obslogging "caatsm/internal/observability/logging"
|
||||
obsmetrics "caatsm/internal/observability/metrics"
|
||||
"caatsm/internal/observability/telemetry"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -28,6 +29,7 @@ type Consumer struct {
|
||||
processor *app.MessageProcessor
|
||||
cfg *config.Config
|
||||
logger *zap.Logger
|
||||
telemetry telemetry.Recorder
|
||||
subject string
|
||||
consumerName string
|
||||
mode string
|
||||
@@ -53,6 +55,7 @@ func ProvideConsumer(
|
||||
js nats.JetStreamContext,
|
||||
processor *app.MessageProcessor,
|
||||
cfg *config.Config,
|
||||
rec telemetry.Recorder,
|
||||
logger *zap.Logger,
|
||||
) (*Consumer, error) {
|
||||
subject := cfg.EffectiveSubscriptionTopic()
|
||||
@@ -108,6 +111,7 @@ func ProvideConsumer(
|
||||
processor: processor,
|
||||
cfg: cfg,
|
||||
logger: logger,
|
||||
telemetry: rec,
|
||||
subject: subject,
|
||||
consumerName: consumerName,
|
||||
mode: mode,
|
||||
@@ -238,7 +242,7 @@ func (c *Consumer) validateDLQ() {
|
||||
|
||||
// Ensure the DLQ subject is actually bound to a JetStream stream. This avoids
|
||||
// the opaque `nats: no response from stream` error later when publishing.
|
||||
obsmetrics.RecordJSAPICall("dlq_validate_stream")
|
||||
c.telemetry.RecordJSAPICall("dlq_validate_stream")
|
||||
streamName, err := c.js.StreamNameBySubject(subject)
|
||||
if err != nil || strings.TrimSpace(streamName) == "" {
|
||||
c.logger.Warn("DLQ subject not bound to any JetStream stream; DLQ routing disabled",
|
||||
@@ -342,7 +346,7 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
|
||||
if isPermanent {
|
||||
result = obsmetrics.ResultPermanentFail
|
||||
}
|
||||
obsmetrics.RecordMessageHandled(c.streamName, c.consumerName, result, elapsed)
|
||||
c.telemetry.RecordMessageHandled(ctx, c.streamName, c.consumerName, result, elapsed)
|
||||
|
||||
if isPermanent {
|
||||
c.consecutiveProcessErrors = 0
|
||||
@@ -376,7 +380,7 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// Transient error: request redelivery with optional delay
|
||||
obsmetrics.RecordRetry(c.streamName, c.consumerName, obsmetrics.RetryReasonProcessorError)
|
||||
c.telemetry.RecordRetry(ctx, c.streamName, c.consumerName, obsmetrics.RetryReasonProcessorError)
|
||||
if nakErr := c.nakWithStrategy(msg); nakErr != nil {
|
||||
c.logger.Error("Failed to NAK message", zap.Error(nakErr))
|
||||
}
|
||||
@@ -393,7 +397,7 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
|
||||
c.logger.Error("Failed to ACK message", zap.Error(ackErr))
|
||||
} else {
|
||||
elapsed := time.Since(start)
|
||||
obsmetrics.RecordMessageHandled(c.streamName, c.consumerName, "ok", elapsed)
|
||||
c.telemetry.RecordMessageHandled(ctx, c.streamName, c.consumerName, "ok", elapsed)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -531,6 +535,10 @@ func (c *Consumer) recordConsumerMetrics(ctx context.Context, info *nats.Consume
|
||||
if c.delivered != nil {
|
||||
c.delivered.Record(ctx, int64(info.Delivered.Stream))
|
||||
}
|
||||
|
||||
// Export an explicit pending messages gauge for Prometheus-based lag /
|
||||
// backlog alerts.
|
||||
obsmetrics.RecordNATSConsumerPending(c.streamName, c.consumerName, info.NumPending)
|
||||
}
|
||||
|
||||
// routeToDLQ publishes a copy of the failed message to the configured DLQ subject,
|
||||
@@ -578,14 +586,14 @@ func (c *Consumer) routeToDLQ(ctx context.Context, msg *nats.Msg, cause error) e
|
||||
// unavailable. Surface this explicitly to make operational diagnosis
|
||||
// easier.
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
obsmetrics.RecordDLQPublishFailure(c.streamName, c.consumerName)
|
||||
c.telemetry.RecordDLQPublishFailure(ctx, c.streamName, c.consumerName)
|
||||
return fmt.Errorf("publish to dlq subject %s: no JetStream stream found for subject or JetStream unavailable: %w", c.dlqSubject, err)
|
||||
}
|
||||
obsmetrics.RecordDLQPublishFailure(c.streamName, c.consumerName)
|
||||
c.telemetry.RecordDLQPublishFailure(ctx, c.streamName, c.consumerName)
|
||||
return fmt.Errorf("publish to dlq subject %s: %w", c.dlqSubject, err)
|
||||
}
|
||||
|
||||
obsmetrics.RecordDLQMessage(c.streamName, c.consumerName)
|
||||
c.telemetry.RecordDLQMessage(ctx, c.streamName, c.consumerName)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ const (
|
||||
MetricDBQueryLatencySeconds = "caatsm_db_query_latency_seconds"
|
||||
MetricDLQMessagesTotal = "caatsm_dlq_messages_total"
|
||||
MetricDLQPublishFailures = "caatsm_dlq_publish_failures_total"
|
||||
MetricNATSConsumerPending = "caatsm_nats_consumer_pending_messages"
|
||||
|
||||
// Common label keys.
|
||||
LabelStatus = "status"
|
||||
@@ -72,6 +73,9 @@ var (
|
||||
// Database metrics.
|
||||
dbQueriesTotal *prometheus.CounterVec
|
||||
dbQueryLatency *prometheus.HistogramVec
|
||||
|
||||
// NATS consumer lag metrics.
|
||||
natsConsumerPending *prometheus.GaugeVec
|
||||
)
|
||||
|
||||
func initCollectors() {
|
||||
@@ -138,6 +142,11 @@ func initCollectors() {
|
||||
Buckets: prometheus.DefBuckets,
|
||||
}, []string{LabelOperation})
|
||||
|
||||
natsConsumerPending = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Name: MetricNATSConsumerPending,
|
||||
Help: "Approximate number of pending messages for a JetStream consumer, labelled by stream and consumer.",
|
||||
}, []string{LabelStream, LabelConsumer})
|
||||
|
||||
registry.MustRegister(
|
||||
processedCounter,
|
||||
failureCounter,
|
||||
@@ -150,6 +159,7 @@ func initCollectors() {
|
||||
dlqPublishFailures,
|
||||
dbQueriesTotal,
|
||||
dbQueryLatency,
|
||||
natsConsumerPending,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -226,6 +236,13 @@ func RecordJSAPICall(operation string) {
|
||||
jsAPICallsTotal.WithLabelValues(labelValue(operation)).Inc()
|
||||
}
|
||||
|
||||
// RecordNATSConsumerPending records the current pending message count for a
|
||||
// JetStream consumer as a gauge, enabling backlog / lag alerts.
|
||||
func RecordNATSConsumerPending(stream, consumer string, pending uint64) {
|
||||
ensureCollectors()
|
||||
natsConsumerPending.WithLabelValues(labelValue(stream), labelValue(consumer)).Set(float64(pending))
|
||||
}
|
||||
|
||||
func labelValue(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"caatsm/internal/infra/config"
|
||||
obsmetrics "caatsm/internal/observability/metrics"
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
)
|
||||
|
||||
// Recorder provides a thin abstraction over telemetry backends (OpenTelemetry,
|
||||
// Prometheus, etc.) so that application code does not need to import concrete
|
||||
// metric libraries directly.
|
||||
type Recorder interface {
|
||||
// RecordProcessingResult captures the final processing status of a telegram
|
||||
// along with the parser latency.
|
||||
RecordProcessingResult(ctx context.Context, status, category string, parseLatency time.Duration)
|
||||
|
||||
// RecordPublishFailure increments the publish failure counter for the given
|
||||
// category.
|
||||
RecordPublishFailure(ctx context.Context, category string)
|
||||
|
||||
// RecordFailure records a high-level failure bucket (parser, repository,
|
||||
// publisher, etc.).
|
||||
RecordFailure(stage string)
|
||||
|
||||
// RecordMessageHandled tracks end-to-end message handling for a particular
|
||||
// stream/consumer pair.
|
||||
RecordMessageHandled(ctx context.Context, stream, consumer, result string, elapsed time.Duration)
|
||||
|
||||
// RecordRetry records a retry (negative acknowledgement) reason.
|
||||
RecordRetry(ctx context.Context, stream, consumer, reason string)
|
||||
|
||||
// RecordDLQMessage records a successfully routed DLQ message.
|
||||
RecordDLQMessage(ctx context.Context, stream, consumer string)
|
||||
|
||||
// RecordDLQPublishFailure records a DLQ publish failure.
|
||||
RecordDLQPublishFailure(ctx context.Context, stream, consumer string)
|
||||
|
||||
// RecordJSAPICall records a JetStream API call.
|
||||
RecordJSAPICall(operation string)
|
||||
}
|
||||
|
||||
// ProvideRecorder wires a composite Recorder based on configuration flags.
|
||||
// - When telemetry is enabled, an OpenTelemetry-backed recorder is included.
|
||||
// - When metrics are enabled, a Prometheus-backed recorder is included.
|
||||
// - When neither is enabled, a noop recorder is returned.
|
||||
func ProvideRecorder(cfg *config.Config) Recorder {
|
||||
if cfg == nil {
|
||||
return NewNoop()
|
||||
}
|
||||
|
||||
var recorders []Recorder
|
||||
|
||||
if cfg.Telemetry.Enabled {
|
||||
recorders = append(recorders, newOTelRecorder())
|
||||
}
|
||||
|
||||
if !cfg.Monitoring.Disabled && cfg.Monitoring.EnableMetrics {
|
||||
recorders = append(recorders, newPromRecorder())
|
||||
}
|
||||
|
||||
if len(recorders) == 0 {
|
||||
return NewNoop()
|
||||
}
|
||||
return NewComposite(recorders...)
|
||||
}
|
||||
|
||||
// noopRecorder implements Recorder but performs no operations.
|
||||
type noopRecorder struct{}
|
||||
|
||||
func NewNoop() Recorder {
|
||||
return &noopRecorder{}
|
||||
}
|
||||
|
||||
func (n *noopRecorder) RecordProcessingResult(ctx context.Context, status, category string, parseLatency time.Duration) {
|
||||
}
|
||||
|
||||
func (n *noopRecorder) RecordPublishFailure(ctx context.Context, category string) {
|
||||
}
|
||||
|
||||
func (n *noopRecorder) RecordFailure(stage string) {
|
||||
}
|
||||
|
||||
func (n *noopRecorder) RecordMessageHandled(ctx context.Context, stream, consumer, result string, elapsed time.Duration) {
|
||||
}
|
||||
|
||||
func (n *noopRecorder) RecordRetry(ctx context.Context, stream, consumer, reason string) {
|
||||
}
|
||||
|
||||
func (n *noopRecorder) RecordDLQMessage(ctx context.Context, stream, consumer string) {
|
||||
}
|
||||
|
||||
func (n *noopRecorder) RecordDLQPublishFailure(ctx context.Context, stream, consumer string) {
|
||||
}
|
||||
|
||||
func (n *noopRecorder) RecordJSAPICall(operation string) {
|
||||
}
|
||||
|
||||
// compositeRecorder fans out all calls to a slice of underlying recorders.
|
||||
type compositeRecorder struct {
|
||||
recorders []Recorder
|
||||
}
|
||||
|
||||
func NewComposite(recorders ...Recorder) Recorder {
|
||||
// Filter out nils defensively.
|
||||
var filtered []Recorder
|
||||
for _, r := range recorders {
|
||||
if r != nil {
|
||||
filtered = append(filtered, r)
|
||||
}
|
||||
}
|
||||
if len(filtered) == 0 {
|
||||
return NewNoop()
|
||||
}
|
||||
return &compositeRecorder{recorders: filtered}
|
||||
}
|
||||
|
||||
func (c *compositeRecorder) RecordProcessingResult(ctx context.Context, status, category string, parseLatency time.Duration) {
|
||||
for _, r := range c.recorders {
|
||||
r.RecordProcessingResult(ctx, status, category, parseLatency)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *compositeRecorder) RecordPublishFailure(ctx context.Context, category string) {
|
||||
for _, r := range c.recorders {
|
||||
r.RecordPublishFailure(ctx, category)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *compositeRecorder) RecordFailure(stage string) {
|
||||
for _, r := range c.recorders {
|
||||
r.RecordFailure(stage)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *compositeRecorder) RecordMessageHandled(ctx context.Context, stream, consumer, result string, elapsed time.Duration) {
|
||||
for _, r := range c.recorders {
|
||||
r.RecordMessageHandled(ctx, stream, consumer, result, elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *compositeRecorder) RecordRetry(ctx context.Context, stream, consumer, reason string) {
|
||||
for _, r := range c.recorders {
|
||||
r.RecordRetry(ctx, stream, consumer, reason)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *compositeRecorder) RecordDLQMessage(ctx context.Context, stream, consumer string) {
|
||||
for _, r := range c.recorders {
|
||||
r.RecordDLQMessage(ctx, stream, consumer)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *compositeRecorder) RecordDLQPublishFailure(ctx context.Context, stream, consumer string) {
|
||||
for _, r := range c.recorders {
|
||||
r.RecordDLQPublishFailure(ctx, stream, consumer)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *compositeRecorder) RecordJSAPICall(operation string) {
|
||||
for _, r := range c.recorders {
|
||||
r.RecordJSAPICall(operation)
|
||||
}
|
||||
}
|
||||
|
||||
// promRecorder delegates to the Prometheus metrics helpers in the
|
||||
// internal/observability/metrics package.
|
||||
type promRecorder struct{}
|
||||
|
||||
func newPromRecorder() Recorder {
|
||||
return &promRecorder{}
|
||||
}
|
||||
|
||||
func (p *promRecorder) RecordProcessingResult(ctx context.Context, status, category string, parseLatency time.Duration) {
|
||||
if parseLatency < 0 {
|
||||
parseLatency = 0
|
||||
}
|
||||
obsmetrics.RecordProcessed(status, category, parseLatency)
|
||||
}
|
||||
|
||||
func (p *promRecorder) RecordPublishFailure(ctx context.Context, category string) {
|
||||
// Prometheus metrics currently only expose failures via caatsm_failures_total,
|
||||
// so we record the publisher failure there.
|
||||
obsmetrics.RecordFailure("publisher")
|
||||
}
|
||||
|
||||
func (p *promRecorder) RecordFailure(stage string) {
|
||||
obsmetrics.RecordFailure(stage)
|
||||
}
|
||||
|
||||
func (p *promRecorder) RecordMessageHandled(ctx context.Context, stream, consumer, result string, elapsed time.Duration) {
|
||||
obsmetrics.RecordMessageHandled(stream, consumer, result, elapsed)
|
||||
}
|
||||
|
||||
func (p *promRecorder) RecordRetry(ctx context.Context, stream, consumer, reason string) {
|
||||
obsmetrics.RecordRetry(stream, consumer, reason)
|
||||
}
|
||||
|
||||
func (p *promRecorder) RecordDLQMessage(ctx context.Context, stream, consumer string) {
|
||||
obsmetrics.RecordDLQMessage(stream, consumer)
|
||||
}
|
||||
|
||||
func (p *promRecorder) RecordDLQPublishFailure(ctx context.Context, stream, consumer string) {
|
||||
obsmetrics.RecordDLQPublishFailure(stream, consumer)
|
||||
}
|
||||
|
||||
func (p *promRecorder) RecordJSAPICall(operation string) {
|
||||
obsmetrics.RecordJSAPICall(operation)
|
||||
}
|
||||
|
||||
// otelRecorder creates and records OpenTelemetry metrics for the CAATSM
|
||||
// processor. It intentionally focuses on a small set of high-value metrics to
|
||||
// avoid duplicating the full Prometheus surface.
|
||||
type otelRecorder struct {
|
||||
meter metric.Meter
|
||||
|
||||
messageStatusAttrKey attribute.Key
|
||||
messageCategoryAttrKey attribute.Key
|
||||
|
||||
messageProcessedCounter metric.Int64Counter
|
||||
messagePublishFailCounter metric.Int64Counter
|
||||
parseLatencyHistogram metric.Float64Histogram
|
||||
}
|
||||
|
||||
func newOTelRecorder() Recorder {
|
||||
meter := otel.Meter("caatsm/app")
|
||||
|
||||
statusKey := attribute.Key("message.status")
|
||||
categoryKey := attribute.Key("message.category")
|
||||
|
||||
messageProcessedCounter, _ := meter.Int64Counter(
|
||||
"caatsm_messages_processed_total",
|
||||
metric.WithDescription("Total number of telegrams processed by the CAATSM processor."),
|
||||
)
|
||||
messagePublishFailCounter, _ := meter.Int64Counter(
|
||||
"caatsm_publish_failures_total",
|
||||
metric.WithDescription("Total number of telegram publish failures."),
|
||||
)
|
||||
parseLatencyHistogram, _ := meter.Float64Histogram(
|
||||
"caatsm_parse_duration_seconds",
|
||||
metric.WithDescription("Latency of parsing a telegram, in seconds."),
|
||||
metric.WithUnit("s"),
|
||||
)
|
||||
|
||||
return &otelRecorder{
|
||||
meter: meter,
|
||||
messageStatusAttrKey: statusKey,
|
||||
messageCategoryAttrKey: categoryKey,
|
||||
messageProcessedCounter: messageProcessedCounter,
|
||||
messagePublishFailCounter: messagePublishFailCounter,
|
||||
parseLatencyHistogram: parseLatencyHistogram,
|
||||
}
|
||||
}
|
||||
|
||||
func (o *otelRecorder) RecordProcessingResult(ctx context.Context, status, category string, parseLatency time.Duration) {
|
||||
if parseLatency < 0 {
|
||||
parseLatency = 0
|
||||
}
|
||||
o.messageProcessedCounter.Add(ctx, 1,
|
||||
metric.WithAttributes(
|
||||
o.messageStatusAttrKey.String(status),
|
||||
o.messageCategoryAttrKey.String(category),
|
||||
),
|
||||
)
|
||||
o.parseLatencyHistogram.Record(ctx, parseLatency.Seconds(),
|
||||
metric.WithAttributes(
|
||||
o.messageStatusAttrKey.String(status),
|
||||
o.messageCategoryAttrKey.String(category),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
func (o *otelRecorder) RecordPublishFailure(ctx context.Context, category string) {
|
||||
o.messagePublishFailCounter.Add(ctx, 1,
|
||||
metric.WithAttributes(
|
||||
o.messageCategoryAttrKey.String(category),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
func (o *otelRecorder) RecordFailure(stage string) {
|
||||
// OpenTelemetry does not currently publish a dedicated failure counter; the
|
||||
// Prometheus surface captures this. This method is a no-op here.
|
||||
}
|
||||
|
||||
func (o *otelRecorder) RecordMessageHandled(ctx context.Context, stream, consumer, result string, elapsed time.Duration) {
|
||||
// High-cardinality stream/consumer labels are exposed via Prometheus
|
||||
// metrics; OTEL can rely on traces and existing consumer metrics.
|
||||
}
|
||||
|
||||
func (o *otelRecorder) RecordRetry(ctx context.Context, stream, consumer, reason string) {
|
||||
}
|
||||
|
||||
func (o *otelRecorder) RecordDLQMessage(ctx context.Context, stream, consumer string) {
|
||||
}
|
||||
|
||||
func (o *otelRecorder) RecordDLQPublishFailure(ctx context.Context, stream, consumer string) {
|
||||
}
|
||||
|
||||
func (o *otelRecorder) RecordJSAPICall(operation string) {
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user