windy/go-caatsm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ Revise observability features by introducing new health and liveness endpoints (/livez and /readyz), updating Prometheus metrics to track NATS consumer pending messages, and enhancing telemetry integration for better monitoring. Update documentation to reflect these changes and ensure consistency in metric naming conventions.

Browse Source
This commit is contained in:
windyboy
2025-11-16 13:14:46 +08:00
parent 3d8572a69f
commit e27328378a
17 changed files with 582 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/infra/buildinfo/buildinfo.go
+19
View File
@@ -0,0 +1,19 @@
package buildinfo
// Version, Commit, and BuiltAt are populated via -ldflags at build time. They
// default to development-friendly values when not provided.
//
// Example:
// go build -ldflags "\
// -X 'caatsm/internal/infra/buildinfo.Version=v0.4.3' \
// -X 'caatsm/internal/infra/buildinfo.Commit=abc1234' \
// -X 'caatsm/internal/infra/buildinfo.BuiltAt=2025-11-16T08:35:00Z' \
// "
var (
Version = "dev"
Commit = "unknown"
BuiltAt = ""
)
internal/infra/monitoring/server.go
+61 -14
View File
@@ -1,6 +1,7 @@
package monitoring
import (
"caatsm/internal/infra/buildinfo"
"caatsm/internal/infra/config"
obsmetrics "caatsm/internal/observability/metrics"
"context"
@@ -47,11 +48,13 @@ func ProvideServer(
routes := 0
if cfg.Monitoring.EnableHealth {
// Liveness: basic process check. For now this reuses the same implementation
// as readiness but can diverge in the future if we need a cheaper liveness probe.
// Liveness: cheap process check that does not hit external dependencies.
mux.HandleFunc("/livez", server.handleLive)
// Backward-compatible health endpoint. For now this keeps the same
// semantics as readiness but will remain stable for existing users.
mux.HandleFunc("/healthz", server.handleHealth)
// Readiness: alias to the same implementation so consumers can adopt /readyz
// without breaking existing /healthz users.
// Readiness: dependency-aware check intended for load balancers and
// orchestrators.
mux.HandleFunc("/readyz", server.handleHealth)
routes++
}
@@ -111,33 +114,52 @@ func (s *Server) Shutdown(ctx context.Context) error {
func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
status := http.StatusOK
result := map[string]interface{}{
"postgres": "ok",
"nats": "ok",
deps := map[string]map[string]interface{}{
"postgres": {
"status": "ok",
},
"nats": {
"status": "ok",
},
}
ctx, cancel := context.WithTimeout(r.Context(), s.healthTimeout())
defer cancel()
if s.pool == nil {
result["postgres"] = "unconfigured"
status = http.StatusServiceUnavailable
} else if err := s.pool.Ping(ctx); err != nil {
result["postgres"] = err.Error()
deps["postgres"]["status"] = "unconfigured"
status = http.StatusServiceUnavailable
} else {
start := time.Now()
if err := s.pool.Ping(ctx); err != nil {
deps["postgres"]["status"] = err.Error()
status = http.StatusServiceUnavailable
} else {
deps["postgres"]["latency_ms"] = time.Since(start).Milliseconds()
}
}
if s.conn == nil {
result["nats"] = "unconfigured"
deps["nats"]["status"] = "unconfigured"
status = http.StatusServiceUnavailable
} else if s.conn.Status() != nats.CONNECTED {
result["nats"] = s.conn.Status().String()
deps["nats"]["status"] = s.conn.Status().String()
status = http.StatusServiceUnavailable
}
payload := map[string]interface{}{
"status": httpStatusLabel(status),
"build": map[string]interface{}{
"version": buildinfo.Version,
"rev": buildinfo.Commit,
"built_at": buildinfo.BuiltAt,
},
"dependencies": deps,
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(result)
_ = json.NewEncoder(w).Encode(payload)
}
func (s *Server) healthTimeout() time.Duration {
@@ -147,3 +169,28 @@ func (s *Server) healthTimeout() time.Duration {
}
return timeout
}
// handleLive reports basic process liveness and build information without
// consulting external dependencies. It is suitable for liveness probes.
func (s *Server) handleLive(w http.ResponseWriter, r *http.Request) {
payload := map[string]interface{}{
"status": "ok",
"build": map[string]interface{}{
"version": buildinfo.Version,
"rev": buildinfo.Commit,
"built_at": buildinfo.BuiltAt,
},
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_ = json.NewEncoder(w).Encode(payload)
}
func httpStatusLabel(code int) string {
if code >= 200 && code < 300 {
return "ok"
}
return "error"
}
internal/infra/nats/consumer.go
+15 -7
View File
@@ -5,6 +5,7 @@ import (
"caatsm/internal/infra/config"
obslogging "caatsm/internal/observability/logging"
obsmetrics "caatsm/internal/observability/metrics"
"caatsm/internal/observability/telemetry"
"context"
"encoding/json"
"errors"
@@ -28,6 +29,7 @@ type Consumer struct {
processor *app.MessageProcessor
cfg *config.Config
logger *zap.Logger
telemetry telemetry.Recorder
subject string
consumerName string
mode string
@@ -53,6 +55,7 @@ func ProvideConsumer(
js nats.JetStreamContext,
processor *app.MessageProcessor,
cfg *config.Config,
rec telemetry.Recorder,
logger *zap.Logger,
) (*Consumer, error) {
subject := cfg.EffectiveSubscriptionTopic()
@@ -108,6 +111,7 @@ func ProvideConsumer(
processor: processor,
cfg: cfg,
logger: logger,
telemetry: rec,
subject: subject,
consumerName: consumerName,
mode: mode,
@@ -238,7 +242,7 @@ func (c *Consumer) validateDLQ() {
// Ensure the DLQ subject is actually bound to a JetStream stream. This avoids
// the opaque `nats: no response from stream` error later when publishing.
obsmetrics.RecordJSAPICall("dlq_validate_stream")
c.telemetry.RecordJSAPICall("dlq_validate_stream")
streamName, err := c.js.StreamNameBySubject(subject)
if err != nil || strings.TrimSpace(streamName) == "" {
c.logger.Warn("DLQ subject not bound to any JetStream stream; DLQ routing disabled",
@@ -342,7 +346,7 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
if isPermanent {
result = obsmetrics.ResultPermanentFail
}
obsmetrics.RecordMessageHandled(c.streamName, c.consumerName, result, elapsed)
c.telemetry.RecordMessageHandled(ctx, c.streamName, c.consumerName, result, elapsed)
if isPermanent {
c.consecutiveProcessErrors = 0
@@ -376,7 +380,7 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
}
// Transient error: request redelivery with optional delay
obsmetrics.RecordRetry(c.streamName, c.consumerName, obsmetrics.RetryReasonProcessorError)
c.telemetry.RecordRetry(ctx, c.streamName, c.consumerName, obsmetrics.RetryReasonProcessorError)
if nakErr := c.nakWithStrategy(msg); nakErr != nil {
c.logger.Error("Failed to NAK message", zap.Error(nakErr))
}
@@ -393,7 +397,7 @@ func (c *Consumer) startJetStream(ctx context.Context) error {
c.logger.Error("Failed to ACK message", zap.Error(ackErr))
} else {
elapsed := time.Since(start)
obsmetrics.RecordMessageHandled(c.streamName, c.consumerName, "ok", elapsed)
c.telemetry.RecordMessageHandled(ctx, c.streamName, c.consumerName, "ok", elapsed)
}
}
}
@@ -531,6 +535,10 @@ func (c *Consumer) recordConsumerMetrics(ctx context.Context, info *nats.Consume
if c.delivered != nil {
c.delivered.Record(ctx, int64(info.Delivered.Stream))
}
// Export an explicit pending messages gauge for Prometheus-based lag /
// backlog alerts.
obsmetrics.RecordNATSConsumerPending(c.streamName, c.consumerName, info.NumPending)
}
// routeToDLQ publishes a copy of the failed message to the configured DLQ subject,
@@ -578,14 +586,14 @@ func (c *Consumer) routeToDLQ(ctx context.Context, msg *nats.Msg, cause error) e
// unavailable. Surface this explicitly to make operational diagnosis
// easier.
if errors.Is(err, nats.ErrNoResponders) {
obsmetrics.RecordDLQPublishFailure(c.streamName, c.consumerName)
c.telemetry.RecordDLQPublishFailure(ctx, c.streamName, c.consumerName)
return fmt.Errorf("publish to dlq subject %s: no JetStream stream found for subject or JetStream unavailable: %w", c.dlqSubject, err)
}
obsmetrics.RecordDLQPublishFailure(c.streamName, c.consumerName)
c.telemetry.RecordDLQPublishFailure(ctx, c.streamName, c.consumerName)
return fmt.Errorf("publish to dlq subject %s: %w", c.dlqSubject, err)
}
obsmetrics.RecordDLQMessage(c.streamName, c.consumerName)
c.telemetry.RecordDLQMessage(ctx, c.streamName, c.consumerName)
return nil
}