windy/go-caatsm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ Integrate monitoring server for observability, adding health and metrics endpoints. Update configuration to enable monitoring features and enhance README with deployment examples for Kubernetes and systemd. Refactor application initialization to include monitoring server setup and improve error handling in message processing metrics.

Browse Source
This commit is contained in:
windyboy
2025-11-15 16:30:29 +08:00
parent c3f98e9c7c
commit 53208997e8
17 changed files with 1092 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/app/processor.go
+21 -9
View File
@@ -4,6 +4,7 @@ import (
"caatsm/internal/adapter"
"caatsm/internal/adapter/parser"
"caatsm/internal/domain"
obsmetrics "caatsm/internal/observability/metrics"
"context"
"fmt"
"strings"
@@ -122,13 +123,15 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
zap.String("content_preview", truncateContent(parsed.Content, 256)),
zap.Error(parseErr),
)
parseLatencyHistogram.Record(ctx, float64(parsed.ParsedAt.Sub(receivedAt).Milliseconds()),
latency := parsed.ParsedAt.Sub(receivedAt)
parseLatencyHistogram.Record(ctx, float64(latency.Milliseconds()),
metric.WithAttributes(
messageStatusAttrKey.String(string(parsed.Status)),
messageCategoryAttrKey.String(parsed.Category),
),
)
recordProcessedMetric(ctx, parsed)
obsmetrics.RecordFailure("parser")
recordProcessedMetric(ctx, parsed, latency)
return Permanent(fmt.Errorf("parser error: %w", parseErr))
}
parsed.ErrorReason = ""
@@ -153,13 +156,15 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
parsed.Status = domain.MessageStatusRepositoryFail
parseLatencyHistogram.Record(ctx, float64(parsed.ParsedAt.Sub(receivedAt).Milliseconds()),
latency := parsed.ParsedAt.Sub(receivedAt)
parseLatencyHistogram.Record(ctx, float64(latency.Milliseconds()),
metric.WithAttributes(
messageStatusAttrKey.String(string(parsed.Status)),
messageCategoryAttrKey.String(parsed.Category),
),
)
recordProcessedMetric(ctx, parsed)
obsmetrics.RecordFailure("repository")
recordProcessedMetric(ctx, parsed, latency)
return fmt.Errorf("failed to insert message: %w", err)
}
@@ -180,13 +185,15 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
messageCategoryAttrKey.String(parsed.Category),
),
)
parseLatencyHistogram.Record(ctx, float64(parsed.ParsedAt.Sub(receivedAt).Milliseconds()),
latency := parsed.ParsedAt.Sub(receivedAt)
parseLatencyHistogram.Record(ctx, float64(latency.Milliseconds()),
metric.WithAttributes(
messageStatusAttrKey.String(string(parsed.Status)),
messageCategoryAttrKey.String(parsed.Category),
),
)
recordProcessedMetric(ctx, parsed)
obsmetrics.RecordFailure("publisher")
recordProcessedMetric(ctx, parsed, latency)
p.persistRaw(ctx, parsed)
// Mark as permanent so the consumer will ack instead of retrying
pubSpan.End()
@@ -194,13 +201,14 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
}
pubSpan.End()
parseLatencyHistogram.Record(ctx, float64(parsed.ParsedAt.Sub(receivedAt).Milliseconds()),
latency := parsed.ParsedAt.Sub(receivedAt)
parseLatencyHistogram.Record(ctx, float64(latency.Milliseconds()),
metric.WithAttributes(
messageStatusAttrKey.String(string(parsed.Status)),
messageCategoryAttrKey.String(parsed.Category),
),
)
recordProcessedMetric(ctx, parsed)
recordProcessedMetric(ctx, parsed, latency)
return nil
}
@@ -242,7 +250,7 @@ func truncateContent(content string, limit int) string {
return content[:limit-3] + "..."
}
func recordProcessedMetric(ctx context.Context, msg *domain.ParsedMessage) {
func recordProcessedMetric(ctx context.Context, msg *domain.ParsedMessage, elapsed time.Duration) {
if msg == nil {
return
}
@@ -252,4 +260,8 @@ func recordProcessedMetric(ctx context.Context, msg *domain.ParsedMessage) {
messageCategoryAttrKey.String(msg.Category),
),
)
if elapsed < 0 {
elapsed = 0
}
obsmetrics.RecordProcessed(string(msg.Status), msg.Category, elapsed)
}
internal/domain/pln_test.go
+51
View File
@@ -0,0 +1,51 @@
package domain
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("ScheduleLine", func() {
var base ScheduleLine
BeforeEach(func() {
base = ScheduleLine{
Index: "001",
Date: "30OCT",
Task: "H/G",
FlightNumber: []string{
"CA1014",
},
AircraftReg: "B2458",
PassengerConfig: "1/1",
ILS: "ILS(0)",
Waypoints: []WayPoint{
{Airport: "ZBTJ", DepartureTime: "0100", ArrivalTime: "0200"},
},
Comments: "all green",
}
})
It("passes validation when all required fields exist", func() {
Expect(base.Validate()).To(Succeed())
})
DescribeTable("required field validation",
func(mutator func(line *ScheduleLine), expected string) {
line := base
mutator(&line)
err := line.Validate()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(expected))
},
Entry("missing date", func(line *ScheduleLine) {
line.Date = ""
}, "date is required"),
Entry("missing flight number", func(line *ScheduleLine) {
line.FlightNumber = nil
}, "flight number is required"),
Entry("missing aircraft registration", func(line *ScheduleLine) {
line.AircraftReg = ""
}, "aircraft registration is required"),
)
})
internal/infra/config/config.go
+42 -6
View File
@@ -14,12 +14,13 @@ import (
// Config holds all application configuration
type Config struct {
NATS NATSConfig `koanf:"nats"`
Postgres PostgresConfig `koanf:"postgres"`
App AppConfig `koanf:"app"`
Log LogConfig `koanf:"log"`
Publisher PublisherConfig `koanf:"publisher"`
Telemetry TelemetryConfig `koanf:"telemetry"`
NATS NATSConfig `koanf:"nats"`
Postgres PostgresConfig `koanf:"postgres"`
App AppConfig `koanf:"app"`
Log LogConfig `koanf:"log"`
Publisher PublisherConfig `koanf:"publisher"`
Telemetry TelemetryConfig `koanf:"telemetry"`
Monitoring MonitoringConfig `koanf:"monitoring"`
// Legacy fields for backward compatibility during migration
Subscription SubscriptionConfig `koanf:"subscription"`
Timeouts TimeoutsConfig `koanf:"timeouts"`
@@ -92,6 +93,17 @@ type TelemetryConfig struct {
Insecure bool `koanf:"insecure"`
}
// MonitoringConfig controls the lightweight HTTP server that exposes health and metrics endpoints.
type MonitoringConfig struct {
Disabled bool `koanf:"disabled"`
Addr string `koanf:"addr"`
EnableMetrics bool `koanf:"enable_metrics"`
EnableHealth bool `koanf:"enable_health"`
ReadTimeout time.Duration `koanf:"read_timeout"`
WriteTimeout time.Duration `koanf:"write_timeout"`
HealthTimeout time.Duration `koanf:"health_timeout"`
}
// SubscriptionConfig holds subscription configuration (legacy)
type SubscriptionConfig struct {
Topic string `koanf:"topic"`
@@ -212,6 +224,21 @@ func LoadConfig() (*Config, error) {
cfg.Telemetry.Endpoint = ""
}
if !cfg.Monitoring.Disabled && cfg.Monitoring.Addr == "" && !cfg.Monitoring.EnableHealth && !cfg.Monitoring.EnableMetrics {
cfg.Monitoring.Addr = ":2112"
cfg.Monitoring.EnableHealth = true
cfg.Monitoring.EnableMetrics = true
}
if cfg.Monitoring.ReadTimeout == 0 {
cfg.Monitoring.ReadTimeout = 5 * time.Second
}
if cfg.Monitoring.WriteTimeout == 0 {
cfg.Monitoring.WriteTimeout = 5 * time.Second
}
if cfg.Monitoring.HealthTimeout == 0 {
cfg.Monitoring.HealthTimeout = 2 * time.Second
}
// Validate configuration
if err := cfg.Validate(); err != nil {
return nil, fmt.Errorf("config validation failed: %w", err)
@@ -292,6 +319,15 @@ func (c *Config) Validate() error {
if c.Telemetry.Endpoint == "" && c.Telemetry.Enabled {
return fmt.Errorf("telemetry.endpoint is required when telemetry.enabled=true")
}
if c.Monitoring.ReadTimeout < 0 {
return fmt.Errorf("monitoring.read_timeout must be >= 0")
}
if c.Monitoring.WriteTimeout < 0 {
return fmt.Errorf("monitoring.write_timeout must be >= 0")
}
if c.Monitoring.HealthTimeout < 0 {
return fmt.Errorf("monitoring.health_timeout must be >= 0")
}
return nil
}
internal/infra/monitoring/server.go
+144
View File
@@ -0,0 +1,144 @@
package monitoring
import (
"caatsm/internal/infra/config"
obsmetrics "caatsm/internal/observability/metrics"
"context"
"encoding/json"
"errors"
"net/http"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/nats-io/nats.go"
"go.uber.org/zap"
)
// Server exposes /healthz and /metrics endpoints for basic operations and observability checks.
type Server struct {
cfg config.MonitoringConfig
logger *zap.Logger
pool *pgxpool.Pool
conn *nats.Conn
httpServer *http.Server
}
// ProvideServer wires a monitoring server if enabled in configuration.
func ProvideServer(
cfg *config.Config,
logger *zap.Logger,
pool *pgxpool.Pool,
conn *nats.Conn,
) (*Server, error) {
if cfg == nil || logger == nil || cfg.Monitoring.Disabled {
return nil, nil
}
if cfg.Monitoring.Addr == "" {
return nil, nil
}
mux := http.NewServeMux()
server := &Server{
cfg: cfg.Monitoring,
logger: logger,
pool: pool,
conn: conn,
}
routes := 0
if cfg.Monitoring.EnableHealth {
mux.HandleFunc("/healthz", server.handleHealth)
routes++
}
if cfg.Monitoring.EnableMetrics {
mux.Handle("/metrics", obsmetrics.Handler())
routes++
}
if routes == 0 {
return nil, nil
}
httpServer := &http.Server{
Addr: cfg.Monitoring.Addr,
Handler: mux,
ReadHeaderTimeout: 3 * time.Second,
ReadTimeout: cfg.Monitoring.ReadTimeout,
WriteTimeout: cfg.Monitoring.WriteTimeout,
}
server.httpServer = httpServer
return server, nil
}
// Start launches the monitoring HTTP server in the background.
func (s *Server) Start(ctx context.Context) error {
if s == nil || s.httpServer == nil {
return nil
}
go func() {
<-ctx.Done()
_ = s.Shutdown(context.Background())
}()
go func() {
s.logger.Info("Monitoring server listening",
zap.String("addr", s.httpServer.Addr),
zap.Bool("metrics", s.cfg.EnableMetrics),
zap.Bool("health", s.cfg.EnableHealth),
)
if err := s.httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
s.logger.Error("Monitoring server exited", zap.Error(err))
}
}()
return nil
}
// Shutdown gracefully stops the HTTP server.
func (s *Server) Shutdown(ctx context.Context) error {
if s == nil || s.httpServer == nil {
return nil
}
return s.httpServer.Shutdown(ctx)
}
func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
status := http.StatusOK
result := map[string]interface{}{
"postgres": "ok",
"nats": "ok",
}
ctx, cancel := context.WithTimeout(r.Context(), s.healthTimeout())
defer cancel()
if s.pool == nil {
result["postgres"] = "unconfigured"
status = http.StatusServiceUnavailable
} else if err := s.pool.Ping(ctx); err != nil {
result["postgres"] = err.Error()
status = http.StatusServiceUnavailable
}
if s.conn == nil {
result["nats"] = "unconfigured"
status = http.StatusServiceUnavailable
} else if s.conn.Status() != nats.CONNECTED {
result["nats"] = s.conn.Status().String()
status = http.StatusServiceUnavailable
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(result)
}
func (s *Server) healthTimeout() time.Duration {
timeout := s.cfg.HealthTimeout
if timeout <= 0 {
timeout = 2 * time.Second
}
return timeout
}
internal/observability/metrics/metrics.go
+71
View File
@@ -0,0 +1,71 @@
package metrics
import (
"math"
"net/http"
"strings"
"sync"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
var (
once sync.Once
registry *prometheus.Registry
processedCounter *prometheus.CounterVec
failureCounter *prometheus.CounterVec
parseLatency *prometheus.HistogramVec
)
func initCollectors() {
registry = prometheus.NewRegistry()
processedCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "caatsm_processed_total",
Help: "Count of telegrams processed by status and category.",
}, []string{"status", "category"})
failureCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "caatsm_failures_total",
Help: "Count of processor failures by stage (parser, repository, publisher).",
}, []string{"stage"})
parseLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "caatsm_parse_latency_seconds",
Help: "Latency between reception and parse completion.",
Buckets: prometheus.DefBuckets,
}, []string{"status", "category"})
registry.MustRegister(processedCounter, failureCounter, parseLatency)
}
func ensureCollectors() {
once.Do(initCollectors)
}
// Handler exposes the Prometheus metrics registry.
func Handler() http.Handler {
ensureCollectors()
return promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
}
// RecordProcessed tracks the final status of a telegram along with the parse latency.
func RecordProcessed(status, category string, elapsed time.Duration) {
ensureCollectors()
processedCounter.WithLabelValues(labelValue(status), labelValue(category)).Inc()
seconds := math.Max(elapsed.Seconds(), 0)
parseLatency.WithLabelValues(labelValue(status), labelValue(category)).Observe(seconds)
}
// RecordFailure increments the failure counter for the supplied stage.
func RecordFailure(stage string) {
ensureCollectors()
failureCounter.WithLabelValues(labelValue(stage)).Inc()
}
func labelValue(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return "unknown"
}
return strings.ToLower(value)
}