✨ Integrate monitoring server for observability, adding health and metrics endpoints. Update configuration to enable monitoring features and enhance README with deployment examples for Kubernetes and systemd. Refactor application initialization to include monitoring server setup and improve error handling in message processing metrics.

2025-11-15 16:30:29 +08:00
parent c3f98e9c7c
commit 53208997e8
17 changed files with 1092 additions and 182 deletions
@@ -14,12 +14,13 @@ import (

 // Config holds all application configuration
 type Config struct {
-	NATS      NATSConfig      `koanf:"nats"`
-	Postgres  PostgresConfig  `koanf:"postgres"`
-	App       AppConfig       `koanf:"app"`
-	Log       LogConfig       `koanf:"log"`
-	Publisher PublisherConfig `koanf:"publisher"`
-	Telemetry TelemetryConfig `koanf:"telemetry"`
+	NATS       NATSConfig       `koanf:"nats"`
+	Postgres   PostgresConfig   `koanf:"postgres"`
+	App        AppConfig        `koanf:"app"`
+	Log        LogConfig        `koanf:"log"`
+	Publisher  PublisherConfig  `koanf:"publisher"`
+	Telemetry  TelemetryConfig  `koanf:"telemetry"`
+	Monitoring MonitoringConfig `koanf:"monitoring"`
 	// Legacy fields for backward compatibility during migration
 	Subscription SubscriptionConfig `koanf:"subscription"`
 	Timeouts     TimeoutsConfig     `koanf:"timeouts"`
@@ -92,6 +93,17 @@ type TelemetryConfig struct {
 	Insecure bool   `koanf:"insecure"`
 }

+// MonitoringConfig controls the lightweight HTTP server that exposes health and metrics endpoints.
+type MonitoringConfig struct {
+	Disabled      bool          `koanf:"disabled"`
+	Addr          string        `koanf:"addr"`
+	EnableMetrics bool          `koanf:"enable_metrics"`
+	EnableHealth  bool          `koanf:"enable_health"`
+	ReadTimeout   time.Duration `koanf:"read_timeout"`
+	WriteTimeout  time.Duration `koanf:"write_timeout"`
+	HealthTimeout time.Duration `koanf:"health_timeout"`
+}
+
 // SubscriptionConfig holds subscription configuration (legacy)
 type SubscriptionConfig struct {
 	Topic      string `koanf:"topic"`
@@ -212,6 +224,21 @@ func LoadConfig() (*Config, error) {
 		cfg.Telemetry.Endpoint = ""
 	}

+	if !cfg.Monitoring.Disabled && cfg.Monitoring.Addr == "" && !cfg.Monitoring.EnableHealth && !cfg.Monitoring.EnableMetrics {
+		cfg.Monitoring.Addr = ":2112"
+		cfg.Monitoring.EnableHealth = true
+		cfg.Monitoring.EnableMetrics = true
+	}
+	if cfg.Monitoring.ReadTimeout == 0 {
+		cfg.Monitoring.ReadTimeout = 5 * time.Second
+	}
+	if cfg.Monitoring.WriteTimeout == 0 {
+		cfg.Monitoring.WriteTimeout = 5 * time.Second
+	}
+	if cfg.Monitoring.HealthTimeout == 0 {
+		cfg.Monitoring.HealthTimeout = 2 * time.Second
+	}
+
 	// Validate configuration
 	if err := cfg.Validate(); err != nil {
 		return nil, fmt.Errorf("config validation failed: %w", err)
@@ -292,6 +319,15 @@ func (c *Config) Validate() error {
 	if c.Telemetry.Endpoint == "" && c.Telemetry.Enabled {
 		return fmt.Errorf("telemetry.endpoint is required when telemetry.enabled=true")
 	}
+	if c.Monitoring.ReadTimeout < 0 {
+		return fmt.Errorf("monitoring.read_timeout must be >= 0")
+	}
+	if c.Monitoring.WriteTimeout < 0 {
+		return fmt.Errorf("monitoring.write_timeout must be >= 0")
+	}
+	if c.Monitoring.HealthTimeout < 0 {
+		return fmt.Errorf("monitoring.health_timeout must be >= 0")
+	}
 	return nil
 }

@@ -0,0 +1,144 @@
+package monitoring
+
+import (
+	"caatsm/internal/infra/config"
+	obsmetrics "caatsm/internal/observability/metrics"
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/nats-io/nats.go"
+	"go.uber.org/zap"
+)
+
+// Server exposes /healthz and /metrics endpoints for basic operations and observability checks.
+type Server struct {
+	cfg        config.MonitoringConfig
+	logger     *zap.Logger
+	pool       *pgxpool.Pool
+	conn       *nats.Conn
+	httpServer *http.Server
+}
+
+// ProvideServer wires a monitoring server if enabled in configuration.
+func ProvideServer(
+	cfg *config.Config,
+	logger *zap.Logger,
+	pool *pgxpool.Pool,
+	conn *nats.Conn,
+) (*Server, error) {
+	if cfg == nil || logger == nil || cfg.Monitoring.Disabled {
+		return nil, nil
+	}
+	if cfg.Monitoring.Addr == "" {
+		return nil, nil
+	}
+
+	mux := http.NewServeMux()
+	server := &Server{
+		cfg:    cfg.Monitoring,
+		logger: logger,
+		pool:   pool,
+		conn:   conn,
+	}
+
+	routes := 0
+	if cfg.Monitoring.EnableHealth {
+		mux.HandleFunc("/healthz", server.handleHealth)
+		routes++
+	}
+	if cfg.Monitoring.EnableMetrics {
+		mux.Handle("/metrics", obsmetrics.Handler())
+		routes++
+	}
+
+	if routes == 0 {
+		return nil, nil
+	}
+
+	httpServer := &http.Server{
+		Addr:              cfg.Monitoring.Addr,
+		Handler:           mux,
+		ReadHeaderTimeout: 3 * time.Second,
+		ReadTimeout:       cfg.Monitoring.ReadTimeout,
+		WriteTimeout:      cfg.Monitoring.WriteTimeout,
+	}
+	server.httpServer = httpServer
+
+	return server, nil
+}
+
+// Start launches the monitoring HTTP server in the background.
+func (s *Server) Start(ctx context.Context) error {
+	if s == nil || s.httpServer == nil {
+		return nil
+	}
+
+	go func() {
+		<-ctx.Done()
+		_ = s.Shutdown(context.Background())
+	}()
+
+	go func() {
+		s.logger.Info("Monitoring server listening",
+			zap.String("addr", s.httpServer.Addr),
+			zap.Bool("metrics", s.cfg.EnableMetrics),
+			zap.Bool("health", s.cfg.EnableHealth),
+		)
+		if err := s.httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			s.logger.Error("Monitoring server exited", zap.Error(err))
+		}
+	}()
+
+	return nil
+}
+
+// Shutdown gracefully stops the HTTP server.
+func (s *Server) Shutdown(ctx context.Context) error {
+	if s == nil || s.httpServer == nil {
+		return nil
+	}
+	return s.httpServer.Shutdown(ctx)
+}
+
+func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
+	status := http.StatusOK
+	result := map[string]interface{}{
+		"postgres": "ok",
+		"nats":     "ok",
+	}
+
+	ctx, cancel := context.WithTimeout(r.Context(), s.healthTimeout())
+	defer cancel()
+
+	if s.pool == nil {
+		result["postgres"] = "unconfigured"
+		status = http.StatusServiceUnavailable
+	} else if err := s.pool.Ping(ctx); err != nil {
+		result["postgres"] = err.Error()
+		status = http.StatusServiceUnavailable
+	}
+
+	if s.conn == nil {
+		result["nats"] = "unconfigured"
+		status = http.StatusServiceUnavailable
+	} else if s.conn.Status() != nats.CONNECTED {
+		result["nats"] = s.conn.Status().String()
+		status = http.StatusServiceUnavailable
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(result)
+}
+
+func (s *Server) healthTimeout() time.Duration {
+	timeout := s.cfg.HealthTimeout
+	if timeout <= 0 {
+		timeout = 2 * time.Second
+	}
+	return timeout
+}