go-caatsm/internal/infra/nats/monitor.go

package nats

import (
	"caatsm/internal/infra/config"
	obsmetrics "caatsm/internal/infra/metrics"
	"context"
	"sync"
	"time"

	"github.com/nats-io/nats.go"
	"go.uber.org/zap"
)

// ConsumerMonitor handles health monitoring and stats emission for the consumer.
type ConsumerMonitor struct {
	logger *zap.Logger
	cfg    *config.Config
	js     nats.JetStreamContext

	// Configuration
	streamName      string
	consumerName    string
	monitorInterval time.Duration

	// State
	lastMessageTime     time.Time
	lastMessageSequence uint64
	messageGapMutex     sync.RWMutex
}

// NewConsumerMonitor creates a new ConsumerMonitor.
func NewConsumerMonitor(
	logger *zap.Logger,
	cfg *config.Config,
	js nats.JetStreamContext,
	streamName string,
	consumerName string,
	monitorInterval time.Duration,
) *ConsumerMonitor {
	return &ConsumerMonitor{
		logger:          logger,
		cfg:             cfg,
		js:              js,
		streamName:      streamName,
		consumerName:    consumerName,
		monitorInterval: monitorInterval,
	}
}

// Start begins the monitoring loop.
func (m *ConsumerMonitor) Start(ctx context.Context) {
	ticker := time.NewTicker(m.monitorInterval)
	defer ticker.Stop()

	// Record initial metric (0) to ensure it appears in Prometheus even before first tick
	m.logger.Info("Starting NATS consumer stats emission goroutine, recording initial pending metric",
		zap.String("stream", m.streamName),
		zap.String("consumer", m.consumerName),
		zap.Duration("interval", m.monitorInterval),
	)
	obsmetrics.RecordNATSConsumerPending(m.streamName, m.consumerName, 0)

	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			m.emitStats()
		}
	}
}

// emitStats gathers and records consumer statistics.
func (m *ConsumerMonitor) emitStats() {
	if m.js != nil {
		info, err := m.js.ConsumerInfo(m.streamName, m.consumerName)
		if err != nil {
			m.logger.Warn("Failed to fetch consumer info for pending messages metric",
				zap.String("stream", m.streamName),
				zap.String("consumer", m.consumerName),
				zap.Error(err),
			)
		} else {
			// Record pending messages for monitoring
			m.logger.Debug("Recording NATS consumer pending messages metric",
				zap.String("stream", m.streamName),
				zap.String("consumer", m.consumerName),
				zap.Uint64("pending", info.NumPending),
			)
			obsmetrics.RecordNATSConsumerPending(m.streamName, m.consumerName, info.NumPending)
		}
	}

	// Record AFTN health metrics
	gapSeconds := m.GetMessageGapSeconds()
	healthy := m.IsHealthy()

	obsmetrics.RecordMessageGap(m.streamName, m.consumerName, gapSeconds)
	obsmetrics.RecordSerialReaderHealth(m.streamName, m.consumerName, healthy)

	if !healthy {
		m.logger.Warn("Serial reader appears stalled - no messages received recently",
			zap.String("stream", m.streamName),
			zap.String("consumer", m.consumerName),
			zap.Float64("gap_seconds", gapSeconds),
			zap.Duration("threshold", m.cfg.AFTN.MessageGapThreshold),
		)
	}
}

// TrackMessage updates health metrics based on a received message.
func (m *ConsumerMonitor) TrackMessage(msg *nats.Msg) {
	if msg == nil {
		return
	}

	m.messageGapMutex.Lock()
	defer m.messageGapMutex.Unlock()

	now := time.Now()
	m.lastMessageTime = now

	// Extract sequence number from message metadata
	if meta, err := msg.Metadata(); err == nil {
		currentSeq := meta.Sequence.Stream

		// Detect sequence gaps if we have a previous sequence
		if m.lastMessageSequence > 0 && m.cfg.AFTN.EnableSequenceGapDetection {
			if currentSeq > m.lastMessageSequence+1 {
				gapSize := currentSeq - m.lastMessageSequence - 1
				m.logger.Warn("Message sequence gap detected",
					zap.String("stream", m.streamName),
					zap.String("consumer", m.consumerName),
					zap.Uint64("last_sequence", m.lastMessageSequence),
					zap.Uint64("current_sequence", currentSeq),
					zap.Uint64("gap_size", gapSize),
				)
				obsmetrics.RecordSequenceGap(m.streamName, m.consumerName, gapSize)
			}
		}

		m.lastMessageSequence = currentSeq
	}
}

// GetMessageGapSeconds returns the number of seconds since the last message was received.
func (m *ConsumerMonitor) GetMessageGapSeconds() float64 {
	m.messageGapMutex.RLock()
	defer m.messageGapMutex.RUnlock()

	if m.lastMessageTime.IsZero() {
		return 0
	}

	return time.Since(m.lastMessageTime).Seconds()
}

// IsHealthy returns true if messages are being received within the threshold.
func (m *ConsumerMonitor) IsHealthy() bool {
	m.messageGapMutex.RLock()
	defer m.messageGapMutex.RUnlock()

	// If we haven't received any messages yet, consider it healthy (initial state)
	if m.lastMessageTime.IsZero() {
		return true
	}

	gap := time.Since(m.lastMessageTime)
	return gap < m.cfg.AFTN.MessageGapThreshold
}

// RecordInitialPending logs and records the initial pending messages count.
// This is exposed to allow recording immediately upon startup.
func (m *ConsumerMonitor) RecordInitialPending(pending uint64) {
	m.logger.Info("Recording initial NATS consumer pending messages metric",
		zap.String("stream", m.streamName),
		zap.String("consumer", m.consumerName),
		zap.Uint64("pending", pending),
	)
	obsmetrics.RecordNATSConsumerPending(m.streamName, m.consumerName, pending)
}