# Prometheus Alert Rules for CAATSM AFTN Telegram Processor # # Installation: # 1. Copy this file to your Prometheus server's rules directory # 2. Add to prometheus.yml: # rule_files: # - "prometheus-alerts.yml" # 3. Reload Prometheus configuration # # Alert Severity Levels: # - critical: Immediate action required (pages on-call) # - warning: Investigation needed (notify team channel) groups: - name: caatsm_aftn_health interval: 30s rules: # Critical: Serial reader has stopped publishing messages - alert: SerialReaderStalled expr: caatsm_serial_reader_healthy == 0 for: 2m labels: severity: critical component: serial_reader annotations: summary: "Serial reader stalled for {{ $labels.stream }}/{{ $labels.consumer }}" description: | No messages have been received from the serial reader for more than 2 minutes. This indicates the serial port reader may have crashed or the hardware connection is broken. Current gap: {{ with query "caatsm_message_gap_seconds{stream=\"" }}{{ . | first | value | humanizeDuration }}{{ end }} Stream: {{ $labels.stream }} Consumer: {{ $labels.consumer }} ACTION REQUIRED: 1. Check serial reader process status 2. Verify serial port connection 3. Check hardware status 4. Review serial reader logs # Warning: Message gap is growing but not yet critical - alert: HighMessageGap expr: caatsm_message_gap_seconds > 60 and caatsm_serial_reader_healthy == 1 for: 1m labels: severity: warning component: serial_reader annotations: summary: "High message gap detected: {{ $labels.stream }}/{{ $labels.consumer }}" description: | Message gap is {{ $value }}s but still below critical threshold. This may indicate slow message processing or reduced incoming message rate. Stream: {{ $labels.stream }} Consumer: {{ $labels.consumer }} # Warning: Sequence gaps detected (missing messages) - alert: MessageSequenceGaps expr: rate(caatsm_message_sequence_gap_total[5m]) > 0 for: 2m labels: severity: warning component: serial_reader annotations: summary: "Message sequence gaps detected: {{ $labels.stream }}/{{ $labels.consumer }}" description: | Missing message sequence numbers detected at {{ $value | humanize }} gaps/sec. This indicates messages are being lost or skipped in the stream. Stream: {{ $labels.stream }} Consumer: {{ $labels.consumer }} Rate: {{ $value | humanize }} gaps/sec Possible causes: - Serial reader buffer overflow - Network packet loss (if messages forwarded over network) - Stream retention limits exceeded - Consumer processing too slow # Warning: High AFTN validation error rate - alert: HighAFTNValidationErrorRate expr: | ( sum(rate(caatsm_aftn_validation_errors_total[5m])) by (stream, consumer) / sum(rate(caatsm_processed_total[5m])) by (stream, consumer) ) > 0.05 for: 5m labels: severity: warning component: aftn_validator annotations: summary: "High AFTN validation error rate: {{ $value | humanizePercentage }}" description: | More than 5% of incoming telegrams are failing AFTN protocol validation. Current error rate: {{ $value | humanizePercentage }} This may indicate: - Upstream system sending malformed telegrams - Serial port data corruption - Configuration mismatch Check DLQ for error details and patterns. # Warning: Consumer lag is growing - alert: ConsumerLagGrowing expr: | deriv(caatsm_nats_consumer_pending_messages[5m]) > 10 for: 3m labels: severity: warning component: consumer annotations: summary: "Consumer lag growing: {{ $labels.stream }}/{{ $labels.consumer }}" description: | Consumer pending messages is growing at {{ $value | humanize }} msgs/sec. Current pending: {{ with query "caatsm_nats_consumer_pending_messages" }}{{ . | first | value }}{{ end }} This indicates the consumer cannot keep up with incoming message rate. Stream: {{ $labels.stream }} Consumer: {{ $labels.consumer }} # Critical: Consumer critically behind - alert: ConsumerCriticallyBehind expr: caatsm_nats_consumer_pending_messages > 5000 for: 5m labels: severity: critical component: consumer annotations: summary: "Consumer critically behind: {{ $value }} pending messages" description: | Consumer has {{ $value }} pending messages - critically behind. This will cause message processing delays and may trigger stream retention limits. Stream: {{ $labels.stream }} Consumer: {{ $labels.consumer }} Pending: {{ $value }} ACTION REQUIRED: 1. Check processor performance and errors 2. Check database connection and performance 3. Consider scaling consumers horizontally 4. Review stream retention settings # Warning: High processing failure rate - alert: HighProcessingFailureRate expr: | ( sum(rate(caatsm_messages_total{result="fail"}[5m])) by (stream, consumer) / sum(rate(caatsm_messages_total[5m])) by (stream, consumer) ) > 0.10 for: 5m labels: severity: warning component: processor annotations: summary: "High processing failure rate: {{ $value | humanizePercentage }}" description: | More than 10% of messages are failing to process. Current failure rate: {{ $value | humanizePercentage }} Stream: {{ $labels.stream }} Consumer: {{ $labels.consumer }} Check application logs for error details. # Warning: High publish failure rate - alert: HighPublishFailureRate expr: | sum(rate(caatsm_publish_failures_total[5m])) by (category) / sum(rate(caatsm_processed_total[5m])) by (category) > 0.05 for: 5m labels: severity: warning component: publisher annotations: summary: "High publish failure rate for {{ $labels.category }}: {{ $value | humanizePercentage }}" description: | More than 5% of {{ $labels.category }} messages failing to publish. Current failure rate: {{ $value | humanizePercentage }} Category: {{ $labels.category }} Check NATS JetStream connectivity and publisher logs. # Warning: DLQ publish failures (messages lost) - alert: DLQPublishFailures expr: rate(caatsm_dlq_publish_failures_total[5m]) > 0 for: 2m labels: severity: warning component: dlq annotations: summary: "DLQ publish failures detected" description: | Failed messages cannot be published to DLQ - messages may be lost! Failure rate: {{ $value | humanize }} msgs/sec Stream: {{ $labels.stream }} Consumer: {{ $labels.consumer }} Check DLQ subject configuration and NATS JetStream health. - name: caatsm_aftn_validation_details interval: 1m rules: # Recording rule: AFTN error rate by type - record: caatsm:aftn_validation_error_rate:5m expr: | rate(caatsm_aftn_validation_errors_total[5m]) # Recording rule: Total processing rate - record: caatsm:processing_rate:5m expr: | sum(rate(caatsm_processed_total[5m])) by (stream, consumer, status) # Recording rule: Average message gap - record: caatsm:message_gap_seconds:avg expr: | avg(caatsm_message_gap_seconds) by (stream, consumer)