✨ Enhance observability and error handling in NATS integration. Introduce comprehensive OpenTelemetry support with environment-based sampling and semantic attributes for tracing and metrics. Implement an advisory dead-letter queue (DLQ) handler for managing message delivery failures. Update NATS consumer to utilize structured logging and improve error handling strategies. Refactor configuration files for OpenTelemetry collector in both development and production environments, ensuring robust telemetry integration. Enhance documentation to reflect new features and best practices for observability.

configs/otel-collector.dev.yaml

@@ -3,27 +3,66 @@ receivers:
     protocols:
       http:
         endpoint: 0.0.0.0:4318
+        max_request_body_size: 20971520  # 20MB
+        max_concurrent_streams: 16
       grpc:
         endpoint: 0.0.0.0:4317
+        max_recv_msg_size: 4194304  # 4MB
+        max_concurrent_streams: 16
+
+processors:
+  batch:
+    send_batch_size: 1024
+    timeout: 1s
+    send_batch_max_size: 2048
+
+  resource:
+    attributes:
+      - key: service.instance.id
+        value: "${POD_NAME}"
+        action: upsert
+      - key: k8s.pod.name
+        value: "${POD_NAME}"
+        action: upsert
+      - key: k8s.namespace.name
+        value: "${NAMESPACE}"
+        action: upsert
 
 exporters:
   logging:
     loglevel: info
+    sampling_initial: 10
+    sampling_thereafter: 100
+
   otlphttp/jaeger:
     endpoint: http://jaeger:4318
     tls:
       insecure: true
+    sending_queue:
+      queue_size: 10000
+    retry_on_failure:
+      enabled: true
+      initial_interval: 5s
+      max_interval: 30s
+      max_elapsed_time: 300s
+
   prometheus:
     endpoint: "0.0.0.0:8889"
     const_labels:
       source: "otel-collector"
+    sending_queue:
+      queue_size: 10000
+    retry_on_failure:
+      enabled: true
 
 service:
   pipelines:
     traces:
       receivers: [otlp]
+      processors: [resource, batch]
       exporters: [logging, otlphttp/jaeger]
     metrics:
       receivers: [otlp]
+      processors: [resource, batch]
       exporters: [logging, prometheus]
 

configs/otel-collector.prod.yaml

@@ -0,0 +1,85 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+        tls:
+          cert_file: /etc/ssl/certs/otel-collector.crt
+          key_file: /etc/ssl/private/otel-collector.key
+        auth:
+          authenticator: bearer_token
+      http:
+        endpoint: 0.0.0.0:4318
+        tls:
+          cert_file: /etc/ssl/certs/otel-collector.crt
+          key_file: /etc/ssl/private/otel-collector.key
+        auth:
+          authenticator: bearer_token
+
+processors:
+  batch:
+    send_batch_size: 1024
+    timeout: 1s
+    send_batch_max_size: 2048
+
+  resource:
+    attributes:
+      - key: service.instance.id
+        value: "${POD_NAME}"
+        action: upsert
+      - key: k8s.pod.name
+        value: "${POD_NAME}"
+        action: upsert
+      - key: k8s.namespace.name
+        value: "${NAMESPACE}"
+        action: upsert
+
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13133
+  pprof:
+    endpoint: :1888
+  zpages:
+    endpoint: :55679
+  bearer_token:
+    token: ${OTEL_COLLECTOR_BEARER_TOKEN}
+
+exporters:
+  otlphttp/jaeger:
+    endpoint: https://jaeger.production.company.com:4318
+    headers:
+      authorization: "Bearer ${JAEGER_API_TOKEN}"
+    tls:
+      insecure: false
+      ca_file: /etc/ssl/certs/ca.pem
+    sending_queue:
+      queue_size: 10000
+    retry_on_failure:
+      enabled: true
+      initial_interval: 5s
+      max_interval: 30s
+      max_elapsed_time: 300s
+
+  prometheusremotewrite:
+    endpoint: https://prometheus-prod.company.com/api/v1/write
+    headers:
+      authorization: "Bearer ${PROMETHEUS_API_TOKEN}"
+    tls:
+      insecure: false
+      ca_file: /etc/ssl/certs/ca.pem
+    sending_queue:
+      queue_size: 10000
+    retry_on_failure:
+      enabled: true
+
+service:
+  extensions: [health_check, pprof, zpages, bearer_token]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [resource, batch]
+      exporters: [otlphttp/jaeger]
+    metrics:
+      receivers: [otlp]
+      processors: [resource, batch]
+      exporters: [prometheusremotewrite]