✨ Enhance observability and error handling in NATS integration. Introduce comprehensive OpenTelemetry support with environment-based sampling and semantic attributes for tracing and metrics. Implement an advisory dead-letter queue (DLQ) handler for managing message delivery failures. Update NATS consumer to utilize structured logging and improve error handling strategies. Refactor configuration files for OpenTelemetry collector in both development and production environments, ensuring robust telemetry integration. Enhance documentation to reflect new features and best practices for observability.
This commit is contained in:
windyboy
@@ -0,0 +1,248 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"caatsm/internal/infra/buildinfo"
|
||||
"caatsm/internal/infra/config"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
|
||||
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
var (
|
||||
// instanceID stores a unique identifier for this running process instance.
|
||||
// It is initialized once at startup and remains stable for the lifetime of the process.
|
||||
instanceID string
|
||||
instanceIDOnce sync.Once
|
||||
)
|
||||
|
||||
// InitOTEL initializes the OpenTelemetry SDK with proper resource attributes,
|
||||
// sampling configuration, and exporters. This should be called once at application startup.
|
||||
// logger is optional; if provided, sensitive telemetry configuration will be logged at debug level.
|
||||
func InitOTEL(ctx context.Context, cfg *config.Config, logger *zap.Logger) error {
|
||||
if !cfg.Telemetry.Enabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Log sensitive telemetry configuration to internal debug logs only
|
||||
if logger != nil {
|
||||
logger.Debug("Initializing OpenTelemetry",
|
||||
zap.String("telemetry.endpoint", cfg.Telemetry.Endpoint),
|
||||
zap.Bool("telemetry.insecure", cfg.Telemetry.Insecure),
|
||||
)
|
||||
}
|
||||
|
||||
// Create resource with comprehensive service information
|
||||
res, err := createResource(ctx, cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create OTEL resource: %w", err)
|
||||
}
|
||||
|
||||
// Initialize tracing
|
||||
if err := initTracing(ctx, cfg, res); err != nil {
|
||||
return fmt.Errorf("failed to initialize tracing: %w", err)
|
||||
}
|
||||
|
||||
// Initialize metrics
|
||||
if err := initMetrics(ctx, cfg, res); err != nil {
|
||||
return fmt.Errorf("failed to initialize metrics: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getInstanceID returns a unique identifier for this running process instance.
|
||||
// It checks environment variables (POD_NAME, CONTAINER_ID, HOSTNAME) first, then generates
|
||||
// a UUID if no environment variable is available. The ID is initialized once and remains
|
||||
// stable for the lifetime of the process.
|
||||
func getInstanceID() string {
|
||||
instanceIDOnce.Do(func() {
|
||||
// Check for Kubernetes pod name first (most common in containerized deployments)
|
||||
if podName := os.Getenv("POD_NAME"); podName != "" {
|
||||
instanceID = podName
|
||||
return
|
||||
}
|
||||
|
||||
// Check for container ID (Docker, containerd, etc.)
|
||||
if containerID := os.Getenv("CONTAINER_ID"); containerID != "" {
|
||||
instanceID = containerID
|
||||
return
|
||||
}
|
||||
|
||||
// Check for HOSTNAME (often set in containers)
|
||||
if hostname := os.Getenv("HOSTNAME"); hostname != "" {
|
||||
// Use hostname if it's not a generic default
|
||||
if hostname != "localhost" && hostname != "localhost.localdomain" {
|
||||
instanceID = hostname
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Generate a UUID for this process instance
|
||||
instanceID = uuid.NewString()
|
||||
})
|
||||
|
||||
return instanceID
|
||||
}
|
||||
|
||||
// createResource creates a resource with standard and custom attributes.
|
||||
// Sensitive infrastructure details (endpoint, insecure flag) are excluded from resource
|
||||
// attributes to prevent leakage. These values are logged internally at debug level if
|
||||
// a logger is provided to InitOTEL.
|
||||
func createResource(ctx context.Context, cfg *config.Config) (*resource.Resource, error) {
|
||||
// Get the runtime instance ID (falls back to buildinfo.Commit if needed)
|
||||
runtimeInstanceID := getInstanceID()
|
||||
if runtimeInstanceID == "" {
|
||||
// Final fallback to build commit if somehow instance ID is empty
|
||||
runtimeInstanceID = buildinfo.Commit
|
||||
}
|
||||
|
||||
attrs := []attribute.KeyValue{
|
||||
// Standard semantic conventions
|
||||
semconv.ServiceName("caatsm"),
|
||||
semconv.ServiceVersion(buildinfo.Version),
|
||||
semconv.ServiceInstanceID(runtimeInstanceID),
|
||||
semconv.ServiceNamespace("airport"),
|
||||
|
||||
// Custom attributes
|
||||
attribute.String("service.component", "receiver"),
|
||||
attribute.String("service.environment", getEnvironment()),
|
||||
attribute.String("build.commit", buildinfo.Commit),
|
||||
attribute.String("build.built_at", buildinfo.BuiltAt),
|
||||
}
|
||||
|
||||
// Add non-sensitive indicator for telemetry endpoint configuration
|
||||
// (without exposing the actual endpoint value)
|
||||
if cfg.Telemetry.Endpoint != "" {
|
||||
attrs = append(attrs, attribute.Bool("telemetry.endpoint.configured", true))
|
||||
} else {
|
||||
attrs = append(attrs, attribute.Bool("telemetry.endpoint.configured", false))
|
||||
}
|
||||
|
||||
return resource.New(ctx, resource.WithAttributes(attrs...))
|
||||
}
|
||||
|
||||
// initTracing sets up the trace provider with appropriate sampling
|
||||
func initTracing(ctx context.Context, cfg *config.Config, res *resource.Resource) error {
|
||||
var traceExporterOptions []otlptracehttp.Option
|
||||
|
||||
traceExporterOptions = append(traceExporterOptions, otlptracehttp.WithEndpoint(cfg.Telemetry.Endpoint))
|
||||
|
||||
if cfg.Telemetry.Insecure {
|
||||
traceExporterOptions = append(traceExporterOptions, otlptracehttp.WithTLSClientConfig(&tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
MinVersion: tls.VersionTLS13,
|
||||
}))
|
||||
}
|
||||
|
||||
traceExporter, err := otlptracehttp.New(ctx, traceExporterOptions...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create trace exporter: %w", err)
|
||||
}
|
||||
|
||||
// Configure sampling based on environment
|
||||
sampler := getSamplerForEnvironment(getEnvironment())
|
||||
|
||||
tracerProvider := sdktrace.NewTracerProvider(
|
||||
sdktrace.WithBatcher(traceExporter,
|
||||
sdktrace.WithBatchTimeout(1*time.Second),
|
||||
sdktrace.WithMaxExportBatchSize(512),
|
||||
sdktrace.WithMaxQueueSize(2048),
|
||||
),
|
||||
sdktrace.WithResource(res),
|
||||
sdktrace.WithSampler(sdktrace.ParentBased(sampler)),
|
||||
)
|
||||
|
||||
otel.SetTracerProvider(tracerProvider)
|
||||
return nil
|
||||
}
|
||||
|
||||
// initMetrics sets up the meter provider
|
||||
func initMetrics(ctx context.Context, cfg *config.Config, res *resource.Resource) error {
|
||||
var metricExporterOptions []otlpmetrichttp.Option
|
||||
|
||||
metricExporterOptions = append(metricExporterOptions, otlpmetrichttp.WithEndpoint(cfg.Telemetry.Endpoint))
|
||||
|
||||
if cfg.Telemetry.Insecure {
|
||||
metricExporterOptions = append(metricExporterOptions, otlpmetrichttp.WithTLSClientConfig(&tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
MinVersion: tls.VersionTLS12,
|
||||
}))
|
||||
}
|
||||
|
||||
metricExporter, err := otlpmetrichttp.New(ctx, metricExporterOptions...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create metric exporter: %w", err)
|
||||
}
|
||||
|
||||
meterProvider := sdkmetric.NewMeterProvider(
|
||||
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExporter,
|
||||
sdkmetric.WithInterval(30*time.Second),
|
||||
)),
|
||||
sdkmetric.WithResource(res),
|
||||
)
|
||||
|
||||
otel.SetMeterProvider(meterProvider)
|
||||
return nil
|
||||
}
|
||||
|
||||
// getSamplerForEnvironment returns appropriate sampling strategy for each environment
|
||||
func getSamplerForEnvironment(env string) sdktrace.Sampler {
|
||||
switch env {
|
||||
case "prod", "production":
|
||||
// 1% sampling in production to control costs and performance
|
||||
return sdktrace.TraceIDRatioBased(0.01)
|
||||
case "staging":
|
||||
// 10% sampling in staging for better observability
|
||||
return sdktrace.TraceIDRatioBased(0.1)
|
||||
case "test", "testing":
|
||||
// Always sample in testing for complete coverage
|
||||
return sdktrace.AlwaysSample()
|
||||
default:
|
||||
// 100% sampling in development for debugging
|
||||
return sdktrace.AlwaysSample()
|
||||
}
|
||||
}
|
||||
|
||||
// ShutdownOTEL gracefully shuts down the OTEL providers
|
||||
func ShutdownOTEL(ctx context.Context) error {
|
||||
var errs []error
|
||||
|
||||
if tracerProvider, ok := otel.GetTracerProvider().(*sdktrace.TracerProvider); ok {
|
||||
if err := tracerProvider.Shutdown(ctx); err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to shutdown tracer provider: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
if meterProvider, ok := otel.GetMeterProvider().(*sdkmetric.MeterProvider); ok {
|
||||
if err := meterProvider.Shutdown(ctx); err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to shutdown meter provider: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
if len(errs) > 0 {
|
||||
return fmt.Errorf("OTEL shutdown errors: %v", errs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getEnvironment returns the current environment from GO_ENV
|
||||
func getEnvironment() string {
|
||||
if env := os.Getenv("GO_ENV"); env != "" {
|
||||
return env
|
||||
}
|
||||
return "dev"
|
||||
}
|
||||
Reference in New Issue
Block a user