Update dependencies, enhance NATS consumer configuration, and improve error handling in message processing. Introduce telemetry support with OpenTelemetry for tracing and metrics. Refactor README to include new configuration options and update tests for improved coverage of error scenarios.

This commit is contained in:
windyboy
2025-11-15 09:01:24 +08:00
parent 31839cf21f
commit 359e7694ab
31 changed files with 1326 additions and 667 deletions
+71 -2
View File
@@ -80,6 +80,8 @@ Configuration is loaded from TOML files and environment variables. The configura
url = "nats://localhost:4222" url = "nats://localhost:4222"
stream = "TELEGRAM" stream = "TELEGRAM"
consumer = "telegram-consumer" consumer = "telegram-consumer"
client = "serial-client"
cluster = "tele-cluster"
[nats.stream_limits] [nats.stream_limits]
max_msgs = 100000 max_msgs = 100000
@@ -93,10 +95,16 @@ replicas = 1
max_deliver = 5 max_deliver = 5
ack_wait = "30s" ack_wait = "30s"
max_ack_pending = 1024 max_ack_pending = 1024
deliver_policy = "all" # all,new,last,last_per_subject,sequence,time
replay_policy = "instant" # instant or original
backoff = ["5s", "30s", "2m"] # optional JetStream redelivery delays
start_sequence = 0
start_time = ""
[subscription] [subscription]
# Optional. Defaults to "telegram.>" when omitted. # Optional. Defaults to "telegram.>" when omitted.
topic = "telegram.serial" topic = "telegram.serial"
queue_group = "tele-queue"
[publisher] [publisher]
topic = "telegram.json" topic = "telegram.json"
@@ -115,6 +123,11 @@ monitor_interval = "30s"
level = "info" level = "info"
format = "json" format = "json"
[telemetry]
enabled = false
endpoint = "http://otel-collector:4318"
insecure = true
### Timeouts and Ack Wait ### Timeouts and Ack Wait
`[timeouts]` is optional, but if you plan to tune JetStream redelivery you should set `timeouts.ack_wait` and/or `[nats.consumer].ack_wait`. When neither is specified the application defaults both values to `30s`, ensuring predictable redelivery timing. `[timeouts]` is optional, but if you plan to tune JetStream redelivery you should set `timeouts.ack_wait` and/or `[nats.consumer].ack_wait`. When neither is specified the application defaults both values to `30s`, ensuring predictable redelivery timing.
@@ -168,10 +181,48 @@ task run-dev
./bin/receiver listen --help ./bin/receiver listen --help
Flags: Flags:
-n, --nats string Nats server address (default: "nats://localhost:4222") -n, --nats-url string NATS server address
-t, --topic string Nats topic to listen to (default: "telegram.serial") -t, --subject string NATS subject to listen to
--stream string JetStream stream name
--consumer string JetStream durable consumer
--publisher-topic string Subject used by the publisher
--postgres-url string PostgreSQL connection URL
--log-level string Logger level (debug|info|warn|error)
--replay-from string Deliver policy override (all|new|last|seq:<n>|time:<RFC3339>)
--ack-wait duration Ack wait override (e.g. 45s)
--telemetry-enabled Enable OpenTelemetry exporters
--telemetry-endpoint string
OTLP collector endpoint
--telemetry-insecure Send OTLP traffic without TLS
``` ```
Critical overrides stay available through CLI flags; advanced tuning such as stream retention, consumer backoff, and copy counts are configured via the TOML file or `CAATSM_` environment variables.
| CLI flag | Config key | Purpose |
|---------------------|------------------------|----------------------------------------|
| `--nats-url` | `nats.url` | Point to a different NATS cluster |
| `--subject` | `subscription.topic` | Change the subscribed subject filter |
| `--stream` | `nats.stream` | Bind to another JetStream stream |
| `--consumer` | `nats.consumer` | Override the durable consumer name |
| `--publisher-topic` | `publisher.topic` | Publish parsed output to a new subject |
| `--postgres-url` | `postgres.url` | Redirect persistence to another DB |
| `--log-level` | `log.level` | Adjust runtime logging verbosity |
| `--telemetry-*` | `telemetry.*` | Toggle tracing/metrics exporters |
#### Replay & Backoff
- `--replay-from seq:12345` replays from a specific JetStream sequence, while `--replay-from time:2024-11-15T08:00:00Z` starts at a timestamp.
- Configure server-side retry delays with `[nats.consumer].backoff = ["5s", "30s", "2m"]`; each duration becomes the delay before the next delivery attempt.
- Combine `backoff` with `--ack-wait` to increase acknowledgement windows (e.g., `--ack-wait 2m`).
### Telemetry
- Enable tracing/metrics via `[telemetry] enabled = true` and set `endpoint` to your OTLP/HTTP collector (e.g., `http://otel-collector:4318`).
- CLI overrides:
- `--telemetry-enabled` flips the feature on/off.
- `--telemetry-endpoint` and `--telemetry-insecure` adjust the OTLP HTTP endpoint and TLS behavior.
- When enabled the app emits OpenTelemetry traces (parser/repository/publisher spans) and JetStream metrics (ack pending, deliveries) for dashboards and alerts.
## Development ## Development
### Project Structure ### Project Structure
@@ -196,6 +247,8 @@ Dependencies are managed using Google Wire. To add a new dependency:
2. Add it to `pkg/di/wire.go` 2. Add it to `pkg/di/wire.go`
3. Run `wire ./pkg/di` to regenerate `wire_gen.go` 3. Run `wire ./pkg/di` to regenerate `wire_gen.go`
> If the `wire` binary is missing, install it with `go install github.com/google/wire/cmd/wire@v0.7.0` and ensure `$GOPATH/bin` is on your `PATH` (or run it directly via the absolute path).
### Testing ### Testing
```bash ```bash
@@ -204,6 +257,9 @@ go test ./...
# Run tests with coverage # Run tests with coverage
task coverage task coverage
# Run all Ginkgo suites (requires go install github.com/onsi/ginkgo/v2/ginkgo@latest)
ginkgo -r
``` ```
## Message Flow ## Message Flow
@@ -216,6 +272,19 @@ task coverage
3. **ACK/NAK** is sent based on processing success/failure 3. **ACK/NAK** is sent based on processing success/failure
4. **Retry Logic** handles transient failures automatically 4. **Retry Logic** handles transient failures automatically
### Failure Buckets
Messages that cannot be parsed or fail to publish are written to `aviation.telegrams_raw` with a status:
| Status | Description |
|-----------------------|--------------------------------------------------|
| `parsed` | Successfully parsed and stored |
| `header_error` | Header invalid (missing start indicator, etc.) |
| `body_error` | Body pattern did not match any known format |
| `publish_error` | Downstream publisher returned an error |
Each entry stores the raw payload, received timestamp, and metadata to aid replay or manual inspection.
## Message Parsing ## Message Parsing
The system supports parsing of aviation telegram messages in the standard ICAO format. All messages follow a common header structure, followed by a message body that varies by message type. The system supports parsing of aviation telegram messages in the standard ICAO format. All messages follow a common header structure, followed by a message body that varies by message type.
+226 -11
View File
@@ -8,10 +8,21 @@ import (
"fmt" "fmt"
"os" "os"
"os/signal" "os/signal"
"strconv"
"strings"
"syscall" "syscall"
"time" "time"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
"go.opentelemetry.io/otel/propagation"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
) )
func main() { func main() {
@@ -32,19 +43,60 @@ func setupApp() *cli.App {
Usage: "Listen to nats messages", Usage: "Listen to nats messages",
Flags: []cli.Flag{ Flags: []cli.Flag{
&cli.StringFlag{ &cli.StringFlag{
Name: "nats", Name: "nats-url",
Aliases: []string{"n"}, Aliases: []string{"n"},
Usage: "Nats server address", Usage: "NATS server address",
Value: "nats://localhost:4222",
EnvVars: []string{"NATS_SERVER"}, EnvVars: []string{"NATS_SERVER"},
}, },
&cli.StringFlag{ &cli.StringFlag{
Name: "topic", Name: "subject",
Aliases: []string{"t"}, Aliases: []string{"t"},
Usage: "Nats topic to listen to", Usage: "NATS subject to listen to",
Value: "telegram.serial",
EnvVars: []string{"NATS_SUBJECT"}, EnvVars: []string{"NATS_SUBJECT"},
}, },
&cli.StringFlag{
Name: "stream",
Usage: "NATS JetStream stream name",
},
&cli.StringFlag{
Name: "consumer",
Usage: "NATS JetStream durable consumer",
},
&cli.StringFlag{
Name: "publisher-topic",
Usage: "Subject used by the publisher",
},
&cli.StringFlag{
Name: "postgres-url",
Usage: "PostgreSQL connection URL",
EnvVars: []string{
"POSTGRES_URL",
},
},
&cli.StringFlag{
Name: "log-level",
Usage: "Logger level (debug, info, warn, error)",
},
&cli.StringFlag{
Name: "replay-from",
Usage: "Override deliver policy: all|new|last|seq:<n>|time:<RFC3339>",
},
&cli.DurationFlag{
Name: "ack-wait",
Usage: "Override consumer ack wait duration",
},
&cli.BoolFlag{
Name: "telemetry-enabled",
Usage: "Enable OpenTelemetry exporters",
},
&cli.StringFlag{
Name: "telemetry-endpoint",
Usage: "OpenTelemetry collector OTLP endpoint",
},
&cli.BoolFlag{
Name: "telemetry-insecure",
Usage: "Send OTLP data without TLS",
},
}, },
Action: executeListen, Action: executeListen,
}, },
@@ -58,12 +110,16 @@ func executeListen(c *cli.Context) error {
return fmt.Errorf("failed to load config: %w", err) return fmt.Errorf("failed to load config: %w", err)
} }
if flagURL := c.String("nats"); flagURL != "" { applyCLIOverrides(cfg, c)
cfg.NATS.URL = flagURL
}
if flagTopic := c.String("topic"); flagTopic != "" { shutdownTelemetry := func(context.Context) error { return nil }
cfg.Subscription.Topic = flagTopic if cfg.Telemetry.Enabled {
var telErr error
shutdownTelemetry, telErr = initTelemetry(context.Background(), cfg)
if telErr != nil {
return fmt.Errorf("failed to initialize telemetry: %w", telErr)
}
defer shutdownTelemetry(context.Background())
} }
// Initialize dependencies using Wire // Initialize dependencies using Wire
@@ -123,3 +179,162 @@ func executeListen(c *cli.Context) error {
return runErr return runErr
} }
func applyCLIOverrides(cfg *config.Config, c *cli.Context) {
if cfg == nil || c == nil {
return
}
if flagURL := c.String("nats-url"); flagURL != "" {
cfg.NATS.URL = flagURL
}
if flagTopic := c.String("subject"); flagTopic != "" {
cfg.Subscription.Topic = flagTopic
}
if stream := c.String("stream"); stream != "" {
cfg.NATS.Stream = stream
}
if consumer := c.String("consumer"); consumer != "" {
cfg.NATS.Consumer = consumer
}
if publisherTopic := c.String("publisher-topic"); publisherTopic != "" {
cfg.Publisher.Topic = publisherTopic
}
if pgURL := c.String("postgres-url"); pgURL != "" {
cfg.Postgres.URL = pgURL
}
if logLevel := c.String("log-level"); logLevel != "" {
cfg.Log.Level = logLevel
}
if replay := c.String("replay-from"); replay != "" {
applyReplayOverride(cfg, replay)
}
if c.IsSet("ack-wait") {
if ack := c.Duration("ack-wait"); ack > 0 {
cfg.Timeouts.AckWait = ack
cfg.NATS.ConsumerRules.AckWait = ack
}
}
if c.IsSet("telemetry-enabled") {
cfg.Telemetry.Enabled = c.Bool("telemetry-enabled")
}
if endpoint := c.String("telemetry-endpoint"); endpoint != "" {
cfg.Telemetry.Endpoint = endpoint
}
if c.IsSet("telemetry-insecure") {
cfg.Telemetry.Insecure = c.Bool("telemetry-insecure")
}
}
func applyReplayOverride(cfg *config.Config, value string) {
if cfg == nil {
return
}
lower := strings.ToLower(strings.TrimSpace(value))
switch {
case lower == "new":
cfg.NATS.ConsumerRules.DeliverPolicy = "new"
cfg.NATS.ConsumerRules.StartSequence = 0
cfg.NATS.ConsumerRules.StartTime = ""
case lower == "all":
cfg.NATS.ConsumerRules.DeliverPolicy = "all"
cfg.NATS.ConsumerRules.StartSequence = 0
cfg.NATS.ConsumerRules.StartTime = ""
case lower == "last":
cfg.NATS.ConsumerRules.DeliverPolicy = "last"
cfg.NATS.ConsumerRules.StartSequence = 0
cfg.NATS.ConsumerRules.StartTime = ""
case lower == "last_per_subject":
cfg.NATS.ConsumerRules.DeliverPolicy = "last_per_subject"
cfg.NATS.ConsumerRules.StartSequence = 0
cfg.NATS.ConsumerRules.StartTime = ""
case strings.HasPrefix(lower, "seq:"):
seqStr := strings.TrimPrefix(lower, "seq:")
if seq, err := strconv.ParseUint(seqStr, 10, 64); err == nil {
cfg.NATS.ConsumerRules.DeliverPolicy = "sequence"
cfg.NATS.ConsumerRules.StartSequence = seq
cfg.NATS.ConsumerRules.StartTime = ""
}
case strings.HasPrefix(lower, "time:"):
ts := strings.TrimSpace(value[5:])
if _, err := time.Parse(time.RFC3339, ts); err == nil {
cfg.NATS.ConsumerRules.DeliverPolicy = "time"
cfg.NATS.ConsumerRules.StartSequence = 0
cfg.NATS.ConsumerRules.StartTime = ts
}
}
}
func initTelemetry(ctx context.Context, cfg *config.Config) (func(context.Context) error, error) {
if cfg == nil || !cfg.Telemetry.Enabled {
return func(context.Context) error { return nil }, nil
}
if cfg.Telemetry.Endpoint == "" {
return nil, fmt.Errorf("telemetry endpoint is required when telemetry.enabled=true")
}
traceOpts := []otlptracehttp.Option{
otlptracehttp.WithEndpoint(cfg.Telemetry.Endpoint),
otlptracehttp.WithURLPath("/v1/traces"),
}
metricOpts := []otlpmetrichttp.Option{
otlpmetrichttp.WithEndpoint(cfg.Telemetry.Endpoint),
otlpmetrichttp.WithURLPath("/v1/metrics"),
}
if cfg.Telemetry.Insecure {
traceOpts = append(traceOpts, otlptracehttp.WithInsecure())
metricOpts = append(metricOpts, otlpmetrichttp.WithInsecure())
}
traceExporter, err := otlptracehttp.New(ctx, traceOpts...)
if err != nil {
return nil, fmt.Errorf("init trace exporter: %w", err)
}
metricExporter, err := otlpmetrichttp.New(ctx, metricOpts...)
if err != nil {
return nil, fmt.Errorf("init metric exporter: %w", err)
}
env := os.Getenv("GO_ENV")
if env == "" {
env = "dev"
}
res, err := resource.New(ctx,
resource.WithFromEnv(),
resource.WithProcess(),
resource.WithOS(),
resource.WithHost(),
resource.WithAttributes(
semconv.ServiceName("caatsm"),
attribute.String("deployment.environment", env),
),
)
if err != nil {
return nil, fmt.Errorf("build telemetry resource: %w", err)
}
tp := sdktrace.NewTracerProvider(
sdktrace.WithBatcher(traceExporter),
sdktrace.WithResource(res),
)
mp := sdkmetric.NewMeterProvider(
sdkmetric.WithResource(res),
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExporter)),
)
otel.SetTracerProvider(tp)
otel.SetMeterProvider(mp)
otel.SetTextMapPropagator(propagation.TraceContext{})
shutdown := func(ctx context.Context) error {
errs := []error{}
if err := mp.Shutdown(ctx); err != nil {
errs = append(errs, err)
}
if err := tp.Shutdown(ctx); err != nil {
errs = append(errs, err)
}
return errors.Join(errs...)
}
return shutdown, nil
}
+13
View File
@@ -0,0 +1,13 @@
package main
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestMainCmd(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "cmd/main Suite")
}
+114
View File
@@ -0,0 +1,114 @@
package main
import (
"flag"
"time"
appconfig "caatsm/internal/infra/config"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/urfave/cli/v2"
)
var _ = Describe("CLI overrides", func() {
var (
cfg *appconfig.Config
flagSet *flag.FlagSet
args []string
)
BeforeEach(func() {
cfg = &appconfig.Config{
NATS: appconfig.NATSConfig{
URL: "nats://base:4222",
Stream: "BASE_STREAM",
Consumer: "base-consumer",
},
Subscription: appconfig.SubscriptionConfig{
Topic: "base.subject",
},
Publisher: appconfig.PublisherConfig{
Topic: "base.publisher",
},
Postgres: appconfig.PostgresConfig{
URL: "postgres://base",
},
Log: appconfig.LogConfig{
Level: "info",
},
Telemetry: appconfig.TelemetryConfig{
Enabled: false,
Endpoint: "",
Insecure: false,
},
}
flagSet = flag.NewFlagSet("test", flag.ContinueOnError)
flagSet.String("nats-url", "", "")
flagSet.String("subject", "", "")
flagSet.String("stream", "", "")
flagSet.String("consumer", "", "")
flagSet.String("publisher-topic", "", "")
flagSet.String("postgres-url", "", "")
flagSet.String("log-level", "", "")
flagSet.String("replay-from", "", "")
flagSet.Duration("ack-wait", 0, "")
flagSet.Bool("telemetry-enabled", false, "")
flagSet.String("telemetry-endpoint", "", "")
flagSet.Bool("telemetry-insecure", false, "")
args = []string{
"--nats-url", "nats://override:4222",
"--subject", "override.subject",
"--stream", "STREAM",
"--consumer", "CONSUMER",
"--publisher-topic", "pub.topic",
"--postgres-url", "postgres://override",
"--log-level", "debug",
"--replay-from", "seq:42",
"--ack-wait", "45s",
"--telemetry-enabled",
"--telemetry-endpoint", "http://otel:4318",
"--telemetry-insecure",
}
})
JustBeforeEach(func() {
err := flagSet.Parse(args)
Expect(err).NotTo(HaveOccurred())
ctx := cli.NewContext(cli.NewApp(), flagSet, nil)
applyCLIOverrides(cfg, ctx)
})
It("applies overrides for all critical flags", func() {
Expect(cfg.NATS.URL).To(Equal("nats://override:4222"))
Expect(cfg.Subscription.Topic).To(Equal("override.subject"))
Expect(cfg.NATS.Stream).To(Equal("STREAM"))
Expect(cfg.NATS.Consumer).To(Equal("CONSUMER"))
Expect(cfg.Publisher.Topic).To(Equal("pub.topic"))
Expect(cfg.Postgres.URL).To(Equal("postgres://override"))
Expect(cfg.Log.Level).To(Equal("debug"))
Expect(cfg.NATS.ConsumerRules.DeliverPolicy).To(Equal("sequence"))
Expect(cfg.NATS.ConsumerRules.StartSequence).To(Equal(uint64(42)))
Expect(cfg.NATS.ConsumerRules.AckWait).To(Equal(45 * time.Second))
Expect(cfg.Telemetry.Enabled).To(BeTrue())
Expect(cfg.Telemetry.Endpoint).To(Equal("http://otel:4318"))
Expect(cfg.Telemetry.Insecure).To(BeTrue())
})
Context("with time-based replay override", func() {
BeforeEach(func() {
args = []string{
"--replay-from", "time:2024-11-15T08:00:00Z",
}
})
It("configures deliver policy and start time", func() {
Expect(cfg.NATS.ConsumerRules.DeliverPolicy).To(Equal("time"))
Expect(cfg.NATS.ConsumerRules.StartTime).To(Equal("2024-11-15T08:00:00Z"))
Expect(cfg.NATS.ConsumerRules.StartSequence).To(Equal(uint64(0)))
})
})
})
+10 -4
View File
@@ -17,10 +17,15 @@ replicas = 1
max_deliver = 5 max_deliver = 5
ack_wait = "30s" ack_wait = "30s"
max_ack_pending = 1024 max_ack_pending = 1024
deliver_policy = "all"
replay_policy = "instant"
backoff = ["5s", "30s", "2m"]
start_sequence = 0
start_time = ""
[subscription] [subscription]
topic = "telegram.serial" topic = "telegram.serial"
queue = "tele-queue" queue_group = "tele-queue"
[publisher] [publisher]
topic = "telegram.json" topic = "telegram.json"
@@ -45,6 +50,7 @@ monitor_interval = "30s"
level = "info" level = "info"
format = "json" format = "json"
[hasura] [telemetry]
endpoint = "http://localhost:8080/v1/graphql" enabled = false
secret = "aviation-test" endpoint = "http://otel-collector:4318"
insecure = true
+20 -10
View File
@@ -11,23 +11,32 @@ require (
github.com/knadh/koanf/providers/file v1.2.0 github.com/knadh/koanf/providers/file v1.2.0
github.com/knadh/koanf/v2 v2.3.0 github.com/knadh/koanf/v2 v2.3.0
github.com/nats-io/nats.go v1.47.0 github.com/nats-io/nats.go v1.47.0
github.com/onsi/ginkgo/v2 v2.25.1 github.com/onsi/ginkgo/v2 v2.27.2
github.com/onsi/gomega v1.38.2 github.com/onsi/gomega v1.38.2
github.com/spf13/viper v1.21.0
github.com/urfave/cli/v2 v2.27.7 github.com/urfave/cli/v2 v2.27.7
go.opentelemetry.io/otel v1.38.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0
go.opentelemetry.io/otel/metric v1.38.0
go.opentelemetry.io/otel/sdk v1.38.0
go.opentelemetry.io/otel/sdk/metric v1.38.0
go.opentelemetry.io/otel/trace v1.38.0
go.uber.org/zap v1.27.0 go.uber.org/zap v1.27.0
gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1
) )
require ( require (
github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/google/go-cmp v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect github.com/google/pprof v0.0.0-20251007162407-5df77e3f7d1d // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect
@@ -38,21 +47,22 @@ require (
github.com/nats-io/nkeys v0.4.11 // indirect github.com/nats-io/nkeys v0.4.11 // indirect
github.com/nats-io/nuid v1.0.1 // indirect github.com/nats-io/nuid v1.0.1 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sagikazarmark/locafero v0.12.0 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect
go.uber.org/automaxprocs v1.6.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
go.uber.org/multierr v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.44.0 // indirect golang.org/x/crypto v0.44.0 // indirect
golang.org/x/mod v0.30.0 // indirect
golang.org/x/net v0.47.0 // indirect golang.org/x/net v0.47.0 // indirect
golang.org/x/sync v0.18.0 // indirect golang.org/x/sync v0.18.0 // indirect
golang.org/x/sys v0.38.0 // indirect golang.org/x/sys v0.38.0 // indirect
golang.org/x/text v0.31.0 // indirect golang.org/x/text v0.31.0 // indirect
golang.org/x/tools v0.39.0 // indirect golang.org/x/tools v0.39.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect
google.golang.org/grpc v1.76.0 // indirect
google.golang.org/protobuf v1.36.10 // indirect
) )
+13
View File
@@ -0,0 +1,13 @@
package mapper
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestMapper(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Adapter Mapper Suite")
}
+31 -6
View File
@@ -43,6 +43,11 @@ func (m *TelegramMapper) ToDBRow(msg *domain.ParsedMessage) ([]interface{}, erro
// SecondaryAddresses is already a string, so we can use it directly // SecondaryAddresses is already a string, so we can use it directly
secondaryAddresses := msg.SecondaryAddresses secondaryAddresses := msg.SecondaryAddresses
status := msg.Status
if status == "" {
status = domain.MessageStatusUnknown
}
return []interface{}{ return []interface{}{
msgUUID, // uuid msgUUID, // uuid
msg.MessageID, // message_id msg.MessageID, // message_id
@@ -55,6 +60,8 @@ func (m *TelegramMapper) ToDBRow(msg *domain.ParsedMessage) ([]interface{}, erro
msg.Category, // category msg.Category, // category
msg.Content, // content (TEXT, original message) msg.Content, // content (TEXT, original message)
bodyDataJSON, // body_data (JSONB) bodyDataJSON, // body_data (JSONB)
string(status), // status
msg.ErrorReason, // error_reason
msg.ReceivedAt, // received_at msg.ReceivedAt, // received_at
msg.ParsedAt, // parsed_at msg.ParsedAt, // parsed_at
msg.DispatchedAt, // dispatched_at msg.DispatchedAt, // dispatched_at
@@ -64,7 +71,7 @@ func (m *TelegramMapper) ToDBRow(msg *domain.ParsedMessage) ([]interface{}, erro
// FromDBRow converts a database row to a domain.ParsedMessage // FromDBRow converts a database row to a domain.ParsedMessage
func (m *TelegramMapper) FromDBRow(row []interface{}) (*domain.ParsedMessage, error) { func (m *TelegramMapper) FromDBRow(row []interface{}) (*domain.ParsedMessage, error) {
const expectedColumns = 15 const expectedColumns = 17
if len(row) < expectedColumns { if len(row) < expectedColumns {
return nil, fmt.Errorf("expected %d columns, got %d", expectedColumns, len(row)) return nil, fmt.Errorf("expected %d columns, got %d", expectedColumns, len(row))
} }
@@ -94,6 +101,19 @@ func (m *TelegramMapper) FromDBRow(row []interface{}) (*domain.ParsedMessage, er
return time.Time{} return time.Time{}
} }
toBool := func(v interface{}) bool {
switch val := v.(type) {
case bool:
return val
case *bool:
return val != nil && *val
case int64:
return val != 0
default:
return false
}
}
var bodyData interface{} var bodyData interface{}
if raw := row[10]; raw != nil { if raw := row[10]; raw != nil {
switch val := raw.(type) { switch val := raw.(type) {
@@ -108,7 +128,10 @@ func (m *TelegramMapper) FromDBRow(row []interface{}) (*domain.ParsedMessage, er
} }
} }
needDispatch, _ := row[14].(bool) status := domain.MessageStatusUnknown
if rawStatus := toString(row[11]); rawStatus != "" {
status = domain.MessageStatus(rawStatus)
}
return &domain.ParsedMessage{ return &domain.ParsedMessage{
Uuid: msgUUID.String(), Uuid: msgUUID.String(),
@@ -122,9 +145,11 @@ func (m *TelegramMapper) FromDBRow(row []interface{}) (*domain.ParsedMessage, er
Category: toString(row[8]), Category: toString(row[8]),
Content: toString(row[9]), Content: toString(row[9]),
BodyData: bodyData, BodyData: bodyData,
ReceivedAt: parseTime(row[11]), Status: status,
ParsedAt: parseTime(row[12]), ErrorReason: toString(row[12]),
DispatchedAt: parseTime(row[13]), ReceivedAt: parseTime(row[13]),
NeedDispatch: needDispatch, ParsedAt: parseTime(row[14]),
DispatchedAt: parseTime(row[15]),
NeedDispatch: toBool(row[16]),
}, nil }, nil
} }
+29 -32
View File
@@ -1,36 +1,38 @@
package mapper package mapper
import ( import (
"testing"
"time" "time"
"caatsm/internal/domain" "caatsm/internal/domain"
"github.com/google/uuid" "github.com/google/uuid"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
) )
func TestTelegramMapper_ToDBRow_GeneratesUUIDWhenEmpty(t *testing.T) { var _ = Describe("TelegramMapper", func() {
mapper := NewTelegramMapper() var mapper *TelegramMapper
BeforeEach(func() {
mapper = NewTelegramMapper()
})
Describe("ToDBRow", func() {
It("generates a UUID when missing", func() {
msg := &domain.ParsedMessage{} msg := &domain.ParsedMessage{}
row, err := mapper.ToDBRow(msg) row, err := mapper.ToDBRow(msg)
if err != nil { Expect(err).NotTo(HaveOccurred())
t.Fatalf("unexpected error: %v", err)
}
value, ok := row[0].(uuid.UUID) value, ok := row[0].(uuid.UUID)
if !ok { Expect(ok).To(BeTrue())
t.Fatalf("expected first column to be uuid.UUID, got %T", row[0]) Expect(value).NotTo(Equal(uuid.Nil))
} })
if value == uuid.Nil { })
t.Fatalf("expected generated uuid to be non-nil")
}
}
func TestTelegramMapper_FromDBRow_RoundTrip(t *testing.T) { Describe("FromDBRow", func() {
mapper := NewTelegramMapper() It("round-trips telegram data", func() {
now := time.Now().UTC() now := time.Now().UTC()
original := &domain.ParsedMessage{ original := &domain.ParsedMessage{
Uuid: uuid.NewString(), Uuid: uuid.NewString(),
MessageID: "TMQ1324", MessageID: "TMQ1324",
@@ -47,25 +49,20 @@ func TestTelegramMapper_FromDBRow_RoundTrip(t *testing.T) {
ParsedAt: now, ParsedAt: now,
DispatchedAt: now, DispatchedAt: now,
NeedDispatch: true, NeedDispatch: true,
Status: domain.MessageStatusParsed,
} }
row, err := mapper.ToDBRow(original) row, err := mapper.ToDBRow(original)
if err != nil { Expect(err).NotTo(HaveOccurred())
t.Fatalf("unexpected error: %v", err) Expect(row).To(HaveLen(17))
}
roundTrip, err := mapper.FromDBRow(row) roundTrip, err := mapper.FromDBRow(row)
if err != nil { Expect(err).NotTo(HaveOccurred())
t.Fatalf("unexpected error reading row: %v", err)
}
if roundTrip.Uuid != original.Uuid { Expect(roundTrip.Uuid).To(Equal(original.Uuid))
t.Fatalf("expected uuid %s, got %s", original.Uuid, roundTrip.Uuid) Expect(roundTrip.MessageID).To(Equal(original.MessageID))
} Expect(roundTrip.NeedDispatch).To(Equal(original.NeedDispatch))
if roundTrip.MessageID != original.MessageID { Expect(roundTrip.Status).To(Equal(original.Status))
t.Fatalf("expected message_id %s, got %s", original.MessageID, roundTrip.MessageID) })
} })
if roundTrip.NeedDispatch != original.NeedDispatch { })
t.Fatalf("expected need_dispatch %v, got %v", original.NeedDispatch, roundTrip.NeedDispatch)
}
}
+1 -2
View File
@@ -14,8 +14,7 @@ func NewAviationParser() *AviationParser {
} }
// Parse parses a raw message string and returns a ParsedMessage // Parse parses a raw message string and returns a ParsedMessage
func (p *AviationParser) Parse(rawText string) *domain.ParsedMessage { func (p *AviationParser) Parse(rawText string) (*domain.ParsedMessage, error) {
// Use the existing Parse function from internal/parsers // Use the existing Parse function from internal/parsers
return parsers.Parse(rawText) return parsers.Parse(rawText)
} }
+1 -3
View File
@@ -5,7 +5,5 @@ import "caatsm/internal/domain"
// Parser defines the interface for parsing raw telegram messages // Parser defines the interface for parsing raw telegram messages
type Parser interface { type Parser interface {
// Parse parses a raw message string and returns a ParsedMessage // Parse parses a raw message string and returns a ParsedMessage
Parse(rawText string) *domain.ParsedMessage Parse(rawText string) (*domain.ParsedMessage, error)
// TODO: consider returning (*domain.ParsedMessage, error) to surface parse failures explicitly.
} }
+3
View File
@@ -12,5 +12,8 @@ type Repository interface {
// InsertBatch inserts multiple telegram messages in a batch // InsertBatch inserts multiple telegram messages in a batch
InsertBatch(ctx context.Context, msgs []*domain.ParsedMessage) error InsertBatch(ctx context.Context, msgs []*domain.ParsedMessage) error
// InsertRaw captures an unparsed or failed telegram for later analysis.
InsertRaw(ctx context.Context, msg *domain.ParsedMessage) error
} }
+15 -23
View File
@@ -2,32 +2,24 @@ package app
import ( import (
"errors" "errors"
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
) )
func TestPermanentWrapsError(t *testing.T) { var _ = Describe("Permanent errors", func() {
It("wraps errors and reports permanence", func() {
base := errors.New("boom") base := errors.New("boom")
perr := Permanent(base) perr := Permanent(base)
if perr == nil { Expect(perr).NotTo(BeNil())
t.Fatalf("expected wrapped error, got nil") Expect(IsPermanent(perr)).To(BeTrue())
} Expect(errors.Is(perr, base)).To(BeTrue())
if !IsPermanent(perr) { Expect(errors.Is(base, perr)).To(BeFalse())
t.Fatalf("expected IsPermanent to be true") })
}
if !errors.Is(perr, base) {
t.Fatalf("expected wrapped error to unwrap to base")
}
if errors.Is(base, perr) {
t.Fatalf("expected base not to consider wrapper as same")
}
}
func TestPermanentNil(t *testing.T) { It("treats nil as non-permanent", func() {
if Permanent(nil) != nil { Expect(Permanent(nil)).To(BeNil())
t.Fatalf("Permanent(nil) should return nil") Expect(IsPermanent(nil)).To(BeFalse())
} })
if IsPermanent(nil) { })
t.Fatalf("IsPermanent(nil) should be false")
}
}
+82 -10
View File
@@ -3,11 +3,16 @@ package app
import ( import (
"caatsm/internal/adapter" "caatsm/internal/adapter"
"caatsm/internal/adapter/parser" "caatsm/internal/adapter/parser"
"caatsm/internal/domain"
"context" "context"
"fmt" "fmt"
"strings" "strings"
"time" "time"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap" "go.uber.org/zap"
) )
@@ -40,11 +45,20 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
return Permanent(fmt.Errorf("empty message")) return Permanent(fmt.Errorf("empty message"))
} }
tracer := otel.Tracer("caatsm/app")
ctx, span := tracer.Start(ctx, "MessageProcessor.Handle")
defer span.End()
span.SetAttributes(attribute.String("nats.msg_id", msgID))
receivedAt := time.Now() receivedAt := time.Now()
parsed := p.parser.Parse(string(raw)) parsed, parseErr := p.parser.Parse(string(raw))
if parsed == nil { if parsed == nil {
return Permanent(fmt.Errorf("parser returned nil")) parsed = domain.NewParsedMessage()
parsed.Content = string(raw)
parsed.ErrorReason = "parser returned nil"
parsed.Status = domain.MessageStatusBodyError
parseErr = fmt.Errorf("parser returned nil")
} }
if msgID != "" { if msgID != "" {
@@ -60,16 +74,38 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
if parsed.ParsedAt.IsZero() { if parsed.ParsedAt.IsZero() {
parsed.ParsedAt = time.Now() parsed.ParsedAt = time.Now()
} }
if parsed.Status == domain.MessageStatusUnknown {
if parseErr == nil {
parsed.Status = domain.MessageStatusParsed
} else {
parsed.Status = domain.MessageStatusBodyError
}
}
if parseErr != nil || !parsed.Parsed {
if parsed.ErrorReason == "" && parseErr != nil {
parsed.ErrorReason = parseErr.Error()
}
span.RecordError(parseErr)
span.SetStatus(codes.Error, parseErr.Error())
p.persistRaw(ctx, parsed)
p.logger.Warn("Message failed to parse",
zap.String("msg_id", msgID),
zap.String("status", string(parsed.Status)),
zap.String("content_preview", truncateContent(parsed.Content, 256)),
zap.Error(parseErr),
)
return Permanent(fmt.Errorf("parser error: %w", parseErr))
}
parsed.ErrorReason = ""
// Log parsing result // Log parsing result
if !parsed.Parsed { span.SetAttributes(
p.logger.Warn("Message not parsed", attribute.String("telegram.status", string(parsed.Status)),
zap.String("msg_id", msgID), attribute.Bool("telegram.parsed", parsed.Parsed),
zap.String("message_id", parsed.MessageID), attribute.String("telegram.category", parsed.Category),
zap.String("category", parsed.Category),
zap.String("content_preview", truncateContent(parsed.Content, 256)),
) )
} else {
p.logger.Info("Message parsed successfully", p.logger.Info("Message parsed successfully",
zap.String("msg_id", msgID), zap.String("msg_id", msgID),
zap.String("message_id", parsed.MessageID), zap.String("message_id", parsed.MessageID),
@@ -77,27 +113,63 @@ func (p *MessageProcessor) Handle(ctx context.Context, raw []byte, msgID string)
zap.Time("received_at", parsed.ReceivedAt), zap.Time("received_at", parsed.ReceivedAt),
zap.Time("parsed_at", parsed.ParsedAt), zap.Time("parsed_at", parsed.ParsedAt),
) )
}
// Insert into database // Insert into database
if err := p.repository.InsertOne(ctx, parsed); err != nil { if err := p.repository.InsertOne(ctx, parsed); err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("failed to insert message: %w", err) return fmt.Errorf("failed to insert message: %w", err)
} }
// Publish parsed message // Publish parsed message
_, pubSpan := tracer.Start(ctx, "Publisher.Publish")
if err := p.publisher.Publish(parsed); err != nil { if err := p.publisher.Publish(parsed); err != nil {
// Log error but don't fail the entire operation // Log error but don't fail the entire operation
p.logger.Error("Failed to publish message", p.logger.Error("Failed to publish message",
zap.String("msg_id", msgID), zap.String("msg_id", msgID),
zap.Error(err), zap.Error(err),
) )
pubSpan.RecordError(err)
pubSpan.SetStatus(codes.Error, err.Error())
parsed.Status = domain.MessageStatusPublishFail
parsed.ErrorReason = err.Error()
p.persistRaw(ctx, parsed)
// Mark as permanent so the consumer will ack instead of retrying // Mark as permanent so the consumer will ack instead of retrying
pubSpan.End()
return Permanent(fmt.Errorf("failed to publish message: %w", err)) return Permanent(fmt.Errorf("failed to publish message: %w", err))
} }
pubSpan.End()
return nil return nil
} }
func (p *MessageProcessor) persistRaw(ctx context.Context, msg *domain.ParsedMessage) {
if msg == nil || p.repository == nil {
return
}
if msg.Content == "" && msg.BodyData != nil {
msg.Content = fmt.Sprintf("%v", msg.BodyData)
}
if msg.ReceivedAt.IsZero() {
msg.ReceivedAt = time.Now()
}
if err := p.repository.InsertRaw(ctx, msg); err != nil {
p.logger.Error("Failed to persist raw telegram",
zap.String("message_id", msg.MessageID),
zap.String("status", string(msg.Status)),
zap.Error(err),
)
} else {
if span := trace.SpanFromContext(ctx); span.IsRecording() {
span.AddEvent("raw telegram persisted",
trace.WithAttributes(
attribute.String("telegram.status", string(msg.Status)),
attribute.String("telegram.message_id", msg.MessageID),
))
}
}
}
func truncateContent(content string, limit int) string { func truncateContent(content string, limit int) string {
if limit <= 0 || len(content) <= limit { if limit <= 0 || len(content) <= limit {
return content return content
+118 -126
View File
@@ -4,7 +4,6 @@ import (
"context" "context"
"errors" "errors"
"strings" "strings"
"testing"
"time" "time"
"caatsm/internal/adapter" "caatsm/internal/adapter"
@@ -12,166 +11,137 @@ import (
"caatsm/internal/domain" "caatsm/internal/domain"
"github.com/google/uuid" "github.com/google/uuid"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"go.uber.org/zap" "go.uber.org/zap"
"go.uber.org/zap/zaptest/observer" "go.uber.org/zap/zaptest/observer"
) )
func TestHandleEmptyMessageIsPermanent(t *testing.T) { var _ = Describe("MessageProcessor", func() {
proc := newTestProcessor(&stubParser{}, &stubRepository{}, &stubPublisher{}) var (
err := proc.Handle(context.Background(), nil, "id-1") repo *stubRepository
if err == nil || !IsPermanent(err) { pub *stubPublisher
t.Fatalf("expected permanent error for empty message, got %v", err) proc *MessageProcessor
} ctx context.Context
} parserStub *stubParser
)
func TestHandleNilParserResultIsPermanent(t *testing.T) { BeforeEach(func() {
proc := newTestProcessor(&stubParser{value: nil}, &stubRepository{}, &stubPublisher{}) repo = &stubRepository{}
err := proc.Handle(context.Background(), []byte("payload"), "id-2") pub = &stubPublisher{}
if err == nil || !IsPermanent(err) { parserStub = &stubParser{}
t.Fatalf("expected permanent error for nil parser result, got %v", err) proc = newTestProcessor(parserStub, repo, pub)
} ctx = context.Background()
} })
func TestHandleSuccessDoesNotOverwriteUuid(t *testing.T) { Describe("Handle", func() {
It("returns a permanent error when payload is empty", func() {
err := proc.Handle(ctx, nil, "id-1")
Expect(err).To(HaveOccurred())
Expect(IsPermanent(err)).To(BeTrue())
})
It("records raw messages when parser returns nil", func() {
parserStub.value = nil
err := proc.Handle(ctx, []byte("payload"), "id-2")
Expect(err).To(HaveOccurred())
Expect(IsPermanent(err)).To(BeTrue())
Expect(repo.rawCount()).To(Equal(1))
})
It("preserves UUIDs and appends nats message id comment", func() {
originalUUID := uuid.NewString() originalUUID := uuid.NewString()
parsed := &domain.ParsedMessage{Uuid: originalUUID, Parsed: true} parserStub.value = &domain.ParsedMessage{Uuid: originalUUID, Parsed: true, Status: domain.MessageStatusParsed}
repo := &stubRepository{} Expect(proc.Handle(ctx, []byte("payload"), "msg-123")).To(Succeed())
pub := &stubPublisher{}
proc := newTestProcessor(&stubParser{value: parsed}, repo, pub)
const msgID = "msg-123" Expect(repo.last()).NotTo(BeNil())
if err := proc.Handle(context.Background(), []byte("payload"), msgID); err != nil { Expect(repo.last().Uuid).To(Equal(originalUUID))
t.Fatalf("expected success, got %v", err) Expect(repo.last().Comments).To(ContainSubstring("nats_msg_id=msg-123"))
} Expect(pub.last).NotTo(BeNil())
})
if repo.last() == nil { It("treats publisher failures as permanent and stores raw entries", func() {
t.Fatalf("expected message to be inserted") parserStub.value = &domain.ParsedMessage{Parsed: true, Status: domain.MessageStatusParsed}
} pub.err = errors.New("publish failed")
if repo.last().Uuid != originalUUID {
t.Fatalf("expected uuid to remain %s, got %s", originalUUID, repo.last().Uuid)
}
if !strings.Contains(repo.last().Comments, "nats_msg_id=msg-123") {
t.Fatalf("expected comments to contain msg id, got %q", repo.last().Comments)
}
if pub.last == nil {
t.Fatalf("expected publisher to receive message")
}
}
func TestHandlePublisherErrorIsPermanent(t *testing.T) { err := proc.Handle(ctx, []byte("payload"), "id-3")
parsed := &domain.ParsedMessage{Parsed: true} Expect(err).To(HaveOccurred())
Expect(IsPermanent(err)).To(BeTrue())
Expect(repo.last()).NotTo(BeNil())
Expect(repo.rawCount()).To(Equal(1))
Expect(repo.lastRaw().Status).To(Equal(domain.MessageStatusPublishFail))
})
repo := &stubRepository{} It("sets timestamps when missing", func() {
pub := &stubPublisher{err: errors.New("publish failed")} parserStub.value = &domain.ParsedMessage{
proc := newTestProcessor(&stubParser{value: parsed}, repo, pub)
err := proc.Handle(context.Background(), []byte("payload"), "id-3")
if err == nil {
t.Fatalf("expected error when publisher fails")
}
if !IsPermanent(err) {
t.Fatalf("publisher failure should be permanent")
}
if repo.last() == nil {
t.Fatalf("expected message to insert before publish failure")
}
}
func TestMessageProcessor_Handle_SetsReceivedAndParsedAtWhenZero(t *testing.T) {
parsed := &domain.ParsedMessage{
Uuid: uuid.NewString(), Uuid: uuid.NewString(),
Parsed: true, Parsed: true,
Status: domain.MessageStatusParsed,
} }
repo := &stubRepository{} pub.err = nil
publisher := &stubPublisher{}
logger := zap.NewNop()
processor := NewMessageProcessor(&stubParser{value: parsed}, repo, publisher, logger)
start := time.Now() start := time.Now()
if err := processor.Handle(context.Background(), []byte("raw"), "msg-4"); err != nil { Expect(proc.Handle(ctx, []byte("payload"), "msg-4")).To(Succeed())
t.Fatalf("unexpected error: %v", err)
}
saved := repo.last() saved := repo.last()
if saved == nil { Expect(saved).NotTo(BeNil())
t.Fatal("expected repository to receive a message") Expect(saved.ReceivedAt).NotTo(BeZero())
} Expect(saved.ParsedAt).NotTo(BeZero())
if saved.ReceivedAt.IsZero() || saved.ParsedAt.IsZero() { Expect(saved.ReceivedAt.After(start.Add(-time.Second))).To(BeTrue())
t.Fatalf("expected timestamps to be set, got received=%v parsed=%v", saved.ReceivedAt, saved.ParsedAt) Expect(saved.ParsedAt.After(start.Add(-time.Second))).To(BeTrue())
} })
if saved.ReceivedAt.Before(start.Add(-time.Second)) || saved.ParsedAt.Before(start.Add(-time.Second)) {
t.Fatalf("timestamps look stale: received=%v parsed=%v", saved.ReceivedAt, saved.ParsedAt)
}
}
func TestMessageProcessor_Handle_DoesNotOverrideExistingTimestamps(t *testing.T) { It("does not override provided timestamps", func() {
received := time.Now().Add(-2 * time.Minute) received := time.Now().Add(-2 * time.Minute)
parsedAt := time.Now().Add(-time.Minute) parsedAt := time.Now().Add(-1 * time.Minute)
parserStub.value = &domain.ParsedMessage{
parsed := &domain.ParsedMessage{
Uuid: uuid.NewString(), Uuid: uuid.NewString(),
Parsed: true, Parsed: true,
Status: domain.MessageStatusParsed,
ReceivedAt: received, ReceivedAt: received,
ParsedAt: parsedAt, ParsedAt: parsedAt,
} }
repo := &stubRepository{}
publisher := &stubPublisher{}
logger := zap.NewNop()
processor := NewMessageProcessor(&stubParser{value: parsed}, repo, publisher, logger)
if err := processor.Handle(context.Background(), []byte("raw"), "msg-5"); err != nil { Expect(proc.Handle(ctx, []byte("payload"), "msg-5")).To(Succeed())
t.Fatalf("unexpected error: %v", err) Expect(repo.last().ReceivedAt).To(Equal(received))
} Expect(repo.last().ParsedAt).To(Equal(parsedAt))
saved := repo.last() })
if saved.ReceivedAt != received {
t.Fatalf("expected received_at to remain %v, got %v", received, saved.ReceivedAt)
}
if saved.ParsedAt != parsedAt {
t.Fatalf("expected parsed_at to remain %v, got %v", parsedAt, saved.ParsedAt)
}
}
func TestContentPreview_TruncatesLongContent(t *testing.T) { It("logs truncated previews when parsing fails", func() {
longContent := strings.Repeat("a", 1024)
preview := truncateContent(longContent, 256)
if len(preview) != 256 {
t.Fatalf("expected preview length 256, got %d", len(preview))
}
if !strings.HasSuffix(preview, "...") {
t.Fatalf("expected preview to end with ellipsis, got %q", preview[len(preview)-10:])
}
}
func TestMessageProcessor_Handle_NotParsedLogsPreviewOnly(t *testing.T) {
core, logs := observer.New(zap.WarnLevel) core, logs := observer.New(zap.WarnLevel)
logger := zap.New(core) logger := zap.New(core)
parserStub = &stubParser{
parser := &stubParser{
value: &domain.ParsedMessage{ value: &domain.ParsedMessage{
Content: strings.Repeat("x", 1024), Content: strings.Repeat("x", 1024),
Parsed: false, Parsed: false,
Status: domain.MessageStatusBodyError,
ErrorReason: "parse failure",
}, },
err: errors.New("parse failure"),
} }
repo := &stubRepository{} proc = NewMessageProcessor(parserStub, repo, pub, logger)
publisher := &stubPublisher{}
processor := NewMessageProcessor(parser, repo, publisher, logger)
if err := processor.Handle(context.Background(), []byte("raw"), "msg-6"); err != nil { err := proc.Handle(ctx, []byte("raw"), "msg-6")
t.Fatalf("unexpected error: %v", err) Expect(err).To(HaveOccurred())
} Expect(IsPermanent(err)).To(BeTrue())
entries := logs.FilterMessage("Message not parsed").All() entries := logs.FilterMessage("Message failed to parse").All()
if len(entries) == 0 { Expect(entries).NotTo(BeEmpty())
t.Fatal("expected a warning log for unparsed message")
}
preview, ok := entries[0].ContextMap()["content_preview"].(string) preview, ok := entries[0].ContextMap()["content_preview"].(string)
if !ok { Expect(ok).To(BeTrue())
t.Fatal("expected content_preview field in log") Expect(len(preview)).To(BeNumerically("<=", 256))
} })
if len(preview) > 256 { })
t.Fatalf("expected preview <= 256 chars, got %d", len(preview))
} Describe("truncateContent", func() {
} It("keeps length at limit with ellipsis", func() {
longContent := strings.Repeat("a", 1024)
Expect(truncateContent(longContent, 256)).To(HaveLen(256))
Expect(truncateContent(longContent, 256)).To(HaveSuffix("..."))
})
})
})
func newTestProcessor(p parser.Parser, repo adapter.Repository, pub adapter.Publisher) *MessageProcessor { func newTestProcessor(p parser.Parser, repo adapter.Repository, pub adapter.Publisher) *MessageProcessor {
return NewMessageProcessor(p, repo, pub, zap.NewNop()) return NewMessageProcessor(p, repo, pub, zap.NewNop())
@@ -179,15 +149,18 @@ func newTestProcessor(p parser.Parser, repo adapter.Repository, pub adapter.Publ
type stubParser struct { type stubParser struct {
value *domain.ParsedMessage value *domain.ParsedMessage
err error
} }
func (s *stubParser) Parse(rawText string) *domain.ParsedMessage { func (s *stubParser) Parse(rawText string) (*domain.ParsedMessage, error) {
return s.value return s.value, s.err
} }
type stubRepository struct { type stubRepository struct {
inserted []*domain.ParsedMessage inserted []*domain.ParsedMessage
raw []*domain.ParsedMessage
err error err error
rawErr error
} }
func (s *stubRepository) InsertOne(ctx context.Context, msg *domain.ParsedMessage) error { func (s *stubRepository) InsertOne(ctx context.Context, msg *domain.ParsedMessage) error {
@@ -202,6 +175,14 @@ func (s *stubRepository) InsertBatch(ctx context.Context, msgs []*domain.ParsedM
return errors.New("not implemented") return errors.New("not implemented")
} }
func (s *stubRepository) InsertRaw(ctx context.Context, msg *domain.ParsedMessage) error {
if s.rawErr != nil {
return s.rawErr
}
s.raw = append(s.raw, msg)
return nil
}
func (s *stubRepository) last() *domain.ParsedMessage { func (s *stubRepository) last() *domain.ParsedMessage {
if len(s.inserted) == 0 { if len(s.inserted) == 0 {
return nil return nil
@@ -209,6 +190,17 @@ func (s *stubRepository) last() *domain.ParsedMessage {
return s.inserted[len(s.inserted)-1] return s.inserted[len(s.inserted)-1]
} }
func (s *stubRepository) lastRaw() *domain.ParsedMessage {
if len(s.raw) == 0 {
return nil
}
return s.raw[len(s.raw)-1]
}
func (s *stubRepository) rawCount() int {
return len(s.raw)
}
type stubPublisher struct { type stubPublisher struct {
last interface{} last interface{}
err error err error
+13
View File
@@ -0,0 +1,13 @@
package app
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestApp(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "App Suite")
}
-124
View File
@@ -1,124 +0,0 @@
package config
import (
"fmt"
"os"
"regexp"
"strings"
"time"
"github.com/spf13/viper"
)
var MyConfig *Config
type Config struct {
Nats NatsConfig
Subscription SubscriptionConfig
Publisher PublisherConfig
Timeouts TimeoutsConfig
Hasura HasuraConfig
}
type NatsConfig struct {
Client string
URL string
Cluster string
}
type SubscriptionConfig struct {
Topic string `mapstructure:"topic"`
QueueGroup string `mapstructure:"queue_group"`
}
type PublisherConfig struct {
Topic string `mapstructure:"topic"`
}
type TimeoutsConfig struct {
Server time.Duration `mapstructure:"server"`
ReconnectWait time.Duration `mapstructure:"reconnect_wait"`
Close time.Duration `mapstructure:"close"`
AckWait time.Duration `mapstructure:"ack_wait"`
}
type BodyConfig struct {
Patterns []PatternConfig
}
type PatternConfig struct {
Pattern string
Comments string
Expression *regexp.Regexp
}
type HasuraConfig struct {
Endpoint string
Secret string
}
const (
EnvProd = "prod"
EnvDev = "dev"
EnvTest = "test"
)
func SetMyConfig(cfg *Config) {
MyConfig = cfg
}
func GetMyConfig() *Config {
if MyConfig == nil {
cfg, err := LoadConfig()
if err != nil {
fmt.Printf("error loading config: %v", err)
}
MyConfig = cfg
}
return MyConfig
}
// LoadConfig loads the configuration from a file
func LoadConfig() (*Config, error) {
// log := utils.Logger
env := os.Getenv("GO_ENV")
if env == "" {
env = "dev"
}
// log.Infof("Environment: %s", env)
viper.SetConfigType("toml")
viper.SetConfigName("config." + env)
viper.AddConfigPath("configs")
viper.SetEnvPrefix("tele")
viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
if err := viper.ReadInConfig(); err != nil {
// log.Errorf("error reading config file for environment '%s': %v", env, err)
return nil, fmt.Errorf("error reading config file for environment '%s': %w", env, err)
}
var config Config
if err := viper.Unmarshal(&config); err != nil {
// log.Errorf("unable to decode config into struct for environment '%s': %v", env, err)
return nil, fmt.Errorf("unable to decode config into struct for environment '%s': %w", env, err)
}
return &config, nil
}
// ValidateConfig validates the loaded configuration
func ValidateConfig(cfg *Config) error {
// log := utils.Logger
if cfg.Nats.Client == "" {
return fmt.Errorf("nats client is required")
}
if cfg.Nats.URL == "" {
return fmt.Errorf("nats URL is required")
}
if cfg.Subscription.Topic == "" {
return fmt.Errorf("subscription topic is required")
}
// fmt.Println("config validation passed")
return nil
}
-112
View File
@@ -1,112 +0,0 @@
package config
import (
"os"
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/spf13/viper"
)
func TestConfig(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Config Suite")
}
var _ = Describe("Config", func() {
var originalEnv string
BeforeEach(func() {
// Save the original GO_ENV value
originalEnv = os.Getenv("GO_ENV")
// Set up a temporary configuration file for testing
viper.Reset()
viper.SetConfigType("toml")
configContent := `
[nats]
client = "test-client"
url = "nats://localhost:4222"
cluster = "test-cluster"
[subscription]
topic = "example-topic"
queue_group = "example-group"
[timeouts]
server_timeout = "30s"
reconnect_wait = "10s"
close_timeout = "10s"
ack_wait_timeout = "5s"
[hasura]
endpoint = "http://localhost:8080/v1/graphql"
secret = "aviation-test"
`
tmpFile, err := os.CreateTemp("", "config.*.toml")
Expect(err).NotTo(HaveOccurred())
_, err = tmpFile.Write([]byte(configContent))
Expect(err).NotTo(HaveOccurred())
err = tmpFile.Close()
Expect(err).NotTo(HaveOccurred())
viper.SetConfigFile(tmpFile.Name())
err = viper.ReadInConfig()
Expect(err).NotTo(HaveOccurred())
// Load the configuration
MyConfig = &Config{}
err = viper.Unmarshal(MyConfig)
Expect(err).NotTo(HaveOccurred())
})
AfterEach(func() {
// Restore the original GO_ENV value
os.Setenv("GO_ENV", originalEnv)
})
Context("Loading configuration", func() {
It("should load the configuration correctly", func() {
cfg := GetMyConfig()
Expect(cfg).NotTo(BeNil())
Expect(cfg.Nats.Client).To(Equal("test-client"))
Expect(cfg.Nats.URL).To(Equal("nats://localhost:4222"))
Expect(cfg.Subscription.Topic).To(Equal("example-topic"))
Expect(cfg.Subscription.QueueGroup).To(Equal("example-group"))
Expect(cfg.Hasura.Endpoint).To(Equal("http://localhost:8080/v1/graphql"))
Expect(cfg.Hasura.Secret).To(Equal("aviation-test"))
})
})
Context("Validating configuration", func() {
It("should validate a valid configuration", func() {
cfg := GetMyConfig()
err := ValidateConfig(cfg)
Expect(err).NotTo(HaveOccurred())
})
It("should return an error for missing NATS client", func() {
cfg := GetMyConfig()
cfg.Nats.Client = ""
err := ValidateConfig(cfg)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(Equal("nats client is required"))
})
It("should return an error for missing NATS URL", func() {
cfg := GetMyConfig()
cfg.Nats.URL = ""
err := ValidateConfig(cfg)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(Equal("nats URL is required"))
})
It("should return an error for missing subscription topic", func() {
cfg := GetMyConfig()
cfg.Subscription.Topic = ""
err := ValidateConfig(cfg)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(Equal("subscription topic is required"))
})
})
})
+15 -1
View File
@@ -84,6 +84,18 @@ DispatchedAt: time.Time{}.
NeedDispatch: false. NeedDispatch: false.
*/ */
// ParsedMessage holds the parsed data from an aviation message
type MessageStatus string
const (
MessageStatusUnknown MessageStatus = "unknown"
MessageStatusParsed MessageStatus = "parsed"
MessageStatusHeaderError MessageStatus = "header_error"
MessageStatusBodyError MessageStatus = "body_error"
MessageStatusRepositoryFail MessageStatus = "repository_error"
MessageStatusPublishFail MessageStatus = "publish_error"
)
// ParsedMessage holds the parsed data from an aviation message // ParsedMessage holds the parsed data from an aviation message
type ParsedMessage struct { type ParsedMessage struct {
// StartIndicator string `json:"startIndicator"` // 电报开始标识: The start of the message indicator (e.g., 'ZCZC'). // StartIndicator string `json:"startIndicator"` // 电报开始标识: The start of the message indicator (e.g., 'ZCZC').
@@ -105,7 +117,8 @@ type ParsedMessage struct {
NeedDispatch bool `json:"needDispatch"` // 需要分发: Indicates if the message needs to be dispatched. NeedDispatch bool `json:"needDispatch"` // 需要分发: Indicates if the message needs to be dispatched.
Parsed bool `json:"parsed"` // 解析: Indicates if the message has been parsed. Parsed bool `json:"parsed"` // 解析: Indicates if the message has been parsed.
Comments string `json:"comments,omitempty"` // 备注: Additional comments. Comments string `json:"comments,omitempty"` // 备注: Additional comments.
Status MessageStatus
ErrorReason string `json:"errorReason,omitempty"`
} }
// NewParsedMessage initializes a ParsedMessage with default values // NewParsedMessage initializes a ParsedMessage with default values
@@ -113,6 +126,7 @@ func NewParsedMessage() *ParsedMessage {
return &ParsedMessage{ return &ParsedMessage{
// SecondaryAddresses: []string{}, // SecondaryAddresses: []string{},
Parsed: false, Parsed: false,
Status: MessageStatusUnknown,
} }
} }
+1 -1
View File
@@ -1,8 +1,8 @@
package iface package iface
import ( import (
"caatsm/internal/config"
"caatsm/internal/domain" "caatsm/internal/domain"
"caatsm/internal/infra/config"
) )
type MessageHandler interface { type MessageHandler interface {
+45
View File
@@ -19,6 +19,7 @@ type Config struct {
App AppConfig `koanf:"app"` App AppConfig `koanf:"app"`
Log LogConfig `koanf:"log"` Log LogConfig `koanf:"log"`
Publisher PublisherConfig `koanf:"publisher"` Publisher PublisherConfig `koanf:"publisher"`
Telemetry TelemetryConfig `koanf:"telemetry"`
// Legacy fields for backward compatibility during migration // Legacy fields for backward compatibility during migration
Subscription SubscriptionConfig `koanf:"subscription"` Subscription SubscriptionConfig `koanf:"subscription"`
Timeouts TimeoutsConfig `koanf:"timeouts"` Timeouts TimeoutsConfig `koanf:"timeouts"`
@@ -51,6 +52,11 @@ type ConsumerRulesConfig struct {
MaxDeliver int `koanf:"max_deliver"` MaxDeliver int `koanf:"max_deliver"`
AckWait time.Duration `koanf:"ack_wait"` AckWait time.Duration `koanf:"ack_wait"`
MaxAckPending int `koanf:"max_ack_pending"` MaxAckPending int `koanf:"max_ack_pending"`
DeliverPolicy string `koanf:"deliver_policy"`
ReplayPolicy string `koanf:"replay_policy"`
Backoff []time.Duration `koanf:"backoff"`
StartSequence uint64 `koanf:"start_sequence"`
StartTime string `koanf:"start_time"`
} }
// PostgresConfig holds PostgreSQL configuration // PostgresConfig holds PostgreSQL configuration
@@ -78,6 +84,13 @@ type PublisherConfig struct {
Topic string `koanf:"topic"` Topic string `koanf:"topic"`
} }
// TelemetryConfig controls tracing/metrics exporters.
type TelemetryConfig struct {
Enabled bool `koanf:"enabled"`
Endpoint string `koanf:"endpoint"`
Insecure bool `koanf:"insecure"`
}
// SubscriptionConfig holds subscription configuration (legacy) // SubscriptionConfig holds subscription configuration (legacy)
type SubscriptionConfig struct { type SubscriptionConfig struct {
Topic string `koanf:"topic"` Topic string `koanf:"topic"`
@@ -183,6 +196,15 @@ func LoadConfig() (*Config, error) {
if cfg.NATS.ConsumerRules.MaxAckPending == 0 { if cfg.NATS.ConsumerRules.MaxAckPending == 0 {
cfg.NATS.ConsumerRules.MaxAckPending = 1024 cfg.NATS.ConsumerRules.MaxAckPending = 1024
} }
if cfg.NATS.ConsumerRules.DeliverPolicy == "" {
cfg.NATS.ConsumerRules.DeliverPolicy = "all"
}
if cfg.NATS.ConsumerRules.ReplayPolicy == "" {
cfg.NATS.ConsumerRules.ReplayPolicy = "instant"
}
if cfg.Telemetry.Endpoint == "" {
cfg.Telemetry.Endpoint = ""
}
// Validate configuration // Validate configuration
if err := cfg.Validate(); err != nil { if err := cfg.Validate(); err != nil {
@@ -236,6 +258,29 @@ func (c *Config) Validate() error {
if c.NATS.ConsumerRules.MaxAckPending < 0 { if c.NATS.ConsumerRules.MaxAckPending < 0 {
return fmt.Errorf("nats.consumer.max_ack_pending must be >= 0") return fmt.Errorf("nats.consumer.max_ack_pending must be >= 0")
} }
switch strings.ToLower(c.NATS.ConsumerRules.DeliverPolicy) {
case "", "all", "new", "last", "last_per_subject", "sequence", "time":
default:
return fmt.Errorf("nats.consumer.deliver_policy must be one of all,new,last,last_per_subject,sequence,time")
}
switch strings.ToLower(c.NATS.ConsumerRules.ReplayPolicy) {
case "", "instant", "original":
default:
return fmt.Errorf("nats.consumer.replay_policy must be instant or original")
}
if c.NATS.ConsumerRules.StartTime != "" {
if _, err := time.Parse(time.RFC3339, c.NATS.ConsumerRules.StartTime); err != nil {
return fmt.Errorf("nats.consumer.start_time must be RFC3339: %w", err)
}
}
for _, d := range c.NATS.ConsumerRules.Backoff {
if d < 0 {
return fmt.Errorf("nats.consumer.backoff durations must be >= 0")
}
}
if c.Telemetry.Endpoint == "" && c.Telemetry.Enabled {
return fmt.Errorf("telemetry.endpoint is required when telemetry.enabled=true")
}
return nil return nil
} }
+26 -23
View File
@@ -3,35 +3,38 @@ package config
import ( import (
"os" "os"
"path/filepath" "path/filepath"
"testing"
"time" "time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
) )
func TestLoadConfig_DefaultAckWait(t *testing.T) { var _ = Describe("LoadConfig", func() {
t.Setenv("GO_ENV", "testdefaults") var (
originalWD string
)
wd, err := os.Getwd() BeforeEach(func() {
if err != nil { Expect(os.Setenv("GO_ENV", "testdefaults")).To(Succeed())
t.Fatalf("failed to get working dir: %v", err)
} var err error
repoRoot := filepath.Clean(filepath.Join(wd, "..", "..", "..")) originalWD, err = os.Getwd()
if err := os.Chdir(repoRoot); err != nil { Expect(err).NotTo(HaveOccurred())
t.Fatalf("failed to chdir to repo root: %v", err)
} repoRoot := filepath.Clean(filepath.Join(originalWD, "..", "..", ".."))
t.Cleanup(func() { Expect(os.Chdir(repoRoot)).To(Succeed())
_ = os.Chdir(wd)
}) })
AfterEach(func() {
Expect(os.Chdir(originalWD)).To(Succeed())
})
It("defaults ack waits when not provided", func() {
cfg, err := LoadConfig() cfg, err := LoadConfig()
if err != nil { Expect(err).NotTo(HaveOccurred())
t.Fatalf("failed to load config: %v", err)
}
want := 30 * time.Second want := 30 * time.Second
if cfg.Timeouts.AckWait != want { Expect(cfg.Timeouts.AckWait).To(Equal(want))
t.Fatalf("expected timeouts.ack_wait to default to %v, got %v", want, cfg.Timeouts.AckWait) Expect(cfg.NATS.ConsumerRules.AckWait).To(Equal(want))
} })
if cfg.NATS.ConsumerRules.AckWait != want { })
t.Fatalf("expected consumer ack_wait to default to %v, got %v", want, cfg.NATS.ConsumerRules.AckWait)
}
}
+13
View File
@@ -0,0 +1,13 @@
package config
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestInfraConfig(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Infra Config Suite")
}
+135 -3
View File
@@ -6,9 +6,14 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"strings"
"time" "time"
"github.com/nats-io/nats.go" "github.com/nats-io/nats.go"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/metric"
"go.uber.org/zap" "go.uber.org/zap"
) )
@@ -21,6 +26,11 @@ type Consumer struct {
logger *zap.Logger logger *zap.Logger
subject string subject string
consumerName string consumerName string
meter metric.Meter
ackPending metric.Int64Histogram
redelivered metric.Int64Histogram
pending metric.Int64Histogram
delivered metric.Int64Histogram
} }
// ProvideConsumer creates a NATS consumer // ProvideConsumer creates a NATS consumer
@@ -47,6 +57,7 @@ func ProvideConsumer(
subject: subject, subject: subject,
consumerName: consumerName, consumerName: consumerName,
} }
consumer.initMetrics()
// Create consumer if it doesn't exist // Create consumer if it doesn't exist
if err := consumer.ensureConsumer(); err != nil { if err := consumer.ensureConsumer(); err != nil {
@@ -74,12 +85,28 @@ func (c *Consumer) ensureConsumer() error {
consumerConfig := &nats.ConsumerConfig{ consumerConfig := &nats.ConsumerConfig{
Durable: c.consumerName, Durable: c.consumerName,
DeliverPolicy: nats.DeliverAllPolicy, DeliverPolicy: mapDeliverPolicy(c.cfg.NATS.ConsumerRules.DeliverPolicy),
AckPolicy: nats.AckExplicitPolicy, AckPolicy: nats.AckExplicitPolicy,
AckWait: ackWait, AckWait: ackWait,
ReplayPolicy: mapReplayPolicy(c.cfg.NATS.ConsumerRules.ReplayPolicy),
MaxDeliver: c.cfg.NATS.ConsumerRules.MaxDeliver, MaxDeliver: c.cfg.NATS.ConsumerRules.MaxDeliver,
MaxAckPending: c.cfg.NATS.ConsumerRules.MaxAckPending, MaxAckPending: c.cfg.NATS.ConsumerRules.MaxAckPending,
FilterSubject: c.subject, FilterSubject: c.subject,
BackOff: c.cfg.NATS.ConsumerRules.Backoff,
}
if consumerConfig.DeliverPolicy == nats.DeliverByStartSequencePolicy && c.cfg.NATS.ConsumerRules.StartSequence > 0 {
consumerConfig.OptStartSeq = c.cfg.NATS.ConsumerRules.StartSequence
}
if consumerConfig.DeliverPolicy == nats.DeliverByStartTimePolicy && strings.TrimSpace(c.cfg.NATS.ConsumerRules.StartTime) != "" {
startTime, err := time.Parse(time.RFC3339, c.cfg.NATS.ConsumerRules.StartTime)
if err != nil {
c.logger.Warn("Invalid start time, falling back to deliver policy defaults",
zap.String("start_time", c.cfg.NATS.ConsumerRules.StartTime),
zap.Error(err),
)
} else {
consumerConfig.OptStartTime = &startTime
}
} }
_, err := c.js.AddConsumer(streamName, consumerConfig) _, err := c.js.AddConsumer(streamName, consumerConfig)
@@ -93,6 +120,8 @@ func (c *Consumer) ensureConsumer() error {
zap.String("stream", streamName), zap.String("stream", streamName),
zap.String("subject", c.subject), zap.String("subject", c.subject),
zap.Duration("ack_wait", ackWait), zap.Duration("ack_wait", ackWait),
zap.String("deliver_policy", c.cfg.NATS.ConsumerRules.DeliverPolicy),
zap.String("replay_policy", c.cfg.NATS.ConsumerRules.ReplayPolicy),
) )
} }
@@ -133,6 +162,9 @@ func (c *Consumer) Start(ctx context.Context) error {
zap.Duration("batch_timeout", batchTimeout), zap.Duration("batch_timeout", batchTimeout),
zap.Int("max_deliver", c.cfg.NATS.ConsumerRules.MaxDeliver), zap.Int("max_deliver", c.cfg.NATS.ConsumerRules.MaxDeliver),
zap.Duration("ack_wait", c.cfg.NATS.ConsumerRules.AckWait), zap.Duration("ack_wait", c.cfg.NATS.ConsumerRules.AckWait),
zap.String("deliver_policy", c.cfg.NATS.ConsumerRules.DeliverPolicy),
zap.String("replay_policy", c.cfg.NATS.ConsumerRules.ReplayPolicy),
zap.Int("backoff_steps", len(c.cfg.NATS.ConsumerRules.Backoff)),
) )
statsCtx, statsCancel := context.WithCancel(ctx) statsCtx, statsCancel := context.WithCancel(ctx)
@@ -177,8 +209,8 @@ func (c *Consumer) Start(ctx context.Context) error {
continue continue
} }
// Transient error: request redelivery // Transient error: request redelivery with optional delay
if nakErr := msg.Nak(); nakErr != nil { if nakErr := c.nakWithStrategy(msg); nakErr != nil {
c.logger.Error("Failed to NAK message", zap.Error(nakErr)) c.logger.Error("Failed to NAK message", zap.Error(nakErr))
} }
continue continue
@@ -221,6 +253,7 @@ func (c *Consumer) emitConsumerStats(ctx context.Context, streamName string) {
zap.Uint64("delivered_consumer_seq", uint64(info.Delivered.Consumer)), zap.Uint64("delivered_consumer_seq", uint64(info.Delivered.Consumer)),
zap.Uint64("delivered_stream_seq", uint64(info.Delivered.Stream)), zap.Uint64("delivered_stream_seq", uint64(info.Delivered.Stream)),
) )
c.recordConsumerMetrics(ctx, info)
} }
} }
} }
@@ -254,10 +287,80 @@ func (c *Consumer) Shutdown(ctx context.Context) error {
} }
} }
func (c *Consumer) initMetrics() {
meter := otel.Meter("caatsm/nats")
c.meter = meter
if hist, err := meter.Int64Histogram("nats.consumer.ack_pending"); err == nil {
c.ackPending = hist
}
if hist, err := meter.Int64Histogram("nats.consumer.redelivered"); err == nil {
c.redelivered = hist
}
if hist, err := meter.Int64Histogram("nats.consumer.pending"); err == nil {
c.pending = hist
}
if hist, err := meter.Int64Histogram("nats.consumer.delivered"); err == nil {
c.delivered = hist
}
}
func (c *Consumer) recordConsumerMetrics(ctx context.Context, info *nats.ConsumerInfo) {
if info == nil {
return
}
if c.ackPending != nil {
c.ackPending.Record(ctx, int64(info.NumAckPending))
}
if c.redelivered != nil {
c.redelivered.Record(ctx, int64(info.NumRedelivered))
}
if c.pending != nil {
c.pending.Record(ctx, int64(info.NumPending))
}
if c.delivered != nil {
c.delivered.Record(ctx, int64(info.Delivered.Stream))
}
}
func (c *Consumer) nakWithStrategy(msg *nats.Msg) error {
backoff := c.cfg.NATS.ConsumerRules.Backoff
if len(backoff) == 0 {
return msg.Nak()
}
meta, err := msg.Metadata()
if err != nil {
c.logger.Warn("Failed to read metadata for backoff strategy", zap.Error(err))
return msg.Nak()
}
attempt := int(meta.NumDelivered)
index := attempt - 1
if index < 0 {
index = 0
}
if index >= len(backoff) {
index = len(backoff) - 1
}
delay := backoff[index]
if delay <= 0 {
return msg.Nak()
}
return msg.NakWithDelay(delay)
}
// processMessage processes a single message // processMessage processes a single message
func (c *Consumer) processMessage(ctx context.Context, msg *nats.Msg) error { func (c *Consumer) processMessage(ctx context.Context, msg *nats.Msg) error {
ctx, span := otel.Tracer("caatsm/nats").Start(ctx, "Consumer.processMessage")
defer span.End()
span.SetAttributes(attribute.String("nats.subject", msg.Subject))
msgID, source, err := c.resolveMsgID(msg) msgID, source, err := c.resolveMsgID(msg)
if err != nil { if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("unable to resolve message id: %w", err) return fmt.Errorf("unable to resolve message id: %w", err)
} }
if source != "header" { if source != "header" {
@@ -276,9 +379,12 @@ func (c *Consumer) processMessage(ctx context.Context, msg *nats.Msg) error {
// Call processor // Call processor
if err := c.processor.Handle(ctx, msg.Data, msgID); err != nil { if err := c.processor.Handle(ctx, msg.Data, msgID); err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("processor error: %w", err) return fmt.Errorf("processor error: %w", err)
} }
span.SetAttributes(attribute.String("telegram.msg_id", msgID))
return nil return nil
} }
@@ -294,3 +400,29 @@ func (c *Consumer) resolveMsgID(msg *nats.Msg) (string, string, error) {
return fmt.Sprintf("js-%d", meta.Sequence.Stream), "metadata", nil return fmt.Sprintf("js-%d", meta.Sequence.Stream), "metadata", nil
} }
func mapDeliverPolicy(value string) nats.DeliverPolicy {
switch strings.ToLower(value) {
case "new":
return nats.DeliverNewPolicy
case "last":
return nats.DeliverLastPolicy
case "last_per_subject":
return nats.DeliverLastPerSubjectPolicy
case "sequence":
return nats.DeliverByStartSequencePolicy
case "time":
return nats.DeliverByStartTimePolicy
default:
return nats.DeliverAllPolicy
}
}
func mapReplayPolicy(value string) nats.ReplayPolicy {
switch strings.ToLower(value) {
case "original":
return nats.ReplayOriginalPolicy
default:
return nats.ReplayInstantPolicy
}
}
+93 -4
View File
@@ -5,10 +5,16 @@ import (
"caatsm/internal/adapter/mapper" "caatsm/internal/adapter/mapper"
"caatsm/internal/domain" "caatsm/internal/domain"
"context" "context"
"encoding/json"
"fmt" "fmt"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool" "github.com/jackc/pgx/v5/pgxpool"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.uber.org/zap" "go.uber.org/zap"
) )
@@ -30,8 +36,14 @@ func ProvideRepository(pool *pgxpool.Pool, logger *zap.Logger) (adapter.Reposito
// InsertOne inserts a single telegram message // InsertOne inserts a single telegram message
func (r *Repository) InsertOne(ctx context.Context, msg *domain.ParsedMessage) error { func (r *Repository) InsertOne(ctx context.Context, msg *domain.ParsedMessage) error {
ctx, span := otel.Tracer("caatsm/postgres").Start(ctx, "Repository.InsertOne")
defer span.End()
span.SetAttributes(attribute.String("db.table", "aviation.telegrams"))
row, err := r.mapper.ToDBRow(msg) row, err := r.mapper.ToDBRow(msg)
if err != nil { if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("failed to map message to DB row: %w", err) return fmt.Errorf("failed to map message to DB row: %w", err)
} }
@@ -39,18 +51,21 @@ func (r *Repository) InsertOne(ctx context.Context, msg *domain.ParsedMessage) e
INSERT INTO aviation.telegrams ( INSERT INTO aviation.telegrams (
uuid, message_id, date_time, priority_indicator, primary_address, uuid, message_id, date_time, priority_indicator, primary_address,
secondary_addresses, originator, originator_date_time, category, secondary_addresses, originator, originator_date_time, category,
content, body_data, received_at, parsed_at, dispatched_at, need_dispatch content, body_data, status, error_reason,
received_at, parsed_at, dispatched_at, need_dispatch
) VALUES ( ) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15 $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17
) )
ON CONFLICT (uuid) DO NOTHING ON CONFLICT (uuid) DO NOTHING
` `
tag, err := r.pool.Exec(ctx, query, tag, err := r.pool.Exec(ctx, query,
row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8],
row[9], row[10], row[11], row[12], row[13], row[14], row[9], row[10], row[11], row[12], row[13], row[14], row[15], row[16],
) )
if err != nil { if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("failed to insert message: %w", err) return fmt.Errorf("failed to insert message: %w", err)
} }
@@ -72,6 +87,10 @@ func (r *Repository) InsertOne(ctx context.Context, msg *domain.ParsedMessage) e
// InsertBatch inserts multiple telegram messages in a batch using CopyFrom // InsertBatch inserts multiple telegram messages in a batch using CopyFrom
func (r *Repository) InsertBatch(ctx context.Context, msgs []*domain.ParsedMessage) error { func (r *Repository) InsertBatch(ctx context.Context, msgs []*domain.ParsedMessage) error {
ctx, span := otel.Tracer("caatsm/postgres").Start(ctx, "Repository.InsertBatch")
defer span.End()
span.SetAttributes(attribute.String("db.table", "aviation.telegrams"))
if len(msgs) == 0 { if len(msgs) == 0 {
return nil return nil
} }
@@ -93,14 +112,18 @@ func (r *Repository) InsertBatch(ctx context.Context, msgs []*domain.ParsedMessa
[]string{ []string{
"uuid", "message_id", "date_time", "priority_indicator", "primary_address", "uuid", "message_id", "date_time", "priority_indicator", "primary_address",
"secondary_addresses", "originator", "originator_date_time", "category", "secondary_addresses", "originator", "originator_date_time", "category",
"content", "body_data", "received_at", "parsed_at", "dispatched_at", "need_dispatch", "content", "body_data", "status", "error_reason",
"received_at", "parsed_at", "dispatched_at", "need_dispatch",
}, },
pgx.CopyFromRows(rows), pgx.CopyFromRows(rows),
) )
if err != nil { if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("failed to batch insert messages: %w", err) return fmt.Errorf("failed to batch insert messages: %w", err)
} }
span.SetAttributes(attribute.Int64("db.inserted", copyCount))
r.logger.Info("Batch inserted messages", r.logger.Info("Batch inserted messages",
zap.Int("count", int(copyCount)), zap.Int("count", int(copyCount)),
zap.Int("attempted", len(msgs)), zap.Int("attempted", len(msgs)),
@@ -108,3 +131,69 @@ func (r *Repository) InsertBatch(ctx context.Context, msgs []*domain.ParsedMessa
return nil return nil
} }
// InsertRaw inserts a failed telegram into aviation.telegrams_raw for post-processing.
func (r *Repository) InsertRaw(ctx context.Context, msg *domain.ParsedMessage) error {
ctx, span := otel.Tracer("caatsm/postgres").Start(ctx, "Repository.InsertRaw")
defer span.End()
span.SetAttributes(attribute.String("db.table", "aviation.telegrams_raw"))
if msg == nil {
err := fmt.Errorf("message is nil")
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("message is nil")
}
if msg.Uuid == "" {
msg.Uuid = uuid.NewString()
}
if msg.ReceivedAt.IsZero() {
msg.ReceivedAt = time.Now()
}
metadata := map[string]interface{}{
"message_id": msg.MessageID,
"category": msg.Category,
"comments": msg.Comments,
}
metadataJSON, err := json.Marshal(metadata)
if err != nil {
return fmt.Errorf("failed to marshal metadata: %w", err)
}
query := `
INSERT INTO aviation.telegrams_raw (
uuid, status, error_reason, content, received_at, metadata
) VALUES (
$1, $2, $3, $4, $5, $6
)
ON CONFLICT (uuid) DO UPDATE
SET status = EXCLUDED.status,
error_reason = EXCLUDED.error_reason,
content = EXCLUDED.content,
received_at = EXCLUDED.received_at,
metadata = EXCLUDED.metadata
`
_, err = r.pool.Exec(ctx, query,
msg.Uuid,
string(msg.Status),
msg.ErrorReason,
msg.Content,
msg.ReceivedAt,
metadataJSON,
)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return fmt.Errorf("failed to insert raw telegram: %w", err)
}
span.SetAttributes(attribute.String("telegram.uuid", msg.Uuid), attribute.String("telegram.status", string(msg.Status)))
r.logger.Debug("Persisted raw telegram",
zap.String("uuid", msg.Uuid),
zap.String("status", string(msg.Status)),
)
return nil
}
+18 -7
View File
@@ -3,6 +3,7 @@ package parsers
import ( import (
"caatsm/internal/domain" "caatsm/internal/domain"
"caatsm/pkg/utils" "caatsm/pkg/utils"
"errors"
"fmt" "fmt"
"regexp" "regexp"
"strings" "strings"
@@ -51,6 +52,10 @@ var (
performancePattern, performancePattern,
regPattern, regPattern,
reroutePattern} reroutePattern}
// ErrHeaderParse indicates an invalid header section.
ErrHeaderParse = errors.New("invalid telegram header")
// ErrBodyParse indicates a failure matching the telegram body.
ErrBodyParse = errors.New("invalid telegram body")
) )
type BodyParser struct { type BodyParser struct {
@@ -194,27 +199,33 @@ func (parser *BodyParser) createBodyData(data map[string]string) (string, interf
} }
} }
func Parse(rawText string) *domain.ParsedMessage { func Parse(rawText string) (*domain.ParsedMessage, error) {
message, err := ParseHeader(rawText) message, err := ParseHeader(rawText)
if err != nil { if err != nil {
msg := domain.NewParsedMessage() msg := domain.NewParsedMessage()
msg.Content = rawText msg.Content = rawText
return msg msg.Comments = err.Error()
msg.ErrorReason = err.Error()
msg.Status = domain.MessageStatusHeaderError
return msg, fmt.Errorf("%w: %w", ErrHeaderParse, err)
} }
bodyParser := NewBodyParser(message.Body) bodyParser := NewBodyParser(message.Body)
category, bodyData, err := bodyParser.Parse() category, bodyData, bodyErr := bodyParser.Parse()
message.Category = category message.Category = category
message.ParsedAt = time.Now() message.ParsedAt = time.Now()
if err != nil { if bodyErr != nil {
message.Comments = err.Error() message.Comments = bodyErr.Error()
return &message message.ErrorReason = bodyErr.Error()
message.Status = domain.MessageStatusBodyError
return &message, fmt.Errorf("%w: %w", ErrBodyParse, bodyErr)
} }
message.Parsed = true message.Parsed = true
message.Status = domain.MessageStatusParsed
message.BodyData = bodyData message.BodyData = bodyData
message.Uuid = uuid.New().String() message.Uuid = uuid.New().String()
return &message return &message, nil
} }
func cleanMessage(text string) string { func cleanMessage(text string) string {
+4 -2
View File
@@ -282,7 +282,8 @@ FF ZBTJZPZX
NNNN NNNN
` `
It("should parse the whole message correctly", func() { It("should parse the whole message correctly", func() {
parsedMessage := Parse(message) parsedMessage, err := Parse(message)
Expect(err).ToNot(HaveOccurred())
Expect(parsedMessage).ToNot(BeNil()) Expect(parsedMessage).ToNot(BeNil())
Expect(parsedMessage.Parsed).To(BeTrue()) Expect(parsedMessage.Parsed).To(BeTrue())
Expect(parsedMessage.MessageID).To(Equal("TMQ2526")) Expect(parsedMessage.MessageID).To(Equal("TMQ2526"))
@@ -333,7 +334,8 @@ GG ZBTJZPZX
NNNN NNNN
` `
It("should parse the whole message correctly", func() { It("should parse the whole message correctly", func() {
parsedMessage := Parse(message) parsedMessage, err := Parse(message)
Expect(err).ToNot(HaveOccurred())
Expect(parsedMessage).ToNot(BeNil()) Expect(parsedMessage).ToNot(BeNil())
Expect(parsedMessage.Parsed).To(BeTrue()) Expect(parsedMessage.Parsed).To(BeTrue())
Expect(parsedMessage.MessageID).To(Equal("TMQ2617")) Expect(parsedMessage.MessageID).To(Equal("TMQ2617"))
+11
View File
@@ -12,6 +12,8 @@ CREATE TABLE aviation.telegrams (
category VARCHAR(255), category VARCHAR(255),
content TEXT, content TEXT,
body_data JSONB, body_data JSONB,
status VARCHAR(32) NOT NULL DEFAULT 'parsed',
error_reason TEXT,
received_at TIMESTAMP NOT NULL, received_at TIMESTAMP NOT NULL,
parsed_at TIMESTAMP, parsed_at TIMESTAMP,
dispatched_at TIMESTAMP, dispatched_at TIMESTAMP,
@@ -24,3 +26,12 @@ CREATE INDEX idx_telegrams_date_time ON aviation.telegrams (date_time);
CREATE INDEX idx_telegrams_priority_indicator ON aviation.telegrams (priority_indicator); CREATE INDEX idx_telegrams_priority_indicator ON aviation.telegrams (priority_indicator);
CREATE INDEX idx_telegrams_primary_address ON aviation.telegrams (primary_address); CREATE INDEX idx_telegrams_primary_address ON aviation.telegrams (primary_address);
CREATE INDEX idx_telegrams_received_at ON aviation.telegrams (received_at); CREATE INDEX idx_telegrams_received_at ON aviation.telegrams (received_at);
CREATE TABLE IF NOT EXISTS aviation.telegrams_raw (
uuid UUID PRIMARY KEY,
status VARCHAR(32) NOT NULL,
error_reason TEXT,
content TEXT NOT NULL,
received_at TIMESTAMP NOT NULL,
metadata JSONB
);
-60
View File
@@ -1,60 +0,0 @@
package di
import (
"caatsm/internal/adapter/parser"
"caatsm/internal/app"
"caatsm/internal/infra/config"
"caatsm/internal/infra/log"
"caatsm/internal/infra/nats"
"caatsm/internal/infra/postgres"
"errors"
)
// InitializeAppWithConfig wires dependencies using the provided config.
func InitializeAppWithConfig(cfg *config.Config) (*app.MessageProcessor, *nats.Consumer, error) {
if cfg == nil {
return nil, nil, errors.New("config is required")
}
logger, err := log.ProvideLogger(cfg)
if err != nil {
return nil, nil, err
}
pool, err := postgres.ProvideDB(cfg)
if err != nil {
return nil, nil, err
}
repo, err := postgres.ProvideRepository(pool, logger)
if err != nil {
return nil, nil, err
}
conn, err := nats.ProvideNATSConn(cfg, logger)
{
if err != nil {
return nil, nil, err
}
}
js, err := nats.ProvideJetStream(conn, cfg, logger)
if err != nil {
return nil, nil, err
}
publisher, err := nats.ProvidePublisher(js, cfg, logger)
if err != nil {
return nil, nil, err
}
messageParser := parser.ProvideParser()
processor := app.NewMessageProcessor(messageParser, repo, publisher, logger)
consumer, err := nats.ProvideConsumer(conn, js, processor, cfg, logger)
if err != nil {
return nil, nil, err
}
return processor, consumer, nil
}
+37 -4
View File
@@ -16,10 +16,23 @@ import (
// InitializeApp initializes the application with all dependencies // InitializeApp initializes the application with all dependencies
func InitializeApp() (*app.MessageProcessor, *nats.Consumer, error) { func InitializeApp() (*app.MessageProcessor, *nats.Consumer, error) {
wire.Build( comps, err := buildAppComponents()
// Config if err != nil {
config.ProvideConfig, return nil, nil, err
}
return comps.Processor, comps.Consumer, nil
}
// InitializeAppWithConfig wires dependencies using a pre-loaded configuration.
func InitializeAppWithConfig(cfg *config.Config) (*app.MessageProcessor, *nats.Consumer, error) {
comps, err := buildAppComponentsWithConfig(cfg)
if err != nil {
return nil, nil, err
}
return comps.Processor, comps.Consumer, nil
}
var runtimeSet = wire.NewSet(
// Logger // Logger
log.ProvideLogger, log.ProvideLogger,
@@ -40,6 +53,26 @@ func InitializeApp() (*app.MessageProcessor, *nats.Consumer, error) {
// Consumer // Consumer
nats.ProvideConsumer, nats.ProvideConsumer,
)
func buildAppComponents() (*appComponents, error) {
wire.Build(
config.ProvideConfig,
runtimeSet,
wire.Struct(new(appComponents), "*"),
) )
return nil, nil, nil return nil, nil
}
func buildAppComponentsWithConfig(cfg *config.Config) (*appComponents, error) {
wire.Build(
runtimeSet,
wire.Struct(new(appComponents), "*"),
)
return nil, nil
}
type appComponents struct {
Processor *app.MessageProcessor
Consumer *nats.Consumer
} }
+103 -32
View File
@@ -1,5 +1,6 @@
// Code generated by Wire. DO NOT EDIT. // Code generated by Wire. DO NOT EDIT.
//go:generate go run -mod=mod github.com/google/wire/cmd/wire
//go:build !wireinject //go:build !wireinject
// +build !wireinject // +build !wireinject
@@ -12,44 +13,114 @@ import (
"caatsm/internal/infra/log" "caatsm/internal/infra/log"
"caatsm/internal/infra/nats" "caatsm/internal/infra/nats"
"caatsm/internal/infra/postgres" "caatsm/internal/infra/postgres"
"github.com/google/wire"
) )
// Injectors from wire.go:
func buildAppComponents() (*appComponents, error) {
parserParser := parser.ProvideParser()
configConfig, err := config.ProvideConfig()
if err != nil {
return nil, err
}
pool, err := postgres.ProvideDB(configConfig)
if err != nil {
return nil, err
}
logger, err := log.ProvideLogger(configConfig)
if err != nil {
return nil, err
}
repository, err := postgres.ProvideRepository(pool, logger)
if err != nil {
return nil, err
}
conn, err := nats.ProvideNATSConn(configConfig, logger)
if err != nil {
return nil, err
}
jetStreamContext, err := nats.ProvideJetStream(conn, configConfig, logger)
if err != nil {
return nil, err
}
publisher, err := nats.ProvidePublisher(jetStreamContext, configConfig, logger)
if err != nil {
return nil, err
}
messageProcessor := app.NewMessageProcessor(parserParser, repository, publisher, logger)
consumer, err := nats.ProvideConsumer(conn, jetStreamContext, messageProcessor, configConfig, logger)
if err != nil {
return nil, err
}
diAppComponents := &appComponents{
Processor: messageProcessor,
Consumer: consumer,
}
return diAppComponents, nil
}
func buildAppComponentsWithConfig(cfg *config.Config) (*appComponents, error) {
parserParser := parser.ProvideParser()
pool, err := postgres.ProvideDB(cfg)
if err != nil {
return nil, err
}
logger, err := log.ProvideLogger(cfg)
if err != nil {
return nil, err
}
repository, err := postgres.ProvideRepository(pool, logger)
if err != nil {
return nil, err
}
conn, err := nats.ProvideNATSConn(cfg, logger)
if err != nil {
return nil, err
}
jetStreamContext, err := nats.ProvideJetStream(conn, cfg, logger)
if err != nil {
return nil, err
}
publisher, err := nats.ProvidePublisher(jetStreamContext, cfg, logger)
if err != nil {
return nil, err
}
messageProcessor := app.NewMessageProcessor(parserParser, repository, publisher, logger)
consumer, err := nats.ProvideConsumer(conn, jetStreamContext, messageProcessor, cfg, logger)
if err != nil {
return nil, err
}
diAppComponents := &appComponents{
Processor: messageProcessor,
Consumer: consumer,
}
return diAppComponents, nil
}
// wire.go:
// InitializeApp initializes the application with all dependencies // InitializeApp initializes the application with all dependencies
func InitializeApp() (*app.MessageProcessor, *nats.Consumer, error) { func InitializeApp() (*app.MessageProcessor, *nats.Consumer, error) {
configConfig, err := config.ProvideConfig() comps, err := buildAppComponents()
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
zapLogger, err := log.ProvideLogger(configConfig) return comps.Processor, comps.Consumer, nil
if err != nil {
return nil, nil, err
}
pgxpoolPool, err := postgres.ProvideDB(configConfig)
if err != nil {
return nil, nil, err
}
adapterRepository, err := postgres.ProvideRepository(pgxpoolPool, zapLogger)
if err != nil {
return nil, nil, err
}
natsConn, err := nats.ProvideNATSConn(configConfig, zapLogger)
if err != nil {
return nil, nil, err
}
natsJetStreamContext, err := nats.ProvideJetStream(natsConn, configConfig, zapLogger)
if err != nil {
return nil, nil, err
}
adapterPublisher, err := nats.ProvidePublisher(natsJetStreamContext, configConfig, zapLogger)
if err != nil {
return nil, nil, err
}
parserParser := parser.ProvideParser()
appMessageProcessor := app.NewMessageProcessor(parserParser, adapterRepository, adapterPublisher, zapLogger)
natsConsumer, err := nats.ProvideConsumer(natsConn, natsJetStreamContext, appMessageProcessor, configConfig, zapLogger)
if err != nil {
return nil, nil, err
}
return appMessageProcessor, natsConsumer, nil
} }
// InitializeAppWithConfig wires dependencies using a pre-loaded configuration.
func InitializeAppWithConfig(cfg *config.Config) (*app.MessageProcessor, *nats.Consumer, error) {
comps, err := buildAppComponentsWithConfig(cfg)
if err != nil {
return nil, nil, err
}
return comps.Processor, comps.Consumer, nil
}
var runtimeSet = wire.NewSet(log.ProvideLogger, postgres.ProvideDB, postgres.ProvideRepository, nats.ProvideNATSConn, nats.ProvideJetStream, nats.ProvidePublisher, parser.ProvideParser, app.NewMessageProcessor, nats.ProvideConsumer)
type appComponents struct {
Processor *app.MessageProcessor
Consumer *nats.Consumer
}