From 4f16bd6d908460c90d50e2f8f5f7c4ac37d4754a Mon Sep 17 00:00:00 2001 From: windyboy Date: Sat, 15 Nov 2025 09:46:29 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Introduce=20Docker=20Compose=20deve?= =?UTF-8?q?lopment=20stack=20with=20TimescaleDB=20and=20NATS=20services.?= =?UTF-8?q?=20Update=20README=20with=20usage=20instructions=20for=20local?= =?UTF-8?q?=20development.=20Upgrade=20Go=20version=20to=201.25.0=20and=20?= =?UTF-8?q?update=20dependencies.=20Refactor=20configuration=20for=20NATS?= =?UTF-8?q?=20consumer=20rules=20and=20enhance=20Telegram=20database=20sch?= =?UTF-8?q?ema=20with=20TimescaleDB=20support.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 26 ++++++++++++-- Taskfile.yml | 25 +++++++++++++ configs/config.dev.toml | 4 +-- configs/otel-collector.dev.yaml | 21 +++++++++++ docker-compose.dev.yml | 59 +++++++++++++++++++++++++++++++ go.mod | 4 +-- internal/infra/config/config.go | 2 +- internal/repository/telegrams.ddl | 43 ++++++++++++---------- 8 files changed, 159 insertions(+), 25 deletions(-) create mode 100644 configs/otel-collector.dev.yaml create mode 100644 docker-compose.dev.yml diff --git a/README.md b/README.md index 17bd3ca..c2a2020 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ This project follows Clean Architecture principles with clear separation of conc - **Clean Architecture**: Clear separation between domain, application, and infrastructure layers - **NATS JetStream**: Reliable message streaming with automatic retries and dead-letter queues -- **PostgreSQL**: High-performance data persistence using pgx with batch operations +- **TimescaleDB (PostgreSQL)**: High-performance persistence using pgx with batch operations and hypertables - **Dependency Injection**: Google Wire for compile-time dependency injection - **Configuration Management**: Koanf for flexible configuration loading (file + environment variables) - **Structured Logging**: Zap logger with configurable levels and formats @@ -91,7 +91,7 @@ discard = "old" storage = "file" replicas = 1 -[nats.consumer] +[nats.consumer_rules] max_deliver = 5 ack_wait = "30s" max_ack_pending = 1024 @@ -225,6 +225,28 @@ Critical overrides stay available through CLI flags; advanced tuning such as str ## Development +### Docker Compose Dev Stack + +For a local stack running TimescaleDB + NATS (matching `config.dev.toml`), use `docker-compose.dev.yml`: + +```bash +docker compose -f docker-compose.dev.yml up -d postgres nats +docker compose -f docker-compose.dev.yml up app +``` + +- `postgres` uses TimescaleDB, seeding the `aviation` schema via `internal/repository/telegrams.ddl` (extension + hypertable). +- `app` mounts the repo so code changes are picked up by `go run ./cmd/main listen`. +- `nats` exposes 4222 (client) and 8222 (monitoring); `nats-box` is available for JetStream inspection (`docker compose exec nats-box sh`). + +Prefer to run the Go binary on your host for quicker iteration: + +```bash +docker compose -f docker-compose.dev.yml up -d postgres nats +GO_ENV=dev CAATSM_POSTGRES_URL=postgres://caatsm:caatsm@localhost:5432/aviation?sslmode=disable go run ./cmd/main listen +``` + +Bring everything down with `docker compose -f docker-compose.dev.yml down -v` when finished. + ### Project Structure - **Domain Layer** (`internal/domain`): Pure business logic and domain models diff --git a/Taskfile.yml b/Taskfile.yml index d8534a2..d6311f3 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -104,6 +104,31 @@ tasks: - echo "Linting code..." - golangci-lint run + up: + desc: Start TimescaleDB + NATS dev stack + cmds: + - echo "Starting dev infrastructure..." + - podman compose -f docker-compose.dev.yml up -d + + down: + desc: Stop dev compose stack and remove containers + cmds: + - echo "Stopping dev infrastructure..." + - podman compose -f docker-compose.dev.yml down + + dev-run: + desc: Run receiver locally against dev stack + deps: + - up + cmds: + - > + CAATSM_NATS_URL=nats://localhost:4222 + CAATSM_POSTGRES_URL=postgres://caatsm:caatsm@localhost:5432/aviation?sslmode=disable + CAATSM_TELEMETRY_ENABLED=true + CAATSM_TELEMETRY_ENDPOINT=localhost:4318 + GO_ENV=dev + go run ./cmd/main listen + help: desc: Show this help message diff --git a/configs/config.dev.toml b/configs/config.dev.toml index e007cf2..48d4944 100644 --- a/configs/config.dev.toml +++ b/configs/config.dev.toml @@ -13,7 +13,7 @@ discard = "old" storage = "file" replicas = 1 -[nats.consumer] +[nats.consumer_rules] max_deliver = 5 ack_wait = "30s" max_ack_pending = 1024 @@ -52,5 +52,5 @@ format = "json" [telemetry] enabled = false -endpoint = "http://otel-collector:4318" +endpoint = "otel-collector:4318" insecure = true \ No newline at end of file diff --git a/configs/otel-collector.dev.yaml b/configs/otel-collector.dev.yaml new file mode 100644 index 0000000..8c58222 --- /dev/null +++ b/configs/otel-collector.dev.yaml @@ -0,0 +1,21 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +exporters: + logging: + loglevel: info + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [logging] + metrics: + receivers: [otlp] + exporters: [logging] + diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..5d039e2 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,59 @@ +services: + + postgres: + image: timescale/timescaledb:2.15.2-pg16 + environment: + POSTGRES_USER: caatsm + POSTGRES_PASSWORD: caatsm + POSTGRES_DB: aviation + ports: + - "5432:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + - ./internal/repository/telegrams.ddl:/docker-entrypoint-initdb.d/01-telegrams.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 5s + timeout: 5s + retries: 10 + networks: + - devnet + + nats: + image: nats:2.10-alpine + command: ["-js", "--http_port=8222", "-DV"] + ports: + - "4222:4222" + - "8222:8222" + networks: + - devnet + + nats-box: + image: synadia/nats-box:latest + entrypoint: ["/bin/sh", "-c", "sleep infinity"] + depends_on: + - nats + networks: + - devnet + environment: + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.99.0 + command: + - "--config=/etc/otelcol/config.yaml" + volumes: + - ./configs/otel-collector.dev.yaml:/etc/otelcol/config.yaml:ro + ports: + - "4317:4317" + - "4318:4318" + networks: + - devnet + +volumes: + postgres-data: + +networks: + devnet: + driver: bridge + diff --git a/go.mod b/go.mod index 0e2ce23..60a0e6e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module caatsm -go 1.24.0 +go 1.25.0 require ( github.com/google/uuid v1.6.0 @@ -35,7 +35,7 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20251007162407-5df77e3f7d1d // indirect + github.com/google/pprof v0.0.0-20251114195745-4902fdda35c8 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect diff --git a/internal/infra/config/config.go b/internal/infra/config/config.go index c995fc8..282118a 100644 --- a/internal/infra/config/config.go +++ b/internal/infra/config/config.go @@ -31,7 +31,7 @@ type NATSConfig struct { Stream string `koanf:"stream"` Consumer string `koanf:"consumer"` StreamLimits StreamLimitsConfig `koanf:"stream_limits"` - ConsumerRules ConsumerRulesConfig `koanf:"consumer"` + ConsumerRules ConsumerRulesConfig `koanf:"consumer_rules"` // Legacy fields Client string `koanf:"client"` Cluster string `koanf:"cluster"` diff --git a/internal/repository/telegrams.ddl b/internal/repository/telegrams.ddl index 9bbcbef..85f9f7e 100644 --- a/internal/repository/telegrams.ddl +++ b/internal/repository/telegrams.ddl @@ -1,37 +1,44 @@ CREATE SCHEMA IF NOT EXISTS aviation; +CREATE EXTENSION IF NOT EXISTS timescaledb; + CREATE TABLE aviation.telegrams ( - uuid UUID PRIMARY KEY, - message_id VARCHAR(255) , - date_time VARCHAR(255) , - priority_indicator VARCHAR(255) , - primary_address VARCHAR(255) , - secondary_addresses TEXT, - originator VARCHAR(255), - originator_date_time VARCHAR(255), - category VARCHAR(255), + uuid UUID NOT NULL, + message_id TEXT, + date_time TEXT, + priority_indicator TEXT, + primary_address TEXT, + secondary_addresses TEXT, + originator TEXT, + originator_date_time TEXT, + category TEXT, content TEXT, body_data JSONB, - status VARCHAR(32) NOT NULL DEFAULT 'parsed', + status TEXT NOT NULL DEFAULT 'parsed', error_reason TEXT, - received_at TIMESTAMP NOT NULL, - parsed_at TIMESTAMP, - dispatched_at TIMESTAMP, - need_dispatch BOOLEAN + received_at TIMESTAMPTZ NOT NULL, + parsed_at TIMESTAMPTZ, + dispatched_at TIMESTAMPTZ, + need_dispatch BOOLEAN, + PRIMARY KEY (uuid, received_at) ); +SELECT create_hypertable('aviation.telegrams', 'received_at', if_not_exists => TRUE); + -- Indexes for better query performance CREATE INDEX idx_telegrams_message_id ON aviation.telegrams (message_id); CREATE INDEX idx_telegrams_date_time ON aviation.telegrams (date_time); CREATE INDEX idx_telegrams_priority_indicator ON aviation.telegrams (priority_indicator); CREATE INDEX idx_telegrams_primary_address ON aviation.telegrams (primary_address); CREATE INDEX idx_telegrams_received_at ON aviation.telegrams (received_at); +CREATE INDEX idx_telegrams_uuid ON aviation.telegrams (uuid); CREATE TABLE IF NOT EXISTS aviation.telegrams_raw ( - uuid UUID PRIMARY KEY, - status VARCHAR(32) NOT NULL, + uuid UUID NOT NULL, + status TEXT NOT NULL, error_reason TEXT, content TEXT NOT NULL, - received_at TIMESTAMP NOT NULL, - metadata JSONB + received_at TIMESTAMPTZ NOT NULL, + metadata JSONB, + PRIMARY KEY (uuid, received_at) );