✨ Introduce Docker Compose development stack with TimescaleDB and NATS services. Update README with usage instructions for local development. Upgrade Go version to 1.25.0 and update dependencies. Refactor configuration for NATS consumer rules and enhance Telegram database schema with TimescaleDB support.
This commit is contained in:
windyboy
@@ -32,7 +32,7 @@ This project follows Clean Architecture principles with clear separation of conc
|
||||
|
||||
- **Clean Architecture**: Clear separation between domain, application, and infrastructure layers
|
||||
- **NATS JetStream**: Reliable message streaming with automatic retries and dead-letter queues
|
||||
- **PostgreSQL**: High-performance data persistence using pgx with batch operations
|
||||
- **TimescaleDB (PostgreSQL)**: High-performance persistence using pgx with batch operations and hypertables
|
||||
- **Dependency Injection**: Google Wire for compile-time dependency injection
|
||||
- **Configuration Management**: Koanf for flexible configuration loading (file + environment variables)
|
||||
- **Structured Logging**: Zap logger with configurable levels and formats
|
||||
@@ -91,7 +91,7 @@ discard = "old"
|
||||
storage = "file"
|
||||
replicas = 1
|
||||
|
||||
[nats.consumer]
|
||||
[nats.consumer_rules]
|
||||
max_deliver = 5
|
||||
ack_wait = "30s"
|
||||
max_ack_pending = 1024
|
||||
@@ -225,6 +225,28 @@ Critical overrides stay available through CLI flags; advanced tuning such as str
|
||||
|
||||
## Development
|
||||
|
||||
### Docker Compose Dev Stack
|
||||
|
||||
For a local stack running TimescaleDB + NATS (matching `config.dev.toml`), use `docker-compose.dev.yml`:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.dev.yml up -d postgres nats
|
||||
docker compose -f docker-compose.dev.yml up app
|
||||
```
|
||||
|
||||
- `postgres` uses TimescaleDB, seeding the `aviation` schema via `internal/repository/telegrams.ddl` (extension + hypertable).
|
||||
- `app` mounts the repo so code changes are picked up by `go run ./cmd/main listen`.
|
||||
- `nats` exposes 4222 (client) and 8222 (monitoring); `nats-box` is available for JetStream inspection (`docker compose exec nats-box sh`).
|
||||
|
||||
Prefer to run the Go binary on your host for quicker iteration:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.dev.yml up -d postgres nats
|
||||
GO_ENV=dev CAATSM_POSTGRES_URL=postgres://caatsm:caatsm@localhost:5432/aviation?sslmode=disable go run ./cmd/main listen
|
||||
```
|
||||
|
||||
Bring everything down with `docker compose -f docker-compose.dev.yml down -v` when finished.
|
||||
|
||||
### Project Structure
|
||||
|
||||
- **Domain Layer** (`internal/domain`): Pure business logic and domain models
|
||||
|
||||
@@ -104,6 +104,31 @@ tasks:
|
||||
- echo "Linting code..."
|
||||
- golangci-lint run
|
||||
|
||||
up:
|
||||
desc: Start TimescaleDB + NATS dev stack
|
||||
cmds:
|
||||
- echo "Starting dev infrastructure..."
|
||||
- podman compose -f docker-compose.dev.yml up -d
|
||||
|
||||
down:
|
||||
desc: Stop dev compose stack and remove containers
|
||||
cmds:
|
||||
- echo "Stopping dev infrastructure..."
|
||||
- podman compose -f docker-compose.dev.yml down
|
||||
|
||||
dev-run:
|
||||
desc: Run receiver locally against dev stack
|
||||
deps:
|
||||
- up
|
||||
cmds:
|
||||
- >
|
||||
CAATSM_NATS_URL=nats://localhost:4222
|
||||
CAATSM_POSTGRES_URL=postgres://caatsm:caatsm@localhost:5432/aviation?sslmode=disable
|
||||
CAATSM_TELEMETRY_ENABLED=true
|
||||
CAATSM_TELEMETRY_ENDPOINT=localhost:4318
|
||||
GO_ENV=dev
|
||||
go run ./cmd/main listen
|
||||
|
||||
|
||||
help:
|
||||
desc: Show this help message
|
||||
|
||||
@@ -13,7 +13,7 @@ discard = "old"
|
||||
storage = "file"
|
||||
replicas = 1
|
||||
|
||||
[nats.consumer]
|
||||
[nats.consumer_rules]
|
||||
max_deliver = 5
|
||||
ack_wait = "30s"
|
||||
max_ack_pending = 1024
|
||||
@@ -52,5 +52,5 @@ format = "json"
|
||||
|
||||
[telemetry]
|
||||
enabled = false
|
||||
endpoint = "http://otel-collector:4318"
|
||||
endpoint = "otel-collector:4318"
|
||||
insecure = true
|
||||
@@ -0,0 +1,21 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
|
||||
exporters:
|
||||
logging:
|
||||
loglevel: info
|
||||
|
||||
service:
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
exporters: [logging]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
exporters: [logging]
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
services:
|
||||
|
||||
postgres:
|
||||
image: timescale/timescaledb:2.15.2-pg16
|
||||
environment:
|
||||
POSTGRES_USER: caatsm
|
||||
POSTGRES_PASSWORD: caatsm
|
||||
POSTGRES_DB: aviation
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
- ./internal/repository/telegrams.ddl:/docker-entrypoint-initdb.d/01-telegrams.sql:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
networks:
|
||||
- devnet
|
||||
|
||||
nats:
|
||||
image: nats:2.10-alpine
|
||||
command: ["-js", "--http_port=8222", "-DV"]
|
||||
ports:
|
||||
- "4222:4222"
|
||||
- "8222:8222"
|
||||
networks:
|
||||
- devnet
|
||||
|
||||
nats-box:
|
||||
image: synadia/nats-box:latest
|
||||
entrypoint: ["/bin/sh", "-c", "sleep infinity"]
|
||||
depends_on:
|
||||
- nats
|
||||
networks:
|
||||
- devnet
|
||||
environment:
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.99.0
|
||||
command:
|
||||
- "--config=/etc/otelcol/config.yaml"
|
||||
volumes:
|
||||
- ./configs/otel-collector.dev.yaml:/etc/otelcol/config.yaml:ro
|
||||
ports:
|
||||
- "4317:4317"
|
||||
- "4318:4318"
|
||||
networks:
|
||||
- devnet
|
||||
|
||||
volumes:
|
||||
postgres-data:
|
||||
|
||||
networks:
|
||||
devnet:
|
||||
driver: bridge
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module caatsm
|
||||
|
||||
go 1.24.0
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/google/uuid v1.6.0
|
||||
@@ -35,7 +35,7 @@ require (
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20251007162407-5df77e3f7d1d // indirect
|
||||
github.com/google/pprof v0.0.0-20251114195745-4902fdda35c8 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
|
||||
@@ -31,7 +31,7 @@ type NATSConfig struct {
|
||||
Stream string `koanf:"stream"`
|
||||
Consumer string `koanf:"consumer"`
|
||||
StreamLimits StreamLimitsConfig `koanf:"stream_limits"`
|
||||
ConsumerRules ConsumerRulesConfig `koanf:"consumer"`
|
||||
ConsumerRules ConsumerRulesConfig `koanf:"consumer_rules"`
|
||||
// Legacy fields
|
||||
Client string `koanf:"client"`
|
||||
Cluster string `koanf:"cluster"`
|
||||
|
||||
@@ -1,37 +1,44 @@
|
||||
CREATE SCHEMA IF NOT EXISTS aviation;
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS timescaledb;
|
||||
|
||||
CREATE TABLE aviation.telegrams (
|
||||
uuid UUID PRIMARY KEY,
|
||||
message_id VARCHAR(255) ,
|
||||
date_time VARCHAR(255) ,
|
||||
priority_indicator VARCHAR(255) ,
|
||||
primary_address VARCHAR(255) ,
|
||||
uuid UUID NOT NULL,
|
||||
message_id TEXT,
|
||||
date_time TEXT,
|
||||
priority_indicator TEXT,
|
||||
primary_address TEXT,
|
||||
secondary_addresses TEXT,
|
||||
originator VARCHAR(255),
|
||||
originator_date_time VARCHAR(255),
|
||||
category VARCHAR(255),
|
||||
originator TEXT,
|
||||
originator_date_time TEXT,
|
||||
category TEXT,
|
||||
content TEXT,
|
||||
body_data JSONB,
|
||||
status VARCHAR(32) NOT NULL DEFAULT 'parsed',
|
||||
status TEXT NOT NULL DEFAULT 'parsed',
|
||||
error_reason TEXT,
|
||||
received_at TIMESTAMP NOT NULL,
|
||||
parsed_at TIMESTAMP,
|
||||
dispatched_at TIMESTAMP,
|
||||
need_dispatch BOOLEAN
|
||||
received_at TIMESTAMPTZ NOT NULL,
|
||||
parsed_at TIMESTAMPTZ,
|
||||
dispatched_at TIMESTAMPTZ,
|
||||
need_dispatch BOOLEAN,
|
||||
PRIMARY KEY (uuid, received_at)
|
||||
);
|
||||
|
||||
SELECT create_hypertable('aviation.telegrams', 'received_at', if_not_exists => TRUE);
|
||||
|
||||
-- Indexes for better query performance
|
||||
CREATE INDEX idx_telegrams_message_id ON aviation.telegrams (message_id);
|
||||
CREATE INDEX idx_telegrams_date_time ON aviation.telegrams (date_time);
|
||||
CREATE INDEX idx_telegrams_priority_indicator ON aviation.telegrams (priority_indicator);
|
||||
CREATE INDEX idx_telegrams_primary_address ON aviation.telegrams (primary_address);
|
||||
CREATE INDEX idx_telegrams_received_at ON aviation.telegrams (received_at);
|
||||
CREATE INDEX idx_telegrams_uuid ON aviation.telegrams (uuid);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS aviation.telegrams_raw (
|
||||
uuid UUID PRIMARY KEY,
|
||||
status VARCHAR(32) NOT NULL,
|
||||
uuid UUID NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
error_reason TEXT,
|
||||
content TEXT NOT NULL,
|
||||
received_at TIMESTAMP NOT NULL,
|
||||
metadata JSONB
|
||||
received_at TIMESTAMPTZ NOT NULL,
|
||||
metadata JSONB,
|
||||
PRIMARY KEY (uuid, received_at)
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user