✨ Upgrade Go version to 1.23.0 and update dependencies. Introduce new application structure with Clean Architecture principles, including message processing, NATS integration, and PostgreSQL repository. Add configuration management using Koanf and structured logging with Zap. Remove legacy GraphQL integration and related files. Implement dependency injection with Google Wire.
This commit is contained in:
windyboy
@@ -0,0 +1,169 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/knadh/koanf/v2"
|
||||
"github.com/knadh/koanf/parsers/toml"
|
||||
"github.com/knadh/koanf/providers/file"
|
||||
envprovider "github.com/knadh/koanf/providers/env"
|
||||
)
|
||||
|
||||
// Config holds all application configuration
|
||||
type Config struct {
|
||||
NATS NATSConfig `koanf:"nats"`
|
||||
Postgres PostgresConfig `koanf:"postgres"`
|
||||
App AppConfig `koanf:"app"`
|
||||
Log LogConfig `koanf:"log"`
|
||||
Publisher PublisherConfig `koanf:"publisher"`
|
||||
// Legacy fields for backward compatibility during migration
|
||||
Subscription SubscriptionConfig `koanf:"subscription"`
|
||||
Timeouts TimeoutsConfig `koanf:"timeouts"`
|
||||
}
|
||||
|
||||
// NATSConfig holds NATS/JetStream configuration
|
||||
type NATSConfig struct {
|
||||
URL string `koanf:"url"`
|
||||
Stream string `koanf:"stream"`
|
||||
Consumer string `koanf:"consumer"`
|
||||
// Legacy fields
|
||||
Client string `koanf:"client"`
|
||||
Cluster string `koanf:"cluster"`
|
||||
}
|
||||
|
||||
// PostgresConfig holds PostgreSQL configuration
|
||||
type PostgresConfig struct {
|
||||
URL string `koanf:"url"`
|
||||
MaxConns int32 `koanf:"max_conns"`
|
||||
MinConns int32 `koanf:"min_conns"`
|
||||
}
|
||||
|
||||
// AppConfig holds application-level configuration
|
||||
type AppConfig struct {
|
||||
BatchSize int `koanf:"batch_size"`
|
||||
BatchTimeout time.Duration `koanf:"batch_timeout"`
|
||||
}
|
||||
|
||||
// LogConfig holds logging configuration
|
||||
type LogConfig struct {
|
||||
Level string `koanf:"level"`
|
||||
Format string `koanf:"format"` // json or console
|
||||
}
|
||||
|
||||
// PublisherConfig holds publisher configuration
|
||||
type PublisherConfig struct {
|
||||
Topic string `koanf:"topic"`
|
||||
}
|
||||
|
||||
// SubscriptionConfig holds subscription configuration (legacy)
|
||||
type SubscriptionConfig struct {
|
||||
Topic string `koanf:"topic"`
|
||||
QueueGroup string `koanf:"queue_group"`
|
||||
}
|
||||
|
||||
// TimeoutsConfig holds timeout configuration (legacy)
|
||||
type TimeoutsConfig struct {
|
||||
Server time.Duration `koanf:"server"`
|
||||
ReconnectWait time.Duration `koanf:"reconnect_wait"`
|
||||
Close time.Duration `koanf:"close"`
|
||||
AckWait time.Duration `koanf:"ack_wait"`
|
||||
}
|
||||
|
||||
// LoadConfig loads configuration from file and environment variables
|
||||
func LoadConfig() (*Config, error) {
|
||||
k := koanf.New(".")
|
||||
|
||||
// Determine environment
|
||||
env := os.Getenv("GO_ENV")
|
||||
if env == "" {
|
||||
env = "dev"
|
||||
}
|
||||
|
||||
// Load from TOML file
|
||||
configFile := fmt.Sprintf("configs/config.%s.toml", env)
|
||||
if err := k.Load(file.Provider(configFile), toml.Parser()); err != nil {
|
||||
return nil, fmt.Errorf("error loading config file '%s': %w", configFile, err)
|
||||
}
|
||||
|
||||
// Load from environment variables with CAATSM_ prefix
|
||||
envProvider := envprovider.Provider("CAATSM_", ".", func(s string) string {
|
||||
// Convert CAATSM_NATS_URL to nats.url
|
||||
s = strings.TrimPrefix(s, "CAATSM_")
|
||||
return strings.ToLower(strings.ReplaceAll(s, "_", "."))
|
||||
})
|
||||
if err := k.Load(envProvider, nil); err != nil {
|
||||
// Environment variables are optional, so we don't fail if they're not present
|
||||
// This allows the config to work with just the file
|
||||
}
|
||||
|
||||
// Unmarshal into Config struct
|
||||
var cfg Config
|
||||
if err := k.Unmarshal("", &cfg); err != nil {
|
||||
return nil, fmt.Errorf("error unmarshaling config: %w", err)
|
||||
}
|
||||
|
||||
// Set defaults
|
||||
if cfg.App.BatchSize == 0 {
|
||||
cfg.App.BatchSize = 50
|
||||
}
|
||||
if cfg.App.BatchTimeout == 0 {
|
||||
cfg.App.BatchTimeout = 2 * time.Second
|
||||
}
|
||||
if cfg.Postgres.MaxConns == 0 {
|
||||
cfg.Postgres.MaxConns = 10
|
||||
}
|
||||
if cfg.Postgres.MinConns == 0 {
|
||||
cfg.Postgres.MinConns = 2
|
||||
}
|
||||
if cfg.Log.Level == "" {
|
||||
cfg.Log.Level = "info"
|
||||
}
|
||||
if cfg.Log.Format == "" {
|
||||
cfg.Log.Format = "json"
|
||||
}
|
||||
if cfg.NATS.Stream == "" {
|
||||
cfg.NATS.Stream = "TELEGRAM"
|
||||
}
|
||||
if cfg.NATS.Consumer == "" {
|
||||
cfg.NATS.Consumer = "telegram-consumer"
|
||||
}
|
||||
|
||||
// Validate configuration
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("config validation failed: %w", err)
|
||||
}
|
||||
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
// Validate validates the configuration
|
||||
func (c *Config) Validate() error {
|
||||
if c.NATS.URL == "" {
|
||||
return fmt.Errorf("nats.url is required")
|
||||
}
|
||||
if c.Subscription.Topic == "" && c.NATS.Stream == "" {
|
||||
return fmt.Errorf("subscription.topic or nats.stream is required")
|
||||
}
|
||||
if c.Publisher.Topic == "" {
|
||||
return fmt.Errorf("publisher.topic is required")
|
||||
}
|
||||
if c.Postgres.URL == "" {
|
||||
return fmt.Errorf("postgres.url is required")
|
||||
}
|
||||
if c.App.BatchSize <= 0 {
|
||||
return fmt.Errorf("app.batch_size must be greater than 0")
|
||||
}
|
||||
if c.App.BatchTimeout <= 0 {
|
||||
return fmt.Errorf("app.batch_timeout must be greater than 0")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProvideConfig is a Wire provider function
|
||||
func ProvideConfig() (*Config, error) {
|
||||
return LoadConfig()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"caatsm/internal/infra/config"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
)
|
||||
|
||||
// ProvideLogger creates a zap logger based on configuration
|
||||
func ProvideLogger(cfg *config.Config) (*zap.Logger, error) {
|
||||
var zapConfig zap.Config
|
||||
|
||||
if cfg.Log.Format == "console" {
|
||||
zapConfig = zap.NewDevelopmentConfig()
|
||||
} else {
|
||||
zapConfig = zap.NewProductionConfig()
|
||||
}
|
||||
|
||||
// Set log level
|
||||
switch cfg.Log.Level {
|
||||
case "debug":
|
||||
zapConfig.Level = zap.NewAtomicLevelAt(zapcore.DebugLevel)
|
||||
case "info":
|
||||
zapConfig.Level = zap.NewAtomicLevelAt(zapcore.InfoLevel)
|
||||
case "warn":
|
||||
zapConfig.Level = zap.NewAtomicLevelAt(zapcore.WarnLevel)
|
||||
case "error":
|
||||
zapConfig.Level = zap.NewAtomicLevelAt(zapcore.ErrorLevel)
|
||||
default:
|
||||
zapConfig.Level = zap.NewAtomicLevelAt(zapcore.InfoLevel)
|
||||
}
|
||||
|
||||
// Build logger
|
||||
logger, err := zapConfig.Build(zap.AddCaller(), zap.AddStacktrace(zapcore.ErrorLevel))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Note: We still replace global logger for backward compatibility with parsers package
|
||||
// This will be removed once parsers are fully migrated to use dependency injection
|
||||
zap.ReplaceGlobals(logger)
|
||||
|
||||
return logger, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
package nats
|
||||
|
||||
import (
|
||||
"caatsm/internal/app"
|
||||
"caatsm/internal/infra/config"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"go.uber.org/zap"
|
||||
"github.com/nats-io/nats.go"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Consumer handles NATS JetStream message consumption
|
||||
type Consumer struct {
|
||||
js nats.JetStreamContext
|
||||
processor *app.MessageProcessor
|
||||
cfg *config.Config
|
||||
logger *zap.Logger
|
||||
subject string
|
||||
consumerName string
|
||||
}
|
||||
|
||||
// ProvideConsumer creates a NATS consumer
|
||||
func ProvideConsumer(
|
||||
js nats.JetStreamContext,
|
||||
processor *app.MessageProcessor,
|
||||
cfg *config.Config,
|
||||
logger *zap.Logger,
|
||||
) (*Consumer, error) {
|
||||
subject := cfg.Subscription.Topic
|
||||
if subject == "" {
|
||||
subject = "telegram.>"
|
||||
}
|
||||
|
||||
consumerName := cfg.NATS.Consumer
|
||||
if consumerName == "" {
|
||||
consumerName = "telegram-consumer"
|
||||
}
|
||||
|
||||
consumer := &Consumer{
|
||||
js: js,
|
||||
processor: processor,
|
||||
cfg: cfg,
|
||||
logger: logger,
|
||||
subject: subject,
|
||||
consumerName: consumerName,
|
||||
}
|
||||
|
||||
// Create consumer if it doesn't exist
|
||||
if err := consumer.ensureConsumer(); err != nil {
|
||||
return nil, fmt.Errorf("failed to ensure consumer: %w", err)
|
||||
}
|
||||
|
||||
return consumer, nil
|
||||
}
|
||||
|
||||
// ensureConsumer creates the consumer if it doesn't exist
|
||||
func (c *Consumer) ensureConsumer() error {
|
||||
streamName := c.cfg.NATS.Stream
|
||||
if streamName == "" {
|
||||
streamName = "TELEGRAM"
|
||||
}
|
||||
|
||||
consumerConfig := &nats.ConsumerConfig{
|
||||
Durable: c.consumerName,
|
||||
DeliverPolicy: nats.DeliverAllPolicy,
|
||||
AckPolicy: nats.AckExplicitPolicy,
|
||||
AckWait: c.cfg.Timeouts.AckWait,
|
||||
MaxDeliver: 5, // Maximum number of delivery attempts
|
||||
FilterSubject: c.subject,
|
||||
}
|
||||
|
||||
_, err := c.js.AddConsumer(streamName, consumerConfig)
|
||||
if err != nil && err != nats.ErrConsumerNameAlreadyInUse {
|
||||
return fmt.Errorf("failed to create consumer: %w", err)
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
c.logger.Info("Created JetStream consumer",
|
||||
zap.String("consumer", c.consumerName),
|
||||
zap.String("stream", streamName),
|
||||
zap.String("subject", c.subject),
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Start starts consuming messages
|
||||
func (c *Consumer) Start(ctx context.Context) error {
|
||||
streamName := c.cfg.NATS.Stream
|
||||
if streamName == "" {
|
||||
streamName = "TELEGRAM"
|
||||
}
|
||||
|
||||
// Create pull subscription
|
||||
sub, err := c.js.PullSubscribe(c.subject, c.consumerName, nats.Bind(streamName, c.consumerName))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create pull subscription: %w", err)
|
||||
}
|
||||
defer sub.Unsubscribe()
|
||||
|
||||
c.logger.Info("Started consuming messages",
|
||||
zap.String("subject", c.subject),
|
||||
zap.String("consumer", c.consumerName),
|
||||
zap.String("stream", streamName),
|
||||
)
|
||||
|
||||
batchSize := c.cfg.App.BatchSize
|
||||
if batchSize == 0 {
|
||||
batchSize = 50
|
||||
}
|
||||
batchTimeout := c.cfg.App.BatchTimeout
|
||||
if batchTimeout == 0 {
|
||||
batchTimeout = 2 * time.Second
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
c.logger.Info("Stopping consumer", zap.Error(ctx.Err()))
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
// Fetch messages in batch
|
||||
msgs, err := sub.Fetch(batchSize, nats.MaxWait(batchTimeout))
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrTimeout) {
|
||||
// Timeout is expected when no messages are available
|
||||
continue
|
||||
}
|
||||
c.logger.Error("Failed to fetch messages", zap.Error(err))
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
// Process each message
|
||||
for _, msg := range msgs {
|
||||
if err := c.processMessage(ctx, msg); err != nil {
|
||||
c.logger.Error("Failed to process message",
|
||||
zap.String("subject", msg.Subject),
|
||||
zap.Error(err),
|
||||
)
|
||||
// NAK the message to retry
|
||||
if nakErr := msg.Nak(); nakErr != nil {
|
||||
c.logger.Error("Failed to NAK message", zap.Error(nakErr))
|
||||
}
|
||||
} else {
|
||||
// ACK the message
|
||||
if ackErr := msg.Ack(); ackErr != nil {
|
||||
c.logger.Error("Failed to ACK message", zap.Error(ackErr))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// processMessage processes a single message
|
||||
func (c *Consumer) processMessage(ctx context.Context, msg *nats.Msg) error {
|
||||
msgID := msg.Header.Get("Nats-Msg-Id")
|
||||
if msgID == "" {
|
||||
// Use reply subject or generate a simple ID
|
||||
if msg.Reply != "" {
|
||||
msgID = msg.Reply
|
||||
} else {
|
||||
msgID = fmt.Sprintf("msg-%d", time.Now().UnixNano())
|
||||
}
|
||||
}
|
||||
|
||||
c.logger.Debug("Processing message",
|
||||
zap.String("subject", msg.Subject),
|
||||
zap.String("msg_id", msgID),
|
||||
zap.Int("data_size", len(msg.Data)),
|
||||
)
|
||||
|
||||
// Call processor
|
||||
if err := c.processor.Handle(ctx, msg.Data, msgID); err != nil {
|
||||
return fmt.Errorf("processor error: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package nats
|
||||
|
||||
import (
|
||||
"caatsm/internal/infra/config"
|
||||
"fmt"
|
||||
"go.uber.org/zap"
|
||||
"github.com/nats-io/nats.go"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ProvideJetStream creates a NATS JetStream connection
|
||||
func ProvideJetStream(cfg *config.Config, logger *zap.Logger) (nats.JetStreamContext, error) {
|
||||
// Connect to NATS
|
||||
nc, err := nats.Connect(
|
||||
cfg.NATS.URL,
|
||||
nats.RetryOnFailedConnect(true),
|
||||
nats.Timeout(cfg.Timeouts.Server),
|
||||
nats.ReconnectWait(cfg.Timeouts.ReconnectWait),
|
||||
nats.DisconnectErrHandler(func(nc *nats.Conn, err error) {
|
||||
if err != nil {
|
||||
logger.Warn("NATS disconnected", zap.Error(err))
|
||||
}
|
||||
}),
|
||||
nats.ReconnectHandler(func(nc *nats.Conn) {
|
||||
logger.Info("NATS reconnected", zap.String("url", nc.ConnectedUrl()))
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to NATS: %w", err)
|
||||
}
|
||||
|
||||
// Get JetStream context
|
||||
js, err := nc.JetStream()
|
||||
if err != nil {
|
||||
nc.Close()
|
||||
return nil, fmt.Errorf("failed to get JetStream context: %w", err)
|
||||
}
|
||||
|
||||
// Create stream if it doesn't exist
|
||||
streamName := cfg.NATS.Stream
|
||||
subject := cfg.Subscription.Topic
|
||||
if subject == "" {
|
||||
subject = "telegram.>"
|
||||
}
|
||||
|
||||
streamConfig := &nats.StreamConfig{
|
||||
Name: streamName,
|
||||
Subjects: []string{subject},
|
||||
Retention: nats.LimitsPolicy,
|
||||
MaxAge: 24 * time.Hour,
|
||||
Storage: nats.FileStorage,
|
||||
Replicas: 1,
|
||||
}
|
||||
|
||||
_, err = js.AddStream(streamConfig)
|
||||
if err != nil && err != nats.ErrStreamNameAlreadyInUse {
|
||||
nc.Close()
|
||||
return nil, fmt.Errorf("failed to create stream: %w", err)
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
logger.Info("Created JetStream", zap.String("stream", streamName), zap.String("subject", subject))
|
||||
}
|
||||
|
||||
return js, nil
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package nats
|
||||
|
||||
import (
|
||||
"caatsm/internal/adapter"
|
||||
"caatsm/internal/infra/config"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"go.uber.org/zap"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// Publisher publishes messages to NATS JetStream
|
||||
type Publisher struct {
|
||||
js nats.JetStreamContext
|
||||
cfg *config.Config
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// ProvidePublisher creates a NATS publisher
|
||||
func ProvidePublisher(
|
||||
js nats.JetStreamContext,
|
||||
cfg *config.Config,
|
||||
logger *zap.Logger,
|
||||
) (adapter.Publisher, error) {
|
||||
return &Publisher{
|
||||
js: js,
|
||||
cfg: cfg,
|
||||
logger: logger,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Publish publishes a message
|
||||
func (p *Publisher) Publish(message interface{}) error {
|
||||
topic := p.cfg.Publisher.Topic
|
||||
if topic == "" {
|
||||
return fmt.Errorf("publisher topic is not configured")
|
||||
}
|
||||
|
||||
// Marshal message to JSON
|
||||
messageBytes, err := json.Marshal(message)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal message: %w", err)
|
||||
}
|
||||
|
||||
// Publish to JetStream
|
||||
_, err = p.js.Publish(topic, messageBytes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to publish message: %w", err)
|
||||
}
|
||||
|
||||
p.logger.Debug("Published message",
|
||||
zap.String("topic", topic),
|
||||
zap.Int("size", len(messageBytes)),
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"caatsm/internal/infra/config"
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// ProvideDB creates a PostgreSQL connection pool
|
||||
func ProvideDB(cfg *config.Config) (*pgxpool.Pool, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
poolConfig, err := pgxpool.ParseConfig(cfg.Postgres.URL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse postgres URL: %w", err)
|
||||
}
|
||||
|
||||
poolConfig.MaxConns = int32(cfg.Postgres.MaxConns)
|
||||
poolConfig.MinConns = int32(cfg.Postgres.MinConns)
|
||||
poolConfig.MaxConnLifetime = time.Hour
|
||||
poolConfig.MaxConnIdleTime = time.Minute * 30
|
||||
|
||||
pool, err := pgxpool.NewWithConfig(ctx, poolConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create connection pool: %w", err)
|
||||
}
|
||||
|
||||
// Test connection
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
return nil, fmt.Errorf("failed to ping database: %w", err)
|
||||
}
|
||||
|
||||
return pool, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"caatsm/internal/adapter"
|
||||
"caatsm/internal/adapter/mapper"
|
||||
"caatsm/internal/domain"
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// Repository implements the adapter.Repository interface using PostgreSQL
|
||||
type Repository struct {
|
||||
pool *pgxpool.Pool
|
||||
mapper *mapper.TelegramMapper
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// ProvideRepository creates a PostgreSQL repository
|
||||
func ProvideRepository(pool *pgxpool.Pool, logger *zap.Logger) (adapter.Repository, error) {
|
||||
return &Repository{
|
||||
pool: pool,
|
||||
mapper: mapper.NewTelegramMapper(),
|
||||
logger: logger,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// InsertOne inserts a single telegram message
|
||||
func (r *Repository) InsertOne(ctx context.Context, msg *domain.ParsedMessage) error {
|
||||
row, err := r.mapper.ToDBRow(msg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to map message to DB row: %w", err)
|
||||
}
|
||||
|
||||
query := `
|
||||
INSERT INTO aviation.telegrams (
|
||||
uuid, message_id, date_time, priority_indicator, primary_address,
|
||||
secondary_addresses, originator, originator_date_time, category,
|
||||
content, body_data, received_at, parsed_at, dispatched_at, need_dispatch
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15
|
||||
)
|
||||
ON CONFLICT (uuid) DO NOTHING
|
||||
`
|
||||
|
||||
_, err = r.pool.Exec(ctx, query,
|
||||
row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8],
|
||||
row[9], row[10], row[11], row[12], row[13], row[14],
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to insert message: %w", err)
|
||||
}
|
||||
|
||||
r.logger.Debug("Inserted message",
|
||||
zap.String("uuid", msg.Uuid),
|
||||
zap.String("message_id", msg.MessageID),
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// InsertBatch inserts multiple telegram messages in a batch using CopyFrom
|
||||
func (r *Repository) InsertBatch(ctx context.Context, msgs []*domain.ParsedMessage) error {
|
||||
if len(msgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Convert messages to rows
|
||||
rows := make([][]interface{}, len(msgs))
|
||||
for i, msg := range msgs {
|
||||
row, err := r.mapper.ToDBRow(msg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to map message %d to DB row: %w", i, err)
|
||||
}
|
||||
rows[i] = row
|
||||
}
|
||||
|
||||
// Use CopyFrom for efficient batch insert
|
||||
copyCount, err := r.pool.CopyFrom(
|
||||
ctx,
|
||||
pgx.Identifier{"aviation", "telegrams"},
|
||||
[]string{
|
||||
"uuid", "message_id", "date_time", "priority_indicator", "primary_address",
|
||||
"secondary_addresses", "originator", "originator_date_time", "category",
|
||||
"content", "body_data", "received_at", "parsed_at", "dispatched_at", "need_dispatch",
|
||||
},
|
||||
pgx.CopyFromRows(rows),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to batch insert messages: %w", err)
|
||||
}
|
||||
|
||||
r.logger.Info("Batch inserted messages",
|
||||
zap.Int("count", int(copyCount)),
|
||||
zap.Int("attempted", len(msgs)),
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user