use submodule
This commit is contained in:
fengzhiqiang
+1
Submodule vendor/github.com/segmentio/kafka-go added at 963714c486
-34
@@ -1,34 +0,0 @@
|
||||
# Compiled Object files, Static and Dynamic libs (Shared Objects)
|
||||
*.o
|
||||
*.a
|
||||
*.so
|
||||
|
||||
# Folders
|
||||
_obj
|
||||
_test
|
||||
|
||||
# Architecture specific extensions/prefixes
|
||||
*.[568vq]
|
||||
[568vq].out
|
||||
|
||||
*.cgo1.go
|
||||
*.cgo2.c
|
||||
_cgo_defun.c
|
||||
_cgo_gotypes.go
|
||||
_cgo_export.*
|
||||
|
||||
_testmain.go
|
||||
|
||||
*.exe
|
||||
*.test
|
||||
*.prof
|
||||
/kafkacli
|
||||
|
||||
# Emacs
|
||||
*~
|
||||
|
||||
# Goland
|
||||
.idea
|
||||
|
||||
# govendor
|
||||
/vendor/*/
|
||||
-21
@@ -1,21 +0,0 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2017 Segment
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
-301
@@ -1,301 +0,0 @@
|
||||
# kafka-go [](https://circleci.com/gh/segmentio/kafka-go) [](https://goreportcard.com/report/github.com/segmentio/kafka-go) [](https://godoc.org/github.com/segmentio/kafka-go)
|
||||
|
||||
## Motivations
|
||||
|
||||
We rely on both Go and Kafka a lot at Segment. Unfortunately, the state of the Go
|
||||
client libraries for Kafka at the time of this writing was not ideal. The available
|
||||
options were:
|
||||
|
||||
- [sarama](https://github.com/Shopify/sarama), which is by far the most popular
|
||||
but is quite difficult to work with. It is poorly documented, the API exposes
|
||||
low level concepts of the Kafka protocol, and it doesn't support recent Go features
|
||||
like [contexts](https://golang.org/pkg/context/). It also passes all values as
|
||||
pointers which causes large numbers of dynamic memory allocations, more frequent
|
||||
garbage collections, and higher memory usage.
|
||||
|
||||
- [confluent-kafka-go](https://github.com/confluentinc/confluent-kafka-go) is a
|
||||
cgo based wrapper around [librdkafka](https://github.com/edenhill/librdkafka),
|
||||
which means it introduces a dependency to a C library on all Go code that uses
|
||||
the package. It has much better documentation than sarama but still lacks support
|
||||
for Go contexts.
|
||||
|
||||
- [goka](https://github.com/lovoo/goka) is a more recent Kafka client for Go
|
||||
which focuses on a specific usage pattern. It provides abstractions for using Kafka
|
||||
as a message passing bus between services rather than an ordered log of events, but
|
||||
this is not the typical use case of Kafka for us at Segment. The package also
|
||||
depends on sarama for all interactions with Kafka.
|
||||
|
||||
This is where `kafka-go` comes into play. It provides both low and high level
|
||||
APIs for interacting with Kafka, mirroring concepts and implementing interfaces of
|
||||
the Go standard library to make it easy to use and integrate with existing
|
||||
software.
|
||||
|
||||
## Connection [](https://godoc.org/github.com/segmentio/kafka-go#Conn)
|
||||
|
||||
The `Conn` type is the core of the `kafka-go` package. It wraps around a raw
|
||||
network connection to expose a low-level API to a Kafka server.
|
||||
|
||||
Here are some examples showing typical use of a connection object:
|
||||
```go
|
||||
// to produce messages
|
||||
topic := "my-topic"
|
||||
partition := 0
|
||||
|
||||
conn, _ := kafka.DialLeader(context.Background(), "tcp", "localhost:9092", topic, partition)
|
||||
|
||||
conn.SetWriteDeadline(time.Now().Add(10*time.Second))
|
||||
conn.WriteMessages(
|
||||
kafka.Message{Value: []byte("one!")},
|
||||
kafka.Message{Value: []byte("two!")},
|
||||
kafka.Message{Value: []byte("three!")},
|
||||
)
|
||||
|
||||
conn.Close()
|
||||
```
|
||||
```go
|
||||
// to consume messages
|
||||
topic := "my-topic"
|
||||
partition := 0
|
||||
|
||||
conn, _ := kafka.DialLeader(context.Background(), "tcp", "localhost:9092", topic, partition)
|
||||
|
||||
conn.SetReadDeadline(time.Now().Add(10*time.Second))
|
||||
batch := conn.ReadBatch(10e3, 1e6) // fetch 10KB min, 1MB max
|
||||
|
||||
b := make([]byte, 10e3) // 10KB max per message
|
||||
for {
|
||||
_, err := batch.Read(b)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
fmt.Println(string(b))
|
||||
}
|
||||
|
||||
batch.Close()
|
||||
conn.Close()
|
||||
```
|
||||
|
||||
Because it is low level, the `Conn` type turns out to be a great building block
|
||||
for higher level abstractions, like the `Reader` for example.
|
||||
|
||||
## Reader [](https://godoc.org/github.com/segmentio/kafka-go#Reader)
|
||||
|
||||
A `Reader` is another concept exposed by the `kafka-go` package, which intends
|
||||
to make it simpler to implement the typical use case of consuming from a single
|
||||
topic-partition pair.
|
||||
A `Reader` also automatically handles reconnections and offset management, and
|
||||
exposes an API that supports asynchronous cancellations and timeouts using Go
|
||||
contexts.
|
||||
|
||||
```go
|
||||
// make a new reader that consumes from topic-A, partition 0, at offset 42
|
||||
r := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: []string{"localhost:9092"},
|
||||
Topic: "topic-A",
|
||||
Partition: 0,
|
||||
MinBytes: 10e3, // 10KB
|
||||
MaxBytes: 10e6, // 10MB
|
||||
})
|
||||
r.SetOffset(42)
|
||||
|
||||
for {
|
||||
m, err := r.ReadMessage(context.Background())
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
fmt.Printf("message at offset %d: %s = %s\n", m.Offset, string(m.Key), string(m.Value))
|
||||
}
|
||||
|
||||
r.Close()
|
||||
```
|
||||
|
||||
### Consumer Groups
|
||||
|
||||
```kafka-go``` also supports Kafka consumer groups including broker managed offsets.
|
||||
To enable consumer groups, simplify specify the GroupID in the ReaderConfig.
|
||||
|
||||
ReadMessage automatically commits offsets when using consumer groups.
|
||||
|
||||
```go
|
||||
// make a new reader that consumes from topic-A
|
||||
r := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: []string{"localhost:9092"},
|
||||
GroupID: "consumer-group-id",
|
||||
Topic: "topic-A",
|
||||
MinBytes: 10e3, // 10KB
|
||||
MaxBytes: 10e6, // 10MB
|
||||
})
|
||||
|
||||
for {
|
||||
m, err := r.ReadMessage(context.Background())
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
fmt.Printf("message at topic/partition/offset %v/%v/%v: %s = %s\n", m.Topic, m.Partition, m.Offset, string(m.Key), string(m.Value))
|
||||
}
|
||||
|
||||
r.Close()
|
||||
```
|
||||
|
||||
There are a number of limitations when using consumer groups:
|
||||
|
||||
* ```(*Reader).SetOffset``` will return an error when GroupID is set
|
||||
* ```(*Reader).Offset``` will always return ```-1``` when GroupID is set
|
||||
* ```(*Reader).Lag``` will always return ```-1``` when GroupID is set
|
||||
* ```(*Reader).ReadLag``` will return an error when GroupID is set
|
||||
* ```(*Reader).Stats``` will return a partition of ```-1``` when GroupID is set
|
||||
|
||||
### Explicit Commits
|
||||
|
||||
```kafka-go``` also supports explicit commits. Instead of calling ```ReadMessage```,
|
||||
call ```FetchMessage``` followed by ```CommitMessages```.
|
||||
|
||||
```go
|
||||
ctx := context.Background()
|
||||
for {
|
||||
m, err := r.FetchMessage(ctx)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
fmt.Printf("message at topic/partition/offset %v/%v/%v: %s = %s\n", m.Topic, m.Partition, m.Offset, string(m.Key), string(m.Value))
|
||||
r.CommitMessages(ctx, m)
|
||||
}
|
||||
```
|
||||
|
||||
### Managing Commits
|
||||
|
||||
By default, CommitMessages will synchronously commit offsets to Kafka. For
|
||||
improved performance, you can instead periodically commit offsets to Kafka
|
||||
by setting CommitInterval on the ReaderConfig.
|
||||
|
||||
|
||||
```go
|
||||
// make a new reader that consumes from topic-A
|
||||
r := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: []string{"localhost:9092"},
|
||||
GroupID: "consumer-group-id",
|
||||
Topic: "topic-A",
|
||||
MinBytes: 10e3, // 10KB
|
||||
MaxBytes: 10e6, // 10MB
|
||||
CommitInterval: time.Second, // flushes commits to Kafka every second
|
||||
})
|
||||
```
|
||||
|
||||
## Writer [](https://godoc.org/github.com/segmentio/kafka-go#Writer)
|
||||
|
||||
To produce messages to Kafka, a program may use the low-level `Conn` API, but
|
||||
the package also provides a higher level `Writer` type which is more appropriate
|
||||
to use in most cases as it provides additional features:
|
||||
|
||||
- Automatic retries and reconnections on errors.
|
||||
- Configurable distribution of messages across available partitions.
|
||||
- Synchronous or asynchronous writes of messages to Kafka.
|
||||
- Asynchronous cancellation using contexts.
|
||||
- Flushing of pending messages on close to support graceful shutdowns.
|
||||
|
||||
```go
|
||||
// make a writer that produces to topic-A, using the least-bytes distribution
|
||||
w := kafka.NewWriter(kafka.WriterConfig{
|
||||
Brokers: []string{"localhost:9092"},
|
||||
Topic: "topic-A",
|
||||
Balancer: &kafka.LeastBytes{},
|
||||
})
|
||||
|
||||
w.WriteMessages(context.Background(),
|
||||
kafka.Message{
|
||||
Key: []byte("Key-A"),
|
||||
Value: []byte("Hello World!"),
|
||||
},
|
||||
kafka.Message{
|
||||
Key: []byte("Key-B"),
|
||||
Value: []byte("One!"),
|
||||
},
|
||||
kafka.Message{
|
||||
Key: []byte("Key-C"),
|
||||
Value: []byte("Two!"),
|
||||
},
|
||||
)
|
||||
|
||||
w.Close()
|
||||
```
|
||||
|
||||
**Note:** Even though kafka.Message contain ```Topic``` and ```Partition``` fields, they **MUST NOT** be
|
||||
set when writing messages. They are intended for read use only.
|
||||
|
||||
### Compatibility with Sarama
|
||||
|
||||
If you're switching from Sarama and need/want to use the same algorithm for message
|
||||
partitioning, you can use the ```kafka.Hash``` balancer. ```kafka.Hash``` routes
|
||||
messages to the same partitions that sarama's default partitioner would route to.
|
||||
|
||||
```go
|
||||
w := kafka.NewWriter(kafka.WriterConfig{
|
||||
Brokers: []string{"localhost:9092"},
|
||||
Topic: "topic-A",
|
||||
Balancer: &kafka.Hash{},
|
||||
})
|
||||
```
|
||||
|
||||
### Compression
|
||||
|
||||
Compression can be enable on the writer :
|
||||
|
||||
```go
|
||||
w := kafka.NewWriter(kafka.WriterConfig{
|
||||
Brokers: []string{"localhost:9092"},
|
||||
Topic: "topic-A",
|
||||
CompressionCodec: snappy.NewCompressionCodec(),
|
||||
})
|
||||
```
|
||||
|
||||
The reader will by default figure out if the consumed messages are compressed by intepreting the message attributes.
|
||||
|
||||
## TLS Support
|
||||
|
||||
For a bare bones Conn type or in the Reader/Writer configs you can specify a dialer option for TLS support. If the TLS field is nil, it will not connect with TLS.
|
||||
|
||||
### Connection
|
||||
|
||||
```go
|
||||
dialer := &kafka.Dialer{
|
||||
Timeout: 10 * time.Second,
|
||||
DualStack: true,
|
||||
TLS: &tls.Config{...tls config...},
|
||||
}
|
||||
|
||||
conn, err := dialer.DialContext(ctx, "tcp", "localhost:9093")
|
||||
```
|
||||
|
||||
### Reader
|
||||
|
||||
```go
|
||||
dialer := &kafka.Dialer{
|
||||
Timeout: 10 * time.Second,
|
||||
DualStack: true,
|
||||
TLS: &tls.Config{...tls config...},
|
||||
}
|
||||
|
||||
r := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: []string{"localhost:9093"},
|
||||
GroupID: "consumer-group-id",
|
||||
Topic: "topic-A",
|
||||
Dialer: dialer,
|
||||
})
|
||||
```
|
||||
|
||||
### Writer
|
||||
|
||||
```go
|
||||
dialer := &kafka.Dialer{
|
||||
Timeout: 10 * time.Second,
|
||||
DualStack: true,
|
||||
TLS: &tls.Config{...tls config...},
|
||||
}
|
||||
|
||||
w := kafka.NewWriter(kafka.WriterConfig{
|
||||
Brokers: []string{"localhost:9093"},
|
||||
Topic: "topic-A",
|
||||
Balancer: &kafka.Hash{},
|
||||
Dialer: dialer,
|
||||
})
|
||||
```
|
||||
-160
@@ -1,160 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"hash"
|
||||
"hash/fnv"
|
||||
"sort"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// The Balancer interface provides an abstraction of the message distribution
|
||||
// logic used by Writer instances to route messages to the partitions available
|
||||
// on a kafka cluster.
|
||||
//
|
||||
// Instances of Balancer do not have to be safe to use concurrently by multiple
|
||||
// goroutines, the Writer implementation ensures that calls to Balance are
|
||||
// synchronized.
|
||||
type Balancer interface {
|
||||
// Balance receives a message and a set of available partitions and
|
||||
// returns the partition number that the message should be routed to.
|
||||
//
|
||||
// An application should refrain from using a balancer to manage multiple
|
||||
// sets of partitions (from different topics for examples), use one balancer
|
||||
// instance for each partition set, so the balancer can detect when the
|
||||
// partitions change and assume that the kafka topic has been rebalanced.
|
||||
Balance(msg Message, partitions ...int) (partition int)
|
||||
}
|
||||
|
||||
// BalancerFunc is an implementation of the Balancer interface that makes it
|
||||
// possible to use regular functions to distribute messages across partitions.
|
||||
type BalancerFunc func(Message, ...int) int
|
||||
|
||||
// Balance calls f, satisfies the Balancer interface.
|
||||
func (f BalancerFunc) Balance(msg Message, partitions ...int) int {
|
||||
return f(msg, partitions...)
|
||||
}
|
||||
|
||||
// RoundRobin is an Balancer implementation that equally distributes messages
|
||||
// across all available partitions.
|
||||
type RoundRobin struct {
|
||||
offset uint64
|
||||
}
|
||||
|
||||
// Balance satisfies the Balancer interface.
|
||||
func (rr *RoundRobin) Balance(msg Message, partitions ...int) int {
|
||||
length := uint64(len(partitions))
|
||||
offset := rr.offset
|
||||
rr.offset++
|
||||
return partitions[offset%length]
|
||||
}
|
||||
|
||||
// LeastBytes is a Balancer implementation that routes messages to the partition
|
||||
// that has received the least amount of data.
|
||||
//
|
||||
// Note that no coordination is done between multiple producers, having good
|
||||
// balancing relies on the fact that each producer using a LeastBytes balancer
|
||||
// should produce well balanced messages.
|
||||
type LeastBytes struct {
|
||||
counters []leastBytesCounter
|
||||
}
|
||||
|
||||
type leastBytesCounter struct {
|
||||
partition int
|
||||
bytes uint64
|
||||
}
|
||||
|
||||
// Balance satisfies the Balancer interface.
|
||||
func (lb *LeastBytes) Balance(msg Message, partitions ...int) int {
|
||||
for _, p := range partitions {
|
||||
if c := lb.counterOf(p); c == nil {
|
||||
lb.counters = lb.makeCounters(partitions...)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
minBytes := lb.counters[0].bytes
|
||||
minIndex := 0
|
||||
|
||||
for i, c := range lb.counters[1:] {
|
||||
if c.bytes < minBytes {
|
||||
minIndex = i + 1
|
||||
minBytes = c.bytes
|
||||
}
|
||||
}
|
||||
|
||||
c := &lb.counters[minIndex]
|
||||
c.bytes += uint64(len(msg.Key)) + uint64(len(msg.Value))
|
||||
return c.partition
|
||||
}
|
||||
|
||||
func (lb *LeastBytes) counterOf(partition int) *leastBytesCounter {
|
||||
i := sort.Search(len(lb.counters), func(i int) bool {
|
||||
return lb.counters[i].partition >= partition
|
||||
})
|
||||
if i == len(lb.counters) || lb.counters[i].partition != partition {
|
||||
return nil
|
||||
}
|
||||
return &lb.counters[i]
|
||||
}
|
||||
|
||||
func (lb *LeastBytes) makeCounters(partitions ...int) (counters []leastBytesCounter) {
|
||||
counters = make([]leastBytesCounter, len(partitions))
|
||||
|
||||
for i, p := range partitions {
|
||||
counters[i].partition = p
|
||||
}
|
||||
|
||||
sort.Slice(counters, func(i int, j int) bool {
|
||||
return counters[i].partition < counters[j].partition
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
fnv1aPool = &sync.Pool{
|
||||
New: func() interface{} {
|
||||
return fnv.New32a()
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
// Hash is a Balancer that uses the provided hash function to determine which
|
||||
// partition to route messages to. This ensures that messages with the same key
|
||||
// are routed to the same partition.
|
||||
//
|
||||
// The logic to calculate the partition is:
|
||||
//
|
||||
// hasher.Sum32() % len(partitions) => partition
|
||||
//
|
||||
// By default, Hash uses the FNV-1a algorithm. This is the same algorithm used
|
||||
// by the Sarama Producer and ensures that messages produced by kafka-go will
|
||||
// be delivered to the same topics that the Sarama producer would be delivered to
|
||||
type Hash struct {
|
||||
rr RoundRobin
|
||||
Hasher hash.Hash32
|
||||
}
|
||||
|
||||
func (h *Hash) Balance(msg Message, partitions ...int) (partition int) {
|
||||
if msg.Key == nil {
|
||||
return h.rr.Balance(msg, partitions...)
|
||||
}
|
||||
|
||||
hasher := h.Hasher
|
||||
if hasher == nil {
|
||||
hasher = fnv1aPool.Get().(hash.Hash32)
|
||||
defer fnv1aPool.Put(hasher)
|
||||
}
|
||||
|
||||
hasher.Reset()
|
||||
if _, err := hasher.Write(msg.Key); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// uses same algorithm that Sarama's hashPartitioner uses
|
||||
partition = int(hasher.Sum32()) % len(partitions)
|
||||
if partition < 0 {
|
||||
partition = -partition
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
-213
@@ -1,213 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A Batch is an iterator over a sequence of messages fetched from a kafka
|
||||
// server.
|
||||
//
|
||||
// Batches are created by calling (*Conn).ReadBatch. They hold a internal lock
|
||||
// on the connection, which is released when the batch is closed. Failing to
|
||||
// call a batch's Close method will likely result in a dead-lock when trying to
|
||||
// use the connection.
|
||||
//
|
||||
// Batches are safe to use concurrently from multiple goroutines.
|
||||
type Batch struct {
|
||||
mutex sync.Mutex
|
||||
conn *Conn
|
||||
lock *sync.Mutex
|
||||
msgs *messageSetReader
|
||||
deadline time.Time
|
||||
throttle time.Duration
|
||||
topic string
|
||||
partition int
|
||||
offset int64
|
||||
highWaterMark int64
|
||||
err error
|
||||
}
|
||||
|
||||
// Throttle gives the throttling duration applied by the kafka server on the
|
||||
// connection.
|
||||
func (batch *Batch) Throttle() time.Duration {
|
||||
return batch.throttle
|
||||
}
|
||||
|
||||
// Watermark returns the current highest watermark in a partition.
|
||||
func (batch *Batch) HighWaterMark() int64 {
|
||||
return batch.highWaterMark
|
||||
}
|
||||
|
||||
// Offset returns the offset of the next message in the batch.
|
||||
func (batch *Batch) Offset() int64 {
|
||||
batch.mutex.Lock()
|
||||
offset := batch.offset
|
||||
batch.mutex.Unlock()
|
||||
return offset
|
||||
}
|
||||
|
||||
// Close closes the batch, releasing the connection lock and returning an error
|
||||
// if reading the batch failed for any reason.
|
||||
func (batch *Batch) Close() error {
|
||||
batch.mutex.Lock()
|
||||
err := batch.close()
|
||||
batch.mutex.Unlock()
|
||||
return err
|
||||
}
|
||||
|
||||
func (batch *Batch) close() (err error) {
|
||||
conn := batch.conn
|
||||
lock := batch.lock
|
||||
|
||||
batch.conn = nil
|
||||
batch.lock = nil
|
||||
if batch.msgs != nil {
|
||||
batch.msgs.discard()
|
||||
}
|
||||
|
||||
if err = batch.err; err == io.EOF {
|
||||
err = nil
|
||||
}
|
||||
|
||||
if conn != nil {
|
||||
conn.rdeadline.unsetConnReadDeadline()
|
||||
conn.mutex.Lock()
|
||||
conn.offset = batch.offset
|
||||
conn.mutex.Unlock()
|
||||
|
||||
if err != nil {
|
||||
if _, ok := err.(Error); !ok && err != io.ErrShortBuffer {
|
||||
conn.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if lock != nil {
|
||||
lock.Unlock()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Read reads the value of the next message from the batch into b, returning the
|
||||
// number of bytes read, or an error if the next message couldn't be read.
|
||||
//
|
||||
// If an error is returned the batch cannot be used anymore and calling Read
|
||||
// again will keep returning that error. All errors except io.EOF (indicating
|
||||
// that the program consumed all messages from the batch) are also returned by
|
||||
// Close.
|
||||
//
|
||||
// The method fails with io.ErrShortBuffer if the buffer passed as argument is
|
||||
// too small to hold the message value.
|
||||
func (batch *Batch) Read(b []byte) (int, error) {
|
||||
n := 0
|
||||
|
||||
batch.mutex.Lock()
|
||||
offset := batch.offset
|
||||
|
||||
_, _, err := batch.readMessage(
|
||||
func(r *bufio.Reader, size int, nbytes int) (int, error) {
|
||||
if nbytes < 0 {
|
||||
return size, nil
|
||||
}
|
||||
return discardN(r, size, nbytes)
|
||||
},
|
||||
func(r *bufio.Reader, size int, nbytes int) (int, error) {
|
||||
if nbytes < 0 {
|
||||
return size, nil
|
||||
}
|
||||
n = nbytes // return value
|
||||
if nbytes > len(b) {
|
||||
nbytes = len(b)
|
||||
}
|
||||
nbytes, err := io.ReadFull(r, b[:nbytes])
|
||||
if err != nil {
|
||||
return size - nbytes, err
|
||||
}
|
||||
return discardN(r, size-nbytes, n-nbytes)
|
||||
},
|
||||
)
|
||||
|
||||
if err == nil && n > len(b) {
|
||||
n, err = len(b), io.ErrShortBuffer
|
||||
batch.err = io.ErrShortBuffer
|
||||
batch.offset = offset // rollback
|
||||
}
|
||||
|
||||
batch.mutex.Unlock()
|
||||
return n, err
|
||||
}
|
||||
|
||||
// ReadMessage reads and return the next message from the batch.
|
||||
//
|
||||
// Because this method allocate memory buffers for the message key and value
|
||||
// it is less memory-efficient than Read, but has the advantage of never
|
||||
// failing with io.ErrShortBuffer.
|
||||
func (batch *Batch) ReadMessage() (Message, error) {
|
||||
msg := Message{}
|
||||
batch.mutex.Lock()
|
||||
|
||||
offset, timestamp, err := batch.readMessage(
|
||||
func(r *bufio.Reader, size int, nbytes int) (remain int, err error) {
|
||||
msg.Key, remain, err = readNewBytes(r, size, nbytes)
|
||||
return
|
||||
},
|
||||
func(r *bufio.Reader, size int, nbytes int) (remain int, err error) {
|
||||
msg.Value, remain, err = readNewBytes(r, size, nbytes)
|
||||
return
|
||||
},
|
||||
)
|
||||
|
||||
batch.mutex.Unlock()
|
||||
msg.Topic = batch.topic
|
||||
msg.Partition = batch.partition
|
||||
msg.Offset = offset
|
||||
msg.Time = timestampToTime(timestamp)
|
||||
|
||||
return msg, err
|
||||
}
|
||||
|
||||
func (batch *Batch) readMessage(
|
||||
key func(*bufio.Reader, int, int) (int, error),
|
||||
val func(*bufio.Reader, int, int) (int, error),
|
||||
) (offset int64, timestamp int64, err error) {
|
||||
if err = batch.err; err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
offset, timestamp, err = batch.msgs.readMessage(batch.offset, key, val)
|
||||
switch err {
|
||||
case nil:
|
||||
batch.offset = offset + 1
|
||||
case errShortRead:
|
||||
// As an "optimization" kafka truncates the returned response after
|
||||
// producing MaxBytes, which could then cause the code to return
|
||||
// errShortRead.
|
||||
err = batch.msgs.discard()
|
||||
switch {
|
||||
case err != nil:
|
||||
batch.err = err
|
||||
case batch.msgs.remaining() == 0:
|
||||
// Because we use the adjusted deadline we could end up returning
|
||||
// before the actual deadline occurred. This is necessary otherwise
|
||||
// timing out the connection for real could end up leaving it in an
|
||||
// unpredictable state, which would require closing it.
|
||||
// This design decision was made to maximize the chances of keeping
|
||||
// the connection open, the trade off being to lose precision on the
|
||||
// read deadline management.
|
||||
if !batch.deadline.IsZero() && time.Now().After(batch.deadline) {
|
||||
err = RequestTimedOut
|
||||
} else {
|
||||
err = io.EOF
|
||||
}
|
||||
batch.err = err
|
||||
}
|
||||
default:
|
||||
batch.err = err
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
-39
@@ -1,39 +0,0 @@
|
||||
package kafka
|
||||
|
||||
// A commit represents the instruction of publishing an update of the last
|
||||
// offset read by a program for a topic and partition.
|
||||
type commit struct {
|
||||
topic string
|
||||
partition int
|
||||
offset int64
|
||||
}
|
||||
|
||||
// makeCommit builds a commit value from a message, the resulting commit takes
|
||||
// its topic, partition, and offset from the message.
|
||||
func makeCommit(msg Message) commit {
|
||||
return commit{
|
||||
topic: msg.Topic,
|
||||
partition: msg.Partition,
|
||||
offset: msg.Offset + 1,
|
||||
}
|
||||
}
|
||||
|
||||
// makeCommits generates a slice of commits from a list of messages, it extracts
|
||||
// the topic, partition, and offset of each message and builds the corresponding
|
||||
// commit slice.
|
||||
func makeCommits(msgs ...Message) []commit {
|
||||
commits := make([]commit, len(msgs))
|
||||
|
||||
for i, m := range msgs {
|
||||
commits[i] = makeCommit(m)
|
||||
}
|
||||
|
||||
return commits
|
||||
}
|
||||
|
||||
// commitRequest is the data type exchanged between the CommitMessages method
|
||||
// and internals of the reader's implementation.
|
||||
type commitRequest struct {
|
||||
commits []commit
|
||||
errch chan<- error
|
||||
}
|
||||
-52
@@ -1,52 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var errUnknownCodec = errors.New("invalid codec")
|
||||
|
||||
var codecs = make(map[int8]CompressionCodec)
|
||||
var codecsMutex sync.RWMutex
|
||||
|
||||
// RegisterCompressionCodec registers a compression codec so it can be used by a Writer.
|
||||
func RegisterCompressionCodec(codec func() CompressionCodec) {
|
||||
c := codec()
|
||||
codecsMutex.Lock()
|
||||
codecs[c.Code()] = c
|
||||
codecsMutex.Unlock()
|
||||
}
|
||||
|
||||
// resolveCodec looks up a codec by Code()
|
||||
func resolveCodec(code int8) (codec CompressionCodec, err error) {
|
||||
codecsMutex.RLock()
|
||||
codec = codecs[code]
|
||||
codecsMutex.RUnlock()
|
||||
|
||||
if codec == nil {
|
||||
err = errUnknownCodec
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// CompressionCodec represents a compression codec to encode and decode
|
||||
// the messages.
|
||||
// See : https://cwiki.apache.org/confluence/display/KAFKA/Compression
|
||||
//
|
||||
// A CompressionCodec must be safe for concurrent access by multiple go
|
||||
// routines.
|
||||
type CompressionCodec interface {
|
||||
// Code returns the compression codec code
|
||||
Code() int8
|
||||
|
||||
// Encode encodes the src data
|
||||
Encode(src []byte) ([]byte, error)
|
||||
|
||||
// Decode decodes the src data
|
||||
Decode(src []byte) ([]byte, error)
|
||||
}
|
||||
|
||||
const compressionCodecMask int8 = 0x03
|
||||
const DefaultCompressionLevel int = -1
|
||||
const CompressionNoneCode = 0
|
||||
-1074
File diff suppressed because it is too large
Load Diff
-80
@@ -1,80 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"hash/crc32"
|
||||
"sync"
|
||||
)
|
||||
|
||||
func crc32OfMessage(magicByte int8, attributes int8, timestamp int64, key []byte, value []byte) uint32 {
|
||||
b := acquireCrc32Buffer()
|
||||
b.writeInt8(magicByte)
|
||||
b.writeInt8(attributes)
|
||||
if magicByte != 0 {
|
||||
b.writeInt64(timestamp)
|
||||
}
|
||||
b.writeBytes(key)
|
||||
b.writeBytes(value)
|
||||
sum := b.sum
|
||||
releaseCrc32Buffer(b)
|
||||
return sum
|
||||
}
|
||||
|
||||
type crc32Buffer struct {
|
||||
sum uint32
|
||||
buf bytes.Buffer
|
||||
}
|
||||
|
||||
func (c *crc32Buffer) writeInt8(i int8) {
|
||||
c.buf.Truncate(0)
|
||||
c.buf.WriteByte(byte(i))
|
||||
c.update()
|
||||
}
|
||||
|
||||
func (c *crc32Buffer) writeInt32(i int32) {
|
||||
a := [4]byte{}
|
||||
binary.BigEndian.PutUint32(a[:], uint32(i))
|
||||
c.buf.Truncate(0)
|
||||
c.buf.Write(a[:])
|
||||
c.update()
|
||||
}
|
||||
|
||||
func (c *crc32Buffer) writeInt64(i int64) {
|
||||
a := [8]byte{}
|
||||
binary.BigEndian.PutUint64(a[:], uint64(i))
|
||||
c.buf.Truncate(0)
|
||||
c.buf.Write(a[:])
|
||||
c.update()
|
||||
}
|
||||
|
||||
func (c *crc32Buffer) writeBytes(b []byte) {
|
||||
if b == nil {
|
||||
c.writeInt32(-1)
|
||||
} else {
|
||||
c.writeInt32(int32(len(b)))
|
||||
}
|
||||
c.sum = crc32Update(c.sum, b)
|
||||
}
|
||||
|
||||
func (c *crc32Buffer) update() {
|
||||
c.sum = crc32Update(c.sum, c.buf.Bytes())
|
||||
}
|
||||
|
||||
func crc32Update(sum uint32, b []byte) uint32 {
|
||||
return crc32.Update(sum, crc32.IEEETable, b)
|
||||
}
|
||||
|
||||
var crc32BufferPool = sync.Pool{
|
||||
New: func() interface{} { return &crc32Buffer{} },
|
||||
}
|
||||
|
||||
func acquireCrc32Buffer() *crc32Buffer {
|
||||
c := crc32BufferPool.Get().(*crc32Buffer)
|
||||
c.sum = 0
|
||||
return c
|
||||
}
|
||||
|
||||
func releaseCrc32Buffer(b *crc32Buffer) {
|
||||
crc32BufferPool.Put(b)
|
||||
}
|
||||
-267
@@ -1,267 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ConfigEntry struct {
|
||||
ConfigName string
|
||||
ConfigValue string
|
||||
}
|
||||
|
||||
func (c ConfigEntry) toCreateTopicsRequestV0ConfigEntry() createTopicsRequestV0ConfigEntry {
|
||||
return createTopicsRequestV0ConfigEntry{
|
||||
ConfigName: c.ConfigName,
|
||||
ConfigValue: c.ConfigValue,
|
||||
}
|
||||
}
|
||||
|
||||
type createTopicsRequestV0ConfigEntry struct {
|
||||
ConfigName string
|
||||
ConfigValue string
|
||||
}
|
||||
|
||||
func (t createTopicsRequestV0ConfigEntry) size() int32 {
|
||||
return sizeofString(t.ConfigName) +
|
||||
sizeofString(t.ConfigValue)
|
||||
}
|
||||
|
||||
func (t createTopicsRequestV0ConfigEntry) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.ConfigName)
|
||||
writeString(w, t.ConfigValue)
|
||||
}
|
||||
|
||||
type ReplicaAssignment struct {
|
||||
Partition int
|
||||
Replicas int
|
||||
}
|
||||
|
||||
func (a ReplicaAssignment) toCreateTopicsRequestV0ReplicaAssignment() createTopicsRequestV0ReplicaAssignment {
|
||||
return createTopicsRequestV0ReplicaAssignment{
|
||||
Partition: int32(a.Partition),
|
||||
Replicas: int32(a.Replicas),
|
||||
}
|
||||
}
|
||||
|
||||
type createTopicsRequestV0ReplicaAssignment struct {
|
||||
Partition int32
|
||||
Replicas int32
|
||||
}
|
||||
|
||||
func (t createTopicsRequestV0ReplicaAssignment) size() int32 {
|
||||
return sizeofInt32(t.Partition) +
|
||||
sizeofInt32(t.Replicas)
|
||||
}
|
||||
|
||||
func (t createTopicsRequestV0ReplicaAssignment) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, t.Partition)
|
||||
writeInt32(w, t.Replicas)
|
||||
}
|
||||
|
||||
type TopicConfig struct {
|
||||
// Topic name
|
||||
Topic string
|
||||
|
||||
// NumPartitions created. -1 indicates unset.
|
||||
NumPartitions int
|
||||
|
||||
// ReplicationFactor for the topic. -1 indicates unset.
|
||||
ReplicationFactor int
|
||||
|
||||
// ReplicaAssignments among kafka brokers for this topic partitions. If this
|
||||
// is set num_partitions and replication_factor must be unset.
|
||||
ReplicaAssignments []ReplicaAssignment
|
||||
|
||||
// ConfigEntries holds topic level configuration for topic to be set.
|
||||
ConfigEntries []ConfigEntry
|
||||
}
|
||||
|
||||
func (t TopicConfig) toCreateTopicsRequestV0Topic() createTopicsRequestV0Topic {
|
||||
var requestV0ReplicaAssignments []createTopicsRequestV0ReplicaAssignment
|
||||
for _, a := range t.ReplicaAssignments {
|
||||
requestV0ReplicaAssignments = append(
|
||||
requestV0ReplicaAssignments,
|
||||
a.toCreateTopicsRequestV0ReplicaAssignment())
|
||||
}
|
||||
var requestV0ConfigEntries []createTopicsRequestV0ConfigEntry
|
||||
for _, c := range t.ConfigEntries {
|
||||
requestV0ConfigEntries = append(
|
||||
requestV0ConfigEntries,
|
||||
c.toCreateTopicsRequestV0ConfigEntry())
|
||||
}
|
||||
|
||||
return createTopicsRequestV0Topic{
|
||||
Topic: t.Topic,
|
||||
NumPartitions: int32(t.NumPartitions),
|
||||
ReplicationFactor: int16(t.ReplicationFactor),
|
||||
ReplicaAssignments: requestV0ReplicaAssignments,
|
||||
ConfigEntries: requestV0ConfigEntries,
|
||||
}
|
||||
}
|
||||
|
||||
type createTopicsRequestV0Topic struct {
|
||||
// Topic name
|
||||
Topic string
|
||||
|
||||
// NumPartitions created. -1 indicates unset.
|
||||
NumPartitions int32
|
||||
|
||||
// ReplicationFactor for the topic. -1 indicates unset.
|
||||
ReplicationFactor int16
|
||||
|
||||
// ReplicaAssignments among kafka brokers for this topic partitions. If this
|
||||
// is set num_partitions and replication_factor must be unset.
|
||||
ReplicaAssignments []createTopicsRequestV0ReplicaAssignment
|
||||
|
||||
// ConfigEntries holds topic level configuration for topic to be set.
|
||||
ConfigEntries []createTopicsRequestV0ConfigEntry
|
||||
}
|
||||
|
||||
func (t createTopicsRequestV0Topic) size() int32 {
|
||||
return sizeofString(t.Topic) +
|
||||
sizeofInt32(t.NumPartitions) +
|
||||
sizeofInt16(t.ReplicationFactor) +
|
||||
sizeofArray(len(t.ReplicaAssignments), func(i int) int32 { return t.ReplicaAssignments[i].size() }) +
|
||||
sizeofArray(len(t.ConfigEntries), func(i int) int32 { return t.ConfigEntries[i].size() })
|
||||
}
|
||||
|
||||
func (t createTopicsRequestV0Topic) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.Topic)
|
||||
writeInt32(w, t.NumPartitions)
|
||||
writeInt16(w, t.ReplicationFactor)
|
||||
writeArray(w, len(t.ReplicaAssignments), func(i int) { t.ReplicaAssignments[i].writeTo(w) })
|
||||
writeArray(w, len(t.ConfigEntries), func(i int) { t.ConfigEntries[i].writeTo(w) })
|
||||
}
|
||||
|
||||
// See http://kafka.apache.org/protocol.html#The_Messages_CreateTopics
|
||||
type createTopicsRequestV0 struct {
|
||||
// Topics contains n array of single topic creation requests. Can not
|
||||
// have multiple entries for the same topic.
|
||||
Topics []createTopicsRequestV0Topic
|
||||
|
||||
// Timeout ms to wait for a topic to be completely created on the
|
||||
// controller node. Values <= 0 will trigger topic creation and return immediately
|
||||
Timeout int32
|
||||
}
|
||||
|
||||
func (t createTopicsRequestV0) size() int32 {
|
||||
return sizeofArray(len(t.Topics), func(i int) int32 { return t.Topics[i].size() }) +
|
||||
sizeofInt32(t.Timeout)
|
||||
}
|
||||
|
||||
func (t createTopicsRequestV0) writeTo(w *bufio.Writer) {
|
||||
writeArray(w, len(t.Topics), func(i int) { t.Topics[i].writeTo(w) })
|
||||
writeInt32(w, t.Timeout)
|
||||
}
|
||||
|
||||
type createTopicsResponseV0TopicError struct {
|
||||
// Topic name
|
||||
Topic string
|
||||
|
||||
// ErrorCode holds response error code
|
||||
ErrorCode int16
|
||||
}
|
||||
|
||||
func (t createTopicsResponseV0TopicError) size() int32 {
|
||||
return sizeofString(t.Topic) +
|
||||
sizeofInt16(t.ErrorCode)
|
||||
}
|
||||
|
||||
func (t createTopicsResponseV0TopicError) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.Topic)
|
||||
writeInt16(w, t.ErrorCode)
|
||||
}
|
||||
|
||||
func (t *createTopicsResponseV0TopicError) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readString(r, size, &t.Topic); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readInt16(r, remain, &t.ErrorCode); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// See http://kafka.apache.org/protocol.html#The_Messages_CreateTopics
|
||||
type createTopicsResponseV0 struct {
|
||||
TopicErrors []createTopicsResponseV0TopicError
|
||||
}
|
||||
|
||||
func (t createTopicsResponseV0) size() int32 {
|
||||
return sizeofArray(len(t.TopicErrors), func(i int) int32 { return t.TopicErrors[i].size() })
|
||||
}
|
||||
|
||||
func (t createTopicsResponseV0) writeTo(w *bufio.Writer) {
|
||||
writeArray(w, len(t.TopicErrors), func(i int) { t.TopicErrors[i].writeTo(w) })
|
||||
}
|
||||
|
||||
func (t *createTopicsResponseV0) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
fn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {
|
||||
var topic createTopicsResponseV0TopicError
|
||||
if fnRemain, fnErr = (&topic).readFrom(r, size); err != nil {
|
||||
return
|
||||
}
|
||||
t.TopicErrors = append(t.TopicErrors, topic)
|
||||
return
|
||||
}
|
||||
if remain, err = readArrayWith(r, size, fn); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (c *Conn) createTopics(request createTopicsRequestV0) (createTopicsResponseV0, error) {
|
||||
var response createTopicsResponseV0
|
||||
|
||||
err := c.writeOperation(
|
||||
func(deadline time.Time, id int32) error {
|
||||
if request.Timeout == 0 {
|
||||
now := time.Now()
|
||||
deadline = adjustDeadlineForRTT(deadline, now, defaultRTT)
|
||||
request.Timeout = milliseconds(deadlineToTimeout(deadline, now))
|
||||
}
|
||||
return c.writeRequest(createTopicsRequest, v0, id, request)
|
||||
},
|
||||
func(deadline time.Time, size int) error {
|
||||
return expectZeroSize(func() (remain int, err error) {
|
||||
return (&response).readFrom(&c.rbuf, size)
|
||||
}())
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return response, err
|
||||
}
|
||||
for _, tr := range response.TopicErrors {
|
||||
if tr.ErrorCode != 0 {
|
||||
return response, Error(tr.ErrorCode)
|
||||
}
|
||||
}
|
||||
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// CreateTopics creates one topic per provided configuration with idempotent
|
||||
// operational semantics. In other words, if CreateTopics is invoked with a
|
||||
// configuration for an existing topic, it will have no effect.
|
||||
func (c *Conn) CreateTopics(topics ...TopicConfig) error {
|
||||
var requestV0Topics []createTopicsRequestV0Topic
|
||||
for _, t := range topics {
|
||||
requestV0Topics = append(
|
||||
requestV0Topics,
|
||||
t.toCreateTopicsRequestV0Topic())
|
||||
}
|
||||
|
||||
_, err := c.createTopics(createTopicsRequestV0{
|
||||
Topics: requestV0Topics,
|
||||
})
|
||||
|
||||
switch err {
|
||||
case TopicAlreadyExists:
|
||||
// ok
|
||||
return nil
|
||||
default:
|
||||
return err
|
||||
}
|
||||
}
|
||||
-124
@@ -1,124 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"time"
|
||||
)
|
||||
|
||||
// See http://kafka.apache.org/protocol.html#The_Messages_DeleteTopics
|
||||
type deleteTopicsRequestV1 struct {
|
||||
// Topics holds the topic names
|
||||
Topics []string
|
||||
|
||||
// Timeout holds the time in ms to wait for a topic to be completely deleted
|
||||
// on the controller node. Values <= 0 will trigger topic deletion and return
|
||||
// immediately.
|
||||
Timeout int32
|
||||
}
|
||||
|
||||
func (t deleteTopicsRequestV1) size() int32 {
|
||||
return sizeofStringArray(t.Topics) +
|
||||
sizeofInt32(t.Timeout)
|
||||
}
|
||||
|
||||
func (t deleteTopicsRequestV1) writeTo(w *bufio.Writer) {
|
||||
writeStringArray(w, t.Topics)
|
||||
writeInt32(w, t.Timeout)
|
||||
}
|
||||
|
||||
type deleteTopicsResponseV1 struct {
|
||||
// ThrottleTimeMS holds the duration in milliseconds for which the request
|
||||
// was throttled due to quota violation (Zero if the request did not violate
|
||||
// any quota)
|
||||
ThrottleTimeMS int32
|
||||
|
||||
// TopicErrorCodes holds per topic error codes
|
||||
TopicErrorCodes []deleteTopicsResponseV1TopicErrorCode
|
||||
}
|
||||
|
||||
func (t deleteTopicsResponseV1) size() int32 {
|
||||
return sizeofInt32(t.ThrottleTimeMS) +
|
||||
sizeofArray(len(t.TopicErrorCodes), func(i int) int32 { return t.TopicErrorCodes[i].size() })
|
||||
}
|
||||
|
||||
func (t *deleteTopicsResponseV1) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readInt32(r, size, &t.ThrottleTimeMS); err != nil {
|
||||
return
|
||||
}
|
||||
fn := func(withReader *bufio.Reader, withSize int) (fnRemain int, fnErr error) {
|
||||
var item deleteTopicsResponseV1TopicErrorCode
|
||||
if fnRemain, fnErr = (&item).readFrom(withReader, withSize); err != nil {
|
||||
return
|
||||
}
|
||||
t.TopicErrorCodes = append(t.TopicErrorCodes, item)
|
||||
return
|
||||
}
|
||||
if remain, err = readArrayWith(r, remain, fn); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (t deleteTopicsResponseV1) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, t.ThrottleTimeMS)
|
||||
writeArray(w, len(t.TopicErrorCodes), func(i int) { t.TopicErrorCodes[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type deleteTopicsResponseV1TopicErrorCode struct {
|
||||
// Topic holds the topic name
|
||||
Topic string
|
||||
|
||||
// ErrorCode holds the error code
|
||||
ErrorCode int16
|
||||
}
|
||||
|
||||
func (t deleteTopicsResponseV1TopicErrorCode) size() int32 {
|
||||
return sizeofString(t.Topic) +
|
||||
sizeofInt16(t.ErrorCode)
|
||||
}
|
||||
|
||||
func (t *deleteTopicsResponseV1TopicErrorCode) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readString(r, size, &t.Topic); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readInt16(r, remain, &t.ErrorCode); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (t deleteTopicsResponseV1TopicErrorCode) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.Topic)
|
||||
writeInt16(w, t.ErrorCode)
|
||||
}
|
||||
|
||||
// deleteTopics deletes the specified topics.
|
||||
//
|
||||
// See http://kafka.apache.org/protocol.html#The_Messages_DeleteTopics
|
||||
func (c *Conn) deleteTopics(request deleteTopicsRequestV1) (deleteTopicsResponseV1, error) {
|
||||
var response deleteTopicsResponseV1
|
||||
err := c.writeOperation(
|
||||
func(deadline time.Time, id int32) error {
|
||||
if request.Timeout == 0 {
|
||||
now := time.Now()
|
||||
deadline = adjustDeadlineForRTT(deadline, now, defaultRTT)
|
||||
request.Timeout = milliseconds(deadlineToTimeout(deadline, now))
|
||||
}
|
||||
return c.writeRequest(deleteTopicsRequest, v1, id, request)
|
||||
},
|
||||
func(deadline time.Time, size int) error {
|
||||
return expectZeroSize(func() (remain int, err error) {
|
||||
return (&response).readFrom(&c.rbuf, size)
|
||||
}())
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return deleteTopicsResponseV1{}, err
|
||||
}
|
||||
for _, c := range response.TopicErrorCodes {
|
||||
if c.ErrorCode != 0 {
|
||||
return response, Error(c.ErrorCode)
|
||||
}
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
-186
@@ -1,186 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import "bufio"
|
||||
|
||||
// See http://kafka.apache.org/protocol.html#The_Messages_DescribeGroups
|
||||
type describeGroupsRequestV1 struct {
|
||||
// List of groupIds to request metadata for (an empty groupId array
|
||||
// will return empty group metadata).
|
||||
GroupIDs []string
|
||||
}
|
||||
|
||||
func (t describeGroupsRequestV1) size() int32 {
|
||||
return sizeofStringArray(t.GroupIDs)
|
||||
}
|
||||
|
||||
func (t describeGroupsRequestV1) writeTo(w *bufio.Writer) {
|
||||
writeStringArray(w, t.GroupIDs)
|
||||
}
|
||||
|
||||
type describeGroupsResponseMemberV1 struct {
|
||||
// MemberID assigned by the group coordinator
|
||||
MemberID string
|
||||
|
||||
// ClientID used in the member's latest join group request
|
||||
ClientID string
|
||||
|
||||
// ClientHost used in the request session corresponding to the member's
|
||||
// join group.
|
||||
ClientHost string
|
||||
|
||||
// MemberMetadata the metadata corresponding to the current group protocol
|
||||
// in use (will only be present if the group is stable).
|
||||
MemberMetadata []byte
|
||||
|
||||
// MemberAssignments provided by the group leader (will only be present if
|
||||
// the group is stable).
|
||||
//
|
||||
// See consumer groups section of https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol
|
||||
MemberAssignments []byte
|
||||
}
|
||||
|
||||
func (t describeGroupsResponseMemberV1) size() int32 {
|
||||
return sizeofString(t.MemberID) +
|
||||
sizeofString(t.ClientID) +
|
||||
sizeofString(t.ClientHost) +
|
||||
sizeofBytes(t.MemberMetadata) +
|
||||
sizeofBytes(t.MemberAssignments)
|
||||
}
|
||||
|
||||
func (t describeGroupsResponseMemberV1) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.MemberID)
|
||||
writeString(w, t.ClientID)
|
||||
writeString(w, t.ClientHost)
|
||||
writeBytes(w, t.MemberMetadata)
|
||||
writeBytes(w, t.MemberAssignments)
|
||||
}
|
||||
|
||||
func (t *describeGroupsResponseMemberV1) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readString(r, size, &t.MemberID); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readString(r, remain, &t.ClientID); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readString(r, remain, &t.ClientHost); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readBytes(r, remain, &t.MemberMetadata); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readBytes(r, remain, &t.MemberAssignments); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type describeGroupsResponseGroupV1 struct {
|
||||
// ErrorCode holds response error code
|
||||
ErrorCode int16
|
||||
|
||||
// GroupID holds the unique group identifier
|
||||
GroupID string
|
||||
|
||||
// State holds current state of the group (one of: Dead, Stable, AwaitingSync,
|
||||
// PreparingRebalance, or empty if there is no active group)
|
||||
State string
|
||||
|
||||
// ProtocolType holds the current group protocol type (will be empty if there is
|
||||
// no active group)
|
||||
ProtocolType string
|
||||
|
||||
// Protocol holds the current group protocol (only provided if the group is Stable)
|
||||
Protocol string
|
||||
|
||||
// Members contains the current group members (only provided if the group is not Dead)
|
||||
Members []describeGroupsResponseMemberV1
|
||||
}
|
||||
|
||||
func (t describeGroupsResponseGroupV1) size() int32 {
|
||||
return sizeofInt16(t.ErrorCode) +
|
||||
sizeofString(t.GroupID) +
|
||||
sizeofString(t.State) +
|
||||
sizeofString(t.ProtocolType) +
|
||||
sizeofString(t.Protocol) +
|
||||
sizeofArray(len(t.Members), func(i int) int32 { return t.Members[i].size() })
|
||||
}
|
||||
|
||||
func (t describeGroupsResponseGroupV1) writeTo(w *bufio.Writer) {
|
||||
writeInt16(w, t.ErrorCode)
|
||||
writeString(w, t.GroupID)
|
||||
writeString(w, t.State)
|
||||
writeString(w, t.ProtocolType)
|
||||
writeString(w, t.Protocol)
|
||||
writeArray(w, len(t.Members), func(i int) { t.Members[i].writeTo(w) })
|
||||
}
|
||||
|
||||
func (t *describeGroupsResponseGroupV1) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readInt16(r, size, &t.ErrorCode); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readString(r, remain, &t.GroupID); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readString(r, remain, &t.State); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readString(r, remain, &t.ProtocolType); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readString(r, remain, &t.Protocol); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
fn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {
|
||||
item := describeGroupsResponseMemberV1{}
|
||||
if fnRemain, fnErr = (&item).readFrom(r, size); err != nil {
|
||||
return
|
||||
}
|
||||
t.Members = append(t.Members, item)
|
||||
return
|
||||
}
|
||||
if remain, err = readArrayWith(r, remain, fn); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
type describeGroupsResponseV1 struct {
|
||||
// Duration in milliseconds for which the request was throttled due
|
||||
// to quota violation (Zero if the request did not violate any quota)
|
||||
ThrottleTimeMS int32
|
||||
|
||||
// Groups holds selected group information
|
||||
Groups []describeGroupsResponseGroupV1
|
||||
}
|
||||
|
||||
func (t describeGroupsResponseV1) size() int32 {
|
||||
return sizeofInt32(t.ThrottleTimeMS) +
|
||||
sizeofArray(len(t.Groups), func(i int) int32 { return t.Groups[i].size() })
|
||||
}
|
||||
|
||||
func (t describeGroupsResponseV1) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, t.ThrottleTimeMS)
|
||||
writeArray(w, len(t.Groups), func(i int) { t.Groups[i].writeTo(w) })
|
||||
}
|
||||
|
||||
func (t *describeGroupsResponseV1) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readInt32(r, size, &t.ThrottleTimeMS); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
fn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {
|
||||
item := describeGroupsResponseGroupV1{}
|
||||
if fnRemain, fnErr = (&item).readFrom(r, size); fnErr != nil {
|
||||
return
|
||||
}
|
||||
t.Groups = append(t.Groups, item)
|
||||
return
|
||||
}
|
||||
if remain, err = readArrayWith(r, remain, fn); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
-365
@@ -1,365 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The Dialer type mirrors the net.Dialer API but is designed to open kafka
|
||||
// connections instead of raw network connections.
|
||||
type Dialer struct {
|
||||
// Unique identifier for client connections established by this Dialer.
|
||||
ClientID string
|
||||
|
||||
// Timeout is the maximum amount of time a dial will wait for a connect to
|
||||
// complete. If Deadline is also set, it may fail earlier.
|
||||
//
|
||||
// The default is no timeout.
|
||||
//
|
||||
// When dialing a name with multiple IP addresses, the timeout may be
|
||||
// divided between them.
|
||||
//
|
||||
// With or without a timeout, the operating system may impose its own
|
||||
// earlier timeout. For instance, TCP timeouts are often around 3 minutes.
|
||||
Timeout time.Duration
|
||||
|
||||
// Deadline is the absolute point in time after which dials will fail.
|
||||
// If Timeout is set, it may fail earlier.
|
||||
// Zero means no deadline, or dependent on the operating system as with the
|
||||
// Timeout option.
|
||||
Deadline time.Time
|
||||
|
||||
// LocalAddr is the local address to use when dialing an address.
|
||||
// The address must be of a compatible type for the network being dialed.
|
||||
// If nil, a local address is automatically chosen.
|
||||
LocalAddr net.Addr
|
||||
|
||||
// DualStack enables RFC 6555-compliant "Happy Eyeballs" dialing when the
|
||||
// network is "tcp" and the destination is a host name with both IPv4 and
|
||||
// IPv6 addresses. This allows a client to tolerate networks where one
|
||||
// address family is silently broken.
|
||||
DualStack bool
|
||||
|
||||
// FallbackDelay specifies the length of time to wait before spawning a
|
||||
// fallback connection, when DualStack is enabled.
|
||||
// If zero, a default delay of 300ms is used.
|
||||
FallbackDelay time.Duration
|
||||
|
||||
// KeepAlive specifies the keep-alive period for an active network
|
||||
// connection.
|
||||
// If zero, keep-alives are not enabled. Network protocols that do not
|
||||
// support keep-alives ignore this field.
|
||||
KeepAlive time.Duration
|
||||
|
||||
// Resolver optionally specifies an alternate resolver to use.
|
||||
Resolver Resolver
|
||||
|
||||
// TLS enables Dialer to open secure connections. If nil, standard net.Conn
|
||||
// will be used.
|
||||
TLS *tls.Config
|
||||
}
|
||||
|
||||
// Dial connects to the address on the named network.
|
||||
func (d *Dialer) Dial(network string, address string) (*Conn, error) {
|
||||
return d.DialContext(context.Background(), network, address)
|
||||
}
|
||||
|
||||
// DialContext connects to the address on the named network using the provided
|
||||
// context.
|
||||
//
|
||||
// The provided Context must be non-nil. If the context expires before the
|
||||
// connection is complete, an error is returned. Once successfully connected,
|
||||
// any expiration of the context will not affect the connection.
|
||||
//
|
||||
// When using TCP, and the host in the address parameter resolves to multiple
|
||||
// network addresses, any dial timeout (from d.Timeout or ctx) is spread over
|
||||
// each consecutive dial, such that each is given an appropriate fraction of the
|
||||
// time to connect. For example, if a host has 4 IP addresses and the timeout is
|
||||
// 1 minute, the connect to each single address will be given 15 seconds to
|
||||
// complete before trying the next one.
|
||||
func (d *Dialer) DialContext(ctx context.Context, network string, address string) (*Conn, error) {
|
||||
if d.Timeout != 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, d.Timeout)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
if !d.Deadline.IsZero() {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithDeadline(ctx, d.Deadline)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
c, err := d.dialContext(ctx, network, address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewConnWith(c, ConnConfig{ClientID: d.ClientID}), nil
|
||||
}
|
||||
|
||||
// DialLeader opens a connection to the leader of the partition for a given
|
||||
// topic.
|
||||
//
|
||||
// The address given to the DialContext method may not be the one that the
|
||||
// connection will end up being established to, because the dialer will lookup
|
||||
// the partition leader for the topic and return a connection to that server.
|
||||
// The original address is only used as a mechanism to discover the
|
||||
// configuration of the kafka cluster that we're connecting to.
|
||||
func (d *Dialer) DialLeader(ctx context.Context, network string, address string, topic string, partition int) (*Conn, error) {
|
||||
p, err := d.LookupPartition(ctx, network, address, topic, partition)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return d.DialPartition(ctx, network, address, p)
|
||||
}
|
||||
|
||||
// DialPartition opens a connection to the leader of the partition specified by partition
|
||||
// descriptor. It's strongly advised to use descriptor of the partition that comes out of
|
||||
// functions LookupPartition or LookupPartitions.
|
||||
func (d *Dialer) DialPartition(ctx context.Context, network string, address string, partition Partition) (*Conn, error) {
|
||||
c, err := d.dialContext(ctx, network, net.JoinHostPort(partition.Leader.Host, strconv.Itoa(partition.Leader.Port)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return NewConnWith(c, ConnConfig{
|
||||
ClientID: d.ClientID,
|
||||
Topic: partition.Topic,
|
||||
Partition: partition.ID,
|
||||
}), nil
|
||||
}
|
||||
|
||||
// LookupLeader searches for the kafka broker that is the leader of the
|
||||
// partition for a given topic, returning a Broker value representing it.
|
||||
func (d *Dialer) LookupLeader(ctx context.Context, network string, address string, topic string, partition int) (Broker, error) {
|
||||
p, err := d.LookupPartition(ctx, network, address, topic, partition)
|
||||
return p.Leader, err
|
||||
}
|
||||
|
||||
// LookupPartition searches for the description of specified partition id.
|
||||
func (d *Dialer) LookupPartition(ctx context.Context, network string, address string, topic string, partition int) (Partition, error) {
|
||||
c, err := d.DialContext(ctx, network, address)
|
||||
if err != nil {
|
||||
return Partition{}, err
|
||||
}
|
||||
defer c.Close()
|
||||
|
||||
brkch := make(chan Partition, 1)
|
||||
errch := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
for attempt := 0; true; attempt++ {
|
||||
if attempt != 0 {
|
||||
sleep(ctx, backoff(attempt, 100*time.Millisecond, 10*time.Second))
|
||||
}
|
||||
|
||||
partitions, err := c.ReadPartitions(topic)
|
||||
if err != nil {
|
||||
if isTemporary(err) {
|
||||
continue
|
||||
}
|
||||
errch <- err
|
||||
return
|
||||
}
|
||||
|
||||
for _, p := range partitions {
|
||||
if p.ID == partition {
|
||||
brkch <- p
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
errch <- UnknownTopicOrPartition
|
||||
}()
|
||||
|
||||
var prt Partition
|
||||
select {
|
||||
case prt = <-brkch:
|
||||
case err = <-errch:
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
}
|
||||
return prt, err
|
||||
}
|
||||
|
||||
// LookupPartitions returns the list of partitions that exist for the given topic.
|
||||
func (d *Dialer) LookupPartitions(ctx context.Context, network string, address string, topic string) ([]Partition, error) {
|
||||
conn, err := d.DialContext(ctx, network, address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
prtch := make(chan []Partition, 1)
|
||||
errch := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
if prt, err := conn.ReadPartitions(topic); err != nil {
|
||||
errch <- err
|
||||
} else {
|
||||
prtch <- prt
|
||||
}
|
||||
}()
|
||||
|
||||
var prt []Partition
|
||||
select {
|
||||
case prt = <-prtch:
|
||||
case err = <-errch:
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
}
|
||||
return prt, err
|
||||
}
|
||||
|
||||
// connectTLS returns a tls.Conn that has already completed the Handshake
|
||||
func (d *Dialer) connectTLS(ctx context.Context, conn net.Conn, config *tls.Config) (tlsConn *tls.Conn, err error) {
|
||||
tlsConn = tls.Client(conn, config)
|
||||
errch := make(chan error)
|
||||
|
||||
go func() {
|
||||
defer close(errch)
|
||||
errch <- tlsConn.Handshake()
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
conn.Close()
|
||||
tlsConn.Close()
|
||||
<-errch // ignore possible error from Handshake
|
||||
err = ctx.Err()
|
||||
|
||||
case err = <-errch:
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (d *Dialer) dialContext(ctx context.Context, network string, address string) (net.Conn, error) {
|
||||
if r := d.Resolver; r != nil {
|
||||
host, port := splitHostPort(address)
|
||||
addrs, err := r.LookupHost(ctx, host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(addrs) != 0 {
|
||||
address = addrs[0]
|
||||
}
|
||||
if len(port) != 0 {
|
||||
address, _ = splitHostPort(address)
|
||||
address = net.JoinHostPort(address, port)
|
||||
}
|
||||
}
|
||||
|
||||
conn, err := (&net.Dialer{
|
||||
LocalAddr: d.LocalAddr,
|
||||
DualStack: d.DualStack,
|
||||
FallbackDelay: d.FallbackDelay,
|
||||
KeepAlive: d.KeepAlive,
|
||||
}).DialContext(ctx, network, address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if d.TLS != nil {
|
||||
c := d.TLS
|
||||
// If no ServerName is set, infer the ServerName
|
||||
// from the hostname we're connecting to.
|
||||
if c.ServerName == "" {
|
||||
c = d.TLS.Clone()
|
||||
// Copied from tls.go in the standard library.
|
||||
colonPos := strings.LastIndex(address, ":")
|
||||
if colonPos == -1 {
|
||||
colonPos = len(address)
|
||||
}
|
||||
hostname := address[:colonPos]
|
||||
c.ServerName = hostname
|
||||
}
|
||||
return d.connectTLS(ctx, conn, c)
|
||||
}
|
||||
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
// DefaultDialer is the default dialer used when none is specified.
|
||||
var DefaultDialer = &Dialer{
|
||||
Timeout: 10 * time.Second,
|
||||
DualStack: true,
|
||||
}
|
||||
|
||||
// Dial is a convenience wrapper for DefaultDialer.Dial.
|
||||
func Dial(network string, address string) (*Conn, error) {
|
||||
return DefaultDialer.Dial(network, address)
|
||||
}
|
||||
|
||||
// DialContext is a convenience wrapper for DefaultDialer.DialContext.
|
||||
func DialContext(ctx context.Context, network string, address string) (*Conn, error) {
|
||||
return DefaultDialer.DialContext(ctx, network, address)
|
||||
}
|
||||
|
||||
// DialLeader is a convenience wrapper for DefaultDialer.DialLeader.
|
||||
func DialLeader(ctx context.Context, network string, address string, topic string, partition int) (*Conn, error) {
|
||||
return DefaultDialer.DialLeader(ctx, network, address, topic, partition)
|
||||
}
|
||||
|
||||
// DialPartition is a convenience wrapper for DefaultDialer.DialPartition.
|
||||
func DialPartition(ctx context.Context, network string, address string, partition Partition) (*Conn, error) {
|
||||
return DefaultDialer.DialPartition(ctx, network, address, partition)
|
||||
}
|
||||
|
||||
// LookupPartition is a convenience wrapper for DefaultDialer.LookupPartition.
|
||||
func LookupPartition(ctx context.Context, network string, address string, topic string, partition int) (Partition, error) {
|
||||
return DefaultDialer.LookupPartition(ctx, network, address, topic, partition)
|
||||
}
|
||||
|
||||
// LookupPartitions is a convenience wrapper for DefaultDialer.LookupPartitions.
|
||||
func LookupPartitions(ctx context.Context, network string, address string, topic string) ([]Partition, error) {
|
||||
return DefaultDialer.LookupPartitions(ctx, network, address, topic)
|
||||
}
|
||||
|
||||
// The Resolver interface is used as an abstraction to provide service discovery
|
||||
// of the hosts of a kafka cluster.
|
||||
type Resolver interface {
|
||||
// LookupHost looks up the given host using the local resolver.
|
||||
// It returns a slice of that host's addresses.
|
||||
LookupHost(ctx context.Context, host string) (addrs []string, err error)
|
||||
}
|
||||
|
||||
func sleep(ctx context.Context, duration time.Duration) bool {
|
||||
if duration == 0 {
|
||||
select {
|
||||
default:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
timer := time.NewTimer(duration)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func backoff(attempt int, min time.Duration, max time.Duration) time.Duration {
|
||||
d := time.Duration(attempt*attempt) * min
|
||||
if d > max {
|
||||
d = max
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func splitHostPort(s string) (host string, port string) {
|
||||
host, port, _ = net.SplitHostPort(s)
|
||||
if len(host) == 0 && len(port) == 0 {
|
||||
host = s
|
||||
}
|
||||
return
|
||||
}
|
||||
-50
@@ -1,50 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import "bufio"
|
||||
|
||||
func discardN(r *bufio.Reader, sz int, n int) (int, error) {
|
||||
var err error
|
||||
if n <= sz {
|
||||
n, err = r.Discard(n)
|
||||
} else {
|
||||
n, err = r.Discard(sz)
|
||||
if err == nil {
|
||||
err = errShortRead
|
||||
}
|
||||
}
|
||||
return sz - n, err
|
||||
}
|
||||
|
||||
func discardInt8(r *bufio.Reader, sz int) (int, error) {
|
||||
return discardN(r, sz, 1)
|
||||
}
|
||||
|
||||
func discardInt16(r *bufio.Reader, sz int) (int, error) {
|
||||
return discardN(r, sz, 2)
|
||||
}
|
||||
|
||||
func discardInt32(r *bufio.Reader, sz int) (int, error) {
|
||||
return discardN(r, sz, 4)
|
||||
}
|
||||
|
||||
func discardInt64(r *bufio.Reader, sz int) (int, error) {
|
||||
return discardN(r, sz, 8)
|
||||
}
|
||||
|
||||
func discardString(r *bufio.Reader, sz int) (int, error) {
|
||||
return readStringWith(r, sz, func(r *bufio.Reader, sz int, n int) (int, error) {
|
||||
if n < 0 {
|
||||
return sz, nil
|
||||
}
|
||||
return discardN(r, sz, n)
|
||||
})
|
||||
}
|
||||
|
||||
func discardBytes(r *bufio.Reader, sz int) (int, error) {
|
||||
return readBytesWith(r, sz, func(r *bufio.Reader, sz int, n int) (int, error) {
|
||||
if n < 0 {
|
||||
return sz, nil
|
||||
}
|
||||
return discardN(r, sz, n)
|
||||
})
|
||||
}
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
version: "3"
|
||||
services:
|
||||
kafka:
|
||||
image: wurstmeister/kafka:0.11.0.1
|
||||
restart: on-failure:3
|
||||
links:
|
||||
- zookeeper
|
||||
ports:
|
||||
- "9092:9092"
|
||||
environment:
|
||||
KAFKA_VERSION: '0.11.0.1'
|
||||
KAFKA_BROKER_ID: 1
|
||||
KAFKA_CREATE_TOPICS: 'test-writer-0:3:1,test-writer-1:3:1'
|
||||
KAFKA_DELETE_TOPIC_ENABLE: 'true'
|
||||
KAFKA_ADVERTISED_HOST_NAME: 'localhost'
|
||||
KAFKA_ADVERTISED_PORT: '9092'
|
||||
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
|
||||
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
|
||||
KAFKA_MESSAGE_MAX_BYTES: 200000000
|
||||
|
||||
zookeeper:
|
||||
image: wurstmeister/zookeeper
|
||||
ports:
|
||||
- "2181:2181"
|
||||
-360
@@ -1,360 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// Error represents the different error codes that may be returned by kafka.
|
||||
type Error int
|
||||
|
||||
const (
|
||||
Unknown Error = -1
|
||||
OffsetOutOfRange Error = 1
|
||||
InvalidMessage Error = 2
|
||||
UnknownTopicOrPartition Error = 3
|
||||
InvalidMessageSize Error = 4
|
||||
LeaderNotAvailable Error = 5
|
||||
NotLeaderForPartition Error = 6
|
||||
RequestTimedOut Error = 7
|
||||
BrokerNotAvailable Error = 8
|
||||
ReplicaNotAvailable Error = 9
|
||||
MessageSizeTooLarge Error = 10
|
||||
StaleControllerEpoch Error = 11
|
||||
OffsetMetadataTooLarge Error = 12
|
||||
GroupLoadInProgress Error = 14
|
||||
GroupCoordinatorNotAvailable Error = 15
|
||||
NotCoordinatorForGroup Error = 16
|
||||
InvalidTopic Error = 17
|
||||
RecordListTooLarge Error = 18
|
||||
NotEnoughReplicas Error = 19
|
||||
NotEnoughReplicasAfterAppend Error = 20
|
||||
InvalidRequiredAcks Error = 21
|
||||
IllegalGeneration Error = 22
|
||||
InconsistentGroupProtocol Error = 23
|
||||
InvalidGroupId Error = 24
|
||||
UnknownMemberId Error = 25
|
||||
InvalidSessionTimeout Error = 26
|
||||
RebalanceInProgress Error = 27
|
||||
InvalidCommitOffsetSize Error = 28
|
||||
TopicAuthorizationFailed Error = 29
|
||||
GroupAuthorizationFailed Error = 30
|
||||
ClusterAuthorizationFailed Error = 31
|
||||
InvalidTimestamp Error = 32
|
||||
UnsupportedSASLMechanism Error = 33
|
||||
IllegalSASLState Error = 34
|
||||
UnsupportedVersion Error = 35
|
||||
TopicAlreadyExists Error = 36
|
||||
InvalidPartitionNumber Error = 37
|
||||
InvalidReplicationFactor Error = 38
|
||||
InvalidReplicaAssignment Error = 39
|
||||
InvalidConfiguration Error = 40
|
||||
NotController Error = 41
|
||||
InvalidRequest Error = 42
|
||||
UnsupportedForMessageFormat Error = 43
|
||||
PolicyViolation Error = 44
|
||||
OutOfOrderSequenceNumber Error = 45
|
||||
DuplicateSequenceNumber Error = 46
|
||||
InvalidProducerEpoch Error = 47
|
||||
InvalidTransactionState Error = 48
|
||||
InvalidProducerIDMapping Error = 49
|
||||
InvalidTransactionTimeout Error = 50
|
||||
ConcurrentTransactions Error = 51
|
||||
TransactionCoordinatorFenced Error = 52
|
||||
TransactionalIDAuthorizationFailed Error = 53
|
||||
SecurityDisabled Error = 54
|
||||
BrokerAuthorizationFailed Error = 55
|
||||
)
|
||||
|
||||
// Error satisfies the error interface.
|
||||
func (e Error) Error() string {
|
||||
return fmt.Sprintf("[%d] %s: %s", e, e.Title(), e.Description())
|
||||
}
|
||||
|
||||
// Timeout returns true if the error was due to a timeout.
|
||||
func (e Error) Timeout() bool {
|
||||
return e == RequestTimedOut
|
||||
}
|
||||
|
||||
// Temporary returns true if the operation that generated the error may succeed
|
||||
// if retried at a later time.
|
||||
func (e Error) Temporary() bool {
|
||||
return e == LeaderNotAvailable ||
|
||||
e == BrokerNotAvailable ||
|
||||
e == ReplicaNotAvailable ||
|
||||
e == GroupLoadInProgress ||
|
||||
e == GroupCoordinatorNotAvailable ||
|
||||
e == RebalanceInProgress ||
|
||||
e.Timeout()
|
||||
}
|
||||
|
||||
// Title returns a human readable title for the error.
|
||||
func (e Error) Title() string {
|
||||
switch e {
|
||||
case Unknown:
|
||||
return "Unknown"
|
||||
case OffsetOutOfRange:
|
||||
return "Offset Out Of Range"
|
||||
case InvalidMessage:
|
||||
return "Invalid Message"
|
||||
case UnknownTopicOrPartition:
|
||||
return "Unknown Topic Or Partition"
|
||||
case InvalidMessageSize:
|
||||
return "Invalid Message Size"
|
||||
case LeaderNotAvailable:
|
||||
return "Leader Not Available"
|
||||
case NotLeaderForPartition:
|
||||
return "Not Leader For Partition"
|
||||
case RequestTimedOut:
|
||||
return "Request Timed Out"
|
||||
case BrokerNotAvailable:
|
||||
return "Broker Not Available"
|
||||
case ReplicaNotAvailable:
|
||||
return "Replica Not Available"
|
||||
case MessageSizeTooLarge:
|
||||
return "Message Size Too Large"
|
||||
case StaleControllerEpoch:
|
||||
return "Stale Controller Epoch"
|
||||
case OffsetMetadataTooLarge:
|
||||
return "Offset Metadata Too Large"
|
||||
case GroupLoadInProgress:
|
||||
return "Group Load In Progress"
|
||||
case GroupCoordinatorNotAvailable:
|
||||
return "Group Coordinator Not Available"
|
||||
case NotCoordinatorForGroup:
|
||||
return "Not Coordinator For Group"
|
||||
case InvalidTopic:
|
||||
return "Invalid Topic"
|
||||
case RecordListTooLarge:
|
||||
return "Record List Too Large"
|
||||
case NotEnoughReplicas:
|
||||
return "Not Enough Replicas"
|
||||
case NotEnoughReplicasAfterAppend:
|
||||
return "Not Enough Replicas After Append"
|
||||
case InvalidRequiredAcks:
|
||||
return "Invalid Required Acks"
|
||||
case IllegalGeneration:
|
||||
return "Illegal Generation"
|
||||
case InconsistentGroupProtocol:
|
||||
return "Inconsistent Group Protocol"
|
||||
case InvalidGroupId:
|
||||
return "Invalid Group ID"
|
||||
case UnknownMemberId:
|
||||
return "Unknown Member ID"
|
||||
case InvalidSessionTimeout:
|
||||
return "Invalid Session Timeout"
|
||||
case RebalanceInProgress:
|
||||
return "Rebalance In Progress"
|
||||
case InvalidCommitOffsetSize:
|
||||
return "Invalid Commit Offset Size"
|
||||
case TopicAuthorizationFailed:
|
||||
return "Topic Authorization Failed"
|
||||
case GroupAuthorizationFailed:
|
||||
return "Group Authorization Failed"
|
||||
case ClusterAuthorizationFailed:
|
||||
return "Cluster Authorization Failed"
|
||||
case InvalidTimestamp:
|
||||
return "Invalid Timestamp"
|
||||
case UnsupportedSASLMechanism:
|
||||
return "Unsupported SASL Mechanism"
|
||||
case IllegalSASLState:
|
||||
return "Illegal SASL State"
|
||||
case UnsupportedVersion:
|
||||
return "Unsupported Version"
|
||||
case TopicAlreadyExists:
|
||||
return "Topic Already Exists"
|
||||
case InvalidPartitionNumber:
|
||||
return "Invalid Partition Number"
|
||||
case InvalidReplicationFactor:
|
||||
return "Invalid Replication Factor"
|
||||
case InvalidReplicaAssignment:
|
||||
return "Invalid Replica Assignment"
|
||||
case InvalidConfiguration:
|
||||
return "Invalid Configuration"
|
||||
case NotController:
|
||||
return "Not Controller"
|
||||
case InvalidRequest:
|
||||
return "Invalid Request"
|
||||
case UnsupportedForMessageFormat:
|
||||
return "Unsupported For Message Format"
|
||||
case PolicyViolation:
|
||||
return "Policy Violation"
|
||||
case OutOfOrderSequenceNumber:
|
||||
return "Out Of Order Sequence Number"
|
||||
case DuplicateSequenceNumber:
|
||||
return "Duplicate Sequence Number"
|
||||
case InvalidProducerEpoch:
|
||||
return "Invalid Producer Epoch"
|
||||
case InvalidTransactionState:
|
||||
return "Invalid Transaction State"
|
||||
case InvalidProducerIDMapping:
|
||||
return "Invalid Producer ID Mapping"
|
||||
case InvalidTransactionTimeout:
|
||||
return "Invalid Transaction Timeout"
|
||||
case ConcurrentTransactions:
|
||||
return "Concurrent Transactions"
|
||||
case TransactionCoordinatorFenced:
|
||||
return "Transaction Coordinator Fenced"
|
||||
case TransactionalIDAuthorizationFailed:
|
||||
return "Transactional ID Authorization Failed"
|
||||
case SecurityDisabled:
|
||||
return "Security Disabled"
|
||||
case BrokerAuthorizationFailed:
|
||||
return "Broker Authorization Failed"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Description returns a human readable description of cause of the error.
|
||||
func (e Error) Description() string {
|
||||
switch e {
|
||||
case Unknown:
|
||||
return "an unexpected server error occurred"
|
||||
case OffsetOutOfRange:
|
||||
return "the requested offset is outside the range of offsets maintained by the server for the given topic/partition"
|
||||
case InvalidMessage:
|
||||
return "the message contents does not match its CRC"
|
||||
case UnknownTopicOrPartition:
|
||||
return "the request is for a topic or partition that does not exist on this broker"
|
||||
case InvalidMessageSize:
|
||||
return "the message has a negative size"
|
||||
case LeaderNotAvailable:
|
||||
return "the cluster is in the middle of a leadership election and there is currently no leader for this partition and hence it is unavailable for writes"
|
||||
case NotLeaderForPartition:
|
||||
return "the client attempted to send messages to a replica that is not the leader for some partition, the client's metadata are likely out of date"
|
||||
case RequestTimedOut:
|
||||
return "the request exceeded the user-specified time limit in the request"
|
||||
case BrokerNotAvailable:
|
||||
return "not a client facing error and is used mostly by tools when a broker is not alive"
|
||||
case ReplicaNotAvailable:
|
||||
return "a replica is expected on a broker, but is not (this can be safely ignored)"
|
||||
case MessageSizeTooLarge:
|
||||
return "the server has a configurable maximum message size to avoid unbounded memory allocation and the client attempted to produce a message larger than this maximum"
|
||||
case StaleControllerEpoch:
|
||||
return "internal error code for broker-to-broker communication"
|
||||
case OffsetMetadataTooLarge:
|
||||
return "the client specified a string larger than configured maximum for offset metadata"
|
||||
case GroupLoadInProgress:
|
||||
return "the broker returns this error code for an offset fetch request if it is still loading offsets (after a leader change for that offsets topic partition), or in response to group membership requests (such as heartbeats) when group metadata is being loaded by the coordinator"
|
||||
case GroupCoordinatorNotAvailable:
|
||||
return "the broker returns this error code for group coordinator requests, offset commits, and most group management requests if the offsets topic has not yet been created, or if the group coordinator is not active"
|
||||
case NotCoordinatorForGroup:
|
||||
return "the broker returns this error code if it receives an offset fetch or commit request for a group that it is not a coordinator for"
|
||||
case InvalidTopic:
|
||||
return "a request which attempted to access an invalid topic (e.g. one which has an illegal name), or if an attempt was made to write to an internal topic (such as the consumer offsets topic)"
|
||||
case RecordListTooLarge:
|
||||
return "a message batch in a produce request exceeds the maximum configured segment size"
|
||||
case NotEnoughReplicas:
|
||||
return "the number of in-sync replicas is lower than the configured minimum and requiredAcks is -1"
|
||||
case NotEnoughReplicasAfterAppend:
|
||||
return "the message was written to the log, but with fewer in-sync replicas than required."
|
||||
case InvalidRequiredAcks:
|
||||
return "the requested requiredAcks is invalid (anything other than -1, 1, or 0)"
|
||||
case IllegalGeneration:
|
||||
return "the generation id provided in the request is not the current generation"
|
||||
case InconsistentGroupProtocol:
|
||||
return "the member provided a protocol type or set of protocols which is not compatible with the current group"
|
||||
case InvalidGroupId:
|
||||
return "the group id is empty or null"
|
||||
case UnknownMemberId:
|
||||
return "the member id is not in the current generation"
|
||||
case InvalidSessionTimeout:
|
||||
return "the requested session timeout is outside of the allowed range on the broker"
|
||||
case RebalanceInProgress:
|
||||
return "the coordinator has begun rebalancing the group, the client should rejoin the group"
|
||||
case InvalidCommitOffsetSize:
|
||||
return "an offset commit was rejected because of oversize metadata"
|
||||
case TopicAuthorizationFailed:
|
||||
return "the client is not authorized to access the requested topic"
|
||||
case GroupAuthorizationFailed:
|
||||
return "the client is not authorized to access a particular group id"
|
||||
case ClusterAuthorizationFailed:
|
||||
return "the client is not authorized to use an inter-broker or administrative API"
|
||||
case InvalidTimestamp:
|
||||
return "the timestamp of the message is out of acceptable range"
|
||||
case UnsupportedSASLMechanism:
|
||||
return "the broker does not support the requested SASL mechanism"
|
||||
case IllegalSASLState:
|
||||
return "the request is not valid given the current SASL state"
|
||||
case UnsupportedVersion:
|
||||
return "the version of API is not supported"
|
||||
case TopicAlreadyExists:
|
||||
return "a topic with this name already exists"
|
||||
case InvalidPartitionNumber:
|
||||
return "the number of partitions is invalid"
|
||||
case InvalidReplicationFactor:
|
||||
return "the replication-factor is invalid"
|
||||
case InvalidReplicaAssignment:
|
||||
return "the replica assignment is invalid"
|
||||
case InvalidConfiguration:
|
||||
return "the configuration is invalid"
|
||||
case NotController:
|
||||
return "this is not the correct controller for this cluster"
|
||||
case InvalidRequest:
|
||||
return "this most likely occurs because of a request being malformed by the client library or the message was sent to an incompatible broker, se the broker logs for more details"
|
||||
case UnsupportedForMessageFormat:
|
||||
return "the message format version on the broker does not support the request"
|
||||
case PolicyViolation:
|
||||
return "the request parameters do not satisfy the configured policy"
|
||||
case OutOfOrderSequenceNumber:
|
||||
return "the broker received an out of order sequence number"
|
||||
case DuplicateSequenceNumber:
|
||||
return "the broker received a duplicate sequence number"
|
||||
case InvalidProducerEpoch:
|
||||
return "the producer attempted an operation with an old epoch, either there is a newer producer with the same transactional ID, or the producer's transaction has been expired by the broker"
|
||||
case InvalidTransactionState:
|
||||
return "the producer attempted a transactional operation in an invalid state"
|
||||
case InvalidProducerIDMapping:
|
||||
return "the producer attempted to use a producer id which is not currently assigned to its transactional ID"
|
||||
case InvalidTransactionTimeout:
|
||||
return "the transaction timeout is larger than the maximum value allowed by the broker (as configured by max.transaction.timeout.ms)"
|
||||
case ConcurrentTransactions:
|
||||
return "the producer attempted to update a transaction while another concurrent operation on the same transaction was ongoing"
|
||||
case TransactionCoordinatorFenced:
|
||||
return "the transaction coordinator sending a WriteTxnMarker is no longer the current coordinator for a given producer"
|
||||
case TransactionalIDAuthorizationFailed:
|
||||
return "the transactional ID authorization failed"
|
||||
case SecurityDisabled:
|
||||
return "the security features are disabled"
|
||||
case BrokerAuthorizationFailed:
|
||||
return "the broker authorization failed"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func isTimeout(err error) bool {
|
||||
e, ok := err.(interface {
|
||||
Timeout() bool
|
||||
})
|
||||
return ok && e.Timeout()
|
||||
}
|
||||
|
||||
func isTemporary(err error) bool {
|
||||
e, ok := err.(interface {
|
||||
Temporary() bool
|
||||
})
|
||||
return ok && e.Temporary()
|
||||
}
|
||||
|
||||
func silentEOF(err error) error {
|
||||
if err == io.EOF {
|
||||
err = nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func dontExpectEOF(err error) error {
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func coalesceErrors(errs ...error) error {
|
||||
for _, err := range errs {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
-101
@@ -1,101 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import "bufio"
|
||||
|
||||
type fetchRequestV2 struct {
|
||||
ReplicaID int32
|
||||
MaxWaitTime int32
|
||||
MinBytes int32
|
||||
Topics []fetchRequestTopicV2
|
||||
}
|
||||
|
||||
func (r fetchRequestV2) size() int32 {
|
||||
return 4 + 4 + 4 + sizeofArray(len(r.Topics), func(i int) int32 { return r.Topics[i].size() })
|
||||
}
|
||||
|
||||
func (r fetchRequestV2) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, r.ReplicaID)
|
||||
writeInt32(w, r.MaxWaitTime)
|
||||
writeInt32(w, r.MinBytes)
|
||||
writeArray(w, len(r.Topics), func(i int) { r.Topics[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type fetchRequestTopicV2 struct {
|
||||
TopicName string
|
||||
Partitions []fetchRequestPartitionV2
|
||||
}
|
||||
|
||||
func (t fetchRequestTopicV2) size() int32 {
|
||||
return sizeofString(t.TopicName) +
|
||||
sizeofArray(len(t.Partitions), func(i int) int32 { return t.Partitions[i].size() })
|
||||
}
|
||||
|
||||
func (t fetchRequestTopicV2) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.TopicName)
|
||||
writeArray(w, len(t.Partitions), func(i int) { t.Partitions[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type fetchRequestPartitionV2 struct {
|
||||
Partition int32
|
||||
FetchOffset int64
|
||||
MaxBytes int32
|
||||
}
|
||||
|
||||
func (p fetchRequestPartitionV2) size() int32 {
|
||||
return 4 + 8 + 4
|
||||
}
|
||||
|
||||
func (p fetchRequestPartitionV2) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, p.Partition)
|
||||
writeInt64(w, p.FetchOffset)
|
||||
writeInt32(w, p.MaxBytes)
|
||||
}
|
||||
|
||||
type fetchResponseV2 struct {
|
||||
ThrottleTime int32
|
||||
Topics []fetchResponseTopicV2
|
||||
}
|
||||
|
||||
func (r fetchResponseV2) size() int32 {
|
||||
return 4 + sizeofArray(len(r.Topics), func(i int) int32 { return r.Topics[i].size() })
|
||||
}
|
||||
|
||||
func (r fetchResponseV2) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, r.ThrottleTime)
|
||||
writeArray(w, len(r.Topics), func(i int) { r.Topics[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type fetchResponseTopicV2 struct {
|
||||
TopicName string
|
||||
Partitions []fetchResponsePartitionV2
|
||||
}
|
||||
|
||||
func (t fetchResponseTopicV2) size() int32 {
|
||||
return sizeofString(t.TopicName) +
|
||||
sizeofArray(len(t.Partitions), func(i int) int32 { return t.Partitions[i].size() })
|
||||
}
|
||||
|
||||
func (t fetchResponseTopicV2) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.TopicName)
|
||||
writeArray(w, len(t.Partitions), func(i int) { t.Partitions[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type fetchResponsePartitionV2 struct {
|
||||
Partition int32
|
||||
ErrorCode int16
|
||||
HighwaterMarkOffset int64
|
||||
MessageSetSize int32
|
||||
MessageSet messageSet
|
||||
}
|
||||
|
||||
func (p fetchResponsePartitionV2) size() int32 {
|
||||
return 4 + 2 + 8 + 4 + p.MessageSet.size()
|
||||
}
|
||||
|
||||
func (p fetchResponsePartitionV2) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, p.Partition)
|
||||
writeInt16(w, p.ErrorCode)
|
||||
writeInt64(w, p.HighwaterMarkOffset)
|
||||
writeInt32(w, p.MessageSetSize)
|
||||
p.MessageSet.writeTo(w)
|
||||
}
|
||||
-86
@@ -1,86 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
)
|
||||
|
||||
// FindCoordinatorRequestV0 requests the coordinator for the specified group or transaction
|
||||
//
|
||||
// See http://kafka.apache.org/protocol.html#The_Messages_FindCoordinator
|
||||
type findCoordinatorRequestV0 struct {
|
||||
// CoordinatorKey holds id to use for finding the coordinator (for groups, this is
|
||||
// the groupId, for transactional producers, this is the transactional id)
|
||||
CoordinatorKey string
|
||||
}
|
||||
|
||||
func (t findCoordinatorRequestV0) size() int32 {
|
||||
return sizeofString(t.CoordinatorKey)
|
||||
}
|
||||
|
||||
func (t findCoordinatorRequestV0) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.CoordinatorKey)
|
||||
}
|
||||
|
||||
type findCoordinatorResponseCoordinatorV0 struct {
|
||||
// NodeID holds the broker id.
|
||||
NodeID int32
|
||||
|
||||
// Host of the broker
|
||||
Host string
|
||||
|
||||
// Port on which broker accepts requests
|
||||
Port int32
|
||||
}
|
||||
|
||||
func (t findCoordinatorResponseCoordinatorV0) size() int32 {
|
||||
return sizeofInt32(t.NodeID) +
|
||||
sizeofString(t.Host) +
|
||||
sizeofInt32(t.Port)
|
||||
}
|
||||
|
||||
func (t findCoordinatorResponseCoordinatorV0) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, t.NodeID)
|
||||
writeString(w, t.Host)
|
||||
writeInt32(w, t.Port)
|
||||
}
|
||||
|
||||
func (t *findCoordinatorResponseCoordinatorV0) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readInt32(r, size, &t.NodeID); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readString(r, remain, &t.Host); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readInt32(r, remain, &t.Port); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type findCoordinatorResponseV0 struct {
|
||||
// ErrorCode holds response error code
|
||||
ErrorCode int16
|
||||
|
||||
// Coordinator holds host and port information for the coordinator
|
||||
Coordinator findCoordinatorResponseCoordinatorV0
|
||||
}
|
||||
|
||||
func (t findCoordinatorResponseV0) size() int32 {
|
||||
return sizeofInt16(t.ErrorCode) +
|
||||
t.Coordinator.size()
|
||||
}
|
||||
|
||||
func (t findCoordinatorResponseV0) writeTo(w *bufio.Writer) {
|
||||
writeInt16(w, t.ErrorCode)
|
||||
t.Coordinator.writeTo(w)
|
||||
}
|
||||
|
||||
func (t *findCoordinatorResponseV0) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readInt16(r, size, &t.ErrorCode); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = (&t.Coordinator).readFrom(r, remain); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
-187
@@ -1,187 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import "sort"
|
||||
|
||||
// GroupMember describes a single participant in a consumer group.
|
||||
type GroupMember struct {
|
||||
// ID is the unique ID for this member as taken from the JoinGroup response.
|
||||
ID string
|
||||
|
||||
// Topics is a list of topics that this member is consuming.
|
||||
Topics []string
|
||||
|
||||
// UserData contains any information that the GroupBalancer sent to the
|
||||
// consumer group coordinator.
|
||||
UserData []byte
|
||||
}
|
||||
|
||||
// GroupMemberAssignments holds MemberID => topic => partitions
|
||||
type GroupMemberAssignments map[string]map[string][]int
|
||||
|
||||
// GroupBalancer encapsulates the client side rebalancing logic
|
||||
type GroupBalancer interface {
|
||||
// ProtocolName of the GroupBalancer
|
||||
ProtocolName() string
|
||||
|
||||
// UserData provides the GroupBalancer an opportunity to embed custom
|
||||
// UserData into the metadata.
|
||||
//
|
||||
// Will be used by JoinGroup to begin the consumer group handshake.
|
||||
//
|
||||
// See https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-JoinGroupRequest
|
||||
UserData() ([]byte, error)
|
||||
|
||||
// DefineMemberships returns which members will be consuming
|
||||
// which topic partitions
|
||||
AssignGroups(members []GroupMember, partitions []Partition) GroupMemberAssignments
|
||||
}
|
||||
|
||||
// RangeGroupBalancer groups consumers by partition
|
||||
//
|
||||
// Example: 5 partitions, 2 consumers
|
||||
// C0: [0, 1, 2]
|
||||
// C1: [3, 4]
|
||||
//
|
||||
// Example: 6 partitions, 3 consumers
|
||||
// C0: [0, 1]
|
||||
// C1: [2, 3]
|
||||
// C2: [4, 5]
|
||||
//
|
||||
type RangeGroupBalancer struct{}
|
||||
|
||||
func (r RangeGroupBalancer) ProtocolName() string {
|
||||
return "range"
|
||||
}
|
||||
|
||||
func (r RangeGroupBalancer) UserData() ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r RangeGroupBalancer) AssignGroups(members []GroupMember, topicPartitions []Partition) GroupMemberAssignments {
|
||||
groupAssignments := GroupMemberAssignments{}
|
||||
membersByTopic := findMembersByTopic(members)
|
||||
|
||||
for topic, members := range membersByTopic {
|
||||
partitions := findPartitions(topic, topicPartitions)
|
||||
partitionCount := len(partitions)
|
||||
memberCount := len(members)
|
||||
|
||||
for memberIndex, member := range members {
|
||||
assignmentsByTopic, ok := groupAssignments[member.ID]
|
||||
if !ok {
|
||||
assignmentsByTopic = map[string][]int{}
|
||||
groupAssignments[member.ID] = assignmentsByTopic
|
||||
}
|
||||
|
||||
minIndex := memberIndex * partitionCount / memberCount
|
||||
maxIndex := (memberIndex + 1) * partitionCount / memberCount
|
||||
|
||||
for partitionIndex, partition := range partitions {
|
||||
if partitionIndex >= minIndex && partitionIndex < maxIndex {
|
||||
assignmentsByTopic[topic] = append(assignmentsByTopic[topic], partition)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return groupAssignments
|
||||
}
|
||||
|
||||
// RoundrobinGroupBalancer divides partitions evenly among consumers
|
||||
//
|
||||
// Example: 5 partitions, 2 consumers
|
||||
// C0: [0, 2, 4]
|
||||
// C1: [1, 3]
|
||||
//
|
||||
// Example: 6 partitions, 3 consumers
|
||||
// C0: [0, 3]
|
||||
// C1: [1, 4]
|
||||
// C2: [2, 5]
|
||||
//
|
||||
type RoundRobinGroupBalancer struct{}
|
||||
|
||||
func (r RoundRobinGroupBalancer) ProtocolName() string {
|
||||
return "roundrobin"
|
||||
}
|
||||
|
||||
func (r RoundRobinGroupBalancer) UserData() ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r RoundRobinGroupBalancer) AssignGroups(members []GroupMember, topicPartitions []Partition) GroupMemberAssignments {
|
||||
groupAssignments := GroupMemberAssignments{}
|
||||
membersByTopic := findMembersByTopic(members)
|
||||
for topic, members := range membersByTopic {
|
||||
partitionIDs := findPartitions(topic, topicPartitions)
|
||||
memberCount := len(members)
|
||||
|
||||
for memberIndex, member := range members {
|
||||
assignmentsByTopic, ok := groupAssignments[member.ID]
|
||||
if !ok {
|
||||
assignmentsByTopic = map[string][]int{}
|
||||
groupAssignments[member.ID] = assignmentsByTopic
|
||||
}
|
||||
|
||||
for partitionIndex, partition := range partitionIDs {
|
||||
if (partitionIndex % memberCount) == memberIndex {
|
||||
assignmentsByTopic[topic] = append(assignmentsByTopic[topic], partition)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return groupAssignments
|
||||
}
|
||||
|
||||
// findPartitions extracts the partition ids associated with the topic from the
|
||||
// list of Partitions provided
|
||||
func findPartitions(topic string, partitions []Partition) []int {
|
||||
var ids []int
|
||||
for _, partition := range partitions {
|
||||
if partition.Topic == topic {
|
||||
ids = append(ids, partition.ID)
|
||||
}
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
// findMembersByTopic groups the memberGroupMetadata by topic
|
||||
func findMembersByTopic(members []GroupMember) map[string][]GroupMember {
|
||||
membersByTopic := map[string][]GroupMember{}
|
||||
for _, member := range members {
|
||||
for _, topic := range member.Topics {
|
||||
membersByTopic[topic] = append(membersByTopic[topic], member)
|
||||
}
|
||||
}
|
||||
|
||||
// normalize ordering of members to enabling grouping across topics by partitions
|
||||
//
|
||||
// Want:
|
||||
// C0 [T0/P0, T1/P0]
|
||||
// C1 [T0/P1, T1/P1]
|
||||
//
|
||||
// Not:
|
||||
// C0 [T0/P0, T1/P1]
|
||||
// C1 [T0/P1, T1/P0]
|
||||
//
|
||||
// Even though the later is still round robin, the partitions are crossed
|
||||
//
|
||||
for _, members := range membersByTopic {
|
||||
sort.Slice(members, func(i, j int) bool {
|
||||
return members[i].ID < members[j].ID
|
||||
})
|
||||
}
|
||||
|
||||
return membersByTopic
|
||||
}
|
||||
|
||||
// findGroupBalancer returns the GroupBalancer with the specified protocolName
|
||||
// from the slice provided
|
||||
func findGroupBalancer(protocolName string, balancers []GroupBalancer) (GroupBalancer, bool) {
|
||||
for _, balancer := range balancers {
|
||||
if balancer.ProtocolName() == protocolName {
|
||||
return balancer, true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
-46
@@ -1,46 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import "bufio"
|
||||
|
||||
type heartbeatRequestV0 struct {
|
||||
// GroupID holds the unique group identifier
|
||||
GroupID string
|
||||
|
||||
// GenerationID holds the generation of the group.
|
||||
GenerationID int32
|
||||
|
||||
// MemberID assigned by the group coordinator
|
||||
MemberID string
|
||||
}
|
||||
|
||||
func (t heartbeatRequestV0) size() int32 {
|
||||
return sizeofString(t.GroupID) +
|
||||
sizeofInt32(t.GenerationID) +
|
||||
sizeofString(t.MemberID)
|
||||
}
|
||||
|
||||
func (t heartbeatRequestV0) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.GroupID)
|
||||
writeInt32(w, t.GenerationID)
|
||||
writeString(w, t.MemberID)
|
||||
}
|
||||
|
||||
type heartbeatResponseV0 struct {
|
||||
// ErrorCode holds response error code
|
||||
ErrorCode int16
|
||||
}
|
||||
|
||||
func (t heartbeatResponseV0) size() int32 {
|
||||
return sizeofInt16(t.ErrorCode)
|
||||
}
|
||||
|
||||
func (t heartbeatResponseV0) writeTo(w *bufio.Writer) {
|
||||
writeInt16(w, t.ErrorCode)
|
||||
}
|
||||
|
||||
func (t *heartbeatResponseV0) readFrom(r *bufio.Reader, sz int) (remain int, err error) {
|
||||
if remain, err = readInt16(r, sz, &t.ErrorCode); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
-202
@@ -1,202 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
)
|
||||
|
||||
type memberGroupMetadata struct {
|
||||
// MemberID assigned by the group coordinator or null if joining for the
|
||||
// first time.
|
||||
MemberID string
|
||||
Metadata groupMetadata
|
||||
}
|
||||
|
||||
type groupMetadata struct {
|
||||
Version int16
|
||||
Topics []string
|
||||
UserData []byte
|
||||
}
|
||||
|
||||
func (t groupMetadata) size() int32 {
|
||||
return sizeofInt16(t.Version) +
|
||||
sizeofStringArray(t.Topics) +
|
||||
sizeofBytes(t.UserData)
|
||||
}
|
||||
|
||||
func (t groupMetadata) writeTo(w *bufio.Writer) {
|
||||
writeInt16(w, t.Version)
|
||||
writeStringArray(w, t.Topics)
|
||||
writeBytes(w, t.UserData)
|
||||
}
|
||||
|
||||
func (t groupMetadata) bytes() []byte {
|
||||
buf := bytes.NewBuffer(nil)
|
||||
w := bufio.NewWriter(buf)
|
||||
t.writeTo(w)
|
||||
w.Flush()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func (t *groupMetadata) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readInt16(r, size, &t.Version); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readStringArray(r, remain, &t.Topics); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readBytes(r, remain, &t.UserData); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type joinGroupRequestGroupProtocolV1 struct {
|
||||
ProtocolName string
|
||||
ProtocolMetadata []byte
|
||||
}
|
||||
|
||||
func (t joinGroupRequestGroupProtocolV1) size() int32 {
|
||||
return sizeofString(t.ProtocolName) +
|
||||
sizeofBytes(t.ProtocolMetadata)
|
||||
}
|
||||
|
||||
func (t joinGroupRequestGroupProtocolV1) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.ProtocolName)
|
||||
writeBytes(w, t.ProtocolMetadata)
|
||||
}
|
||||
|
||||
type joinGroupRequestV1 struct {
|
||||
// GroupID holds the unique group identifier
|
||||
GroupID string
|
||||
|
||||
// SessionTimeout holds the coordinator considers the consumer dead if it
|
||||
// receives no heartbeat after this timeout in ms.
|
||||
SessionTimeout int32
|
||||
|
||||
// RebalanceTimeout holds the maximum time that the coordinator will wait
|
||||
// for each member to rejoin when rebalancing the group in ms
|
||||
RebalanceTimeout int32
|
||||
|
||||
// MemberID assigned by the group coordinator or the zero string if joining
|
||||
// for the first time.
|
||||
MemberID string
|
||||
|
||||
// ProtocolType holds the unique name for class of protocols implemented by group
|
||||
ProtocolType string
|
||||
|
||||
// GroupProtocols holds the list of protocols that the member supports
|
||||
GroupProtocols []joinGroupRequestGroupProtocolV1
|
||||
}
|
||||
|
||||
func (t joinGroupRequestV1) size() int32 {
|
||||
return sizeofString(t.GroupID) +
|
||||
sizeofInt32(t.SessionTimeout) +
|
||||
sizeofInt32(t.RebalanceTimeout) +
|
||||
sizeofString(t.MemberID) +
|
||||
sizeofString(t.ProtocolType) +
|
||||
sizeofArray(len(t.GroupProtocols), func(i int) int32 { return t.GroupProtocols[i].size() })
|
||||
}
|
||||
|
||||
func (t joinGroupRequestV1) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.GroupID)
|
||||
writeInt32(w, t.SessionTimeout)
|
||||
writeInt32(w, t.RebalanceTimeout)
|
||||
writeString(w, t.MemberID)
|
||||
writeString(w, t.ProtocolType)
|
||||
writeArray(w, len(t.GroupProtocols), func(i int) { t.GroupProtocols[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type joinGroupResponseMemberV1 struct {
|
||||
// MemberID assigned by the group coordinator
|
||||
MemberID string
|
||||
MemberMetadata []byte
|
||||
}
|
||||
|
||||
func (t joinGroupResponseMemberV1) size() int32 {
|
||||
return sizeofString(t.MemberID) +
|
||||
sizeofBytes(t.MemberMetadata)
|
||||
}
|
||||
|
||||
func (t joinGroupResponseMemberV1) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.MemberID)
|
||||
writeBytes(w, t.MemberMetadata)
|
||||
}
|
||||
|
||||
func (t *joinGroupResponseMemberV1) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readString(r, size, &t.MemberID); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readBytes(r, remain, &t.MemberMetadata); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type joinGroupResponseV1 struct {
|
||||
// ErrorCode holds response error code
|
||||
ErrorCode int16
|
||||
|
||||
// GenerationID holds the generation of the group.
|
||||
GenerationID int32
|
||||
|
||||
// GroupProtocol holds the group protocol selected by the coordinator
|
||||
GroupProtocol string
|
||||
|
||||
// LeaderID holds the leader of the group
|
||||
LeaderID string
|
||||
|
||||
// MemberID assigned by the group coordinator
|
||||
MemberID string
|
||||
Members []joinGroupResponseMemberV1
|
||||
}
|
||||
|
||||
func (t joinGroupResponseV1) size() int32 {
|
||||
return sizeofInt16(t.ErrorCode) +
|
||||
sizeofInt32(t.GenerationID) +
|
||||
sizeofString(t.GroupProtocol) +
|
||||
sizeofString(t.LeaderID) +
|
||||
sizeofString(t.MemberID) +
|
||||
sizeofArray(len(t.MemberID), func(i int) int32 { return t.Members[i].size() })
|
||||
}
|
||||
|
||||
func (t joinGroupResponseV1) writeTo(w *bufio.Writer) {
|
||||
writeInt16(w, t.ErrorCode)
|
||||
writeInt32(w, t.GenerationID)
|
||||
writeString(w, t.GroupProtocol)
|
||||
writeString(w, t.LeaderID)
|
||||
writeString(w, t.MemberID)
|
||||
writeArray(w, len(t.Members), func(i int) { t.Members[i].writeTo(w) })
|
||||
}
|
||||
|
||||
func (t *joinGroupResponseV1) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readInt16(r, size, &t.ErrorCode); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readInt32(r, remain, &t.GenerationID); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readString(r, remain, &t.GroupProtocol); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readString(r, remain, &t.LeaderID); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readString(r, remain, &t.MemberID); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
fn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {
|
||||
var item joinGroupResponseMemberV1
|
||||
if fnRemain, fnErr = (&item).readFrom(r, size); fnErr != nil {
|
||||
return
|
||||
}
|
||||
t.Members = append(t.Members, item)
|
||||
return
|
||||
}
|
||||
if remain, err = readArrayWith(r, remain, fn); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import "bufio"
|
||||
|
||||
type leaveGroupRequestV0 struct {
|
||||
// GroupID holds the unique group identifier
|
||||
GroupID string
|
||||
|
||||
// MemberID assigned by the group coordinator or the zero string if joining
|
||||
// for the first time.
|
||||
MemberID string
|
||||
}
|
||||
|
||||
func (t leaveGroupRequestV0) size() int32 {
|
||||
return sizeofString(t.GroupID) +
|
||||
sizeofString(t.MemberID)
|
||||
}
|
||||
|
||||
func (t leaveGroupRequestV0) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.GroupID)
|
||||
writeString(w, t.MemberID)
|
||||
}
|
||||
|
||||
type leaveGroupResponseV0 struct {
|
||||
// ErrorCode holds response error code
|
||||
ErrorCode int16
|
||||
}
|
||||
|
||||
func (t leaveGroupResponseV0) size() int32 {
|
||||
return sizeofInt16(t.ErrorCode)
|
||||
}
|
||||
|
||||
func (t leaveGroupResponseV0) writeTo(w *bufio.Writer) {
|
||||
writeInt16(w, t.ErrorCode)
|
||||
}
|
||||
|
||||
func (t *leaveGroupResponseV0) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readInt16(r, size, &t.ErrorCode); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
-87
@@ -1,87 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
)
|
||||
|
||||
type listGroupsRequestV1 struct {
|
||||
}
|
||||
|
||||
func (t listGroupsRequestV1) size() int32 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (t listGroupsRequestV1) writeTo(w *bufio.Writer) {
|
||||
}
|
||||
|
||||
type ListGroupsResponseGroupV1 struct {
|
||||
// GroupID holds the unique group identifier
|
||||
GroupID string
|
||||
ProtocolType string
|
||||
}
|
||||
|
||||
func (t ListGroupsResponseGroupV1) size() int32 {
|
||||
return sizeofString(t.GroupID) +
|
||||
sizeofString(t.ProtocolType)
|
||||
}
|
||||
|
||||
func (t ListGroupsResponseGroupV1) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.GroupID)
|
||||
writeString(w, t.ProtocolType)
|
||||
}
|
||||
|
||||
func (t *ListGroupsResponseGroupV1) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readString(r, size, &t.GroupID); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readString(r, remain, &t.ProtocolType); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type listGroupsResponseV1 struct {
|
||||
// ThrottleTimeMS holds the duration in milliseconds for which the request
|
||||
// was throttled due to quota violation (Zero if the request did not violate
|
||||
// any quota)
|
||||
ThrottleTimeMS int32
|
||||
|
||||
// ErrorCode holds response error code
|
||||
ErrorCode int16
|
||||
Groups []ListGroupsResponseGroupV1
|
||||
}
|
||||
|
||||
func (t listGroupsResponseV1) size() int32 {
|
||||
return sizeofInt32(t.ThrottleTimeMS) +
|
||||
sizeofInt16(t.ErrorCode) +
|
||||
sizeofArray(len(t.Groups), func(i int) int32 { return t.Groups[i].size() })
|
||||
}
|
||||
|
||||
func (t listGroupsResponseV1) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, t.ThrottleTimeMS)
|
||||
writeInt16(w, t.ErrorCode)
|
||||
writeArray(w, len(t.Groups), func(i int) { t.Groups[i].writeTo(w) })
|
||||
}
|
||||
|
||||
func (t *listGroupsResponseV1) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readInt32(r, size, &t.ThrottleTimeMS); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readInt16(r, remain, &t.ErrorCode); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
fn := func(withReader *bufio.Reader, withSize int) (fnRemain int, fnErr error) {
|
||||
var item ListGroupsResponseGroupV1
|
||||
if fnRemain, fnErr = (&item).readFrom(withReader, withSize); err != nil {
|
||||
return
|
||||
}
|
||||
t.Groups = append(t.Groups, item)
|
||||
return
|
||||
}
|
||||
if remain, err = readArrayWith(r, remain, fn); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
-105
@@ -1,105 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import "bufio"
|
||||
|
||||
type listOffsetRequestV1 struct {
|
||||
ReplicaID int32
|
||||
Topics []listOffsetRequestTopicV1
|
||||
}
|
||||
|
||||
func (r listOffsetRequestV1) size() int32 {
|
||||
return 4 + sizeofArray(len(r.Topics), func(i int) int32 { return r.Topics[i].size() })
|
||||
}
|
||||
|
||||
func (r listOffsetRequestV1) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, r.ReplicaID)
|
||||
writeArray(w, len(r.Topics), func(i int) { r.Topics[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type listOffsetRequestTopicV1 struct {
|
||||
TopicName string
|
||||
Partitions []listOffsetRequestPartitionV1
|
||||
}
|
||||
|
||||
func (t listOffsetRequestTopicV1) size() int32 {
|
||||
return sizeofString(t.TopicName) +
|
||||
sizeofArray(len(t.Partitions), func(i int) int32 { return t.Partitions[i].size() })
|
||||
}
|
||||
|
||||
func (t listOffsetRequestTopicV1) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.TopicName)
|
||||
writeArray(w, len(t.Partitions), func(i int) { t.Partitions[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type listOffsetRequestPartitionV1 struct {
|
||||
Partition int32
|
||||
Time int64
|
||||
}
|
||||
|
||||
func (p listOffsetRequestPartitionV1) size() int32 {
|
||||
return 4 + 8
|
||||
}
|
||||
|
||||
func (p listOffsetRequestPartitionV1) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, p.Partition)
|
||||
writeInt64(w, p.Time)
|
||||
}
|
||||
|
||||
type listOffsetResponseV1 []listOffsetResponseTopicV1
|
||||
|
||||
func (r listOffsetResponseV1) size() int32 {
|
||||
return sizeofArray(len(r), func(i int) int32 { return r[i].size() })
|
||||
}
|
||||
|
||||
func (r listOffsetResponseV1) writeTo(w *bufio.Writer) {
|
||||
writeArray(w, len(r), func(i int) { r[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type listOffsetResponseTopicV1 struct {
|
||||
TopicName string
|
||||
PartitionOffsets []partitionOffsetV1
|
||||
}
|
||||
|
||||
func (t listOffsetResponseTopicV1) size() int32 {
|
||||
return sizeofString(t.TopicName) +
|
||||
sizeofArray(len(t.PartitionOffsets), func(i int) int32 { return t.PartitionOffsets[i].size() })
|
||||
}
|
||||
|
||||
func (t listOffsetResponseTopicV1) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.TopicName)
|
||||
writeArray(w, len(t.PartitionOffsets), func(i int) { t.PartitionOffsets[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type partitionOffsetV1 struct {
|
||||
Partition int32
|
||||
ErrorCode int16
|
||||
Timestamp int64
|
||||
Offset int64
|
||||
}
|
||||
|
||||
func (p partitionOffsetV1) size() int32 {
|
||||
return 4 + 2 + 8 + 8
|
||||
}
|
||||
|
||||
func (p partitionOffsetV1) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, p.Partition)
|
||||
writeInt16(w, p.ErrorCode)
|
||||
writeInt64(w, p.Timestamp)
|
||||
writeInt64(w, p.Offset)
|
||||
}
|
||||
|
||||
func (p *partitionOffsetV1) readFrom(r *bufio.Reader, sz int) (remain int, err error) {
|
||||
if remain, err = readInt32(r, sz, &p.Partition); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readInt16(r, remain, &p.ErrorCode); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readInt64(r, remain, &p.Timestamp); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readInt64(r, remain, &p.Offset); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
-253
@@ -1,253 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Message is a data structure representing kafka messages.
|
||||
type Message struct {
|
||||
// Topic is reads only and MUST NOT be set when writing messages
|
||||
Topic string
|
||||
|
||||
// Partition is reads only and MUST NOT be set when writing messages
|
||||
Partition int
|
||||
Offset int64
|
||||
Key []byte
|
||||
Value []byte
|
||||
|
||||
// If not set at the creation, Time will be automatically set when
|
||||
// writing the message.
|
||||
Time time.Time
|
||||
}
|
||||
|
||||
func (msg Message) item() messageSetItem {
|
||||
item := messageSetItem{
|
||||
Offset: msg.Offset,
|
||||
Message: msg.message(),
|
||||
}
|
||||
item.MessageSize = item.Message.size()
|
||||
return item
|
||||
}
|
||||
|
||||
func (msg Message) message() message {
|
||||
m := message{
|
||||
MagicByte: 1,
|
||||
Key: msg.Key,
|
||||
Value: msg.Value,
|
||||
Timestamp: timestamp(msg.Time),
|
||||
}
|
||||
m.CRC = m.crc32()
|
||||
return m
|
||||
}
|
||||
|
||||
type message struct {
|
||||
CRC int32
|
||||
MagicByte int8
|
||||
Attributes int8
|
||||
Timestamp int64
|
||||
Key []byte
|
||||
Value []byte
|
||||
}
|
||||
|
||||
func (m message) crc32() int32 {
|
||||
return int32(crc32OfMessage(m.MagicByte, m.Attributes, m.Timestamp, m.Key, m.Value))
|
||||
}
|
||||
|
||||
func (m message) size() int32 {
|
||||
size := 4 + 1 + 1 + sizeofBytes(m.Key) + sizeofBytes(m.Value)
|
||||
if m.MagicByte != 0 {
|
||||
size += 8 // Timestamp
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
func (m message) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, m.CRC)
|
||||
writeInt8(w, m.MagicByte)
|
||||
writeInt8(w, m.Attributes)
|
||||
if m.MagicByte != 0 {
|
||||
writeInt64(w, m.Timestamp)
|
||||
}
|
||||
writeBytes(w, m.Key)
|
||||
writeBytes(w, m.Value)
|
||||
}
|
||||
|
||||
type messageSetItem struct {
|
||||
Offset int64
|
||||
MessageSize int32
|
||||
Message message
|
||||
}
|
||||
|
||||
func (m messageSetItem) size() int32 {
|
||||
return 8 + 4 + m.Message.size()
|
||||
}
|
||||
|
||||
func (m messageSetItem) writeTo(w *bufio.Writer) {
|
||||
writeInt64(w, m.Offset)
|
||||
writeInt32(w, m.MessageSize)
|
||||
m.Message.writeTo(w)
|
||||
}
|
||||
|
||||
type messageSet []messageSetItem
|
||||
|
||||
func (s messageSet) size() (size int32) {
|
||||
for _, m := range s {
|
||||
size += m.size()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s messageSet) writeTo(w *bufio.Writer) {
|
||||
for _, m := range s {
|
||||
m.writeTo(w)
|
||||
}
|
||||
}
|
||||
|
||||
type messageSetReader struct {
|
||||
*readerStack
|
||||
}
|
||||
|
||||
type readerStack struct {
|
||||
reader *bufio.Reader
|
||||
remain int
|
||||
base int64
|
||||
parent *readerStack
|
||||
}
|
||||
|
||||
func newMessageSetReader(reader *bufio.Reader, remain int) *messageSetReader {
|
||||
return &messageSetReader{&readerStack{
|
||||
reader: reader,
|
||||
remain: remain,
|
||||
}}
|
||||
}
|
||||
|
||||
func (r *messageSetReader) readMessage(min int64,
|
||||
key func(*bufio.Reader, int, int) (int, error),
|
||||
val func(*bufio.Reader, int, int) (int, error),
|
||||
) (offset int64, timestamp int64, err error) {
|
||||
for r.readerStack != nil {
|
||||
if r.remain == 0 {
|
||||
r.readerStack = r.parent
|
||||
continue
|
||||
}
|
||||
|
||||
var attributes int8
|
||||
if offset, attributes, timestamp, r.remain, err = readMessageHeader(r.reader, r.remain); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// if the message is compressed, decompress it and push a new reader
|
||||
// onto the stack.
|
||||
code := attributes & compressionCodecMask
|
||||
if code != 0 {
|
||||
var codec CompressionCodec
|
||||
if codec, err = resolveCodec(attributes); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// discard next four bytes...will be -1 to indicate null key
|
||||
if r.remain, err = discardN(r.reader, r.remain, 4); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// read and decompress the contained message set.
|
||||
var decompressed []byte
|
||||
if r.remain, err = readBytesWith(r.reader, r.remain, func(r *bufio.Reader, sz, n int) (remain int, err error) {
|
||||
var value []byte
|
||||
if value, remain, err = readNewBytes(r, sz, n); err != nil {
|
||||
return
|
||||
}
|
||||
decompressed, err = codec.Decode(value)
|
||||
return
|
||||
}); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// the compressed message's offset will be equal to the offset of
|
||||
// the last message in the set. within the compressed set, the
|
||||
// offsets will be relative, so we have to scan through them to
|
||||
// get the base offset. for example, if there are four compressed
|
||||
// messages at offsets 10-13, then the container message will have
|
||||
// offset 13 and the contained messages will be 0,1,2,3. the base
|
||||
// offset for the container, then is 13-3=10.
|
||||
if offset, err = extractOffset(offset, decompressed); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
r.readerStack = &readerStack{
|
||||
reader: bufio.NewReader(bytes.NewReader(decompressed)),
|
||||
remain: len(decompressed),
|
||||
base: offset,
|
||||
parent: r.readerStack,
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// adjust the offset in case we're reading compressed messages. the
|
||||
// base will be zero otherwise.
|
||||
offset += r.base
|
||||
|
||||
// When the messages are compressed kafka may return messages at an
|
||||
// earlier offset than the one that was requested, it's the client's
|
||||
// responsibility to ignore those.
|
||||
if offset < min {
|
||||
if r.remain, err = discardBytes(r.reader, r.remain); err != nil {
|
||||
return
|
||||
}
|
||||
if r.remain, err = discardBytes(r.reader, r.remain); err != nil {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if r.remain, err = readBytesWith(r.reader, r.remain, key); err != nil {
|
||||
return
|
||||
}
|
||||
r.remain, err = readBytesWith(r.reader, r.remain, val)
|
||||
return
|
||||
}
|
||||
|
||||
err = errShortRead
|
||||
return
|
||||
}
|
||||
|
||||
func (r *messageSetReader) remaining() (remain int) {
|
||||
for s := r.readerStack; s != nil; s = s.parent {
|
||||
remain += s.remain
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (r *messageSetReader) discard() (err error) {
|
||||
if r.readerStack == nil {
|
||||
return
|
||||
}
|
||||
// rewind up to the top-most reader b/c it's the only one that's doing
|
||||
// actual i/o. the rest are byte buffers that have been pushed on the stack
|
||||
// while reading compressed message sets.
|
||||
for r.parent != nil {
|
||||
r.readerStack = r.parent
|
||||
}
|
||||
r.remain, err = discardN(r.reader, r.remain, r.remain)
|
||||
return
|
||||
}
|
||||
|
||||
func extractOffset(base int64, msgSet []byte) (offset int64, err error) {
|
||||
r, remain := bufio.NewReader(bytes.NewReader(msgSet)), len(msgSet)
|
||||
for remain > 0 {
|
||||
if remain, err = readInt64(r, remain, &offset); err != nil {
|
||||
return
|
||||
}
|
||||
var sz int32
|
||||
if remain, err = readInt32(r, remain, &sz); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = discardN(r, remain, int(sz)); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
offset = base - offset
|
||||
return
|
||||
}
|
||||
-89
@@ -1,89 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import "bufio"
|
||||
|
||||
type topicMetadataRequestV1 []string
|
||||
|
||||
func (r topicMetadataRequestV1) size() int32 {
|
||||
return sizeofStringArray([]string(r))
|
||||
}
|
||||
|
||||
func (r topicMetadataRequestV1) writeTo(w *bufio.Writer) {
|
||||
writeStringArray(w, []string(r))
|
||||
}
|
||||
|
||||
type metadataResponseV1 struct {
|
||||
Brokers []brokerMetadataV1
|
||||
ControllerID int32
|
||||
Topics []topicMetadataV1
|
||||
}
|
||||
|
||||
func (r metadataResponseV1) size() int32 {
|
||||
n1 := sizeofArray(len(r.Brokers), func(i int) int32 { return r.Brokers[i].size() })
|
||||
n2 := sizeofArray(len(r.Topics), func(i int) int32 { return r.Topics[i].size() })
|
||||
return 4 + n1 + n2
|
||||
}
|
||||
|
||||
func (r metadataResponseV1) writeTo(w *bufio.Writer) {
|
||||
writeArray(w, len(r.Brokers), func(i int) { r.Brokers[i].writeTo(w) })
|
||||
writeInt32(w, r.ControllerID)
|
||||
writeArray(w, len(r.Topics), func(i int) { r.Topics[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type brokerMetadataV1 struct {
|
||||
NodeID int32
|
||||
Host string
|
||||
Port int32
|
||||
Rack string
|
||||
}
|
||||
|
||||
func (b brokerMetadataV1) size() int32 {
|
||||
return 4 + 4 + sizeofString(b.Host) + sizeofString(b.Rack)
|
||||
}
|
||||
|
||||
func (b brokerMetadataV1) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, b.NodeID)
|
||||
writeString(w, b.Host)
|
||||
writeInt32(w, b.Port)
|
||||
writeString(w, b.Rack)
|
||||
}
|
||||
|
||||
type topicMetadataV1 struct {
|
||||
TopicErrorCode int16
|
||||
TopicName string
|
||||
Internal bool
|
||||
Partitions []partitionMetadataV1
|
||||
}
|
||||
|
||||
func (t topicMetadataV1) size() int32 {
|
||||
return 2 + 1 +
|
||||
sizeofString(t.TopicName) +
|
||||
sizeofArray(len(t.Partitions), func(i int) int32 { return t.Partitions[i].size() })
|
||||
}
|
||||
|
||||
func (t topicMetadataV1) writeTo(w *bufio.Writer) {
|
||||
writeInt16(w, t.TopicErrorCode)
|
||||
writeString(w, t.TopicName)
|
||||
writeBool(w, t.Internal)
|
||||
writeArray(w, len(t.Partitions), func(i int) { t.Partitions[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type partitionMetadataV1 struct {
|
||||
PartitionErrorCode int16
|
||||
PartitionID int32
|
||||
Leader int32
|
||||
Replicas []int32
|
||||
Isr []int32
|
||||
}
|
||||
|
||||
func (p partitionMetadataV1) size() int32 {
|
||||
return 2 + 4 + 4 + sizeofInt32Array(p.Replicas) + sizeofInt32Array(p.Isr)
|
||||
}
|
||||
|
||||
func (p partitionMetadataV1) writeTo(w *bufio.Writer) {
|
||||
writeInt16(w, p.PartitionErrorCode)
|
||||
writeInt32(w, p.PartitionID)
|
||||
writeInt32(w, p.Leader)
|
||||
writeInt32Array(w, p.Replicas)
|
||||
writeInt32Array(w, p.Isr)
|
||||
}
|
||||
-167
@@ -1,167 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import "bufio"
|
||||
|
||||
type offsetCommitRequestV2Partition struct {
|
||||
// Partition ID
|
||||
Partition int32
|
||||
|
||||
// Offset to be committed
|
||||
Offset int64
|
||||
|
||||
// Metadata holds any associated metadata the client wants to keep
|
||||
Metadata string
|
||||
}
|
||||
|
||||
func (t offsetCommitRequestV2Partition) size() int32 {
|
||||
return sizeofInt32(t.Partition) +
|
||||
sizeofInt64(t.Offset) +
|
||||
sizeofString(t.Metadata)
|
||||
}
|
||||
|
||||
func (t offsetCommitRequestV2Partition) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, t.Partition)
|
||||
writeInt64(w, t.Offset)
|
||||
writeString(w, t.Metadata)
|
||||
}
|
||||
|
||||
type offsetCommitRequestV2Topic struct {
|
||||
// Topic name
|
||||
Topic string
|
||||
|
||||
// Partitions to commit offsets
|
||||
Partitions []offsetCommitRequestV2Partition
|
||||
}
|
||||
|
||||
func (t offsetCommitRequestV2Topic) size() int32 {
|
||||
return sizeofString(t.Topic) +
|
||||
sizeofArray(len(t.Partitions), func(i int) int32 { return t.Partitions[i].size() })
|
||||
}
|
||||
|
||||
func (t offsetCommitRequestV2Topic) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.Topic)
|
||||
writeArray(w, len(t.Partitions), func(i int) { t.Partitions[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type offsetCommitRequestV2 struct {
|
||||
// GroupID holds the unique group identifier
|
||||
GroupID string
|
||||
|
||||
// GenerationID holds the generation of the group.
|
||||
GenerationID int32
|
||||
|
||||
// MemberID assigned by the group coordinator
|
||||
MemberID string
|
||||
|
||||
// RetentionTime holds the time period in ms to retain the offset.
|
||||
RetentionTime int64
|
||||
|
||||
// Topics to commit offsets
|
||||
Topics []offsetCommitRequestV2Topic
|
||||
}
|
||||
|
||||
func (t offsetCommitRequestV2) size() int32 {
|
||||
return sizeofString(t.GroupID) +
|
||||
sizeofInt32(t.GenerationID) +
|
||||
sizeofString(t.MemberID) +
|
||||
sizeofInt64(t.RetentionTime) +
|
||||
sizeofArray(len(t.Topics), func(i int) int32 { return t.Topics[i].size() })
|
||||
}
|
||||
|
||||
func (t offsetCommitRequestV2) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.GroupID)
|
||||
writeInt32(w, t.GenerationID)
|
||||
writeString(w, t.MemberID)
|
||||
writeInt64(w, t.RetentionTime)
|
||||
writeArray(w, len(t.Topics), func(i int) { t.Topics[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type offsetCommitResponseV2PartitionResponse struct {
|
||||
Partition int32
|
||||
|
||||
// ErrorCode holds response error code
|
||||
ErrorCode int16
|
||||
}
|
||||
|
||||
func (t offsetCommitResponseV2PartitionResponse) size() int32 {
|
||||
return sizeofInt32(t.Partition) +
|
||||
sizeofInt16(t.ErrorCode)
|
||||
}
|
||||
|
||||
func (t offsetCommitResponseV2PartitionResponse) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, t.Partition)
|
||||
writeInt16(w, t.ErrorCode)
|
||||
}
|
||||
|
||||
func (t *offsetCommitResponseV2PartitionResponse) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readInt32(r, size, &t.Partition); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readInt16(r, remain, &t.ErrorCode); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type offsetCommitResponseV2Response struct {
|
||||
Topic string
|
||||
PartitionResponses []offsetCommitResponseV2PartitionResponse
|
||||
}
|
||||
|
||||
func (t offsetCommitResponseV2Response) size() int32 {
|
||||
return sizeofString(t.Topic) +
|
||||
sizeofArray(len(t.PartitionResponses), func(i int) int32 { return t.PartitionResponses[i].size() })
|
||||
}
|
||||
|
||||
func (t offsetCommitResponseV2Response) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.Topic)
|
||||
writeArray(w, len(t.PartitionResponses), func(i int) { t.PartitionResponses[i].writeTo(w) })
|
||||
}
|
||||
|
||||
func (t *offsetCommitResponseV2Response) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readString(r, size, &t.Topic); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
fn := func(r *bufio.Reader, withSize int) (fnRemain int, fnErr error) {
|
||||
item := offsetCommitResponseV2PartitionResponse{}
|
||||
if fnRemain, fnErr = (&item).readFrom(r, withSize); fnErr != nil {
|
||||
return
|
||||
}
|
||||
t.PartitionResponses = append(t.PartitionResponses, item)
|
||||
return
|
||||
}
|
||||
if remain, err = readArrayWith(r, remain, fn); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
type offsetCommitResponseV2 struct {
|
||||
Responses []offsetCommitResponseV2Response
|
||||
}
|
||||
|
||||
func (t offsetCommitResponseV2) size() int32 {
|
||||
return sizeofArray(len(t.Responses), func(i int) int32 { return t.Responses[i].size() })
|
||||
}
|
||||
|
||||
func (t offsetCommitResponseV2) writeTo(w *bufio.Writer) {
|
||||
writeArray(w, len(t.Responses), func(i int) { t.Responses[i].writeTo(w) })
|
||||
}
|
||||
|
||||
func (t *offsetCommitResponseV2) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
fn := func(r *bufio.Reader, withSize int) (fnRemain int, fnErr error) {
|
||||
item := offsetCommitResponseV2Response{}
|
||||
if fnRemain, fnErr = (&item).readFrom(r, withSize); fnErr != nil {
|
||||
return
|
||||
}
|
||||
t.Responses = append(t.Responses, item)
|
||||
return
|
||||
}
|
||||
if remain, err = readArrayWith(r, size, fn); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
-168
@@ -1,168 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
)
|
||||
|
||||
type offsetFetchRequestV1Topic struct {
|
||||
// Topic name
|
||||
Topic string
|
||||
|
||||
// Partitions to fetch offsets
|
||||
Partitions []int32
|
||||
}
|
||||
|
||||
func (t offsetFetchRequestV1Topic) size() int32 {
|
||||
return sizeofString(t.Topic) +
|
||||
sizeofInt32Array(t.Partitions)
|
||||
}
|
||||
|
||||
func (t offsetFetchRequestV1Topic) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.Topic)
|
||||
writeInt32Array(w, t.Partitions)
|
||||
}
|
||||
|
||||
type offsetFetchRequestV1 struct {
|
||||
// GroupID holds the unique group identifier
|
||||
GroupID string
|
||||
|
||||
// Topics to fetch offsets.
|
||||
Topics []offsetFetchRequestV1Topic
|
||||
}
|
||||
|
||||
func (t offsetFetchRequestV1) size() int32 {
|
||||
return sizeofString(t.GroupID) +
|
||||
sizeofArray(len(t.Topics), func(i int) int32 { return t.Topics[i].size() })
|
||||
}
|
||||
|
||||
func (t offsetFetchRequestV1) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.GroupID)
|
||||
writeArray(w, len(t.Topics), func(i int) { t.Topics[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type offsetFetchResponseV1PartitionResponse struct {
|
||||
// Partition ID
|
||||
Partition int32
|
||||
|
||||
// Offset of last committed message
|
||||
Offset int64
|
||||
|
||||
// Metadata client wants to keep
|
||||
Metadata string
|
||||
|
||||
// ErrorCode holds response error code
|
||||
ErrorCode int16
|
||||
}
|
||||
|
||||
func (t offsetFetchResponseV1PartitionResponse) size() int32 {
|
||||
return sizeofInt32(t.Partition) +
|
||||
sizeofInt64(t.Offset) +
|
||||
sizeofString(t.Metadata) +
|
||||
sizeofInt16(t.ErrorCode)
|
||||
}
|
||||
|
||||
func (t offsetFetchResponseV1PartitionResponse) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, t.Partition)
|
||||
writeInt64(w, t.Offset)
|
||||
writeString(w, t.Metadata)
|
||||
writeInt16(w, t.ErrorCode)
|
||||
}
|
||||
|
||||
func (t *offsetFetchResponseV1PartitionResponse) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readInt32(r, size, &t.Partition); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readInt64(r, remain, &t.Offset); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readString(r, remain, &t.Metadata); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readInt16(r, remain, &t.ErrorCode); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type offsetFetchResponseV1Response struct {
|
||||
// Topic name
|
||||
Topic string
|
||||
|
||||
// PartitionResponses holds offsets by partition
|
||||
PartitionResponses []offsetFetchResponseV1PartitionResponse
|
||||
}
|
||||
|
||||
func (t offsetFetchResponseV1Response) size() int32 {
|
||||
return sizeofString(t.Topic) +
|
||||
sizeofArray(len(t.PartitionResponses), func(i int) int32 { return t.PartitionResponses[i].size() })
|
||||
}
|
||||
|
||||
func (t offsetFetchResponseV1Response) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.Topic)
|
||||
writeArray(w, len(t.PartitionResponses), func(i int) { t.PartitionResponses[i].writeTo(w) })
|
||||
}
|
||||
|
||||
func (t *offsetFetchResponseV1Response) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
if remain, err = readString(r, size, &t.Topic); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
fn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {
|
||||
item := offsetFetchResponseV1PartitionResponse{}
|
||||
if fnRemain, fnErr = (&item).readFrom(r, size); err != nil {
|
||||
return
|
||||
}
|
||||
t.PartitionResponses = append(t.PartitionResponses, item)
|
||||
return
|
||||
}
|
||||
if remain, err = readArrayWith(r, remain, fn); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
type offsetFetchResponseV1 struct {
|
||||
// Responses holds topic partition offsets
|
||||
Responses []offsetFetchResponseV1Response
|
||||
}
|
||||
|
||||
func (t offsetFetchResponseV1) size() int32 {
|
||||
return sizeofArray(len(t.Responses), func(i int) int32 { return t.Responses[i].size() })
|
||||
}
|
||||
|
||||
func (t offsetFetchResponseV1) writeTo(w *bufio.Writer) {
|
||||
writeArray(w, len(t.Responses), func(i int) { t.Responses[i].writeTo(w) })
|
||||
}
|
||||
|
||||
func (t *offsetFetchResponseV1) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
fn := func(r *bufio.Reader, withSize int) (fnRemain int, fnErr error) {
|
||||
item := offsetFetchResponseV1Response{}
|
||||
if fnRemain, fnErr = (&item).readFrom(r, withSize); fnErr != nil {
|
||||
return
|
||||
}
|
||||
t.Responses = append(t.Responses, item)
|
||||
return
|
||||
}
|
||||
if remain, err = readArrayWith(r, size, fn); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func findOffset(topic string, partition int32, response offsetFetchResponseV1) (int64, bool) {
|
||||
for _, r := range response.Responses {
|
||||
if r.Topic != topic {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, pr := range r.PartitionResponses {
|
||||
if pr.Partition == partition {
|
||||
return pr.Offset, true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0, false
|
||||
}
|
||||
-113
@@ -1,113 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import "bufio"
|
||||
|
||||
type produceRequestV2 struct {
|
||||
RequiredAcks int16
|
||||
Timeout int32
|
||||
Topics []produceRequestTopicV2
|
||||
}
|
||||
|
||||
func (r produceRequestV2) size() int32 {
|
||||
return 2 + 4 + sizeofArray(len(r.Topics), func(i int) int32 { return r.Topics[i].size() })
|
||||
}
|
||||
|
||||
func (r produceRequestV2) writeTo(w *bufio.Writer) {
|
||||
writeInt16(w, r.RequiredAcks)
|
||||
writeInt32(w, r.Timeout)
|
||||
writeArray(w, len(r.Topics), func(i int) { r.Topics[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type produceRequestTopicV2 struct {
|
||||
TopicName string
|
||||
Partitions []produceRequestPartitionV2
|
||||
}
|
||||
|
||||
func (t produceRequestTopicV2) size() int32 {
|
||||
return sizeofString(t.TopicName) +
|
||||
sizeofArray(len(t.Partitions), func(i int) int32 { return t.Partitions[i].size() })
|
||||
}
|
||||
|
||||
func (t produceRequestTopicV2) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.TopicName)
|
||||
writeArray(w, len(t.Partitions), func(i int) { t.Partitions[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type produceRequestPartitionV2 struct {
|
||||
Partition int32
|
||||
MessageSetSize int32
|
||||
MessageSet messageSet
|
||||
}
|
||||
|
||||
func (p produceRequestPartitionV2) size() int32 {
|
||||
return 4 + 4 + p.MessageSet.size()
|
||||
}
|
||||
|
||||
func (p produceRequestPartitionV2) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, p.Partition)
|
||||
writeInt32(w, p.MessageSetSize)
|
||||
p.MessageSet.writeTo(w)
|
||||
}
|
||||
|
||||
type produceResponseV2 struct {
|
||||
ThrottleTime int32
|
||||
Topics []produceResponseTopicV2
|
||||
}
|
||||
|
||||
func (r produceResponseV2) size() int32 {
|
||||
return 4 + sizeofArray(len(r.Topics), func(i int) int32 { return r.Topics[i].size() })
|
||||
}
|
||||
|
||||
func (r produceResponseV2) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, r.ThrottleTime)
|
||||
writeArray(w, len(r.Topics), func(i int) { r.Topics[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type produceResponseTopicV2 struct {
|
||||
TopicName string
|
||||
Partitions []produceResponsePartitionV2
|
||||
}
|
||||
|
||||
func (t produceResponseTopicV2) size() int32 {
|
||||
return sizeofString(t.TopicName) +
|
||||
sizeofArray(len(t.Partitions), func(i int) int32 { return t.Partitions[i].size() })
|
||||
}
|
||||
|
||||
func (t produceResponseTopicV2) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.TopicName)
|
||||
writeArray(w, len(t.Partitions), func(i int) { t.Partitions[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type produceResponsePartitionV2 struct {
|
||||
Partition int32
|
||||
ErrorCode int16
|
||||
Offset int64
|
||||
Timestamp int64
|
||||
}
|
||||
|
||||
func (p produceResponsePartitionV2) size() int32 {
|
||||
return 4 + 2 + 8 + 8
|
||||
}
|
||||
|
||||
func (p produceResponsePartitionV2) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, p.Partition)
|
||||
writeInt16(w, p.ErrorCode)
|
||||
writeInt64(w, p.Offset)
|
||||
writeInt64(w, p.Timestamp)
|
||||
}
|
||||
|
||||
func (p *produceResponsePartitionV2) readFrom(r *bufio.Reader, sz int) (remain int, err error) {
|
||||
if remain, err = readInt32(r, sz, &p.Partition); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readInt16(r, remain, &p.ErrorCode); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readInt64(r, remain, &p.Offset); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readInt64(r, remain, &p.Timestamp); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
-84
@@ -1,84 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type apiKey int16
|
||||
|
||||
const (
|
||||
produceRequest apiKey = 0
|
||||
fetchRequest apiKey = 1
|
||||
listOffsetRequest apiKey = 2
|
||||
metadataRequest apiKey = 3
|
||||
offsetCommitRequest apiKey = 8
|
||||
offsetFetchRequest apiKey = 9
|
||||
groupCoordinatorRequest apiKey = 10
|
||||
joinGroupRequest apiKey = 11
|
||||
heartbeatRequest apiKey = 12
|
||||
leaveGroupRequest apiKey = 13
|
||||
syncGroupRequest apiKey = 14
|
||||
describeGroupsRequest apiKey = 15
|
||||
listGroupsRequest apiKey = 16
|
||||
createTopicsRequest apiKey = 19
|
||||
deleteTopicsRequest apiKey = 20
|
||||
)
|
||||
|
||||
type apiVersion int16
|
||||
|
||||
const (
|
||||
v0 apiVersion = 0
|
||||
v1 apiVersion = 1
|
||||
v2 apiVersion = 2
|
||||
v3 apiVersion = 3
|
||||
)
|
||||
|
||||
type requestHeader struct {
|
||||
Size int32
|
||||
ApiKey int16
|
||||
ApiVersion int16
|
||||
CorrelationID int32
|
||||
ClientID string
|
||||
}
|
||||
|
||||
func (h requestHeader) size() int32 {
|
||||
return 4 + 2 + 2 + 4 + sizeofString(h.ClientID)
|
||||
}
|
||||
|
||||
func (h requestHeader) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, h.Size)
|
||||
writeInt16(w, h.ApiKey)
|
||||
writeInt16(w, h.ApiVersion)
|
||||
writeInt32(w, h.CorrelationID)
|
||||
writeString(w, h.ClientID)
|
||||
}
|
||||
|
||||
type request interface {
|
||||
size() int32
|
||||
writeTo(*bufio.Writer)
|
||||
}
|
||||
|
||||
func makeInt8(b []byte) int8 {
|
||||
return int8(b[0])
|
||||
}
|
||||
|
||||
func makeInt16(b []byte) int16 {
|
||||
return int16(binary.BigEndian.Uint16(b))
|
||||
}
|
||||
|
||||
func makeInt32(b []byte) int32 {
|
||||
return int32(binary.BigEndian.Uint32(b))
|
||||
}
|
||||
|
||||
func makeInt64(b []byte) int64 {
|
||||
return int64(binary.BigEndian.Uint64(b))
|
||||
}
|
||||
|
||||
func expectZeroSize(sz int, err error) error {
|
||||
if err == nil && sz != 0 {
|
||||
err = fmt.Errorf("reading a response left %d unread bytes", sz)
|
||||
}
|
||||
return err
|
||||
}
|
||||
-376
@@ -1,376 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
type readable interface {
|
||||
readFrom(*bufio.Reader, int) (int, error)
|
||||
}
|
||||
|
||||
var errShortRead = errors.New("not enough bytes available to load the response")
|
||||
|
||||
func peekRead(r *bufio.Reader, sz int, n int, f func([]byte)) (int, error) {
|
||||
if n > sz {
|
||||
return sz, errShortRead
|
||||
}
|
||||
b, err := r.Peek(n)
|
||||
if err != nil {
|
||||
return sz, err
|
||||
}
|
||||
f(b)
|
||||
return discardN(r, sz, n)
|
||||
}
|
||||
|
||||
func readInt8(r *bufio.Reader, sz int, v *int8) (int, error) {
|
||||
return peekRead(r, sz, 1, func(b []byte) { *v = makeInt8(b) })
|
||||
}
|
||||
|
||||
func readInt16(r *bufio.Reader, sz int, v *int16) (int, error) {
|
||||
return peekRead(r, sz, 2, func(b []byte) { *v = makeInt16(b) })
|
||||
}
|
||||
|
||||
func readInt32(r *bufio.Reader, sz int, v *int32) (int, error) {
|
||||
return peekRead(r, sz, 4, func(b []byte) { *v = makeInt32(b) })
|
||||
}
|
||||
|
||||
func readInt64(r *bufio.Reader, sz int, v *int64) (int, error) {
|
||||
return peekRead(r, sz, 8, func(b []byte) { *v = makeInt64(b) })
|
||||
}
|
||||
|
||||
func readBool(r *bufio.Reader, sz int, v *bool) (int, error) {
|
||||
return peekRead(r, sz, 1, func(b []byte) { *v = b[0] != 0 })
|
||||
}
|
||||
|
||||
func readString(r *bufio.Reader, sz int, v *string) (int, error) {
|
||||
return readStringWith(r, sz, func(r *bufio.Reader, sz int, n int) (remain int, err error) {
|
||||
*v, remain, err = readNewString(r, sz, n)
|
||||
return
|
||||
})
|
||||
}
|
||||
|
||||
func readStringWith(r *bufio.Reader, sz int, cb func(*bufio.Reader, int, int) (int, error)) (int, error) {
|
||||
var err error
|
||||
var len int16
|
||||
|
||||
if sz, err = readInt16(r, sz, &len); err != nil {
|
||||
return sz, err
|
||||
}
|
||||
|
||||
n := int(len)
|
||||
if n > sz {
|
||||
return sz, errShortRead
|
||||
}
|
||||
|
||||
return cb(r, sz, n)
|
||||
}
|
||||
|
||||
func readNewString(r *bufio.Reader, sz int, n int) (string, int, error) {
|
||||
b, sz, err := readNewBytes(r, sz, n)
|
||||
return string(b), sz, err
|
||||
}
|
||||
|
||||
func readBytes(r *bufio.Reader, sz int, v *[]byte) (int, error) {
|
||||
return readBytesWith(r, sz, func(r *bufio.Reader, sz int, n int) (remain int, err error) {
|
||||
*v, remain, err = readNewBytes(r, sz, n)
|
||||
return
|
||||
})
|
||||
}
|
||||
|
||||
func readBytesWith(r *bufio.Reader, sz int, cb func(*bufio.Reader, int, int) (int, error)) (int, error) {
|
||||
var err error
|
||||
var len int32
|
||||
|
||||
if sz, err = readInt32(r, sz, &len); err != nil {
|
||||
return sz, err
|
||||
}
|
||||
|
||||
n := int(len)
|
||||
if n > sz {
|
||||
return sz, errShortRead
|
||||
}
|
||||
|
||||
return cb(r, sz, n)
|
||||
}
|
||||
|
||||
func readNewBytes(r *bufio.Reader, sz int, n int) ([]byte, int, error) {
|
||||
var err error
|
||||
var b []byte
|
||||
|
||||
if n > 0 {
|
||||
b = make([]byte, n)
|
||||
n, err = io.ReadFull(r, b)
|
||||
b = b[:n]
|
||||
sz -= n
|
||||
}
|
||||
|
||||
return b, sz, err
|
||||
}
|
||||
|
||||
func readArrayWith(r *bufio.Reader, sz int, cb func(*bufio.Reader, int) (int, error)) (int, error) {
|
||||
var err error
|
||||
var len int32
|
||||
|
||||
if sz, err = readInt32(r, sz, &len); err != nil {
|
||||
return sz, err
|
||||
}
|
||||
|
||||
for n := int(len); n > 0; n-- {
|
||||
if sz, err = cb(r, sz); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return sz, err
|
||||
}
|
||||
|
||||
func readStringArray(r *bufio.Reader, sz int, v *[]string) (remain int, err error) {
|
||||
var content []string
|
||||
fn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {
|
||||
var value string
|
||||
if fnRemain, fnErr = readString(r, size, &value); fnErr != nil {
|
||||
return
|
||||
}
|
||||
content = append(content, value)
|
||||
return
|
||||
}
|
||||
if remain, err = readArrayWith(r, sz, fn); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
*v = content
|
||||
return
|
||||
}
|
||||
|
||||
func readMapStringInt32(r *bufio.Reader, sz int, v *map[string][]int32) (remain int, err error) {
|
||||
var len int32
|
||||
if remain, err = readInt32(r, sz, &len); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
content := make(map[string][]int32, len)
|
||||
for i := 0; i < int(len); i++ {
|
||||
var key string
|
||||
var values []int32
|
||||
|
||||
if remain, err = readString(r, remain, &key); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
fn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {
|
||||
var value int32
|
||||
if fnRemain, fnErr = readInt32(r, size, &value); fnErr != nil {
|
||||
return
|
||||
}
|
||||
values = append(values, value)
|
||||
return
|
||||
}
|
||||
if remain, err = readArrayWith(r, remain, fn); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
content[key] = values
|
||||
}
|
||||
*v = content
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func read(r *bufio.Reader, sz int, a interface{}) (int, error) {
|
||||
switch v := a.(type) {
|
||||
case *int8:
|
||||
return readInt8(r, sz, v)
|
||||
case *int16:
|
||||
return readInt16(r, sz, v)
|
||||
case *int32:
|
||||
return readInt32(r, sz, v)
|
||||
case *int64:
|
||||
return readInt64(r, sz, v)
|
||||
case *bool:
|
||||
return readBool(r, sz, v)
|
||||
case *string:
|
||||
return readString(r, sz, v)
|
||||
case *[]byte:
|
||||
return readBytes(r, sz, v)
|
||||
}
|
||||
switch v := reflect.ValueOf(a).Elem(); v.Kind() {
|
||||
case reflect.Struct:
|
||||
return readStruct(r, sz, v)
|
||||
case reflect.Slice:
|
||||
return readSlice(r, sz, v)
|
||||
default:
|
||||
panic(fmt.Sprintf("unsupported type: %T", a))
|
||||
}
|
||||
}
|
||||
|
||||
func readAll(r *bufio.Reader, sz int, ptrs ...interface{}) (int, error) {
|
||||
var err error
|
||||
|
||||
for _, ptr := range ptrs {
|
||||
if sz, err = readPtr(r, sz, ptr); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return sz, err
|
||||
}
|
||||
|
||||
func readPtr(r *bufio.Reader, sz int, ptr interface{}) (int, error) {
|
||||
switch v := ptr.(type) {
|
||||
case *int8:
|
||||
return readInt8(r, sz, v)
|
||||
case *int16:
|
||||
return readInt16(r, sz, v)
|
||||
case *int32:
|
||||
return readInt32(r, sz, v)
|
||||
case *int64:
|
||||
return readInt64(r, sz, v)
|
||||
case *string:
|
||||
return readString(r, sz, v)
|
||||
case *[]byte:
|
||||
return readBytes(r, sz, v)
|
||||
case readable:
|
||||
return v.readFrom(r, sz)
|
||||
default:
|
||||
panic(fmt.Sprintf("unsupported type: %T", v))
|
||||
}
|
||||
}
|
||||
|
||||
func readStruct(r *bufio.Reader, sz int, v reflect.Value) (int, error) {
|
||||
var err error
|
||||
for i, n := 0, v.NumField(); i != n; i++ {
|
||||
if sz, err = read(r, sz, v.Field(i).Addr().Interface()); err != nil {
|
||||
return sz, err
|
||||
}
|
||||
}
|
||||
return sz, nil
|
||||
}
|
||||
|
||||
func readSlice(r *bufio.Reader, sz int, v reflect.Value) (int, error) {
|
||||
var err error
|
||||
var len int32
|
||||
|
||||
if sz, err = readInt32(r, sz, &len); err != nil {
|
||||
return sz, err
|
||||
}
|
||||
|
||||
if n := int(len); n < 0 {
|
||||
v.Set(reflect.Zero(v.Type()))
|
||||
} else {
|
||||
v.Set(reflect.MakeSlice(v.Type(), n, n))
|
||||
|
||||
for i := 0; i != n; i++ {
|
||||
if sz, err = read(r, sz, v.Index(i).Addr().Interface()); err != nil {
|
||||
return sz, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return sz, nil
|
||||
}
|
||||
|
||||
func readFetchResponseHeader(r *bufio.Reader, size int) (throttle int32, watermark int64, remain int, err error) {
|
||||
var n int32
|
||||
var p struct {
|
||||
Partition int32
|
||||
ErrorCode int16
|
||||
HighwaterMarkOffset int64
|
||||
MessageSetSize int32
|
||||
}
|
||||
|
||||
if remain, err = readInt32(r, size, &throttle); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = readInt32(r, remain, &n); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// This error should never trigger, unless there's a bug in the kafka client
|
||||
// or server.
|
||||
if n != 1 {
|
||||
err = fmt.Errorf("1 kafka topic was expected in the fetch response but the client received %d", n)
|
||||
return
|
||||
}
|
||||
|
||||
// We ignore the topic name because we've requests messages for a single
|
||||
// topic, unless there's a bug in the kafka server we will have received
|
||||
// the name of the topic that we requested.
|
||||
if remain, err = discardString(r, remain); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = readInt32(r, remain, &n); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// This error should never trigger, unless there's a bug in the kafka client
|
||||
// or server.
|
||||
if n != 1 {
|
||||
err = fmt.Errorf("1 kafka partition was expected in the fetch response but the client received %d", n)
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = read(r, remain, &p); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if p.ErrorCode != 0 {
|
||||
err = Error(p.ErrorCode)
|
||||
return
|
||||
}
|
||||
|
||||
// This error should never trigger, unless there's a bug in the kafka client
|
||||
// or server.
|
||||
if remain != int(p.MessageSetSize) {
|
||||
err = fmt.Errorf("the size of the message set in a fetch response doesn't match the number of remaining bytes (message set size = %d, remaining bytes = %d)", p.MessageSetSize, remain)
|
||||
return
|
||||
}
|
||||
|
||||
watermark = p.HighwaterMarkOffset
|
||||
return
|
||||
}
|
||||
|
||||
func readMessageHeader(r *bufio.Reader, sz int) (offset int64, attributes int8, timestamp int64, remain int, err error) {
|
||||
var version int8
|
||||
|
||||
if remain, err = readInt64(r, sz, &offset); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// On discarding the message size and CRC:
|
||||
// ---------------------------------------
|
||||
//
|
||||
// - Not sure why kafka gives the message size here, we already have the
|
||||
// number of remaining bytes in the response and kafka should only truncate
|
||||
// the trailing message.
|
||||
//
|
||||
// - TCP is already taking care of ensuring data integrity, no need to
|
||||
// waste resources doing it a second time so we just skip the message CRC.
|
||||
//
|
||||
if remain, err = discardN(r, remain, 8); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = readInt8(r, remain, &version); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if remain, err = readInt8(r, remain, &attributes); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
switch version {
|
||||
case 0:
|
||||
case 1:
|
||||
remain, err = readInt64(r, remain, ×tamp)
|
||||
default:
|
||||
err = fmt.Errorf("unsupported message version %d found in fetch response", version)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
-1895
File diff suppressed because it is too large
Load Diff
-61
@@ -1,61 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// runGroup is a collection of goroutines working together. If any one goroutine
|
||||
// stops, then all goroutines will be stopped.
|
||||
//
|
||||
// A zero runGroup is valid
|
||||
type runGroup struct {
|
||||
initOnce sync.Once
|
||||
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
func (r *runGroup) init() {
|
||||
if r.cancel == nil {
|
||||
r.ctx, r.cancel = context.WithCancel(context.Background())
|
||||
}
|
||||
}
|
||||
|
||||
func (r *runGroup) WithContext(ctx context.Context) *runGroup {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
return &runGroup{
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
}
|
||||
|
||||
// Wait blocks until all function calls have returned.
|
||||
func (r *runGroup) Wait() {
|
||||
r.wg.Wait()
|
||||
}
|
||||
|
||||
// Stop stops the goroutines and waits for them to complete
|
||||
func (r *runGroup) Stop() {
|
||||
r.initOnce.Do(r.init)
|
||||
r.cancel()
|
||||
r.Wait()
|
||||
}
|
||||
|
||||
// Go calls the given function in a new goroutine.
|
||||
//
|
||||
// The first call to return a non-nil error cancels the group; its error will be
|
||||
// returned by Wait.
|
||||
func (r *runGroup) Go(f func(stop <-chan struct{})) {
|
||||
r.initOnce.Do(r.init)
|
||||
|
||||
r.wg.Add(1)
|
||||
go func() {
|
||||
defer r.wg.Done()
|
||||
defer r.cancel()
|
||||
|
||||
f(r.ctx.Done())
|
||||
}()
|
||||
}
|
||||
-73
@@ -1,73 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import "fmt"
|
||||
|
||||
type sizable interface {
|
||||
size() int32
|
||||
}
|
||||
|
||||
func sizeof(a interface{}) int32 {
|
||||
switch v := a.(type) {
|
||||
case int8:
|
||||
return 1
|
||||
case int16:
|
||||
return 2
|
||||
case int32:
|
||||
return 4
|
||||
case int64:
|
||||
return 8
|
||||
case string:
|
||||
return sizeofString(v)
|
||||
case bool:
|
||||
return 1
|
||||
case []byte:
|
||||
return sizeofBytes(v)
|
||||
case sizable:
|
||||
return v.size()
|
||||
}
|
||||
panic(fmt.Sprintf("unsupported type: %T", a))
|
||||
}
|
||||
|
||||
func sizeofInt8(_ int8) int32 {
|
||||
return 1
|
||||
}
|
||||
|
||||
func sizeofInt16(_ int16) int32 {
|
||||
return 2
|
||||
}
|
||||
|
||||
func sizeofInt32(_ int32) int32 {
|
||||
return 4
|
||||
}
|
||||
|
||||
func sizeofInt64(_ int64) int32 {
|
||||
return 8
|
||||
}
|
||||
|
||||
func sizeofString(s string) int32 {
|
||||
return 2 + int32(len(s))
|
||||
}
|
||||
|
||||
func sizeofBool(_ bool) int32 {
|
||||
return 1
|
||||
}
|
||||
|
||||
func sizeofBytes(b []byte) int32 {
|
||||
return 4 + int32(len(b))
|
||||
}
|
||||
|
||||
func sizeofArray(n int, f func(int) int32) int32 {
|
||||
s := int32(4)
|
||||
for i := 0; i != n; i++ {
|
||||
s += f(i)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func sizeofInt32Array(a []int32) int32 {
|
||||
return 4 + (4 * int32(len(a)))
|
||||
}
|
||||
|
||||
func sizeofStringArray(a []string) int32 {
|
||||
return sizeofArray(len(a), func(i int) int32 { return sizeofString(a[i]) })
|
||||
}
|
||||
-186
@@ -1,186 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SummaryStats is a data structure that carries a summary of observed values.
|
||||
// The average, minimum, and maximum are reported.
|
||||
type SummaryStats struct {
|
||||
Avg int64 `metric:"avg" type:"gauge"`
|
||||
Min int64 `metric:"min" type:"gauge"`
|
||||
Max int64 `metric:"max" type:"gauge"`
|
||||
}
|
||||
|
||||
// DurationStats is a data structure that carries a summary of observed duration
|
||||
// values. The average, minimum, and maximum are reported.
|
||||
type DurationStats struct {
|
||||
Avg time.Duration `metric:"avg" type:"gauge"`
|
||||
Min time.Duration `metric:"min" type:"gauge"`
|
||||
Max time.Duration `metric:"max" type:"gauge"`
|
||||
}
|
||||
|
||||
// counter is an atomic incrementing counter which gets reset on snapshot.
|
||||
//
|
||||
// Since atomic is used to mutate the statistic the value must be 64-bit aligned.
|
||||
// See https://golang.org/pkg/sync/atomic/#pkg-note-BUG
|
||||
type counter int64
|
||||
|
||||
func (c *counter) ptr() *int64 {
|
||||
return (*int64)(c)
|
||||
}
|
||||
|
||||
func (c *counter) observe(v int64) {
|
||||
atomic.AddInt64(c.ptr(), v)
|
||||
}
|
||||
|
||||
func (c *counter) snapshot() int64 {
|
||||
p := c.ptr()
|
||||
v := atomic.LoadInt64(p)
|
||||
atomic.AddInt64(p, -v)
|
||||
return v
|
||||
}
|
||||
|
||||
// gauge is an atomic integer that may be set to any arbitrary value, the value
|
||||
// does not change after a snapshot.
|
||||
//
|
||||
// Since atomic is used to mutate the statistic the value must be 64-bit aligned.
|
||||
// See https://golang.org/pkg/sync/atomic/#pkg-note-BUG
|
||||
type gauge int64
|
||||
|
||||
func (g *gauge) ptr() *int64 {
|
||||
return (*int64)(g)
|
||||
}
|
||||
|
||||
func (g *gauge) observe(v int64) {
|
||||
atomic.StoreInt64(g.ptr(), v)
|
||||
}
|
||||
|
||||
func (g *gauge) snapshot() int64 {
|
||||
return atomic.LoadInt64(g.ptr())
|
||||
}
|
||||
|
||||
// minimum is an atomic integral type that keeps track of the minimum of all
|
||||
// values that it observed between snapshots.
|
||||
//
|
||||
// Since atomic is used to mutate the statistic the value must be 64-bit aligned.
|
||||
// See https://golang.org/pkg/sync/atomic/#pkg-note-BUG
|
||||
type minimum int64
|
||||
|
||||
func (m *minimum) ptr() *int64 {
|
||||
return (*int64)(m)
|
||||
}
|
||||
|
||||
func (m *minimum) observe(v int64) {
|
||||
for {
|
||||
ptr := m.ptr()
|
||||
min := atomic.LoadInt64(ptr)
|
||||
|
||||
if min >= 0 && min <= v {
|
||||
break
|
||||
}
|
||||
|
||||
if atomic.CompareAndSwapInt64(ptr, min, v) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *minimum) snapshot() int64 {
|
||||
p := m.ptr()
|
||||
v := atomic.LoadInt64(p)
|
||||
atomic.CompareAndSwapInt64(p, v, -1)
|
||||
if v < 0 {
|
||||
v = 0
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// maximum is an atomic integral type that keeps track of the maximum of all
|
||||
// values that it observed between snapshots.
|
||||
//
|
||||
// Since atomic is used to mutate the statistic the value must be 64-bit aligned.
|
||||
// See https://golang.org/pkg/sync/atomic/#pkg-note-BUG
|
||||
type maximum int64
|
||||
|
||||
func (m *maximum) ptr() *int64 {
|
||||
return (*int64)(m)
|
||||
}
|
||||
|
||||
func (m *maximum) observe(v int64) {
|
||||
for {
|
||||
ptr := m.ptr()
|
||||
max := atomic.LoadInt64(ptr)
|
||||
|
||||
if max >= 0 && max >= v {
|
||||
break
|
||||
}
|
||||
|
||||
if atomic.CompareAndSwapInt64(ptr, max, v) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *maximum) snapshot() int64 {
|
||||
p := m.ptr()
|
||||
v := atomic.LoadInt64(p)
|
||||
atomic.CompareAndSwapInt64(p, v, -1)
|
||||
if v < 0 {
|
||||
v = 0
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
type summary struct {
|
||||
min minimum
|
||||
max maximum
|
||||
sum counter
|
||||
count counter
|
||||
}
|
||||
|
||||
func makeSummary() summary {
|
||||
return summary{
|
||||
min: -1,
|
||||
max: -1,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *summary) observe(v int64) {
|
||||
s.min.observe(v)
|
||||
s.max.observe(v)
|
||||
s.sum.observe(v)
|
||||
s.count.observe(1)
|
||||
}
|
||||
|
||||
func (s *summary) observeDuration(v time.Duration) {
|
||||
s.observe(int64(v))
|
||||
}
|
||||
|
||||
func (s *summary) snapshot() SummaryStats {
|
||||
avg := int64(0)
|
||||
min := s.min.snapshot()
|
||||
max := s.max.snapshot()
|
||||
sum := s.sum.snapshot()
|
||||
count := s.count.snapshot()
|
||||
|
||||
if count != 0 {
|
||||
avg = int64(float64(sum) / float64(count))
|
||||
}
|
||||
|
||||
return SummaryStats{
|
||||
Avg: avg,
|
||||
Min: min,
|
||||
Max: max,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *summary) snapshotDuration() DurationStats {
|
||||
summary := s.snapshot()
|
||||
return DurationStats{
|
||||
Avg: time.Duration(summary.Avg),
|
||||
Min: time.Duration(summary.Min),
|
||||
Max: time.Duration(summary.Max),
|
||||
}
|
||||
}
|
||||
-141
@@ -1,141 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
)
|
||||
|
||||
type groupAssignment struct {
|
||||
Version int16
|
||||
Topics map[string][]int32
|
||||
UserData []byte
|
||||
}
|
||||
|
||||
func (t groupAssignment) size() int32 {
|
||||
sz := sizeofInt16(t.Version) + sizeofInt16(int16(len(t.Topics)))
|
||||
|
||||
for topic, partitions := range t.Topics {
|
||||
sz += sizeofString(topic) + sizeofInt32Array(partitions)
|
||||
}
|
||||
|
||||
return sz + sizeofBytes(t.UserData)
|
||||
}
|
||||
|
||||
func (t groupAssignment) writeTo(w *bufio.Writer) {
|
||||
writeInt16(w, t.Version)
|
||||
writeInt32(w, int32(len(t.Topics)))
|
||||
|
||||
for topic, partitions := range t.Topics {
|
||||
writeString(w, topic)
|
||||
writeInt32Array(w, partitions)
|
||||
}
|
||||
|
||||
writeBytes(w, t.UserData)
|
||||
}
|
||||
|
||||
func (t *groupAssignment) readFrom(r *bufio.Reader, size int) (remain int, err error) {
|
||||
// I came across this case when testing for compatibility with bsm/sarama-cluster. It
|
||||
// appears in some cases, sarama-cluster can send a nil array entry. Admittedly, I
|
||||
// didn't look too closely at it.
|
||||
if size == 0 {
|
||||
t.Topics = map[string][]int32{}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
if remain, err = readInt16(r, size, &t.Version); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readMapStringInt32(r, remain, &t.Topics); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readBytes(r, remain, &t.UserData); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (t groupAssignment) bytes() []byte {
|
||||
buf := bytes.NewBuffer(nil)
|
||||
w := bufio.NewWriter(buf)
|
||||
t.writeTo(w)
|
||||
w.Flush()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
type syncGroupRequestGroupAssignmentV0 struct {
|
||||
// MemberID assigned by the group coordinator
|
||||
MemberID string
|
||||
|
||||
// MemberAssignments holds client encoded assignments
|
||||
//
|
||||
// See consumer groups section of https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol
|
||||
MemberAssignments []byte
|
||||
}
|
||||
|
||||
func (t syncGroupRequestGroupAssignmentV0) size() int32 {
|
||||
return sizeofString(t.MemberID) +
|
||||
sizeofBytes(t.MemberAssignments)
|
||||
}
|
||||
|
||||
func (t syncGroupRequestGroupAssignmentV0) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.MemberID)
|
||||
writeBytes(w, t.MemberAssignments)
|
||||
}
|
||||
|
||||
type syncGroupRequestV0 struct {
|
||||
// GroupID holds the unique group identifier
|
||||
GroupID string
|
||||
|
||||
// GenerationID holds the generation of the group.
|
||||
GenerationID int32
|
||||
|
||||
// MemberID assigned by the group coordinator
|
||||
MemberID string
|
||||
|
||||
GroupAssignments []syncGroupRequestGroupAssignmentV0
|
||||
}
|
||||
|
||||
func (t syncGroupRequestV0) size() int32 {
|
||||
return sizeofString(t.GroupID) +
|
||||
sizeofInt32(t.GenerationID) +
|
||||
sizeofString(t.MemberID) +
|
||||
sizeofArray(len(t.GroupAssignments), func(i int) int32 { return t.GroupAssignments[i].size() })
|
||||
}
|
||||
|
||||
func (t syncGroupRequestV0) writeTo(w *bufio.Writer) {
|
||||
writeString(w, t.GroupID)
|
||||
writeInt32(w, t.GenerationID)
|
||||
writeString(w, t.MemberID)
|
||||
writeArray(w, len(t.GroupAssignments), func(i int) { t.GroupAssignments[i].writeTo(w) })
|
||||
}
|
||||
|
||||
type syncGroupResponseV0 struct {
|
||||
// ErrorCode holds response error code
|
||||
ErrorCode int16
|
||||
|
||||
// MemberAssignments holds client encoded assignments
|
||||
//
|
||||
// See consumer groups section of https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol
|
||||
MemberAssignments []byte
|
||||
}
|
||||
|
||||
func (t syncGroupResponseV0) size() int32 {
|
||||
return sizeofInt16(t.ErrorCode) +
|
||||
sizeofBytes(t.MemberAssignments)
|
||||
}
|
||||
|
||||
func (t syncGroupResponseV0) writeTo(w *bufio.Writer) {
|
||||
writeInt16(w, t.ErrorCode)
|
||||
writeBytes(w, t.MemberAssignments)
|
||||
}
|
||||
|
||||
func (t *syncGroupResponseV0) readFrom(r *bufio.Reader, sz int) (remain int, err error) {
|
||||
if remain, err = readInt16(r, sz, &t.ErrorCode); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = readBytes(r, remain, &t.MemberAssignments); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
-55
@@ -1,55 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"math"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
maxTimeout = time.Duration(math.MaxInt32) * time.Millisecond
|
||||
minTimeout = time.Duration(math.MinInt32) * time.Millisecond
|
||||
defaultRTT = 1 * time.Second
|
||||
)
|
||||
|
||||
func timestamp(t time.Time) int64 {
|
||||
if t.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return t.UnixNano() / int64(time.Millisecond)
|
||||
}
|
||||
|
||||
func timestampToTime(t int64) time.Time {
|
||||
return time.Unix(t/1000, (t%1000)*int64(time.Millisecond))
|
||||
}
|
||||
|
||||
func duration(ms int32) time.Duration {
|
||||
return time.Duration(ms) * time.Millisecond
|
||||
}
|
||||
|
||||
func milliseconds(d time.Duration) int32 {
|
||||
switch {
|
||||
case d > maxTimeout:
|
||||
d = maxTimeout
|
||||
case d < minTimeout:
|
||||
d = minTimeout
|
||||
}
|
||||
return int32(d / time.Millisecond)
|
||||
}
|
||||
|
||||
func deadlineToTimeout(deadline time.Time, now time.Time) time.Duration {
|
||||
if deadline.IsZero() {
|
||||
return maxTimeout
|
||||
}
|
||||
return deadline.Sub(now)
|
||||
}
|
||||
|
||||
func adjustDeadlineForRTT(deadline time.Time, now time.Time, rtt time.Duration) time.Time {
|
||||
if !deadline.IsZero() {
|
||||
timeout := deadline.Sub(now)
|
||||
if timeout < rtt {
|
||||
rtt = timeout / 4
|
||||
}
|
||||
deadline = deadline.Add(-rtt)
|
||||
}
|
||||
return deadline
|
||||
}
|
||||
-288
@@ -1,288 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
type writable interface {
|
||||
writeTo(*bufio.Writer)
|
||||
}
|
||||
|
||||
func writeInt8(w *bufio.Writer, i int8) {
|
||||
w.WriteByte(byte(i))
|
||||
}
|
||||
|
||||
func writeInt16(w *bufio.Writer, i int16) {
|
||||
var b [2]byte
|
||||
binary.BigEndian.PutUint16(b[:], uint16(i))
|
||||
w.WriteByte(b[0])
|
||||
w.WriteByte(b[1])
|
||||
}
|
||||
|
||||
func writeInt32(w *bufio.Writer, i int32) {
|
||||
var b [4]byte
|
||||
binary.BigEndian.PutUint32(b[:], uint32(i))
|
||||
w.WriteByte(b[0])
|
||||
w.WriteByte(b[1])
|
||||
w.WriteByte(b[2])
|
||||
w.WriteByte(b[3])
|
||||
}
|
||||
|
||||
func writeInt64(w *bufio.Writer, i int64) {
|
||||
var b [8]byte
|
||||
binary.BigEndian.PutUint64(b[:], uint64(i))
|
||||
w.WriteByte(b[0])
|
||||
w.WriteByte(b[1])
|
||||
w.WriteByte(b[2])
|
||||
w.WriteByte(b[3])
|
||||
w.WriteByte(b[4])
|
||||
w.WriteByte(b[5])
|
||||
w.WriteByte(b[6])
|
||||
w.WriteByte(b[7])
|
||||
}
|
||||
|
||||
func writeString(w *bufio.Writer, s string) {
|
||||
writeInt16(w, int16(len(s)))
|
||||
w.WriteString(s)
|
||||
}
|
||||
|
||||
func writeBytes(w *bufio.Writer, b []byte) {
|
||||
n := len(b)
|
||||
if b == nil {
|
||||
n = -1
|
||||
}
|
||||
writeInt32(w, int32(n))
|
||||
w.Write(b)
|
||||
}
|
||||
|
||||
func writeBool(w *bufio.Writer, b bool) {
|
||||
v := int8(0)
|
||||
if b {
|
||||
v = 1
|
||||
}
|
||||
writeInt8(w, v)
|
||||
}
|
||||
|
||||
func writeArrayLen(w *bufio.Writer, n int) {
|
||||
writeInt32(w, int32(n))
|
||||
}
|
||||
|
||||
func writeArray(w *bufio.Writer, n int, f func(int)) {
|
||||
writeArrayLen(w, n)
|
||||
for i := 0; i != n; i++ {
|
||||
f(i)
|
||||
}
|
||||
}
|
||||
|
||||
func writeStringArray(w *bufio.Writer, a []string) {
|
||||
writeArray(w, len(a), func(i int) { writeString(w, a[i]) })
|
||||
}
|
||||
|
||||
func writeInt32Array(w *bufio.Writer, a []int32) {
|
||||
writeArray(w, len(a), func(i int) { writeInt32(w, a[i]) })
|
||||
}
|
||||
|
||||
func write(w *bufio.Writer, a interface{}) {
|
||||
switch v := a.(type) {
|
||||
case int8:
|
||||
writeInt8(w, v)
|
||||
case int16:
|
||||
writeInt16(w, v)
|
||||
case int32:
|
||||
writeInt32(w, v)
|
||||
case int64:
|
||||
writeInt64(w, v)
|
||||
case string:
|
||||
writeString(w, v)
|
||||
case []byte:
|
||||
writeBytes(w, v)
|
||||
case bool:
|
||||
writeBool(w, v)
|
||||
case writable:
|
||||
v.writeTo(w)
|
||||
default:
|
||||
panic(fmt.Sprintf("unsupported type: %T", a))
|
||||
}
|
||||
}
|
||||
|
||||
// The functions bellow are used as optimizations to avoid dynamic memory
|
||||
// allocations that occur when building the data structures representing the
|
||||
// kafka protocol requests.
|
||||
|
||||
func writeFetchRequestV2(w *bufio.Writer, correlationID int32, clientID, topic string, partition int32, offset int64, minBytes, maxBytes int, maxWait time.Duration) error {
|
||||
h := requestHeader{
|
||||
ApiKey: int16(fetchRequest),
|
||||
ApiVersion: int16(v2),
|
||||
CorrelationID: correlationID,
|
||||
ClientID: clientID,
|
||||
}
|
||||
h.Size = (h.size() - 4) +
|
||||
4 + // replica ID
|
||||
4 + // max wait time
|
||||
4 + // min bytes
|
||||
4 + // topic array length
|
||||
sizeofString(topic) +
|
||||
4 + // partition array length
|
||||
4 + // partition
|
||||
8 + // offset
|
||||
4 // max bytes
|
||||
|
||||
h.writeTo(w)
|
||||
writeInt32(w, -1) // replica ID
|
||||
writeInt32(w, milliseconds(maxWait))
|
||||
writeInt32(w, int32(minBytes))
|
||||
|
||||
// topic array
|
||||
writeArrayLen(w, 1)
|
||||
writeString(w, topic)
|
||||
|
||||
// partition array
|
||||
writeArrayLen(w, 1)
|
||||
writeInt32(w, partition)
|
||||
writeInt64(w, offset)
|
||||
writeInt32(w, int32(maxBytes))
|
||||
|
||||
return w.Flush()
|
||||
}
|
||||
|
||||
func writeListOffsetRequestV1(w *bufio.Writer, correlationID int32, clientID, topic string, partition int32, time int64) error {
|
||||
h := requestHeader{
|
||||
ApiKey: int16(listOffsetRequest),
|
||||
ApiVersion: int16(v1),
|
||||
CorrelationID: correlationID,
|
||||
ClientID: clientID,
|
||||
}
|
||||
h.Size = (h.size() - 4) +
|
||||
4 + // replica ID
|
||||
4 + // topic array length
|
||||
sizeofString(topic) + // topic
|
||||
4 + // partition array length
|
||||
4 + // partition
|
||||
8 // time
|
||||
|
||||
h.writeTo(w)
|
||||
writeInt32(w, -1) // replica ID
|
||||
|
||||
// topic array
|
||||
writeArrayLen(w, 1)
|
||||
writeString(w, topic)
|
||||
|
||||
// partition array
|
||||
writeArrayLen(w, 1)
|
||||
writeInt32(w, partition)
|
||||
writeInt64(w, time)
|
||||
|
||||
return w.Flush()
|
||||
}
|
||||
|
||||
func writeProduceRequestV2(w *bufio.Writer, codec CompressionCodec, correlationID int32, clientID, topic string, partition int32, timeout time.Duration, requiredAcks int16, msgs ...Message) error {
|
||||
var size int32
|
||||
attributes := int8(CompressionNoneCode)
|
||||
|
||||
// if compressing, replace the slice of messages with a single compressed
|
||||
// message set.
|
||||
if codec != nil {
|
||||
var err error
|
||||
if msgs, err = compress(codec, msgs...); err != nil {
|
||||
return err
|
||||
}
|
||||
attributes = codec.Code()
|
||||
}
|
||||
|
||||
for _, msg := range msgs {
|
||||
size += 8 + // offset
|
||||
4 + // message size
|
||||
4 + // crc
|
||||
1 + // magic byte
|
||||
1 + // attributes
|
||||
8 + // timestamp
|
||||
sizeofBytes(msg.Key) +
|
||||
sizeofBytes(msg.Value)
|
||||
}
|
||||
|
||||
h := requestHeader{
|
||||
ApiKey: int16(produceRequest),
|
||||
ApiVersion: int16(v2),
|
||||
CorrelationID: correlationID,
|
||||
ClientID: clientID,
|
||||
}
|
||||
h.Size = (h.size() - 4) +
|
||||
2 + // required acks
|
||||
4 + // timeout
|
||||
4 + // topic array length
|
||||
sizeofString(topic) + // topic
|
||||
4 + // partition array length
|
||||
4 + // partition
|
||||
4 + // message set size
|
||||
size
|
||||
|
||||
h.writeTo(w)
|
||||
writeInt16(w, requiredAcks) // required acks
|
||||
writeInt32(w, milliseconds(timeout))
|
||||
|
||||
// topic array
|
||||
writeArrayLen(w, 1)
|
||||
writeString(w, topic)
|
||||
|
||||
// partition array
|
||||
writeArrayLen(w, 1)
|
||||
writeInt32(w, partition)
|
||||
writeInt32(w, size)
|
||||
|
||||
for _, msg := range msgs {
|
||||
writeMessage(w, msg.Offset, attributes, msg.Time, msg.Key, msg.Value)
|
||||
}
|
||||
|
||||
return w.Flush()
|
||||
}
|
||||
|
||||
func compress(codec CompressionCodec, msgs ...Message) ([]Message, error) {
|
||||
estimatedLen := 0
|
||||
for _, msg := range msgs {
|
||||
estimatedLen += int(msgSize(msg.Key, msg.Value))
|
||||
}
|
||||
buf := &bytes.Buffer{}
|
||||
buf.Grow(estimatedLen)
|
||||
bufWriter := bufio.NewWriter(buf)
|
||||
for offset, msg := range msgs {
|
||||
writeMessage(bufWriter, int64(offset), CompressionNoneCode, msg.Time, msg.Key, msg.Value)
|
||||
}
|
||||
bufWriter.Flush()
|
||||
|
||||
compressed, err := codec.Encode(buf.Bytes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return []Message{{Value: compressed}}, nil
|
||||
}
|
||||
|
||||
const magicByte = 1 // compatible with kafka 0.10.0.0+
|
||||
|
||||
func writeMessage(w *bufio.Writer, offset int64, attributes int8, time time.Time, key, value []byte) {
|
||||
timestamp := timestamp(time)
|
||||
crc32 := crc32OfMessage(magicByte, attributes, timestamp, key, value)
|
||||
size := msgSize(key, value)
|
||||
|
||||
writeInt64(w, offset)
|
||||
writeInt32(w, size)
|
||||
writeInt32(w, int32(crc32))
|
||||
writeInt8(w, magicByte)
|
||||
writeInt8(w, attributes)
|
||||
writeInt64(w, timestamp)
|
||||
writeBytes(w, key)
|
||||
writeBytes(w, value)
|
||||
}
|
||||
|
||||
func msgSize(key, value []byte) int32 {
|
||||
return 4 + // crc
|
||||
1 + // magic byte
|
||||
1 + // attributes
|
||||
8 + // timestamp
|
||||
sizeofBytes(key) +
|
||||
sizeofBytes(value)
|
||||
}
|
||||
-734
@@ -1,734 +0,0 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"math/rand"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The Writer type provides the implementation of a producer of kafka messages
|
||||
// that automatically distributes messages across partitions of a single topic
|
||||
// using a configurable balancing policy.
|
||||
//
|
||||
// Instances of Writer are safe to use concurrently from multiple goroutines.
|
||||
type Writer struct {
|
||||
config WriterConfig
|
||||
|
||||
mutex sync.RWMutex
|
||||
closed bool
|
||||
|
||||
join sync.WaitGroup
|
||||
msgs chan writerMessage
|
||||
done chan struct{}
|
||||
|
||||
// writer stats are all made of atomic values, no need for synchronization.
|
||||
// Use a pointer to ensure 64-bit alignment of the values.
|
||||
stats *writerStats
|
||||
}
|
||||
|
||||
// WriterConfig is a configuration type used to create new instances of Writer.
|
||||
type WriterConfig struct {
|
||||
// The list of brokers used to discover the partitions available on the
|
||||
// kafka cluster.
|
||||
//
|
||||
// This field is required, attempting to create a writer with an empty list
|
||||
// of brokers will panic.
|
||||
Brokers []string
|
||||
|
||||
// The topic that the writer will produce messages to.
|
||||
//
|
||||
// This field is required, attempting to create a writer with an empty topic
|
||||
// will panic.
|
||||
Topic string
|
||||
|
||||
// The dialer used by the writer to establish connections to the kafka
|
||||
// cluster.
|
||||
//
|
||||
// If nil, the default dialer is used instead.
|
||||
Dialer *Dialer
|
||||
|
||||
// The balancer used to distribute messages across partitions.
|
||||
//
|
||||
// The default is to use a round-robin distribution.
|
||||
Balancer Balancer
|
||||
|
||||
// Limit on how many attempts will be made to deliver a message.
|
||||
//
|
||||
// The default is to try at most 10 times.
|
||||
MaxAttempts int
|
||||
|
||||
// A hint on the capacity of the writer's internal message queue.
|
||||
//
|
||||
// The default is to use a queue capacity of 100 messages.
|
||||
QueueCapacity int
|
||||
|
||||
// Limit on how many messages will be buffered before being sent to a
|
||||
// partition.
|
||||
//
|
||||
// The default is to use a target batch size of 100 messages.
|
||||
BatchSize int
|
||||
|
||||
// Time limit on how often incomplete message batches will be flushed to
|
||||
// kafka.
|
||||
//
|
||||
// The default is to flush at least every second.
|
||||
BatchTimeout time.Duration
|
||||
|
||||
// Timeout for read operations performed by the Writer.
|
||||
//
|
||||
// Defaults to 10 seconds.
|
||||
ReadTimeout time.Duration
|
||||
|
||||
// Timeout for write operation performed by the Writer.
|
||||
//
|
||||
// Defaults to 10 seconds.
|
||||
WriteTimeout time.Duration
|
||||
|
||||
// This interval defines how often the list of partitions is refreshed from
|
||||
// kafka. It allows the writer to automatically handle when new partitions
|
||||
// are added to a topic.
|
||||
//
|
||||
// The default is to refresh partitions every 15 seconds.
|
||||
RebalanceInterval time.Duration
|
||||
|
||||
// Number of acknowledges from partition replicas required before receiving
|
||||
// a response to a produce request (default to -1, which means to wait for
|
||||
// all replicas).
|
||||
RequiredAcks int
|
||||
|
||||
// Setting this flag to true causes the WriteMessages method to never block.
|
||||
// It also means that errors are ignored since the caller will not receive
|
||||
// the returned value. Use this only if you don't care about guarantees of
|
||||
// whether the messages were written to kafka.
|
||||
Async bool
|
||||
|
||||
// CompressionCodec set the codec to be used to compress Kafka messages.
|
||||
// Note that messages are allowed to overwrite the compression codec individually.
|
||||
CompressionCodec
|
||||
|
||||
// If not nil, specifies a logger used to report internal changes within the
|
||||
// writer.
|
||||
Logger *log.Logger
|
||||
|
||||
// ErrorLogger is the logger used to report errors. If nil, the writer falls
|
||||
// back to using Logger instead.
|
||||
ErrorLogger *log.Logger
|
||||
|
||||
newPartitionWriter func(partition int, config WriterConfig, stats *writerStats) partitionWriter
|
||||
}
|
||||
|
||||
// WriterStats is a data structure returned by a call to Writer.Stats that
|
||||
// exposes details about the behavior of the writer.
|
||||
type WriterStats struct {
|
||||
Dials int64 `metric:"kafka.writer.dial.count" type:"counter"`
|
||||
Writes int64 `metric:"kafka.writer.write.count" type:"counter"`
|
||||
Messages int64 `metric:"kafka.writer.message.count" type:"counter"`
|
||||
Bytes int64 `metric:"kafka.writer.message.bytes" type:"counter"`
|
||||
Rebalances int64 `metric:"kafka.writer.rebalance.count" type:"counter"`
|
||||
Errors int64 `metric:"kafka.writer.error.count" type:"counter"`
|
||||
|
||||
DialTime DurationStats `metric:"kafka.writer.dial.seconds"`
|
||||
WriteTime DurationStats `metric:"kafka.writer.write.seconds"`
|
||||
WaitTime DurationStats `metric:"kafka.writer.wait.seconds"`
|
||||
Retries SummaryStats `metric:"kafka.writer.retries.count"`
|
||||
BatchSize SummaryStats `metric:"kafka.writer.batch.size"`
|
||||
|
||||
MaxAttempts int64 `metric:"kafka.writer.attempts.max" type:"gauge"`
|
||||
MaxBatchSize int64 `metric:"kafka.writer.batch.max" type:"gauge"`
|
||||
BatchTimeout time.Duration `metric:"kafka.writer.batch.timeout" type:"gauge"`
|
||||
ReadTimeout time.Duration `metric:"kafka.writer.read.timeout" type:"gauge"`
|
||||
WriteTimeout time.Duration `metric:"kafka.writer.write.timeout" type:"gauge"`
|
||||
RebalanceInterval time.Duration `metric:"kafka.writer.rebalance.interval" type:"gauge"`
|
||||
RequiredAcks int64 `metric:"kafka.writer.acks.required" type:"gauge"`
|
||||
Async bool `metric:"kafka.writer.async" type:"gauge"`
|
||||
QueueLength int64 `metric:"kafka.writer.queue.length" type:"gauge"`
|
||||
QueueCapacity int64 `metric:"kafka.writer.queue.capacity" type:"gauge"`
|
||||
|
||||
ClientID string `tag:"client_id"`
|
||||
Topic string `tag:"topic"`
|
||||
}
|
||||
|
||||
// writerStats is a struct that contains statistics on a writer.
|
||||
//
|
||||
// Since atomic is used to mutate the statistics the values must be 64-bit aligned.
|
||||
// This is easily accomplished by always allocating this struct directly, (i.e. using a pointer to the struct).
|
||||
// See https://golang.org/pkg/sync/atomic/#pkg-note-BUG
|
||||
type writerStats struct {
|
||||
dials counter
|
||||
writes counter
|
||||
messages counter
|
||||
bytes counter
|
||||
rebalances counter
|
||||
errors counter
|
||||
dialTime summary
|
||||
writeTime summary
|
||||
waitTime summary
|
||||
retries summary
|
||||
batchSize summary
|
||||
}
|
||||
|
||||
// NewWriter creates and returns a new Writer configured with config.
|
||||
func NewWriter(config WriterConfig) *Writer {
|
||||
if len(config.Brokers) == 0 {
|
||||
panic("cannot create a kafka writer with an empty list of brokers")
|
||||
}
|
||||
|
||||
if len(config.Topic) == 0 {
|
||||
panic("cannot create a kafka writer with an empty topic")
|
||||
}
|
||||
|
||||
if config.Dialer == nil {
|
||||
config.Dialer = DefaultDialer
|
||||
}
|
||||
|
||||
if config.Balancer == nil {
|
||||
config.Balancer = &RoundRobin{}
|
||||
}
|
||||
|
||||
if config.newPartitionWriter == nil {
|
||||
config.newPartitionWriter = func(partition int, config WriterConfig, stats *writerStats) partitionWriter {
|
||||
return newWriter(partition, config, stats)
|
||||
}
|
||||
}
|
||||
|
||||
if config.MaxAttempts == 0 {
|
||||
config.MaxAttempts = 10
|
||||
}
|
||||
|
||||
if config.QueueCapacity == 0 {
|
||||
config.QueueCapacity = 100
|
||||
}
|
||||
|
||||
if config.BatchSize == 0 {
|
||||
config.BatchSize = 100
|
||||
}
|
||||
|
||||
if config.BatchTimeout == 0 {
|
||||
config.BatchTimeout = 1 * time.Second
|
||||
}
|
||||
|
||||
if config.ReadTimeout == 0 {
|
||||
config.ReadTimeout = 10 * time.Second
|
||||
}
|
||||
|
||||
if config.WriteTimeout == 0 {
|
||||
config.WriteTimeout = 10 * time.Second
|
||||
}
|
||||
|
||||
if config.RebalanceInterval == 0 {
|
||||
config.RebalanceInterval = 15 * time.Second
|
||||
}
|
||||
|
||||
w := &Writer{
|
||||
config: config,
|
||||
msgs: make(chan writerMessage, config.QueueCapacity),
|
||||
done: make(chan struct{}),
|
||||
stats: &writerStats{
|
||||
dialTime: makeSummary(),
|
||||
writeTime: makeSummary(),
|
||||
waitTime: makeSummary(),
|
||||
retries: makeSummary(),
|
||||
},
|
||||
}
|
||||
|
||||
w.join.Add(1)
|
||||
go w.run()
|
||||
return w
|
||||
}
|
||||
|
||||
// WriteMessages writes a batch of messages to the kafka topic configured on this
|
||||
// writer.
|
||||
//
|
||||
// Unless the writer was configured to write messages asynchronously, the method
|
||||
// blocks until all messages have been written, or until the maximum number of
|
||||
// attempts was reached.
|
||||
//
|
||||
// When the method returns an error, there's no way to know yet which messages
|
||||
// have succeeded of failed.
|
||||
//
|
||||
// The context passed as first argument may also be used to asynchronously
|
||||
// cancel the operation. Note that in this case there are no guarantees made on
|
||||
// whether messages were written to kafka. The program should assume that the
|
||||
// whole batch failed and re-write the messages later (which could then cause
|
||||
// duplicates).
|
||||
func (w *Writer) WriteMessages(ctx context.Context, msgs ...Message) error {
|
||||
if len(msgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var res = make(chan error, len(msgs))
|
||||
var err error
|
||||
|
||||
t0 := time.Now()
|
||||
|
||||
for attempt := 0; attempt < w.config.MaxAttempts; attempt++ {
|
||||
w.mutex.RLock()
|
||||
|
||||
if w.closed {
|
||||
w.mutex.RUnlock()
|
||||
return io.ErrClosedPipe
|
||||
}
|
||||
|
||||
for _, msg := range msgs {
|
||||
select {
|
||||
case w.msgs <- writerMessage{
|
||||
msg: msg,
|
||||
res: res,
|
||||
}:
|
||||
case <-ctx.Done():
|
||||
w.mutex.RUnlock()
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
w.mutex.RUnlock()
|
||||
|
||||
if w.config.Async {
|
||||
break
|
||||
}
|
||||
|
||||
var retry []Message
|
||||
|
||||
for i := 0; i != len(msgs); i++ {
|
||||
select {
|
||||
case e := <-res:
|
||||
if e != nil {
|
||||
if we, ok := e.(*writerError); ok {
|
||||
w.stats.retries.observe(1)
|
||||
retry, err = append(retry, we.msg), we.err
|
||||
} else {
|
||||
err = e
|
||||
}
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
if msgs = retry; len(msgs) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
timer := time.NewTimer(backoff(attempt+1, 100*time.Millisecond, 1*time.Second))
|
||||
select {
|
||||
case <-timer.C:
|
||||
// Only clear the error (so we retry the loop) if we have more retries, otherwise
|
||||
// we risk silencing the error.
|
||||
if attempt < w.config.MaxAttempts-1 {
|
||||
err = nil
|
||||
}
|
||||
case <-ctx.Done():
|
||||
err = ctx.Err()
|
||||
case <-w.done:
|
||||
err = io.ErrClosedPipe
|
||||
}
|
||||
timer.Stop()
|
||||
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
t1 := time.Now()
|
||||
w.stats.writeTime.observeDuration(t1.Sub(t0))
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Stats returns a snapshot of the writer stats since the last time the method
|
||||
// was called, or since the writer was created if it is called for the first
|
||||
// time.
|
||||
//
|
||||
// A typical use of this method is to spawn a goroutine that will periodically
|
||||
// call Stats on a kafka writer and report the metrics to a stats collection
|
||||
// system.
|
||||
func (w *Writer) Stats() WriterStats {
|
||||
return WriterStats{
|
||||
Dials: w.stats.dials.snapshot(),
|
||||
Writes: w.stats.writes.snapshot(),
|
||||
Messages: w.stats.messages.snapshot(),
|
||||
Bytes: w.stats.bytes.snapshot(),
|
||||
Rebalances: w.stats.rebalances.snapshot(),
|
||||
Errors: w.stats.errors.snapshot(),
|
||||
DialTime: w.stats.dialTime.snapshotDuration(),
|
||||
WriteTime: w.stats.writeTime.snapshotDuration(),
|
||||
WaitTime: w.stats.waitTime.snapshotDuration(),
|
||||
Retries: w.stats.retries.snapshot(),
|
||||
BatchSize: w.stats.batchSize.snapshot(),
|
||||
MaxAttempts: int64(w.config.MaxAttempts),
|
||||
MaxBatchSize: int64(w.config.BatchSize),
|
||||
BatchTimeout: w.config.BatchTimeout,
|
||||
ReadTimeout: w.config.ReadTimeout,
|
||||
WriteTimeout: w.config.WriteTimeout,
|
||||
RebalanceInterval: w.config.RebalanceInterval,
|
||||
RequiredAcks: int64(w.config.RequiredAcks),
|
||||
Async: w.config.Async,
|
||||
QueueLength: int64(len(w.msgs)),
|
||||
QueueCapacity: int64(cap(w.msgs)),
|
||||
ClientID: w.config.Dialer.ClientID,
|
||||
Topic: w.config.Topic,
|
||||
}
|
||||
}
|
||||
|
||||
// Close flushes all buffered messages and closes the writer. The call to Close
|
||||
// aborts any concurrent calls to WriteMessages, which then return with the
|
||||
// io.ErrClosedPipe error.
|
||||
func (w *Writer) Close() (err error) {
|
||||
w.mutex.Lock()
|
||||
|
||||
if !w.closed {
|
||||
w.closed = true
|
||||
close(w.msgs)
|
||||
close(w.done)
|
||||
}
|
||||
|
||||
w.mutex.Unlock()
|
||||
w.join.Wait()
|
||||
return
|
||||
}
|
||||
|
||||
func (w *Writer) run() {
|
||||
defer w.join.Done()
|
||||
|
||||
ticker := time.NewTicker(w.config.RebalanceInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
var rebalance = true
|
||||
var writers = make(map[int]partitionWriter)
|
||||
var partitions []int
|
||||
var err error
|
||||
|
||||
for {
|
||||
if rebalance {
|
||||
w.stats.rebalances.observe(1)
|
||||
rebalance = false
|
||||
|
||||
var newPartitions []int
|
||||
var oldPartitions = partitions
|
||||
|
||||
if newPartitions, err = w.partitions(); err == nil {
|
||||
for _, partition := range diffp(oldPartitions, newPartitions) {
|
||||
w.close(writers[partition])
|
||||
delete(writers, partition)
|
||||
}
|
||||
|
||||
for _, partition := range diffp(newPartitions, oldPartitions) {
|
||||
writers[partition] = w.open(partition)
|
||||
}
|
||||
|
||||
partitions = newPartitions
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case wm, ok := <-w.msgs:
|
||||
if !ok {
|
||||
for _, writer := range writers {
|
||||
w.close(writer)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if len(partitions) != 0 {
|
||||
selectedPartition := w.config.Balancer.Balance(wm.msg, partitions...)
|
||||
writers[selectedPartition].messages() <- wm
|
||||
} else {
|
||||
// No partitions were found because the topic doesn't exist.
|
||||
if err == nil {
|
||||
err = fmt.Errorf("failed to find any partitions for topic %s", w.config.Topic)
|
||||
}
|
||||
|
||||
wm.res <- &writerError{msg: wm.msg, err: err}
|
||||
}
|
||||
|
||||
case <-ticker.C:
|
||||
rebalance = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Writer) partitions() (partitions []int, err error) {
|
||||
for _, broker := range shuffledStrings(w.config.Brokers) {
|
||||
var conn *Conn
|
||||
var plist []Partition
|
||||
|
||||
if conn, err = w.config.Dialer.Dial("tcp", broker); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
conn.SetReadDeadline(time.Now().Add(w.config.ReadTimeout))
|
||||
plist, err = conn.ReadPartitions(w.config.Topic)
|
||||
conn.Close()
|
||||
|
||||
if err == nil {
|
||||
partitions = make([]int, len(plist))
|
||||
for i, p := range plist {
|
||||
partitions[i] = p.ID
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
sort.Ints(partitions)
|
||||
return
|
||||
}
|
||||
|
||||
func (w *Writer) open(partition int) partitionWriter {
|
||||
return w.config.newPartitionWriter(partition, w.config, w.stats)
|
||||
}
|
||||
|
||||
func (w *Writer) close(writer partitionWriter) {
|
||||
w.join.Add(1)
|
||||
go func() {
|
||||
writer.close()
|
||||
w.join.Done()
|
||||
}()
|
||||
}
|
||||
|
||||
func diffp(new []int, old []int) (diff []int) {
|
||||
for _, p := range new {
|
||||
if i := sort.SearchInts(old, p); i == len(old) || old[i] != p {
|
||||
diff = append(diff, p)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type partitionWriter interface {
|
||||
messages() chan<- writerMessage
|
||||
close()
|
||||
}
|
||||
|
||||
type writer struct {
|
||||
brokers []string
|
||||
topic string
|
||||
partition int
|
||||
requiredAcks int
|
||||
batchSize int
|
||||
batchTimeout time.Duration
|
||||
writeTimeout time.Duration
|
||||
dialer *Dialer
|
||||
msgs chan writerMessage
|
||||
join sync.WaitGroup
|
||||
stats *writerStats
|
||||
codec CompressionCodec
|
||||
logger *log.Logger
|
||||
errorLogger *log.Logger
|
||||
}
|
||||
|
||||
func newWriter(partition int, config WriterConfig, stats *writerStats) *writer {
|
||||
w := &writer{
|
||||
brokers: config.Brokers,
|
||||
topic: config.Topic,
|
||||
partition: partition,
|
||||
requiredAcks: config.RequiredAcks,
|
||||
batchSize: config.BatchSize,
|
||||
batchTimeout: config.BatchTimeout,
|
||||
writeTimeout: config.WriteTimeout,
|
||||
dialer: config.Dialer,
|
||||
msgs: make(chan writerMessage, config.QueueCapacity),
|
||||
stats: stats,
|
||||
codec: config.CompressionCodec,
|
||||
logger: config.Logger,
|
||||
errorLogger: config.ErrorLogger,
|
||||
}
|
||||
w.join.Add(1)
|
||||
go w.run()
|
||||
return w
|
||||
}
|
||||
|
||||
func (w *writer) close() {
|
||||
close(w.msgs)
|
||||
w.join.Wait()
|
||||
}
|
||||
|
||||
func (w *writer) messages() chan<- writerMessage {
|
||||
return w.msgs
|
||||
}
|
||||
|
||||
func (w *writer) withLogger(do func(*log.Logger)) {
|
||||
if w.logger != nil {
|
||||
do(w.logger)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *writer) withErrorLogger(do func(*log.Logger)) {
|
||||
if w.errorLogger != nil {
|
||||
do(w.errorLogger)
|
||||
} else {
|
||||
w.withLogger(do)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *writer) run() {
|
||||
defer w.join.Done()
|
||||
|
||||
ticker := time.NewTicker(w.batchTimeout / 10)
|
||||
defer ticker.Stop()
|
||||
|
||||
var conn *Conn
|
||||
var done bool
|
||||
var batch = make([]Message, 0, w.batchSize)
|
||||
var resch = make([](chan<- error), 0, w.batchSize)
|
||||
var lastFlushAt = time.Now()
|
||||
|
||||
defer func() {
|
||||
if conn != nil {
|
||||
conn.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
for !done {
|
||||
var mustFlush bool
|
||||
|
||||
select {
|
||||
case wm, ok := <-w.msgs:
|
||||
if !ok {
|
||||
done, mustFlush = true, true
|
||||
} else {
|
||||
batch = append(batch, wm.msg)
|
||||
resch = append(resch, wm.res)
|
||||
mustFlush = len(batch) >= w.batchSize
|
||||
}
|
||||
|
||||
case now := <-ticker.C:
|
||||
mustFlush = now.Sub(lastFlushAt) > w.batchTimeout
|
||||
}
|
||||
|
||||
if mustFlush {
|
||||
lastFlushAt = time.Now()
|
||||
|
||||
if len(batch) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
var err error
|
||||
if conn, err = w.write(conn, batch, resch); err != nil {
|
||||
if conn != nil {
|
||||
conn.Close()
|
||||
conn = nil
|
||||
}
|
||||
}
|
||||
|
||||
for i := range batch {
|
||||
batch[i] = Message{}
|
||||
}
|
||||
|
||||
for i := range resch {
|
||||
resch[i] = nil
|
||||
}
|
||||
|
||||
batch = batch[:0]
|
||||
resch = resch[:0]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *writer) dial() (conn *Conn, err error) {
|
||||
for _, broker := range shuffledStrings(w.brokers) {
|
||||
t0 := time.Now()
|
||||
if conn, err = w.dialer.DialLeader(context.Background(), "tcp", broker, w.topic, w.partition); err == nil {
|
||||
t1 := time.Now()
|
||||
w.stats.dials.observe(1)
|
||||
w.stats.dialTime.observeDuration(t1.Sub(t0))
|
||||
conn.SetRequiredAcks(w.requiredAcks)
|
||||
break
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (w *writer) write(conn *Conn, batch []Message, resch [](chan<- error)) (ret *Conn, err error) {
|
||||
w.stats.writes.observe(1)
|
||||
if conn == nil {
|
||||
if conn, err = w.dial(); err != nil {
|
||||
w.stats.errors.observe(1)
|
||||
w.withErrorLogger(func(logger *log.Logger) {
|
||||
logger.Printf("error dialing kafka brokers for topic %s (partition %d): %s", w.topic, w.partition, err)
|
||||
})
|
||||
for i, res := range resch {
|
||||
res <- &writerError{msg: batch[i], err: err}
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
t0 := time.Now()
|
||||
conn.SetWriteDeadline(time.Now().Add(w.writeTimeout))
|
||||
|
||||
if _, err = conn.WriteCompressedMessages(w.codec, batch...); err != nil {
|
||||
w.stats.errors.observe(1)
|
||||
w.withErrorLogger(func(logger *log.Logger) {
|
||||
logger.Printf("error writing messages to %s (partition %d): %s", w.topic, w.partition, err)
|
||||
})
|
||||
for i, res := range resch {
|
||||
res <- &writerError{msg: batch[i], err: err}
|
||||
}
|
||||
} else {
|
||||
for _, m := range batch {
|
||||
w.stats.messages.observe(1)
|
||||
w.stats.bytes.observe(int64(len(m.Key) + len(m.Value)))
|
||||
}
|
||||
for _, res := range resch {
|
||||
res <- nil
|
||||
}
|
||||
}
|
||||
|
||||
t1 := time.Now()
|
||||
w.stats.waitTime.observeDuration(t1.Sub(t0))
|
||||
w.stats.batchSize.observe(int64(len(batch)))
|
||||
|
||||
ret = conn
|
||||
return
|
||||
}
|
||||
|
||||
type writerMessage struct {
|
||||
msg Message
|
||||
res chan<- error
|
||||
}
|
||||
|
||||
type writerError struct {
|
||||
msg Message
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *writerError) Cause() error {
|
||||
return e.err
|
||||
}
|
||||
|
||||
func (e *writerError) Error() string {
|
||||
return e.err.Error()
|
||||
}
|
||||
|
||||
func (e *writerError) Temporary() bool {
|
||||
return isTemporary(e.err)
|
||||
}
|
||||
|
||||
func (e *writerError) Timeout() bool {
|
||||
return isTimeout(e.err)
|
||||
}
|
||||
|
||||
func shuffledStrings(list []string) []string {
|
||||
shuffledList := make([]string, len(list))
|
||||
copy(shuffledList, list)
|
||||
|
||||
shufflerMutex.Lock()
|
||||
|
||||
for i := range shuffledList {
|
||||
j := shuffler.Intn(i + 1)
|
||||
shuffledList[i], shuffledList[j] = shuffledList[j], shuffledList[i]
|
||||
}
|
||||
|
||||
shufflerMutex.Unlock()
|
||||
return shuffledList
|
||||
}
|
||||
|
||||
var (
|
||||
shufflerMutex = sync.Mutex{}
|
||||
shuffler = rand.New(rand.NewSource(time.Now().Unix()))
|
||||
)
|
||||
Reference in New Issue
Block a user