add dep management
This commit is contained in:
fengzhiqiang
+253
@@ -0,0 +1,253 @@
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Message is a data structure representing kafka messages.
|
||||
type Message struct {
|
||||
// Topic is reads only and MUST NOT be set when writing messages
|
||||
Topic string
|
||||
|
||||
// Partition is reads only and MUST NOT be set when writing messages
|
||||
Partition int
|
||||
Offset int64
|
||||
Key []byte
|
||||
Value []byte
|
||||
|
||||
// If not set at the creation, Time will be automatically set when
|
||||
// writing the message.
|
||||
Time time.Time
|
||||
}
|
||||
|
||||
func (msg Message) item() messageSetItem {
|
||||
item := messageSetItem{
|
||||
Offset: msg.Offset,
|
||||
Message: msg.message(),
|
||||
}
|
||||
item.MessageSize = item.Message.size()
|
||||
return item
|
||||
}
|
||||
|
||||
func (msg Message) message() message {
|
||||
m := message{
|
||||
MagicByte: 1,
|
||||
Key: msg.Key,
|
||||
Value: msg.Value,
|
||||
Timestamp: timestamp(msg.Time),
|
||||
}
|
||||
m.CRC = m.crc32()
|
||||
return m
|
||||
}
|
||||
|
||||
type message struct {
|
||||
CRC int32
|
||||
MagicByte int8
|
||||
Attributes int8
|
||||
Timestamp int64
|
||||
Key []byte
|
||||
Value []byte
|
||||
}
|
||||
|
||||
func (m message) crc32() int32 {
|
||||
return int32(crc32OfMessage(m.MagicByte, m.Attributes, m.Timestamp, m.Key, m.Value))
|
||||
}
|
||||
|
||||
func (m message) size() int32 {
|
||||
size := 4 + 1 + 1 + sizeofBytes(m.Key) + sizeofBytes(m.Value)
|
||||
if m.MagicByte != 0 {
|
||||
size += 8 // Timestamp
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
func (m message) writeTo(w *bufio.Writer) {
|
||||
writeInt32(w, m.CRC)
|
||||
writeInt8(w, m.MagicByte)
|
||||
writeInt8(w, m.Attributes)
|
||||
if m.MagicByte != 0 {
|
||||
writeInt64(w, m.Timestamp)
|
||||
}
|
||||
writeBytes(w, m.Key)
|
||||
writeBytes(w, m.Value)
|
||||
}
|
||||
|
||||
type messageSetItem struct {
|
||||
Offset int64
|
||||
MessageSize int32
|
||||
Message message
|
||||
}
|
||||
|
||||
func (m messageSetItem) size() int32 {
|
||||
return 8 + 4 + m.Message.size()
|
||||
}
|
||||
|
||||
func (m messageSetItem) writeTo(w *bufio.Writer) {
|
||||
writeInt64(w, m.Offset)
|
||||
writeInt32(w, m.MessageSize)
|
||||
m.Message.writeTo(w)
|
||||
}
|
||||
|
||||
type messageSet []messageSetItem
|
||||
|
||||
func (s messageSet) size() (size int32) {
|
||||
for _, m := range s {
|
||||
size += m.size()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s messageSet) writeTo(w *bufio.Writer) {
|
||||
for _, m := range s {
|
||||
m.writeTo(w)
|
||||
}
|
||||
}
|
||||
|
||||
type messageSetReader struct {
|
||||
*readerStack
|
||||
}
|
||||
|
||||
type readerStack struct {
|
||||
reader *bufio.Reader
|
||||
remain int
|
||||
base int64
|
||||
parent *readerStack
|
||||
}
|
||||
|
||||
func newMessageSetReader(reader *bufio.Reader, remain int) *messageSetReader {
|
||||
return &messageSetReader{&readerStack{
|
||||
reader: reader,
|
||||
remain: remain,
|
||||
}}
|
||||
}
|
||||
|
||||
func (r *messageSetReader) readMessage(min int64,
|
||||
key func(*bufio.Reader, int, int) (int, error),
|
||||
val func(*bufio.Reader, int, int) (int, error),
|
||||
) (offset int64, timestamp int64, err error) {
|
||||
for r.readerStack != nil {
|
||||
if r.remain == 0 {
|
||||
r.readerStack = r.parent
|
||||
continue
|
||||
}
|
||||
|
||||
var attributes int8
|
||||
if offset, attributes, timestamp, r.remain, err = readMessageHeader(r.reader, r.remain); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// if the message is compressed, decompress it and push a new reader
|
||||
// onto the stack.
|
||||
code := attributes & compressionCodecMask
|
||||
if code != 0 {
|
||||
var codec CompressionCodec
|
||||
if codec, err = resolveCodec(attributes); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// discard next four bytes...will be -1 to indicate null key
|
||||
if r.remain, err = discardN(r.reader, r.remain, 4); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// read and decompress the contained message set.
|
||||
var decompressed []byte
|
||||
if r.remain, err = readBytesWith(r.reader, r.remain, func(r *bufio.Reader, sz, n int) (remain int, err error) {
|
||||
var value []byte
|
||||
if value, remain, err = readNewBytes(r, sz, n); err != nil {
|
||||
return
|
||||
}
|
||||
decompressed, err = codec.Decode(value)
|
||||
return
|
||||
}); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// the compressed message's offset will be equal to the offset of
|
||||
// the last message in the set. within the compressed set, the
|
||||
// offsets will be relative, so we have to scan through them to
|
||||
// get the base offset. for example, if there are four compressed
|
||||
// messages at offsets 10-13, then the container message will have
|
||||
// offset 13 and the contained messages will be 0,1,2,3. the base
|
||||
// offset for the container, then is 13-3=10.
|
||||
if offset, err = extractOffset(offset, decompressed); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
r.readerStack = &readerStack{
|
||||
reader: bufio.NewReader(bytes.NewReader(decompressed)),
|
||||
remain: len(decompressed),
|
||||
base: offset,
|
||||
parent: r.readerStack,
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// adjust the offset in case we're reading compressed messages. the
|
||||
// base will be zero otherwise.
|
||||
offset += r.base
|
||||
|
||||
// When the messages are compressed kafka may return messages at an
|
||||
// earlier offset than the one that was requested, it's the client's
|
||||
// responsibility to ignore those.
|
||||
if offset < min {
|
||||
if r.remain, err = discardBytes(r.reader, r.remain); err != nil {
|
||||
return
|
||||
}
|
||||
if r.remain, err = discardBytes(r.reader, r.remain); err != nil {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if r.remain, err = readBytesWith(r.reader, r.remain, key); err != nil {
|
||||
return
|
||||
}
|
||||
r.remain, err = readBytesWith(r.reader, r.remain, val)
|
||||
return
|
||||
}
|
||||
|
||||
err = errShortRead
|
||||
return
|
||||
}
|
||||
|
||||
func (r *messageSetReader) remaining() (remain int) {
|
||||
for s := r.readerStack; s != nil; s = s.parent {
|
||||
remain += s.remain
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (r *messageSetReader) discard() (err error) {
|
||||
if r.readerStack == nil {
|
||||
return
|
||||
}
|
||||
// rewind up to the top-most reader b/c it's the only one that's doing
|
||||
// actual i/o. the rest are byte buffers that have been pushed on the stack
|
||||
// while reading compressed message sets.
|
||||
for r.parent != nil {
|
||||
r.readerStack = r.parent
|
||||
}
|
||||
r.remain, err = discardN(r.reader, r.remain, r.remain)
|
||||
return
|
||||
}
|
||||
|
||||
func extractOffset(base int64, msgSet []byte) (offset int64, err error) {
|
||||
r, remain := bufio.NewReader(bytes.NewReader(msgSet)), len(msgSet)
|
||||
for remain > 0 {
|
||||
if remain, err = readInt64(r, remain, &offset); err != nil {
|
||||
return
|
||||
}
|
||||
var sz int32
|
||||
if remain, err = readInt32(r, remain, &sz); err != nil {
|
||||
return
|
||||
}
|
||||
if remain, err = discardN(r, remain, int(sz)); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
offset = base - offset
|
||||
return
|
||||
}
|
||||
Reference in New Issue
Block a user