mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-12-19 13:37:49 +08:00
dc784bf217
* listing files to convert to parquet * write parquet files * save logs into parquet files * pass by value * compact logs into parquet format * can skip existing files * refactor * refactor * fix compilation * when no partition found * refactor * add untested parquet file read * rename package * refactor * rename files * remove unused * add merged log read func * parquet wants to know the file size * rewind by time * pass in stop ts * add stop ts * adjust log * minor * adjust log * skip .parquet files when reading message logs * skip non message files * Update subscriber_record.go * send messages * skip message data with only ts * skip non log files * update parquet-go package * ensure a valid record type * add new field to a record type * Update read_parquet_to_log.go * fix parquet file name generation * separating reading parquet and logs * add key field * add skipped logs * use in memory cache * refactor * refactor * refactor * refactor, and change compact log * refactor * rename * refactor * fix format * prefix v to version directory
62 lines
2.2 KiB
Go
62 lines
2.2 KiB
Go
package sub_client
|
|
|
|
import (
|
|
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
|
|
"google.golang.org/grpc"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
type SubscriberConfiguration struct {
|
|
ClientId string
|
|
ConsumerGroup string
|
|
ConsumerGroupInstanceId string
|
|
GrpcDialOption grpc.DialOption
|
|
MaxPartitionCount int32 // how many partitions to process concurrently
|
|
PerPartitionConcurrency int32 // how many messages to process concurrently per partition
|
|
}
|
|
|
|
type ContentConfiguration struct {
|
|
Topic topic.Topic
|
|
Filter string
|
|
StartTime time.Time
|
|
StopTime time.Time
|
|
}
|
|
|
|
type OnEachMessageFunc func(key, value []byte) (err error)
|
|
type OnCompletionFunc func()
|
|
|
|
type TopicSubscriber struct {
|
|
SubscriberConfig *SubscriberConfiguration
|
|
ContentConfig *ContentConfiguration
|
|
brokerPartitionAssignmentChan chan *mq_pb.SubscriberToSubCoordinatorResponse
|
|
brokerPartitionAssignmentAckChan chan *mq_pb.SubscriberToSubCoordinatorRequest
|
|
OnEachMessageFunc OnEachMessageFunc
|
|
OnCompletionFunc OnCompletionFunc
|
|
bootstrapBrokers []string
|
|
waitForMoreMessage bool
|
|
activeProcessors map[topic.Partition]*ProcessorState
|
|
activeProcessorsLock sync.Mutex
|
|
}
|
|
|
|
func NewTopicSubscriber(bootstrapBrokers []string, subscriber *SubscriberConfiguration, content *ContentConfiguration) *TopicSubscriber {
|
|
return &TopicSubscriber{
|
|
SubscriberConfig: subscriber,
|
|
ContentConfig: content,
|
|
brokerPartitionAssignmentChan: make(chan *mq_pb.SubscriberToSubCoordinatorResponse, 1024),
|
|
brokerPartitionAssignmentAckChan: make(chan *mq_pb.SubscriberToSubCoordinatorRequest, 1024),
|
|
bootstrapBrokers: bootstrapBrokers,
|
|
waitForMoreMessage: true,
|
|
activeProcessors: make(map[topic.Partition]*ProcessorState),
|
|
}
|
|
}
|
|
|
|
func (sub *TopicSubscriber) SetEachMessageFunc(onEachMessageFn OnEachMessageFunc) {
|
|
sub.OnEachMessageFunc = onEachMessageFn
|
|
}
|
|
|
|
func (sub *TopicSubscriber) SetCompletionFunc(onCompletionFn OnCompletionFunc) {
|
|
sub.OnCompletionFunc = onCompletionFn
|
|
}
|