mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-12-19 13:37:49 +08:00
dc784bf217
* listing files to convert to parquet * write parquet files * save logs into parquet files * pass by value * compact logs into parquet format * can skip existing files * refactor * refactor * fix compilation * when no partition found * refactor * add untested parquet file read * rename package * refactor * rename files * remove unused * add merged log read func * parquet wants to know the file size * rewind by time * pass in stop ts * add stop ts * adjust log * minor * adjust log * skip .parquet files when reading message logs * skip non message files * Update subscriber_record.go * send messages * skip message data with only ts * skip non log files * update parquet-go package * ensure a valid record type * add new field to a record type * Update read_parquet_to_log.go * fix parquet file name generation * separating reading parquet and logs * add key field * add skipped logs * use in memory cache * refactor * refactor * refactor * refactor, and change compact log * refactor * rename * refactor * fix format * prefix v to version directory
70 lines
1.7 KiB
Go
70 lines
1.7 KiB
Go
package topic
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
|
|
jsonpb "google.golang.org/protobuf/encoding/protojson"
|
|
)
|
|
|
|
type Topic struct {
|
|
Namespace string
|
|
Name string
|
|
}
|
|
|
|
func NewTopic(namespace string, name string) Topic {
|
|
return Topic{
|
|
Namespace: namespace,
|
|
Name: name,
|
|
}
|
|
}
|
|
func FromPbTopic(topic *mq_pb.Topic) Topic {
|
|
return Topic{
|
|
Namespace: topic.Namespace,
|
|
Name: topic.Name,
|
|
}
|
|
}
|
|
|
|
func (t Topic) ToPbTopic() *mq_pb.Topic {
|
|
return &mq_pb.Topic{
|
|
Namespace: t.Namespace,
|
|
Name: t.Name,
|
|
}
|
|
}
|
|
|
|
func (t Topic) String() string {
|
|
return fmt.Sprintf("%s.%s", t.Namespace, t.Name)
|
|
}
|
|
|
|
func (t Topic) Dir() string {
|
|
return fmt.Sprintf("%s/%s/%s", filer.TopicsDir, t.Namespace, t.Name)
|
|
}
|
|
|
|
func (t Topic) ReadConfFile(client filer_pb.SeaweedFilerClient) (*mq_pb.ConfigureTopicResponse, error) {
|
|
data, err := filer.ReadInsideFiler(client, t.Dir(), filer.TopicConfFile)
|
|
if errors.Is(err, filer_pb.ErrNotFound) {
|
|
return nil, err
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read topic.conf of %v: %v", t, err)
|
|
}
|
|
// parse into filer conf object
|
|
conf := &mq_pb.ConfigureTopicResponse{}
|
|
if err = jsonpb.Unmarshal(data, conf); err != nil {
|
|
return nil, fmt.Errorf("unmarshal topic %v conf: %v", t, err)
|
|
}
|
|
return conf, nil
|
|
}
|
|
|
|
func (t Topic) WriteConfFile(client filer_pb.SeaweedFilerClient, conf *mq_pb.ConfigureTopicResponse) error {
|
|
var buf bytes.Buffer
|
|
filer.ProtoToText(&buf, conf)
|
|
if err := filer.SaveInsideFiler(client, t.Dir(), filer.TopicConfFile, buf.Bytes()); err != nil {
|
|
return fmt.Errorf("save topic %v conf: %v", t, err)
|
|
}
|
|
return nil
|
|
}
|