2020-04-16 17:21:23 +08:00
|
|
|
syntax = "proto3";
|
|
|
|
|
|
|
|
package messaging_pb;
|
|
|
|
|
2024-04-13 13:27:16 +08:00
|
|
|
import "schema.proto";
|
|
|
|
|
2022-07-29 15:17:28 +08:00
|
|
|
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb";
|
2022-07-03 15:29:25 +08:00
|
|
|
option java_package = "seaweedfs.mq";
|
2023-12-12 04:05:54 +08:00
|
|
|
option java_outer_classname = "MessageQueueProto";
|
2020-04-16 17:21:23 +08:00
|
|
|
|
|
|
|
//////////////////////////////////////////////////
|
|
|
|
|
|
|
|
service SeaweedMessaging {
|
|
|
|
|
2022-08-01 04:23:44 +08:00
|
|
|
// control plane
|
2022-07-11 15:20:27 +08:00
|
|
|
rpc FindBrokerLeader (FindBrokerLeaderRequest) returns (FindBrokerLeaderResponse) {
|
|
|
|
}
|
2020-05-05 17:05:28 +08:00
|
|
|
|
2023-09-15 14:49:05 +08:00
|
|
|
// control plane for balancer
|
2023-12-12 04:05:54 +08:00
|
|
|
rpc PublisherToPubBalancer (stream PublisherToPubBalancerRequest) returns (stream PublisherToPubBalancerResponse) {
|
2023-09-15 14:49:05 +08:00
|
|
|
}
|
2023-12-12 04:05:54 +08:00
|
|
|
rpc BalanceTopics (BalanceTopicsRequest) returns (BalanceTopicsResponse) {
|
2023-10-01 04:18:49 +08:00
|
|
|
}
|
2023-12-12 04:05:54 +08:00
|
|
|
|
2023-08-21 13:53:05 +08:00
|
|
|
// control plane for topic partitions
|
2023-12-12 04:05:54 +08:00
|
|
|
rpc ListTopics (ListTopicsRequest) returns (ListTopicsResponse) {
|
2023-08-21 13:53:05 +08:00
|
|
|
}
|
2023-09-27 06:17:33 +08:00
|
|
|
rpc ConfigureTopic (ConfigureTopicRequest) returns (ConfigureTopicResponse) {
|
2023-09-25 05:22:11 +08:00
|
|
|
}
|
2023-12-12 04:05:54 +08:00
|
|
|
rpc LookupTopicBrokers (LookupTopicBrokersRequest) returns (LookupTopicBrokersResponse) {
|
2023-08-21 13:53:05 +08:00
|
|
|
}
|
2023-12-12 04:05:54 +08:00
|
|
|
|
|
|
|
// invoked by the balancer, running on each broker
|
2023-08-21 13:53:05 +08:00
|
|
|
rpc AssignTopicPartitions (AssignTopicPartitionsRequest) returns (AssignTopicPartitionsResponse) {
|
|
|
|
}
|
2023-12-12 04:05:54 +08:00
|
|
|
rpc ClosePublishers(ClosePublishersRequest) returns (ClosePublishersResponse) {
|
|
|
|
}
|
|
|
|
rpc CloseSubscribers(CloseSubscribersRequest) returns (CloseSubscribersResponse) {
|
|
|
|
}
|
|
|
|
|
|
|
|
// subscriber connects to broker balancer, which coordinates with the subscribers
|
|
|
|
rpc SubscriberToSubCoordinator (stream SubscriberToSubCoordinatorRequest) returns (stream SubscriberToSubCoordinatorResponse) {
|
2023-08-21 13:53:05 +08:00
|
|
|
}
|
|
|
|
|
2023-12-12 04:05:54 +08:00
|
|
|
// data plane for each topic partition
|
2024-01-06 07:14:25 +08:00
|
|
|
rpc PublishMessage (stream PublishMessageRequest) returns (stream PublishMessageResponse) {
|
2022-08-01 04:23:44 +08:00
|
|
|
}
|
2024-05-18 09:17:23 +08:00
|
|
|
rpc SubscribeMessage (stream SubscribeMessageRequest) returns (stream SubscribeMessageResponse) {
|
2023-08-27 04:39:21 +08:00
|
|
|
}
|
2024-03-11 05:34:28 +08:00
|
|
|
// The lead broker asks a follower broker to follow itself
|
2024-03-21 03:25:40 +08:00
|
|
|
rpc PublishFollowMe (stream PublishFollowMeRequest) returns (stream PublishFollowMeResponse) {
|
2024-03-11 05:34:28 +08:00
|
|
|
}
|
2024-05-18 09:17:23 +08:00
|
|
|
rpc SubscribeFollowMe (stream SubscribeFollowMeRequest) returns (SubscribeFollowMeResponse) {
|
|
|
|
}
|
2020-04-16 17:21:23 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
//////////////////////////////////////////////////
|
|
|
|
|
2022-07-11 03:11:37 +08:00
|
|
|
message FindBrokerLeaderRequest {
|
2022-07-11 15:20:27 +08:00
|
|
|
string filer_group = 1;
|
2020-05-05 17:05:28 +08:00
|
|
|
}
|
|
|
|
|
2022-07-11 03:11:37 +08:00
|
|
|
message FindBrokerLeaderResponse {
|
2020-05-05 17:05:28 +08:00
|
|
|
string broker = 1;
|
|
|
|
}
|
2022-07-11 15:20:27 +08:00
|
|
|
|
2023-08-21 13:53:05 +08:00
|
|
|
message Topic {
|
|
|
|
string namespace = 1;
|
|
|
|
string name = 2;
|
|
|
|
}
|
2022-07-11 15:20:27 +08:00
|
|
|
message Partition {
|
|
|
|
int32 ring_size = 1;
|
|
|
|
int32 range_start = 2;
|
|
|
|
int32 range_stop = 3;
|
2023-12-12 04:05:54 +08:00
|
|
|
int64 unix_time_ns = 4;
|
2022-07-11 15:20:27 +08:00
|
|
|
}
|
2022-08-01 04:23:44 +08:00
|
|
|
|
2024-01-06 07:03:16 +08:00
|
|
|
message Offset {
|
|
|
|
Topic topic = 1;
|
|
|
|
repeated PartitionOffset partition_offsets = 2;
|
|
|
|
}
|
|
|
|
|
2024-01-08 16:03:08 +08:00
|
|
|
enum PartitionOffsetStartType {
|
|
|
|
EARLIEST = 0;
|
|
|
|
EARLIEST_IN_MEMORY = 1;
|
|
|
|
LATEST = 2;
|
|
|
|
}
|
|
|
|
|
2024-01-06 07:03:16 +08:00
|
|
|
message PartitionOffset {
|
|
|
|
Partition partition = 1;
|
2024-01-08 16:03:08 +08:00
|
|
|
int64 start_ts_ns = 2;
|
|
|
|
int64 stop_ts_ns = 3;
|
|
|
|
PartitionOffsetStartType start_type = 4;
|
2024-01-06 07:03:16 +08:00
|
|
|
}
|
|
|
|
|
2023-09-15 14:49:05 +08:00
|
|
|
//////////////////////////////////////////////////
|
|
|
|
message BrokerStats {
|
2023-09-20 05:02:08 +08:00
|
|
|
int32 cpu_usage_percent = 1;
|
|
|
|
map<string, TopicPartitionStats> stats = 2;
|
2023-09-15 14:49:05 +08:00
|
|
|
}
|
2023-09-20 05:02:08 +08:00
|
|
|
message TopicPartitionStats {
|
|
|
|
Topic topic = 1;
|
|
|
|
Partition partition = 2;
|
2024-03-28 14:42:19 +08:00
|
|
|
int32 publisher_count = 3;
|
|
|
|
int32 subscriber_count = 4;
|
2024-05-19 15:46:12 +08:00
|
|
|
string follower = 5;
|
2023-09-20 05:02:08 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2023-12-12 04:05:54 +08:00
|
|
|
message PublisherToPubBalancerRequest {
|
2023-09-15 14:49:05 +08:00
|
|
|
message InitMessage {
|
|
|
|
string broker = 1;
|
|
|
|
}
|
|
|
|
oneof message {
|
|
|
|
InitMessage init = 1;
|
|
|
|
BrokerStats stats = 2;
|
|
|
|
}
|
|
|
|
}
|
2023-12-12 04:05:54 +08:00
|
|
|
message PublisherToPubBalancerResponse {
|
2023-09-15 14:49:05 +08:00
|
|
|
}
|
2023-12-12 04:05:54 +08:00
|
|
|
|
|
|
|
message BalanceTopicsRequest {
|
|
|
|
}
|
|
|
|
message BalanceTopicsResponse {
|
|
|
|
}
|
|
|
|
|
2023-09-15 14:49:05 +08:00
|
|
|
//////////////////////////////////////////////////
|
2023-09-27 06:17:33 +08:00
|
|
|
message ConfigureTopicRequest {
|
2023-09-25 05:22:11 +08:00
|
|
|
Topic topic = 1;
|
|
|
|
int32 partition_count = 2;
|
2024-04-13 13:27:16 +08:00
|
|
|
schema_pb.RecordType record_type = 3;
|
2023-09-25 05:22:11 +08:00
|
|
|
}
|
2023-09-27 06:17:33 +08:00
|
|
|
message ConfigureTopicResponse {
|
2023-09-25 05:22:11 +08:00
|
|
|
repeated BrokerPartitionAssignment broker_partition_assignments = 2;
|
2024-04-13 13:27:16 +08:00
|
|
|
schema_pb.RecordType record_type = 3;
|
2023-09-25 05:22:11 +08:00
|
|
|
}
|
2023-09-25 12:19:51 +08:00
|
|
|
message ListTopicsRequest {
|
|
|
|
}
|
|
|
|
message ListTopicsResponse {
|
|
|
|
repeated Topic topics = 1;
|
|
|
|
}
|
2023-08-28 09:59:04 +08:00
|
|
|
message LookupTopicBrokersRequest {
|
2023-08-21 13:53:05 +08:00
|
|
|
Topic topic = 1;
|
|
|
|
}
|
2023-08-28 09:59:04 +08:00
|
|
|
message LookupTopicBrokersResponse {
|
2023-08-21 13:53:05 +08:00
|
|
|
Topic topic = 1;
|
2023-08-28 09:59:04 +08:00
|
|
|
repeated BrokerPartitionAssignment broker_partition_assignments = 2;
|
2023-08-21 13:53:05 +08:00
|
|
|
}
|
2023-08-28 09:59:04 +08:00
|
|
|
message BrokerPartitionAssignment {
|
|
|
|
Partition partition = 1;
|
|
|
|
string leader_broker = 2;
|
2024-05-19 15:46:12 +08:00
|
|
|
string follower_broker = 3;
|
2023-08-21 13:53:05 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
message AssignTopicPartitionsRequest {
|
|
|
|
Topic topic = 1;
|
2023-08-28 09:59:04 +08:00
|
|
|
repeated BrokerPartitionAssignment broker_partition_assignments = 2;
|
2023-08-21 13:53:05 +08:00
|
|
|
bool is_leader = 3;
|
2023-12-12 04:05:54 +08:00
|
|
|
bool is_draining = 4;
|
2023-08-21 13:53:05 +08:00
|
|
|
}
|
|
|
|
message AssignTopicPartitionsResponse {
|
|
|
|
}
|
|
|
|
|
2023-12-12 04:05:54 +08:00
|
|
|
message SubscriberToSubCoordinatorRequest {
|
|
|
|
message InitMessage {
|
|
|
|
string consumer_group = 1;
|
2023-12-29 03:56:37 +08:00
|
|
|
string consumer_group_instance_id = 2;
|
2023-12-12 04:05:54 +08:00
|
|
|
Topic topic = 3;
|
2024-05-20 05:52:38 +08:00
|
|
|
// The consumer group instance will be assigned at most max_partition_count partitions.
|
|
|
|
// If the number of partitions is less than the sum of max_partition_count,
|
|
|
|
// the consumer group instance may be assigned partitions less than max_partition_count.
|
|
|
|
// Default is 1.
|
2024-05-14 09:30:10 +08:00
|
|
|
int32 max_partition_count = 4;
|
2024-05-20 05:52:38 +08:00
|
|
|
// If consumer group instance changes, wait for rebalance_seconds before reassigning partitions
|
|
|
|
// Exception: if adding a new consumer group instance and sum of max_partition_count equals the number of partitions,
|
|
|
|
// the rebalance will happen immediately.
|
|
|
|
// Default is 10 seconds.
|
|
|
|
int32 rebalance_seconds = 5;
|
2023-12-12 04:05:54 +08:00
|
|
|
}
|
|
|
|
message AckMessage {
|
|
|
|
Partition partition = 1;
|
|
|
|
int64 ts_ns = 2;
|
|
|
|
}
|
|
|
|
oneof message {
|
|
|
|
InitMessage init = 1;
|
|
|
|
AckMessage ack = 2;
|
|
|
|
}
|
2023-08-21 13:53:05 +08:00
|
|
|
}
|
2023-12-12 04:05:54 +08:00
|
|
|
message SubscriberToSubCoordinatorResponse {
|
|
|
|
message Assignment {
|
|
|
|
int64 generation = 1;
|
2024-03-25 03:57:09 +08:00
|
|
|
repeated BrokerPartitionAssignment partition_assignments = 2;
|
2023-12-12 04:05:54 +08:00
|
|
|
}
|
|
|
|
oneof message {
|
|
|
|
Assignment assignment = 1;
|
|
|
|
}
|
2023-08-21 13:53:05 +08:00
|
|
|
}
|
|
|
|
|
2022-08-01 04:23:44 +08:00
|
|
|
//////////////////////////////////////////////////
|
2024-04-01 07:35:46 +08:00
|
|
|
message ControlMessage {
|
|
|
|
bool is_close = 1;
|
|
|
|
}
|
2023-08-27 04:39:21 +08:00
|
|
|
message DataMessage {
|
|
|
|
bytes key = 1;
|
|
|
|
bytes value = 2;
|
2023-12-12 04:05:54 +08:00
|
|
|
int64 ts_ns = 3;
|
2024-04-01 07:35:46 +08:00
|
|
|
ControlMessage ctrl = 4;
|
2023-08-27 04:39:21 +08:00
|
|
|
}
|
2024-01-06 07:14:25 +08:00
|
|
|
message PublishMessageRequest {
|
2022-08-01 04:23:44 +08:00
|
|
|
message InitMessage {
|
2023-08-27 04:39:21 +08:00
|
|
|
Topic topic = 1;
|
|
|
|
Partition partition = 2;
|
2023-09-07 13:39:46 +08:00
|
|
|
int32 ack_interval = 3;
|
2024-05-19 15:46:12 +08:00
|
|
|
string follower_broker = 4;
|
2024-03-31 15:19:16 +08:00
|
|
|
string publisher_name = 5; // for debugging
|
2023-08-21 13:53:05 +08:00
|
|
|
}
|
|
|
|
oneof message {
|
|
|
|
InitMessage init = 1;
|
|
|
|
DataMessage data = 2;
|
|
|
|
}
|
|
|
|
int64 sequence = 3;
|
2022-08-01 04:23:44 +08:00
|
|
|
}
|
2024-01-06 07:14:25 +08:00
|
|
|
message PublishMessageResponse {
|
2022-08-01 04:23:44 +08:00
|
|
|
int64 ack_sequence = 1;
|
2023-08-21 13:53:05 +08:00
|
|
|
string error = 2;
|
2023-12-12 04:05:54 +08:00
|
|
|
bool should_close = 3;
|
2022-08-01 04:23:44 +08:00
|
|
|
}
|
2024-03-11 05:34:28 +08:00
|
|
|
message PublishFollowMeRequest {
|
2024-03-21 03:25:40 +08:00
|
|
|
message InitMessage {
|
|
|
|
Topic topic = 1;
|
|
|
|
Partition partition = 2;
|
|
|
|
}
|
|
|
|
message FlushMessage {
|
|
|
|
int64 ts_ns = 1;
|
|
|
|
}
|
|
|
|
message CloseMessage {
|
|
|
|
}
|
|
|
|
oneof message {
|
|
|
|
InitMessage init = 1;
|
|
|
|
DataMessage data = 2;
|
|
|
|
FlushMessage flush = 3;
|
|
|
|
CloseMessage close = 4;
|
|
|
|
}
|
2024-03-11 05:34:28 +08:00
|
|
|
}
|
|
|
|
message PublishFollowMeResponse {
|
2024-03-21 03:25:40 +08:00
|
|
|
int64 ack_ts_ns = 1;
|
2024-03-11 05:34:28 +08:00
|
|
|
}
|
2024-01-06 07:14:25 +08:00
|
|
|
message SubscribeMessageRequest {
|
2023-10-01 12:47:57 +08:00
|
|
|
message InitMessage {
|
2023-09-01 15:36:51 +08:00
|
|
|
string consumer_group = 1;
|
|
|
|
string consumer_id = 2;
|
2023-09-05 12:43:50 +08:00
|
|
|
string client_id = 3;
|
2023-10-01 04:18:49 +08:00
|
|
|
Topic topic = 4;
|
2024-01-06 07:24:14 +08:00
|
|
|
PartitionOffset partition_offset = 5;
|
|
|
|
string filter = 6;
|
2024-05-19 15:46:12 +08:00
|
|
|
string follower_broker = 7;
|
2024-05-21 00:33:37 +08:00
|
|
|
int32 concurrency = 8;
|
2023-10-01 04:18:49 +08:00
|
|
|
}
|
|
|
|
message AckMessage {
|
|
|
|
int64 sequence = 1;
|
2024-05-21 00:33:37 +08:00
|
|
|
bytes key = 2;
|
2023-10-01 04:18:49 +08:00
|
|
|
}
|
|
|
|
oneof message {
|
2023-10-01 12:47:57 +08:00
|
|
|
InitMessage init = 1;
|
|
|
|
AckMessage ack = 2;
|
2023-08-27 04:39:21 +08:00
|
|
|
}
|
|
|
|
}
|
2024-01-06 07:14:25 +08:00
|
|
|
message SubscribeMessageResponse {
|
2024-04-01 07:35:46 +08:00
|
|
|
message SubscribeCtrlMessage {
|
2023-08-27 04:39:21 +08:00
|
|
|
string error = 1;
|
2023-10-01 04:18:49 +08:00
|
|
|
bool is_end_of_stream = 2;
|
|
|
|
bool is_end_of_topic = 3;
|
2023-08-27 04:39:21 +08:00
|
|
|
}
|
|
|
|
oneof message {
|
2024-04-01 07:35:46 +08:00
|
|
|
SubscribeCtrlMessage ctrl = 1;
|
2023-08-27 04:39:21 +08:00
|
|
|
DataMessage data = 2;
|
|
|
|
}
|
|
|
|
}
|
2024-05-18 09:17:23 +08:00
|
|
|
message SubscribeFollowMeRequest {
|
|
|
|
message InitMessage {
|
|
|
|
Topic topic = 1;
|
|
|
|
Partition partition = 2;
|
2024-05-19 15:46:12 +08:00
|
|
|
string consumer_group = 3;
|
2024-05-18 09:17:23 +08:00
|
|
|
}
|
|
|
|
message AckMessage {
|
|
|
|
int64 ts_ns = 1;
|
|
|
|
}
|
|
|
|
message CloseMessage {
|
|
|
|
}
|
|
|
|
oneof message {
|
|
|
|
InitMessage init = 1;
|
|
|
|
AckMessage ack = 2;
|
|
|
|
CloseMessage close = 3;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
message SubscribeFollowMeResponse {
|
|
|
|
int64 ack_ts_ns = 1;
|
|
|
|
}
|
2023-12-12 04:05:54 +08:00
|
|
|
message ClosePublishersRequest {
|
|
|
|
Topic topic = 1;
|
|
|
|
int64 unix_time_ns = 2;
|
|
|
|
}
|
|
|
|
message ClosePublishersResponse {
|
|
|
|
}
|
|
|
|
message CloseSubscribersRequest {
|
|
|
|
Topic topic = 1;
|
|
|
|
int64 unix_time_ns = 2;
|
|
|
|
}
|
|
|
|
message CloseSubscribersResponse {
|
|
|
|
}
|