1
0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-12-30 21:25:32 +08:00
seaweedfs/weed/mq/pub_balancer/allocate.go

163 lines
4.9 KiB
Go
Raw Normal View History

Merge accumulated changes related to message queue (#5098) * balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * tracking topic=>broker * merge * comment
2023-12-12 04:05:54 +08:00
package pub_balancer
import (
cmap "github.com/orcaman/concurrent-map/v2"
Merge accumulated changes related to message queue (#5098) * balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * tracking topic=>broker * merge * comment
2023-12-12 04:05:54 +08:00
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
2023-09-25 06:08:44 +08:00
"math/rand"
2024-01-11 14:36:17 +08:00
"time"
)
2024-01-17 00:40:23 +08:00
func AllocateTopicPartitions(brokers cmap.ConcurrentMap[string, *BrokerStats], partitionCount int32) (assignments []*mq_pb.BrokerPartitionAssignment) {
2023-09-25 06:08:44 +08:00
// divide the ring into partitions
2024-01-11 14:36:17 +08:00
now := time.Now().UnixNano()
2023-09-25 06:08:44 +08:00
rangeSize := MaxPartitionCount / partitionCount
2023-09-25 12:19:51 +08:00
for i := int32(0); i < partitionCount; i++ {
2023-09-25 06:08:44 +08:00
assignment := &mq_pb.BrokerPartitionAssignment{
Partition: &mq_pb.Partition{
RingSize: MaxPartitionCount,
2023-09-25 06:08:44 +08:00
RangeStart: int32(i * rangeSize),
RangeStop: int32((i + 1) * rangeSize),
2024-01-11 14:36:17 +08:00
UnixTimeNs: now,
},
2023-09-25 06:08:44 +08:00
}
if i == partitionCount-1 {
assignment.Partition.RangeStop = MaxPartitionCount
}
assignments = append(assignments, assignment)
}
2023-09-25 06:08:44 +08:00
2024-03-08 02:45:38 +08:00
EnsureAssignmentsToActiveBrokers(brokers, 1, assignments)
2023-09-25 06:08:44 +08:00
Merge accumulated changes related to message queue (#5098) * balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * tracking topic=>broker * merge * comment
2023-12-12 04:05:54 +08:00
glog.V(0).Infof("allocate topic partitions %d: %v", len(assignments), assignments)
2023-09-25 06:08:44 +08:00
return
}
2024-03-01 06:51:06 +08:00
// randomly pick n brokers, which may contain duplicates
2023-09-25 06:08:44 +08:00
// TODO pick brokers based on the broker stats
2023-09-25 12:19:51 +08:00
func pickBrokers(brokers cmap.ConcurrentMap[string, *BrokerStats], count int32) []string {
2023-09-25 06:08:44 +08:00
candidates := make([]string, 0, brokers.Count())
for brokerStatsItem := range brokers.IterBuffered() {
candidates = append(candidates, brokerStatsItem.Key)
}
pickedBrokers := make([]string, 0, count)
2023-09-25 12:19:51 +08:00
for i := int32(0); i < count; i++ {
2024-03-01 06:51:06 +08:00
p := rand.Intn(len(candidates))
2023-09-25 06:08:44 +08:00
pickedBrokers = append(pickedBrokers, candidates[p])
}
return pickedBrokers
}
2024-01-21 17:27:22 +08:00
2024-03-01 06:51:06 +08:00
// reservoir sampling select N brokers from the active brokers, with exclusion of the excluded brokers
func pickBrokersExcluded(brokers []string, count int, excludedLeadBroker string, excludedBrokers []string) []string {
// convert the excluded brokers to a map
excludedBrokerMap := make(map[string]bool)
for _, broker := range excludedBrokers {
excludedBrokerMap[broker] = true
}
if excludedLeadBroker != "" {
excludedBrokerMap[excludedLeadBroker] = true
}
pickedBrokers := make([]string, 0, count)
for i, broker := range brokers {
if _, found := excludedBrokerMap[broker]; found {
2024-01-21 17:27:22 +08:00
continue
}
2024-03-01 06:51:06 +08:00
if len(pickedBrokers) < count {
pickedBrokers = append(pickedBrokers, broker)
} else {
j := rand.Intn(i + 1)
if j < count {
pickedBrokers[j] = broker
}
2024-01-21 17:27:22 +08:00
}
}
2024-03-01 06:51:06 +08:00
// shuffle the picked brokers
count = len(pickedBrokers)
for i := 0; i < count; i++ {
j := rand.Intn(count)
pickedBrokers[i], pickedBrokers[j] = pickedBrokers[j], pickedBrokers[i]
}
return pickedBrokers
}
2024-03-01 12:50:57 +08:00
// EnsureAssignmentsToActiveBrokers ensures the assignments are assigned to active brokers
2024-03-01 06:51:06 +08:00
func EnsureAssignmentsToActiveBrokers(activeBrokers cmap.ConcurrentMap[string, *BrokerStats], followerCount int, assignments []*mq_pb.BrokerPartitionAssignment) (hasChanges bool) {
2024-03-08 02:45:38 +08:00
glog.V(0).Infof("EnsureAssignmentsToActiveBrokers: activeBrokers: %v, followerCount: %d, assignments: %v", activeBrokers.Count(), followerCount, assignments)
2024-03-01 06:51:06 +08:00
candidates := make([]string, 0, activeBrokers.Count())
for brokerStatsItem := range activeBrokers.IterBuffered() {
candidates = append(candidates, brokerStatsItem.Key)
2024-01-21 17:27:22 +08:00
}
2024-03-01 06:51:06 +08:00
for _, assignment := range assignments {
// count how many brokers are needed
count := 0
if assignment.LeaderBroker == "" {
count++
} else if _, found := activeBrokers.Get(assignment.LeaderBroker); !found {
assignment.LeaderBroker = ""
count++
}
2024-03-17 01:51:47 +08:00
for i := 0; i < followerCount; i++ {
2024-03-01 06:51:06 +08:00
if i >= len(assignment.FollowerBrokers) {
count++
continue
}
if assignment.FollowerBrokers[i] == "" {
count++
} else if _, found := activeBrokers.Get(assignment.FollowerBrokers[i]); !found {
assignment.FollowerBrokers[i] = ""
count++
}
2024-02-05 05:38:00 +08:00
}
2024-03-01 06:51:06 +08:00
if count > 0 {
pickedBrokers := pickBrokersExcluded(candidates, count, assignment.LeaderBroker, assignment.FollowerBrokers)
i := 0
if assignment.LeaderBroker == "" {
2024-03-08 02:45:38 +08:00
if i < len(pickedBrokers) {
assignment.LeaderBroker = pickedBrokers[i]
i++
hasChanges = true
}
2024-03-01 06:51:06 +08:00
}
2024-03-01 21:59:40 +08:00
hasEmptyFollowers := false
2024-03-01 06:51:06 +08:00
j := 0
2024-03-17 01:51:47 +08:00
for ; j < len(assignment.FollowerBrokers); j++ {
2024-03-01 06:51:06 +08:00
if assignment.FollowerBrokers[j] == "" {
2024-03-01 12:50:57 +08:00
hasChanges = true
2024-03-01 21:59:40 +08:00
if i < len(pickedBrokers) {
assignment.FollowerBrokers[j] = pickedBrokers[i]
i++
} else {
hasEmptyFollowers = true
}
}
}
if hasEmptyFollowers {
var followerBrokers []string
for _, follower := range assignment.FollowerBrokers {
if follower != "" {
followerBrokers = append(followerBrokers, follower)
}
2024-03-01 06:51:06 +08:00
}
2024-03-01 21:59:40 +08:00
assignment.FollowerBrokers = followerBrokers
2024-03-01 06:51:06 +08:00
}
2024-03-01 21:59:40 +08:00
2024-03-01 06:51:06 +08:00
if i < len(pickedBrokers) {
assignment.FollowerBrokers = append(assignment.FollowerBrokers, pickedBrokers[i:]...)
2024-03-01 12:50:57 +08:00
hasChanges = true
2024-03-01 06:51:06 +08:00
}
}
2024-01-23 03:04:24 +08:00
}
2024-03-08 02:45:38 +08:00
glog.V(0).Infof("EnsureAssignmentsToActiveBrokers: activeBrokers: %v, followerCount: %d, assignments: %v hasChanges: %v", activeBrokers.Count(), followerCount, assignments, hasChanges)
2024-01-23 03:04:24 +08:00
return
2024-01-21 17:27:22 +08:00
}