1
0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-12-26 02:37:55 +08:00
seaweedfs/weed/cluster/cluster.go

356 lines
9.4 KiB
Go
Raw Normal View History

2021-11-08 15:52:40 +08:00
package cluster
2021-11-04 15:54:38 +08:00
import (
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
2021-11-04 15:54:38 +08:00
"math"
"sync"
"time"
)
2021-11-08 16:09:11 +08:00
const (
2022-04-02 07:55:26 +08:00
MasterType = "master"
VolumeServerType = "volumeServer"
FilerType = "filer"
BrokerType = "broker"
2021-11-08 16:09:11 +08:00
)
2022-07-11 05:01:53 +08:00
type FilerGroupName string
2022-07-03 15:29:25 +08:00
type DataCenter string
type Rack string
2022-07-11 05:01:53 +08:00
type Leaders struct {
leaders [3]pb.ServerAddress
}
2021-11-04 15:54:38 +08:00
type ClusterNode struct {
Address pb.ServerAddress
Version string
counter int
CreatedTs time.Time
DataCenter DataCenter
Rack Rack
2021-11-04 15:54:38 +08:00
}
2022-07-11 05:01:53 +08:00
type GroupMembers struct {
members map[pb.ServerAddress]*ClusterNode
leaders *Leaders
}
type ClusterNodeGroups struct {
groupMembers map[FilerGroupName]*GroupMembers
sync.RWMutex
}
2021-11-04 15:54:38 +08:00
type Cluster struct {
2022-07-11 05:01:53 +08:00
filerGroups *ClusterNodeGroups
brokerGroups *ClusterNodeGroups
2021-11-04 15:54:38 +08:00
}
2022-07-11 05:01:53 +08:00
func newClusterNodeGroups() *ClusterNodeGroups {
return &ClusterNodeGroups{
groupMembers: map[FilerGroupName]*GroupMembers{},
2021-11-04 15:54:38 +08:00
}
}
2022-07-11 05:01:53 +08:00
func (g *ClusterNodeGroups) getGroupMembers(filerGroup FilerGroupName, createIfNotFound bool) *GroupMembers {
2022-07-11 15:20:27 +08:00
members, found := g.groupMembers[filerGroup]
2022-05-02 12:59:16 +08:00
if !found && createIfNotFound {
2022-07-11 15:20:27 +08:00
members = &GroupMembers{
2022-07-11 03:11:37 +08:00
members: make(map[pb.ServerAddress]*ClusterNode),
2022-05-02 12:59:16 +08:00
leaders: &Leaders{},
}
2022-07-11 15:20:27 +08:00
g.groupMembers[filerGroup] = members
2022-05-02 12:59:16 +08:00
}
2022-07-11 15:20:27 +08:00
return members
2022-05-02 12:59:16 +08:00
}
2022-07-11 05:01:53 +08:00
func (m *GroupMembers) addMember(dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) *ClusterNode {
if existingNode, found := m.members[address]; found {
existingNode.counter++
return nil
}
t := &ClusterNode{
Address: address,
Version: version,
counter: 1,
CreatedTs: time.Now(),
DataCenter: dataCenter,
Rack: rack,
}
m.members[address] = t
return t
}
func (m *GroupMembers) removeMember(address pb.ServerAddress) bool {
if existingNode, found := m.members[address]; !found {
return false
} else {
existingNode.counter--
if existingNode.counter <= 0 {
delete(m.members, address)
return true
}
}
return false
}
func (g *ClusterNodeGroups) AddClusterNode(filerGroup FilerGroupName, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
g.Lock()
defer g.Unlock()
m := g.getGroupMembers(filerGroup, true)
if t := m.addMember(dataCenter, rack, address, version); t != nil {
return ensureGroupLeaders(m, true, filerGroup, nodeType, address)
}
return nil
}
func (g *ClusterNodeGroups) RemoveClusterNode(filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
g.Lock()
defer g.Unlock()
m := g.getGroupMembers(filerGroup, false)
if m == nil {
return nil
}
if m.removeMember(address) {
return ensureGroupLeaders(m, false, filerGroup, nodeType, address)
}
return nil
}
func (g *ClusterNodeGroups) ListClusterNode(filerGroup FilerGroupName) (nodes []*ClusterNode) {
g.Lock()
defer g.Unlock()
m := g.getGroupMembers(filerGroup, false)
if m == nil {
return nil
}
for _, node := range m.members {
nodes = append(nodes, node)
}
return
}
func (g *ClusterNodeGroups) IsOneLeader(filerGroup FilerGroupName, address pb.ServerAddress) bool {
g.Lock()
defer g.Unlock()
m := g.getGroupMembers(filerGroup, false)
if m == nil {
return false
}
return m.leaders.isOneLeader(address)
}
2022-07-11 15:20:27 +08:00
func (g *ClusterNodeGroups) ListClusterNodeLeaders(filerGroup FilerGroupName) (nodes []pb.ServerAddress) {
g.Lock()
defer g.Unlock()
m := g.getGroupMembers(filerGroup, false)
if m == nil {
return nil
}
return m.leaders.GetLeaders()
}
2022-07-11 05:01:53 +08:00
func NewCluster() *Cluster {
return &Cluster{
filerGroups: newClusterNodeGroups(),
brokerGroups: newClusterNodeGroups(),
}
}
func (cluster *Cluster) getGroupMembers(filerGroup FilerGroupName, nodeType string, createIfNotFound bool) *GroupMembers {
2021-11-04 15:54:38 +08:00
switch nodeType {
2021-11-08 16:09:11 +08:00
case FilerType:
2022-07-11 05:01:53 +08:00
return cluster.filerGroups.getGroupMembers(filerGroup, createIfNotFound)
2021-11-10 00:50:55 +08:00
case BrokerType:
2022-07-11 05:01:53 +08:00
return cluster.brokerGroups.getGroupMembers(filerGroup, createIfNotFound)
}
return nil
}
2022-07-03 15:29:25 +08:00
2022-07-11 05:01:53 +08:00
func (cluster *Cluster) AddClusterNode(ns, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
filerGroup := FilerGroupName(ns)
switch nodeType {
case FilerType:
return cluster.filerGroups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
case BrokerType:
return cluster.brokerGroups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
2021-11-08 16:09:11 +08:00
case MasterType:
2022-04-07 00:17:04 +08:00
return []*master_pb.KeepConnectedResponse{
{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
NodeType: nodeType,
Address: string(address),
IsAdd: true,
},
},
}
2021-11-04 15:54:38 +08:00
}
2021-11-06 19:07:38 +08:00
return nil
2021-11-04 15:54:38 +08:00
}
2022-07-11 05:01:53 +08:00
func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
filerGroup := FilerGroupName(ns)
2021-11-04 15:54:38 +08:00
switch nodeType {
2021-11-08 16:09:11 +08:00
case FilerType:
2022-07-11 05:01:53 +08:00
return cluster.filerGroups.RemoveClusterNode(filerGroup, nodeType, address)
2021-11-10 00:50:55 +08:00
case BrokerType:
2022-07-11 05:01:53 +08:00
return cluster.brokerGroups.RemoveClusterNode(filerGroup, nodeType, address)
2021-11-08 16:09:11 +08:00
case MasterType:
2022-04-07 00:17:04 +08:00
return []*master_pb.KeepConnectedResponse{
{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
NodeType: nodeType,
Address: string(address),
IsAdd: false,
},
},
}
2021-11-04 15:54:38 +08:00
}
2021-11-06 19:07:38 +08:00
return nil
2021-11-04 15:54:38 +08:00
}
2022-07-11 05:01:53 +08:00
func (cluster *Cluster) ListClusterNode(filerGroup FilerGroupName, nodeType string) (nodes []*ClusterNode) {
2021-11-04 15:54:38 +08:00
switch nodeType {
2021-11-08 16:09:11 +08:00
case FilerType:
2022-07-11 05:01:53 +08:00
return cluster.filerGroups.ListClusterNode(filerGroup)
2021-11-10 00:50:55 +08:00
case BrokerType:
2022-07-11 05:01:53 +08:00
return cluster.brokerGroups.ListClusterNode(filerGroup)
2021-11-08 16:09:11 +08:00
case MasterType:
2021-11-04 15:54:38 +08:00
}
return
}
2021-11-07 05:23:35 +08:00
2022-07-11 15:20:27 +08:00
func (cluster *Cluster) ListClusterNodeLeaders(filerGroup FilerGroupName, nodeType string) (nodes []pb.ServerAddress) {
switch nodeType {
case FilerType:
return cluster.filerGroups.ListClusterNodeLeaders(filerGroup)
case BrokerType:
return cluster.brokerGroups.ListClusterNodeLeaders(filerGroup)
case MasterType:
}
return
}
2022-07-11 05:01:53 +08:00
func (cluster *Cluster) IsOneLeader(filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) bool {
switch nodeType {
case FilerType:
return cluster.filerGroups.IsOneLeader(filerGroup, address)
case BrokerType:
return cluster.brokerGroups.IsOneLeader(filerGroup, address)
case MasterType:
2022-05-02 12:59:16 +08:00
}
2022-07-11 05:01:53 +08:00
return false
2021-11-07 05:23:35 +08:00
}
2021-11-04 15:54:38 +08:00
2022-07-11 05:01:53 +08:00
func ensureGroupLeaders(m *GroupMembers, isAdd bool, filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
2021-11-04 15:54:38 +08:00
if isAdd {
2022-07-11 05:01:53 +08:00
if m.leaders.addLeaderIfVacant(address) {
2021-11-04 15:54:38 +08:00
// has added the address as one leader
2021-11-06 19:07:38 +08:00
result = append(result, &master_pb.KeepConnectedResponse{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
2022-05-02 12:59:16 +08:00
FilerGroup: string(filerGroup),
NodeType: nodeType,
Address: string(address),
IsLeader: true,
IsAdd: true,
2021-11-06 19:07:38 +08:00
},
})
} else {
result = append(result, &master_pb.KeepConnectedResponse{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
2022-05-02 12:59:16 +08:00
FilerGroup: string(filerGroup),
NodeType: nodeType,
Address: string(address),
IsLeader: false,
IsAdd: true,
2021-11-06 19:07:38 +08:00
},
})
2021-11-04 15:54:38 +08:00
}
} else {
2022-07-11 05:01:53 +08:00
if m.leaders.removeLeaderIfExists(address) {
2021-11-06 19:07:38 +08:00
result = append(result, &master_pb.KeepConnectedResponse{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
2022-05-02 12:59:16 +08:00
FilerGroup: string(filerGroup),
NodeType: nodeType,
Address: string(address),
IsLeader: true,
IsAdd: false,
2021-11-06 19:07:38 +08:00
},
})
2021-11-04 15:54:38 +08:00
// pick the freshest one, since it is less likely to go away
var shortestDuration int64 = math.MaxInt64
now := time.Now()
var candidateAddress pb.ServerAddress
2022-07-11 05:01:53 +08:00
for _, node := range m.members {
if m.leaders.isOneLeader(node.Address) {
2021-11-04 15:54:38 +08:00
continue
}
2022-05-31 07:23:52 +08:00
duration := now.Sub(node.CreatedTs).Nanoseconds()
2021-11-04 15:54:38 +08:00
if duration < shortestDuration {
shortestDuration = duration
candidateAddress = node.Address
}
}
if candidateAddress != "" {
2022-07-11 05:01:53 +08:00
m.leaders.addLeaderIfVacant(candidateAddress)
2021-11-06 19:07:38 +08:00
// added a new leader
result = append(result, &master_pb.KeepConnectedResponse{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
NodeType: nodeType,
Address: string(candidateAddress),
IsLeader: true,
IsAdd: true,
},
})
2021-11-04 15:54:38 +08:00
}
2021-11-06 19:07:38 +08:00
} else {
result = append(result, &master_pb.KeepConnectedResponse{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
2022-05-02 12:59:16 +08:00
FilerGroup: string(filerGroup),
NodeType: nodeType,
Address: string(address),
IsLeader: false,
IsAdd: false,
2021-11-06 19:07:38 +08:00
},
})
2021-11-04 15:54:38 +08:00
}
}
2021-11-06 19:07:38 +08:00
return
2021-11-04 15:54:38 +08:00
}
func (leaders *Leaders) addLeaderIfVacant(address pb.ServerAddress) (hasChanged bool) {
if leaders.isOneLeader(address) {
return
}
for i := 0; i < len(leaders.leaders); i++ {
if leaders.leaders[i] == "" {
leaders.leaders[i] = address
hasChanged = true
return
}
}
return
}
func (leaders *Leaders) removeLeaderIfExists(address pb.ServerAddress) (hasChanged bool) {
if !leaders.isOneLeader(address) {
return
}
for i := 0; i < len(leaders.leaders); i++ {
if leaders.leaders[i] == address {
leaders.leaders[i] = ""
hasChanged = true
return
}
}
return
}
func (leaders *Leaders) isOneLeader(address pb.ServerAddress) bool {
for i := 0; i < len(leaders.leaders); i++ {
if leaders.leaders[i] == address {
return true
}
}
return false
}
func (leaders *Leaders) GetLeaders() (addresses []pb.ServerAddress) {
for i := 0; i < len(leaders.leaders); i++ {
if leaders.leaders[i] != "" {
addresses = append(addresses, leaders.leaders[i])
}
}
return
}