2022-04-04 16:50:56 +08:00
|
|
|
package weed_server
|
|
|
|
|
|
|
|
// https://yusufs.medium.com/creating-distributed-kv-database-by-implementing-raft-consensus-using-golang-d0884eef2e28
|
|
|
|
// https://github.com/Jille/raft-grpc-example/blob/cd5bcab0218f008e044fbeee4facdd01b06018ad/application.go#L18
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
transport "github.com/Jille/raft-grpc-transport"
|
2022-10-07 19:20:34 +08:00
|
|
|
"github.com/armon/go-metrics"
|
|
|
|
"github.com/armon/go-metrics/prometheus"
|
2022-04-04 16:50:56 +08:00
|
|
|
"github.com/hashicorp/raft"
|
2022-08-31 01:12:02 +08:00
|
|
|
boltdb "github.com/hashicorp/raft-boltdb/v2"
|
2022-07-29 15:17:28 +08:00
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
2022-10-07 19:20:34 +08:00
|
|
|
"github.com/seaweedfs/seaweedfs/weed/stats"
|
2022-04-04 16:50:56 +08:00
|
|
|
"google.golang.org/grpc"
|
|
|
|
"math/rand"
|
|
|
|
"os"
|
2022-04-06 15:18:43 +08:00
|
|
|
"path"
|
2022-04-04 16:50:56 +08:00
|
|
|
"path/filepath"
|
2022-04-05 18:42:19 +08:00
|
|
|
"sort"
|
|
|
|
"strings"
|
2022-04-04 16:50:56 +08:00
|
|
|
"time"
|
|
|
|
)
|
|
|
|
|
2022-04-06 15:18:43 +08:00
|
|
|
const (
|
2022-04-07 00:17:04 +08:00
|
|
|
ldbFile = "logs.dat"
|
|
|
|
sdbFile = "stable.dat"
|
|
|
|
updatePeersTimeout = 15 * time.Minute
|
2022-04-06 15:18:43 +08:00
|
|
|
)
|
|
|
|
|
2022-04-05 18:42:19 +08:00
|
|
|
func getPeerIdx(self pb.ServerAddress, mapPeers map[string]pb.ServerAddress) int {
|
|
|
|
peers := make([]pb.ServerAddress, 0, len(mapPeers))
|
|
|
|
for _, peer := range mapPeers {
|
|
|
|
peers = append(peers, peer)
|
|
|
|
}
|
|
|
|
sort.Slice(peers, func(i, j int) bool {
|
|
|
|
return strings.Compare(string(peers[i]), string(peers[j])) < 0
|
|
|
|
})
|
|
|
|
for i, peer := range peers {
|
|
|
|
if string(peer) == string(self) {
|
|
|
|
return i
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return -1
|
|
|
|
}
|
|
|
|
|
2022-04-05 16:17:53 +08:00
|
|
|
func (s *RaftServer) AddPeersConfiguration() (cfg raft.Configuration) {
|
|
|
|
for _, peer := range s.peers {
|
|
|
|
cfg.Servers = append(cfg.Servers, raft.Server{
|
|
|
|
Suffrage: raft.Voter,
|
2022-04-11 13:44:44 +08:00
|
|
|
ID: raft.ServerID(peer),
|
2022-04-05 16:17:53 +08:00
|
|
|
Address: raft.ServerAddress(peer.ToGrpcAddress()),
|
|
|
|
})
|
|
|
|
}
|
|
|
|
return cfg
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *RaftServer) UpdatePeers() {
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case isLeader := <-s.RaftHashicorp.LeaderCh():
|
|
|
|
if isLeader {
|
2022-04-05 18:42:19 +08:00
|
|
|
peerLeader := string(s.serverAddr)
|
2022-04-05 16:17:53 +08:00
|
|
|
existsPeerName := make(map[string]bool)
|
|
|
|
for _, server := range s.RaftHashicorp.GetConfiguration().Configuration().Servers {
|
|
|
|
if string(server.ID) == peerLeader {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
existsPeerName[string(server.ID)] = true
|
|
|
|
}
|
|
|
|
for _, peer := range s.peers {
|
2022-04-11 13:44:44 +08:00
|
|
|
peerName := string(peer)
|
|
|
|
if peerName == peerLeader || existsPeerName[peerName] {
|
2022-04-05 16:17:53 +08:00
|
|
|
continue
|
|
|
|
}
|
2022-04-11 13:44:44 +08:00
|
|
|
glog.V(0).Infof("adding new peer: %s", peerName)
|
2022-04-05 16:17:53 +08:00
|
|
|
s.RaftHashicorp.AddVoter(
|
2022-04-11 13:44:44 +08:00
|
|
|
raft.ServerID(peerName), raft.ServerAddress(peer.ToGrpcAddress()), 0, 0)
|
2022-04-05 16:17:53 +08:00
|
|
|
}
|
2022-08-31 01:12:02 +08:00
|
|
|
for peer := range existsPeerName {
|
2022-04-05 16:17:53 +08:00
|
|
|
if _, found := s.peers[peer]; !found {
|
|
|
|
glog.V(0).Infof("removing old peer: %s", peer)
|
|
|
|
s.RaftHashicorp.RemoveServer(raft.ServerID(peer), 0, 0)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if _, found := s.peers[peerLeader]; !found {
|
|
|
|
glog.V(0).Infof("removing old leader peer: %s", peerLeader)
|
|
|
|
s.RaftHashicorp.RemoveServer(raft.ServerID(peerLeader), 0, 0)
|
|
|
|
}
|
|
|
|
}
|
2022-04-07 00:17:04 +08:00
|
|
|
return
|
|
|
|
case <-time.After(updatePeersTimeout):
|
|
|
|
return
|
2022-04-05 16:17:53 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-04-04 20:51:51 +08:00
|
|
|
func NewHashicorpRaftServer(option *RaftServerOption) (*RaftServer, error) {
|
2022-04-04 16:50:56 +08:00
|
|
|
s := &RaftServer{
|
|
|
|
peers: option.Peers,
|
|
|
|
serverAddr: option.ServerAddr,
|
|
|
|
dataDir: option.DataDir,
|
|
|
|
topo: option.Topo,
|
|
|
|
}
|
|
|
|
|
|
|
|
c := raft.DefaultConfig()
|
2022-04-05 16:50:39 +08:00
|
|
|
c.LocalID = raft.ServerID(s.serverAddr) // TODO maybee the IP:port address will change
|
2022-04-04 16:50:56 +08:00
|
|
|
c.HeartbeatTimeout = time.Duration(float64(option.HeartbeatInterval) * (rand.Float64()*0.25 + 1))
|
|
|
|
c.ElectionTimeout = option.ElectionTimeout
|
2022-04-04 20:51:51 +08:00
|
|
|
if c.LeaderLeaseTimeout > c.HeartbeatTimeout {
|
|
|
|
c.LeaderLeaseTimeout = c.HeartbeatTimeout
|
|
|
|
}
|
2022-04-04 16:50:56 +08:00
|
|
|
if glog.V(4) {
|
2022-04-04 20:51:51 +08:00
|
|
|
c.LogLevel = "Debug"
|
2022-04-04 16:50:56 +08:00
|
|
|
} else if glog.V(2) {
|
2022-04-04 20:51:51 +08:00
|
|
|
c.LogLevel = "Info"
|
2022-04-04 16:50:56 +08:00
|
|
|
} else if glog.V(1) {
|
2022-04-04 20:51:51 +08:00
|
|
|
c.LogLevel = "Warn"
|
2022-04-04 16:50:56 +08:00
|
|
|
} else if glog.V(0) {
|
2022-04-04 20:51:51 +08:00
|
|
|
c.LogLevel = "Error"
|
2022-04-04 16:50:56 +08:00
|
|
|
}
|
|
|
|
|
2023-03-21 21:36:13 +08:00
|
|
|
if err := raft.ValidateConfig(c); err != nil {
|
|
|
|
return nil, fmt.Errorf(`raft.ValidateConfig: %v`, err)
|
|
|
|
}
|
|
|
|
|
2022-04-06 15:18:43 +08:00
|
|
|
if option.RaftBootstrap {
|
|
|
|
os.RemoveAll(path.Join(s.dataDir, ldbFile))
|
|
|
|
os.RemoveAll(path.Join(s.dataDir, sdbFile))
|
2022-06-21 16:47:51 +08:00
|
|
|
os.RemoveAll(path.Join(s.dataDir, "snapshots"))
|
|
|
|
}
|
|
|
|
if err := os.MkdirAll(path.Join(s.dataDir, "snapshots"), os.ModePerm); err != nil {
|
|
|
|
return nil, err
|
2022-04-06 15:18:43 +08:00
|
|
|
}
|
2022-04-04 16:50:56 +08:00
|
|
|
baseDir := s.dataDir
|
|
|
|
|
2022-04-06 15:18:43 +08:00
|
|
|
ldb, err := boltdb.NewBoltStore(filepath.Join(baseDir, ldbFile))
|
2022-04-04 16:50:56 +08:00
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf(`boltdb.NewBoltStore(%q): %v`, filepath.Join(baseDir, "logs.dat"), err)
|
|
|
|
}
|
|
|
|
|
2022-04-06 15:18:43 +08:00
|
|
|
sdb, err := boltdb.NewBoltStore(filepath.Join(baseDir, sdbFile))
|
2022-04-04 16:50:56 +08:00
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf(`boltdb.NewBoltStore(%q): %v`, filepath.Join(baseDir, "stable.dat"), err)
|
|
|
|
}
|
|
|
|
|
|
|
|
fss, err := raft.NewFileSnapshotStore(baseDir, 3, os.Stderr)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf(`raft.NewFileSnapshotStore(%q, ...): %v`, baseDir, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
s.TransportManager = transport.New(raft.ServerAddress(s.serverAddr), []grpc.DialOption{option.GrpcDialOption})
|
|
|
|
|
|
|
|
stateMachine := StateMachine{topo: option.Topo}
|
2022-04-04 20:51:51 +08:00
|
|
|
s.RaftHashicorp, err = raft.NewRaft(c, &stateMachine, ldb, sdb, fss, s.TransportManager.Transport())
|
2022-04-04 16:50:56 +08:00
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("raft.NewRaft: %v", err)
|
|
|
|
}
|
2022-04-05 16:17:53 +08:00
|
|
|
if option.RaftBootstrap || len(s.RaftHashicorp.GetConfiguration().Configuration().Servers) == 0 {
|
|
|
|
cfg := s.AddPeersConfiguration()
|
2022-04-05 18:42:19 +08:00
|
|
|
// Need to get lock, in case all servers do this at the same time.
|
|
|
|
peerIdx := getPeerIdx(s.serverAddr, s.peers)
|
2022-09-15 18:12:30 +08:00
|
|
|
timeSleep := time.Duration(float64(c.LeaderLeaseTimeout) * (rand.Float64()*0.25 + 1) * float64(peerIdx))
|
|
|
|
glog.V(0).Infof("Bootstrapping idx: %d sleep: %v new cluster: %+v", peerIdx, timeSleep, cfg)
|
|
|
|
time.Sleep(timeSleep)
|
2022-04-04 20:51:51 +08:00
|
|
|
f := s.RaftHashicorp.BootstrapCluster(cfg)
|
2022-04-04 16:50:56 +08:00
|
|
|
if err := f.Error(); err != nil {
|
|
|
|
return nil, fmt.Errorf("raft.Raft.BootstrapCluster: %v", err)
|
|
|
|
}
|
2022-04-05 16:17:53 +08:00
|
|
|
} else {
|
|
|
|
go s.UpdatePeers()
|
2022-04-04 16:50:56 +08:00
|
|
|
}
|
2022-04-05 16:17:53 +08:00
|
|
|
|
2022-04-04 20:51:51 +08:00
|
|
|
ticker := time.NewTicker(c.HeartbeatTimeout * 10)
|
|
|
|
if glog.V(4) {
|
|
|
|
go func() {
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ticker.C:
|
|
|
|
cfuture := s.RaftHashicorp.GetConfiguration()
|
|
|
|
if err = cfuture.Error(); err != nil {
|
|
|
|
glog.Fatalf("error getting config: %s", err)
|
|
|
|
}
|
|
|
|
configuration := cfuture.Configuration()
|
|
|
|
glog.V(4).Infof("Showing peers known by %s:\n%+v", s.RaftHashicorp.String(), configuration.Servers)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
}
|
2022-04-04 16:50:56 +08:00
|
|
|
|
2022-10-07 19:20:34 +08:00
|
|
|
// Configure a prometheus sink as the raft metrics sink
|
|
|
|
if sink, err := prometheus.NewPrometheusSinkFrom(prometheus.PrometheusOpts{
|
|
|
|
Registerer: stats.Gather,
|
|
|
|
}); err != nil {
|
|
|
|
return nil, fmt.Errorf("NewPrometheusSink: %v", err)
|
|
|
|
} else {
|
|
|
|
metricsConf := metrics.DefaultConfig(stats.Namespace)
|
|
|
|
metricsConf.EnableRuntimeMetrics = false
|
|
|
|
if _, err = metrics.NewGlobal(metricsConf, sink); err != nil {
|
|
|
|
return nil, fmt.Errorf("metrics.NewGlobal: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-04-04 16:50:56 +08:00
|
|
|
return s, nil
|
|
|
|
}
|