2013-12-02 17:37:36 +08:00
|
|
|
package weed_server
|
|
|
|
|
|
|
|
import (
|
2022-08-24 14:18:21 +08:00
|
|
|
"context"
|
2015-01-08 16:19:32 +08:00
|
|
|
"fmt"
|
2014-10-27 02:34:55 +08:00
|
|
|
"net/http"
|
|
|
|
"net/http/httputil"
|
|
|
|
"net/url"
|
2019-06-05 16:30:24 +08:00
|
|
|
"os"
|
|
|
|
"regexp"
|
|
|
|
"strings"
|
2014-10-27 02:34:55 +08:00
|
|
|
"sync"
|
2019-06-05 16:30:24 +08:00
|
|
|
"time"
|
2014-10-27 02:34:55 +08:00
|
|
|
|
2022-08-24 14:18:21 +08:00
|
|
|
"github.com/seaweedfs/seaweedfs/weed/stats"
|
|
|
|
|
2022-07-29 15:17:28 +08:00
|
|
|
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
2021-11-29 16:49:49 +08:00
|
|
|
|
2020-01-30 01:09:55 +08:00
|
|
|
"github.com/gorilla/mux"
|
2022-04-07 00:17:04 +08:00
|
|
|
hashicorpRaft "github.com/hashicorp/raft"
|
2022-07-28 03:12:40 +08:00
|
|
|
"github.com/seaweedfs/raft"
|
2020-01-30 01:09:55 +08:00
|
|
|
"google.golang.org/grpc"
|
|
|
|
|
2022-07-29 15:17:28 +08:00
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/security"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/sequence"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/shell"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/topology"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
2013-12-02 17:37:36 +08:00
|
|
|
)
|
|
|
|
|
2019-11-11 09:15:17 +08:00
|
|
|
const (
|
2022-08-01 15:51:41 +08:00
|
|
|
SequencerType = "master.sequencer.type"
|
|
|
|
SequencerSnowflakeId = "master.sequencer.sequencer_snowflake_id"
|
2019-11-11 09:15:17 +08:00
|
|
|
)
|
|
|
|
|
2019-06-23 18:08:27 +08:00
|
|
|
type MasterOption struct {
|
2021-09-13 13:47:52 +08:00
|
|
|
Master pb.ServerAddress
|
2020-07-14 22:34:16 +08:00
|
|
|
MetaFolder string
|
2021-08-13 08:54:34 +08:00
|
|
|
VolumeSizeLimitMB uint32
|
2020-07-14 22:34:16 +08:00
|
|
|
VolumePreallocate bool
|
2020-06-05 01:52:01 +08:00
|
|
|
// PulseSeconds int
|
2019-06-23 18:08:27 +08:00
|
|
|
DefaultReplicaPlacement string
|
|
|
|
GarbageThreshold float64
|
|
|
|
WhiteList []string
|
|
|
|
DisableHttp bool
|
|
|
|
MetricsAddress string
|
|
|
|
MetricsIntervalSec int
|
2021-08-13 09:10:59 +08:00
|
|
|
IsFollower bool
|
2019-06-23 18:08:27 +08:00
|
|
|
}
|
|
|
|
|
2013-12-02 17:37:36 +08:00
|
|
|
type MasterServer struct {
|
2021-12-05 16:42:25 +08:00
|
|
|
master_pb.UnimplementedSeaweedServer
|
2019-06-23 18:08:27 +08:00
|
|
|
option *MasterOption
|
|
|
|
guard *security.Guard
|
|
|
|
|
|
|
|
preallocateSize int64
|
2013-12-02 17:37:36 +08:00
|
|
|
|
2021-05-06 18:46:14 +08:00
|
|
|
Topo *topology.Topology
|
|
|
|
vg *topology.VolumeGrowth
|
|
|
|
vgCh chan *topology.VolumeGrowRequest
|
2014-02-05 17:54:52 +08:00
|
|
|
|
2020-12-13 13:19:40 +08:00
|
|
|
boundedLeaderChan chan int
|
2022-04-07 18:31:37 +08:00
|
|
|
|
2018-07-28 14:09:55 +08:00
|
|
|
// notifying clients
|
|
|
|
clientChansLock sync.RWMutex
|
2021-11-06 19:07:38 +08:00
|
|
|
clientChans map[string]chan *master_pb.KeepConnectedResponse
|
2019-02-19 04:11:52 +08:00
|
|
|
|
2019-10-29 21:28:28 +08:00
|
|
|
grpcDialOption grpc.DialOption
|
2019-07-28 18:58:13 +08:00
|
|
|
|
|
|
|
MasterClient *wdclient.MasterClient
|
2020-04-13 08:48:39 +08:00
|
|
|
|
2020-07-14 22:34:16 +08:00
|
|
|
adminLocks *AdminLocks
|
2021-11-03 14:38:45 +08:00
|
|
|
|
2021-11-08 15:52:40 +08:00
|
|
|
Cluster *cluster.Cluster
|
2013-12-02 17:37:36 +08:00
|
|
|
}
|
|
|
|
|
2022-03-27 04:33:17 +08:00
|
|
|
func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.ServerAddress) *MasterServer {
|
2017-01-09 03:01:46 +08:00
|
|
|
|
2020-01-30 01:09:55 +08:00
|
|
|
v := util.GetViper()
|
2019-02-14 16:08:20 +08:00
|
|
|
signingKey := v.GetString("jwt.signing.key")
|
2019-05-04 23:42:25 +08:00
|
|
|
v.SetDefault("jwt.signing.expires_after_seconds", 10)
|
|
|
|
expiresAfterSec := v.GetInt("jwt.signing.expires_after_seconds")
|
2019-02-14 16:08:20 +08:00
|
|
|
|
2019-06-06 15:29:02 +08:00
|
|
|
readSigningKey := v.GetString("jwt.signing.read.key")
|
|
|
|
v.SetDefault("jwt.signing.read.expires_after_seconds", 60)
|
|
|
|
readExpiresAfterSec := v.GetInt("jwt.signing.read.expires_after_seconds")
|
|
|
|
|
2020-04-02 03:18:40 +08:00
|
|
|
v.SetDefault("master.replication.treat_replication_as_minimums", false)
|
|
|
|
replicationAsMin := v.GetBool("master.replication.treat_replication_as_minimums")
|
|
|
|
|
2021-05-06 18:46:14 +08:00
|
|
|
v.SetDefault("master.volume_growth.copy_1", 7)
|
|
|
|
v.SetDefault("master.volume_growth.copy_2", 6)
|
|
|
|
v.SetDefault("master.volume_growth.copy_3", 3)
|
|
|
|
v.SetDefault("master.volume_growth.copy_other", 1)
|
|
|
|
v.SetDefault("master.volume_growth.threshold", 0.9)
|
|
|
|
|
2017-01-09 03:01:46 +08:00
|
|
|
var preallocateSize int64
|
2019-06-23 18:08:27 +08:00
|
|
|
if option.VolumePreallocate {
|
|
|
|
preallocateSize = int64(option.VolumeSizeLimitMB) * (1 << 20)
|
2017-01-09 03:01:46 +08:00
|
|
|
}
|
2019-07-28 18:58:13 +08:00
|
|
|
|
2020-01-30 01:09:55 +08:00
|
|
|
grpcDialOption := security.LoadClientTLS(v, "grpc.master")
|
2013-12-02 17:37:36 +08:00
|
|
|
ms := &MasterServer{
|
2019-06-23 18:08:27 +08:00
|
|
|
option: option,
|
|
|
|
preallocateSize: preallocateSize,
|
2021-07-01 16:21:14 +08:00
|
|
|
vgCh: make(chan *topology.VolumeGrowRequest, 1<<6),
|
2021-11-06 19:07:38 +08:00
|
|
|
clientChans: make(map[string]chan *master_pb.KeepConnectedResponse),
|
2019-10-29 21:28:28 +08:00
|
|
|
grpcDialOption: grpcDialOption,
|
2023-08-24 22:08:56 +08:00
|
|
|
MasterClient: wdclient.NewMasterClient(grpcDialOption, "", cluster.MasterType, option.Master, "", "", *pb.NewServiceDiscoveryFromMap(peers)),
|
2020-05-26 09:39:53 +08:00
|
|
|
adminLocks: NewAdminLocks(),
|
2021-11-08 15:52:40 +08:00
|
|
|
Cluster: cluster.NewCluster(),
|
2013-12-02 17:37:36 +08:00
|
|
|
}
|
2020-12-13 13:19:40 +08:00
|
|
|
ms.boundedLeaderChan = make(chan int, 16)
|
2022-04-07 18:31:37 +08:00
|
|
|
|
2022-08-27 01:18:49 +08:00
|
|
|
ms.MasterClient.SetOnPeerUpdateFn(ms.OnPeerUpdate)
|
2019-11-04 16:45:38 +08:00
|
|
|
|
|
|
|
seq := ms.createSequencer(option)
|
|
|
|
if nil == seq {
|
|
|
|
glog.Fatalf("create sequencer failed.")
|
|
|
|
}
|
2020-06-05 01:52:01 +08:00
|
|
|
ms.Topo = topology.NewTopology("topo", seq, uint64(ms.option.VolumeSizeLimitMB)*1024*1024, 5, replicationAsMin)
|
2014-04-13 16:29:52 +08:00
|
|
|
ms.vg = topology.NewDefaultVolumeGrowth()
|
2019-06-23 18:08:27 +08:00
|
|
|
glog.V(0).Infoln("Volume Size Limit is", ms.option.VolumeSizeLimitMB, "MB")
|
2013-12-02 17:37:36 +08:00
|
|
|
|
2019-06-23 18:08:27 +08:00
|
|
|
ms.guard = security.NewGuard(ms.option.WhiteList, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec)
|
2015-01-06 06:20:04 +08:00
|
|
|
|
2020-12-14 16:24:41 +08:00
|
|
|
handleStaticResources2(r)
|
|
|
|
r.HandleFunc("/", ms.proxyToLeader(ms.uiStatusHandler))
|
|
|
|
r.HandleFunc("/ui/index.html", ms.uiStatusHandler)
|
2019-06-23 18:08:27 +08:00
|
|
|
if !ms.option.DisableHttp {
|
2019-03-02 21:10:05 +08:00
|
|
|
r.HandleFunc("/dir/assign", ms.proxyToLeader(ms.guard.WhiteList(ms.dirAssignHandler)))
|
2019-07-28 18:58:13 +08:00
|
|
|
r.HandleFunc("/dir/lookup", ms.guard.WhiteList(ms.dirLookupHandler))
|
2019-03-02 21:10:05 +08:00
|
|
|
r.HandleFunc("/dir/status", ms.proxyToLeader(ms.guard.WhiteList(ms.dirStatusHandler)))
|
|
|
|
r.HandleFunc("/col/delete", ms.proxyToLeader(ms.guard.WhiteList(ms.collectionDeleteHandler)))
|
|
|
|
r.HandleFunc("/vol/grow", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeGrowHandler)))
|
|
|
|
r.HandleFunc("/vol/status", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeStatusHandler)))
|
|
|
|
r.HandleFunc("/vol/vacuum", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeVacuumHandler)))
|
|
|
|
r.HandleFunc("/submit", ms.guard.WhiteList(ms.submitFromMasterServerHandler))
|
2020-02-22 13:45:03 +08:00
|
|
|
/*
|
|
|
|
r.HandleFunc("/stats/health", ms.guard.WhiteList(statsHealthHandler))
|
|
|
|
r.HandleFunc("/stats/counter", ms.guard.WhiteList(statsCounterHandler))
|
|
|
|
r.HandleFunc("/stats/memory", ms.guard.WhiteList(statsMemoryHandler))
|
|
|
|
*/
|
2019-07-28 18:58:13 +08:00
|
|
|
r.HandleFunc("/{fileId}", ms.redirectHandler)
|
2019-03-22 00:49:04 +08:00
|
|
|
}
|
2013-12-02 17:37:36 +08:00
|
|
|
|
2021-05-06 18:46:14 +08:00
|
|
|
ms.Topo.StartRefreshWritableVolumes(
|
|
|
|
ms.grpcDialOption,
|
|
|
|
ms.option.GarbageThreshold,
|
|
|
|
v.GetFloat64("master.volume_growth.threshold"),
|
|
|
|
ms.preallocateSize,
|
|
|
|
)
|
|
|
|
|
|
|
|
ms.ProcessGrowRequest()
|
2013-12-02 17:37:36 +08:00
|
|
|
|
2021-08-13 09:10:59 +08:00
|
|
|
if !option.IsFollower {
|
|
|
|
ms.startAdminScripts()
|
|
|
|
}
|
2019-06-05 16:30:24 +08:00
|
|
|
|
2013-12-02 17:37:36 +08:00
|
|
|
return ms
|
|
|
|
}
|
2014-02-05 17:54:52 +08:00
|
|
|
|
|
|
|
func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) {
|
2022-04-04 20:51:51 +08:00
|
|
|
var raftServerName string
|
2022-08-25 00:49:05 +08:00
|
|
|
|
|
|
|
ms.Topo.RaftServerAccessLock.Lock()
|
2022-04-04 20:51:51 +08:00
|
|
|
if raftServer.raftServer != nil {
|
|
|
|
ms.Topo.RaftServer = raftServer.raftServer
|
|
|
|
ms.Topo.RaftServer.AddEventListener(raft.LeaderChangeEventType, func(e raft.Event) {
|
|
|
|
glog.V(0).Infof("leader change event: %+v => %+v", e.PrevValue(), e.Value())
|
|
|
|
stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", e.Value())).Inc()
|
|
|
|
if ms.Topo.RaftServer.Leader() != "" {
|
2022-10-07 00:30:30 +08:00
|
|
|
glog.V(0).Infof("[%s] %s becomes leader.", ms.Topo.RaftServer.Name(), ms.Topo.RaftServer.Leader())
|
2022-04-04 20:51:51 +08:00
|
|
|
}
|
|
|
|
})
|
2022-10-07 00:30:30 +08:00
|
|
|
raftServerName = fmt.Sprintf("[%s]", ms.Topo.RaftServer.Name())
|
2022-04-04 20:51:51 +08:00
|
|
|
} else if raftServer.RaftHashicorp != nil {
|
|
|
|
ms.Topo.HashicorpRaft = raftServer.RaftHashicorp
|
|
|
|
leaderCh := raftServer.RaftHashicorp.LeaderCh()
|
2022-10-07 00:30:30 +08:00
|
|
|
prevLeader, _ := ms.Topo.HashicorpRaft.LeaderWithID()
|
|
|
|
raftServerName = ms.Topo.HashicorpRaft.String()
|
2022-04-04 20:51:51 +08:00
|
|
|
go func() {
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case isLeader := <-leaderCh:
|
2022-10-07 00:30:30 +08:00
|
|
|
ms.Topo.RaftServerAccessLock.RLock()
|
|
|
|
leader, _ := ms.Topo.HashicorpRaft.LeaderWithID()
|
|
|
|
ms.Topo.RaftServerAccessLock.RUnlock()
|
2022-04-04 20:51:51 +08:00
|
|
|
glog.V(0).Infof("is leader %+v change event: %+v => %+v", isLeader, prevLeader, leader)
|
|
|
|
stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", leader)).Inc()
|
|
|
|
prevLeader = leader
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
}
|
2022-08-25 00:49:05 +08:00
|
|
|
ms.Topo.RaftServerAccessLock.Unlock()
|
|
|
|
|
2014-03-16 14:03:49 +08:00
|
|
|
if ms.Topo.IsLeader() {
|
2022-10-07 00:30:30 +08:00
|
|
|
glog.V(0).Infof("%s I am the leader!", raftServerName)
|
2014-02-10 15:37:29 +08:00
|
|
|
} else {
|
2022-10-07 00:30:30 +08:00
|
|
|
var raftServerLeader string
|
2022-08-25 00:49:05 +08:00
|
|
|
ms.Topo.RaftServerAccessLock.RLock()
|
2022-10-07 00:30:30 +08:00
|
|
|
if ms.Topo.RaftServer != nil {
|
|
|
|
raftServerLeader = ms.Topo.RaftServer.Leader()
|
|
|
|
} else if ms.Topo.HashicorpRaft != nil {
|
|
|
|
raftServerName = ms.Topo.HashicorpRaft.String()
|
|
|
|
raftServerLeaderAddr, _ := ms.Topo.HashicorpRaft.LeaderWithID()
|
|
|
|
raftServerLeader = string(raftServerLeaderAddr)
|
2014-04-12 07:23:58 +08:00
|
|
|
}
|
2022-08-25 00:49:05 +08:00
|
|
|
ms.Topo.RaftServerAccessLock.RUnlock()
|
2022-10-07 00:30:30 +08:00
|
|
|
glog.V(0).Infof("%s %s - is the leader.", raftServerName, raftServerLeader)
|
2014-02-10 15:37:29 +08:00
|
|
|
}
|
2014-02-05 17:54:52 +08:00
|
|
|
}
|
|
|
|
|
2020-05-25 21:00:12 +08:00
|
|
|
func (ms *MasterServer) proxyToLeader(f http.HandlerFunc) http.HandlerFunc {
|
2014-02-05 17:54:52 +08:00
|
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
2014-03-16 14:03:49 +08:00
|
|
|
if ms.Topo.IsLeader() {
|
2014-02-05 17:54:52 +08:00
|
|
|
f(w, r)
|
2022-04-07 22:37:40 +08:00
|
|
|
return
|
|
|
|
}
|
2022-08-25 00:49:05 +08:00
|
|
|
|
|
|
|
// get the current raft leader
|
|
|
|
leaderAddr, _ := ms.Topo.MaybeLeader()
|
2023-07-05 02:45:21 +08:00
|
|
|
raftServerLeader := leaderAddr.ToHttpAddress()
|
2022-04-07 22:37:40 +08:00
|
|
|
if raftServerLeader == "" {
|
2021-08-13 09:10:59 +08:00
|
|
|
f(w, r)
|
2022-04-07 22:37:40 +08:00
|
|
|
return
|
|
|
|
}
|
2022-08-25 00:49:05 +08:00
|
|
|
|
2022-04-07 22:37:40 +08:00
|
|
|
ms.boundedLeaderChan <- 1
|
|
|
|
defer func() { <-ms.boundedLeaderChan }()
|
|
|
|
targetUrl, err := url.Parse("http://" + raftServerLeader)
|
|
|
|
if err != nil {
|
|
|
|
writeJsonError(w, r, http.StatusInternalServerError,
|
|
|
|
fmt.Errorf("Leader URL http://%s Parse Error: %v", raftServerLeader, err))
|
|
|
|
return
|
|
|
|
}
|
2022-08-25 00:49:05 +08:00
|
|
|
|
|
|
|
// proxy to leader
|
2022-04-07 22:37:40 +08:00
|
|
|
glog.V(4).Infoln("proxying to leader", raftServerLeader)
|
|
|
|
proxy := httputil.NewSingleHostReverseProxy(targetUrl)
|
|
|
|
director := proxy.Director
|
|
|
|
proxy.Director = func(req *http.Request) {
|
|
|
|
actualHost, err := security.GetActualRemoteHost(req)
|
|
|
|
if err == nil {
|
|
|
|
req.Header.Set("HTTP_X_FORWARDED_FOR", actualHost)
|
|
|
|
}
|
|
|
|
director(req)
|
2014-02-05 17:54:52 +08:00
|
|
|
}
|
2022-04-07 22:37:40 +08:00
|
|
|
proxy.Transport = util.Transport
|
|
|
|
proxy.ServeHTTP(w, r)
|
2014-02-05 17:54:52 +08:00
|
|
|
}
|
|
|
|
}
|
2019-06-05 16:30:24 +08:00
|
|
|
|
|
|
|
func (ms *MasterServer) startAdminScripts() {
|
2020-01-30 01:09:55 +08:00
|
|
|
v := util.GetViper()
|
2019-06-05 16:30:24 +08:00
|
|
|
adminScripts := v.GetString("master.maintenance.scripts")
|
|
|
|
if adminScripts == "" {
|
2021-12-21 05:34:57 +08:00
|
|
|
return
|
2019-06-05 16:30:24 +08:00
|
|
|
}
|
2021-11-29 16:49:49 +08:00
|
|
|
glog.V(0).Infof("adminScripts: %v", adminScripts)
|
2019-06-05 16:30:24 +08:00
|
|
|
|
2019-11-12 14:21:43 +08:00
|
|
|
v.SetDefault("master.maintenance.sleep_minutes", 17)
|
|
|
|
sleepMinutes := v.GetInt("master.maintenance.sleep_minutes")
|
|
|
|
|
2019-06-05 16:30:24 +08:00
|
|
|
scriptLines := strings.Split(adminScripts, "\n")
|
2020-04-24 04:37:31 +08:00
|
|
|
if !strings.Contains(adminScripts, "lock") {
|
|
|
|
scriptLines = append(append([]string{}, "lock"), scriptLines...)
|
|
|
|
scriptLines = append(scriptLines, "unlock")
|
|
|
|
}
|
2019-06-05 16:30:24 +08:00
|
|
|
|
2021-09-13 13:47:52 +08:00
|
|
|
masterAddress := string(ms.option.Master)
|
2019-06-05 16:30:24 +08:00
|
|
|
|
|
|
|
var shellOptions shell.ShellOptions
|
2020-01-30 01:09:55 +08:00
|
|
|
shellOptions.GrpcDialOption = security.LoadClientTLS(v, "grpc.master")
|
2019-06-05 16:30:24 +08:00
|
|
|
shellOptions.Masters = &masterAddress
|
2019-11-12 14:21:43 +08:00
|
|
|
|
2020-04-09 14:57:15 +08:00
|
|
|
shellOptions.Directory = "/"
|
2022-05-03 22:12:33 +08:00
|
|
|
emptyFilerGroup := ""
|
|
|
|
shellOptions.FilerGroup = &emptyFilerGroup
|
2019-06-05 16:30:24 +08:00
|
|
|
|
2022-01-12 17:11:25 +08:00
|
|
|
commandEnv := shell.NewCommandEnv(&shellOptions)
|
2019-06-05 16:30:24 +08:00
|
|
|
|
|
|
|
reg, _ := regexp.Compile(`'.*?'|".*?"|\S+`)
|
|
|
|
|
2021-11-06 08:52:15 +08:00
|
|
|
go commandEnv.MasterClient.KeepConnectedToMaster()
|
2019-06-05 16:30:24 +08:00
|
|
|
|
|
|
|
go func() {
|
2021-12-08 03:44:08 +08:00
|
|
|
for {
|
|
|
|
time.Sleep(time.Duration(sleepMinutes) * time.Minute)
|
2022-08-12 06:03:26 +08:00
|
|
|
if ms.Topo.IsLeader() && ms.MasterClient.GetMaster() != "" {
|
2022-07-11 05:01:53 +08:00
|
|
|
shellOptions.FilerAddress = ms.GetOneFiler(cluster.FilerGroupName(*shellOptions.FilerGroup))
|
2022-01-12 17:11:25 +08:00
|
|
|
if shellOptions.FilerAddress == "" {
|
|
|
|
continue
|
|
|
|
}
|
2019-06-05 16:30:24 +08:00
|
|
|
for _, line := range scriptLines {
|
2020-04-24 05:01:46 +08:00
|
|
|
for _, c := range strings.Split(line, ";") {
|
|
|
|
processEachCmd(reg, c, commandEnv)
|
2019-06-05 16:30:24 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
}
|
2019-11-04 16:45:38 +08:00
|
|
|
|
2020-04-24 05:01:46 +08:00
|
|
|
func processEachCmd(reg *regexp.Regexp, line string, commandEnv *shell.CommandEnv) {
|
|
|
|
cmds := reg.FindAllString(line, -1)
|
|
|
|
if len(cmds) == 0 {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
args := make([]string, len(cmds[1:]))
|
|
|
|
for i := range args {
|
|
|
|
args[i] = strings.Trim(string(cmds[1+i]), "\"'")
|
|
|
|
}
|
2023-03-20 12:20:46 +08:00
|
|
|
cmd := cmds[0]
|
2020-04-24 05:01:46 +08:00
|
|
|
|
|
|
|
for _, c := range shell.Commands {
|
|
|
|
if c.Name() == cmd {
|
|
|
|
glog.V(0).Infof("executing: %s %v", cmd, args)
|
|
|
|
if err := c.Do(args, commandEnv, os.Stdout); err != nil {
|
|
|
|
glog.V(0).Infof("error: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-11-04 16:45:38 +08:00
|
|
|
func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer {
|
|
|
|
var seq sequence.Sequencer
|
2020-01-30 01:09:55 +08:00
|
|
|
v := util.GetViper()
|
2019-11-15 14:20:38 +08:00
|
|
|
seqType := strings.ToLower(v.GetString(SequencerType))
|
|
|
|
glog.V(1).Infof("[%s] : [%s]", SequencerType, seqType)
|
2019-11-11 09:15:17 +08:00
|
|
|
switch strings.ToLower(seqType) {
|
2021-03-25 18:49:26 +08:00
|
|
|
case "snowflake":
|
|
|
|
var err error
|
2021-07-24 11:54:03 +08:00
|
|
|
snowflakeId := v.GetInt(SequencerSnowflakeId)
|
2021-09-13 13:47:52 +08:00
|
|
|
seq, err = sequence.NewSnowflakeSequencer(string(option.Master), snowflakeId)
|
2021-03-25 18:49:26 +08:00
|
|
|
if err != nil {
|
|
|
|
glog.Error(err)
|
|
|
|
seq = nil
|
|
|
|
}
|
2019-11-04 16:45:38 +08:00
|
|
|
default:
|
|
|
|
seq = sequence.NewMemorySequencer()
|
|
|
|
}
|
|
|
|
return seq
|
|
|
|
}
|
2022-04-07 00:17:04 +08:00
|
|
|
|
2022-05-31 12:27:48 +08:00
|
|
|
func (ms *MasterServer) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, startFrom time.Time) {
|
2022-08-25 00:49:05 +08:00
|
|
|
ms.Topo.RaftServerAccessLock.RLock()
|
|
|
|
defer ms.Topo.RaftServerAccessLock.RUnlock()
|
|
|
|
|
2022-04-07 00:17:04 +08:00
|
|
|
if update.NodeType != cluster.MasterType || ms.Topo.HashicorpRaft == nil {
|
|
|
|
return
|
|
|
|
}
|
2022-04-07 21:52:13 +08:00
|
|
|
glog.V(4).Infof("OnPeerUpdate: %+v", update)
|
|
|
|
|
2022-04-07 00:17:04 +08:00
|
|
|
peerAddress := pb.ServerAddress(update.Address)
|
2022-04-07 18:31:37 +08:00
|
|
|
peerName := string(peerAddress)
|
2022-08-24 14:18:21 +08:00
|
|
|
if ms.Topo.HashicorpRaft.State() != hashicorpRaft.Leader {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if update.IsAdd {
|
2022-08-01 15:51:41 +08:00
|
|
|
raftServerFound := false
|
|
|
|
for _, server := range ms.Topo.HashicorpRaft.GetConfiguration().Configuration().Servers {
|
|
|
|
if string(server.ID) == peerName {
|
|
|
|
raftServerFound = true
|
2022-04-07 18:31:37 +08:00
|
|
|
}
|
|
|
|
}
|
2022-08-01 15:51:41 +08:00
|
|
|
if !raftServerFound {
|
|
|
|
glog.V(0).Infof("adding new raft server: %s", peerName)
|
|
|
|
ms.Topo.HashicorpRaft.AddVoter(
|
|
|
|
hashicorpRaft.ServerID(peerName),
|
|
|
|
hashicorpRaft.ServerAddress(peerAddress.ToGrpcAddress()), 0, 0)
|
2022-04-07 00:17:04 +08:00
|
|
|
}
|
2022-08-24 14:18:21 +08:00
|
|
|
} else {
|
|
|
|
pb.WithMasterClient(false, peerAddress, ms.grpcDialOption, true, func(client master_pb.SeaweedClient) error {
|
2022-08-24 14:28:16 +08:00
|
|
|
ctx, cancel := context.WithTimeout(context.TODO(), 15*time.Second)
|
2022-08-24 14:18:21 +08:00
|
|
|
defer cancel()
|
|
|
|
if _, err := client.Ping(ctx, &master_pb.PingRequest{Target: string(peerAddress), TargetType: cluster.MasterType}); err != nil {
|
|
|
|
glog.V(0).Infof("master %s didn't respond to pings. remove raft server", peerName)
|
|
|
|
if err := ms.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
|
|
|
|
_, err := client.RaftRemoveServer(context.Background(), &master_pb.RaftRemoveServerRequest{
|
|
|
|
Id: peerName,
|
|
|
|
Force: false,
|
|
|
|
})
|
|
|
|
return err
|
|
|
|
}); err != nil {
|
|
|
|
glog.Warningf("failed removing old raft server: %v", err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
glog.V(0).Infof("master %s successfully responded to ping", peerName)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
})
|
2022-04-07 00:17:04 +08:00
|
|
|
}
|
|
|
|
}
|
2022-10-07 00:30:30 +08:00
|
|
|
|
|
|
|
func (ms *MasterServer) Shutdown() {
|
|
|
|
if ms.Topo == nil || ms.Topo.HashicorpRaft == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if ms.Topo.HashicorpRaft.State() == hashicorpRaft.Leader {
|
|
|
|
ms.Topo.HashicorpRaft.LeadershipTransfer()
|
|
|
|
}
|
|
|
|
ms.Topo.HashicorpRaft.Shutdown()
|
|
|
|
}
|