2016-06-03 09:09:14 +08:00
|
|
|
package command
|
2012-08-07 16:29:22 +08:00
|
|
|
|
|
|
|
import (
|
2019-06-05 16:30:24 +08:00
|
|
|
"net/http"
|
|
|
|
"os"
|
|
|
|
"runtime"
|
|
|
|
"strconv"
|
|
|
|
"strings"
|
|
|
|
|
2019-02-19 14:38:14 +08:00
|
|
|
"github.com/chrislusf/raft/protobuf"
|
2020-04-28 14:10:23 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/util/grace"
|
2020-01-30 01:09:55 +08:00
|
|
|
"github.com/gorilla/mux"
|
|
|
|
"google.golang.org/grpc/reflection"
|
|
|
|
|
2019-02-19 14:38:14 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
2020-03-04 16:39:47 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/pb"
|
2019-02-19 14:38:14 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
|
2019-02-19 04:11:52 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/security"
|
2019-02-19 14:38:14 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/server"
|
2019-11-29 10:33:18 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/storage/backend"
|
2019-02-19 14:38:14 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/util"
|
2012-08-07 16:29:22 +08:00
|
|
|
)
|
|
|
|
|
2019-06-23 18:08:27 +08:00
|
|
|
var (
|
|
|
|
m MasterOptions
|
|
|
|
)
|
|
|
|
|
|
|
|
type MasterOptions struct {
|
|
|
|
port *int
|
|
|
|
ip *string
|
|
|
|
ipBind *string
|
|
|
|
metaFolder *string
|
|
|
|
peers *string
|
|
|
|
volumeSizeLimitMB *uint
|
|
|
|
volumePreallocate *bool
|
2020-06-05 01:52:01 +08:00
|
|
|
// pulseSeconds *int
|
2019-06-23 18:08:27 +08:00
|
|
|
defaultReplication *string
|
|
|
|
garbageThreshold *float64
|
|
|
|
whiteList *string
|
|
|
|
disableHttp *bool
|
|
|
|
metricsAddress *string
|
|
|
|
metricsIntervalSec *int
|
|
|
|
}
|
|
|
|
|
2012-08-07 16:29:22 +08:00
|
|
|
func init() {
|
2012-09-04 10:18:02 +08:00
|
|
|
cmdMaster.Run = runMaster // break init cycle
|
2019-06-23 18:08:27 +08:00
|
|
|
m.port = cmdMaster.Flag.Int("port", 9333, "http listen port")
|
2020-04-19 06:20:20 +08:00
|
|
|
m.ip = cmdMaster.Flag.String("ip", util.DetectedHostAddress(), "master <ip>|<server> address")
|
2019-06-23 18:08:27 +08:00
|
|
|
m.ipBind = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
|
|
|
|
m.metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
|
2020-04-14 03:58:45 +08:00
|
|
|
m.peers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095")
|
2019-06-23 18:08:27 +08:00
|
|
|
m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
|
|
|
|
m.volumePreallocate = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
|
2020-06-05 01:52:01 +08:00
|
|
|
// m.pulseSeconds = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
|
2019-06-23 18:08:27 +08:00
|
|
|
m.defaultReplication = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.")
|
|
|
|
m.garbageThreshold = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
|
|
|
|
m.whiteList = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
|
|
|
|
m.disableHttp = cmdMaster.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
|
|
|
|
m.metricsAddress = cmdMaster.Flag.String("metrics.address", "", "Prometheus gateway address")
|
|
|
|
m.metricsIntervalSec = cmdMaster.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
|
2012-08-07 16:29:22 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
var cmdMaster = &Command{
|
2012-09-04 10:18:02 +08:00
|
|
|
UsageLine: "master -port=9333",
|
|
|
|
Short: "start a master server",
|
2019-02-10 13:07:12 +08:00
|
|
|
Long: `start a master server to provide volume=>location mapping service and sequence number of file ids
|
|
|
|
|
|
|
|
The configuration file "security.toml" is read from ".", "$HOME/.seaweedfs/", or "/etc/seaweedfs/", in that order.
|
|
|
|
|
|
|
|
The example security.toml configuration file can be generated by "weed scaffold -config=security"
|
2012-08-07 16:29:22 +08:00
|
|
|
|
|
|
|
`,
|
|
|
|
}
|
|
|
|
|
|
|
|
var (
|
2019-06-23 18:08:27 +08:00
|
|
|
masterCpuProfile = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file")
|
|
|
|
masterMemProfile = cmdMaster.Flag.String("memprofile", "", "memory profile output file")
|
2012-08-07 16:29:22 +08:00
|
|
|
)
|
|
|
|
|
|
|
|
func runMaster(cmd *Command, args []string) bool {
|
2019-02-19 04:11:52 +08:00
|
|
|
|
2019-06-05 16:30:24 +08:00
|
|
|
util.LoadConfiguration("security", false)
|
|
|
|
util.LoadConfiguration("master", false)
|
2019-02-19 04:11:52 +08:00
|
|
|
|
2019-06-23 18:08:27 +08:00
|
|
|
runtime.GOMAXPROCS(runtime.NumCPU())
|
2020-04-28 14:10:23 +08:00
|
|
|
grace.SetupProfiling(*masterCpuProfile, *masterMemProfile)
|
2017-06-22 16:33:58 +08:00
|
|
|
|
2019-06-23 18:08:27 +08:00
|
|
|
if err := util.TestFolderWritable(*m.metaFolder); err != nil {
|
|
|
|
glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *m.metaFolder, err)
|
2013-12-10 05:27:09 +08:00
|
|
|
}
|
2019-07-28 16:55:05 +08:00
|
|
|
|
|
|
|
var masterWhiteList []string
|
2019-06-23 18:08:27 +08:00
|
|
|
if *m.whiteList != "" {
|
|
|
|
masterWhiteList = strings.Split(*m.whiteList, ",")
|
2013-08-14 00:31:19 +08:00
|
|
|
}
|
2019-06-23 18:08:27 +08:00
|
|
|
if *m.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
|
2017-07-17 12:40:47 +08:00
|
|
|
glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
|
|
|
|
}
|
2013-08-14 00:31:19 +08:00
|
|
|
|
2019-07-28 16:55:05 +08:00
|
|
|
startMaster(m, masterWhiteList)
|
2014-02-15 09:10:49 +08:00
|
|
|
|
2019-07-28 16:55:05 +08:00
|
|
|
return true
|
|
|
|
}
|
2014-04-26 13:09:42 +08:00
|
|
|
|
2019-07-28 16:55:05 +08:00
|
|
|
func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
2019-07-28 18:58:13 +08:00
|
|
|
|
2020-01-30 01:09:55 +08:00
|
|
|
backend.LoadConfiguration(util.GetViper())
|
2019-11-29 10:33:18 +08:00
|
|
|
|
2019-07-28 18:58:13 +08:00
|
|
|
myMasterAddress, peers := checkPeers(*masterOption.ip, *masterOption.port, *masterOption.peers)
|
|
|
|
|
2019-07-28 16:55:05 +08:00
|
|
|
r := mux.NewRouter()
|
2019-07-28 18:58:13 +08:00
|
|
|
ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), peers)
|
2019-07-28 16:55:05 +08:00
|
|
|
listeningAddress := *masterOption.ipBind + ":" + strconv.Itoa(*masterOption.port)
|
2020-06-02 15:10:35 +08:00
|
|
|
glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
|
2019-01-19 06:14:47 +08:00
|
|
|
masterListener, e := util.NewListener(listeningAddress, 0)
|
2014-03-21 02:07:15 +08:00
|
|
|
if e != nil {
|
2015-01-14 09:04:41 +08:00
|
|
|
glog.Fatalf("Master startup error: %v", e)
|
2012-09-29 01:21:06 +08:00
|
|
|
}
|
2019-07-28 16:55:05 +08:00
|
|
|
// start raftServer
|
2020-01-30 01:09:55 +08:00
|
|
|
raftServer := weed_server.NewRaftServer(security.LoadClientTLS(util.GetViper(), "grpc.master"),
|
2020-06-05 01:52:01 +08:00
|
|
|
peers, myMasterAddress, *masterOption.metaFolder, ms.Topo, 5)
|
2019-07-28 16:55:05 +08:00
|
|
|
if raftServer == nil {
|
|
|
|
glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717", *masterOption.metaFolder)
|
|
|
|
}
|
|
|
|
ms.SetRaftServer(raftServer)
|
|
|
|
r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
|
|
|
|
// starting grpc server
|
|
|
|
grpcPort := *masterOption.port + 10000
|
|
|
|
grpcL, err := util.NewListener(*masterOption.ipBind+":"+strconv.Itoa(grpcPort), 0)
|
|
|
|
if err != nil {
|
|
|
|
glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
|
|
|
|
}
|
|
|
|
// Create your protocol servers.
|
2020-03-04 16:39:47 +08:00
|
|
|
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
|
2019-07-28 16:55:05 +08:00
|
|
|
master_pb.RegisterSeaweedServer(grpcS, ms)
|
|
|
|
protobuf.RegisterRaftServer(grpcS, raftServer)
|
|
|
|
reflection.Register(grpcS)
|
2020-06-02 15:10:35 +08:00
|
|
|
glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOption.ipBind, grpcPort)
|
2019-07-28 16:55:05 +08:00
|
|
|
go grpcS.Serve(grpcL)
|
2017-01-10 17:01:12 +08:00
|
|
|
|
2019-07-28 18:58:13 +08:00
|
|
|
go ms.MasterClient.KeepConnectedToMaster()
|
|
|
|
|
2019-01-19 06:14:47 +08:00
|
|
|
// start http server
|
2017-01-10 17:01:12 +08:00
|
|
|
httpS := &http.Server{Handler: r}
|
2019-07-28 16:55:05 +08:00
|
|
|
go httpS.Serve(masterListener)
|
2017-01-10 17:01:12 +08:00
|
|
|
|
2019-07-28 16:55:05 +08:00
|
|
|
select {}
|
2012-08-07 16:29:22 +08:00
|
|
|
}
|
2018-08-13 05:25:31 +08:00
|
|
|
|
|
|
|
func checkPeers(masterIp string, masterPort int, peers string) (masterAddress string, cleanedPeers []string) {
|
2020-04-14 03:58:45 +08:00
|
|
|
glog.V(0).Infof("current: %s:%d peers:%s", masterIp, masterPort, peers)
|
2018-08-13 05:25:31 +08:00
|
|
|
masterAddress = masterIp + ":" + strconv.Itoa(masterPort)
|
|
|
|
if peers != "" {
|
|
|
|
cleanedPeers = strings.Split(peers, ",")
|
|
|
|
}
|
|
|
|
|
|
|
|
hasSelf := false
|
|
|
|
for _, peer := range cleanedPeers {
|
|
|
|
if peer == masterAddress {
|
|
|
|
hasSelf = true
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if !hasSelf {
|
2019-07-28 18:58:13 +08:00
|
|
|
cleanedPeers = append(cleanedPeers, masterAddress)
|
2018-08-13 05:25:31 +08:00
|
|
|
}
|
2019-07-28 18:58:13 +08:00
|
|
|
if len(cleanedPeers)%2 == 0 {
|
2018-08-13 05:25:31 +08:00
|
|
|
glog.Fatalf("Only odd number of masters are supported!")
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
2019-06-23 18:08:27 +08:00
|
|
|
|
|
|
|
func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOption {
|
|
|
|
return &weed_server.MasterOption{
|
2020-04-19 06:17:27 +08:00
|
|
|
Host: *m.ip,
|
2019-06-23 18:08:27 +08:00
|
|
|
Port: *m.port,
|
|
|
|
MetaFolder: *m.metaFolder,
|
|
|
|
VolumeSizeLimitMB: *m.volumeSizeLimitMB,
|
|
|
|
VolumePreallocate: *m.volumePreallocate,
|
2020-06-05 01:52:01 +08:00
|
|
|
// PulseSeconds: *m.pulseSeconds,
|
2019-06-23 18:08:27 +08:00
|
|
|
DefaultReplicaPlacement: *m.defaultReplication,
|
|
|
|
GarbageThreshold: *m.garbageThreshold,
|
|
|
|
WhiteList: whiteList,
|
|
|
|
DisableHttp: *m.disableHttp,
|
|
|
|
MetricsAddress: *m.metricsAddress,
|
|
|
|
MetricsIntervalSec: *m.metricsIntervalSec,
|
|
|
|
}
|
2019-06-24 06:30:16 +08:00
|
|
|
}
|