seaweedfs/weed/command/volume.go

412 lines
15 KiB
Go
Raw Normal View History

package command
import (
2019-11-17 11:40:36 +08:00
"fmt"
2018-10-11 15:08:13 +08:00
"net/http"
2020-06-10 17:52:24 +08:00
httppprof "net/http/pprof"
"os"
2018-10-11 15:08:13 +08:00
"runtime/pprof"
2012-08-24 13:46:54 +08:00
"strconv"
"strings"
2018-10-11 15:08:13 +08:00
"time"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/spf13/viper"
2019-11-17 11:40:36 +08:00
"google.golang.org/grpc"
2020-06-11 04:10:10 +08:00
"github.com/chrislusf/seaweedfs/weed/util/grace"
2020-03-04 16:39:47 +08:00
"github.com/chrislusf/seaweedfs/weed/pb"
2020-01-03 16:37:24 +08:00
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util/httpdown"
"google.golang.org/grpc/reflection"
"github.com/chrislusf/seaweedfs/weed/glog"
2018-10-11 16:16:33 +08:00
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
weed_server "github.com/chrislusf/seaweedfs/weed/server"
2020-09-25 01:21:23 +08:00
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
2018-10-11 15:08:13 +08:00
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/util"
)
2015-01-13 16:27:51 +08:00
var (
v VolumeServerOptions
)
type VolumeServerOptions struct {
port *int
2021-09-12 17:25:15 +08:00
portGrpc *int
publicPort *int
folders []string
folderMaxLimits []int
idxFolder *string
ip *string
publicUrl *string
bindIp *string
mastersString *string
masters []pb.ServerAddress
idleConnectionTimeout *int
dataCenter *string
rack *string
whiteList []string
indexType *string
diskType *string
fixJpgOrientation *bool
readMode *string
cpuProfile *string
memProfile *string
compactionMBPerSecond *int
fileSizeLimitMB *int
concurrentUploadLimitMB *int
concurrentDownloadLimitMB *int
pprof *bool
preStopSeconds *int
metricsHttpPort *int
2020-06-11 04:10:10 +08:00
// pulseSeconds *int
2022-05-20 18:18:20 +08:00
enableTcp *bool
inflightUploadDataTimeout *time.Duration
2015-01-13 16:27:51 +08:00
}
func init() {
2012-08-24 13:46:54 +08:00
cmdVolume.Run = runVolume // break init cycle
2015-01-13 16:27:51 +08:00
v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
2021-09-21 05:05:59 +08:00
v.portGrpc = cmdVolume.Flag.Int("port.grpc", 0, "grpc listen port")
v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
v.bindIp = cmdVolume.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
v.mastersString = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
v.preStopSeconds = cmdVolume.Flag.Int("preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
// v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
2017-01-10 17:01:12 +08:00
v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds")
2015-01-13 16:27:51 +08:00
v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
2021-02-22 18:03:12 +08:00
v.diskType = cmdVolume.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
2021-07-04 06:55:56 +08:00
v.readMode = cmdVolume.Flag.String("readMode", "proxy", "[local|proxy|redirect] how to deal with non-local volume: 'not found|proxy to remote node|redirect volume location'.")
2017-06-22 16:33:58 +08:00
v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
v.concurrentUploadLimitMB = cmdVolume.Flag.Int("concurrentUploadLimitMB", 256, "limit total concurrent upload size")
v.concurrentDownloadLimitMB = cmdVolume.Flag.Int("concurrentDownloadLimitMB", 256, "limit total concurrent download size")
2020-06-10 17:52:24 +08:00
v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
2020-09-24 20:45:39 +08:00
v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
2022-02-28 15:11:09 +08:00
v.enableTcp = cmdVolume.Flag.Bool("tcp", false, "<experimental> enable tcp port")
2022-05-20 18:18:20 +08:00
v.inflightUploadDataTimeout = cmdVolume.Flag.Duration("inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers")
}
var cmdVolume = &Command{
2012-09-26 16:55:56 +08:00
UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333",
2012-08-24 13:46:54 +08:00
Short: "start a volume server",
Long: `start a volume server to provide storage spaces
`,
}
var (
volumeFolders = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
2022-01-14 05:03:04 +08:00
maxVolumeCounts = cmdVolume.Flag.String("max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured as free disk space divided by volume size.")
2013-08-14 00:22:06 +08:00
volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
minFreeSpacePercent = cmdVolume.Flag.String("minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly (deprecated, use minFreeSpace instead).")
minFreeSpace = cmdVolume.Flag.String("minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.")
)
func runVolume(cmd *Command, args []string) bool {
2019-02-19 04:11:52 +08:00
util.LoadConfiguration("security", false)
2019-02-19 04:11:52 +08:00
2020-06-10 17:52:24 +08:00
// If --pprof is set we assume the caller wants to be able to collect
// cpu and memory profiles via go tool pprof
if !*v.pprof {
grace.SetupProfiling(*v.cpuProfile, *v.memProfile)
}
2015-01-13 16:27:51 +08:00
2020-09-25 01:21:23 +08:00
go stats_collect.StartMetricsServer(*v.metricsHttpPort)
2021-04-27 10:37:24 +08:00
minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent)
v.masters = pb.ServerAddresses(*v.mastersString).ToAddresses()
2021-04-27 10:37:24 +08:00
v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, minFreeSpaces)
return true
}
2021-04-27 10:37:24 +08:00
func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption string, minFreeSpaces []util.MinFreeSpace) {
2019-11-17 11:40:36 +08:00
// Set multiple folders and each folder's max volume count limit'
v.folders = strings.Split(volumeFolders, ",")
2020-06-21 23:38:00 +08:00
for _, folder := range v.folders {
if err := util.TestFolderWritable(util.ResolvePath(folder)); err != nil {
2020-06-21 23:38:00 +08:00
glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
}
}
// set max
maxCountStrings := strings.Split(maxVolumeCounts, ",")
for _, maxString := range maxCountStrings {
if max, e := strconv.Atoi(maxString); e == nil {
2015-01-13 16:27:51 +08:00
v.folderMaxLimits = append(v.folderMaxLimits, max)
} else {
glog.Fatalf("The max specified in -max not a valid number %s", maxString)
}
}
if len(v.folderMaxLimits) == 1 && len(v.folders) > 1 {
for i := 0; i < len(v.folders)-1; i++ {
v.folderMaxLimits = append(v.folderMaxLimits, v.folderMaxLimits[0])
}
}
2015-01-13 16:27:51 +08:00
if len(v.folders) != len(v.folderMaxLimits) {
glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(v.folders), len(v.folderMaxLimits))
}
2020-06-21 23:38:00 +08:00
2021-04-27 10:37:24 +08:00
if len(minFreeSpaces) == 1 && len(v.folders) > 1 {
for i := 0; i < len(v.folders)-1; i++ {
2021-04-27 10:37:24 +08:00
minFreeSpaces = append(minFreeSpaces, minFreeSpaces[0])
}
}
2021-04-27 10:37:24 +08:00
if len(v.folders) != len(minFreeSpaces) {
glog.Fatalf("%d directories by -dir, but only %d minFreeSpacePercent is set by -minFreeSpacePercent", len(v.folders), len(minFreeSpaces))
}
2015-01-13 16:36:44 +08:00
2020-12-14 14:49:56 +08:00
// set disk types
2021-02-16 18:47:02 +08:00
var diskTypes []types.DiskType
2020-12-14 14:49:56 +08:00
diskTypeStrings := strings.Split(*v.diskType, ",")
for _, diskTypeString := range diskTypeStrings {
2021-02-16 18:47:02 +08:00
diskTypes = append(diskTypes, types.ToDiskType(diskTypeString))
2020-12-14 14:49:56 +08:00
}
if len(diskTypes) == 1 && len(v.folders) > 1 {
for i := 0; i < len(v.folders)-1; i++ {
diskTypes = append(diskTypes, diskTypes[0])
}
}
if len(v.folders) != len(diskTypes) {
glog.Fatalf("%d directories by -dir, but only %d disk types is set by -disk", len(v.folders), len(diskTypes))
}
2019-11-17 11:40:36 +08:00
// security related white list configuration
if volumeWhiteListOption != "" {
v.whiteList = strings.Split(volumeWhiteListOption, ",")
2015-01-13 16:27:51 +08:00
}
2012-09-26 18:27:10 +08:00
if *v.ip == "" {
*v.ip = util.DetectedHostAddress()
2020-04-10 14:42:59 +08:00
glog.V(0).Infof("detected volume server ip address: %v", *v.ip)
2012-09-26 17:29:16 +08:00
}
if *v.bindIp == "" {
*v.bindIp = *v.ip
}
if *v.publicPort == 0 {
*v.publicPort = *v.port
}
2021-09-21 05:05:59 +08:00
if *v.portGrpc == 0 {
*v.portGrpc = 10000 + *v.port
}
2015-04-09 02:08:08 +08:00
if *v.publicUrl == "" {
2021-09-08 10:29:42 +08:00
*v.publicUrl = util.JoinHostPort(*v.ip, *v.publicPort)
2015-04-09 02:08:08 +08:00
}
volumeMux := http.NewServeMux()
publicVolumeMux := volumeMux
2019-11-17 11:40:36 +08:00
if v.isSeparatedPublicPort() {
publicVolumeMux = http.NewServeMux()
}
2012-08-24 13:46:54 +08:00
2020-06-10 17:52:24 +08:00
if *v.pprof {
volumeMux.HandleFunc("/debug/pprof/", httppprof.Index)
volumeMux.HandleFunc("/debug/pprof/cmdline", httppprof.Cmdline)
volumeMux.HandleFunc("/debug/pprof/profile", httppprof.Profile)
volumeMux.HandleFunc("/debug/pprof/symbol", httppprof.Symbol)
volumeMux.HandleFunc("/debug/pprof/trace", httppprof.Trace)
}
volumeNeedleMapKind := storage.NeedleMapInMemory
switch *v.indexType {
case "leveldb":
volumeNeedleMapKind = storage.NeedleMapLevelDb
case "leveldbMedium":
volumeNeedleMapKind = storage.NeedleMapLevelDbMedium
case "leveldbLarge":
volumeNeedleMapKind = storage.NeedleMapLevelDbLarge
}
volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
*v.ip, *v.port, *v.portGrpc, *v.publicUrl,
2021-04-27 10:37:24 +08:00
v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes,
2020-12-14 14:29:52 +08:00
*v.idxFolder,
volumeNeedleMapKind,
v.masters, 5, *v.dataCenter, *v.rack,
2015-01-13 16:27:51 +08:00
v.whiteList,
*v.fixJpgOrientation, *v.readMode,
*v.compactionMBPerSecond,
2020-01-03 16:37:24 +08:00
*v.fileSizeLimitMB,
2021-04-02 17:22:26 +08:00
int64(*v.concurrentUploadLimitMB)*1024*1024,
int64(*v.concurrentDownloadLimitMB)*1024*1024,
2022-05-20 18:18:20 +08:00
*v.inflightUploadDataTimeout,
)
2019-11-17 11:40:36 +08:00
// starting grpc server
grpcS := v.startGrpcService(volumeServer)
// starting public http server
var publicHttpDown httpdown.Server
if v.isSeparatedPublicPort() {
publicHttpDown = v.startPublicHttpService(publicVolumeMux)
if nil == publicHttpDown {
glog.Fatalf("start public http service failed")
}
}
// starting tcp server
2021-03-08 06:45:36 +08:00
if *v.enableTcp {
go v.startTcpService(volumeServer)
}
2019-11-17 11:40:36 +08:00
// starting the cluster http server
clusterHttpServer := v.startClusterHttpService(volumeMux)
2020-09-14 12:25:51 +08:00
stopChan := make(chan bool)
grace.OnInterrupt(func() {
fmt.Println("volume server has been killed")
2019-11-17 11:40:36 +08:00
2020-09-14 12:25:51 +08:00
// Stop heartbeats
if !volumeServer.StopHeartbeat() {
volumeServer.SetStopping()
2020-09-14 12:25:51 +08:00
glog.V(0).Infof("stop send heartbeat and wait %d seconds until shutdown ...", *v.preStopSeconds)
time.Sleep(time.Duration(*v.preStopSeconds) * time.Second)
}
2019-11-17 11:40:36 +08:00
2020-09-14 12:25:51 +08:00
shutdown(publicHttpDown, clusterHttpServer, grpcS, volumeServer)
stopChan <- true
2020-09-14 03:41:26 +08:00
})
2019-11-17 11:40:36 +08:00
2020-09-14 12:25:51 +08:00
select {
case <-stopChan:
}
2019-11-17 11:40:36 +08:00
2020-09-14 03:41:26 +08:00
}
2019-11-17 11:40:36 +08:00
2020-09-14 12:25:51 +08:00
func shutdown(publicHttpDown httpdown.Server, clusterHttpServer httpdown.Server, grpcS *grpc.Server, volumeServer *weed_server.VolumeServer) {
2019-11-17 11:40:36 +08:00
2020-09-14 03:41:26 +08:00
// firstly, stop the public http service to prevent from receiving new user request
if nil != publicHttpDown {
glog.V(0).Infof("stop public http server ... ")
if err := publicHttpDown.Stop(); err != nil {
glog.Warningf("stop the public http server failed, %v", err)
}
}
2014-05-13 15:04:28 +08:00
2020-09-14 03:41:26 +08:00
glog.V(0).Infof("graceful stop cluster http server ... ")
if err := clusterHttpServer.Stop(); err != nil {
glog.Warningf("stop the cluster http server failed, %v", err)
2019-11-17 11:40:36 +08:00
}
2020-09-14 03:41:26 +08:00
glog.V(0).Infof("graceful stop gRPC ...")
grpcS.GracefulStop()
volumeServer.Shutdown()
pprof.StopCPUProfile()
2019-11-17 11:40:36 +08:00
}
// check whether configure the public port
func (v VolumeServerOptions) isSeparatedPublicPort() bool {
return *v.publicPort != *v.port
}
func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerServer) *grpc.Server {
2021-09-12 17:25:15 +08:00
grpcPort := *v.portGrpc
2021-09-08 10:29:42 +08:00
grpcL, err := util.NewListener(util.JoinHostPort(*v.bindIp, grpcPort), 0)
2018-10-11 16:16:33 +08:00
if err != nil {
glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
}
2020-03-04 16:39:47 +08:00
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.volume"))
2019-11-17 11:40:36 +08:00
volume_server_pb.RegisterVolumeServerServer(grpcS, vs)
2018-10-11 16:16:33 +08:00
reflection.Register(grpcS)
2019-11-17 11:40:36 +08:00
go func() {
if err := grpcS.Serve(grpcL); err != nil {
glog.Fatalf("start gRPC service failed, %s", err)
2019-02-25 16:43:36 +08:00
}
2019-11-17 11:40:36 +08:00
}()
return grpcS
}
func (v VolumeServerOptions) startPublicHttpService(handler http.Handler) httpdown.Server {
2021-09-08 10:29:42 +08:00
publicListeningAddress := util.JoinHostPort(*v.bindIp, *v.publicPort)
2020-06-02 15:10:35 +08:00
glog.V(0).Infoln("Start Seaweed volume server", util.Version(), "public at", publicListeningAddress)
2019-11-17 11:40:36 +08:00
publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
if e != nil {
glog.Fatalf("Volume server listener error:%v", e)
}
pubHttp := httpdown.HTTP{StopTimeout: 5 * time.Minute, KillTimeout: 5 * time.Minute}
publicHttpDown := pubHttp.Serve(&http.Server{Handler: handler}, publicListener)
go func() {
if err := publicHttpDown.Wait(); err != nil {
glog.Errorf("public http down wait failed, %v", err)
2019-02-25 16:43:36 +08:00
}
2019-11-17 11:40:36 +08:00
}()
return publicHttpDown
}
func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpdown.Server {
var (
certFile, keyFile string
)
if viper.GetString("https.volume.key") != "" {
certFile = viper.GetString("https.volume.cert")
keyFile = viper.GetString("https.volume.key")
}
2021-09-08 10:29:42 +08:00
listeningAddress := util.JoinHostPort(*v.bindIp, *v.port)
2020-06-02 15:10:35 +08:00
glog.V(0).Infof("Start Seaweed volume server %s at %s", util.Version(), listeningAddress)
2019-11-17 11:40:36 +08:00
listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
if e != nil {
glog.Fatalf("Volume server listener error:%v", e)
2012-08-24 13:46:54 +08:00
}
2019-11-17 11:40:36 +08:00
httpDown := httpdown.HTTP{
KillTimeout: time.Minute,
2022-02-15 00:42:27 +08:00
StopTimeout: 30 * time.Second,
2019-11-17 11:40:36 +08:00
CertFile: certFile,
KeyFile: keyFile}
httpS := &http.Server{Handler: handler}
if viper.GetString("https.volume.ca") != "" {
clientCertFile := viper.GetString("https.volume.ca")
httpS.TLSConfig = security.LoadClientTLSHTTP(clientCertFile)
}
clusterHttpServer := httpDown.Serve(httpS, listener)
2019-11-17 11:40:36 +08:00
go func() {
if e := clusterHttpServer.Wait(); e != nil {
glog.Fatalf("Volume server fail to serve: %v", e)
}
}()
return clusterHttpServer
}
func (v VolumeServerOptions) startTcpService(volumeServer *weed_server.VolumeServer) {
listeningAddress := util.JoinHostPort(*v.bindIp, *v.port+20000)
glog.V(0).Infoln("Start Seaweed volume server", util.Version(), "tcp at", listeningAddress)
listener, e := util.NewListener(listeningAddress, 0)
if e != nil {
glog.Fatalf("Volume server listener error on %s:%v", listeningAddress, e)
}
defer listener.Close()
for {
c, err := listener.Accept()
if err != nil {
fmt.Println(err)
return
}
go volumeServer.HandleTcpConnection(c)
}
}