diff --git a/go.mod b/go.mod index 53188faf2..8fdbf5f60 100644 --- a/go.mod +++ b/go.mod @@ -90,6 +90,7 @@ require ( gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect modernc.org/sqlite v1.10.7 + github.com/posener/complete v1.2.3 ) // replace github.com/seaweedfs/fuse => /Users/chris/go/src/github.com/seaweedfs/fuse diff --git a/go.sum b/go.sum index c96845979..f23ca0020 100644 --- a/go.sum +++ b/go.sum @@ -364,10 +364,12 @@ github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= @@ -571,6 +573,8 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/pquerna/cachecontrol v0.1.0 h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8cTqKc= github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= diff --git a/k8s/seaweedfs/Chart.yaml b/k8s/seaweedfs/Chart.yaml index cc8e4bdca..83b0148e7 100644 --- a/k8s/seaweedfs/Chart.yaml +++ b/k8s/seaweedfs/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -appVersion: "2.60" -version: "2.60" +appVersion: "2.61" +version: "2.61" diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml index 3f1c6da1c..adc0f1d2c 100644 --- a/k8s/seaweedfs/values.yaml +++ b/k8s/seaweedfs/values.yaml @@ -4,7 +4,6 @@ global: registry: "" repository: "" imageName: chrislusf/seaweedfs - # imageTag: "2.60" - started using {.Chart.appVersion} imagePullPolicy: IfNotPresent imagePullSecrets: imagepullsecret restartPolicy: Always diff --git a/weed/command/autocomplete.go b/weed/command/autocomplete.go new file mode 100644 index 000000000..9a545a183 --- /dev/null +++ b/weed/command/autocomplete.go @@ -0,0 +1,109 @@ +package command + +import ( + "fmt" + flag "github.com/chrislusf/seaweedfs/weed/util/fla9" + "github.com/posener/complete" + completeinstall "github.com/posener/complete/cmd/install" + "runtime" +) + +func AutocompleteMain(commands []*Command) bool { + subCommands := make(map[string]complete.Command) + helpSubCommands := make(map[string]complete.Command) + for _, cmd := range commands { + flags := make(map[string]complete.Predictor) + cmd.Flag.VisitAll(func(flag *flag.Flag) { + flags["-"+flag.Name] = complete.PredictAnything + }) + + subCommands[cmd.Name()] = complete.Command{ + Flags: flags, + } + helpSubCommands[cmd.Name()] = complete.Command{} + } + subCommands["help"] = complete.Command{Sub: helpSubCommands} + + globalFlags := make(map[string]complete.Predictor) + flag.VisitAll(func(flag *flag.Flag) { + globalFlags["-"+flag.Name] = complete.PredictAnything + }) + + weedCmd := complete.Command{ + Sub: subCommands, + Flags: globalFlags, + GlobalFlags: complete.Flags{"-h": complete.PredictNothing}, + } + cmp := complete.New("weed", weedCmd) + + return cmp.Complete() +} + +func installAutoCompletion() bool { + if runtime.GOOS == "windows" { + fmt.Println("windows is not supported") + return false + } + + err := completeinstall.Install("weed") + if err != nil { + fmt.Printf("install failed! %s\n", err) + return false + } + fmt.Printf("autocompletion is enabled. Please restart your shell.\n") + return true +} + +func uninstallAutoCompletion() bool { + if runtime.GOOS == "windows" { + fmt.Println("windows is not supported") + return false + } + + err := completeinstall.Uninstall("weed") + if err != nil { + fmt.Printf("uninstall failed! %s\n", err) + return false + } + fmt.Printf("autocompletion is disable. Please restart your shell.\n") + return true +} + +var cmdAutocomplete = &Command{ + Run: runAutocomplete, + UsageLine: "autocomplete", + Short: "install autocomplete", + Long: `weed autocomplete is installed in the shell. + + Supported shells are bash, zsh, and fish. + Windows is not supported. + +`, +} + +func runAutocomplete(cmd *Command, args []string) bool { + if len(args) != 0 { + cmd.Usage() + } + + return installAutoCompletion() +} + +var cmdUnautocomplete = &Command{ + Run: runUnautocomplete, + UsageLine: "autocomplete.uninstall", + Short: "uninstall autocomplete", + Long: `weed autocomplete is uninstalled in the shell. + + Windows is not supported. + +`, +} + +func runUnautocomplete(cmd *Command, args []string) bool { + if len(args) != 0 { + cmd.Usage() + } + + return uninstallAutoCompletion() +} diff --git a/weed/command/command.go b/weed/command/command.go index 0bac56442..9ae93fe61 100644 --- a/weed/command/command.go +++ b/weed/command/command.go @@ -8,6 +8,8 @@ import ( ) var Commands = []*Command{ + cmdAutocomplete, + cmdUnautocomplete, cmdBackup, cmdBenchmark, cmdCompact, diff --git a/weed/command/filer_backup.go b/weed/command/filer_backup.go index 888b46fe7..fc4dd8298 100644 --- a/weed/command/filer_backup.go +++ b/weed/command/filer_backup.go @@ -52,11 +52,11 @@ var cmdFilerBackup = &Command{ func runFilerBackup(cmd *Command, args []string) bool { - grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") - util.LoadConfiguration("security", false) util.LoadConfiguration("replication", true) + grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") + for { err := doFilerBackup(grpcDialOption, &filerBackupOptions) if err != nil { diff --git a/weed/command/filer_meta_backup.go b/weed/command/filer_meta_backup.go index ba0b44659..28bd367e7 100644 --- a/weed/command/filer_meta_backup.go +++ b/weed/command/filer_meta_backup.go @@ -53,6 +53,7 @@ The backup writes to another filer store specified in a backup_filer.toml. func runFilerMetaBackup(cmd *Command, args []string) bool { + util.LoadConfiguration("security", false) metaBackup.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") // load backup_filer.toml diff --git a/weed/command/filer_meta_tail.go b/weed/command/filer_meta_tail.go index 8451ffd78..76699bb5e 100644 --- a/weed/command/filer_meta_tail.go +++ b/weed/command/filer_meta_tail.go @@ -45,6 +45,7 @@ var ( func runFilerMetaTail(cmd *Command, args []string) bool { + util.LoadConfiguration("security", false) grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") var filterFunc func(dir, fname string) bool diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go index 211c34aea..7cfc8a7fe 100644 --- a/weed/command/filer_sync.go +++ b/weed/command/filer_sync.go @@ -89,6 +89,7 @@ var cmdFilerSynchronize = &Command{ func runFilerSynchronize(cmd *Command, args []string) bool { + util.LoadConfiguration("security", false) grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") grace.SetupProfiling(*syncCpuProfile, *syncMemProfile) diff --git a/weed/command/iam.go b/weed/command/iam.go index 17d0832cb..ed4eea543 100644 --- a/weed/command/iam.go +++ b/weed/command/iam.go @@ -49,6 +49,7 @@ func (iamopt *IamOptions) startIamServer() bool { return false } + util.LoadConfiguration("security", false) grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") for { err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { diff --git a/weed/command/scaffold/master.toml b/weed/command/scaffold/master.toml index cdd92016c..020f48e36 100644 --- a/weed/command/scaffold/master.toml +++ b/weed/command/scaffold/master.toml @@ -11,7 +11,7 @@ scripts = """ ec.encode -fullPercent=95 -quietFor=1h ec.rebuild -force ec.balance -force - volume.deleteEmpty -quietFor=24h + volume.deleteEmpty -quietFor=24h -force volume.balance -force volume.fix.replication unlock diff --git a/weed/command/upload.go b/weed/command/upload.go index ccdec561f..9ae1befab 100644 --- a/weed/command/upload.go +++ b/weed/command/upload.go @@ -71,13 +71,13 @@ func runUpload(cmd *Command, args []string) bool { util.LoadConfiguration("security", false) grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") - defaultCollection, err := readMasterConfiguration(grpcDialOption, *upload.master) + defaultReplication, err := readMasterConfiguration(grpcDialOption, *upload.master) if err != nil { fmt.Printf("upload: %v", err) return false } if *upload.replication == "" { - *upload.replication = defaultCollection + *upload.replication = defaultReplication } if len(args) == 0 { diff --git a/weed/shell/command_volume_balance.go b/weed/shell/command_volume_balance.go index ad7da0e44..6da128c68 100644 --- a/weed/shell/command_volume_balance.go +++ b/weed/shell/command_volume_balance.go @@ -120,6 +120,21 @@ func balanceVolumeServers(commandEnv *CommandEnv, diskTypes []types.DiskType, vo func balanceVolumeServersByDiskType(commandEnv *CommandEnv, diskType types.DiskType, volumeReplicas map[uint32][]*VolumeReplica, nodes []*Node, volumeSizeLimit uint64, collection string, applyBalancing bool) error { + // balance read only volumes + for _, n := range nodes { + n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool { + if collection != "ALL_COLLECTIONS" { + if v.Collection != collection { + return false + } + } + return v.DiskType == string(diskType) && (v.ReadOnly || v.Size >= volumeSizeLimit) + }) + } + if err := balanceSelectedVolume(commandEnv, volumeReplicas, nodes, capacityByMaxVolumeCount(diskType), sortReadOnlyVolumes, applyBalancing); err != nil { + return err + } + // balance writable volumes for _, n := range nodes { n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool { @@ -135,21 +150,6 @@ func balanceVolumeServersByDiskType(commandEnv *CommandEnv, diskType types.DiskT return err } - // balance readable volumes - for _, n := range nodes { - n.selectVolumes(func(v *master_pb.VolumeInformationMessage) bool { - if collection != "ALL_COLLECTIONS" { - if v.Collection != collection { - return false - } - } - return v.DiskType == string(diskType) && (v.ReadOnly || v.Size >= volumeSizeLimit) - }) - } - if err := balanceSelectedVolume(commandEnv, volumeReplicas, nodes, capacityByMaxVolumeCount(diskType), sortReadOnlyVolumes, applyBalancing); err != nil { - return err - } - return nil } diff --git a/weed/shell/command_volume_delete_empty.go b/weed/shell/command_volume_delete_empty.go index 07c6d5b2c..079915f66 100644 --- a/weed/shell/command_volume_delete_empty.go +++ b/weed/shell/command_volume_delete_empty.go @@ -3,11 +3,10 @@ package shell import ( "flag" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "github.com/chrislusf/seaweedfs/weed/storage/needle" "io" "log" "time" - - "github.com/chrislusf/seaweedfs/weed/storage/needle" ) func init() { @@ -24,7 +23,7 @@ func (c *commandVolumeDeleteEmpty) Name() string { func (c *commandVolumeDeleteEmpty) Help() string { return `delete empty volumes from all volume servers - volume.deleteEmpty -quietFor=24h + volume.deleteEmpty -quietFor=24h -force This command deletes all empty volumes from one volume server. @@ -39,6 +38,7 @@ func (c *commandVolumeDeleteEmpty) Do(args []string, commandEnv *CommandEnv, wri volDeleteCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) quietPeriod := volDeleteCommand.Duration("quietFor", 24*time.Hour, "select empty volumes with no recent writes, avoid newly created ones") + applyBalancing := volDeleteCommand.Bool("force", false, "apply to delete empty volumes") if err = volDeleteCommand.Parse(args); err != nil { return nil } @@ -55,10 +55,15 @@ func (c *commandVolumeDeleteEmpty) Do(args []string, commandEnv *CommandEnv, wri eachDataNode(topologyInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) { for _, diskInfo := range dn.DiskInfos { for _, v := range diskInfo.VolumeInfos { - if v.Size <= 8 && v.ModifiedAtSecond + quietSeconds < nowUnixSeconds { - log.Printf("deleting empty volume %d from %s", v.Id, dn.Id) - if deleteErr := deleteVolume(commandEnv.option.GrpcDialOption, needle.VolumeId(v.Id), dn.Id); deleteErr != nil { - err = deleteErr + if v.Size <= 8 && v.ModifiedAtSecond+quietSeconds < nowUnixSeconds { + if *applyBalancing { + log.Printf("deleting empty volume %d from %s", v.Id, dn.Id) + if deleteErr := deleteVolume(commandEnv.option.GrpcDialOption, needle.VolumeId(v.Id), dn.Id); deleteErr != nil { + err = deleteErr + } + continue + } else { + log.Printf("empty volume %d from %s", v.Id, dn.Id) } } } diff --git a/weed/shell/command_volume_fix_replication.go b/weed/shell/command_volume_fix_replication.go index 9e6280788..7e7c0a93d 100644 --- a/weed/shell/command_volume_fix_replication.go +++ b/weed/shell/command_volume_fix_replication.go @@ -160,7 +160,7 @@ func (c *commandVolumeFixReplication) fixUnderReplicatedVolumes(commandEnv *Comm for _, vid := range underReplicatedVolumeIds { for i := 0; i < retryCount+1; i++ { if err = c.fixOneUnderReplicatedVolume(commandEnv, writer, takeAction, volumeReplicas, vid, allLocations); err == nil { - continue + break } } } diff --git a/weed/util/constants.go b/weed/util/constants.go index 2636adf45..108dfb2e4 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -5,7 +5,7 @@ import ( ) var ( - VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 60) + VERSION = fmt.Sprintf("%s %.02f", sizeLimit, 2.61) COMMIT = "" ) diff --git a/weed/weed.go b/weed/weed.go index 91c17d9ff..068d2077c 100644 --- a/weed/weed.go +++ b/weed/weed.go @@ -46,6 +46,11 @@ func main() { glog.MaxSize = 1024 * 1024 * 32 rand.Seed(time.Now().UnixNano()) flag.Usage = usage + + if command.AutocompleteMain(commands) { + return + } + flag.Parse() args := flag.Args()