2016-06-03 09:09:14 +08:00
|
|
|
package command
|
2015-05-26 15:58:41 +08:00
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
2022-11-14 16:19:27 +08:00
|
|
|
|
2022-07-29 15:17:28 +08:00
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
2019-04-19 12:43:36 +08:00
|
|
|
|
2022-07-29 15:17:28 +08:00
|
|
|
"github.com/seaweedfs/seaweedfs/weed/security"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
2015-05-26 15:58:41 +08:00
|
|
|
|
2022-07-29 15:17:28 +08:00
|
|
|
"github.com/seaweedfs/seaweedfs/weed/operation"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/storage"
|
2015-05-26 15:58:41 +08:00
|
|
|
)
|
|
|
|
|
|
|
|
var (
|
|
|
|
s BackupOptions
|
|
|
|
)
|
|
|
|
|
|
|
|
type BackupOptions struct {
|
2019-08-16 11:05:22 +08:00
|
|
|
master *string
|
|
|
|
collection *string
|
|
|
|
dir *string
|
|
|
|
volumeId *int
|
|
|
|
ttl *string
|
|
|
|
replication *string
|
2015-05-26 15:58:41 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
cmdBackup.Run = runBackup // break init cycle
|
|
|
|
s.master = cmdBackup.Flag.String("server", "localhost:9333", "SeaweedFS master location")
|
|
|
|
s.collection = cmdBackup.Flag.String("collection", "", "collection name")
|
|
|
|
s.dir = cmdBackup.Flag.String("dir", ".", "directory to store volume data files")
|
|
|
|
s.volumeId = cmdBackup.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.")
|
2019-08-16 11:05:22 +08:00
|
|
|
s.ttl = cmdBackup.Flag.String("ttl", "", `backup volume's time to live, format:
|
|
|
|
3m: 3 minutes
|
|
|
|
4h: 4 hours
|
|
|
|
5d: 5 days
|
|
|
|
6w: 6 weeks
|
|
|
|
7M: 7 months
|
|
|
|
8y: 8 years
|
|
|
|
default is the same with origin`)
|
|
|
|
s.replication = cmdBackup.Flag.String("replication", "", "backup volume's replication, default is the same with origin")
|
2015-05-26 15:58:41 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
var cmdBackup = &Command{
|
|
|
|
UsageLine: "backup -dir=. -volumeId=234 -server=localhost:9333",
|
|
|
|
Short: "incrementally backup a volume to local folder",
|
|
|
|
Long: `Incrementally backup volume data.
|
2019-04-11 17:18:53 +08:00
|
|
|
|
2015-05-26 15:58:41 +08:00
|
|
|
It is expected that you use this inside a script, to loop through
|
|
|
|
all possible volume ids that needs to be backup to local folder.
|
2019-04-11 17:18:53 +08:00
|
|
|
|
2015-05-26 15:58:41 +08:00
|
|
|
The volume id does not need to exist locally or even remotely.
|
|
|
|
This will help to backup future new volumes.
|
2019-04-11 17:18:53 +08:00
|
|
|
|
2015-05-26 15:58:41 +08:00
|
|
|
Usually backing up is just copying the .dat (and .idx) files.
|
2019-01-17 09:17:19 +08:00
|
|
|
But it's tricky to incrementally copy the differences.
|
2019-04-11 17:18:53 +08:00
|
|
|
|
2015-05-26 15:58:41 +08:00
|
|
|
The complexity comes when there are multiple addition, deletion and compaction.
|
2019-02-06 21:59:15 +08:00
|
|
|
This tool will handle them correctly and efficiently, avoiding unnecessary data transportation.
|
2015-05-26 15:58:41 +08:00
|
|
|
`,
|
|
|
|
}
|
|
|
|
|
|
|
|
func runBackup(cmd *Command, args []string) bool {
|
2019-02-19 04:11:52 +08:00
|
|
|
|
2019-06-05 16:30:24 +08:00
|
|
|
util.LoadConfiguration("security", false)
|
2020-01-30 01:09:55 +08:00
|
|
|
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
|
2019-02-19 04:11:52 +08:00
|
|
|
|
2015-05-26 15:58:41 +08:00
|
|
|
if *s.volumeId == -1 {
|
|
|
|
return false
|
|
|
|
}
|
2019-04-19 12:43:36 +08:00
|
|
|
vid := needle.VolumeId(*s.volumeId)
|
2015-05-26 15:58:41 +08:00
|
|
|
|
|
|
|
// find volume location, replication, ttl info
|
2021-09-13 13:47:52 +08:00
|
|
|
lookup, err := operation.LookupVolumeId(func() pb.ServerAddress { return pb.ServerAddress(*s.master) }, grpcDialOption, vid.String())
|
2015-05-26 15:58:41 +08:00
|
|
|
if err != nil {
|
|
|
|
fmt.Printf("Error looking up volume %d: %v\n", vid, err)
|
|
|
|
return true
|
|
|
|
}
|
2021-09-13 13:47:52 +08:00
|
|
|
volumeServer := lookup.Locations[0].ServerAddress()
|
2015-05-26 15:58:41 +08:00
|
|
|
|
2019-02-19 04:11:52 +08:00
|
|
|
stats, err := operation.GetVolumeSyncStatus(volumeServer, grpcDialOption, uint32(vid))
|
2015-05-26 15:58:41 +08:00
|
|
|
if err != nil {
|
|
|
|
fmt.Printf("Error get volume %d status: %v\n", vid, err)
|
|
|
|
return true
|
|
|
|
}
|
2019-08-16 11:05:22 +08:00
|
|
|
var ttl *needle.TTL
|
|
|
|
if *s.ttl != "" {
|
|
|
|
ttl, err = needle.ReadTTL(*s.ttl)
|
|
|
|
if err != nil {
|
|
|
|
fmt.Printf("Error generate volume %d ttl %s: %v\n", vid, *s.ttl, err)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
ttl, err = needle.ReadTTL(stats.Ttl)
|
|
|
|
if err != nil {
|
|
|
|
fmt.Printf("Error get volume %d ttl %s: %v\n", vid, stats.Ttl, err)
|
|
|
|
return true
|
|
|
|
}
|
2015-05-26 15:58:41 +08:00
|
|
|
}
|
2019-12-24 04:48:20 +08:00
|
|
|
var replication *super_block.ReplicaPlacement
|
2019-08-16 11:05:22 +08:00
|
|
|
if *s.replication != "" {
|
2019-12-24 04:48:20 +08:00
|
|
|
replication, err = super_block.NewReplicaPlacementFromString(*s.replication)
|
2019-08-16 11:05:22 +08:00
|
|
|
if err != nil {
|
|
|
|
fmt.Printf("Error generate volume %d replication %s : %v\n", vid, *s.replication, err)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
} else {
|
2019-12-24 04:48:20 +08:00
|
|
|
replication, err = super_block.NewReplicaPlacementFromString(stats.Replication)
|
2019-08-16 11:05:22 +08:00
|
|
|
if err != nil {
|
|
|
|
fmt.Printf("Error get volume %d replication %s : %v\n", vid, stats.Replication, err)
|
|
|
|
return true
|
|
|
|
}
|
2015-05-26 15:58:41 +08:00
|
|
|
}
|
2022-11-14 16:19:27 +08:00
|
|
|
v, err := storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0, 0)
|
2015-05-26 15:58:41 +08:00
|
|
|
if err != nil {
|
|
|
|
fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2019-04-20 03:29:49 +08:00
|
|
|
if v.SuperBlock.CompactionRevision < uint16(stats.CompactRevision) {
|
2022-05-14 04:46:52 +08:00
|
|
|
if err = v.Compact2(0, 0, nil); err != nil {
|
2019-03-26 00:16:12 +08:00
|
|
|
fmt.Printf("Compact Volume before synchronizing %v\n", err)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
if err = v.CommitCompact(); err != nil {
|
|
|
|
fmt.Printf("Commit Compact before synchronizing %v\n", err)
|
|
|
|
return true
|
|
|
|
}
|
2019-04-20 03:29:49 +08:00
|
|
|
v.SuperBlock.CompactionRevision = uint16(stats.CompactRevision)
|
2019-10-29 15:35:16 +08:00
|
|
|
v.DataBackend.WriteAt(v.SuperBlock.Bytes(), 0)
|
2019-03-26 00:16:12 +08:00
|
|
|
}
|
|
|
|
|
2019-04-19 15:39:34 +08:00
|
|
|
datSize, _, _ := v.FileStat()
|
|
|
|
|
|
|
|
if datSize > stats.TailOffset {
|
2019-03-26 00:16:12 +08:00
|
|
|
// remove the old data
|
|
|
|
v.Destroy()
|
|
|
|
// recreate an empty volume
|
2022-11-14 16:19:27 +08:00
|
|
|
v, err = storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0, 0)
|
2019-03-26 00:16:12 +08:00
|
|
|
if err != nil {
|
|
|
|
fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
defer v.Close()
|
|
|
|
|
2019-04-18 13:04:49 +08:00
|
|
|
if err := v.IncrementalBackup(volumeServer, grpcDialOption); err != nil {
|
2015-05-26 15:58:41 +08:00
|
|
|
fmt.Printf("Error synchronizing volume %d: %v\n", vid, err)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
return true
|
|
|
|
}
|