mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-11-28 21:39:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #145 from dyeldandi/incrementalExport

Browse Source 
Using LastModified in export to export only new files and fill up ctime and mtime in tar archive
This commit is contained in:
chrislusf 2015-05-20 09:38:52 -07:00
commit ebf8e9f663
1 changed files with 27 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
go/weed/export.go
View File

@ -21,10 +21,11 @@ func init() {
const (
defaultFnFormat = `{{.Mime}}/{{.Id}}:{{.Name}}`
timeFormat = "2006-01-02T15:04:05"
)
var cmdExport = &Command{
UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}}",
UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}} -newer='"+timeFormat+"'",
Short: "list or export files from one volume data file",
Long: `List all files in a volume, or Export all files in a volume to a tar file if the output is specified.
@ -39,19 +40,32 @@ var (
exportVolumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.")
dest = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout")
format = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename format, default to {{.Mime}}/{{.Id}}:{{.Name}}")
newer = cmdExport.Flag.String("newer", "", "export only files newer than this time, default is all files. Must be specified in RFC3339 without timezone")
tarFh *tar.Writer
tarHeader tar.Header
fnTmpl *template.Template
fnTmplBuf = bytes.NewBuffer(nil)
newerThan time.Time
newerThanUnix int64 = -1
localLocation, _ = time.LoadLocation("Local")
)
func runExport(cmd *Command, args []string) bool {
var err error
if *newer != "" {
if newerThan, err = time.ParseInLocation(timeFormat, *newer, localLocation); err != nil {
fmt.Println("cannot parse 'newer' argument: " + err.Error())
return false
}
newerThanUnix = newerThan.Unix()
}
if *exportVolumeId == -1 {
return false
}
var err error
if *dest != "" {
if *dest != "-" && !strings.HasSuffix(*dest, ".tar") {
fmt.Println("the output file", *dest, "should be '-' or end with .tar")
@ -109,6 +123,11 @@ func runExport(cmd *Command, args []string) bool {
glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
n.Id, offset, n.Size, n.DiskSize(), n.IsGzipped(), ok, nv)
if ok && nv.Size > 0 && int64(nv.Offset)*8 == offset {
if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
n.LastModified, newerThanUnix)
return nil
}
return walker(vid, n, version)
}
if !ok {
@ -151,6 +170,12 @@ func walker(vid storage.VolumeId, n *storage.Needle, version storage.Version) (e
}
tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data))
if n.HasLastModifiedDate() {
tarHeader.ModTime = time.Unix(int64(n.LastModified), 0)
} else {
tarHeader.ModTime = time.Unix(0,0)
}
tarHeader.ChangeTime = tarHeader.ModTime
if err = tarFh.WriteHeader(&tarHeader); err != nil {
return err
}