seaweedfs/weed/filer/embedded_filer/directory_in_map.go

package embedded_filer

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var writeLock sync.Mutex //serialize changes to dir.log

type DirectoryEntryInMap struct {
	sync.Mutex
	Name           string
	Parent         *DirectoryEntryInMap
	subDirectories map[string]*DirectoryEntryInMap
	Id             filer.DirectoryId
}

func (de *DirectoryEntryInMap) getChild(dirName string) (*DirectoryEntryInMap, bool) {
	de.Lock()
	defer de.Unlock()
	child, ok := de.subDirectories[dirName]
	return child, ok
}
func (de *DirectoryEntryInMap) addChild(dirName string, child *DirectoryEntryInMap) {
	de.Lock()
	defer de.Unlock()
	de.subDirectories[dirName] = child
}
func (de *DirectoryEntryInMap) removeChild(dirName string) {
	de.Lock()
	defer de.Unlock()
	delete(de.subDirectories, dirName)
}
func (de *DirectoryEntryInMap) hasChildren() bool {
	de.Lock()
	defer de.Unlock()
	return len(de.subDirectories) > 0
}
func (de *DirectoryEntryInMap) children() (dirNames []filer.DirectoryEntry) {
	de.Lock()
	defer de.Unlock()
	for k, v := range de.subDirectories {
		dirNames = append(dirNames, filer.DirectoryEntry{Name: k, Id: v.Id})
	}
	return dirNames
}

type DirectoryManagerInMap struct {
	Root      *DirectoryEntryInMap
	max       filer.DirectoryId
	logFile   *os.File
	isLoading bool
}

func (dm *DirectoryManagerInMap) newDirectoryEntryInMap(parent *DirectoryEntryInMap, name string) (d *DirectoryEntryInMap, err error) {
	d = &DirectoryEntryInMap{Name: name, Parent: parent, subDirectories: make(map[string]*DirectoryEntryInMap)}
	var parts []string
	for p := d; p != nil && p.Name != ""; p = p.Parent {
		parts = append(parts, p.Name)
	}
	n := len(parts)
	if n <= 0 {
		return nil, fmt.Errorf("Failed to create folder %s/%s", parent.Name, name)
	}
	for i := 0; i < n/2; i++ {
		parts[i], parts[n-1-i] = parts[n-1-i], parts[i]
	}
	dm.max++
	d.Id = dm.max
	dm.log("add", "/"+strings.Join(parts, "/"), strconv.Itoa(int(d.Id)))
	return d, nil
}

func (dm *DirectoryManagerInMap) log(words ...string) {
	if !dm.isLoading {
		dm.logFile.WriteString(strings.Join(words, "\t") + "\n")
	}
}

func NewDirectoryManagerInMap(dirLogFile string) (dm *DirectoryManagerInMap, err error) {
	dm = &DirectoryManagerInMap{}
	//dm.Root do not use newDirectoryEntryInMap, since dm.max will be changed
	dm.Root = &DirectoryEntryInMap{subDirectories: make(map[string]*DirectoryEntryInMap)}
	if dm.logFile, err = os.OpenFile(dirLogFile, os.O_RDWR|os.O_CREATE, 0644); err != nil {
		return nil, fmt.Errorf("cannot write directory log file %s: %v", dirLogFile, err)
	}
	return dm, dm.load()
}

func (dm *DirectoryManagerInMap) processEachLine(line string) error {
	if strings.HasPrefix(line, "#") {
		return nil
	}
	if line == "" {
		return nil
	}
	parts := strings.Split(line, "\t")
	if len(parts) == 0 {
		return nil
	}
	switch parts[0] {
	case "add":
		v, pe := strconv.Atoi(parts[2])
		if pe != nil {
			return pe
		}
		if e := dm.loadDirectory(parts[1], filer.DirectoryId(v)); e != nil {
			return e
		}
	case "mov":
		newName := ""
		if len(parts) >= 4 {
			newName = parts[3]
		}
		if e := dm.MoveUnderDirectory(parts[1], parts[2], newName); e != nil {
			return e
		}
	case "del":
		if e := dm.DeleteDirectory(parts[1]); e != nil {
			return e
		}
	default:
		fmt.Printf("line %s has %s!\n", line, parts[0])
		return nil
	}
	return nil
}
func (dm *DirectoryManagerInMap) load() error {
	dm.max = 0
	lines := bufio.NewReader(dm.logFile)
	dm.isLoading = true
	defer func() { dm.isLoading = false }()
	for {
		line, err := util.Readln(lines)
		if err != nil && err != io.EOF {
			return err
		}
		if pe := dm.processEachLine(string(line)); pe != nil {
			return pe
		}
		if err == io.EOF {
			return nil
		}
	}
}

func (dm *DirectoryManagerInMap) findDirectory(dirPath string) (*DirectoryEntryInMap, error) {
	if dirPath == "" {
		return dm.Root, nil
	}
	dirPath = CleanFilePath(dirPath)
	if dirPath == "/" {
		return dm.Root, nil
	}
	parts := strings.Split(dirPath, "/")
	dir := dm.Root
	for i := 1; i < len(parts); i++ {
		if sub, ok := dir.getChild(parts[i]); ok {
			dir = sub
		} else {
			return dm.Root, fmt.Errorf("Directory %s Not Found", dirPath)
		}
	}
	return dir, nil
}
func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (filer.DirectoryId, error) {
	d, e := dm.findDirectory(dirPath)
	if e == nil {
		return d.Id, nil
	}
	return dm.Root.Id, e
}

func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId filer.DirectoryId) error {
	dirPath = CleanFilePath(dirPath)
	if dirPath == "/" {
		return nil
	}
	parts := strings.Split(dirPath, "/")
	dir := dm.Root
	for i := 1; i < len(parts); i++ {
		sub, ok := dir.getChild(parts[i])
		if !ok {
			writeLock.Lock()
			if sub2, createdByOtherThread := dir.getChild(parts[i]); createdByOtherThread {
				sub = sub2
			} else {
				if i != len(parts)-1 {
					writeLock.Unlock()
					return fmt.Errorf("%s should be created after parent %s", dirPath, parts[i])
				}
				var err error
				sub, err = dm.newDirectoryEntryInMap(dir, parts[i])
				if err != nil {
					writeLock.Unlock()
					return err
				}
				if sub.Id != dirId {
					writeLock.Unlock()
					// the dir.log should be the same order as in-memory directory id
					return fmt.Errorf("%s should be have id %v instead of %v", dirPath, sub.Id, dirId)
				}
				dir.addChild(parts[i], sub)
			}
			writeLock.Unlock()
		}
		dir = sub
	}
	return nil
}

func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEntryInMap, created bool) {
	dirPath = CleanFilePath(dirPath)
	if dirPath == "/" {
		return dm.Root, false
	}
	parts := strings.Split(dirPath, "/")
	dir = dm.Root
	for i := 1; i < len(parts); i++ {
		sub, ok := dir.getChild(parts[i])
		if !ok {
			writeLock.Lock()
			if sub2, createdByOtherThread := dir.getChild(parts[i]); createdByOtherThread {
				sub = sub2
			} else {
				var err error
				sub, err = dm.newDirectoryEntryInMap(dir, parts[i])
				if err != nil {
					writeLock.Unlock()
					return nil, false
				}
				dir.addChild(parts[i], sub)
				created = true
			}
			writeLock.Unlock()
		}
		dir = sub
	}
	return dir, created
}

func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (filer.DirectoryId, error) {
	dir, _ := dm.makeDirectory(dirPath)
	return dir.Id, nil
}

func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParentDirPath string, newName string) error {
	writeLock.Lock()
	defer writeLock.Unlock()
	oldDir, oe := dm.findDirectory(oldDirPath)
	if oe != nil {
		return oe
	}
	parentDir, pe := dm.findDirectory(newParentDirPath)
	if pe != nil {
		return pe
	}
	dm.log("mov", oldDirPath, newParentDirPath, newName)
	oldDir.Parent.removeChild(oldDir.Name)
	if newName == "" {
		newName = oldDir.Name
	}
	parentDir.addChild(newName, oldDir)
	oldDir.Name = newName
	oldDir.Parent = parentDir
	return nil
}

func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryEntry, err error) {
	d, e := dm.findDirectory(dirPath)
	if e != nil {
		return dirNames, e
	}
	return d.children(), nil
}
func (dm *DirectoryManagerInMap) DeleteDirectory(dirPath string) error {
	writeLock.Lock()
	defer writeLock.Unlock()
	if dirPath == "/" {
		return fmt.Errorf("Can not delete %s", dirPath)
	}
	d, e := dm.findDirectory(dirPath)
	if e != nil {
		return e
	}
	if d.hasChildren() {
		return fmt.Errorf("dir %s still has sub directories", dirPath)
	}
	d.Parent.removeChild(d.Name)
	d.Parent = nil
	dm.log("del", dirPath)
	return nil
}

func CleanFilePath(fp string) string {
	ret := filepath.Clean(fp)
	if os.PathSeparator == '\\' {
		return strings.Replace(ret, "\\", "/", -1)
	}
	return ret
}