2020-04-22 09:50:30 +08:00
|
|
|
package meta_cache
|
|
|
|
|
|
|
|
import (
|
2020-04-30 09:20:54 +08:00
|
|
|
"context"
|
2020-09-01 15:21:19 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/filer"
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/filer/leveldb"
|
2020-04-22 09:50:30 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
2021-10-17 14:33:45 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
2020-04-30 09:20:54 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/util"
|
2020-06-20 00:45:27 +08:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/util/bounded_tree"
|
2021-07-01 16:19:28 +08:00
|
|
|
"os"
|
2020-04-22 09:50:30 +08:00
|
|
|
)
|
|
|
|
|
2020-07-20 15:17:50 +08:00
|
|
|
// need to have logic similar to FilerStoreWrapper
|
|
|
|
// e.g. fill fileId field for chunks
|
|
|
|
|
2020-04-22 09:50:30 +08:00
|
|
|
type MetaCache struct {
|
2020-09-24 18:06:44 +08:00
|
|
|
localStore filer.VirtualFilerStore
|
2021-07-01 16:19:28 +08:00
|
|
|
// sync.RWMutex
|
2020-06-20 00:45:27 +08:00
|
|
|
visitedBoundary *bounded_tree.BoundedTree
|
2020-09-03 15:07:22 +08:00
|
|
|
uidGidMapper *UidGidMapper
|
2020-10-26 10:24:15 +08:00
|
|
|
invalidateFunc func(util.FullPath)
|
2020-04-22 09:50:30 +08:00
|
|
|
}
|
|
|
|
|
2020-10-26 10:24:15 +08:00
|
|
|
func NewMetaCache(dbFolder string, baseDir util.FullPath, uidGidMapper *UidGidMapper, invalidateFunc func(util.FullPath)) *MetaCache {
|
2020-04-22 09:50:30 +08:00
|
|
|
return &MetaCache{
|
2020-09-23 07:24:13 +08:00
|
|
|
localStore: openMetaStore(dbFolder),
|
2020-10-13 12:58:37 +08:00
|
|
|
visitedBoundary: bounded_tree.NewBoundedTree(baseDir),
|
2020-09-03 15:07:22 +08:00
|
|
|
uidGidMapper: uidGidMapper,
|
2021-01-28 02:28:37 +08:00
|
|
|
invalidateFunc: func(fullpath util.FullPath) {
|
|
|
|
invalidateFunc(fullpath)
|
|
|
|
},
|
2020-04-22 09:50:30 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-09-24 18:06:44 +08:00
|
|
|
func openMetaStore(dbFolder string) filer.VirtualFilerStore {
|
2020-04-22 09:50:30 +08:00
|
|
|
|
2020-04-22 13:00:34 +08:00
|
|
|
os.RemoveAll(dbFolder)
|
2020-04-22 09:50:30 +08:00
|
|
|
os.MkdirAll(dbFolder, 0755)
|
|
|
|
|
|
|
|
store := &leveldb.LevelDBStore{}
|
2020-04-22 13:00:34 +08:00
|
|
|
config := &cacheConfig{
|
|
|
|
dir: dbFolder,
|
|
|
|
}
|
2020-04-22 09:50:30 +08:00
|
|
|
|
|
|
|
if err := store.Initialize(config, ""); err != nil {
|
|
|
|
glog.Fatalf("Failed to initialize metadata cache store for %s: %+v", store.GetName(), err)
|
|
|
|
}
|
|
|
|
|
2020-09-24 18:06:44 +08:00
|
|
|
return filer.NewFilerStoreWrapper(store)
|
2020-04-22 09:50:30 +08:00
|
|
|
|
2020-04-30 04:26:02 +08:00
|
|
|
}
|
2020-04-30 09:20:54 +08:00
|
|
|
|
2020-09-01 15:21:19 +08:00
|
|
|
func (mc *MetaCache) InsertEntry(ctx context.Context, entry *filer.Entry) error {
|
2021-07-01 16:19:28 +08:00
|
|
|
//mc.Lock()
|
|
|
|
//defer mc.Unlock()
|
2020-08-24 06:48:02 +08:00
|
|
|
return mc.doInsertEntry(ctx, entry)
|
|
|
|
}
|
|
|
|
|
2020-09-01 15:21:19 +08:00
|
|
|
func (mc *MetaCache) doInsertEntry(ctx context.Context, entry *filer.Entry) error {
|
2020-09-23 07:24:13 +08:00
|
|
|
return mc.localStore.InsertEntry(ctx, entry)
|
2020-04-30 09:20:54 +08:00
|
|
|
}
|
|
|
|
|
2020-09-03 15:07:22 +08:00
|
|
|
func (mc *MetaCache) AtomicUpdateEntryFromFiler(ctx context.Context, oldPath util.FullPath, newEntry *filer.Entry) error {
|
2021-07-01 16:19:28 +08:00
|
|
|
//mc.Lock()
|
|
|
|
//defer mc.Unlock()
|
2020-06-20 00:45:27 +08:00
|
|
|
|
|
|
|
oldDir, _ := oldPath.DirAndName()
|
|
|
|
if mc.visitedBoundary.HasVisited(util.FullPath(oldDir)) {
|
|
|
|
if oldPath != "" {
|
2020-08-18 01:07:34 +08:00
|
|
|
if newEntry != nil && oldPath == newEntry.FullPath {
|
2020-08-18 01:03:34 +08:00
|
|
|
// skip the unnecessary deletion
|
|
|
|
// leave the update to the following InsertEntry operation
|
|
|
|
} else {
|
2021-05-14 13:41:19 +08:00
|
|
|
glog.V(3).Infof("DeleteEntry %s", oldPath)
|
2020-09-23 07:24:13 +08:00
|
|
|
if err := mc.localStore.DeleteEntry(ctx, oldPath); err != nil {
|
2020-08-18 01:03:34 +08:00
|
|
|
return err
|
|
|
|
}
|
2020-06-20 00:45:27 +08:00
|
|
|
}
|
2020-04-30 09:20:54 +08:00
|
|
|
}
|
2020-07-14 22:34:16 +08:00
|
|
|
} else {
|
2020-06-20 00:45:27 +08:00
|
|
|
// println("unknown old directory:", oldDir)
|
2020-04-30 09:20:54 +08:00
|
|
|
}
|
2020-06-20 00:45:27 +08:00
|
|
|
|
2020-04-30 09:20:54 +08:00
|
|
|
if newEntry != nil {
|
2020-06-20 00:45:27 +08:00
|
|
|
newDir, _ := newEntry.DirAndName()
|
|
|
|
if mc.visitedBoundary.HasVisited(util.FullPath(newDir)) {
|
2020-10-26 10:24:15 +08:00
|
|
|
glog.V(3).Infof("InsertEntry %s/%s", newDir, newEntry.Name())
|
2020-09-23 07:24:13 +08:00
|
|
|
if err := mc.localStore.InsertEntry(ctx, newEntry); err != nil {
|
2020-06-20 00:45:27 +08:00
|
|
|
return err
|
|
|
|
}
|
2020-04-30 09:20:54 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-09-01 15:21:19 +08:00
|
|
|
func (mc *MetaCache) UpdateEntry(ctx context.Context, entry *filer.Entry) error {
|
2021-07-01 16:19:28 +08:00
|
|
|
//mc.Lock()
|
|
|
|
//defer mc.Unlock()
|
2020-09-23 07:24:13 +08:00
|
|
|
return mc.localStore.UpdateEntry(ctx, entry)
|
2020-04-30 09:20:54 +08:00
|
|
|
}
|
|
|
|
|
2020-09-01 15:21:19 +08:00
|
|
|
func (mc *MetaCache) FindEntry(ctx context.Context, fp util.FullPath) (entry *filer.Entry, err error) {
|
2021-07-01 16:19:28 +08:00
|
|
|
//mc.RLock()
|
|
|
|
//defer mc.RUnlock()
|
2020-09-23 07:24:13 +08:00
|
|
|
entry, err = mc.localStore.FindEntry(ctx, fp)
|
2020-07-20 15:17:50 +08:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2020-09-03 15:07:22 +08:00
|
|
|
mc.mapIdFromFilerToLocal(entry)
|
2020-07-20 15:17:50 +08:00
|
|
|
return
|
2020-04-30 09:20:54 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
func (mc *MetaCache) DeleteEntry(ctx context.Context, fp util.FullPath) (err error) {
|
2021-07-01 16:19:28 +08:00
|
|
|
//mc.Lock()
|
|
|
|
//defer mc.Unlock()
|
2020-09-23 07:24:13 +08:00
|
|
|
return mc.localStore.DeleteEntry(ctx, fp)
|
2020-04-30 09:20:54 +08:00
|
|
|
}
|
|
|
|
|
2021-10-17 14:33:45 +08:00
|
|
|
func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, client filer_pb.FilerClient, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) error {
|
2021-07-01 16:19:28 +08:00
|
|
|
//mc.RLock()
|
|
|
|
//defer mc.RUnlock()
|
2020-07-20 15:17:50 +08:00
|
|
|
|
2020-10-14 02:21:13 +08:00
|
|
|
if !mc.visitedBoundary.HasVisited(dirPath) {
|
2021-10-17 14:33:45 +08:00
|
|
|
glog.V(2).Infof("visit unsynchronized dir: %v", dirPath)
|
|
|
|
// this should not happen often
|
|
|
|
// unless, e.g., moving a deep directory
|
|
|
|
EnsureVisited(mc, client, dirPath)
|
2020-10-14 02:21:13 +08:00
|
|
|
}
|
|
|
|
|
2021-01-16 15:56:24 +08:00
|
|
|
_, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *filer.Entry) bool {
|
2020-09-03 15:07:22 +08:00
|
|
|
mc.mapIdFromFilerToLocal(entry)
|
2021-01-16 15:56:24 +08:00
|
|
|
return eachEntryFunc(entry)
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
return err
|
2020-07-20 15:17:50 +08:00
|
|
|
}
|
2021-01-16 15:56:24 +08:00
|
|
|
return err
|
2020-04-30 09:20:54 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
func (mc *MetaCache) Shutdown() {
|
2021-07-01 16:19:28 +08:00
|
|
|
//mc.Lock()
|
|
|
|
//defer mc.Unlock()
|
2020-09-23 07:24:13 +08:00
|
|
|
mc.localStore.Shutdown()
|
2020-04-30 09:20:54 +08:00
|
|
|
}
|
2020-09-03 15:07:22 +08:00
|
|
|
|
|
|
|
func (mc *MetaCache) mapIdFromFilerToLocal(entry *filer.Entry) {
|
|
|
|
entry.Attr.Uid, entry.Attr.Gid = mc.uidGidMapper.FilerToLocal(entry.Attr.Uid, entry.Attr.Gid)
|
|
|
|
}
|