From f01d5616b3dba0779d2adbe361271a8f9626e8ee Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 13 May 2018 14:02:29 -0700 Subject: [PATCH] add better listing directory entries --- weed/filer2/embedded/embedded_store.go | 2 +- weed/filer2/filer.go | 4 ++-- weed/filer2/filer_structure.go | 2 +- weed/filer2/memdb/memdb_store.go | 23 ++++++++++++++++++++--- weed/filer2/memdb/memdb_store_test.go | 15 +++++++++++---- 5 files changed, 35 insertions(+), 11 deletions(-) diff --git a/weed/filer2/embedded/embedded_store.go b/weed/filer2/embedded/embedded_store.go index a2a45807a..a69190e32 100644 --- a/weed/filer2/embedded/embedded_store.go +++ b/weed/filer2/embedded/embedded_store.go @@ -37,6 +37,6 @@ func (filer *EmbeddedStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2 return nil, nil } -func (filer *EmbeddedStore) ListDirectoryEntries(fullpath filer2.FullPath) (entries []*filer2.Entry, err error) { +func (filer *EmbeddedStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { return nil, nil } diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index 39be69d3a..ad7a3d906 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -117,11 +117,11 @@ func (f *Filer) DeleteEntry(p FullPath) (fileEntry *Entry, err error) { return f.store.DeleteEntry(p) } -func (f *Filer) ListDirectoryEntries(p FullPath) ([]*Entry, error) { +func (f *Filer) ListDirectoryEntries(p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) { if strings.HasSuffix(string(p), "/") { p = p[0:len(p)-1] } - return f.store.ListDirectoryEntries(p) + return f.store.ListDirectoryEntries(p, startFileName, inclusive, limit) } func (f *Filer) cacheGetDirectory(dirpath string) (*Entry) { diff --git a/weed/filer2/filer_structure.go b/weed/filer2/filer_structure.go index c6a3f817d..c31d878cf 100644 --- a/weed/filer2/filer_structure.go +++ b/weed/filer2/filer_structure.go @@ -62,5 +62,5 @@ type FilerStore interface { FindEntry(FullPath) (found bool, entry *Entry, err error) DeleteEntry(FullPath) (fileEntry *Entry, err error) - ListDirectoryEntries(dirPath FullPath) ([]*Entry, error) + ListDirectoryEntries(dirPath FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) } diff --git a/weed/filer2/memdb/memdb_store.go b/weed/filer2/memdb/memdb_store.go index 47e9934aa..edddcca96 100644 --- a/weed/filer2/memdb/memdb_store.go +++ b/weed/filer2/memdb/memdb_store.go @@ -60,9 +60,18 @@ func (filer *MemDbStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.En return entry, nil } -func (filer *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath) (entries []*filer2.Entry, err error) { - filer.tree.AscendGreaterOrEqual(Entry{&filer2.Entry{FullPath: fullpath}}, +func (filer *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { + + startFrom := string(fullpath) + if startFileName != "" { + startFrom = startFrom + "/" + startFileName + } + + filer.tree.AscendGreaterOrEqual(Entry{&filer2.Entry{FullPath: filer2.FullPath(startFrom)}}, func(item btree.Item) bool { + if limit <= 0 { + return false + } entry := item.(Entry).Entry // println("checking", entry.FullPath) if entry.FullPath == fullpath { @@ -70,7 +79,14 @@ func (filer *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath) (entries // println("skipping the folder", entry.FullPath) return true } - dir, _ := entry.FullPath.DirAndName() + dir, name := entry.FullPath.DirAndName() + if name == startFileName { + if inclusive { + limit-- + entries = append(entries, entry) + } + return true + } if !strings.HasPrefix(dir, string(fullpath)) { // println("directory is:", dir, "fullpath:", fullpath) // println("breaking from", entry.FullPath) @@ -83,6 +99,7 @@ func (filer *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath) (entries } // now process the directory items // println("adding entry", entry.FullPath) + limit-- entries = append(entries, entry) return true }, diff --git a/weed/filer2/memdb/memdb_store_test.go b/weed/filer2/memdb/memdb_store_test.go index 9c7810d4d..3fefbf5cd 100644 --- a/weed/filer2/memdb/memdb_store_test.go +++ b/weed/filer2/memdb/memdb_store_test.go @@ -72,7 +72,7 @@ func TestCreateFileAndList(t *testing.T) { filer.CreateEntry(entry2) // checking the 2 files - entries, err := filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/")) + entries, err := filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/"), "", false, 100) if err != nil { t.Errorf("list entries: %v", err) @@ -94,8 +94,15 @@ func TestCreateFileAndList(t *testing.T) { return } + // checking the offset + entries, err = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/"), "file1.jpg", false, 100) + if len(entries) != 1 { + t.Errorf("list entries count: %v", len(entries)) + return + } + // checking one upper directory - entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is")) + entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100) if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return @@ -113,7 +120,7 @@ func TestCreateFileAndList(t *testing.T) { filer.CreateEntry(entry3) // checking one upper directory - entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is")) + entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100) if len(entries) != 2 { t.Errorf("list entries count: %v", len(entries)) return @@ -121,7 +128,7 @@ func TestCreateFileAndList(t *testing.T) { // delete file and count filer.DeleteEntry(file3Path) - entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is")) + entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100) if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return