mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-11-28 05:15:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

s3 fix get list of dir object key with slash suffix (#4391)

Browse Source 
* s3 fix get list of dir object key with slash suffix
https://github.com/seaweedfs/seaweedfs/issues/3086

* list only entry dir eq prefix

---------

Co-authored-by: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.co>
This commit is contained in:
Konstantin Lebedev 2023-04-12 11:52:35 +05:00 committed by GitHub
parent ccc030b245
commit 44ad07276c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 22 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
weed/s3api/s3api_objects_list_handlers.go
View File

@ -141,7 +141,8 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
var doErr error var doErr error
var nextMarker string var nextMarker string
cursor := &ListingCursor{ cursor := &ListingCursor{
maxKeys: maxKeys, maxKeys: maxKeys,
prefixEndsOnDelimiter: strings.HasSuffix(originalPrefix, "/"),
} }
// check filer // check filer
@ -199,7 +200,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
nextMarker = requestDir + "/" + nextMarker nextMarker = requestDir + "/" + nextMarker
} }
break break
} else if empty { } else if empty || strings.HasSuffix(originalPrefix, "/") {
nextMarker = "" nextMarker = ""
break break
} else { } else {
@ -227,8 +228,9 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
} }
type ListingCursor struct { type ListingCursor struct {
maxKeys int maxKeys int
isTruncated bool isTruncated bool
prefixEndsOnDelimiter bool
} }
// the prefix and marker may be in different directories // the prefix and marker may be in different directories
@ -236,7 +238,7 @@ type ListingCursor struct {
func normalizePrefixMarker(prefix, marker string) (alignedDir, alignedPrefix, alignedMarker string) { func normalizePrefixMarker(prefix, marker string) (alignedDir, alignedPrefix, alignedMarker string) {
// alignedDir should not end with "/" // alignedDir should not end with "/"
// alignedDir, alignedPrefix, alignedMarker should only have "/" in middle // alignedDir, alignedPrefix, alignedMarker should only have "/" in middle
prefix = strings.TrimLeft(prefix, "/") prefix = strings.Trim(prefix, "/")
marker = strings.TrimLeft(marker, "/") marker = strings.TrimLeft(marker, "/")
if prefix == "" { if prefix == "" {
return "", "", marker return "", "", marker
@ -264,6 +266,7 @@ func normalizePrefixMarker(prefix, marker string) (alignedDir, alignedPrefix, al
} }
return return
} }
func toDirAndName(dirAndName string) (dir, name string) { func toDirAndName(dirAndName string) (dir, name string) {
sepIndex := strings.LastIndex(dirAndName, "/") sepIndex := strings.LastIndex(dirAndName, "/")
if sepIndex >= 0 { if sepIndex >= 0 {
@ -273,6 +276,7 @@ func toDirAndName(dirAndName string) (dir, name string) {
} }
return return
} }
func toParentAndDescendants(dirAndName string) (dir, name string) { func toParentAndDescendants(dirAndName string) (dir, name string) {
sepIndex := strings.Index(dirAndName, "/") sepIndex := strings.Index(dirAndName, "/")
if sepIndex >= 0 { if sepIndex >= 0 {
@ -287,7 +291,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
// invariants // invariants
// prefix and marker should be under dir, marker may contain "/" // prefix and marker should be under dir, marker may contain "/"
// maxKeys should be updated for each recursion // maxKeys should be updated for each recursion
// glog.V(4).Infof("doListFilerEntries dir: %s, prefix: %s, marker %s, maxKeys: %d, prefixEndsOnDelimiter: %+v", dir, prefix, marker, cursor.maxKeys, cursor.prefixEndsOnDelimiter)
if prefix == "/" && delimiter == "/" { if prefix == "/" && delimiter == "/" {
return return
} }
@ -319,6 +323,9 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
StartFromFileName: marker, StartFromFileName: marker,
InclusiveStartFrom: inclusiveStartFrom, InclusiveStartFrom: inclusiveStartFrom,
} }
if cursor.prefixEndsOnDelimiter {
request.Limit = uint32(1)
}
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
@ -344,8 +351,15 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
} }
entry := resp.Entry entry := resp.Entry
nextMarker = entry.Name nextMarker = entry.Name
if cursor.prefixEndsOnDelimiter {
if entry.Name == prefix && entry.IsDirectory {
cursor.prefixEndsOnDelimiter = false
} else {
continue
}
}
if entry.IsDirectory { if entry.IsDirectory {
// println("ListEntries", dir, "dir:", entry.Name) // glog.V(4).Infof("List Dir Entries %s, file: %s, maxKeys %d", dir, entry.Name, cursor.maxKeys)
if entry.Name == s3_constants.MultipartUploadsFolder { // FIXME no need to apply to all directories. this extra also affects maxKeys if entry.Name == s3_constants.MultipartUploadsFolder { // FIXME no need to apply to all directories. this extra also affects maxKeys
continue continue
} }
@ -375,7 +389,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
} }
} else { } else {
eachEntryFn(dir, entry) eachEntryFn(dir, entry)
// println("ListEntries", dir, "file:", entry.Name, "maxKeys", cursor.maxKeys) // glog.V(4).Infof("List File Entries %s, file: %s, maxKeys %d", dir, entry.Name, cursor.maxKeys)
} }
} }
return return