mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/chrislusf/seaweedfs synced 2024-09-19 07:21:00 +02:00
Code Issues Projects Releases Wiki Activity

Fix s3api_object_list_handlers returning contents less than the specified limit when more data actually exists (#4240)

Browse source 
Fix when the stored data is actually enough but s3api_object_list_handlers returns less than the specified limit

Signed-off-by: changlin.shi <changlin.shi@ly.com>
This commit is contained in:
LHHDZ 2023-02-22 15:08:52 +08:00 committed by GitHub
parent ef2f741823
commit db5515eada
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 47 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
weed/s3api/s3api_objects_list_handlers.go
View file

@ -146,57 +146,65 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
// check filer // check filer
err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
for {
nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, cursor, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) { empty := true
if entry.IsDirectory { nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, cursor, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) {
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html empty = false
if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter. if entry.IsDirectory {
commonPrefixes = append(commonPrefixes, PrefixEntry{ // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter.
}) commonPrefixes = append(commonPrefixes, PrefixEntry{
//All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns. Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
cursor.maxKeys-- })
} else if entry.IsDirectoryKeyObject() { //All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns.
cursor.maxKeys--
} else if entry.IsDirectoryKeyObject() {
contents = append(contents, ListEntry{
Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
ETag: "\"" + filer.ETag(entry) + "\"",
Owner: CanonicalUser{
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
DisplayName: entry.Attributes.UserName,
},
StorageClass: "STANDARD",
})
cursor.maxKeys--
}
} else {
storageClass := "STANDARD"
if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
storageClass = string(v)
}
contents = append(contents, ListEntry{ contents = append(contents, ListEntry{
Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
ETag: "\"" + filer.ETag(entry) + "\"", ETag: "\"" + filer.ETag(entry) + "\"",
Size: int64(filer.FileSize(entry)),
Owner: CanonicalUser{ Owner: CanonicalUser{
ID: fmt.Sprintf("%x", entry.Attributes.Uid), ID: fmt.Sprintf("%x", entry.Attributes.Uid),
DisplayName: entry.Attributes.UserName, DisplayName: entry.Attributes.UserName,
}, },
StorageClass: "STANDARD", StorageClass: StorageClass(storageClass),
}) })
cursor.maxKeys-- cursor.maxKeys--
} }
} else { })
storageClass := "STANDARD" if doErr != nil {
if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok { return doErr
storageClass = string(v)
}
contents = append(contents, ListEntry{
Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
ETag: "\"" + filer.ETag(entry) + "\"",
Size: int64(filer.FileSize(entry)),
Owner: CanonicalUser{
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
DisplayName: entry.Attributes.UserName,
},
StorageClass: StorageClass(storageClass),
})
cursor.maxKeys--
} }
})
if doErr != nil {
return doErr
}
if !cursor.isTruncated { if cursor.isTruncated {
nextMarker = "" if requestDir != "" {
} else { nextMarker = requestDir + "/" + nextMarker
if requestDir != "" { }
nextMarker = requestDir + "/" + nextMarker break
} else if empty {
nextMarker = ""
break
} else {
// start next loop
marker = nextMarker
} }
} }