From b9c1f3e9de49dec66d34913442def65f6d9da238 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 21 Sep 2020 10:51:24 -0700 Subject: [PATCH] s3: fixes for list multipart upload --- weed/s3api/filer_multipart.go | 23 ++++++++++++++++--- weed/s3api/s3api_object_copy_handlers.go | 2 +- weed/s3api/s3api_object_multipart_handlers.go | 2 +- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index d1f0155f0..f882592c1 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -24,6 +24,9 @@ type InitiateMultipartUploadResult struct { } func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInput) (output *InitiateMultipartUploadResult, code s3err.ErrorCode) { + + glog.V(2).Infof("createMultipartUpload input %v", input) + uploadId, _ := uuid.NewRandom() uploadIdString := uploadId.String() @@ -55,6 +58,8 @@ type CompleteMultipartUploadResult struct { func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploadInput) (output *CompleteMultipartUploadResult, code s3err.ErrorCode) { + glog.V(2).Infof("completeMultipartUpload input %v", input) + uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId entries, _, err := s3a.list(uploadDirectory, "", "", false, 0) @@ -123,6 +128,8 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput) (output *s3.AbortMultipartUploadOutput, code s3err.ErrorCode) { + glog.V(2).Infof("abortMultipartUpload input %v", input) + exists, err := s3a.exists(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true) if err != nil { glog.V(1).Infof("bucket %s abort upload %s: %v", *input.Bucket, *input.UploadId, err) @@ -159,6 +166,8 @@ type ListMultipartUploadsResult struct { func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput) (output *ListMultipartUploadsResult, code s3err.ErrorCode) { // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html + glog.V(2).Infof("listMultipartUploads input %v", input) + output = &ListMultipartUploadsResult{ Bucket: input.Bucket, Delimiter: input.Delimiter, @@ -168,7 +177,7 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput Prefix: input.Prefix, } - entries, isLast, err := s3a.list(s3a.genUploadsFolder(*input.Bucket), *input.Prefix, *input.KeyMarker, true, uint32(*input.MaxUploads)) + entries, isLast, err := s3a.list(s3a.genUploadsFolder(*input.Bucket), "", *input.UploadIdMarker, false, uint32(*input.MaxUploads)) if err != nil { glog.Errorf("listMultipartUploads %s error: %v", *input.Bucket, err) return @@ -177,9 +186,15 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput for _, entry := range entries { if entry.Extended != nil { - key := entry.Extended["key"] + key := string(entry.Extended["key"]) + if *input.KeyMarker != "" && *input.KeyMarker != key { + continue + } + if *input.Prefix != "" && !strings.HasPrefix(key, *input.Prefix) { + continue + } output.Upload = append(output.Upload, &s3.MultipartUpload{ - Key: objectKey(aws.String(string(key))), + Key: objectKey(aws.String(key)), UploadId: aws.String(entry.Name), }) if !isLast { @@ -209,6 +224,8 @@ type ListPartsResult struct { func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListPartsResult, code s3err.ErrorCode) { // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html + glog.V(2).Infof("listObjectParts input %v", input) + output = &ListPartsResult{ Bucket: input.Bucket, Key: objectKey(input.Key), diff --git a/weed/s3api/s3api_object_copy_handlers.go b/weed/s3api/s3api_object_copy_handlers.go index 9f88e205a..99a852c0c 100644 --- a/weed/s3api/s3api_object_copy_handlers.go +++ b/weed/s3api/s3api_object_copy_handlers.go @@ -116,7 +116,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req rangeHeader := r.Header.Get("x-amz-copy-source-range") dstUrl := fmt.Sprintf("http://%s%s/%s/%04d.part?collection=%s", - s3a.option.Filer, s3a.genUploadsFolder(dstBucket), uploadID, partID-1, dstBucket) + s3a.option.Filer, s3a.genUploadsFolder(dstBucket), uploadID, partID, dstBucket) srcUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer, s3a.option.BucketsPath, srcBucket, srcObject) diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 7dca98c75..0c0e8b245 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -200,7 +200,7 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ defer dataReader.Close() uploadUrl := fmt.Sprintf("http://%s%s/%s/%04d.part?collection=%s", - s3a.option.Filer, s3a.genUploadsFolder(bucket), uploadID, partID-1, bucket) + s3a.option.Filer, s3a.genUploadsFolder(bucket), uploadID, partID, bucket) etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)