From 3e3e0fa2402ee29aabf523dc20aa5cfd5b22d569 Mon Sep 17 00:00:00 2001
From: adasauce <60991921+adasauce@users.noreply.github.com>
Date: Wed, 6 Mar 2024 11:24:44 -0400
Subject: [PATCH] feat: add support for S3 ListObjects and ListObjectsV2
 (#5350)

---
 weed/s3api/s3api_objects_list_handlers.go | 67 +++++++++++++++--------
 1 file changed, 44 insertions(+), 23 deletions(-)

diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go
index 78b77a044..b2ad915b9 100644
--- a/weed/s3api/s3api_objects_list_handlers.go
+++ b/weed/s3api/s3api_objects_list_handlers.go
@@ -47,10 +47,6 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
 		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
 		return
 	}
-	if delimiter != "" && delimiter != "/" {
-		s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
-		return
-	}
 
 	marker := continuationToken
 	if continuationToken == "" {
@@ -103,10 +99,6 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
 		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
 		return
 	}
-	if delimiter != "" && delimiter != "/" {
-		s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
-		return
-	}
 
 	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)
 
@@ -171,22 +163,51 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 						cursor.maxKeys--
 					}
 				} else {
-					storageClass := "STANDARD"
-					if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
-						storageClass = string(v)
+					var delimiterFound bool
+					if delimiter != "" {
+						// keys that contain the same string between the prefix and the first occurrence of the delimiter are grouped together as a commonPrefix.
+						// extract the string between the prefix and the delimiter and add it to the commonPrefixes if it's unique.
+						fullPath := fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):]
+						delimitedPath := strings.SplitN(fullPath, delimiter, 2)
+						if len(delimitedPath) == 2 {
+
+							// S3 clients expect the delimited prefix to contain the delimiter.
+							delimitedPrefix := delimitedPath[0] + delimiter
+
+							for i := range commonPrefixes {
+								if commonPrefixes[i].Prefix == delimitedPrefix {
+									delimiterFound = true
+									break
+								}
+							}
+
+							if !delimiterFound {
+								commonPrefixes = append(commonPrefixes, PrefixEntry{
+									Prefix: delimitedPrefix,
+								})
+								cursor.maxKeys--
+								delimiterFound = true
+							}
+						}
+					}
+					if !delimiterFound {
+						storageClass := "STANDARD"
+						if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
+							storageClass = string(v)
+						}
+						contents = append(contents, ListEntry{
+							Key:          fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
+							LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
+							ETag:         "\"" + filer.ETag(entry) + "\"",
+							Size:         int64(filer.FileSize(entry)),
+							Owner: CanonicalUser{
+								ID:          fmt.Sprintf("%x", entry.Attributes.Uid),
+								DisplayName: entry.Attributes.UserName,
+							},
+							StorageClass: StorageClass(storageClass),
+						})
+						cursor.maxKeys--
 					}
-					contents = append(contents, ListEntry{
-						Key:          fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
-						LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
-						ETag:         "\"" + filer.ETag(entry) + "\"",
-						Size:         int64(filer.FileSize(entry)),
-						Owner: CanonicalUser{
-							ID:          fmt.Sprintf("%x", entry.Attributes.Uid),
-							DisplayName: entry.Attributes.UserName,
-						},
-						StorageClass: StorageClass(storageClass),
-					})
-					cursor.maxKeys--
 				}
 			})
 			if doErr != nil {