mirror of
https://github.com/chrislusf/seaweedfs
synced 2025-07-24 20:42:47 +02:00
* add object versioning * add missing file * Update weed/s3api/s3api_object_versioning.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update weed/s3api/s3api_object_versioning.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update weed/s3api/s3api_object_versioning.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * ListObjectVersionsResult is better to show multiple version entries * fix test * Update weed/s3api/s3api_object_handlers_put.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update weed/s3api/s3api_object_versioning.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * multiple improvements * move PutBucketVersioningHandler into weed/s3api/s3api_bucket_handlers.go file * duplicated code for reading bucket config, versioningEnabled, etc. try to use functions * opportunity to cache bucket config * error handling if bucket is not found * in case bucket is not found * fix build * add object versioning tests * remove non-existent tests * add tests * add versioning tests * skip a new test * ensure .versions directory exists before saving info into it * fix creating version entry * logging on creating version directory * Update s3api_object_versioning_test.go * retry and wait for directory creation * revert add more logging * Update s3api_object_versioning.go * more debug messages * clean up logs, and touch directory correctly * log the .versions creation and then parent directory listing * use mkFile instead of touch touch is for update * clean up data * add versioning test in go * change location * if modified, latest version is moved to .versions directory, and create a new latest version Core versioning functionality: WORKING TestVersioningBasicWorkflow - PASS TestVersioningDeleteMarkers - PASS TestVersioningMultipleVersionsSameObject - PASS TestVersioningDeleteAndRecreate - PASS TestVersioningListWithPagination - PASS ❌ Some advanced features still failing: ETag calculation issues (using mtime instead of proper MD5) Specific version retrieval (EOF error) Version deletion (internal errors) Concurrent operations (race conditions) * calculate multi chunk md5 Test Results - All Passing: ✅ TestBucketListReturnDataVersioning - PASS ✅ TestVersioningCreateObjectsInOrder - PASS ✅ TestVersioningBasicWorkflow - PASS ✅ TestVersioningMultipleVersionsSameObject - PASS ✅ TestVersioningDeleteMarkers - PASS * dedupe * fix TestVersioningErrorCases * fix eof error of reading old versions * get specific version also check current version * enable integration tests for versioning * trigger action to work for now * Fix GitHub Actions S3 versioning tests workflow - Fix syntax error (incorrect indentation) - Update directory paths from weed/s3api/versioning_tests/ to test/s3/versioning/ - Add push trigger for add-object-versioning branch to enable CI during development - Update artifact paths to match correct directory structure * Improve CI robustness for S3 versioning tests Makefile improvements: - Increase server startup timeout from 30s to 90s for CI environments - Add progressive timeout reporting (logs at 30s, full logs at 90s) - Better error handling with server logs on failure - Add server PID tracking for debugging - Improved test failure reporting GitHub Actions workflow improvements: - Increase job timeouts to account for CI environment delays - Add system information logging (memory, disk space) - Add detailed failure reporting with server logs - Add process and network diagnostics on failure - Better error messaging and log collection These changes should resolve the 'Server failed to start within 30 seconds' issue that was causing the CI tests to fail. * adjust testing volume size * Update Makefile * Update Makefile * Update Makefile * Update Makefile * Update s3-versioning-tests.yml * Update s3api_object_versioning.go * Update Makefile * do not clean up * log received version id * more logs * printout response * print out list version response * use tmp files when put versioned object * change to versions folder layout * Delete weed-test.log * test with mixed versioned and unversioned objects * remove versionDirCache * remove unused functions * remove unused function * remove fallback checking * minor --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
124 lines
3.6 KiB
Go
124 lines
3.6 KiB
Go
package s3api
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
)
|
|
|
|
func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error {
|
|
|
|
return filer_pb.Mkdir(context.Background(), s3a, parentDirectoryPath, dirName, fn)
|
|
|
|
}
|
|
|
|
func (s3a *S3ApiServer) mkFile(parentDirectoryPath string, fileName string, chunks []*filer_pb.FileChunk, fn func(entry *filer_pb.Entry)) error {
|
|
|
|
return filer_pb.MkFile(context.Background(), s3a, parentDirectoryPath, fileName, chunks, fn)
|
|
|
|
}
|
|
|
|
func (s3a *S3ApiServer) list(parentDirectoryPath, prefix, startFrom string, inclusive bool, limit uint32) (entries []*filer_pb.Entry, isLast bool, err error) {
|
|
|
|
err = filer_pb.List(context.Background(), s3a, parentDirectoryPath, prefix, func(entry *filer_pb.Entry, isLastEntry bool) error {
|
|
entries = append(entries, entry)
|
|
if isLastEntry {
|
|
isLast = true
|
|
}
|
|
return nil
|
|
}, startFrom, inclusive, limit)
|
|
|
|
if len(entries) == 0 {
|
|
isLast = true
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
func (s3a *S3ApiServer) rm(parentDirectoryPath, entryName string, isDeleteData, isRecursive bool) error {
|
|
|
|
return s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
|
|
|
err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
})
|
|
|
|
}
|
|
|
|
func doDeleteEntry(client filer_pb.SeaweedFilerClient, parentDirectoryPath string, entryName string, isDeleteData bool, isRecursive bool) error {
|
|
request := &filer_pb.DeleteEntryRequest{
|
|
Directory: parentDirectoryPath,
|
|
Name: entryName,
|
|
IsDeleteData: isDeleteData,
|
|
IsRecursive: isRecursive,
|
|
IgnoreRecursiveError: true,
|
|
}
|
|
|
|
glog.V(1).Infof("delete entry %v/%v: %v", parentDirectoryPath, entryName, request)
|
|
if resp, err := client.DeleteEntry(context.Background(), request); err != nil {
|
|
glog.V(0).Infof("delete entry %v: %v", request, err)
|
|
return fmt.Errorf("delete entry %s/%s: %v", parentDirectoryPath, entryName, err)
|
|
} else {
|
|
if resp.Error != "" {
|
|
return fmt.Errorf("delete entry %s/%s: %v", parentDirectoryPath, entryName, resp.Error)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isDirectory bool) (exists bool, err error) {
|
|
|
|
return filer_pb.Exists(context.Background(), s3a, parentDirectoryPath, entryName, isDirectory)
|
|
|
|
}
|
|
|
|
func (s3a *S3ApiServer) touch(parentDirectoryPath string, entryName string, entry *filer_pb.Entry) (err error) {
|
|
|
|
return filer_pb.Touch(context.Background(), s3a, parentDirectoryPath, entryName, entry)
|
|
|
|
}
|
|
|
|
func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry *filer_pb.Entry, err error) {
|
|
fullPath := util.NewFullPath(parentDirectoryPath, entryName)
|
|
return filer_pb.GetEntry(context.Background(), s3a, fullPath)
|
|
}
|
|
|
|
func (s3a *S3ApiServer) updateEntry(parentDirectoryPath string, newEntry *filer_pb.Entry) error {
|
|
updateEntryRequest := &filer_pb.UpdateEntryRequest{
|
|
Directory: parentDirectoryPath,
|
|
Entry: newEntry,
|
|
}
|
|
|
|
err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
|
err := filer_pb.UpdateEntry(context.Background(), client, updateEntryRequest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
})
|
|
return err
|
|
}
|
|
|
|
func (s3a *S3ApiServer) getCollectionName(bucket string) string {
|
|
if s3a.option.FilerGroup != "" {
|
|
return fmt.Sprintf("%s_%s", s3a.option.FilerGroup, bucket)
|
|
}
|
|
return bucket
|
|
}
|
|
|
|
func objectKey(key *string) *string {
|
|
if strings.HasPrefix(*key, "/") {
|
|
t := (*key)[1:]
|
|
return &t
|
|
}
|
|
return key
|
|
}
|