From f5dc251b1e2d4b6278bd406d225b5a8358d9c0af Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Sat, 20 Apr 2024 01:24:18 +0500 Subject: [PATCH] s3 PutBucketLifecycle https://github.com/seaweedfs/seaweedfs/issues/4533 --- go.mod | 1 + go.sum | 2 + weed/s3api/s3api_bucket_handlers.go | 97 ++++++++++++++++++++++++++--- 3 files changed, 91 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index c965346e9..68a8835d1 100644 --- a/go.mod +++ b/go.mod @@ -150,6 +150,7 @@ require ( github.com/hanwen/go-fuse/v2 v2.5.0 github.com/hashicorp/raft v1.6.1 github.com/hashicorp/raft-boltdb/v2 v2.3.0 + github.com/minio/minio-go/v7 v7.0.66 github.com/orcaman/concurrent-map/v2 v2.0.1 github.com/rabbitmq/amqp091-go v1.9.0 github.com/rclone/rclone v1.66.0 diff --git a/go.sum b/go.sum index 9cce26431..d6a23668f 100644 --- a/go.sum +++ b/go.sum @@ -660,6 +660,8 @@ github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= +github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw= +github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 04e1e00a4..d8c42eb98 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -1,15 +1,18 @@ package s3api import ( + "bytes" "context" "encoding/xml" "errors" "fmt" "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" + "github.com/minio/minio-go/v7/pkg/lifecycle" "github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket" "github.com/seaweedfs/seaweedfs/weed/util" "math" "net/http" + "strings" "time" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -325,19 +328,22 @@ func (s3a *S3ApiServer) GetBucketLifecycleConfigurationHandler(w http.ResponseWr s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchLifecycleConfiguration) return } - response := Lifecycle{} - for prefix, internalTtl := range ttls { + response := lifecycle.Configuration{} + for locationPrefix, internalTtl := range ttls { ttl, _ := needle.ReadTTL(internalTtl) days := int(ttl.Minutes() / 60 / 24) if days == 0 { continue } - response.Rules = append(response.Rules, Rule{ - Status: Enabled, Filter: Filter{ - Prefix: Prefix{string: prefix, set: true}, - set: true, - }, - Expiration: Expiration{Days: days, set: true}, + rulePrefix, found := strings.CutPrefix(locationPrefix, fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket)) + if !found { + continue + } + response.Rules = append(response.Rules, lifecycle.Rule{ + ID: rulePrefix, + Status: "Enabled", + Prefix: rulePrefix, + Expiration: lifecycle.Expiration{Days: lifecycle.ExpirationDays(days)}, }) } writeSuccessResponseXML(w, r, response) @@ -346,9 +352,82 @@ func (s3a *S3ApiServer) GetBucketLifecycleConfigurationHandler(w http.ResponseWr // PutBucketLifecycleConfigurationHandler Put Bucket Lifecycle configuration // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWriter, r *http.Request) { + // collect parameters + bucket, _ := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("PutBucketLifecycleConfigurationHandler %s", bucket) - s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) + if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, err) + return + } + lifecycleConfig := &lifecycle.Configuration{} + if err := xmlDecoder(r.Body, lifecycleConfig, r.ContentLength); err != nil { + glog.Warningf("PutBucketLifecycleConfigurationHandler xml decode: %s", err) + s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML) + return + } + + fc, err := filer.ReadFilerConf(s3a.option.Filer, s3a.option.GrpcDialOption, nil) + if err != nil { + glog.Errorf("PutBucketLifecycleConfigurationHandler read filer config: %s", err) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } + collectionTtls := fc.GetCollectionTtls(bucket) + changed := false + + for _, rule := range lifecycleConfig.Rules { + if !strings.EqualFold(rule.Status, "Enabled") { + continue + } + if rule.Expiration.Days == 0 { + continue + } + + var rulePrefix string + switch { + case len(rule.RuleFilter.Prefix) > 0: + rulePrefix = rule.RuleFilter.Prefix + case len(rule.Prefix) > 0: + rulePrefix = rule.Prefix + } + + if len(rulePrefix) == 0 { + continue + } + locConf := &filer_pb.FilerConf_PathConf{ + LocationPrefix: fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, bucket, rulePrefix), + Collection: bucket, + Ttl: fmt.Sprintf("%dd", rule.Expiration.Days), + } + if ttl, ok := collectionTtls[locConf.LocationPrefix]; ok && ttl == locConf.Ttl { + continue + } + if err := fc.AddLocationConf(locConf); err != nil { + glog.Errorf("PutBucketLifecycleConfigurationHandler add location config: %s", err) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } + changed = true + } + + if changed { + var buf bytes.Buffer + if err := fc.ToText(&buf); err != nil { + glog.Errorf("PutBucketLifecycleConfigurationHandler save config to text: %s", err) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + } + if err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + return filer.SaveInsideFiler(client, filer.DirectoryEtcSeaweedFS, filer.FilerConfName, buf.Bytes()) + }); err != nil { + glog.Errorf("PutBucketLifecycleConfigurationHandler save config inside filer: %s", err) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } + } + + writeSuccessResponseEmpty(w, r) } // DeleteBucketLifecycleHandler Delete Bucket Lifecycle