From 94a01fcfcb473057028c6e24b49032f0897be958 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 15 Sep 2021 23:04:16 -0700 Subject: [PATCH] filer.remote.gateway: add options to include or exclude new bucket names to mirror --- weed/command/filer_remote_gateway.go | 4 ++++ weed/command/filer_remote_gateway_buckets.go | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/weed/command/filer_remote_gateway.go b/weed/command/filer_remote_gateway.go index 1b44f31ce..be5ab3fe2 100644 --- a/weed/command/filer_remote_gateway.go +++ b/weed/command/filer_remote_gateway.go @@ -22,6 +22,8 @@ type RemoteGatewayOptions struct { timeAgo *time.Duration createBucketAt *string createBucketRandomSuffix *bool + include *string + exclude *string mappings *remote_pb.RemoteStorageMapping remoteConfs map[string]*remote_pb.RemoteConf @@ -50,6 +52,8 @@ func init() { remoteGatewayOptions.createBucketRandomSuffix = cmdFilerRemoteGateway.Flag.Bool("createBucketWithRandomSuffix", true, "add randomized suffix to bucket name to avoid conflicts") remoteGatewayOptions.readChunkFromFiler = cmdFilerRemoteGateway.Flag.Bool("filerProxy", false, "read file chunks from filer instead of volume servers") remoteGatewayOptions.timeAgo = cmdFilerRemoteGateway.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"") + remoteGatewayOptions.include = cmdFilerRemoteGateway.Flag.String("include", "", "pattens of new bucket names, e.g., s3*") + remoteGatewayOptions.exclude = cmdFilerRemoteGateway.Flag.String("exclude", "", "pattens of new bucket names, e.g., local*") } var cmdFilerRemoteGateway = &Command{ diff --git a/weed/command/filer_remote_gateway_buckets.go b/weed/command/filer_remote_gateway_buckets.go index e16e4f731..bd3e76859 100644 --- a/weed/command/filer_remote_gateway_buckets.go +++ b/weed/command/filer_remote_gateway_buckets.go @@ -13,6 +13,7 @@ import ( "github.com/golang/protobuf/proto" "math" "math/rand" + "path/filepath" "strings" "time" ) @@ -75,6 +76,16 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour } bucketName := strings.ToLower(entry.Name) + if *option.include != "" { + if ok, _ := filepath.Match(*option.include, entry.Name); !ok { + return nil + } + } + if *option.exclude != "" { + if ok, _ := filepath.Match(*option.exclude, entry.Name); ok { + return nil + } + } if *option.createBucketRandomSuffix { // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html if len(bucketName)+5 > 63 {