diff --git a/docker/README.md b/docker/README.md index 524f5d4d0..288d87158 100644 --- a/docker/README.md +++ b/docker/README.md @@ -28,6 +28,13 @@ cd $GOPATH/src/github.com/chrislusf/seaweedfs/docker make ``` +### S3 cmd + +list +``` +s3cmd --no-ssl --host=127.0.0.1:8333 ls s3:// +``` + ## Build and push a multiarch build Make sure that `docker buildx` is supported (might be an experimental docker feature) diff --git a/docker/compose/local-dev-compose.yml b/docker/compose/local-dev-compose.yml index 01d0594a6..5df1e7332 100644 --- a/docker/compose/local-dev-compose.yml +++ b/docker/compose/local-dev-compose.yml @@ -29,7 +29,7 @@ services: - 8111:8111 - 8888:8888 - 18888:18888 - command: '-v=1 filer -master="master:9333" -iam' + command: '-v=1 filer -ip.bind=0.0.0.0 -master="master:9333" -iam -iam.ip=filer' depends_on: - master - volume @@ -41,7 +41,7 @@ services: image: chrislusf/seaweedfs:local ports: - 8333:8333 - command: '-v=1 s3 -filer="filer:8888"' + command: '-v=1 s3 -filer="filer:8888" -ip.bind=s3' depends_on: - master - volume diff --git a/docker/seaweedfs-dev-compose.yml b/docker/seaweedfs-dev-compose.yml index 2382fb17d..0b11e72e4 100644 --- a/docker/seaweedfs-dev-compose.yml +++ b/docker/seaweedfs-dev-compose.yml @@ -20,7 +20,7 @@ services: ports: - 8888:8888 - 18888:18888 - command: 'filer -master="master:9333"' + command: 'filer -master="master:9333" -ip.bind=0.0.0.0' depends_on: - master - volume @@ -28,7 +28,7 @@ services: image: chrislusf/seaweedfs:dev # use a remote dev image ports: - 8333:8333 - command: 's3 -filer="filer:8888"' + command: 's3 -filer="filer:8888" -ip.bind=0.0.0.0' depends_on: - master - volume diff --git a/weed/cluster/cluster.go b/weed/cluster/cluster.go index 6c24df44c..ad6e6b879 100644 --- a/weed/cluster/cluster.go +++ b/weed/cluster/cluster.go @@ -46,8 +46,6 @@ func NewCluster() *Cluster { } func (cluster *Cluster) getFilers(filerGroup FilerGroup, createIfNotFound bool) *Filers { - cluster.filersLock.Lock() - defer cluster.filersLock.Unlock() filers, found := cluster.filerGroup2filers[filerGroup] if !found && createIfNotFound { filers = &Filers{ @@ -63,6 +61,8 @@ func (cluster *Cluster) AddClusterNode(ns, nodeType string, address pb.ServerAdd filerGroup := FilerGroup(ns) switch nodeType { case FilerType: + cluster.filersLock.Lock() + defer cluster.filersLock.Unlock() filers := cluster.getFilers(filerGroup, true) if existingNode, found := filers.filers[address]; found { existingNode.counter++ @@ -115,6 +115,8 @@ func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb filerGroup := FilerGroup(ns) switch nodeType { case FilerType: + cluster.filersLock.Lock() + defer cluster.filersLock.Unlock() filers := cluster.getFilers(filerGroup, false) if filers == nil { return nil @@ -165,12 +167,12 @@ func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb func (cluster *Cluster) ListClusterNode(filerGroup FilerGroup, nodeType string) (nodes []*ClusterNode) { switch nodeType { case FilerType: + cluster.filersLock.RLock() + defer cluster.filersLock.RUnlock() filers := cluster.getFilers(filerGroup, false) if filers == nil { return } - cluster.filersLock.RLock() - defer cluster.filersLock.RUnlock() for _, node := range filers.filers { nodes = append(nodes, node) } diff --git a/weed/cluster/cluster_test.go b/weed/cluster/cluster_test.go index ccaccf6f7..1187642de 100644 --- a/weed/cluster/cluster_test.go +++ b/weed/cluster/cluster_test.go @@ -3,6 +3,8 @@ package cluster import ( "github.com/chrislusf/seaweedfs/weed/pb" "github.com/stretchr/testify/assert" + "strconv" + "sync" "testing" ) @@ -45,3 +47,35 @@ func TestClusterAddRemoveNodes(t *testing.T) { c.RemoveClusterNode("", "filer", pb.ServerAddress("111:1")) } + +func TestConcurrentAddRemoveNodes(t *testing.T) { + c := NewCluster() + var wg sync.WaitGroup + for i := 0; i < 50; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + address := strconv.Itoa(i) + c.AddClusterNode("", "filer", pb.ServerAddress(address), "23.45") + }(i) + } + wg.Wait() + + for i := 0; i < 50; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + address := strconv.Itoa(i) + node := c.RemoveClusterNode("", "filer", pb.ServerAddress(address)) + + if len(node) == 0 { + t.Errorf("TestConcurrentAddRemoveNodes: node[%s] not found", address) + return + } else if node[0].ClusterNodeUpdate.Address != address { + t.Errorf("TestConcurrentAddRemoveNodes: expect:%s, actual:%s", address, node[0].ClusterNodeUpdate.Address) + return + } + }(i) + } + wg.Wait() +} diff --git a/weed/command/update_full.go b/weed/command/update_full.go index 529f38219..185203aee 100644 --- a/weed/command/update_full.go +++ b/weed/command/update_full.go @@ -1,5 +1,5 @@ -//go:build elastic && ydb && gocdk && hdfs -// +build elastic,ydb,gocdk,hdfs +//go:build elastic && ydb && gocdk && tikv +// +build elastic,ydb,gocdk,tikv package command diff --git a/weed/s3api/s3api_object_copy_handlers.go b/weed/s3api/s3api_object_copy_handlers.go index 6b67ef337..950e7a8fb 100644 --- a/weed/s3api/s3api_object_copy_handlers.go +++ b/weed/s3api/s3api_object_copy_handlers.go @@ -45,7 +45,12 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource) return } - entry.Extended = processMetadataBytes(r.Header, entry.Extended, replaceMeta, replaceTagging) + entry.Extended, err = processMetadataBytes(r.Header, entry.Extended, replaceMeta, replaceTagging) + if err != nil { + glog.Errorf("CopyObjectHandler ValidateTags error %s: %v", r.URL, err) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag) + return + } err = s3a.touch(dir, name, entry) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource) @@ -252,7 +257,7 @@ func processMetadata(reqHeader, existing http.Header, replaceMeta, replaceTaggin return } -func processMetadataBytes(reqHeader http.Header, existing map[string][]byte, replaceMeta, replaceTagging bool) (metadata map[string][]byte) { +func processMetadataBytes(reqHeader http.Header, existing map[string][]byte, replaceMeta, replaceTagging bool) (metadata map[string][]byte, err error) { metadata = make(map[string][]byte) if sc := existing[s3_constants.AmzStorageClass]; len(sc) > 0 { @@ -277,16 +282,18 @@ func processMetadataBytes(reqHeader http.Header, existing map[string][]byte, rep } } } - if replaceTagging { if tags := reqHeader.Get(s3_constants.AmzObjectTagging); tags != "" { - for _, v := range strings.Split(tags, "&") { - tag := strings.Split(v, "=") - if len(tag) == 2 { - metadata[s3_constants.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1]) - } else if len(tag) == 1 { - metadata[s3_constants.AmzObjectTagging+"-"+tag[0]] = nil - } + parsedTags, err := parseTagsHeader(tags) + if err != nil { + return nil, err + } + err = ValidateTags(parsedTags) + if err != nil { + return nil, err + } + for k, v := range parsedTags { + metadata[s3_constants.AmzObjectTagging+"-"+k] = []byte(v) } } } else { diff --git a/weed/s3api/s3api_object_copy_handlers_test.go b/weed/s3api/s3api_object_copy_handlers_test.go index 610b29a6b..29d519c24 100644 --- a/weed/s3api/s3api_object_copy_handlers_test.go +++ b/weed/s3api/s3api_object_copy_handlers_test.go @@ -332,6 +332,19 @@ var processMetadataBytesTestCases = []struct { "X-Amz-Tagging-type": "request", }, }, + + { + 108, + H{ + "User-Agent": "firefox", + "X-Amz-Meta-My-Meta": "request", + "X-Amz-Tagging": "A=B&a=b&type=request*", + s3_constants.AmzUserMetaDirective: DirectiveReplace, + s3_constants.AmzObjectTaggingDirective: DirectiveReplace, + }, + H{}, + H{}, + }, } func TestProcessMetadata(t *testing.T) { @@ -339,7 +352,6 @@ func TestProcessMetadata(t *testing.T) { reqHeader := transferHToHeader(tc.request) existing := transferHToHeader(tc.existing) replaceMeta, replaceTagging := replaceDirective(reqHeader) - err := processMetadata(reqHeader, existing, replaceMeta, replaceTagging, func(_ string, _ string) (tags map[string]string, err error) { return tc.getTags, nil }, "", "") @@ -367,7 +379,7 @@ func TestProcessMetadataBytes(t *testing.T) { reqHeader := transferHToHeader(tc.request) existing := transferHToBytesArr(tc.existing) replaceMeta, replaceTagging := replaceDirective(reqHeader) - extends := processMetadataBytes(reqHeader, existing, replaceMeta, replaceTagging) + extends, _ := processMetadataBytes(reqHeader, existing, replaceMeta, replaceTagging) result := transferBytesArrToH(extends) fmtTagging(result, tc.want) diff --git a/weed/s3api/s3api_object_tagging_handlers.go b/weed/s3api/s3api_object_tagging_handlers.go index 9fde0309c..1791d7dc8 100644 --- a/weed/s3api/s3api_object_tagging_handlers.go +++ b/weed/s3api/s3api_object_tagging_handlers.go @@ -62,23 +62,12 @@ func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.R return } tags := tagging.ToTags() - if len(tags) > 10 { - glog.Errorf("PutObjectTaggingHandler tags %s: %d tags more than 10", r.URL, len(tags)) + err = ValidateTags(tags) + if err != nil { + glog.Errorf("PutObjectTaggingHandler ValidateTags error %s: %v", r.URL, err) s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag) return } - for k, v := range tags { - if len(k) > 128 { - glog.Errorf("PutObjectTaggingHandler tags %s: tag key %s longer than 128", r.URL, k) - s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag) - return - } - if len(v) > 256 { - glog.Errorf("PutObjectTaggingHandler tags %s: tag value %s longer than 256", r.URL, v) - s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag) - return - } - } if err = s3a.setTags(dir, name, tagging.ToTags()); err != nil { if err == filer_pb.ErrNotFound { diff --git a/weed/s3api/tags.go b/weed/s3api/tags.go index 979e5a80c..d49db6894 100644 --- a/weed/s3api/tags.go +++ b/weed/s3api/tags.go @@ -2,6 +2,9 @@ package s3api import ( "encoding/xml" + "fmt" + "regexp" + "strings" ) type Tag struct { @@ -37,3 +40,40 @@ func FromTags(tags map[string]string) (t *Tagging) { } return } + +func parseTagsHeader(tags string) (map[string]string, error) { + parsedTags := make(map[string]string) + for _, v := range strings.Split(tags, "&") { + tag := strings.Split(v, "=") + if len(tag) == 2 { + parsedTags[tag[0]] = tag[1] + } else if len(tag) == 1 { + parsedTags[tag[0]] = "" + } + } + return parsedTags, nil +} + +func ValidateTags(tags map[string]string) error { + if len(tags) > 10 { + return fmt.Errorf("validate tags: %d tags more than 10", len(tags)) + } + for k, v := range tags { + if len(k) > 128 { + return fmt.Errorf("validate tags: tag key longer than 128") + } + validateKey, err := regexp.MatchString(`^([\p{L}\p{Z}\p{N}_.:/=+\-@]*)$`, k) + if !validateKey || err != nil { + return fmt.Errorf("validate tags key %s error, incorrect key", k) + } + if len(v) > 256 { + return fmt.Errorf("validate tags: tag value longer than 256") + } + validateValue, err := regexp.MatchString(`^([\p{L}\p{Z}\p{N}_.:/=+\-@]*)$`, v) + if !validateValue || err != nil { + return fmt.Errorf("validate tags value %s error, incorrect value", v) + } + } + + return nil +} diff --git a/weed/s3api/tags_test.go b/weed/s3api/tags_test.go index d8beb1922..fb464fcae 100644 --- a/weed/s3api/tags_test.go +++ b/weed/s3api/tags_test.go @@ -50,3 +50,65 @@ func TestXMLMarshall(t *testing.T) { assert.Equal(t, expected, actual) } + +type TestTags map[string]string + +var ValidateTagsTestCases = []struct { + testCaseID int + tags TestTags + wantErrString string +}{ + { + 1, + TestTags{"key-1": "value-1"}, + "", + }, + { + 2, + TestTags{"key-1": "valueOver256R59YI9bahPwAVqvLeKCvM2S1RjzgP8fNDKluCbol0XTTFY6VcMwTBmdnqjsddilXztSGfEoZS1wDAIMBA0rW0CLNSoE2zNg4TT0vDbLHEtZBoZjdZ5E0JNIAqwb9ptIk2VizYmhWjb1G4rJ0CqDGWxcy3usXaQg6Dk6kU8N4hlqwYWeGw7uqdghcQ3ScfF02nHW9QFMN7msLR5fe90mbFBBp3Tjq34i0LEr4By2vxoRa2RqdBhEJhi23Tm"}, + "validate tags: tag value longer than 256", + }, + { + 3, + TestTags{"keyLenOver128a5aUUGcPexMELsz3RyROzIzfO6BKABeApH2nbbagpOxZh2MgBWYDZtFxQaCuQeP1xR7dUJLwfFfDHguVIyxvTStGDk51BemKETIwZ0zkhR7lhfHBp2y0nFnV": "value-1"}, + "validate tags: tag key longer than 128", + }, + { + 4, + TestTags{"key-1*": "value-1"}, + "validate tags key key-1* error, incorrect key", + }, + { + 5, + TestTags{"key-1": "value-1?"}, + "validate tags value value-1? error, incorrect value", + }, + { + 6, + TestTags{ + "key-1": "value", + "key-2": "value", + "key-3": "value", + "key-4": "value", + "key-5": "value", + "key-6": "value", + "key-7": "value", + "key-8": "value", + "key-9": "value", + "key-10": "value", + "key-11": "value", + }, + "validate tags: 11 tags more than 10", + }, +} + +func TestValidateTags(t *testing.T) { + for _, testCase := range ValidateTagsTestCases { + err := ValidateTags(testCase.tags) + if testCase.wantErrString == "" { + assert.NoErrorf(t, err, "no error") + } else { + assert.EqualError(t, err, testCase.wantErrString) + } + } +}