From 86cdb2a3e5e341804005e2cd3556e533170d2733 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 11 Nov 2020 02:01:24 -0800 Subject: [PATCH 1/2] adjust logs --- weed/s3api/s3api_object_copy_handlers.go | 3 +++ weed/s3api/s3api_object_handlers.go | 2 +- weed/s3api/s3api_object_multipart_handlers.go | 16 +++++++++------- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/weed/s3api/s3api_object_copy_handlers.go b/weed/s3api/s3api_object_copy_handlers.go index 99a852c0c..ca578e7e5 100644 --- a/weed/s3api/s3api_object_copy_handlers.go +++ b/weed/s3api/s3api_object_copy_handlers.go @@ -2,6 +2,7 @@ package s3api import ( "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "net/http" "net/url" @@ -47,6 +48,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request } defer util.CloseResponse(resp) + glog.V(2).Infof("copy from %s to %s", srcUrl, dstUrl) etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body) if errCode != s3err.ErrNone { @@ -127,6 +129,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req } defer dataReader.Close() + glog.V(2).Infof("copy from %s to %s", srcUrl, dstUrl) etag, errCode := s3a.putToFiler(r, dstUrl, dataReader) if errCode != s3err.ErrNone { diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index fe134c102..7ea49f2c6 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -323,7 +323,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader resp_body, ra_err := ioutil.ReadAll(resp.Body) if ra_err != nil { - glog.Errorf("upload to filer response read: %v", ra_err) + glog.Errorf("upload to filer response read %d: %v", resp.StatusCode, ra_err) return etag, s3err.ErrInternalError } var ret weed_server.FilerPostResult diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 0c0e8b245..4ddb24e31 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -2,6 +2,7 @@ package s3api import ( "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "net/http" "net/url" @@ -28,13 +29,13 @@ func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http Key: objectKey(aws.String(object)), }) + glog.V(2).Info("NewMultipartUploadHandler", string(encodeResponse(response)), errCode) + if errCode != s3err.ErrNone { writeErrorResponse(w, errCode, r.URL) return } - // println("NewMultipartUploadHandler", string(encodeResponse(response))) - writeSuccessResponseXML(w, encodeResponse(response)) } @@ -52,7 +53,7 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r UploadId: aws.String(uploadID), }) - // println("CompleteMultipartUploadHandler", string(encodeResponse(response)), errCode) + glog.V(2).Info("CompleteMultipartUploadHandler", string(encodeResponse(response)), errCode) if errCode != s3err.ErrNone { writeErrorResponse(w, errCode, r.URL) @@ -81,7 +82,7 @@ func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *ht return } - // println("AbortMultipartUploadHandler", string(encodeResponse(response))) + glog.V(2).Info("AbortMultipartUploadHandler", string(encodeResponse(response))) writeSuccessResponseXML(w, encodeResponse(response)) @@ -114,13 +115,14 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht UploadIdMarker: aws.String(uploadIDMarker), }) + glog.V(2).Info("ListMultipartUploadsHandler", string(encodeResponse(response)), errCode) + if errCode != s3err.ErrNone { writeErrorResponse(w, errCode, r.URL) return } // TODO handle encodingType - // println("ListMultipartUploadsHandler", string(encodeResponse(response))) writeSuccessResponseXML(w, encodeResponse(response)) } @@ -147,13 +149,13 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re UploadId: aws.String(uploadID), }) + glog.V(2).Info("ListObjectPartsHandler", string(encodeResponse(response)), errCode) + if errCode != s3err.ErrNone { writeErrorResponse(w, errCode, r.URL) return } - // println("ListObjectPartsHandler", string(encodeResponse(response))) - writeSuccessResponseXML(w, encodeResponse(response)) } From 81024761cdb94ea75cf7e6917c026eb5cb84330c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 11 Nov 2020 02:02:30 -0800 Subject: [PATCH 2/2] add local registry docker cluster --- docker/Makefile | 3 + docker/local-registry-compose.yml | 95 +++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 docker/local-registry-compose.yml diff --git a/docker/Makefile b/docker/Makefile index 166188bc3..8ab83ca18 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -12,6 +12,9 @@ build: dev: build docker-compose -f local-dev-compose.yml -p seaweedfs up +dev_registry: build + docker-compose -f local-registry-compose.yml -p seaweedfs up + cluster: build docker-compose -f local-cluster-compose.yml -p seaweedfs up diff --git a/docker/local-registry-compose.yml b/docker/local-registry-compose.yml new file mode 100644 index 000000000..da86b4d74 --- /dev/null +++ b/docker/local-registry-compose.yml @@ -0,0 +1,95 @@ +version: '2' + +services: + master: + image: chrislusf/seaweedfs:local + ports: + - 9333:9333 + - 19333:19333 + command: "master -ip=master -volumeSizeLimitMB=1024" + volume: + image: chrislusf/seaweedfs:local + ports: + - 8080:8080 + - 18080:18080 + command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1" + volumes: + - type: bind + source: /Volumes/mobile_disk/data + target: /data + depends_on: + - master + filer: + image: chrislusf/seaweedfs:local + ports: + - 8888:8888 + - 18888:18888 + command: 'filer -master="master:9333"' + depends_on: + - master + - volume + s3: + image: chrislusf/seaweedfs:local + ports: + - 8333:8333 + command: '-v 9 s3 -filer="filer:8888"' + depends_on: + - master + - volume + - filer + minio: + image: minio/minio + ports: + - 9000:9000 + command: 'minio server /data' + environment: + MINIO_ACCESS_KEY: "some_access_key1" + MINIO_SECRET_KEY: "some_secret_key1" + depends_on: + - master + registry1: + image: registry:2 + environment: + REGISTRY_HTTP_ADDR: "0.0.0.0:5001" # seaweedfs s3 + REGISTRY_LOG_LEVEL: "debug" + REGISTRY_STORAGE: "s3" + REGISTRY_STORAGE_S3_REGION: "us-east-1" + REGISTRY_STORAGE_S3_REGIONENDPOINT: "http://s3:8333" + REGISTRY_STORAGE_S3_BUCKET: "registry" + REGISTRY_STORAGE_S3_ACCESSKEY: "some_access_key1" + REGISTRY_STORAGE_S3_SECRETKEY: "some_secret_key1" + REGISTRY_STORAGE_S3_V4AUTH: "true" + REGISTRY_STORAGE_S3_SECURE: "false" + REGISTRY_STORAGE_S3_SKIPVERIFY: "true" + REGISTRY_STORAGE_S3_ROOTDIRECTORY: "/" + REGISTRY_STORAGE_DELETE_ENABLED: "true" + REGISTRY_STORAGE_REDIRECT_DISABLE: "true" + REGISTRY_VALIDATION_DISABLED: "true" + ports: + - 5001:5001 + depends_on: + - s3 + - minio + registry2: + image: registry:2 + environment: + REGISTRY_HTTP_ADDR: "0.0.0.0:5002" # minio + REGISTRY_LOG_LEVEL: "debug" + REGISTRY_STORAGE: "s3" + REGISTRY_STORAGE_S3_REGION: "us-east-1" + REGISTRY_STORAGE_S3_REGIONENDPOINT: "http://minio:9000" + REGISTRY_STORAGE_S3_BUCKET: "registry" + REGISTRY_STORAGE_S3_ACCESSKEY: "some_access_key1" + REGISTRY_STORAGE_S3_SECRETKEY: "some_secret_key1" + REGISTRY_STORAGE_S3_V4AUTH: "true" + REGISTRY_STORAGE_S3_SECURE: "false" + REGISTRY_STORAGE_S3_SKIPVERIFY: "true" + REGISTRY_STORAGE_S3_ROOTDIRECTORY: "/" + REGISTRY_STORAGE_DELETE_ENABLED: "true" + REGISTRY_STORAGE_REDIRECT_DISABLE: "true" + REGISTRY_VALIDATION_DISABLED: "true" + ports: + - 5002:5002 + depends_on: + - s3 + - minio \ No newline at end of file