From 1a41691b4c17b36b8ad39491ce579547e1ee4c04 Mon Sep 17 00:00:00 2001 From: liubaojiang <1838095916@qq.com> Date: Fri, 20 May 2022 14:33:47 +0800 Subject: [PATCH 01/38] exclude replication from the concurrentUploadLimitMB --- weed/server/volume_server.go | 2 ++ weed/server/volume_server_handlers.go | 15 ++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index 477a3709c..e557cf76b 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -24,6 +24,7 @@ type VolumeServer struct { inFlightDownloadDataSize int64 concurrentUploadLimit int64 concurrentDownloadLimit int64 + inFlightUploadDataLimitCond *sync.Cond inFlightDownloadDataLimitCond *sync.Cond SeedMasterNodes []pb.ServerAddress @@ -84,6 +85,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, fileSizeLimitBytes: int64(fileSizeLimitMB) * 1024 * 1024, isHeartbeating: true, stopChan: make(chan bool), + inFlightUploadDataLimitCond: sync.NewCond(new(sync.Mutex)), inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)), concurrentUploadLimit: concurrentUploadLimit, concurrentDownloadLimit: concurrentDownloadLimit, diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index 49bc297fb..afd32fed4 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -1,7 +1,6 @@ package weed_server import ( - "fmt" "net/http" "strconv" "strings" @@ -60,13 +59,15 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque contentLength := getContentLength(r) // exclude the replication from the concurrentUploadLimitMB - if vs.concurrentUploadLimit != 0 && r.URL.Query().Get("type") != "replicate" && - atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit { - err := fmt.Errorf("reject because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) - glog.V(1).Infof("too many requests: %v", err) - writeJsonError(w, r, http.StatusTooManyRequests, err) - return + if r.URL.Query().Get("type") != "replicate" { //Non-Replication + vs.inFlightUploadDataLimitCond.L.Lock() + for vs.concurrentUploadLimit != 0 && atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit { + glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) + vs.inFlightUploadDataLimitCond.Wait() + } + vs.inFlightUploadDataLimitCond.L.Unlock() } + atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength) defer func() { atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength) From 71b2e6223e07eaa5d70efdc8ccbe7f39ce6a0169 Mon Sep 17 00:00:00 2001 From: liubaojiang <1838095916@qq.com> Date: Fri, 20 May 2022 15:19:35 +0800 Subject: [PATCH 02/38] add inFlightUploadDataLimitCond signal --- weed/server/volume_server_handlers.go | 1 + 1 file changed, 1 insertion(+) diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index afd32fed4..c199fa46a 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -71,6 +71,7 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength) defer func() { atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength) + vs.inFlightUploadDataLimitCond.Signal() }() // processs uploads From 076e48a6761396034b6c8132278330593ca698c5 Mon Sep 17 00:00:00 2001 From: liubaojiang <1838095916@qq.com> Date: Fri, 20 May 2022 18:18:20 +0800 Subject: [PATCH 03/38] add inflight upload data wait timeout --- weed/command/server.go | 1 + weed/command/volume.go | 5 ++++- weed/server/volume_server.go | 4 ++++ weed/server/volume_server_handlers.go | 28 +++++++++++++++++++-------- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/weed/command/server.go b/weed/command/server.go index 4b6b6c642..405d432f0 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -131,6 +131,7 @@ func init() { serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile") serverOptions.v.idxFolder = cmdServer.Flag.String("volume.dir.idx", "", "directory to store .idx files") serverOptions.v.enableTcp = cmdServer.Flag.Bool("volume.tcp", false, " enable tcp port") + serverOptions.v.inflightUploadDataTimeout = cmdServer.Flag.Duration("volume.inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers") s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port") s3Options.portGrpc = cmdServer.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port") diff --git a/weed/command/volume.go b/weed/command/volume.go index b1455352c..158bdf162 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -65,7 +65,8 @@ type VolumeServerOptions struct { preStopSeconds *int metricsHttpPort *int // pulseSeconds *int - enableTcp *bool + enableTcp *bool + inflightUploadDataTimeout *time.Duration } func init() { @@ -96,6 +97,7 @@ func init() { v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files") v.enableTcp = cmdVolume.Flag.Bool("tcp", false, " enable tcp port") + v.inflightUploadDataTimeout = cmdVolume.Flag.Duration("inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers") } var cmdVolume = &Command{ @@ -244,6 +246,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v *v.fileSizeLimitMB, int64(*v.concurrentUploadLimitMB)*1024*1024, int64(*v.concurrentDownloadLimitMB)*1024*1024, + *v.inflightUploadDataTimeout, ) // starting grpc server grpcS := v.startGrpcService(volumeServer) diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index e557cf76b..abb30229a 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -3,6 +3,7 @@ package weed_server import ( "net/http" "sync" + "time" "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" @@ -26,6 +27,7 @@ type VolumeServer struct { concurrentDownloadLimit int64 inFlightUploadDataLimitCond *sync.Cond inFlightDownloadDataLimitCond *sync.Cond + inflightUploadDataTimeout time.Duration SeedMasterNodes []pb.ServerAddress currentMaster pb.ServerAddress @@ -61,6 +63,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, fileSizeLimitMB int, concurrentUploadLimit int64, concurrentDownloadLimit int64, + inflightUploadDataTimeout time.Duration, ) *VolumeServer { v := util.GetViper() @@ -89,6 +92,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)), concurrentUploadLimit: concurrentUploadLimit, concurrentDownloadLimit: concurrentDownloadLimit, + inflightUploadDataTimeout: inflightUploadDataTimeout, } vs.SeedMasterNodes = masterNodes diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index c199fa46a..aa231e650 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -1,10 +1,12 @@ package weed_server import ( + "fmt" "net/http" "strconv" "strings" "sync/atomic" + "time" "github.com/chrislusf/seaweedfs/weed/util" @@ -55,22 +57,32 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque vs.guard.WhiteList(vs.DeleteHandler)(w, r) case "PUT", "POST": - // wait until in flight data is less than the limit contentLength := getContentLength(r) - + startTime := time.Now() + vs.inFlightUploadDataLimitCond.L.Lock() // exclude the replication from the concurrentUploadLimitMB - if r.URL.Query().Get("type") != "replicate" { //Non-Replication - vs.inFlightUploadDataLimitCond.L.Lock() - for vs.concurrentUploadLimit != 0 && atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit { + if r.URL.Query().Get("type") != "replicate" { + for vs.concurrentUploadLimit != 0 && vs.inFlightUploadDataSize > vs.concurrentUploadLimit { + //wait timeout + if startTime.Add(vs.inflightUploadDataTimeout).Before(time.Now()) { + err := fmt.Errorf("reject because inflight upload data %d > %d, and wait timeout", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) + vs.inFlightUploadDataLimitCond.L.Unlock() + glog.V(1).Infof("too many requests: %v", err) + writeJsonError(w, r, http.StatusTooManyRequests, err) + return + } + glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) vs.inFlightUploadDataLimitCond.Wait() } - vs.inFlightUploadDataLimitCond.L.Unlock() } + vs.inFlightUploadDataSize += contentLength + vs.inFlightUploadDataLimitCond.L.Unlock() - atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength) defer func() { - atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength) + vs.inFlightUploadDataLimitCond.L.Lock() + vs.inFlightUploadDataSize -= contentLength + vs.inFlightUploadDataLimitCond.L.Unlock() vs.inFlightUploadDataLimitCond.Signal() }() From f0ee3e6f2129eb2637ea0fdb18540f18147bf474 Mon Sep 17 00:00:00 2001 From: liubaojiang <1838095916@qq.com> Date: Tue, 31 May 2022 09:40:25 +0800 Subject: [PATCH 04/38] reduce the scope of inFlightUploadDataLimitCond lock --- weed/server/volume_server_handlers.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index aa231e650..7e4c11fed 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -58,31 +58,27 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque case "PUT", "POST": contentLength := getContentLength(r) - startTime := time.Now() - vs.inFlightUploadDataLimitCond.L.Lock() // exclude the replication from the concurrentUploadLimitMB if r.URL.Query().Get("type") != "replicate" { + startTime := time.Now() + vs.inFlightUploadDataLimitCond.L.Lock() for vs.concurrentUploadLimit != 0 && vs.inFlightUploadDataSize > vs.concurrentUploadLimit { - //wait timeout + //wait timeout check if startTime.Add(vs.inflightUploadDataTimeout).Before(time.Now()) { - err := fmt.Errorf("reject because inflight upload data %d > %d, and wait timeout", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) vs.inFlightUploadDataLimitCond.L.Unlock() + err := fmt.Errorf("reject because inflight upload data %d > %d, and wait timeout", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) glog.V(1).Infof("too many requests: %v", err) writeJsonError(w, r, http.StatusTooManyRequests, err) return } - glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) vs.inFlightUploadDataLimitCond.Wait() } - } - vs.inFlightUploadDataSize += contentLength - vs.inFlightUploadDataLimitCond.L.Unlock() - - defer func() { - vs.inFlightUploadDataLimitCond.L.Lock() - vs.inFlightUploadDataSize -= contentLength vs.inFlightUploadDataLimitCond.L.Unlock() + } + atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength) + defer func() { + atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength) vs.inFlightUploadDataLimitCond.Signal() }() From 2da2ff00a3f346335c1d61324697e72c03e4bf3d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 15:19:43 +0000 Subject: [PATCH 05/38] Bump google.golang.org/api from 0.82.0 to 0.83.0 Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.82.0 to 0.83.0. - [Release notes](https://github.com/googleapis/google-api-go-client/releases) - [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.82.0...v0.83.0) --- updated-dependencies: - dependency-name: google.golang.org/api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 2fbff4d37..0e3a4cfce 100644 --- a/go.mod +++ b/go.mod @@ -123,15 +123,15 @@ require ( golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd golang.org/x/image v0.0.0-20200119044424-58c23975cae1 - golang.org/x/net v0.0.0-20220526153639-5463443f8c37 + golang.org/x/net v0.0.0-20220607020251-c690dde0001d golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401 // indirect golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect - google.golang.org/api v0.82.0 + google.golang.org/api v0.83.0 google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220527130721-00d5c0f3be58 // indirect + google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 // indirect google.golang.org/grpc v1.47.0 google.golang.org/protobuf v1.28.0 gopkg.in/inf.v0 v0.9.1 // indirect @@ -215,7 +215,7 @@ require ( go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.21.0 // indirect golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect - golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 // indirect + golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 6fc6a2787..e5101b0d5 100644 --- a/go.sum +++ b/go.sum @@ -1059,8 +1059,8 @@ golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220420153159-1850ba15e1be/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220526153639-5463443f8c37 h1:lUkvobShwKsOesNfWWlCS5q7fnbG1MEliIzwu886fn8= -golang.org/x/net v0.0.0-20220526153639-5463443f8c37/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1095,8 +1095,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 h1:w8s32wxx3sY+OjLlv9qltkLU5yvJzxjjgiHWLjdIcw4= -golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1334,8 +1334,8 @@ google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRR google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= google.golang.org/api v0.76.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= -google.golang.org/api v0.82.0 h1:h6EGeZuzhoKSS7BUznzkW+2wHZ+4Ubd6rsVvvh3dRkw= -google.golang.org/api v0.82.0/go.mod h1:Ld58BeTlL9DIYr2M2ajvoSqmGLei0BMn+kVBmkam1os= +google.golang.org/api v0.83.0 h1:pMvST+6v+46Gabac4zlJlalxZjCeRcepwg2EdBU+nCc= +google.golang.org/api v0.83.0/go.mod h1:CNywQoj/AfhTw26ZWAa6LwOv+6WFxHmeLPZq2uncLZk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1438,8 +1438,8 @@ google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46/go.mod h1:8w6bsBMX google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220527130721-00d5c0f3be58 h1:a221mAAEAzq4Lz6ZWRkcS8ptb2mxoxYSt4N68aRyQHM= -google.golang.org/genproto v0.0.0-20220527130721-00d5c0f3be58/go.mod h1:yKyY4AMRwFiC8yMMNaMi+RkCnjZJt9LoWuvhXjMs+To= +google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 h1:qRu95HZ148xXw+XeZ3dvqe85PxH4X8+jIo0iRPKcEnM= +google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8/go.mod h1:yKyY4AMRwFiC8yMMNaMi+RkCnjZJt9LoWuvhXjMs+To= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1513,6 +1513,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From d4156e4f26255cd872e5d054f9ebf48b5652f4a0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 15:20:00 +0000 Subject: [PATCH 06/38] Bump github.com/ydb-platform/ydb-go-sdk/v3 from 3.25.3 to 3.26.10 Bumps [github.com/ydb-platform/ydb-go-sdk/v3](https://github.com/ydb-platform/ydb-go-sdk) from 3.25.3 to 3.26.10. - [Release notes](https://github.com/ydb-platform/ydb-go-sdk/releases) - [Changelog](https://github.com/ydb-platform/ydb-go-sdk/blob/master/CHANGELOG.md) - [Commits](https://github.com/ydb-platform/ydb-go-sdk/compare/v3.25.3...v3.26.10) --- updated-dependencies: - dependency-name: github.com/ydb-platform/ydb-go-sdk/v3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 5 +++-- go.sum | 11 +++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 2fbff4d37..f1463cf30 100644 --- a/go.mod +++ b/go.mod @@ -155,7 +155,7 @@ require ( github.com/hashicorp/raft v1.3.9 github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 - github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3 + github.com/ydb-platform/ydb-go-sdk/v3 v3.26.10 ) require ( @@ -190,6 +190,7 @@ require ( github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/goidentity/v6 v6.0.1 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect + github.com/jonboulle/clockwork v0.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/cpuid/v2 v2.0.6 // indirect github.com/mattn/go-colorable v0.1.12 // indirect @@ -206,7 +207,7 @@ require ( github.com/subosito/gotenv v1.3.0 // indirect github.com/tinylib/msgp v1.1.6 // indirect github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect - github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc // indirect + github.com/ydb-platform/ydb-go-genproto v0.0.0-20220531094121-36ca6bddb9f7 // indirect github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect go.etcd.io/etcd/api/v3 v3.5.4 // indirect diff --git a/go.sum b/go.sum index 6fc6a2787..a523b1204 100644 --- a/go.sum +++ b/go.sum @@ -579,6 +579,8 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -868,12 +870,14 @@ github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCO github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD1vqadsDQUva6t2O9MbnyvoOgo8nFNPaOIH5U8= github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE= -github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc h1:xvTP0fhYNm+Ws+xC34jzF9EdorPUKkucJr0TyybqVSk= github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc/go.mod h1:cc138nptTn9eKptCQl/grxP6pBKpo/bnXDiOxuVZtps= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20220531094121-36ca6bddb9f7 h1:S3bwscnat3pa188pvEuXDUstxPjCnGpn5bZJuRMkL+g= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20220531094121-36ca6bddb9f7/go.mod h1:cc138nptTn9eKptCQl/grxP6pBKpo/bnXDiOxuVZtps= github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 h1:EYSI1kulnHb0H0zt3yOw4cRj4ABMSMGwNe43D+fX7e4= github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2/go.mod h1:Xfjce+VMU9yJVr1lj60yK2fFPWjB4jr/4cp3K7cjzi4= -github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3 h1:yyMw+sTDSqGVDG8ivAw8V/kXfaulXsgftSG+4nCmseA= github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88= +github.com/ydb-platform/ydb-go-sdk/v3 v3.26.10 h1:2M6L2NX8l103qNDM1i97gh81Y2EJEAEyON3/N5zQV14= +github.com/ydb-platform/ydb-go-sdk/v3 v3.26.10/go.mod h1:1U2nGytADgY4/U7OykO3LfzeS3/zz4zqg7wlUA7XHfc= github.com/ydb-platform/ydb-go-yc v0.8.3 h1:92UUUMsfvtMl6mho8eQ9lbkiPrF3a9CT+RrVRAKNRwo= github.com/ydb-platform/ydb-go-yc v0.8.3/go.mod h1:zUolAFGzJ5XG8uwiseTLr9Lapm7L7hdVdZgLSuv9FXE= github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg= @@ -1513,7 +1517,10 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From a6470579690a37318e3d49846c6c71dae31d8dfe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 15:20:05 +0000 Subject: [PATCH 07/38] Bump github.com/linxGnu/grocksdb from 1.7.2 to 1.7.3 Bumps [github.com/linxGnu/grocksdb](https://github.com/linxGnu/grocksdb) from 1.7.2 to 1.7.3. - [Release notes](https://github.com/linxGnu/grocksdb/releases) - [Commits](https://github.com/linxGnu/grocksdb/compare/v1.7.2...v1.7.3) --- updated-dependencies: - dependency-name: github.com/linxGnu/grocksdb dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 2fbff4d37..4afb13b29 100644 --- a/go.mod +++ b/go.mod @@ -68,7 +68,7 @@ require ( github.com/klauspost/reedsolomon v1.9.16 github.com/kurin/blazer v0.5.3 github.com/lib/pq v1.10.6 - github.com/linxGnu/grocksdb v1.7.2 + github.com/linxGnu/grocksdb v1.7.3 github.com/magiconair/properties v1.8.6 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-ieproxy v0.0.3 // indirect diff --git a/go.sum b/go.sum index 6fc6a2787..1eef8a2a7 100644 --- a/go.sum +++ b/go.sum @@ -636,8 +636,8 @@ github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/linxGnu/grocksdb v1.7.2 h1:5SVU++WAyOzk4jG0pXrI9plvZEPRCRbh4UrDhAxrktM= -github.com/linxGnu/grocksdb v1.7.2/go.mod h1:3tKVazPI6orwnNtFAOSRZwhi6bMOkc3+kREb8qpKaSE= +github.com/linxGnu/grocksdb v1.7.3 h1:S9XiU4FviunvjNdNG+kWe2BoOy/2EKZSdDyeGmL0vDs= +github.com/linxGnu/grocksdb v1.7.3/go.mod h1:G4zrMNj2CP2aCXF61jbmZH81tu+kU3qU4rYpOU8WOL8= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -1513,7 +1513,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From ba16477cef4fecadb9f26ef9c366a9e2fc8168bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 15:20:11 +0000 Subject: [PATCH 08/38] Bump github.com/Shopify/sarama from 1.34.0 to 1.34.1 Bumps [github.com/Shopify/sarama](https://github.com/Shopify/sarama) from 1.34.0 to 1.34.1. - [Release notes](https://github.com/Shopify/sarama/releases) - [Changelog](https://github.com/Shopify/sarama/blob/main/CHANGELOG.md) - [Commits](https://github.com/Shopify/sarama/compare/v1.34.0...v1.34.1) --- updated-dependencies: - dependency-name: github.com/Shopify/sarama dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 15 +++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 2fbff4d37..a86779642 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.15.0 github.com/OneOfOne/xxhash v1.2.8 - github.com/Shopify/sarama v1.34.0 + github.com/Shopify/sarama v1.34.1 github.com/aws/aws-sdk-go v1.44.14 github.com/beorn7/perks v1.0.1 // indirect github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 @@ -64,7 +64,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/karlseguin/ccache/v2 v2.0.8 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.15.1 // indirect + github.com/klauspost/compress v1.15.6 // indirect github.com/klauspost/reedsolomon v1.9.16 github.com/kurin/blazer v0.5.3 github.com/lib/pq v1.10.6 diff --git a/go.sum b/go.sum index 6fc6a2787..496670f16 100644 --- a/go.sum +++ b/go.sum @@ -128,10 +128,10 @@ github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= -github.com/Shopify/sarama v1.34.0 h1:j4zTaFHFnfvuV2fdLZyXqIg0Tu4Mzl9f064Z5/H+o4o= -github.com/Shopify/sarama v1.34.0/go.mod h1:V2ceE9UupUf4/oP1Z38SI49fAnD0/MtkqDDHvolIeeQ= -github.com/Shopify/toxiproxy/v2 v2.3.0 h1:62YkpiP4bzdhKMH+6uC5E95y608k3zDwdzuBMsnn3uQ= -github.com/Shopify/toxiproxy/v2 v2.3.0/go.mod h1:KvQTtB6RjCJY4zqNJn7C7JDFgsG5uoHYDirfUfpIm0c= +github.com/Shopify/sarama v1.34.1 h1:pVCQO7BMAK3s1jWhgi5v1W6lwZ6Veiekfc2vsgRS06Y= +github.com/Shopify/sarama v1.34.1/go.mod h1:NZSNswsnStpq8TUdFaqnpXm2Do6KRzTIjdBdVlL1YRM= +github.com/Shopify/toxiproxy/v2 v2.4.0 h1:O1e4Jfvr/hefNTNu+8VtdEG5lSeamJRo4aKhMOKNM64= +github.com/Shopify/toxiproxy/v2 v2.4.0/go.mod h1:3ilnjng821bkozDRxNoo64oI/DKqM+rOyJzb564+bvg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -603,9 +603,9 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY= +github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/cpuid/v2 v2.0.6 h1:dQ5ueTiftKxp0gyjKSx5+8BtPWkyQbd95m8Gys/RarI= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/reedsolomon v1.9.16 h1:mR0AwphBwqFv/I3B9AHtNKvzuowI1vrj8/3UX4XRmHA= @@ -745,6 +745,7 @@ github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3O github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -1513,6 +1514,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 1d7c0e65a0a2097147c457a251a6556faf42b304 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 15:20:23 +0000 Subject: [PATCH 09/38] Bump github.com/aws/aws-sdk-go from 1.44.14 to 1.44.32 Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.14 to 1.44.32. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.44.14...v1.44.32) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2fbff4d37..9485dc401 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/Azure/azure-storage-blob-go v0.15.0 github.com/OneOfOne/xxhash v1.2.8 github.com/Shopify/sarama v1.34.0 - github.com/aws/aws-sdk-go v1.44.14 + github.com/aws/aws-sdk-go v1.44.32 github.com/beorn7/perks v1.0.1 // indirect github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 github.com/bwmarrin/snowflake v0.3.0 diff --git a/go.sum b/go.sum index 6fc6a2787..71b3d3f2b 100644 --- a/go.sum +++ b/go.sum @@ -150,8 +150,8 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.44.14 h1:qd7/muV1rElsbvkK9D1nHUzBoDlEw2etfeo4IE82eSQ= -github.com/aws/aws-sdk-go v1.44.14/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.32 h1:x5hBtpY/02sgRL158zzTclcCLwh3dx3YlSl1rAH4Op0= +github.com/aws/aws-sdk-go v1.44.32/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go-v2 v1.16.2 h1:fqlCk6Iy3bnCumtrLz9r3mJ/2gUT0pJ0wLFVIdWh+JA= github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1/go.mod h1:n8Bs1ElDD2wJ9kCRTczA83gYbBmjSwZp3umc6zF4EeM= @@ -1513,6 +1513,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 2b159c331d5ec9a09aae831725fbb02238abe9dd Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 14 Jun 2022 13:16:09 +0500 Subject: [PATCH 10/38] fix build container --- .github/workflows/container_latest.yml | 2 +- .github/workflows/container_release1.yml | 2 +- .github/workflows/container_release2.yml | 2 +- .github/workflows/container_release3.yml | 2 +- .github/workflows/container_release4.yml | 2 +- .github/workflows/container_release5.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index 04939b8d6..35dcea714 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -4,7 +4,7 @@ on: push: tags: - '*' - workflow_dispatch: [] + workflow_dispatch: {} permissions: contents: read diff --git a/.github/workflows/container_release1.yml b/.github/workflows/container_release1.yml index b8ff81a1b..1bcf768cd 100644 --- a/.github/workflows/container_release1.yml +++ b/.github/workflows/container_release1.yml @@ -4,7 +4,7 @@ on: push: tags: - '*' - workflow_dispatch: [] + workflow_dispatch: {} permissions: contents: read diff --git a/.github/workflows/container_release2.yml b/.github/workflows/container_release2.yml index a02ab4f87..c58bb2b40 100644 --- a/.github/workflows/container_release2.yml +++ b/.github/workflows/container_release2.yml @@ -4,7 +4,7 @@ on: push: tags: - '*' - workflow_dispatch: [] + workflow_dispatch: {} permissions: contents: read diff --git a/.github/workflows/container_release3.yml b/.github/workflows/container_release3.yml index 104a26ef6..5ff6cd497 100644 --- a/.github/workflows/container_release3.yml +++ b/.github/workflows/container_release3.yml @@ -4,7 +4,7 @@ on: push: tags: - '*' - workflow_dispatch: [] + workflow_dispatch: {} permissions: contents: read diff --git a/.github/workflows/container_release4.yml b/.github/workflows/container_release4.yml index 92d776f79..0788abeca 100644 --- a/.github/workflows/container_release4.yml +++ b/.github/workflows/container_release4.yml @@ -4,7 +4,7 @@ on: push: tags: - '*' - workflow_dispatch: [] + workflow_dispatch: {} permissions: contents: read diff --git a/.github/workflows/container_release5.yml b/.github/workflows/container_release5.yml index 820527f5b..e536a20b7 100644 --- a/.github/workflows/container_release5.yml +++ b/.github/workflows/container_release5.yml @@ -4,7 +4,7 @@ on: push: tags: - '*' - workflow_dispatch: [] + workflow_dispatch: {} permissions: contents: read From cc5d7710880491b7c72ca0dbd60008a9c3a84f08 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 14 Jun 2022 01:37:28 -0700 Subject: [PATCH 11/38] Update container_dev.yml --- .github/workflows/container_dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index ac93e4537..d8a2312ea 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -3,7 +3,7 @@ name: "docker: build dev containers" on: push: branches: [ master ] - workflow_dispatch: [] + workflow_dispatch: {} permissions: contents: read From 322dbcf372d75f144b01a17b2a19195fe43b038a Mon Sep 17 00:00:00 2001 From: guol-fnst Date: Tue, 14 Jun 2022 19:01:57 +0800 Subject: [PATCH 12/38] add update feature --- weed/command/command.go | 4 +- weed/command/update.go | 386 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 389 insertions(+), 1 deletion(-) create mode 100644 weed/command/update.go diff --git a/weed/command/command.go b/weed/command/command.go index dbc18a053..b16de3eba 100644 --- a/weed/command/command.go +++ b/weed/command/command.go @@ -2,9 +2,10 @@ package command import ( "fmt" - flag "github.com/chrislusf/seaweedfs/weed/util/fla9" "os" "strings" + + flag "github.com/chrislusf/seaweedfs/weed/util/fla9" ) var Commands = []*Command{ @@ -37,6 +38,7 @@ var Commands = []*Command{ cmdServer, cmdShell, cmdUpload, + cmdUpdate, cmdVersion, cmdVolume, cmdWebDav, diff --git a/weed/command/update.go b/weed/command/update.go new file mode 100644 index 000000000..bda904a85 --- /dev/null +++ b/weed/command/update.go @@ -0,0 +1,386 @@ +package command + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/util" + "golang.org/x/net/context/ctxhttp" +) + +// Release collects data about a single release on GitHub. +type Release struct { + Name string `json:"name"` + TagName string `json:"tag_name"` + Draft bool `json:"draft"` + PreRelease bool `json:"prerelease"` + PublishedAt time.Time `json:"published_at"` + Assets []Asset `json:"assets"` + + Version string `json:"-"` // set manually in the code +} + +// Asset is a file uploaded and attached to a release. +type Asset struct { + ID int `json:"id"` + Name string `json:"name"` + URL string `json:"url"` +} + +var ( + updateOpt UpdateOptions +) + +type UpdateOptions struct { + Output *string +} + +func init() { + updateOpt.Output = cmdUpdate.Flag.String("output", "weed", "Save the latest weed as `filename`") + cmdUpdate.Run = runUpdate +} + +var cmdUpdate = &Command{ + UsageLine: "update [-output=weed]", + Short: "get latest stable version from https://github.com/chrislusf/seaweedfs", + Long: `get latest stable version from https://github.com/chrislusf/seaweedfs`, +} + +func runUpdate(cmd *Command, args []string) bool { + weedPath := *updateOpt.Output + if weedPath == "" { + file, err := os.Executable() + if err != nil { + glog.Fatalf("unable to find executable:%s", err) + return false + } + + *updateOpt.Output = file + } + + fi, err := os.Lstat(*updateOpt.Output) + if err != nil { + dirname := filepath.Dir(*updateOpt.Output) + di, err := os.Lstat(dirname) + if err != nil { + glog.Fatalf("unable to find directory:%s", dirname) + return false + } + if !di.Mode().IsDir() { + glog.Fatalf("output parent path %v is not a directory, use --output to specify a different file path", dirname) + return false + } + } else { + if !fi.Mode().IsRegular() { + glog.Fatalf("output path %v is not a normal file, use --output to specify a different file path", updateOpt.Output) + return false + } + } + + glog.V(0).Infof("writing weed to %v\n", *updateOpt.Output) + + v, err := downloadLatestStableRelease(context.Background(), *updateOpt.Output) + if err != nil { + glog.Fatalf("unable to update weed: %v", err) + return false + } + + glog.V(0).Infof("successfully updated weed to version %v\n", v) + + return true +} + +func downloadLatestStableRelease(ctx context.Context, target string) (version string, err error) { + currentVersion := util.VERSION_NUMBER + largeDiskSuffix := "" + if util.VolumeSizeLimitGB == 8000 { + largeDiskSuffix = "_large_disk" + } + + rel, err := GitHubLatestRelease(ctx, "chrislusf", "seaweedfs") + if err != nil { + return "", err + } + + if rel.Version == currentVersion { + glog.V(0).Infof("weed is up to date\n") + return currentVersion, nil + } + + glog.V(0).Infof("latest version is %v\n", rel.Version) + + ext := "tar.gz" + if runtime.GOOS == "windows" { + ext = "zip" + } + + suffix := fmt.Sprintf("%s_%s%s.%s", runtime.GOOS, runtime.GOARCH, largeDiskSuffix, ext) + md5Filename := fmt.Sprintf("%s.md5", suffix) + _, md5Val, err := getGithubDataFile(ctx, rel.Assets, md5Filename) + if err != nil { + return "", err + } + + downloadFilename, buf, err := getGithubDataFile(ctx, rel.Assets, suffix) + if err != nil { + return "", err + } + + md5Ctx := md5.New() + md5Ctx.Write(buf) + binaryMd5 := md5Ctx.Sum(nil) + if hex.EncodeToString(binaryMd5) != string(md5Val[0:32]) { + glog.Errorf("md5:'%s' '%s'", hex.EncodeToString(binaryMd5), string(md5Val[0:32])) + err = errors.New("binary md5sum doesn't match") + return "", err + } + + err = extractToFile(buf, downloadFilename, target) + if err != nil { + return "", err + } + + return rel.Version, nil +} + +func (r Release) String() string { + return fmt.Sprintf("%v %v, %d assets", + r.TagName, + r.PublishedAt.Local().Format("2006-01-02 15:04:05"), + len(r.Assets)) +} + +const githubAPITimeout = 30 * time.Second + +// githubError is returned by the GitHub API, e.g. for rate-limiting. +type githubError struct { + Message string +} + +// GitHubLatestRelease uses the GitHub API to get information about the latest +// release of a repository. +func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, error) { + ctx, cancel := context.WithTimeout(ctx, githubAPITimeout) + defer cancel() + + url := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/latest", owner, repo) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return Release{}, err + } + + // pin API version 3 + req.Header.Set("Accept", "application/vnd.github.v3+json") + + res, err := ctxhttp.Do(ctx, http.DefaultClient, req) + if err != nil { + return Release{}, err + } + + if res.StatusCode != http.StatusOK { + content := res.Header.Get("Content-Type") + if strings.Contains(content, "application/json") { + // try to decode error message + var msg githubError + jerr := json.NewDecoder(res.Body).Decode(&msg) + if jerr == nil { + return Release{}, fmt.Errorf("unexpected status %v (%v) returned, message:\n %v", res.StatusCode, res.Status, msg.Message) + } + } + + _ = res.Body.Close() + return Release{}, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status) + } + + buf, err := ioutil.ReadAll(res.Body) + if err != nil { + _ = res.Body.Close() + return Release{}, err + } + + err = res.Body.Close() + if err != nil { + return Release{}, err + } + + var release Release + err = json.Unmarshal(buf, &release) + if err != nil { + return Release{}, err + } + + if release.TagName == "" { + return Release{}, errors.New("tag name for latest release is empty") + } + + release.Version = release.TagName + + return release, nil +} + +func getGithubData(ctx context.Context, url string) ([]byte, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + // request binary data + req.Header.Set("Accept", "application/octet-stream") + + res, err := ctxhttp.Do(ctx, http.DefaultClient, req) + if err != nil { + return nil, err + } + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status) + } + + buf, err := ioutil.ReadAll(res.Body) + if err != nil { + _ = res.Body.Close() + return nil, err + } + + err = res.Body.Close() + if err != nil { + return nil, err + } + + return buf, nil +} + +func getGithubDataFile(ctx context.Context, assets []Asset, suffix string) (filename string, data []byte, err error) { + var url string + for _, a := range assets { + if strings.HasSuffix(a.Name, suffix) { + url = a.URL + filename = a.Name + break + } + } + + if url == "" { + return "", nil, fmt.Errorf("unable to find file with suffix %v", suffix) + } + + glog.V(0).Infof("download %v\n", filename) + data, err = getGithubData(ctx, url) + if err != nil { + return "", nil, err + } + + return filename, data, nil +} + +func extractToFile(buf []byte, filename, target string) error { + var rd io.Reader = bytes.NewReader(buf) + + switch filepath.Ext(filename) { + case ".gz": + gr, err := gzip.NewReader(rd) + if err != nil { + return err + } + defer gr.Close() + trd := tar.NewReader(gr) + hdr, terr := trd.Next() + if terr != nil { + glog.Errorf("uncompress file(%s) failed:%s", hdr.Name, terr) + return terr + } + rd = trd + case ".zip": + zrd, err := zip.NewReader(bytes.NewReader(buf), int64(len(buf))) + if err != nil { + return err + } + + if len(zrd.File) != 1 { + return errors.New("ZIP archive contains more than one file") + } + + file, err := zrd.File[0].Open() + if err != nil { + return err + } + + defer func() { + _ = file.Close() + }() + + rd = file + } + + // Write everything to a temp file + dir := filepath.Dir(target) + new, err := ioutil.TempFile(dir, "weed") + if err != nil { + return err + } + + n, err := io.Copy(new, rd) + if err != nil { + _ = new.Close() + _ = os.Remove(new.Name()) + return err + } + if err = new.Sync(); err != nil { + return err + } + if err = new.Close(); err != nil { + return err + } + + mode := os.FileMode(0755) + // attempt to find the original mode + if fi, err := os.Lstat(target); err == nil { + mode = fi.Mode() + } + + // Remove the original binary. + if err := removeWeedBinary(dir, target); err != nil { + return err + } + + // Rename the temp file to the final location atomically. + if err := os.Rename(new.Name(), target); err != nil { + return err + } + + glog.V(0).Infof("saved %d bytes in %v\n", n, target) + return os.Chmod(target, mode) +} + +// Rename (rather than remove) the running version. The running binary will be locked +// on Windows and cannot be removed while still executing. +func removeWeedBinary(dir, target string) error { + if runtime.GOOS == "linux" { + return nil + } + backup := filepath.Join(dir, filepath.Base(target)+".bak") + if _, err := os.Stat(backup); err == nil { + _ = os.Remove(backup) + } + if err := os.Rename(target, backup); err != nil { + return fmt.Errorf("unable to rename target file: %v", err) + } + return nil +} From 78fc9cdfa0cdd973d68be432cdcefb804ae5b723 Mon Sep 17 00:00:00 2001 From: guol-fnst Date: Tue, 14 Jun 2022 19:10:54 +0800 Subject: [PATCH 13/38] add "copied from https://github.com/restic/restic/tree/master/internal/selfupdate" --- weed/command/command.go | 2 +- weed/command/update.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/weed/command/command.go b/weed/command/command.go index b16de3eba..7635405dc 100644 --- a/weed/command/command.go +++ b/weed/command/command.go @@ -37,8 +37,8 @@ var Commands = []*Command{ cmdScaffold, cmdServer, cmdShell, - cmdUpload, cmdUpdate, + cmdUpload, cmdVersion, cmdVolume, cmdWebDav, diff --git a/weed/command/update.go b/weed/command/update.go index bda904a85..64a664923 100644 --- a/weed/command/update.go +++ b/weed/command/update.go @@ -25,6 +25,8 @@ import ( "golang.org/x/net/context/ctxhttp" ) +//copied from https://github.com/restic/restic/tree/master/internal/selfupdate + // Release collects data about a single release on GitHub. type Release struct { Name string `json:"name"` @@ -59,8 +61,8 @@ func init() { var cmdUpdate = &Command{ UsageLine: "update [-output=weed]", - Short: "get latest stable version from https://github.com/chrislusf/seaweedfs", - Long: `get latest stable version from https://github.com/chrislusf/seaweedfs`, + Short: "get latest version from https://github.com/chrislusf/seaweedfs", + Long: `get latest version from https://github.com/chrislusf/seaweedfs`, } func runUpdate(cmd *Command, args []string) bool { From 14d82c3deaee2b7eecd95c5432c9a60b23a94033 Mon Sep 17 00:00:00 2001 From: "zhihao.qu" Date: Tue, 14 Jun 2022 19:46:02 +0800 Subject: [PATCH 14/38] feat(filer.sync): add offset to path. --- weed/command/filer_sync.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go index 7aa9c1e8d..dc7c569c3 100644 --- a/weed/command/filer_sync.go +++ b/weed/command/filer_sync.go @@ -182,7 +182,7 @@ func doSubscribeFilerMetaChanges(clientId int32, grpcDialOption grpc.DialOption, // if first time, start from now // if has previously synced, resume from that point of time - sourceFilerOffsetTsNs, err := getOffset(grpcDialOption, targetFiler, SyncKeyPrefix, sourceFilerSignature) + sourceFilerOffsetTsNs, err := getOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature) if err != nil { return err } @@ -214,7 +214,7 @@ func doSubscribeFilerMetaChanges(clientId int32, grpcDialOption grpc.DialOption, now := time.Now().Nanosecond() glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, lastTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9)) lastLogTsNs = now - return setOffset(grpcDialOption, targetFiler, SyncKeyPrefix, sourceFilerSignature, lastTsNs) + return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, lastTsNs) }) return pb.FollowMetadata(sourceFiler, grpcDialOption, "syncTo_"+string(targetFiler), clientId, @@ -226,6 +226,16 @@ const ( SyncKeyPrefix = "sync." ) +// When each business is distinguished according to path, and offsets need to be maintained separately. +func getSignaturePrefixByPath(path string) string { + // compatible historical version + if path == "/" { + return SyncKeyPrefix + } else { + return SyncKeyPrefix + path + } +} + func getOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32) (lastOffsetTsNs int64, readErr error) { readErr = pb.WithFilerClient(false, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { From 28cda854eb1dc3fa648ff4950cabf6bfafe05902 Mon Sep 17 00:00:00 2001 From: "yulai.li" Date: Wed, 15 Jun 2022 00:30:49 +0800 Subject: [PATCH 15/38] Add filer command line parameter to let Filer UI show/hide directory delete button --- weed/command/filer.go | 3 ++ weed/server/filer_server.go | 3 +- weed/server/filer_server_handlers_read_dir.go | 7 +++- weed/server/filer_ui/filer.html | 40 +++++++++---------- 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/weed/command/filer.go b/weed/command/filer.go index 2c91e6fec..e558b16aa 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -55,6 +55,7 @@ type FilerOptions struct { debug *bool debugPort *int localSocket *string + showUIDirectoryDelete *bool } func init() { @@ -82,6 +83,7 @@ func init() { f.debug = cmdFiler.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:/debug/pprof/goroutine?debug=2") f.debugPort = cmdFiler.Flag.Int("debug.port", 6060, "http port for debugging") f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-.sock") + f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", false, "enable filer UI show delete directory button") // start s3 on filer filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway") @@ -216,6 +218,7 @@ func (fo *FilerOptions) startFiler() { Cipher: *fo.cipher, SaveToFilerLimit: int64(*fo.saveToFilerLimit), ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024, + ShowUIDirectoryDelete: *fo.showUIDirectoryDelete, }) if nfs_err != nil { glog.Fatalf("Filer startup error: %v", nfs_err) diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 8908b5e5f..6bf0261ee 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -3,7 +3,6 @@ package weed_server import ( "context" "fmt" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "net/http" "os" "sync" @@ -17,6 +16,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/util" @@ -67,6 +67,7 @@ type FilerOption struct { Cipher bool SaveToFilerLimit int64 ConcurrentUploadLimit int64 + ShowUIDirectoryDelete bool } type FilerServer struct { diff --git a/weed/server/filer_server_handlers_read_dir.go b/weed/server/filer_server_handlers_read_dir.go index 8382cfc76..eaf17fa18 100644 --- a/weed/server/filer_server_handlers_read_dir.go +++ b/weed/server/filer_server_handlers_read_dir.go @@ -73,7 +73,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque return } - ui.StatusTpl.Execute(w, struct { + err = ui.StatusTpl.Execute(w, struct { Path string Breadcrumbs []ui.Breadcrumb Entries interface{} @@ -81,6 +81,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque LastFileName string ShouldDisplayLoadMore bool EmptyFolder bool + ShowDirectoryDelete bool }{ path, ui.ToBreadcrumb(path), @@ -89,5 +90,9 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque lastFileName, shouldDisplayLoadMore, emptyFolder, + fs.option.ShowUIDirectoryDelete, }) + if err != nil { + glog.V(0).Infof("Template Execute Error: %v", err) + } } diff --git a/weed/server/filer_ui/filer.html b/weed/server/filer_ui/filer.html index f9c35440e..c9d832e8f 100644 --- a/weed/server/filer_ui/filer.html +++ b/weed/server/filer_ui/filer.html @@ -109,38 +109,37 @@
- {{if .EmptyFolder}} + {{ if .EmptyFolder }}
+
- {{else}} + {{ else }} - {{$path := .Path }} + {{ $path := .Path }} + {{ $showDirDel := .ShowDirectoryDelete }} {{ range $entry_index, $entry := .Entries }} {{ end }}
- {{if $entry.IsDirectory}} + {{ if $entry.IsDirectory }}   {{ $entry.Name }} - {{else}} + {{ else }} {{ $entry.Name }} - {{end}} + {{ end }} - {{if $entry.IsDirectory}} - {{else}} + {{ if not $entry.IsDirectory }} {{ $entry.Mime }}  - {{end}} + {{ end }} - {{if $entry.IsDirectory}} - {{else}} + {{ if not $entry.IsDirectory }} {{ $entry.Size | humanizeBytes }}  - {{end}} + {{ end }} {{ $entry.Timestamp.Format "2006-01-02 15:04" }} @@ -150,31 +149,32 @@ - {{if $entry.IsDirectory}} -
- {{end}} + {{ end }}
- {{if .ShouldDisplayLoadMore}} + {{ if .ShouldDisplayLoadMore }}
- Date: Tue, 14 Jun 2022 12:14:52 -0700 Subject: [PATCH 17/38] 3.11 --- k8s/helm_charts2/Chart.yaml | 4 ++-- weed/util/constants.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/helm_charts2/Chart.yaml b/k8s/helm_charts2/Chart.yaml index edb4d7f72..5e5fc59b4 100644 --- a/k8s/helm_charts2/Chart.yaml +++ b/k8s/helm_charts2/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -appVersion: "3.10" -version: "3.10" +appVersion: "3.11" +version: "3.11" diff --git a/weed/util/constants.go b/weed/util/constants.go index 213eafae0..c0fea8b17 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -5,7 +5,7 @@ import ( ) var ( - VERSION_NUMBER = fmt.Sprintf("%.02f", 3.10) + VERSION_NUMBER = fmt.Sprintf("%.02f", 3.11) VERSION = sizeLimit + " " + VERSION_NUMBER COMMIT = "" ) From 42d04c581b174d21ecbd137bac87256adeaa9fd3 Mon Sep 17 00:00:00 2001 From: "zhihao.qu" Date: Wed, 15 Jun 2022 11:33:18 +0800 Subject: [PATCH 18/38] feat(filer.sync): add metricsServer in filer.sync. Metrics include: (1) the offset of the filer.sync (2) the last send timestamp of the filer subscription --- weed/command/filer_sync.go | 11 ++++++++++- weed/server/filer_grpc_server_sub_meta.go | 4 ++++ weed/stats/metrics.go | 18 ++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go index dc7c569c3..dca4e693f 100644 --- a/weed/command/filer_sync.go +++ b/weed/command/filer_sync.go @@ -12,6 +12,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/replication/sink/filersink" "github.com/chrislusf/seaweedfs/weed/replication/source" "github.com/chrislusf/seaweedfs/weed/security" + statsCollect "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util/grace" "google.golang.org/grpc" @@ -40,6 +41,7 @@ type SyncOptions struct { bFromTsMs *int64 aProxyByFiler *bool bProxyByFiler *bool + metricsHttpPort *int clientId int32 } @@ -72,6 +74,7 @@ func init() { syncOptions.bFromTsMs = cmdFilerSynchronize.Flag.Int64("b.fromTsMs", 0, "synchronization from timestamp on filer B. The unit is millisecond") syncCpuProfile = cmdFilerSynchronize.Flag.String("cpuprofile", "", "cpu profile output file") syncMemProfile = cmdFilerSynchronize.Flag.String("memprofile", "", "memory profile output file") + syncOptions.metricsHttpPort = cmdFilerSynchronize.Flag.Int("metricsPort", 0, "metrics listen port") syncOptions.clientId = util.RandomInt32() } @@ -103,6 +106,9 @@ func runFilerSynchronize(cmd *Command, args []string) bool { filerA := pb.ServerAddress(*syncOptions.filerA) filerB := pb.ServerAddress(*syncOptions.filerB) + // start filer.sync metrics server + go statsCollect.StartMetricsServer(*syncOptions.metricsHttpPort) + // read a filer signature aFilerSignature, aFilerErr := replication.ReadFilerSignature(grpcDialOption, filerA) if aFilerErr != nil { @@ -210,14 +216,17 @@ func doSubscribeFilerMetaChanges(clientId int32, grpcDialOption grpc.DialOption, } var lastLogTsNs = time.Now().Nanosecond() + var clientName = fmt.Sprintf("syncTo_%s_From_%s", string(targetFiler), string(sourceFiler)) processEventFnWithOffset := pb.AddOffsetFunc(processEventFn, 3*time.Second, func(counter int64, lastTsNs int64) error { now := time.Now().Nanosecond() glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, lastTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9)) lastLogTsNs = now + // collect synchronous offset + statsCollect.FilerSyncOffsetGauge.WithLabelValues(sourceFiler.String(), targetFiler.String(), clientName, sourcePath).Set(float64(lastTsNs)) return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, lastTsNs) }) - return pb.FollowMetadata(sourceFiler, grpcDialOption, "syncTo_"+string(targetFiler), clientId, + return pb.FollowMetadata(sourceFiler, grpcDialOption, clientName, clientId, sourcePath, nil, sourceFilerOffsetTsNs, 0, targetFilerSignature, processEventFnWithOffset, pb.RetryForeverOnError) } diff --git a/weed/server/filer_grpc_server_sub_meta.go b/weed/server/filer_grpc_server_sub_meta.go index 0540400a3..da710234b 100644 --- a/weed/server/filer_grpc_server_sub_meta.go +++ b/weed/server/filer_grpc_server_sub_meta.go @@ -2,6 +2,7 @@ package weed_server import ( "fmt" + "github.com/chrislusf/seaweedfs/weed/stats" "strings" "time" @@ -229,6 +230,9 @@ func (fs *FilerServer) eachEventNotificationFn(req *filer_pb.SubscribeMetadataRe } } + // collect timestamps for path + stats.FilerServerLastSendTsOfSubscribeGauge.WithLabelValues(fs.option.Host.String(), req.ClientName, req.PathPrefix).Set(float64(tsNs)) + message := &filer_pb.SubscribeMetadataResponse{ Directory: dirPath, EventNotification: eventNotification, diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index 943aafff9..207b37c81 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -77,6 +77,14 @@ var ( Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24), }, []string{"type"}) + FilerServerLastSendTsOfSubscribeGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "SeaweedFS", + Subsystem: "filer", + Name: "last_send_timestamp_of_subscribe", + Help: "The last send timestamp of the filer subscription.", + }, []string{"sourceFiler", "clientName", "path"}) + FilerStoreCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "SeaweedFS", @@ -94,6 +102,14 @@ var ( Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24), }, []string{"store", "type"}) + FilerSyncOffsetGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "SeaweedFS", + Subsystem: "filerSync", + Name: "sync_offset", + Help: "The offset of the filer synchronization service.", + }, []string{"sourceFiler", "targetFiler", "clientName", "path"}) + VolumeServerRequestCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "SeaweedFS", @@ -179,6 +195,8 @@ func init() { Gather.MustRegister(FilerRequestHistogram) Gather.MustRegister(FilerStoreCounter) Gather.MustRegister(FilerStoreHistogram) + Gather.MustRegister(FilerSyncOffsetGauge) + Gather.MustRegister(FilerServerLastSendTsOfSubscribeGauge) Gather.MustRegister(collectors.NewGoCollector()) Gather.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) From b90d213a87f94d22743000a977901fcb376ec07b Mon Sep 17 00:00:00 2001 From: "yulai.li" Date: Wed, 15 Jun 2022 11:37:37 +0800 Subject: [PATCH 19/38] Change ui.deleteDir default to true --- weed/command/filer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/command/filer.go b/weed/command/filer.go index e558b16aa..c9f9a1956 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -83,7 +83,7 @@ func init() { f.debug = cmdFiler.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:/debug/pprof/goroutine?debug=2") f.debugPort = cmdFiler.Flag.Int("debug.port", 6060, "http port for debugging") f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-.sock") - f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", false, "enable filer UI show delete directory button") + f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button") // start s3 on filer filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway") From 4d0d1848c648c3e2e6596451fa20eabc468239dc Mon Sep 17 00:00:00 2001 From: "zhihao.qu" Date: Wed, 15 Jun 2022 13:33:20 +0800 Subject: [PATCH 20/38] fix(filer.sync): modify clientName format : from -> to --- weed/command/filer_sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go index dca4e693f..b7da1baf9 100644 --- a/weed/command/filer_sync.go +++ b/weed/command/filer_sync.go @@ -216,7 +216,7 @@ func doSubscribeFilerMetaChanges(clientId int32, grpcDialOption grpc.DialOption, } var lastLogTsNs = time.Now().Nanosecond() - var clientName = fmt.Sprintf("syncTo_%s_From_%s", string(targetFiler), string(sourceFiler)) + var clientName = fmt.Sprintf("syncFrom_%s_To_%s", string(sourceFiler), string(targetFiler)) processEventFnWithOffset := pb.AddOffsetFunc(processEventFn, 3*time.Second, func(counter int64, lastTsNs int64) error { now := time.Now().Nanosecond() glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, lastTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9)) From 695e8c75551b7de9f47723b682f0cb0ab4b74002 Mon Sep 17 00:00:00 2001 From: guol-fnst Date: Wed, 15 Jun 2022 15:26:04 +0800 Subject: [PATCH 21/38] fix minor bug --- weed/command/update.go | 44 +++++++++++++++--------------------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/weed/command/update.go b/weed/command/update.go index 64a664923..945ecc293 100644 --- a/weed/command/update.go +++ b/weed/command/update.go @@ -9,7 +9,6 @@ import ( "crypto/md5" "encoding/hex" "encoding/json" - "errors" "fmt" "io" "io/ioutil" @@ -46,6 +45,13 @@ type Asset struct { URL string `json:"url"` } +const githubAPITimeout = 30 * time.Second + +// githubError is returned by the GitHub API, e.g. for rate-limiting. +type githubError struct { + Message string +} + var ( updateOpt UpdateOptions ) @@ -66,17 +72,6 @@ var cmdUpdate = &Command{ } func runUpdate(cmd *Command, args []string) bool { - weedPath := *updateOpt.Output - if weedPath == "" { - file, err := os.Executable() - if err != nil { - glog.Fatalf("unable to find executable:%s", err) - return false - } - - *updateOpt.Output = file - } - fi, err := os.Lstat(*updateOpt.Output) if err != nil { dirname := filepath.Dir(*updateOpt.Output) @@ -91,7 +86,7 @@ func runUpdate(cmd *Command, args []string) bool { } } else { if !fi.Mode().IsRegular() { - glog.Fatalf("output path %v is not a normal file, use --output to specify a different file path", updateOpt.Output) + glog.Fatalf("output path %v is not a normal file, use --output to specify a different file path", *updateOpt.Output) return false } } @@ -111,11 +106,6 @@ func runUpdate(cmd *Command, args []string) bool { func downloadLatestStableRelease(ctx context.Context, target string) (version string, err error) { currentVersion := util.VERSION_NUMBER - largeDiskSuffix := "" - if util.VolumeSizeLimitGB == 8000 { - largeDiskSuffix = "_large_disk" - } - rel, err := GitHubLatestRelease(ctx, "chrislusf", "seaweedfs") if err != nil { return "", err @@ -128,6 +118,11 @@ func downloadLatestStableRelease(ctx context.Context, target string) (version st glog.V(0).Infof("latest version is %v\n", rel.Version) + largeDiskSuffix := "" + if util.VolumeSizeLimitGB == 8000 { + largeDiskSuffix = "_large_disk" + } + ext := "tar.gz" if runtime.GOOS == "windows" { ext = "zip" @@ -150,7 +145,7 @@ func downloadLatestStableRelease(ctx context.Context, target string) (version st binaryMd5 := md5Ctx.Sum(nil) if hex.EncodeToString(binaryMd5) != string(md5Val[0:32]) { glog.Errorf("md5:'%s' '%s'", hex.EncodeToString(binaryMd5), string(md5Val[0:32])) - err = errors.New("binary md5sum doesn't match") + err = fmt.Errorf("binary md5sum doesn't match") return "", err } @@ -169,13 +164,6 @@ func (r Release) String() string { len(r.Assets)) } -const githubAPITimeout = 30 * time.Second - -// githubError is returned by the GitHub API, e.g. for rate-limiting. -type githubError struct { - Message string -} - // GitHubLatestRelease uses the GitHub API to get information about the latest // release of a repository. func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, error) { @@ -229,7 +217,7 @@ func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, erro } if release.TagName == "" { - return Release{}, errors.New("tag name for latest release is empty") + return Release{}, fmt.Errorf("tag name for latest release is empty") } release.Version = release.TagName @@ -316,7 +304,7 @@ func extractToFile(buf []byte, filename, target string) error { } if len(zrd.File) != 1 { - return errors.New("ZIP archive contains more than one file") + return fmt.Errorf("ZIP archive contains more than one file") } file, err := zrd.File[0].Open() From 358c0c0b37a2b6ce7eb136dc55dabef61ef4528d Mon Sep 17 00:00:00 2001 From: "yulai.li" Date: Wed, 15 Jun 2022 15:53:31 +0800 Subject: [PATCH 22/38] Fix: provide filer.ui.deleteDir options for server command --- weed/command/server.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/weed/command/server.go b/weed/command/server.go index 4b6b6c642..ba71a44bd 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -2,8 +2,6 @@ package command import ( "fmt" - "github.com/chrislusf/seaweedfs/weed/pb" - "github.com/chrislusf/seaweedfs/weed/util/grace" "net/http" "os" "strings" @@ -12,7 +10,9 @@ import ( stats_collect "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/util/grace" ) type ServerOptions struct { @@ -114,6 +114,7 @@ func init() { filerOptions.saveToFilerLimit = cmdServer.Flag.Int("filer.saveToFilerLimit", 0, "Small files smaller than this limit can be cached in filer store.") filerOptions.concurrentUploadLimitMB = cmdServer.Flag.Int("filer.concurrentUploadLimitMB", 64, "limit total concurrent upload size") filerOptions.localSocket = cmdServer.Flag.String("filer.localSocket", "", "default to /tmp/seaweedfs-filer-.sock") + filerOptions.showUIDirectoryDelete = cmdServer.Flag.Bool("filer.ui.deleteDir", true, "enable filer UI show delete directory button") serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port") From 31a929f7f206b21cd2d62ad4d897820ebbaad50c Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Wed, 15 Jun 2022 15:36:43 +0500 Subject: [PATCH 23/38] logging processRangeRequest errors --- weed/server/common.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/weed/server/common.go b/weed/server/common.go index 39a8637ac..f02ec67ac 100644 --- a/weed/server/common.go +++ b/weed/server/common.go @@ -284,6 +284,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64 if rangeReq == "" { w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) if err := writeFn(bufferedWriter, 0, totalSize); err != nil { + glog.Errorf("processRangeRequest headers: %+v err: %v", w.Header(), err) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -294,6 +295,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64 //mostly copy from src/pkg/net/http/fs.go ranges, err := parseRange(rangeReq, totalSize) if err != nil { + glog.Errorf("processRangeRequest headers: %+v err: %v", w.Header(), err) http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable) return } @@ -326,6 +328,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64 w.WriteHeader(http.StatusPartialContent) err = writeFn(bufferedWriter, ra.start, ra.length) if err != nil { + glog.Errorf("processRangeRequest headers: %+v err: %v", w.Header(), err) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -365,6 +368,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64 } w.WriteHeader(http.StatusPartialContent) if _, err := io.CopyN(bufferedWriter, sendContent, sendSize); err != nil { + glog.Errorf("processRangeRequest err: %v", err) http.Error(w, "Internal Error", http.StatusInternalServerError) return } From 338705f375a37c0bb49aed2cea4d3f89d98ef70e Mon Sep 17 00:00:00 2001 From: ningfd Date: Wed, 15 Jun 2022 19:20:13 +0800 Subject: [PATCH 24/38] fix(wdclient): GetLocations return --- weed/wdclient/vid_map.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/wdclient/vid_map.go b/weed/wdclient/vid_map.go index cdd783d91..f7a9a0f1a 100644 --- a/weed/wdclient/vid_map.go +++ b/weed/wdclient/vid_map.go @@ -133,7 +133,7 @@ func (vc *vidMap) GetLocations(vid uint32) (locations []Location, found bool) { return } locations, found = vc.ecVid2Locations[vid] - return + return locations, found && len(locations) > 0 } func (vc *vidMap) addLocation(vid uint32, location Location) { From 3076ac101ead13d58e7b6c065af65437e3aff9cd Mon Sep 17 00:00:00 2001 From: liubaojiang <1838095916@qq.com> Date: Thu, 16 Jun 2022 09:58:44 +0800 Subject: [PATCH 25/38] move vs.concurrentUploadLimit != 0 out of the lock --- weed/server/volume_server_handlers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index 7e4c11fed..ef37b48f5 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -59,10 +59,10 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque contentLength := getContentLength(r) // exclude the replication from the concurrentUploadLimitMB - if r.URL.Query().Get("type") != "replicate" { + if r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 { startTime := time.Now() vs.inFlightUploadDataLimitCond.L.Lock() - for vs.concurrentUploadLimit != 0 && vs.inFlightUploadDataSize > vs.concurrentUploadLimit { + for vs.inFlightUploadDataSize > vs.concurrentUploadLimit { //wait timeout check if startTime.Add(vs.inflightUploadDataTimeout).Before(time.Now()) { vs.inFlightUploadDataLimitCond.L.Unlock() From 1dbb925d298572afb09257ba3555374032799580 Mon Sep 17 00:00:00 2001 From: guol-fnst Date: Wed, 15 Jun 2022 15:51:37 +0800 Subject: [PATCH 26/38] remove unused function remove log when weed is up to date add check whether weed is full version --- weed/command/update.go | 26 ++++++++++++-------------- weed/command/update_full.go | 9 +++++++++ 2 files changed, 21 insertions(+), 14 deletions(-) create mode 100644 weed/command/update_full.go diff --git a/weed/command/update.go b/weed/command/update.go index 945ecc293..3f575d513 100644 --- a/weed/command/update.go +++ b/weed/command/update.go @@ -52,6 +52,9 @@ type githubError struct { Message string } +//default version is not full version +var isFullVersion = false + var ( updateOpt UpdateOptions ) @@ -91,16 +94,11 @@ func runUpdate(cmd *Command, args []string) bool { } } - glog.V(0).Infof("writing weed to %v\n", *updateOpt.Output) - v, err := downloadLatestStableRelease(context.Background(), *updateOpt.Output) if err != nil { - glog.Fatalf("unable to update weed: %v", err) + glog.Fatalf("unable to update weed to version %s: %v", err, v) return false } - - glog.V(0).Infof("successfully updated weed to version %v\n", v) - return true } @@ -123,12 +121,17 @@ func downloadLatestStableRelease(ctx context.Context, target string) (version st largeDiskSuffix = "_large_disk" } + fullSuffix := "" + if isFullVersion { + fullSuffix = "_full" + } + ext := "tar.gz" if runtime.GOOS == "windows" { ext = "zip" } - suffix := fmt.Sprintf("%s_%s%s.%s", runtime.GOOS, runtime.GOARCH, largeDiskSuffix, ext) + suffix := fmt.Sprintf("%s_%s%s%s.%s", runtime.GOOS, runtime.GOARCH, fullSuffix, largeDiskSuffix, ext) md5Filename := fmt.Sprintf("%s.md5", suffix) _, md5Val, err := getGithubDataFile(ctx, rel.Assets, md5Filename) if err != nil { @@ -152,18 +155,13 @@ func downloadLatestStableRelease(ctx context.Context, target string) (version st err = extractToFile(buf, downloadFilename, target) if err != nil { return "", err + } else { + glog.V(0).Infof("successfully updated weed to version %v\n", rel.Version) } return rel.Version, nil } -func (r Release) String() string { - return fmt.Sprintf("%v %v, %d assets", - r.TagName, - r.PublishedAt.Local().Format("2006-01-02 15:04:05"), - len(r.Assets)) -} - // GitHubLatestRelease uses the GitHub API to get information about the latest // release of a repository. func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, error) { diff --git a/weed/command/update_full.go b/weed/command/update_full.go new file mode 100644 index 000000000..529f38219 --- /dev/null +++ b/weed/command/update_full.go @@ -0,0 +1,9 @@ +//go:build elastic && ydb && gocdk && hdfs +// +build elastic,ydb,gocdk,hdfs + +package command + +//set true if gtags are set +func init() { + isFullVersion = true +} From 4bc879264335ed0f96fc9604d261efe37554e470 Mon Sep 17 00:00:00 2001 From: guol-fnst Date: Thu, 16 Jun 2022 11:19:49 +0800 Subject: [PATCH 27/38] support specific version of weed --- weed/command/update.go | 55 +++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/weed/command/update.go b/weed/command/update.go index 3f575d513..2f25cbe32 100644 --- a/weed/command/update.go +++ b/weed/command/update.go @@ -60,18 +60,20 @@ var ( ) type UpdateOptions struct { - Output *string + Output *string + Version *string } func init() { - updateOpt.Output = cmdUpdate.Flag.String("output", "weed", "Save the latest weed as `filename`") + updateOpt.Output = cmdUpdate.Flag.String("output", "weed", "Save the weed as `filename` or /path/to/dir/filename.") + updateOpt.Version = cmdUpdate.Flag.String("version", "0", "The version of weed you want to download. If not specified, get the latest version.") cmdUpdate.Run = runUpdate } var cmdUpdate = &Command{ UsageLine: "update [-output=weed]", - Short: "get latest version from https://github.com/chrislusf/seaweedfs", - Long: `get latest version from https://github.com/chrislusf/seaweedfs`, + Short: "get latest or specific version from https://github.com/chrislusf/seaweedfs", + Long: `get latest or specific version from https://github.com/chrislusf/seaweedfs`, } func runUpdate(cmd *Command, args []string) bool { @@ -80,41 +82,45 @@ func runUpdate(cmd *Command, args []string) bool { dirname := filepath.Dir(*updateOpt.Output) di, err := os.Lstat(dirname) if err != nil { - glog.Fatalf("unable to find directory:%s", dirname) + glog.Errorf("unable to find directory:%s", dirname) return false } if !di.Mode().IsDir() { - glog.Fatalf("output parent path %v is not a directory, use --output to specify a different file path", dirname) + glog.Errorf("output parent path %v is not a directory, use --output to specify a different file path", dirname) return false } } else { if !fi.Mode().IsRegular() { - glog.Fatalf("output path %v is not a normal file, use --output to specify a different file path", *updateOpt.Output) + glog.Errorf("output path %v is not a normal file, use --output to specify a different file path", *updateOpt.Output) return false } } - v, err := downloadLatestStableRelease(context.Background(), *updateOpt.Output) + _, err = downloadRelease(context.Background(), *updateOpt.Output, *updateOpt.Version) if err != nil { - glog.Fatalf("unable to update weed to version %s: %v", err, v) + glog.Errorf("unable to download weed: %v", err) return false } return true } -func downloadLatestStableRelease(ctx context.Context, target string) (version string, err error) { +func downloadRelease(ctx context.Context, target string, ver string) (version string, err error) { currentVersion := util.VERSION_NUMBER - rel, err := GitHubLatestRelease(ctx, "chrislusf", "seaweedfs") + rel, err := GitHubLatestRelease(ctx, ver, "chrislusf", "seaweedfs") if err != nil { return "", err } if rel.Version == currentVersion { - glog.V(0).Infof("weed is up to date\n") + if ver == "0" { + glog.V(0).Infof("weed is up to date") + } else { + glog.V(0).Infof("no need to download the same version of weed ") + } return currentVersion, nil } - glog.V(0).Infof("latest version is %v\n", rel.Version) + glog.V(0).Infof("download version: %s", rel.Version) largeDiskSuffix := "" if util.VolumeSizeLimitGB == 8000 { @@ -162,13 +168,13 @@ func downloadLatestStableRelease(ctx context.Context, target string) (version st return rel.Version, nil } -// GitHubLatestRelease uses the GitHub API to get information about the latest +// GitHubLatestRelease uses the GitHub API to get information about the specific // release of a repository. -func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, error) { +func GitHubLatestRelease(ctx context.Context, ver string, owner, repo string) (Release, error) { ctx, cancel := context.WithTimeout(ctx, githubAPITimeout) defer cancel() - url := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/latest", owner, repo) + url := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases", owner, repo) req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return Release{}, err @@ -209,17 +215,28 @@ func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, erro } var release Release - err = json.Unmarshal(buf, &release) + var releaseList []Release + err = json.Unmarshal(buf, &releaseList) if err != nil { return Release{}, err } + if ver == "0" { + release = releaseList[0] + glog.V(0).Infof("latest version is %v\n", release.TagName) + } else { + for _, r := range releaseList { + if r.TagName == ver { + release = r + break + } + } + } if release.TagName == "" { - return Release{}, fmt.Errorf("tag name for latest release is empty") + return Release{}, fmt.Errorf("can not find the specific version") } release.Version = release.TagName - return release, nil } From 62f74f5e3cce0668478c179f81fd1a9c5916e455 Mon Sep 17 00:00:00 2001 From: zzq09494 Date: Thu, 16 Jun 2022 13:21:25 +0800 Subject: [PATCH 28/38] add bucket label to s3 prometheus metrics --- weed/s3api/stats.go | 6 ++++-- weed/stats/metrics.go | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/weed/s3api/stats.go b/weed/s3api/stats.go index 973d8c0eb..003807a25 100644 --- a/weed/s3api/stats.go +++ b/weed/s3api/stats.go @@ -1,6 +1,7 @@ package s3api import ( + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" stats_collect "github.com/chrislusf/seaweedfs/weed/stats" "net/http" "strconv" @@ -27,11 +28,12 @@ func (r *StatusRecorder) Flush() { func track(f http.HandlerFunc, action string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + bucket, _ := s3_constants.GetBucketAndObject(r) w.Header().Set("Server", "SeaweedFS S3") recorder := NewStatusResponseWriter(w) start := time.Now() f(recorder, r) - stats_collect.S3RequestHistogram.WithLabelValues(action).Observe(time.Since(start).Seconds()) - stats_collect.S3RequestCounter.WithLabelValues(action, strconv.Itoa(recorder.Status)).Inc() + stats_collect.S3RequestHistogram.WithLabelValues(action, bucket).Observe(time.Since(start).Seconds()) + stats_collect.S3RequestCounter.WithLabelValues(action, strconv.Itoa(recorder.Status), bucket).Inc() } } diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index 207b37c81..f0b810608 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -173,7 +173,8 @@ var ( Subsystem: "s3", Name: "request_total", Help: "Counter of s3 requests.", - }, []string{"type", "code"}) + }, []string{"type", "code", "bucket"}) + S3RequestHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "SeaweedFS", @@ -181,7 +182,7 @@ var ( Name: "request_seconds", Help: "Bucketed histogram of s3 request processing time.", Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24), - }, []string{"type"}) + }, []string{"type", "bucket"}) ) func init() { From 0a613876caa22542cac3bd99c19f44e64846a52f Mon Sep 17 00:00:00 2001 From: zzq09494 Date: Thu, 16 Jun 2022 13:50:16 +0800 Subject: [PATCH 29/38] add bucket label to the grafana dashboard --- other/metrics/grafana_seaweedfs.json | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/other/metrics/grafana_seaweedfs.json b/other/metrics/grafana_seaweedfs.json index 3b9b222b4..d10fb7c51 100644 --- a/other/metrics/grafana_seaweedfs.json +++ b/other/metrics/grafana_seaweedfs.json @@ -539,11 +539,12 @@ "step": 60 }, { - "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))", + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))", "format": "time_series", "hide": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "{{bucket}} {{type}}", "refId": "B", "step": 60 } @@ -645,11 +646,12 @@ "step": 60 }, { - "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))", + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type,bucket))", "format": "time_series", "hide": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "{{bucket}} {{type}}", "refId": "B", "step": 60 } @@ -751,11 +753,11 @@ "step": 60 }, { - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))", "format": "time_series", "hide": false, "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "{{bucket}} {{type}}", "refId": "B", "step": 60 } @@ -864,7 +866,7 @@ "expr": "rate(SeaweedFS_s3_request_total[1m])", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "{{bucket}} {{type}}", "refId": "A", "step": 30 } From 6449114e5e88953b0fe22b50b46d91a7bed52523 Mon Sep 17 00:00:00 2001 From: zzq09494 Date: Thu, 16 Jun 2022 13:52:36 +0800 Subject: [PATCH 30/38] format --- other/metrics/grafana_seaweedfs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/other/metrics/grafana_seaweedfs.json b/other/metrics/grafana_seaweedfs.json index d10fb7c51..88844b3c3 100644 --- a/other/metrics/grafana_seaweedfs.json +++ b/other/metrics/grafana_seaweedfs.json @@ -646,7 +646,7 @@ "step": 60 }, { - "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type,bucket))", + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))", "format": "time_series", "hide": false, "interval": "", From df0ce31a2ee87bf4550da20cf7c1095d154b24e3 Mon Sep 17 00:00:00 2001 From: liubaojiang <1838095916@qq.com> Date: Thu, 16 Jun 2022 14:07:11 +0800 Subject: [PATCH 31/38] add condition when inFlightUploadDataLimitCond signal --- weed/server/volume_server_handlers.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index ef37b48f5..293f36f14 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -79,7 +79,9 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength) defer func() { atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength) - vs.inFlightUploadDataLimitCond.Signal() + if vs.concurrentUploadLimit != 0 { + vs.inFlightUploadDataLimitCond.Signal() + } }() // processs uploads From e024586ff11d025ce2dd27822a2b04e01260fb6c Mon Sep 17 00:00:00 2001 From: chrislu Date: Wed, 15 Jun 2022 23:19:39 -0700 Subject: [PATCH 32/38] adjust conditions --- weed/server/volume_server_handlers.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index 293f36f14..f219dc9b9 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -59,7 +59,8 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque contentLength := getContentLength(r) // exclude the replication from the concurrentUploadLimitMB - if r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 { + shouldWatchUploadLimit := r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 + if shouldWatchUploadLimit { startTime := time.Now() vs.inFlightUploadDataLimitCond.L.Lock() for vs.inFlightUploadDataSize > vs.concurrentUploadLimit { @@ -79,7 +80,7 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength) defer func() { atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength) - if vs.concurrentUploadLimit != 0 { + if shouldWatchUploadLimit { vs.inFlightUploadDataLimitCond.Signal() } }() From 995e221e97d2cd9982d07c699afec4815d45e557 Mon Sep 17 00:00:00 2001 From: guol-fnst Date: Thu, 16 Jun 2022 15:56:34 +0800 Subject: [PATCH 33/38] remove -output, add -dir,-name for more flexibility --- weed/command/update.go | 64 ++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 37 deletions(-) diff --git a/weed/command/update.go b/weed/command/update.go index 2f25cbe32..df5988b6e 100644 --- a/weed/command/update.go +++ b/weed/command/update.go @@ -60,13 +60,17 @@ var ( ) type UpdateOptions struct { - Output *string + dir *string + name *string Version *string } func init() { - updateOpt.Output = cmdUpdate.Flag.String("output", "weed", "Save the weed as `filename` or /path/to/dir/filename.") - updateOpt.Version = cmdUpdate.Flag.String("version", "0", "The version of weed you want to download. If not specified, get the latest version.") + path, _ := os.Executable() + _, name := filepath.Split(path) + updateOpt.dir = cmdUpdate.Flag.String("dir", filepath.Dir(path), "directory to save new weed.") + updateOpt.name = cmdUpdate.Flag.String("name", name, "name of new weed.On windows, name shouldn't be same to the orignial name.") + updateOpt.Version = cmdUpdate.Flag.String("version", "0", "specific version of weed you want to download. If not specified, get the latest version.") cmdUpdate.Run = runUpdate } @@ -77,26 +81,33 @@ var cmdUpdate = &Command{ } func runUpdate(cmd *Command, args []string) bool { - fi, err := os.Lstat(*updateOpt.Output) - if err != nil { - dirname := filepath.Dir(*updateOpt.Output) - di, err := os.Lstat(dirname) - if err != nil { - glog.Errorf("unable to find directory:%s", dirname) - return false - } - if !di.Mode().IsDir() { - glog.Errorf("output parent path %v is not a directory, use --output to specify a different file path", dirname) + path, _ := os.Executable() + _, name := filepath.Split(path) + + if *updateOpt.dir != "" { + if err := util.TestFolderWritable(util.ResolvePath(*updateOpt.dir)); err != nil { + glog.Fatalf("Check Folder(-dir) Writable %s : %s", *updateOpt.dir, err) return false } } else { - if !fi.Mode().IsRegular() { - glog.Errorf("output path %v is not a normal file, use --output to specify a different file path", *updateOpt.Output) + *updateOpt.dir = filepath.Dir(path) + } + + if *updateOpt.name == "" { + *updateOpt.name = name + } + + if runtime.GOOS == "windows" { + if *updateOpt.name == name || *updateOpt.name == "" { + glog.Fatalf("On windows, name of the new weed shouldn't be same to the orignial name.") return false } } - _, err = downloadRelease(context.Background(), *updateOpt.Output, *updateOpt.Version) + target := *updateOpt.dir + "/" + *updateOpt.name + glog.V(0).Infof("new weed will be saved to %s", target) + + _, err := downloadRelease(context.Background(), target, *updateOpt.Version) if err != nil { glog.Errorf("unable to download weed: %v", err) return false @@ -360,11 +371,6 @@ func extractToFile(buf []byte, filename, target string) error { mode = fi.Mode() } - // Remove the original binary. - if err := removeWeedBinary(dir, target); err != nil { - return err - } - // Rename the temp file to the final location atomically. if err := os.Rename(new.Name(), target); err != nil { return err @@ -373,19 +379,3 @@ func extractToFile(buf []byte, filename, target string) error { glog.V(0).Infof("saved %d bytes in %v\n", n, target) return os.Chmod(target, mode) } - -// Rename (rather than remove) the running version. The running binary will be locked -// on Windows and cannot be removed while still executing. -func removeWeedBinary(dir, target string) error { - if runtime.GOOS == "linux" { - return nil - } - backup := filepath.Join(dir, filepath.Base(target)+".bak") - if _, err := os.Stat(backup); err == nil { - _ = os.Remove(backup) - } - if err := os.Rename(target, backup); err != nil { - return fmt.Errorf("unable to rename target file: %v", err) - } - return nil -} From aaa9938353a02ca38cfe4a309b857c1ff8f738ca Mon Sep 17 00:00:00 2001 From: chrislu Date: Thu, 16 Jun 2022 01:26:36 -0700 Subject: [PATCH 34/38] Revert "adjust conditions" This reverts commit e024586ff11d025ce2dd27822a2b04e01260fb6c. --- weed/server/volume_server_handlers.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index f219dc9b9..293f36f14 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -59,8 +59,7 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque contentLength := getContentLength(r) // exclude the replication from the concurrentUploadLimitMB - shouldWatchUploadLimit := r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 - if shouldWatchUploadLimit { + if r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 { startTime := time.Now() vs.inFlightUploadDataLimitCond.L.Lock() for vs.inFlightUploadDataSize > vs.concurrentUploadLimit { @@ -80,7 +79,7 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength) defer func() { atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength) - if shouldWatchUploadLimit { + if vs.concurrentUploadLimit != 0 { vs.inFlightUploadDataLimitCond.Signal() } }() From cd078471c4a3f588d452d1ae92b7489a5186957b Mon Sep 17 00:00:00 2001 From: guol-fnst Date: Thu, 16 Jun 2022 16:52:57 +0800 Subject: [PATCH 35/38] fix path bug on windows fix usage --- weed/command/update.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/weed/command/update.go b/weed/command/update.go index df5988b6e..8e0a76016 100644 --- a/weed/command/update.go +++ b/weed/command/update.go @@ -69,13 +69,13 @@ func init() { path, _ := os.Executable() _, name := filepath.Split(path) updateOpt.dir = cmdUpdate.Flag.String("dir", filepath.Dir(path), "directory to save new weed.") - updateOpt.name = cmdUpdate.Flag.String("name", name, "name of new weed.On windows, name shouldn't be same to the orignial name.") + updateOpt.name = cmdUpdate.Flag.String("name", name, "name of new weed. On windows, name shouldn't be same to the orignial name.") updateOpt.Version = cmdUpdate.Flag.String("version", "0", "specific version of weed you want to download. If not specified, get the latest version.") cmdUpdate.Run = runUpdate } var cmdUpdate = &Command{ - UsageLine: "update [-output=weed]", + UsageLine: "weed update -dir=/path/to/dir -name=name -version=x.xx", Short: "get latest or specific version from https://github.com/chrislusf/seaweedfs", Long: `get latest or specific version from https://github.com/chrislusf/seaweedfs`, } @@ -97,14 +97,15 @@ func runUpdate(cmd *Command, args []string) bool { *updateOpt.name = name } + target := filepath.Join(*updateOpt.dir, *updateOpt.name) + if runtime.GOOS == "windows" { - if *updateOpt.name == name || *updateOpt.name == "" { + if target == path { glog.Fatalf("On windows, name of the new weed shouldn't be same to the orignial name.") return false } } - target := *updateOpt.dir + "/" + *updateOpt.name glog.V(0).Infof("new weed will be saved to %s", target) _, err := downloadRelease(context.Background(), target, *updateOpt.Version) From 625fd16a2e5eb676e8c9eaa0268f78246c69b1ef Mon Sep 17 00:00:00 2001 From: chrislu Date: Thu, 16 Jun 2022 15:39:29 -0700 Subject: [PATCH 36/38] reduce upfront memory usage for low density volume --- weed/storage/needle_map/compact_map.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/storage/needle_map/compact_map.go b/weed/storage/needle_map/compact_map.go index 3d2047f99..ccce8f108 100644 --- a/weed/storage/needle_map/compact_map.go +++ b/weed/storage/needle_map/compact_map.go @@ -8,7 +8,7 @@ import ( ) const ( - batch = 100000 + batch = 10000 ) type SectionalNeedleId uint32 From c7c20881f5543e9ab2f413d32ad2943c3d1c4828 Mon Sep 17 00:00:00 2001 From: guol-fnst Date: Fri, 17 Jun 2022 12:40:54 +0800 Subject: [PATCH 37/38] remove "weed" from UsageLine, or weed will not show usage of weed update --- weed/command/update.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/command/update.go b/weed/command/update.go index 8e0a76016..2d0dc42ad 100644 --- a/weed/command/update.go +++ b/weed/command/update.go @@ -75,7 +75,7 @@ func init() { } var cmdUpdate = &Command{ - UsageLine: "weed update -dir=/path/to/dir -name=name -version=x.xx", + UsageLine: "update [-dir=/path/to/dir] [-name=name] [-version=x.xx]", Short: "get latest or specific version from https://github.com/chrislusf/seaweedfs", Long: `get latest or specific version from https://github.com/chrislusf/seaweedfs`, } From fdacd94af57a0b9040a396154e7c322bc56dd16e Mon Sep 17 00:00:00 2001 From: geekboood Date: Sun, 19 Jun 2022 16:54:04 +0800 Subject: [PATCH 38/38] fix: invalid chunk data when failed to read manifests --- weed/filer/filechunk_manifest.go | 7 ++++--- weed/filer/filechunks.go | 8 ++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/weed/filer/filechunk_manifest.go b/weed/filer/filechunk_manifest.go index 091bbee5a..4eb657dfa 100644 --- a/weed/filer/filechunk_manifest.go +++ b/weed/filer/filechunk_manifest.go @@ -3,7 +3,6 @@ package filer import ( "bytes" "fmt" - "github.com/chrislusf/seaweedfs/weed/wdclient" "io" "math" "net/url" @@ -11,6 +10,8 @@ import ( "sync" "time" + "github.com/chrislusf/seaweedfs/weed/wdclient" + "github.com/golang/protobuf/proto" "github.com/chrislusf/seaweedfs/weed/glog" @@ -63,14 +64,14 @@ func ResolveChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chun resolvedChunks, err := ResolveOneChunkManifest(lookupFileIdFn, chunk) if err != nil { - return chunks, nil, err + return dataChunks, nil, err } manifestChunks = append(manifestChunks, chunk) // recursive subDataChunks, subManifestChunks, subErr := ResolveChunkManifest(lookupFileIdFn, resolvedChunks, startOffset, stopOffset) if subErr != nil { - return chunks, nil, subErr + return dataChunks, nil, subErr } dataChunks = append(dataChunks, subDataChunks...) manifestChunks = append(manifestChunks, subManifestChunks...) diff --git a/weed/filer/filechunks.go b/weed/filer/filechunks.go index 208ef8095..48b344bf8 100644 --- a/weed/filer/filechunks.go +++ b/weed/filer/filechunks.go @@ -3,11 +3,12 @@ package filer import ( "bytes" "fmt" - "github.com/chrislusf/seaweedfs/weed/wdclient" - "golang.org/x/exp/slices" "math" "sync" + "github.com/chrislusf/seaweedfs/weed/wdclient" + "golang.org/x/exp/slices" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -248,6 +249,9 @@ func MergeIntoVisibles(visibles []VisibleInterval, chunk *filer_pb.FileChunk) (n func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles []VisibleInterval, err error) { chunks, _, err = ResolveChunkManifest(lookupFileIdFn, chunks, startOffset, stopOffset) + if err != nil { + return + } visibles2 := readResolvedChunks(chunks)