From a37f0692961e66d8f02db164a6a19c04c4cdbd46 Mon Sep 17 00:00:00 2001 From: LazyDBA247-Anyvision Date: Thu, 17 Dec 2020 19:41:33 +0200 Subject: [PATCH 1/3] fix Argument Name, and use the correct one. --- docker/Dockerfile.go_build | 4 ++-- docker/Dockerfile.go_build_large | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile.go_build b/docker/Dockerfile.go_build index a5f8725e0..726046b56 100644 --- a/docker/Dockerfile.go_build +++ b/docker/Dockerfile.go_build @@ -2,8 +2,8 @@ FROM frolvlad/alpine-glibc as builder RUN apk add git go g++ RUN mkdir -p /go/src/github.com/chrislusf/ RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs -ARG branch=${branch:-master} -RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $ARG +ARG BRANCH=${BRANCH:-master} +RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \ && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \ && go install -ldflags "${LDFLAGS}" diff --git a/docker/Dockerfile.go_build_large b/docker/Dockerfile.go_build_large index c3ea4e606..8fc85e868 100644 --- a/docker/Dockerfile.go_build_large +++ b/docker/Dockerfile.go_build_large @@ -2,8 +2,8 @@ FROM frolvlad/alpine-glibc as builder RUN apk add git go g++ RUN mkdir -p /go/src/github.com/chrislusf/ RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs -ARG branch=${branch:-master} -RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $ARG +ARG BRANCH=${BRANCH:-master} +RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \ && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \ && go install -tags 5BytesOffset -ldflags "${LDFLAGS}" From 986cbdf7d95bbb0839ab77bb7e42e4b9baad0a69 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 17 Dec 2020 12:46:42 -0800 Subject: [PATCH 2/3] Revert "Merge pull request #1683 from qieqieplus/master" This reverts commit 8cb67952db822ef232dd2701ce78a5486d54b4ea, reversing changes made to 200e56215a4fdc4ec424f05fb975cdc876d7a2b4. --- weed/storage/disk_location.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index 359d0cec2..2d4d120af 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -99,14 +99,12 @@ func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind Ne } // avoid loading one volume more than once - l.volumesLock.Lock() - if _, found := l.volumes[vid]; found { - l.volumesLock.Unlock() + l.volumesLock.RLock() + _, found := l.volumes[vid] + l.volumesLock.RUnlock() + if found { glog.V(1).Infof("loaded volume, %v", vid) return true - } else { - l.volumes[vid] = nil - l.volumesLock.Unlock() } // load the volume @@ -115,7 +113,7 @@ func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind Ne glog.V(0).Infof("new volume %s error %s", volumeName, e) return false } - + l.SetVolume(vid, v) size, _, _ := v.FileStat() From e2076201d79e84df0407a9e1e535bf966452bba0 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 17 Dec 2020 13:03:39 -0800 Subject: [PATCH 3/3] volume: avoid reprocessing the same volume fix https://github.com/chrislusf/seaweedfs/issues/1682 --- weed/storage/disk_location.go | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index 2d4d120af..9b2ab69fe 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -53,7 +53,7 @@ func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpacePercent float32 } func volumeIdFromFileName(filename string) (needle.VolumeId, string, error) { - if strings.HasSuffix(filename, ".idx") || strings.HasSuffix(filename, ".vif") { + if isValidVolume(filename) { base := filename[:len(filename)-4] collection, volumeId, err := parseCollectionVolumeId(base) return volumeId, collection, err @@ -71,15 +71,26 @@ func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeI return collection, vol, err } +func isValidVolume(basename string) bool { + return strings.HasSuffix(basename, ".idx") || strings.HasSuffix(basename, ".vif") +} + +func getValidVolumeName(basename string) string { + if isValidVolume(basename) { + return basename[:len(basename)-4] + } + return "" +} + func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapType) bool { basename := fileInfo.Name() if fileInfo.IsDir() { return false } - if !strings.HasSuffix(basename, ".idx") && !strings.HasSuffix(basename, ".vif") { + volumeName := getValidVolumeName(basename) + if volumeName == "" { return false } - volumeName := basename[:len(basename)-4] // check for incomplete volume noteFile := l.Directory + "/" + volumeName + ".note" @@ -126,9 +137,17 @@ func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, con task_queue := make(chan os.FileInfo, 10*concurrency) go func() { + foundVolumeNames := make(map[string]bool) if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil { for _, fi := range fileInfos { - task_queue <- fi + volumeName := getValidVolumeName(fi.Name()) + if volumeName == "" { + continue + } + if _, found := foundVolumeNames[volumeName]; !found { + foundVolumeNames[volumeName] = true + task_queue <- fi + } } } close(task_queue)