mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/chrislusf/seaweedfs synced 2024-06-30 22:31:06 +02:00
Code Issues Projects Releases Wiki Activity

Fix race conditions during in-flight size checks (#3505)

Browse source
This commit is contained in:
Patrick Schmidt 2022-08-25 05:03:34 +02:00 committed by GitHub
parent 7b424a54dc
commit 2930263dfd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 22 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
weed/filer/reader_cache.go
View file

@ -186,8 +186,8 @@ func (s *SingleChunkCacher) destroy() {
if s.data != nil { if s.data != nil {
mem.Free(s.data) mem.Free(s.data)
s.data = nil s.data = nil
close(s.cacheStartedCh)
} }
close(s.cacheStartedCh)
} }
func (s *SingleChunkCacher) readChunkAt(buf []byte, offset int64) (int, error) { func (s *SingleChunkCacher) readChunkAt(buf []byte, offset int64) (int, error) {

13
weed/server/filer_server_handlers.go
View file

@ -2,14 +2,15 @@ package weed_server
import ( import (
"errors" "errors"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"net/http" "net/http"
"strings" "strings"
"sync/atomic" "sync/atomic"
"time" "time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/stats"
) )
@ -63,9 +64,11 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
// wait until in flight data is less than the limit // wait until in flight data is less than the limit
contentLength := getContentLength(r) contentLength := getContentLength(r)
fs.inFlightDataLimitCond.L.Lock() fs.inFlightDataLimitCond.L.Lock()
for fs.option.ConcurrentUploadLimit != 0 && atomic.LoadInt64(&fs.inFlightDataSize) > fs.option.ConcurrentUploadLimit { inFlightDataSize := atomic.LoadInt64(&fs.inFlightDataSize)
glog.V(4).Infof("wait because inflight data %d > %d", fs.inFlightDataSize, fs.option.ConcurrentUploadLimit) for fs.option.ConcurrentUploadLimit != 0 && inFlightDataSize > fs.option.ConcurrentUploadLimit {
glog.V(4).Infof("wait because inflight data %d > %d", inFlightDataSize, fs.option.ConcurrentUploadLimit)
fs.inFlightDataLimitCond.Wait() fs.inFlightDataLimitCond.Wait()
inFlightDataSize = atomic.LoadInt64(&fs.inFlightDataSize)
} }
fs.inFlightDataLimitCond.L.Unlock() fs.inFlightDataLimitCond.L.Unlock()
atomic.AddInt64(&fs.inFlightDataSize, contentLength) atomic.AddInt64(&fs.inFlightDataSize, contentLength)

20
weed/server/volume_server_handlers.go
View file

@ -45,15 +45,17 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
case "GET", "HEAD": case "GET", "HEAD":
stats.ReadRequest() stats.ReadRequest()
vs.inFlightDownloadDataLimitCond.L.Lock() vs.inFlightDownloadDataLimitCond.L.Lock()
for vs.concurrentDownloadLimit != 0 && atomic.LoadInt64(&vs.inFlightDownloadDataSize) > vs.concurrentDownloadLimit { inFlightDownloadSize := atomic.LoadInt64(&vs.inFlightDownloadDataSize)
for vs.concurrentDownloadLimit != 0 && inFlightDownloadSize > vs.concurrentDownloadLimit {
select { select {
case <-r.Context().Done(): case <-r.Context().Done():
glog.V(4).Infof("request cancelled from %s: %v", r.RemoteAddr, r.Context().Err()) glog.V(4).Infof("request cancelled from %s: %v", r.RemoteAddr, r.Context().Err())
return return
default: default:
glog.V(4).Infof("wait because inflight download data %d > %d", vs.inFlightDownloadDataSize, vs.concurrentDownloadLimit) glog.V(4).Infof("wait because inflight download data %d > %d", inFlightDownloadSize, vs.concurrentDownloadLimit)
vs.inFlightDownloadDataLimitCond.Wait() vs.inFlightDownloadDataLimitCond.Wait()
} }
inFlightDownloadSize = atomic.LoadInt64(&vs.inFlightDownloadDataSize)
} }
vs.inFlightDownloadDataLimitCond.L.Unlock() vs.inFlightDownloadDataLimitCond.L.Unlock()
vs.GetOrHeadHandler(w, r) vs.GetOrHeadHandler(w, r)
@ -66,17 +68,19 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
if r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 { if r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 {
startTime := time.Now() startTime := time.Now()
vs.inFlightUploadDataLimitCond.L.Lock() vs.inFlightUploadDataLimitCond.L.Lock()
for vs.inFlightUploadDataSize > vs.concurrentUploadLimit { inFlightUploadDataSize := atomic.LoadInt64(&vs.inFlightUploadDataSize)
for inFlightUploadDataSize > vs.concurrentUploadLimit {
//wait timeout check //wait timeout check
if startTime.Add(vs.inflightUploadDataTimeout).Before(time.Now()) { if startTime.Add(vs.inflightUploadDataTimeout).Before(time.Now()) {
vs.inFlightUploadDataLimitCond.L.Unlock() vs.inFlightUploadDataLimitCond.L.Unlock()
err := fmt.Errorf("reject because inflight upload data %d > %d, and wait timeout", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) err := fmt.Errorf("reject because inflight upload data %d > %d, and wait timeout", inFlightUploadDataSize, vs.concurrentUploadLimit)
glog.V(1).Infof("too many requests: %v", err) glog.V(1).Infof("too many requests: %v", err)
writeJsonError(w, r, http.StatusTooManyRequests, err) writeJsonError(w, r, http.StatusTooManyRequests, err)
return return
} }
glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit) glog.V(4).Infof("wait because inflight upload data %d > %d", inFlightUploadDataSize, vs.concurrentUploadLimit)
vs.inFlightUploadDataLimitCond.Wait() vs.inFlightUploadDataLimitCond.Wait()
inFlightUploadDataSize = atomic.LoadInt64(&vs.inFlightUploadDataSize)
} }
vs.inFlightUploadDataLimitCond.L.Unlock() vs.inFlightUploadDataLimitCond.L.Unlock()
} }
@ -121,9 +125,11 @@ func (vs *VolumeServer) publicReadOnlyHandler(w http.ResponseWriter, r *http.Req
case "GET", "HEAD": case "GET", "HEAD":
stats.ReadRequest() stats.ReadRequest()
vs.inFlightDownloadDataLimitCond.L.Lock() vs.inFlightDownloadDataLimitCond.L.Lock()
for vs.concurrentDownloadLimit != 0 && atomic.LoadInt64(&vs.inFlightDownloadDataSize) > vs.concurrentDownloadLimit { inFlightDownloadSize := atomic.LoadInt64(&vs.inFlightDownloadDataSize)
glog.V(4).Infof("wait because inflight download data %d > %d", vs.inFlightDownloadDataSize, vs.concurrentDownloadLimit) for vs.concurrentDownloadLimit != 0 && inFlightDownloadSize > vs.concurrentDownloadLimit {
glog.V(4).Infof("wait because inflight download data %d > %d", inFlightDownloadSize, vs.concurrentDownloadLimit)
vs.inFlightDownloadDataLimitCond.Wait() vs.inFlightDownloadDataLimitCond.Wait()
inFlightDownloadSize = atomic.LoadInt64(&vs.inFlightDownloadDataSize)
} }
vs.inFlightDownloadDataLimitCond.L.Unlock() vs.inFlightDownloadDataLimitCond.L.Unlock()
vs.GetOrHeadHandler(w, r) vs.GetOrHeadHandler(w, r)