mirror of
https://github.com/chrislusf/seaweedfs
synced 2024-06-28 21:31:56 +02:00
mount: less channel waiting
This commit is contained in:
parent
c31b254248
commit
3bf0116de1
|
@ -2,11 +2,17 @@ package filesys
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
|
||||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
|
||||||
"io"
|
"io"
|
||||||
|
"runtime"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||||
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
concurrentWriterLimit = runtime.NumCPU()
|
||||||
)
|
)
|
||||||
|
|
||||||
type ContinuousDirtyPages struct {
|
type ContinuousDirtyPages struct {
|
||||||
|
@ -15,17 +21,26 @@ type ContinuousDirtyPages struct {
|
||||||
writeWaitGroup sync.WaitGroup
|
writeWaitGroup sync.WaitGroup
|
||||||
chunkSaveErrChan chan error
|
chunkSaveErrChan chan error
|
||||||
chunkSaveErrChanClosed bool
|
chunkSaveErrChanClosed bool
|
||||||
|
lastErr error
|
||||||
lock sync.Mutex
|
lock sync.Mutex
|
||||||
collection string
|
collection string
|
||||||
replication string
|
replication string
|
||||||
}
|
}
|
||||||
|
|
||||||
func newDirtyPages(file *File) *ContinuousDirtyPages {
|
func newDirtyPages(file *File) *ContinuousDirtyPages {
|
||||||
return &ContinuousDirtyPages{
|
dirtyPages := &ContinuousDirtyPages{
|
||||||
intervals: &ContinuousIntervals{},
|
intervals: &ContinuousIntervals{},
|
||||||
f: file,
|
f: file,
|
||||||
chunkSaveErrChan: make(chan error, 8),
|
chunkSaveErrChan: make(chan error, concurrentWriterLimit),
|
||||||
}
|
}
|
||||||
|
go func() {
|
||||||
|
for t := range dirtyPages.chunkSaveErrChan {
|
||||||
|
if t != nil {
|
||||||
|
dirtyPages.lastErr = t
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return dirtyPages
|
||||||
}
|
}
|
||||||
|
|
||||||
func (pages *ContinuousDirtyPages) AddPage(offset int64, data []byte) {
|
func (pages *ContinuousDirtyPages) AddPage(offset int64, data []byte) {
|
||||||
|
@ -105,7 +120,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64,
|
||||||
chunk.Mtime = mtime
|
chunk.Mtime = mtime
|
||||||
pages.collection, pages.replication = collection, replication
|
pages.collection, pages.replication = collection, replication
|
||||||
pages.f.addChunks([]*filer_pb.FileChunk{chunk})
|
pages.f.addChunks([]*filer_pb.FileChunk{chunk})
|
||||||
pages.chunkSaveErrChan <- nil
|
glog.V(0).Infof("%s saveToStorage [%d,%d)", pages.f.fullpath(), offset, offset+size)
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -208,25 +208,17 @@ func (fh *FileHandle) doFlush(ctx context.Context, header fuse.Header) error {
|
||||||
|
|
||||||
fh.dirtyPages.saveExistingPagesToStorage()
|
fh.dirtyPages.saveExistingPagesToStorage()
|
||||||
|
|
||||||
var err error
|
|
||||||
go func() {
|
|
||||||
for t := range fh.dirtyPages.chunkSaveErrChan {
|
|
||||||
if t != nil {
|
|
||||||
err = t
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
fh.dirtyPages.writeWaitGroup.Wait()
|
fh.dirtyPages.writeWaitGroup.Wait()
|
||||||
|
|
||||||
if err != nil {
|
if fh.dirtyPages.lastErr != nil {
|
||||||
return err
|
return fh.dirtyPages.lastErr
|
||||||
}
|
}
|
||||||
|
|
||||||
if !fh.f.dirtyMetadata {
|
if !fh.f.dirtyMetadata {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
err = fh.f.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
err := fh.f.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||||
|
|
||||||
if fh.f.entry.Attributes != nil {
|
if fh.f.entry.Attributes != nil {
|
||||||
fh.f.entry.Attributes.Mime = fh.contentType
|
fh.f.entry.Attributes.Mime = fh.contentType
|
||||||
|
|
40
weed/util/limiter.go
Normal file
40
weed/util/limiter.go
Normal file
|
@ -0,0 +1,40 @@
|
||||||
|
package util
|
||||||
|
|
||||||
|
// initial version comes from https://github.com/korovkin/limiter/blob/master/limiter.go
|
||||||
|
|
||||||
|
// LimitedConcurrentExecutor object
|
||||||
|
type LimitedConcurrentExecutor struct {
|
||||||
|
limit int
|
||||||
|
tokenChan chan int
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewLimitedConcurrentExecutor(limit int) *LimitedConcurrentExecutor {
|
||||||
|
|
||||||
|
// allocate a limiter instance
|
||||||
|
c := &LimitedConcurrentExecutor{
|
||||||
|
limit: limit,
|
||||||
|
tokenChan: make(chan int, limit),
|
||||||
|
}
|
||||||
|
|
||||||
|
// allocate the tokenChan:
|
||||||
|
for i := 0; i < c.limit; i++ {
|
||||||
|
c.tokenChan <- i
|
||||||
|
}
|
||||||
|
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute adds a function to the execution queue.
|
||||||
|
// if num of go routines allocated by this instance is < limit
|
||||||
|
// launch a new go routine to execute job
|
||||||
|
// else wait until a go routine becomes available
|
||||||
|
func (c *LimitedConcurrentExecutor) Execute(job func()) {
|
||||||
|
token := <-c.tokenChan
|
||||||
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
c.tokenChan <- token
|
||||||
|
}()
|
||||||
|
// run the job
|
||||||
|
job()
|
||||||
|
}()
|
||||||
|
}
|
Loading…
Reference in a new issue