FUSE: add configurable in memory chunk cache size

weed/command/mount.go

@@ -9,6 +9,7 @@ type MountOptions struct {
 	replication                 *string
 	ttlSec                      *int
 	chunkSizeLimitMB            *int
+	chunkCacheCountLimit        *int64
 	dataCenter                  *string
 	allowOthers                 *bool
 	umaskString                 *string
@@ -32,6 +33,7 @@ func init() {
 	mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.")
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
 	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 4, "local write buffer size, also chunk large files")
+	mountOptions.chunkCacheCountLimit = cmdMount.Flag.Int64("chunkCacheCountLimit", 1000, "number of file chunks to cache in memory")
 	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
 	mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
 	mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111")

weed/command/mount_std.go

@@ -168,6 +168,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		Replication:                 *option.replication,
 		TtlSec:                      int32(*option.ttlSec),
 		ChunkSizeLimit:              int64(chunkSizeLimitMB) * 1024 * 1024,
+		ChunkCacheCountLimit:        *option.chunkCacheCountLimit,
 		DataCenter:                  *option.dataCenter,
 		DirListCacheLimit:           *option.dirListCacheLimit,
 		EntryCacheTtl:               3 * time.Second,

weed/filesys/reader_at.go

@@ -57,6 +57,7 @@ func NewChunkReaderAtFromClient(filerClient filer_pb.FilerClient, chunkViews []*
 			return
 		},
 		bufferOffset: -1,
+		chunkCache:   chunkCache,
 	}
 }
 

weed/filesys/wfs.go

@@ -22,17 +22,18 @@ import (
 )
 
 type Option struct {
-	FilerGrpcAddress   string
+	FilerGrpcAddress     string
-	GrpcDialOption     grpc.DialOption
+	GrpcDialOption       grpc.DialOption
-	FilerMountRootPath string
+	FilerMountRootPath   string
-	Collection         string
+	Collection           string
-	Replication        string
+	Replication          string
-	TtlSec             int32
+	TtlSec               int32
-	ChunkSizeLimit     int64
+	ChunkSizeLimit       int64
-	DataCenter         string
+	ChunkCacheCountLimit int64
-	DirListCacheLimit  int64
+	DataCenter           string
-	EntryCacheTtl      time.Duration
+	DirListCacheLimit    int64
-	Umask              os.FileMode
+	EntryCacheTtl        time.Duration
+	Umask                os.FileMode
 
 	MountUid   uint32
 	MountGid   uint32
@@ -81,7 +82,7 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 				return make([]byte, option.ChunkSizeLimit)
 			},
 		},
-		chunkCache: pb_cache.NewChunkCache(),
+		chunkCache: pb_cache.NewChunkCache(option.ChunkCacheCountLimit),
 	}
 
 	wfs.root = &Dir{name: wfs.option.FilerMountRootPath, wfs: wfs}

weed/pb/pb_cache/chunk_cache.go

@@ -11,9 +11,13 @@ type ChunkCache struct {
 	cache *ccache.Cache
 }
 
-func NewChunkCache() *ChunkCache {
+func NewChunkCache(maxEntries int64) *ChunkCache {
+	pruneCount := maxEntries >> 3
+	if pruneCount <= 0 {
+		pruneCount = 500
+	}
 	return &ChunkCache{
-		cache: ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
+		cache: ccache.New(ccache.Configure().MaxSize(maxEntries).ItemsToPrune(uint32(pruneCount))),
 	}
 }
 

weed/server/webdav_server.go

@@ -99,7 +99,7 @@ type WebDavFile struct {
 func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error) {
 	return &WebDavFileSystem{
 		option:     option,
-		chunkCache: pb_cache.NewChunkCache(),
+		chunkCache: pb_cache.NewChunkCache(1000),
 	}, nil
 }