filer: increase directory listing pagination size

weed/filer2/filer.go

@@ -3,18 +3,22 @@ package filer2
 import (
 	"context"
 	"fmt"
-	"google.golang.org/grpc"
 	"math"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
+	"google.golang.org/grpc"
+
+	"github.com/karlseguin/ccache"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
-	"github.com/karlseguin/ccache"
 )
 
+const PaginationSize = 1024 * 256
+
 var (
 	OS_UID = uint32(os.Getuid())
 	OS_GID = uint32(os.Getgid())
@@ -32,7 +36,7 @@ func NewFiler(masters []string, grpcDialOption grpc.DialOption) *Filer {
 	f := &Filer{
 		directoryCache:     ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
 		MasterClient:       wdclient.NewMasterClient(context.Background(), grpcDialOption, "filer", masters),
-		fileIdDeletionChan: make(chan string, 4096),
+		fileIdDeletionChan: make(chan string, PaginationSize),
 		GrpcDialOption:     grpcDialOption,
 	}
 
@@ -217,7 +221,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p FullPath, isRecurs
 		lastFileName := ""
 		includeLastFile := false
 		for limit > 0 {
-			entries, err := f.ListDirectoryEntries(ctx, p, lastFileName, includeLastFile, 1024)
+			entries, err := f.ListDirectoryEntries(ctx, p, lastFileName, includeLastFile, PaginationSize)
 			if err != nil {
 				glog.Errorf("list folder %s: %v", p, err)
 				return fmt.Errorf("list folder %s: %v", p, err)
@@ -241,7 +245,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p FullPath, isRecurs
 				}
 			}
 
-			if len(entries) < 1024 {
+			if len(entries) < PaginationSize {
 				break
 			}
 		}

weed/filer2/filer_client_util.go

@@ -128,8 +128,6 @@ func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath
 
 	err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
-		paginationLimit := 1024 * 256
-
 		lastEntryName := ""
 
 		for {
@@ -137,7 +135,7 @@ func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath
 			request := &filer_pb.ListEntriesRequest{
 				Directory:         fullDirPath,
 				StartFromFileName: lastEntryName,
-				Limit:             uint32(paginationLimit),
+				Limit:             PaginationSize,
 			}
 
 			glog.V(3).Infof("read directory: %v", request)
@@ -151,7 +149,7 @@ func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath
 				lastEntryName = entry.Name
 			}
 
-			if len(resp.Entries) < paginationLimit {
+			if len(resp.Entries) < PaginationSize {
 				break
 			}
 

weed/filesys/dir.go

@@ -214,7 +214,6 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
 
 	err = dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
-		paginationLimit := 1024 * 256
 		remaining := dir.wfs.option.DirListingLimit
 
 		lastEntryName := ""
@@ -224,7 +223,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
 			request := &filer_pb.ListEntriesRequest{
 				Directory:         dir.Path,
 				StartFromFileName: lastEntryName,
-				Limit:             uint32(paginationLimit),
+				Limit:             filer2.PaginationSize,
 			}
 
 			glog.V(4).Infof("read directory: %v", request)
@@ -250,7 +249,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
 
 			remaining -= len(resp.Entries)
 
-			if len(resp.Entries) < paginationLimit {
+			if len(resp.Entries) < filer2.PaginationSize {
 				break
 			}
 

weed/operation/delete_content.go

@@ -107,7 +107,7 @@ func DeleteFilesWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []str
 		ret = append(ret, result...)
 	}
 
-	glog.V(0).Infof("deleted %d items", len(ret))
+	glog.V(1).Infof("deleted %d items", len(ret))
 
 	return ret, err
 }

weed/server/filer_grpc_server.go

@@ -40,7 +40,7 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie
 		limit = fs.option.DirListingLimit
 	}
 
-	paginationLimit := 1024 * 256
+	paginationLimit := filer2.PaginationSize
 	if limit < paginationLimit {
 		paginationLimit = limit
 	}