From 003d48da21b0ecd0758d79bfed234cb2bf820398 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 15 Aug 2020 19:55:28 -0700 Subject: [PATCH] adjust logs --- weed/filer2/reader_at.go | 4 ++-- weed/filesys/dir.go | 4 ++-- weed/filesys/dirty_page.go | 4 ++-- weed/filesys/file.go | 10 +++++----- weed/filesys/filehandle.go | 12 ++++++------ weed/filesys/meta_cache/meta_cache_init.go | 2 +- weed/filesys/wfs.go | 11 +++++------ weed/filesys/wfs_deletion.go | 2 +- weed/operation/upload_content.go | 4 +++- weed/pb/filer_pb/filer_client.go | 2 +- weed/pb/filer_pb/filer_pb_helper.go | 6 +++--- weed/pb/filer_pb/filer_pb_helper_test.go | 2 +- weed/util/chunk_cache/chunk_cache.go | 2 +- 13 files changed, 33 insertions(+), 32 deletions(-) diff --git a/weed/filer2/reader_at.go b/weed/filer2/reader_at.go index 2f65761cc..aee631705 100644 --- a/weed/filer2/reader_at.go +++ b/weed/filer2/reader_at.go @@ -112,12 +112,12 @@ func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) { func (c *ChunkReadAt) fetchChunkData(chunkView *ChunkView) (data []byte, err error) { - glog.V(4).Infof("fetchChunkData %s [%d,%d)\n", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size)) + glog.V(5).Infof("fetchChunkData %s [%d,%d)\n", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size)) hasDataInCache := false chunkData := c.chunkCache.GetChunk(chunkView.FileId, chunkView.ChunkSize) if chunkData != nil { - glog.V(4).Infof("cache hit %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size)) + glog.V(5).Infof("cache hit %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size)) hasDataInCache = true } else { chunkData, err = c.doFetchFullChunkData(chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped) diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 0bfb009f0..578c40014 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -240,7 +240,7 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse. return nil, fuse.ENOENT } } else { - glog.V(4).Infof("dir Lookup cache hit %s", fullFilePath) + glog.V(5).Infof("dir Lookup cache hit %s", fullFilePath) } if entry != nil { @@ -268,7 +268,7 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse. func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { - glog.V(4).Infof("dir ReadDirAll %s", dir.FullPath()) + glog.V(5).Infof("dir ReadDirAll %s", dir.FullPath()) processEachEntryFn := func(entry *filer_pb.Entry, isLast bool) error { fullpath := util.NewFullPath(dir.FullPath(), entry.Name) diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index 8b7d92ffb..ba8f7ec41 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -35,7 +35,7 @@ func (pages *ContinuousDirtyPages) AddPage(offset int64, data []byte) (chunks [] pages.lock.Lock() defer pages.lock.Unlock() - glog.V(4).Infof("%s AddPage [%d,%d) of %d bytes", pages.f.fullpath(), offset, offset+int64(len(data)), pages.f.entry.Attributes.FileSize) + glog.V(5).Infof("%s AddPage [%d,%d) of %d bytes", pages.f.fullpath(), offset, offset+int64(len(data)), pages.f.entry.Attributes.FileSize) if len(data) > int(pages.f.wfs.option.ChunkSizeLimit) { // this is more than what buffer can hold. @@ -127,7 +127,7 @@ func (pages *ContinuousDirtyPages) saveExistingLargestPageToStorage() (chunk *fi chunk, err = pages.saveToStorage(maxList.ToReader(), maxList.Offset(), chunkSize) if err == nil { hasSavedData = true - glog.V(4).Infof("%s saveToStorage %s [%d,%d) of %d bytes", pages.f.fullpath(), chunk.FileId, maxList.Offset(), maxList.Offset()+chunkSize, fileSize) + glog.V(4).Infof("saveToStorage %s %s [%d,%d) of %d bytes", pages.f.fullpath(), chunk.GetFileIdString(), maxList.Offset(), maxList.Offset()+chunkSize, fileSize) return } else { glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.f.fullpath(), maxList.Offset(), maxList.Offset()+chunkSize, err) diff --git a/weed/filesys/file.go b/weed/filesys/file.go index b6242c774..ec7ece604 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -85,7 +85,7 @@ func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { - glog.V(4).Infof("file %v open %+v", file.fullpath(), req) + glog.V(5).Infof("file %v open %+v", file.fullpath(), req) file.isOpen++ @@ -93,7 +93,7 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op resp.Handle = fuse.HandleID(handle.handle) - glog.V(4).Infof("%v file open handle id = %d", file.fullpath(), handle.handle) + glog.V(5).Infof("%v file open handle id = %d", file.fullpath(), handle.handle) return handle, nil @@ -101,7 +101,7 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error { - glog.V(4).Infof("%v file setattr %+v, old:%+v", file.fullpath(), req, file.entry.Attributes) + glog.V(5).Infof("%v file setattr %+v, old:%+v", file.fullpath(), req, file.entry.Attributes) if err := file.maybeLoadEntry(ctx); err != nil { return err @@ -121,10 +121,10 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f int64Size = int64(req.Size) - chunk.Offset if int64Size > 0 { chunks = append(chunks, chunk) - glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk, chunk.Size, int64Size) + glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size) chunk.Size = uint64(int64Size) } else { - glog.V(4).Infof("truncated whole chunk %+v\n", chunk) + glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString()) truncatedChunks = append(truncatedChunks, chunk) } } diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index 42a0b2446..94029f61c 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -54,7 +54,7 @@ var _ = fs.HandleReleaser(&FileHandle{}) func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { - glog.V(2).Infof("%s read fh %d: [%d,%d)", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size)) + glog.V(5).Infof("%s read fh %d: [%d,%d)", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size)) buff := make([]byte, req.Size) @@ -125,7 +125,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f copy(data, req.Data) fh.f.entry.Attributes.FileSize = uint64(max(req.Offset+int64(len(data)), int64(fh.f.entry.Attributes.FileSize))) - glog.V(2).Infof("%v write [%d,%d)", fh.f.fullpath(), req.Offset, req.Offset+int64(len(req.Data))) + glog.V(5).Infof("%v write [%d,%d)", fh.f.fullpath(), req.Offset, req.Offset+int64(len(req.Data))) chunks, err := fh.dirtyPages.AddPage(req.Offset, data) if err != nil { @@ -153,7 +153,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error { - glog.V(4).Infof("%v release fh %d", fh.f.fullpath(), fh.handle) + glog.V(4).Infof("Release %v fh %d", fh.f.fullpath(), fh.handle) fh.f.isOpen-- @@ -170,7 +170,7 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { // fflush works at fh level // send the data to the OS - glog.V(4).Infof("%s fh %d flush %v", fh.f.fullpath(), fh.handle, req) + glog.V(5).Infof("Flush %s fh %d %v", fh.f.fullpath(), fh.handle, req) chunks, err := fh.dirtyPages.FlushToStorage() if err != nil { @@ -213,7 +213,7 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { glog.V(4).Infof("%s set chunks: %v", fh.f.fullpath(), len(fh.f.entry.Chunks)) for i, chunk := range fh.f.entry.Chunks { - glog.V(4).Infof("%s chunks %d: %v [%d,%d)", fh.f.fullpath(), i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size)) + glog.V(4).Infof("%s chunks %d: %v [%d,%d)", fh.f.fullpath(), i, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size)) } chunks, garbages := filer2.CompactFileChunks(filer2.LookupFn(fh.f.wfs), fh.f.entry.Chunks) @@ -238,7 +238,7 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { fh.f.wfs.deleteFileChunks(garbages) for i, chunk := range garbages { - glog.V(4).Infof("garbage %s chunks %d: %v [%d,%d)", fh.f.fullpath(), i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size)) + glog.V(4).Infof("garbage %s chunks %d: %v [%d,%d)", fh.f.fullpath(), i, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size)) } return nil diff --git a/weed/filesys/meta_cache/meta_cache_init.go b/weed/filesys/meta_cache/meta_cache_init.go index 662a60fe0..cd98f4a7c 100644 --- a/weed/filesys/meta_cache/meta_cache_init.go +++ b/weed/filesys/meta_cache/meta_cache_init.go @@ -14,7 +14,7 @@ func EnsureVisited(mc *MetaCache, client filer_pb.FilerClient, dirPath util.Full mc.visitedBoundary.EnsureVisited(dirPath, func(path util.FullPath) (childDirectories []string, err error) { - glog.V(4).Infof("ReadDirAllEntries %s ...", path) + glog.V(5).Infof("ReadDirAllEntries %s ...", path) err = filer_pb.ReadDirAllEntries(client, dirPath, "", func(pbEntry *filer_pb.Entry, isLast bool) error { entry := filer2.FromPbEntry(string(dirPath), pbEntry) diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index 22f0b655a..eb7042663 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -113,7 +113,7 @@ func (wfs *WFS) Root() (fs.Node, error) { func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHandle) { fullpath := file.fullpath() - glog.V(4).Infof("%s AcquireHandle uid=%d gid=%d", fullpath, uid, gid) + glog.V(4).Infof("AcquireHandle %s uid=%d gid=%d", fullpath, uid, gid) wfs.handlesLock.Lock() defer wfs.handlesLock.Unlock() @@ -127,7 +127,6 @@ func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHand fileHandle = newFileHandle(file, uid, gid) wfs.handles[inodeId] = fileHandle fileHandle.handle = inodeId - glog.V(4).Infof("%s new fh %d", fullpath, fileHandle.handle) return } @@ -136,7 +135,7 @@ func (wfs *WFS) ReleaseHandle(fullpath util.FullPath, handleId fuse.HandleID) { wfs.handlesLock.Lock() defer wfs.handlesLock.Unlock() - glog.V(4).Infof("%s ReleaseHandle id %d current handles length %d", fullpath, handleId, len(wfs.handles)) + glog.V(5).Infof("%s ReleaseHandle id %d current handles length %d", fullpath, handleId, len(wfs.handles)) delete(wfs.handles, fullpath.AsInode()) @@ -146,7 +145,7 @@ func (wfs *WFS) ReleaseHandle(fullpath util.FullPath, handleId fuse.HandleID) { // Statfs is called to obtain file system metadata. Implements fuse.FSStatfser func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.StatfsResponse) error { - glog.V(4).Infof("reading fs stats: %+v", req) + glog.V(5).Infof("reading fs stats: %+v", req) if wfs.stats.lastChecked < time.Now().Unix()-20 { @@ -158,13 +157,13 @@ func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse. Ttl: fmt.Sprintf("%ds", wfs.option.TtlSec), } - glog.V(4).Infof("reading filer stats: %+v", request) + glog.V(5).Infof("reading filer stats: %+v", request) resp, err := client.Statistics(context.Background(), request) if err != nil { glog.V(0).Infof("reading filer stats %v: %v", request, err) return err } - glog.V(4).Infof("read filer stats: %+v", resp) + glog.V(5).Infof("read filer stats: %+v", resp) wfs.stats.TotalSize = resp.TotalSize wfs.stats.UsedSize = resp.UsedSize diff --git a/weed/filesys/wfs_deletion.go b/weed/filesys/wfs_deletion.go index bf21b1808..203ebdad1 100644 --- a/weed/filesys/wfs_deletion.go +++ b/weed/filesys/wfs_deletion.go @@ -38,7 +38,7 @@ func (wfs *WFS) deleteFileIds(grpcDialOption grpc.DialOption, client filer_pb.Se m := make(map[string]operation.LookupResult) - glog.V(4).Infof("remove file lookup volume id locations: %v", vids) + glog.V(5).Infof("deleteFileIds lookup volume id locations: %v", vids) resp, err := client.LookupVolume(context.Background(), &filer_pb.LookupVolumeRequest{ VolumeIds: vids, }) diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index e1914f20a..f59c7e1a9 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -33,6 +33,7 @@ type UploadResult struct { } func (uploadResult *UploadResult) ToPbFileChunk(fileId string, offset int64) *filer_pb.FileChunk { + fid, _ := filer_pb.ToFileIdObject(fileId) return &filer_pb.FileChunk{ FileId: fileId, Offset: offset, @@ -41,6 +42,7 @@ func (uploadResult *UploadResult) ToPbFileChunk(fileId string, offset int64) *fi ETag: uploadResult.ETag, CipherKey: uploadResult.CipherKey, IsCompressed: uploadResult.Gzip > 0, + Fid: fid, } } @@ -84,7 +86,7 @@ func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, } func retriedUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) { - for i:=0; i< 3; i++ { + for i := 0; i < 3; i++ { uploadResult, err = doUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt) if err == nil { return diff --git a/weed/pb/filer_pb/filer_client.go b/weed/pb/filer_pb/filer_client.go index 6605202e0..c5a8c311a 100644 --- a/weed/pb/filer_pb/filer_client.go +++ b/weed/pb/filer_pb/filer_client.go @@ -83,7 +83,7 @@ func doList(filerClient FilerClient, fullDirPath util.FullPath, prefix string, f InclusiveStartFrom: inclusive, } - glog.V(4).Infof("read directory: %v", request) + glog.V(5).Infof("read directory: %v", request) ctx, cancel := context.WithCancel(context.Background()) stream, err := client.ListEntries(ctx, request) if err != nil { diff --git a/weed/pb/filer_pb/filer_pb_helper.go b/weed/pb/filer_pb/filer_pb_helper.go index 96ab2154f..2dc1ebaf8 100644 --- a/weed/pb/filer_pb/filer_pb_helper.go +++ b/weed/pb/filer_pb/filer_pb_helper.go @@ -10,7 +10,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage/needle" ) -func toFileIdObject(fileIdStr string) (*FileId, error) { +func ToFileIdObject(fileIdStr string) (*FileId, error) { t, err := needle.ParseFileIdFromString(fileIdStr) if err != nil { return nil, err @@ -43,14 +43,14 @@ func BeforeEntrySerialization(chunks []*FileChunk) { for _, chunk := range chunks { if chunk.FileId != "" { - if fid, err := toFileIdObject(chunk.FileId); err == nil { + if fid, err := ToFileIdObject(chunk.FileId); err == nil { chunk.Fid = fid chunk.FileId = "" } } if chunk.SourceFileId != "" { - if fid, err := toFileIdObject(chunk.SourceFileId); err == nil { + if fid, err := ToFileIdObject(chunk.SourceFileId); err == nil { chunk.SourceFid = fid chunk.SourceFileId = "" } diff --git a/weed/pb/filer_pb/filer_pb_helper_test.go b/weed/pb/filer_pb/filer_pb_helper_test.go index d4468c011..0009afdbe 100644 --- a/weed/pb/filer_pb/filer_pb_helper_test.go +++ b/weed/pb/filer_pb/filer_pb_helper_test.go @@ -9,7 +9,7 @@ import ( func TestFileIdSize(t *testing.T) { fileIdStr := "11745,0293434534cbb9892b" - fid, _ := toFileIdObject(fileIdStr) + fid, _ := ToFileIdObject(fileIdStr) bytes, _ := proto.Marshal(fid) println(len(fileIdStr)) diff --git a/weed/util/chunk_cache/chunk_cache.go b/weed/util/chunk_cache/chunk_cache.go index 17b64fb6c..b54b40dbb 100644 --- a/weed/util/chunk_cache/chunk_cache.go +++ b/weed/util/chunk_cache/chunk_cache.go @@ -89,7 +89,7 @@ func (c *ChunkCache) SetChunk(fileId string, data []byte) { c.Lock() defer c.Unlock() - glog.V(4).Infof("SetChunk %s size %d\n", fileId, len(data)) + glog.V(5).Infof("SetChunk %s size %d\n", fileId, len(data)) c.doSetChunk(fileId, data) }