From fa0cfdfb7f5f152930349ee26fa070f32efd044f Mon Sep 17 00:00:00 2001 From: Radtoo <56451+Radtoo@users.noreply.github.com> Date: Fri, 21 Jan 2022 06:38:34 +0100 Subject: [PATCH 1/5] Added basic deletion capabilities to fsck. --- weed/shell/command_volume_fsck.go | 137 +++++++++++++++++++++--------- 1 file changed, 98 insertions(+), 39 deletions(-) diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 2d570fef3..4daab0f76 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -6,7 +6,10 @@ import ( "flag" "fmt" "io" + "io/ioutil" "math" + "net/http" + "net/url" "os" "path/filepath" "sync" @@ -61,7 +64,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. verbose := fsckCommand.Bool("v", false, "verbose mode") findMissingChunksInFiler := fsckCommand.Bool("findMissingChunksInFiler", false, "see \"help volume.fsck\"") findMissingChunksInFilerPath := fsckCommand.String("findMissingChunksInFilerPath", "/", "used together with findMissingChunksInFiler") - applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, " delete data not referenced by the filer") + applyPurging := fsckCommand.Bool("forcePurge", false, " after detection, delete missing data from volumes / delete missing file entries from filer") if err = fsckCommand.Parse(args); err != nil { return nil } @@ -98,7 +101,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. if *findMissingChunksInFiler { // collect all filer file ids and paths - if err = c.collectFilerFileIdAndPaths(volumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose, applyPurging); err != nil { + if err = c.collectFilerFileIdAndPaths(volumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose); err != nil { return fmt.Errorf("collectFilerFileIdAndPaths: %v", err) } // for each volume, check filer file ids @@ -107,10 +110,10 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. } } else { // collect all filer file ids - if err = c.collectFilerFileIds(tempFolder, volumeIdToVInfo, *verbose, writer); err != nil { + if err = c.collectFilerFileIds(volumeIdToVInfo, tempFolder, writer, *verbose); err != nil { return fmt.Errorf("failed to collect file ids from filer: %v", err) } - // volume file ids substract filer file ids + // volume file ids subtract filer file ids if err = c.findExtraChunksInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, applyPurging); err != nil { return fmt.Errorf("findExtraChunksInVolumeServers: %v", err) } @@ -119,7 +122,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. return nil } -func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool, applyPurging *bool) error { +func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool) error { if verbose { fmt.Fprintf(writer, "checking each file from filer ...\n") @@ -185,7 +188,7 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging *bool) error { for volumeId, vinfo := range volumeIdToVInfo { - checkErr := c.oneVolumeFileIdsCheckOneVolume(tempFolder, volumeId, writer, verbose) + checkErr := c.oneVolumeFileIdsCheckOneVolume(tempFolder, volumeId, writer, verbose, applyPurging) if checkErr != nil { return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr) } @@ -194,7 +197,9 @@ func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInf } func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging *bool) error { + var totalInUseCount, totalOrphanChunkCount, totalOrphanDataSize uint64 + for volumeId, vinfo := range volumeIdToVInfo { inUseCount, orphanFileIds, orphanDataSize, checkErr := c.oneVolumeFileIdsSubtractFilerFileIds(tempFolder, volumeId, writer, verbose) if checkErr != nil { @@ -210,32 +215,52 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[u } } - if *applyPurging && len(orphanFileIds) > 0 { - if vinfo.isEcVolume { - fmt.Fprintf(writer, "Skip purging for Erasure Coded volume %d.\n", volumeId) - continue - } - if vinfo.isReadOnly { - fmt.Fprintf(writer, "Skip purging for read only volume %d.\n", volumeId) - continue - } - if inUseCount == 0 { - if err := deleteVolume(c.env.option.GrpcDialOption, needle.VolumeId(volumeId), vinfo.server); err != nil { - return fmt.Errorf("delete volume %d: %v", volumeId, err) + if len(orphanFileIds) > 0 { + + if *applyPurging { + if verbose { + fmt.Fprintf(writer, "purging process for volume %d", volumeId) + } + + if vinfo.isEcVolume { + fmt.Fprintf(writer, "skip purging for Erasure Coded volume %d.\n", volumeId) + continue + } + + needleVID := needle.VolumeId(volumeId) + + if vinfo.isReadOnly { + err := markVolumeWritable(c.env.option.GrpcDialOption, needleVID, vinfo.server, true) + if err != nil { + return fmt.Errorf("mark volume %d read/write: %v", volumeId, err) + } + + fmt.Fprintf(writer, "temporarily marked %d on server %v writable for forced purge\n", volumeId, vinfo.server) + defer markVolumeWritable(c.env.option.GrpcDialOption, needleVID, vinfo.server, false) + } + + fmt.Fprintf(writer, "marked %d on server %v writable for forced purge\n", volumeId, vinfo.server) + + // FIXME Does not delete all replicas of this volumeId on all volume servers. Using fileId based deletion in all cases for now. + // if inUseCount < 1 { + // if verbose { + // fmt.Fprintf(writer, "removing empty (all content orphaned) volume %d\n", volumeId) + // } + // if err := deleteVolume(c.env.option.GrpcDialOption, needleVID, vinfo.server); err != nil { + // return fmt.Errorf("removing empty volume %d: %v", volumeId, err) + // } + // } else { + + if verbose { + fmt.Fprintf(writer, "purging files from volume %d\n", volumeId) } - } else { if err := c.purgeFileIdsForOneVolume(volumeId, orphanFileIds, writer); err != nil { - return fmt.Errorf("purge for volume %d: %v", volumeId, err) + return fmt.Errorf("purging volume %d: %v", volumeId, err) } } } } - if totalOrphanChunkCount == 0 { - fmt.Fprintf(writer, "no orphan data\n") - return nil - } - if !*applyPurging { pct := float64(totalOrphanChunkCount*100) / (float64(totalOrphanChunkCount + totalInUseCount)) fmt.Fprintf(writer, "\nTotal\t\tentries:%d\torphan:%d\t%.2f%%\t%dB\n", @@ -243,6 +268,12 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[u fmt.Fprintf(writer, "This could be normal if multiple filers or no filers are used.\n") } + + if totalOrphanChunkCount == 0 { + fmt.Fprintf(writer, "no orphan data\n") + //return nil + } + return nil } @@ -283,7 +314,7 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, volumeId } -func (c *commandVolumeFsck) collectFilerFileIds(tempFolder string, volumeIdToServer map[uint32]VInfo, verbose bool, writer io.Writer) error { +func (c *commandVolumeFsck) collectFilerFileIds(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool) error { if verbose { fmt.Fprintf(writer, "collecting file ids from filer ...\n") @@ -333,10 +364,10 @@ func (c *commandVolumeFsck) collectFilerFileIds(tempFolder string, volumeIdToSer }) } -func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, volumeId uint32, writer io.Writer, verbose bool) (err error) { +func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, volumeId uint32, writer io.Writer, verbose bool, applyPurging *bool) (err error) { if verbose { - fmt.Fprintf(writer, "find missing file chuns in volume %d ...\n", volumeId) + fmt.Fprintf(writer, "find missing file chunks in volume %d ...\n", volumeId) } db := needle_map.NewMemDb() @@ -366,11 +397,7 @@ func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, vo for { readSize, err = io.ReadFull(br, buffer) if err != nil || readSize != 16 { - if err == io.EOF { - return nil - } else { - break - } + break } item.fileKey = util.BytesToUint64(buffer[:8]) @@ -386,14 +413,46 @@ func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, vo } item.path = util.FullPath(string(pathBytes)) - if _, found := db.Get(types.NeedleId(item.fileKey)); !found { - fmt.Fprintf(writer, "%d,%x%08x in %s %d not found\n", volumeId, item.fileKey, item.cookie, item.path, pathSize) + needleId := types.NeedleId(item.fileKey) + if _, found := db.Get(needleId); !found { + fmt.Fprintf(writer, "%s\n", item.path) + + if *applyPurging { + req, err := http.NewRequest(http.MethodDelete, "", nil) + // defining the URL this way automatically escapes complex path names + req.URL = &url.URL{ + Scheme: "http", + Host: c.env.option.FilerAddress.ToHttpAddress(), + Path: string(item.path), + } + if verbose { + fmt.Printf("full HTTP delete request to be sent: %v\n", req) + } + if err != nil { + fmt.Errorf("HTTP delete request error: %v\n", err) + } + + client := &http.Client{} + + resp, err := client.Do(req) + if err != nil { + fmt.Errorf("DELETE fetch error: %v\n", err) + } + defer resp.Body.Close() + + _, err = ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Errorf("DELETE response error: %v\n", err) + } + + if verbose { + fmt.Println("delete response Status : ", resp.Status) + fmt.Println("delete response Headers : ", resp.Header) + } + } } - } - - return - + return nil } func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(tempFolder string, volumeId uint32, writer io.Writer, verbose bool) (inUseCount uint64, orphanFileIds []string, orphanDataSize uint64, err error) { From 6b17f45da2302b6a36908ff3fc9b0b7606fd677d Mon Sep 17 00:00:00 2001 From: Radtoo <56451+Radtoo@users.noreply.github.com> Date: Sun, 6 Feb 2022 23:22:04 +0100 Subject: [PATCH 2/5] Also delete paths for which a volume is entirely absent. --- weed/shell/command_volume_fsck.go | 147 +++++++++++++++--------------- 1 file changed, 72 insertions(+), 75 deletions(-) diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 4daab0f76..410b51db3 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -101,11 +101,11 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. if *findMissingChunksInFiler { // collect all filer file ids and paths - if err = c.collectFilerFileIdAndPaths(volumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose); err != nil { + if err = c.collectFilerFileIdAndPaths(volumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose, *applyPurging); err != nil { return fmt.Errorf("collectFilerFileIdAndPaths: %v", err) } // for each volume, check filer file ids - if err = c.findFilerChunksMissingInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, applyPurging); err != nil { + if err = c.findFilerChunksMissingInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, *applyPurging); err != nil { return fmt.Errorf("findFilerChunksMissingInVolumeServers: %v", err) } } else { @@ -114,7 +114,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. return fmt.Errorf("failed to collect file ids from filer: %v", err) } // volume file ids subtract filer file ids - if err = c.findExtraChunksInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, applyPurging); err != nil { + if err = c.findExtraChunksInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, *applyPurging); err != nil { return fmt.Errorf("findExtraChunksInVolumeServers: %v", err) } } @@ -122,7 +122,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. return nil } -func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool) error { +func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool, applyPurging bool) error { if verbose { fmt.Fprintf(writer, "checking each file from filer ...\n") @@ -179,13 +179,17 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint // fmt.Fprintf(writer, "%d,%x%08x %d %s\n", i.vid, i.fileKey, i.cookie, len(i.path), i.path) } else { fmt.Fprintf(writer, "%d,%x%08x %s volume not found\n", i.vid, i.fileKey, i.cookie, i.path) + if applyPurging { + fmt.Printf("deleting path %s for volume not found", i.path) + c.httpDelete(i.path, verbose) + } } } }) } -func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging *bool) error { +func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging bool) error { for volumeId, vinfo := range volumeIdToVInfo { checkErr := c.oneVolumeFileIdsCheckOneVolume(tempFolder, volumeId, writer, verbose, applyPurging) @@ -196,7 +200,7 @@ func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInf return nil } -func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging *bool) error { +func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging bool) error { var totalInUseCount, totalOrphanChunkCount, totalOrphanDataSize uint64 @@ -215,53 +219,41 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[u } } - if len(orphanFileIds) > 0 { + if applyPurging && len(orphanFileIds) > 0 { + if verbose { + fmt.Fprintf(writer, "purging process for volume %d", volumeId) + } - if *applyPurging { - if verbose { - fmt.Fprintf(writer, "purging process for volume %d", volumeId) + if vinfo.isEcVolume { + fmt.Fprintf(writer, "skip purging for Erasure Coded volume %d.\n", volumeId) + continue + } + + needleVID := needle.VolumeId(volumeId) + + if vinfo.isReadOnly { + err := markVolumeWritable(c.env.option.GrpcDialOption, needleVID, vinfo.server, true) + if err != nil { + return fmt.Errorf("mark volume %d read/write: %v", volumeId, err) } - if vinfo.isEcVolume { - fmt.Fprintf(writer, "skip purging for Erasure Coded volume %d.\n", volumeId) - continue - } + fmt.Fprintf(writer, "temporarily marked %d on server %v writable for forced purge\n", volumeId, vinfo.server) + defer markVolumeWritable(c.env.option.GrpcDialOption, needleVID, vinfo.server, false) + } - needleVID := needle.VolumeId(volumeId) + fmt.Fprintf(writer, "marked %d on server %v writable for forced purge\n", volumeId, vinfo.server) - if vinfo.isReadOnly { - err := markVolumeWritable(c.env.option.GrpcDialOption, needleVID, vinfo.server, true) - if err != nil { - return fmt.Errorf("mark volume %d read/write: %v", volumeId, err) - } + if verbose { + fmt.Fprintf(writer, "purging files from volume %d\n", volumeId) + } - fmt.Fprintf(writer, "temporarily marked %d on server %v writable for forced purge\n", volumeId, vinfo.server) - defer markVolumeWritable(c.env.option.GrpcDialOption, needleVID, vinfo.server, false) - } - - fmt.Fprintf(writer, "marked %d on server %v writable for forced purge\n", volumeId, vinfo.server) - - // FIXME Does not delete all replicas of this volumeId on all volume servers. Using fileId based deletion in all cases for now. - // if inUseCount < 1 { - // if verbose { - // fmt.Fprintf(writer, "removing empty (all content orphaned) volume %d\n", volumeId) - // } - // if err := deleteVolume(c.env.option.GrpcDialOption, needleVID, vinfo.server); err != nil { - // return fmt.Errorf("removing empty volume %d: %v", volumeId, err) - // } - // } else { - - if verbose { - fmt.Fprintf(writer, "purging files from volume %d\n", volumeId) - } - if err := c.purgeFileIdsForOneVolume(volumeId, orphanFileIds, writer); err != nil { - return fmt.Errorf("purging volume %d: %v", volumeId, err) - } + if err := c.purgeFileIdsForOneVolume(volumeId, orphanFileIds, writer); err != nil { + return fmt.Errorf("purging volume %d: %v", volumeId, err) } } } - if !*applyPurging { + if !applyPurging { pct := float64(totalOrphanChunkCount*100) / (float64(totalOrphanChunkCount + totalInUseCount)) fmt.Fprintf(writer, "\nTotal\t\tentries:%d\torphan:%d\t%.2f%%\t%dB\n", totalOrphanChunkCount+totalInUseCount, totalOrphanChunkCount, pct, totalOrphanDataSize) @@ -364,7 +356,7 @@ func (c *commandVolumeFsck) collectFilerFileIds(volumeIdToServer map[uint32]VInf }) } -func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, volumeId uint32, writer io.Writer, verbose bool, applyPurging *bool) (err error) { +func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, volumeId uint32, writer io.Writer, verbose bool, applyPurging bool) (err error) { if verbose { fmt.Fprintf(writer, "find missing file chunks in volume %d ...\n", volumeId) @@ -417,44 +409,49 @@ func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, vo if _, found := db.Get(needleId); !found { fmt.Fprintf(writer, "%s\n", item.path) - if *applyPurging { - req, err := http.NewRequest(http.MethodDelete, "", nil) + if applyPurging { // defining the URL this way automatically escapes complex path names - req.URL = &url.URL{ - Scheme: "http", - Host: c.env.option.FilerAddress.ToHttpAddress(), - Path: string(item.path), - } - if verbose { - fmt.Printf("full HTTP delete request to be sent: %v\n", req) - } - if err != nil { - fmt.Errorf("HTTP delete request error: %v\n", err) - } - - client := &http.Client{} - - resp, err := client.Do(req) - if err != nil { - fmt.Errorf("DELETE fetch error: %v\n", err) - } - defer resp.Body.Close() - - _, err = ioutil.ReadAll(resp.Body) - if err != nil { - fmt.Errorf("DELETE response error: %v\n", err) - } - - if verbose { - fmt.Println("delete response Status : ", resp.Status) - fmt.Println("delete response Headers : ", resp.Header) - } + c.httpDelete(item.path, verbose) } } } return nil } +func (c *commandVolumeFsck) httpDelete(path util.FullPath, verbose bool) { + req, err := http.NewRequest(http.MethodDelete, "", nil) + + req.URL = &url.URL{ + Scheme: "http", + Host: c.env.option.FilerAddress.ToHttpAddress(), + Path: string(path), + } + if verbose { + fmt.Printf("full HTTP delete request to be sent: %v\n", req) + } + if err != nil { + fmt.Errorf("HTTP delete request error: %v\n", err) + } + + client := &http.Client{} + + resp, err := client.Do(req) + if err != nil { + fmt.Errorf("DELETE fetch error: %v\n", err) + } + defer resp.Body.Close() + + _, err = ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Errorf("DELETE response error: %v\n", err) + } + + if verbose { + fmt.Println("delete response Status : ", resp.Status) + fmt.Println("delete response Headers : ", resp.Header) + } +} + func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(tempFolder string, volumeId uint32, writer io.Writer, verbose bool) (inUseCount uint64, orphanFileIds []string, orphanDataSize uint64, err error) { db := needle_map.NewMemDb() From fbb14e0ea8145777486af160958d6c90a4569415 Mon Sep 17 00:00:00 2001 From: Radtoo <56451+Radtoo@users.noreply.github.com> Date: Sun, 6 Feb 2022 23:46:52 +0100 Subject: [PATCH 3/5] Adding separate toggle to purge absent vols While this toggle is basically required to clean out entries for deleted volumes, having a separate description + toggling this separately seems like a good idea so people get a chance to check if their volumes are all mounted/connected as expected. Also renamed forcePurge to just purge. --- weed/shell/command_volume_fsck.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 410b51db3..6a4ab2559 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -64,7 +64,8 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. verbose := fsckCommand.Bool("v", false, "verbose mode") findMissingChunksInFiler := fsckCommand.Bool("findMissingChunksInFiler", false, "see \"help volume.fsck\"") findMissingChunksInFilerPath := fsckCommand.String("findMissingChunksInFilerPath", "/", "used together with findMissingChunksInFiler") - applyPurging := fsckCommand.Bool("forcePurge", false, " after detection, delete missing data from volumes / delete missing file entries from filer") + applyPurging := fsckCommand.Bool("purge", false, " after detection, delete missing data from volumes / delete missing file entries from filer") + purgeAbsent := fsckCommand.Bool("purgeAbsent", false, " delete missing file entries from filer if the corresponding volume is missing for any reason, please ensure all still existing/expected volumes are connected! used together with findMissingChunksInFiler") if err = fsckCommand.Parse(args); err != nil { return nil } @@ -101,7 +102,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. if *findMissingChunksInFiler { // collect all filer file ids and paths - if err = c.collectFilerFileIdAndPaths(volumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose, *applyPurging); err != nil { + if err = c.collectFilerFileIdAndPaths(volumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose, *purgeAbsent); err != nil { return fmt.Errorf("collectFilerFileIdAndPaths: %v", err) } // for each volume, check filer file ids @@ -122,7 +123,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. return nil } -func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool, applyPurging bool) error { +func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool, purgeAbsent bool) error { if verbose { fmt.Fprintf(writer, "checking each file from filer ...\n") @@ -179,8 +180,8 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint // fmt.Fprintf(writer, "%d,%x%08x %d %s\n", i.vid, i.fileKey, i.cookie, len(i.path), i.path) } else { fmt.Fprintf(writer, "%d,%x%08x %s volume not found\n", i.vid, i.fileKey, i.cookie, i.path) - if applyPurging { - fmt.Printf("deleting path %s for volume not found", i.path) + if purgeAbsent { + fmt.Printf("deleting path %s after volume not found", i.path) c.httpDelete(i.path, verbose) } } From fbe7ed7927e63bdecf3527e5d96501e084b388fc Mon Sep 17 00:00:00 2001 From: Radtoo <56451+Radtoo@users.noreply.github.com> Date: Thu, 10 Feb 2022 20:01:48 +0100 Subject: [PATCH 4/5] Revert the the flag to "reallyDeleteFromVolume". --- weed/shell/command_volume_fsck.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 6a4ab2559..005dc98c2 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -64,7 +64,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. verbose := fsckCommand.Bool("v", false, "verbose mode") findMissingChunksInFiler := fsckCommand.Bool("findMissingChunksInFiler", false, "see \"help volume.fsck\"") findMissingChunksInFilerPath := fsckCommand.String("findMissingChunksInFilerPath", "/", "used together with findMissingChunksInFiler") - applyPurging := fsckCommand.Bool("purge", false, " after detection, delete missing data from volumes / delete missing file entries from filer") + applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, " after detection, delete missing data from volumes / delete missing file entries from filer") purgeAbsent := fsckCommand.Bool("purgeAbsent", false, " delete missing file entries from filer if the corresponding volume is missing for any reason, please ensure all still existing/expected volumes are connected! used together with findMissingChunksInFiler") if err = fsckCommand.Parse(args); err != nil { return nil From 724ce04b1c6ab3f397e0a33add28b07906179482 Mon Sep 17 00:00:00 2001 From: Radtoo <56451+Radtoo@users.noreply.github.com> Date: Sun, 13 Feb 2022 00:53:35 +0100 Subject: [PATCH 5/5] We picked the second flag's name. --- weed/shell/command_volume_fsck.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 005dc98c2..e73f60a2b 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -65,7 +65,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. findMissingChunksInFiler := fsckCommand.Bool("findMissingChunksInFiler", false, "see \"help volume.fsck\"") findMissingChunksInFilerPath := fsckCommand.String("findMissingChunksInFilerPath", "/", "used together with findMissingChunksInFiler") applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, " after detection, delete missing data from volumes / delete missing file entries from filer") - purgeAbsent := fsckCommand.Bool("purgeAbsent", false, " delete missing file entries from filer if the corresponding volume is missing for any reason, please ensure all still existing/expected volumes are connected! used together with findMissingChunksInFiler") + purgeAbsent := fsckCommand.Bool("reallyDeleteFilerEntries", false, " delete missing file entries from filer if the corresponding volume is missing for any reason, please ensure all still existing/expected volumes are connected! used together with findMissingChunksInFiler") if err = fsckCommand.Parse(args); err != nil { return nil }