From 981c981fa8862c34f0a56ec2823f66a029c17826 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Tue, 17 Mar 2015 11:15:37 -0700 Subject: [PATCH] one-off fix for issues/93 https://github.com/chrislusf/weed-fs/issues/93 --- unmaintained/README.txt | 9 +++ unmaintained/fix_dat/fix_dat.go | 130 ++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 unmaintained/README.txt create mode 100644 unmaintained/fix_dat/fix_dat.go diff --git a/unmaintained/README.txt b/unmaintained/README.txt new file mode 100644 index 000000000..f34136959 --- /dev/null +++ b/unmaintained/README.txt @@ -0,0 +1,9 @@ +Here is the folder of unmaintained go code. + +Although its content is useful sometimes, +Its content would not be released. +And the code can be ugly, e.g., not checking exceptions. + +If any code is used often, it will be productionized. +So it is helpful if anyone wants to cleanup a bit. + diff --git a/unmaintained/fix_dat/fix_dat.go b/unmaintained/fix_dat/fix_dat.go new file mode 100644 index 000000000..486da4a78 --- /dev/null +++ b/unmaintained/fix_dat/fix_dat.go @@ -0,0 +1,130 @@ +package main + +import ( + "flag" + "fmt" + "io" + "os" + "path" + "strconv" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" +) + +var ( + fixVolumePath = flag.String("dir", "/tmp", "data directory to store files") + fixVolumeCollection = flag.String("collection", "", "the volume collection name") + fixVolumeId = flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.") +) + +/* +This is to resolve an one-time issue that caused inconsistency with .dat and .idx files. +1. fix the .dat file, a new .dat_fixed file will be generated. + go run fix_dat.go -volumeId=9 -dir=/Users/chrislu/Downloads +2. move the original .dat and .idx files to some backup folder, and rename .dat_fixed to .dat file + mv 9.dat_fixed 9.dat +3. fix the .idx file with the "weed fix" + weed fix -volumeId=9 -dir=/Users/chrislu/Downloads +*/ +func main() { + flag.Parse() + fileName := strconv.Itoa(*fixVolumeId) + if *fixVolumeCollection != "" { + fileName = *fixVolumeCollection + "_" + fileName + } + indexFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".idx"), os.O_RDONLY, 0644) + if err != nil { + glog.Fatalf("Read Volume Index [ERROR] %s\n", err) + } + defer indexFile.Close() + datFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".dat"), os.O_RDONLY, 0644) + if err != nil { + glog.Fatalf("Read Volume Data [ERROR] %s\n", err) + } + defer datFile.Close() + + newDatFile, err := os.Create(path.Join(*fixVolumePath, fileName+".dat_fixed")) + if err != nil { + glog.Fatalf("Write New Volume Data [ERROR] %s\n", err) + } + defer newDatFile.Close() + + header := make([]byte, storage.SuperBlockSize) + datFile.Read(header) + newDatFile.Write(header) + + iterateEntries(datFile, indexFile, func(n *storage.Needle, offset int64) { + fmt.Printf("file id=%d name=%s size=%d dataSize=%d\n", n.Id, string(n.Name), n.Size, n.DataSize) + s, e := n.Append(newDatFile, storage.Version2) + fmt.Printf("size %d error %v\n", s, e) + }) + +} + +func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needle, offset int64)) { + // start to read index file + var readerOffset int64 + bytes := make([]byte, 16) + count, _ := idxFile.ReadAt(bytes, readerOffset) + readerOffset += int64(count) + + // start to read dat file + offset := int64(storage.SuperBlockSize) + version := storage.Version2 + n, rest, err := storage.ReadNeedleHeader(datFile, version, offset) + if err != nil { + fmt.Printf("cannot read needle header: %v", err) + return + } + fmt.Printf("Needle %+v, rest %d\n", n, rest) + for n != nil && count > 0 { + // parse index file entry + key := util.BytesToUint64(bytes[0:8]) + offsetFromIndex := util.BytesToUint32(bytes[8:12]) + sizeFromIndex := util.BytesToUint32(bytes[12:16]) + count, _ = idxFile.ReadAt(bytes, readerOffset) + readerOffset += int64(count) + + if offsetFromIndex != 0 && offset != int64(offsetFromIndex)*8 { + //t := offset + offset = int64(offsetFromIndex) * 8 + //fmt.Printf("Offset change %d => %d\n", t, offset) + } + + fmt.Printf("key: %d offsetFromIndex %d n.Size %d sizeFromIndex:%d\n", key, offsetFromIndex, n.Size, sizeFromIndex) + + padding := storage.NeedlePaddingSize - ((sizeFromIndex + storage.NeedleHeaderSize + storage.NeedleChecksumSize) % storage.NeedlePaddingSize) + rest = sizeFromIndex + storage.NeedleChecksumSize + padding + + func() { + defer func() { + if r := recover(); r != nil { + fmt.Println("Recovered in f", r) + } + }() + if err = n.ReadNeedleBody(datFile, version, offset+int64(storage.NeedleHeaderSize), rest); err != nil { + fmt.Printf("cannot read needle body: offset %d body %d %v\n", offset, rest, err) + } + }() + + if n.Size <= n.DataSize { + continue + } + visitNeedle(n, offset) + + offset += int64(storage.NeedleHeaderSize) + int64(rest) + //fmt.Printf("==> new entry offset %d\n", offset) + if n, rest, err = storage.ReadNeedleHeader(datFile, version, offset); err != nil { + if err == io.EOF { + return + } + + fmt.Printf("cannot read needle header: %v\n", err) + return + } + //fmt.Printf("new entry needle size:%d rest:%d\n", n.Size, rest) + } + +}