diff --git a/unmaintained/fix_dat/fix_dat.go b/unmaintained/fix_dat/fix_dat.go index ae45f7cc8..84a2264ed 100644 --- a/unmaintained/fix_dat/fix_dat.go +++ b/unmaintained/fix_dat/fix_dat.go @@ -107,7 +107,7 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl fmt.Println("Recovered in f", r) } }() - if _, err = n.ReadNeedleBody(datFile, version, offset+int64(storage.NeedleHeaderSize), rest, false); err != nil { + if err = n.ReadNeedleBody(datFile, version, offset+int64(storage.NeedleHeaderSize), rest, false); err != nil { fmt.Printf("cannot read needle body: offset %d body %d %v\n", offset, rest, err) } }() diff --git a/weed/storage/needle_read_write.go b/weed/storage/needle_read_write.go index c7115f800..2f26147d6 100644 --- a/weed/storage/needle_read_write.go +++ b/weed/storage/needle_read_write.go @@ -226,20 +226,20 @@ func ReadNeedleHeader(r *os.File, version Version, offset int64) (n *Needle, bod //n should be a needle already read the header //the input stream will read until next file entry -func (n *Needle) ReadNeedleBody(r *os.File, version Version, offset int64, bodyLength uint32) (bytes []byte, err error) { +func (n *Needle) ReadNeedleBody(r *os.File, version Version, offset int64, bodyLength uint32) (err error) { if bodyLength <= 0 { - return + return nil } switch version { case Version1: - bytes = make([]byte, bodyLength) + bytes := make([]byte, bodyLength) if _, err = r.ReadAt(bytes, offset); err != nil { return } n.Data = bytes[:n.Size] n.Checksum = NewCRC(n.Data) case Version2: - bytes = make([]byte, bodyLength) + bytes := make([]byte, bodyLength) if _, err = r.ReadAt(bytes, offset); err != nil { return } diff --git a/weed/storage/volume.go b/weed/storage/volume.go index 43b239fd2..4c125fa4e 100644 --- a/weed/storage/volume.go +++ b/weed/storage/volume.go @@ -64,22 +64,14 @@ func readIndexEntryAtOffset(indexFile *os.File, offset int64, v Version) (bytes _, err = indexFile.ReadAt(bytes, offset) return } -func verifyNeedleIntegrity(datFile *os.File, v Version, offset int64, key uint64) error { - if n, bodyLength, err := ReadNeedleHeader(datFile, v, offset); err != nil { - return fmt.Errorf("can not read needle header: %v", err) - } else { - if n.Id != key { - return fmt.Errorf("index key %#x does not match needle's Id %#x", key, n.Id) - } else { - if bytes, err := n.ReadNeedleBody(datFile, v, offset+int64(NeedleHeaderSize), bodyLength); err != nil { - return fmt.Errorf("dat file's body reading failed: %v", err) - } else { - checksum := util.BytesToUint32(bytes[n.Size : n.Size+NeedleChecksumSize]) - if n.Checksum.Value() != checksum { - return fmt.Errorf("CRC check failed") - } - } - } +func verifyNeedleIntegrity(datFile *os.File, v Version, offset int64, key uint64, size uint32) error { + n := new(Needle) + err := n.ReadData(datFile, offset, size, v) + if err != nil { + return err + } + if n.Id != key { + return fmt.Errorf("index key %#x does not match needle's Id %#x", key, n.Id) } return nil } @@ -94,8 +86,8 @@ func volumeDataIntegrityChecking(v *Volume, indexFile *os.File) error { if lastIdxEntry, e = readIndexEntryAtOffset(indexFile, indexSize-NeedleIndexSize, v.Version()); e != nil { return fmt.Errorf("readLastIndexEntry failed: %v", e) } - key, offset, _ := idxFileEntry(lastIdxEntry) - if e = verifyNeedleIntegrity(v.dataFile, v.Version(), int64(offset)*NeedlePaddingSize, key); e != nil { + key, offset, size := idxFileEntry(lastIdxEntry) + if e = verifyNeedleIntegrity(v.dataFile, v.Version(), int64(offset)*NeedlePaddingSize, key, size); e != nil { return fmt.Errorf("verifyNeedleIntegrity failed: %v", e) } } else { @@ -397,7 +389,7 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId, } for n != nil { if readNeedleBody { - if _, err = n.ReadNeedleBody(v.dataFile, version, offset+int64(NeedleHeaderSize), rest); err != nil { + if err = n.ReadNeedleBody(v.dataFile, version, offset+int64(NeedleHeaderSize), rest); err != nil { glog.V(0).Infof("cannot read needle body: %v", err) //err = fmt.Errorf("cannot read needle body: %v", err) //return