mirror of
https://github.com/chrislusf/seaweedfs
synced 2024-07-05 08:36:55 +02:00
fix needle Append return offset to avoid uint32 overflow
This commit is contained in:
parent
7081261fca
commit
27093bc2e5
|
@ -3,6 +3,7 @@ package storage
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||||
|
@ -27,8 +28,8 @@ func (n *Needle) DiskSize(version Version) int64 {
|
||||||
return getActualSize(n.Size, version)
|
return getActualSize(n.Size, version)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Needle) Append(w *os.File, version Version) (offset Offset, size uint32, actualSize int64, err error) {
|
func (n *Needle) Append(w *os.File, version Version) (offset uint64, size uint32, actualSize int64, err error) {
|
||||||
if end, e := w.Seek(0, 2); e == nil {
|
if end, e := w.Seek(0, io.SeekEnd); e == nil {
|
||||||
defer func(w *os.File, off int64) {
|
defer func(w *os.File, off int64) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if te := w.Truncate(end); te != nil {
|
if te := w.Truncate(end); te != nil {
|
||||||
|
@ -36,7 +37,7 @@ func (n *Needle) Append(w *os.File, version Version) (offset Offset, size uint32
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}(w, end)
|
}(w, end)
|
||||||
offset = Offset(end)
|
offset = uint64(end)
|
||||||
} else {
|
} else {
|
||||||
err = fmt.Errorf("Cannot Read Current Volume Position: %v", e)
|
err = fmt.Errorf("Cannot Read Current Volume Position: %v", e)
|
||||||
return
|
return
|
||||||
|
@ -175,7 +176,7 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version
|
||||||
}
|
}
|
||||||
n.ParseNeedleHeader(bytes)
|
n.ParseNeedleHeader(bytes)
|
||||||
if n.Size != size {
|
if n.Size != size {
|
||||||
return fmt.Errorf("File Entry Not Found. Needle id %d expected size %d Memory %d", n.Id, n.Size, size)
|
return fmt.Errorf("File Entry Not Found. offset %d, Needle id %d expected size %d Memory %d", offset, n.Id, n.Size, size)
|
||||||
}
|
}
|
||||||
switch version {
|
switch version {
|
||||||
case Version1:
|
case Version1:
|
||||||
|
|
62
weed/storage/needle_read_write_test.go
Normal file
62
weed/storage/needle_read_write_test.go
Normal file
|
@ -0,0 +1,62 @@
|
||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/rand"
|
||||||
|
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAppend(t *testing.T) {
|
||||||
|
n := &Needle{
|
||||||
|
|
||||||
|
Cookie: types.Cookie(123), // Cookie Cookie `comment:"random number to mitigate brute force lookups"`
|
||||||
|
Id: types.NeedleId(123), // Id NeedleId `comment:"needle id"`
|
||||||
|
Size: 8, // Size uint32 `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`
|
||||||
|
DataSize: 4, // DataSize uint32 `comment:"Data size"` //version2
|
||||||
|
Data: []byte("abcd"), // Data []byte `comment:"The actual file data"`
|
||||||
|
Flags: 0, // Flags byte `comment:"boolean flags"` //version2
|
||||||
|
NameSize: 0, // NameSize uint8 //version2
|
||||||
|
Name: nil, // Name []byte `comment:"maximum 256 characters"` //version2
|
||||||
|
MimeSize: 0, // MimeSize uint8 //version2
|
||||||
|
Mime: nil, // Mime []byte `comment:"maximum 256 characters"` //version2
|
||||||
|
PairsSize: 0, // PairsSize uint16 //version2
|
||||||
|
Pairs: nil, // Pairs []byte `comment:"additional name value pairs, json format, maximum 6
|
||||||
|
LastModified: 123, // LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes
|
||||||
|
Ttl: nil, // Ttl *TTL
|
||||||
|
Checksum: 123, // Checksum CRC `comment:"CRC32 to check integrity"`
|
||||||
|
AppendAtNs: 123, // AppendAtNs uint64 `comment:"append timestamp in nano seconds"` //version3
|
||||||
|
Padding: nil, // Padding []byte `comment:"Aligned to 8 bytes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
tempFile, err := ioutil.TempFile("", ".dat")
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Fail TempFile. %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
uint8 : 0 to 255
|
||||||
|
uint16 : 0 to 65535
|
||||||
|
uint32 : 0 to 4294967295
|
||||||
|
uint64 : 0 to 18446744073709551615
|
||||||
|
int8 : -128 to 127
|
||||||
|
int16 : -32768 to 32767
|
||||||
|
int32 : -2147483648 to 2147483647
|
||||||
|
int64 : -9223372036854775808 to 9223372036854775807
|
||||||
|
*/
|
||||||
|
|
||||||
|
fileSize := int64(4294967295) + 10000
|
||||||
|
io.CopyN(tempFile, rand.Reader, fileSize)
|
||||||
|
defer func() {
|
||||||
|
tempFile.Close()
|
||||||
|
os.Remove(tempFile.Name())
|
||||||
|
}()
|
||||||
|
|
||||||
|
offset, _, _, _ := n.Append(tempFile, CurrentVersion)
|
||||||
|
if offset != uint64(fileSize) {
|
||||||
|
t.Errorf("Fail to Append Needle.")
|
||||||
|
}
|
||||||
|
}
|
|
@ -76,7 +76,7 @@ func (v *Volume) AppendBlob(b []byte) (offset int64, err error) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *Volume) writeNeedle(n *Needle) (offset Offset, size uint32, err error) {
|
func (v *Volume) writeNeedle(n *Needle) (offset uint64, size uint32, err error) {
|
||||||
glog.V(4).Infof("writing needle %s", NewFileIdFromNeedle(v.Id, n).String())
|
glog.V(4).Infof("writing needle %s", NewFileIdFromNeedle(v.Id, n).String())
|
||||||
if v.readOnly {
|
if v.readOnly {
|
||||||
err = fmt.Errorf("%s is read-only", v.dataFile.Name())
|
err = fmt.Errorf("%s is read-only", v.dataFile.Name())
|
||||||
|
@ -96,8 +96,8 @@ func (v *Volume) writeNeedle(n *Needle) (offset Offset, size uint32, err error)
|
||||||
}
|
}
|
||||||
|
|
||||||
nv, ok := v.nm.Get(n.Id)
|
nv, ok := v.nm.Get(n.Id)
|
||||||
if !ok || Offset(nv.Offset)*NeedlePaddingSize < offset {
|
if !ok || uint64(nv.Offset)*NeedlePaddingSize < offset {
|
||||||
if err = v.nm.Put(n.Id, offset/NeedlePaddingSize, n.Size); err != nil {
|
if err = v.nm.Put(n.Id, Offset(offset/NeedlePaddingSize), n.Size); err != nil {
|
||||||
glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err)
|
glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -124,7 +124,7 @@ func (v *Volume) deleteNeedle(n *Needle) (uint32, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return size, err
|
return size, err
|
||||||
}
|
}
|
||||||
if err = v.nm.Delete(n.Id, offset/NeedlePaddingSize); err != nil {
|
if err = v.nm.Delete(n.Id, Offset(offset/NeedlePaddingSize)); err != nil {
|
||||||
return size, err
|
return size, err
|
||||||
}
|
}
|
||||||
return size, err
|
return size, err
|
||||||
|
|
Loading…
Reference in a new issue