From 3ece066700169013497390aaca8d1d86bf7ecb2d Mon Sep 17 00:00:00 2001 From: chrislusf Date: Mon, 6 Apr 2015 14:17:36 -0700 Subject: [PATCH] change count to uint64 to fix #109 fix https://github.com/chrislusf/weed-fs/issues/109 --- go/compress/delta_binary_pack32.go | 32 +++++++++++++++++++ go/operation/assign_file_id.go | 6 ++-- go/operation/submit.go | 2 +- go/sequence/memory_sequencer.go | 2 +- go/sequence/sequence.go | 2 +- go/topology/topology.go | 2 +- go/topology/volume_layout.go | 2 +- go/weed/weed_server/master_server_handlers.go | 4 +-- 8 files changed, 42 insertions(+), 10 deletions(-) create mode 100644 go/compress/delta_binary_pack32.go diff --git a/go/compress/delta_binary_pack32.go b/go/compress/delta_binary_pack32.go new file mode 100644 index 000000000..42ae8d42d --- /dev/null +++ b/go/compress/delta_binary_pack32.go @@ -0,0 +1,32 @@ +package compress + +import ( + "github.com/reducedb/encoding/cursor" + "github.com/reducedb/encoding/delta/bp32" +) + +// Compress compresses in[]int32 to out[]int32 +func Compress32(in []int32) (out []int32, err error) { + out = make([]int32, len(in)*2) + inpos := cursor.New() + outpos := cursor.New() + + if err = bp32.New().Compress(in, inpos, len(in), out, outpos); err != nil { + return nil, err + } + + return out[:outpos.Get()], nil +} + +// Uncompress uncompresses in[]int32 to out[]int32 +func Uncompress32(in []int32, buffer []int32) (out []int32, err error) { + out = buffer + inpos := cursor.New() + outpos := cursor.New() + + if err = bp32.New().Uncompress(in, inpos, len(in), out, outpos); err != nil { + return nil, err + } + + return out[:outpos.Get()], nil +} diff --git a/go/operation/assign_file_id.go b/go/operation/assign_file_id.go index 31766b053..02a0de976 100644 --- a/go/operation/assign_file_id.go +++ b/go/operation/assign_file_id.go @@ -15,13 +15,13 @@ type AssignResult struct { Fid string `json:"fid,omitempty"` Url string `json:"url,omitempty"` PublicUrl string `json:"publicUrl,omitempty"` - Count int `json:"count,omitempty"` + Count uint64 `json:"count,omitempty"` Error string `json:"error,omitempty"` } -func Assign(server string, count int, replication string, collection string, ttl string) (*AssignResult, error) { +func Assign(server string, count uint64, replication string, collection string, ttl string) (*AssignResult, error) { values := make(url.Values) - values.Add("count", strconv.Itoa(count)) + values.Add("count", strconv.FormatUint(count, 10)) if replication != "" { values.Add("replication", replication) } diff --git a/go/operation/submit.go b/go/operation/submit.go index 2519d29f8..ca151dfb9 100644 --- a/go/operation/submit.go +++ b/go/operation/submit.go @@ -43,7 +43,7 @@ func SubmitFiles(master string, files []FilePart, for index, file := range files { results[index].FileName = file.FileName } - ret, err := Assign(master, len(files), replication, collection, ttl) + ret, err := Assign(master, uint64(len(files)), replication, collection, ttl) if err != nil { for index, _ := range files { results[index].Error = err.Error() diff --git a/go/sequence/memory_sequencer.go b/go/sequence/memory_sequencer.go index c7ee1ae8f..d727dc723 100644 --- a/go/sequence/memory_sequencer.go +++ b/go/sequence/memory_sequencer.go @@ -15,7 +15,7 @@ func NewMemorySequencer() (m *MemorySequencer) { return } -func (m *MemorySequencer) NextFileId(count int) (uint64, int) { +func (m *MemorySequencer) NextFileId(count uint64) (uint64, uint64) { m.sequenceLock.Lock() defer m.sequenceLock.Unlock() ret := m.counter diff --git a/go/sequence/sequence.go b/go/sequence/sequence.go index 1aa167b6b..fbdc3b8ef 100644 --- a/go/sequence/sequence.go +++ b/go/sequence/sequence.go @@ -1,7 +1,7 @@ package sequence type Sequencer interface { - NextFileId(count int) (uint64, int) + NextFileId(count uint64) (uint64, uint64) SetMax(uint64) Peek() uint64 } diff --git a/go/topology/topology.go b/go/topology/topology.go index d96847ed2..f12261081 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -115,7 +115,7 @@ func (t *Topology) HasWritableVolume(option *VolumeGrowOption) bool { return vl.GetActiveVolumeCount(option) > 0 } -func (t *Topology) PickForWrite(count int, option *VolumeGrowOption) (string, int, *DataNode, error) { +func (t *Topology) PickForWrite(count uint64, option *VolumeGrowOption) (string, uint64, *DataNode, error) { vid, count, datanodes, err := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl).PickForWrite(count, option) if err != nil || datanodes.Length() == 0 { return "", 0, nil, errors.New("No writable volumes available!") diff --git a/go/topology/volume_layout.go b/go/topology/volume_layout.go index e5667e322..14e112141 100644 --- a/go/topology/volume_layout.go +++ b/go/topology/volume_layout.go @@ -87,7 +87,7 @@ func (vl *VolumeLayout) ListVolumeServers() (nodes []*DataNode) { return } -func (vl *VolumeLayout) PickForWrite(count int, option *VolumeGrowOption) (*storage.VolumeId, int, *VolumeLocationList, error) { +func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (*storage.VolumeId, uint64, *VolumeLocationList, error) { len_writers := len(vl.writables) if len_writers <= 0 { glog.V(0).Infoln("No more writable volumes!") diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index f8e81152d..097aea503 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -69,8 +69,8 @@ func (ms *MasterServer) volumeLookupHandler(w http.ResponseWriter, r *http.Reque func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) { stats.AssignRequest() - requestedCount, e := strconv.Atoi(r.FormValue("count")) - if e != nil { + requestedCount, e := strconv.ParseUint(r.FormValue("count"), 10, 64) + if e != nil || requestedCount == 0 { requestedCount = 1 }