mirror of
https://github.com/chrislusf/seaweedfs
synced 2024-09-12 12:00:45 +02:00
Revert "Merge pull request #1479 from LIBA-S/fix_oversized"
This reverts commitbd11f0b3e4
, reversing changes made toec5b9f1e91
.
This commit is contained in:
parent
4a1fe4b8e2
commit
3e52329cee
|
@ -172,10 +172,10 @@ func vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeLayout *VolumeL
|
|||
for vid, locationList := range tmpMap {
|
||||
|
||||
volumeLayout.accessLock.RLock()
|
||||
isReadOnly := volumeLayout.readonlyVolumes.IsTrue(vid)
|
||||
isReadOnly, hasValue := volumeLayout.readonlyVolumes[vid]
|
||||
volumeLayout.accessLock.RUnlock()
|
||||
|
||||
if isReadOnly {
|
||||
if hasValue && isReadOnly {
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
|
@ -13,100 +13,14 @@ import (
|
|||
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
|
||||
)
|
||||
|
||||
type copyState int
|
||||
|
||||
const (
|
||||
noCopies copyState = 0 + iota
|
||||
insufficientCopies
|
||||
enoughCopies
|
||||
)
|
||||
|
||||
type volumeState string
|
||||
|
||||
const (
|
||||
readOnlyState volumeState = "ReadOnly"
|
||||
oversizedState = "Oversized"
|
||||
)
|
||||
|
||||
type stateIndicator func(copyState) bool
|
||||
|
||||
func ExistCopies() stateIndicator {
|
||||
return func(state copyState) bool { return state != noCopies }
|
||||
}
|
||||
|
||||
func NoCopies() stateIndicator {
|
||||
return func(state copyState) bool { return state == noCopies }
|
||||
}
|
||||
|
||||
type volumesBinaryState struct {
|
||||
copyCount int
|
||||
name volumeState // the name for volume state (eg. "Readonly", "Oversized")
|
||||
indicator stateIndicator // indicate whether the volumes should be marked as `name`
|
||||
copyMap map[needle.VolumeId]*VolumeLocationList
|
||||
}
|
||||
|
||||
func NewVolumesBinaryState(name volumeState, copyCount int, indicator stateIndicator) *volumesBinaryState {
|
||||
return &volumesBinaryState{
|
||||
copyCount: copyCount,
|
||||
name: name,
|
||||
indicator: indicator,
|
||||
copyMap: make(map[needle.VolumeId]*VolumeLocationList),
|
||||
}
|
||||
}
|
||||
|
||||
func (v *volumesBinaryState) Dump() (res []uint32) {
|
||||
for vid, list := range v.copyMap {
|
||||
if v.indicator(v.copyState(list)) {
|
||||
res = append(res, uint32(vid))
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (v *volumesBinaryState) IsTrue(vid needle.VolumeId) bool {
|
||||
list, _ := v.copyMap[vid]
|
||||
return v.indicator(v.copyState(list))
|
||||
}
|
||||
|
||||
func (v *volumesBinaryState) Add(vid needle.VolumeId, dn *DataNode) {
|
||||
list, _ := v.copyMap[vid]
|
||||
if list != nil {
|
||||
list.Set(dn)
|
||||
return
|
||||
}
|
||||
list = NewVolumeLocationList()
|
||||
list.Set(dn)
|
||||
v.copyMap[vid] = list
|
||||
}
|
||||
|
||||
func (v *volumesBinaryState) Remove(vid needle.VolumeId, dn *DataNode) {
|
||||
list, _ := v.copyMap[vid]
|
||||
if list != nil {
|
||||
list.Remove(dn)
|
||||
if list.Length() == 0 {
|
||||
delete(v.copyMap, vid)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (v *volumesBinaryState) copyState(list *VolumeLocationList) copyState {
|
||||
if list == nil {
|
||||
return noCopies
|
||||
}
|
||||
if list.Length() < v.copyCount {
|
||||
return insufficientCopies
|
||||
}
|
||||
return enoughCopies
|
||||
}
|
||||
|
||||
// mapping from volume to its locations, inverted from server to volume
|
||||
type VolumeLayout struct {
|
||||
rp *super_block.ReplicaPlacement
|
||||
ttl *needle.TTL
|
||||
vid2location map[needle.VolumeId]*VolumeLocationList
|
||||
writables []needle.VolumeId // transient array of writable volume id
|
||||
readonlyVolumes *volumesBinaryState // readonly volumes
|
||||
oversizedVolumes *volumesBinaryState // oversized volumes
|
||||
writables []needle.VolumeId // transient array of writable volume id
|
||||
readonlyVolumes map[needle.VolumeId]bool // transient set of readonly volumes
|
||||
oversizedVolumes map[needle.VolumeId]bool // set of oversized volumes
|
||||
volumeSizeLimit uint64
|
||||
replicationAsMin bool
|
||||
accessLock sync.RWMutex
|
||||
|
@ -124,8 +38,8 @@ func NewVolumeLayout(rp *super_block.ReplicaPlacement, ttl *needle.TTL, volumeSi
|
|||
ttl: ttl,
|
||||
vid2location: make(map[needle.VolumeId]*VolumeLocationList),
|
||||
writables: *new([]needle.VolumeId),
|
||||
readonlyVolumes: NewVolumesBinaryState(readOnlyState, rp.GetCopyCount(), ExistCopies()),
|
||||
oversizedVolumes: NewVolumesBinaryState(oversizedState, rp.GetCopyCount(), ExistCopies()),
|
||||
readonlyVolumes: make(map[needle.VolumeId]bool),
|
||||
oversizedVolumes: make(map[needle.VolumeId]bool),
|
||||
volumeSizeLimit: volumeSizeLimit,
|
||||
replicationAsMin: replicationAsMin,
|
||||
}
|
||||
|
@ -140,7 +54,7 @@ func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) {
|
|||
defer vl.accessLock.Unlock()
|
||||
|
||||
defer vl.ensureCorrectWritables(v)
|
||||
defer vl.rememberOversizedVolume(v, dn)
|
||||
defer vl.rememberOversizedVolume(v)
|
||||
|
||||
if _, ok := vl.vid2location[v.Id]; !ok {
|
||||
vl.vid2location[v.Id] = NewVolumeLocationList()
|
||||
|
@ -152,26 +66,24 @@ func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) {
|
|||
if vInfo.ReadOnly {
|
||||
glog.V(1).Infof("vid %d removed from writable", v.Id)
|
||||
vl.removeFromWritable(v.Id)
|
||||
vl.readonlyVolumes.Add(v.Id, dn)
|
||||
vl.readonlyVolumes[v.Id] = true
|
||||
return
|
||||
} else {
|
||||
vl.readonlyVolumes.Remove(v.Id, dn)
|
||||
delete(vl.readonlyVolumes, v.Id)
|
||||
}
|
||||
} else {
|
||||
glog.V(1).Infof("vid %d removed from writable", v.Id)
|
||||
vl.removeFromWritable(v.Id)
|
||||
vl.readonlyVolumes.Remove(v.Id, dn)
|
||||
delete(vl.readonlyVolumes, v.Id)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (vl *VolumeLayout) rememberOversizedVolume(v *storage.VolumeInfo, dn *DataNode) {
|
||||
func (vl *VolumeLayout) rememberOversizedVolume(v *storage.VolumeInfo) {
|
||||
if vl.isOversized(v) {
|
||||
vl.oversizedVolumes.Add(v.Id, dn)
|
||||
} else {
|
||||
vl.oversizedVolumes.Remove(v.Id, dn)
|
||||
vl.oversizedVolumes[v.Id] = true
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -187,8 +99,6 @@ func (vl *VolumeLayout) UnRegisterVolume(v *storage.VolumeInfo, dn *DataNode) {
|
|||
|
||||
if location.Remove(dn) {
|
||||
|
||||
vl.readonlyVolumes.Remove(v.Id, dn)
|
||||
vl.oversizedVolumes.Remove(v.Id, dn)
|
||||
vl.ensureCorrectWritables(v)
|
||||
|
||||
if location.Length() == 0 {
|
||||
|
@ -200,7 +110,7 @@ func (vl *VolumeLayout) UnRegisterVolume(v *storage.VolumeInfo, dn *DataNode) {
|
|||
|
||||
func (vl *VolumeLayout) ensureCorrectWritables(v *storage.VolumeInfo) {
|
||||
if vl.enoughCopies(v.Id) && vl.isWritable(v) {
|
||||
if vl.oversizedVolumes.IsTrue(v.Id) {
|
||||
if _, ok := vl.oversizedVolumes[v.Id]; !ok {
|
||||
vl.setVolumeWritable(v.Id)
|
||||
}
|
||||
} else {
|
||||
|
@ -405,7 +315,7 @@ func (vl *VolumeLayout) Stats() *VolumeLayoutStats {
|
|||
size, fileCount := vll.Stats(vid, freshThreshold)
|
||||
ret.FileCount += uint64(fileCount)
|
||||
ret.UsedSize += size
|
||||
if vl.readonlyVolumes.IsTrue(vid) {
|
||||
if vl.readonlyVolumes[vid] {
|
||||
ret.TotalSize += size
|
||||
} else {
|
||||
ret.TotalSize += vl.volumeSizeLimit
|
||||
|
|
|
@ -1,113 +0,0 @@
|
|||
package topology
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
)
|
||||
|
||||
func TestVolumesBinaryState(t *testing.T) {
|
||||
vids := []needle.VolumeId{
|
||||
needle.VolumeId(1),
|
||||
needle.VolumeId(2),
|
||||
needle.VolumeId(3),
|
||||
needle.VolumeId(4),
|
||||
needle.VolumeId(5),
|
||||
}
|
||||
|
||||
dns := []*DataNode{
|
||||
&DataNode{
|
||||
Ip: "127.0.0.1",
|
||||
Port: 8081,
|
||||
},
|
||||
&DataNode{
|
||||
Ip: "127.0.0.1",
|
||||
Port: 8082,
|
||||
},
|
||||
&DataNode{
|
||||
Ip: "127.0.0.1",
|
||||
Port: 8083,
|
||||
},
|
||||
}
|
||||
|
||||
state_exist := NewVolumesBinaryState(readOnlyState, 3, ExistCopies())
|
||||
state_exist.Add(vids[0], dns[0])
|
||||
state_exist.Add(vids[0], dns[1])
|
||||
state_exist.Add(vids[1], dns[2])
|
||||
state_exist.Add(vids[2], dns[1])
|
||||
state_exist.Add(vids[4], dns[1])
|
||||
state_exist.Add(vids[4], dns[2])
|
||||
|
||||
state_no := NewVolumesBinaryState(readOnlyState, 3, NoCopies())
|
||||
state_no.Add(vids[0], dns[0])
|
||||
state_no.Add(vids[0], dns[1])
|
||||
state_no.Add(vids[3], dns[1])
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
state *volumesBinaryState
|
||||
expectResult []bool
|
||||
update func()
|
||||
expectResultAfterUpdate []bool
|
||||
}{
|
||||
{
|
||||
name: "mark true when exist copies",
|
||||
state: state_exist,
|
||||
expectResult: []bool{true, true, true, false, true},
|
||||
update: func() {
|
||||
state_exist.Remove(vids[0], dns[2])
|
||||
state_exist.Remove(vids[1], dns[2])
|
||||
state_exist.Remove(vids[3], dns[2])
|
||||
state_exist.Remove(vids[4], dns[1])
|
||||
state_exist.Remove(vids[4], dns[2])
|
||||
},
|
||||
expectResultAfterUpdate: []bool{true, false, true, false, false},
|
||||
},
|
||||
{
|
||||
name: "mark true when inexist copies",
|
||||
state: state_no,
|
||||
expectResult: []bool{false, true, true, false, true},
|
||||
update: func() {
|
||||
state_no.Remove(vids[0], dns[2])
|
||||
state_no.Remove(vids[1], dns[2])
|
||||
state_no.Add(vids[2], dns[1])
|
||||
state_no.Remove(vids[3], dns[1])
|
||||
state_no.Remove(vids[4], dns[2])
|
||||
},
|
||||
expectResultAfterUpdate: []bool{false, true, false, true, true},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var result []bool
|
||||
for index, _ := range vids {
|
||||
result = append(result, test.state.IsTrue(vids[index]))
|
||||
}
|
||||
if len(result) != len(test.expectResult) {
|
||||
t.Fatalf("len(result) != len(expectResult), got %d, expected %d\n",
|
||||
len(result), len(test.expectResult))
|
||||
}
|
||||
for index, val := range result {
|
||||
if val != test.expectResult[index] {
|
||||
t.Fatalf("result not matched, index %d, got %v, expect %v\n",
|
||||
index, val, test.expectResult[index])
|
||||
}
|
||||
}
|
||||
test.update()
|
||||
var updateResult []bool
|
||||
for index, _ := range vids {
|
||||
updateResult = append(updateResult, test.state.IsTrue(vids[index]))
|
||||
}
|
||||
if len(updateResult) != len(test.expectResultAfterUpdate) {
|
||||
t.Fatalf("len(updateResult) != len(expectResultAfterUpdate), got %d, expected %d\n",
|
||||
len(updateResult), len(test.expectResultAfterUpdate))
|
||||
}
|
||||
for index, val := range updateResult {
|
||||
if val != test.expectResultAfterUpdate[index] {
|
||||
t.Fatalf("update result not matched, index %d, got %v, expect %v\n",
|
||||
index, val, test.expectResultAfterUpdate[index])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue