mirror of
https://github.com/chrislusf/seaweedfs
synced 2024-06-17 08:00:28 +02:00
avoid repeated calls to heavy-weighted viper
This commit is contained in:
parent
31f1f96038
commit
55976ae04a
|
@ -97,6 +97,11 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
|
||||||
v.SetDefault("master.volume_growth.copy_3", 3)
|
v.SetDefault("master.volume_growth.copy_3", 3)
|
||||||
v.SetDefault("master.volume_growth.copy_other", 1)
|
v.SetDefault("master.volume_growth.copy_other", 1)
|
||||||
v.SetDefault("master.volume_growth.threshold", 0.9)
|
v.SetDefault("master.volume_growth.threshold", 0.9)
|
||||||
|
topology.VolumeGrowStrategy.Copy1Count = v.GetInt("master.volume_growth.copy_1")
|
||||||
|
topology.VolumeGrowStrategy.Copy2Count = v.GetInt("master.volume_growth.copy_2")
|
||||||
|
topology.VolumeGrowStrategy.Copy3Count = v.GetInt("master.volume_growth.copy_3")
|
||||||
|
topology.VolumeGrowStrategy.CopyOtherCount = v.GetInt("master.volume_growth.copy_other")
|
||||||
|
topology.VolumeGrowStrategy.Threshold = v.GetFloat64("master.volume_growth.threshold")
|
||||||
|
|
||||||
var preallocateSize int64
|
var preallocateSize int64
|
||||||
if option.VolumePreallocate {
|
if option.VolumePreallocate {
|
||||||
|
@ -151,7 +156,7 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
|
||||||
ms.Topo.StartRefreshWritableVolumes(
|
ms.Topo.StartRefreshWritableVolumes(
|
||||||
ms.grpcDialOption,
|
ms.grpcDialOption,
|
||||||
ms.option.GarbageThreshold,
|
ms.option.GarbageThreshold,
|
||||||
v.GetFloat64("master.volume_growth.threshold"),
|
topology.VolumeGrowStrategy.Threshold,
|
||||||
ms.preallocateSize,
|
ms.preallocateSize,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -15,7 +15,6 @@ import (
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -31,6 +30,24 @@ type VolumeGrowRequest struct {
|
||||||
Count int
|
Count int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type volumeGrowthStrategy struct {
|
||||||
|
Copy1Count int
|
||||||
|
Copy2Count int
|
||||||
|
Copy3Count int
|
||||||
|
CopyOtherCount int
|
||||||
|
Threshold float64
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
VolumeGrowStrategy = volumeGrowthStrategy{
|
||||||
|
Copy1Count: 7,
|
||||||
|
Copy2Count: 6,
|
||||||
|
Copy3Count: 3,
|
||||||
|
CopyOtherCount: 1,
|
||||||
|
Threshold: 0.9,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
type VolumeGrowOption struct {
|
type VolumeGrowOption struct {
|
||||||
Collection string `json:"collection,omitempty"`
|
Collection string `json:"collection,omitempty"`
|
||||||
ReplicaPlacement *super_block.ReplicaPlacement `json:"replication,omitempty"`
|
ReplicaPlacement *super_block.ReplicaPlacement `json:"replication,omitempty"`
|
||||||
|
@ -52,11 +69,6 @@ func (o *VolumeGrowOption) String() string {
|
||||||
return string(blob)
|
return string(blob)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (o *VolumeGrowOption) Threshold() float64 {
|
|
||||||
v := util.GetViper()
|
|
||||||
return v.GetFloat64("master.volume_growth.threshold")
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewDefaultVolumeGrowth() *VolumeGrowth {
|
func NewDefaultVolumeGrowth() *VolumeGrowth {
|
||||||
return &VolumeGrowth{}
|
return &VolumeGrowth{}
|
||||||
}
|
}
|
||||||
|
@ -64,16 +76,14 @@ func NewDefaultVolumeGrowth() *VolumeGrowth {
|
||||||
// one replication type may need rp.GetCopyCount() actual volumes
|
// one replication type may need rp.GetCopyCount() actual volumes
|
||||||
// given copyCount, how many logical volumes to create
|
// given copyCount, how many logical volumes to create
|
||||||
func (vg *VolumeGrowth) findVolumeCount(copyCount int) (count int) {
|
func (vg *VolumeGrowth) findVolumeCount(copyCount int) (count int) {
|
||||||
v := util.GetViper()
|
|
||||||
switch copyCount {
|
switch copyCount {
|
||||||
case 1:
|
case 1: count = VolumeGrowStrategy.Copy1Count
|
||||||
count = v.GetInt("master.volume_growth.copy_1")
|
|
||||||
case 2:
|
case 2:
|
||||||
count = v.GetInt("master.volume_growth.copy_2")
|
count = VolumeGrowStrategy.Copy2Count
|
||||||
case 3:
|
case 3:
|
||||||
count = v.GetInt("master.volume_growth.copy_3")
|
count = VolumeGrowStrategy.Copy3Count
|
||||||
default:
|
default:
|
||||||
count = v.GetInt("master.volume_growth.copy_other")
|
count = VolumeGrowStrategy.CopyOtherCount
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,7 +4,6 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/sequence"
|
"github.com/seaweedfs/seaweedfs/weed/sequence"
|
||||||
|
@ -420,8 +419,7 @@ func TestPickForWrite(t *testing.T) {
|
||||||
Rack: "",
|
Rack: "",
|
||||||
DataNode: "",
|
DataNode: "",
|
||||||
}
|
}
|
||||||
v := util.GetViper()
|
VolumeGrowStrategy.Threshold = 0.9
|
||||||
v.Set("master.volume_growth.threshold", 0.9)
|
|
||||||
for _, rpStr := range []string{"001", "010", "100"} {
|
for _, rpStr := range []string{"001", "010", "100"} {
|
||||||
rp, _ := super_block.NewReplicaPlacementFromString(rpStr)
|
rp, _ := super_block.NewReplicaPlacementFromString(rpStr)
|
||||||
vl := topo.GetVolumeLayout("test", rp, needle.EMPTY_TTL, types.HardDriveType)
|
vl := topo.GetVolumeLayout("test", rp, needle.EMPTY_TTL, types.HardDriveType)
|
||||||
|
|
|
@ -303,7 +303,7 @@ func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (vi
|
||||||
// check whether picked file is close to full
|
// check whether picked file is close to full
|
||||||
dn := locationList.Head()
|
dn := locationList.Head()
|
||||||
info, _ := dn.GetVolumesById(vid)
|
info, _ := dn.GetVolumesById(vid)
|
||||||
if float64(info.Size) > float64(vl.volumeSizeLimit)*option.Threshold() {
|
if float64(info.Size) > float64(vl.volumeSizeLimit)*VolumeGrowStrategy.Threshold {
|
||||||
shouldGrow = true
|
shouldGrow = true
|
||||||
}
|
}
|
||||||
return vid, count, locationList.Copy(), shouldGrow, nil
|
return vid, count, locationList.Copy(), shouldGrow, nil
|
||||||
|
@ -334,7 +334,7 @@ func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (vi
|
||||||
vid, locationList = writableVolumeId, volumeLocationList.Copy()
|
vid, locationList = writableVolumeId, volumeLocationList.Copy()
|
||||||
// check whether picked file is close to full
|
// check whether picked file is close to full
|
||||||
info, _ := dn.GetVolumesById(writableVolumeId)
|
info, _ := dn.GetVolumesById(writableVolumeId)
|
||||||
if float64(info.Size) > float64(vl.volumeSizeLimit)*option.Threshold() {
|
if float64(info.Size) > float64(vl.volumeSizeLimit)*VolumeGrowStrategy.Threshold {
|
||||||
shouldGrow = true
|
shouldGrow = true
|
||||||
}
|
}
|
||||||
counter = count
|
counter = count
|
||||||
|
@ -370,7 +370,6 @@ func (vl *VolumeLayout) GetActiveVolumeCount(option *VolumeGrowOption) (total, a
|
||||||
return len(vl.writables), len(vl.writables), len(vl.crowded)
|
return len(vl.writables), len(vl.writables), len(vl.crowded)
|
||||||
}
|
}
|
||||||
total = len(vl.writables)
|
total = len(vl.writables)
|
||||||
threshold := option.Threshold()
|
|
||||||
for _, v := range vl.writables {
|
for _, v := range vl.writables {
|
||||||
for _, dn := range vl.vid2location[v].list {
|
for _, dn := range vl.vid2location[v].list {
|
||||||
if dn.GetDataCenter().Id() == NodeId(option.DataCenter) {
|
if dn.GetDataCenter().Id() == NodeId(option.DataCenter) {
|
||||||
|
@ -382,7 +381,7 @@ func (vl *VolumeLayout) GetActiveVolumeCount(option *VolumeGrowOption) (total, a
|
||||||
}
|
}
|
||||||
active++
|
active++
|
||||||
info, _ := dn.GetVolumesById(v)
|
info, _ := dn.GetVolumesById(v)
|
||||||
if float64(info.Size) > float64(vl.volumeSizeLimit)*threshold {
|
if float64(info.Size) > float64(vl.volumeSizeLimit)* VolumeGrowStrategy.Threshold{
|
||||||
crowded++
|
crowded++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue