mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/chrislusf/seaweedfs synced 2024-05-03 18:10:12 +02:00
Code Issues Projects Releases Wiki Activity

avoid repeated calls to heavy-weighted viper

Browse source
This commit is contained in:
chrislu 2024-04-18 09:09:45 -07:00
parent 31f1f96038
commit 55976ae04a
4 changed files with 32 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
weed/server/master_server.go
View file

@ -97,6 +97,11 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
v.SetDefault("master.volume_growth.copy_3", 3)
v.SetDefault("master.volume_growth.copy_other", 1)
v.SetDefault("master.volume_growth.threshold", 0.9)
topology.VolumeGrowStrategy.Copy1Count = v.GetInt("master.volume_growth.copy_1")
topology.VolumeGrowStrategy.Copy2Count = v.GetInt("master.volume_growth.copy_2")
topology.VolumeGrowStrategy.Copy3Count = v.GetInt("master.volume_growth.copy_3")
topology.VolumeGrowStrategy.CopyOtherCount = v.GetInt("master.volume_growth.copy_other")
topology.VolumeGrowStrategy.Threshold = v.GetFloat64("master.volume_growth.threshold")
var preallocateSize int64
if option.VolumePreallocate {
@ -151,7 +156,7 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
ms.Topo.StartRefreshWritableVolumes(
ms.grpcDialOption,
ms.option.GarbageThreshold,
v.GetFloat64("master.volume_growth.threshold"),
topology.VolumeGrowStrategy.Threshold,
ms.preallocateSize,
)

34
weed/topology/volume_growth.go
View file

@ -15,7 +15,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/util"
)
/*
@ -31,6 +30,24 @@ type VolumeGrowRequest struct {
Count int
}
type volumeGrowthStrategy struct {
Copy1Count int
Copy2Count int
Copy3Count int
CopyOtherCount int
Threshold float64
}
var (
VolumeGrowStrategy = volumeGrowthStrategy{
Copy1Count: 7,
Copy2Count: 6,
Copy3Count: 3,
CopyOtherCount: 1,
Threshold: 0.9,
}
)
type VolumeGrowOption struct {
Collection string `json:"collection,omitempty"`
ReplicaPlacement *super_block.ReplicaPlacement `json:"replication,omitempty"`
@ -52,11 +69,6 @@ func (o *VolumeGrowOption) String() string {
return string(blob)
}
func (o *VolumeGrowOption) Threshold() float64 {
v := util.GetViper()
return v.GetFloat64("master.volume_growth.threshold")
}
func NewDefaultVolumeGrowth() *VolumeGrowth {
return &VolumeGrowth{}
}
@ -64,16 +76,14 @@ func NewDefaultVolumeGrowth() *VolumeGrowth {
// one replication type may need rp.GetCopyCount() actual volumes
// given copyCount, how many logical volumes to create
func (vg *VolumeGrowth) findVolumeCount(copyCount int) (count int) {
v := util.GetViper()
switch copyCount {
case 1:
count = v.GetInt("master.volume_growth.copy_1")
case 1: count = VolumeGrowStrategy.Copy1Count
case 2:
count = v.GetInt("master.volume_growth.copy_2")
count = VolumeGrowStrategy.Copy2Count
case 3:
count = v.GetInt("master.volume_growth.copy_3")
count = VolumeGrowStrategy.Copy3Count
default:
count = v.GetInt("master.volume_growth.copy_other")
count = VolumeGrowStrategy.CopyOtherCount
}
return
}

4
weed/topology/volume_growth_test.go
View file

@ -4,7 +4,6 @@ import (
"encoding/json"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/util"
"testing"
"github.com/seaweedfs/seaweedfs/weed/sequence"
@ -420,8 +419,7 @@ func TestPickForWrite(t *testing.T) {
Rack: "",
DataNode: "",
}
v := util.GetViper()
v.Set("master.volume_growth.threshold", 0.9)
VolumeGrowStrategy.Threshold = 0.9
for _, rpStr := range []string{"001", "010", "100"} {
rp, _ := super_block.NewReplicaPlacementFromString(rpStr)
vl := topo.GetVolumeLayout("test", rp, needle.EMPTY_TTL, types.HardDriveType)

7
weed/topology/volume_layout.go
View file

@ -303,7 +303,7 @@ func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (vi
// check whether picked file is close to full
dn := locationList.Head()
info, _ := dn.GetVolumesById(vid)
if float64(info.Size) > float64(vl.volumeSizeLimit)*option.Threshold() {
if float64(info.Size) > float64(vl.volumeSizeLimit)*VolumeGrowStrategy.Threshold {
shouldGrow = true
}
return vid, count, locationList.Copy(), shouldGrow, nil
@ -334,7 +334,7 @@ func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (vi
vid, locationList = writableVolumeId, volumeLocationList.Copy()
// check whether picked file is close to full
info, _ := dn.GetVolumesById(writableVolumeId)
if float64(info.Size) > float64(vl.volumeSizeLimit)*option.Threshold() {
if float64(info.Size) > float64(vl.volumeSizeLimit)*VolumeGrowStrategy.Threshold {
shouldGrow = true
}
counter = count
@ -370,7 +370,6 @@ func (vl *VolumeLayout) GetActiveVolumeCount(option *VolumeGrowOption) (total, a
return len(vl.writables), len(vl.writables), len(vl.crowded)
}
total = len(vl.writables)
threshold := option.Threshold()
for _, v := range vl.writables {
for _, dn := range vl.vid2location[v].list {
if dn.GetDataCenter().Id() == NodeId(option.DataCenter) {
@ -382,7 +381,7 @@ func (vl *VolumeLayout) GetActiveVolumeCount(option *VolumeGrowOption) (total, a
}
active++
info, _ := dn.GetVolumesById(v)
if float64(info.Size) > float64(vl.volumeSizeLimit)*threshold {
if float64(info.Size) > float64(vl.volumeSizeLimit)* VolumeGrowStrategy.Threshold{
crowded++
}
}