mirror of
https://github.com/chrislusf/seaweedfs
synced 2024-09-12 12:00:45 +02:00
356 lines
9.4 KiB
Go
356 lines
9.4 KiB
Go
package cluster
|
|
|
|
import (
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"math"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
MasterType = "master"
|
|
VolumeServerType = "volumeServer"
|
|
FilerType = "filer"
|
|
BrokerType = "broker"
|
|
)
|
|
|
|
type FilerGroupName string
|
|
type DataCenter string
|
|
type Rack string
|
|
|
|
type Leaders struct {
|
|
leaders [3]pb.ServerAddress
|
|
}
|
|
type ClusterNode struct {
|
|
Address pb.ServerAddress
|
|
Version string
|
|
counter int
|
|
CreatedTs time.Time
|
|
DataCenter DataCenter
|
|
Rack Rack
|
|
}
|
|
type GroupMembers struct {
|
|
members map[pb.ServerAddress]*ClusterNode
|
|
leaders *Leaders
|
|
}
|
|
type ClusterNodeGroups struct {
|
|
groupMembers map[FilerGroupName]*GroupMembers
|
|
sync.RWMutex
|
|
}
|
|
type Cluster struct {
|
|
filerGroups *ClusterNodeGroups
|
|
brokerGroups *ClusterNodeGroups
|
|
}
|
|
|
|
func newClusterNodeGroups() *ClusterNodeGroups {
|
|
return &ClusterNodeGroups{
|
|
groupMembers: map[FilerGroupName]*GroupMembers{},
|
|
}
|
|
}
|
|
func (g *ClusterNodeGroups) getGroupMembers(filerGroup FilerGroupName, createIfNotFound bool) *GroupMembers {
|
|
members, found := g.groupMembers[filerGroup]
|
|
if !found && createIfNotFound {
|
|
members = &GroupMembers{
|
|
members: make(map[pb.ServerAddress]*ClusterNode),
|
|
leaders: &Leaders{},
|
|
}
|
|
g.groupMembers[filerGroup] = members
|
|
}
|
|
return members
|
|
}
|
|
|
|
func (m *GroupMembers) addMember(dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) *ClusterNode {
|
|
if existingNode, found := m.members[address]; found {
|
|
existingNode.counter++
|
|
return nil
|
|
}
|
|
t := &ClusterNode{
|
|
Address: address,
|
|
Version: version,
|
|
counter: 1,
|
|
CreatedTs: time.Now(),
|
|
DataCenter: dataCenter,
|
|
Rack: rack,
|
|
}
|
|
m.members[address] = t
|
|
return t
|
|
}
|
|
func (m *GroupMembers) removeMember(address pb.ServerAddress) bool {
|
|
if existingNode, found := m.members[address]; !found {
|
|
return false
|
|
} else {
|
|
existingNode.counter--
|
|
if existingNode.counter <= 0 {
|
|
delete(m.members, address)
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (g *ClusterNodeGroups) AddClusterNode(filerGroup FilerGroupName, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
|
|
g.Lock()
|
|
defer g.Unlock()
|
|
m := g.getGroupMembers(filerGroup, true)
|
|
if t := m.addMember(dataCenter, rack, address, version); t != nil {
|
|
return ensureGroupLeaders(m, true, filerGroup, nodeType, address)
|
|
}
|
|
return nil
|
|
}
|
|
func (g *ClusterNodeGroups) RemoveClusterNode(filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
|
|
g.Lock()
|
|
defer g.Unlock()
|
|
m := g.getGroupMembers(filerGroup, false)
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
if m.removeMember(address) {
|
|
return ensureGroupLeaders(m, false, filerGroup, nodeType, address)
|
|
}
|
|
return nil
|
|
}
|
|
func (g *ClusterNodeGroups) ListClusterNode(filerGroup FilerGroupName) (nodes []*ClusterNode) {
|
|
g.Lock()
|
|
defer g.Unlock()
|
|
m := g.getGroupMembers(filerGroup, false)
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
for _, node := range m.members {
|
|
nodes = append(nodes, node)
|
|
}
|
|
return
|
|
}
|
|
func (g *ClusterNodeGroups) IsOneLeader(filerGroup FilerGroupName, address pb.ServerAddress) bool {
|
|
g.Lock()
|
|
defer g.Unlock()
|
|
m := g.getGroupMembers(filerGroup, false)
|
|
if m == nil {
|
|
return false
|
|
}
|
|
return m.leaders.isOneLeader(address)
|
|
}
|
|
func (g *ClusterNodeGroups) ListClusterNodeLeaders(filerGroup FilerGroupName) (nodes []pb.ServerAddress) {
|
|
g.Lock()
|
|
defer g.Unlock()
|
|
m := g.getGroupMembers(filerGroup, false)
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
return m.leaders.GetLeaders()
|
|
}
|
|
|
|
func NewCluster() *Cluster {
|
|
return &Cluster{
|
|
filerGroups: newClusterNodeGroups(),
|
|
brokerGroups: newClusterNodeGroups(),
|
|
}
|
|
}
|
|
|
|
func (cluster *Cluster) getGroupMembers(filerGroup FilerGroupName, nodeType string, createIfNotFound bool) *GroupMembers {
|
|
switch nodeType {
|
|
case FilerType:
|
|
return cluster.filerGroups.getGroupMembers(filerGroup, createIfNotFound)
|
|
case BrokerType:
|
|
return cluster.brokerGroups.getGroupMembers(filerGroup, createIfNotFound)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (cluster *Cluster) AddClusterNode(ns, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
|
|
filerGroup := FilerGroupName(ns)
|
|
switch nodeType {
|
|
case FilerType:
|
|
return cluster.filerGroups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
|
|
case BrokerType:
|
|
return cluster.brokerGroups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
|
|
case MasterType:
|
|
return []*master_pb.KeepConnectedResponse{
|
|
{
|
|
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
|
NodeType: nodeType,
|
|
Address: string(address),
|
|
IsAdd: true,
|
|
},
|
|
},
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
|
|
filerGroup := FilerGroupName(ns)
|
|
switch nodeType {
|
|
case FilerType:
|
|
return cluster.filerGroups.RemoveClusterNode(filerGroup, nodeType, address)
|
|
case BrokerType:
|
|
return cluster.brokerGroups.RemoveClusterNode(filerGroup, nodeType, address)
|
|
case MasterType:
|
|
return []*master_pb.KeepConnectedResponse{
|
|
{
|
|
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
|
NodeType: nodeType,
|
|
Address: string(address),
|
|
IsAdd: false,
|
|
},
|
|
},
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (cluster *Cluster) ListClusterNode(filerGroup FilerGroupName, nodeType string) (nodes []*ClusterNode) {
|
|
switch nodeType {
|
|
case FilerType:
|
|
return cluster.filerGroups.ListClusterNode(filerGroup)
|
|
case BrokerType:
|
|
return cluster.brokerGroups.ListClusterNode(filerGroup)
|
|
case MasterType:
|
|
}
|
|
return
|
|
}
|
|
|
|
func (cluster *Cluster) ListClusterNodeLeaders(filerGroup FilerGroupName, nodeType string) (nodes []pb.ServerAddress) {
|
|
switch nodeType {
|
|
case FilerType:
|
|
return cluster.filerGroups.ListClusterNodeLeaders(filerGroup)
|
|
case BrokerType:
|
|
return cluster.brokerGroups.ListClusterNodeLeaders(filerGroup)
|
|
case MasterType:
|
|
}
|
|
return
|
|
}
|
|
|
|
func (cluster *Cluster) IsOneLeader(filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) bool {
|
|
switch nodeType {
|
|
case FilerType:
|
|
return cluster.filerGroups.IsOneLeader(filerGroup, address)
|
|
case BrokerType:
|
|
return cluster.brokerGroups.IsOneLeader(filerGroup, address)
|
|
case MasterType:
|
|
}
|
|
return false
|
|
}
|
|
|
|
func ensureGroupLeaders(m *GroupMembers, isAdd bool, filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
|
|
if isAdd {
|
|
if m.leaders.addLeaderIfVacant(address) {
|
|
// has added the address as one leader
|
|
result = append(result, &master_pb.KeepConnectedResponse{
|
|
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
|
FilerGroup: string(filerGroup),
|
|
NodeType: nodeType,
|
|
Address: string(address),
|
|
IsLeader: true,
|
|
IsAdd: true,
|
|
},
|
|
})
|
|
} else {
|
|
result = append(result, &master_pb.KeepConnectedResponse{
|
|
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
|
FilerGroup: string(filerGroup),
|
|
NodeType: nodeType,
|
|
Address: string(address),
|
|
IsLeader: false,
|
|
IsAdd: true,
|
|
},
|
|
})
|
|
}
|
|
} else {
|
|
if m.leaders.removeLeaderIfExists(address) {
|
|
|
|
result = append(result, &master_pb.KeepConnectedResponse{
|
|
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
|
FilerGroup: string(filerGroup),
|
|
NodeType: nodeType,
|
|
Address: string(address),
|
|
IsLeader: true,
|
|
IsAdd: false,
|
|
},
|
|
})
|
|
|
|
// pick the freshest one, since it is less likely to go away
|
|
var shortestDuration int64 = math.MaxInt64
|
|
now := time.Now()
|
|
var candidateAddress pb.ServerAddress
|
|
for _, node := range m.members {
|
|
if m.leaders.isOneLeader(node.Address) {
|
|
continue
|
|
}
|
|
duration := now.Sub(node.CreatedTs).Nanoseconds()
|
|
if duration < shortestDuration {
|
|
shortestDuration = duration
|
|
candidateAddress = node.Address
|
|
}
|
|
}
|
|
if candidateAddress != "" {
|
|
m.leaders.addLeaderIfVacant(candidateAddress)
|
|
// added a new leader
|
|
result = append(result, &master_pb.KeepConnectedResponse{
|
|
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
|
NodeType: nodeType,
|
|
Address: string(candidateAddress),
|
|
IsLeader: true,
|
|
IsAdd: true,
|
|
},
|
|
})
|
|
}
|
|
} else {
|
|
result = append(result, &master_pb.KeepConnectedResponse{
|
|
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
|
FilerGroup: string(filerGroup),
|
|
NodeType: nodeType,
|
|
Address: string(address),
|
|
IsLeader: false,
|
|
IsAdd: false,
|
|
},
|
|
})
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
func (leaders *Leaders) addLeaderIfVacant(address pb.ServerAddress) (hasChanged bool) {
|
|
if leaders.isOneLeader(address) {
|
|
return
|
|
}
|
|
for i := 0; i < len(leaders.leaders); i++ {
|
|
if leaders.leaders[i] == "" {
|
|
leaders.leaders[i] = address
|
|
hasChanged = true
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}
|
|
func (leaders *Leaders) removeLeaderIfExists(address pb.ServerAddress) (hasChanged bool) {
|
|
if !leaders.isOneLeader(address) {
|
|
return
|
|
}
|
|
for i := 0; i < len(leaders.leaders); i++ {
|
|
if leaders.leaders[i] == address {
|
|
leaders.leaders[i] = ""
|
|
hasChanged = true
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}
|
|
func (leaders *Leaders) isOneLeader(address pb.ServerAddress) bool {
|
|
for i := 0; i < len(leaders.leaders); i++ {
|
|
if leaders.leaders[i] == address {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
func (leaders *Leaders) GetLeaders() (addresses []pb.ServerAddress) {
|
|
for i := 0; i < len(leaders.leaders); i++ {
|
|
if leaders.leaders[i] != "" {
|
|
addresses = append(addresses, leaders.leaders[i])
|
|
}
|
|
}
|
|
return
|
|
}
|