mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/chrislusf/seaweedfs synced 2024-05-08 20:41:20 +02:00
Code Issues Projects Releases Wiki Activity

fix master maintenance logic

Browse source 
Signed-off-by: Lei Liu <lei01.liu@horizon.ai>
This commit is contained in:
Lei Liu 2019-11-12 14:21:43 +08:00
parent ee90236a97
commit 46755ea1e1
8 changed files with 50 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
unmaintained/remove_duplicate_fids/remove_duplicate_fids.go
View file

@ -26,9 +26,9 @@ type VolumeFileScanner4SeeDat struct {
version needle.Version
block storage.SuperBlock
dir string
hashes map[string]bool
dat *os.File
dir string
hashes map[string]bool
dat *os.File
datBackend backend.DataStorageBackend
}

2
weed/command/scaffold.go
View file

@ -346,6 +346,8 @@ scripts = """
"""
sleep_minutes = 17 # sleep minutes between each script execution
filer_url = "http://localhost:8888/"
sequencer_type = memory # Choose [memory|etcd] type for storing the file id sequence
# when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence

24
weed/command/shell.go
View file

@ -2,9 +2,6 @@ package command
import (
"fmt"
"net/url"
"strconv"
"strings"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/shell"
@ -37,7 +34,7 @@ func runShell(command *Command, args []string) bool {
shellOptions.GrpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
var filerPwdErr error
shellOptions.FilerHost, shellOptions.FilerPort, shellOptions.Directory, filerPwdErr = parseFilerUrl(*shellInitialFilerUrl)
shellOptions.FilerHost, shellOptions.FilerPort, shellOptions.Directory, filerPwdErr = util.ParseFilerUrl(*shellInitialFilerUrl)
if filerPwdErr != nil {
fmt.Printf("failed to parse url filer.url=%s : %v\n", *shellInitialFilerUrl, filerPwdErr)
return false
@ -48,22 +45,3 @@ func runShell(command *Command, args []string) bool {
return true
}
func parseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error) {
if !strings.HasPrefix(entryPath, "http://") && !strings.HasPrefix(entryPath, "https://") {
entryPath = "http://" + entryPath
}
var u *url.URL
u, err = url.Parse(entryPath)
if err != nil {
return
}
filerServer = u.Hostname()
portString := u.Port()
if portString != "" {
filerPort, err = strconv.ParseInt(portString, 10, 32)
}
path = u.Path
return
}

14
weed/operation/assign_file_id.go
View file

@ -47,13 +47,13 @@ func Assign(server string, grpcDialOption grpc.DialOption, primaryRequest *Volum
lastError = WithMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
req := &master_pb.AssignRequest{
Count: primaryRequest.Count,
Replication: primaryRequest.Replication,
Collection: primaryRequest.Collection,
Ttl: primaryRequest.Ttl,
DataCenter: primaryRequest.DataCenter,
Rack: primaryRequest.Rack,
DataNode: primaryRequest.DataNode,
Count: primaryRequest.Count,
Replication: primaryRequest.Replication,
Collection: primaryRequest.Collection,
Ttl: primaryRequest.Ttl,
DataCenter: primaryRequest.DataCenter,
Rack: primaryRequest.Rack,
DataNode: primaryRequest.DataNode,
WritableVolumeCount: primaryRequest.WritableVolumeCount,
}
resp, grpcErr := masterClient.Assign(context.Background(), req)

20
weed/server/master_server.go
View file

@ -182,16 +182,21 @@ func (ms *MasterServer) proxyToLeader(f func(w http.ResponseWriter, r *http.Requ
}
func (ms *MasterServer) startAdminScripts() {
var err error
v := viper.GetViper()
adminScripts := v.GetString("master.maintenance.scripts")
v.SetDefault("master.maintenance.sleep_minutes", 17)
sleepMinutes := v.GetInt("master.maintenance.sleep_minutes")
glog.V(0).Infof("adminScripts:\n%v", adminScripts)
if adminScripts == "" {
return
}
v.SetDefault("master.maintenance.sleep_minutes", 17)
sleepMinutes := v.GetInt("master.maintenance.sleep_minutes")
v.SetDefault("master.maintenance.filer_url", "http://localhost:8888/")
filerURL := v.GetString("master.maintenance.filer_url")
scriptLines := strings.Split(adminScripts, "\n")
masterAddress := "localhost:" + strconv.Itoa(ms.option.Port)
@ -199,9 +204,12 @@ func (ms *MasterServer) startAdminScripts() {
var shellOptions shell.ShellOptions
shellOptions.GrpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "master")
shellOptions.Masters = &masterAddress
shellOptions.FilerHost = "localhost"
shellOptions.FilerPort = 8888
shellOptions.Directory = "/"
shellOptions.FilerHost, shellOptions.FilerPort, shellOptions.Directory, err = util.ParseFilerUrl(filerURL)
if err != nil {
glog.V(0).Infof("failed to parse master.maintenance.filer_url=%s : %v\n", filerURL, err)
return
}
commandEnv := shell.NewCommandEnv(shellOptions)

4
weed/storage/backend/memory_map/memory_map_backend.go
View file

@ -17,14 +17,14 @@ type MemoryMappedFile struct {
func NewMemoryMappedFile(f *os.File, memoryMapSizeMB uint32) *MemoryMappedFile {
mmf := &MemoryMappedFile{
mm : new(MemoryMap),
mm: new(MemoryMap),
}
mmf.mm.CreateMemoryMap(f, 1024*1024*uint64(memoryMapSizeMB))
return mmf
}
func (mmf *MemoryMappedFile) ReadAt(p []byte, off int64) (n int, err error) {
readBytes, e := mmf.mm.ReadMemory(uint64(off), uint64(len(p)))
readBytes, e := mmf.mm.ReadMemory(uint64(off), uint64(len(p)))
if e != nil {
return 0, e
}

1
weed/util/config.go
View file

@ -45,4 +45,3 @@ func LoadConfiguration(configFileName string, required bool) (loaded bool) {
func Config() Configuration {
return viper.GetViper()
}

21
weed/util/parse.go
View file

@ -1,7 +1,9 @@
package util
import (
"net/url"
"strconv"
"strings"
)
func ParseInt(text string, defaultValue int) int {
@ -24,3 +26,22 @@ func ParseUint64(text string, defaultValue uint64) uint64 {
}
return count
}
func ParseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error) {
if !strings.HasPrefix(entryPath, "http://") && !strings.HasPrefix(entryPath, "https://") {
entryPath = "http://" + entryPath
}
var u *url.URL
u, err = url.Parse(entryPath)
if err != nil {
return
}
filerServer = u.Hostname()
portString := u.Port()
if portString != "" {
filerPort, err = strconv.ParseInt(portString, 10, 32)
}
path = u.Path
return
}