From e666aeece2778812a4a9d3fc4daaaac86fe4a412 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 23 Mar 2020 21:26:15 -0700 Subject: [PATCH] simplify parsing filer host and port --- weed/shell/command_bucket_create.go | 9 ++------- weed/shell/command_bucket_delete.go | 8 +++----- weed/shell/command_bucket_list.go | 8 +++----- weed/shell/command_fs_cat.go | 8 +++----- weed/shell/command_fs_cd.go | 10 ++-------- weed/shell/command_fs_du.go | 28 ++++++---------------------- weed/shell/command_fs_ls.go | 8 +++----- weed/shell/command_fs_meta_cat.go | 6 ++---- weed/shell/command_fs_meta_load.go | 9 ++------- weed/shell/command_fs_meta_notify.go | 4 ++-- weed/shell/command_fs_meta_save.go | 8 ++++---- weed/shell/command_fs_mv.go | 6 +++--- weed/shell/command_fs_tree.go | 4 ++-- weed/shell/commands.go | 27 +++++++-------------------- 14 files changed, 44 insertions(+), 99 deletions(-) diff --git a/weed/shell/command_bucket_create.go b/weed/shell/command_bucket_create.go index 3546528aa..52d96e4c3 100644 --- a/weed/shell/command_bucket_create.go +++ b/weed/shell/command_bucket_create.go @@ -43,16 +43,11 @@ func (c *commandBucketCreate) Do(args []string, commandEnv *CommandEnv, writer i return fmt.Errorf("empty bucket name") } - filerServer, filerPort, _, parseErr := commandEnv.parseUrl(findInputDirectory(bucketCommand.Args())) - if parseErr != nil { - return parseErr - } - - err = commandEnv.withFilerClient(filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error { + err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { - return fmt.Errorf("get filer %s:%d configuration: %v", filerServer, filerPort, err) + return fmt.Errorf("get filer configuration: %v", err) } filerBucketsPath := resp.DirBuckets diff --git a/weed/shell/command_bucket_delete.go b/weed/shell/command_bucket_delete.go index 509b3c1de..07c2e74ac 100644 --- a/weed/shell/command_bucket_delete.go +++ b/weed/shell/command_bucket_delete.go @@ -38,19 +38,17 @@ func (c *commandBucketDelete) Do(args []string, commandEnv *CommandEnv, writer i return fmt.Errorf("empty bucket name") } - filerServer, filerPort, _, parseErr := commandEnv.parseUrl(findInputDirectory(bucketCommand.Args())) + _, parseErr := commandEnv.parseUrl(findInputDirectory(bucketCommand.Args())) if parseErr != nil { return parseErr } - filerClient := commandEnv.getFilerClient(filerServer, filerPort) - var filerBucketsPath string - filerBucketsPath, err = readFilerBucketsPath(filerClient) + filerBucketsPath, err = readFilerBucketsPath(commandEnv) if err != nil { return fmt.Errorf("read buckets: %v", err) } - return filer_pb.Remove(filerClient, filerBucketsPath, *bucketName, false, true, true) + return filer_pb.Remove(commandEnv, filerBucketsPath, *bucketName, false, true, true) } diff --git a/weed/shell/command_bucket_list.go b/weed/shell/command_bucket_list.go index 486d40fba..b982ff646 100644 --- a/weed/shell/command_bucket_list.go +++ b/weed/shell/command_bucket_list.go @@ -34,20 +34,18 @@ func (c *commandBucketList) Do(args []string, commandEnv *CommandEnv, writer io. return nil } - filerServer, filerPort, _, parseErr := commandEnv.parseUrl(findInputDirectory(bucketCommand.Args())) + _, parseErr := commandEnv.parseUrl(findInputDirectory(bucketCommand.Args())) if parseErr != nil { return parseErr } - filerClient := commandEnv.getFilerClient(filerServer, filerPort) - var filerBucketsPath string - filerBucketsPath, err = readFilerBucketsPath(filerClient) + filerBucketsPath, err = readFilerBucketsPath(commandEnv) if err != nil { return fmt.Errorf("read buckets: %v", err) } - err = filer_pb.List(filerClient, filerBucketsPath, "", func(entry *filer_pb.Entry, isLast bool) { + err = filer_pb.List(commandEnv, filerBucketsPath, "", func(entry *filer_pb.Entry, isLast bool) { if entry.Attributes.Replication == "" || entry.Attributes.Replication == "000" { fmt.Fprintf(writer, " %s\n", entry.Name) } else { diff --git a/weed/shell/command_fs_cat.go b/weed/shell/command_fs_cat.go index b1d4eea14..7177d8ac3 100644 --- a/weed/shell/command_fs_cat.go +++ b/weed/shell/command_fs_cat.go @@ -30,20 +30,18 @@ func (c *commandFsCat) Help() string { func (c *commandFsCat) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - input := findInputDirectory(args) - - filerServer, filerPort, path, err := commandEnv.parseUrl(input) + path, err := commandEnv.parseUrl(findInputDirectory(args)) if err != nil { return err } - if commandEnv.isDirectory(filerServer, filerPort, path) { + if commandEnv.isDirectory(path) { return fmt.Errorf("%s is a directory", path) } dir, name := util.FullPath(path).DirAndName() - return commandEnv.withFilerClient(filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error { + return commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.LookupDirectoryEntryRequest{ Name: name, diff --git a/weed/shell/command_fs_cd.go b/weed/shell/command_fs_cd.go index 377fd40f7..2cc28f7a2 100644 --- a/weed/shell/command_fs_cd.go +++ b/weed/shell/command_fs_cd.go @@ -30,25 +30,19 @@ func (c *commandFsCd) Help() string { func (c *commandFsCd) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - input := findInputDirectory(args) - - filerServer, filerPort, path, err := commandEnv.parseUrl(input) + path, err := commandEnv.parseUrl(findInputDirectory(args)) if err != nil { return err } if path == "/" { - commandEnv.option.FilerHost = filerServer - commandEnv.option.FilerPort = filerPort commandEnv.option.Directory = "/" return nil } - err = commandEnv.checkDirectory(filerServer, filerPort, path) + err = commandEnv.checkDirectory(path) if err == nil { - commandEnv.option.FilerHost = filerServer - commandEnv.option.FilerPort = filerPort commandEnv.option.Directory = path } diff --git a/weed/shell/command_fs_du.go b/weed/shell/command_fs_du.go index 0372ba95f..f3d479614 100644 --- a/weed/shell/command_fs_du.go +++ b/weed/shell/command_fs_du.go @@ -32,18 +32,18 @@ func (c *commandFsDu) Help() string { func (c *commandFsDu) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - filerServer, filerPort, path, err := commandEnv.parseUrl(findInputDirectory(args)) + path, err := commandEnv.parseUrl(findInputDirectory(args)) if err != nil { return err } - if commandEnv.isDirectory(filerServer, filerPort, path) { + if commandEnv.isDirectory(path) { path = path + "/" } var blockCount, byteCount uint64 dir, name := util.FullPath(path).DirAndName() - blockCount, byteCount, err = duTraverseDirectory(writer, commandEnv.getFilerClient(filerServer, filerPort), dir, name) + blockCount, byteCount, err = duTraverseDirectory(writer, commandEnv, dir, name) if name == "" && err == nil { fmt.Fprintf(writer, "block:%4d\tbyte:%10d\t%s\n", blockCount, byteCount, dir) @@ -78,29 +78,13 @@ func duTraverseDirectory(writer io.Writer, filerClient filer_pb.FilerClient, dir return } -func (env *CommandEnv) withFilerClient(filerServer string, filerPort int64, fn func(filer_pb.SeaweedFilerClient) error) error { +func (env *CommandEnv) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error { - filerGrpcAddress := fmt.Sprintf("%s:%d", filerServer, filerPort+10000) + filerGrpcAddress := fmt.Sprintf("%s:%d", env.option.FilerHost, env.option.FilerPort+10000) return pb.WithGrpcFilerClient(filerGrpcAddress, env.option.GrpcDialOption, fn) } -type commandFilerClient struct { - env *CommandEnv - filerServer string - filerPort int64 -} - -func (env *CommandEnv) getFilerClient(filerServer string, filerPort int64) *commandFilerClient { - return &commandFilerClient{ - env: env, - filerServer: filerServer, - filerPort: filerPort, - } -} -func (c *commandFilerClient) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error { - return c.env.withFilerClient(c.filerServer, c.filerPort, fn) -} -func (c *commandFilerClient) AdjustedUrl(hostAndPort string) string { +func (env *CommandEnv) AdjustedUrl(hostAndPort string) string { return hostAndPort } diff --git a/weed/shell/command_fs_ls.go b/weed/shell/command_fs_ls.go index 66adf057e..be531e980 100644 --- a/weed/shell/command_fs_ls.go +++ b/weed/shell/command_fs_ls.go @@ -50,21 +50,19 @@ func (c *commandFsLs) Do(args []string, commandEnv *CommandEnv, writer io.Writer } } - input := findInputDirectory(args) - - filerServer, filerPort, path, err := commandEnv.parseUrl(input) + path, err := commandEnv.parseUrl(findInputDirectory(args)) if err != nil { return err } - if commandEnv.isDirectory(filerServer, filerPort, path) { + if commandEnv.isDirectory(path) { path = path + "/" } dir, name := util.FullPath(path).DirAndName() entryCount := 0 - err = filer_pb.ReadDirAllEntries(commandEnv.getFilerClient(filerServer, filerPort), util.FullPath(dir), name, func(entry *filer_pb.Entry, isLast bool) { + err = filer_pb.ReadDirAllEntries(commandEnv, util.FullPath(dir), name, func(entry *filer_pb.Entry, isLast bool) { if !showHidden && strings.HasPrefix(entry.Name, ".") { return diff --git a/weed/shell/command_fs_meta_cat.go b/weed/shell/command_fs_meta_cat.go index cbbca746c..0679ec075 100644 --- a/weed/shell/command_fs_meta_cat.go +++ b/weed/shell/command_fs_meta_cat.go @@ -31,16 +31,14 @@ func (c *commandFsMetaCat) Help() string { func (c *commandFsMetaCat) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - input := findInputDirectory(args) - - filerServer, filerPort, path, err := commandEnv.parseUrl(input) + path, err := commandEnv.parseUrl(findInputDirectory(args)) if err != nil { return err } dir, name := util.FullPath(path).DirAndName() - return commandEnv.withFilerClient(filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error { + return commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.LookupDirectoryEntryRequest{ Name: name, diff --git a/weed/shell/command_fs_meta_load.go b/weed/shell/command_fs_meta_load.go index a19e9d3ce..69ae9454c 100644 --- a/weed/shell/command_fs_meta_load.go +++ b/weed/shell/command_fs_meta_load.go @@ -37,11 +37,6 @@ func (c *commandFsMetaLoad) Do(args []string, commandEnv *CommandEnv, writer io. return nil } - filerServer, filerPort, path, err := commandEnv.parseUrl(findInputDirectory(nil)) - if err != nil { - return err - } - fileName := args[len(args)-1] dst, err := os.OpenFile(fileName, os.O_RDONLY, 0644) @@ -52,7 +47,7 @@ func (c *commandFsMetaLoad) Do(args []string, commandEnv *CommandEnv, writer io. var dirCount, fileCount uint64 - err = commandEnv.withFilerClient(filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error { + err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { sizeBuf := make([]byte, 4) @@ -98,7 +93,7 @@ func (c *commandFsMetaLoad) Do(args []string, commandEnv *CommandEnv, writer io. if err == nil { fmt.Fprintf(writer, "\ntotal %d directories, %d files", dirCount, fileCount) - fmt.Fprintf(writer, "\n%s is loaded to http://%s:%d%s\n", fileName, filerServer, filerPort, path) + fmt.Fprintf(writer, "\n%s is loaded.\n", fileName) } return err diff --git a/weed/shell/command_fs_meta_notify.go b/weed/shell/command_fs_meta_notify.go index 995ea16a2..b361b61a6 100644 --- a/weed/shell/command_fs_meta_notify.go +++ b/weed/shell/command_fs_meta_notify.go @@ -32,7 +32,7 @@ func (c *commandFsMetaNotify) Help() string { func (c *commandFsMetaNotify) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - filerServer, filerPort, path, err := commandEnv.parseUrl(findInputDirectory(args)) + path, err := commandEnv.parseUrl(findInputDirectory(args)) if err != nil { return err } @@ -43,7 +43,7 @@ func (c *commandFsMetaNotify) Do(args []string, commandEnv *CommandEnv, writer i var dirCount, fileCount uint64 - err = doTraverseBFS(writer, commandEnv.getFilerClient(filerServer, filerPort), util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) { + err = doTraverseBFS(writer, commandEnv, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) { if entry.IsDirectory { dirCount++ diff --git a/weed/shell/command_fs_meta_save.go b/weed/shell/command_fs_meta_save.go index 4314542bd..a07a94ccb 100644 --- a/weed/shell/command_fs_meta_save.go +++ b/weed/shell/command_fs_meta_save.go @@ -52,7 +52,7 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io. return nil } - filerServer, filerPort, path, parseErr := commandEnv.parseUrl(findInputDirectory(fsMetaSaveCommand.Args())) + path, parseErr := commandEnv.parseUrl(findInputDirectory(fsMetaSaveCommand.Args())) if parseErr != nil { return parseErr } @@ -61,7 +61,7 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io. fileName := *outputFileName if fileName == "" { fileName = fmt.Sprintf("%s-%d-%4d%02d%02d-%02d%02d%02d.meta", - filerServer, filerPort, t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) + commandEnv.option.FilerHost, commandEnv.option.FilerPort, t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) } dst, openErr := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) @@ -85,7 +85,7 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io. var dirCount, fileCount uint64 - err = doTraverseBFS(writer, commandEnv.getFilerClient(filerServer, filerPort), util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) { + err = doTraverseBFS(writer, commandEnv, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) { protoMessage := &filer_pb.FullEntry{ Dir: string(parentPath), @@ -118,7 +118,7 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io. if err == nil { fmt.Fprintf(writer, "total %d directories, %d files\n", dirCount, fileCount) - fmt.Fprintf(writer, "meta data for http://%s:%d%s is saved to %s\n", filerServer, filerPort, path, fileName) + fmt.Fprintf(writer, "meta data for %s is saved to %s\n", path, fileName) } return err diff --git a/weed/shell/command_fs_mv.go b/weed/shell/command_fs_mv.go index 148ac6e2f..c7c0984fc 100644 --- a/weed/shell/command_fs_mv.go +++ b/weed/shell/command_fs_mv.go @@ -37,12 +37,12 @@ func (c *commandFsMv) Help() string { func (c *commandFsMv) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - filerServer, filerPort, sourcePath, err := commandEnv.parseUrl(args[0]) + sourcePath, err := commandEnv.parseUrl(args[0]) if err != nil { return err } - _, _, destinationPath, err := commandEnv.parseUrl(args[1]) + destinationPath, err := commandEnv.parseUrl(args[1]) if err != nil { return err } @@ -51,7 +51,7 @@ func (c *commandFsMv) Do(args []string, commandEnv *CommandEnv, writer io.Writer destinationDir, destinationName := util.FullPath(destinationPath).DirAndName() - return commandEnv.withFilerClient(filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error { + return commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { // collect destination entry info destinationRequest := &filer_pb.LookupDirectoryEntryRequest{ diff --git a/weed/shell/command_fs_tree.go b/weed/shell/command_fs_tree.go index 0982082db..b0752ea03 100644 --- a/weed/shell/command_fs_tree.go +++ b/weed/shell/command_fs_tree.go @@ -30,14 +30,14 @@ func (c *commandFsTree) Help() string { func (c *commandFsTree) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { - filerServer, filerPort, path, err := commandEnv.parseUrl(findInputDirectory(args)) + path, err := commandEnv.parseUrl(findInputDirectory(args)) if err != nil { return err } dir, name := util.FullPath(path).DirAndName() - dirCount, fCount, terr := treeTraverseDirectory(writer, commandEnv.getFilerClient(filerServer, filerPort), util.FullPath(dir), name, newPrefix(), -1) + dirCount, fCount, terr := treeTraverseDirectory(writer, commandEnv, util.FullPath(dir), name, newPrefix(), -1) if terr == nil { fmt.Fprintf(writer, "%d directories, %d files\n", dirCount, fCount) diff --git a/weed/shell/commands.go b/weed/shell/commands.go index 660929ec7..7e240de5f 100644 --- a/weed/shell/commands.go +++ b/weed/shell/commands.go @@ -48,7 +48,7 @@ func NewCommandEnv(options ShellOptions) *CommandEnv { } } -func (ce *CommandEnv) parseUrl(input string) (filerServer string, filerPort int64, path string, err error) { +func (ce *CommandEnv) parseUrl(input string) (path string, err error) { if strings.HasPrefix(input, "http") { err = fmt.Errorf("http://: prefix is not supported any more") return @@ -56,35 +56,22 @@ func (ce *CommandEnv) parseUrl(input string) (filerServer string, filerPort int6 if !strings.HasPrefix(input, "/") { input = filepath.ToSlash(filepath.Join(ce.option.Directory, input)) } - return ce.option.FilerHost, ce.option.FilerPort, input, err + return input, err } -func (ce *CommandEnv) isDirectory(filerServer string, filerPort int64, path string) bool { +func (ce *CommandEnv) isDirectory(path string) bool { - return ce.checkDirectory(filerServer, filerPort, path) == nil + return ce.checkDirectory(path) == nil } -func (ce *CommandEnv) checkDirectory(filerServer string, filerPort int64, path string) error { +func (ce *CommandEnv) checkDirectory(path string) error { dir, name := util.FullPath(path).DirAndName() - return ce.withFilerClient(filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error { + _, err := filer_pb.Exists(ce, dir, name, true) - resp, lookupErr := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{ - Directory: dir, - Name: name, - }) - if lookupErr != nil { - return lookupErr - } - - if !resp.Entry.IsDirectory { - return fmt.Errorf("not a directory") - } - - return nil - }) + return err }