package command import ( "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/server" "github.com/spf13/viper" ) func init() { cmdFilerExport.Run = runFilerExport // break init cycle } var cmdFilerExport = &Command{ UsageLine: "filer.export -sourceStore=mysql -targetStroe=cassandra", Short: "export meta data in filer store", Long: `Iterate the file tree and export all metadata out Both source and target store: * should be a store name already specified in filer.toml * do not need to be enabled state If target store is empty, only the directory tree will be listed. `, } var ( // filerExportOutputFile = cmdFilerExport.Flag.String("output", "", "the output file. If empty, only list out the directory tree") filerExportSourceStore = cmdFilerExport.Flag.String("sourceStore", "", "the source store name in filer.toml") filerExportTargetStore = cmdFilerExport.Flag.String("targetStore", "", "the target store name in filer.toml") dirListLimit = cmdFilerExport.Flag.Int("dirListLimit", 100000, "limit directory list size") ) type statistics struct { directoryCount int fileCount int } func runFilerExport(cmd *Command, args []string) bool { weed_server.LoadConfiguration("filer", true) config := viper.GetViper() var sourceStore, targetStore filer2.FilerStore for _, store := range filer2.Stores { if store.GetName() == *filerExportSourceStore { viperSub := config.Sub(store.GetName()) if err := store.Initialize(viperSub); err != nil { glog.Fatalf("Failed to initialize store for %s: %+v", store.GetName(), err) } else { sourceStore = store } break } } for _, store := range filer2.Stores { if store.GetName() == *filerExportTargetStore { viperSub := config.Sub(store.GetName()) if err := store.Initialize(viperSub); err != nil { glog.Fatalf("Failed to initialize store for %s: %+v", store.GetName(), err) } else { targetStore = store } break } } if sourceStore == nil { glog.Errorf("Failed to find source store %s", *filerExportSourceStore) println("existing data sources are:") for _, store := range filer2.Stores { println(" " + store.GetName()) } return false } stat := statistics{} var fn func(level int, entry *filer2.Entry) error if targetStore == nil { fn = printout } else { fn = func(level int, entry *filer2.Entry) error { return targetStore.InsertEntry(entry) } } doTraverse(&stat, sourceStore, filer2.FullPath("/"), 0, fn) glog.Infof("processed %d directories, %d files", stat.directoryCount, stat.fileCount) return true } func doTraverse(stat *statistics, filerStore filer2.FilerStore, parentPath filer2.FullPath, level int, fn func(level int, entry *filer2.Entry) error) { limit := *dirListLimit lastEntryName := "" for { entries, err := filerStore.ListDirectoryEntries(parentPath, lastEntryName, false, limit) if err != nil { break } for _, entry := range entries { if fnErr := fn(level, entry); fnErr != nil { glog.Errorf("failed to process entry: %s", entry.FullPath) } if entry.IsDirectory() { stat.directoryCount++ doTraverse(stat, filerStore, entry.FullPath, level+1, fn) } else { stat.fileCount++ } } if len(entries) < limit { break } } } func printout(level int, entry *filer2.Entry) error { for i := 0; i < level; i++ { print(" ") } println(entry.FullPath.Name()) return nil }