From 3885374edf80c5c07551c2a43948614b98936fe5 Mon Sep 17 00:00:00 2001 From: chrislu Date: Thu, 21 Apr 2022 01:10:46 -0700 Subject: [PATCH] conditionally build elastic, gocdk to reduce binary size --- Makefile | 11 +++ weed/command/filer_meta_tail.go | 72 ---------------- weed/command/filer_meta_tail_elastic.go | 82 +++++++++++++++++++ weed/command/filer_meta_tail_non_elastic.go | 14 ++++ weed/filer/elastic/v7/doc.go | 9 ++ weed/filer/elastic/v7/elastic_store.go | 3 + weed/filer/elastic/v7/elastic_store_kv.go | 3 + weed/notification/gocdk_pub_sub/doc.go | 9 ++ .../gocdk_pub_sub/gocdk_pub_sub.go | 3 + .../sub/notification_gocdk_pub_sub.go | 3 + 10 files changed, 137 insertions(+), 72 deletions(-) create mode 100644 Makefile create mode 100644 weed/command/filer_meta_tail_elastic.go create mode 100644 weed/command/filer_meta_tail_non_elastic.go create mode 100644 weed/filer/elastic/v7/doc.go create mode 100644 weed/notification/gocdk_pub_sub/doc.go diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..844c67172 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +BINARY = weed + +SOURCE_DIR = . + +all: install + +install: + cd weed; go install + +full_install: + cd weed; go install -tags "elastic gocdk" diff --git a/weed/command/filer_meta_tail.go b/weed/command/filer_meta_tail.go index 51c4e7128..7dbeee444 100644 --- a/weed/command/filer_meta_tail.go +++ b/weed/command/filer_meta_tail.go @@ -1,12 +1,9 @@ package command import ( - "context" "fmt" "github.com/chrislusf/seaweedfs/weed/pb" "github.com/golang/protobuf/jsonpb" - jsoniter "github.com/json-iterator/go" - elastic "github.com/olivere/elastic/v7" "os" "path/filepath" "strings" @@ -124,72 +121,3 @@ func runFilerMetaTail(cmd *Command, args []string) bool { return true } - -type EsDocument struct { - Dir string `json:"dir,omitempty"` - Name string `json:"name,omitempty"` - IsDirectory bool `json:"isDir,omitempty"` - Size uint64 `json:"size,omitempty"` - Uid uint32 `json:"uid,omitempty"` - Gid uint32 `json:"gid,omitempty"` - UserName string `json:"userName,omitempty"` - Collection string `json:"collection,omitempty"` - Crtime int64 `json:"crtime,omitempty"` - Mtime int64 `json:"mtime,omitempty"` - Mime string `json:"mime,omitempty"` -} - -func toEsEntry(event *filer_pb.EventNotification) (*EsDocument, string) { - entry := event.NewEntry - dir, name := event.NewParentPath, entry.Name - id := util.Md5String([]byte(util.NewFullPath(dir, name))) - esEntry := &EsDocument{ - Dir: dir, - Name: name, - IsDirectory: entry.IsDirectory, - Size: entry.Attributes.FileSize, - Uid: entry.Attributes.Uid, - Gid: entry.Attributes.Gid, - UserName: entry.Attributes.UserName, - Collection: entry.Attributes.Collection, - Crtime: entry.Attributes.Crtime, - Mtime: entry.Attributes.Mtime, - Mime: entry.Attributes.Mime, - } - return esEntry, id -} - -func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) { - options := []elastic.ClientOptionFunc{} - options = append(options, elastic.SetURL(strings.Split(servers, ",")...)) - options = append(options, elastic.SetSniff(false)) - options = append(options, elastic.SetHealthcheck(false)) - client, err := elastic.NewClient(options...) - if err != nil { - return nil, err - } - return func(resp *filer_pb.SubscribeMetadataResponse) error { - event := resp.EventNotification - if event.OldEntry != nil && - (event.NewEntry == nil || resp.Directory != event.NewParentPath || event.OldEntry.Name != event.NewEntry.Name) { - // delete or not update the same file - dir, name := resp.Directory, event.OldEntry.Name - id := util.Md5String([]byte(util.NewFullPath(dir, name))) - println("delete", id) - _, err := client.Delete().Index(esIndex).Id(id).Do(context.Background()) - return err - } - if event.NewEntry != nil { - // add a new file or update the same file - esEntry, id := toEsEntry(event) - value, err := jsoniter.Marshal(esEntry) - if err != nil { - return err - } - println(string(value)) - _, err = client.Index().Index(esIndex).Id(id).BodyJson(string(value)).Do(context.Background()) - return err - } - return nil - }, nil -} diff --git a/weed/command/filer_meta_tail_elastic.go b/weed/command/filer_meta_tail_elastic.go new file mode 100644 index 000000000..4c5b606a3 --- /dev/null +++ b/weed/command/filer_meta_tail_elastic.go @@ -0,0 +1,82 @@ +//go:build elastic +// +build elastic + +package command + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" + jsoniter "github.com/json-iterator/go" + elastic "github.com/olivere/elastic/v7" + "strings" +) + +type EsDocument struct { + Dir string `json:"dir,omitempty"` + Name string `json:"name,omitempty"` + IsDirectory bool `json:"isDir,omitempty"` + Size uint64 `json:"size,omitempty"` + Uid uint32 `json:"uid,omitempty"` + Gid uint32 `json:"gid,omitempty"` + UserName string `json:"userName,omitempty"` + Collection string `json:"collection,omitempty"` + Crtime int64 `json:"crtime,omitempty"` + Mtime int64 `json:"mtime,omitempty"` + Mime string `json:"mime,omitempty"` +} + +func toEsEntry(event *filer_pb.EventNotification) (*EsDocument, string) { + entry := event.NewEntry + dir, name := event.NewParentPath, entry.Name + id := util.Md5String([]byte(util.NewFullPath(dir, name))) + esEntry := &EsDocument{ + Dir: dir, + Name: name, + IsDirectory: entry.IsDirectory, + Size: entry.Attributes.FileSize, + Uid: entry.Attributes.Uid, + Gid: entry.Attributes.Gid, + UserName: entry.Attributes.UserName, + Collection: entry.Attributes.Collection, + Crtime: entry.Attributes.Crtime, + Mtime: entry.Attributes.Mtime, + Mime: entry.Attributes.Mime, + } + return esEntry, id +} + +func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) { + options := []elastic.ClientOptionFunc{} + options = append(options, elastic.SetURL(strings.Split(servers, ",")...)) + options = append(options, elastic.SetSniff(false)) + options = append(options, elastic.SetHealthcheck(false)) + client, err := elastic.NewClient(options...) + if err != nil { + return nil, err + } + return func(resp *filer_pb.SubscribeMetadataResponse) error { + event := resp.EventNotification + if event.OldEntry != nil && + (event.NewEntry == nil || resp.Directory != event.NewParentPath || event.OldEntry.Name != event.NewEntry.Name) { + // delete or not update the same file + dir, name := resp.Directory, event.OldEntry.Name + id := util.Md5String([]byte(util.NewFullPath(dir, name))) + println("delete", id) + _, err := client.Delete().Index(esIndex).Id(id).Do(context.Background()) + return err + } + if event.NewEntry != nil { + // add a new file or update the same file + esEntry, id := toEsEntry(event) + value, err := jsoniter.Marshal(esEntry) + if err != nil { + return err + } + println(string(value)) + _, err = client.Index().Index(esIndex).Id(id).BodyJson(string(value)).Do(context.Background()) + return err + } + return nil + }, nil +} diff --git a/weed/command/filer_meta_tail_non_elastic.go b/weed/command/filer_meta_tail_non_elastic.go new file mode 100644 index 000000000..f78f3ee09 --- /dev/null +++ b/weed/command/filer_meta_tail_non_elastic.go @@ -0,0 +1,14 @@ +//go:build !elastic +// +build !elastic + +package command + +import ( + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" +) + +func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) { + return func(resp *filer_pb.SubscribeMetadataResponse) error { + return nil + }, nil +} diff --git a/weed/filer/elastic/v7/doc.go b/weed/filer/elastic/v7/doc.go new file mode 100644 index 000000000..704bbf6de --- /dev/null +++ b/weed/filer/elastic/v7/doc.go @@ -0,0 +1,9 @@ +/* + +Package elastic is for elastic filer store. + +The referenced "github.com/olivere/elastic/v7" library is too big when compiled. +So this is only compiled in "make full_install". + +*/ +package elastic diff --git a/weed/filer/elastic/v7/elastic_store.go b/weed/filer/elastic/v7/elastic_store.go index a16e5ebca..cb2c66f5a 100644 --- a/weed/filer/elastic/v7/elastic_store.go +++ b/weed/filer/elastic/v7/elastic_store.go @@ -1,3 +1,6 @@ +//go:build elastic +// +build elastic + package elastic import ( diff --git a/weed/filer/elastic/v7/elastic_store_kv.go b/weed/filer/elastic/v7/elastic_store_kv.go index 99c03314e..43835c153 100644 --- a/weed/filer/elastic/v7/elastic_store_kv.go +++ b/weed/filer/elastic/v7/elastic_store_kv.go @@ -1,3 +1,6 @@ +//go:build elastic +// +build elastic + package elastic import ( diff --git a/weed/notification/gocdk_pub_sub/doc.go b/weed/notification/gocdk_pub_sub/doc.go new file mode 100644 index 000000000..d7fbb9f78 --- /dev/null +++ b/weed/notification/gocdk_pub_sub/doc.go @@ -0,0 +1,9 @@ +/* + +Package gocdk_pub_sub is for Azure Service Bus and RabbitMQ. + +The referenced "gocloud.dev/pubsub" library is too big when compiled. +So this is only compiled in "make full_install". + +*/ +package gocdk_pub_sub diff --git a/weed/notification/gocdk_pub_sub/gocdk_pub_sub.go b/weed/notification/gocdk_pub_sub/gocdk_pub_sub.go index 01c4d901f..f31b6997e 100644 --- a/weed/notification/gocdk_pub_sub/gocdk_pub_sub.go +++ b/weed/notification/gocdk_pub_sub/gocdk_pub_sub.go @@ -1,3 +1,6 @@ +//go:build gocdk +// +build gocdk + // Package gocdk_pub_sub supports the Go CDK (Cloud Development Kit) PubSub API, // which in turn supports many providers, including Amazon SNS/SQS, Azure Service Bus, // Google Cloud PubSub, and RabbitMQ. diff --git a/weed/replication/sub/notification_gocdk_pub_sub.go b/weed/replication/sub/notification_gocdk_pub_sub.go index b16eec2e1..cb690e3ce 100644 --- a/weed/replication/sub/notification_gocdk_pub_sub.go +++ b/weed/replication/sub/notification_gocdk_pub_sub.go @@ -1,3 +1,6 @@ +//go:build gocdk +// +build gocdk + package sub import (