From 2b580a7566536fb229a50af94b67d9df46724f34 Mon Sep 17 00:00:00 2001 From: chrislu Date: Wed, 17 Aug 2022 12:42:03 -0700 Subject: [PATCH] also migrate jsonpb --- weed/command/filer_meta_tail.go | 7 ++---- weed/filer/entry_codec.go | 2 +- weed/filer/filer_conf.go | 12 +++------- weed/filer/s3iam_conf.go | 23 ++++++++++++++------ weed/mq/broker/broker_segment_serde.go | 4 ++-- weed/notification/aws_sqs/aws_sqs_pub.go | 7 ++++-- weed/pb/proto_read_write_test.go | 2 +- weed/replication/sub/notification_aws_sqs.go | 2 +- weed/s3api/auth_credentials_test.go | 2 +- weed/shell/command_fs_meta_cat.go | 20 ++--------------- weed/shell/command_remote_configure.go | 10 +-------- weed/shell/command_remote_mount.go | 13 +---------- weed/storage/volume_info/volume_info.go | 15 ++++++------- weed/util/skiplist/Makefile | 2 +- weed/util/skiplist/skiplist.pb.go | 11 +++------- 15 files changed, 47 insertions(+), 85 deletions(-) diff --git a/weed/command/filer_meta_tail.go b/weed/command/filer_meta_tail.go index a433d1b07..58a496ff4 100644 --- a/weed/command/filer_meta_tail.go +++ b/weed/command/filer_meta_tail.go @@ -2,8 +2,8 @@ package command import ( "fmt" + "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/pb" - "google.golang.org/protobuf/jsonpb" "os" "path/filepath" "strings" @@ -88,11 +88,8 @@ func runFilerMetaTail(cmd *Command, args []string) bool { return false } - jsonpbMarshaler := jsonpb.Marshaler{ - EmitDefaults: false, - } eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error { - jsonpbMarshaler.Marshal(os.Stdout, resp) + filer.ProtoToText(os.Stdout, resp) fmt.Fprintln(os.Stdout) return nil } diff --git a/weed/filer/entry_codec.go b/weed/filer/entry_codec.go index 786ecea49..ce9c0484b 100644 --- a/weed/filer/entry_codec.go +++ b/weed/filer/entry_codec.go @@ -21,7 +21,7 @@ func (entry *Entry) DecodeAttributesAndChunks(blob []byte) error { message := &filer_pb.Entry{} - if err := proto.UnmarshalMerge(blob, message); err != nil { + if err := proto.Unmarshal(blob, message); err != nil { return fmt.Errorf("decoding value blob for %s: %v", entry.FullPath, err) } diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go index 59abaa5ab..30161cfaf 100644 --- a/weed/filer/filer_conf.go +++ b/weed/filer/filer_conf.go @@ -13,7 +13,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/viant/ptrie" - "google.golang.org/protobuf/jsonpb" + jsonpb "google.golang.org/protobuf/encoding/protojson" ) const ( @@ -93,7 +93,7 @@ func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*file func (fc *FilerConf) LoadFromBytes(data []byte) (err error) { conf := &filer_pb.FilerConf{} - if err := jsonpb.Unmarshal(bytes.NewReader(data), conf); err != nil { + if err := jsonpb.Unmarshal(data, conf); err != nil { return err } @@ -181,11 +181,5 @@ func (fc *FilerConf) ToProto() *filer_pb.FilerConf { } func (fc *FilerConf) ToText(writer io.Writer) error { - - m := jsonpb.Marshaler{ - EmitDefaults: false, - Indent: " ", - } - - return m.Marshal(writer, fc.ToProto()) + return ProtoToText(writer, fc.ToProto()) } diff --git a/weed/filer/s3iam_conf.go b/weed/filer/s3iam_conf.go index 6240058a4..24ed46be5 100644 --- a/weed/filer/s3iam_conf.go +++ b/weed/filer/s3iam_conf.go @@ -1,17 +1,16 @@ package filer import ( - "bytes" "fmt" "io" "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" - "google.golang.org/protobuf/jsonpb" + jsonpb "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" ) func ParseS3ConfigurationFromBytes[T proto.Message](content []byte, config T) error { - if err := jsonpb.Unmarshal(bytes.NewBuffer(content), config); err != nil { + if err := jsonpb.Unmarshal(content, config); err != nil { return err } return nil @@ -19,12 +18,22 @@ func ParseS3ConfigurationFromBytes[T proto.Message](content []byte, config T) er func ProtoToText(writer io.Writer, config proto.Message) error { - m := jsonpb.Marshaler{ - EmitDefaults: false, - Indent: " ", + m := jsonpb.MarshalOptions{ + EmitUnpopulated: true, + Indent: " ", } - return m.Marshal(writer, config) + text, marshalErr := m.Marshal(config) + if marshalErr != nil { + return fmt.Errorf("marshal proto message: %v", marshalErr) + } + + _, writeErr := writer.Write(text) + if writeErr != nil { + return fmt.Errorf("fail to write proto message: %v", writeErr) + } + + return writeErr } // CheckDuplicateAccessKey returns an error message when s3cfg has duplicate access keys diff --git a/weed/mq/broker/broker_segment_serde.go b/weed/mq/broker/broker_segment_serde.go index 44bc1b7d3..a8b1d05bc 100644 --- a/weed/mq/broker/broker_segment_serde.go +++ b/weed/mq/broker/broker_segment_serde.go @@ -8,7 +8,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" - "google.golang.org/protobuf/jsonpb" + jsonpb "google.golang.org/protobuf/encoding/protojson" "time" ) @@ -60,7 +60,7 @@ func (broker *MessageQueueBroker) readSegmentOnFiler(segment *mq.Segment) (info // parse into filer conf object info = &mq_pb.SegmentInfo{} - if err = jsonpb.Unmarshal(bytes.NewReader(data), info); err != nil { + if err = jsonpb.Unmarshal(data, info); err != nil { return err } found = true diff --git a/weed/notification/aws_sqs/aws_sqs_pub.go b/weed/notification/aws_sqs/aws_sqs_pub.go index f647134c7..c9e674257 100644 --- a/weed/notification/aws_sqs/aws_sqs_pub.go +++ b/weed/notification/aws_sqs/aws_sqs_pub.go @@ -70,7 +70,10 @@ func (k *AwsSqsPub) initialize(awsAccessKeyId, awsSecretAccessKey, region, queue func (k *AwsSqsPub) SendMessage(key string, message proto.Message) (err error) { - text := proto.MarshalTextString(message) + text, err := proto.Marshal(message) + if err != nil { + return fmt.Errorf("send message marshal %+v: %v", message, err) + } _, err = k.svc.SendMessage(&sqs.SendMessageInput{ DelaySeconds: aws.Int64(10), @@ -80,7 +83,7 @@ func (k *AwsSqsPub) SendMessage(key string, message proto.Message) (err error) { StringValue: aws.String(key), }, }, - MessageBody: aws.String(text), + MessageBody: aws.String(string(text)), QueueUrl: &k.queueUrl, }) diff --git a/weed/pb/proto_read_write_test.go b/weed/pb/proto_read_write_test.go index f4751c3bb..ac3b960ee 100644 --- a/weed/pb/proto_read_write_test.go +++ b/weed/pb/proto_read_write_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" - "google.golang.org/protobuf/jsonpb" + jsonpb "google.golang.org/protobuf/encoding/protojson" ) func TestJsonpMarshalUnmarshal(t *testing.T) { diff --git a/weed/replication/sub/notification_aws_sqs.go b/weed/replication/sub/notification_aws_sqs.go index 146f42c72..0201d4f9c 100644 --- a/weed/replication/sub/notification_aws_sqs.go +++ b/weed/replication/sub/notification_aws_sqs.go @@ -98,7 +98,7 @@ func (k *AwsSqsInput) ReceiveMessage() (key string, message *filer_pb.EventNotif key = *keyValue.StringValue text := *result.Messages[0].Body message = &filer_pb.EventNotification{} - err = proto.UnmarshalText(text, message) + err = proto.Unmarshal([]byte(text), message) // delete the message _, err = k.svc.DeleteMessage(&sqs.DeleteMessageInput{ diff --git a/weed/s3api/auth_credentials_test.go b/weed/s3api/auth_credentials_test.go index dacbc775d..09692eb81 100644 --- a/weed/s3api/auth_credentials_test.go +++ b/weed/s3api/auth_credentials_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" "testing" - "google.golang.org/protobuf/jsonpb" + jsonpb "google.golang.org/protobuf/encoding/protojson" "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" ) diff --git a/weed/shell/command_fs_meta_cat.go b/weed/shell/command_fs_meta_cat.go index 4f3b3551e..3ae106415 100644 --- a/weed/shell/command_fs_meta_cat.go +++ b/weed/shell/command_fs_meta_cat.go @@ -2,8 +2,7 @@ package shell import ( "fmt" - "golang.org/x/exp/slices" - "google.golang.org/protobuf/jsonpb" + "github.com/seaweedfs/seaweedfs/weed/filer" "google.golang.org/protobuf/proto" "io" @@ -50,22 +49,7 @@ func (c *commandFsMetaCat) Do(args []string, commandEnv *CommandEnv, writer io.W return err } - m := jsonpb.Marshaler{ - EmitDefaults: true, - Indent: " ", - } - slices.SortFunc(respLookupEntry.Entry.Chunks, func(a, b *filer_pb.FileChunk) bool { - if a.Offset == b.Offset { - return a.Mtime < b.Mtime - } - return a.Offset < b.Offset - }) - text, marshalErr := m.MarshalToString(respLookupEntry.Entry) - if marshalErr != nil { - return fmt.Errorf("marshal meta: %v", marshalErr) - } - - fmt.Fprintf(writer, "%s\n", text) + filer.ProtoToText(writer, respLookupEntry.Entry) bytes, _ := proto.Marshal(respLookupEntry.Entry) gzippedBytes, _ := util.GzipData(bytes) diff --git a/weed/shell/command_remote_configure.go b/weed/shell/command_remote_configure.go index 306858271..4491dc456 100644 --- a/weed/shell/command_remote_configure.go +++ b/weed/shell/command_remote_configure.go @@ -9,7 +9,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb" "github.com/seaweedfs/seaweedfs/weed/remote_storage" "github.com/seaweedfs/seaweedfs/weed/util" - "google.golang.org/protobuf/jsonpb" "google.golang.org/protobuf/proto" "io" "regexp" @@ -159,15 +158,8 @@ func (c *commandRemoteConfigure) listExistingRemoteStorages(commandEnv *CommandE conf.TencentSecretKey = strings.Repeat("*", len(conf.TencentSecretKey)) conf.WasabiSecretKey = strings.Repeat("*", len(conf.WasabiSecretKey)) - m := jsonpb.Marshaler{ - EmitDefaults: false, - Indent: " ", - } + return filer.ProtoToText(writer, conf) - err := m.Marshal(writer, conf) - fmt.Fprintln(writer) - - return err }) } diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go index ef837a821..0f74729f1 100644 --- a/weed/shell/command_remote_mount.go +++ b/weed/shell/command_remote_mount.go @@ -9,7 +9,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb" "github.com/seaweedfs/seaweedfs/weed/remote_storage" "github.com/seaweedfs/seaweedfs/weed/util" - "google.golang.org/protobuf/jsonpb" "google.golang.org/protobuf/proto" "io" "os" @@ -101,17 +100,7 @@ func listExistingRemoteStorageMounts(commandEnv *CommandEnv, writer io.Writer) ( } func jsonPrintln(writer io.Writer, message proto.Message) error { - if message == nil { - return nil - } - m := jsonpb.Marshaler{ - EmitDefaults: false, - Indent: " ", - } - - err := m.Marshal(writer, message) - fmt.Fprintln(writer) - return err + return filer.ProtoToText(writer, message) } func syncMetadata(commandEnv *CommandEnv, writer io.Writer, dir string, nonEmpty bool, remoteConf *remote_pb.RemoteConf, remote *remote_pb.RemoteStorageLocation) error { diff --git a/weed/storage/volume_info/volume_info.go b/weed/storage/volume_info/volume_info.go index 8002e9c19..7f8663be2 100644 --- a/weed/storage/volume_info/volume_info.go +++ b/weed/storage/volume_info/volume_info.go @@ -1,11 +1,10 @@ package volume_info import ( - "bytes" "fmt" "os" - "google.golang.org/protobuf/jsonpb" + jsonpb "google.golang.org/protobuf/encoding/protojson" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" @@ -44,7 +43,7 @@ func MaybeLoadVolumeInfo(fileName string) (volumeInfo *volume_server_pb.VolumeIn } glog.V(1).Infof("maybeLoadVolumeInfo Unmarshal volume info %v", fileName) - if err = jsonpb.Unmarshal(bytes.NewReader(tierData), volumeInfo); err != nil { + if err = jsonpb.Unmarshal(tierData, volumeInfo); err != nil { glog.Warningf("unmarshal error: %v", err) err = fmt.Errorf("unmarshal error: %v", err) return @@ -65,17 +64,17 @@ func SaveVolumeInfo(fileName string, volumeInfo *volume_server_pb.VolumeInfo) er return fmt.Errorf("%s not writable", fileName) } - m := jsonpb.Marshaler{ - EmitDefaults: true, - Indent: " ", + m := jsonpb.MarshalOptions{ + EmitUnpopulated: true, + Indent: " ", } - text, marshalErr := m.MarshalToString(volumeInfo) + text, marshalErr := m.Marshal(volumeInfo) if marshalErr != nil { return fmt.Errorf("marshal to %s: %v", fileName, marshalErr) } - writeErr := util.WriteFile(fileName, []byte(text), 0755) + writeErr := util.WriteFile(fileName, text, 0755) if writeErr != nil { return fmt.Errorf("fail to write %s : %v", fileName, writeErr) } diff --git a/weed/util/skiplist/Makefile b/weed/util/skiplist/Makefile index af4afe639..7c4652cba 100644 --- a/weed/util/skiplist/Makefile +++ b/weed/util/skiplist/Makefile @@ -3,4 +3,4 @@ all: gen .PHONY : gen gen: - protoc skiplist.proto --go_out=plugins=grpc:. --go_opt=paths=source_relative + protoc skiplist.proto --go_out=. --go-grpc_out=. --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative diff --git a/weed/util/skiplist/skiplist.pb.go b/weed/util/skiplist/skiplist.pb.go index ef3827a65..de3efa513 100644 --- a/weed/util/skiplist/skiplist.pb.go +++ b/weed/util/skiplist/skiplist.pb.go @@ -1,13 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.3 +// protoc-gen-go v1.26.0 +// protoc v3.17.3 // source: skiplist.proto package skiplist import ( - proto "google.golang.org/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -21,10 +20,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type SkipListProto struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -325,7 +320,7 @@ var file_skiplist_proto_rawDesc = []byte{ 0x22, 0x25, 0x0a, 0x0d, 0x4e, 0x61, 0x6d, 0x65, 0x42, 0x61, 0x74, 0x63, 0x68, 0x44, 0x61, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x42, 0x33, 0x5a, 0x31, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x75, 0x74, 0x69, 0x6c, 0x2f, 0x73, 0x6b, 0x69, 0x70, 0x6c, 0x69, 0x73, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,