mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/chrislusf/seaweedfs synced 2024-05-04 10:31:06 +02:00
Code Issues Projects Releases Wiki Activity

also migrate jsonpb

Browse source
This commit is contained in:
chrislu 2022-08-17 12:42:03 -07:00
parent eaeb141b09
commit 2b580a7566
15 changed files with 47 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
weed/command/filer_meta_tail.go
View file

@ -2,8 +2,8 @@ package command
import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb"
"google.golang.org/protobuf/jsonpb"
"os"
"path/filepath"
"strings"
@ -88,11 +88,8 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
return false
}
jsonpbMarshaler := jsonpb.Marshaler{
EmitDefaults: false,
}
eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
jsonpbMarshaler.Marshal(os.Stdout, resp)
filer.ProtoToText(os.Stdout, resp)
fmt.Fprintln(os.Stdout)
return nil
}

2
weed/filer/entry_codec.go
View file

@ -21,7 +21,7 @@ func (entry *Entry) DecodeAttributesAndChunks(blob []byte) error {
message := &filer_pb.Entry{}
if err := proto.UnmarshalMerge(blob, message); err != nil {
if err := proto.Unmarshal(blob, message); err != nil {
return fmt.Errorf("decoding value blob for %s: %v", entry.FullPath, err)
}

12
weed/filer/filer_conf.go
View file

@ -13,7 +13,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/viant/ptrie"
"google.golang.org/protobuf/jsonpb"
jsonpb "google.golang.org/protobuf/encoding/protojson"
)
const (
@ -93,7 +93,7 @@ func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*file
func (fc *FilerConf) LoadFromBytes(data []byte) (err error) {
conf := &filer_pb.FilerConf{}
if err := jsonpb.Unmarshal(bytes.NewReader(data), conf); err != nil {
if err := jsonpb.Unmarshal(data, conf); err != nil {
return err
}
@ -181,11 +181,5 @@ func (fc *FilerConf) ToProto() *filer_pb.FilerConf {
}
func (fc *FilerConf) ToText(writer io.Writer) error {
m := jsonpb.Marshaler{
EmitDefaults: false,
Indent: " ",
}
return m.Marshal(writer, fc.ToProto())
return ProtoToText(writer, fc.ToProto())
}

23
weed/filer/s3iam_conf.go
View file

@ -1,17 +1,16 @@
package filer
import (
"bytes"
"fmt"
"io"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
"google.golang.org/protobuf/jsonpb"
jsonpb "google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/proto"
)
func ParseS3ConfigurationFromBytes[T proto.Message](content []byte, config T) error {
if err := jsonpb.Unmarshal(bytes.NewBuffer(content), config); err != nil {
if err := jsonpb.Unmarshal(content, config); err != nil {
return err
}
return nil
@ -19,12 +18,22 @@ func ParseS3ConfigurationFromBytes[T proto.Message](content []byte, config T) er
func ProtoToText(writer io.Writer, config proto.Message) error {
m := jsonpb.Marshaler{
EmitDefaults: false,
Indent: " ",
m := jsonpb.MarshalOptions{
EmitUnpopulated: true,
Indent: " ",
}
return m.Marshal(writer, config)
text, marshalErr := m.Marshal(config)
if marshalErr != nil {
return fmt.Errorf("marshal proto message: %v", marshalErr)
}
_, writeErr := writer.Write(text)
if writeErr != nil {
return fmt.Errorf("fail to write proto message: %v", writeErr)
}
return writeErr
}
// CheckDuplicateAccessKey returns an error message when s3cfg has duplicate access keys

4
weed/mq/broker/broker_segment_serde.go
View file

@ -8,7 +8,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
"google.golang.org/protobuf/jsonpb"
jsonpb "google.golang.org/protobuf/encoding/protojson"
"time"
)
@ -60,7 +60,7 @@ func (broker *MessageQueueBroker) readSegmentOnFiler(segment *mq.Segment) (info
// parse into filer conf object
info = &mq_pb.SegmentInfo{}
if err = jsonpb.Unmarshal(bytes.NewReader(data), info); err != nil {
if err = jsonpb.Unmarshal(data, info); err != nil {
return err
}
found = true

7
weed/notification/aws_sqs/aws_sqs_pub.go
View file

@ -70,7 +70,10 @@ func (k *AwsSqsPub) initialize(awsAccessKeyId, awsSecretAccessKey, region, queue
func (k *AwsSqsPub) SendMessage(key string, message proto.Message) (err error) {
text := proto.MarshalTextString(message)
text, err := proto.Marshal(message)
if err != nil {
return fmt.Errorf("send message marshal %+v: %v", message, err)
}
_, err = k.svc.SendMessage(&sqs.SendMessageInput{
DelaySeconds: aws.Int64(10),
@ -80,7 +83,7 @@ func (k *AwsSqsPub) SendMessage(key string, message proto.Message) (err error) {
StringValue: aws.String(key),
},
},
MessageBody: aws.String(text),
MessageBody: aws.String(string(text)),
QueueUrl: &k.queueUrl,
})

2
weed/pb/proto_read_write_test.go
View file

@ -5,7 +5,7 @@ import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"google.golang.org/protobuf/jsonpb"
jsonpb "google.golang.org/protobuf/encoding/protojson"
)
func TestJsonpMarshalUnmarshal(t *testing.T) {

2
weed/replication/sub/notification_aws_sqs.go
View file

@ -98,7 +98,7 @@ func (k *AwsSqsInput) ReceiveMessage() (key string, message *filer_pb.EventNotif
key = *keyValue.StringValue
text := *result.Messages[0].Body
message = &filer_pb.EventNotification{}
err = proto.UnmarshalText(text, message)
err = proto.Unmarshal([]byte(text), message)
// delete the message
_, err = k.svc.DeleteMessage(&sqs.DeleteMessageInput{

2
weed/s3api/auth_credentials_test.go
View file

@ -5,7 +5,7 @@ import (
"github.com/stretchr/testify/assert"
"testing"
"google.golang.org/protobuf/jsonpb"
jsonpb "google.golang.org/protobuf/encoding/protojson"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
)

20
weed/shell/command_fs_meta_cat.go
View file

@ -2,8 +2,7 @@ package shell
import (
"fmt"
"golang.org/x/exp/slices"
"google.golang.org/protobuf/jsonpb"
"github.com/seaweedfs/seaweedfs/weed/filer"
"google.golang.org/protobuf/proto"
"io"
@ -50,22 +49,7 @@ func (c *commandFsMetaCat) Do(args []string, commandEnv *CommandEnv, writer io.W
return err
}
m := jsonpb.Marshaler{
EmitDefaults: true,
Indent: " ",
}
slices.SortFunc(respLookupEntry.Entry.Chunks, func(a, b *filer_pb.FileChunk) bool {
if a.Offset == b.Offset {
return a.Mtime < b.Mtime
}
return a.Offset < b.Offset
})
text, marshalErr := m.MarshalToString(respLookupEntry.Entry)
if marshalErr != nil {
return fmt.Errorf("marshal meta: %v", marshalErr)
}
fmt.Fprintf(writer, "%s\n", text)
filer.ProtoToText(writer, respLookupEntry.Entry)
bytes, _ := proto.Marshal(respLookupEntry.Entry)
gzippedBytes, _ := util.GzipData(bytes)

10
weed/shell/command_remote_configure.go
View file

@ -9,7 +9,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/jsonpb"
"google.golang.org/protobuf/proto"
"io"
"regexp"
@ -159,15 +158,8 @@ func (c *commandRemoteConfigure) listExistingRemoteStorages(commandEnv *CommandE
conf.TencentSecretKey = strings.Repeat("*", len(conf.TencentSecretKey))
conf.WasabiSecretKey = strings.Repeat("*", len(conf.WasabiSecretKey))
m := jsonpb.Marshaler{
EmitDefaults: false,
Indent: " ",
}
return filer.ProtoToText(writer, conf)
err := m.Marshal(writer, conf)
fmt.Fprintln(writer)
return err
})
}

13
weed/shell/command_remote_mount.go
View file

@ -9,7 +9,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/jsonpb"
"google.golang.org/protobuf/proto"
"io"
"os"
@ -101,17 +100,7 @@ func listExistingRemoteStorageMounts(commandEnv *CommandEnv, writer io.Writer) (
}
func jsonPrintln(writer io.Writer, message proto.Message) error {
if message == nil {
return nil
}
m := jsonpb.Marshaler{
EmitDefaults: false,
Indent: " ",
}
err := m.Marshal(writer, message)
fmt.Fprintln(writer)
return err
return filer.ProtoToText(writer, message)
}
func syncMetadata(commandEnv *CommandEnv, writer io.Writer, dir string, nonEmpty bool, remoteConf *remote_pb.RemoteConf, remote *remote_pb.RemoteStorageLocation) error {

15
weed/storage/volume_info/volume_info.go
View file

@ -1,11 +1,10 @@
package volume_info
import (
"bytes"
"fmt"
"os"
"google.golang.org/protobuf/jsonpb"
jsonpb "google.golang.org/protobuf/encoding/protojson"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
@ -44,7 +43,7 @@ func MaybeLoadVolumeInfo(fileName string) (volumeInfo *volume_server_pb.VolumeIn
}
glog.V(1).Infof("maybeLoadVolumeInfo Unmarshal volume info %v", fileName)
if err = jsonpb.Unmarshal(bytes.NewReader(tierData), volumeInfo); err != nil {
if err = jsonpb.Unmarshal(tierData, volumeInfo); err != nil {
glog.Warningf("unmarshal error: %v", err)
err = fmt.Errorf("unmarshal error: %v", err)
return
@ -65,17 +64,17 @@ func SaveVolumeInfo(fileName string, volumeInfo *volume_server_pb.VolumeInfo) er
return fmt.Errorf("%s not writable", fileName)
}
m := jsonpb.Marshaler{
EmitDefaults: true,
Indent: " ",
m := jsonpb.MarshalOptions{
EmitUnpopulated: true,
Indent: " ",
}
text, marshalErr := m.MarshalToString(volumeInfo)
text, marshalErr := m.Marshal(volumeInfo)
if marshalErr != nil {
return fmt.Errorf("marshal to %s: %v", fileName, marshalErr)
}
writeErr := util.WriteFile(fileName, []byte(text), 0755)
writeErr := util.WriteFile(fileName, text, 0755)
if writeErr != nil {
return fmt.Errorf("fail to write %s : %v", fileName, writeErr)
}

2
weed/util/skiplist/Makefile
View file

@ -3,4 +3,4 @@ all: gen
.PHONY : gen
gen:
protoc skiplist.proto --go_out=plugins=grpc:. --go_opt=paths=source_relative
protoc skiplist.proto --go_out=. --go-grpc_out=. --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative

11
weed/util/skiplist/skiplist.pb.go
View file

@ -1,13 +1,12 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.25.0
// protoc v3.12.3
// protoc-gen-go v1.26.0
// protoc v3.17.3
// source: skiplist.proto
package skiplist
import (
proto "google.golang.org/protobuf/proto"
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
@ -21,10 +20,6 @@ const (
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
// This is a compile-time assertion that a sufficiently up-to-date version
// of the legacy proto package is being used.
const _ = proto.ProtoPackageIsVersion4
type SkipListProto struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@ -325,7 +320,7 @@ var file_skiplist_proto_rawDesc = []byte{
0x22, 0x25, 0x0a, 0x0d, 0x4e, 0x61, 0x6d, 0x65, 0x42, 0x61, 0x74, 0x63, 0x68, 0x44, 0x61, 0x74,
0x61, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c,
0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x42, 0x33, 0x5a, 0x31, 0x67, 0x69, 0x74, 0x68, 0x75,
0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f,
0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f,
0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x75,
0x74, 0x69, 0x6c, 0x2f, 0x73, 0x6b, 0x69, 0x70, 0x6c, 0x69, 0x73, 0x74, 0x62, 0x06, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x33,