diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 5983c84d8..8df46e917 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -100,6 +100,7 @@ message FileChunk { FileId fid = 7; FileId source_fid = 8; bytes cipher_key = 9; + bool is_gzipped = 10; } message FileId { diff --git a/unmaintained/repeated_vacuum/repeated_vacuum.go b/unmaintained/repeated_vacuum/repeated_vacuum.go index ebe5d8225..c2f626f72 100644 --- a/unmaintained/repeated_vacuum/repeated_vacuum.go +++ b/unmaintained/repeated_vacuum/repeated_vacuum.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "flag" "fmt" "log" @@ -31,11 +30,10 @@ func main() { data := make([]byte, 1024) rand.Read(data) - reader := bytes.NewReader(data) targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid) - _, err = operation.Upload(targetUrl, fmt.Sprintf("test%d", i), false, reader, false, "bench/test", nil, assignResult.Auth) + _, err = operation.UploadData(targetUrl, fmt.Sprintf("test%d", i), false, data, false, "bench/test", nil, assignResult.Auth) if err != nil { log.Fatalf("upload: %v", err) } diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index 50a120875..1162bb204 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -38,7 +38,6 @@ type CopyOptions struct { masterClient *wdclient.MasterClient concurrenctFiles *int concurrenctChunks *int - compressionLevel *int grpcDialOption grpc.DialOption masters []string cipher bool @@ -54,7 +53,6 @@ func init() { copy.maxMB = cmdCopy.Flag.Int("maxMB", 32, "split files larger than the limit") copy.concurrenctFiles = cmdCopy.Flag.Int("c", 8, "concurrent file copy goroutines") copy.concurrenctChunks = cmdCopy.Flag.Int("concurrentChunks", 8, "concurrent chunk copy goroutines for each file") - copy.compressionLevel = cmdCopy.Flag.Int("compressionLevel", 9, "local file compression level 1 ~ 9") } var cmdCopy = &Command{ @@ -270,6 +268,10 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err // upload the file content fileName := filepath.Base(f.Name()) mimeType := detectMimeType(f) + data, err := ioutil.ReadAll(f) + if err != nil { + return err + } var chunks []*filer_pb.FileChunk var assignResult *filer_pb.AssignVolumeResponse @@ -303,7 +305,7 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId - uploadResult, err := operation.UploadWithLocalCompressionLevel(targetUrl, fileName, worker.options.cipher, f, false, mimeType, nil, security.EncodedJwt(assignResult.Auth), *worker.options.compressionLevel) + uploadResult, err := operation.UploadData(targetUrl, fileName, worker.options.cipher, data, false, mimeType, nil, security.EncodedJwt(assignResult.Auth)) if err != nil { return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err) } @@ -317,8 +319,9 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err Offset: 0, Size: uint64(uploadResult.Size), Mtime: time.Now().UnixNano(), - ETag: uploadResult.ETag, + ETag: uploadResult.Md5, CipherKey: uploadResult.CipherKey, + IsGzipped: uploadResult.Gzip > 0, }) fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName) @@ -429,6 +432,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File, Mtime: time.Now().UnixNano(), ETag: uploadResult.ETag, CipherKey: uploadResult.CipherKey, + IsGzipped: uploadResult.Gzip > 0, } fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size)) }(i) diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go index 98a965337..711488df1 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer2/filechunks.go @@ -72,6 +72,7 @@ type ChunkView struct { LogicOffset int64 IsFullChunk bool CipherKey []byte + isGzipped bool } func ViewFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views []*ChunkView) { @@ -87,6 +88,7 @@ func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int stop := offset + int64(size) for _, chunk := range visibles { + if chunk.start <= offset && offset < chunk.stop && offset < stop { isFullChunk := chunk.isFullChunk && chunk.start == offset && chunk.stop <= stop views = append(views, &ChunkView{ @@ -96,6 +98,7 @@ func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int LogicOffset: offset, IsFullChunk: isFullChunk, CipherKey: chunk.cipherKey, + isGzipped: chunk.isGzipped, }) offset = min(chunk.stop, stop) } @@ -122,7 +125,7 @@ var bufPool = sync.Pool{ func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.FileChunk) []VisibleInterval { - newV := newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Mtime, true, chunk.CipherKey) + newV := newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Mtime, true, chunk.CipherKey, chunk.IsGzipped) length := len(visibles) if length == 0 { @@ -136,11 +139,11 @@ func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb. logPrintf(" before", visibles) for _, v := range visibles { if v.start < chunk.Offset && chunk.Offset < v.stop { - newVisibles = append(newVisibles, newVisibleInterval(v.start, chunk.Offset, v.fileId, v.modifiedTime, false, v.cipherKey)) + newVisibles = append(newVisibles, newVisibleInterval(v.start, chunk.Offset, v.fileId, v.modifiedTime, false, v.cipherKey, v.isGzipped)) } chunkStop := chunk.Offset + int64(chunk.Size) if v.start < chunkStop && chunkStop < v.stop { - newVisibles = append(newVisibles, newVisibleInterval(chunkStop, v.stop, v.fileId, v.modifiedTime, false, v.cipherKey)) + newVisibles = append(newVisibles, newVisibleInterval(chunkStop, v.stop, v.fileId, v.modifiedTime, false, v.cipherKey, v.isGzipped)) } if chunkStop <= v.start || v.stop <= chunk.Offset { newVisibles = append(newVisibles, v) @@ -171,6 +174,7 @@ func NonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []Vi var newVisibles []VisibleInterval for _, chunk := range chunks { + newVisibles = MergeIntoVisibles(visibles, newVisibles, chunk) t := visibles[:0] visibles = newVisibles @@ -193,9 +197,10 @@ type VisibleInterval struct { fileId string isFullChunk bool cipherKey []byte + isGzipped bool } -func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool, cipherKey []byte) VisibleInterval { +func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool, cipherKey []byte, isGzipped bool) VisibleInterval { return VisibleInterval{ start: start, stop: stop, @@ -203,6 +208,7 @@ func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, is modifiedTime: modifiedTime, isFullChunk: isFullChunk, cipherKey: cipherKey, + isGzipped: isGzipped, } } diff --git a/weed/filer2/filer_client_util.go b/weed/filer2/filer_client_util.go index 98bb57779..1c1fa6a5b 100644 --- a/weed/filer2/filer_client_util.go +++ b/weed/filer2/filer_client_util.go @@ -70,7 +70,7 @@ func ReadIntoBuffer(filerClient FilerClient, fullFilePath FullPath, buff []byte, volumeServerAddress := filerClient.AdjustedUrl(locations.Locations[0].Url) var n int64 - n, err = util.ReadUrl(fmt.Sprintf("http://%s/%s", volumeServerAddress, chunkView.FileId), chunkView.CipherKey, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), buff[chunkView.LogicOffset-baseOffset:chunkView.LogicOffset-baseOffset+int64(chunkView.Size)]) + n, err = util.ReadUrl(fmt.Sprintf("http://%s/%s", volumeServerAddress, chunkView.FileId), chunkView.CipherKey, chunkView.isGzipped, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), buff[chunkView.LogicOffset-baseOffset:chunkView.LogicOffset-baseOffset+int64(chunkView.Size)]) if err != nil { diff --git a/weed/filer2/stream.go b/weed/filer2/stream.go index 0a1f943ea..381d99144 100644 --- a/weed/filer2/stream.go +++ b/weed/filer2/stream.go @@ -26,8 +26,9 @@ func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*f } for _, chunkView := range chunkViews { + urlString := fileId2Url[chunkView.FileId] - err := util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), func(data []byte) { + err := util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.isGzipped, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), func(data []byte) { w.Write(data) }) if err != nil { diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 1b11ddb9e..483229b3f 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -35,8 +35,6 @@ var _ = fs.NodeForgetter(&Dir{}) func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error { - glog.V(3).Infof("dir Attr %s, existing attr: %+v", dir.Path, attr) - // https://github.com/bazil/fuse/issues/196 attr.Valid = time.Second diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index ff3b8f885..7e33c97a7 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -191,6 +191,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64, Mtime: time.Now().UnixNano(), ETag: uploadResult.ETag, CipherKey: uploadResult.CipherKey, + IsGzipped: uploadResult.Gzip > 0, }, nil } diff --git a/weed/operation/submit.go b/weed/operation/submit.go index 8c7ed5d7b..5e4dc4374 100644 --- a/weed/operation/submit.go +++ b/weed/operation/submit.go @@ -1,8 +1,6 @@ package operation import ( - "bytes" - "google.golang.org/grpc" "io" "mime" "net/url" @@ -11,6 +9,8 @@ import ( "strconv" "strings" + "google.golang.org/grpc" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/security" ) @@ -52,7 +52,7 @@ func SubmitFiles(master string, grpcDialOption grpc.DialOption, files []FilePart } ret, err := Assign(master, grpcDialOption, ar) if err != nil { - for index, _ := range files { + for index := range files { results[index].Error = err.Error() } return results, err @@ -214,12 +214,11 @@ func upload_chunked_file_manifest(fileUrl string, manifest *ChunkManifest, jwt s if e != nil { return e } - bufReader := bytes.NewReader(buf) glog.V(4).Info("Uploading chunks manifest ", manifest.Name, " to ", fileUrl, "...") u, _ := url.Parse(fileUrl) q := u.Query() q.Set("cm", "true") u.RawQuery = q.Encode() - _, e = Upload(u.String(), manifest.Name, false, bufReader, false, "application/json", nil, jwt) + _, e = UploadData(u.String(), manifest.Name, false, buf, false, "application/json", nil, jwt) return e } diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index e27ff4da7..a83317975 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -3,7 +3,7 @@ package operation import ( "bytes" "compress/flate" - "compress/gzip" + "crypto/md5" "encoding/json" "errors" "fmt" @@ -28,6 +28,8 @@ type UploadResult struct { ETag string `json:"eTag,omitempty"` CipherKey []byte `json:"cipherKey,omitempty"` Mime string `json:"mime,omitempty"` + Gzip uint32 `json:"gzip,omitempty"` + Md5 string `json:"md5,omitempty"` } var ( @@ -43,22 +45,28 @@ func init() { var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") // Upload sends a POST request to a volume server to upload the content with adjustable compression level -func UploadWithLocalCompressionLevel(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt, compressionLevel int) (*UploadResult, error) { - if compressionLevel < 1 { - compressionLevel = 1 +func UploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) { + hash := md5.New() + hash.Write(data) + uploadResult, err = doUploadData(uploadUrl, filename, cipher, data, isInputGzipped, mtype, pairMap, jwt) + if uploadResult != nil { + uploadResult.Md5 = fmt.Sprintf("%x", hash.Sum(nil)) } - if compressionLevel > 9 { - compressionLevel = 9 - } - return doUpload(uploadUrl, filename, cipher, reader, isInputGzipped, mtype, pairMap, compressionLevel, jwt) + return } // Upload sends a POST request to a volume server to upload the content with fast compression -func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) { - return doUpload(uploadUrl, filename, cipher, reader, isInputGzipped, mtype, pairMap, flate.BestSpeed, jwt) +func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) { + hash := md5.New() + reader = io.TeeReader(reader, hash) + uploadResult, err = doUpload(uploadUrl, filename, cipher, reader, isInputGzipped, mtype, pairMap, flate.BestSpeed, jwt) + if uploadResult != nil { + uploadResult.Md5 = fmt.Sprintf("%x", hash.Sum(nil)) + } + return } -func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputGzipped bool, mtype string, pairMap map[string]string, compression int, jwt security.EncodedJwt) (*UploadResult, error) { +func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) { contentIsGzipped := isInputGzipped shouldGzipNow := false if !isInputGzipped { @@ -67,33 +75,131 @@ func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, contentIsGzipped = true } } - // encrypt data - var cipherKey util.CipherKey + var clearDataLen int - var err error - if cipher { - cipherKey, reader, clearDataLen, _, err = util.EncryptReader(reader) - if err != nil { - return nil, err + + // gzip if possible + // this could be double copying + clearDataLen = len(data) + if shouldGzipNow { + data, err = util.GzipData(data) + } else if isInputGzipped { + // just to get the clear data length + clearData, err := util.UnGzipData(data) + if err == nil { + clearDataLen = len(clearData) } } - // upload data - uploadResult, err := upload_content(uploadUrl, func(w io.Writer) (err error) { - if shouldGzipNow { - gzWriter, _ := gzip.NewWriterLevel(w, compression) - _, err = io.Copy(gzWriter, reader) - gzWriter.Close() - } else { - _, err = io.Copy(w, reader) - } - return - }, filename, contentIsGzipped, mtype, pairMap, jwt) + if cipher { + // encrypt(gzip(data)) - // remember cipher key - if uploadResult != nil && cipherKey != nil { - uploadResult.CipherKey = cipherKey - uploadResult.Size = uint32(clearDataLen) + // encrypt + cipherKey := util.GenCipherKey() + encryptedData, encryptionErr := util.Encrypt(data, cipherKey) + if encryptionErr != nil { + err = fmt.Errorf("encrypt input: %v", encryptionErr) + return + } + + // upload data + uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) { + _, err = w.Write(encryptedData) + return + }, "", false, "", nil, jwt) + if uploadResult != nil { + uploadResult.Name = filename + uploadResult.Mime = mtype + uploadResult.CipherKey = cipherKey + } + } else { + // upload data + uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) { + _, err = w.Write(data) + return + }, filename, contentIsGzipped, mtype, pairMap, jwt) + } + + uploadResult.Size = uint32(clearDataLen) + if contentIsGzipped { + uploadResult.Gzip = 1 + } + + return uploadResult, err +} + +func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputGzipped bool, mtype string, pairMap map[string]string, compression int, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) { + contentIsGzipped := isInputGzipped + shouldGzipNow := false + if !isInputGzipped { + if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); mtype == "" || iAmSure && shouldBeZipped { + shouldGzipNow = true + contentIsGzipped = true + } + } + + var clearDataLen int + + // gzip if possible + // this could be double copying + data, readErr := ioutil.ReadAll(reader) + if readErr != nil { + err = fmt.Errorf("read input: %v", readErr) + return + } + clearDataLen = len(data) + if shouldGzipNow { + data, err = util.GzipData(data) + } else if isInputGzipped { + // just to get the clear data length + clearData, err := util.UnGzipData(data) + if err == nil { + clearDataLen = len(clearData) + } + } + + println("data size", clearDataLen) + + if cipher { + // encrypt(gzip(data)) + + // encrypt + cipherKey := util.GenCipherKey() + encryptedData, encryptionErr := util.Encrypt(data, cipherKey) + if encryptionErr != nil { + err = fmt.Errorf("encrypt input: %v", encryptionErr) + return + } + println("encrypted data size", len(encryptedData)) + + // upload data + uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) { + n, err := w.Write(encryptedData) + println("writtern data size", n) + return + }, "", false, "", nil, jwt) + if uploadResult != nil { + uploadResult.Name = filename + uploadResult.Mime = mtype + uploadResult.CipherKey = cipherKey + uploadResult.Size = uint32(clearDataLen) + } + } else { + // upload data + uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) { + n, err := w.Write(data) + println("written data size", n) + return + }, filename, contentIsGzipped, mtype, pairMap, jwt) + } + + if uploadResult == nil { + return + } + + uploadResult.Size = uint32(clearDataLen) + if contentIsGzipped { + uploadResult.Gzip = 1 } return uploadResult, err diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index 5983c84d8..8df46e917 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -100,6 +100,7 @@ message FileChunk { FileId fid = 7; FileId source_fid = 8; bytes cipher_key = 9; + bool is_gzipped = 10; } message FileId { diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index 92a12321c..9cf659ece 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -288,6 +288,7 @@ type FileChunk struct { Fid *FileId `protobuf:"bytes,7,opt,name=fid" json:"fid,omitempty"` SourceFid *FileId `protobuf:"bytes,8,opt,name=source_fid,json=sourceFid" json:"source_fid,omitempty"` CipherKey []byte `protobuf:"bytes,9,opt,name=cipher_key,json=cipherKey,proto3" json:"cipher_key,omitempty"` + IsGzipped bool `protobuf:"varint,10,opt,name=is_gzipped,json=isGzipped" json:"is_gzipped,omitempty"` } func (m *FileChunk) Reset() { *m = FileChunk{} } @@ -358,6 +359,13 @@ func (m *FileChunk) GetCipherKey() []byte { return nil } +func (m *FileChunk) GetIsGzipped() bool { + if m != nil { + return m.IsGzipped + } + return false +} + type FileId struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` FileKey uint64 `protobuf:"varint,2,opt,name=file_key,json=fileKey" json:"file_key,omitempty"` @@ -1610,114 +1618,115 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1742 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0x4f, 0x6f, 0xdc, 0xc6, - 0x15, 0x37, 0xf7, 0x3f, 0xdf, 0xee, 0x3a, 0xd2, 0xac, 0x9c, 0xac, 0xd7, 0x92, 0xab, 0xd0, 0x75, - 0xaa, 0xc2, 0x86, 0x6a, 0xa8, 0x39, 0x24, 0x4d, 0x7b, 0xb0, 0x65, 0xb9, 0x30, 0x62, 0x3b, 0x2e, - 0x65, 0x17, 0x29, 0x0a, 0x94, 0xa0, 0xc8, 0xd9, 0xd5, 0x54, 0x24, 0x87, 0x19, 0x0e, 0x2d, 0xb9, - 0xdf, 0xa3, 0x97, 0x02, 0x3d, 0xf4, 0x7b, 0x14, 0xbd, 0x14, 0x05, 0xfa, 0x39, 0x7a, 0xec, 0xa1, - 0x9f, 0xa1, 0x98, 0x37, 0x24, 0x77, 0xb8, 0x5c, 0x49, 0x49, 0x8b, 0xdc, 0x38, 0xef, 0xdf, 0xbc, - 0xf9, 0xbd, 0xbf, 0xbb, 0x30, 0x9c, 0xb3, 0x88, 0x8a, 0xfd, 0x54, 0x70, 0xc9, 0xc9, 0x00, 0x0f, - 0x5e, 0x7a, 0xe2, 0x7c, 0x05, 0x77, 0x5e, 0x70, 0x7e, 0x96, 0xa7, 0x4f, 0x99, 0xa0, 0x81, 0xe4, - 0xe2, 0xfd, 0x51, 0x22, 0xc5, 0x7b, 0x97, 0x7e, 0x93, 0xd3, 0x4c, 0x92, 0x6d, 0xb0, 0xc3, 0x92, - 0x31, 0xb5, 0x76, 0xad, 0x3d, 0xdb, 0x5d, 0x12, 0x08, 0x81, 0x4e, 0xe2, 0xc7, 0x74, 0xda, 0x42, - 0x06, 0x7e, 0x3b, 0x47, 0xb0, 0xbd, 0xde, 0x60, 0x96, 0xf2, 0x24, 0xa3, 0xe4, 0x3e, 0x74, 0xa9, - 0x22, 0xa0, 0xb5, 0xe1, 0xc1, 0x07, 0xfb, 0xa5, 0x2b, 0xfb, 0x5a, 0x4e, 0x73, 0x9d, 0xbf, 0x59, - 0x40, 0x5e, 0xb0, 0x4c, 0x2a, 0x22, 0xa3, 0xd9, 0xb7, 0xf3, 0xe7, 0x43, 0xe8, 0xa5, 0x82, 0xce, - 0xd9, 0x45, 0xe1, 0x51, 0x71, 0x22, 0x0f, 0x61, 0x33, 0x93, 0xbe, 0x90, 0xcf, 0x04, 0x8f, 0x9f, - 0xb1, 0x88, 0xbe, 0x52, 0x4e, 0xb7, 0x51, 0xa4, 0xc9, 0x20, 0xfb, 0x40, 0x58, 0x12, 0x44, 0x79, - 0xc6, 0xde, 0xd1, 0xe3, 0x92, 0x3b, 0xed, 0xec, 0x5a, 0x7b, 0x03, 0x77, 0x0d, 0x87, 0x6c, 0x41, - 0x37, 0x62, 0x31, 0x93, 0xd3, 0xee, 0xae, 0xb5, 0x37, 0x76, 0xf5, 0xc1, 0xf9, 0x39, 0x4c, 0x6a, - 0xfe, 0x7f, 0xb7, 0xe7, 0xff, 0xb9, 0x05, 0x5d, 0x24, 0x54, 0x18, 0x5b, 0x4b, 0x8c, 0xc9, 0xc7, - 0x30, 0x62, 0x99, 0xb7, 0x04, 0xa2, 0x85, 0xbe, 0x0d, 0x59, 0x56, 0x61, 0x4e, 0x1e, 0x40, 0x2f, - 0x38, 0xcd, 0x93, 0xb3, 0x6c, 0xda, 0xde, 0x6d, 0xef, 0x0d, 0x0f, 0x26, 0xcb, 0x8b, 0xd4, 0x43, - 0x0f, 0x15, 0xcf, 0x2d, 0x44, 0xc8, 0x67, 0x00, 0xbe, 0x94, 0x82, 0x9d, 0xe4, 0x92, 0x66, 0xf8, - 0xd2, 0xe1, 0xc1, 0xd4, 0x50, 0xc8, 0x33, 0xfa, 0xb8, 0xe2, 0xbb, 0x86, 0x2c, 0xf9, 0x1c, 0x06, - 0xf4, 0x42, 0xd2, 0x24, 0xa4, 0xe1, 0xb4, 0x8b, 0x17, 0xed, 0xac, 0xbc, 0x68, 0xff, 0xa8, 0xe0, - 0xeb, 0xf7, 0x55, 0xe2, 0xb3, 0x2f, 0x60, 0x5c, 0x63, 0x91, 0x0d, 0x68, 0x9f, 0xd1, 0x32, 0xaa, - 0xea, 0x53, 0x21, 0xfb, 0xce, 0x8f, 0x72, 0x9d, 0x60, 0x23, 0x57, 0x1f, 0x7e, 0xd6, 0xfa, 0xcc, - 0x72, 0x9e, 0x82, 0xfd, 0x2c, 0x8f, 0xa2, 0x4a, 0x31, 0x64, 0xa2, 0x54, 0x0c, 0x99, 0x58, 0xa2, - 0xdc, 0xba, 0x12, 0xe5, 0xbf, 0x5a, 0xb0, 0x79, 0xf4, 0x8e, 0x26, 0xf2, 0x15, 0x97, 0x6c, 0xce, - 0x02, 0x5f, 0x32, 0x9e, 0x90, 0x87, 0x60, 0xf3, 0x28, 0xf4, 0xae, 0x0c, 0xd3, 0x80, 0x47, 0x85, - 0xd7, 0x0f, 0xc1, 0x4e, 0xe8, 0xb9, 0x77, 0xe5, 0x75, 0x83, 0x84, 0x9e, 0x6b, 0xe9, 0x7b, 0x30, - 0x0e, 0x69, 0x44, 0x25, 0xf5, 0xaa, 0xe8, 0xa8, 0xd0, 0x8d, 0x34, 0xf1, 0x50, 0x87, 0xe3, 0x13, - 0xf8, 0x40, 0x99, 0x4c, 0x7d, 0x41, 0x13, 0xe9, 0xa5, 0xbe, 0x3c, 0xc5, 0x98, 0xd8, 0xee, 0x38, - 0xa1, 0xe7, 0xaf, 0x91, 0xfa, 0xda, 0x97, 0xa7, 0xce, 0x1f, 0x5b, 0x60, 0x57, 0xc1, 0x24, 0x1f, - 0x41, 0x5f, 0x5d, 0xeb, 0xb1, 0xb0, 0x40, 0xa2, 0xa7, 0x8e, 0xcf, 0x43, 0x55, 0x15, 0x7c, 0x3e, - 0xcf, 0xa8, 0x44, 0xf7, 0xda, 0x6e, 0x71, 0x52, 0x99, 0x95, 0xb1, 0x3f, 0xe8, 0x42, 0xe8, 0xb8, - 0xf8, 0xad, 0x10, 0x8f, 0x25, 0x8b, 0x29, 0x5e, 0xd8, 0x76, 0xf5, 0x81, 0x4c, 0xa0, 0x4b, 0x3d, - 0xe9, 0x2f, 0x30, 0xc3, 0x6d, 0xb7, 0x43, 0xdf, 0xf8, 0x0b, 0xf2, 0x43, 0xb8, 0x99, 0xf1, 0x5c, - 0x04, 0xd4, 0x2b, 0xaf, 0xed, 0x21, 0x77, 0xa4, 0xa9, 0xcf, 0xf4, 0xe5, 0x0e, 0xb4, 0xe7, 0x2c, - 0x9c, 0xf6, 0x11, 0x98, 0x8d, 0x7a, 0x12, 0x3e, 0x0f, 0x5d, 0xc5, 0x24, 0x3f, 0x01, 0xa8, 0x2c, - 0x85, 0xd3, 0xc1, 0x25, 0xa2, 0x76, 0x69, 0x37, 0x24, 0x3b, 0x00, 0x01, 0x4b, 0x4f, 0xa9, 0xf0, - 0x54, 0xc2, 0xd8, 0x98, 0x1c, 0xb6, 0xa6, 0x7c, 0x49, 0xdf, 0x3b, 0x5f, 0x43, 0xaf, 0xb8, 0xfd, - 0x0e, 0xd8, 0xef, 0x78, 0x94, 0xc7, 0x15, 0x2a, 0x63, 0x77, 0xa0, 0x09, 0xcf, 0x43, 0x72, 0x1b, - 0xb0, 0x0d, 0xa2, 0x8d, 0x16, 0x62, 0x80, 0x00, 0x7e, 0x49, 0xb1, 0x91, 0x04, 0x9c, 0x9f, 0x31, - 0x0d, 0x4e, 0xdf, 0x2d, 0x4e, 0xce, 0x7f, 0x5a, 0x70, 0xb3, 0x5e, 0x0d, 0xea, 0x0a, 0xb4, 0x82, - 0x50, 0x5a, 0x68, 0x06, 0xcd, 0x1e, 0xd7, 0xe0, 0x6c, 0x99, 0x70, 0x96, 0x2a, 0x31, 0x0f, 0xf5, - 0x05, 0x63, 0xad, 0xf2, 0x92, 0x87, 0x54, 0x25, 0x73, 0xce, 0x42, 0xc4, 0x7f, 0xec, 0xaa, 0x4f, - 0x45, 0x59, 0xb0, 0xb0, 0xe8, 0x2e, 0xea, 0x13, 0xdd, 0x13, 0x68, 0xb7, 0xa7, 0x23, 0xaa, 0x4f, - 0x2a, 0xa2, 0xb1, 0xa2, 0xf6, 0x75, 0x98, 0xd4, 0x37, 0xd9, 0x85, 0xa1, 0xa0, 0x69, 0x54, 0x24, - 0x37, 0xa2, 0x6b, 0xbb, 0x26, 0x89, 0xdc, 0x05, 0x08, 0x78, 0x14, 0xd1, 0x00, 0x05, 0x6c, 0x14, - 0x30, 0x28, 0x2a, 0xb1, 0xa4, 0x8c, 0xbc, 0x8c, 0x06, 0x53, 0xd8, 0xb5, 0xf6, 0xba, 0x6e, 0x4f, - 0xca, 0xe8, 0x98, 0x06, 0xea, 0x1d, 0x79, 0x46, 0x85, 0x87, 0xfd, 0x69, 0x88, 0x7a, 0x03, 0x45, - 0xc0, 0x2e, 0xba, 0x03, 0xb0, 0x10, 0x3c, 0x4f, 0x35, 0x77, 0xb4, 0xdb, 0x56, 0xad, 0x1a, 0x29, - 0xc8, 0xbe, 0x0f, 0x37, 0xb3, 0xf7, 0x71, 0xc4, 0x92, 0x33, 0x4f, 0xfa, 0x62, 0x41, 0xe5, 0x74, - 0xac, 0x53, 0xbc, 0xa0, 0xbe, 0x41, 0xa2, 0x93, 0x02, 0x39, 0x14, 0xd4, 0x97, 0xf4, 0x3b, 0x4c, - 0xa5, 0x6f, 0x57, 0xfc, 0xe4, 0x16, 0xf4, 0xb8, 0x47, 0x2f, 0x82, 0xa8, 0xa8, 0xc1, 0x2e, 0x3f, - 0xba, 0x08, 0x22, 0xe7, 0x01, 0x4c, 0x6a, 0x37, 0x16, 0x7d, 0x7b, 0x0b, 0xba, 0x54, 0x08, 0x5e, - 0x76, 0x19, 0x7d, 0x70, 0x7e, 0x03, 0xe4, 0x6d, 0x1a, 0x7e, 0x1f, 0xee, 0x39, 0xb7, 0x60, 0x52, - 0x33, 0xad, 0xfd, 0x70, 0xfe, 0x61, 0x01, 0x79, 0x8a, 0xcd, 0xe2, 0xff, 0x9b, 0xd3, 0xaa, 0x7c, - 0xd5, 0x0c, 0xd1, 0xcd, 0x28, 0xf4, 0xa5, 0x5f, 0x4c, 0xb8, 0x11, 0xcb, 0xb4, 0xfd, 0xa7, 0xbe, - 0xf4, 0x8b, 0x49, 0x23, 0x68, 0x90, 0x0b, 0x35, 0xf4, 0x30, 0x09, 0x71, 0xd2, 0xb8, 0x25, 0x89, - 0x7c, 0x0a, 0x1f, 0xb2, 0x45, 0xc2, 0x05, 0x5d, 0x8a, 0x79, 0x1a, 0xaa, 0x1e, 0x0a, 0x6f, 0x69, - 0x6e, 0xa5, 0x70, 0x84, 0xc8, 0x3d, 0x80, 0x49, 0xed, 0x19, 0x57, 0xc2, 0xfc, 0x27, 0x0b, 0xa6, - 0x8f, 0x25, 0x8f, 0x59, 0xe0, 0x52, 0xe5, 0x7c, 0xed, 0xe9, 0xf7, 0x60, 0xac, 0xda, 0xf5, 0xea, - 0xf3, 0x47, 0x3c, 0x0a, 0x97, 0xe3, 0xf0, 0x36, 0xa8, 0x8e, 0xed, 0x19, 0x28, 0xf4, 0x79, 0x14, - 0x62, 0x26, 0xde, 0x03, 0xd5, 0x56, 0x0d, 0x7d, 0xbd, 0x18, 0x8c, 0x12, 0x7a, 0x5e, 0xd3, 0x57, - 0x42, 0xa8, 0xaf, 0x7b, 0x71, 0x3f, 0xa1, 0xe7, 0x4a, 0xdf, 0xb9, 0x03, 0xb7, 0xd7, 0xf8, 0x56, - 0x84, 0xeb, 0x9f, 0x16, 0x4c, 0x1e, 0x67, 0x19, 0x5b, 0x24, 0xbf, 0xc6, 0xb6, 0x53, 0x3a, 0xbd, - 0x05, 0xdd, 0x80, 0xe7, 0x89, 0x44, 0x67, 0xbb, 0xae, 0x3e, 0xac, 0x54, 0x62, 0xab, 0x51, 0x89, - 0x2b, 0xb5, 0xdc, 0x6e, 0xd6, 0xb2, 0x51, 0xab, 0x9d, 0x5a, 0xad, 0xfe, 0x00, 0x86, 0x2a, 0xc8, - 0x5e, 0x40, 0x13, 0x49, 0x45, 0xd1, 0xc8, 0x41, 0x91, 0x0e, 0x91, 0xa2, 0x04, 0xcc, 0x81, 0xa3, - 0x7b, 0x39, 0xa4, 0xcb, 0x69, 0xf3, 0x2f, 0x0b, 0xb6, 0xea, 0x4f, 0x29, 0x62, 0x76, 0xe9, 0xe0, - 0x51, 0xad, 0x4c, 0x44, 0xc5, 0x3b, 0xd4, 0xa7, 0x6a, 0x0a, 0x69, 0x7e, 0x12, 0xb1, 0xc0, 0x53, - 0x0c, 0xed, 0xbf, 0xad, 0x29, 0x6f, 0x45, 0xb4, 0x44, 0xa5, 0x63, 0xa2, 0x42, 0xa0, 0xe3, 0xe7, - 0xf2, 0xb4, 0x1c, 0x3e, 0xea, 0x7b, 0x05, 0xa9, 0xde, 0x75, 0x48, 0xf5, 0x9b, 0x48, 0x55, 0x99, - 0x36, 0x30, 0x33, 0xed, 0x53, 0x98, 0xe8, 0xed, 0xb5, 0x1e, 0xae, 0x1d, 0x80, 0x6a, 0x8e, 0x64, - 0x53, 0x4b, 0x37, 0xb3, 0x72, 0x90, 0x64, 0xce, 0x2f, 0xc0, 0x7e, 0xc1, 0xb5, 0xdd, 0x8c, 0x3c, - 0x02, 0x3b, 0x2a, 0x0f, 0x28, 0x3a, 0x3c, 0x20, 0xcb, 0x1a, 0x2f, 0xe5, 0xdc, 0xa5, 0x90, 0xf3, - 0x05, 0x0c, 0x4a, 0x72, 0x89, 0x99, 0x75, 0x19, 0x66, 0xad, 0x15, 0xcc, 0x9c, 0xbf, 0x5b, 0xb0, - 0x55, 0x77, 0xb9, 0x08, 0xcb, 0x5b, 0x18, 0x57, 0x57, 0x78, 0xb1, 0x9f, 0x16, 0xbe, 0x3c, 0x32, - 0x7d, 0x69, 0xaa, 0x55, 0x0e, 0x66, 0x2f, 0xfd, 0x54, 0xe7, 0xf2, 0x28, 0x32, 0x48, 0xb3, 0x37, - 0xb0, 0xd9, 0x10, 0x59, 0xb3, 0xba, 0xfd, 0xd8, 0x5c, 0xdd, 0x6a, 0xeb, 0x67, 0xa5, 0x6d, 0xee, - 0x73, 0x9f, 0xc3, 0x47, 0xba, 0x1d, 0x1c, 0x56, 0x31, 0x2c, 0xb1, 0xaf, 0x87, 0xda, 0x5a, 0x0d, - 0xb5, 0x33, 0x83, 0x69, 0x53, 0xb5, 0x28, 0xbf, 0x05, 0x6c, 0x1e, 0x4b, 0x5f, 0xb2, 0x4c, 0xb2, - 0xa0, 0xfa, 0x0d, 0xb1, 0x92, 0x1b, 0xd6, 0x75, 0x13, 0xb1, 0x59, 0x87, 0x1b, 0xd0, 0x96, 0xb2, - 0xcc, 0x5f, 0xf5, 0xa9, 0xa2, 0x40, 0xcc, 0x9b, 0x8a, 0x18, 0x7c, 0x0f, 0x57, 0xa9, 0x7c, 0x90, - 0x5c, 0xfa, 0x91, 0xde, 0x38, 0x3a, 0xb8, 0x71, 0xd8, 0x48, 0xc1, 0x95, 0x43, 0x0f, 0xe5, 0x50, - 0x73, 0xbb, 0x7a, 0x1f, 0x51, 0x04, 0x64, 0xee, 0x00, 0x60, 0xa9, 0xea, 0x2a, 0xeb, 0x69, 0x5d, - 0x45, 0x39, 0x54, 0x04, 0xe7, 0x2e, 0x6c, 0xff, 0x92, 0x4a, 0xb5, 0x3b, 0x89, 0x43, 0x9e, 0xcc, - 0xd9, 0x22, 0x17, 0xbe, 0x11, 0x0a, 0xe7, 0xdf, 0x16, 0xec, 0x5c, 0x22, 0x50, 0x3c, 0x78, 0x0a, - 0xfd, 0xd8, 0xcf, 0x24, 0x15, 0x65, 0x95, 0x94, 0xc7, 0x55, 0x28, 0x5a, 0xd7, 0x41, 0xd1, 0x6e, - 0x40, 0x71, 0x0b, 0x7a, 0xb1, 0x7f, 0xe1, 0xc5, 0x27, 0xc5, 0x72, 0xd4, 0x8d, 0xfd, 0x8b, 0x97, - 0x27, 0xd8, 0xd9, 0x98, 0xf0, 0x4e, 0xf2, 0xe0, 0x8c, 0xca, 0xac, 0xea, 0x6c, 0x4c, 0x3c, 0xd1, - 0x14, 0xf5, 0x68, 0x25, 0xf0, 0x4d, 0x4e, 0x73, 0x9a, 0x15, 0xbd, 0x42, 0x0d, 0xc7, 0x5f, 0x21, - 0x01, 0x97, 0x29, 0x5c, 0x1d, 0xb1, 0x4b, 0x0c, 0xdc, 0xe2, 0x74, 0xf0, 0x97, 0x01, 0x8c, 0x8e, - 0xa9, 0x7f, 0x4e, 0x69, 0x88, 0x0f, 0x26, 0x8b, 0xb2, 0xd0, 0xea, 0xbf, 0x6c, 0xc9, 0xfd, 0xd5, - 0x8a, 0x5a, 0xfb, 0x53, 0x7a, 0xf6, 0xc9, 0x75, 0x62, 0x45, 0xce, 0xde, 0x20, 0xaf, 0x60, 0x68, - 0xfc, 0x74, 0x24, 0xdb, 0x86, 0x62, 0xe3, 0x17, 0xf1, 0x6c, 0xe7, 0x12, 0x6e, 0x69, 0xed, 0x91, - 0x45, 0x5e, 0xc0, 0xd0, 0x58, 0x69, 0x4c, 0x7b, 0xcd, 0xdd, 0xca, 0xb4, 0xb7, 0x66, 0x0f, 0x72, - 0x6e, 0x28, 0x6b, 0xc6, 0x62, 0x62, 0x5a, 0x6b, 0xae, 0x42, 0xa6, 0xb5, 0x75, 0xdb, 0x0c, 0x5a, - 0x33, 0xf6, 0x00, 0xd3, 0x5a, 0x73, 0xcb, 0x31, 0xad, 0xad, 0x59, 0x1e, 0x9c, 0x1b, 0xe4, 0x6b, - 0x98, 0x1c, 0x4b, 0x41, 0xfd, 0x78, 0xc9, 0x5e, 0x41, 0xf0, 0x7f, 0xb0, 0xba, 0x67, 0x3d, 0xb2, - 0xc8, 0xef, 0x60, 0xb3, 0x31, 0xe5, 0x89, 0xb3, 0xd4, 0xbc, 0x6c, 0x3d, 0x99, 0xdd, 0xbb, 0x52, - 0xa6, 0xf2, 0xfc, 0x2b, 0x18, 0x99, 0xc3, 0x95, 0x18, 0x4e, 0xad, 0xd9, 0x1f, 0x66, 0x77, 0x2f, - 0x63, 0x9b, 0x06, 0xcd, 0xfe, 0x6e, 0x1a, 0x5c, 0x33, 0xe1, 0x4c, 0x83, 0xeb, 0xc6, 0x82, 0x73, - 0x83, 0xfc, 0x16, 0x36, 0x56, 0xfb, 0x2c, 0xf9, 0x78, 0x15, 0xba, 0x46, 0xfb, 0x9e, 0x39, 0x57, - 0x89, 0x54, 0xc6, 0x9f, 0x03, 0x2c, 0xdb, 0x27, 0xb9, 0xb3, 0xd4, 0x69, 0xb4, 0xef, 0xd9, 0xf6, - 0x7a, 0x66, 0x65, 0xea, 0xf7, 0x70, 0x6b, 0x6d, 0x8f, 0x22, 0x46, 0x01, 0x5e, 0xd5, 0xe5, 0x66, - 0x3f, 0xba, 0x56, 0xae, 0xbc, 0xeb, 0xc9, 0x5d, 0xd8, 0xc8, 0x74, 0x8b, 0x98, 0x67, 0xfb, 0x41, - 0xc4, 0x68, 0x22, 0x9f, 0x00, 0x6a, 0xbc, 0x16, 0x5c, 0xf2, 0x93, 0x1e, 0xfe, 0xdd, 0xf6, 0xd3, - 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xf6, 0xb5, 0x1d, 0x19, 0x7d, 0x13, 0x00, 0x00, + // 1759 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0x4f, 0x6f, 0xdb, 0xc8, + 0x15, 0x0f, 0x25, 0x4b, 0x16, 0x9f, 0xa4, 0xac, 0x3d, 0x72, 0xb2, 0x8a, 0x62, 0xa7, 0x5e, 0xa6, + 0xd9, 0xba, 0x48, 0xe0, 0x06, 0xee, 0x1e, 0x76, 0xbb, 0xed, 0x21, 0x71, 0x9c, 0x45, 0xd0, 0x24, + 0x9b, 0xd2, 0x49, 0xb1, 0x45, 0x81, 0x12, 0x34, 0x39, 0x96, 0xa7, 0xa6, 0x38, 0xdc, 0x99, 0x61, + 0x6c, 0xef, 0x47, 0x29, 0xd0, 0x43, 0xbf, 0x43, 0x8f, 0x45, 0x2f, 0x45, 0x81, 0x7e, 0x8e, 0x1e, + 0x7b, 0xe8, 0x67, 0x28, 0xe6, 0x0d, 0x49, 0x0d, 0x45, 0xd9, 0xde, 0xed, 0x62, 0x6f, 0x9c, 0xf7, + 0x6f, 0xde, 0xfc, 0xde, 0x5f, 0x09, 0xfa, 0xc7, 0x2c, 0xa1, 0x62, 0x37, 0x13, 0x5c, 0x71, 0xd2, + 0xc3, 0x43, 0x90, 0x1d, 0x79, 0x5f, 0xc2, 0xdd, 0x97, 0x9c, 0x9f, 0xe6, 0xd9, 0x33, 0x26, 0x68, + 0xa4, 0xb8, 0xb8, 0x38, 0x48, 0x95, 0xb8, 0xf0, 0xe9, 0xd7, 0x39, 0x95, 0x8a, 0x6c, 0x82, 0x1b, + 0x97, 0x8c, 0xb1, 0xb3, 0xed, 0xec, 0xb8, 0xfe, 0x9c, 0x40, 0x08, 0xac, 0xa4, 0xe1, 0x8c, 0x8e, + 0x5b, 0xc8, 0xc0, 0x6f, 0xef, 0x00, 0x36, 0x97, 0x1b, 0x94, 0x19, 0x4f, 0x25, 0x25, 0x0f, 0xa0, + 0x43, 0x35, 0x01, 0xad, 0xf5, 0xf7, 0x3e, 0xd8, 0x2d, 0x5d, 0xd9, 0x35, 0x72, 0x86, 0xeb, 0xfd, + 0xdd, 0x01, 0xf2, 0x92, 0x49, 0xa5, 0x89, 0x8c, 0xca, 0x6f, 0xe7, 0xcf, 0x6d, 0xe8, 0x66, 0x82, + 0x1e, 0xb3, 0xf3, 0xc2, 0xa3, 0xe2, 0x44, 0x1e, 0xc1, 0xba, 0x54, 0xa1, 0x50, 0xcf, 0x05, 0x9f, + 0x3d, 0x67, 0x09, 0x7d, 0xad, 0x9d, 0x6e, 0xa3, 0x48, 0x93, 0x41, 0x76, 0x81, 0xb0, 0x34, 0x4a, + 0x72, 0xc9, 0xde, 0xd3, 0xc3, 0x92, 0x3b, 0x5e, 0xd9, 0x76, 0x76, 0x7a, 0xfe, 0x12, 0x0e, 0xd9, + 0x80, 0x4e, 0xc2, 0x66, 0x4c, 0x8d, 0x3b, 0xdb, 0xce, 0xce, 0xd0, 0x37, 0x07, 0xef, 0x97, 0x30, + 0xaa, 0xf9, 0xff, 0xdd, 0x9e, 0xff, 0xe7, 0x16, 0x74, 0x90, 0x50, 0x61, 0xec, 0xcc, 0x31, 0x26, + 0x1f, 0xc1, 0x80, 0xc9, 0x60, 0x0e, 0x44, 0x0b, 0x7d, 0xeb, 0x33, 0x59, 0x61, 0x4e, 0x1e, 0x42, + 0x37, 0x3a, 0xc9, 0xd3, 0x53, 0x39, 0x6e, 0x6f, 0xb7, 0x77, 0xfa, 0x7b, 0xa3, 0xf9, 0x45, 0xfa, + 0xa1, 0xfb, 0x9a, 0xe7, 0x17, 0x22, 0xe4, 0x53, 0x80, 0x50, 0x29, 0xc1, 0x8e, 0x72, 0x45, 0x25, + 0xbe, 0xb4, 0xbf, 0x37, 0xb6, 0x14, 0x72, 0x49, 0x9f, 0x54, 0x7c, 0xdf, 0x92, 0x25, 0x9f, 0x41, + 0x8f, 0x9e, 0x2b, 0x9a, 0xc6, 0x34, 0x1e, 0x77, 0xf0, 0xa2, 0xad, 0x85, 0x17, 0xed, 0x1e, 0x14, + 0x7c, 0xf3, 0xbe, 0x4a, 0x7c, 0xf2, 0x39, 0x0c, 0x6b, 0x2c, 0xb2, 0x06, 0xed, 0x53, 0x5a, 0x46, + 0x55, 0x7f, 0x6a, 0x64, 0xdf, 0x87, 0x49, 0x6e, 0x12, 0x6c, 0xe0, 0x9b, 0xc3, 0x2f, 0x5a, 0x9f, + 0x3a, 0xde, 0x33, 0x70, 0x9f, 0xe7, 0x49, 0x52, 0x29, 0xc6, 0x4c, 0x94, 0x8a, 0x31, 0x13, 0x73, + 0x94, 0x5b, 0x57, 0xa2, 0xfc, 0x37, 0x07, 0xd6, 0x0f, 0xde, 0xd3, 0x54, 0xbd, 0xe6, 0x8a, 0x1d, + 0xb3, 0x28, 0x54, 0x8c, 0xa7, 0xe4, 0x11, 0xb8, 0x3c, 0x89, 0x83, 0x2b, 0xc3, 0xd4, 0xe3, 0x49, + 0xe1, 0xf5, 0x23, 0x70, 0x53, 0x7a, 0x16, 0x5c, 0x79, 0x5d, 0x2f, 0xa5, 0x67, 0x46, 0xfa, 0x3e, + 0x0c, 0x63, 0x9a, 0x50, 0x45, 0x83, 0x2a, 0x3a, 0x3a, 0x74, 0x03, 0x43, 0xdc, 0x37, 0xe1, 0xf8, + 0x18, 0x3e, 0xd0, 0x26, 0xb3, 0x50, 0xd0, 0x54, 0x05, 0x59, 0xa8, 0x4e, 0x30, 0x26, 0xae, 0x3f, + 0x4c, 0xe9, 0xd9, 0x1b, 0xa4, 0xbe, 0x09, 0xd5, 0x89, 0xf7, 0xd7, 0x16, 0xb8, 0x55, 0x30, 0xc9, + 0x87, 0xb0, 0xaa, 0xaf, 0x0d, 0x58, 0x5c, 0x20, 0xd1, 0xd5, 0xc7, 0x17, 0xb1, 0xae, 0x0a, 0x7e, + 0x7c, 0x2c, 0xa9, 0x42, 0xf7, 0xda, 0x7e, 0x71, 0xd2, 0x99, 0x25, 0xd9, 0x37, 0xa6, 0x10, 0x56, + 0x7c, 0xfc, 0xd6, 0x88, 0xcf, 0x14, 0x9b, 0x51, 0xbc, 0xb0, 0xed, 0x9b, 0x03, 0x19, 0x41, 0x87, + 0x06, 0x2a, 0x9c, 0x62, 0x86, 0xbb, 0xfe, 0x0a, 0x7d, 0x1b, 0x4e, 0xc9, 0x8f, 0xe1, 0xa6, 0xe4, + 0xb9, 0x88, 0x68, 0x50, 0x5e, 0xdb, 0x45, 0xee, 0xc0, 0x50, 0x9f, 0x9b, 0xcb, 0x3d, 0x68, 0x1f, + 0xb3, 0x78, 0xbc, 0x8a, 0xc0, 0xac, 0xd5, 0x93, 0xf0, 0x45, 0xec, 0x6b, 0x26, 0xf9, 0x19, 0x40, + 0x65, 0x29, 0x1e, 0xf7, 0x2e, 0x11, 0x75, 0x4b, 0xbb, 0x31, 0xd9, 0x02, 0x88, 0x58, 0x76, 0x42, + 0x45, 0xa0, 0x13, 0xc6, 0xc5, 0xe4, 0x70, 0x0d, 0xe5, 0xd7, 0xf4, 0x42, 0xb3, 0x99, 0x0c, 0xa6, + 0xdf, 0xb0, 0x2c, 0xa3, 0xf1, 0x18, 0x10, 0x61, 0x97, 0xc9, 0x2f, 0x0c, 0xc1, 0xfb, 0x0a, 0xba, + 0x85, 0x73, 0x77, 0xc1, 0x7d, 0xcf, 0x93, 0x7c, 0x56, 0x81, 0x36, 0xf4, 0x7b, 0x86, 0xf0, 0x22, + 0x26, 0x77, 0x00, 0xbb, 0x24, 0x5e, 0xd1, 0x42, 0x88, 0x10, 0x5f, 0x7d, 0xc1, 0x6d, 0xe8, 0x46, + 0x9c, 0x9f, 0x32, 0x83, 0xdd, 0xaa, 0x5f, 0x9c, 0xbc, 0xff, 0xb6, 0xe0, 0x66, 0xbd, 0x58, 0xf4, + 0x15, 0x68, 0x05, 0x91, 0x76, 0xd0, 0x0c, 0x9a, 0x3d, 0xac, 0xa1, 0xdd, 0xb2, 0xd1, 0x2e, 0x55, + 0x66, 0x3c, 0x36, 0x17, 0x0c, 0x8d, 0xca, 0x2b, 0x1e, 0x53, 0x9d, 0xeb, 0x39, 0x8b, 0x31, 0x3c, + 0x43, 0x5f, 0x7f, 0x6a, 0xca, 0x94, 0xc5, 0x45, 0xf3, 0xd1, 0x9f, 0xe8, 0x9e, 0x40, 0xbb, 0x5d, + 0x13, 0x70, 0x73, 0xd2, 0x01, 0x9f, 0x69, 0xea, 0xaa, 0x89, 0xa2, 0xfe, 0x26, 0xdb, 0xd0, 0x17, + 0x34, 0x4b, 0x8a, 0xdc, 0x47, 0xf0, 0x5d, 0xdf, 0x26, 0x91, 0x7b, 0x00, 0x11, 0x4f, 0x12, 0x1a, + 0xa1, 0x80, 0x8b, 0x02, 0x16, 0x45, 0xe7, 0x9d, 0x52, 0x49, 0x20, 0x69, 0x84, 0x50, 0x77, 0xfc, + 0xae, 0x52, 0xc9, 0x21, 0x8d, 0xf4, 0x3b, 0x72, 0x49, 0x45, 0x80, 0xed, 0xab, 0x8f, 0x7a, 0x3d, + 0x4d, 0xc0, 0x26, 0xbb, 0x05, 0x30, 0x15, 0x3c, 0xcf, 0x0c, 0x77, 0xb0, 0xdd, 0xd6, 0x9d, 0x1c, + 0x29, 0xc8, 0x7e, 0x00, 0x37, 0xe5, 0xc5, 0x2c, 0x61, 0xe9, 0x69, 0xa0, 0x42, 0x31, 0xa5, 0x6a, + 0x3c, 0x34, 0x15, 0x50, 0x50, 0xdf, 0x22, 0xd1, 0xcb, 0x80, 0xec, 0x0b, 0x1a, 0x2a, 0xfa, 0x1d, + 0x86, 0xd6, 0xb7, 0xeb, 0x0d, 0xe4, 0x16, 0x74, 0x79, 0x40, 0xcf, 0xa3, 0xa4, 0x28, 0xd1, 0x0e, + 0x3f, 0x38, 0x8f, 0x12, 0xef, 0x21, 0x8c, 0x6a, 0x37, 0x16, 0x6d, 0x7d, 0x03, 0x3a, 0x54, 0x08, + 0x5e, 0x36, 0x21, 0x73, 0xf0, 0x7e, 0x07, 0xe4, 0x5d, 0x16, 0xff, 0x10, 0xee, 0x79, 0xb7, 0x60, + 0x54, 0x33, 0x6d, 0xfc, 0xf0, 0xfe, 0xe9, 0x00, 0x79, 0x86, 0xbd, 0xe4, 0xfb, 0x8d, 0x71, 0x5d, + 0xdd, 0x7a, 0xc4, 0x98, 0x5e, 0x15, 0x87, 0x2a, 0x2c, 0x06, 0xe0, 0x80, 0x49, 0x63, 0xff, 0x59, + 0xa8, 0xc2, 0x62, 0x10, 0x09, 0x1a, 0xe5, 0x42, 0xcf, 0x44, 0x4c, 0x42, 0x1c, 0x44, 0x7e, 0x49, + 0x22, 0x9f, 0xc0, 0x6d, 0x36, 0x4d, 0xb9, 0xa0, 0x73, 0xb1, 0xc0, 0x40, 0xd5, 0x45, 0xe1, 0x0d, + 0xc3, 0xad, 0x14, 0x0e, 0x10, 0xb9, 0x87, 0x30, 0xaa, 0x3d, 0xe3, 0x4a, 0x98, 0xff, 0xe4, 0xc0, + 0xf8, 0x89, 0xe2, 0x33, 0x16, 0xf9, 0x54, 0x3b, 0x5f, 0x7b, 0xfa, 0x7d, 0x18, 0xea, 0x6e, 0xbe, + 0xf8, 0xfc, 0x01, 0x4f, 0xe2, 0xf9, 0xb4, 0xbc, 0x03, 0xba, 0xa1, 0x07, 0x16, 0x0a, 0xab, 0x3c, + 0x89, 0x31, 0x13, 0xef, 0x83, 0xee, 0xba, 0x96, 0xbe, 0xd9, 0x1b, 0x06, 0x29, 0x3d, 0xab, 0xe9, + 0x6b, 0x21, 0xd4, 0x37, 0xad, 0x7a, 0x35, 0xa5, 0x67, 0x5a, 0xdf, 0xbb, 0x0b, 0x77, 0x96, 0xf8, + 0x56, 0x84, 0xeb, 0x5f, 0x0e, 0x8c, 0x9e, 0x48, 0xc9, 0xa6, 0xe9, 0x6f, 0xb1, 0xed, 0x94, 0x4e, + 0x6f, 0x40, 0x27, 0xe2, 0x79, 0xaa, 0xd0, 0xd9, 0x8e, 0x6f, 0x0e, 0x0b, 0x95, 0xd8, 0x6a, 0x54, + 0xe2, 0x42, 0x2d, 0xb7, 0x9b, 0xb5, 0x6c, 0xd5, 0xea, 0x4a, 0xad, 0x56, 0x7f, 0x04, 0x7d, 0x1d, + 0xe4, 0x20, 0xa2, 0xa9, 0xa2, 0xa2, 0xe8, 0xf3, 0xa0, 0x49, 0xfb, 0x48, 0xd1, 0x02, 0xf6, 0x3c, + 0x32, 0xad, 0x1e, 0xb2, 0xf9, 0x30, 0xfa, 0xb7, 0x03, 0x1b, 0xf5, 0xa7, 0x14, 0x31, 0xbb, 0x74, + 0x2e, 0xe9, 0x56, 0x26, 0x92, 0xe2, 0x1d, 0xfa, 0x53, 0x37, 0x85, 0x2c, 0x3f, 0x4a, 0x58, 0x14, + 0x68, 0x86, 0xf1, 0xdf, 0x35, 0x94, 0x77, 0x22, 0x99, 0xa3, 0xb2, 0x62, 0xa3, 0x42, 0x60, 0x25, + 0xcc, 0xd5, 0x49, 0x39, 0x9b, 0xf4, 0xf7, 0x02, 0x52, 0xdd, 0xeb, 0x90, 0x5a, 0x6d, 0x22, 0x55, + 0x65, 0x5a, 0xcf, 0xce, 0xb4, 0x4f, 0x60, 0x64, 0x96, 0xdb, 0x7a, 0xb8, 0xb6, 0x00, 0xaa, 0x39, + 0x22, 0xc7, 0x8e, 0x69, 0x66, 0xe5, 0x20, 0x91, 0xde, 0xaf, 0xc0, 0x7d, 0xc9, 0x8d, 0x5d, 0x49, + 0x1e, 0x83, 0x9b, 0x94, 0x07, 0x14, 0xed, 0xef, 0x91, 0x79, 0x8d, 0x97, 0x72, 0xfe, 0x5c, 0xc8, + 0xfb, 0x1c, 0x7a, 0x25, 0xb9, 0xc4, 0xcc, 0xb9, 0x0c, 0xb3, 0xd6, 0x02, 0x66, 0xde, 0x3f, 0x1c, + 0xd8, 0xa8, 0xbb, 0x5c, 0x84, 0xe5, 0x1d, 0x0c, 0xab, 0x2b, 0x82, 0x59, 0x98, 0x15, 0xbe, 0x3c, + 0xb6, 0x7d, 0x69, 0xaa, 0x55, 0x0e, 0xca, 0x57, 0x61, 0x66, 0x72, 0x79, 0x90, 0x58, 0xa4, 0xc9, + 0x5b, 0x58, 0x6f, 0x88, 0x2c, 0xd9, 0xec, 0x7e, 0x6a, 0x6f, 0x76, 0xb5, 0xed, 0xb4, 0xd2, 0xb6, + 0xd7, 0xbd, 0xcf, 0xe0, 0x43, 0xd3, 0x0e, 0xf6, 0xab, 0x18, 0x96, 0xd8, 0xd7, 0x43, 0xed, 0x2c, + 0x86, 0xda, 0x9b, 0xc0, 0xb8, 0xa9, 0x5a, 0x94, 0xdf, 0x14, 0xd6, 0x0f, 0x55, 0xa8, 0x98, 0x54, + 0x2c, 0xaa, 0x7e, 0x62, 0x2c, 0xe4, 0x86, 0x73, 0xdd, 0x44, 0x6c, 0xd6, 0xe1, 0x1a, 0xb4, 0x95, + 0x2a, 0xf3, 0x57, 0x7f, 0xea, 0x28, 0x10, 0xfb, 0xa6, 0x22, 0x06, 0x3f, 0xc0, 0x55, 0x3a, 0x1f, + 0x14, 0x57, 0x61, 0x62, 0x36, 0x8e, 0x15, 0xdc, 0x38, 0x5c, 0xa4, 0xe0, 0xca, 0x61, 0x86, 0x72, + 0x6c, 0xb8, 0x1d, 0xb3, 0x8f, 0x68, 0x02, 0x32, 0xb7, 0x00, 0xb0, 0x54, 0x4d, 0x95, 0x75, 0x8d, + 0xae, 0xa6, 0xec, 0x6b, 0x82, 0x77, 0x0f, 0x36, 0xbf, 0xa0, 0x4a, 0xef, 0x4e, 0x62, 0x9f, 0xa7, + 0xc7, 0x6c, 0x9a, 0x8b, 0xd0, 0x0a, 0x85, 0xf7, 0x1f, 0x07, 0xb6, 0x2e, 0x11, 0x28, 0x1e, 0x3c, + 0x86, 0xd5, 0x59, 0x28, 0x15, 0x15, 0x65, 0x95, 0x94, 0xc7, 0x45, 0x28, 0x5a, 0xd7, 0x41, 0xd1, + 0x6e, 0x40, 0x71, 0x0b, 0xba, 0xb3, 0xf0, 0x3c, 0x98, 0x1d, 0x15, 0xcb, 0x51, 0x67, 0x16, 0x9e, + 0xbf, 0x3a, 0xc2, 0xce, 0xc6, 0x44, 0x70, 0x94, 0x47, 0xa7, 0x54, 0xc9, 0xaa, 0xb3, 0x31, 0xf1, + 0xd4, 0x50, 0xf4, 0xa3, 0xb5, 0xc0, 0xd7, 0x39, 0xcd, 0xa9, 0x2c, 0x7a, 0x85, 0x1e, 0x8e, 0xbf, + 0x41, 0x02, 0x2e, 0x53, 0xb8, 0x59, 0x62, 0x97, 0xe8, 0xf9, 0xc5, 0x69, 0xef, 0x2f, 0x3d, 0x18, + 0x1c, 0xd2, 0xf0, 0x8c, 0xd2, 0x18, 0x1f, 0x4c, 0xa6, 0x65, 0xa1, 0xd5, 0x7f, 0xf8, 0x92, 0x07, + 0x8b, 0x15, 0xb5, 0xf4, 0x97, 0xf6, 0xe4, 0xe3, 0xeb, 0xc4, 0x8a, 0x9c, 0xbd, 0x41, 0x5e, 0x43, + 0xdf, 0xfa, 0x65, 0x49, 0x36, 0x2d, 0xc5, 0xc6, 0x0f, 0xe6, 0xc9, 0xd6, 0x25, 0xdc, 0xd2, 0xda, + 0x63, 0x87, 0xbc, 0x84, 0xbe, 0xb5, 0xd2, 0xd8, 0xf6, 0x9a, 0xbb, 0x95, 0x6d, 0x6f, 0xc9, 0x1e, + 0xe4, 0xdd, 0xd0, 0xd6, 0xac, 0xc5, 0xc4, 0xb6, 0xd6, 0x5c, 0x85, 0x6c, 0x6b, 0xcb, 0xb6, 0x19, + 0xb4, 0x66, 0xed, 0x01, 0xb6, 0xb5, 0xe6, 0x96, 0x63, 0x5b, 0x5b, 0xb2, 0x3c, 0x78, 0x37, 0xc8, + 0x57, 0x30, 0x3a, 0x54, 0x82, 0x86, 0xb3, 0x39, 0x7b, 0x01, 0xc1, 0xff, 0xc3, 0xea, 0x8e, 0xf3, + 0xd8, 0x21, 0x7f, 0x80, 0xf5, 0xc6, 0x94, 0x27, 0xde, 0x5c, 0xf3, 0xb2, 0xf5, 0x64, 0x72, 0xff, + 0x4a, 0x99, 0xca, 0xf3, 0x2f, 0x61, 0x60, 0x0f, 0x57, 0x62, 0x39, 0xb5, 0x64, 0x7f, 0x98, 0xdc, + 0xbb, 0x8c, 0x6d, 0x1b, 0xb4, 0xfb, 0xbb, 0x6d, 0x70, 0xc9, 0x84, 0xb3, 0x0d, 0x2e, 0x1b, 0x0b, + 0xde, 0x0d, 0xf2, 0x7b, 0x58, 0x5b, 0xec, 0xb3, 0xe4, 0xa3, 0x45, 0xe8, 0x1a, 0xed, 0x7b, 0xe2, + 0x5d, 0x25, 0x52, 0x19, 0x7f, 0x01, 0x30, 0x6f, 0x9f, 0xe4, 0xee, 0x5c, 0xa7, 0xd1, 0xbe, 0x27, + 0x9b, 0xcb, 0x99, 0x95, 0xa9, 0x3f, 0xc2, 0xad, 0xa5, 0x3d, 0x8a, 0x58, 0x05, 0x78, 0x55, 0x97, + 0x9b, 0xfc, 0xe4, 0x5a, 0xb9, 0xf2, 0xae, 0xa7, 0xf7, 0x60, 0x4d, 0x9a, 0x16, 0x71, 0x2c, 0x77, + 0xa3, 0x84, 0xd1, 0x54, 0x3d, 0x05, 0xd4, 0x78, 0x23, 0xb8, 0xe2, 0x47, 0x5d, 0xfc, 0x37, 0xee, + 0xe7, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x35, 0x0b, 0x9e, 0x2e, 0x9c, 0x13, 0x00, 0x00, } diff --git a/weed/pb/master_pb/master.pb.go b/weed/pb/master_pb/master.pb.go index 95c9533a1..c33e2b768 100644 --- a/weed/pb/master_pb/master.pb.go +++ b/weed/pb/master_pb/master.pb.go @@ -428,12 +428,10 @@ type VolumeEcShardInformationMessage struct { EcIndexBits uint32 `protobuf:"varint,3,opt,name=ec_index_bits,json=ecIndexBits" json:"ec_index_bits,omitempty"` } -func (m *VolumeEcShardInformationMessage) Reset() { *m = VolumeEcShardInformationMessage{} } -func (m *VolumeEcShardInformationMessage) String() string { return proto.CompactTextString(m) } -func (*VolumeEcShardInformationMessage) ProtoMessage() {} -func (*VolumeEcShardInformationMessage) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{4} -} +func (m *VolumeEcShardInformationMessage) Reset() { *m = VolumeEcShardInformationMessage{} } +func (m *VolumeEcShardInformationMessage) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardInformationMessage) ProtoMessage() {} +func (*VolumeEcShardInformationMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } func (m *VolumeEcShardInformationMessage) GetId() uint32 { if m != nil { @@ -1424,12 +1422,10 @@ type GetMasterConfigurationResponse struct { MetricsIntervalSeconds uint32 `protobuf:"varint,2,opt,name=metrics_interval_seconds,json=metricsIntervalSeconds" json:"metrics_interval_seconds,omitempty"` } -func (m *GetMasterConfigurationResponse) Reset() { *m = GetMasterConfigurationResponse{} } -func (m *GetMasterConfigurationResponse) String() string { return proto.CompactTextString(m) } -func (*GetMasterConfigurationResponse) ProtoMessage() {} -func (*GetMasterConfigurationResponse) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{32} -} +func (m *GetMasterConfigurationResponse) Reset() { *m = GetMasterConfigurationResponse{} } +func (m *GetMasterConfigurationResponse) String() string { return proto.CompactTextString(m) } +func (*GetMasterConfigurationResponse) ProtoMessage() {} +func (*GetMasterConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{32} } func (m *GetMasterConfigurationResponse) GetMetricsAddress() string { if m != nil { diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go index 588b18f2e..56baa0cf7 100644 --- a/weed/pb/volume_server_pb/volume_server.pb.go +++ b/weed/pb/volume_server_pb/volume_server.pb.go @@ -1035,12 +1035,10 @@ func (m *VolumeEcShardsGenerateRequest) GetCollection() string { type VolumeEcShardsGenerateResponse struct { } -func (m *VolumeEcShardsGenerateResponse) Reset() { *m = VolumeEcShardsGenerateResponse{} } -func (m *VolumeEcShardsGenerateResponse) String() string { return proto.CompactTextString(m) } -func (*VolumeEcShardsGenerateResponse) ProtoMessage() {} -func (*VolumeEcShardsGenerateResponse) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{41} -} +func (m *VolumeEcShardsGenerateResponse) Reset() { *m = VolumeEcShardsGenerateResponse{} } +func (m *VolumeEcShardsGenerateResponse) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardsGenerateResponse) ProtoMessage() {} +func (*VolumeEcShardsGenerateResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{41} } type VolumeEcShardsRebuildRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` @@ -1413,12 +1411,10 @@ func (m *VolumeEcShardsToVolumeRequest) GetCollection() string { type VolumeEcShardsToVolumeResponse struct { } -func (m *VolumeEcShardsToVolumeResponse) Reset() { *m = VolumeEcShardsToVolumeResponse{} } -func (m *VolumeEcShardsToVolumeResponse) String() string { return proto.CompactTextString(m) } -func (*VolumeEcShardsToVolumeResponse) ProtoMessage() {} -func (*VolumeEcShardsToVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{57} -} +func (m *VolumeEcShardsToVolumeResponse) Reset() { *m = VolumeEcShardsToVolumeResponse{} } +func (m *VolumeEcShardsToVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardsToVolumeResponse) ProtoMessage() {} +func (*VolumeEcShardsToVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{57} } type ReadVolumeFileStatusRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` @@ -2085,10 +2081,8 @@ type QueryRequest_InputSerialization_JSONInput struct { func (m *QueryRequest_InputSerialization_JSONInput) Reset() { *m = QueryRequest_InputSerialization_JSONInput{} } -func (m *QueryRequest_InputSerialization_JSONInput) String() string { - return proto.CompactTextString(m) -} -func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {} +func (m *QueryRequest_InputSerialization_JSONInput) String() string { return proto.CompactTextString(m) } +func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {} func (*QueryRequest_InputSerialization_JSONInput) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{70, 1, 1} } diff --git a/weed/replication/sink/azuresink/azure_sink.go b/weed/replication/sink/azuresink/azure_sink.go index 66e7b9c3e..89e04922f 100644 --- a/weed/replication/sink/azuresink/azure_sink.go +++ b/weed/replication/sink/azuresink/azure_sink.go @@ -115,7 +115,7 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry) error { } var writeErr error - readErr := util.ReadUrlAsStream(fileUrl, nil, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) { + readErr := util.ReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) { _, writeErr = appendBlobURL.AppendBlock(context.Background(), bytes.NewReader(data), azblob.AppendBlobAccessConditions{}, nil) }) diff --git a/weed/replication/sink/b2sink/b2_sink.go b/weed/replication/sink/b2sink/b2_sink.go index 0f8313638..df0653f73 100644 --- a/weed/replication/sink/b2sink/b2_sink.go +++ b/weed/replication/sink/b2sink/b2_sink.go @@ -103,7 +103,7 @@ func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry) error { } var writeErr error - readErr := util.ReadUrlAsStream(fileUrl, nil, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) { + readErr := util.ReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) { _, err := writer.Write(data) if err != nil { writeErr = err diff --git a/weed/replication/sink/filersink/fetch_write.go b/weed/replication/sink/filersink/fetch_write.go index 512fbd46b..07218b9b3 100644 --- a/weed/replication/sink/filersink/fetch_write.go +++ b/weed/replication/sink/filersink/fetch_write.go @@ -51,6 +51,7 @@ func (fs *FilerSink) replicateOneChunk(sourceChunk *filer_pb.FileChunk, dir stri ETag: sourceChunk.ETag, SourceFileId: sourceChunk.GetFileIdString(), CipherKey: sourceChunk.CipherKey, + IsGzipped: sourceChunk.IsGzipped, }, nil } diff --git a/weed/replication/sink/gcssink/gcs_sink.go b/weed/replication/sink/gcssink/gcs_sink.go index dd8567e0e..694399274 100644 --- a/weed/replication/sink/gcssink/gcs_sink.go +++ b/weed/replication/sink/gcssink/gcs_sink.go @@ -101,7 +101,7 @@ func (g *GcsSink) CreateEntry(key string, entry *filer_pb.Entry) error { return err } - err = util.ReadUrlAsStream(fileUrl, nil, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) { + err = util.ReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) { wc.Write(data) }) diff --git a/weed/replication/sink/s3sink/s3_write.go b/weed/replication/sink/s3sink/s3_write.go index dcc041642..854688b1e 100644 --- a/weed/replication/sink/s3sink/s3_write.go +++ b/weed/replication/sink/s3sink/s3_write.go @@ -162,6 +162,6 @@ func (s3sink *S3Sink) buildReadSeeker(chunk *filer2.ChunkView) (io.ReadSeeker, e return nil, err } buf := make([]byte, chunk.Size) - util.ReadUrl(fileUrl, nil, false, chunk.Offset, int(chunk.Size), buf) + util.ReadUrl(fileUrl, nil, false,false, chunk.Offset, int(chunk.Size), buf) return bytes.NewReader(buf), nil } diff --git a/weed/server/common.go b/weed/server/common.go index 5904e2c47..e06142d7f 100644 --- a/weed/server/common.go +++ b/weed/server/common.go @@ -1,7 +1,6 @@ package weed_server import ( - "bytes" "encoding/json" "errors" "fmt" @@ -136,7 +135,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st } debug("upload file to store", url) - uploadResult, err := operation.Upload(url, pu.FileName, false, bytes.NewReader(pu.Data), pu.IsGzipped, pu.MimeType, pu.PairMap, assignResult.Auth) + uploadResult, err := operation.UploadData(url, pu.FileName, false, pu.Data, pu.IsGzipped, pu.MimeType, pu.PairMap, assignResult.Auth) if err != nil { writeJsonError(w, r, http.StatusInternalServerError, err) return diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 641a367b1..bce3c78bc 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -2,6 +2,7 @@ package weed_server import ( "context" + "fmt" "io" "net/http" "path" @@ -87,6 +88,9 @@ func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r if fileName != "" { fileName = path.Base(fileName) } + contentType := part1.Header.Get("Content-Type") + + fmt.Printf("autochunk part header: %+v\n", part1.Header) var fileChunks []*filer_pb.FileChunk @@ -102,7 +106,7 @@ func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r } // upload the chunk to the volume server - uploadResult, uploadErr := fs.doUpload(urlLocation, w, r, limitedReader, "", "", nil, auth) + uploadResult, uploadErr := fs.doUpload(urlLocation, w, r, limitedReader, fileName, contentType, nil, auth) if uploadErr != nil { return nil, uploadErr } @@ -121,6 +125,7 @@ func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r Mtime: time.Now().UnixNano(), ETag: uploadResult.ETag, CipherKey: uploadResult.CipherKey, + IsGzipped: uploadResult.Gzip > 0, }, ) @@ -154,6 +159,7 @@ func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r Replication: replication, Collection: collection, TtlSec: int32(util.ParseInt(r.URL.Query().Get("ttl"), 0)), + Mime: contentType, }, Chunks: fileChunks, } diff --git a/weed/server/filer_server_handlers_write_cipher.go b/weed/server/filer_server_handlers_write_cipher.go index bd2b52fb3..3aebc4b61 100644 --- a/weed/server/filer_server_handlers_write_cipher.go +++ b/weed/server/filer_server_handlers_write_cipher.go @@ -1,7 +1,6 @@ package weed_server import ( - "bytes" "context" "fmt" "net/http" @@ -28,7 +27,7 @@ func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *ht glog.V(4).Infof("write %s to %v", r.URL.Path, urlLocation) - // Note: gzip(cipher(data)), cipher data first, then gzip + // Note: encrypt(gzip(data)), encrypt data first, then gzip sizeLimit := int64(fs.option.MaxMB) * 1024 * 1024 @@ -41,7 +40,7 @@ func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *ht pu.MimeType = http.DetectContentType(uncompressedData) } - uploadResult, uploadError := operation.Upload(urlLocation, pu.FileName, true, bytes.NewReader(uncompressedData), false, pu.MimeType, pu.PairMap, auth) + uploadResult, uploadError := operation.UploadData(urlLocation, pu.FileName, true, uncompressedData, false, pu.MimeType, pu.PairMap, auth) if uploadError != nil { return nil, fmt.Errorf("upload to volume server: %v", uploadError) } @@ -53,11 +52,14 @@ func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *ht Offset: 0, Size: uint64(uploadResult.Size), Mtime: time.Now().UnixNano(), - ETag: uploadResult.ETag, + ETag: uploadResult.Md5, CipherKey: uploadResult.CipherKey, + IsGzipped: uploadResult.Gzip > 0, }, } + fmt.Printf("uploaded: %+v\n", uploadResult) + path := r.URL.Path if strings.HasSuffix(path, "/") { if pu.FileName != "" { diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index 9451fdf99..1fb0912c5 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -1,7 +1,6 @@ package weed_server import ( - "bytes" "context" "fmt" "io" @@ -418,8 +417,7 @@ func (f *WebDavFile) Write(buf []byte) (int, error) { } fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) - bufReader := bytes.NewReader(buf) - uploadResult, err := operation.Upload(fileUrl, f.name, f.fs.option.Cipher, bufReader, false, "", nil, auth) + uploadResult, err := operation.UploadData(fileUrl, f.name, f.fs.option.Cipher, buf, false, "", nil, auth) if err != nil { glog.V(0).Infof("upload data %v to %s: %v", f.name, fileUrl, err) return 0, fmt.Errorf("upload data: %v", err) @@ -436,6 +434,7 @@ func (f *WebDavFile) Write(buf []byte) (int, error) { Mtime: time.Now().UnixNano(), ETag: uploadResult.ETag, CipherKey: uploadResult.CipherKey, + IsGzipped: uploadResult.Gzip > 0, } f.entry.Chunks = append(f.entry.Chunks, chunk) diff --git a/weed/topology/store_replicate.go b/weed/topology/store_replicate.go index 332ec4d65..4acc71be5 100644 --- a/weed/topology/store_replicate.go +++ b/weed/topology/store_replicate.go @@ -1,7 +1,6 @@ package topology import ( - "bytes" "encoding/json" "errors" "fmt" @@ -73,7 +72,7 @@ func ReplicatedWrite(masterNode string, s *storage.Store, } // volume server do not know about encryption - _, err := operation.Upload(u.String(), string(n.Name), false, bytes.NewReader(n.Data), n.IsGzipped(), string(n.Mime), pairMap, jwt) + _, err := operation.UploadData(u.String(), string(n.Name), false, n.Data, n.IsGzipped(), string(n.Mime), pairMap, jwt) return err }); err != nil { size = 0 diff --git a/weed/util/http_util.go b/weed/util/http_util.go index 833db910c..750516b92 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -189,11 +189,11 @@ func NormalizeUrl(url string) string { return "http://" + url } -func ReadUrl(fileUrl string, cipherKey []byte, isFullChunk bool, offset int64, size int, buf []byte) (int64, error) { +func ReadUrl(fileUrl string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int, buf []byte) (int64, error) { if cipherKey != nil { var n int - err := readEncryptedUrl(fileUrl, cipherKey, offset, size, func(data []byte) { + err := readEncryptedUrl(fileUrl, cipherKey, isGzipped, offset, size, func(data []byte) { n = copy(buf, data) }) return int64(n), err @@ -258,10 +258,10 @@ func ReadUrl(fileUrl string, cipherKey []byte, isFullChunk bool, offset int64, s return n, err } -func ReadUrlAsStream(fileUrl string, cipherKey []byte, isFullChunk bool, offset int64, size int, fn func(data []byte)) error { +func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) error { if cipherKey != nil { - return readEncryptedUrl(fileUrl, cipherKey, offset, size, fn) + return readEncryptedUrl(fileUrl, cipherKey, isContentGzipped, offset, size, fn) } req, err := http.NewRequest("GET", fileUrl, nil) @@ -300,7 +300,7 @@ func ReadUrlAsStream(fileUrl string, cipherKey []byte, isFullChunk bool, offset } -func readEncryptedUrl(fileUrl string, cipherKey []byte, offset int64, size int, fn func(data []byte)) error { +func readEncryptedUrl(fileUrl string, cipherKey []byte, isContentGzipped bool, offset int64, size int, fn func(data []byte)) error { encryptedData, err := Get(fileUrl) if err != nil { return fmt.Errorf("fetch %s: %v", fileUrl, err) @@ -309,6 +309,12 @@ func readEncryptedUrl(fileUrl string, cipherKey []byte, offset int64, size int, if err != nil { return fmt.Errorf("decrypt %s: %v", fileUrl, err) } + if isContentGzipped { + decryptedData, err = UnGzipData(decryptedData) + if err != nil { + return fmt.Errorf("unzip decrypt %s: %v", fileUrl, err) + } + } if len(decryptedData) < int(offset)+size { return fmt.Errorf("read decrypted %s size %d [%d, %d)", fileUrl, len(decryptedData), offset, int(offset)+size) }