From b4b407e4038943ca5b7dc440d2848f23c11b73ca Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 27 May 2019 11:59:03 -0700 Subject: [PATCH] add grpc ec shard read --- weed/pb/volume_server.proto | 20 +- weed/pb/volume_server_pb/volume_server.pb.go | 368 +++++++++++++------ weed/server/volume_grpc_copy.go | 5 +- weed/server/volume_grpc_erasure_coding.go | 59 ++- weed/server/volume_grpc_tail.go | 5 +- weed/shell/command_ec_encode.go | 6 +- weed/storage/erasure_coding/ec_locate.go | 12 + weed/storage/erasure_coding/ec_shard.go | 6 + weed/storage/erasure_coding/ec_test.go | 19 +- weed/storage/erasure_coding/ec_volume.go | 13 +- weed/storage/needle/needle_read_write.go | 16 +- weed/storage/store_ec.go | 57 ++- 12 files changed, 418 insertions(+), 168 deletions(-) diff --git a/weed/pb/volume_server.proto b/weed/pb/volume_server.proto index 0f21c8ba5..e8d4ae61e 100644 --- a/weed/pb/volume_server.proto +++ b/weed/pb/volume_server.proto @@ -58,6 +58,8 @@ service VolumeServer { } rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) { } + rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) { + } } @@ -213,7 +215,7 @@ message VolumeEcShardsGenerateResponse { message VolumeEcShardsCopyRequest { uint32 volume_id = 1; string collection = 2; - repeated uint32 ec_indexes = 3; + repeated uint32 shard_ids = 3; string source_data_node = 5; } message VolumeEcShardsCopyResponse { @@ -222,7 +224,7 @@ message VolumeEcShardsCopyResponse { message VolumeEcShardsDeleteRequest { uint32 volume_id = 1; bool should_delete_ecx = 2; - repeated uint32 ec_indexes = 3; + repeated uint32 shard_ids = 3; } message VolumeEcShardsDeleteResponse { } @@ -230,18 +232,28 @@ message VolumeEcShardsDeleteResponse { message VolumeEcShardsMountRequest { uint32 volume_id = 1; string collection = 2; - repeated uint32 ec_indexes = 3; + repeated uint32 shard_ids = 3; } message VolumeEcShardsMountResponse { } message VolumeEcShardsUnmountRequest { uint32 volume_id = 1; - repeated uint32 ec_indexes = 3; + repeated uint32 shard_ids = 3; } message VolumeEcShardsUnmountResponse { } +message VolumeEcShardReadRequest { + uint32 volume_id = 1; + uint32 shard_id = 2; + int64 offset = 3; + int64 size = 4; +} +message VolumeEcShardReadResponse { + bytes data = 1; +} + message ReadVolumeFileStatusRequest { uint32 volume_id = 1; } diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go index f572fe401..180fbc0af 100644 --- a/weed/pb/volume_server_pb/volume_server.pb.go +++ b/weed/pb/volume_server_pb/volume_server.pb.go @@ -53,6 +53,8 @@ It has these top-level messages: VolumeEcShardsMountResponse VolumeEcShardsUnmountRequest VolumeEcShardsUnmountResponse + VolumeEcShardReadRequest + VolumeEcShardReadResponse ReadVolumeFileStatusRequest ReadVolumeFileStatusResponse DiskStatus @@ -811,7 +813,7 @@ func (*VolumeEcShardsGenerateResponse) Descriptor() ([]byte, []int) { return fil type VolumeEcShardsCopyRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"` - EcIndexes []uint32 `protobuf:"varint,3,rep,packed,name=ec_indexes,json=ecIndexes" json:"ec_indexes,omitempty"` + ShardIds []uint32 `protobuf:"varint,3,rep,packed,name=shard_ids,json=shardIds" json:"shard_ids,omitempty"` SourceDataNode string `protobuf:"bytes,5,opt,name=source_data_node,json=sourceDataNode" json:"source_data_node,omitempty"` } @@ -834,9 +836,9 @@ func (m *VolumeEcShardsCopyRequest) GetCollection() string { return "" } -func (m *VolumeEcShardsCopyRequest) GetEcIndexes() []uint32 { +func (m *VolumeEcShardsCopyRequest) GetShardIds() []uint32 { if m != nil { - return m.EcIndexes + return m.ShardIds } return nil } @@ -859,7 +861,7 @@ func (*VolumeEcShardsCopyResponse) Descriptor() ([]byte, []int) { return fileDes type VolumeEcShardsDeleteRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` ShouldDeleteEcx bool `protobuf:"varint,2,opt,name=should_delete_ecx,json=shouldDeleteEcx" json:"should_delete_ecx,omitempty"` - EcIndexes []uint32 `protobuf:"varint,3,rep,packed,name=ec_indexes,json=ecIndexes" json:"ec_indexes,omitempty"` + ShardIds []uint32 `protobuf:"varint,3,rep,packed,name=shard_ids,json=shardIds" json:"shard_ids,omitempty"` } func (m *VolumeEcShardsDeleteRequest) Reset() { *m = VolumeEcShardsDeleteRequest{} } @@ -881,9 +883,9 @@ func (m *VolumeEcShardsDeleteRequest) GetShouldDeleteEcx() bool { return false } -func (m *VolumeEcShardsDeleteRequest) GetEcIndexes() []uint32 { +func (m *VolumeEcShardsDeleteRequest) GetShardIds() []uint32 { if m != nil { - return m.EcIndexes + return m.ShardIds } return nil } @@ -899,7 +901,7 @@ func (*VolumeEcShardsDeleteResponse) Descriptor() ([]byte, []int) { return fileD type VolumeEcShardsMountRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"` - EcIndexes []uint32 `protobuf:"varint,3,rep,packed,name=ec_indexes,json=ecIndexes" json:"ec_indexes,omitempty"` + ShardIds []uint32 `protobuf:"varint,3,rep,packed,name=shard_ids,json=shardIds" json:"shard_ids,omitempty"` } func (m *VolumeEcShardsMountRequest) Reset() { *m = VolumeEcShardsMountRequest{} } @@ -921,9 +923,9 @@ func (m *VolumeEcShardsMountRequest) GetCollection() string { return "" } -func (m *VolumeEcShardsMountRequest) GetEcIndexes() []uint32 { +func (m *VolumeEcShardsMountRequest) GetShardIds() []uint32 { if m != nil { - return m.EcIndexes + return m.ShardIds } return nil } @@ -937,8 +939,8 @@ func (*VolumeEcShardsMountResponse) ProtoMessage() {} func (*VolumeEcShardsMountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{41} } type VolumeEcShardsUnmountRequest struct { - VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` - EcIndexes []uint32 `protobuf:"varint,3,rep,packed,name=ec_indexes,json=ecIndexes" json:"ec_indexes,omitempty"` + VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` + ShardIds []uint32 `protobuf:"varint,3,rep,packed,name=shard_ids,json=shardIds" json:"shard_ids,omitempty"` } func (m *VolumeEcShardsUnmountRequest) Reset() { *m = VolumeEcShardsUnmountRequest{} } @@ -953,9 +955,9 @@ func (m *VolumeEcShardsUnmountRequest) GetVolumeId() uint32 { return 0 } -func (m *VolumeEcShardsUnmountRequest) GetEcIndexes() []uint32 { +func (m *VolumeEcShardsUnmountRequest) GetShardIds() []uint32 { if m != nil { - return m.EcIndexes + return m.ShardIds } return nil } @@ -968,6 +970,62 @@ func (m *VolumeEcShardsUnmountResponse) String() string { return prot func (*VolumeEcShardsUnmountResponse) ProtoMessage() {} func (*VolumeEcShardsUnmountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{43} } +type VolumeEcShardReadRequest struct { + VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` + ShardId uint32 `protobuf:"varint,2,opt,name=shard_id,json=shardId" json:"shard_id,omitempty"` + Offset int64 `protobuf:"varint,3,opt,name=offset" json:"offset,omitempty"` + Size int64 `protobuf:"varint,4,opt,name=size" json:"size,omitempty"` +} + +func (m *VolumeEcShardReadRequest) Reset() { *m = VolumeEcShardReadRequest{} } +func (m *VolumeEcShardReadRequest) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardReadRequest) ProtoMessage() {} +func (*VolumeEcShardReadRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{44} } + +func (m *VolumeEcShardReadRequest) GetVolumeId() uint32 { + if m != nil { + return m.VolumeId + } + return 0 +} + +func (m *VolumeEcShardReadRequest) GetShardId() uint32 { + if m != nil { + return m.ShardId + } + return 0 +} + +func (m *VolumeEcShardReadRequest) GetOffset() int64 { + if m != nil { + return m.Offset + } + return 0 +} + +func (m *VolumeEcShardReadRequest) GetSize() int64 { + if m != nil { + return m.Size + } + return 0 +} + +type VolumeEcShardReadResponse struct { + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` +} + +func (m *VolumeEcShardReadResponse) Reset() { *m = VolumeEcShardReadResponse{} } +func (m *VolumeEcShardReadResponse) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardReadResponse) ProtoMessage() {} +func (*VolumeEcShardReadResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{45} } + +func (m *VolumeEcShardReadResponse) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + type ReadVolumeFileStatusRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` } @@ -975,7 +1033,7 @@ type ReadVolumeFileStatusRequest struct { func (m *ReadVolumeFileStatusRequest) Reset() { *m = ReadVolumeFileStatusRequest{} } func (m *ReadVolumeFileStatusRequest) String() string { return proto.CompactTextString(m) } func (*ReadVolumeFileStatusRequest) ProtoMessage() {} -func (*ReadVolumeFileStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{44} } +func (*ReadVolumeFileStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{46} } func (m *ReadVolumeFileStatusRequest) GetVolumeId() uint32 { if m != nil { @@ -998,7 +1056,7 @@ type ReadVolumeFileStatusResponse struct { func (m *ReadVolumeFileStatusResponse) Reset() { *m = ReadVolumeFileStatusResponse{} } func (m *ReadVolumeFileStatusResponse) String() string { return proto.CompactTextString(m) } func (*ReadVolumeFileStatusResponse) ProtoMessage() {} -func (*ReadVolumeFileStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{45} } +func (*ReadVolumeFileStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{47} } func (m *ReadVolumeFileStatusResponse) GetVolumeId() uint32 { if m != nil { @@ -1066,7 +1124,7 @@ type DiskStatus struct { func (m *DiskStatus) Reset() { *m = DiskStatus{} } func (m *DiskStatus) String() string { return proto.CompactTextString(m) } func (*DiskStatus) ProtoMessage() {} -func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{46} } +func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{48} } func (m *DiskStatus) GetDir() string { if m != nil { @@ -1109,7 +1167,7 @@ type MemStatus struct { func (m *MemStatus) Reset() { *m = MemStatus{} } func (m *MemStatus) String() string { return proto.CompactTextString(m) } func (*MemStatus) ProtoMessage() {} -func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{47} } +func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{49} } func (m *MemStatus) GetGoroutines() int32 { if m != nil { @@ -1205,6 +1263,8 @@ func init() { proto.RegisterType((*VolumeEcShardsMountResponse)(nil), "volume_server_pb.VolumeEcShardsMountResponse") proto.RegisterType((*VolumeEcShardsUnmountRequest)(nil), "volume_server_pb.VolumeEcShardsUnmountRequest") proto.RegisterType((*VolumeEcShardsUnmountResponse)(nil), "volume_server_pb.VolumeEcShardsUnmountResponse") + proto.RegisterType((*VolumeEcShardReadRequest)(nil), "volume_server_pb.VolumeEcShardReadRequest") + proto.RegisterType((*VolumeEcShardReadResponse)(nil), "volume_server_pb.VolumeEcShardReadResponse") proto.RegisterType((*ReadVolumeFileStatusRequest)(nil), "volume_server_pb.ReadVolumeFileStatusRequest") proto.RegisterType((*ReadVolumeFileStatusResponse)(nil), "volume_server_pb.ReadVolumeFileStatusResponse") proto.RegisterType((*DiskStatus)(nil), "volume_server_pb.DiskStatus") @@ -1247,6 +1307,7 @@ type VolumeServerClient interface { VolumeEcShardsDelete(ctx context.Context, in *VolumeEcShardsDeleteRequest, opts ...grpc.CallOption) (*VolumeEcShardsDeleteResponse, error) VolumeEcShardsMount(ctx context.Context, in *VolumeEcShardsMountRequest, opts ...grpc.CallOption) (*VolumeEcShardsMountResponse, error) VolumeEcShardsUnmount(ctx context.Context, in *VolumeEcShardsUnmountRequest, opts ...grpc.CallOption) (*VolumeEcShardsUnmountResponse, error) + VolumeEcShardRead(ctx context.Context, in *VolumeEcShardReadRequest, opts ...grpc.CallOption) (VolumeServer_VolumeEcShardReadClient, error) } type volumeServerClient struct { @@ -1524,6 +1585,38 @@ func (c *volumeServerClient) VolumeEcShardsUnmount(ctx context.Context, in *Volu return out, nil } +func (c *volumeServerClient) VolumeEcShardRead(ctx context.Context, in *VolumeEcShardReadRequest, opts ...grpc.CallOption) (VolumeServer_VolumeEcShardReadClient, error) { + stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[3], c.cc, "/volume_server_pb.VolumeServer/VolumeEcShardRead", opts...) + if err != nil { + return nil, err + } + x := &volumeServerVolumeEcShardReadClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type VolumeServer_VolumeEcShardReadClient interface { + Recv() (*VolumeEcShardReadResponse, error) + grpc.ClientStream +} + +type volumeServerVolumeEcShardReadClient struct { + grpc.ClientStream +} + +func (x *volumeServerVolumeEcShardReadClient) Recv() (*VolumeEcShardReadResponse, error) { + m := new(VolumeEcShardReadResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + // Server API for VolumeServer service type VolumeServerServer interface { @@ -1552,6 +1645,7 @@ type VolumeServerServer interface { VolumeEcShardsDelete(context.Context, *VolumeEcShardsDeleteRequest) (*VolumeEcShardsDeleteResponse, error) VolumeEcShardsMount(context.Context, *VolumeEcShardsMountRequest) (*VolumeEcShardsMountResponse, error) VolumeEcShardsUnmount(context.Context, *VolumeEcShardsUnmountRequest) (*VolumeEcShardsUnmountResponse, error) + VolumeEcShardRead(*VolumeEcShardReadRequest, VolumeServer_VolumeEcShardReadServer) error } func RegisterVolumeServerServer(s *grpc.Server, srv VolumeServerServer) { @@ -1963,6 +2057,27 @@ func _VolumeServer_VolumeEcShardsUnmount_Handler(srv interface{}, ctx context.Co return interceptor(ctx, in, info, handler) } +func _VolumeServer_VolumeEcShardRead_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(VolumeEcShardReadRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(VolumeServerServer).VolumeEcShardRead(m, &volumeServerVolumeEcShardReadServer{stream}) +} + +type VolumeServer_VolumeEcShardReadServer interface { + Send(*VolumeEcShardReadResponse) error + grpc.ServerStream +} + +type volumeServerVolumeEcShardReadServer struct { + grpc.ServerStream +} + +func (x *volumeServerVolumeEcShardReadServer) Send(m *VolumeEcShardReadResponse) error { + return x.ServerStream.SendMsg(m) +} + var _VolumeServer_serviceDesc = grpc.ServiceDesc{ ServiceName: "volume_server_pb.VolumeServer", HandlerType: (*VolumeServerServer)(nil), @@ -2060,6 +2175,11 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{ Handler: _VolumeServer_VolumeTailSender_Handler, ServerStreams: true, }, + { + StreamName: "VolumeEcShardRead", + Handler: _VolumeServer_VolumeEcShardRead_Handler, + ServerStreams: true, + }, }, Metadata: "volume_server.proto", } @@ -2067,109 +2187,113 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("volume_server.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1655 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xc4, 0x59, 0x5b, 0x4f, 0xdc, 0xd6, - 0x16, 0xc6, 0x99, 0x01, 0x86, 0x35, 0x90, 0xc0, 0x86, 0xc0, 0x60, 0x02, 0x21, 0x4e, 0x4e, 0x32, - 0xb9, 0x41, 0x4e, 0xa2, 0x73, 0x4e, 0xce, 0x39, 0x0f, 0x6d, 0x42, 0xd2, 0x16, 0xa9, 0x49, 0x24, - 0x93, 0x44, 0xbd, 0x44, 0xb2, 0x36, 0xf6, 0x22, 0x58, 0x78, 0xec, 0x89, 0xbd, 0x4d, 0x21, 0x6a, - 0x9f, 0xda, 0xaa, 0xef, 0xfd, 0x07, 0x55, 0xdf, 0xfa, 0xd0, 0xd7, 0xfe, 0xaa, 0xfe, 0x82, 0xbe, - 0x54, 0xfb, 0x62, 0xcf, 0xf8, 0xc6, 0x98, 0x26, 0x52, 0xdf, 0xf6, 0xac, 0xbd, 0xee, 0x5e, 0x97, - 0xfd, 0x01, 0xcc, 0x1f, 0x06, 0x5e, 0xdc, 0x43, 0x2b, 0xc2, 0xf0, 0x10, 0xc3, 0x8d, 0x7e, 0x18, - 0xb0, 0x80, 0xcc, 0x66, 0x88, 0x56, 0x7f, 0xd7, 0xd8, 0x04, 0xf2, 0x90, 0x32, 0x7b, 0xff, 0x11, - 0x7a, 0xc8, 0xd0, 0xc4, 0x37, 0x31, 0x46, 0x8c, 0x2c, 0x43, 0x6b, 0xcf, 0xf5, 0xd0, 0x72, 0x9d, - 0xa8, 0xa3, 0xad, 0x37, 0xba, 0x53, 0xe6, 0x24, 0xff, 0xbd, 0xed, 0x44, 0xc6, 0x33, 0x98, 0xcf, - 0x08, 0x44, 0xfd, 0xc0, 0x8f, 0x90, 0xdc, 0x87, 0xc9, 0x10, 0xa3, 0xd8, 0x63, 0x52, 0xa0, 0x7d, - 0x77, 0x6d, 0x23, 0x6f, 0x6b, 0x23, 0x15, 0x89, 0x3d, 0x66, 0x26, 0xec, 0x86, 0x0b, 0xd3, 0xc3, - 0x17, 0x64, 0x09, 0x26, 0x95, 0xed, 0x8e, 0xb6, 0xae, 0x75, 0xa7, 0xcc, 0x09, 0x69, 0x9a, 0x2c, - 0xc2, 0x44, 0xc4, 0x28, 0x8b, 0xa3, 0xce, 0x99, 0x75, 0xad, 0x3b, 0x6e, 0xaa, 0x5f, 0x64, 0x01, - 0xc6, 0x31, 0x0c, 0x83, 0xb0, 0xd3, 0x10, 0xec, 0xf2, 0x07, 0x21, 0xd0, 0x8c, 0xdc, 0xb7, 0xd8, - 0x69, 0xae, 0x6b, 0xdd, 0x19, 0x53, 0x9c, 0x8d, 0x49, 0x18, 0x7f, 0xdc, 0xeb, 0xb3, 0x63, 0xe3, - 0x3f, 0xd0, 0x79, 0x49, 0xed, 0x38, 0xee, 0xbd, 0x14, 0x3e, 0x6e, 0xed, 0xa3, 0x7d, 0x90, 0xc4, - 0xbe, 0x02, 0x53, 0xca, 0x73, 0xe5, 0xc1, 0x8c, 0xd9, 0x92, 0x84, 0x6d, 0xc7, 0xf8, 0x10, 0x96, - 0x4b, 0x04, 0x55, 0x0e, 0x2e, 0xc3, 0xcc, 0x6b, 0x1a, 0xee, 0xd2, 0xd7, 0x68, 0x85, 0x94, 0xb9, - 0x81, 0x90, 0xd6, 0xcc, 0x69, 0x45, 0x34, 0x39, 0xcd, 0xf8, 0x12, 0xf4, 0x8c, 0x86, 0xa0, 0xd7, - 0xa7, 0x36, 0xab, 0x63, 0x9c, 0xac, 0x43, 0xbb, 0x1f, 0x22, 0xf5, 0xbc, 0xc0, 0xa6, 0x0c, 0x45, - 0x16, 0x1a, 0xe6, 0x30, 0xc9, 0x58, 0x85, 0x95, 0x52, 0xe5, 0xd2, 0x41, 0xe3, 0x7e, 0xce, 0xfb, - 0xa0, 0xd7, 0x73, 0x6b, 0x99, 0x36, 0x2e, 0x14, 0xbc, 0x16, 0x92, 0x4a, 0xef, 0x7f, 0x73, 0xb7, - 0x1e, 0x52, 0x3f, 0xee, 0xd7, 0x52, 0x9c, 0xf7, 0x38, 0x11, 0x4d, 0x35, 0x2f, 0xc9, 0xe2, 0xd8, - 0x0a, 0x3c, 0x0f, 0x6d, 0xe6, 0x06, 0x7e, 0xa2, 0x76, 0x0d, 0xc0, 0x4e, 0x89, 0xaa, 0x54, 0x86, - 0x28, 0x86, 0x0e, 0x9d, 0xa2, 0xa8, 0x52, 0xfb, 0x8b, 0x06, 0xe7, 0x1f, 0xa8, 0xa4, 0x49, 0xc3, - 0xb5, 0x3e, 0x40, 0xd6, 0xe4, 0x99, 0xbc, 0xc9, 0xfc, 0x07, 0x6a, 0x14, 0x3e, 0x10, 0xe7, 0x08, - 0xb1, 0xef, 0xb9, 0x36, 0x15, 0x2a, 0x9a, 0x42, 0xc5, 0x30, 0x89, 0xcc, 0x42, 0x83, 0x31, 0xaf, - 0x33, 0x2e, 0x6e, 0xf8, 0xd1, 0xe8, 0xc0, 0x62, 0xde, 0x57, 0x15, 0xc6, 0xbf, 0x61, 0x49, 0x52, - 0x76, 0x8e, 0x7d, 0x7b, 0x47, 0x74, 0x43, 0xad, 0xa4, 0xff, 0xa1, 0x41, 0xa7, 0x28, 0xa8, 0xaa, - 0xf8, 0x5d, 0x33, 0x70, 0xda, 0xf8, 0xc8, 0x45, 0x68, 0x33, 0xea, 0x7a, 0x56, 0xb0, 0xb7, 0x17, - 0x21, 0xeb, 0x4c, 0xac, 0x6b, 0xdd, 0xa6, 0x09, 0x9c, 0xf4, 0x4c, 0x50, 0xc8, 0x75, 0x98, 0xb5, - 0x65, 0x25, 0x5b, 0x21, 0x1e, 0xba, 0x11, 0xd7, 0x3c, 0x29, 0x1c, 0x3b, 0x67, 0x27, 0x15, 0x2e, - 0xc9, 0xc4, 0x80, 0x19, 0xd7, 0x39, 0xb2, 0xc4, 0x00, 0x11, 0xed, 0xdf, 0x12, 0xda, 0xda, 0xae, - 0x73, 0xf4, 0x91, 0xeb, 0xe1, 0x0e, 0x9f, 0x02, 0x2f, 0xe1, 0x82, 0x0c, 0x7e, 0xdb, 0xb7, 0x43, - 0xec, 0xa1, 0xcf, 0xa8, 0xb7, 0x15, 0xf4, 0x8f, 0x6b, 0x95, 0xc0, 0x32, 0xb4, 0x22, 0xd7, 0xb7, - 0xd1, 0xf2, 0xe5, 0x18, 0x6a, 0x9a, 0x93, 0xe2, 0xf7, 0xd3, 0xc8, 0x78, 0x08, 0xab, 0x15, 0x7a, - 0x55, 0x66, 0x2f, 0xc1, 0xb4, 0x70, 0xcc, 0x0e, 0x7c, 0x86, 0x3e, 0x13, 0xba, 0xa7, 0xcd, 0x36, - 0xa7, 0x6d, 0x49, 0x92, 0xf1, 0x4f, 0x20, 0x52, 0xc7, 0x93, 0x20, 0xf6, 0xeb, 0xb5, 0xe6, 0x79, - 0x98, 0xcf, 0x88, 0xa8, 0xda, 0xb8, 0x07, 0x0b, 0x92, 0xfc, 0xc2, 0xef, 0xd5, 0xd6, 0xb5, 0x04, - 0xe7, 0x73, 0x42, 0x4a, 0xdb, 0xdd, 0xc4, 0x48, 0x76, 0x4f, 0x9c, 0xa8, 0x6c, 0x31, 0xf1, 0x20, - 0xbb, 0x2a, 0x8c, 0x5f, 0x35, 0x98, 0x4b, 0xc6, 0x48, 0xcd, 0xac, 0x9f, 0xb2, 0xec, 0x1a, 0x95, - 0x65, 0xd7, 0x1c, 0x94, 0x5d, 0x17, 0x66, 0xa3, 0x20, 0x0e, 0x6d, 0xb4, 0x1c, 0xca, 0xa8, 0xe5, - 0x07, 0x0e, 0xaa, 0xaa, 0x3c, 0x2b, 0xe9, 0x8f, 0x28, 0xa3, 0x4f, 0x03, 0x07, 0x8d, 0x0f, 0x92, - 0x8f, 0x92, 0xf9, 0x9a, 0xd7, 0x61, 0xce, 0xa3, 0x11, 0xb3, 0x68, 0xbf, 0x8f, 0xbe, 0x63, 0x51, - 0xc6, 0x4b, 0x42, 0x13, 0x25, 0x71, 0x96, 0x5f, 0x3c, 0x10, 0xf4, 0x07, 0xec, 0x69, 0x64, 0xfc, - 0xa8, 0xc1, 0x39, 0x2e, 0xcb, 0x4b, 0xb0, 0x56, 0xbc, 0xb3, 0xd0, 0xc0, 0x23, 0xa6, 0x02, 0xe5, - 0x47, 0xb2, 0x09, 0xf3, 0xaa, 0xd6, 0xdd, 0xc0, 0x1f, 0xb4, 0x41, 0x43, 0x08, 0x92, 0xc1, 0x55, - 0xda, 0x09, 0x17, 0xa1, 0x1d, 0xb1, 0xa0, 0x9f, 0x74, 0x55, 0x53, 0x76, 0x15, 0x27, 0xc9, 0xae, - 0x32, 0xfe, 0x05, 0xb3, 0x03, 0x9f, 0xea, 0x57, 0xe8, 0xb7, 0x5a, 0x32, 0x74, 0x9e, 0x53, 0xd7, - 0xdb, 0x41, 0xdf, 0xc1, 0xf0, 0x1d, 0x3b, 0x87, 0xdc, 0x81, 0x05, 0xd7, 0xf1, 0xd0, 0x62, 0x6e, - 0x0f, 0x83, 0x98, 0x59, 0x11, 0xda, 0x81, 0xef, 0x44, 0x49, 0x74, 0xfc, 0xee, 0xb9, 0xbc, 0xda, - 0x91, 0x37, 0xc6, 0x77, 0xe9, 0x04, 0x1b, 0xf6, 0x62, 0xb0, 0x87, 0x7d, 0x44, 0xae, 0x70, 0x1f, - 0xa9, 0x83, 0xa1, 0x0a, 0x63, 0x5a, 0x12, 0x3f, 0x11, 0x34, 0x9e, 0x1f, 0xc5, 0xb4, 0x1b, 0x38, - 0xc7, 0xc2, 0xa3, 0x69, 0x13, 0x24, 0xe9, 0x61, 0xe0, 0x1c, 0x8b, 0x51, 0x12, 0x59, 0xe2, 0x13, - 0xdb, 0xfb, 0xb1, 0x7f, 0x20, 0xbc, 0x69, 0x99, 0x6d, 0x37, 0xfa, 0x94, 0x46, 0x6c, 0x8b, 0x93, - 0x8c, 0xdf, 0x34, 0x58, 0x1e, 0xb8, 0x61, 0xa2, 0x8d, 0xee, 0xe1, 0xdf, 0x90, 0x0e, 0x2e, 0xa1, - 0x6a, 0x39, 0xf3, 0xe8, 0x52, 0xe5, 0x4e, 0xe4, 0x9d, 0x9a, 0xf8, 0xe2, 0x46, 0x2c, 0xf4, 0x12, - 0xc7, 0x55, 0x8b, 0xbe, 0x4a, 0x46, 0xd9, 0x63, 0x7b, 0x67, 0x9f, 0x86, 0x4e, 0xf4, 0x31, 0xfa, - 0x18, 0x52, 0xf6, 0x5e, 0xd6, 0xa4, 0xb1, 0x0e, 0x6b, 0x55, 0xda, 0x95, 0xfd, 0x9f, 0xd2, 0xbc, - 0x26, 0x2c, 0xef, 0x6d, 0x54, 0xac, 0x02, 0xa0, 0x6d, 0xb9, 0xbe, 0x83, 0x47, 0xc8, 0x53, 0xda, - 0xe8, 0xce, 0x98, 0x53, 0x68, 0x6f, 0x4b, 0xc2, 0x29, 0xa6, 0x42, 0x9a, 0xc1, 0xac, 0x8b, 0x2a, - 0x82, 0xef, 0x35, 0x58, 0xc9, 0x5e, 0xd7, 0x9f, 0x9c, 0xe4, 0x06, 0xcc, 0x45, 0xfb, 0x41, 0xec, - 0x39, 0x96, 0x23, 0x84, 0x2c, 0xb4, 0x8f, 0x44, 0x28, 0x2d, 0xf3, 0x9c, 0xbc, 0x90, 0xca, 0x1e, - 0xdb, 0x47, 0x23, 0xe2, 0x31, 0xd6, 0x92, 0x65, 0x97, 0x77, 0x43, 0xf9, 0x79, 0x94, 0x8f, 0xa2, - 0xf6, 0xe2, 0x79, 0xc7, 0x4c, 0x8b, 0x97, 0x5f, 0x99, 0x65, 0xe5, 0xd8, 0x17, 0x79, 0xc7, 0x4f, - 0xb1, 0xc7, 0x46, 0x99, 0xbe, 0x98, 0x2f, 0xef, 0xfc, 0xba, 0xfb, 0x1f, 0xac, 0x98, 0x48, 0x1d, - 0xc9, 0x24, 0x1e, 0x0e, 0xf5, 0x1f, 0x57, 0xbf, 0x9f, 0x81, 0x0b, 0xe5, 0xc2, 0x75, 0x1e, 0x58, - 0xff, 0x07, 0x3d, 0x7d, 0xc0, 0xf0, 0xfe, 0x8f, 0x18, 0xed, 0xf5, 0xd3, 0x09, 0x20, 0x07, 0xc5, - 0x92, 0x7a, 0xcd, 0x3c, 0x4f, 0xee, 0x93, 0x31, 0x50, 0x78, 0xfd, 0x34, 0x0a, 0xaf, 0x1f, 0x6e, - 0xc0, 0xa1, 0xac, 0xca, 0x80, 0x5c, 0x13, 0x4b, 0x0e, 0x65, 0x55, 0x06, 0x52, 0x61, 0x61, 0x60, - 0x5c, 0x1a, 0x50, 0xfc, 0xc2, 0xc0, 0x2a, 0x80, 0xda, 0x21, 0xb1, 0x9f, 0xbc, 0xe6, 0xa6, 0xe4, - 0x06, 0x89, 0xfd, 0xca, 0x45, 0x36, 0x59, 0xb9, 0xc8, 0xb2, 0x65, 0xd6, 0x2a, 0x4c, 0x93, 0xcf, - 0x00, 0x1e, 0xb9, 0xd1, 0x81, 0x4c, 0x32, 0xdf, 0x9c, 0x8e, 0x1b, 0x2a, 0x38, 0xc0, 0x8f, 0x9c, - 0x42, 0x3d, 0x4f, 0xa5, 0x8e, 0x1f, 0x39, 0x34, 0x8c, 0x23, 0x74, 0x54, 0x76, 0xc4, 0x99, 0xd3, - 0xf6, 0x42, 0x44, 0x95, 0x00, 0x71, 0x36, 0x7e, 0xd6, 0x60, 0xea, 0x09, 0xf6, 0x94, 0xe6, 0x35, - 0x80, 0xd7, 0x41, 0x18, 0xc4, 0xcc, 0xf5, 0x51, 0x2e, 0xfa, 0x71, 0x73, 0x88, 0xf2, 0xd7, 0xed, - 0x08, 0xa8, 0x8a, 0xde, 0x9e, 0x4a, 0xa6, 0x38, 0x73, 0xda, 0x3e, 0xd2, 0xbe, 0xca, 0x9f, 0x38, - 0x73, 0xa0, 0x1b, 0x31, 0x6a, 0x1f, 0x88, 0x64, 0x35, 0x4d, 0xf9, 0xe3, 0xee, 0x0f, 0xf3, 0x30, - 0x3d, 0x3c, 0xda, 0xc9, 0x2b, 0x68, 0x0f, 0x21, 0x74, 0x72, 0xa5, 0x08, 0xc4, 0x8b, 0x88, 0x5f, - 0xff, 0xc7, 0x08, 0x2e, 0xd5, 0x18, 0x63, 0xc4, 0x87, 0xb9, 0x02, 0x02, 0x26, 0x37, 0x8a, 0xd2, - 0x55, 0xf8, 0x5a, 0xbf, 0x59, 0x8b, 0x37, 0xb5, 0xc7, 0x60, 0xbe, 0x04, 0xd2, 0x92, 0x5b, 0x23, - 0xb4, 0x64, 0x60, 0xb5, 0x7e, 0xbb, 0x26, 0x77, 0x6a, 0xf5, 0x0d, 0x90, 0x22, 0xde, 0x25, 0x37, - 0x47, 0xaa, 0x19, 0xe0, 0x69, 0xfd, 0x56, 0x3d, 0xe6, 0xca, 0x40, 0x25, 0x12, 0x1e, 0x19, 0x68, - 0x06, 0x6b, 0x8f, 0x0c, 0x34, 0x07, 0xaf, 0xc7, 0xc8, 0x01, 0xcc, 0xe6, 0x51, 0x32, 0xb9, 0x5e, - 0xf5, 0xa7, 0x9b, 0x02, 0x08, 0xd7, 0x6f, 0xd4, 0x61, 0x4d, 0x8d, 0x21, 0x9c, 0xcd, 0x22, 0x59, - 0x72, 0xad, 0x28, 0x5f, 0x8a, 0xcb, 0xf5, 0xee, 0x68, 0xc6, 0xe1, 0x98, 0xf2, 0xe8, 0xb6, 0x2c, - 0xa6, 0x0a, 0xe8, 0x5c, 0x16, 0x53, 0x15, 0x58, 0x36, 0xc6, 0xc8, 0xd7, 0x09, 0x64, 0xca, 0xa1, - 0x3e, 0xb2, 0x51, 0xa5, 0xa6, 0x1c, 0x76, 0xea, 0x9b, 0xb5, 0xf9, 0x13, 0xdb, 0x77, 0x34, 0xde, - 0xeb, 0x43, 0xe0, 0xaf, 0xac, 0xd7, 0x8b, 0x70, 0xb2, 0xac, 0xd7, 0xcb, 0x10, 0xe4, 0x18, 0xd9, - 0x85, 0x99, 0x0c, 0x1c, 0x24, 0x57, 0xab, 0x24, 0xb3, 0xcb, 0x59, 0xbf, 0x36, 0x92, 0x2f, 0xb5, - 0x61, 0x25, 0xd3, 0x4b, 0x8d, 0xab, 0x4a, 0xe7, 0xb2, 0xf3, 0xea, 0xea, 0x28, 0xb6, 0xd4, 0xc0, - 0xe7, 0x00, 0x03, 0xf4, 0x46, 0x2e, 0x57, 0xc9, 0x0d, 0x7f, 0x8a, 0x2b, 0x27, 0x33, 0xa5, 0xaa, - 0xbf, 0x82, 0x85, 0xb2, 0x4d, 0x4f, 0x4a, 0xba, 0xf0, 0x84, 0xe7, 0x84, 0xbe, 0x51, 0x97, 0x3d, - 0x35, 0xfc, 0x02, 0x5a, 0x09, 0x76, 0x23, 0x97, 0x8a, 0xd2, 0x39, 0xac, 0xa9, 0x1b, 0x27, 0xb1, - 0x0c, 0x55, 0x53, 0x2f, 0x69, 0x9c, 0x01, 0xa8, 0xaa, 0x6e, 0x9c, 0x02, 0xfc, 0xab, 0x6e, 0x9c, - 0x22, 0x46, 0x13, 0xe6, 0xde, 0x24, 0xb8, 0x7a, 0x18, 0x83, 0x94, 0x0e, 0xd9, 0x2a, 0x88, 0x55, - 0x3a, 0x64, 0xab, 0x61, 0xcd, 0x18, 0xf9, 0x06, 0x16, 0xcb, 0xa1, 0x07, 0xa9, 0x6c, 0xbf, 0x0a, - 0x08, 0xa4, 0xdf, 0xa9, 0x2f, 0x90, 0x59, 0x2b, 0x05, 0xcc, 0x50, 0x1d, 0x71, 0x09, 0xf8, 0xa9, - 0x8e, 0xb8, 0x14, 0x86, 0x88, 0x1a, 0x2d, 0x03, 0x00, 0x65, 0x35, 0x7a, 0x02, 0x5e, 0xd1, 0x37, - 0xea, 0xb2, 0x67, 0xf6, 0x59, 0xf1, 0x7d, 0x4f, 0x46, 0xfa, 0x9f, 0x19, 0x55, 0xb7, 0x6b, 0x72, - 0xa7, 0x56, 0xdf, 0x26, 0xe3, 0x38, 0xf7, 0xb4, 0x27, 0x23, 0x03, 0xc8, 0x8d, 0xb0, 0xcd, 0xda, - 0xfc, 0x89, 0xed, 0xdd, 0x09, 0xf1, 0x4f, 0x96, 0x7b, 0x7f, 0x06, 0x00, 0x00, 0xff, 0xff, 0xb2, - 0x60, 0x85, 0x8e, 0x7b, 0x19, 0x00, 0x00, + // 1724 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xc4, 0x59, 0xdd, 0x6e, 0xd4, 0xc6, + 0x17, 0x8f, 0xd9, 0x4d, 0x76, 0x73, 0x36, 0x81, 0x64, 0x12, 0x92, 0x8d, 0x03, 0x61, 0x31, 0xfc, + 0x21, 0x04, 0x48, 0xf8, 0x83, 0xda, 0xd2, 0xf6, 0xa2, 0x85, 0x40, 0xdb, 0x48, 0x05, 0x24, 0x07, + 0x10, 0x55, 0x91, 0xac, 0x89, 0x3d, 0x21, 0x56, 0xbc, 0xf6, 0xe2, 0x19, 0xa7, 0x09, 0x6a, 0x7b, + 0x43, 0x9f, 0xa0, 0x2f, 0xd0, 0x8b, 0xde, 0xf5, 0xa2, 0xb7, 0x7d, 0xaa, 0x3e, 0x40, 0xd5, 0x9b, + 0x6a, 0x3e, 0xec, 0xf5, 0x67, 0xd6, 0x29, 0x48, 0xbd, 0x9b, 0x3d, 0x73, 0xbe, 0x7d, 0xce, 0x99, + 0xf9, 0xcd, 0xc2, 0xdc, 0x41, 0xe0, 0x45, 0x7d, 0x62, 0x51, 0x12, 0x1e, 0x90, 0x70, 0x7d, 0x10, + 0x06, 0x2c, 0x40, 0x33, 0x19, 0xa2, 0x35, 0xd8, 0x31, 0x36, 0x00, 0xdd, 0xc7, 0xcc, 0xde, 0x7b, + 0x40, 0x3c, 0xc2, 0x88, 0x49, 0x5e, 0x47, 0x84, 0x32, 0xb4, 0x04, 0xed, 0x5d, 0xd7, 0x23, 0x96, + 0xeb, 0xd0, 0xae, 0xd6, 0x6b, 0xac, 0x4e, 0x9a, 0x2d, 0xfe, 0x7b, 0xcb, 0xa1, 0xc6, 0x13, 0x98, + 0xcb, 0x08, 0xd0, 0x41, 0xe0, 0x53, 0x82, 0xee, 0x42, 0x2b, 0x24, 0x34, 0xf2, 0x98, 0x14, 0xe8, + 0xdc, 0x5e, 0x59, 0xcf, 0xdb, 0x5a, 0x4f, 0x44, 0x22, 0x8f, 0x99, 0x31, 0xbb, 0xe1, 0xc2, 0x54, + 0x7a, 0x03, 0x2d, 0x42, 0x4b, 0xd9, 0xee, 0x6a, 0x3d, 0x6d, 0x75, 0xd2, 0x9c, 0x90, 0xa6, 0xd1, + 0x02, 0x4c, 0x50, 0x86, 0x59, 0x44, 0xbb, 0xa7, 0x7a, 0xda, 0xea, 0xb8, 0xa9, 0x7e, 0xa1, 0x79, + 0x18, 0x27, 0x61, 0x18, 0x84, 0xdd, 0x86, 0x60, 0x97, 0x3f, 0x10, 0x82, 0x26, 0x75, 0xdf, 0x90, + 0x6e, 0xb3, 0xa7, 0xad, 0x4e, 0x9b, 0x62, 0x6d, 0xb4, 0x60, 0xfc, 0x61, 0x7f, 0xc0, 0x8e, 0x8c, + 0x8f, 0xa0, 0xfb, 0x1c, 0xdb, 0x51, 0xd4, 0x7f, 0x2e, 0x7c, 0xdc, 0xdc, 0x23, 0xf6, 0x7e, 0x1c, + 0xfb, 0x32, 0x4c, 0x2a, 0xcf, 0x95, 0x07, 0xd3, 0x66, 0x5b, 0x12, 0xb6, 0x1c, 0xe3, 0x73, 0x58, + 0x2a, 0x11, 0x54, 0x39, 0xb8, 0x04, 0xd3, 0xaf, 0x70, 0xb8, 0x83, 0x5f, 0x11, 0x2b, 0xc4, 0xcc, + 0x0d, 0x84, 0xb4, 0x66, 0x4e, 0x29, 0xa2, 0xc9, 0x69, 0xc6, 0xb7, 0xa0, 0x67, 0x34, 0x04, 0xfd, + 0x01, 0xb6, 0x59, 0x1d, 0xe3, 0xa8, 0x07, 0x9d, 0x41, 0x48, 0xb0, 0xe7, 0x05, 0x36, 0x66, 0x44, + 0x64, 0xa1, 0x61, 0xa6, 0x49, 0xc6, 0x79, 0x58, 0x2e, 0x55, 0x2e, 0x1d, 0x34, 0xee, 0xe6, 0xbc, + 0x0f, 0xfa, 0x7d, 0xb7, 0x96, 0x69, 0xe3, 0x5c, 0xc1, 0x6b, 0x21, 0xa9, 0xf4, 0x7e, 0x9c, 0xdb, + 0xf5, 0x08, 0xf6, 0xa3, 0x41, 0x2d, 0xc5, 0x79, 0x8f, 0x63, 0xd1, 0x44, 0xf3, 0xa2, 0x2c, 0x8e, + 0xcd, 0xc0, 0xf3, 0x88, 0xcd, 0xdc, 0xc0, 0x8f, 0xd5, 0xae, 0x00, 0xd8, 0x09, 0x51, 0x95, 0x4a, + 0x8a, 0x62, 0xe8, 0xd0, 0x2d, 0x8a, 0x2a, 0xb5, 0xbf, 0x69, 0x70, 0xf6, 0x9e, 0x4a, 0x9a, 0x34, + 0x5c, 0xeb, 0x03, 0x64, 0x4d, 0x9e, 0xca, 0x9b, 0xcc, 0x7f, 0xa0, 0x46, 0xe1, 0x03, 0x71, 0x8e, + 0x90, 0x0c, 0x3c, 0xd7, 0xc6, 0x42, 0x45, 0x53, 0xa8, 0x48, 0x93, 0xd0, 0x0c, 0x34, 0x18, 0xf3, + 0xba, 0xe3, 0x62, 0x87, 0x2f, 0x8d, 0x2e, 0x2c, 0xe4, 0x7d, 0x55, 0x61, 0x7c, 0x08, 0x8b, 0x92, + 0xb2, 0x7d, 0xe4, 0xdb, 0xdb, 0xa2, 0x1b, 0x6a, 0x25, 0xfd, 0x6f, 0x0d, 0xba, 0x45, 0x41, 0x55, + 0xc5, 0xef, 0x9a, 0x81, 0x93, 0xc6, 0x87, 0x2e, 0x40, 0x87, 0x61, 0xd7, 0xb3, 0x82, 0xdd, 0x5d, + 0x4a, 0x58, 0x77, 0xa2, 0xa7, 0xad, 0x36, 0x4d, 0xe0, 0xa4, 0x27, 0x82, 0x82, 0xae, 0xc1, 0x8c, + 0x2d, 0x2b, 0xd9, 0x0a, 0xc9, 0x81, 0x4b, 0xb9, 0xe6, 0x96, 0x70, 0xec, 0x8c, 0x1d, 0x57, 0xb8, + 0x24, 0x23, 0x03, 0xa6, 0x5d, 0xe7, 0xd0, 0x12, 0x03, 0x44, 0xb4, 0x7f, 0x5b, 0x68, 0xeb, 0xb8, + 0xce, 0xe1, 0x17, 0xae, 0x47, 0xb6, 0xf9, 0x14, 0x78, 0x0e, 0xe7, 0x64, 0xf0, 0x5b, 0xbe, 0x1d, + 0x92, 0x3e, 0xf1, 0x19, 0xf6, 0x36, 0x83, 0xc1, 0x51, 0xad, 0x12, 0x58, 0x82, 0x36, 0x75, 0x7d, + 0x9b, 0x58, 0xbe, 0x1c, 0x43, 0x4d, 0xb3, 0x25, 0x7e, 0x3f, 0xa6, 0xc6, 0x7d, 0x38, 0x5f, 0xa1, + 0x57, 0x65, 0xf6, 0x22, 0x4c, 0x09, 0xc7, 0xec, 0xc0, 0x67, 0xc4, 0x67, 0x42, 0xf7, 0x94, 0xd9, + 0xe1, 0xb4, 0x4d, 0x49, 0x32, 0xfe, 0x0f, 0x48, 0xea, 0x78, 0x14, 0x44, 0x7e, 0xbd, 0xd6, 0x3c, + 0x0b, 0x73, 0x19, 0x11, 0x55, 0x1b, 0x77, 0x60, 0x5e, 0x92, 0x9f, 0xf9, 0xfd, 0xda, 0xba, 0x16, + 0xe1, 0x6c, 0x4e, 0x48, 0x69, 0xbb, 0x1d, 0x1b, 0xc9, 0x9e, 0x13, 0xc7, 0x2a, 0x5b, 0x88, 0x3d, + 0xc8, 0x1e, 0x15, 0xc6, 0xef, 0x1a, 0xcc, 0xc6, 0x63, 0xa4, 0x66, 0xd6, 0x4f, 0x58, 0x76, 0x8d, + 0xca, 0xb2, 0x6b, 0x0e, 0xcb, 0x6e, 0x15, 0x66, 0x68, 0x10, 0x85, 0x36, 0xb1, 0x1c, 0xcc, 0xb0, + 0xe5, 0x07, 0x0e, 0x51, 0x55, 0x79, 0x5a, 0xd2, 0x1f, 0x60, 0x86, 0x1f, 0x07, 0x0e, 0x31, 0x3e, + 0x8b, 0x3f, 0x4a, 0xe6, 0x6b, 0x5e, 0x83, 0x59, 0x0f, 0x53, 0x66, 0xe1, 0xc1, 0x80, 0xf8, 0x8e, + 0x85, 0x19, 0x2f, 0x09, 0x4d, 0x94, 0xc4, 0x69, 0xbe, 0x71, 0x4f, 0xd0, 0xef, 0xb1, 0xc7, 0xd4, + 0xf8, 0x59, 0x83, 0x33, 0x5c, 0x96, 0x97, 0x60, 0xad, 0x78, 0x67, 0xa0, 0x41, 0x0e, 0x99, 0x0a, + 0x94, 0x2f, 0xd1, 0x06, 0xcc, 0xa9, 0x5a, 0x77, 0x03, 0x7f, 0xd8, 0x06, 0x0d, 0x21, 0x88, 0x86, + 0x5b, 0x49, 0x27, 0x5c, 0x80, 0x0e, 0x65, 0xc1, 0x20, 0xee, 0xaa, 0xa6, 0xec, 0x2a, 0x4e, 0x92, + 0x5d, 0x65, 0x7c, 0x00, 0x33, 0x43, 0x9f, 0xea, 0x57, 0xe8, 0x5b, 0x2d, 0x1e, 0x3a, 0x4f, 0xb1, + 0xeb, 0x6d, 0x13, 0xdf, 0x21, 0xe1, 0x3b, 0x76, 0x0e, 0xba, 0x05, 0xf3, 0xae, 0xe3, 0x11, 0x8b, + 0xb9, 0x7d, 0x12, 0x44, 0xcc, 0xa2, 0xc4, 0x0e, 0x7c, 0x87, 0xc6, 0xd1, 0xf1, 0xbd, 0xa7, 0x72, + 0x6b, 0x5b, 0xee, 0x18, 0x3f, 0x25, 0x13, 0x2c, 0xed, 0xc5, 0xf0, 0x1c, 0xf6, 0x09, 0xe1, 0x0a, + 0xf7, 0x08, 0x76, 0x48, 0xa8, 0xc2, 0x98, 0x92, 0xc4, 0xaf, 0x04, 0x8d, 0xe7, 0x47, 0x31, 0xed, + 0x04, 0xce, 0x91, 0xf0, 0x68, 0xca, 0x04, 0x49, 0xba, 0x1f, 0x38, 0x47, 0x62, 0x94, 0x50, 0x4b, + 0x7c, 0x62, 0x7b, 0x2f, 0xf2, 0xf7, 0x85, 0x37, 0x6d, 0xb3, 0xe3, 0xd2, 0xaf, 0x31, 0x65, 0x9b, + 0x9c, 0x64, 0xfc, 0xa1, 0xc1, 0xd2, 0xd0, 0x0d, 0x93, 0xd8, 0xc4, 0x3d, 0xf8, 0x0f, 0xd2, 0xc1, + 0x25, 0x54, 0x2d, 0x67, 0x2e, 0x5d, 0xaa, 0xdc, 0x91, 0xdc, 0x53, 0x13, 0x5f, 0xec, 0x88, 0x03, + 0xbd, 0xc4, 0x71, 0xd5, 0xa2, 0x2f, 0xe3, 0x51, 0xf6, 0xd0, 0xde, 0xde, 0xc3, 0xa1, 0x43, 0xbf, + 0x24, 0x3e, 0x09, 0x31, 0x7b, 0x2f, 0xc7, 0xa4, 0xd1, 0x83, 0x95, 0x2a, 0xed, 0xca, 0xfe, 0x2f, + 0x49, 0x5e, 0x63, 0x96, 0xf7, 0x36, 0x2a, 0x96, 0x61, 0x92, 0x72, 0x8d, 0xe2, 0x6e, 0xdb, 0xe8, + 0x35, 0xb8, 0xb0, 0x20, 0x6c, 0x39, 0xf4, 0x04, 0x33, 0x21, 0xc9, 0x5f, 0xd6, 0x41, 0xe5, 0xff, + 0x5b, 0x0d, 0x96, 0xb3, 0xdb, 0xf5, 0xe7, 0x26, 0x5a, 0x83, 0x59, 0xba, 0x17, 0x44, 0x9e, 0x63, + 0x39, 0x42, 0xc8, 0x22, 0xf6, 0xa1, 0x08, 0xa4, 0x6d, 0x9e, 0x91, 0x1b, 0x52, 0xd9, 0x43, 0xfb, + 0xf0, 0xd8, 0x68, 0x8c, 0x95, 0xf8, 0xa0, 0xcb, 0x3b, 0xa1, 0xbc, 0x3c, 0xc8, 0xc7, 0x50, 0xfb, + 0xd0, 0x79, 0xa7, 0x2c, 0x8b, 0x3b, 0x5f, 0x99, 0x5d, 0xe5, 0xd6, 0x8b, 0xbc, 0xdb, 0x27, 0x38, + 0xc1, 0x8e, 0x37, 0x7c, 0x21, 0x5f, 0xd6, 0xf9, 0x63, 0xee, 0xc7, 0x78, 0xaa, 0x28, 0x06, 0x93, + 0x60, 0xa7, 0x76, 0x37, 0x2b, 0xb3, 0x22, 0x1b, 0xd3, 0x66, 0x4b, 0x59, 0xe5, 0xb0, 0x45, 0xcd, + 0x60, 0x79, 0x1f, 0x54, 0xbf, 0x32, 0x00, 0xa5, 0xa1, 0x00, 0xca, 0x46, 0xae, 0xec, 0xa5, 0x7d, + 0x35, 0xd6, 0x10, 0x34, 0x79, 0x55, 0xaa, 0x69, 0x26, 0xd6, 0xc6, 0x27, 0xb0, 0xcc, 0x79, 0xa4, + 0x90, 0xb8, 0xe1, 0xd4, 0xbf, 0x05, 0xfe, 0x79, 0x0a, 0xce, 0x95, 0x0b, 0xd7, 0xb9, 0x09, 0x7e, + 0x0a, 0x7a, 0x72, 0xd3, 0xe2, 0x83, 0x8a, 0x32, 0xdc, 0x1f, 0x24, 0xa3, 0x4a, 0x4e, 0xb4, 0x45, + 0x75, 0xed, 0x7a, 0x1a, 0xef, 0xc7, 0xf3, 0xaa, 0x70, 0x4d, 0x6b, 0x14, 0xae, 0x69, 0xdc, 0x80, + 0x83, 0x59, 0x95, 0x01, 0x79, 0x9e, 0x2d, 0x3a, 0x98, 0x55, 0x19, 0x48, 0x84, 0x85, 0x81, 0x71, + 0x69, 0x40, 0xf1, 0x0b, 0x03, 0xe7, 0x01, 0xd4, 0x61, 0x17, 0xf9, 0xf1, 0xb5, 0x73, 0x52, 0x1e, + 0x75, 0x91, 0x5f, 0x79, 0xe2, 0xb6, 0x2a, 0x4f, 0xdc, 0x6c, 0x4f, 0xb4, 0x0b, 0x63, 0xef, 0x05, + 0xc0, 0x03, 0x97, 0xee, 0xcb, 0x24, 0xf3, 0x23, 0xde, 0x71, 0x43, 0x85, 0x5b, 0xf8, 0x92, 0x53, + 0xb0, 0xe7, 0xa9, 0xd4, 0xf1, 0x25, 0xff, 0xe2, 0x11, 0x25, 0x8e, 0xca, 0x8e, 0x58, 0x73, 0xda, + 0x6e, 0x48, 0x88, 0x4a, 0x80, 0x58, 0x1b, 0xbf, 0x6a, 0x30, 0xf9, 0x88, 0xf4, 0x95, 0xe6, 0x15, + 0x80, 0x57, 0x41, 0x18, 0x44, 0xcc, 0xf5, 0x89, 0xbc, 0x91, 0x8c, 0x9b, 0x29, 0xca, 0xbf, 0xb7, + 0x23, 0x4a, 0x96, 0x78, 0xbb, 0x2a, 0x99, 0x62, 0xcd, 0x69, 0x7b, 0x04, 0x0f, 0x54, 0xfe, 0xc4, + 0x9a, 0x23, 0x72, 0xca, 0xb0, 0xbd, 0x2f, 0x92, 0xd5, 0x34, 0xe5, 0x8f, 0xdb, 0x7f, 0xcd, 0xc1, + 0x54, 0xfa, 0x0c, 0x42, 0x2f, 0xa1, 0x93, 0x7a, 0x4a, 0x40, 0x97, 0x8b, 0x2f, 0x06, 0xc5, 0xa7, + 0x09, 0xfd, 0x7f, 0x23, 0xb8, 0x54, 0x27, 0x8f, 0x21, 0x1f, 0x66, 0x0b, 0x50, 0x1d, 0xad, 0x15, + 0xa5, 0xab, 0x1e, 0x02, 0xf4, 0xeb, 0xb5, 0x78, 0x13, 0x7b, 0x0c, 0xe6, 0x4a, 0xb0, 0x37, 0xba, + 0x31, 0x42, 0x4b, 0x06, 0xff, 0xeb, 0x37, 0x6b, 0x72, 0x27, 0x56, 0x5f, 0x03, 0x2a, 0x02, 0x73, + 0x74, 0x7d, 0xa4, 0x9a, 0x21, 0xf0, 0xd7, 0x6f, 0xd4, 0x63, 0xae, 0x0c, 0x54, 0x42, 0xf6, 0x91, + 0x81, 0x66, 0x1e, 0x05, 0x46, 0x06, 0x9a, 0x7b, 0x07, 0x18, 0x43, 0xfb, 0x30, 0x93, 0x87, 0xf3, + 0xe8, 0x5a, 0xd5, 0x1b, 0x53, 0xe1, 0xb5, 0x40, 0x5f, 0xab, 0xc3, 0x9a, 0x18, 0x23, 0x70, 0x3a, + 0x0b, 0xb9, 0xd1, 0xd5, 0xa2, 0x7c, 0xe9, 0x03, 0x82, 0xbe, 0x3a, 0x9a, 0x31, 0x1d, 0x53, 0x1e, + 0x86, 0x97, 0xc5, 0x54, 0x81, 0xf1, 0xcb, 0x62, 0xaa, 0x42, 0xf5, 0xc6, 0x18, 0xfa, 0x3e, 0xc6, + 0x76, 0x39, 0x78, 0x8a, 0xd6, 0xab, 0xd4, 0x94, 0xe3, 0x63, 0x7d, 0xa3, 0x36, 0x7f, 0x6c, 0xfb, + 0x96, 0xc6, 0x7b, 0x3d, 0x85, 0x52, 0xcb, 0x7a, 0xbd, 0x88, 0x7b, 0xcb, 0x7a, 0xbd, 0x0c, 0xea, + 0x8e, 0xa1, 0x1d, 0x98, 0xce, 0xe0, 0x56, 0x74, 0xa5, 0x4a, 0x32, 0x7b, 0x97, 0xd0, 0xaf, 0x8e, + 0xe4, 0x4b, 0x6c, 0x58, 0xf1, 0xf4, 0x52, 0xe3, 0xaa, 0xd2, 0xb9, 0xec, 0xbc, 0xba, 0x32, 0x8a, + 0x2d, 0x31, 0xf0, 0x0d, 0xc0, 0x10, 0x66, 0xa2, 0x4b, 0x55, 0x72, 0xe9, 0x4f, 0x71, 0xf9, 0x78, + 0xa6, 0x44, 0xf5, 0x77, 0x30, 0x5f, 0x76, 0xd2, 0xa3, 0x92, 0x2e, 0x3c, 0xe6, 0x3a, 0xa1, 0xaf, + 0xd7, 0x65, 0x4f, 0x0c, 0x3f, 0x83, 0x76, 0x0c, 0x32, 0xd1, 0xc5, 0xa2, 0x74, 0x0e, 0x14, 0xeb, + 0xc6, 0x71, 0x2c, 0xa9, 0x6a, 0xea, 0xc7, 0x8d, 0x33, 0x44, 0x7f, 0xd5, 0x8d, 0x53, 0xc0, 0xa9, + 0xd5, 0x8d, 0x53, 0x04, 0x93, 0xc2, 0xdc, 0xeb, 0xf8, 0x01, 0x20, 0x0d, 0x96, 0x4a, 0x87, 0x6c, + 0x15, 0x16, 0x2c, 0x1d, 0xb2, 0xd5, 0xf8, 0x6b, 0x0c, 0xfd, 0x00, 0x0b, 0xe5, 0x18, 0x09, 0x55, + 0xb6, 0x5f, 0x05, 0x56, 0xd3, 0x6f, 0xd5, 0x17, 0xc8, 0x1c, 0x2b, 0x05, 0x78, 0x53, 0x1d, 0x71, + 0x09, 0x4a, 0xab, 0x8e, 0xb8, 0x14, 0x31, 0x89, 0x1a, 0x2d, 0x43, 0x2b, 0x65, 0x35, 0x7a, 0x0c, + 0xb4, 0xd2, 0xd7, 0xeb, 0xb2, 0x67, 0xce, 0xb3, 0x22, 0x1c, 0x41, 0x23, 0xfd, 0xcf, 0x8c, 0xaa, + 0x9b, 0x35, 0xb9, 0x13, 0xab, 0x6f, 0xe2, 0x71, 0x9c, 0xc3, 0x22, 0x68, 0x64, 0x00, 0xb9, 0x11, + 0xb6, 0x51, 0x9b, 0x3f, 0xb1, 0x3d, 0x88, 0x1f, 0xe0, 0x52, 0x30, 0x03, 0xad, 0x8d, 0xd0, 0x93, + 0xc2, 0x42, 0xfa, 0xf5, 0x5a, 0xbc, 0xc3, 0x0e, 0xda, 0x99, 0x10, 0xff, 0x3f, 0xdd, 0xf9, 0x27, + 0x00, 0x00, 0xff, 0xff, 0x67, 0x03, 0x26, 0xea, 0x96, 0x1a, 0x00, 0x00, } diff --git a/weed/server/volume_grpc_copy.go b/weed/server/volume_grpc_copy.go index 7b681aa53..e5a3d6edf 100644 --- a/weed/server/volume_grpc_copy.go +++ b/weed/server/volume_grpc_copy.go @@ -16,6 +16,8 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" ) +const BufferSizeLimit = 1024 * 1024 * 2 + // VolumeCopy copy the .idx .dat files, and mount the volume func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.VolumeCopyRequest) (*volume_server_pb.VolumeCopyResponse, error) { @@ -190,7 +192,6 @@ func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream v bytesToRead := int64(req.StopOffset) - const BufferSize = 1024 * 1024 * 2 var fileName = v.FileName() + req.Ext file, err := os.Open(fileName) if err != nil { @@ -198,7 +199,7 @@ func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream v } defer file.Close() - buffer := make([]byte, BufferSize) + buffer := make([]byte, BufferSizeLimit) for bytesToRead > 0 { bytesread, err := file.Read(buffer) diff --git a/weed/server/volume_grpc_erasure_coding.go b/weed/server/volume_grpc_erasure_coding.go index f82b07e29..aa0f80442 100644 --- a/weed/server/volume_grpc_erasure_coding.go +++ b/weed/server/volume_grpc_erasure_coding.go @@ -3,6 +3,7 @@ package weed_server import ( "context" "fmt" + "io" "math" "os" @@ -71,8 +72,8 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv } // copy ec data slices - for _, ecIndex := range req.EcIndexes { - if err := vs.doCopyFile(ctx, client, req.VolumeId, math.MaxUint32, math.MaxInt64, baseFileName, erasure_coding.ToExt(int(ecIndex))); err != nil { + for _, shardId := range req.ShardIds { + if err := vs.doCopyFile(ctx, client, req.VolumeId, math.MaxUint32, math.MaxInt64, baseFileName, erasure_coding.ToExt(int(shardId))); err != nil { return err } } @@ -95,8 +96,8 @@ func (vs *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_se } baseFileName := v.FileName() - for _, shardIndex := range req.EcIndexes { - if err := os.Remove(baseFileName + erasure_coding.ToExt(int(shardIndex))); err != nil { + for _, shardId := range req.ShardIds { + if err := os.Remove(baseFileName + erasure_coding.ToExt(int(shardId))); err != nil { return nil, err } } @@ -112,7 +113,7 @@ func (vs *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_se func (vs *VolumeServer) VolumeEcShardsMount(ctx context.Context, req *volume_server_pb.VolumeEcShardsMountRequest) (*volume_server_pb.VolumeEcShardsMountResponse, error) { - for _, shardId := range req.EcIndexes { + for _, shardId := range req.ShardIds { err := vs.store.MountEcShards(req.Collection, needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId)) if err != nil { @@ -131,7 +132,7 @@ func (vs *VolumeServer) VolumeEcShardsMount(ctx context.Context, req *volume_ser func (vs *VolumeServer) VolumeEcShardsUnmount(ctx context.Context, req *volume_server_pb.VolumeEcShardsUnmountRequest) (*volume_server_pb.VolumeEcShardsUnmountResponse, error) { - for _, shardId := range req.EcIndexes { + for _, shardId := range req.ShardIds { err := vs.store.UnmountEcShards(needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId)) if err != nil { @@ -147,3 +148,49 @@ func (vs *VolumeServer) VolumeEcShardsUnmount(ctx context.Context, req *volume_s return &volume_server_pb.VolumeEcShardsUnmountResponse{}, nil } + +func (vs *VolumeServer) VolumeEcShardRead(req *volume_server_pb.VolumeEcShardReadRequest, stream volume_server_pb.VolumeServer_VolumeEcShardReadServer) error { + + ecShards, found := vs.store.HasEcShard(needle.VolumeId(req.VolumeId)) + if !found { + return fmt.Errorf("not found ec volume id %d", req.VolumeId) + } + ecShard, found := ecShards.FindEcVolumeShard(erasure_coding.ShardId(req.ShardId)) + if !found { + return fmt.Errorf("not found ec shard %d.%d", req.VolumeId, req.ShardId) + } + + buffer := make([]byte, BufferSizeLimit) + startOffset, bytesToRead := req.Offset, req.Size + + for bytesToRead > 0 { + bytesread, err := ecShard.ReadAt(buffer, startOffset) + + // println(fileName, "read", bytesread, "bytes, with target", bytesToRead) + + if err != nil { + if err != io.EOF { + return err + } + // println(fileName, "read", bytesread, "bytes, with target", bytesToRead, "err", err.Error()) + break + } + + if int64(bytesread) > bytesToRead { + bytesread = int(bytesToRead) + } + err = stream.Send(&volume_server_pb.VolumeEcShardReadResponse{ + Data: buffer[:bytesread], + }) + if err != nil { + // println("sending", bytesread, "bytes err", err.Error()) + return err + } + + bytesToRead -= int64(bytesread) + + } + + return nil + +} diff --git a/weed/server/volume_grpc_tail.go b/weed/server/volume_grpc_tail.go index 667131e9f..698bad5b8 100644 --- a/weed/server/volume_grpc_tail.go +++ b/weed/server/volume_grpc_tail.go @@ -69,12 +69,11 @@ func sendNeedlesSince(stream volume_server_pb.VolumeServer_VolumeTailSenderServe err = storage.ScanVolumeFileNeedleFrom(v.Version(), v.DataFile(), foundOffset.ToAcutalOffset(), func(needleHeader, needleBody []byte, needleAppendAtNs uint64) error { - blockSizeLimit := 1024 * 1024 * 2 isLastChunk := false // need to send body by chunks - for i := 0; i < len(needleBody); i += blockSizeLimit { - stopOffset := i + blockSizeLimit + for i := 0; i < len(needleBody); i += BufferSizeLimit { + stopOffset := i + BufferSizeLimit if stopOffset >= len(needleBody) { isLastChunk = true stopOffset = len(needleBody) diff --git a/weed/shell/command_ec_encode.go b/weed/shell/command_ec_encode.go index e0d028392..499c8a32e 100644 --- a/weed/shell/command_ec_encode.go +++ b/weed/shell/command_ec_encode.go @@ -208,7 +208,7 @@ func oneServerCopyEcShardsFromSource(ctx context.Context, grpcDialOption grpc.Di _, copyErr := volumeServerClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{ VolumeId: uint32(volumeId), Collection: collection, - EcIndexes: copiedShardIds, + ShardIds: copiedShardIds, SourceDataNode: existingLocation.Url, }) if copyErr != nil { @@ -219,7 +219,7 @@ func oneServerCopyEcShardsFromSource(ctx context.Context, grpcDialOption grpc.Di _, mountErr := volumeServerClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{ VolumeId: uint32(volumeId), Collection: collection, - EcIndexes: copiedShardIds, + ShardIds: copiedShardIds, }) if mountErr != nil { return mountErr @@ -243,7 +243,7 @@ func sourceServerDeleteEcShards(ctx context.Context, grpcDialOption grpc.DialOpt return operation.WithVolumeServerClient(sourceLocation.Url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { _, deleteErr := volumeServerClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ VolumeId: uint32(volumeId), - EcIndexes: toBeDeletedShardIds, + ShardIds: toBeDeletedShardIds, ShouldDeleteEcx: shouldDeleteEcx, }) return deleteErr diff --git a/weed/storage/erasure_coding/ec_locate.go b/weed/storage/erasure_coding/ec_locate.go index ee8af3382..562966f8f 100644 --- a/weed/storage/erasure_coding/ec_locate.go +++ b/weed/storage/erasure_coding/ec_locate.go @@ -69,3 +69,15 @@ func locateOffsetWithinBlocks(blockLength int64, offset int64) (blockIndex int, innerBlockOffset = offset % blockLength return } + +func (interval Interval) ToShardIdAndOffset(largeBlockSize, smallBlockSize int64) (ShardId, int64) { + ecFileOffset := interval.InnerBlockOffset + rowIndex := interval.BlockIndex / DataShardsCount + if interval.IsLargeBlock { + ecFileOffset += int64(rowIndex) * largeBlockSize + } else { + ecFileOffset += int64(interval.LargeBlockRowsCount)*largeBlockSize + int64(rowIndex)*smallBlockSize + } + ecFileIndex := interval.BlockIndex % DataShardsCount + return ShardId(ecFileIndex), ecFileOffset +} diff --git a/weed/storage/erasure_coding/ec_shard.go b/weed/storage/erasure_coding/ec_shard.go index 084d3f03b..fa4bfcecd 100644 --- a/weed/storage/erasure_coding/ec_shard.go +++ b/weed/storage/erasure_coding/ec_shard.go @@ -105,3 +105,9 @@ func (shard *EcVolumeShard) findNeedleFromEcx(needleId types.NeedleId) (offset t err = fmt.Errorf("needle id %d not found", needleId) return } + +func (shard *EcVolumeShard) ReadAt(buf []byte, offset int64) (int, error) { + + return shard.ecdFile.ReadAt(buf, offset) + +} diff --git a/weed/storage/erasure_coding/ec_test.go b/weed/storage/erasure_coding/ec_test.go index 83b0bc23a..e2e872dbe 100644 --- a/weed/storage/erasure_coding/ec_test.go +++ b/weed/storage/erasure_coding/ec_test.go @@ -105,10 +105,8 @@ func readEcFile(datSize int64, ecFiles []*os.File, offset types.Offset, size uin intervals := LocateData(largeBlockSize, smallBlockSize, datSize, offset.ToAcutalOffset(), size) - nLargeBlockRows := int(datSize / (largeBlockSize * DataShardsCount)) - for i, interval := range intervals { - if d, e := readOneInterval(interval, ecFiles, nLargeBlockRows); e != nil { + if d, e := readOneInterval(interval, ecFiles); e != nil { return nil, e } else { if i == 0 { @@ -122,21 +120,14 @@ func readEcFile(datSize int64, ecFiles []*os.File, offset types.Offset, size uin return data, nil } -func readOneInterval(interval Interval, ecFiles []*os.File, nLargeBlockRows int) (data []byte, err error) { - ecFileOffset := interval.InnerBlockOffset - rowIndex := interval.BlockIndex / DataShardsCount - if interval.IsLargeBlock { - ecFileOffset += int64(rowIndex) * largeBlockSize - } else { - ecFileOffset += int64(nLargeBlockRows)*largeBlockSize + int64(rowIndex)*smallBlockSize - } +func readOneInterval(interval Interval, ecFiles []*os.File) (data []byte, err error) { - ecFileIndex := interval.BlockIndex % DataShardsCount + ecFileOffset, ecFileIndex := interval.ToShardIdAndOffset(largeBlockSize, smallBlockSize) data = make([]byte, interval.Size) err = readFromFile(ecFiles[ecFileIndex], data, ecFileOffset) { // do some ec testing - ecData, err := readFromOtherEcFiles(ecFiles, ecFileIndex, ecFileOffset, interval.Size) + ecData, err := readFromOtherEcFiles(ecFiles, int(ecFileIndex), ecFileOffset, interval.Size) if err != nil { return nil, fmt.Errorf("ec reconstruct error: %v", err) } @@ -198,7 +189,7 @@ func TestLocateData(t *testing.T) { if len(intervals) != 1 { t.Errorf("unexpected interval size %d", len(intervals)) } - if !intervals[0].sameAs(Interval{0, 0, 1, false}) { + if !intervals[0].sameAs(Interval{0, 0, 1, false, 1}) { t.Errorf("unexpected interval %+v", intervals[0]) } diff --git a/weed/storage/erasure_coding/ec_volume.go b/weed/storage/erasure_coding/ec_volume.go index d57a28449..881e88d5f 100644 --- a/weed/storage/erasure_coding/ec_volume.go +++ b/weed/storage/erasure_coding/ec_volume.go @@ -6,6 +6,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/storage/needle" + "github.com/chrislusf/seaweedfs/weed/storage/types" ) type EcVolumeShards []*EcVolumeShard @@ -71,19 +72,17 @@ func (shards *EcVolumeShards) ToVolumeEcShardInformationMessage() (messages []*m return } -func (shards *EcVolumeShards) ReadEcShardNeedle(n *needle.Needle) (int, error) { +func (shards *EcVolumeShards) LocateEcShardNeedle(n *needle.Needle) (offset types.Offset, size uint32, intervals []Interval, err error) { shard := (*shards)[0] // find the needle from ecx file - offset, size, err := shard.findNeedleFromEcx(n.Id) + offset, size, err = shard.findNeedleFromEcx(n.Id) if err != nil { - return 0, err + return types.Offset{}, 0, nil, err } // calculate the locations in the ec shards - intervals := LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, shard.ecxFileSize, offset.ToAcutalOffset(), size) + intervals = LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, shard.ecxFileSize, offset.ToAcutalOffset(), size) - // TODO read the intervals - - return len(intervals), nil + return } diff --git a/weed/storage/needle/needle_read_write.go b/weed/storage/needle/needle_read_write.go index 85f335892..f2d699d11 100644 --- a/weed/storage/needle/needle_read_write.go +++ b/weed/storage/needle/needle_read_write.go @@ -170,11 +170,8 @@ func ReadNeedleBlob(r *os.File, offset int64, size uint32, version Version) (dat return dataSlice, err } -func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version) (err error) { - bytes, err := ReadNeedleBlob(r, offset, size, version) - if err != nil { - return err - } +// ReadBytes hydrates the needle from the bytes buffer, with only n.Id is set. +func (n *Needle) ReadBytes(bytes []byte, offset int64, size uint32, version Version) (err error) { n.ParseNeedleHeader(bytes) if n.Size != size { return fmt.Errorf("File Entry Not Found. offset %d, Needle id %d expected size %d Memory %d", offset, n.Id, n.Size, size) @@ -203,6 +200,15 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version return nil } +// ReadData hydrates the needle from the file, with only n.Id is set. +func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version) (err error) { + bytes, err := ReadNeedleBlob(r, offset, size, version) + if err != nil { + return err + } + return n.ReadBytes(bytes, offset, size, version) +} + func (n *Needle) ParseNeedleHeader(bytes []byte) { n.Cookie = BytesToCookie(bytes[0:CookieSize]) n.Id = BytesToNeedleId(bytes[CookieSize : CookieSize+NeedleIdSize]) diff --git a/weed/storage/store_ec.go b/weed/storage/store_ec.go index ed7c6484b..7514339c9 100644 --- a/weed/storage/store_ec.go +++ b/weed/storage/store_ec.go @@ -89,9 +89,62 @@ func (s *Store) HasEcShard(vid needle.VolumeId) (erasure_coding.EcVolumeShards, func (s *Store) ReadEcShardNeedle(vid needle.VolumeId, n *needle.Needle) (int, error) { for _, location := range s.Locations { - if ecShards, found := location.HasEcShard(vid); found { - return ecShards.ReadEcShardNeedle(n) + if localEcShards, found := location.HasEcShard(vid); found { + + offset, size, intervals, err := localEcShards.LocateEcShardNeedle(n) + if err != nil { + return 0, err + } + + bytes, err := s.ReadEcShardIntervals(vid, localEcShards, intervals) + if err != nil { + return 0, fmt.Errorf("ReadEcShardIntervals: %v", err) + } + + version := needle.CurrentVersion + + err = n.ReadBytes(bytes, offset.ToAcutalOffset(), size, version) + if err != nil { + return 0, fmt.Errorf("readbytes: %v", err) + } + + return len(bytes), nil } } return 0, fmt.Errorf("ec shard %d not found", vid) } + +func (s *Store) ReadEcShardIntervals(vid needle.VolumeId, localEcShards erasure_coding.EcVolumeShards, intervals []erasure_coding.Interval) (data []byte, err error) { + for i, interval := range intervals { + if d, e := s.readOneEcShardInterval(vid, localEcShards, interval); e != nil { + return nil, e + } else { + if i == 0 { + data = d + } else { + data = append(data, d...) + } + } + } + return +} + +func (s *Store) readOneEcShardInterval(vid needle.VolumeId, localEcShards erasure_coding.EcVolumeShards, interval erasure_coding.Interval) (data []byte, err error) { + shardId, actualOffset := interval.ToShardIdAndOffset(erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize) + data = make([]byte, interval.Size) + if shard, found := localEcShards.FindEcVolumeShard(shardId); found { + if _, err = shard.ReadAt(data, actualOffset); err != nil { + return + } + } else { + s.readOneRemoteEcShardInterval(vid, shardId, data, actualOffset) + } + return +} + +func (s *Store) readOneRemoteEcShardInterval(vid needle.VolumeId, shardId erasure_coding.ShardId, buf []byte, offset int64) (n int, err error) { + + + + return +}