From 17ac1290c0087a8264827e2ab1e9662f3ab4d257 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 21 May 2019 22:41:20 -0700 Subject: [PATCH] volume: load ec shards during heartbeats to master --- unmaintained/see_idx/see_idx.go | 4 +- weed/command/server.go | 2 +- weed/pb/master.proto | 13 + weed/pb/master_pb/master.pb.go | 304 ++++++++++++-------- weed/server/master_grpc_server.go | 4 +- weed/server/volume_grpc_client_to_master.go | 38 ++- weed/storage/disk_location.go | 69 +++-- weed/storage/disk_location_ec.go | 84 ++++++ weed/storage/disk_location_ec_test.go | 17 ++ weed/storage/erasure_coding/3.ec07 | 0 weed/storage/erasure_coding/3.ecx | 0 weed/storage/erasure_coding/ec_encoder.go | 4 +- weed/storage/erasure_coding/ec_volume.go | 108 +++++++ weed/storage/idx/walk.go | 54 ++++ weed/storage/needle_map.go | 7 - weed/storage/needle_map_leveldb.go | 6 +- weed/storage/needle_map_memory.go | 40 +-- weed/storage/needle_map_metric.go | 3 +- weed/storage/store.go | 7 + weed/storage/store_ec.go | 21 ++ weed/storage/volume_backup.go | 5 +- weed/storage/volume_checking.go | 3 +- weed/storage/volume_vacuum.go | 5 +- 23 files changed, 591 insertions(+), 207 deletions(-) create mode 100644 weed/storage/disk_location_ec.go create mode 100644 weed/storage/disk_location_ec_test.go create mode 100644 weed/storage/erasure_coding/3.ec07 create mode 100644 weed/storage/erasure_coding/3.ecx create mode 100644 weed/storage/erasure_coding/ec_volume.go create mode 100644 weed/storage/idx/walk.go create mode 100644 weed/storage/store_ec.go diff --git a/unmaintained/see_idx/see_idx.go b/unmaintained/see_idx/see_idx.go index 23ca04c2e..777af1821 100644 --- a/unmaintained/see_idx/see_idx.go +++ b/unmaintained/see_idx/see_idx.go @@ -8,7 +8,7 @@ import ( "strconv" "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/storage" + "github.com/chrislusf/seaweedfs/weed/storage/idx" "github.com/chrislusf/seaweedfs/weed/storage/types" ) @@ -35,7 +35,7 @@ func main() { } defer indexFile.Close() - storage.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error { + idx.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error { fmt.Printf("key:%v offset:%v size:%v\n", key, offset, size) return nil }) diff --git a/weed/command/server.go b/weed/command/server.go index e015e8832..630ba72a7 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -222,7 +222,7 @@ func runServer(cmd *Command, args []string) bool { glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err) } // Create your protocol servers. - glog.V(0).Infof("grpc config %+v", viper.Sub("grpc")) + glog.V(1).Infof("grpc config %+v", viper.Sub("grpc")) grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "master")) master_pb.RegisterSeaweedServer(grpcS, ms) protobuf.RegisterRaftServer(grpcS, raftServer) diff --git a/weed/pb/master.proto b/weed/pb/master.proto index e6d34b826..0968d2dcf 100644 --- a/weed/pb/master.proto +++ b/weed/pb/master.proto @@ -38,6 +38,13 @@ message Heartbeat { // delta volumes repeated VolumeShortInformationMessage new_volumes = 10; repeated VolumeShortInformationMessage deleted_volumes = 11; + + // erasure coding + repeated VolumeEcShardInformationMessage ec_shards = 16; + // delta erasure coding shards + repeated VolumeEcShardInformationMessage new_ec_shards = 17; + repeated VolumeEcShardInformationMessage deleted_ec_shards = 18; + } message HeartbeatResponse { @@ -67,6 +74,12 @@ message VolumeShortInformationMessage { uint32 ttl = 10; } +message VolumeEcShardInformationMessage { + uint32 id = 1; + string collection = 2; + uint32 ec_index = 3; +} + message Empty { } diff --git a/weed/pb/master_pb/master.pb.go b/weed/pb/master_pb/master.pb.go index 4c2e893a6..0c7a9837e 100644 --- a/weed/pb/master_pb/master.pb.go +++ b/weed/pb/master_pb/master.pb.go @@ -13,6 +13,7 @@ It has these top-level messages: HeartbeatResponse VolumeInformationMessage VolumeShortInformationMessage + VolumeEcShardInformationMessage Empty SuperBlockExtra ClientListenRequest @@ -72,6 +73,11 @@ type Heartbeat struct { // delta volumes NewVolumes []*VolumeShortInformationMessage `protobuf:"bytes,10,rep,name=new_volumes,json=newVolumes" json:"new_volumes,omitempty"` DeletedVolumes []*VolumeShortInformationMessage `protobuf:"bytes,11,rep,name=deleted_volumes,json=deletedVolumes" json:"deleted_volumes,omitempty"` + // erasure coding + EcShards []*VolumeEcShardInformationMessage `protobuf:"bytes,16,rep,name=ec_shards,json=ecShards" json:"ec_shards,omitempty"` + // delta erasure coding shards + NewEcShards []*VolumeEcShardInformationMessage `protobuf:"bytes,17,rep,name=new_ec_shards,json=newEcShards" json:"new_ec_shards,omitempty"` + DeletedEcShards []*VolumeEcShardInformationMessage `protobuf:"bytes,18,rep,name=deleted_ec_shards,json=deletedEcShards" json:"deleted_ec_shards,omitempty"` } func (m *Heartbeat) Reset() { *m = Heartbeat{} } @@ -156,6 +162,27 @@ func (m *Heartbeat) GetDeletedVolumes() []*VolumeShortInformationMessage { return nil } +func (m *Heartbeat) GetEcShards() []*VolumeEcShardInformationMessage { + if m != nil { + return m.EcShards + } + return nil +} + +func (m *Heartbeat) GetNewEcShards() []*VolumeEcShardInformationMessage { + if m != nil { + return m.NewEcShards + } + return nil +} + +func (m *Heartbeat) GetDeletedEcShards() []*VolumeEcShardInformationMessage { + if m != nil { + return m.DeletedEcShards + } + return nil +} + type HeartbeatResponse struct { VolumeSizeLimit uint64 `protobuf:"varint,1,opt,name=volumeSizeLimit" json:"volumeSizeLimit,omitempty"` Leader string `protobuf:"bytes,3,opt,name=leader" json:"leader,omitempty"` @@ -324,13 +351,45 @@ func (m *VolumeShortInformationMessage) GetTtl() uint32 { return 0 } +type VolumeEcShardInformationMessage struct { + Id uint32 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"` + Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"` + EcIndex uint32 `protobuf:"varint,3,opt,name=ec_index,json=ecIndex" json:"ec_index,omitempty"` +} + +func (m *VolumeEcShardInformationMessage) Reset() { *m = VolumeEcShardInformationMessage{} } +func (m *VolumeEcShardInformationMessage) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardInformationMessage) ProtoMessage() {} +func (*VolumeEcShardInformationMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } + +func (m *VolumeEcShardInformationMessage) GetId() uint32 { + if m != nil { + return m.Id + } + return 0 +} + +func (m *VolumeEcShardInformationMessage) GetCollection() string { + if m != nil { + return m.Collection + } + return "" +} + +func (m *VolumeEcShardInformationMessage) GetEcIndex() uint32 { + if m != nil { + return m.EcIndex + } + return 0 +} + type Empty struct { } func (m *Empty) Reset() { *m = Empty{} } func (m *Empty) String() string { return proto.CompactTextString(m) } func (*Empty) ProtoMessage() {} -func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } +func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } type SuperBlockExtra struct { ErasureCoding *SuperBlockExtra_ErasureCoding `protobuf:"bytes,1,opt,name=erasure_coding,json=erasureCoding" json:"erasure_coding,omitempty"` @@ -339,7 +398,7 @@ type SuperBlockExtra struct { func (m *SuperBlockExtra) Reset() { *m = SuperBlockExtra{} } func (m *SuperBlockExtra) String() string { return proto.CompactTextString(m) } func (*SuperBlockExtra) ProtoMessage() {} -func (*SuperBlockExtra) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } +func (*SuperBlockExtra) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } func (m *SuperBlockExtra) GetErasureCoding() *SuperBlockExtra_ErasureCoding { if m != nil { @@ -358,7 +417,7 @@ func (m *SuperBlockExtra_ErasureCoding) Reset() { *m = SuperBlockExtra_E func (m *SuperBlockExtra_ErasureCoding) String() string { return proto.CompactTextString(m) } func (*SuperBlockExtra_ErasureCoding) ProtoMessage() {} func (*SuperBlockExtra_ErasureCoding) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{5, 0} + return fileDescriptor0, []int{6, 0} } func (m *SuperBlockExtra_ErasureCoding) GetData() uint32 { @@ -389,7 +448,7 @@ type ClientListenRequest struct { func (m *ClientListenRequest) Reset() { *m = ClientListenRequest{} } func (m *ClientListenRequest) String() string { return proto.CompactTextString(m) } func (*ClientListenRequest) ProtoMessage() {} -func (*ClientListenRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } +func (*ClientListenRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } func (m *ClientListenRequest) GetName() string { if m != nil { @@ -408,7 +467,7 @@ type VolumeLocation struct { func (m *VolumeLocation) Reset() { *m = VolumeLocation{} } func (m *VolumeLocation) String() string { return proto.CompactTextString(m) } func (*VolumeLocation) ProtoMessage() {} -func (*VolumeLocation) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } +func (*VolumeLocation) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } func (m *VolumeLocation) GetUrl() string { if m != nil { @@ -446,7 +505,7 @@ type LookupVolumeRequest struct { func (m *LookupVolumeRequest) Reset() { *m = LookupVolumeRequest{} } func (m *LookupVolumeRequest) String() string { return proto.CompactTextString(m) } func (*LookupVolumeRequest) ProtoMessage() {} -func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } +func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } func (m *LookupVolumeRequest) GetVolumeIds() []string { if m != nil { @@ -469,7 +528,7 @@ type LookupVolumeResponse struct { func (m *LookupVolumeResponse) Reset() { *m = LookupVolumeResponse{} } func (m *LookupVolumeResponse) String() string { return proto.CompactTextString(m) } func (*LookupVolumeResponse) ProtoMessage() {} -func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } +func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } func (m *LookupVolumeResponse) GetVolumeIdLocations() []*LookupVolumeResponse_VolumeIdLocation { if m != nil { @@ -488,7 +547,7 @@ func (m *LookupVolumeResponse_VolumeIdLocation) Reset() { *m = LookupVol func (m *LookupVolumeResponse_VolumeIdLocation) String() string { return proto.CompactTextString(m) } func (*LookupVolumeResponse_VolumeIdLocation) ProtoMessage() {} func (*LookupVolumeResponse_VolumeIdLocation) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{9, 0} + return fileDescriptor0, []int{10, 0} } func (m *LookupVolumeResponse_VolumeIdLocation) GetVolumeId() string { @@ -520,7 +579,7 @@ type Location struct { func (m *Location) Reset() { *m = Location{} } func (m *Location) String() string { return proto.CompactTextString(m) } func (*Location) ProtoMessage() {} -func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } +func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} } func (m *Location) GetUrl() string { if m != nil { @@ -549,7 +608,7 @@ type AssignRequest struct { func (m *AssignRequest) Reset() { *m = AssignRequest{} } func (m *AssignRequest) String() string { return proto.CompactTextString(m) } func (*AssignRequest) ProtoMessage() {} -func (*AssignRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} } +func (*AssignRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} } func (m *AssignRequest) GetCount() uint64 { if m != nil { @@ -612,7 +671,7 @@ type AssignResponse struct { func (m *AssignResponse) Reset() { *m = AssignResponse{} } func (m *AssignResponse) String() string { return proto.CompactTextString(m) } func (*AssignResponse) ProtoMessage() {} -func (*AssignResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} } +func (*AssignResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} } func (m *AssignResponse) GetFid() string { if m != nil { @@ -665,7 +724,7 @@ type StatisticsRequest struct { func (m *StatisticsRequest) Reset() { *m = StatisticsRequest{} } func (m *StatisticsRequest) String() string { return proto.CompactTextString(m) } func (*StatisticsRequest) ProtoMessage() {} -func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} } +func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} } func (m *StatisticsRequest) GetReplication() string { if m != nil { @@ -700,7 +759,7 @@ type StatisticsResponse struct { func (m *StatisticsResponse) Reset() { *m = StatisticsResponse{} } func (m *StatisticsResponse) String() string { return proto.CompactTextString(m) } func (*StatisticsResponse) ProtoMessage() {} -func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} } +func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} } func (m *StatisticsResponse) GetReplication() string { if m != nil { @@ -752,7 +811,7 @@ type StorageType struct { func (m *StorageType) Reset() { *m = StorageType{} } func (m *StorageType) String() string { return proto.CompactTextString(m) } func (*StorageType) ProtoMessage() {} -func (*StorageType) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} } +func (*StorageType) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} } func (m *StorageType) GetReplication() string { if m != nil { @@ -775,7 +834,7 @@ type Collection struct { func (m *Collection) Reset() { *m = Collection{} } func (m *Collection) String() string { return proto.CompactTextString(m) } func (*Collection) ProtoMessage() {} -func (*Collection) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} } +func (*Collection) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} } func (m *Collection) GetName() string { if m != nil { @@ -790,7 +849,7 @@ type CollectionListRequest struct { func (m *CollectionListRequest) Reset() { *m = CollectionListRequest{} } func (m *CollectionListRequest) String() string { return proto.CompactTextString(m) } func (*CollectionListRequest) ProtoMessage() {} -func (*CollectionListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} } +func (*CollectionListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} } type CollectionListResponse struct { Collections []*Collection `protobuf:"bytes,1,rep,name=collections" json:"collections,omitempty"` @@ -799,7 +858,7 @@ type CollectionListResponse struct { func (m *CollectionListResponse) Reset() { *m = CollectionListResponse{} } func (m *CollectionListResponse) String() string { return proto.CompactTextString(m) } func (*CollectionListResponse) ProtoMessage() {} -func (*CollectionListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} } +func (*CollectionListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} } func (m *CollectionListResponse) GetCollections() []*Collection { if m != nil { @@ -815,7 +874,7 @@ type CollectionDeleteRequest struct { func (m *CollectionDeleteRequest) Reset() { *m = CollectionDeleteRequest{} } func (m *CollectionDeleteRequest) String() string { return proto.CompactTextString(m) } func (*CollectionDeleteRequest) ProtoMessage() {} -func (*CollectionDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} } +func (*CollectionDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} } func (m *CollectionDeleteRequest) GetName() string { if m != nil { @@ -830,7 +889,7 @@ type CollectionDeleteResponse struct { func (m *CollectionDeleteResponse) Reset() { *m = CollectionDeleteResponse{} } func (m *CollectionDeleteResponse) String() string { return proto.CompactTextString(m) } func (*CollectionDeleteResponse) ProtoMessage() {} -func (*CollectionDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} } +func (*CollectionDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} } // // volume related @@ -847,7 +906,7 @@ type DataNodeInfo struct { func (m *DataNodeInfo) Reset() { *m = DataNodeInfo{} } func (m *DataNodeInfo) String() string { return proto.CompactTextString(m) } func (*DataNodeInfo) ProtoMessage() {} -func (*DataNodeInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} } +func (*DataNodeInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} } func (m *DataNodeInfo) GetId() string { if m != nil { @@ -903,7 +962,7 @@ type RackInfo struct { func (m *RackInfo) Reset() { *m = RackInfo{} } func (m *RackInfo) String() string { return proto.CompactTextString(m) } func (*RackInfo) ProtoMessage() {} -func (*RackInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} } +func (*RackInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} } func (m *RackInfo) GetId() string { if m != nil { @@ -959,7 +1018,7 @@ type DataCenterInfo struct { func (m *DataCenterInfo) Reset() { *m = DataCenterInfo{} } func (m *DataCenterInfo) String() string { return proto.CompactTextString(m) } func (*DataCenterInfo) ProtoMessage() {} -func (*DataCenterInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} } +func (*DataCenterInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} } func (m *DataCenterInfo) GetId() string { if m != nil { @@ -1015,7 +1074,7 @@ type TopologyInfo struct { func (m *TopologyInfo) Reset() { *m = TopologyInfo{} } func (m *TopologyInfo) String() string { return proto.CompactTextString(m) } func (*TopologyInfo) ProtoMessage() {} -func (*TopologyInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} } +func (*TopologyInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} } func (m *TopologyInfo) GetId() string { if m != nil { @@ -1065,7 +1124,7 @@ type VolumeListRequest struct { func (m *VolumeListRequest) Reset() { *m = VolumeListRequest{} } func (m *VolumeListRequest) String() string { return proto.CompactTextString(m) } func (*VolumeListRequest) ProtoMessage() {} -func (*VolumeListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} } +func (*VolumeListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} } type VolumeListResponse struct { TopologyInfo *TopologyInfo `protobuf:"bytes,1,opt,name=topology_info,json=topologyInfo" json:"topology_info,omitempty"` @@ -1075,7 +1134,7 @@ type VolumeListResponse struct { func (m *VolumeListResponse) Reset() { *m = VolumeListResponse{} } func (m *VolumeListResponse) String() string { return proto.CompactTextString(m) } func (*VolumeListResponse) ProtoMessage() {} -func (*VolumeListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} } +func (*VolumeListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} } func (m *VolumeListResponse) GetTopologyInfo() *TopologyInfo { if m != nil { @@ -1096,6 +1155,7 @@ func init() { proto.RegisterType((*HeartbeatResponse)(nil), "master_pb.HeartbeatResponse") proto.RegisterType((*VolumeInformationMessage)(nil), "master_pb.VolumeInformationMessage") proto.RegisterType((*VolumeShortInformationMessage)(nil), "master_pb.VolumeShortInformationMessage") + proto.RegisterType((*VolumeEcShardInformationMessage)(nil), "master_pb.VolumeEcShardInformationMessage") proto.RegisterType((*Empty)(nil), "master_pb.Empty") proto.RegisterType((*SuperBlockExtra)(nil), "master_pb.SuperBlockExtra") proto.RegisterType((*SuperBlockExtra_ErasureCoding)(nil), "master_pb.SuperBlockExtra.ErasureCoding") @@ -1494,98 +1554,104 @@ var _Seaweed_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("master.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1482 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xd4, 0x58, 0x4d, 0x6f, 0xdb, 0x46, - 0x13, 0x36, 0xa9, 0x0f, 0x4b, 0xa3, 0xef, 0xb5, 0x93, 0x30, 0xca, 0x9b, 0x44, 0x61, 0x2e, 0xca, - 0xfb, 0xe1, 0x37, 0x75, 0x0f, 0x3d, 0xb4, 0x45, 0x90, 0x38, 0x0e, 0x6a, 0xc4, 0x69, 0x12, 0x2a, - 0x49, 0x81, 0x02, 0x05, 0xbb, 0x22, 0xd7, 0x0e, 0x61, 0x8a, 0x64, 0xc9, 0x95, 0x63, 0xe5, 0xd2, - 0x43, 0x7b, 0x2c, 0xda, 0x43, 0xff, 0x44, 0x7f, 0x45, 0x2f, 0x3d, 0xe6, 0xc7, 0x14, 0xe8, 0xbd, - 0x40, 0xb1, 0x1f, 0x24, 0x97, 0x94, 0x6c, 0xa7, 0x05, 0x72, 0xc8, 0x6d, 0x77, 0x66, 0x76, 0x76, - 0xf6, 0x99, 0xe1, 0x33, 0x23, 0x41, 0x7b, 0x86, 0x13, 0x4a, 0xe2, 0xad, 0x28, 0x0e, 0x69, 0x88, - 0x9a, 0x62, 0x67, 0x47, 0x53, 0xf3, 0x4d, 0x05, 0x9a, 0x9f, 0x11, 0x1c, 0xd3, 0x29, 0xc1, 0x14, - 0x75, 0x41, 0xf7, 0x22, 0x43, 0x1b, 0x69, 0xe3, 0xa6, 0xa5, 0x7b, 0x11, 0x42, 0x50, 0x8d, 0xc2, - 0x98, 0x1a, 0xfa, 0x48, 0x1b, 0x77, 0x2c, 0xbe, 0x46, 0x57, 0x01, 0xa2, 0xf9, 0xd4, 0xf7, 0x1c, - 0x7b, 0x1e, 0xfb, 0x46, 0x85, 0xdb, 0x36, 0x85, 0xe4, 0x79, 0xec, 0xa3, 0x31, 0xf4, 0x67, 0xf8, - 0xc4, 0x3e, 0x0e, 0xfd, 0xf9, 0x8c, 0xd8, 0x4e, 0x38, 0x0f, 0xa8, 0x51, 0xe5, 0xc7, 0xbb, 0x33, - 0x7c, 0xf2, 0x82, 0x8b, 0x77, 0x98, 0x14, 0x8d, 0x58, 0x54, 0x27, 0xf6, 0x81, 0xe7, 0x13, 0xfb, - 0x88, 0x2c, 0x8c, 0xda, 0x48, 0x1b, 0x57, 0x2d, 0x98, 0xe1, 0x93, 0x07, 0x9e, 0x4f, 0x1e, 0x92, - 0x05, 0xba, 0x0e, 0x2d, 0x17, 0x53, 0x6c, 0x3b, 0x24, 0xa0, 0x24, 0x36, 0xea, 0xfc, 0x2e, 0x60, - 0xa2, 0x1d, 0x2e, 0x61, 0xf1, 0xc5, 0xd8, 0x39, 0x32, 0xd6, 0xb9, 0x86, 0xaf, 0x59, 0x7c, 0xd8, - 0x9d, 0x79, 0x81, 0xcd, 0x23, 0x6f, 0xf0, 0xab, 0x9b, 0x5c, 0xf2, 0x84, 0x85, 0xff, 0x29, 0xac, - 0x8b, 0xd8, 0x12, 0xa3, 0x39, 0xaa, 0x8c, 0x5b, 0xdb, 0x37, 0xb7, 0x32, 0x34, 0xb6, 0x44, 0x78, - 0x7b, 0xc1, 0x41, 0x18, 0xcf, 0x30, 0xf5, 0xc2, 0xe0, 0x11, 0x49, 0x12, 0x7c, 0x48, 0xac, 0xf4, - 0x0c, 0xda, 0x83, 0x56, 0x40, 0x5e, 0xd9, 0xa9, 0x0b, 0xe0, 0x2e, 0xc6, 0x4b, 0x2e, 0x26, 0x2f, - 0xc3, 0x98, 0xae, 0xf0, 0x03, 0x01, 0x79, 0xf5, 0x42, 0xba, 0x7a, 0x0a, 0x3d, 0x97, 0xf8, 0x84, - 0x12, 0x37, 0x73, 0xd7, 0xfa, 0x9b, 0xee, 0xba, 0xd2, 0x81, 0x74, 0x69, 0x3e, 0x87, 0x41, 0x96, - 0x4c, 0x8b, 0x24, 0x51, 0x18, 0x24, 0x04, 0x8d, 0xa1, 0x27, 0xfc, 0x4f, 0xbc, 0xd7, 0x64, 0xdf, - 0x9b, 0x79, 0x94, 0x67, 0xb8, 0x6a, 0x95, 0xc5, 0xe8, 0x22, 0xd4, 0x7d, 0x82, 0x5d, 0x12, 0xcb, - 0xb4, 0xca, 0x9d, 0xf9, 0xbb, 0x0e, 0xc6, 0x69, 0xd0, 0xf0, 0x9a, 0x71, 0xb9, 0xc7, 0x8e, 0xa5, - 0x7b, 0x2e, 0xcb, 0x49, 0xe2, 0xbd, 0x26, 0xbc, 0x66, 0xaa, 0x16, 0x5f, 0xa3, 0x6b, 0x00, 0x4e, - 0xe8, 0xfb, 0xc4, 0x61, 0x07, 0xa5, 0x73, 0x45, 0xc2, 0x72, 0xc6, 0xcb, 0x20, 0x2f, 0x97, 0xaa, - 0xd5, 0x64, 0x12, 0x51, 0x29, 0x37, 0xa0, 0x2d, 0x1e, 0x2a, 0x0d, 0x44, 0xa5, 0xb4, 0x84, 0x4c, - 0x98, 0xfc, 0x17, 0x50, 0x0a, 0xe6, 0x74, 0x91, 0x19, 0xd6, 0xb9, 0x61, 0x5f, 0x6a, 0xee, 0x2d, - 0x52, 0xeb, 0x2b, 0xd0, 0x8c, 0x09, 0x76, 0xed, 0x30, 0xf0, 0x17, 0xbc, 0x78, 0x1a, 0x56, 0x83, - 0x09, 0x1e, 0x07, 0xfe, 0x02, 0xfd, 0x07, 0x06, 0x31, 0x89, 0x7c, 0xcf, 0xc1, 0x76, 0xe4, 0x63, - 0x87, 0xcc, 0x48, 0x90, 0xd6, 0x51, 0x5f, 0x2a, 0x9e, 0xa4, 0x72, 0x64, 0xc0, 0xfa, 0x31, 0x89, - 0x13, 0xf6, 0xac, 0x26, 0x37, 0x49, 0xb7, 0xa8, 0x0f, 0x15, 0x4a, 0x7d, 0x03, 0xb8, 0x94, 0x2d, - 0xd1, 0x2d, 0xe8, 0x3b, 0xe1, 0x2c, 0xc2, 0x0e, 0xb5, 0x63, 0x72, 0xec, 0xf1, 0x43, 0x2d, 0xae, - 0xee, 0x49, 0xb9, 0x25, 0xc5, 0xe6, 0x2f, 0x1a, 0x5c, 0x3d, 0x33, 0xf5, 0x4b, 0xb0, 0x9f, 0x07, - 0xf1, 0xbb, 0x7a, 0x95, 0xb9, 0x0e, 0xb5, 0xdd, 0x59, 0x44, 0x17, 0xe6, 0xaf, 0x1a, 0xf4, 0x26, - 0xf3, 0x88, 0xc4, 0xf7, 0xfc, 0xd0, 0x39, 0xda, 0x3d, 0xa1, 0x31, 0x46, 0x8f, 0xa1, 0x4b, 0x62, - 0x9c, 0xcc, 0x63, 0x96, 0x11, 0xd7, 0x0b, 0x0e, 0x79, 0xc4, 0xc5, 0x12, 0x2f, 0x9d, 0xd9, 0xda, - 0x15, 0x07, 0x76, 0xb8, 0xbd, 0xd5, 0x21, 0xea, 0x76, 0xf8, 0x25, 0x74, 0x0a, 0x7a, 0x56, 0x6e, - 0x8c, 0x10, 0x24, 0x12, 0x7c, 0xcd, 0xea, 0x38, 0xc2, 0xb1, 0x47, 0x17, 0x92, 0xb8, 0xe4, 0x8e, - 0x95, 0x99, 0xe4, 0x25, 0xcf, 0x4d, 0x8c, 0xca, 0xa8, 0xc2, 0xa8, 0x41, 0x48, 0xf6, 0xdc, 0xc4, - 0xbc, 0x05, 0x1b, 0x3b, 0xbe, 0x47, 0x02, 0xba, 0xef, 0x25, 0x94, 0x04, 0x16, 0xf9, 0x66, 0x4e, - 0x12, 0xca, 0x6e, 0x08, 0xf0, 0x8c, 0x48, 0x5a, 0xe4, 0x6b, 0xf3, 0x5b, 0xe8, 0x8a, 0xf4, 0xec, - 0x87, 0x0e, 0x4f, 0x0b, 0x03, 0x86, 0xf1, 0xa1, 0x30, 0x62, 0xcb, 0x12, 0x51, 0xea, 0x65, 0xa2, - 0xbc, 0x0c, 0x0d, 0xce, 0x24, 0x79, 0x28, 0xeb, 0x8c, 0x1c, 0x3c, 0x37, 0xc9, 0xeb, 0xdd, 0x15, - 0xea, 0x2a, 0x57, 0xb7, 0xd2, 0x8f, 0xdd, 0x73, 0x13, 0xf3, 0x19, 0x6c, 0xec, 0x87, 0xe1, 0xd1, - 0x3c, 0x12, 0x61, 0xa4, 0xb1, 0x16, 0x5f, 0xa8, 0x8d, 0x2a, 0xec, 0xce, 0xec, 0x85, 0xa5, 0x22, - 0xd1, 0xcb, 0x45, 0x62, 0xfe, 0xa1, 0xc1, 0x66, 0xd1, 0xad, 0xe4, 0x90, 0xaf, 0x61, 0x23, 0xf3, - 0x6b, 0xfb, 0xf2, 0xcd, 0xe2, 0x82, 0xd6, 0xf6, 0x6d, 0x25, 0x99, 0xab, 0x4e, 0xa7, 0xb4, 0xea, - 0xa6, 0x60, 0x59, 0x83, 0xe3, 0x92, 0x24, 0x19, 0x9e, 0x40, 0xbf, 0x6c, 0xc6, 0x3e, 0xd3, 0xec, - 0x56, 0x89, 0x6c, 0x23, 0x3d, 0x89, 0x3e, 0x80, 0x66, 0x1e, 0x88, 0xce, 0x03, 0xd9, 0x28, 0x04, - 0x22, 0xef, 0xca, 0xad, 0xd0, 0x26, 0xd4, 0x48, 0x1c, 0x87, 0x29, 0xbd, 0x89, 0x8d, 0xf9, 0x31, - 0x34, 0xfe, 0x71, 0x16, 0xcd, 0x37, 0x1a, 0x74, 0xee, 0x26, 0x89, 0x77, 0x98, 0x95, 0xcb, 0x26, - 0xd4, 0x04, 0xf9, 0x08, 0x92, 0x15, 0x1b, 0x34, 0x82, 0x96, 0xfc, 0xca, 0x14, 0xe8, 0x55, 0xd1, - 0xb9, 0x1f, 0xb0, 0xfc, 0xf2, 0xaa, 0x22, 0x34, 0xc6, 0x27, 0xa5, 0xf6, 0x58, 0x3b, 0xb5, 0x3d, - 0xd6, 0x95, 0xf6, 0x78, 0x05, 0x9a, 0xfc, 0x50, 0x10, 0xba, 0x44, 0xf6, 0xcd, 0x06, 0x13, 0x7c, - 0x1e, 0xba, 0xc4, 0xfc, 0x59, 0x83, 0x6e, 0xfa, 0x1a, 0x99, 0xf9, 0x3e, 0x54, 0x0e, 0x32, 0xf4, - 0xd9, 0x32, 0xc5, 0x48, 0x3f, 0x0d, 0xa3, 0xa5, 0x91, 0x20, 0x43, 0xa4, 0xaa, 0x22, 0x92, 0x25, - 0xa3, 0xa6, 0x24, 0x83, 0x85, 0x8c, 0xe7, 0xf4, 0x65, 0x1a, 0x32, 0x5b, 0x9b, 0x87, 0x30, 0x98, - 0x50, 0x4c, 0xbd, 0x84, 0x7a, 0x4e, 0x92, 0xc2, 0x5c, 0x02, 0x54, 0x3b, 0x0f, 0x50, 0xfd, 0x34, - 0x40, 0x2b, 0x19, 0xa0, 0xe6, 0x6f, 0x1a, 0x20, 0xf5, 0x26, 0x09, 0xc1, 0x3b, 0xb8, 0x8a, 0x41, - 0x46, 0x43, 0x8a, 0x7d, 0x9b, 0xf7, 0x4a, 0xd9, 0xf1, 0xb8, 0x84, 0xb5, 0x63, 0x96, 0xa5, 0x79, - 0x42, 0x5c, 0xa1, 0x15, 0xed, 0xae, 0xc1, 0x04, 0x5c, 0x59, 0xec, 0x96, 0xf5, 0x52, 0xb7, 0x34, - 0xef, 0x42, 0x6b, 0x42, 0xc3, 0x18, 0x1f, 0x92, 0x67, 0x8b, 0xe8, 0x6d, 0xa2, 0x97, 0xd1, 0xe9, - 0x39, 0x10, 0x23, 0x80, 0x9d, 0x3c, 0xfa, 0x55, 0x04, 0x78, 0x09, 0x2e, 0xe4, 0x16, 0x8c, 0x2f, - 0x65, 0x5e, 0xcc, 0xa7, 0x70, 0xb1, 0xac, 0x90, 0x30, 0x7e, 0x04, 0xad, 0x1c, 0x92, 0x94, 0x3b, - 0x2e, 0x28, 0x9f, 0x6c, 0x7e, 0xce, 0x52, 0x2d, 0xcd, 0xff, 0xc1, 0xa5, 0x5c, 0x75, 0x9f, 0x93, - 0xe0, 0x59, 0xdc, 0x3c, 0x04, 0x63, 0xd9, 0x5c, 0xc4, 0x60, 0xfe, 0xa4, 0x43, 0xfb, 0xbe, 0xac, - 0x76, 0xd6, 0x54, 0x95, 0x36, 0xda, 0xe4, 0x6d, 0xf4, 0x06, 0xb4, 0x0b, 0xa3, 0xab, 0x98, 0x62, - 0x5a, 0xc7, 0xca, 0xdc, 0xba, 0x6a, 0xc2, 0xad, 0x70, 0xb3, 0xf2, 0x84, 0xfb, 0x6f, 0x18, 0x1c, - 0xc4, 0x84, 0x2c, 0x0f, 0xc3, 0x55, 0xab, 0xc7, 0x14, 0xaa, 0xed, 0x16, 0x6c, 0x60, 0x87, 0x7a, - 0xc7, 0x25, 0x6b, 0x91, 0xfb, 0x81, 0x50, 0xa9, 0xf6, 0x0f, 0xb2, 0x40, 0xbd, 0xe0, 0x20, 0x4c, - 0x8c, 0xfa, 0xdb, 0x0f, 0xb3, 0xf2, 0x35, 0x4c, 0x93, 0x98, 0xdf, 0xeb, 0xd0, 0xb0, 0xb0, 0x73, - 0xf4, 0x7e, 0xa3, 0x71, 0x07, 0x7a, 0x19, 0xab, 0x15, 0x00, 0xb9, 0xa4, 0x00, 0xa2, 0x26, 0xde, - 0xea, 0xb8, 0xca, 0x2e, 0x31, 0xff, 0xd4, 0xa0, 0x7b, 0x3f, 0x63, 0xce, 0xf7, 0x1b, 0x8c, 0x6d, - 0x00, 0x46, 0xf5, 0x05, 0x1c, 0xd4, 0xd6, 0x98, 0xa6, 0xdb, 0x6a, 0xc6, 0x72, 0x95, 0x98, 0x3f, - 0xea, 0xd0, 0x7e, 0x16, 0x46, 0xa1, 0x1f, 0x1e, 0x2e, 0xde, 0xef, 0xd7, 0xef, 0xc2, 0x40, 0xe9, - 0x8a, 0x05, 0x10, 0x2e, 0x97, 0x8a, 0x21, 0x4f, 0xb6, 0xd5, 0x73, 0x0b, 0xfb, 0xc4, 0xdc, 0x80, - 0x81, 0x9c, 0xf0, 0x14, 0x72, 0xfb, 0x4e, 0x03, 0xa4, 0x4a, 0x25, 0xb3, 0x7d, 0x02, 0x1d, 0x2a, - 0xb1, 0xe3, 0xf7, 0xc9, 0x21, 0x57, 0xad, 0x3d, 0x15, 0x5b, 0xab, 0x4d, 0x55, 0xa4, 0xff, 0x0f, - 0x9b, 0xf2, 0x65, 0x8c, 0xed, 0x6d, 0x9f, 0xfd, 0x14, 0xb3, 0x67, 0x53, 0x89, 0xf0, 0xa0, 0xf4, - 0x23, 0xed, 0xd1, 0x74, 0xfb, 0x87, 0x1a, 0xac, 0x4f, 0x08, 0x7e, 0x45, 0x88, 0x8b, 0xf6, 0xa0, - 0x33, 0x21, 0x81, 0x9b, 0xff, 0x84, 0xdf, 0x54, 0x2e, 0xcd, 0xa4, 0xc3, 0x7f, 0xad, 0x92, 0x66, - 0xac, 0xb8, 0x36, 0xd6, 0x6e, 0x6b, 0xe8, 0x09, 0x74, 0x1e, 0x12, 0x12, 0xed, 0x84, 0x41, 0x40, - 0x1c, 0x4a, 0x5c, 0x74, 0x4d, 0xe5, 0xe6, 0xe5, 0xc1, 0x78, 0x78, 0x79, 0x89, 0x6c, 0xd2, 0x39, - 0x4a, 0x7a, 0x7c, 0x0a, 0x6d, 0x75, 0x1e, 0x2c, 0x38, 0x5c, 0x31, 0xbd, 0x0e, 0xaf, 0x9f, 0x33, - 0x48, 0x9a, 0x6b, 0xe8, 0x0e, 0xd4, 0xc5, 0x80, 0x82, 0x0c, 0xc5, 0xb8, 0x30, 0x81, 0x15, 0xe2, - 0x2a, 0x4e, 0x33, 0xe6, 0x1a, 0x7a, 0x08, 0x90, 0xb7, 0x78, 0xa4, 0xe2, 0xb2, 0x34, 0x63, 0x0c, - 0xaf, 0x9e, 0xa2, 0xcd, 0x9c, 0x7d, 0x01, 0xdd, 0x62, 0xb3, 0x43, 0xa3, 0x95, 0xfd, 0x4c, 0xa9, - 0xa1, 0xe1, 0x8d, 0x33, 0x2c, 0x32, 0xc7, 0x5f, 0x41, 0xbf, 0xdc, 0xc3, 0x90, 0xb9, 0xf2, 0x60, - 0xa1, 0x1f, 0x0e, 0x6f, 0x9e, 0x69, 0xa3, 0x82, 0x90, 0x97, 0x71, 0x01, 0x84, 0xa5, 0x9a, 0x2f, - 0x80, 0xb0, 0x5c, 0xfb, 0xe6, 0xda, 0xb4, 0xce, 0xff, 0x54, 0xfa, 0xf0, 0xaf, 0x00, 0x00, 0x00, - 0xff, 0xff, 0x94, 0xee, 0xe2, 0x89, 0x64, 0x12, 0x00, 0x00, + // 1572 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xd4, 0x58, 0xcd, 0x6e, 0xdc, 0x46, + 0x12, 0x16, 0xe7, 0x4f, 0x33, 0x35, 0x9a, 0xd1, 0x4c, 0x4b, 0xb6, 0xa9, 0xf1, 0xda, 0x1e, 0xd3, + 0x97, 0xb1, 0x77, 0x57, 0xeb, 0xd5, 0x1e, 0xf6, 0xb0, 0x1b, 0x18, 0xb6, 0x2c, 0x27, 0x82, 0xe5, + 0x3f, 0x8e, 0xed, 0x00, 0x01, 0x02, 0x86, 0x22, 0x4b, 0x32, 0x21, 0x0e, 0xc9, 0x90, 0x3d, 0xb2, + 0xc6, 0x97, 0x1c, 0x92, 0x63, 0x90, 0x1c, 0xf2, 0x12, 0x79, 0x88, 0x20, 0x97, 0x1c, 0xf3, 0x30, + 0x01, 0x72, 0x0f, 0x10, 0xf4, 0x0f, 0xc9, 0x26, 0x67, 0x24, 0xd9, 0x01, 0x7c, 0xf0, 0xad, 0xbb, + 0xaa, 0xba, 0xba, 0xfa, 0xeb, 0xea, 0xaf, 0x8a, 0x84, 0x95, 0x89, 0x9d, 0x50, 0x8c, 0x37, 0xa3, + 0x38, 0xa4, 0x21, 0x69, 0x89, 0x99, 0x15, 0xed, 0x1b, 0x3f, 0xd5, 0xa1, 0xf5, 0x09, 0xda, 0x31, + 0xdd, 0x47, 0x9b, 0x92, 0x2e, 0x54, 0xbc, 0x48, 0xd7, 0x86, 0xda, 0xa8, 0x65, 0x56, 0xbc, 0x88, + 0x10, 0xa8, 0x45, 0x61, 0x4c, 0xf5, 0xca, 0x50, 0x1b, 0x75, 0x4c, 0x3e, 0x26, 0x57, 0x00, 0xa2, + 0xe9, 0xbe, 0xef, 0x39, 0xd6, 0x34, 0xf6, 0xf5, 0x2a, 0xb7, 0x6d, 0x09, 0xc9, 0x8b, 0xd8, 0x27, + 0x23, 0xe8, 0x4d, 0xec, 0x13, 0xeb, 0x38, 0xf4, 0xa7, 0x13, 0xb4, 0x9c, 0x70, 0x1a, 0x50, 0xbd, + 0xc6, 0x97, 0x77, 0x27, 0xf6, 0xc9, 0x4b, 0x2e, 0xde, 0x66, 0x52, 0x32, 0x64, 0x51, 0x9d, 0x58, + 0x07, 0x9e, 0x8f, 0xd6, 0x11, 0xce, 0xf4, 0xfa, 0x50, 0x1b, 0xd5, 0x4c, 0x98, 0xd8, 0x27, 0x0f, + 0x3c, 0x1f, 0x1f, 0xe2, 0x8c, 0x5c, 0x83, 0xb6, 0x6b, 0x53, 0xdb, 0x72, 0x30, 0xa0, 0x18, 0xeb, + 0x0d, 0xbe, 0x17, 0x30, 0xd1, 0x36, 0x97, 0xb0, 0xf8, 0x62, 0xdb, 0x39, 0xd2, 0x97, 0xb9, 0x86, + 0x8f, 0x59, 0x7c, 0xb6, 0x3b, 0xf1, 0x02, 0x8b, 0x47, 0xde, 0xe4, 0x5b, 0xb7, 0xb8, 0xe4, 0x29, + 0x0b, 0xff, 0x23, 0x58, 0x16, 0xb1, 0x25, 0x7a, 0x6b, 0x58, 0x1d, 0xb5, 0xb7, 0x6e, 0x6c, 0x66, + 0x68, 0x6c, 0x8a, 0xf0, 0x76, 0x83, 0x83, 0x30, 0x9e, 0xd8, 0xd4, 0x0b, 0x83, 0x47, 0x98, 0x24, + 0xf6, 0x21, 0x9a, 0xe9, 0x1a, 0xb2, 0x0b, 0xed, 0x00, 0x5f, 0x5b, 0xa9, 0x0b, 0xe0, 0x2e, 0x46, + 0x73, 0x2e, 0xc6, 0xaf, 0xc2, 0x98, 0x2e, 0xf0, 0x03, 0x01, 0xbe, 0x7e, 0x29, 0x5d, 0x3d, 0x83, + 0x55, 0x17, 0x7d, 0xa4, 0xe8, 0x66, 0xee, 0xda, 0xef, 0xe8, 0xae, 0x2b, 0x1d, 0xa4, 0x2e, 0x3f, + 0x86, 0x16, 0x3a, 0x56, 0xf2, 0xca, 0x8e, 0xdd, 0x44, 0xef, 0x71, 0x67, 0xb7, 0xe6, 0x9c, 0xed, + 0x38, 0x63, 0x66, 0xb0, 0xc0, 0x5d, 0x13, 0x85, 0x2a, 0x21, 0x8f, 0xa1, 0xc3, 0x8e, 0x99, 0x3b, + 0xeb, 0xbf, 0xb3, 0x33, 0x86, 0xd3, 0x4e, 0xea, 0xef, 0x25, 0xf4, 0xd3, 0xb3, 0xe6, 0x3e, 0xc9, + 0x3b, 0xfb, 0x4c, 0x01, 0x4b, 0xfd, 0x1a, 0x2f, 0xa0, 0x9f, 0x65, 0xaf, 0x89, 0x49, 0x14, 0x06, + 0x09, 0x92, 0x11, 0xac, 0x0a, 0x40, 0xc7, 0xde, 0x1b, 0xdc, 0xf3, 0x26, 0x1e, 0xe5, 0x29, 0x5d, + 0x33, 0xcb, 0x62, 0x72, 0x11, 0x1a, 0x3e, 0xda, 0x2e, 0xc6, 0x32, 0x8f, 0xe5, 0xcc, 0xf8, 0xad, + 0x02, 0xfa, 0x69, 0xb9, 0xc0, 0x1f, 0x89, 0xcb, 0x3d, 0x76, 0xcc, 0x8a, 0xe7, 0xb2, 0x24, 0x4c, + 0xbc, 0x37, 0xc8, 0x1f, 0x49, 0xcd, 0xe4, 0x63, 0x72, 0x15, 0xc0, 0x09, 0x7d, 0x1f, 0x1d, 0xb6, + 0x50, 0x3a, 0x57, 0x24, 0x2c, 0x49, 0x79, 0xde, 0xe7, 0xef, 0xa3, 0x66, 0xb6, 0x98, 0x44, 0x3c, + 0x8d, 0xeb, 0xb0, 0x22, 0x4e, 0x2a, 0x0d, 0xc4, 0xd3, 0x68, 0x0b, 0x99, 0x30, 0xf9, 0x07, 0x90, + 0x14, 0xd1, 0xfd, 0x59, 0x66, 0xd8, 0xe0, 0x86, 0x3d, 0xa9, 0xb9, 0x37, 0x4b, 0xad, 0x2f, 0x43, + 0x2b, 0x46, 0xdb, 0xb5, 0xc2, 0xc0, 0x9f, 0xf1, 0xd7, 0xd2, 0x34, 0x9b, 0x4c, 0xf0, 0x24, 0xf0, + 0x67, 0xe4, 0xef, 0xd0, 0x8f, 0x31, 0xf2, 0x3d, 0xc7, 0xb6, 0x22, 0xdf, 0x76, 0x70, 0x82, 0x41, + 0xfa, 0x70, 0x7a, 0x52, 0xf1, 0x34, 0x95, 0x13, 0x1d, 0x96, 0x8f, 0x31, 0x4e, 0xd8, 0xb1, 0x5a, + 0xdc, 0x24, 0x9d, 0x92, 0x1e, 0x54, 0x29, 0xf5, 0x75, 0xe0, 0x52, 0x36, 0x24, 0x37, 0xa1, 0xe7, + 0x84, 0x93, 0xc8, 0x76, 0xa8, 0x15, 0xe3, 0xb1, 0xc7, 0x17, 0xb5, 0xb9, 0x7a, 0x55, 0xca, 0x4d, + 0x29, 0x36, 0x7e, 0xd4, 0xe0, 0xca, 0x99, 0xb9, 0x3e, 0x07, 0xfb, 0x79, 0x10, 0xbf, 0xaf, 0x53, + 0x19, 0x3e, 0x5c, 0x3b, 0x27, 0x4f, 0xcf, 0x89, 0xb5, 0x32, 0x17, 0xeb, 0x06, 0x34, 0xd1, 0xb1, + 0xbc, 0xc0, 0xc5, 0x13, 0x7e, 0x92, 0x8e, 0xb9, 0x8c, 0xce, 0x2e, 0x9b, 0x1a, 0xcb, 0x50, 0xdf, + 0x99, 0x44, 0x74, 0x66, 0xfc, 0xac, 0xc1, 0xea, 0x78, 0x1a, 0x61, 0x7c, 0xcf, 0x0f, 0x9d, 0xa3, + 0x9d, 0x13, 0x1a, 0xdb, 0xe4, 0x09, 0x74, 0x31, 0xb6, 0x93, 0x69, 0xcc, 0xee, 0xdf, 0xf5, 0x82, + 0x43, 0xbe, 0x67, 0x91, 0x41, 0x4a, 0x6b, 0x36, 0x77, 0xc4, 0x82, 0x6d, 0x6e, 0x6f, 0x76, 0x50, + 0x9d, 0x0e, 0x3e, 0x83, 0x4e, 0x41, 0xcf, 0x92, 0x9b, 0xf1, 0xad, 0x3c, 0x0b, 0x1f, 0xb3, 0x57, + 0x13, 0xd9, 0xb1, 0x47, 0x67, 0xb2, 0x2e, 0xc8, 0x19, 0x4b, 0x6a, 0x49, 0xfb, 0x9e, 0x9b, 0xe8, + 0xd5, 0x61, 0x95, 0x31, 0xaf, 0x90, 0xec, 0xba, 0x89, 0x71, 0x13, 0xd6, 0xb6, 0x7d, 0x0f, 0x03, + 0xba, 0xe7, 0x25, 0x14, 0x03, 0x13, 0xbf, 0x9c, 0x62, 0x42, 0xd9, 0x0e, 0x81, 0x3d, 0x41, 0x59, + 0x75, 0xf8, 0xd8, 0xf8, 0x0a, 0xba, 0x02, 0xe2, 0xbd, 0xd0, 0xe1, 0xc0, 0xb2, 0x6b, 0x60, 0xe5, + 0x46, 0x18, 0xb1, 0x61, 0xa9, 0x0e, 0x55, 0xca, 0x75, 0x68, 0x03, 0x9a, 0x9c, 0xa8, 0xf3, 0x50, + 0x96, 0x19, 0xf7, 0x7a, 0x6e, 0x92, 0xbf, 0x2e, 0x57, 0xa8, 0x6b, 0x5c, 0xdd, 0x4e, 0xb9, 0xd4, + 0x73, 0x13, 0xe3, 0x39, 0xac, 0xed, 0x85, 0xe1, 0xd1, 0x34, 0x12, 0x61, 0xa4, 0xb1, 0x16, 0x4f, + 0xa8, 0x0d, 0xab, 0x6c, 0xcf, 0xec, 0x84, 0xe7, 0x5d, 0xb3, 0xf1, 0xbb, 0x06, 0xeb, 0x45, 0xb7, + 0x92, 0xb1, 0xbe, 0x80, 0xb5, 0xcc, 0xaf, 0xe5, 0xcb, 0x33, 0x8b, 0x0d, 0xda, 0x5b, 0xb7, 0x95, + 0xcb, 0x5c, 0xb4, 0x3a, 0xad, 0x5a, 0x6e, 0x0a, 0x96, 0xd9, 0x3f, 0x2e, 0x49, 0x92, 0xc1, 0x09, + 0xf4, 0xca, 0x66, 0x8c, 0x14, 0xb2, 0x5d, 0x25, 0xb2, 0xcd, 0x74, 0x25, 0xf9, 0x37, 0xb4, 0xf2, + 0x40, 0x2a, 0x3c, 0x90, 0xb5, 0x42, 0x20, 0x72, 0xaf, 0xdc, 0x8a, 0xac, 0x43, 0x1d, 0xe3, 0x38, + 0x4c, 0xc9, 0x54, 0x4c, 0x8c, 0xff, 0x41, 0xf3, 0x2f, 0xdf, 0xa2, 0xf1, 0xab, 0x06, 0x9d, 0xbb, + 0x49, 0xe2, 0x1d, 0x66, 0xe9, 0xb2, 0x0e, 0x75, 0x41, 0x75, 0x82, 0xd2, 0xc5, 0x84, 0x0c, 0xa1, + 0x2d, 0xdf, 0xb4, 0x02, 0xbd, 0x2a, 0x3a, 0x97, 0x2e, 0xe4, 0x3b, 0xaf, 0x89, 0xd0, 0x18, 0x7b, + 0x95, 0xba, 0x8f, 0xfa, 0xa9, 0xdd, 0x47, 0x43, 0xe9, 0x3e, 0x2e, 0x43, 0x8b, 0x2f, 0x0a, 0x42, + 0x17, 0x65, 0x5b, 0xd2, 0x64, 0x82, 0xc7, 0xa1, 0x8b, 0xc6, 0x0f, 0x1a, 0x74, 0xd3, 0xd3, 0xc8, + 0x9b, 0xef, 0x41, 0xf5, 0x20, 0x43, 0x9f, 0x0d, 0x53, 0x8c, 0x2a, 0xa7, 0x61, 0x34, 0xd7, 0x71, + 0x65, 0x88, 0xd4, 0x54, 0x44, 0xb2, 0xcb, 0xa8, 0x2b, 0x97, 0xc1, 0x42, 0xb6, 0xa7, 0xf4, 0x55, + 0x1a, 0x32, 0x1b, 0x1b, 0x87, 0xd0, 0x1f, 0x53, 0x9b, 0x7a, 0x09, 0xf5, 0x9c, 0x24, 0x85, 0xb9, + 0x04, 0xa8, 0x76, 0x1e, 0xa0, 0x95, 0xd3, 0x00, 0xad, 0x66, 0x80, 0x1a, 0xbf, 0x68, 0x40, 0xd4, + 0x9d, 0x24, 0x04, 0xef, 0x61, 0x2b, 0x06, 0x19, 0x0d, 0xa9, 0xed, 0x5b, 0xbc, 0x32, 0xcb, 0xfa, + 0xca, 0x25, 0xac, 0xf8, 0xb3, 0x5b, 0x9a, 0x26, 0xe8, 0x0a, 0xad, 0x28, 0xae, 0x4d, 0x26, 0xe0, + 0xca, 0x62, 0x6d, 0x6e, 0x94, 0x6a, 0xb3, 0x71, 0x17, 0xda, 0x63, 0x1a, 0xc6, 0xf6, 0x21, 0x3e, + 0x9f, 0x45, 0x6f, 0x13, 0xbd, 0x8c, 0xae, 0x92, 0x03, 0x31, 0x04, 0xd8, 0xce, 0xa3, 0x5f, 0x44, + 0x80, 0x97, 0xe0, 0x42, 0x6e, 0xc1, 0xf8, 0x52, 0xde, 0x8b, 0xf1, 0x0c, 0x2e, 0x96, 0x15, 0x12, + 0xc6, 0xff, 0x42, 0x3b, 0x87, 0x24, 0xe5, 0x8e, 0x0b, 0xca, 0x93, 0xcd, 0xd7, 0x99, 0xaa, 0xa5, + 0xf1, 0x4f, 0xb8, 0x94, 0xab, 0xee, 0x73, 0x12, 0x3c, 0x8b, 0x9b, 0x07, 0xa0, 0xcf, 0x9b, 0x8b, + 0x18, 0x8c, 0xef, 0x2b, 0xb0, 0x72, 0x5f, 0x66, 0x3b, 0x2b, 0x8b, 0x4a, 0x21, 0x6c, 0xf1, 0x42, + 0x78, 0x1d, 0x56, 0x0a, 0x5f, 0x06, 0xa2, 0x67, 0x6a, 0x1f, 0x2b, 0x9f, 0x05, 0x8b, 0x3e, 0x20, + 0xaa, 0xdc, 0xac, 0xfc, 0x01, 0x71, 0x0b, 0xfa, 0x07, 0x31, 0xe2, 0xfc, 0xb7, 0x46, 0xcd, 0x5c, + 0x65, 0x0a, 0xd5, 0x76, 0x13, 0xd6, 0x6c, 0x87, 0x7a, 0xc7, 0x25, 0x6b, 0x71, 0xf7, 0x7d, 0xa1, + 0x52, 0xed, 0x1f, 0x64, 0x81, 0x7a, 0xc1, 0x41, 0x98, 0xe8, 0x8d, 0xb7, 0xff, 0x56, 0x90, 0xa7, + 0x61, 0x9a, 0xc4, 0xf8, 0xa6, 0x02, 0x4d, 0xd3, 0x76, 0x8e, 0x3e, 0x6c, 0x34, 0xee, 0xc0, 0x6a, + 0xc6, 0x6a, 0x05, 0x40, 0x2e, 0x29, 0x80, 0xa8, 0x17, 0x6f, 0x76, 0x5c, 0x65, 0x96, 0x18, 0x7f, + 0x68, 0xd0, 0xbd, 0x9f, 0x31, 0xe7, 0x87, 0x0d, 0xc6, 0x16, 0x00, 0xa3, 0xfa, 0x02, 0x0e, 0x6a, + 0x69, 0x4c, 0xaf, 0xdb, 0x6c, 0xc5, 0x72, 0x94, 0x18, 0xdf, 0x55, 0x60, 0xe5, 0x79, 0x18, 0x85, + 0x7e, 0x78, 0x38, 0xfb, 0xb0, 0x4f, 0xbf, 0x03, 0x7d, 0xa5, 0x2a, 0x16, 0x40, 0xd8, 0x28, 0x25, + 0x43, 0x7e, 0xd9, 0xe6, 0xaa, 0x5b, 0x98, 0x27, 0xc6, 0x1a, 0xf4, 0x65, 0x87, 0xa7, 0x90, 0xdb, + 0xd7, 0x1a, 0x10, 0x55, 0x2a, 0x99, 0xed, 0xff, 0xd0, 0xa1, 0x12, 0x3b, 0xbe, 0x9f, 0x6c, 0x72, + 0xd5, 0xdc, 0x53, 0xb1, 0x35, 0x57, 0xa8, 0x8a, 0xf4, 0xbf, 0x60, 0x5d, 0x9e, 0x8c, 0xb1, 0xbd, + 0xe5, 0xb3, 0x0f, 0x3f, 0x6b, 0xb2, 0x2f, 0x11, 0xee, 0x97, 0x3e, 0x09, 0x1f, 0xed, 0x6f, 0x7d, + 0x5b, 0x87, 0xe5, 0x31, 0xda, 0xaf, 0x11, 0x5d, 0xb2, 0x0b, 0x9d, 0x31, 0x06, 0x6e, 0xfe, 0x87, + 0x64, 0x5d, 0xd9, 0x34, 0x93, 0x0e, 0xfe, 0xb6, 0x48, 0x9a, 0xb1, 0xe2, 0xd2, 0x48, 0xbb, 0xad, + 0x91, 0xa7, 0xd0, 0x79, 0x88, 0x18, 0x6d, 0x87, 0x41, 0x80, 0x0e, 0x45, 0x97, 0x5c, 0x55, 0xb9, + 0x79, 0xbe, 0x31, 0x1e, 0x6c, 0xcc, 0x91, 0x4d, 0xda, 0x47, 0x49, 0x8f, 0xcf, 0x60, 0x45, 0xed, + 0x07, 0x0b, 0x0e, 0x17, 0x74, 0xaf, 0x83, 0x6b, 0xe7, 0x34, 0x92, 0xc6, 0x12, 0xb9, 0x03, 0x0d, + 0xd1, 0xa0, 0x10, 0x5d, 0x31, 0x2e, 0x74, 0x60, 0x85, 0xb8, 0x8a, 0xdd, 0x8c, 0xb1, 0x44, 0x1e, + 0x02, 0xe4, 0x25, 0x9e, 0xa8, 0xb8, 0xcc, 0xf5, 0x18, 0x83, 0x2b, 0xa7, 0x68, 0x33, 0x67, 0x9f, + 0x42, 0xb7, 0x58, 0xec, 0xc8, 0x70, 0x61, 0x3d, 0x53, 0x72, 0x68, 0x70, 0xfd, 0x0c, 0x8b, 0xcc, + 0xf1, 0xe7, 0xd0, 0x2b, 0xd7, 0x30, 0x62, 0x2c, 0x5c, 0x58, 0xa8, 0x87, 0x83, 0x1b, 0x67, 0xda, + 0xa8, 0x20, 0xe4, 0x69, 0x5c, 0x00, 0x61, 0x2e, 0xe7, 0x0b, 0x20, 0xcc, 0xe7, 0xbe, 0xb1, 0xb4, + 0xdf, 0xe0, 0xff, 0xec, 0xfe, 0xf3, 0x67, 0x00, 0x00, 0x00, 0xff, 0xff, 0x63, 0xc4, 0xec, 0xbe, + 0xc3, 0x13, 0x00, 0x00, } diff --git a/weed/server/master_grpc_server.go b/weed/server/master_grpc_server.go index 1cc3fef7a..34962c83c 100644 --- a/weed/server/master_grpc_server.go +++ b/weed/server/master_grpc_server.go @@ -87,7 +87,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ } // update master internal volume layouts t.IncrementalSyncDataNodeRegistration(heartbeat.NewVolumes, heartbeat.DeletedVolumes, dn) - } else { + } else if len(heartbeat.Volumes) > 0 { // process heartbeat.Volumes newVolumes, deletedVolumes := t.SyncDataNodeRegistration(heartbeat.Volumes, dn) @@ -99,6 +99,8 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ glog.V(0).Infof("master see deleted volume %d from %s", uint32(v.Id), dn.Url()) message.DeletedVids = append(message.DeletedVids, uint32(v.Id)) } + } else if len(heartbeat.EcShards) > 0 { + glog.V(0).Infof("master recieved ec shards from %s: %+v", dn.Url(), heartbeat.EcShards) } if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 { diff --git a/weed/server/volume_grpc_client_to_master.go b/weed/server/volume_grpc_client_to_master.go index 0d4532485..6364cbc10 100644 --- a/weed/server/volume_grpc_client_to_master.go +++ b/weed/server/volume_grpc_client_to_master.go @@ -94,7 +94,13 @@ func (vs *VolumeServer) doHeartbeat(ctx context.Context, masterNode, masterGrpcA return "", err } - tickChan := time.Tick(sleepInterval) + if err = stream.Send(vs.store.CollectErasureCodingHeartbeat()); err != nil { + glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err) + return "", err + } + + volumeTickChan := time.Tick(sleepInterval) + ecShardTickChan := time.Tick(17 * sleepInterval) for { select { @@ -109,6 +115,17 @@ func (vs *VolumeServer) doHeartbeat(ctx context.Context, masterNode, masterGrpcA glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err) return "", err } + case ecShardMessage := <-vs.store.NewEcShardsChan: + deltaBeat := &master_pb.Heartbeat{ + NewEcShards: []*master_pb.VolumeEcShardInformationMessage{ + &ecShardMessage, + }, + } + glog.V(1).Infof("volume server %s:%d adds ec shard %d:%d", vs.store.Ip, vs.store.Port, ecShardMessage.Id, ecShardMessage.EcIndex) + if err = stream.Send(deltaBeat); err != nil { + glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err) + return "", err + } case volumeMessage := <-vs.store.DeletedVolumesChan: deltaBeat := &master_pb.Heartbeat{ DeletedVolumes: []*master_pb.VolumeShortInformationMessage{ @@ -120,12 +137,29 @@ func (vs *VolumeServer) doHeartbeat(ctx context.Context, masterNode, masterGrpcA glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err) return "", err } - case <-tickChan: + case ecShardMessage := <-vs.store.DeletedEcShardsChan: + deltaBeat := &master_pb.Heartbeat{ + DeletedEcShards: []*master_pb.VolumeEcShardInformationMessage{ + &ecShardMessage, + }, + } + glog.V(1).Infof("volume server %s:%d deletes ec shard %d:%d", vs.store.Ip, vs.store.Port, ecShardMessage.Id, ecShardMessage.EcIndex) + if err = stream.Send(deltaBeat); err != nil { + glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err) + return "", err + } + case <-volumeTickChan: glog.V(4).Infof("volume server %s:%d heartbeat", vs.store.Ip, vs.store.Port) if err = stream.Send(vs.store.CollectHeartbeat()); err != nil { glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err) return "", err } + case <-ecShardTickChan: + glog.V(4).Infof("volume server %s:%d ec heartbeat", vs.store.Ip, vs.store.Port) + if err = stream.Send(vs.store.CollectErasureCodingHeartbeat()); err != nil { + glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err) + return "", err + } case err = <-doneChan: return } diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index a4a3c519e..bd6d467d9 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -9,6 +9,7 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" "github.com/chrislusf/seaweedfs/weed/storage/needle" ) @@ -17,43 +18,52 @@ type DiskLocation struct { MaxVolumeCount int volumes map[needle.VolumeId]*Volume sync.RWMutex + + // erasure coding + ecShards map[needle.VolumeId]erasure_coding.EcVolumeShards + ecShardsLock sync.RWMutex } func NewDiskLocation(dir string, maxVolumeCount int) *DiskLocation { location := &DiskLocation{Directory: dir, MaxVolumeCount: maxVolumeCount} location.volumes = make(map[needle.VolumeId]*Volume) + location.ecShards = make(map[needle.VolumeId]erasure_coding.EcVolumeShards) return location } func (l *DiskLocation) volumeIdFromPath(dir os.FileInfo) (needle.VolumeId, string, error) { name := dir.Name() if !dir.IsDir() && strings.HasSuffix(name, ".dat") { - collection := "" base := name[:len(name)-len(".dat")] - i := strings.LastIndex(base, "_") - if i > 0 { - collection, base = base[0:i], base[i+1:] - } - vol, err := needle.NewVolumeId(base) - return vol, collection, err + collection, volumeId, err := parseCollectionVolumeId(base) + return volumeId, collection, err } return 0, "", fmt.Errorf("Path is not a volume: %s", name) } -func (l *DiskLocation) loadExistingVolume(dir os.FileInfo, needleMapKind NeedleMapType, mutex *sync.RWMutex) { - name := dir.Name() - if !dir.IsDir() && strings.HasSuffix(name, ".dat") { - vid, collection, err := l.volumeIdFromPath(dir) +func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeId, err error) { + i := strings.LastIndex(base, "_") + if i > 0 { + collection, base = base[0:i], base[i+1:] + } + vol, err := needle.NewVolumeId(base) + return collection, vol, err +} + +func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapType) { + name := fileInfo.Name() + if !fileInfo.IsDir() && strings.HasSuffix(name, ".dat") { + vid, collection, err := l.volumeIdFromPath(fileInfo) if err == nil { - mutex.RLock() + l.RLock() _, found := l.volumes[vid] - mutex.RUnlock() + l.RUnlock() if !found { if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0); e == nil { - mutex.Lock() + l.Lock() l.volumes[vid] = v - mutex.Unlock() + l.Unlock() size, _, _ := v.FileStat() glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s", l.Directory+"/"+name, v.ReplicaPlacement, v.Version(), size, v.Ttl.String()) @@ -80,13 +90,12 @@ func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, con }() var wg sync.WaitGroup - var mutex sync.RWMutex for workerNum := 0; workerNum < concurrency; workerNum++ { wg.Add(1) go func() { defer wg.Done() for dir := range task_queue { - l.loadExistingVolume(dir, needleMapKind, &mutex) + l.loadExistingVolume(dir, needleMapKind) } }() } @@ -95,12 +104,13 @@ func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, con } func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapType) { - l.Lock() - defer l.Unlock() l.concurrentLoadingVolumes(needleMapKind, 10) + glog.V(0).Infof("Store started on dir: %s with %d volumes max %d", l.Directory, len(l.volumes), l.MaxVolumeCount) + + l.loadAllEcShards() + glog.V(0).Infof("Store started on dir: %s with %d ec shards", l.Directory, len(l.ecShards)) - glog.V(0).Infoln("Store started on dir:", l.Directory, "with", len(l.volumes), "volumes", "max", l.MaxVolumeCount) } func (l *DiskLocation) DeleteCollectionFromDiskLocation(collection string) (e error) { @@ -132,12 +142,11 @@ func (l *DiskLocation) deleteVolumeById(vid needle.VolumeId) (e error) { } func (l *DiskLocation) LoadVolume(vid needle.VolumeId, needleMapKind NeedleMapType) bool { - if dirs, err := ioutil.ReadDir(l.Directory); err == nil { - for _, dir := range dirs { - volId, _, err := l.volumeIdFromPath(dir) + if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil { + for _, fileInfo := range fileInfos { + volId, _, err := l.volumeIdFromPath(fileInfo) if vid == volId && err == nil { - var mutex sync.RWMutex - l.loadExistingVolume(dir, needleMapKind, &mutex) + l.loadExistingVolume(fileInfo, needleMapKind) return true } } @@ -194,10 +203,16 @@ func (l *DiskLocation) VolumesLen() int { func (l *DiskLocation) Close() { l.Lock() - defer l.Unlock() - for _, v := range l.volumes { v.Close() } + l.Unlock() + + l.ecShardsLock.Lock() + for _, shards := range l.ecShards { + shards.Close() + } + l.ecShardsLock.Unlock() + return } diff --git a/weed/storage/disk_location_ec.go b/weed/storage/disk_location_ec.go new file mode 100644 index 000000000..3b2f1ec02 --- /dev/null +++ b/weed/storage/disk_location_ec.go @@ -0,0 +1,84 @@ +package storage + +import ( + "fmt" + "io/ioutil" + "path" + "regexp" + "sort" + "strconv" + + "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" + "github.com/chrislusf/seaweedfs/weed/storage/needle" +) + +var ( + re = regexp.MustCompile("\\.ec[0-9][0-9]") +) + +func (l *DiskLocation) loadEcShards(baseName string, shards []string, collection string, vid needle.VolumeId) (err error){ + + for _, shard := range shards{ + shardId, err := strconv.ParseInt(path.Ext(shard)[3:], 10, 64) + if err != nil { + return fmt.Errorf("failed to parse ec shard name %v: %v", shard, err) + } + ecVolumeShard, err := erasure_coding.NewEcVolumeShard(l.Directory, collection, vid, int(shardId)) + if err != nil { + return fmt.Errorf("failed to create ec shard %v: %v", shard, err) + } + l.ecShardsLock.Lock() + l.ecShards[vid] = append(l.ecShards[vid], ecVolumeShard) + l.ecShardsLock.Unlock() + } + + return nil +} + +func (l *DiskLocation) loadAllEcShards() (err error){ + + fileInfos, err := ioutil.ReadDir(l.Directory) + if err != nil { + return fmt.Errorf("load all ec shards in dir %s: %v", l.Directory, err) + } + + sort.Slice(fileInfos, func(i, j int) bool { + return fileInfos[i].Name() < fileInfos[j].Name() + }) + + var sameVolumeShards []string + var prevVolumeId needle.VolumeId + for _, fileInfo := range fileInfos{ + if fileInfo.IsDir(){ + continue + } + ext := path.Ext(fileInfo.Name()) + name := fileInfo.Name() + baseName := name[:len(name)-len(ext)] + + collection, volumeId, err := parseCollectionVolumeId(baseName) + if err != nil { + continue + } + + if re.MatchString(ext){ + if prevVolumeId == 0 || volumeId == prevVolumeId { + sameVolumeShards = append(sameVolumeShards, fileInfo.Name()) + }else{ + sameVolumeShards = []string{fileInfo.Name()} + } + prevVolumeId = volumeId + continue + } + + if ext == ".ecx" && volumeId == prevVolumeId{ + if err = l.loadEcShards(baseName, sameVolumeShards, collection, volumeId);err!=nil{ + return fmt.Errorf("loadEcShards collection:%v volumeId:%d : %v", collection, volumeId, err) + } + prevVolumeId = volumeId + continue + } + + } + return nil +} diff --git a/weed/storage/disk_location_ec_test.go b/weed/storage/disk_location_ec_test.go new file mode 100644 index 000000000..99cb242a4 --- /dev/null +++ b/weed/storage/disk_location_ec_test.go @@ -0,0 +1,17 @@ +package storage + +import ( + "testing" +) + +func TestLoadingEcShards(t *testing.T) { + dl := NewDiskLocation("./erasure_coding", 100) + err := dl.loadAllEcShards() + if err != nil { + t.Errorf("load all ec shards: %v", err) + } + + if len(dl.ecShards)!=1 { + t.Errorf("loading err") + } +} \ No newline at end of file diff --git a/weed/storage/erasure_coding/3.ec07 b/weed/storage/erasure_coding/3.ec07 new file mode 100644 index 000000000..e69de29bb diff --git a/weed/storage/erasure_coding/3.ecx b/weed/storage/erasure_coding/3.ecx new file mode 100644 index 000000000..e69de29bb diff --git a/weed/storage/erasure_coding/ec_encoder.go b/weed/storage/erasure_coding/ec_encoder.go index f4aa17e9d..54e757678 100644 --- a/weed/storage/erasure_coding/ec_encoder.go +++ b/weed/storage/erasure_coding/ec_encoder.go @@ -6,7 +6,7 @@ import ( "os" "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/storage" + "github.com/chrislusf/seaweedfs/weed/storage/idx" "github.com/chrislusf/seaweedfs/weed/storage/needle_map" "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/klauspost/reedsolomon" @@ -190,7 +190,7 @@ func readCompactMap(baseFileName string) (*needle_map.CompactMap, error) { defer indexFile.Close() cm := needle_map.NewCompactMap() - err = storage.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error { + err = idx.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error { if !offset.IsZero() && size != types.TombstoneFileSize { cm.Set(key, offset, size) } else { diff --git a/weed/storage/erasure_coding/ec_volume.go b/weed/storage/erasure_coding/ec_volume.go new file mode 100644 index 000000000..db0f911e4 --- /dev/null +++ b/weed/storage/erasure_coding/ec_volume.go @@ -0,0 +1,108 @@ +package erasure_coding + +import ( + "fmt" + "os" + "path" + "strconv" + + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "github.com/chrislusf/seaweedfs/weed/storage/needle" +) + +type EcVolumeShard struct { + VolumeId needle.VolumeId + ShardId uint8 + Collection string + dir string + ecdFile *os.File + ecxFile *os.File +} +type EcVolumeShards []*EcVolumeShard + +func NewEcVolumeShard(dirname string, collection string, id needle.VolumeId, shardId int) (v *EcVolumeShard, e error) { + + v = &EcVolumeShard{dir: dirname, Collection: collection, VolumeId: id, ShardId: uint8(shardId)} + + baseFileName := v.FileName() + if v.ecxFile, e = os.OpenFile(baseFileName+".ecx", os.O_RDONLY, 0644); e != nil { + return nil, fmt.Errorf("cannot read ec volume index %s.ecx: %v", baseFileName, e) + } + if v.ecdFile, e = os.OpenFile(baseFileName+ToExt(shardId), os.O_RDONLY, 0644); e != nil { + return nil, fmt.Errorf("cannot read ec volume shard %s.%s: %v", baseFileName, ToExt(shardId), e) + } + + return +} + +func (shards *EcVolumeShards) AddEcVolumeShard(ecVolumeShard *EcVolumeShard) bool { + for _, s := range *shards { + if s.ShardId == ecVolumeShard.ShardId { + return false + } + } + *shards = append(*shards, ecVolumeShard) + return true +} + +func (shards *EcVolumeShards) DeleteEcVolumeShard(ecVolumeShard *EcVolumeShard) bool { + foundPosition := -1 + for i, s := range *shards { + if s.ShardId == ecVolumeShard.ShardId { + foundPosition = i + } + } + if foundPosition < 0 { + return false + } + + *shards = append((*shards)[:foundPosition], (*shards)[foundPosition+1:]...) + return true +} + +func (shards *EcVolumeShards) Close() { + for _, s := range *shards { + s.Close() + } +} + +func (shards *EcVolumeShards) ToVolumeInformationMessage() (messages []*master_pb.VolumeEcShardInformationMessage) { + for _, s := range *shards { + m := &master_pb.VolumeEcShardInformationMessage{ + Id: uint32(s.VolumeId), + Collection: s.Collection, + EcIndex: uint32(s.ShardId), + } + messages = append(messages, m) + } + return +} + +func (v *EcVolumeShard) String() string { + return fmt.Sprintf("ec shard %v:%v, dir:%s, Collection:%s", v.VolumeId, v.ShardId, v.dir, v.Collection) +} + +func (v *EcVolumeShard) FileName() (fileName string) { + return EcShardFileName(v.Collection, v.dir, int(v.VolumeId)) +} + +func EcShardFileName(collection string, dir string, id int) (fileName string) { + idString := strconv.Itoa(id) + if collection == "" { + fileName = path.Join(dir, idString) + } else { + fileName = path.Join(dir, collection+"_"+idString) + } + return +} + +func (v *EcVolumeShard) Close() { + if v.ecdFile != nil { + _ = v.ecdFile.Close() + v.ecdFile = nil + } + if v.ecxFile != nil { + _ = v.ecxFile.Close() + v.ecxFile = nil + } +} diff --git a/weed/storage/idx/walk.go b/weed/storage/idx/walk.go new file mode 100644 index 000000000..c674a2aeb --- /dev/null +++ b/weed/storage/idx/walk.go @@ -0,0 +1,54 @@ +package idx + +import ( + "io" + "os" + + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/storage/types" + "github.com/chrislusf/seaweedfs/weed/util" +) + +// walks through the index file, calls fn function with each key, offset, size +// stops with the error returned by the fn function +func WalkIndexFile(r *os.File, fn func(key types.NeedleId, offset types.Offset, size uint32) error) error { + var readerOffset int64 + bytes := make([]byte, types.NeedleMapEntrySize*RowsToRead) + count, e := r.ReadAt(bytes, readerOffset) + glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e) + readerOffset += int64(count) + var ( + key types.NeedleId + offset types.Offset + size uint32 + i int + ) + + for count > 0 && e == nil || e == io.EOF { + for i = 0; i+types.NeedleMapEntrySize <= count; i += types.NeedleMapEntrySize { + key, offset, size = IdxFileEntry(bytes[i : i+types.NeedleMapEntrySize]) + if e = fn(key, offset, size); e != nil { + return e + } + } + if e == io.EOF { + return nil + } + count, e = r.ReadAt(bytes, readerOffset) + glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e) + readerOffset += int64(count) + } + return e +} + +func IdxFileEntry(bytes []byte) (key types.NeedleId, offset types.Offset, size uint32) { + key = types.BytesToNeedleId(bytes[:types.NeedleIdSize]) + offset = types.BytesToOffset(bytes[types.NeedleIdSize : types.NeedleIdSize+types.OffsetSize]) + size = util.BytesToUint32(bytes[types.NeedleIdSize+types.OffsetSize : types.NeedleIdSize+types.OffsetSize+types.SizeSize]) + return +} + +const ( + RowsToRead = 1024 +) + diff --git a/weed/storage/needle_map.go b/weed/storage/needle_map.go index f2c88093f..876839be1 100644 --- a/weed/storage/needle_map.go +++ b/weed/storage/needle_map.go @@ -8,7 +8,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage/needle_map" . "github.com/chrislusf/seaweedfs/weed/storage/types" - "github.com/chrislusf/seaweedfs/weed/util" ) type NeedleMapType int @@ -55,12 +54,6 @@ func (nm *baseNeedleMapper) IndexFileName() string { return nm.indexFile.Name() } -func IdxFileEntry(bytes []byte) (key NeedleId, offset Offset, size uint32) { - key = BytesToNeedleId(bytes[:NeedleIdSize]) - offset = BytesToOffset(bytes[NeedleIdSize : NeedleIdSize+OffsetSize]) - size = util.BytesToUint32(bytes[NeedleIdSize+OffsetSize : NeedleIdSize+OffsetSize+SizeSize]) - return -} func (nm *baseNeedleMapper) appendToIndexFile(key NeedleId, offset Offset, size uint32) error { bytes := needle_map.ToBytes(key, offset, size) diff --git a/weed/storage/needle_map_leveldb.go b/weed/storage/needle_map_leveldb.go index 602c90ddf..c2c6fcf19 100644 --- a/weed/storage/needle_map_leveldb.go +++ b/weed/storage/needle_map_leveldb.go @@ -2,10 +2,12 @@ package storage import ( "fmt" - "github.com/syndtr/goleveldb/leveldb/opt" "os" "path/filepath" + "github.com/chrislusf/seaweedfs/weed/storage/idx" + "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/storage/needle_map" . "github.com/chrislusf/seaweedfs/weed/storage/types" @@ -64,7 +66,7 @@ func generateLevelDbFile(dbFileName string, indexFile *os.File) error { return err } defer db.Close() - return WalkIndexFile(indexFile, func(key NeedleId, offset Offset, size uint32) error { + return idx.WalkIndexFile(indexFile, func(key NeedleId, offset Offset, size uint32) error { if !offset.IsZero() && size != TombstoneFileSize { levelDbWrite(db, key, offset, size) } else { diff --git a/weed/storage/needle_map_memory.go b/weed/storage/needle_map_memory.go index bda4a5c8f..ee639a7e6 100644 --- a/weed/storage/needle_map_memory.go +++ b/weed/storage/needle_map_memory.go @@ -1,10 +1,10 @@ package storage import ( - "io" "os" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/storage/idx" "github.com/chrislusf/seaweedfs/weed/storage/needle_map" . "github.com/chrislusf/seaweedfs/weed/storage/types" ) @@ -30,10 +30,6 @@ func NewBtreeNeedleMap(file *os.File) *NeedleMap { return nm } -const ( - RowsToRead = 1024 -) - func LoadCompactNeedleMap(file *os.File) (*NeedleMap, error) { nm := NewCompactNeedleMap(file) return doLoading(file, nm) @@ -45,7 +41,7 @@ func LoadBtreeNeedleMap(file *os.File) (*NeedleMap, error) { } func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) { - e := WalkIndexFile(file, func(key NeedleId, offset Offset, size uint32) error { + e := idx.WalkIndexFile(file, func(key NeedleId, offset Offset, size uint32) error { nm.MaybeSetMaxFileKey(key) if !offset.IsZero() && size != TombstoneFileSize { nm.FileCounter++ @@ -68,38 +64,6 @@ func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) { return nm, e } -// walks through the index file, calls fn function with each key, offset, size -// stops with the error returned by the fn function -func WalkIndexFile(r *os.File, fn func(key NeedleId, offset Offset, size uint32) error) error { - var readerOffset int64 - bytes := make([]byte, NeedleMapEntrySize*RowsToRead) - count, e := r.ReadAt(bytes, readerOffset) - glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e) - readerOffset += int64(count) - var ( - key NeedleId - offset Offset - size uint32 - i int - ) - - for count > 0 && e == nil || e == io.EOF { - for i = 0; i+NeedleMapEntrySize <= count; i += NeedleMapEntrySize { - key, offset, size = IdxFileEntry(bytes[i : i+NeedleMapEntrySize]) - if e = fn(key, offset, size); e != nil { - return e - } - } - if e == io.EOF { - return nil - } - count, e = r.ReadAt(bytes, readerOffset) - glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e) - readerOffset += int64(count) - } - return e -} - func (nm *NeedleMap) Put(key NeedleId, offset Offset, size uint32) error { _, oldSize := nm.m.Set(NeedleId(key), offset, size) nm.logPut(key, oldSize, size) diff --git a/weed/storage/needle_map_metric.go b/weed/storage/needle_map_metric.go index 55b877db6..f7372105f 100644 --- a/weed/storage/needle_map_metric.go +++ b/weed/storage/needle_map_metric.go @@ -5,6 +5,7 @@ import ( "os" "sync/atomic" + "github.com/chrislusf/seaweedfs/weed/storage/idx" . "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/willf/bloom" ) @@ -119,7 +120,7 @@ func reverseWalkIndexFile(r *os.File, initFn func(entryCount int64), fn func(key return e } for i := int(nextBatchSize) - 1; i >= 0; i-- { - key, offset, size := IdxFileEntry(bytes[i*NeedleMapEntrySize : i*NeedleMapEntrySize+NeedleMapEntrySize]) + key, offset, size := idx.IdxFileEntry(bytes[i*NeedleMapEntrySize : i*NeedleMapEntrySize+NeedleMapEntrySize]) if e = fn(key, offset, size); e != nil { return e } diff --git a/weed/storage/store.go b/weed/storage/store.go index bde6d872a..d9c9b0e35 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -30,6 +30,8 @@ type Store struct { NeedleMapType NeedleMapType NewVolumesChan chan master_pb.VolumeShortInformationMessage DeletedVolumesChan chan master_pb.VolumeShortInformationMessage + NewEcShardsChan chan master_pb.VolumeEcShardInformationMessage + DeletedEcShardsChan chan master_pb.VolumeEcShardInformationMessage } func (s *Store) String() (str string) { @@ -47,6 +49,10 @@ func NewStore(port int, ip, publicUrl string, dirnames []string, maxVolumeCounts } s.NewVolumesChan = make(chan master_pb.VolumeShortInformationMessage, 3) s.DeletedVolumesChan = make(chan master_pb.VolumeShortInformationMessage, 3) + + s.NewEcShardsChan = make(chan master_pb.VolumeEcShardInformationMessage, 3) + s.DeletedEcShardsChan = make(chan master_pb.VolumeEcShardInformationMessage, 3) + return } func (s *Store) AddVolume(volumeId needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string, preallocate int64) error { @@ -186,6 +192,7 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat { } } + func (s *Store) Close() { for _, location := range s.Locations { location.Close() diff --git a/weed/storage/store_ec.go b/weed/storage/store_ec.go new file mode 100644 index 000000000..43d851faa --- /dev/null +++ b/weed/storage/store_ec.go @@ -0,0 +1,21 @@ +package storage + +import ( + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" +) + +func (s *Store) CollectErasureCodingHeartbeat() *master_pb.Heartbeat { + var ecShardMessages []*master_pb.VolumeEcShardInformationMessage + for _, location := range s.Locations { + location.ecShardsLock.RLock() + for _, ecShards := range location.ecShards { + ecShardMessages = append(ecShardMessages, ecShards.ToVolumeInformationMessage()...) + } + location.ecShardsLock.RUnlock() + } + + return &master_pb.Heartbeat{ + EcShards: ecShardMessages, + } + +} diff --git a/weed/storage/volume_backup.go b/weed/storage/volume_backup.go index b7ca3c023..f56c40019 100644 --- a/weed/storage/volume_backup.go +++ b/weed/storage/volume_backup.go @@ -8,6 +8,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" + "github.com/chrislusf/seaweedfs/weed/storage/idx" "github.com/chrislusf/seaweedfs/weed/storage/needle" . "github.com/chrislusf/seaweedfs/weed/storage/types" "google.golang.org/grpc" @@ -142,7 +143,7 @@ func (v *Volume) locateLastAppendEntry() (Offset, error) { if n != NeedleMapEntrySize { return Offset{}, fmt.Errorf("file %s read error: %v", indexFile.Name(), e) } - _, offset, _ := IdxFileEntry(bytes) + _, offset, _ := idx.IdxFileEntry(bytes) return offset, nil } @@ -230,7 +231,7 @@ func (v *Volume) readAppendAtNsForIndexEntry(indexFile *os.File, bytes []byte, m if _, readErr := indexFile.ReadAt(bytes, m*NeedleMapEntrySize); readErr != nil && readErr != io.EOF { return Offset{}, readErr } - _, offset, _ := IdxFileEntry(bytes) + _, offset, _ := idx.IdxFileEntry(bytes) return offset, nil } diff --git a/weed/storage/volume_checking.go b/weed/storage/volume_checking.go index 7bb998fa6..8f930546f 100644 --- a/weed/storage/volume_checking.go +++ b/weed/storage/volume_checking.go @@ -4,6 +4,7 @@ import ( "fmt" "os" + "github.com/chrislusf/seaweedfs/weed/storage/idx" "github.com/chrislusf/seaweedfs/weed/storage/needle" . "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/chrislusf/seaweedfs/weed/util" @@ -21,7 +22,7 @@ func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (lastAppendAtNs uin if lastIdxEntry, e = readIndexEntryAtOffset(indexFile, indexSize-NeedleMapEntrySize); e != nil { return 0, fmt.Errorf("readLastIndexEntry %s failed: %v", indexFile.Name(), e) } - key, offset, size := IdxFileEntry(lastIdxEntry) + key, offset, size := idx.IdxFileEntry(lastIdxEntry) if offset.IsZero() { return 0, nil } diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go index bbe17071e..f89f2cf0a 100644 --- a/weed/storage/volume_vacuum.go +++ b/weed/storage/volume_vacuum.go @@ -6,6 +6,7 @@ import ( "time" "github.com/chrislusf/seaweedfs/weed/glog" + idx2 "github.com/chrislusf/seaweedfs/weed/storage/idx" "github.com/chrislusf/seaweedfs/weed/storage/needle" . "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/chrislusf/seaweedfs/weed/util" @@ -143,7 +144,7 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI if IdxEntry, err = readIndexEntryAtOffset(oldIdxFile, idxOffset); err != nil { return fmt.Errorf("readIndexEntry %s at offset %d failed: %v", oldIdxFileName, idxOffset, err) } - key, offset, size := IdxFileEntry(IdxEntry) + key, offset, size := idx2.IdxFileEntry(IdxEntry) glog.V(4).Infof("key %d offset %d size %d", key, offset, size) if _, found := incrementedHasUpdatedIndexEntry[key]; !found { incrementedHasUpdatedIndexEntry[key] = keyField{ @@ -329,7 +330,7 @@ func (v *Volume) copyDataBasedOnIndexFile(dstName, idxName string) (err error) { dst.Write(v.SuperBlock.Bytes()) newOffset := int64(v.SuperBlock.BlockSize()) - WalkIndexFile(oldIndexFile, func(key NeedleId, offset Offset, size uint32) error { + idx2.WalkIndexFile(oldIndexFile, func(key NeedleId, offset Offset, size uint32) error { if offset.IsZero() || size == TombstoneFileSize { return nil }