From 178eb8cce10e23631df46148a0a7d59d68b301bd Mon Sep 17 00:00:00 2001 From: yah01 Date: Mon, 28 Mar 2022 16:43:27 +0800 Subject: [PATCH] Modify proto to support replicas (#16218) Signed-off-by: yah01 --- internal/proto/query_coord.proto | 2 +- internal/proto/querypb/query_coord.pb.go | 328 +++++++++++------------ internal/querycoord/cluster.go | 20 +- internal/querycoord/meta.go | 114 +++++--- internal/querycoord/replica.go | 91 +++++++ 5 files changed, 340 insertions(+), 215 deletions(-) create mode 100644 internal/querycoord/replica.go diff --git a/internal/proto/query_coord.proto b/internal/proto/query_coord.proto index d8a41464ba..e1c08a8931 100644 --- a/internal/proto/query_coord.proto +++ b/internal/proto/query_coord.proto @@ -349,10 +349,10 @@ message ReplicaInfo { // ReplicaGroup int64 collection_id = 2; repeated int64 partition_ids = 3; // empty indicates to load collection repeated ShardReplica shard_replicas = 4; + repeated int64 node_ids = 5; // include leaders } message ShardReplica { int64 leader = 1; string dm_channel_name = 2; - repeated int64 node_ids = 3; // include leader } \ No newline at end of file diff --git a/internal/proto/querypb/query_coord.pb.go b/internal/proto/querypb/query_coord.pb.go index e2d60d320a..dd5a76c488 100644 --- a/internal/proto/querypb/query_coord.pb.go +++ b/internal/proto/querypb/query_coord.pb.go @@ -2477,6 +2477,7 @@ type ReplicaInfo struct { CollectionId int64 `protobuf:"varint,2,opt,name=collection_id,json=collectionId,proto3" json:"collection_id,omitempty"` PartitionIds []int64 `protobuf:"varint,3,rep,packed,name=partition_ids,json=partitionIds,proto3" json:"partition_ids,omitempty"` ShardReplicas []*ShardReplica `protobuf:"bytes,4,rep,name=shard_replicas,json=shardReplicas,proto3" json:"shard_replicas,omitempty"` + NodeIds []int64 `protobuf:"varint,5,rep,packed,name=node_ids,json=nodeIds,proto3" json:"node_ids,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2535,10 +2536,16 @@ func (m *ReplicaInfo) GetShardReplicas() []*ShardReplica { return nil } +func (m *ReplicaInfo) GetNodeIds() []int64 { + if m != nil { + return m.NodeIds + } + return nil +} + type ShardReplica struct { Leader int64 `protobuf:"varint,1,opt,name=leader,proto3" json:"leader,omitempty"` DmChannelName string `protobuf:"bytes,2,opt,name=dm_channel_name,json=dmChannelName,proto3" json:"dm_channel_name,omitempty"` - NodeIds []int64 `protobuf:"varint,3,rep,packed,name=node_ids,json=nodeIds,proto3" json:"node_ids,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2583,13 +2590,6 @@ func (m *ShardReplica) GetDmChannelName() string { return "" } -func (m *ShardReplica) GetNodeIds() []int64 { - if m != nil { - return m.NodeIds - } - return nil -} - func init() { proto.RegisterEnum("milvus.proto.query.PartitionState", PartitionState_name, PartitionState_value) proto.RegisterEnum("milvus.proto.query.TriggerCondition", TriggerCondition_name, TriggerCondition_value) @@ -2640,162 +2640,162 @@ var fileDescriptor_aab7cc9a69ed26e8 = []byte{ 0xf5, 0xee, 0xf9, 0xb0, 0x3d, 0x6f, 0x3e, 0x5d, 0x4e, 0xfc, 0x9b, 0xcc, 0x6f, 0x37, 0xf1, 0x76, 0xd6, 0x49, 0xc8, 0xb2, 0x4e, 0x70, 0x40, 0x62, 0x05, 0x1c, 0xd6, 0x36, 0xf1, 0x9a, 0x6c, 0xbc, 0xa6, 0xed, 0x04, 0x88, 0x22, 0x35, 0x3d, 0xd3, 0x35, 0xe3, 0x56, 0xfa, 0x63, 0xd2, 0xd5, 0xb3, - 0x89, 0xc3, 0x95, 0x03, 0x20, 0x21, 0xc4, 0x15, 0x21, 0x4e, 0x20, 0xc8, 0x61, 0x2f, 0x9c, 0x91, - 0xe0, 0xc2, 0xbf, 0x81, 0x38, 0x20, 0xc1, 0x3f, 0xc0, 0x11, 0x81, 0xea, 0xa3, 0x7b, 0xfa, 0xa3, - 0xda, 0xee, 0xb1, 0x95, 0x4d, 0x84, 0xb8, 0x4d, 0xbd, 0x7e, 0xf5, 0xde, 0xab, 0xf7, 0x5d, 0xf5, - 0x06, 0x96, 0x9e, 0x4e, 0xb0, 0x7f, 0xac, 0x0f, 0x3c, 0xcf, 0x37, 0xd7, 0xc7, 0xbe, 0x17, 0x78, - 0x08, 0x39, 0x96, 0xfd, 0xe9, 0x84, 0xf0, 0xd5, 0x3a, 0xfb, 0xde, 0x6b, 0x0c, 0x3c, 0xc7, 0xf1, - 0x5c, 0x0e, 0xeb, 0x35, 0xe2, 0x18, 0xbd, 0x96, 0xe5, 0x06, 0xd8, 0x77, 0x0d, 0x3b, 0xfc, 0x4a, - 0x06, 0x47, 0xd8, 0x31, 0xc4, 0xaa, 0x63, 0x1a, 0x81, 0x11, 0xa7, 0xaf, 0xfe, 0x50, 0x81, 0x95, - 0x83, 0x23, 0xef, 0xd9, 0x96, 0x67, 0xdb, 0x78, 0x10, 0x58, 0x9e, 0x4b, 0x34, 0xfc, 0x74, 0x82, - 0x49, 0x80, 0x6e, 0x43, 0xa5, 0x6f, 0x10, 0xdc, 0x55, 0x56, 0x95, 0x1b, 0xf5, 0x8d, 0xb7, 0xd6, - 0x13, 0x92, 0x08, 0x11, 0xee, 0x93, 0xd1, 0xa6, 0x41, 0xb0, 0xc6, 0x30, 0x11, 0x82, 0x8a, 0xd9, - 0xdf, 0xdd, 0xee, 0x96, 0x56, 0x95, 0x1b, 0x65, 0x8d, 0xfd, 0x46, 0xef, 0x42, 0x73, 0x10, 0xd1, - 0xde, 0xdd, 0x26, 0xdd, 0xf2, 0x6a, 0xf9, 0x46, 0x59, 0x4b, 0x02, 0xd5, 0xdf, 0x2a, 0xf0, 0x7f, - 0x19, 0x31, 0xc8, 0xd8, 0x73, 0x09, 0x46, 0x77, 0x60, 0x9e, 0x04, 0x46, 0x30, 0x21, 0x42, 0x92, - 0xff, 0x97, 0x4a, 0x72, 0xc0, 0x50, 0x34, 0x81, 0x9a, 0x65, 0x5b, 0x92, 0xb0, 0x45, 0x5f, 0x82, - 0x0b, 0x96, 0x7b, 0x1f, 0x3b, 0x9e, 0x7f, 0xac, 0x8f, 0xb1, 0x3f, 0xc0, 0x6e, 0x60, 0x8c, 0x70, - 0x28, 0xe3, 0x72, 0xf8, 0x6d, 0x7f, 0xfa, 0x49, 0xfd, 0x8d, 0x02, 0x17, 0xa9, 0xa4, 0xfb, 0x86, - 0x1f, 0x58, 0xaf, 0x40, 0x5f, 0x2a, 0x34, 0xe2, 0x32, 0x76, 0xcb, 0xec, 0x5b, 0x02, 0x46, 0x71, - 0xc6, 0x21, 0x7b, 0x7a, 0xb6, 0x0a, 0x13, 0x37, 0x01, 0x53, 0x7f, 0x2d, 0x0c, 0x1b, 0x97, 0xf3, - 0x3c, 0x0a, 0x4d, 0xf3, 0x2c, 0x65, 0x79, 0x9e, 0x45, 0x9d, 0x7f, 0x57, 0xe0, 0xe2, 0xc7, 0x9e, - 0x61, 0x4e, 0x0d, 0xff, 0xf9, 0xab, 0xf3, 0x1b, 0x30, 0xcf, 0xa3, 0xa4, 0x5b, 0x61, 0xbc, 0xd6, - 0x92, 0xbc, 0x44, 0x04, 0x4d, 0x25, 0x3c, 0x60, 0x00, 0x4d, 0x6c, 0x42, 0x6b, 0xd0, 0xf2, 0xf1, - 0xd8, 0xb6, 0x06, 0x86, 0xee, 0x4e, 0x9c, 0x3e, 0xf6, 0xbb, 0xd5, 0x55, 0xe5, 0x46, 0x55, 0x6b, - 0x0a, 0xe8, 0x1e, 0x03, 0xaa, 0xbf, 0x54, 0xa0, 0xab, 0x61, 0x1b, 0x1b, 0x04, 0xbf, 0xce, 0xc3, - 0xae, 0xc0, 0xbc, 0xeb, 0x99, 0x78, 0x77, 0x9b, 0x1d, 0xb6, 0xac, 0x89, 0x95, 0xfa, 0x37, 0x61, - 0x88, 0x37, 0xdc, 0xaf, 0x63, 0xc6, 0xaa, 0x9e, 0xc1, 0x58, 0xea, 0x9f, 0xa6, 0x56, 0x78, 0xd3, - 0x4f, 0x3a, 0xb5, 0x54, 0x35, 0x61, 0xa9, 0xef, 0xc1, 0xa5, 0x2d, 0x1f, 0x1b, 0x01, 0xfe, 0x36, - 0xad, 0x06, 0x5b, 0x47, 0x86, 0xeb, 0x62, 0x3b, 0x3c, 0x42, 0x9a, 0xb9, 0x22, 0x61, 0xde, 0x85, - 0x85, 0xb1, 0xef, 0x3d, 0x3f, 0x8e, 0xe4, 0x0e, 0x97, 0xea, 0xef, 0x14, 0xe8, 0xc9, 0x68, 0x9f, - 0x27, 0x71, 0x5c, 0x85, 0xa6, 0x28, 0x6b, 0x9c, 0x1a, 0xe3, 0x59, 0xd3, 0x1a, 0x4f, 0x63, 0x1c, - 0xd0, 0x6d, 0xb8, 0xc0, 0x91, 0x7c, 0x4c, 0x26, 0x76, 0x10, 0xe1, 0x96, 0x19, 0x2e, 0x62, 0xdf, - 0x34, 0xf6, 0x49, 0xec, 0x50, 0x5f, 0x2a, 0x70, 0x69, 0x07, 0x07, 0x91, 0x11, 0x29, 0x57, 0xfc, - 0x86, 0xe6, 0xe2, 0xcf, 0x14, 0xe8, 0xc9, 0x64, 0x3d, 0x8f, 0x5a, 0x1f, 0xc1, 0x4a, 0xc4, 0x43, - 0x37, 0x31, 0x19, 0xf8, 0xd6, 0x98, 0x39, 0x33, 0xcb, 0xcc, 0xf5, 0x8d, 0xab, 0xeb, 0xd9, 0xce, - 0x61, 0x3d, 0x2d, 0xc1, 0xc5, 0x88, 0xc4, 0x76, 0x8c, 0x82, 0xfa, 0x53, 0x05, 0x2e, 0xee, 0xe0, - 0xe0, 0x00, 0x8f, 0x1c, 0xec, 0x06, 0xbb, 0xee, 0xd0, 0x3b, 0xbb, 0x5e, 0x2f, 0x03, 0x10, 0x41, - 0x27, 0xaa, 0x1a, 0x31, 0x48, 0x11, 0x1d, 0xb3, 0x26, 0x25, 0x2d, 0xcf, 0x79, 0x74, 0xf7, 0x15, - 0xa8, 0x5a, 0xee, 0xd0, 0x0b, 0x55, 0x75, 0x45, 0xa6, 0xaa, 0x38, 0x33, 0x8e, 0xad, 0xfe, 0xab, - 0x04, 0x2b, 0x1f, 0x9a, 0xa6, 0x2c, 0xec, 0x66, 0xd7, 0xcb, 0x34, 0xba, 0x4b, 0xf1, 0xe8, 0x2e, - 0xe4, 0x73, 0x99, 0x90, 0xaa, 0xcc, 0x10, 0x52, 0xd5, 0xbc, 0x90, 0x42, 0x3b, 0xd0, 0x24, 0x18, - 0x3f, 0xd1, 0xc7, 0x1e, 0x61, 0x3e, 0xd1, 0x9d, 0x67, 0xa7, 0x51, 0x93, 0xa7, 0x89, 0x1a, 0xcc, - 0xfb, 0x64, 0xb4, 0x2f, 0x30, 0xb5, 0x06, 0xdd, 0x18, 0xae, 0xd0, 0x03, 0x58, 0x19, 0xd9, 0x5e, - 0xdf, 0xb0, 0x75, 0x82, 0x0d, 0x1b, 0x9b, 0xba, 0xb0, 0x37, 0xe9, 0x2e, 0x14, 0x53, 0xf8, 0x05, - 0xbe, 0xfd, 0x80, 0xed, 0x16, 0x1f, 0x88, 0xfa, 0x57, 0x05, 0x2e, 0x69, 0xd8, 0xf1, 0x3e, 0xc5, - 0xff, 0xad, 0x26, 0x50, 0x7f, 0xae, 0x40, 0x83, 0x56, 0xe1, 0xfb, 0x38, 0x30, 0xa8, 0x26, 0xd0, - 0x07, 0x50, 0xb3, 0x3d, 0xc3, 0xd4, 0x83, 0xe3, 0x31, 0x3f, 0x5a, 0x2b, 0x7d, 0x34, 0xae, 0x3d, - 0xba, 0xe9, 0xf0, 0x78, 0x8c, 0xb5, 0x45, 0x5b, 0xfc, 0xca, 0x1c, 0xa3, 0x54, 0x20, 0x7b, 0x95, - 0x65, 0x9d, 0x64, 0x19, 0x56, 0xbe, 0x63, 0x04, 0x83, 0xa3, 0x6d, 0x47, 0x88, 0x49, 0x5e, 0x8f, - 0xce, 0x8b, 0x14, 0xcd, 0x28, 0xb4, 0xab, 0x32, 0x4f, 0xa3, 0xd7, 0x9f, 0xf5, 0x87, 0xc2, 0x0c, - 0xb1, 0xd0, 0x8e, 0x75, 0x15, 0xf3, 0x67, 0x69, 0x01, 0xb7, 0xa0, 0x89, 0x9f, 0x0f, 0xec, 0x89, - 0x89, 0x75, 0xce, 0x9d, 0xfb, 0xf9, 0x65, 0x09, 0xf7, 0xb8, 0x9b, 0x37, 0xc4, 0xa6, 0x5d, 0x21, - 0x03, 0x37, 0xb5, 0x83, 0x03, 0xa3, 0xbb, 0xc8, 0xc4, 0x58, 0xcd, 0x33, 0x75, 0xe8, 0x1f, 0xdc, - 0xdc, 0x74, 0xa5, 0xfe, 0x5b, 0x81, 0x4b, 0xdc, 0x4c, 0xd8, 0x0e, 0x8c, 0xd7, 0x6b, 0xa9, 0xc8, - 0x0a, 0x95, 0x19, 0xad, 0x10, 0xd3, 0x40, 0x6d, 0x66, 0x0d, 0xfc, 0xa2, 0x02, 0x6d, 0xa1, 0x5e, - 0x8a, 0xc1, 0xe2, 0xe7, 0x2d, 0xa8, 0x45, 0xc5, 0x46, 0x34, 0x43, 0x53, 0x00, 0x5a, 0x85, 0x7a, - 0xcc, 0x7b, 0xc4, 0x41, 0xe3, 0xa0, 0x42, 0xa7, 0x0d, 0x5b, 0x87, 0x4a, 0xac, 0x75, 0x78, 0x1b, - 0x60, 0x68, 0x4f, 0xc8, 0x91, 0x1e, 0x58, 0x0e, 0x16, 0x0d, 0x5c, 0x8d, 0x41, 0x0e, 0x2d, 0x07, - 0xa3, 0x0f, 0xa1, 0xd1, 0xb7, 0x5c, 0xdb, 0x1b, 0xe9, 0x63, 0x23, 0x38, 0x22, 0xdd, 0xf9, 0x5c, - 0x7f, 0xb9, 0x6b, 0x61, 0xdb, 0xdc, 0x64, 0xb8, 0x5a, 0x9d, 0xef, 0xd9, 0xa7, 0x5b, 0xd0, 0x65, - 0xa8, 0xbb, 0x13, 0x47, 0xf7, 0x86, 0xba, 0xef, 0x3d, 0xa3, 0x1e, 0xc7, 0x58, 0xb8, 0x13, 0xe7, - 0x93, 0xa1, 0xe6, 0x3d, 0x23, 0xe8, 0xeb, 0x50, 0xa3, 0xe5, 0x8e, 0xd8, 0xde, 0x88, 0x74, 0x17, - 0x0b, 0xd1, 0x9f, 0x6e, 0xa0, 0xbb, 0x4d, 0xea, 0x47, 0x6c, 0x77, 0xad, 0xd8, 0xee, 0x68, 0x03, - 0xba, 0x06, 0xad, 0x81, 0xe7, 0x8c, 0x0d, 0xa6, 0xa1, 0xbb, 0xbe, 0xe7, 0x74, 0x81, 0xc5, 0x6a, - 0x0a, 0x8a, 0xee, 0x42, 0xdd, 0x72, 0x4d, 0xfc, 0x5c, 0x44, 0x4d, 0x9d, 0xf1, 0x59, 0x93, 0x99, - 0xfc, 0x21, 0x1e, 0x30, 0x5e, 0xbb, 0x14, 0x9d, 0xd9, 0x1d, 0xac, 0xf0, 0x27, 0x41, 0xef, 0x40, - 0x43, 0x18, 0x55, 0x27, 0xd6, 0x0b, 0xdc, 0x6d, 0x70, 0x43, 0x0a, 0xd8, 0x81, 0xf5, 0x02, 0xab, - 0xbf, 0x2f, 0xc1, 0x52, 0x86, 0x08, 0x6d, 0x85, 0x87, 0x0c, 0x12, 0x3a, 0x47, 0xb8, 0xa4, 0x24, - 0xb1, 0x6b, 0xf4, 0x6d, 0x1a, 0xd1, 0x26, 0x7e, 0xce, 0x7c, 0x63, 0x51, 0xab, 0x73, 0x18, 0x23, - 0x40, 0x6d, 0xcc, 0xa5, 0x77, 0x0d, 0x07, 0x8b, 0x56, 0xb5, 0xc6, 0x20, 0x7b, 0x86, 0x83, 0x29, - 0x6d, 0x2e, 0x62, 0xe8, 0x19, 0xe1, 0x92, 0x7e, 0xe9, 0x4f, 0x2c, 0xc6, 0x95, 0x7b, 0x46, 0xb8, - 0x44, 0xdb, 0xd0, 0xe0, 0x24, 0xc7, 0x86, 0x6f, 0x38, 0xa1, 0x5f, 0xbc, 0x23, 0x0d, 0xd7, 0x7b, - 0xf8, 0xf8, 0xa1, 0x61, 0x4f, 0xf0, 0xbe, 0x61, 0xf9, 0x1a, 0xd7, 0xe3, 0x3e, 0xdb, 0x85, 0x6e, - 0x40, 0x87, 0x53, 0x19, 0x5a, 0x36, 0x16, 0x1e, 0x46, 0x33, 0x52, 0x4d, 0x6b, 0x31, 0xf8, 0x5d, - 0xcb, 0xc6, 0xdc, 0x89, 0xa2, 0x23, 0x30, 0xb5, 0x2d, 0x72, 0x1f, 0x62, 0x10, 0xa6, 0xb4, 0x7f, - 0x94, 0x60, 0x99, 0x86, 0x52, 0x58, 0x82, 0xcf, 0x9e, 0x4d, 0xde, 0x06, 0x30, 0x49, 0xa0, 0x27, - 0x32, 0x4a, 0xcd, 0x24, 0xc1, 0x1e, 0x4f, 0x2a, 0x1f, 0x84, 0x09, 0xa3, 0x9c, 0xdf, 0xbc, 0xa6, - 0x42, 0x3b, 0x9b, 0xba, 0xcf, 0x74, 0x7b, 0xbf, 0x0a, 0x4d, 0xe2, 0x4d, 0xfc, 0x01, 0xd6, 0x13, - 0x97, 0xad, 0x06, 0x07, 0xee, 0xc9, 0x73, 0xde, 0xbc, 0xf4, 0x15, 0x21, 0x96, 0xbc, 0x16, 0x66, - 0x4e, 0x5e, 0x7f, 0x51, 0x60, 0x45, 0x5c, 0x4c, 0xcf, 0xaf, 0xed, 0xbc, 0xdc, 0x1d, 0x66, 0xaa, - 0xf2, 0x09, 0x97, 0x9c, 0x4a, 0x81, 0xca, 0x5b, 0x95, 0x54, 0xde, 0x64, 0xa3, 0x3f, 0x9f, 0x6e, - 0xf4, 0xd5, 0x9f, 0x29, 0xb0, 0xf2, 0x91, 0xe1, 0x9a, 0xde, 0x70, 0x78, 0xfe, 0x03, 0x6e, 0x45, - 0x01, 0xbf, 0x3b, 0x4b, 0x23, 0x9f, 0xd8, 0xa4, 0xfe, 0xa8, 0x04, 0x88, 0x5a, 0x63, 0xd3, 0xb0, - 0x0d, 0x77, 0x80, 0xcf, 0x2e, 0xcd, 0x1a, 0xb4, 0x12, 0x3e, 0x14, 0xbd, 0x36, 0xc6, 0x9d, 0x88, - 0xa0, 0x7b, 0xd0, 0xea, 0x73, 0x56, 0xba, 0x8f, 0x0d, 0xe2, 0xb9, 0xcc, 0x0e, 0xad, 0x8d, 0x77, - 0x65, 0x62, 0x1f, 0xfa, 0xd6, 0x68, 0x84, 0xfd, 0x2d, 0xcf, 0x35, 0x79, 0x8b, 0xdd, 0xec, 0x87, - 0x62, 0xd2, 0xad, 0xe8, 0x0a, 0xd4, 0xa7, 0x01, 0x15, 0xf6, 0x42, 0x10, 0x45, 0x14, 0x41, 0xef, - 0xc1, 0x52, 0xb2, 0xfb, 0x9e, 0x1a, 0xae, 0x43, 0xe2, 0x8d, 0x35, 0x35, 0xce, 0x0f, 0x00, 0x45, - 0xdd, 0x1d, 0x6b, 0x22, 0x58, 0x76, 0x2c, 0xf2, 0x98, 0xf0, 0x16, 0xd4, 0xcc, 0x70, 0xa7, 0xb8, - 0xda, 0x4f, 0x01, 0x34, 0xba, 0xb8, 0x84, 0x3a, 0x75, 0x74, 0x6c, 0x86, 0xf5, 0x93, 0x03, 0x3f, - 0x66, 0x30, 0xf5, 0xb3, 0x12, 0x74, 0xe2, 0x1d, 0x7d, 0x61, 0xde, 0xaf, 0xe6, 0x69, 0xe1, 0x84, - 0xeb, 0x4b, 0xe5, 0x1c, 0xd7, 0x97, 0xec, 0xf5, 0xaa, 0x7a, 0xb6, 0xeb, 0x95, 0xfa, 0x2b, 0x05, - 0xda, 0xa9, 0x9b, 0x7c, 0xba, 0x93, 0x51, 0xb2, 0x9d, 0xcc, 0x57, 0xa1, 0x4a, 0xcb, 0x3b, 0x66, - 0x4a, 0x6a, 0xa5, 0xd9, 0xca, 0xde, 0x07, 0x34, 0xbe, 0x01, 0xdd, 0x82, 0x65, 0xc9, 0xb3, 0xae, - 0x30, 0x25, 0xca, 0xbe, 0xea, 0xaa, 0x7f, 0xa8, 0x40, 0x3d, 0xa6, 0x8f, 0x53, 0x9a, 0xb0, 0x22, - 0xf7, 0x94, 0xd4, 0xf1, 0xca, 0xd9, 0xe3, 0xe5, 0xbc, 0x6b, 0xa2, 0x4b, 0xb0, 0xe8, 0x60, 0x87, - 0xd7, 0x37, 0x51, 0x6c, 0x1d, 0xec, 0xd0, 0xea, 0x46, 0x3f, 0xd1, 0x0e, 0x8a, 0xb5, 0x4f, 0x3c, - 0xa3, 0x2f, 0xb8, 0x13, 0x87, 0x35, 0x4f, 0xc9, 0xd2, 0xbe, 0x70, 0x42, 0x69, 0x5f, 0x4c, 0x96, - 0xf6, 0x44, 0x38, 0xd4, 0xd2, 0xe1, 0x50, 0xb4, 0x2f, 0xba, 0x0d, 0xcb, 0x03, 0xf6, 0x0c, 0x67, - 0x6e, 0x1e, 0x6f, 0x45, 0x9f, 0xba, 0x75, 0xd6, 0x83, 0xc8, 0x3e, 0xa1, 0xbb, 0xd4, 0xb9, 0x44, - 0x07, 0xc4, 0xac, 0xdc, 0x60, 0x56, 0x96, 0x77, 0x0e, 0xc2, 0x36, 0xdc, 0xc8, 0x61, 0x4e, 0x64, - 0xab, 0x74, 0x47, 0xd6, 0x3c, 0x6b, 0x47, 0x76, 0x05, 0xea, 0xe1, 0xa3, 0xb8, 0x65, 0x92, 0x6e, - 0x8b, 0xa7, 0x27, 0x01, 0xda, 0x35, 0x09, 0x53, 0xbe, 0x47, 0xef, 0x4b, 0x26, 0xe9, 0xb6, 0xd9, - 0xd7, 0x05, 0x66, 0x31, 0x93, 0xa8, 0x2f, 0xcb, 0xd0, 0x9a, 0xd6, 0xeb, 0xc2, 0xd9, 0xa0, 0xc8, - 0x84, 0x62, 0x0f, 0x3a, 0xd3, 0x57, 0x33, 0xa6, 0xa8, 0x13, 0x5b, 0x8e, 0xf4, 0x7b, 0x59, 0x7b, - 0x9c, 0x0a, 0xbb, 0xc4, 0xf5, 0xbc, 0x32, 0xd3, 0xf5, 0xfc, 0x7c, 0x0f, 0xd9, 0xe8, 0x0e, 0x5c, - 0xf4, 0x79, 0xbb, 0x60, 0xea, 0x89, 0x63, 0xf3, 0xca, 0x7b, 0x21, 0xfc, 0xb8, 0x1f, 0x3f, 0x7e, - 0x4e, 0x24, 0x2f, 0xe4, 0x45, 0x72, 0xda, 0x8c, 0x8b, 0x69, 0x33, 0xaa, 0x0f, 0x60, 0xf9, 0x81, - 0x4b, 0x26, 0x7d, 0x32, 0xf0, 0xad, 0x3e, 0x0e, 0x6f, 0x9d, 0x85, 0xec, 0xd5, 0x83, 0x45, 0x91, - 0x8b, 0xb9, 0xad, 0x6a, 0x5a, 0xb4, 0x56, 0x7f, 0xa2, 0xc0, 0x4a, 0x96, 0x2e, 0x73, 0x85, 0x69, - 0xa0, 0x2b, 0x89, 0x40, 0xff, 0x2e, 0x2c, 0x4f, 0xc9, 0xeb, 0x09, 0xca, 0xf5, 0x8d, 0xeb, 0x32, - 0xa3, 0x48, 0x04, 0xd7, 0xd0, 0x94, 0x46, 0x08, 0x53, 0xff, 0xa9, 0xc0, 0x92, 0x08, 0x19, 0x0a, - 0x1b, 0xb1, 0xfb, 0x3a, 0x2d, 0x3e, 0x9e, 0x6b, 0x5b, 0x6e, 0xd4, 0x38, 0x8a, 0x33, 0x72, 0xa0, - 0x68, 0x1c, 0x3f, 0x82, 0xb6, 0x40, 0x8a, 0x6a, 0x48, 0xc1, 0x56, 0xa5, 0xc5, 0xf7, 0x45, 0xd5, - 0x63, 0x0d, 0x5a, 0xde, 0x70, 0x18, 0xe7, 0xc7, 0x93, 0x60, 0x53, 0x40, 0x05, 0xc3, 0x6f, 0x41, - 0x27, 0x44, 0x9b, 0xb5, 0x6a, 0xb5, 0xc5, 0xc6, 0xe8, 0xbd, 0xed, 0xc7, 0x0a, 0x74, 0x93, 0x35, - 0x2c, 0x76, 0xfc, 0xd9, 0xbb, 0xa4, 0xaf, 0x25, 0x5f, 0x5d, 0xd7, 0x4e, 0x90, 0x67, 0xca, 0x27, - 0x7c, 0x7b, 0xfd, 0xa3, 0x02, 0x75, 0x4d, 0xb8, 0x1d, 0x65, 0xff, 0x36, 0xc0, 0xd4, 0x31, 0xc3, - 0x9a, 0x12, 0xf9, 0x25, 0x35, 0x4e, 0xcc, 0x19, 0x2c, 0x53, 0x52, 0x54, 0x18, 0xd2, 0x34, 0x19, - 0x50, 0xf7, 0xce, 0xbc, 0x7e, 0x99, 0xb4, 0x6a, 0xb7, 0xc8, 0x91, 0xe1, 0x9b, 0xba, 0x20, 0x1e, - 0xaa, 0x53, 0xda, 0xdb, 0x1f, 0x50, 0x4c, 0x21, 0xa6, 0xd6, 0x24, 0xb1, 0x15, 0x51, 0x2d, 0x68, - 0xc4, 0x3f, 0x53, 0x3f, 0xb6, 0xb1, 0x61, 0x62, 0x3f, 0xf4, 0x63, 0xbe, 0x42, 0xd7, 0xa0, 0x6d, - 0x3a, 0xa1, 0xff, 0xf2, 0xfa, 0xc3, 0xdb, 0x9a, 0x66, 0x54, 0x47, 0x58, 0x0d, 0x8a, 0x27, 0xd0, - 0x72, 0x22, 0x81, 0xde, 0x7c, 0x01, 0xad, 0x64, 0xe6, 0x42, 0x0d, 0x58, 0xdc, 0xf3, 0x82, 0x6f, - 0x3e, 0xb7, 0x48, 0xd0, 0x99, 0x43, 0x2d, 0x80, 0x3d, 0x2f, 0xd8, 0xf7, 0x31, 0xc1, 0x6e, 0xd0, - 0x51, 0x10, 0xc0, 0xfc, 0x27, 0xee, 0xb6, 0x45, 0x9e, 0x74, 0x4a, 0x68, 0x59, 0xf4, 0x16, 0x86, - 0xbd, 0x2b, 0xd2, 0x41, 0xa7, 0x4c, 0xb7, 0x47, 0xab, 0x0a, 0xea, 0x40, 0x23, 0x42, 0xd9, 0xd9, - 0x7f, 0xd0, 0xa9, 0xa2, 0x1a, 0x54, 0xf9, 0xcf, 0xf9, 0x9b, 0x26, 0x74, 0xd2, 0xad, 0x2b, 0xa5, - 0xf9, 0xc0, 0xbd, 0xe7, 0x7a, 0xcf, 0x22, 0x50, 0x67, 0x0e, 0xd5, 0x61, 0x41, 0x5c, 0x07, 0x3a, - 0x0a, 0x6a, 0x43, 0x3d, 0xd6, 0x89, 0x77, 0x4a, 0x14, 0xb0, 0xe3, 0x8f, 0x07, 0xa2, 0x27, 0xe7, - 0x22, 0x50, 0x17, 0xdf, 0xf6, 0x9e, 0xb9, 0x9d, 0xca, 0xcd, 0x4d, 0x58, 0x0c, 0x53, 0x2a, 0x45, - 0xe5, 0xd4, 0x5d, 0xba, 0xec, 0xcc, 0xa1, 0x25, 0x68, 0x26, 0x26, 0x99, 0x1d, 0x05, 0x21, 0x68, - 0x25, 0xa7, 0xcc, 0x9d, 0xd2, 0xc6, 0x9f, 0xeb, 0x00, 0xbc, 0xed, 0xf4, 0x3c, 0xdf, 0x44, 0x63, - 0x40, 0x3b, 0x38, 0xa0, 0x25, 0xd5, 0x73, 0xc3, 0x72, 0x48, 0xd0, 0xed, 0x9c, 0xee, 0x2c, 0x8b, - 0x2a, 0x44, 0xed, 0x5d, 0xcb, 0xd9, 0x91, 0x42, 0x57, 0xe7, 0x90, 0xc3, 0x38, 0x1e, 0x5a, 0x0e, - 0x3e, 0xb4, 0x06, 0x4f, 0xa2, 0x7e, 0x35, 0x9f, 0x63, 0x0a, 0x35, 0xe4, 0x98, 0x2a, 0x5d, 0x62, - 0x71, 0x10, 0xf8, 0x96, 0x3b, 0x0a, 0x07, 0x25, 0xea, 0x1c, 0x7a, 0x0a, 0x17, 0x76, 0x30, 0xe3, - 0x6e, 0x91, 0xc0, 0x1a, 0x90, 0x90, 0xe1, 0x46, 0x3e, 0xc3, 0x0c, 0xf2, 0x8c, 0x2c, 0x6d, 0x68, - 0xa7, 0xfe, 0xd5, 0x81, 0x6e, 0xca, 0xe3, 0x46, 0xf6, 0x0f, 0x94, 0xde, 0x7b, 0x85, 0x70, 0x23, - 0x6e, 0x16, 0xb4, 0x92, 0xff, 0x78, 0x40, 0x5f, 0xc8, 0x23, 0x90, 0x99, 0xfd, 0xf6, 0x6e, 0x16, - 0x41, 0x8d, 0x58, 0x3d, 0xe2, 0xfe, 0x74, 0x1a, 0x2b, 0xe9, 0x40, 0xbd, 0x77, 0xd2, 0x8c, 0x4a, - 0x9d, 0x43, 0xdf, 0x87, 0xa5, 0xcc, 0x84, 0x1a, 0x7d, 0x51, 0x46, 0x3e, 0x6f, 0x90, 0x7d, 0x1a, - 0x87, 0x47, 0xe9, 0x68, 0xc8, 0x97, 0x3e, 0xf3, 0x57, 0x85, 0xe2, 0xd2, 0xc7, 0xc8, 0x9f, 0x24, - 0xfd, 0xcc, 0x1c, 0x26, 0x80, 0xb2, 0x33, 0x6a, 0xf4, 0xbe, 0x8c, 0x45, 0xee, 0x9c, 0xbc, 0xb7, - 0x5e, 0x14, 0x3d, 0x32, 0xf9, 0x84, 0x45, 0x6b, 0xfa, 0xde, 0x25, 0x65, 0x9b, 0x3b, 0x97, 0x96, - 0xb3, 0xcd, 0x1f, 0x0d, 0x73, 0xa7, 0x4e, 0x8e, 0x3e, 0xe5, 0xb6, 0x92, 0x8e, 0x6b, 0xe5, 0x4e, - 0x2d, 0x9f, 0xa4, 0xaa, 0x73, 0xe8, 0x30, 0x91, 0x84, 0xd1, 0xb5, 0x3c, 0x9f, 0x48, 0xbe, 0x97, - 0x9c, 0x66, 0x2e, 0x1d, 0x60, 0x07, 0x07, 0xf7, 0x71, 0xe0, 0x5b, 0x03, 0x92, 0x26, 0x2a, 0x16, - 0x53, 0x84, 0x90, 0xe8, 0xf5, 0x53, 0xf1, 0x42, 0xb1, 0x37, 0x5e, 0x02, 0xd4, 0x98, 0xcd, 0x68, - 0x7d, 0xf8, 0x5f, 0x1a, 0x7f, 0x05, 0x69, 0xfc, 0x31, 0xb4, 0x53, 0x73, 0x6f, 0x79, 0x1a, 0x97, - 0x0f, 0xc7, 0x4f, 0x73, 0x90, 0x3e, 0xa0, 0xec, 0x54, 0x57, 0x1e, 0x58, 0xb9, 0xd3, 0xdf, 0xd3, - 0x78, 0x3c, 0x86, 0x76, 0x6a, 0x84, 0x29, 0x3f, 0x81, 0x7c, 0xce, 0x59, 0xe0, 0x04, 0xd9, 0xc9, - 0x9b, 0xfc, 0x04, 0xb9, 0x13, 0xba, 0xd3, 0x78, 0x3c, 0xe4, 0x83, 0xe1, 0xe8, 0x3e, 0x70, 0x3d, - 0x2f, 0x3a, 0x53, 0x8f, 0xab, 0xaf, 0x3f, 0x5f, 0xbf, 0xfa, 0x7a, 0xf6, 0x18, 0xda, 0xa9, 0xa7, - 0x73, 0xb9, 0x75, 0xe5, 0xef, 0xeb, 0xa7, 0x51, 0xff, 0x1c, 0x33, 0xf0, 0xab, 0xce, 0x95, 0x9b, - 0x5f, 0x7e, 0xb4, 0x31, 0xb2, 0x82, 0xa3, 0x49, 0x9f, 0x9e, 0xf2, 0x16, 0xc7, 0x7c, 0xdf, 0xf2, - 0xc4, 0xaf, 0x5b, 0x61, 0xd2, 0xb8, 0xc5, 0x28, 0xdd, 0x62, 0xd2, 0x8e, 0xfb, 0xfd, 0x79, 0xb6, - 0xbc, 0xf3, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x27, 0xc3, 0xf4, 0xc1, 0xa0, 0x2c, 0x00, 0x00, + 0x89, 0xc3, 0x95, 0x03, 0x20, 0x21, 0xc4, 0x15, 0x21, 0x4e, 0x20, 0xc8, 0x61, 0x2f, 0x9c, 0x39, + 0x70, 0xe1, 0x9f, 0xe0, 0x80, 0x38, 0x20, 0xc1, 0x3f, 0xc0, 0x11, 0x81, 0xea, 0xa3, 0x7b, 0xfa, + 0xa3, 0xda, 0xee, 0xb1, 0x95, 0x4d, 0x84, 0xb8, 0x4d, 0xbd, 0x7e, 0xf5, 0xde, 0xab, 0xf7, 0x5d, + 0xf5, 0x06, 0x96, 0x9e, 0x4e, 0xb0, 0x7f, 0xac, 0x0f, 0x3c, 0xcf, 0x37, 0xd7, 0xc7, 0xbe, 0x17, + 0x78, 0x08, 0x39, 0x96, 0xfd, 0xe9, 0x84, 0xf0, 0xd5, 0x3a, 0xfb, 0xde, 0x6b, 0x0c, 0x3c, 0xc7, + 0xf1, 0x5c, 0x0e, 0xeb, 0x35, 0xe2, 0x18, 0xbd, 0x96, 0xe5, 0x06, 0xd8, 0x77, 0x0d, 0x3b, 0xfc, + 0x4a, 0x06, 0x47, 0xd8, 0x31, 0xc4, 0xaa, 0x63, 0x1a, 0x81, 0x11, 0xa7, 0xaf, 0xfe, 0x50, 0x81, + 0x95, 0x83, 0x23, 0xef, 0xd9, 0x96, 0x67, 0xdb, 0x78, 0x10, 0x58, 0x9e, 0x4b, 0x34, 0xfc, 0x74, + 0x82, 0x49, 0x80, 0x6e, 0x43, 0xa5, 0x6f, 0x10, 0xdc, 0x55, 0x56, 0x95, 0x1b, 0xf5, 0x8d, 0xb7, + 0xd6, 0x13, 0x92, 0x08, 0x11, 0xee, 0x93, 0xd1, 0xa6, 0x41, 0xb0, 0xc6, 0x30, 0x11, 0x82, 0x8a, + 0xd9, 0xdf, 0xdd, 0xee, 0x96, 0x56, 0x95, 0x1b, 0x65, 0x8d, 0xfd, 0x46, 0xef, 0x42, 0x73, 0x10, + 0xd1, 0xde, 0xdd, 0x26, 0xdd, 0xf2, 0x6a, 0xf9, 0x46, 0x59, 0x4b, 0x02, 0xd5, 0xdf, 0x2a, 0xf0, + 0x7f, 0x19, 0x31, 0xc8, 0xd8, 0x73, 0x09, 0x46, 0x77, 0x60, 0x9e, 0x04, 0x46, 0x30, 0x21, 0x42, + 0x92, 0xff, 0x97, 0x4a, 0x72, 0xc0, 0x50, 0x34, 0x81, 0x9a, 0x65, 0x5b, 0x92, 0xb0, 0x45, 0x5f, + 0x82, 0x0b, 0x96, 0x7b, 0x1f, 0x3b, 0x9e, 0x7f, 0xac, 0x8f, 0xb1, 0x3f, 0xc0, 0x6e, 0x60, 0x8c, + 0x70, 0x28, 0xe3, 0x72, 0xf8, 0x6d, 0x7f, 0xfa, 0x49, 0xfd, 0x8d, 0x02, 0x17, 0xa9, 0xa4, 0xfb, + 0x86, 0x1f, 0x58, 0xaf, 0x40, 0x5f, 0x2a, 0x34, 0xe2, 0x32, 0x76, 0xcb, 0xec, 0x5b, 0x02, 0x46, + 0x71, 0xc6, 0x21, 0x7b, 0x7a, 0xb6, 0x0a, 0x13, 0x37, 0x01, 0x53, 0x7f, 0x2d, 0x0c, 0x1b, 0x97, + 0xf3, 0x3c, 0x0a, 0x4d, 0xf3, 0x2c, 0x65, 0x79, 0x9e, 0x45, 0x9d, 0x7f, 0x57, 0xe0, 0xe2, 0xc7, + 0x9e, 0x61, 0x4e, 0x0d, 0xff, 0xf9, 0xab, 0xf3, 0x1b, 0x30, 0xcf, 0xa3, 0xa4, 0x5b, 0x61, 0xbc, + 0xd6, 0x92, 0xbc, 0x44, 0x04, 0x4d, 0x25, 0x3c, 0x60, 0x00, 0x4d, 0x6c, 0x42, 0x6b, 0xd0, 0xf2, + 0xf1, 0xd8, 0xb6, 0x06, 0x86, 0xee, 0x4e, 0x9c, 0x3e, 0xf6, 0xbb, 0xd5, 0x55, 0xe5, 0x46, 0x55, + 0x6b, 0x0a, 0xe8, 0x1e, 0x03, 0xaa, 0xbf, 0x54, 0xa0, 0xab, 0x61, 0x1b, 0x1b, 0x04, 0xbf, 0xce, + 0xc3, 0xae, 0xc0, 0xbc, 0xeb, 0x99, 0x78, 0x77, 0x9b, 0x1d, 0xb6, 0xac, 0x89, 0x95, 0xfa, 0x37, + 0x61, 0x88, 0x37, 0xdc, 0xaf, 0x63, 0xc6, 0xaa, 0x9e, 0xc1, 0x58, 0xea, 0x1f, 0xa7, 0x56, 0x78, + 0xd3, 0x4f, 0x3a, 0xb5, 0x54, 0x35, 0x61, 0xa9, 0xef, 0xc1, 0xa5, 0x2d, 0x1f, 0x1b, 0x01, 0xfe, + 0x36, 0xad, 0x06, 0x5b, 0x47, 0x86, 0xeb, 0x62, 0x3b, 0x3c, 0x42, 0x9a, 0xb9, 0x22, 0x61, 0xde, + 0x85, 0x85, 0xb1, 0xef, 0x3d, 0x3f, 0x8e, 0xe4, 0x0e, 0x97, 0xea, 0xef, 0x14, 0xe8, 0xc9, 0x68, + 0x9f, 0x27, 0x71, 0x5c, 0x85, 0xa6, 0x28, 0x6b, 0x9c, 0x1a, 0xe3, 0x59, 0xd3, 0x1a, 0x4f, 0x63, + 0x1c, 0xd0, 0x6d, 0xb8, 0xc0, 0x91, 0x7c, 0x4c, 0x26, 0x76, 0x10, 0xe1, 0x96, 0x19, 0x2e, 0x62, + 0xdf, 0x34, 0xf6, 0x49, 0xec, 0x50, 0x5f, 0x2a, 0x70, 0x69, 0x07, 0x07, 0x91, 0x11, 0x29, 0x57, + 0xfc, 0x86, 0xe6, 0xe2, 0xcf, 0x14, 0xe8, 0xc9, 0x64, 0x3d, 0x8f, 0x5a, 0x1f, 0xc1, 0x4a, 0xc4, + 0x43, 0x37, 0x31, 0x19, 0xf8, 0xd6, 0x98, 0x39, 0x33, 0xcb, 0xcc, 0xf5, 0x8d, 0xab, 0xeb, 0xd9, + 0xce, 0x61, 0x3d, 0x2d, 0xc1, 0xc5, 0x88, 0xc4, 0x76, 0x8c, 0x82, 0xfa, 0x53, 0x05, 0x2e, 0xee, + 0xe0, 0xe0, 0x00, 0x8f, 0x1c, 0xec, 0x06, 0xbb, 0xee, 0xd0, 0x3b, 0xbb, 0x5e, 0x2f, 0x03, 0x10, + 0x41, 0x27, 0xaa, 0x1a, 0x31, 0x48, 0x11, 0x1d, 0xb3, 0x26, 0x25, 0x2d, 0xcf, 0x79, 0x74, 0xf7, + 0x15, 0xa8, 0x5a, 0xee, 0xd0, 0x0b, 0x55, 0x75, 0x45, 0xa6, 0xaa, 0x38, 0x33, 0x8e, 0xad, 0xfe, + 0xab, 0x04, 0x2b, 0x1f, 0x9a, 0xa6, 0x2c, 0xec, 0x66, 0xd7, 0xcb, 0x34, 0xba, 0x4b, 0xf1, 0xe8, + 0x2e, 0xe4, 0x73, 0x99, 0x90, 0xaa, 0xcc, 0x10, 0x52, 0xd5, 0xbc, 0x90, 0x42, 0x3b, 0xd0, 0x24, + 0x18, 0x3f, 0xd1, 0xc7, 0x1e, 0x61, 0x3e, 0xd1, 0x9d, 0x67, 0xa7, 0x51, 0x93, 0xa7, 0x89, 0x1a, + 0xcc, 0xfb, 0x64, 0xb4, 0x2f, 0x30, 0xb5, 0x06, 0xdd, 0x18, 0xae, 0xd0, 0x03, 0x58, 0x19, 0xd9, + 0x5e, 0xdf, 0xb0, 0x75, 0x82, 0x0d, 0x1b, 0x9b, 0xba, 0xb0, 0x37, 0xe9, 0x2e, 0x14, 0x53, 0xf8, + 0x05, 0xbe, 0xfd, 0x80, 0xed, 0x16, 0x1f, 0x88, 0xfa, 0x57, 0x05, 0x2e, 0x69, 0xd8, 0xf1, 0x3e, + 0xc5, 0xff, 0xad, 0x26, 0x50, 0x7f, 0xae, 0x40, 0x83, 0x56, 0xe1, 0xfb, 0x38, 0x30, 0xa8, 0x26, + 0xd0, 0x07, 0x50, 0xb3, 0x3d, 0xc3, 0xd4, 0x83, 0xe3, 0x31, 0x3f, 0x5a, 0x2b, 0x7d, 0x34, 0xae, + 0x3d, 0xba, 0xe9, 0xf0, 0x78, 0x8c, 0xb5, 0x45, 0x5b, 0xfc, 0xca, 0x1c, 0xa3, 0x54, 0x20, 0x7b, + 0x95, 0x65, 0x9d, 0x64, 0x19, 0x56, 0xbe, 0x63, 0x04, 0x83, 0xa3, 0x6d, 0x47, 0x88, 0x49, 0x5e, + 0x8f, 0xce, 0x8b, 0x14, 0xcd, 0x28, 0xb4, 0xab, 0x32, 0x4f, 0xa3, 0xd7, 0x9f, 0xf5, 0x87, 0xc2, + 0x0c, 0xb1, 0xd0, 0x8e, 0x75, 0x15, 0xf3, 0x67, 0x69, 0x01, 0xb7, 0xa0, 0x89, 0x9f, 0x0f, 0xec, + 0x89, 0x89, 0x75, 0xce, 0x9d, 0xfb, 0xf9, 0x65, 0x09, 0xf7, 0xb8, 0x9b, 0x37, 0xc4, 0xa6, 0x5d, + 0x21, 0x03, 0x37, 0xb5, 0x83, 0x03, 0xa3, 0xbb, 0xc8, 0xc4, 0x58, 0xcd, 0x33, 0x75, 0xe8, 0x1f, + 0xdc, 0xdc, 0x74, 0xa5, 0xfe, 0x5b, 0x81, 0x4b, 0xdc, 0x4c, 0xd8, 0x0e, 0x8c, 0xd7, 0x6b, 0xa9, + 0xc8, 0x0a, 0x95, 0x19, 0xad, 0x10, 0xd3, 0x40, 0x6d, 0x66, 0x0d, 0xfc, 0xa2, 0x02, 0x6d, 0xa1, + 0x5e, 0x8a, 0xc1, 0xe2, 0xe7, 0x2d, 0xa8, 0x45, 0xc5, 0x46, 0x34, 0x43, 0x53, 0x00, 0x5a, 0x85, + 0x7a, 0xcc, 0x7b, 0xc4, 0x41, 0xe3, 0xa0, 0x42, 0xa7, 0x0d, 0x5b, 0x87, 0x4a, 0xac, 0x75, 0x78, + 0x1b, 0x60, 0x68, 0x4f, 0xc8, 0x91, 0x1e, 0x58, 0x0e, 0x16, 0x0d, 0x5c, 0x8d, 0x41, 0x0e, 0x2d, + 0x07, 0xa3, 0x0f, 0xa1, 0xd1, 0xb7, 0x5c, 0xdb, 0x1b, 0xe9, 0x63, 0x23, 0x38, 0x22, 0xdd, 0xf9, + 0x5c, 0x7f, 0xb9, 0x6b, 0x61, 0xdb, 0xdc, 0x64, 0xb8, 0x5a, 0x9d, 0xef, 0xd9, 0xa7, 0x5b, 0xd0, + 0x65, 0xa8, 0xbb, 0x13, 0x47, 0xf7, 0x86, 0xba, 0xef, 0x3d, 0xa3, 0x1e, 0xc7, 0x58, 0xb8, 0x13, + 0xe7, 0x93, 0xa1, 0xe6, 0x3d, 0x23, 0xe8, 0xeb, 0x50, 0xa3, 0xe5, 0x8e, 0xd8, 0xde, 0x88, 0x74, + 0x17, 0x0b, 0xd1, 0x9f, 0x6e, 0xa0, 0xbb, 0x4d, 0xea, 0x47, 0x6c, 0x77, 0xad, 0xd8, 0xee, 0x68, + 0x03, 0xba, 0x06, 0xad, 0x81, 0xe7, 0x8c, 0x0d, 0xa6, 0xa1, 0xbb, 0xbe, 0xe7, 0x74, 0x81, 0xc5, + 0x6a, 0x0a, 0x8a, 0xee, 0x42, 0xdd, 0x72, 0x4d, 0xfc, 0x5c, 0x44, 0x4d, 0x9d, 0xf1, 0x59, 0x93, + 0x99, 0xfc, 0x21, 0x1e, 0x30, 0x5e, 0xbb, 0x14, 0x9d, 0xd9, 0x1d, 0xac, 0xf0, 0x27, 0x41, 0xef, + 0x40, 0x43, 0x18, 0x55, 0x27, 0xd6, 0x0b, 0xdc, 0x6d, 0x70, 0x43, 0x0a, 0xd8, 0x81, 0xf5, 0x02, + 0xab, 0xbf, 0x2f, 0xc1, 0x52, 0x86, 0x08, 0x6d, 0x85, 0x87, 0x0c, 0x12, 0x3a, 0x47, 0xb8, 0xa4, + 0x24, 0xb1, 0x6b, 0xf4, 0x6d, 0x1a, 0xd1, 0x26, 0x7e, 0xce, 0x7c, 0x63, 0x51, 0xab, 0x73, 0x18, + 0x23, 0x40, 0x6d, 0xcc, 0xa5, 0x77, 0x0d, 0x07, 0x8b, 0x56, 0xb5, 0xc6, 0x20, 0x7b, 0x86, 0x83, + 0x29, 0x6d, 0x2e, 0x62, 0xe8, 0x19, 0xe1, 0x92, 0x7e, 0xe9, 0x4f, 0x2c, 0xc6, 0x95, 0x7b, 0x46, + 0xb8, 0x44, 0xdb, 0xd0, 0xe0, 0x24, 0xc7, 0x86, 0x6f, 0x38, 0xa1, 0x5f, 0xbc, 0x23, 0x0d, 0xd7, + 0x7b, 0xf8, 0xf8, 0xa1, 0x61, 0x4f, 0xf0, 0xbe, 0x61, 0xf9, 0x1a, 0xd7, 0xe3, 0x3e, 0xdb, 0x85, + 0x6e, 0x40, 0x87, 0x53, 0x19, 0x5a, 0x36, 0x16, 0x1e, 0x46, 0x33, 0x52, 0x4d, 0x6b, 0x31, 0xf8, + 0x5d, 0xcb, 0xc6, 0xdc, 0x89, 0xa2, 0x23, 0x30, 0xb5, 0x2d, 0x72, 0x1f, 0x62, 0x10, 0xa6, 0xb4, + 0x7f, 0x94, 0x60, 0x99, 0x86, 0x52, 0x58, 0x82, 0xcf, 0x9e, 0x4d, 0xde, 0x06, 0x30, 0x49, 0xa0, + 0x27, 0x32, 0x4a, 0xcd, 0x24, 0xc1, 0x1e, 0x4f, 0x2a, 0x1f, 0x84, 0x09, 0xa3, 0x9c, 0xdf, 0xbc, + 0xa6, 0x42, 0x3b, 0x9b, 0xba, 0xcf, 0x74, 0x7b, 0xbf, 0x0a, 0x4d, 0xe2, 0x4d, 0xfc, 0x01, 0xd6, + 0x13, 0x97, 0xad, 0x06, 0x07, 0xee, 0xc9, 0x73, 0xde, 0xbc, 0xf4, 0x15, 0x21, 0x96, 0xbc, 0x16, + 0x66, 0x4e, 0x5e, 0x7f, 0x51, 0x60, 0x45, 0x5c, 0x4c, 0xcf, 0xaf, 0xed, 0xbc, 0xdc, 0x1d, 0x66, + 0xaa, 0xf2, 0x09, 0x97, 0x9c, 0x4a, 0x81, 0xca, 0x5b, 0x95, 0x54, 0xde, 0x64, 0xa3, 0x3f, 0x9f, + 0x6e, 0xf4, 0xd5, 0x9f, 0x29, 0xb0, 0xf2, 0x91, 0xe1, 0x9a, 0xde, 0x70, 0x78, 0xfe, 0x03, 0x6e, + 0x45, 0x01, 0xbf, 0x3b, 0x4b, 0x23, 0x9f, 0xd8, 0xa4, 0xfe, 0xa8, 0x04, 0x88, 0x5a, 0x63, 0xd3, + 0xb0, 0x0d, 0x77, 0x80, 0xcf, 0x2e, 0xcd, 0x1a, 0xb4, 0x12, 0x3e, 0x14, 0xbd, 0x36, 0xc6, 0x9d, + 0x88, 0xa0, 0x7b, 0xd0, 0xea, 0x73, 0x56, 0xba, 0x8f, 0x0d, 0xe2, 0xb9, 0xcc, 0x0e, 0xad, 0x8d, + 0x77, 0x65, 0x62, 0x1f, 0xfa, 0xd6, 0x68, 0x84, 0xfd, 0x2d, 0xcf, 0x35, 0x79, 0x8b, 0xdd, 0xec, + 0x87, 0x62, 0xd2, 0xad, 0xe8, 0x0a, 0xd4, 0xa7, 0x01, 0x15, 0xf6, 0x42, 0x10, 0x45, 0x14, 0x41, + 0xef, 0xc1, 0x52, 0xb2, 0xfb, 0x9e, 0x1a, 0xae, 0x43, 0xe2, 0x8d, 0x35, 0x35, 0xce, 0x0f, 0x00, + 0x45, 0xdd, 0x1d, 0x6b, 0x22, 0x58, 0x76, 0x2c, 0xf2, 0x98, 0xf0, 0x16, 0xd4, 0xcc, 0x70, 0xa7, + 0xb8, 0xda, 0x4f, 0x01, 0x34, 0xba, 0xb8, 0x84, 0x3a, 0x75, 0x74, 0x6c, 0x86, 0xf5, 0x93, 0x03, + 0x3f, 0x66, 0x30, 0xf5, 0xb3, 0x12, 0x74, 0xe2, 0x1d, 0x7d, 0x61, 0xde, 0xaf, 0xe6, 0x69, 0xe1, + 0x84, 0xeb, 0x4b, 0xe5, 0x1c, 0xd7, 0x97, 0xec, 0xf5, 0xaa, 0x7a, 0xb6, 0xeb, 0x95, 0xfa, 0x2b, + 0x05, 0xda, 0xa9, 0x9b, 0x7c, 0xba, 0x93, 0x51, 0xb2, 0x9d, 0xcc, 0x57, 0xa1, 0x4a, 0xcb, 0x3b, + 0x66, 0x4a, 0x6a, 0xa5, 0xd9, 0xca, 0xde, 0x07, 0x34, 0xbe, 0x01, 0xdd, 0x82, 0x65, 0xc9, 0xb3, + 0xae, 0x30, 0x25, 0xca, 0xbe, 0xea, 0xaa, 0x7f, 0xa8, 0x40, 0x3d, 0xa6, 0x8f, 0x53, 0x9a, 0xb0, + 0x22, 0xf7, 0x94, 0xd4, 0xf1, 0xca, 0xd9, 0xe3, 0xe5, 0xbc, 0x6b, 0xa2, 0x4b, 0xb0, 0xe8, 0x60, + 0x87, 0xd7, 0x37, 0x51, 0x6c, 0x1d, 0xec, 0xd0, 0xea, 0x46, 0x3f, 0xd1, 0x0e, 0x8a, 0xb5, 0x4f, + 0x3c, 0xa3, 0x2f, 0xb8, 0x13, 0x87, 0x35, 0x4f, 0xc9, 0xd2, 0xbe, 0x70, 0x42, 0x69, 0x5f, 0x4c, + 0x96, 0xf6, 0x44, 0x38, 0xd4, 0xd2, 0xe1, 0x50, 0xb4, 0x2f, 0xba, 0x0d, 0xcb, 0x03, 0xf6, 0x0c, + 0x67, 0x6e, 0x1e, 0x6f, 0x45, 0x9f, 0xba, 0x75, 0xd6, 0x83, 0xc8, 0x3e, 0xa1, 0xbb, 0xd4, 0xb9, + 0x44, 0x07, 0xc4, 0xac, 0xdc, 0x60, 0x56, 0x96, 0x77, 0x0e, 0xc2, 0x36, 0xdc, 0xc8, 0x61, 0x4e, + 0x64, 0xab, 0x74, 0x47, 0xd6, 0x3c, 0x6b, 0x47, 0x76, 0x05, 0xea, 0xe1, 0xa3, 0xb8, 0x65, 0x92, + 0x6e, 0x8b, 0xa7, 0x27, 0x01, 0xda, 0x35, 0x09, 0x53, 0xbe, 0x47, 0xef, 0x4b, 0x26, 0xe9, 0xb6, + 0xd9, 0xd7, 0x05, 0x66, 0x31, 0x93, 0xa8, 0x2f, 0xcb, 0xd0, 0x9a, 0xd6, 0xeb, 0xc2, 0xd9, 0xa0, + 0xc8, 0x84, 0x62, 0x0f, 0x3a, 0xd3, 0x57, 0x33, 0xa6, 0xa8, 0x13, 0x5b, 0x8e, 0xf4, 0x7b, 0x59, + 0x7b, 0x9c, 0x0a, 0xbb, 0xc4, 0xf5, 0xbc, 0x32, 0xd3, 0xf5, 0xfc, 0x7c, 0x0f, 0xd9, 0xe8, 0x0e, + 0x5c, 0xf4, 0x79, 0xbb, 0x60, 0xea, 0x89, 0x63, 0xf3, 0xca, 0x7b, 0x21, 0xfc, 0xb8, 0x1f, 0x3f, + 0x7e, 0x4e, 0x24, 0x2f, 0xe4, 0x45, 0x72, 0xda, 0x8c, 0x8b, 0x69, 0x33, 0xaa, 0x0f, 0x60, 0xf9, + 0x81, 0x4b, 0x26, 0x7d, 0x32, 0xf0, 0xad, 0x3e, 0x0e, 0x6f, 0x9d, 0x85, 0xec, 0xd5, 0x83, 0x45, + 0x91, 0x8b, 0xb9, 0xad, 0x6a, 0x5a, 0xb4, 0x56, 0x7f, 0xa2, 0xc0, 0x4a, 0x96, 0x2e, 0x73, 0x85, + 0x69, 0xa0, 0x2b, 0x89, 0x40, 0xff, 0x2e, 0x2c, 0x4f, 0xc9, 0xeb, 0x09, 0xca, 0xf5, 0x8d, 0xeb, + 0x32, 0xa3, 0x48, 0x04, 0xd7, 0xd0, 0x94, 0x46, 0x08, 0x53, 0xff, 0xa9, 0xc0, 0x92, 0x08, 0x19, + 0x0a, 0x1b, 0xb1, 0xfb, 0x3a, 0x2d, 0x3e, 0x9e, 0x6b, 0x5b, 0x6e, 0xd4, 0x38, 0x8a, 0x33, 0x72, + 0xa0, 0x68, 0x1c, 0x3f, 0x82, 0xb6, 0x40, 0x8a, 0x6a, 0x48, 0xc1, 0x56, 0xa5, 0xc5, 0xf7, 0x45, + 0xd5, 0x63, 0x0d, 0x5a, 0xde, 0x70, 0x18, 0xe7, 0xc7, 0x93, 0x60, 0x53, 0x40, 0x05, 0xc3, 0x6f, + 0x41, 0x27, 0x44, 0x9b, 0xb5, 0x6a, 0xb5, 0xc5, 0xc6, 0xe8, 0xbd, 0xed, 0xc7, 0x0a, 0x74, 0x93, + 0x35, 0x2c, 0x76, 0xfc, 0xd9, 0xbb, 0xa4, 0xaf, 0x25, 0x5f, 0x5d, 0xd7, 0x4e, 0x90, 0x67, 0xca, + 0x27, 0x7c, 0x7b, 0xfd, 0xb3, 0x02, 0x75, 0x4d, 0xb8, 0x1d, 0x65, 0xff, 0x36, 0xc0, 0xd4, 0x31, + 0xc3, 0x9a, 0x12, 0xf9, 0x25, 0x35, 0x4e, 0xcc, 0x19, 0x2c, 0x53, 0x52, 0x54, 0x18, 0xd2, 0x34, + 0x19, 0x50, 0xf7, 0xce, 0xbc, 0x7e, 0x99, 0xb4, 0x6a, 0xb7, 0xc8, 0x91, 0xe1, 0x9b, 0xba, 0x20, + 0x1e, 0xaa, 0x53, 0xda, 0xdb, 0x1f, 0x50, 0x4c, 0x21, 0xa6, 0xd6, 0x24, 0xb1, 0x55, 0x32, 0xe1, + 0x55, 0x93, 0x09, 0x6f, 0x0f, 0x1a, 0xf1, 0x9d, 0xd4, 0xc5, 0x6d, 0x6c, 0x98, 0xd8, 0x0f, 0x5d, + 0x9c, 0xaf, 0xd0, 0x35, 0x68, 0x9b, 0x4e, 0xe8, 0xda, 0xbc, 0x34, 0xf1, 0x8e, 0xa7, 0x19, 0x95, + 0x18, 0x5a, 0x9e, 0x6e, 0xbe, 0x80, 0x56, 0x32, 0x73, 0xa1, 0x06, 0x2c, 0xee, 0x79, 0xc1, 0x37, + 0x9f, 0x5b, 0x24, 0xe8, 0xcc, 0xa1, 0x16, 0xc0, 0x9e, 0x17, 0xec, 0xfb, 0x98, 0x60, 0x37, 0xe8, + 0x28, 0x08, 0x60, 0xfe, 0x13, 0x77, 0xdb, 0x22, 0x4f, 0x3a, 0x25, 0xb4, 0x2c, 0x7a, 0x0b, 0xc3, + 0xde, 0x15, 0xe9, 0xa0, 0x53, 0xa6, 0xdb, 0xa3, 0x55, 0x05, 0x75, 0xa0, 0x11, 0xa1, 0xec, 0xec, + 0x3f, 0xe8, 0x54, 0x51, 0x0d, 0xaa, 0xfc, 0xe7, 0xfc, 0x4d, 0x13, 0x3a, 0xe9, 0xd6, 0x95, 0xd2, + 0x7c, 0xe0, 0xde, 0x73, 0xbd, 0x67, 0x11, 0xa8, 0x33, 0x87, 0xea, 0xb0, 0x20, 0xae, 0x03, 0x1d, + 0x05, 0xb5, 0xa1, 0x1e, 0xeb, 0xc4, 0x3b, 0x25, 0x0a, 0xd8, 0xf1, 0xc7, 0x03, 0xd1, 0x93, 0x73, + 0x11, 0xa8, 0x8b, 0x6f, 0x7b, 0xcf, 0xdc, 0x4e, 0xe5, 0xe6, 0x26, 0x2c, 0x86, 0x29, 0x95, 0xa2, + 0x72, 0xea, 0x2e, 0x5d, 0x76, 0xe6, 0xd0, 0x12, 0x34, 0x13, 0x93, 0xcc, 0x8e, 0x82, 0x10, 0xb4, + 0x92, 0x53, 0xe6, 0x4e, 0x69, 0xe3, 0x4f, 0x75, 0x00, 0xde, 0x76, 0x7a, 0x9e, 0x6f, 0xa2, 0x31, + 0xa0, 0x1d, 0x1c, 0xd0, 0x92, 0xea, 0xb9, 0x61, 0x39, 0x24, 0xe8, 0x76, 0x4e, 0x77, 0x96, 0x45, + 0x15, 0xa2, 0xf6, 0xae, 0xe5, 0xec, 0x48, 0xa1, 0xab, 0x73, 0xc8, 0x61, 0x1c, 0x0f, 0x2d, 0x07, + 0x1f, 0x5a, 0x83, 0x27, 0x51, 0xbf, 0x9a, 0xcf, 0x31, 0x85, 0x1a, 0x72, 0x4c, 0x95, 0x2e, 0xb1, + 0x38, 0x08, 0x7c, 0xcb, 0x1d, 0x85, 0x83, 0x12, 0x75, 0x0e, 0x3d, 0x85, 0x0b, 0x3b, 0x98, 0x71, + 0xb7, 0x48, 0x60, 0x0d, 0x48, 0xc8, 0x70, 0x23, 0x9f, 0x61, 0x06, 0x79, 0x46, 0x96, 0x36, 0xb4, + 0x53, 0xff, 0xea, 0x40, 0x37, 0xe5, 0x71, 0x23, 0xfb, 0x07, 0x4a, 0xef, 0xbd, 0x42, 0xb8, 0x11, + 0x37, 0x0b, 0x5a, 0xc9, 0x7f, 0x3c, 0xa0, 0x2f, 0xe4, 0x11, 0xc8, 0xcc, 0x7e, 0x7b, 0x37, 0x8b, + 0xa0, 0x46, 0xac, 0x1e, 0x71, 0x7f, 0x3a, 0x8d, 0x95, 0x74, 0xa0, 0xde, 0x3b, 0x69, 0x46, 0xa5, + 0xce, 0xa1, 0xef, 0xc3, 0x52, 0x66, 0x42, 0x8d, 0xbe, 0x28, 0x23, 0x9f, 0x37, 0xc8, 0x3e, 0x8d, + 0xc3, 0xa3, 0x74, 0x34, 0xe4, 0x4b, 0x9f, 0xf9, 0xab, 0x42, 0x71, 0xe9, 0x63, 0xe4, 0x4f, 0x92, + 0x7e, 0x66, 0x0e, 0x13, 0x40, 0xd9, 0x19, 0x35, 0x7a, 0x5f, 0xc6, 0x22, 0x77, 0x4e, 0xde, 0x5b, + 0x2f, 0x8a, 0x1e, 0x99, 0x7c, 0xc2, 0xa2, 0x35, 0x7d, 0xef, 0x92, 0xb2, 0xcd, 0x9d, 0x4b, 0xcb, + 0xd9, 0xe6, 0x8f, 0x86, 0xb9, 0x53, 0x27, 0x47, 0x9f, 0x72, 0x5b, 0x49, 0xc7, 0xb5, 0x72, 0xa7, + 0x96, 0x4f, 0x52, 0xd5, 0x39, 0x74, 0x98, 0x48, 0xc2, 0xe8, 0x5a, 0x9e, 0x4f, 0x24, 0xdf, 0x4b, + 0x4e, 0x33, 0x97, 0x0e, 0xb0, 0x83, 0x83, 0xfb, 0x38, 0xf0, 0xad, 0x01, 0x49, 0x13, 0x15, 0x8b, + 0x29, 0x42, 0x48, 0xf4, 0xfa, 0xa9, 0x78, 0xa1, 0xd8, 0x1b, 0x2f, 0x01, 0x6a, 0xcc, 0x66, 0xb4, + 0x3e, 0xfc, 0x2f, 0x8d, 0xbf, 0x82, 0x34, 0xfe, 0x18, 0xda, 0xa9, 0xb9, 0xb7, 0x3c, 0x8d, 0xcb, + 0x87, 0xe3, 0xa7, 0x39, 0x48, 0x1f, 0x50, 0x76, 0xaa, 0x2b, 0x0f, 0xac, 0xdc, 0xe9, 0xef, 0x69, + 0x3c, 0x1e, 0x43, 0x3b, 0x35, 0xc2, 0x94, 0x9f, 0x40, 0x3e, 0xe7, 0x2c, 0x70, 0x82, 0xec, 0xe4, + 0x4d, 0x7e, 0x82, 0xdc, 0x09, 0xdd, 0x69, 0x3c, 0x1e, 0xf2, 0xc1, 0x70, 0x74, 0x1f, 0xb8, 0x9e, + 0x17, 0x9d, 0xa9, 0xc7, 0xd5, 0xd7, 0x9f, 0xaf, 0x5f, 0x7d, 0x3d, 0x7b, 0x0c, 0xed, 0xd4, 0xd3, + 0xb9, 0xdc, 0xba, 0xf2, 0xf7, 0xf5, 0xd3, 0xa8, 0x7f, 0x8e, 0x19, 0xf8, 0x55, 0xe7, 0xca, 0xcd, + 0x2f, 0x3f, 0xda, 0x18, 0x59, 0xc1, 0xd1, 0xa4, 0x4f, 0x4f, 0x79, 0x8b, 0x63, 0xbe, 0x6f, 0x79, + 0xe2, 0xd7, 0xad, 0x30, 0x69, 0xdc, 0x62, 0x94, 0x6e, 0x31, 0x69, 0xc7, 0xfd, 0xfe, 0x3c, 0x5b, + 0xde, 0xf9, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x52, 0x32, 0xc1, 0x89, 0xa0, 0x2c, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/internal/querycoord/cluster.go b/internal/querycoord/cluster.go index 7d65d67973..fed9efcac2 100644 --- a/internal/querycoord/cluster.go +++ b/internal/querycoord/cluster.go @@ -21,6 +21,7 @@ import ( "encoding/json" "errors" "fmt" + "math/rand" "path/filepath" "strconv" "sync" @@ -75,8 +76,7 @@ type Cluster interface { allocateSegmentsToQueryNode(ctx context.Context, reqs []*querypb.LoadSegmentsRequest, wait bool, excludeNodeIDs []int64, includeNodeIDs []int64) error allocateChannelsToQueryNode(ctx context.Context, reqs []*querypb.WatchDmChannelsRequest, wait bool, excludeNodeIDs []int64) error - assignNodesToReplicas(ctx context.Context, reqs []*querypb.LoadSegmentsRequest, replicaIds []int64) error - assignSegmentsToReplica(ctx context.Context, reqs []*querypb.LoadSegmentsRequest, replicaID int64, wait bool) error + assignNodesToReplicas(ctx context.Context, replicas []*querypb.ReplicaInfo) error getSessionVersion() int64 @@ -704,12 +704,16 @@ func (c *queryNodeCluster) allocateChannelsToQueryNode(ctx context.Context, reqs return c.channelAllocator(ctx, reqs, c, c.clusterMeta, wait, excludeNodeIDs) } -func (c *queryNodeCluster) assignNodesToReplicas(ctx context.Context, reqs []*querypb.LoadSegmentsRequest, replicaIds []int64) error { - // todo(yah01) - return nil -} +func (c *queryNodeCluster) assignNodesToReplicas(ctx context.Context, replicas []*querypb.ReplicaInfo) error { + nodes := c.onlineNodeIDs() + if len(nodes) < len(replicas) { + return errors.New("no enough nodes to create replicas") + } + + for _, node := range nodes { + idx := rand.Int() % len(replicas) + replicas[idx].NodeIds = append(replicas[idx].NodeIds, node) + } -func (c *queryNodeCluster) assignSegmentsToReplica(ctx context.Context, reqs []*querypb.LoadSegmentsRequest, replicaID int64, wait bool) error { - // todo(yah01) return nil } diff --git a/internal/querycoord/meta.go b/internal/querycoord/meta.go index 9f8be6f6eb..0923d9794f 100644 --- a/internal/querycoord/meta.go +++ b/internal/querycoord/meta.go @@ -45,6 +45,7 @@ const ( collectionMetaPrefix = "queryCoord-collectionMeta" dmChannelMetaPrefix = "queryCoord-dmChannelWatchInfo" deltaChannelMetaPrefix = "queryCoord-deltaChannel" + ReplicaMetaPrefix = "queryCoord-ReplicaMeta" ) type col2SegmentInfos = map[UniqueID][]*querypb.SegmentInfo @@ -91,11 +92,12 @@ type Meta interface { getWatchedChannelsByNodeID(nodeID int64) *querypb.UnsubscribeChannelInfo - addReplica(replicaNum int32, collectionID int64, partitionIds []int64) error - setReplicaInfo(replicaID int64, info *querypb.ReplicaInfo) error + generateReplica(collectionID int64, partitionIds []int64) (*querypb.ReplicaInfo, error) + addReplica(replica *querypb.ReplicaInfo) error + setReplicaInfo(info *querypb.ReplicaInfo) error getReplicaByID(replicaID int64) (*querypb.ReplicaInfo, error) - getReplicasByNodeID(nodeID int64, collectionID int64) (*querypb.ReplicaInfo, error) getReplicasByCollectionID(collectionID int64) ([]*querypb.ReplicaInfo, error) + getReplicasByNodeID(nodeID int64) ([]*querypb.ReplicaInfo, error) } // MetaReplica records the current load information on all querynodes @@ -120,7 +122,7 @@ type MetaReplica struct { segmentsInfo *segmentsInfo //partitionStates map[UniqueID]*querypb.PartitionStates - // replicas *ReplicaInfos + replicas *ReplicaInfos } func newMeta(ctx context.Context, kv kv.MetaKv, factory msgstream.Factory, idAllocator func() (UniqueID, error)) (Meta, error) { @@ -145,7 +147,7 @@ func newMeta(ctx context.Context, kv kv.MetaKv, factory msgstream.Factory, idAll queryStreams: queryMsgStream, segmentsInfo: newSegmentsInfo(kv), - // replicas: NewReplicaInfos(), + replicas: NewReplicaInfos(), } err := m.reloadFromKV() @@ -301,6 +303,7 @@ func (m *MetaReplica) addCollection(collectionID UniqueID, loadType querypb.Load PartitionStates: partitionStates, LoadType: loadType, Schema: schema, + // ReplicaIds: replicas, } err := saveGlobalCollectionInfo(collectionID, newCollection, m.client) if err != nil { @@ -1022,55 +1025,72 @@ func (m *MetaReplica) getWatchedChannelsByNodeID(nodeID int64) *querypb.Unsubscr return unsubscribeChannelInfo } -func (m *MetaReplica) addReplica(replicaNum int32, collectionID int64, partitionIds []int64) error { - // collection, err := meta.getCollectionInfoByID(collectionId) - // if err != nil { - // return err - // } +func (m *MetaReplica) generateReplica(collectionID int64, partitionIds []int64) (*querypb.ReplicaInfo, error) { + id, err := m.idAllocator() + if err != nil { + return nil, err + } - // replicas := make([]*querypb.ReplicaInfo, 0, replicaNum) - - // for i := 0; i < int(replicaNum); i++ { - // replicaId, err := meta.idAllocator() - // if err != nil { - // return err - // } - - // replicas = append(replicas, &querypb.ReplicaInfo{ - // ReplicaId: replicaId, - // CollectionId: collectionId, - // PartitionIds: partitionIds, - // ShardReplicas: make([]*querypb.ShardReplica, 0), - // }) - // } - - // for _, replica := range replicas { - // meta.replicas.Insert(replica.ReplicaId, replica) - // } - - return nil - // todo(yah01) + return &querypb.ReplicaInfo{ + ReplicaId: id, + CollectionId: collectionID, + PartitionIds: partitionIds, + ShardReplicas: make([]*querypb.ShardReplica, 0), + NodeIds: make([]int64, 0), + }, nil } -func (m *MetaReplica) setReplicaInfo(replicaID int64, info *querypb.ReplicaInfo) error { +func (m *MetaReplica) addReplica(replica *querypb.ReplicaInfo) error { + err := saveReplicaInfo(replica, m.client) + if err != nil { + return err + } + + m.replicas.Insert(replica) + return nil +} + +func (m *MetaReplica) setReplicaInfo(info *querypb.ReplicaInfo) error { + err := saveReplicaInfo(info, m.client) + if err != nil { + return err + } + + m.replicas.Insert(info) return nil - // todo(yah01) } func (m *MetaReplica) getReplicaByID(replicaID int64) (*querypb.ReplicaInfo, error) { - return nil, nil - // todo(yah01) - // return meta.replicas.Get(replicaID) + replica, ok := m.replicas.Get(replicaID) + if !ok { + return nil, errors.New("replica not found") + } + + return replica, nil } -func (m *MetaReplica) getReplicasByNodeID(nodeID int64, collectionID int64) (*querypb.ReplicaInfo, error) { - return nil, nil - // todo(yah01) +func (m *MetaReplica) getReplicasByNodeID(nodeID int64) ([]*querypb.ReplicaInfo, error) { + replicas := m.replicas.GetReplicasByNodeID(nodeID) + return replicas, nil } func (m *MetaReplica) getReplicasByCollectionID(collectionID int64) ([]*querypb.ReplicaInfo, error) { - return nil, nil - // todo(sunby) + collection, err := m.getCollectionInfoByID(collectionID) + if err != nil { + return nil, err + } + + replicas := make([]*querypb.ReplicaInfo, 0, len(collection.ReplicaIds)) + for _, replicaID := range collection.ReplicaIds { + replica, err := m.getReplicaByID(replicaID) + if err != nil { + return nil, err + } + + replicas = append(replicas, replica) + } + + return replicas, nil } //func (m *MetaReplica) printMeta() { @@ -1127,6 +1147,16 @@ func saveDmChannelWatchInfos(infos []*querypb.DmChannelWatchInfo, kv kv.MetaKv) return kv.MultiSave(kvs) } +func saveReplicaInfo(info *querypb.ReplicaInfo, kv kv.MetaKv) error { + infoBytes, err := proto.Marshal(info) + if err != nil { + return err + } + + key := fmt.Sprintf("%s/%d", ReplicaMetaPrefix, info.ReplicaId) + return kv.Save(key, string(infoBytes)) +} + func removeCollectionMeta(collectionID UniqueID, kv kv.MetaKv) error { var prefixes []string collectionInfosPrefix := fmt.Sprintf("%s/%d", collectionMetaPrefix, collectionID) diff --git a/internal/querycoord/replica.go b/internal/querycoord/replica.go new file mode 100644 index 0000000000..fa43fb271a --- /dev/null +++ b/internal/querycoord/replica.go @@ -0,0 +1,91 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package querycoord + +import ( + "sync" + + "github.com/milvus-io/milvus/internal/proto/querypb" +) + +type replicaSlice = []*querypb.ReplicaInfo + +type ReplicaInfos struct { + globalGuard sync.RWMutex // We have to make sure atomically update replicas and index + + // Persistent Info + replicas map[UniqueID]*querypb.ReplicaInfo // replica_id -> *ReplicaInfo + + // Non-persistent info + nodeIndex map[UniqueID][]*querypb.ReplicaInfo // node_id -> []*ReplicaInfo +} + +func NewReplicaInfos() *ReplicaInfos { + return &ReplicaInfos{ + globalGuard: sync.RWMutex{}, + replicas: make(map[int64]*querypb.ReplicaInfo), + nodeIndex: make(map[int64][]*querypb.ReplicaInfo), + } +} + +func (rep *ReplicaInfos) Get(replicaID UniqueID) (*querypb.ReplicaInfo, bool) { + rep.globalGuard.RLock() + defer rep.globalGuard.RUnlock() + + info, ok := rep.replicas[replicaID] + return info, ok +} + +// Make sure atomically update replica and index +func (rep *ReplicaInfos) Insert(info *querypb.ReplicaInfo) { + rep.globalGuard.Lock() + defer rep.globalGuard.Unlock() + + old, ok := rep.replicas[info.ReplicaId] + // This updates ReplicaInfo, not inserts a new one + // No need to update nodeIndex + if ok { + *old = *info + return + } + + rep.replicas[info.ReplicaId] = info + + for _, nodeID := range info.NodeIds { + replicas, ok := rep.nodeIndex[nodeID] + if !ok { + replicas = make([]*querypb.ReplicaInfo, 0) + rep.nodeIndex[nodeID] = replicas + } + + replicas = append(replicas, info) + rep.nodeIndex[nodeID] = replicas + } +} + +func (rep *ReplicaInfos) GetReplicasByNodeID(nodeID UniqueID) []*querypb.ReplicaInfo { + rep.globalGuard.RLock() + defer rep.globalGuard.RUnlock() + + // Avoid to create entry if nodeID not found + replicas, ok := rep.nodeIndex[nodeID] + if !ok { + return nil + } + + return replicas +}