From e4c51aae366e96fc58148cfb0d30ae35c165f28b Mon Sep 17 00:00:00 2001 From: xige-16 Date: Sat, 19 Jun 2021 11:45:09 +0800 Subject: [PATCH] Add recovery logic for querynode and queryservice (#5843) * merge milvus/recovery2 Signed-off-by: xige-16 * add recovery logic in queryservice Signed-off-by: xige-16 * debug smoke case Signed-off-by: xige-16 * add etcd to querynode Signed-off-by: xige-16 * fix release partition error Signed-off-by: xige-16 * fix load balance error Signed-off-by: xige-16 * debug querynode down and recovery Signed-off-by: xige-16 * add log Signed-off-by: xige-16 * fix showCollection Signed-off-by: xige-16 * skip smoke test search without insert Signed-off-by: xige-16 --- internal/distributed/querynode/service.go | 8 +- internal/msgstream/msg.go | 9 +- internal/proto/internal.proto | 5 + internal/proto/internalpb/internal.pb.go | 298 +++--- internal/proto/query_service.proto | 14 +- internal/proto/querypb/query_service.pb.go | 319 +++--- internal/querynode/collection_replica.go | 11 +- .../querynode/flow_graph_service_time_node.go | 2 +- internal/querynode/historical.go | 11 +- internal/querynode/query_node.go | 111 ++- internal/querynode/query_node_test.go | 12 +- internal/querynode/search_collection.go | 61 +- internal/querynode/segment_loader.go | 41 +- internal/querynode/streaming.go | 5 +- internal/querynode/tsafe.go | 12 +- internal/queryservice/cluster.go | 233 ++++- internal/queryservice/impl.go | 161 ++- internal/queryservice/meta.go | 269 ++++- internal/queryservice/meta_test.go | 5 +- internal/queryservice/param_table.go | 14 + internal/queryservice/querynode.go | 133 ++- internal/queryservice/queryservice.go | 192 +++- internal/queryservice/task.go | 936 +++++++++++------- internal/queryservice/task_scheduler.go | 574 +++++++++-- .../collection/test_load_collection.py | 1 + tests/python_test/entity/test_insert.py | 1 + tests/python_test/entity/test_search.py | 1 + 27 files changed, 2495 insertions(+), 944 deletions(-) diff --git a/internal/distributed/querynode/service.go b/internal/distributed/querynode/service.go index 16d03c6ccb..964518f729 100644 --- a/internal/distributed/querynode/service.go +++ b/internal/distributed/querynode/service.go @@ -89,10 +89,6 @@ func (s *Server) init() error { closer := trace.InitTracing(fmt.Sprintf("query_node ip: %s, port: %d", Params.QueryNodeIP, Params.QueryNodePort)) s.closer = closer - if err := s.querynode.Register(); err != nil { - return err - } - log.Debug("QueryNode", zap.Int("port", Params.QueryNodePort)) s.wg.Add(1) go s.startGrpcLoop(Params.QueryNodePort) @@ -101,6 +97,10 @@ func (s *Server) init() error { if err != nil { return err } + + if err := s.querynode.Register(); err != nil { + return err + } // --- QueryService --- log.Debug("QueryNode start to new QueryServiceClient", zap.Any("QueryServiceAddress", Params.QueryServiceAddress)) queryService, err := qsc.NewClient(qn.Params.MetaRootPath, qn.Params.EtcdEndpoints, 3*time.Second) diff --git a/internal/msgstream/msg.go b/internal/msgstream/msg.go index 7e15bbd539..141cfe0736 100644 --- a/internal/msgstream/msg.go +++ b/internal/msgstream/msg.go @@ -20,7 +20,6 @@ import ( "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/proto/internalpb" - "github.com/milvus-io/milvus/internal/proto/querypb" ) type MsgType = commonpb.MsgType @@ -888,7 +887,7 @@ func (sim *SegmentInfoMsg) Unmarshal(input MarshalType) (TsMsg, error) { /////////////////////////////////////////LoadBalanceSegments////////////////////////////////////////// type LoadBalanceSegmentsMsg struct { BaseMsg - querypb.LoadBalanceSegments + internalpb.LoadBalanceSegmentsRequest } func (l *LoadBalanceSegmentsMsg) TraceCtx() context.Context { @@ -909,7 +908,7 @@ func (l *LoadBalanceSegmentsMsg) Type() MsgType { func (l *LoadBalanceSegmentsMsg) Marshal(input TsMsg) (MarshalType, error) { load := input.(*LoadBalanceSegmentsMsg) - loadReq := &load.LoadBalanceSegments + loadReq := &load.LoadBalanceSegmentsRequest mb, err := proto.Marshal(loadReq) if err != nil { return nil, err @@ -918,7 +917,7 @@ func (l *LoadBalanceSegmentsMsg) Marshal(input TsMsg) (MarshalType, error) { } func (l *LoadBalanceSegmentsMsg) Unmarshal(input MarshalType) (TsMsg, error) { - loadReq := querypb.LoadBalanceSegments{} + loadReq := internalpb.LoadBalanceSegmentsRequest{} in, err := ConvertToByteArray(input) if err != nil { return nil, err @@ -927,7 +926,7 @@ func (l *LoadBalanceSegmentsMsg) Unmarshal(input MarshalType) (TsMsg, error) { if err != nil { return nil, err } - loadMsg := &LoadBalanceSegmentsMsg{LoadBalanceSegments: loadReq} + loadMsg := &LoadBalanceSegmentsMsg{LoadBalanceSegmentsRequest: loadReq} loadMsg.BeginTimestamp = loadReq.Base.Timestamp loadMsg.EndTimestamp = loadReq.Base.Timestamp diff --git a/internal/proto/internal.proto b/internal/proto/internal.proto index 0c9ef8880b..762f0a0598 100644 --- a/internal/proto/internal.proto +++ b/internal/proto/internal.proto @@ -174,6 +174,11 @@ message DeleteRequest { repeated int64 primary_keys = 5; } +message LoadBalanceSegmentsRequest { + common.MsgBase base = 1; + repeated int64 segmentIDs = 2; +} + message LoadIndex { common.MsgBase base = 1; int64 segmentID = 2; diff --git a/internal/proto/internalpb/internal.pb.go b/internal/proto/internalpb/internal.pb.go index 32b639ecda..92136ba2b9 100644 --- a/internal/proto/internalpb/internal.pb.go +++ b/internal/proto/internalpb/internal.pb.go @@ -1487,6 +1487,53 @@ func (m *DeleteRequest) GetPrimaryKeys() []int64 { return nil } +type LoadBalanceSegmentsRequest struct { + Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` + SegmentIDs []int64 `protobuf:"varint,2,rep,packed,name=segmentIDs,proto3" json:"segmentIDs,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *LoadBalanceSegmentsRequest) Reset() { *m = LoadBalanceSegmentsRequest{} } +func (m *LoadBalanceSegmentsRequest) String() string { return proto.CompactTextString(m) } +func (*LoadBalanceSegmentsRequest) ProtoMessage() {} +func (*LoadBalanceSegmentsRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_41f4a519b878ee3b, []int{21} +} + +func (m *LoadBalanceSegmentsRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_LoadBalanceSegmentsRequest.Unmarshal(m, b) +} +func (m *LoadBalanceSegmentsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_LoadBalanceSegmentsRequest.Marshal(b, m, deterministic) +} +func (m *LoadBalanceSegmentsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_LoadBalanceSegmentsRequest.Merge(m, src) +} +func (m *LoadBalanceSegmentsRequest) XXX_Size() int { + return xxx_messageInfo_LoadBalanceSegmentsRequest.Size(m) +} +func (m *LoadBalanceSegmentsRequest) XXX_DiscardUnknown() { + xxx_messageInfo_LoadBalanceSegmentsRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_LoadBalanceSegmentsRequest proto.InternalMessageInfo + +func (m *LoadBalanceSegmentsRequest) GetBase() *commonpb.MsgBase { + if m != nil { + return m.Base + } + return nil +} + +func (m *LoadBalanceSegmentsRequest) GetSegmentIDs() []int64 { + if m != nil { + return m.SegmentIDs + } + return nil +} + type LoadIndex struct { Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` SegmentID int64 `protobuf:"varint,2,opt,name=segmentID,proto3" json:"segmentID,omitempty"` @@ -1503,7 +1550,7 @@ func (m *LoadIndex) Reset() { *m = LoadIndex{} } func (m *LoadIndex) String() string { return proto.CompactTextString(m) } func (*LoadIndex) ProtoMessage() {} func (*LoadIndex) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{21} + return fileDescriptor_41f4a519b878ee3b, []int{22} } func (m *LoadIndex) XXX_Unmarshal(b []byte) error { @@ -1583,7 +1630,7 @@ func (m *SegmentStatisticsUpdates) Reset() { *m = SegmentStatisticsUpdat func (m *SegmentStatisticsUpdates) String() string { return proto.CompactTextString(m) } func (*SegmentStatisticsUpdates) ProtoMessage() {} func (*SegmentStatisticsUpdates) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{22} + return fileDescriptor_41f4a519b878ee3b, []int{23} } func (m *SegmentStatisticsUpdates) XXX_Unmarshal(b []byte) error { @@ -1665,7 +1712,7 @@ func (m *SegmentStatistics) Reset() { *m = SegmentStatistics{} } func (m *SegmentStatistics) String() string { return proto.CompactTextString(m) } func (*SegmentStatistics) ProtoMessage() {} func (*SegmentStatistics) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{23} + return fileDescriptor_41f4a519b878ee3b, []int{24} } func (m *SegmentStatistics) XXX_Unmarshal(b []byte) error { @@ -1712,7 +1759,7 @@ func (m *SegmentFlushCompletedMsg) Reset() { *m = SegmentFlushCompletedM func (m *SegmentFlushCompletedMsg) String() string { return proto.CompactTextString(m) } func (*SegmentFlushCompletedMsg) ProtoMessage() {} func (*SegmentFlushCompletedMsg) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{24} + return fileDescriptor_41f4a519b878ee3b, []int{25} } func (m *SegmentFlushCompletedMsg) XXX_Unmarshal(b []byte) error { @@ -1759,7 +1806,7 @@ func (m *IndexStats) Reset() { *m = IndexStats{} } func (m *IndexStats) String() string { return proto.CompactTextString(m) } func (*IndexStats) ProtoMessage() {} func (*IndexStats) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{25} + return fileDescriptor_41f4a519b878ee3b, []int{26} } func (m *IndexStats) XXX_Unmarshal(b []byte) error { @@ -1807,7 +1854,7 @@ func (m *FieldStats) Reset() { *m = FieldStats{} } func (m *FieldStats) String() string { return proto.CompactTextString(m) } func (*FieldStats) ProtoMessage() {} func (*FieldStats) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{26} + return fileDescriptor_41f4a519b878ee3b, []int{27} } func (m *FieldStats) XXX_Unmarshal(b []byte) error { @@ -1863,7 +1910,7 @@ func (m *SegmentStats) Reset() { *m = SegmentStats{} } func (m *SegmentStats) String() string { return proto.CompactTextString(m) } func (*SegmentStats) ProtoMessage() {} func (*SegmentStats) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{27} + return fileDescriptor_41f4a519b878ee3b, []int{28} } func (m *SegmentStats) XXX_Unmarshal(b []byte) error { @@ -1925,7 +1972,7 @@ func (m *QueryNodeStats) Reset() { *m = QueryNodeStats{} } func (m *QueryNodeStats) String() string { return proto.CompactTextString(m) } func (*QueryNodeStats) ProtoMessage() {} func (*QueryNodeStats) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{28} + return fileDescriptor_41f4a519b878ee3b, []int{29} } func (m *QueryNodeStats) XXX_Unmarshal(b []byte) error { @@ -1981,7 +2028,7 @@ func (m *MsgPosition) Reset() { *m = MsgPosition{} } func (m *MsgPosition) String() string { return proto.CompactTextString(m) } func (*MsgPosition) ProtoMessage() {} func (*MsgPosition) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{29} + return fileDescriptor_41f4a519b878ee3b, []int{30} } func (m *MsgPosition) XXX_Unmarshal(b []byte) error { @@ -2044,7 +2091,7 @@ func (m *ChannelTimeTickMsg) Reset() { *m = ChannelTimeTickMsg{} } func (m *ChannelTimeTickMsg) String() string { return proto.CompactTextString(m) } func (*ChannelTimeTickMsg) ProtoMessage() {} func (*ChannelTimeTickMsg) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{30} + return fileDescriptor_41f4a519b878ee3b, []int{31} } func (m *ChannelTimeTickMsg) XXX_Unmarshal(b []byte) error { @@ -2116,6 +2163,7 @@ func init() { proto.RegisterType((*RetrieveRequest)(nil), "milvus.proto.internal.RetrieveRequest") proto.RegisterType((*RetrieveResults)(nil), "milvus.proto.internal.RetrieveResults") proto.RegisterType((*DeleteRequest)(nil), "milvus.proto.internal.DeleteRequest") + proto.RegisterType((*LoadBalanceSegmentsRequest)(nil), "milvus.proto.internal.LoadBalanceSegmentsRequest") proto.RegisterType((*LoadIndex)(nil), "milvus.proto.internal.LoadIndex") proto.RegisterType((*SegmentStatisticsUpdates)(nil), "milvus.proto.internal.SegmentStatisticsUpdates") proto.RegisterType((*SegmentStatistics)(nil), "milvus.proto.internal.SegmentStatistics") @@ -2131,118 +2179,120 @@ func init() { func init() { proto.RegisterFile("internal.proto", fileDescriptor_41f4a519b878ee3b) } var fileDescriptor_41f4a519b878ee3b = []byte{ - // 1807 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x58, 0xcd, 0x93, 0x1b, 0x47, - 0x15, 0x67, 0x24, 0xed, 0x4a, 0x7a, 0x9a, 0x5d, 0xcb, 0xed, 0xb5, 0x33, 0x6b, 0x3b, 0x89, 0x32, - 0xe1, 0x63, 0x89, 0x0b, 0xaf, 0xd9, 0x00, 0x49, 0x51, 0x14, 0x4e, 0xbc, 0x4a, 0x8c, 0xca, 0x59, - 0xd7, 0x32, 0x72, 0x52, 0x05, 0x97, 0xa9, 0x96, 0xa6, 0x57, 0x9a, 0x64, 0xbe, 0xe8, 0x6e, 0xd9, - 0x56, 0x4e, 0x1c, 0xb8, 0x51, 0x70, 0xa0, 0x8a, 0x13, 0xff, 0x03, 0x57, 0x4e, 0x7c, 0x54, 0x4e, - 0x54, 0xf1, 0x17, 0xf0, 0x6f, 0x70, 0xe4, 0x02, 0xd5, 0xaf, 0x7b, 0x3e, 0xa4, 0x9d, 0x5d, 0x36, - 0xeb, 0xa2, 0x20, 0x05, 0xb7, 0xe9, 0xf7, 0x5e, 0x7f, 0xfc, 0x7e, 0xef, 0xf5, 0x7b, 0x6f, 0x1a, - 0xb6, 0xc3, 0x44, 0x32, 0x9e, 0xd0, 0xe8, 0x6e, 0xc6, 0x53, 0x99, 0x92, 0xeb, 0x71, 0x18, 0x3d, - 0x5d, 0x08, 0x3d, 0xba, 0x9b, 0x2b, 0x6f, 0xda, 0xd3, 0x34, 0x8e, 0xd3, 0x44, 0x8b, 0x6f, 0xda, - 0x62, 0x3a, 0x67, 0x31, 0xd5, 0x23, 0xf7, 0x0f, 0x16, 0x6c, 0x1d, 0xa6, 0x71, 0x96, 0x26, 0x2c, - 0x91, 0xa3, 0xe4, 0x24, 0x25, 0x37, 0x60, 0x33, 0x49, 0x03, 0x36, 0x1a, 0x3a, 0xd6, 0xc0, 0xda, - 0x6b, 0x7a, 0x66, 0x44, 0x08, 0xb4, 0x78, 0x1a, 0x31, 0xa7, 0x31, 0xb0, 0xf6, 0xba, 0x1e, 0x7e, - 0x93, 0xfb, 0x00, 0x42, 0x52, 0xc9, 0xfc, 0x69, 0x1a, 0x30, 0xa7, 0x39, 0xb0, 0xf6, 0xb6, 0x0f, - 0x06, 0x77, 0x6b, 0x4f, 0x71, 0x77, 0xac, 0x0c, 0x0f, 0xd3, 0x80, 0x79, 0x5d, 0x91, 0x7f, 0x92, - 0x77, 0x00, 0xd8, 0x73, 0xc9, 0xa9, 0x1f, 0x26, 0x27, 0xa9, 0xd3, 0x1a, 0x34, 0xf7, 0x7a, 0x07, - 0xaf, 0xad, 0x2e, 0x60, 0x0e, 0xff, 0x88, 0x2d, 0x3f, 0xa2, 0xd1, 0x82, 0x1d, 0xd3, 0x90, 0x7b, - 0x5d, 0x9c, 0xa4, 0x8e, 0xeb, 0xfe, 0xd5, 0x82, 0x2b, 0x05, 0x00, 0xdc, 0x43, 0x90, 0xef, 0xc2, - 0x06, 0x6e, 0x81, 0x08, 0x7a, 0x07, 0x5f, 0x3e, 0xe3, 0x44, 0x2b, 0xb8, 0x3d, 0x3d, 0x85, 0x7c, - 0x08, 0xd7, 0xc4, 0x62, 0x32, 0xcd, 0x55, 0x3e, 0x4a, 0x85, 0xd3, 0xc0, 0xa3, 0x5d, 0x6c, 0x25, - 0x52, 0x5d, 0xc0, 0x1c, 0xe9, 0x4d, 0xd8, 0x54, 0x2b, 0x2d, 0x04, 0xb2, 0xd4, 0x3b, 0xb8, 0x55, - 0x0b, 0x72, 0x8c, 0x26, 0x9e, 0x31, 0x75, 0x6f, 0xc1, 0xee, 0x43, 0x26, 0xd7, 0xd0, 0x79, 0xec, - 0x27, 0x0b, 0x26, 0xa4, 0x51, 0x3e, 0x09, 0x63, 0xf6, 0x24, 0x9c, 0x7e, 0x72, 0x38, 0xa7, 0x49, - 0xc2, 0xa2, 0x5c, 0xf9, 0x32, 0xdc, 0x7a, 0xc8, 0x70, 0x42, 0x28, 0x64, 0x38, 0x15, 0x6b, 0xea, - 0xeb, 0x70, 0xed, 0x21, 0x93, 0xc3, 0x60, 0x4d, 0xfc, 0x11, 0x74, 0x1e, 0x2b, 0x67, 0xab, 0x30, - 0xf8, 0x0e, 0xb4, 0x69, 0x10, 0x70, 0x26, 0x84, 0x61, 0xf1, 0x76, 0xed, 0x89, 0xdf, 0xd5, 0x36, - 0x5e, 0x6e, 0x5c, 0x17, 0x26, 0xee, 0xc7, 0x00, 0xa3, 0x24, 0x94, 0xc7, 0x94, 0xd3, 0x58, 0x9c, - 0x19, 0x60, 0x43, 0xb0, 0x85, 0xa4, 0x5c, 0xfa, 0x19, 0xda, 0x19, 0xca, 0x2f, 0x10, 0x0d, 0x3d, - 0x9c, 0xa6, 0x57, 0x77, 0x7f, 0x04, 0x30, 0x96, 0x3c, 0x4c, 0x66, 0x1f, 0x84, 0x42, 0xaa, 0xbd, - 0x9e, 0x2a, 0x3b, 0x05, 0xa2, 0xb9, 0xd7, 0xf5, 0xcc, 0xa8, 0xe2, 0x8e, 0xc6, 0xc5, 0xdd, 0x71, - 0x1f, 0x7a, 0x39, 0xdd, 0x47, 0x62, 0x46, 0xee, 0x41, 0x6b, 0x42, 0x05, 0x3b, 0x97, 0x9e, 0x23, - 0x31, 0x7b, 0x40, 0x05, 0xf3, 0xd0, 0xd2, 0xfd, 0xac, 0x01, 0x2f, 0x1d, 0x72, 0x86, 0xc1, 0x1f, - 0x45, 0x6c, 0x2a, 0xc3, 0x34, 0x31, 0xdc, 0x7f, 0xfe, 0xd5, 0xc8, 0x4b, 0xd0, 0x0e, 0x26, 0x7e, - 0x42, 0xe3, 0x9c, 0xec, 0xcd, 0x60, 0xf2, 0x98, 0xc6, 0x8c, 0x7c, 0x15, 0xb6, 0xa7, 0xc5, 0xfa, - 0x4a, 0x82, 0x31, 0xd7, 0xf5, 0xd6, 0xa4, 0xca, 0x55, 0xc1, 0x64, 0x34, 0x74, 0x5a, 0xe8, 0x06, - 0xfc, 0x26, 0x2e, 0xd8, 0xa5, 0xd5, 0x68, 0xe8, 0x6c, 0xa0, 0x6e, 0x45, 0xa6, 0x48, 0xd5, 0x39, - 0xc4, 0xd9, 0x1c, 0x58, 0x7b, 0xb6, 0x67, 0x46, 0xe4, 0x1e, 0x5c, 0x7b, 0x1a, 0x72, 0xb9, 0xa0, - 0x91, 0x89, 0x2b, 0xb5, 0x8b, 0x70, 0xda, 0xc8, 0x7c, 0x9d, 0x8a, 0x1c, 0xc0, 0x4e, 0x36, 0x5f, - 0x8a, 0x70, 0xba, 0x36, 0xa5, 0x83, 0x53, 0x6a, 0x75, 0xee, 0x67, 0x16, 0x5c, 0x1f, 0xf2, 0x34, - 0xfb, 0x22, 0x53, 0xe8, 0xfe, 0xa2, 0x01, 0x37, 0x74, 0x24, 0x1c, 0x53, 0x2e, 0xc3, 0x7f, 0x13, - 0x8a, 0xaf, 0xc1, 0x95, 0x72, 0x57, 0x6d, 0x50, 0x0f, 0xe3, 0x2b, 0xb0, 0x9d, 0xe5, 0xe7, 0xd0, - 0x76, 0x2d, 0xb4, 0xdb, 0x2a, 0xa4, 0x2b, 0x68, 0x37, 0xce, 0x41, 0xbb, 0x59, 0x13, 0x30, 0x03, - 0xe8, 0x15, 0x0b, 0x8d, 0x86, 0x4e, 0x1b, 0x4d, 0xaa, 0x22, 0xf7, 0xe7, 0x0d, 0xd8, 0x51, 0x4e, - 0xfd, 0x3f, 0x1b, 0x8a, 0x8d, 0x3f, 0x36, 0x80, 0xe8, 0xe8, 0x18, 0x25, 0x01, 0x7b, 0xfe, 0x9f, - 0xe4, 0xe2, 0x65, 0x80, 0x93, 0x90, 0x45, 0x41, 0x95, 0x87, 0x2e, 0x4a, 0x5e, 0x88, 0x03, 0x07, - 0xda, 0xb8, 0x48, 0x81, 0x3f, 0x1f, 0xaa, 0x2a, 0xa0, 0x3b, 0x02, 0x53, 0x05, 0x3a, 0x17, 0xae, - 0x02, 0x38, 0xcd, 0x54, 0x81, 0xdf, 0x36, 0x61, 0x6b, 0x94, 0x08, 0xc6, 0xe5, 0xff, 0x72, 0x20, - 0x91, 0xdb, 0xd0, 0x15, 0x6c, 0x16, 0xab, 0xc6, 0x64, 0xe8, 0x74, 0x50, 0x5f, 0x0a, 0x94, 0x76, - 0xaa, 0x33, 0xeb, 0x68, 0xe8, 0x74, 0xb5, 0x6b, 0x0b, 0x01, 0x79, 0x05, 0x40, 0x86, 0x31, 0x13, - 0x92, 0xc6, 0x99, 0x70, 0x60, 0xd0, 0xdc, 0x6b, 0x79, 0x15, 0x89, 0xaa, 0x02, 0x3c, 0x7d, 0x36, - 0x1a, 0x0a, 0xa7, 0x37, 0x68, 0xaa, 0x32, 0xae, 0x47, 0xe4, 0x5b, 0xd0, 0xe1, 0xe9, 0x33, 0x3f, - 0xa0, 0x92, 0x3a, 0x36, 0x3a, 0x6f, 0xb7, 0x96, 0xec, 0x07, 0x51, 0x3a, 0xf1, 0xda, 0x3c, 0x7d, - 0x36, 0xa4, 0x92, 0xba, 0x7f, 0x6b, 0xc0, 0xd6, 0x98, 0x51, 0x3e, 0x9d, 0x5f, 0xde, 0x61, 0x5f, - 0x87, 0x3e, 0x67, 0x62, 0x11, 0x49, 0xbf, 0x84, 0xa5, 0x3d, 0x77, 0x45, 0xcb, 0x0f, 0x0b, 0x70, - 0x39, 0xe5, 0xcd, 0x73, 0x28, 0x6f, 0xd5, 0x50, 0xee, 0x82, 0x5d, 0xe1, 0x57, 0x38, 0x1b, 0x08, - 0x7d, 0x45, 0x46, 0xfa, 0xd0, 0x0c, 0x44, 0x84, 0x1e, 0xeb, 0x7a, 0xea, 0x93, 0xdc, 0x81, 0xab, - 0x59, 0x44, 0xa7, 0x6c, 0x9e, 0x46, 0x01, 0xe3, 0xfe, 0x8c, 0xa7, 0x8b, 0x0c, 0xdd, 0x65, 0x7b, - 0xfd, 0x8a, 0xe2, 0xa1, 0x92, 0x93, 0xb7, 0xa0, 0x13, 0x88, 0xc8, 0x97, 0xcb, 0x8c, 0xa1, 0xcb, - 0xb6, 0xcf, 0xc0, 0x3e, 0x14, 0xd1, 0x93, 0x65, 0xc6, 0xbc, 0x76, 0xa0, 0x3f, 0xc8, 0x3d, 0xd8, - 0x11, 0x8c, 0x87, 0x34, 0x0a, 0x3f, 0x65, 0x81, 0xcf, 0x9e, 0x67, 0xdc, 0xcf, 0x22, 0x9a, 0xa0, - 0x67, 0x6d, 0x8f, 0x94, 0xba, 0xf7, 0x9e, 0x67, 0xfc, 0x38, 0xa2, 0x89, 0xfb, 0x8f, 0x0a, 0xe9, - 0x8a, 0x1f, 0x71, 0x09, 0xd2, 0x2f, 0xd3, 0x49, 0xd5, 0x7a, 0xaa, 0x59, 0xef, 0xa9, 0x57, 0xa1, - 0x17, 0x33, 0xc9, 0xc3, 0xa9, 0x66, 0x44, 0x5f, 0x20, 0xd0, 0x22, 0x84, 0x4d, 0xa0, 0x35, 0x0f, - 0xa5, 0x76, 0x85, 0xed, 0xe1, 0x37, 0xf9, 0x1e, 0xdc, 0x14, 0x8c, 0x46, 0x2c, 0xf0, 0x8b, 0x68, - 0x17, 0xbe, 0x40, 0xa4, 0x2c, 0x70, 0x36, 0xd1, 0x69, 0x8e, 0xb6, 0x18, 0x17, 0x06, 0x63, 0xa3, - 0x27, 0xfb, 0x70, 0xad, 0x38, 0x56, 0x65, 0x9a, 0xee, 0x63, 0x48, 0xa9, 0x2a, 0x26, 0xbc, 0x0d, - 0xce, 0x2c, 0x4a, 0x27, 0x34, 0xf2, 0x4f, 0xed, 0x8a, 0xf9, 0xab, 0xe9, 0xdd, 0xd0, 0xfa, 0xf1, - 0xda, 0x96, 0xee, 0x6f, 0x1a, 0x70, 0xc5, 0x53, 0x58, 0xd8, 0x53, 0xf6, 0x85, 0x0f, 0xfc, 0x37, - 0xa0, 0x19, 0x06, 0x02, 0x03, 0xbf, 0x77, 0xe0, 0xac, 0x9e, 0xdb, 0xfc, 0x74, 0x8e, 0x86, 0xc2, - 0x53, 0x46, 0xe4, 0x75, 0xd8, 0x4a, 0x17, 0x32, 0x5b, 0x48, 0x1f, 0x13, 0x7f, 0xde, 0x25, 0xda, - 0x5a, 0xf8, 0x3e, 0xca, 0xdc, 0xdf, 0x37, 0xab, 0xec, 0xfc, 0xb7, 0x46, 0xa8, 0x81, 0xdd, 0xba, - 0x08, 0xec, 0xfb, 0xd0, 0xd3, 0x78, 0x75, 0x7e, 0xdc, 0xc0, 0xfc, 0xf8, 0x4a, 0xed, 0x1c, 0xe4, - 0x40, 0xe5, 0x46, 0x4f, 0x57, 0x60, 0xa1, 0xbe, 0xc9, 0xf7, 0xe1, 0xd6, 0xe9, 0xc8, 0xe6, 0x86, - 0xa3, 0x3c, 0xb4, 0x77, 0xd7, 0x43, 0x3b, 0x27, 0x31, 0x20, 0xdf, 0x84, 0x9d, 0x4a, 0x6c, 0x97, - 0x13, 0x4d, 0x93, 0x5e, 0xea, 0xca, 0x29, 0x97, 0x8f, 0xee, 0xbf, 0x58, 0xb0, 0x35, 0x64, 0x11, - 0x93, 0x2f, 0x10, 0xdb, 0x35, 0xc5, 0xb6, 0x51, 0x5b, 0x6c, 0x57, 0xaa, 0x59, 0xf3, 0xfc, 0x6a, - 0xd6, 0x3a, 0x55, 0xcd, 0x5e, 0x03, 0x3b, 0xe3, 0x61, 0x4c, 0xf9, 0xd2, 0xff, 0x84, 0x2d, 0xf3, - 0xf8, 0xee, 0x19, 0xd9, 0x23, 0xb6, 0x14, 0xee, 0xdf, 0x2d, 0xe8, 0x7e, 0x90, 0xd2, 0x00, 0x7b, - 0xb2, 0x4b, 0x20, 0x59, 0x29, 0xc6, 0x8d, 0x9a, 0x62, 0x5c, 0xb4, 0x55, 0xf9, 0xf1, 0xcb, 0x3e, - 0xab, 0xd2, 0x2f, 0xb5, 0x56, 0xfb, 0xa5, 0x57, 0xa1, 0x17, 0xaa, 0x03, 0xf9, 0x19, 0x95, 0x73, - 0x7d, 0xee, 0xae, 0x07, 0x28, 0x3a, 0x56, 0x12, 0xd5, 0x50, 0xe5, 0x06, 0xd8, 0x50, 0x6d, 0x5e, - 0xb8, 0xa1, 0x32, 0x8b, 0x60, 0x43, 0xf5, 0xa7, 0x06, 0x38, 0xc6, 0xb3, 0xe5, 0xab, 0xc2, 0x87, - 0x59, 0x80, 0x8f, 0x1b, 0xb7, 0xa1, 0x5b, 0x78, 0xdd, 0xfc, 0xd4, 0x97, 0x02, 0x45, 0xfd, 0x11, - 0x8b, 0x53, 0xbe, 0x1c, 0x87, 0x9f, 0x32, 0x03, 0xbc, 0x22, 0x51, 0xd8, 0x1e, 0x2f, 0x62, 0x2f, - 0x7d, 0x26, 0x4c, 0x56, 0xca, 0x87, 0x0a, 0xdb, 0x14, 0xdb, 0x60, 0x5f, 0x79, 0x0a, 0x91, 0xb7, - 0x3c, 0xd0, 0x22, 0xf5, 0x27, 0x4e, 0x76, 0xa1, 0xc3, 0x92, 0x40, 0x6b, 0x37, 0x50, 0xdb, 0x66, - 0x49, 0x80, 0xaa, 0x11, 0x6c, 0x9b, 0xd7, 0x84, 0x54, 0x60, 0x86, 0x32, 0x79, 0xc9, 0x3d, 0xe3, - 0x09, 0xe7, 0x48, 0xcc, 0x8e, 0x8d, 0xa5, 0xb7, 0xa5, 0x1f, 0x14, 0xcc, 0x90, 0xbc, 0x07, 0xb6, - 0xda, 0xa5, 0x58, 0xa8, 0x7d, 0xe1, 0x85, 0x7a, 0x2c, 0x09, 0xf2, 0x81, 0xfb, 0x2b, 0x0b, 0xae, - 0x9e, 0xa2, 0xf0, 0x12, 0x71, 0xf4, 0x08, 0x3a, 0x63, 0x36, 0x53, 0x4b, 0xe4, 0x6f, 0x24, 0xfb, - 0x67, 0x3d, 0xb9, 0x9d, 0xe1, 0x30, 0xaf, 0x58, 0xc0, 0xfd, 0xb8, 0x70, 0xeb, 0xfb, 0xd1, 0x42, - 0xcc, 0x0f, 0xd3, 0x38, 0x53, 0xf7, 0x35, 0xb8, 0xd4, 0x03, 0xc7, 0xf9, 0x21, 0xee, 0xfe, 0xcc, - 0x02, 0xc0, 0xcb, 0x83, 0x5b, 0x9f, 0x0a, 0x4c, 0xeb, 0x32, 0x81, 0xa9, 0xba, 0x9e, 0x64, 0x11, - 0xfb, 0x9c, 0x45, 0x54, 0x96, 0xb9, 0x49, 0x98, 0xdd, 0x49, 0xb2, 0x88, 0x3d, 0xad, 0x32, 0x30, - 0x85, 0xfb, 0x4b, 0x0b, 0x00, 0x93, 0xab, 0x3e, 0xc6, 0x7a, 0xf5, 0xb3, 0xce, 0xff, 0x5d, 0x69, - 0xac, 0x5e, 0xbf, 0x07, 0xf9, 0xf5, 0x13, 0xe8, 0x8f, 0x66, 0x1d, 0x86, 0xc2, 0x1f, 0x25, 0x78, - 0x73, 0x43, 0xb5, 0x0f, 0x7e, 0x6d, 0x81, 0x5d, 0x71, 0x95, 0x58, 0xa5, 0xd1, 0x5a, 0xcf, 0x14, - 0xd8, 0x11, 0xa9, 0xdb, 0xe3, 0x8b, 0xca, 0x85, 0x8a, 0xcb, 0x0b, 0xb5, 0x0b, 0x1d, 0xa4, 0xa4, - 0x72, 0xa3, 0x12, 0x73, 0xa3, 0xee, 0xc0, 0x55, 0xce, 0xa6, 0x2c, 0x91, 0xd1, 0xd2, 0x8f, 0xd3, - 0x20, 0x3c, 0x09, 0x59, 0x80, 0xf7, 0xaa, 0xe3, 0xf5, 0x73, 0xc5, 0x91, 0x91, 0xbb, 0x7f, 0xb6, - 0x60, 0xfb, 0x87, 0x0b, 0xc6, 0x97, 0x8f, 0xd3, 0x80, 0xe9, 0x93, 0x7d, 0xfe, 0x90, 0x78, 0x07, - 0xb1, 0x18, 0x7a, 0x74, 0xb8, 0xbe, 0xfe, 0xaf, 0xc3, 0x55, 0x78, 0x1d, 0x61, 0x42, 0x54, 0x51, - 0xac, 0x7f, 0x41, 0x2f, 0x42, 0x71, 0xe9, 0x58, 0x53, 0x36, 0x35, 0xc5, 0x3f, 0xb5, 0xa0, 0x57, - 0xb9, 0x98, 0x2a, 0xdd, 0x9b, 0xda, 0xa0, 0x4b, 0x8a, 0x85, 0x09, 0xb7, 0x37, 0x2d, 0x1f, 0x9a, - 0xc8, 0x0e, 0x6c, 0xc4, 0x62, 0x66, 0x3c, 0x6e, 0x7b, 0x7a, 0x40, 0x6e, 0x42, 0x27, 0x16, 0x33, - 0xec, 0xd4, 0x4d, 0x96, 0x2e, 0xc6, 0xca, 0x6d, 0x45, 0x45, 0x31, 0xc9, 0xaa, 0x14, 0xb8, 0xbf, - 0xb3, 0x80, 0x98, 0xa6, 0xe1, 0x85, 0x5e, 0x11, 0x31, 0x60, 0xab, 0x8f, 0x65, 0x0d, 0xdd, 0x39, - 0x55, 0x65, 0x6b, 0xe5, 0xae, 0x79, 0xaa, 0xdc, 0xdd, 0x81, 0xab, 0x01, 0x3b, 0xa1, 0xaa, 0xbf, - 0x59, 0x3f, 0x72, 0xdf, 0x28, 0x9e, 0xe4, 0xf2, 0x37, 0xde, 0x86, 0x6e, 0xf1, 0x78, 0x4f, 0xfa, - 0x60, 0x8f, 0x92, 0x50, 0xe2, 0xaf, 0x44, 0x98, 0xcc, 0xfa, 0x5f, 0x22, 0x3d, 0x68, 0xff, 0x80, - 0xd1, 0x48, 0xce, 0x97, 0x7d, 0x8b, 0xd8, 0xd0, 0x79, 0x77, 0x92, 0xa4, 0x3c, 0xa6, 0x51, 0xbf, - 0xf1, 0xe0, 0xad, 0x1f, 0x7f, 0x7b, 0x16, 0xca, 0xf9, 0x62, 0xa2, 0x90, 0xec, 0x6b, 0x68, 0xdf, - 0x08, 0x53, 0xf3, 0xb5, 0x9f, 0x7b, 0x6d, 0x1f, 0xd1, 0x16, 0xc3, 0x6c, 0x32, 0xd9, 0x44, 0xc9, - 0x9b, 0xff, 0x0c, 0x00, 0x00, 0xff, 0xff, 0xef, 0x4f, 0x87, 0x49, 0xe2, 0x18, 0x00, 0x00, + // 1832 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x59, 0x4b, 0x93, 0x23, 0x47, + 0x11, 0xa6, 0x25, 0xcd, 0x48, 0x4a, 0xf5, 0xcc, 0x6a, 0x6b, 0xc7, 0xeb, 0x9e, 0xd9, 0xf5, 0x5a, + 0x6e, 0xf3, 0x18, 0xbc, 0xc1, 0xce, 0x32, 0x06, 0xec, 0x20, 0x08, 0xd6, 0x9e, 0x91, 0xbd, 0x28, + 0xd6, 0xb3, 0x31, 0xb4, 0xd6, 0x8e, 0x80, 0x4b, 0x47, 0xa9, 0xbb, 0x46, 0x6a, 0xbb, 0x5f, 0x74, + 0x95, 0x76, 0x57, 0x3e, 0x71, 0xe0, 0x46, 0xc0, 0x81, 0x08, 0x4e, 0xfc, 0x07, 0xae, 0x9c, 0x78, + 0x84, 0x4f, 0x44, 0xf0, 0x0b, 0xf8, 0x1b, 0x1c, 0xb9, 0x40, 0x54, 0x56, 0xf5, 0x43, 0x9a, 0x9e, + 0x61, 0xac, 0x0d, 0x02, 0x1c, 0x70, 0xeb, 0xca, 0xcc, 0x7a, 0xe4, 0x97, 0x5f, 0x65, 0xa6, 0x4a, + 0xb0, 0x1d, 0xc4, 0x82, 0x65, 0x31, 0x0d, 0xef, 0xa5, 0x59, 0x22, 0x12, 0xf2, 0x52, 0x14, 0x84, + 0x4f, 0xe7, 0x5c, 0x8d, 0xee, 0xe5, 0xca, 0x3d, 0xd3, 0x4b, 0xa2, 0x28, 0x89, 0x95, 0x78, 0xcf, + 0xe4, 0xde, 0x8c, 0x45, 0x54, 0x8d, 0xec, 0x3f, 0x18, 0xb0, 0x75, 0x9c, 0x44, 0x69, 0x12, 0xb3, + 0x58, 0x8c, 0xe2, 0xb3, 0x84, 0xdc, 0x84, 0xcd, 0x38, 0xf1, 0xd9, 0x68, 0x68, 0x19, 0x03, 0x63, + 0xbf, 0xe9, 0xe8, 0x11, 0x21, 0xd0, 0xca, 0x92, 0x90, 0x59, 0x8d, 0x81, 0xb1, 0xdf, 0x75, 0xf0, + 0x9b, 0x3c, 0x00, 0xe0, 0x82, 0x0a, 0xe6, 0x7a, 0x89, 0xcf, 0xac, 0xe6, 0xc0, 0xd8, 0xdf, 0x3e, + 0x1c, 0xdc, 0xab, 0x3d, 0xc5, 0xbd, 0xb1, 0x34, 0x3c, 0x4e, 0x7c, 0xe6, 0x74, 0x79, 0xfe, 0x49, + 0xde, 0x01, 0x60, 0xcf, 0x45, 0x46, 0xdd, 0x20, 0x3e, 0x4b, 0xac, 0xd6, 0xa0, 0xb9, 0xdf, 0x3b, + 0x7c, 0x6d, 0x79, 0x01, 0x7d, 0xf8, 0x47, 0x6c, 0xf1, 0x11, 0x0d, 0xe7, 0xec, 0x94, 0x06, 0x99, + 0xd3, 0xc5, 0x49, 0xf2, 0xb8, 0xf6, 0x5f, 0x0d, 0xb8, 0x56, 0x38, 0x80, 0x7b, 0x70, 0xf2, 0x5d, + 0xd8, 0xc0, 0x2d, 0xd0, 0x83, 0xde, 0xe1, 0x97, 0x2f, 0x38, 0xd1, 0x92, 0xdf, 0x8e, 0x9a, 0x42, + 0x3e, 0x84, 0x1b, 0x7c, 0x3e, 0xf1, 0x72, 0x95, 0x8b, 0x52, 0x6e, 0x35, 0xf0, 0x68, 0x57, 0x5b, + 0x89, 0x54, 0x17, 0xd0, 0x47, 0x7a, 0x13, 0x36, 0xe5, 0x4a, 0x73, 0x8e, 0x28, 0xf5, 0x0e, 0x6f, + 0xd5, 0x3a, 0x39, 0x46, 0x13, 0x47, 0x9b, 0xda, 0xb7, 0x60, 0xf7, 0x21, 0x13, 0x2b, 0xde, 0x39, + 0xec, 0x27, 0x73, 0xc6, 0x85, 0x56, 0x3e, 0x09, 0x22, 0xf6, 0x24, 0xf0, 0x3e, 0x39, 0x9e, 0xd1, + 0x38, 0x66, 0x61, 0xae, 0x7c, 0x05, 0x6e, 0x3d, 0x64, 0x38, 0x21, 0xe0, 0x22, 0xf0, 0xf8, 0x8a, + 0xfa, 0x25, 0xb8, 0xf1, 0x90, 0x89, 0xa1, 0xbf, 0x22, 0xfe, 0x08, 0x3a, 0x8f, 0x65, 0xb0, 0x25, + 0x0d, 0xbe, 0x03, 0x6d, 0xea, 0xfb, 0x19, 0xe3, 0x5c, 0xa3, 0x78, 0xbb, 0xf6, 0xc4, 0xef, 0x2a, + 0x1b, 0x27, 0x37, 0xae, 0xa3, 0x89, 0xfd, 0x31, 0xc0, 0x28, 0x0e, 0xc4, 0x29, 0xcd, 0x68, 0xc4, + 0x2f, 0x24, 0xd8, 0x10, 0x4c, 0x2e, 0x68, 0x26, 0xdc, 0x14, 0xed, 0x34, 0xe4, 0x57, 0x60, 0x43, + 0x0f, 0xa7, 0xa9, 0xd5, 0xed, 0x1f, 0x01, 0x8c, 0x45, 0x16, 0xc4, 0xd3, 0x0f, 0x02, 0x2e, 0xe4, + 0x5e, 0x4f, 0xa5, 0x9d, 0x74, 0xa2, 0xb9, 0xdf, 0x75, 0xf4, 0xa8, 0x12, 0x8e, 0xc6, 0xd5, 0xc3, + 0xf1, 0x00, 0x7a, 0x39, 0xdc, 0x27, 0x7c, 0x4a, 0xee, 0x43, 0x6b, 0x42, 0x39, 0xbb, 0x14, 0x9e, + 0x13, 0x3e, 0x3d, 0xa2, 0x9c, 0x39, 0x68, 0x69, 0x7f, 0xd6, 0x80, 0x97, 0x8f, 0x33, 0x86, 0xe4, + 0x0f, 0x43, 0xe6, 0x89, 0x20, 0x89, 0x35, 0xf6, 0x9f, 0x7f, 0x35, 0xf2, 0x32, 0xb4, 0xfd, 0x89, + 0x1b, 0xd3, 0x28, 0x07, 0x7b, 0xd3, 0x9f, 0x3c, 0xa6, 0x11, 0x23, 0x5f, 0x85, 0x6d, 0xaf, 0x58, + 0x5f, 0x4a, 0x90, 0x73, 0x5d, 0x67, 0x45, 0x2a, 0x43, 0xe5, 0x4f, 0x46, 0x43, 0xab, 0x85, 0x61, + 0xc0, 0x6f, 0x62, 0x83, 0x59, 0x5a, 0x8d, 0x86, 0xd6, 0x06, 0xea, 0x96, 0x64, 0x12, 0x54, 0x95, + 0x43, 0xac, 0xcd, 0x81, 0xb1, 0x6f, 0x3a, 0x7a, 0x44, 0xee, 0xc3, 0x8d, 0xa7, 0x41, 0x26, 0xe6, + 0x34, 0xd4, 0xbc, 0x92, 0xbb, 0x70, 0xab, 0x8d, 0xc8, 0xd7, 0xa9, 0xc8, 0x21, 0xec, 0xa4, 0xb3, + 0x05, 0x0f, 0xbc, 0x95, 0x29, 0x1d, 0x9c, 0x52, 0xab, 0xb3, 0x3f, 0x33, 0xe0, 0xa5, 0x61, 0x96, + 0xa4, 0x5f, 0x64, 0x08, 0xed, 0x5f, 0x34, 0xe0, 0xa6, 0x62, 0xc2, 0x29, 0xcd, 0x44, 0xf0, 0x6f, + 0xf2, 0xe2, 0x6b, 0x70, 0xad, 0xdc, 0x55, 0x19, 0xd4, 0xbb, 0xf1, 0x15, 0xd8, 0x4e, 0xf3, 0x73, + 0x28, 0xbb, 0x16, 0xda, 0x6d, 0x15, 0xd2, 0x25, 0x6f, 0x37, 0x2e, 0xf1, 0x76, 0xb3, 0x86, 0x30, + 0x03, 0xe8, 0x15, 0x0b, 0x8d, 0x86, 0x56, 0x1b, 0x4d, 0xaa, 0x22, 0xfb, 0xe7, 0x0d, 0xd8, 0x91, + 0x41, 0xfd, 0x3f, 0x1a, 0x12, 0x8d, 0x3f, 0x36, 0x80, 0x28, 0x76, 0x8c, 0x62, 0x9f, 0x3d, 0xff, + 0x4f, 0x62, 0xf1, 0x0a, 0xc0, 0x59, 0xc0, 0x42, 0xbf, 0x8a, 0x43, 0x17, 0x25, 0x2f, 0x84, 0x81, + 0x05, 0x6d, 0x5c, 0xa4, 0xf0, 0x3f, 0x1f, 0xca, 0x2a, 0xa0, 0x3a, 0x02, 0x5d, 0x05, 0x3a, 0x57, + 0xae, 0x02, 0x38, 0x4d, 0x57, 0x81, 0xdf, 0x36, 0x61, 0x6b, 0x14, 0x73, 0x96, 0x89, 0xff, 0x65, + 0x22, 0x91, 0xdb, 0xd0, 0xe5, 0x6c, 0x1a, 0xc9, 0xc6, 0x64, 0x68, 0x75, 0x50, 0x5f, 0x0a, 0xa4, + 0xd6, 0x53, 0x99, 0x75, 0x34, 0xb4, 0xba, 0x2a, 0xb4, 0x85, 0x80, 0xdc, 0x01, 0x10, 0x41, 0xc4, + 0xb8, 0xa0, 0x51, 0xca, 0x2d, 0x18, 0x34, 0xf7, 0x5b, 0x4e, 0x45, 0x22, 0xab, 0x40, 0x96, 0x3c, + 0x1b, 0x0d, 0xb9, 0xd5, 0x1b, 0x34, 0x65, 0x19, 0x57, 0x23, 0xf2, 0x2d, 0xe8, 0x64, 0xc9, 0x33, + 0xd7, 0xa7, 0x82, 0x5a, 0x26, 0x06, 0x6f, 0xb7, 0x16, 0xec, 0xa3, 0x30, 0x99, 0x38, 0xed, 0x2c, + 0x79, 0x36, 0xa4, 0x82, 0xda, 0x7f, 0x6b, 0xc0, 0xd6, 0x98, 0xd1, 0xcc, 0x9b, 0xad, 0x1f, 0xb0, + 0xaf, 0x43, 0x3f, 0x63, 0x7c, 0x1e, 0x0a, 0xb7, 0x74, 0x4b, 0x45, 0xee, 0x9a, 0x92, 0x1f, 0x17, + 0xce, 0xe5, 0x90, 0x37, 0x2f, 0x81, 0xbc, 0x55, 0x03, 0xb9, 0x0d, 0x66, 0x05, 0x5f, 0x6e, 0x6d, + 0xa0, 0xeb, 0x4b, 0x32, 0xd2, 0x87, 0xa6, 0xcf, 0x43, 0x8c, 0x58, 0xd7, 0x91, 0x9f, 0xe4, 0x2e, + 0x5c, 0x4f, 0x43, 0xea, 0xb1, 0x59, 0x12, 0xfa, 0x2c, 0x73, 0xa7, 0x59, 0x32, 0x4f, 0x31, 0x5c, + 0xa6, 0xd3, 0xaf, 0x28, 0x1e, 0x4a, 0x39, 0x79, 0x0b, 0x3a, 0x3e, 0x0f, 0x5d, 0xb1, 0x48, 0x19, + 0x86, 0x6c, 0xfb, 0x02, 0xdf, 0x87, 0x3c, 0x7c, 0xb2, 0x48, 0x99, 0xd3, 0xf6, 0xd5, 0x07, 0xb9, + 0x0f, 0x3b, 0x9c, 0x65, 0x01, 0x0d, 0x83, 0x4f, 0x99, 0xef, 0xb2, 0xe7, 0x69, 0xe6, 0xa6, 0x21, + 0x8d, 0x31, 0xb2, 0xa6, 0x43, 0x4a, 0xdd, 0x7b, 0xcf, 0xd3, 0xec, 0x34, 0xa4, 0xb1, 0xfd, 0x8f, + 0x0a, 0xe8, 0x12, 0x1f, 0xbe, 0x06, 0xe8, 0xeb, 0x74, 0x52, 0xb5, 0x91, 0x6a, 0xd6, 0x47, 0xea, + 0x55, 0xe8, 0x45, 0x4c, 0x64, 0x81, 0xa7, 0x10, 0x51, 0x17, 0x08, 0x94, 0x08, 0xdd, 0x26, 0xd0, + 0x9a, 0x05, 0x42, 0x85, 0xc2, 0x74, 0xf0, 0x9b, 0x7c, 0x0f, 0xf6, 0x38, 0xa3, 0x21, 0xf3, 0xdd, + 0x82, 0xed, 0xdc, 0xe5, 0xe8, 0x29, 0xf3, 0xad, 0x4d, 0x0c, 0x9a, 0xa5, 0x2c, 0xc6, 0x85, 0xc1, + 0x58, 0xeb, 0xc9, 0x01, 0xdc, 0x28, 0x8e, 0x55, 0x99, 0xa6, 0xfa, 0x18, 0x52, 0xaa, 0x8a, 0x09, + 0x6f, 0x83, 0x35, 0x0d, 0x93, 0x09, 0x0d, 0xdd, 0x73, 0xbb, 0x62, 0xfe, 0x6a, 0x3a, 0x37, 0x95, + 0x7e, 0xbc, 0xb2, 0xa5, 0xfd, 0x9b, 0x06, 0x5c, 0x73, 0xa4, 0x2f, 0xec, 0x29, 0xfb, 0xc2, 0x13, + 0xff, 0x0d, 0x68, 0x06, 0x3e, 0x47, 0xe2, 0xf7, 0x0e, 0xad, 0xe5, 0x73, 0xeb, 0x1f, 0x9d, 0xa3, + 0x21, 0x77, 0xa4, 0x11, 0x79, 0x1d, 0xb6, 0x92, 0xb9, 0x48, 0xe7, 0xc2, 0xc5, 0xc4, 0x9f, 0x77, + 0x89, 0xa6, 0x12, 0xbe, 0x8f, 0x32, 0xfb, 0xf7, 0xcd, 0x2a, 0x3a, 0xff, 0xad, 0x0c, 0xd5, 0x6e, + 0xb7, 0xae, 0xe2, 0xf6, 0x03, 0xe8, 0x29, 0x7f, 0x55, 0x7e, 0xdc, 0xc0, 0xfc, 0x78, 0xa7, 0x76, + 0x0e, 0x62, 0x20, 0x73, 0xa3, 0xa3, 0x2a, 0x30, 0x97, 0xdf, 0xe4, 0xfb, 0x70, 0xeb, 0x3c, 0xb3, + 0x33, 0x8d, 0x51, 0x4e, 0xed, 0xdd, 0x55, 0x6a, 0xe7, 0x20, 0xfa, 0xe4, 0x9b, 0xb0, 0x53, 0xe1, + 0x76, 0x39, 0x51, 0x37, 0xe9, 0xa5, 0xae, 0x9c, 0xb2, 0x3e, 0xbb, 0xff, 0x62, 0xc0, 0xd6, 0x90, + 0x85, 0x4c, 0xbc, 0x00, 0xb7, 0x6b, 0x8a, 0x6d, 0xa3, 0xb6, 0xd8, 0x2e, 0x55, 0xb3, 0xe6, 0xe5, + 0xd5, 0xac, 0x75, 0xae, 0x9a, 0xbd, 0x06, 0x66, 0x9a, 0x05, 0x11, 0xcd, 0x16, 0xee, 0x27, 0x6c, + 0x91, 0xf3, 0xbb, 0xa7, 0x65, 0x8f, 0xd8, 0x82, 0xdb, 0x31, 0xec, 0x7d, 0x90, 0x50, 0xff, 0x88, + 0x86, 0x34, 0xf6, 0x98, 0x76, 0x93, 0xaf, 0xef, 0xd9, 0x1d, 0x80, 0x0a, 0x92, 0x0d, 0xdc, 0xb0, + 0x22, 0xb1, 0xff, 0x6e, 0x40, 0x57, 0x6e, 0x88, 0x3d, 0xe0, 0x1a, 0xeb, 0x2f, 0x15, 0xff, 0x46, + 0x4d, 0xf1, 0x2f, 0xda, 0xb8, 0x1c, 0xae, 0xb2, 0xaf, 0xab, 0xf4, 0x67, 0xad, 0xe5, 0xfe, 0xec, + 0x55, 0xe8, 0x05, 0xf2, 0x40, 0x6e, 0x4a, 0xc5, 0x4c, 0xe1, 0xd4, 0x75, 0x00, 0x45, 0xa7, 0x52, + 0x22, 0x1b, 0xb8, 0xdc, 0x00, 0x1b, 0xb8, 0xcd, 0x2b, 0x37, 0x70, 0x7a, 0x11, 0x6c, 0xe0, 0xfe, + 0xd4, 0x00, 0x4b, 0x43, 0x5c, 0xbe, 0x62, 0x7c, 0x98, 0xfa, 0xf8, 0x98, 0x72, 0x1b, 0xba, 0x05, + 0xcb, 0xf4, 0x23, 0x42, 0x29, 0x90, 0xb8, 0x9e, 0xb0, 0x28, 0xc9, 0x16, 0xe3, 0xe0, 0x53, 0xa6, + 0x1d, 0xaf, 0x48, 0xa4, 0x6f, 0x8f, 0xe7, 0x91, 0x93, 0x3c, 0xe3, 0x3a, 0x0b, 0xe6, 0x43, 0xe9, + 0x9b, 0x87, 0x6d, 0xb7, 0x2b, 0x99, 0x81, 0x9e, 0xb7, 0x1c, 0x50, 0x22, 0xf9, 0xcb, 0x9f, 0xec, + 0x42, 0x87, 0xc5, 0xbe, 0xd2, 0x6e, 0xa0, 0xb6, 0xcd, 0x62, 0x1f, 0x55, 0x23, 0xd8, 0xd6, 0xaf, + 0x17, 0x09, 0xc7, 0x8c, 0xa8, 0xf3, 0xa0, 0x7d, 0xc1, 0x93, 0xd1, 0x09, 0x9f, 0x9e, 0x6a, 0x4b, + 0x67, 0x4b, 0x3d, 0x60, 0xe8, 0x21, 0x79, 0x0f, 0x4c, 0xb9, 0x4b, 0xb1, 0x50, 0xfb, 0xca, 0x0b, + 0xf5, 0x58, 0xec, 0xe7, 0x03, 0xfb, 0x57, 0x06, 0x5c, 0x3f, 0x07, 0xe1, 0x1a, 0x3c, 0x7a, 0x04, + 0x9d, 0x31, 0x9b, 0xca, 0x25, 0xf2, 0x37, 0x99, 0x83, 0x8b, 0x9e, 0xf8, 0x2e, 0x08, 0x98, 0x53, + 0x2c, 0x60, 0x7f, 0x5c, 0x84, 0xf5, 0xfd, 0x70, 0xce, 0x67, 0xc7, 0x49, 0x94, 0xca, 0xfc, 0xe0, + 0xaf, 0xf5, 0xa0, 0x72, 0x39, 0xc5, 0xed, 0x9f, 0x19, 0x00, 0x78, 0x79, 0x70, 0xeb, 0x73, 0xc4, + 0x34, 0xd6, 0x21, 0xa6, 0xec, 0xb2, 0xe2, 0x79, 0xe4, 0x66, 0x2c, 0xa4, 0xa2, 0xcc, 0x85, 0x5c, + 0xef, 0x4e, 0xe2, 0x79, 0xe4, 0x28, 0x55, 0x9e, 0x20, 0xec, 0x5f, 0x1a, 0x00, 0x98, 0xcc, 0xd5, + 0x31, 0x56, 0xab, 0xad, 0x71, 0xf9, 0xcf, 0xa3, 0xc6, 0xf2, 0xf5, 0x3b, 0xca, 0xaf, 0x1f, 0xc7, + 0x78, 0x34, 0xeb, 0x7c, 0x28, 0xe2, 0x51, 0x3a, 0xaf, 0x6f, 0xa8, 0x8a, 0xc1, 0xaf, 0x0d, 0x30, + 0x2b, 0xa1, 0xe2, 0xcb, 0x30, 0x1a, 0xab, 0x99, 0x02, 0x3b, 0x30, 0x79, 0x7b, 0x5c, 0x5e, 0xb9, + 0x50, 0x51, 0x79, 0xa1, 0x76, 0xa1, 0x83, 0x90, 0x54, 0x6e, 0x54, 0xac, 0x6f, 0xd4, 0x5d, 0xb8, + 0x9e, 0x31, 0x8f, 0xc5, 0x22, 0x5c, 0xb8, 0x51, 0xe2, 0x07, 0x67, 0x01, 0xf3, 0xf1, 0x5e, 0x75, + 0x9c, 0x7e, 0xae, 0x38, 0xd1, 0x72, 0xfb, 0xcf, 0x06, 0x6c, 0xff, 0x70, 0xce, 0xb2, 0xc5, 0xe3, + 0xc4, 0x67, 0xea, 0x64, 0x9f, 0x9f, 0x12, 0xef, 0xa0, 0x2f, 0x1a, 0x1e, 0x45, 0xd7, 0xd7, 0xff, + 0x35, 0x5d, 0xb9, 0xd3, 0xe1, 0x9a, 0xa2, 0x12, 0x62, 0xf5, 0x93, 0xf7, 0x2a, 0x10, 0x97, 0x81, + 0xd5, 0x65, 0x5a, 0x41, 0xfc, 0x53, 0x03, 0x7a, 0x95, 0x8b, 0x29, 0xcb, 0x8b, 0xae, 0x45, 0xaa, + 0x84, 0x19, 0x98, 0x70, 0x7b, 0x5e, 0xf9, 0xb0, 0x45, 0x76, 0x60, 0x23, 0xe2, 0x53, 0x1d, 0x71, + 0xd3, 0x51, 0x03, 0xb2, 0x07, 0x9d, 0x88, 0x4f, 0xf1, 0x97, 0x81, 0xce, 0xd2, 0xc5, 0x58, 0x86, + 0xad, 0xa8, 0x60, 0x3a, 0x59, 0x95, 0x02, 0xfb, 0x77, 0x06, 0x10, 0xdd, 0xa4, 0xbc, 0xd0, 0xab, + 0x25, 0x12, 0xb6, 0xfa, 0x38, 0xd7, 0x50, 0x9d, 0x5a, 0x55, 0xb6, 0x52, 0x5e, 0x9b, 0xe7, 0xca, + 0xeb, 0x5d, 0xb8, 0xee, 0xb3, 0x33, 0x2a, 0xfb, 0xa9, 0xd5, 0x23, 0xf7, 0xb5, 0xe2, 0x49, 0x2e, + 0x7f, 0xe3, 0x6d, 0xe8, 0x16, 0x7f, 0x16, 0x90, 0x3e, 0x98, 0xa3, 0x38, 0x10, 0xf8, 0xd3, 0x25, + 0x88, 0xa7, 0xfd, 0x2f, 0x91, 0x1e, 0xb4, 0x7f, 0xc0, 0x68, 0x28, 0x66, 0x8b, 0xbe, 0x41, 0x4c, + 0xe8, 0xbc, 0x3b, 0x89, 0x93, 0x2c, 0xa2, 0x61, 0xbf, 0x71, 0xf4, 0xd6, 0x8f, 0xbf, 0x3d, 0x0d, + 0xc4, 0x6c, 0x3e, 0x91, 0x9e, 0x1c, 0x28, 0xd7, 0xbe, 0x11, 0x24, 0xfa, 0xeb, 0x20, 0x8f, 0xda, + 0x01, 0x7a, 0x5b, 0x0c, 0xd3, 0xc9, 0x64, 0x13, 0x25, 0x6f, 0xfe, 0x33, 0x00, 0x00, 0xff, 0xff, + 0x57, 0x6c, 0x28, 0x16, 0x52, 0x19, 0x00, 0x00, } diff --git a/internal/proto/query_service.proto b/internal/proto/query_service.proto index 4ef8feff4d..8a35ddd1a6 100644 --- a/internal/proto/query_service.proto +++ b/internal/proto/query_service.proto @@ -88,6 +88,7 @@ message ReleaseCollectionRequest { common.MsgBase base = 1; int64 dbID = 2; int64 collectionID = 3; + int64 nodeID = 4; } message LoadPartitionsRequest { @@ -103,6 +104,7 @@ message ReleasePartitionsRequest { int64 dbID = 2; int64 collectionID = 3; repeated int64 partitionIDs = 4; + int64 nodeID = 5; } message CreateQueryChannelRequest { @@ -266,7 +268,8 @@ message CollectionInfo { int64 collectionID = 1; repeated int64 partitionIDs = 2; repeated DmChannelInfo channel_infos = 3; - schema.CollectionSchema schema = 6; + bool load_collection = 4; + schema.CollectionSchema schema = 5; } message HandoffSegments { @@ -285,9 +288,8 @@ message LoadBalanceSegmentInfo { bool valid_info = 8; } -message LoadBalanceSegments { +message LoadBalanceRequest { common.MsgBase base = 1; - repeated LoadBalanceSegmentInfo infos = 2; -} - - + repeated int64 source_nodeIDs = 2; + TriggerCondition balance_reason = 3; +} \ No newline at end of file diff --git a/internal/proto/querypb/query_service.pb.go b/internal/proto/querypb/query_service.pb.go index 277ab1a3d7..4bda52587a 100644 --- a/internal/proto/querypb/query_service.pb.go +++ b/internal/proto/querypb/query_service.pb.go @@ -493,6 +493,7 @@ type ReleaseCollectionRequest struct { Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` DbID int64 `protobuf:"varint,2,opt,name=dbID,proto3" json:"dbID,omitempty"` CollectionID int64 `protobuf:"varint,3,opt,name=collectionID,proto3" json:"collectionID,omitempty"` + NodeID int64 `protobuf:"varint,4,opt,name=nodeID,proto3" json:"nodeID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -544,6 +545,13 @@ func (m *ReleaseCollectionRequest) GetCollectionID() int64 { return 0 } +func (m *ReleaseCollectionRequest) GetNodeID() int64 { + if m != nil { + return m.NodeID + } + return 0 +} + type LoadPartitionsRequest struct { Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` DbID int64 `protobuf:"varint,2,opt,name=dbID,proto3" json:"dbID,omitempty"` @@ -620,6 +628,7 @@ type ReleasePartitionsRequest struct { DbID int64 `protobuf:"varint,2,opt,name=dbID,proto3" json:"dbID,omitempty"` CollectionID int64 `protobuf:"varint,3,opt,name=collectionID,proto3" json:"collectionID,omitempty"` PartitionIDs []int64 `protobuf:"varint,4,rep,packed,name=partitionIDs,proto3" json:"partitionIDs,omitempty"` + NodeID int64 `protobuf:"varint,5,opt,name=nodeID,proto3" json:"nodeID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -678,6 +687,13 @@ func (m *ReleasePartitionsRequest) GetPartitionIDs() []int64 { return nil } +func (m *ReleasePartitionsRequest) GetNodeID() int64 { + if m != nil { + return m.NodeID + } + return 0 +} + type CreateQueryChannelRequest struct { CollectionID int64 `protobuf:"varint,1,opt,name=collectionID,proto3" json:"collectionID,omitempty"` ProxyID int64 `protobuf:"varint,2,opt,name=proxyID,proto3" json:"proxyID,omitempty"` @@ -1708,7 +1724,8 @@ type CollectionInfo struct { CollectionID int64 `protobuf:"varint,1,opt,name=collectionID,proto3" json:"collectionID,omitempty"` PartitionIDs []int64 `protobuf:"varint,2,rep,packed,name=partitionIDs,proto3" json:"partitionIDs,omitempty"` ChannelInfos []*DmChannelInfo `protobuf:"bytes,3,rep,name=channel_infos,json=channelInfos,proto3" json:"channel_infos,omitempty"` - Schema *schemapb.CollectionSchema `protobuf:"bytes,6,opt,name=schema,proto3" json:"schema,omitempty"` + LoadCollection bool `protobuf:"varint,4,opt,name=load_collection,json=loadCollection,proto3" json:"load_collection,omitempty"` + Schema *schemapb.CollectionSchema `protobuf:"bytes,5,opt,name=schema,proto3" json:"schema,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1760,6 +1777,13 @@ func (m *CollectionInfo) GetChannelInfos() []*DmChannelInfo { return nil } +func (m *CollectionInfo) GetLoadCollection() bool { + if m != nil { + return m.LoadCollection + } + return false +} + func (m *CollectionInfo) GetSchema() *schemapb.CollectionSchema { if m != nil { return m.Schema @@ -1909,53 +1933,61 @@ func (m *LoadBalanceSegmentInfo) GetValidInfo() bool { return false } -type LoadBalanceSegments struct { - Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` - Infos []*LoadBalanceSegmentInfo `protobuf:"bytes,2,rep,name=infos,proto3" json:"infos,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` +type LoadBalanceRequest struct { + Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` + SourceNodeIDs []int64 `protobuf:"varint,2,rep,packed,name=source_nodeIDs,json=sourceNodeIDs,proto3" json:"source_nodeIDs,omitempty"` + BalanceReason TriggerCondition `protobuf:"varint,3,opt,name=balance_reason,json=balanceReason,proto3,enum=milvus.proto.query.TriggerCondition" json:"balance_reason,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (m *LoadBalanceSegments) Reset() { *m = LoadBalanceSegments{} } -func (m *LoadBalanceSegments) String() string { return proto.CompactTextString(m) } -func (*LoadBalanceSegments) ProtoMessage() {} -func (*LoadBalanceSegments) Descriptor() ([]byte, []int) { +func (m *LoadBalanceRequest) Reset() { *m = LoadBalanceRequest{} } +func (m *LoadBalanceRequest) String() string { return proto.CompactTextString(m) } +func (*LoadBalanceRequest) ProtoMessage() {} +func (*LoadBalanceRequest) Descriptor() ([]byte, []int) { return fileDescriptor_5fcb6756dc1afb8d, []int{29} } -func (m *LoadBalanceSegments) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_LoadBalanceSegments.Unmarshal(m, b) +func (m *LoadBalanceRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_LoadBalanceRequest.Unmarshal(m, b) } -func (m *LoadBalanceSegments) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_LoadBalanceSegments.Marshal(b, m, deterministic) +func (m *LoadBalanceRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_LoadBalanceRequest.Marshal(b, m, deterministic) } -func (m *LoadBalanceSegments) XXX_Merge(src proto.Message) { - xxx_messageInfo_LoadBalanceSegments.Merge(m, src) +func (m *LoadBalanceRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_LoadBalanceRequest.Merge(m, src) } -func (m *LoadBalanceSegments) XXX_Size() int { - return xxx_messageInfo_LoadBalanceSegments.Size(m) +func (m *LoadBalanceRequest) XXX_Size() int { + return xxx_messageInfo_LoadBalanceRequest.Size(m) } -func (m *LoadBalanceSegments) XXX_DiscardUnknown() { - xxx_messageInfo_LoadBalanceSegments.DiscardUnknown(m) +func (m *LoadBalanceRequest) XXX_DiscardUnknown() { + xxx_messageInfo_LoadBalanceRequest.DiscardUnknown(m) } -var xxx_messageInfo_LoadBalanceSegments proto.InternalMessageInfo +var xxx_messageInfo_LoadBalanceRequest proto.InternalMessageInfo -func (m *LoadBalanceSegments) GetBase() *commonpb.MsgBase { +func (m *LoadBalanceRequest) GetBase() *commonpb.MsgBase { if m != nil { return m.Base } return nil } -func (m *LoadBalanceSegments) GetInfos() []*LoadBalanceSegmentInfo { +func (m *LoadBalanceRequest) GetSourceNodeIDs() []int64 { if m != nil { - return m.Infos + return m.SourceNodeIDs } return nil } +func (m *LoadBalanceRequest) GetBalanceReason() TriggerCondition { + if m != nil { + return m.BalanceReason + } + return TriggerCondition_handoff +} + func init() { proto.RegisterEnum("milvus.proto.query.PartitionState", PartitionState_name, PartitionState_value) proto.RegisterEnum("milvus.proto.query.TriggerCondition", TriggerCondition_name, TriggerCondition_value) @@ -1989,129 +2021,132 @@ func init() { proto.RegisterType((*CollectionInfo)(nil), "milvus.proto.query.CollectionInfo") proto.RegisterType((*HandoffSegments)(nil), "milvus.proto.query.HandoffSegments") proto.RegisterType((*LoadBalanceSegmentInfo)(nil), "milvus.proto.query.LoadBalanceSegmentInfo") - proto.RegisterType((*LoadBalanceSegments)(nil), "milvus.proto.query.LoadBalanceSegments") + proto.RegisterType((*LoadBalanceRequest)(nil), "milvus.proto.query.LoadBalanceRequest") } func init() { proto.RegisterFile("query_service.proto", fileDescriptor_5fcb6756dc1afb8d) } var fileDescriptor_5fcb6756dc1afb8d = []byte{ - // 1856 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x58, 0xcd, 0x73, 0x1c, 0x47, + // 1898 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x59, 0xcd, 0x73, 0x1c, 0x47, 0x15, 0xd7, 0xec, 0xae, 0x56, 0xda, 0xb7, 0x5f, 0x93, 0xb6, 0x2d, 0xd6, 0x4b, 0x9c, 0x38, 0xe3, - 0x04, 0x3b, 0x0a, 0x91, 0x53, 0x4a, 0xa0, 0xc8, 0x81, 0x2a, 0x2c, 0x6d, 0x2c, 0x16, 0x12, 0x45, - 0x8c, 0x4c, 0x28, 0x5c, 0x2e, 0x96, 0xd1, 0x4c, 0x6b, 0x35, 0x95, 0x99, 0xee, 0xd5, 0x74, 0xaf, - 0x65, 0xfb, 0x00, 0x45, 0x15, 0x17, 0x4e, 0x9c, 0x38, 0x71, 0xe2, 0x42, 0xc1, 0x81, 0xff, 0x81, - 0x7f, 0x04, 0x28, 0x8a, 0xdc, 0x38, 0x73, 0xa7, 0xa6, 0xbb, 0x67, 0x76, 0x3e, 0x7a, 0xa5, 0xb5, - 0x64, 0x47, 0x29, 0x8a, 0xdb, 0xf4, 0xeb, 0xd7, 0xef, 0xbb, 0x7f, 0xf3, 0xfa, 0xc1, 0x95, 0xe3, - 0x29, 0x8e, 0x9e, 0x8e, 0x18, 0x8e, 0x1e, 0xfb, 0x2e, 0xde, 0x98, 0x44, 0x94, 0x53, 0x84, 0x42, - 0x3f, 0x78, 0x3c, 0x65, 0x72, 0xb5, 0x21, 0x38, 0xfa, 0x2d, 0x97, 0x86, 0x21, 0x25, 0x92, 0xd6, - 0x6f, 0x65, 0x39, 0xfa, 0x1d, 0x9f, 0x70, 0x1c, 0x11, 0x27, 0x48, 0x76, 0x99, 0x7b, 0x84, 0x43, - 0x47, 0xad, 0x90, 0xe7, 0x70, 0x27, 0xaf, 0xc1, 0xfa, 0x25, 0x5c, 0xb1, 0xf1, 0xd8, 0x67, 0x1c, - 0x47, 0xbb, 0xd4, 0xc3, 0x36, 0x3e, 0x9e, 0x62, 0xc6, 0xd1, 0x7b, 0x50, 0x3b, 0x70, 0x18, 0xee, - 0x19, 0x37, 0x8d, 0x3b, 0xcd, 0xcd, 0x57, 0x37, 0x72, 0x76, 0x28, 0x03, 0x3e, 0x61, 0xe3, 0x2d, - 0x87, 0x61, 0x5b, 0x70, 0xa2, 0x6f, 0xc3, 0x8a, 0xe3, 0x79, 0x11, 0x66, 0xac, 0x57, 0x39, 0xe5, - 0xd0, 0x3d, 0xc9, 0x63, 0x27, 0xcc, 0xd6, 0x6f, 0x0d, 0xb8, 0x9a, 0xb7, 0x80, 0x4d, 0x28, 0x61, - 0x18, 0xbd, 0x0f, 0x75, 0xc6, 0x1d, 0x3e, 0x65, 0xca, 0x88, 0xaf, 0x6b, 0xe5, 0xed, 0x0b, 0x16, - 0x5b, 0xb1, 0xa2, 0x2d, 0x68, 0xfa, 0xc4, 0xe7, 0xa3, 0x89, 0x13, 0x39, 0x61, 0x62, 0xc9, 0x1b, - 0xf9, 0x93, 0x69, 0x8c, 0x86, 0xc4, 0xe7, 0x7b, 0x82, 0xd1, 0x06, 0x3f, 0xfd, 0xb6, 0x7e, 0x06, - 0x6b, 0xfb, 0x47, 0xf4, 0x64, 0x9b, 0x06, 0x01, 0x76, 0xb9, 0x4f, 0x09, 0x3b, 0x7f, 0x54, 0x10, - 0xd4, 0xbc, 0x83, 0xe1, 0x40, 0x18, 0x52, 0xb5, 0xc5, 0xb7, 0xc5, 0xe1, 0x6b, 0x25, 0xf9, 0x17, - 0xf1, 0xf9, 0x4d, 0x68, 0xbb, 0xa9, 0xac, 0xe1, 0x20, 0xf6, 0xba, 0x7a, 0xa7, 0x6a, 0xe7, 0x89, - 0xd6, 0xaf, 0x0c, 0xb8, 0x16, 0xab, 0xdd, 0x73, 0x22, 0xee, 0xbf, 0x78, 0xaf, 0x90, 0x05, 0xad, - 0xac, 0xc2, 0x5e, 0x55, 0xec, 0xe5, 0x68, 0xd6, 0xb1, 0x8c, 0x6c, 0xd6, 0x84, 0x8b, 0x38, 0x6e, - 0x41, 0x6b, 0x92, 0x88, 0x9a, 0xf9, 0x9d, 0xa3, 0x59, 0x7f, 0x35, 0xe0, 0xda, 0xc7, 0xd4, 0xf1, - 0x66, 0xd1, 0xfe, 0xd2, 0xdd, 0x46, 0xdf, 0x85, 0xba, 0xbc, 0x87, 0xbd, 0x9a, 0xd0, 0xf5, 0x56, - 0x5e, 0x97, 0xba, 0xa3, 0x33, 0x0b, 0xf7, 0x05, 0xc1, 0x56, 0x87, 0xac, 0x5f, 0x1b, 0xd0, 0xb3, - 0x71, 0x80, 0x1d, 0x86, 0x2f, 0xd1, 0x0b, 0xeb, 0x0b, 0x15, 0xc9, 0x4b, 0x2c, 0xa0, 0x52, 0xc6, - 0x6b, 0xe5, 0x8c, 0x67, 0xa2, 0xbd, 0x7c, 0x9e, 0x68, 0xff, 0x69, 0x16, 0xed, 0xaf, 0xb8, 0xa7, - 0xd6, 0x4f, 0xe1, 0xfa, 0x76, 0x84, 0x1d, 0x8e, 0x7f, 0x14, 0xff, 0x18, 0xb6, 0x8f, 0x1c, 0x42, - 0x70, 0x90, 0x98, 0x5a, 0x54, 0x62, 0x68, 0x94, 0xf4, 0x60, 0x65, 0x12, 0xd1, 0x27, 0x4f, 0x53, - 0xfb, 0x92, 0xa5, 0xf5, 0x07, 0x03, 0xfa, 0x3a, 0xd9, 0x17, 0xb9, 0xae, 0xb7, 0xa1, 0x1b, 0x49, - 0xe3, 0x46, 0xae, 0x94, 0x27, 0xb4, 0x36, 0xec, 0x8e, 0x22, 0x2b, 0x2d, 0xe8, 0x2d, 0xe8, 0x44, - 0x98, 0x4d, 0x83, 0x19, 0x5f, 0x55, 0xf0, 0xb5, 0x25, 0x55, 0xb1, 0x59, 0x7f, 0x36, 0xe0, 0xfa, - 0x0e, 0xe6, 0x69, 0x96, 0x62, 0x75, 0xf8, 0x2b, 0x9a, 0xaa, 0x10, 0xba, 0x05, 0x3b, 0xd1, 0x4d, - 0x68, 0x66, 0x58, 0x54, 0x7e, 0xb2, 0x24, 0xf4, 0x1d, 0x58, 0x8e, 0x43, 0x87, 0x85, 0x45, 0x9d, - 0x4d, 0x6b, 0xa3, 0xdc, 0x0d, 0x6c, 0xe4, 0xa5, 0xda, 0xf2, 0x80, 0xf5, 0x17, 0x03, 0xfa, 0xba, - 0xd0, 0x5c, 0x24, 0x7d, 0x0f, 0x61, 0x2d, 0x35, 0x6e, 0xe4, 0x61, 0xe6, 0x46, 0xfe, 0x44, 0x5c, - 0x0e, 0x81, 0xbb, 0xcd, 0xcd, 0x5b, 0x67, 0x9b, 0xc7, 0xec, 0x6b, 0xa9, 0x88, 0x41, 0x46, 0x82, - 0xe5, 0xc3, 0xb5, 0x1d, 0xcc, 0xf7, 0xf1, 0x38, 0xc4, 0x84, 0x0f, 0xc9, 0x21, 0x3d, 0x7f, 0x16, - 0x5f, 0x03, 0x60, 0x4a, 0x4e, 0xfa, 0x4b, 0xc8, 0x50, 0xac, 0xbf, 0x55, 0xa0, 0x99, 0x51, 0x84, - 0x5e, 0x85, 0x46, 0xba, 0xab, 0x92, 0x30, 0x23, 0x94, 0xf2, 0x5f, 0xd1, 0xe4, 0xbf, 0x90, 0xc8, - 0x6a, 0x39, 0x91, 0x6b, 0x50, 0x27, 0xd4, 0xc3, 0xc3, 0x81, 0xf8, 0x01, 0x54, 0x6d, 0xb5, 0x42, - 0xd7, 0x61, 0x35, 0xc4, 0xe1, 0x88, 0xf9, 0xcf, 0xb0, 0x00, 0xab, 0xaa, 0xbd, 0x12, 0xe2, 0x70, - 0xdf, 0x7f, 0x86, 0xe3, 0x2d, 0x32, 0x0d, 0x47, 0x11, 0x3d, 0x61, 0xbd, 0xba, 0xdc, 0x22, 0xd3, - 0xd0, 0xa6, 0x27, 0x0c, 0xdd, 0x00, 0xf0, 0x89, 0x87, 0x9f, 0x8c, 0x88, 0x13, 0xe2, 0xde, 0x8a, - 0xb8, 0x1a, 0x0d, 0x41, 0xd9, 0x75, 0x42, 0x1c, 0x5f, 0x6a, 0xb1, 0x18, 0x0e, 0x7a, 0xab, 0xf2, - 0xa0, 0x5a, 0xc6, 0xae, 0xaa, 0x0b, 0x35, 0x1c, 0xf4, 0x1a, 0xf2, 0x5c, 0x4a, 0x40, 0x1f, 0x41, - 0x5b, 0xf9, 0x3d, 0x92, 0x55, 0x07, 0xa2, 0xea, 0x6e, 0xea, 0xd2, 0xaa, 0x02, 0x28, 0x6b, 0xae, - 0xc5, 0x32, 0xab, 0xf8, 0x6f, 0xb5, 0x56, 0xcc, 0xe5, 0x45, 0xca, 0xee, 0x5b, 0xb0, 0xec, 0x93, - 0x43, 0x9a, 0x54, 0xd9, 0xeb, 0xa7, 0x98, 0x23, 0x94, 0x49, 0x6e, 0xeb, 0xef, 0x06, 0xac, 0xdd, - 0xf3, 0x3c, 0x1d, 0x32, 0x3e, 0x7f, 0x4d, 0xcd, 0xf2, 0x57, 0xc9, 0xe5, 0x6f, 0x11, 0x74, 0x78, - 0x07, 0x5e, 0x29, 0xa0, 0x9e, 0x2a, 0x83, 0x86, 0x6d, 0xe6, 0x71, 0x6f, 0x38, 0x40, 0x6f, 0x83, - 0x99, 0x47, 0xbe, 0xe1, 0x40, 0x14, 0x46, 0xc3, 0xee, 0xe6, 0xb0, 0x6f, 0x38, 0xb0, 0xfe, 0x69, - 0xc0, 0x75, 0x1b, 0x87, 0xf4, 0x31, 0xfe, 0xdf, 0xf5, 0xf1, 0x5f, 0x15, 0x58, 0xfb, 0x89, 0xc3, - 0xdd, 0xa3, 0x41, 0xa8, 0x88, 0xec, 0x72, 0x1c, 0x2c, 0x5c, 0xf1, 0x5a, 0xf9, 0x8a, 0xa7, 0x65, - 0xba, 0xac, 0x2b, 0xd3, 0xf8, 0xe1, 0xb5, 0xf1, 0x59, 0xe2, 0xef, 0xac, 0x4c, 0x33, 0xcd, 0x4a, - 0xfd, 0x1c, 0xcd, 0x0a, 0xda, 0x86, 0x36, 0x7e, 0xe2, 0x06, 0x53, 0x0f, 0x8f, 0xa4, 0xf6, 0x15, - 0xa1, 0xfd, 0x35, 0x8d, 0xf6, 0xec, 0x1d, 0x69, 0xa9, 0x43, 0x43, 0x71, 0x55, 0xfe, 0x6d, 0x40, - 0x57, 0xed, 0xc6, 0xfd, 0xdd, 0x02, 0xa8, 0x58, 0x08, 0x47, 0xa5, 0x1c, 0x8e, 0x45, 0x82, 0x9a, - 0xfc, 0x6f, 0x6b, 0x99, 0xff, 0xed, 0x0d, 0x80, 0xc3, 0x60, 0xca, 0x8e, 0x46, 0xdc, 0x0f, 0x13, - 0x4c, 0x6c, 0x08, 0xca, 0x03, 0x3f, 0xc4, 0xe8, 0x1e, 0xb4, 0x0e, 0x7c, 0x12, 0xd0, 0xf1, 0x68, - 0xe2, 0xf0, 0xa3, 0x18, 0x19, 0xe7, 0xb9, 0x7b, 0xdf, 0xc7, 0x81, 0xb7, 0x25, 0x78, 0xed, 0xa6, - 0x3c, 0xb3, 0x17, 0x1f, 0xb1, 0xfe, 0x58, 0x81, 0x2b, 0xb1, 0x9b, 0xca, 0xe3, 0x97, 0x50, 0x50, - 0x1f, 0x26, 0xa5, 0x50, 0x9d, 0xff, 0x5f, 0x2c, 0xc4, 0xbb, 0x5c, 0x0e, 0xe7, 0x79, 0x29, 0xa0, - 0x1f, 0x42, 0x27, 0xa0, 0x8e, 0x37, 0x72, 0x29, 0xf1, 0x44, 0x26, 0x44, 0x04, 0x3b, 0x9b, 0x6f, - 0xea, 0x4c, 0x78, 0x10, 0xf9, 0xe3, 0x31, 0x8e, 0xb6, 0x13, 0x5e, 0xbb, 0x1d, 0x88, 0x77, 0x92, - 0x5a, 0x0a, 0x04, 0x55, 0x8d, 0xf0, 0xcb, 0x8b, 0x55, 0x52, 0x03, 0xd5, 0x53, 0x7a, 0xae, 0xda, - 0x02, 0x3d, 0xd7, 0xb2, 0xe6, 0x21, 0x90, 0xef, 0x04, 0xea, 0xa5, 0x4e, 0xe0, 0x01, 0xb4, 0x53, - 0x5c, 0x11, 0x45, 0x7f, 0x0b, 0xda, 0xd2, 0xac, 0x51, 0x1c, 0x09, 0xec, 0x25, 0x3d, 0xb3, 0x24, - 0x7e, 0x2c, 0x68, 0xb1, 0xd4, 0x14, 0xb7, 0xe4, 0x4f, 0xa9, 0x61, 0x67, 0x28, 0xd6, 0xef, 0x0c, - 0x30, 0xb3, 0x88, 0x2c, 0x24, 0x2f, 0xd2, 0x8c, 0xdf, 0x86, 0xae, 0x1c, 0x01, 0xcd, 0x60, 0x51, - 0xb5, 0xc7, 0xc7, 0x59, 0x71, 0x03, 0xf4, 0x01, 0xac, 0x49, 0xc6, 0x12, 0x8c, 0xca, 0x36, 0xf9, - 0xaa, 0xd8, 0xb5, 0x0b, 0x58, 0xfa, 0x0f, 0x03, 0x3a, 0xb3, 0xc2, 0x59, 0xd8, 0xaa, 0x05, 0xde, - 0xd8, 0xe8, 0x3e, 0xb4, 0x95, 0x0d, 0xa3, 0x6c, 0xe1, 0xbf, 0xa1, 0xab, 0xba, 0x5c, 0xc4, 0xed, - 0x56, 0x06, 0x12, 0x2f, 0x0a, 0x86, 0xd6, 0x2f, 0xa0, 0xfb, 0x7d, 0x87, 0x78, 0xf4, 0xf0, 0x30, - 0xa9, 0xd7, 0x73, 0x14, 0xea, 0x87, 0xf9, 0x76, 0xe3, 0x39, 0x2e, 0xaf, 0xf5, 0xfb, 0x0a, 0xac, - 0xc5, 0xb4, 0x2d, 0x27, 0x70, 0x88, 0x8b, 0x17, 0x6f, 0x32, 0x5f, 0x0c, 0x9c, 0xde, 0x82, 0x36, - 0xa3, 0xd3, 0xc8, 0xc5, 0xa3, 0x5c, 0xaf, 0xd9, 0x92, 0xc4, 0x5d, 0x79, 0xdf, 0x6e, 0x00, 0x78, - 0x8c, 0x27, 0x1c, 0x0a, 0x5f, 0x3d, 0xc6, 0xd5, 0xf6, 0xeb, 0xd0, 0x54, 0x32, 0x3c, 0x4a, 0xb0, - 0x48, 0xc3, 0xaa, 0x0d, 0x92, 0x34, 0xa0, 0x44, 0xb4, 0xa5, 0xf1, 0x79, 0xb1, 0xbb, 0x22, 0x76, - 0x57, 0x3c, 0xc6, 0xc5, 0xd6, 0x0d, 0x80, 0xc7, 0x4e, 0xe0, 0x7b, 0xa2, 0x06, 0x44, 0xeb, 0xb9, - 0x6a, 0x37, 0x04, 0x25, 0x0e, 0x81, 0xf5, 0x1b, 0x43, 0xe2, 0x6e, 0x3e, 0x3a, 0xe7, 0x49, 0xd1, - 0xf7, 0xf2, 0x29, 0x5a, 0xd7, 0xa5, 0x48, 0x9f, 0x07, 0x95, 0xa9, 0xf5, 0x67, 0xd0, 0xc9, 0x3f, - 0x4c, 0x50, 0x0b, 0x56, 0x77, 0x29, 0xff, 0xe8, 0x89, 0xcf, 0xb8, 0xb9, 0x84, 0x3a, 0x00, 0xbb, - 0x94, 0xef, 0x45, 0x98, 0x61, 0xc2, 0x4d, 0x03, 0x01, 0xd4, 0x3f, 0x25, 0x03, 0x9f, 0x7d, 0x6e, - 0x56, 0xd0, 0x15, 0xf5, 0x92, 0x73, 0x82, 0x21, 0xf9, 0x04, 0x87, 0x34, 0x7a, 0x6a, 0x56, 0xe3, - 0xe3, 0xe9, 0xaa, 0x86, 0x4c, 0x68, 0xa5, 0x2c, 0x3b, 0x7b, 0x3f, 0x36, 0x97, 0x51, 0x03, 0x96, - 0xe5, 0x67, 0x7d, 0xfd, 0x53, 0x30, 0x8b, 0xc8, 0x8b, 0x9a, 0xb0, 0x72, 0x24, 0x2b, 0xd7, 0x5c, - 0x42, 0x5d, 0x68, 0x06, 0x33, 0xeb, 0x4d, 0x23, 0x26, 0x8c, 0xa3, 0x89, 0xab, 0xc0, 0xd7, 0xac, - 0xc4, 0xda, 0xe2, 0x04, 0x0e, 0xe8, 0x09, 0x31, 0xab, 0xeb, 0x3f, 0x80, 0x56, 0xb6, 0x1d, 0x47, - 0xab, 0x50, 0xdb, 0xa5, 0x04, 0x9b, 0x4b, 0xb1, 0xd8, 0x9d, 0x88, 0x9e, 0xf8, 0x64, 0x2c, 0x7d, - 0xb8, 0x1f, 0xd1, 0x67, 0x98, 0x98, 0x95, 0x78, 0x83, 0x61, 0x27, 0x88, 0x37, 0xaa, 0xf1, 0x46, - 0xbc, 0xc0, 0x9e, 0x59, 0xdb, 0xfc, 0x0f, 0x40, 0x4b, 0x80, 0xd7, 0xbe, 0x1c, 0x12, 0xa3, 0x09, - 0xa0, 0x1d, 0xcc, 0xb7, 0x69, 0x38, 0xa1, 0x24, 0xd1, 0xc0, 0xd0, 0x7b, 0x73, 0x06, 0xaa, 0x65, - 0x56, 0x65, 0x74, 0xff, 0x1b, 0x73, 0x4e, 0x14, 0xd8, 0xad, 0x25, 0x14, 0x0a, 0x8d, 0xf1, 0xdf, - 0xfe, 0x81, 0xef, 0x7e, 0x9e, 0x8c, 0x04, 0x4e, 0xd1, 0x58, 0x60, 0x4d, 0x34, 0x16, 0x6e, 0xae, - 0x5a, 0xec, 0xf3, 0xc8, 0x27, 0xe3, 0xe4, 0x45, 0x62, 0x2d, 0xa1, 0x63, 0xb8, 0x1a, 0xbf, 0x56, - 0xb8, 0xc3, 0x7d, 0xc6, 0x7d, 0x97, 0x25, 0x0a, 0x37, 0xe7, 0x2b, 0x2c, 0x31, 0x3f, 0xa7, 0x4a, - 0x17, 0x5a, 0xd9, 0x81, 0x37, 0xba, 0xad, 0x2b, 0x60, 0xcd, 0x50, 0xbe, 0x7f, 0xe7, 0x6c, 0xc6, - 0x54, 0x49, 0x00, 0xdd, 0xc2, 0x90, 0x19, 0x69, 0x2f, 0x8a, 0x7e, 0xd2, 0xdd, 0x7f, 0x67, 0x21, - 0xde, 0x54, 0x9b, 0x0f, 0x9d, 0xfc, 0x60, 0x17, 0xbd, 0x3d, 0x4f, 0x40, 0x69, 0xa8, 0xd6, 0x5f, - 0x5f, 0x84, 0x35, 0x55, 0xf5, 0x10, 0x3a, 0xf9, 0x29, 0xa4, 0x5e, 0x95, 0x76, 0x52, 0xd9, 0x3f, - 0xed, 0xc5, 0x69, 0x2d, 0xa1, 0x9f, 0xc3, 0x2b, 0xa5, 0xd1, 0x1f, 0xfa, 0xa6, 0x3e, 0xea, 0xfa, - 0x09, 0xe1, 0x59, 0x1a, 0x94, 0xf5, 0xb3, 0x28, 0xce, 0xb7, 0xbe, 0x34, 0xeb, 0x5d, 0xdc, 0xfa, - 0x8c, 0xf8, 0xd3, 0xac, 0x7f, 0x6e, 0x0d, 0x53, 0x40, 0xe5, 0xa1, 0x20, 0x7a, 0x57, 0xa7, 0x62, - 0xee, 0x60, 0xb2, 0xbf, 0xb1, 0x28, 0x7b, 0x9a, 0xf2, 0xa9, 0x80, 0x84, 0xe2, 0xfc, 0x4c, 0xab, - 0x76, 0xee, 0x3c, 0x50, 0xaf, 0x76, 0xfe, 0x8c, 0x4c, 0x16, 0x75, 0x7e, 0x90, 0xa1, 0xcf, 0x95, - 0x76, 0x70, 0xa5, 0x2f, 0x6a, 0xfd, 0x5c, 0xc4, 0x5a, 0xda, 0xfc, 0x62, 0x15, 0x1a, 0xc2, 0x79, - 0x01, 0x08, 0xff, 0x07, 0xdd, 0x17, 0x0f, 0xba, 0x8f, 0xa0, 0x5b, 0x18, 0x07, 0xe9, 0xf1, 0x50, - 0x3f, 0x33, 0x3a, 0xeb, 0x62, 0x1c, 0x00, 0x2a, 0xcf, 0x62, 0xf4, 0x15, 0x3a, 0x77, 0x66, 0x73, - 0x96, 0x8e, 0x47, 0xd0, 0x2d, 0xcc, 0x42, 0xf4, 0x1e, 0xe8, 0x07, 0x26, 0x67, 0x49, 0xff, 0x0c, - 0x5a, 0xd9, 0x57, 0xb1, 0xfe, 0xa7, 0xa4, 0x79, 0x37, 0x5f, 0x3e, 0x28, 0xbd, 0x7c, 0xd0, 0x7e, - 0x04, 0xdd, 0xc2, 0x43, 0x58, 0x1f, 0x79, 0xfd, 0x6b, 0xf9, 0x2c, 0xe9, 0x5f, 0x1e, 0xcc, 0x6c, - 0x7d, 0xf0, 0x70, 0x73, 0xec, 0xf3, 0xa3, 0xe9, 0x41, 0x6c, 0xc4, 0x5d, 0x79, 0xf2, 0x5d, 0x9f, - 0xaa, 0xaf, 0xbb, 0xc9, 0x7d, 0xbb, 0x2b, 0x84, 0xdd, 0x15, 0xc2, 0x26, 0x07, 0x07, 0x75, 0xb1, - 0x7c, 0xff, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x28, 0x13, 0x87, 0x0e, 0xa2, 0x20, 0x00, 0x00, + 0x18, 0x3b, 0x0a, 0x91, 0x53, 0x4a, 0xa0, 0xc8, 0x81, 0x83, 0xa5, 0x8d, 0xc5, 0x42, 0xa2, 0x88, + 0x91, 0x09, 0x85, 0xcb, 0xc5, 0x30, 0x3b, 0xd3, 0xda, 0x9d, 0xca, 0x4c, 0xf7, 0x6a, 0x7a, 0xd6, + 0xb2, 0x7d, 0x80, 0xa2, 0x8a, 0x03, 0x37, 0x4e, 0x9c, 0xe0, 0xc2, 0x85, 0x2a, 0x0e, 0xfc, 0x03, + 0x9c, 0xf2, 0x8f, 0x40, 0x15, 0x45, 0x6e, 0x9c, 0xb9, 0x53, 0xd3, 0xdd, 0x33, 0x3b, 0x1f, 0xbd, + 0xd2, 0x5a, 0xb2, 0x63, 0x17, 0xc5, 0x6d, 0xfb, 0xf5, 0xeb, 0xf7, 0xdd, 0xbf, 0x79, 0xfd, 0x16, + 0x2e, 0x1d, 0xcf, 0x70, 0xf8, 0xc4, 0x62, 0x38, 0x7c, 0xe4, 0x39, 0x78, 0x6b, 0x1a, 0xd2, 0x88, + 0x22, 0x14, 0x78, 0xfe, 0xa3, 0x19, 0x13, 0xab, 0x2d, 0xce, 0xd1, 0x6f, 0x39, 0x34, 0x08, 0x28, + 0x11, 0xb4, 0x7e, 0x2b, 0xcb, 0xd1, 0xef, 0x78, 0x24, 0xc2, 0x21, 0xb1, 0xfd, 0x64, 0x97, 0x39, + 0x13, 0x1c, 0xd8, 0x72, 0x85, 0x5c, 0x3b, 0xb2, 0xf3, 0x1a, 0x8c, 0x5f, 0xc1, 0x25, 0x13, 0x8f, + 0x3d, 0x16, 0xe1, 0x70, 0x9f, 0xba, 0xd8, 0xc4, 0xc7, 0x33, 0xcc, 0x22, 0xf4, 0x3e, 0xd4, 0x46, + 0x36, 0xc3, 0x3d, 0xed, 0xba, 0x76, 0xbb, 0xb9, 0xfd, 0xfa, 0x56, 0xce, 0x0e, 0x69, 0xc0, 0xa7, + 0x6c, 0xbc, 0x63, 0x33, 0x6c, 0x72, 0x4e, 0xf4, 0x5d, 0x58, 0xb3, 0x5d, 0x37, 0xc4, 0x8c, 0xf5, + 0x2a, 0xa7, 0x1c, 0xba, 0x2b, 0x78, 0xcc, 0x84, 0xd9, 0xf8, 0x9d, 0x06, 0x97, 0xf3, 0x16, 0xb0, + 0x29, 0x25, 0x0c, 0xa3, 0x0f, 0xa0, 0xce, 0x22, 0x3b, 0x9a, 0x31, 0x69, 0xc4, 0x37, 0x95, 0xf2, + 0x0e, 0x39, 0x8b, 0x29, 0x59, 0xd1, 0x0e, 0x34, 0x3d, 0xe2, 0x45, 0xd6, 0xd4, 0x0e, 0xed, 0x20, + 0xb1, 0xe4, 0xad, 0xfc, 0xc9, 0x34, 0x46, 0x43, 0xe2, 0x45, 0x07, 0x9c, 0xd1, 0x04, 0x2f, 0xfd, + 0x6d, 0xfc, 0x1c, 0x36, 0x0e, 0x27, 0xf4, 0x64, 0x97, 0xfa, 0x3e, 0x76, 0x22, 0x8f, 0x12, 0x76, + 0xfe, 0xa8, 0x20, 0xa8, 0xb9, 0xa3, 0xe1, 0x80, 0x1b, 0x52, 0x35, 0xf9, 0x6f, 0x23, 0x82, 0x6f, + 0x94, 0xe4, 0x5f, 0xc4, 0xe7, 0xb7, 0xa1, 0xed, 0xa4, 0xb2, 0x86, 0x83, 0xd8, 0xeb, 0xea, 0xed, + 0xaa, 0x99, 0x27, 0x1a, 0xbf, 0xd6, 0xe0, 0x4a, 0xac, 0xf6, 0xc0, 0x0e, 0x23, 0xef, 0xf9, 0x7b, + 0x85, 0x0c, 0x68, 0x65, 0x15, 0xf6, 0xaa, 0x7c, 0x2f, 0x47, 0x33, 0x8e, 0x45, 0x64, 0xb3, 0x26, + 0x5c, 0xc4, 0x71, 0x03, 0x5a, 0xd3, 0x44, 0xd4, 0xdc, 0xef, 0x1c, 0xcd, 0xf8, 0x52, 0x83, 0x2b, + 0x9f, 0x50, 0xdb, 0x9d, 0x47, 0xfb, 0x6b, 0x77, 0x1b, 0x7d, 0x1f, 0xea, 0xe2, 0x1e, 0xf6, 0x6a, + 0x5c, 0xd7, 0xcd, 0xbc, 0x2e, 0x79, 0x47, 0xe7, 0x16, 0x1e, 0x72, 0x82, 0x29, 0x0f, 0x19, 0x7f, + 0xd4, 0xa0, 0x67, 0x62, 0x1f, 0xdb, 0x0c, 0xbf, 0x4c, 0x2f, 0x36, 0xa0, 0x4e, 0xa8, 0x8b, 0x87, + 0x03, 0xee, 0x45, 0xd5, 0x94, 0x2b, 0xe3, 0x2b, 0x19, 0xe1, 0x97, 0x58, 0x58, 0xa5, 0x4a, 0xa8, + 0x95, 0x2b, 0x21, 0x93, 0x85, 0xd5, 0xf3, 0x64, 0xe1, 0xcb, 0x79, 0x16, 0x5e, 0x75, 0x4f, 0xe7, + 0x99, 0x5a, 0xcd, 0x65, 0xea, 0x67, 0x70, 0x75, 0x37, 0xc4, 0x76, 0x84, 0x7f, 0x1c, 0x7f, 0x48, + 0x76, 0x27, 0x36, 0x21, 0xd8, 0x4f, 0x5c, 0x28, 0x2a, 0xd7, 0x14, 0xca, 0x7b, 0xb0, 0x36, 0x0d, + 0xe9, 0xe3, 0x27, 0xa9, 0xdd, 0xc9, 0xd2, 0xf8, 0x93, 0x06, 0x7d, 0x95, 0xec, 0x8b, 0x5c, 0xef, + 0x5b, 0xd0, 0x0d, 0x85, 0x71, 0x96, 0x23, 0xe4, 0x71, 0xad, 0x0d, 0xb3, 0x23, 0xc9, 0x52, 0x0b, + 0xba, 0x09, 0x9d, 0x10, 0xb3, 0x99, 0x3f, 0xe7, 0xab, 0x72, 0xbe, 0xb6, 0xa0, 0x4a, 0x36, 0xe3, + 0x2f, 0x1a, 0x5c, 0xdd, 0xc3, 0x51, 0x9a, 0xbd, 0x58, 0x1d, 0x7e, 0x35, 0x53, 0x68, 0x04, 0xd0, + 0x2d, 0xd8, 0x89, 0xae, 0x43, 0x33, 0xc3, 0x22, 0xf3, 0x93, 0x25, 0xa1, 0xef, 0xc1, 0x6a, 0x1c, + 0x3a, 0xcc, 0x2d, 0xea, 0x6c, 0x1b, 0x5b, 0xe5, 0xee, 0x61, 0x2b, 0x2f, 0xd5, 0x14, 0x07, 0x8c, + 0xbf, 0x6a, 0xd0, 0x57, 0x85, 0xe6, 0x22, 0xe9, 0x7b, 0x00, 0x1b, 0xa9, 0x71, 0x96, 0x8b, 0x99, + 0x13, 0x7a, 0x53, 0x7e, 0x69, 0x38, 0x4e, 0x37, 0xb7, 0x6f, 0x9c, 0x6d, 0x1e, 0x33, 0xaf, 0xa4, + 0x22, 0x06, 0x19, 0x09, 0x86, 0x07, 0x57, 0xf6, 0x70, 0x74, 0x88, 0xc7, 0x01, 0x26, 0xd1, 0x90, + 0x1c, 0xd1, 0xf3, 0x67, 0xf1, 0x0d, 0x00, 0x26, 0xe5, 0xa4, 0x9f, 0x90, 0x0c, 0xc5, 0xf8, 0x7b, + 0x05, 0x9a, 0x19, 0x45, 0xe8, 0x75, 0x68, 0xa4, 0xbb, 0x32, 0x09, 0x73, 0x42, 0x29, 0xff, 0x15, + 0x45, 0xfe, 0x0b, 0x89, 0xac, 0x96, 0x13, 0xb9, 0x00, 0x6a, 0xd1, 0x55, 0x58, 0x0f, 0x70, 0x60, + 0x31, 0xef, 0x29, 0x96, 0x57, 0x7b, 0x2d, 0xc0, 0xc1, 0xa1, 0xf7, 0x14, 0xc7, 0x5b, 0x64, 0x16, + 0x58, 0x21, 0x3d, 0x61, 0xbd, 0xba, 0xd8, 0x22, 0xb3, 0xc0, 0xa4, 0x27, 0x0c, 0x5d, 0x03, 0xf0, + 0x88, 0x8b, 0x1f, 0x5b, 0xc4, 0x0e, 0x70, 0x6f, 0x8d, 0x5f, 0x8d, 0x06, 0xa7, 0xec, 0xdb, 0x01, + 0x8e, 0x2f, 0x35, 0x5f, 0x0c, 0x07, 0xbd, 0x75, 0x71, 0x50, 0x2e, 0x63, 0x57, 0xe5, 0x85, 0x1a, + 0x0e, 0x7a, 0x0d, 0x71, 0x2e, 0x25, 0xa0, 0x8f, 0xa1, 0x2d, 0xfd, 0xb6, 0x44, 0xd5, 0x01, 0xaf, + 0xba, 0xeb, 0xaa, 0xb4, 0xca, 0x00, 0x8a, 0x9a, 0x6b, 0xb1, 0xcc, 0xca, 0xf8, 0x8d, 0x06, 0x1b, + 0xc5, 0x5c, 0x5e, 0xa4, 0xec, 0xbe, 0x03, 0xab, 0x1e, 0x39, 0xa2, 0x49, 0x95, 0xbd, 0x79, 0x8a, + 0x39, 0x5c, 0x99, 0xe0, 0x36, 0xfe, 0xa1, 0xc1, 0xc6, 0x5d, 0xd7, 0x55, 0x21, 0xe3, 0xb3, 0xd7, + 0xd4, 0x3c, 0x7f, 0x95, 0x5c, 0xfe, 0x96, 0x41, 0x87, 0x77, 0xe1, 0xb5, 0x02, 0xea, 0xc9, 0x32, + 0x68, 0x98, 0x7a, 0x1e, 0xf7, 0x86, 0x03, 0xf4, 0x0e, 0xe8, 0x79, 0xe4, 0x93, 0x98, 0xdf, 0x30, + 0xbb, 0x39, 0xec, 0x1b, 0x0e, 0x8c, 0x7f, 0x6a, 0x70, 0xd5, 0xc4, 0x01, 0x7d, 0x84, 0xff, 0x77, + 0x7d, 0xfc, 0x57, 0x05, 0x36, 0x7e, 0x6a, 0x47, 0xce, 0x64, 0x10, 0x48, 0x22, 0x7b, 0x39, 0x0e, + 0x16, 0xae, 0x78, 0xad, 0x7c, 0xc5, 0xd3, 0x32, 0x5d, 0x55, 0x95, 0x69, 0xfc, 0x50, 0xdb, 0xfa, + 0x3c, 0xf1, 0x77, 0x5e, 0xa6, 0x99, 0x26, 0xa6, 0x7e, 0x8e, 0x26, 0x06, 0xed, 0x42, 0x1b, 0x3f, + 0x76, 0xfc, 0x99, 0x8b, 0x2d, 0xa1, 0x7d, 0x8d, 0x6b, 0x7f, 0x43, 0xa1, 0x3d, 0x7b, 0x47, 0x5a, + 0xf2, 0xd0, 0x90, 0x5f, 0x95, 0x7f, 0x6b, 0xd0, 0x95, 0xbb, 0x71, 0xdf, 0xb7, 0x04, 0x2a, 0x16, + 0xc2, 0x51, 0x29, 0x87, 0x63, 0x99, 0xa0, 0x26, 0xdf, 0xdb, 0x5a, 0xe6, 0x7b, 0x7b, 0x0d, 0xe0, + 0xc8, 0x9f, 0xb1, 0x89, 0x15, 0x79, 0x41, 0x82, 0x89, 0x0d, 0x4e, 0xb9, 0xef, 0x05, 0x18, 0xdd, + 0x85, 0xd6, 0xc8, 0x23, 0x3e, 0x1d, 0x5b, 0x53, 0x3b, 0x9a, 0xc4, 0xc8, 0xb8, 0xc8, 0xdd, 0x7b, + 0x1e, 0xf6, 0xdd, 0x1d, 0xce, 0x6b, 0x36, 0xc5, 0x99, 0x83, 0xf8, 0x88, 0xf1, 0xe7, 0x0a, 0x5c, + 0x8a, 0xdd, 0x94, 0x1e, 0xbf, 0x80, 0x82, 0xfa, 0x28, 0x29, 0x85, 0xea, 0xe2, 0xef, 0x62, 0x21, + 0xde, 0xe5, 0x72, 0x38, 0xcf, 0xcb, 0x02, 0xfd, 0x08, 0x3a, 0x3e, 0xb5, 0x5d, 0xcb, 0xa1, 0xc4, + 0xe5, 0x99, 0xe0, 0x11, 0xec, 0x6c, 0xbf, 0xad, 0x32, 0xe1, 0x7e, 0xe8, 0x8d, 0xc7, 0x38, 0xdc, + 0x4d, 0x78, 0xcd, 0xb6, 0xcf, 0xdf, 0x55, 0x72, 0xc9, 0x11, 0x54, 0x36, 0xc8, 0x2f, 0x2e, 0x56, + 0x49, 0x0d, 0x54, 0x4f, 0xe9, 0xb9, 0x6a, 0x4b, 0xf4, 0x5c, 0xab, 0x8a, 0xb6, 0x39, 0xdf, 0x09, + 0xd4, 0x4b, 0x9d, 0xc0, 0x7d, 0x68, 0xa7, 0xb8, 0xc2, 0x8b, 0xfe, 0x06, 0xb4, 0x85, 0x59, 0x56, + 0x1c, 0x09, 0xec, 0x26, 0x3d, 0xb3, 0x20, 0x7e, 0xc2, 0x69, 0xb1, 0xd4, 0x14, 0xb7, 0xc4, 0x47, + 0xa9, 0x61, 0x66, 0x28, 0xc6, 0xef, 0x35, 0xd0, 0xb3, 0x88, 0xcc, 0x25, 0x2f, 0xd3, 0x8c, 0xdf, + 0x82, 0xae, 0x18, 0x19, 0xcd, 0x61, 0x51, 0xb6, 0xc7, 0xc7, 0x59, 0x71, 0x03, 0xf4, 0x21, 0x6c, + 0x08, 0xc6, 0x12, 0x8c, 0x8a, 0x36, 0xf9, 0x32, 0xdf, 0x35, 0x0b, 0x58, 0xfa, 0xdb, 0x0a, 0x74, + 0xe6, 0x85, 0xb3, 0xb4, 0x55, 0x4b, 0xbc, 0xc9, 0xd1, 0x3d, 0x68, 0x4b, 0x1b, 0xac, 0x6c, 0xe1, + 0xbf, 0xa5, 0xaa, 0xba, 0x5c, 0xc4, 0xcd, 0x56, 0x06, 0x12, 0xf9, 0x03, 0x41, 0x96, 0x6f, 0x62, + 0x00, 0xcf, 0xfd, 0xba, 0xd9, 0xf1, 0x73, 0x2f, 0xfe, 0x8b, 0x3e, 0xfd, 0x7e, 0x09, 0xdd, 0x1f, + 0xd8, 0xc4, 0xa5, 0x47, 0x47, 0x49, 0x61, 0x9f, 0xa3, 0xa2, 0x3f, 0xca, 0xf7, 0x25, 0xcf, 0x70, + 0xcb, 0x8d, 0x3f, 0x54, 0x60, 0x23, 0xa6, 0xed, 0xd8, 0xbe, 0x4d, 0x1c, 0xbc, 0x7c, 0x37, 0xfa, + 0x7c, 0x70, 0xf7, 0x06, 0xb4, 0x19, 0x9d, 0x85, 0x0e, 0xb6, 0x72, 0x4d, 0x69, 0x4b, 0x10, 0xf7, + 0xc5, 0xc5, 0xbc, 0x06, 0xe0, 0xb2, 0xc8, 0xca, 0xbd, 0x3b, 0x1b, 0x2e, 0x8b, 0xe4, 0xf6, 0x9b, + 0xd0, 0x94, 0x32, 0x5c, 0x4a, 0x30, 0xff, 0x78, 0xad, 0x9b, 0x20, 0x48, 0x03, 0x4a, 0x78, 0xff, + 0x1a, 0x9f, 0xe7, 0xbb, 0x6b, 0x7c, 0x77, 0xcd, 0x65, 0x11, 0xdf, 0xba, 0x06, 0xf0, 0xc8, 0xf6, + 0x3d, 0x97, 0x17, 0x0b, 0xef, 0x51, 0xd7, 0xcd, 0x06, 0xa7, 0xc4, 0x21, 0x30, 0xfe, 0xa6, 0x01, + 0xca, 0x44, 0xe7, 0xfc, 0x98, 0x73, 0x13, 0x3a, 0x39, 0x3f, 0xd3, 0x41, 0x5a, 0xd6, 0x51, 0x16, + 0x83, 0xe6, 0x48, 0xa8, 0xb2, 0x42, 0x6c, 0x33, 0x4a, 0x78, 0xd0, 0x96, 0x06, 0xcd, 0x51, 0x62, + 0x66, 0x7c, 0x74, 0xf3, 0x29, 0x74, 0xf2, 0x4f, 0x1e, 0xd4, 0x82, 0xf5, 0x7d, 0x1a, 0x7d, 0xfc, + 0xd8, 0x63, 0x91, 0xbe, 0x82, 0x3a, 0x00, 0xfb, 0x34, 0x3a, 0x08, 0x31, 0xc3, 0x24, 0xd2, 0x35, + 0x04, 0x50, 0xff, 0x8c, 0x0c, 0x3c, 0xf6, 0x85, 0x5e, 0x41, 0x97, 0xe4, 0x1b, 0xd1, 0xf6, 0x87, + 0xe4, 0x53, 0x1c, 0xd0, 0xf0, 0x89, 0x5e, 0x8d, 0x8f, 0xa7, 0xab, 0x1a, 0xd2, 0xa1, 0x95, 0xb2, + 0xec, 0x1d, 0xfc, 0x44, 0x5f, 0x45, 0x0d, 0x58, 0x15, 0x3f, 0xeb, 0x9b, 0x9f, 0x81, 0x5e, 0x34, + 0x0f, 0x35, 0x61, 0x6d, 0x22, 0x4a, 0x5d, 0x5f, 0x41, 0x5d, 0x68, 0xfa, 0xf3, 0xc0, 0xea, 0x5a, + 0x4c, 0x18, 0x87, 0x53, 0x47, 0x86, 0x58, 0xaf, 0xc4, 0xda, 0xe2, 0x58, 0x0d, 0xe8, 0x09, 0xd1, + 0xab, 0x9b, 0x3f, 0x84, 0x56, 0xb6, 0xd1, 0x47, 0xeb, 0x50, 0xdb, 0xa7, 0x04, 0xeb, 0x2b, 0xb1, + 0xd8, 0xbd, 0x90, 0x9e, 0x78, 0x64, 0x2c, 0x7c, 0xb8, 0x17, 0xd2, 0xa7, 0x98, 0xe8, 0x95, 0x78, + 0x83, 0x61, 0xdb, 0x8f, 0x37, 0xaa, 0xf1, 0x46, 0xbc, 0xc0, 0xae, 0x5e, 0xdb, 0xfe, 0x0f, 0x40, + 0x8b, 0xc3, 0xe2, 0xa1, 0x18, 0x57, 0xa3, 0x29, 0xa0, 0x3d, 0x1c, 0xed, 0xd2, 0x60, 0x4a, 0x49, + 0xa2, 0x81, 0xa1, 0xf7, 0x17, 0x8c, 0x76, 0xcb, 0xac, 0xd2, 0xe8, 0xfe, 0xb7, 0x16, 0x9c, 0x28, + 0xb0, 0x1b, 0x2b, 0x28, 0xe0, 0x1a, 0xe3, 0x3e, 0xe2, 0xbe, 0xe7, 0x7c, 0x91, 0x0c, 0x1b, 0x4e, + 0xd1, 0x58, 0x60, 0x4d, 0x34, 0x16, 0xae, 0xba, 0x5c, 0x1c, 0x46, 0xa1, 0x47, 0xc6, 0xc9, 0x5b, + 0xc7, 0x58, 0x41, 0xc7, 0x70, 0x39, 0x7e, 0x07, 0x45, 0x76, 0xe4, 0xb1, 0xc8, 0x73, 0x58, 0xa2, + 0x70, 0x7b, 0xb1, 0xc2, 0x12, 0xf3, 0x33, 0xaa, 0x74, 0xa0, 0x95, 0x1d, 0xbd, 0xa3, 0x5b, 0xaa, + 0x12, 0x56, 0xfc, 0x3d, 0xd0, 0xbf, 0x7d, 0x36, 0x63, 0xaa, 0xc4, 0x87, 0x6e, 0x61, 0xdc, 0x8d, + 0x36, 0x95, 0xe0, 0xa7, 0x9c, 0xb9, 0xf7, 0xdf, 0x5d, 0x8a, 0x37, 0xd5, 0xe6, 0x41, 0x27, 0x3f, + 0x62, 0x46, 0xef, 0x2c, 0x12, 0x50, 0x1a, 0xe3, 0xf5, 0x37, 0x97, 0x61, 0x4d, 0x55, 0x3d, 0x80, + 0x4e, 0x7e, 0xee, 0xa9, 0x56, 0xa5, 0x9c, 0x8d, 0xf6, 0x4f, 0x7b, 0xcb, 0x1a, 0x2b, 0xe8, 0x17, + 0xf0, 0x5a, 0x69, 0xd8, 0x88, 0xbe, 0xad, 0x8e, 0xba, 0x7a, 0x26, 0x79, 0x96, 0x06, 0x69, 0x7d, + 0xe6, 0x2b, 0xb9, 0xd0, 0xfa, 0xd2, 0xd4, 0x79, 0x79, 0xeb, 0x33, 0xe2, 0x4f, 0xb3, 0xfe, 0x99, + 0x35, 0xcc, 0x00, 0x95, 0xc7, 0x8d, 0xe8, 0x3d, 0x95, 0x8a, 0x85, 0x23, 0xcf, 0xfe, 0xd6, 0xb2, + 0xec, 0x69, 0xca, 0x67, 0x1c, 0x12, 0x8a, 0x93, 0x39, 0xa5, 0xda, 0x85, 0x93, 0x46, 0xb5, 0xda, + 0xc5, 0xd3, 0x37, 0x51, 0xd4, 0xf9, 0x11, 0x89, 0x3a, 0x57, 0xca, 0x91, 0x98, 0xba, 0xa8, 0xd5, + 0x13, 0x17, 0x63, 0x65, 0xfb, 0xab, 0x75, 0x68, 0x70, 0xe7, 0x39, 0x20, 0xfc, 0x1f, 0x74, 0x9f, + 0x3f, 0xe8, 0x3e, 0x84, 0x6e, 0x61, 0xd0, 0xa4, 0xc6, 0x43, 0xf5, 0x34, 0xea, 0xac, 0x8b, 0x31, + 0x02, 0x54, 0x9e, 0xf2, 0xa8, 0x2b, 0x74, 0xe1, 0x34, 0xe8, 0x2c, 0x1d, 0x0f, 0xa1, 0x5b, 0x98, + 0xb2, 0xa8, 0x3d, 0x50, 0x8f, 0x62, 0xce, 0x92, 0xfe, 0x39, 0xb4, 0xb2, 0xef, 0x6d, 0xf5, 0x47, + 0x49, 0xf1, 0x22, 0x7f, 0xf9, 0xa0, 0xf4, 0xe2, 0x41, 0xfb, 0x21, 0x74, 0x0b, 0x4f, 0x6c, 0x75, + 0xe4, 0xd5, 0xef, 0xf0, 0xb3, 0xa4, 0x7f, 0x7d, 0x30, 0xb3, 0xf3, 0xe1, 0x83, 0xed, 0xb1, 0x17, + 0x4d, 0x66, 0xa3, 0xd8, 0x88, 0x3b, 0xe2, 0xe4, 0x7b, 0x1e, 0x95, 0xbf, 0xee, 0x24, 0xf7, 0xed, + 0x0e, 0x17, 0x76, 0x87, 0x0b, 0x9b, 0x8e, 0x46, 0x75, 0xbe, 0xfc, 0xe0, 0xbf, 0x01, 0x00, 0x00, + 0xff, 0xff, 0xcb, 0x4e, 0xe3, 0xca, 0x2c, 0x21, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/internal/querynode/collection_replica.go b/internal/querynode/collection_replica.go index 00bc4ab1bf..84955ed7c1 100644 --- a/internal/querynode/collection_replica.go +++ b/internal/querynode/collection_replica.go @@ -25,6 +25,7 @@ import "C" import ( "errors" "fmt" + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/proto/datapb" "strconv" "sync" @@ -92,6 +93,8 @@ type collectionReplica struct { segments map[UniqueID]*Segment excludedSegments map[UniqueID][]*datapb.SegmentInfo // map[collectionID]segmentIDs + + etcdKV *etcdkv.EtcdKV } //----------------------------------------------------------------------------------------------------- collection @@ -396,6 +399,11 @@ func (colReplica *collectionReplica) removeSegmentPrivate(segmentID UniqueID) er partition.removeSegmentID(segmentID) delete(colReplica.segments, segmentID) deleteSegment(segment) + key := fmt.Sprintf("%s/%d", queryNodeSegmentMetaPrefix, segmentID) + err = colReplica.etcdKV.Remove(key) + if err != nil { + log.Error("error when remove segment info from etcd") + } return nil } @@ -551,7 +559,7 @@ func (colReplica *collectionReplica) freeAll() { colReplica.segments = make(map[UniqueID]*Segment) } -func newCollectionReplica() ReplicaInterface { +func newCollectionReplica(etcdKv *etcdkv.EtcdKV) ReplicaInterface { collections := make(map[UniqueID]*Collection) partitions := make(map[UniqueID]*Partition) segments := make(map[UniqueID]*Segment) @@ -563,6 +571,7 @@ func newCollectionReplica() ReplicaInterface { segments: segments, excludedSegments: excludedSegments, + etcdKV: etcdKv, } return replica diff --git a/internal/querynode/flow_graph_service_time_node.go b/internal/querynode/flow_graph_service_time_node.go index 82db7420b4..66ef3d408a 100644 --- a/internal/querynode/flow_graph_service_time_node.go +++ b/internal/querynode/flow_graph_service_time_node.go @@ -69,7 +69,7 @@ func (stNode *serviceTimeNode) Operate(in []flowgraph.Msg) []flowgraph.Msg { // zap.Int64("tSafe", int64(serviceTimeMsg.timeRange.timestampMax)), // zap.Any("collectionID", stNode.collectionID), // zap.Any("id", id), - // zap.Any("channel", channelTmp), + // zap.Any("channel", stNode.vChannel), //) //if err := stNode.sendTimeTick(serviceTimeMsg.timeRange.timestampMax); err != nil { diff --git a/internal/querynode/historical.go b/internal/querynode/historical.go index 9ab0d970e5..0a8becee3c 100644 --- a/internal/querynode/historical.go +++ b/internal/querynode/historical.go @@ -16,6 +16,7 @@ import ( "errors" "fmt" + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "go.uber.org/zap" "github.com/milvus-io/milvus/internal/log" @@ -27,15 +28,19 @@ type historical struct { replica ReplicaInterface loader *segmentLoader statsService *statsService + + //TODO + globalSealedSegments []UniqueID } func newHistorical(ctx context.Context, masterService types.MasterService, dataService types.DataService, indexService types.IndexService, - factory msgstream.Factory) *historical { - replica := newCollectionReplica() - loader := newSegmentLoader(ctx, masterService, indexService, dataService, replica) + factory msgstream.Factory, + etcdKV *etcdkv.EtcdKV) *historical { + replica := newCollectionReplica(etcdKV) + loader := newSegmentLoader(ctx, masterService, indexService, dataService, replica, etcdKV) ss := newStatsService(ctx, replica, loader.indexLoader.fieldStatsChan, factory) return &historical{ diff --git a/internal/querynode/query_node.go b/internal/querynode/query_node.go index 3b8c024a10..18ccf1d702 100644 --- a/internal/querynode/query_node.go +++ b/internal/querynode/query_node.go @@ -27,19 +27,19 @@ import "C" import ( "context" "errors" - "fmt" + "github.com/milvus-io/milvus/internal/kv" + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" + "github.com/milvus-io/milvus/internal/util/retry" + "go.etcd.io/etcd/clientv3" + "go.uber.org/zap" "math/rand" "strconv" "sync/atomic" "time" - "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/msgstream" - "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/internalpb" - queryPb "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/types" "github.com/milvus-io/milvus/internal/util/sessionutil" "github.com/milvus-io/milvus/internal/util/typeutil" @@ -70,6 +70,9 @@ type QueryNode struct { scheduler *taskScheduler session *sessionutil.Session + + minioKV kv.BaseKV // minio minioKV + etcdKV *etcdkv.EtcdKV } func NewQueryNode(ctx context.Context, queryNodeID UniqueID, factory msgstream.Factory) *QueryNode { @@ -111,57 +114,75 @@ func (node *QueryNode) Register() error { node.session = sessionutil.NewSession(node.queryNodeLoopCtx, Params.MetaRootPath, Params.EtcdEndpoints) node.session.Init(typeutil.QueryNodeRole, Params.QueryNodeIP+":"+strconv.FormatInt(Params.QueryNodePort, 10), false) Params.QueryNodeID = node.session.ServerID + log.Debug("query nodeID", zap.Int64("nodeID", Params.QueryNodeID)) return nil } func (node *QueryNode) Init() error { - ctx := context.Background() + //ctx := context.Background() + connectEtcdFn := func() error { + etcdClient, err := clientv3.New(clientv3.Config{Endpoints: Params.EtcdEndpoints}) + if err != nil { + return err + } + etcdKV := etcdkv.NewEtcdKV(etcdClient, Params.MetaRootPath) + node.etcdKV = etcdKV + return err + } + log.Debug("queryNode try to connect etcd") + err := retry.Retry(100000, time.Millisecond*200, connectEtcdFn) + if err != nil { + log.Debug("queryNode try to connect etcd failed", zap.Error(err)) + return err + } + log.Debug("queryNode try to connect etcd success") node.historical = newHistorical(node.queryNodeLoopCtx, node.masterService, node.dataService, node.indexService, - node.msFactory) - node.streaming = newStreaming(node.queryNodeLoopCtx, node.msFactory) + node.msFactory, + node.etcdKV) + node.streaming = newStreaming(node.queryNodeLoopCtx, node.msFactory, node.etcdKV) C.SegcoreInit() - registerReq := &queryPb.RegisterNodeRequest{ - Base: &commonpb.MsgBase{ - SourceID: Params.QueryNodeID, - }, - Address: &commonpb.Address{ - Ip: Params.QueryNodeIP, - Port: Params.QueryNodePort, - }, - } - - resp, err := node.queryService.RegisterNode(ctx, registerReq) - if err != nil { - log.Debug("QueryNode RegisterNode failed", zap.Error(err)) - panic(err) - } - if resp.Status.ErrorCode != commonpb.ErrorCode_Success { - log.Debug("QueryNode RegisterNode failed", zap.Any("Reason", resp.Status.Reason)) - panic(resp.Status.Reason) - } - log.Debug("QueryNode RegisterNode success") - - for _, kv := range resp.InitParams.StartParams { - switch kv.Key { - case "StatsChannelName": - Params.StatsChannelName = kv.Value - case "TimeTickChannelName": - Params.QueryTimeTickChannelName = kv.Value - case "SearchChannelName": - Params.SearchChannelNames = append(Params.SearchChannelNames, kv.Value) - case "SearchResultChannelName": - Params.SearchResultChannelNames = append(Params.SearchResultChannelNames, kv.Value) - default: - return fmt.Errorf("Invalid key: %v", kv.Key) - } - } - - log.Debug("QueryNode Init ", zap.Int64("QueryNodeID", Params.QueryNodeID), zap.Any("searchChannelNames", Params.SearchChannelNames)) + //registerReq := &queryPb.RegisterNodeRequest{ + // Base: &commonpb.MsgBase{ + // SourceID: Params.QueryNodeID, + // }, + // Address: &commonpb.Address{ + // Ip: Params.QueryNodeIP, + // Port: Params.QueryNodePort, + // }, + //} + // + //resp, err := node.queryService.RegisterNode(ctx, registerReq) + //if err != nil { + // log.Debug("QueryNode RegisterNode failed", zap.Error(err)) + // panic(err) + //} + //if resp.Status.ErrorCode != commonpb.ErrorCode_Success { + // log.Debug("QueryNode RegisterNode failed", zap.Any("Reason", resp.Status.Reason)) + // panic(resp.Status.Reason) + //} + //log.Debug("QueryNode RegisterNode success") + // + //for _, kv := range resp.InitParams.StartParams { + // switch kv.Key { + // case "StatsChannelName": + // Params.StatsChannelName = kv.Value + // case "TimeTickChannelName": + // Params.QueryTimeTickChannelName = kv.Value + // case "SearchChannelName": + // Params.SearchChannelNames = append(Params.SearchChannelNames, kv.Value) + // case "SearchResultChannelName": + // Params.SearchResultChannelNames = append(Params.SearchResultChannelNames, kv.Value) + // default: + // return fmt.Errorf("Invalid key: %v", kv.Key) + // } + //} + // + //log.Debug("QueryNode Init ", zap.Int64("QueryNodeID", Params.QueryNodeID), zap.Any("searchChannelNames", Params.SearchChannelNames)) if node.masterService == nil { log.Error("null master service detected") diff --git a/internal/querynode/query_node_test.go b/internal/querynode/query_node_test.go index b8214c70a0..3351ccd0a6 100644 --- a/internal/querynode/query_node_test.go +++ b/internal/querynode/query_node_test.go @@ -20,7 +20,9 @@ import ( "time" "github.com/stretchr/testify/assert" + "go.etcd.io/etcd/clientv3" + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/msgstream" "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/etcdpb" @@ -163,6 +165,12 @@ func newQueryNodeMock() *QueryNode { }() } + etcdClient, err := clientv3.New(clientv3.Config{Endpoints: Params.EtcdEndpoints}) + if err != nil { + panic(err) + } + etcdKV := etcdkv.NewEtcdKV(etcdClient, Params.MetaRootPath) + msFactory, err := newMessageStreamFactory() if err != nil { panic(err) @@ -172,8 +180,8 @@ func newQueryNodeMock() *QueryNode { if err != nil { panic(err) } - svr.historical = newHistorical(svr.queryNodeLoopCtx, nil, nil, nil, svr.msFactory) - svr.streaming = newStreaming(ctx, msFactory) + svr.historical = newHistorical(svr.queryNodeLoopCtx, nil, nil, nil, svr.msFactory, etcdKV) + svr.streaming = newStreaming(ctx, msFactory, etcdKV) return svr } diff --git a/internal/querynode/search_collection.go b/internal/querynode/search_collection.go index 787b28a35a..35687a34b4 100644 --- a/internal/querynode/search_collection.go +++ b/internal/querynode/search_collection.go @@ -227,33 +227,34 @@ func (s *searchCollection) consumeSearch() { } func (s *searchCollection) loadBalance(msg *msgstream.LoadBalanceSegmentsMsg) { - log.Debug("consume load balance message", - zap.Int64("msgID", msg.ID())) - nodeID := Params.QueryNodeID - for _, info := range msg.Infos { - segmentID := info.SegmentID - if nodeID == info.SourceNodeID { - err := s.historical.replica.removeSegment(segmentID) - if err != nil { - log.Error("loadBalance failed when remove segment", - zap.Error(err), - zap.Any("segmentID", segmentID)) - } - } - if nodeID == info.DstNodeID { - segment, err := s.historical.replica.getSegmentByID(segmentID) - if err != nil { - log.Error("loadBalance failed when making segment on service", - zap.Error(err), - zap.Any("segmentID", segmentID)) - continue // not return, try to load balance all segment - } - segment.setOnService(true) - } - } - log.Debug("load balance done", - zap.Int64("msgID", msg.ID()), - zap.Int("num of segment", len(msg.Infos))) + //TODO:: get loadBalance info from etcd + //log.Debug("consume load balance message", + // zap.Int64("msgID", msg.ID())) + //nodeID := Params.QueryNodeID + //for _, info := range msg.Infos { + // segmentID := info.SegmentID + // if nodeID == info.SourceNodeID { + // err := s.historical.replica.removeSegment(segmentID) + // if err != nil { + // log.Error("loadBalance failed when remove segment", + // zap.Error(err), + // zap.Any("segmentID", segmentID)) + // } + // } + // if nodeID == info.DstNodeID { + // segment, err := s.historical.replica.getSegmentByID(segmentID) + // if err != nil { + // log.Error("loadBalance failed when making segment on service", + // zap.Error(err), + // zap.Any("segmentID", segmentID)) + // continue // not return, try to load balance all segment + // } + // segment.setOnService(true) + // } + //} + //log.Debug("load balance done", + // zap.Int64("msgID", msg.ID()), + // zap.Int("num of segment", len(msg.Infos))) } func (s *searchCollection) receiveSearch(msg *msgstream.SearchMsg) { @@ -346,9 +347,9 @@ func (s *searchCollection) doUnsolvedMsgSearch() { zap.Any("tSafe", st)) s.setServiceableTime(serviceTime) - log.Debug("query node::doUnsolvedMsgSearch: setServiceableTime", - zap.Any("serviceTime", st), - ) + //log.Debug("query node::doUnsolvedMsgSearch: setServiceableTime", + // zap.Any("serviceTime", st), + //) searchMsg := make([]*msgstream.SearchMsg, 0) tempMsg := s.popAllUnsolvedMsg() diff --git a/internal/querynode/segment_loader.go b/internal/querynode/segment_loader.go index b9a7d4e851..640bbc2c1b 100644 --- a/internal/querynode/segment_loader.go +++ b/internal/querynode/segment_loader.go @@ -17,9 +17,11 @@ import ( "fmt" "strconv" + "github.com/golang/protobuf/proto" "go.uber.org/zap" "github.com/milvus-io/milvus/internal/kv" + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" minioKV "github.com/milvus-io/milvus/internal/kv/minio" "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/proto/commonpb" @@ -29,13 +31,19 @@ import ( "github.com/milvus-io/milvus/internal/types" ) +const ( + queryServiceSegmentMetaPrefix = "queryService-segmentMeta" + queryNodeSegmentMetaPrefix = "queryNode-segmentMeta" +) + // segmentLoader is only responsible for loading the field data from binlog type segmentLoader struct { historicalReplica ReplicaInterface dataService types.DataService - kv kv.BaseKV // minio kv + minioKV kv.BaseKV // minio minioKV + etcdKV *etcdkv.EtcdKV indexLoader *indexLoader } @@ -100,6 +108,30 @@ func (loader *segmentLoader) loadSegment(req *queryPb.LoadSegmentsRequest, onSer if err != nil { deleteSegment(segment) log.Error(err.Error()) + continue + } + if onService { + key := fmt.Sprintf("%s/%d", queryServiceSegmentMetaPrefix, segmentID) + value, err := loader.etcdKV.Load(key) + if err != nil { + deleteSegment(segment) + log.Error("error when load segment info from etcd", zap.Any("error", err.Error())) + continue + } + segmentInfo := &queryPb.SegmentInfo{} + err = proto.UnmarshalText(value, segmentInfo) + if err != nil { + deleteSegment(segment) + log.Error("error when unmarshal segment info from etcd", zap.Any("error", err.Error())) + continue + } + segmentInfo.SegmentState = queryPb.SegmentState_sealed + newKey := fmt.Sprintf("%s/%d", queryNodeSegmentMetaPrefix, segmentID) + err = loader.etcdKV.Save(newKey, proto.MarshalTextString(segmentInfo)) + if err != nil { + deleteSegment(segment) + log.Error("error when update segment info to etcd", zap.Any("error", err.Error())) + } } } @@ -211,7 +243,7 @@ func (loader *segmentLoader) loadSegmentFieldsData(segment *Segment, binlogPaths Value: make([]byte, 0), } for _, path := range paths { - binLog, err := loader.kv.Load(path) + binLog, err := loader.minioKV.Load(path) if err != nil { // TODO: return or continue? return err @@ -273,7 +305,7 @@ func (loader *segmentLoader) loadSegmentFieldsData(segment *Segment, binlogPaths return nil } -func newSegmentLoader(ctx context.Context, masterService types.MasterService, indexService types.IndexService, dataService types.DataService, replica ReplicaInterface) *segmentLoader { +func newSegmentLoader(ctx context.Context, masterService types.MasterService, indexService types.IndexService, dataService types.DataService, replica ReplicaInterface, etcdKV *etcdkv.EtcdKV) *segmentLoader { option := &minioKV.Option{ Address: Params.MinioEndPoint, AccessKeyID: Params.MinioAccessKeyID, @@ -294,7 +326,8 @@ func newSegmentLoader(ctx context.Context, masterService types.MasterService, in dataService: dataService, - kv: client, + minioKV: client, + etcdKV: etcdKV, indexLoader: iLoader, } diff --git a/internal/querynode/streaming.go b/internal/querynode/streaming.go index cf23896956..00a89db2d0 100644 --- a/internal/querynode/streaming.go +++ b/internal/querynode/streaming.go @@ -16,6 +16,7 @@ import ( "errors" "fmt" + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "go.uber.org/zap" "github.com/milvus-io/milvus/internal/log" @@ -32,8 +33,8 @@ type streaming struct { msFactory msgstream.Factory } -func newStreaming(ctx context.Context, factory msgstream.Factory) *streaming { - replica := newCollectionReplica() +func newStreaming(ctx context.Context, factory msgstream.Factory, etcdKV *etcdkv.EtcdKV) *streaming { + replica := newCollectionReplica(etcdKV) tReplica := newTSafeReplica() newDS := newDataSyncService(ctx, replica, tReplica, factory) diff --git a/internal/querynode/tsafe.go b/internal/querynode/tsafe.go index ef878aa617..01768de10b 100644 --- a/internal/querynode/tsafe.go +++ b/internal/querynode/tsafe.go @@ -16,8 +16,6 @@ import ( "math" "sync" - "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/log" ) @@ -107,11 +105,11 @@ func (ts *tSafe) start() { watcher.notify() } - log.Debug("set tSafe done", - zap.Any("id", m.id), - zap.Any("channel", ts.channel), - zap.Any("t", m.t), - zap.Any("tSafe", ts.tSafe)) + //log.Debug("set tSafe done", + // zap.Any("id", m.id), + // zap.Any("channel", ts.channel), + // zap.Any("t", m.t), + // zap.Any("tSafe", ts.tSafe)) ts.tSafeMu.Unlock() } } diff --git a/internal/queryservice/cluster.go b/internal/queryservice/cluster.go index 56196a2c0e..3485fde320 100644 --- a/internal/queryservice/cluster.go +++ b/internal/queryservice/cluster.go @@ -13,37 +13,106 @@ package queryservice import ( "context" + "encoding/json" "errors" "fmt" + "path/filepath" + "strconv" "sync" + "github.com/golang/protobuf/proto" "go.uber.org/zap" + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/util/sessionutil" +) + +const ( + queryNodeMetaPrefix = "queryService-queryNodeMeta" + queryNodeInfoPrefix = "queryService-queryNodeInfo" ) type queryNodeCluster struct { + client *etcdkv.EtcdKV + sync.RWMutex clusterMeta *meta nodes map[int64]*queryNode } -func newQueryNodeCluster(clusterMeta *meta) *queryNodeCluster { +func newQueryNodeCluster(clusterMeta *meta, kv *etcdkv.EtcdKV) (*queryNodeCluster, error) { nodes := make(map[int64]*queryNode) - return &queryNodeCluster{ + c := &queryNodeCluster{ + client: kv, clusterMeta: clusterMeta, nodes: nodes, } + err := c.reloadFromKV() + if err != nil { + return nil, err + } + + return c, nil } -func (c *queryNodeCluster) GetComponentInfos(ctx context.Context) []*internalpb.ComponentInfo { +func (c *queryNodeCluster) reloadFromKV() error { + nodeIDs := make([]UniqueID, 0) + keys, values, err := c.client.LoadWithPrefix(queryNodeInfoPrefix) + if err != nil { + return err + } + for index := range keys { + nodeID, err := strconv.ParseInt(filepath.Base(keys[index]), 10, 64) + if err != nil { + return err + } + nodeIDs = append(nodeIDs, nodeID) + session := &sessionutil.Session{} + err = json.Unmarshal([]byte(values[index]), session) + if err != nil { + return err + } + err = c.RegisterNode(session, nodeID) + if err != nil { + return err + } + } + for _, nodeID := range nodeIDs { + infoPrefix := fmt.Sprintf("%s/%d", queryNodeMetaPrefix, nodeID) + collectionKeys, collectionValues, err := c.client.LoadWithPrefix(infoPrefix) + if err != nil { + return err + } + for index := range collectionKeys { + collectionID, err := strconv.ParseInt(filepath.Base(collectionKeys[index]), 10, 64) + if err != nil { + return err + } + collectionInfo := &querypb.CollectionInfo{} + err = proto.UnmarshalText(collectionValues[index], collectionInfo) + if err != nil { + return err + } + c.nodes[nodeID].collectionInfos[collectionID] = collectionInfo + } + } + return nil +} + +func (c *queryNodeCluster) GetComponentInfos(ctx context.Context) ([]*internalpb.ComponentInfo, error) { c.RLock() defer c.RUnlock() subComponentInfos := make([]*internalpb.ComponentInfo, 0) - for nodeID, node := range c.nodes { + nodeIDs, err := c.getOnServiceNodeIDs() + if err != nil { + return nil, err + } + for _, nodeID := range nodeIDs { + node := c.nodes[nodeID] componentStates, err := node.client.GetComponentStates(ctx) if err != nil { subComponentInfos = append(subComponentInfos, &internalpb.ComponentInfo{ @@ -55,49 +124,59 @@ func (c *queryNodeCluster) GetComponentInfos(ctx context.Context) []*internalpb. subComponentInfos = append(subComponentInfos, componentStates.State) } - return subComponentInfos + return subComponentInfos, nil } func (c *queryNodeCluster) LoadSegments(ctx context.Context, nodeID int64, in *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { c.Lock() defer c.Unlock() + if node, ok := c.nodes[nodeID]; ok { - //TODO::etcd - log.Debug("load segment infos", zap.Any("infos", in)) + if !node.isOnService() { + return nil, errors.New("node offline") + } + segmentInfos := make(map[UniqueID]*querypb.SegmentInfo) for _, info := range in.Infos { segmentID := info.SegmentID - if info, ok := c.clusterMeta.segmentInfos[segmentID]; ok { - info.SegmentState = querypb.SegmentState_sealing + segmentInfo, err := c.clusterMeta.getSegmentInfoByID(segmentID) + if err == nil { + segmentInfos[segmentID] = proto.Clone(segmentInfo).(*querypb.SegmentInfo) + if in.LoadCondition != querypb.TriggerCondition_loadBalance { + segmentInfo.SegmentState = querypb.SegmentState_sealing + segmentInfo.NodeID = nodeID + } + } else { + segmentInfo = &querypb.SegmentInfo{ + SegmentID: segmentID, + CollectionID: info.CollectionID, + PartitionID: info.PartitionID, + NodeID: nodeID, + SegmentState: querypb.SegmentState_sealing, + } } - segmentInfo := &querypb.SegmentInfo{ - SegmentID: segmentID, - CollectionID: info.CollectionID, - PartitionID: info.PartitionID, - NodeID: nodeID, - SegmentState: querypb.SegmentState_sealing, - } - c.clusterMeta.segmentInfos[segmentID] = segmentInfo + c.clusterMeta.setSegmentInfo(segmentID, segmentInfo) } status, err := node.client.LoadSegments(ctx, in) if err == nil && status.ErrorCode == commonpb.ErrorCode_Success { for _, info := range in.Infos { - if !c.clusterMeta.hasCollection(info.CollectionID) { - c.clusterMeta.addCollection(info.CollectionID, in.Schema) - } - + c.clusterMeta.addCollection(info.CollectionID, in.Schema) c.clusterMeta.addPartition(info.CollectionID, info.PartitionID) - if !node.hasCollection(info.CollectionID) { - node.addCollection(info.CollectionID, in.Schema) - } + node.addCollection(info.CollectionID, in.Schema) node.addPartition(info.CollectionID, info.PartitionID) } - return status, err - } - for _, info := range in.Infos { - segmentID := info.SegmentID - c.clusterMeta.deleteSegmentInfoByID(segmentID) + } else { + for _, info := range in.Infos { + segmentID := info.SegmentID + if _, ok = segmentInfos[segmentID]; ok { + c.clusterMeta.setSegmentInfo(segmentID, segmentInfos[segmentID]) + continue + } + c.clusterMeta.removeSegmentInfo(segmentID) + c.clusterMeta.deleteSegmentInfoByID(segmentID) + } } + return status, err } return nil, errors.New("Can't find query node by nodeID ") @@ -108,6 +187,15 @@ func (c *queryNodeCluster) ReleaseSegments(ctx context.Context, nodeID int64, in defer c.Unlock() if node, ok := c.nodes[nodeID]; ok { + if !node.isOnService() { + return nil, errors.New("node offline") + } + for _, segmentID := range in.SegmentIDs { + err := c.clusterMeta.removeSegmentInfo(segmentID) + if err != nil { + log.Error("remove segmentInfo Error", zap.Any("error", err.Error()), zap.Int64("segmentID", segmentID)) + } + } status, err := node.client.ReleaseSegments(ctx, in) if err == nil && status.ErrorCode == commonpb.ErrorCode_Success { for _, segmentID := range in.SegmentIDs { @@ -123,7 +211,11 @@ func (c *queryNodeCluster) ReleaseSegments(ctx context.Context, nodeID int64, in func (c *queryNodeCluster) WatchDmChannels(ctx context.Context, nodeID int64, in *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) { c.Lock() defer c.Unlock() + if node, ok := c.nodes[nodeID]; ok { + if !node.isOnService() { + return nil, errors.New("node offline") + } channels := make([]string, 0) for _, info := range in.Infos { channels = append(channels, info.ChannelName) @@ -133,14 +225,13 @@ func (c *queryNodeCluster) WatchDmChannels(ctx context.Context, nodeID int64, in log.Debug("queryNode watch dm channel done") if err == nil && status.ErrorCode == commonpb.ErrorCode_Success { collectionID := in.CollectionID - if !c.clusterMeta.hasCollection(collectionID) { - c.clusterMeta.addCollection(collectionID, in.Schema) - } + c.clusterMeta.addCollection(collectionID, in.Schema) c.clusterMeta.addDmChannel(collectionID, nodeID, channels) - if !node.hasCollection(collectionID) { - node.addCollection(collectionID, in.Schema) - } + + node.addCollection(collectionID, in.Schema) node.addDmChannel(collectionID, channels) + } else { + } return status, err } @@ -151,8 +242,6 @@ func (c *queryNodeCluster) hasWatchedQueryChannel(ctx context.Context, nodeID in c.Lock() defer c.Unlock() - //TODO::should reopen - //collectionID = 0 return c.nodes[nodeID].hasWatchedQueryChannel(collectionID) } @@ -237,8 +326,13 @@ func (c *queryNodeCluster) getSegmentInfo(ctx context.Context, in *querypb.GetSe defer c.Unlock() segmentInfos := make([]*querypb.SegmentInfo, 0) - for _, node := range c.nodes { - res, err := node.client.GetSegmentInfo(ctx, in) + nodes, err := c.getOnServiceNodeIDs() + if err != nil { + log.Warn(err.Error()) + return segmentInfos, nil + } + for _, nodeID := range nodes { + res, err := c.nodes[nodeID].client.GetSegmentInfo(ctx, in) if err != nil { return nil, err } @@ -284,16 +378,69 @@ func (c *queryNodeCluster) getNumSegments(nodeID int64) (int, error) { return numSegment, nil } -func (c *queryNodeCluster) RegisterNode(ip string, port int64, id UniqueID) error { - node, err := newQueryNode(ip, port, id) +func (c *queryNodeCluster) RegisterNode(session *sessionutil.Session, id UniqueID) error { + c.Lock() + defer c.Unlock() + + sessionJSON, err := json.Marshal(session) if err != nil { return err } - c.Lock() - defer c.Unlock() + key := fmt.Sprintf("%s/%d", queryNodeInfoPrefix, id) + err = c.client.Save(key, string(sessionJSON)) + if err != nil { + return err + } + node, err := newQueryNode(session.Address, id, c.client) + if err != nil { + return err + } + log.Debug("register a new query node", zap.Int64("nodeID", id), zap.String("address", session.Address)) + if _, ok := c.nodes[id]; !ok { c.nodes[id] = node return nil } + return fmt.Errorf("node %d alredy exists in cluster", id) } + +func (c *queryNodeCluster) removeNodeInfo(nodeID int64) error { + key := fmt.Sprintf("%s/%d", queryNodeInfoPrefix, nodeID) + return c.client.Remove(key) +} + +func (c *queryNodeCluster) onServiceNodeIDs() ([]int64, error) { + c.Lock() + defer c.Unlock() + + return c.getOnServiceNodeIDs() +} + +func (c *queryNodeCluster) getOnServiceNodeIDs() ([]int64, error) { + nodeIDs := make([]int64, 0) + for nodeID, node := range c.nodes { + if node.isOnService() { + nodeIDs = append(nodeIDs, nodeID) + } + } + if len(nodeIDs) == 0 { + return nil, errors.New("no queryNode is alive") + } + + return nodeIDs, nil +} + +func (c *queryNodeCluster) printMeta() { + for id, node := range c.nodes { + if node.isOnService() { + for collectionID, info := range node.collectionInfos { + log.Debug("queryService cluster info: collectionInfo", zap.Int64("nodeID", id), zap.Int64("collectionID", collectionID), zap.Any("info", info)) + } + + for collectionID, info := range node.watchedQueryChannels { + log.Debug("queryService cluster info: watchedQueryChannelInfo", zap.Int64("nodeID", id), zap.Int64("collectionID", collectionID), zap.Any("info", info)) + } + } + } +} diff --git a/internal/queryservice/impl.go b/internal/queryservice/impl.go index d3326484e5..a482501682 100644 --- a/internal/queryservice/impl.go +++ b/internal/queryservice/impl.go @@ -15,7 +15,6 @@ import ( "context" "errors" "fmt" - "strconv" "go.uber.org/zap" @@ -24,6 +23,7 @@ import ( "github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/proto/milvuspb" "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/util/sessionutil" ) func (qs *QueryService) GetComponentStates(ctx context.Context) (*internalpb.ComponentStates, error) { @@ -31,13 +31,22 @@ func (qs *QueryService) GetComponentStates(ctx context.Context) (*internalpb.Com NodeID: Params.QueryServiceID, StateCode: qs.stateCode.Load().(internalpb.StateCode), } - subComponentInfos := qs.cluster.GetComponentInfos(ctx) + + //subComponentInfos, err := qs.cluster.GetComponentInfos(ctx) + //if err != nil { + // return &internalpb.ComponentStates{ + // Status: &commonpb.Status{ + // ErrorCode: commonpb.ErrorCode_UnexpectedError, + // Reason: err.Error(), + // }, + // }, err + //} return &internalpb.ComponentStates{ Status: &commonpb.Status{ ErrorCode: commonpb.ErrorCode_Success, }, - State: serviceComponentInfo, - SubcomponentStates: subComponentInfos, + State: serviceComponentInfo, + //SubcomponentStates: subComponentInfos, }, nil } @@ -77,7 +86,11 @@ func (qs *QueryService) RegisterNode(ctx context.Context, req *querypb.RegisterN }, err } - err := qs.cluster.RegisterNode(req.Address.Ip, req.Address.Port, req.Base.SourceID) + session := &sessionutil.Session{ + ServerID: nodeID, + Address: fmt.Sprintf("%s:%d", req.Address.Ip, req.Address.Port), + } + err := qs.cluster.RegisterNode(session, req.Base.SourceID) if err != nil { log.Debug("register query node new NodeClient failed", zap.Any("QueryNodeID", nodeID), zap.String("address", req.Address.String())) return &querypb.RegisterNodeResponse{ @@ -101,9 +114,9 @@ func (qs *QueryService) RegisterNode(ctx context.Context, req *querypb.RegisterN func (qs *QueryService) ShowCollections(ctx context.Context, req *querypb.ShowCollectionsRequest) (*querypb.ShowCollectionsResponse, error) { dbID := req.DbID - log.Debug("show collection start, dbID = ", zap.String("dbID", strconv.FormatInt(dbID, 10))) + log.Debug("show collection start", zap.Int64("dbID", dbID)) collectionIDs := qs.meta.showCollections() - log.Debug("show collection end") + log.Debug("show collection end", zap.Int64s("collections", collectionIDs)) return &querypb.ShowCollectionsResponse{ Status: &commonpb.Status{ ErrorCode: commonpb.ErrorCode_Success, @@ -114,7 +127,7 @@ func (qs *QueryService) ShowCollections(ctx context.Context, req *querypb.ShowCo func (qs *QueryService) LoadCollection(ctx context.Context, req *querypb.LoadCollectionRequest) (*commonpb.Status, error) { collectionID := req.CollectionID - schema := req.Schema + //schema := req.Schema log.Debug("LoadCollectionRequest received", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Int64("collectionID", collectionID), zap.Stringer("schema", req.Schema)) status := &commonpb.Status{ @@ -122,19 +135,21 @@ func (qs *QueryService) LoadCollection(ctx context.Context, req *querypb.LoadCol } hasCollection := qs.meta.hasCollection(collectionID) - if !hasCollection { - err := qs.meta.addCollection(collectionID, schema) - if err != nil { - log.Error(err.Error()) - return status, err - } + if hasCollection { + status.ErrorCode = commonpb.ErrorCode_Success + status.Reason = "collection has been loaded" + return status, nil } + //err := qs.meta.addCollection(collectionID, schema) + //if err != nil { + // log.Error(err.Error()) + // return status, err + //} loadCollectionTask := &LoadCollectionTask{ BaseTask: BaseTask{ - ctx: qs.loopCtx, - Condition: NewTaskCondition(qs.loopCtx), - + ctx: qs.loopCtx, + Condition: NewTaskCondition(qs.loopCtx), triggerCondition: querypb.TriggerCondition_grpcRequest, }, LoadCollectionRequest: req, @@ -145,11 +160,12 @@ func (qs *QueryService) LoadCollection(ctx context.Context, req *querypb.LoadCol } qs.scheduler.Enqueue([]task{loadCollectionTask}) - err := loadCollectionTask.WaitToFinish() - if err != nil { - status.Reason = err.Error() - return status, err - } + //err := loadCollectionTask.WaitToFinish() + //if err != nil { + // status.Reason = err.Error() + // return status, err + //} + //qs.meta.setLoadCollection(collectionID, true) log.Debug("LoadCollectionRequest completed", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Int64("collectionID", collectionID)) status.ErrorCode = commonpb.ErrorCode_Success @@ -171,9 +187,8 @@ func (qs *QueryService) ReleaseCollection(ctx context.Context, req *querypb.Rele releaseCollectionTask := &ReleaseCollectionTask{ BaseTask: BaseTask{ - ctx: qs.loopCtx, - Condition: NewTaskCondition(qs.loopCtx), - + ctx: qs.loopCtx, + Condition: NewTaskCondition(qs.loopCtx), triggerCondition: querypb.TriggerCondition_grpcRequest, }, ReleaseCollectionRequest: req, @@ -189,11 +204,14 @@ func (qs *QueryService) ReleaseCollection(ctx context.Context, req *querypb.Rele } log.Debug("ReleaseCollectionRequest completed", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Int64("collectionID", collectionID)) + qs.meta.printMeta() + qs.cluster.printMeta() return status, nil } func (qs *QueryService) ShowPartitions(ctx context.Context, req *querypb.ShowPartitionsRequest) (*querypb.ShowPartitionsResponse, error) { collectionID := req.CollectionID + log.Debug("show partitions start, ", zap.Int64("collectionID", collectionID)) partitionIDs, err := qs.meta.showPartitions(collectionID) if err != nil { return &querypb.ShowPartitionsResponse{ @@ -204,6 +222,8 @@ func (qs *QueryService) ShowPartitions(ctx context.Context, req *querypb.ShowPar }, err } + log.Debug("show partitions end", zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", partitionIDs)) + return &querypb.ShowPartitionsResponse{ Status: &commonpb.Status{ ErrorCode: commonpb.ErrorCode_Success, @@ -213,50 +233,18 @@ func (qs *QueryService) ShowPartitions(ctx context.Context, req *querypb.ShowPar } func (qs *QueryService) LoadPartitions(ctx context.Context, req *querypb.LoadPartitionsRequest) (*commonpb.Status, error) { - //TODO::suggest different partitions have different dm channel collectionID := req.CollectionID partitionIDs := req.PartitionIDs log.Debug("LoadPartitionRequest received", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", partitionIDs)) - status, err := LoadPartitionMetaCheck(qs.meta, req) - if err != nil { - return status, err - } - - loadPartitionTask := &LoadPartitionTask{ - BaseTask: BaseTask{ - ctx: qs.loopCtx, - Condition: NewTaskCondition(qs.loopCtx), - - triggerCondition: querypb.TriggerCondition_grpcRequest, - }, - LoadPartitionsRequest: req, - masterService: qs.masterServiceClient, - dataService: qs.dataServiceClient, - cluster: qs.cluster, - meta: qs.meta, - } - qs.scheduler.Enqueue([]task{loadPartitionTask}) - - err = loadPartitionTask.WaitToFinish() - if err != nil { - status.ErrorCode = commonpb.ErrorCode_UnexpectedError - status.Reason = err.Error() - return status, err - } - - log.Debug("LoadPartitionRequest completed", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Int64("collectionID", req.CollectionID)) - return status, nil -} - -func LoadPartitionMetaCheck(meta *meta, req *querypb.LoadPartitionsRequest) (*commonpb.Status, error) { - //dbID := req.DbID - collectionID := req.CollectionID - partitionIDs := req.PartitionIDs - schema := req.Schema - status := &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UnexpectedError, } + hasCollection := qs.meta.hasCollection(collectionID) + if hasCollection && qs.meta.collectionInfos[collectionID].LoadCollection { + status.ErrorCode = commonpb.ErrorCode_Success + status.Reason = "collection has been loaded" + return status, nil + } if len(partitionIDs) == 0 { err := errors.New("partitionIDs are empty") @@ -264,30 +252,39 @@ func LoadPartitionMetaCheck(meta *meta, req *querypb.LoadPartitionsRequest) (*co return status, err } - hasCollection := meta.hasCollection(collectionID) - if !hasCollection { - err := meta.addCollection(collectionID, schema) - if err != nil { - status.Reason = err.Error() - return status, err - } - } - partitionIDsToLoad := make([]UniqueID, 0) for _, partitionID := range partitionIDs { - hasPartition := meta.hasPartition(collectionID, partitionID) + hasPartition := qs.meta.hasPartition(collectionID, partitionID) if !hasPartition { - err := meta.addPartition(collectionID, partitionID) - if err != nil { - status.Reason = err.Error() - return status, err - } partitionIDsToLoad = append(partitionIDsToLoad, partitionID) } } req.PartitionIDs = partitionIDsToLoad + if len(req.PartitionIDs) > 0 { + loadPartitionTask := &LoadPartitionTask{ + BaseTask: BaseTask{ + ctx: qs.loopCtx, + Condition: NewTaskCondition(qs.loopCtx), + triggerCondition: querypb.TriggerCondition_grpcRequest, + }, + LoadPartitionsRequest: req, + dataService: qs.dataServiceClient, + cluster: qs.cluster, + meta: qs.meta, + } + qs.scheduler.Enqueue([]task{loadPartitionTask}) + + //err := loadPartitionTask.WaitToFinish() + //if err != nil { + // status.ErrorCode = commonpb.ErrorCode_UnexpectedError + // status.Reason = err.Error() + // return status, err + //} + } + status.ErrorCode = commonpb.ErrorCode_Success + log.Debug("LoadPartitionRequest completed", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Int64("collectionID", req.CollectionID)) return status, nil } @@ -311,9 +308,8 @@ func (qs *QueryService) ReleasePartitions(ctx context.Context, req *querypb.Rele req.PartitionIDs = toReleasedPartitionID releasePartitionTask := &ReleasePartitionTask{ BaseTask: BaseTask{ - ctx: qs.loopCtx, - Condition: NewTaskCondition(qs.loopCtx), - + ctx: qs.loopCtx, + Condition: NewTaskCondition(qs.loopCtx), triggerCondition: querypb.TriggerCondition_grpcRequest, }, ReleasePartitionsRequest: req, @@ -329,7 +325,8 @@ func (qs *QueryService) ReleasePartitions(ctx context.Context, req *querypb.Rele } } log.Debug("ReleasePartitionRequest completed", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", partitionIDs)) - //TODO:: queryNodeCluster cancel subscribe dmChannels + qs.meta.printMeta() + qs.cluster.printMeta() return status, nil } diff --git a/internal/queryservice/meta.go b/internal/queryservice/meta.go index 30a023a2ce..82bf57c318 100644 --- a/internal/queryservice/meta.go +++ b/internal/queryservice/meta.go @@ -13,17 +13,29 @@ package queryservice import ( "errors" + "fmt" + "path/filepath" "strconv" "sync" + "github.com/golang/protobuf/proto" "go.uber.org/zap" + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/proto/schemapb" ) +const ( + collectionMetaPrefix = "queryService-collectionMeta" + segmentMetaPrefix = "queryService-segmentMeta" + queryChannelMetaPrefix = "queryService-queryChannel" +) + type meta struct { + client *etcdkv.EtcdKV // client of a reliable kv service, i.e. etcd client + sync.RWMutex collectionInfos map[UniqueID]*querypb.CollectionInfo segmentInfos map[UniqueID]*querypb.SegmentInfo @@ -32,17 +44,82 @@ type meta struct { partitionStates map[UniqueID]querypb.PartitionState } -func newMeta() *meta { +func newMeta(kv *etcdkv.EtcdKV) (*meta, error) { collectionInfos := make(map[UniqueID]*querypb.CollectionInfo) segmentInfos := make(map[UniqueID]*querypb.SegmentInfo) queryChannelInfos := make(map[UniqueID]*querypb.QueryChannelInfo) partitionStates := make(map[UniqueID]querypb.PartitionState) - return &meta{ + + m := &meta{ + client: kv, collectionInfos: collectionInfos, segmentInfos: segmentInfos, queryChannelInfos: queryChannelInfos, partitionStates: partitionStates, } + + err := m.reloadFromKV() + if err != nil { + return nil, err + } + + return m, nil +} + +func (m *meta) reloadFromKV() error { + collectionKeys, collectionValues, err := m.client.LoadWithPrefix(collectionMetaPrefix) + if err != nil { + return err + } + for index := range collectionKeys { + collectionID, err := strconv.ParseInt(filepath.Base(collectionKeys[index]), 10, 64) + if err != nil { + return err + } + collectionInfo := &querypb.CollectionInfo{} + err = proto.UnmarshalText(collectionValues[index], collectionInfo) + if err != nil { + return err + } + m.collectionInfos[collectionID] = collectionInfo + } + + segmentKeys, segmentValues, err := m.client.LoadWithPrefix(segmentMetaPrefix) + if err != nil { + return err + } + for index := range segmentKeys { + segmentID, err := strconv.ParseInt(filepath.Base(segmentKeys[index]), 10, 64) + if err != nil { + return err + } + segmentInfo := &querypb.SegmentInfo{} + err = proto.UnmarshalText(segmentValues[index], segmentInfo) + if err != nil { + return err + } + m.segmentInfos[segmentID] = segmentInfo + } + + queryChannelKeys, queryChannelValues, err := m.client.LoadWithPrefix(queryChannelMetaPrefix) + if err != nil { + return nil + } + for index := range queryChannelKeys { + collectionID, err := strconv.ParseInt(filepath.Base(queryChannelKeys[index]), 10, 64) + if err != nil { + return err + } + queryChannelInfo := &querypb.QueryChannelInfo{} + err = proto.UnmarshalText(queryChannelValues[index], queryChannelInfo) + if err != nil { + return err + } + m.queryChannelInfos[collectionID] = queryChannelInfo + } + //TODO::update partition states + + return nil } func (m *meta) showCollections() []UniqueID { @@ -107,6 +184,10 @@ func (m *meta) addCollection(collectionID UniqueID, schema *schemapb.CollectionS Schema: schema, } m.collectionInfos[collectionID] = newCollection + err := m.saveCollectionInfo(collectionID, newCollection) + if err != nil { + log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID)) + } return nil } @@ -126,6 +207,10 @@ func (m *meta) addPartition(collectionID UniqueID, partitionID UniqueID) error { col.PartitionIDs = append(col.PartitionIDs, partitionID) m.partitionStates[partitionID] = querypb.PartitionState_NotPresent log.Debug("add a partition to meta", zap.Int64s("partitionIDs", col.PartitionIDs)) + err := m.saveCollectionInfo(collectionID, col) + if err != nil { + log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID)) + } return nil } return errors.New("addPartition: can't find collection when add partition") @@ -134,21 +219,50 @@ func (m *meta) addPartition(collectionID UniqueID, partitionID UniqueID) error { func (m *meta) deleteSegmentInfoByID(segmentID UniqueID) { m.Lock() defer m.Unlock() - delete(m.segmentInfos, segmentID) + + if _, ok := m.segmentInfos[segmentID]; ok { + err := m.removeSegmentInfo(segmentID) + if err != nil { + log.Error("remove segmentInfo error", zap.Any("error", err.Error()), zap.Int64("segmentID", segmentID)) + } + delete(m.segmentInfos, segmentID) + } +} + +func (m *meta) deleteSegmentInfoByNodeID(nodeID UniqueID) { + m.Lock() + defer m.Unlock() + + for segmentID, info := range m.segmentInfos { + if info.NodeID == nodeID { + err := m.removeSegmentInfo(segmentID) + if err != nil { + log.Error("remove segmentInfo error", zap.Any("error", err.Error()), zap.Int64("segmentID", segmentID)) + } + delete(m.segmentInfos, segmentID) + } + } } func (m *meta) setSegmentInfo(segmentID UniqueID, info *querypb.SegmentInfo) { m.Lock() defer m.Unlock() + err := m.saveSegmentInfo(segmentID, info) + if err != nil { + log.Error("save segmentInfo error", zap.Any("error", err.Error()), zap.Int64("segmentID", segmentID)) + } m.segmentInfos[segmentID] = info } func (m *meta) getSegmentInfos(segmentIDs []UniqueID) ([]*querypb.SegmentInfo, error) { + m.Lock() + defer m.Unlock() + segmentInfos := make([]*querypb.SegmentInfo, 0) for _, segmentID := range segmentIDs { if info, ok := m.segmentInfos[segmentID]; ok { - segmentInfos = append(segmentInfos, info) + segmentInfos = append(segmentInfos, proto.Clone(info).(*querypb.SegmentInfo)) continue } return nil, errors.New("segment not exist") @@ -156,17 +270,51 @@ func (m *meta) getSegmentInfos(segmentIDs []UniqueID) ([]*querypb.SegmentInfo, e return segmentInfos, nil } +func (m *meta) hasSegmentInfo(segmentID UniqueID) bool { + m.RLock() + defer m.RUnlock() + + if _, ok := m.segmentInfos[segmentID]; ok { + return true + } + + return false +} + func (m *meta) getSegmentInfoByID(segmentID UniqueID) (*querypb.SegmentInfo, error) { m.Lock() defer m.Unlock() if info, ok := m.segmentInfos[segmentID]; ok { - return info, nil + + return proto.Clone(info).(*querypb.SegmentInfo), nil } return nil, errors.New("getSegmentInfoByID: can't find segmentID in segmentInfos") } +func (m *meta) getCollectionInfoByID(collectionID UniqueID) (*querypb.CollectionInfo, error) { + m.Lock() + defer m.Unlock() + + if info, ok := m.collectionInfos[collectionID]; ok { + return proto.Clone(info).(*querypb.CollectionInfo), nil + } + + return nil, errors.New("getCollectionInfoByID: can't find collectionID in collectionInfo") +} + +func (m *meta) getQueryChannelInfoByID(collectionID UniqueID) (*querypb.QueryChannelInfo, error) { + m.Lock() + defer m.Unlock() + + if info, ok := m.queryChannelInfos[collectionID]; ok { + return proto.Clone(info).(*querypb.QueryChannelInfo), nil + } + + return nil, errors.New("getQueryChannelInfoByID: can't find collectionID in queryChannelInfo") +} + func (m *meta) updatePartitionState(partitionID UniqueID, state querypb.PartitionState) error { m.Lock() defer m.Unlock() @@ -201,10 +349,19 @@ func (m *meta) releaseCollection(collectionID UniqueID) { } for id, info := range m.segmentInfos { if info.CollectionID == collectionID { + err := m.removeSegmentInfo(id) + if err != nil { + log.Error("remove segmentInfo error", zap.Any("error", err.Error()), zap.Int64("segmentID", id)) + } delete(m.segmentInfos, id) } } + delete(m.queryChannelInfos, collectionID) + err := m.removeCollectionInfo(collectionID) + if err != nil { + log.Error("remove collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID)) + } } func (m *meta) releasePartition(collectionID UniqueID, partitionID UniqueID) { @@ -300,6 +457,43 @@ func (m *meta) addDmChannel(collectionID UniqueID, nodeID int64, channels []stri } info.ChannelInfos = append(info.ChannelInfos, newChannelInfo) } + + err := m.saveCollectionInfo(collectionID, info) + if err != nil { + log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID)) + } + } + + return errors.New("addDmChannels: can't find collection in collectionInfos") +} + +func (m *meta) removeDmChannel(collectionID UniqueID, nodeID int64, channels []string) error { + m.Lock() + defer m.Unlock() + + if info, ok := m.collectionInfos[collectionID]; ok { + for _, channelInfo := range info.ChannelInfos { + if channelInfo.NodeIDLoaded == nodeID { + newChannelIDs := make([]string, 0) + for _, channelID := range channelInfo.ChannelIDs { + findChannel := false + for _, channel := range channels { + if channelID == channel { + findChannel = true + } + } + if !findChannel { + newChannelIDs = append(newChannelIDs, channelID) + } + } + channelInfo.ChannelIDs = newChannelIDs + } + } + + err := m.saveCollectionInfo(collectionID, info) + if err != nil { + log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID)) + } } return errors.New("addDmChannels: can't find collection in collectionInfos") @@ -330,3 +524,68 @@ func (m *meta) GetQueryChannel(collectionID UniqueID) (string, string) { //TODO::return channel according collectionID return allocatedQueryChannel, allocatedQueryResultChannel } + +func (m *meta) saveCollectionInfo(collectionID UniqueID, info *querypb.CollectionInfo) error { + infoBytes := proto.MarshalTextString(info) + + key := fmt.Sprintf("%s/%d", collectionMetaPrefix, collectionID) + return m.client.Save(key, infoBytes) +} + +func (m *meta) removeCollectionInfo(collectionID UniqueID) error { + key := fmt.Sprintf("%s/%d", collectionMetaPrefix, collectionID) + return m.client.Remove(key) +} + +func (m *meta) saveSegmentInfo(segmentID UniqueID, info *querypb.SegmentInfo) error { + infoBytes := proto.MarshalTextString(info) + + key := fmt.Sprintf("%s/%d", segmentMetaPrefix, segmentID) + return m.client.Save(key, infoBytes) +} + +func (m *meta) removeSegmentInfo(segmentID UniqueID) error { + key := fmt.Sprintf("%s/%d", segmentMetaPrefix, segmentID) + return m.client.Remove(key) +} + +func (m *meta) saveQueryChannelInfo(collectionID UniqueID, info *querypb.QueryChannelInfo) error { + infoBytes := proto.MarshalTextString(info) + + key := fmt.Sprintf("%s/%d", queryChannelMetaPrefix, collectionID) + return m.client.Save(key, infoBytes) +} + +func (m *meta) removeQueryChannelInfo(collectionID UniqueID) error { + key := fmt.Sprintf("%s/%d", queryChannelMetaPrefix, collectionID) + return m.client.Remove(key) +} + +func (m *meta) setLoadCollection(collectionID UniqueID, state bool) error { + m.Lock() + defer m.Unlock() + + if info, ok := m.collectionInfos[collectionID]; ok { + info.LoadCollection = state + err := m.saveCollectionInfo(collectionID, info) + if err != nil { + log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID)) + } + } + + return errors.New("setLoadCollection: can't find collection in collectionInfos") +} + +func (m *meta) printMeta() { + for id, info := range m.collectionInfos { + log.Debug("queryService meta: collectionInfo", zap.Int64("collectionID", id), zap.Any("info", info)) + } + + for id, info := range m.segmentInfos { + log.Debug("queryService meta: segmentInfo", zap.Int64("segmentID", id), zap.Any("info", info)) + } + + for id, info := range m.queryChannelInfos { + log.Debug("queryService meta: queryChannelInfo", zap.Int64("collectionID", id), zap.Any("info", info)) + } +} diff --git a/internal/queryservice/meta_test.go b/internal/queryservice/meta_test.go index bd96d1ce57..af32034788 100644 --- a/internal/queryservice/meta_test.go +++ b/internal/queryservice/meta_test.go @@ -19,8 +19,9 @@ import ( ) func TestReplica_Release(t *testing.T) { - meta := newMeta() - err := meta.addCollection(1, nil) + meta, err := newMeta(nil) + assert.Nil(t, err) + err = meta.addCollection(1, nil) require.NoError(t, err) collections := meta.showCollections() diff --git a/internal/queryservice/param_table.go b/internal/queryservice/param_table.go index 2782f5a2c8..414274c8e0 100644 --- a/internal/queryservice/param_table.go +++ b/internal/queryservice/param_table.go @@ -50,6 +50,7 @@ type ParamTable struct { // --- ETCD --- EtcdEndpoints []string MetaRootPath string + KvRootPath string } var Params ParamTable @@ -86,6 +87,7 @@ func (p *ParamTable) Init() { // --- ETCD --- p.initEtcdEndpoints() p.initMetaRootPath() + p.initKvRootPath() }) } @@ -194,3 +196,15 @@ func (p *ParamTable) initMetaRootPath() { } p.MetaRootPath = path.Join(rootPath, subPath) } + +func (p *ParamTable) initKvRootPath() { + rootPath, err := p.Load("etcd.rootPath") + if err != nil { + panic(err) + } + subPath, err := p.Load("etcd.kvSubPath") + if err != nil { + panic(err) + } + p.KvRootPath = path.Join(rootPath, subPath) +} diff --git a/internal/queryservice/querynode.go b/internal/queryservice/querynode.go index cf71033094..3e73d3363b 100644 --- a/internal/queryservice/querynode.go +++ b/internal/queryservice/querynode.go @@ -17,53 +17,50 @@ import ( "sync" "time" + "github.com/golang/protobuf/proto" + "go.uber.org/zap" + nodeclient "github.com/milvus-io/milvus/internal/distributed/querynode/client" + etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" + "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/proto/schemapb" "github.com/milvus-io/milvus/internal/types" ) type queryNode struct { - id int64 - address struct { - ip string - port int64 - } - client types.QueryNode + id int64 + address string + client types.QueryNode + kvClient *etcdkv.EtcdKV - //mu sync.Mutex // guards segments and channels2Col - //nodeMeta *meta sync.RWMutex collectionInfos map[UniqueID]*querypb.CollectionInfo watchedQueryChannels map[UniqueID]*querypb.QueryChannelInfo - //segments map[UniqueID][]UniqueID - //channels2Col map[UniqueID][]string + onService bool } -func newQueryNode(ip string, port int64, id UniqueID) (*queryNode, error) { - client, err := nodeclient.NewClient(fmt.Sprintf("%s:%d", ip, port), 3*time.Second) +func newQueryNode(address string, id UniqueID, kv *etcdkv.EtcdKV) (*queryNode, error) { + client, err := nodeclient.NewClient(address, 3*time.Second) if err != nil { return nil, err } if err := client.Init(); err != nil { return nil, err } - if err := client.Start(); err != nil { return nil, err } collectionInfo := make(map[UniqueID]*querypb.CollectionInfo) watchedChannels := make(map[UniqueID]*querypb.QueryChannelInfo) return &queryNode{ - id: id, - address: struct { - ip string - port int64 - }{ip: ip, port: port}, + id: id, + address: address, client: client, + kvClient: kv, collectionInfos: collectionInfo, watchedQueryChannels: watchedChannels, - //nodeMeta: newMetaReplica(), + onService: true, }, nil } @@ -107,12 +104,26 @@ func (qn *queryNode) addCollection(collectionID UniqueID, schema *schemapb.Colle Schema: schema, } qn.collectionInfos[collectionID] = newCollection + err := qn.saveCollectionInfo(collectionID, newCollection) + if err != nil { + log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID)) + } return nil } return errors.New("addCollection: collection already exists") } +func (qn *queryNode) getCollectionInfoByID(collectionID UniqueID) (*querypb.CollectionInfo, error) { + qn.Lock() + defer qn.Lock() + + if _, ok := qn.collectionInfos[collectionID]; ok { + return proto.Clone(qn.collectionInfos[collectionID]).(*querypb.CollectionInfo), nil + } + return nil, errors.New("addPartition: can't find collection") +} + func (qn *queryNode) addPartition(collectionID UniqueID, partitionID UniqueID) error { qn.Lock() defer qn.Unlock() @@ -123,6 +134,10 @@ func (qn *queryNode) addPartition(collectionID UniqueID, partitionID UniqueID) e } } col.PartitionIDs = append(col.PartitionIDs, partitionID) + err := qn.saveCollectionInfo(collectionID, col) + if err != nil { + log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID)) + } return nil } return errors.New("addPartition: can't find collection when add partition") @@ -131,9 +146,14 @@ func (qn *queryNode) addPartition(collectionID UniqueID, partitionID UniqueID) e func (qn *queryNode) releaseCollection(collectionID UniqueID) { qn.Lock() defer qn.Unlock() - delete(qn.collectionInfos, collectionID) - //TODO::should reopen - //collectionID = 0 + if _, ok := qn.collectionInfos[collectionID]; ok { + err := qn.removeCollectionInfo(collectionID) + if err != nil { + log.Error("remove collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID)) + } + delete(qn.collectionInfos, collectionID) + } + delete(qn.watchedQueryChannels, collectionID) } @@ -149,6 +169,10 @@ func (qn *queryNode) releasePartition(collectionID UniqueID, partitionID UniqueI } } info.PartitionIDs = newPartitionIDs + err := qn.removeCollectionInfo(collectionID) + if err != nil { + log.Error("remove collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID)) + } } } @@ -206,12 +230,46 @@ func (qn *queryNode) addDmChannel(collectionID UniqueID, channels []string) erro } info.ChannelInfos = append(info.ChannelInfos, newChannelInfo) } + err := qn.saveCollectionInfo(collectionID, info) + if err != nil { + log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID)) + } } return errors.New("addDmChannels: can't find collection in watchedQueryChannel") } -//TODO::removeDmChannels +func (qn *queryNode) removeDmChannel(collectionID UniqueID, channels []string) error { + qn.Lock() + defer qn.Unlock() + + if info, ok := qn.collectionInfos[collectionID]; ok { + for _, channelInfo := range info.ChannelInfos { + if channelInfo.NodeIDLoaded == qn.id { + newChannelIDs := make([]string, 0) + for _, channelID := range channelInfo.ChannelIDs { + findChannel := false + for _, channel := range channels { + if channelID == channel { + findChannel = true + } + } + if !findChannel { + newChannelIDs = append(newChannelIDs, channelID) + } + } + channelInfo.ChannelIDs = newChannelIDs + } + } + + err := qn.saveCollectionInfo(collectionID, info) + if err != nil { + log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID)) + } + } + + return errors.New("addDmChannels: can't find collection in watchedQueryChannel") +} func (qn *queryNode) hasWatchedQueryChannel(collectionID UniqueID) bool { qn.RLock() @@ -236,5 +294,32 @@ func (qn *queryNode) removeQueryChannel(collectionID UniqueID) error { defer qn.Unlock() delete(qn.watchedQueryChannels, collectionID) - return nil + + return errors.New("removeQueryChannel: can't find collection in watchedQueryChannel") +} + +func (qn *queryNode) saveCollectionInfo(collectionID UniqueID, info *querypb.CollectionInfo) error { + infoBytes := proto.MarshalTextString(info) + + key := fmt.Sprintf("%s/%d/%d", queryNodeMetaPrefix, qn.id, collectionID) + return qn.kvClient.Save(key, infoBytes) +} + +func (qn *queryNode) removeCollectionInfo(collectionID UniqueID) error { + key := fmt.Sprintf("%s/%d/%d", queryNodeMetaPrefix, qn.id, collectionID) + return qn.kvClient.Remove(key) +} + +func (qn *queryNode) setNodeState(onService bool) { + qn.Lock() + defer qn.Unlock() + + qn.onService = onService +} + +func (qn *queryNode) isOnService() bool { + qn.Lock() + defer qn.Unlock() + + return qn.onService } diff --git a/internal/queryservice/queryservice.go b/internal/queryservice/queryservice.go index ad482ffa46..9b09cb25cb 100644 --- a/internal/queryservice/queryservice.go +++ b/internal/queryservice/queryservice.go @@ -14,17 +14,25 @@ package queryservice import ( "context" "math/rand" + "path/filepath" "strconv" + "sync" "sync/atomic" "time" + "github.com/coreos/etcd/mvcc/mvccpb" + "github.com/golang/protobuf/proto" + "go.etcd.io/etcd/clientv3" "go.uber.org/zap" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/msgstream" + "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/internalpb" + "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/types" + "github.com/milvus-io/milvus/internal/util/retry" "github.com/milvus-io/milvus/internal/util/sessionutil" "github.com/milvus-io/milvus/internal/util/typeutil" ) @@ -39,7 +47,8 @@ type queryChannelInfo struct { type QueryService struct { loopCtx context.Context loopCancel context.CancelFunc - kvBase *etcdkv.EtcdKV + loopWg sync.WaitGroup + kvClient *etcdkv.EtcdKV queryServiceID uint64 meta *meta @@ -49,7 +58,8 @@ type QueryService struct { dataServiceClient types.DataService masterServiceClient types.MasterService - session *sessionutil.Session + session *sessionutil.Session + eventChan <-chan *sessionutil.SessionEvent stateCode atomic.Value isInit atomic.Value @@ -67,7 +77,33 @@ func (qs *QueryService) Register() error { } func (qs *QueryService) Init() error { + connectEtcdFn := func() error { + etcdClient, err := clientv3.New(clientv3.Config{Endpoints: Params.EtcdEndpoints}) + if err != nil { + return err + } + etcdKV := etcdkv.NewEtcdKV(etcdClient, Params.MetaRootPath) + qs.kvClient = etcdKV + metaKV, err := newMeta(etcdKV) + if err != nil { + return err + } + qs.meta = metaKV + qs.cluster, err = newQueryNodeCluster(metaKV, etcdKV) + if err != nil { + return err + } + qs.scheduler, err = NewTaskScheduler(qs.loopCtx, metaKV, qs.cluster, etcdKV, qs.masterServiceClient, qs.dataServiceClient) + return err + } + log.Debug("queryService try to connect etcd") + err := retry.Retry(100000, time.Millisecond*200, connectEtcdFn) + if err != nil { + log.Debug("queryService try to connect etcd failed", zap.Error(err)) + return err + } + log.Debug("queryService try to connect etcd success") return nil } @@ -75,6 +111,13 @@ func (qs *QueryService) Start() error { qs.scheduler.Start() log.Debug("start scheduler ...") qs.UpdateStateCode(internalpb.StateCode_Healthy) + + qs.loopWg.Add(1) + go qs.watchNodeLoop() + + qs.loopWg.Add(1) + go qs.watchMetaLoop() + return nil } @@ -83,6 +126,8 @@ func (qs *QueryService) Stop() error { log.Debug("close scheduler ...") qs.loopCancel() qs.UpdateStateCode(internalpb.StateCode_Abnormal) + + qs.loopWg.Wait() return nil } @@ -105,16 +150,11 @@ func NewQueryService(ctx context.Context, factory msgstream.Factory) (*QueryServ }) ctx1, cancel := context.WithCancel(ctx) - meta := newMeta() service := &QueryService{ loopCtx: ctx1, loopCancel: cancel, - meta: meta, msFactory: factory, } - //TODO::set etcd kvbase - service.scheduler = NewTaskScheduler(ctx1, meta, service.kvBase) - service.cluster = newQueryNodeCluster(meta) service.UpdateStateCode(internalpb.StateCode_Abnormal) log.Debug("QueryService", zap.Any("queryChannels", queryChannels)) @@ -128,3 +168,141 @@ func (qs *QueryService) SetMasterService(masterService types.MasterService) { func (qs *QueryService) SetDataService(dataService types.DataService) { qs.dataServiceClient = dataService } + +func (qs *QueryService) watchNodeLoop() { + ctx, cancel := context.WithCancel(qs.loopCtx) + defer cancel() + defer qs.loopWg.Done() + log.Debug("QueryService start watch node loop") + + clusterStartSession, version, _ := qs.session.GetSessions(typeutil.QueryNodeRole) + sessionMap := make(map[int64]*sessionutil.Session) + for _, session := range clusterStartSession { + nodeID := session.ServerID + sessionMap[nodeID] = session + } + for nodeID, session := range sessionMap { + if _, ok := qs.cluster.nodes[nodeID]; !ok { + serverID := session.ServerID + err := qs.cluster.RegisterNode(session, serverID) + if err != nil { + log.Error("register queryNode error", zap.Any("error", err.Error())) + } + log.Debug("QueryService", zap.Any("Add QueryNode, session serverID", serverID)) + } + } + for nodeID := range qs.cluster.nodes { + if _, ok := sessionMap[nodeID]; !ok { + qs.cluster.nodes[nodeID].setNodeState(false) + loadBalanceSegment := &querypb.LoadBalanceRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_LoadBalanceSegments, + SourceID: qs.session.ServerID, + }, + SourceNodeIDs: []int64{nodeID}, + } + + loadBalanceTask := &LoadBalanceTask{ + BaseTask: BaseTask{ + ctx: qs.loopCtx, + Condition: NewTaskCondition(qs.loopCtx), + triggerCondition: querypb.TriggerCondition_nodeDown, + }, + LoadBalanceRequest: loadBalanceSegment, + master: qs.masterServiceClient, + dataService: qs.dataServiceClient, + cluster: qs.cluster, + meta: qs.meta, + } + qs.scheduler.Enqueue([]task{loadBalanceTask}) + } + } + + qs.eventChan = qs.session.WatchServices(typeutil.QueryNodeRole, version+1) + for { + select { + case <-ctx.Done(): + return + case event := <-qs.eventChan: + switch event.EventType { + case sessionutil.SessionAddEvent: + serverID := event.Session.ServerID + err := qs.cluster.RegisterNode(event.Session, serverID) + if err != nil { + log.Error(err.Error()) + } + log.Debug("QueryService", zap.Any("Add QueryNode, session serverID", serverID)) + case sessionutil.SessionDelEvent: + serverID := event.Session.ServerID + log.Debug("QueryService", zap.Any("The QueryNode crashed with ID", serverID)) + qs.cluster.nodes[serverID].setNodeState(false) + loadBalanceSegment := &querypb.LoadBalanceRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_LoadBalanceSegments, + SourceID: qs.session.ServerID, + }, + SourceNodeIDs: []int64{serverID}, + BalanceReason: querypb.TriggerCondition_nodeDown, + } + + loadBalanceTask := &LoadBalanceTask{ + BaseTask: BaseTask{ + ctx: qs.loopCtx, + Condition: NewTaskCondition(qs.loopCtx), + triggerCondition: querypb.TriggerCondition_nodeDown, + }, + LoadBalanceRequest: loadBalanceSegment, + master: qs.masterServiceClient, + dataService: qs.dataServiceClient, + cluster: qs.cluster, + meta: qs.meta, + } + qs.scheduler.Enqueue([]task{loadBalanceTask}) + err := loadBalanceTask.WaitToFinish() + if err != nil { + log.Error(err.Error()) + } + log.Debug("load balance done after queryNode down", zap.Int64s("nodeIDs", loadBalanceTask.SourceNodeIDs)) + //TODO::remove nodeInfo and clear etcd + } + } + } +} + +func (qs *QueryService) watchMetaLoop() { + ctx, cancel := context.WithCancel(qs.loopCtx) + + defer cancel() + defer qs.loopWg.Done() + log.Debug("QueryService start watch meta loop") + + watchChan := qs.meta.client.WatchWithPrefix("queryNode-segmentMeta") + + for { + select { + case <-ctx.Done(): + return + case resp := <-watchChan: + log.Debug("segment meta updated.") + for _, event := range resp.Events { + segmentID, err := strconv.ParseInt(filepath.Base(string(event.Kv.Key)), 10, 64) + if err != nil { + log.Error("watch meta loop error when get segmentID", zap.Any("error", err.Error())) + } + segmentInfo := &querypb.SegmentInfo{} + err = proto.UnmarshalText(string(event.Kv.Value), segmentInfo) + if err != nil { + log.Error("watch meta loop error when unmarshal", zap.Any("error", err.Error())) + } + switch event.Type { + case mvccpb.PUT: + //TODO:: + qs.meta.setSegmentInfo(segmentID, segmentInfo) + case mvccpb.DELETE: + //TODO:: + } + } + } + } + +} diff --git a/internal/queryservice/task.go b/internal/queryservice/task.go index a2de58ea2d..227ea5ed3a 100644 --- a/internal/queryservice/task.go +++ b/internal/queryservice/task.go @@ -13,10 +13,9 @@ package queryservice import ( "context" - "errors" "fmt" - "sort" + "github.com/golang/protobuf/proto" "go.uber.org/zap" "github.com/milvus-io/milvus/internal/log" @@ -28,6 +27,22 @@ import ( "github.com/milvus-io/milvus/internal/types" ) +const ( + triggerTaskPrefix = "queryService-triggerTask" + activeTaskPrefix = "queryService-activeTask" + taskInfoPrefix = "queryService-taskInfo" + loadBalanceInfoPrefix = "queryService-loadBalanceInfo" +) + +type taskState int + +const ( + taskUndo taskState = 0 + taskDoing taskState = 1 + taskDone taskState = 3 + taskExpired taskState = 4 +) + type task interface { TraceCtx() context.Context ID() UniqueID // return ReqId @@ -43,6 +58,11 @@ type task interface { GetParentTask() task GetChildTask() []task AddChildTask(t task) + IsValid() bool + Reschedule() ([]task, error) + Marshal() string + State() taskState + SetState(state taskState) } type BaseTask struct { @@ -50,6 +70,7 @@ type BaseTask struct { ctx context.Context cancel context.CancelFunc result *commonpb.Status + state taskState taskID UniqueID triggerCondition querypb.TriggerCondition @@ -85,17 +106,34 @@ func (bt *BaseTask) AddChildTask(t task) { bt.childTasks = append(bt.childTasks, t) } +func (bt *BaseTask) IsValid() bool { + return true +} + +func (bt *BaseTask) Reschedule() ([]task, error) { + return nil, nil +} + +func (bt *BaseTask) State() taskState { + return bt.state +} + +func (bt *BaseTask) SetState(state taskState) { + bt.state = state +} + //************************grpcTask***************************// type LoadCollectionTask struct { BaseTask *querypb.LoadCollectionRequest - masterService types.MasterService - dataService types.DataService - cluster *queryNodeCluster - meta *meta - toWatchPosition map[string]*internalpb.MsgPosition - excludeSegment map[string][]UniqueID - watchNeeded bool + masterService types.MasterService + dataService types.DataService + cluster *queryNodeCluster + meta *meta +} + +func (lct *LoadCollectionTask) Marshal() string { + return proto.MarshalTextString(lct.LoadCollectionRequest) } func (lct *LoadCollectionTask) Type() commonpb.MsgType { @@ -122,6 +160,7 @@ func (lct *LoadCollectionTask) Execute(ctx context.Context) error { ErrorCode: commonpb.ErrorCode_UnexpectedError, } + lct.meta.addCollection(collectionID, lct.Schema) showPartitionRequest := &milvuspb.ShowPartitionsRequest{ Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_ShowPartitions, @@ -148,7 +187,6 @@ func (lct *LoadCollectionTask) Execute(ctx context.Context) error { CollectionID: collectionID, PartitionID: partitionID, } - //recoveryInfo, err := mockGetRecoveryInfoFromDataService(lct.ctx, lct.masterService, lct.dataService, getRecoveryInfoRequest) recoveryInfo, err := lct.dataService.GetRecoveryInfo(lct.ctx, getRecoveryInfoRequest) if err != nil { status.Reason = err.Error() @@ -230,27 +268,29 @@ func (lct *LoadCollectionTask) Execute(ctx context.Context) error { }, LoadSegmentsRequest: loadSegmentsRequest, + meta: lct.meta, cluster: lct.cluster, } lct.AddChildTask(loadSegmentTask) - log.Debug("add a loadSegmentTask to loadCollectionTask's childTask", zap.Any("load segment Task", loadSegmentTask)) + log.Debug("loadCollectionTask: add a loadSegmentTask to loadCollectionTask's childTask", zap.Any("task", loadSegmentTask)) } - for channel, nodeID := range watchRequest2Nodes { + for index, nodeID := range watchRequest2Nodes { + channel := channelsToWatch[index] watchRequests[channel].NodeID = nodeID watchDmChannelTask := &WatchDmChannelTask{ BaseTask: BaseTask{ - ctx: lct.ctx, - Condition: NewTaskCondition(lct.ctx), - + ctx: lct.ctx, + Condition: NewTaskCondition(lct.ctx), triggerCondition: querypb.TriggerCondition_grpcRequest, }, WatchDmChannelsRequest: watchRequests[channel], + meta: lct.meta, cluster: lct.cluster, } //TODO::open when queryNode watchDmChannel work lct.AddChildTask(watchDmChannelTask) - log.Debug("add a watchDmChannelTask to loadCollectionTask's childTask", zap.Any("task", watchDmChannelTask)) + log.Debug("loadCollectionTask: add a watchDmChannelTask to loadCollectionTask's childTask", zap.Any("task", watchDmChannelTask)) } for nodeID, watched := range watchQueryChannelInfo { @@ -266,9 +306,8 @@ func (lct *LoadCollectionTask) Execute(ctx context.Context) error { } watchQueryChannelTask := &WatchQueryChannelTask{ BaseTask: BaseTask{ - ctx: lct.ctx, - Condition: NewTaskCondition(lct.ctx), - + ctx: lct.ctx, + Condition: NewTaskCondition(lct.ctx), triggerCondition: querypb.TriggerCondition_grpcRequest, }, @@ -276,7 +315,7 @@ func (lct *LoadCollectionTask) Execute(ctx context.Context) error { cluster: lct.cluster, } lct.AddChildTask(watchQueryChannelTask) - log.Debug("add a watchQueryChannelTask to loadCollectionTask's childTask") + log.Debug("loadCollectionTask: add a watchQueryChannelTask to loadCollectionTask's childTask", zap.Any("task", watchQueryChannelTask)) } } @@ -288,6 +327,8 @@ func (lct *LoadCollectionTask) Execute(ctx context.Context) error { func (lct *LoadCollectionTask) PostExecute(ctx context.Context) error { collectionID := lct.CollectionID + lct.meta.addCollection(collectionID, lct.Schema) + lct.meta.collectionInfos[collectionID].LoadCollection = true log.Debug("LoadCollectionTask postExecute done", zap.Int64("msgID", lct.ID()), zap.Int64("collectionID", collectionID)) @@ -297,10 +338,13 @@ func (lct *LoadCollectionTask) PostExecute(ctx context.Context) error { type ReleaseCollectionTask struct { BaseTask *querypb.ReleaseCollectionRequest - nodeID int64 cluster *queryNodeCluster } +func (rct *ReleaseCollectionTask) Marshal() string { + return proto.MarshalTextString(rct.ReleaseCollectionRequest) +} + func (rct *ReleaseCollectionTask) Type() commonpb.MsgType { return rct.Base.MsgType } @@ -322,26 +366,30 @@ func (rct *ReleaseCollectionTask) Execute(ctx context.Context) error { status := &commonpb.Status{ ErrorCode: commonpb.ErrorCode_Success, } - if rct.nodeID <= 0 { - for nodeID := range rct.cluster.nodes { + + if rct.NodeID <= 0 { + for nodeID, node := range rct.cluster.nodes { + if !node.isOnService() { + continue + } + req := proto.Clone(rct.ReleaseCollectionRequest).(*querypb.ReleaseCollectionRequest) + req.NodeID = nodeID releaseCollectionTask := &ReleaseCollectionTask{ BaseTask: BaseTask{ ctx: rct.ctx, Condition: NewTaskCondition(rct.ctx), triggerCondition: querypb.TriggerCondition_grpcRequest, }, - - ReleaseCollectionRequest: rct.ReleaseCollectionRequest, - nodeID: nodeID, + ReleaseCollectionRequest: req, cluster: rct.cluster, } rct.AddChildTask(releaseCollectionTask) - log.Debug("add a releaseCollectionTask to releaseCollectionTask's childTask") + log.Debug("ReleaseCollectionTask: add a releaseCollectionTask to releaseCollectionTask's childTask", zap.Any("task", releaseCollectionTask)) } } else { - _, err := rct.cluster.releaseCollection(ctx, rct.nodeID, rct.ReleaseCollectionRequest) + _, err := rct.cluster.releaseCollection(ctx, rct.NodeID, rct.ReleaseCollectionRequest) if err != nil { - log.Error("release collection end, node occur error", zap.String("nodeID", fmt.Sprintln(rct.nodeID))) + log.Error("ReleaseCollectionTask: release collection end, node occur error", zap.String("nodeID", fmt.Sprintln(rct.NodeID))) status.ErrorCode = commonpb.ErrorCode_UnexpectedError status.Reason = err.Error() rct.result = status @@ -353,7 +401,7 @@ func (rct *ReleaseCollectionTask) Execute(ctx context.Context) error { log.Debug("ReleaseCollectionTask Execute done", zap.Int64("msgID", rct.ID()), zap.Int64("collectionID", collectionID), - zap.Int64("nodeID", rct.nodeID)) + zap.Int64("nodeID", rct.NodeID)) return nil } @@ -363,17 +411,20 @@ func (rct *ReleaseCollectionTask) PostExecute(ctx context.Context) error { log.Debug("ReleaseCollectionTask postExecute done", zap.Int64("msgID", rct.ID()), zap.Int64("collectionID", collectionID), - zap.Int64("nodeID", rct.nodeID)) + zap.Int64("nodeID", rct.NodeID)) return nil } type LoadPartitionTask struct { BaseTask *querypb.LoadPartitionsRequest - masterService types.MasterService - dataService types.DataService - cluster *queryNodeCluster - meta *meta + dataService types.DataService + cluster *queryNodeCluster + meta *meta +} + +func (lpt *LoadPartitionTask) Marshal() string { + return proto.MarshalTextString(lpt.LoadPartitionsRequest) } func (lpt *LoadPartitionTask) Type() commonpb.MsgType { @@ -396,6 +447,10 @@ func (lpt *LoadPartitionTask) Execute(ctx context.Context) error { collectionID := lpt.CollectionID partitionIDs := lpt.PartitionIDs + lpt.meta.addCollection(collectionID, lpt.Schema) + for _, id := range partitionIDs { + lpt.meta.addPartition(collectionID, id) + } status := &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UnexpectedError, } @@ -410,7 +465,6 @@ func (lpt *LoadPartitionTask) Execute(ctx context.Context) error { CollectionID: collectionID, PartitionID: partitionID, } - //recoveryInfo, err := mockGetRecoveryInfoFromDataService(lpt.ctx, lpt.masterService, lpt.dataService, getRecoveryInfoRequest) recoveryInfo, err := lpt.dataService.GetRecoveryInfo(lpt.ctx, getRecoveryInfoRequest) if err != nil { status.Reason = err.Error() @@ -442,13 +496,13 @@ func (lpt *LoadPartitionTask) Execute(ctx context.Context) error { } channelsToWatch = append(channelsToWatch, channel) watchRequests = append(watchRequests, watchRequest) - log.Debug("set watchDmChannelsRequests", zap.Any("request", watchRequest), zap.Int64("collectionID", collectionID)) + log.Debug("LoadPartitionTask: set watchDmChannelsRequests", zap.Any("request", watchRequest), zap.Int64("collectionID", collectionID)) } } segment2Nodes := shuffleSegmentsToQueryNode(segmentsToLoad, lpt.cluster) watchRequest2Nodes := shuffleChannelsToQueryNode(channelsToWatch, lpt.cluster) - log.Debug("watch request to node", zap.Any("request map", watchRequest2Nodes), zap.Int64("collectionID", collectionID)) + log.Debug("LoadPartitionTask: watch request to node", zap.Any("request map", watchRequest2Nodes), zap.Int64("collectionID", collectionID)) watchQueryChannelInfo := make(map[int64]bool) node2Segments := make(map[int64][]*querypb.SegmentLoadInfo) @@ -488,28 +542,27 @@ func (lpt *LoadPartitionTask) Execute(ctx context.Context) error { }, LoadSegmentsRequest: loadSegmentsRequest, + meta: lpt.meta, cluster: lpt.cluster, } lpt.AddChildTask(loadSegmentTask) - log.Debug("add a loadSegmentTask to loadPartitionTask's childTask") + log.Debug("LoadPartitionTask: add a loadSegmentTask to loadPartitionTask's childTask") } - index := 0 - for _, nodeID := range watchRequest2Nodes { + for index, nodeID := range watchRequest2Nodes { watchRequests[index].NodeID = nodeID watchDmChannelTask := &WatchDmChannelTask{ BaseTask: BaseTask{ - ctx: lpt.ctx, - Condition: NewTaskCondition(lpt.ctx), - + ctx: lpt.ctx, + Condition: NewTaskCondition(lpt.ctx), triggerCondition: querypb.TriggerCondition_grpcRequest, }, WatchDmChannelsRequest: watchRequests[index], + meta: lpt.meta, cluster: lpt.cluster, } lpt.AddChildTask(watchDmChannelTask) - log.Debug("add a watchDmChannelTask to loadPartitionTask's childTask") - index++ + log.Debug("LoadPartitionTask: add a watchDmChannelTask to loadPartitionTask's childTask", zap.Any("task", watchDmChannelTask)) } for nodeID, watched := range watchQueryChannelInfo { @@ -525,9 +578,8 @@ func (lpt *LoadPartitionTask) Execute(ctx context.Context) error { } watchQueryChannelTask := &WatchQueryChannelTask{ BaseTask: BaseTask{ - ctx: lpt.ctx, - Condition: NewTaskCondition(lpt.ctx), - + ctx: lpt.ctx, + Condition: NewTaskCondition(lpt.ctx), triggerCondition: querypb.TriggerCondition_grpcRequest, }, @@ -535,7 +587,7 @@ func (lpt *LoadPartitionTask) Execute(ctx context.Context) error { cluster: lpt.cluster, } lpt.AddChildTask(watchQueryChannelTask) - log.Debug("add a watchQueryChannelTask to loadPartitionTask's childTask") + log.Debug("LoadPartitionTask: add a watchQueryChannelTask to loadPartitionTask's childTask", zap.Any("task", watchQueryChannelTask)) } } @@ -562,10 +614,13 @@ func (lpt *LoadPartitionTask) PostExecute(ctx context.Context) error { type ReleasePartitionTask struct { BaseTask *querypb.ReleasePartitionsRequest - nodeID int64 cluster *queryNodeCluster } +func (rpt *ReleasePartitionTask) Marshal() string { + return proto.MarshalTextString(rpt.ReleasePartitionsRequest) +} + func (rpt *ReleasePartitionTask) Type() commonpb.MsgType { return rpt.Base.MsgType } @@ -588,24 +643,26 @@ func (rpt *ReleasePartitionTask) Execute(ctx context.Context) error { status := &commonpb.Status{ ErrorCode: commonpb.ErrorCode_Success, } - if rpt.nodeID <= 0 { + + if rpt.NodeID <= 0 { for nodeID := range rpt.cluster.nodes { + req := proto.Clone(rpt.ReleasePartitionsRequest).(*querypb.ReleasePartitionsRequest) + req.NodeID = nodeID releasePartitionTask := &ReleasePartitionTask{ BaseTask: BaseTask{ ctx: rpt.ctx, - Condition: rpt.Condition, + Condition: NewTaskCondition(rpt.ctx), triggerCondition: querypb.TriggerCondition_grpcRequest, }, - ReleasePartitionsRequest: rpt.ReleasePartitionsRequest, - nodeID: nodeID, + ReleasePartitionsRequest: req, cluster: rpt.cluster, } rpt.AddChildTask(releasePartitionTask) - log.Debug("add a releasePartitionTask to releasePartitionTask's childTask") + log.Debug("ReleasePartitionTask: add a releasePartitionTask to releasePartitionTask's childTask", zap.Any("task", releasePartitionTask)) } } else { - status, err := rpt.cluster.releasePartitions(ctx, rpt.nodeID, rpt.ReleasePartitionsRequest) + status, err := rpt.cluster.releasePartitions(ctx, rpt.NodeID, rpt.ReleasePartitionsRequest) if err != nil { rpt.result = status return err @@ -617,7 +674,7 @@ func (rpt *ReleasePartitionTask) Execute(ctx context.Context) error { zap.Int64("msgID", rpt.ID()), zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", partitionIDs), - zap.Int64("nodeID", rpt.nodeID)) + zap.Int64("nodeID", rpt.NodeID)) return nil } @@ -629,7 +686,7 @@ func (rpt *ReleasePartitionTask) PostExecute(ctx context.Context) error { zap.Int64("msgID", rpt.ID()), zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", partitionIDs), - zap.Int64("nodeID", rpt.nodeID)) + zap.Int64("nodeID", rpt.NodeID)) return nil } @@ -637,24 +694,38 @@ func (rpt *ReleasePartitionTask) PostExecute(ctx context.Context) error { type LoadSegmentTask struct { BaseTask *querypb.LoadSegmentsRequest + meta *meta cluster *queryNodeCluster } +func (lst *LoadSegmentTask) Marshal() string { + return proto.MarshalTextString(lst.LoadSegmentsRequest) +} + +func (lst *LoadSegmentTask) IsValid() bool { + return lst.ctx != nil && lst.cluster.nodes[lst.NodeID].isOnService() +} + func (lst *LoadSegmentTask) Type() commonpb.MsgType { return lst.Base.MsgType } + func (lst *LoadSegmentTask) Timestamp() Timestamp { return lst.Base.Timestamp } + func (lst *LoadSegmentTask) PreExecute(ctx context.Context) error { segmentIDs := make([]UniqueID, 0) for _, info := range lst.Infos { segmentIDs = append(segmentIDs, info.SegmentID) } log.Debug("start do loadSegmentTask", - zap.Int64s("segmentIDs", segmentIDs)) + zap.Int64s("segmentIDs", segmentIDs), + zap.Int64("loaded nodeID", lst.NodeID), + zap.Int64("taskID", lst.ID())) return nil } + func (lst *LoadSegmentTask) Execute(ctx context.Context) error { status, err := lst.cluster.LoadSegments(lst.ctx, lst.NodeID, lst.LoadSegmentsRequest) if err != nil { @@ -663,32 +734,111 @@ func (lst *LoadSegmentTask) Execute(ctx context.Context) error { } lst.result = status - log.Debug("loadSegmentTask Execute done") + log.Debug("loadSegmentTask Execute done", + zap.Int64("taskID", lst.ID())) return nil } func (lst *LoadSegmentTask) PostExecute(ctx context.Context) error { - log.Debug("loadSegmentTask postExecute done") + log.Debug("loadSegmentTask postExecute done", + zap.Int64("taskID", lst.ID())) return nil } +func (lst *LoadSegmentTask) Reschedule() ([]task, error) { + segmentIDs := make([]UniqueID, 0) + collectionID := lst.Infos[0].CollectionID + reScheduledTask := make([]task, 0) + for _, info := range lst.Infos { + segmentID := info.SegmentID + segmentIDs = append(segmentIDs, segmentID) + } + segment2Nodes := shuffleSegmentsToQueryNode(segmentIDs, lst.cluster) + node2segmentInfos := make(map[int64][]*querypb.SegmentLoadInfo) + for _, info := range lst.Infos { + segmentID := info.SegmentID + nodeID := segment2Nodes[segmentID] + if _, ok := node2segmentInfos[nodeID]; !ok { + node2segmentInfos[nodeID] = make([]*querypb.SegmentLoadInfo, 0) + } + node2segmentInfos[nodeID] = append(node2segmentInfos[nodeID], info) + } + + for nodeID, infos := range node2segmentInfos { + loadSegmentTask := &LoadSegmentTask{ + BaseTask: lst.BaseTask, + LoadSegmentsRequest: &querypb.LoadSegmentsRequest{ + Base: lst.Base, + NodeID: nodeID, + Infos: infos, + Schema: lst.Schema, + LoadCondition: lst.LoadCondition, + }, + meta: lst.meta, + cluster: lst.cluster, + } + reScheduledTask = append(reScheduledTask, loadSegmentTask) + log.Debug("LoadSegmentTask: add a loadSegmentTask to RescheduleTasks", zap.Any("task", loadSegmentTask)) + + hasWatchQueryChannel := lst.cluster.hasWatchedQueryChannel(lst.ctx, nodeID, collectionID) + if !hasWatchQueryChannel { + queryChannel, queryResultChannel := lst.meta.GetQueryChannel(collectionID) + + addQueryChannelRequest := &querypb.AddQueryChannelRequest{ + Base: lst.Base, + NodeID: nodeID, + CollectionID: collectionID, + RequestChannelID: queryChannel, + ResultChannelID: queryResultChannel, + } + watchQueryChannelTask := &WatchQueryChannelTask{ + BaseTask: BaseTask{ + ctx: lst.ctx, + Condition: NewTaskCondition(lst.ctx), + triggerCondition: querypb.TriggerCondition_grpcRequest, + }, + + AddQueryChannelRequest: addQueryChannelRequest, + cluster: lst.cluster, + } + reScheduledTask = append(reScheduledTask, watchQueryChannelTask) + log.Debug("LoadSegmentTask: add a watchQueryChannelTask to RescheduleTasks", zap.Any("task", watchQueryChannelTask)) + } + } + + return reScheduledTask, nil +} + type ReleaseSegmentTask struct { BaseTask *querypb.ReleaseSegmentsRequest cluster *queryNodeCluster } +func (rst *ReleaseSegmentTask) Marshal() string { + return proto.MarshalTextString(rst.ReleaseSegmentsRequest) +} + +func (rst *ReleaseSegmentTask) IsValid() bool { + return rst.ctx != nil && rst.cluster.nodes[rst.NodeID].isOnService() +} + func (rst *ReleaseSegmentTask) Type() commonpb.MsgType { return rst.Base.MsgType } + func (rst *ReleaseSegmentTask) Timestamp() Timestamp { return rst.Base.Timestamp } + func (rst *ReleaseSegmentTask) PreExecute(ctx context.Context) error { segmentIDs := rst.SegmentIDs log.Debug("start do releaseSegmentTask", - zap.Int64s("segmentIDs", segmentIDs)) + zap.Int64s("segmentIDs", segmentIDs), + zap.Int64("loaded nodeID", rst.NodeID), + zap.Int64("taskID", rst.ID())) return nil } + func (rst *ReleaseSegmentTask) Execute(ctx context.Context) error { status, err := rst.cluster.ReleaseSegments(rst.ctx, rst.NodeID, rst.ReleaseSegmentsRequest) if err != nil { @@ -698,28 +848,42 @@ func (rst *ReleaseSegmentTask) Execute(ctx context.Context) error { rst.result = status log.Debug("releaseSegmentTask Execute done", - zap.Int64s("segmentIDs", rst.SegmentIDs)) + zap.Int64s("segmentIDs", rst.SegmentIDs), + zap.Int64("taskID", rst.ID())) return nil } + func (rst *ReleaseSegmentTask) PostExecute(ctx context.Context) error { segmentIDs := rst.SegmentIDs log.Debug("releaseSegmentTask postExecute done", - zap.Int64s("segmentIDs", segmentIDs)) + zap.Int64s("segmentIDs", segmentIDs), + zap.Int64("taskID", rst.ID())) return nil } type WatchDmChannelTask struct { BaseTask *querypb.WatchDmChannelsRequest + meta *meta cluster *queryNodeCluster } +func (wdt *WatchDmChannelTask) Marshal() string { + return proto.MarshalTextString(wdt.WatchDmChannelsRequest) +} + +func (wdt *WatchDmChannelTask) IsValid() bool { + return wdt.ctx != nil && wdt.cluster.nodes[wdt.NodeID].isOnService() +} + func (wdt *WatchDmChannelTask) Type() commonpb.MsgType { return wdt.Base.MsgType } + func (wdt *WatchDmChannelTask) Timestamp() Timestamp { return wdt.Base.Timestamp } + func (wdt *WatchDmChannelTask) PreExecute(ctx context.Context) error { channelInfos := wdt.Infos channels := make([]string, 0) @@ -727,9 +891,12 @@ func (wdt *WatchDmChannelTask) PreExecute(ctx context.Context) error { channels = append(channels, info.ChannelName) } log.Debug("start do watchDmChannelTask", - zap.Strings("dmChannels", channels)) + zap.Strings("dmChannels", channels), + zap.Int64("loaded nodeID", wdt.NodeID), + zap.Int64("taskID", wdt.ID())) return nil } + func (wdt *WatchDmChannelTask) Execute(ctx context.Context) error { status, err := wdt.cluster.WatchDmChannels(wdt.ctx, wdt.NodeID, wdt.WatchDmChannelsRequest) if err != nil { @@ -738,208 +905,393 @@ func (wdt *WatchDmChannelTask) Execute(ctx context.Context) error { } wdt.result = status - log.Debug("watchDmChannelsTask Execute done") + log.Debug("watchDmChannelsTask Execute done", + zap.Int64("taskID", wdt.ID())) return nil } + func (wdt *WatchDmChannelTask) PostExecute(ctx context.Context) error { - log.Debug("watchDmChannelTask postExecute done") + log.Debug("watchDmChannelTask postExecute done", + zap.Int64("taskID", wdt.ID())) return nil } +func (wdt *WatchDmChannelTask) Reschedule() ([]task, error) { + collectionID := wdt.CollectionID + channelIDs := make([]string, 0) + reScheduledTask := make([]task, 0) + for _, info := range wdt.Infos { + channelID := info.ChannelName + channelIDs = append(channelIDs, channelID) + } + + channel2Nodes := shuffleChannelsToQueryNode(channelIDs, wdt.cluster) + node2channelInfos := make(map[int64][]*datapb.VchannelInfo) + for index, info := range wdt.Infos { + nodeID := channel2Nodes[index] + if _, ok := node2channelInfos[nodeID]; !ok { + node2channelInfos[nodeID] = make([]*datapb.VchannelInfo, 0) + } + node2channelInfos[nodeID] = append(node2channelInfos[nodeID], info) + } + + for nodeID, infos := range node2channelInfos { + loadSegmentTask := &WatchDmChannelTask{ + BaseTask: wdt.BaseTask, + WatchDmChannelsRequest: &querypb.WatchDmChannelsRequest{ + Base: wdt.Base, + NodeID: nodeID, + CollectionID: wdt.CollectionID, + PartitionID: wdt.PartitionID, + Infos: infos, + Schema: wdt.Schema, + ExcludeInfos: wdt.ExcludeInfos, + }, + meta: wdt.meta, + cluster: wdt.cluster, + } + reScheduledTask = append(reScheduledTask, loadSegmentTask) + log.Debug("WatchDmChannelTask: add a watchDmChannelTask to RescheduleTasks", zap.Any("task", loadSegmentTask)) + + hasWatchQueryChannel := wdt.cluster.hasWatchedQueryChannel(wdt.ctx, nodeID, collectionID) + if !hasWatchQueryChannel { + queryChannel, queryResultChannel := wdt.meta.GetQueryChannel(collectionID) + + addQueryChannelRequest := &querypb.AddQueryChannelRequest{ + Base: wdt.Base, + NodeID: nodeID, + CollectionID: collectionID, + RequestChannelID: queryChannel, + ResultChannelID: queryResultChannel, + } + watchQueryChannelTask := &WatchQueryChannelTask{ + BaseTask: BaseTask{ + ctx: wdt.ctx, + Condition: NewTaskCondition(wdt.ctx), + triggerCondition: querypb.TriggerCondition_grpcRequest, + }, + + AddQueryChannelRequest: addQueryChannelRequest, + cluster: wdt.cluster, + } + reScheduledTask = append(reScheduledTask, watchQueryChannelTask) + log.Debug("WatchDmChannelTask: add a watchQueryChannelTask to RescheduleTasks", zap.Any("task", watchQueryChannelTask)) + } + } + + return reScheduledTask, nil +} + type WatchQueryChannelTask struct { BaseTask *querypb.AddQueryChannelRequest cluster *queryNodeCluster } -func (aqt *WatchQueryChannelTask) Type() commonpb.MsgType { - return aqt.Base.MsgType +func (wqt *WatchQueryChannelTask) Marshal() string { + return proto.MarshalTextString(wqt.AddQueryChannelRequest) } -func (aqt *WatchQueryChannelTask) Timestamp() Timestamp { - return aqt.Base.Timestamp + +func (wqt *WatchQueryChannelTask) IsValid() bool { + return wqt.ctx != nil && wqt.cluster.nodes[wqt.NodeID].isOnService() } -func (aqt *WatchQueryChannelTask) PreExecute(ctx context.Context) error { + +func (wqt *WatchQueryChannelTask) Type() commonpb.MsgType { + return wqt.Base.MsgType +} + +func (wqt *WatchQueryChannelTask) Timestamp() Timestamp { + return wqt.Base.Timestamp +} + +func (wqt *WatchQueryChannelTask) PreExecute(ctx context.Context) error { log.Debug("start do WatchQueryChannelTask", - zap.Int64("collectionID", aqt.CollectionID), - zap.String("queryChannel", aqt.RequestChannelID), - zap.String("queryResultChannel", aqt.ResultChannelID)) + zap.Int64("collectionID", wqt.CollectionID), + zap.String("queryChannel", wqt.RequestChannelID), + zap.String("queryResultChannel", wqt.ResultChannelID), + zap.Int64("loaded nodeID", wqt.NodeID), + zap.Int64("taskID", wqt.ID())) return nil } -func (aqt *WatchQueryChannelTask) Execute(ctx context.Context) error { - status, err := aqt.cluster.AddQueryChannel(aqt.ctx, aqt.NodeID, aqt.AddQueryChannelRequest) + +func (wqt *WatchQueryChannelTask) Execute(ctx context.Context) error { + status, err := wqt.cluster.AddQueryChannel(wqt.ctx, wqt.NodeID, wqt.AddQueryChannelRequest) if err != nil { - aqt.result = status + wqt.result = status return err } - aqt.result = status + wqt.result = status log.Debug("watchQueryChannelTask Execute done", - zap.Int64("collectionID", aqt.CollectionID), - zap.String("queryChannel", aqt.RequestChannelID), - zap.String("queryResultChannel", aqt.ResultChannelID)) + zap.Int64("collectionID", wqt.CollectionID), + zap.String("queryChannel", wqt.RequestChannelID), + zap.String("queryResultChannel", wqt.ResultChannelID), + zap.Int64("taskID", wqt.ID())) return nil } -func (aqt *WatchQueryChannelTask) PostExecute(ctx context.Context) error { +func (wqt *WatchQueryChannelTask) PostExecute(ctx context.Context) error { log.Debug("WatchQueryChannelTask postExecute done", - zap.Int64("collectionID", aqt.CollectionID), - zap.String("queryChannel", aqt.RequestChannelID), - zap.String("queryResultChannel", aqt.ResultChannelID)) + zap.Int64("collectionID", wqt.CollectionID), + zap.String("queryChannel", wqt.RequestChannelID), + zap.String("queryResultChannel", wqt.ResultChannelID), + zap.Int64("taskID", wqt.ID())) return nil } -func mockGetRecoveryInfoFromDataService(ctx context.Context, - master types.MasterService, - dataService types.DataService, - req *datapb.GetRecoveryInfoRequest) (*datapb.GetRecoveryInfoResponse, error) { - segmentIDs := make([]UniqueID, 0) - showSegmentRequest := &milvuspb.ShowSegmentsRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_ShowSegments, - }, - CollectionID: req.CollectionID, - PartitionID: req.PartitionID, +//****************************handoff task********************************// +type HandoffTask struct { +} + +//*********************** ***load balance task*** ************************// +type LoadBalanceTask struct { + BaseTask + *querypb.LoadBalanceRequest + master types.MasterService + dataService types.DataService + cluster *queryNodeCluster + meta *meta +} + +func (lbt *LoadBalanceTask) Marshal() string { + return proto.MarshalTextString(lbt.LoadBalanceRequest) +} + +func (lbt *LoadBalanceTask) Type() commonpb.MsgType { + return lbt.Base.MsgType +} + +func (lbt *LoadBalanceTask) Timestamp() Timestamp { + return lbt.Base.Timestamp +} + +func (lbt *LoadBalanceTask) PreExecute(ctx context.Context) error { + log.Debug("start do LoadBalanceTask", + zap.Int64s("sourceNodeIDs", lbt.SourceNodeIDs), + zap.Any("balanceReason", lbt.BalanceReason), + zap.Int64("taskID", lbt.ID())) + return nil +} + +func (lbt *LoadBalanceTask) Execute(ctx context.Context) error { + status := &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, } - showSegmentsResponse, err := master.ShowSegments(ctx, showSegmentRequest) - if err != nil { - return nil, err + if lbt.triggerCondition == querypb.TriggerCondition_nodeDown { + for _, nodeID := range lbt.SourceNodeIDs { + lbt.meta.deleteSegmentInfoByNodeID(nodeID) + collectionInfos := lbt.cluster.nodes[nodeID].collectionInfos + for collectionID, info := range collectionInfos { + loadCollection := lbt.meta.collectionInfos[collectionID].LoadCollection + schema := lbt.meta.collectionInfos[collectionID].Schema + partitionIDs := info.PartitionIDs + + segmentsToLoad := make([]UniqueID, 0) + segment2BingLog := make(map[UniqueID]*querypb.SegmentLoadInfo) + channelsToWatch := make([]string, 0) + watchRequestsInPartition := make([]*querypb.WatchDmChannelsRequest, 0) + watchRequestsInCollection := make(map[string]*querypb.WatchDmChannelsRequest) + + dmChannels, err := lbt.meta.getDmChannelsByNodeID(collectionID, nodeID) + if err != nil { + status.Reason = err.Error() + lbt.result = status + return err + } + + for _, partitionID := range partitionIDs { + getRecoveryInfo := &datapb.GetRecoveryInfoRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_LoadBalanceSegments, + }, + CollectionID: collectionID, + PartitionID: partitionID, + } + recoveryInfo, err := lbt.dataService.GetRecoveryInfo(lbt.ctx, getRecoveryInfo) + if err != nil { + status.Reason = err.Error() + lbt.result = status + return err + } + + for _, channelInfo := range recoveryInfo.Channels { + for _, channel := range dmChannels { + if channelInfo.ChannelName == channel { + watchRequest := &querypb.WatchDmChannelsRequest{ + Base: lbt.Base, + CollectionID: collectionID, + Infos: []*datapb.VchannelInfo{channelInfo}, + Schema: schema, + } + if loadCollection { + if _, ok := watchRequestsInCollection[channel]; !ok { + watchRequestsInCollection[channel] = watchRequest + channelsToWatch = append(channelsToWatch, channel) + } else { + oldInfo := watchRequestsInCollection[channel].Infos[0] + newInfo := mergeVChannelInfo(oldInfo, channelInfo) + watchRequestsInCollection[channel].Infos = []*datapb.VchannelInfo{newInfo} + } + } else { + watchRequest.PartitionID = partitionID + channelsToWatch = append(channelsToWatch, channel) + watchRequestsInPartition = append(watchRequestsInPartition, watchRequest) + } + break + } + } + } + + for _, binlog := range recoveryInfo.Binlogs { + segmentID := binlog.SegmentID + if lbt.meta.hasSegmentInfo(segmentID) { + continue + } + segmentLoadInfo := &querypb.SegmentLoadInfo{ + SegmentID: segmentID, + PartitionID: partitionID, + CollectionID: collectionID, + BinlogPaths: make([]*datapb.FieldBinlog, 0), + } + segmentLoadInfo.BinlogPaths = append(segmentLoadInfo.BinlogPaths, binlog.FieldBinlogs...) + segmentsToLoad = append(segmentsToLoad, segmentID) + segment2BingLog[segmentID] = segmentLoadInfo + } + } + + segment2Nodes := shuffleSegmentsToQueryNode(segmentsToLoad, lbt.cluster) + watchRequest2Nodes := shuffleChannelsToQueryNode(channelsToWatch, lbt.cluster) + + watchQueryChannelInfo := make(map[int64]bool) + node2Segments := make(map[int64][]*querypb.SegmentLoadInfo) + for segmentID, id := range segment2Nodes { + if _, ok := node2Segments[id]; !ok { + node2Segments[id] = make([]*querypb.SegmentLoadInfo, 0) + } + node2Segments[id] = append(node2Segments[id], segment2BingLog[segmentID]) + if lbt.cluster.hasWatchedQueryChannel(lbt.ctx, id, collectionID) { + watchQueryChannelInfo[id] = true + continue + } + watchQueryChannelInfo[id] = false + } + for _, id := range watchRequest2Nodes { + if lbt.cluster.hasWatchedQueryChannel(lbt.ctx, id, collectionID) { + watchQueryChannelInfo[id] = true + continue + } + watchQueryChannelInfo[id] = false + } + + for id, segmentInfos := range node2Segments { + loadSegmentsRequest := &querypb.LoadSegmentsRequest{ + Base: lbt.Base, + NodeID: id, + Infos: segmentInfos, + Schema: schema, + LoadCondition: querypb.TriggerCondition_grpcRequest, + } + + loadSegmentTask := &LoadSegmentTask{ + BaseTask: BaseTask{ + ctx: lbt.ctx, + Condition: NewTaskCondition(lbt.ctx), + triggerCondition: querypb.TriggerCondition_grpcRequest, + }, + + LoadSegmentsRequest: loadSegmentsRequest, + meta: lbt.meta, + cluster: lbt.cluster, + } + lbt.AddChildTask(loadSegmentTask) + log.Debug("LoadBalanceTask: add a loadSegmentTask to loadBalanceTask's childTask", zap.Any("task", loadSegmentTask)) + } + + for index, id := range watchRequest2Nodes { + var watchRequest *querypb.WatchDmChannelsRequest + if loadCollection { + channel := channelsToWatch[index] + watchRequest = watchRequestsInCollection[channel] + } else { + watchRequest = watchRequestsInPartition[index] + } + watchRequest.NodeID = id + watchDmChannelTask := &WatchDmChannelTask{ + BaseTask: BaseTask{ + ctx: lbt.ctx, + Condition: NewTaskCondition(lbt.ctx), + triggerCondition: querypb.TriggerCondition_grpcRequest, + }, + WatchDmChannelsRequest: watchRequest, + meta: lbt.meta, + cluster: lbt.cluster, + } + lbt.AddChildTask(watchDmChannelTask) + log.Debug("LoadBalanceTask: add a watchDmChannelTask to loadBalanceTask's childTask", zap.Any("task", watchDmChannelTask)) + } + + for id, watched := range watchQueryChannelInfo { + if !watched { + queryChannel, queryResultChannel := lbt.meta.GetQueryChannel(collectionID) + + addQueryChannelRequest := &querypb.AddQueryChannelRequest{ + Base: lbt.Base, + NodeID: id, + CollectionID: collectionID, + RequestChannelID: queryChannel, + ResultChannelID: queryResultChannel, + } + watchQueryChannelTask := &WatchQueryChannelTask{ + BaseTask: BaseTask{ + ctx: lbt.ctx, + Condition: NewTaskCondition(lbt.ctx), + triggerCondition: querypb.TriggerCondition_grpcRequest, + }, + + AddQueryChannelRequest: addQueryChannelRequest, + cluster: lbt.cluster, + } + lbt.AddChildTask(watchQueryChannelTask) + log.Debug("LoadBalanceTask: add a watchQueryChannelTask to loadBalanceTask's childTask", zap.Any("task", watchQueryChannelTask)) + } + } + } + } } - segmentIDs = append(segmentIDs, showSegmentsResponse.SegmentIDs...) - getSegmentStatesResponse, err := dataService.GetSegmentStates(ctx, &datapb.GetSegmentStatesRequest{ - SegmentIDs: segmentIDs, - }) - if err != nil { - return nil, err - } - log.Debug("get segmentStates from dataService", zap.Any("states", getSegmentStatesResponse)) - segmentStates := make(map[UniqueID]*datapb.SegmentStateInfo) - channel2Segments := make(map[string][]UniqueID) - for _, state := range getSegmentStatesResponse.States { - segmentID := state.SegmentID - segmentStates[segmentID] = state - if state.StartPosition == nil || len(state.StartPosition.ChannelName) == 0 { + + //TODO:: + //if lbt.triggerCondition == querypb.TriggerCondition_loadBalance { + // return nil + //} + + log.Debug("LoadBalanceTask Execute done", + zap.Int64s("sourceNodeIDs", lbt.SourceNodeIDs), + zap.Any("balanceReason", lbt.BalanceReason), + zap.Int64("taskID", lbt.ID())) + return nil +} + +func (lbt *LoadBalanceTask) PostExecute(ctx context.Context) error { + log.Debug("LoadBalanceTask postExecute done", + zap.Int64s("sourceNodeIDs", lbt.SourceNodeIDs), + zap.Any("balanceReason", lbt.BalanceReason), + zap.Int64("taskID", lbt.ID())) + return nil +} + +func shuffleChannelsToQueryNode(dmChannels []string, cluster *queryNodeCluster) []int64 { + maxNumChannels := 0 + for nodeID, node := range cluster.nodes { + if !node.onService { continue } - channelName := state.StartPosition.ChannelName - if _, ok := channel2Segments[channelName]; !ok { - segments := make([]UniqueID, 0) - segments = append(segments, segmentID) - channel2Segments[channelName] = segments - } else { - channel2Segments[channelName] = append(channel2Segments[channelName], segmentID) - } - } - channelInfos := make([]*datapb.VchannelInfo, 0) - segmentBinlogs := make([]*datapb.SegmentBinlogs, 0) - for channel, segmentIDs := range channel2Segments { - channelInfo := &datapb.VchannelInfo{ - CollectionID: req.CollectionID, - ChannelName: channel, - SeekPosition: &internalpb.MsgPosition{ - ChannelName: channel, - }, - UnflushedSegments: make([]*datapb.SegmentInfo, 0), - FlushedSegments: make([]UniqueID, 0), - } - sort.Slice(segmentIDs, func(i, j int) bool { - return segmentIDs[i] < segmentIDs[j] - }) - for _, id := range segmentIDs { - if segmentStates[id].State == commonpb.SegmentState_Flushed { - channelInfo.FlushedSegments = append(channelInfo.FlushedSegments, id) - channelInfo.SeekPosition = segmentStates[id].EndPosition - continue - } - if segmentStates[id].StartPosition != nil { - checkpoint := &datapb.SegmentInfo{ - ID: id, - DmlPosition: segmentStates[id].StartPosition, - } - channelInfo.UnflushedSegments = append(channelInfo.UnflushedSegments, checkpoint) - if checkpoint.DmlPosition.Timestamp < channelInfo.SeekPosition.Timestamp || channelInfo.SeekPosition.Timestamp == 0 { - channelInfo.SeekPosition = checkpoint.DmlPosition - } - } - } - channelInfos = append(channelInfos, channelInfo) - - for _, id := range channelInfo.FlushedSegments { - segmentBinlog := &datapb.SegmentBinlogs{ - SegmentID: id, - FieldBinlogs: make([]*datapb.FieldBinlog, 0), - } - insertBinlogPathRequest := &datapb.GetInsertBinlogPathsRequest{ - SegmentID: id, - } - - pathResponse, err := dataService.GetInsertBinlogPaths(ctx, insertBinlogPathRequest) - if err != nil || pathResponse.Status.ErrorCode != commonpb.ErrorCode_Success { - return nil, err - } - if len(pathResponse.FieldIDs) != len(pathResponse.Paths) || len(pathResponse.FieldIDs) <= 0 { - return nil, errors.New("illegal GetInsertBinlogPathsResponse") - } - - for index, fieldID := range pathResponse.FieldIDs { - if len(pathResponse.Paths[index].Values) == 0 { - continue - } - fieldBingLog := &datapb.FieldBinlog{ - FieldID: fieldID, - Binlogs: pathResponse.Paths[index].Values, - } - segmentBinlog.FieldBinlogs = append(segmentBinlog.FieldBinlogs, fieldBingLog) - log.Debug("get binlog path from data service", - zap.Int64("collectionID", req.CollectionID), - zap.Int64("partitionID", req.PartitionID), - zap.Int64("segmentID", id), - zap.Int64("fieldID", fieldID), - zap.Strings("binlogPath", pathResponse.Paths[index].Values)) - } - segmentBinlogs = append(segmentBinlogs, segmentBinlog) - } - } - if len(channelInfos) == 0 { - // get physical channels - desColReq := &milvuspb.DescribeCollectionRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_DescribeCollection, - }, - CollectionID: req.CollectionID, - } - desColRsp, err := master.DescribeCollection(ctx, desColReq) - if err != nil { - log.Error("get physical channels failed, err = " + err.Error()) - return nil, err - } - for _, channel := range desColRsp.VirtualChannelNames { - channelInfo := &datapb.VchannelInfo{ - CollectionID: req.CollectionID, - ChannelName: channel, - } - channelInfos = append(channelInfos, channelInfo) - } - } - - return &datapb.GetRecoveryInfoResponse{ - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_Success, - }, - Channels: channelInfos, - Binlogs: segmentBinlogs, - }, nil -} - -func shuffleChannelsToQueryNode(dmChannels []string, cluster *queryNodeCluster) map[string]int64 { - maxNumChannels := 0 - for nodeID := range cluster.nodes { numChannels, _ := cluster.getNumDmChannels(nodeID) if numChannels > maxNumChannels { maxNumChannels = numChannels } } - res := make(map[string]int64) + res := make([]int64, 0) if len(dmChannels) == 0 { return res } @@ -949,20 +1301,26 @@ func shuffleChannelsToQueryNode(dmChannels []string, cluster *queryNodeCluster) for { lastOffset := offset if !loopAll { - for id := range cluster.nodes { + for id, node := range cluster.nodes { + if !node.isOnService() { + continue + } numSegments, _ := cluster.getNumSegments(id) if numSegments >= maxNumChannels { continue } - res[dmChannels[offset]] = id + res = append(res, id) offset++ if offset == len(dmChannels) { return res } } } else { - for id := range cluster.nodes { - res[dmChannels[offset]] = id + for id, node := range cluster.nodes { + if !node.isOnService() { + continue + } + res = append(res, id) offset++ if offset == len(dmChannels) { return res @@ -977,7 +1335,10 @@ func shuffleChannelsToQueryNode(dmChannels []string, cluster *queryNodeCluster) func shuffleSegmentsToQueryNode(segmentIDs []UniqueID, cluster *queryNodeCluster) map[UniqueID]int64 { maxNumSegments := 0 - for nodeID := range cluster.nodes { + for nodeID, node := range cluster.nodes { + if !node.isOnService() { + continue + } numSegments, _ := cluster.getNumSegments(nodeID) if numSegments > maxNumSegments { maxNumSegments = numSegments @@ -994,7 +1355,10 @@ func shuffleSegmentsToQueryNode(segmentIDs []UniqueID, cluster *queryNodeCluster for { lastOffset := offset if !loopAll { - for id := range cluster.nodes { + for id, node := range cluster.nodes { + if !node.isOnService() { + continue + } numSegments, _ := cluster.getNumSegments(id) if numSegments >= maxNumSegments { continue @@ -1006,7 +1370,10 @@ func shuffleSegmentsToQueryNode(segmentIDs []UniqueID, cluster *queryNodeCluster } } } else { - for id := range cluster.nodes { + for id, node := range cluster.nodes { + if !node.isOnService() { + continue + } res[segmentIDs[offset]] = id offset++ if offset == len(segmentIDs) { @@ -1051,100 +1418,3 @@ func mergeVChannelInfo(info1 *datapb.VchannelInfo, info2 *datapb.VchannelInfo) * FlushedSegments: flushedSegments, } } - -//****************************handoff task********************************// -type HandoffTask struct { -} - -//*********************** ***load balance task*** ************************// -type loadBalanceTask struct { -} - -//func watchDmChannels(ctx context.Context, -// dataService types.DataService, -// cluster *queryNodeCluster, -// meta *meta, -// dbID UniqueID, -// collectionID UniqueID, -// msgBase *commonpb.MsgBase, -// toWatchPosition map[string]*internalpb.MsgPosition, -// excludeSegment map[string][]UniqueID) error { -// col, err := meta.getCollectionByID(0, collectionID) -// if err != nil { -// return err -// } -// channelRequest := datapb.GetInsertChannelsRequest{ -// DbID: dbID, -// CollectionID: collectionID, -// } -// resp, err := dataService.GetInsertChannels(ctx, &channelRequest) -// if err != nil { -// return err -// } -// if len(resp.Values) == 0 { -// err = errors.New("haven't assign dm channel to collection") -// return err -// } -// -// dmChannels := resp.Values -// channelsWithoutPos := make([]string, 0) -// for _, channel := range dmChannels { -// findChannel := false -// ChannelsWithPos := col.dmChannels -// for _, ch := range ChannelsWithPos { -// if channel == ch { -// findChannel = true -// break -// } -// } -// if !findChannel { -// channelsWithoutPos = append(channelsWithoutPos, channel) -// } -// } -// -// err = meta.addDmChannels(dbID, collectionID, channelsWithoutPos) -// if err != nil { -// return err -// } -// //for _, ch := range channelsWithoutPos { -// // pos := &internalpb.MsgPosition{ -// // ChannelName: ch, -// // } -// // err = meta.addDmChannels(dbID, collectionID, chs) -// // if err != nil { -// // return err -// // } -// //} -// -// nodeID2Channels := shuffleChannelsToQueryNode(dbID, dmChannels, cluster) -// for nodeID, channels := range nodeID2Channels { -// //node := queryNodes[nodeID] -// watchDmChannelsInfo := make([]*querypb.WatchDmChannelInfo, 0) -// for _, ch := range channels { -// info := &querypb.WatchDmChannelInfo{ -// ChannelID: ch, -// //Pos: col.dmChannels2Pos[ch], -// //ExcludedSegments: col.excludeSegmentIds, -// } -// if _, ok := toWatchPosition[ch]; ok { -// info.Pos = toWatchPosition[ch] -// info.ExcludedSegments = excludeSegment[ch] -// } -// watchDmChannelsInfo = append(watchDmChannelsInfo, info) -// } -// request := &querypb.WatchDmChannelsRequest{ -// Base: msgBase, -// CollectionID: collectionID, -// ChannelIDs: channels, -// Infos: watchDmChannelsInfo, -// } -// _, err := cluster.WatchDmChannels(ctx, nodeID, request) -// if err != nil { -// return err -// } -// cluster.AddDmChannels(dbID, nodeID, channels, collectionID) -// log.Debug("query node ", zap.String("nodeID", strconv.FormatInt(nodeID, 10)), zap.String("watch channels", fmt.Sprintln(channels))) -// } -// -// return nil -//} diff --git a/internal/queryservice/task_scheduler.go b/internal/queryservice/task_scheduler.go index c163dad2c0..91d5927961 100644 --- a/internal/queryservice/task_scheduler.go +++ b/internal/queryservice/task_scheduler.go @@ -14,14 +14,24 @@ package queryservice import ( "container/list" "context" + "errors" + "fmt" + "path/filepath" + "strconv" "sync" + "github.com/golang/protobuf/proto" "github.com/opentracing/opentracing-go" "go.uber.org/zap" + "github.com/milvus-io/milvus/internal/allocator" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/types" "github.com/milvus-io/milvus/internal/util/trace" + "github.com/milvus-io/milvus/internal/util/tsoutil" oplog "github.com/opentracing/opentracing-go/log" ) @@ -31,7 +41,6 @@ type TaskQueue struct { maxTask int64 taskChan chan int // to block scheduler - scheduler *TaskScheduler sync.Mutex } @@ -77,6 +86,15 @@ func (queue *TaskQueue) addTask(tasks []task) { } } +func (queue *TaskQueue) addTaskToFront(t task) { + queue.taskChan <- 1 + if queue.tasks.Len() == 0 { + queue.tasks.PushBack(t) + } else { + queue.tasks.PushFront(t) + } +} + func (queue *TaskQueue) PopTask() task { queue.Lock() defer queue.Unlock() @@ -92,12 +110,11 @@ func (queue *TaskQueue) PopTask() task { return ft.Value.(task) } -func NewTaskQueue(scheduler *TaskScheduler) *TaskQueue { +func NewTaskQueue() *TaskQueue { return &TaskQueue{ - tasks: list.New(), - maxTask: 1024, - taskChan: make(chan int, 1024), - scheduler: scheduler, + tasks: list.New(), + maxTask: 1024, + taskChan: make(chan int, 1024), } } @@ -105,144 +122,557 @@ type TaskScheduler struct { triggerTaskQueue *TaskQueue activateTaskChan chan task meta *meta + cluster *queryNodeCluster taskIDAllocator func() (UniqueID, error) - kvBase *etcdkv.EtcdKV + client *etcdkv.EtcdKV + + master types.MasterService + dataService types.DataService wg sync.WaitGroup ctx context.Context cancel context.CancelFunc } -func NewTaskScheduler(ctx context.Context, meta *meta, kv *etcdkv.EtcdKV) *TaskScheduler { +func NewTaskScheduler(ctx context.Context, meta *meta, cluster *queryNodeCluster, kv *etcdkv.EtcdKV, master types.MasterService, dataService types.DataService) (*TaskScheduler, error) { ctx1, cancel := context.WithCancel(ctx) taskChan := make(chan task, 1024) s := &TaskScheduler{ ctx: ctx1, cancel: cancel, meta: meta, + cluster: cluster, activateTaskChan: taskChan, - kvBase: kv, + client: kv, + master: master, + dataService: dataService, + } + s.triggerTaskQueue = NewTaskQueue() + idAllocator := allocator.NewGlobalIDAllocator("idTimestamp", tsoutil.NewTSOKVBase(Params.EtcdEndpoints, Params.KvRootPath, "queryService task id")) + if err := idAllocator.Initialize(); err != nil { + log.Debug("QueryService idAllocator initialize failed", zap.Error(err)) + return nil, err + } + s.taskIDAllocator = func() (UniqueID, error) { + return idAllocator.AllocOne() + } + err := s.reloadFromKV() + if err != nil { + return nil, err } - s.triggerTaskQueue = NewTaskQueue(s) - //TODO::add etcd - //idAllocator := allocator.NewGlobalIDAllocator("queryService taskID", s.kvBase) - //s.taskIDAllocator = func() (UniqueID, error) { - // return idAllocator.AllocOne() - //} - return s + return s, nil +} + +func (scheduler *TaskScheduler) reloadFromKV() error { + triggerTaskIDKeys, triggerTaskValues, err := scheduler.client.LoadWithPrefix(triggerTaskPrefix) + if err != nil { + return err + } + activeTaskIDKeys, activeTaskValues, err := scheduler.client.LoadWithPrefix(activeTaskPrefix) + if err != nil { + return err + } + taskInfoKeys, taskInfoValues, err := scheduler.client.LoadWithPrefix(taskInfoPrefix) + if err != nil { + return err + } + + triggerTasks := make(map[int64]task) + for index := range triggerTaskIDKeys { + taskID, err := strconv.ParseInt(filepath.Base(triggerTaskIDKeys[index]), 10, 64) + if err != nil { + return err + } + t, err := scheduler.unmarshalTask(triggerTaskValues[index]) + if err != nil { + return err + } + triggerTasks[taskID] = t + } + + activeTasks := make(map[int64]task) + for index := range activeTaskIDKeys { + taskID, err := strconv.ParseInt(filepath.Base(activeTaskIDKeys[index]), 10, 64) + if err != nil { + return err + } + t, err := scheduler.unmarshalTask(activeTaskValues[index]) + if err != nil { + return err + } + activeTasks[taskID] = t + } + + taskInfos := make(map[int64]taskState) + for index := range taskInfoKeys { + taskID, err := strconv.ParseInt(filepath.Base(taskInfoKeys[index]), 10, 64) + if err != nil { + return err + } + value, err := strconv.ParseInt(taskInfoValues[index], 10, 64) + if err != nil { + return err + } + state := taskState(value) + taskInfos[taskID] = state + } + + var doneTriggerTask task = nil + for id, t := range triggerTasks { + if taskInfos[id] == taskDone { + doneTriggerTask = t + for _, childTask := range activeTasks { + t.AddChildTask(childTask) + } + continue + } + scheduler.triggerTaskQueue.addTask([]task{t}) + } + + if doneTriggerTask != nil { + scheduler.triggerTaskQueue.addTaskToFront(doneTriggerTask) + } + + return nil +} + +func (scheduler *TaskScheduler) unmarshalTask(t string) (task, error) { + header := commonpb.MsgHeader{} + err := proto.UnmarshalText(t, &header) + if err != nil { + return nil, fmt.Errorf("Failed to unmarshal message header, err %s ", err.Error()) + } + var newTask task + switch header.Base.MsgType { + case commonpb.MsgType_LoadCollection: + loadReq := querypb.LoadCollectionRequest{} + err = proto.UnmarshalText(t, &loadReq) + if err != nil { + log.Error(err.Error()) + } + loadCollectionTask := &LoadCollectionTask{ + BaseTask: BaseTask{ + ctx: scheduler.ctx, + Condition: NewTaskCondition(scheduler.ctx), + triggerCondition: querypb.TriggerCondition_grpcRequest, + }, + LoadCollectionRequest: &loadReq, + masterService: scheduler.master, + dataService: scheduler.dataService, + cluster: scheduler.cluster, + meta: scheduler.meta, + } + newTask = loadCollectionTask + case commonpb.MsgType_LoadPartitions: + loadReq := querypb.LoadPartitionsRequest{} + err = proto.UnmarshalText(t, &loadReq) + if err != nil { + log.Error(err.Error()) + } + loadPartitionTask := &LoadPartitionTask{ + BaseTask: BaseTask{ + ctx: scheduler.ctx, + Condition: NewTaskCondition(scheduler.ctx), + triggerCondition: querypb.TriggerCondition_grpcRequest, + }, + LoadPartitionsRequest: &loadReq, + dataService: scheduler.dataService, + cluster: scheduler.cluster, + meta: scheduler.meta, + } + newTask = loadPartitionTask + case commonpb.MsgType_ReleaseCollection: + loadReq := querypb.ReleaseCollectionRequest{} + err = proto.UnmarshalText(t, &loadReq) + if err != nil { + log.Error(err.Error()) + } + releaseCollectionTask := &ReleaseCollectionTask{ + BaseTask: BaseTask{ + ctx: scheduler.ctx, + Condition: NewTaskCondition(scheduler.ctx), + triggerCondition: querypb.TriggerCondition_grpcRequest, + }, + ReleaseCollectionRequest: &loadReq, + cluster: scheduler.cluster, + } + newTask = releaseCollectionTask + case commonpb.MsgType_ReleasePartitions: + loadReq := querypb.ReleasePartitionsRequest{} + err = proto.UnmarshalText(t, &loadReq) + if err != nil { + log.Error(err.Error()) + } + releasePartitionTask := &ReleasePartitionTask{ + BaseTask: BaseTask{ + ctx: scheduler.ctx, + Condition: NewTaskCondition(scheduler.ctx), + triggerCondition: querypb.TriggerCondition_grpcRequest, + }, + ReleasePartitionsRequest: &loadReq, + cluster: scheduler.cluster, + } + newTask = releasePartitionTask + case commonpb.MsgType_LoadSegments: + loadReq := querypb.LoadSegmentsRequest{} + err = proto.UnmarshalText(t, &loadReq) + if err != nil { + log.Error(err.Error()) + } + loadSegmentTask := &LoadSegmentTask{ + BaseTask: BaseTask{ + ctx: scheduler.ctx, + Condition: NewTaskCondition(scheduler.ctx), + triggerCondition: querypb.TriggerCondition_grpcRequest, + }, + LoadSegmentsRequest: &loadReq, + cluster: scheduler.cluster, + meta: scheduler.meta, + } + newTask = loadSegmentTask + case commonpb.MsgType_ReleaseSegments: + loadReq := querypb.ReleaseSegmentsRequest{} + err = proto.UnmarshalText(t, &loadReq) + if err != nil { + log.Error(err.Error()) + } + releaseSegmentTask := &ReleaseSegmentTask{ + BaseTask: BaseTask{ + ctx: scheduler.ctx, + Condition: NewTaskCondition(scheduler.ctx), + triggerCondition: querypb.TriggerCondition_grpcRequest, + }, + ReleaseSegmentsRequest: &loadReq, + cluster: scheduler.cluster, + } + newTask = releaseSegmentTask + case commonpb.MsgType_WatchDmChannels: + loadReq := querypb.WatchDmChannelsRequest{} + err = proto.UnmarshalText(t, &loadReq) + if err != nil { + log.Error(err.Error()) + } + watchDmChannelTask := &WatchDmChannelTask{ + BaseTask: BaseTask{ + ctx: scheduler.ctx, + Condition: NewTaskCondition(scheduler.ctx), + triggerCondition: querypb.TriggerCondition_grpcRequest, + }, + WatchDmChannelsRequest: &loadReq, + cluster: scheduler.cluster, + meta: scheduler.meta, + } + newTask = watchDmChannelTask + case commonpb.MsgType_WatchQueryChannels: + loadReq := querypb.AddQueryChannelRequest{} + err = proto.UnmarshalText(t, &loadReq) + if err != nil { + log.Error(err.Error()) + } + watchQueryChannelTask := &WatchQueryChannelTask{ + BaseTask: BaseTask{ + ctx: scheduler.ctx, + Condition: NewTaskCondition(scheduler.ctx), + triggerCondition: querypb.TriggerCondition_grpcRequest, + }, + AddQueryChannelRequest: &loadReq, + cluster: scheduler.cluster, + } + newTask = watchQueryChannelTask + case commonpb.MsgType_LoadBalanceSegments: + loadReq := querypb.LoadBalanceRequest{} + err = proto.UnmarshalText(t, &loadReq) + if err != nil { + log.Error(err.Error()) + } + loadBalanceTask := &LoadBalanceTask{ + BaseTask: BaseTask{ + ctx: scheduler.ctx, + Condition: NewTaskCondition(scheduler.ctx), + triggerCondition: loadReq.BalanceReason, + }, + LoadBalanceRequest: &loadReq, + master: scheduler.master, + dataService: scheduler.dataService, + cluster: scheduler.cluster, + meta: scheduler.meta, + } + newTask = loadBalanceTask + default: + err = errors.New("inValid msg type when unMarshal task") + log.Error(err.Error()) + return nil, err + } + + return newTask, nil } func (scheduler *TaskScheduler) Enqueue(tasks []task) { - //TODO::open when add etcd - //for _, t := range tasks { - // id, err := scheduler.taskIDAllocator() - // if err != nil { - // log.Error(err.Error()) - // } - // t.SetID(id) - //} + for _, t := range tasks { + id, err := scheduler.taskIDAllocator() + if err != nil { + log.Error(err.Error()) + } + t.SetID(id) + kvs := make(map[string]string) + taskKey := fmt.Sprintf("%s/%d", triggerTaskPrefix, t.ID()) + kvs[taskKey] = t.Marshal() + stateKey := fmt.Sprintf("%s/%d", taskInfoPrefix, t.ID()) + kvs[stateKey] = string(taskUndo) + err = scheduler.client.MultiSave(kvs) + if err != nil { + log.Error("error when save trigger task to etcd", zap.Int64("taskID", t.ID())) + } + log.Debug("EnQueue a triggerTask and save to etcd", zap.Int64("taskID", t.ID())) + } + scheduler.triggerTaskQueue.addTask(tasks) } -func (scheduler *TaskScheduler) processTask(t task) { +func (scheduler *TaskScheduler) processTask(t task) error { span, ctx := trace.StartSpanFromContext(t.TraceCtx(), opentracing.Tags{ "Type": t.Type(), "ID": t.ID(), }) defer span.Finish() - span.LogFields(oplog.Int64("scheduler process PreExecute", t.ID())) - err := t.PreExecute(ctx) + span.LogFields(oplog.Int64("processTask: scheduler process PreExecute", t.ID())) + key := fmt.Sprintf("%s/%d", taskInfoPrefix, t.ID()) + err := scheduler.client.Save(key, string(taskDoing)) - defer func() { - t.Notify(err) - }() if err != nil { - log.Debug("preExecute err", zap.String("reason", err.Error())) + log.Debug("processTask: update task state err", zap.String("reason", err.Error())) trace.LogError(span, err) - return + return err } - span.LogFields(oplog.Int64("scheduler process Execute", t.ID())) + err = t.PreExecute(ctx) + if err != nil { + log.Debug("processTask: preExecute err", zap.String("reason", err.Error())) + trace.LogError(span, err) + return err + } + + span.LogFields(oplog.Int64("processTask: scheduler process Execute", t.ID())) err = t.Execute(ctx) if err != nil { - log.Debug("execute err", zap.String("reason", err.Error())) + log.Debug("processTask: execute err", zap.String("reason", err.Error())) trace.LogError(span, err) - return + return err } - span.LogFields(oplog.Int64("scheduler process PostExecute", t.ID())) + span.LogFields(oplog.Int64("processTask: scheduler process PostExecute", t.ID())) err = t.PostExecute(ctx) + if err != nil { + log.Debug("processTask: postExecute err", zap.String("reason", err.Error())) + trace.LogError(span, err) + return err + } + + for _, childTask := range t.GetChildTask() { + if childTask == nil { + log.Error("processTask: child task equal nil") + continue + } + + id, err := scheduler.taskIDAllocator() + if err != nil { + return err + } + childTask.SetID(id) + kvs := make(map[string]string) + taskKey := fmt.Sprintf("%s/%d", activeTaskPrefix, childTask.ID()) + kvs[taskKey] = t.Marshal() + stateKey := fmt.Sprintf("%s/%d", taskInfoPrefix, childTask.ID()) + kvs[stateKey] = string(taskUndo) + err = scheduler.client.MultiSave(kvs) + if err != nil { + return err + } + log.Debug("processTask: save active task to etcd", zap.Int64("parent taskID", t.ID()), zap.Int64("child taskID", childTask.ID())) + } + + err = scheduler.client.Save(key, string(taskDone)) + if err != nil { + log.Debug("processTask: update task state err", zap.String("reason", err.Error())) + trace.LogError(span, err) + return err + } + return err } func (scheduler *TaskScheduler) scheduleLoop() { defer scheduler.wg.Done() - var w sync.WaitGroup + activeTaskWg := &sync.WaitGroup{} + var err error = nil for { select { case <-scheduler.ctx.Done(): return case <-scheduler.triggerTaskQueue.Chan(): t := scheduler.triggerTaskQueue.PopTask() - log.Debug("pop a triggerTask from triggerTaskQueue") - scheduler.processTask(t) - //TODO::add active task to etcd - w.Add(2) - go scheduler.addActivateTask(&w, t) - //TODO::handle active task return error, maybe node down... - go scheduler.processActivateTask(&w) - w.Wait() - //TODO:: delete trigger task from etcd + log.Debug("scheduleLoop: pop a triggerTask from triggerTaskQueue", zap.Int64("taskID", t.ID())) + if t.State() < taskDone { + err = scheduler.processTask(t) + if err != nil { + log.Error("scheduleLoop: process task error", zap.Any("error", err.Error())) + continue + } + } + log.Debug("scheduleLoop: num of child task", zap.Int("num child task", len(t.GetChildTask()))) + for _, childTask := range t.GetChildTask() { + if childTask != nil { + log.Debug("scheduleLoop: add a activate task to activateChan", zap.Int64("taskID", childTask.ID())) + scheduler.activateTaskChan <- childTask + activeTaskWg.Add(1) + go scheduler.waitActivateTaskDone(activeTaskWg, childTask) + } + } + activeTaskWg.Wait() + if t.Type() == commonpb.MsgType_ReleaseCollection || t.Type() == commonpb.MsgType_ReleasePartitions { + t.Notify(err) + } + keys := make([]string, 0) + taskKey := fmt.Sprintf("%s/%d", triggerTaskPrefix, t.ID()) + stateKey := fmt.Sprintf("%s/%d", taskInfoPrefix, t.ID()) + keys = append(keys, taskKey) + keys = append(keys, stateKey) + err = scheduler.client.MultiRemove(keys) + if err != nil { + log.Error("scheduleLoop: error when remove trigger task to etcd", zap.Int64("taskID", t.ID())) + } + log.Debug("scheduleLoop: trigger task done and delete from etcd", zap.Int64("taskID", t.ID())) } } } -func (scheduler *TaskScheduler) addActivateTask(wg *sync.WaitGroup, t task) { - defer wg.Done() - var activeTaskWg sync.WaitGroup - log.Debug("num of child task", zap.Int("num child task", len(t.GetChildTask()))) - for _, childTask := range t.GetChildTask() { - if childTask != nil { - log.Debug("add a activate task to activateChan") - scheduler.activateTaskChan <- childTask - activeTaskWg.Add(1) - go scheduler.waitActivateTaskDone(&activeTaskWg, childTask) - } - } - scheduler.activateTaskChan <- nil - activeTaskWg.Wait() -} - func (scheduler *TaskScheduler) waitActivateTaskDone(wg *sync.WaitGroup, t task) { defer wg.Done() err := t.WaitToFinish() if err != nil { - //TODO:: redo task - log.Error("waitActivateTaskDone: activate task return err") + log.Debug("waitActivateTaskDone: activate task return err", zap.Any("error", err.Error()), zap.Int64("taskID", t.ID())) + redoFunc1 := func() { + if !t.IsValid() { + reScheduledTasks, err := t.Reschedule() + if err != nil { + log.Error(err.Error()) + return + } + removes := make([]string, 0) + taskKey := fmt.Sprintf("%s/%d", activeTaskPrefix, t.ID()) + removes = append(removes, taskKey) + stateKey := fmt.Sprintf("%s/%d", taskInfoPrefix, t.ID()) + removes = append(removes, stateKey) + + saves := make(map[string]string) + reSchedID := make([]int64, 0) + for _, rt := range reScheduledTasks { + if rt != nil { + id, err := scheduler.taskIDAllocator() + if err != nil { + log.Error(err.Error()) + continue + } + rt.SetID(id) + taskKey := fmt.Sprintf("%s/%d", activeTaskPrefix, rt.ID()) + saves[taskKey] = rt.Marshal() + stateKey := fmt.Sprintf("%s/%d", taskInfoPrefix, rt.ID()) + saves[stateKey] = string(taskUndo) + reSchedID = append(reSchedID, rt.ID()) + } + } + err = scheduler.client.MultiSaveAndRemove(saves, removes) + if err != nil { + log.Error("waitActivateTaskDone: error when save and remove task from etcd") + } + log.Debug("waitActivateTaskDone: delete failed active task and save reScheduled task to etcd", zap.Int64("failed taskID", t.ID()), zap.Int64s("reScheduled taskIDs", reSchedID)) + + for _, rt := range reScheduledTasks { + if rt != nil { + log.Debug("waitActivateTaskDone: add a reScheduled active task to activateChan", zap.Int64("taskID", rt.ID())) + scheduler.activateTaskChan <- rt + wg.Add(1) + go scheduler.waitActivateTaskDone(wg, rt) + } + } + //delete task from etcd + } else { + log.Debug("waitActivateTaskDone: retry the active task", zap.Int64("taskID", t.ID())) + scheduler.activateTaskChan <- t + wg.Add(1) + go scheduler.waitActivateTaskDone(wg, t) + } + } + + redoFunc2 := func() { + if t.IsValid() { + scheduler.activateTaskChan <- t + wg.Add(1) + go scheduler.waitActivateTaskDone(wg, t) + } + } + + switch t.Type() { + case commonpb.MsgType_LoadSegments: + redoFunc1() + case commonpb.MsgType_WatchDmChannels: + redoFunc1() + case commonpb.MsgType_WatchQueryChannels: + redoFunc2() + case commonpb.MsgType_ReleaseSegments: + redoFunc2() + case commonpb.MsgType_ReleaseCollection: + redoFunc2() + case commonpb.MsgType_ReleasePartitions: + redoFunc2() + default: + //TODO:: case commonpb.MsgType_RemoveDmChannels: + } + } else { + keys := make([]string, 0) + taskKey := fmt.Sprintf("%s/%d", activeTaskPrefix, t.ID()) + stateKey := fmt.Sprintf("%s/%d", taskInfoPrefix, t.ID()) + keys = append(keys, taskKey) + keys = append(keys, stateKey) + err = scheduler.client.MultiRemove(keys) + if err != nil { + log.Error("waitActivateTaskDone: error when remove task from etcd", zap.Int64("taskID", t.ID())) + } + log.Debug("waitActivateTaskDone: delete activate task from etcd", zap.Int64("taskID", t.ID())) } - log.Debug("one activate task done") + log.Debug("waitActivateTaskDone: one activate task done", zap.Int64("taskID", t.ID())) } -func (scheduler *TaskScheduler) processActivateTask(wg *sync.WaitGroup) { - defer wg.Done() +func (scheduler *TaskScheduler) processActivateTaskLoop() { + defer scheduler.wg.Done() for { select { case <-scheduler.ctx.Done(): return case t := <-scheduler.activateTaskChan: if t == nil { - return + log.Error("processActivateTaskLoop: pop a nil active task", zap.Int64("taskID", t.ID())) + continue } - log.Debug("pop a activate task from activateChan") - scheduler.processTask(t) - //TODO:: delete active task from etcd + stateKey := fmt.Sprintf("%s/%d", taskInfoPrefix, t.ID()) + err := scheduler.client.Save(stateKey, string(taskDoing)) + if err != nil { + t.Notify(err) + continue + } + log.Debug("processActivateTaskLoop: pop a active task from activateChan", zap.Int64("taskID", t.ID())) + go func() { + err := scheduler.processTask(t) + t.Notify(err) + }() } } } func (scheduler *TaskScheduler) Start() error { - scheduler.wg.Add(1) + scheduler.wg.Add(2) go scheduler.scheduleLoop() + go scheduler.processActivateTaskLoop() return nil } diff --git a/tests/python_test/collection/test_load_collection.py b/tests/python_test/collection/test_load_collection.py index 7763c4244e..ad6fba3e49 100644 --- a/tests/python_test/collection/test_load_collection.py +++ b/tests/python_test/collection/test_load_collection.py @@ -499,6 +499,7 @@ class TestLoadPartition: connect.create_index(binary_collection, default_binary_vec_field_name, get_binary_index) connect.load_partitions(binary_collection, [default_tag]) + @pytest.mark.skip("xige-16-search-without-insert") @pytest.mark.tags(CaseLabel.tags_smoke) def test_load_empty_partition(self, connect, collection): ''' diff --git a/tests/python_test/entity/test_insert.py b/tests/python_test/entity/test_insert.py index 7b3a1c8a46..32ead0ebe5 100644 --- a/tests/python_test/entity/test_insert.py +++ b/tests/python_test/entity/test_insert.py @@ -885,6 +885,7 @@ class TestInsertMultiCollections: stats = connect.get_collection_stats(collection_name) assert stats[row_count] == 1 + @pytest.mark.skip("xige-16-search-without-insert") @pytest.mark.timeout(ADD_TIMEOUT) @pytest.mark.tags(CaseLabel.tags_smoke) def test_insert_entity_search_entity_another(self, connect, collection): diff --git a/tests/python_test/entity/test_search.py b/tests/python_test/entity/test_search.py index 3700191455..f690c2783f 100644 --- a/tests/python_test/entity/test_search.py +++ b/tests/python_test/entity/test_search.py @@ -1154,6 +1154,7 @@ class TestSearchDSL(object): assert len(res) == nq assert len(res[0]) == 0 + @pytest.mark.skip("xige-16-search-without-insert") @pytest.mark.tags(CaseLabel.tags_smoke) def test_query_complex_dsl(self, connect, collection): '''