admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-07 01:28:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Remove Load Delta Channel Request (#18497)

Browse Source 
Fix #18393

remove watch delta channel task related logic

Co-authored-by: xiaofan-luan <xiaofan.luan@zilliz.com>
Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>

Co-authored-by: Congqi Xia <congqi.xia@zilliz.com>
This commit is contained in:
Xiaofan 2022-08-04 17:50:37 +08:00 committed by GitHub
parent c345df41a7
commit b427450593
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 468 additions and 1090 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
cmd/milvus/mck.go
View File

@ -624,21 +624,8 @@ func (c *mck) unmarshalTask(taskID int64, t string) (string, []int64, []int64, e
log.Info("WatchDmChannels", zap.String("detail", fmt.Sprintf("+%v", loadReq))) log.Info("WatchDmChannels", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
return "WatchDmChannels", removeRepeatElement(partitionIDs), removeRepeatElement(segmentIDs), nil return "WatchDmChannels", removeRepeatElement(partitionIDs), removeRepeatElement(segmentIDs), nil
case commonpb.MsgType_WatchDeltaChannels: case commonpb.MsgType_WatchDeltaChannels:
loadReq := querypb.WatchDeltaChannelsRequest{} log.Warn("legacy WatchDeltaChannels type found, ignore")
err = proto.Unmarshal([]byte(t), &loadReq) return "WatchQueryChannels", emptyInt64(), emptyInt64(), nil
if err != nil {
return errReturn(taskID, "WatchDeltaChannelsRequest", err)
}
var partitionIDs []int64
var segmentIDs []int64
if loadReq.LoadMeta != nil {
partitionIDs = append(partitionIDs, loadReq.LoadMeta.PartitionIDs...)
}
pids, sids := c.extractVchannelInfo(taskID, loadReq.Infos)
partitionIDs = append(partitionIDs, pids...)
segmentIDs = append(segmentIDs, sids...)
log.Info("WatchDeltaChannels", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
return "WatchDeltaChannels", removeRepeatElement(partitionIDs), removeRepeatElement(segmentIDs), nil
case commonpb.MsgType_WatchQueryChannels: case commonpb.MsgType_WatchQueryChannels:
log.Warn("legacy WatchQueryChannels type found, ignore") log.Warn("legacy WatchQueryChannels type found, ignore")
return "WatchQueryChannels", emptyInt64(), emptyInt64(), nil return "WatchQueryChannels", emptyInt64(), emptyInt64(), nil

14
internal/distributed/querynode/client/client.go
View File

@ -152,20 +152,6 @@ func (c *Client) WatchDmChannels(ctx context.Context, req *querypb.WatchDmChanne
return ret.(*commonpb.Status), err return ret.(*commonpb.Status), err
} }
// WatchDeltaChannels watches the channels about data manipulation.
func (c *Client) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) {
ret, err := c.grpcClient.ReCall(ctx, func(client interface{}) (interface{}, error) {
if !funcutil.CheckCtxValid(ctx) {
return nil, ctx.Err()
}
return client.(querypb.QueryNodeClient).WatchDeltaChannels(ctx, req)
})
if err != nil || ret == nil {
return nil, err
}
return ret.(*commonpb.Status), err
}
// LoadSegments loads the segments to search. // LoadSegments loads the segments to search.
func (c *Client) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { func (c *Client) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) {
ret, err := c.grpcClient.ReCall(ctx, func(client interface{}) (interface{}, error) { ret, err := c.grpcClient.ReCall(ctx, func(client interface{}) (interface{}, error) {

3
internal/distributed/querynode/client/client_test.go
View File

@ -90,9 +90,6 @@ func Test_NewClient(t *testing.T) {
r12, err := client.GetMetrics(ctx, nil) r12, err := client.GetMetrics(ctx, nil)
retCheck(retNotNil, r12, err) retCheck(retNotNil, r12, err)
r13, err := client.WatchDeltaChannels(ctx, nil)
retCheck(retNotNil, r13, err)
r14, err := client.Search(ctx, nil) r14, err := client.Search(ctx, nil)
retCheck(retNotNil, r14, err) retCheck(retNotNil, r14, err)

6
internal/distributed/querynode/service.go
View File

@ -262,12 +262,6 @@ func (s *Server) WatchDmChannels(ctx context.Context, req *querypb.WatchDmChanne
return s.querynode.WatchDmChannels(ctx, req) return s.querynode.WatchDmChannels(ctx, req)
} }
// WatchDeltaChannels watches the channels about data manipulation.
func (s *Server) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) {
// ignore ctx
return s.querynode.WatchDeltaChannels(ctx, req)
}
// LoadSegments loads the segments to search. // LoadSegments loads the segments to search.
func (s *Server) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { func (s *Server) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) {
// ignore ctx // ignore ctx

4
internal/distributed/querynode/service_test.go
View File

@ -82,10 +82,6 @@ func (m *MockQueryNode) WatchDmChannels(ctx context.Context, req *querypb.WatchD
return m.status, m.err return m.status, m.err
} }
func (m *MockQueryNode) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) {
return m.status, m.err
}
func (m *MockQueryNode) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { func (m *MockQueryNode) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) {
return m.status, m.err return m.status, m.err
} }

10
internal/proto/query_coord.proto
View File

@ -41,7 +41,6 @@ service QueryNode {
rpc GetStatisticsChannel(internal.GetStatisticsChannelRequest) returns(milvus.StringResponse){} rpc GetStatisticsChannel(internal.GetStatisticsChannelRequest) returns(milvus.StringResponse){}
rpc WatchDmChannels(WatchDmChannelsRequest) returns (common.Status) {} rpc WatchDmChannels(WatchDmChannelsRequest) returns (common.Status) {}
rpc WatchDeltaChannels(WatchDeltaChannelsRequest) returns (common.Status) {}
rpc LoadSegments(LoadSegmentsRequest) returns (common.Status) {} rpc LoadSegments(LoadSegmentsRequest) returns (common.Status) {}
rpc ReleaseCollection(ReleaseCollectionRequest) returns (common.Status) {} rpc ReleaseCollection(ReleaseCollectionRequest) returns (common.Status) {}
rpc ReleasePartitions(ReleasePartitionsRequest) returns (common.Status) {} rpc ReleasePartitions(ReleasePartitionsRequest) returns (common.Status) {}
@ -185,15 +184,6 @@ message WatchDmChannelsRequest {
map<int64, data.SegmentInfo> segment_infos = 10; map<int64, data.SegmentInfo> segment_infos = 10;
} }
message WatchDeltaChannelsRequest {
common.MsgBase base = 1;
int64 nodeID = 2;
int64 collectionID = 3;
repeated data.VchannelInfo infos = 4;
int64 replica_id = 5;
LoadMetaInfo load_meta = 9;
}
message SegmentLoadInfo { message SegmentLoadInfo {
int64 segmentID = 1; int64 segmentID = 1;
int64 partitionID = 2; int64 partitionID = 2;

527
internal/proto/querypb/query_coord.pb.go
View File

@ -1283,85 +1283,6 @@ func (m *WatchDmChannelsRequest) GetSegmentInfos() map[int64]*datapb.SegmentInfo
return nil return nil
} }
type WatchDeltaChannelsRequest struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
NodeID int64 `protobuf:"varint,2,opt,name=nodeID,proto3" json:"nodeID,omitempty"`
CollectionID int64 `protobuf:"varint,3,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
Infos []*datapb.VchannelInfo `protobuf:"bytes,4,rep,name=infos,proto3" json:"infos,omitempty"`
ReplicaId int64 `protobuf:"varint,5,opt,name=replica_id,json=replicaId,proto3" json:"replica_id,omitempty"`
LoadMeta *LoadMetaInfo `protobuf:"bytes,9,opt,name=load_meta,json=loadMeta,proto3" json:"load_meta,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *WatchDeltaChannelsRequest) Reset() { *m = WatchDeltaChannelsRequest{} }
func (m *WatchDeltaChannelsRequest) String() string { return proto.CompactTextString(m) }
func (*WatchDeltaChannelsRequest) ProtoMessage() {}
func (*WatchDeltaChannelsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{18}
}
func (m *WatchDeltaChannelsRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_WatchDeltaChannelsRequest.Unmarshal(m, b)
}
func (m *WatchDeltaChannelsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_WatchDeltaChannelsRequest.Marshal(b, m, deterministic)
}
func (m *WatchDeltaChannelsRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_WatchDeltaChannelsRequest.Merge(m, src)
}
func (m *WatchDeltaChannelsRequest) XXX_Size() int {
return xxx_messageInfo_WatchDeltaChannelsRequest.Size(m)
}
func (m *WatchDeltaChannelsRequest) XXX_DiscardUnknown() {
xxx_messageInfo_WatchDeltaChannelsRequest.DiscardUnknown(m)
}
var xxx_messageInfo_WatchDeltaChannelsRequest proto.InternalMessageInfo
func (m *WatchDeltaChannelsRequest) GetBase() *commonpb.MsgBase {
if m != nil {
return m.Base
}
return nil
}
func (m *WatchDeltaChannelsRequest) GetNodeID() int64 {
if m != nil {
return m.NodeID
}
return 0
}
func (m *WatchDeltaChannelsRequest) GetCollectionID() int64 {
if m != nil {
return m.CollectionID
}
return 0
}
func (m *WatchDeltaChannelsRequest) GetInfos() []*datapb.VchannelInfo {
if m != nil {
return m.Infos
}
return nil
}
func (m *WatchDeltaChannelsRequest) GetReplicaId() int64 {
if m != nil {
return m.ReplicaId
}
return 0
}
func (m *WatchDeltaChannelsRequest) GetLoadMeta() *LoadMetaInfo {
if m != nil {
return m.LoadMeta
}
return nil
}
type SegmentLoadInfo struct { type SegmentLoadInfo struct {
SegmentID int64 `protobuf:"varint,1,opt,name=segmentID,proto3" json:"segmentID,omitempty"` SegmentID int64 `protobuf:"varint,1,opt,name=segmentID,proto3" json:"segmentID,omitempty"`
PartitionID int64 `protobuf:"varint,2,opt,name=partitionID,proto3" json:"partitionID,omitempty"` PartitionID int64 `protobuf:"varint,2,opt,name=partitionID,proto3" json:"partitionID,omitempty"`
@ -1385,7 +1306,7 @@ func (m *SegmentLoadInfo) Reset() { *m = SegmentLoadInfo{} }
func (m *SegmentLoadInfo) String() string { return proto.CompactTextString(m) } func (m *SegmentLoadInfo) String() string { return proto.CompactTextString(m) }
func (*SegmentLoadInfo) ProtoMessage() {} func (*SegmentLoadInfo) ProtoMessage() {}
func (*SegmentLoadInfo) Descriptor() ([]byte, []int) { func (*SegmentLoadInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{19} return fileDescriptor_aab7cc9a69ed26e8, []int{18}
} }
func (m *SegmentLoadInfo) XXX_Unmarshal(b []byte) error { func (m *SegmentLoadInfo) XXX_Unmarshal(b []byte) error {
@ -1515,7 +1436,7 @@ func (m *FieldIndexInfo) Reset() { *m = FieldIndexInfo{} }
func (m *FieldIndexInfo) String() string { return proto.CompactTextString(m) } func (m *FieldIndexInfo) String() string { return proto.CompactTextString(m) }
func (*FieldIndexInfo) ProtoMessage() {} func (*FieldIndexInfo) ProtoMessage() {}
func (*FieldIndexInfo) Descriptor() ([]byte, []int) { func (*FieldIndexInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{20} return fileDescriptor_aab7cc9a69ed26e8, []int{19}
} }
func (m *FieldIndexInfo) XXX_Unmarshal(b []byte) error { func (m *FieldIndexInfo) XXX_Unmarshal(b []byte) error {
@ -1611,7 +1532,7 @@ func (m *LoadSegmentsRequest) Reset() { *m = LoadSegmentsRequest{} }
func (m *LoadSegmentsRequest) String() string { return proto.CompactTextString(m) } func (m *LoadSegmentsRequest) String() string { return proto.CompactTextString(m) }
func (*LoadSegmentsRequest) ProtoMessage() {} func (*LoadSegmentsRequest) ProtoMessage() {}
func (*LoadSegmentsRequest) Descriptor() ([]byte, []int) { func (*LoadSegmentsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{21} return fileDescriptor_aab7cc9a69ed26e8, []int{20}
} }
func (m *LoadSegmentsRequest) XXX_Unmarshal(b []byte) error { func (m *LoadSegmentsRequest) XXX_Unmarshal(b []byte) error {
@ -1713,7 +1634,7 @@ func (m *ReleaseSegmentsRequest) Reset() { *m = ReleaseSegmentsRequest{}
func (m *ReleaseSegmentsRequest) String() string { return proto.CompactTextString(m) } func (m *ReleaseSegmentsRequest) String() string { return proto.CompactTextString(m) }
func (*ReleaseSegmentsRequest) ProtoMessage() {} func (*ReleaseSegmentsRequest) ProtoMessage() {}
func (*ReleaseSegmentsRequest) Descriptor() ([]byte, []int) { func (*ReleaseSegmentsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{22} return fileDescriptor_aab7cc9a69ed26e8, []int{21}
} }
func (m *ReleaseSegmentsRequest) XXX_Unmarshal(b []byte) error { func (m *ReleaseSegmentsRequest) XXX_Unmarshal(b []byte) error {
@ -1798,7 +1719,7 @@ func (m *SearchRequest) Reset() { *m = SearchRequest{} }
func (m *SearchRequest) String() string { return proto.CompactTextString(m) } func (m *SearchRequest) String() string { return proto.CompactTextString(m) }
func (*SearchRequest) ProtoMessage() {} func (*SearchRequest) ProtoMessage() {}
func (*SearchRequest) Descriptor() ([]byte, []int) { func (*SearchRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{23} return fileDescriptor_aab7cc9a69ed26e8, []int{22}
} }
func (m *SearchRequest) XXX_Unmarshal(b []byte) error { func (m *SearchRequest) XXX_Unmarshal(b []byte) error {
@ -1869,7 +1790,7 @@ func (m *QueryRequest) Reset() { *m = QueryRequest{} }
func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (m *QueryRequest) String() string { return proto.CompactTextString(m) }
func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) ProtoMessage() {}
func (*QueryRequest) Descriptor() ([]byte, []int) { func (*QueryRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{24} return fileDescriptor_aab7cc9a69ed26e8, []int{23}
} }
func (m *QueryRequest) XXX_Unmarshal(b []byte) error { func (m *QueryRequest) XXX_Unmarshal(b []byte) error {
@ -1938,7 +1859,7 @@ func (m *SyncReplicaSegmentsRequest) Reset() { *m = SyncReplicaSegmentsR
func (m *SyncReplicaSegmentsRequest) String() string { return proto.CompactTextString(m) } func (m *SyncReplicaSegmentsRequest) String() string { return proto.CompactTextString(m) }
func (*SyncReplicaSegmentsRequest) ProtoMessage() {} func (*SyncReplicaSegmentsRequest) ProtoMessage() {}
func (*SyncReplicaSegmentsRequest) Descriptor() ([]byte, []int) { func (*SyncReplicaSegmentsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{25} return fileDescriptor_aab7cc9a69ed26e8, []int{24}
} }
func (m *SyncReplicaSegmentsRequest) XXX_Unmarshal(b []byte) error { func (m *SyncReplicaSegmentsRequest) XXX_Unmarshal(b []byte) error {
@ -1993,7 +1914,7 @@ func (m *ReplicaSegmentsInfo) Reset() { *m = ReplicaSegmentsInfo{} }
func (m *ReplicaSegmentsInfo) String() string { return proto.CompactTextString(m) } func (m *ReplicaSegmentsInfo) String() string { return proto.CompactTextString(m) }
func (*ReplicaSegmentsInfo) ProtoMessage() {} func (*ReplicaSegmentsInfo) ProtoMessage() {}
func (*ReplicaSegmentsInfo) Descriptor() ([]byte, []int) { func (*ReplicaSegmentsInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{26} return fileDescriptor_aab7cc9a69ed26e8, []int{25}
} }
func (m *ReplicaSegmentsInfo) XXX_Unmarshal(b []byte) error { func (m *ReplicaSegmentsInfo) XXX_Unmarshal(b []byte) error {
@ -2049,7 +1970,7 @@ func (m *HandoffSegmentsRequest) Reset() { *m = HandoffSegmentsRequest{}
func (m *HandoffSegmentsRequest) String() string { return proto.CompactTextString(m) } func (m *HandoffSegmentsRequest) String() string { return proto.CompactTextString(m) }
func (*HandoffSegmentsRequest) ProtoMessage() {} func (*HandoffSegmentsRequest) ProtoMessage() {}
func (*HandoffSegmentsRequest) Descriptor() ([]byte, []int) { func (*HandoffSegmentsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{27} return fileDescriptor_aab7cc9a69ed26e8, []int{26}
} }
func (m *HandoffSegmentsRequest) XXX_Unmarshal(b []byte) error { func (m *HandoffSegmentsRequest) XXX_Unmarshal(b []byte) error {
@ -2107,7 +2028,7 @@ func (m *LoadBalanceRequest) Reset() { *m = LoadBalanceRequest{} }
func (m *LoadBalanceRequest) String() string { return proto.CompactTextString(m) } func (m *LoadBalanceRequest) String() string { return proto.CompactTextString(m) }
func (*LoadBalanceRequest) ProtoMessage() {} func (*LoadBalanceRequest) ProtoMessage() {}
func (*LoadBalanceRequest) Descriptor() ([]byte, []int) { func (*LoadBalanceRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{28} return fileDescriptor_aab7cc9a69ed26e8, []int{27}
} }
func (m *LoadBalanceRequest) XXX_Unmarshal(b []byte) error { func (m *LoadBalanceRequest) XXX_Unmarshal(b []byte) error {
@ -2185,7 +2106,7 @@ func (m *DmChannelWatchInfo) Reset() { *m = DmChannelWatchInfo{} }
func (m *DmChannelWatchInfo) String() string { return proto.CompactTextString(m) } func (m *DmChannelWatchInfo) String() string { return proto.CompactTextString(m) }
func (*DmChannelWatchInfo) ProtoMessage() {} func (*DmChannelWatchInfo) ProtoMessage() {}
func (*DmChannelWatchInfo) Descriptor() ([]byte, []int) { func (*DmChannelWatchInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{29} return fileDescriptor_aab7cc9a69ed26e8, []int{28}
} }
func (m *DmChannelWatchInfo) XXX_Unmarshal(b []byte) error { func (m *DmChannelWatchInfo) XXX_Unmarshal(b []byte) error {
@ -2256,7 +2177,7 @@ func (m *QueryChannelInfo) Reset() { *m = QueryChannelInfo{} }
func (m *QueryChannelInfo) String() string { return proto.CompactTextString(m) } func (m *QueryChannelInfo) String() string { return proto.CompactTextString(m) }
func (*QueryChannelInfo) ProtoMessage() {} func (*QueryChannelInfo) ProtoMessage() {}
func (*QueryChannelInfo) Descriptor() ([]byte, []int) { func (*QueryChannelInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{30} return fileDescriptor_aab7cc9a69ed26e8, []int{29}
} }
func (m *QueryChannelInfo) XXX_Unmarshal(b []byte) error { func (m *QueryChannelInfo) XXX_Unmarshal(b []byte) error {
@ -2325,7 +2246,7 @@ func (m *PartitionStates) Reset() { *m = PartitionStates{} }
func (m *PartitionStates) String() string { return proto.CompactTextString(m) } func (m *PartitionStates) String() string { return proto.CompactTextString(m) }
func (*PartitionStates) ProtoMessage() {} func (*PartitionStates) ProtoMessage() {}
func (*PartitionStates) Descriptor() ([]byte, []int) { func (*PartitionStates) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{31} return fileDescriptor_aab7cc9a69ed26e8, []int{30}
} }
func (m *PartitionStates) XXX_Unmarshal(b []byte) error { func (m *PartitionStates) XXX_Unmarshal(b []byte) error {
@ -2393,7 +2314,7 @@ func (m *SegmentInfo) Reset() { *m = SegmentInfo{} }
func (m *SegmentInfo) String() string { return proto.CompactTextString(m) } func (m *SegmentInfo) String() string { return proto.CompactTextString(m) }
func (*SegmentInfo) ProtoMessage() {} func (*SegmentInfo) ProtoMessage() {}
func (*SegmentInfo) Descriptor() ([]byte, []int) { func (*SegmentInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{32} return fileDescriptor_aab7cc9a69ed26e8, []int{31}
} }
func (m *SegmentInfo) XXX_Unmarshal(b []byte) error { func (m *SegmentInfo) XXX_Unmarshal(b []byte) error {
@ -2538,7 +2459,7 @@ func (m *CollectionInfo) Reset() { *m = CollectionInfo{} }
func (m *CollectionInfo) String() string { return proto.CompactTextString(m) } func (m *CollectionInfo) String() string { return proto.CompactTextString(m) }
func (*CollectionInfo) ProtoMessage() {} func (*CollectionInfo) ProtoMessage() {}
func (*CollectionInfo) Descriptor() ([]byte, []int) { func (*CollectionInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{33} return fileDescriptor_aab7cc9a69ed26e8, []int{32}
} }
func (m *CollectionInfo) XXX_Unmarshal(b []byte) error { func (m *CollectionInfo) XXX_Unmarshal(b []byte) error {
@ -2634,7 +2555,7 @@ func (m *UnsubscribeChannels) Reset() { *m = UnsubscribeChannels{} }
func (m *UnsubscribeChannels) String() string { return proto.CompactTextString(m) } func (m *UnsubscribeChannels) String() string { return proto.CompactTextString(m) }
func (*UnsubscribeChannels) ProtoMessage() {} func (*UnsubscribeChannels) ProtoMessage() {}
func (*UnsubscribeChannels) Descriptor() ([]byte, []int) { func (*UnsubscribeChannels) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{34} return fileDescriptor_aab7cc9a69ed26e8, []int{33}
} }
func (m *UnsubscribeChannels) XXX_Unmarshal(b []byte) error { func (m *UnsubscribeChannels) XXX_Unmarshal(b []byte) error {
@ -2681,7 +2602,7 @@ func (m *UnsubscribeChannelInfo) Reset() { *m = UnsubscribeChannelInfo{}
func (m *UnsubscribeChannelInfo) String() string { return proto.CompactTextString(m) } func (m *UnsubscribeChannelInfo) String() string { return proto.CompactTextString(m) }
func (*UnsubscribeChannelInfo) ProtoMessage() {} func (*UnsubscribeChannelInfo) ProtoMessage() {}
func (*UnsubscribeChannelInfo) Descriptor() ([]byte, []int) { func (*UnsubscribeChannelInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{35} return fileDescriptor_aab7cc9a69ed26e8, []int{34}
} }
func (m *UnsubscribeChannelInfo) XXX_Unmarshal(b []byte) error { func (m *UnsubscribeChannelInfo) XXX_Unmarshal(b []byte) error {
@ -2731,7 +2652,7 @@ func (m *SegmentChangeInfo) Reset() { *m = SegmentChangeInfo{} }
func (m *SegmentChangeInfo) String() string { return proto.CompactTextString(m) } func (m *SegmentChangeInfo) String() string { return proto.CompactTextString(m) }
func (*SegmentChangeInfo) ProtoMessage() {} func (*SegmentChangeInfo) ProtoMessage() {}
func (*SegmentChangeInfo) Descriptor() ([]byte, []int) { func (*SegmentChangeInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{36} return fileDescriptor_aab7cc9a69ed26e8, []int{35}
} }
func (m *SegmentChangeInfo) XXX_Unmarshal(b []byte) error { func (m *SegmentChangeInfo) XXX_Unmarshal(b []byte) error {
@ -2792,7 +2713,7 @@ func (m *SealedSegmentsChangeInfo) Reset() { *m = SealedSegmentsChangeIn
func (m *SealedSegmentsChangeInfo) String() string { return proto.CompactTextString(m) } func (m *SealedSegmentsChangeInfo) String() string { return proto.CompactTextString(m) }
func (*SealedSegmentsChangeInfo) ProtoMessage() {} func (*SealedSegmentsChangeInfo) ProtoMessage() {}
func (*SealedSegmentsChangeInfo) Descriptor() ([]byte, []int) { func (*SealedSegmentsChangeInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{37} return fileDescriptor_aab7cc9a69ed26e8, []int{36}
} }
func (m *SealedSegmentsChangeInfo) XXX_Unmarshal(b []byte) error { func (m *SealedSegmentsChangeInfo) XXX_Unmarshal(b []byte) error {
@ -2851,7 +2772,6 @@ func init() {
proto.RegisterType((*LoadMetaInfo)(nil), "milvus.proto.query.LoadMetaInfo") proto.RegisterType((*LoadMetaInfo)(nil), "milvus.proto.query.LoadMetaInfo")
proto.RegisterType((*WatchDmChannelsRequest)(nil), "milvus.proto.query.WatchDmChannelsRequest") proto.RegisterType((*WatchDmChannelsRequest)(nil), "milvus.proto.query.WatchDmChannelsRequest")
proto.RegisterMapType((map[int64]*datapb.SegmentInfo)(nil), "milvus.proto.query.WatchDmChannelsRequest.SegmentInfosEntry") proto.RegisterMapType((map[int64]*datapb.SegmentInfo)(nil), "milvus.proto.query.WatchDmChannelsRequest.SegmentInfosEntry")
proto.RegisterType((*WatchDeltaChannelsRequest)(nil), "milvus.proto.query.WatchDeltaChannelsRequest")
proto.RegisterType((*SegmentLoadInfo)(nil), "milvus.proto.query.SegmentLoadInfo") proto.RegisterType((*SegmentLoadInfo)(nil), "milvus.proto.query.SegmentLoadInfo")
proto.RegisterType((*FieldIndexInfo)(nil), "milvus.proto.query.FieldIndexInfo") proto.RegisterType((*FieldIndexInfo)(nil), "milvus.proto.query.FieldIndexInfo")
proto.RegisterType((*LoadSegmentsRequest)(nil), "milvus.proto.query.LoadSegmentsRequest") proto.RegisterType((*LoadSegmentsRequest)(nil), "milvus.proto.query.LoadSegmentsRequest")
@ -2876,195 +2796,192 @@ func init() {
func init() { proto.RegisterFile("query_coord.proto", fileDescriptor_aab7cc9a69ed26e8) } func init() { proto.RegisterFile("query_coord.proto", fileDescriptor_aab7cc9a69ed26e8) }
var fileDescriptor_aab7cc9a69ed26e8 = []byte{ var fileDescriptor_aab7cc9a69ed26e8 = []byte{
// 3003 bytes of a gzipped FileDescriptorProto // 2951 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x3a, 0x49, 0x6f, 0x1c, 0xc7, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x3a, 0x49, 0x6f, 0x1c, 0xc7,
0xd5, 0xec, 0x59, 0xc8, 0x99, 0x37, 0x5b, 0xab, 0x28, 0x51, 0xe3, 0xf9, 0x24, 0x9b, 0x6e, 0x59, 0xd5, 0xec, 0xd9, 0xe7, 0xcd, 0xd6, 0x2a, 0x4a, 0xd4, 0x78, 0x3e, 0xc9, 0xa6, 0x5b, 0x96, 0xcd,
0x36, 0x3f, 0xda, 0xa6, 0x64, 0xca, 0x31, 0xec, 0xc4, 0x06, 0x22, 0x91, 0x16, 0xcd, 0x48, 0xa2, 0x8f, 0xb6, 0x29, 0x99, 0xf2, 0x67, 0xd8, 0x5f, 0x6c, 0x20, 0x12, 0x69, 0xd1, 0x8c, 0x24, 0x9a,
0x99, 0x1e, 0xc9, 0x09, 0x04, 0x03, 0x93, 0x9e, 0xe9, 0x9a, 0x61, 0x43, 0xbd, 0x8c, 0xba, 0x7a, 0xe9, 0x91, 0x9c, 0x40, 0x30, 0x30, 0xe9, 0x99, 0xae, 0x19, 0x36, 0xd4, 0xcb, 0xa8, 0xab, 0x87,
0x28, 0xd1, 0xb9, 0x06, 0x01, 0xb2, 0x1d, 0x82, 0x5c, 0x03, 0x23, 0x87, 0xe4, 0x10, 0x20, 0x46, 0x32, 0x9d, 0x6b, 0x10, 0x20, 0xdb, 0x21, 0xc8, 0x35, 0x08, 0x72, 0x48, 0x0e, 0x01, 0x62, 0xe4,
0x2e, 0xb9, 0x04, 0x08, 0x82, 0xdc, 0x72, 0x0d, 0x90, 0x1f, 0xe0, 0x63, 0x0e, 0xc9, 0x35, 0x87, 0x92, 0x4b, 0x80, 0x20, 0xc8, 0x2d, 0xd7, 0x00, 0xf9, 0x01, 0x39, 0xe6, 0x90, 0x5c, 0x73, 0x08,
0xdc, 0x82, 0xda, 0x7a, 0x9b, 0x6e, 0xcd, 0x90, 0xb4, 0xbc, 0x00, 0xb9, 0x75, 0xbf, 0x7a, 0x55, 0x72, 0x09, 0x6a, 0xeb, 0x6d, 0xba, 0xc5, 0x11, 0x69, 0x79, 0x01, 0x72, 0xeb, 0x7a, 0xf5, 0xaa,
0xef, 0xd5, 0xdb, 0x5f, 0x55, 0xc1, 0x99, 0x87, 0x13, 0xec, 0x1f, 0xf5, 0x06, 0x9e, 0xe7, 0x9b, 0xde, 0xab, 0xb7, 0xbf, 0xaa, 0x86, 0x33, 0x0f, 0x67, 0xd8, 0x3f, 0x1a, 0x8c, 0x3c, 0xcf, 0x37,
0x1b, 0x63, 0xdf, 0x0b, 0x3c, 0x84, 0x1c, 0xcb, 0x3e, 0x9c, 0x10, 0xfe, 0xb7, 0xc1, 0xc6, 0x3b, 0x37, 0xa6, 0xbe, 0x17, 0x78, 0x08, 0x39, 0x96, 0x7d, 0x38, 0x23, 0x7c, 0xb4, 0xc1, 0xe6, 0x7b,
0xf5, 0x81, 0xe7, 0x38, 0x9e, 0xcb, 0x61, 0x9d, 0x7a, 0x1c, 0xa3, 0xd3, 0xb4, 0xdc, 0x00, 0xfb, 0xcd, 0x91, 0xe7, 0x38, 0x9e, 0xcb, 0x61, 0xbd, 0x66, 0x1c, 0xa3, 0xd7, 0xb6, 0xdc, 0x00, 0xfb,
0xae, 0x61, 0xcb, 0x51, 0x32, 0x38, 0xc0, 0x8e, 0x21, 0xfe, 0x54, 0xd3, 0x08, 0x8c, 0xf8, 0xfa, 0xae, 0x61, 0xcb, 0x59, 0x32, 0x3a, 0xc0, 0x8e, 0x21, 0x46, 0xaa, 0x69, 0x04, 0x46, 0x7c, 0x7f,
0xda, 0x0f, 0x14, 0x58, 0xe9, 0x1e, 0x78, 0x8f, 0xb6, 0x3c, 0xdb, 0xc6, 0x83, 0xc0, 0xf2, 0x5c, 0xed, 0x3b, 0x0a, 0xac, 0xf4, 0x0f, 0xbc, 0x47, 0x5b, 0x9e, 0x6d, 0xe3, 0x51, 0x60, 0x79, 0x2e,
0xa2, 0xe3, 0x87, 0x13, 0x4c, 0x02, 0x74, 0x15, 0x4a, 0x7d, 0x83, 0xe0, 0xb6, 0xb2, 0xaa, 0xac, 0xd1, 0xf1, 0xc3, 0x19, 0x26, 0x01, 0xba, 0x0a, 0xa5, 0xa1, 0x41, 0x70, 0x57, 0x59, 0x55, 0xd6,
0xd5, 0x36, 0x2f, 0x6c, 0x24, 0x38, 0x11, 0x2c, 0xdc, 0x21, 0xa3, 0x1b, 0x06, 0xc1, 0x3a, 0xc3, 0x1a, 0x9b, 0x17, 0x36, 0x12, 0x9c, 0x08, 0x16, 0xee, 0x90, 0xc9, 0x0d, 0x83, 0x60, 0x9d, 0x61,
0x44, 0x08, 0x4a, 0x66, 0x7f, 0x77, 0xbb, 0x5d, 0x58, 0x55, 0xd6, 0x8a, 0x3a, 0xfb, 0x46, 0x2f, 0x22, 0x04, 0x25, 0x73, 0xb8, 0xbb, 0xdd, 0x2d, 0xac, 0x2a, 0x6b, 0x45, 0x9d, 0x7d, 0xa3, 0x17,
0x40, 0x63, 0x10, 0xae, 0xbd, 0xbb, 0x4d, 0xda, 0xc5, 0xd5, 0xe2, 0x5a, 0x51, 0x4f, 0x02, 0xb5, 0xa0, 0x35, 0x0a, 0xf7, 0xde, 0xdd, 0x26, 0xdd, 0xe2, 0x6a, 0x71, 0xad, 0xa8, 0x27, 0x81, 0xda,
0x4f, 0x15, 0x38, 0x3f, 0xc5, 0x06, 0x19, 0x7b, 0x2e, 0xc1, 0xe8, 0x1a, 0x2c, 0x92, 0xc0, 0x08, 0x5f, 0x15, 0x38, 0x3f, 0xc7, 0x06, 0x99, 0x7a, 0x2e, 0xc1, 0xe8, 0x1a, 0x54, 0x48, 0x60, 0x04,
0x26, 0x44, 0x70, 0xf2, 0x7f, 0x99, 0x9c, 0x74, 0x19, 0x8a, 0x2e, 0x50, 0xa7, 0xc9, 0x16, 0x32, 0x33, 0x22, 0x38, 0xf9, 0x9f, 0x4c, 0x4e, 0xfa, 0x0c, 0x45, 0x17, 0xa8, 0xf3, 0x64, 0x0b, 0x19,
0xc8, 0xa2, 0xd7, 0xe0, 0xac, 0xe5, 0xde, 0xc1, 0x8e, 0xe7, 0x1f, 0xf5, 0xc6, 0xd8, 0x1f, 0x60, 0x64, 0xd1, 0x6b, 0x70, 0xd6, 0x72, 0xef, 0x60, 0xc7, 0xf3, 0x8f, 0x06, 0x53, 0xec, 0x8f, 0xb0,
0x37, 0x30, 0x46, 0x58, 0xf2, 0xb8, 0x2c, 0xc7, 0xf6, 0xa3, 0x21, 0xf4, 0x06, 0x9c, 0xe7, 0x5a, 0x1b, 0x18, 0x13, 0x2c, 0x79, 0x5c, 0x96, 0x73, 0xfb, 0xd1, 0x14, 0x7a, 0x03, 0xce, 0x73, 0x2d,
0x22, 0xd8, 0x3f, 0xb4, 0x06, 0xb8, 0x67, 0x1c, 0x1a, 0x96, 0x6d, 0xf4, 0x6d, 0xdc, 0x2e, 0xad, 0x11, 0xec, 0x1f, 0x5a, 0x23, 0x3c, 0x30, 0x0e, 0x0d, 0xcb, 0x36, 0x86, 0x36, 0xee, 0x96, 0x56,
0x16, 0xd7, 0x2a, 0xfa, 0x39, 0x36, 0xdc, 0xe5, 0xa3, 0xd7, 0xe5, 0xa0, 0xf6, 0x1b, 0x05, 0xce, 0x8b, 0x6b, 0x35, 0xfd, 0x1c, 0x9b, 0xee, 0xf3, 0xd9, 0xeb, 0x72, 0x52, 0xfb, 0xa5, 0x02, 0xe7,
0xd1, 0x1d, 0xee, 0x1b, 0x7e, 0x60, 0x3d, 0x05, 0x39, 0x6b, 0x50, 0x8f, 0xef, 0xad, 0x5d, 0x64, 0xe8, 0x09, 0xf7, 0x0d, 0x3f, 0xb0, 0x9e, 0x82, 0x9c, 0x35, 0x68, 0xc6, 0xcf, 0xd6, 0x2d, 0xb2,
0x63, 0x09, 0x18, 0xc5, 0x19, 0x4b, 0xf2, 0x54, 0x26, 0x25, 0xb6, 0xcd, 0x04, 0x4c, 0xfb, 0xb5, 0xb9, 0x04, 0x8c, 0xe2, 0x4c, 0x25, 0x79, 0x2a, 0x93, 0x12, 0x3b, 0x66, 0x02, 0xa6, 0xfd, 0x42,
0x30, 0x88, 0x38, 0x9f, 0xa7, 0x51, 0x44, 0x9a, 0x66, 0x61, 0x9a, 0xe6, 0x09, 0xd4, 0xa0, 0xfd, 0x18, 0x44, 0x9c, 0xcf, 0xd3, 0x28, 0x22, 0x4d, 0xb3, 0x30, 0x4f, 0xf3, 0x04, 0x6a, 0xd0, 0xfe,
0x43, 0x81, 0x73, 0xb7, 0x3d, 0xc3, 0x8c, 0x0c, 0xe6, 0xf3, 0x17, 0xe7, 0x3b, 0xb0, 0xc8, 0xbd, 0xa6, 0xc0, 0xb9, 0xdb, 0x9e, 0x61, 0x46, 0x06, 0xf3, 0xd9, 0x8b, 0xf3, 0x1d, 0xa8, 0x70, 0xef,
0xab, 0x5d, 0x62, 0xb4, 0x2e, 0x27, 0x69, 0x09, 0xcf, 0x8b, 0x38, 0xec, 0x32, 0x80, 0x2e, 0x26, 0xea, 0x96, 0x18, 0xad, 0xcb, 0x49, 0x5a, 0xc2, 0xf3, 0x22, 0x0e, 0xfb, 0x0c, 0xa0, 0x8b, 0x45,
0xa1, 0xcb, 0xd0, 0xf4, 0xf1, 0xd8, 0xb6, 0x06, 0x46, 0xcf, 0x9d, 0x38, 0x7d, 0xec, 0xb7, 0xcb, 0xe8, 0x32, 0xb4, 0x7d, 0x3c, 0xb5, 0xad, 0x91, 0x31, 0x70, 0x67, 0xce, 0x10, 0xfb, 0xdd, 0xf2,
0xab, 0xca, 0x5a, 0x59, 0x6f, 0x08, 0xe8, 0x1e, 0x03, 0x6a, 0xbf, 0x54, 0xa0, 0xad, 0x63, 0x1b, 0xaa, 0xb2, 0x56, 0xd6, 0x5b, 0x02, 0xba, 0xc7, 0x80, 0xda, 0x4f, 0x15, 0xe8, 0xea, 0xd8, 0xc6,
0x1b, 0x04, 0x7f, 0x91, 0x9b, 0x5d, 0x81, 0x45, 0xd7, 0x33, 0xf1, 0xee, 0x36, 0xdb, 0x6c, 0x51, 0x06, 0xc1, 0x9f, 0xe7, 0x61, 0x57, 0xa0, 0xe2, 0x7a, 0x26, 0xde, 0xdd, 0x66, 0x87, 0x2d, 0xea,
0x17, 0x7f, 0xda, 0x7f, 0x14, 0x38, 0xbb, 0x83, 0x03, 0xaa, 0x75, 0x8b, 0x04, 0xd6, 0x20, 0x34, 0x62, 0xa4, 0xfd, 0x4b, 0x81, 0xb3, 0x3b, 0x38, 0xa0, 0x5a, 0xb7, 0x48, 0x60, 0x8d, 0x42, 0xb3,
0xeb, 0x77, 0xa0, 0xe8, 0xe3, 0x87, 0x82, 0xb3, 0x97, 0x93, 0x9c, 0x85, 0x41, 0x2a, 0x6b, 0xa6, 0x7e, 0x07, 0x8a, 0x3e, 0x7e, 0x28, 0x38, 0x7b, 0x39, 0xc9, 0x59, 0x18, 0xa4, 0xb2, 0x56, 0xea,
0x4e, 0xe7, 0xa1, 0xe7, 0xa1, 0x6e, 0x3a, 0x76, 0x6f, 0x70, 0x60, 0xb8, 0x2e, 0xb6, 0xb9, 0xdd, 0x74, 0x1d, 0x7a, 0x1e, 0x9a, 0xa6, 0x63, 0x0f, 0x46, 0x07, 0x86, 0xeb, 0x62, 0x9b, 0xdb, 0x4d,
0x54, 0xf5, 0x9a, 0xe9, 0xd8, 0x5b, 0x02, 0x84, 0x9e, 0x05, 0x20, 0x78, 0xe4, 0x60, 0x37, 0x88, 0x5d, 0x6f, 0x98, 0x8e, 0xbd, 0x25, 0x40, 0xe8, 0x59, 0x00, 0x82, 0x27, 0x0e, 0x76, 0x83, 0x28,
0xe2, 0x4a, 0x0c, 0x82, 0xd6, 0xe1, 0xcc, 0xd0, 0xf7, 0x9c, 0x1e, 0x39, 0x30, 0x7c, 0xb3, 0x67, 0xae, 0xc4, 0x20, 0x68, 0x1d, 0xce, 0x8c, 0x7d, 0xcf, 0x19, 0x90, 0x03, 0xc3, 0x37, 0x07, 0x36,
0x63, 0xc3, 0xc4, 0x3e, 0xe3, 0xbe, 0xa2, 0xb7, 0xe8, 0x40, 0x97, 0xc2, 0x6f, 0x33, 0x30, 0xba, 0x36, 0x4c, 0xec, 0x33, 0xee, 0x6b, 0x7a, 0x87, 0x4e, 0xf4, 0x29, 0xfc, 0x36, 0x03, 0xa3, 0x6b,
0x06, 0x65, 0x32, 0xf0, 0xc6, 0x98, 0xe9, 0xa0, 0xb9, 0x79, 0x71, 0x63, 0x3a, 0xee, 0x6e, 0x6c, 0x50, 0x26, 0x23, 0x6f, 0x8a, 0x99, 0x0e, 0xda, 0x9b, 0x17, 0x37, 0xe6, 0xe3, 0xee, 0xc6, 0xb6,
0x1b, 0x81, 0xd1, 0xa5, 0x48, 0x3a, 0xc7, 0xd5, 0x7e, 0x52, 0xe0, 0x46, 0xf8, 0x25, 0xf7, 0xe9, 0x11, 0x18, 0x7d, 0x8a, 0xa4, 0x73, 0x5c, 0xed, 0x07, 0x05, 0x6e, 0x84, 0x5f, 0x70, 0x9f, 0x8e,
0x98, 0xa1, 0x96, 0x3f, 0x1b, 0x43, 0x5d, 0xcc, 0x32, 0xd4, 0xbf, 0x44, 0x86, 0xfa, 0x65, 0x17, 0x19, 0x6a, 0xf9, 0xd3, 0x31, 0xd4, 0x4a, 0x96, 0xa1, 0xfe, 0x31, 0x32, 0xd4, 0x2f, 0xba, 0x40,
0x48, 0x64, 0xcc, 0xe5, 0x84, 0x31, 0xff, 0x56, 0x81, 0x67, 0x76, 0x70, 0x10, 0xb2, 0x4f, 0x6d, 0x22, 0x63, 0x2e, 0x27, 0x8c, 0xf9, 0x57, 0x0a, 0x3c, 0xb3, 0x83, 0x83, 0x90, 0x7d, 0x6a, 0x9b,
0x13, 0x7f, 0x49, 0x03, 0xf5, 0x27, 0x0a, 0x74, 0xb2, 0x78, 0x3d, 0x4d, 0xb0, 0xbe, 0x0f, 0x2b, 0xf8, 0x0b, 0x1a, 0xa8, 0x3f, 0x51, 0xa0, 0x97, 0xc5, 0xeb, 0x69, 0x82, 0xf5, 0x7d, 0x58, 0x09,
0x21, 0x8d, 0x9e, 0x89, 0xc9, 0xc0, 0xb7, 0xc6, 0x4c, 0x8d, 0xcc, 0xfd, 0x6a, 0x9b, 0x97, 0xb2, 0x69, 0x0c, 0x4c, 0x4c, 0x46, 0xbe, 0x35, 0x65, 0x6a, 0x64, 0xee, 0xd7, 0xd8, 0xbc, 0x94, 0xe5,
0xdc, 0x22, 0xcd, 0xc1, 0xb9, 0x70, 0x89, 0xed, 0xd8, 0x0a, 0xda, 0xcf, 0x14, 0x38, 0x47, 0xdd, 0x16, 0x69, 0x0e, 0xce, 0x85, 0x5b, 0x6c, 0xc7, 0x76, 0xd0, 0x7e, 0xa4, 0xc0, 0x39, 0xea, 0xee,
0x5d, 0xf8, 0xa7, 0x3b, 0xf4, 0x4e, 0x2e, 0xd7, 0xa4, 0xe7, 0x17, 0xa6, 0x3c, 0x7f, 0x0e, 0x19, 0xc2, 0x3f, 0xdd, 0xb1, 0x77, 0x72, 0xb9, 0x26, 0x3d, 0xbf, 0x30, 0xe7, 0xf9, 0x0b, 0xc8, 0x98,
0xb3, 0xca, 0x27, 0xcd, 0xcf, 0x69, 0x64, 0xf7, 0x35, 0x28, 0x5b, 0xee, 0xd0, 0x93, 0xa2, 0x7a, 0x55, 0x3e, 0x69, 0x7e, 0x4e, 0x23, 0xbb, 0xff, 0x83, 0xb2, 0xe5, 0x8e, 0x3d, 0x29, 0xaa, 0xe7,
0x2e, 0x4b, 0x54, 0x71, 0x62, 0x1c, 0x5b, 0x73, 0x39, 0x17, 0x51, 0x28, 0x3a, 0x85, 0xb9, 0xa5, 0xb2, 0x44, 0x15, 0x27, 0xc6, 0xb1, 0x35, 0x97, 0x73, 0x11, 0x85, 0xa2, 0x53, 0x98, 0x5b, 0xfa,
0xb7, 0x5d, 0xc8, 0xd8, 0xf6, 0x4f, 0x15, 0x38, 0x3f, 0x45, 0xf0, 0x34, 0xfb, 0x7e, 0x1b, 0x16, 0xd8, 0x85, 0x8c, 0x63, 0xff, 0x50, 0x81, 0xf3, 0x73, 0x04, 0x4f, 0x73, 0xee, 0xb7, 0xa1, 0xc2,
0x59, 0x80, 0x95, 0x1b, 0x7f, 0x21, 0x73, 0xe3, 0x31, 0x72, 0xb7, 0x2d, 0x12, 0xe8, 0x62, 0x8e, 0x02, 0xac, 0x3c, 0xf8, 0x0b, 0x99, 0x07, 0x8f, 0x91, 0xbb, 0x6d, 0x91, 0x40, 0x17, 0x6b, 0x34,
0xe6, 0x81, 0x9a, 0x1e, 0xa3, 0xa1, 0x5f, 0x84, 0xfd, 0x9e, 0x6b, 0x38, 0x5c, 0x00, 0x55, 0xbd, 0x0f, 0xd4, 0xf4, 0x1c, 0x0d, 0xfd, 0x22, 0xec, 0x0f, 0x5c, 0xc3, 0xe1, 0x02, 0xa8, 0xeb, 0x0d,
0x26, 0x60, 0x7b, 0x86, 0x83, 0xd1, 0x33, 0x50, 0xa1, 0x2e, 0xdb, 0xb3, 0x4c, 0xa9, 0xfe, 0x25, 0x01, 0xdb, 0x33, 0x1c, 0x8c, 0x9e, 0x81, 0x1a, 0x75, 0xd9, 0x81, 0x65, 0x4a, 0xf5, 0x57, 0x99,
0xe6, 0xc2, 0x26, 0x41, 0x17, 0x01, 0xd8, 0x90, 0x61, 0x9a, 0x3e, 0xcf, 0x0a, 0x55, 0xbd, 0x4a, 0x0b, 0x9b, 0x04, 0x5d, 0x04, 0x60, 0x53, 0x86, 0x69, 0xfa, 0x3c, 0x2b, 0xd4, 0xf5, 0x3a, 0x85,
0x21, 0xd7, 0x29, 0x40, 0xfb, 0xb9, 0x02, 0x75, 0x1a, 0xb3, 0xef, 0xe0, 0xc0, 0xa0, 0x7a, 0x40, 0x5c, 0xa7, 0x00, 0xed, 0xc7, 0x0a, 0x34, 0x69, 0xcc, 0xbe, 0x83, 0x03, 0x83, 0xea, 0x01, 0xbd,
0x6f, 0x41, 0xd5, 0xf6, 0x0c, 0xb3, 0x17, 0x1c, 0x8d, 0x39, 0xa9, 0x66, 0x5a, 0xd6, 0x7c, 0x0b, 0x05, 0x75, 0xdb, 0x33, 0xcc, 0x41, 0x70, 0x34, 0xe5, 0xa4, 0xda, 0x69, 0x59, 0xf3, 0x23, 0xd0,
0x74, 0xd2, 0xdd, 0xa3, 0x31, 0xd6, 0x2b, 0xb6, 0xf8, 0x9a, 0x47, 0xde, 0x53, 0xae, 0x5c, 0xcc, 0x45, 0x77, 0x8f, 0xa6, 0x58, 0xaf, 0xd9, 0xe2, 0x6b, 0x11, 0x79, 0xcf, 0xb9, 0x72, 0x31, 0xc3,
0x70, 0xe5, 0x1f, 0x96, 0x61, 0xe5, 0x3b, 0x46, 0x30, 0x38, 0xd8, 0x76, 0x64, 0x72, 0x3b, 0xb9, 0x95, 0xbf, 0x5b, 0x86, 0x95, 0x6f, 0x18, 0xc1, 0xe8, 0x60, 0xdb, 0x91, 0xc9, 0xed, 0xe4, 0x46,
0x11, 0x44, 0xb1, 0xad, 0x10, 0x8f, 0x6d, 0x9f, 0x59, 0xec, 0x0c, 0xed, 0xbc, 0x9c, 0x65, 0xe7, 0x10, 0xc5, 0xb6, 0x42, 0x3c, 0xb6, 0x7d, 0x6a, 0xb1, 0x33, 0xb4, 0xf3, 0x72, 0x96, 0x9d, 0xd3,
0xb4, 0xc1, 0xd8, 0xf8, 0x40, 0xa8, 0x2a, 0x66, 0xe7, 0xb1, 0x1c, 0xb4, 0x78, 0x92, 0x1c, 0xb4, 0x06, 0x63, 0xe3, 0x03, 0xa1, 0xaa, 0x98, 0x9d, 0xc7, 0x72, 0x50, 0xe5, 0x24, 0x39, 0x68, 0x0b,
0x05, 0x0d, 0xfc, 0x78, 0x60, 0x4f, 0xa8, 0xce, 0x19, 0xf5, 0x25, 0x46, 0xfd, 0xd9, 0x0c, 0xea, 0x5a, 0xf8, 0xa3, 0x91, 0x3d, 0xa3, 0x3a, 0x67, 0xd4, 0xab, 0x8c, 0xfa, 0xb3, 0x19, 0xd4, 0xe3,
0x71, 0x27, 0xab, 0x8b, 0x49, 0xbb, 0x82, 0x07, 0xae, 0x6a, 0x07, 0x07, 0x46, 0xbb, 0xc2, 0xd8, 0x4e, 0xd6, 0x14, 0x8b, 0x76, 0x05, 0x0f, 0x5c, 0xd5, 0x0e, 0x0e, 0x8c, 0x6e, 0x8d, 0xb1, 0xb1,
0x58, 0xcd, 0x53, 0xb5, 0xb4, 0x0f, 0xae, 0x6e, 0xfa, 0x87, 0x2e, 0x40, 0x55, 0x64, 0xbc, 0xdd, 0x9a, 0xa7, 0x6a, 0x69, 0x1f, 0x5c, 0xdd, 0x74, 0x84, 0x2e, 0x40, 0x5d, 0x64, 0xbc, 0xdd, 0xed,
0xed, 0x76, 0x95, 0x89, 0x2f, 0x02, 0x20, 0x03, 0x1a, 0x22, 0x02, 0x09, 0x0e, 0x81, 0x71, 0xf8, 0x6e, 0x9d, 0x89, 0x2f, 0x02, 0x20, 0x03, 0x5a, 0x22, 0x02, 0x09, 0x0e, 0x81, 0x71, 0xf8, 0x76,
0x76, 0x16, 0x81, 0x6c, 0x65, 0xc7, 0x39, 0x27, 0xef, 0xba, 0x81, 0x7f, 0xa4, 0xd7, 0x49, 0x0c, 0x16, 0x81, 0x6c, 0x65, 0xc7, 0x39, 0x27, 0xef, 0xba, 0x81, 0x7f, 0xa4, 0x37, 0x49, 0x0c, 0xd4,
0xd4, 0xe9, 0xc1, 0x99, 0x29, 0x14, 0xa4, 0x42, 0xf1, 0x01, 0x3e, 0x62, 0x06, 0x52, 0xd4, 0xe9, 0x1b, 0xc0, 0x99, 0x39, 0x14, 0xa4, 0x42, 0xf1, 0x01, 0x3e, 0x62, 0x06, 0x52, 0xd4, 0xe9, 0x27,
0x27, 0x7a, 0x1d, 0xca, 0x87, 0x86, 0x3d, 0xc1, 0xcc, 0x00, 0x66, 0xcb, 0x88, 0x23, 0x7f, 0xbd, 0x7a, 0x1d, 0xca, 0x87, 0x86, 0x3d, 0xc3, 0xcc, 0x00, 0x8e, 0x97, 0x11, 0x47, 0xfe, 0xff, 0xc2,
0xf0, 0xa6, 0xa2, 0x7d, 0x5c, 0x80, 0x67, 0x38, 0x6f, 0xd8, 0x0e, 0x8c, 0x2f, 0xd6, 0x16, 0x43, 0x9b, 0x8a, 0xf6, 0x87, 0x12, 0x74, 0xc4, 0x14, 0x95, 0x01, 0xf3, 0x8f, 0x0b, 0x50, 0x0f, 0x23,
0x3b, 0x2b, 0x1d, 0xcb, 0xce, 0x2e, 0x02, 0xc8, 0x62, 0xc5, 0x32, 0x45, 0x7a, 0x0f, 0xb5, 0x64, 0xab, 0xa0, 0x12, 0x01, 0xd0, 0x2a, 0x34, 0x62, 0xd6, 0x21, 0x4c, 0x2e, 0x0e, 0x5a, 0xc8, 0xee,
0x26, 0x4d, 0xa0, 0x7a, 0x5c, 0x13, 0xd0, 0xfe, 0x5c, 0x82, 0x96, 0x90, 0x1d, 0xc5, 0x60, 0x01, 0x64, 0x9e, 0x2c, 0xc5, 0xf2, 0xe4, 0x45, 0x80, 0xb1, 0x3d, 0x23, 0x07, 0x83, 0xc0, 0x72, 0xb0,
0xe4, 0x02, 0x54, 0xc3, 0xd4, 0x23, 0xd4, 0x10, 0x01, 0xd0, 0x2a, 0xd4, 0x62, 0xee, 0x23, 0xe4, 0xc8, 0xd3, 0x75, 0x06, 0xb9, 0x6b, 0x39, 0x18, 0x5d, 0x87, 0xe6, 0xd0, 0x72, 0x6d, 0x6f, 0x32,
0x10, 0x07, 0xcd, 0x25, 0x0c, 0x59, 0x48, 0x94, 0x62, 0x85, 0xc4, 0x45, 0x80, 0xa1, 0x3d, 0x21, 0x98, 0x1a, 0xc1, 0x01, 0xe9, 0x56, 0x72, 0xed, 0xe1, 0xa6, 0x85, 0x6d, 0xf3, 0x06, 0xc3, 0xd5,
0x07, 0xbd, 0xc0, 0x72, 0xb0, 0xdc, 0x29, 0x83, 0xdc, 0xb5, 0x1c, 0x8c, 0xae, 0x43, 0xbd, 0x6f, 0x1b, 0x7c, 0xcd, 0x3e, 0x5d, 0x82, 0x9e, 0x85, 0x86, 0x3b, 0x73, 0x06, 0xde, 0x78, 0xe0, 0x7b,
0xb9, 0xb6, 0x37, 0xea, 0x8d, 0x8d, 0xe0, 0x80, 0xb4, 0x17, 0x73, 0x1d, 0xe6, 0xa6, 0x85, 0x6d, 0x8f, 0xa8, 0x45, 0x31, 0x12, 0xee, 0xcc, 0x79, 0x7f, 0xac, 0x7b, 0x8f, 0x68, 0x64, 0xab, 0xd3,
0xf3, 0x06, 0xc3, 0xd5, 0x6b, 0x7c, 0xce, 0x3e, 0x9d, 0x82, 0x9e, 0x85, 0x9a, 0x3b, 0x71, 0x7a, 0x18, 0x47, 0x6c, 0x6f, 0x42, 0xba, 0xb5, 0x85, 0xf6, 0x8f, 0x16, 0xd0, 0xd5, 0x26, 0xb6, 0x03,
0xde, 0xb0, 0xe7, 0x7b, 0x8f, 0xa8, 0xcb, 0x31, 0x12, 0xee, 0xc4, 0x79, 0x7f, 0xa8, 0x7b, 0x8f, 0x83, 0xad, 0xae, 0x2f, 0xb6, 0x3a, 0x5c, 0x80, 0x5e, 0x84, 0xf6, 0xc8, 0x73, 0xa6, 0x06, 0x93,
0x68, 0xe8, 0xaf, 0xd2, 0x24, 0x40, 0x6c, 0x6f, 0x44, 0xda, 0x95, 0xb9, 0xd6, 0x8f, 0x26, 0xd0, 0xd0, 0x4d, 0xdf, 0x73, 0x98, 0x39, 0x15, 0xf5, 0x14, 0x14, 0x6d, 0x41, 0xc3, 0x72, 0x4d, 0xfc,
0xd9, 0x26, 0x35, 0x33, 0x36, 0xbb, 0x3a, 0xdf, 0xec, 0x70, 0x02, 0x7a, 0x11, 0x9a, 0x03, 0xcf, 0x91, 0xb0, 0xb9, 0x06, 0xa3, 0xa3, 0x65, 0xd9, 0x1c, 0x23, 0xb4, 0x4b, 0x71, 0x99, 0xd6, 0xc1,
0x19, 0x1b, 0x4c, 0x42, 0x37, 0x7d, 0xcf, 0x61, 0xfe, 0x56, 0xd4, 0x53, 0x50, 0xb4, 0x05, 0x35, 0x92, 0x9f, 0x84, 0x06, 0x5c, 0x69, 0xba, 0xc4, 0xfa, 0x18, 0x77, 0x9b, 0x5c, 0x8b, 0x02, 0xd6,
0xcb, 0x35, 0xf1, 0x63, 0xe1, 0x94, 0x35, 0x46, 0x47, 0xcb, 0x52, 0x39, 0x23, 0xb4, 0x4b, 0x71, 0xb7, 0x3e, 0xc6, 0xb4, 0x06, 0xb4, 0x5c, 0x82, 0xfd, 0x40, 0x56, 0xe4, 0xdd, 0x16, 0x8b, 0xca,
0x99, 0xd2, 0xc1, 0x92, 0x9f, 0x84, 0x66, 0x24, 0xe9, 0xdb, 0xc4, 0xfa, 0x08, 0xb7, 0xeb, 0x5c, 0x2d, 0x0e, 0x15, 0x96, 0xac, 0xfd, 0xa6, 0x00, 0xed, 0x24, 0x21, 0xd4, 0x85, 0xea, 0x98, 0x41,
0x8b, 0x02, 0xd6, 0xb5, 0x3e, 0xc2, 0xb4, 0x48, 0xb6, 0x5c, 0x82, 0xfd, 0x40, 0xb6, 0x2c, 0xed, 0xa4, 0xf5, 0xc8, 0x21, 0x25, 0x8b, 0x5d, 0xda, 0x1c, 0x0f, 0x18, 0x2f, 0xcc, 0x78, 0x6a, 0x7a,
0x06, 0x4b, 0x5b, 0x0d, 0x0e, 0x15, 0xbe, 0xa4, 0xfd, 0xbe, 0x00, 0xcd, 0x24, 0x21, 0xd4, 0x86, 0x83, 0xc3, 0xd8, 0x06, 0xd4, 0x08, 0xf8, 0xf1, 0x58, 0x22, 0x28, 0x32, 0x92, 0x75, 0x06, 0x61,
0xa5, 0x21, 0x83, 0x48, 0xeb, 0x91, 0xbf, 0x94, 0x2c, 0x76, 0x8d, 0xbe, 0x4d, 0x63, 0x9e, 0x89, 0x69, 0xa0, 0x0b, 0x55, 0x7e, 0x0c, 0x69, 0x3a, 0x72, 0x48, 0x67, 0x86, 0x33, 0x8b, 0x51, 0xe5,
0x1f, 0x33, 0xe3, 0xa9, 0xe8, 0x35, 0x0e, 0x63, 0x0b, 0x50, 0x23, 0xe0, 0xdb, 0x63, 0x99, 0xb2, 0xa6, 0x23, 0x87, 0x68, 0x1b, 0x9a, 0x7c, 0xcb, 0xa9, 0xe1, 0x1b, 0x8e, 0x34, 0x9c, 0xe7, 0x33,
0xc8, 0x48, 0x56, 0x19, 0x84, 0xe5, 0xc9, 0x36, 0x2c, 0xf1, 0x6d, 0x48, 0xd3, 0x91, 0xbf, 0x74, 0x23, 0xeb, 0x2d, 0x7c, 0xf4, 0x01, 0x75, 0x8e, 0x7d, 0xc3, 0xf2, 0x75, 0x2e, 0xe8, 0x7d, 0xb6,
0xa4, 0x3f, 0xb1, 0x18, 0x55, 0x6e, 0x3a, 0xf2, 0x17, 0x6d, 0x43, 0x9d, 0x2f, 0x39, 0x36, 0x7c, 0x0a, 0xad, 0x81, 0xca, 0x77, 0x19, 0x5b, 0x36, 0x16, 0x26, 0x58, 0x65, 0xb9, 0xa6, 0xcd, 0xe0,
0xc3, 0x91, 0x86, 0xf3, 0x7c, 0xa6, 0xbb, 0xdf, 0xc2, 0x47, 0x1f, 0xd0, 0xe8, 0xb1, 0x6f, 0x58, 0x37, 0x2d, 0x1b, 0x73, 0x2b, 0x0b, 0x8f, 0xc0, 0x44, 0x5b, 0xe3, 0x46, 0xc6, 0x20, 0x54, 0xb0,
0xbe, 0xce, 0x05, 0xbd, 0xcf, 0x66, 0xa1, 0x35, 0x50, 0xf9, 0x2a, 0x43, 0xcb, 0xc6, 0xc2, 0x04, 0xda, 0x5f, 0x8a, 0xb0, 0x4c, 0x7d, 0x4d, 0xb8, 0xdd, 0x29, 0x02, 0xff, 0x45, 0x00, 0x93, 0x04,
0x97, 0x58, 0x32, 0x6e, 0x32, 0xf8, 0x4d, 0xcb, 0xc6, 0xdc, 0xca, 0xc2, 0x2d, 0x30, 0xd1, 0x56, 0x83, 0x44, 0xf0, 0xaf, 0x9b, 0x24, 0xd8, 0xe3, 0xf1, 0xff, 0x2d, 0x19, 0xb7, 0x8b, 0xf9, 0xa5,
0xb8, 0x91, 0x31, 0x08, 0x15, 0xac, 0xf6, 0xf7, 0x22, 0x2c, 0x53, 0x5f, 0x13, 0x6e, 0x77, 0x8a, 0x5c, 0xca, 0xf7, 0xe7, 0x63, 0xf7, 0x89, 0x1a, 0xdd, 0x4b, 0xd0, 0x22, 0xde, 0xcc, 0x1f, 0xe1,
0x68, 0x74, 0x11, 0xc0, 0x24, 0x41, 0x2f, 0x11, 0x91, 0xaa, 0x26, 0x09, 0xf6, 0x78, 0x50, 0x7a, 0x41, 0xa2, 0xe8, 0x6e, 0x72, 0xe0, 0x5e, 0x76, 0x7a, 0xaa, 0x64, 0x36, 0xdc, 0xb1, 0xf8, 0x5d,
0x4b, 0x06, 0x9c, 0x62, 0x7e, 0xad, 0x9b, 0xf2, 0xfd, 0xe9, 0xe4, 0x76, 0xa2, 0x93, 0x80, 0x4b, 0x3d, 0x5d, 0xfc, 0xae, 0xa5, 0xe3, 0xf7, 0x2d, 0xe8, 0x30, 0xf7, 0x1b, 0x4c, 0x3d, 0xc2, 0x7b,
0xd0, 0x20, 0xde, 0xc4, 0x1f, 0xe0, 0x5e, 0xa2, 0x2b, 0xa9, 0x73, 0xe0, 0x5e, 0x76, 0xcc, 0x5c, 0x17, 0xe1, 0xb5, 0x5a, 0x4e, 0xef, 0x7a, 0x87, 0x4c, 0xf6, 0x05, 0xaa, 0xde, 0x66, 0x4b, 0xe5,
0xcc, 0x3c, 0x91, 0x88, 0x45, 0xb7, 0xa5, 0xd3, 0x25, 0xb8, 0x4a, 0x3a, 0xc1, 0xdd, 0x82, 0x16, 0x90, 0x68, 0x3f, 0x29, 0xc0, 0x8a, 0xe8, 0x85, 0x4e, 0xaf, 0xd8, 0xbc, 0x8c, 0x2e, 0xa3, 0x66,
0x73, 0xbf, 0xde, 0xd8, 0x23, 0xbc, 0xb9, 0x13, 0x5e, 0xab, 0xe5, 0x34, 0xf7, 0x77, 0xc8, 0x68, 0xf1, 0x31, 0xdd, 0x45, 0x69, 0x81, 0x2c, 0x5f, 0xce, 0xc8, 0xf2, 0xc9, 0x0a, 0xbb, 0x32, 0x57,
0x5f, 0xa0, 0xea, 0x4d, 0x36, 0x55, 0xfe, 0x12, 0xed, 0x17, 0x05, 0x58, 0x11, 0xcd, 0xe2, 0xe9, 0x61, 0x87, 0xfd, 0x72, 0xf5, 0x09, 0xfa, 0xe5, 0xbf, 0x2b, 0xd0, 0xea, 0x63, 0xc3, 0x1f, 0x1d,
0x15, 0x9b, 0x97, 0x66, 0x64, 0xd4, 0x2c, 0x3e, 0xa1, 0xfd, 0x2a, 0xcd, 0x51, 0x06, 0x95, 0x33, 0x48, 0x61, 0xbc, 0x11, 0xbf, 0x24, 0x78, 0x21, 0x47, 0xd0, 0x89, 0x25, 0x5f, 0x9e, 0xdb, 0x81,
0xca, 0xa0, 0x64, 0x0b, 0xb2, 0x38, 0xd5, 0x82, 0x84, 0x07, 0x0a, 0x4b, 0xc7, 0x38, 0x50, 0xf8, 0x7f, 0x28, 0xd0, 0xfc, 0x3a, 0x9d, 0x92, 0x87, 0x7d, 0x33, 0x7e, 0xd8, 0x17, 0x73, 0x0e, 0xab,
0xa7, 0x02, 0x8d, 0x2e, 0x36, 0xfc, 0xc1, 0x81, 0x14, 0xc6, 0x1b, 0xf1, 0x53, 0x94, 0x17, 0x72, 0xe3, 0xc0, 0xb7, 0xf0, 0x21, 0xfe, 0xd2, 0x1d, 0xf7, 0x4f, 0x0a, 0xf4, 0xfa, 0x47, 0xee, 0x48,
0x04, 0x9d, 0x98, 0xf2, 0xd5, 0x39, 0x3e, 0xf9, 0x97, 0x02, 0xf5, 0x6f, 0xd3, 0x21, 0xb9, 0xd9, 0xe7, 0x1e, 0x75, 0x7a, 0xb3, 0xbf, 0x04, 0xad, 0xc3, 0x44, 0x1f, 0x50, 0x60, 0xe1, 0xbf, 0x79,
0x37, 0xe3, 0x9b, 0x7d, 0x31, 0x67, 0xb3, 0x3a, 0x0e, 0x7c, 0x0b, 0x1f, 0xe2, 0xaf, 0xdc, 0x76, 0x18, 0x6f, 0x04, 0x74, 0x50, 0xe5, 0xdd, 0x84, 0x38, 0xac, 0x0c, 0x70, 0x2f, 0x65, 0x71, 0x9d,
0xff, 0xaa, 0x40, 0xa7, 0x7b, 0xe4, 0x0e, 0x74, 0xee, 0x51, 0xa7, 0x37, 0xfb, 0x4b, 0xd0, 0x38, 0x62, 0x8e, 0x05, 0x88, 0x8e, 0x9f, 0x04, 0x6a, 0x3e, 0x2c, 0x67, 0xe0, 0xa1, 0xf3, 0x50, 0x15,
0x4c, 0x34, 0x4a, 0x05, 0x16, 0xfe, 0xeb, 0x87, 0xf1, 0x4e, 0x49, 0x07, 0x55, 0xd6, 0x43, 0x62, 0x3d, 0x87, 0x48, 0x64, 0xdc, 0x0f, 0x4d, 0xaa, 0x9d, 0xa8, 0x6b, 0xb6, 0xcc, 0xf9, 0x22, 0xc8,
0xb3, 0x32, 0xc0, 0xbd, 0x94, 0xc5, 0x75, 0x8a, 0x39, 0x16, 0x20, 0x5a, 0x7e, 0x12, 0xa8, 0xf9, 0x44, 0xcf, 0x41, 0x23, 0x2c, 0x0e, 0xcd, 0x39, 0xf5, 0x98, 0x44, 0xfb, 0xbd, 0x02, 0x2b, 0xef,
0xb0, 0x9c, 0x81, 0x87, 0xce, 0xc3, 0x92, 0x68, 0xca, 0x44, 0x22, 0xe3, 0x7e, 0x68, 0x52, 0xed, 0x19, 0xae, 0xe9, 0x8d, 0xc7, 0xa7, 0x97, 0xdc, 0x16, 0x24, 0xea, 0xc6, 0x45, 0x3b, 0xd2, 0xc4,
0x44, 0xc7, 0x0a, 0x96, 0x39, 0x5d, 0x04, 0x99, 0xe8, 0x39, 0xa8, 0x85, 0xd5, 0xb3, 0x39, 0xa5, 0x22, 0xf4, 0x32, 0x9c, 0xf1, 0x79, 0x04, 0x33, 0x93, 0xa2, 0x2d, 0xea, 0xaa, 0x9c, 0x08, 0x45,
0x1e, 0x93, 0x68, 0x7f, 0x52, 0x60, 0xe5, 0x3d, 0xc3, 0x35, 0xbd, 0xe1, 0xf0, 0xf4, 0x92, 0xdb, 0xf6, 0xeb, 0x02, 0x20, 0x1a, 0x75, 0x6f, 0x18, 0xb6, 0xe1, 0x8e, 0xf0, 0xc9, 0x59, 0xbf, 0x0c,
0x82, 0x44, 0x61, 0x3d, 0x6f, 0xcb, 0x9e, 0x98, 0x84, 0x5e, 0x86, 0x33, 0x3e, 0x8f, 0x60, 0x66, 0xed, 0x44, 0xae, 0x08, 0x2f, 0xee, 0xe3, 0xc9, 0x82, 0xa0, 0x5b, 0xd0, 0x1e, 0x72, 0x52, 0x03,
0x52, 0xb4, 0x45, 0x5d, 0x95, 0x03, 0xa1, 0xc8, 0x7e, 0x57, 0x00, 0x44, 0xa3, 0xee, 0x0d, 0xc3, 0x1f, 0x1b, 0xc4, 0x73, 0x59, 0x10, 0x6c, 0x67, 0x37, 0x9f, 0x77, 0x7d, 0x6b, 0x32, 0xc1, 0xfe,
0x36, 0xdc, 0x01, 0x3e, 0x39, 0xeb, 0x97, 0xa1, 0x99, 0xc8, 0x15, 0xe1, 0xcd, 0x46, 0x3c, 0x59, 0x96, 0xe7, 0x9a, 0x3c, 0x5a, 0xb7, 0x86, 0x92, 0x4d, 0xba, 0x94, 0x2a, 0x27, 0x4a, 0x9c, 0xb2,
0x10, 0x74, 0x0b, 0x9a, 0x7d, 0x4e, 0xaa, 0xe7, 0x63, 0x83, 0x78, 0x2e, 0x0b, 0x82, 0xcd, 0xec, 0xe9, 0x81, 0x30, 0x73, 0x32, 0x51, 0x10, 0x6c, 0xd8, 0x91, 0x20, 0xa2, 0xa8, 0xa9, 0xf2, 0x89,
0xee, 0xfc, 0xae, 0x6f, 0x8d, 0x46, 0xd8, 0xdf, 0xf2, 0x5c, 0x93, 0x47, 0xeb, 0x46, 0x5f, 0xb2, 0x7e, 0xfe, 0xdd, 0x43, 0x46, 0x22, 0xd3, 0x7e, 0xab, 0x00, 0x0a, 0xcb, 0x7f, 0xd6, 0x0c, 0x30,
0x49, 0xa7, 0x52, 0xe5, 0x44, 0x89, 0x53, 0x76, 0x85, 0x10, 0x66, 0x4e, 0x26, 0x0a, 0x82, 0x0d, 0x0b, 0x4b, 0x2f, 0x55, 0x32, 0x82, 0xf7, 0x05, 0xa8, 0x9b, 0x72, 0xa5, 0xf0, 0x88, 0x08, 0x40,
0x3b, 0x12, 0x44, 0x14, 0x35, 0x55, 0x3e, 0xd0, 0xcd, 0x3f, 0x9c, 0xc9, 0x48, 0x64, 0xda, 0x1f, 0x7d, 0x86, 0x1f, 0x63, 0x40, 0xb3, 0x1e, 0x36, 0x65, 0xb5, 0xcd, 0x81, 0xb7, 0x19, 0x2c, 0x99,
0x14, 0x40, 0x61, 0x7f, 0xc4, 0x3a, 0x12, 0x66, 0x61, 0xe9, 0xa9, 0x4a, 0x46, 0xf0, 0xbe, 0x00, 0x07, 0x4b, 0xe9, 0x3c, 0x18, 0x6f, 0xad, 0xcb, 0x89, 0xd6, 0x5a, 0xfb, 0xa4, 0x00, 0x2a, 0x8b,
0x55, 0x53, 0xce, 0x14, 0x1e, 0x11, 0x01, 0xa8, 0xcf, 0xf0, 0x6d, 0xf4, 0x68, 0xd6, 0xc3, 0xa6, 0x68, 0x5b, 0x51, 0x7f, 0xb7, 0x10, 0xd3, 0x97, 0xa0, 0x25, 0x9e, 0xb6, 0x12, 0x8c, 0x37, 0x1f,
0xac, 0xb6, 0x39, 0xf0, 0x36, 0x83, 0x25, 0xf3, 0x60, 0x29, 0x9d, 0x07, 0xe3, 0x67, 0x0f, 0xe5, 0xc6, 0x36, 0x43, 0x57, 0xe1, 0x2c, 0x47, 0xf2, 0x31, 0x99, 0xd9, 0x51, 0xa1, 0xc9, 0xab, 0x3e,
0xc4, 0xd9, 0x83, 0xf6, 0x49, 0x01, 0x54, 0x16, 0xd1, 0xb6, 0xa2, 0xc6, 0x64, 0x2e, 0xa6, 0x2f, 0xf4, 0x90, 0x87, 0x52, 0x3a, 0x25, 0x57, 0xdc, 0x83, 0x95, 0x89, 0xed, 0x0d, 0x0d, 0x7b, 0x90,
0x41, 0x43, 0xdc, 0xfd, 0x25, 0x18, 0xaf, 0x3f, 0x8c, 0x2d, 0x86, 0xae, 0xc2, 0x59, 0x8e, 0xe4, 0x54, 0x0f, 0xd7, 0xe1, 0x02, 0x16, 0x7f, 0x96, 0x2f, 0xef, 0xc7, 0x75, 0x48, 0xd0, 0x0e, 0xed,
0x63, 0x32, 0xb1, 0xa3, 0x42, 0x93, 0x57, 0x7d, 0xe8, 0x21, 0x0f, 0xa5, 0x74, 0x48, 0xce, 0xb8, 0xe4, 0xf0, 0x83, 0xb0, 0x10, 0x10, 0xb7, 0xa6, 0x8b, 0xd4, 0x01, 0x4d, 0xba, 0x50, 0x8e, 0xb4,
0x07, 0x2b, 0x23, 0xdb, 0xeb, 0x1b, 0x76, 0x2f, 0xa9, 0x9e, 0x9c, 0xa6, 0x6a, 0xda, 0xe2, 0xcf, 0x9f, 0x29, 0xd0, 0x49, 0xdd, 0x8e, 0xa5, 0x1b, 0x26, 0x65, 0xbe, 0x61, 0x7a, 0x13, 0xca, 0xb4,
0xf2, 0xe9, 0xdd, 0xb8, 0x0e, 0x09, 0xda, 0xa1, 0xad, 0x2e, 0x7e, 0x10, 0x16, 0x02, 0xe2, 0x58, 0x8b, 0xe0, 0xf1, 0xae, 0x9d, 0x5d, 0xcc, 0x27, 0x77, 0xd5, 0xf9, 0x02, 0x74, 0x05, 0x96, 0x33,
0x79, 0x9e, 0x3a, 0xa0, 0x4e, 0x27, 0xca, 0x3f, 0xed, 0x63, 0x05, 0x5a, 0xa9, 0xe3, 0xc3, 0x74, 0xde, 0x51, 0x84, 0x0d, 0xa0, 0xf9, 0x67, 0x14, 0xed, 0x77, 0x25, 0x68, 0xc4, 0xe4, 0x71, 0x4c,
0xc3, 0xa4, 0x4c, 0x37, 0x4c, 0x6f, 0x42, 0x99, 0x76, 0x11, 0x3c, 0xde, 0x35, 0xb3, 0x8b, 0xf9, 0xaf, 0xb7, 0xc8, 0x75, 0x47, 0xea, 0x78, 0xc5, 0xf9, 0xe3, 0xe5, 0x3c, 0x24, 0x50, 0xbb, 0x73,
0xe4, 0xaa, 0x3a, 0x9f, 0x80, 0xae, 0xc0, 0x72, 0xc6, 0x45, 0x93, 0xb0, 0x01, 0x34, 0x7d, 0xcf, 0xb0, 0xc3, 0xab, 0x64, 0x51, 0xb2, 0x3b, 0xd8, 0x61, 0xcd, 0x07, 0x35, 0xc9, 0x99, 0xc3, 0xbb,
0xa4, 0xfd, 0xb1, 0x04, 0xb5, 0x98, 0x3c, 0x66, 0xf4, 0x7a, 0xf3, 0x9c, 0x07, 0xa5, 0xb6, 0x57, 0x34, 0xee, 0x4e, 0x55, 0x77, 0xe6, 0xb0, 0x1e, 0x2d, 0xd9, 0x20, 0x54, 0x1f, 0xd3, 0x20, 0xd4,
0x9c, 0xde, 0x5e, 0xce, 0x4d, 0x0b, 0xb5, 0x3b, 0x07, 0x3b, 0xbc, 0x4a, 0x16, 0x25, 0xbb, 0x83, 0x92, 0x0d, 0x42, 0xc2, 0x8f, 0xea, 0x69, 0x3f, 0x5a, 0xb4, 0xfd, 0xba, 0x0a, 0xcb, 0x23, 0x1f,
0x1d, 0xd6, 0x7c, 0x50, 0x93, 0x9c, 0x38, 0xbc, 0x4b, 0xe3, 0xee, 0xb4, 0xe4, 0x4e, 0x1c, 0xd6, 0x1b, 0x01, 0x36, 0x6f, 0x1c, 0x6d, 0x85, 0x53, 0xdd, 0x06, 0xcb, 0xab, 0x59, 0x53, 0xe8, 0x66,
0xa3, 0x25, 0x1b, 0x84, 0xa5, 0x27, 0x34, 0x08, 0x95, 0x64, 0x83, 0x90, 0xf0, 0xa3, 0x6a, 0xda, 0x74, 0x4d, 0xc0, 0xb5, 0xdc, 0x64, 0x5a, 0xce, 0xee, 0x3f, 0x84, 0x6e, 0xb8, 0x92, 0x65, 0x78,
0x8f, 0xe6, 0x6d, 0xbf, 0xae, 0xc2, 0xf2, 0xc0, 0xc7, 0x46, 0x80, 0xcd, 0x1b, 0x47, 0x5b, 0xe1, 0x66, 0xa3, 0x74, 0xe3, 0xd7, 0x3a, 0x51, 0xe3, 0xf7, 0x1c, 0x34, 0x64, 0xf6, 0xa4, 0xee, 0xde,
0x50, 0xbb, 0xc6, 0xf2, 0x6a, 0xd6, 0x10, 0xba, 0x19, 0x9d, 0xa3, 0x70, 0x2d, 0xd7, 0x99, 0x96, 0xe6, 0x91, 0x4f, 0xc6, 0x02, 0x93, 0x24, 0x82, 0x41, 0x27, 0x19, 0x0c, 0xfe, 0x5c, 0x84, 0x76,
0xb3, 0xfb, 0x0f, 0xa1, 0x1b, 0xae, 0x64, 0x19, 0x9e, 0xd9, 0x5f, 0xba, 0xf1, 0x6b, 0x9c, 0xa8, 0x54, 0xf2, 0x2f, 0x1c, 0x0a, 0x16, 0x79, 0x0f, 0xdc, 0x03, 0x35, 0x4a, 0xa8, 0x4c, 0x4a, 0x8f,
0xf1, 0x7b, 0x0e, 0x6a, 0xd1, 0x69, 0x02, 0x69, 0x37, 0x79, 0xe4, 0x0b, 0x8f, 0x13, 0x48, 0x22, 0xed, 0x5a, 0xd2, 0x17, 0xd0, 0x9d, 0x69, 0xca, 0xe7, 0x12, 0x57, 0x7c, 0xa5, 0x27, 0xba, 0xe2,
0x18, 0xb4, 0x92, 0xc1, 0xe0, 0x6f, 0x45, 0x68, 0x46, 0x25, 0xff, 0xdc, 0xa1, 0x60, 0x9e, 0x0b, 0x3b, 0xe5, 0xd3, 0xc9, 0x35, 0x38, 0x17, 0x26, 0xd1, 0xc4, 0xb1, 0x79, 0x45, 0x7d, 0x56, 0x4e,
0xd3, 0x3d, 0x50, 0xa3, 0x84, 0xca, 0xa4, 0xf4, 0xc4, 0xae, 0x25, 0x7d, 0x42, 0xdf, 0x1a, 0xa7, 0xee, 0xc7, 0x8f, 0x9f, 0xe3, 0xc6, 0xd5, 0x3c, 0x37, 0x4e, 0xab, 0xb1, 0x36, 0xa7, 0xc6, 0xf9,
0x7c, 0x2e, 0x71, 0x06, 0x5a, 0x3a, 0xd6, 0x19, 0xe8, 0x29, 0xef, 0x96, 0xae, 0xc1, 0xb9, 0x30, 0x17, 0x9c, 0x7a, 0xd6, 0x0b, 0xce, 0x3d, 0x58, 0xbe, 0xe7, 0x92, 0xd9, 0x90, 0x8c, 0x7c, 0x6b,
0x89, 0x26, 0xb6, 0xcd, 0x2b, 0xea, 0xb3, 0x72, 0x70, 0x3f, 0xbe, 0xfd, 0x1c, 0x37, 0x5e, 0xca, 0x88, 0xc3, 0xd2, 0x72, 0x11, 0xb5, 0xf6, 0xa0, 0x96, 0xaa, 0x4e, 0xc3, 0xb1, 0xf6, 0x7d, 0x05,
0x73, 0xe3, 0xb4, 0x1a, 0x2b, 0x53, 0x6a, 0x9c, 0xbe, 0xe2, 0xaa, 0x66, 0x5d, 0x71, 0xdd, 0x83, 0x56, 0xe6, 0xf7, 0x65, 0x16, 0x13, 0x05, 0x03, 0x25, 0x11, 0x0c, 0xbe, 0x09, 0xcb, 0xd1, 0xf6,
0xe5, 0x7b, 0x2e, 0x99, 0xf4, 0xc9, 0xc0, 0xb7, 0xfa, 0x38, 0x2c, 0x2d, 0xe7, 0x51, 0x6b, 0x07, 0xc9, 0xba, 0x37, 0xa7, 0xb2, 0xcb, 0x60, 0x5c, 0x47, 0xd1, 0x1e, 0x12, 0xa6, 0xfd, 0x53, 0x09,
0x2a, 0xa9, 0xea, 0x34, 0xfc, 0xd7, 0x7e, 0xac, 0xc0, 0xca, 0xf4, 0xba, 0xcc, 0x62, 0xa2, 0x60, 0x2f, 0xd1, 0x28, 0x6c, 0xc2, 0xae, 0x06, 0x69, 0x82, 0xf2, 0x5c, 0xdb, 0x72, 0xc3, 0x16, 0x55,
0xa0, 0x24, 0x82, 0xc1, 0x77, 0x61, 0x39, 0x5a, 0x3e, 0x59, 0xf7, 0xe6, 0x54, 0x76, 0x19, 0x8c, 0x9c, 0x91, 0x03, 0x45, 0x8b, 0xfa, 0x1e, 0x74, 0x04, 0x52, 0x98, 0x67, 0x16, 0xac, 0xac, 0xda,
0xeb, 0x28, 0x5a, 0x43, 0xc2, 0xb4, 0x7f, 0x2b, 0xe1, 0x29, 0x23, 0x85, 0x8d, 0xd8, 0xd9, 0x29, 0x7c, 0x5d, 0x98, 0x61, 0x2e, 0x43, 0xdb, 0x1b, 0x8f, 0xe3, 0xf4, 0x78, 0xa0, 0x6c, 0x09, 0xa8,
0x4d, 0x50, 0x9e, 0x6b, 0x5b, 0x6e, 0xd8, 0xa2, 0x8a, 0x3d, 0x72, 0xa0, 0x68, 0x51, 0xdf, 0x83, 0x20, 0xf8, 0x35, 0x50, 0x25, 0xda, 0x93, 0x66, 0xb6, 0x8e, 0x58, 0x18, 0x56, 0x68, 0xdf, 0x53,
0x96, 0x40, 0x0a, 0xf3, 0xcc, 0x9c, 0x95, 0x55, 0x93, 0xcf, 0x0b, 0x33, 0xcc, 0x65, 0x68, 0x7a, 0xa0, 0x9b, 0xcc, 0x73, 0xb1, 0xe3, 0x3f, 0x79, 0x9d, 0xf6, 0x95, 0xe4, 0x6b, 0xc7, 0xe5, 0xc7,
0xc3, 0x61, 0x9c, 0x1e, 0x0f, 0x94, 0x0d, 0x01, 0x15, 0x04, 0xbf, 0x05, 0xaa, 0x44, 0x3b, 0x6e, 0xf0, 0x13, 0xd1, 0x11, 0xf7, 0x09, 0xeb, 0x5f, 0x85, 0x7a, 0xd8, 0x3e, 0xa0, 0x06, 0x54, 0xef,
0x66, 0x6b, 0x89, 0x89, 0x61, 0x85, 0xf6, 0x23, 0x05, 0xda, 0xc9, 0x3c, 0x17, 0xdb, 0xfe, 0xf1, 0xb9, 0xb7, 0x5c, 0xef, 0x91, 0xab, 0x2e, 0xa1, 0x2a, 0x14, 0xaf, 0xdb, 0xb6, 0xaa, 0xa0, 0x16,
0xeb, 0xb4, 0x6f, 0x24, 0xaf, 0x83, 0x2e, 0x3f, 0x81, 0x9f, 0x88, 0x8e, 0x38, 0x4f, 0x58, 0xff, 0xd4, 0xfb, 0x81, 0x8f, 0x0d, 0xc7, 0x72, 0x27, 0x6a, 0x01, 0xb5, 0x01, 0xde, 0xb3, 0x48, 0xe0,
0x26, 0x54, 0xc3, 0xf6, 0x01, 0xd5, 0x60, 0xe9, 0x9e, 0x7b, 0xcb, 0xf5, 0x1e, 0xb9, 0xea, 0x02, 0xf9, 0xd6, 0xc8, 0xb0, 0xd5, 0xe2, 0xfa, 0xc7, 0xd0, 0x4e, 0x7a, 0x3d, 0x6a, 0x42, 0x6d, 0xcf,
0x5a, 0x82, 0xe2, 0x75, 0xdb, 0x56, 0x15, 0xd4, 0x80, 0x6a, 0x37, 0xf0, 0xb1, 0xe1, 0x58, 0xee, 0x0b, 0xde, 0xfd, 0xc8, 0x22, 0x81, 0xba, 0x44, 0xf1, 0xf7, 0xbc, 0x60, 0xdf, 0xc7, 0x04, 0xbb,
0x48, 0x2d, 0xa0, 0x26, 0xc0, 0x7b, 0x16, 0x09, 0x3c, 0xdf, 0x1a, 0x18, 0xb6, 0x5a, 0x5c, 0xff, 0x81, 0xaa, 0x20, 0x80, 0xca, 0xfb, 0xee, 0xb6, 0x45, 0x1e, 0xa8, 0x05, 0xb4, 0x2c, 0x92, 0xb2,
0x08, 0x9a, 0x49, 0xaf, 0x47, 0x75, 0xa8, 0xec, 0x79, 0xc1, 0xbb, 0x8f, 0x2d, 0x12, 0xa8, 0x0b, 0x61, 0xef, 0x0a, 0x57, 0x52, 0x8b, 0x74, 0x79, 0x38, 0x2a, 0x21, 0x15, 0x9a, 0x21, 0xca, 0xce,
0x14, 0x7f, 0xcf, 0x0b, 0xf6, 0x7d, 0x4c, 0xb0, 0x1b, 0xa8, 0x0a, 0x02, 0x58, 0x7c, 0xdf, 0xdd, 0xfe, 0x3d, 0xb5, 0x8c, 0xea, 0x50, 0xe6, 0x9f, 0x95, 0x75, 0x13, 0xd4, 0x74, 0x45, 0x49, 0xf7,
0xb6, 0xc8, 0x03, 0xb5, 0x80, 0x96, 0x45, 0x52, 0x36, 0xec, 0x5d, 0xe1, 0x4a, 0x6a, 0x91, 0x4e, 0xe4, 0x87, 0x08, 0x41, 0xea, 0x12, 0x3d, 0x99, 0x28, 0xe9, 0x55, 0x05, 0x75, 0xa0, 0x11, 0x2b,
0x0f, 0xff, 0x4a, 0x48, 0x85, 0x7a, 0x88, 0xb2, 0xb3, 0x7f, 0x4f, 0x2d, 0xa3, 0x2a, 0x94, 0xf9, 0x90, 0xd5, 0x02, 0x05, 0xec, 0xf8, 0xd3, 0x91, 0x28, 0x95, 0x39, 0x0b, 0x54, 0xef, 0xdb, 0x54,
0xe7, 0xe2, 0xba, 0x09, 0x6a, 0xba, 0xa2, 0xa4, 0x6b, 0xf2, 0x4d, 0x84, 0x20, 0x75, 0x81, 0xee, 0x12, 0xa5, 0xf5, 0x1b, 0x50, 0x93, 0xe1, 0x88, 0xa2, 0x0a, 0x11, 0xd1, 0xa1, 0xba, 0x84, 0xce,
0x4c, 0x94, 0xf4, 0xaa, 0x82, 0x5a, 0x50, 0x8b, 0x15, 0xc8, 0x6a, 0x81, 0x02, 0x76, 0xfc, 0xf1, 0x40, 0x2b, 0xf1, 0xee, 0xac, 0x2a, 0x08, 0x41, 0x3b, 0xf9, 0x3f, 0x84, 0x5a, 0xd8, 0xfc, 0x79,
0x40, 0x94, 0xca, 0x9c, 0x05, 0xaa, 0xf7, 0x6d, 0x2a, 0x89, 0xd2, 0xfa, 0x0d, 0xa8, 0xc8, 0x70, 0x13, 0x80, 0xd7, 0x6b, 0x9e, 0xe7, 0x9b, 0x68, 0x0a, 0x68, 0x07, 0x07, 0x34, 0x17, 0x79, 0xae,
0x44, 0x51, 0x85, 0x88, 0xe8, 0xaf, 0xba, 0x80, 0xce, 0x40, 0x23, 0x71, 0x31, 0xaf, 0x2a, 0x08, 0xcc, 0x23, 0x04, 0x5d, 0xcd, 0x7f, 0x9a, 0x4f, 0xa1, 0x0a, 0x56, 0x7b, 0x79, 0xad, 0x6b, 0x0a,
0x41, 0x33, 0xf9, 0x60, 0x44, 0x2d, 0x6c, 0xfe, 0xaa, 0x0e, 0xc0, 0xeb, 0x35, 0xcf, 0xf3, 0x4d, 0x5d, 0x5b, 0x42, 0x0e, 0xa3, 0x78, 0xd7, 0x72, 0xf0, 0x5d, 0x6b, 0xf4, 0x20, 0x2c, 0xf4, 0xf2,
0x34, 0x06, 0xb4, 0x83, 0x03, 0x9a, 0x8b, 0x3c, 0x57, 0xe6, 0x11, 0x82, 0xae, 0xe6, 0xbf, 0x5d, 0x29, 0xa6, 0x50, 0x25, 0xc5, 0x54, 0xd8, 0x17, 0x83, 0x7e, 0xe0, 0x5b, 0xee, 0x44, 0xbe, 0x5e,
0x48, 0xa1, 0x0a, 0x56, 0x3b, 0x79, 0xad, 0x6b, 0x0a, 0x5d, 0x5b, 0x40, 0x0e, 0xa3, 0x78, 0xd7, 0x69, 0x4b, 0xe8, 0x61, 0xea, 0x57, 0x04, 0x49, 0x70, 0x73, 0x91, 0xbf, 0x0f, 0x4e, 0x46, 0xd2,
0x72, 0xf0, 0x5d, 0x6b, 0xf0, 0x20, 0x2c, 0xf4, 0xf2, 0x29, 0xa6, 0x50, 0x25, 0xc5, 0x54, 0xd8, 0x86, 0x4e, 0xea, 0xbf, 0x25, 0xb4, 0x9e, 0xfd, 0x00, 0x96, 0xf5, 0x8f, 0x55, 0xef, 0xe5, 0x85,
0x17, 0x3f, 0xdd, 0xc0, 0xb7, 0xdc, 0x91, 0xbc, 0xde, 0xd3, 0x16, 0xd0, 0xc3, 0xd4, 0x5b, 0x0d, 0x70, 0x43, 0x6a, 0x16, 0xb4, 0x93, 0xff, 0xe6, 0xa0, 0xff, 0xcd, 0xdb, 0x60, 0xee, 0x09, 0xbe,
0x49, 0x70, 0x73, 0x9e, 0xe7, 0x19, 0x27, 0x23, 0x69, 0x43, 0x2b, 0xf5, 0xb0, 0x0b, 0xad, 0x67, 0xb7, 0xbe, 0x08, 0x6a, 0x48, 0xea, 0x3e, 0xb7, 0xa7, 0xe3, 0x48, 0x65, 0xfe, 0xfe, 0xd0, 0x7b,
0xdf, 0x10, 0x66, 0x3d, 0x42, 0xeb, 0xbc, 0x3c, 0x17, 0x6e, 0x48, 0xcd, 0x82, 0x66, 0xf2, 0xf1, 0xdc, 0xc3, 0xa1, 0xb6, 0x84, 0xbe, 0x05, 0x67, 0xe6, 0x7e, 0x14, 0x40, 0xaf, 0x64, 0xf7, 0xeb,
0x12, 0xfa, 0xff, 0xbc, 0x05, 0xa6, 0xde, 0x28, 0x74, 0xd6, 0xe7, 0x41, 0x0d, 0x49, 0xdd, 0xe7, 0xd9, 0xff, 0x13, 0x1c, 0x47, 0xe1, 0x7e, 0xda, 0x1b, 0xf2, 0xb9, 0x9f, 0xfb, 0xa9, 0x66, 0x71,
0xf6, 0x34, 0x8b, 0x54, 0xe6, 0xfb, 0x90, 0xce, 0x93, 0x6e, 0x56, 0xb5, 0x05, 0xf4, 0x3d, 0x38, 0xee, 0x63, 0xdb, 0x3f, 0x8e, 0xfb, 0x27, 0xa6, 0x30, 0x63, 0x6e, 0x93, 0xee, 0x1c, 0x5e, 0xcd,
0x33, 0xf5, 0x92, 0x02, 0xbd, 0x92, 0xdd, 0xaf, 0x67, 0x3f, 0xb8, 0x98, 0x45, 0xe1, 0x7e, 0xda, 0x22, 0x91, 0xfb, 0xb7, 0x42, 0x6f, 0x63, 0x51, 0xf4, 0xb8, 0x75, 0x25, 0x1f, 0xc4, 0xb3, 0x85,
0x1b, 0xf2, 0xb9, 0x9f, 0x7a, 0x75, 0x34, 0x3f, 0xf7, 0xb1, 0xe5, 0x9f, 0xc4, 0xfd, 0xb1, 0x29, 0x96, 0xf9, 0x88, 0x9f, 0x6d, 0x5d, 0xd9, 0xef, 0xeb, 0xda, 0x12, 0xba, 0x9b, 0x88, 0x86, 0xe8,
0x4c, 0x98, 0xdb, 0xa4, 0x3b, 0x87, 0x57, 0xb3, 0x48, 0xe4, 0x3e, 0xe7, 0xe8, 0x6c, 0xcc, 0x8b, 0xc5, 0x3c, 0xe5, 0x24, 0xef, 0x13, 0x8e, 0x93, 0xdb, 0x00, 0x60, 0x07, 0x07, 0x77, 0x70, 0xe0,
0x1e, 0xb7, 0xae, 0xe4, 0x8b, 0x81, 0x6c, 0xa1, 0x65, 0xbe, 0x72, 0xc8, 0xb6, 0xae, 0xec, 0x07, 0x5b, 0x23, 0x92, 0xde, 0x54, 0x0c, 0x22, 0x04, 0xb9, 0xe9, 0x4b, 0xc7, 0xe2, 0x85, 0x6c, 0x0f,
0x08, 0xda, 0x02, 0xba, 0x9b, 0x88, 0x86, 0xe8, 0xc5, 0x3c, 0xe5, 0x24, 0xcf, 0x13, 0x66, 0xc9, 0xa1, 0xb1, 0x83, 0x03, 0x71, 0x39, 0x44, 0x50, 0xee, 0x4a, 0x89, 0x21, 0x49, 0xac, 0x1d, 0x8f,
0xad, 0x07, 0xb0, 0x83, 0x83, 0x3b, 0x38, 0xf0, 0xad, 0x01, 0x49, 0x2f, 0x2a, 0x7e, 0x22, 0x04, 0x18, 0x8f, 0x28, 0xa9, 0xf7, 0x79, 0x94, 0x2b, 0xdb, 0xf9, 0xbf, 0x06, 0xb2, 0x23, 0x4a, 0xce,
0xb9, 0xe8, 0x4b, 0x33, 0xf1, 0x42, 0xb6, 0xfb, 0x50, 0xdb, 0xc1, 0x81, 0x38, 0x1c, 0x22, 0x28, 0x83, 0xbf, 0xb6, 0xb4, 0xf9, 0xef, 0x06, 0xd4, 0x59, 0x8a, 0xa0, 0xa9, 0xe7, 0xbf, 0x19, 0xe2,
0x77, 0xa6, 0xc4, 0x90, 0x24, 0xd6, 0x66, 0x23, 0xc6, 0x23, 0x4a, 0xea, 0x01, 0x03, 0xca, 0x95, 0x29, 0x64, 0x88, 0x0f, 0xa1, 0x93, 0x7a, 0xee, 0xcd, 0xd6, 0x67, 0xf6, 0x9b, 0xf0, 0x71, 0x26,
0xed, 0xf4, 0xb3, 0x8a, 0xec, 0x88, 0x92, 0xf3, 0x22, 0x42, 0x5b, 0xd8, 0xfc, 0xb4, 0x0e, 0x55, 0xff, 0x01, 0xff, 0x9b, 0x21, 0xac, 0x2c, 0x5f, 0xca, 0xf3, 0xa4, 0xd4, 0xad, 0xe2, 0xe7, 0x1f,
0x96, 0x22, 0x68, 0xea, 0xf9, 0x5f, 0x86, 0x78, 0x0a, 0x19, 0xe2, 0x43, 0x68, 0xa5, 0xee, 0xc3, 0xe4, 0x9e, 0x7e, 0x12, 0xf8, 0x10, 0x3a, 0xa9, 0x37, 0x98, 0x6c, 0xc9, 0x67, 0x3f, 0xd4, 0x1c,
0xb3, 0xf5, 0x99, 0x7d, 0x69, 0x3e, 0xcb, 0xe4, 0xfb, 0x80, 0xa6, 0x6f, 0xb4, 0xb3, 0x43, 0x45, 0xb7, 0xfb, 0x67, 0x18, 0x2d, 0x4d, 0x58, 0xce, 0xb8, 0x59, 0x47, 0x99, 0x11, 0x3e, 0xff, 0x0a,
0xee, 0xcd, 0xf7, 0x2c, 0x1a, 0x1f, 0xf0, 0x27, 0x25, 0x61, 0xf5, 0xfa, 0x52, 0x9e, 0xb7, 0xa6, 0xfe, 0xf8, 0x03, 0xb5, 0x12, 0xe6, 0x8e, 0xd6, 0xf2, 0x98, 0x4c, 0xff, 0xb1, 0xd9, 0x7b, 0x65,
0x4e, 0x2e, 0xbf, 0xf8, 0x40, 0xfa, 0xf4, 0x13, 0xcd, 0x87, 0xd0, 0x4a, 0xdd, 0xf3, 0x64, 0x6b, 0xb1, 0xdf, 0x3b, 0xc3, 0x03, 0xf5, 0xa1, 0xc2, 0x1f, 0x75, 0xd0, 0xf3, 0xd9, 0x8d, 0x43, 0xec,
0x37, 0xfb, 0x32, 0x68, 0xd6, 0xea, 0x9f, 0x63, 0x44, 0x36, 0x61, 0x39, 0xe3, 0xf4, 0x1e, 0x65, 0xc1, 0xa7, 0x77, 0xdc, 0xb3, 0x10, 0x99, 0xd9, 0x01, 0x61, 0x9b, 0x96, 0x59, 0x24, 0x43, 0x99,
0x66, 0x91, 0xfc, 0x63, 0xfe, 0xd9, 0x1b, 0x6a, 0x24, 0x5c, 0x0a, 0xad, 0xe5, 0x31, 0x99, 0x7e, 0x6f, 0x82, 0xf1, 0x97, 0x98, 0xde, 0xf1, 0x8f, 0x2f, 0x72, 0xd3, 0x6f, 0x03, 0xe2, 0xe5, 0x98,
0x36, 0xdb, 0x79, 0x65, 0xbe, 0x37, 0xb6, 0xe1, 0x86, 0xba, 0xb0, 0xc8, 0x2f, 0x8e, 0xd0, 0xf3, 0x3b, 0xb6, 0x26, 0x33, 0xdf, 0xe0, 0x66, 0x9a, 0x17, 0x9f, 0xe6, 0x51, 0x25, 0xc5, 0xd7, 0x9e,
0xd9, 0xcd, 0x49, 0xec, 0x52, 0xa9, 0x33, 0xeb, 0xea, 0x89, 0x4c, 0xec, 0x80, 0xb0, 0x45, 0xcb, 0x60, 0x45, 0x28, 0xa6, 0xa7, 0x9d, 0xcf, 0x6e, 0xbc, 0x7e, 0x7f, 0x73, 0x62, 0x05, 0x07, 0xb3,
0x2c, 0x5a, 0xa2, 0xcc, 0x7b, 0xc7, 0xf8, 0x6d, 0x4f, 0x67, 0xf6, 0x05, 0x8f, 0x5c, 0xf4, 0xfb, 0x21, 0x35, 0x86, 0x2b, 0x1c, 0xf3, 0x55, 0xcb, 0x13, 0x5f, 0x57, 0x24, 0x97, 0x57, 0xd8, 0x4e,
0x80, 0x78, 0xc9, 0xe7, 0x0e, 0xad, 0xd1, 0xc4, 0x37, 0xb8, 0x99, 0xe6, 0xc5, 0xc0, 0x69, 0x54, 0x57, 0x98, 0x20, 0xa7, 0xc3, 0x61, 0x85, 0x0d, 0xaf, 0xfd, 0x27, 0x00, 0x00, 0xff, 0xff, 0x55,
0x49, 0xf1, 0xb5, 0x63, 0xcc, 0x08, 0xc5, 0xf4, 0xb4, 0x73, 0xe6, 0x8d, 0xd7, 0xef, 0x6f, 0x8e, 0x43, 0xa4, 0x8c, 0xaf, 0x30, 0x00, 0x00,
0xac, 0xe0, 0x60, 0xd2, 0xa7, 0xc6, 0x70, 0x85, 0x63, 0xbe, 0x6a, 0x79, 0xe2, 0xeb, 0x8a, 0xe4,
0xf2, 0x0a, 0x5b, 0xe9, 0x0a, 0x13, 0xe4, 0xb8, 0xdf, 0x5f, 0x64, 0xbf, 0xd7, 0xfe, 0x1b, 0x00,
0x00, 0xff, 0xff, 0xcc, 0xd9, 0x54, 0x42, 0x34, 0x32, 0x00, 0x00,
} }
// Reference imports to suppress errors if they are not otherwise used. // Reference imports to suppress errors if they are not otherwise used.
@ -3663,7 +3580,6 @@ type QueryNodeClient interface {
GetTimeTickChannel(ctx context.Context, in *internalpb.GetTimeTickChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) GetTimeTickChannel(ctx context.Context, in *internalpb.GetTimeTickChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error)
GetStatisticsChannel(ctx context.Context, in *internalpb.GetStatisticsChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) GetStatisticsChannel(ctx context.Context, in *internalpb.GetStatisticsChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error)
WatchDmChannels(ctx context.Context, in *WatchDmChannelsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) WatchDmChannels(ctx context.Context, in *WatchDmChannelsRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
WatchDeltaChannels(ctx context.Context, in *WatchDeltaChannelsRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
LoadSegments(ctx context.Context, in *LoadSegmentsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) LoadSegments(ctx context.Context, in *LoadSegmentsRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
ReleaseCollection(ctx context.Context, in *ReleaseCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) ReleaseCollection(ctx context.Context, in *ReleaseCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
ReleasePartitions(ctx context.Context, in *ReleasePartitionsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) ReleasePartitions(ctx context.Context, in *ReleasePartitionsRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
@ -3722,15 +3638,6 @@ func (c *queryNodeClient) WatchDmChannels(ctx context.Context, in *WatchDmChanne
return out, nil return out, nil
} }
func (c *queryNodeClient) WatchDeltaChannels(ctx context.Context, in *WatchDeltaChannelsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
out := new(commonpb.Status)
err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryNode/WatchDeltaChannels", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *queryNodeClient) LoadSegments(ctx context.Context, in *LoadSegmentsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) { func (c *queryNodeClient) LoadSegments(ctx context.Context, in *LoadSegmentsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
out := new(commonpb.Status) out := new(commonpb.Status)
err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryNode/LoadSegments", in, out, opts...) err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryNode/LoadSegments", in, out, opts...)
@ -3836,7 +3743,6 @@ type QueryNodeServer interface {
GetTimeTickChannel(context.Context, *internalpb.GetTimeTickChannelRequest) (*milvuspb.StringResponse, error) GetTimeTickChannel(context.Context, *internalpb.GetTimeTickChannelRequest) (*milvuspb.StringResponse, error)
GetStatisticsChannel(context.Context, *internalpb.GetStatisticsChannelRequest) (*milvuspb.StringResponse, error) GetStatisticsChannel(context.Context, *internalpb.GetStatisticsChannelRequest) (*milvuspb.StringResponse, error)
WatchDmChannels(context.Context, *WatchDmChannelsRequest) (*commonpb.Status, error) WatchDmChannels(context.Context, *WatchDmChannelsRequest) (*commonpb.Status, error)
WatchDeltaChannels(context.Context, *WatchDeltaChannelsRequest) (*commonpb.Status, error)
LoadSegments(context.Context, *LoadSegmentsRequest) (*commonpb.Status, error) LoadSegments(context.Context, *LoadSegmentsRequest) (*commonpb.Status, error)
ReleaseCollection(context.Context, *ReleaseCollectionRequest) (*commonpb.Status, error) ReleaseCollection(context.Context, *ReleaseCollectionRequest) (*commonpb.Status, error)
ReleasePartitions(context.Context, *ReleasePartitionsRequest) (*commonpb.Status, error) ReleasePartitions(context.Context, *ReleasePartitionsRequest) (*commonpb.Status, error)
@ -3867,9 +3773,6 @@ func (*UnimplementedQueryNodeServer) GetStatisticsChannel(ctx context.Context, r
func (*UnimplementedQueryNodeServer) WatchDmChannels(ctx context.Context, req *WatchDmChannelsRequest) (*commonpb.Status, error) { func (*UnimplementedQueryNodeServer) WatchDmChannels(ctx context.Context, req *WatchDmChannelsRequest) (*commonpb.Status, error) {
return nil, status.Errorf(codes.Unimplemented, "method WatchDmChannels not implemented") return nil, status.Errorf(codes.Unimplemented, "method WatchDmChannels not implemented")
} }
func (*UnimplementedQueryNodeServer) WatchDeltaChannels(ctx context.Context, req *WatchDeltaChannelsRequest) (*commonpb.Status, error) {
return nil, status.Errorf(codes.Unimplemented, "method WatchDeltaChannels not implemented")
}
func (*UnimplementedQueryNodeServer) LoadSegments(ctx context.Context, req *LoadSegmentsRequest) (*commonpb.Status, error) { func (*UnimplementedQueryNodeServer) LoadSegments(ctx context.Context, req *LoadSegmentsRequest) (*commonpb.Status, error) {
return nil, status.Errorf(codes.Unimplemented, "method LoadSegments not implemented") return nil, status.Errorf(codes.Unimplemented, "method LoadSegments not implemented")
} }
@ -3980,24 +3883,6 @@ func _QueryNode_WatchDmChannels_Handler(srv interface{}, ctx context.Context, de
return interceptor(ctx, in, info, handler) return interceptor(ctx, in, info, handler)
} }
func _QueryNode_WatchDeltaChannels_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(WatchDeltaChannelsRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(QueryNodeServer).WatchDeltaChannels(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/milvus.proto.query.QueryNode/WatchDeltaChannels",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(QueryNodeServer).WatchDeltaChannels(ctx, req.(*WatchDeltaChannelsRequest))
}
return interceptor(ctx, in, info, handler)
}
func _QueryNode_LoadSegments_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { func _QueryNode_LoadSegments_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(LoadSegmentsRequest) in := new(LoadSegmentsRequest)
if err := dec(in); err != nil { if err := dec(in); err != nil {
@ -4216,10 +4101,6 @@ var _QueryNode_serviceDesc = grpc.ServiceDesc{
MethodName: "WatchDmChannels", MethodName: "WatchDmChannels",
Handler: _QueryNode_WatchDmChannels_Handler, Handler: _QueryNode_WatchDmChannels_Handler,
}, },
{
MethodName: "WatchDeltaChannels",
Handler: _QueryNode_WatchDeltaChannels_Handler,
},
{ {
MethodName: "LoadSegments", MethodName: "LoadSegments",
Handler: _QueryNode_LoadSegments_Handler, Handler: _QueryNode_LoadSegments_Handler,

5
internal/proxy/query_node_mock_test.go
View File

@ -75,11 +75,6 @@ func (m *QueryNodeMock) WatchDmChannels(ctx context.Context, req *querypb.WatchD
return nil, nil return nil, nil
} }
// TODO
func (m *QueryNodeMock) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) {
return nil, nil
}
// TODO // TODO
func (m *QueryNodeMock) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { func (m *QueryNodeMock) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) {
return nil, nil return nil, nil

35
internal/querycoord/cluster.go
View File

@ -58,8 +58,6 @@ type Cluster interface {
// Channel // Channel
WatchDmChannels(ctx context.Context, nodeID int64, in *querypb.WatchDmChannelsRequest) error WatchDmChannels(ctx context.Context, nodeID int64, in *querypb.WatchDmChannelsRequest) error
WatchDeltaChannels(ctx context.Context, nodeID int64, in *querypb.WatchDeltaChannelsRequest) error
HasWatchedDeltaChannel(ctx context.Context, nodeID int64, collectionID UniqueID) bool
// Node // Node
RegisterNode(ctx context.Context, session *sessionutil.Session, id UniqueID, state nodeState) error RegisterNode(ctx context.Context, session *sessionutil.Session, id UniqueID, state nodeState) error
@ -209,9 +207,6 @@ func (c *queryNodeCluster) LoadSegments(ctx context.Context, nodeID int64, in *q
if targetNode != nil { if targetNode != nil {
collectionID := in.CollectionID collectionID := in.CollectionID
// if node has watched the collection's deltaChannel
// then the node should recover part delete log from dmChanel
if c.HasWatchedDeltaChannel(ctx, nodeID, collectionID) {
// get all deltaChannelInfo of the collection from meta // get all deltaChannelInfo of the collection from meta
deltaChannelInfos, err := c.clusterMeta.getDeltaChannelsByCollectionID(collectionID) deltaChannelInfos, err := c.clusterMeta.getDeltaChannelsByCollectionID(collectionID)
if err != nil { if err != nil {
@ -248,9 +243,8 @@ func (c *queryNodeCluster) LoadSegments(ctx context.Context, nodeID int64, in *q
return err return err
} }
} }
}
err := targetNode.loadSegments(ctx, in) err = targetNode.loadSegments(ctx, in)
if err != nil { if err != nil {
log.Warn("loadSegments: queryNode load segments error", zap.Int64("nodeID", nodeID), zap.String("error info", err.Error())) log.Warn("loadSegments: queryNode load segments error", zap.Int64("nodeID", nodeID), zap.String("error info", err.Error()))
return err return err
@ -328,33 +322,6 @@ func (c *queryNodeCluster) WatchDmChannels(ctx context.Context, nodeID int64, in
return fmt.Errorf("watchDmChannels: can't find QueryNode by nodeID, nodeID = %d", nodeID) return fmt.Errorf("watchDmChannels: can't find QueryNode by nodeID, nodeID = %d", nodeID)
} }
func (c *queryNodeCluster) WatchDeltaChannels(ctx context.Context, nodeID int64, in *querypb.WatchDeltaChannelsRequest) error {
c.RLock()
var targetNode Node
if node, ok := c.nodes[nodeID]; ok {
targetNode = node
}
c.RUnlock()
if targetNode != nil {
err := targetNode.watchDeltaChannels(ctx, in)
if err != nil {
return err
}
return nil
}
return fmt.Errorf("watchDeltaChannels: can't find QueryNode by nodeID, nodeID = %d", nodeID)
}
func (c *queryNodeCluster) HasWatchedDeltaChannel(ctx context.Context, nodeID int64, collectionID UniqueID) bool {
c.RLock()
defer c.RUnlock()
return c.nodes[nodeID].hasWatchedDeltaChannel(collectionID)
}
func (c *queryNodeCluster) ReleaseCollection(ctx context.Context, nodeID int64, in *querypb.ReleaseCollectionRequest) error { func (c *queryNodeCluster) ReleaseCollection(ctx context.Context, nodeID int64, in *querypb.ReleaseCollectionRequest) error {
c.RLock() c.RLock()
var targetNode Node var targetNode Node

12
internal/querycoord/cluster_test.go
View File

@ -495,20 +495,10 @@ func TestGrpcRequest(t *testing.T) {
CollectionID: defaultCollectionID, CollectionID: defaultCollectionID,
} }
err := cluster.LoadSegments(baseCtx, nodeID, loadSegmentReq) err := cluster.LoadSegments(baseCtx, nodeID, loadSegmentReq)
assert.Equal(t, 0, len(loadSegmentReq.DeltaPositions)) assert.Equal(t, 1, len(loadSegmentReq.DeltaPositions))
assert.Nil(t, err) assert.Nil(t, err)
}) })
t.Run("Test WatchDeletaChannel", func(t *testing.T) {
watchDeltaChannelReq := &querypb.WatchDeltaChannelsRequest{
CollectionID: defaultCollectionID,
Infos: deltaChannelInfo,
}
err := cluster.WatchDeltaChannels(baseCtx, nodeID, watchDeltaChannelReq)
assert.Nil(t, err)
assert.Equal(t, true, cluster.HasWatchedDeltaChannel(baseCtx, nodeID, defaultCollectionID))
})
t.Run("Test LoadSegmentsAfterWatchDeltaChannel", func(t *testing.T) { t.Run("Test LoadSegmentsAfterWatchDeltaChannel", func(t *testing.T) {
segmentLoadInfo := &querypb.SegmentLoadInfo{ segmentLoadInfo := &querypb.SegmentLoadInfo{
SegmentID: defaultSegmentID, SegmentID: defaultSegmentID,

7
internal/querycoord/mock_querynode_client_test.go
View File

@ -26,7 +26,6 @@ import (
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/milvuspb" "github.com/milvus-io/milvus/internal/proto/milvuspb"
"github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/proto/querypb"
@ -43,7 +42,6 @@ type queryNodeClientMock struct {
} }
func newQueryNodeTest(ctx context.Context, address string, id UniqueID, kv *etcdkv.EtcdKV) (Node, error) { func newQueryNodeTest(ctx context.Context, address string, id UniqueID, kv *etcdkv.EtcdKV) (Node, error) {
watchedDeltaChannels := make(map[UniqueID][]*datapb.VchannelInfo)
childCtx, cancel := context.WithCancel(ctx) childCtx, cancel := context.WithCancel(ctx)
client, err := newQueryNodeClientMock(childCtx, address) client, err := newQueryNodeClientMock(childCtx, address)
if err != nil { if err != nil {
@ -57,7 +55,6 @@ func newQueryNodeTest(ctx context.Context, address string, id UniqueID, kv *etcd
address: address, address: address,
client: client, client: client,
kvClient: kv, kvClient: kv,
watchedDeltaChannels: watchedDeltaChannels,
} }
return node, nil return node, nil
@ -120,10 +117,6 @@ func (client *queryNodeClientMock) WatchDmChannels(ctx context.Context, req *que
return client.grpcClient.WatchDmChannels(ctx, req) return client.grpcClient.WatchDmChannels(ctx, req)
} }
func (client *queryNodeClientMock) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) {
return client.grpcClient.WatchDeltaChannels(ctx, req)
}
func (client *queryNodeClientMock) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { func (client *queryNodeClientMock) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) {
return client.grpcClient.LoadSegments(ctx, req) return client.grpcClient.LoadSegments(ctx, req)
} }

4
internal/querycoord/mock_querynode_server_test.go
View File

@ -207,10 +207,6 @@ func (qs *queryNodeServerMock) WatchDmChannels(ctx context.Context, req *querypb
return qs.watchDmChannels() return qs.watchDmChannels()
} }
func (qs *queryNodeServerMock) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) {
return qs.watchDeltaChannels()
}
func (qs *queryNodeServerMock) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { func (qs *queryNodeServerMock) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) {
sizePerRecord, err := typeutil.EstimateSizePerRecord(req.Schema) sizePerRecord, err := typeutil.EstimateSizePerRecord(req.Schema)
if err != nil { if err != nil {

48
internal/querycoord/querynode.go
View File

@ -29,7 +29,6 @@ import (
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/milvuspb" "github.com/milvus-io/milvus/internal/proto/milvuspb"
"github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/proto/querypb"
@ -47,12 +46,7 @@ type Node interface {
releasePartitions(ctx context.Context, in *querypb.ReleasePartitionsRequest) error releasePartitions(ctx context.Context, in *querypb.ReleasePartitionsRequest) error
watchDmChannels(ctx context.Context, in *querypb.WatchDmChannelsRequest) error watchDmChannels(ctx context.Context, in *querypb.WatchDmChannelsRequest) error
watchDeltaChannels(ctx context.Context, in *querypb.WatchDeltaChannelsRequest) error
syncReplicaSegments(ctx context.Context, in *querypb.SyncReplicaSegmentsRequest) error syncReplicaSegments(ctx context.Context, in *querypb.SyncReplicaSegmentsRequest) error
//removeDmChannel(collectionID UniqueID, channels []string) error
hasWatchedDeltaChannel(collectionID UniqueID) bool
setState(state nodeState) setState(state nodeState)
getState() nodeState getState() nodeState
isOnline() bool isOnline() bool
@ -75,7 +69,6 @@ type queryNode struct {
kvClient *etcdkv.EtcdKV kvClient *etcdkv.EtcdKV
sync.RWMutex sync.RWMutex
watchedDeltaChannels map[UniqueID][]*datapb.VchannelInfo
state nodeState state nodeState
stateLock sync.RWMutex stateLock sync.RWMutex
@ -86,7 +79,6 @@ type queryNode struct {
} }
func newQueryNode(ctx context.Context, address string, id UniqueID, kv *etcdkv.EtcdKV) (Node, error) { func newQueryNode(ctx context.Context, address string, id UniqueID, kv *etcdkv.EtcdKV) (Node, error) {
watchedDeltaChannels := make(map[UniqueID][]*datapb.VchannelInfo)
childCtx, cancel := context.WithCancel(ctx) childCtx, cancel := context.WithCancel(ctx)
client, err := nodeclient.NewClient(childCtx, address) client, err := nodeclient.NewClient(childCtx, address)
if err != nil { if err != nil {
@ -100,7 +92,6 @@ func newQueryNode(ctx context.Context, address string, id UniqueID, kv *etcdkv.E
address: address, address: address,
client: client, client: client,
kvClient: kv, kvClient: kv,
watchedDeltaChannels: watchedDeltaChannels,
state: disConnect, state: disConnect,
} }
@ -127,30 +118,12 @@ func (qn *queryNode) start() error {
} }
func (qn *queryNode) stop() { func (qn *queryNode) stop() {
//qn.stateLock.Lock()
//defer qn.stateLock.Unlock()
//qn.state = offline
if qn.client != nil { if qn.client != nil {
qn.client.Stop() qn.client.Stop()
} }
qn.cancel() qn.cancel()
} }
func (qn *queryNode) hasWatchedDeltaChannel(collectionID UniqueID) bool {
qn.RLock()
defer qn.RUnlock()
_, ok := qn.watchedDeltaChannels[collectionID]
return ok
}
func (qn *queryNode) setDeltaChannelInfo(collectionID int64, infos []*datapb.VchannelInfo) {
qn.Lock()
defer qn.Unlock()
qn.watchedDeltaChannels[collectionID] = infos
}
func (qn *queryNode) setState(state nodeState) { func (qn *queryNode) setState(state nodeState) {
qn.stateLock.Lock() qn.stateLock.Lock()
defer qn.stateLock.Unlock() defer qn.stateLock.Unlock()
@ -196,22 +169,6 @@ func (qn *queryNode) watchDmChannels(ctx context.Context, in *querypb.WatchDmCha
return nil return nil
} }
func (qn *queryNode) watchDeltaChannels(ctx context.Context, in *querypb.WatchDeltaChannelsRequest) error {
if !qn.isOnline() {
return errors.New("WatchDeltaChannels: queryNode is offline")
}
status, err := qn.client.WatchDeltaChannels(qn.ctx, in)
if err != nil {
return err
}
if status.ErrorCode != commonpb.ErrorCode_Success {
return errors.New(status.Reason)
}
qn.setDeltaChannelInfo(in.CollectionID, in.Infos)
return err
}
func (qn *queryNode) releaseCollection(ctx context.Context, in *querypb.ReleaseCollectionRequest) error { func (qn *queryNode) releaseCollection(ctx context.Context, in *querypb.ReleaseCollectionRequest) error {
if !qn.isOnline() { if !qn.isOnline() {
log.Warn("ReleaseCollection: the QueryNode has been offline, the release request is no longer needed", zap.Int64("nodeID", qn.id)) log.Warn("ReleaseCollection: the QueryNode has been offline, the release request is no longer needed", zap.Int64("nodeID", qn.id))
@ -225,11 +182,6 @@ func (qn *queryNode) releaseCollection(ctx context.Context, in *querypb.ReleaseC
if status.ErrorCode != commonpb.ErrorCode_Success { if status.ErrorCode != commonpb.ErrorCode_Success {
return errors.New(status.Reason) return errors.New(status.Reason)
} }
qn.Lock()
delete(qn.watchedDeltaChannels, in.CollectionID)
qn.Unlock()
return nil return nil
} }

93
internal/querycoord/task.go
View File

@ -1359,7 +1359,7 @@ func (lst *loadSegmentTask) execute(ctx context.Context) error {
err := lst.cluster.LoadSegments(ctx, lst.DstNodeID, lst.LoadSegmentsRequest) err := lst.cluster.LoadSegments(ctx, lst.DstNodeID, lst.LoadSegmentsRequest)
if err != nil { if err != nil {
log.Warn("loadSegmentTask: loadSegment occur error", zap.Int64("taskID", lst.getTaskID())) log.Warn("loadSegmentTask: loadSegment occur error", zap.Int64("taskID", lst.getTaskID()), zap.Error(err))
lst.setResultInfo(err) lst.setResultInfo(err)
return err return err
} }
@ -1606,84 +1606,6 @@ func (wdt *watchDmChannelTask) reschedule(ctx context.Context) ([]task, error) {
return reScheduledTasks, nil return reScheduledTasks, nil
} }
type watchDeltaChannelTask struct {
*baseTask
*querypb.WatchDeltaChannelsRequest
cluster Cluster
}
func (wdt *watchDeltaChannelTask) msgBase() *commonpb.MsgBase {
return wdt.Base
}
func (wdt *watchDeltaChannelTask) marshal() ([]byte, error) {
return proto.Marshal(wdt.WatchDeltaChannelsRequest)
}
func (wdt *watchDeltaChannelTask) isValid() bool {
online, err := wdt.cluster.IsOnline(wdt.NodeID)
if err != nil {
return false
}
return wdt.ctx != nil && online
}
func (wdt *watchDeltaChannelTask) msgType() commonpb.MsgType {
return wdt.Base.MsgType
}
func (wdt *watchDeltaChannelTask) timestamp() Timestamp {
return wdt.Base.Timestamp
}
func (wdt *watchDeltaChannelTask) updateTaskProcess() {
parentTask := wdt.getParentTask()
if parentTask == nil {
log.Warn("watchDeltaChannel: parentTask should not be nil")
return
}
parentTask.updateTaskProcess()
}
func (wdt *watchDeltaChannelTask) preExecute(context.Context) error {
channelInfos := wdt.Infos
channels := make([]string, 0)
for _, info := range channelInfos {
channels = append(channels, info.ChannelName)
}
wdt.setResultInfo(nil)
log.Info("start do watchDeltaChannelTask",
zap.Strings("deltaChannels", channels),
zap.Int64("loaded nodeID", wdt.NodeID),
zap.Int64("taskID", wdt.getTaskID()),
zap.Int64("msgID", wdt.GetBase().GetMsgID()))
return nil
}
func (wdt *watchDeltaChannelTask) execute(ctx context.Context) error {
defer wdt.reduceRetryCount()
err := wdt.cluster.WatchDeltaChannels(wdt.ctx, wdt.NodeID, wdt.WatchDeltaChannelsRequest)
if err != nil {
log.Warn("watchDeltaChannelTask: watchDeltaChannel occur error", zap.Int64("taskID", wdt.getTaskID()), zap.Error(err))
wdt.setResultInfo(err)
return err
}
log.Info("watchDeltaChannelsTask Execute done",
zap.Int64("taskID", wdt.getTaskID()),
zap.Int64("msgID", wdt.GetBase().GetMsgID()))
return nil
}
func (wdt *watchDeltaChannelTask) postExecute(context.Context) error {
log.Info("watchDeltaChannelTask postExecute done",
zap.Int64("taskID", wdt.getTaskID()),
zap.Int64("msgID", wdt.GetBase().GetMsgID()))
return nil
}
//****************************handoff task********************************// //****************************handoff task********************************//
type handoffTask struct { type handoffTask struct {
*baseTask *baseTask
@ -2160,8 +2082,7 @@ func (lbt *loadBalanceTask) processNodeDownLoadBalance(ctx context.Context) erro
lbt.addChildTask(internalTask) lbt.addChildTask(internalTask)
log.Info("loadBalanceTask: add a childTask", log.Info("loadBalanceTask: add a childTask",
zap.Int64("taskID", lbt.getTaskID()), zap.Int64("taskID", lbt.getTaskID()),
zap.String("taskType", internalTask.msgType().String()), zap.Any("task", internalTask))
zap.Int64("destNode", getDstNodeIDByTask(internalTask)))
} }
log.Info("loadBalanceTask: assign child task done", zap.Int64("taskID", lbt.getTaskID()), zap.Int64s("sourceNodeIDs", lbt.SourceNodeIDs)) log.Info("loadBalanceTask: assign child task done", zap.Int64("taskID", lbt.getTaskID()), zap.Int64s("sourceNodeIDs", lbt.SourceNodeIDs))
@ -2630,13 +2551,13 @@ func mergeWatchDeltaChannelInfo(infos []*datapb.VchannelInfo) []*datapb.Vchannel
} }
} }
var result []*datapb.VchannelInfo var result []*datapb.VchannelInfo
for _, index := range minPositions { for channel, index := range minPositions {
result = append(result, infos[index]) result = append(result, infos[index])
} log.Info("merge delta channels finished", zap.String("channel", channel),
log.Info("merge delta channels finished", zap.Any("merged info", infos[index]),
zap.Any("origin info length", len(infos)),
zap.Any("merged info length", len(result)),
) )
}
return result return result
} }

81
internal/querycoord/task_scheduler.go
View File

@ -406,19 +406,7 @@ func (scheduler *TaskScheduler) unmarshalTask(taskID UniqueID, t string) (task,
} }
newTask = watchDmChannelTask newTask = watchDmChannelTask
case commonpb.MsgType_WatchDeltaChannels: case commonpb.MsgType_WatchDeltaChannels:
//TODO::trigger condition may be different log.Warn("legacy WatchDeltaChannels type found, ignore")
loadReq := querypb.WatchDeltaChannelsRequest{}
err = proto.Unmarshal([]byte(t), &loadReq)
reviseWatchDeltaChannelsRequest(&loadReq)
if err != nil {
return nil, err
}
watchDeltaChannelTask := &watchDeltaChannelTask{
baseTask: baseTask,
WatchDeltaChannelsRequest: &loadReq,
cluster: scheduler.cluster,
}
newTask = watchDeltaChannelTask
case commonpb.MsgType_WatchQueryChannels: case commonpb.MsgType_WatchQueryChannels:
//Deprecated WatchQueryChannel //Deprecated WatchQueryChannel
log.Warn("legacy WatchQueryChannels type found, ignore") log.Warn("legacy WatchQueryChannels type found, ignore")
@ -524,8 +512,6 @@ func (scheduler *TaskScheduler) processTask(t task) error {
protoSize = proto.Size(childTask.(*loadSegmentTask).LoadSegmentsRequest) protoSize = proto.Size(childTask.(*loadSegmentTask).LoadSegmentsRequest)
case commonpb.MsgType_WatchDmChannels: case commonpb.MsgType_WatchDmChannels:
protoSize = proto.Size(childTask.(*watchDmChannelTask).WatchDmChannelsRequest) protoSize = proto.Size(childTask.(*watchDmChannelTask).WatchDmChannelsRequest)
case commonpb.MsgType_WatchDeltaChannels:
protoSize = proto.Size(childTask.(*watchDeltaChannelTask).WatchDeltaChannelsRequest)
default: default:
//TODO:: //TODO::
} }
@ -728,17 +714,6 @@ func (scheduler *TaskScheduler) scheduleLoop() {
if len(childTasks) != 0 { if len(childTasks) != 0 {
// include loadSegment, watchDmChannel, releaseCollection, releasePartition, releaseSegment // include loadSegment, watchDmChannel, releaseCollection, releasePartition, releaseSegment
processInternalTaskFn(childTasks, triggerTask) processInternalTaskFn(childTasks, triggerTask)
if triggerTask.getResultInfo().ErrorCode == commonpb.ErrorCode_Success {
// derivedInternalTasks include watchDeltaChannel, watchQueryChannel
// derivedInternalTasks generate from loadSegment and watchDmChannel reqs
derivedInternalTasks, err := generateDerivedInternalTasks(triggerTask, scheduler.meta, scheduler.cluster)
if err != nil {
log.Error("scheduleLoop: generate derived watchDeltaChannel and watchcQueryChannel tasks failed", zap.Int64("triggerTaskID", triggerTask.getTaskID()), zap.Error(err))
triggerTask.setResultInfo(err)
} else {
processInternalTaskFn(derivedInternalTasks, triggerTask)
}
}
} }
// triggerTask may be LoadCollection, LoadPartitions, LoadBalance, Handoff // triggerTask may be LoadCollection, LoadPartitions, LoadBalance, Handoff
@ -1115,57 +1090,3 @@ func updateSegmentInfoFromTask(ctx context.Context, triggerTask task, meta Meta)
return nil return nil
} }
// generateDerivedInternalTasks generate watchDeltaChannel and watchQueryChannel tasks
func generateDerivedInternalTasks(triggerTask task, meta Meta, cluster Cluster) ([]task, error) {
var derivedInternalTasks []task
watchDeltaChannelInfo := make(map[int64]map[UniqueID]UniqueID)
addChannelWatchInfoFn := func(nodeID int64, collectionID UniqueID, replicaID UniqueID, watchInfo map[int64]map[UniqueID]UniqueID) {
if _, ok := watchInfo[nodeID]; !ok {
watchInfo[nodeID] = make(map[UniqueID]UniqueID)
}
watchInfo[nodeID][collectionID] = replicaID
}
for _, childTask := range triggerTask.getChildTask() {
if childTask.msgType() == commonpb.MsgType_LoadSegments {
loadSegmentTask := childTask.(*loadSegmentTask)
collectionID := loadSegmentTask.CollectionID
replicaID := loadSegmentTask.GetReplicaID()
nodeID := loadSegmentTask.DstNodeID
if !cluster.HasWatchedDeltaChannel(triggerTask.traceCtx(), nodeID, collectionID) {
addChannelWatchInfoFn(nodeID, collectionID, replicaID, watchDeltaChannelInfo)
}
}
}
for nodeID, collectionIDs := range watchDeltaChannelInfo {
for collectionID, replicaID := range collectionIDs {
deltaChannelInfo, err := meta.getDeltaChannelsByCollectionID(collectionID)
if err != nil {
return nil, err
}
msgBase := proto.Clone(triggerTask.msgBase()).(*commonpb.MsgBase)
msgBase.MsgType = commonpb.MsgType_WatchDeltaChannels
watchDeltaRequest := &querypb.WatchDeltaChannelsRequest{
Base: msgBase,
CollectionID: collectionID,
Infos: deltaChannelInfo,
ReplicaId: replicaID,
}
watchDeltaRequest.NodeID = nodeID
baseTask := newBaseTask(triggerTask.traceCtx(), triggerTask.getTriggerCondition())
baseTask.setParentTask(triggerTask)
watchDeltaTask := &watchDeltaChannelTask{
baseTask: baseTask,
WatchDeltaChannelsRequest: watchDeltaRequest,
cluster: cluster,
}
derivedInternalTasks = append(derivedInternalTasks, watchDeltaTask)
}
}
return derivedInternalTasks, nil
}

64
internal/querycoord/task_scheduler_test.go
View File

@ -365,27 +365,6 @@ func TestUnMarshalTask(t *testing.T) {
assert.Nil(t, task2) assert.Nil(t, task2)
}) })
t.Run("Test watchDeltaChannelTask", func(t *testing.T) {
watchTask := &watchDeltaChannelTask{
WatchDeltaChannelsRequest: &querypb.WatchDeltaChannelsRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_WatchDeltaChannels,
},
},
}
blobs, err := watchTask.marshal()
assert.Nil(t, err)
err = kv.Save("testMarshalWatchDeltaChannel", string(blobs))
assert.Nil(t, err)
defer kv.RemoveWithPrefix("testMarshalWatchDeltaChannel")
value, err := kv.Load("testMarshalWatchDeltaChannel")
assert.Nil(t, err)
task, err := taskScheduler.unmarshalTask(1007, value)
assert.Nil(t, err)
assert.Equal(t, task.msgType(), commonpb.MsgType_WatchDeltaChannels)
})
t.Run("Test loadBalanceTask", func(t *testing.T) { t.Run("Test loadBalanceTask", func(t *testing.T) {
loadBalanceTask := &loadBalanceTask{ loadBalanceTask := &loadBalanceTask{
LoadBalanceRequest: &querypb.LoadBalanceRequest{ LoadBalanceRequest: &querypb.LoadBalanceRequest{
@ -525,49 +504,6 @@ func Test_saveInternalTaskToEtcd(t *testing.T) {
}) })
} }
func Test_generateDerivedInternalTasks(t *testing.T) {
refreshParams()
baseCtx := context.Background()
queryCoord, err := startQueryCoord(baseCtx)
assert.Nil(t, err)
node1, err := startQueryNodeServer(baseCtx)
assert.Nil(t, err)
waitQueryNodeOnline(queryCoord.cluster, node1.queryNodeID)
vChannelInfos, _, err := queryCoord.broker.getRecoveryInfo(baseCtx, defaultCollectionID, defaultPartitionID)
assert.NoError(t, err)
deltaChannelInfos := make([]*datapb.VchannelInfo, len(vChannelInfos))
for i, info := range vChannelInfos {
deltaInfo, err := generateWatchDeltaChannelInfo(info)
assert.NoError(t, err)
deltaChannelInfos[i] = deltaInfo
}
queryCoord.meta.setDeltaChannel(defaultCollectionID, deltaChannelInfos)
loadCollectionTask := genLoadCollectionTask(baseCtx, queryCoord)
loadSegmentTask := genLoadSegmentTask(baseCtx, queryCoord, node1.queryNodeID)
loadCollectionTask.addChildTask(loadSegmentTask)
loadSegmentTask.setParentTask(loadCollectionTask)
watchDmChannelTask := genWatchDmChannelTask(baseCtx, queryCoord, node1.queryNodeID)
loadCollectionTask.addChildTask(watchDmChannelTask)
watchDmChannelTask.setParentTask(loadCollectionTask)
derivedTasks, err := generateDerivedInternalTasks(loadCollectionTask, queryCoord.meta, queryCoord.cluster)
assert.Nil(t, err)
assert.Equal(t, 1, len(derivedTasks))
for _, internalTask := range derivedTasks {
matchType := internalTask.msgType() == commonpb.MsgType_WatchDeltaChannels
assert.Equal(t, true, matchType)
if internalTask.msgType() == commonpb.MsgType_WatchDeltaChannels {
assert.Equal(t, node1.queryNodeID, internalTask.(*watchDeltaChannelTask).NodeID)
}
}
queryCoord.Stop()
err = removeAllSession()
assert.Nil(t, err)
}
func TestTaskScheduler_BindContext(t *testing.T) { func TestTaskScheduler_BindContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
s := &TaskScheduler{ s := &TaskScheduler{

32
internal/querycoord/task_test.go
View File

@ -190,12 +190,14 @@ func genWatchDmChannelTask(ctx context.Context, queryCoord *QueryCoord, nodeID i
queryCoord.meta.addCollection(defaultCollectionID, querypb.LoadType_LoadCollection, schema) queryCoord.meta.addCollection(defaultCollectionID, querypb.LoadType_LoadCollection, schema)
return watchDmChannelTask return watchDmChannelTask
} }
func genLoadSegmentTask(ctx context.Context, queryCoord *QueryCoord, nodeID int64) *loadSegmentTask { func genLoadSegmentTask(ctx context.Context, queryCoord *QueryCoord, nodeID int64) *loadSegmentTask {
schema := genDefaultCollectionSchema(false) schema := genDefaultCollectionSchema(false)
segmentInfo := &querypb.SegmentLoadInfo{ segmentInfo := &querypb.SegmentLoadInfo{
SegmentID: defaultSegmentID, SegmentID: defaultSegmentID,
PartitionID: defaultPartitionID, PartitionID: defaultPartitionID,
CollectionID: defaultCollectionID, CollectionID: defaultCollectionID,
InsertChannel: "by-dev-rootcoord-dml_1_2021v1",
} }
req := &querypb.LoadSegmentsRequest{ req := &querypb.LoadSegmentsRequest{
Base: &commonpb.MsgBase{ Base: &commonpb.MsgBase{
@ -246,29 +248,18 @@ func genLoadSegmentTask(ctx context.Context, queryCoord *QueryCoord, nodeID int6
loadSegmentTask.setParentTask(parentTask) loadSegmentTask.setParentTask(parentTask)
queryCoord.meta.addCollection(defaultCollectionID, querypb.LoadType_LoadCollection, schema) queryCoord.meta.addCollection(defaultCollectionID, querypb.LoadType_LoadCollection, schema)
return loadSegmentTask
}
func genWatchDeltaChannelTask(ctx context.Context, queryCoord *QueryCoord, nodeID int64) *watchDeltaChannelTask { deltaChannelInfo := []*datapb.VchannelInfo{
req := &querypb.WatchDeltaChannelsRequest{ {
Base: &commonpb.MsgBase{ CollectionID: defaultCollectionID,
MsgType: commonpb.MsgType_WatchDeltaChannels, ChannelName: "by-dev-rootcoord-delta_1_2021v1",
SeekPosition: &internalpb.MsgPosition{
ChannelName: "by-dev-rootcoord-dml_1",
}, },
NodeID: nodeID,
CollectionID: defaultCollectionID,
LoadMeta: &querypb.LoadMetaInfo{
LoadType: querypb.LoadType_LoadCollection,
CollectionID: defaultCollectionID,
PartitionIDs: []int64{defaultPartitionID},
}, },
} }
baseTask := newBaseTask(ctx, querypb.TriggerCondition_GrpcRequest) queryCoord.meta.setDeltaChannel(defaultCollectionID, deltaChannelInfo)
baseTask.taskID = 300 return loadSegmentTask
return &watchDeltaChannelTask{
baseTask: baseTask,
WatchDeltaChannelsRequest: req,
cluster: queryCoord.cluster,
}
} }
func waitTaskFinalState(t task, state taskState) { func waitTaskFinalState(t task, state taskState) {
@ -1399,9 +1390,6 @@ func TestUpdateTaskProcessWhenLoadSegment(t *testing.T) {
assert.Nil(t, err) assert.Nil(t, err)
assert.Equal(t, int64(0), collectionInfo.InMemoryPercentage) assert.Equal(t, int64(0), collectionInfo.InMemoryPercentage)
watchDeltaChannel := genWatchDeltaChannelTask(ctx, queryCoord, node1.queryNodeID)
watchDeltaChannel.setParentTask(loadCollectionTask)
queryCoord.scheduler.processTask(watchDeltaChannel)
collectionInfo, err = queryCoord.meta.getCollectionInfoByID(defaultCollectionID) collectionInfo, err = queryCoord.meta.getCollectionInfoByID(defaultCollectionID)
assert.Nil(t, err) assert.Nil(t, err)
assert.Equal(t, int64(0), collectionInfo.InMemoryPercentage) assert.Equal(t, int64(0), collectionInfo.InMemoryPercentage)

7
internal/querycoord/task_util.go
View File

@ -63,13 +63,6 @@ func thinWatchDmChannelsRequest(request *querypb.WatchDmChannelsRequest) *queryp
return cloned return cloned
} }
// reviseWatchDeltaChannelsRequest will revise the WatchDeltaChannelsRequest for upgrade compatibility from 2.0.2
func reviseWatchDeltaChannelsRequest(req *querypb.WatchDeltaChannelsRequest) {
for _, vChannel := range req.GetInfos() {
reviseVChannelInfo(vChannel)
}
}
// reviseVChannelInfo will revise the datapb.VchannelInfo for upgrade compatibility from 2.0.2 // reviseVChannelInfo will revise the datapb.VchannelInfo for upgrade compatibility from 2.0.2
func reviseVChannelInfo(vChannel *datapb.VchannelInfo) { func reviseVChannelInfo(vChannel *datapb.VchannelInfo) {
removeDuplicateSegmentIDFn := func(ids []int64) []int64 { removeDuplicateSegmentIDFn := func(ids []int64) []int64 {

29
internal/querycoord/util.go
View File

@ -21,7 +21,6 @@ import (
"sort" "sort"
"github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/milvuspb" "github.com/milvus-io/milvus/internal/proto/milvuspb"
"github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/proto/querypb"
@ -81,34 +80,6 @@ func getFieldSizeFromFieldBinlog(fieldBinlog *datapb.FieldBinlog) int64 {
} }
func getDstNodeIDByTask(t task) int64 {
var nodeID int64
switch t.msgType() {
case commonpb.MsgType_LoadSegments:
loadSegment := t.(*loadSegmentTask)
nodeID = loadSegment.DstNodeID
case commonpb.MsgType_WatchDmChannels:
watchDmChannel := t.(*watchDmChannelTask)
nodeID = watchDmChannel.NodeID
case commonpb.MsgType_WatchDeltaChannels:
watchDeltaChannel := t.(*watchDeltaChannelTask)
nodeID = watchDeltaChannel.NodeID
case commonpb.MsgType_ReleaseCollection:
releaseCollection := t.(*releaseCollectionTask)
nodeID = releaseCollection.NodeID
case commonpb.MsgType_ReleasePartitions:
releasePartition := t.(*releasePartitionTask)
nodeID = releasePartition.NodeID
case commonpb.MsgType_ReleaseSegments:
releaseSegment := t.(*releaseSegmentTask)
nodeID = releaseSegment.NodeID
default:
//TODO::
}
return nodeID
}
// syncReplicaSegments syncs the segments distribution of replica to shard leaders // syncReplicaSegments syncs the segments distribution of replica to shard leaders
// only syncs the segments in shards if not nil // only syncs the segments in shards if not nil
func syncReplicaSegments(ctx context.Context, meta Meta, cluster Cluster, replicaID UniqueID, shards ...string) error { func syncReplicaSegments(ctx context.Context, meta Meta, cluster Cluster, replicaID UniqueID, shards ...string) error {

52
internal/querynode/impl.go
View File

@ -325,58 +325,6 @@ func (node *QueryNode) WatchDmChannels(ctx context.Context, in *queryPb.WatchDmC
return waitFunc() return waitFunc()
} }
// WatchDeltaChannels create consumers on dmChannels to receive Incremental data，which is the important part of real-time query
func (node *QueryNode) WatchDeltaChannels(ctx context.Context, in *queryPb.WatchDeltaChannelsRequest) (*commonpb.Status, error) {
code := node.stateCode.Load().(internalpb.StateCode)
if code != internalpb.StateCode_Healthy {
err := fmt.Errorf("query node %d is not ready", Params.QueryNodeCfg.GetNodeID())
status := &commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError,
Reason: err.Error(),
}
return status, nil
}
dct := &watchDeltaChannelsTask{
baseTask: baseTask{
ctx: ctx,
done: make(chan error),
},
req: in,
node: node,
}
err := node.scheduler.queue.Enqueue(dct)
if err != nil {
status := &commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError,
Reason: err.Error(),
}
log.Warn(err.Error())
return status, nil
}
log.Info("watchDeltaChannelsTask Enqueue done", zap.Int64("collectionID", in.CollectionID), zap.Int64("nodeID", Params.QueryNodeCfg.GetNodeID()))
waitFunc := func() (*commonpb.Status, error) {
err = dct.WaitToFinish()
if err != nil {
status := &commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError,
Reason: err.Error(),
}
log.Warn(err.Error())
return status, nil
}
log.Info("watchDeltaChannelsTask WaitToFinish done", zap.Int64("collectionID", in.CollectionID), zap.Int64("nodeID", Params.QueryNodeCfg.GetNodeID()))
return &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
}, nil
}
return waitFunc()
}
// LoadSegments load historical data into query node, historical data can be vector data or index // LoadSegments load historical data into query node, historical data can be vector data or index
func (node *QueryNode) LoadSegments(ctx context.Context, in *queryPb.LoadSegmentsRequest) (*commonpb.Status, error) { func (node *QueryNode) LoadSegments(ctx context.Context, in *queryPb.LoadSegmentsRequest) (*commonpb.Status, error) {
code := node.stateCode.Load().(internalpb.StateCode) code := node.stateCode.Load().(internalpb.StateCode)

2
internal/querynode/query_shard_service.go
View File

@ -75,7 +75,7 @@ func (q *queryShardService) addQueryShard(collectionID UniqueID, channel Channel
q.queryShardsMu.Lock() q.queryShardsMu.Lock()
defer q.queryShardsMu.Unlock() defer q.queryShardsMu.Unlock()
if _, ok := q.queryShards[channel]; ok { if _, ok := q.queryShards[channel]; ok {
return errors.New(fmt.Sprintln("query shard(channel) ", channel, " already exists")) return nil
} }
qs, err := newQueryShard( qs, err := newQueryShard(
q.ctx, q.ctx,

54
internal/querynode/segment_loader.go
View File

@ -24,7 +24,6 @@ import (
"runtime" "runtime"
"runtime/debug" "runtime/debug"
"strconv" "strconv"
"sync"
"github.com/panjf2000/ants/v2" "github.com/panjf2000/ants/v2"
"go.uber.org/zap" "go.uber.org/zap"
@ -84,6 +83,7 @@ func (loader *segmentLoader) LoadSegment(req *querypb.LoadSegmentsRequest, segme
segmentNum := len(req.Infos) segmentNum := len(req.Infos)
if segmentNum == 0 { if segmentNum == 0 {
log.Warn("find no valid segment target, skip load segment", zap.Any("request", req))
return nil return nil
} }
@ -156,7 +156,7 @@ func (loader *segmentLoader) LoadSegment(req *querypb.LoadSegmentsRequest, segme
newSegments[segmentID] = segment newSegments[segmentID] = segment
} }
loadSegmentFunc := func(idx int) error { loadFileFunc := func(idx int) error {
loadInfo := req.Infos[idx] loadInfo := req.Infos[idx]
collectionID := loadInfo.CollectionID collectionID := loadInfo.CollectionID
partitionID := loadInfo.PartitionID partitionID := loadInfo.PartitionID
@ -164,7 +164,7 @@ func (loader *segmentLoader) LoadSegment(req *querypb.LoadSegmentsRequest, segme
segment := newSegments[segmentID] segment := newSegments[segmentID]
tr := timerecord.NewTimeRecorder("loadDurationPerSegment") tr := timerecord.NewTimeRecorder("loadDurationPerSegment")
err := loader.loadSegmentInternal(segment, loadInfo) err := loader.loadFiles(segment, loadInfo)
if err != nil { if err != nil {
log.Error("load segment failed when load data into memory", log.Error("load segment failed when load data into memory",
zap.Int64("collectionID", collectionID), zap.Int64("collectionID", collectionID),
@ -186,8 +186,7 @@ func (loader *segmentLoader) LoadSegment(req *querypb.LoadSegmentsRequest, segme
zap.Int("segmentNum", segmentNum), zap.Int("segmentNum", segmentNum),
zap.Int("concurrencyLevel", concurrencyLevel)) zap.Int("concurrencyLevel", concurrencyLevel))
err = funcutil.ProcessFuncParallel(segmentNum, err = funcutil.ProcessFuncParallel(segmentNum,
concurrencyLevel, concurrencyLevel, loadFileFunc, "loadSegmentFunc")
loadSegmentFunc, "loadSegmentFunc")
if err != nil { if err != nil {
segmentGC() segmentGC()
return err return err
@ -211,7 +210,7 @@ func (loader *segmentLoader) LoadSegment(req *querypb.LoadSegmentsRequest, segme
return nil return nil
} }
func (loader *segmentLoader) loadSegmentInternal(segment *Segment, func (loader *segmentLoader) loadFiles(segment *Segment,
loadInfo *querypb.SegmentLoadInfo) error { loadInfo *querypb.SegmentLoadInfo) error {
collectionID := loadInfo.CollectionID collectionID := loadInfo.CollectionID
partitionID := loadInfo.PartitionID partitionID := loadInfo.PartitionID
@ -643,7 +642,7 @@ func (loader *segmentLoader) loadDeltaLogs(segment *Segment, deltaLogs []*datapb
} }
func (loader *segmentLoader) FromDmlCPLoadDelete(ctx context.Context, collectionID int64, position *internalpb.MsgPosition) error { func (loader *segmentLoader) FromDmlCPLoadDelete(ctx context.Context, collectionID int64, position *internalpb.MsgPosition) error {
log.Info("from dml check point load delete", zap.Any("position", position), zap.Any("msg id", position.MsgID)) log.Info("from dml check point load delete", zap.Any("position", position))
stream, err := loader.factory.NewMsgStream(ctx) stream, err := loader.factory.NewMsgStream(ctx)
if err != nil { if err != nil {
return err return err
@ -682,7 +681,7 @@ func (loader *segmentLoader) FromDmlCPLoadDelete(ctx context.Context, collection
} }
log.Info("start read delta msg from seek position to last position", log.Info("start read delta msg from seek position to last position",
zap.Int64("Collection ID", collectionID), zap.String("channel", pChannelName)) zap.Int64("Collection ID", collectionID), zap.String("channel", pChannelName), zap.Any("seek pos", position), zap.Any("last msg", lastMsgID))
hasMore := true hasMore := true
for hasMore { for hasMore {
select { select {
@ -744,43 +743,18 @@ func (loader *segmentLoader) FromDmlCPLoadDelete(ctx context.Context, collection
} }
offset := segment.segmentPreDelete(len(pks)) offset := segment.segmentPreDelete(len(pks))
delData.deleteOffset[segmentID] = offset delData.deleteOffset[segmentID] = offset
timestamps := delData.deleteTimestamps[segmentID]
err = segment.segmentDelete(offset, pks, timestamps)
if err != nil {
log.Warn("QueryNode: segment delete failed", zap.Int64("segment", segmentID), zap.Error(err))
return err
}
} }
wg := sync.WaitGroup{}
for segmentID := range delData.deleteOffset {
wg.Add(1)
go deletePk(loader.metaReplica, delData, segmentID, &wg)
}
wg.Wait()
log.Info("from dml check point load done", zap.Any("msg id", position.GetMsgID())) log.Info("from dml check point load done", zap.Any("msg id", position.GetMsgID()))
return nil return nil
} }
func deletePk(replica ReplicaInterface, deleteData *deleteData, segmentID UniqueID, wg *sync.WaitGroup) {
defer wg.Done()
log.Debug("QueryNode::iNode::delete", zap.Any("SegmentID", segmentID))
targetSegment, err := replica.getSegmentByID(segmentID, segmentTypeSealed)
if err != nil {
log.Error(err.Error())
return
}
if targetSegment.segmentType != segmentTypeSealed {
return
}
ids := deleteData.deleteIDs[segmentID]
timestamps := deleteData.deleteTimestamps[segmentID]
offset := deleteData.deleteOffset[segmentID]
err = targetSegment.segmentDelete(offset, ids, timestamps)
if err != nil {
log.Warn("QueryNode: targetSegmentDelete failed", zap.Error(err))
return
}
log.Debug("Do delete done", zap.Int("len", len(deleteData.deleteIDs[segmentID])), zap.Int64("segmentID", segmentID), zap.Any("segmentType", targetSegment.segmentType))
}
// JoinIDPath joins ids to path format. // JoinIDPath joins ids to path format.
func JoinIDPath(ids ...UniqueID) string { func JoinIDPath(ids ...UniqueID) string {
idStr := make([]string, 0, len(ids)) idStr := make([]string, 0, len(ids))
@ -819,6 +793,8 @@ func (loader *segmentLoader) checkSegmentSize(collectionID UniqueID, segmentLoad
loadingUsage := usedMemAfterLoad + uint64( loadingUsage := usedMemAfterLoad + uint64(
float64(maxSegmentSize)*float64(concurrency)*Params.QueryNodeCfg.LoadMemoryUsageFactor) float64(maxSegmentSize)*float64(concurrency)*Params.QueryNodeCfg.LoadMemoryUsageFactor)
log.Debug("predict memory usage while loading (in MiB)", log.Debug("predict memory usage while loading (in MiB)",
zap.Int64("collectionID", collectionID),
zap.Int("concurrency", concurrency),
zap.Uint64("usage", toMB(loadingUsage)), zap.Uint64("usage", toMB(loadingUsage)),
zap.Uint64("usageAfterLoad", toMB(usedMemAfterLoad))) zap.Uint64("usageAfterLoad", toMB(usedMemAfterLoad)))

2
internal/querynode/shard_cluster.go
View File

@ -907,7 +907,7 @@ func (sc *ShardCluster) Search(ctx context.Context, req *querypb.SearchRequest,
wg.Wait() wg.Wait()
if err != nil { if err != nil {
log.Error(err.Error()) log.Error("failed to do search", zap.Any("req", req), zap.Error(err))
return nil, err return nil, err
} }

129
internal/querynode/task.go
View File

@ -31,6 +31,7 @@ import (
queryPb "github.com/milvus-io/milvus/internal/proto/querypb" queryPb "github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/util/funcutil" "github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus/internal/util/typeutil" "github.com/milvus-io/milvus/internal/util/typeutil"
"golang.org/x/sync/errgroup"
) )
type task interface { type task interface {
@ -90,12 +91,6 @@ type watchDmChannelsTask struct {
node *QueryNode node *QueryNode
} }
type watchDeltaChannelsTask struct {
baseTask
req *queryPb.WatchDeltaChannelsRequest
node *QueryNode
}
type loadSegmentsTask struct { type loadSegmentsTask struct {
baseTask baseTask
req *queryPb.LoadSegmentsRequest req *queryPb.LoadSegmentsRequest
@ -386,10 +381,8 @@ func (w *watchDmChannelsTask) Execute(ctx context.Context) (err error) {
// add tsafe watch in query shard if exists // add tsafe watch in query shard if exists
for _, dmlChannel := range vChannels { for _, dmlChannel := range vChannels {
if !w.node.queryShardService.hasQueryShard(dmlChannel) {
w.node.queryShardService.addQueryShard(collectionID, dmlChannel, w.req.GetReplicaID()) w.node.queryShardService.addQueryShard(collectionID, dmlChannel, w.req.GetReplicaID())
} }
}
// start flow graphs // start flow graphs
for _, fg := range channel2FlowGraph { for _, fg := range channel2FlowGraph {
@ -400,63 +393,50 @@ func (w *watchDmChannelsTask) Execute(ctx context.Context) (err error) {
return nil return nil
} }
// watchDeltaChannelsTask // internal helper function to subscribe delta channel
func (w *watchDeltaChannelsTask) Execute(ctx context.Context) error { func (l *loadSegmentsTask) watchDeltaChannel(vchanName []string) error {
collectionID := w.req.CollectionID collectionID := l.req.CollectionID
var vDeltaChannels, pDeltaChannels []string
// get all vChannels VPDeltaChannels := make(map[string]string)
vDeltaChannels := make([]Channel, 0) for _, v := range vchanName {
pDeltaChannels := make([]Channel, 0) dc, err := funcutil.ConvertChannelName(v, Params.CommonCfg.RootCoordDml, Params.CommonCfg.RootCoordDelta)
VPDeltaChannels := make(map[string]string) // map[vChannel]pChannel if err != nil {
vChannel2SeekPosition := make(map[string]*internalpb.MsgPosition) log.Warn("watchDeltaChannels, failed to convert deltaChannel from dmlChannel", zap.String("DmlChannel", v), zap.Error(err))
for _, info := range w.req.Infos { return err
v := info.ChannelName }
p := funcutil.ToPhysicalChannel(info.ChannelName) p := funcutil.ToPhysicalChannel(dc)
vDeltaChannels = append(vDeltaChannels, v) vDeltaChannels = append(vDeltaChannels, dc)
pDeltaChannels = append(pDeltaChannels, p) pDeltaChannels = append(pDeltaChannels, p)
VPDeltaChannels[v] = p VPDeltaChannels[dc] = p
vChannel2SeekPosition[v] = info.SeekPosition
} }
log.Info("Starting WatchDeltaChannels ...", log.Info("Starting WatchDeltaChannels ...",
zap.Any("collectionID", collectionID), zap.Int64("collectionID", collectionID),
zap.Any("vDeltaChannels", vDeltaChannels), zap.Any("channels", VPDeltaChannels),
zap.Any("pChannels", pDeltaChannels),
)
if len(VPDeltaChannels) != len(vDeltaChannels) {
return errors.New("get physical channels failed, illegal channel length, collectionID = " + fmt.Sprintln(collectionID))
}
log.Info("Get physical channels done",
zap.Any("collectionID", collectionID),
) )
if hasColl := w.node.metaReplica.hasCollection(collectionID); !hasColl { coll, err := l.node.metaReplica.getCollectionByID(collectionID)
return fmt.Errorf("cannot find collection with collectionID, %d", collectionID)
}
coll, err := w.node.metaReplica.getCollectionByID(collectionID)
if err != nil { if err != nil {
return err return err
} }
channel2FlowGraph, err := w.node.dataSyncService.addFlowGraphsForDeltaChannels(collectionID, vDeltaChannels) channel2FlowGraph, err := l.node.dataSyncService.addFlowGraphsForDeltaChannels(collectionID, vDeltaChannels)
if err != nil { if err != nil {
log.Warn("watchDeltaChannel, add flowGraph for deltaChannel failed", zap.Int64("collectionID", collectionID), zap.Strings("vDeltaChannels", vDeltaChannels), zap.Error(err)) log.Warn("watchDeltaChannel, add flowGraph for deltaChannel failed", zap.Int64("collectionID", collectionID), zap.Strings("vDeltaChannels", vDeltaChannels), zap.Error(err))
return err return err
} }
consumeSubName := funcutil.GenChannelSubName(Params.CommonCfg.QueryNodeSubName, collectionID, Params.QueryNodeCfg.GetNodeID()) consumeSubName := funcutil.GenChannelSubName(Params.CommonCfg.QueryNodeSubName, collectionID, Params.QueryNodeCfg.GetNodeID())
// channels as consumer // channels as consumer
for channel, fg := range channel2FlowGraph { for channel, fg := range channel2FlowGraph {
pchannel := VPDeltaChannels[channel]
// use pChannel to consume // use pChannel to consume
err = fg.consumeFlowGraphFromLatest(VPDeltaChannels[channel], consumeSubName) err = fg.consumeFlowGraphFromLatest(pchannel, consumeSubName)
if err != nil { if err != nil {
log.Error("msgStream as consumer failed for deltaChannels", zap.Int64("collectionID", collectionID), zap.Strings("vDeltaChannels", vDeltaChannels)) log.Error("msgStream as consumer failed for deltaChannels", zap.Int64("collectionID", collectionID), zap.Strings("vDeltaChannels", vDeltaChannels))
break break
} }
err = w.node.loader.FromDmlCPLoadDelete(w.ctx, collectionID, vChannel2SeekPosition[channel])
if err != nil {
log.Error("watchDeltaChannelsTask from dml cp load delete failed", zap.Int64("collectionID", collectionID), zap.Strings("vDeltaChannels", vDeltaChannels))
break
}
} }
if err != nil { if err != nil {
log.Warn("watchDeltaChannel, add flowGraph for deltaChannel failed", zap.Int64("collectionID", collectionID), zap.Strings("vDeltaChannels", vDeltaChannels), zap.Error(err)) log.Warn("watchDeltaChannel, add flowGraph for deltaChannel failed", zap.Int64("collectionID", collectionID), zap.Strings("vDeltaChannels", vDeltaChannels), zap.Error(err))
for _, fg := range channel2FlowGraph { for _, fg := range channel2FlowGraph {
@ -466,7 +446,7 @@ func (w *watchDeltaChannelsTask) Execute(ctx context.Context) error {
for channel := range channel2FlowGraph { for channel := range channel2FlowGraph {
gcChannels = append(gcChannels, channel) gcChannels = append(gcChannels, channel)
} }
w.node.dataSyncService.removeFlowGraphsByDeltaChannels(gcChannels) l.node.dataSyncService.removeFlowGraphsByDeltaChannels(gcChannels)
return err return err
} }
@ -478,18 +458,20 @@ func (w *watchDeltaChannelsTask) Execute(ctx context.Context) error {
// create tSafe // create tSafe
for _, channel := range vDeltaChannels { for _, channel := range vDeltaChannels {
w.node.tSafeReplica.addTSafe(channel) l.node.tSafeReplica.addTSafe(channel)
} }
// add tsafe watch in query shard if exists // add tsafe watch in query shard if exists, we find no way to handle it if query shard not exist
for _, channel := range vDeltaChannels { for _, channel := range vDeltaChannels {
dmlChannel, err := funcutil.ConvertChannelName(channel, Params.CommonCfg.RootCoordDelta, Params.CommonCfg.RootCoordDml) dmlChannel, err := funcutil.ConvertChannelName(channel, Params.CommonCfg.RootCoordDelta, Params.CommonCfg.RootCoordDml)
if err != nil { if err != nil {
log.Warn("failed to convert delta channel to dml", zap.String("channel", channel), zap.Error(err)) log.Error("failed to convert delta channel to dml", zap.String("channel", channel), zap.Error(err))
continue panic(err)
} }
if !w.node.queryShardService.hasQueryShard(dmlChannel) { err = l.node.queryShardService.addQueryShard(collectionID, dmlChannel, l.req.GetReplicaID())
w.node.queryShardService.addQueryShard(collectionID, dmlChannel, w.req.GetReplicaId()) if err != nil {
log.Error("failed to add shard Service to query shard", zap.String("channel", channel), zap.Error(err))
panic(err)
} }
} }
@ -503,7 +485,6 @@ func (w *watchDeltaChannelsTask) Execute(ctx context.Context) error {
} }
// loadSegmentsTask // loadSegmentsTask
func (l *loadSegmentsTask) PreExecute(ctx context.Context) error { func (l *loadSegmentsTask) PreExecute(ctx context.Context) error {
log.Info("LoadSegmentTask PreExecute start", zap.Int64("msgID", l.req.Base.MsgID)) log.Info("LoadSegmentTask PreExecute start", zap.Int64("msgID", l.req.Base.MsgID))
var err error var err error
@ -527,7 +508,7 @@ func (l *loadSegmentsTask) PreExecute(ctx context.Context) error {
if !has { if !has {
filteredInfos = append(filteredInfos, info) filteredInfos = append(filteredInfos, info)
} else { } else {
log.Debug("ignore segment that is already loaded", zap.Int64("segmentID", info.SegmentID)) log.Debug("ignore segment that is already loaded", zap.Int64("collectionID", info.SegmentID), zap.Int64("segmentID", info.SegmentID))
} }
} }
l.req.Infos = filteredInfos l.req.Infos = filteredInfos
@ -536,38 +517,52 @@ func (l *loadSegmentsTask) PreExecute(ctx context.Context) error {
} }
func (l *loadSegmentsTask) Execute(ctx context.Context) error { func (l *loadSegmentsTask) Execute(ctx context.Context) error {
// TODO: support db
log.Info("LoadSegmentTask Execute start", zap.Int64("msgID", l.req.Base.MsgID)) log.Info("LoadSegmentTask Execute start", zap.Int64("msgID", l.req.Base.MsgID))
err := l.node.loader.LoadSegment(l.req, segmentTypeSealed) err := l.node.loader.LoadSegment(l.req, segmentTypeSealed)
if err != nil { if err != nil {
log.Warn(err.Error()) log.Warn("failed to load segment", zap.Int64("collectionID", l.req.CollectionID),
zap.Int64("replicaID", l.req.ReplicaID), zap.Error(err))
return err
}
vchanName := make([]string, 0)
for _, deltaPosition := range l.req.DeltaPositions {
vchanName = append(vchanName, deltaPosition.ChannelName)
}
// TODO delta channel need to released 1. if other watchDeltaChannel fail 2. when segment release
err = l.watchDeltaChannel(vchanName)
if err != nil {
// roll back
for _, segment := range l.req.Infos {
l.node.metaReplica.removeSegment(segment.SegmentID, segmentTypeSealed)
}
log.Warn("failed to watch Delta channel while load segment", zap.Int64("collectionID", l.req.CollectionID),
zap.Int64("replicaID", l.req.ReplicaID), zap.Error(err))
return err return err
} }
// reload delete log from cp to latest position runningGroup, groupCtx := errgroup.WithContext(l.ctx)
for _, deltaPosition := range l.req.DeltaPositions { for _, deltaPosition := range l.req.DeltaPositions {
err = l.node.loader.FromDmlCPLoadDelete(ctx, l.req.CollectionID, deltaPosition) pos := deltaPosition
runningGroup.Go(func() error {
// reload data from dml channel
return l.node.loader.FromDmlCPLoadDelete(groupCtx, l.req.CollectionID, pos)
})
}
err = runningGroup.Wait()
if err != nil { if err != nil {
for _, segment := range l.req.Infos { for _, segment := range l.req.Infos {
l.node.metaReplica.removeSegment(segment.SegmentID, segmentTypeSealed) l.node.metaReplica.removeSegment(segment.SegmentID, segmentTypeSealed)
} }
log.Warn("LoadSegmentTask from delta check point load delete failed", zap.Int64("msgID", l.req.Base.MsgID), zap.Error(err)) log.Warn("failed to load delete data while load segment", zap.Int64("collectionID", l.req.CollectionID),
zap.Int64("replicaID", l.req.ReplicaID), zap.Error(err))
return err return err
} }
}
log.Info("LoadSegmentTask Execute done", zap.Int64("msgID", l.req.Base.MsgID)) log.Info("LoadSegmentTask Execute done", zap.Int64("collectionID", l.req.CollectionID),
zap.Int64("replicaID", l.req.ReplicaID), zap.Int64("msgID", l.req.Base.MsgID))
return nil return nil
} }
type ReplicaType int
const (
replicaNone ReplicaType = iota
replicaStreaming
replicaHistorical
)
func (r *releaseCollectionTask) Execute(ctx context.Context) error { func (r *releaseCollectionTask) Execute(ctx context.Context) error {
log.Info("Execute release collection task", zap.Any("collectionID", r.req.CollectionID)) log.Info("Execute release collection task", zap.Any("collectionID", r.req.CollectionID))

200
internal/querynode/task_test.go
View File

@ -21,11 +21,8 @@ import (
"testing" "testing"
"time" "time"
"github.com/apache/pulsar-client-go/pulsar"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/milvus-io/milvus/internal/mq/msgstream" "github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper/rmq"
"github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/proto/internalpb"
@ -33,6 +30,8 @@ import (
"github.com/milvus-io/milvus/internal/proto/schemapb" "github.com/milvus-io/milvus/internal/proto/schemapb"
"github.com/milvus-io/milvus/internal/util/funcutil" "github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus/internal/util/typeutil" "github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
func TestTask_watchDmChannelsTask(t *testing.T) { func TestTask_watchDmChannelsTask(t *testing.T) {
@ -269,93 +268,6 @@ func TestTask_watchDmChannelsTask(t *testing.T) {
}) })
} }
func TestTask_watchDeltaChannelsTask(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
genWatchDeltaChannelsRequest := func() *querypb.WatchDeltaChannelsRequest {
req := &querypb.WatchDeltaChannelsRequest{
Base: genCommonMsgBase(commonpb.MsgType_WatchDeltaChannels),
CollectionID: defaultCollectionID,
}
return req
}
t.Run("test timestamp", func(t *testing.T) {
timestamp := Timestamp(1000)
task := watchDeltaChannelsTask{
baseTask: baseTask{
ts: timestamp,
},
req: genWatchDeltaChannelsRequest(),
}
resT := task.Timestamp()
assert.Equal(t, timestamp, resT)
})
t.Run("test OnEnqueue", func(t *testing.T) {
task := watchDeltaChannelsTask{
req: genWatchDeltaChannelsRequest(),
}
err := task.OnEnqueue()
assert.NoError(t, err)
task.req.Base = nil
err = task.OnEnqueue()
assert.NoError(t, err)
})
t.Run("test execute", func(t *testing.T) {
node, err := genSimpleQueryNode(ctx)
assert.NoError(t, err)
task := watchDeltaChannelsTask{
req: genWatchDeltaChannelsRequest(),
node: node,
}
task.ctx = ctx
task.req.Infos = []*datapb.VchannelInfo{
{
CollectionID: defaultCollectionID,
ChannelName: defaultDeltaChannel,
SeekPosition: &internalpb.MsgPosition{
ChannelName: defaultDMLChannel,
MsgID: pulsar.EarliestMessageID().Serialize(),
MsgGroup: defaultSubName,
Timestamp: 0,
},
},
}
err = task.Execute(ctx)
assert.NoError(t, err)
})
t.Run("test execute without init collection", func(t *testing.T) {
node, err := genSimpleQueryNode(ctx)
assert.NoError(t, err)
task := watchDeltaChannelsTask{
req: genWatchDeltaChannelsRequest(),
node: node,
}
task.ctx = ctx
task.req.Infos = []*datapb.VchannelInfo{
{
CollectionID: defaultCollectionID,
ChannelName: defaultDeltaChannel,
SeekPosition: &internalpb.MsgPosition{
ChannelName: defaultDeltaChannel,
MsgID: []byte{1, 2, 3, 4, 5, 6, 7, 8},
MsgGroup: defaultSubName,
Timestamp: 0,
},
},
}
task.req.CollectionID++
err = task.Execute(ctx)
assert.Error(t, err)
})
}
func TestTask_loadSegmentsTask(t *testing.T) { func TestTask_loadSegmentsTask(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
@ -375,6 +287,7 @@ func TestTask_loadSegmentsTask(t *testing.T) {
task := loadSegmentsTask{ task := loadSegmentsTask{
baseTask: baseTask{ baseTask: baseTask{
ts: timestamp, ts: timestamp,
ctx: ctx,
}, },
req: genLoadEmptySegmentsRequest(), req: genLoadEmptySegmentsRequest(),
} }
@ -384,6 +297,9 @@ func TestTask_loadSegmentsTask(t *testing.T) {
t.Run("test OnEnqueue", func(t *testing.T) { t.Run("test OnEnqueue", func(t *testing.T) {
task := loadSegmentsTask{ task := loadSegmentsTask{
baseTask: baseTask{
ctx: ctx,
},
req: genLoadEmptySegmentsRequest(), req: genLoadEmptySegmentsRequest(),
} }
err := task.OnEnqueue() err := task.OnEnqueue()
@ -417,6 +333,9 @@ func TestTask_loadSegmentsTask(t *testing.T) {
} }
task := loadSegmentsTask{ task := loadSegmentsTask{
baseTask: baseTask{
ctx: ctx,
},
req: req, req: req,
node: node, node: node,
} }
@ -428,6 +347,8 @@ func TestTask_loadSegmentsTask(t *testing.T) {
node, err := genSimpleQueryNode(ctx) node, err := genSimpleQueryNode(ctx)
assert.NoError(t, err) assert.NoError(t, err)
node.metaReplica.removeSegment(defaultSegmentID, segmentTypeSealed)
fieldBinlog, statsLog, err := saveBinLog(ctx, defaultCollectionID, defaultPartitionID, defaultSegmentID, defaultMsgLength, schema) fieldBinlog, statsLog, err := saveBinLog(ctx, defaultCollectionID, defaultPartitionID, defaultSegmentID, defaultMsgLength, schema)
assert.NoError(t, err) assert.NoError(t, err)
@ -446,6 +367,9 @@ func TestTask_loadSegmentsTask(t *testing.T) {
} }
task := loadSegmentsTask{ task := loadSegmentsTask{
baseTask: baseTask{
ctx: ctx,
},
req: req, req: req,
node: node, node: node,
} }
@ -556,6 +480,9 @@ func TestTask_loadSegmentsTask(t *testing.T) {
} }
task := loadSegmentsTask{ task := loadSegmentsTask{
baseTask: baseTask{
ctx: ctx,
},
req: req, req: req,
node: node, node: node,
} }
@ -584,6 +511,9 @@ func TestTask_loadSegmentsTask(t *testing.T) {
assert.NoError(t, err) assert.NoError(t, err)
task := loadSegmentsTask{ task := loadSegmentsTask{
baseTask: baseTask{
ctx: ctx,
},
req: genLoadEmptySegmentsRequest(), req: genLoadEmptySegmentsRequest(),
node: node, node: node,
} }
@ -606,6 +536,94 @@ func TestTask_loadSegmentsTask(t *testing.T) {
}) })
} }
func TestTask_loadSegmentsTaskLoadDelta(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
schema := genTestCollectionSchema()
t.Run("test repeated load delta channel", func(t *testing.T) {
node, err := genSimpleQueryNode(ctx)
assert.NoError(t, err)
vDmChannel := "by-dev-rootcoord-dml-test_2_2021v2"
segmentLoadInfo := &querypb.SegmentLoadInfo{
SegmentID: UniqueID(1000),
PartitionID: defaultPartitionID,
CollectionID: defaultCollectionID,
}
loadReq := &querypb.LoadSegmentsRequest{
Base: genCommonMsgBase(commonpb.MsgType_LoadSegments),
Schema: schema,
Infos: []*querypb.SegmentLoadInfo{segmentLoadInfo},
DeltaPositions: []*internalpb.MsgPosition{
{
ChannelName: vDmChannel,
MsgID: rmq.SerializeRmqID(0),
Timestamp: 100,
},
},
}
task := loadSegmentsTask{
baseTask: baseTask{
ctx: ctx,
},
req: loadReq,
node: node,
}
// execute loadSegmentsTask twice
err = task.PreExecute(ctx)
assert.NoError(t, err)
err = task.Execute(ctx)
assert.NoError(t, err)
// expected only one segment in replica
num := node.metaReplica.getSegmentNum(segmentTypeSealed)
assert.Equal(t, 2, num)
// load second segments with same channel
loadReq = &querypb.LoadSegmentsRequest{
Base: genCommonMsgBase(commonpb.MsgType_LoadSegments),
Schema: schema,
Infos: []*querypb.SegmentLoadInfo{
{
SegmentID: UniqueID(1001),
PartitionID: defaultPartitionID,
CollectionID: defaultCollectionID,
},
},
DeltaPositions: []*internalpb.MsgPosition{
{
ChannelName: vDmChannel,
MsgID: rmq.SerializeRmqID(0),
Timestamp: 100,
},
},
}
task = loadSegmentsTask{
baseTask: baseTask{
ctx: ctx,
},
req: loadReq,
node: node,
}
// execute loadSegmentsTask twice
err = task.PreExecute(ctx)
assert.NoError(t, err)
err = task.Execute(ctx)
assert.NoError(t, err)
num = node.metaReplica.getSegmentNum(segmentTypeSealed)
assert.Equal(t, 3, num)
ok := node.queryShardService.hasQueryShard(vDmChannel)
assert.True(t, ok)
assert.Equal(t, len(node.dataSyncService.dmlChannel2FlowGraph), 0)
assert.Equal(t, len(node.dataSyncService.deltaChannel2FlowGraph), 1)
})
}
func TestTask_releaseCollectionTask(t *testing.T) { func TestTask_releaseCollectionTask(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()

1
internal/types/types.go
View File

@ -1223,7 +1223,6 @@ type QueryNode interface {
TimeTickProvider TimeTickProvider
WatchDmChannels(ctx context.Context, req *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) WatchDmChannels(ctx context.Context, req *querypb.WatchDmChannelsRequest) (*commonpb.Status, error)
WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error)
// LoadSegments notifies QueryNode to load the sealed segments from storage. The load tasks are sync to this // LoadSegments notifies QueryNode to load the sealed segments from storage. The load tasks are sync to this
// rpc, QueryNode will return after all the sealed segments are loaded. // rpc, QueryNode will return after all the sealed segments are loaded.
// //

6
internal/util/funcutil/parallel.go
View File

@ -48,12 +48,10 @@ func ProcessFuncParallel(total, maxParallel int, f ProcessFunc, fname string) er
t := time.Now() t := time.Now()
defer func() { defer func() {
log.Debug(fname, zap.Any("time cost", time.Since(t))) log.Debug(fname, zap.Any("total", total), zap.Any("time cost", time.Since(t)))
}() }()
nPerBatch := (total + maxParallel - 1) / maxParallel nPerBatch := (total + maxParallel - 1) / maxParallel
log.Debug(fname, zap.Any("total", total))
log.Debug(fname, zap.Any("nPerBatch", nPerBatch))
quit := make(chan bool) quit := make(chan bool)
errc := make(chan error) errc := make(chan error)
@ -105,8 +103,6 @@ func ProcessFuncParallel(total, maxParallel int, f ProcessFunc, fname string) er
routineNum++ routineNum++
} }
log.Debug(fname, zap.Any("NumOfGoRoutines", routineNum))
if routineNum <= 0 { if routineNum <= 0 {
return nil return nil
} }

4
internal/util/mock/grpc_querynode_client.go
View File

@ -53,10 +53,6 @@ func (m *GrpcQueryNodeClient) WatchDmChannels(ctx context.Context, in *querypb.W
return &commonpb.Status{}, m.Err return &commonpb.Status{}, m.Err
} }
func (m *GrpcQueryNodeClient) WatchDeltaChannels(ctx context.Context, in *querypb.WatchDeltaChannelsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
return &commonpb.Status{}, m.Err
}
func (m *GrpcQueryNodeClient) LoadSegments(ctx context.Context, in *querypb.LoadSegmentsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) { func (m *GrpcQueryNodeClient) LoadSegments(ctx context.Context, in *querypb.LoadSegmentsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
return &commonpb.Status{}, m.Err return &commonpb.Status{}, m.Err
} }

4
internal/util/mock/querynode_client.go
View File

@ -69,10 +69,6 @@ func (q QueryNodeClient) WatchDmChannels(ctx context.Context, req *querypb.Watch
return q.grpcClient.WatchDmChannels(ctx, req) return q.grpcClient.WatchDmChannels(ctx, req)
} }
func (q QueryNodeClient) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) {
return q.grpcClient.WatchDeltaChannels(ctx, req)
}
func (q QueryNodeClient) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { func (q QueryNodeClient) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) {
return q.grpcClient.LoadSegments(ctx, req) return q.grpcClient.LoadSegments(ctx, req)
} }