From b427450593245f1ce49e9df2d1d2700be581e75c Mon Sep 17 00:00:00 2001 From: Xiaofan <83447078+xiaofan-luan@users.noreply.github.com> Date: Thu, 4 Aug 2022 17:50:37 +0800 Subject: [PATCH] Remove Load Delta Channel Request (#18497) Fix #18393 remove watch delta channel task related logic Co-authored-by: xiaofan-luan Signed-off-by: Congqi Xia Co-authored-by: Congqi Xia --- cmd/milvus/mck.go | 17 +- .../distributed/querynode/client/client.go | 14 - .../querynode/client/client_test.go | 3 - internal/distributed/querynode/service.go | 6 - .../distributed/querynode/service_test.go | 4 - internal/proto/query_coord.proto | 10 - internal/proto/querypb/query_coord.pb.go | 527 +++++++----------- internal/proxy/query_node_mock_test.go | 5 - internal/querycoord/cluster.go | 93 +--- internal/querycoord/cluster_test.go | 12 +- .../querycoord/mock_querynode_client_test.go | 19 +- .../querycoord/mock_querynode_server_test.go | 4 - internal/querycoord/querynode.go | 66 +-- internal/querycoord/task.go | 93 +--- internal/querycoord/task_scheduler.go | 81 +-- internal/querycoord/task_scheduler_test.go | 64 --- internal/querycoord/task_test.go | 38 +- internal/querycoord/task_util.go | 7 - internal/querycoord/util.go | 29 - internal/querynode/impl.go | 52 -- internal/querynode/query_shard_service.go | 2 +- internal/querynode/segment_loader.go | 54 +- internal/querynode/shard_cluster.go | 2 +- internal/querynode/task.go | 139 +++-- internal/querynode/task_test.go | 202 ++++--- internal/types/types.go | 1 - internal/util/funcutil/parallel.go | 6 +- internal/util/mock/grpc_querynode_client.go | 4 - internal/util/mock/querynode_client.go | 4 - 29 files changed, 468 insertions(+), 1090 deletions(-) diff --git a/cmd/milvus/mck.go b/cmd/milvus/mck.go index bda3019c26..2660fc6e15 100644 --- a/cmd/milvus/mck.go +++ b/cmd/milvus/mck.go @@ -624,21 +624,8 @@ func (c *mck) unmarshalTask(taskID int64, t string) (string, []int64, []int64, e log.Info("WatchDmChannels", zap.String("detail", fmt.Sprintf("+%v", loadReq))) return "WatchDmChannels", removeRepeatElement(partitionIDs), removeRepeatElement(segmentIDs), nil case commonpb.MsgType_WatchDeltaChannels: - loadReq := querypb.WatchDeltaChannelsRequest{} - err = proto.Unmarshal([]byte(t), &loadReq) - if err != nil { - return errReturn(taskID, "WatchDeltaChannelsRequest", err) - } - var partitionIDs []int64 - var segmentIDs []int64 - if loadReq.LoadMeta != nil { - partitionIDs = append(partitionIDs, loadReq.LoadMeta.PartitionIDs...) - } - pids, sids := c.extractVchannelInfo(taskID, loadReq.Infos) - partitionIDs = append(partitionIDs, pids...) - segmentIDs = append(segmentIDs, sids...) - log.Info("WatchDeltaChannels", zap.String("detail", fmt.Sprintf("+%v", loadReq))) - return "WatchDeltaChannels", removeRepeatElement(partitionIDs), removeRepeatElement(segmentIDs), nil + log.Warn("legacy WatchDeltaChannels type found, ignore") + return "WatchQueryChannels", emptyInt64(), emptyInt64(), nil case commonpb.MsgType_WatchQueryChannels: log.Warn("legacy WatchQueryChannels type found, ignore") return "WatchQueryChannels", emptyInt64(), emptyInt64(), nil diff --git a/internal/distributed/querynode/client/client.go b/internal/distributed/querynode/client/client.go index a3c9678968..b2059e0b4d 100644 --- a/internal/distributed/querynode/client/client.go +++ b/internal/distributed/querynode/client/client.go @@ -152,20 +152,6 @@ func (c *Client) WatchDmChannels(ctx context.Context, req *querypb.WatchDmChanne return ret.(*commonpb.Status), err } -// WatchDeltaChannels watches the channels about data manipulation. -func (c *Client) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) { - ret, err := c.grpcClient.ReCall(ctx, func(client interface{}) (interface{}, error) { - if !funcutil.CheckCtxValid(ctx) { - return nil, ctx.Err() - } - return client.(querypb.QueryNodeClient).WatchDeltaChannels(ctx, req) - }) - if err != nil || ret == nil { - return nil, err - } - return ret.(*commonpb.Status), err -} - // LoadSegments loads the segments to search. func (c *Client) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { ret, err := c.grpcClient.ReCall(ctx, func(client interface{}) (interface{}, error) { diff --git a/internal/distributed/querynode/client/client_test.go b/internal/distributed/querynode/client/client_test.go index 5b8b0b1295..8c3c03f9e0 100644 --- a/internal/distributed/querynode/client/client_test.go +++ b/internal/distributed/querynode/client/client_test.go @@ -90,9 +90,6 @@ func Test_NewClient(t *testing.T) { r12, err := client.GetMetrics(ctx, nil) retCheck(retNotNil, r12, err) - r13, err := client.WatchDeltaChannels(ctx, nil) - retCheck(retNotNil, r13, err) - r14, err := client.Search(ctx, nil) retCheck(retNotNil, r14, err) diff --git a/internal/distributed/querynode/service.go b/internal/distributed/querynode/service.go index 5608dc5750..207a52ce30 100644 --- a/internal/distributed/querynode/service.go +++ b/internal/distributed/querynode/service.go @@ -262,12 +262,6 @@ func (s *Server) WatchDmChannels(ctx context.Context, req *querypb.WatchDmChanne return s.querynode.WatchDmChannels(ctx, req) } -// WatchDeltaChannels watches the channels about data manipulation. -func (s *Server) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) { - // ignore ctx - return s.querynode.WatchDeltaChannels(ctx, req) -} - // LoadSegments loads the segments to search. func (s *Server) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { // ignore ctx diff --git a/internal/distributed/querynode/service_test.go b/internal/distributed/querynode/service_test.go index 72760bb198..fdc56581ef 100644 --- a/internal/distributed/querynode/service_test.go +++ b/internal/distributed/querynode/service_test.go @@ -82,10 +82,6 @@ func (m *MockQueryNode) WatchDmChannels(ctx context.Context, req *querypb.WatchD return m.status, m.err } -func (m *MockQueryNode) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) { - return m.status, m.err -} - func (m *MockQueryNode) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { return m.status, m.err } diff --git a/internal/proto/query_coord.proto b/internal/proto/query_coord.proto index 46bc0c9251..4d3f08db7d 100644 --- a/internal/proto/query_coord.proto +++ b/internal/proto/query_coord.proto @@ -41,7 +41,6 @@ service QueryNode { rpc GetStatisticsChannel(internal.GetStatisticsChannelRequest) returns(milvus.StringResponse){} rpc WatchDmChannels(WatchDmChannelsRequest) returns (common.Status) {} - rpc WatchDeltaChannels(WatchDeltaChannelsRequest) returns (common.Status) {} rpc LoadSegments(LoadSegmentsRequest) returns (common.Status) {} rpc ReleaseCollection(ReleaseCollectionRequest) returns (common.Status) {} rpc ReleasePartitions(ReleasePartitionsRequest) returns (common.Status) {} @@ -185,15 +184,6 @@ message WatchDmChannelsRequest { map segment_infos = 10; } -message WatchDeltaChannelsRequest { - common.MsgBase base = 1; - int64 nodeID = 2; - int64 collectionID = 3; - repeated data.VchannelInfo infos = 4; - int64 replica_id = 5; - LoadMetaInfo load_meta = 9; -} - message SegmentLoadInfo { int64 segmentID = 1; int64 partitionID = 2; diff --git a/internal/proto/querypb/query_coord.pb.go b/internal/proto/querypb/query_coord.pb.go index 7052d40529..d25223dce3 100644 --- a/internal/proto/querypb/query_coord.pb.go +++ b/internal/proto/querypb/query_coord.pb.go @@ -1283,85 +1283,6 @@ func (m *WatchDmChannelsRequest) GetSegmentInfos() map[int64]*datapb.SegmentInfo return nil } -type WatchDeltaChannelsRequest struct { - Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` - NodeID int64 `protobuf:"varint,2,opt,name=nodeID,proto3" json:"nodeID,omitempty"` - CollectionID int64 `protobuf:"varint,3,opt,name=collectionID,proto3" json:"collectionID,omitempty"` - Infos []*datapb.VchannelInfo `protobuf:"bytes,4,rep,name=infos,proto3" json:"infos,omitempty"` - ReplicaId int64 `protobuf:"varint,5,opt,name=replica_id,json=replicaId,proto3" json:"replica_id,omitempty"` - LoadMeta *LoadMetaInfo `protobuf:"bytes,9,opt,name=load_meta,json=loadMeta,proto3" json:"load_meta,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *WatchDeltaChannelsRequest) Reset() { *m = WatchDeltaChannelsRequest{} } -func (m *WatchDeltaChannelsRequest) String() string { return proto.CompactTextString(m) } -func (*WatchDeltaChannelsRequest) ProtoMessage() {} -func (*WatchDeltaChannelsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{18} -} - -func (m *WatchDeltaChannelsRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_WatchDeltaChannelsRequest.Unmarshal(m, b) -} -func (m *WatchDeltaChannelsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_WatchDeltaChannelsRequest.Marshal(b, m, deterministic) -} -func (m *WatchDeltaChannelsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_WatchDeltaChannelsRequest.Merge(m, src) -} -func (m *WatchDeltaChannelsRequest) XXX_Size() int { - return xxx_messageInfo_WatchDeltaChannelsRequest.Size(m) -} -func (m *WatchDeltaChannelsRequest) XXX_DiscardUnknown() { - xxx_messageInfo_WatchDeltaChannelsRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_WatchDeltaChannelsRequest proto.InternalMessageInfo - -func (m *WatchDeltaChannelsRequest) GetBase() *commonpb.MsgBase { - if m != nil { - return m.Base - } - return nil -} - -func (m *WatchDeltaChannelsRequest) GetNodeID() int64 { - if m != nil { - return m.NodeID - } - return 0 -} - -func (m *WatchDeltaChannelsRequest) GetCollectionID() int64 { - if m != nil { - return m.CollectionID - } - return 0 -} - -func (m *WatchDeltaChannelsRequest) GetInfos() []*datapb.VchannelInfo { - if m != nil { - return m.Infos - } - return nil -} - -func (m *WatchDeltaChannelsRequest) GetReplicaId() int64 { - if m != nil { - return m.ReplicaId - } - return 0 -} - -func (m *WatchDeltaChannelsRequest) GetLoadMeta() *LoadMetaInfo { - if m != nil { - return m.LoadMeta - } - return nil -} - type SegmentLoadInfo struct { SegmentID int64 `protobuf:"varint,1,opt,name=segmentID,proto3" json:"segmentID,omitempty"` PartitionID int64 `protobuf:"varint,2,opt,name=partitionID,proto3" json:"partitionID,omitempty"` @@ -1385,7 +1306,7 @@ func (m *SegmentLoadInfo) Reset() { *m = SegmentLoadInfo{} } func (m *SegmentLoadInfo) String() string { return proto.CompactTextString(m) } func (*SegmentLoadInfo) ProtoMessage() {} func (*SegmentLoadInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{19} + return fileDescriptor_aab7cc9a69ed26e8, []int{18} } func (m *SegmentLoadInfo) XXX_Unmarshal(b []byte) error { @@ -1515,7 +1436,7 @@ func (m *FieldIndexInfo) Reset() { *m = FieldIndexInfo{} } func (m *FieldIndexInfo) String() string { return proto.CompactTextString(m) } func (*FieldIndexInfo) ProtoMessage() {} func (*FieldIndexInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{20} + return fileDescriptor_aab7cc9a69ed26e8, []int{19} } func (m *FieldIndexInfo) XXX_Unmarshal(b []byte) error { @@ -1611,7 +1532,7 @@ func (m *LoadSegmentsRequest) Reset() { *m = LoadSegmentsRequest{} } func (m *LoadSegmentsRequest) String() string { return proto.CompactTextString(m) } func (*LoadSegmentsRequest) ProtoMessage() {} func (*LoadSegmentsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{21} + return fileDescriptor_aab7cc9a69ed26e8, []int{20} } func (m *LoadSegmentsRequest) XXX_Unmarshal(b []byte) error { @@ -1713,7 +1634,7 @@ func (m *ReleaseSegmentsRequest) Reset() { *m = ReleaseSegmentsRequest{} func (m *ReleaseSegmentsRequest) String() string { return proto.CompactTextString(m) } func (*ReleaseSegmentsRequest) ProtoMessage() {} func (*ReleaseSegmentsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{22} + return fileDescriptor_aab7cc9a69ed26e8, []int{21} } func (m *ReleaseSegmentsRequest) XXX_Unmarshal(b []byte) error { @@ -1798,7 +1719,7 @@ func (m *SearchRequest) Reset() { *m = SearchRequest{} } func (m *SearchRequest) String() string { return proto.CompactTextString(m) } func (*SearchRequest) ProtoMessage() {} func (*SearchRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{23} + return fileDescriptor_aab7cc9a69ed26e8, []int{22} } func (m *SearchRequest) XXX_Unmarshal(b []byte) error { @@ -1869,7 +1790,7 @@ func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{24} + return fileDescriptor_aab7cc9a69ed26e8, []int{23} } func (m *QueryRequest) XXX_Unmarshal(b []byte) error { @@ -1938,7 +1859,7 @@ func (m *SyncReplicaSegmentsRequest) Reset() { *m = SyncReplicaSegmentsR func (m *SyncReplicaSegmentsRequest) String() string { return proto.CompactTextString(m) } func (*SyncReplicaSegmentsRequest) ProtoMessage() {} func (*SyncReplicaSegmentsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{25} + return fileDescriptor_aab7cc9a69ed26e8, []int{24} } func (m *SyncReplicaSegmentsRequest) XXX_Unmarshal(b []byte) error { @@ -1993,7 +1914,7 @@ func (m *ReplicaSegmentsInfo) Reset() { *m = ReplicaSegmentsInfo{} } func (m *ReplicaSegmentsInfo) String() string { return proto.CompactTextString(m) } func (*ReplicaSegmentsInfo) ProtoMessage() {} func (*ReplicaSegmentsInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{26} + return fileDescriptor_aab7cc9a69ed26e8, []int{25} } func (m *ReplicaSegmentsInfo) XXX_Unmarshal(b []byte) error { @@ -2049,7 +1970,7 @@ func (m *HandoffSegmentsRequest) Reset() { *m = HandoffSegmentsRequest{} func (m *HandoffSegmentsRequest) String() string { return proto.CompactTextString(m) } func (*HandoffSegmentsRequest) ProtoMessage() {} func (*HandoffSegmentsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{27} + return fileDescriptor_aab7cc9a69ed26e8, []int{26} } func (m *HandoffSegmentsRequest) XXX_Unmarshal(b []byte) error { @@ -2107,7 +2028,7 @@ func (m *LoadBalanceRequest) Reset() { *m = LoadBalanceRequest{} } func (m *LoadBalanceRequest) String() string { return proto.CompactTextString(m) } func (*LoadBalanceRequest) ProtoMessage() {} func (*LoadBalanceRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{28} + return fileDescriptor_aab7cc9a69ed26e8, []int{27} } func (m *LoadBalanceRequest) XXX_Unmarshal(b []byte) error { @@ -2185,7 +2106,7 @@ func (m *DmChannelWatchInfo) Reset() { *m = DmChannelWatchInfo{} } func (m *DmChannelWatchInfo) String() string { return proto.CompactTextString(m) } func (*DmChannelWatchInfo) ProtoMessage() {} func (*DmChannelWatchInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{29} + return fileDescriptor_aab7cc9a69ed26e8, []int{28} } func (m *DmChannelWatchInfo) XXX_Unmarshal(b []byte) error { @@ -2256,7 +2177,7 @@ func (m *QueryChannelInfo) Reset() { *m = QueryChannelInfo{} } func (m *QueryChannelInfo) String() string { return proto.CompactTextString(m) } func (*QueryChannelInfo) ProtoMessage() {} func (*QueryChannelInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{30} + return fileDescriptor_aab7cc9a69ed26e8, []int{29} } func (m *QueryChannelInfo) XXX_Unmarshal(b []byte) error { @@ -2325,7 +2246,7 @@ func (m *PartitionStates) Reset() { *m = PartitionStates{} } func (m *PartitionStates) String() string { return proto.CompactTextString(m) } func (*PartitionStates) ProtoMessage() {} func (*PartitionStates) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{31} + return fileDescriptor_aab7cc9a69ed26e8, []int{30} } func (m *PartitionStates) XXX_Unmarshal(b []byte) error { @@ -2393,7 +2314,7 @@ func (m *SegmentInfo) Reset() { *m = SegmentInfo{} } func (m *SegmentInfo) String() string { return proto.CompactTextString(m) } func (*SegmentInfo) ProtoMessage() {} func (*SegmentInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{32} + return fileDescriptor_aab7cc9a69ed26e8, []int{31} } func (m *SegmentInfo) XXX_Unmarshal(b []byte) error { @@ -2538,7 +2459,7 @@ func (m *CollectionInfo) Reset() { *m = CollectionInfo{} } func (m *CollectionInfo) String() string { return proto.CompactTextString(m) } func (*CollectionInfo) ProtoMessage() {} func (*CollectionInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{33} + return fileDescriptor_aab7cc9a69ed26e8, []int{32} } func (m *CollectionInfo) XXX_Unmarshal(b []byte) error { @@ -2634,7 +2555,7 @@ func (m *UnsubscribeChannels) Reset() { *m = UnsubscribeChannels{} } func (m *UnsubscribeChannels) String() string { return proto.CompactTextString(m) } func (*UnsubscribeChannels) ProtoMessage() {} func (*UnsubscribeChannels) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{34} + return fileDescriptor_aab7cc9a69ed26e8, []int{33} } func (m *UnsubscribeChannels) XXX_Unmarshal(b []byte) error { @@ -2681,7 +2602,7 @@ func (m *UnsubscribeChannelInfo) Reset() { *m = UnsubscribeChannelInfo{} func (m *UnsubscribeChannelInfo) String() string { return proto.CompactTextString(m) } func (*UnsubscribeChannelInfo) ProtoMessage() {} func (*UnsubscribeChannelInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{35} + return fileDescriptor_aab7cc9a69ed26e8, []int{34} } func (m *UnsubscribeChannelInfo) XXX_Unmarshal(b []byte) error { @@ -2731,7 +2652,7 @@ func (m *SegmentChangeInfo) Reset() { *m = SegmentChangeInfo{} } func (m *SegmentChangeInfo) String() string { return proto.CompactTextString(m) } func (*SegmentChangeInfo) ProtoMessage() {} func (*SegmentChangeInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{36} + return fileDescriptor_aab7cc9a69ed26e8, []int{35} } func (m *SegmentChangeInfo) XXX_Unmarshal(b []byte) error { @@ -2792,7 +2713,7 @@ func (m *SealedSegmentsChangeInfo) Reset() { *m = SealedSegmentsChangeIn func (m *SealedSegmentsChangeInfo) String() string { return proto.CompactTextString(m) } func (*SealedSegmentsChangeInfo) ProtoMessage() {} func (*SealedSegmentsChangeInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{37} + return fileDescriptor_aab7cc9a69ed26e8, []int{36} } func (m *SealedSegmentsChangeInfo) XXX_Unmarshal(b []byte) error { @@ -2851,7 +2772,6 @@ func init() { proto.RegisterType((*LoadMetaInfo)(nil), "milvus.proto.query.LoadMetaInfo") proto.RegisterType((*WatchDmChannelsRequest)(nil), "milvus.proto.query.WatchDmChannelsRequest") proto.RegisterMapType((map[int64]*datapb.SegmentInfo)(nil), "milvus.proto.query.WatchDmChannelsRequest.SegmentInfosEntry") - proto.RegisterType((*WatchDeltaChannelsRequest)(nil), "milvus.proto.query.WatchDeltaChannelsRequest") proto.RegisterType((*SegmentLoadInfo)(nil), "milvus.proto.query.SegmentLoadInfo") proto.RegisterType((*FieldIndexInfo)(nil), "milvus.proto.query.FieldIndexInfo") proto.RegisterType((*LoadSegmentsRequest)(nil), "milvus.proto.query.LoadSegmentsRequest") @@ -2876,195 +2796,192 @@ func init() { func init() { proto.RegisterFile("query_coord.proto", fileDescriptor_aab7cc9a69ed26e8) } var fileDescriptor_aab7cc9a69ed26e8 = []byte{ - // 3003 bytes of a gzipped FileDescriptorProto + // 2951 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x3a, 0x49, 0x6f, 0x1c, 0xc7, - 0xd5, 0xec, 0x59, 0xc8, 0x99, 0x37, 0x5b, 0xab, 0x28, 0x51, 0xe3, 0xf9, 0x24, 0x9b, 0x6e, 0x59, - 0x36, 0x3f, 0xda, 0xa6, 0x64, 0xca, 0x31, 0xec, 0xc4, 0x06, 0x22, 0x91, 0x16, 0xcd, 0x48, 0xa2, - 0x99, 0x1e, 0xc9, 0x09, 0x04, 0x03, 0x93, 0x9e, 0xe9, 0x9a, 0x61, 0x43, 0xbd, 0x8c, 0xba, 0x7a, - 0x28, 0xd1, 0xb9, 0x06, 0x01, 0xb2, 0x1d, 0x82, 0x5c, 0x03, 0x23, 0x87, 0xe4, 0x10, 0x20, 0x46, - 0x2e, 0xb9, 0x04, 0x08, 0x82, 0xdc, 0x72, 0x0d, 0x90, 0x1f, 0xe0, 0x63, 0x0e, 0xc9, 0x35, 0x87, - 0xdc, 0x82, 0xda, 0x7a, 0x9b, 0x6e, 0xcd, 0x90, 0xb4, 0xbc, 0x00, 0xb9, 0x75, 0xbf, 0x7a, 0x55, - 0xef, 0xd5, 0xdb, 0x5f, 0x55, 0xc1, 0x99, 0x87, 0x13, 0xec, 0x1f, 0xf5, 0x06, 0x9e, 0xe7, 0x9b, - 0x1b, 0x63, 0xdf, 0x0b, 0x3c, 0x84, 0x1c, 0xcb, 0x3e, 0x9c, 0x10, 0xfe, 0xb7, 0xc1, 0xc6, 0x3b, - 0xf5, 0x81, 0xe7, 0x38, 0x9e, 0xcb, 0x61, 0x9d, 0x7a, 0x1c, 0xa3, 0xd3, 0xb4, 0xdc, 0x00, 0xfb, - 0xae, 0x61, 0xcb, 0x51, 0x32, 0x38, 0xc0, 0x8e, 0x21, 0xfe, 0x54, 0xd3, 0x08, 0x8c, 0xf8, 0xfa, - 0xda, 0x0f, 0x14, 0x58, 0xe9, 0x1e, 0x78, 0x8f, 0xb6, 0x3c, 0xdb, 0xc6, 0x83, 0xc0, 0xf2, 0x5c, - 0xa2, 0xe3, 0x87, 0x13, 0x4c, 0x02, 0x74, 0x15, 0x4a, 0x7d, 0x83, 0xe0, 0xb6, 0xb2, 0xaa, 0xac, - 0xd5, 0x36, 0x2f, 0x6c, 0x24, 0x38, 0x11, 0x2c, 0xdc, 0x21, 0xa3, 0x1b, 0x06, 0xc1, 0x3a, 0xc3, - 0x44, 0x08, 0x4a, 0x66, 0x7f, 0x77, 0xbb, 0x5d, 0x58, 0x55, 0xd6, 0x8a, 0x3a, 0xfb, 0x46, 0x2f, - 0x40, 0x63, 0x10, 0xae, 0xbd, 0xbb, 0x4d, 0xda, 0xc5, 0xd5, 0xe2, 0x5a, 0x51, 0x4f, 0x02, 0xb5, - 0x4f, 0x15, 0x38, 0x3f, 0xc5, 0x06, 0x19, 0x7b, 0x2e, 0xc1, 0xe8, 0x1a, 0x2c, 0x92, 0xc0, 0x08, - 0x26, 0x44, 0x70, 0xf2, 0x7f, 0x99, 0x9c, 0x74, 0x19, 0x8a, 0x2e, 0x50, 0xa7, 0xc9, 0x16, 0x32, - 0xc8, 0xa2, 0xd7, 0xe0, 0xac, 0xe5, 0xde, 0xc1, 0x8e, 0xe7, 0x1f, 0xf5, 0xc6, 0xd8, 0x1f, 0x60, - 0x37, 0x30, 0x46, 0x58, 0xf2, 0xb8, 0x2c, 0xc7, 0xf6, 0xa3, 0x21, 0xf4, 0x06, 0x9c, 0xe7, 0x5a, - 0x22, 0xd8, 0x3f, 0xb4, 0x06, 0xb8, 0x67, 0x1c, 0x1a, 0x96, 0x6d, 0xf4, 0x6d, 0xdc, 0x2e, 0xad, - 0x16, 0xd7, 0x2a, 0xfa, 0x39, 0x36, 0xdc, 0xe5, 0xa3, 0xd7, 0xe5, 0xa0, 0xf6, 0x1b, 0x05, 0xce, - 0xd1, 0x1d, 0xee, 0x1b, 0x7e, 0x60, 0x3d, 0x05, 0x39, 0x6b, 0x50, 0x8f, 0xef, 0xad, 0x5d, 0x64, - 0x63, 0x09, 0x18, 0xc5, 0x19, 0x4b, 0xf2, 0x54, 0x26, 0x25, 0xb6, 0xcd, 0x04, 0x4c, 0xfb, 0xb5, - 0x30, 0x88, 0x38, 0x9f, 0xa7, 0x51, 0x44, 0x9a, 0x66, 0x61, 0x9a, 0xe6, 0x09, 0xd4, 0xa0, 0xfd, - 0x43, 0x81, 0x73, 0xb7, 0x3d, 0xc3, 0x8c, 0x0c, 0xe6, 0xf3, 0x17, 0xe7, 0x3b, 0xb0, 0xc8, 0xbd, - 0xab, 0x5d, 0x62, 0xb4, 0x2e, 0x27, 0x69, 0x09, 0xcf, 0x8b, 0x38, 0xec, 0x32, 0x80, 0x2e, 0x26, - 0xa1, 0xcb, 0xd0, 0xf4, 0xf1, 0xd8, 0xb6, 0x06, 0x46, 0xcf, 0x9d, 0x38, 0x7d, 0xec, 0xb7, 0xcb, - 0xab, 0xca, 0x5a, 0x59, 0x6f, 0x08, 0xe8, 0x1e, 0x03, 0x6a, 0xbf, 0x54, 0xa0, 0xad, 0x63, 0x1b, - 0x1b, 0x04, 0x7f, 0x91, 0x9b, 0x5d, 0x81, 0x45, 0xd7, 0x33, 0xf1, 0xee, 0x36, 0xdb, 0x6c, 0x51, - 0x17, 0x7f, 0xda, 0x7f, 0x14, 0x38, 0xbb, 0x83, 0x03, 0xaa, 0x75, 0x8b, 0x04, 0xd6, 0x20, 0x34, - 0xeb, 0x77, 0xa0, 0xe8, 0xe3, 0x87, 0x82, 0xb3, 0x97, 0x93, 0x9c, 0x85, 0x41, 0x2a, 0x6b, 0xa6, - 0x4e, 0xe7, 0xa1, 0xe7, 0xa1, 0x6e, 0x3a, 0x76, 0x6f, 0x70, 0x60, 0xb8, 0x2e, 0xb6, 0xb9, 0xdd, - 0x54, 0xf5, 0x9a, 0xe9, 0xd8, 0x5b, 0x02, 0x84, 0x9e, 0x05, 0x20, 0x78, 0xe4, 0x60, 0x37, 0x88, - 0xe2, 0x4a, 0x0c, 0x82, 0xd6, 0xe1, 0xcc, 0xd0, 0xf7, 0x9c, 0x1e, 0x39, 0x30, 0x7c, 0xb3, 0x67, - 0x63, 0xc3, 0xc4, 0x3e, 0xe3, 0xbe, 0xa2, 0xb7, 0xe8, 0x40, 0x97, 0xc2, 0x6f, 0x33, 0x30, 0xba, - 0x06, 0x65, 0x32, 0xf0, 0xc6, 0x98, 0xe9, 0xa0, 0xb9, 0x79, 0x71, 0x63, 0x3a, 0xee, 0x6e, 0x6c, - 0x1b, 0x81, 0xd1, 0xa5, 0x48, 0x3a, 0xc7, 0xd5, 0x7e, 0x52, 0xe0, 0x46, 0xf8, 0x25, 0xf7, 0xe9, - 0x98, 0xa1, 0x96, 0x3f, 0x1b, 0x43, 0x5d, 0xcc, 0x32, 0xd4, 0xbf, 0x44, 0x86, 0xfa, 0x65, 0x17, - 0x48, 0x64, 0xcc, 0xe5, 0x84, 0x31, 0xff, 0x56, 0x81, 0x67, 0x76, 0x70, 0x10, 0xb2, 0x4f, 0x6d, - 0x13, 0x7f, 0x49, 0x03, 0xf5, 0x27, 0x0a, 0x74, 0xb2, 0x78, 0x3d, 0x4d, 0xb0, 0xbe, 0x0f, 0x2b, - 0x21, 0x8d, 0x9e, 0x89, 0xc9, 0xc0, 0xb7, 0xc6, 0x4c, 0x8d, 0xcc, 0xfd, 0x6a, 0x9b, 0x97, 0xb2, - 0xdc, 0x22, 0xcd, 0xc1, 0xb9, 0x70, 0x89, 0xed, 0xd8, 0x0a, 0xda, 0xcf, 0x14, 0x38, 0x47, 0xdd, - 0x5d, 0xf8, 0xa7, 0x3b, 0xf4, 0x4e, 0x2e, 0xd7, 0xa4, 0xe7, 0x17, 0xa6, 0x3c, 0x7f, 0x0e, 0x19, - 0xb3, 0xca, 0x27, 0xcd, 0xcf, 0x69, 0x64, 0xf7, 0x35, 0x28, 0x5b, 0xee, 0xd0, 0x93, 0xa2, 0x7a, - 0x2e, 0x4b, 0x54, 0x71, 0x62, 0x1c, 0x5b, 0x73, 0x39, 0x17, 0x51, 0x28, 0x3a, 0x85, 0xb9, 0xa5, - 0xb7, 0x5d, 0xc8, 0xd8, 0xf6, 0x4f, 0x15, 0x38, 0x3f, 0x45, 0xf0, 0x34, 0xfb, 0x7e, 0x1b, 0x16, - 0x59, 0x80, 0x95, 0x1b, 0x7f, 0x21, 0x73, 0xe3, 0x31, 0x72, 0xb7, 0x2d, 0x12, 0xe8, 0x62, 0x8e, - 0xe6, 0x81, 0x9a, 0x1e, 0xa3, 0xa1, 0x5f, 0x84, 0xfd, 0x9e, 0x6b, 0x38, 0x5c, 0x00, 0x55, 0xbd, - 0x26, 0x60, 0x7b, 0x86, 0x83, 0xd1, 0x33, 0x50, 0xa1, 0x2e, 0xdb, 0xb3, 0x4c, 0xa9, 0xfe, 0x25, - 0xe6, 0xc2, 0x26, 0x41, 0x17, 0x01, 0xd8, 0x90, 0x61, 0x9a, 0x3e, 0xcf, 0x0a, 0x55, 0xbd, 0x4a, - 0x21, 0xd7, 0x29, 0x40, 0xfb, 0xb9, 0x02, 0x75, 0x1a, 0xb3, 0xef, 0xe0, 0xc0, 0xa0, 0x7a, 0x40, - 0x6f, 0x41, 0xd5, 0xf6, 0x0c, 0xb3, 0x17, 0x1c, 0x8d, 0x39, 0xa9, 0x66, 0x5a, 0xd6, 0x7c, 0x0b, - 0x74, 0xd2, 0xdd, 0xa3, 0x31, 0xd6, 0x2b, 0xb6, 0xf8, 0x9a, 0x47, 0xde, 0x53, 0xae, 0x5c, 0xcc, - 0x70, 0xe5, 0x1f, 0x96, 0x61, 0xe5, 0x3b, 0x46, 0x30, 0x38, 0xd8, 0x76, 0x64, 0x72, 0x3b, 0xb9, - 0x11, 0x44, 0xb1, 0xad, 0x10, 0x8f, 0x6d, 0x9f, 0x59, 0xec, 0x0c, 0xed, 0xbc, 0x9c, 0x65, 0xe7, - 0xb4, 0xc1, 0xd8, 0xf8, 0x40, 0xa8, 0x2a, 0x66, 0xe7, 0xb1, 0x1c, 0xb4, 0x78, 0x92, 0x1c, 0xb4, - 0x05, 0x0d, 0xfc, 0x78, 0x60, 0x4f, 0xa8, 0xce, 0x19, 0xf5, 0x25, 0x46, 0xfd, 0xd9, 0x0c, 0xea, - 0x71, 0x27, 0xab, 0x8b, 0x49, 0xbb, 0x82, 0x07, 0xae, 0x6a, 0x07, 0x07, 0x46, 0xbb, 0xc2, 0xd8, - 0x58, 0xcd, 0x53, 0xb5, 0xb4, 0x0f, 0xae, 0x6e, 0xfa, 0x87, 0x2e, 0x40, 0x55, 0x64, 0xbc, 0xdd, - 0xed, 0x76, 0x95, 0x89, 0x2f, 0x02, 0x20, 0x03, 0x1a, 0x22, 0x02, 0x09, 0x0e, 0x81, 0x71, 0xf8, - 0x76, 0x16, 0x81, 0x6c, 0x65, 0xc7, 0x39, 0x27, 0xef, 0xba, 0x81, 0x7f, 0xa4, 0xd7, 0x49, 0x0c, - 0xd4, 0xe9, 0xc1, 0x99, 0x29, 0x14, 0xa4, 0x42, 0xf1, 0x01, 0x3e, 0x62, 0x06, 0x52, 0xd4, 0xe9, - 0x27, 0x7a, 0x1d, 0xca, 0x87, 0x86, 0x3d, 0xc1, 0xcc, 0x00, 0x66, 0xcb, 0x88, 0x23, 0x7f, 0xbd, - 0xf0, 0xa6, 0xa2, 0x7d, 0x5c, 0x80, 0x67, 0x38, 0x6f, 0xd8, 0x0e, 0x8c, 0x2f, 0xd6, 0x16, 0x43, - 0x3b, 0x2b, 0x1d, 0xcb, 0xce, 0x2e, 0x02, 0xc8, 0x62, 0xc5, 0x32, 0x45, 0x7a, 0x0f, 0xb5, 0x64, - 0x26, 0x4d, 0xa0, 0x7a, 0x5c, 0x13, 0xd0, 0xfe, 0x5c, 0x82, 0x96, 0x90, 0x1d, 0xc5, 0x60, 0x01, - 0xe4, 0x02, 0x54, 0xc3, 0xd4, 0x23, 0xd4, 0x10, 0x01, 0xd0, 0x2a, 0xd4, 0x62, 0xee, 0x23, 0xe4, - 0x10, 0x07, 0xcd, 0x25, 0x0c, 0x59, 0x48, 0x94, 0x62, 0x85, 0xc4, 0x45, 0x80, 0xa1, 0x3d, 0x21, - 0x07, 0xbd, 0xc0, 0x72, 0xb0, 0xdc, 0x29, 0x83, 0xdc, 0xb5, 0x1c, 0x8c, 0xae, 0x43, 0xbd, 0x6f, - 0xb9, 0xb6, 0x37, 0xea, 0x8d, 0x8d, 0xe0, 0x80, 0xb4, 0x17, 0x73, 0x1d, 0xe6, 0xa6, 0x85, 0x6d, - 0xf3, 0x06, 0xc3, 0xd5, 0x6b, 0x7c, 0xce, 0x3e, 0x9d, 0x82, 0x9e, 0x85, 0x9a, 0x3b, 0x71, 0x7a, - 0xde, 0xb0, 0xe7, 0x7b, 0x8f, 0xa8, 0xcb, 0x31, 0x12, 0xee, 0xc4, 0x79, 0x7f, 0xa8, 0x7b, 0x8f, - 0x68, 0xe8, 0xaf, 0xd2, 0x24, 0x40, 0x6c, 0x6f, 0x44, 0xda, 0x95, 0xb9, 0xd6, 0x8f, 0x26, 0xd0, - 0xd9, 0x26, 0x35, 0x33, 0x36, 0xbb, 0x3a, 0xdf, 0xec, 0x70, 0x02, 0x7a, 0x11, 0x9a, 0x03, 0xcf, - 0x19, 0x1b, 0x4c, 0x42, 0x37, 0x7d, 0xcf, 0x61, 0xfe, 0x56, 0xd4, 0x53, 0x50, 0xb4, 0x05, 0x35, - 0xcb, 0x35, 0xf1, 0x63, 0xe1, 0x94, 0x35, 0x46, 0x47, 0xcb, 0x52, 0x39, 0x23, 0xb4, 0x4b, 0x71, - 0x99, 0xd2, 0xc1, 0x92, 0x9f, 0x84, 0x66, 0x24, 0xe9, 0xdb, 0xc4, 0xfa, 0x08, 0xb7, 0xeb, 0x5c, - 0x8b, 0x02, 0xd6, 0xb5, 0x3e, 0xc2, 0xb4, 0x48, 0xb6, 0x5c, 0x82, 0xfd, 0x40, 0xb6, 0x2c, 0xed, - 0x06, 0x4b, 0x5b, 0x0d, 0x0e, 0x15, 0xbe, 0xa4, 0xfd, 0xbe, 0x00, 0xcd, 0x24, 0x21, 0xd4, 0x86, - 0xa5, 0x21, 0x83, 0x48, 0xeb, 0x91, 0xbf, 0x94, 0x2c, 0x76, 0x8d, 0xbe, 0x4d, 0x63, 0x9e, 0x89, - 0x1f, 0x33, 0xe3, 0xa9, 0xe8, 0x35, 0x0e, 0x63, 0x0b, 0x50, 0x23, 0xe0, 0xdb, 0x63, 0x99, 0xb2, - 0xc8, 0x48, 0x56, 0x19, 0x84, 0xe5, 0xc9, 0x36, 0x2c, 0xf1, 0x6d, 0x48, 0xd3, 0x91, 0xbf, 0x74, - 0xa4, 0x3f, 0xb1, 0x18, 0x55, 0x6e, 0x3a, 0xf2, 0x17, 0x6d, 0x43, 0x9d, 0x2f, 0x39, 0x36, 0x7c, - 0xc3, 0x91, 0x86, 0xf3, 0x7c, 0xa6, 0xbb, 0xdf, 0xc2, 0x47, 0x1f, 0xd0, 0xe8, 0xb1, 0x6f, 0x58, - 0xbe, 0xce, 0x05, 0xbd, 0xcf, 0x66, 0xa1, 0x35, 0x50, 0xf9, 0x2a, 0x43, 0xcb, 0xc6, 0xc2, 0x04, - 0x97, 0x58, 0x32, 0x6e, 0x32, 0xf8, 0x4d, 0xcb, 0xc6, 0xdc, 0xca, 0xc2, 0x2d, 0x30, 0xd1, 0x56, - 0xb8, 0x91, 0x31, 0x08, 0x15, 0xac, 0xf6, 0xf7, 0x22, 0x2c, 0x53, 0x5f, 0x13, 0x6e, 0x77, 0x8a, - 0x68, 0x74, 0x11, 0xc0, 0x24, 0x41, 0x2f, 0x11, 0x91, 0xaa, 0x26, 0x09, 0xf6, 0x78, 0x50, 0x7a, - 0x4b, 0x06, 0x9c, 0x62, 0x7e, 0xad, 0x9b, 0xf2, 0xfd, 0xe9, 0xe4, 0x76, 0xa2, 0x93, 0x80, 0x4b, - 0xd0, 0x20, 0xde, 0xc4, 0x1f, 0xe0, 0x5e, 0xa2, 0x2b, 0xa9, 0x73, 0xe0, 0x5e, 0x76, 0xcc, 0x5c, - 0xcc, 0x3c, 0x91, 0x88, 0x45, 0xb7, 0xa5, 0xd3, 0x25, 0xb8, 0x4a, 0x3a, 0xc1, 0xdd, 0x82, 0x16, - 0x73, 0xbf, 0xde, 0xd8, 0x23, 0xbc, 0xb9, 0x13, 0x5e, 0xab, 0xe5, 0x34, 0xf7, 0x77, 0xc8, 0x68, - 0x5f, 0xa0, 0xea, 0x4d, 0x36, 0x55, 0xfe, 0x12, 0xed, 0x17, 0x05, 0x58, 0x11, 0xcd, 0xe2, 0xe9, - 0x15, 0x9b, 0x97, 0x66, 0x64, 0xd4, 0x2c, 0x3e, 0xa1, 0xfd, 0x2a, 0xcd, 0x51, 0x06, 0x95, 0x33, - 0xca, 0xa0, 0x64, 0x0b, 0xb2, 0x38, 0xd5, 0x82, 0x84, 0x07, 0x0a, 0x4b, 0xc7, 0x38, 0x50, 0xf8, - 0xa7, 0x02, 0x8d, 0x2e, 0x36, 0xfc, 0xc1, 0x81, 0x14, 0xc6, 0x1b, 0xf1, 0x53, 0x94, 0x17, 0x72, - 0x04, 0x9d, 0x98, 0xf2, 0xd5, 0x39, 0x3e, 0xf9, 0x97, 0x02, 0xf5, 0x6f, 0xd3, 0x21, 0xb9, 0xd9, - 0x37, 0xe3, 0x9b, 0x7d, 0x31, 0x67, 0xb3, 0x3a, 0x0e, 0x7c, 0x0b, 0x1f, 0xe2, 0xaf, 0xdc, 0x76, - 0xff, 0xaa, 0x40, 0xa7, 0x7b, 0xe4, 0x0e, 0x74, 0xee, 0x51, 0xa7, 0x37, 0xfb, 0x4b, 0xd0, 0x38, - 0x4c, 0x34, 0x4a, 0x05, 0x16, 0xfe, 0xeb, 0x87, 0xf1, 0x4e, 0x49, 0x07, 0x55, 0xd6, 0x43, 0x62, - 0xb3, 0x32, 0xc0, 0xbd, 0x94, 0xc5, 0x75, 0x8a, 0x39, 0x16, 0x20, 0x5a, 0x7e, 0x12, 0xa8, 0xf9, - 0xb0, 0x9c, 0x81, 0x87, 0xce, 0xc3, 0x92, 0x68, 0xca, 0x44, 0x22, 0xe3, 0x7e, 0x68, 0x52, 0xed, - 0x44, 0xc7, 0x0a, 0x96, 0x39, 0x5d, 0x04, 0x99, 0xe8, 0x39, 0xa8, 0x85, 0xd5, 0xb3, 0x39, 0xa5, - 0x1e, 0x93, 0x68, 0x7f, 0x52, 0x60, 0xe5, 0x3d, 0xc3, 0x35, 0xbd, 0xe1, 0xf0, 0xf4, 0x92, 0xdb, - 0x82, 0x44, 0x61, 0x3d, 0x6f, 0xcb, 0x9e, 0x98, 0x84, 0x5e, 0x86, 0x33, 0x3e, 0x8f, 0x60, 0x66, - 0x52, 0xb4, 0x45, 0x5d, 0x95, 0x03, 0xa1, 0xc8, 0x7e, 0x57, 0x00, 0x44, 0xa3, 0xee, 0x0d, 0xc3, - 0x36, 0xdc, 0x01, 0x3e, 0x39, 0xeb, 0x97, 0xa1, 0x99, 0xc8, 0x15, 0xe1, 0xcd, 0x46, 0x3c, 0x59, - 0x10, 0x74, 0x0b, 0x9a, 0x7d, 0x4e, 0xaa, 0xe7, 0x63, 0x83, 0x78, 0x2e, 0x0b, 0x82, 0xcd, 0xec, - 0xee, 0xfc, 0xae, 0x6f, 0x8d, 0x46, 0xd8, 0xdf, 0xf2, 0x5c, 0x93, 0x47, 0xeb, 0x46, 0x5f, 0xb2, - 0x49, 0xa7, 0x52, 0xe5, 0x44, 0x89, 0x53, 0x76, 0x85, 0x10, 0x66, 0x4e, 0x26, 0x0a, 0x82, 0x0d, - 0x3b, 0x12, 0x44, 0x14, 0x35, 0x55, 0x3e, 0xd0, 0xcd, 0x3f, 0x9c, 0xc9, 0x48, 0x64, 0xda, 0x1f, - 0x14, 0x40, 0x61, 0x7f, 0xc4, 0x3a, 0x12, 0x66, 0x61, 0xe9, 0xa9, 0x4a, 0x46, 0xf0, 0xbe, 0x00, - 0x55, 0x53, 0xce, 0x14, 0x1e, 0x11, 0x01, 0xa8, 0xcf, 0xf0, 0x6d, 0xf4, 0x68, 0xd6, 0xc3, 0xa6, - 0xac, 0xb6, 0x39, 0xf0, 0x36, 0x83, 0x25, 0xf3, 0x60, 0x29, 0x9d, 0x07, 0xe3, 0x67, 0x0f, 0xe5, - 0xc4, 0xd9, 0x83, 0xf6, 0x49, 0x01, 0x54, 0x16, 0xd1, 0xb6, 0xa2, 0xc6, 0x64, 0x2e, 0xa6, 0x2f, - 0x41, 0x43, 0xdc, 0xfd, 0x25, 0x18, 0xaf, 0x3f, 0x8c, 0x2d, 0x86, 0xae, 0xc2, 0x59, 0x8e, 0xe4, - 0x63, 0x32, 0xb1, 0xa3, 0x42, 0x93, 0x57, 0x7d, 0xe8, 0x21, 0x0f, 0xa5, 0x74, 0x48, 0xce, 0xb8, - 0x07, 0x2b, 0x23, 0xdb, 0xeb, 0x1b, 0x76, 0x2f, 0xa9, 0x9e, 0x9c, 0xa6, 0x6a, 0xda, 0xe2, 0xcf, - 0xf2, 0xe9, 0xdd, 0xb8, 0x0e, 0x09, 0xda, 0xa1, 0xad, 0x2e, 0x7e, 0x10, 0x16, 0x02, 0xe2, 0x58, - 0x79, 0x9e, 0x3a, 0xa0, 0x4e, 0x27, 0xca, 0x3f, 0xed, 0x63, 0x05, 0x5a, 0xa9, 0xe3, 0xc3, 0x74, - 0xc3, 0xa4, 0x4c, 0x37, 0x4c, 0x6f, 0x42, 0x99, 0x76, 0x11, 0x3c, 0xde, 0x35, 0xb3, 0x8b, 0xf9, - 0xe4, 0xaa, 0x3a, 0x9f, 0x80, 0xae, 0xc0, 0x72, 0xc6, 0x45, 0x93, 0xb0, 0x01, 0x34, 0x7d, 0xcf, - 0xa4, 0xfd, 0xb1, 0x04, 0xb5, 0x98, 0x3c, 0x66, 0xf4, 0x7a, 0xf3, 0x9c, 0x07, 0xa5, 0xb6, 0x57, - 0x9c, 0xde, 0x5e, 0xce, 0x4d, 0x0b, 0xb5, 0x3b, 0x07, 0x3b, 0xbc, 0x4a, 0x16, 0x25, 0xbb, 0x83, - 0x1d, 0xd6, 0x7c, 0x50, 0x93, 0x9c, 0x38, 0xbc, 0x4b, 0xe3, 0xee, 0xb4, 0xe4, 0x4e, 0x1c, 0xd6, - 0xa3, 0x25, 0x1b, 0x84, 0xa5, 0x27, 0x34, 0x08, 0x95, 0x64, 0x83, 0x90, 0xf0, 0xa3, 0x6a, 0xda, - 0x8f, 0xe6, 0x6d, 0xbf, 0xae, 0xc2, 0xf2, 0xc0, 0xc7, 0x46, 0x80, 0xcd, 0x1b, 0x47, 0x5b, 0xe1, - 0x50, 0xbb, 0xc6, 0xf2, 0x6a, 0xd6, 0x10, 0xba, 0x19, 0x9d, 0xa3, 0x70, 0x2d, 0xd7, 0x99, 0x96, - 0xb3, 0xfb, 0x0f, 0xa1, 0x1b, 0xae, 0x64, 0x19, 0x9e, 0xd9, 0x5f, 0xba, 0xf1, 0x6b, 0x9c, 0xa8, - 0xf1, 0x7b, 0x0e, 0x6a, 0xd1, 0x69, 0x02, 0x69, 0x37, 0x79, 0xe4, 0x0b, 0x8f, 0x13, 0x48, 0x22, - 0x18, 0xb4, 0x92, 0xc1, 0xe0, 0x6f, 0x45, 0x68, 0x46, 0x25, 0xff, 0xdc, 0xa1, 0x60, 0x9e, 0x0b, - 0xd3, 0x3d, 0x50, 0xa3, 0x84, 0xca, 0xa4, 0xf4, 0xc4, 0xae, 0x25, 0x7d, 0x42, 0xdf, 0x1a, 0xa7, - 0x7c, 0x2e, 0x71, 0x06, 0x5a, 0x3a, 0xd6, 0x19, 0xe8, 0x29, 0xef, 0x96, 0xae, 0xc1, 0xb9, 0x30, - 0x89, 0x26, 0xb6, 0xcd, 0x2b, 0xea, 0xb3, 0x72, 0x70, 0x3f, 0xbe, 0xfd, 0x1c, 0x37, 0x5e, 0xca, - 0x73, 0xe3, 0xb4, 0x1a, 0x2b, 0x53, 0x6a, 0x9c, 0xbe, 0xe2, 0xaa, 0x66, 0x5d, 0x71, 0xdd, 0x83, - 0xe5, 0x7b, 0x2e, 0x99, 0xf4, 0xc9, 0xc0, 0xb7, 0xfa, 0x38, 0x2c, 0x2d, 0xe7, 0x51, 0x6b, 0x07, - 0x2a, 0xa9, 0xea, 0x34, 0xfc, 0xd7, 0x7e, 0xac, 0xc0, 0xca, 0xf4, 0xba, 0xcc, 0x62, 0xa2, 0x60, - 0xa0, 0x24, 0x82, 0xc1, 0x77, 0x61, 0x39, 0x5a, 0x3e, 0x59, 0xf7, 0xe6, 0x54, 0x76, 0x19, 0x8c, - 0xeb, 0x28, 0x5a, 0x43, 0xc2, 0xb4, 0x7f, 0x2b, 0xe1, 0x29, 0x23, 0x85, 0x8d, 0xd8, 0xd9, 0x29, - 0x4d, 0x50, 0x9e, 0x6b, 0x5b, 0x6e, 0xd8, 0xa2, 0x8a, 0x3d, 0x72, 0xa0, 0x68, 0x51, 0xdf, 0x83, - 0x96, 0x40, 0x0a, 0xf3, 0xcc, 0x9c, 0x95, 0x55, 0x93, 0xcf, 0x0b, 0x33, 0xcc, 0x65, 0x68, 0x7a, - 0xc3, 0x61, 0x9c, 0x1e, 0x0f, 0x94, 0x0d, 0x01, 0x15, 0x04, 0xbf, 0x05, 0xaa, 0x44, 0x3b, 0x6e, - 0x66, 0x6b, 0x89, 0x89, 0x61, 0x85, 0xf6, 0x23, 0x05, 0xda, 0xc9, 0x3c, 0x17, 0xdb, 0xfe, 0xf1, - 0xeb, 0xb4, 0x6f, 0x24, 0xaf, 0x83, 0x2e, 0x3f, 0x81, 0x9f, 0x88, 0x8e, 0x38, 0x4f, 0x58, 0xff, - 0x26, 0x54, 0xc3, 0xf6, 0x01, 0xd5, 0x60, 0xe9, 0x9e, 0x7b, 0xcb, 0xf5, 0x1e, 0xb9, 0xea, 0x02, - 0x5a, 0x82, 0xe2, 0x75, 0xdb, 0x56, 0x15, 0xd4, 0x80, 0x6a, 0x37, 0xf0, 0xb1, 0xe1, 0x58, 0xee, - 0x48, 0x2d, 0xa0, 0x26, 0xc0, 0x7b, 0x16, 0x09, 0x3c, 0xdf, 0x1a, 0x18, 0xb6, 0x5a, 0x5c, 0xff, - 0x08, 0x9a, 0x49, 0xaf, 0x47, 0x75, 0xa8, 0xec, 0x79, 0xc1, 0xbb, 0x8f, 0x2d, 0x12, 0xa8, 0x0b, - 0x14, 0x7f, 0xcf, 0x0b, 0xf6, 0x7d, 0x4c, 0xb0, 0x1b, 0xa8, 0x0a, 0x02, 0x58, 0x7c, 0xdf, 0xdd, - 0xb6, 0xc8, 0x03, 0xb5, 0x80, 0x96, 0x45, 0x52, 0x36, 0xec, 0x5d, 0xe1, 0x4a, 0x6a, 0x91, 0x4e, - 0x0f, 0xff, 0x4a, 0x48, 0x85, 0x7a, 0x88, 0xb2, 0xb3, 0x7f, 0x4f, 0x2d, 0xa3, 0x2a, 0x94, 0xf9, - 0xe7, 0xe2, 0xba, 0x09, 0x6a, 0xba, 0xa2, 0xa4, 0x6b, 0xf2, 0x4d, 0x84, 0x20, 0x75, 0x81, 0xee, - 0x4c, 0x94, 0xf4, 0xaa, 0x82, 0x5a, 0x50, 0x8b, 0x15, 0xc8, 0x6a, 0x81, 0x02, 0x76, 0xfc, 0xf1, - 0x40, 0x94, 0xca, 0x9c, 0x05, 0xaa, 0xf7, 0x6d, 0x2a, 0x89, 0xd2, 0xfa, 0x0d, 0xa8, 0xc8, 0x70, - 0x44, 0x51, 0x85, 0x88, 0xe8, 0xaf, 0xba, 0x80, 0xce, 0x40, 0x23, 0x71, 0x31, 0xaf, 0x2a, 0x08, - 0x41, 0x33, 0xf9, 0x60, 0x44, 0x2d, 0x6c, 0xfe, 0xaa, 0x0e, 0xc0, 0xeb, 0x35, 0xcf, 0xf3, 0x4d, - 0x34, 0x06, 0xb4, 0x83, 0x03, 0x9a, 0x8b, 0x3c, 0x57, 0xe6, 0x11, 0x82, 0xae, 0xe6, 0xbf, 0x5d, - 0x48, 0xa1, 0x0a, 0x56, 0x3b, 0x79, 0xad, 0x6b, 0x0a, 0x5d, 0x5b, 0x40, 0x0e, 0xa3, 0x78, 0xd7, - 0x72, 0xf0, 0x5d, 0x6b, 0xf0, 0x20, 0x2c, 0xf4, 0xf2, 0x29, 0xa6, 0x50, 0x25, 0xc5, 0x54, 0xd8, - 0x17, 0x3f, 0xdd, 0xc0, 0xb7, 0xdc, 0x91, 0xbc, 0xde, 0xd3, 0x16, 0xd0, 0xc3, 0xd4, 0x5b, 0x0d, - 0x49, 0x70, 0x73, 0x9e, 0xe7, 0x19, 0x27, 0x23, 0x69, 0x43, 0x2b, 0xf5, 0xb0, 0x0b, 0xad, 0x67, - 0xdf, 0x10, 0x66, 0x3d, 0x42, 0xeb, 0xbc, 0x3c, 0x17, 0x6e, 0x48, 0xcd, 0x82, 0x66, 0xf2, 0xf1, - 0x12, 0xfa, 0xff, 0xbc, 0x05, 0xa6, 0xde, 0x28, 0x74, 0xd6, 0xe7, 0x41, 0x0d, 0x49, 0xdd, 0xe7, - 0xf6, 0x34, 0x8b, 0x54, 0xe6, 0xfb, 0x90, 0xce, 0x93, 0x6e, 0x56, 0xb5, 0x05, 0xf4, 0x3d, 0x38, - 0x33, 0xf5, 0x92, 0x02, 0xbd, 0x92, 0xdd, 0xaf, 0x67, 0x3f, 0xb8, 0x98, 0x45, 0xe1, 0x7e, 0xda, - 0x1b, 0xf2, 0xb9, 0x9f, 0x7a, 0x75, 0x34, 0x3f, 0xf7, 0xb1, 0xe5, 0x9f, 0xc4, 0xfd, 0xb1, 0x29, - 0x4c, 0x98, 0xdb, 0xa4, 0x3b, 0x87, 0x57, 0xb3, 0x48, 0xe4, 0x3e, 0xe7, 0xe8, 0x6c, 0xcc, 0x8b, - 0x1e, 0xb7, 0xae, 0xe4, 0x8b, 0x81, 0x6c, 0xa1, 0x65, 0xbe, 0x72, 0xc8, 0xb6, 0xae, 0xec, 0x07, - 0x08, 0xda, 0x02, 0xba, 0x9b, 0x88, 0x86, 0xe8, 0xc5, 0x3c, 0xe5, 0x24, 0xcf, 0x13, 0x66, 0xc9, - 0xad, 0x07, 0xb0, 0x83, 0x83, 0x3b, 0x38, 0xf0, 0xad, 0x01, 0x49, 0x2f, 0x2a, 0x7e, 0x22, 0x04, - 0xb9, 0xe8, 0x4b, 0x33, 0xf1, 0x42, 0xb6, 0xfb, 0x50, 0xdb, 0xc1, 0x81, 0x38, 0x1c, 0x22, 0x28, - 0x77, 0xa6, 0xc4, 0x90, 0x24, 0xd6, 0x66, 0x23, 0xc6, 0x23, 0x4a, 0xea, 0x01, 0x03, 0xca, 0x95, - 0xed, 0xf4, 0xb3, 0x8a, 0xec, 0x88, 0x92, 0xf3, 0x22, 0x42, 0x5b, 0xd8, 0xfc, 0xb4, 0x0e, 0x55, - 0x96, 0x22, 0x68, 0xea, 0xf9, 0x5f, 0x86, 0x78, 0x0a, 0x19, 0xe2, 0x43, 0x68, 0xa5, 0xee, 0xc3, - 0xb3, 0xf5, 0x99, 0x7d, 0x69, 0x3e, 0xcb, 0xe4, 0xfb, 0x80, 0xa6, 0x6f, 0xb4, 0xb3, 0x43, 0x45, - 0xee, 0xcd, 0xf7, 0x2c, 0x1a, 0x1f, 0xf0, 0x27, 0x25, 0x61, 0xf5, 0xfa, 0x52, 0x9e, 0xb7, 0xa6, - 0x4e, 0x2e, 0xbf, 0xf8, 0x40, 0xfa, 0xf4, 0x13, 0xcd, 0x87, 0xd0, 0x4a, 0xdd, 0xf3, 0x64, 0x6b, - 0x37, 0xfb, 0x32, 0x68, 0xd6, 0xea, 0x9f, 0x63, 0x44, 0x36, 0x61, 0x39, 0xe3, 0xf4, 0x1e, 0x65, - 0x66, 0x91, 0xfc, 0x63, 0xfe, 0xd9, 0x1b, 0x6a, 0x24, 0x5c, 0x0a, 0xad, 0xe5, 0x31, 0x99, 0x7e, - 0x36, 0xdb, 0x79, 0x65, 0xbe, 0x37, 0xb6, 0xe1, 0x86, 0xba, 0xb0, 0xc8, 0x2f, 0x8e, 0xd0, 0xf3, - 0xd9, 0xcd, 0x49, 0xec, 0x52, 0xa9, 0x33, 0xeb, 0xea, 0x89, 0x4c, 0xec, 0x80, 0xb0, 0x45, 0xcb, - 0x2c, 0x5a, 0xa2, 0xcc, 0x7b, 0xc7, 0xf8, 0x6d, 0x4f, 0x67, 0xf6, 0x05, 0x8f, 0x5c, 0xf4, 0xfb, - 0x80, 0x78, 0xc9, 0xe7, 0x0e, 0xad, 0xd1, 0xc4, 0x37, 0xb8, 0x99, 0xe6, 0xc5, 0xc0, 0x69, 0x54, - 0x49, 0xf1, 0xb5, 0x63, 0xcc, 0x08, 0xc5, 0xf4, 0xb4, 0x73, 0xe6, 0x8d, 0xd7, 0xef, 0x6f, 0x8e, - 0xac, 0xe0, 0x60, 0xd2, 0xa7, 0xc6, 0x70, 0x85, 0x63, 0xbe, 0x6a, 0x79, 0xe2, 0xeb, 0x8a, 0xe4, - 0xf2, 0x0a, 0x5b, 0xe9, 0x0a, 0x13, 0xe4, 0xb8, 0xdf, 0x5f, 0x64, 0xbf, 0xd7, 0xfe, 0x1b, 0x00, - 0x00, 0xff, 0xff, 0xcc, 0xd9, 0x54, 0x42, 0x34, 0x32, 0x00, 0x00, + 0xd5, 0xec, 0xd9, 0xe7, 0xcd, 0xd6, 0x2a, 0x4a, 0xd4, 0x78, 0x3e, 0xc9, 0xa6, 0x5b, 0x96, 0xcd, + 0x8f, 0xb6, 0x29, 0x99, 0xf2, 0x67, 0xd8, 0x5f, 0x6c, 0x20, 0x12, 0x69, 0xd1, 0x8c, 0x24, 0x9a, + 0xe9, 0x91, 0x9c, 0x40, 0x30, 0x30, 0xe9, 0x99, 0xae, 0x19, 0x36, 0xd4, 0xcb, 0xa8, 0xab, 0x87, + 0x32, 0x9d, 0x6b, 0x10, 0x20, 0xdb, 0x21, 0xc8, 0x35, 0x08, 0x72, 0x48, 0x0e, 0x01, 0x62, 0xe4, + 0x92, 0x4b, 0x80, 0x20, 0xc8, 0x2d, 0xd7, 0x00, 0xf9, 0x01, 0x39, 0xe6, 0x90, 0x5c, 0x73, 0x08, + 0x72, 0x09, 0x6a, 0xeb, 0x6d, 0xba, 0xc5, 0x11, 0x69, 0x79, 0x01, 0x72, 0xeb, 0x7a, 0xf5, 0xaa, + 0xde, 0xab, 0xb7, 0xbf, 0xaa, 0x86, 0x33, 0x0f, 0x67, 0xd8, 0x3f, 0x1a, 0x8c, 0x3c, 0xcf, 0x37, + 0x37, 0xa6, 0xbe, 0x17, 0x78, 0x08, 0x39, 0x96, 0x7d, 0x38, 0x23, 0x7c, 0xb4, 0xc1, 0xe6, 0x7b, + 0xcd, 0x91, 0xe7, 0x38, 0x9e, 0xcb, 0x61, 0xbd, 0x66, 0x1c, 0xa3, 0xd7, 0xb6, 0xdc, 0x00, 0xfb, + 0xae, 0x61, 0xcb, 0x59, 0x32, 0x3a, 0xc0, 0x8e, 0x21, 0x46, 0xaa, 0x69, 0x04, 0x46, 0x7c, 0x7f, + 0xed, 0x3b, 0x0a, 0xac, 0xf4, 0x0f, 0xbc, 0x47, 0x5b, 0x9e, 0x6d, 0xe3, 0x51, 0x60, 0x79, 0x2e, + 0xd1, 0xf1, 0xc3, 0x19, 0x26, 0x01, 0xba, 0x0a, 0xa5, 0xa1, 0x41, 0x70, 0x57, 0x59, 0x55, 0xd6, + 0x1a, 0x9b, 0x17, 0x36, 0x12, 0x9c, 0x08, 0x16, 0xee, 0x90, 0xc9, 0x0d, 0x83, 0x60, 0x9d, 0x61, + 0x22, 0x04, 0x25, 0x73, 0xb8, 0xbb, 0xdd, 0x2d, 0xac, 0x2a, 0x6b, 0x45, 0x9d, 0x7d, 0xa3, 0x17, + 0xa0, 0x35, 0x0a, 0xf7, 0xde, 0xdd, 0x26, 0xdd, 0xe2, 0x6a, 0x71, 0xad, 0xa8, 0x27, 0x81, 0xda, + 0x5f, 0x15, 0x38, 0x3f, 0xc7, 0x06, 0x99, 0x7a, 0x2e, 0xc1, 0xe8, 0x1a, 0x54, 0x48, 0x60, 0x04, + 0x33, 0x22, 0x38, 0xf9, 0x9f, 0x4c, 0x4e, 0xfa, 0x0c, 0x45, 0x17, 0xa8, 0xf3, 0x64, 0x0b, 0x19, + 0x64, 0xd1, 0x6b, 0x70, 0xd6, 0x72, 0xef, 0x60, 0xc7, 0xf3, 0x8f, 0x06, 0x53, 0xec, 0x8f, 0xb0, + 0x1b, 0x18, 0x13, 0x2c, 0x79, 0x5c, 0x96, 0x73, 0xfb, 0xd1, 0x14, 0x7a, 0x03, 0xce, 0x73, 0x2d, + 0x11, 0xec, 0x1f, 0x5a, 0x23, 0x3c, 0x30, 0x0e, 0x0d, 0xcb, 0x36, 0x86, 0x36, 0xee, 0x96, 0x56, + 0x8b, 0x6b, 0x35, 0xfd, 0x1c, 0x9b, 0xee, 0xf3, 0xd9, 0xeb, 0x72, 0x52, 0xfb, 0xa5, 0x02, 0xe7, + 0xe8, 0x09, 0xf7, 0x0d, 0x3f, 0xb0, 0x9e, 0x82, 0x9c, 0x35, 0x68, 0xc6, 0xcf, 0xd6, 0x2d, 0xb2, + 0xb9, 0x04, 0x8c, 0xe2, 0x4c, 0x25, 0x79, 0x2a, 0x93, 0x12, 0x3b, 0x66, 0x02, 0xa6, 0xfd, 0x42, + 0x18, 0x44, 0x9c, 0xcf, 0xd3, 0x28, 0x22, 0x4d, 0xb3, 0x30, 0x4f, 0xf3, 0x04, 0x6a, 0xd0, 0xfe, + 0xa6, 0xc0, 0xb9, 0xdb, 0x9e, 0x61, 0x46, 0x06, 0xf3, 0xd9, 0x8b, 0xf3, 0x1d, 0xa8, 0x70, 0xef, + 0xea, 0x96, 0x18, 0xad, 0xcb, 0x49, 0x5a, 0xc2, 0xf3, 0x22, 0x0e, 0xfb, 0x0c, 0xa0, 0x8b, 0x45, + 0xe8, 0x32, 0xb4, 0x7d, 0x3c, 0xb5, 0xad, 0x91, 0x31, 0x70, 0x67, 0xce, 0x10, 0xfb, 0xdd, 0xf2, + 0xaa, 0xb2, 0x56, 0xd6, 0x5b, 0x02, 0xba, 0xc7, 0x80, 0xda, 0x4f, 0x15, 0xe8, 0xea, 0xd8, 0xc6, + 0x06, 0xc1, 0x9f, 0xe7, 0x61, 0x57, 0xa0, 0xe2, 0x7a, 0x26, 0xde, 0xdd, 0x66, 0x87, 0x2d, 0xea, + 0x62, 0xa4, 0xfd, 0x4b, 0x81, 0xb3, 0x3b, 0x38, 0xa0, 0x5a, 0xb7, 0x48, 0x60, 0x8d, 0x42, 0xb3, + 0x7e, 0x07, 0x8a, 0x3e, 0x7e, 0x28, 0x38, 0x7b, 0x39, 0xc9, 0x59, 0x18, 0xa4, 0xb2, 0x56, 0xea, + 0x74, 0x1d, 0x7a, 0x1e, 0x9a, 0xa6, 0x63, 0x0f, 0x46, 0x07, 0x86, 0xeb, 0x62, 0x9b, 0xdb, 0x4d, + 0x5d, 0x6f, 0x98, 0x8e, 0xbd, 0x25, 0x40, 0xe8, 0x59, 0x00, 0x82, 0x27, 0x0e, 0x76, 0x83, 0x28, + 0xae, 0xc4, 0x20, 0x68, 0x1d, 0xce, 0x8c, 0x7d, 0xcf, 0x19, 0x90, 0x03, 0xc3, 0x37, 0x07, 0x36, + 0x36, 0x4c, 0xec, 0x33, 0xee, 0x6b, 0x7a, 0x87, 0x4e, 0xf4, 0x29, 0xfc, 0x36, 0x03, 0xa3, 0x6b, + 0x50, 0x26, 0x23, 0x6f, 0x8a, 0x99, 0x0e, 0xda, 0x9b, 0x17, 0x37, 0xe6, 0xe3, 0xee, 0xc6, 0xb6, + 0x11, 0x18, 0x7d, 0x8a, 0xa4, 0x73, 0x5c, 0xed, 0x07, 0x05, 0x6e, 0x84, 0x5f, 0x70, 0x9f, 0x8e, + 0x19, 0x6a, 0xf9, 0xd3, 0x31, 0xd4, 0x4a, 0x96, 0xa1, 0xfe, 0x31, 0x32, 0xd4, 0x2f, 0xba, 0x40, + 0x22, 0x63, 0x2e, 0x27, 0x8c, 0xf9, 0x57, 0x0a, 0x3c, 0xb3, 0x83, 0x83, 0x90, 0x7d, 0x6a, 0x9b, + 0xf8, 0x0b, 0x1a, 0xa8, 0x3f, 0x51, 0xa0, 0x97, 0xc5, 0xeb, 0x69, 0x82, 0xf5, 0x7d, 0x58, 0x09, + 0x69, 0x0c, 0x4c, 0x4c, 0x46, 0xbe, 0x35, 0x65, 0x6a, 0x64, 0xee, 0xd7, 0xd8, 0xbc, 0x94, 0xe5, + 0x16, 0x69, 0x0e, 0xce, 0x85, 0x5b, 0x6c, 0xc7, 0x76, 0xd0, 0x7e, 0xa4, 0xc0, 0x39, 0xea, 0xee, + 0xc2, 0x3f, 0xdd, 0xb1, 0x77, 0x72, 0xb9, 0x26, 0x3d, 0xbf, 0x30, 0xe7, 0xf9, 0x0b, 0xc8, 0x98, + 0x55, 0x3e, 0x69, 0x7e, 0x4e, 0x23, 0xbb, 0xff, 0x83, 0xb2, 0xe5, 0x8e, 0x3d, 0x29, 0xaa, 0xe7, + 0xb2, 0x44, 0x15, 0x27, 0xc6, 0xb1, 0x35, 0x97, 0x73, 0x11, 0x85, 0xa2, 0x53, 0x98, 0x5b, 0xfa, + 0xd8, 0x85, 0x8c, 0x63, 0xff, 0x50, 0x81, 0xf3, 0x73, 0x04, 0x4f, 0x73, 0xee, 0xb7, 0xa1, 0xc2, + 0x02, 0xac, 0x3c, 0xf8, 0x0b, 0x99, 0x07, 0x8f, 0x91, 0xbb, 0x6d, 0x91, 0x40, 0x17, 0x6b, 0x34, + 0x0f, 0xd4, 0xf4, 0x1c, 0x0d, 0xfd, 0x22, 0xec, 0x0f, 0x5c, 0xc3, 0xe1, 0x02, 0xa8, 0xeb, 0x0d, + 0x01, 0xdb, 0x33, 0x1c, 0x8c, 0x9e, 0x81, 0x1a, 0x75, 0xd9, 0x81, 0x65, 0x4a, 0xf5, 0x57, 0x99, + 0x0b, 0x9b, 0x04, 0x5d, 0x04, 0x60, 0x53, 0x86, 0x69, 0xfa, 0x3c, 0x2b, 0xd4, 0xf5, 0x3a, 0x85, + 0x5c, 0xa7, 0x00, 0xed, 0xc7, 0x0a, 0x34, 0x69, 0xcc, 0xbe, 0x83, 0x03, 0x83, 0xea, 0x01, 0xbd, + 0x05, 0x75, 0xdb, 0x33, 0xcc, 0x41, 0x70, 0x34, 0xe5, 0xa4, 0xda, 0x69, 0x59, 0xf3, 0x23, 0xd0, + 0x45, 0x77, 0x8f, 0xa6, 0x58, 0xaf, 0xd9, 0xe2, 0x6b, 0x11, 0x79, 0xcf, 0xb9, 0x72, 0x31, 0xc3, + 0x95, 0xbf, 0x5b, 0x86, 0x95, 0x6f, 0x18, 0xc1, 0xe8, 0x60, 0xdb, 0x91, 0xc9, 0xed, 0xe4, 0x46, + 0x10, 0xc5, 0xb6, 0x42, 0x3c, 0xb6, 0x7d, 0x6a, 0xb1, 0x33, 0xb4, 0xf3, 0x72, 0x96, 0x9d, 0xd3, + 0x06, 0x63, 0xe3, 0x03, 0xa1, 0xaa, 0x98, 0x9d, 0xc7, 0x72, 0x50, 0xe5, 0x24, 0x39, 0x68, 0x0b, + 0x5a, 0xf8, 0xa3, 0x91, 0x3d, 0xa3, 0x3a, 0x67, 0xd4, 0xab, 0x8c, 0xfa, 0xb3, 0x19, 0xd4, 0xe3, + 0x4e, 0xd6, 0x14, 0x8b, 0x76, 0x05, 0x0f, 0x5c, 0xd5, 0x0e, 0x0e, 0x8c, 0x6e, 0x8d, 0xb1, 0xb1, + 0x9a, 0xa7, 0x6a, 0x69, 0x1f, 0x5c, 0xdd, 0x74, 0x84, 0x2e, 0x40, 0x5d, 0x64, 0xbc, 0xdd, 0xed, + 0x6e, 0x9d, 0x89, 0x2f, 0x02, 0x20, 0x03, 0x5a, 0x22, 0x02, 0x09, 0x0e, 0x81, 0x71, 0xf8, 0x76, + 0x16, 0x81, 0x6c, 0x65, 0xc7, 0x39, 0x27, 0xef, 0xba, 0x81, 0x7f, 0xa4, 0x37, 0x49, 0x0c, 0xd4, + 0x1b, 0xc0, 0x99, 0x39, 0x14, 0xa4, 0x42, 0xf1, 0x01, 0x3e, 0x62, 0x06, 0x52, 0xd4, 0xe9, 0x27, + 0x7a, 0x1d, 0xca, 0x87, 0x86, 0x3d, 0xc3, 0xcc, 0x00, 0x8e, 0x97, 0x11, 0x47, 0xfe, 0xff, 0xc2, + 0x9b, 0x8a, 0xf6, 0x87, 0x12, 0x74, 0xc4, 0x14, 0x95, 0x01, 0xf3, 0x8f, 0x0b, 0x50, 0x0f, 0x23, + 0xab, 0xa0, 0x12, 0x01, 0xd0, 0x2a, 0x34, 0x62, 0xd6, 0x21, 0x4c, 0x2e, 0x0e, 0x5a, 0xc8, 0xee, + 0x64, 0x9e, 0x2c, 0xc5, 0xf2, 0xe4, 0x45, 0x80, 0xb1, 0x3d, 0x23, 0x07, 0x83, 0xc0, 0x72, 0xb0, + 0xc8, 0xd3, 0x75, 0x06, 0xb9, 0x6b, 0x39, 0x18, 0x5d, 0x87, 0xe6, 0xd0, 0x72, 0x6d, 0x6f, 0x32, + 0x98, 0x1a, 0xc1, 0x01, 0xe9, 0x56, 0x72, 0xed, 0xe1, 0xa6, 0x85, 0x6d, 0xf3, 0x06, 0xc3, 0xd5, + 0x1b, 0x7c, 0xcd, 0x3e, 0x5d, 0x82, 0x9e, 0x85, 0x86, 0x3b, 0x73, 0x06, 0xde, 0x78, 0xe0, 0x7b, + 0x8f, 0xa8, 0x45, 0x31, 0x12, 0xee, 0xcc, 0x79, 0x7f, 0xac, 0x7b, 0x8f, 0x68, 0x64, 0xab, 0xd3, + 0x18, 0x47, 0x6c, 0x6f, 0x42, 0xba, 0xb5, 0x85, 0xf6, 0x8f, 0x16, 0xd0, 0xd5, 0x26, 0xb6, 0x03, + 0x83, 0xad, 0xae, 0x2f, 0xb6, 0x3a, 0x5c, 0x80, 0x5e, 0x84, 0xf6, 0xc8, 0x73, 0xa6, 0x06, 0x93, + 0xd0, 0x4d, 0xdf, 0x73, 0x98, 0x39, 0x15, 0xf5, 0x14, 0x14, 0x6d, 0x41, 0xc3, 0x72, 0x4d, 0xfc, + 0x91, 0xb0, 0xb9, 0x06, 0xa3, 0xa3, 0x65, 0xd9, 0x1c, 0x23, 0xb4, 0x4b, 0x71, 0x99, 0xd6, 0xc1, + 0x92, 0x9f, 0x84, 0x06, 0x5c, 0x69, 0xba, 0xc4, 0xfa, 0x18, 0x77, 0x9b, 0x5c, 0x8b, 0x02, 0xd6, + 0xb7, 0x3e, 0xc6, 0xb4, 0x06, 0xb4, 0x5c, 0x82, 0xfd, 0x40, 0x56, 0xe4, 0xdd, 0x16, 0x8b, 0xca, + 0x2d, 0x0e, 0x15, 0x96, 0xac, 0xfd, 0xa6, 0x00, 0xed, 0x24, 0x21, 0xd4, 0x85, 0xea, 0x98, 0x41, + 0xa4, 0xf5, 0xc8, 0x21, 0x25, 0x8b, 0x5d, 0xda, 0x1c, 0x0f, 0x18, 0x2f, 0xcc, 0x78, 0x6a, 0x7a, + 0x83, 0xc3, 0xd8, 0x06, 0xd4, 0x08, 0xf8, 0xf1, 0x58, 0x22, 0x28, 0x32, 0x92, 0x75, 0x06, 0x61, + 0x69, 0xa0, 0x0b, 0x55, 0x7e, 0x0c, 0x69, 0x3a, 0x72, 0x48, 0x67, 0x86, 0x33, 0x8b, 0x51, 0xe5, + 0xa6, 0x23, 0x87, 0x68, 0x1b, 0x9a, 0x7c, 0xcb, 0xa9, 0xe1, 0x1b, 0x8e, 0x34, 0x9c, 0xe7, 0x33, + 0x23, 0xeb, 0x2d, 0x7c, 0xf4, 0x01, 0x75, 0x8e, 0x7d, 0xc3, 0xf2, 0x75, 0x2e, 0xe8, 0x7d, 0xb6, + 0x0a, 0xad, 0x81, 0xca, 0x77, 0x19, 0x5b, 0x36, 0x16, 0x26, 0x58, 0x65, 0xb9, 0xa6, 0xcd, 0xe0, + 0x37, 0x2d, 0x1b, 0x73, 0x2b, 0x0b, 0x8f, 0xc0, 0x44, 0x5b, 0xe3, 0x46, 0xc6, 0x20, 0x54, 0xb0, + 0xda, 0x5f, 0x8a, 0xb0, 0x4c, 0x7d, 0x4d, 0xb8, 0xdd, 0x29, 0x02, 0xff, 0x45, 0x00, 0x93, 0x04, + 0x83, 0x44, 0xf0, 0xaf, 0x9b, 0x24, 0xd8, 0xe3, 0xf1, 0xff, 0x2d, 0x19, 0xb7, 0x8b, 0xf9, 0xa5, + 0x5c, 0xca, 0xf7, 0xe7, 0x63, 0xf7, 0x89, 0x1a, 0xdd, 0x4b, 0xd0, 0x22, 0xde, 0xcc, 0x1f, 0xe1, + 0x41, 0xa2, 0xe8, 0x6e, 0x72, 0xe0, 0x5e, 0x76, 0x7a, 0xaa, 0x64, 0x36, 0xdc, 0xb1, 0xf8, 0x5d, + 0x3d, 0x5d, 0xfc, 0xae, 0xa5, 0xe3, 0xf7, 0x2d, 0xe8, 0x30, 0xf7, 0x1b, 0x4c, 0x3d, 0xc2, 0x7b, + 0x17, 0xe1, 0xb5, 0x5a, 0x4e, 0xef, 0x7a, 0x87, 0x4c, 0xf6, 0x05, 0xaa, 0xde, 0x66, 0x4b, 0xe5, + 0x90, 0x68, 0x3f, 0x29, 0xc0, 0x8a, 0xe8, 0x85, 0x4e, 0xaf, 0xd8, 0xbc, 0x8c, 0x2e, 0xa3, 0x66, + 0xf1, 0x31, 0xdd, 0x45, 0x69, 0x81, 0x2c, 0x5f, 0xce, 0xc8, 0xf2, 0xc9, 0x0a, 0xbb, 0x32, 0x57, + 0x61, 0x87, 0xfd, 0x72, 0xf5, 0x09, 0xfa, 0xe5, 0xbf, 0x2b, 0xd0, 0xea, 0x63, 0xc3, 0x1f, 0x1d, + 0x48, 0x61, 0xbc, 0x11, 0xbf, 0x24, 0x78, 0x21, 0x47, 0xd0, 0x89, 0x25, 0x5f, 0x9e, 0xdb, 0x81, + 0x7f, 0x28, 0xd0, 0xfc, 0x3a, 0x9d, 0x92, 0x87, 0x7d, 0x33, 0x7e, 0xd8, 0x17, 0x73, 0x0e, 0xab, + 0xe3, 0xc0, 0xb7, 0xf0, 0x21, 0xfe, 0xd2, 0x1d, 0xf7, 0x4f, 0x0a, 0xf4, 0xfa, 0x47, 0xee, 0x48, + 0xe7, 0x1e, 0x75, 0x7a, 0xb3, 0xbf, 0x04, 0xad, 0xc3, 0x44, 0x1f, 0x50, 0x60, 0xe1, 0xbf, 0x79, + 0x18, 0x6f, 0x04, 0x74, 0x50, 0xe5, 0xdd, 0x84, 0x38, 0xac, 0x0c, 0x70, 0x2f, 0x65, 0x71, 0x9d, + 0x62, 0x8e, 0x05, 0x88, 0x8e, 0x9f, 0x04, 0x6a, 0x3e, 0x2c, 0x67, 0xe0, 0xa1, 0xf3, 0x50, 0x15, + 0x3d, 0x87, 0x48, 0x64, 0xdc, 0x0f, 0x4d, 0xaa, 0x9d, 0xa8, 0x6b, 0xb6, 0xcc, 0xf9, 0x22, 0xc8, + 0x44, 0xcf, 0x41, 0x23, 0x2c, 0x0e, 0xcd, 0x39, 0xf5, 0x98, 0x44, 0xfb, 0xbd, 0x02, 0x2b, 0xef, + 0x19, 0xae, 0xe9, 0x8d, 0xc7, 0xa7, 0x97, 0xdc, 0x16, 0x24, 0xea, 0xc6, 0x45, 0x3b, 0xd2, 0xc4, + 0x22, 0xf4, 0x32, 0x9c, 0xf1, 0x79, 0x04, 0x33, 0x93, 0xa2, 0x2d, 0xea, 0xaa, 0x9c, 0x08, 0x45, + 0xf6, 0xeb, 0x02, 0x20, 0x1a, 0x75, 0x6f, 0x18, 0xb6, 0xe1, 0x8e, 0xf0, 0xc9, 0x59, 0xbf, 0x0c, + 0xed, 0x44, 0xae, 0x08, 0x2f, 0xee, 0xe3, 0xc9, 0x82, 0xa0, 0x5b, 0xd0, 0x1e, 0x72, 0x52, 0x03, + 0x1f, 0x1b, 0xc4, 0x73, 0x59, 0x10, 0x6c, 0x67, 0x37, 0x9f, 0x77, 0x7d, 0x6b, 0x32, 0xc1, 0xfe, + 0x96, 0xe7, 0x9a, 0x3c, 0x5a, 0xb7, 0x86, 0x92, 0x4d, 0xba, 0x94, 0x2a, 0x27, 0x4a, 0x9c, 0xb2, + 0xe9, 0x81, 0x30, 0x73, 0x32, 0x51, 0x10, 0x6c, 0xd8, 0x91, 0x20, 0xa2, 0xa8, 0xa9, 0xf2, 0x89, + 0x7e, 0xfe, 0xdd, 0x43, 0x46, 0x22, 0xd3, 0x7e, 0xab, 0x00, 0x0a, 0xcb, 0x7f, 0xd6, 0x0c, 0x30, + 0x0b, 0x4b, 0x2f, 0x55, 0x32, 0x82, 0xf7, 0x05, 0xa8, 0x9b, 0x72, 0xa5, 0xf0, 0x88, 0x08, 0x40, + 0x7d, 0x86, 0x1f, 0x63, 0x40, 0xb3, 0x1e, 0x36, 0x65, 0xb5, 0xcd, 0x81, 0xb7, 0x19, 0x2c, 0x99, + 0x07, 0x4b, 0xe9, 0x3c, 0x18, 0x6f, 0xad, 0xcb, 0x89, 0xd6, 0x5a, 0xfb, 0xa4, 0x00, 0x2a, 0x8b, + 0x68, 0x5b, 0x51, 0x7f, 0xb7, 0x10, 0xd3, 0x97, 0xa0, 0x25, 0x9e, 0xb6, 0x12, 0x8c, 0x37, 0x1f, + 0xc6, 0x36, 0x43, 0x57, 0xe1, 0x2c, 0x47, 0xf2, 0x31, 0x99, 0xd9, 0x51, 0xa1, 0xc9, 0xab, 0x3e, + 0xf4, 0x90, 0x87, 0x52, 0x3a, 0x25, 0x57, 0xdc, 0x83, 0x95, 0x89, 0xed, 0x0d, 0x0d, 0x7b, 0x90, + 0x54, 0x0f, 0xd7, 0xe1, 0x02, 0x16, 0x7f, 0x96, 0x2f, 0xef, 0xc7, 0x75, 0x48, 0xd0, 0x0e, 0xed, + 0xe4, 0xf0, 0x83, 0xb0, 0x10, 0x10, 0xb7, 0xa6, 0x8b, 0xd4, 0x01, 0x4d, 0xba, 0x50, 0x8e, 0xb4, + 0x9f, 0x29, 0xd0, 0x49, 0xdd, 0x8e, 0xa5, 0x1b, 0x26, 0x65, 0xbe, 0x61, 0x7a, 0x13, 0xca, 0xb4, + 0x8b, 0xe0, 0xf1, 0xae, 0x9d, 0x5d, 0xcc, 0x27, 0x77, 0xd5, 0xf9, 0x02, 0x74, 0x05, 0x96, 0x33, + 0xde, 0x51, 0x84, 0x0d, 0xa0, 0xf9, 0x67, 0x14, 0xed, 0x77, 0x25, 0x68, 0xc4, 0xe4, 0x71, 0x4c, + 0xaf, 0xb7, 0xc8, 0x75, 0x47, 0xea, 0x78, 0xc5, 0xf9, 0xe3, 0xe5, 0x3c, 0x24, 0x50, 0xbb, 0x73, + 0xb0, 0xc3, 0xab, 0x64, 0x51, 0xb2, 0x3b, 0xd8, 0x61, 0xcd, 0x07, 0x35, 0xc9, 0x99, 0xc3, 0xbb, + 0x34, 0xee, 0x4e, 0x55, 0x77, 0xe6, 0xb0, 0x1e, 0x2d, 0xd9, 0x20, 0x54, 0x1f, 0xd3, 0x20, 0xd4, + 0x92, 0x0d, 0x42, 0xc2, 0x8f, 0xea, 0x69, 0x3f, 0x5a, 0xb4, 0xfd, 0xba, 0x0a, 0xcb, 0x23, 0x1f, + 0x1b, 0x01, 0x36, 0x6f, 0x1c, 0x6d, 0x85, 0x53, 0xdd, 0x06, 0xcb, 0xab, 0x59, 0x53, 0xe8, 0x66, + 0x74, 0x4d, 0xc0, 0xb5, 0xdc, 0x64, 0x5a, 0xce, 0xee, 0x3f, 0x84, 0x6e, 0xb8, 0x92, 0x65, 0x78, + 0x66, 0xa3, 0x74, 0xe3, 0xd7, 0x3a, 0x51, 0xe3, 0xf7, 0x1c, 0x34, 0x64, 0xf6, 0xa4, 0xee, 0xde, + 0xe6, 0x91, 0x4f, 0xc6, 0x02, 0x93, 0x24, 0x82, 0x41, 0x27, 0x19, 0x0c, 0xfe, 0x5c, 0x84, 0x76, + 0x54, 0xf2, 0x2f, 0x1c, 0x0a, 0x16, 0x79, 0x0f, 0xdc, 0x03, 0x35, 0x4a, 0xa8, 0x4c, 0x4a, 0x8f, + 0xed, 0x5a, 0xd2, 0x17, 0xd0, 0x9d, 0x69, 0xca, 0xe7, 0x12, 0x57, 0x7c, 0xa5, 0x27, 0xba, 0xe2, + 0x3b, 0xe5, 0xd3, 0xc9, 0x35, 0x38, 0x17, 0x26, 0xd1, 0xc4, 0xb1, 0x79, 0x45, 0x7d, 0x56, 0x4e, + 0xee, 0xc7, 0x8f, 0x9f, 0xe3, 0xc6, 0xd5, 0x3c, 0x37, 0x4e, 0xab, 0xb1, 0x36, 0xa7, 0xc6, 0xf9, + 0x17, 0x9c, 0x7a, 0xd6, 0x0b, 0xce, 0x3d, 0x58, 0xbe, 0xe7, 0x92, 0xd9, 0x90, 0x8c, 0x7c, 0x6b, + 0x88, 0xc3, 0xd2, 0x72, 0x11, 0xb5, 0xf6, 0xa0, 0x96, 0xaa, 0x4e, 0xc3, 0xb1, 0xf6, 0x7d, 0x05, + 0x56, 0xe6, 0xf7, 0x65, 0x16, 0x13, 0x05, 0x03, 0x25, 0x11, 0x0c, 0xbe, 0x09, 0xcb, 0xd1, 0xf6, + 0xc9, 0xba, 0x37, 0xa7, 0xb2, 0xcb, 0x60, 0x5c, 0x47, 0xd1, 0x1e, 0x12, 0xa6, 0xfd, 0x53, 0x09, + 0x2f, 0xd1, 0x28, 0x6c, 0xc2, 0xae, 0x06, 0x69, 0x82, 0xf2, 0x5c, 0xdb, 0x72, 0xc3, 0x16, 0x55, + 0x9c, 0x91, 0x03, 0x45, 0x8b, 0xfa, 0x1e, 0x74, 0x04, 0x52, 0x98, 0x67, 0x16, 0xac, 0xac, 0xda, + 0x7c, 0x5d, 0x98, 0x61, 0x2e, 0x43, 0xdb, 0x1b, 0x8f, 0xe3, 0xf4, 0x78, 0xa0, 0x6c, 0x09, 0xa8, + 0x20, 0xf8, 0x35, 0x50, 0x25, 0xda, 0x93, 0x66, 0xb6, 0x8e, 0x58, 0x18, 0x56, 0x68, 0xdf, 0x53, + 0xa0, 0x9b, 0xcc, 0x73, 0xb1, 0xe3, 0x3f, 0x79, 0x9d, 0xf6, 0x95, 0xe4, 0x6b, 0xc7, 0xe5, 0xc7, + 0xf0, 0x13, 0xd1, 0x11, 0xf7, 0x09, 0xeb, 0x5f, 0x85, 0x7a, 0xd8, 0x3e, 0xa0, 0x06, 0x54, 0xef, + 0xb9, 0xb7, 0x5c, 0xef, 0x91, 0xab, 0x2e, 0xa1, 0x2a, 0x14, 0xaf, 0xdb, 0xb6, 0xaa, 0xa0, 0x16, + 0xd4, 0xfb, 0x81, 0x8f, 0x0d, 0xc7, 0x72, 0x27, 0x6a, 0x01, 0xb5, 0x01, 0xde, 0xb3, 0x48, 0xe0, + 0xf9, 0xd6, 0xc8, 0xb0, 0xd5, 0xe2, 0xfa, 0xc7, 0xd0, 0x4e, 0x7a, 0x3d, 0x6a, 0x42, 0x6d, 0xcf, + 0x0b, 0xde, 0xfd, 0xc8, 0x22, 0x81, 0xba, 0x44, 0xf1, 0xf7, 0xbc, 0x60, 0xdf, 0xc7, 0x04, 0xbb, + 0x81, 0xaa, 0x20, 0x80, 0xca, 0xfb, 0xee, 0xb6, 0x45, 0x1e, 0xa8, 0x05, 0xb4, 0x2c, 0x92, 0xb2, + 0x61, 0xef, 0x0a, 0x57, 0x52, 0x8b, 0x74, 0x79, 0x38, 0x2a, 0x21, 0x15, 0x9a, 0x21, 0xca, 0xce, + 0xfe, 0x3d, 0xb5, 0x8c, 0xea, 0x50, 0xe6, 0x9f, 0x95, 0x75, 0x13, 0xd4, 0x74, 0x45, 0x49, 0xf7, + 0xe4, 0x87, 0x08, 0x41, 0xea, 0x12, 0x3d, 0x99, 0x28, 0xe9, 0x55, 0x05, 0x75, 0xa0, 0x11, 0x2b, + 0x90, 0xd5, 0x02, 0x05, 0xec, 0xf8, 0xd3, 0x91, 0x28, 0x95, 0x39, 0x0b, 0x54, 0xef, 0xdb, 0x54, + 0x12, 0xa5, 0xf5, 0x1b, 0x50, 0x93, 0xe1, 0x88, 0xa2, 0x0a, 0x11, 0xd1, 0xa1, 0xba, 0x84, 0xce, + 0x40, 0x2b, 0xf1, 0xee, 0xac, 0x2a, 0x08, 0x41, 0x3b, 0xf9, 0x3f, 0x84, 0x5a, 0xd8, 0xfc, 0x79, + 0x13, 0x80, 0xd7, 0x6b, 0x9e, 0xe7, 0x9b, 0x68, 0x0a, 0x68, 0x07, 0x07, 0x34, 0x17, 0x79, 0xae, + 0xcc, 0x23, 0x04, 0x5d, 0xcd, 0x7f, 0x9a, 0x4f, 0xa1, 0x0a, 0x56, 0x7b, 0x79, 0xad, 0x6b, 0x0a, + 0x5d, 0x5b, 0x42, 0x0e, 0xa3, 0x78, 0xd7, 0x72, 0xf0, 0x5d, 0x6b, 0xf4, 0x20, 0x2c, 0xf4, 0xf2, + 0x29, 0xa6, 0x50, 0x25, 0xc5, 0x54, 0xd8, 0x17, 0x83, 0x7e, 0xe0, 0x5b, 0xee, 0x44, 0xbe, 0x5e, + 0x69, 0x4b, 0xe8, 0x61, 0xea, 0x57, 0x04, 0x49, 0x70, 0x73, 0x91, 0xbf, 0x0f, 0x4e, 0x46, 0xd2, + 0x86, 0x4e, 0xea, 0xbf, 0x25, 0xb4, 0x9e, 0xfd, 0x00, 0x96, 0xf5, 0x8f, 0x55, 0xef, 0xe5, 0x85, + 0x70, 0x43, 0x6a, 0x16, 0xb4, 0x93, 0xff, 0xe6, 0xa0, 0xff, 0xcd, 0xdb, 0x60, 0xee, 0x09, 0xbe, + 0xb7, 0xbe, 0x08, 0x6a, 0x48, 0xea, 0x3e, 0xb7, 0xa7, 0xe3, 0x48, 0x65, 0xfe, 0xfe, 0xd0, 0x7b, + 0xdc, 0xc3, 0xa1, 0xb6, 0x84, 0xbe, 0x05, 0x67, 0xe6, 0x7e, 0x14, 0x40, 0xaf, 0x64, 0xf7, 0xeb, + 0xd9, 0xff, 0x13, 0x1c, 0x47, 0xe1, 0x7e, 0xda, 0x1b, 0xf2, 0xb9, 0x9f, 0xfb, 0xa9, 0x66, 0x71, + 0xee, 0x63, 0xdb, 0x3f, 0x8e, 0xfb, 0x27, 0xa6, 0x30, 0x63, 0x6e, 0x93, 0xee, 0x1c, 0x5e, 0xcd, + 0x22, 0x91, 0xfb, 0xb7, 0x42, 0x6f, 0x63, 0x51, 0xf4, 0xb8, 0x75, 0x25, 0x1f, 0xc4, 0xb3, 0x85, + 0x96, 0xf9, 0x88, 0x9f, 0x6d, 0x5d, 0xd9, 0xef, 0xeb, 0xda, 0x12, 0xba, 0x9b, 0x88, 0x86, 0xe8, + 0xc5, 0x3c, 0xe5, 0x24, 0xef, 0x13, 0x8e, 0x93, 0xdb, 0x00, 0x60, 0x07, 0x07, 0x77, 0x70, 0xe0, + 0x5b, 0x23, 0x92, 0xde, 0x54, 0x0c, 0x22, 0x04, 0xb9, 0xe9, 0x4b, 0xc7, 0xe2, 0x85, 0x6c, 0x0f, + 0xa1, 0xb1, 0x83, 0x03, 0x71, 0x39, 0x44, 0x50, 0xee, 0x4a, 0x89, 0x21, 0x49, 0xac, 0x1d, 0x8f, + 0x18, 0x8f, 0x28, 0xa9, 0xf7, 0x79, 0x94, 0x2b, 0xdb, 0xf9, 0xbf, 0x06, 0xb2, 0x23, 0x4a, 0xce, + 0x83, 0xbf, 0xb6, 0xb4, 0xf9, 0xef, 0x06, 0xd4, 0x59, 0x8a, 0xa0, 0xa9, 0xe7, 0xbf, 0x19, 0xe2, + 0x29, 0x64, 0x88, 0x0f, 0xa1, 0x93, 0x7a, 0xee, 0xcd, 0xd6, 0x67, 0xf6, 0x9b, 0xf0, 0x71, 0x26, + 0xff, 0x01, 0xff, 0x9b, 0x21, 0xac, 0x2c, 0x5f, 0xca, 0xf3, 0xa4, 0xd4, 0xad, 0xe2, 0xe7, 0x1f, + 0xe4, 0x9e, 0x7e, 0x12, 0xf8, 0x10, 0x3a, 0xa9, 0x37, 0x98, 0x6c, 0xc9, 0x67, 0x3f, 0xd4, 0x1c, + 0xb7, 0xfb, 0x67, 0x18, 0x2d, 0x4d, 0x58, 0xce, 0xb8, 0x59, 0x47, 0x99, 0x11, 0x3e, 0xff, 0x0a, + 0xfe, 0xf8, 0x03, 0xb5, 0x12, 0xe6, 0x8e, 0xd6, 0xf2, 0x98, 0x4c, 0xff, 0xb1, 0xd9, 0x7b, 0x65, + 0xb1, 0xdf, 0x3b, 0xc3, 0x03, 0xf5, 0xa1, 0xc2, 0x1f, 0x75, 0xd0, 0xf3, 0xd9, 0x8d, 0x43, 0xec, + 0xc1, 0xa7, 0x77, 0xdc, 0xb3, 0x10, 0x99, 0xd9, 0x01, 0x61, 0x9b, 0x96, 0x59, 0x24, 0x43, 0x99, + 0x6f, 0x82, 0xf1, 0x97, 0x98, 0xde, 0xf1, 0x8f, 0x2f, 0x72, 0xd3, 0x6f, 0x03, 0xe2, 0xe5, 0x98, + 0x3b, 0xb6, 0x26, 0x33, 0xdf, 0xe0, 0x66, 0x9a, 0x17, 0x9f, 0xe6, 0x51, 0x25, 0xc5, 0xd7, 0x9e, + 0x60, 0x45, 0x28, 0xa6, 0xa7, 0x9d, 0xcf, 0x6e, 0xbc, 0x7e, 0x7f, 0x73, 0x62, 0x05, 0x07, 0xb3, + 0x21, 0x35, 0x86, 0x2b, 0x1c, 0xf3, 0x55, 0xcb, 0x13, 0x5f, 0x57, 0x24, 0x97, 0x57, 0xd8, 0x4e, + 0x57, 0x98, 0x20, 0xa7, 0xc3, 0x61, 0x85, 0x0d, 0xaf, 0xfd, 0x27, 0x00, 0x00, 0xff, 0xff, 0x55, + 0x43, 0xa4, 0x8c, 0xaf, 0x30, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -3663,7 +3580,6 @@ type QueryNodeClient interface { GetTimeTickChannel(ctx context.Context, in *internalpb.GetTimeTickChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) GetStatisticsChannel(ctx context.Context, in *internalpb.GetStatisticsChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) WatchDmChannels(ctx context.Context, in *WatchDmChannelsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) - WatchDeltaChannels(ctx context.Context, in *WatchDeltaChannelsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) LoadSegments(ctx context.Context, in *LoadSegmentsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) ReleaseCollection(ctx context.Context, in *ReleaseCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) ReleasePartitions(ctx context.Context, in *ReleasePartitionsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) @@ -3722,15 +3638,6 @@ func (c *queryNodeClient) WatchDmChannels(ctx context.Context, in *WatchDmChanne return out, nil } -func (c *queryNodeClient) WatchDeltaChannels(ctx context.Context, in *WatchDeltaChannelsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) { - out := new(commonpb.Status) - err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryNode/WatchDeltaChannels", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *queryNodeClient) LoadSegments(ctx context.Context, in *LoadSegmentsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) { out := new(commonpb.Status) err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryNode/LoadSegments", in, out, opts...) @@ -3836,7 +3743,6 @@ type QueryNodeServer interface { GetTimeTickChannel(context.Context, *internalpb.GetTimeTickChannelRequest) (*milvuspb.StringResponse, error) GetStatisticsChannel(context.Context, *internalpb.GetStatisticsChannelRequest) (*milvuspb.StringResponse, error) WatchDmChannels(context.Context, *WatchDmChannelsRequest) (*commonpb.Status, error) - WatchDeltaChannels(context.Context, *WatchDeltaChannelsRequest) (*commonpb.Status, error) LoadSegments(context.Context, *LoadSegmentsRequest) (*commonpb.Status, error) ReleaseCollection(context.Context, *ReleaseCollectionRequest) (*commonpb.Status, error) ReleasePartitions(context.Context, *ReleasePartitionsRequest) (*commonpb.Status, error) @@ -3867,9 +3773,6 @@ func (*UnimplementedQueryNodeServer) GetStatisticsChannel(ctx context.Context, r func (*UnimplementedQueryNodeServer) WatchDmChannels(ctx context.Context, req *WatchDmChannelsRequest) (*commonpb.Status, error) { return nil, status.Errorf(codes.Unimplemented, "method WatchDmChannels not implemented") } -func (*UnimplementedQueryNodeServer) WatchDeltaChannels(ctx context.Context, req *WatchDeltaChannelsRequest) (*commonpb.Status, error) { - return nil, status.Errorf(codes.Unimplemented, "method WatchDeltaChannels not implemented") -} func (*UnimplementedQueryNodeServer) LoadSegments(ctx context.Context, req *LoadSegmentsRequest) (*commonpb.Status, error) { return nil, status.Errorf(codes.Unimplemented, "method LoadSegments not implemented") } @@ -3980,24 +3883,6 @@ func _QueryNode_WatchDmChannels_Handler(srv interface{}, ctx context.Context, de return interceptor(ctx, in, info, handler) } -func _QueryNode_WatchDeltaChannels_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(WatchDeltaChannelsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(QueryNodeServer).WatchDeltaChannels(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/milvus.proto.query.QueryNode/WatchDeltaChannels", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(QueryNodeServer).WatchDeltaChannels(ctx, req.(*WatchDeltaChannelsRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _QueryNode_LoadSegments_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(LoadSegmentsRequest) if err := dec(in); err != nil { @@ -4216,10 +4101,6 @@ var _QueryNode_serviceDesc = grpc.ServiceDesc{ MethodName: "WatchDmChannels", Handler: _QueryNode_WatchDmChannels_Handler, }, - { - MethodName: "WatchDeltaChannels", - Handler: _QueryNode_WatchDeltaChannels_Handler, - }, { MethodName: "LoadSegments", Handler: _QueryNode_LoadSegments_Handler, diff --git a/internal/proxy/query_node_mock_test.go b/internal/proxy/query_node_mock_test.go index 467b57fb9d..edb0cd2b6c 100644 --- a/internal/proxy/query_node_mock_test.go +++ b/internal/proxy/query_node_mock_test.go @@ -75,11 +75,6 @@ func (m *QueryNodeMock) WatchDmChannels(ctx context.Context, req *querypb.WatchD return nil, nil } -// TODO -func (m *QueryNodeMock) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) { - return nil, nil -} - // TODO func (m *QueryNodeMock) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { return nil, nil diff --git a/internal/querycoord/cluster.go b/internal/querycoord/cluster.go index 057208875e..2341a960ed 100644 --- a/internal/querycoord/cluster.go +++ b/internal/querycoord/cluster.go @@ -58,8 +58,6 @@ type Cluster interface { // Channel WatchDmChannels(ctx context.Context, nodeID int64, in *querypb.WatchDmChannelsRequest) error - WatchDeltaChannels(ctx context.Context, nodeID int64, in *querypb.WatchDeltaChannelsRequest) error - HasWatchedDeltaChannel(ctx context.Context, nodeID int64, collectionID UniqueID) bool // Node RegisterNode(ctx context.Context, session *sessionutil.Session, id UniqueID, state nodeState) error @@ -209,48 +207,44 @@ func (c *queryNodeCluster) LoadSegments(ctx context.Context, nodeID int64, in *q if targetNode != nil { collectionID := in.CollectionID - // if node has watched the collection's deltaChannel - // then the node should recover part delete log from dmChanel - if c.HasWatchedDeltaChannel(ctx, nodeID, collectionID) { - // get all deltaChannelInfo of the collection from meta - deltaChannelInfos, err := c.clusterMeta.getDeltaChannelsByCollectionID(collectionID) + // get all deltaChannelInfo of the collection from meta + deltaChannelInfos, err := c.clusterMeta.getDeltaChannelsByCollectionID(collectionID) + if err != nil { + // this case should not happen + // deltaChannelInfos should have been set to meta before executing child tasks + log.Error("loadSegments: failed to get deltaChannelInfo from meta", zap.Error(err)) + return err + } + deltaChannel2Info := make(map[string]*datapb.VchannelInfo, len(deltaChannelInfos)) + for _, info := range deltaChannelInfos { + deltaChannel2Info[info.ChannelName] = info + } + + // check delta channel which should be reloaded + reloadDeltaChannels := make(map[string]struct{}) + for _, segment := range in.Infos { + // convert vChannel to deltaChannel + deltaChannelName, err := funcutil.ConvertChannelName(segment.InsertChannel, Params.CommonCfg.RootCoordDml, Params.CommonCfg.RootCoordDelta) if err != nil { - // this case should not happen - // deltaChannelInfos should have been set to meta before executing child tasks - log.Error("loadSegments: failed to get deltaChannelInfo from meta", zap.Error(err)) return err } - deltaChannel2Info := make(map[string]*datapb.VchannelInfo, len(deltaChannelInfos)) - for _, info := range deltaChannelInfos { - deltaChannel2Info[info.ChannelName] = info - } - // check delta channel which should be reloaded - reloadDeltaChannels := make(map[string]struct{}) - for _, segment := range in.Infos { - // convert vChannel to deltaChannel - deltaChannelName, err := funcutil.ConvertChannelName(segment.InsertChannel, Params.CommonCfg.RootCoordDml, Params.CommonCfg.RootCoordDelta) - if err != nil { - return err - } + reloadDeltaChannels[deltaChannelName] = struct{}{} + } - reloadDeltaChannels[deltaChannelName] = struct{}{} - } - - in.DeltaPositions = make([]*internalpb.MsgPosition, 0) - for deltaChannelName := range reloadDeltaChannels { - if info, ok := deltaChannel2Info[deltaChannelName]; ok { - in.DeltaPositions = append(in.DeltaPositions, info.SeekPosition) - } else { - // this case should not happen - err = fmt.Errorf("loadSegments: can't find deltaChannelInfo, channel name = %s", deltaChannelName) - log.Error(err.Error()) - return err - } + in.DeltaPositions = make([]*internalpb.MsgPosition, 0) + for deltaChannelName := range reloadDeltaChannels { + if info, ok := deltaChannel2Info[deltaChannelName]; ok { + in.DeltaPositions = append(in.DeltaPositions, info.SeekPosition) + } else { + // this case should not happen + err = fmt.Errorf("loadSegments: can't find deltaChannelInfo, channel name = %s", deltaChannelName) + log.Error(err.Error()) + return err } } - err := targetNode.loadSegments(ctx, in) + err = targetNode.loadSegments(ctx, in) if err != nil { log.Warn("loadSegments: queryNode load segments error", zap.Int64("nodeID", nodeID), zap.String("error info", err.Error())) return err @@ -328,33 +322,6 @@ func (c *queryNodeCluster) WatchDmChannels(ctx context.Context, nodeID int64, in return fmt.Errorf("watchDmChannels: can't find QueryNode by nodeID, nodeID = %d", nodeID) } -func (c *queryNodeCluster) WatchDeltaChannels(ctx context.Context, nodeID int64, in *querypb.WatchDeltaChannelsRequest) error { - c.RLock() - var targetNode Node - if node, ok := c.nodes[nodeID]; ok { - targetNode = node - } - c.RUnlock() - - if targetNode != nil { - err := targetNode.watchDeltaChannels(ctx, in) - if err != nil { - return err - } - - return nil - } - - return fmt.Errorf("watchDeltaChannels: can't find QueryNode by nodeID, nodeID = %d", nodeID) -} - -func (c *queryNodeCluster) HasWatchedDeltaChannel(ctx context.Context, nodeID int64, collectionID UniqueID) bool { - c.RLock() - defer c.RUnlock() - - return c.nodes[nodeID].hasWatchedDeltaChannel(collectionID) -} - func (c *queryNodeCluster) ReleaseCollection(ctx context.Context, nodeID int64, in *querypb.ReleaseCollectionRequest) error { c.RLock() var targetNode Node diff --git a/internal/querycoord/cluster_test.go b/internal/querycoord/cluster_test.go index ec590f794e..f25bf38944 100644 --- a/internal/querycoord/cluster_test.go +++ b/internal/querycoord/cluster_test.go @@ -495,20 +495,10 @@ func TestGrpcRequest(t *testing.T) { CollectionID: defaultCollectionID, } err := cluster.LoadSegments(baseCtx, nodeID, loadSegmentReq) - assert.Equal(t, 0, len(loadSegmentReq.DeltaPositions)) + assert.Equal(t, 1, len(loadSegmentReq.DeltaPositions)) assert.Nil(t, err) }) - t.Run("Test WatchDeletaChannel", func(t *testing.T) { - watchDeltaChannelReq := &querypb.WatchDeltaChannelsRequest{ - CollectionID: defaultCollectionID, - Infos: deltaChannelInfo, - } - err := cluster.WatchDeltaChannels(baseCtx, nodeID, watchDeltaChannelReq) - assert.Nil(t, err) - assert.Equal(t, true, cluster.HasWatchedDeltaChannel(baseCtx, nodeID, defaultCollectionID)) - }) - t.Run("Test LoadSegmentsAfterWatchDeltaChannel", func(t *testing.T) { segmentLoadInfo := &querypb.SegmentLoadInfo{ SegmentID: defaultSegmentID, diff --git a/internal/querycoord/mock_querynode_client_test.go b/internal/querycoord/mock_querynode_client_test.go index 6f860f1538..d0aa97c7b5 100644 --- a/internal/querycoord/mock_querynode_client_test.go +++ b/internal/querycoord/mock_querynode_client_test.go @@ -26,7 +26,6 @@ import ( etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/proto/commonpb" - "github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/proto/milvuspb" "github.com/milvus-io/milvus/internal/proto/querypb" @@ -43,7 +42,6 @@ type queryNodeClientMock struct { } func newQueryNodeTest(ctx context.Context, address string, id UniqueID, kv *etcdkv.EtcdKV) (Node, error) { - watchedDeltaChannels := make(map[UniqueID][]*datapb.VchannelInfo) childCtx, cancel := context.WithCancel(ctx) client, err := newQueryNodeClientMock(childCtx, address) if err != nil { @@ -51,13 +49,12 @@ func newQueryNodeTest(ctx context.Context, address string, id UniqueID, kv *etcd return nil, err } node := &queryNode{ - ctx: childCtx, - cancel: cancel, - id: id, - address: address, - client: client, - kvClient: kv, - watchedDeltaChannels: watchedDeltaChannels, + ctx: childCtx, + cancel: cancel, + id: id, + address: address, + client: client, + kvClient: kv, } return node, nil @@ -120,10 +117,6 @@ func (client *queryNodeClientMock) WatchDmChannels(ctx context.Context, req *que return client.grpcClient.WatchDmChannels(ctx, req) } -func (client *queryNodeClientMock) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) { - return client.grpcClient.WatchDeltaChannels(ctx, req) -} - func (client *queryNodeClientMock) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { return client.grpcClient.LoadSegments(ctx, req) } diff --git a/internal/querycoord/mock_querynode_server_test.go b/internal/querycoord/mock_querynode_server_test.go index ba547d9c0c..8aef9ebb8b 100644 --- a/internal/querycoord/mock_querynode_server_test.go +++ b/internal/querycoord/mock_querynode_server_test.go @@ -207,10 +207,6 @@ func (qs *queryNodeServerMock) WatchDmChannels(ctx context.Context, req *querypb return qs.watchDmChannels() } -func (qs *queryNodeServerMock) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) { - return qs.watchDeltaChannels() -} - func (qs *queryNodeServerMock) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { sizePerRecord, err := typeutil.EstimateSizePerRecord(req.Schema) if err != nil { diff --git a/internal/querycoord/querynode.go b/internal/querycoord/querynode.go index d561594915..6ce38f765d 100644 --- a/internal/querycoord/querynode.go +++ b/internal/querycoord/querynode.go @@ -29,7 +29,6 @@ import ( etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/proto/commonpb" - "github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/proto/milvuspb" "github.com/milvus-io/milvus/internal/proto/querypb" @@ -47,12 +46,7 @@ type Node interface { releasePartitions(ctx context.Context, in *querypb.ReleasePartitionsRequest) error watchDmChannels(ctx context.Context, in *querypb.WatchDmChannelsRequest) error - watchDeltaChannels(ctx context.Context, in *querypb.WatchDeltaChannelsRequest) error syncReplicaSegments(ctx context.Context, in *querypb.SyncReplicaSegmentsRequest) error - //removeDmChannel(collectionID UniqueID, channels []string) error - - hasWatchedDeltaChannel(collectionID UniqueID) bool - setState(state nodeState) getState() nodeState isOnline() bool @@ -75,9 +69,8 @@ type queryNode struct { kvClient *etcdkv.EtcdKV sync.RWMutex - watchedDeltaChannels map[UniqueID][]*datapb.VchannelInfo - state nodeState - stateLock sync.RWMutex + state nodeState + stateLock sync.RWMutex totalMem uint64 memUsage uint64 @@ -86,7 +79,6 @@ type queryNode struct { } func newQueryNode(ctx context.Context, address string, id UniqueID, kv *etcdkv.EtcdKV) (Node, error) { - watchedDeltaChannels := make(map[UniqueID][]*datapb.VchannelInfo) childCtx, cancel := context.WithCancel(ctx) client, err := nodeclient.NewClient(childCtx, address) if err != nil { @@ -94,14 +86,13 @@ func newQueryNode(ctx context.Context, address string, id UniqueID, kv *etcdkv.E return nil, err } node := &queryNode{ - ctx: childCtx, - cancel: cancel, - id: id, - address: address, - client: client, - kvClient: kv, - watchedDeltaChannels: watchedDeltaChannels, - state: disConnect, + ctx: childCtx, + cancel: cancel, + id: id, + address: address, + client: client, + kvClient: kv, + state: disConnect, } return node, nil @@ -127,30 +118,12 @@ func (qn *queryNode) start() error { } func (qn *queryNode) stop() { - //qn.stateLock.Lock() - //defer qn.stateLock.Unlock() - //qn.state = offline if qn.client != nil { qn.client.Stop() } qn.cancel() } -func (qn *queryNode) hasWatchedDeltaChannel(collectionID UniqueID) bool { - qn.RLock() - defer qn.RUnlock() - - _, ok := qn.watchedDeltaChannels[collectionID] - return ok -} - -func (qn *queryNode) setDeltaChannelInfo(collectionID int64, infos []*datapb.VchannelInfo) { - qn.Lock() - defer qn.Unlock() - - qn.watchedDeltaChannels[collectionID] = infos -} - func (qn *queryNode) setState(state nodeState) { qn.stateLock.Lock() defer qn.stateLock.Unlock() @@ -196,22 +169,6 @@ func (qn *queryNode) watchDmChannels(ctx context.Context, in *querypb.WatchDmCha return nil } -func (qn *queryNode) watchDeltaChannels(ctx context.Context, in *querypb.WatchDeltaChannelsRequest) error { - if !qn.isOnline() { - return errors.New("WatchDeltaChannels: queryNode is offline") - } - - status, err := qn.client.WatchDeltaChannels(qn.ctx, in) - if err != nil { - return err - } - if status.ErrorCode != commonpb.ErrorCode_Success { - return errors.New(status.Reason) - } - qn.setDeltaChannelInfo(in.CollectionID, in.Infos) - return err -} - func (qn *queryNode) releaseCollection(ctx context.Context, in *querypb.ReleaseCollectionRequest) error { if !qn.isOnline() { log.Warn("ReleaseCollection: the QueryNode has been offline, the release request is no longer needed", zap.Int64("nodeID", qn.id)) @@ -225,11 +182,6 @@ func (qn *queryNode) releaseCollection(ctx context.Context, in *querypb.ReleaseC if status.ErrorCode != commonpb.ErrorCode_Success { return errors.New(status.Reason) } - - qn.Lock() - delete(qn.watchedDeltaChannels, in.CollectionID) - qn.Unlock() - return nil } diff --git a/internal/querycoord/task.go b/internal/querycoord/task.go index fd99d7ca0f..43d86ec5f4 100644 --- a/internal/querycoord/task.go +++ b/internal/querycoord/task.go @@ -1359,7 +1359,7 @@ func (lst *loadSegmentTask) execute(ctx context.Context) error { err := lst.cluster.LoadSegments(ctx, lst.DstNodeID, lst.LoadSegmentsRequest) if err != nil { - log.Warn("loadSegmentTask: loadSegment occur error", zap.Int64("taskID", lst.getTaskID())) + log.Warn("loadSegmentTask: loadSegment occur error", zap.Int64("taskID", lst.getTaskID()), zap.Error(err)) lst.setResultInfo(err) return err } @@ -1606,84 +1606,6 @@ func (wdt *watchDmChannelTask) reschedule(ctx context.Context) ([]task, error) { return reScheduledTasks, nil } -type watchDeltaChannelTask struct { - *baseTask - *querypb.WatchDeltaChannelsRequest - cluster Cluster -} - -func (wdt *watchDeltaChannelTask) msgBase() *commonpb.MsgBase { - return wdt.Base -} - -func (wdt *watchDeltaChannelTask) marshal() ([]byte, error) { - return proto.Marshal(wdt.WatchDeltaChannelsRequest) -} - -func (wdt *watchDeltaChannelTask) isValid() bool { - online, err := wdt.cluster.IsOnline(wdt.NodeID) - if err != nil { - return false - } - - return wdt.ctx != nil && online -} - -func (wdt *watchDeltaChannelTask) msgType() commonpb.MsgType { - return wdt.Base.MsgType -} - -func (wdt *watchDeltaChannelTask) timestamp() Timestamp { - return wdt.Base.Timestamp -} - -func (wdt *watchDeltaChannelTask) updateTaskProcess() { - parentTask := wdt.getParentTask() - if parentTask == nil { - log.Warn("watchDeltaChannel: parentTask should not be nil") - return - } - parentTask.updateTaskProcess() -} - -func (wdt *watchDeltaChannelTask) preExecute(context.Context) error { - channelInfos := wdt.Infos - channels := make([]string, 0) - for _, info := range channelInfos { - channels = append(channels, info.ChannelName) - } - wdt.setResultInfo(nil) - log.Info("start do watchDeltaChannelTask", - zap.Strings("deltaChannels", channels), - zap.Int64("loaded nodeID", wdt.NodeID), - zap.Int64("taskID", wdt.getTaskID()), - zap.Int64("msgID", wdt.GetBase().GetMsgID())) - return nil -} - -func (wdt *watchDeltaChannelTask) execute(ctx context.Context) error { - defer wdt.reduceRetryCount() - - err := wdt.cluster.WatchDeltaChannels(wdt.ctx, wdt.NodeID, wdt.WatchDeltaChannelsRequest) - if err != nil { - log.Warn("watchDeltaChannelTask: watchDeltaChannel occur error", zap.Int64("taskID", wdt.getTaskID()), zap.Error(err)) - wdt.setResultInfo(err) - return err - } - - log.Info("watchDeltaChannelsTask Execute done", - zap.Int64("taskID", wdt.getTaskID()), - zap.Int64("msgID", wdt.GetBase().GetMsgID())) - return nil -} - -func (wdt *watchDeltaChannelTask) postExecute(context.Context) error { - log.Info("watchDeltaChannelTask postExecute done", - zap.Int64("taskID", wdt.getTaskID()), - zap.Int64("msgID", wdt.GetBase().GetMsgID())) - return nil -} - //****************************handoff task********************************// type handoffTask struct { *baseTask @@ -2160,8 +2082,7 @@ func (lbt *loadBalanceTask) processNodeDownLoadBalance(ctx context.Context) erro lbt.addChildTask(internalTask) log.Info("loadBalanceTask: add a childTask", zap.Int64("taskID", lbt.getTaskID()), - zap.String("taskType", internalTask.msgType().String()), - zap.Int64("destNode", getDstNodeIDByTask(internalTask))) + zap.Any("task", internalTask)) } log.Info("loadBalanceTask: assign child task done", zap.Int64("taskID", lbt.getTaskID()), zap.Int64s("sourceNodeIDs", lbt.SourceNodeIDs)) @@ -2630,13 +2551,13 @@ func mergeWatchDeltaChannelInfo(infos []*datapb.VchannelInfo) []*datapb.Vchannel } } var result []*datapb.VchannelInfo - for _, index := range minPositions { + for channel, index := range minPositions { result = append(result, infos[index]) + log.Info("merge delta channels finished", zap.String("channel", channel), + zap.Any("merged info", infos[index]), + ) } - log.Info("merge delta channels finished", - zap.Any("origin info length", len(infos)), - zap.Any("merged info length", len(result)), - ) + return result } diff --git a/internal/querycoord/task_scheduler.go b/internal/querycoord/task_scheduler.go index 4b4ca0d7cb..3e377609cc 100644 --- a/internal/querycoord/task_scheduler.go +++ b/internal/querycoord/task_scheduler.go @@ -406,19 +406,7 @@ func (scheduler *TaskScheduler) unmarshalTask(taskID UniqueID, t string) (task, } newTask = watchDmChannelTask case commonpb.MsgType_WatchDeltaChannels: - //TODO::trigger condition may be different - loadReq := querypb.WatchDeltaChannelsRequest{} - err = proto.Unmarshal([]byte(t), &loadReq) - reviseWatchDeltaChannelsRequest(&loadReq) - if err != nil { - return nil, err - } - watchDeltaChannelTask := &watchDeltaChannelTask{ - baseTask: baseTask, - WatchDeltaChannelsRequest: &loadReq, - cluster: scheduler.cluster, - } - newTask = watchDeltaChannelTask + log.Warn("legacy WatchDeltaChannels type found, ignore") case commonpb.MsgType_WatchQueryChannels: //Deprecated WatchQueryChannel log.Warn("legacy WatchQueryChannels type found, ignore") @@ -524,8 +512,6 @@ func (scheduler *TaskScheduler) processTask(t task) error { protoSize = proto.Size(childTask.(*loadSegmentTask).LoadSegmentsRequest) case commonpb.MsgType_WatchDmChannels: protoSize = proto.Size(childTask.(*watchDmChannelTask).WatchDmChannelsRequest) - case commonpb.MsgType_WatchDeltaChannels: - protoSize = proto.Size(childTask.(*watchDeltaChannelTask).WatchDeltaChannelsRequest) default: //TODO:: } @@ -728,17 +714,6 @@ func (scheduler *TaskScheduler) scheduleLoop() { if len(childTasks) != 0 { // include loadSegment, watchDmChannel, releaseCollection, releasePartition, releaseSegment processInternalTaskFn(childTasks, triggerTask) - if triggerTask.getResultInfo().ErrorCode == commonpb.ErrorCode_Success { - // derivedInternalTasks include watchDeltaChannel, watchQueryChannel - // derivedInternalTasks generate from loadSegment and watchDmChannel reqs - derivedInternalTasks, err := generateDerivedInternalTasks(triggerTask, scheduler.meta, scheduler.cluster) - if err != nil { - log.Error("scheduleLoop: generate derived watchDeltaChannel and watchcQueryChannel tasks failed", zap.Int64("triggerTaskID", triggerTask.getTaskID()), zap.Error(err)) - triggerTask.setResultInfo(err) - } else { - processInternalTaskFn(derivedInternalTasks, triggerTask) - } - } } // triggerTask may be LoadCollection, LoadPartitions, LoadBalance, Handoff @@ -1115,57 +1090,3 @@ func updateSegmentInfoFromTask(ctx context.Context, triggerTask task, meta Meta) return nil } - -// generateDerivedInternalTasks generate watchDeltaChannel and watchQueryChannel tasks -func generateDerivedInternalTasks(triggerTask task, meta Meta, cluster Cluster) ([]task, error) { - var derivedInternalTasks []task - watchDeltaChannelInfo := make(map[int64]map[UniqueID]UniqueID) - addChannelWatchInfoFn := func(nodeID int64, collectionID UniqueID, replicaID UniqueID, watchInfo map[int64]map[UniqueID]UniqueID) { - if _, ok := watchInfo[nodeID]; !ok { - watchInfo[nodeID] = make(map[UniqueID]UniqueID) - } - - watchInfo[nodeID][collectionID] = replicaID - } - - for _, childTask := range triggerTask.getChildTask() { - if childTask.msgType() == commonpb.MsgType_LoadSegments { - loadSegmentTask := childTask.(*loadSegmentTask) - collectionID := loadSegmentTask.CollectionID - replicaID := loadSegmentTask.GetReplicaID() - nodeID := loadSegmentTask.DstNodeID - if !cluster.HasWatchedDeltaChannel(triggerTask.traceCtx(), nodeID, collectionID) { - addChannelWatchInfoFn(nodeID, collectionID, replicaID, watchDeltaChannelInfo) - } - } - } - - for nodeID, collectionIDs := range watchDeltaChannelInfo { - for collectionID, replicaID := range collectionIDs { - deltaChannelInfo, err := meta.getDeltaChannelsByCollectionID(collectionID) - if err != nil { - return nil, err - } - msgBase := proto.Clone(triggerTask.msgBase()).(*commonpb.MsgBase) - msgBase.MsgType = commonpb.MsgType_WatchDeltaChannels - watchDeltaRequest := &querypb.WatchDeltaChannelsRequest{ - Base: msgBase, - CollectionID: collectionID, - Infos: deltaChannelInfo, - - ReplicaId: replicaID, - } - watchDeltaRequest.NodeID = nodeID - baseTask := newBaseTask(triggerTask.traceCtx(), triggerTask.getTriggerCondition()) - baseTask.setParentTask(triggerTask) - watchDeltaTask := &watchDeltaChannelTask{ - baseTask: baseTask, - WatchDeltaChannelsRequest: watchDeltaRequest, - cluster: cluster, - } - derivedInternalTasks = append(derivedInternalTasks, watchDeltaTask) - } - } - - return derivedInternalTasks, nil -} diff --git a/internal/querycoord/task_scheduler_test.go b/internal/querycoord/task_scheduler_test.go index d05f49e6ae..b5379c2124 100644 --- a/internal/querycoord/task_scheduler_test.go +++ b/internal/querycoord/task_scheduler_test.go @@ -365,27 +365,6 @@ func TestUnMarshalTask(t *testing.T) { assert.Nil(t, task2) }) - t.Run("Test watchDeltaChannelTask", func(t *testing.T) { - watchTask := &watchDeltaChannelTask{ - WatchDeltaChannelsRequest: &querypb.WatchDeltaChannelsRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_WatchDeltaChannels, - }, - }, - } - blobs, err := watchTask.marshal() - assert.Nil(t, err) - err = kv.Save("testMarshalWatchDeltaChannel", string(blobs)) - assert.Nil(t, err) - defer kv.RemoveWithPrefix("testMarshalWatchDeltaChannel") - value, err := kv.Load("testMarshalWatchDeltaChannel") - assert.Nil(t, err) - - task, err := taskScheduler.unmarshalTask(1007, value) - assert.Nil(t, err) - assert.Equal(t, task.msgType(), commonpb.MsgType_WatchDeltaChannels) - }) - t.Run("Test loadBalanceTask", func(t *testing.T) { loadBalanceTask := &loadBalanceTask{ LoadBalanceRequest: &querypb.LoadBalanceRequest{ @@ -525,49 +504,6 @@ func Test_saveInternalTaskToEtcd(t *testing.T) { }) } -func Test_generateDerivedInternalTasks(t *testing.T) { - refreshParams() - baseCtx := context.Background() - queryCoord, err := startQueryCoord(baseCtx) - assert.Nil(t, err) - node1, err := startQueryNodeServer(baseCtx) - assert.Nil(t, err) - waitQueryNodeOnline(queryCoord.cluster, node1.queryNodeID) - vChannelInfos, _, err := queryCoord.broker.getRecoveryInfo(baseCtx, defaultCollectionID, defaultPartitionID) - assert.NoError(t, err) - deltaChannelInfos := make([]*datapb.VchannelInfo, len(vChannelInfos)) - for i, info := range vChannelInfos { - deltaInfo, err := generateWatchDeltaChannelInfo(info) - assert.NoError(t, err) - deltaChannelInfos[i] = deltaInfo - } - - queryCoord.meta.setDeltaChannel(defaultCollectionID, deltaChannelInfos) - - loadCollectionTask := genLoadCollectionTask(baseCtx, queryCoord) - loadSegmentTask := genLoadSegmentTask(baseCtx, queryCoord, node1.queryNodeID) - loadCollectionTask.addChildTask(loadSegmentTask) - loadSegmentTask.setParentTask(loadCollectionTask) - watchDmChannelTask := genWatchDmChannelTask(baseCtx, queryCoord, node1.queryNodeID) - loadCollectionTask.addChildTask(watchDmChannelTask) - watchDmChannelTask.setParentTask(loadCollectionTask) - - derivedTasks, err := generateDerivedInternalTasks(loadCollectionTask, queryCoord.meta, queryCoord.cluster) - assert.Nil(t, err) - assert.Equal(t, 1, len(derivedTasks)) - for _, internalTask := range derivedTasks { - matchType := internalTask.msgType() == commonpb.MsgType_WatchDeltaChannels - assert.Equal(t, true, matchType) - if internalTask.msgType() == commonpb.MsgType_WatchDeltaChannels { - assert.Equal(t, node1.queryNodeID, internalTask.(*watchDeltaChannelTask).NodeID) - } - } - - queryCoord.Stop() - err = removeAllSession() - assert.Nil(t, err) -} - func TestTaskScheduler_BindContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) s := &TaskScheduler{ diff --git a/internal/querycoord/task_test.go b/internal/querycoord/task_test.go index b842a6eb94..124b51315b 100644 --- a/internal/querycoord/task_test.go +++ b/internal/querycoord/task_test.go @@ -190,12 +190,14 @@ func genWatchDmChannelTask(ctx context.Context, queryCoord *QueryCoord, nodeID i queryCoord.meta.addCollection(defaultCollectionID, querypb.LoadType_LoadCollection, schema) return watchDmChannelTask } + func genLoadSegmentTask(ctx context.Context, queryCoord *QueryCoord, nodeID int64) *loadSegmentTask { schema := genDefaultCollectionSchema(false) segmentInfo := &querypb.SegmentLoadInfo{ - SegmentID: defaultSegmentID, - PartitionID: defaultPartitionID, - CollectionID: defaultCollectionID, + SegmentID: defaultSegmentID, + PartitionID: defaultPartitionID, + CollectionID: defaultCollectionID, + InsertChannel: "by-dev-rootcoord-dml_1_2021v1", } req := &querypb.LoadSegmentsRequest{ Base: &commonpb.MsgBase{ @@ -246,29 +248,18 @@ func genLoadSegmentTask(ctx context.Context, queryCoord *QueryCoord, nodeID int6 loadSegmentTask.setParentTask(parentTask) queryCoord.meta.addCollection(defaultCollectionID, querypb.LoadType_LoadCollection, schema) - return loadSegmentTask -} -func genWatchDeltaChannelTask(ctx context.Context, queryCoord *QueryCoord, nodeID int64) *watchDeltaChannelTask { - req := &querypb.WatchDeltaChannelsRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_WatchDeltaChannels, - }, - NodeID: nodeID, - CollectionID: defaultCollectionID, - LoadMeta: &querypb.LoadMetaInfo{ - LoadType: querypb.LoadType_LoadCollection, + deltaChannelInfo := []*datapb.VchannelInfo{ + { CollectionID: defaultCollectionID, - PartitionIDs: []int64{defaultPartitionID}, + ChannelName: "by-dev-rootcoord-delta_1_2021v1", + SeekPosition: &internalpb.MsgPosition{ + ChannelName: "by-dev-rootcoord-dml_1", + }, }, } - baseTask := newBaseTask(ctx, querypb.TriggerCondition_GrpcRequest) - baseTask.taskID = 300 - return &watchDeltaChannelTask{ - baseTask: baseTask, - WatchDeltaChannelsRequest: req, - cluster: queryCoord.cluster, - } + queryCoord.meta.setDeltaChannel(defaultCollectionID, deltaChannelInfo) + return loadSegmentTask } func waitTaskFinalState(t task, state taskState) { @@ -1399,9 +1390,6 @@ func TestUpdateTaskProcessWhenLoadSegment(t *testing.T) { assert.Nil(t, err) assert.Equal(t, int64(0), collectionInfo.InMemoryPercentage) - watchDeltaChannel := genWatchDeltaChannelTask(ctx, queryCoord, node1.queryNodeID) - watchDeltaChannel.setParentTask(loadCollectionTask) - queryCoord.scheduler.processTask(watchDeltaChannel) collectionInfo, err = queryCoord.meta.getCollectionInfoByID(defaultCollectionID) assert.Nil(t, err) assert.Equal(t, int64(0), collectionInfo.InMemoryPercentage) diff --git a/internal/querycoord/task_util.go b/internal/querycoord/task_util.go index 0048addcbc..4971a3063e 100644 --- a/internal/querycoord/task_util.go +++ b/internal/querycoord/task_util.go @@ -63,13 +63,6 @@ func thinWatchDmChannelsRequest(request *querypb.WatchDmChannelsRequest) *queryp return cloned } -// reviseWatchDeltaChannelsRequest will revise the WatchDeltaChannelsRequest for upgrade compatibility from 2.0.2 -func reviseWatchDeltaChannelsRequest(req *querypb.WatchDeltaChannelsRequest) { - for _, vChannel := range req.GetInfos() { - reviseVChannelInfo(vChannel) - } -} - // reviseVChannelInfo will revise the datapb.VchannelInfo for upgrade compatibility from 2.0.2 func reviseVChannelInfo(vChannel *datapb.VchannelInfo) { removeDuplicateSegmentIDFn := func(ids []int64) []int64 { diff --git a/internal/querycoord/util.go b/internal/querycoord/util.go index c6ece959ce..9a8857676c 100644 --- a/internal/querycoord/util.go +++ b/internal/querycoord/util.go @@ -21,7 +21,6 @@ import ( "sort" "github.com/milvus-io/milvus/internal/log" - "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/proto/milvuspb" "github.com/milvus-io/milvus/internal/proto/querypb" @@ -81,34 +80,6 @@ func getFieldSizeFromFieldBinlog(fieldBinlog *datapb.FieldBinlog) int64 { } -func getDstNodeIDByTask(t task) int64 { - var nodeID int64 - switch t.msgType() { - case commonpb.MsgType_LoadSegments: - loadSegment := t.(*loadSegmentTask) - nodeID = loadSegment.DstNodeID - case commonpb.MsgType_WatchDmChannels: - watchDmChannel := t.(*watchDmChannelTask) - nodeID = watchDmChannel.NodeID - case commonpb.MsgType_WatchDeltaChannels: - watchDeltaChannel := t.(*watchDeltaChannelTask) - nodeID = watchDeltaChannel.NodeID - case commonpb.MsgType_ReleaseCollection: - releaseCollection := t.(*releaseCollectionTask) - nodeID = releaseCollection.NodeID - case commonpb.MsgType_ReleasePartitions: - releasePartition := t.(*releasePartitionTask) - nodeID = releasePartition.NodeID - case commonpb.MsgType_ReleaseSegments: - releaseSegment := t.(*releaseSegmentTask) - nodeID = releaseSegment.NodeID - default: - //TODO:: - } - - return nodeID -} - // syncReplicaSegments syncs the segments distribution of replica to shard leaders // only syncs the segments in shards if not nil func syncReplicaSegments(ctx context.Context, meta Meta, cluster Cluster, replicaID UniqueID, shards ...string) error { diff --git a/internal/querynode/impl.go b/internal/querynode/impl.go index 1354194ec6..6e2f5b9228 100644 --- a/internal/querynode/impl.go +++ b/internal/querynode/impl.go @@ -325,58 +325,6 @@ func (node *QueryNode) WatchDmChannels(ctx context.Context, in *queryPb.WatchDmC return waitFunc() } -// WatchDeltaChannels create consumers on dmChannels to receive Incremental data，which is the important part of real-time query -func (node *QueryNode) WatchDeltaChannels(ctx context.Context, in *queryPb.WatchDeltaChannelsRequest) (*commonpb.Status, error) { - code := node.stateCode.Load().(internalpb.StateCode) - if code != internalpb.StateCode_Healthy { - err := fmt.Errorf("query node %d is not ready", Params.QueryNodeCfg.GetNodeID()) - status := &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_UnexpectedError, - Reason: err.Error(), - } - return status, nil - } - dct := &watchDeltaChannelsTask{ - baseTask: baseTask{ - ctx: ctx, - done: make(chan error), - }, - req: in, - node: node, - } - - err := node.scheduler.queue.Enqueue(dct) - if err != nil { - status := &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_UnexpectedError, - Reason: err.Error(), - } - log.Warn(err.Error()) - return status, nil - } - - log.Info("watchDeltaChannelsTask Enqueue done", zap.Int64("collectionID", in.CollectionID), zap.Int64("nodeID", Params.QueryNodeCfg.GetNodeID())) - - waitFunc := func() (*commonpb.Status, error) { - err = dct.WaitToFinish() - if err != nil { - status := &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_UnexpectedError, - Reason: err.Error(), - } - log.Warn(err.Error()) - return status, nil - } - - log.Info("watchDeltaChannelsTask WaitToFinish done", zap.Int64("collectionID", in.CollectionID), zap.Int64("nodeID", Params.QueryNodeCfg.GetNodeID())) - return &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_Success, - }, nil - } - - return waitFunc() -} - // LoadSegments load historical data into query node, historical data can be vector data or index func (node *QueryNode) LoadSegments(ctx context.Context, in *queryPb.LoadSegmentsRequest) (*commonpb.Status, error) { code := node.stateCode.Load().(internalpb.StateCode) diff --git a/internal/querynode/query_shard_service.go b/internal/querynode/query_shard_service.go index 5f5ef1e9c3..8a2bc010bf 100644 --- a/internal/querynode/query_shard_service.go +++ b/internal/querynode/query_shard_service.go @@ -75,7 +75,7 @@ func (q *queryShardService) addQueryShard(collectionID UniqueID, channel Channel q.queryShardsMu.Lock() defer q.queryShardsMu.Unlock() if _, ok := q.queryShards[channel]; ok { - return errors.New(fmt.Sprintln("query shard(channel) ", channel, " already exists")) + return nil } qs, err := newQueryShard( q.ctx, diff --git a/internal/querynode/segment_loader.go b/internal/querynode/segment_loader.go index ae6a2f0843..9baadd3f9b 100644 --- a/internal/querynode/segment_loader.go +++ b/internal/querynode/segment_loader.go @@ -24,7 +24,6 @@ import ( "runtime" "runtime/debug" "strconv" - "sync" "github.com/panjf2000/ants/v2" "go.uber.org/zap" @@ -84,6 +83,7 @@ func (loader *segmentLoader) LoadSegment(req *querypb.LoadSegmentsRequest, segme segmentNum := len(req.Infos) if segmentNum == 0 { + log.Warn("find no valid segment target, skip load segment", zap.Any("request", req)) return nil } @@ -156,7 +156,7 @@ func (loader *segmentLoader) LoadSegment(req *querypb.LoadSegmentsRequest, segme newSegments[segmentID] = segment } - loadSegmentFunc := func(idx int) error { + loadFileFunc := func(idx int) error { loadInfo := req.Infos[idx] collectionID := loadInfo.CollectionID partitionID := loadInfo.PartitionID @@ -164,7 +164,7 @@ func (loader *segmentLoader) LoadSegment(req *querypb.LoadSegmentsRequest, segme segment := newSegments[segmentID] tr := timerecord.NewTimeRecorder("loadDurationPerSegment") - err := loader.loadSegmentInternal(segment, loadInfo) + err := loader.loadFiles(segment, loadInfo) if err != nil { log.Error("load segment failed when load data into memory", zap.Int64("collectionID", collectionID), @@ -186,8 +186,7 @@ func (loader *segmentLoader) LoadSegment(req *querypb.LoadSegmentsRequest, segme zap.Int("segmentNum", segmentNum), zap.Int("concurrencyLevel", concurrencyLevel)) err = funcutil.ProcessFuncParallel(segmentNum, - concurrencyLevel, - loadSegmentFunc, "loadSegmentFunc") + concurrencyLevel, loadFileFunc, "loadSegmentFunc") if err != nil { segmentGC() return err @@ -211,7 +210,7 @@ func (loader *segmentLoader) LoadSegment(req *querypb.LoadSegmentsRequest, segme return nil } -func (loader *segmentLoader) loadSegmentInternal(segment *Segment, +func (loader *segmentLoader) loadFiles(segment *Segment, loadInfo *querypb.SegmentLoadInfo) error { collectionID := loadInfo.CollectionID partitionID := loadInfo.PartitionID @@ -643,7 +642,7 @@ func (loader *segmentLoader) loadDeltaLogs(segment *Segment, deltaLogs []*datapb } func (loader *segmentLoader) FromDmlCPLoadDelete(ctx context.Context, collectionID int64, position *internalpb.MsgPosition) error { - log.Info("from dml check point load delete", zap.Any("position", position), zap.Any("msg id", position.MsgID)) + log.Info("from dml check point load delete", zap.Any("position", position)) stream, err := loader.factory.NewMsgStream(ctx) if err != nil { return err @@ -682,7 +681,7 @@ func (loader *segmentLoader) FromDmlCPLoadDelete(ctx context.Context, collection } log.Info("start read delta msg from seek position to last position", - zap.Int64("Collection ID", collectionID), zap.String("channel", pChannelName)) + zap.Int64("Collection ID", collectionID), zap.String("channel", pChannelName), zap.Any("seek pos", position), zap.Any("last msg", lastMsgID)) hasMore := true for hasMore { select { @@ -744,43 +743,18 @@ func (loader *segmentLoader) FromDmlCPLoadDelete(ctx context.Context, collection } offset := segment.segmentPreDelete(len(pks)) delData.deleteOffset[segmentID] = offset + timestamps := delData.deleteTimestamps[segmentID] + err = segment.segmentDelete(offset, pks, timestamps) + if err != nil { + log.Warn("QueryNode: segment delete failed", zap.Int64("segment", segmentID), zap.Error(err)) + return err + } } - wg := sync.WaitGroup{} - for segmentID := range delData.deleteOffset { - wg.Add(1) - go deletePk(loader.metaReplica, delData, segmentID, &wg) - } - wg.Wait() log.Info("from dml check point load done", zap.Any("msg id", position.GetMsgID())) return nil } -func deletePk(replica ReplicaInterface, deleteData *deleteData, segmentID UniqueID, wg *sync.WaitGroup) { - defer wg.Done() - log.Debug("QueryNode::iNode::delete", zap.Any("SegmentID", segmentID)) - targetSegment, err := replica.getSegmentByID(segmentID, segmentTypeSealed) - if err != nil { - log.Error(err.Error()) - return - } - - if targetSegment.segmentType != segmentTypeSealed { - return - } - - ids := deleteData.deleteIDs[segmentID] - timestamps := deleteData.deleteTimestamps[segmentID] - offset := deleteData.deleteOffset[segmentID] - - err = targetSegment.segmentDelete(offset, ids, timestamps) - if err != nil { - log.Warn("QueryNode: targetSegmentDelete failed", zap.Error(err)) - return - } - log.Debug("Do delete done", zap.Int("len", len(deleteData.deleteIDs[segmentID])), zap.Int64("segmentID", segmentID), zap.Any("segmentType", targetSegment.segmentType)) -} - // JoinIDPath joins ids to path format. func JoinIDPath(ids ...UniqueID) string { idStr := make([]string, 0, len(ids)) @@ -819,6 +793,8 @@ func (loader *segmentLoader) checkSegmentSize(collectionID UniqueID, segmentLoad loadingUsage := usedMemAfterLoad + uint64( float64(maxSegmentSize)*float64(concurrency)*Params.QueryNodeCfg.LoadMemoryUsageFactor) log.Debug("predict memory usage while loading (in MiB)", + zap.Int64("collectionID", collectionID), + zap.Int("concurrency", concurrency), zap.Uint64("usage", toMB(loadingUsage)), zap.Uint64("usageAfterLoad", toMB(usedMemAfterLoad))) diff --git a/internal/querynode/shard_cluster.go b/internal/querynode/shard_cluster.go index 39a08b8dee..d8d7ce545d 100644 --- a/internal/querynode/shard_cluster.go +++ b/internal/querynode/shard_cluster.go @@ -907,7 +907,7 @@ func (sc *ShardCluster) Search(ctx context.Context, req *querypb.SearchRequest, wg.Wait() if err != nil { - log.Error(err.Error()) + log.Error("failed to do search", zap.Any("req", req), zap.Error(err)) return nil, err } diff --git a/internal/querynode/task.go b/internal/querynode/task.go index b00b8e6c33..9e2117512d 100644 --- a/internal/querynode/task.go +++ b/internal/querynode/task.go @@ -31,6 +31,7 @@ import ( queryPb "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/util/funcutil" "github.com/milvus-io/milvus/internal/util/typeutil" + "golang.org/x/sync/errgroup" ) type task interface { @@ -90,12 +91,6 @@ type watchDmChannelsTask struct { node *QueryNode } -type watchDeltaChannelsTask struct { - baseTask - req *queryPb.WatchDeltaChannelsRequest - node *QueryNode -} - type loadSegmentsTask struct { baseTask req *queryPb.LoadSegmentsRequest @@ -386,9 +381,7 @@ func (w *watchDmChannelsTask) Execute(ctx context.Context) (err error) { // add tsafe watch in query shard if exists for _, dmlChannel := range vChannels { - if !w.node.queryShardService.hasQueryShard(dmlChannel) { - w.node.queryShardService.addQueryShard(collectionID, dmlChannel, w.req.GetReplicaID()) - } + w.node.queryShardService.addQueryShard(collectionID, dmlChannel, w.req.GetReplicaID()) } // start flow graphs @@ -400,63 +393,50 @@ func (w *watchDmChannelsTask) Execute(ctx context.Context) (err error) { return nil } -// watchDeltaChannelsTask -func (w *watchDeltaChannelsTask) Execute(ctx context.Context) error { - collectionID := w.req.CollectionID - - // get all vChannels - vDeltaChannels := make([]Channel, 0) - pDeltaChannels := make([]Channel, 0) - VPDeltaChannels := make(map[string]string) // map[vChannel]pChannel - vChannel2SeekPosition := make(map[string]*internalpb.MsgPosition) - for _, info := range w.req.Infos { - v := info.ChannelName - p := funcutil.ToPhysicalChannel(info.ChannelName) - vDeltaChannels = append(vDeltaChannels, v) +// internal helper function to subscribe delta channel +func (l *loadSegmentsTask) watchDeltaChannel(vchanName []string) error { + collectionID := l.req.CollectionID + var vDeltaChannels, pDeltaChannels []string + VPDeltaChannels := make(map[string]string) + for _, v := range vchanName { + dc, err := funcutil.ConvertChannelName(v, Params.CommonCfg.RootCoordDml, Params.CommonCfg.RootCoordDelta) + if err != nil { + log.Warn("watchDeltaChannels, failed to convert deltaChannel from dmlChannel", zap.String("DmlChannel", v), zap.Error(err)) + return err + } + p := funcutil.ToPhysicalChannel(dc) + vDeltaChannels = append(vDeltaChannels, dc) pDeltaChannels = append(pDeltaChannels, p) - VPDeltaChannels[v] = p - vChannel2SeekPosition[v] = info.SeekPosition + VPDeltaChannels[dc] = p } log.Info("Starting WatchDeltaChannels ...", - zap.Any("collectionID", collectionID), - zap.Any("vDeltaChannels", vDeltaChannels), - zap.Any("pChannels", pDeltaChannels), - ) - if len(VPDeltaChannels) != len(vDeltaChannels) { - return errors.New("get physical channels failed, illegal channel length, collectionID = " + fmt.Sprintln(collectionID)) - } - log.Info("Get physical channels done", - zap.Any("collectionID", collectionID), + zap.Int64("collectionID", collectionID), + zap.Any("channels", VPDeltaChannels), ) - if hasColl := w.node.metaReplica.hasCollection(collectionID); !hasColl { - return fmt.Errorf("cannot find collection with collectionID, %d", collectionID) - } - coll, err := w.node.metaReplica.getCollectionByID(collectionID) + coll, err := l.node.metaReplica.getCollectionByID(collectionID) if err != nil { return err } - channel2FlowGraph, err := w.node.dataSyncService.addFlowGraphsForDeltaChannels(collectionID, vDeltaChannels) + channel2FlowGraph, err := l.node.dataSyncService.addFlowGraphsForDeltaChannels(collectionID, vDeltaChannels) if err != nil { log.Warn("watchDeltaChannel, add flowGraph for deltaChannel failed", zap.Int64("collectionID", collectionID), zap.Strings("vDeltaChannels", vDeltaChannels), zap.Error(err)) return err } consumeSubName := funcutil.GenChannelSubName(Params.CommonCfg.QueryNodeSubName, collectionID, Params.QueryNodeCfg.GetNodeID()) + // channels as consumer for channel, fg := range channel2FlowGraph { + pchannel := VPDeltaChannels[channel] // use pChannel to consume - err = fg.consumeFlowGraphFromLatest(VPDeltaChannels[channel], consumeSubName) + err = fg.consumeFlowGraphFromLatest(pchannel, consumeSubName) if err != nil { log.Error("msgStream as consumer failed for deltaChannels", zap.Int64("collectionID", collectionID), zap.Strings("vDeltaChannels", vDeltaChannels)) break } - err = w.node.loader.FromDmlCPLoadDelete(w.ctx, collectionID, vChannel2SeekPosition[channel]) - if err != nil { - log.Error("watchDeltaChannelsTask from dml cp load delete failed", zap.Int64("collectionID", collectionID), zap.Strings("vDeltaChannels", vDeltaChannels)) - break - } } + if err != nil { log.Warn("watchDeltaChannel, add flowGraph for deltaChannel failed", zap.Int64("collectionID", collectionID), zap.Strings("vDeltaChannels", vDeltaChannels), zap.Error(err)) for _, fg := range channel2FlowGraph { @@ -466,7 +446,7 @@ func (w *watchDeltaChannelsTask) Execute(ctx context.Context) error { for channel := range channel2FlowGraph { gcChannels = append(gcChannels, channel) } - w.node.dataSyncService.removeFlowGraphsByDeltaChannels(gcChannels) + l.node.dataSyncService.removeFlowGraphsByDeltaChannels(gcChannels) return err } @@ -478,18 +458,20 @@ func (w *watchDeltaChannelsTask) Execute(ctx context.Context) error { // create tSafe for _, channel := range vDeltaChannels { - w.node.tSafeReplica.addTSafe(channel) + l.node.tSafeReplica.addTSafe(channel) } - // add tsafe watch in query shard if exists + // add tsafe watch in query shard if exists, we find no way to handle it if query shard not exist for _, channel := range vDeltaChannels { dmlChannel, err := funcutil.ConvertChannelName(channel, Params.CommonCfg.RootCoordDelta, Params.CommonCfg.RootCoordDml) if err != nil { - log.Warn("failed to convert delta channel to dml", zap.String("channel", channel), zap.Error(err)) - continue + log.Error("failed to convert delta channel to dml", zap.String("channel", channel), zap.Error(err)) + panic(err) } - if !w.node.queryShardService.hasQueryShard(dmlChannel) { - w.node.queryShardService.addQueryShard(collectionID, dmlChannel, w.req.GetReplicaId()) + err = l.node.queryShardService.addQueryShard(collectionID, dmlChannel, l.req.GetReplicaID()) + if err != nil { + log.Error("failed to add shard Service to query shard", zap.String("channel", channel), zap.Error(err)) + panic(err) } } @@ -503,7 +485,6 @@ func (w *watchDeltaChannelsTask) Execute(ctx context.Context) error { } // loadSegmentsTask - func (l *loadSegmentsTask) PreExecute(ctx context.Context) error { log.Info("LoadSegmentTask PreExecute start", zap.Int64("msgID", l.req.Base.MsgID)) var err error @@ -527,7 +508,7 @@ func (l *loadSegmentsTask) PreExecute(ctx context.Context) error { if !has { filteredInfos = append(filteredInfos, info) } else { - log.Debug("ignore segment that is already loaded", zap.Int64("segmentID", info.SegmentID)) + log.Debug("ignore segment that is already loaded", zap.Int64("collectionID", info.SegmentID), zap.Int64("segmentID", info.SegmentID)) } } l.req.Infos = filteredInfos @@ -536,38 +517,52 @@ func (l *loadSegmentsTask) PreExecute(ctx context.Context) error { } func (l *loadSegmentsTask) Execute(ctx context.Context) error { - // TODO: support db log.Info("LoadSegmentTask Execute start", zap.Int64("msgID", l.req.Base.MsgID)) err := l.node.loader.LoadSegment(l.req, segmentTypeSealed) if err != nil { - log.Warn(err.Error()) + log.Warn("failed to load segment", zap.Int64("collectionID", l.req.CollectionID), + zap.Int64("replicaID", l.req.ReplicaID), zap.Error(err)) + return err + } + vchanName := make([]string, 0) + for _, deltaPosition := range l.req.DeltaPositions { + vchanName = append(vchanName, deltaPosition.ChannelName) + } + // TODO delta channel need to released 1. if other watchDeltaChannel fail 2. when segment release + err = l.watchDeltaChannel(vchanName) + if err != nil { + // roll back + for _, segment := range l.req.Infos { + l.node.metaReplica.removeSegment(segment.SegmentID, segmentTypeSealed) + } + log.Warn("failed to watch Delta channel while load segment", zap.Int64("collectionID", l.req.CollectionID), + zap.Int64("replicaID", l.req.ReplicaID), zap.Error(err)) return err } - // reload delete log from cp to latest position + runningGroup, groupCtx := errgroup.WithContext(l.ctx) for _, deltaPosition := range l.req.DeltaPositions { - err = l.node.loader.FromDmlCPLoadDelete(ctx, l.req.CollectionID, deltaPosition) - if err != nil { - for _, segment := range l.req.Infos { - l.node.metaReplica.removeSegment(segment.SegmentID, segmentTypeSealed) - } - log.Warn("LoadSegmentTask from delta check point load delete failed", zap.Int64("msgID", l.req.Base.MsgID), zap.Error(err)) - return err + pos := deltaPosition + runningGroup.Go(func() error { + // reload data from dml channel + return l.node.loader.FromDmlCPLoadDelete(groupCtx, l.req.CollectionID, pos) + }) + } + err = runningGroup.Wait() + if err != nil { + for _, segment := range l.req.Infos { + l.node.metaReplica.removeSegment(segment.SegmentID, segmentTypeSealed) } + log.Warn("failed to load delete data while load segment", zap.Int64("collectionID", l.req.CollectionID), + zap.Int64("replicaID", l.req.ReplicaID), zap.Error(err)) + return err } - log.Info("LoadSegmentTask Execute done", zap.Int64("msgID", l.req.Base.MsgID)) + log.Info("LoadSegmentTask Execute done", zap.Int64("collectionID", l.req.CollectionID), + zap.Int64("replicaID", l.req.ReplicaID), zap.Int64("msgID", l.req.Base.MsgID)) return nil } -type ReplicaType int - -const ( - replicaNone ReplicaType = iota - replicaStreaming - replicaHistorical -) - func (r *releaseCollectionTask) Execute(ctx context.Context) error { log.Info("Execute release collection task", zap.Any("collectionID", r.req.CollectionID)) diff --git a/internal/querynode/task_test.go b/internal/querynode/task_test.go index 946b436bd1..d3fea898e4 100644 --- a/internal/querynode/task_test.go +++ b/internal/querynode/task_test.go @@ -21,11 +21,8 @@ import ( "testing" "time" - "github.com/apache/pulsar-client-go/pulsar" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/milvus-io/milvus/internal/mq/msgstream" + "github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper/rmq" "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/proto/internalpb" @@ -33,6 +30,8 @@ import ( "github.com/milvus-io/milvus/internal/proto/schemapb" "github.com/milvus-io/milvus/internal/util/funcutil" "github.com/milvus-io/milvus/internal/util/typeutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestTask_watchDmChannelsTask(t *testing.T) { @@ -269,93 +268,6 @@ func TestTask_watchDmChannelsTask(t *testing.T) { }) } -func TestTask_watchDeltaChannelsTask(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - genWatchDeltaChannelsRequest := func() *querypb.WatchDeltaChannelsRequest { - req := &querypb.WatchDeltaChannelsRequest{ - Base: genCommonMsgBase(commonpb.MsgType_WatchDeltaChannels), - CollectionID: defaultCollectionID, - } - return req - } - - t.Run("test timestamp", func(t *testing.T) { - timestamp := Timestamp(1000) - task := watchDeltaChannelsTask{ - baseTask: baseTask{ - ts: timestamp, - }, - req: genWatchDeltaChannelsRequest(), - } - resT := task.Timestamp() - assert.Equal(t, timestamp, resT) - }) - - t.Run("test OnEnqueue", func(t *testing.T) { - task := watchDeltaChannelsTask{ - req: genWatchDeltaChannelsRequest(), - } - err := task.OnEnqueue() - assert.NoError(t, err) - task.req.Base = nil - err = task.OnEnqueue() - assert.NoError(t, err) - }) - - t.Run("test execute", func(t *testing.T) { - node, err := genSimpleQueryNode(ctx) - assert.NoError(t, err) - - task := watchDeltaChannelsTask{ - req: genWatchDeltaChannelsRequest(), - node: node, - } - task.ctx = ctx - task.req.Infos = []*datapb.VchannelInfo{ - { - CollectionID: defaultCollectionID, - ChannelName: defaultDeltaChannel, - SeekPosition: &internalpb.MsgPosition{ - ChannelName: defaultDMLChannel, - MsgID: pulsar.EarliestMessageID().Serialize(), - MsgGroup: defaultSubName, - Timestamp: 0, - }, - }, - } - err = task.Execute(ctx) - assert.NoError(t, err) - }) - - t.Run("test execute without init collection", func(t *testing.T) { - node, err := genSimpleQueryNode(ctx) - assert.NoError(t, err) - - task := watchDeltaChannelsTask{ - req: genWatchDeltaChannelsRequest(), - node: node, - } - task.ctx = ctx - task.req.Infos = []*datapb.VchannelInfo{ - { - CollectionID: defaultCollectionID, - ChannelName: defaultDeltaChannel, - SeekPosition: &internalpb.MsgPosition{ - ChannelName: defaultDeltaChannel, - MsgID: []byte{1, 2, 3, 4, 5, 6, 7, 8}, - MsgGroup: defaultSubName, - Timestamp: 0, - }, - }, - } - task.req.CollectionID++ - err = task.Execute(ctx) - assert.Error(t, err) - }) -} - func TestTask_loadSegmentsTask(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -374,7 +286,8 @@ func TestTask_loadSegmentsTask(t *testing.T) { timestamp := Timestamp(1000) task := loadSegmentsTask{ baseTask: baseTask{ - ts: timestamp, + ts: timestamp, + ctx: ctx, }, req: genLoadEmptySegmentsRequest(), } @@ -384,6 +297,9 @@ func TestTask_loadSegmentsTask(t *testing.T) { t.Run("test OnEnqueue", func(t *testing.T) { task := loadSegmentsTask{ + baseTask: baseTask{ + ctx: ctx, + }, req: genLoadEmptySegmentsRequest(), } err := task.OnEnqueue() @@ -417,6 +333,9 @@ func TestTask_loadSegmentsTask(t *testing.T) { } task := loadSegmentsTask{ + baseTask: baseTask{ + ctx: ctx, + }, req: req, node: node, } @@ -428,6 +347,8 @@ func TestTask_loadSegmentsTask(t *testing.T) { node, err := genSimpleQueryNode(ctx) assert.NoError(t, err) + node.metaReplica.removeSegment(defaultSegmentID, segmentTypeSealed) + fieldBinlog, statsLog, err := saveBinLog(ctx, defaultCollectionID, defaultPartitionID, defaultSegmentID, defaultMsgLength, schema) assert.NoError(t, err) @@ -446,6 +367,9 @@ func TestTask_loadSegmentsTask(t *testing.T) { } task := loadSegmentsTask{ + baseTask: baseTask{ + ctx: ctx, + }, req: req, node: node, } @@ -556,6 +480,9 @@ func TestTask_loadSegmentsTask(t *testing.T) { } task := loadSegmentsTask{ + baseTask: baseTask{ + ctx: ctx, + }, req: req, node: node, } @@ -584,6 +511,9 @@ func TestTask_loadSegmentsTask(t *testing.T) { assert.NoError(t, err) task := loadSegmentsTask{ + baseTask: baseTask{ + ctx: ctx, + }, req: genLoadEmptySegmentsRequest(), node: node, } @@ -606,6 +536,94 @@ func TestTask_loadSegmentsTask(t *testing.T) { }) } +func TestTask_loadSegmentsTaskLoadDelta(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + schema := genTestCollectionSchema() + + t.Run("test repeated load delta channel", func(t *testing.T) { + node, err := genSimpleQueryNode(ctx) + assert.NoError(t, err) + vDmChannel := "by-dev-rootcoord-dml-test_2_2021v2" + + segmentLoadInfo := &querypb.SegmentLoadInfo{ + SegmentID: UniqueID(1000), + PartitionID: defaultPartitionID, + CollectionID: defaultCollectionID, + } + loadReq := &querypb.LoadSegmentsRequest{ + Base: genCommonMsgBase(commonpb.MsgType_LoadSegments), + Schema: schema, + Infos: []*querypb.SegmentLoadInfo{segmentLoadInfo}, + DeltaPositions: []*internalpb.MsgPosition{ + { + ChannelName: vDmChannel, + MsgID: rmq.SerializeRmqID(0), + Timestamp: 100, + }, + }, + } + + task := loadSegmentsTask{ + baseTask: baseTask{ + ctx: ctx, + }, + req: loadReq, + node: node, + } + // execute loadSegmentsTask twice + err = task.PreExecute(ctx) + assert.NoError(t, err) + err = task.Execute(ctx) + assert.NoError(t, err) + // expected only one segment in replica + num := node.metaReplica.getSegmentNum(segmentTypeSealed) + assert.Equal(t, 2, num) + + // load second segments with same channel + loadReq = &querypb.LoadSegmentsRequest{ + Base: genCommonMsgBase(commonpb.MsgType_LoadSegments), + Schema: schema, + Infos: []*querypb.SegmentLoadInfo{ + { + SegmentID: UniqueID(1001), + PartitionID: defaultPartitionID, + CollectionID: defaultCollectionID, + }, + }, + DeltaPositions: []*internalpb.MsgPosition{ + { + ChannelName: vDmChannel, + MsgID: rmq.SerializeRmqID(0), + Timestamp: 100, + }, + }, + } + + task = loadSegmentsTask{ + baseTask: baseTask{ + ctx: ctx, + }, + req: loadReq, + node: node, + } + // execute loadSegmentsTask twice + err = task.PreExecute(ctx) + assert.NoError(t, err) + err = task.Execute(ctx) + assert.NoError(t, err) + + num = node.metaReplica.getSegmentNum(segmentTypeSealed) + assert.Equal(t, 3, num) + + ok := node.queryShardService.hasQueryShard(vDmChannel) + assert.True(t, ok) + + assert.Equal(t, len(node.dataSyncService.dmlChannel2FlowGraph), 0) + assert.Equal(t, len(node.dataSyncService.deltaChannel2FlowGraph), 1) + }) +} + func TestTask_releaseCollectionTask(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/internal/types/types.go b/internal/types/types.go index ea6eda06ad..1f492afa94 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1223,7 +1223,6 @@ type QueryNode interface { TimeTickProvider WatchDmChannels(ctx context.Context, req *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) - WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) // LoadSegments notifies QueryNode to load the sealed segments from storage. The load tasks are sync to this // rpc, QueryNode will return after all the sealed segments are loaded. // diff --git a/internal/util/funcutil/parallel.go b/internal/util/funcutil/parallel.go index 03afada3cf..3a58b8f1ed 100644 --- a/internal/util/funcutil/parallel.go +++ b/internal/util/funcutil/parallel.go @@ -48,12 +48,10 @@ func ProcessFuncParallel(total, maxParallel int, f ProcessFunc, fname string) er t := time.Now() defer func() { - log.Debug(fname, zap.Any("time cost", time.Since(t))) + log.Debug(fname, zap.Any("total", total), zap.Any("time cost", time.Since(t))) }() nPerBatch := (total + maxParallel - 1) / maxParallel - log.Debug(fname, zap.Any("total", total)) - log.Debug(fname, zap.Any("nPerBatch", nPerBatch)) quit := make(chan bool) errc := make(chan error) @@ -105,8 +103,6 @@ func ProcessFuncParallel(total, maxParallel int, f ProcessFunc, fname string) er routineNum++ } - log.Debug(fname, zap.Any("NumOfGoRoutines", routineNum)) - if routineNum <= 0 { return nil } diff --git a/internal/util/mock/grpc_querynode_client.go b/internal/util/mock/grpc_querynode_client.go index b05bbb4fe0..7906662f0d 100644 --- a/internal/util/mock/grpc_querynode_client.go +++ b/internal/util/mock/grpc_querynode_client.go @@ -53,10 +53,6 @@ func (m *GrpcQueryNodeClient) WatchDmChannels(ctx context.Context, in *querypb.W return &commonpb.Status{}, m.Err } -func (m *GrpcQueryNodeClient) WatchDeltaChannels(ctx context.Context, in *querypb.WatchDeltaChannelsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) { - return &commonpb.Status{}, m.Err -} - func (m *GrpcQueryNodeClient) LoadSegments(ctx context.Context, in *querypb.LoadSegmentsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) { return &commonpb.Status{}, m.Err } diff --git a/internal/util/mock/querynode_client.go b/internal/util/mock/querynode_client.go index 87095a4588..3ac2b80bd8 100644 --- a/internal/util/mock/querynode_client.go +++ b/internal/util/mock/querynode_client.go @@ -69,10 +69,6 @@ func (q QueryNodeClient) WatchDmChannels(ctx context.Context, req *querypb.Watch return q.grpcClient.WatchDmChannels(ctx, req) } -func (q QueryNodeClient) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) { - return q.grpcClient.WatchDeltaChannels(ctx, req) -} - func (q QueryNodeClient) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { return q.grpcClient.LoadSegments(ctx, req) }