From e40d95e093e9694cd5bca592f57b62c837095333 Mon Sep 17 00:00:00 2001 From: wei liu Date: Thu, 15 Jun 2023 14:24:38 +0800 Subject: [PATCH] add search/query request cost metrics on qn (#24413) Signed-off-by: Wei Liu --- internal/proto/internal.proto | 14 + internal/proto/internalpb/internal.pb.go | 355 ++++++++++++------- internal/querynodev2/delegator/delegator.go | 2 +- internal/querynodev2/segments/result.go | 17 + internal/querynodev2/segments/result_test.go | 34 ++ internal/querynodev2/segments/utils.go | 14 + internal/querynodev2/services.go | 51 ++- internal/querynodev2/tasks/scheduler.go | 22 +- internal/querynodev2/tasks/task.go | 15 +- 9 files changed, 356 insertions(+), 168 deletions(-) diff --git a/internal/proto/internal.proto b/internal/proto/internal.proto index c379a4f9fd..ce525c162e 100644 --- a/internal/proto/internal.proto +++ b/internal/proto/internal.proto @@ -117,6 +117,15 @@ message SearchResults { bytes sliced_blob = 10; int64 sliced_num_count = 11; int64 sliced_offset = 12; + + // search request cost + CostAggregation costAggregation = 13; +} + +message CostAggregation { + int64 responseTime = 1; + int64 serviceTime = 2; + int64 totalNQ = 3; } message RetrieveRequest { @@ -136,6 +145,8 @@ message RetrieveRequest { int64 iteration_extension_reduce_rate = 14; } + + message RetrieveResults { common.MsgBase base = 1; common.Status status = 2; @@ -145,6 +156,9 @@ message RetrieveResults { repeated int64 sealed_segmentIDs_retrieved = 6; repeated string channelIDs_retrieved = 7; repeated int64 global_sealed_segmentIDs = 8; + + // query request cost + CostAggregation costAggregation = 13; } message LoadIndex { diff --git a/internal/proto/internalpb/internal.pb.go b/internal/proto/internalpb/internal.pb.go index 44d2211672..d971a98578 100644 --- a/internal/proto/internalpb/internal.pb.go +++ b/internal/proto/internalpb/internal.pb.go @@ -908,12 +908,14 @@ type SearchResults struct { ChannelIDsSearched []string `protobuf:"bytes,8,rep,name=channelIDs_searched,json=channelIDsSearched,proto3" json:"channelIDs_searched,omitempty"` GlobalSealedSegmentIDs []int64 `protobuf:"varint,9,rep,packed,name=global_sealed_segmentIDs,json=globalSealedSegmentIDs,proto3" json:"global_sealed_segmentIDs,omitempty"` // schema.SearchResultsData inside - SlicedBlob []byte `protobuf:"bytes,10,opt,name=sliced_blob,json=slicedBlob,proto3" json:"sliced_blob,omitempty"` - SlicedNumCount int64 `protobuf:"varint,11,opt,name=sliced_num_count,json=slicedNumCount,proto3" json:"sliced_num_count,omitempty"` - SlicedOffset int64 `protobuf:"varint,12,opt,name=sliced_offset,json=slicedOffset,proto3" json:"sliced_offset,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + SlicedBlob []byte `protobuf:"bytes,10,opt,name=sliced_blob,json=slicedBlob,proto3" json:"sliced_blob,omitempty"` + SlicedNumCount int64 `protobuf:"varint,11,opt,name=sliced_num_count,json=slicedNumCount,proto3" json:"sliced_num_count,omitempty"` + SlicedOffset int64 `protobuf:"varint,12,opt,name=sliced_offset,json=slicedOffset,proto3" json:"sliced_offset,omitempty"` + // search request cost + CostAggregation *CostAggregation `protobuf:"bytes,13,opt,name=costAggregation,proto3" json:"costAggregation,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *SearchResults) Reset() { *m = SearchResults{} } @@ -1025,6 +1027,68 @@ func (m *SearchResults) GetSlicedOffset() int64 { return 0 } +func (m *SearchResults) GetCostAggregation() *CostAggregation { + if m != nil { + return m.CostAggregation + } + return nil +} + +type CostAggregation struct { + ResponseTime int64 `protobuf:"varint,1,opt,name=responseTime,proto3" json:"responseTime,omitempty"` + ServiceTime int64 `protobuf:"varint,2,opt,name=serviceTime,proto3" json:"serviceTime,omitempty"` + TotalNQ int64 `protobuf:"varint,3,opt,name=totalNQ,proto3" json:"totalNQ,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CostAggregation) Reset() { *m = CostAggregation{} } +func (m *CostAggregation) String() string { return proto.CompactTextString(m) } +func (*CostAggregation) ProtoMessage() {} +func (*CostAggregation) Descriptor() ([]byte, []int) { + return fileDescriptor_41f4a519b878ee3b, []int{14} +} + +func (m *CostAggregation) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CostAggregation.Unmarshal(m, b) +} +func (m *CostAggregation) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CostAggregation.Marshal(b, m, deterministic) +} +func (m *CostAggregation) XXX_Merge(src proto.Message) { + xxx_messageInfo_CostAggregation.Merge(m, src) +} +func (m *CostAggregation) XXX_Size() int { + return xxx_messageInfo_CostAggregation.Size(m) +} +func (m *CostAggregation) XXX_DiscardUnknown() { + xxx_messageInfo_CostAggregation.DiscardUnknown(m) +} + +var xxx_messageInfo_CostAggregation proto.InternalMessageInfo + +func (m *CostAggregation) GetResponseTime() int64 { + if m != nil { + return m.ResponseTime + } + return 0 +} + +func (m *CostAggregation) GetServiceTime() int64 { + if m != nil { + return m.ServiceTime + } + return 0 +} + +func (m *CostAggregation) GetTotalNQ() int64 { + if m != nil { + return m.TotalNQ + } + return 0 +} + type RetrieveRequest struct { Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` ReqID int64 `protobuf:"varint,2,opt,name=reqID,proto3" json:"reqID,omitempty"` @@ -1049,7 +1113,7 @@ func (m *RetrieveRequest) Reset() { *m = RetrieveRequest{} } func (m *RetrieveRequest) String() string { return proto.CompactTextString(m) } func (*RetrieveRequest) ProtoMessage() {} func (*RetrieveRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{14} + return fileDescriptor_41f4a519b878ee3b, []int{15} } func (m *RetrieveRequest) XXX_Unmarshal(b []byte) error { @@ -1177,16 +1241,18 @@ type RetrieveResults struct { SealedSegmentIDsRetrieved []int64 `protobuf:"varint,6,rep,packed,name=sealed_segmentIDs_retrieved,json=sealedSegmentIDsRetrieved,proto3" json:"sealed_segmentIDs_retrieved,omitempty"` ChannelIDsRetrieved []string `protobuf:"bytes,7,rep,name=channelIDs_retrieved,json=channelIDsRetrieved,proto3" json:"channelIDs_retrieved,omitempty"` GlobalSealedSegmentIDs []int64 `protobuf:"varint,8,rep,packed,name=global_sealed_segmentIDs,json=globalSealedSegmentIDs,proto3" json:"global_sealed_segmentIDs,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + // query request cost + CostAggregation *CostAggregation `protobuf:"bytes,13,opt,name=costAggregation,proto3" json:"costAggregation,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *RetrieveResults) Reset() { *m = RetrieveResults{} } func (m *RetrieveResults) String() string { return proto.CompactTextString(m) } func (*RetrieveResults) ProtoMessage() {} func (*RetrieveResults) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{15} + return fileDescriptor_41f4a519b878ee3b, []int{16} } func (m *RetrieveResults) XXX_Unmarshal(b []byte) error { @@ -1263,6 +1329,13 @@ func (m *RetrieveResults) GetGlobalSealedSegmentIDs() []int64 { return nil } +func (m *RetrieveResults) GetCostAggregation() *CostAggregation { + if m != nil { + return m.CostAggregation + } + return nil +} + type LoadIndex struct { Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` SegmentID int64 `protobuf:"varint,2,opt,name=segmentID,proto3" json:"segmentID,omitempty"` @@ -1279,7 +1352,7 @@ func (m *LoadIndex) Reset() { *m = LoadIndex{} } func (m *LoadIndex) String() string { return proto.CompactTextString(m) } func (*LoadIndex) ProtoMessage() {} func (*LoadIndex) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{16} + return fileDescriptor_41f4a519b878ee3b, []int{17} } func (m *LoadIndex) XXX_Unmarshal(b []byte) error { @@ -1354,7 +1427,7 @@ func (m *IndexStats) Reset() { *m = IndexStats{} } func (m *IndexStats) String() string { return proto.CompactTextString(m) } func (*IndexStats) ProtoMessage() {} func (*IndexStats) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{17} + return fileDescriptor_41f4a519b878ee3b, []int{18} } func (m *IndexStats) XXX_Unmarshal(b []byte) error { @@ -1402,7 +1475,7 @@ func (m *FieldStats) Reset() { *m = FieldStats{} } func (m *FieldStats) String() string { return proto.CompactTextString(m) } func (*FieldStats) ProtoMessage() {} func (*FieldStats) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{18} + return fileDescriptor_41f4a519b878ee3b, []int{19} } func (m *FieldStats) XXX_Unmarshal(b []byte) error { @@ -1458,7 +1531,7 @@ func (m *SegmentStats) Reset() { *m = SegmentStats{} } func (m *SegmentStats) String() string { return proto.CompactTextString(m) } func (*SegmentStats) ProtoMessage() {} func (*SegmentStats) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{19} + return fileDescriptor_41f4a519b878ee3b, []int{20} } func (m *SegmentStats) XXX_Unmarshal(b []byte) error { @@ -1521,7 +1594,7 @@ func (m *ChannelTimeTickMsg) Reset() { *m = ChannelTimeTickMsg{} } func (m *ChannelTimeTickMsg) String() string { return proto.CompactTextString(m) } func (*ChannelTimeTickMsg) ProtoMessage() {} func (*ChannelTimeTickMsg) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{20} + return fileDescriptor_41f4a519b878ee3b, []int{21} } func (m *ChannelTimeTickMsg) XXX_Unmarshal(b []byte) error { @@ -1587,7 +1660,7 @@ func (m *CredentialInfo) Reset() { *m = CredentialInfo{} } func (m *CredentialInfo) String() string { return proto.CompactTextString(m) } func (*CredentialInfo) ProtoMessage() {} func (*CredentialInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{21} + return fileDescriptor_41f4a519b878ee3b, []int{22} } func (m *CredentialInfo) XXX_Unmarshal(b []byte) error { @@ -1655,7 +1728,7 @@ func (m *ListPolicyRequest) Reset() { *m = ListPolicyRequest{} } func (m *ListPolicyRequest) String() string { return proto.CompactTextString(m) } func (*ListPolicyRequest) ProtoMessage() {} func (*ListPolicyRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{22} + return fileDescriptor_41f4a519b878ee3b, []int{23} } func (m *ListPolicyRequest) XXX_Unmarshal(b []byte) error { @@ -1697,7 +1770,7 @@ func (m *ListPolicyResponse) Reset() { *m = ListPolicyResponse{} } func (m *ListPolicyResponse) String() string { return proto.CompactTextString(m) } func (*ListPolicyResponse) ProtoMessage() {} func (*ListPolicyResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{23} + return fileDescriptor_41f4a519b878ee3b, []int{24} } func (m *ListPolicyResponse) XXX_Unmarshal(b []byte) error { @@ -1751,7 +1824,7 @@ func (m *ShowConfigurationsRequest) Reset() { *m = ShowConfigurationsReq func (m *ShowConfigurationsRequest) String() string { return proto.CompactTextString(m) } func (*ShowConfigurationsRequest) ProtoMessage() {} func (*ShowConfigurationsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{24} + return fileDescriptor_41f4a519b878ee3b, []int{25} } func (m *ShowConfigurationsRequest) XXX_Unmarshal(b []byte) error { @@ -1798,7 +1871,7 @@ func (m *ShowConfigurationsResponse) Reset() { *m = ShowConfigurationsRe func (m *ShowConfigurationsResponse) String() string { return proto.CompactTextString(m) } func (*ShowConfigurationsResponse) ProtoMessage() {} func (*ShowConfigurationsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{25} + return fileDescriptor_41f4a519b878ee3b, []int{26} } func (m *ShowConfigurationsResponse) XXX_Unmarshal(b []byte) error { @@ -1845,7 +1918,7 @@ func (m *Rate) Reset() { *m = Rate{} } func (m *Rate) String() string { return proto.CompactTextString(m) } func (*Rate) ProtoMessage() {} func (*Rate) Descriptor() ([]byte, []int) { - return fileDescriptor_41f4a519b878ee3b, []int{26} + return fileDescriptor_41f4a519b878ee3b, []int{27} } func (m *Rate) XXX_Unmarshal(b []byte) error { @@ -1896,6 +1969,7 @@ func init() { proto.RegisterType((*CreateIndexRequest)(nil), "milvus.proto.internal.CreateIndexRequest") proto.RegisterType((*SearchRequest)(nil), "milvus.proto.internal.SearchRequest") proto.RegisterType((*SearchResults)(nil), "milvus.proto.internal.SearchResults") + proto.RegisterType((*CostAggregation)(nil), "milvus.proto.internal.CostAggregation") proto.RegisterType((*RetrieveRequest)(nil), "milvus.proto.internal.RetrieveRequest") proto.RegisterType((*RetrieveResults)(nil), "milvus.proto.internal.RetrieveResults") proto.RegisterType((*LoadIndex)(nil), "milvus.proto.internal.LoadIndex") @@ -1914,120 +1988,125 @@ func init() { func init() { proto.RegisterFile("internal.proto", fileDescriptor_41f4a519b878ee3b) } var fileDescriptor_41f4a519b878ee3b = []byte{ - // 1835 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x58, 0xcd, 0x6f, 0x1c, 0x49, - 0x15, 0xa7, 0xe7, 0x7b, 0xde, 0x8c, 0x9d, 0x71, 0xc5, 0x09, 0x1d, 0x27, 0xbb, 0xf6, 0x36, 0x48, - 0x98, 0x45, 0x1b, 0x2f, 0x5e, 0xed, 0x86, 0x03, 0x02, 0xc5, 0x6e, 0xaf, 0x35, 0xda, 0x71, 0x70, - 0x7a, 0xa2, 0x95, 0xe0, 0xd2, 0xaa, 0x99, 0x7e, 0x1e, 0x17, 0xe9, 0x2f, 0x57, 0x55, 0xc7, 0x76, - 0xce, 0xdc, 0x90, 0xb8, 0x71, 0x41, 0x82, 0x33, 0x42, 0xe2, 0x0c, 0x37, 0xfe, 0xaf, 0x15, 0x07, - 0x54, 0x1f, 0x3d, 0x1f, 0xb6, 0x63, 0xd9, 0x8e, 0x80, 0xdd, 0x5b, 0xbd, 0xf7, 0x7e, 0xf5, 0xaa, - 0xea, 0xbd, 0x57, 0xbf, 0x7e, 0xd5, 0xb0, 0xcc, 0x52, 0x89, 0x3c, 0xa5, 0xf1, 0xd3, 0x9c, 0x67, - 0x32, 0x23, 0x0f, 0x12, 0x16, 0xbf, 0x29, 0x84, 0x91, 0x9e, 0x96, 0xc6, 0xb5, 0xee, 0x38, 0x4b, - 0x92, 0x2c, 0x35, 0xea, 0xb5, 0xae, 0x18, 0x1f, 0x63, 0x42, 0x8d, 0xe4, 0x3d, 0x86, 0x47, 0xfb, - 0x28, 0x5f, 0xb1, 0x04, 0x5f, 0xb1, 0xf1, 0xeb, 0xdd, 0x63, 0x9a, 0xa6, 0x18, 0x07, 0x78, 0x52, - 0xa0, 0x90, 0xde, 0x07, 0xf0, 0x78, 0x1f, 0xe5, 0x50, 0x52, 0xc9, 0x84, 0x64, 0x63, 0x71, 0xc1, - 0xfc, 0x00, 0xee, 0xef, 0xa3, 0xf4, 0xa3, 0x0b, 0xea, 0xaf, 0xa1, 0xf5, 0x22, 0x8b, 0xb0, 0x9f, - 0x1e, 0x65, 0xe4, 0x0b, 0x68, 0xd2, 0x28, 0xe2, 0x28, 0x84, 0xeb, 0x6c, 0x38, 0x9b, 0x9d, 0xed, - 0x27, 0x4f, 0x17, 0xf6, 0x68, 0x77, 0xf6, 0xdc, 0x60, 0x82, 0x12, 0x4c, 0x08, 0xd4, 0x78, 0x16, - 0xa3, 0x5b, 0xd9, 0x70, 0x36, 0xdb, 0x81, 0x1e, 0x7b, 0xbf, 0x05, 0xe8, 0xa7, 0x4c, 0x1e, 0x52, - 0x4e, 0x13, 0x41, 0x1e, 0x42, 0x23, 0x55, 0xab, 0xf8, 0xda, 0x71, 0x35, 0xb0, 0x12, 0xf1, 0xa1, - 0x2b, 0x24, 0xe5, 0x32, 0xcc, 0x35, 0xce, 0xad, 0x6c, 0x54, 0x37, 0x3b, 0xdb, 0x1f, 0x5d, 0xb9, - 0xec, 0x57, 0x78, 0xfe, 0x35, 0x8d, 0x0b, 0x3c, 0xa4, 0x8c, 0x07, 0x1d, 0x3d, 0xcd, 0x78, 0xf7, - 0x7e, 0x0d, 0x30, 0x94, 0x9c, 0xa5, 0x93, 0x01, 0x13, 0x52, 0xad, 0xf5, 0x46, 0xe1, 0xd4, 0x21, - 0xaa, 0x9b, 0xed, 0xc0, 0x4a, 0xe4, 0x33, 0x68, 0x08, 0x49, 0x65, 0x21, 0xf4, 0x3e, 0x3b, 0xdb, - 0x8f, 0xaf, 0x5c, 0x65, 0xa8, 0x21, 0x81, 0x85, 0x7a, 0x7f, 0xaf, 0xc0, 0xea, 0x42, 0x54, 0x6d, - 0xdc, 0xc8, 0xa7, 0x50, 0x1b, 0x51, 0x81, 0xd7, 0x06, 0xea, 0x40, 0x4c, 0x76, 0xa8, 0xc0, 0x40, - 0x23, 0x55, 0x94, 0xa2, 0x51, 0xdf, 0xd7, 0xab, 0x57, 0x03, 0x3d, 0x26, 0x1e, 0x74, 0xc7, 0x59, - 0x1c, 0xe3, 0x58, 0xb2, 0x2c, 0xed, 0xfb, 0x6e, 0x55, 0xdb, 0x16, 0x74, 0x0a, 0x93, 0x53, 0x2e, - 0x99, 0x11, 0x85, 0x5b, 0xdb, 0xa8, 0x2a, 0xcc, 0xbc, 0x8e, 0xfc, 0x18, 0x7a, 0x92, 0xd3, 0x37, - 0x18, 0x87, 0x92, 0x25, 0x28, 0x24, 0x4d, 0x72, 0xb7, 0xbe, 0xe1, 0x6c, 0xd6, 0x82, 0x7b, 0x46, - 0xff, 0xaa, 0x54, 0x93, 0x2d, 0xb8, 0x3f, 0x29, 0x28, 0xa7, 0xa9, 0x44, 0x9c, 0x43, 0x37, 0x34, - 0x9a, 0x4c, 0x4d, 0xb3, 0x09, 0x3f, 0x81, 0x15, 0x05, 0xcb, 0x0a, 0x39, 0x07, 0x6f, 0x6a, 0x78, - 0xcf, 0x1a, 0xa6, 0x60, 0xef, 0x1f, 0x0e, 0x3c, 0xb8, 0x10, 0x2f, 0x91, 0x67, 0xa9, 0xc0, 0x3b, - 0x04, 0xec, 0x2e, 0x09, 0x23, 0xcf, 0xa0, 0xae, 0x46, 0xc2, 0xad, 0xde, 0xb4, 0x94, 0x0c, 0xde, - 0xfb, 0x8b, 0x03, 0x64, 0x97, 0x23, 0x95, 0xf8, 0x3c, 0x66, 0xf4, 0x3d, 0xf2, 0xfc, 0x7d, 0x68, - 0x46, 0xa3, 0x30, 0xa5, 0x49, 0x79, 0x21, 0x1a, 0xd1, 0xe8, 0x05, 0x4d, 0x90, 0xfc, 0x08, 0xee, - 0xcd, 0x12, 0x6b, 0x00, 0x55, 0x0d, 0x58, 0x9e, 0xa9, 0x35, 0x70, 0x15, 0xea, 0x54, 0xed, 0xc1, - 0xad, 0x69, 0xb3, 0x11, 0x3c, 0x01, 0x3d, 0x9f, 0x67, 0xf9, 0x7f, 0x6b, 0x77, 0xd3, 0x45, 0xab, - 0xf3, 0x8b, 0xfe, 0xd9, 0x81, 0x95, 0xe7, 0xb1, 0x44, 0xfe, 0x2d, 0x0d, 0xca, 0xbf, 0x2a, 0x65, - 0xd6, 0xfa, 0x69, 0x84, 0x67, 0xff, 0xcf, 0x0d, 0x7e, 0x00, 0x70, 0xc4, 0x30, 0x8e, 0x0c, 0xc6, - 0xec, 0xb2, 0xad, 0x35, 0xda, 0x5c, 0x5e, 0xff, 0xfa, 0x35, 0xd7, 0xbf, 0x71, 0xc5, 0xf5, 0x77, - 0xa1, 0xa9, 0x9d, 0xf4, 0x7d, 0x7d, 0xe9, 0xaa, 0x41, 0x29, 0x2a, 0xf2, 0xc4, 0x33, 0xc9, 0x69, - 0x49, 0x9e, 0xad, 0x1b, 0x93, 0xa7, 0x9e, 0x66, 0xc9, 0xf3, 0x9b, 0x1a, 0x2c, 0x0d, 0x91, 0xf2, - 0xf1, 0xf1, 0xdd, 0x83, 0xb7, 0x0a, 0x75, 0x8e, 0x27, 0x53, 0x6e, 0x33, 0xc2, 0xf4, 0xc4, 0xd5, - 0x6b, 0x4e, 0x5c, 0xbb, 0x01, 0xe1, 0xd5, 0xaf, 0x20, 0xbc, 0x1e, 0x54, 0x23, 0x11, 0xeb, 0x80, - 0xb5, 0x03, 0x35, 0x54, 0x34, 0x95, 0xc7, 0x74, 0x8c, 0xc7, 0x59, 0x1c, 0x21, 0x0f, 0x27, 0x3c, - 0x2b, 0x0c, 0x4d, 0x75, 0x83, 0xde, 0x9c, 0x61, 0x5f, 0xe9, 0xc9, 0x33, 0x68, 0x45, 0x22, 0x0e, - 0xe5, 0x79, 0x8e, 0x6e, 0x6b, 0xc3, 0xd9, 0x5c, 0x7e, 0xc7, 0x31, 0x7d, 0x11, 0xbf, 0x3a, 0xcf, - 0x31, 0x68, 0x46, 0x66, 0x40, 0x3e, 0x85, 0x55, 0x81, 0x9c, 0xd1, 0x98, 0xbd, 0xc5, 0x28, 0xc4, - 0xb3, 0x9c, 0x87, 0x79, 0x4c, 0x53, 0xb7, 0xad, 0x17, 0x22, 0x33, 0xdb, 0xde, 0x59, 0xce, 0x0f, - 0x63, 0x9a, 0x92, 0x4d, 0xe8, 0x65, 0x85, 0xcc, 0x0b, 0x19, 0xea, 0xbc, 0x89, 0x90, 0x45, 0x2e, - 0xe8, 0x13, 0x2d, 0x1b, 0xfd, 0x97, 0x5a, 0xdd, 0x8f, 0xae, 0x24, 0xf1, 0xce, 0xad, 0x48, 0xbc, - 0x7b, 0x3b, 0x12, 0x5f, 0xba, 0x9a, 0xc4, 0xc9, 0x32, 0x54, 0xd2, 0x13, 0x77, 0x59, 0xa7, 0xa6, - 0x92, 0x9e, 0xa8, 0x44, 0xca, 0x2c, 0x7f, 0xed, 0xde, 0x33, 0x89, 0x54, 0x63, 0xf2, 0x21, 0x40, - 0x82, 0x92, 0xb3, 0xb1, 0x0a, 0x8b, 0xdb, 0xd3, 0x79, 0x98, 0xd3, 0x90, 0x1f, 0xc2, 0x12, 0x9b, - 0xa4, 0x19, 0xc7, 0x7d, 0x9e, 0x9d, 0xb2, 0x74, 0xe2, 0xae, 0x6c, 0x38, 0x9b, 0xad, 0x60, 0x51, - 0xe9, 0xfd, 0xbb, 0x3a, 0x2b, 0x3e, 0x51, 0xc4, 0x52, 0xfc, 0xaf, 0x3e, 0x13, 0xd3, 0x8a, 0xad, - 0xce, 0x57, 0xec, 0x3a, 0x74, 0xcc, 0x11, 0x4c, 0x65, 0xd4, 0x2e, 0x9d, 0x6a, 0x1d, 0x3a, 0x69, - 0x91, 0x84, 0x27, 0x05, 0x72, 0x86, 0xc2, 0xde, 0x65, 0x48, 0x8b, 0xe4, 0xa5, 0xd1, 0x90, 0xfb, - 0x50, 0x97, 0x59, 0x1e, 0xbe, 0xb6, 0x57, 0x59, 0xc5, 0xea, 0x2b, 0xf2, 0x73, 0x58, 0x13, 0x48, - 0x63, 0x8c, 0x42, 0x81, 0x93, 0x04, 0x53, 0xd9, 0xf7, 0x45, 0x28, 0xf4, 0xb1, 0x31, 0x72, 0x9b, - 0xba, 0x18, 0x5c, 0x83, 0x18, 0x4e, 0x01, 0x43, 0x6b, 0x57, 0xb9, 0x1e, 0x9b, 0x9e, 0x6d, 0x61, - 0x5a, 0x4b, 0x37, 0x37, 0x64, 0x66, 0x9a, 0x4e, 0xf8, 0x19, 0xb8, 0x93, 0x38, 0x1b, 0xd1, 0x38, - 0xbc, 0xb4, 0xaa, 0xdb, 0xd6, 0x8b, 0x3d, 0x34, 0xf6, 0xe1, 0x85, 0x25, 0xd5, 0xf1, 0x44, 0xcc, - 0xc6, 0x18, 0x85, 0xa3, 0x38, 0x1b, 0xb9, 0xa0, 0x8b, 0x1a, 0x8c, 0x6a, 0x27, 0xce, 0x46, 0xaa, - 0x98, 0x2d, 0x40, 0x85, 0x61, 0x9c, 0x15, 0xa9, 0xd4, 0x25, 0x5a, 0x0d, 0x96, 0x8d, 0xfe, 0x45, - 0x91, 0xec, 0x2a, 0x2d, 0xf9, 0x01, 0x2c, 0x59, 0x64, 0x76, 0x74, 0x24, 0x50, 0xea, 0xda, 0xac, - 0x06, 0x5d, 0xa3, 0xfc, 0x95, 0xd6, 0x79, 0xff, 0xac, 0xc1, 0xbd, 0x40, 0x45, 0x17, 0xdf, 0xe0, - 0x77, 0x89, 0x7d, 0xde, 0xc5, 0x02, 0x8d, 0x5b, 0xb1, 0x40, 0xf3, 0xc6, 0x2c, 0xd0, 0xba, 0x15, - 0x0b, 0xb4, 0x6f, 0xc7, 0x02, 0xf0, 0x0e, 0x16, 0x58, 0x85, 0x7a, 0xcc, 0x12, 0x56, 0x26, 0xd8, - 0x08, 0x97, 0xef, 0x75, 0xf7, 0x8a, 0x7b, 0x4d, 0x1e, 0x41, 0x8b, 0x09, 0x5b, 0x1f, 0x4b, 0x1a, - 0xd0, 0x64, 0xc2, 0x14, 0xc6, 0x1e, 0xac, 0x33, 0x89, 0x9c, 0xea, 0xcf, 0x29, 0x9e, 0x49, 0x4c, - 0x85, 0x1a, 0x71, 0x8c, 0x8a, 0x31, 0x86, 0x9c, 0x4a, 0xb4, 0xcc, 0xf3, 0x64, 0x0a, 0xdb, 0x2b, - 0x51, 0x81, 0x06, 0x05, 0x54, 0xa2, 0xf7, 0xd7, 0xea, 0x7c, 0xe9, 0x7c, 0x0b, 0xb8, 0xe3, 0x63, - 0xa8, 0xb2, 0xc8, 0x74, 0x27, 0x9d, 0x6d, 0x77, 0xd1, 0x8f, 0x7d, 0xc4, 0xf5, 0x7d, 0x11, 0x28, - 0x10, 0xf9, 0x25, 0x74, 0x6c, 0x19, 0x44, 0x54, 0x52, 0x5d, 0x62, 0x9d, 0xed, 0x0f, 0xaf, 0x9c, - 0xa3, 0xeb, 0xc2, 0xa7, 0x92, 0x06, 0xa6, 0xbb, 0x10, 0x6a, 0x4c, 0x7e, 0x01, 0x8f, 0x2f, 0x33, - 0x0a, 0xb7, 0xe1, 0x88, 0xdc, 0x86, 0xae, 0xac, 0x47, 0x17, 0x29, 0xa5, 0x8c, 0x57, 0x44, 0x7e, - 0x0a, 0xab, 0x73, 0x9c, 0x32, 0x9b, 0xd8, 0xd4, 0xa4, 0x32, 0xc7, 0x37, 0xb3, 0x29, 0xd7, 0xb1, - 0x4a, 0xeb, 0x3a, 0x56, 0xf1, 0xbe, 0x71, 0xa0, 0x3d, 0xc8, 0x68, 0xa4, 0x3b, 0xb4, 0x3b, 0x24, - 0xe9, 0x09, 0xb4, 0xa7, 0x6b, 0xd9, 0x3b, 0x3e, 0x53, 0x28, 0xeb, 0xb4, 0xc9, 0xb2, 0x9d, 0xd9, - 0x5c, 0xd7, 0x35, 0xd7, 0x3d, 0xd5, 0x16, 0xbb, 0xa7, 0x75, 0xe8, 0x30, 0xb5, 0xa1, 0x30, 0xa7, - 0xf2, 0xd8, 0x5c, 0xf3, 0x76, 0x00, 0x5a, 0x75, 0xa8, 0x34, 0xaa, 0xbd, 0x2a, 0x01, 0xba, 0xbd, - 0x6a, 0xdc, 0xb8, 0xbd, 0xb2, 0x4e, 0x74, 0x7b, 0xf5, 0x3b, 0x47, 0x3d, 0x84, 0x23, 0x3c, 0x53, - 0x45, 0x74, 0xd9, 0xa9, 0x73, 0x17, 0xa7, 0x8a, 0x7f, 0x14, 0xff, 0x72, 0x8c, 0xa9, 0x9c, 0x65, - 0x42, 0xd8, 0xe0, 0x90, 0xb4, 0x48, 0x02, 0x63, 0xb2, 0x59, 0x10, 0xde, 0x1f, 0x1c, 0x00, 0x5d, - 0x4a, 0x66, 0x1b, 0x17, 0x89, 0xd0, 0xb9, 0xbe, 0xf1, 0xac, 0x2c, 0x86, 0x6e, 0xa7, 0x0c, 0xdd, - 0x35, 0x2f, 0xad, 0xe9, 0xcf, 0x8e, 0xd9, 0xe1, 0x6d, 0x74, 0xf5, 0xd8, 0xfb, 0xa3, 0x03, 0x5d, - 0xbb, 0x3b, 0xb3, 0xa5, 0x85, 0x2c, 0x3b, 0x17, 0xb3, 0xac, 0xbf, 0xcc, 0x49, 0xc6, 0xcf, 0x43, - 0xc1, 0xde, 0xa2, 0xdd, 0x10, 0x18, 0xd5, 0x90, 0xbd, 0x45, 0xc5, 0x38, 0x3a, 0x24, 0xd9, 0xa9, - 0xb0, 0xf7, 0xb2, 0xa9, 0xc2, 0x90, 0x9d, 0x0a, 0xc5, 0x7a, 0x1c, 0xc7, 0x98, 0xca, 0xf8, 0x3c, - 0x4c, 0xb2, 0x88, 0x1d, 0x31, 0x8c, 0x74, 0x35, 0xb4, 0x82, 0x5e, 0x69, 0x38, 0xb0, 0x7a, 0xf5, - 0x80, 0x25, 0xf6, 0x17, 0x49, 0xf9, 0x9f, 0xe5, 0x40, 0x4c, 0xee, 0x50, 0xb5, 0x2a, 0xc4, 0xc6, - 0x8f, 0x2a, 0x44, 0xf3, 0x6b, 0xa3, 0x1d, 0x2c, 0xe8, 0x54, 0x13, 0x35, 0xe5, 0x61, 0x13, 0xc7, - 0x5a, 0x30, 0xa7, 0x51, 0x3b, 0x8f, 0xf0, 0x88, 0x16, 0xf1, 0x3c, 0x5f, 0xd7, 0x0c, 0x5f, 0x5b, - 0xc3, 0xc2, 0xd3, 0x7b, 0x79, 0x97, 0x63, 0x84, 0xa9, 0x64, 0x34, 0xd6, 0x3f, 0x74, 0xd6, 0xa0, - 0x55, 0x08, 0x95, 0x86, 0xc4, 0xec, 0xbc, 0x1d, 0x4c, 0x65, 0xf2, 0x09, 0x10, 0x4c, 0xc7, 0xfc, - 0x3c, 0x57, 0x15, 0x94, 0x53, 0x21, 0x4e, 0x33, 0x1e, 0xd9, 0xb7, 0xcf, 0xca, 0xd4, 0x72, 0x68, - 0x0d, 0xe4, 0x21, 0x34, 0x24, 0xa6, 0x34, 0x95, 0xf6, 0x8e, 0x59, 0xc9, 0x32, 0xbd, 0x28, 0x72, - 0xe4, 0x36, 0xa6, 0x4d, 0x26, 0x86, 0x4a, 0x54, 0x2f, 0x27, 0x71, 0x4c, 0xb7, 0x3f, 0xff, 0x62, - 0xe6, 0xbe, 0x6e, 0x5e, 0x4e, 0x46, 0x5d, 0xfa, 0xf6, 0xf6, 0x60, 0x65, 0xc0, 0x84, 0x3c, 0xcc, - 0x62, 0x36, 0x3e, 0xbf, 0x73, 0x1f, 0xe0, 0xfd, 0xde, 0x01, 0x32, 0xef, 0xc7, 0xfe, 0x78, 0x98, - 0x71, 0xbc, 0x73, 0x73, 0x8e, 0xff, 0x08, 0xba, 0xb9, 0x76, 0x13, 0xb2, 0xf4, 0x28, 0x2b, 0xb3, - 0xd7, 0x31, 0x3a, 0x15, 0x5b, 0xa1, 0xde, 0x7b, 0x2a, 0x98, 0x21, 0xcf, 0x62, 0x34, 0xc9, 0x6b, - 0x07, 0x6d, 0xa5, 0x09, 0x94, 0xc2, 0x9b, 0xc0, 0xa3, 0xe1, 0x71, 0x76, 0xba, 0x9b, 0xa5, 0x47, - 0x6c, 0x52, 0x98, 0x0f, 0xd9, 0x7b, 0x3c, 0xa0, 0x5d, 0x68, 0xe6, 0x54, 0xaa, 0x3b, 0x65, 0x73, - 0x54, 0x8a, 0xde, 0x9f, 0x1c, 0x58, 0xbb, 0x6a, 0xa5, 0xf7, 0x39, 0xfe, 0x3e, 0x2c, 0x8d, 0x8d, - 0x3b, 0xe3, 0xed, 0xe6, 0x3f, 0xe6, 0x16, 0xe7, 0x79, 0x7b, 0x50, 0x53, 0x9f, 0x6b, 0xb2, 0x05, - 0x15, 0x2e, 0xf5, 0x0e, 0x96, 0xb7, 0xd7, 0xdf, 0xc1, 0x14, 0x0a, 0xa8, 0x5f, 0x5b, 0x15, 0x2e, - 0x49, 0x17, 0x1c, 0xae, 0x4f, 0xea, 0x04, 0x0e, 0xff, 0xf8, 0x6f, 0x0e, 0xb4, 0x4a, 0x33, 0x59, - 0x81, 0x25, 0xdf, 0x1f, 0xec, 0x4e, 0xb9, 0xaa, 0xf7, 0x3d, 0xd2, 0x83, 0xae, 0xef, 0x0f, 0x0e, - 0xcb, 0x1e, 0xad, 0xe7, 0x90, 0x2e, 0xb4, 0x7c, 0x7f, 0xa0, 0xc9, 0xa7, 0x57, 0xb1, 0xd2, 0x97, - 0x71, 0x21, 0x8e, 0x7b, 0xd5, 0xa9, 0x83, 0x24, 0xa7, 0xc6, 0x41, 0x8d, 0x2c, 0x41, 0xdb, 0x3f, - 0x18, 0xf4, 0x53, 0x81, 0x5c, 0xf6, 0xea, 0x56, 0xf4, 0x31, 0x46, 0x89, 0xbd, 0x06, 0xb9, 0x07, - 0x1d, 0xff, 0x60, 0xb0, 0x53, 0xc4, 0xaf, 0xd5, 0x77, 0xac, 0xd7, 0xd4, 0xf6, 0x97, 0x03, 0xd3, - 0x71, 0xf7, 0x5a, 0xda, 0xfd, 0xcb, 0x81, 0x7a, 0x03, 0x9c, 0xf7, 0xda, 0x3b, 0xcf, 0x7e, 0xf3, - 0xf9, 0x84, 0xc9, 0xe3, 0x62, 0xa4, 0x02, 0xb4, 0x65, 0xce, 0xfa, 0x09, 0xcb, 0xec, 0x68, 0xab, - 0x3c, 0xef, 0x96, 0x3e, 0xfe, 0x54, 0xcc, 0x47, 0xa3, 0x86, 0xd6, 0x7c, 0xf6, 0x9f, 0x00, 0x00, - 0x00, 0xff, 0xff, 0x56, 0x6c, 0x12, 0x3b, 0x2a, 0x16, 0x00, 0x00, + // 1906 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x58, 0x4b, 0x6f, 0x1c, 0xb9, + 0x11, 0x4e, 0xcf, 0x7b, 0x6a, 0x46, 0xd2, 0x88, 0x2b, 0x3b, 0xed, 0xc7, 0xae, 0xb5, 0x9d, 0x20, + 0x51, 0x36, 0x58, 0x7b, 0xa3, 0xc5, 0xae, 0x73, 0x08, 0x12, 0xd8, 0x6a, 0xad, 0x30, 0xd8, 0x91, + 0x23, 0xf7, 0x18, 0x0b, 0x24, 0x97, 0x06, 0x67, 0xba, 0x34, 0x62, 0xdc, 0x2f, 0x91, 0x6c, 0x3d, + 0x7c, 0xce, 0x2d, 0x40, 0x6e, 0xb9, 0x04, 0x48, 0x7e, 0x40, 0x80, 0x9c, 0x93, 0x5b, 0xfe, 0x41, + 0x7e, 0x90, 0x4f, 0x01, 0x1f, 0x3d, 0x0f, 0x69, 0x2c, 0x48, 0x72, 0x1e, 0x9b, 0x1b, 0x59, 0xf5, + 0xb1, 0x48, 0x56, 0x15, 0x3f, 0x16, 0x09, 0xab, 0x2c, 0x95, 0xc8, 0x53, 0x1a, 0x3f, 0xce, 0x79, + 0x26, 0x33, 0x72, 0x27, 0x61, 0xf1, 0x49, 0x21, 0x4c, 0xef, 0x71, 0xa9, 0xbc, 0xdf, 0x1d, 0x67, + 0x49, 0x92, 0xa5, 0x46, 0x7c, 0xbf, 0x2b, 0xc6, 0x47, 0x98, 0x50, 0xd3, 0xf3, 0x1e, 0xc0, 0xbd, + 0x3d, 0x94, 0xaf, 0x58, 0x82, 0xaf, 0xd8, 0xf8, 0xf5, 0xce, 0x11, 0x4d, 0x53, 0x8c, 0x03, 0x3c, + 0x2e, 0x50, 0x48, 0xef, 0x43, 0x78, 0xb0, 0x87, 0x72, 0x28, 0xa9, 0x64, 0x42, 0xb2, 0xb1, 0xb8, + 0xa0, 0xbe, 0x03, 0x1f, 0xec, 0xa1, 0xf4, 0xa3, 0x0b, 0xe2, 0x6f, 0xa0, 0xf5, 0x22, 0x8b, 0xb0, + 0x9f, 0x1e, 0x66, 0xe4, 0x4b, 0x68, 0xd2, 0x28, 0xe2, 0x28, 0x84, 0xeb, 0x6c, 0x3a, 0x5b, 0x9d, + 0xed, 0x87, 0x8f, 0x17, 0xd6, 0x68, 0x57, 0xf6, 0xcc, 0x60, 0x82, 0x12, 0x4c, 0x08, 0xd4, 0x78, + 0x16, 0xa3, 0x5b, 0xd9, 0x74, 0xb6, 0xda, 0x81, 0x6e, 0x7b, 0xbf, 0x01, 0xe8, 0xa7, 0x4c, 0x1e, + 0x50, 0x4e, 0x13, 0x41, 0xee, 0x42, 0x23, 0x55, 0xb3, 0xf8, 0xda, 0x70, 0x35, 0xb0, 0x3d, 0xe2, + 0x43, 0x57, 0x48, 0xca, 0x65, 0x98, 0x6b, 0x9c, 0x5b, 0xd9, 0xac, 0x6e, 0x75, 0xb6, 0x3f, 0x5e, + 0x3a, 0xed, 0xd7, 0x78, 0xfe, 0x0d, 0x8d, 0x0b, 0x3c, 0xa0, 0x8c, 0x07, 0x1d, 0x3d, 0xcc, 0x58, + 0xf7, 0x7e, 0x05, 0x30, 0x94, 0x9c, 0xa5, 0x93, 0x01, 0x13, 0x52, 0xcd, 0x75, 0xa2, 0x70, 0x6a, + 0x13, 0xd5, 0xad, 0x76, 0x60, 0x7b, 0xe4, 0x73, 0x68, 0x08, 0x49, 0x65, 0x21, 0xf4, 0x3a, 0x3b, + 0xdb, 0x0f, 0x96, 0xce, 0x32, 0xd4, 0x90, 0xc0, 0x42, 0xbd, 0xbf, 0x56, 0x60, 0x63, 0xc1, 0xab, + 0xd6, 0x6f, 0xe4, 0x33, 0xa8, 0x8d, 0xa8, 0xc0, 0x2b, 0x1d, 0xb5, 0x2f, 0x26, 0xcf, 0xa9, 0xc0, + 0x40, 0x23, 0x95, 0x97, 0xa2, 0x51, 0xdf, 0xd7, 0xb3, 0x57, 0x03, 0xdd, 0x26, 0x1e, 0x74, 0xc7, + 0x59, 0x1c, 0xe3, 0x58, 0xb2, 0x2c, 0xed, 0xfb, 0x6e, 0x55, 0xeb, 0x16, 0x64, 0x0a, 0x93, 0x53, + 0x2e, 0x99, 0xe9, 0x0a, 0xb7, 0xb6, 0x59, 0x55, 0x98, 0x79, 0x19, 0xf9, 0x11, 0xf4, 0x24, 0xa7, + 0x27, 0x18, 0x87, 0x92, 0x25, 0x28, 0x24, 0x4d, 0x72, 0xb7, 0xbe, 0xe9, 0x6c, 0xd5, 0x82, 0x35, + 0x23, 0x7f, 0x55, 0x8a, 0xc9, 0x13, 0xf8, 0x60, 0x52, 0x50, 0x4e, 0x53, 0x89, 0x38, 0x87, 0x6e, + 0x68, 0x34, 0x99, 0xaa, 0x66, 0x03, 0x7e, 0x0c, 0xeb, 0x0a, 0x96, 0x15, 0x72, 0x0e, 0xde, 0xd4, + 0xf0, 0x9e, 0x55, 0x4c, 0xc1, 0xde, 0xdf, 0x1c, 0xb8, 0x73, 0xc1, 0x5f, 0x22, 0xcf, 0x52, 0x81, + 0xb7, 0x70, 0xd8, 0x6d, 0x02, 0x46, 0x9e, 0x42, 0x5d, 0xb5, 0x84, 0x5b, 0xbd, 0x6e, 0x2a, 0x19, + 0xbc, 0xf7, 0x67, 0x07, 0xc8, 0x0e, 0x47, 0x2a, 0xf1, 0x59, 0xcc, 0xe8, 0x7b, 0xc4, 0xf9, 0xbb, + 0xd0, 0x8c, 0x46, 0x61, 0x4a, 0x93, 0xf2, 0x40, 0x34, 0xa2, 0xd1, 0x0b, 0x9a, 0x20, 0xf9, 0x21, + 0xac, 0xcd, 0x02, 0x6b, 0x00, 0x55, 0x0d, 0x58, 0x9d, 0x89, 0x35, 0x70, 0x03, 0xea, 0x54, 0xad, + 0xc1, 0xad, 0x69, 0xb5, 0xe9, 0x78, 0x02, 0x7a, 0x3e, 0xcf, 0xf2, 0xff, 0xd4, 0xea, 0xa6, 0x93, + 0x56, 0xe7, 0x27, 0xfd, 0x93, 0x03, 0xeb, 0xcf, 0x62, 0x89, 0xfc, 0x5b, 0xea, 0x94, 0x7f, 0x54, + 0xca, 0xa8, 0xf5, 0xd3, 0x08, 0xcf, 0xfe, 0x97, 0x0b, 0xfc, 0x10, 0xe0, 0x90, 0x61, 0x1c, 0x19, + 0x8c, 0x59, 0x65, 0x5b, 0x4b, 0xb4, 0xba, 0x3c, 0xfe, 0xf5, 0x2b, 0x8e, 0x7f, 0x63, 0xc9, 0xf1, + 0x77, 0xa1, 0xa9, 0x8d, 0xf4, 0x7d, 0x7d, 0xe8, 0xaa, 0x41, 0xd9, 0x55, 0xe4, 0x89, 0x67, 0x92, + 0xd3, 0x92, 0x3c, 0x5b, 0xd7, 0x26, 0x4f, 0x3d, 0xcc, 0x92, 0xe7, 0xdb, 0x1a, 0xac, 0x0c, 0x91, + 0xf2, 0xf1, 0xd1, 0xed, 0x9d, 0xb7, 0x01, 0x75, 0x8e, 0xc7, 0x53, 0x6e, 0x33, 0x9d, 0xe9, 0x8e, + 0xab, 0x57, 0xec, 0xb8, 0x76, 0x0d, 0xc2, 0xab, 0x2f, 0x21, 0xbc, 0x1e, 0x54, 0x23, 0x11, 0x6b, + 0x87, 0xb5, 0x03, 0xd5, 0x54, 0x34, 0x95, 0xc7, 0x74, 0x8c, 0x47, 0x59, 0x1c, 0x21, 0x0f, 0x27, + 0x3c, 0x2b, 0x0c, 0x4d, 0x75, 0x83, 0xde, 0x9c, 0x62, 0x4f, 0xc9, 0xc9, 0x53, 0x68, 0x45, 0x22, + 0x0e, 0xe5, 0x79, 0x8e, 0x6e, 0x6b, 0xd3, 0xd9, 0x5a, 0x7d, 0xc7, 0x36, 0x7d, 0x11, 0xbf, 0x3a, + 0xcf, 0x31, 0x68, 0x46, 0xa6, 0x41, 0x3e, 0x83, 0x0d, 0x81, 0x9c, 0xd1, 0x98, 0xbd, 0xc1, 0x28, + 0xc4, 0xb3, 0x9c, 0x87, 0x79, 0x4c, 0x53, 0xb7, 0xad, 0x27, 0x22, 0x33, 0xdd, 0xee, 0x59, 0xce, + 0x0f, 0x62, 0x9a, 0x92, 0x2d, 0xe8, 0x65, 0x85, 0xcc, 0x0b, 0x19, 0xea, 0xb8, 0x89, 0x90, 0x45, + 0x2e, 0xe8, 0x1d, 0xad, 0x1a, 0xf9, 0x57, 0x5a, 0xdc, 0x8f, 0x96, 0x92, 0x78, 0xe7, 0x46, 0x24, + 0xde, 0xbd, 0x19, 0x89, 0xaf, 0x2c, 0x27, 0x71, 0xb2, 0x0a, 0x95, 0xf4, 0xd8, 0x5d, 0xd5, 0xa1, + 0xa9, 0xa4, 0xc7, 0x2a, 0x90, 0x32, 0xcb, 0x5f, 0xbb, 0x6b, 0x26, 0x90, 0xaa, 0x4d, 0x3e, 0x02, + 0x48, 0x50, 0x72, 0x36, 0x56, 0x6e, 0x71, 0x7b, 0x3a, 0x0e, 0x73, 0x12, 0xf2, 0x7d, 0x58, 0x61, + 0x93, 0x34, 0xe3, 0xb8, 0xc7, 0xb3, 0x53, 0x96, 0x4e, 0xdc, 0xf5, 0x4d, 0x67, 0xab, 0x15, 0x2c, + 0x0a, 0xbd, 0x7f, 0xce, 0x25, 0x9f, 0x28, 0x62, 0x29, 0xfe, 0x5b, 0xd7, 0xc4, 0x34, 0x63, 0xab, + 0xf3, 0x19, 0xfb, 0x08, 0x3a, 0x66, 0x0b, 0x26, 0x33, 0x6a, 0x97, 0x76, 0xf5, 0x08, 0x3a, 0x69, + 0x91, 0x84, 0xc7, 0x05, 0x72, 0x86, 0xc2, 0x9e, 0x65, 0x48, 0x8b, 0xe4, 0xa5, 0x91, 0x90, 0x0f, + 0xa0, 0x2e, 0xb3, 0x3c, 0x7c, 0x6d, 0x8f, 0xb2, 0xf2, 0xd5, 0xd7, 0xe4, 0x67, 0x70, 0x5f, 0x20, + 0x8d, 0x31, 0x0a, 0x05, 0x4e, 0x12, 0x4c, 0x65, 0xdf, 0x17, 0xa1, 0xd0, 0xdb, 0xc6, 0xc8, 0x6d, + 0xea, 0x64, 0x70, 0x0d, 0x62, 0x38, 0x05, 0x0c, 0xad, 0x5e, 0xc5, 0x7a, 0x6c, 0x6a, 0xb6, 0x85, + 0x61, 0x2d, 0x5d, 0xdc, 0x90, 0x99, 0x6a, 0x3a, 0xe0, 0xa7, 0xe0, 0x4e, 0xe2, 0x6c, 0x44, 0xe3, + 0xf0, 0xd2, 0xac, 0x6e, 0x5b, 0x4f, 0x76, 0xd7, 0xe8, 0x87, 0x17, 0xa6, 0x54, 0xdb, 0x13, 0x31, + 0x1b, 0x63, 0x14, 0x8e, 0xe2, 0x6c, 0xe4, 0x82, 0x4e, 0x6a, 0x30, 0xa2, 0xe7, 0x71, 0x36, 0x52, + 0xc9, 0x6c, 0x01, 0xca, 0x0d, 0xe3, 0xac, 0x48, 0xa5, 0x4e, 0xd1, 0x6a, 0xb0, 0x6a, 0xe4, 0x2f, + 0x8a, 0x64, 0x47, 0x49, 0xc9, 0xf7, 0x60, 0xc5, 0x22, 0xb3, 0xc3, 0x43, 0x81, 0x52, 0xe7, 0x66, + 0x35, 0xe8, 0x1a, 0xe1, 0x2f, 0xb5, 0x8c, 0x1c, 0x28, 0x6e, 0x15, 0xf2, 0xd9, 0x64, 0xc2, 0x71, + 0x42, 0xd5, 0xd9, 0xd6, 0x39, 0xd9, 0xd9, 0xfe, 0xc1, 0xe3, 0xa5, 0xc5, 0xf1, 0xe3, 0x9d, 0x45, + 0x74, 0x70, 0x71, 0xb8, 0x77, 0x0c, 0x6b, 0x17, 0x30, 0x8a, 0x4e, 0xb8, 0x2d, 0x42, 0x54, 0x8a, + 0xdb, 0x0a, 0x74, 0x41, 0x46, 0x36, 0xa1, 0x23, 0x90, 0x9f, 0xb0, 0xb1, 0x81, 0x18, 0x1a, 0x9b, + 0x17, 0x29, 0x1a, 0x96, 0x99, 0xa4, 0xf1, 0x8b, 0x97, 0x36, 0x65, 0xca, 0xae, 0xf7, 0xf7, 0x1a, + 0xac, 0x05, 0x2a, 0x45, 0xf0, 0x04, 0xff, 0x9f, 0x28, 0xf4, 0x5d, 0x54, 0xd6, 0xb8, 0x11, 0x95, + 0x35, 0xaf, 0x4d, 0x65, 0xad, 0x1b, 0x51, 0x59, 0xfb, 0x66, 0x54, 0x06, 0xef, 0xa0, 0xb2, 0x0d, + 0xa8, 0xc7, 0x2c, 0x61, 0x65, 0x96, 0x9a, 0xce, 0x65, 0x72, 0xea, 0x2e, 0x21, 0x27, 0x72, 0x0f, + 0x5a, 0x4c, 0xd8, 0x24, 0x5f, 0xd1, 0x80, 0x26, 0x13, 0x26, 0xbb, 0x77, 0xe1, 0x11, 0x93, 0xc8, + 0x75, 0x82, 0x85, 0x78, 0x26, 0x31, 0x15, 0xaa, 0xc5, 0x31, 0x2a, 0xc6, 0x18, 0x72, 0x2a, 0xd1, + 0xd2, 0xe7, 0xc3, 0x29, 0x6c, 0xb7, 0x44, 0x05, 0x1a, 0x14, 0x50, 0x89, 0xde, 0xdb, 0xea, 0x7c, + 0xea, 0x7c, 0x0b, 0x08, 0xf0, 0x13, 0xa8, 0xb2, 0xc8, 0x94, 0x58, 0x9d, 0x6d, 0x77, 0xd1, 0x8e, + 0x7d, 0x89, 0xf6, 0x7d, 0x11, 0x28, 0x10, 0xf9, 0x05, 0x74, 0x6c, 0x1a, 0x44, 0x54, 0x52, 0x9d, + 0x62, 0x9d, 0xed, 0x8f, 0x96, 0x8e, 0xd1, 0x79, 0xe1, 0x53, 0x49, 0x03, 0x53, 0x22, 0x09, 0xd5, + 0x26, 0x3f, 0x87, 0x07, 0x97, 0x69, 0x91, 0x5b, 0x77, 0x44, 0x6e, 0x43, 0x67, 0xd6, 0xbd, 0x8b, + 0xbc, 0x58, 0xfa, 0x2b, 0x22, 0x3f, 0x81, 0x8d, 0x39, 0x62, 0x9c, 0x0d, 0x6c, 0x6a, 0x66, 0x9c, + 0x23, 0xcd, 0xd9, 0x90, 0xab, 0xa8, 0xb1, 0x75, 0x25, 0x35, 0xfe, 0xfb, 0xa9, 0xea, 0xad, 0x03, + 0xed, 0x41, 0x46, 0x23, 0x5d, 0xb8, 0xde, 0x22, 0xec, 0x0f, 0xa1, 0x3d, 0x5d, 0xbd, 0x65, 0x8d, + 0x99, 0x40, 0x69, 0xa7, 0xb5, 0xa7, 0x2d, 0x58, 0xe7, 0x8a, 0xd1, 0xb9, 0xa2, 0xb2, 0xb6, 0x58, + 0x54, 0x3e, 0x82, 0x0e, 0x53, 0x0b, 0x0a, 0x73, 0x2a, 0x8f, 0x0c, 0x71, 0xb4, 0x03, 0xd0, 0xa2, + 0x03, 0x25, 0x51, 0x55, 0x67, 0x09, 0xd0, 0x55, 0x67, 0xe3, 0xda, 0x55, 0xa7, 0x35, 0xa2, 0xab, + 0xce, 0xdf, 0x3a, 0x00, 0x7a, 0xe3, 0x2a, 0x2d, 0x2f, 0x1b, 0x75, 0x6e, 0x63, 0x54, 0x31, 0x9a, + 0xba, 0x96, 0x38, 0xc6, 0x54, 0xce, 0x62, 0x2b, 0xac, 0x73, 0x48, 0x5a, 0x24, 0x81, 0x51, 0xd9, + 0xb8, 0x0a, 0xef, 0xf7, 0x0e, 0x80, 0x4e, 0x4e, 0xb3, 0x8c, 0x8b, 0xd4, 0xea, 0x5c, 0x5d, 0x8f, + 0x57, 0x16, 0x5d, 0xf7, 0xbc, 0x74, 0xdd, 0x15, 0x0f, 0xd0, 0x69, 0x7a, 0xcc, 0x36, 0x6f, 0xbd, + 0xab, 0xdb, 0xde, 0x1f, 0x1c, 0xe8, 0xda, 0xd5, 0x99, 0x25, 0x2d, 0x44, 0xd9, 0xb9, 0x18, 0x65, + 0x5d, 0xb0, 0x24, 0x19, 0x3f, 0x0f, 0x05, 0x7b, 0x53, 0xde, 0x5b, 0x60, 0x44, 0x43, 0xf6, 0x06, + 0x15, 0x87, 0x69, 0x97, 0x64, 0xa7, 0xa2, 0xbc, 0xb7, 0x94, 0x1b, 0xb2, 0x53, 0xa1, 0x78, 0x94, + 0xe3, 0x18, 0x53, 0x19, 0x9f, 0x87, 0x49, 0x16, 0xb1, 0x43, 0x86, 0x91, 0xce, 0x86, 0x56, 0xd0, + 0x2b, 0x15, 0xfb, 0x56, 0xae, 0xde, 0xf5, 0xc4, 0xfe, 0x1c, 0x95, 0xdf, 0x4f, 0xfb, 0x62, 0x72, + 0x8b, 0xac, 0x55, 0x2e, 0x36, 0x76, 0x54, 0x22, 0x9a, 0x1f, 0x9f, 0x76, 0xb0, 0x20, 0x53, 0xb5, + 0xe5, 0x94, 0xd9, 0x8d, 0x1f, 0x6b, 0xc1, 0x9c, 0x44, 0xad, 0x3c, 0xc2, 0x43, 0x5a, 0xc4, 0xf3, + 0x37, 0x40, 0xcd, 0xdc, 0x00, 0x56, 0xb1, 0xf0, 0x23, 0xb1, 0xba, 0xc3, 0x31, 0xc2, 0x54, 0x32, + 0x1a, 0xeb, 0x7f, 0xae, 0xfb, 0xd0, 0x2a, 0x84, 0x0a, 0x83, 0xad, 0x06, 0xda, 0xc1, 0xb4, 0x4f, + 0x3e, 0x05, 0x82, 0xe9, 0x98, 0x9f, 0xe7, 0x2a, 0x83, 0x72, 0x2a, 0xc4, 0x69, 0xc6, 0x23, 0xfb, + 0x24, 0x5c, 0x9f, 0x6a, 0x0e, 0xac, 0x82, 0xdc, 0x85, 0x86, 0xc4, 0x94, 0xa6, 0xd2, 0x9e, 0x31, + 0xdb, 0xb3, 0x77, 0x87, 0x28, 0x72, 0xe4, 0xd6, 0xa7, 0x4d, 0x26, 0x86, 0xaa, 0xab, 0x1e, 0x94, + 0xe2, 0x88, 0x6e, 0x7f, 0xf1, 0xe5, 0xcc, 0x7c, 0xdd, 0x3c, 0x28, 0x8d, 0xb8, 0xb4, 0xed, 0xed, + 0xc2, 0xfa, 0x80, 0x09, 0x79, 0x90, 0xc5, 0x6c, 0x7c, 0x7e, 0xeb, 0xca, 0xc2, 0xfb, 0x9d, 0x03, + 0x64, 0xde, 0x8e, 0xfd, 0x8f, 0x99, 0xdd, 0x1a, 0xce, 0xf5, 0x6f, 0x8d, 0x8f, 0xa1, 0x9b, 0x6b, + 0x33, 0x21, 0x4b, 0x0f, 0xb3, 0x32, 0x7a, 0x1d, 0x23, 0x53, 0xbe, 0x15, 0xea, 0x19, 0xac, 0x9c, + 0x19, 0xf2, 0x2c, 0x46, 0x13, 0xbc, 0x76, 0xd0, 0x56, 0x92, 0x40, 0x09, 0xbc, 0x09, 0xdc, 0x1b, + 0x1e, 0x65, 0xa7, 0x3b, 0x59, 0x7a, 0xc8, 0x26, 0x85, 0xb9, 0x1a, 0xdf, 0xe3, 0x5f, 0xc1, 0x85, + 0x66, 0x4e, 0xa5, 0x3a, 0x53, 0x36, 0x46, 0x65, 0xd7, 0xfb, 0xa3, 0x03, 0xf7, 0x97, 0xcd, 0xf4, + 0x3e, 0xdb, 0xdf, 0x83, 0x95, 0xb1, 0x31, 0x67, 0xac, 0x5d, 0xff, 0xbf, 0x72, 0x71, 0x9c, 0xb7, + 0x0b, 0x35, 0x55, 0x00, 0x90, 0x27, 0x50, 0xe1, 0x52, 0xaf, 0x60, 0x75, 0xfb, 0xd1, 0x3b, 0x98, + 0x42, 0x01, 0xf5, 0x23, 0xb4, 0xc2, 0x25, 0xe9, 0x82, 0xc3, 0xf5, 0x4e, 0x9d, 0xc0, 0xe1, 0x9f, + 0xfc, 0xc5, 0x81, 0x56, 0xa9, 0x26, 0xeb, 0xb0, 0xe2, 0xfb, 0x83, 0x9d, 0x29, 0x57, 0xf5, 0xbe, + 0x43, 0x7a, 0xd0, 0xf5, 0xfd, 0xc1, 0x41, 0x59, 0xf5, 0xf5, 0x1c, 0xd2, 0x85, 0x96, 0xef, 0x0f, + 0x34, 0xf9, 0xf4, 0x2a, 0xb6, 0xf7, 0x55, 0x5c, 0x88, 0xa3, 0x5e, 0x75, 0x6a, 0x20, 0xc9, 0xa9, + 0x31, 0x50, 0x23, 0x2b, 0xd0, 0xf6, 0xf7, 0x07, 0xfd, 0x54, 0x20, 0x97, 0xbd, 0xba, 0xed, 0xfa, + 0x18, 0xa3, 0xc4, 0x5e, 0x83, 0xac, 0x41, 0xc7, 0xdf, 0x1f, 0x3c, 0x2f, 0xe2, 0xd7, 0xea, 0x1e, + 0xeb, 0x35, 0xb5, 0xfe, 0xe5, 0xc0, 0x3c, 0x44, 0x7a, 0x2d, 0x6d, 0xfe, 0xe5, 0x40, 0x3d, 0x8d, + 0xce, 0x7b, 0xed, 0xe7, 0x4f, 0x7f, 0xfd, 0xc5, 0x84, 0xc9, 0xa3, 0x62, 0xa4, 0x1c, 0xf4, 0xc4, + 0xec, 0xf5, 0x53, 0x96, 0xd9, 0xd6, 0x93, 0x72, 0xbf, 0x4f, 0xf4, 0xf6, 0xa7, 0xdd, 0x7c, 0x34, + 0x6a, 0x68, 0xc9, 0xe7, 0xff, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x70, 0x7b, 0x95, 0x84, 0x41, 0x17, + 0x00, 0x00, } diff --git a/internal/querynodev2/delegator/delegator.go b/internal/querynodev2/delegator/delegator.go index 63076d16f3..dc948990fc 100644 --- a/internal/querynodev2/delegator/delegator.go +++ b/internal/querynodev2/delegator/delegator.go @@ -264,7 +264,7 @@ func (sd *shardDelegator) Query(ctx context.Context, req *querypb.QueryRequest) } if !funcutil.SliceContain(req.GetDmlChannels(), sd.vchannelName) { - log.Warn("deletgator received query request not belongs to it", + log.Warn("delegator received query request not belongs to it", zap.Strings("reqChannels", req.GetDmlChannels()), ) return nil, fmt.Errorf("dml channel not match, delegator channel %s, search channels %v", sd.vchannelName, req.GetDmlChannels()) diff --git a/internal/querynodev2/segments/result.go b/internal/querynodev2/segments/result.go index f8716cc91f..8a4fa1450c 100644 --- a/internal/querynodev2/segments/result.go +++ b/internal/querynodev2/segments/result.go @@ -22,6 +22,7 @@ import ( "math" "github.com/golang/protobuf/proto" + "github.com/samber/lo" "go.uber.org/zap" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" @@ -66,6 +67,14 @@ func ReduceSearchResults(ctx context.Context, results []*internalpb.SearchResult return nil, err } + requestCosts := lo.FilterMap(results, func(result *internalpb.SearchResults, _ int) (*internalpb.CostAggregation, bool) { + if result.CostAggregation == nil { + return nil, false + } + return result.CostAggregation, true + }) + searchResults.CostAggregation = mergeRequestCost(requestCosts) + return searchResults, nil } @@ -274,6 +283,14 @@ func MergeInternalRetrieveResult(ctx context.Context, retrieveResults []*interna log.Debug("skip duplicated query result while reducing internal.RetrieveResults", zap.Int64("dupCount", skipDupCnt)) } + requestCosts := lo.FilterMap(retrieveResults, func(result *internalpb.RetrieveResults, _ int) (*internalpb.CostAggregation, bool) { + if result.CostAggregation == nil { + return nil, false + } + return result.CostAggregation, true + }) + ret.CostAggregation = mergeRequestCost(requestCosts) + return ret, nil } diff --git a/internal/querynodev2/segments/result_test.go b/internal/querynodev2/segments/result_test.go index 18f1427260..2747a23ea5 100644 --- a/internal/querynodev2/segments/result_test.go +++ b/internal/querynodev2/segments/result_test.go @@ -22,6 +22,7 @@ import ( "sort" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" @@ -574,6 +575,39 @@ func (suite *ResultSuite) TestSort() { suite.Equal([]byte{2, 3, 4, 5, 6, 7, 8, 9}, result.FieldsData[7].GetVectors().GetBinaryVector()) } +func TestResult_MergeRequestCost(t *testing.T) { + costs := []*internalpb.CostAggregation{ + { + ResponseTime: 11, + ServiceTime: 12, + TotalNQ: 13, + }, + + { + ResponseTime: 21, + ServiceTime: 22, + TotalNQ: 23, + }, + + { + ResponseTime: 31, + ServiceTime: 32, + TotalNQ: 33, + }, + + { + ResponseTime: 41, + ServiceTime: 42, + TotalNQ: 43, + }, + } + + channelCost := mergeRequestCost(costs) + assert.Equal(t, int64(41), channelCost.ResponseTime) + assert.Equal(t, int64(42), channelCost.ServiceTime) + assert.Equal(t, int64(43), channelCost.TotalNQ) +} + func TestResult(t *testing.T) { suite.Run(t, new(ResultSuite)) } diff --git a/internal/querynodev2/segments/utils.go b/internal/querynodev2/segments/utils.go index 43bdfe4b6f..33af9c4422 100644 --- a/internal/querynodev2/segments/utils.go +++ b/internal/querynodev2/segments/utils.go @@ -12,6 +12,7 @@ import ( "go.uber.org/zap" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" + "github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/pkg/common" "github.com/milvus-io/milvus/pkg/log" @@ -282,3 +283,16 @@ func fillFieldData(ctx context.Context, vcm storage.ChunkManager, dataPath strin return fmt.Errorf("invalid data type: %s", fieldData.Type.String()) } } + +// mergeRequestCost merge the costs of request, the cost may came from different worker in same channel +// or different channel in same collection, for now we just choose the part with the highest response time +func mergeRequestCost(requestCosts []*internalpb.CostAggregation) *internalpb.CostAggregation { + var result *internalpb.CostAggregation + for _, cost := range requestCosts { + if result == nil || result.ResponseTime < cost.ResponseTime { + result = cost + } + } + + return result +} diff --git a/internal/querynodev2/services.go b/internal/querynodev2/services.go index 8aea1e62db..df27a3a578 100644 --- a/internal/querynodev2/services.go +++ b/internal/querynodev2/services.go @@ -701,29 +701,29 @@ func (node *QueryNode) SearchSegments(ctx context.Context, req *querypb.SearchRe searchCtx, cancel := context.WithCancel(ctx) defer cancel() - tr := timerecord.NewTimeRecorder("searchChannel") - log.Debug("search channel...") + tr := timerecord.NewTimeRecorder("searchSegments") + log.Debug("search segments...") collection := node.manager.Collection.Get(req.Req.GetCollectionID()) if collection == nil { - log.Warn("failed to search channel", zap.Error(segments.ErrCollectionNotFound)) + log.Warn("failed to search segments", zap.Error(segments.ErrCollectionNotFound)) return nil, segments.WrapCollectionNotFound(req.GetReq().GetCollectionID()) } task := tasks.NewSearchTask(searchCtx, collection, node.manager, req) if !node.scheduler.Add(task) { err := merr.WrapErrTaskQueueFull() - log.Warn("failed to search channel", zap.Error(err)) + log.Warn("failed to search segments", zap.Error(err)) return nil, err } err := task.Wait() if err != nil { - log.Warn("failed to search channel", zap.Error(err)) + log.Warn("failed to search segments", zap.Error(err)) return nil, err } - tr.CtxElapse(ctx, fmt.Sprintf("search channel done, channel = %s, segmentIDs = %v", + tr.CtxElapse(ctx, fmt.Sprintf("search segments done, channel = %s, segmentIDs = %v", channel, req.GetSegmentIDs(), )) @@ -732,7 +732,13 @@ func (node *QueryNode) SearchSegments(ctx context.Context, req *querypb.SearchRe latency := tr.ElapseSpan() metrics.QueryNodeSQReqLatency.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.SearchLabel, metrics.FromLeader).Observe(float64(latency.Milliseconds())) metrics.QueryNodeSQCount.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.SearchLabel, metrics.SuccessLabel).Inc() - return task.Result(), nil + + result := task.Result() + if result.CostAggregation != nil { + // update channel's response time + result.CostAggregation.ResponseTime = int64(latency) + } + return result, nil } // Search performs replica search tasks. @@ -813,7 +819,6 @@ func (node *QueryNode) Search(ctx context.Context, req *querypb.SearchRequest) ( } runningGp.Go(func() error { - ret, err := node.searchChannel(runningCtx, req, ch) mu.Lock() defer mu.Unlock() @@ -835,24 +840,27 @@ func (node *QueryNode) Search(ctx context.Context, req *querypb.SearchRequest) ( } tr := timerecord.NewTimeRecorderWithTrace(ctx, "searchRequestReduce") - ret, err := segments.ReduceSearchResults(ctx, toReduceResults, req.Req.GetNq(), req.Req.GetTopk(), req.Req.GetMetricType()) + result, err := segments.ReduceSearchResults(ctx, toReduceResults, req.Req.GetNq(), req.Req.GetTopk(), req.Req.GetMetricType()) if err != nil { log.Warn("failed to reduce search results", zap.Error(err)) failRet.Status.ErrorCode = commonpb.ErrorCode_UnexpectedError failRet.Status.Reason = err.Error() return failRet, nil } - metrics.QueryNodeReduceLatency.WithLabelValues( - fmt.Sprint(paramtable.GetNodeID()), - metrics.SearchLabel). + metrics.QueryNodeReduceLatency.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.SearchLabel). Observe(float64(tr.ElapseSpan().Milliseconds())) - if !req.FromShardLeader { - collector.Rate.Add(metricsinfo.NQPerSecond, float64(req.GetReq().GetNq())) - collector.Rate.Add(metricsinfo.SearchThroughput, float64(proto.Size(req))) - metrics.QueryNodeExecuteCounter.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), metrics.SearchLabel).Add(float64(proto.Size(req))) + collector.Rate.Add(metricsinfo.NQPerSecond, float64(req.GetReq().GetNq())) + collector.Rate.Add(metricsinfo.SearchThroughput, float64(proto.Size(req))) + metrics.QueryNodeExecuteCounter.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), metrics.SearchLabel). + Add(float64(proto.Size(req))) + + if result.CostAggregation != nil { + // update channel's response time + currentTotalNQ := node.scheduler.GetWaitingTaskTotalNQ() + result.CostAggregation.TotalNQ = currentTotalNQ } - return ret, nil + return result, nil } // only used for delegator query segments from worker @@ -882,7 +890,7 @@ func (node *QueryNode) QuerySegments(ctx context.Context, req *querypb.QueryRequ } defer node.lifetime.Done() - log.Debug("start do query with channel", + log.Debug("start do query segments", zap.Bool("fromShardLeader", req.GetFromShardLeader()), zap.Int64s("segmentIDs", req.GetSegmentIDs()), ) @@ -890,7 +898,7 @@ func (node *QueryNode) QuerySegments(ctx context.Context, req *querypb.QueryRequ queryCtx, cancel := context.WithCancel(ctx) defer cancel() - tr := timerecord.NewTimeRecorder("queryChannel") + tr := timerecord.NewTimeRecorder("querySegments") results, err := node.querySegments(queryCtx, req) if err != nil { log.Warn("failed to query channel", zap.Error(err)) @@ -910,6 +918,11 @@ func (node *QueryNode) QuerySegments(ctx context.Context, req *querypb.QueryRequ latency := tr.ElapseSpan() metrics.QueryNodeSQReqLatency.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.QueryLabel, metrics.FromLeader).Observe(float64(latency.Milliseconds())) metrics.QueryNodeSQCount.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.QueryLabel, metrics.SuccessLabel).Inc() + results.CostAggregation = &internalpb.CostAggregation{ + ServiceTime: latency.Milliseconds(), + ResponseTime: latency.Milliseconds(), + TotalNQ: 0, + } return results, nil } diff --git a/internal/querynodev2/tasks/scheduler.go b/internal/querynodev2/tasks/scheduler.go index 6d99758d30..559b367b3d 100644 --- a/internal/querynodev2/tasks/scheduler.go +++ b/internal/querynodev2/tasks/scheduler.go @@ -9,6 +9,7 @@ import ( "github.com/milvus-io/milvus/pkg/util/conc" "github.com/milvus-io/milvus/pkg/util/metricsinfo" "github.com/milvus-io/milvus/pkg/util/paramtable" + "go.uber.org/atomic" ) const ( @@ -20,10 +21,8 @@ type Scheduler struct { mergingSearchTasks []*SearchTask mergedSearchTasks chan *SearchTask - queryProcessQueue chan *QueryTask - queryWaitQueue chan *QueryTask - - pool *conc.Pool[any] + pool *conc.Pool[any] + waitingTaskTotalNQ atomic.Int64 } func NewScheduler() *Scheduler { @@ -33,9 +32,9 @@ func NewScheduler() *Scheduler { searchWaitQueue: make(chan *SearchTask, maxWaitTaskNum), mergingSearchTasks: make([]*SearchTask, 0), mergedSearchTasks: make(chan *SearchTask), - // queryProcessQueue: make(chan), - pool: conc.NewPool[any](maxReadConcurrency, conc.WithPreAlloc(true)), + pool: conc.NewPool[any](maxReadConcurrency, conc.WithPreAlloc(true)), + waitingTaskTotalNQ: *atomic.NewInt64(0), } } @@ -46,6 +45,7 @@ func (s *Scheduler) Add(task Task) bool { select { case s.searchWaitQueue <- t: collector.Counter.Inc(metricsinfo.ReadyQueueType, 1) + s.waitingTaskTotalNQ.Add(t.nq) metrics.QueryNodeReadTaskUnsolveLen.WithLabelValues(fmt.Sprint(paramtable.GetNodeID())).Inc() default: return false @@ -165,6 +165,11 @@ func (s *Scheduler) process(t Task) { metrics.QueryNodeReadTaskConcurrency.WithLabelValues(fmt.Sprint(paramtable.GetNodeID())).Dec() return nil, err }) + + switch t := t.(type) { + case *SearchTask: + s.waitingTaskTotalNQ.Sub(t.nq) + } } // mergeTasks merge the given task with one of merged tasks, @@ -174,6 +179,7 @@ func (s *Scheduler) mergeTasks(t Task) { merged := false for _, task := range s.mergingSearchTasks { if task.Merge(t) { + s.waitingTaskTotalNQ.Sub(t.nq) merged = true break } @@ -183,3 +189,7 @@ func (s *Scheduler) mergeTasks(t Task) { } } } + +func (s *Scheduler) GetWaitingTaskTotalNQ() int64 { + return s.waitingTaskTotalNQ.Load() +} diff --git a/internal/querynodev2/tasks/task.go b/internal/querynodev2/tasks/task.go index f617e15e14..0d657eb849 100644 --- a/internal/querynodev2/tasks/task.go +++ b/internal/querynodev2/tasks/task.go @@ -63,8 +63,7 @@ func NewSearchTask(ctx context.Context, originTopks: []int64{req.GetReq().GetTopk()}, originNqs: []int64{req.GetReq().GetNq()}, notifier: make(chan error, 1), - - tr: timerecord.NewTimeRecorderWithTrace(ctx, "searchTask"), + tr: timerecord.NewTimeRecorderWithTrace(ctx, "searchTask"), } } @@ -86,6 +85,8 @@ func (t *SearchTask) Execute() error { zap.String("shard", t.req.GetDmlChannels()[0]), ) + executeRecord := timerecord.NewTimeRecorderWithTrace(t.ctx, "searchTaskExecute") + req := t.req t.combinePlaceHolderGroups() searchReq, err := segments.NewSearchRequest(t.collection, req, t.placeholderGroup) @@ -135,12 +136,15 @@ func (t *SearchTask) Execute() error { TopK: t.originTopks[i], SlicedOffset: 1, SlicedNumCount: 1, + CostAggregation: &internalpb.CostAggregation{ + ServiceTime: executeRecord.ElapseSpan().Milliseconds(), + }, } } return nil } - tr := timerecord.NewTimeRecorderWithTrace(t.ctx, "searchTaskReduce") + reduceRecord := timerecord.NewTimeRecorderWithTrace(t.ctx, "searchTaskReduce") blobs, err := segments.ReduceSearchResultsAndFillData( searchReq.Plan(), results, @@ -174,7 +178,7 @@ func (t *SearchTask) Execute() error { metrics.QueryNodeReduceLatency.WithLabelValues( fmt.Sprint(paramtable.GetNodeID()), metrics.SearchLabel). - Observe(float64(tr.ElapseSpan().Milliseconds())) + Observe(float64(reduceRecord.ElapseSpan().Milliseconds())) task.result = &internalpb.SearchResults{ Status: util.WrapStatus(commonpb.ErrorCode_Success, ""), @@ -184,6 +188,9 @@ func (t *SearchTask) Execute() error { SlicedBlob: bs, SlicedOffset: 1, SlicedNumCount: 1, + CostAggregation: &internalpb.CostAggregation{ + ServiceTime: executeRecord.ElapseSpan().Milliseconds(), + }, } } return nil