From c12abf4e2a914f374d9739f52b59dab59a719ccc Mon Sep 17 00:00:00 2001 From: Buqian Zheng Date: Tue, 18 Mar 2025 17:20:16 +0800 Subject: [PATCH] enhance: improve sparse query nnz metric (#40713) add query type and field id label; add metric for hybrid search issue: https://github.com/milvus-io/milvus/issues/35853 Signed-off-by: Buqian Zheng --- internal/proxy/task_search.go | 7 ++++++- internal/querynodev2/delegator/delegator_data.go | 2 +- pkg/metrics/metrics.go | 1 + pkg/metrics/proxy_metrics.go | 2 +- pkg/metrics/querynode_metrics.go | 1 + pkg/util/typeutil/schema.go | 5 +++++ 6 files changed, 15 insertions(+), 3 deletions(-) diff --git a/internal/proxy/task_search.go b/internal/proxy/task_search.go index 7a09a3a8a8..64b378434a 100644 --- a/internal/proxy/task_search.go +++ b/internal/proxy/task_search.go @@ -418,6 +418,9 @@ func (t *searchTask) initAdvancedSearchRequest(ctx context.Context) error { if err != nil { return err } + if typeutil.IsFieldSparseFloatVector(t.schema.CollectionSchema, internalSubReq.FieldId) { + metrics.ProxySearchSparseNumNonZeros.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), t.collectionName, metrics.HybridSearchLabel, strconv.FormatInt(internalSubReq.FieldId, 10)).Observe(float64(typeutil.EstimateSparseVectorNNZFromPlaceholderGroup(internalSubReq.PlaceholderGroup, int(internalSubReq.GetNq())))) + } t.SearchRequest.SubReqs[index] = internalSubReq t.queryInfos[index] = queryInfo log.Debug("proxy init search request", @@ -510,7 +513,9 @@ func (t *searchTask) initSearchRequest(ctx context.Context) error { if err != nil { return err } - metrics.ProxySearchSparseNumNonZeros.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), t.collectionName).Observe(float64(typeutil.EstimateSparseVectorNNZFromPlaceholderGroup(t.request.PlaceholderGroup, int(t.request.GetNq())))) + if typeutil.IsFieldSparseFloatVector(t.schema.CollectionSchema, t.SearchRequest.FieldId) { + metrics.ProxySearchSparseNumNonZeros.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), t.collectionName, metrics.SearchLabel, strconv.FormatInt(t.SearchRequest.FieldId, 10)).Observe(float64(typeutil.EstimateSparseVectorNNZFromPlaceholderGroup(t.request.PlaceholderGroup, int(t.request.GetNq())))) + } t.SearchRequest.PlaceholderGroup = t.request.PlaceholderGroup t.SearchRequest.Topk = queryInfo.GetTopk() t.SearchRequest.MetricType = queryInfo.GetMetricType() diff --git a/internal/querynodev2/delegator/delegator_data.go b/internal/querynodev2/delegator/delegator_data.go index 66211b5f2d..2872a3a2d5 100644 --- a/internal/querynodev2/delegator/delegator_data.go +++ b/internal/querynodev2/delegator/delegator_data.go @@ -1063,7 +1063,7 @@ func (sd *shardDelegator) buildBM25IDF(req *internalpb.SearchRequest) (float64, } for _, idf := range idfSparseVector { - metrics.QueryNodeSearchFTSNumTokens.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), fmt.Sprint(sd.collectionID)).Observe(float64(typeutil.SparseFloatRowElementCount(idf))) + metrics.QueryNodeSearchFTSNumTokens.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), fmt.Sprint(sd.collectionID), fmt.Sprint(req.GetFieldId())).Observe(float64(typeutil.SparseFloatRowElementCount(idf))) } err = SetBM25Params(req, avgdl) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index ea0a66277f..b2f7ffc371 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -95,6 +95,7 @@ const ( indexTaskStatusLabelName = "index_task_status" msgTypeLabelName = "msg_type" collectionIDLabelName = "collection_id" + fieldIDLabelName = "field_id" channelNameLabelName = "channel_name" functionLabelName = "function_name" queryTypeLabelName = "query_type" diff --git a/pkg/metrics/proxy_metrics.go b/pkg/metrics/proxy_metrics.go index 4e05574694..8473273003 100644 --- a/pkg/metrics/proxy_metrics.go +++ b/pkg/metrics/proxy_metrics.go @@ -426,7 +426,7 @@ var ( Name: "search_sparse_num_non_zeros", Help: "the number of non-zeros in each sparse search task", Buckets: buckets, - }, []string{nodeIDLabelName, collectionName}) + }, []string{nodeIDLabelName, collectionName, queryTypeLabelName, fieldIDLabelName}) // ProxyQueueTaskNum records task number of queue in Proxy. ProxyQueueTaskNum = prometheus.NewGaugeVec( diff --git a/pkg/metrics/querynode_metrics.go b/pkg/metrics/querynode_metrics.go index b09bd138ce..9f011b5a54 100644 --- a/pkg/metrics/querynode_metrics.go +++ b/pkg/metrics/querynode_metrics.go @@ -360,6 +360,7 @@ var ( }, []string{ nodeIDLabelName, collectionIDLabelName, + fieldIDLabelName, }) QueryNodeSearchGroupSize = prometheus.NewHistogramVec( diff --git a/pkg/util/typeutil/schema.go b/pkg/util/typeutil/schema.go index 99bc8e80c8..75fb28f375 100644 --- a/pkg/util/typeutil/schema.go +++ b/pkg/util/typeutil/schema.go @@ -1239,6 +1239,11 @@ func GetPrimaryFieldSchema(schema *schemapb.CollectionSchema) (*schemapb.FieldSc return nil, errors.New("primary field is not found") } +func IsFieldSparseFloatVector(schema *schemapb.CollectionSchema, fieldID int64) bool { + fieldSchema := GetField(schema, fieldID) + return fieldSchema != nil && IsSparseFloatVectorType(fieldSchema.DataType) +} + // GetPartitionKeyFieldSchema get partition field schema from collection schema func GetPartitionKeyFieldSchema(schema *schemapb.CollectionSchema) (*schemapb.FieldSchema, error) { for _, fieldSchema := range schema.GetFields() {