From 97757405de8fef34229c4e23f5242b90d776a762 Mon Sep 17 00:00:00 2001 From: "zhenshan.cao" Date: Fri, 29 Apr 2022 15:35:47 +0800 Subject: [PATCH] Fix bug: wrong metrics name and forget to register metrics (#16714) Signed-off-by: zhenshan.cao --- internal/datacoord/meta.go | 3 +++ internal/metrics/datacoord_metrics.go | 8 ++++++++ internal/metrics/rootcoord_metrics.go | 5 +++-- internal/proxy/impl.go | 10 ++++++++-- internal/querynode/historical.go | 6 +++--- 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/internal/datacoord/meta.go b/internal/datacoord/meta.go index d8a54cc983..22f98246c3 100644 --- a/internal/datacoord/meta.go +++ b/internal/datacoord/meta.go @@ -91,6 +91,7 @@ func (m *meta) reloadFromKV() error { } } metrics.DataCoordNumStoredRows.WithLabelValues().Set(float64(numStoredRows)) + metrics.DataCoordNumStoredRowsCounter.WithLabelValues().Add(float64(numStoredRows)) return nil } @@ -224,6 +225,7 @@ func (m *meta) SetState(segmentID UniqueID, state commonpb.SegmentState) error { metrics.DataCoordNumSegments.WithLabelValues(string(state)).Inc() if state == commonpb.SegmentState_Flushed { metrics.DataCoordNumStoredRows.WithLabelValues().Add(float64(curSegInfo.GetNumOfRows())) + metrics.DataCoordNumStoredRowsCounter.WithLabelValues().Add(float64(curSegInfo.GetNumOfRows())) } else if oldState == commonpb.SegmentState_Flushed { metrics.DataCoordNumStoredRows.WithLabelValues().Sub(float64(curSegInfo.GetNumOfRows())) } @@ -380,6 +382,7 @@ func (m *meta) UpdateFlushSegmentsInfo( metrics.DataCoordNumSegments.WithLabelValues(string(newSegmentState)).Inc() if newSegmentState == commonpb.SegmentState_Flushed { metrics.DataCoordNumStoredRows.WithLabelValues().Add(float64(clonedSegment.GetNumOfRows())) + metrics.DataCoordNumStoredRowsCounter.WithLabelValues().Add(float64(clonedSegment.GetNumOfRows())) } else if oldSegmentState == commonpb.SegmentState_Flushed { metrics.DataCoordNumStoredRows.WithLabelValues().Sub(float64(segment.GetNumOfRows())) } diff --git a/internal/metrics/datacoord_metrics.go b/internal/metrics/datacoord_metrics.go index 602baf277f..1e78969fae 100644 --- a/internal/metrics/datacoord_metrics.go +++ b/internal/metrics/datacoord_metrics.go @@ -68,6 +68,14 @@ var ( Help: "number of stored rows", }, []string{}) + DataCoordNumStoredRowsCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: milvusNamespace, + Subsystem: typeutil.DataCoordRole, + Name: "stored_rows_count", + Help: "count of all stored rows ever", + }, []string{}) + DataCoordSyncEpoch = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: milvusNamespace, diff --git a/internal/metrics/rootcoord_metrics.go b/internal/metrics/rootcoord_metrics.go index 198ad6c8fb..293bdc1dc8 100644 --- a/internal/metrics/rootcoord_metrics.go +++ b/internal/metrics/rootcoord_metrics.go @@ -114,7 +114,7 @@ var ( prometheus.GaugeOpts{ Namespace: milvusNamespace, Subsystem: typeutil.RootCoordRole, - Name: "msgstream_num", + Name: "msgstream_obj_num", Help: "number of message streams", }) @@ -136,7 +136,8 @@ func RegisterRootCoord(registry *prometheus.Registry) { registry.MustRegister(RootCoordInsertChannelTimeTick) registry.MustRegister(RootCoordSyncTimeTickLatency) - // for DDL latency + // for DDL + registry.MustRegister(RootCoordDDLReqCounter) registry.MustRegister(RootCoordDDLReqLatency) // for allocator diff --git a/internal/proxy/impl.go b/internal/proxy/impl.go index c672433e61..3546953646 100644 --- a/internal/proxy/impl.go +++ b/internal/proxy/impl.go @@ -2455,6 +2455,7 @@ func (node *Proxy) Search(ctx context.Context, request *milvuspb.SearchRequest) }, }, nil } + tr.Record("search request enqueue") log.Debug( rpcEnqueued(method), @@ -2500,6 +2501,9 @@ func (node *Proxy) Search(ctx context.Context, request *milvuspb.SearchRequest) }, nil } + span := tr.Record("wait search result") + metrics.ProxyWaitForSearchResultLatency.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.GetNodeID(), 10), + metrics.SearchLabel).Observe(float64(span.Milliseconds())) log.Debug( rpcDone(method), zap.String("traceID", traceID), @@ -2517,7 +2521,6 @@ func (node *Proxy) Search(ctx context.Context, request *milvuspb.SearchRequest) metrics.ProxyDQLFunctionCall.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.GetNodeID(), 10), method, metrics.SuccessLabel).Inc() - metrics.ProxySearchVectors.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.GetNodeID(), 10)).Add(float64(qt.result.GetResults().GetNumQueries())) searchDur := tr.ElapseSpan().Milliseconds() metrics.ProxySearchLatency.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.GetNodeID(), 10), @@ -2681,6 +2684,7 @@ func (node *Proxy) Query(ctx context.Context, request *milvuspb.QueryRequest) (* }, }, nil } + tr.Record("query request enqueue") log.Debug( rpcEnqueued(method), @@ -2716,7 +2720,9 @@ func (node *Proxy) Query(ctx context.Context, request *milvuspb.QueryRequest) (* }, }, nil } - + span := tr.Record("wait query result") + metrics.ProxyWaitForSearchResultLatency.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.GetNodeID(), 10), + metrics.QueryLabel).Observe(float64(span.Milliseconds())) log.Debug( rpcDone(method), zap.String("traceID", traceID), diff --git a/internal/querynode/historical.go b/internal/querynode/historical.go index e19d033465..22cbd603f4 100644 --- a/internal/querynode/historical.go +++ b/internal/querynode/historical.go @@ -252,9 +252,9 @@ func (h *historical) searchSegments(segIDs []UniqueID, searchReqs []*searchReque searchResult, err := seg.search(plan, searchReqs, []Timestamp{searchTs}) // update metrics - metrics.QueryNodeSQSegmentLatency.WithLabelValues(metrics.SearchLabel, - metrics.SealedSegmentLabel, - fmt.Sprint(Params.QueryNodeCfg.GetNodeID())).Observe(float64(tr.ElapseSpan().Milliseconds())) + metrics.QueryNodeSQSegmentLatency.WithLabelValues(fmt.Sprint(Params.QueryNodeCfg.GetNodeID()), + metrics.SearchLabel, + metrics.SealedSegmentLabel).Observe(float64(tr.ElapseSpan().Milliseconds())) // write back result into list lock.Lock()