From c8a78465b4427681500032d2525ae45f16720002 Mon Sep 17 00:00:00 2001
From: "yihao.dai" <yihao.dai@zilliz.com>
Date: Sun, 21 Jan 2024 14:44:59 +0800
Subject: [PATCH] enhance: Add a counter monitoring for the rate-limit requests
 (#30109)

Add a counter monitoring metric for the ratelimited rpc requests with
labels: proxy nodeID, rpc request type, and state.

issue: https://github.com/milvus-io/milvus/issues/30052

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
---
 internal/proxy/rate_limit_interceptor.go |  7 +++++++
 pkg/metrics/proxy_metrics.go             | 10 ++++++++++
 2 files changed, 17 insertions(+)

diff --git a/internal/proxy/rate_limit_interceptor.go b/internal/proxy/rate_limit_interceptor.go
index 218ccea094..ad79ba58f7 100644
--- a/internal/proxy/rate_limit_interceptor.go
+++ b/internal/proxy/rate_limit_interceptor.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"strconv"
 
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
@@ -27,7 +28,9 @@ import (
 	"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
 	"github.com/milvus-io/milvus/internal/proto/internalpb"
 	"github.com/milvus-io/milvus/internal/types"
+	"github.com/milvus-io/milvus/pkg/metrics"
 	"github.com/milvus-io/milvus/pkg/util/merr"
+	"github.com/milvus-io/milvus/pkg/util/paramtable"
 )
 
 // RateLimitInterceptor returns a new unary server interceptors that performs request rate limiting.
@@ -39,12 +42,16 @@ func RateLimitInterceptor(limiter types.Limiter) grpc.UnaryServerInterceptor {
 		}
 
 		err = limiter.Check(collectionID, rt, n)
+		nodeID := strconv.FormatInt(paramtable.GetNodeID(), 10)
+		metrics.ProxyRateLimitReqCount.WithLabelValues(nodeID, rt.String(), metrics.TotalLabel).Inc()
 		if err != nil {
+			metrics.ProxyRateLimitReqCount.WithLabelValues(nodeID, rt.String(), metrics.FailLabel).Inc()
 			rsp := getFailedResponse(req, err)
 			if rsp != nil {
 				return rsp, nil
 			}
 		}
+		metrics.ProxyRateLimitReqCount.WithLabelValues(nodeID, rt.String(), metrics.SuccessLabel).Inc()
 		return handler(ctx, req)
 	}
 }
diff --git a/pkg/metrics/proxy_metrics.go b/pkg/metrics/proxy_metrics.go
index 9e82173f9a..e2349adff9 100644
--- a/pkg/metrics/proxy_metrics.go
+++ b/pkg/metrics/proxy_metrics.go
@@ -297,6 +297,15 @@ var (
 		}, []string{
 			nodeIDLabelName,
 		})
+
+	// ProxyRateLimitReqCount integrates a counter monitoring metric for the rate-limit rpc requests.
+	ProxyRateLimitReqCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: milvusNamespace,
+			Subsystem: typeutil.ProxyRole,
+			Name:      "rate_limit_req_count",
+			Help:      "count of operation executed",
+		}, []string{nodeIDLabelName, msgTypeLabelName, statusLabelName})
 )
 
 // RegisterProxy registers Proxy metrics
@@ -341,6 +350,7 @@ func RegisterProxy(registry *prometheus.Registry) {
 
 	registry.MustRegister(ProxyWorkLoadScore)
 	registry.MustRegister(ProxyExecutingTotalNq)
+	registry.MustRegister(ProxyRateLimitReqCount)
 }
 
 func CleanupCollectionMetrics(nodeID int64, collection string) {