enhance: [2.5] Add accesslog field for template value length info (#44723) (#44791)

Cherry-pick from master
pr: #44723 
Related to #36672

Add accesslog field displaying value length for search/query request may
help developers debug related issues

---------

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>

internal/proxy/accesslog/info/grpc_info.go

@@ -34,6 +34,7 @@ import (
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
 	"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
+	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
 	"github.com/milvus-io/milvus/internal/proxy/connection"
 	"github.com/milvus-io/milvus/pkg/v2/util/merr"
 	"github.com/milvus-io/milvus/pkg/v2/util/requestutil"
@@ -356,3 +357,17 @@ func (i *GrpcAccessInfo) QueryParams() string {
 func (i *GrpcAccessInfo) SetActualConsistencyLevel(acl commonpb.ConsistencyLevel) {
 	i.actualConsistencyLevel = &acl
 }
+
+func (i *GrpcAccessInfo) TemplateValueLength() string {
+	templateValues, ok := requestutil.GetExprTemplateValues(i.req)
+	if !ok {
+		return NotAny
+	}
+
+	// get length only
+	m := lo.MapValues(templateValues, func(tv *schemapb.TemplateValue, _ string) int {
+		return getLengthFromTemplateValue(tv)
+	})
+
+	return fmt.Sprint(m)
+}

internal/proxy/accesslog/info/grpc_info_test.go

@@ -32,6 +32,7 @@ import (
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
 	"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
+	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
 	"github.com/milvus-io/milvus/internal/proxy/connection"
 	"github.com/milvus-io/milvus/pkg/v2/util"
 	"github.com/milvus-io/milvus/pkg/v2/util/crypto"
@@ -281,6 +282,30 @@ func (s *GrpcAccessInfoSuite) TestQueryParams() {
 	s.Equal(kvsToString(params), Get(s.info, "$query_params")[0])
 }
 
+func (s *GrpcAccessInfoSuite) TestTemplateValueLength() {
+	// params := []*commonpb.KeyValuePair{{Key: "test_key", Value: "test_value"}}
+	exprTemplValues := map[string]*schemapb.TemplateValue{
+		"store_id": {
+			Val: &schemapb.TemplateValue_ArrayVal{
+				ArrayVal: &schemapb.TemplateArrayValue{
+					Data: &schemapb.TemplateArrayValue_LongData{
+						LongData: &schemapb.LongArray{
+							Data: []int64{0, 1},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	s.info.req = &milvuspb.SearchRequest{
+		Dsl:                "store_id in {store_id}",
+		ExprTemplateValues: exprTemplValues,
+	}
+
+	s.Equal(`map[store_id:2]`, Get(s.info, "$template_value_length")[0])
+}
+
 func TestGrpcAccssInfo(t *testing.T) {
 	suite.Run(t, new(GrpcAccessInfoSuite))
 }

internal/proxy/accesslog/info/info.go

@@ -20,6 +20,7 @@ import "github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
 
 const (
 	Unknown            = "Unknown"
+	NotAny             = "n/a"
 	timeFormat         = "2006/01/02 15:04:05.000 -07:00"
 	ClientRequestIDKey = "client_request_id"
 )
@@ -28,31 +29,32 @@ type getMetricFunc func(i AccessInfo) string
 
 // supported metrics
 var MetricFuncMap = map[string]getMetricFunc{
-	"$method_name":       getMethodName,
+	"$method_name":           getMethodName,
-	"$method_status":     getMethodStatus,
+	"$method_status":         getMethodStatus,
-	"$trace_id":          getTraceID,
+	"$trace_id":              getTraceID,
-	"$user_addr":         getAddr,
+	"$user_addr":             getAddr,
-	"$user_name":         getUserName,
+	"$user_name":             getUserName,
-	"$response_size":     getResponseSize,
+	"$response_size":         getResponseSize,
-	"$error_code":        getErrorCode,
+	"$error_code":            getErrorCode,
-	"$error_msg":         getErrorMsg,
+	"$error_msg":             getErrorMsg,
-	"$error_type":        getErrorType,
+	"$error_type":            getErrorType,
-	"$database_name":     getDbName,
+	"$database_name":         getDbName,
-	"$collection_name":   getCollectionName,
+	"$collection_name":       getCollectionName,
-	"$partition_name":    getPartitionName,
+	"$partition_name":        getPartitionName,
-	"$time_cost":         getTimeCost,
+	"$time_cost":             getTimeCost,
-	"$time_now":          getTimeNow,
+	"$time_now":              getTimeNow,
-	"$time_start":        getTimeStart,
+	"$time_start":            getTimeStart,
-	"$time_end":          getTimeEnd,
+	"$time_end":              getTimeEnd,
-	"$method_expr":       getExpr,
+	"$method_expr":           getExpr,
-	"$output_fields":     getOutputFields,
+	"$output_fields":         getOutputFields,
-	"$sdk_version":       getSdkVersion,
+	"$sdk_version":           getSdkVersion,
-	"$cluster_prefix":    getClusterPrefix,
+	"$cluster_prefix":        getClusterPrefix,
-	"$consistency_level": getConsistencyLevel,
+	"$consistency_level":     getConsistencyLevel,
-	"$anns_field":        getAnnsField,
+	"$anns_field":            getAnnsField,
-	"$nq":                getNq,
+	"$nq":                    getNq,
-	"$search_params":     getSearchParams,
+	"$search_params":         getSearchParams,
-	"$query_params":      getQueryParams,
+	"$query_params":          getQueryParams,
+	"$template_value_length": getTemplateValueLength,
 }
 
 type AccessInfo interface {
@@ -80,6 +82,7 @@ type AccessInfo interface {
 	NQ() string
 	SearchParams() string
 	QueryParams() string
+	TemplateValueLength() string
 	SetActualConsistencyLevel(commonpb.ConsistencyLevel)
 }
 
@@ -195,3 +198,7 @@ func getSearchParams(i AccessInfo) string {
 func getQueryParams(i AccessInfo) string {
 	return i.QueryParams()
 }
+
+func getTemplateValueLength(i AccessInfo) string {
+	return i.TemplateValueLength()
+}

internal/proxy/accesslog/info/restful_info.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
 	"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
+	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
 	"github.com/milvus-io/milvus/pkg/v2/util/requestutil"
 )
 
@@ -267,3 +268,17 @@ func (i *RestfulInfo) QueryParams() string {
 func (i *RestfulInfo) SetActualConsistencyLevel(acl commonpb.ConsistencyLevel) {
 	i.actualConsistencyLevel = &acl
 }
+
+func (i *RestfulInfo) TemplateValueLength() string {
+	templateValues, ok := requestutil.GetExprTemplateValues(i.req)
+	if !ok {
+		return NotAny
+	}
+
+	// get length only
+	m := lo.MapValues(templateValues, func(tv *schemapb.TemplateValue, _ string) int {
+		return getLengthFromTemplateValue(tv)
+	})
+
+	return fmt.Sprint(m)
+}

internal/proxy/accesslog/info/restful_info_test.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
 	"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
+	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
 	"github.com/milvus-io/milvus/pkg/v2/util/merr"
 	"github.com/milvus-io/milvus/pkg/v2/util/paramtable"
 )
@@ -253,6 +254,29 @@ func (s *RestfulAccessInfoSuite) TestQueryParams() {
 	s.Equal(kvsToString(params), Get(s.info, "$query_params")[0])
 }
 
+func (s *RestfulAccessInfoSuite) TestTemplateValueLength() {
+	exprTemplValues := map[string]*schemapb.TemplateValue{
+		"store_id": {
+			Val: &schemapb.TemplateValue_ArrayVal{
+				ArrayVal: &schemapb.TemplateArrayValue{
+					Data: &schemapb.TemplateArrayValue_LongData{
+						LongData: &schemapb.LongArray{
+							Data: []int64{0, 1},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	s.info.req = &milvuspb.SearchRequest{
+		Dsl:                "store_id in {store_id}",
+		ExprTemplateValues: exprTemplValues,
+	}
+
+	s.Equal(`map[store_id:2]`, Get(s.info, "$template_value_length")[0])
+}
+
 func TestRestfulAccessInfo(t *testing.T) {
 	suite.Run(t, new(RestfulAccessInfoSuite))
 }

internal/proxy/accesslog/info/util.go

@@ -26,6 +26,7 @@ import (
 	"google.golang.org/grpc/metadata"
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
+	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
 	"github.com/milvus-io/milvus/pkg/v2/util"
 	"github.com/milvus-io/milvus/pkg/v2/util/crypto"
 	"github.com/milvus-io/milvus/pkg/v2/util/funcutil"
@@ -101,6 +102,29 @@ func getAnnsFieldFromKvs(kvs []*commonpb.KeyValuePair) string {
 	return field
 }
 
+func getLengthFromTemplateValue(tv *schemapb.TemplateValue) int {
+	arrayValues := tv.GetArrayVal()
+	// other single value data type
+	if arrayValues == nil {
+		return 1
+	}
+	switch arrayValues.GetData().(type) {
+	case *schemapb.TemplateArrayValue_BoolData:
+		return len(arrayValues.GetBoolData().GetData())
+	case *schemapb.TemplateArrayValue_LongData:
+		return len(arrayValues.GetLongData().GetData())
+	case *schemapb.TemplateArrayValue_DoubleData:
+		return len(arrayValues.GetDoubleData().GetData())
+	case *schemapb.TemplateArrayValue_StringData:
+		return len(arrayValues.GetStringData().GetData())
+	case *schemapb.TemplateArrayValue_JsonData:
+		return len(arrayValues.GetJsonData().GetData())
+	default:
+		// undefined
+		return -1
+	}
+}
+
 func listToString(strs []string) string {
 	result := "["
 	for i, str := range strs {

internal/proxy/accesslog/info/util_test.go

@@ -20,6 +20,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
 )
 
 func TestGetSdkTypeByUserAgent(t *testing.T) {
@@ -45,3 +47,105 @@ func TestGetSdkTypeByUserAgent(t *testing.T) {
 	_, ok = getSdkTypeByUserAgent([]string{"invalid_type"})
 	assert.False(t, ok)
 }
+
+func TestGetLengthFromTemplateValue(t *testing.T) {
+	t.Run("bool_array", func(t *testing.T) {
+		tv := &schemapb.TemplateValue{
+			Val: &schemapb.TemplateValue_ArrayVal{
+				ArrayVal: &schemapb.TemplateArrayValue{
+					Data: &schemapb.TemplateArrayValue_BoolData{
+						BoolData: &schemapb.BoolArray{
+							Data: []bool{true, false},
+						},
+					},
+				},
+			},
+		}
+
+		assert.Equal(t, 2, getLengthFromTemplateValue(tv))
+	})
+
+	t.Run("string_array", func(t *testing.T) {
+		tv := &schemapb.TemplateValue{
+			Val: &schemapb.TemplateValue_ArrayVal{
+				ArrayVal: &schemapb.TemplateArrayValue{
+					Data: &schemapb.TemplateArrayValue_StringData{
+						StringData: &schemapb.StringArray{
+							Data: []string{"foo", "bar"},
+						},
+					},
+				},
+			},
+		}
+
+		assert.Equal(t, 2, getLengthFromTemplateValue(tv))
+	})
+
+	t.Run("long_array", func(t *testing.T) {
+		tv := &schemapb.TemplateValue{
+			Val: &schemapb.TemplateValue_ArrayVal{
+				ArrayVal: &schemapb.TemplateArrayValue{
+					Data: &schemapb.TemplateArrayValue_LongData{
+						LongData: &schemapb.LongArray{
+							Data: []int64{0, 1},
+						},
+					},
+				},
+			},
+		}
+
+		assert.Equal(t, 2, getLengthFromTemplateValue(tv))
+	})
+
+	t.Run("double_array", func(t *testing.T) {
+		tv := &schemapb.TemplateValue{
+			Val: &schemapb.TemplateValue_ArrayVal{
+				ArrayVal: &schemapb.TemplateArrayValue{
+					Data: &schemapb.TemplateArrayValue_DoubleData{
+						DoubleData: &schemapb.DoubleArray{
+							Data: []float64{0, 1},
+						},
+					},
+				},
+			},
+		}
+
+		assert.Equal(t, 2, getLengthFromTemplateValue(tv))
+	})
+
+	t.Run("json_array", func(t *testing.T) {
+		tv := &schemapb.TemplateValue{
+			Val: &schemapb.TemplateValue_ArrayVal{
+				ArrayVal: &schemapb.TemplateArrayValue{
+					Data: &schemapb.TemplateArrayValue_JsonData{
+						JsonData: &schemapb.JSONArray{
+							Data: [][]byte{[]byte("{}"), []byte("{}")},
+						},
+					},
+				},
+			},
+		}
+
+		assert.Equal(t, 2, getLengthFromTemplateValue(tv))
+	})
+
+	t.Run("nil", func(t *testing.T) {
+		tv := &schemapb.TemplateValue{
+			Val: &schemapb.TemplateValue_ArrayVal{
+				ArrayVal: &schemapb.TemplateArrayValue{},
+			},
+		}
+
+		assert.Equal(t, -1, getLengthFromTemplateValue(tv))
+	})
+
+	t.Run("primitive", func(t *testing.T) {
+		tv := &schemapb.TemplateValue{
+			Val: &schemapb.TemplateValue_StringVal{
+				StringVal: "foo",
+			},
+		}
+
+		assert.Equal(t, 1, getLengthFromTemplateValue(tv))
+	})
+}

pkg/util/requestutil/getter.go

@@ -18,7 +18,10 @@
 
 package requestutil
 
-import "github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
+import (
+	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
+	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
+)
 
 type CollectionNameGetter interface {
 	GetCollectionName() string
@@ -168,6 +171,19 @@ func GetConsistencyLevelFromRequst(req interface{}) (commonpb.ConsistencyLevel,
 	return getter.GetConsistencyLevel(), true
 }
 
+// TemplateValuesGetter is the common interface for getting expr template values from milvus requests.
+type TemplateValuesGetter interface {
+	GetExprTemplateValues() map[string]*schemapb.TemplateValue
+}
+
+func GetExprTemplateValues(req any) (map[string]*schemapb.TemplateValue, bool) {
+	getter, ok := req.(TemplateValuesGetter)
+	if !ok {
+		return nil, false
+	}
+	return getter.GetExprTemplateValues(), true
+}
+
 var TraceLogBaseInfoFuncMap = map[string]func(interface{}) (any, bool){
 	"collection_name": GetCollectionNameFromRequest,
 	"db_name":         GetDbNameFromRequest,