diff --git a/internal/proxy/accesslog/info/grpc_info.go b/internal/proxy/accesslog/info/grpc_info.go index c07d13fc1c..7e65ea0d23 100644 --- a/internal/proxy/accesslog/info/grpc_info.go +++ b/internal/proxy/accesslog/info/grpc_info.go @@ -34,6 +34,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/internal/proxy/connection" "github.com/milvus-io/milvus/pkg/v2/util/logutil" "github.com/milvus-io/milvus/pkg/v2/util/merr" @@ -368,3 +369,17 @@ func (i *GrpcAccessInfo) ClientRequestTime() string { func (i *GrpcAccessInfo) SetActualConsistencyLevel(acl commonpb.ConsistencyLevel) { i.actualConsistencyLevel = &acl } + +func (i *GrpcAccessInfo) TemplateValueLength() string { + templateValues, ok := requestutil.GetExprTemplateValues(i.req) + if !ok { + return NotAny + } + + // get length only + m := lo.MapValues(templateValues, func(tv *schemapb.TemplateValue, _ string) int { + return getLengthFromTemplateValue(tv) + }) + + return fmt.Sprint(m) +} diff --git a/internal/proxy/accesslog/info/grpc_info_test.go b/internal/proxy/accesslog/info/grpc_info_test.go index d559fe3ea9..f9e1404bde 100644 --- a/internal/proxy/accesslog/info/grpc_info_test.go +++ b/internal/proxy/accesslog/info/grpc_info_test.go @@ -34,6 +34,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/internal/proxy/connection" "github.com/milvus-io/milvus/pkg/v2/common" "github.com/milvus-io/milvus/pkg/v2/util" @@ -295,6 +296,30 @@ func (s *GrpcAccessInfoSuite) TestClientRequestTime() { s.NotEqual(Unknown, Get(s.info, "$client_request_time")[0]) } +func (s *GrpcAccessInfoSuite) TestTemplateValueLength() { + // params := []*commonpb.KeyValuePair{{Key: "test_key", Value: "test_value"}} + exprTemplValues := map[string]*schemapb.TemplateValue{ + "store_id": { + Val: &schemapb.TemplateValue_ArrayVal{ + ArrayVal: &schemapb.TemplateArrayValue{ + Data: &schemapb.TemplateArrayValue_LongData{ + LongData: &schemapb.LongArray{ + Data: []int64{0, 1}, + }, + }, + }, + }, + }, + } + + s.info.req = &milvuspb.SearchRequest{ + Dsl: "store_id in {store_id}", + ExprTemplateValues: exprTemplValues, + } + + s.Equal(`map[store_id:2]`, Get(s.info, "$template_value_length")[0]) +} + func TestGrpcAccssInfo(t *testing.T) { suite.Run(t, new(GrpcAccessInfoSuite)) } diff --git a/internal/proxy/accesslog/info/info.go b/internal/proxy/accesslog/info/info.go index 3bfc1b9d89..ac6c2f0ff9 100644 --- a/internal/proxy/accesslog/info/info.go +++ b/internal/proxy/accesslog/info/info.go @@ -20,6 +20,7 @@ import "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" const ( Unknown = "Unknown" + NotAny = "n/a" timeFormat = "2006/01/02 15:04:05.000 -07:00" ClientRequestIDKey = "client_request_id" ) @@ -28,32 +29,33 @@ type getMetricFunc func(i AccessInfo) string // supported metrics var MetricFuncMap = map[string]getMetricFunc{ - "$method_name": getMethodName, - "$method_status": getMethodStatus, - "$trace_id": getTraceID, - "$user_addr": getAddr, - "$user_name": getUserName, - "$response_size": getResponseSize, - "$error_code": getErrorCode, - "$error_msg": getErrorMsg, - "$error_type": getErrorType, - "$database_name": getDbName, - "$collection_name": getCollectionName, - "$partition_name": getPartitionName, - "$time_cost": getTimeCost, - "$time_now": getTimeNow, - "$time_start": getTimeStart, - "$time_end": getTimeEnd, - "$method_expr": getExpr, - "$output_fields": getOutputFields, - "$sdk_version": getSdkVersion, - "$cluster_prefix": getClusterPrefix, - "$consistency_level": getConsistencyLevel, - "$anns_field": getAnnsField, - "$nq": getNq, - "$search_params": getSearchParams, - "$query_params": getQueryParams, - "$client_request_time": getClientRequestTime, + "$method_name": getMethodName, + "$method_status": getMethodStatus, + "$trace_id": getTraceID, + "$user_addr": getAddr, + "$user_name": getUserName, + "$response_size": getResponseSize, + "$error_code": getErrorCode, + "$error_msg": getErrorMsg, + "$error_type": getErrorType, + "$database_name": getDbName, + "$collection_name": getCollectionName, + "$partition_name": getPartitionName, + "$time_cost": getTimeCost, + "$time_now": getTimeNow, + "$time_start": getTimeStart, + "$time_end": getTimeEnd, + "$method_expr": getExpr, + "$output_fields": getOutputFields, + "$sdk_version": getSdkVersion, + "$cluster_prefix": getClusterPrefix, + "$consistency_level": getConsistencyLevel, + "$anns_field": getAnnsField, + "$nq": getNq, + "$search_params": getSearchParams, + "$query_params": getQueryParams, + "$client_request_time": getClientRequestTime, + "$template_value_length": getTemplateValueLength, } type AccessInfo interface { @@ -82,6 +84,7 @@ type AccessInfo interface { SearchParams() string QueryParams() string ClientRequestTime() string + TemplateValueLength() string SetActualConsistencyLevel(commonpb.ConsistencyLevel) } @@ -201,3 +204,7 @@ func getQueryParams(i AccessInfo) string { func getClientRequestTime(i AccessInfo) string { return i.ClientRequestTime() } + +func getTemplateValueLength(i AccessInfo) string { + return i.TemplateValueLength() +} diff --git a/internal/proxy/accesslog/info/restful_info.go b/internal/proxy/accesslog/info/restful_info.go index ed44b70187..18ca58c7d0 100644 --- a/internal/proxy/accesslog/info/restful_info.go +++ b/internal/proxy/accesslog/info/restful_info.go @@ -27,6 +27,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/pkg/v2/util/requestutil" ) @@ -271,3 +272,17 @@ func (i *RestfulInfo) ClientRequestTime() string { func (i *RestfulInfo) SetActualConsistencyLevel(acl commonpb.ConsistencyLevel) { i.actualConsistencyLevel = &acl } + +func (i *RestfulInfo) TemplateValueLength() string { + templateValues, ok := requestutil.GetExprTemplateValues(i.req) + if !ok { + return NotAny + } + + // get length only + m := lo.MapValues(templateValues, func(tv *schemapb.TemplateValue, _ string) int { + return getLengthFromTemplateValue(tv) + }) + + return fmt.Sprint(m) +} diff --git a/internal/proxy/accesslog/info/restful_info_test.go b/internal/proxy/accesslog/info/restful_info_test.go index 014a33ed19..af525d8c56 100644 --- a/internal/proxy/accesslog/info/restful_info_test.go +++ b/internal/proxy/accesslog/info/restful_info_test.go @@ -27,6 +27,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/pkg/v2/util/merr" "github.com/milvus-io/milvus/pkg/v2/util/paramtable" ) @@ -253,6 +254,29 @@ func (s *RestfulAccessInfoSuite) TestQueryParams() { s.Equal(kvsToString(params), Get(s.info, "$query_params")[0]) } +func (s *RestfulAccessInfoSuite) TestTemplateValueLength() { + exprTemplValues := map[string]*schemapb.TemplateValue{ + "store_id": { + Val: &schemapb.TemplateValue_ArrayVal{ + ArrayVal: &schemapb.TemplateArrayValue{ + Data: &schemapb.TemplateArrayValue_LongData{ + LongData: &schemapb.LongArray{ + Data: []int64{0, 1}, + }, + }, + }, + }, + }, + } + + s.info.req = &milvuspb.SearchRequest{ + Dsl: "store_id in {store_id}", + ExprTemplateValues: exprTemplValues, + } + + s.Equal(`map[store_id:2]`, Get(s.info, "$template_value_length")[0]) +} + func TestRestfulAccessInfo(t *testing.T) { suite.Run(t, new(RestfulAccessInfoSuite)) } diff --git a/internal/proxy/accesslog/info/util.go b/internal/proxy/accesslog/info/util.go index c290a6d2ba..a985929ca0 100644 --- a/internal/proxy/accesslog/info/util.go +++ b/internal/proxy/accesslog/info/util.go @@ -27,6 +27,7 @@ import ( "google.golang.org/grpc/metadata" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/pkg/v2/util" "github.com/milvus-io/milvus/pkg/v2/util/crypto" "github.com/milvus-io/milvus/pkg/v2/util/funcutil" @@ -102,6 +103,29 @@ func getAnnsFieldFromKvs(kvs []*commonpb.KeyValuePair) string { return field } +func getLengthFromTemplateValue(tv *schemapb.TemplateValue) int { + arrayValues := tv.GetArrayVal() + // other single value data type + if arrayValues == nil { + return 1 + } + switch arrayValues.GetData().(type) { + case *schemapb.TemplateArrayValue_BoolData: + return len(arrayValues.GetBoolData().GetData()) + case *schemapb.TemplateArrayValue_LongData: + return len(arrayValues.GetLongData().GetData()) + case *schemapb.TemplateArrayValue_DoubleData: + return len(arrayValues.GetDoubleData().GetData()) + case *schemapb.TemplateArrayValue_StringData: + return len(arrayValues.GetStringData().GetData()) + case *schemapb.TemplateArrayValue_JsonData: + return len(arrayValues.GetJsonData().GetData()) + default: + // undefined + return -1 + } +} + func listToString(strs []string) string { result := "[" for i, str := range strs { diff --git a/internal/proxy/accesslog/info/util_test.go b/internal/proxy/accesslog/info/util_test.go index 10bd17990c..47f633c697 100644 --- a/internal/proxy/accesslog/info/util_test.go +++ b/internal/proxy/accesslog/info/util_test.go @@ -20,6 +20,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" ) func TestGetSdkTypeByUserAgent(t *testing.T) { @@ -45,3 +47,105 @@ func TestGetSdkTypeByUserAgent(t *testing.T) { _, ok = getSdkTypeByUserAgent([]string{"invalid_type"}) assert.False(t, ok) } + +func TestGetLengthFromTemplateValue(t *testing.T) { + t.Run("bool_array", func(t *testing.T) { + tv := &schemapb.TemplateValue{ + Val: &schemapb.TemplateValue_ArrayVal{ + ArrayVal: &schemapb.TemplateArrayValue{ + Data: &schemapb.TemplateArrayValue_BoolData{ + BoolData: &schemapb.BoolArray{ + Data: []bool{true, false}, + }, + }, + }, + }, + } + + assert.Equal(t, 2, getLengthFromTemplateValue(tv)) + }) + + t.Run("string_array", func(t *testing.T) { + tv := &schemapb.TemplateValue{ + Val: &schemapb.TemplateValue_ArrayVal{ + ArrayVal: &schemapb.TemplateArrayValue{ + Data: &schemapb.TemplateArrayValue_StringData{ + StringData: &schemapb.StringArray{ + Data: []string{"foo", "bar"}, + }, + }, + }, + }, + } + + assert.Equal(t, 2, getLengthFromTemplateValue(tv)) + }) + + t.Run("long_array", func(t *testing.T) { + tv := &schemapb.TemplateValue{ + Val: &schemapb.TemplateValue_ArrayVal{ + ArrayVal: &schemapb.TemplateArrayValue{ + Data: &schemapb.TemplateArrayValue_LongData{ + LongData: &schemapb.LongArray{ + Data: []int64{0, 1}, + }, + }, + }, + }, + } + + assert.Equal(t, 2, getLengthFromTemplateValue(tv)) + }) + + t.Run("double_array", func(t *testing.T) { + tv := &schemapb.TemplateValue{ + Val: &schemapb.TemplateValue_ArrayVal{ + ArrayVal: &schemapb.TemplateArrayValue{ + Data: &schemapb.TemplateArrayValue_DoubleData{ + DoubleData: &schemapb.DoubleArray{ + Data: []float64{0, 1}, + }, + }, + }, + }, + } + + assert.Equal(t, 2, getLengthFromTemplateValue(tv)) + }) + + t.Run("json_array", func(t *testing.T) { + tv := &schemapb.TemplateValue{ + Val: &schemapb.TemplateValue_ArrayVal{ + ArrayVal: &schemapb.TemplateArrayValue{ + Data: &schemapb.TemplateArrayValue_JsonData{ + JsonData: &schemapb.JSONArray{ + Data: [][]byte{[]byte("{}"), []byte("{}")}, + }, + }, + }, + }, + } + + assert.Equal(t, 2, getLengthFromTemplateValue(tv)) + }) + + t.Run("nil", func(t *testing.T) { + tv := &schemapb.TemplateValue{ + Val: &schemapb.TemplateValue_ArrayVal{ + ArrayVal: &schemapb.TemplateArrayValue{}, + }, + } + + assert.Equal(t, -1, getLengthFromTemplateValue(tv)) + }) + + t.Run("primitive", func(t *testing.T) { + tv := &schemapb.TemplateValue{ + Val: &schemapb.TemplateValue_StringVal{ + StringVal: "foo", + }, + } + + assert.Equal(t, 1, getLengthFromTemplateValue(tv)) + }) +} diff --git a/pkg/util/requestutil/getter.go b/pkg/util/requestutil/getter.go index 8bcec4a815..7fc293dcf1 100644 --- a/pkg/util/requestutil/getter.go +++ b/pkg/util/requestutil/getter.go @@ -18,7 +18,10 @@ package requestutil -import "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" +import ( + "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" +) type CollectionNameGetter interface { GetCollectionName() string @@ -168,6 +171,19 @@ func GetConsistencyLevelFromRequst(req interface{}) (commonpb.ConsistencyLevel, return getter.GetConsistencyLevel(), true } +// TemplateValuesGetter is the common interface for getting expr template values from milvus requests. +type TemplateValuesGetter interface { + GetExprTemplateValues() map[string]*schemapb.TemplateValue +} + +func GetExprTemplateValues(req any) (map[string]*schemapb.TemplateValue, bool) { + getter, ok := req.(TemplateValuesGetter) + if !ok { + return nil, false + } + return getter.GetExprTemplateValues(), true +} + var TraceLogBaseInfoFuncMap = map[string]func(interface{}) (any, bool){ "collection_name": GetCollectionNameFromRequest, "db_name": GetDbNameFromRequest,