admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-07 01:28:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: [2.5] Add accesslog field for template value length info (#44723) (#44791)

Browse Source 
Cherry-pick from master
pr: #44723 
Related to #36672

Add accesslog field displaying value length for search/query request may
help developers debug related issues

---------

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
This commit is contained in:
congqixia 2025-10-13 14:29:58 +08:00 committed by GitHub
parent 17a91f3bdf
commit c30cb6c283
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 256 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
internal/proxy/accesslog/info/grpc_info.go
View File

@ -34,6 +34,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/internal/proxy/connection"
"github.com/milvus-io/milvus/pkg/v2/util/merr"
"github.com/milvus-io/milvus/pkg/v2/util/requestutil"
@ -356,3 +357,17 @@ func (i *GrpcAccessInfo) QueryParams() string {
func (i *GrpcAccessInfo) SetActualConsistencyLevel(acl commonpb.ConsistencyLevel) {
i.actualConsistencyLevel = &acl
}
func (i *GrpcAccessInfo) TemplateValueLength() string {
templateValues, ok := requestutil.GetExprTemplateValues(i.req)
if !ok {
return NotAny
}
// get length only
m := lo.MapValues(templateValues, func(tv *schemapb.TemplateValue, _ string) int {
return getLengthFromTemplateValue(tv)
})
return fmt.Sprint(m)
}

25
internal/proxy/accesslog/info/grpc_info_test.go
View File

@ -32,6 +32,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/internal/proxy/connection"
"github.com/milvus-io/milvus/pkg/v2/util"
"github.com/milvus-io/milvus/pkg/v2/util/crypto"
@ -281,6 +282,30 @@ func (s *GrpcAccessInfoSuite) TestQueryParams() {
s.Equal(kvsToString(params), Get(s.info, "$query_params")[0])
}
func (s *GrpcAccessInfoSuite) TestTemplateValueLength() {
// params := []*commonpb.KeyValuePair{{Key: "test_key", Value: "test_value"}}
exprTemplValues := map[string]*schemapb.TemplateValue{
"store_id": {
Val: &schemapb.TemplateValue_ArrayVal{
ArrayVal: &schemapb.TemplateArrayValue{
Data: &schemapb.TemplateArrayValue_LongData{
LongData: &schemapb.LongArray{
Data: []int64{0, 1},
},
},
},
},
},
}
s.info.req = &milvuspb.SearchRequest{
Dsl: "store_id in {store_id}",
ExprTemplateValues: exprTemplValues,
}
s.Equal(`map[store_id:2]`, Get(s.info, "$template_value_length")[0])
}
func TestGrpcAccssInfo(t *testing.T) {
suite.Run(t, new(GrpcAccessInfoSuite))
}

57
internal/proxy/accesslog/info/info.go
View File

@ -20,6 +20,7 @@ import "github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
const (
Unknown = "Unknown"
NotAny = "n/a"
timeFormat = "2006/01/02 15:04:05.000 -07:00"
ClientRequestIDKey = "client_request_id"
)
@ -28,31 +29,32 @@ type getMetricFunc func(i AccessInfo) string
// supported metrics
var MetricFuncMap = map[string]getMetricFunc{
"$method_name": getMethodName,
"$method_status": getMethodStatus,
"$trace_id": getTraceID,
"$user_addr": getAddr,
"$user_name": getUserName,
"$response_size": getResponseSize,
"$error_code": getErrorCode,
"$error_msg": getErrorMsg,
"$error_type": getErrorType,
"$database_name": getDbName,
"$collection_name": getCollectionName,
"$partition_name": getPartitionName,
"$time_cost": getTimeCost,
"$time_now": getTimeNow,
"$time_start": getTimeStart,
"$time_end": getTimeEnd,
"$method_expr": getExpr,
"$output_fields": getOutputFields,
"$sdk_version": getSdkVersion,
"$cluster_prefix": getClusterPrefix,
"$consistency_level": getConsistencyLevel,
"$anns_field": getAnnsField,
"$nq": getNq,
"$search_params": getSearchParams,
"$query_params": getQueryParams,
"$method_name": getMethodName,
"$method_status": getMethodStatus,
"$trace_id": getTraceID,
"$user_addr": getAddr,
"$user_name": getUserName,
"$response_size": getResponseSize,
"$error_code": getErrorCode,
"$error_msg": getErrorMsg,
"$error_type": getErrorType,
"$database_name": getDbName,
"$collection_name": getCollectionName,
"$partition_name": getPartitionName,
"$time_cost": getTimeCost,
"$time_now": getTimeNow,
"$time_start": getTimeStart,
"$time_end": getTimeEnd,
"$method_expr": getExpr,
"$output_fields": getOutputFields,
"$sdk_version": getSdkVersion,
"$cluster_prefix": getClusterPrefix,
"$consistency_level": getConsistencyLevel,
"$anns_field": getAnnsField,
"$nq": getNq,
"$search_params": getSearchParams,
"$query_params": getQueryParams,
"$template_value_length": getTemplateValueLength,
}
type AccessInfo interface {
@ -80,6 +82,7 @@ type AccessInfo interface {
NQ() string
SearchParams() string
QueryParams() string
TemplateValueLength() string
SetActualConsistencyLevel(commonpb.ConsistencyLevel)
}
@ -195,3 +198,7 @@ func getSearchParams(i AccessInfo) string {
func getQueryParams(i AccessInfo) string {
return i.QueryParams()
}
func getTemplateValueLength(i AccessInfo) string {
return i.TemplateValueLength()
}

15
internal/proxy/accesslog/info/restful_info.go
View File

@ -27,6 +27,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/pkg/v2/util/requestutil"
)
@ -267,3 +268,17 @@ func (i *RestfulInfo) QueryParams() string {
func (i *RestfulInfo) SetActualConsistencyLevel(acl commonpb.ConsistencyLevel) {
i.actualConsistencyLevel = &acl
}
func (i *RestfulInfo) TemplateValueLength() string {
templateValues, ok := requestutil.GetExprTemplateValues(i.req)
if !ok {
return NotAny
}
// get length only
m := lo.MapValues(templateValues, func(tv *schemapb.TemplateValue, _ string) int {
return getLengthFromTemplateValue(tv)
})
return fmt.Sprint(m)
}

24
internal/proxy/accesslog/info/restful_info_test.go
View File

@ -27,6 +27,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/pkg/v2/util/merr"
"github.com/milvus-io/milvus/pkg/v2/util/paramtable"
)
@ -253,6 +254,29 @@ func (s *RestfulAccessInfoSuite) TestQueryParams() {
s.Equal(kvsToString(params), Get(s.info, "$query_params")[0])
}
func (s *RestfulAccessInfoSuite) TestTemplateValueLength() {
exprTemplValues := map[string]*schemapb.TemplateValue{
"store_id": {
Val: &schemapb.TemplateValue_ArrayVal{
ArrayVal: &schemapb.TemplateArrayValue{
Data: &schemapb.TemplateArrayValue_LongData{
LongData: &schemapb.LongArray{
Data: []int64{0, 1},
},
},
},
},
},
}
s.info.req = &milvuspb.SearchRequest{
Dsl: "store_id in {store_id}",
ExprTemplateValues: exprTemplValues,
}
s.Equal(`map[store_id:2]`, Get(s.info, "$template_value_length")[0])
}
func TestRestfulAccessInfo(t *testing.T) {
suite.Run(t, new(RestfulAccessInfoSuite))
}

24
internal/proxy/accesslog/info/util.go
View File

@ -26,6 +26,7 @@ import (
"google.golang.org/grpc/metadata"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/pkg/v2/util"
"github.com/milvus-io/milvus/pkg/v2/util/crypto"
"github.com/milvus-io/milvus/pkg/v2/util/funcutil"
@ -101,6 +102,29 @@ func getAnnsFieldFromKvs(kvs []*commonpb.KeyValuePair) string {
return field
}
func getLengthFromTemplateValue(tv *schemapb.TemplateValue) int {
arrayValues := tv.GetArrayVal()
// other single value data type
if arrayValues == nil {
return 1
}
switch arrayValues.GetData().(type) {
case *schemapb.TemplateArrayValue_BoolData:
return len(arrayValues.GetBoolData().GetData())
case *schemapb.TemplateArrayValue_LongData:
return len(arrayValues.GetLongData().GetData())
case *schemapb.TemplateArrayValue_DoubleData:
return len(arrayValues.GetDoubleData().GetData())
case *schemapb.TemplateArrayValue_StringData:
return len(arrayValues.GetStringData().GetData())
case *schemapb.TemplateArrayValue_JsonData:
return len(arrayValues.GetJsonData().GetData())
default:
// undefined
return -1
}
}
func listToString(strs []string) string {
result := "["
for i, str := range strs {

104
internal/proxy/accesslog/info/util_test.go
View File

@ -20,6 +20,8 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
)
func TestGetSdkTypeByUserAgent(t *testing.T) {
@ -45,3 +47,105 @@ func TestGetSdkTypeByUserAgent(t *testing.T) {
_, ok = getSdkTypeByUserAgent([]string{"invalid_type"})
assert.False(t, ok)
}
func TestGetLengthFromTemplateValue(t *testing.T) {
t.Run("bool_array", func(t *testing.T) {
tv := &schemapb.TemplateValue{
Val: &schemapb.TemplateValue_ArrayVal{
ArrayVal: &schemapb.TemplateArrayValue{
Data: &schemapb.TemplateArrayValue_BoolData{
BoolData: &schemapb.BoolArray{
Data: []bool{true, false},
},
},
},
},
}
assert.Equal(t, 2, getLengthFromTemplateValue(tv))
})
t.Run("string_array", func(t *testing.T) {
tv := &schemapb.TemplateValue{
Val: &schemapb.TemplateValue_ArrayVal{
ArrayVal: &schemapb.TemplateArrayValue{
Data: &schemapb.TemplateArrayValue_StringData{
StringData: &schemapb.StringArray{
Data: []string{"foo", "bar"},
},
},
},
},
}
assert.Equal(t, 2, getLengthFromTemplateValue(tv))
})
t.Run("long_array", func(t *testing.T) {
tv := &schemapb.TemplateValue{
Val: &schemapb.TemplateValue_ArrayVal{
ArrayVal: &schemapb.TemplateArrayValue{
Data: &schemapb.TemplateArrayValue_LongData{
LongData: &schemapb.LongArray{
Data: []int64{0, 1},
},
},
},
},
}
assert.Equal(t, 2, getLengthFromTemplateValue(tv))
})
t.Run("double_array", func(t *testing.T) {
tv := &schemapb.TemplateValue{
Val: &schemapb.TemplateValue_ArrayVal{
ArrayVal: &schemapb.TemplateArrayValue{
Data: &schemapb.TemplateArrayValue_DoubleData{
DoubleData: &schemapb.DoubleArray{
Data: []float64{0, 1},
},
},
},
},
}
assert.Equal(t, 2, getLengthFromTemplateValue(tv))
})
t.Run("json_array", func(t *testing.T) {
tv := &schemapb.TemplateValue{
Val: &schemapb.TemplateValue_ArrayVal{
ArrayVal: &schemapb.TemplateArrayValue{
Data: &schemapb.TemplateArrayValue_JsonData{
JsonData: &schemapb.JSONArray{
Data: [][]byte{[]byte("{}"), []byte("{}")},
},
},
},
},
}
assert.Equal(t, 2, getLengthFromTemplateValue(tv))
})
t.Run("nil", func(t *testing.T) {
tv := &schemapb.TemplateValue{
Val: &schemapb.TemplateValue_ArrayVal{
ArrayVal: &schemapb.TemplateArrayValue{},
},
}
assert.Equal(t, -1, getLengthFromTemplateValue(tv))
})
t.Run("primitive", func(t *testing.T) {
tv := &schemapb.TemplateValue{
Val: &schemapb.TemplateValue_StringVal{
StringVal: "foo",
},
}
assert.Equal(t, 1, getLengthFromTemplateValue(tv))
})
}

18
pkg/util/requestutil/getter.go
View File

@ -18,7 +18,10 @@
package requestutil
import "github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
import (
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
)
type CollectionNameGetter interface {
GetCollectionName() string
@ -168,6 +171,19 @@ func GetConsistencyLevelFromRequst(req interface{}) (commonpb.ConsistencyLevel,
return getter.GetConsistencyLevel(), true
}
// TemplateValuesGetter is the common interface for getting expr template values from milvus requests.
type TemplateValuesGetter interface {
GetExprTemplateValues() map[string]*schemapb.TemplateValue
}
func GetExprTemplateValues(req any) (map[string]*schemapb.TemplateValue, bool) {
getter, ok := req.(TemplateValuesGetter)
if !ok {
return nil, false
}
return getter.GetExprTemplateValues(), true
}
var TraceLogBaseInfoFuncMap = map[string]func(interface{}) (any, bool){
"collection_name": GetCollectionNameFromRequest,
"db_name": GetDbNameFromRequest,