admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-08 01:58:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: Avoid convert body byte slice to string in httpserver (#40405)

Browse Source 
The convertion of byte slice to string may copy the underline data which
may cause extra memory and cpu time for httpserver

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
This commit is contained in:
congqixia 2025-03-06 16:28:02 +08:00 committed by GitHub
parent 5c5273f95e
commit 7fbeb5624e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 27 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/distributed/proxy/httpserver/handler_v1.go
View File

@ -755,7 +755,7 @@ func (h *HandlersV1) insert(c *gin.Context) {
return nil, RestRequestInterceptorErr return nil, RestRequestInterceptorErr
} }
body, _ := c.Get(gin.BodyBytesKey) body, _ := c.Get(gin.BodyBytesKey)
err, httpReq.Data, _ = checkAndSetData(string(body.([]byte)), collSchema) err, httpReq.Data, _ = checkAndSetData(body.([]byte), collSchema)
if err != nil { if err != nil {
log.Warn("high level restful api, fail to deal with insert data", zap.Any("body", body), zap.Error(err)) log.Warn("high level restful api, fail to deal with insert data", zap.Any("body", body), zap.Error(err))
HTTPAbortReturn(c, http.StatusOK, gin.H{ HTTPAbortReturn(c, http.StatusOK, gin.H{
@ -861,7 +861,7 @@ func (h *HandlersV1) upsert(c *gin.Context) {
} }
} }
body, _ := c.Get(gin.BodyBytesKey) body, _ := c.Get(gin.BodyBytesKey)
err, httpReq.Data, _ = checkAndSetData(string(body.([]byte)), collSchema) err, httpReq.Data, _ = checkAndSetData(body.([]byte), collSchema)
if err != nil { if err != nil {
log.Warn("high level restful api, fail to deal with upsert data", zap.Any("body", body), zap.Error(err)) log.Warn("high level restful api, fail to deal with upsert data", zap.Any("body", body), zap.Error(err))
HTTPAbortReturn(c, http.StatusOK, gin.H{ HTTPAbortReturn(c, http.StatusOK, gin.H{

4
internal/distributed/proxy/httpserver/handler_v2.go
View File

@ -922,7 +922,7 @@ func (h *HandlersV2) insert(ctx context.Context, c *gin.Context, anyReq any, dbN
} }
body, _ := c.Get(gin.BodyBytesKey) body, _ := c.Get(gin.BodyBytesKey)
var validDataMap map[string][]bool var validDataMap map[string][]bool
err, httpReq.Data, validDataMap = checkAndSetData(string(body.([]byte)), collSchema) err, httpReq.Data, validDataMap = checkAndSetData(body.([]byte), collSchema)
if err != nil { if err != nil {
log.Ctx(ctx).Warn("high level restful api, fail to deal with insert data", zap.Error(err), zap.String("body", string(body.([]byte)))) log.Ctx(ctx).Warn("high level restful api, fail to deal with insert data", zap.Error(err), zap.String("body", string(body.([]byte))))
HTTPAbortReturn(c, http.StatusOK, gin.H{ HTTPAbortReturn(c, http.StatusOK, gin.H{
@ -996,7 +996,7 @@ func (h *HandlersV2) upsert(ctx context.Context, c *gin.Context, anyReq any, dbN
} }
body, _ := c.Get(gin.BodyBytesKey) body, _ := c.Get(gin.BodyBytesKey)
var validDataMap map[string][]bool var validDataMap map[string][]bool
err, httpReq.Data, validDataMap = checkAndSetData(string(body.([]byte)), collSchema) err, httpReq.Data, validDataMap = checkAndSetData(body.([]byte), collSchema)
if err != nil { if err != nil {
log.Ctx(ctx).Warn("high level restful api, fail to deal with upsert data", zap.Any("body", body), zap.Error(err)) log.Ctx(ctx).Warn("high level restful api, fail to deal with upsert data", zap.Any("body", body), zap.Error(err))
HTTPAbortReturn(c, http.StatusOK, gin.H{ HTTPAbortReturn(c, http.StatusOK, gin.H{

4
internal/distributed/proxy/httpserver/utils.go
View File

@ -283,10 +283,10 @@ func printIndexes(indexes []*milvuspb.IndexDescription) []gin.H {
// --------------------- insert param --------------------- // // --------------------- insert param --------------------- //
func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error, []map[string]interface{}, map[string][]bool) { func checkAndSetData(body []byte, collSchema *schemapb.CollectionSchema) (error, []map[string]interface{}, map[string][]bool) {
var reallyDataArray []map[string]interface{} var reallyDataArray []map[string]interface{}
validDataMap := make(map[string][]bool) validDataMap := make(map[string][]bool)
dataResult := gjson.Get(body, HTTPRequestData) dataResult := gjson.GetBytes(body, HTTPRequestData)
dataResultArray := dataResult.Array() dataResultArray := dataResult.Array()
if len(dataResultArray) == 0 { if len(dataResultArray) == 0 {
return merr.ErrMissingRequiredParameters, reallyDataArray, validDataMap return merr.ErrMissingRequiredParameters, reallyDataArray, validDataMap

42
internal/distributed/proxy/httpserver/utils_test.go
View File

@ -598,7 +598,7 @@ func TestPrimaryField(t *testing.T) {
func TestAnyToColumns(t *testing.T) { func TestAnyToColumns(t *testing.T) {
t.Run("insert with dynamic field", func(t *testing.T) { t.Run("insert with dynamic field", func(t *testing.T) {
body := "{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}" body := []byte("{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}")
req := InsertReq{} req := InsertReq{}
coll := generateCollectionSchema(schemapb.DataType_Int64, false, true) coll := generateCollectionSchema(schemapb.DataType_Int64, false, true)
var err error var err error
@ -615,7 +615,7 @@ func TestAnyToColumns(t *testing.T) {
}) })
t.Run("upsert with dynamic field", func(t *testing.T) { t.Run("upsert with dynamic field", func(t *testing.T) {
body := "{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}" body := []byte("{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}")
req := InsertReq{} req := InsertReq{}
coll := generateCollectionSchema(schemapb.DataType_Int64, false, true) coll := generateCollectionSchema(schemapb.DataType_Int64, false, true)
var err error var err error
@ -632,7 +632,7 @@ func TestAnyToColumns(t *testing.T) {
}) })
t.Run("insert with dynamic field, but pass pk when autoid==true", func(t *testing.T) { t.Run("insert with dynamic field, but pass pk when autoid==true", func(t *testing.T) {
body := "{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}" body := []byte("{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}")
req := InsertReq{} req := InsertReq{}
coll := generateCollectionSchema(schemapb.DataType_Int64, true, true) coll := generateCollectionSchema(schemapb.DataType_Int64, true, true)
var err error var err error
@ -647,7 +647,7 @@ func TestAnyToColumns(t *testing.T) {
}) })
t.Run("pass more field", func(t *testing.T) { t.Run("pass more field", func(t *testing.T) {
body := "{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}" body := []byte("{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}")
coll := generateCollectionSchema(schemapb.DataType_Int64, true, false) coll := generateCollectionSchema(schemapb.DataType_Int64, true, false)
var err error var err error
err, _, _ = checkAndSetData(body, coll) err, _, _ = checkAndSetData(body, coll)
@ -656,7 +656,7 @@ func TestAnyToColumns(t *testing.T) {
}) })
t.Run("insert with autoid==false", func(t *testing.T) { t.Run("insert with autoid==false", func(t *testing.T) {
body := "{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}" body := []byte("{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}")
req := InsertReq{} req := InsertReq{}
coll := generateCollectionSchema(schemapb.DataType_Int64, false, false) coll := generateCollectionSchema(schemapb.DataType_Int64, false, false)
var err error var err error
@ -672,7 +672,7 @@ func TestAnyToColumns(t *testing.T) {
}) })
t.Run("insert with autoid==false but has no pk", func(t *testing.T) { t.Run("insert with autoid==false but has no pk", func(t *testing.T) {
body := "{\"data\": { \"book_intro\": [0.1, 0.2], \"word_count\": 2}}" body := []byte("{\"data\": { \"book_intro\": [0.1, 0.2], \"word_count\": 2}}")
coll := generateCollectionSchema(schemapb.DataType_Int64, false, false) coll := generateCollectionSchema(schemapb.DataType_Int64, false, false)
var err error var err error
err, _, _ = checkAndSetData(body, coll) err, _, _ = checkAndSetData(body, coll)
@ -681,7 +681,7 @@ func TestAnyToColumns(t *testing.T) {
}) })
t.Run("insert with autoid==true", func(t *testing.T) { t.Run("insert with autoid==true", func(t *testing.T) {
body := "{\"data\": { \"book_intro\": [0.1, 0.2], \"word_count\": 2}}" body := []byte("{\"data\": { \"book_intro\": [0.1, 0.2], \"word_count\": 2}}")
req := InsertReq{} req := InsertReq{}
coll := generateCollectionSchema(schemapb.DataType_Int64, true, false) coll := generateCollectionSchema(schemapb.DataType_Int64, true, false)
var err error var err error
@ -696,7 +696,7 @@ func TestAnyToColumns(t *testing.T) {
}) })
t.Run("upsert with autoid==true", func(t *testing.T) { t.Run("upsert with autoid==true", func(t *testing.T) {
body := "{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}" body := []byte("{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}")
req := InsertReq{} req := InsertReq{}
coll := generateCollectionSchema(schemapb.DataType_Int64, true, false) coll := generateCollectionSchema(schemapb.DataType_Int64, true, false)
var err error var err error
@ -712,7 +712,7 @@ func TestAnyToColumns(t *testing.T) {
}) })
t.Run("upsert with autoid==false", func(t *testing.T) { t.Run("upsert with autoid==false", func(t *testing.T) {
body := "{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}" body := []byte("{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}")
req := InsertReq{} req := InsertReq{}
coll := generateCollectionSchema(schemapb.DataType_Int64, true, false) coll := generateCollectionSchema(schemapb.DataType_Int64, true, false)
var err error var err error
@ -730,7 +730,7 @@ func TestAnyToColumns(t *testing.T) {
func TestCheckAndSetData(t *testing.T) { func TestCheckAndSetData(t *testing.T) {
t.Run("invalid field name with dynamic field", func(t *testing.T) { t.Run("invalid field name with dynamic field", func(t *testing.T) {
body := "{\"data\": {\"id\": 0,\"$meta\": 2,\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}" body := []byte("{\"data\": {\"id\": 0,\"$meta\": 2,\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}")
coll := generateCollectionSchema(schemapb.DataType_Int64, false, true) coll := generateCollectionSchema(schemapb.DataType_Int64, false, true)
var err error var err error
err, _, _ = checkAndSetData(body, coll) err, _, _ = checkAndSetData(body, coll)
@ -738,7 +738,7 @@ func TestCheckAndSetData(t *testing.T) {
assert.Equal(t, true, strings.HasPrefix(err.Error(), "use the invalid field name")) assert.Equal(t, true, strings.HasPrefix(err.Error(), "use the invalid field name"))
}) })
t.Run("without vector", func(t *testing.T) { t.Run("without vector", func(t *testing.T) {
body := "{\"data\": {}}" body := []byte("{\"data\": {}}")
var err error var err error
primaryField := generatePrimaryField(schemapb.DataType_Int64, true) primaryField := generatePrimaryField(schemapb.DataType_Int64, true)
floatVectorField := generateVectorFieldSchema(schemapb.DataType_FloatVector) floatVectorField := generateVectorFieldSchema(schemapb.DataType_FloatVector)
@ -800,7 +800,7 @@ func TestCheckAndSetData(t *testing.T) {
t.Run("with pk when autoID == True when upsert", func(t *testing.T) { t.Run("with pk when autoID == True when upsert", func(t *testing.T) {
arrayFieldName := "array-int64" arrayFieldName := "array-int64"
body := "{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}" body := []byte("{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}")
coll := generateCollectionSchema(schemapb.DataType_Int64, true, false) coll := generateCollectionSchema(schemapb.DataType_Int64, true, false)
coll.Fields = append(coll.Fields, &schemapb.FieldSchema{ coll.Fields = append(coll.Fields, &schemapb.FieldSchema{
Name: arrayFieldName, Name: arrayFieldName,
@ -815,7 +815,7 @@ func TestCheckAndSetData(t *testing.T) {
t.Run("without pk when autoID == True when insert", func(t *testing.T) { t.Run("without pk when autoID == True when insert", func(t *testing.T) {
arrayFieldName := "array-int64" arrayFieldName := "array-int64"
body := "{\"data\": {\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}" body := []byte("{\"data\": {\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}")
coll := generateCollectionSchema(schemapb.DataType_Int64, true, false) coll := generateCollectionSchema(schemapb.DataType_Int64, true, false)
coll.Fields = append(coll.Fields, &schemapb.FieldSchema{ coll.Fields = append(coll.Fields, &schemapb.FieldSchema{
Name: arrayFieldName, Name: arrayFieldName,
@ -830,7 +830,7 @@ func TestCheckAndSetData(t *testing.T) {
t.Run("with pk when autoID == false", func(t *testing.T) { t.Run("with pk when autoID == false", func(t *testing.T) {
arrayFieldName := "array-int64" arrayFieldName := "array-int64"
body := "{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}" body := []byte("{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}")
coll := generateCollectionSchema(schemapb.DataType_Int64, false, false) coll := generateCollectionSchema(schemapb.DataType_Int64, false, false)
coll.Fields = append(coll.Fields, &schemapb.FieldSchema{ coll.Fields = append(coll.Fields, &schemapb.FieldSchema{
Name: arrayFieldName, Name: arrayFieldName,
@ -846,7 +846,7 @@ func TestCheckAndSetData(t *testing.T) {
func TestInsertWithInt64(t *testing.T) { func TestInsertWithInt64(t *testing.T) {
arrayFieldName := "array-int64" arrayFieldName := "array-int64"
body := "{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}" body := []byte("{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}")
coll := generateCollectionSchema(schemapb.DataType_Int64, false, true) coll := generateCollectionSchema(schemapb.DataType_Int64, false, true)
coll.Fields = append(coll.Fields, &schemapb.FieldSchema{ coll.Fields = append(coll.Fields, &schemapb.FieldSchema{
Name: arrayFieldName, Name: arrayFieldName,
@ -875,7 +875,7 @@ func TestInsertWithNullableField(t *testing.T) {
DataType: schemapb.DataType_Int64, DataType: schemapb.DataType_Int64,
Nullable: true, Nullable: true,
}) })
body := "{\"data\": [{\"book_id\": 9999999999999999, \"\nullable\": null,\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]},{\"book_id\": 1, \"nullable\": 1,\"book_intro\": [0.3, 0.4], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}]" body := []byte("{\"data\": [{\"book_id\": 9999999999999999, \"\nullable\": null,\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]},{\"book_id\": 1, \"nullable\": 1,\"book_intro\": [0.3, 0.4], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}]")
err, data, validData := checkAndSetData(body, coll) err, data, validData := checkAndSetData(body, coll)
assert.Equal(t, nil, err) assert.Equal(t, nil, err)
assert.Equal(t, 2, len(data)) assert.Equal(t, 2, len(data))
@ -911,7 +911,7 @@ func TestInsertWithDefaultValueField(t *testing.T) {
}, },
}, },
}) })
body := "{\"data\": [{\"book_id\": 9999999999999999, \"\fid\": null,\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]},{\"book_id\": 1, \"fid\": 1,\"book_intro\": [0.3, 0.4], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}]" body := []byte("{\"data\": [{\"book_id\": 9999999999999999, \"\fid\": null,\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]},{\"book_id\": 1, \"fid\": 1,\"book_intro\": [0.3, 0.4], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}]")
err, data, validData := checkAndSetData(body, coll) err, data, validData := checkAndSetData(body, coll)
assert.Equal(t, nil, err) assert.Equal(t, nil, err)
assert.Equal(t, 2, len(data)) assert.Equal(t, 2, len(data))
@ -2074,7 +2074,7 @@ func newRowsWithArray(results []map[string]interface{}) []map[string]interface{}
func TestArray(t *testing.T) { func TestArray(t *testing.T) {
body, _ := generateRequestBody(schemapb.DataType_Int64) body, _ := generateRequestBody(schemapb.DataType_Int64)
collectionSchema := generateCollectionSchema(schemapb.DataType_Int64, false, true) collectionSchema := generateCollectionSchema(schemapb.DataType_Int64, false, true)
err, rows, validRows := checkAndSetData(string(body), collectionSchema) err, rows, validRows := checkAndSetData(body, collectionSchema)
assert.Equal(t, nil, err) assert.Equal(t, nil, err)
assert.Equal(t, 0, len(validRows)) assert.Equal(t, 0, len(validRows))
assert.Equal(t, true, compareRows(rows, generateRawRows(schemapb.DataType_Int64), compareRow)) assert.Equal(t, true, compareRows(rows, generateRawRows(schemapb.DataType_Int64), compareRow))
@ -2084,7 +2084,7 @@ func TestArray(t *testing.T) {
body, _ = generateRequestBodyWithArray(schemapb.DataType_Int64) body, _ = generateRequestBodyWithArray(schemapb.DataType_Int64)
collectionSchema = newCollectionSchemaWithArray(generateCollectionSchema(schemapb.DataType_Int64, false, true)) collectionSchema = newCollectionSchemaWithArray(generateCollectionSchema(schemapb.DataType_Int64, false, true))
err, rows, validRows = checkAndSetData(string(body), collectionSchema) err, rows, validRows = checkAndSetData(body, collectionSchema)
assert.Equal(t, nil, err) assert.Equal(t, nil, err)
assert.Equal(t, 0, len(validRows)) assert.Equal(t, 0, len(validRows))
assert.Equal(t, true, compareRows(rows, newRowsWithArray(generateRawRows(schemapb.DataType_Int64)), compareRow)) assert.Equal(t, true, compareRows(rows, newRowsWithArray(generateRawRows(schemapb.DataType_Int64)), compareRow))
@ -2171,7 +2171,7 @@ func TestVector(t *testing.T) {
}, },
EnableDynamicField: true, EnableDynamicField: true,
} }
err, rows, validRows := checkAndSetData(string(body), collectionSchema) err, rows, validRows := checkAndSetData(body, collectionSchema)
assert.Equal(t, nil, err) assert.Equal(t, nil, err)
for i, row := range rows { for i, row := range rows {
assert.Equal(t, 2, len(row[floatVector].([]float32))) assert.Equal(t, 2, len(row[floatVector].([]float32)))
@ -2207,7 +2207,7 @@ func TestVector(t *testing.T) {
} }
row[field] = value row[field] = value
body, _ = wrapRequestBody([]map[string]interface{}{row}) body, _ = wrapRequestBody([]map[string]interface{}{row})
err, _, _ = checkAndSetData(string(body), collectionSchema) err, _, _ = checkAndSetData(body, collectionSchema)
assert.Error(t, err) assert.Error(t, err)
} }