admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: Avoid convert body byte slice to string in httpserver (#40405)

Browse Source 
The convertion of byte slice to string may copy the underline data which
may cause extra memory and cpu time for httpserver

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
This commit is contained in:
congqixia 2025-03-06 16:28:02 +08:00 committed by GitHub
parent 5c5273f95e
commit 7fbeb5624e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 27 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/distributed/proxy/httpserver/handler_v1.go
View File

@ -755,7 +755,7 @@ func (h *HandlersV1) insert(c *gin.Context) {
return nil, RestRequestInterceptorErr
}
body, _ := c.Get(gin.BodyBytesKey)
err, httpReq.Data, _ = checkAndSetData(string(body.([]byte)), collSchema)
err, httpReq.Data, _ = checkAndSetData(body.([]byte), collSchema)
if err != nil {
log.Warn("high level restful api, fail to deal with insert data", zap.Any("body", body), zap.Error(err))
HTTPAbortReturn(c, http.StatusOK, gin.H{
@ -861,7 +861,7 @@ func (h *HandlersV1) upsert(c *gin.Context) {
}
}
body, _ := c.Get(gin.BodyBytesKey)
err, httpReq.Data, _ = checkAndSetData(string(body.([]byte)), collSchema)
err, httpReq.Data, _ = checkAndSetData(body.([]byte), collSchema)
if err != nil {
log.Warn("high level restful api, fail to deal with upsert data", zap.Any("body", body), zap.Error(err))
HTTPAbortReturn(c, http.StatusOK, gin.H{

4
internal/distributed/proxy/httpserver/handler_v2.go
View File

@ -922,7 +922,7 @@ func (h *HandlersV2) insert(ctx context.Context, c *gin.Context, anyReq any, dbN
}
body, _ := c.Get(gin.BodyBytesKey)
var validDataMap map[string][]bool
err, httpReq.Data, validDataMap = checkAndSetData(string(body.([]byte)), collSchema)
err, httpReq.Data, validDataMap = checkAndSetData(body.([]byte), collSchema)
if err != nil {
log.Ctx(ctx).Warn("high level restful api, fail to deal with insert data", zap.Error(err), zap.String("body", string(body.([]byte))))
HTTPAbortReturn(c, http.StatusOK, gin.H{
@ -996,7 +996,7 @@ func (h *HandlersV2) upsert(ctx context.Context, c *gin.Context, anyReq any, dbN
}
body, _ := c.Get(gin.BodyBytesKey)
var validDataMap map[string][]bool
err, httpReq.Data, validDataMap = checkAndSetData(string(body.([]byte)), collSchema)
err, httpReq.Data, validDataMap = checkAndSetData(body.([]byte), collSchema)
if err != nil {
log.Ctx(ctx).Warn("high level restful api, fail to deal with upsert data", zap.Any("body", body), zap.Error(err))
HTTPAbortReturn(c, http.StatusOK, gin.H{

4
internal/distributed/proxy/httpserver/utils.go
View File

@ -283,10 +283,10 @@ func printIndexes(indexes []*milvuspb.IndexDescription) []gin.H {
// --------------------- insert param --------------------- //
func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error, []map[string]interface{}, map[string][]bool) {
func checkAndSetData(body []byte, collSchema *schemapb.CollectionSchema) (error, []map[string]interface{}, map[string][]bool) {
var reallyDataArray []map[string]interface{}
validDataMap := make(map[string][]bool)
dataResult := gjson.Get(body, HTTPRequestData)
dataResult := gjson.GetBytes(body, HTTPRequestData)
dataResultArray := dataResult.Array()
if len(dataResultArray) == 0 {
return merr.ErrMissingRequiredParameters, reallyDataArray, validDataMap

42
internal/distributed/proxy/httpserver/utils_test.go
View File

@ -598,7 +598,7 @@ func TestPrimaryField(t *testing.T) {
func TestAnyToColumns(t *testing.T) {
t.Run("insert with dynamic field", func(t *testing.T) {
body := "{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}"
body := []byte("{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}")
req := InsertReq{}
coll := generateCollectionSchema(schemapb.DataType_Int64, false, true)
var err error
@ -615,7 +615,7 @@ func TestAnyToColumns(t *testing.T) {
})
t.Run("upsert with dynamic field", func(t *testing.T) {
body := "{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}"
body := []byte("{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}")
req := InsertReq{}
coll := generateCollectionSchema(schemapb.DataType_Int64, false, true)
var err error
@ -632,7 +632,7 @@ func TestAnyToColumns(t *testing.T) {
})
t.Run("insert with dynamic field, but pass pk when autoid==true", func(t *testing.T) {
body := "{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}"
body := []byte("{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}")
req := InsertReq{}
coll := generateCollectionSchema(schemapb.DataType_Int64, true, true)
var err error
@ -647,7 +647,7 @@ func TestAnyToColumns(t *testing.T) {
})
t.Run("pass more field", func(t *testing.T) {
body := "{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}"
body := []byte("{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}")
coll := generateCollectionSchema(schemapb.DataType_Int64, true, false)
var err error
err, _, _ = checkAndSetData(body, coll)
@ -656,7 +656,7 @@ func TestAnyToColumns(t *testing.T) {
})
t.Run("insert with autoid==false", func(t *testing.T) {
body := "{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}"
body := []byte("{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}")
req := InsertReq{}
coll := generateCollectionSchema(schemapb.DataType_Int64, false, false)
var err error
@ -672,7 +672,7 @@ func TestAnyToColumns(t *testing.T) {
})
t.Run("insert with autoid==false but has no pk", func(t *testing.T) {
body := "{\"data\": { \"book_intro\": [0.1, 0.2], \"word_count\": 2}}"
body := []byte("{\"data\": { \"book_intro\": [0.1, 0.2], \"word_count\": 2}}")
coll := generateCollectionSchema(schemapb.DataType_Int64, false, false)
var err error
err, _, _ = checkAndSetData(body, coll)
@ -681,7 +681,7 @@ func TestAnyToColumns(t *testing.T) {
})
t.Run("insert with autoid==true", func(t *testing.T) {
body := "{\"data\": { \"book_intro\": [0.1, 0.2], \"word_count\": 2}}"
body := []byte("{\"data\": { \"book_intro\": [0.1, 0.2], \"word_count\": 2}}")
req := InsertReq{}
coll := generateCollectionSchema(schemapb.DataType_Int64, true, false)
var err error
@ -696,7 +696,7 @@ func TestAnyToColumns(t *testing.T) {
})
t.Run("upsert with autoid==true", func(t *testing.T) {
body := "{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}"
body := []byte("{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}")
req := InsertReq{}
coll := generateCollectionSchema(schemapb.DataType_Int64, true, false)
var err error
@ -712,7 +712,7 @@ func TestAnyToColumns(t *testing.T) {
})
t.Run("upsert with autoid==false", func(t *testing.T) {
body := "{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}"
body := []byte("{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}")
req := InsertReq{}
coll := generateCollectionSchema(schemapb.DataType_Int64, true, false)
var err error
@ -730,7 +730,7 @@ func TestAnyToColumns(t *testing.T) {
func TestCheckAndSetData(t *testing.T) {
t.Run("invalid field name with dynamic field", func(t *testing.T) {
body := "{\"data\": {\"id\": 0,\"$meta\": 2,\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}"
body := []byte("{\"data\": {\"id\": 0,\"$meta\": 2,\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}")
coll := generateCollectionSchema(schemapb.DataType_Int64, false, true)
var err error
err, _, _ = checkAndSetData(body, coll)
@ -738,7 +738,7 @@ func TestCheckAndSetData(t *testing.T) {
assert.Equal(t, true, strings.HasPrefix(err.Error(), "use the invalid field name"))
})
t.Run("without vector", func(t *testing.T) {
body := "{\"data\": {}}"
body := []byte("{\"data\": {}}")
var err error
primaryField := generatePrimaryField(schemapb.DataType_Int64, true)
floatVectorField := generateVectorFieldSchema(schemapb.DataType_FloatVector)
@ -800,7 +800,7 @@ func TestCheckAndSetData(t *testing.T) {
t.Run("with pk when autoID == True when upsert", func(t *testing.T) {
arrayFieldName := "array-int64"
body := "{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}"
body := []byte("{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}")
coll := generateCollectionSchema(schemapb.DataType_Int64, true, false)
coll.Fields = append(coll.Fields, &schemapb.FieldSchema{
Name: arrayFieldName,
@ -815,7 +815,7 @@ func TestCheckAndSetData(t *testing.T) {
t.Run("without pk when autoID == True when insert", func(t *testing.T) {
arrayFieldName := "array-int64"
body := "{\"data\": {\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}"
body := []byte("{\"data\": {\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}")
coll := generateCollectionSchema(schemapb.DataType_Int64, true, false)
coll.Fields = append(coll.Fields, &schemapb.FieldSchema{
Name: arrayFieldName,
@ -830,7 +830,7 @@ func TestCheckAndSetData(t *testing.T) {
t.Run("with pk when autoID == false", func(t *testing.T) {
arrayFieldName := "array-int64"
body := "{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}"
body := []byte("{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}")
coll := generateCollectionSchema(schemapb.DataType_Int64, false, false)
coll.Fields = append(coll.Fields, &schemapb.FieldSchema{
Name: arrayFieldName,
@ -846,7 +846,7 @@ func TestCheckAndSetData(t *testing.T) {
func TestInsertWithInt64(t *testing.T) {
arrayFieldName := "array-int64"
body := "{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}"
body := []byte("{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}")
coll := generateCollectionSchema(schemapb.DataType_Int64, false, true)
coll.Fields = append(coll.Fields, &schemapb.FieldSchema{
Name: arrayFieldName,
@ -875,7 +875,7 @@ func TestInsertWithNullableField(t *testing.T) {
DataType: schemapb.DataType_Int64,
Nullable: true,
})
body := "{\"data\": [{\"book_id\": 9999999999999999, \"\nullable\": null,\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]},{\"book_id\": 1, \"nullable\": 1,\"book_intro\": [0.3, 0.4], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}]"
body := []byte("{\"data\": [{\"book_id\": 9999999999999999, \"\nullable\": null,\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]},{\"book_id\": 1, \"nullable\": 1,\"book_intro\": [0.3, 0.4], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}]")
err, data, validData := checkAndSetData(body, coll)
assert.Equal(t, nil, err)
assert.Equal(t, 2, len(data))
@ -911,7 +911,7 @@ func TestInsertWithDefaultValueField(t *testing.T) {
},
},
})
body := "{\"data\": [{\"book_id\": 9999999999999999, \"\fid\": null,\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]},{\"book_id\": 1, \"fid\": 1,\"book_intro\": [0.3, 0.4], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}]"
body := []byte("{\"data\": [{\"book_id\": 9999999999999999, \"\fid\": null,\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]},{\"book_id\": 1, \"fid\": 1,\"book_intro\": [0.3, 0.4], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}]")
err, data, validData := checkAndSetData(body, coll)
assert.Equal(t, nil, err)
assert.Equal(t, 2, len(data))
@ -2074,7 +2074,7 @@ func newRowsWithArray(results []map[string]interface{}) []map[string]interface{}
func TestArray(t *testing.T) {
body, _ := generateRequestBody(schemapb.DataType_Int64)
collectionSchema := generateCollectionSchema(schemapb.DataType_Int64, false, true)
err, rows, validRows := checkAndSetData(string(body), collectionSchema)
err, rows, validRows := checkAndSetData(body, collectionSchema)
assert.Equal(t, nil, err)
assert.Equal(t, 0, len(validRows))
assert.Equal(t, true, compareRows(rows, generateRawRows(schemapb.DataType_Int64), compareRow))
@ -2084,7 +2084,7 @@ func TestArray(t *testing.T) {
body, _ = generateRequestBodyWithArray(schemapb.DataType_Int64)
collectionSchema = newCollectionSchemaWithArray(generateCollectionSchema(schemapb.DataType_Int64, false, true))
err, rows, validRows = checkAndSetData(string(body), collectionSchema)
err, rows, validRows = checkAndSetData(body, collectionSchema)
assert.Equal(t, nil, err)
assert.Equal(t, 0, len(validRows))
assert.Equal(t, true, compareRows(rows, newRowsWithArray(generateRawRows(schemapb.DataType_Int64)), compareRow))
@ -2171,7 +2171,7 @@ func TestVector(t *testing.T) {
},
EnableDynamicField: true,
}
err, rows, validRows := checkAndSetData(string(body), collectionSchema)
err, rows, validRows := checkAndSetData(body, collectionSchema)
assert.Equal(t, nil, err)
for i, row := range rows {
assert.Equal(t, 2, len(row[floatVector].([]float32)))
@ -2207,7 +2207,7 @@ func TestVector(t *testing.T) {
}
row[field] = value
body, _ = wrapRequestBody([]map[string]interface{}{row})
err, _, _ = checkAndSetData(string(body), collectionSchema)
err, _, _ = checkAndSetData(body, collectionSchema)
assert.Error(t, err)
}