diff --git a/internal/distributed/proxy/httpserver/handler_v1.go b/internal/distributed/proxy/httpserver/handler_v1.go index 53a6187107..9980a41a19 100644 --- a/internal/distributed/proxy/httpserver/handler_v1.go +++ b/internal/distributed/proxy/httpserver/handler_v1.go @@ -755,7 +755,7 @@ func (h *HandlersV1) insert(c *gin.Context) { return nil, RestRequestInterceptorErr } body, _ := c.Get(gin.BodyBytesKey) - err, httpReq.Data, _ = checkAndSetData(string(body.([]byte)), collSchema) + err, httpReq.Data, _ = checkAndSetData(body.([]byte), collSchema) if err != nil { log.Warn("high level restful api, fail to deal with insert data", zap.Any("body", body), zap.Error(err)) HTTPAbortReturn(c, http.StatusOK, gin.H{ @@ -861,7 +861,7 @@ func (h *HandlersV1) upsert(c *gin.Context) { } } body, _ := c.Get(gin.BodyBytesKey) - err, httpReq.Data, _ = checkAndSetData(string(body.([]byte)), collSchema) + err, httpReq.Data, _ = checkAndSetData(body.([]byte), collSchema) if err != nil { log.Warn("high level restful api, fail to deal with upsert data", zap.Any("body", body), zap.Error(err)) HTTPAbortReturn(c, http.StatusOK, gin.H{ diff --git a/internal/distributed/proxy/httpserver/handler_v2.go b/internal/distributed/proxy/httpserver/handler_v2.go index 2532e907fc..d541868f4c 100644 --- a/internal/distributed/proxy/httpserver/handler_v2.go +++ b/internal/distributed/proxy/httpserver/handler_v2.go @@ -922,7 +922,7 @@ func (h *HandlersV2) insert(ctx context.Context, c *gin.Context, anyReq any, dbN } body, _ := c.Get(gin.BodyBytesKey) var validDataMap map[string][]bool - err, httpReq.Data, validDataMap = checkAndSetData(string(body.([]byte)), collSchema) + err, httpReq.Data, validDataMap = checkAndSetData(body.([]byte), collSchema) if err != nil { log.Ctx(ctx).Warn("high level restful api, fail to deal with insert data", zap.Error(err), zap.String("body", string(body.([]byte)))) HTTPAbortReturn(c, http.StatusOK, gin.H{ @@ -996,7 +996,7 @@ func (h *HandlersV2) upsert(ctx context.Context, c *gin.Context, anyReq any, dbN } body, _ := c.Get(gin.BodyBytesKey) var validDataMap map[string][]bool - err, httpReq.Data, validDataMap = checkAndSetData(string(body.([]byte)), collSchema) + err, httpReq.Data, validDataMap = checkAndSetData(body.([]byte), collSchema) if err != nil { log.Ctx(ctx).Warn("high level restful api, fail to deal with upsert data", zap.Any("body", body), zap.Error(err)) HTTPAbortReturn(c, http.StatusOK, gin.H{ diff --git a/internal/distributed/proxy/httpserver/utils.go b/internal/distributed/proxy/httpserver/utils.go index c42eaa4765..1b32098f80 100644 --- a/internal/distributed/proxy/httpserver/utils.go +++ b/internal/distributed/proxy/httpserver/utils.go @@ -283,10 +283,10 @@ func printIndexes(indexes []*milvuspb.IndexDescription) []gin.H { // --------------------- insert param --------------------- // -func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error, []map[string]interface{}, map[string][]bool) { +func checkAndSetData(body []byte, collSchema *schemapb.CollectionSchema) (error, []map[string]interface{}, map[string][]bool) { var reallyDataArray []map[string]interface{} validDataMap := make(map[string][]bool) - dataResult := gjson.Get(body, HTTPRequestData) + dataResult := gjson.GetBytes(body, HTTPRequestData) dataResultArray := dataResult.Array() if len(dataResultArray) == 0 { return merr.ErrMissingRequiredParameters, reallyDataArray, validDataMap diff --git a/internal/distributed/proxy/httpserver/utils_test.go b/internal/distributed/proxy/httpserver/utils_test.go index 340a00c8e1..834ca2e59e 100644 --- a/internal/distributed/proxy/httpserver/utils_test.go +++ b/internal/distributed/proxy/httpserver/utils_test.go @@ -598,7 +598,7 @@ func TestPrimaryField(t *testing.T) { func TestAnyToColumns(t *testing.T) { t.Run("insert with dynamic field", func(t *testing.T) { - body := "{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}" + body := []byte("{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}") req := InsertReq{} coll := generateCollectionSchema(schemapb.DataType_Int64, false, true) var err error @@ -615,7 +615,7 @@ func TestAnyToColumns(t *testing.T) { }) t.Run("upsert with dynamic field", func(t *testing.T) { - body := "{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}" + body := []byte("{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}") req := InsertReq{} coll := generateCollectionSchema(schemapb.DataType_Int64, false, true) var err error @@ -632,7 +632,7 @@ func TestAnyToColumns(t *testing.T) { }) t.Run("insert with dynamic field, but pass pk when autoid==true", func(t *testing.T) { - body := "{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}" + body := []byte("{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}") req := InsertReq{} coll := generateCollectionSchema(schemapb.DataType_Int64, true, true) var err error @@ -647,7 +647,7 @@ func TestAnyToColumns(t *testing.T) { }) t.Run("pass more field", func(t *testing.T) { - body := "{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}" + body := []byte("{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}") coll := generateCollectionSchema(schemapb.DataType_Int64, true, false) var err error err, _, _ = checkAndSetData(body, coll) @@ -656,7 +656,7 @@ func TestAnyToColumns(t *testing.T) { }) t.Run("insert with autoid==false", func(t *testing.T) { - body := "{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}" + body := []byte("{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}") req := InsertReq{} coll := generateCollectionSchema(schemapb.DataType_Int64, false, false) var err error @@ -672,7 +672,7 @@ func TestAnyToColumns(t *testing.T) { }) t.Run("insert with autoid==false but has no pk", func(t *testing.T) { - body := "{\"data\": { \"book_intro\": [0.1, 0.2], \"word_count\": 2}}" + body := []byte("{\"data\": { \"book_intro\": [0.1, 0.2], \"word_count\": 2}}") coll := generateCollectionSchema(schemapb.DataType_Int64, false, false) var err error err, _, _ = checkAndSetData(body, coll) @@ -681,7 +681,7 @@ func TestAnyToColumns(t *testing.T) { }) t.Run("insert with autoid==true", func(t *testing.T) { - body := "{\"data\": { \"book_intro\": [0.1, 0.2], \"word_count\": 2}}" + body := []byte("{\"data\": { \"book_intro\": [0.1, 0.2], \"word_count\": 2}}") req := InsertReq{} coll := generateCollectionSchema(schemapb.DataType_Int64, true, false) var err error @@ -696,7 +696,7 @@ func TestAnyToColumns(t *testing.T) { }) t.Run("upsert with autoid==true", func(t *testing.T) { - body := "{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}" + body := []byte("{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}") req := InsertReq{} coll := generateCollectionSchema(schemapb.DataType_Int64, true, false) var err error @@ -712,7 +712,7 @@ func TestAnyToColumns(t *testing.T) { }) t.Run("upsert with autoid==false", func(t *testing.T) { - body := "{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}" + body := []byte("{\"data\": {\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2}}") req := InsertReq{} coll := generateCollectionSchema(schemapb.DataType_Int64, true, false) var err error @@ -730,7 +730,7 @@ func TestAnyToColumns(t *testing.T) { func TestCheckAndSetData(t *testing.T) { t.Run("invalid field name with dynamic field", func(t *testing.T) { - body := "{\"data\": {\"id\": 0,\"$meta\": 2,\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}" + body := []byte("{\"data\": {\"id\": 0,\"$meta\": 2,\"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}") coll := generateCollectionSchema(schemapb.DataType_Int64, false, true) var err error err, _, _ = checkAndSetData(body, coll) @@ -738,7 +738,7 @@ func TestCheckAndSetData(t *testing.T) { assert.Equal(t, true, strings.HasPrefix(err.Error(), "use the invalid field name")) }) t.Run("without vector", func(t *testing.T) { - body := "{\"data\": {}}" + body := []byte("{\"data\": {}}") var err error primaryField := generatePrimaryField(schemapb.DataType_Int64, true) floatVectorField := generateVectorFieldSchema(schemapb.DataType_FloatVector) @@ -800,7 +800,7 @@ func TestCheckAndSetData(t *testing.T) { t.Run("with pk when autoID == True when upsert", func(t *testing.T) { arrayFieldName := "array-int64" - body := "{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}" + body := []byte("{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}") coll := generateCollectionSchema(schemapb.DataType_Int64, true, false) coll.Fields = append(coll.Fields, &schemapb.FieldSchema{ Name: arrayFieldName, @@ -815,7 +815,7 @@ func TestCheckAndSetData(t *testing.T) { t.Run("without pk when autoID == True when insert", func(t *testing.T) { arrayFieldName := "array-int64" - body := "{\"data\": {\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}" + body := []byte("{\"data\": {\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}") coll := generateCollectionSchema(schemapb.DataType_Int64, true, false) coll.Fields = append(coll.Fields, &schemapb.FieldSchema{ Name: arrayFieldName, @@ -830,7 +830,7 @@ func TestCheckAndSetData(t *testing.T) { t.Run("with pk when autoID == false", func(t *testing.T) { arrayFieldName := "array-int64" - body := "{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}" + body := []byte("{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}") coll := generateCollectionSchema(schemapb.DataType_Int64, false, false) coll.Fields = append(coll.Fields, &schemapb.FieldSchema{ Name: arrayFieldName, @@ -846,7 +846,7 @@ func TestCheckAndSetData(t *testing.T) { func TestInsertWithInt64(t *testing.T) { arrayFieldName := "array-int64" - body := "{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}" + body := []byte("{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}") coll := generateCollectionSchema(schemapb.DataType_Int64, false, true) coll.Fields = append(coll.Fields, &schemapb.FieldSchema{ Name: arrayFieldName, @@ -875,7 +875,7 @@ func TestInsertWithNullableField(t *testing.T) { DataType: schemapb.DataType_Int64, Nullable: true, }) - body := "{\"data\": [{\"book_id\": 9999999999999999, \"\nullable\": null,\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]},{\"book_id\": 1, \"nullable\": 1,\"book_intro\": [0.3, 0.4], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}]" + body := []byte("{\"data\": [{\"book_id\": 9999999999999999, \"\nullable\": null,\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]},{\"book_id\": 1, \"nullable\": 1,\"book_intro\": [0.3, 0.4], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}]") err, data, validData := checkAndSetData(body, coll) assert.Equal(t, nil, err) assert.Equal(t, 2, len(data)) @@ -911,7 +911,7 @@ func TestInsertWithDefaultValueField(t *testing.T) { }, }, }) - body := "{\"data\": [{\"book_id\": 9999999999999999, \"\fid\": null,\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]},{\"book_id\": 1, \"fid\": 1,\"book_intro\": [0.3, 0.4], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}]" + body := []byte("{\"data\": [{\"book_id\": 9999999999999999, \"\fid\": null,\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]},{\"book_id\": 1, \"fid\": 1,\"book_intro\": [0.3, 0.4], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}]") err, data, validData := checkAndSetData(body, coll) assert.Equal(t, nil, err) assert.Equal(t, 2, len(data)) @@ -2074,7 +2074,7 @@ func newRowsWithArray(results []map[string]interface{}) []map[string]interface{} func TestArray(t *testing.T) { body, _ := generateRequestBody(schemapb.DataType_Int64) collectionSchema := generateCollectionSchema(schemapb.DataType_Int64, false, true) - err, rows, validRows := checkAndSetData(string(body), collectionSchema) + err, rows, validRows := checkAndSetData(body, collectionSchema) assert.Equal(t, nil, err) assert.Equal(t, 0, len(validRows)) assert.Equal(t, true, compareRows(rows, generateRawRows(schemapb.DataType_Int64), compareRow)) @@ -2084,7 +2084,7 @@ func TestArray(t *testing.T) { body, _ = generateRequestBodyWithArray(schemapb.DataType_Int64) collectionSchema = newCollectionSchemaWithArray(generateCollectionSchema(schemapb.DataType_Int64, false, true)) - err, rows, validRows = checkAndSetData(string(body), collectionSchema) + err, rows, validRows = checkAndSetData(body, collectionSchema) assert.Equal(t, nil, err) assert.Equal(t, 0, len(validRows)) assert.Equal(t, true, compareRows(rows, newRowsWithArray(generateRawRows(schemapb.DataType_Int64)), compareRow)) @@ -2171,7 +2171,7 @@ func TestVector(t *testing.T) { }, EnableDynamicField: true, } - err, rows, validRows := checkAndSetData(string(body), collectionSchema) + err, rows, validRows := checkAndSetData(body, collectionSchema) assert.Equal(t, nil, err) for i, row := range rows { assert.Equal(t, 2, len(row[floatVector].([]float32))) @@ -2207,7 +2207,7 @@ func TestVector(t *testing.T) { } row[field] = value body, _ = wrapRequestBody([]map[string]interface{}{row}) - err, _, _ = checkAndSetData(string(body), collectionSchema) + err, _, _ = checkAndSetData(body, collectionSchema) assert.Error(t, err) }