admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: Fix inserting geometry data bug by restful (#44479)

Browse Source 
issue: #43427

---------

Signed-off-by: Cai Zhang <cai.zhang@zilliz.com>
This commit is contained in:
cai.zhang 2025-09-19 17:18:02 +08:00 committed by GitHub
parent 5b8288a0ef
commit fa54cdadac
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 18 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
internal/distributed/proxy/httpserver/utils.go
View File

@ -19,7 +19,6 @@ package httpserver
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"math"
"reflect"
@ -31,8 +30,6 @@ import (
"github.com/gin-gonic/gin"
"github.com/spf13/cast"
"github.com/tidwall/gjson"
"github.com/twpayne/go-geom/encoding/wkb"
"github.com/twpayne/go-geom/encoding/wkt"
"go.uber.org/zap"
"google.golang.org/protobuf/proto"
@ -549,24 +546,7 @@ func checkAndSetData(body []byte, collSchema *schemapb.CollectionSchema) (error,
case schemapb.DataType_JSON:
reallyData[fieldName] = []byte(dataString)
case schemapb.DataType_Geometry:
// treat as string(wkt) data,the string data must be valid
WktString, err := base64.StdEncoding.DecodeString(dataString)
if err != nil {
log.Warn("proxy can not decode datastring with base64", zap.String("WktString:", dataString))
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
geomT, err := wkt.Unmarshal(string(WktString))
if err != nil {
log.Warn("proxy change wkt to geomtyr failed!!", zap.String("WktString:", dataString))
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
// translate the wkt bytes to wkb bytes ,store the bytes in LittleEndian which cpp core used as well
dataWkbBytes, err := wkb.Marshal(geomT, wkb.NDR)
if err != nil {
log.Warn("proxy change geomtry to wkb failed!!", zap.String("WktString:", dataString))
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = dataWkbBytes
reallyData[fieldName] = dataString
case schemapb.DataType_Float:
result, err := cast.ToFloat32E(dataString)
if err != nil {
@ -775,7 +755,7 @@ func anyToColumns(rows []map[string]interface{}, validDataMap map[string][]bool,
case schemapb.DataType_JSON:
data = make([][]byte, 0, rowsLen)
case schemapb.DataType_Geometry:
data = make([][]byte, 0, rowsLen)
data = make([]string, 0, rowsLen)
case schemapb.DataType_FloatVector:
data = make([][]float32, 0, rowsLen)
dim, _ := getDim(field)
@ -866,7 +846,7 @@ func anyToColumns(rows []map[string]interface{}, validDataMap map[string][]bool,
case schemapb.DataType_JSON:
nameColumns[field.Name] = append(nameColumns[field.Name].([][]byte), candi.v.Interface().([]byte))
case schemapb.DataType_Geometry:
nameColumns[field.Name] = append(nameColumns[field.Name].([][]byte), candi.v.Interface().([]byte))
nameColumns[field.Name] = append(nameColumns[field.Name].([]string), candi.v.Interface().(string))
case schemapb.DataType_FloatVector:
nameColumns[field.Name] = append(nameColumns[field.Name].([][]float32), candi.v.Interface().([]float32))
case schemapb.DataType_BinaryVector:
@ -1034,9 +1014,9 @@ func anyToColumns(rows []map[string]interface{}, validDataMap map[string][]bool,
case schemapb.DataType_Geometry:
colData.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_GeometryData{
GeometryData: &schemapb.GeometryArray{
Data: column.([][]byte),
Data: &schemapb.ScalarField_GeometryWktData{
GeometryWktData: &schemapb.GeometryWktArray{
Data: column.([]string),
},
},
},
@ -1310,7 +1290,7 @@ func buildQueryResp(rowsNum int64, needFields []string, fieldDataList []*schemap
case schemapb.DataType_JSON:
rowsNum = int64(len(fieldDataList[0].GetScalars().GetJsonData().Data))
case schemapb.DataType_Geometry:
rowsNum = int64(len(fieldDataList[0].GetScalars().GetGeometryData().Data))
rowsNum = int64(len(fieldDataList[0].GetScalars().GetGeometryWktData().Data))
case schemapb.DataType_BinaryVector:
rowsNum = int64(len(fieldDataList[0].GetVectors().GetBinaryVector())*8) / fieldDataList[0].GetVectors().GetDim()
case schemapb.DataType_FloatVector:
@ -1466,7 +1446,7 @@ func buildQueryResp(rowsNum int64, needFields []string, fieldDataList []*schemap
row[fieldDataList[j].FieldName] = nil
continue
}
row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetGeometryData().Data[i]
row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetGeometryWktData().Data[i]
default:
row[fieldDataList[j].FieldName] = ""
}

21
internal/distributed/proxy/httpserver/utils_test.go
View File

@ -1230,9 +1230,9 @@ func compareRow(m1 map[string]interface{}, m2 map[string]interface{}) bool {
return false
}
} else if key == "field-geometry" {
arr1 := value.([]uint8)
arr2 := string(m2[key].([]byte))
if arr2 != (string)(arr1) {
arr1 := value.(string)
arr2 := m2[key].(string)
if arr2 != arr1 {
return false
}
} else if strings.HasPrefix(key, "array-") {
@ -1600,13 +1600,12 @@ func newFieldData(fieldDatas []*schemapb.FieldData, firstFieldType schemapb.Data
FieldName: "field-geometry",
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_GeometryData{
GeometryData: &schemapb.GeometryArray{
// WKT: POINT (30.123 -10.456)
Data: [][]byte{
[]byte(`POINT (30.123 -10.456)`),
[]byte(`POINT (30.123 -10.456)`),
[]byte(`POINT (30.123 -10.456)`),
Data: &schemapb.ScalarField_GeometryWktData{
GeometryWktData: &schemapb.GeometryWktArray{
Data: []string{
`POINT (30.123 -10.456)`,
`POINT (30.123 -10.456)`,
`POINT (30.123 -10.456)`,
// wkb:{0x01, 0x01, 0x00, 0x00, 0x00, 0xD2, 0x4A, 0x4D, 0x6A, 0x8B, 0x3C, 0x5C, 0x0A, 0x0D, 0x1B, 0x4F, 0x4F, 0x9A, 0x3D, 0x4},
},
},
@ -1919,7 +1918,7 @@ func newSearchResult(results []map[string]interface{}) []map[string]interface{}
result["field-varchar"] = strconv.Itoa(i)
result["field-string"] = strconv.Itoa(i)
result["field-json"] = []byte(`{"XXX": 0}`)
result["field-geometry"] = []byte(`POINT (30.123 -10.456)`)
result["field-geometry"] = `POINT (30.123 -10.456)`
result["field-array"] = []bool{true}
result["array-bool"] = []bool{true}
result["array-int8"] = []int32{0}