admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: add some check fieldata dim (#31564) (#31899)

Browse Source 
issue: https://github.com/milvus-io/milvus/issues/30138
issue: https://github.com/milvus-io/milvus/issues/31724

pr: https://github.com/milvus-io/milvus/pull/31564

---------

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
Co-authored-by: cqy123456 <39671710+cqy123456@users.noreply.github.com>
This commit is contained in:
yihao.dai 2024-04-05 10:04:16 +08:00 committed by GitHub
parent ba36f66a5c
commit fa447532d7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 102 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
internal/proxy/validate_util.go
View File

@ -115,7 +115,10 @@ func (v *validateUtil) checkAligned(data []*schemapb.FieldData, schema *typeutil
msg := fmt.Sprintf("the num_rows (%d) of field (%s) is not equal to passed num_rows (%d)", fieldNumRows, fieldName, passedNumRows)
return merr.WrapErrParameterInvalid(passedNumRows, numRows, msg)
}
errDimMismatch := func(fieldName string, dataDim int64, schemaDim int64) error {
msg := fmt.Sprintf("the dim (%d) of field data(%s) is not equal to schema dim (%d)", dataDim, fieldName, schemaDim)
return merr.WrapErrParameterInvalid(dataDim, schemaDim, msg)
}
for _, field := range data {
switch field.GetType() {
case schemapb.DataType_FloatVector:
@ -133,6 +136,10 @@ func (v *validateUtil) checkAligned(data []*schemapb.FieldData, schema *typeutil
if err != nil {
return err
}
dataDim := field.GetVectors().Dim
if dataDim != dim {
return errDimMismatch(field.GetFieldName(), dataDim, dim)
}
if n != numRows {
return errNumRowsMismatch(field.GetFieldName(), n, numRows)
@ -148,6 +155,10 @@ func (v *validateUtil) checkAligned(data []*schemapb.FieldData, schema *typeutil
if err != nil {
return err
}
dataDim := field.GetVectors().Dim
if dataDim != dim {
return errDimMismatch(field.GetFieldName(), dataDim, dim)
}
n, err := funcutil.GetNumRowsOfBinaryVectorField(field.GetVectors().GetBinaryVector(), dim)
if err != nil {

90
internal/proxy/validate_util_test.go
View File

@ -316,6 +316,48 @@ func Test_validateUtil_checkAligned(t *testing.T) {
assert.Error(t, err)
})
t.Run("field_data dim not match schema dim", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_FloatVector,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Data: &schemapb.VectorField_FloatVector{
FloatVector: &schemapb.FloatArray{
Data: []float32{1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8},
},
},
Dim: 16,
},
},
},
}
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: "8",
},
},
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)
v := newValidateUtil()
err = v.checkAligned(data, h, 1)
assert.Error(t, err)
})
t.Run("invalid num rows", func(t *testing.T) {
data := []*schemapb.FieldData{
{
@ -328,6 +370,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
Data: []float32{1.1, 2.2},
},
},
Dim: 8,
},
},
},
@ -369,6 +412,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
Data: []float32{1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8},
},
},
Dim: 8,
},
},
},
@ -445,6 +489,46 @@ func Test_validateUtil_checkAligned(t *testing.T) {
assert.Error(t, err)
})
t.Run("field data dim not match schema dim", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_BinaryVector,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Data: &schemapb.VectorField_BinaryVector{
BinaryVector: []byte("66666666"),
},
Dim: 128,
},
},
},
}
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: "8",
},
},
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)
v := newValidateUtil()
err = v.checkAligned(data, h, 100)
assert.Error(t, err)
})
t.Run("invalid num rows", func(t *testing.T) {
data := []*schemapb.FieldData{
{
@ -455,6 +539,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
Data: &schemapb.VectorField_BinaryVector{
BinaryVector: []byte("not128"),
},
Dim: 128,
},
},
},
@ -494,6 +579,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
Data: &schemapb.VectorField_BinaryVector{
BinaryVector: []byte{'1', '2'},
},
Dim: 8,
},
},
},
@ -575,6 +661,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
Type: schemapb.DataType_FloatVector,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: 8,
Data: &schemapb.VectorField_FloatVector{
FloatVector: &schemapb.FloatArray{
Data: generateFloatVectors(10, 8),
@ -588,6 +675,7 @@ func Test_validateUtil_checkAligned(t *testing.T) {
Type: schemapb.DataType_BinaryVector,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: 8,
Data: &schemapb.VectorField_BinaryVector{
BinaryVector: generateBinaryVectors(10, 8),
},
@ -1616,6 +1704,7 @@ func Test_validateUtil_Validate(t *testing.T) {
Data: generateFloatVectors(2, 8),
},
},
Dim: 8,
},
},
},
@ -1627,6 +1716,7 @@ func Test_validateUtil_Validate(t *testing.T) {
Data: &schemapb.VectorField_BinaryVector{
BinaryVector: generateBinaryVectors(2, 8),
},
Dim: 8,
},
},
},