admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: missing to support compact for Array type (#29505)

Browse Source 
the array type can't be compacted, the system could continue with the
inserted segments, but these segments can be never compacted

fix #29503

---------

Signed-off-by: yah01 <yah2er0ne@outlook.com>
This commit is contained in:
yah01 2023-12-28 15:42:48 +08:00 committed by GitHub
parent 8a4c0d4a3f
commit a8a0aa9357
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 49 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
internal/datanode/compactor.go
View File

@ -761,6 +761,24 @@ func interface2FieldData(schemaDataType schemapb.DataType, content []interface{}
}
rst = data
case schemapb.DataType_Array:
data := &storage.ArrayFieldData{
Data: make([]*schemapb.ScalarField, 0, len(content)),
}
if len(content) > 0 {
data.ElementType = content[0].(*schemapb.ScalarField).GetArrayData().GetElementType()
}
for _, c := range content {
r, ok := c.(*schemapb.ScalarField)
if !ok {
return nil, errTransferType
}
data.Data = append(data.Data, r)
}
rst = data
case schemapb.DataType_FloatVector:
data := &storage.FloatVectorFieldData{
Data: []float32{},

33
internal/datanode/compactor_test.go
View File

@ -99,9 +99,26 @@ func TestCompactionTaskInnerMethods(t *testing.T) {
{true, schemapb.DataType_Double, []interface{}{float64(1), float64(2)}, "valid float64"},
{true, schemapb.DataType_VarChar, []interface{}{"test1", "test2"}, "valid varChar"},
{true, schemapb.DataType_JSON, []interface{}{[]byte("{\"key\":\"value\"}"), []byte("{\"hello\":\"world\"}")}, "valid json"},
{true, schemapb.DataType_Array, []interface{}{
&schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: []int32{1, 2},
},
},
},
&schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: []int32{3, 4},
},
},
},
}, "valid array"},
{true, schemapb.DataType_FloatVector, []interface{}{[]float32{1.0, 2.0}}, "valid floatvector"},
{true, schemapb.DataType_BinaryVector, []interface{}{[]byte{255}}, "valid binaryvector"},
{true, schemapb.DataType_Float16Vector, []interface{}{[]byte{255, 255, 255, 255}}, "valid float16vector"},
{false, schemapb.DataType_Bool, []interface{}{1, 2}, "invalid bool"},
{false, schemapb.DataType_Int8, []interface{}{nil, nil}, "invalid int8"},
{false, schemapb.DataType_Int16, []interface{}{nil, nil}, "invalid int16"},
@ -114,9 +131,21 @@ func TestCompactionTaskInnerMethods(t *testing.T) {
{false, schemapb.DataType_FloatVector, []interface{}{nil, nil}, "invalid floatvector"},
{false, schemapb.DataType_BinaryVector, []interface{}{nil, nil}, "invalid binaryvector"},
{false, schemapb.DataType_Float16Vector, []interface{}{nil, nil}, "invalid float16vector"},
{false, schemapb.DataType_None, nil, "invalid data type"},
}
// make sure all new data types missed to handle would throw unexpected error
// todo(yah01): enable this after the BF16 vector type ready
// for typeName, typeValue := range schemapb.DataType_value {
// tests = append(tests, struct {
// isvalid bool
// tp schemapb.DataType
// content []interface{}
// description string
// }{false, schemapb.DataType(typeValue), []interface{}{nil, nil}, "invalid " + typeName})
// }
for _, test := range tests {
t.Run(test.description, func(t *testing.T) {
if test.isvalid {
@ -125,7 +154,7 @@ func TestCompactionTaskInnerMethods(t *testing.T) {
assert.Equal(t, 2, fd.RowNum())
} else {
fd, err := interface2FieldData(test.tp, test.content, 2)
assert.Error(t, err)
assert.ErrorIs(t, err, errTransferType)
assert.Nil(t, fd)
}
})