From a8a0aa93572be3187280fd2c99cf46c0ab3f087a Mon Sep 17 00:00:00 2001 From: yah01 Date: Thu, 28 Dec 2023 15:42:48 +0800 Subject: [PATCH] fix: missing to support compact for Array type (#29505) the array type can't be compacted, the system could continue with the inserted segments, but these segments can be never compacted fix #29503 --------- Signed-off-by: yah01 --- internal/datanode/compactor.go | 18 ++++++++++++++++ internal/datanode/compactor_test.go | 33 +++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/internal/datanode/compactor.go b/internal/datanode/compactor.go index 9060558027..9af0c5c701 100644 --- a/internal/datanode/compactor.go +++ b/internal/datanode/compactor.go @@ -761,6 +761,24 @@ func interface2FieldData(schemaDataType schemapb.DataType, content []interface{} } rst = data + case schemapb.DataType_Array: + data := &storage.ArrayFieldData{ + Data: make([]*schemapb.ScalarField, 0, len(content)), + } + + if len(content) > 0 { + data.ElementType = content[0].(*schemapb.ScalarField).GetArrayData().GetElementType() + } + + for _, c := range content { + r, ok := c.(*schemapb.ScalarField) + if !ok { + return nil, errTransferType + } + data.Data = append(data.Data, r) + } + rst = data + case schemapb.DataType_FloatVector: data := &storage.FloatVectorFieldData{ Data: []float32{}, diff --git a/internal/datanode/compactor_test.go b/internal/datanode/compactor_test.go index e99dadfe3d..8560ac8444 100644 --- a/internal/datanode/compactor_test.go +++ b/internal/datanode/compactor_test.go @@ -99,9 +99,26 @@ func TestCompactionTaskInnerMethods(t *testing.T) { {true, schemapb.DataType_Double, []interface{}{float64(1), float64(2)}, "valid float64"}, {true, schemapb.DataType_VarChar, []interface{}{"test1", "test2"}, "valid varChar"}, {true, schemapb.DataType_JSON, []interface{}{[]byte("{\"key\":\"value\"}"), []byte("{\"hello\":\"world\"}")}, "valid json"}, + {true, schemapb.DataType_Array, []interface{}{ + &schemapb.ScalarField{ + Data: &schemapb.ScalarField_IntData{ + IntData: &schemapb.IntArray{ + Data: []int32{1, 2}, + }, + }, + }, + &schemapb.ScalarField{ + Data: &schemapb.ScalarField_IntData{ + IntData: &schemapb.IntArray{ + Data: []int32{3, 4}, + }, + }, + }, + }, "valid array"}, {true, schemapb.DataType_FloatVector, []interface{}{[]float32{1.0, 2.0}}, "valid floatvector"}, {true, schemapb.DataType_BinaryVector, []interface{}{[]byte{255}}, "valid binaryvector"}, {true, schemapb.DataType_Float16Vector, []interface{}{[]byte{255, 255, 255, 255}}, "valid float16vector"}, + {false, schemapb.DataType_Bool, []interface{}{1, 2}, "invalid bool"}, {false, schemapb.DataType_Int8, []interface{}{nil, nil}, "invalid int8"}, {false, schemapb.DataType_Int16, []interface{}{nil, nil}, "invalid int16"}, @@ -114,9 +131,21 @@ func TestCompactionTaskInnerMethods(t *testing.T) { {false, schemapb.DataType_FloatVector, []interface{}{nil, nil}, "invalid floatvector"}, {false, schemapb.DataType_BinaryVector, []interface{}{nil, nil}, "invalid binaryvector"}, {false, schemapb.DataType_Float16Vector, []interface{}{nil, nil}, "invalid float16vector"}, - {false, schemapb.DataType_None, nil, "invalid data type"}, } + // make sure all new data types missed to handle would throw unexpected error + // todo(yah01): enable this after the BF16 vector type ready + // for typeName, typeValue := range schemapb.DataType_value { + // tests = append(tests, struct { + // isvalid bool + + // tp schemapb.DataType + // content []interface{} + + // description string + // }{false, schemapb.DataType(typeValue), []interface{}{nil, nil}, "invalid " + typeName}) + // } + for _, test := range tests { t.Run(test.description, func(t *testing.T) { if test.isvalid { @@ -125,7 +154,7 @@ func TestCompactionTaskInnerMethods(t *testing.T) { assert.Equal(t, 2, fd.RowNum()) } else { fd, err := interface2FieldData(test.tp, test.content, 2) - assert.Error(t, err) + assert.ErrorIs(t, err, errTransferType) assert.Nil(t, fd) } })