diff --git a/internal/core/src/common/FieldData.cpp b/internal/core/src/common/FieldData.cpp index c10a59dbfe..28a0f9e07a 100644 --- a/internal/core/src/common/FieldData.cpp +++ b/internal/core/src/common/FieldData.cpp @@ -15,9 +15,11 @@ // limitations under the License. #include "common/FieldData.h" +#include #include "arrow/array/array_binary.h" #include "arrow/chunked_array.h" +#include "bitset/detail/element_wise.h" #include "common/Array.h" #include "common/EasyAssert.h" #include "common/Exception.h" @@ -67,13 +69,11 @@ FieldDataImpl::FillFieldData( element_count * dim_, data_.data() + length_ * dim_); - ssize_t byte_count = (element_count + 7) / 8; // Note: if 'nullable == true` and valid_data is nullptr // means null_count == 0, will fill it with 0xFF - if (valid_data == nullptr) { - valid_data_.assign(byte_count, 0xFF); - } else { - std::copy_n(valid_data, byte_count, valid_data_.data()); + if (valid_data != nullptr) { + bitset::detail::ElementWiseBitsetPolicy::op_copy( + valid_data, 0, valid_data_.data(), length_, element_count); } length_ += element_count; diff --git a/internal/core/src/common/FieldDataInterface.h b/internal/core/src/common/FieldDataInterface.h index fbedac00b2..5a1c0a2a9d 100644 --- a/internal/core/src/common/FieldDataInterface.h +++ b/internal/core/src/common/FieldDataInterface.h @@ -341,7 +341,7 @@ class FieldDataImpl : public FieldDataBase { if (IsVectorDataType(data_type)) { PanicInfo(NotImplemented, "vector type not support null"); } - valid_data_.resize((num_rows_ + 7) / 8); + valid_data_.resize((num_rows_ + 7) / 8, 0xFF); } } @@ -478,7 +478,7 @@ class FieldDataImpl : public FieldDataBase { data_.resize(num_rows_ * dim_); } if (nullable_) { - valid_data_.resize((num_rows_ + 7) / 8); + valid_data_.resize((num_rows_ + 7) / 8, 0xFF); } } @@ -496,7 +496,7 @@ class FieldDataImpl : public FieldDataBase { num_rows_ = num_rows; data_.resize(num_rows_ * dim_); if (nullable_) { - valid_data_.resize((num_rows + 7) / 8); + valid_data_.resize((num_rows + 7) / 8, 0xFF); } } } @@ -594,10 +594,9 @@ class FieldDataStringImpl : public FieldDataImpl { } if (IsNullable()) { auto valid_data = array->null_bitmap_data(); - if (valid_data == nullptr) { - valid_data_.resize((n + 7) / 8, 0xFF); - } else { - std::copy_n(valid_data, (n + 7) / 8, valid_data_.data()); + if (valid_data != nullptr) { + bitset::detail::ElementWiseBitsetPolicy::op_copy( + valid_data, 0, valid_data_.data(), length_, n); } } length_ += n; @@ -658,7 +657,8 @@ class FieldDataJsonImpl : public FieldDataImpl { resize_field_data(length_ + element_count); } - valid_data_.assign((element_count + 7) / 8, 0x00); + bitset::detail::ElementWiseBitsetPolicy::op_fill( + valid_data_.data(), length_, element_count, false); length_ += element_count; } @@ -685,10 +685,9 @@ class FieldDataJsonImpl : public FieldDataImpl { } if (IsNullable()) { auto valid_data = array->null_bitmap_data(); - if (valid_data == nullptr) { - valid_data_.assign((n + 7) / 8, 0xFF); - } else { - std::copy_n(valid_data, (n + 7) / 8, valid_data_.data()); + if (valid_data != nullptr) { + bitset::detail::ElementWiseBitsetPolicy::op_copy( + valid_data, 0, valid_data_.data(), length_, n); } } length_ += n; diff --git a/internal/core/unittest/test_data_codec.cpp b/internal/core/unittest/test_data_codec.cpp index 3873adebf1..b07ae9261e 100644 --- a/internal/core/unittest/test_data_codec.cpp +++ b/internal/core/unittest/test_data_codec.cpp @@ -64,7 +64,7 @@ TEST(storage, InsertDataBoolNullable) { FixedVector data = {true, false, false, false, true}; auto field_data = milvus::storage::CreateFieldData(storage::DataType::BOOL, true); - uint8_t* valid_data = new uint8_t[1]{0x13}; + uint8_t* valid_data = new uint8_t[1]{0xF3}; field_data->FillFieldData(data.data(), valid_data, data.size()); @@ -131,7 +131,7 @@ TEST(storage, InsertDataInt8Nullable) { FixedVector data = {1, 2, 3, 4, 5}; auto field_data = milvus::storage::CreateFieldData(storage::DataType::INT8, true); - uint8_t* valid_data = new uint8_t[1]{0x13}; + uint8_t* valid_data = new uint8_t[1]{0xF3}; field_data->FillFieldData(data.data(), valid_data, data.size()); auto payload_reader = @@ -195,7 +195,7 @@ TEST(storage, InsertDataInt16Nullable) { FixedVector data = {1, 2, 3, 4, 5}; auto field_data = milvus::storage::CreateFieldData(storage::DataType::INT16, true); - uint8_t* valid_data = new uint8_t[1]{0x13}; + uint8_t* valid_data = new uint8_t[1]{0xF3}; field_data->FillFieldData(data.data(), valid_data, data.size()); auto payload_reader = @@ -259,7 +259,7 @@ TEST(storage, InsertDataInt32Nullable) { FixedVector data = {1, 2, 3, 4, 5}; auto field_data = milvus::storage::CreateFieldData(storage::DataType::INT32, true); - uint8_t* valid_data = new uint8_t[1]{0x13}; + uint8_t* valid_data = new uint8_t[1]{0xF3}; field_data->FillFieldData(data.data(), valid_data, data.size()); auto payload_reader = @@ -323,7 +323,7 @@ TEST(storage, InsertDataInt64Nullable) { FixedVector data = {1, 2, 3, 4, 5}; auto field_data = milvus::storage::CreateFieldData(storage::DataType::INT64, true); - uint8_t* valid_data = new uint8_t[1]{0x13}; + uint8_t* valid_data = new uint8_t[1]{0xF3}; field_data->FillFieldData(data.data(), valid_data, data.size()); auto payload_reader = @@ -393,7 +393,7 @@ TEST(storage, InsertDataStringNullable) { "test1", "test2", "test3", "test4", "test5"}; auto field_data = milvus::storage::CreateFieldData(storage::DataType::STRING, true); - uint8_t* valid_data = new uint8_t[1]{0x13}; + uint8_t* valid_data = new uint8_t[1]{0xF3}; field_data->FillFieldData(data.data(), valid_data, data.size()); auto payload_reader = @@ -460,7 +460,7 @@ TEST(storage, InsertDataFloatNullable) { FixedVector data = {1, 2, 3, 4, 5}; auto field_data = milvus::storage::CreateFieldData(storage::DataType::FLOAT, true); - std::array valid_data = {0x13}; + std::array valid_data = {0xF3}; field_data->FillFieldData(data.data(), valid_data.data(), data.size()); auto payload_reader = @@ -523,7 +523,7 @@ TEST(storage, InsertDataDoubleNullable) { FixedVector data = {1, 2, 3, 4, 5}; auto field_data = milvus::storage::CreateFieldData(storage::DataType::DOUBLE, true); - uint8_t* valid_data = new uint8_t[1]{0x13}; + uint8_t* valid_data = new uint8_t[1]{0xF3}; field_data->FillFieldData(data.data(), valid_data, data.size()); auto payload_reader = @@ -774,7 +774,7 @@ TEST(storage, InsertDataStringArrayNullable) { FixedVector data = {string_array, int_array}; auto field_data = milvus::storage::CreateFieldData(storage::DataType::ARRAY, true); - uint8_t* valid_data = new uint8_t[1]{0x01}; + uint8_t* valid_data = new uint8_t[1]{0xFD}; field_data->FillFieldData(data.data(), valid_data, data.size()); auto payload_reader = @@ -812,7 +812,7 @@ TEST(storage, InsertDataJsonNullable) { Json(simdjson::padded_string(std::string("A")))}; auto field_data = milvus::storage::CreateFieldData(storage::DataType::JSON, true); - uint8_t* valid_data = new uint8_t[1]{0x00}; + uint8_t* valid_data = new uint8_t[1]{0xFC}; field_data->FillFieldData(data.data(), valid_data, data.size()); auto payload_reader = @@ -842,7 +842,7 @@ TEST(storage, InsertDataJsonFillWithNull) { auto field_data = milvus::storage::CreateFieldData(storage::DataType::JSON, true); int64_t size = 2; - uint8_t* valid_data = new uint8_t[1]{0x00}; + uint8_t* valid_data = new uint8_t[1]{0xFC}; field_data->FillFieldData(std::nullopt, size); auto payload_reader =