admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: Fix the bug of valid data write corruption (#42556)

Browse Source 
issue: https://github.com/milvus-io/milvus/issues/42554

---------

Signed-off-by: sunby <sunbingyi1992@gmail.com>
This commit is contained in:
Bingyi Sun 2025-06-10 14:22:34 +08:00 committed by GitHub
parent 2861096734
commit b3ecf77a66
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 27 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
internal/core/src/common/FieldData.cpp
View File

@ -15,9 +15,11 @@
// limitations under the License.
#include "common/FieldData.h"
#include <cstdint>
#include "arrow/array/array_binary.h"
#include "arrow/chunked_array.h"
#include "bitset/detail/element_wise.h"
#include "common/Array.h"
#include "common/EasyAssert.h"
#include "common/Exception.h"
@ -67,13 +69,11 @@ FieldDataImpl<Type, is_type_entire_row>::FillFieldData(
element_count * dim_,
data_.data() + length_ * dim_);
ssize_t byte_count = (element_count + 7) / 8;
// Note: if 'nullable == true` and valid_data is nullptr
// means null_count == 0, will fill it with 0xFF
if (valid_data == nullptr) {
valid_data_.assign(byte_count, 0xFF);
} else {
std::copy_n(valid_data, byte_count, valid_data_.data());
if (valid_data != nullptr) {
bitset::detail::ElementWiseBitsetPolicy<uint8_t>::op_copy(
valid_data, 0, valid_data_.data(), length_, element_count);
}
length_ += element_count;

23
internal/core/src/common/FieldDataInterface.h
View File

@ -341,7 +341,7 @@ class FieldDataImpl : public FieldDataBase {
if (IsVectorDataType(data_type)) {
PanicInfo(NotImplemented, "vector type not support null");
}
valid_data_.resize((num_rows_ + 7) / 8);
valid_data_.resize((num_rows_ + 7) / 8, 0xFF);
}
}
@ -478,7 +478,7 @@ class FieldDataImpl : public FieldDataBase {
data_.resize(num_rows_ * dim_);
}
if (nullable_) {
valid_data_.resize((num_rows_ + 7) / 8);
valid_data_.resize((num_rows_ + 7) / 8, 0xFF);
}
}
@ -496,7 +496,7 @@ class FieldDataImpl : public FieldDataBase {
num_rows_ = num_rows;
data_.resize(num_rows_ * dim_);
if (nullable_) {
valid_data_.resize((num_rows + 7) / 8);
valid_data_.resize((num_rows + 7) / 8, 0xFF);
}
}
}
@ -594,10 +594,9 @@ class FieldDataStringImpl : public FieldDataImpl<std::string, true> {
}
if (IsNullable()) {
auto valid_data = array->null_bitmap_data();
if (valid_data == nullptr) {
valid_data_.resize((n + 7) / 8, 0xFF);
} else {
std::copy_n(valid_data, (n + 7) / 8, valid_data_.data());
if (valid_data != nullptr) {
bitset::detail::ElementWiseBitsetPolicy<uint8_t>::op_copy(
valid_data, 0, valid_data_.data(), length_, n);
}
}
length_ += n;
@ -658,7 +657,8 @@ class FieldDataJsonImpl : public FieldDataImpl<Json, true> {
resize_field_data(length_ + element_count);
}
valid_data_.assign((element_count + 7) / 8, 0x00);
bitset::detail::ElementWiseBitsetPolicy<uint8_t>::op_fill(
valid_data_.data(), length_, element_count, false);
length_ += element_count;
}
@ -685,10 +685,9 @@ class FieldDataJsonImpl : public FieldDataImpl<Json, true> {
}
if (IsNullable()) {
auto valid_data = array->null_bitmap_data();
if (valid_data == nullptr) {
valid_data_.assign((n + 7) / 8, 0xFF);
} else {
std::copy_n(valid_data, (n + 7) / 8, valid_data_.data());
if (valid_data != nullptr) {
bitset::detail::ElementWiseBitsetPolicy<uint8_t>::op_copy(
valid_data, 0, valid_data_.data(), length_, n);
}
}
length_ += n;

22
internal/core/unittest/test_data_codec.cpp
View File

@ -64,7 +64,7 @@ TEST(storage, InsertDataBoolNullable) {
FixedVector<bool> data = {true, false, false, false, true};
auto field_data =
milvus::storage::CreateFieldData(storage::DataType::BOOL, true);
uint8_t* valid_data = new uint8_t[1]{0x13};
uint8_t* valid_data = new uint8_t[1]{0xF3};
field_data->FillFieldData(data.data(), valid_data, data.size());
@ -131,7 +131,7 @@ TEST(storage, InsertDataInt8Nullable) {
FixedVector<int8_t> data = {1, 2, 3, 4, 5};
auto field_data =
milvus::storage::CreateFieldData(storage::DataType::INT8, true);
uint8_t* valid_data = new uint8_t[1]{0x13};
uint8_t* valid_data = new uint8_t[1]{0xF3};
field_data->FillFieldData(data.data(), valid_data, data.size());
auto payload_reader =
@ -195,7 +195,7 @@ TEST(storage, InsertDataInt16Nullable) {
FixedVector<int16_t> data = {1, 2, 3, 4, 5};
auto field_data =
milvus::storage::CreateFieldData(storage::DataType::INT16, true);
uint8_t* valid_data = new uint8_t[1]{0x13};
uint8_t* valid_data = new uint8_t[1]{0xF3};
field_data->FillFieldData(data.data(), valid_data, data.size());
auto payload_reader =
@ -259,7 +259,7 @@ TEST(storage, InsertDataInt32Nullable) {
FixedVector<int32_t> data = {1, 2, 3, 4, 5};
auto field_data =
milvus::storage::CreateFieldData(storage::DataType::INT32, true);
uint8_t* valid_data = new uint8_t[1]{0x13};
uint8_t* valid_data = new uint8_t[1]{0xF3};
field_data->FillFieldData(data.data(), valid_data, data.size());
auto payload_reader =
@ -323,7 +323,7 @@ TEST(storage, InsertDataInt64Nullable) {
FixedVector<int64_t> data = {1, 2, 3, 4, 5};
auto field_data =
milvus::storage::CreateFieldData(storage::DataType::INT64, true);
uint8_t* valid_data = new uint8_t[1]{0x13};
uint8_t* valid_data = new uint8_t[1]{0xF3};
field_data->FillFieldData(data.data(), valid_data, data.size());
auto payload_reader =
@ -393,7 +393,7 @@ TEST(storage, InsertDataStringNullable) {
"test1", "test2", "test3", "test4", "test5"};
auto field_data =
milvus::storage::CreateFieldData(storage::DataType::STRING, true);
uint8_t* valid_data = new uint8_t[1]{0x13};
uint8_t* valid_data = new uint8_t[1]{0xF3};
field_data->FillFieldData(data.data(), valid_data, data.size());
auto payload_reader =
@ -460,7 +460,7 @@ TEST(storage, InsertDataFloatNullable) {
FixedVector<float> data = {1, 2, 3, 4, 5};
auto field_data =
milvus::storage::CreateFieldData(storage::DataType::FLOAT, true);
std::array<uint8_t, 1> valid_data = {0x13};
std::array<uint8_t, 1> valid_data = {0xF3};
field_data->FillFieldData(data.data(), valid_data.data(), data.size());
auto payload_reader =
@ -523,7 +523,7 @@ TEST(storage, InsertDataDoubleNullable) {
FixedVector<double> data = {1, 2, 3, 4, 5};
auto field_data =
milvus::storage::CreateFieldData(storage::DataType::DOUBLE, true);
uint8_t* valid_data = new uint8_t[1]{0x13};
uint8_t* valid_data = new uint8_t[1]{0xF3};
field_data->FillFieldData(data.data(), valid_data, data.size());
auto payload_reader =
@ -774,7 +774,7 @@ TEST(storage, InsertDataStringArrayNullable) {
FixedVector<Array> data = {string_array, int_array};
auto field_data =
milvus::storage::CreateFieldData(storage::DataType::ARRAY, true);
uint8_t* valid_data = new uint8_t[1]{0x01};
uint8_t* valid_data = new uint8_t[1]{0xFD};
field_data->FillFieldData(data.data(), valid_data, data.size());
auto payload_reader =
@ -812,7 +812,7 @@ TEST(storage, InsertDataJsonNullable) {
Json(simdjson::padded_string(std::string("A")))};
auto field_data =
milvus::storage::CreateFieldData(storage::DataType::JSON, true);
uint8_t* valid_data = new uint8_t[1]{0x00};
uint8_t* valid_data = new uint8_t[1]{0xFC};
field_data->FillFieldData(data.data(), valid_data, data.size());
auto payload_reader =
@ -842,7 +842,7 @@ TEST(storage, InsertDataJsonFillWithNull) {
auto field_data =
milvus::storage::CreateFieldData(storage::DataType::JSON, true);
int64_t size = 2;
uint8_t* valid_data = new uint8_t[1]{0x00};
uint8_t* valid_data = new uint8_t[1]{0xFC};
field_data->FillFieldData(std::nullopt, size);
auto payload_reader =