admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: fix mmap failed when string field all value is empty (#31406)

Browse Source 
#31162

Signed-off-by: luzhang <luzhang@zilliz.com>
Co-authored-by: luzhang <luzhang@zilliz.com>
This commit is contained in:
zhagnlu 2024-03-21 10:41:07 +08:00 committed by GitHub
parent 1f43be4a3c
commit cf5109ec17
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 32 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
internal/core/src/mmap/Column.h
View File

@ -41,6 +41,15 @@
namespace milvus {
/*
* If string field's value all empty, need a string padding to avoid
* mmap failing because size_ is zero which causing invalid arguement
* array has the same problem
* TODO: remove it when support NULL value
*/
constexpr size_t STRING_PADDING = 1;
constexpr size_t ARRAY_PADDING = 1;
class ColumnBase {
public:
// memory mode ctor
@ -48,10 +57,7 @@ class ColumnBase {
: type_size_(datatype_is_sparse_vector(field_meta.get_data_type())
? 1
: field_meta.get_sizeof()) {
// simdjson requires a padding following the json data
padding_ = field_meta.get_data_type() == DataType::JSON
? simdjson::SIMDJSON_PADDING
: 0;
SetPaddingSize(field_meta.get_data_type());
if (datatype_is_variable(field_meta.get_data_type())) {
return;
@ -78,9 +84,7 @@ class ColumnBase {
? 1
: field_meta.get_sizeof()),
num_rows_(size / type_size_) {
padding_ = field_meta.get_data_type() == DataType::JSON
? simdjson::SIMDJSON_PADDING
: 0;
SetPaddingSize(field_meta.get_data_type());
size_ = size;
cap_size_ = size;
@ -105,7 +109,7 @@ class ColumnBase {
num_rows_(size / datatype_sizeof(data_type, dim)),
size_(size),
cap_size_(size) {
padding_ = data_type == DataType::JSON ? simdjson::SIMDJSON_PADDING : 0;
SetPaddingSize(data_type);
data_ = static_cast<char*>(mmap(nullptr,
cap_size_ + padding_,
@ -194,6 +198,26 @@ class ColumnBase {
num_rows_++;
}
void
SetPaddingSize(const DataType& type) {
switch (type) {
case DataType::JSON:
// simdjson requires a padding following the json data
padding_ = simdjson::SIMDJSON_PADDING;
break;
case DataType::VARCHAR:
case DataType::STRING:
padding_ = STRING_PADDING;
break;
case DataType::ARRAY:
padding_ = ARRAY_PADDING;
break;
default:
padding_ = 0;
break;
}
}
protected:
// only for memory mode, not mmap
void