diff --git a/internal/core/src/mmap/Column.h b/internal/core/src/mmap/Column.h index 941908095c..9a2052ddef 100644 --- a/internal/core/src/mmap/Column.h +++ b/internal/core/src/mmap/Column.h @@ -41,6 +41,15 @@ namespace milvus { +/* +* If string field's value all empty, need a string padding to avoid +* mmap failing because size_ is zero which causing invalid arguement +* array has the same problem +* TODO: remove it when support NULL value +*/ +constexpr size_t STRING_PADDING = 1; +constexpr size_t ARRAY_PADDING = 1; + class ColumnBase { public: // memory mode ctor @@ -48,10 +57,7 @@ class ColumnBase { : type_size_(datatype_is_sparse_vector(field_meta.get_data_type()) ? 1 : field_meta.get_sizeof()) { - // simdjson requires a padding following the json data - padding_ = field_meta.get_data_type() == DataType::JSON - ? simdjson::SIMDJSON_PADDING - : 0; + SetPaddingSize(field_meta.get_data_type()); if (datatype_is_variable(field_meta.get_data_type())) { return; @@ -78,9 +84,7 @@ class ColumnBase { ? 1 : field_meta.get_sizeof()), num_rows_(size / type_size_) { - padding_ = field_meta.get_data_type() == DataType::JSON - ? simdjson::SIMDJSON_PADDING - : 0; + SetPaddingSize(field_meta.get_data_type()); size_ = size; cap_size_ = size; @@ -105,7 +109,7 @@ class ColumnBase { num_rows_(size / datatype_sizeof(data_type, dim)), size_(size), cap_size_(size) { - padding_ = data_type == DataType::JSON ? simdjson::SIMDJSON_PADDING : 0; + SetPaddingSize(data_type); data_ = static_cast(mmap(nullptr, cap_size_ + padding_, @@ -194,6 +198,26 @@ class ColumnBase { num_rows_++; } + void + SetPaddingSize(const DataType& type) { + switch (type) { + case DataType::JSON: + // simdjson requires a padding following the json data + padding_ = simdjson::SIMDJSON_PADDING; + break; + case DataType::VARCHAR: + case DataType::STRING: + padding_ = STRING_PADDING; + break; + case DataType::ARRAY: + padding_ = ARRAY_PADDING; + break; + default: + padding_ = 0; + break; + } + } + protected: // only for memory mode, not mmap void